commit 1d3f9ce2a5311f80fcaade85928bc6ec5975c8dc
Author: DynamoRIO Developers <vmware.com>
Date:   Tue Feb 10 21:30:46 2009 +0000

    initial open source import from formerly proprietary VMware sources
    
    SVN-Revision: 1

diff --git a/License.txt b/License.txt
new file mode 100644
index 000000000..15ca5e302
--- /dev/null
+++ b/License.txt
@@ -0,0 +1,27 @@
+Copyright (c) 2000-2009 VMware, Inc. licensed under the terms of the BSD.  All other rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of VMware, Inc. nor the names of its contributors may be
+  used to endorse or promote products derived from this software without
+  specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
diff --git a/Makefile b/Makefile
new file mode 100644
index 000000000..f44f79d69
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,170 @@
+# **********************************************************
+# Copyright 2008 VMware, Inc.  All rights reserved. -- VMware Confidential
+# **********************************************************
+
+###########################################################################
+###########################################################################
+###
+### Top-level Makefile for building DynamoRIO
+###
+### Default build is a GoVirtual release package.
+###
+### The build directory is NOT automatically updated:
+### the user should invoke as "make clean zip" (or just "make")
+### in order to update an existing build directory.
+###
+### WARNING: do NOT include any shell-invoking values prior to
+### the currently-under-cygwin test below!
+
+.PHONY: default zip clean dynamorio-all
+
+default: clean dynamorio-all
+
+# Only Windows shoud have the OS env var set
+ifeq ($(OS),)
+  MACHINE := linux
+else
+  MACHINE := win32
+endif
+
+# build team provides these:
+# OBJDIR               = ignored
+# BUILD_ROOT           = pass to makezip.sh
+# PUBLISH_DIR          = pass to makezip.sh
+# BUILD_NUMBER         = -DUNIQUE_BUILD_NUMBER via makezip.sh
+# PRODUCT_BUILD_NUMBER = -DBUILD_NUMBER via makezip.sh
+
+# we no longer support viper
+BUILD_TYPE := vmap
+
+BUILD_ROOT := $(CURDIR)/zipbuild
+
+# used if invoker doesn't specify
+NO_BUILD_NUMBER := 16
+
+# our BUILD_NUMBER goes into the windows x.y.z.w version and so must
+# be <64K, which is vmware's PRODUCT_BUILD_NUMBER.  but we're supposed
+# to record the potentially-larger unique BUILD_NUMBER somewhere as well,
+# requiring us to pass in two build numbers to makezip.sh.
+ifneq ($(BUILD_NUMBER),)
+  UNIQUE_BUILD_NUMBER := $(BUILD_NUMBER)
+else
+  UNIQUE_BUILD_NUMBER := $(NO_BUILD_NUMBER)
+endif
+
+# part-of-version build number: must be <64K
+ifneq ($(PRODUCT_BUILD_NUMBER),)
+  VER_BUILD_NUMBER := $(PRODUCT_BUILD_NUMBER)
+else
+  VER_BUILD_NUMBER := $(NO_BUILD_NUMBER)
+endif
+
+# keep in sync with CURRENT_API_VERSION in core/x86/instrument.h
+# and _USES_DR_VERSION_ in dr_api.h (PR 250952) and compatibility
+# check in core/x86/instrument.c (OLDEST_COMPATIBLE_VERSION, etc.)
+# and api/docs/footer.html
+# and tools/DRgui/DynamoRIO.rc
+VERSION_NUMBER := 1.3.1
+
+MAKEZIP_ARGS := $(BUILD_TYPE) $(BUILD_ROOT) $(VER_BUILD_NUMBER) \
+                $(UNIQUE_BUILD_NUMBER) $(VERSION_NUMBER) $(PUBLISH_DIR)
+
+ifndef DYNAMORIO_TOOLS
+  DYNAMORIO_TOOLS := tools
+endif
+
+##################################################
+ifeq ($(MACHINE), win32)
+
+  # To support building with either a local cygwin 
+  # or a toolchain cygwin, we do NOT
+  # invoke any commands that require sh or bash to avoid cygwin
+  # conflicts before we've identified whether we're using local
+  # cygwin or need to invoke our batch file to set up
+  # for toolchain cygwin.
+  #
+  # The standard VMware Windows build uses a non-cygwin make.exe
+  # from toolchain, with the path set such that it finds
+  # toolchain/win32/bin/sh.exe as its shell.  We do NOT want
+  # to execute that shell and want to use our toolchain cygwin
+  # instead.  But, for invocations on a machine that
+  # has a local cygwin, we want to use that.  Fundamentally
+  # we want to know whether there is a local cygwin install
+  # that we can use, not whether the make.exe is cygwin or not.
+  # That's difficult to check w/o invoking utility programs.
+  # Next best would be to identify whether any other running
+  # processes are cygwin, since those can conflict with toolchain
+  # cygwin.  That's also hard to do, so we settle for checking
+  # whether make.exe was invoked from a cygwin shell.
+  # Unfortunately $OSTYPE and $MACHTYPE are not exported, so
+  # we instead check whether cmd's %ComSpec% is set (it is even if
+  # invoking make.exe from the Run box).
+  # Note that we can't check whether PATH uses ; or : since
+  # toolchain make.exe converts a Unix PATH to Windows format.
+
+  ifeq ($(ComSpec),)
+    LOCAL_CYGWIN := 1
+  else
+    LOCAL_CYGWIN := 0
+  endif
+
+  ifeq ($(LOCAL_CYGWIN),1)
+    # Currently under cygwin: keep using that cygwin.
+    # We assume all utilities are on the PATH.
+zip:
+	$(DYNAMORIO_TOOLS)/makezip.sh $(MAKEZIP_ARGS)
+
+clean:
+	$(RM) -rf $(BUILD_ROOT) $(PUBLISH_DIR)
+
+  else
+
+    # No local cygwin: set up for toolchain cygwin.
+    SHELL   := cmd.exe
+    ifndef DYNAMORIO_BATCH
+      DYNAMORIO_BATCH := batch
+    endif
+
+zip:
+	$(DYNAMORIO_BATCH)\makezip.bat $(MAKEZIP_ARGS)
+
+    # For some reason I can't get direct invocation of rd to
+    # work with toolchain make: it messes up its CreateProcess
+    # invocation.
+clean:
+	-$(DYNAMORIO_BATCH)\rmtree.bat $(subst /,\,$(BUILD_ROOT))
+	-$(DYNAMORIO_BATCH)\rmtree.bat $(subst /,\,$(PUBLISH_DIR))
+
+  endif
+
+##################################################
+else
+
+  ifndef DYNAMORIO_MAKE
+    DYNAMORIO_MAKE := make
+  endif
+  # we need the x64 version of strip in order to operate on x64 libs
+  ARCH := x64
+  include $(DYNAMORIO_MAKE)/compiler.mk
+
+  # Set up for scripts like makezip.sh and for auto-generated
+  # api/docs/latex/Makefile to use toolchain utils.
+  # For scripts we could set env vars whenever make invokes a script
+  #   TAR=$(TAR) PERL=$(PERL) ... <script>
+  # and then have the script explicitly invoke $TAR, but it's less work
+  # to set up the path (esp. for auto-gen latex/Makefile).
+  #
+  # PR 340793: this older doxygen doesn't quite match newer local
+  # doxygen, but it's close enough
+  #
+  PATH := $(BINUTILS)/$(BINPREFIX)-linux/bin:$(COREUTILS):$(TC_AWK):$(TC_DIFF):$(TC_GREP):$(TC_PERL):$(TC_SED):$(TC_XARGS):$(TC_DOXYGEN):$(TC_MOGRIFY):$(TC_FIG2DEV):$(TC_GS):$(TC_TETEX):$(TC_TAR):$(TC_GZIP):$(TC_ZIP):$(TC_MAKE):$(TC_FIND)
+
+zip:
+	$(DYNAMORIO_TOOLS)/makezip.sh $(MAKEZIP_ARGS)
+
+clean:
+	$(RM) -rf $(BUILD_ROOT) $(PUBLISH_DIR)
+
+endif
+##################################################
+dynamorio-all: zip
diff --git a/README b/README
new file mode 100644
index 000000000..300fa4f4c
--- /dev/null
+++ b/README
@@ -0,0 +1,41 @@
+Copyright (c) 2000-2009 VMware, Inc. licensed under the terms of the BSD.  All other rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of VMware, Inc. nor the names of its contributors may be
+  used to endorse or promote products derived from this software without
+  specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+
+**********************************************************
+
+DynamoRIO is a runtime code manipulation system that supports code
+transformations on any part of a program, while it executes. DynamoRIO
+exports an interface for building dynamic tools for a wide variety of uses:
+program analysis and understanding, profiling, instrumentation,
+optimization, translation, etc. Unlike many dynamic tool systems, DynamoRIO
+is not limited to insertion of callouts/trampolines and allows arbitrary
+modifications to application instructions via a powerful IA-32/AMD64
+instruction manipulation library. DynamoRIO provides efficient,
+transparent, and comprehensive manipulation of unmodified applications
+running on stock operating systems (Windows or Linux) and commodity IA-32
+and AMD64 hardware.
diff --git a/README_code_standards.html b/README_code_standards.html
new file mode 100644
index 000000000..ede80e2d6
--- /dev/null
+++ b/README_code_standards.html
@@ -0,0 +1,707 @@
+<!doctype html public "-//IETF//DTD HTML//EN">
+<html>
+<head>
+<title>DynamoRIO Coding Standards</title>
+</head>
+
+<!--------------------------------------------------------------------------
+Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice,
+  this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of VMware, Inc. nor the names of its contributors may be
+  used to endorse or promote products derived from this software without
+  specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+-------------------------------------------------------------------------->
+
+<body bgcolor="ffffff" text="000000" link="000088" vlink="006600">
+
+<h1>DynamoRIO Coding Standards</h1>
+
+<ul>
+<li><a href="#Style">Coding Style</a></li>
+<li><a href="#Content">Code Content</a></li>
+</ul>
+
+<!--------------------------------------------------------------------------->
+<a name="Style">&nbsp;</a>
+<table bgcolor="#6699cc" width=100% cellspacing=0 cellpadding=5 border=0>
+<tr><td>&nbsp;<font size=+2>
+<strong>Coding Style</strong>
+</font></td></tr></table>
+<!--------------------------------------------------------------------------->
+
+<p>The overall goal is to <strong>make the source code as readable and
+as self-documenting as possible</strong>.  Everyone following the same
+style guidelines is an important part of keeping the code consistent and
+maintainable.</p>
+
+
+<h3>Naming Conventions</h3>
+
+<ol>
+
+<li><p>Variable and function names use only lowercase letters.  Multi-word
+function and variable names are all lowercase with underscores delimiting
+words.  Do not use CamelCase for names, unless mirroring Windows-defined
+data structures.
+<pre>
+<em>GOOD</em>: instr_get_target()
+<font color=gray><em>BAD</em>:  instrGetTarget()</font>
+</pre>
+</p></li>
+
+<li><p>Type names are all lowercase, with underscores dividing words, and
+ending in <tt>_t</tt>:
+<pre>
+instr_t
+build_bb_t
+</pre>
+</p></li>
+
+<li><p>The name of a struct in a typedef should be the type name with an
+underscore prefixed:
+<pre>
+typedef struct _build_bb_t {
+    ...
+} build_bb_t;
+</pre>
+</p></li>
+
+<li><p>Constants should be in all capital letters, with underscores dividing
+words.  Enum members should use a common descriptive prefix.
+<pre>
+static const int MAX_SIZE = 256;
+enum {
+    DUMPCORE_DEADLOCK           = 0x0004,
+    DUMPCORE_ASSERTION          = 0x0008,
+};
+</pre>
+</p></li>
+
+<li><p>Preprocessor defines and macros should be in all capital letters, with
+underscores dividing words.
+<pre>
+#ifdef WINDOWS
+# define IF_WINDOWS(x) x
+#else
+# define IF_WINDOWS(x)
+#endif
+</pre>
+</p></li>
+
+<li><p>Functions that operate on a data structure should contain that
+structure as a prefix.  For example, all of the routines that operate on
+the <tt>instr_t</tt> struct begin with <tt>instr_</tt>.
+</p></li>
+
+<li><p>Use <tt>static</tt> when possible for every function or variable
+that is not needed outside of its own file.
+</p></li>
+
+</ol>
+
+
+<h3>Types</h3>
+
+<ol>
+
+<li><p>See above for naming conventions for types.
+</p></li>
+
+<li><p>When declaring a function with no arguments, always explicitly use
+the <tt>void</tt> keyword.  Otherwise the compiler will not be able to
+check whether you are incorrectly passing arguments to that function.
+<pre>
+<em>GOOD</em>: int foo(void);
+<font color=gray><em>BAD</em>:  int foo();</font>
+</pre>
+</p></li>
+
+<li><p>Use the <tt>IN</tt>, <tt>OUT</tt>, and <tt>INOUT</tt> labels to
+describe function parameters.  This is a recent addition to DynamoRIO so
+you will see many older functions without these labels, but use them on all
+new functions.
+<pre>
+<em>GOOD</em>: int foo(IN int length, OUT char *buf);
+<font color=gray><em>BAD</em>:  int foo(int length, char *buf);</font>
+</pre>
+</p></li>
+
+<li><p>Only use boolean types as conditionals.  This means using explicit NULL
+comparisons and result comparisons.  In particular with functions like
+strcmp() and memcmp(), the use of ! is counter-intuitive.
+<pre>
+<em>GOOD</em>: if (p == NULL) ...
+<font color=gray><em>BAD</em>:  if (p)</font>
+<em>GOOD</em>: if (p != NULL) ...
+<font color=gray><em>BAD</em>:  if (!p)</font>
+<em>GOOD</em>: if (strncmp(...) == 0) ...
+<font color=gray><em>BAD</em>:  if (!strncmp(...))</font>
+</pre>
+</p></li>
+
+<li><p>It's much easier to read <tt>if (i == 0)</tt> than <tt>if (0 == i)
+</tt>.  The compiler, with all warnings turned on (which we have), will
+warn you if you use assignment rather than equality.
+<pre>
+<em>GOOD</em>: if (i == 0) ...
+<font color=gray><em>BAD</em>:  if (0 == i)</font>
+</pre>
+</p></li>
+
+<li><p>Use the <tt>TEST</tt> and related macros for testing bits.
+<pre>
+<em>GOOD</em>: if (TEST(BITMASK, x))
+<font color=gray><em>BAD</em>:  if ((x & BITMASK) != 0)</font>
+</pre>
+</p></li>
+
+<li><p>Write code that is 32-bit and 64-bit aware:
+  <ul>
+  <li><p>Use int and uint for 32-bit integers.  Do not use long as its size is
+  64-bit for Linux but 32-bit for Windows.  We assume that int is a 32-bit
+  type.
+  <li><p>Use int64 and uint64 for 64-bit integers.  Use <tt>INT64_FORMAT</tt>
+  and related macros for printing 64-bit integers.
+  <li><p>Use ptr_uint_t and ptr_int_t for pointer-sized integers.
+  <li><p>Use size_t for sizes of memory regions.
+  <li><p>Use reg_t for register-sized values.
+  <li><p>Use <tt>ASSERT_TRUNCATE</tt> macros when casting to a smaller type.
+  <li><p>Use <tt>PFX</tt> and other printf macros for printing pointer-sized variables.
+  </ul>
+</p></li>
+
+<li><p>Invalid addresses, either pointers to our data structures or
+application addresses that we're manipulating, have the value NULL, not 0.
+0 is only for arithmetic types.  
+</p></li>
+
+<li><p><tt>const</tt> makes code easier to read and lets the compiler complain
+about errors and generate better code.  It is also required for the most
+efficient self-protection.  Use whenever possible.
+</p></li>
+
+<li><p>Place * prefixing variable names (C style), not suffixing type names
+(Java style):
+<pre>
+<em>GOOD</em>: char *foo;
+<font color=gray><em>BAD</em>:  char* foo;</font>
+</pre>
+</p></li>
+
+</ol>
+
+
+<h3>Commenting Conventions</h3>
+
+<ol>
+
+<li><p>This is C, so <tt>/* */</tt> comments are preferable to <tt>//</tt>.
+Put stars on each line of a multi-line comment, like this:
+<pre>
+/* multi-line
+ * comment
+ * with stars
+ */
+</pre>
+Generally the trailing <tt>*/</tt> should be on its own line, but it can
+optionally be on the end of the precending line.
+</p></li>
+
+<li><p>Make liberal use of comments.  However, too many comments can impair
+readability.  Choose self-descriptive function and variable names to reduce
+the number of comments needed.
+</p></li>
+
+<li><p>Do not use large, clunky function headers that simply duplicate
+information in the code itself.  Such headers tend to contain stale,
+incorrect information, for two reasons: the code is often updated without
+maintaining the header, and since the headers are a pain to type they are
+often copied from other functions and not completely modified for their new
+home.  They also make it harder to see the code or to group related
+functions, as they take up so much screen space.  It is better to have
+leaner, more maintainable, and more readable implementation files by using
+self-descriptive function and parameter names and placing comments for
+function parameters next to the parameters themselves.
+
+<pre>
+<em>GOOD</em>: /* Retrieves the name of the logfile for a particular thread. 
+       * Returns false if no such thread exists.
+       */
+      bool get_logfile(IN thread_id_t thread, 
+                       OUT char *fname,
+                       IN size_t fname_len)
+
+<font color=gray><em>BAD</em>:  /*------------------------------------------------------
+       * Name: get_logfile
+       *
+       * Purpose:
+       *    Retrieves the name of the logfile for a particular thread. 
+       *
+       * Parameters:
+       *    [IN]  thread    = which thread
+       *    [OUT] fname     = where to store the logfile name
+       *    [IN]  fname_len = the size of the fname buffer
+       *
+       * Returns:
+       *     True if successful.
+       *     False if no such thread exists.
+       *
+       * Side effects:
+       *     None.
+       * ------------------------------------------------------
+       */
+      bool get_logfile(thread_id_t thread, char *fname, size_t fname_len)
+  </font>
+</pre>
+</p></li>
+
+<li><p>Use doxygen comments on all function and type declarations that are
+exported as part of the API.  For comments starting with <tt>/**</tt>,
+leave the rest of the first line empty, unless the entire comment is a
+single line.  Some examples:
+<pre>
+DR_API
+/**
+ * Returns the entry point of the function with the given name in the module
+ * with the given base. Returns NULL on failure.
+ * \note Currently Windows only.
+ */
+generic_func_t
+dr_get_proc_address(IN module_handle_t lib, IN const char *name);
+
+/**
+ * Data structure passed with a signal event.  Contains the machine
+ * context at the signal interruption point and other signal
+ * information.
+ */
+typedef struct _dr_siginfo_t {
+    int sig;                /**< The signal number. */
+    void *drcontext;        /**< The context of the thread receiving the signal. */
+    dr_mcontext_t mcontext; /**< The machine state at the signal interruption point. */
+    siginfo_t siginfo;      /**< The signal information provided by the kernel. */
+} dr_siginfo_t;
+</pre>
+</p></li>
+
+<li><p><strong>NEVER</strong> check in commented-out code.  This is
+unacceptable.  If you feel strongly that you need to leave code in that is
+disabled, use conditional compilation (e.g., <tt>#if
+DISABLED_UNTIL_BUG_812_IS_FIXED</tt>), and explain why the code is
+disabled.
+</p></li>
+
+<li><p>Sloppy comments full of misspelled words, etc. are an indication of
+carelessness.  We do not want carelessly written code, and we do not
+want carelessly written comments.
+</p></li>
+
+<li><p>Use <tt>FIXME</tt> in comments to indicate missing features, code that
+could be optimized, or anything else that may warrant re-examination later.
+</p></li>
+
+</ol>
+
+
+<h3>Warnings Are Errors</h3>
+
+<ol>
+
+<li><p> Uninitialized variables warning (W4701 for cl): Don't initialize
+variables when you don't need to, so that we can still have good warnings
+about uninitialized variables in the future.  Only if the compiler can't
+analyze code properly is it better to err on the side of a deterministic
+bug and set to 0 or {0}.
+
+<p> Use do {} while () loops to help the compiler figure out that variables
+will get initialized.  The generated code on those constructs is faster and
+better predicted (although optimizations should be able to transform simple
+loops).
+</p></li>
+
+<li><p> For suggested use of static analysis tools: PreFAST or /analyze for
+new code, refer to case 3966.
+</p></li>
+
+</ol>
+
+
+<h3>Program Structure</h3>
+
+<ol>
+
+<li><p>Keep the line length to 90 characters or less.
+</p></li>
+
+<li><p>Use an indentation level of 4 spaces (no tabs, always expand them to
+spaces when saving the file).  <strong>WARNING</strong>: Emacs defaults are
+not always correct here.  Make sure your .emacs contains the following:
+
+<pre>
+; always expand tabs to spaces
+(setq-default indent-tabs-mode 'nil)
+
+; want "gnu" style but indent of 4:
+(setq c-basic-offset 4)
+(add-hook 'c-mode-hook '(lambda ()
+			  (setq c-basic-offset 4)))
+</pre>
+</p></li>
+
+<li><p>K&amp;R-style braces: opening braces at the end of the line preceding
+the new code block, closing braces on their own line at the same
+indentation as the line preceding the code block.  Functions are an
+exception -- see below.
+</p></li>
+
+<li><p>Functions should have their type on a separate line from their name.
+Place the function's opening brace on a line by itself at zero
+indentation.
+<br><em>Example</em>:<br>
+<pre>
+int
+foo(int x, int y)
+{
+    return 42;
+}
+</pre>
+</p></li>
+
+<li><p>Put spaces after commas in parameter and argument lists
+<pre>
+<em>GOOD</em>: foo(x, y, z);
+<font color=gray><em>BAD</em>:  foo(x,y,z);</font>
+</pre>
+</p></li>
+
+<li><p>Do not put spaces between a function name or a <tt>for</tt>,
+<tt>if</tt>, <tt>while</tt>, <tt>do</tt>, or <tt>switch</tt> and the
+following parenthesis, or before the closing parenthesis.<br>
+<pre>
+<em>GOOD</em>: if (x == 6)
+<font color=gray><em>BAD</em>:  if( x==6 )</font>
+</pre>
+</p></li>
+
+<li><p>Always put the body of an if or loop on a separate line from the
+line containing the keyword.
+<pre>
+<em>GOOD</em>: if (x == 6)
+          y = 5;
+
+<font color=gray><em>BAD</em>:  if (x==6) y = 5;</font>
+</pre>
+</p></li>
+
+<li><p>A multi-line (not just multi-statement) body (of an if, loop, etc.)
+should always be surrounded with braces (to avoid errors in later
+statements arising from indentation mistakes).
+</p></li>
+
+<li><p>Statements should always begin on a new line (do not put multiple
+statements on the same line).
+</p></li>
+
+<li><p>Indent nested preprocessor statements.  The # character must be in
+the first column, but the rest of the statement can be indented.
+<br><em>Example:</em></br>
+<pre>
+#ifdef OUTERDEF
+#  ifdef INNERDEF
+#    define INSIDE
+#  endif
+#else
+#  define OUTSIDE
+#endif
+</pre>
+</p></li>
+
+<li><p>To make the code easier to read, use the <tt>DODEBUG</tt>,
+<tt>DOSTATS</tt>, or <tt>DOLOG</tt> macros, or the <tt>IF_WINDOWS</tt> and
+related macros, rather than ifdefs, for common defines.
+</p></li>
+
+<li><p> Do not use DO_ONCE(SYSLOG_INTERNAL.  Instead use two new macros:
+DODEBUG_ONCE and SYSLOG_INTERNAL_*_ONCE.
+</p></li>
+
+<li><p>Use <em>make ugly</em> to examine the code for known poor coding
+patterns.
+</p></li>
+
+</ol>
+
+
+<h3>File Organization</h3>
+
+<ol>
+
+<li><p>Write OS-independent code as much as possible and keep it in the
+base <tt>core/</tt> directory.  If code must diverge for Windows versus
+Linux, provide an OS-independent interface documented
+in <tt>core/os_shared.h</tt> and implemented separately
+in <tt>core/linux/</tt> and <tt>core/windows/</tt>.
+</p></li>
+
+</ol>
+
+<!--------------------------------------------------------------------------->
+<a name="Content">&nbsp;</a>
+<table bgcolor="#6699cc" width=100% cellspacing=0 cellpadding=5 border=0>
+<tr><td>&nbsp;<font size=+2>
+<strong>Code Content</strong>
+</font></td></tr></table>
+<!--------------------------------------------------------------------------->
+
+
+<h3>Adding New Features</h3>
+
+<ol>
+
+<li><p>A new feature should be added under its own preprocessor define as well
+as under its own off-by-default runtime option, to facilitate incremental
+development and testing.  Once it is working properly, the ifdefs can be
+removed.  Once it is proven to be worthwhile and robust, the option can be
+turned on by default.
+</p></li>
+
+<li><p>Code reviews are required for any commit, regardless of how trivial.
+</p></li>
+
+<li><p>Use assertions whenever an assumption is made.  We have several types
+of ASSERT macros: use the appropriate one.  For any API-visible function,
+use CLIENT_ASSERT to provide an understandable string to the user.
+</p></li>
+
+<li><p>Add a new statistic to <tt>core/lib/statsx.h</tt> for anything we might
+want to keep track of.
+</p></li>
+
+<li><p>Be careful with locks.  Make sure to follow our deadlock-avoidance
+scheme when adding a new lock: add it to the list in <tt>core/utils.h</tt>.
+</p></li>
+
+</ol>
+
+
+<h3>Security</h3>
+
+<ol>
+
+<li><p>Be security aware. Carefully watch your buffers and input validation to
+avoid introducing vulnerabilities.
+</p></li>
+
+<li><p>Do not use unsafe string routines.  E.g., use <tt>snprintf</tt> instead
+of <tt>sprintf</tt>.
+</p></li>
+
+<li><p>Variable placement: <strong>.data is read-only!</strong>
+</p></li>
+
+  <p>
+  For self-protection from inadvertent or malicious application activity,
+  DynamoRIO uses a non-standard data section layout with five separate
+  sections:
+
+  <ol></p></li>
+
+  <li><p><em>.rdata</em> = read-only data<br> Mark all non-written
+  variables as <tt>const</tt> to get them into this section.
+  </p></li>
+
+  <li><p><em>.data</em> = rarely-written data<br> Variables that are
+  written only during initialization, cleanup, or a handful of times during
+  execution. This section is write-protected by default and only
+  unprotected for small, infrequent windows of time.  This is the default
+  location for a variable (even if in .bss, if on Windows, though not on
+  Linux currently due to PR 213296), so if you write to your new
+  default-location variable after initialization <strong>DynamoRIO will
+  crash</strong>!  In debug builds a special self-protection message will
+  alert you to crashes coming from attempts to write to .data.
+
+  </p><p>If you must write to a variable after initialization,
+  you can either unprotect it with this pair of macros (however, see the
+  next item):
+  <pre>
+    SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+    SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+  </pre>
+  Or you can move it to another section, probably .fspdata, like this:
+  <pre>
+    DECLARE_FREQPROT_VAR(static vartype myvar, initial-value);
+  </pre>
+  </p>
+
+  </p><p>
+  Only unprotect if you write <strong>very infrequently</strong>, as in
+  zero times on any critical path.  Keep an eye on the stats for
+  unprotection calls and make sure you're not affecting performance and not
+  opening up too many windows where the entire .data section is
+  unprotected.
+
+  </p><p> Do not keep pointers in a data section other than .data.  If they
+  must be writable, use indirection and keep them on the heap.
+
+  </p><p> Think about whether non-pointers (booleans, counters) can be
+  exploited before throwing them into unprotected sections.  Can an
+  overwrite of the variable in question cause us to stop protecting the
+  application, or to lose control of the application?
+
+  </p><p> Place all locks in .cspdata.
+
+  </p><p> Be aware that DO_ONCE introduces a window where .data is unprotected
+  (see PR 212510).
+
+  </p></li>
+
+  <li><p><em>.fspdata</em> = frequently-written self-protected data<br>
+  Variables that are written enough that we do not want to unprotect .data
+  every time. We place them in a separate section called .fspdata (for
+  "frequently self-protected data", referring to the number of times we
+  need to change the page protections, which is proportional to the number
+  of post-init writes) both for security and performance reasons. We moved
+  any pointers in this category into structures on the heap, enabling us to
+  move their base pointers into .data. Currently the heap is better
+  protected than our data sections due to its better randomization and its
+  guard pages.  Try not to put new variables here as we hope to eventually
+  eliminate this section.  Note that this is not yet protected at all: PR
+  212508.
+  </p></li>
+
+  <li><p><em>.cspdata</em> = context-switch-written self-protected data<br>
+  This is our section for locks, which are written so frequently that
+  unprotecting them is best done at each context switch.  Note that this is
+  not yet protected at all: PR 212508.
+  </p></li>
+
+  <li><p><em>.nspdata</em> = not self-protected data<br> Debug-build-only
+  variables, statistics, or data that is not persistent across code cache
+  execution fall into this category of variables that we never
+  protect.  This section is always writable.
+  </p></li>
+  </ol>
+
+  </p><p>DynamoRIO has several other features for self-protection:
+  <ol>
+  
+  <li><p>The base of its heap is randomized.  This is why we consider
+  writable pointers in the heap to be safer than writable pointers in a
+  data section.
+  </p></li>
+
+  <li><p>Guard pages protect both ends of every stack and heap unit to
+  protect against sequential overwrites.
+  </p></li>
+
+  <li><p>No state is kept on the DynamoRIO stack across code cache
+  execution: execution starts from a clean slate at the base of the stack
+  on every exit from the code cache.
+  </p></li>
+
+  <li><p>Generated code is made read-only after initialization.
+  </p></li>
+
+  <li><p>The options are made read-only after initialization.
+  Now that we have .data read-only, this is simply part of that feature.
+  </p></li>
+
+  <li><p>The base of the dynamorio.dll library is randomized.
+  </p></li>
+
+  </ol>
+
+</p></li>
+
+</ol>
+
+
+<h3>Transparency</h3>
+
+<p>
+The DynamoRIO core must be absolutely transparent.  It cannot interfere
+with the application's semantics.  This means that extra care must be taken
+when using <strong>any</strong> operating system or shared library
+resources that might interact with the application's use of those
+resources.  See the API documentation for more details.  The following
+guidelines list some common mistakes to avoid.
+</p>
+
+<ol>
+
+<li><p>Do not introduce new user library dependencies.  On Windows we only
+use ntdll.dll, and there we only call routines that we are sure are
+re-entrant.  On Linux we're not quite there yet: we still use some C
+library routines.  But we're working toward libc independence, so avoid
+adding any new library calls.
+</p></li>
+
+<li><p> Never call any potentially non-reentrant library routines.  These
+include malloc, calloc, fprintf, etc., and everything that calls these
+routines, such as strdup.  Usually you'll be violating the prior point if
+you use these, but keep in mind that many ntdll.dll routines are
+non-reentrant because they acquire locks or allocate memory.  Use raw
+system calls instead of library routines (yes we have hit many bugs due to
+using library routines).
+</p></li>
+
+<li><p>Do not make any alertable system calls on Windows.
+This includes any Wait* API routine.  Use the system calls instead, which
+let you specify whether it should be alertable or not.
+</p></li>
+
+<li><p> For memory allocation, use either the local heap (via
+<tt>heap_alloc</tt>) or the global heap (via
+<tt>global_heap_alloc</tt>). (In certain circumstances, such as inside
+signal handlers, other heaps should be used.)  Always de-allocate
+memory.  We cannot assume we will last as long as the application, since
+we may be detached before the application exits.
+</p></li>
+
+<li><p> For I/O, use the interfaces exported by the OS subdirectories.  Do not
+use <tt>stderr</tt> or <tt>stdout</tt> -- due to <tt>FILE</tt> problems on windows we use
+<tt>HANDLE</tt> on windows and hide the distinction behind the
+<tt>file_t</tt> type.  Use the <tt>STDERR</tt> and <tt>STDOUT</tt> macros.
+</p></li>
+
+<li><p>DO NOT USE FLOATING-POINT OPERATIONS!
+DynamoRIO does not save or restore floating-point state on context
+switches.  You can use the routines <tt>save_fpstate</tt> and
+<tt>restore_fpstate</tt> (includes mmx & sse state) around your code if
+you really need floating point operations.
+</p></li>
+
+<li><p>DO NOT HOLD LOCKS ACROSS CODE CACHE EXECUTION!
+Doing so violates the assumptions of our synchronization algorithms.
+</p></li>
+
+</ol>
+
+<!--------------------------------------------------------------------------->
+<table bgcolor="#6699cc" width=100% cellspacing=0 cellpadding=5 border=0>
+<tr><td>&nbsp;</td></tr></table>
+<!--------------------------------------------------------------------------->
+
+</body>
+</html>
diff --git a/api/docs/API.doxy b/api/docs/API.doxy
new file mode 100644
index 000000000..692fe45c1
--- /dev/null
+++ b/api/docs/API.doxy
@@ -0,0 +1,257 @@
+# **********************************************************
+# Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+# **********************************************************/
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# Doxyfile 1.5.1
+
+#---------------------------------------------------------------------------
+# Project related configuration options
+#---------------------------------------------------------------------------
+PROJECT_NAME           = "DynamoRIO API"
+PROJECT_NUMBER         = 
+OUTPUT_DIRECTORY       = 
+CREATE_SUBDIRS         = NO
+OUTPUT_LANGUAGE        = English
+USE_WINDOWS_ENCODING   = NO
+BRIEF_MEMBER_DESC      = YES
+REPEAT_BRIEF           = YES
+ABBREVIATE_BRIEF       = 
+ALWAYS_DETAILED_SEC    = NO
+INLINE_INHERITED_MEMB  = NO
+FULL_PATH_NAMES        = NO
+STRIP_FROM_PATH        =
+STRIP_FROM_INC_PATH    = 
+SHORT_NAMES            = NO
+JAVADOC_AUTOBRIEF      = NO
+MULTILINE_CPP_IS_BRIEF = NO
+DETAILS_AT_TOP         = YES
+INHERIT_DOCS           = YES
+SEPARATE_MEMBER_PAGES  = NO
+TAB_SIZE               = 8
+# if you add/change aliases make sure you also change the sed lines in the Makefile
+ALIASES                = DynamoRIO=DynamoRIO Client=Client Clients=Clients client=client clients=clients
+OPTIMIZE_OUTPUT_FOR_C  = YES
+OPTIMIZE_OUTPUT_JAVA   = NO
+BUILTIN_STL_SUPPORT    = NO
+DISTRIBUTE_GROUP_DOC   = YES
+SUBGROUPING            = YES
+#---------------------------------------------------------------------------
+# Build related configuration options
+#---------------------------------------------------------------------------
+EXTRACT_ALL            = NO
+EXTRACT_PRIVATE        = NO
+EXTRACT_STATIC         = NO
+EXTRACT_LOCAL_CLASSES  = YES
+EXTRACT_LOCAL_METHODS  = NO
+HIDE_UNDOC_MEMBERS     = YES
+HIDE_UNDOC_CLASSES     = YES
+HIDE_FRIEND_COMPOUNDS  = NO
+HIDE_IN_BODY_DOCS      = NO
+INTERNAL_DOCS          = NO
+CASE_SENSE_NAMES       = YES
+HIDE_SCOPE_NAMES       = NO
+SHOW_INCLUDE_FILES     = YES
+INLINE_INFO            = YES
+SORT_MEMBER_DOCS       = YES
+SORT_BRIEF_DOCS        = NO
+SORT_BY_SCOPE_NAME     = NO
+GENERATE_TODOLIST      = YES
+GENERATE_TESTLIST      = YES
+GENERATE_BUGLIST       = YES
+GENERATE_DEPRECATEDLIST= YES
+# choices: linux, VMsafe, NYI_kernel_mediated_transfer_events
+ENABLED_SECTIONS       = 
+MAX_INITIALIZER_LINES  = 30
+SHOW_USED_FILES        = YES
+SHOW_DIRECTORIES       = NO
+FILE_VERSION_FILTER    = 
+#---------------------------------------------------------------------------
+# configuration options related to warning and progress messages
+#---------------------------------------------------------------------------
+QUIET                  = NO
+WARNINGS               = YES
+WARN_IF_UNDOCUMENTED   = YES
+WARN_IF_DOC_ERROR      = YES
+WARN_NO_PARAMDOC       = NO
+WARN_FORMAT            = "$file:$line: $text"
+WARN_LOGFILE           = 
+#---------------------------------------------------------------------------
+# configuration options related to the input files
+#---------------------------------------------------------------------------
+INPUT                  = .
+FILE_PATTERNS          = dr_*.h *.dox
+RECURSIVE              = NO
+EXCLUDE                = 
+EXCLUDE_SYMLINKS       = NO
+EXCLUDE_PATTERNS       = 
+EXAMPLE_PATH           = 
+EXAMPLE_PATTERNS       = *
+EXAMPLE_RECURSIVE      = NO
+IMAGE_PATH             = genimages images
+INPUT_FILTER           = 
+FILTER_PATTERNS        = 
+FILTER_SOURCE_FILES    = NO
+#---------------------------------------------------------------------------
+# configuration options related to source browsing
+#---------------------------------------------------------------------------
+SOURCE_BROWSER         = NO
+INLINE_SOURCES         = NO
+STRIP_CODE_COMMENTS    = YES
+REFERENCED_BY_RELATION = NO
+REFERENCES_RELATION    = NO
+REFERENCES_LINK_SOURCE = YES
+USE_HTAGS              = NO
+VERBATIM_HEADERS       = NO
+#---------------------------------------------------------------------------
+# configuration options related to the alphabetical class index
+#---------------------------------------------------------------------------
+ALPHABETICAL_INDEX     = NO
+COLS_IN_ALPHA_INDEX    = 5
+IGNORE_PREFIX          = 
+#---------------------------------------------------------------------------
+# configuration options related to the HTML output
+#---------------------------------------------------------------------------
+GENERATE_HTML          = YES
+HTML_OUTPUT            = html
+HTML_FILE_EXTENSION    = .html
+HTML_HEADER            = header.html
+HTML_FOOTER            = footer.html
+HTML_STYLESHEET        = htmlstyle.css
+HTML_ALIGN_MEMBERS     = YES
+GENERATE_HTMLHELP      = NO
+CHM_FILE               = 
+HHC_LOCATION           = 
+GENERATE_CHI           = NO
+BINARY_TOC             = NO
+TOC_EXPAND             = YES
+DISABLE_INDEX          = YES
+ENUM_VALUES_PER_LINE   = 1
+GENERATE_TREEVIEW      = YES
+TREEVIEW_WIDTH         = 250
+#---------------------------------------------------------------------------
+# configuration options related to the LaTeX output
+#---------------------------------------------------------------------------
+GENERATE_LATEX         = YES
+LATEX_OUTPUT           = latex
+LATEX_CMD_NAME         = latex
+MAKEINDEX_CMD_NAME     = makeindex
+COMPACT_LATEX          = NO
+PAPER_TYPE             = letter
+EXTRA_PACKAGES         = 
+LATEX_HEADER           = 
+PDF_HYPERLINKS         = YES
+USE_PDFLATEX           = YES
+LATEX_BATCHMODE        = NO
+LATEX_HIDE_INDICES     = NO
+#---------------------------------------------------------------------------
+# configuration options related to the RTF output
+#---------------------------------------------------------------------------
+GENERATE_RTF           = YES
+RTF_OUTPUT             = rtf
+COMPACT_RTF            = NO
+RTF_HYPERLINKS         = YES
+RTF_STYLESHEET_FILE    = 
+RTF_EXTENSIONS_FILE    = 
+#---------------------------------------------------------------------------
+# configuration options related to the man page output
+#---------------------------------------------------------------------------
+GENERATE_MAN           = NO
+MAN_OUTPUT             = man
+MAN_EXTENSION          = .3
+MAN_LINKS              = NO
+#---------------------------------------------------------------------------
+# configuration options related to the XML output
+#---------------------------------------------------------------------------
+GENERATE_XML           = NO
+XML_OUTPUT             = xml
+XML_SCHEMA             = 
+XML_DTD                = 
+XML_PROGRAMLISTING     = YES
+#---------------------------------------------------------------------------
+# configuration options for the AutoGen Definitions output
+#---------------------------------------------------------------------------
+GENERATE_AUTOGEN_DEF   = NO
+#---------------------------------------------------------------------------
+# configuration options related to the Perl module output
+#---------------------------------------------------------------------------
+GENERATE_PERLMOD       = NO
+PERLMOD_LATEX          = NO
+PERLMOD_PRETTY         = YES
+PERLMOD_MAKEVAR_PREFIX = 
+#---------------------------------------------------------------------------
+# Configuration options related to the preprocessor   
+#---------------------------------------------------------------------------
+ENABLE_PREPROCESSING   = YES
+MACRO_EXPANSION        = YES
+EXPAND_ONLY_PREDEF     = YES
+SEARCH_INCLUDES        = YES
+INCLUDE_PATH           = 
+INCLUDE_FILE_PATTERNS  = 
+PREDEFINED             = WINDOWS LINUX X64
+EXPAND_AS_DEFINED      = IF_X64_ELSE
+SKIP_FUNCTION_MACROS   = YES
+#---------------------------------------------------------------------------
+# Configuration::additions related to external references   
+#---------------------------------------------------------------------------
+TAGFILES               = 
+GENERATE_TAGFILE       = 
+ALLEXTERNALS           = NO
+EXTERNAL_GROUPS        = YES
+PERL_PATH              = /usr/bin/perl
+#---------------------------------------------------------------------------
+# Configuration options related to the dot tool   
+#---------------------------------------------------------------------------
+CLASS_DIAGRAMS         = NO
+HIDE_UNDOC_RELATIONS   = YES
+HAVE_DOT               = NO
+CLASS_GRAPH            = YES
+COLLABORATION_GRAPH    = YES
+GROUP_GRAPHS           = YES
+UML_LOOK               = NO
+TEMPLATE_RELATIONS     = NO
+INCLUDE_GRAPH          = YES
+INCLUDED_BY_GRAPH      = YES
+CALL_GRAPH             = NO
+CALLER_GRAPH           = NO
+GRAPHICAL_HIERARCHY    = YES
+DIRECTORY_GRAPH        = YES
+DOT_IMAGE_FORMAT       = png
+DOT_PATH               = 
+DOTFILE_DIRS           = 
+MAX_DOT_GRAPH_WIDTH    = 1024
+MAX_DOT_GRAPH_HEIGHT   = 1024
+MAX_DOT_GRAPH_DEPTH    = 1000
+DOT_TRANSPARENT        = NO
+DOT_MULTI_TARGETS      = NO
+GENERATE_LEGEND        = YES
+DOT_CLEANUP            = YES
+#---------------------------------------------------------------------------
+# Configuration::additions related to the search engine   
+#---------------------------------------------------------------------------
+SEARCHENGINE           = YES
diff --git a/api/docs/Makefile b/api/docs/Makefile
new file mode 100644
index 000000000..552315234
--- /dev/null
+++ b/api/docs/Makefile
@@ -0,0 +1,243 @@
+# **********************************************************
+# Copyright (c) 2007-2009 VMware, Inc.  All rights reserved.
+# **********************************************************/
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+.PHONY: all clean mkimages
+
+ifndef DYNAMORIO_MAKE
+DYNAMORIO_MAKE := ../../make
+endif
+
+include $(DYNAMORIO_MAKE)/compiler.mk
+
+ifndef DYNAMORIO_HOME
+DYNAMORIO_HOME := ../..
+endif
+
+ifndef DYNAMORIO_LIBUTIL
+DYNAMORIO_LIBUTIL := ../../libutil
+endif
+
+ifeq ($(MACHINE), linux)
+DYNAMORIO_DIR_BASE := $(DYNAMORIO_HOME)/exports/x86_$(MACHINE)_
+else
+DYNAMORIO_DIR_BASE := $(DYNAMORIO_HOME)/exports/x86_$(MACHINE)_
+endif
+
+# allow yes or 1 for DEBUG or INTERNAL
+USE_DEBUG=$(DEBUG)
+ifeq ($(DEBUG),yes)
+USE_DEBUG := 1
+endif
+USE_INTERNAL=$(INTERNAL)
+ifeq ($(INTERNAL),yes)
+USE_INTERNAL := 1
+endif
+
+INTERNAL_EXISTS := $(wildcard $(DYNAMORIO_DIR_BASE)intr/dr_api.h)
+RELEASE_EXISTS  := $(wildcard $(DYNAMORIO_DIR_BASE)rel/dr_api.h)
+DEBUG_EXISTS    := $(wildcard $(DYNAMORIO_DIR_BASE)dbg/dr_api.h)
+
+ifeq ($(USE_DEBUG),1)
+RELEASE_EXISTS  := ""
+INTERNAL_EXISTS := ""
+else
+ifeq ($(USE_INTERNAL),1)
+RELEASE_EXISTS  := ""
+DEBUG_EXISTS    := ""
+endif
+endif
+
+ifeq ($(RELEASE_EXISTS), $(DYNAMORIO_DIR_BASE)rel/dr_api.h)
+BUILD := rel
+else
+ifeq ($(INTERNAL_EXISTS), $(DYNAMORIO_DIR_BASE)intr/dr_api.h)
+BUILD := intr
+else
+ifeq ($(DEBUG_EXISTS), $(DYNAMORIO_DIR_BASE)dbg/dr_api.h)
+BUILD := dbg
+else
+$(error API header files do not exist)
+endif
+endif
+endif
+
+ifeq ($(MACHINE), linux)
+EXPORT_DIR := $(DYNAMORIO_DIR_BASE)$(BUILD)
+else
+EXPORT_DIR := $(shell $(CYGPATH) -u $(DYNAMORIO_DIR_BASE)$(BUILD))
+endif
+HEADERS    := $(wildcard $(EXPORT_DIR)/dr_*.h)
+
+# We need to escape EXPORT_DIR so we can pass it to sed.
+# Use sed to put in the escape characters.
+EXPORT_ESC := $(shell $(ECHO) "$(EXPORT_DIR)" | $(SED) 's/\./\\\./g')
+EXPORT_ESC := $(shell $(ECHO) "$(EXPORT_ESC)" | $(SED) 's/\//\\\//g')
+EXPORT_SED := "s/\(INPUT[ \t]*=\)/\1 $(EXPORT_ESC)/g"
+
+# We also need to escape the path to libutil
+LIBUTIL_DIR := $(DYNAMORIO_LIBUTIL)
+ifeq ($(MACHINE), win32)
+LIBUTIL_DIR := $(shell $(CYGPATH) -u $(DYNAMORIO_LIBUTIL))
+endif
+LIBUTIL_ESC := $(shell $(ECHO) "$(LIBUTIL_DIR)" | $(SED) 's/\./\\\./g')
+LIBUTIL_ESC := $(shell $(ECHO) "$(LIBUTIL_ESC)" | $(SED) 's/\//\\\//g')
+LIBUTIL_SED := "s/\(INPUT[ \t]*=\)/\1 $(LIBUTIL_ESC)/g"
+
+
+all: $(HEADERS) mkimages Doxyfile
+	@$(ECHO) "** Using headers in $(EXPORT_DIR) **"
+ifdef VMSAFE
+	@$(ECHO) "Use the macro \\DynamoRIO to allow name substitution"
+# we don't do -i to allow function names and macros
+	$(GREP) -E -n -H "(^|[^\\])DynamoRIO" *.dox; if [ $$? == 0 ]; then false; fi
+	$(GREP) -E -n -i -H "(^|[^_\\-])client($$|[^_])" *.dox; if [ $$? == 0 ]; then false; fi
+endif
+
+# run doxygen and fail on warnings
+	$(DOXYGEN) 2>&1 | $(TEE) doxygen.out
+	$(GREP) -i warning doxygen.out; if [ $$? == 0 ]; then false; fi
+	$(RM) doxygen.out
+
+# doxygen does not put header.html on the top-level frame so we add favicon manually
+	$(SED) -i 's/<title>/<link rel="shortcut icon" type="image\/x-icon" href="favicon.ico" \/>\n<title>/' html/index.html
+# add our logo to treeview pane (yes there is no closing </body></html>)
+	$(ECHO) '<p>&nbsp;<br><img src="drlogo.png"></p>' >> html/tree.html
+# workaround for doxygen bug
+	$(CP) html/htmlstyle.css html/doxygen.css
+# get our samples in place
+	$(MKDIR) -p html/samples
+	$(CP) -a ../samples/*.c html/samples/
+# not exporting custom stats for VMsafe, security api for VMAP
+# FIXME: these exclusions are duplicated in tools/makezip.sh: combine somehow?
+ifdef VMSAFE
+# -f because of perforce file protections being carried over
+	$(RM) html/samples/stats.c html/samples/tracedump.c
+	$(CP) -a ../samples/MF_moduledb-sample.config html/samples/
+	$(CP) -a ../samples/VIPA_test.exe html/samples/
+else
+	$(RM) html/samples/MF_moduledb*
+ifeq ($(MACHINE), linux)
+# no stats sample on linux
+	$(RM) html/samples/stats.c
+endif
+endif
+	$(CP) -a ../samples/Makefile html/samples/
+# IE for some reason won't initially load an .ico in a web page from a server,
+# though will show it via file: or after pointing straight at image from server.
+# doxygen fails to copy image referenced only in footer.html
+	$(CP) images/favicon.png html/
+
+pdf: all
+	-$(CP) images/vmware-logo-crop.png latex/vmware-logo.png
+	./fix-latex.pl
+	$(MAKE) -C latex
+
+# note that we can have "a" vs "an" issues
+# thus, going w/ a consonant-onset acronym:
+# VIPER = VMsafe In-ProcEss Runtime
+VMSAFE_NAME=VIPER
+# Viper In-Process Agent: again, starting w/ a consonant
+VMSAFE_CLIENT=VIPA
+
+Doxyfile: API.doxy Makefile
+	$(SED) $(EXPORT_SED) $< > $@
+	$(SED) -i $(LIBUTIL_SED) $@
+ifdef VMSAFE
+	$(SED) -i 's/\(PROJECT_NAME[ \t]*=\).*/\1 "VMsafe In-Process API"/' $@
+	$(SED) -i 's/\(ENABLED_SECTIONS[ \t]*=\)/\1 vmsafe/' $@
+	$(SED) -i 's/DynamoRIO=DynamoRIO/DynamoRIO=${VMSAFE_NAME}/' $@
+	$(SED) -i 's/client=client/client=${VMSAFE_CLIENT}/' $@
+	$(SED) -i 's/clients=clients/clients=${VMSAFE_CLIENT}s/' $@
+	$(SED) -i 's/Client=Client/Client=${VMSAFE_CLIENT}/' $@
+	$(SED) -i 's/Clients=Clients/Clients=${VMSAFE_CLIENT}s/' $@
+else
+	$(SED) -i 's/\(ENABLED_SECTIONS[ \t]*=\)/\1 linux/' $@
+endif
+# "doxygen -u" seems to preserve our non-default settings, while eliminating
+# warnings about deprecated options due to differing versions of doxygen
+	$(DOXYGEN) -u
+
+ifndef HEADER_DIR
+update_headers:
+	@$(ECHO) "must set HEADER_DIR variable"; false
+else
+update_headers:
+# even w/ no extension given, my cygwin perl makes .bak files!
+	cd $(HEADER_DIR); \
+	  $(PERL) -pi -e 's|\bDynamoRIO\b|${VMSAFE_NAME}|g' *.h; \
+	  $(PERL) -pi -e 's|\bDR\b|${VMSAFE_NAME}|g' *.h;        \
+	  $(PERL) -pi -e 's|\bclient(s?)\b|${VMSAFE_CLIENT}\1|gi' *.h; \
+	  $(RM) *.bak
+endif
+
+####################
+# Figures from xfig
+#
+# Requires fig2dev from the transfig package and mogrify from the ImageMagick
+# package to be installed.
+ifdef VMSAFE
+FIGXL8 := $(SED) 's|DynamoRIO|${VMSAFE_NAME}|g'
+else
+FIGXL8 := $(CAT)
+endif
+GENIMGDIR := genimages
+FIGS := $(wildcard *.fig)
+FIG_IMAGES := $(patsubst %.fig,$(GENIMGDIR)/%.eps,$(FIGS)) \
+              $(patsubst %.fig,$(GENIMGDIR)/%.png,$(FIGS))
+mkimages: $(GENIMGDIR) $(FIG_IMAGES)
+$(GENIMGDIR):
+	if ! test -d $(GENIMGDIR); then $(MKDIR) -p $(GENIMGDIR); fi
+# fig2dev w/ pipe or redirect crashes sometimes on cygwin so
+# we use ugly temp files
+$(GENIMGDIR)/%.post.fig: %.fig
+	${FIGXL8} $< > $@
+$(GENIMGDIR)/%.eps: $(GENIMGDIR)/%.post.fig
+	$(FIG2DEV) -L eps $< $@
+# Because fig2dev does not perform anti-aliasing, converting directly
+# to an image format can result in very poor quality lines and text.
+# This problem is compounded if that image is subsequently resized
+# without any anti-aliasing, such as by your web browser.  The
+# solution is to magnify the vector data to at least 4x and then
+# generate a lossless bitmap format such as TIFF; then have a real
+# image manipulator resize it to the final size (cannot make
+# browser-resizable vector graphics).
+# See also http://www.burningcutlery.com/derek/bargraph/
+%.png: %.tiff
+	$(MOGRIFY) -resize 33% -format png $<
+# ImageMagick 6.0.4 produces foo.png.0 while 6.3.0 creates foo-0.png
+	@(if test -e $*-1.png; then $(RM) $*-1.png; $(MV) $*-0.png $@; else \
+	  $(RM) $*.png.1; $(MV) $*.png.0 $@; fi)
+$(GENIMGDIR)/%.tiff: $(GENIMGDIR)/%.post.fig
+	$(FIG2DEV) -L tiff -m 4 $< $@
+####################
+
+clean:
+	$(RM) -r html latex rtf Doxyfile $(GENIMGDIR)
+
diff --git a/api/docs/bt.dox b/api/docs/bt.dox
new file mode 100644
index 000000000..188e82e2f
--- /dev/null
+++ b/api/docs/bt.dox
@@ -0,0 +1,978 @@
+/* **********************************************************
+ * Copyright (c) 2007-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/**
+***************************************************************************
+***************************************************************************
+\page API_BT Code Manipulation API
+
+The Code Manipulation API exposes the full power of \DynamoRIO, allowing
+tools to observe and modify the application's actual code stream as it
+executes.  Modifications are not limited to trampoline insertion and can
+include arbitrary changes.  We divide the API description into the
+following sections:
+
+ - \ref sec_IR
+ - \ref sec_events_bt
+ - \ref sec_decode
+ - \ref sec_64bit
+ - \ref sec_IR_utils
+ - \ref sec_translation
+ - \ref bt_examples
+\ifnot vmsafe - \ref sec_startstop
+\endif
+
+***************************************************************************
+\htmlonly
+<table width=100% bgcolor="#000000" cellspacing=0 cellpadding=2 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td></td></tr></table></td></tr></table></td></tr></table>
+\endhtmlonly
+\section sec_IR Instruction Representation
+
+The primary data structures involved in instruction manipulation are
+the \c instr_t, which represents a single IA-32 instruction, and the \c
+instrlist_t, which is a linked list of instructions.  The header files
+dr_ir_instrlist.h and dr_ir_instr.h list a number of functions that
+operate on these data structures, including:
+
+- Routines to create new instructions.
+- Routines to iterate over an instruction's operands.
+- Routines to iterate over an \c instrlist_t.
+- Routines to insert and remove an \c instr_t from an \c instrlist_t.
+
+As we will see in the the \ref sec_events_bt section that follows, a
+\client usually interacts with \c instrlist_t's in the form of \e basic \e
+blocks or \e traces.  A basic block is a sequence of instructions that
+terminates with a control transfer operation.  Traces are
+frequently-executed sequences of basic blocks that \DynamoRIO forms
+dynamically as the application executes, i.e., \e hot code.
+Collectively, we refer to basic blocks and traces as \e fragments.
+Both basic blocks and traces present a linear view of control flow.
+In other words, instruction sequences have a single entrance and one
+or more exits.  This representation greatly simplifies analysis and is
+a primary contributor to \DynamoRIO's efficiency.
+
+
+\if level_of_detail
+********************
+\subsection sec_IR_adaptive Adaptive Level of Detail
+
+It is costly to decode IA-32 instructions.  Fortunately, \DynamoRIO is
+often only interested in high-level information for a subset of
+instructions, such as just the control-flow instructions.  Each
+instruction can be at one of five levels of detail.  To simplify \clients,
+\DynamoRIO only passes them instructions at Level 3 or Level 4.
+Internally, \DynamoRIO uses all five levels:
+
+\par Level 0: Raw Bundle
+
+At Level 0, the \c instr_t data structure holds raw bytes for group of
+instructions decoded only enough to determine the final instruction
+boundary:
+
+<table border=0 cellpadding=2 cellspacing=1>
+<tr bgcolor="ffcc99"><td><tt>
+8d 34 01 8b 46 0c 2b 46 1c 0f b7 4e 08 c1 e1 07 3b c1 0f 8d a2 0a 00 00
+</tt></td></tr>
+</table>
+
+\par Level 1: Raw Individual
+
+At Level 1, each \c instr_t holds only one instruction, but has no more
+information than raw bytes:
+
+<table border=0 cellpadding=2 cellspacing=1>
+<tr><td bgcolor="ffcc99"><tt>8d 34 01</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>8b 46 0c</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>2b 46 1c</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>0f b7 4e 08</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>c1 e1 07</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>3b c1</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>0f 8d a2 0a 00 00</tt></td></tr>
+</table>
+
+\par Level 2: Opcode and Eflags
+
+At Level 2, \c instr_t has been decoded just enough to determine opcode and
+flags effects (flags are important when analyzing IA-32 code).  Raw bytes
+are still used for encoding.  The flags effects below are only shown for
+reading (R) or writing (W) the six arithmetic flags (Carry, Parity, Adjust,
+Zero, Sign, and Overflow).
+
+<table border=0 cellpadding=2 cellspacing=1>
+<tr><td bgcolor="ffcc99"><tt>8d 34 01</tt></td>
+  <td bgcolor="ccccff"><tt>lea</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>-</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;</td><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>8b 46 0c</tt></td>
+  <td bgcolor="ccccff"><tt>mov</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>-</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;</td><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>2b 46 1c</tt></td>
+  <td bgcolor="ccccff"><tt>sub</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>WCPAZSO</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;</td><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>0f b7 4e 08</tt></td>
+  <td bgcolor="ccccff"><tt>movzx</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>-</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;</td><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>c1 e1 07</tt></td>
+  <td bgcolor="ccccff"><tt>shl</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>WCPAZSO</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;</td><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>3b c1</tt></td>
+  <td bgcolor="ccccff"><tt>cmp</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>WCPAZSO</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;</td><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>0f 8d a2 0a 00 00</tt></td>
+  <td bgcolor="ccccff"><tt>jnl</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>RSO</tt></td></tr>
+</table>
+
+\par Level 3: Operands
+
+A Level 3 \c instr_t contains dynamically allocated arrays of source and
+destination operands (dynamic because IA-32 is so variable) that are now
+filled in.  Raw bytes are still valid and are used for encoding.  This
+level combines high-level information with quick encoding.
+
+<table border=0 cellpadding=2 cellspacing=1>
+<tr><td bgcolor="ffcc99"><tt>8d 34 01</tt></td>
+  <td bgcolor="ccccff"><tt>lea</tt></td>
+  <td bgcolor="ccff66"><tt>(%ecx,%eax,1) => %esi</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>-</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;</td><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>8b 46 0c</tt></td>
+  <td bgcolor="ccccff"><tt>mov</tt></td>
+  <td bgcolor="ccff66"><tt>0xc(%esi) => %eax</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>-</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;</td><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>2b 46 1c</tt></td>
+  <td bgcolor="ccccff"><tt>sub</tt></td>
+  <td bgcolor="ccff66"><tt>0x1c(%esi) %eax => %eax</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>WCPAZSO</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;</td><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>0f b7 4e 08</tt></td>
+  <td bgcolor="ccccff"><tt>movzx</tt></td>
+  <td bgcolor="ccff66"><tt>0x8(%esi) => %ecx</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>-</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;</td><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>c1 e1 07</tt></td>
+  <td bgcolor="ccccff"><tt>shl</tt></td>
+  <td bgcolor="ccff66"><tt>$0x07 %ecx => %ecx</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>WCPAZSO</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;</td><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>3b c1</tt></td>
+  <td bgcolor="ccccff"><tt>cmp</tt></td>
+  <td bgcolor="ccff66"><tt>%eax %ecx</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>WCPAZSO</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;</td><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>0f 8d a2 0a 00 00</tt></td>
+  <td bgcolor="ccccff"><tt>jnl</tt></td>
+  <td bgcolor="ccff66"><tt>$0x77f52269</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>RSO</tt></td></tr>
+</table>
+
+\par Level 4: Modified Operands
+
+At the highest level, \c instr_t has been modified at the operand level, or
+has been created from operands, such that the raw bytes are no longer
+valid.  The \c instr_t must be fully encoded from its operands.
+
+<table border=0 cellpadding=2 cellspacing=1>
+<tr><td>&nbsp;</td>
+  <td bgcolor="ccccff"><tt>lea</tt></td>
+  <td bgcolor="ccff66"><tt>(%ecx,%eax,1) => \e %edi</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>-</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;</td><td>&nbsp;|</td></tr>
+<tr><td>&nbsp;</td>
+  <td bgcolor="ccccff"><tt>mov</tt></td>
+  <td bgcolor="ccff66"><tt>0xc(\e %edi) => %eax</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>-</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;</td><td>&nbsp;|</td></tr>
+<tr><td>&nbsp;</td>
+  <td bgcolor="ccccff"><tt>sub</tt></td>
+  <td bgcolor="ccff66"><tt>0x1c(\e %edi) %eax => %eax</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>WCPAZSO</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;</td><td>&nbsp;|</td></tr>
+<tr><td>&nbsp;</td>
+  <td bgcolor="ccccff"><tt>movzx</tt></td>
+  <td bgcolor="ccff66"><tt>0x8(\e %edi) => %ecx</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>-</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;</td><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>c1 e1 07</tt></td>
+  <td bgcolor="ccccff"><tt>shl</tt></td>
+  <td bgcolor="ccff66"><tt>$0x07 %ecx => %ecx</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>WCPAZSO</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;</td><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>3b c1</tt></td>
+  <td bgcolor="ccccff"><tt>cmp</tt></td>
+  <td bgcolor="ccff66"><tt>%eax %ecx</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>WCPAZSO</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;</td><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>0f 8d a2 0a 00 00</tt></td>
+  <td bgcolor="ccccff"><tt>jnl</tt></td>
+  <td bgcolor="ccff66"><tt>$0x77f52269</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>RSO</tt></td></tr>
+</table>
+
+\par Basic Block Example
+
+As an example of using different levels of detail, a basic block in
+\DynamoRIO is represented using a Level 0 \c instr_t for all non-control-flow
+instructions, and a Level 3 \c instr_t for the block-ending control-flow
+instruction:
+
+<table border=0 cellpadding=2 cellspacing=1>
+<tr bgcolor="ffcc99"><td colspan=4><tt>
+8d 34 01 8b 46 0c 2b 46 1c 0f b7 4e 08 c1 e1 07 3b c1
+</tt></td></tr>
+<tr align="middle" bgcolor="ffffff"><td>&nbsp;|</td></tr>
+<tr><td bgcolor="ffcc99"><tt>0f 8d a2 0a 00 00</tt></td>
+  <td bgcolor="ccccff"><tt>jnl&nbsp;&nbsp;&nbsp;&nbsp;</tt></td>
+  <td bgcolor="ccff66"><tt>$0x77f52269&nbsp;&nbsp;&nbsp;&nbsp;</tt></td>
+  <td bgcolor="ccccff" align="middle"><tt>&nbsp;&nbsp;RSO&nbsp;&nbsp;</tt></td></tr>
+</table>
+
+However, when a \client registers for the basic block event, \DynamoRIO
+passes an \c instrlist_t of all Level 3 \c instr_t's, for simplicity.
+\endif
+
+
+***************************************************************************
+\htmlonly
+<table width=100% bgcolor="#000000" cellspacing=0 cellpadding=2 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td></td></tr></table></td></tr></table></td></tr></table>
+\endhtmlonly
+\section sec_events_bt Events
+
+The core of a \client's interaction with \DynamoRIO occurs through <em>
+event hooks</em>: the \client registers its own callback routine (or \e
+hook) for each event it is interested in.  \DynamoRIO calls the
+\client's event hooks at appropriate times, giving the \client access to
+key actions during execution of the application.  The \ref sec_events
+section describes events common to the entire \DynamoRIO API. Here we
+discuss the events specific to the Code Manipulation portion.
+
+\DynamoRIO provides two events related to fragment creation: one for
+basic blocks and one for traces (see dr_register_bb_event() and
+dr_register_trace_event()).  Through these fragment-creation hooks,
+the \client has the ability to inspect and modify any piece of code
+that \DynamoRIO emits before it executes.  Using the basic block hook,
+a \client sees \e all application code.  The trace-creation hook
+provides a mechanism for \clients to instrument only
+frequently-executed code paths.
+
+The code manipulation API is highly efficient in that fragment
+creation comprises a small part of \DynamoRIO's overhead.  A \client's
+analysis time rarely impacts execution time; only instrumentation code
+inserted by the \client affects efficiency.
+
+
+******************** \subsection sec_events_bb Basic Block Creation
+
+Through the basic block creation event, registered via
+dr_register_bb_event(), the \client has the ability to inspect and transform
+any piece of code prior to its execution.  The \client's hook receives
+three parameters:
+
+\code
+dr_emit_flags_t new_block(void *drcontext, void *tag, instrlist_t *bb, 
+                          bool for_trace, bool translating);
+\endcode 
+
+ - \c drcontext is a pointer to the input program's machine context.
+   \Clients should not inspect or modify the context; it is provided as
+   an opaque pointer (i.e., <tt>void *</tt>) to be passed to API
+   routines that require access to this internal data.
+
+ - \c tag is a unique identifier for the basic block fragment.
+
+ - \c bb is a pointer to the list of instructions that comprise the
+   basic block.  \Clients can examine, manipulate, or completely
+   replace the instructions in the list.
+
+ - \c for_trace indicates whether this callback is for a new basic block
+   (false) or for adding a basic block to a trace being created (true).
+   The \client has the opportunity to either include the same modifications
+   made to the standalone basic block, or to use different modifications,
+   for the code in the trace.
+
+ - \c translating indicates whether this callback is for basic block
+   creation (false) or is for address translation (true). This is further
+   explained in \ref sec_translation.
+
+The return value of the basic block callback should generally be
+DR_EMIT_DEFAULT; however, time-varying instrumentation or complex code
+transformations may need to return DR_EMIT_STORE_TRANSLATIONS.  See \ref
+sec_translation for further details.
+
+To iterate over instructions in an \c instrlist_t, use the \ref
+instrlist_first(), \ref instrlist_last(), and \ref instr_get_next()
+routines.  For example:
+
+\code
+dr_emit_flags_t new_block(void *drcontext, void *tag, instrlist_t *bb, 
+                          bool for_trace, bool translating)
+{
+  instr_t *instr, *next;
+  for (instr = instrlist_first(bb);
+       instr != instrlist_last(bb);
+       instr = next) {
+    next = instr_get_next(instr);
+    /* do some processing on instr */
+  }
+  return DR_EMIT_DEFAULT;
+}
+\endcode
+
+********************
+\subsection sec_Meta Meta-Instructions
+
+\DynamoRIO performs some processing on the basic block after the
+\client has finished with it, primarily modifying branches to ensure
+that \DynamoRIO retains control after execution.  It is important that
+the \client mark any control-flow instructions it does not want treated
+as application instructions as \e meta-instructions. Doing so informs
+\DynamoRIO that these instructions should execute natively rather than
+being trapped and redirected to new basic block fragments.
+Meta instructions are marked using these API routines:
+
+\code
+instr_set_ok_to_mangle()
+instrlist_meta_preinsert()
+instrlist_meta_postinsert()
+instrlist_meta_append()
+\endcode
+
+Through meta instructions, a \client can add its own internal control flow
+or make a call to a native routine that will not be brought into the code
+cache by \DynamoRIO.  However, such native calls need to be careful to
+remain transparent (see \ref sec_clean_call).
+
+While \DynamoRIO attempts to support arbitrary code transformations, its
+internal operation requires that we impose the following limitations:
+
+ - If there is more than one non-meta branch, only the last can be
+   conditional.
+ - A non-meta conditional branch must be the final instruction in the
+   block.
+ - There can only be one indirect branch (call, jump, or return) in 
+   a basic block, and it must be the final non-meta branch in the block.
+ - The exit control-flow of a block ending in a system call cannot be
+   changed.
+
+********************
+\subsection sec_events_trace Trace Creation
+
+\DynamoRIO provides access to traces primarily through the trace-creation
+event, registered via dr_register_trace_event().  It is important to note
+that \clients are not
+required to employ the trace-creation event to ensure full instrumentation.
+Rather, it is sufficient to perform all code modification using the basic
+block event.  Any basic blocks that \DynamoRIO chooses to place in a trace
+will contain all \client modifications (unless the \client behaves
+differently in the basic block hook when its \c for_trace parameter is
+true).  The trace-creation event provides
+a mechanism for \clients to instrument \e hot code separately.
+
+The parameters to the trace-creation event hook are nearly identical
+to those of the basic block hook:
+
+\code
+dr_emit_flags_t new_trace(void *drcontext, void *tag, instrlist_t *trace,
+                          bool translating);
+\endcode 
+
+ - \c drcontext is a pointer to the input program's machine context.
+   \Clients should not inspect or modify the context; it is provided as
+   an opaque pointer (i.e., <tt>void *</tt>) to be passed to API
+   routines that require access to this internal data.
+
+ - \c tag is a unique identifier for the trace fragment.
+
+ - \c bb is a pointer to the list of instructions that comprise the
+   trace.  \Clients can examine, manipulate, or completely replace the
+   instructions in the list.
+
+ - \c translating indicates whether this callback is for trace creation
+   (false) or is for address translation (true). This is further explained
+   in \ref sec_translation.
+
+The return value of the trace callback should generally be DR_EMIT_DEFAULT;
+however, time-varying instrumentation or complex code transformations may
+need to return DR_EMIT_STORE_TRANSLATIONS.  See \ref sec_translation for
+further details.
+
+\DynamoRIO calls the \client-supplied event hook each time a trace is
+created, just before the trace is emitted into the code cache.
+Additionally, as each constituent basic block is added to the trace,
+\DynamoRIO calls the basic block creation hook with the \p for_trace
+parameter set to true.  In order to preserve basic block instrumentation
+inside of traces, a \client need only act identically with respect to the
+\p for_trace parameter; it can ignore the trace event if its goal is to
+place instrumentation on all code.
+
+The constituent basic blocks will be stitched together prior to insertion
+in the code cache: conditional branches will be realigned so that their
+fall-through target remains on the trace, and inlined indirect branches
+will be preceded by a comparison against the on-trace target.
+
+If the basic block callback behaves differently based on the \p for_trace
+parameter, different instrumentation will exist in the trace as opposed to
+the standalone basic block.  If the basic block corresponds to the
+application code at the start of the trace (i.e., it is a trace head), the
+trace will shadow the basic block and the trace will be executed
+preferentially.  If #dr_delete_fragment() is called, it will also delete
+the trace first and may leave the basic block in place.  The flush routines
+(#dr_flush_region(), #dr_delay_flush_region(), #dr_unlink_flush_region()),
+however, will delete traces and basic blocks alike.
+
+********************
+\subsection sec_events_translation State Restoration
+
+If a \client is only adding instrumentation (meta instructions) that do not
+reference application memory, and is not reordering or removing application
+instructions, then it need not register for this event.  If, however, a
+\client is modifying application code or adding instructions that could
+fault, the \client must be capable of restoring the original context.
+\DynamoRIO calls the state restoration event, registered via
+dr_register_restore_state_event(), whenever it needs to translate a code
+cache context to an original application context:
+
+\code
+void restore_state(void *drcontext, void *tag, dr_mcontext_t *mcontext,
+                   bool restore_memory, bool app_code_consistent)
+\endcode
+
+See \ref sec_translation for further details.
+
+********************
+\subsection sec_events_del Basic Block and Trace Deletion
+
+\DynamoRIO can also provide notification of fragment deletion via
+dr_register_delete_event().  The signature for this event callback is:
+
+\code
+void fragment_deleted(void *drcontext, void *tag);
+\endcode
+
+\DynamoRIO calls this event hook each time it deletes a fragment from
+the code cache.  Such information may be needed if the \client
+maintains its own data structures about emitted fragment code that
+must be consistent across fragment deletions.
+
+
+***************************************************************************
+\htmlonly
+<table width=100% bgcolor="#000000" cellspacing=0 cellpadding=2 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td></td></tr></table></td></tr></table></td></tr></table>
+\endhtmlonly
+\section sec_decode Decoding and Encoding
+
+As discussed in \ref sec_events_bb and \ref sec_events_trace, a
+\client's primary interface to code inspection and manipulation is via
+the basic block and trace hooks.  However, \DynamoRIO also
+exports a rich set of functions and data structures to decode and
+encode instructions directly.  The following subsections overview this
+functionality.
+
+
+********************
+\subsection sec_Decoding Decoding
+
+\DynamoRIO provides several routines for decoding and disassembling
+IA-32 instructions.  The most common method for decoding is the
+decode() routine, which populates an \c instr_t data structure with all
+information about the instruction (e.g., opcode and operand
+information).
+
+When decoding instructions, \clients must explicitly manage the \c
+instr_t data structure.  For example, the following code shows how to
+use the instr_init(), instr_reset(), and instr_free() routines to
+decode a sequence of arbritrary instructions:
+
+\code
+instr_t instr;
+instr_init(&instr);
+do {
+  instr_reset(dcontext, &instr);
+  pc = decode(dcontext, pc, &instr);
+  /* check for invalid instr */
+  if (pc == NULL)
+    break;
+  if (instr_writes_memory(&instr)) {
+    /* do some processing */
+  }
+} while (pc < stop_pc);
+instr_free(dcontext, &instr);
+\endcode
+
+The 64-bit build of \DynamoRIO is capable of decoding 32-bit code as well.
+See \ref sec_64bit for full details.
+
+\subsection sec_InstrGen Instruction Generation
+
+\Clients can construct instructions from scratch in two different ways:
+
+ -# Using the INSTR_CREATE_opcode macros that fill
+    in implicit operands automatically:
+    \code
+instr_t *instr = INSTR_CREATE_dec(dcontext, opnd_create_reg(REG_EDX));
+    \endcode
+ -# Specifying the opcode and all operands (including implicit
+    operands):
+    \code
+instr_t *instr = instr_create(dcontext);
+instr_set_opcode(instr, OP_dec);
+instr_set_num_opnds(dcontext, instr, 1, 1);
+instr_set_dst(instr, 0, opnd_create_reg(REG_EDX));
+instr_set_src(instr, 0, opnd_create_reg(REG_EDX));
+    \endcode
+
+When using the second method, the exact order of operands and their
+sizes must match the templates that \DynamoRIO uses.  The
+INSTR_CREATE_ macros in dr_ir_macros.h should be consulted to
+determine the order.
+
+
+\subsection sec_Encoding Encoding
+
+\DynamoRIO's encoding routines take an instruction or list of
+instructions and encode them into the corresponding IA-32 bit pattern:
+
+\code instr_encode(), instrlist_encode() \endcode
+
+When encoding a control transfer instruction that targets another
+instruction, two encoding passes are performed: one to find the offset
+of the target instruction, and the other to link the control transfer
+to the proper target offset.
+
+The 64-bit build of \DynamoRIO is capable of encoding 32-bit code as well.
+See \ref sec_64bit for details.
+
+***************************************************************************
+\htmlonly
+<table width=100% bgcolor="#000000" cellspacing=0 cellpadding=2 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td></td></tr></table></td></tr></table></td></tr></table>
+\endhtmlonly
+\section sec_64bit 64-bit Versus 32-bit Instructions
+
+The 64-bit build of \DynamoRIO uses 64-bit decoding and encoding by
+default, while the 32-bit build uses 32-bit.  The 64-bit build is also
+capable of decoding and encoding 32-bit instructions.  Each thread has a
+current mode that is used to determine how to interpret instructions while
+decoding, whose default matches the \DynamoRIO build.  The set_x86_mode()
+routine changes the current mode, while get_x86_mode() queries the current
+mode.
+
+Each instruction contains a flag indicating whether it should be encoded in
+32-bit or 64-bit mode.  When an instruction is created or decoded, the
+instruction's flag is set to the thread's current mode.  It can be queried
+with instr_get_x86_mode() and changed with instr_set_x86_mode().
+
+
+***************************************************************************
+\htmlonly
+<table width=100% bgcolor="#000000" cellspacing=0 cellpadding=2 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td></td></tr></table></td></tr></table></td></tr></table>
+\endhtmlonly
+\section sec_IR_utils Utilities
+
+In addition to instruction decoding and encoding, the API includes
+several higher-level routines to facilitate code instrumentation.
+These include the following:
+
+- Routines to insert clean calls to \client-defined functions.
+- Routines to instrument control-flow instructions.
+- Routines to spill registers to \DynamoRIO's thread-private spill
+  slots.
+- Routines to quickly save and restore arithmetic flags,
+  floating-point state, and MMX/SSE registers.
+
+The following subsections describe these routines in more detail.
+
+********************
+\subsection sec_clean_call Clean Calls
+
+To make it easy to insert code into the application instruction
+stream, \DynamoRIO provides a <em>clean call</em> mechanism, which
+allows insertion of a transparent call to a \client routine.
+The dr_insert_clean_call() routine takes care of switching to a clean
+stack, setting up arguments to a call and making the call, optionally
+preserving floating point state, and preserving application state across
+the entire sequence.
+
+Here is an example of inserting a clean call to the \c at_mbr function:
+
+\code
+if (instr_is_mbr(instr)) {
+  app_pc address = instr_get_app_pc(instr);
+  uint opcode = instr_get_opcode(instr);
+  instr_t *nxt = instr_get_next(instr);
+  dr_insert_clean_call(drcontext, ilist, nxt, (void *) at_mbr,
+                       false/*don't need to save fp state*/,
+                       2 /* 2 parameters */,
+                       /* opcode is 1st parameter */
+                       OPND_CREATE_INT32(opcode),
+                       /* address is 2nd parameter */
+                       OPND_CREATE_INTPTR(address));
+}
+\endcode
+
+Through this mechanism, \clients can write analysis code in C or other
+high-level languages and easily insert calls to these routines in the
+instruction stream.  Note, however, that saving and restoring machine state
+is an expensive operation.  Performance-critical operations should be
+inlined for maximum efficiency.
+
+The stack that \DynamoRIO switches to for clean calls is relatively small:
+only 20KB by default.  \Clients can increase the size of the stack with the
+\ref op_stack_size "-stack_size" runtime option.  \Clients should also
+avoid keeping persistent state on the clean call stack, as it is wiped
+clean at the start of each clean call.
+
+The saved interrupted application state can be accessed using
+dr_get_mcontext() and modified using dr_set_mcontext().
+
+For performance reasons, clean calls do not save or restore floating point,
+MMX, or SSE state by default.  If the clean callee is using floating point
+or multimedia operations, it should request that the clean call mechanism
+preserve the floating point state through the appropriate parameter to
+dr_insert_clean_call().  See also
+\ref sec_trans_floating_point "Floating Point State, MMX, and SSE Transparency".
+
+If more detailed control over the call sequence is desired, it can be
+broken down into its constituent pieces:
+
+ - dr_prepare_for_call()
+ - Optionally, dr_insert_save_fpstate()
+ - dr_insert_call()
+ - Optionally, dr_insert_restore_fpstate()
+ - dr_cleanup_after_call()
+
+********************
+\subsection sec_state State Preservation
+
+To facilitate code transformations, \DynamoRIO makes available its register
+spill slots and other state preservation functionality.  It exports API
+routines for saving and restoring registers to and from thread-local spill
+slots:
+
+\code dr_save_reg(), dr_restore_reg(), and dr_reg_spill_slot_opnd() \endcode
+
+The values stored in these spill slots remain valid until the next non-meta
+(i.e. application) instruction and as such can be accessed from clean calls using:
+
+\code dr_read_saved_reg(), dr_write_saved_reg() \endcode
+
+For longer term persistance \DynamoRIO also provides a generic dedicated 
+thread-local storage field for use by \clients, making it easy to write 
+thread-aware \clients.  From C code, use:
+
+\code dr_get_tls_field(), dr_set_tls_field() \endcode
+
+To access this thread-local field from the code cache, use the following
+routines to generate the necessary code:
+
+\code dr_insert_read_tls_field(), dr_insert_write_tls_field() \endcode
+
+Since saving and restoring the \c eflags register is required for almost
+all code transformations, and since it is difficult to do so efficiently,
+we export routines that use our efficient method of arithmetic flag
+preservation:
+
+\code dr_save_arith_flags(), dr_restore_arith_flags() \endcode
+
+As just discussed in \ref sec_clean_call, we also export convenience
+routines for making \e clean (i.e., transparent) native calls from the code
+cache, as well as floating point and multimedia state preservation.
+
+
+********************
+\subsection sec_branch_instru Branch Instrumentation
+
+\DynamoRIO provides explicit support for instrumenting call
+instructions, direct (or unconditional) branches, indirect (or
+multi-way) branches, and conditional branches.  These convenience
+routines insert clean calls to \client-provided methods, passing as
+arguments the instruction pc and target pc of each control transfer,
+along with taken or not taken information for conditional branches:
+
+\code
+dr_insert_call_instrumentation()
+dr_insert_ubr_instrumentation()
+dr_insert_mbr_instrumentation()
+dr_insert_cbr_instrumentation()
+\endcode
+
+
+********************
+\subsection sec_adaptive Dynamic Instrumentation
+
+\DynamoRIO allows a \client to dynamically adjust its instrumentation
+by providing routines to flush all cached fragments corresponding to
+an application code region:
+
+\code
+dr_flush_region()
+dr_unlink_flush_region()
+dr_delay_flush_region()
+\endcode
+
+In order to directly modify the instrumentation on a particular fragment
+(as opposed to replacing instrumentation on all copies of fragments
+corresponding to particular application code), \DynamoRIO also supports
+directly replacing an existing fragment with a new \c instrlist_t:
+
+\code
+dr_replace_fragment()
+\endcode
+
+However, this routine is only supported when running with the \ref
+op_thread_priv "-thread_private" runtime option, and it replaces the
+fragment for the current thread only.  A \client can call this routine even
+while inside the to-be-replaced fragment (e.g., in a clean call from inside
+the fragment).  In this scenario, the old fragment is executed to
+completion and the new code is inserted before the next execution.
+
+For example usage, see the \client sample \ref sec_ex3.
+
+********************
+\subsection sec_custom_traces Custom Traces
+
+\DynamoRIO combines frequently executed sequences of basic blocks into
+<em>traces</em>.  It uses a simple profiling scheme based on <em>trace
+heads</em>, which are the targets of backward branches or exits from
+existing traces.  Execution counters are kept for each trace head.  Once a
+head crosses a threshold, the next sequence of basic blocks that are
+executed becomes a new trace.
+
+\DynamoRIO allows a \client to build custom traces by marking its own trace
+heads (<em>in addition</em> to \DynamoRIO's normal trace heads) and
+deciding when to end traces.  If a \client registers for the following
+event, \DynamoRIO will call its hook before extending a trace (with tag \c
+trace_tag) with a new basic block (with tag \c next_tag):
+
+\code
+int query_end_trace(void *drcontext, void *trace_tag, void *next_tag);
+\endcode
+
+The \client hook returns one of these values:
+ - CUSTOM_TRACE_DR_DECIDES = use standard termination criteria
+ - CUSTOM_TRACE_END_NOW = end trace now
+ - CUSTOM_TRACE_CONTINUE = do not end trace
+
+If using standard termination criteria, \DynamoRIO ends the trace if it
+reaches a trace head or another trace (or certain corner-case basic blocks
+that cannot be part of a trace).
+
+The \client can also mark any basic block as a trace head with
+\code dr_mark_trace_head() \endcode
+
+For example usage, see the callee-inlining \client sample \ref sec_ex5.
+
+\if custom_stubs
+********************
+\subsection sec_custom_stubs Custom Exit Stubs
+
+An exit cti can be given an instrlist_t to be prepended to the standard
+exit stub.  There are set and get methods for this custom exit stub code:
+
+\code
+void instr_set_exit_stub_code(instr_t *instr, instrlist_t *stub);
+instrlist_t *instr_exit_stub_code(instr_t *instr);
+\endcode
+
+When a fragment is re-decoded, e.g. when being appended to a trace or when
+re-decoded using dr_decode_fragment, the custom stubs are regenerated and
+added to the owning exit cti's.
+\endif
+
+
+\if prefixes
+********************
+\subsection sec_prefixes Prefixes
+
+Some code manipulations need to store a target address in a register and
+then jump there, but need the register to be restored as well.  \DynamoRIO
+provides a single-instruction prefix that is placed on all fragments (basic
+blocks as well as traces) that restores ecx.  It is on traces for internal
+\DynamoRIO use.  To have it added to basic blocks as well, call this routine
+during initialization:
+
+\code
+void dr_add_prefixes_to_basic_blocks();
+\endcode
+
+To have a cti target the prefix rather than the normal entry, use these set
+and get routines:
+
+\code
+bool instr_branch_targets_prefix(instr_t *instr);
+void instr_branch_set_prefix_target(instr_t *instr, bool val);
+\endcode
+\endif
+
+
+***************************************************************************
+\htmlonly
+<table width=100% bgcolor="#000000" cellspacing=0 cellpadding=2 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td></td></tr></table></td></tr></table></td></tr></table>
+\endhtmlonly
+\section sec_translation State Translation
+
+To support transparent fault handling, \DynamoRIO must translate a fault in
+the code cache into a fault at the corresponding application address.
+\DynamoRIO must also be able to translate when a suspended thread is
+examined by the application or by \DynamoRIO itself for internal
+synchronization purposes.
+
+If a \client is only adding observational instrumentation (i.e., \ref
+sec_Meta) (which should not fault) and is not modifying, reordering, or
+removing application instructions, these details can be ignored.  In that
+case the \client's basic block and trace callbacks should return
+#DR_EMIT_DEFAULT in addition to being deterministic and idempotent (i.e.,
+\DynamoRIO should be able to repeatedly call the callback and receive back
+the same resulting instruction list, with no net state changes to the
+\client).
+
+If a \client is performing modifications, then in order for \DynamoRIO to
+properly translate a code cache address the \client must use
+instr_set_translation() (chainable via INSTR_XL8()) in the basic block and
+trace creation callbacks to set the corresponding application address for
+each modified instruction and each added non-meta instruction.  The
+translation value is the application address that should be presented to
+the application as the faulting address, or the application address that
+should be restarted after a suspend.
+
+There are two methods for using the translated addresses:
+
+-# Return #DR_EMIT_STORE_TRANSLATIONS from the basic block creation
+   callback.  DR will then store the translation addresses and use
+   the stored information on a fault.  The basic block callback for
+   \p tag will not be called with \p translating set to true.  Note
+   that unless #DR_EMIT_STORE_TRANSLATIONS is also returned for \p
+   for_trace calls (or #DR_EMIT_STORE_TRANSLATIONS is returned in
+   the trace callback), each constituent block comprising the trace
+   will need to be re-created with both \p for_trace and \p
+   translating set to true.  Storing translations uses additional
+   memory that can be significant: up to 20% in some cases, as it
+   prevents DR from using its simple data structures and forces it
+   to fall back to its complex, corner-case design.  This is why DR
+   does not store all translations by default.
+-# Return #DR_EMIT_DEFAULT from the basic block or trace creation callback.
+   \DynamoRIO will then call the callback again during fault translation
+   with \p translating set to true.  All modifications to the instruction
+   list that were performed on the creation callback must be repeated on
+   the translating callback.  This option is only posible when basic block
+   modifications are deterministic and idempotent, but it saves memory.
+   Naturally, global state changes triggered by block creation should be
+   wrapped in checks for \p translating being false.  Even in this case,
+   instr_set_translation() should be called for non-meta instructions even
+   when \p translating is false, as \DynamoRIO may decide to store the
+   translations at creation time for reasons of its own.
+
+Furthermore, if the \client's modifications change any part of the machine
+state besides the program counter, the \client should use
+dr_register_restore_state_event() (see \ref sec_events_translation) to
+restore the registers to their original application values.
+
+For meta instructions that do not reference application memory (i.e., they
+should not fault), leave the translation field as NULL.  A NULL value
+instructs \DynamoRIO to use the subsequent non-meta instruction's
+translation as the application address, and to fail when translating the
+full state.  Since the full state will only be needed when relocating a
+thread (as stated, there will not be a fault here), failure indicates that
+this is not a valid relocation point, and \DynamoRIO's thread
+synchronization scheme will use another spot.  If the translation field is
+set to a non-NULL value, the \client should be willing to also restore the
+rest of the machine state at that point (restore spilled registers, etc.)
+via #dr_register_restore_state_event().  This is necessary for meta
+instructions that reference application memory.  \DynamoRIO takes care of
+such potentially-faulting instructions added by its own API routines
+(#dr_insert_clean_call() arguments that reference application data,
+#dr_insert_mbr_instrumentation()'s read of application indirect branch
+data, etc.)
+
+
+\ifnot vmsafe
+***************************************************************************
+\htmlonly
+<table width=100% bgcolor="#000000" cellspacing=0 cellpadding=2 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td></td></tr></table></td></tr></table></td></tr></table>
+\endhtmlonly
+\section sec_startstop Running a Subset of an Application
+
+An alternative to running an entire application under \DynamoRIO control is to
+use the Application Interface to specify a portion of the application to
+run.  This interface consists of the following routines:
+
+\code
+dr_app_setup()
+dr_app_start()
+dr_app_stop()
+dr_app_cleanup()
+dr_app_take_over()
+\endcode
+
+When building an executable that links with the \DynamoRIO library, you may
+need to explicitly link with the C library prior to the \DynamoRIO library:
+
+\code /link /nodefaultlib libcmt.lib dynamorio.lib \endcode
+
+\endif
+
+***************************************************************************
+****************************************************************************
+****************************************************************************
+*/
diff --git a/api/docs/client.fig b/api/docs/client.fig
new file mode 100644
index 000000000..856248b28
--- /dev/null
+++ b/api/docs/client.fig
@@ -0,0 +1,40 @@
+#FIG 3.2
+Landscape
+Center
+Inches
+Letter  
+100.00
+Single
+-2
+1200 2
+0 32 #000000
+0 33 #eeeeee
+0 34 #ffffaa
+0 35 #0000ff
+0 36 #6666ff
+0 37 #aaaaff
+0 38 #00ffff
+0 39 #77ff00
+0 40 #aaffaa
+0 41 #dd00ff
+0 42 #ffaaaa
+0 43 #cccccc
+0 44 #aaaaaa
+2 2 0 1 0 33 54 0 20 0.000 0 0 -1 0 0 5
+	 600 750 3600 750 3600 0 600 0 600 750
+2 1 0 1 0 7 50 0 -1 0.000 1 0 -1 1 0 2
+	1 1 1.00 60.00 90.00
+	 2100 750 2100 1200
+2 2 0 1 0 34 51 0 20 0.000 0 0 -1 0 0 5
+	 2775 1950 3600 1950 3600 1200 2775 1200 2775 1950
+2 2 0 1 0 42 51 0 20 0.000 0 0 -1 0 0 5
+	 600 1950 2775 1950 2775 1200 600 1200 600 1950
+2 1 0 1 0 7 50 0 -1 0.000 1 0 -1 1 0 2
+	1 1 1.00 60.00 90.00
+	 2100 1950 2100 2400
+2 2 0 1 0 44 54 0 20 0.000 0 0 -1 0 0 5
+	 600 3150 3600 3150 3600 2400 600 2400 600 3150
+4 0 0 50 0 0 12 0.0000 4 180 1425 1425 375 running application\001
+4 0 0 50 0 0 12 0.0000 4 180 945 1275 1575 DynamoRIO\001
+4 0 0 50 0 0 12 0.0000 4 135 420 3000 1575 client\001
+4 0 0 50 0 0 12 0.0000 4 180 1380 1425 2850 hardware platform\001
diff --git a/api/docs/fix-latex.pl b/api/docs/fix-latex.pl
new file mode 100755
index 000000000..d4d91c3a1
--- /dev/null
+++ b/api/docs/fix-latex.pl
@@ -0,0 +1,115 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2007 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# Script to post-process the latex auto-generated with doxygen.  It
+# looks like doxygen is pretty rigid about where it puts chapters and
+# sections; we'll clean up the latex ourselves before generating the
+# pdf.  Our approach is to try to replicate the structure of the html
+# output.
+
+$html_dir = "./html";
+$latex_dir = "./latex";
+
+$html_file = "$html_dir/tree.html";
+$latex_file = "$latex_dir/refman.tex";
+
+$latex_style = "$latex_dir/doxygen.sty";
+
+#
+# Read the first part of the current latex file, up until the first
+# \chapter declaration.  That portion will remain the same.
+#
+$latex = "";
+open (IN, $latex_file) || die "ERROR: can't open $latex_file\n";
+while (<IN>) {
+    last if ($_ =~ /^\\chapter/);
+    if (/Generated by Doxygen/) {
+        $latex .= "{\\small VMware Proprietary and Confidential}\\\\\n";
+        # eat the next two lines
+        $_ = <IN>; # vspace line
+        $_ = <IN>;
+        die "ERROR: unexpected line" unless (/^{\\small/);
+    } else {
+        $latex .= $_;
+    }
+}
+close (IN);
+
+open (OUT, ">$latex_file") || die "ERROR: can't overwrite $latex_file\n";
+print OUT $latex;
+
+#
+# Now parse the html file and match its layout.
+#
+open (IN, $html_file) || die "ERROR: can't open $html_file\n";
+
+$level = -1;
+while (<IN>) {
+    if ($_ =~ /<p>.*href=\"(.+)\.html\".*>(.*)<\/a><\/p>/) {
+        if (-e "$latex_dir/$1.tex") {
+            if ($level == 1) {
+                print OUT "\\chapter{$2}\n";
+            }
+            print OUT "\\input{$1}\n";
+        }
+    }
+    elsif ($_ =~ /<div/) {
+        $level++;
+    }
+    elsif ($_ =~ /<\/div/) {
+        $level--;
+    }
+}
+
+print OUT "\\printindex\n\\end{document}\n";
+
+close (IN);
+close (OUT);
+
+# now process the style file
+$latex = "";
+open (IN, $latex_style) || die "ERROR: can't open $latex_style\n";
+while (<IN>) {
+    # put in our logo and statement in the footers
+    if (/^\\rfoot/) {
+        $latex .= "\\rfoot[\\fancyplain{}{\\bfseries\\scriptsize \\raisebox{1ex}{VMware Proprietary and Confidential\\ \\ \\ } \\includegraphics[width=13ex]{vmware-logo} }]{}\n";
+    } elsif (/^\\lfoot/) {
+        $latex .= "\\lfoot[]{\\fancyplain{}{\\bfseries\\scriptsize \\includegraphics[width=13ex]{vmware-logo} \\raisebox{1ex}{\\ \\ \\ VMware Proprietary and Confidential} }}\n";
+    } else {
+        $latex .= $_;
+    }
+}
+close (IN);
+open (OUT, ">$latex_style") || die "ERROR: can't overwrite $latex_style\n";
+print OUT $latex;
+close (OUT);
+
diff --git a/api/docs/flow-highlevel.fig b/api/docs/flow-highlevel.fig
new file mode 100644
index 000000000..91946cc32
--- /dev/null
+++ b/api/docs/flow-highlevel.fig
@@ -0,0 +1,103 @@
+#FIG 3.2
+Landscape
+Center
+Inches
+Letter  
+100.00
+Single
+-2
+1200 2
+0 32 #000000
+0 33 #dfdfdf
+0 34 #ffffae
+0 35 #0000ff
+0 36 #6165ff
+0 37 #aeaaff
+0 38 #00ffff
+0 39 #71ff00
+0 40 #aeffae
+0 41 #df00ff
+0 42 #ffaaae
+0 43 #cfcfcf
+0 44 #aeaaae
+0 45 #efefef
+2 1 1 1 0 7 52 0 -1 4.000 1 0 -1 1 0 2
+	1 0 1.00 60.00 120.00
+	 675 1275 675 450
+2 1 0 1 0 7 52 0 -1 0.000 0 0 -1 1 0 3
+	1 1 1.00 60.00 120.00
+	 675 3450 675 1275 1725 1275
+2 1 0 1 0 7 52 0 -1 0.000 0 0 -1 1 0 3
+	1 1 1.00 60.00 120.00
+	 825 3450 825 1425 1725 1425
+2 1 1 1 0 7 52 0 -1 4.000 1 0 -1 1 0 2
+	1 0 1.00 60.00 120.00
+	 825 1425 1875 450
+2 1 1 1 0 7 52 0 -1 4.000 1 0 -1 1 0 2
+	1 0 1.00 60.00 120.00
+	 975 1575 3075 450
+2 1 0 1 0 7 52 0 -1 0.000 0 0 -1 1 0 3
+	1 1 1.00 60.00 120.00
+	 975 3450 975 1575 1725 1575
+2 1 0 1 0 7 52 0 -1 0.000 0 0 -1 1 0 2
+	1 1 1.00 60.00 120.00
+	 1050 450 2100 1200
+2 1 0 1 0 7 52 0 -1 0.000 1 0 -1 0 1 2
+	1 1 1.00 60.00 120.00
+	 2250 1200 2250 450
+2 1 0 1 0 7 52 0 -1 0.000 0 0 -1 1 0 2
+	1 1 1.00 60.00 120.00
+	 3450 450 2400 1200
+2 2 0 1 0 45 54 0 20 0.000 0 0 -1 0 0 5
+	 300 450 4200 450 4200 0 300 0 300 450
+2 1 0 1 0 7 52 0 -1 0.000 0 0 -1 0 0 2
+	 2925 0 2925 450
+2 1 0 1 0 7 52 0 -1 0.000 0 0 -1 0 0 2
+	 1575 0 1575 450
+2 2 0 1 0 43 54 0 20 0.000 0 0 -1 0 0 5
+	 4200 3900 300 3900 300 3450 4200 3450 4200 3900
+2 2 0 1 0 44 54 0 20 0.000 0 0 -1 0 0 5
+	 4200 4350 300 4350 300 3900 4200 3900 4200 4350
+2 1 0 1 0 7 52 0 -1 0.000 1 0 -1 0 1 2
+	1 1 1.00 60.00 120.00
+	 3150 1350 2775 1350
+2 1 0 1 0 42 52 0 -1 0.000 0 0 -1 1 0 2
+	1 1 1.00 60.00 120.00
+	 3150 1500 2775 1500
+2 1 0 1 0 42 52 0 -1 0.000 0 0 -1 1 0 2
+	1 1 1.00 60.00 120.00
+	 2175 1650 2175 2250
+2 1 0 1 0 42 52 0 -1 0.000 0 0 -1 1 0 2
+	1 1 1.00 60.00 120.00
+	 2325 2250 2325 1650
+2 1 0 1 0 42 52 0 -1 0.000 0 0 -1 1 0 4
+	1 1 1.00 60.00 120.00
+	 1725 2550 1425 2550 1425 2400 1725 2400
+2 2 0 1 0 37 54 0 20 0.000 0 0 -1 0 0 5
+	 1725 2700 2775 2700 2775 2250 1725 2250 1725 2700
+2 2 0 1 0 40 54 0 20 0.000 0 0 -1 0 0 5
+	 3150 1650 4200 1650 4200 1200 3150 1200 3150 1650
+2 2 0 1 0 34 54 0 20 0.000 0 0 -1 0 0 5
+	 1725 1650 2775 1650 2775 1200 1725 1200 1725 1650
+2 2 2 1 0 33 54 0 20 3.000 0 0 -1 0 0 5
+	 1725 1800 4200 1800 4200 2100 1725 2100 1725 1800
+2 2 0 1 0 42 56 0 20 0.000 0 0 -1 0 0 5
+	 75 975 4425 975 4425 2925 75 2925 75 975
+2 1 0 1 0 42 52 0 -1 0.000 0 0 -1 1 0 2
+	1 1 1.00 60.00 120.00
+	 2175 2700 2175 3450
+2 1 0 1 0 42 52 0 -1 0.000 0 0 -1 1 0 2
+	1 1 1.00 60.00 120.00
+	 2325 3450 2325 2700
+4 0 0 50 0 0 12 0.0000 4 180 1965 1275 300 multi-threaded application\001
+4 0 0 50 0 0 12 0.0000 4 180 1305 1650 3750 operating system\001
+4 0 0 50 0 0 12 0.0000 4 135 705 1950 4200 hardware\001
+4 0 0 50 0 0 12 0.0000 4 180 645 1950 1500 dispatch\001
+4 0 0 50 0 0 12 0.0000 4 135 855 3300 1425 basic block\001
+4 0 0 50 0 0 12 0.0000 4 135 510 3375 1575 builder\001
+4 0 0 50 0 0 12 0.0000 4 135 855 1875 2550 code cache\001
+4 0 0 50 0 0 12 0.0000 4 135 1125 2475 2025 context switch\001
+4 0 0 56 0 2 14 0.0000 4 195 1200 3075 2550 DynamoRIO\001
+4 0 0 50 0 0 12 0.0000 4 180 960 2475 3225 system calls\001
+4 0 0 50 0 0 12 1.5708 4 135 1215 375 2550 kernel-mediated\001
+4 0 0 50 0 0 12 1.5708 4 135 1260 525 2550 control transfers\001
diff --git a/api/docs/flow.fig b/api/docs/flow.fig
new file mode 100644
index 000000000..81cf76cf6
--- /dev/null
+++ b/api/docs/flow.fig
@@ -0,0 +1,112 @@
+#FIG 3.2
+Landscape
+Center
+Inches
+Letter  
+100.00
+Single
+-2
+1200 2
+0 32 #000000
+0 33 #dddddd
+0 34 #ffffaa
+0 35 #0000ff
+0 36 #6666ff
+0 37 #aaaaff
+0 38 #00ffff
+0 39 #77ff00
+0 40 #aaffaa
+0 41 #dd00ff
+0 42 #ffaaaa
+0 43 #cccccc
+0 44 #aaaaaa
+2 1 0 1 0 7 50 0 -1 0.000 1 0 -1 1 0 2
+	1 1 1.00 60.00 90.00
+	 3750 750 3750 450
+2 1 0 1 0 7 50 0 -1 0.000 1 0 -1 1 0 2
+	1 1 1.00 60.00 90.00
+	 4050 450 4050 750
+2 1 0 1 0 7 50 0 -1 0.000 1 0 -1 1 0 2
+	1 1 1.00 60.00 90.00
+	 5250 450 5250 750
+2 1 0 1 0 7 50 0 -1 0.000 1 0 -1 1 0 2
+	1 1 1.00 60.00 90.00
+	 4950 750 4950 450
+2 1 0 1 0 7 51 0 -1 0.000 1 0 -1 1 0 2
+	1 1 1.00 60.00 90.00
+	 1950 2250 4350 2250
+2 1 0 1 0 7 51 0 -1 0.000 1 0 -1 1 0 4
+	1 1 1.00 60.00 90.00
+	 7050 2850 7050 3225 3150 3225 3150 2850
+2 1 0 1 0 7 51 0 -1 0.000 1 0 -1 1 0 4
+	1 1 1.00 60.00 90.00
+	 1200 2850 1200 3075 750 3075 750 2850
+2 1 0 1 0 7 51 0 -1 0.000 1 0 -1 1 0 4
+	1 1 1.00 60.00 90.00
+	 5400 2850 5400 3075 4950 3075 4950 2850
+2 1 0 1 0 7 51 0 -1 0.000 1 0 -1 1 0 3
+	1 1 1.00 60.00 90.00
+	 3000 975 1575 975 1575 2100
+2 1 0 1 0 7 52 0 -1 4.000 1 0 -1 1 0 4
+	1 1 1.00 60.00 90.00
+	 4575 2850 4575 3075 3825 3075 3825 2850
+2 1 0 1 0 7 52 0 -1 4.000 1 0 -1 1 0 4
+	1 1 1.00 60.00 90.00
+	 6450 2850 6450 3075 5700 3075 5700 2850
+2 1 0 1 0 7 51 0 -1 0.000 1 0 -1 1 0 2
+	1 1 1.00 60.00 90.00
+	 4050 2625 4350 2625
+2 1 0 1 0 7 51 0 -1 0.000 1 0 -1 1 0 2
+	1 1 1.00 60.00 90.00
+	 1950 2625 2250 2625
+2 1 0 1 0 7 51 0 -1 4.000 1 0 -1 1 0 2
+	1 1 1.00 60.00 90.00
+	 5925 2625 6225 2625
+2 1 0 1 0 7 51 0 -1 0.000 1 0 -1 1 0 2
+	1 1 1.00 60.00 90.00
+	 5475 1200 5475 2100
+2 1 0 1 0 7 50 0 -1 0.000 1 0 -1 1 0 3
+	1 1 1.00 60.00 90.00
+	 1875 450 1875 825 3000 825
+2 2 0 1 0 34 51 0 20 0.000 0 0 -1 0 0 5
+	 3000 1200 5775 1200 5775 750 3000 750 3000 1200
+2 2 0 1 0 37 54 0 20 0.000 0 0 -1 0 0 5
+	 0 2850 0 2100 1950 2100 1950 2850 0 2850
+2 2 0 1 0 42 54 0 20 0.000 0 0 -1 0 0 5
+	 4050 2850 2250 2850 2250 2400 4050 2400 4050 2850
+2 2 0 1 0 37 54 0 20 0.000 0 0 -1 0 0 5
+	 4350 2850 4350 2100 5925 2100 5925 2850 4350 2850
+2 2 0 1 0 42 54 0 20 0.000 0 0 -1 0 0 5
+	 6225 2850 7800 2850 7800 2400 6225 2400 6225 2850
+2 2 0 1 0 40 54 0 20 0.000 0 0 -1 0 0 5
+	 2850 450 4500 450 4500 0 2850 0 2850 450
+2 2 0 1 0 40 54 0 20 0.000 0 0 -1 0 0 5
+	 6075 0 4800 0 4800 450 6075 450 6075 0
+2 2 0 0 7 33 54 0 20 0.000 0 0 -1 0 0 5
+	 0 1800 7800 1800 7800 1500 0 1500 0 1800
+2 1 0 1 0 7 50 0 -1 0.000 1 0 -1 1 0 3
+	1 1 1.00 60.00 90.00
+	 1875 2100 1875 1125 3000 1125
+2 1 0 1 0 7 50 0 -1 0.000 1 0 -1 1 0 2
+	1 1 1.00 60.00 90.00
+	 3150 2400 3150 1200
+2 1 0 1 0 7 51 0 -1 0.000 1 0 -1 1 0 2
+	1 1 1.00 60.00 90.00
+	 5625 2100 5625 1200
+2 1 0 1 0 7 52 0 -1 4.000 1 0 -1 1 0 4
+	1 1 1.00 60.00 90.00
+	 2475 2850 2475 3075 1725 3075 1725 2850
+4 0 0 50 0 0 12 0.0000 4 135 1875 75 2400 BASIC BLOCK CACHE\001
+4 0 0 50 0 0 12 0.0000 4 135 1245 300 2550 non-control-flow\001
+4 0 0 50 0 0 12 0.0000 4 135 900 525 2700 instructions\001
+4 0 0 50 0 0 12 0.0000 4 135 1275 4500 2400 TRACE CACHE\001
+4 0 0 50 0 0 12 0.0000 4 135 1245 4500 2550 non-control-flow\001
+4 0 0 50 0 0 12 0.0000 4 135 900 4725 2700 instructions\001
+4 0 0 50 0 0 12 0.0000 4 135 570 1575 375 START\001
+4 0 0 50 0 0 12 0.0000 4 135 1410 3000 278 basic block builder\001
+4 0 0 50 0 0 12 0.0000 4 180 645 4050 1012 dispatch\001
+4 0 0 50 0 0 12 0.0000 4 135 1050 4950 278 trace selector\001
+4 0 0 50 0 0 12 0.0000 4 135 1125 3750 1725 context switch\001
+4 0 0 50 0 0 12 0.0000 4 180 1665 2325 2677 indirect branch lookup\001
+4 0 0 50 0 0 12 0.0000 4 135 1125 6480 2602 indirect branch\001
+4 0 0 50 0 0 12 0.0000 4 180 1155 6495 2767 stays on trace?\001
diff --git a/api/docs/footer.html b/api/docs/footer.html
new file mode 100644
index 000000000..c84ca3bd9
--- /dev/null
+++ b/api/docs/footer.html
@@ -0,0 +1,5 @@
+<hr size="1"><address style="text-align: center;"><small>
+<img border=0 src="favicon.png"> &nbsp; $projectname version 1.3.1 --- $datetime 
+&nbsp; <img border=0 src="favicon.png">
+</body>
+</html>
diff --git a/api/docs/header.html b/api/docs/header.html
new file mode 100644
index 000000000..af289ffad
--- /dev/null
+++ b/api/docs/header.html
@@ -0,0 +1,7 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+<html><head><meta http-equiv="Content-Type" content="text/html;charset=iso-8859-1">
+<title>$title</title>
+<link href="$relpath$doxygen.css" rel="stylesheet" type="text/css">
+<link href="$relpath$tabs.css" rel="stylesheet" type="text/css">
+<link rel="shortcut icon" type="image/x-icon" href="favicon.ico" />
+</head><body>
diff --git a/api/docs/htmlstyle.css b/api/docs/htmlstyle.css
new file mode 100644
index 000000000..5d07aa095
--- /dev/null
+++ b/api/docs/htmlstyle.css
@@ -0,0 +1,359 @@
+BODY,H1,H2,H3,H4,H5,H6,P,CENTER,TD,TH,UL,DL,DIV {
+	font-family: Geneva, Arial, Helvetica, sans-serif;
+}
+BODY,TD {
+       font-size: 90%;
+}
+H1 {
+	text-align: center;
+       font-size: 160%;
+}
+H2 {
+       font-size: 130%;
+}
+H3 {
+       font-size: 110%;
+}
+CAPTION { font-weight: bold }
+DIV.qindex {
+	width: 100%;
+	background-color: #e8eef2;
+	border: 1px solid #84b0c7;
+	text-align: center;
+	margin: 2px;
+	padding: 2px;
+	line-height: 140%;
+}
+DIV.nav {
+	width: 100%;
+	background-color: #e8eef2;
+	border: 1px solid #84b0c7;
+	text-align: center;
+	margin: 2px;
+	padding: 2px;
+	line-height: 140%;
+}
+DIV.navtab {
+       background-color: #e8eef2;
+       border: 1px solid #84b0c7;
+       text-align: center;
+       margin: 2px;
+       margin-right: 15px;
+       padding: 2px;
+}
+TD.navtab {
+       font-size: 70%;
+}
+A.qindex {
+       text-decoration: none;
+       font-weight: bold;
+       color: #1A419D;
+}
+A.qindex:visited {
+       text-decoration: none;
+       font-weight: bold;
+       color: #1A419D
+}
+A.qindex:hover {
+	text-decoration: none;
+	background-color: #ddddff;
+}
+A.qindexHL {
+	text-decoration: none;
+	font-weight: bold;
+	background-color: #6666cc;
+	color: #ffffff;
+	border: 1px double #9295C2;
+}
+A.qindexHL:hover {
+	text-decoration: none;
+	background-color: #6666cc;
+	color: #ffffff;
+}
+A.qindexHL:visited { text-decoration: none; background-color: #6666cc; color: #ffffff }
+A.el { text-decoration: none; font-weight: bold }
+A.elRef { font-weight: bold }
+A.code:link { text-decoration: none; font-weight: normal; color: #0000FF}
+A.code:visited { text-decoration: none; font-weight: normal; color: #0000FF}
+A.codeRef:link { font-weight: normal; color: #0000FF}
+A.codeRef:visited { font-weight: normal; color: #0000FF}
+A:hover { text-decoration: none; background-color: #f2f2ff }
+DL.el { margin-left: -1cm }
+.fragment {
+       font-family: monospace, fixed;
+       font-size: 95%;
+}
+PRE.fragment {
+	border: 1px solid #CCCCCC;
+	background-color: #f5f5f5;
+	margin-top: 4px;
+	margin-bottom: 4px;
+	margin-left: 2px;
+	margin-right: 8px;
+	padding-left: 6px;
+	padding-right: 6px;
+	padding-top: 4px;
+	padding-bottom: 4px;
+}
+DIV.ah { background-color: black; font-weight: bold; color: #ffffff; margin-bottom: 3px; margin-top: 3px }
+
+DIV.groupHeader {
+       margin-left: 16px;
+       margin-top: 12px;
+       margin-bottom: 6px;
+       font-weight: bold;
+}
+DIV.groupText { margin-left: 16px; font-style: italic; font-size: 90% }
+BODY {
+	background: white;
+	color: black;
+	margin-right: 20px;
+	margin-left: 20px;
+}
+TD.indexkey {
+	background-color: #e8eef2;
+	font-weight: bold;
+	padding-right  : 10px;
+	padding-top    : 2px;
+	padding-left   : 10px;
+	padding-bottom : 2px;
+	margin-left    : 0px;
+	margin-right   : 0px;
+	margin-top     : 2px;
+	margin-bottom  : 2px;
+	border: 1px solid #CCCCCC;
+}
+TD.indexvalue {
+	background-color: #e8eef2;
+	font-style: italic;
+	padding-right  : 10px;
+	padding-top    : 2px;
+	padding-left   : 10px;
+	padding-bottom : 2px;
+	margin-left    : 0px;
+	margin-right   : 0px;
+	margin-top     : 2px;
+	margin-bottom  : 2px;
+	border: 1px solid #CCCCCC;
+}
+TR.memlist {
+   background-color: #f0f0f0; 
+}
+P.formulaDsp { text-align: center; }
+IMG.formulaDsp { }
+IMG.formulaInl { vertical-align: middle; }
+SPAN.keyword       { color: #008000 }
+SPAN.keywordtype   { color: #604020 }
+SPAN.keywordflow   { color: #e08000 }
+SPAN.comment       { color: #800000 }
+SPAN.preprocessor  { color: #806020 }
+SPAN.stringliteral { color: #002080 }
+SPAN.charliteral   { color: #008080 }
+.mdescLeft {
+       padding: 0px 8px 4px 8px;
+	font-size: 80%;
+	font-style: italic;
+	background-color: #FAFAFA;
+	border-top: 1px none #E0E0E0;
+	border-right: 1px none #E0E0E0;
+	border-bottom: 1px none #E0E0E0;
+	border-left: 1px none #E0E0E0;
+	margin: 0px;
+}
+.mdescRight {
+       padding: 0px 8px 4px 8px;
+	font-size: 80%;
+	font-style: italic;
+	background-color: #FAFAFA;
+	border-top: 1px none #E0E0E0;
+	border-right: 1px none #E0E0E0;
+	border-bottom: 1px none #E0E0E0;
+	border-left: 1px none #E0E0E0;
+	margin: 0px;
+}
+.memItemLeft {
+	padding: 1px 0px 0px 8px;
+	margin: 4px;
+	border-top-width: 1px;
+	border-right-width: 1px;
+	border-bottom-width: 1px;
+	border-left-width: 1px;
+	border-top-color: #E0E0E0;
+	border-right-color: #E0E0E0;
+	border-bottom-color: #E0E0E0;
+	border-left-color: #E0E0E0;
+	border-top-style: solid;
+	border-right-style: none;
+	border-bottom-style: none;
+	border-left-style: none;
+	background-color: #FAFAFA;
+	font-size: 80%;
+}
+.memItemRight {
+	padding: 1px 8px 0px 8px;
+	margin: 4px;
+	border-top-width: 1px;
+	border-right-width: 1px;
+	border-bottom-width: 1px;
+	border-left-width: 1px;
+	border-top-color: #E0E0E0;
+	border-right-color: #E0E0E0;
+	border-bottom-color: #E0E0E0;
+	border-left-color: #E0E0E0;
+	border-top-style: solid;
+	border-right-style: none;
+	border-bottom-style: none;
+	border-left-style: none;
+	background-color: #FAFAFA;
+	font-size: 80%;
+}
+.memTemplItemLeft {
+	padding: 1px 0px 0px 8px;
+	margin: 4px;
+	border-top-width: 1px;
+	border-right-width: 1px;
+	border-bottom-width: 1px;
+	border-left-width: 1px;
+	border-top-color: #E0E0E0;
+	border-right-color: #E0E0E0;
+	border-bottom-color: #E0E0E0;
+	border-left-color: #E0E0E0;
+	border-top-style: none;
+	border-right-style: none;
+	border-bottom-style: none;
+	border-left-style: none;
+	background-color: #FAFAFA;
+	font-size: 80%;
+}
+.memTemplItemRight {
+	padding: 1px 8px 0px 8px;
+	margin: 4px;
+	border-top-width: 1px;
+	border-right-width: 1px;
+	border-bottom-width: 1px;
+	border-left-width: 1px;
+	border-top-color: #E0E0E0;
+	border-right-color: #E0E0E0;
+	border-bottom-color: #E0E0E0;
+	border-left-color: #E0E0E0;
+	border-top-style: none;
+	border-right-style: none;
+	border-bottom-style: none;
+	border-left-style: none;
+	background-color: #FAFAFA;
+	font-size: 80%;
+}
+.memTemplParams {
+	padding: 1px 0px 0px 8px;
+	margin: 4px;
+	border-top-width: 1px;
+	border-right-width: 1px;
+	border-bottom-width: 1px;
+	border-left-width: 1px;
+	border-top-color: #E0E0E0;
+	border-right-color: #E0E0E0;
+	border-bottom-color: #E0E0E0;
+	border-left-color: #E0E0E0;
+	border-top-style: solid;
+	border-right-style: none;
+	border-bottom-style: none;
+	border-left-style: none;
+       color: #606060;
+	background-color: #FAFAFA;
+	font-size: 80%;
+}
+.search     { color: #003399;
+              font-weight: bold;
+}
+FORM.search {
+              margin-bottom: 0px;
+              margin-top: 0px;
+}
+INPUT.search { font-size: 75%;
+               color: #000080;
+               font-weight: normal;
+               background-color: #e8eef2;
+}
+TD.tiny      { font-size: 75%;
+}
+a {
+	color: #1A41A8;
+}
+a:visited {
+	color: #2A3798;
+}
+.dirtab { padding: 4px;
+          border-collapse: collapse;
+          border: 1px solid #84b0c7;
+}
+TH.dirtab { background: #e8eef2;
+            font-weight: bold;
+}
+HR { height: 1px;
+     border: none;
+     border-top: 1px solid black;
+}
+
+/* Style for detailed member documentation */
+.memtemplate {
+  font-size: 80%;
+  color: #606060;
+  font-weight: normal;
+  margin-left: 3px;
+} 
+.memnav { 
+  background-color: #e8eef2;
+  border: 1px solid #84b0c7;
+  text-align: center;
+  margin: 2px;
+  margin-right: 15px;
+  padding: 2px;
+}
+.memitem {
+  padding: 4px;
+  background-color: #eef3f5;
+  border-width: 1px;
+  border-style: solid;
+  border-color: #dedeee;
+  -moz-border-radius: 8px 8px 8px 8px;
+}
+.memname {
+  white-space: nowrap;
+  font-weight: bold;
+}
+.memdoc{
+  padding-left: 10px;
+}
+.memproto {
+  background-color: #d5e1e8;
+  width: 100%;
+  border-width: 1px;
+  border-style: solid;
+  border-color: #84b0c7;
+  font-weight: bold;
+  -moz-border-radius: 8px 8px 8px 8px;
+}
+.paramkey {
+  text-align: right;
+}
+.paramtype {
+  white-space: nowrap;
+}
+.paramname {
+  color: #602020;
+  font-style: italic;
+  white-space: nowrap;
+}
+/* End Styling for detailed member documentation */
+
+/* for the tree view */
+.ftvtree {
+	font-family: sans-serif;
+	margin:0.5em;
+}
+.directory { font-size: 9pt; font-weight: bold; }
+.directory h3 { margin: 0px; margin-top: 1em; font-size: 11pt; }
+.directory > h3 { margin-top: 0; }
+.directory p { margin: 0px; white-space: nowrap; }
+.directory div { display: none; margin: 0px; }
+.directory img { vertical-align: -30%; }
diff --git a/api/docs/images/drlogo-16x16.ico b/api/docs/images/drlogo-16x16.ico
new file mode 100644
index 000000000..f2ffe3407
Binary files /dev/null and b/api/docs/images/drlogo-16x16.ico differ
diff --git a/api/docs/images/drlogo-16x16.png b/api/docs/images/drlogo-16x16.png
new file mode 100644
index 000000000..5657537bb
Binary files /dev/null and b/api/docs/images/drlogo-16x16.png differ
diff --git a/api/docs/images/drlogo-16x16.xcf b/api/docs/images/drlogo-16x16.xcf
new file mode 100644
index 000000000..2c52af769
Binary files /dev/null and b/api/docs/images/drlogo-16x16.xcf differ
diff --git a/api/docs/images/drlogo.png b/api/docs/images/drlogo.png
new file mode 100644
index 000000000..bc07d86f9
Binary files /dev/null and b/api/docs/images/drlogo.png differ
diff --git a/api/docs/images/drlogo.xcf b/api/docs/images/drlogo.xcf
new file mode 100644
index 000000000..66850a636
Binary files /dev/null and b/api/docs/images/drlogo.xcf differ
diff --git a/api/docs/images/favicon-rio.ico b/api/docs/images/favicon-rio.ico
new file mode 100644
index 000000000..2b4cbeaf7
Binary files /dev/null and b/api/docs/images/favicon-rio.ico differ
diff --git a/api/docs/images/favicon-rio.png b/api/docs/images/favicon-rio.png
new file mode 100644
index 000000000..7e2c8b42f
Binary files /dev/null and b/api/docs/images/favicon-rio.png differ
diff --git a/api/docs/images/favicon.ico b/api/docs/images/favicon.ico
new file mode 100644
index 000000000..f2ffe3407
Binary files /dev/null and b/api/docs/images/favicon.ico differ
diff --git a/api/docs/images/favicon.png b/api/docs/images/favicon.png
new file mode 100644
index 000000000..5657537bb
Binary files /dev/null and b/api/docs/images/favicon.png differ
diff --git a/api/docs/images/vmware-logo-crop.png b/api/docs/images/vmware-logo-crop.png
new file mode 100644
index 000000000..452ce0af6
Binary files /dev/null and b/api/docs/images/vmware-logo-crop.png differ
diff --git a/api/docs/images/vmware-logo.eps b/api/docs/images/vmware-logo.eps
new file mode 100644
index 000000000..65a4c59f9
--- /dev/null
+++ b/api/docs/images/vmware-logo.eps
@@ -0,0 +1,1401 @@
+%!PS-Adobe-3.0 EPSF-3.0
+%%Creator: (ImageMagick)
+%%Title: (vmware-logo.eps)
+%%CreationDate: (Wed Oct 10 12:21:19 2007)
+%%BoundingBox: 0 0 187 72
+%%HiResBoundingBox: 0 0 187 72
+%%DocumentData: Clean7Bit
+%%LanguageLevel: 1
+%%Pages: 1
+%%EndComments
+
+%%BeginDefaults
+%%EndDefaults
+
+%%BeginProlog
+%
+% Display a color image.  The image is displayed in color on
+% Postscript viewers or printers that support color, otherwise
+% it is displayed as grayscale.
+%
+/DirectClassPacket
+{
+  %
+  % Get a DirectClass packet.
+  %
+  % Parameters:
+  %   red.
+  %   green.
+  %   blue.
+  %   length: number of pixels minus one of this color (optional).
+  %
+  currentfile color_packet readhexstring pop pop
+  compression 0 eq
+  {
+    /number_pixels 3 def
+  }
+  {
+    currentfile byte readhexstring pop 0 get
+    /number_pixels exch 1 add 3 mul def
+  } ifelse
+  0 3 number_pixels 1 sub
+  {
+    pixels exch color_packet putinterval
+  } for
+  pixels 0 number_pixels getinterval
+} bind def
+
+/DirectClassImage
+{
+  %
+  % Display a DirectClass image.
+  %
+  systemdict /colorimage known
+  {
+    columns rows 8
+    [
+      columns 0 0
+      rows neg 0 rows
+    ]
+    { DirectClassPacket } false 3 colorimage
+  }
+  {
+    %
+    % No colorimage operator;  convert to grayscale.
+    %
+    columns rows 8
+    [
+      columns 0 0
+      rows neg 0 rows
+    ]
+    { GrayDirectClassPacket } image
+  } ifelse
+} bind def
+
+/GrayDirectClassPacket
+{
+  %
+  % Get a DirectClass packet;  convert to grayscale.
+  %
+  % Parameters:
+  %   red
+  %   green
+  %   blue
+  %   length: number of pixels minus one of this color (optional).
+  %
+  currentfile color_packet readhexstring pop pop
+  color_packet 0 get 0.299 mul
+  color_packet 1 get 0.587 mul add
+  color_packet 2 get 0.114 mul add
+  cvi
+  /gray_packet exch def
+  compression 0 eq
+  {
+    /number_pixels 1 def
+  }
+  {
+    currentfile byte readhexstring pop 0 get
+    /number_pixels exch 1 add def
+  } ifelse
+  0 1 number_pixels 1 sub
+  {
+    pixels exch gray_packet put
+  } for
+  pixels 0 number_pixels getinterval
+} bind def
+
+/GrayPseudoClassPacket
+{
+  %
+  % Get a PseudoClass packet;  convert to grayscale.
+  %
+  % Parameters:
+  %   index: index into the colormap.
+  %   length: number of pixels minus one of this color (optional).
+  %
+  currentfile byte readhexstring pop 0 get
+  /offset exch 3 mul def
+  /color_packet colormap offset 3 getinterval def
+  color_packet 0 get 0.299 mul
+  color_packet 1 get 0.587 mul add
+  color_packet 2 get 0.114 mul add
+  cvi
+  /gray_packet exch def
+  compression 0 eq
+  {
+    /number_pixels 1 def
+  }
+  {
+    currentfile byte readhexstring pop 0 get
+    /number_pixels exch 1 add def
+  } ifelse
+  0 1 number_pixels 1 sub
+  {
+    pixels exch gray_packet put
+  } for
+  pixels 0 number_pixels getinterval
+} bind def
+
+/PseudoClassPacket
+{
+  %
+  % Get a PseudoClass packet.
+  %
+  % Parameters:
+  %   index: index into the colormap.
+  %   length: number of pixels minus one of this color (optional).
+  %
+  currentfile byte readhexstring pop 0 get
+  /offset exch 3 mul def
+  /color_packet colormap offset 3 getinterval def
+  compression 0 eq
+  {
+    /number_pixels 3 def
+  }
+  {
+    currentfile byte readhexstring pop 0 get
+    /number_pixels exch 1 add 3 mul def
+  } ifelse
+  0 3 number_pixels 1 sub
+  {
+    pixels exch color_packet putinterval
+  } for
+  pixels 0 number_pixels getinterval
+} bind def
+
+/PseudoClassImage
+{
+  %
+  % Display a PseudoClass image.
+  %
+  % Parameters:
+  %   class: 0-PseudoClass or 1-Grayscale.
+  %
+  currentfile buffer readline pop
+  token pop /class exch def pop
+  class 0 gt
+  {
+    currentfile buffer readline pop
+    token pop /depth exch def pop
+    /grays columns 8 add depth sub depth mul 8 idiv string def
+    columns rows depth
+    [
+      columns 0 0
+      rows neg 0 rows
+    ]
+    { currentfile grays readhexstring pop } image
+  }
+  {
+    %
+    % Parameters:
+    %   colors: number of colors in the colormap.
+    %   colormap: red, green, blue color packets.
+    %
+    currentfile buffer readline pop
+    token pop /colors exch def pop
+    /colors colors 3 mul def
+    /colormap colors string def
+    currentfile colormap readhexstring pop pop
+    systemdict /colorimage known
+    {
+      columns rows 8
+      [
+        columns 0 0
+        rows neg 0 rows
+      ]
+      { PseudoClassPacket } false 3 colorimage
+    }
+    {
+      %
+      % No colorimage operator;  convert to grayscale.
+      %
+      columns rows 8
+      [
+        columns 0 0
+        rows neg 0 rows
+      ]
+      { GrayPseudoClassPacket } image
+    } ifelse
+  } ifelse
+} bind def
+
+/DisplayImage
+{
+  %
+  % Display a DirectClass or PseudoClass image.
+  %
+  % Parameters:
+  %   x & y translation.
+  %   x & y scale.
+  %   label pointsize.
+  %   image label.
+  %   image columns & rows.
+  %   class: 0-DirectClass or 1-PseudoClass.
+  %   compression: 0-none or 1-RunlengthEncoded.
+  %   hex color packets.
+  %
+  gsave
+  /buffer 512 string def
+  /byte 1 string def
+  /color_packet 3 string def
+  /pixels 768 string def
+
+  currentfile buffer readline pop
+  token pop /x exch def
+  token pop /y exch def pop
+  x y translate
+  currentfile buffer readline pop
+  token pop /x exch def
+  token pop /y exch def pop
+  currentfile buffer readline pop
+  token pop /pointsize exch def pop
+  /Times-Roman findfont pointsize scalefont setfont
+  x y scale
+  currentfile buffer readline pop
+  token pop /columns exch def
+  token pop /rows exch def pop
+  currentfile buffer readline pop
+  token pop /class exch def pop
+  currentfile buffer readline pop
+  token pop /compression exch def pop
+  class 0 gt { PseudoClassImage } { DirectClassImage } ifelse
+  grestore
+} bind def
+%%EndProlog
+%%Page:  1 1
+%%PageBoundingBox: 0 0 187 72
+userdict begin
+DisplayImage
+0 0
+187 72
+12.000000
+187 72
+0
+0
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFDFFFBF5FFFFFFF8FFFF
+FDFEFFFFFDF4FFFBF8F1FFFFFFFFFBFAFAEEFFFFEFFEFFF4F6FCFCF6FDFFF7FFFFF5FFFD
+FFFDFFFFFDFFFFFDFFFFFDFFFFFDFFFFFDFFFFFDFFFFFDFFFFFAF8FCF7FBF8FFFFF8FFFF
+FFFFFBFFFDF1FFF9EEFFFBF4FFFFFBFFFEF9FFFBF5FFFEF6FFFAF5FFFDFBFFFBFCFFFEFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFDFFFDF9FFFFFFF8FFFFFDFEFFFFFDF4FFFFFDE8FBFFBFD8FF
+A5BDE98FA9CC93AAD399B0E299B2EA93B1E38CADD88BB7D28BB7D28BB7D28BB7D28BB7D2
+8BB7D28BB7D28BB7D293B1D38CB6DC88BBE683B5DA84B0CD8FB3CDA4C8E8C9F3FFF5FFFF
+FBFFFFFCFEFDFFFFFDFFFFFDFFFFFFFEFFFFFCFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFA
+FFFEFCF9FFFFFDFEFFFFFDF6FEFFFFC0D7E96593C7739EC86992B07093B37293BC7192BF
+7699C1759CBB6F95C46F95C46F95C46F95C46F95C46F95C46F95C46F95C47496B97097BE
+6B98BF6D9ABB79A1BB739AB96293BC67A3D7E7F6FDF7FFFFF7FFFFFBFFFFFBFCFFFCFFFF
+FBFFFFF9FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFBFCFEFFFEFBFEFDFBFBFFFFFEFEFEFFFEF8F7FCFF
+8EACC4779AAECDEBF3EBFFFEF4FFFDF7FFFFF3FEFFF2FFFEECFDF3F9FFFFF9FFFFF9FFFF
+F9FFFFF9FFFFF9FFFFF9FFFFF9FFFFFBFFFFF8FFFFF2FCFFF5FDFFFCFFFFECF5FEA8C2DB
+6D94BDC5D2DBF6FFFFF7FCFFFEFFFFFFFEFFFEFEFEFEFEFFFCFDFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+F9FDFFFFFEFBFEFDFBFBFFFFFEFEFEFFFEFAF2FCFF7295B198A9C3F2FEFFFEFFFBFFFDF6
+FFFBFAFFFAFDFFFCFBFFFEF8FFFDF8FFFDF8FFFDF8FFFDF8FFFDF8FFFDF8FFFDF8FFFDF8
+FFF9F9FFFFFDFFFFFBFFFFF6FFFDF4FFFFFFCDDFF57B9DC3C0C9D2FBFFFFFFFFFFFFFEFB
+FFFEF8FFFDF5FFFEF8FFFEF8FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF9FFFFFFFFFDFEFDFBFCFFFFFEFEFE
+FFFEFBF4FFFF6E99BB9CAFD0F5FFFFFEFFFFFFFBF9FFF6FBFFF8FFFFFCFFFEFFFFFFFCFF
+FFFCFFFFFCFFFFFCFFFFFCFFFFFCFFFFFCFFFFFCFFFFFDF7FFFEFBFFFFFAFFFEF4FFFAF0
+FFFEFFD7E4F77894B9BECAD8F6FFFFFCFFFFFFFFFDFFFCF6FFFEF6FFFEF6FFFEF6FFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFF9FFFFFFFFFDFFFEFBFBFFFFFFFFFFFFFEFBF1FFFF709EC293AFC7EEFFFF
+F9FFFBFFFFF4FFFCF8FFFEFFF5FFFFF2FFFFF9FFFDF9FFFDF9FFFDF9FFFDF9FFFDF9FFFD
+F9FFFDF9FFFDFEFFFFF8FFFFF4FFFFF9FFFFFEFFFFFAFEFFCDDDF67A98BEBCD4ECE7FCFF
+F3FFFFF5FEFFFAFEFFFEFFFFFEFFFDF9FEFAFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF7FFFFFEFDFBFFFEFB
+FEFFFFFFFFFFFDFCFAEAFDFF6C9CC295B0CDEBFFFFF8FFFFFFFFF7FEFCFDFAFEFFC1D1DE
+95ADB78EB2D48EB2D48EB2D48EB2D48EB2D48EB2D48EB2D48EB2D495B4C98FB4CE8AB3CF
+93B8D295B5CC93B2D187B0DE6598D17A9EC094B3D298B2C99EB1C2B6C4CFE3F0F8F0FDFF
+F5FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFCFFFBFBFFFBFBFFFFFFFDFFFFFDFBFEFEF2F1FFFF7B93C3
+8EBBDCF3F9FFFFFDFFF2FAFDF5FFFF9EC1E96598CD7292A96899C16899C16899C16899C1
+6899C16899C16899C16899C16A9BBC669ABF6599C06995B8749BBC6791B7639BCE579DDB
+6796CE829AB67297B27799B26895BF68A0D1F0FFFFFBFFF8FFFFFDF8F8F6FFFFFDFFFFFF
+FFFFFFFFFFFFFCFCFEFFFFFFFEFFFDFEFFFFFEFEFFFEFEFFFDFFFAFBFEF5FDFFFCFCFFFF
+FCFDFFFCFEFDFFFFFFFDFBFCFFFFFFFEFFFFFBFFFFF7FEFFFFFEFFFEFEFEFDFDFBFFFFFB
+FEFFFBFBFDFCFBFFFFFCFFFFFFFCFFFEF8FAFDFBFEFFFFFFFFFFFFFFFFFDFFFFFAFDFAF3
+FFFEFFFEFFFFFCFDFFFBFCFFFEFFFFFEFFFFFCFDFFFEFFFFFFFEFDFFFEFFFFFFFFFDFEFF
+FDFEFFFEFFFFFFFDFEFDFCFAF8FFFFF9FFFFF9FEFFFCFFFFFCFFFFFCFDFFFFFEFFFFFBF8
+FFFEF8F7F6F4FEFFFFFCFFFFF8F9FBFFFFFFFFFFFDFEFAF7FCFFFFFCFDFFFFFEFFFFFDFF
+FFFDFFFFFEFFFAFBFDFCFFFDF5FFFFF8FFFFF9FFFFFCFDFFFEFCFDFFFBFFFCF9FFFEFEFF
+FEFDFFFFFAFFFFFAFCFFFDFCFDFDFFFAFFFFF9FFFFF8FFFFFCFFFFFCFFFFFEFFFFFEFFFF
+FEFFFFFBFCFEFCFFFFFCFFFFFBFDFCFCFCFCFFFFFDFFFEFCFAFAFAFCFFFFF9FFFFF8FFFF
+FEFDF9FDFCF8FFFFFDF6F8F7FAFEFDFAFEFFFCFFFFFDFFFEFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFBFCFF
+F6FBFEF5FCFFFDFCFFFFFDFEFBFFF5F0FFFF7696BF8EAEC5FFFDFFFFFBFBFFFEF9FEFFFD
+8099B880A5CFE5F7FFFEFFFFFEFFFFFEFFFFFEFFFFFEFFFFFEFFFFFEFFFFFEFFFFFCFFFF
+F8FFFFF9FFFFFEFFFFFFFEFDFEFFFFD3E4F87D9BBDAECFF2F1FCFFEDFFFFE4FAFFA6C8EB
+6593C4D4DCE7FCFDF8FFFFFDFAFAF8FFFFFDFDFDFDFDFDFDFFFFFFFFFFFFFFFFFFFFFFFA
+FDFBFEFCF9FFFFFEFFFFFFFAFEFFF8FEFFFDFBFCFFFFFFFFFEFEFCFFFFFDFFFCFBFFFEFF
+FFFEFFFFFEFFFDFEFFFFFEFFFFFCFBFFFDF8FFFFFAFFFEF9FCFCFAFEFEFFFFFEFFFEFCFF
+FDFCFFFEFFFFFCFFFFFBFFFFFAFEFFFCFFFDFEFFFDFDFBFCFEFCFDFFFEFFFFFEFFFFFFFF
+FFFFFFFDFEFFFEFFFFFFFBF8FFFEFDFFFEFFFFFFFFFEFFFFFEFEFFFFFFFFFFFEFFFAFDFF
+FCFFFFFEFFFFFEFEFFFFFEFFFFFEFFFEFCFDFFFEFFFFFFFFFDFDFDFDFDFDFFFFFFFFFFFF
+FFFFFDFFFEFBFFFEFBFCFFFFFAFCFBFFFFFFFFFEFFFFFDFEFFFFFDFEFDF9FEFFFAFCFFFF
+FBFDFCFFFFFFFFFFFDFFFEFFFFFEFFFFFEFFFFFEFFFFFDFFFFFEFFFFFEFFFFFEFDFCFBF7
+FBFBF9FEFFFDFCFFFDFCFCFAFFFFFFFBFBFBFCFCFCFFFFFFFFFFFFFEFEFEF9F9F7FFFFFD
+FFFFFDFFFEFBFFFEFBFFFFFBFFFFFFFAFBFDF5F9FAFFFFFBFFFEFAFFFFFBFFFFFDFFFFFD
+FBFDFCFEFFFFFBFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF9FBFFFDFFFFFCFFFEF8FFFFFEFBFDFFF0
+EAFEFD7897B68EB3CDFEFEFFFBF4FBFCFFFFF8FFFF728FB18AB3DFF0FFFFFFFFFDFFFFFD
+FFFFFDFFFFFDFFFFFDFFFFFDFFFFFDFFFFFDFFFFFAFFFFFDFFFFFFFFFEFAFFFCF7FFFEFF
+D8E9FB698AA9B5C8D6FFFCF0F8FFFBF9FFFFCBE3FF7195C5C5C7D6FAF8FBFFFFFDFEFEFC
+FFFFFDFBFBFBFFFFFFFFFFFFFFFFFFFEFEFFFFFBF2FBF5F5FFFAFFFFFAFEFFFFF7FFFFF6
+FFFFFBFBFAFFFFFFFBFFFBF8FEFAF7FDF9F6FFFEFBFFFAF7FFFEFBFFFDFBFFFDFDFFFAF7
+FFFAF4FFFFF8FFFFF8FFFFFBFEFCFDFAF8FBFBFCFEFCFFFFF8FCFDF5F9FAFBFFFFFBFFFF
+F1FAF9F6FFFEFFFEF8FFFFFAFFFFFAFFFFFBFFFFFBFCFCFAFFFFFDFFFFFFFFFCF9FFFEFD
+FFFEFDFFFFFDFFFEFFFCFCFEFAFAFCFEFFFFFFFEFDFEFDFBFFFEFFFFFCF9FFF9F6FFFEFB
+FDFBFCFEFFFFF8FFFFFCFFFFFDFCFAFFFBF7FFFEFBFDFCFAFFFBF8FFFDF9FFFFFFF8F8F6
+FFFFFBFFFFF8FFFFF8FDFAF3FFFDF8FFFEFAFFFFFAFFFEF8FFFEF8FFFEFAFFFEFDFFFEFF
+FFFEFFFFFAFAFFFEFBFCFCFCFAFCFBFFFFFDFFFDF6FFFDF4FEFBF2FCFCF4FFFAF6FFFEFB
+FDFCFAFFFFFFFEFEFEFEFDFBFFFEFBFFFEFAFFFEFBFFFEFBFBF7F4FDF9F6FFFEFBFFFEFB
+FFFEFAFFFEFAFFFFFDFEFDFBFFFEFAFFFBF8FFFFFBFAFAF8FEFFFFFCFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFF9EFFFFBF1FFFBFCFFFBFFFFFDFAFFFFEFF4FFFF7993B491B5CDFFFDFFFFFAFF
+FBFFFFF6FFFF7996B892B7E1EDFBFFFAFEFFFAFEFFFAFEFFFAFEFFFAFEFFFAFEFFFAFEFF
+FAFEFFF7FFFFF5FFFFF6FFFFFCFFFFFEFFFFF4FEFFD1F0FF70A1CCC0CEDBFFFDF1FDFFF7
+FDFFFCD5E9FF7197C6BEC5D5FCFFFFFFFFFDFFFFFDFFFFFDFFFFFFFFFFFFFFFFFFFDFDFF
+FEFEFFFFFFFAFFFDFFFFFCFFFFFEFFFCFBF6FBFBF1FFFFFBFFFEFFFFFDFAFFFFFBFFFEFF
+FFFFFFFFFEFFFFFEFFFFFDFDFFFCFAFFFDFFFFFEFFFFFFFDFFFEFAFDFEF9FFFFFFFEFDFF
+FEFFFFFCFFFFFBFDFAFFFFFBFFFFFBF9FAF5FCFFFDFBFFFFF7FFFFFFFFFAFCFDF7F7F8F3
+FFFFFBFFFFFBFFFFFBFFFFFBFCFDF8FFFFFFFFFFFFFFFEFFFFFCFBFFFEFDFFFEFFFEFFFF
+FEFFFFFFFAF4FFFEFBFFFFFFFFFEFFFFFAF7FFFEF8FEFCFDFAFFFFEFFCFFFCFFFFFFFEFA
+FFFEFAFFFFFDFEFFFFFFFFFFFDF8F4FAF8F9FDFDFBFEFFFAFCFFF4F7FAEFFFFFF8FFFFFB
+FAF6F5FDF8F4FFFDF6FFFCF6FFFAF7FDFEFFFBFEFFFFFEFFFFFCFAFFFFF8FEFFFFF6FFFF
+FBFFFFFFFFFAFFFEF4FFFEF3FFFFFAFFFEFBFFFEFDF7F5F6FFFFFFFFFFFFFFFEFFFFFEFD
+FFFDFAFFFDFEFFFFFFFFFFFFFFFFFFFBF9FAFDF8F5FFFEF8FFFCF6FFFFFFFFFFFDFFFFFD
+FDFCFAFFFFFFFEFFFFFCFFFFFCFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFCEEFFFFF4F8FFFFF8FFFF
+F3FBFFF4FFFFE8FFFF7194CA95B4C9FEF9FFFFFCFFFBFFFFEFF7F98299B894B2D6F9FEFF
+FFFEFDFFFEFDFFFEFDFFFEFDFFFEFDFFFEFDFFFEFDFFFEFDF5FAFFFBFFFFFEFDFFFFF7F4
+FFFDF4FFFFFFCFE4F76D95B8B4C4E5FFFAFCFAFFFFFAFFFFCEE6FF6B97C4B8CAD6F4FFFF
+FFFFFDFFFFFDFFFFFDFFFFFFFFFFFFFDFDFFFFFFFFFFFFFFF8FFFFF1F7FFF5F8FFF8FAFF
+FEFFFFFDFEF6FFFFFBFFFEFFF8F7F5FEFFFFF5FCFFF2FCFFF6FFFFF8FFFFFBFFFFFEFFFF
+F9FFFFF5FCFFF7FFFFF4FDFFF5FFFFF7FFFFF5FFFFF7FFFFF8FEFEFCFFFDFBFCF6FCF9F2
+FFFFFAFEFFFFF5FDFFF7FFFFF5FFFFF7FFFFF8FFFFF9FFFFFBFFFFF3F7F6FCFEFBFEFFFD
+EDF9FFF7FFFFFCFFFFFFFEFFFFFBFAFDFCFAFEFEFFFCFDFFFFF3EBFFFFFFEDF7FFF2FBFF
+FFFFFFFFFEFAFEFCFDFBFFFFF4FFFFFBFFFFFFFEFAFFFFFDF1FAFFEEFBFFF9FFFFFFFFFD
+FFFEFFFFFFFFFBFFFFF7FFFFF5FFFFF5FFFFF8FFFFF9FFFFFBFFFFFFFEFFFFFDFCFBFEFF
+EFFDFFF2FFFFFCFFFFFFF6F1FFFCF3F9FFFFEEFFFFEBFDFFF2FAFCFFFFFBFEFFFFF7FFFF
+F3FAFFFCFFFFFEFFFFFEFEFFFAFAFCFEFFFFFCFFFFF7FEFFF8FFFFF3FDFFF3FFFFF5FFFF
+F8FFFFFCFFFFFFFEFDFDF3F1FAF9F7FFFFFFFBFFFFEFF6FCF0F9FFF1F8FEFBFFFFFBFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFF5FFFFEEFFFFB4D4EB96B6DD97B6D58CAFC38CB8DD6597DE6FA9D1
+97AECD9CB0CB92B5CB97BBD36294C76EA3D992ADC08EB2D48EB2D48EB2D48EB2D48EB2D4
+8EB2D48EB2D48EB2D4A5B5C5D2DFEFFBFFFFFFFBF8FFFCF4FFFFFDD3E9F7729BB9B2CBF4
+FDFBFFF8FFFFFAFFFFCEE6FF6D9CC6BACED9EDFFFFFFFFFDFEFEFCFCFCFAFFFFFFFCFCFC
+FCFCFEFFFFFFFFFFFFBDD8EBA2BAD4A5B6D4C6D2E8F8FFFFFCFFFDFEFEFCFFFDFFFCFFFF
+F7FFFFC2D4E2A5BED499B4CF96B1CC99B1C998AEC596ABC6BED4ECEBFFFFDDF6FFAFCADD
+8CA6BD7D97B28AA5C2A9BDC8D1E2EAF2FDFFF9FFFFF9FFFFCBD7E39AADBE8EA4BB7595AC
+94B3C8B0CBDED5EBF9F4FFFFF8FFFFFBFFFFE5E9EA99B7D1B1C7DCDEEAF6FCFFFFFFFEFF
+FFFDFCFFFEFFFFFFFFFFFCF7EDF7FFA3BFD7A3C1D9DDEBF6FFFFFFFDFBFCFCFFFFF8FFFF
+FAFAFAFFFEFBF1F8FEB1CADE9EBCD4CADCEAFCFFFFFFFAFAFEFFFFF5FFFFEAFFFFC8E5F7
+A1BDD296B0C79BB4CAA7BFD9CCDCECF7FFFFF1FFFFA1BFD998B6CEE6F3FCFFFEF8FFFFF6
+EDFAFFACCBE0A2C2D9CFE2F1F7FFFFD4E4F3ADC5DD9CB5CBBCCDDDF9FFFFFDFDFFFFFFFF
+F9FFFFEDFEFFD0E9FFAAC2DA97B1CA90AEC89DBBD5B8D0E8DEEEFEF9FFFFFFFEFFFFFBF8
+FCFFFFE0EEF9B4C9DCAABFD2BCCCDBF1F8FEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF0FFFFC7E3F1
+7093B16689B3799AB97293A279A3BC6895CE5B9FD07395BB6D8DB36694B573A4C5508CC8
+619FDC7498B0669AC9669AC9669AC9669AC9669AC9669AC9669AC9669AC96C8FB583A4C5
+D6ECFFF9FFFFFFFDFEF9FFFFCCEBFF6C9EC1B1CBEEFFFFFFFBFFFFFFFFFDCFE0F46D92BC
+BCC8D4F7FFFFFFFFFDFFFFFDFEFEFCFFFFFFFAFAFAFFFFFFFFFFFFE6E6E874A2C66690BA
+698AB58CA5C4E1F3FFF8FFFFFEFFFFFFFDFFF8FFFFD8E8F589A5BD6B93B66494BC6596BF
+6B98C16F9BC26D94BD729ABE82AACD76A1C16E99BB6F9ABD6691BB6C97C26C97B77499B6
+B7D5EFD0EAFF94AEC77596B77BA2CB638FBE6499C56798C16893B687A9C5BBD1E6F3FFFF
+F8FFFFC1C4C96494BA7599B9B8CEE3F8FFFFFFFFFFFEFAF9FFFCFBFCFAFBFFFEFDB7CEE0
+6797BF6596BFA4C0D8FBFFFFFFFEFDFDFBFCFBFCFEFEFAF7FFFFFDC9DBE9739CBC6391B5
+9EBAD0F6FFFFFFFBF8FCFFFFD4E6FA92B3D4709AC26B98C26E9BC26591B46894C1789ABD
+9CB6D1A1C4E26B9BC16E9BBCC6DCEAF7F6F2FFFEFAD3E5F37098BB6591B6A1C0DDC9DEF9
+90AED06A93BF6892B87D9BB7E6F2FFFDFCFFFFFEFFD7E3F1A0BEDA6D97BD6D95B96C98BF
+6796C06695BF7199BD89A7C3BDD0E1F6FFFFFFFDF8F7FCFFC0D6EB86A9C7799DBD97B0C6
+E3ECF5FFFEFDFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFCFEFF97A4B48297B2DAF1FFF1FFFFF2FFFFEBFFFF
+7C97B593B7D1F8FBFFFCFEFFF2FFFFEAFCFF7D9BBF8EACD0FCFFFFFFFDFDFFFDFDFFFDFD
+FFFDFDFFFDFDFFFDFDFFFDFDFFFDFDAFD5FA7394B5B6C8DEFDFEFFFFFDF6FFFFFDD2E8F5
+6E97B3B3CEE3FFFEF5FDFFF4FFFCF2DDE6F77A97BFC4C6D5FCFFFFFFFFFDFFFFFDFFFFFD
+FFFFFFFCFCFCFFFFFFFBFBFDCACACC5E98C66B9FD16C97CA7494BAC7DEEEF3FEFFFEFFFF
+FFFDFFF3FFFFC4D9EA7194B26094BC64A0D2629FD55F9BCF629CCE6B9BCC6598C76EA1CC
+6498C06297C16CA0CF6398CA699ED265A1D36096C46493BD709AC2709AC46494C55F97D2
+62A2E25B9CD465A2D8679BCA6A91B87D9BB7CBDCF0F7FFFFCACED769A1D0628FB69FBDD7
+F0FCFFFFFFFFFFFBF8FFFEFBFFFEFDFCFCFE8DABC5639FD1609FD479A0C1EFF9FFFFFEFD
+FDF9F6FFFEFBFFFEF8FFFFFFAAC1D36195BD67A0CD9DC0DCF4FEFFFFFEFADADEE78CA1BE
+688DBA6798D06198D0639BCC699FCB609BD36897C36E94B877A4CB609AC86196BEB5D0E3
+FEFFFDFFFCF9CFE4F76D9DC56194C381A7CE89A7CB6187B4699DD6679DCC6F95B9EEFFFF
+FEFFFFE6E7EC93A4B66F95B9669CCB6699C66EA4D366A0D2649ED0699CC9658DB1859FB8
+C5D6E8FFFCF6F4F9FFBEDAF28EBAE188B5DCA0BEDAE3ECF5FDF8F5FFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FEFEFE92AFD96A9CC1FBFFFAFFFCFFF8FFFFFEFFFA6C96C68EB2D4F9FFFDFFFCFFFFFDF8
+F9FFFF6F95C48BB7D2FFFDFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+D4E5F97197BBB1CCE1FCFFFFFAFEFDFFFFFDDAE4F06999C7B3CCE2FEFEFFFCFFFFFFFCF7
+D3E4F4659BC9B6CBDEFFFFFFFFFFFFFFFFFFFFFFFFFEFEFEFFFFFFFFFFFFF6F6F6E4E4E4
+6E98C06F9FCD6195C46F9BC29DB5CDF5FEFFFCFFFFFBFEFFFEFFFD85A9DB6D99BE61A0C9
+6395D07899BA5E9CC1629DD96395C8669BCF679FD06394BC7198B7759CBB6493BD669FD4
+7595BE6D99CA6498D16398CE6891BD7495B8819EBE7596B76A98B96395CA69A1DE619DCF
+6397BF83A1C7F4FFFFF2F4E97097B46096C276A2C9EBF1FDFFFBF9FDFEFFF7FEFFFFFEF8
+E7F3FF7895B76594C26BA0D26B98BFCDE8FDF4FCFEFFFFF6FFFDFFFBFCFFF2FFFF779EBB
+699EC66894B7D3E3F3FFFBFAFCFBFF7B9DC0679FD2608AB0A0B1C3DCE5EEF5FAFFFFFEFF
+E8F2FBAEC3D4678EAF609AC861A0D56B9AC4B3C4D6FFFDFDFFFCFDD7E5EE6C95B1639BCA
+699CD56389BA7D9CB99DBBC6BFD1DDFAFAFAFFF5F1FEFEFF8BA9CB6196C8629ACB90C0E8
+EEF5FDFBFEFFFBF9FFEFEDFFB3BCDB728FB96496C95F9DD6DAEAFAF5FFFFE1EBF48897B8
+99B0D991ACC1E9FAFFFBFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFEFE92AFD96A9CC1FBFFFAFFFCFF
+F8FFFFFEFFFA6C96C68EB2D4F9FFFDFFFCFFFFFDF8F9FFFF6F95C48BB7D2FFFDFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD4E5F97197BBB1CCE1FCFFFFFAFEFD
+FFFFFDDAE4F06999C7B3CCE2FEFEFFFCFFFFFFFCF7D3E4F4659BC9B6CBDEFFFFFFFFFFFF
+FFFFFFFFFFFFFCFCFCFEFEFEFFFFFFFDFDFDF6F6F6769ABE6B9AC66196C86D9EC97D9BB7
+EBF7FFFEFFFFFFFFFFE9F3F57098CB719EC55997C06B9AD083A0BE65A0C26097CF769FCD
+618DBA618DB499BCD8CCE6F7D7ECFDB2D0EC7298BD6F92BA6E9AC9699ED46395C6749ABF
+A3BDD8C8DEF6D0E8FF99BFD66E99C65D90C9629AC96798C07696BCEAF7FFFEFFF88AB1CE
+6499C36E9BC4C9D5E5FFFCFDFFFEFFFCFFFFFFFFF8BFD4F37898BE6995C26CA0CF6A97BE
+A4C3D8F8FFFFFFFFF6FEFDFBFBFFFFD4E7F8769CC06798C37DA5C8E5F5FFFFFDFAC4CDDC
+749BC2629ACB8FB3D5E1EDFBFBFFFFF9FCFFFFFEFFFBFFFFE3F1FCA0C0D9679CC65C98CC
+6D9CC6B5C7DBFFFDFEFFFAFBD9E7F2739CBA679DCB6694C88BACD7C7DDF4EEFFFFFBFFFF
+FFFEFCFFFEFFC5D4E96D93BA6895BFA2C0DCECF8FFF5FCF4FEFFF8FFFFF6FFFFFBF0FBFF
+ACC9D96B9AB864A0C4A5B5CCF8FFFFFCFFFFF0F3FFE8F3FFF2FFFFFBFFFFFBF9FEFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFEFEFE92AFD96A9CC1FBFFFAFFFCFFF8FFFFFEFFFA6C96C68EB2D4F9FFFD
+FFFCFFFFFDF8F9FFFF6F95C48BB7D2FFFDFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFD4E5F97197BBB1CCE1FCFFFFFAFEFDFFFFFDDAE4F06999C7B3CCE2FEFEFF
+FCFFFFFFFCF7D3E4F4659BC9B6CBDEFFFFFFFFFFFFFFFFFFFFFFFFFEFEFEFDFDFDFEFEFE
+FFFFFFFFFFFFA7C5E16B96C0629CCE6199CA6C93B4CCDCEBFDFEFFFFFAF6C3D6E56092C5
+6D9DC55D98C26F98C4A9BFD66C9EC16395C67396BE6D93B887A9C5E1F4FFFBFFFFFEFEFE
+F5FEFFADBED26F95BA6899C4679DCC6394BD91B0CDDCEDFDF7FFFFF7FFFFE3FBFF97B5D9
+6993C56A9FCB6B9CC46C8CB3D9E8FBFCFFFAB2D7F16599C16695BFA0B4CCFFFEFFFFFEFF
+FEFFFFFFFFFB87ADD47095BF759FC97DACD66D9AC17FA1BCF5FFFFFFFFFAFDFDF5F8FFFF
+A4BED7759CC56691BEA2C3E2F5FFFFFFFFF689A4C16798C36094C3C0DAF3FCFFFFFFFEFF
+FEFFFFFFFEFCFFFFFBFCFFFFD6E9FA749FC25A92C16B9CC5B5CADFFFFEFFFFFDFED5E7F5
+6E97B76699C66891BDB9D0F0F1FDFFF9FFFFFFFFF6FBF7F4F7FFFF92B0D26594C27399BE
+DBE2ECFFFBECFFFEFFFFFDFAFFFEF8FDF8F2FEFFFFDDEEFE83A2C16F99C37D94B4E0EDF5
+FFFFF8FFFCFDF7F2F8FFFFFBFFFCF7FFF6FCFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFEFE92AFD96A9CC1
+FBFFFAFFFCFFF8FFFFFEFFFA6C96C68EB2D4F9FFFDFFFCFFFFFDF8F9FFFF6F95C48BB7D2
+FFFDFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD4E5F97197BBB1CCE1
+FCFFFFFAFEFDFFFFFDDAE4F06999C7B3CCE2FEFEFFFCFFFFFFFCF7D3E4F4659BC9B6CBDE
+FFFFFFFFFFFFFEFEFEFFFFFFFFFFFFFFFFFFFDFDFDFEFEFEFFFFFFDEEFFF6F93B7639DCF
+5794CA73A0C99DB3C8FCFFFFFFFEFB99B7D16299CF6495BE6CA1CB7798B9DFECFC6B97BA
+6E9BC56D96C27198BF97B6D2F4FFFFFFFFFAFDF4EBFFFEFFD2D9E3729DC06498C0659BC9
+6997BBABC6D9F8FFFFFFFFFDF7F8F3F8FFFFB2C6E17499C56D9EC76B9CC47294BAD8E9FD
+F7FEF7D7F6FF6F9BBE6495C083A2C1F2FAFFFEFAFBFFFFFFF3F8FB679BCB6690B89ABCDF
+A5CBEF6F9BC07299B8DCEDFDFFFFFFFEFEF4F4FFFF83A4C5729ACB6E90BDCDE3FBF8FFFF
+ECEFE66E9AC15E96C76F9EC8E6F7FFFFFEFBFCF7F3FFFFFFFEFDF9FFFBF3FFFEFAF2FBFF
+8DAFCA6499C56899C4B3C9E0FFFEFFFFFDFED3E6F56A92B66E9DC97FA1C7E7F6FFFEFFFF
+FCF7F1FDF8F2FDFEFFE3F4FF7096BD6799CA7EA5CCF5FFFFFFFCF3FFFDFFF8F5FCFFFDFD
+FEF9F6FEFFFFF5FFFFA7BEDE7693BF7094C4C0D6E4FFFFFAFFF9F2FFFEF8FFFCF3FFFEFB
+FFFBFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFEFEFE92AFD96A9CC1FBFFFAFFFCFFF8FFFFFEFFFA6C96C6
+8EB2D4F9FFFDFFFCFFFFFDF8F9FFFF6F95C48BB7D2FFFDFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFD4E5F97197BBB1CCE1FCFFFFFAFEFDFFFFFDDAE4F06999C7
+B3CCE2FEFEFFFCFFFFFFFCF7D3E4F4659BC9B6CBDEFFFFFFFFFFFFFDFDFDFDFDFDFFFFFF
+FFFFFFFEFEFEFCFCFCFEFEFEF9FFFF90ABC8689ECD5C9BD16C9FCC7795AFF3FFFFF9FDFF
+7A9EC263A1D26695BF6F9CC694AEC5F8FFFF6C93BA709DC4679BCD6292C08BB0CDEEFFFF
+FFFFFAFFFBF2FFFFFFD4DEE8719DC26498C0699FCB6F9CBDB3CADAFCFFFFFFFEF5FFFBF1
+FBFCF7B8C4DC7596BF6798C16599C17497BFD9EAFEF7FEF7F0FFFF89ABC66998C26D97BF
+D1E2F6FDFBFEFEFFFFCADCE6609DD36991B5CEE3F8D3EBFF6F97BA6F9BC0A9C1D9FCFFFF
+FFFFF6DCF2FF719BC36C96C8819DC4F0FBFFFBFFFFDBE2DB6599C8609ACC88B2D8F8FFFF
+FFF8F2FEF9F5FFFFFFFFFCF9FFFEF8FFF9F2FFFFFFA9C2D86D9EC96598C3B3CBE3FFFEFF
+FEFEFCD7EAFB6D94BD719DCA8BA9CBF7FFFFFFFEFBFFFBF2FCFFFFF8FFFFD0E5FA698FB4
+6B9BCB80AFDBCAEBFFCFE4F7CFE9F6CDE4ECD7ECEDD3E9E6CDE7E8C8E8F59DC5DF6B97BA
+6899D1ABCBE2F8FFFFF5F1EEFFFEFBF6F9F2FBFFFFF9F8FFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFEFE
+92AFD96A9CC1FBFFFAFFFCFFF8FFFFFEFFFA6C96C68EB2D4F9FFFDFFFCFFFFFDF8F9FFFF
+6F95C48BB7D2FFFDFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD4E5F9
+7197BBB1CCE1FCFFFFFAFEFDFFFFFDDAE4F06999C7B3CCE2FEFEFFFCFFFFFFFCF7D3E4F4
+659BC9B6CBDEFFFFFFFFFFFFFDFDFDFAFAFAFDFDFDFFFFFFFFFFFFFFFFFFFFFFFFFFFDFF
+C5D7EB6B98C264A0D45C92C07198B9C9E2F8C9DCEB6C93BC5D9CC7719DC46D94BFC3D9E6
+FBFFFF6F95C26A9CC1619DD36197C68DB6D6EEFFFFFFFFFAFFFEF6FBFFFFD8E8F76D99BE
+6396C1659BC96E9ABDB5CCDEFCFFFFFFFDF7FFFEF6FFFFFABCC8DE7497C1659AC66499C5
+6E93BDD3E4F8FBFFF9FBFFFFB7CCDD719EC55E92C2A6C1DEFEFFFFF9FFFF95B5CA5D9CD1
+7D9CB9F4F9FFF3FBFF7FA2C26D9CC682A3C2F7FFFFFFFFFAB4CFE2699AC56594C2A5BAD7
+FCFFFFFEFFFFD7E2E46298C6649CCB92B6D6FBFFFFFFFDF6FFFFFBFEFFFFFFFFFAFFFFFF
+FFFBF5FFFEFBB2C7DA6899C2679AC7B5CDE5FFFBFBFFFFFAD7EAFB6D94BF6D99C88FAECD
+F7FFFFFFFCF8FFFAF4FBFFFFF5FFFFD0E6FD789CC06B9AC86B9ECD6A99C36D99BC6999C1
+6C9ABE6996B76B98B96F9DBF6D9EC6689CCC5E96C96494C5B4D4E9F7FFFFFBFBFDFEFFFF
+F8FFFEF9FFFFFBFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFEFE92AFD96A9CC1FBFFFAFFFCFFF8FFFF
+FEFFFA6C96C68EB2D4F9FFFDFFFCFFFFFDF8F9FFFF6F95C48BB7D2FFFDFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD4E5F97197BBB1CCE1FCFFFFFAFEFDFFFFFD
+DAE4F06999C7B3CCE2FEFEFFFCFFFFFFFCF7D3E4F4659BC9B6CBDEFFFFFFFFFFFFFFFFFF
+FDFDFDFDFDFDFEFEFEFFFFFFFFFFFFFFFFFFFFF7F4EAF3FF779DC1679DCC6094C376A2C7
+9CC0E093B6D46E94C15A98BD769CC180A1CCEAFCFFFBFFFF6C95C9659CC369A1D46B9CC7
+8FB3CDF2FFFFFFFFF4FFFDF4FCFFFFD6E7F76C96BE6598C56197C66B97BEB5CEE4FBFFFF
+FEFDFBFFFCF9FEFFFBBACAE36E94C3659DCC669FCC7097C2D4E5F9FFFFF6FFFEF6E8F1F8
+76A1C45B97CD81A5C9F7FBFFE6F3FC709DBE5B99CAA1B7CCFFFAF4FFFEFDAAC8E4669AC9
+749ABFE1F0FFFDFDFF8CAAC4659BCA699AC3D3E4F6FFFEFDFFFFFFD9E5F16598C36699C4
+8EB0CBF5F8FDFFFEF8FEFFFFFAFEFFFFFEF8F5FEFFFFFFFBFFFCF9B4C7D86596BF679BCA
+B6CCE3FFFBFAFFFFF8D0E3F47096C56D98CB95B6D5F4FFFFFFFFFDFFFCFBFFFCF4F6FBFF
+D0E6FD749BC26493C16895BF6F95B97796B27197C6749AC97195C77293C67596CB6F93C7
+6C91C57398CC769ABAC5DCE4F9FEFAFFFCFFFEFBFFFCFFFDFCFFF8FFFEFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFEFEFE92AFD96A9CC1FBFFFAFFFCFFF8FFFFFEFFFA6C96C68EB2D4F9FFFDFFFCFF
+FFFDF8F9FFFF6F95C48BB7D2FFFDFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFD4E5F97197BBB1CCE1FCFFFFFAFEFDFFFFFDDAE4F06999C7B3CCE2FEFEFFFCFFFF
+FFFCF7D3E4F4659BC9B6CBDEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFEFEFFFFFF
+FFFFFFFFFAF3FBFFFF9ABBDC689BC86BA0CC6494BC73A1C56D9AC1749AC75F9BBD7195B9
+9DBAE4F4FFFFFCFFFF6893CA67A1C96696C46E9ABF93AFC5F9FFFFFFFEEFFFF8EAFEFFFF
+D5E5F46C96C06F9FCF669BCD6B98C2B2CCE5F8FFFFFFFEFFFFFFFFFBFFFEB7CBE66A94C4
+609CCE649ECE7097C2D0DFF2FFFFF5FFFBECFEFFFF7AA0C464A1D86E95BCE7EEF8D1E1EE
+6499C1649ECCC7D7E6FFF9EAFFFCF6D3EEFF6096C56E98C2BFD1E7F9F8FE7493B0649ECC
+73A5CAF2FFFFFFFBF6FFFEFFD8E5F66F9CC36A97BE94B2CAFBFFFFFFFFFAF6FBFEFBFFFF
+FFFAF4F0FFFFFEFFFFFCFBF9BACDDE6A9BC4689CCBB1C7DEFFFCF8FFFFF3CFE0F2789ECD
+6A98CA8BAECCEBFCFFFCFDFFFFFAFEFFFBE7FFFFFDCDE5FD6E99C46999C986ACD1D2E8FD
+DCE6F0D4E7EED3E3F0DBE8F9D7E2F8D6E1F7DEE9FBDAE6F2D9E6EFD9F0FFF8FFFFFFFDF4
+FFF6FDFFFAFFFEFDF8FFFFF3FFFDF4FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFCFF8EBAE16B96B6FBFFFD
+FCFFF8F8FFFFF2F9FF9BA6B87B93B7D3ECFFF4FFFEFFFFFAF6F9FF7E95BE95AFD0F2FFFF
+FFFDFDFFFDFDFFFDFDFFFDFDFFFDFDFFFDFDFFFDFDFFFDFDDAE4F07B96B1ACCBE8F2FFFF
+F8FFFFEEFFFFB6D4EC7190A5C5CFCEF1FAF7FFFFF8FFFDF9D4E1F17B99BBB5C8E6FFFCFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFEFFF9F9FFD8DFF17C95B4
+6D99C8669AD3689ACF6A94C4669ECF6A93BF7E93B0E1E8F0FBFFFDF1FFFF6D8EAF6E98D2
+5E95CB719CC698AEC5FCFFFFFFFDFDFFFCFBFCFFFFDAE4EE6B9AC66298C7679DCC6A96BB
+B7D0E6F5FFFFF7FCFFFBFFFFF5FFFFAFC7DF729CC46499CB649AC96E96BAD2E7FCF7FFFF
+F9FFFFF3FDF5C6D9D77795B96A98C998CDEC8CB9D07E9CC26F93B3E7FAFFFCFCFAFFFFFA
+F2FFFF7E9EB7739DC37AA5D0B8D9F87BA1C56C92B6ADC7DEF8FFFFFFFCF8FFFCF9F6FAFB
+6F99B2689ABB6899BAEBFFFFFCF8F7FFFBF4FFFDFFF0FFFFFFF8E9FFFFF8F1FFFF84AFD2
+5D95C66695C3B4C9E6FBFBFFFEFEFED2EBFF6D9DC36999BF9DB8CDFAFEFFFFFEFBFFFFFD
+FBFFFFFEFFFFECF4FF7791B46997C97BA2CBF8FFFFFFFBE8F8FFFFFBFEFFFEFCFDFFFDF9
+FFFDFAFFFEFDFEFFFFFBFFFFFEFFFFFBFAFFFFFBFFFFFDFFFFFCFFFFFFFFFCFFFDFBFFFB
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFCFD89B6DD6895B4FBFFFFFCFDF7F8FFFFFBFFFFC3CFDF679CC4
+699ECA719BC17390AE749CBF5791C36DA6DB6590BB669AC9669AC9669AC9669AC9669AC9
+669AC9669AC9669AC97298BC5E94C367A1D17098BB7797B0729ABD6596BE7EAED4EAF3FC
+F8FFFFFFFFFFFFFFFFD4E7F8749BBCAECAE2FDFCFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFEFEFEFFFFFFF3FAFF92ABC16995BC6199C86397C6709FC9619BCD
+6C97C298ACC7F4F9FFFAFFF9F1FFFF7498BA6394CE619DD36F9CC695ADC7F7FEFFFFFEFD
+FEFAF9FCFFFFDEE7F06F9CC5689BC86FA2CF6F97BAB8CBDCFCFFFFFFFFFDFFFFFDFBFFFF
+B3C5D9769CC06A9ECD6A9ECD7498BAD3E3F3FFFEFFFAFAFFFCFFFAE5F3F37E9BBD6795C7
+7DB5DA76A7C86F94C18FADC7F0FEFFFFFFFBFDFDF5F5FFFFA1BFD76D97BD6E9AC787B1D7
+6C96BC749BBCD3E9FEFCFFFFFFFDF9FFFBF8FBFFFF88ABCB6A97BE709FC9B3D1EBFEFFFF
+FFF8EDFFFBF4FCFFFFFFFEF8FEFFFFD1E4F5779FC36A9ECD719CC6BDCFE5FFFEFFFFFFFD
+CFE8FC6495BD6599C198B6CEF8FFFFFFFEFBFFFFFDF7FDFDFAFCFBF4F8FF98AFCE6793C2
+729DC8D0E2F6FFFEFBFFF9F6FFFDF8FFFCF6FFFCF6FFFDFAFFFFFFF7FEFFE7F3FFE2E9F3
+FBFFFFFEFEFFFFFEFFFFFEFFFFFFFFFBFDFAF8FDF9FFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFDFD85B2D9
+6A97B8FCFFFFFDFEF9F8FFFFFCFFFFEDF7FFAACCD88FB3D598B5DF93AAC997BAD05F94B3
+74A4CC8FAAD58EB2D48EB2D48EB2D48EB2D48EB2D48EB2D48EB2D48EB2D484A7C76497C4
+76A9D693B2CE9DB0C194B0C69BBDD8C9E9FFFBFEFFFBFFFFFDFCFFFEFFFFD0E7F96F9CBB
+AFCFE4FCFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF8FFFFF8
+FBFFFFB7CDDA6B97B4629CC4619BC36498BF619ECD6D98C2C3D5EDFFFEFFF9FAF4F2FFFF
+749BBC6599D35D9AD16899C498B3CEF7FFFFFFFEFFFEFAF9FEFFFFE0E7ED6D9AC35D90BB
+6396C17094B4BCCCDBFDFDFDFFFEF8FFFEF8FEFCFFBAC7D7749BBC6197C56397C67598B8
+D8E2EEFFFBF8FFFBFFFFFFFAF9FFFF9EB7D66C9ACB629CCA6399C76896CAC0D3E4F7FFFF
+FFFFFAFAF7F0F7FFFFC9E5FB6D95B96998C46599C86493BD85A8C8F1FFFFFFFFFFFFFDFA
+FEFDFBFBFFFFC2D7F67699C36393C1759DC1EFFFFFFFFEF9FFFBEFFBFBF1F8FFFFE0EDFE
+9BB5D0638FB66BA0CC6A95B8B7CAD9FFFFFDFFFDF9D1E8FA6095BF669FCC95B8D4F7FFFF
+FFFEFCFFFEFBFBFFFFFFFFFFFAFBFFBBCDE3678EB76392BE96BADADDF2FFFFFCF4FFFCF6
+FFF9F4FDF9F8FCFFFFF2FFFFBED3E686A0B78897AAE0ECFCF9FFFFF8F9FBFFFFFDFEFDFB
+FDFDFDFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFB86B3DA6F9BBEFCFFFFFFFFFDF9FEFFFEFFFF
+F9FFFFFFFFF6F8FDFFFCFEFFFFFEFFF5FFFF7C99A997ADC2FCFEFFFFFEFDFFFEFDFFFEFD
+FFFEFDFFFEFDFFFEFDFFFEFDFFFEFDD0E5FA789BBBABCCEBF5FFFFFFFEFFFCFFFFF5FEFF
+FBFFFFFFFDFEFFFEFFFDF7F7FFFEFFCEE3F47099B9AFCADFFDFCFAFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFBFBF8F3FEFFFFDDEFFB789FBC6398C26BA1CD
+6397BF609BC5749ABFE6F2FFFFFDFFF9F6EFF4FFFF6990AF6599D2609CD06796C098B6D0
+F4FFFFFFFFFFFFFFFDFEFEFEDEE2E56C9DC8669CCB689ECC6F97BBB9CEDFF9FDFFFFFEFB
+FEFDF9FDFDFFBFD0E06E99BC619DCF659FCF7098BBD9E7F4FFFDFCFFFCFDFFFAF7FFFEFF
+C9DAF47099C56199CC639AD06B9DD2EAF3FCFEFFFFFFFFF8FFFCF5F9FFFFE3FAFF7DA1C3
+6E9BC5649ED06897C1A4C2DCF7FFFFFFFFFDFFFEFAFFFFFFF9FEFFF8FFFF8DA8C56A99C3
+6396C17BA2C1C1D8E8F4FFFFF4FFFFC6E2FF99B7DB6D93BA6494C26BA3D26896BAB1CBDA
+FBFFFFFFFBF4D5EAFB6498C067A0CD92B7D4F7FFFFFFFEFCFFFEFBF8FDFFFEFEFEFFFFFF
+E6F2FF91B2D16693BD608DB684ACD0C8D2DCE9F5FFF2FFFFEBFFFFBFD9F093B5D1789FC0
+6A95B88091ADDEEDFFF7FFFFFAFEFDFFFFFAFEFDF9FFFEFFFFFDFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFA86B6DC709CBFFCFFFFFFFFFDFBFFFFFBFBF9FBFFFFFFFEFFFAFFFFFBFFFBFAFCF9
+F1FFFF7595BC93B1CDEEF9FDFAFEFFFAFEFFFAFEFFFAFEFFFAFEFFFAFEFFFAFEFFFAFEFF
+D4E6FA6F92B2ACCEEAF7FFFFFFFBF8FFFFFFF7FCFFFFFFFFFFFEEFFFFFF3FFF6EDFFFEFB
+D9EAFA7B9CBFB6C8E0FFFAFAFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FEFFFFFFFAFEFFFEFFF7FFFF99B6D66691BE6B99CA6F9AC76798B89EBDD9F8FFFFFFFCFF
+FFFEF5F7FFFF799BB66393C36A9BC66F97BB9FB9D0EFF9FFFEFFFFFFFFFDFEFEFEE2E6E7
+5F90BB699FCE679FCE6793B8B7D0E6F8FFFFFBFFFFF8FDFFF9FFFFC0D6EB6492B6629ED0
+65A1D36893B6D3E6F7FCFFFFFFFCFAFFFCFDFFFDFFE7F3FF7899BA6B99CB6695C981ACD7
+FEFEFFFFFFFDFFFFFAFFFFF8FCFFFFF3FFFF9CBAD66B92B96397C6749CBFD2E4F8F9FCFF
+FFFEFDFFFBF8FFFFFFFAFEFFFFFEF9DBECF684ABC85E92BA6699C47BA7CE688EB37598BE
+74A0CF5A85B06B96C178ABDA659BCA6898BEB3CFE4F3FFFFFFFDF4D8EAF86595BB5F93BB
+91B1CAF7FFFFFFFFFDFEFDFBFCFFFFFFFFFDFFFFFBF9FFFFCCE3F58BAFCF6F9BC26390BA
+709CBF6995BA6794BB6C9CC46E9EC66898C06696BC6B98BFC7D9F1F2FFFFF8FFFFFAFFFB
+FFFFFAFFFEFBFFFDFFFEFBFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFA86B6DC6D99BEF8FBFFFFFDFE
+FEFFFFFFFDF8FEFFFDFFFDFFFCFFFFFBFFF0FFFFF6F4FFFF7697C492B3D4F1FFFFFFFFFD
+FFFFFDFFFFFDFFFFFDFFFFFDFFFFFDFFFFFDFFFFFDDFE9F5778FA7B4CDE3FEFFFFFFF6EE
+FFFDFAFAF9F7FFFEFAFFFFF3FFFFF8FFFFFAFCFFFFCBDFF77193B9BBC9E3FFF9FBFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFAFCFFFFFDFFFDF6FDFEFDFFCDDEFA
+86A3CB6C8DB87491B97DA1AFD1E6F7F7F9FFFFF6F9FFFDF7F1FAF9A2BDCE6D93B76D90B0
+88A4BCC3D4E4F9FFFFFEFFFFFDFDFBFEFFFFF8FCFD81A5C76E98BE6491BA7CA3C4D4E9FC
+F8FFFFFBFFFFF7FFFFF8FFFFD6EBFC80A4C46697BF6899C183A6C2E2F4FFFCFFFFFEFFFA
+FFFCFFFFF8FFFBFFFFA0B7C97A9AC16E8EB7B6D2EAFFFCFBFFFFFFFCFEFBFFFEF9FEFDF8
+FBFFFFC8DAEE7B98B86D93B892ACC5F8FFFFFFFFFFFFFEFFF9F9FBFEFEFEFFFFFDFFFFF3
+FBFFFFD1E8F69BBEDA719BC1628DB86A96C56D99CA6892BA7CA0C4AED1F1AAD0F46B95BB
+7296B6BFD6E8F8FFFFFFFDF4DDEBF6789FBE6C95B5A3B8C9F9FDFFFFFFFDFDFEFFFAFBFF
+FDFCFAFFFFFAFEFFFBF7FFFFBED5E79EBCD87597BA6DA0CB689DC9699ECA6A9DC86595BD
+6F97BA9ABDDBCAE9FFF1FDFFF8FFFFF6FAFDFEFEFCFFFEFBFEFAF9FFFEFFFDFAFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFF785B7DA6B97BCFAFDFFFFFDFFFEFEFFFFFEF8FFFFFAFFFDF9FFFFFA
+FFFFF3FFFDF1F9FCFF7D98B37B9DB9DEF3FFFEFFFFFEFFFFFEFFFFFEFFFFFEFFFFFEFFFF
+FEFFFFFEFFFFCEDEEE7E9CB8AECCE8F6FFFFFCFBF9FCFFFFF7FFFFF6FDFFF2FFFFF0FFFF
+F0FFFFE3F6FFA2C6E66794BDC2DAF2FFF7F5FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFF9FFFFFFFEFFFFF8F6FFFEFFF4FCFFC7D9EFA1B5CEA5B7CFC9DDDCF7FFFF
+FFFDFFFFFCFFFFFEFAFCFFFDE1F3FDB4CFE49FB4C7BFCFDEEEF7FFFEFFFFFFFFFDFCFCFA
+FEFFFFFBFFFFDDE9F9AFC4D9A4BED7C4D9ECF8FFFFFFFFFDFDFCF8FCFEFDFFFFFFF8FFFF
+CBDEEDA3BED3A6C1D6CEE0EEFBFFFFFFFEFBF8FFFFFEFCFFFEF7FEFFFFF8DBE9EAB0BFDC
+AAB8D3ECF7F9FFFCFBFEFFFFFAFFFFFDFEF9FFFFF6FFFEFBF2FAFFB2C6DEA8BED5D1DBE7
+FFFEFFFFFEFDFEFFFFF6FBFFFEFEFEFFFEF8FBF7F4FFFFFDFEFFFFEDF4FCDCECFBB9CFE6
+9BB9D58FB0CFB7CEE0DBEBFAF3FFFFEAF7FFB1C3D7AFC0D2E9EDF8FFFDFFFFFDF4EEF8FF
+B0CEE6A8C4DAD5DFE8FFFCF9FFFFFDFBFFFFFFFEFFFFFEFFFFFEF9FFFFF6FFFFFAFBFFFF
+F1FDFFE2F1FFB0CAE19BB7CF8FADC7A2BED6BFD8EEDBECFCF1FBFFFBFFFFFCFFFDFEFFFD
+FFFFFDFFFDFDFFFDFFFDF9FAFFFDFFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFEF685B7DA6B97BE
+FEFEFFFFFDFFFEFCFFFFFFF6FFFFF8F8FFFBF3FEFFFEFEFFFFFBFDF7FFFFA3CCDE6B9BC1
+7196C26899C16899C16899C16899C16899C16899C16899C16899C1709FCB5998CE5C9DD3
+6D9AC17294B06E99BB6A9BC36997BB7593B57294B9799ABB799CBA6293BB6BA4D1E1FFFF
+FFFFF8FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF8FFFFF9F9F1FFFDF1
+FFFCF1FEFFFFF5FFFFF0FEFFF7FFFFF8FFFAF4F8F9FFFBFFFFFBFFFDF8F5FBFDF8F5FFFF
+ECFEFFF7FFFFF8FFFFFCFFFFFFFFFFFFFFFDFFFFFDFBFFFFF9FEFFFFFDFDF5F8FFF7FFFF
+F9FFFFFFFEFBFFF8EFFFFCF3FFFDF7FFFDF6FFFDFAFCFFFFF4FDFFF6FFFFFEFFFFFFFBF5
+FFFDF1F3FFFFF7FBFFFFFDFFFCFAEBFBFFFAF5FCFFFBFEFFFFFFF6FFFEFFFCFFFFF9FFFF
+FEFFFDFFFDF4FFF8F1FCFFFFEFFEFFF1FAFFFEFFFFFFF8F5FFFBF8FAFDFFF9FFFFFFFFFF
+FFFDF4FFFDFFFFFBFFFFFBF8FFFBF4FFF9F4FEFFFDEEF9FBF2FFFFF7FDFDFFFFFBF9F2EA
+FFFEFBFFFEFFFAF9FFFFFBFFFFF7F4FFFBF4F8FFFFE9FFFFE9FCFFFEFFFFFFFDF6FFFDFA
+F8FFFFFFFDFFFFFFFFFFFFFAFFFFF3F9F5E9FFFEF8F9F7FAFEFEFFFFFCFBFFFFFFFCFFFF
+FCFFFFFEFFFFFFFEFBFFFCF4FFFBEFFFFFF3FFFCF1FFFCF9FFFCFDFFFBFFFFFDFFFFFDFF
+FEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFEFAF78BB6D6649ABEF5FFFFFFFCE8FFFCF9FDFBFFFFF8FA
+FFFFF8FEFFF9FDFFFCFCFFFFF9FFFFEAF4FFC4D0E0A3AEC08EB2D48EB2D48EB2D48EB2D4
+8EB2D48EB2D48EB2D48EB2D486B1D4688EBB86A8CD92B2C196B4BF98B3CE94B1CF91B0C5
+8CAEC995B5CE97B2C799B0C2B6C6D3E0EDF6F9FFFFFBFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFB
+8EB3D06A9CBFF4FFFFFFFFF4F8FFFFF2FFFFFDFBFFFFFDF4FFFDF7FFFEF9FEFEFCFEFFFF
+FCFFFFFBFFFFF2F9FFF9FFFDF9FFFDF9FFFDF9FFFDF9FFFDF9FFFDF9FFFDF9FFFDC1E4FF
+809EC4B2CBE9ECFFFFF2FFFFF1FEFFF2FFFFF2FFFFF1FFFFF2FFFFF4FFFFF3FDFFF7FEFF
+FCFFFFFEFFFFF9F9F9FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFF8AADCD709EC0F2FFFFFFFEF5F2FAFC
+F8FFFFFFFCF9FFFEFFFFFFFFFEFFFFF9FDFFF6FBFEF7FFFFF8FFFFF8FFFFFFFCFFFFFCFF
+FFFCFFFFFCFFFFFCFFFFFCFFFFFCFFFFFCFFCFEBFF788FAFB7C8DCF8FFFFFBFFF9FBFAFF
+FFFDFFFDFCF7FFFFFFFDFDFDFDFDFDFFFFFDFFFFFDFFFEFBFFFEFBFFFEFAFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFF7FFFF8AACCF749ABEF0FEFFFFFEF8FFFEF5FFFFF4FFFCE8FFFDFFFFFDFFFFFEFF
+FFFFFDFFFFFDFFFFFBFCFDF7F9FCF5FFFDF8FFFDF8FFFDF8FFFDF8FFFDF8FFFDF8FFFDF8
+FFFDF8CFEDFF7C93B3BCCBE0F8FFFEFCFCF2FFFDFDFFFCFDFFFBF0FFFEF8FFFEF8FFFCF8
+FFFDFAFFFDFAFCFBF9FCFCFCFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF5FFFF9FB9D47798B7C8DEF3
+F0FEFFF2FFFFF4FFFFF6FFFFFEFEFFFBFEFFF9FCFFFBFEFFFCFFFFFCFFFFFBFFFFFBFFFD
+F9FFFFF9FFFFF9FFFFF9FFFFF9FFFFF9FFFFF9FFFFF9FFFFABCDE97791B4BBCDE5F5FFFF
+FCFFFAFFFEFFFFFCFFFFFCF6FFFDFAFFFFFDFFFFFFFEFFFFFCFFFFFBFFFFF8FFFFF7FEFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFCFFFFCBD8E1899FB67A99B8749ABE719FC36B9DC0659BBF709CC3
+6D99BE6A96BB6A96B96A97B86996B76B98B76F9ABA6F95C46F95C46F95C46F95C46F95C4
+6F95C46F95C46F95C47998B488A0C2D6E8FFF7FFFFFBFFFBFDFDFFFFFEFFFFFFFDFCFDFF
+FEFFFFFCFFFFFBFFFFFBFFFFF9FFFFF8FFFFF8FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFCFFFAF8FCFB
+CBD8E1A0BAD58BADD08BAECE8FB4D18DB8D885B7DA84B6D986B6DC88B8DE89B6DD86B3DA
+87B4DB8BB7DE8BB7D28BB7D28BB7D28BB7D28BB7D28BB7D28BB7D28BB7D2ADC0D1D4E2FC
+F7FFFFFBFFFBFDFEF6FFFDFFFFFEFFFEFFFAFEFFFFFEFFFFFDFEFFFBFBFBFBFBFBFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF2FFF8FCFFFAFCFFFFF5FFFFF7FFFFFFFEFFFFFEFB
+FFFBF8FFFFF8FFFFF8FEFDF8FFFFFAFFFEFBFFFDFDFFFAFBFFFAFDFFFDFFFFFDFFFFFDFF
+FFFDFFFFFDFFFFFDFFFFFDFFFFFDFFF8FFFFFBFEFFFEFEFFFAF8ECFFFFF1FFFCFCFFFDFF
+FEFBF6FFFFFDFFFEFBFFFEFBFFFEFAFFFEF8FFFEF8FFFDF6FFFCF5FFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF
+
+end
+%%PageTrailer
+%%Trailer
+%%EOF
diff --git a/api/docs/images/vmware-logo.ico b/api/docs/images/vmware-logo.ico
new file mode 100644
index 000000000..8b93853b0
Binary files /dev/null and b/api/docs/images/vmware-logo.ico differ
diff --git a/api/docs/images/vmware-logo.png b/api/docs/images/vmware-logo.png
new file mode 100644
index 000000000..9603bd171
Binary files /dev/null and b/api/docs/images/vmware-logo.png differ
diff --git a/api/docs/interpose.fig b/api/docs/interpose.fig
new file mode 100644
index 000000000..6f3564006
--- /dev/null
+++ b/api/docs/interpose.fig
@@ -0,0 +1,39 @@
+#FIG 3.2
+Landscape
+Center
+Inches
+Letter  
+100.00
+Single
+-2
+1200 2
+0 32 #000000
+0 33 #eeeeee
+0 34 #ffffaa
+0 35 #0000ff
+0 36 #6666ff
+0 37 #aaaaff
+0 38 #00ffff
+0 39 #77ff00
+0 40 #aaffaa
+0 41 #dd00ff
+0 42 #ffaaaa
+0 43 #cccccc
+0 44 #aaaaaa
+2 2 0 1 0 42 51 0 20 0.000 0 0 -1 0 0 5
+	 600 1950 3600 1950 3600 1200 600 1200 600 1950
+2 2 0 1 0 44 54 0 20 0.000 0 0 -1 0 0 5
+	 600 3150 3600 3150 3600 2400 600 2400 600 3150
+2 1 0 1 0 7 50 0 -1 0.000 1 0 -1 1 0 2
+	1 1 1.00 60.00 90.00
+	 2100 750 2100 1200
+2 1 0 1 0 7 50 0 -1 0.000 1 0 -1 1 0 2
+	1 1 1.00 60.00 90.00
+	 2100 1950 2100 2400
+2 2 0 1 0 33 54 0 20 0.000 0 0 -1 0 0 5
+	 600 750 3600 750 3600 0 600 0 600 750
+4 0 0 50 0 0 12 0.0000 4 180 1425 1425 375 running application\001
+4 0 0 50 0 0 12 0.0000 4 180 1380 1425 2775 hardware platform\001
+4 0 0 50 0 0 12 0.0000 4 180 2790 750 1650 observe/manipulate every instruction\001
+4 0 0 50 0 0 12 0.0000 4 180 1890 1125 1800 in the running application\001
+4 0 0 50 0 0 12 0.0000 4 180 1950 1125 1425 runtime code manipulator:\001
diff --git a/api/docs/intro.dox b/api/docs/intro.dox
new file mode 100644
index 000000000..a8b65cd6e
--- /dev/null
+++ b/api/docs/intro.dox
@@ -0,0 +1,1250 @@
+/* **********************************************************
+ * Copyright (c) 2007-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/**
+ ****************************************************************************
+ ****************************************************************************
+ ****************************************************************************
+\if vmsafe
+\mainpage VMsafe In-Process API
+\section VMsafe_intro Introduction
+This document describes a component of the VMsafe model that provides control 
+at a process level and explains how to use it.   For the rest of the document 
+this component shall be referred to as the VMsafe In-Process Runtime or VIPER.
+
+VIPER allows a user specified (via a Guest Agent) in-process agent (referred to
+as \client from now on) to monitor and modify the execution of any given
+process.  VIPER is different from the security agent on the security monitor VM
+or an in-guest agent as neither of them provides direct control at the
+process level because they are unaware of processes.  VIPER on the other hand
+is fully aware of process and resides within the address space of each one that
+is intended to be monitored or controlled.  It provides a finer grain of
+control as it allows the user to decide which processes and what aspects of the
+processes' execution to monitor and control.  Also, a \client is an optional
+component for any VMsafe-based solution.  The figure below gives an idea of how
+VIPER fits in with other VMsafe components.  All user- or ISV-provided parts of a
+VMsafe solution are shown in blue and the VMsafe components in gray.
+
+\image html viper.png
+\image rtf viper.png
+\image latex viper.eps
+
+\else
+\mainpage The \DynamoRIO API
+\endif
+
+\image html drlogo.png
+\image rtf drlogo.png
+
+\DynamoRIO is a <em>runtime code manipulation system</em> that supports
+code transformations on any part of a program, <em>while it executes</em>.
+\DynamoRIO gives complete control over the runtime code stream and does not
+limit transformations to trampoline insertion.  \DynamoRIO exports an
+interface for building dynamic tools for a wide variety of uses: program
+analysis and understanding, profiling, instrumentation, optimization,
+translation, etc. \DynamoRIO provides efficient, transparent, and
+comprehensive manipulation of an unmodified application running on a stock
+operating system (Windows\if linux or Linux\endif) and commodity IA-32 and
+AMD64 hardware.  See \ref sec_limit_platforms for supported platforms.
+
+This document describes the \DynamoRIO system and the various API's that it
+exports for building custom tools.  It is divided into the following
+sections:
+
+- \subpage using
+  <br>How to deploy and make use of \DynamoRIO.
+\if vmsafe
+- \subpage API_security
+  <br>An interface for customizing and applying the award-winning Memory
+  Firewall security software.
+
+- \subpage API_probe
+  <br>A powerful probe insertion interface that can operate with the code cache
+  or in a lighter-weight but more restrictive mode without code cache.
+\endif
+- \subpage API_BT
+  <br>\DynamoRIO's full runtime code manipulation interface.
+
+\ifnot vmsafe 
+- \subpage page_standalone
+  <br>\DynamoRIO can be used as a standalone library for IA-32/AMD64
+  disassembly, decoding, encoding, and general instruction manipulation.
+\endif
+
+- \subpage API_tutorial
+  <br>Gives a few short tutorials on using \DynamoRIO.
+
+- \subpage API_samples
+  <br>Shows some sample use cases and reference implementations.
+
+- \subpage overview
+  <br>A description of the implementation of the \DynamoRIO system.
+
+- \subpage release_notes
+  <br>Release notes for this release, including changes since prior
+  releases \ifnot vmsafe and plans for future releases \endif.
+
+- \subpage page_license
+
+\image html vmware-logo.png
+\image rtf vmware-logo.png
+
+
+***************************************************************************
+***************************************************************************
+\page overview \DynamoRIO System Details
+
+\DynamoRIO is a system for runtime code manipulation that is efficient,
+transparent, and comprehensive, able to observe and manipulate every
+executed instruction in an unmodified application running on a stock
+operating system and commodity hardware.
+
+***************************************************************************
+\section sec_intro Introduction
+
+\DynamoRIO operates in user mode on a target process.  It acts as a
+<em>process virtual machine</em>, interposing between the application and
+the operating system.  It has a complete view of the application code
+stream and acts as a runtime control point, allowing custom tools to be
+embedded inside it:
+
+  \image html interpose.png
+  \image rtf interpose.png
+  \image latex interpose.eps "Flow chart" width=10cm
+
+The application itself, along with the underlying operating system and
+hardware, remain unchanged.  \DynamoRIO operates in native (non-virtual)
+environments as well as inside guest operating systems running on virtual
+machines.  Tools created on top of \DynamoRIO will operate without change
+whether the underlying operating system is native or a virtual machine
+guest.
+
+***************************************************************************
+\section sec_system System Operation
+
+\DynamoRIO operates by shifting an application's execution from its original
+instructions to a <em>code cache</em>, where the instructions can be freely
+modified. \DynamoRIO occupies the address space with the application and has
+full control over execution, taking over whenever control leaves the code
+cache or when the operating system directly transfers control to the
+application (<em>kernel-mediated control transfers</em>):
+
+  \image html flow-highlevel.png
+  \image rtf flow-highlevel.png
+  \image latex flow-highlevel.eps "Flow chart" width=10cm
+
+\DynamoRIO copies the application code one <em>dynamic basic block</em> at a
+time into its basic block code cache. A block that directly targets another
+block already resident in the cache is linked to that block to avoid the
+cost of returning to the \DynamoRIO dispatcher.
+
+Frequently executed sequences of basic blocks are combined into
+<em>traces</em>, which are placed in a separate code cache. \DynamoRIO makes
+these traces available via its interface for convenient access to hot
+application code streams.
+
+The following figure shows the flow of control between the components of
+\DynamoRIO and its code caches:
+
+  \image html flow.png
+  \image rtf flow.png
+  \image latex flow.eps "Flow chart" width=15cm
+
+The context switch is between \DynamoRIO's operational state and the machine
+state of the application: both are still within the same process.
+
+Indirect branches require dynamic resolution of their targets, which is
+performed via an inlined table lookup or a compare to a known target
+inlined into a trace.
+
+\section sec_sys_transp Transparency
+
+Transparency is an important requirement for \DynamoRIO and its \clients.
+The subject is fully covered in \subpage transparency.
+
+\ifnot vmsafe
+***************************************************************************
+\section sec_refs References
+
+The canonical reference for \DynamoRIO is:
+
+- Derek Bruening.<br>
+  <a href="http://www.burningcutlery.com/derek/phd.html">
+  Efficient, Transparent, and Comprehensive Runtime Code Manipulation</a>.<br>
+  Ph.D. Thesis, MIT, September 2004.
+
+Other publications describing \DynamoRIO include:
+
+- Derek Bruening and Vladimir Kiriansky.<br>
+  <a href="http://www.burningcutlery.com/derek/docs/procshared-VEE08.pdf">
+  Process-Shared and Persistent Code Caches</a>.<br>
+  International Conference on Virtual Execution Environments (VEE-08), March 2008.<br>
+
+- Derek Bruening, Vladimir Kiriansky, Timothy Garnett, and Sanjeev Banerji.<br>
+  <a href="http://www.burningcutlery.com/derek/docs/threadshared-CGO06.pdf">
+  Thread-Shared Software Code Caches</a>.<br>
+  International Symposium on Code Generation and Optimization (CGO-06), March 2006.<br>
+
+- Derek Bruening and Saman Amarasinghe. <br>
+  <a href="http://www.burningcutlery.com/derek/docs/cacheconscap-CGO05.pdf">
+  Maintaining Consistency and Bounding Capacity of Software Code Caches</a>.<br>
+  International Symposium on Code Generation and Optimization (CGO-05), March 2005. <br>
+
+- Gregory Sullivan, Derek Bruening, Iris Baron, Timothy Garnett, and 
+  Saman Amarasinghe. <br>
+  <a href="http://www.burningcutlery.com/derek/docs/IVME03.pdf">
+  Dynamic Native Optimization of Interpreters</a>. <br>
+  ACM Workshop on Interpreters, Virtual Machines and Emulators (IVME-03), June 2003.<br>
+
+- Derek Bruening, Timothy Garnett, and Saman Amarasinghe. <br>
+  <a href="http://www.burningcutlery.com/derek/docs/adaptive-CGO03.pdf">
+  An Infrastructure for Adaptive Dynamic Optimization</a>. <br>
+  International Symposium on Code Generation and Optimization (CGO-03), March 2003. <br>
+
+- Derek Bruening, Evelyn Duesterwald, and Saman Amarasinghe.<br>
+  <a href="http://www.burningcutlery.com/derek/docs/win32-FDDO.pdf">
+  Design and Implementation of a Dynamic Optimization Framework for Windows</a>.<br>
+  4th ACM Workshop on Feedback-Directed and Dynamic
+  Optimization (FDDO-4), December 2001.<br>
+
+\endif
+
+\image html favicon.ico
+
+****************************************************************************
+****************************************************************************
+*/
+/* It's good to use separate C comments: we've hit some sort of doxygen
+ * internal buffering error before if one comment gets too long.
+ */
+/**
+***************************************************************************
+***************************************************************************
+\page using Usage Model for \DynamoRIO
+
+This section gives an overview of how to use \DynamoRIO, divided into the
+following sub-sections:
+
+\if vmsafe 
+ - \ref sec_deploy
+ - \ref dr_api
+ - \ref dr_modes
+ - \ref sec_events
+ - \ref sec_utils
+ - \ref sec_extlibs
+ - \ref sec_comm
+ - \ref sec_options
+ - \ref sec_debugging
+\else
+ - \ref sec_deploy
+ - \ref sec_events
+ - \ref sec_utils
+ - \ref sec_extlibs
+ - \ref sec_comm
+ - \ref sec_options
+ - \ref sec_debugging
+\endif
+
+\ifnot vmsafe
+\DynamoRIO exports a rich Application Programming Interface (API) to the
+user for building a \DynamoRIO <em>\client</em>. A \DynamoRIO \client is a
+program that is coupled with \DynamoRIO in order to jointly operate on an
+input program binary:
+
+  \image html \client.png
+  \image rtf \client.png
+  \image latex \client.eps "\DynamoRIO \client" width=10cm
+
+To interact with the \client, \DynamoRIO provides specific events that a
+\client can intercept.  Event interception functions, if supplied by a user
+\client, are called by \DynamoRIO at appropriate times.
+
+\DynamoRIO can alternatively be used as a third-party disassembly library
+(see \ref page_standalone).
+\endif
+
+\section sec_deploy Deployment
+\if vmsafe
+A \DynamoRIO application consists of a guest agent and one or more \clients.  Each of
+these must be linked against the necessary \DynamoRIO dynamic libraries, which
+are provided as part of the SDK.  These \DynamoRIO dynamic libraries must be
+installed when the \DynamoRIO application is installed on a guest OS.  
+
+Any \DynamoRIO application wanting to get control of a process must
+first register a \client (which is a self-contained executable
+library) for that process.  Otherwise \DynamoRIO will not be
+initialized for that process.  This is done by the guest agent, which
+can either be a service or a standalone executable.  The guest agent
+should use dr_register_process() and dr_register_client() to register
+a process name for which control is desired, which \clients to use for
+each process, and which mode to run the \clients in (more on this
+below).  \DynamoRIO will be initialized in the specified mode for
+registered processes started subsequently.  \DynamoRIO will then load
+the corresponding \clients into such processes and call each \client's
+initialization routine, dr_init().  There are three modes of operation
+for \DynamoRIO which map to the three API's provided by \DynamoRIO
+(see below).  Depending upon the mode chosen the usage model will
+change.
+
+In addition to registering the processes for which control is desired, the 
+guest agent can also unregister them as needed using dr_unregister_process().
+The guest agent can also send update messages to processes currently
+initialized with \DynamoRIO and a \client using dr_nudge_process().
+For example, this can be used by the guest agent when it has obtained new
+information (data, libraries, etc.), from a management server or over the
+internet from a security vendor, to let \clients know about them. Note
+that nudging 64-bit processes is not yet supported.
+
+Process registration, unregistration, and nudging require administrative
+privileges.  On Windows Vista, if UAC is enabled, process registration must be
+performed by an elevated (runas admin) process. When using \c drdeploy.exe,
+be sure that the cmd shell being used was started with elevated
+permissions.
+
+\else
+
+Once the \DynamoRIO distribution contents are unpacked (see \ref sec_package),
+deployment is handled by the configuration API and library (see
+dr_config.h) or the \c drdeploy.exe tool\if linux on Windows and the
+\c drdeploy script on Linux\endif.  \c drdeploy.exe is built on top of the
+configuration library and exposes all of its functionality.
+
+\ifnot vmsafe 
+When using \DynamoRIO as a third-party disassembly library (see \ref
+page_standalone), no deployment is needed, as \DynamoRIO does not control a
+target application when used as a regular library.
+\endif
+
+\if linux
+\subsection win_deploy Windows Deployment
+\endif
+
+To run a process under \DynamoRIO, a \client for that process must
+first be registered. Once unpacked, \c drdeploy.exe (or the configuration
+API) can be used to register processes and \clients,
+unregister processes or \clients, and \ref sec_comm
+"nudge" running processes.  Registration
+information is stored in the Windows registry 
+(\c \\HKLM\\SOFTWARE\\VMware\\DynamoRIO) and as such is
+persistent across reboots.  Processes are identified by executable
+name for registration, unregistration, and nudging.  Run \c drdeploy.exe
+with no arguments for help on its usage.
+
+As an example, assume you have unpacked the \DynamoRIO distribution to
+C:\\dr.  Register \c calc.exe to run with the bbsize sample \client
+using the following command:
+\if vmsafe
+\code
+C:\dr\bin\drdeploy.exe -reg calc.exe -root C:\dr -mode code -client C:\dr\docs\html\samples\bbsize.dll 0 ""
+\endcode
+\else
+\code
+C:\dr\bin\drdeploy.exe -reg calc.exe -root C:\dr -client C:\dr\docs\html\samples\bbsize.dll 0 ""
+\endcode
+\endif
+The next time \c calc.exe is started, it will run under \DynamoRIO
+with the bbsize \client. 
+
+To unregister \c calc.exe, issue the following command:
+\code
+C:\dr\bin\drdeploy.exe -unreg calc.exe
+\endcode
+
+To \ref sec_comm "nudge" all instances of \c calc.exe running under
+\DynamoRIO with argument "5", use:
+\code
+C:\dr\bin\drdeploy.exe -nudge calc.exe 0 5
+\endcode
+This will result in a nudge event with argument=5 delivered to the
+\client callback registered with dr_register_nudge_event() in all
+\c calc.exe processes running under \DynamoRIO.  The third argument,
+0, is an ID supplied at registration which uniquely identifies the
+target \client (see dr_deploy.h for details).  Note that nudging
+64-bit applications is not yet supported.
+
+To view 32-bit or WOW64 processes running under \DynamoRIO the
+\c bin\bin32\drview.exe tool can be used.  Note that the tool does
+not yet support showing the status of 64-bit applications.
+
+\attention
+Note that on Windows NT a reboot is required after the first process
+is registered before any applications will run under \DynamoRIO.
+
+\DynamoRIO uses the
+\c \\HKLM\\SOFTWARE\\Microsoft\\Windows\\WindowsNT\\CurrentVersion\\AppInit_DLLs
+key to inject into new processes.  For injection to work, the registered
+process must statically link to user32.dll (only a few small
+non-graphical windows applications don't link user32.dll).  If a
+target application does not link to user32.dll, \DynamoRIO can still
+inject if the process is launched with the appropriate drinject.exe tool (in the 
+bin/bin32 and bin/bin64 directories) or the parent process (usually cmd.exe or 
+explorer.exe for user launched processes) is running under \DynamoRIO.
+The drinject.exe tool uses the configuration information set by
+drdeploy.exe for the target application.
+
+\attention
+Process registration, unregistration, and nudging require administrative
+privileges.  On Windows Vista, if UAC is enabled, process registration must be
+performed by an elevated (runas admin) process. When using \c drdeploy.exe,
+be sure that the cmd shell being used was started with elevated
+permissions.
+
+\if linux
+\subsection lin_deploy Linux Deployment
+
+Once \DynamoRIO has been unpacked and the target machine configured, the
+\c drdeploy script can be used to run applications
+under \DynamoRIO.  The \ref limits_linux_preload "LD_PRELOAD"
+based \c drdeploy script and the \ref sec_startstop "app_start()/app_stop()"
+interface are currently the only supported methods of running
+applications under \DynamoRIO on Linux.  
+
+As an example of drdeploy usage, the following command runs \c ls
+under \DynamoRIO with the bbsize sample \client:
+\code
+% bin/drdeploy -client docs/html/samples/bbsize.so ls
+\endcode
+Run \c drdeploy with no options to get a list of the options and
+environment variable shortcuts it supports.
+
+\endif
+\endif
+
+\subsection multi_client Multiple \Clients
+
+\DynamoRIO does support multiple \clients.  It is each \client's
+responsibility, however, to ensure compatibility with other \clients.
+\DynamoRIO makes no attempt to force cooperation among \clients.  For
+example, instruction stream modifcations made by one \client are
+visible to other \clients.  Systems employing multiple \clients must
+be aware of such interactions and design accordingly.
+
+\Client registration requires users to specify the \em priority of each
+\client.  \DynamoRIO calls each \client's
+dr_init() routine sequentially according to this priority.  \Clients
+with a numerically lower priority value are called first and therefore
+given the first opportunity to register callbacks (the \client with
+priority 0 is called first).  Since \DynamoRIO delivers event
+callbacks sequentially, \client priority and the order of event
+registration is important.  For a given event, the \em first
+registered callback is called \em last.  This scheme gives precedence
+to the first registered callback since that callback is given the final
+opportunity to modify the instruction stream or influence \DynamoRIO's
+operation. 
+
+\if vmsafe
+***************************************************************************
+\section dr_api API
+\DynamoRIO's API is divided into the following interfaces:
+- \ref API_security
+  <br>An interface for customizing and applying the award-winning Memory
+  Firewall security software.
+- \ref API_probe
+  <br>A powerful probe insertion interface that can operate with the code cache
+  (\ref DR_MODE_CODE_MANIPULATION) or in a lighter-weight but more restrictive
+  mode without the code cache (\ref DR_MODE_PROBE).
+- \ref API_BT
+  <br>The full runtime code manipulation interface.  This includes the Probe
+  API, i.e., probes in the code cache.
+\endif
+
+\if vmsafe
+***************************************************************************
+\section dr_modes Modes
+
+The different modes a \client can request \DynamoRIO to operate in are listed
+below.  These are not to be confused with the different \ref dr_api provided by
+\DynamoRIO.  Each mode of \DynamoRIO's operation maps to one or more API.
+
+  - The Memory Firewall mode (#DR_MODE_MEMORY_FIREWALL) maps exclusively to the Memory Firewall API.
+  - The Probe mode maps to the Probe API (#DR_MODE_PROBE) without code cache, i.e., a lighter-weight but restrictive mode.
+  - The Code Manipulation mode maps to the Code Manipulation API (#DR_MODE_CODE_MANIPULATION) and the Probe API (#DR_MODE_PROBE) with code cache.
+  .
+
+Below we describe parts of the system that are common to all modes and
+interfaces.  Each interface adds its own events and utilities, described in
+the section for that interface.
+\endif
+
+***************************************************************************
+\section sec_events Common Events
+
+A \client's primary interaction with the \DynamoRIO system is via a
+set of event callbacks.  These events include the following:
+
+  - Basic block and trace creation or deletion
+
+  - Process initialization and exit
+
+  - Thread initialization and exit
+\if linux
+  - Fork child initialization (Linux-only); meant to be used for
+    re-initialization of data structures and creation of new log files
+\endif
+  - Application library load and unload
+\ifnot NYI_kernel_mediated_transfer_events
+  - Application fault or exception \if linux (currently Windows only) \endif
+\else
+  - Kernel-mediated control transfers:
+    - Application fault or exception
+    - Application APC (Asynchronous Procedure Call) or callback (Windows)
+    - Application signal (Linux)
+\endif
+  - System call interception: pre-system call, post-system call, and system
+    call filtering by number
+
+  - Signal interception (Linux-only)
+
+  - Nudge received - see \ref sec_comm \if linux (currently Windows only) \endif
+
+Typically, a \client will register for the desired events at
+initialization in its dr_init() routine.  \DynamoRIO then calls the
+registered functions at the appropriate times.  Each event has a
+specific registration routine (e.g., dr_register_thread_init_event())
+and an associated unregistration routine.  The header file dr_events.h
+contains the declarations for all registration and unregistration
+routines.
+
+Note that \clients are allowed to register multiple callbacks for the
+same event.  \DynamoRIO also supports mutiple \clients, each of which
+can register for the same event.  In this case, \DynamoRIO sequences
+event callbacks in reverse order of when they were registered.  In
+other words, the first registered callback receives event notification
+last.  This scheme gives priority to a callback registered earlier,
+since it can override or modify the actions of \clients registered
+later.  Note that \DynamoRIO calls each \client's dr_init() routine
+according to the \client's priority (see \ref multi_client and
+dr_register_client() in the deployment API).
+
+Systems registering multiple callbacks for a single event should be
+aware that \client modifications are visible in subsequent callbacks.
+\DynamoRIO makes no attempt to mitigate interference among callback
+functions.  It is the responsibility of a \client to ensure
+compatibility among its callback functions and the callback functions
+of other \clients.
+
+\Clients can also unregister a callback using the appropriate
+unregister routine (see dr_events.h).  While unusual, it is possible for
+one callback routine to unregister another.  In this case, \DynamoRIO
+still calls routines that were registered before the event.
+Unregistration takes effect before the next event.
+
+On Linux, an exec (SYS_execve) does NOT result in an exit event, but it
+WILL result in the \client library being reloaded and its dr_init() routine
+being called again.  The system call events can be used for notification of
+SYS_execve.
+
+***************************************************************************
+\section sec_utils Common Utilities
+
+\DynamoRIO provides \clients with a powerful library of utilities for
+custom runtime code transformations.  The interface includes explicit
+support for creating \e transparent \clients.  See the section on
+\ref transparency for a full discussion of the importance of remaining
+transparent when operating in the same process as the application.
+\DynamoRIO provides common resources \clients can use to avoid reliance on
+shared libraries that may be in use by the application.  The \client should
+only use external resources through \DynamoRIO's own API, through direct
+system calls, or via an external agent in a separate process that
+communicates with the \client (see \ref sec_comm).  Third-party libraries
+can be used if they are linked statically and there is no possibility of
+global resource conflicts (e.g., a third-party library's memory allocation
+must be wrapped: see \ref sec_extlibs).  \if vmsafe For the Probe API,
+these restrictions can be relaxed (see \ref sec_trans_probe). \endif
+
+\DynamoRIO's API provides:
+
+ - Memory allocation: both thread-private (faster as it incurs no
+   synchronization costs) and thread-shared
+ - Thread-local storage
+ - Thread-local stack separate from the application stack
+ - Simple mutexes
+ - File creation, reading, and writing
+ - Address space querying
+ - Application module iterator \if linux (currently Windows only) \endif
+ - Processor feature identification
+\if future - FIXME: try...except, or safe_rw? \endif
+
+See dr_tools.h and dr_proc.h for specifics of each routine.
+
+Another class of utilities provided by \DynamoRIO are structures and
+routines for decoding, encoding, and manipulating IA-32 and
+AMD64 instructions. These are described in \ref sec_IR.
+
+\anchor subsec_forwards
+In addition\if linux, on Windows,\endif \DynamoRIO provides a number
+of utility functions that it fowards to a core Windows system library
+that we believe to be safe for \clients to use:
+
+ - wcstoul
+ - wcstombs
+ - wcstol
+ - wcsstr
+ - wcsspn
+ - wcsrchr
+ - wcspbrk
+ - wcsncpy
+ - wcsncmp
+ - wcsncat
+ - wcslen
+ - wcscspn
+ - wcscpy
+ - wcscmp
+ - wcschr
+ - wcscat
+ - towupper
+ - towlower
+ - toupper
+ - tolower
+ - tan
+ - strtoul
+ - strtol
+ - strstr
+ - strspn
+ - strrchr
+ - strpbrk
+ - strncpy
+ - strncmp
+ - strncat
+ - strlen
+ - strcspn
+ - strcmp
+ - strchr
+ - sscanf
+ - sqrt
+ - sprintf
+ - sin
+ - qsort
+ - pow
+ - memset
+ - memmove
+ - memcpy
+ - memcmp
+ - memchr
+ - mbstowcs
+ - log
+ - labs
+ - isxdigit
+ - iswxdigit
+ - iswspace
+ - iswlower
+ - iswdigit
+ - iswctype
+ - iswalpha
+ - isupper
+ - isspace
+ - ispunct
+ - isprint
+ - islower
+ - isgraph
+ - isdigit
+ - iscntrl
+ - isalpha
+ - isalnum
+ - floor
+ - fabs
+ - cos
+ - ceil
+ - atol
+ - atoi
+ - atan
+ - abs
+ - _wtol
+ - _wtoi64
+ - _wtoi
+ - _wcsupr
+ - _wcsnicmp
+ - _wcslwr
+ - _wcsicmp
+ - _vsnprintf
+ - _ultow
+ - _ultoa
+ - _ui64toa
+ - _toupper
+ - _tolower
+ - _strupr
+ - _strnicmp
+ - _strlwr
+ - _stricmp
+ - _strcmpi
+ - _snwprintf
+ - _snprintf
+ - _memicmp
+ - _memccpy
+ - _ltow
+ - _ltoa
+ - _itow
+ - _itoa
+ - _i64tow
+ - _i64toa
+ - _ftol
+ - _fltused
+ - _chkstk
+ - _aullshr
+ - _aullrem
+ - _aulldiv
+ - _atoi64
+ - _allshr
+ - _allshl
+ - _allrem
+ - _allmul
+ - _alldiv
+ - __toascii
+ - __iscsymf
+ - __iscsym
+ - __isascii
+
+In general, these routines match their standard C library counterparts.  However, be
+warned that some of these may be more limited.  In particular, _vsnprintf
+and _snprintf do not support floating-point values.  \DynamoRIO provides
+its own snprintf() that does support floating-point values, but does
+not support printing wide characters.  When printing floating-point values 
+be sure to \ref sec_trans_floating_point "save the application's floating 
+point state" so as to avoid corrupting it.
+
+***************************************************************************
+\section sec_extlibs Using External Libraries
+
+\Clients are free to use external libraries as long as those libraries
+do not use any global user-mode resources that would interfere with
+the running application.  Furthermore, \clients should link statically
+to all libraries in order to prevent re-entrancy problems (see \ref
+sec_trans_resource).
+
+Most libraries need to allocate and free memory.  For transparency,
+\clients should use \DynamoRIO's internal memory pool.
+
+\if linux
+On Linux, ld provides the -wrap option, which allows us to override
+the C library's memory heap allocation routines with our own.  For
+convenience, \DynamoRIO exports __wrap_malloc(), __wrap_realloc(), and
+__wrap_free() for this purpose.  These routines behave like their C
+library counterparts, but operate on \DynamoRIO's global memory pool.
+Use the -Xlinker flag with gcc to replace the libc routines with
+\DynamoRIO's _wrap routines, e.g.,
+
+\code
+  gcc -Xlinker -wrap=malloc -Xlinker -wrap=realloc -Xlinker -wrap=free ...
+\endcode
+
+\subsection subsec_cpp C++ \Clients
+
+The ability to override the memory allocation routines makes it
+convenient to develop C++ \clients that use the \em new and \em delete
+operators (as long as those operators are implemented using malloc and
+free).  In particular, heap allocation is required to use the C++
+Standard Template Library containers. When developing a C++ \client,
+make sure to link statically to libstdc++.  This is most easily
+accomplished by specifying the path to the static version of the
+library on the gcc command line.  gcc's -print-file-name option is
+useful for discovering this path, e.g.,
+
+\code
+  g++ -print-file-name=libstdc++.a
+\endcode
+
+A full gcc command line for building a C++ \client might look something like
+this: 
+
+\code
+g++ -o my-client.so -I<header dir> \
+  -fPIC -shared -nodefaultlibs \
+  -Xlinker -wrap=malloc -Xlinker -wrap=realloc -Xlinker -wrap=free \
+  `g++ -print-file-name=libstdc++.a` \
+  `g++ -print-file-name=libgcc.a` \
+  `g++ -print-file-name=libgcc_eh.a` \
+  my-client.cpp
+\endcode
+
+See also the stl_test.cpp sample and Makefile provided with this
+documentation.
+
+Be aware that we have successfully built and tested several small C++
+\clients using the gcc toolchain, but have not performed extensive testing.
+Our \clients successfully compile and run linked with the g++ libraries
+included in Red Hat Enterprise Linux 4, but on later distributions the
+pre-built libraries may not work and you may need to build static versions
+from the gcc sources with the appropriate flags (including -fPIC).
+
+We do not recommend that a \client or its libraries invoke their own system
+calls as this bypasses \DynamoRIO's monitoring of changes to the process
+address space and changes to threads or control flow.  Such system calls
+will also not work properly on Linux when using sysenter on some systems.
+If you see an assert to that effect in debug build, try the \ref
+op_sysenter "-sysenter_is_int80" option.
+
+\endif
+
+***************************************************************************
+\section sec_comm Communication
+
+Due to transparency limitations (see \ref transparency),
+\DynamoRIO can only support certain communication channels in and out of the
+target application process.  These include:
+
+ - \DynamoRIO deployment control and runtime options: see \ref sec_deploy
+   and \ref sec_options.  In particular, the deployment API allows users to
+   pass up-front runtime information to the \client.
+ - Nudges: Since polling requires extra threads, and \DynamoRIO tries not
+   to create permanent extra threads (see \ref sec_trans_thread
+   "Thread Transparency"), a mechanism called \e nudges are the preferred mechanism
+   for pushing data into the process. Nudges are used to notify \DynamoRIO
+   that it needs to re-read its options, or perform some other action.
+   \DynamoRIO also provides a custom nudge event that can be used by
+   \clients. See dr_nudge_process() and dr_register_nudge_event(). \if linux
+   Nudges are currently only supported on Windows. \endif
+ - Files can be used to send data out.  An external process can wait on
+   the file.
+
+***************************************************************************
+\section sec_options Fine-Tuning \DynamoRIO: Runtime Parameters
+
+\DynamoRIO's behavior can be fine-tuned using runtime parameters.  Options
+are specified via \c drdeploy.exe or dr_register_process() \if linux on
+Windows and via the \c drdeploy script on Linux\endif.  See \ref sec_deploy.
+
+ - \b -opt_memory: \anchor op_memory
+   Reduce memory usage, but potentially at the cost of performance.  This
+   option can result in memory savings as high as 20%, and usually incurs
+   no noticable performance degradation.  However, it conflicts with the
+   \ref op_enable "-enable_full_api option" and cannot be used with
+   dr_unlink_flush_region().
+
+\if expose_opt_speed
+ - \b -opt_speed: \anchor op_speed
+   By default, \DynamoRIO provides a more straightforward code stream to
+   \clients in lieu of performance optimizations.  This option attempts
+   to obtain higher performance with potential loss of \client simplicity.
+   In particular, unconditional branches and in some cases indirect calls
+   may be elided in basic blocks.  See also \ref sec_limit_perf.
+   Note that dr_insert_mbr_instrumentation() is not supported when -opt_speed
+   is specified.
+\endif
+
+ - \b -stack_size \e \<number\>: \anchor op_stack_size 
+   \DynamoRIO's per-thread stack is limited to 20KB by default (this may
+   seem small, but this is much larger than its size when no \client is
+   present).  This parameter can be used to increase the size; however,
+   larger stack sizes use significantly more memory when targeting
+   applications with hundreds of threads.  The parameter can take a 'K'
+   suffix, and must be a multiple of the page size (4K).  This stack is
+   used \if vmsafe for probe callbacks and by the Code Manipulation
+   API \else by the \endif routines dr_insert_clean_call(),
+   dr_swap_to_clean_stack(), dr_prepare_for_call(),
+   dr_insert_call_instrumentation(), dr_insert_mbr_instrumentation(),
+   dr_insert_cbr_instrumentation(), and dr_insert_ubr_instrumentation().
+   The stack is started fresh for each use, so <em>no persistent state may be
+   stored on it</em>.
+\if vmsafe
+Options available only in Code Manipulation mode and Memory Firewall mode
+(see \ref dr_modes):
+\endif
+ - \b -thread_private: \anchor op_thread_priv
+   By default, \DynamoRIO's code caches are shared across threads.  This
+   option requests code caches that are private to each thread.  For
+   applications with many threads, thread-private code caches use more
+   memory.  However, they can be more efficient, particularly when
+   inserting thread-specific instrumentation.
+
+ - \b -disable_traces:
+   By default, \DynamoRIO builds both a <em>basic block</em> code cache and
+   a <em>trace</em> code cache (see \ref sec_IR).  This option disables
+   trace building, which can have a negative performance impact.
+   When traces are disabled, dr_register_trace_event() has no effect.
+   \DynamoRIO tries to keep traces transparent to a \client who is
+   interested in all code and not only hot code, so there is rarely a
+   reason to disable traces.
+   \if internal_comment
+     if we expose -enable_traces, note that it must be specified BEFORE
+     -thread_private as today it turns on -shared_traces
+   \endif
+
+ - \b -enable_full_api: \anchor op_enable
+   \DynamoRIO's default internal options balance performance with API
+   usability.  A few API functions, such as dr_unlink_flush_region(),
+   are incompatible with this default mode.  \Client users can gain 
+   access to the entire set of API functions with -enable_full_api.
+   Note that this option may result in a small performance degradation.
+
+ - \b -max_bb_instrs: 
+   \DynamoRIO stops building a basic block if it hits this application
+   instruction count limit before hitting control flow or other block
+   termination conditions.  The default value is 1024; lower it if
+   extensive \client instrumentation is running into code cache size
+   limit asserts.
+
+ - \b -max_trace_bbs: 
+   \DynamoRIO will not build a trace with larger than this number of
+   constituent basic block.  The default value is 128; lower it if
+   extensive \client instrumentation is running into code cache size
+   limit asserts.
+
+ - \b -sysenter_is_int80: \anchor op_sysenter
+   This option only applies to Linux.  If sysenter is the system call
+   gateway, \DynamoRIO normally hooks the vsyscall vdso page when it can.
+   This option requests that \DynamoRIO convert sysenter into int 0x80
+   instead.  See \ref sec_extlibs.
+\if pcache
+FIXME: expose -desktop and pcache location configuration
+\endif
+
+\if cache_sizing
+FIXME: users may want control over adaptive wset cache management,
+particularly for thread-private to avoid deletions, but also for shared if
+they want to shrink memory usage
+\endif
+
+ - \b -syntax_intel: \anchor op_syntax_intel
+    This option causes \DynamoRIO to output all disassembly using Intel
+    syntax rather than the default AT&T-style syntax.
+
+\ifnot vmsafe
+ - \b -tracedump_text and \b -tracedump_binary:
+    These options cause \DynamoRIO to output all traces that were created
+    to the log file \e traces-shared.0.TID.html, where \e
+    TID is the thread id of the initial thread; any thread-private traces
+    (see \ref op_thread_priv "-thread_private option") produce per-thread
+    files \e traces.TID.html.
+    Traces are logged whenever they are flushed from the cache (which can
+    be during execution or at the latest at program termination).  The two
+    options select either a text dump or a binary dump.  The text dump
+    takes up considerable room and time to dump, while the binary dump
+    requires more effort to examine.  The binary trace dump format is
+    documented in dr_tools.h, and a sample reader is provided with this
+    distribution.
+
+ - \b -tracedump_origins
+   When selected by itself with neither -tracedump_text nor
+   -tracedump_binary, dumps only a text list of the constituent basic block
+   tags of each trace to the trace log file.  When combined with either of
+   -tracedump_text or -tracedump_binary, adds a full disassembly of the
+   constituent basic blocks to the selected dump.
+\endif
+
+\if profiling
+FIXME PR 225255: profiling options
+\endif
+
+Options controlling notifications from \DynamoRIO:
+
+ - \b -msgbox_mask \e 0xN: \anchor op_msgbox_mask
+   Controls whether \DynamoRIO uses pop-up message boxes when
+   presenting information. \if linux Currently Windows only. \endif
+   The mask takes the following bitfields:
+   - INFORMATION = 0x1
+   - WARNING     = 0x2
+   - ERROR       = 0x4
+   - CRITICAL    = 0x8
+   .
+   dr_messagebox() is not affected by -msgbox_mask. For the
+   provided Windows debug build -msgbox_mask defaults to 0xC.
+
+\attention
+   On Vista most Windows services are currently unable to display
+   message boxes (see \ref limits_vista_service_messagebox
+   "Limitations"). Since these services also don't have an associated
+   console for stderr printing, the \ref op_loglevel "-loglevel"
+   and \ref op_logmask "-logmask" options should be used
+   instead. For the messages that would be displayed by -msgbox_mask,
+   setting any bit in -logmask is sufficient for the message to be
+   included in the logfile.
+
+ - \b -stderr_mask \e 0xN:
+   Parallel to -msgbox_mask, but controls \DynamoRIO's output to standard
+   error.  This option takes the same bitfields as -msgbox_mask.  The API
+   routine dr_is_notify_on() can be used to determine if -stderr_mask is
+   non-zero.  Messages printed to stderr will only be visible for
+   applications that have an attached console.  For the provided Linux
+   debug builds, -stderr_mask defaults to 0xF; for the Linux release
+   builds, its default is 0xE.
+
+\ifnot vmsafe
+Options aiding in debugging:
+
+ - \b -pause_on_error: \anchor op_pause
+   For Linux builds only, this option requests that when \DynamoRIO
+   encounters an assert or crash that it suspend the process so that a
+   debugger can be attached.
+
+ - \b -no_hide: \anchor op_no_hide
+    By default, \DynamoRIO hides itself from the Windows module list, for
+    transparency.  However, this makes it more difficult to debug a process
+    under \DynamoRIO's control.  The option -no_hide turns off this module
+    hiding. \if linux This option is for Windows only. \endif
+\endif
+
+Options available only in the debug build of \DynamoRIO:
+
+\anchor op_loglevel
+ - \b -loglevel \e N:
+    If N is greater than 0, \DynamoRIO prints out a log of its actions.
+    The greater the value of N, the more information \DynamoRIO prints.
+    Useful ranges are from 1 to 6.  Verbosity is set to 0 by default, i.e.,
+    no log written.  All log files are kept in a log directory.  There is
+    one directory per address space per run.  The directories are named \e
+    app.NNN, where \e app is the application name and \e NNN is a number
+    that is incremented with each directory created.  \if Linux On
+    Windows the \else The \endif directories are located by default in
+    a subdirectory \e logs of the \DynamoRIO home directory as
+    specified in the dr_register_process() or \c drdeploy.exe
+    configuration for the target application.  \if linux On Linux the
+    \c drdeploy script allows for setting the logging directory. \endif
+    There is one main log file per directory named
+    \e app.0.TID.html, where \e TID is the thread identifier of the initial
+    thread.  There is also a log file per thread, named \e log.N.TID.html,
+    where \e N is the thread's creation ordinal and \e TID is its thread
+    identifier.  The loglevel may be changed during program execution, but
+    if it began at 0 then it cannot be raised later.  The -logmask
+    parameter can be used to control which \DynamoRIO modules output data
+    to the log files. dr_log() allows the \client to write to the above
+    logfiles.
+
+\anchor op_logmask
+ - \b -logmask \e 0xN:
+    Selects which \DynamoRIO modules print out logging information, at the
+    -loglevel level.  The mask is a combination of the LOG_ bitfields
+    listed in dr_tools.h (#LOG_ALL selects all modules).
+
+ - \b -ignore_assert_list \b '*': \anchor op_ignore_assert
+    Ignores all \DynamoRIO asserts of the form "<file>:1234". * may be
+    replaced by a ; separated lists of individual asserts to ignore
+    "foo.c:333;bar.c:12".
+
+***************************************************************************
+\section sec_debugging Diagnosing and Reporting Problems
+
+When using a complex system like \DynamoRIO, problems can be challenging to
+diagnose.  This section contains some debugging tips.
+
+\subsection sec_diagnosing Debugging Tips
+
+ - \DynamoRIO disables itself when Windows is booted in safe mode (without
+networking).  Thus, if a crash occurs in a Windows service under
+\DynamoRIO, rebooting in safe mode will allow recovery.
+
+ - If the \client library doesn't seem to function for a given process, it is
+   likely that the \client library wasn't loaded due to errors.  
+
+   \par
+   One of the common situations where this happens is when the target 
+   application runs as a different user than the user who created the \client 
+   library.  This results in the application process not having the right 
+   permissions to access the \client library.
+
+   \par 
+   Try running the process under the debug mode of \DynamoRIO (see 
+   dr_register_process()), where diagnostic messages are raised on errors like
+   \client library permissions.  To see all messages, set the notification 
+   options like -msgbox_mask and -stderr_mask options to 0xf (see \ref 
+   sec_options).  This will alert you to the problem.
+
+ - \DynamoRIO asserts of the form "<file>:1234" can be suppressed with
+   the \ref op_ignore_assert "-ignore_assert_list '*'" option. * may
+   be replaced by a ; separated lists of individual asserts to
+   suppress as so "-ignore_assert_list 'foo.c:333;bar.c:12'".
+
+\ifnot vmsafe
+ - A process under control of \DynamoRIO can be executed within a
+   debugger.  On Windows it is best to set the \ref op_no_hide "-no_hide"
+   option so the debugger can see the \DynamoRIO library.
+
+   \par
+   To attach to a process on Windows, use the \ref op_msgbox_mask
+   "-msgbox_mask" option and attach the debugger while the dialog box has
+   paused the application.  On Linux, use the \ref op_pause
+   "-pause_on_error" option to allow attaching a debugger when a problem
+   occurs.  To run an application on Linux under a debugger from the start
+   you can perform the work of the \c drdeploy script by setting the
+   LD_LIBRARY_PATH and LD_PRELOAD environment variables from within the
+   debugger.
+
+   \par
+   For debugging on Windows we recommend using windbg version 6.3.0017
+   (\b not the newer versions 6.4 through 6.8, as they have problems
+   displaying callstacks involving \DynamoRIO code).
+
+   \par
+   Take care when setting breakpoints in an application running under
+   \DynamoRIO.  If the debugger inserts \c int3 it can find its way into
+   the code cache and cause errors as the debugger will not realize that
+   the resulting trap is from a breakpoint.  Use read watchpoints on the
+   code in question instead.
+
+   \par
+   On Windows, if an application invokes OuputDebugString() while under a
+   debugger, \DynamoRIO can end up losing control of the application.
+\endif
+
+ - The \DynamoRIO header files have typedefs that may conflict with other
+   header files wrapped in ifndef DR_DO_NOT_DEFINE_<type> to make it
+   easier to work around such conflicts.
+
+\subsection sec_reporting Reporting Problems
+
+For questions as well as bug reports, send email to <a
+href="mailto:dynamorio-govirtual@vmware.com">dynamorio-govirtual@vmware.com</a>.
+For bug reports, please include a detailed description of the problem (is
+it an application crash?  a \DynamoRIO crash?  a hang?  a debug build
+assert?) and how to reproduce it.
+
+****************************************************************************
+****************************************************************************
+*/
+/* It's good to use separate C comments: we've hit some sort of doxygen
+ * internal buffering error before if one comment gets too long.
+ */
+/**
+***************************************************************************
+***************************************************************************
+\ifnot vmsafe
+\page page_standalone IA-32/AMD64 Disassembly Library
+
+\DynamoRIO can be used as a standalone library for IA-32/AMD64
+disassembly, decoding, encoding, and general instruction manipulation,
+independently of controlling a target application.  When used in this way,
+all aspects of \DynamoRIO's API routines that apply to instrumentation or
+application control are not applicable; however, the full, rich instruction
+set API is enabled.  For further information see the following sections of
+the Code Manipulation API:
+
+ - \ref sec_IR
+ - \ref sec_decode
+
+\section sec_standalone Using \DynamoRIO as a Standalone Library
+
+\DynamoRIO can be used as a regular third-party library for a standalone
+application (instead of a \client that operates on a target program).  In
+order to use the \DynamoRIO API in such a situation, a dummy context must
+be created.  This routine creates such a context, and initializes
+\DynamoRIO for standalone use:
+
+\code dr_standalone_init() \endcode
+
+Note that the returned context cannot be used as the drcontext for a thread
+running under \DynamoRIO control!  It is only for standalone programs that
+wish to use \DynamoRIO as a library of routines for IA-32 instruction
+manipulation or other purposes.
+
+Since the \DynamoRIO library on Windows includes or forwards
+implementations of certain C library routines (see 
+\ref subsec_forwards "C library utilities"), standalone applications
+linking to both \DynamoRIO and the C library may experience linker errors
+when building and floating point problems when running.  To avoid these
+problems, explicitly list the C runtime library on the command line:
+
+\code /link /nodefaultlib libcmt.lib dynamorio.lib \endcode
+
+\DynamoRIO writes to stderr and stdout using raw system calls, which can
+interfere with the buffering of library routines.  When mixing use of
+printf or fprintf with \DynamoRIO output (including not only dr_printf()
+and dr_fprintf() but also passing STDOUT or STDERR to routines like
+disassemble()), you may need to flush between library printing and
+\DynamoRIO printing (e.g., using fflush(stdout)) to ensure that the library
+output is visible.
+
+The binary tracedump reader (\ref sec_ex8) is an example of use of
+\DynamoRIO as a standalone library.
+
+\endif
+
+*/
+
+
+===========================================================================
+INTERNAL NOTES
+
+TODO:
+try "favico.ico" w/o /, once have write access
+this doesn't have ico, try it:
+  http://engweb.vmware.com/~slarsen/dr_api/
+this already has ico:
+  http://vmweb.vmware.com/~slarsen/dr_api/
+
+TODO:
+* move event details to header files
+* remove trace hook from bt.dox?
+* 64-bit?
+* \client limitations: mention arithmetic flag assumptions?
+* do bb mod limits apply to traces too?
+* add notes on actually building VIPA:
+  - need /noentry /nodefaultlib
+    (xref case 202210 on .E if don't have /noentry)
+
+PDF output:
+* vmware-logo.eps is ugly: couldn't get a nice conversion: took from
+  openoffice of VMsafe DevGuide
+* update all eps captions and sizes
+
+PDF and RTF output:
+* intro pages are at end!
+
+RTF output:
+* images in separate files
+* no title (need template?)
+
+TODO:
+See using.html from the MIT release, which had these sections:
+    - What's in the Release Distribution
+    - What Does and Doesn't Work
+    - Using the GUI
+    - Running \DynamoRIO from the Command Line
+    - Environment Variables
+    - Runtime Options
+    - Profiling
+    - Application Interface
+
+--------------------------------------------------
+Complaints about doxygen treeview:
+* Treeview uses frames, which are horrible for bookmarking and history tracking
+* Current page not tracked in treeview
+* Globals list still uses tabs: not that bad though
+
+--------------------------------------------------
+\page => under a separate top-level chapter "Related Pages"
+example:
+  \page page1 A documentation page
+    Leading text.
+    \section sec An example section
+    This page contains the subsections \ref subsection1 and \ref subsection2.
+    For more info see page \ref page2.
+    \subsection subsection1 The first subsection
+    Text.
+    \subsection subsection2 The second subsection
+    More text.
+  \page page2 Another page
+    Even more info.
+
+groups => under a separate top-level chapter "Modules"
+  \defgroup IntVariables Global integer variables
+  \ingroup A
+
+===========================================================================
diff --git a/api/docs/license.dox b/api/docs/license.dox
new file mode 100644
index 000000000..e5667cfed
--- /dev/null
+++ b/api/docs/license.dox
@@ -0,0 +1,69 @@
+/* **********************************************************
+ * Copyright (c) 2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/**
+***************************************************************************
+***************************************************************************
+***************************************************************************
+
+\page page_license License
+
+Copyright (c) 2000-2009 VMware, Inc. licensed under the terms of the BSD.  All other rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ - Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+
+ - Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+ - Neither the name of VMware, Inc. nor the names of its contributors may be
+   used to endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.
+
+*/
+
diff --git a/api/docs/memfirewall.dox b/api/docs/memfirewall.dox
new file mode 100644
index 000000000..5f509803e
--- /dev/null
+++ b/api/docs/memfirewall.dox
@@ -0,0 +1,165 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/**
+***************************************************************************
+***************************************************************************
+\if vmsafe
+\page API_security Memory Firewall API
+
+The Memory Firewall API provides an interface for using and
+customizing the award-winning Memory Firewall software originally
+developed by Determina, Inc.  Memory Firewall is a security technique
+that leverages the \DynamoRIO runtime code manipulation system
+to control what code is allowed to execute ("Restricted Code Origins"
+or RCO) and how said code can be reached ("Restricted Control
+Transfers" or RCT). The former provides protection against code
+injection attacks while the latter provides protection against code
+reuse attacks. Together these types of attacks account for most
+critical exploits seen in the wild.
+
+The Memory Firewall API comes with a built in set of policies and
+exemptions based off of a simplified version of those used in
+Determina's VPS product.  These policies and exemptions are not
+configurable and should be suitable for most applications.  Some
+applications or modules may use code constructs that violate these
+policies and in these cases a \client can be used to exempt these
+violations.  The \client is also given control over the remediation
+taken when a violation occurs.  Most standard windows services 
+(other than winlogon.exe: see \ref limits_winlogon "Limitations") and
+applications shouldn't receive violations in
+and of themselves. However, poorly written 3rd party plugins (such as
+printer drivers in the spoolsv.exe service and plugins to explorer.exe
+and Internet Explorer) can be a source of violations.  Third party
+software that is injected into all processes (such as certain
+anti-virus and anti-cheating products) can also be a source of
+violations.
+
+***************************************************************************
+\htmlonly
+<table width=100% bgcolor="#000000" cellspacing=0 cellpadding=2 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td></td></tr></table></td></tr></table></td></tr></table>
+\endhtmlonly
+\section sec_MF_events Events
+
+The security violation event occurs when a violation of the built in
+policies is detected.  The \client registers a callback routine (or
+hook) to receive notification of this event (see
+dr_register_security_event()).  The hook routine receives seven
+parameters that describe the violation and give control over the
+remediation actions taken :
+
+\code
+void security_violation(void *drcontext, void *source_tag,
+                        app_pc source_pc, app_pc target_pc, 
+                        dr_security_violation_type_t violation,
+                        dr_mcontext_t *mcontext,
+                        dr_security_violation_action_t *action);
+\endcode 
+
+ - \c drcontext is a pointer to the input program's binary translation
+   context. It is critical for correct program execution that the 
+   input program's context remains intact. Thus, the user is not expected
+   to inspect or modify the context and it is passed as an opaque pointer
+   (i.e., <tt>void *</tt>).
+
+ - \c source_tag is the unique tag of the fragment of code in the
+   code cache that was the source of the violations. Each fragment of
+   code within \DynamoRIO's code cache is identified by a unique \e
+   tag which is the application starting address of that block of
+   code.
+
+ - \c source_pc is the address of the source instruction for the violation.
+
+ - \c target_pc is the address of the target instruction for the violation.
+
+ - \c violation is a value from the \c dr_security_violation_type_t enum in
+   dr_events.h that describes the type of violation that occurred
+   including whether the violation is from the RCO or RCT policies.
+   See the enum for a fuller description of the violation types.
+
+ - \c mcontext is the application's context at the time of the
+   violation.  This is an IN/OUT parameter to allow the application's
+   state to be changed as part of the remediation.
+
+ - \c action is an IN/OUT parameter that selects the remediation action
+   to take.  The remediations are drawn from the \c
+   dr_security_violation_action_t enum in dr_events.h and include such
+   actions as continuing as if no violation occurred, throwing an
+   exception at the target, etc.  See the enum for a fuller
+   description of possible remediations. 
+
+***************************************************************************
+\htmlonly
+<table width=100% bgcolor="#000000" cellspacing=0 cellpadding=2 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td></td></tr></table></td></tr></table></td></tr></table>
+\endhtmlonly
+\section sec_MF_utils Utilities
+
+The Memory Firewall API also includes the ability to generate a
+forensics report when a violation is received that includes detailed
+information about the source and target addresses, as well as general
+information on the process and machine as a whole.  The forensics
+report is generated in an xml block described by
+dr_forensics-1.0.dtd.
+
+\code
+void dr_write_forensics_report(void *dcontext, (file_t) file,
+                               dr_security_violation_type_t violation,
+                               dr_security_violation_action_t action,
+                               const char *violation_name);
+\endcode
+
+
+***************************************************************************
+\htmlonly
+<table width=100% bgcolor="#000000" cellspacing=0 cellpadding=2 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td></td></tr></table></td></tr></table></td></tr></table>
+\endhtmlonly
+
+\section sec_MF_examples Examples
+
+An example Memory Firewall \client implementation is provided at <a
+href="samples/MF_moduledb.c">samples/MF_moduledb.c</a>. See \ref
+sec_mf_samples for a description.
+
+\endif
+****************************************************************************
+****************************************************************************
+*/
+
diff --git a/api/docs/probe.dox b/api/docs/probe.dox
new file mode 100644
index 000000000..0ef8765e7
--- /dev/null
+++ b/api/docs/probe.dox
@@ -0,0 +1,101 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/**
+***************************************************************************
+***************************************************************************
+\if vmsafe
+\page API_probe Probe API
+
+The Probe API provides an interface that enables a \client to insert
+instructions at any arbitrary point in the executable instructions of a
+process.  This allows for monitoring and changing program behavior.  The usage
+model for the Probe API is described below.
+
+In dr_init(), a \client should call dr_register_probes() to register (with
+\DynamoRIO) all the points in the current proces at which it would like to
+insert a probe.  A probe insertion point may be a raw virtual address, an
+executable library name and an exported address or an executable library name
+and offset.  For each probe insertion point the \client should provide
+a callback function.  If a probe or its callback function is invalid, the
+registration will faill for that probe (see dr_register_probes() for details).
+
+After successful registration, \DynamoRIO will invoke a probe's callback
+function using an alternate stack whenever that probe's insertion point
+executes.  With the callback, \DynamoRIO passes a pointer argument of type \ref
+dr_mcontext_t (the process's register context at that point of execution).
+Within the callback function, the \client can perform its monitoring and
+control operations using a subset of \DynamoRIO's API routines (e.g., functions
+to collect information about the process, communicate with external processes,
+etc.).  See \ref sec_utils and \ref sec_comm.
+
+When a callback function completes, it returns a code to indicate its
+status and to request that \DynamoRIO perform certain actions on its
+behalf.  Example actions include: sending messages to an event log,
+killing the current thread or process, raising an exception, or
+changing control flow.  These capabilities enable a callback function to have
+more influence over the process's control flow.  Once the callback function
+exits and \DynamoRIO has finished the requested actions, the process continues
+execution.
+
+\Clients can determine the status of each probe by calling
+dr_get_probe_status() at any time during process execution.  \Clients can also
+add, remove or update probes at any time during process execution via
+subsequent calls to dr_register_probes().  A new list of probes completely
+replaces the existing probes.  That is, existing probes not specified in the
+new list are removed from the process.
+
+The flexibility of doing probe registration at any time during execution means
+that probe registration is not a synchronous operation.  In other words, probes
+may not yet be registered when dr_register_probes() returns.  After an update,
+the \client should use dr_get_probe_status() to ascertain the status of the
+requested probe registrations.
+
+Probe updates can be used by the \client as part of an internally or an
+externally triggered event.  An internal event is triggered by the \client 
+itself, based on analysis of process behavior for example.  Here the \client
+might decide to change callback functions to another one or remove a probe or
+add a new one.  An externally triggered event is caused by the Guest Agent via
+dr_nudge_process().  The Guest Agent might have downloaded new versions of some
+callback functions, have obtained data regarding new probes, etc.  It notifies
+the \client which may choose use the new callbacks or probes using
+dr_register_probes().
+
+At process exit all probes are removed automatically.  Probes are removed
+automatically at a few other scenarios too (see dr_register_probes()) for
+details.
+
+\endif
+****************************************************************************
+****************************************************************************
+*/
+
diff --git a/api/docs/release.dox b/api/docs/release.dox
new file mode 100644
index 000000000..e2aa71089
--- /dev/null
+++ b/api/docs/release.dox
@@ -0,0 +1,546 @@
+/* **********************************************************
+ * Copyright (c) 2008-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/**
+***************************************************************************
+***************************************************************************
+***************************************************************************
+
+\page release_notes Release Notes for Version 1.3.1
+
+This section is divided into the following subsections:
+
+ - \ref sec_package
+\ifnot vmsafe
+ - \ref sec_changes
+\endif
+ - \ref sec_limits
+\ifnot vmsafe 
+ - \ref sec_future
+\endif
+
+***************************************************************************
+\htmlonly
+<table width=100% bgcolor="#000000" cellspacing=0 cellpadding=2 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td></td></tr></table></td></tr></table></td></tr></table>
+\endhtmlonly
+\section sec_package Distribution Contents
+
+The following are part of the \DynamoRIO release distribution:
+
+\if profiling
+  FIXME PR 225255: profile library too
+\endif
+
+ - Four different \DynamoRIO libraries: debug and release for each of
+   32-bit and 64-bit. 
+   The debug library enables assertion messages to more easily diagnose
+   API usage errors.
+ - The \DynamoRIO deployment library and command-line utility
+   \c drdeploy.exe, for configuring applications to execute under
+   \DynamoRIO control. \if linux Windows package only. \endif
+ - A utility \c drinject.exe to launch applications under \DynamoRIO
+   control.  \if linux Windows package only. \endif
+ - A utility \c drview.exe for viewing which 32-bit or WOW64 processes
+   are running under \DynamoRIO control.  Note that the tool does
+   not yet support showing the status of 64-bit applications.
+   \if linux Windows package only. \endif
+\if linux
+ - The \c drdeploy script for configuring and running applications under
+   \DynamoRIO on Linux (Linux package only).
+\endif
+ - Header files for the \DynamoRIO APIs.
+ - This documentation.
+ - \ref API_samples "Sample \clients".
+\ifnot vmsafe
+ - A graphical statistics viewer, usable with the \DynamoRIO debug
+   library to see internal statistics as well as custom statistics defined
+   by a \client (see \ref sec_ex7). \if linux Windows package only. \endif
+\endif
+\ifnot vmsafe
+ - A binary tracedump reader, which also functions as a sample
+   \client using \DynamoRIO as a standalone library (see \ref
+   sec_standalone).
+\endif
+
+\ifnot vmsafe
+***************************************************************************
+\htmlonly
+<table width=100% bgcolor="#000000" cellspacing=0 cellpadding=2 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td></td></tr></table></td></tr></table></td></tr></table>
+\endhtmlonly
+\section sec_changes Changes Since Prior Releases
+
+We have re-numbered the \DynamoRIO versions as follows:
+
+ - 1.0.0 = 0.9.6 build 9600
+ - 1.1.0 = 0.9.6 build 9601
+ - 1.2.0 = 0.9.6 build 9602
+
+The current version is 1.3.1 and is backward compatible with the 0.9.6
+releases.  1.3.1 is identical to 1.3.0 but is under a BSD license (see \ref
+page_license).
+
+The changes between version 1.3.0 and version 1.2.0 (0.9.6 9602) are:
+
+ - Version numbering shift to 1.x.y instead of 0.9.6 960x
+ - New system call pre, post, and filter events, and new system call
+   parameter and result access, along with a feature to chain system calls:
+   - dr_register_pre_syscall_event()
+   - dr_register_post_syscall_event()
+   - dr_register_filter_syscall_event()
+   - dr_syscall_get_param()
+   - dr_syscall_set_param()
+   - dr_syscall_set_sysnum()
+   - dr_syscall_get_result()
+   - dr_syscall_set_result()
+   - dr_syscall_invoke_another()
+   - dr_is_wow64()
+ - New signal event for Linux
+   - dr_register_signal_event()
+ - New option \ref op_pause "-pause_on_error", and error messages to
+   stdout by default for release builds, to improve Linux debugging
+ - New routines for memory allocation and memory manipulation:
+   - dr_nonheap_alloc()
+   - dr_nonheap_free()
+   - dr_memory_protect()
+ - New option \ref op_syntax_intel "-syntax_intel" for Intel-style disassembly
+ - New option \ref op_sysenter "-sysenter_is_int80"
+ - The parameter to an application's system call (normally kept in the eax
+   register) can now be freely changed in basic blocks on all platforms
+ - Added support for 64-bit -thread_private
+ - Added corner-case undocumented IA-32 opcode support
+ - Fixed bug running multi-threaded 64-bit Linux apps
+ - Fixed bugs in 64-bit Linux signal handling
+ - Fixed bug running -thread_private debug build
+ - Fixed bug running 32-bit apps on 64-bit Linux on AMD processors
+ - Fixed bug where OS_OPEN_APPEND overwrote instead of appending on Windows
+
+The changes between the 0.9.6 release builds 9602 and 9601 are:
+
+ - Performance improvements for both the base \DynamoRIO system and for
+   \client instrumentation when running on Pentium M, Core, and Core 2
+   processors.
+ - 64-bit bug fixes
+ - Added several convenience routines:
+   - get_register_name() 
+   - reg_to_pointer_sized()
+   - reg_is_gpr()
+   - reg_is_segment()
+   - reg_32_to_8()
+ - Disassembly now expands immed sizes to match operands
+ - Fixed bug in instr_is_nop()
+
+The changes between the 0.9.6 release builds 9601 and 9600 are:
+
+ - The Windows registry key used is now "VMware, Inc." instead of "VMware"
+ - Added large file support (see #DR_FILE_ALLOW_LARGE)
+ - Added support for decoding from a copy of code: decode_from_copy() and
+   disassemble_from_copy().
+ - Changed the default options to favor performance, and added the
+   \ref op_memory "-opt_memory" runtime option to prioritize memory instead.
+
+Release 0.9.6 is \b not backward compatible with prior releases 0.9.1-0.9.5.
+
+The major changes between the 0.9.6 and 0.9.5 releases include 64-bit
+support, multiple \clients, state translation, trace contents, and Linux
+module events and fast system calls:
+
+ - 64-bit applications and \clients are now supported.  This changed
+   several function signatures: 
+   - instr_encode()
+   - decode_next_pc()
+   - decode_sizeof()
+   - decode_eflags_usage()
+   - instr_init()
+   - The binary trace dump format changed.
+   .
+   Several new functions were added:
+   - set_x86_mode()
+   - get_x86_mode()
+   - instr_set_x86_mode()
+   - instr_get_x86_mode()
+   - opnd_create_rel_addr()
+   - opnd_create_far_rel_addr()
+   - opnd_is_rel_addr()
+   - opnd_is_near_rel_addr()
+   - opnd_is_far_rel_addr()
+   - instr_has_rel_addr_reference()
+   - instr_get_rel_addr_target()
+   - instr_get_rel_addr_dst_idx()
+   - instr_get_rel_addr_src_idx()
+   - instr_shrink_to_32_bits()
+   - opnd_shrink_to_32_bits()
+   - reg_32_to_64()
+   - reg_64_to_32()
+   - reg_is_extended()
+   - reg_parameter_num()
+   .
+   To build a 64-bit \client, set the \p X86_64 preprocessor define before
+   including the \DynamoRIO header files, and link with the 64-bit build of
+   \DynamoRIO (for a 32-bit \client, set \p X86_32).
+ - Multiple \clients are now supported.  This changed the signatures of
+   dr_init(), dr_get_options(), and dr_get_client_path().  It also changed
+   how \clients are deployed and nudged, and how events are unregistered:
+   explicit unregistration routines are now used.
+ - State translation in the presence of \clients is now fully supported.
+   This changed the signature for the basic block and trace event callbacks
+   (see dr_register_bb_event() and dr_register_trace_event()), added a
+   new event dr_register_restore_state_event(), and added new functions
+   instr_set_translation(), instr_set_meta_no_translation(), and INSTR_XL8().
+ - The trace callback (#dr_register_trace_event()) now presents original
+   application code to the \client, rather than code that has already
+   been modified for execution in the code cache.  The \client also has
+   flexibility in which instrumentation is included from constituent
+   basic blocks added to a trace (the \p for_trace parameter: see
+   #dr_register_bb_event()).
+ - Fast system calls (syscall and sysenter) are now supported on Linux.
+ - Module load/unload events and module iteration are now supported on Linux.
+ - System calls for 32-bit applications on 64-bit kernels are no longer
+   hidden by vsyscall; related functions were removed:
+   instr_is_lol64_syscall(), instr_is_32on64_syscall().
+ - Due to both 64-bit support and full WOW64 (32-bit applications on 64-bit
+   Windows) support, xmm registers were added to dr_mcontext_t, and a
+   new function dr_mcontext_xmm_fields_valid() was added.
+ - Far instr-type operands are now supported: opnd_create_far_instr(),
+   opnd_is_near_instr(), opnd_is_far_instr().
+ - Miscellaneous new functions were added:
+   - instr_convert_short_meta_jmp_to_long()
+   - instr_reads_from_reg()
+   - LOCK()
+   - OPND_CREATE_INT_32OR8()
+   - OPND_CREATE_INT_16OR8()
+   - instrlist_meta_append()
+   - dr_using_all_private_caches()
+ - The type of nudge arguments was changed from (void *) to uint64.
+ - The signature of dr_lookup_module() changed.  It no longer has an
+   IMAGE_SECTION_HEADER out argument.  See dr_lookup_module_section()
+   for that functionality.
+ - The disassemble-from-address routines now return NULL when pointed at
+   invalid instructions (matching the decode routines).
+ - The routines to access \DynamoRIO tls slots from the cache were changed.
+   dr_insert_write_temp_tls was eliminated in favor of a generalized #dr_save_reg
+   with more flexibility on which slot can be used.  #dr_save_arith_flags was
+   similarly generalized.  Slots are now guaranteed to remain valid until the
+   next non-meta instruction allowing access to saved registers during clean
+   calls via #dr_read_saved_reg and #dr_write_saved_reg.  #dr_insert_mbr_instrumentation
+   also now requires caller to specify the spill slot to be clobbered
+   which must be less than dr_max_opnd_accessible_spill_slot().
+
+The major changes between the 0.9.5 and 0.9.4 releases are:
+
+ - The basic block hook (\ref sec_events_bb) passes completely unmodified
+   application code to the \client (no mangling or elision).
+ - The old \client hook exports have been replaced with an explicit event
+   registration model.
+ - Instruction representation simplification: the \client only sees fully
+   decoded instructions.
+ - Easier-to-use clean calls (see #dr_insert_clean_call).
+ - Library support (-wrap on linux, ntdll on windows: see \ref sec_extlibs
+   and \ref sec_utils).
+ - Some features have been removed (these are easily implemented by a
+   \client): there is no more edge-counting profile build, no more
+   custom exit stubs, and no more prefixes.
+ - Infrastructure improvements:
+   - Thread-shared caches (can still request thread-private: \ref
+     op_thread_priv "-thread_private option").  Note that there are
+     some subtle changes stemming from using thread-shared: in particular,
+     note that the context passed to the deletion event may be NULL
+     (see #dr_register_delete_event).
+   - Direct access to TLS slots (#dr_save_reg, dr_insert_write_temp_tls,
+     #dr_insert_write_tls_field).
+ - Module events (#dr_register_module_load_event),
+   module iteration (#dr_module_iterator_start, #dr_lookup_module, etc.),
+   and memory querying (#dr_query_memory, #dr_virtual_query).
+ - The full API is now documented in html and pdf for easy browsing.
+ - Numerous type and routine name changes.
+
+\endif
+
+***************************************************************************
+\if future
+FIXME not done yet
+
+  - see \clients/mit_release/using.html: what's included, etc.
+  - DRcontrol: specify target app and runtime options (TODO: case 10647:
+                                                       external vs internal ops) 
+  - GUI: just stats viewer, or control too?
+  - DRview
+  - native_exec (TODO: specify on module load; TODO: case 10846 eliding problems)
+  - multiple \clients (TODO: multiple non-mutating, one mutating RIO \client;
+                      multiple probe \clients w/ chaining; one prog shep \client?)
+  - detach (TODO: ask each \client to clean up)
+  - updating: load new library  
+
+Options for communication in and out of the process are discussed in \ref
+sec_comm.
+\endif
+
+
+***************************************************************************
+\htmlonly
+<table width=100% bgcolor="#000000" cellspacing=0 cellpadding=2 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td></td></tr></table></td></tr></table></td></tr></table>
+\endhtmlonly
+\section sec_limits Limitations
+
+\subsection sec_limit_clients \Client Limitations
+The most important limitation on a \client is that it remain transparent.
+This is described fully in \ref transparency.
+Here we summarize the key points to remain transparent:
+  - The \client should be a self-contained library with linkage to nothing
+other than \DynamoRIO libraries, unless those other libraries are
+statically linked and have all of their own resource usage made transparent
+(see \ref sec_extlibs).  Linkage to other shared libraries is likely to
+result in using the libraries used by the process being controlled, thereby
+leading to unpredictable re-entrancy problems.
+  - Currently communication API provided by \DynamoRIO is limited to file
+I/O.  The \client should only use this and not any system libraries
+directly to avoid non-deterministic and unpredictable re-entrancy problems.
+This is because the process can be in the middle of using them.  If a
+\client has to avail of more system services, it can include its own system
+call entry functions, statically link against these or request that these
+be added to \DynamoRIO.  Non system resource related functionality like
+string manipulation, etc. can be statically linked to a \client or by
+defining users' own routines.
+\if vmsafe
+  - The security agent on the security monitor VM has no direct
+    relationship with \DynamoRIO.  However, the VMsafe solution provider
+    can architect their own communication channel between the security
+    agent on the security monitor VM and the \client by using an agent on
+    the \client's VM as an intermediary.
+ \anchor limits_winlogon
+  - There is a known false positive in the anti-piracy DRM code in the
+    Windows Genuine Advantage component of winlogon.exe.  This code
+    performs deliberate obfuscation and violations of standard coding
+    behavior.
+\endif
+\if vmsafe
+  - The \client can run in only one mode at a time, which cannot be changed during the
+execution of the process.  Mode change can be done only on a subsequent
+application restart.
+  - The \client should call only the API corresponding to the mode it was started
+in.  Mixing API from different modes can lead to unspecified results and is
+not supported.
+  - Probe API allows only one registration of probes, which must occur in
+    dr_init().  
+  - Probes can be specified only with library names and offsets from the base
+    of the library in memory (\ref DR_PROBE_ADDR_LIB_OFFS).
+  - When a probe callback finishes execution it can't request \DynamoRIO to
+    perform any actions like logging, control flow changes, etc.
+  - Probe locations can't include system call wrappers in ntdll.dll, i.e.,
+    ntdll!Nt* routines can't be probe locations.
+\endif
+
+\subsection sec_limit_platforms Platform Limitations
+  - \DynamoRIO currently supports the following NT-based 32-bit
+    Windows systems : NT (all service packs), 2000 (all service
+    packs), XP (32-bit, service packs 0-3), 2003 (32-bit, service
+    packs 0-2), and Vista (32-bit, service packs 0-1). It does not
+    support Windows 95, 98, ME or Server 2008.
+  - This release of \DynamoRIO has limited support for running 32-bit
+    Windows applications on the following 64-bit Windows operating
+    systems : Windows XP Professional x64 (service pack 2), Windows 2003
+    x64 (service pack 2), Vista x64 (service packs 0-1). Only the 32-bit
+    code will be seen, and children processes created will not be injected
+    into.  On 64-bit Windows 32-bit applications are automatically run
+    through the Windows-On-Windows or WOW64 emulator so system call and
+    indirect call processing \clients must be aware of
+    #instr_is_wow64_syscall().
+  - This release of \DynamoRIO has experimental support for running
+    64-bit Windows applications, using the 64-bit \DynamoRIO build, on
+    the following 64-bit Windows systems: Windows XP Professional x64
+    (service pack 2), Windows 2003
+    x64 (service pack 2), Vista x64 (service packs 0-1). It does not
+    support any Itanium based Windows systems.  For 64-bit
+    applications \clients must be compiled for 64-bit with a
+    fixed preferred address in the lower 2GB of the address space (e.g.,
+    with the link options /base:"0x72000000" /FIXED), and multiple
+    \clients with conflicting base addresses aren't supported.  See <a
+    href="samples/Makefile">samples/Makefile</a> for an example.  The
+    target application may fail to initialize properly or load
+    \DynamoRIO if there is an address conflict with the preferred
+    bases of the \client or \DynamoRIO dlls, but given the choice of
+    bases and the fact that \DynamoRIO is loaded quite early we expect
+    this to only very rarely be a problem.  This limitation will be
+    addressed in future releases.  This release also does not support
+    thread-private caches with 64-bit applications.
+  - This release does not fully support applications that mix 32-bit and
+    64-bit code.  Future releases will support such mixtures.
+  - When running a cygwin application under control of \DynamoRIO,
+    stderr and stdout output from \DynamoRIO or its \clients may not
+    be visible.
+\if linux
+  - This release of \DynamoRIO has limited support for most 32-bit Linux
+    distributions running on Intel-compatible hardware, including
+    Ubuntu 7.04, Ubuntu 8.04, and Fedora 9.  It may not support
+    applications using segment registers beyond the standard threading
+    library usage.  Future releases will match the Windows version in
+    comprehensive support.
+  - This release of \DynamoRIO has limited support for running 32-bit
+    Linux applications on 64-bit Linux operating systems on
+    AMD64-compatible hardware.
+  - This release of \DynamoRIO has experimental support for running 64-bit
+    Linux applications, using the 64-bit \DynamoRIO build.  Similarly to
+    64-bit Windows, \clients must set a preferred base address in the lower
+    2GB of the address space, and multiple \clients with conflicting base
+    addresses aren't supported.  See <a
+    href="samples/Makefile">samples/Makefile</a> for an example.
+    This limitation will be addressed in future releases.
+\endif
+
+\subsection sec_limit_perf Performance Limitations
+
+  - In order to present a more straightforward code stream to \clients,
+    \DynamoRIO has several optimizations disabled in this release.
+    System-call-heavy applications are the ones most likely to be affected.
+    Future releases may allow \clients to choose performance versus
+    visibility.  This release does provide the \ref op_memory "-opt_memory option"
+    to enable prioritizing memory usage.
+  - The performance \if vmsafe of Code Manipulation Mode and Memory 
+    Firewall Mode \endif when starting up large desktop applications
+    may be noticeably worse than native.  Upcoming releases will
+    address this issue. \if vmsafe Note that Probe Mode incurs
+    significantly less overhead than the other modes. \endif
+  - The performance \if vmsafe of Code Manipulation Mode \endif when
+    running  Java, .Net, Flash or similar managed execution
+    applications can be noticeably worse then native.  This can
+    include applications that load these components as in-process
+    plugins (such as Internet Explorer). Upcoming releases will
+    address this issue.  \if vmsafe Probe Mode and Memory Firewall
+    Mode are not affected. \endif
+  - When using \if vmsafe Code Manipulation Mode or Memory Firewall
+    Mode \else \DynamoRIO \endif on all or many processes on a system
+    simultaneously, memory usage may become a factor.  Upcoming
+    releases will address this issue.  \if vmsafe Note that Probe Mode
+    incurs significantly less overhead than the other modes. \endif
+
+\subsection sec_limit_deploy Deployment Limitations
+
+  - The dr_config.lib library is not multi-thread safe.  Users of the
+    library should ensure that no more then one thread accesses the
+    library at a time.
+  - Other installed software that uses hooks may not always be
+    interoperable with \DynamoRIO.
+  - Other installed software may conflict with \DynamoRIO's use of the
+    \c \\HKLM\\SOFTWARE\\Microsoft\\Windows\\WindowsNT\\CurrentVersion\\AppInit_DLLs
+    registry key (only very rarely an issue).
+  - Cygwin processes do not work with \DynamoRIO due to cygwin's
+    implementation of fork being incompatible with \DynamoRIO.
+  - Though \DynamoRIO supports running on Windows NT, the provided
+    installer will only run on Windows 2000 and higher.  After
+    installation on a 2000 or higher machine the \ref sec_package may
+    be copied to the NT machine for use there.
+  - On Windows NT, a reboot is required after the initial
+    dr_register_process() in order for \DynamoRIO to take control of any
+    applications.  The reboot will not change or undo the registration.
+  - A Windows application that does not statically link with
+    user32.dll will not be run under control of \DynamoRIO unless its
+    parent process (typically cmd.exe or explorer.exe, for manually
+    launched applications) is already under \DynamoRIO control
+    \if vmsafe (the parent can be in any mode)\endif or the drinject.exe
+    utility is used to launch the application. Only some small
+    non-graphical applications do not link with user32.dll.
+  - When invoking an application in any way that is not from a parent
+    process under \DynamoRIO control, \DynamoRIO takes control a little
+    later and in some rare cases the application has already created a new
+    thread.  This is in violation of the Windows specficiations, but cygwin
+    processes do this.  This new thread will not be under \DynamoRIO
+    control and can cause a variety of problems.  In such cases it is best
+    to invoke from a parent process under \DynamoRIO control.  However, for
+    32-bit applications on 64-bit Windows operating systems, \DynamoRIO
+    takes over later even from the parent.  Future releases will address
+    this limitation.
+  - This release does not support running some Windows services under
+    control of \DynamoRIO: System, smss.exe, csrss.exe, and protected
+    processes on Windows Vista.
+  - This release does not support nudging 64-bit processes.
+\anchor limits_vista_service_messagebox
+  - On Windows Vista most services are run in a separate session as a
+    security feature.  This means that neither \DynamoRIO nor its
+    \client will be able to display a messagebox when running in said
+    services (they have no associated visible window station). See
+    dr_messagebox().
+  - On Windows Vista \if vmsafe in Code Manipulation Mode \endif the code from
+    certain dlls (found mostly in services) and the code subsequently
+    called from those dlls is run natively and is not visible to the
+    instrumentation APIs.  This only applies to dlls that have a .pexe
+    section (only 13 dlls have such a section is a standard Vista
+    install) which seems to be associated with a likely obfuscation
+    method involving kernel side components that this release has no
+    other workaround for.
+\if linux
+\anchor limits_linux_preload
+  - On Linux, the only provided deployment method in this release (aside
+    from the \ref sec_startstop "app_start() app_stop()" interface, which
+    requires soure code modification) is LD_PRELOAD.  This means that
+    static binaries cannot be injected into.  Also, suid-root binaries
+    disallow absolute paths in LD_PRELOAD and ignore LD_LIBRARY_PATH, so
+    alternate methods of specifying the path are needed there.  One method
+    is to place libdrpreload.so's full path in /etc/ld.so.preload and copy
+    libdynamorio.so to /usr/lib or some other system search directory.
+\endif
+
+\ifnot vmsafe 
+***************************************************************************
+\htmlonly
+<table width=100% bgcolor="#000000" cellspacing=0 cellpadding=2 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td><table width=100% bgcolor="#0000C0" cellspacing=0 cellpadding=1 border=0>
+  <tr><td></td></tr></table></td></tr></table></td></tr></table>
+\endhtmlonly
+\section sec_future Plans for Future Releases
+
+We hope to include the following major features in future releases:
+
+ - Persistent and process-shared code caches
+ - Auto-inlining of instrumentation callouts
+ - \Client threads
+
+We also plan to add a number of secondary features, including:
+
+ - More flexible (earlier, or later via attach) Linux injection
+ - Full control over trace building
+ - \Client exception event for \client fault recovery/remediation
+
+Send comments or requests to <a
+href="mailto:dynamorio-govirtual@vmware.com">dynamorio-govirtual@vmware.com</a>.
+\endif
+
+*/
+
diff --git a/api/docs/samples.dox b/api/docs/samples.dox
new file mode 100644
index 000000000..d5c77ef1c
--- /dev/null
+++ b/api/docs/samples.dox
@@ -0,0 +1,526 @@
+/* **********************************************************
+ * Copyright (c) 2007-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/**
+\page API_samples Sample Use Cases
+
+We provide several examples to illustrate how the \DynamoRIO API is used to
+build a \DynamoRIO \client.
+
+\if vmsafe
+\section probe_samples Probe API Examples
+
+Not available yet.
+\endif
+
+\if vmsafe
+\anchor sec_mf_samples
+\section mf_samples Memory Firewall Examples
+
+An example Memory Firewall \client implementation is provided at <a
+href="samples/MF_moduledb.c">samples/MF_moduledb.c</a> (see that file
+for additional documentation). The MF_moduledb sample \client reads in
+a list of module names and options from a config file and exempts security
+violations associated with those modules based on the type of
+violation and the options in the file. In this manner violations from
+dlls known to have certain types of false positives can be avoided.
+It also supports dynamically changing its exemptions with a nudge
+handler that triggers re-reading the config file.
+
+The MF_moduledb sample \client is provided with a configuration file <a
+href="samples/MF_moduledb-sample.config">samples/MF_moduledb-sample.config</a>
+and a violation generating executable <a
+href="samples/VIPA_test.exe">samples/VIPA_test.exe.c</a> for easy
+testing. To use :
+
+Use drdeploy.exe to configure VIPA_test.exe to run under security_api mode with
+the appropriate options.
+drdeploy.exe -reg VIPA_test.exe -root \<root path\> -mode security_api -client \<path
+to MF_moduledb.dll\> -ops "-client_ops \<path to MF_moduledb-sample.config\>"
+
+Then run VIPA_test.exe. Clicking on the heap attack button should produce messages
+that a potential security violation is being allowed.  Clicking on the stack attack
+button should produce a message that a potential security violation is being blocked
+by killing the process.
+
+Use 'drdeploy.exe -nudge VIPA_text.exe' to nudge the process to re-read the
+configuration file.
+
+\endif
+
+\section bt_examples Code Manipulation API Examples
+
+
+********************
+\subsection sec_ex1 Instruction Counting
+
+We now illustrate how to use the above API to implement a simple
+instrumentation \client for counting the number of executed call and
+return instructions in the input program.  Full code for this example is
+in the file <a href="samples/countcalls.c">samples/countcalls.c</a>.
+
+The \client maintains set of three counters: num_direct_calls,
+num_indirect_calls, and num_returns to count three different
+types of instructions during execution.  It keeps both thread-private
+and global versions of these counters.  The \client initializes
+everything by supplying the following dr_init routine:
+
+\code
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+  /* register events */
+  dr_register_exit_event(event_exit);
+  dr_register_thread_init_event(event_thread_init);
+  dr_register_thread_exit_event(event_thread_exit);
+  dr_register_bb_event(event_basic_block);
+
+  /* make it easy to tell, by looking at log file, which \client executed */
+  dr_log(NULL, LOG_ALL, 1, "\Client 'countcalls' initializing\n");
+}
+\endcode
+
+The \client provides an event_exit routine that displays the final 
+values of the global counters as well as a thread_exit routine that
+shows the counter totals on a per-thread basis.
+
+The \client keeps track of each thread's instruction counts separately. 
+To do this, it creates a data structure that will be separately allocated 
+for each thread:
+
+\code
+typedef struct {
+  int num_direct_calls;
+  int num_indirect_calls;
+  int num_returns;
+} per_thread_t;
+\endcode
+
+Now the thread hooks are used to initialize the data structure and to
+display the thread-private totals :
+
+\code
+static void event_thread_init(void *drcontext)
+{
+  /* create an instance of our data structure for this thread */
+  per_thread *data = (per_thread *)
+    dr_thread_alloc(drcontext, sizeof(per_thread));
+  /* store it in the slot provided in the drcontext */
+  dr_set_tls_field(drcontext, data);
+  data->num_direct_calls = 0;
+  data->num_indirect_calls = 0;
+  data->num_returns = 0;
+  dr_log(drcontext, LOG_ALL, 1, "countcalls: set up for thread %d\n",
+         dr_get_thread_id(drcontext));
+}
+
+static void event_thread_exit(void *drcontext)
+{
+  per_thread *data = (per_thread *) dr_get_tls_field(drcontext);
+
+  ... // string formatting and displaying
+
+  /* clean up memory */
+  dr_thread_free(drcontext, data, sizeof(per_thread));
+}
+\endcode
+
+The real work is done in the basic block hook.  We simply look for
+the instructions we're interested in and insert an increment of the
+appropriate thread-local and global counters, remembering to save the
+flags, of course.  This sample has separate paths for incrementing the 
+thread private counts for shared vs. thread-private caches (see the 
+-thread_private option) to illustrate the differences in targeting
+for them.  Note that the shared path would work fine with private
+caches.
+
+\code
+static void
+insert_counter_update(void *drcontext, instrlist_t *bb, instr_t *where, int offset)
+{
+  /* Since the inc instruction clobbers 5 of the arithmetic eflags,
+   * we have to save them around the inc. We could be more efficient
+   * by not bothering to save the overflow flag and constructing our
+   * own sequence of instructions to save the other 5 flags (using
+   * lahf) or by doing a liveness analysis on the flags and saving
+   * only if live.
+   */
+  dr_save_arith_flags(drcontext, bb, where, SPILL_SLOT_1);
+
+  /* Increment the global counter using the lock prefix to make it atomic
+   * across threads. It would be cheaper to aggregate the thread counters
+   * in the exit events, but this sample is intended to illustrate inserted
+   * instrumentation.
+   */
+  instrlist_meta_preinsert(bb, where, LOCK(INSTR_CREATE_inc
+    (drcontext, OPND_CREATE_ABSMEM(((byte *)&global_count) + offset, OPSZ_4))));
+
+  /* Increment the thread private counter. */
+  if (dr_using_all_private_caches()) {
+    per_thread_t *data = (per_thread_t *) dr_get_tls_field(drcontext);
+    /* private caches - we can use an absolute address */
+    instrlist_meta_preinsert(bb, where, INSTR_CREATE_inc(drcontext, 
+        OPND_CREATE_ABSMEM(((byte *)&data) + offset, OPSZ_4)));
+  } else {
+    /* shared caches - we must indirect via thread local storage */
+    /* We spill xbx to use a scratch register (we could do a liveness
+     * analysis to try and find a dead register to use). Note that xax
+     * is currently holding the saved eflags. */
+    dr_save_reg(drcontext, bb, where, REG_XBX, SPILL_SLOT_2);
+    dr_insert_read_tls_field(drcontext, bb, where, REG_XBX);
+    instrlist_meta_preinsert(bb, where,
+        INSTR_CREATE_inc(drcontext, OPND_CREATE_MEM32(REG_XBX, offset)));
+    dr_restore_reg(drcontext, bb, where, REG_XBX, SPILL_SLOT_2);
+  }
+
+  /* restore flags */
+  dr_restore_arith_flags(drcontext, bb, where, SPILL_SLOT_1);
+}
+
+static dr_emit_flags_t
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating)
+{
+  instr_t *instr, *next_instr;
+
+  ... // some logging
+
+  for (instr = instrlist_first(bb); instr != NULL; instr = next_instr) {
+    /* grab next now so we don't go over instructions we insert */
+    next_instr = instr_get_next(instr);
+
+    /* instrument calls and returns -- ignore far calls/rets */
+    if (instr_is_call_direct(instr)) {
+      insert_counter_update(drcontext, bb, instr,
+                            offsetof(per_thread_t, num_direct_calls));
+    } else if (instr_is_call_indirect(instr)) {
+      insert_counter_update(drcontext, bb, instr,
+                            offsetof(per_thread_t, num_indirect_calls));
+    } else if (instr_is_return(instr)) {
+      insert_counter_update(drcontext, bb, instr,
+                            offsetof(per_thread_t, num_returns));
+    }
+  }
+
+  ... // some logging
+
+  return DR_EMIT_DEFAULT;
+}
+
+\endcode
+
+\par Building the Example
+
+To build the \client instrumentation application, the
+\client program file (say instrcalls.c) should include the main header file:
+
+\code
+#include "dr_api.h"
+\endcode
+
+The \client's target operating system and architecture must
+be specified by setting pre-processor defines before including the
+\DynamoRIO header files.  The appropriate library must then be linked
+with.  The define choices are:
+
+-# \p WINDOWS or \p LINUX
+-# \p X86_32 or \p X86_64
+
+We can then build the \client shared library in Windows:
+
+\code
+> cl instrcalls.c /I$DYNAMORIO_HOME/include /link /NODEFAULTLIB /NOENTRY /libpath:$DYNAMORIO_HOME/bin dynamorio.lib /dll /out:instrcalls.dll 
+\endcode
+
+Be sure to include /NODEFAULTLIB and /NOENTRY!  Not including them will
+result in an unsafe \client library with the potential to deadlock or
+crash.  See \ref transparency for further information on transparency.
+
+\if linux
+To build in Linux:
+
+\code
+> gcc -shared -nostartfiles -DLINUX -I$DYNAMORIO_HOME/include instrcalls.c -o instrcalls.so 
+\endcode
+\endif
+
+The result is a shared library instrcalls.dll. To invoke the \client library,
+follow the instructions under \ref sec_deploy.
+
+
+********************
+\subsection sec_ex2 Instruction Profiling
+
+The next example shows how to use the provided control flow
+instrumentation routines, which allow more sophisticated profiling than
+simply counting instructions.  Full code for this example is
+in the file <a href="samples/instrcalls.c">samples/instrcalls.c</a>.
+
+As in the previous example, the \client is interested in direct and
+indirect calls and returns.  The \client wants to analyze the target
+address of each dynamic instance of a call or return.  For our example,
+we simply dump the data in text format to a separate file for each
+thread.  Since FILE cannot be exported from a DLL on Windows, we use the
+\DynamoRIO-provided file_t type that hides the distinction between FILE and
+HANDLE \if linux to allow the same code to work on Linux and Windows
+\endif.  We make use of the thread initialization and exit routines to
+open and close the file.  We store the file for a thread in the user
+slot in the drcontext.
+
+\code
+static void event_thread_init(void *drcontext)
+{
+  /* we're going to dump our data to a per-thread file */
+  file_t f;
+  char logname[512];
+
+  ... // filename generation
+
+  f = dr_open_file(fname, false/*write*/);
+  DR_ASSERT(f != INVALID_FILE);
+
+  /* store it in the slot provided in the drcontext */
+  dr_set_tls_field(drcontext, (void *)f);
+
+  ... // logging
+}
+
+static void event_thread_exit(void *drcontext)
+{
+  file_t f = (file_t)(ptr_uint_t) dr_get_tls_field(drcontext);
+  dr_close_file(f);
+}
+\endcode
+
+The basic block hook inserts a call to a procedure for each
+type of instruction, using the API-provided
+dr_insert_call_instrumentation and dr_insert_mbr_instrumentation
+routines, which insert calls to procedures with a certain signature.
+
+\code
+static dr_emit_flags_t 
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating)
+{
+  instr_t *instr, *next_instr;
+
+  ... // logging
+
+  for (instr = instrlist_first(bb); instr != NULL; instr = next_instr) {
+    next_instr = instr_get_next(instr);
+    if (!instr_opcode_valid(instr))
+        continue;
+    /* instrument calls and returns -- ignore far calls/rets */
+    if (instr_is_call_direct(instr)) {
+        dr_insert_call_instrumentation(drcontext, bb, instr, (app_pc)at_call);
+    } else if (instr_is_call_indirect(instr)) {
+        dr_insert_mbr_instrumentation(drcontext, bb, instr, (app_pc)at_call_ind,
+                                      SPILL_SLOT_1);
+    } else if (instr_is_return(instr)) {
+        dr_insert_mbr_instrumentation(drcontext, bb, instr, (app_pc)at_return,
+                                      SPILL_SLOT_1);
+    }
+  }
+  return DR_EMIT_DEFAULT;
+}
+\endcode
+
+These procedures look like this :
+
+\code
+static void
+at_call(app_pc instr_addr, app_pc target_addr)
+{
+  file_t f = (file_t)(ptr_uint_t) dr_get_tls_field(dr_get_current_drcontext());
+  dr_mcontext_t mc;
+  dr_get_mcontext(dr_get_current_drcontext(), &mc, NULL);
+  dr_fprintf(f, "CALL @ "PFX" to "PFX", TOS is "PFX"\n",
+             instr_addr, target_addr, mc.xsp);
+}
+
+static void
+at_call_ind(app_pc instr_addr, app_pc target_addr)
+{
+  file_t f = (file_t)(ptr_uint_t) dr_get_tls_field(dr_get_current_drcontext());
+  dr_fprintf(f, "CALL INDIRECT @ "PFX" to "PFX"\n", instr_addr, target_addr);
+}
+
+static void
+at_return(app_pc instr_addr, app_pc target_addr)
+{
+  file_t f = (file_t)(ptr_uint_t) dr_get_tls_field(dr_get_current_drcontext());
+  dr_fprintf(f, "RETURN @ "PFX" to "PFX"\n", instr_addr, target_addr);
+}
+\endcode
+
+The address of the instruction and the address of its target are both
+provided.  These routines could perform some sort of analysis based on
+these addresses.  In our example we simply print out the data.
+
+
+********************
+\subsection sec_ex3 Modifying Existing Instrumentation
+
+In this example, we show how to update or replace existing
+instrumentation after it executes.  This ability is useful for \clients
+performing adaptive optimization.  In this example, however, we are
+interested in recording the direction of all conditional branches, but
+wish to remove the overhead of instrumentation once we've gathered
+that information.  This code could form part of a dynamic CFG builder,
+where we want to observe the control-flow edges that execute at
+runtime, but remove the instrumentation after it executes.
+
+While \DynamoRIO supports direct fragment replacement, another method
+for re-instrumentation is to flush the fragment from the code cache
+and rebuild it in the basic block event callback.  In other words, we
+take the following approach:
+
+-# In the basic block event callback, insert separate instrumentation
+   for the taken and fall-through edges.
+-# When the basic block executes, note the direction taken and flush
+   the fragment from the code cache.
+-# When the basic block event triggers again, insert instrumentation
+   only for the unseen edge.  After both edges have triggered, remove
+   all instrumentation for the cbr.
+
+We insert separate clean calls for the taken and fall-through cases.
+In each clean call, we record the observed direction and immediately
+flush the basic block using dr_flush_region().  Since that routine
+removes the calling block, we redirect execution to the target or
+fall-through address with dr_redirect_execution().  The file <a
+href="samples/cbr.c">samples/cbr.c</a> contains the full code for this
+sample.
+
+********************
+\subsection sec_ex4 Optimization
+
+For the next example we consider a \client application for a simple
+optimization. The optimizer replaces every increment/decrement operation
+with a corresponding add/subtract operation if running on a Pentium 4,
+where the add/subtract is less expensive.  For optimizations, we are
+less concerned with covering all the code that is executed; on the
+contrary, in order to amortize the optimization overhead, we only want
+to apply the optimization to hot code.  Thus, we apply the optimization
+at the trace level rather than the basic block level.  Full code for
+this example is in the file <a
+href="samples/inc2add.c">samples/inc2add.c</a>.
+
+
+********************
+\subsection sec_ex5 Custom Tracing
+
+This example demonstrates the custom tracing interface. It changes
+\DynamoRIO's tracing behavior to favor making traces that start at a call
+and end right after a return.  It demonstrates the use of both custom trace
+API elements :
+
+\code
+int query_end_trace(void *drcontext, void *trace_tag, void *next_tag);
+bool dr_mark_trace_head(void *drcontext, void *tag);
+\endcode
+
+Full code for this example is in the file <a
+href="samples/inline.c">samples/inline.c</a>.
+
+
+********************
+\subsection sec_ex6 Use of Floating Point Operation in a \Client
+
+Because saving the floating point state is very expensive, \DynamoRIO seeks 
+to do so on an as needed basis.  If a \client wishes to use floating point
+operations it must save and restore the application's floating point state
+around the usage.  For an inserted clean call out of the code cache, this
+can be conveniently done using dr_insert_clean_call() and passing true for
+the save_fpstate parameter.  It can also be done explicitly using these
+routines:
+
+\code
+void proc_save_fpstate(byte *buf);
+void proc_restore_fpstate(byte *buf);
+\endcode
+
+These routines must be used if floating point operations are performed in
+non-inserted-call locations, such as event callbacks.  Note that there are
+restrictions on how these methods may be called: see the documentation in
+the header files for additional information.  Note also that the floating
+point state must be saved around calls to our provided printing routines
+when they are used to print floats.  However, it is not necessary to save
+and restore the floating point state around floating point operations if
+they are being used in the initialization or termination routines.
+
+This example \client counts the number of basic blocks processed and keeps
+statistics on their average size using floating point operations. Full code
+for this example is in the file <a
+href="samples/bbsize.c">samples/bbsize.c</a>.
+
+
+\ifnot vmsafe
+********************
+\subsection sec_ex7 Use of Custom \Client Statistics with the Windows GUI
+
+The new Windows GUI will display custom \client statistics, if they are
+placed in shared memory with a certain name.  The sample <a
+href="samples/stats.c">samples/stats.c</a> gives code for the
+protocol used in the form of a sample \client that counts total
+instructions, floating-point instructions, and system calls.
+
+\endif
+
+\ifnot vmsafe
+********************
+\subsection sec_ex8 Use of Standalone API
+
+The binary tracedump reader also functions as an example of
+\ref page_standalone : <a href="samples/tracedump.c">samples/tracedump.c</a>.
+
+\endif
+
+********************
+\subsection sec_ex9 Other examples
+
+The sample <a href="samples/strace.c">samples/strace.c</a> displays how to
+use the system call events and API routines.
+
+The sample <a href="samples/signal.c">samples/signal.c</a> demonstrates how to
+use the signal event.
+
+The sample <a href="samples/prefetch.c">samples/prefetch.c</a> demonstrates
+modifying the dynamic code stream for compatibilty between different
+processor types. 
+
+The sample <a href="samples/div.c">samples/div.c</a> demonstrates
+profiling the types of values fed to a particular opcode.
+
+The sample <a href="samples/empty.c">samples/empty.c</a> is provided
+as an example \client that does nothing.
+
+ */
diff --git a/api/docs/transp.dox b/api/docs/transp.dox
new file mode 100644
index 000000000..fecddb4df
--- /dev/null
+++ b/api/docs/transp.dox
@@ -0,0 +1,327 @@
+/* **********************************************************
+ * Copyright (c) 2007-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/**
+ ****************************************************************************
+ ****************************************************************************
+ ****************************************************************************
+\page transparency \Client Transparency
+
+\DynamoRIO must avoid interfering with the semantics of a program while it
+executes.  Shifting execution from the original application code into a
+cache that occupies the application's own address space provides
+flexibility but complicates transparency.  To achieve transparency,
+\DynamoRIO cannot make any assumptions about a program's stack usage, heap
+usage, or any of its dependencies on the instruction set architecture or
+operating system.  \DynamoRIO's transparency restrictions necessarily apply
+to a \client as\if vmsafe well (though they are lessened when using the Probe API in
+lightweight mode).\else well.\endif
+
+Contents:
+- \ref sec_trans_resource
+- \ref sec_trans_unmod
+- \ref sec_trans_pretend
+\if vmsafe
+- \ref sec_trans_probe
+\endif
+
+We first describe each of the three transparency categories for \DynamoRIO
+and their ramifications for \clients using the code cache.  \if vmsafe
+Finally, we discuss transparency issues for the Probe API, which has
+fewer limitations. \endif
+
+***************************************************************************
+\section sec_trans_resource Resource Usage Conflicts
+
+Ideally, \DynamoRIO and its \client's resources should be completely disjoint
+from the application's, to avoid conflicts in the usage of libraries, heap,
+input/output, and locks.  Library conflicts are the most relevant to a
+\client.
+
+\par Library Transparency
+
+Sharing libraries with the application can cause problems with re-entrancy
+and corruption of persistent state like error codes.  \DynamoRIO's dispatch
+code can execute at arbitrary points in the middle of application code; a
+\client's instrumentation is similarly executed.  If both the application
+and \DynamoRIO use the same non-re-entrant library routine, \DynamoRIO might
+call the routine while the application is inside it, causing incorrect
+behavior.  We have learned this lesson the hard way, having run into it
+several times.  The solution is for \DynamoRIO's external resources to come
+only from system calls and never from user libraries.  This is
+straightforward to accomplish on Linux, and most operating systems, where
+the system call interface is a standard mechanism for requesting services
+(\e a in the figure below).  However, on Windows, the documented method of
+interacting with the operating system is not via system calls but instead
+through an application programming interface (the {\em Win32 API}) built
+with user libraries on top of the system call interface (\e b in the
+figure).  If \DynamoRIO uses this interface, re-entrancy and other resource
+usage conflicts can, and will, occur.  To achieve full transparency on
+Windows, the system call interface (\e c in the figure) must be used,
+rather than the API layer.
+
+  \image html windows.png
+  \image rtf windows.png
+  \image latex windows.eps "Native API" width=10cm
+
+\DynamoRIO provides access to resources to \clients via a cross-platform
+API.  We do not recommend that a \client invoke its own system calls as
+this bypasses \DynamoRIO's monitoring of changes to the process address
+space and changes to threads or control flow.
+
+We provide limited support to assist in using statically linked libraries
+in a \client: see \ref sec_extlibs.
+
+In addition to Library Transparency, several other types of transparency
+are of concern to a \client:
+
+\par Heap Transparency
+
+Sharing heap allocation routines with the application violates Library
+Transparency.  Most heap allocation routines are not re-entrant
+(they are thread-safe, but not re-entrant).  Additionally, \DynamoRIO
+should not interfere with the data layout of the application (Data
+Transparency: see below) or with application memory bugs (Error
+Transparency: see below).  \DynamoRIO obtains its memory directly from
+system calls and parcels it out internally with a custom memory manager.
+It exports its heap allocation routines through its API to ensure that
+\clients maintain Heap Transparency (see \ref sec_utils).
+
+\par Input/Output Transparency
+
+\DynamoRIO uses its own input/output routines to avoid interfering with
+the application's buffering.  As with heap transparency, \DynamoRIO
+exports its input/output routines to \clients to ensure that transparency
+is not violated (see \ref sec_utils).
+
+\par Synchronization Transparency
+
+\DynamoRIO and its \clients must avoid acquiring locks that the application
+also acquires, such as the \p LoaderLock on Windows.  Additionally, there
+are restrictions imposed by \DynamoRIO on when its own locks can be
+acquired, to allow it to safely synchronize with multiple threads:
+
+***************************************************************************
+\section sec_trans_unmod Leaving the Application Unchanged When Possible
+
+As many aspects of the application as possible should be left unchanged:
+
+\anchor sec_trans_floating_point
+\par Floating Point State, MMX, and SSE Transparency
+
+Because it is expensive to do so and rarely necessary, \DynamoRIO does
+\e NOT save or restore the floating point state or MMX (64-bit) registers
+during a context switch away from the application.  If at any time a
+\client wishes to use floating point or multimedia operations, it
+must explicitly preserve the state.  The dr_insert_clean_call() routine
+takes a boolean indicating whether floating point state should be preserved
+across the call; this is the most convenient method for saving the state.
+
+The state can alternatively be saved explicitly from C code using:
+
+\code 
+proc_save_fpstate(byte *buf)
+proc_restore_fpstate(byte *buf)
+\endcode
+
+Saving can be done from inlined code in the code cache using:
+
+\code dr_insert_save_fpstate(), dr_insert_restore_fpstate() \endcode
+
+These routines require a buffer that is 16-byte-aligned and of a certain
+size (512 bytes for processors with the FXSR feature, and 108 bytes for
+those without).  Here is a sample usage:
+
+\code
+byte fp_raw[512 + 16];
+byte *fp_align = (byte *) ( (((ptr_uint_t)fp_raw) + 16) & ((ptr_uint_t)-16) );
+proc_save_fpstate(fp_align);
+\endcode
+
+Note that floating point operations include almost any operation that
+acts on a float, even printing one with %%f.
+
+The XMM (128-bit) registers are saved by \DynamoRIO on context switches and
+clean calls only for a 64-bit Windows kernel (whether in a 32-bit process
+(WOW64) or a 64-bit process) or for 64-bit Linux applications.  For a
+64-bit Windows kernel, xmm0 through xmm5 are preserved; for a 64-bit Linux
+application, xmm0 through xmm15 are preserved.  For all other platforms, it
+is up to the \client to preserve the xmm registers, and for 64-bit Windows
+kernels, to preserve the rest of the xmm registers beyond xmm5, if any code
+is invoked that modifies them.  See also the dr_mcontext_xmm_fields_valid()
+routine.
+
+\anchor sec_trans_thread
+\par Thread Transparency
+
+For full transparency, if \DynamoRIO or a \client creates extra threads
+they should be hidden from any introspection performed by the
+application.
+
+\par Executable Transparency
+
+The program binary and shared library files on disk should not be
+modified.
+
+\par Data Transparency
+
+\DynamoRIO leaves application data unmodified, including heap layout.
+
+\par Stack Transparency
+
+The application stack must look exactly like it does natively.  It is
+tempting to use the application stack for scratch space, but we have seen
+applications like Microsoft Office access data beyond the top of the stack
+(i.e., the application stores data on the top of the stack, moves the stack
+pointer to the previous location, and then accesses the data).  Using the
+application stack for scratch space would clobber such data.  Additionally,
+hand-crafted code might use the stack pointer as a general-purpose
+register.  Other and better options for temporary space are available.
+\DynamoRIO provides thread-local storage through its API.
+
+\DynamoRIO provides a separate stack to use for itself and for the \client,
+and never assumes even that the application stack is valid.  Many
+applications examine their stack and may not work properly if something
+is slightly different than expected.
+
+***************************************************************************
+\section sec_trans_pretend Pretending The Application Is Unchanged When It Is Not
+
+For changes that are necessary (such as executing out of a code cache),
+\DynamoRIO must warp events like interrupts, signals, and exceptions such
+that they appear to have occurred natively.
+
+\par Cache Consistency
+
+\DynamoRIO must keep its cached copies of the application code consistent
+with the actual copy in memory.  If the application unloads a shared
+library and loads a new one in its place, or modifies its own code,
+\DynamoRIO must change its code cache to reflect those changes to avoid
+incorrectly executing stale code.  If the \client needs to modify
+application code, it should do so through the basic block event, rather
+than directly.  
+\if future
+FIXME: if we provide routine to modify app memory should
+talk about it here.
+\endif
+
+\par Address Space Transparency
+
+\DynamoRIO must pretend that it is not perturbing the application's
+address space.  An application bug that writes to invalid memory and
+generates an exception should do the same thing under \DynamoRIO, even if
+we have allocated memory at that location that would natively have been
+invalid.  This requires protecting all \DynamoRIO memory from inadvertent
+(or malicious) writes by the application.  Furthermore, \DynamoRIO
+hides itself from introspection by manipulating memory queries.
+
+\par Application Address Transparency
+
+Although the application's code is moved into a cache, every address
+manipulated by the application must remain an original application
+address.  \DynamoRIO must translate indirect branch targets from
+application addresses to code cache addresses, and conversely if a code
+cache address is ever exposed to the application, \DynamoRIO must
+translate it back to its original application address.  The latter occurs
+when the operating system hands a machine context to a signal or
+exception handler.  In that case both the faulting or interrupted address
+and the complete register state must be made to look like the signal or
+exception occurred natively, rather than inside the code cache where it
+actually occurred.
+
+To save space, \DynamoRIO does not store any mappings from code cache
+state to application state.  Since our cache consistency guarantees that
+the original application code cannot have changed since we built a
+fragment, we re-create the fragment from the original code, applying all
+the same transformations we applied when we first copied it into our code
+cache.  We then walk through the reproduction and the code cache version
+in lockstep until we reach the target instruction.  In order to
+accomplish this algorithm in code that has been modified or re-arranged
+by a \client, \DynamoRIO needs \client support.  We have not yet implemented
+this support.  \if future FIXME: say anything else? \endif
+
+\par Error Transparency
+
+Application errors under \DynamoRIO must occur as they would natively.  We
+accomplish this by maintaining Heap Transparency, Data Transparency,
+Stack Transparency, and Address Space Transparency, We have seen many
+cases of applications that access invalid memory natively, handle the
+exception, and carry on.  Without Error Transparency such applications
+would not work properly under \DynamoRIO.
+
+\par Timing Transparency
+
+We would like to make it impossible for the application to determine
+whether it is executing inside of \DynamoRIO.  However, this may not be
+attainable for some aspects of execution, such as the exact timing of
+certain operations.  This brings efficiency into the transparency
+equation.
+
+Changing the timing of multi-threaded applications can uncover behavior
+that does not normally happen natively.  We have encountered race
+conditions while executing under \DynamoRIO that are difficult to
+reproduce outside of our system.  These are not strictly speaking
+transparency violations, as the errors \e could have occurred without
+us, but are best avoided.
+
+\par Debugging Transparency
+
+A debugger should be able to attach to a process under \DynamoRIO's
+control just like it would natively.  Previously discussed transparency
+issues overlap with Debugging Transparency.  Stack Transparency and Data
+Transparency ensure that callstacks and application memory show up
+correctly.  However, \DynamoRIO can't entirely hide the fact that the
+application is running out of a code cache from the debugger.  The
+debugger will see the wrong eip value when breaking in and execution
+break points will not work correctly (and in fact can lead to
+corruption of the \DynamoRIO code cache).  Debugging Tools for Windows and
+\p gdb are known to work with \DynamoRIO with the exception of the
+issues noted above. 
+
+\if vmsafe
+***************************************************************************
+\section sec_trans_probe Transparency of Probe API \Clients
+
+For the Probe API, any particular probe routine is interrupting the
+application at a single, known point.  If the \client is certain that this
+is a safe point for invoking shared resources or performing other actions
+that are risky when attempted at arbitrary points, then they may be
+invoked.  However, be warned that it can be difficult to be certain that
+library routines are safe, as they may themselves acquire locks or call
+other routines that are not safe.
+
+\endif
+
+***************************************************************************
+****************************************************************************
+****************************************************************************
+*/
diff --git a/api/docs/tutorial.dox b/api/docs/tutorial.dox
new file mode 100644
index 000000000..c28c32852
--- /dev/null
+++ b/api/docs/tutorial.dox
@@ -0,0 +1,1188 @@
+/* **********************************************************
+ * Copyright (c) 2008-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/**
+\page API_tutorial API Usage Tutorial
+
+Below we provide a few short tutorials on using the \DynamoRIO API to write
+several \clients.
+
+- A \client that tracks the average dynamic basic block size
+demonstrating some simple instrumentation : \subpage API_tutorial_bbdynsize1 "average_bb_size".
+
+- A \client that steals a register from the application demonstrating
+some simple state translation : \subpage API_tutorial_steal_reg1 "steal_reg".
+
+- A \client that adjusts prefetch instructions demonstrating some simple
+app modifications : \subpage API_tutorial_prefetch1 "prefetch".
+
+**/
+
+
+/**
+\page API_tutorial_bbdynsize1
+
+In this tutorial we'll create a simple \client that will compute the
+average size of basic blocks executed (in instructions) weighted by
+their execution frequency. We start with an empty \DynamoRIO \client.
+
+\code
+#include "dr_api.h"
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    /* empty */
+}
+\endcode
+
+[ \ref API_tutorial "prev" | \subpage API_tutorial_bbdynsize2 "next" ]
+
+\page API_tutorial_bbdynsize2
+Now we register for events we'll need.  For our first pass we only
+need to register for the bb event (to add our instrumentation) and the
+exit event (to display the results).  We don't need to register for
+trace events (as we will add our instrumentation to the constituent
+bbs) and we can ignore the \p for_trace and \p translating arguments
+to the basic block callback as our changes are deterministic and
+idempotent so \DynamoRIO default translation will work fine.
+
+\code
+#include "dr_api.h"
+
++ static void
++ event_exit(void);
++ static dr_emit_flags_t
++ event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
++                   bool for_trace, bool translating);
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
++     /* register events */
++     dr_register_exit_event(event_exit);
++     dr_register_bb_event(event_basic_block);
+}
+
++ static void
++ event_exit(void)
++ {
++     /* empty */
++ }
++ 
++ static dr_emit_flags_t
++ event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
++                   bool for_trace, bool translating)
++ {
++     /* empty */
++     return DR_EMIT_DEFAULT;
++ }
+\endcode
+
+[ \ref API_tutorial_bbdynsize1 "prev" | \subpage API_tutorial_bbdynsize3 "next" ]
+
+\page API_tutorial_bbdynsize3
+Next we add code to track the size of blocks as we build them for our
+as-built count.  This will give us the number of blocks built and
+their average size.  Next we will add the dynamic execution counts.
+
+\code
+#include "dr_api.h"
+
++ #ifdef WINDOWS
++ # define DISPLAY_STRING(msg) dr_messagebox(msg)
++ #else
++ # define DISPLAY_STRING(msg) dr_printf("%s\n", msg)
++ #endif
+
++ typedef struct bb_counts { 
++    uint64 blocks;
++    uint64 total_size;
++ } bb_counts;
+
++ static bb_counts counts_as_built;
++ void *as_built_lock;
+
+static void
+event_exit(void);
+static dr_emit_flags_t
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating);
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    /* register events */
+    dr_register_exit_event(event_exit);
+    dr_register_bb_event(event_basic_block);
++ 
++     /* initialize lock */
++     as_built_lock = dr_mutex_create();
+}
+
+static void
+event_exit(void)
+{
++     /* Display results - we must first snpritnf the string as on windows
++      * dr_printf(), dr_messagebox() and dr_fprintf() can't print floats. */
++     char msg[512];
++     int len;
++     len = snprintf(msg, sizeof(msg)/sizeof(msg[0]),
++                    "Number of basic blocks built : %"UINT64_FORMAT_CODE"\n"
++                    "     Average size            : %5.2lf instructions\n",
++                    counts_as_built.blocks,
++                    counts_as_built.total_size / (double)counts_as_built.blocks);
++     DR_ASSERT(len > 0);
++     msg[sizeof(msg)/sizeof(msg[0])-1] = '\0'; /* NUll terminate */
++     DISPLAY_STRING(msg);
++ 
++      /* free mutex */
++      dr_mutex_destroy(as_built_lock);
+}
+
+static dr_emit_flags_t
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating)
+{
++     uint num_instructions = 0;
++     instr_t *instr;
++ 
++     /* count the number of instructions in this block */
++     for (instr = instrlist_first(bb); instr != NULL; instr = instr_get_next(instr)) {
++         num_instructions++;
++     }
++
++     /* update the as-built counts */
++     dr_mutex_lock(as_built_lock);
++     counts_as_built.blocks++;
++     counts_as_built.total_size += num_instructions;
++     dr_mutex_unlock(as_built_lock);
+    return DR_EMIT_DEFAULT;
+}
+\endcode
+
+[ \ref API_tutorial_bbdynsize2 "prev" | \subpage API_tutorial_bbdynsize4 "next" ]
+
+\page API_tutorial_bbdynsize4
+Now we add instrumentation to gather the dynamic counts.  For the
+first pass we'll use a clean call to increment the counters from each
+block.  However this approach is very slow since for each basic block
+execution a context switch out of the cache will occur.  We'll look at
+other approaches next.
+
+\code
+#include "dr_api.h"
+
+#ifdef WINDOWS
+# define DISPLAY_STRING(msg) dr_messagebox(msg)
+#else
+# define DISPLAY_STRING(msg) dr_printf("%s\n", msg)
+#endif
+
+typedef struct bb_counts { 
+   uint64 blocks;
+   uint64 total_size;
+} bb_counts;
+
+static bb_counts counts_as_built;
+void *as_built_lock;
+
++ static bb_counts counts_dynamic;
++ void *count_lock;
+
+static void
+event_exit(void);
+static dr_emit_flags_t
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating);
++ static void
++ clean_call(uint instruction_count);
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    /* register events */
+    dr_register_exit_event(event_exit);
+    dr_register_bb_event(event_basic_block);
+
+    /* initialize lock */
+    as_built_lock = dr_mutex_create();
++     count_lock = dr_mutex_create();
+}
+
+static void
+event_exit(void)
+{
+    /* Display results - we must first snpritnf the string as on windows
+     * dr_printf(), dr_messagebox() and dr_fprintf() can't print floats. */
+    char msg[512];
+    int len;
+    len = snprintf(msg, sizeof(msg)/sizeof(msg[0]),
+                   "Number of blocks built : %"UINT64_FORMAT_CODE"\n"
+                   "     Average size      : %5.2lf instructions\n"
++                    "Number of blocks executed  : %"UINT64_FORMAT_CODE"\n"
++                    "     Average weighted size : %5.2lf instructions\n",
+                   counts_as_built.blocks,
+                   counts_as_built.total_size / (double)counts_as_built.blocks,
++                    counts_dynamic.blocks,
++                    counts_dynamic.total_size / (double)counts_dynamic.blocks);
+    DR_ASSERT(len > 0);
+    msg[sizeof(msg)/sizeof(msg[0])-1] = '\0';
+    DISPLAY_STRING(msg);
+
+    /* free mutex */
+    dr_mutex_destroy(as_built_lock);
++     dr_mutex_destroy(count_lock);
+}
+
++ static void
++ clean_call(uint instruction_count)
++ {
++     dr_mutex_lock(count_lock);
++     counts_dynamic.blocks++;
++     counts_dynamic.total_size += instruction_count;
++     dr_mutex_unlock(count_lock);
++ }
+
+static dr_emit_flags_t
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating)
+{
+    uint num_instructions = 0;
+    instr_t *instr;
+
+    /* count the number of instructions in this block */
+    for (instr = instrlist_first(bb); instr != NULL; instr = instr_get_next(instr)) {
+        num_instructions++;
+    }
+
+    /* update the as-built counts */
+    dr_mutex_lock(as_built_lock);
+    counts_as_built.blocks++;
+    counts_as_built.total_size += num_instructions;
+    dr_mutex_unlock(as_built_lock);
+
++     /* insert clean call */
++     dr_insert_clean_call(drcontext, bb, instrlist_first(bb), clean_call, false, 1,
++                          OPND_CREATE_INT32(num_instructions));
++ 
+    return DR_EMIT_DEFAULT;
+}
+\endcode
+
+[ \ref API_tutorial_bbdynsize3 "prev" | \subpage API_tutorial_bbdynsize5 "next" ]
+
+\page API_tutorial_bbdynsize5
+In this variation we use inlined atomic instructions to adjust the counters.  On 
+a single core box this form should be quite fast, but on a multi-core box the 
+cache line locking for atomic operations will slow things dramatically.  We'll 
+look at forms more suitable for multi-core further on.
+
+\code
+#include "dr_api.h"
+
+#ifdef WINDOWS
+# define DISPLAY_STRING(msg) dr_messagebox(msg)
+#else
+# define DISPLAY_STRING(msg) dr_printf("%s\n", msg)
+#endif
+
+typedef struct bb_counts { 
+   uint64 blocks;
+   uint64 total_size;
+} bb_counts;
+
+static bb_counts counts_as_built;
+void *as_built_lock;
+
+static bb_counts counts_dynamic;
+
+static void
+event_exit(void);
+static dr_emit_flags_t
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating);
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    /* register events */
+    dr_register_exit_event(event_exit);
+    dr_register_bb_event(event_basic_block);
+
+    /* initialize lock */
+    as_built_lock = dr_mutex_create();
+}
+
+static void
+event_exit(void)
+{
+    /* Display results - we must first snpritnf the string as on windows
+     * dr_printf(), dr_messagebox() and dr_fprintf() can't print floats. */
+    char msg[512];
+    int len;
+    len = snprintf(msg, sizeof(msg)/sizeof(msg[0]),
+                   "Number of blocks built : %"UINT64_FORMAT_CODE"\n"
+                   "     Average size      : %5.2lf instructions\n"
+                   "Number of blocks executed  : %"UINT64_FORMAT_CODE"\n"
+                   "     Average weighted size : %5.2lf instructions\n",
+                   counts_as_built.blocks,
+                   counts_as_built.total_size / (double)counts_as_built.blocks,
+                   counts_dynamic.blocks,
+                   counts_dynamic.total_size / (double)counts_dynamic.blocks);
+    DR_ASSERT(len > 0);
+    msg[sizeof(msg)/sizeof(msg[0])-1] = '\0';
+    DISPLAY_STRING(msg);
+
+    /* free mutex */
+    dr_mutex_destroy(as_built_lock);
+}
+
+static dr_emit_flags_t
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating)
+{
+    uint num_instructions = 0;
+    instr_t *instr, *where = NULL;
+
+    /* count the number of instructions in this block */
+    for (instr = instrlist_first(bb); instr != NULL; instr = instr_get_next(instr)) {
+        num_instructions++;
+    }
+
+    /* update the as-built counts */
+    dr_mutex_lock(as_built_lock);
+    counts_as_built.blocks++;
+    counts_as_built.total_size += num_instructions;
+    dr_mutex_unlock(as_built_lock);
+
++     /* increment counters */
++     where = instrlist_first(bb);
++     dr_save_arith_flags(drcontext, bb, where, SPILL_SLOT_1);
++ #ifdef X86_32
++     /* Since the counters are 64-bit we must use an add + an addc to increment.
++      * The operations is still effectively atomic since we're only increasing
++      * the count. */
++     instrlist_meta_preinsert(bb, where,
++         LOCK(INSTR_CREATE_add(drcontext,
++                               OPND_CREATE_ABSMEM((byte *)&counts_dynamic.blocks, OPSZ_4),
++                               OPND_CREATE_INT8(1))));
++     instrlist_meta_preinsert(bb, where,
++         LOCK(INSTR_CREATE_adc(drcontext,
++                               OPND_CREATE_ABSMEM((byte *)&counts_dynamic.blocks + 4, OPSZ_4),
++                               OPND_CREATE_INT8(0))));
++ 
++     instrlist_meta_preinsert(bb, where,
++         LOCK(INSTR_CREATE_add(drcontext,
++                               OPND_CREATE_ABSMEM((byte *)&counts_dynamic.total_size, OPSZ_4),
++                               OPND_CREATE_INT_32OR8(num_instructions))));
++     instrlist_meta_preinsert(bb, where,
++         LOCK(INSTR_CREATE_adc(drcontext,
++                               OPND_CREATE_ABSMEM((byte *)&counts_dynamic.total_size + 4, OPSZ_4),
++                               OPND_CREATE_INT8(0))));
++ #else /* X86_64 */
++     instrlist_meta_preinsert(bb, where,
++         LOCK(INSTR_CREATE_inc(drcontext,
++                               OPND_CREATE_ABSMEM((byte *)&counts_dynamic.blocks, OPSZ_8))));
++     instrlist_meta_preinsert(bb, where,
++         LOCK(INSTR_CREATE_add(drcontext,
++                               OPND_CREATE_ABSMEM((byte *)&counts_dynamic.total_size, OPSZ_8),
++                               OPND_CREATE_INT_32OR8(num_instructions))));
++ #endif
++     dr_restore_arith_flags(drcontext, bb, where, SPILL_SLOT_1);
++ 
+    return DR_EMIT_DEFAULT;
+}
+\endcode
+
+[ \ref API_tutorial_bbdynsize4 "prev" | \subpage API_tutorial_bbdynsize6 "next" ]
+
+\page API_tutorial_bbdynsize6
+Next up we add an optimization to only save and restore the flags if we can't find a spot 
+within the block where the flags are dead.
+
+\code
+#include "dr_api.h"
+
+#ifdef WINDOWS
+# define DISPLAY_STRING(msg) dr_messagebox(msg)
+#else
+# define DISPLAY_STRING(msg) dr_printf("%s\n", msg)
+#endif
+
++ #define TESTALL(mask, var) (((mask) & (var)) == (mask))
++ #define TESTANY(mask, var) (((mask) & (var)) != 0)
+
+typedef struct bb_counts { 
+   uint64 blocks;
+   uint64 total_size;
+} bb_counts;
+
+
+static bb_counts counts_as_built;
+void *as_built_lock;
+
+static bb_counts counts_dynamic;
+
++ /* Protected by the as_built_lock */
++ static uint64 bbs_eflags_saved;
+
+
+static void
+event_exit(void);
+static dr_emit_flags_t
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating);
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    /* register events */
+    dr_register_exit_event(event_exit);
+    dr_register_bb_event(event_basic_block);
+
+    /* initialize lock */
+    as_built_lock = dr_mutex_create();
+}
+
+static void
+event_exit(void)
+{
+    /* Display results - we must first snpritnf the string as on windows
+     * dr_printf(), dr_messagebox() and dr_fprintf() can't print floats. */
+    char msg[512];
+    int len;
+    len = snprintf(msg, sizeof(msg)/sizeof(msg[0]),
+                   "Number of blocks built  : %"UINT64_FORMAT_CODE"\n"
+                   "     Average size       : %5.2lf instructions\n"
++                    "     Num saved eflags : %"UINT64_FORMAT_CODE"\n"
+                   "Number of blocks executed  : %"UINT64_FORMAT_CODE"\n"
+                   "     Average weighted size : %5.2lf instructions\n",
+                   counts_as_built.blocks,
+                   counts_as_built.total_size / (double)counts_as_built.blocks,
++                    bbs_eflags_saved,
+                   counts_dynamic.blocks,
+                   counts_dynamic.total_size / (double)counts_dynamic.blocks);
+    DR_ASSERT(len > 0);
+    msg[sizeof(msg)/sizeof(msg[0])-1] = '\0';
+    DISPLAY_STRING(msg);
+
+    /* free mutex */
+    dr_mutex_destroy(as_built_lock);
+}
+
+static dr_emit_flags_t
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating)
+{
+    uint num_instructions = 0;
+    instr_t *instr, *where = NULL;
++     bool eflags_saved = true;
+
+    /* count the number of instructions in this block */
+    for (instr = instrlist_first(bb); instr != NULL; instr = instr_get_next(instr)) {
++         /* Since it doesn't matter where we insert, look for a place
++          * where the eflags are dead. */
++         uint flags = instr_get_arith_flags(instr);
++         if (TESTALL(EFLAGS_WRITE_6, flags) && !TESTANY(EFLAGS_READ_6, flags)) {
++             where = instr;
++             eflags_saved = false;
++         }
+        num_instructions++;
+    }
+
+    /* update the as-built counts */
+    dr_mutex_lock(as_built_lock);
+    counts_as_built.blocks++;
+    counts_as_built.total_size += num_instructions;
++     if (eflags_saved)
++         bbs_eflags_saved++;
+    dr_mutex_unlock(as_built_lock);
+
+    /* increment counters */
++     if (eflags_saved) {
++         where = instrlist_first(bb);
+        dr_save_arith_flags(drcontext, bb, where, SPILL_SLOT_1);
++     }
+#ifdef X86_32
+    /* Since the counters are 64-bit we must use an add an addc to increment.
+     * The operations is still effectively atomic since we're only increasing
+     * the count. */
+    instrlist_meta_preinsert(bb, where,
+        LOCK(INSTR_CREATE_add(drcontext,
+                              OPND_CREATE_ABSMEM((byte *)&counts_dynamic.blocks, OPSZ_4),
+                              OPND_CREATE_INT8(1))));
+    instrlist_meta_preinsert(bb, where,
+        LOCK(INSTR_CREATE_adc(drcontext,
+                              OPND_CREATE_ABSMEM((byte *)&counts_dynamic.blocks + 4, OPSZ_4),
+                              OPND_CREATE_INT8(0))));
+
+    instrlist_meta_preinsert(bb, where,
+        LOCK(INSTR_CREATE_add(drcontext,
+                              OPND_CREATE_ABSMEM((byte *)&counts_dynamic.total_size, OPSZ_4),
+                              OPND_CREATE_INT_32OR8(num_instructions))));
+    instrlist_meta_preinsert(bb, where,
+        LOCK(INSTR_CREATE_adc(drcontext,
+                              OPND_CREATE_ABSMEM((byte *)&counts_dynamic.total_size + 4, OPSZ_4),
+                              OPND_CREATE_INT8(0))));
+#else /* X86_64 */
+    instrlist_meta_preinsert(bb, where,
+        LOCK(INSTR_CREATE_inc(drcontext,
+                              OPND_CREATE_ABSMEM((byte *)&counts_dynamic.blocks, OPSZ_8))));
+    instrlist_meta_preinsert(bb, where,
+        LOCK(INSTR_CREATE_add(drcontext,
+                              OPND_CREATE_ABSMEM((byte *)&counts_dynamic.total_size, OPSZ_8),
+                              OPND_CREATE_INT_32OR8(num_instructions))));
+#endif
++     if (eflags_saved) {
+        dr_restore_arith_flags(drcontext, bb, where, SPILL_SLOT_1);
++     }
+
+    return DR_EMIT_DEFAULT;
+}
+\endcode
+
+[ \ref API_tutorial_bbdynsize5 "prev" | \subpage API_tutorial_bbdynsize7 "next" ]
+
+\page API_tutorial_bbdynsize7
+We now have a pretty fast implementation, probably about as fast as we can get 
+on a single core machine.  However, on multi-core systems the cache 
+line locking for the atomic operations becomes a bottle neck.  To get around that 
+we can use thread-private counts and aggregate them.  With thread-private counts
+no locked instructions are needed, however the extra indirection does has its
+own cost of extra added instructions to spill a register and get the tls pointer 
+so on single core this will be slower.
+
+\code
+#include "dr_api.h"
++ #include <stddef.h>
+
+#ifdef WINDOWS
+# define DISPLAY_STRING(msg) dr_messagebox(msg)
+#else
+# define DISPLAY_STRING(msg) dr_printf("%s\n", msg)
+#endif
+
+#define TESTALL(mask, var) (((mask) & (var)) == (mask))
+#define TESTANY(mask, var) (((mask) & (var)) != 0)
+
+typedef struct bb_counts { 
+   uint64 blocks;
+   uint64 total_size;
+} bb_counts;
+
+static bb_counts counts_as_built;
+void *as_built_lock;
+
+static bb_counts counts_dynamic;
++ void *count_lock;
+
+/* Protected by the as_built_lock */
+static uint64 bbs_eflags_saved;
+
+
+static void
+event_exit(void);
+static dr_emit_flags_t
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating);
++ static void
++ event_thread_init(void *drcontext);
++ static void
++ event_thread_exit(void *drcontext);
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    /* register events */
+    dr_register_exit_event(event_exit);
+    dr_register_bb_event(event_basic_block);
++     dr_register_thread_init_event(event_thread_init);
++     dr_register_thread_exit_event(event_thread_exit);
+
+    /* initialize lock */
+    as_built_lock = dr_mutex_create();
++     count_lock = dr_mutex_create();
+}
+
+static void
+event_exit(void)
+{
+    /* Display results - we must first snpritnf the string as on windows
+     * dr_printf(), dr_messagebox() and dr_fprintf() can't print floats. */
+    char msg[512];
+    int len;
+    len = snprintf(msg, sizeof(msg)/sizeof(msg[0]),
+                   "Number of blocks built  : %"UINT64_FORMAT_CODE"\n"
+                   "     Average size       : %5.2lf instructions\n"
+                   "     Num saved eflags : %"UINT64_FORMAT_CODE"\n"
+                   "Number of blocks executed  : %"UINT64_FORMAT_CODE"\n"
+                   "     Average weighted size : %5.2lf instructions\n",
+                   counts_as_built.blocks,
+                   counts_as_built.total_size / (double)counts_as_built.blocks,
+                   bbs_eflags_saved,
+                   counts_dynamic.blocks,
+                   counts_dynamic.total_size / (double)counts_dynamic.blocks);
+    DR_ASSERT(len > 0);
+    msg[sizeof(msg)/sizeof(msg[0])-1] = '\0';
+    DISPLAY_STRING(msg);
+
+    /* free mutex */
+    dr_mutex_destroy(as_built_lock);
++     dr_mutex_destroy(count_lock);
+}
+
++ static void 
++ event_thread_init(void *drcontext)
++ {
++     /* create an instance of our data structure for this thread */
++     bb_counts *counts = (bb_counts *)dr_thread_alloc(drcontext, sizeof(bb_counts));
++     /* store it in the slot provided in the drcontext */
++     dr_set_tls_field(drcontext, counts);
++     memset(counts, 0, sizeof(bb_counts));
++ }
++ 
++ static void 
++ event_thread_exit(void *drcontext)
++ {
++     bb_counts *counts = (bb_counts *) dr_get_tls_field(drcontext);
++     /* NOTE - if we so choose we could report per-thread sizes here. */
++     dr_mutex_lock(count_lock);
++     counts_dynamic.blocks += counts->blocks;
++     counts_dynamic.total_size += counts->total_size;
++     dr_mutex_unlock(count_lock);
++     dr_thread_free(drcontext, counts, sizeof(bb_counts));
++ }
+
+static dr_emit_flags_t
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating)
+{
+    uint num_instructions = 0;
+    instr_t *instr, *where = NULL;
+    bool eflags_saved = true;
++     bb_counts *counts = (bb_counts *) dr_get_tls_field(drcontext);
+
+    /* count the number of instructions in this block */
+    for (instr = instrlist_first(bb); instr != NULL; instr = instr_get_next(instr)) {
+        /* Since it doesn't matter where we insert, look for a place 
+         * where the eflags are dead. */
+        uint flags = instr_get_arith_flags(instr);
+        if (TESTALL(EFLAGS_WRITE_6, flags) && !TESTANY(EFLAGS_READ_6, flags)) {
+            where = instr;
+            eflags_saved = false;
+        }
+        num_instructions++;
+    }
+
+    /* update the as-built counts */
+    dr_mutex_lock(as_built_lock);
+    counts_as_built.blocks++;
+    counts_as_built.total_size += num_instructions;
+    if (eflags_saved)
+        bbs_eflags_saved++;
+    dr_mutex_unlock(as_built_lock);
+
+    /* increment counters */
+    if (eflags_saved) {
+        where = instrlist_first(bb);
+        dr_save_arith_flags(drcontext, bb, where, SPILL_SLOT_1);
+    }
++     /* Spill a register to get a pointer to our TLS structure. */
++     dr_save_reg(drcontext, bb, where, REG_XDI, SPILL_SLOT_2);
++     dr_insert_read_tls_field(drcontext, bb, where, REG_XDI);
+#ifdef X86_32
+    /* Since the counters are 64-bit we must use an add an addc to increment.
+     * The operations is still effectively atomic since we're only increasing
+     * the count. */
+    instrlist_meta_preinsert(bb, where,
+!         INSTR_CREATE_add(drcontext,
+!                          OPND_CREATE_MEM32(REG_XDI, offsetof(bb_counts, blocks)),
+                         OPND_CREATE_INT8(1)));
+    instrlist_meta_preinsert(bb, where,
+!         INSTR_CREATE_adc(drcontext,
+!                          OPND_CREATE_MEM32(REG_XDI, offsetof(bb_counts, blocks)+4),
+                         OPND_CREATE_INT8(0)));
+
+    instrlist_meta_preinsert(bb, where,
+!         INSTR_CREATE_add(drcontext,
+!                          OPND_CREATE_MEM32(REG_XDI, offsetof(bb_counts, total_size)),
+                         OPND_CREATE_INT_32OR8(num_instructions)));
+    instrlist_meta_preinsert(bb, where,
+!         INSTR_CREATE_adc(drcontext,
+!                          OPND_CREATE_MEM32(REG_XDI, offsetof(bb_counts, total_size)+4),
+                         OPND_CREATE_INT8(0)));
+#else /* X86_64 */
+    instrlist_meta_preinsert(bb, where,
+!        INSTR_CREATE_inc(drcontext,
+!                         OPND_CREATE_MEM64(REG_XDI, offsetof(bb_counts, blocks))));
+    instrlist_meta_preinsert(bb, where,
+!        INSTR_CREATE_add(drcontext,
+!                         OPND_CREATE_MEM64(REG_XDI, offsetof(bb_counts, total_size)),
+                        OPND_CREATE_INT_32OR8(num_instructions)));
+#endif
++     /* Restore spilled register. */
++     dr_restore_reg(drcontext, bb, where, REG_XDI, SPILL_SLOT_2);
+    if (eflags_saved) {
+        dr_restore_arith_flags(drcontext, bb, where, SPILL_SLOT_1);
+    }
+
+    return DR_EMIT_DEFAULT;
+}
+\endcode
+
+[ \ref API_tutorial_bbdynsize6 "prev" | \subpage API_tutorial_bbdynsize8 "next" ]
+
+\page API_tutorial_bbdynsize8
+Our final optimization is to note that if we're running with thread-private caches 
+(i.e. with the -thread_private option to \DynamoRIO) we can use absolute addressing 
+on the per-thread counters instead of indirecting via TLS which should greatly speed 
+things up.  The downside to -thread_private is that on applications with many threads 
+memory use can get quite high, potentially enough to impact performance of the system.
+
+\code
+#include "dr_api.h"
+#include <stddef.h>
+
+#ifdef WINDOWS
+# define DISPLAY_STRING(msg) dr_messagebox(msg)
+#else
+# define DISPLAY_STRING(msg) dr_printf("%s\n", msg)
+#endif
+
+#define TESTALL(mask, var) (((mask) & (var)) == (mask))
+#define TESTANY(mask, var) (((mask) & (var)) != 0)
+
+typedef struct bb_counts { 
+   uint64 blocks;
+   uint64 total_size;
+} bb_counts;
+
+static bb_counts counts_as_built;
+void *as_built_lock;
+
+static bb_counts counts_dynamic;
+void *count_lock;
+
+/* Protected by the as_built_lock */
+static uint64 bbs_eflags_saved;
+
+
+static void
+event_exit(void);
+static dr_emit_flags_t
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating);
+static void
+event_thread_init(void *drcontext);
+static void
+event_thread_exit(void *drcontext);
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    /* register events */
+    dr_register_exit_event(event_exit);
+    dr_register_bb_event(event_basic_block);
+    dr_register_thread_init_event(event_thread_init);
+    dr_register_thread_exit_event(event_thread_exit);
+
+    /* initialize lock */
+    as_built_lock = dr_mutex_create();
+    count_lock = dr_mutex_create();
+}
+
+static void
+event_exit(void)
+{
+    /* Display results - we must first snpritnf the string as on windows
+     * dr_printf(), dr_messagebox() and dr_fprintf() can't print floats. */
+    char msg[512];
+    int len;
+    len = snprintf(msg, sizeof(msg)/sizeof(msg[0]),
+                   "Number of blocks built  : %"UINT64_FORMAT_CODE"\n"
+                   "     Average size       : %5.2lf instructions\n"
+                   "     Num saved eflags : %"UINT64_FORMAT_CODE"\n"
+                   "Number of blocks executed  : %"UINT64_FORMAT_CODE"\n"
+                   "     Average weighted size : %5.2lf instructions\n",
+                   counts_as_built.blocks,
+                   counts_as_built.total_size / (double)counts_as_built.blocks,
+                   bbs_eflags_saved,
+                   counts_dynamic.blocks,
+                   counts_dynamic.total_size / (double)counts_dynamic.blocks);
+    DR_ASSERT(len > 0);
+    msg[sizeof(msg)/sizeof(msg[0])-1] = '\0';
+    DISPLAY_STRING(msg);
+
+    /* free mutex */
+    dr_mutex_destroy(as_built_lock);
+    dr_mutex_destroy(count_lock);
+}
+
+static void 
+event_thread_init(void *drcontext)
+{
+    /* create an instance of our data structure for this thread */
+    bb_counts *counts = (bb_counts *)dr_thread_alloc(drcontext, sizeof(bb_counts));
+    /* store it in the slot provided in the drcontext */
+    dr_set_tls_field(drcontext, counts);
+    memset(counts, 0, sizeof(bb_counts));
+}
+
+static void 
+event_thread_exit(void *drcontext)
+{
+    bb_counts *counts = (bb_counts *) dr_get_tls_field(drcontext);
+    /* NOTE - if we so choose we could report per-thread sizes here. */
+    dr_mutex_lock(count_lock);
+    counts_dynamic.blocks += counts->blocks;
+    counts_dynamic.total_size += counts->total_size;
+    dr_mutex_unlock(count_lock);
+    dr_thread_free(drcontext, counts, sizeof(bb_counts));
+}
+
+static dr_emit_flags_t
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating)
+{
+    uint num_instructions = 0;
+    instr_t *instr, *where = NULL;
+    bool eflags_saved = true;
+    bb_counts *counts = (bb_counts *) dr_get_tls_field(drcontext);
+
+    /* count the number of instructions in this block */
+    for (instr = instrlist_first(bb); instr != NULL; instr = instr_get_next(instr)) {
+        /* Since it doesn't matter where we insert, look for a place 
+         * where the eflags are dead. */
+        uint flags = instr_get_arith_flags(instr);
+        if (TESTALL(EFLAGS_WRITE_6, flags) && !TESTANY(EFLAGS_READ_6, flags)) {
+            where = instr;
+            eflags_saved = false;
+        }
+        num_instructions++;
+    }
+
+    /* update the as-built counts */
+    dr_mutex_lock(as_built_lock);
+    counts_as_built.blocks++;
+    counts_as_built.total_size += num_instructions;
+    if (eflags_saved)
+        bbs_eflags_saved++;
+    dr_mutex_unlock(as_built_lock);
+
+    /* increment counters */
+    if (eflags_saved) {
+        where = instrlist_first(bb);
+        dr_save_arith_flags(drcontext, bb, where, SPILL_SLOT_1);
+    }
++     /* If all thread-private caches we can use direct addressing. */
++     if (!dr_using_all_private_caches()) {
+        /* Spill a register to get a pointer to our TLS structure. */
+        dr_save_reg(drcontext, bb, where, REG_XDI, SPILL_SLOT_2);
+        dr_insert_read_tls_field(drcontext, bb, where, REG_XDI);
++     }
+#ifdef X86_32
+    /* Since the counters are 64-bit we must use an add an addc to increment.
+     * The operations is still effectively atomic since we're only increasing
+     * the count. */
+    instrlist_meta_preinsert(bb, where,
+        INSTR_CREATE_add(drcontext,
+!                          dr_using_all_private_caches() ?
+!                            OPND_CREATE_ABSMEM((byte *)&counts->blocks, OPSZ_4) : 
+!                            OPND_CREATE_MEM32(REG_XDI, offsetof(bb_counts, blocks)),
+                         OPND_CREATE_INT8(1)));
+    instrlist_meta_preinsert(bb, where,
+        INSTR_CREATE_adc(drcontext,
+!                          dr_using_all_private_caches() ?
+!                            OPND_CREATE_ABSMEM((byte *)&counts->blocks +4, OPSZ_4) :
+!                            OPND_CREATE_MEM32(REG_XDI, offsetof(bb_counts, blocks)+4),
+                         OPND_CREATE_INT8(0)));
+
+    instrlist_meta_preinsert(bb, where,
+        INSTR_CREATE_add(drcontext,
+!                          dr_using_all_private_caches() ?
+!                            OPND_CREATE_ABSMEM((byte *)&counts->total_size, OPSZ_4) :
+!                            OPND_CREATE_MEM32(REG_XDI, offsetof(bb_counts, total_size)),
+                         OPND_CREATE_INT_32OR8(num_instructions)));
+    instrlist_meta_preinsert(bb, where,
+        INSTR_CREATE_adc(drcontext,
+!                          dr_using_all_private_caches() ?
+!                            OPND_CREATE_ABSMEM((byte *)&counts->total_size +4, OPSZ_4) :
+!                            OPND_CREATE_MEM32(REG_XDI, offsetof(bb_counts, total_size)+4),
+                         OPND_CREATE_INT8(0)));
+#else /* X86_64 */
+    instrlist_meta_preinsert(bb, where,
+       INSTR_CREATE_inc(drcontext,
+!                          dr_using_all_private_caches() ?
+!                            OPND_CREATE_ABSMEM((byte *)&counts->blocks, OPSZ_8) :
+!                            OPND_CREATE_MEM64(REG_XDI, offsetof(bb_counts, blocks))));
+    instrlist_meta_preinsert(bb, where,
+       INSTR_CREATE_add(drcontext,
+!                          dr_using_all_private_caches() ?
+!                            OPND_CREATE_ABSMEM((byte *)&counts->total_size, OPSZ_8) :
+!                            OPND_CREATE_MEM64(REG_XDI, offsetof(bb_counts, total_size)),
+                        OPND_CREATE_INT_32OR8(num_instructions)));
+#endif
++     if (!dr_using_all_private_caches()) {
+        /* Restore spilled register. */
+        dr_restore_reg(drcontext, bb, where, REG_XDI, SPILL_SLOT_2);
++     }
+    if (eflags_saved) {
+        dr_restore_arith_flags(drcontext, bb, where, SPILL_SLOT_1);
+    }
+
+    return DR_EMIT_DEFAULT;
+}
+\endcode
+
+
+[ \ref API_tutorial_bbdynsize7 "prev" | \subpage API_tutorial_bbdynsize9 "next" ]
+
+\page API_tutorial_bbdynsize9
+The final multi-core optimized version.  If we are single core then the LOCK version 
+is probably faster unless we are running with -thread_private in which case it's probably 
+close to equivalent.
+
+\code
+#include "dr_api.h"
+#include <stddef.h>
+
+#ifdef WINDOWS
+# define DISPLAY_STRING(msg) dr_messagebox(msg)
+#else
+# define DISPLAY_STRING(msg) dr_printf("%s\n", msg)
+#endif
+
+#define TESTALL(mask, var) (((mask) & (var)) == (mask))
+#define TESTANY(mask, var) (((mask) & (var)) != 0)
+
+typedef struct bb_counts { 
+   uint64 blocks;
+   uint64 total_size;
+} bb_counts;
+
+static bb_counts counts_as_built;
+void *as_built_lock;
+
+static bb_counts counts_dynamic;
+void *count_lock;
+
+/* Protected by the as_built_lock */
+static uint64 bbs_eflags_saved;
+
+
+static void
+event_exit(void);
+static dr_emit_flags_t
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating);
+static void
+event_thread_init(void *drcontext);
+static void
+event_thread_exit(void *drcontext);
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    /* register events */
+    dr_register_exit_event(event_exit);
+    dr_register_bb_event(event_basic_block);
+    dr_register_thread_init_event(event_thread_init);
+    dr_register_thread_exit_event(event_thread_exit);
+
+    /* initialize lock */
+    as_built_lock = dr_mutex_create();
+    count_lock = dr_mutex_create();
+}
+
+static void
+event_exit(void)
+{
+    /* Display results - we must first snpritnf the string as on windows
+     * dr_printf(), dr_messagebox() and dr_fprintf() can't print floats. */
+    char msg[512];
+    int len;
+    len = snprintf(msg, sizeof(msg)/sizeof(msg[0]),
+                   "Number of blocks built  : %"UINT64_FORMAT_CODE"\n"
+                   "     Average size       : %5.2lf instructions\n"
+                   "     Num saved eflags : %"UINT64_FORMAT_CODE"\n"
+                   "Number of blocks executed  : %"UINT64_FORMAT_CODE"\n"
+                   "     Average weighted size : %5.2lf instructions\n",
+                   counts_as_built.blocks,
+                   counts_as_built.total_size / (double)counts_as_built.blocks,
+                   bbs_eflags_saved,
+                   counts_dynamic.blocks,
+                   counts_dynamic.total_size / (double)counts_dynamic.blocks);
+    DR_ASSERT(len > 0);
+    msg[sizeof(msg)/sizeof(msg[0])-1] = '\0';
+    DISPLAY_STRING(msg);
+
+    /* free mutex */
+    dr_mutex_destroy(as_built_lock);
+    dr_mutex_destroy(count_lock);
+}
+
+static void 
+event_thread_init(void *drcontext)
+{
+    /* create an instance of our data structure for this thread */
+    bb_counts *counts = (bb_counts *)dr_thread_alloc(drcontext, sizeof(bb_counts));
+    /* store it in the slot provided in the drcontext */
+    dr_set_tls_field(drcontext, counts);
+    memset(counts, 0, sizeof(bb_counts));
+}
+
+static void 
+event_thread_exit(void *drcontext)
+{
+    bb_counts *counts = (bb_counts *) dr_get_tls_field(drcontext);
+    /* NOTE - if we so choose we could report per-thread sizes here. */
+    dr_mutex_lock(count_lock);
+    counts_dynamic.blocks += counts->blocks;
+    counts_dynamic.total_size += counts->total_size;
+    dr_mutex_unlock(count_lock);
+    dr_thread_free(drcontext, counts, sizeof(bb_counts));
+}
+
+static dr_emit_flags_t
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating)
+{
+    uint num_instructions = 0;
+    instr_t *instr, *where = NULL;
+    bool eflags_saved = true;
+    bb_counts *counts = (bb_counts *) dr_get_tls_field(drcontext);
+
+    /* count the number of instructions in this block */
+    for (instr = instrlist_first(bb); instr != NULL; instr = instr_get_next(instr)) {
+        /* Since it doesn't matter where we insert, look for a place 
+         * where the eflags are dead. */
+        uint flags = instr_get_arith_flags(instr);
+        if (TESTALL(EFLAGS_WRITE_6, flags) && !TESTANY(EFLAGS_READ_6, flags)) {
+            where = instr;
+            eflags_saved = false;
+        }
+        num_instructions++;
+    }
+
+    /* update the as-built counts */
+    dr_mutex_lock(as_built_lock);
+    counts_as_built.blocks++;
+    counts_as_built.total_size += num_instructions;
+    if (eflags_saved)
+        bbs_eflags_saved++;
+    dr_mutex_unlock(as_built_lock);
+
+    /* increment counters */
+    if (eflags_saved) {
+        where = instrlist_first(bb);
+        dr_save_arith_flags(drcontext, bb, where, SPILL_SLOT_1);
+    }
+    /* If all thread-private caches we can use direct addressing. */
+    if (!dr_using_all_private_caches()) {
+        /* Spill a register to get a pointer to our TLS structure. */
+        dr_save_reg(drcontext, bb, where, REG_XDI, SPILL_SLOT_2);
+        dr_insert_read_tls_field(drcontext, bb, where, REG_XDI);
+    }
+#ifdef X86_32
+    /* Since the counters are 64-bit we must use an add an addc to increment.
+     * The operations is still effectively atomic since we're only increasing
+     * the count. */
+    instrlist_meta_preinsert(bb, where,
+        INSTR_CREATE_add(drcontext,
+                         dr_using_all_private_caches() ?
+                           OPND_CREATE_ABSMEM((byte *)&counts->blocks, OPSZ_4) : 
+                           OPND_CREATE_MEM32(REG_XDI, offsetof(bb_counts, blocks)),
+                         OPND_CREATE_INT8(1)));
+    instrlist_meta_preinsert(bb, where,
+        INSTR_CREATE_adc(drcontext,
+                         dr_using_all_private_caches() ?
+                           OPND_CREATE_ABSMEM((byte *)&counts->blocks +4, OPSZ_4) :
+                           OPND_CREATE_MEM32(REG_XDI, offsetof(bb_counts, blocks)+4),
+                         OPND_CREATE_INT8(0)));
+
+    instrlist_meta_preinsert(bb, where,
+        INSTR_CREATE_add(drcontext,
+                         dr_using_all_private_caches() ?
+                           OPND_CREATE_ABSMEM((byte *)&counts->total_size, OPSZ_4) :
+                           OPND_CREATE_MEM32(REG_XDI, offsetof(bb_counts, total_size)),
+                         OPND_CREATE_INT_32OR8(num_instructions)));
+    instrlist_meta_preinsert(bb, where,
+        INSTR_CREATE_adc(drcontext,
+                         dr_using_all_private_caches() ?
+                           OPND_CREATE_ABSMEM((byte *)&counts->total_size +4, OPSZ_4) :
+                           OPND_CREATE_MEM32(REG_XDI, offsetof(bb_counts, total_size)+4),
+                         OPND_CREATE_INT8(0)));
+#else /* X86_64 */
+    instrlist_meta_preinsert(bb, where,
+       INSTR_CREATE_inc(drcontext,
+                         dr_using_all_private_caches() ?
+                           OPND_CREATE_ABSMEM((byte *)&counts->blocks, OPSZ_8) :
+                           OPND_CREATE_MEM64(REG_XDI, offsetof(bb_counts, blocks))));
+    instrlist_meta_preinsert(bb, where,
+       INSTR_CREATE_add(drcontext,
+                         dr_using_all_private_caches() ?
+                           OPND_CREATE_ABSMEM((byte *)&counts->total_size, OPSZ_8) :
+                           OPND_CREATE_MEM64(REG_XDI, offsetof(bb_counts, total_size)),
+                        OPND_CREATE_INT_32OR8(num_instructions)));
+#endif
+    if (!dr_using_all_private_caches()) {
+        /* Restore spilled register. */
+        dr_restore_reg(drcontext, bb, where, REG_XDI, SPILL_SLOT_2);
+    }
+    if (eflags_saved) {
+        dr_restore_arith_flags(drcontext, bb, where, SPILL_SLOT_1);
+    }
+
+    return DR_EMIT_DEFAULT;
+}
+\endcode
+
+[ \ref API_tutorial_bbdynsize8 "prev" | \ref API_tutorial "back" ]
+
+**/
+
+
+/**
+\page API_tutorial_steal_reg1
+<b>coming soon</b>
+
+[ \ref API_tutorial "back" ]
+**/
+
+
+/**
+\page API_tutorial_prefetch1
+<b>coming soon</b>
+
+[ \ref API_tutorial "back" ]
+**/
diff --git a/api/docs/viper.fig b/api/docs/viper.fig
new file mode 100644
index 000000000..6e7373d57
--- /dev/null
+++ b/api/docs/viper.fig
@@ -0,0 +1,106 @@
+#FIG 3.2  Produced by xfig version 3.2.5
+Landscape
+Center
+Inches
+A4      
+100.00
+Single
+-2
+1200 2
+0 32 #f7cb66
+0 33 #995e3c
+0 34 #6f300f
+0 35 #969696
+0 36 #ccccb1
+0 37 #3366ff
+6 1725 1050 3075 1950
+2 4 0 1 37 37 50 -1 62 0.000 0 0 11 0 0 5
+	 3075 1050 1725 1050 1725 1950 3075 1950 3075 1050
+4 1 7 49 -1 18 12 0.0000 4 210 1095 2400 1725 Agent App\001
+4 1 7 49 -1 18 12 0.0000 4 210 870 2400 1425 Security\001
+-6
+6 1725 2700 3075 3600
+2 4 0 1 35 35 50 -1 62 0.000 0 0 11 0 0 5
+	 3075 2700 1725 2700 1725 3600 3075 3600 3075 2700
+4 1 7 49 -1 18 12 0.0000 4 165 720 2400 3075 VMsafe\001
+4 1 7 49 -1 18 12 0.0000 4 165 1035 2400 3375 Front-End\001
+-6
+6 4200 1200 5550 1950
+2 4 0 1 37 37 50 -1 62 0.000 0 0 11 0 0 5
+	 5550 1200 4200 1200 4200 1950 5550 1950 5550 1200
+4 1 7 49 -1 18 12 0.0000 4 210 615 4875 1800 Agent\001
+4 1 7 49 -1 18 12 0.0000 4 165 615 4875 1500 Guest\001
+-6
+6 4200 2100 5550 2850
+2 4 0 1 37 37 50 -1 62 0.000 0 0 11 0 0 5
+	 5550 2100 4200 2100 4200 2850 5550 2850 5550 2100
+4 1 7 49 -1 18 12 0.0000 4 165 810 4875 2700 Handler\001
+4 1 7 49 -1 18 12 0.0000 4 165 795 4875 2400 Internal\001
+-6
+6 5700 750 6900 2550
+2 1 0 6 8 8 50 -1 -1 0.000 0 0 -1 1 1 2
+	1 1 2.00 150.00 75.00
+	1 1 2.00 150.00 75.00
+	 6300 1350 6300 1800
+2 4 0 1 35 35 50 -1 62 0.000 0 0 7 0 0 5
+	 6750 1800 5850 1800 5850 2400 6750 2400 6750 1800
+2 4 0 1 37 37 50 -1 62 0.000 0 0 7 0 0 5
+	 6675 1350 5925 1350 5925 900 6675 900 6675 1350
+2 4 0 1 36 36 51 -1 62 0.000 0 0 7 0 0 5
+	 6900 750 5700 750 5700 2550 6900 2550 6900 750
+4 1 7 49 -1 18 12 0.0000 4 165 660 6300 2175 VIPER\001
+4 1 7 49 -1 18 12 0.0000 4 165 510 6300 1200 VIPA\001
+-6
+6 7050 750 8250 2550
+2 1 0 6 8 8 50 -1 -1 0.000 0 0 -1 1 1 2
+	1 1 2.00 150.00 75.00
+	1 1 2.00 150.00 75.00
+	 7650 1350 7650 1800
+2 4 0 1 36 36 51 -1 62 0.000 0 0 7 0 0 5
+	 8250 750 7050 750 7050 2550 8250 2550 8250 750
+2 4 0 1 37 37 50 -1 62 0.000 0 0 7 0 0 5
+	 8025 1350 7275 1350 7275 900 8025 900 8025 1350
+2 4 0 1 35 35 50 -1 62 0.000 0 0 7 0 0 5
+	 8100 1800 7200 1800 7200 2400 8100 2400 8100 1800
+4 1 7 49 -1 18 12 0.0000 4 165 510 7650 1200 VIPA\001
+4 1 7 49 -1 18 12 0.0000 4 165 660 7650 2175 VIPER\001
+-6
+6 5475 3450 6825 4350
+2 4 0 1 35 35 50 -1 62 0.000 0 0 11 0 0 5
+	 6825 3450 5475 3450 5475 4350 6825 4350 6825 3450
+4 1 7 49 -1 18 12 0.0000 4 165 720 6150 3825 VMsafe\001
+4 1 7 49 -1 18 12 0.0000 4 165 975 6150 4125 Back-End\001
+-6
+2 4 0 1 34 34 51 -1 62 0.000 0 0 7 0 0 5
+	 8400 3300 4050 3300 4050 2700 8400 2700 8400 3300
+2 4 0 1 33 33 51 -1 62 0.000 0 0 7 0 0 5
+	 3600 4500 1200 4500 1200 3900 3600 3900 3600 4500
+2 4 1 2 0 7 52 -1 20 9.000 0 0 7 0 0 5
+	 8400 2700 4050 2700 4050 600 8400 600 8400 2700
+2 4 0 1 33 33 51 -1 62 0.000 0 0 7 0 0 5
+	 8400 4500 4050 4500 4050 3300 8400 3300 8400 4500
+2 1 0 6 8 8 50 -1 -1 0.000 0 0 -1 1 1 2
+	1 1 2.00 150.00 75.00
+	1 1 2.00 150.00 75.00
+	 4875 2850 5550 3525
+2 4 0 1 32 32 50 -1 62 0.000 0 0 7 0 0 5
+	 8400 5100 1200 5100 1200 4500 8400 4500 8400 5100
+2 4 0 1 34 34 51 -1 62 0.000 0 0 7 0 0 5
+	 3600 3900 1200 3900 1200 2250 3600 2250 3600 3900
+2 1 0 8 8 8 50 -1 -1 0.000 0 0 -1 1 1 2
+	1 1 2.00 150.00 75.00
+	1 1 2.00 150.00 75.00
+	 3075 3150 5475 3900
+2 1 0 6 8 8 50 -1 -1 0.000 0 0 -1 1 1 2
+	1 1 2.00 150.00 75.00
+	1 1 2.00 150.00 75.00
+	 2400 1950 2400 2700
+2 4 0 1 36 36 51 -1 62 0.000 0 0 7 0 0 5
+	 3600 600 1200 600 1200 2250 3600 2250 3600 600
+4 1 7 49 -1 18 12 0.0000 4 165 345 2400 4275 VM\001
+4 1 0 50 -1 18 12 0.0000 4 165 1095 4800 975 Processes\001
+4 1 7 49 -1 18 12 0.0000 4 165 330 6150 3075 OS\001
+4 1 0 49 -1 18 12 0.0000 4 210 4065 4725 4875 Virtual Infrastructure (Single ESX Host)\001
+4 1 7 50 -1 18 12 0.0000 4 165 555 1800 2550 Linux\001
+4 1 0 50 -1 18 12 0.0000 4 210 1755 2400 900 Security Monitor\001
+4 1 7 50 -1 18 12 0.0000 4 165 345 7575 3975 VM\001
diff --git a/api/docs/windows.fig b/api/docs/windows.fig
new file mode 100644
index 000000000..050dddd76
--- /dev/null
+++ b/api/docs/windows.fig
@@ -0,0 +1,74 @@
+#FIG 3.2
+Landscape
+Center
+Inches
+Letter  
+100.00
+Single
+-2
+1200 2
+0 32 #000000
+0 33 #dddddd
+0 34 #ffffaa
+0 35 #0000ff
+0 36 #6666ff
+0 37 #aaaaff
+0 38 #ddddff
+0 39 #77ff00
+0 40 #aaffaa
+0 41 #dd00ff
+0 42 #ffaaaa
+6 0 1350 3150 3525
+2 2 0 1 0 37 54 0 20 0.000 0 0 -1 0 0 5
+	 1050 3150 1050 2700 2850 2700 2850 3150 1050 3150
+2 1 0 1 0 7 51 0 -1 0.000 1 0 -1 1 1 2
+	1 1 1.00 60.00 90.00
+	1 1 1.00 60.00 120.00
+	 1950 1800 1950 2700
+2 2 0 0 7 33 54 0 20 0.000 0 0 -1 0 0 5
+	 750 2400 750 2100 3150 2100 3150 2400 750 2400
+2 2 0 1 0 38 51 0 20 0.000 0 0 -1 0 0 5
+	 1050 1800 1050 1350 2850 1350 2850 1800 1050 1800
+2 1 0 4 41 7 50 0 -1 0.000 0 0 -1 1 0 2
+	1 1 4.00 120.00 180.00
+	 225 2250 750 2250
+4 0 0 50 0 0 12 0.0000 4 180 1305 1350 3000 operating system\001
+4 0 0 50 0 0 12 0.0000 4 180 1560 1200 2325 system call gateway\001
+4 0 0 50 0 0 12 0.0000 4 180 825 1500 1650 application\001
+4 0 0 50 0 0 14 0.0000 4 150 495 1650 3525 Linux\001
+4 0 -1 50 0 0 12 0.0000 4 180 165 37 2295 a)\001
+-6
+6 3600 0 6750 3525
+2 1 0 1 0 7 51 0 -1 0.000 1 0 -1 1 1 2
+	1 1 1.00 60.00 90.00
+	1 1 1.00 60.00 120.00
+	 5550 450 5550 1350
+2 1 0 1 0 7 51 0 -1 0.000 1 0 -1 1 1 2
+	1 1 1.00 60.00 90.00
+	1 1 1.00 60.00 120.00
+	 5550 1800 5550 2700
+2 2 0 1 0 37 54 0 20 0.000 0 0 -1 0 0 5
+	 4650 3150 4650 2700 6450 2700 6450 3150 4650 3150
+2 2 0 0 7 33 54 0 20 0.000 0 0 -1 0 0 5
+	 4350 2400 4350 2100 6750 2100 6750 2400 4350 2400
+2 2 0 0 7 33 54 0 20 0.000 0 0 -1 0 0 5
+	 4350 1050 4350 750 6750 750 6750 1050 4350 1050
+2 2 0 1 0 42 51 0 20 0.000 0 0 -1 0 0 5
+	 4650 1800 4650 1350 6450 1350 6450 1800 4650 1800
+2 2 0 1 0 38 51 0 20 0.000 0 0 -1 0 0 5
+	 4650 450 4650 0 6450 0 6450 450 4650 450
+2 1 0 4 41 7 50 0 -1 0.000 0 0 -1 1 0 2
+	1 1 4.00 120.00 180.00
+	 3825 2250 4350 2250
+2 1 0 4 41 7 50 0 -1 0.000 0 0 -1 1 0 2
+	1 1 4.00 120.00 180.00
+	 3825 900 4350 900
+4 0 0 50 0 0 12 0.0000 4 180 825 5100 300 application\001
+4 0 0 50 0 0 12 0.0000 4 180 1560 4800 2325 system call gateway\001
+4 0 0 50 0 0 12 0.0000 4 180 1305 4950 3000 operating system\001
+4 0 0 50 0 0 12 0.0000 4 135 975 5175 1650 Win32 DLLs\001
+4 0 0 50 0 0 12 0.0000 4 135 840 5100 975 Win32 API\001
+4 0 0 50 0 0 14 0.0000 4 150 810 5175 3525 Windows\001
+4 0 -1 50 0 0 12 0.0000 4 180 165 3623 953 b)\001
+4 0 -1 50 0 0 12 0.0000 4 180 165 3630 2302 c)\001
+-6
diff --git a/api/samples/MF_moduledb-sample.config b/api/samples/MF_moduledb-sample.config
new file mode 100644
index 000000000..4a84fde10
--- /dev/null
+++ b/api/samples/MF_moduledb-sample.config
@@ -0,0 +1 @@
+foo.dll                                                                                                                                                                                                                                                             yyybar.dll                                                                                                                                                                                                                                                             ynnVIPA_Test.exe                                                                                                                                                                                                                                                       nyytest.dll                                                                                                                                                                                                                                                             nnn
\ No newline at end of file
diff --git a/api/samples/MF_moduledb.c b/api/samples/MF_moduledb.c
new file mode 100644
index 000000000..05b7e24a6
--- /dev/null
+++ b/api/samples/MF_moduledb.c
@@ -0,0 +1,341 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Usage :
+ * Should be run in DR_MODE_MEMORY_FIREWALL mode.
+ *
+ * Expects a single -client_ops option: the full absolute path to a configuration file
+ * (for example '-client_ops "C:\MF_moduledb_VIPA\MF_moduledb-sample.config"').
+ *
+ * The configuration file is specified as a series of table_value_t structures defined
+ * below with no line breaks or extra padding.  Each table_value_t structure consists
+ * of a module_name padded with spaces to MAXIMUM_PATH in length followed by three
+ * 'y' or 'n' letters specifying the allow_to_stack, allow_to_heap, and allow_to_here
+ * modes.
+ *
+ * If allow_to_stack is 'y', execution is allowed to go from the module specified to
+ * a violating location on the stack. If allow_to_heap is 'y', execution is allowed to
+ * go from the module specified to a violating location on the heap. If allow_to_here is
+ * 'y', violating transfers targeting the module are allowed.
+ *
+ * A sample config file is included: MF_moduledb-sample.config that is set up to work
+ * with VIPA_test.exe (also in the sample folder). The VIPA_test.exe program has two
+ * buttons, one to generate a stack overflow attack and one to generate a heap
+ * overflow attack.  The sample config file MF_moduledb-sample.config is set up to
+ * allow the heap attack, but not the stack attack. To demonstrate :
+ *
+ * Use drdeploy.exe to configure VIPA_test.exe to run under security_api mode with
+ * the appropriate options.
+ * drdeploy.exe -reg VIPA_test.exe -root <root path> -mode security_api -client <path
+ * to MF_moduledb.dll> -ops "-client_ops <path to MF_moduledb-sample.config>"
+ *
+ * Then run VIPA_test.exe. Clicking on the heap attack button should produce messages
+ * that a potential security violation is being allowed.  Clicking on the stack attack
+ * button should produce a message that a potential security violation is being blocked
+ * by killing the process.
+ * 
+ * Use 'drdeploy.exe -nudge VIPA_text.exe' to nudge the process to re-read the
+ * configuration file.
+ *
+ * The #define VERBOSE and #define VVERBOSE can be adjusted below to generate more
+ * verbose logging.
+ */
+
+#include "dr_api.h"
+
+#define VERBOSE 0
+#define VVERBOSE 0
+#define USE_MESSAGEBOX 1
+
+#define NAME "MF_moduledb"
+
+#ifdef SHOW_RESULTS
+# if defined(WINDOWS) && USE_MESSAGEBOX
+#  define DISPLAY_FUNC dr_messagebox
+# else /* no messageboxes, use dr_printf() instead */
+#  define DISPLAY_FUNC dr_printf
+# endif
+#else
+# define DISPLAY_FUNC dummy_func
+#endif
+
+#if VERBOSE
+# define VDISPLAY_FUNC DISPLAY_FUNC
+#else
+# define VDISPLAY_FUNC dummy_func
+#endif
+#if VVERBOSE
+# define VVDISPLAY_FUNC DISPLAY_FUNC
+#else
+# define VVDISPLAY_FUNC dummy_func
+#endif
+
+/* used to avoid displaying a string */
+static void dummy_func(char *fmt, ...) {}
+
+typedef struct {
+    char module_name[MAXIMUM_PATH];
+
+    /* exempt transfers to violating stack regions from this module */
+    char allow_to_stack; /* either 'y' or 'n' */
+    /* exempt transfers to violating heap regions from this module */
+    char allow_to_heap; /* either 'y' or 'n' */
+    /* exempt violating transfers to this module */
+    char allow_to_here; /* either 'y' or 'n' */
+
+    /* Additional relaxation options such as allowing violating code origins regions
+     * within this module (some .data sections for ex.) could be added here. */
+} table_value_t;
+
+struct _table_entry_t;
+typedef struct _table_entry_t table_entry_t;
+
+struct _table_entry_t {
+    table_value_t value; /* the entry we read out of the config file */
+    table_entry_t *next; /* table is just a singly linked list */
+}; /* table_entry_t */
+
+static const char *table_def_file_name; /* name of configuration file */
+static void *table_lock; /* for multithreaded access to the table */
+static table_entry_t *table = NULL; /* table of relaxations */
+
+static void event_security_violation(void *drcontext, void *source_tag,
+                                     app_pc source_pc, app_pc target_pc,
+                                     dr_security_violation_type_t violation,
+                                     dr_mcontext_t *mcontext,
+                                     dr_security_violation_action_t *action);
+static void event_nudge(void *drcontext, uint64 argument);
+static void event_about_to_terminate_nudges(int total_non_app_threads,
+                                            int live_non_app_threads);
+static void event_exit(void);
+
+static table_entry_t * get_entry_for_address(app_pc addr);
+static void read_table();
+static void free_table();
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    VDISPLAY_FUNC(NAME" initializing.");
+
+    /* register the events we wish to handle */
+    dr_register_security_event(event_security_violation);
+    dr_register_nudge_event(event_nudge, id);
+    dr_register_exit_event(event_exit);
+    
+    /* read the client options */
+    table_def_file_name = dr_get_options(id);
+    if (table_def_file_name == NULL ||
+        table_def_file_name[0] == '\0') {
+        DISPLAY_FUNC(NAME" requires the table name as parameter\n");
+        dr_abort();
+    }
+
+    /* initialize structures */
+    table_lock = dr_mutex_create();
+    read_table();
+}
+
+static void
+event_exit(void)
+{
+    VDISPLAY_FUNC(NAME" exiting.");
+
+    /* free structures */
+    free_table();
+    dr_mutex_destroy(table_lock);
+}
+
+static void
+event_nudge(void *drcontext, uint64 argument)
+{
+    DISPLAY_FUNC(NAME" received nudge event; re-reading config file.");
+
+    /* An external process has nudged us with dr_nudge_process() telling us
+     * to re-read the configuration file. */
+    dr_mutex_lock(table_lock);
+    free_table();
+    read_table();
+    dr_mutex_unlock(table_lock);
+}
+
+#ifdef WINDOWS
+/* Lookup the module containing addr and see if we have a table entry for it */
+static table_entry_t *
+get_entry_for_address(app_pc addr)
+{
+    module_data_t *data = dr_lookup_module(addr);
+    table_entry_t *entry = NULL;
+    if (data != NULL) {
+        entry = table;
+        while (entry != NULL &&
+               _stricmp(dr_module_preferred_name(data), entry->value.module_name) != 0) {
+            entry = entry->next;
+        }
+        dr_free_module_data(data);
+    }
+    return entry;
+}
+#endif
+
+static void
+event_security_violation(void *drcontext, void *source_tag,
+                         app_pc source_pc, app_pc target_pc,
+                         dr_security_violation_type_t violation,
+                         dr_mcontext_t *mcontext,
+                         dr_security_violation_action_t *action)
+{
+    /* A potential security violation was detected. */
+    char *violation_str = NULL; /* a name for this violation */
+    bool allow = false; /* should we let execution continue */
+
+#ifdef WINDOWS
+    /* find the module we came from */
+    app_pc source = (source_pc == NULL ? source_tag : source_pc);
+    table_entry_t *entry;
+
+    /* check our source relaxations */
+    entry = get_entry_for_address(source);
+    if (entry != NULL) {
+        /* we have a match, check the relaxations */
+        if (violation == DR_RCO_STACK_VIOLATION &&
+            (entry->value.allow_to_stack == 'y' || entry->value.allow_to_stack == 'Y')) {
+            allow = true;
+        }
+        if (violation == DR_RCO_HEAP_VIOLATION &&
+            (entry->value.allow_to_heap == 'y' || entry->value.allow_to_heap == 'Y')) {
+            allow = true;
+        }
+    }
+    
+    /* check our target relaxations */
+    entry = get_entry_for_address(target_pc);
+    if (entry != NULL) {
+        /* we have a match, check the relaxations */
+        if ((violation == DR_RCT_RETURN_VIOLATION ||
+             violation == DR_RCT_INDIRECT_CALL_VIOLATION ||
+             violation == DR_RCT_INDIRECT_JUMP_VIOLATION) &&
+            (entry->value.allow_to_here == 'y' || entry->value.allow_to_here == 'Y')) {
+            allow = true;
+        }
+    }
+    
+#endif
+    
+    switch (violation) {
+    case DR_RCO_STACK_VIOLATION:
+        violation_str = "stack execution violation";
+        break;
+    case DR_RCO_HEAP_VIOLATION:
+        violation_str = "heap execution violation";
+        break;
+    case DR_RCT_RETURN_VIOLATION:
+        violation_str = "return target violation";
+        break;
+    case DR_RCT_INDIRECT_CALL_VIOLATION:
+        violation_str = "call transfer violation";
+        break;
+    case DR_RCT_INDIRECT_JUMP_VIOLATION:
+        violation_str = "jump transfer violation";
+        break;
+    default:
+        violation_str = "unknown";
+        break;
+    }
+
+    if (allow)
+        *action = DR_VIOLATION_ACTION_CONTINUE;
+
+    /* could use dr_write_forensics_report() here to log additional information */
+    
+    DISPLAY_FUNC("WARNING - possible security violation \"%s\" detected, %s.",
+                 violation_str, allow ? "allowing" : "blocking");
+}
+
+static void
+read_table()
+{
+    file_t file;
+    bool read_entry = true;
+
+    file = dr_open_file(table_def_file_name, DR_FILE_READ);
+    if (file == INVALID_FILE) {
+        DISPLAY_FUNC(NAME" error opening config file \"%s\"\n", table_def_file_name);
+        return;
+    }
+
+    VVDISPLAY_FUNC(NAME" reading config file: \"%s\"\n", table_def_file_name);
+
+    do {
+        table_entry_t *entry = (table_entry_t *)dr_global_alloc(sizeof(table_entry_t));
+        if (dr_read_file(file, &entry->value, sizeof(table_value_t)) !=
+            sizeof(table_value_t)) {
+            /* end of file */
+            read_entry = false;
+            dr_global_free(entry, sizeof(table_entry_t));
+        } else {
+            int i;
+            /* insert NULL termination for module name (including space padding) */
+            for (i = sizeof(entry->value.module_name) - 1;
+                 i >= 0 && entry->value.module_name[i] == ' '; i--) {
+                entry->value.module_name[i] = '\0';
+            }
+            /* just in case */
+            entry->value.module_name[sizeof(entry->value.module_name)-1] = '\0';
+
+            /* add to the table */
+            entry->next = table;
+            table = entry;
+            VVDISPLAY_FUNC(NAME" read entry for module=\"%s\" to_stack=%s to_heap=%s "
+                           "transfer_to_here=%s\n", entry->value.module_name,
+                           (entry->value.allow_to_stack == 'y' ||
+                            entry->value.allow_to_stack == 'Y') ? "yes" : "no",
+                           (entry->value.allow_to_heap == 'y' ||
+                            entry->value.allow_to_heap == 'Y') ? "yes" : "no",
+                           (entry->value.allow_to_here == 'y' ||
+                            entry->value.allow_to_here == 'Y') ? "yes" : "no");
+        }
+    } while (read_entry);
+    VVDISPLAY_FUNC(NAME" done reading config file.");
+}
+
+static void
+free_table()
+{
+    /* Free all table entries */
+    table_entry_t *next;
+    while (table != NULL) {
+        next = table->next;
+        dr_global_free(table, sizeof(table_entry_t));
+        table = next;
+    }
+}
diff --git a/api/samples/Makefile b/api/samples/Makefile
new file mode 100644
index 000000000..f0a16bfda
--- /dev/null
+++ b/api/samples/Makefile
@@ -0,0 +1,221 @@
+# **********************************************************
+# Copyright (c) 2008-2009 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+###########################################################################
+###########################################################################
+###
+### Makefile for DynamoRIO samples
+### Requires GNU make for conditionals.
+### On Windows requires cygwin cygpath and standard VC command-line setup
+### (cl.exe on path, LIB and INCLUDE environment vars set).
+### Use ARCH=x64 to build for 64bit.
+### Use DEBUG=1 to link with the debug build.
+###
+
+.PHONY: clean all
+
+GCC := gcc
+GXX := g++
+CL := cl
+
+ifndef ARCH
+  # default is x86 instead of x64
+  ARCH := x86
+endif
+ifndef DEBUG
+  # default is release build
+  DEBUG := 0
+endif
+
+ifndef DYNAMORIO_HOME
+  # assume we're in docs/html/samples/
+  DYNAMORIO_HOME := ../../..
+endif
+
+ifeq ($(shell uname -s | cut -c1-5), Linux)
+  MACHINE := linux
+  WD := $(shell pwd)
+else
+  MACHINE := win32
+  WD := $(shell cygpath -w $(shell pwd))
+  # rely on the env vars used for cmd-line building with cl.exe
+  SYS_INCLUDE := /I$(subst ;, /I, $(INCLUDE))
+  SYS_LIB :=  /libpath:$(subst ;, /libpath:,$(LIB))
+  DYNAMORIO_HOME := $(shell cygpath -ma $(DYNAMORIO_HOME))
+endif
+
+ifeq ($(DEBUG), 0)
+LIBDIR := release
+else
+LIBDIR := debug
+endif
+
+ifeq ($(ARCH), x64)
+  DR_LIBPATH := $(DYNAMORIO_HOME)/lib64/$(LIBDIR)
+else
+  DR_LIBPATH := $(DYNAMORIO_HOME)/lib32/$(LIBDIR)
+endif
+DR_LIB := dynamorio.lib
+DR_INCLUDE := $(DYNAMORIO_HOME)/include
+
+ifeq ($(MACHINE), linux)
+  D := -D
+  DBG := -g3
+  SRC_EXCLUDE := stats.c
+  ifeq ($(ARCH), x64)
+    CFLAGS := -m64 $(DBG) $(D)LINUX
+    # Note - it's currently required on 64bit that a client library have a
+    # preferred base address in the lower 2GB, to be reachable by our
+    # library and code cache.  This is due to limitations in the current
+    # experimental 64bit release and will be adressed in later releases.
+    # Naturally for multiple clients different addresses should be used.
+    PREFERRED_BASE := 0x72000000
+    LD_SCRIPT := ldscript
+    LD_USE_SCRIPT := -Xlinker -dT -Xlinker $(LD_SCRIPT)
+  else
+    CFLAGS := -m32 $(DBG) $(D)LINUX
+    LD_USE_SCRIPT :=
+  endif
+  SLIB_SUFFIX := .so
+  SEXE_SUFFIX :=
+  CXXFLAGS += `$(GXX) -print-file-name=libstdc++.a`
+  CXXFLAGS += `$(GXX) -print-file-name=libgcc.a`
+  CXXFLAGS += `$(GXX) -print-file-name=libgcc_eh.a`
+  CXXFLAGS += -Xlinker -wrap=malloc -Xlinker -wrap=realloc -Xlinker -wrap=free
+else
+  D := /D
+  DBG := /Zi
+  CFLAGS := /nologo /GS- $(DBG) $(D)WINDOWS
+  SRC_EXCLUDE := stl_test.cpp
+  ifeq ($(ARCH), x64)
+    # Note - it's currently required on 64bit that /base is used to specify a
+    # preferred base address such that the resulting instrumentation dll is
+    # loaded in the range 0x72000000-0x75000000. To ensure that we also add
+    # /FIXED.  This is due to limitations in the current experimental 64bit release
+    # and will be adressed in later releases.
+    # Naturally for multiple clients different addresses should be used.
+    LFLAGS := /machine:X64 /base:"0x72000000" /FIXED
+    SRC_EXCLUDE += stats.c
+  else
+    LFLAGS :=
+  endif
+  SLIB_SUFFIX := .dll
+  SEXE_SUFFIX := .exe
+endif
+
+# standalone client isn't easy to run in short regressions
+SRCS_STANDALONE := tracedump.c
+SEXES := $(subst .c,$(SEXE_SUFFIX),$(SRCS_STANDALONE))
+
+SRCS_LIB := $(filter-out $(SRC_EXCLUDE) $(EXCLUDE) $(SRCS_STANDALONE),$(wildcard *.c *.cpp))
+SLIBS := $(subst .c,$(SLIB_SUFFIX),$(subst .cpp,$(SLIB_SUFFIX),$(SRCS_LIB)))
+
+SHOW_RESULTS := 1
+ifeq ($(SHOW_RESULTS), 1)
+  CFLAGS += $(D)SHOW_RESULTS
+endif
+ifeq ($(ARCH), x64)
+  CFLAGS += $(D)X86_64
+else
+  CFLAGS += $(D)X86_32
+endif
+
+all: $(SLIBS) $(SEXES)
+
+listsrcs:
+	@echo $(SRCS_LIB)
+
+# shared library clients
+ifeq ($(MACHINE), linux)
+  ifeq ($(ARCH), x64)
+$(LD_SCRIPT):
+    # To set a preferred base address we tweak the default linker script
+	ld --verbose | sed '/SECTIONS/,/^}/!d' | \
+	  sed 's/0x400000/$(PREFERRED_BASE)/g; /^{/a . = $(PREFERRED_BASE);' > $@
+  endif
+
+%.so: %.c $(LD_SCRIPT)
+    # avoid SElinux text relocation security violations by explicitly requesting PIC
+	$(GCC) $(CFLAGS) -fPIC -shared -nostartfiles -I$(DR_INCLUDE) $(SYS_INCLUDE) \
+          $< -o $@ $(LD_USE_SCRIPT)
+    # client libraries need to be executable by all, otherwise the
+    # client library may not be loaded if run as a different user
+	chmod ugo+rx "$(WD)/$@"
+	@echo "---------------------------------------------------------------------------"
+	@echo "Usage: pass to drdeploy :"
+	@echo "  -client $(WD)/$@ 0 \"\""
+
+%.so: %.cpp $(LD_SCRIPT)
+    # avoid SElinux text relocation security violations by explicitly requesting PIC
+    # Note that if we list the source file before the .a files (in CXXFLAGS) we
+    # get ld warnings on x64 so be sure to list source file last.
+	$(GXX) $(CFLAGS) $(CXXFLAGS) -fPIC -shared -nostartfiles -I$(DR_INCLUDE) \
+          $(SYS_INCLUDE) $< -o $@ $(LD_USE_SCRIPT)
+else
+%.dll: %.c
+	$(CL) $(CFLAGS) $< /I$(DR_INCLUDE) $(SYS_INCLUDE) /link $(LFLAGS) $(SYS_LIB) \
+          /libpath:$(DR_LIBPATH) $(DR_LIB) /dll /out:$@ /nodefaultlib /noentry
+endif
+    # client libraries need to be executable by all, otherwise the
+    # client library may not be loaded if run as a different user
+	chmod ugo+rx "$(WD)/$@"
+	@echo "---------------------------------------------------------------------------"
+	@echo "Usage: pass to drdeploy :"
+ifeq ($(MACHINE), linux)
+	@echo "  -client $(WD)/$@ 0 \"\""
+else
+	@echo "  -client '$(WD)\\$@' 0 \"\""
+endif
+
+# standalone programs
+ifeq ($(MACHINE), linux)
+%: %.c
+	$(GCC) $(CFLAGS) -I$(DR_INCLUDE) $(SYS_INCLUDE) -L$(DR_LIBPATH) -ldynamorio \
+          -Xlinker -rpath -Xlinker $(DR_LIBPATH) $< -o $@
+	@echo "rpath is set, but if libdynamorio.so is moved, LD_LIBRARY_PATH will need to be set"
+else
+%.exe: %.c
+	$(CL) $(CFLAGS) $< /I$(DR_INCLUDE) $(SYS_INCLUDE) /link /nodefaultlib $(SYS_LIB) \
+          /libpath:$(DR_LIBPATH) libcmt.lib $(DR_LIB) kernel32.lib user32.lib /out:$@
+	@echo "---------------------------------------------------------------------------"
+	@echo "Make sure the loader finds dynamorio.dll (e.g., copy to same dir)"
+endif
+
+clean:
+ifeq ($(MACHINE), linux)
+	-rm -f *.so
+	-rm -f *.log
+  ifeq ($(ARCH), x64)
+	-rm -f $(LD_SCRIPT)
+  endif
+else
+	-rm -f *.{dll,exp,lib,obj,pdb,ilk,log}
+endif
+	-rm -f $(SEXES)
diff --git a/api/samples/VIPA_test.exe b/api/samples/VIPA_test.exe
new file mode 100755
index 000000000..5e8525ecb
Binary files /dev/null and b/api/samples/VIPA_test.exe differ
diff --git a/api/samples/bbcount.c b/api/samples/bbcount.c
new file mode 100644
index 000000000..47f166365
--- /dev/null
+++ b/api/samples/bbcount.c
@@ -0,0 +1,164 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Code Manipulation API Sample:
+ * bbcount.c
+ *
+ * Reports the dynamic execution count of all basic blocks.
+ * Illustrates how to perform performant inline increments with analysis
+ * on whether flags need to be preserved.
+ */
+
+#include <stddef.h> /* for offsetof */
+#include "dr_api.h"
+
+#ifdef WINDOWS
+# define DISPLAY_STRING(msg) dr_messagebox(msg)
+#else
+# define DISPLAY_STRING(msg) dr_printf("%s\n", msg);
+#endif
+
+#define NULL_TERMINATE(buf) buf[(sizeof(buf)/sizeof(buf[0])) - 1] = '\0'
+
+#define TESTALL(mask, var) (((mask) & (var)) == (mask))
+#define TESTANY(mask, var) (((mask) & (var)) != 0)
+
+/* we only have a global count */
+static int global_count;
+
+/* If being off a little bit is not important, or the target
+ * application is single-threaded or spends most of its time in one
+ * thread, performing a racy inc (i.e., not synchronized among threads)
+ * is three times faster than synchronizing.
+ */
+#define RACY_INC 1
+
+#ifdef SHOW_RESULTS
+/* some meta-stats: static (not per-execution) */
+static int bbs_eflags_saved;
+static int bbs_no_eflags_saved;
+#endif
+
+static void event_exit(void);
+static void event_thread_init(void *drcontext);
+static void event_thread_exit(void *drcontext);
+static dr_emit_flags_t event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                                         bool for_trace, bool translating);
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    /* register events */
+    dr_register_exit_event(event_exit);
+    dr_register_bb_event(event_basic_block);
+
+    /* make it easy to tell, by looking at log file, which client executed */
+    dr_log(NULL, LOG_ALL, 1, "Client 'bbcount' initializing\n");
+#ifdef SHOW_RESULTS
+    /* also give notification to stderr */
+    if (dr_is_notify_on())
+        dr_fprintf(STDERR, "Client bbcount is running\n");
+#endif
+}
+
+static void 
+event_exit(void)
+{
+#ifdef SHOW_RESULTS
+    char msg[512];
+    int len;
+    len = snprintf(msg, sizeof(msg)/sizeof(msg[0]),
+                   "Instrumentation results:\n"
+                   "%10d basic block executions\n"
+                   "%10d basic blocks needed flag saving\n"
+                   "%10d basic blocks did not\n",
+                   global_count, bbs_eflags_saved, bbs_no_eflags_saved);
+    DR_ASSERT(len > 0);
+    NULL_TERMINATE(msg);
+    DISPLAY_STRING(msg);
+#endif /* SHOW_RESULTS */
+}
+
+static dr_emit_flags_t
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating)
+{
+    instr_t *instr, *first = instrlist_first(bb);
+    uint flags;
+    
+#ifdef VERBOSE
+    dr_printf("in dynamorio_basic_block(tag="PFX")\n", tag);
+# ifdef VERBOSE_VERBOSE
+    instrlist_disassemble(drcontext, tag, bb, STDOUT);
+# endif
+#endif
+
+    /* Our inc can go anywhere, so find a spot where flags are dead. */
+    for (instr = first; instr != NULL; instr = instr_get_next(instr)) {
+        flags = instr_get_arith_flags(instr);
+        /* OP_inc doesn't write CF but not worth distinguishing */
+        if (TESTALL(EFLAGS_WRITE_6, flags) && !TESTANY(EFLAGS_READ_6, flags))
+            break;
+    }
+
+    if (instr == NULL)
+        dr_save_arith_flags(drcontext, bb, first, SPILL_SLOT_1);
+    /* Increment the global counter using the lock prefix to make it atomic
+     * across threads.
+     */
+#ifdef RACY_INC
+    instrlist_meta_preinsert
+        (bb, (instr == NULL) ? first : instr,
+         INSTR_CREATE_inc(drcontext, OPND_CREATE_ABSMEM
+                          ((byte *)&global_count, OPSZ_4)));
+#else
+    instrlist_meta_preinsert
+        (bb, (instr == NULL) ? first : instr,
+         LOCK(INSTR_CREATE_inc(drcontext, OPND_CREATE_ABSMEM
+                               ((byte *)&global_count, OPSZ_4))));
+#endif
+    if (instr == NULL)
+        dr_restore_arith_flags(drcontext, bb, first, SPILL_SLOT_1);
+    
+#ifdef SHOW_RESULTS
+    if (instr == NULL)
+        bbs_eflags_saved++;
+    else
+        bbs_no_eflags_saved++;
+#endif
+
+#if defined(VERBOSE) && defined(VERBOSE_VERBOSE)
+    dr_printf("Finished instrumenting dynamorio_basic_block(tag="PFX")\n", tag);
+    instrlist_disassemble(drcontext, tag, bb, STDOUT);
+#endif
+    return DR_EMIT_DEFAULT;
+}
diff --git a/api/samples/bbsize.c b/api/samples/bbsize.c
new file mode 100644
index 000000000..0d15c376f
--- /dev/null
+++ b/api/samples/bbsize.c
@@ -0,0 +1,131 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Code Manipulation API Sample:
+ * bbsize.c
+ *
+ * Reports basic statistics on the sizes of all basic blocks in the
+ * target application.  Illustrates how to preserve floating point
+ * state in an event callback.
+ */
+
+#include "dr_api.h"
+
+#ifdef WINDOWS
+# define DISPLAY_STRING(msg) dr_messagebox(msg)
+#else
+# define DISPLAY_STRING(msg) dr_printf("%s\n", msg);
+#endif
+
+static void *stats_mutex; /* for multithread support */
+static int num_bb;
+static double ave_size;
+static int max_size;
+
+static void event_exit(void);
+static dr_emit_flags_t event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                                         bool for_trace, bool translating);
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    num_bb = 0;
+    ave_size = 0.;
+    max_size = 0;
+    stats_mutex = dr_mutex_create();
+    dr_register_bb_event(event_basic_block);
+    dr_register_exit_event(event_exit);
+#ifdef SHOW_RESULTS
+    if (dr_is_notify_on())
+	dr_fprintf(STDERR, "Client bbsize is running\n");
+#endif
+}
+
+static void 
+event_exit(void)
+{
+#ifdef SHOW_RESULTS
+    char msg[512];
+    int len;
+    /* Note that using %f with dr_printf or dr_fprintf on Windows will print
+     * garbage as they use ntdll._vsnprintf, so we must use snprintf.
+     */
+    len = snprintf(msg, sizeof(msg)/sizeof(msg[0]),
+                   "Number of basic blocks seen: %d\n"
+                   "               Maximum size: %d instructions\n"
+                   "               Average size: %5.1f instructions\n",
+                   num_bb, max_size, ave_size);
+    DR_ASSERT(len > 0);
+    msg[sizeof(msg)/sizeof(msg[0])-1] = '\0';
+    DISPLAY_STRING(msg);
+#endif /* SHOW_RESULTS */
+    dr_mutex_destroy(stats_mutex);
+}
+
+static dr_emit_flags_t
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating)
+{
+    instr_t *instr;
+    int cur_size = 0;
+
+    /* we use fp ops so we have to save fp state */
+    byte fp_raw[512 + 16];
+    byte *fp_align = (byte *) ( (((ptr_uint_t)fp_raw) + 16) & ((ptr_uint_t)-16) );
+
+    if (translating)
+        return DR_EMIT_DEFAULT;
+
+    proc_save_fpstate(fp_align);
+
+    for (instr = instrlist_first(bb); instr != NULL; instr = instr_get_next(instr))
+	cur_size++;
+
+    dr_mutex_lock(stats_mutex);
+#ifdef VERBOSE_VERBOSE
+    dr_fprintf(STDERR,
+	       "Average: cur=%d, old=%8.1f, num=%d, old*num=%8.1f\n"
+	       "\told*num+cur=%8.1f, new=%8.1f\n",
+	       cur_size, ave_size, num_bb, ave_size*num_bb,
+	       (ave_size * num_bb) + cur_size,
+	       ((ave_size * num_bb) + cur_size) / (double) (num_bb+1));
+#endif
+    if (cur_size > max_size)
+	max_size = cur_size;
+    ave_size = ((ave_size * num_bb) + cur_size) / (double) (num_bb+1);
+    num_bb++;
+    dr_mutex_unlock(stats_mutex);
+
+    proc_restore_fpstate(fp_align);
+
+    return DR_EMIT_DEFAULT;
+}
diff --git a/api/samples/cbr.c b/api/samples/cbr.c
new file mode 100644
index 000000000..cad91ed14
--- /dev/null
+++ b/api/samples/cbr.c
@@ -0,0 +1,432 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Code Manipulation API Sample:
+ * cbr.c
+ *
+ * This sample shows how to update or replace instrumented code after
+ * it executes.  We focus on cbr instructions, inserting
+ * instrumentation to record the fallthrough and taken addresses when
+ * they first execute.  After a particular branch first executes, we
+ * re-instrument the basic block to remove the instrumentation for the
+ * direction taken.  If and when we see the other direction, we remove
+ * all instrumentation for that branch.  We design this sample to
+ * avoid the instrumentation overhead for a particular direction until
+ * it is taken.  Furthermore, we remove all overhead for that
+ * direction after it triggers.
+ *
+ * This sample might form part of a dynamic CFG builder, where we want
+ * to record each control-flow edge, but we don't want to pay the
+ * execution overhead of the instrumentation after we've noted the
+ * edge.
+ *
+ * We use the following replacement scheme:
+ * 1) In the BB event, insert instrumentation for both the taken and
+ *    fallthrough edges.
+ * 2) When the BB executes, note the direction taken and flush the
+ *    fragment from the code cache.
+ * 3) When the BB event triggers again, insert new instrumentation.
+ */
+
+#include "dr_api.h"
+
+#define MINSERT instrlist_meta_preinsert
+
+#define ASSERT(x) \
+    do {                                                        \
+        if (!(x)) {                                             \
+            dr_printf("ASSERT failed on line %d", __LINE__);    \
+            dr_flush_file(STDOUT);                              \
+            dr_abort();                                         \
+        }                                                       \
+    } while (0)                             
+
+/* We need a table to store the state of each cbr (i.e., "seen taken
+ * edge", "seen fallthrough edge", or "seen both").  We'll use a
+ * simple hash table.
+ */
+#define HASH_TABLE_SIZE 7919
+
+/* Possible cbr states */
+typedef enum {
+    CBR_NEITHER   = 0x00,
+    CBR_TAKEN     = 0x01,
+    CBR_NOT_TAKEN = 0x10
+} cbr_state_t;
+
+/* Each bucket in the hash table is a list of the following elements.
+ * For each cbr, we store its address and its state.
+ */
+typedef struct _elem_t {
+    struct _elem_t *next;
+    cbr_state_t state;
+    app_pc addr;
+} elem_t;
+
+typedef struct _list_t {
+    elem_t *head;
+    elem_t *tail;
+} list_t;
+
+
+/* We'll use one global hash table */
+typedef list_t **hash_table_t;
+hash_table_t table = NULL;
+
+static
+elem_t *new_elem(app_pc addr, cbr_state_t state)
+{
+    elem_t *elem = (elem_t *)dr_global_alloc(sizeof(elem_t));
+    ASSERT(elem != NULL);
+
+    elem->next  = NULL;
+    elem->addr  = addr;
+    elem->state = state;
+
+    return elem;
+}
+
+static
+void delete_elem(elem_t *elem)
+{
+    dr_global_free(elem, sizeof(elem_t));
+}
+
+static
+void append_elem(list_t *list, elem_t *elem)
+{
+    if (list->head == NULL) {
+        ASSERT(list->tail == NULL);
+        list->head = elem;
+        list->tail = elem;
+    }
+    else {
+        list->tail->next = elem;
+        list->tail = elem;
+    }
+}
+
+static
+elem_t *find_elem(list_t *list, app_pc addr)
+{
+    elem_t *elem = list->head;
+    while (elem != NULL) {
+        if (elem->addr == addr);
+            return elem;
+        elem = elem->next;
+    }
+
+    return NULL;
+}
+
+static
+list_t *new_list()
+{
+    list_t *list = (list_t *)dr_global_alloc(sizeof(list_t));
+    list->head = NULL;
+    list->tail = NULL;
+    return list;
+}
+
+static
+void delete_list(list_t *list)
+{
+    elem_t *iter = list->head;
+    while (iter != NULL) {
+        elem_t *next = iter->next;
+        delete_elem(iter);
+        iter = next;
+    }
+
+    dr_global_free(list, sizeof(list_t));
+}
+
+hash_table_t new_table()
+{
+    int i;
+    hash_table_t table = (hash_table_t)dr_global_alloc
+        (sizeof(list_t *) * HASH_TABLE_SIZE);
+
+    for (i=0; i<HASH_TABLE_SIZE; i++) {
+        table[i] = NULL;
+    }
+
+    return table;
+}
+
+void delete_table(hash_table_t table)
+{
+    int i;
+    for (i=0; i<HASH_TABLE_SIZE; i++) {
+        if (table[i] != NULL) {
+            delete_list(table[i]);
+        }
+    }
+
+    dr_global_free(table, sizeof(list_t *) * HASH_TABLE_SIZE);
+}
+
+static
+uint hash_func(app_pc addr)
+{
+    return ((uint)(((ptr_uint_t)addr) % HASH_TABLE_SIZE));
+}
+
+elem_t *lookup(hash_table_t table, app_pc addr)
+{
+    list_t *list = table[hash_func(addr)];
+    if (list != NULL)
+        return find_elem(list, addr);
+
+    return NULL;
+}
+
+void insert(hash_table_t table, app_pc addr, cbr_state_t state)
+{
+    elem_t *elem = new_elem(addr, state);
+
+    uint index = hash_func(addr);
+    list_t *list = table[index];
+    if (list == NULL) {
+        list = new_list();
+        table[index] = list;
+    }
+
+    append_elem(list, elem);
+}
+
+/*
+ * End hash table implementation 
+ */
+
+/* Clean call for the 'taken' case */
+static void at_taken(app_pc src, app_pc targ)
+{
+    int app_errno;
+    dr_mcontext_t mcontext;
+    void *drcontext = dr_get_current_drcontext();
+
+    /* 
+     * Record the fact that we've seen the taken case.
+     */
+    elem_t *elem = lookup(table, src);
+    ASSERT(elem != NULL);
+    elem->state |= CBR_TAKEN;
+
+    /* Remove the bb from the cache so it will be re-built the next
+     * time it executes.
+     */
+    /* Since the flush will remove the fragment we're already in,
+     * redirect execution to the target address.
+     */
+    dr_flush_region(src, 1);
+    dr_get_mcontext(drcontext, &mcontext, &app_errno);
+    mcontext.pc = targ;
+    dr_redirect_execution(&mcontext, app_errno);
+}
+
+/* Clean call for the 'not taken' case */
+static void at_not_taken(app_pc src, app_pc fall)
+{
+    int app_errno;
+    dr_mcontext_t mcontext;
+    void *drcontext = dr_get_current_drcontext();
+
+    /* 
+     * Record the fact that we've seen the not_taken case.
+     */
+    elem_t *elem = lookup(table, src);
+    ASSERT(elem != NULL);
+    elem->state |= CBR_NOT_TAKEN;
+
+    /* Remove the bb from the cache so it will be re-built the next
+     * time it executes.
+     */
+    /* Since the flush will remove the fragment we're already in,
+     * redirect execution to the fallthrough address.
+     */
+    dr_flush_region(src, 1);
+    dr_get_mcontext(drcontext, &mcontext, &app_errno);
+    mcontext.pc = fall;
+    dr_redirect_execution(&mcontext, app_errno);
+}
+
+
+static dr_emit_flags_t
+bb_event(void *drcontext, void *tag, instrlist_t *bb, bool for_trace, bool translating)
+{
+    instr_t *instr, *next_instr;
+    
+    for (instr = instrlist_first(bb); instr != NULL; instr = next_instr) {
+        next_instr = instr_get_next(instr);
+
+        if (instr_is_cbr(instr)) {
+            /* Conditional branch.  We can determine the target and
+             * fallthrough addresses here, but we want to note the
+             * edge if and when it actually executes at runtime.
+             * Instead of using dr_insert_cbr_instrumentation(), we'll
+             * insert separate instrumentation for the taken and not
+             * taken cases and remove the instrumentation for an edge
+             * after it executes.
+             */
+            cbr_state_t state;
+            bool insert_taken, insert_not_taken;
+            app_pc src = instr_get_app_pc(instr);
+            
+            /* First look up the state of this branch so we
+             * know what instrumentation to insert, if any.
+             */
+            elem_t *elem = lookup(table, src);
+            
+            if (elem == NULL) {
+                state = CBR_NEITHER;
+                insert(table, src, CBR_NEITHER);
+            }
+            else {
+                state = elem->state;
+            }
+            
+            insert_taken = (state & CBR_TAKEN) == 0;
+            insert_not_taken = (state & CBR_NOT_TAKEN) == 0;
+            
+            if (insert_taken || insert_not_taken) {
+                app_pc fall = (app_pc)decode_next_pc(drcontext, (byte *)src);
+                app_pc targ = instr_get_branch_target_pc(instr);
+                
+                /* Redirect the existing cbr to jump to a callout for
+                 * the 'taken' case.  We'll insert a 'not-taken'
+                 * callout at the fallthrough address.
+                 */
+                instr_t *label = INSTR_CREATE_label(drcontext);
+                /* should be meta, and meta-instrs shouldn't have translations */
+                instr_set_meta_no_translation(instr);
+                /* it may not reach (in particular for x64) w/ our added clean call */
+                if (instr_is_cti_short(instr))
+                    instr_convert_short_meta_jmp_to_long(drcontext, bb, instr);
+                instr_set_target(instr, opnd_create_instr(label));
+                
+                if (insert_not_taken) {
+                    /* Callout for the not-taken case.  Insert after
+                     * the cbr (i.e., 3rd argument is NULL). 
+                     */
+                    dr_insert_clean_call(drcontext, bb, NULL, 
+                                         (void*)at_not_taken,
+                                         false /* don't save fp state */, 
+                                         2 /* 2 args for at_not_taken */,
+                                         OPND_CREATE_INTPTR(src),
+                                         OPND_CREATE_INTPTR(fall));
+                }
+                
+                /* After the callout, jump to the original fallthrough
+                 * address.  Note that this is an exit cti, and should
+                 * not be a meta-instruction.  Therefore, we use
+                 * preinsert instead of meta_preinsert, and we must
+                 * set the translation field.  On Windows, this jump
+                 * and the final jump below never execute since the
+                 * at_taken and at_not_taken callouts redirect
+                 * execution and never return.  However, since the API
+                 * expects clients to produced well-formed code, we
+                 * insert explicit exits from the block for Windows as
+                 * well as Linux.
+                 */
+                instrlist_preinsert(bb, NULL,
+                                    INSTR_XL8(INSTR_CREATE_jmp
+                                              (drcontext, opnd_create_pc(fall)), fall));
+                
+                /* label goes before the 'taken' callout */
+                MINSERT(bb, NULL, label);
+                
+                if (insert_taken) {
+                    /* Callout for the taken case */
+                    dr_insert_clean_call(drcontext, bb, NULL,
+                                         (void*)at_taken,
+                                         false /* don't save fp state */,
+                                         2 /* 2 args for at_taken */,
+                                         OPND_CREATE_INTPTR(src),
+                                         OPND_CREATE_INTPTR(targ));
+                }
+                
+                /* After the callout, jump to the original target
+                 * block (this should not be a meta-instruction).
+                 */
+                instrlist_preinsert(bb, NULL,
+                                    INSTR_XL8(INSTR_CREATE_jmp
+                                              (drcontext, opnd_create_pc(targ)), targ));
+            }
+        }
+    }
+    /* since our added instrumentation is not constant, we ask to store
+     * translations now
+     */
+    return DR_EMIT_STORE_TRANSLATIONS;
+}
+
+
+void dr_exit(void)
+{
+#ifdef SHOW_RESULTS
+    /* Print all the cbr's seen over the life of the process, and
+     * whether we saw taken, not taken, or both.
+     */
+    int i;
+    for (i=0; i<HASH_TABLE_SIZE; i++) {
+        if (table[i] != NULL) {
+            elem_t *iter;
+            for (iter = table[i]->head; iter != NULL; iter = iter->next) {
+                cbr_state_t state = iter->state;
+
+                if (state == CBR_TAKEN) {
+                    dr_printf(""PFX": taken\n", iter->addr);
+                }
+                else if (state == CBR_NOT_TAKEN) {
+                    dr_printf(""PFX": not taken\n", iter->addr);
+                }                
+                else {
+                    ASSERT(state == CBR_TAKEN | CBR_NOT_TAKEN);
+                    dr_printf(""PFX": both\n", iter->addr);
+                }
+            }
+        }
+    }
+#endif
+
+    delete_table(table);
+}
+
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    table = new_table();
+    dr_register_bb_event(bb_event);
+    dr_register_exit_event(dr_exit);
+}
diff --git a/api/samples/countcalls.c b/api/samples/countcalls.c
new file mode 100644
index 000000000..49e0b8b86
--- /dev/null
+++ b/api/samples/countcalls.c
@@ -0,0 +1,225 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Code Manipulation API Sample:
+ * countcalls.c
+ *
+ * Reports the dynamic execution count for direct calls, indirect
+ * calls, and returns in the target application.  Illustrates how to
+ * perform performant inline increments and use per-thread data
+ * structures.
+ */
+
+#include <stddef.h> /* for offsetof */
+#include "dr_api.h"
+
+#ifdef WINDOWS
+# define DISPLAY_STRING(msg) dr_messagebox(msg)
+#else
+# define DISPLAY_STRING(msg) dr_printf("%s\n", msg);
+#endif
+
+#define NULL_TERMINATE(buf) buf[(sizeof(buf)/sizeof(buf[0])) - 1] = '\0'
+
+/* keep separate counters for each thread, in this thread-local data
+ * structure:
+ */
+typedef struct {
+    int num_direct_calls;
+    int num_indirect_calls;
+    int num_returns;
+} per_thread_t;
+
+/* keep a global count as well */
+static per_thread_t global_count = {0};
+
+static void event_exit(void);
+static void event_thread_init(void *drcontext);
+static void event_thread_exit(void *drcontext);
+static dr_emit_flags_t event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                                         bool for_trace, bool translating);
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    /* register events */
+    dr_register_exit_event(event_exit);
+    dr_register_thread_init_event(event_thread_init);
+    dr_register_thread_exit_event(event_thread_exit);
+    dr_register_bb_event(event_basic_block);
+
+    /* make it easy to tell, by looking at log file, which client executed */
+    dr_log(NULL, LOG_ALL, 1, "Client 'countcalls' initializing\n");
+#ifdef SHOW_RESULTS
+    /* also give notification to stderr */
+    if (dr_is_notify_on())
+        dr_fprintf(STDERR, "Client countcalls is running\n");
+#endif
+}
+
+static void
+display_results(per_thread_t *data, char *thread_note)
+{
+#ifdef SHOW_RESULTS
+    char msg[512];
+    int len;
+    len = snprintf(msg, sizeof(msg)/sizeof(msg[0]),
+                   "%sInstrumentation results:\n"
+                   "  saw %d direct calls\n"
+                   "  saw %d indirect calls\n"
+                   "  saw %d returns\n",
+                   thread_note, data->num_direct_calls,
+                   data->num_indirect_calls, data->num_returns);
+    DR_ASSERT(len > 0);
+    NULL_TERMINATE(msg);
+    DISPLAY_STRING(msg);
+#endif /* SHOW_RESULTS */
+}
+
+static void 
+event_exit(void)
+{
+    display_results(&global_count, "");
+}
+
+static void 
+event_thread_init(void *drcontext)
+{
+    /* create an instance of our data structure for this thread */
+    per_thread_t *data = (per_thread_t *)
+        dr_thread_alloc(drcontext, sizeof(per_thread_t));
+    /* store it in the slot provided in the drcontext */
+    dr_set_tls_field(drcontext, data);
+    data->num_direct_calls = 0;
+    data->num_indirect_calls = 0;
+    data->num_returns = 0;
+    dr_log(drcontext, LOG_ALL, 1, "countcalls: set up for thread %d\n",
+           dr_get_thread_id(drcontext));
+}
+
+static void 
+event_thread_exit(void *drcontext)
+{
+    per_thread_t *data = (per_thread_t *) dr_get_tls_field(drcontext);
+    char msg[512];
+    int len;
+
+    len = snprintf(msg, sizeof(msg)/sizeof(msg[0]),
+                   "Thread %d exited - ", dr_get_thread_id(drcontext));
+    DR_ASSERT(len > 0);
+    NULL_TERMINATE(msg);
+
+    /* display thread private counts data */
+    display_results(data, msg);
+
+    /* clean up memory */
+    dr_thread_free(drcontext, data, sizeof(per_thread_t));
+}
+
+static void
+insert_counter_update(void *drcontext, instrlist_t *bb, instr_t *where, int offset)
+{
+    /* Since the inc instruction clobbers 5 of the arithmetic eflags,
+     * we have to save them around the inc. We could be more efficient
+     * by not bothering to save the overflow flag and constructing our
+     * own sequence of instructions to save the other 5 flags (using
+     * lahf) or by doing a liveness analysis on the flags and saving
+     * only if live.
+     */
+    dr_save_arith_flags(drcontext, bb, where, SPILL_SLOT_1);
+
+    /* Increment the global counter using the lock prefix to make it atomic
+     * across threads. It would be cheaper to aggregate the thread counters
+     * in the exit events, but this sample is intended to illustrate inserted
+     * instrumentation.
+     */
+    instrlist_meta_preinsert(bb, where, LOCK(INSTR_CREATE_inc
+        (drcontext, OPND_CREATE_ABSMEM(((byte *)&global_count) + offset, OPSZ_4))));
+
+    /* Increment the thread private counter. */
+    if (dr_using_all_private_caches()) {
+        per_thread_t *data = (per_thread_t *) dr_get_tls_field(drcontext);
+        /* private caches - we can use an absolute address */
+        instrlist_meta_preinsert(bb, where, INSTR_CREATE_inc(drcontext, 
+            OPND_CREATE_ABSMEM(((byte *)&data) + offset, OPSZ_4)));
+    } else {
+        /* shared caches - we must indirect via thread local storage */
+        /* We spill xbx to use a scratch register (we could do a liveness
+         * analysis to try and find a dead register to use). Note that xax
+         * is currently holding the saved eflags. */
+        dr_save_reg(drcontext, bb, where, REG_XBX, SPILL_SLOT_2);
+        dr_insert_read_tls_field(drcontext, bb, where, REG_XBX);
+        instrlist_meta_preinsert(bb, where,
+            INSTR_CREATE_inc(drcontext, OPND_CREATE_MEM32(REG_XBX, offset)));
+        dr_restore_reg(drcontext, bb, where, REG_XBX, SPILL_SLOT_2);
+    }
+
+    /* restore flags */
+    dr_restore_arith_flags(drcontext, bb, where, SPILL_SLOT_1);
+}
+
+static dr_emit_flags_t
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating)
+{
+    instr_t *instr, *next_instr;
+    
+#ifdef VERBOSE
+    dr_printf("in dynamorio_basic_block(tag="PFX")\n", tag);
+# ifdef VERBOSE_VERBOSE
+    instrlist_disassemble(drcontext, tag, bb, STDOUT);
+# endif
+#endif
+    
+    for (instr = instrlist_first(bb); instr != NULL; instr = next_instr) {
+        /* grab next now so we don't go over instructions we insert */
+        next_instr = instr_get_next(instr);
+        
+        /* instrument calls and returns -- ignore far calls/rets */
+        if (instr_is_call_direct(instr)) {
+            insert_counter_update(drcontext, bb, instr,
+                                  offsetof(per_thread_t, num_direct_calls));
+        } else if (instr_is_call_indirect(instr)) {
+            insert_counter_update(drcontext, bb, instr,
+                                  offsetof(per_thread_t, num_indirect_calls));
+        } else if (instr_is_return(instr)) {
+            insert_counter_update(drcontext, bb, instr,
+                                  offsetof(per_thread_t, num_returns));
+        }
+    }
+    
+#if defined(VERBOSE) && defined(VERBOSE_VERBOSE)
+    dr_printf("Finished instrumenting dynamorio_basic_block(tag="PFX")\n", tag);
+    instrlist_disassemble(drcontext, tag, bb, STDOUT);
+#endif
+    return DR_EMIT_DEFAULT;
+}
diff --git a/api/samples/div.c b/api/samples/div.c
new file mode 100644
index 000000000..a821171a4
--- /dev/null
+++ b/api/samples/div.c
@@ -0,0 +1,118 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Counts the number of dynamic div instruction for which the
+ * divisor is a power of 2 (these are cases where div could be
+ * strength reduced to a simple shift).  Demonstrates callout
+ * based profiling with live operand values. */
+
+#include "dr_api.h"
+
+#ifdef WINDOWS
+# define DISPLAY_STRING(msg) dr_messagebox(msg)
+#else
+# define DISPLAY_STRING(msg) dr_printf("%s\n", msg);
+#endif
+
+#define NULL_TERMINATE(buf) buf[(sizeof(buf)/sizeof(buf[0])) - 1] = '\0'
+
+static dr_emit_flags_t bb_event(void *drcontext, void *tag, instrlist_t *bb,
+                                bool for_trace, bool translating);
+static void exit_event(void);
+
+static int div_count = 0, div_p2_count = 0;
+static void *count_mutex; /* for multithread support */
+
+DR_EXPORT void
+dr_init(client_id_t id)
+{
+    dr_register_exit_event(exit_event);
+    dr_register_bb_event(bb_event);
+    count_mutex = dr_mutex_create();
+}
+
+static void
+exit_event(void)
+{
+#ifdef SHOW_RESULTS
+    char msg[512];
+    int len;
+    len = snprintf(msg, sizeof(msg)/sizeof(msg[0]),
+                   "Instrumentation results:\n"
+                   "  saw %d div instructions\n"
+                   "  of which %d were powers of 2\n",
+                   div_count, div_p2_count);
+    DR_ASSERT(len > 0);
+    NULL_TERMINATE(msg);
+    DISPLAY_STRING(msg);
+#endif /* SHOW_RESULTS */
+
+    dr_mutex_destroy(count_mutex);
+}
+
+static void
+callback(app_pc addr, uint divisor)
+{
+    /* instead of a lock could use atomic operations to
+     * increment the counters */
+    dr_mutex_lock(count_mutex);
+
+    div_count++;
+
+    /* check for power of 2 */
+    if (divisor & (divisor - 1) != 0)
+        div_p2_count++;
+
+    dr_mutex_unlock(count_mutex);
+}
+
+static dr_emit_flags_t
+bb_event(void* drcontext, void *tag, instrlist_t *bb, bool for_trace, bool translating)
+{
+    instr_t *instr, *next_instr;
+    int opcode;
+
+    for (instr = instrlist_first(bb); instr != NULL; instr = next_instr) {
+        next_instr = instr_get_next(instr);
+        opcode = instr_get_opcode(instr);
+
+        /* if find div, insert a clean call to our instrumentation routine */
+        if (opcode == OP_div) {   
+            dr_insert_clean_call(drcontext, bb, instr, (void *)callback,
+                                 false /*no fp save*/, 2,
+                                 OPND_CREATE_INTPTR(instr_get_app_pc(instr)),
+                                 instr_get_src(instr, 0) /*divisor is 1st src*/);
+        }
+    }
+    return DR_EMIT_DEFAULT;
+}
+
diff --git a/api/samples/empty.c b/api/samples/empty.c
new file mode 100644
index 000000000..8b1583560
--- /dev/null
+++ b/api/samples/empty.c
@@ -0,0 +1,55 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Code Manipulation API Sample:
+ * empty.c
+ *
+ * Serves as an example of an empty client that does nothing but
+ * register for the exit event.
+ */
+
+#include "dr_api.h"
+
+static void event_exit(void);
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    /* empty client */
+    dr_register_exit_event(event_exit);
+}
+
+static void 
+event_exit(void)
+{
+    /* empty client */
+}
diff --git a/api/samples/inc2add.c b/api/samples/inc2add.c
new file mode 100644
index 000000000..cf935aa18
--- /dev/null
+++ b/api/samples/inc2add.c
@@ -0,0 +1,214 @@
+/* **********************************************************
+ * Copyright (c) 2002-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Code Manipulation API Sample:
+ * inc2add.c
+ *
+ * Performs a dynamic optimization: converts the "inc" instruction to
+ * "add 1" whenever worthwhile and feasible without perturbing the
+ * target application's behavior.  Illustrates a
+ * microarchitecture-specific optimization best performed at runtime
+ * when the underlying processor is known.
+ */
+
+#include "dr_api.h"
+
+#ifdef WINDOWS
+# define DISPLAY_STRING(msg) dr_messagebox(msg)
+# define ATOMIC_INC(var) _InterlockedIncrement(&var)
+#else
+# define DISPLAY_STRING(msg) dr_printf("%s\n", msg);
+# define ATOMIC_INC(var) __asm__ __volatile__("lock incl %0" : "=m" (var) : : "memory")
+#endif
+
+static bool enable;
+
+/* use atomic operations to increment these to avoid the hassle of locking. */
+static int num_examined, num_converted;
+
+/* replaces inc with add 1, dec with sub 1 
+ * returns true if successful, false if not
+ */
+static bool
+replace_inc_with_add(void *drcontext, instr_t *inst, instrlist_t *trace);
+
+static dr_emit_flags_t
+event_trace(void *drcontext, void *tag, instrlist_t *trace, bool translating);
+
+static void 
+event_exit(void);
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    dr_register_exit_event(event_exit);
+    dr_register_trace_event(event_trace);
+    /* this optimization is only worthwhile on the Pentium 4, where
+     * an add of 1 is faster than an inc
+     */
+    enable = (proc_get_family() == FAMILY_PENTIUM_4);
+    /* make it easy to tell, by looking at log file, which client executed */
+    dr_log(NULL, LOG_ALL, 1, "Client 'inc2add' initializing\n");
+#ifdef SHOW_RESULTS
+    /* also give notification to stderr */
+    if (dr_is_notify_on())
+	dr_fprintf(STDERR, "Client inc2add is running\n");
+#endif
+    /* initialize our global variables */
+    num_examined = 0;
+    num_converted = 0;
+}
+
+static void 
+event_exit(void)
+{
+#ifdef SHOW_RESULTS
+    char msg[256];
+    int len;
+    if (enable) {
+	len = snprintf(msg, sizeof(msg)/sizeof(msg[0]),
+                       "converted %d out of %d inc/dec to add/sub\n",
+                       num_converted, num_examined);
+    } else {
+	len = snprintf(msg, sizeof(msg)/sizeof(msg[0]),
+                       "decided to keep all original inc/dec\n");
+    }
+    DR_ASSERT(len > 0);
+    msg[sizeof(msg)/sizeof(msg[0])-1] = '\0';
+    DISPLAY_STRING(msg);
+#endif /* SHOW_RESULTS */
+}
+
+/* replaces all inc with add 1, dec with sub 1
+ * if cannot replace (eflags constraints), leaves original instruction alone
+ */
+static dr_emit_flags_t
+event_trace(void *drcontext, void *tag, instrlist_t *trace, bool translating)
+{
+    instr_t *instr, *next_instr;
+    int opcode;
+
+    if (!enable)
+	return DR_EMIT_DEFAULT;
+
+#ifdef VERBOSE
+    dr_printf("in dynamorio_trace(tag="PFX")\n", tag);
+    instrlist_disassemble(drcontext, tag, trace, STDOUT);
+#endif
+
+    for (instr = instrlist_first(trace); instr != NULL; instr = next_instr) {
+	/* grab next now so we don't go over instructions we insert */
+	next_instr = instr_get_next(instr);
+	opcode = instr_get_opcode(instr);
+	if (opcode == OP_inc || opcode == OP_dec) {
+            if (!translating)
+                ATOMIC_INC(num_examined);
+	    if (replace_inc_with_add(drcontext, instr, trace)) {
+                if (!translating)
+                    ATOMIC_INC(num_converted);
+            }
+	}
+    }
+
+#ifdef VERBOSE
+    dr_printf("after dynamorio_trace(tag="PFX"):\n", tag);
+    instrlist_disassemble(drcontext, tag, trace, STDOUT);
+#endif
+
+    return DR_EMIT_DEFAULT;
+}
+
+/* replaces inc with add 1, dec with sub 1 
+ * returns true if successful, false if not
+ */
+static bool
+replace_inc_with_add(void *drcontext, instr_t *instr, instrlist_t *trace)
+{
+    instr_t *in;
+    uint eflags;
+    int opcode = instr_get_opcode(instr);
+    bool ok_to_replace = false;
+
+    DR_ASSERT(opcode == OP_inc || opcode == OP_dec);
+#ifdef VERBOSE
+    dr_print_instr(drcontext, STDOUT, instr, "in replace_inc_with_add:\n\t");
+#endif
+
+    /* add/sub writes CF, inc/dec does not, make sure that's ok */
+    for (in = instr; in != NULL; in = instr_get_next(in)) {
+	eflags = instr_get_eflags(in);
+	if ((eflags & EFLAGS_READ_CF) != 0) {
+#ifdef VERBOSE
+            dr_print_instr(drcontext, STDOUT, in,
+                           "\treads CF => cannot replace inc with add: ");
+#endif
+	    return false;
+	}
+	if (instr_is_exit_cti(in)) {
+	    /* to be more sophisticated, examine instructions at
+	     * target of exit cti (if it is a direct branch).
+	     * for this example, we give up if we hit a branch.
+	     */
+	    return false;
+	}
+	/* if writes but doesn't read, ok */
+	if ((eflags & EFLAGS_WRITE_CF) != 0) {
+	    ok_to_replace = true;
+	    break;
+	}
+    }
+    if (!ok_to_replace) {
+#ifdef VERBOSE
+        dr_printf("\tno write to CF => cannot replace inc with add\n");
+#endif
+	return false;
+    }
+    if (opcode == OP_inc) {
+#ifdef VERBOSE
+        dr_printf("\treplacing inc with add\n");
+#endif
+	in = INSTR_CREATE_add(drcontext, instr_get_dst(instr, 0),
+			      OPND_CREATE_INT8(1));
+    } else {
+#ifdef VERBOSE
+        dr_printf("\treplacing dec with sub\n");
+#endif
+	in = INSTR_CREATE_sub(drcontext, instr_get_dst(instr, 0),
+			      OPND_CREATE_INT8(1));
+    }
+    if (instr_get_prefix_flag(instr, PREFIX_LOCK))
+        instr_set_prefix_flag(in, PREFIX_LOCK);
+    instr_set_translation(in, instr_get_app_pc(instr));
+    instrlist_replace(trace, instr, in);
+    instr_destroy(drcontext, instr);
+    return true;
+}
diff --git a/api/samples/inline.c b/api/samples/inline.c
new file mode 100644
index 000000000..b4c0360b8
--- /dev/null
+++ b/api/samples/inline.c
@@ -0,0 +1,361 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Code Manipulation API Sample:
+ * trace-inline.c
+ *
+ * Uses the custom trace API to inline entire callees into traces.
+ */
+
+#include "dr_api.h"
+#include <string.h> /* memset */
+
+#define VERBOSE 1
+
+#ifdef WINDOWS
+# define DISPLAY_STRING(msg) dr_messagebox(msg)
+#else
+# define DISPLAY_STRING(msg) dr_printf("%s\n", msg);
+#endif
+
+/****************************************************************************/
+/* global */
+
+#ifdef SHOW_RESULTS
+static int num_complete_inlines;
+#endif
+
+static void *htable_mutex; /* for multithread support */
+
+static void event_exit(void);
+static dr_emit_flags_t event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                                         bool for_trace, bool translating);
+static void event_fragment_deleted(void *drcontext, void *tag);
+static dr_custom_trace_action_t
+query_end_trace(void *drcontext, void * trace_tag, void *next_tag);
+
+/****************************************************************************/
+/* hashtable so we know if a particular tag is for a call trace or a
+ * normal back branch trace
+ */
+
+typedef struct _trace_head_entry_t {
+    void *tag;
+    bool is_trace_head;
+    bool has_ret;
+    /* have to end at next block after see return */
+    int end_next;
+    /* some callees are too large to inline, so have size limit */
+    uint size;
+    struct _trace_head_entry_t *next;
+} trace_head_entry_t;
+
+/* global table */
+static trace_head_entry_t **htable;
+
+/* max call-trace size */
+#define INLINE_SIZE_LIMIT (4*1024)
+
+/* no instruction alignment -> use the lsb! */
+#define HASH_MASK(num_bits) ((~0U)>>(32-(num_bits)))
+#define HASH_FUNC_BITS(val, num_bits) ((val) & (HASH_MASK(num_bits)))
+#define HASH_FUNC(val, mask) ((val) & (mask))
+#define HASHTABLE_SIZE(num_bits) (1U << (num_bits))
+
+#define HASH_BITS 13
+#define TABLE_SIZE HASHTABLE_SIZE(HASH_BITS) * sizeof(trace_head_entry_t*)
+
+/* if drcontext == NULL uses global memory */
+static trace_head_entry_t **
+htable_create(void *drcontext)
+{
+    trace_head_entry_t **table = (trace_head_entry_t**) dr_global_alloc(TABLE_SIZE);
+    /* assume during process init so no lock needed */
+    memset(table, 0, TABLE_SIZE);
+    return table;
+}
+
+/* if drcontext == NULL uses global memory */
+static void
+htable_free(void *drcontext, trace_head_entry_t **table)
+{
+    /* assume during process exit so no lock needed */
+    int i;
+    /* clean up memory */
+    for (i = 0; i < HASHTABLE_SIZE(HASH_BITS); i++) {
+	trace_head_entry_t *e = table[i];
+	while (e) {
+	    trace_head_entry_t *nexte = e->next;
+	    dr_global_free(e, sizeof(trace_head_entry_t));
+	    e = nexte;
+	}
+	table[i] = NULL;
+    }
+    dr_global_free(table, TABLE_SIZE);
+}
+
+/* Caller must hold htable_mutex if drcontext == NULL. */
+static trace_head_entry_t *
+add_trace_head_entry(void *drcontext, void *tag)
+{
+    trace_head_entry_t **table = htable;
+    trace_head_entry_t *e;
+    uint hindex;
+    e = (trace_head_entry_t *) dr_global_alloc(sizeof(trace_head_entry_t));
+    e->tag = tag;
+    e->end_next = 0;
+    e->size = 0;
+    e->has_ret = false;
+    e->is_trace_head = false;
+    /* we assume we're masking away top 32 bits for 64-bit, so cast here */
+    hindex = HASH_FUNC_BITS((uint)(ptr_uint_t)tag, HASH_BITS);
+    e->next = table[hindex];
+    table[hindex] = e;
+    return e;
+}
+
+/* Lookup an entry by pc and return a pointer to the corresponding entry .
+ * Returns NULL if no such entry exists.
+ * Caller must hold htable_mutex if drcontext == NULL.
+ */
+static trace_head_entry_t *
+lookup_trace_head_entry(void *drcontext, void *tag)
+{
+    trace_head_entry_t **table = htable;
+    trace_head_entry_t *e;
+    uint hindex;
+    /* we assume we're masking away top 32 bits for 64-bit, so cast here */
+    hindex = HASH_FUNC_BITS((uint)(ptr_uint_t)tag, HASH_BITS);
+    for (e = table[hindex]; e; e = e->next) {
+	if (e->tag == tag)
+	    return e;
+    }
+    return NULL;
+}
+
+/* Lookup an entry by tag and index and delete it.
+ * Returns false if no such entry exists.
+ * Caller must hold htable_mutex if drcontext == NULL.
+ */
+static bool
+remove_trace_head_entry(void *drcontext, void *tag)
+{
+    trace_head_entry_t **table = htable;
+    trace_head_entry_t *e, *prev;
+    uint hindex;
+    /* we assume we're masking away top 32 bits for 64-bit, so cast here */
+    hindex = HASH_FUNC_BITS((uint)(ptr_uint_t)tag, HASH_BITS);
+    for (prev = NULL, e = table[hindex]; e; prev = e, e = e->next) {
+	if (e->tag == tag) {
+	    if (prev)
+		prev->next = e->next;
+	    else
+		table[hindex] = e->next;
+	    dr_global_free(e, sizeof(trace_head_entry_t));
+	    return true;
+	}
+    }
+    return false;
+}
+
+/****************************************************************************/
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    htable_mutex = dr_mutex_create();
+
+    /* global HASH_BITS-bit addressed hash table */
+    htable = htable_create(NULL/*global*/);
+
+    dr_register_exit_event(event_exit);
+    dr_register_bb_event(event_basic_block);
+    dr_register_delete_event(event_fragment_deleted);
+    dr_register_end_trace_event(query_end_trace);
+
+    /* make it easy to tell, by looking at log file, which client executed */
+    dr_log(NULL, LOG_ALL, 1, "Client 'inline' initializing\n");
+#ifdef SHOW_RESULTS
+    /* also give notification to stderr */
+    if (dr_is_notify_on())
+	dr_fprintf(STDERR, "Client inline is running\n");
+    num_complete_inlines = 0;
+#endif
+}
+
+static void 
+event_exit(void)
+{
+#ifdef SHOW_RESULTS
+    /* display the results */
+    char msg[512];
+    int len = snprintf(msg, sizeof(msg)/sizeof(msg[0]),
+                       "Inlining results:\n"
+                       "  Number of complete inlines: %d\n", num_complete_inlines);
+    DR_ASSERT(len > 0);
+    msg[sizeof(msg)/sizeof(msg[0])-1] = '\0';
+    DISPLAY_STRING(msg);
+#endif
+    htable_free(NULL/*global*/, htable);
+    dr_mutex_destroy(htable_mutex);
+}
+
+/****************************************************************************/
+/* the work itself */
+
+static dr_emit_flags_t 
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating)
+{
+    instr_t *instr;
+    trace_head_entry_t *e = NULL;
+    if (translating)
+        return DR_EMIT_DEFAULT;
+    for (instr = instrlist_first(bb); instr != NULL; instr = instr_get_next(instr)) {
+	/* blocks containing calls are trace heads */
+	if (instr_is_call(instr)) {
+	    dr_mark_trace_head(drcontext, tag);
+            dr_mutex_lock(htable_mutex);
+	    e = add_trace_head_entry(NULL, tag);
+	    e->is_trace_head = true;
+            dr_mutex_unlock(htable_mutex);
+#ifdef VERBOSE
+	    dr_log(drcontext, LOG_ALL, 3,
+		   "inline: marking bb "PFX" as trace head\n", tag);
+#endif
+	    /* doesn't matter what's in rest of bb */
+	    return DR_EMIT_DEFAULT;
+	} else if (instr_is_return(instr)) {
+            dr_mutex_lock(htable_mutex);
+            e = add_trace_head_entry(NULL, tag);
+	    e->has_ret = true;
+            dr_mutex_unlock(htable_mutex);
+	}
+    }
+    return DR_EMIT_DEFAULT;
+}
+
+/* to keep the size of our hashtable down */
+static void
+event_fragment_deleted(void *drcontext, void *tag)
+{
+    dr_mutex_lock(htable_mutex);
+    remove_trace_head_entry(NULL, tag);
+    dr_mutex_unlock(htable_mutex);
+}
+
+/* Ask whether to end trace prior to adding next_tag fragment.
+ * Return values:
+ *   CUSTOM_TRACE_DR_DECIDES = use standard termination criteria
+ *   CUSTOM_TRACE_END_NOW   = end trace
+ *   CUSTOM_TRACE_CONTINUE  = do not end trace
+ */
+static dr_custom_trace_action_t
+query_end_trace(void *drcontext, void *trace_tag, void *next_tag)
+{
+    /* if this is a call trace, only end on the block AFTER a return
+     *   (need to get the return inlined!)
+     * if this is a standard back branch trace, end it if we see a
+     *   block with a call (so that we'll go into the call trace).
+     *   otherwise return 0 and let DynamoRIO determine whether to
+     *   terminate the trace now.
+     */
+    trace_head_entry_t *e;
+    dr_mutex_lock(htable_mutex);
+    e = lookup_trace_head_entry(NULL, trace_tag);
+    if (e == NULL || !e->is_trace_head) {
+	e = lookup_trace_head_entry(NULL, next_tag);
+	if (e == NULL || !e->is_trace_head) {
+            dr_mutex_unlock(htable_mutex);
+	    return CUSTOM_TRACE_DR_DECIDES;
+        } else {
+	    /* we've found a call, end this trace now so it won't keep going and
+	     * end up never entering the call trace
+	     */
+#ifdef VERBOSE
+	    dr_log(drcontext, LOG_ALL, 3,
+		   "inline: ending trace "PFX" before block "PFX" containing call\n",
+		   trace_tag, next_tag);
+#endif
+            dr_mutex_unlock(htable_mutex);
+	    return CUSTOM_TRACE_END_NOW;
+	}
+    } else if (e->end_next > 0) {
+	e->end_next--;
+	if (e->end_next == 0) {
+#ifdef VERBOSE
+	    dr_log(drcontext, LOG_ALL, 3,
+		   "inline: ending trace "PFX" before "PFX"\n",
+		   trace_tag, next_tag);
+#endif
+#ifdef SHOW_RESULTS
+	    num_complete_inlines++;
+#endif
+            dr_mutex_unlock(htable_mutex);
+	    return CUSTOM_TRACE_END_NOW;
+	}
+    } else {
+	trace_head_entry_t *nxte = lookup_trace_head_entry(NULL, next_tag);
+	uint size = dr_fragment_size(drcontext, next_tag);
+	e->size += size;
+	if (e->size > INLINE_SIZE_LIMIT) {
+#ifdef VERBOSE
+	    dr_log(drcontext, LOG_ALL, 3,
+		   "inline: ending trace "PFX" before "PFX" because reached size limit\n",
+		   trace_tag, next_tag);
+#endif
+            dr_mutex_unlock(htable_mutex);
+	    return CUSTOM_TRACE_END_NOW;
+	}
+	if (nxte != NULL && nxte->has_ret && !nxte->is_trace_head) {
+	    /* end trace after NEXT block */
+	    e->end_next = 2;
+#ifdef VERBOSE
+	    dr_log(drcontext, LOG_ALL, 3,
+		   "inline: going to be ending trace "PFX" after "PFX"\n",
+		   trace_tag, next_tag);
+#endif
+            dr_mutex_unlock(htable_mutex);	
+            return CUSTOM_TRACE_CONTINUE;
+	}
+    }
+    /* do not end trace */
+#ifdef VERBOSE
+    dr_log(drcontext, LOG_ALL, 3,
+	   "inline: NOT ending trace "PFX" after "PFX"\n", trace_tag, next_tag);
+#endif
+    dr_mutex_unlock(htable_mutex);
+    return CUSTOM_TRACE_CONTINUE;
+}
+
+
+
diff --git a/api/samples/instrcalls.c b/api/samples/instrcalls.c
new file mode 100644
index 000000000..7a1dfb730
--- /dev/null
+++ b/api/samples/instrcalls.c
@@ -0,0 +1,163 @@
+/* **********************************************************
+ * Copyright (c) 2002-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Code Manipulation API Sample:
+ * instrcalls.c
+ *
+ * Instruments direct calls, indirect calls, and returns in the target
+ * application.  For each dynamic execution, the call target and other
+ * key information is written to a log file.  Note that this log file
+ * can become quite large, and this client incurs more overhead than
+ * the other clients due to its log file.
+ */
+
+#include "dr_api.h"
+
+static void event_thread_init(void *drcontext);
+static void event_thread_exit(void *drcontext);
+static dr_emit_flags_t event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                                         bool for_trace, bool translating);
+
+static client_id_t my_id;
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    my_id = id;
+    /* make it easy to tell, by looking at log file, which client executed */
+    dr_log(NULL, LOG_ALL, 1, "Client 'instrcalls' initializing\n");
+    /* also give notification to stderr */
+#ifdef SHOW_RESULTS
+    if (dr_is_notify_on())
+        dr_fprintf(STDERR, "Client instrcalls is running\n");
+#endif
+    dr_register_bb_event(event_basic_block);
+    dr_register_thread_init_event(event_thread_init);
+    dr_register_thread_exit_event(event_thread_exit);
+}
+
+#ifdef WINDOWS
+# define DIRSEP '\\'
+#else
+# define DIRSEP '/'
+#endif
+
+static void
+event_thread_init(void *drcontext)
+{
+    file_t f;
+    char logname[512];
+    char *dirsep;
+    int len;
+    /* We're going to dump our data to a per-thread file.
+     * On Windows we need an absolute path so we place it in
+     * the same directory as our library. We could also pass
+     * in a path and retrieve with dr_get_options().
+     */
+    len = snprintf(logname, sizeof(logname)/sizeof(logname[0]),
+                   "%s", dr_get_client_path(my_id));
+    DR_ASSERT(len > 0);
+    for (dirsep = logname + len; *dirsep != DIRSEP; dirsep--)
+        DR_ASSERT(dirsep > logname);
+    len = snprintf(dirsep + 1, (sizeof(logname) - (dirsep - logname))/sizeof(logname[0]),
+                   "instrcalls.%d.log", dr_get_thread_id(drcontext));
+    DR_ASSERT(len > 0);
+    logname[sizeof(logname)/sizeof(logname[0])-1] = '\0';
+    f = dr_open_file(logname, DR_FILE_WRITE_OVERWRITE);
+    DR_ASSERT(f != INVALID_FILE);
+
+    /* store it in the slot provided in the drcontext */
+    dr_set_tls_field(drcontext, (void *)(ptr_uint_t)f);
+    dr_log(drcontext, LOG_ALL, 1, 
+           "instrcalls: log for thread %d is instrcalls.%03d\n",
+           dr_get_thread_id(drcontext), dr_get_thread_id(drcontext));
+}
+
+static void
+event_thread_exit(void *drcontext)
+{
+    file_t f = (file_t)(ptr_uint_t) dr_get_tls_field(drcontext);
+    dr_close_file(f);
+}
+
+static void
+at_call(app_pc instr_addr, app_pc target_addr)
+{
+    file_t f = (file_t)(ptr_uint_t) dr_get_tls_field(dr_get_current_drcontext());
+    dr_mcontext_t mc;
+    dr_get_mcontext(dr_get_current_drcontext(), &mc, NULL);
+    dr_fprintf(f, "CALL @ "PFX" to "PFX", TOS is "PFX"\n",
+               instr_addr, target_addr, mc.xsp);
+}
+
+static void
+at_call_ind(app_pc instr_addr, app_pc target_addr)
+{
+    file_t f = (file_t)(ptr_uint_t) dr_get_tls_field(dr_get_current_drcontext());
+    dr_fprintf(f, "CALL INDIRECT @ "PFX" to "PFX"\n", instr_addr, target_addr);
+}
+
+static void
+at_return(app_pc instr_addr, app_pc target_addr)
+{
+    file_t f = (file_t)(ptr_uint_t) dr_get_tls_field(dr_get_current_drcontext());
+    dr_fprintf(f, "RETURN @ "PFX" to "PFX"\n", instr_addr, target_addr);
+}
+
+static dr_emit_flags_t 
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating)
+{
+    instr_t *instr, *next_instr;
+#ifdef VERBOSE
+    dr_printf("in dr_basic_block(tag="PFX")\n", tag);
+# if VERBOSE_VERBOSE
+    instrlist_disassemble(drcontext, tag, bb, STDOUT);
+# endif
+#endif
+    for (instr = instrlist_first(bb); instr != NULL; instr = next_instr) {
+        next_instr = instr_get_next(instr);
+        if (!instr_opcode_valid(instr))
+            continue;
+        /* instrument calls and returns -- ignore far calls/rets */
+        if (instr_is_call_direct(instr)) {
+            dr_insert_call_instrumentation(drcontext, bb, instr, (app_pc)at_call);
+        } else if (instr_is_call_indirect(instr)) {
+            dr_insert_mbr_instrumentation(drcontext, bb, instr, (app_pc)at_call_ind,
+                                          SPILL_SLOT_1);
+        } else if (instr_is_return(instr)) {
+            dr_insert_mbr_instrumentation(drcontext, bb, instr, (app_pc)at_return,
+                                          SPILL_SLOT_1);
+        }
+    }
+    return DR_EMIT_DEFAULT;
+}
diff --git a/api/samples/prefetch.c b/api/samples/prefetch.c
new file mode 100644
index 000000000..5d0fb0354
--- /dev/null
+++ b/api/samples/prefetch.c
@@ -0,0 +1,102 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Code Manipulation API Sample:
+ * prefetch.c
+ *
+ * Performs a code-stream compatibility modification.  Programs optimized for AMD
+ * processors are likely to use the prefetch and prefetchw instructions (originally
+ * added as part of the 3D!now extensions).  However, these opcodes cause illegal
+ * instruction faults on most Intel processors.  Here we detect if we are on an Intel
+ * processor and translate (in this case remove) prefetch instructions so the app can
+ * run.
+ */
+
+#include "dr_api.h"
+
+static void event_exit(void);
+static dr_emit_flags_t event_bb(void *drcontext, void *tag, instrlist_t *bb,
+                                bool for_trace, bool translating);
+
+static void *count_mutex;  /* for multithread support */
+static int prefetches_removed = 0, prefetchws_removed = 0;
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    dr_register_exit_event(event_exit);
+    /* Only need to remove prefetches for Intel processors. */
+    if (proc_get_vendor() == VENDOR_INTEL) {
+        dr_register_bb_event(event_bb);
+    }
+
+    /* make it easy to tell, by looking at log file, which client executed */
+    dr_log(NULL, LOG_ALL, 1, "Client 'prefetch' initializing\n");
+
+    /* must create the counting mutex */
+    count_mutex = dr_mutex_create();
+}
+
+static void 
+event_exit(void)
+{
+    dr_log(NULL, LOG_ALL, 1, "Removed %d prefetches and %d prefetchws.\n",
+           prefetches_removed, prefetchws_removed);
+    dr_mutex_destroy(count_mutex);
+}
+
+static dr_emit_flags_t
+event_bb(void *drcontext, void *tag, instrlist_t *bb, bool for_trace, bool translating)
+{
+    instr_t *instr, *next_instr;
+    int opcode;
+
+    /* look for and remove prefetch[w] instructions */
+    for (instr = instrlist_first(bb); instr != NULL; instr = next_instr) {
+	next_instr = instr_get_next(instr);
+	opcode = instr_get_opcode(instr);
+	if (opcode == OP_prefetch || opcode == OP_prefetchw) {
+            instrlist_remove(bb, instr);
+            instr_destroy(drcontext, instr);
+            if (!translating) {
+                /* count_mutex protects the counter values */
+                dr_mutex_lock(count_mutex);
+                if (opcode == OP_prefetch)
+                    prefetches_removed++;
+                else
+                    prefetchws_removed++;
+                dr_mutex_unlock(count_mutex);
+            }
+	}
+    }
+    return DR_EMIT_DEFAULT;
+}
diff --git a/api/samples/signal.c b/api/samples/signal.c
new file mode 100644
index 000000000..2283b8f35
--- /dev/null
+++ b/api/samples/signal.c
@@ -0,0 +1,106 @@
+/* **********************************************************
+ * Copyright (c) 2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Code Manipulation API Sample:
+ * signal.c
+ *
+ * Monitors application signals.
+ */
+
+#include "dr_api.h"
+#include <signal.h>
+
+#ifdef WINDOWS
+# define DISPLAY_STRING(msg) dr_messagebox(msg)
+# define ATOMIC_INC(var) _InterlockedIncrement(&var)
+#else
+# define DISPLAY_STRING(msg) dr_printf("%s\n", msg);
+# define ATOMIC_INC(var) __asm__ __volatile__("lock incl %0" : "=m" (var) : : "memory")
+#endif
+
+static int num_signals;
+
+static void event_exit(void);
+#ifdef LINUX
+static dr_signal_action_t event_signal(void *drcontext, dr_siginfo_t *info);
+#endif
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+#ifdef LINUX
+    dr_register_signal_event(event_signal);
+#endif
+    dr_register_exit_event(event_exit);
+#ifdef SHOW_RESULTS
+    if (dr_is_notify_on())
+	dr_fprintf(STDERR, "Client signal is running\n");
+#endif
+}
+
+static void 
+event_exit(void)
+{
+#ifdef SHOW_RESULTS
+    char msg[512];
+    int len;
+    /* Note that using %f with dr_printf or dr_fprintf on Windows will print
+     * garbage as they use ntdll._vsnprintf, so we must use snprintf.
+     */
+    len = snprintf(msg, sizeof(msg)/sizeof(msg[0]),
+                   "<Number of signals seen: %d>", num_signals);
+    DR_ASSERT(len > 0);
+    msg[sizeof(msg)/sizeof(msg[0])-1] = '\0';
+    DISPLAY_STRING(msg);
+#endif /* SHOW_RESULTS */
+}
+
+#ifdef LINUX
+static
+dr_signal_action_t event_signal(void *drcontext, dr_siginfo_t *info)
+{
+    ATOMIC_INC(num_signals);
+
+    if (info->sig == SIGTERM) {
+        /* Ignore TERM */
+        return DR_SIGNAL_SUPPRESS;
+    } else if (info->sig == SIGSEGV) {
+        /* Skip the faulting instruction.  This is a sample only! */
+        app_pc pc = decode_next_pc(drcontext, info->mcontext.xip);
+        if (pc != NULL)
+            info->mcontext.xip = pc;
+        return DR_SIGNAL_REDIRECT;
+    }
+
+    return DR_SIGNAL_DELIVER;
+}
+#endif
diff --git a/api/samples/stats.c b/api/samples/stats.c
new file mode 100644
index 000000000..761756905
--- /dev/null
+++ b/api/samples/stats.c
@@ -0,0 +1,657 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Code Manipulation API Sample:
+ * stats.c
+ *
+ * Serves as an example of how to create custom statistics and export
+ * them in shared memory (without using any Windows API libraries, and
+ * thus remaining transparent).  These statistics can be viewed using
+ * the provided statistics viewer.  This code also documents the
+ * official shared memory layout required by the statistics viewer.
+ */
+
+#include "dr_api.h"
+#include <stddef.h> /* for offsetof */
+
+#ifndef WINDOWS
+# error WINDOWS-only!
+#endif
+
+#ifdef WINDOWS
+# define DISPLAY_STRING(msg) dr_messagebox(msg)
+#else
+# define DISPLAY_STRING(msg) dr_printf("%s\n", msg);
+#endif
+
+/* We export a set of stats in shared memory.
+ * drgui.exe reads and displays them.
+ */
+const char *stat_names[] = {
+    /* drgui.exe displays CLIENTSTAT_NAME_MAX_LEN chars for each name */
+    "Instructions",
+    "Floating point instrs",
+    "System calls",
+};
+
+/* Be sure to prefix "Global\" to ensure this is visible across sessions,
+ * except on NT where Global\ is not supported.
+ */
+#define CLIENT_SHMEM_KEY_NT_L L"DynamoRIO_Client_Statistics"
+#define CLIENT_SHMEM_KEY_L    L"Global\\DynamoRIO_Client_Statistics"
+#define CLIENTSTAT_NAME_MAX_LEN 50
+#define NUM_STATS (sizeof(stat_names)/sizeof(char*))
+
+/* Statistics are all 64-bit for x64.  At some point we'll add per-stat
+ * typing, but for now we have identical types dependent on the platform.
+ */
+#ifdef X86_64
+# error Statistics sample and graphical viewer are not yet supported for 64-bit
+typedef int64 stats_int_t;
+# define STAT_FORMAT_CODE UINT64_FORMAT_CODE
+#else
+typedef int stats_int_t;
+# define STAT_FORMAT_CODE "d"
+#endif
+
+/* we allocate this struct in the shared memory: */
+typedef struct _client_stats {
+    uint num_stats;
+    bool exited;
+    uint pid;
+    /* we need a copy of all the names here */
+    char names[NUM_STATS][CLIENTSTAT_NAME_MAX_LEN];
+    stats_int_t num_instrs;
+    stats_int_t num_flops;
+    stats_int_t num_syscalls;
+} client_stats;
+
+/* we directly increment the global counters using a lock prefix */
+static client_stats *stats;
+
+/***************************************************************************/
+/* shared memory setup */
+
+/* we have multiple shared memories: one that holds the count of
+ * statistics instances, and then one per statistics struct */
+static HANDLE shared_map_count;
+static PVOID shared_view_count;
+static int * shared_count;
+static HANDLE shared_map;
+static PVOID shared_view;
+#define KEYNAME_MAXLEN 128
+static wchar_t shared_keyname[KEYNAME_MAXLEN];
+
+/* returns current contents of addr and replaces contents with value */
+static __declspec(naked) /* no prolog/epilog */
+uint
+atomic_swap(void *addr, uint value)
+{
+    __asm {
+       /* cl was using ebp even though naked so we hand-code param access */
+        mov eax, dword ptr [esp+8] /* value */
+        mov ecx, dword ptr [esp+4] /* addr */
+        xchg (ecx), eax
+        ret
+    }
+}
+
+typedef LONG NTSTATUS;
+#define NT_SUCCESS(Status) ((Status) >= 0)
+
+#define NT_CURRENT_PROCESS ( (HANDLE) -1 )
+#define NT_CURRENT_THREAD  ( (HANDLE) -2 )
+
+/* from DDK2003SP1/3790.1830/inc/wnet/ntdef.h */
+#define OBJ_CASE_INSENSITIVE    0x00000040L
+#define OBJ_OPENIF              0x00000080L
+
+typedef struct _UNICODE_STRING {
+    /* Length field is size in bytes not counting final 0 */
+    USHORT Length;
+    USHORT MaximumLength;
+    PWSTR  Buffer;
+} UNICODE_STRING;
+typedef UNICODE_STRING *PUNICODE_STRING;
+
+typedef struct _STRING {
+  USHORT  Length;
+  USHORT  MaximumLength;
+  PCHAR  Buffer;
+} ANSI_STRING;
+typedef ANSI_STRING *PANSI_STRING;
+
+typedef struct _OBJECT_ATTRIBUTES {
+    ULONG Length;
+    HANDLE RootDirectory;
+    PUNICODE_STRING ObjectName;
+    ULONG Attributes;
+    PSECURITY_DESCRIPTOR SecurityDescriptor;
+    PVOID SecurityQualityOfService;  // Points to type SECURITY_QUALITY_OF_SERVICE
+} OBJECT_ATTRIBUTES;
+typedef OBJECT_ATTRIBUTES *POBJECT_ATTRIBUTES;
+
+#define InitializeObjectAttributes( p, n, a, r, s ) { \
+    (p)->Length = sizeof( OBJECT_ATTRIBUTES );          \
+    (p)->RootDirectory = r;                             \
+    (p)->Attributes = a;                                \
+    (p)->ObjectName = n;                                \
+    (p)->SecurityDescriptor = s;                        \
+    (p)->SecurityQualityOfService = NULL;               \
+    }
+
+/* from ntddk.h */
+typedef enum _SECTION_INHERIT {
+    ViewShare = 1,
+    ViewUnmap = 2
+} SECTION_INHERIT;
+
+#define DIRECTORY_ALL_ACCESS (STANDARD_RIGHTS_REQUIRED | 0xF)
+#define STATUS_OBJECT_NAME_EXISTS        ((NTSTATUS)0x40000000L)
+
+#define GET_NTDLL(NtFunction, signature) NTSYSAPI NTSTATUS NTAPI NtFunction signature
+GET_NTDLL(NtCreateSection, (OUT PHANDLE SectionHandle,
+                            IN ACCESS_MASK DesiredAccess,
+                            IN POBJECT_ATTRIBUTES ObjectAttributes,
+                            IN PLARGE_INTEGER SectionSize OPTIONAL,
+                            IN ULONG Protect,
+                            IN ULONG Attributes,
+                            IN HANDLE FileHandle));
+GET_NTDLL(NtMapViewOfSection, (IN HANDLE           SectionHandle,
+                               IN HANDLE           ProcessHandle,
+                               IN OUT PVOID       *BaseAddress,
+                               IN ULONG            ZeroBits,
+                               IN SIZE_T           CommitSize,
+                               IN OUT PLARGE_INTEGER  SectionOffset OPTIONAL,
+                               IN OUT PSIZE_T      ViewSize,
+                               IN SECTION_INHERIT  InheritDisposition,
+                               IN ULONG            AllocationType,
+                               IN ULONG            Protect));
+GET_NTDLL(NtUnmapViewOfSection, (IN HANDLE         ProcessHandle,
+                                 IN PVOID          BaseAddress));
+
+GET_NTDLL(NtOpenDirectoryObject, (PHANDLE, ACCESS_MASK ,POBJECT_ATTRIBUTES));
+
+typedef struct _PEB {
+    BOOLEAN InheritedAddressSpace;
+    BOOLEAN ReadImageFileExecOptions;
+    BOOLEAN BeingDebugged;
+    BOOLEAN Spare;
+    HANDLE Mutant;
+    PVOID ImageBaseAddress; 
+    PVOID LoaderData;
+    PVOID ProcessParameters;
+    PVOID SubSystemData;
+    PVOID ProcessHeap;
+    PVOID FastPebLock;
+    PVOID FastPebLockRoutine;
+    PVOID FastPebUnlockRoutine;
+    ULONG EnvironmentUpdateCount;
+    PVOID KernelCallbackTable;
+    PVOID EventLogSection;
+    PVOID EventLog;
+    PVOID FreeList;
+    ULONG TlsExpansionCounter;
+    PVOID TlsBitmap;
+    ULONG TlsBitmapBits[0x2];
+    PVOID ReadOnlySharedMemoryBase;
+    PVOID ReadOnlySharedMemoryHeap;
+    PVOID ReadOnlyStaticServerData;
+    PVOID AnsiCodePageData;
+    PVOID OemCodePageData;
+    PVOID UnicodeCaseTableData;
+    ULONG NumberOfProcessors;
+    ULONG NtGlobalFlag;
+    BYTE Spare2[0x4];
+    LARGE_INTEGER CriticalSectionTimeout;
+    ULONG HeapSegmentReserve;
+    ULONG HeapSegmentCommit; 
+    ULONG HeapDeCommitTotalFreeThreshold;
+    ULONG HeapDeCommitFreeBlockThreshold;
+    ULONG NumberOfHeaps;
+    ULONG MaximumNumberOfHeaps;
+    PVOID *ProcessHeaps;
+    PVOID GdiSharedHandleTable; 
+    PVOID ProcessStarterHelper; 
+    PVOID GdiDCAttributeList; 
+    PRTL_CRITICAL_SECTION LoaderLock;
+    ULONG OSMajorVersion;
+    ULONG OSMinorVersion;
+    ULONG OSBuildNumber;
+    ULONG OSPlatformId;
+    // ...
+} PEB;
+
+typedef struct _TEB {
+    PVOID /* PEXCEPTION_REGISTRATION_RECORD */ ExceptionList;
+    PVOID StackUserTop;  
+    PVOID StackUserBase; 
+    PVOID SubSystemTib;  
+    ULONG FiberData;     
+    PVOID ArbitraryUser; 
+    struct _TEB *Self;   
+    DWORD EnvironmentPtr;
+    DWORD ProcessID;     
+    DWORD ThreadID;      
+    DWORD RpcHandle;     
+    PVOID* TLSArray;     
+    PEB* PEBPtr;       
+    DWORD LastErrorValue;
+    // ...
+} TEB;
+
+TEB *
+get_TEB(void)
+{
+    return (TEB *) __readfsdword(offsetof(TEB, Self));
+}
+
+static uint
+getpid(void)
+{
+    return get_TEB()->ProcessID;
+}
+
+static int
+get_last_error(void)
+{
+    /* RtlGetLastWin32Error is only available on XP+? */
+    return get_TEB()->LastErrorValue;
+}
+
+static void
+set_last_error(int val)
+{
+    get_TEB()->LastErrorValue = val;
+}
+
+static NTSTATUS
+wchar_to_unicode(PUNICODE_STRING dst, PCWSTR src)
+{
+    NTSTATUS res;
+    GET_NTDLL(RtlInitUnicodeString, (IN OUT PUNICODE_STRING DestinationString,
+                                     IN PCWSTR SourceString));
+    res = RtlInitUnicodeString(dst, src);
+    return res;
+}
+
+static bool
+is_windows_NT(void)
+{
+    PEB *peb = get_TEB()->PEBPtr;
+    DR_ASSERT(peb != NULL);
+    return (peb->OSPlatformId == VER_PLATFORM_WIN32_NT && peb->OSMajorVersion == 4);
+}
+
+static HANDLE
+get_BNO_handle(void)
+{
+    NTSTATUS res;
+    HANDLE h;
+    OBJECT_ATTRIBUTES oa;
+    UNICODE_STRING name_unicode;
+    res = wchar_to_unicode(&name_unicode, L"\\BaseNamedObjects");
+    DR_ASSERT(NT_SUCCESS(res));
+    InitializeObjectAttributes(&oa, &name_unicode, OBJ_CASE_INSENSITIVE, NULL, NULL);
+    res = NtOpenDirectoryObject(&h, DIRECTORY_ALL_ACCESS &
+                                ~(DELETE | WRITE_DAC | WRITE_OWNER),
+                                &oa);
+    DR_ASSERT(NT_SUCCESS(res));
+    return h;
+}
+
+static HANDLE
+my_CreateFileMappingW(HANDLE file, SECURITY_ATTRIBUTES *attrb, 
+                      uint prot, uint size_high, uint size_low, wchar_t *name)
+{
+    NTSTATUS res;
+    HANDLE section;
+    UNICODE_STRING section_name_unicode;
+    OBJECT_ATTRIBUTES section_attributes;
+    uint access;
+    LARGE_INTEGER li_size;
+    li_size.HighPart = size_high;
+    li_size.LowPart = size_low;
+    DR_ASSERT(name != NULL);
+    res = wchar_to_unicode(&section_name_unicode, name);
+    DR_ASSERT(NT_SUCCESS(res));
+    if (!NT_SUCCESS(res))
+        return NULL;
+    if (file == INVALID_HANDLE_VALUE) {
+        DR_ASSERT(size_high != 0 || size_low != 0);
+        file = NULL;
+    }
+    /* we ignore dacl for now */
+    InitializeObjectAttributes(&section_attributes, &section_name_unicode,
+                               OBJ_CASE_INSENSITIVE | OBJ_OPENIF,
+                               get_BNO_handle(), NULL);
+    access = STANDARD_RIGHTS_REQUIRED | SECTION_QUERY | SECTION_MAP_READ;
+    if (prot == PAGE_READWRITE)
+        access |= (SECTION_MAP_WRITE | SECTION_MAP_READ);
+    res = NtCreateSection(&section, access,
+                          &section_attributes,
+                          (size_high == 0 && size_low == 0) ? NULL /* entire file */
+                          : &li_size,
+                          prot, SEC_COMMIT, file);
+    if (!NT_SUCCESS(res)) {
+        dr_fprintf(STDERR, "Error in my_CreateFileMappingW: "PFX"\n", res);
+        DR_ASSERT(false);
+        return NULL;
+    }
+    if (res == STATUS_OBJECT_NAME_EXISTS) {
+        set_last_error(ERROR_ALREADY_EXISTS);
+    } else
+        set_last_error(ERROR_SUCCESS);
+    return section;
+}
+
+static void *
+my_MapViewOfFile(HANDLE section, uint file_prot, uint offs_high, uint offs_low,
+                 size_t size)
+{
+    NTSTATUS res;
+    void *map = NULL;
+    uint prot;
+    LARGE_INTEGER li_offs;
+    li_offs.HighPart = offs_high;
+    li_offs.LowPart = offs_low;
+    /* convert FILE_ flags to PAGE_ flags */
+    if ((file_prot & FILE_MAP_WRITE) != 0)
+        prot = PAGE_READWRITE;
+    else if ((file_prot & FILE_MAP_READ) != 0)
+        prot = PAGE_READONLY;
+    else if ((file_prot & FILE_MAP_COPY) != 0)
+        prot = PAGE_WRITECOPY;
+    else
+        prot = PAGE_NOACCESS;
+    res = NtMapViewOfSection(section, NT_CURRENT_PROCESS, &map, 0,
+                             0 /* no commit */, &li_offs, (PSIZE_T) &size, 
+                             ViewUnmap, 0, prot);
+    if (!NT_SUCCESS(res)) {
+        dr_fprintf(STDERR, "Error in my_CreateFileMappingW: "PFX"\n", res);
+        DR_ASSERT(false);
+        return NULL;
+    }
+    return map;
+
+}
+
+static void
+my_UnmapViewOfFile(HANDLE base)
+{
+    NTSTATUS res = NtUnmapViewOfSection(NT_CURRENT_PROCESS, base);
+    DR_ASSERT(NT_SUCCESS(res));
+}
+
+static client_stats *
+shared_memory_init(void)
+{
+    bool is_NT = is_windows_NT();
+    int num;
+    int pos;
+    SECURITY_ATTRIBUTES attrb;
+    SECURITY_DESCRIPTOR descrip;
+    SECURITY_DESCRIPTOR_CONTROL sd_control = SE_DACL_PRESENT;
+    attrb.nLength = sizeof(SECURITY_ATTRIBUTES);
+    attrb.bInheritHandle = FALSE;
+    /* Set security descriptor by hand (WIN32 API routines to do so are
+     * in advapi).  This may not be fully forward compatible.
+     */
+    /* initialize the descriptor */
+    memset(&descrip, 0, sizeof(SECURITY_DESCRIPTOR));
+    /* Set the dacl present flag. DACL will be NULL from memset which is as 
+     * desired since will give full acccess to everyone: security
+     * is not a concern for these stats.
+     */
+    descrip.Control = sd_control;
+    /* set revision appropriately */
+    descrip.Revision = SECURITY_DESCRIPTOR_REVISION;
+    attrb.lpSecurityDescriptor = &descrip;
+    /* We do not want to rely on the registry.
+     * Instead, a piece of shared memory with the key base name holds the
+     * total number of stats instances.
+     */
+    shared_map_count =
+        my_CreateFileMappingW(INVALID_HANDLE_VALUE, &attrb, 
+                              PAGE_READWRITE, 0, sizeof(client_stats), 
+                              is_NT ? CLIENT_SHMEM_KEY_NT_L : CLIENT_SHMEM_KEY_L);
+    DR_ASSERT(shared_map_count != NULL);
+    shared_view_count = 
+        my_MapViewOfFile(shared_map_count, FILE_MAP_READ|FILE_MAP_WRITE, 0, 0, 0);
+    DR_ASSERT(shared_view_count != NULL);
+    shared_count = (int *) shared_view_count;
+    /* ASSUMPTION: memory is initialized to 0!
+     * otherwise our protocol won't work
+     * it's hard to build a protocol to initialize it to 0 -- if you want
+     * to add one, feel free, but make sure it's correct
+     */
+    do {
+        pos = (int) atomic_swap(shared_count, (uint) -1);
+        /* if get -1 back, someone else is looking at it */
+    } while (pos == -1);
+    /* now increment it */
+    atomic_swap(shared_count, pos+1);
+
+    num = 0;
+    while (1) {
+        _snwprintf(shared_keyname, KEYNAME_MAXLEN, L"%s.%03d",
+                   is_NT ? CLIENT_SHMEM_KEY_NT_L : CLIENT_SHMEM_KEY_L, num);
+        shared_map = my_CreateFileMappingW(INVALID_HANDLE_VALUE, &attrb, 
+                                           PAGE_READWRITE, 0, 
+                                           sizeof(client_stats), 
+                                           shared_keyname);
+        if (shared_map != NULL && get_last_error() == ERROR_ALREADY_EXISTS) { 
+            dr_close_file(shared_map); 
+            shared_map = NULL; 
+        }
+        if (shared_map != NULL)
+            break;
+        num++;
+    }
+    dr_log(NULL, LOG_ALL, 1, "Shared memory key is: \"%S\"\n", shared_keyname);
+    dr_fprintf(STDERR, "Shared memory key is: \"%S\"\n", shared_keyname);
+    shared_view = my_MapViewOfFile(shared_map, FILE_MAP_READ|FILE_MAP_WRITE,
+                                   0, 0, 0);
+    DR_ASSERT(shared_view != NULL);
+    return (client_stats *) shared_view;
+}
+
+static void
+shared_memory_exit(void)
+{
+    int pos; 
+    stats->exited = true;
+    /* close down statistics */
+    my_UnmapViewOfFile(shared_view);
+    dr_close_file(shared_map);
+    /* decrement count, then unmap */
+    do {
+        pos = atomic_swap(shared_count, (uint) -1);
+        /* if get -1 back, someone else is looking at it */
+    } while (pos == -1);
+    /* now increment it */
+    atomic_swap(shared_count, pos-1);
+    my_UnmapViewOfFile(shared_view_count);
+    dr_close_file(shared_map_count);
+}
+/***************************************************************************/
+
+static void event_exit(void);
+static dr_emit_flags_t event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                                         bool for_trace, bool translating);
+
+static client_id_t my_id;
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    uint i;
+
+    my_id = id;
+    /* make it easy to tell, by looking at log file, which client executed */
+    dr_log(NULL, LOG_ALL, 1, "Client 'stats' initializing\n");
+
+    stats = shared_memory_init();
+    memset(stats, 0, sizeof(stats));
+    stats->num_stats = NUM_STATS;
+    stats->pid = getpid();
+    for (i=0; i<NUM_STATS; i++) {
+        strncpy(stats->names[i], stat_names[i], CLIENTSTAT_NAME_MAX_LEN);
+        stats->names[i][CLIENTSTAT_NAME_MAX_LEN-1] = '\0';
+    }
+    dr_register_bb_event(event_basic_block);
+    dr_register_exit_event(event_exit);
+}
+
+#ifdef WINDOWS
+# define DIRSEP '\\'
+#else
+# define DIRSEP '/'
+#endif
+
+static void 
+event_exit(void)
+{
+    file_t f;
+    /* display the results */
+    char msg[512];
+    char fname[512];
+    char *dirsep;
+    int len;
+    len = snprintf(msg, sizeof(msg)/sizeof(msg[0]),
+                   "Instrumentation results:\n"
+                   "  saw %" STAT_FORMAT_CODE " flops\n", stats->num_flops);
+    DR_ASSERT(len > 0);
+    msg[sizeof(msg)/sizeof(msg[0])-1] = '\0';
+#ifdef SHOW_RESULTS
+    DISPLAY_STRING(msg);
+#endif /* SHOW_RESULTS */
+
+    /* On Windows we need an absolute path so we place it in
+     * the same directory as our library.
+     */
+    len = snprintf(fname, sizeof(fname)/sizeof(fname[0]),
+                   "%s", dr_get_client_path(my_id));
+    DR_ASSERT(len > 0);
+    for (dirsep = fname + len; *dirsep != DIRSEP; dirsep--)
+        DR_ASSERT(dirsep > fname);
+    len = snprintf(dirsep + 1, (sizeof(fname) - (dirsep - fname))/sizeof(fname[0]),
+                   "stats.%d.log", dr_get_process_id());
+    DR_ASSERT(len > 0);
+    fname[sizeof(fname)/sizeof(fname[0])-1] = '\0';
+    f = dr_open_file(fname, DR_FILE_WRITE_OVERWRITE);
+    DR_ASSERT(f != INVALID_FILE);
+    dr_fprintf(f, "%s\n", msg);
+    dr_close_file(f);
+
+    shared_memory_exit();
+}
+
+static void
+insert_inc(void *drcontext, instrlist_t *bb, instr_t *where,
+           stats_int_t *addr, uint incby)
+{
+    instr_t *inc;
+    opnd_t immed;
+    /* FIXME: we're using stats_int_t, but our inc here does not support 64-bit */
+    if (incby <= 0x7f)
+        immed = OPND_CREATE_INT8(incby);
+    else /* unlikely but possible */
+        immed = OPND_CREATE_INT32(incby);
+    inc = INSTR_CREATE_add(drcontext, OPND_CREATE_MEM32(REG_NULL, (int)addr), immed);
+    /* make it thread-safe (only works if it doesn't straddle a cache line) */
+    instr_set_prefix_flag(inc, PREFIX_LOCK);
+    DR_ASSERT((((ptr_uint_t)addr) & 0x3) == 0); /* 4-aligned => single cache line */
+    instrlist_meta_preinsert(bb, where, inc);
+}
+
+static dr_emit_flags_t 
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                  bool for_trace, bool translating)
+{
+    instr_t *instr;
+    int op;
+    int num_instrs = 0;
+    int num_flops = 0;
+    int num_syscalls = 0;
+#ifdef VERBOSE
+    dr_printf("in dr_basic_block(tag="PFX")\n", tag);
+# ifdef VERBOSE_VERBOSE
+    instrlist_disassemble(drcontext, tag, bb, STDOUT);
+# endif
+#endif
+    /* count up # flops, then do single increment at end */
+    for (instr = instrlist_first(bb); instr != NULL; instr = instr_get_next(instr)) {
+        num_instrs++;
+        op = instr_get_opcode(instr);
+        if (op >= OP_fadd && op <= OP_fcomip) {
+            /* FIXME: exclude loads and stores */
+#ifdef VERBOSE
+            dr_print_instr(drcontext, STDOUT, instr, "Found flop: ");
+#endif
+            num_flops++;
+        }
+        if (instr_is_syscall(instr)) {
+            num_syscalls++;
+        }
+    }
+    if (num_instrs > 0 || num_flops > 0 || num_syscalls > 0) {
+        uint eflags;
+        bool need_to_save = true;
+        /* insert increment at start, for maximum prob. of not needing
+         * to save flags
+         */
+        for (instr = instrlist_first(bb); instr != NULL;
+             instr = instr_get_next(instr)) {
+            eflags = instr_get_eflags(instr);
+            /* could be more sophisticated and look beyond exit */
+            if (instr_is_exit_cti(instr) || (eflags & EFLAGS_READ_6) != 0)
+                break;
+            if ((eflags & EFLAGS_WRITE_6) == EFLAGS_WRITE_6) {
+                need_to_save = false;
+                break;
+            }
+        }
+        instr = instrlist_first(bb);
+        if (need_to_save)
+            dr_save_arith_flags(drcontext, bb, instr, SPILL_SLOT_1);
+        if (num_instrs > 0)
+            insert_inc(drcontext, bb, instr, &stats->num_instrs, num_instrs);
+        if (num_flops > 0)
+            insert_inc(drcontext, bb, instr, &stats->num_flops, num_flops);
+        if (num_syscalls > 0)
+            insert_inc(drcontext, bb, instr, &stats->num_syscalls, num_syscalls);
+        if (need_to_save)
+            dr_restore_arith_flags(drcontext, bb, instr, SPILL_SLOT_1);
+    }
+    return DR_EMIT_DEFAULT;
+}
diff --git a/api/samples/stl_test.cpp b/api/samples/stl_test.cpp
new file mode 100644
index 000000000..99bdb04da
--- /dev/null
+++ b/api/samples/stl_test.cpp
@@ -0,0 +1,131 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* C++ sample client:
+ * stl_test.cpp
+ *
+ * Some simple tests with C++ STL containers to make sure they 
+ * work with a DR client.
+ */
+
+#include "dr_api.h"
+#include <vector>
+#include <list>
+#include <map>
+
+using namespace std;
+
+
+DR_EXPORT void
+dr_init(client_id_t client_id)
+{
+    int i;
+    bool success = true;
+
+    //
+    // Put values in a vector and read them out
+    //
+#ifdef SHOW_RESULTS
+    dr_printf("testing vector...");
+#endif
+
+    vector<int>* v = new vector<int>();
+    for (i=0; i<5; i++) {
+        v->push_back(i);
+    }
+    
+    for (i=0; i<5; i++) {
+#ifdef SHOW_RESULTS
+        dr_printf("%d ", (*v)[i]);
+#endif
+        if ((*v)[i] != i) {
+            success = false;
+        }
+    }    
+    delete v;
+
+    //
+    // Put values in a list and read them out
+    //
+#ifdef SHOW_RESULTS
+    dr_printf("\ntesting list...");
+#endif
+
+    list<int> l;
+    for (i=0; i<5; i++) {
+        l.push_back(i);
+    }
+
+    i = 0;
+    for (list<int>::iterator l_iter = l.begin(); l_iter != l.end(); l_iter++) {
+#ifdef SHOW_RESULTS
+        dr_printf("%d ", *l_iter);
+#endif
+        if (*l_iter != i) {
+            success = false;
+        }
+        i++;
+    }
+
+    //
+    // Put values in a map and read them out
+    //
+#ifdef SHOW_RESULTS
+    dr_printf("\ntesting map...");
+#endif
+
+    map<int,int> m;
+    for (i=0; i<5; i++) {
+        m[i] = i;
+    }
+
+    for (i=0; i<5; i++) {
+#ifdef SHOW_RESULTS
+        dr_printf("%d ", m[i]);
+#endif
+        if (m[i] != i) {
+            success = false;
+        }
+    }
+
+    //
+    // Done; print summary
+    //
+#ifdef SHOW_RESULTS
+    if (success) {
+        dr_printf("\nSUCCESS\n");
+    }
+    else {
+        dr_printf("\nFAILURE\n");
+    }
+#endif
+}
diff --git a/api/samples/strace.c b/api/samples/strace.c
new file mode 100644
index 000000000..ae4f837f4
--- /dev/null
+++ b/api/samples/strace.c
@@ -0,0 +1,315 @@
+/* **********************************************************
+ * Copyright (c) 2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Code Manipulation API Sample:
+ * strace.c
+ *
+ * Monitors system calls.  As an example, we modify SYS_write/NtWriteFile.
+ * On Windows we have to take extra steps to find system call numbers
+ * and handle emulation parameters for WOW64 (Windows-On-Windows: 32-bit
+ * applications on 64-bit Windows).
+ */
+
+#include "dr_api.h"
+#include <string.h> /* memset */
+
+#ifdef LINUX
+# include <syscall.h>
+#endif
+
+#ifdef WINDOWS
+# define DISPLAY_STRING(msg) dr_messagebox(msg)
+# define ATOMIC_INC(var) _InterlockedIncrement(&var)
+#else
+# define DISPLAY_STRING(msg) dr_printf("%s\n", msg);
+# define ATOMIC_INC(var) __asm__ __volatile__("lock incl %0" : "=m" (var) : : "memory")
+#endif
+
+/* Some syscalls have more args, but this is the max we need for SYS_write/NtWriteFile */
+#ifdef WINDOWS
+# define SYS_MAX_ARGS 9
+#else
+# define SYS_MAX_ARGS 3
+#endif
+
+/* Thread-local data structure for storing system call parameters */
+typedef struct {
+    reg_t param[SYS_MAX_ARGS];
+#ifdef WINDOWS
+    reg_t xcx; /* emulation parameter for WOW64 */
+#endif
+    bool repeat;
+} per_thread_t;
+
+/* The system call number of SYS_write/NtWriteFile */
+static int write_sysnum;
+
+static int num_syscalls;
+
+static int get_write_sysnum(void);
+static void event_exit(void);
+static void event_thread_init(void *drcontext);
+static void event_thread_exit(void *drcontext);
+static bool event_filter_syscall(void *drcontext, int sysnum);
+static bool event_pre_syscall(void *drcontext, int sysnum);
+static void event_post_syscall(void *drcontext, int sysnum);
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    write_sysnum = get_write_sysnum();
+    dr_register_thread_init_event(event_thread_init);
+    dr_register_thread_exit_event(event_thread_exit);
+    dr_register_filter_syscall_event(event_filter_syscall);
+    dr_register_pre_syscall_event(event_pre_syscall);
+    dr_register_post_syscall_event(event_post_syscall);
+    dr_register_exit_event(event_exit);
+#ifdef SHOW_RESULTS
+    if (dr_is_notify_on())
+	dr_fprintf(STDERR, "Client strace is running\n");
+#endif
+}
+
+static void 
+show_results(void)
+{
+#ifdef SHOW_RESULTS
+    char msg[512];
+    int len;
+    /* Note that using %f with dr_printf or dr_fprintf on Windows will print
+     * garbage as they use ntdll._vsnprintf, so we must use snprintf.
+     */
+    len = snprintf(msg, sizeof(msg)/sizeof(msg[0]),
+                   "<Number of system calls seen: %d>", num_syscalls);
+    DR_ASSERT(len > 0);
+    msg[sizeof(msg)/sizeof(msg[0])-1] = '\0';
+    DISPLAY_STRING(msg);
+#endif /* SHOW_RESULTS */
+}
+
+static void 
+event_exit(void)
+{
+    show_results();
+}
+
+static void
+event_thread_init(void *drcontext)
+{
+    /* create an instance of our data structure for this thread */
+    per_thread_t *data = (per_thread_t *)
+        dr_thread_alloc(drcontext, sizeof(per_thread_t));
+    memset(data, 0, sizeof(*data));
+    /* store it in the slot provided in the drcontext */
+    dr_set_tls_field(drcontext, data);
+}
+
+static void 
+event_thread_exit(void *drcontext)
+{
+    per_thread_t *data = (per_thread_t *) dr_get_tls_field(drcontext);
+    /* clean up memory */
+    dr_thread_free(drcontext, data, sizeof(per_thread_t));
+}
+
+static bool
+event_filter_syscall(void *drcontext, int sysnum)
+{
+    return true; /* intercept everything */
+}
+
+static bool
+event_pre_syscall(void *drcontext, int sysnum)
+{
+    ATOMIC_INC(num_syscalls);
+#ifdef LINUX
+    if (sysnum == SYS_execve) {
+        /* our stats will be re-set post-execve so display now */
+        show_results();
+# ifdef SHOW_RESULTS
+        dr_fprintf(STDERR, "<---- execve ---->\n");
+# endif
+    }
+#endif
+#ifdef SHOW_RESULTS
+    dr_fprintf(STDERR, "[%d] "PFX" "PFX" "PFX"\n",
+               sysnum, 
+               dr_syscall_get_param(drcontext, 0),
+               dr_syscall_get_param(drcontext, 1),
+               dr_syscall_get_param(drcontext, 2));
+#endif
+    if (sysnum == write_sysnum) {
+        /* store params for access post-syscall */
+        per_thread_t *data = (per_thread_t *) dr_get_tls_field(drcontext);
+        int i;
+#ifdef WINDOWS
+        /* stderr and stdout are identical in our cygwin rxvt shell so for
+         * our example we suppress output starting with 'H' instead
+         */
+        byte *output = (byte *) dr_syscall_get_param(drcontext, 5);
+        size_t len = dr_syscall_get_param(drcontext, 6);
+        byte first;
+        size_t read;
+        bool ok = dr_safe_read(output, 1, &first, &read);
+        if (!ok || read != 1)
+            return true; /* data unreadable: execute normally */
+        if (dr_is_wow64()) {
+            /* store the xcx emulation parameter for wow64 */
+            dr_mcontext_t mc;
+            dr_get_mcontext(drcontext, &mc, NULL);
+            data->xcx = mc.xcx;
+        }
+#endif
+        for (i = 0; i < SYS_MAX_ARGS; i++)
+            data->param[i] = dr_syscall_get_param(drcontext, i);
+        /* suppress stderr */
+        if (dr_syscall_get_param(drcontext, 0) == (reg_t) STDERR
+#ifdef WINDOWS
+            && first == 'H'
+#endif
+            ) {
+            /* pretend it succeeded */
+#ifdef LINUX
+            /* return the #bytes == 3rd param */
+            dr_syscall_set_result(drcontext, dr_syscall_get_param(drcontext, 2));
+#else
+            /* we should also set the IO_STATUS_BLOCK.Information field */
+            dr_syscall_set_result(drcontext, 0);
+#endif
+#ifdef SHOW_RESULTS
+            dr_fprintf(STDERR, "  [%d] => skipped\n", sysnum);
+#endif
+            return false; /* skip syscall */
+        } else if (dr_syscall_get_param(drcontext, 0) == (reg_t) STDOUT) {
+            if (!data->repeat) {
+                /* redirect stdout to stderr (unless it's our repeat) */
+                dr_syscall_set_param(drcontext, 0, (reg_t) STDERR);
+            }
+            /* we're going to repeat this syscall once */
+            data->repeat = !data->repeat;
+        }
+    }
+    return true; /* execute normally */
+}
+
+static void
+event_post_syscall(void *drcontext, int sysnum)
+{
+#ifdef SHOW_RESULTS
+    dr_fprintf(STDERR, "  [%d] => "PFX" ("SZFMT")\n",
+               sysnum, 
+               dr_syscall_get_result(drcontext),
+               (ptr_int_t)dr_syscall_get_result(drcontext));
+#endif
+    if (sysnum == write_sysnum) {
+        per_thread_t *data = (per_thread_t *) dr_get_tls_field(drcontext);
+        /* we repeat a write originally to stdout that we redirected to
+         * stderr: on the repeat we use stdout
+         */
+        if (data->repeat) {
+            /* repeat syscall with stdout */
+            int i;
+            dr_syscall_set_sysnum(drcontext, write_sysnum);
+            dr_syscall_set_param(drcontext, 0, (reg_t) STDOUT);
+            for (i = 1; i < SYS_MAX_ARGS; i++) 
+                dr_syscall_set_param(drcontext, i, data->param[i]);
+#ifdef WINDOWS
+            if (dr_is_wow64()) {
+                /* Set the xcx emulation parameter for wow64: since
+                 * we're executing the same system call again we can
+                 * use that same parameter.  For new system calls we'd
+                 * need to determine the parameter from the ntdll
+                 * wrapper.
+                 */
+                dr_mcontext_t mc;
+                dr_get_mcontext(drcontext, &mc, NULL);
+                mc.xcx = data->xcx;
+                dr_set_mcontext(drcontext, &mc, NULL);
+            }
+#endif
+            dr_syscall_invoke_another(drcontext);
+        }
+    }
+}
+
+static int
+get_write_sysnum(void)
+{
+#ifdef LINUX
+    return SYS_write;
+#else
+    byte *entry;
+    module_data_t *data = dr_lookup_module_by_name("ntdll.dll");
+    DR_ASSERT(data != NULL);
+    entry = (byte *) dr_get_proc_address(data->start, "NtWriteFile");
+    DR_ASSERT(entry != NULL);
+    dr_free_module_data(data);
+    return decode_syscall_num(entry);
+#endif
+}
+
+#ifdef WINDOWS
+/* takes in an Nt syscall wrapper entry point */
+static int
+decode_syscall_num(byte *entry)
+{
+    void *drcontext = dr_get_current_drcontext();
+    int num = -1;
+    byte *pc = entry;
+    uint opc;
+    instr_t instr;
+    instr_init(drcontext, &instr);
+    do {
+        instr_reset(drcontext, &instr);
+        pc = decode(drcontext, pc, &instr);
+        DR_ASSERT(instr_valid(&instr));
+        opc = instr_get_opcode(&instr);
+        /* safety check: should only get 11 or 12 bytes in */
+        if (pc - entry > 20) {
+            DR_ASSERT(false);
+            instr_free(drcontext, &instr);
+            return -1;
+        }
+        /* Note that we'll failed if somebody has hooked the wrapper */
+        if (opc == OP_mov_imm && opnd_is_reg(instr_get_dst(&instr, 0)) &&
+            opnd_get_reg(instr_get_dst(&instr, 0)) == REG_EAX) {
+            DR_ASSERT(opnd_is_immed_int(instr_get_src(&instr, 0)));
+            num = opnd_get_immed_int(instr_get_src(&instr, 0));
+            break;
+        }
+        /* stop at call to vsyscall or at int itself */
+    } while (opc != OP_call_ind && opc != OP_int);
+    instr_free(drcontext, &instr);
+    DR_ASSERT(num > -1);
+    return num;
+}
+#endif /* WINDOWS */
diff --git a/api/samples/tracedump.c b/api/samples/tracedump.c
new file mode 100644
index 000000000..812543caa
--- /dev/null
+++ b/api/samples/tracedump.c
@@ -0,0 +1,253 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Code Manipulation API Sample:
+ * tracedump.c
+ *
+ * Disassembles a trace dump in binary format produced by the
+ * -tracedump_binary option.  Also illustrates the standalone API.
+ */
+
+#include "dr_api.h"
+#include <stdlib.h> /* for realloc */
+#include <assert.h>
+#include <stddef.h> /* for offsetof */
+#include <string.h> /* for memcpy */
+
+#define OUTPUT 0
+#define VERBOSE 0
+#define VERBOSE_VERBOSE 0
+
+/* binary trace dump format is in dr_tools.h */
+
+#define BUF_SIZE 4096
+
+static app_pc
+dis(void *drcontext, app_pc pc, app_pc display_pc)
+{
+    return disassemble_from_copy(drcontext, pc, display_pc, STDOUT,
+                                 true/*show display_pc*/, true/*bytes*/);
+}
+
+/* We use an array of SEPARATE_STUB_MAX_SIZE */
+#define STUB_STRUCT_MAX_SIZE (sizeof(tracedump_stub_data_t) + SEPARATE_STUB_MAX_SIZE)
+#define STUBIDX(stubs, i) \
+    ((tracedump_stub_data_t *)(((byte *)stubs)+(i)*STUB_STRUCT_MAX_SIZE))
+
+static void
+read_data(file_t f, void *drcontext)
+{
+    byte sbuf[BUF_SIZE];
+    byte *dbuf = NULL;
+    int read, dlen = 0, i;
+    byte *p = sbuf, *pc, *next_pc;
+    tracedump_trace_header_t hdrs;
+    tracedump_file_header_t fhdr;
+    tracedump_stub_data_t *stubs;
+    int next_stub_offs;
+    int cur_stub;
+
+    read = dr_read_file(f, &fhdr, sizeof(fhdr));
+    assert(read == sizeof(fhdr));
+    if (fhdr.version != _USES_DR_VERSION_) {
+        dr_fprintf(STDERR, "Error: file version %d does not match tool version %d\n",
+                   fhdr.version, _USES_DR_VERSION_);
+        return;
+    }
+    if (IF_X64_ELSE(!fhdr.x64, fhdr.x64)) {
+        dr_fprintf(STDERR, "Error: file architecture %s does not match tool's %s\n",
+                   fhdr.x64 ? "x64" : "x86", IF_X64_ELSE("x64", "x86"));
+        return;
+    }
+    if (fhdr.linkcount_size != 0 && fhdr.linkcount_size != 4 &&
+        fhdr.linkcount_size != 8) {
+        dr_fprintf(STDERR, "Error: file is not a trace dump\n");
+        return;
+    }
+
+    while (1) {
+        read = dr_read_file(f, &hdrs, sizeof(tracedump_trace_header_t));
+        if (read != sizeof(tracedump_trace_header_t))
+            break; /* assume end of file */
+#ifdef X86_64
+        set_x86_mode(drcontext, !hdrs.x64);
+#endif
+        dr_printf("\nTRACE # %d\n", hdrs.frag_id);
+        dr_printf("Tag = "PFX"\n", hdrs.tag);
+        if (hdrs.num_bbs > 0) {
+            int j;
+            app_pc tag;
+            dr_printf("\nORIGINAL CODE\n");
+            for (j=0; j<hdrs.num_bbs; j++) {
+                read = dr_read_file(f, sbuf, BB_ORIGIN_HEADER_SIZE);
+                assert(read == BB_ORIGIN_HEADER_SIZE);
+                p = sbuf;
+                tag = *((app_pc*)p);
+                p += sizeof(tag);
+                dr_printf("Basic block %d: tag "PFX"\n", j, tag);
+                i = *((int*)p);
+                p += sizeof(i);
+                dr_printf("Size: %d bytes\n", i);
+                if (i >= BUF_SIZE) {
+                    if (i >= dlen)
+                        dbuf = realloc(dbuf, i);
+                    p = dbuf;
+                } else
+                    p = sbuf;
+                read = dr_read_file(f, p, i);
+                assert(read == i);
+                pc = p;
+                while (pc - p < i) {
+                    pc = dis(drcontext, pc, tag + (pc - p));
+                }
+            }
+            dr_printf("END ORIGINAL CODE\n\n");
+        }
+        stubs = dr_global_alloc(hdrs.num_exits * STUB_STRUCT_MAX_SIZE);
+        assert(stubs != NULL);
+        next_stub_offs = hdrs.code_size;
+        if (fhdr.linkcount_size > 0) {
+            dr_printf("Exit stubs:\n");
+        }
+        for (i = 0; i<hdrs.num_exits; i++) {
+            assert(STUB_DATA_FIXED_SIZE+fhdr.linkcount_size < BUF_SIZE);
+            read = dr_read_file(f, sbuf, STUB_DATA_FIXED_SIZE+fhdr.linkcount_size);
+            assert(read == STUB_DATA_FIXED_SIZE+fhdr.linkcount_size);
+            p = sbuf;
+
+            /* We read in whole struct.  The union and code fields are of
+             * course variable-sized so we update them after.
+             */
+            *STUBIDX(stubs,i) = *(tracedump_stub_data_t *)p;
+            p += STUB_DATA_FIXED_SIZE;
+            /* linkcounts are no longer available but we have backward compatibility */
+            if (fhdr.linkcount_size == 8) {
+                STUBIDX(stubs,i)->count.count64 = *((uint64*)p);
+                p += 8;
+                dr_printf("\t#%d: target = "PFX", %s, count = %" UINT64_FORMAT_CODE "\n",
+                          i, STUBIDX(stubs,i)->target,
+                          STUBIDX(stubs,i)->linked ? "not linked" : "linked",
+                          STUBIDX(stubs,i)->count.count64);
+            } else if (fhdr.linkcount_size == 4) {
+                STUBIDX(stubs,i)->count.count32 = *((uint *)p);
+                p += 4;
+                dr_printf("\t#%d: target = "PFX", %s, count = %lu\n",
+                          i, STUBIDX(stubs,i)->target,
+                          STUBIDX(stubs,i)->linked ? "not linked" : "linked",
+                          STUBIDX(stubs,i)->count.count32);
+            } else {
+                dr_printf("\t#%d: target = "PFX", %s\n",
+                          i, STUBIDX(stubs,i)->target,
+                          STUBIDX(stubs,i)->linked ? "not linked" : "linked");
+            }
+            assert(p - sbuf == STUB_DATA_FIXED_SIZE+fhdr.linkcount_size);
+            if (STUBIDX(stubs,i)->stub_pc < hdrs.cache_start_pc ||
+                STUBIDX(stubs,i)->stub_pc >= hdrs.cache_start_pc + hdrs.code_size) {
+                assert(STUBIDX(stubs,i)->stub_size < BUF_SIZE);
+                assert(STUBIDX(stubs,i)->stub_size <= SEPARATE_STUB_MAX_SIZE);
+                read = dr_read_file(f, sbuf, STUBIDX(stubs,i)->stub_size);
+                assert(read == STUBIDX(stubs,i)->stub_size);
+                p = sbuf;
+                memcpy(STUBIDX(stubs,i)->stub_code, p, STUBIDX(stubs,i)->stub_size);
+                p += STUBIDX(stubs,i)->stub_size;
+            } else if (STUBIDX(stubs,i)->stub_pc - hdrs.cache_start_pc < next_stub_offs) {
+                next_stub_offs = STUBIDX(stubs,i)->stub_pc - hdrs.cache_start_pc;
+            }
+        }
+        if (hdrs.code_size >= dlen)
+            dbuf = realloc(dbuf, hdrs.code_size);
+        p = dbuf;
+        read = dr_read_file(f, p, hdrs.code_size);
+        assert(read == hdrs.code_size);
+        pc = p;
+        dr_printf("Size = %d\n", hdrs.code_size);
+        dr_printf("Body:\n");
+        dr_printf("  -------- indirect branch target entry: --------\n");
+        while (pc - p < next_stub_offs) {
+            if (pc - p == hdrs.entry_offs)
+                dr_printf("  -------- normal entry: --------\n");
+            next_pc = decode_next_pc(drcontext, pc);
+            if ((next_pc - p) == hdrs.entry_offs && (next_pc - pc == 6))
+                dr_printf("  -------- prefix entry: --------\n");
+            pc = dis(drcontext, pc, hdrs.cache_start_pc + (pc - p));
+        }
+        /* stubs */
+        for (cur_stub = 0; cur_stub < hdrs.num_exits; cur_stub++) {
+            bool separate = STUBIDX(stubs,cur_stub)->stub_pc < hdrs.cache_start_pc ||
+                STUBIDX(stubs,cur_stub)->stub_pc >= hdrs.cache_start_pc + hdrs.code_size;
+            app_pc stub_pc = (app_pc) STUBIDX(stubs,cur_stub)->stub_code;
+            dr_printf("  -------- exit stub %d: -------- <target: "PFX">\n",
+                      cur_stub, STUBIDX(stubs,cur_stub)->target);
+            next_stub_offs = hdrs.code_size;
+            for (i=cur_stub + 1; i<hdrs.num_exits; i++) {
+                if (STUBIDX(stubs,i)->stub_pc >= hdrs.cache_start_pc &&
+                    STUBIDX(stubs,i)->stub_pc < hdrs.cache_start_pc + hdrs.code_size) {
+                    next_stub_offs = STUBIDX(stubs,i)->stub_pc - hdrs.cache_start_pc;
+                    break;
+                }
+            }
+            if (separate) {
+                app_pc spc = stub_pc;
+                while (spc - stub_pc < STUBIDX(stubs,cur_stub)->stub_size) {
+                    spc = dis(drcontext, spc, STUBIDX(stubs,cur_stub)->stub_pc +
+                              (spc - stub_pc));
+                }
+            } else {
+                while (pc - p < next_stub_offs) {
+                    pc = dis(drcontext, pc, hdrs.cache_start_pc + (pc - p));
+                }
+            }
+        }
+        dr_printf("END TRACE %d\n", hdrs.frag_id);
+    }
+    if (dbuf != NULL)
+        free(dbuf);
+}
+    
+int
+main(int argc, char *argv[])
+{
+    file_t f;
+    void *drcontext = dr_standalone_init();
+    if (argc != 2) {
+        dr_fprintf(STDERR, "Usage: %s <tracefile>\n", argv[0]);
+        return 1;
+    }
+    f = dr_open_file(argv[1], DR_FILE_READ | DR_FILE_ALLOW_LARGE);
+    if (f == INVALID_FILE) {
+        dr_fprintf(STDERR, "Error opening %s\n", argv[1]);
+        return 1;
+    }
+    read_data(f, drcontext);
+    dr_close_file(f);
+    return 0;
+}
diff --git a/batch/common.bat b/batch/common.bat
new file mode 100644
index 000000000..f587008f4
--- /dev/null
+++ b/batch/common.bat
@@ -0,0 +1,72 @@
+REM **********************************************************
+REM Copyright (c) 2008 VMware, Inc.  All rights reserved.
+REM **********************************************************
+REM
+REM Redistribution and use in source and binary forms, with or without
+REM modification, are permitted provided that the following conditions are met:
+REM 
+REM * Redistributions of source code must retain the above copyright notice,
+REM   this list of conditions and the following disclaimer.
+REM 
+REM * Redistributions in binary form must reproduce the above copyright notice,
+REM   this list of conditions and the following disclaimer in the documentation
+REM   and/or other materials provided with the distribution.
+REM 
+REM * Neither the name of VMware, Inc. nor the names of its contributors may be
+REM   used to endorse or promote products derived from this software without
+REM   specific prior written permission.
+REM 
+REM THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+REM AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+REM IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+REM ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+REM FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+REM DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+REM SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+REM CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+REM LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+REM OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+REM DAMAGE.
+
+@echo off
+
+REM This script provides the functionality common to all
+REM the batch scripts in this module: it sets up the cygwin
+REM mount points and provides the proper path to the
+REM cygwin utilities in our local toolchain module.  All
+REM .bat files launching a cygwin command should call this
+REM script first and then invoke a bash shell with the
+REM path set to BASH_PATH.
+
+if {%TCROOT%}=={} goto error
+
+if not {%DYNAMORIO_TOOLCHAIN%}=={} goto set_reg
+
+REM If DYNAMORIO_TOOLCHAIN is not set, provide a default
+REM value relative to the location of this script.
+set CUR_DIR=%CD%
+cd %~dp0\..
+set DYNAMORIO_TOOLCHAIN=%CD%\toolchain
+cd %CUR_DIR%
+
+REM Cygwin utils won't work correctly without the mount points
+REM specified in the registry.  Unmount first and then force 
+REM /, /usr/bin, and /usr/lib to point to our toolchain module.
+:set_reg
+set CYGDIR=%DYNAMORIO_TOOLCHAIN%\win32\cygwin-1.5.25-7
+set CYGBIN=%CYGDIR%\bin
+set CYGX11=%CYGDIR%\usr\X11R6\bin
+
+%CYGBIN%\umount -A
+%CYGBIN%\mount -f %CYGDIR% /
+%CYGBIN%\mount -f %CYGDIR%\bin /usr/bin
+%CYGBIN%\mount -f %CYGDIR%\lib /usr/lib
+
+REM Provide a variable the calling .bat script can use to set
+REM the path when invoking its cygwin command.
+set BASH_PATH='%CYGBIN%':'%CYGX11%':$PATH
+goto :eof
+
+:error
+echo ERROR: you must set TCROOT
+exit /b 1
diff --git a/batch/makedocs.bat b/batch/makedocs.bat
new file mode 100644
index 000000000..63c561739
--- /dev/null
+++ b/batch/makedocs.bat
@@ -0,0 +1,65 @@
+REM **********************************************************
+REM Copyright (c) 2008 VMware, Inc.  All rights reserved.
+REM **********************************************************
+REM
+REM Redistribution and use in source and binary forms, with or without
+REM modification, are permitted provided that the following conditions are met:
+REM 
+REM * Redistributions of source code must retain the above copyright notice,
+REM   this list of conditions and the following disclaimer.
+REM 
+REM * Redistributions in binary form must reproduce the above copyright notice,
+REM   this list of conditions and the following disclaimer in the documentation
+REM   and/or other materials provided with the distribution.
+REM 
+REM * Neither the name of VMware, Inc. nor the names of its contributors may be
+REM   used to endorse or promote products derived from this software without
+REM   specific prior written permission.
+REM 
+REM THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+REM AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+REM IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+REM ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+REM FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+REM DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+REM SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+REM CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+REM LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+REM OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+REM DAMAGE.
+
+@echo off
+
+set CUR_DIR=%CD%
+
+set TARG=
+if {%1}=={vmap} set TARG=VMAP
+if {%1}=={viper} set TARG=VMSAFE
+if {%TARG%}=={} goto usage
+
+cd %~dp0\..
+set DR_HOME=%CD%
+
+call %DR_HOME%\batch\common.bat
+if %errorlevel% neq 0 goto error
+
+cd %DR_HOME%\core
+%CYGBIN%\bash -c "PATH=%BASH_PATH% make clear"
+%CYGBIN%\bash -c "PATH=%BASH_PATH% make %TARG%=1 DEBUG=0 INTERNAL=0 api_headers"
+
+cd %DR_HOME%\api\docs
+%CYGBIN%\bash -c "PATH=%BASH_PATH% make clean"
+%CYGBIN%\bash -c "PATH=%BASH_PATH% make %TARG%=1 pdf"
+
+REM clean up mounts
+%CYGBIN%\umount -A
+
+cd %CUR_DIR%
+goto :eof
+
+:usage
+echo "usage: %0 <viper|vmap>"
+
+:error
+cd %CUR_DIR%
+exit /b 1
diff --git a/batch/makezip.bat b/batch/makezip.bat
new file mode 100644
index 000000000..00c6cbcde
--- /dev/null
+++ b/batch/makezip.bat
@@ -0,0 +1,51 @@
+REM **********************************************************
+REM Copyright (c) 2008 VMware, Inc.  All rights reserved.
+REM **********************************************************
+REM
+REM Redistribution and use in source and binary forms, with or without
+REM modification, are permitted provided that the following conditions are met:
+REM 
+REM * Redistributions of source code must retain the above copyright notice,
+REM   this list of conditions and the following disclaimer.
+REM 
+REM * Redistributions in binary form must reproduce the above copyright notice,
+REM   this list of conditions and the following disclaimer in the documentation
+REM   and/or other materials provided with the distribution.
+REM 
+REM * Neither the name of VMware, Inc. nor the names of its contributors may be
+REM   used to endorse or promote products derived from this software without
+REM   specific prior written permission.
+REM 
+REM THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+REM AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+REM IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+REM ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+REM FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+REM DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+REM SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+REM CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+REM LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+REM OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+REM DAMAGE.
+
+@echo off
+
+set CUR_DIR=%CD%
+cd %~dp0\..
+set DR_HOME=%CD%
+
+call %DR_HOME%\batch\common.bat
+if %errorlevel% neq 0 goto error
+
+cd %DR_HOME%\tools
+%CYGBIN%\bash -c "PATH=%BASH_PATH% ./makezip.sh %1 '%2' %3 %4 %5 %6"
+
+REM clean up mounts
+%CYGBIN%\umount -A
+
+cd %CUR_DIR%
+goto :eof
+
+:error
+cd %CUR_DIR%
+exit /b 1
diff --git a/batch/rmtree.bat b/batch/rmtree.bat
new file mode 100644
index 000000000..d34abdc76
--- /dev/null
+++ b/batch/rmtree.bat
@@ -0,0 +1,33 @@
+REM **********************************************************
+REM Copyright (c) 2008 VMware, Inc.  All rights reserved.
+REM **********************************************************
+REM
+REM Redistribution and use in source and binary forms, with or without
+REM modification, are permitted provided that the following conditions are met:
+REM 
+REM * Redistributions of source code must retain the above copyright notice,
+REM   this list of conditions and the following disclaimer.
+REM 
+REM * Redistributions in binary form must reproduce the above copyright notice,
+REM   this list of conditions and the following disclaimer in the documentation
+REM   and/or other materials provided with the distribution.
+REM 
+REM * Neither the name of VMware, Inc. nor the names of its contributors may be
+REM   used to endorse or promote products derived from this software without
+REM   specific prior written permission.
+REM 
+REM THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+REM AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+REM IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+REM ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+REM FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+REM DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+REM SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+REM CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+REM LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+REM OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+REM DAMAGE.
+
+@echo off
+rd /s /q %1
+exit /b 1
diff --git a/clients/standalone/Makefile b/clients/standalone/Makefile
new file mode 100644
index 000000000..35c2de658
--- /dev/null
+++ b/clients/standalone/Makefile
@@ -0,0 +1,164 @@
+# **********************************************************
+# Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+.PHONY: all clean
+
+ifeq ($(shell uname -s | cut -c1-5), Linux)
+MACHINE = linux
+else
+MACHINE = win32
+endif
+
+ifndef DYNAMORIO_MAKE
+  DYNAMORIO_MAKE := ../../make
+endif
+include $(DYNAMORIO_MAKE)/compiler.mk
+
+# windows vs. linux flags, etc.
+ifeq ($(MACHINE), linux)
+U       =-u 
+I       =-I
+D       =-D
+L       =-L
+INCR    =-c
+OUT     =-o 
+EXEOUT  =-o 
+WARN    =-Wall
+OPT     =-O3
+NOPT    =-O0
+DBG     =-g3
+LD	= /usr/bin/ld
+OBJ_SFX =o
+EXE_SFX =
+LIB_NAME =libdrprivate.so
+else # win32
+# avoid problems w/ line endings on #! files that cygwin's sh has
+SHELL   =bash
+U       =/U
+I       =/I
+D       =/D
+L       =/link /LIBPATH:
+INCR    =/c
+OUT     =/Fo
+EXEOUT  =/Fe
+DLLOUT  =/dll /out:
+LIBOUT  =/implib:
+WARN    =/W3
+OPT     =/O2
+NOPT    =/Od
+DBG     =/Zi
+LD      = link /nologo
+OBJ_SFX =obj
+EXE_SFX =.exe
+LIB_NAME =dynamorio.dll
+endif
+
+# use release build if it exists, unless DEBUG=yes
+ifeq ($(MACHINE), linux)
+DYNAMORIO_DIR_BASE = $(DYNAMORIO_HOME)/exports/x86_linux_
+DYNAMORIO_BUILD_DIR_BASE = $(DYNAMORIO_HOME)/build/x86_linux_
+else
+DYNAMORIO_DIR_BASE = $(DYNAMORIO_HOME)/exports/x86_win32_
+DYNAMORIO_BUILD_DIR_BASE = $(DYNAMORIO_HOME)/build/x86_win32_
+endif
+
+# allow yes or 1 for DEBUG
+ifeq ($(DEBUG),yes)
+RELEASE_EXISTS=""
+else
+ifeq ($(DEBUG),1)
+RELEASE_EXISTS=""
+else
+RELEASE_EXISTS=$(wildcard $(DYNAMORIO_DIR_BASE)rel)
+endif
+endif
+
+ifeq ($(RELEASE_EXISTS), $(DYNAMORIO_DIR_BASE)rel)
+DYNAMORIO_DIR = $(DYNAMORIO_DIR_BASE)rel
+DYNAMORIO_BUILD_DIR = $(DYNAMORIO_BUILD_DIR_BASE)rel
+else
+DYNAMORIO_DIR = $(DYNAMORIO_DIR_BASE)dbg
+DYNAMORIO_BUILD_DIR = $(DYNAMORIO_BUILD_DIR_BASE)dbg
+endif
+
+ifeq ($(MACHINE), linux)
+DYNAMORIO_INC = $(I)$(DYNAMORIO_DIR) 
+else
+DYNAMORIO_INC = $(I)$(subst /,\\,$(shell cygpath -m ${DYNAMORIO_DIR})) $(MSINCLUDE)
+DYNAMORIO_LIBDIR = $(subst /,\\,$(shell cygpath -m ${DYNAMORIO_DIR})) $(MSLIB)
+endif
+
+CC_BASE = $(CC) $(DBG) $I.
+
+# some apps need LINUX defined even if not using API
+ifeq ($(MACHINE), win32)
+# don't need to define WIN32, assumed by ifndef LINUX
+CC_BASE += /MT
+else
+CC_BASE += $(D)LINUX
+endif
+
+CC_BASE_DYNAMORIO = $(CC_BASE) $(DYNAMORIO_INC) $(D)DYNAMO $(D)X86
+
+ifeq ($(MACHINE), linux)
+LINK = -lm
+LINK_DYNAMORIO = $(L)$(DYNAMORIO_DIR) -ldynamorio -lm -Xlinker -rpath -Xlinker $(DYNAMORIO_DIR)
+else # win32
+# /MTd must be included when compiling, not on link line!
+# else get weird link errors
+WINLIBS =  kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib \
+           advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib \
+           odbc32.lib odbccp32.lib imagehlp.lib libcmt.lib
+LINK = $(WINLIBS) /link /subsystem:console
+LINK_DYNAMORIO = $(L)$(DYNAMORIO_LIBDIR) $(WINLIBS) dynamorio.lib /subsystem:console
+endif
+
+ifeq ($(MACHINE), linux)
+# FIXME: re-enable signal tests when have time to fix core signal bugs
+SRCS := tracedump.c
+else
+SRCS := tracedump.c module_rsrc.c winsysnums.c vista_hash.c
+endif
+
+PGMS := $(patsubst %.c,%$(EXE_SFX),$(SRCS))
+
+all: $(PGMS)
+
+ifeq ($(MACHINE), win32)
+# using %$(EXE_SFX) doesn't seem to work well on win32, so we duplicate
+$(PGMS): %.exe: %.c
+	$(CC_BASE_DYNAMORIO) $(D)USE_DYNAMO $(EXEOUT)$@ $< $(LINK_DYNAMORIO)
+else # linux
+$(PGMS): %: %.c
+	$(CC_BASE_DYNAMORIO) $(D)USE_DYNAMO $(EXEOUT)$@ $< $(LINK_DYNAMORIO)
+endif # linux
+
+clean:
+	-rm -f $(PGMS) *.$(OBJ_SFX) *.pdb *.ilk *.opt
diff --git a/clients/standalone/module_rsrc.c b/clients/standalone/module_rsrc.c
new file mode 100644
index 000000000..f31179573
--- /dev/null
+++ b/clients/standalone/module_rsrc.c
@@ -0,0 +1,82 @@
+/* **********************************************************
+ * Copyright (c) 2006 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* FIXME - should use dr_fprintf but can't get that to work, then again this is
+ * standalone so doesn't really matter what we use. */
+
+#include "dynamorio.h"
+#include "dynamorio_mod.h"
+#include "windows.h"
+#include <assert.h>
+#include <stddef.h>
+#include <stdio.h>
+ 
+int
+main(int argc, char *argv[])
+{
+    version_info_t info;
+    HMODULE hmod;
+    
+    void *drcontext = dr_standalone_init();
+    if (argc != 2) {
+        printf("Usage: %s <dll to read>\n", argv[0]);
+        return 1;
+    }
+
+    /* Don't use LOAD_LIBRARY_AS_DATAFILE, it doesn't layout the sections as we expect
+     * use DONT_RESOLVE_DLL_REFERENCES instead */
+    hmod = LoadLibraryEx(argv[1], NULL, DONT_RESOLVE_DLL_REFERENCES);
+    if (hmod == NULL) {
+        printf("Can't find module %s\n", argv[1]);
+        return 2;
+    }
+    
+    /* Need to round off the loaded as data flag to get actual base */
+    if (!get_module_resource_version_info((app_pc)((uint)hmod & 0xfffffffe), &info)) {
+        printf("Failed to read rsrc directory\n");
+        return 3;
+    }
+
+    printf("File Version = %04d.%04d.%04d.%04d, Product Version = %04d.%04d.%04d.%04d\n",
+           info.file_version.version_parts.p1, info.file_version.version_parts.p2,
+           info.file_version.version_parts.p3, info.file_version.version_parts.p4,
+           info.product_version.version_parts.p1, info.product_version.version_parts.p2,
+           info.product_version.version_parts.p3, info.product_version.version_parts.p4);
+    if (info.original_filename != NULL)
+        printf("Original File Name = \"%S\"\n", info.original_filename);
+    if (info.company_name != NULL)
+        printf("Company Name = \"%S\"\n", info.company_name);
+    if (info.product_name != NULL)
+        printf("Product Name = \"%S\"\n", info.product_name);
+
+    return 0;
+}
diff --git a/clients/standalone/vista_hash.c b/clients/standalone/vista_hash.c
new file mode 100644
index 000000000..2c0a4cfca
--- /dev/null
+++ b/clients/standalone/vista_hash.c
@@ -0,0 +1,547 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+#include <stdlib.h> /* for realloc */
+#include <assert.h>
+#include <stddef.h> /* for offsetof */
+#include <string.h> /* for memcpy */
+#include <imagehlp.h>
+
+/*
+ * NOTE - I haven't tried running this with the dynamorio.dll in this directory so it
+ * might not work. I've been using a tot VMAP=1 build of core.
+ *
+ * ls *.dll > LIST
+ * for i in $( cat LIST ) ; do ../vista_hash.exe $i; done > ../OUTPUT
+ */
+
+void
+dummy()
+{}
+
+static bool v = false;
+static bool vv = false;
+
+#define VVERBOSE_PRINT if (vv) printf
+#define VERBOSE_PRINT if (v) printf
+
+#define ASSERT(x) ((x) ? 0 : dr_messagebox("Error %d %s\n", __LINE__, #x) , 0)
+
+#define ALIGN_BACKWARD(x, alignment) (((ptr_uint_t)x) & (~((alignment)-1)))
+#define ALIGN_FORWARD(x, alignment) \
+    ((((ptr_uint_t)x) + ((alignment)-1)) & (~((alignment)-1)))
+
+bool
+compare_pages(void *drcontext, byte *start1, byte *start2, uint start_offset)
+{
+    static byte copy_buf1_i[2*PAGE_SIZE+100];
+    static byte copy_buf2_i[2*PAGE_SIZE+100];
+    static byte buf1_i[2*PAGE_SIZE+100];
+    static byte buf2_i[2*PAGE_SIZE+100];
+    
+    byte *copy1 = (byte *)ALIGN_FORWARD(copy_buf1_i, PAGE_SIZE);
+    byte *copy2 = (byte *)ALIGN_FORWARD(copy_buf2_i, PAGE_SIZE);
+    byte *buf1 = (byte *)ALIGN_FORWARD(buf1_i, PAGE_SIZE);
+    byte *buf2 = (byte *)ALIGN_FORWARD(buf2_i, PAGE_SIZE);
+
+    byte *p1 = copy1+start_offset, *p2 = copy2+start_offset;
+    byte *b1 = buf1, *c1 = buf1, *b2 = buf2, *c2 = buf2;
+
+    int skipped_bytes1 = 0, skipped_identical_bytes1 = 0;
+    int skipped_bytes2 = 0, skipped_identical_bytes2 = 0;
+
+    /* we make a copy (zero extend the page) for decode to avoid walking to next,
+     * potentially invalid page */
+    memcpy(copy1, start1, PAGE_SIZE);
+    memcpy(copy2, start2, PAGE_SIZE);
+
+    /* Note - we do the compare ~1 instruction at a time, would be more effiecient
+     * to do the whole page and compare, but this makes it easier to track down where
+     * the differences originate from. */
+
+    while (p1 < copy1 + PAGE_SIZE) {
+        /* The idea is to do a lightweight decoding of the page and eliminate likely
+         * relocations from the instruction stream.  I expect that relocations within
+         * the stream to be 4-byte immeds or 4-byte displacements with pointer like
+         * values.  For now just using decode_sizeof to get the size of the instuction
+         * and based on that determine how much to chop off the end (instruction format
+         * is [...][disp][imm]) to remove the potential relocation.  With better 
+         * information from decode_sizeof (it knows whether an immed/disp is present or
+         * not and what offset it would be etc.) we could keep more of the bytes but
+         * this works for testing. What we'll miss unless we get really lucky is 
+         * relocs in read only data (const string arrays, for dr the const decode
+         * tables full of pointers for ex.). */
+
+        /* How the assumptions work out.  The assumption that we quickly get to the
+         * appropriate instruction frame seems valid.  Most cases we synchronize very
+         * quickly, usually just a couple of bytes and very rarely more then 20. 
+         * I Haven't seen any relocations in instructions that weren't caught
+         * below.  However, so far only matching ~60% of sibling pages because of
+         * read only data relocations interspersed in the text sections. Instruction
+         * frame alignment isn't an issue that often and the second pass is catching
+         * most of those. */
+
+        while (p1 - copy1 <= p2 - copy2) {
+            uint num_bytes, i, num_prefix = 0;
+            uint size = decode_sizeof(p1, &num_prefix);
+            size -= num_prefix;
+            num_bytes = num_prefix;
+            if (size == 0) {
+                size = 1; /* treat invalid as size == 1 */
+                num_bytes += 1;
+            } else if (size <= 4) {
+                num_bytes += size; /* no 4-byte disp or imm */
+            } else if (size <= 6 ) {
+                num_bytes += size - 4; /* could have 4-byte disp or immed */ 
+            } else if (size <= 7) {
+                num_bytes += size - 5; /* could have 4-byte disp and upto 1-byte immed
+                                        * or 4-byte immed */
+            } else if (size <= 9) {
+                num_bytes += size - 6; /* could have 4-byte disp and upto 2-byte immed
+                                        * or 4-byte immed */
+            } else {
+                num_bytes += size - 8; /* could have 4-byte disp and 4-byte immed */
+            }
+            for (i = 0; i < num_bytes; i++)
+                *b1++ = *p1++;
+            skipped_bytes1 += size - (num_bytes - num_prefix);
+            for (i = 0; i < size - (num_bytes - num_prefix); i++) {
+                if (*p1 == *(copy2+(p1-copy1)))
+                    skipped_identical_bytes1++;
+                p1++;
+            }
+        }
+
+        while (p2 - copy2 < p1 - copy1) {
+            uint num_bytes, i, num_prefix = 0;
+            uint size = decode_sizeof(p2, &num_prefix);
+            size -= num_prefix;
+            num_bytes = num_prefix;
+            if (size == 0) {
+                size = 1; /* treat invalid as size == 1 */
+                num_bytes += 1;
+            } else if (size <= 4) {
+                num_bytes += size;
+            } else if (size <=6 ) {
+                num_bytes += size - 4;
+            } else if (size <= 7) {
+                num_bytes += size - 5;
+            } else if (size <= 9) {
+                num_bytes += size - 6;
+            } else {
+                num_bytes += size - 8;
+            }
+            for (i = 0; i < num_bytes; i++)
+                *b2++ = *p2++;
+            skipped_bytes2 += size - (num_bytes - num_prefix);
+            for (i = 0; i < size - (num_bytes - num_prefix); i++) {
+                if (*p2 == *(copy1+(p2-copy2)))
+                    skipped_identical_bytes2++;
+                p2++;
+            }
+        }
+
+        /* Do check */
+        while (c1 < b1 && c2 < b2) {
+            if (*c1++ != *c2++) {
+                VVERBOSE_PRINT("Mismatch found near offset 0x%04x of page %08x\n",
+                               p1 - copy1, start1);
+                return false;
+            }
+        }
+    }
+    ASSERT(skipped_bytes1 == skipped_bytes2);
+    ASSERT(skipped_identical_bytes1 == skipped_identical_bytes2);
+    VVERBOSE_PRINT("Match found! skipped=%d skipped_identical=%d\n",
+                   skipped_bytes1, skipped_identical_bytes1);
+    return true;
+}
+
+
+/* Note that we should keep an eye for potential additional qualifier
+ * flags.  Alternatively we may simply mask off ~0xff to allow for any
+ * future flags added here.
+ */
+#define PAGE_PROTECTION_QUALIFIERS (PAGE_GUARD | PAGE_NOCACHE | PAGE_WRITECOMBINE)
+
+bool
+prot_is_writable(uint prot) 
+{
+    prot &= ~PAGE_PROTECTION_QUALIFIERS;
+    return (prot == PAGE_READWRITE || prot == PAGE_WRITECOPY ||
+            prot == PAGE_EXECUTE_READWRITE || prot == PAGE_EXECUTE_WRITECOPY);
+}
+
+# define OPT_HDR(nt_hdr_p, field) ((nt_hdr_p)->OptionalHeader.field)
+# define OPT_HDR_P(nt_hdr_p, field) (&((nt_hdr_p)->OptionalHeader.field))
+bool
+get_IAT_section_bounds(app_pc module_base, app_pc *iat_start, app_pc *iat_end)
+{
+    IMAGE_DOS_HEADER *dos = (IMAGE_DOS_HEADER *) module_base;
+    IMAGE_NT_HEADERS *nt = (IMAGE_NT_HEADERS *) (((ptr_uint_t)dos) + dos->e_lfanew);
+    IMAGE_DATA_DIRECTORY *dir;
+    if (dos->e_magic != IMAGE_DOS_SIGNATURE ||
+        nt == NULL || nt->Signature != IMAGE_NT_SIGNATURE)
+        return false;
+    dir = &OPT_HDR(nt, DataDirectory)[IMAGE_DIRECTORY_ENTRY_IAT];
+    *iat_start = module_base + dir->VirtualAddress;
+    *iat_end = module_base + dir->VirtualAddress + dir->Size;
+    return true;
+}
+ 
+int
+usage(char *name)
+{
+    printf("Usage: %s [-v] [-vv] [-no_second_pass] [-second_pass_offset <val>] [-no_assume_IAT_written] [-spin_for_debugger] <dll>\n", name);
+    return -1;
+}
+ 
+int
+main(int argc, char *argv[])
+{
+    byte *dll_1, *dll_2, *p1, *p2, *iat_start1, *iat_end1, *iat_start2, *iat_end2;
+    bool has_iat = false;
+    MEMORY_BASIC_INFORMATION info;
+    void *drcontext = dr_standalone_init();
+    uint writable_pages = 0, reserved_pages = 0, IAT_pages = 0;
+    uint matched_pages = 0, second_matched_pages = 0, unmatched_pages = 0;
+    uint exact_match_pages = 0, exact_no_match_pages = 0;
+    char reloc_file[MAX_PATH] = {0}, orig_file[MAX_PATH], *input_file;
+    uint old_size = 0, new_size = 0;
+    uint old_base = 0, new_base = 0x69000000; /* unlikely to collide */
+
+    /* user specified option defaults */
+    uint arg_offs = 1;
+    bool use_second_pass = true;
+    bool assume_header_match = true;
+    uint second_pass_offset = 16; /* FIXME arbitrary, what's a good choice? */
+    bool assume_IAT_written = true;
+    bool spin_for_debugger = false;
+
+    if (argc < 2)
+        return usage(argv[0]);
+    while (argv[arg_offs][0] == '-') {
+        if (strcmp(argv[arg_offs], "-vv") == 0) {
+            vv = true;
+        } else if (strcmp(argv[arg_offs], "-v") == 0) {
+            v = true;
+        } else if (strcmp(argv[arg_offs], "-no_second_pass") == 0) {
+            use_second_pass = false;
+        } else if (strcmp(argv[arg_offs], "-second_pass_offset") == 0) {
+            if (argc <= arg_offs+1)
+                return usage(argv[0]);
+            second_pass_offset = atoi(argv[++arg_offs]);
+        } else if (strcmp(argv[arg_offs], "-no_assume_IAT_written") == 0) {
+            assume_IAT_written = false;
+        } else if (strcmp(argv[arg_offs], "-spin_for_debugger") == 0) {
+            spin_for_debugger = true;
+        } else {
+            return usage(argv[0]);
+        }
+        arg_offs++;
+    }   
+    input_file = argv[arg_offs++];
+    if (arg_offs != argc)
+        return usage(argv[0]);
+    
+    snprintf(reloc_file, sizeof(reloc_file), "%s.reloc.dll", input_file);
+    reloc_file[sizeof(reloc_file)-1] = '\0';
+    if (!CopyFile(input_file, reloc_file, FALSE)) {
+        LPSTR msg = NULL;
+        uint error = GetLastError();
+        FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_ALLOCATE_BUFFER,
+                      0, GetLastError(), 0, msg, 0, NULL);
+        VERBOSE_PRINT("Copy Error %d (0x%x) = %s\n", error, error, msg);
+        return 1;
+    }
+    snprintf(orig_file, sizeof(orig_file), "%s.orig.dll", input_file);
+    orig_file[sizeof(orig_file)-1] = '\0';
+    if (!CopyFile(input_file, orig_file, FALSE)) {
+        LPSTR msg = NULL;
+        uint error = GetLastError();
+        FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_ALLOCATE_BUFFER,
+                      0, GetLastError(), 0, msg, 0, NULL);
+        VERBOSE_PRINT("Copy Error %d (0x%x) = %s\n", error, error, msg);
+        return 1;
+    }
+    if (ReBaseImage(reloc_file, "", TRUE, FALSE, FALSE, 0, &old_size, &old_base,
+                    &new_size, &new_base, 0)) {
+        VERBOSE_PRINT("Rebased imsage \"%s\" from 0x%08x to 0x%08x\n"
+                      "Size changed from %d bytes to %d bytes\n",
+                      input_file, old_base, new_base, old_size, new_size);
+    } else {
+        LPSTR msg = NULL;
+        uint error = GetLastError();
+        FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_ALLOCATE_BUFFER,
+                      0, GetLastError(), 0, msg, 0, NULL);
+        VERBOSE_PRINT("Rebase Error %d (0x%x) = %s\n", error, error, msg);
+        return 1;
+    }
+    
+    dll_1 = (byte *)ALIGN_BACKWARD(LoadLibraryExA(orig_file, NULL,
+                                                  DONT_RESOLVE_DLL_REFERENCES),
+                                   PAGE_SIZE);
+    p1 = dll_1;
+    dll_2 = (byte *)ALIGN_BACKWARD(LoadLibraryExA(reloc_file, NULL,
+                                                  DONT_RESOLVE_DLL_REFERENCES),
+                                   PAGE_SIZE);
+    p2 = dll_2;
+    VVERBOSE_PRINT("Loaded dll @ 0x%08x and 0x%08x\n", dll_1, dll_2);
+
+    if (dll_1 == NULL || dll_2 == NULL) {
+        VERBOSE_PRINT( "Error loading %s\n", input_file);
+        return 1;
+    }
+    
+    /* Handle the first page specially since I'm seeing problems with a handful of
+     * dlls that aren't really getting rebased. mcupdate_GenuineIntel.dll for ex.
+     * (which does have relocations etc.) not sure what's up, but it's only a couple of
+     * dlls so will ignore them. If we really rebased the header should differ. */
+    if (memcmp(dll_1, dll_2, PAGE_SIZE) == 0) {
+        printf("%s - ERROR during relocating\n", input_file);
+        return 1;
+    } else {
+        exact_no_match_pages++;
+        if (assume_header_match)
+            /* We could modify the hash function to catch header pages. */
+            matched_pages++;
+        else 
+            unmatched_pages++;
+    }
+    p1 += PAGE_SIZE;
+    p2 += PAGE_SIZE;
+
+    if (assume_IAT_written && get_IAT_section_bounds(dll_1, &iat_start1, &iat_end1)) {
+        has_iat = true;
+        ASSERT(get_IAT_section_bounds(dll_2, &iat_start2, &iat_end2) &&
+               iat_start1 - dll_1 == iat_start2 - dll_2 &&
+               iat_end1 - dll_1 == iat_end2 - dll_2);
+    }
+
+    while (dr_virtual_query(p1, &info, sizeof(info)) == sizeof(info) &&
+           info.State != MEM_FREE && info.AllocationBase == dll_1) {
+        /* we only check read-only pages (assumption writable pages aren't shareable) */
+        ASSERT(p1 == info.BaseAddress);
+        if (info.State != MEM_COMMIT) {
+            reserved_pages += info.RegionSize / PAGE_SIZE;
+            VVERBOSE_PRINT("skipping %d reserved pages\n", info.RegionSize / PAGE_SIZE);
+            p1 += info.RegionSize;
+            p2 += info.RegionSize;
+        } else if (!prot_is_writable(info.Protect)) {
+            int i;
+            for (i = 0; i < info.RegionSize / PAGE_SIZE; i++) {
+                bool exact = false;
+                if (assume_IAT_written && has_iat &&
+                    iat_end1 > p1 && iat_start1 < p1 + PAGE_SIZE) {
+                    /* overlaps an IAT page */
+                    IAT_pages++;
+                    p1 += PAGE_SIZE;
+                    p2 += PAGE_SIZE;
+                    continue;
+                }
+                if (memcmp(p1, p2, PAGE_SIZE) == 0) {
+                    VVERBOSE_PRINT("Page Exact Match\n");
+                    exact_match_pages++;
+                    exact = true;
+                } else {
+                    VVERBOSE_PRINT("Page Exact Mismatch\n");
+                    exact_no_match_pages++;
+                }
+                if (compare_pages(drcontext, p1, p2, 0)) {
+                    VVERBOSE_PRINT("Matched page\n");
+                    matched_pages++;
+                } else { 
+                    VVERBOSE_PRINT("Failed to match page\n");
+                    if (use_second_pass &&
+                        compare_pages(drcontext, p1, p2, second_pass_offset)) {
+                        second_matched_pages++;
+                    } else {
+                        unmatched_pages++;
+                    }
+                    ASSERT(!exact);
+                }
+                p1 += PAGE_SIZE;
+                p2 += PAGE_SIZE;
+            }
+        } else {
+            writable_pages += info.RegionSize / PAGE_SIZE;
+            VVERBOSE_PRINT("skipping %d writable pages\n", info.RegionSize / PAGE_SIZE);
+            p1 += info.RegionSize;
+            p2 += info.RegionSize;
+        }
+    }
+
+    VERBOSE_PRINT("%d exact match, %d not exact match\n%d hash_match, %d second_hash_match, %d hash_mismatch\n",
+                  exact_match_pages, exact_no_match_pages, matched_pages, second_matched_pages, unmatched_pages); 
+
+    printf("%s : %d pages - %d w %d res %d IAT = %d same %d differ : %d hash differ %d first hash differ : %d%% found, %d%% found first hash\n",
+           input_file, writable_pages + reserved_pages + IAT_pages + exact_match_pages + exact_no_match_pages,
+           writable_pages, reserved_pages, IAT_pages,
+           exact_match_pages, exact_no_match_pages,
+           unmatched_pages, unmatched_pages + second_matched_pages,
+           (100 * (matched_pages + second_matched_pages - exact_match_pages))/exact_no_match_pages,
+           (100 * (matched_pages - exact_match_pages))/exact_no_match_pages);
+
+    while (spin_for_debugger)
+        Sleep(1000);
+
+    return 0;
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+#if 0
+/* First tried something like this, but we hit too many issues in decode and encode */
+bool
+compare_pages(void *drcontext, byte *start1, byte *start2)
+{
+    byte *p1 = start1, *p2 = start2;
+    int skipped_bytes = 0, identical_skipped_bytes = 0;
+
+    while (p1 < start1 + PAGE_SIZE) {
+        int instr_size = decode_sizeof(p1, NULL _IF_X64(NULL));
+        if (p1 + instr_size > start1 + PAGE_SIZE) {
+            /* We're overlapping the end of the page, skip these. */
+            int end_skip = start1 + PAGE_SIZE - p1;
+            VVERBOSE_PRINT("Passing PAGE_END %d bytes", end_skip);
+            skipped_bytes += end_skip;
+            if (memcmp(p1, p2, end_skip) == 0)
+                identical_skipped_bytes += end_skip;
+            break;
+        }
+        if (decode_sizeof(p2, NULL _IF_X64(NULL)) != instr_size) {
+            VVERBOSE_PRINT("Instruction alignment mismatch\n");
+            return false;
+        }
+        /* assumption - instructions <= 4 bytes in size won't have relocations */
+        if (instr_size < 5) {
+            if (memcmp(p1, p2, instr_size) != 0) {
+                VVERBOSE_PRINT("Difference found in small instr\n");
+                return false;
+            }
+            p1 += size;
+            p2 += size;
+        } else {
+            /* guess if there could be a relocation */
+            instr_t *instr1 = instr_create(drcontext);
+            instr_t *instr2 = instr_create(drcontext);
+            p1 = decode(drcontext, p1, instr1);
+            p2 = decode(drcontext, p2, instr2);
+            if (p1 - start1 != p2 - start2) {
+                VVERBOSE_PRINT("Instruction alignment mismatch on full decode\n");
+                /* Fixme - free instr, don't expect this to happen */
+                return false;
+            }
+            if (instr_get_num_srcs(instr1) != instr_get_num_srcs(instr2) ||
+                instr_get_num_dsts(instr1) != instr_get_num_dsts(instr2)) {
+                VVERBOSE_PRINT("Full decode operand mismatch");
+                return false;
+            }
+            for (i = instr_get_num_srcs(instr1); i > 0; i--) {
+                opnd_t opnd = instr_get_src(instr1, i);
+                if (opnd_is_immed_int(opnd) && opnd_get_immed_int(opnd) > 0x10000) {
+                    instr_set_src(instr1, i, opnd_create_immed_int(opnd_get_immed_int(opnd), opnd_get_size(opnd)));
+                }
+            }
+        }
+    }
+}
+
+void
+modify_instr_for_relocations(void *drcontext, instr_t *inst,
+                             ptr_uint_t *immed, ptr_uint_t *disp)
+{
+    int i;
+    ptr_uint_t limmed = 0, ldisp = 0;
+    for (i = instr_num_srcs(inst) - 1; i >= 0; i--) {
+        opnd_t opnd = instr_get_src(inst, i);
+        if (opnd_is_immed_int(opnd) && opnd_get_immed_int(opnd) > 0x10000) {
+            if (limmed != 0) {
+                ASSERT(false);
+            } else {
+                limmed = opnd_get_immed_int(opnd);
+            }
+            instr_set_src(inst, i, opnd_create_immed_int(0, opnd_get_size(opnd)));
+        }
+        if (opnd_is_base_disp(opnd) && opnd_get_disp(opnd) > 0x10000) {
+            if (ldisp != 0 && ldisp != opnd_get_disp(opnd)) {
+                ASSERT(false);
+            } else {
+                ldisp = opnd_get_disp(opnd);
+            }
+            instr_set_src(inst, i, opnd_create_base_disp(opnd_get_base(opnd), opnd_get_index(opnd), opnd_get_scale(opnd), 0, opnd_get_size(opnd)));
+        }
+    }
+    for (i = instr_num_dsts(inst) - 1; i >= 0; i--) {
+        opnd_t opnd = instr_get_dst(inst, i);
+        ASSERT(!opnd_is_immed(opnd));
+        if (opnd_is_base_disp(opnd) && opnd_get_disp(opnd) > 0x10000) {
+            if (ldisp != 0 && ldisp != opnd_get_disp(opnd)) {
+                ASSERT(false);
+            } else {
+                ldisp = opnd_get_disp(opnd);
+            }
+            instr_set_dst(inst, i, opnd_create_base_disp(opnd_get_base(opnd), opnd_get_index(opnd), opnd_get_scale(opnd), 0, opnd_get_size(opnd)));
+        }
+    }
+    if (limmed != 0)
+        *immed = limmed;
+    if (ldisp != 0)
+        *disp = ldisp; 
+}
+
+#endif
diff --git a/clients/standalone/winsysnums.c b/clients/standalone/winsysnums.c
new file mode 100644
index 000000000..697e17ccb
--- /dev/null
+++ b/clients/standalone/winsysnums.c
@@ -0,0 +1,463 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+
+/* winsysnums.c: analyzes a dll's exported routines, looking for system call
+ * numbers or Ki* routines -- typically pointed at a new ntdll.dll
+ */
+
+/* Uses the DR CLIENT_INTERFACE API, using DR as a standalone library, rather than
+ * being a client library working with DR on a target program.
+ *
+ * To build:
+ * 1) Build DR w/ CLIENT_INTERFACE=1
+ * 2) cl winsysnums.c /I$DYNAMORIO_HOME/exports/x86_win32_dbg /link /libpath:$DYNAMORIO_HOME/exports/x86_win32_dbg dynamorio.lib imagehlp.lib
+ *
+ * To run, you need to put dynamorio.dll into either the current directory
+ * or system32.
+ */
+
+#include "dr_api.h"
+#include <assert.h>
+#include <imagehlp.h>
+#include <stdio.h>
+
+#ifdef LINUX
+# define EXPORT
+#else
+# define EXPORT __declspec(dllexport)
+#endif
+
+/* global params */
+static bool expect_int2e = false;
+static bool expect_sysenter = false;
+static bool expect_wow = false;
+static bool expect_x64 = false;
+static bool verbose = false;
+static bool list_exports = false;
+static bool list_forwards = false;
+static bool list_Ki = false;
+static bool list_syscalls = false;
+static bool ignore_Zw = false;
+
+static void
+common_print(char *fmt, va_list ap)
+{
+    vfprintf(stdout, fmt, ap);
+    fflush(stdout);
+}
+
+static void
+verbose_print(char *fmt, ...)
+{
+    if (verbose) {
+        va_list ap;
+        va_start(ap, fmt);
+        common_print(fmt, ap);
+        va_end(ap);
+    }
+}
+
+static void
+print(char *fmt, ...)
+{
+    va_list ap;
+    va_start(ap, fmt);
+    common_print(fmt, ap);
+    va_end(ap);
+}
+
+#define MAX_INSTRS_BEFORE_SYSCALL 16
+#define MAX_INSTRS_IN_FUNCTION 256
+
+typedef struct {
+    int sysnum;
+    int num_args;
+    int fixup_index; /* WOW dlls only */
+} syscall_info_t;
+
+/* returns false on failure */
+static bool
+decode_function(void *dcontext, byte *entry)
+{
+    byte *pc;
+    int num_instr = 0;
+    bool found_ret = false;
+    instr_t *instr;
+    if (entry == NULL)
+        return false;
+    instr = instr_create(dcontext);
+    pc = entry;
+    while (true) {
+        instr_reset(dcontext, instr);
+        pc = decode(dcontext, pc, instr);
+        dr_print_instr(dcontext, STDOUT, instr, "");
+        if (instr_is_return(instr)) {
+            found_ret = true;
+            break;
+        }
+        num_instr++;
+        if (num_instr > MAX_INSTRS_IN_FUNCTION) {
+            print("ERROR: hit max instr limit %d\n", MAX_INSTRS_IN_FUNCTION);
+            break;
+        }
+    }
+    instr_destroy(dcontext, instr);
+    return found_ret;
+}
+
+static void
+check_Ki(const char *name)
+{
+    /* FIXME: eventually we should automatically analyze these, but
+     * not worth the time at this point.  Once we have automatic
+     * analysis code, should put it into DR debug build init too!  For
+     * now we issue manual instructions about verifying our
+     * assumptions, and look for unknown Ki routines.
+     */
+    if (strcmp(name, "KiUserApcDispatcher") == 0) {
+        print("verify that:\n"
+              "\t1) *esp == call* target (not relied on)\n"
+              "\t2) *(esp+16) == CONTEXT\n");
+    } else if (strcmp(name, "KiUserExceptionDispatcher") == 0) {
+        print("verify that:\n"
+              "\t1) *esp = EXCEPTION_RECORD*\n"
+              "\t2) *(esp+4) == CONTEXT*\n");
+    } else if (strcmp(name, "KiRaiseUserExceptionDispatcher") == 0) {
+        print("we've never seen this guy invoked\n");
+    } else if (strcmp(name, "KiUserCallbackDispatcher") == 0) {
+        print("verify that:\n"
+              "\t1) peb->KernelCallbackTable[*(esp+4)] == call* target (not relied on)\n");
+    } else if (strcmp(name, "KiFastSystemCall") == 0) {
+        print("should be simply \"mov esp,edx; sysenter; ret\"\n");
+    } else if (strcmp(name, "KiFastSystemCallRet") == 0) {
+        print("should be simply \"ret\"\n");
+    } else if (strcmp(name, "KiIntSystemCall") == 0) {
+        print("should be simply \"lea 0x8(esp),edx; int 2e; ret\"\n");
+    } else {
+        print("WARNING!  UNKNOWN Ki ROUTINE!\n");
+    }
+}
+
+/* returns false on failure */
+static bool
+decode_syscall_num(void *dcontext, byte *entry, syscall_info_t *info)
+{
+    /* FIXME: would like to fail gracefully rather than have a DR assertion
+     * on non-code! => use DEBUG=0 INTERNAL=1 DR build!
+     */
+    bool found_syscall = false, found_eax = false, found_edx = false, found_ecx = false;
+    bool found_ret = false;
+    byte *pc;
+    int num_instr = 0;
+    instr_t *instr;
+    if (entry == NULL)
+        return false;
+    info->num_args = -1; /* if find sysnum but not args */
+    info->sysnum = -1;
+    instr = instr_create(dcontext);
+    pc = entry;
+    /* FIXME - we don't support decoding 64bit instructions in 32bit mode, but I want
+     * this to work on 32bit machines.  Hack fix based on the wrapper pattern, we skip
+     * the first instruction (mov r10, rcx) here, the rest should decode ok.
+     * Xref PR 236203. */
+    if (expect_x64 && *pc == 0x4c && *pc+1 == 0x8b && *pc+2 == 0xd1)
+        pc += 3;
+    while (true) {
+        instr_reset(dcontext, instr);
+        pc = decode(dcontext, pc, instr);
+        if (verbose)
+            dr_print_instr(dcontext, STDOUT, instr, "");
+        if (pc == NULL || !instr_valid(instr))
+            break;
+        /* ASSUMPTION: a mov imm of 0x7ffe0300 into edx followed by an
+         * indirect call via edx is a system call on XP and later
+         * On XP SP1 it's call *edx, while on XP SP2 it's call *(edx)
+         * For wow it's a call through fs.
+         * FIXME - core exports various is_*_syscall routines (such as
+         * instr_is_wow64_syscall()) which we could use here instead of
+         * duplicating if they were more flexible about when they could
+         * be called (instr_is_wow64_syscall() for ex. asserts if not
+         * in a wow process).
+         */
+        if (/* int 2e or x64 */
+            (instr_is_syscall(instr) && found_eax && (expect_int2e || expect_x64)) ||
+            /* sysenter case */
+            (expect_sysenter && found_edx && found_eax &&
+             instr_is_call_indirect(instr) &&
+             /* XP SP{0,1}, 2003 SP0: call *edx */
+             ((opnd_is_reg(instr_get_target(instr)) &&
+               opnd_get_reg(instr_get_target(instr)) == REG_EDX) ||
+              /* XP SP2, 2003 SP1: call *(edx) */
+              (opnd_is_base_disp(instr_get_target(instr)) &&
+               opnd_get_base(instr_get_target(instr)) == REG_EDX &&
+               opnd_get_index(instr_get_target(instr)) == REG_NULL &&
+               opnd_get_disp(instr_get_target(instr)) == 0))) ||
+            /* wow case */
+            (expect_wow && found_ecx && found_eax &&
+             instr_is_call_indirect(instr) &&
+             opnd_is_far_base_disp(instr_get_target(instr)) &&
+             opnd_get_base(instr_get_target(instr)) == REG_NULL &&
+             opnd_get_index(instr_get_target(instr)) == REG_NULL &&
+             opnd_get_segment(instr_get_target(instr)) == SEG_FS)) {
+            found_syscall = true;
+        } else if (instr_is_return(instr)) {
+            if (opnd_is_immed_int(instr_get_src(instr, 0)))
+                info->num_args = opnd_get_immed_int(instr_get_src(instr, 0));
+            else
+                info->num_args = 0;
+            found_ret = true;
+            break;
+        } else if (instr_is_cti(instr)) {
+            /* assume not a syscall wrapper if we hit a cti */
+            break; /* give up gracefully */
+        } else if ((!found_eax || !found_edx || !found_ecx) &&
+            instr_get_opcode(instr) == OP_mov_imm &&
+            opnd_is_reg(instr_get_dst(instr, 0))) {
+            if (!found_eax && opnd_get_reg(instr_get_dst(instr, 0)) == REG_EAX) {
+                info->sysnum = opnd_get_immed_int(instr_get_src(instr, 0));
+                found_eax = true;
+            } else if (!found_edx && opnd_get_reg(instr_get_dst(instr, 0)) == REG_EDX) {
+                int imm = opnd_get_immed_int(instr_get_src(instr, 0));
+                if (imm == 0x7ffe0300)
+                    found_edx = true;
+            } else if (!found_ecx && opnd_get_reg(instr_get_dst(instr, 0)) == REG_ECX) {
+                found_ecx = true;
+                info->fixup_index = opnd_get_immed_int(instr_get_src(instr, 0));
+            }
+        } else if (instr_get_opcode(instr) == OP_xor &&
+                   opnd_is_reg(instr_get_src(instr, 0)) &&
+                   opnd_get_reg(instr_get_src(instr, 0)) == REG_ECX &&
+                   opnd_is_reg(instr_get_dst(instr, 0)) &&
+                   opnd_get_reg(instr_get_dst(instr, 0)) == REG_ECX) {
+            /* xor to 0 */
+            found_ecx = true;
+            info->fixup_index = 0;
+        }
+        num_instr++;
+        if (num_instr > MAX_INSTRS_BEFORE_SYSCALL) /* wrappers should be short! */
+            break; /* avoid weird cases like NPXEMULATORTABLE */
+    }
+    instr_destroy(dcontext, instr);
+    return found_syscall;
+}
+
+static void
+process_exports(void *dcontext, char *dllname)
+{
+    LOADED_IMAGE img;
+    IMAGE_EXPORT_DIRECTORY *dir;
+    IMAGE_SECTION_HEADER *sec;
+    DWORD *name, *code;
+    WORD *ordinal;
+    const char *string;
+    uint size;
+    BOOL res;
+    int i;
+    byte *addr, *start_exports, *end_exports;
+    syscall_info_t sysinfo;
+
+    res = MapAndLoad(dllname, NULL, &img, FALSE, TRUE);
+    if (!res) {
+        print("Error loading %s\n", dllname);
+        return;
+    }
+    dir = (IMAGE_EXPORT_DIRECTORY *)
+        ImageDirectoryEntryToData(img.MappedAddress, FALSE,
+                                  IMAGE_DIRECTORY_ENTRY_EXPORT, &size);
+    verbose_print("mapped at 0x%08x, exports is at 0x%08x, size is 0x%x\n",
+                  img.MappedAddress, dir, size);
+    start_exports = (byte *) dir;
+    end_exports = start_exports + size;
+    verbose_print("name=%s, ord base=0x%08x, names=%d 0x%08x\n",
+                  (char *) ImageRvaToVa(img.FileHeader, img.MappedAddress,
+                                        dir->Name, NULL),
+                  dir->Base, dir->NumberOfNames, dir->AddressOfNames);
+
+    /* don't limit functions to lie in .text -- 
+     * for ntdll, some exported routines have their code after .text, inside
+     * ECODE section!
+     */
+    sec = img.Sections;
+    for (i = 0; i < img.NumberOfSections; i++) {
+        verbose_print("Section %d %s: 0x%x + 0x%x == 0x%08x through 0x%08x\n",
+                      i, sec->Name, sec->VirtualAddress, sec->SizeOfRawData,
+                      ImageRvaToVa(img.FileHeader, img.MappedAddress,
+                                   sec->VirtualAddress, NULL),
+                      (uint) ImageRvaToVa(img.FileHeader, img.MappedAddress,
+                                          sec->VirtualAddress, NULL) +
+                      sec->SizeOfRawData);
+        sec++;
+    }
+
+    name = (DWORD *) ImageRvaToVa(img.FileHeader, img.MappedAddress,
+                                  dir->AddressOfNames, NULL);
+    code = (DWORD *) ImageRvaToVa(img.FileHeader, img.MappedAddress,
+                                  dir->AddressOfFunctions, NULL);
+    ordinal = (WORD *) ImageRvaToVa(img.FileHeader, img.MappedAddress,
+                                    dir->AddressOfNameOrdinals, NULL);
+    verbose_print("names: from 0x%08x to 0x%08x\n",
+                  ImageRvaToVa(img.FileHeader, img.MappedAddress, name[0], NULL),
+                  ImageRvaToVa(img.FileHeader, img.MappedAddress,
+                               name[dir->NumberOfNames-1], NULL));
+
+    for (i = 0; i < dir->NumberOfNames; i++) {
+        string = (char *) ImageRvaToVa(img.FileHeader, img.MappedAddress, name[i], NULL);
+        /* ordinal is biased (dir->Base), but don't add base when using as index */
+        assert(dir->NumberOfFunctions > ordinal[i]);
+        /* I don't understand why have to do RVA to VA here, when dumpbin /exports
+         * seems to give the same offsets but by simply adding them to base we
+         * get the appropriate code location -- but that doesn't work here...
+         */
+        addr = ImageRvaToVa(img.FileHeader, img.MappedAddress,
+                            code[ordinal[i]], NULL);
+        verbose_print("name=%s 0x%08x, ord=%d, code=0x%x -> 0x%08x\n",
+                      string, string, ordinal[i], code[ordinal[i]], addr);
+        if (list_exports) {
+            print("ord %3d offs 0x%08x %s\n", ordinal[i],
+                  addr - img.MappedAddress, string);
+        }
+        if (list_Ki && string[0] == 'K' && string[1] == 'i') {
+            print("\n==================================================\n");
+            print("%s\n\n", string);
+            check_Ki(string);
+            print("\ndisassembly:\n");
+            decode_function(dcontext, addr);
+            print(  "==================================================\n");
+        }
+        /* forwarded export points inside exports section */
+        if (addr >= start_exports && addr < end_exports) {
+            if (list_forwards || verbose) {
+                /* I've had issues w/ forwards before, so avoid printing crap */
+                if (addr[0] > 0 && addr[0] < 127)
+                    print("%s is forwarded to %.128s\n", string, addr);
+                else
+                    print("ERROR identifying forwarded entry for %s\n", string);
+            }
+        } else if (list_syscalls) {
+            if (!ignore_Zw || string[0] != 'Z' || string[1] != 'w') {
+                if (decode_syscall_num(dcontext, addr, &sysinfo)) {
+                    if (sysinfo.sysnum == -1) {
+                        /* we expect this sometimes */
+                        if (strcmp(string, "KiFastSystemCall") != 0 &&
+                            strcmp(string, "KiIntSystemCall") != 0) {
+                            print("ERROR: unknown syscall #: %s\n", string);
+                        }
+                    } else {
+                        if (expect_wow) {
+                            print("syscall # 0x%04x %2d args fixup 0x%02x = %s\n",
+                                  sysinfo.sysnum, sysinfo.num_args,
+                                  sysinfo.fixup_index, string);
+                        } else if (expect_x64) {
+                            print("syscall # 0x%04x = %s\n",
+                                  sysinfo.sysnum, string);
+                        } else {
+                            print("syscall # 0x%04x %2d args = %s\n",
+                                  sysinfo.sysnum, sysinfo.num_args, string);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+static void
+usage(char *pgm)
+{
+    print("Usage: %s [-syscalls <-sysenter | -int2e | -wow | -x64> [-ignore_Zw]] | "
+          "-Ki | -exports | -forwards | -v] <dll>\n", pgm);
+    exit(-1);
+}
+
+int
+main(int argc, char *argv[])
+{
+    void *dcontext = dr_standalone_init();
+    OSVERSIONINFO version;
+    int res;
+    char *dll;
+    bool forced = false;
+
+    for (res=1; res < argc; res++) {
+        if (strcmp(argv[res], "-sysenter") == 0) {
+            expect_sysenter = true;
+            forced = true;
+        } else if (strcmp(argv[res], "-int2e") == 0) {
+            expect_int2e = false;
+            forced = true;
+        } else if (strcmp(argv[res], "-wow") == 0) {
+            expect_wow = true;
+            forced = true;
+        } else if (strcmp(argv[res], "-x64") == 0) {
+            expect_x64 = true;
+            /* FIXME - ideally we should switch to 64bit decoding mode here
+             * or compile for 64bit and switch the other options to 32bit
+             * xref case PR 236203.  We hack a fix for -syscalls (see
+             * decode_syscall_num()) but -Ki won't work. */
+            forced = true;
+        } else if (strcmp(argv[res], "-v") == 0) {
+            verbose = true;
+        } else if (strcmp(argv[res], "-exports") == 0) {
+            list_exports = true;
+            list_forwards = true; /* implied */
+        } else if (strcmp(argv[res], "-forwards") == 0) {
+            list_forwards = true;
+        } else if (strcmp(argv[res], "-Ki") == 0) {
+            list_Ki = true;
+        } else if (strcmp(argv[res], "-syscalls") == 0) {
+            list_syscalls = true;
+        } else if (strcmp(argv[res], "-ignore_Zw") == 0) {
+            ignore_Zw = true;
+        } else if (argv[res][0] == '-') {
+            usage(argv[0]);
+            assert(false); /* not reached */
+        } else {
+            break;
+        }
+    }
+    if (res >= argc ||
+        (!list_syscalls && !list_Ki && !list_forwards && !verbose)) {
+        usage(argv[0]);
+        assert(false); /* not reached */
+    }
+    dll = argv[res];
+
+    if (!forced && list_syscalls) {
+        usage(argv[0]);
+        assert(false); /* not reached */
+    }
+
+    process_exports(dcontext, dll);
+    return 0;
+}
diff --git a/core/Makefile b/core/Makefile
new file mode 100644
index 000000000..478f28dd8
--- /dev/null
+++ b/core/Makefile
@@ -0,0 +1,2161 @@
+# **********************************************************
+# Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# Copyright (c) 2003-2007 Determina Corp.
+# Copyright (c) 2001-2003 Massachusetts Institute of Technology
+# Copyright (c) 2001 Hewlett Packard Company
+
+###########################################################################
+###########################################################################
+###
+### Makefile for building DynamoRIO
+###
+### This Makefile's primary target is x86 and x64, either WinNT or Linux.
+### It is intended to be usable either on Linux or on WinNT
+### via cygwin (gnu make is required).  
+###
+### A separate build directory tree is used.
+### This is the only Makefile -- no Makefiles in subdirectories.
+### See the section "Configuration Values" on customizing the build defines.
+###
+### To stick a custom message into the .rsrc properties for windows
+### define SPECIAL_BUILD (i.e. make SPECIAL_BUILD="experimental fix for case xxx")
+
+ETAGS	= /usr/bin/etags
+MAKE	= make
+
+#allow adding to p4 command line, for ex. P4PATH_OVERRIDE='-d $(shell $(CYGPATH) -m $$PWD)'
+#for using the windows native p4 client under cygwin without having to set P4CLIENT
+P4 = p4 $(P4PATH_OVERRIDE)
+
+ifndef ARCH
+# default is x86 instead of x64
+ARCH = x86
+endif
+
+ifndef DYNAMORIO_MAKE
+  DYNAMORIO_MAKE := ../make
+endif
+include $(DYNAMORIO_MAKE)/compiler.mk
+
+# windows vs. linux flags, etc.
+ifeq ($(MACHINE), linux)
+CFLAGS  =-std=gnu99 # -std=c99 doesn't quite work
+# FIXME case 191480: we used to pass -pedantic just to cpp;
+# now w/ no separate cpp step we should eliminate the
+# warnings and pass -pedantic to main gcc
+WARN    =-Wall -Werror
+OPT     =-O3
+DBG     =-g3
+# FIXME: perhaps we should optimize but with -fno-omit-frame-pointer?
+# disable intrinsic to avoid stack overflow in set_dynamo_options(): case 7853
+DBG_OPT =-fno-builtin-strcmp
+OBJ_SFX =o
+EXE_SFX =   # note . would be included here
+ifeq ($(STATIC), 1)
+SO_SFX  =.a
+else # non-STATIC
+SO_SFX  =.so
+endif
+else # win32
+WARN    =/W4 /WX
+OPT     =/O2
+DBG     =/Zi
+LINK_DBG=/debug
+# do not use /O2 on windows, it messes up debug info!
+DBG_OPT =
+# FIXME: further debug checking for CL 13: /RTCc 
+OBJ_SFX =obj
+EXE_SFX =.exe
+SO_SFX  =.dll
+endif
+
+# Our dependence support uses gcc on both platforms
+GCC     =gcc
+DEP_SFX =dep
+DEPENDS =-MM -MG -MP # no system headers, generated get phony targets
+
+# Basic directories
+DYNAMO	     =..
+# src/ or core/
+CORE_DIRNAME =core
+SRC	     =.
+HTML	     =$(DYNAMO)/html
+HTML_BACK    =../$(CORE_DIRNAME)
+EXPORTS_BASE =$(DYNAMO)/exports
+BUILD_BASE   =$(DYNAMO)/build
+ifeq ($(ARCH), x64)
+  INSTALL_LIB_BASE :=$(DYNAMO)/lib64
+else
+  INSTALL_LIB_BASE :=$(DYNAMO)/lib32
+endif
+INSTALL_BIN      =$(DYNAMO)/bin
+INSTALL_INCLUDE  =$(DYNAMO)/include
+# read from env var
+SYSWIDE_DIR  =$(DYNAMORIO_AUTOINJECT)
+ifeq ($(MACHINE), win32)
+SYSWIDE_DIR := $(subst \,/,$(SYSWIDE_DIR))
+# on windows this env var contains the library name
+# on linux it's the directory name
+SYSWIDE_DIR := $(dir $(SYSWIDE_DIR))
+endif
+# we check for empty env var once we know debug status
+
+###########################################################################
+##
+## Target & Compiler/Linker Flags
+
+# target ARCH
+ifeq ($(ARCH), x64)
+ARCH_DEFS = $(D)X86 $(D)X64
+ARCH_DIRNAME := x86
+IS_X86 := 1
+else
+ifeq ($(ARCH), x86)
+ARCH_DEFS = $(D)X86
+ARCH_DIRNAME := x86
+IS_X86 := 1
+else
+$(error Unsupported arch $(ARCH))
+endif
+endif
+ARCH_SRC := $(SRC)/$(ARCH_DIRNAME)
+
+# target OS (properties for OS variants are below)
+ifeq ($(MACHINE), linux)
+OSNAME = linux
+OS_DEFS = $(D)LINUX
+else # win32
+OSNAME = win32
+OS_DEFS = $(D)WINDOWS
+endif
+OS_SRC = $(SRC)/$(OSNAME)
+
+# NOTE we use LIBDIR instead of LIB to avoid shadowing the env. variable
+# LIB when using make runregression on the local host
+LIBDIR = lib
+LIB_SRC = $(SRC)/$(LIBDIR)
+
+ifndef PROGRAMFILES
+ifdef HOMEDRIVE
+PROGRAMFILES = $(HOMEDRIVE)\\PROGRAM FILES
+else
+PROGRAMFILES = C:\\PROGRAM FILES
+endif
+endif
+
+# Include directories
+# NOTE we use INCLUDE_PATHS instead of INCLUDE to avoid shadowing the env.
+# variable INCLUDE when using make runregression on the local host, if buildtools
+# is set then this isn't a problem so it's okay that Buildtools Setup.mk sets INCLUDE
+ifeq ($(MACHINE), linux)
+  INCLUDE_PATHS = $(I)$(SRC) $(I)$(ARCH_SRC) $(I)$(OS_SRC) $(I)$(LIB_SRC)
+else
+  INCLUDE_PATHS = $(I)$(subst /,\\,$(SRC)) $(I)$(subst /,\\,$(ARCH_SRC)) \
+                  $(I)$(subst /,\\,$(OS_SRC)) $(I)$(subst /,\\,$(LIB_SRC)) \
+                  $(MSINCLUDE)
+endif
+
+###########################################################################
+# 
+# Configuration Values
+#
+# The configuration to build can be specified on the cmdline by overriding the default
+# values here or in a file named Makefile.mydefines which we include below.  To specify
+# individual defines from the section "definitions for conditional compilation" set
+# ADD_DEFINES.
+#
+# The common configurations are specfied with :
+# [VPS|VMSAFE|VMAP]=1 INERNAL=[0,1] DEBUG=[0,1]  (default is VMSAFE=1 INTERNAL=1 DEBUG=1)
+#
+# INTERNAL - specifies whether this build is for internal use only or for customer's use
+# DEBUG - specifies a debug build
+# INTERNAL=0 DEBUG=0 is an external release build
+# INTERNAL=0 DEBUG=1 is an external debug build
+# INTERNAL=1 DEBUG=1 is an internal debug build
+#
+# We map VMware OBJDIR as well:
+# release => INTERNAL=0 DEBUG=0
+# beta => INTERNAL=0 DEBUG=1
+# opt => INTERNAL=0 DEBUG=0
+# obj => INTERNAL=1 DEBUG=1
+#
+# VPS, VMSAFE, and VMAP are meta-configurations that control the other configuration
+# options (except INTERNAL and DEBUG) and correspond to the VPS (Determina), VMSAFE, and
+# VMAP products respectively.  They are meant to be mutually exclusive.
+#
+# OS variants:
+# VMKERNEL - ESX or ESXi
+# VMKCLASSIC - ESX classic
+#   FIXME PR 363075: eventually should go away and be runtime-controlled
+#   Currently it's only used for INIT_TAKE_OVER
+# MACOS - Mac OS X
+#
+# individual configuration specifiers used by VPS/VMSAFE/VMAP :
+# PROGRAM_SHEPHERDING - enable execution shepherding for security
+# CLIENT_INTERFACE - are we exposing our client api
+# APP_EXPORTS - are we exporting the start/stop interface
+# STRACE_CLIENT - should we build the strace client
+# HOT_PATCHING_INTERFACE - are we using hot patching functionality (liveshields/probes)
+# PROCESS_CONTROL - enable process control/lockdown features
+# GBOP - enable our generic buffer overflow prevention feature (windows only)
+#
+# See below for additional configuration options not tied to VPS/VMSAFE/VMAP.
+#
+
+DEBUG := 1
+INTERNAL := 1
+
+ifeq ($(OBJDIR),release)
+DEBUG := 0
+INTERNAL := 0
+endif
+ifeq ($(OBJDIR),beta)
+DEBUG := 1
+INTERNAL := 0
+endif
+ifeq ($(OBJDIR),opt)
+DEBUG := 0
+INTERNAL := 0
+endif
+ifeq ($(OBJDIR),obj)
+DEBUG := 1
+INTERNAL := 1
+endif
+
+# default to VMAP=1, but only if neither VPS or VMSAFE is set
+VPS := 0
+VMAP := 0
+VMSAFE := 0
+ifneq ($(VMSAFE), 1)
+ifneq ($(VPS), 1)
+VMAP := 1
+endif
+endif
+
+# default is Linux
+VMKERNEL := 0
+VMKCLASSIC := 0
+MACOS := 0
+
+# we default to everything off
+PROGRAM_SHEPHERDING := 0
+CLIENT_INTERFACE := 0
+APP_EXPORTS := 0
+STRACE_CLIENT := 0
+HOT_PATCHING_INTERFACE := 0
+PROCESS_CONTROL := 0
+ifeq ($(MACHINE), win32)
+GBOP := 0
+endif
+
+# set the VMSAFE/VMAP/VPS configurations
+
+ifeq ($(VMSAFE), 1)
+# client and security support
+PROGRAM_SHEPHERDING := 1
+CLIENT_INTERFACE := 1
+APP_EXPORTS := 1
+HOT_PATCHING_INTERFACE := 1
+endif
+
+ifeq ($(VMAP), 1)
+# client support, no security
+CLIENT_INTERFACE := 1
+APP_EXPORTS := 1
+endif
+
+ifeq ($(VPS), 1)
+# security support, no client
+PROGRAM_SHEPHERDING := 1
+HOT_PATCHING_INTERFACE := 1
+PROCESS_CONTROL := 1
+ifeq ($(MACHINE), win32)
+GBOP := 1
+endif
+endif
+
+# can be used to enable collection of DR timing info in non-internal builds
+KSTATS = 0
+
+# use to build a non-debug yet non-optimized build, for caller profiling
+CALLPROF = 0
+
+ifeq ($(MACHINE), linux)
+# PR 262458: now that we're using gcc >= 3.4 we
+# can use -fvisibility instead of a linker version script.
+# Note that we're still using a linker script to set our preferred
+# base (PR 253624) and will need to for section ordering as well
+# (PR 208267) but those are separate scripts.
+LINKER_VERSION_SCRIPT = 0
+
+KEEP_SYMBOLS_FOR_LIBC_BACKTRACE := 0
+ifeq ($(INTERNAL), 1)
+KEEP_SYMBOLS_FOR_LIBC_BACKTRACE := 1
+endif
+
+endif
+
+# static or shared library [no longer supported, FIXME: maybe remove]
+STATIC = 0
+
+# build an executable for a unit test, rather than a library?
+STANDALONE_UNIT_TEST = 0
+
+ifeq ($(MACHINE), win32)
+# for libc independence
+NOLIBC = 1
+endif
+
+# use dependencies?
+ifeq ($(MACHINE), linux)
+USEDEP = 0 # PR 207890: not always working on linux either
+else
+# FIXME PR 207890: having problems with cygwin make, leave off for now
+USEDEP = 0
+endif
+
+# convenience include file for per-tree customization:
+-include Makefile.mydefines
+
+## split defines from flags so can pass to assembler & makedepend:
+
+# general purpose CFLAGS
+
+###################################
+# definitions for conditional compilation
+#
+# Linux variants
+#    $(D)HAVE_PROC_MAPS - set if /proc/self/maps is available
+#    $(D)HAVE_TLS       - set if any form of ldt or gdt entry can be claimed
+#    $(D)HAVE_TLS_AREA_MULTI - set if a usually-unused set_thread_area slot is available
+#    $(D)HAVE_SIGALTSTACK - set if SYS_sigaltstack is available
+#    $(D)INIT_TAKE_OVER - libdynamorio.so init() takes over so no preload needed
+# not supported but still in code because may be useful later
+#    $(D)STEAL_REGISTER
+#    $(D)DCONTEXT_IN_EDI
+#    $(D)ASSUME_NORMAL_EFLAGS - (NEVER ON) (causes errors w/ Microsoft-compiled apps)
+# internal studies, not for general use
+#    $(D)SHARING_STUDY
+#    $(D)FRAGMENT_SIZES_STUDY
+#    $(D)FOOL_CPUID
+#    $(D)NATIVE_RETURN - should clean this up, as well
+#	NATIVE_RETURN_CALLDEPTH, NATIVE_RETURN_RET_IN_TRACES, 
+#	NATIVE_RETURN_TRY_TO_PUT_APP_RETURN_PC_ON_STACK
+#    $(D)LOAD_DYNAMO_DEBUGBREAK
+# profiling
+#    $(D)PROFILE_LINKCOUNT $(D)LINKCOUNT_64_BITS 
+#    $(D)PROFILE_RDTSC 
+#    $(D)PAPI - now deprecated
+#    $(D)WINDOWS_PC_SAMPLE - on for all Windows builds
+#    $(D)KSTATS - on for INTERNAL, DEBUG, and PROFILE builds, use KSTATS=1 for
+# release builds
+#    $(D)PROGRAM_SHEPHERDING -  (always ON)
+#         currently turns on code origins checks and diagnostics, eventually will also turn
+#         on return-after-call and other restricted control transfer features
+#    $(D)RETURN_AFTER_CALL  - (always ON) return only to instructions after seen calls
+#    $(D)RCT_IND_BRANCH     - (experimental) indirect branch only to address taken entry points
+#    $(D)DGC_DIAGNOSTICS
+#    $(D)CHECK_RETURNS_SSE2 (experimental security feature)
+#    $(D)CHECK_RETURNS_SSE2_EMIT (experimental unfinished)
+#    $(D)DIRECT_CALL_CHECK  (experimental unfinished)
+#    $(D)SIMULATE_ATTACK    - simulate security violations
+#    $(D)GBOP - generic buffer overflow prevention via hooking APIs
+# optimization of application
+#    $(D)SIDELINE
+#    $(D)SIDELINE_COUNT_STUDY
+#    $(D)LOAD_TO_CONST - around loadtoconst.c, $(D)LTC_STATS 
+
+# optimization of dynamo
+#    $(D)AVOID_EFLAGS  (uses instructions that don't modify flags) (defines ASSUME_NORMAL_EFLAGS)
+#    $(D)RETURN_STACK
+#    $(D)TRACE_HEAD_CACHE_INCR   (incompatible with security FIXME:?)
+#    $(D)DISALLOW_CACHE_RESIZING (use as temporary hack when developing)
+# transparency
+#    $(D)NOLIBC = doesn't link libc on windows, currently uses ntdll.dll libc
+#      functions, NOLIBC=0 causes the core to be linked against libc and kernel32.dll
+# external interface
+#    $(D)CLIENT_INTERFACE
+#    $(D)DR_APP_EXPORTS
+#    $(D)CUSTOM_EXIT_STUBS -- optional part of CLIENT_INTERFACE
+#      we may want it for our own internal use too, though
+#    $(D)CUSTOM_TRACES -- optional part of CLIENT_INTERFACE
+#      has some sub-features that are aggressive and not supported by default:
+#      $(D)CUSTOM_TRACES_RET_REMOVAL = support for removing inlined rets
+#      $(D)CLIENT_SIDELINE = allows adaptive interface methods to be called 
+#                            from other threads safetly, performance hit
+#                            requires CLIENT_INTERFACE
+#    $(D)UNSUPPORTED_API -- part of 0.9.4 MIT API but not supported in current API
+#    $(D)NOT_DYNAMORIO_CORE - should be defined by non core components sharing our code
+
+#    $(D)FANCY_COUNTDOWN - (NOT IMPLEMENTED) countdown messagebox
+
+# debugging
+#    $(D)DEBUG for debug builds
+#    $(D)DEBUG_MEMORY (on for DEBUG)
+#    $(D)STACK_GUARD_PAGE (on for DEBUG)
+#    $(D)DEADLOCK_AVOIDANCE (on for DEBUG) - enforce total rank order on locks
+#    $(D)MUTEX_CALLSTACK - enable collecting callstack info, requires DEADLOCK_AVOIDANCE
+#    $(D)HEAP_ACCOUNTING (on for DEBUG)
+
+#    $(D)INTERNAL for features that are not intended to reach customer hands
+#    $(D)VERBOSE=1 for verbose debugging or in situations where normal DEBUG 
+
+# statistics
+#    $(D)HASHTABLE_STATISTICS - IBL table statistics
+
+# target platforms
+#    $(D)WINDOWS (avoid using _WIN32 used by cl)
+#    $(D)LINUX 
+#    note that in many cases we use the else of WINDOWS to mean LINUX and vice versa
+#    we're just starting to add VMKERNEL and MACOS support
+#    $(D)X86
+#    $(D)X64
+
+# support for running in x86 emulator on IA-64
+#    $(D)IA32_ON_IA64
+
+# build script provides these
+#    $(D)BUILD_NUMBER (<64K == vmware's PRODUCT_BUILD_NUMBER)
+#    $(D)UNIQUE_BUILD_NUMBER (== vmware's BUILD_NUMBER)
+#    $(D)VERSION_NUMBER
+#    $(D)VERSION_COMMA_DELIMITED
+
+###################################
+
+# UNIX variants
+ifeq ($(MACHINE), linux)
+  ifeq ($(VMKCLASSIC), 1)
+    VMKERNEL := 1
+  endif
+  ifeq ($(VMKERNEL), 1)
+    GEN_DEFINES += $(D)VMX86_SERVER
+    GEN_DEFINES += $(D)USERLEVEL
+
+    # FIXME PR 361894: ESX4.0+ has HAVE_TLS but not HAVE_TLS_AREA_MULTI:
+    # so won't work w/ pthreads or other app using TLS slot:
+    # not specifying for now since too limiting.
+    # Remove HAVE_TLS_AREA_MULTI, then?
+    # Plus, ESX3.5 does not have any TLS: so need runtime control
+    # over whether use non-TLS get_thread_private_dcontext()
+    # and force thread-private (and not support x64).
+    ifeq ($(VMKCLASSIC), 1)
+      # FIXME: setting this until get drpreload working on esx classic
+      GEN_DEFINES += $(D)INIT_TAKE_OVER
+    endif
+
+
+    # We need these only for the OS dependent files,
+    # yet there are no filename conflicts so always add to include paths
+    VMKERNEL_TREE := $(VMTREE)# path to bora
+    INCLUDE_PATHS += $(I)$(VMKERNEL_TREE)/public \
+		     $(I)$(VMKERNEL_TREE)/lib/public \
+		     $(I)$(VMKERNEL_TREE)/vmkernel/public \
+		     $(I)$(VMKERNEL_TREE)/vmkernel/include/vmkapi \
+		     $(I)$(VMKERNEL_TREE)/vmkernel/include/user \
+		     $(I)$(VMKERNEL_TREE)/vmkernel/include/public \
+		     $(I)$(VMKERNEL_TREE)/lib/vmkuser/include/public \
+		     $(I)$(VMKERNEL_TREE)/vmkernel/distribute \
+		     $(I)$(VMKERNEL_TREE)/vmcore/public
+
+  else
+    ifeq ($(MACOS), 1)
+      # FIXME NYI
+      GEN_DEFINES += $(D)MACOS
+    else
+      # Linux
+      GEN_DEFINES += $(D)HAVE_PROC_MAPS
+      GEN_DEFINES += $(D)HAVE_TLS
+      GEN_DEFINES += $(D)HAVE_TLS_AREA_MULTI
+      GEN_DEFINES += $(D)HAVE_SIGALTSTACK
+    endif
+  endif
+endif
+
+# defines in all builds
+ifeq ($(MACHINE), win32)
+GEN_DEFINES += $(D)WINDOWS_PC_SAMPLE
+endif
+
+# defines that depend on configuration variables
+# N.B.: order is important here as some set values of later vars
+
+ifeq ($(PROGRAM_SHEPHERDING), 1)
+GEN_DEFINES += $(D)PROGRAM_SHEPHERDING
+GEN_DEFINES += $(D)RETURN_AFTER_CALL
+GEN_DEFINES += $(D)RCT_IND_BRANCH
+endif
+
+ifeq ($(CLIENT_INTERFACE), 1)
+# standard client interface features
+GEN_DEFINES += $(D)CLIENT_INTERFACE
+GEN_DEFINES += $(D)DYNAMORIO_IR_EXPORTS
+GEN_DEFINES += $(D)CUSTOM_TRACES
+# PR 200409: not part of our current API, xref PR 215179 on -pad_jmps issues with
+# CUSTOM_EXIT_STUBS
+#GEN_DEFINES += $(D)CUSTOM_EXIT_STUBS
+#GEN_DEFINES += $(D)UNSUPPORTED_API
+endif
+
+ifeq ($(APP_EXPORTS), 1)
+GEN_DEFINES += $(D)DR_APP_EXPORTS
+endif
+
+ifeq ($(HOT_PATCHING_INTERFACE), 1)
+GEN_DEFINES += $(D)HOT_PATCHING_INTERFACE
+endif
+
+ifeq ($(PROCESS_CONTROL), 1)
+GEN_DEFINES += $(D)PROCESS_CONTROL
+endif
+
+ifeq ($(GBOP), 1)
+# FIXME: some of the hooks depend on hotp_only HOT_PATCHING_INTERFACE
+GEN_DEFINES += $(D)GBOP
+endif
+
+# we also define INTERNAL for builds for features that are not intended to reach customer hands
+ifeq ($(DEBUG), 0)
+# a command line override is still possible, but the default release build is NOT internal
+INTERNAL = 0
+else
+# for bug fixing this is useful so we turn on for all debug builds
+GEN_DEFINES += $(D)DEBUG_MEMORY
+GEN_DEFINES += $(D)STACK_GUARD_PAGE
+GEN_DEFINES += $(D)HEAP_ACCOUNTING
+GEN_DEFINES += $(D)DEADLOCK_AVOIDANCE
+GEN_DEFINES += $(D)MUTEX_CALLSTACK # requires DEADLOCK_AVOIDANCE
+# even though only usable in all-private config useful in default builds
+GEN_DEFINES += $(D)SHARING_STUDY
+endif
+
+ifeq ($(INTERNAL), 1)
+GEN_DEFINES += $(D)INTERNAL
+KSTATS = 1
+endif
+
+ifeq ($(EXTERNAL_RELEASE), 1)
+$(error EXTERNAL_RELEASE no longer supported. Update your script.)
+endif
+
+ifeq ($(EXTERNAL_INJECTOR), 1)
+GEN_DEFINES += $(D)EXTERNAL_INJECTOR
+endif
+
+ifeq ($(STANDALONE_UNIT_TEST), 1)
+GEN_DEFINES += $(D)STANDALONE_UNIT_TEST
+endif
+
+NOLIBC_DLL_ENTRY = /entry:DllMain
+ifeq ($(MACHINE), win32)
+ifeq ($(NOLIBC), 1)
+GEN_DEFINES += $(D)NOLIBC
+CORE_DLL_ENTRY = $(NOLIBC_DLL_ENTRY)
+else
+CORE_DLL_ENTRY =
+endif
+endif
+
+# Processing for custom builds
+include Makefile.custom_build
+
+ifneq ($(CUSTOM_PRODUCT_NAME),)
+  GEN_DEFINES += $(D)CUSTOM_PRODUCT_NAME=$(CUSTOM_PRODUCT_NAME)
+endif
+
+ifneq ($(BUILD_NUMBER),)
+  GEN_DEFINES += $(D)BUILD_NUMBER=$(BUILD_NUMBER)
+endif
+
+ifneq ($(UNIQUE_BUILD_NUMBER),)
+  GEN_DEFINES += $(D)UNIQUE_BUILD_NUMBER=$(UNIQUE_BUILD_NUMBER)
+else
+  GEN_DEFINES += $(D)UNIQUE_BUILD_NUMBER=$(BUILD_NUMBER)
+endif
+
+ifneq ($(VERSION_NUMBER),)
+  GEN_DEFINES += $(D)VERSION_NUMBER=$(VERSION_NUMBER)
+endif
+
+ifneq ($(VERSION_COMMA_DELIMITED),)
+  GEN_DEFINES += $(D)VERSION_COMMA_DELIMITED=$(VERSION_COMMA_DELIMITED)
+endif
+
+STATS=$(DEBUG)
+ifeq ($(STATS),1)
+GEN_DEFINES += $(D)HASHTABLE_STATISTICS
+KSTATS = 1
+endif
+
+# no KSTATS for caller profiling: we want to be as close to release
+# build as we can, but w/o optimizations
+ifeq ($(CALLPROF), 1)
+KSTATS=0
+GEN_DEFINES += $(D)CALL_PROFILE
+endif
+
+ifeq ($(KSTATS), 1)
+GEN_DEFINES += $(D)KSTATS
+endif
+
+# choose BUILD_SUFFIX 
+ifeq ($(DEBUG),1)
+BUILD_SUFFIX=dbg
+INSTALL_LIB = $(INSTALL_LIB_BASE)/debug
+else
+ifeq ($(PROFILE),1)
+GEN_DEFINES += $(D)PROFILE
+BUILD_SUFFIX=prof
+INSTALL_LIB = $(INSTALL_LIB_BASE)/profile
+else
+ifeq ($(INTERNAL), 1)
+BUILD_SUFFIX=intr
+INSTALL_LIB = $(INSTALL_LIB_BASE)/internal
+else
+BUILD_SUFFIX=rel
+INSTALL_LIB = $(INSTALL_LIB_BASE)/release
+endif # INTERNAL
+endif # PROFILE
+endif # DEBUG
+
+BUILD = $(BUILD_BASE)/$(ARCH)_$(OSNAME)_$(BUILD_SUFFIX)
+EXPORTS = $(EXPORTS_BASE)/$(ARCH)_$(OSNAME)_$(BUILD_SUFFIX)
+
+ifeq ($(SYSWIDE_DIR),)
+# use install dir if no systemwide env var set
+SYSWIDE_DIR = $(INSTALL_LIB_BASE)
+endif
+
+# target specific CFLAGS
+TGT_DEFINES = $(ARCH_DEFS) $(OS_DEFS)
+ifeq ($(STATIC), 1)
+TGT_DEFINES += $(D)STATIC_LIBRARY
+TGT_CFLAGS =
+else
+ifeq ($(MACHINE), linux)
+
+# FIXME PR 196857: pick gcc, glibc, and binutils targets in
+# the vmware toolchain.
+ifeq ($(ARCH), x64)
+TGT_CFLAGS += -m64
+ASMFLAGS += --64
+else
+TGT_CFLAGS += -m32
+ASMFLAGS += --32
+endif
+
+# don't use -fpic, it's slow?  we want statically linked shared lib?
+TGT_CFLAGS += -fpic
+
+ifeq ($(LINKER_VERSION_SCRIPT), 0)
+# only export functions so marked via attributes
+# only available in gcc 3.4!
+# so we use an ugly linker script via $(EXPORT_LIST) instead by default
+#
+# using "internal" instead of "hidden" b/c we don't need any indirect
+# calls to our non-exported functions
+ifeq ($(KEEP_SYMBOLS_FOR_LIBC_BACKTRACE), 0)
+TGT_CFLAGS += -fvisibility=internal
+endif
+GEN_DEFINES += $(D)USE_VISIBILITY_ATTRIBUTES
+endif
+
+# disable strict aliasing opt in gcc 3.3.3 -- gives warnings and makes bad assumptions
+TGT_CFLAGS += -fno-strict-aliasing
+else
+TGT_CFLAGS =
+# read-only string pooling
+TGT_CFLAGS +=/GF
+endif
+endif
+
+GEN_CFLAGS = $(WARN)
+DEFINES = $(GEN_DEFINES) $(TGT_DEFINES) $(ADD_DEFINES) $(REMOVE_DEFINES)
+API_DEFINES = $(DEFINES) $(D)API_EXPORT_ONLY
+CFLAGS += $(GEN_CFLAGS) $(TGT_CFLAGS)
+
+ifeq ($(MACHINE), win32)
+# FIXME case 191729: we should try to enable this.
+# Currently we get "unresolved external symbol ___security_cookie"
+CFLAGS += /GS-
+FILTER_CC =
+ifneq ($(ARCH), x64)
+# PR 219380: to avoid __ftol2_sse from libc
+$(BUILD)/io.obj: CFLAGS += /QIfist
+# There is no supported way to suppress a "command line" warning
+$(BUILD)/io.obj: FILTER_CC = 2>&1 | $(GREP) -v "D9035 : option 'QIfist'";
+endif
+endif
+
+ifeq ($(DEBUG),1)
+# debug specific flags and defines
+DEFINES += $(D)DEBUG
+CFLAGS += $(DBG) $(DBG_OPT)
+else
+# release specific flags and defines
+DEFINES += $(D)RELEASE 
+ifeq ($(CALLPROF), 1)
+# no opts -- we need to avoid messing up call frame walking
+# FIXME: just disable frame ptr elim opt: but /Oy- ran into some issues
+else
+CFLAGS += $(OPT)
+endif
+ifeq ($(MACHINE), win32)
+# using DBG here won't mess up the optimizations, debug line number info
+# gets a little messed up, but is better than nothing
+CFLAGS += $(DBG)
+endif
+endif
+
+# asm flags
+ifeq ($(IS_X86), 1)
+  ifeq ($(MACHINE), linux)
+    DEFINES += $(D)ASSEMBLE_WITH_GAS
+    ASM_DEFINES := $(patsubst $(D)%, --defsym %=1, $(DEFINES))
+    ASM_DEFINES := $(patsubst $(U)%, --defsym %=0, $(ASM_DEFINES))
+    ASM_NDCP_DEFINES := --defsym NOT_DYNAMORIO_CORE_PROPER=1
+    # We do NOT use --gstabs as it creates two sections, .stab and .stabstr, AFTER our
+    # .lspdata and .nspdata sections.  This messes up our data segment protection scheme
+    # as .data is BEFORE our protected sections and .stab* are writable (older 
+    # versions of as had them read-only).  We have found no way to control the order
+    # with the linker, and we don't want to bother to recognize these particular sections
+    # and omit them from protection, since we don't really need them.
+  else
+    DEFINES += $(D)ASSEMBLE_WITH_MASM
+    ASM_DEFINES := $(DEFINES)
+    ASM_NDCP_DEFINES := $(D)NOT_DYNAMORIO_CORE_PROPER
+    ASMFLAGS += $(INCR)
+    ifeq ($(DEBUG),1)
+      ASMFLAGS += $(DBG)
+    endif
+    # ml can't handle the line number markers
+    ASMPREPROC += $(CPP_NO_LINENUM)
+  endif
+  ASMFLAGS += $(ASM_DEFINES)
+endif
+
+# target-specific linker flags (only used if creating .o instead of .a)
+ifeq ($(MACHINE), linux)
+API_UNDEFS = $(U)dynamorio_app_init $(U)dynamorio_app_exit \
+	     $(U)dynamo_thread_init $(U)dynamo_thread_exit \
+	     $(U)dr_app_start $(U)dr_app_stop \
+	     $(U)dynamorio_app_take_over
+LDFLAGS = $(API_UNDEFS) -s -x -r
+# need libgcc.a for __cmpdi2 (long long conversion functions I think)
+## actually I don't need to explicitly name it if link through gcc
+## GCC_LIB_PATH := $(shell locate libgcc.a)
+CC_LDLIBS	:= -lc -ldl -lm
+# FIXME case 1891/8972: should statically link w/ everything
+# but then we are initialized prior to 
+# libc init and we need to get our env vars directly (case 1891/69)
+#CC_LDLIBS	= $(L)/usr/lib -static-libgcc /usr/lib/libc.a /usr/lib/libdl.a /usr/lib/libm.a lib/papi/libpapi.a
+LD_LDLIBS = $(CC_LDLIBS)
+else # win32
+OUTNAME = dynamorio
+INJECTNAME = drinject
+PREINJECTNAME = drpreinject
+EARLY_INJECT_HELPER1_NAME = drearlyhelp1
+EARLY_INJECT_HELPER2_NAME = drearlyhelp2
+# PR 219380: we export ntdll routines from dynamorio.dll by forwarding them.
+# However, the linker tries to be smart and thinks our _snprintf forward
+# is the same as our own snprintf: it removes the latter and issues a warning.
+# We re-add our own snprintf by listing it in the .def file, and we suppress
+# the warning using the secret option "/ignore".
+ifeq ($(ARCH), x64)
+DEF_SRC := lib/dr_ntdll_x64.def
+# Xref PR 215395, we currently require DynamoRIO to be loaded in a certain address
+# range.  To be double sure we compile it FIXED for now.
+NO_PIC64 := /FIXED
+else
+DEF_SRC := lib/dr_ntdll.def
+NO_PIC64 :=
+endif
+DEF_BLD := $(BUILD)/$(DEF_SRC)
+FORWARD_TO_NTDLL := /def:$(DEF_BLD) /ignore:4197
+ifeq ($(DEBUG), 1)
+# FIXME: we should verify these bases have no conflicts in supported apps
+TGTFLAGS = ${LINK_DBG} /release /opt:ref
+LDFLAGS_B = /base:"0x15000000" $(NO_PIC64)
+else
+TGTFLAGS = ${LINK_DBG} /release /opt:ref /opt:icf 
+LDFLAGS_B = /base:"0x71000000" $(NO_PIC64)
+endif
+MSBUILD := $(subst /,\\,$(BUILD))
+ifeq ($(ARCH), x64)
+LD_MACHINE_FLAGS = /machine:X64
+NTDLL_LIB := ntdll.lib
+NTDLL_LIBPATH := $(NTDLL_LIBPATH_X64)
+else
+LDFLAGS = /machine:I386
+NTDLL_LIB := ntdll.lib
+NTDLL_LIBPATH := $(NTDLL_LIBPATH_X86)
+endif
+# make sure to export dynamo_auto_start (it's now in x86.s and so
+# we can't just say __declspec(dllexport))
+LDFLAGS = $(TGTFLAGS) $(LDFLAGS_B) /dll /incremental:no \
+          $(CORE_DLL_ENTRY) $(LD_MACHINE_FLAGS) \
+          /pdb:"$(MSBUILD)\\$(OUTNAME).pdb" \
+          /out:"$(MSBUILD)\\$(OUTNAME)$(SO_SFX)" \
+          /implib:"$(MSBUILD)\\$(OUTNAME).lib" \
+          /map:"$(MSBUILD)\\$(OUTNAME).map" \
+          /mapinfo:exports $(FORWARD_TO_NTDLL) \
+          /export:dynamo_auto_start \
+          /export:dynamorio_app_take_over
+ifeq ($(APP_EXPORTS), 1)
+LDFLAGS += /export:dr_app_start \
+	   /export:dr_app_take_over
+endif
+# I'm adding libcmt to attempt to make sure we have multithreaded C lib:
+ifeq ($(DEBUG), 1)
+WIN32_C_LIB = libcmtd.lib
+else
+WIN32_C_LIB = libcmt.lib
+endif
+ifeq ($(NOLIBC), 1)
+CORE_WIN32_C_LIB =
+else
+CORE_WIN32_C_LIB = $(WIN32_C_LIB) kernel32.lib
+endif
+# case 4125: we link with msvcrt.lib for vc8 /O2's use of _alldvrm
+# note that _alldvrm is in ntdll.dll on xp+; but for pre-xp uses we
+# stick with the static linking of the impl in msvcrt.lib.
+CC_LDLIBS = $(MSLIB) $(L)$(subst /,\\,$(NTDLL_LIBPATH)) \
+            /NODEFAULTLIB $(CORE_WIN32_C_LIB) $(NTDLL_LIB) msvcrt.lib
+LD_LDLIBS := $(subst /link,,$(CC_LDLIBS))
+INJECTFLAGS = $(TGTFLAGS) /incremental:no $(LD_MACHINE_FLAGS) /subsystem:console \
+          /pdb:"$(MSBUILD)\\$(INJECTNAME).pdb" \
+          /out:"$(MSBUILD)\\$(INJECTNAME).exe" \
+          /map:"$(MSBUILD)\\$(INJECTNAME).map" \
+          /mapinfo:exports
+INJECTLIBS_1 = $(MSLIB) $(L)$(subst /,\\,$(NTDLL_LIBPATH)) /NODEFAULTLIB \
+               $(WIN32_C_LIB) $(NTDLL_LIB) kernel32.lib Advapi32.lib imagehlp.lib
+INJECTLIBS = $(subst /link,,$(INJECTLIBS_1))
+PRE_INJECT_B = /base:"0x14000000"
+# preinjector requires no C library for its unload-self method
+PRE_INJECTFLAGS = $(TGTFLAGS) $(PRE_INJECT_B) /dll /incremental:no $(LD_MACHINE_FLAGS) \
+          $(NOLIBC_DLL_ENTRY) \
+          /pdb:"$(MSBUILD)\\$(PREINJECTNAME).pdb" \
+          /out:"$(MSBUILD)\\$(PREINJECTNAME).dll" \
+          /map:"$(MSBUILD)\\$(PREINJECTNAME).map" \
+          /mapinfo:exports
+# case 4125: we link with msvcrt.lib for vc8 /O2's use of _alldvrm
+PRE_INJECTLIBS_1 = $(MSLIB) $(L)$(subst /,\\,$(NTDLL_LIBPATH)) \
+                   /NODEFAULTLIB $(NTDLL_LIB) kernel32.lib msvcrt.lib \
+                   $(L)$(EXPORTS) $(OUTNAME).lib
+PRE_INJECTLIBS := $(subst /link,,$(PRE_INJECTLIBS_1))
+# release base - 64k 
+EARLY_INJECT_HELPER1_B = /base:"0x70FF0000"
+EARLY_INJECT_HELPER1_FLAGS = $(TGTFLAGS) $(EARLY_INJECT_HELPER1_B) /dll \
+          /incremental:no $(LD_MACHINE_FLAGS) $(NOLIBC_DLL_ENTRY) \
+          /pdb:"$(MSBUILD)\\$(EARLY_INJECT_HELPER1_NAME).pdb" \
+          /out:"$(MSBUILD)\\$(EARLY_INJECT_HELPER1_NAME)$(SO_SFX)" \
+          /map:"$(MSBUILD)\\$(EARLY_INJECT_HELPER1_NAME).map" \
+          /mapinfo:exports
+EARLY_INJECT_HELPER1_LIBS = /NODEFAULTLIB "$(MSBUILD)\\$(EARLY_INJECT_HELPER2_NAME).lib"
+# helper1 base - 64k 
+EARLY_INJECT_HELPER2_B = /base:"0x71110000"
+EARLY_INJECT_HELPER2_FLAGS = $(TGTFLAGS) $(EARLY_INJECT_HELPER2_B) /dll \
+          /incremental:no $(LD_MACHINE_FLAGS) $(NOLIBC_DLL_ENTRY) \
+          /pdb:"$(MSBUILD)\\$(EARLY_INJECT_HELPER2_NAME).pdb" \
+          /out:"$(MSBUILD)\\$(EARLY_INJECT_HELPER2_NAME)$(SO_SFX)" \
+          /implib:"$(MSBUILD)\\$(EARLY_INJECT_HELPER2_NAME).lib" \
+          /map:"$(MSBUILD)\\$(EARLY_INJECT_HELPER2_NAME).map" \
+          /mapinfo:exports 
+EARLY_INJECT_HELPER2_LIBS = /NODEFAULTLIB
+endif
+
+###########################################################################
+##
+## Files
+
+FORTRAN_SRC = $(LIB_SRC)/fortran.c
+FORTRAN_OBJ = $(LIBDIR)/fortran.$(OBJ_SFX)
+
+STANDALONE_NAME = drtest-${UNIT_TEST}
+ifeq ($(MACHINE), linux)
+ifeq ($(STATIC), 1)
+ARCHIVE	= libdynamorio$(SO_SFX)
+else
+ARCHIVE	= libdynamorio$(SO_SFX)
+PRELOAD = libdrpreload$(SO_SFX)
+ifeq ($(LINKER_VERSION_SCRIPT), 1)
+EXPORT_LIST = $(BUILD)/exportlist
+LINKER_EXPORT_FLAGS = -Xlinker --version-script -Xlinker $(EXPORT_LIST)
+endif
+LD_SCRIPT = $(BUILD)/ldscript
+endif
+
+ifeq ($(KEEP_SYMBOLS_FOR_LIBC_BACKTRACE), 1)
+LINKER_EXPORT_FLAGS += -Xlinker --export-dynamic
+endif
+
+else # win32
+ARCHIVE	= $(OUTNAME)$(SO_SFX)
+INJECTOR = $(INJECTNAME)$(EXE_SFX)
+PRELOAD = $(PREINJECTNAME)$(SO_SFX)
+EARLY_INJECT_HELPER1 = $(EARLY_INJECT_HELPER1_NAME)$(SO_SFX)
+EARLY_INJECT_HELPER2 = $(EARLY_INJECT_HELPER2_NAME)$(SO_SFX)
+endif
+STANDALONE_EXE = ${STANDALONE_NAME}${EXE_SFX}
+
+# options.c first so that cl will put lspdata section before nspdata section!
+CORE_SRCFILES = options.c dynamo.c fragment.c fcache.c link.c dispatch.c emit.c utils.c \
+                stats.c heap.c instrlist.c monitor.c perfctr.c vmareas.c rct.c \
+                hotpatch.c hashtable.c module_list.c moduledb.c perscache.c nudge.c \
+                synch.c
+ifeq ($(OSNAME), linux)
+CORE_SRCFILES += io.c
+else
+# PR 219380: ntdll._vnsprintf doesn't support printing floats so we export ours
+ifeq ($(CLIENT_INTERFACE), 1)
+CORE_SRCFILES += io.c
+endif
+endif
+CORE_SRCS := $(patsubst %,$(SRC)/%, $(CORE_SRCFILES))
+CORE_OBJS := $(patsubst %.c,$(BUILD)/%.$(OBJ_SFX), $(CORE_SRCFILES)) 
+
+########################################
+## Arch files
+
+# both win32 and linux use x86.s now
+ifeq ($(IS_X86), 1)
+ARCH_SRCFILES := \
+  arch.c decode.c decode_table.c encode.c emit_utils.c instr.c interp.c \
+  proc.c mangle.c steal_reg.c x86_code.c disassemble.c x86.s \
+  decode_fast.c optimize.c sideline.c instrument.c retcheck.c
+endif # x86
+
+ARCH_SRCS  := $(patsubst %,$(ARCH_SRC)/%, $(ARCH_SRCFILES))
+ARCH_OBJS1 := $(patsubst %.c,$(BUILD)/$(ARCH_DIRNAME)/%.$(OBJ_SFX), $(ARCH_SRCFILES))
+ARCH_OBJS  := $(patsubst %.s,$(BUILD)/$(ARCH_DIRNAME)/%.$(OBJ_SFX), $(ARCH_OBJS1))
+
+########################################
+## OS files
+
+ifeq ($(OSNAME), linux)
+OS_SRCFILES = os.c signal.c module.c pcprofile.c stackdump.c diagnost.c
+  ifeq ($(VMKERNEL), 1)
+    OS_SRCFILES += vmkuw.c
+  endif
+else # win32
+OS_SRCFILES = eventlog.c os.c syscall.c callback.c drmarker.c ntdll.c inject.c \
+              inject_shared.c module.c module_shared.c diagnost.c aslr.c
+endif
+OS_SRCS := $(patsubst %,$(OS_SRC)/%, $(OS_SRCFILES))
+OS_OBJS := $(patsubst %.c,$(BUILD)/$(OSNAME)/%.$(OBJ_SFX), $(OS_SRCFILES))
+
+ifeq ($(OSNAME), linux)
+
+########################################
+## Preload Files
+
+PRELOAD_SRCFILES = preload.c
+PRELOAD_SRCS := $(patsubst %,$(OS_SRC)/%, $(PRELOAD_SRCFILES))
+PRELOAD_OBJS := $(patsubst %.c,$(BUILD)/$(OSNAME)/%.$(OBJ_SFX), $(PRELOAD_SRCFILES)) 
+
+endif #linux
+
+ifeq ($(MACHINE), win32)
+
+########################################
+## Injector Files
+
+# N.B.: *-noncore.c does not exist, this is a hack!
+INJECT_SRCFILES = injector.c inject-noncore.c ntdll-noncore.c inject_shared-noncore.c \
+                  x86-noncore.s
+INJECT_SRC_OBJS := $(patsubst %.s,$(BUILD)/$(ARCH_DIRNAME)/%.$(OBJ_SFX),\
+                     $(patsubst %.c,$(BUILD)/$(OSNAME)/%.$(OBJ_SFX), $(INJECT_SRCFILES)))
+INJECT_RESOURCE_OBJS = $(BUILD)/injector.res
+INJECT_OBJS := $(INJECT_SRC_OBJS) $(INJECT_RESOURCE_OBJS)
+
+########################################
+## Pre-Injector files
+
+# N.B.: *-noncore.c does not exist, this is a hack!
+PRE_INJECT_SRCFILES = pre_inject.c ntdll-noncore.c inject_shared-noncore.c \
+                      drmarker-noncore.c x86-noncore.s pre_inject_asm.s \
+                      module_shared-noncore.c
+PRE_INJECT_SRC_OBJS := $(patsubst %.s,$(BUILD)/$(ARCH_DIRNAME)/%.$(OBJ_SFX),\
+                        $(patsubst %.c,$(BUILD)/$(OSNAME)/%.$(OBJ_SFX),\
+                         $(PRE_INJECT_SRCFILES)))
+PRE_INJECT_RESOURCE_OBJS = $(BUILD)/preinjector.res
+PRE_INJECT_OBJS := $(PRE_INJECT_SRC_OBJS) $(PRE_INJECT_RESOURCE_OBJS)
+
+#######################################
+## Early inject helper dlls files
+
+EARLY_INJECT_HELPER1_SRCFILES = early_inject_helper1.c
+EARLY_INJECT_HELPER1_SRCS = $(patsubst %,$(OS_SRC)/%, $(EARLY_INJECT_HELPER1_SRCFILES))
+EARLY_INJECT_HELPER1_SRC_OBJS = $(patsubst %.c,$(BUILD)/$(OSNAME)/%.$(OBJ_SFX), $(EARLY_INJECT_HELPER1_SRCFILES))
+EARLY_INJECT_HELPER1_RESOURCE_OBJS = $(BUILD)/drearlyhelp1.res
+EARLY_INJECT_HELPER1_OBJS := $(EARLY_INJECT_HELPER1_SRC_OBJS) $(EARLY_INJECT_HELPER1_RESOURCE_OBJS)
+
+EARLY_INJECT_HELPER2_SRCFILES = early_inject_helper2.c
+EARLY_INJECT_HELPER2_SRCS = $(patsubst %,$(OS_SRC)/%, $(EARLY_INJECT_HELPER2_SRCFILES))
+EARLY_INJECT_HELPER2_SRC_OBJS = $(patsubst %.c,$(BUILD)/$(OSNAME)/%.$(OBJ_SFX), $(EARLY_INJECT_HELPER2_SRCFILES))
+EARLY_INJECT_HELPER2_RESOURCE_OBJS = $(BUILD)/drearlyhelp1.res
+EARLY_INJECT_HELPER2_OBJS := $(EARLY_INJECT_HELPER2_SRC_OBJS) $(INJECT_RESOURCE_OBJS)
+
+endif #windows
+
+########################################
+## Buildmark files
+## FIXME: $(BUILD)/
+DEFINES_H = $(BUILD)/defines.h
+POST_DEFINES_H = $(BUILD)/post-defines.h
+BUILDMARK_H =  $(DEFINES_H) $(POST_DEFINES_H)
+BUILDMARK_SRCFILES = buildmark.c
+BUILDMARK_SRCS := $(patsubst %,$(SRC)/%, $(BUILDMARK_SRCFILES))
+BUILDMARK_OBJS := $(patsubst %.c,$(BUILD)/%.$(OBJ_SFX), $(BUILDMARK_SRCFILES)) 
+
+########################################
+## Resource files
+# Event log messages
+# .mc file processed and shared for fmt strings, see below
+ifeq ($(MACHINE), win32)
+MESSAGE_FILES=$(OSNAME)/events.mc
+MESSAGE_H= $(patsubst %.mc,$(BUILD)/%.h, $(MESSAGE_FILES)) 
+# note: the above goes to $(BUILD)/OS
+MESSAGE_RC= $(patsubst %.mc,$(BUILD)/%.rc, $(MESSAGE_FILES)) 
+
+# only one resource file allowed so we include by reference generated message resources
+RESOURCE_OBJS=$(BUILD)/dynamorio.res
+GENERATED_INCLUDES= $(I)$(subst /,\\,$(BUILD)) $(I)$(subst /,\\,$(BUILD)/$(OSNAME))
+else # no resources on linux
+RESOURCE_OBJS=
+GENERATED_INCLUDES=$(I)$(BUILD) # for DEFINES_H
+endif
+
+########################################
+## Syslog fmt string header files (shared)
+MESSAGE_STRINGS_FILES=win32/events.mc
+MESSAGE_STRINGS_H=$(BUILD)/event_strings.h
+
+########################################
+## Everything going into core library
+
+SRCS   := $(CORE_SRCS) $(ARCH_SRCS) $(OS_SRCS) $(BUILDMARK_SRCS)
+OTHER_SRCS := $(FORTRAN_SRC)
+C_SRCS := $(patsubst %.s,, $(SRCS))
+CPPS   := $(patsubst ./%.c,$(BUILD)/%.$(CPP_SFX), $(C_SRCS))
+DEPS   := $(CPPS:.$(CPP_SFX)=.$(DEP_SFX))
+OBJS   := $(CORE_OBJS) $(ARCH_OBJS) $(OS_OBJS) $(BUILDMARK_OBJS) $(RESOURCE_OBJS)
+OTHER_OBJS := $(FORTRAN_OBJ)
+
+ifeq ($(MACHINE), win32)
+PDBS   := $(subst .$(OBJ_SFX),.pdb, $(OBJS))
+endif
+
+########################################
+## All header files
+HEADERS = $(wildcard *.h */*.h)
+
+# script to generate API header files
+GENAPI = lib/genapi.pl
+
+###########################################################################
+##
+## Rules
+
+.PHONY:: all builddir help tags clear clean exports unexport zap html ugly pretty_ugly \
+	really_ugly compare-defines clean-mark buildmark buildnum injector preload \
+	early_inject_helper install dynamorio usedep_defines depend clients symbols \
+	diff diffcheck diffemail diffclean diffnotes diffskel diffprep diffsend \
+	commit runregression specupdate api_headers
+
+###########################################################################
+## Unit tests have all objs except for the current unit tested file
+
+ifeq ($(STANDALONE_UNIT_TEST), 1)
+UNIT_TEST = unitname
+UNIT_OBJS := $(subst $(UNIT_TEST),unit-$(UNIT_TEST), $(OBJS))
+
+# FIXME: should rename all other objects to -standalone.obj to make
+# sure compiled with $(D)STANDALONE_UNIT_TEST without complaining when
+# compiling the library itself, that way we will be able to run the
+# unit tests simultaneously as regression tests too
+
+# even though we don't use the corresponding object we'll depend on it here to capture its dependencies
+$(BUILD)/unit-$(UNIT_TEST).$(OBJ_SFX) : $(BUILD)/$(UNIT_TEST).c
+
+ifneq ($(findstring unit-$(UNIT_TEST),$(OBJS)),)
+# we have older unit tests embedded in the src file itself, which
+# we must copy to create the unit- version
+$(BUILD)/unit-$(UNIT_TEST).$(OBJ_SFX) : $(BUILD)/$(UNIT_TEST).$(OBJ_SFX)
+	$(CP) $< $@
+endif
+
+# the default target convention has to be all for the defines check to work properly
+all: builddir standalone
+
+LOGLEVEL = 0
+UNIT_OPTIONS = -stderr_mask 0x15 ${UNIT_ADD_OPTIONS}
+UNIT_OPTIONS += -detect_mode  # for security unit tests
+ifneq (${LOGLEVEL},)
+  UNIT_OPTIONS += -loglevel ${LOGLEVEL}
+endif
+
+ifeq ($(MACHINE), win32)
+# we need to set any environment variables into the registry with drinject
+UNIT_CMDLINE=${EXPORTS}/${INJECTOR} ${INJECTOR_CMDLINE} -noinject
+UNIT_CMDLINE_DEPENDS=${EXPORTS}/${INJECTOR}
+# FIXME: in fact it will be a good dual purpose test to run all tests under DR
+# after the proper library is used - see comment about -standalone.obj
+else
+UNIT_CMDLINE=
+endif # !win32
+
+unitrun: all standalone ${UNIT_CMDLINE_DEPENDS}
+	export DYNAMORIO_OPTIONS="${UNIT_OPTIONS}"; ${UNIT_CMDLINE} ${BUILD}/${STANDALONE_EXE}
+
+else # !STANDALONE
+
+ifeq ($(STATIC),1)
+all: builddir dynamorio $(EXPORTS)/$(FORTRAN_OBJ)
+else
+ifeq ($(MACHINE), win32)
+all: builddir dynamorio injector early_inject_helper clients
+else #linux
+all: builddir dynamorio $(EXPORTS)/$(FORTRAN_OBJ) \
+     $(EXPORTS)/$(PRELOAD) clients
+endif
+endif # !STATIC
+endif # !STANDALONE
+
+
+builddir:
+	@(if ! test -e $(BUILD_BASE); then $(MKDIR) $(BUILD_BASE); fi)
+	@(if ! test -e $(BUILD); then $(MKDIR) $(BUILD); fi)
+	@(if ! test -e $(BUILD)/$(ARCH_DIRNAME); then $(MKDIR) $(BUILD)/$(ARCH_DIRNAME); fi)
+	@(if ! test -e $(BUILD)/$(OSNAME); then $(MKDIR) $(BUILD)/$(OSNAME); fi)
+	@(if ! test -e $(BUILD)/$(LIBDIR); then $(MKDIR) $(BUILD)/$(LIBDIR); fi)
+	@(if ! test -e $(EXPORTS_BASE); then $(MKDIR) $(EXPORTS_BASE); fi)
+	@(if ! test -e $(EXPORTS); then $(MKDIR) $(EXPORTS); fi)
+
+# creates the TAGS directory under core/ for Emacs use
+ifeq ($(MACHINE), win32)
+ALL_SRCS := $(SRC)/*.[hc] $(LIB_SRC)/*.[hc] $(OS_SRC)/*.{h,c,mc,rc} $(ARCH_SRC)/*.{h,c,asm}
+else
+ALL_SRCS := $(SRC)/*.[hc] $(LIB_SRC)/*.[hc] $(OS_SRC)/*.{h,c} $(ARCH_SRC)/*.{h,c,asm}
+endif
+tags TAGS:
+	$(ETAGS) -o $(SRC)/TAGS $(ALL_SRCS)
+# -cscope in case it's not installed
+# cscope seems to be very flaky, and many queries fail, when the index is built
+# using files passed on command line: so we use cscope.files.
+	$(LS) -1 $(ALL_SRCS) > cscope.files
+	-cscope -b
+
+# blows away build, exports, and install dirs
+clear:
+	$(RM) -r $(BUILD_BASE)
+	$(RM) -r $(EXPORTS_BASE)
+	$(RM) -r $(INSTALL_LIB)
+	$(RM) -r $(INSTALL_BIN)
+	$(RM) -r $(INSTALL_INCLUDE)
+
+# removes .a,.i and .o files and generated header files
+clean:
+	$(RM) $(OBJS) $(UNIT_OBJS) $(OTHER_OBJS) $(CPPS) $(DEPS) $(EXPORTS)/*$(SO_SFX) $(BUILD)/*$(SO_SFX) $(EXPORTS)/*.{$(OBJ_SFX),h} $(BUILD)/*.{$(OBJ_SFX),h} $(BUILD)/*/*.{$(OBJ_SFX),h} $(PRELOAD_OBJS)
+	$(RM) $(BUILDMARK_H)
+ifeq ($(MACHINE), win32)
+	$(RM) $(PDBS) $(BUILD)/*.$(CPP_SFX).bak $(BUILD)/*/*.$(CPP_SFX).bak $(BUILD)/*.{cod,exp,lib,map,pdb,res,rc,bin,exe} $(BUILD)/*/*.{cod,exp,lib,map,pdb,res,rc,bin,exe}
+	$(RM) $(EXPORTS)/*.{cod,exp,lib,map,pdb,res,rc,bin,exe}
+else
+ifeq ($(LINKER_VERSION_SCRIPT), 1)
+	$(RM) $(EXPORT_LIST)
+endif
+endif
+
+exports: unexport all
+unexport:
+	$(RM) $(EXPORTS)/*.{so,a,h,dll,lib} $(BUILD)/*.{so,a,dll,lib}
+
+# removes emacs *~ files
+zap:
+	$(RM) $(SRC)/*~ $(OS_SRC)/*~ $(ARCH_SRC)/*~ $(LIB_SRC)/*~
+
+%.iso: 
+	(cd $(DYNAMO); mkisofs -o $@ -R -J $*)
+	@$(ECHO) "The source files should not be made accessible to unauthorized parties! ***"
+
+# creates html navigation files for the entire Dynamo source tree
+html:
+	@(if ! test -e $(HTML); then $(MKDIR) $(HTML); fi)
+	(cd $(HTML); $(DYNAMORIO_TOOLS)/ctoohtml $(HTML_BACK)/$(SRC)/*.[ch] $(HTML_BACK)/$(OS_SRC)/*.[ch] $(HTML_BACK)/$(ARCH_SRC)/*.[ch] $(HTML_BACK)/$(LIB_SRC)/*.[ch])
+
+# Code cleanliness craze - enforcing the rules, add suggestions to Case #133
+CHECK_SRC=*.[ch] */*.[ch]  # all by default, you can override CHECK_SRC=win32/os.c
+# The output of make ugly rule can easily be used in emacs with C-x `
+# These are as separate rules so that you can disable them easily
+# Use cvs annotate to see what you're responsible for.
+# Don't fix these if someone else maybe working on a file!
+# Note - we use "|| true" below after the grep cmds to avoid annoying make error output.
+# All ugly rules should pass, the short regression verifies this.  Rules that don't quite
+# yet pass, but almost do are in pretty_ugly. Rules that have too many false positives
+# or are too prevelant to really consider fixing are in really_ugly.
+# All rules correspond to coding style directives and rules in docs/codestandards.html
+UGLY_RULE_HEADER = Make ugly rule:
+# NOTE - runregression depends on the exact value of UGLY_RULE_HEADER
+ugly:
+# Rules that should pass.
+	@$(ECHO) "$(UGLY_RULE_HEADER) remove extraneous debugging facilities"
+	-$(GREP) -n -H "TRACELOG([^l]\|NOCHECKIN" $(CHECK_SRC) || true
+	@$(ECHO) "$(UGLY_RULE_HEADER) use SYSLOG_INTERNAL_*_ONCE instead of DO_ONCE(SYSLOG_INTERNAL"
+	-$(GREP) -n -H "DO_ONCE({\?[[:space:]]*SYSLOG_INTERNAL" $(CHECK_SRC) || true
+	@$(ECHO) "$(UGLY_RULE_HEADER) use ASSERT_CURIOSITY_ONCE instead of DO_ONCE(ASSERT"
+# note that ASSERT_NOT_TESTED is already DO_ONCE so shouldn't see DO_ONCE on any ASSERT
+	-$(GREP) -n -H "ONCE({\?[[:space:]]*ASSERT" $(CHECK_SRC) || true
+	@$(ECHO) "$(UGLY_RULE_HEADER) clean TABs w/ M-x untabify ($(CAT) -T | $(GREP) "\^I" to see)"
+	-$(GREP) -n -H "	" $(CHECK_SRC) || true
+	@$(ECHO) "$(UGLY_RULE_HEADER) minor style issues"
+# TODO - add perl rule for multiline if bodies (grep isn't up to the task)
+	@$(ECHO) "$(UGLY_RULE_HEADER) put if body on separate line"
+# since is grep, doesn't handle the if conditional taking more then one line
+	-$(GREP) -n -H "[^[:alnum:]\-_]if[[:space:]]*(.*;" $(CHECK_SRC) || true
+	@$(ECHO) "$(UGLY_RULE_HEADER) put loop body on separate line"
+# since is grep, doesn't handle the while conditional taking more then one line
+# check for while loop
+	-$(GREP) -n -H "^[^}]*[^[:alnum:]\-_]while[[:space:]]*(.*;" $(CHECK_SRC) || true
+# check that the while closing a do loop is on its own line
+	-$(GREP) -n -H "^.*[^}[:space:]]\+.*[^[:alnum:]\-_]while[[:space:]]*(.*;" $(CHECK_SRC) || true
+	-$(GREP) -n -H "[^[:alnum:]\-_]do[[:space:]]*{.*;" $(CHECK_SRC) || true
+# since is grep, doesn't handle the for pre/check/post taking more then one line
+	-$(GREP) -n -H "[^[:alnum:]\-_]for[[:space:]]*(.*;.*;.*).*;" $(CHECK_SRC) || true
+	@$(ECHO) "$(UGLY_RULE_HEADER) (1 == var) vs. (var == 1)"
+	-$(GREP) -n -H "([[:space:]]*\([[:digit:]]\+\|NULL\)[[:space:]]*==[[:space:]]*[^[:digit:]]" $(CHECK_SRC) || true
+# check that empty param list is declared void
+# FIXME: should we only require for decls by only running on header files?
+# But we do want to catch static decls.
+	-$(GREP) -n -H "^[[:alnum:]][[:alnum:]_\* ]\+();" $(CHECK_SRC) || true
+
+pretty_ugly: ugly
+# Rules that should pass (and almost do, all have only a few violations left and
+# no false positives). FIXME - should be cleaned up and moved to ugly xref 8806.
+	@$(ECHO) "$(UGLY_RULE_HEADER) minor style issues"
+	@$(ECHO) "$(UGLY_RULE_HEADER) spacing, if( or if  ( vs if ("
+	-$(GREP) -n -H "[^[:alnum:]\-_]\(do\|if\|while\|for\|else\|switch\)\([[:space:]][[:space:]]\+\)\?[({]" $(CHECK_SRC) || true
+	@$(ECHO) "$(UGLY_RULE_HEADER) spacing, }else or }  else vs } else"
+	-$(GREP) -n -H "[});]\([[:space:]][[:space:]]\+\)\?\(do\|if\|while\|for\|else\|switch\)\([^[:alnum:]\-_]\|$$\)" $(CHECK_SRC) || true
+	@$(ECHO) "$(UGLY_RULE_HEADER) empty param list should be (void) not ()"
+
+really_ugly: pretty_ugly
+# Rules that don't pass at all, either because of excessive violations in the
+# codebase or because of false positives. FIXME - for rules with few or no false
+# positives but numerous violations we could use a count or something to try and
+# keep things from getting worse xref case 8806.
+	@$(ECHO) "$(UGLY_RULE_HEADER) long lines"
+# no false positives, but numerous violations
+	-$(GREP) -n -H ".\{90,\}" $(CHECK_SRC) || true
+	@$(ECHO) "$UGLY_RULE_HEADER) one statement per line"
+# With all the exemptions very few false positives, but still many violations
+# (and prob. some false negatives). The first grep looks for lines with two ; not
+# within quotes. 2nd, 3rd and 4th greps subtract out lines where the extra
+# semicolon is prob. part of a comment. NOTE, for some reason [[:space:]] doesn't
+# work for matching before the * in the 3rd grep. 5th grep subtracts out for (;;)
+# loop usage and 6th grep subtracts out the very common 'case: foo; break;' motif
+# (even though it technically breaks the rule). Final grep subtracts out certain
+# common macros whose usage often breaks the rule.
+	-$(GREP) -n -H -E '^(([^;"]|("([^"]|\\")*"))*;){2,}' $(CHECK_SRC) |\
+	 $(GREP) -vE "^[^;]*;?[^;]*/\*" | $(GREP) -v "^[^;\*=\(]*\*[[:space:]]" |\
+	 $(GREP) -vE  "^[^;]*;?[^;]*//" | $(GREP) -v "for" | $(GREP) -v "break;[^;]*$$" |\
+	 $(GREP) -vE "DODEBUG|DOLOG|DOSTATS|DO_VERBOSE" || true
+	@$(ECHO) "$(UGLY_RULE_HEADER) instr_t* foo vs. instr_t *foo, etc."
+# only a few false positives, but numerous violations
+	-$(GREP) -n -H "[[:alnum:]]\*[^[:alnum:]/\)\(]" $(CHECK_SRC) || true 
+	@$(ECHO) "$(UGLY_RULE_HEADER) use C style comments"
+# only a few false positives, but numerous violations (even allowing for usage
+# inside /* */ comments)
+	-$(GREP) -n -H "//" $(CHECK_SRC) || true
+	@$(ECHO) "$(UGLY_RULE_HEADER) clean up commented out code"
+# mostly false positives (we're too grammatical ;) due to using ; in comments
+	-$(GREP) -n -H ".*//.*;\|.*/\*.*;.*\*/" $(CHECK_SRC) || true
+	@$(ECHO) "$(UGLY_RULE_HEADER) use int/uint instead of long/ulong"
+# some false positives and numerous violations
+	-$(GREP) -n -H "^[^\*]*long" $(CHECK_SRC) || true
+	@$(ECHO) "$(UGLY_RULE_HEADER) spacing, ( foo, bar ) vs. (foo, bar)"
+# Only a few false positives and not that many violations (for  the "( " rule at
+# least). FIXME - these could probably be moved to pretty_ugly at some point.
+	-$(GREP) -n -H "([[:space:]]\+[^\\[:space:]]\+" $(CHECK_SRC) || true
+	-$(GREP) -n -H "[[:space:]])" $(CHECK_SRC) | $(GREP) -v -e "DEBUG_DECLARE" || true
+	@$(ECHO) "$(UGLY_RULE_HEADER) put spaces after commas in arg lists etc."
+# no real false positives, but tons of violations (even ignoring asm listings)
+	-$(GREP) -n -H ",[^[:space:]]" $(CHECK_SRC) || true
+
+$(BUILDMARK_H): 
+	@$(ECHO) -e "#define DYNAMORIO_DEFINES \""$(DEFINES)"\"\n"\
+	"extern const char dynamorio_version_string[];\n"\
+	"extern const char dynamorio_buildmark[];" > $@
+
+# we ignore whitespace since from within emacs the "extern" lines have an initial space.
+# note that "-b" is not sufficient since it complains about no whitespace versus some
+# whitespace: it only suppresses changes in numbers >= 1.
+DIFF_DEFINES := $(DIFF) --ignore-all-space
+compare-defines: $(BUILDMARK_H)
+ifeq ($(IGNOREDEF), y)
+	@$(DIFF_DEFINES) $^ || $(ECHO) Inconsistency ignored
+#still keep POST_DEFINES_H as a mark
+else
+	! test -e $< || $(DIFF_DEFINES) $^ || ($(ECHO) Makefile.mydefines:1:Possibly inconsistent build. Run make clean all; exit 1)
+	$(RM) $(POST_DEFINES_H)
+endif
+
+clean-mark:
+	$(RM) $(BUILDMARK_OBJS)
+
+buildmark: compare-defines clean-mark $(BUILDMARK_OBJS)
+
+# display the build number
+buildnum:
+	@$(ECHO) build number is ${BUILD_NUMBER}
+
+# will always compare defines with current and then recreate buildmark.o 
+
+# FIXME: PR 214218 - For the current cygwin version, 'make clear'
+# fails to remove the build directory.  The problem stems from
+# implicit rules of the form:
+# 
+#  $(BUILD)/%.$(OBJ_SFX): $(SRC)/%.c
+#
+# These rules cause make to hold an open handle to the directory
+# $(BUILD).  To avoid this problem, the best solution is probably to
+# use static patterns:
+#
+#  $(OBJS): $(BUILD)/%.$(OBJ_SFX): $(SRC)/%.c
+#
+# This will require extensive changes, however.  An easier solution is
+# to ifdef out all rules referencing the $(BUILD) dir for 'make
+# clear'.  The disadvantage of this approach is that commands like
+# 'make clear all' don't work as desired.
+ifneq ($(MAKECMDGOALS),clear) 
+
+###########################################################################
+##
+## Non-phony rules
+
+$(MESSAGE_STRINGS_H) : $(MESSAGE_STRINGS_FILES) gen_event_strings.pl
+	$(PERL) gen_event_strings.pl $(MESSAGE_STRINGS_FILES) $(MESSAGE_STRINGS_H)
+
+ifneq ($(STATIC),1) ########################################
+
+# injector
+ifeq ($(MACHINE), win32)
+injector: builddir $(EXPORTS)/$(INJECTOR) $(EXPORTS)/$(ARCHIVE)
+$(EXPORTS)/$(INJECTOR): $(BUILD)/$(INJECTOR)
+	$(CP) $(BUILD)/$(INJECTNAME).{exe,map} $(EXPORTS)
+ifneq ($(DBG),) # sometimes can't build pdb (case 192750)
+	$(CP) $(BUILD)/$(INJECTNAME).pdb $(EXPORTS)
+endif
+$(BUILD)/$(INJECTOR): $(INJECT_OBJS)
+	$(LD) $(INJECTFLAGS) $(INJECT_OBJS) $(INJECTLIBS)
+
+early_inject_helper: builddir $(EXPORTS)/$(EARLY_INJECT_HELPER1) $(EXPORTS)/$(EARLY_INJECT_HELPER2)
+$(EXPORTS)/$(EARLY_INJECT_HELPER1): $(BUILD)/$(EARLY_INJECT_HELPER1)
+	$(CP) $(BUILD)/$(EARLY_INJECT_HELPER1_NAME).{dll,map} $(EXPORTS)
+ifneq ($(DBG),) # sometimes can't build pdb (case 192750)
+	$(CP) $(BUILD)/$(EARLY_INJECT_HELPER1_NAME).pdb $(EXPORTS)
+endif
+$(EXPORTS)/$(EARLY_INJECT_HELPER2): $(BUILD)/$(EARLY_INJECT_HELPER2)
+	$(CP) $(BUILD)/$(EARLY_INJECT_HELPER2_NAME).{dll,map,lib} $(EXPORTS)
+ifneq ($(DBG),) # sometimes can't build pdb (case 192750)
+	$(CP) $(BUILD)/$(EARLY_INJECT_HELPER2_NAME).pdb $(EXPORTS)
+endif
+$(BUILD)/$(EARLY_INJECT_HELPER1): $(EARLY_INJECT_HELPER1_OBJS) $(BUILD)/$(EARLY_INJECT_HELPER2)
+	$(LD) $(EARLY_INJECT_HELPER1_FLAGS) $(EARLY_INJECT_HELPER1_OBJS) $(EARLY_INJECT_HELPER1_LIBS)
+$(BUILD)/$(EARLY_INJECT_HELPER2): $(EARLY_INJECT_HELPER2_OBJS)
+	$(LD) $(EARLY_INJECT_HELPER2_FLAGS) $(EARLY_INJECT_HELPER2_OBJS) $(EARLY_INJECT_HELPER2_LIBS)
+
+
+# pre-injector: separate library + copy of dynamo library in special place
+preload: builddir $(INSTALL_LIB_BASE)/$(PRELOAD) $(EXPORTS)/$(ARCHIVE)
+	@$(ECHO) "installing $(ARCHIVE) for systemwide injection to $(SYSWIDE_DIR)"
+	@(if ! test -e $(SYSWIDE_DIR); then $(MKDIR) $(SYSWIDE_DIR); fi)
+	$(CP) $(EXPORTS)/$(ARCHIVE) $(SYSWIDE_DIR)/$(ARCHIVE)
+$(INSTALL_LIB_BASE)/$(PRELOAD): $(PRE_INJECT_OBJS)
+	@(if ! test -e $(INSTALL_LIB_BASE); then $(MKDIR) $(INSTALL_LIB_BASE); fi)
+	$(LD) $(PRE_INJECTFLAGS) $(PRE_INJECT_OBJS) $(PRE_INJECTLIBS)
+	$(CP) $(BUILD)/$(PREINJECTNAME).{dll,map} $(INSTALL_LIB_BASE)
+	$(CP) $(BUILD)/$(PREINJECTNAME).{dll,map} $(EXPORTS)
+ifneq ($(DBG),) # sometimes can't build pdb (case 192750)
+	$(CP) $(BUILD)/$(PREINJECTNAME).pdb $(INSTALL_LIB_BASE)
+	$(CP) $(BUILD)/$(PREINJECTNAME).pdb $(EXPORTS) 
+endif
+
+# TODO: use a RLL library instead of putting everything in one dll
+$(BUILD)/%.rll : $(BUILD)/$(OSNAME)/%.res
+	$(LD) -dll -noentry -out:$(subst /,\\,$@) $<
+endif
+
+ifeq ($(MACHINE), linux)
+LDDYNAMO = -L$(EXPORTS) -ldynamorio
+
+preload: builddir $(EXPORTS)/$(PRELOAD) $(EXPORTS)/$(ARCHIVE)
+$(EXPORTS)/$(PRELOAD): $(BUILD)/$(PRELOAD)
+# PR 212290: ensure no text relocations (they violate selinux execmod policies)
+	$(READELF) -d $(BUILD)/$(PRELOAD) | $(GREP) -q TEXTREL; \
+	if [ $$? == 0 ]; then \
+	    $(ECHO) "$(BUILD)/$(PRELOAD) has text relocations" && false; \
+	fi
+# also check for execstack
+	$(READELF) -l $(BUILD)/$(PRELOAD) | $(GREP) STACK | $(GREP) -q RWE; \
+	if [ $$? == 0 ]; then \
+	    $(ECHO) "$(BUILD)/$(PRELOAD) has executable stack" && false; \
+	fi
+	$(CP) -a $(BUILD)/$(PRELOAD) $(EXPORTS)
+	@(if ! test -e $(INSTALL_LIB_BASE); then $(MKDIR) $(INSTALL_LIB_BASE); fi)
+	$(CP) -a $(BUILD)/$(PRELOAD) $(INSTALL_LIB_BASE)/$(PRELOAD)
+
+$(BUILD)/$(PRELOAD): $(PRELOAD_OBJS)
+# FIXME case 69/1891: -z initfirst = initialize first at runtime (before libc)
+#	$(CC) -shared -nostartfiles -Xlinker -z -Xlinker initfirst $< $(CC_LDLIBS) $(LDDYNAMO) -o $@
+	$(CC) $(CFLAGS) -shared -nostartfiles $< $(CC_LDLIBS) $(LDDYNAMO) -o $@
+endif
+
+endif # ifneq ($(STATIC),1) ########################################
+
+ifeq ($(MACHINE), win32)
+TYPE_EXE = VFT_APP
+TYPE_DLL = VFT_DLL
+BUILD_RESOURCE_COMMON = $(RC) -r -fo $(subst /,\\,$@) $(INCLUDE_PATHS) $(GENERATED_INCLUDES) $(DEFINES) $(RESOURCE_DEFINES)
+
+# make resources - consumes both a resource and the associated BIN file
+$(BUILD)/dynamorio.res : $(SRC)/$(OSNAME)/resources.rc $(BUILD)/$(OSNAME)/events.rc
+	 $(BUILD_RESOURCE_COMMON) $(D)INCLUDE_EVENTS $(D)FILE_NAME=\"$(ARCHIVE)\" $(D)FILE_TYPE=$(TYPE_DLL) $(D)FILE_DESCRIPTION="PRODUCT_NAME \" core library\"" $<
+
+$(BUILD)/injector.res : $(SRC)/$(OSNAME)/resources.rc
+	$(BUILD_RESOURCE_COMMON) $(D)FILE_NAME=\"$(INJECTOR)\" $(D)FILE_TYPE=$(TYPE_EXE) $(D)FILE_DESCRIPTION="PRODUCT_NAME \" injection utility\"" $<
+
+$(BUILD)/preinjector.res : $(SRC)/$(OSNAME)/resources.rc
+	$(BUILD_RESOURCE_COMMON) $(D)FILE_NAME=\"$(PRELOAD)\" $(D)FILE_TYPE=$(TYPE_DLL) $(D)FILE_DESCRIPTION="PRODUCT_NAME \" systemwide injector dll\"" $<
+
+$(BUILD)/drearlyhelp1.res : $(SRC)/$(OSNAME)/resources.rc
+	$(BUILD_RESOURCE_COMMON) $(D)FILE_NAME=\"$(EARLY_INJECT_HELPER1)\" $(D)FILE_TYPE=$(TYPE_DLL) $(D)FILE_DESCRIPTION="PRODUCT_NAME \" additional win2k injector dll\"" $<
+
+$(BUILD)/drearlyhelp2.res : $(SRC)/$(OSNAME)/resources.rc
+	$(BUILD_RESOURCE_COMMON) $(D)FILE_NAME=\"$(EARLY_INJECT_HELPER2)\" $(D)FILE_TYPE=$(TYPE_DLL) $(D)FILE_DESCRIPTION="PRODUCT_NAME \" additional win2k injector dll\"" $<
+
+# start from message files
+# NOTE: a BIN file is also generated and pointed to by the .rc  $(BUILD)/$(OSNAME)/%.BIN
+$(BUILD)/$(OSNAME)/%.rc $(BUILD)/$(OSNAME)/%.h: $(OSNAME)/%.mc
+	$(MC) -h $(subst /,\\,$(BUILD)/$(OSNAME)) -r $(subst /,\\,$(BUILD)/$(OSNAME)) $<
+endif # ifeq  ($(MACHINE), win32)
+
+ifeq ($(STATIC),1)
+install: all
+else
+# FIXME - should install also copy early inject helper dlls to system32?
+install: all preload
+endif
+ifeq ($(MACHINE), linux)
+	@(if ! test -e $(INSTALL_LIB_BASE); then $(MKDIR) $(INSTALL_LIB_BASE); fi)
+	@(if ! test -e $(SYSWIDE_DIR); then $(MKDIR) $(SYSWIDE_DIR); fi)
+else
+# preload builds install base dir for us
+endif
+	@(if ! test -e $(INSTALL_LIB); then $(MKDIR) $(INSTALL_LIB); fi)
+	@(if ! test -e $(INSTALL_BIN); then $(MKDIR) $(INSTALL_BIN); fi)
+	@(if ! test -e $(INSTALL_INCLUDE); then $(MKDIR) $(INSTALL_INCLUDE); fi)
+	$(CP) $(EXPORTS)/$(ARCHIVE) $(INSTALL_LIB)
+# copy exported .h files if they exist
+	@(junk=`$(LS) $(EXPORTS)/*.h 2>&1`; if [ $$? == 0 ] ; then \
+		$(CP) $(EXPORTS)/*.h $(INSTALL_INCLUDE); fi)
+ifeq ($(MACHINE), win32)
+# get the .lib file too for linking (_app interface)
+	$(CP) $(EXPORTS)/$(subst dll,lib,$(ARCHIVE)) $(INSTALL_LIB)
+ifeq ($(STATIC), 0)
+	$(CP) $(EXPORTS)/$(INJECTOR) $(INSTALL_BIN)
+endif
+endif
+
+# lib helper: fortran
+$(EXPORTS)/$(FORTRAN_OBJ): $(BUILD)/$(FORTRAN_OBJ)
+	$(CP) $(BUILD)/$(FORTRAN_OBJ) $(EXPORTS)
+
+ifeq ($(MACHINE), linux)
+ifeq ($(LINKER_VERSION_SCRIPT), 1)
+# generate linker script to limit exports
+$(BUILD)/$(ARCHIVE): $(EXPORT_LIST)
+$(EXPORT_LIST): $(GENAPI)
+	$(PERL) $(GENAPI) -filter "$(DEFINES)" > $@
+endif
+endif
+
+# dynamo library itself
+$(EXPORTS)/$(ARCHIVE): $(BUILD)/$(ARCHIVE) $(GENAPI)
+ifeq ($(MACHINE), linux)
+# PR 212290: ensure no text relocations (they violate selinux execmod policies)
+	$(READELF) -d $(BUILD)/$(ARCHIVE) | $(GREP) -q TEXTREL; \
+	if [ $$? == 0 ]; then \
+	    $(ECHO) "$(BUILD)/$(ARCHIVE) has text relocations" && false; \
+	fi
+# also check for execstack
+	$(READELF) -l $(BUILD)/$(ARCHIVE) | $(GREP) STACK | $(GREP) -q RWE; \
+	if [ $$? == 0 ]; then \
+	    $(ECHO) "$(BUILD)/$(ARCHIVE) has executable stack" && false; \
+	fi
+	$(CP) -a $(BUILD)/$(ARCHIVE) $(EXPORTS)
+	$(SIZE) $(EXPORTS)/$(ARCHIVE)
+else
+	$(CP) $(BUILD)/$(OUTNAME).{dll,lib,map} $(EXPORTS)
+ifneq ($(DBG),) # sometimes can't build pdb (case 192750)
+	$(CP) $(BUILD)/$(OUTNAME).pdb $(EXPORTS)
+endif
+	$(DUMPBIN) /summary $(EXPORTS)/$(ARCHIVE)
+endif # WINDOWS
+
+ifeq ($(CLIENT_INTERFACE), 1)
+api_headers: $(GENAPI) $(HEADERS) builddir
+# create header files to complete IR API
+	$(RM) $(EXPORTS)/dr_*.h
+	$(CP) $(LIB_SRC)/dr_api.h $(EXPORTS)
+ifeq ($(MACHINE), linux)
+	$(PERL) $(GENAPI) -header $(EXPORTS) "$(API_DEFINES)"
+else
+	$(PERL) $(GENAPI) -header $(EXPORTS) "$(API_DEFINES)"
+endif
+# kill #ifdefs from func declarations in dr_app.h,
+# but not from #defines at top (WINDOWS, etc.)
+# FIXME: make this simpler!  do we really have to write another script?
+# there must be a better way to selectively strip defines from our header files
+	export line=`$(GREP) -n '/\*\*\*\*' $(LIB_SRC)/dr_app.h | $(CUT) -d : -f 1 | $(AWK) '{print $$1 - 1}'`; \
+	$(HEAD) -$$line $(LIB_SRC)/dr_app.h > $(EXPORTS)/dr_app.h; \
+	export rest=`$(WC) $(LIB_SRC)/dr_app.h $(EXPORTS)/dr_app.h | $(XARGS) $(ECHO) | $(AWK) '{print $$1 - $$5}'`; \
+	$(TAIL) -$$rest $(LIB_SRC)/dr_app.h | $(SED) 's/#endif \/\* _DR_APP_H_ \*\///' > ._foo; \
+	$(CPP) $(CPP_KEEP_COMMENTS) $(CPP_NO_LINENUM) $(subst /,-,$(DEFINES)) ._foo >> $(EXPORTS)/dr_app.h; \
+	$(TAIL) -2 $(LIB_SRC)/dr_app.h >> $(EXPORTS)/dr_app.h; \
+	$(SED) -i 's/\r//' $(EXPORTS)/dr_app.h
+	$(RM) ._foo
+# FIXME - would be better if this was just not generated at all when !HOT_PATCING_INTERFACE.
+# Need to add appropriate ifdefs for the core, for now this is the easiest path.
+ifneq ($(HOT_PATCHING_INTERFACE), 1)
+	$(RM) $(EXPORTS)/dr_probe.h
+endif
+else
+ifeq ($(APP_EXPORTS), 1)
+api_headers: $(LIB_SRC)/dr_app.h builddir
+	$(RM) $(EXPORTS)/dr_*.h
+	$(CP) $(LIB_SRC)/dr_app.h $(EXPORTS)
+else
+api_headers:
+endif
+endif
+
+dynamorio: $(DEFINES_H) compare-defines usedep_defines $(EXPORTS)/$(ARCHIVE) api_headers
+
+standalone: $(DEFINES_H) compare-defines usedep_defines $(BUILD)/$(STANDALONE_EXE)
+
+usedep_defines:
+# HACK: 1st clean, while doing deps, decides DEFINES_H does not need to be remade,
+# then does the clean, and then if another target is specified, DEFINES_H is
+# already on prune list and we end up w/ a compilation error since it's missing...
+	(if ! test -e $(DEFINES_H); then $(MAKE) builddir $(DEFINES_H); fi)
+
+ifeq ($(ARCH), x64) # see notes below on PR 253624
+SET_PREFERRED_BASE := 1
+PREFERRED_BASE := 0x71000000
+LINKER_DEFAULT_BASE:=0x400000
+else
+  ifeq ($(VMKERNEL), 1)
+    # we end up with the default executable base (0x08*) so go back to 0 base
+    # (else we fail to load on esx)
+    SET_PREFERRED_BASE := 1
+    PREFERRED_BASE := 0x00000000
+  else
+    # off by default, but can be specified
+    SET_PREFERRED_BASE := 0
+  endif
+LINKER_DEFAULT_BASE:=0x08048000
+endif
+
+# we always use a script for our own library bounds (PR 361594)
+LD_USE_SCRIPT := -Xlinker -dT -Xlinker $(LD_SCRIPT)
+
+$(BUILD)/$(ARCHIVE): $(MESSAGE_STRINGS_H) $(MESSAGE_H) $(OBJS)
+ifeq ($(MACHINE), linux)
+ifeq ($(STATIC), 1)
+	ar r $@ $(OBJS) 
+else
+#	$(LD) -shared -Bsymbolic $(OBJS) $(LD_LDLIBS) -o $@ /usr/lib/gcc-lib/i386-redhat-linux/egcs-2.91.66/libgcc.a
+#	$(CC) -shared -nostartfiles -Xlinker -Bsymbolic -Xlinker -z -Xlinker now $(OBJS) $(CC_LDLIBS) -o $@
+#	$(CC) -nostdlib -shared -nostartfiles -Xlinker -Bsymbolic -Xlinker -rpath -Xlinker $(DYNAMORIO_HOME)/${CORE_DIRNAME}-signals/lib $(OBJS) $(CC_LDLIBS)  -L$(LIB_SRC) -lourlibc -o $@
+# -z now = do non-lazy runtime binding
+	$(MAKE) buildmark
+# PR 361954: set up vars pointing at our own library bounds using the
+# symbols __executable_start and end defined in default script
+	$(LD) --verbose | $(SED) '/SECTIONS/,/^}/!d' | \
+	  $(SED) 's/^}/PROVIDE(dynamorio_so_start = __executable_start); PROVIDE(dynamorio_so_end = end);\n}/;' \
+	  > $(LD_SCRIPT)
+ifeq ($(SET_PREFERRED_BASE), 1)
+# PR 253624: right now our library has to be next to our heap, and our
+# heap needs to be in lower 4GB for our fallback TLS (PR 285410), so we
+# set a preferred base address via a linker script.
+# In order to just tweak the default linker script we start with exactly that.
+	$(SED) -i 's/$(LINKER_DEFAULT_BASE)/$(PREFERRED_BASE)/g; s/^{/{\n . = 0x71000000;/;' $(LD_SCRIPT)
+endif
+	$(CC) $(CFLAGS) -shared -nostartfiles -Xlinker -z -Xlinker now -Xlinker -Bsymbolic $(LINKER_EXPORT_FLAGS) $(OBJS) $(CC_LDLIBS) -o $@ $(LD_USE_SCRIPT)
+endif
+else # win32
+	$(CP) $(DEF_SRC) $(DEF_BLD)
+ifeq ($(CLIENT_INTERFACE), 0)
+# we're not exporting snprintf() so remove from def file */
+	$(SED) -i '/ snprintf/d' $(DEF_BLD)
+endif
+	$(LD) $(LDFLAGS) $(OBJS) $(LD_LDLIBS)
+endif
+
+$(BUILD)/$(STANDALONE_EXE): $(MESSAGE_STRINGS_H) $(MESSAGE_H) $(UNIT_OBJS)
+ifeq ($(MACHINE), linux)
+	$(MAKE) buildmark
+	$(CC) $(UNIT_OBJS) $(CC_LDLIBS) -o $@
+else # win32
+	$(LD) $(TGTFLAGS) $(UNIT_OBJS) $(LD_LDLIBS) /out:$@ \
+		/pdb:"$(MSBUILD)\\$(STANDALONE_NAME).pdb"
+endif
+# FIXME: copy to exports or anything?
+
+# .c -> .o
+$(BUILD)/%.$(OBJ_SFX): $(SRC)/%.c
+ifeq ($(MACHINE), win32)
+	$(CC) $(CFLAGS) $(INCLUDE_PATHS) $(GENERATED_INCLUDES) $(DEFINES) $(INCR) /TC $(subst /,\\,$<) $(OUT)$(subst /,\\,$@) /Fd$(subst /,\\,$(subst .$(OBJ_SFX),.pdb,$@)) $(FILTER_CC)
+else # linux
+	$(CC) $(CFLAGS) $(INCLUDE_PATHS) $(GENERATED_INCLUDES) $(DEFINES) $(INCR) $< $(OUT)$@
+endif
+
+# non-core build of a core file
+# FIXME: this is a hack -- ugly duplication of above rule -- but
+# no simpler alternatives (could create ntdll-noncore.c that defines
+# NOT_DYNAMORIO_CORE_PROPER, or a separate make:
+# make NONCORE=1 BUILD=noncore ADD_DEFINES=NOT_DYNAMORIO_CORE_PROPER)
+#
+# -noncore.c -> -noncore.o
+# FIXME: This rule should use gcc on both Windows and Linux!
+$(BUILD)/%-noncore.$(OBJ_SFX): $(SRC)/%.c
+ifeq ($(MACHINE), win32)
+	$(CC) $(CFLAGS) $(INCLUDE_PATHS) $(GENERATED_INCLUDES) $(DEFINES) $(D)NOT_DYNAMORIO_CORE_PROPER $(INCR) /TC $(subst /,\\,$<) $(OUT)$(subst /,\\,$@) /Fd$(subst /,\\,$(subst .$(OBJ_SFX),.pdb,$@))
+else # linux
+	$(CC) $(CFLAGS) $(INCLUDE_PATHS) $(GENERATED_INCLUDES) $(DEFINES) $(D)NOT_DYNAMORIO_CORE_PROPER $(INCR) $< $(OUT)$@
+endif
+
+# .c -> .cod (and implicitly .$(OBJ_SFX))
+$(BUILD)/%.cod: $(BUILD)/%.c
+ifeq ($(MACHINE), win32)
+	$(CC) $(CFLAGS) $(INCR) /TC $(subst /,\\,$<) $(OUT)$(subst /,\\,$(subst .cod,.$(OBJ_SFX),$@)) /Fd$(subst /,\\,$(subst .cod,.pdb,$@)) /FAcs /Fa$(subst /,\\,$@)
+endif
+
+## We use .asm as a platform-independent assembly file (for cygwin and linux),
+## and generate a .s file in the build directory from the .asm file:
+
+# .asm -> .s
+# we want ".ifdef FOO" to not turn into ".ifdef 1" so we say "-DFOO=FOO":
+ASM_CPP_DEFINES := $(join $(DEFINES),$(patsubst $(D)%,=%,$(DEFINES)))
+$(BUILD)/%.s: $(SRC)/%.asm
+	$(CPP) $(PREPROC) $(ASMPREPROC) $(CPPFLAGS) $(ASM_CPP_DEFINES) $< $(CPPOUT)$@
+# since we don't want to duplicate gas and ml macros we use cpp macros w/
+# the added feature of insertable newlines, needed for asm code, via "@N@":
+	$(SED) -i 's/\@N\@/\n/g' $@
+
+$(BUILD)/%-noncore.s: $(SRC)/%.asm
+	$(CPP) $(PREPROC) $(ASMPREPROC) $(CPPFLAGS) $(ASM_CPP_DEFINES) $(ASM_NDCP_DEFINES) $< $(CPPOUT)$@
+# we can't put in newlines with cpp so we have sed do it
+	$(SED) -i 's/\@N\@/\n/g' $@
+
+# .s -> .o
+$(BUILD)/%.$(OBJ_SFX): $(BUILD)/%.s
+# ml and ml64 ignore /Fo unless it is BEFORE the source file
+	$(AS) $(ASMFLAGS) $(OUT)$@ $<
+
+# .c -> .d
+# see info make Prerequisites
+# cannot depend on builddir, else make will keep rebuilding .dep files, since
+# every time it reads Makefile it will think they need to be remade b/c of builddir
+# always being out-of-date, so instead we test for presence inside the command
+$(BUILD)/%.$(DEP_SFX): $(SRC)/%.c
+	@(if test -e $(BUILD); then $(GCC) $(DEPENDS) $(CPPFLAGS) $(INCLUDE_PATHS) $(GENERATED_INCLUDES) \
+                $(DEFINES) -MQ $(patsubst %.$(DEP_SFX),%.$(OBJ_SFX), $@) -MQ $@ $(SRC)/$*.c > $@ ; fi)
+
+depend: builddir $(DEPS)
+	@$(ECHO) Generated dependency files for make USEDEP=1
+	@$(ECHO) If new files are added use make clean depend.
+
+endif # ifneq ($(MAKECMDGOALS),clear)
+
+# include dependencies
+ifeq ($(USEDEP), 1)
+-include $(DEPS)
+endif
+
+# External modules go here, and will be build if present
+ifeq ($(STRACE_CLIENT), 1)
+CLIENTS += ../clients/stracedr
+endif
+
+.PHONY:: $(CLIENTS)
+
+clients: $(CLIENTS)
+
+$(CLIENTS):
+	@(if test -e $(CLIENTS); then $(MAKE) -C $@ all; fi)
+
+###########################################################################
+# non build related targets follow
+
+###########################################################################
+# this target should be able to run all unit tests
+# currently requires make clean unitall if the core was previously built
+allunit: 
+	-$(MAKE) UNIT_TEST=rct NOLIBC=0 STANDALONE_UNIT_TEST=1 ADD_DEFINES=$(D)RCT_IND_BRANCH UNIT_ADD_OPTIONS='-rct_ind_call 11 -rct_ind_jump 11 -no_diagnostics' unitrun
+	$(MAKE) clean
+	-$(MAKE) UNIT_TEST=options NOLIBC=0 STANDALONE_UNIT_TEST=1 ADD_DEFINES=$(D)OPTIONS_UNIT_TEST unitrun
+	$(MAKE) clean
+	-$(MAKE) UNIT_TEST=vmareas NOLIBC=0 STANDALONE_UNIT_TEST=1 ADD_DEFINES=$(D)VMAREAS_UNIT_TEST unitrun
+
+###########################################################################
+# run a target program with the current library
+ifeq ($(MACHINE), win32)
+# .exe added automatically
+TESTPROG=notepad
+progrun: all
+	${EXPORTS}/${INJECTOR} ${EXPORTS}/${ARCHIVE} ${TESTPROG}${EXE_SFX} ${TESTARGS}
+else
+TESTPROG=ls
+progrun: all
+	export LD_LIBRARY_PATH=${CUR_FULL}/${CORE_DIRNAME}/${EXPORTS}/:$$LD_LIBRARY_PATH \
+	LD_PRELOAD="$(PRELOAD) $(ARCHIVE)"; ${TESTPROG}${EXE_SFX} ${TESTARGS}
+endif
+
+## prefast
+# used only when building dynamorio.sys
+PREFAST_STACKHOG_THRESHOLD=512
+PREFAST_LOG=${BUILD}/prefast.log
+prefast: builddir
+	-prefast /reset /StackHogThreshold=${PREFAST_STACKHOG_THRESHOLD} /log=${PREFAST_LOG} $(MAKE) -k 
+
+prefastlist: prefast
+	prefast /log=${PREFAST_LOG} list | $(TEE) ${BUILD}/prefast.err
+
+prefastview: # shows last results
+	prefast /log=${PREFAST_LOG} view
+
+
+###########################################################################
+# Support for adding symbols to a symbol server
+# you can do this only on Windows
+ifeq ($(MACHINE), win32)
+
+# FIXME: if have never connected this will fail
+# how specify username of guest?
+# NOTE: could add a "logon to symbol server" target which executes:
+#  cmd /c net use \\\\deter-build.determina.com\\symbols "" /user:guest
+SYMBOL_SERVER=\\\\10.1.5.15\\symbols
+
+
+# vlk: I also like my VMs to have host only network so then I do
+#   subst y: z:\shared\symbols   # which would be host shared with /mnt/fileserver/symbols if that were writable
+#   SYMBOL_SERVER=y:\\
+
+SYMBOL_STORE=${SYMBOL_SERVER}\\product
+SYMBOL_ADMIN=$(SYMBOL_STORE)\\000Admin
+
+ifndef BUILD_NUMBER
+  BUILD_NUMBER=custom
+endif
+PRODUCT_NAME="DynamoRIO"
+DATE='$(shell $(DATE) +"Built at %Y-%m-%d %T")'
+COMMENT="Build ${BUILD_NUMBER} by $(USER)@$(COMPUTERNAME)"
+
+symbols:
+	@$(ECHO) "Connecting to $(SYMBOL_SERVER)"; \
+	export sa=$(SYMBOL_ADMIN) ss=$(SYMBOL_STORE); \
+	$(DYNAMORIO_TOOLS)/external/symstore add /r /f ${EXPORTS} \
+	/s ${SYMBOL_STORE} /t ${PRODUCT_NAME} /v ${COMMENT} /c ${DATE} | $(PERL) -e '\
+	$$sa=$$ENV{"sa"}; $$ss=$$ENV{"ss"}; \
+	while (<>) { if (/^Finding ID...\s+(\d+)/) {$$id=$$1;} print $$_; } \
+	open (SA, "< $$sa\\$$id"); print "\nFiles added:\n"; \
+	while (<SA>) { print $$_; if (/(dynamorio.dll.[0-9A-H]+)/) {$$file=$$1;} } close(SA); \
+	if (defined($$file)) { print "\nDR dll: $$ss\\$$file\\dynamorio.dll\n"; } \
+	else { print "\nWarning: no dynamorio.dll seems to have been added!\n"; }'
+
+endif # win32
+
+###########################################################################
+# diff automation
+#
+# usage: prepare a notes file with any comments for the email after a line
+# beginning "toemail", and point to it w/ NOTES=
+#
+# example: make REVIEWER=derek DIFFNUM=2 NOTES=msg diffsend
+# to give better diff base name use CUR_TREE=
+
+DIFF_DIR=..
+# name is picked by current tree name as set above
+
+# default for notes location
+NOTES=$(DIFF_DIR)/diff.notes
+
+DIFFNUM=1
+# specify this if want to add case number or something
+DIFF_PREFIX=
+DIFF_NAME=$(DIFF_PREFIX)$(CUR_TREE)-$(DIFFNUM)
+DIFF_DIFF=$(DIFF_DIR)/$(DIFF_NAME).diff
+DIFF_NOTES=$(DIFF_DIR)/$(DIFF_NAME).notes
+DIFFEMAIL=$(DIFF_DIR)/$(DIFF_NAME).email
+# will have machine name appended to it:
+REGRESSION=$(DIFF_DIR)/$(DIFF_NAME).regr
+
+# obtain remote host from $CVSROOT
+# we don't want to hardcode the cvsroot path so we instead
+# remove the trailing /... in $CVSROOT
+REVIEW_DIR=~/review
+# there's no clean way to do this w/o invoking shell: we split off abs path
+# (yes we assume it's an abs path) from machine: (if there is one), kill the
+# whole abs path, re-add the : (if needed), and then append the review path.
+
+ifeq ($(USECVS), 1)
+REMOTE_DIR=$(patsubst %,%:,$(patsubst /%,,$(subst :, ,$(subst :ext:,,$(CVSROOT)))))$(REVIEW_DIR)
+REMOTE_HOST=$(subst :,,$(subst $(REVIEW_DIR),,$(REMOTE_DIR)))
+else
+# without CVS this is used for storing review comments AS well as a sendmail proxy
+REVIEW_HOST=determina-cvs.eng.vmware.com
+# FQDN for VPN users, no localhost users expected
+REMOTE_HOST=${USER}@$(REVIEW_HOST)
+
+# used for reviews, currently target of scp, FIXME: should change to point to local mount of home share
+REMOTE_DIR=$(REMOTE_HOST):$(REVIEW_DIR)
+endif
+
+REVIEWER=
+REVIEWER_MAIL=$(REVIEWER)@vmware.com
+# only allow reviewers in the dev list
+REVIEWER_CHECK=$(filter ${REVIEWER},${DEVELOPERS})
+CARBON_COPY=dynamorio@vmware.com
+
+# cvs diff -c -p -N
+# FIXME: p4 DIFF doesn't show new files (-N) and doesn't support function info (-p)
+# We get around this by using P4DIFF env. variable to tell p4 to use plain diff.
+# For p4 built-in diff
+#   -dc -dp or -dcp => diff -c (eqv.)
+#   -dp -dc or -dpc => diff    (eqv.)
+# For P4DIFF=diff,ok should there be any difference in the two cases below, the manual
+# is unclear and disagrees slightly with p4 help diff, also unclear... so experimentally
+#   -dc -dp => diff -c
+#   -dp -dc or -dpc or -dcp => diff -c -p
+# Going with -dcp which should work for either diff tool.
+# FIXME: -db -dl may be ok too if perforce handles these better than CVS
+# FIXME: should perhaps just back in the options we want to the P4DIFF def below
+# FIXME: if we move to p5 we can get diffs for new files too, xref case 225545
+DIFF_OPTIONS=-dcp
+
+COMMIT_SCRIPT=${DYNAMORIO_TOOLS}/commitmsg.pl
+
+$(DIFF_DIFF): $(C_SRCS) Makefile 
+	export P4DIFF=diff; $(P4) diff $(DIFF_OPTIONS) > $@ || true
+# p4 doesn't diff added or deleted files. So we add those in manually. Xref PR 225545.
+# show deleted files: Note - uses 'p4 annotate' to display the contents of the file.
+	for i in `$(P4) opened | $(AWK) '/ - delete.*text/ {print $$1}'`; do \
+	  $(ECHO) $$i | $(AWK) '{printf "==== %s - Removing ====\n*** %s\n--- /dev/null\n***************\n",$$1,$$1 >> "$@"}'; \
+	  $(P4) annotate -q $$i | $(SED) 's|^[0-9]*: |- |' >> $@; \
+	done || true
+	$(P4) opened | $(AWK) '/ - delete.*binary/ {printf "==== %s - Removing ====\n*** %s\n--- /dev/null\n***************[Binary]\n",$$1,$$1}' >> $@ || true
+# show added files: Note - the sed \\ to \\\\ is for windows p4 (which gives a
+#                   windows path which needs escaping); the first sed strips off the
+#                   revision number which 'p4 where' chokes on.
+	for i in `$(P4) opened | $(AWK) '/ - add.*text/ {print $$1}' | $(SED) 's|#[0-9]*$$||'`; do \
+	  $(P4) where $$i | $(AWK) '{printf "==== %s#NEW - %s ====\n*** /dev/null\n--- %s\n***************\n",$$1,$$3,$$3 >> "$@"; print $$3}' | \
+	  $(SED) 's|\\|\\\\|g' | $(XARGS) $(CAT) | $(SED) 's|^|+ |' >> $@; \
+	done || true
+	$(P4) opened | $(AWK) '/ - add.*binary/ {print $$1}' | $(SED) 's|#[0-9]*$$||' | $(XARGS) --no-run-if-empty $(P4) where | $(AWK) '{printf "==== %s#NEW - %s ====\n*** /dev/null\n--- %s\n***************[Binary]\n",$$1,$$3,$$3}' >> $@ || true
+# inline comments for a regular patch diff should be the norm for now
+# FIXME: we should also produce web diffs for quick reviews
+
+$(DIFFEMAIL):
+	@(if [ -z $(REVIEWER) ]; then $(ECHO) "must set REVIEWER="; false; fi)
+	@(if [ -z $(REVIEWER_CHECK) ]; then $(ECHO) "must set REVIEWER to one of $(DEVELOPERS)"; false; fi)
+	@$(ECHO) "From: $(USER)@vmware.com" > $@
+	@$(ECHO) "To: $(REVIEWER_MAIL)" >> $@
+	@$(ECHO) "Cc: $(CARBON_COPY)" >> $@
+	@$(ECHO) "Subject: [cr] $(DIFF_NAME)" >> $@
+	@$(ECHO) "" >> $@
+	@$(ECHO) "$(REMOTE_DIR)/$(DIFF_NAME).{diff,notes}" >> $@
+	@$(ECHO) "" >> $@
+	@$(GREP) '^toemail' $(DIFF_NOTES) || ($(ECHO) "missing toemail marker"; false)
+	@$(GREP) -A 50000 '^toemail' $(DIFF_NOTES) | $(GREP) -v toemail >> $@
+	@$(ECHO) "" >> $@
+	@$(ECHO) "----------------------------------------" >> $@
+	@-diffstat -f 1 $(DIFF_DIFF) >> $@
+	@LINES=`$(WC) -l $(DIFF_DIFF)|$(CUT) -f 1 -d ' '`; $(ECHO) $$LINES diff lines >> $@; $(ECHO) Requires $$[($$LINES+500)/500] reviewers unless $(REVIEWER) overrides >> $@
+	@$(ECHO) "This email was generated automatically by $(USER) at `date`" >> $@
+	@$(ECHO) "" >> $@
+	@$(ECHO) Review before sending email
+	$(CAT) $(DIFFEMAIL)
+
+diff: $(DIFF_DIFF)
+
+# FIXME - eventually depend on "runregression" and check its output for failed tests
+diffcheck: ${DIFF_DIFF} diffnotes
+# FIXME: new files fail here
+	@($(GREP) -c '^==== //' ${DIFF_DIFF}; $(GREP) -c '^\.$$' ${DIFF_NOTES}) | $(XARGS) | $(AWK) '{if ($$1 != $$2) { print "ERROR: "$$1" changed files != "$$2" notes entries"; exit -1}}'
+# runregression checks for make ugly failures this way, FIXME - we could use
+# exit $$[1-$$?] after each of the grep rules in ugly: and just use a dependency here
+# instead, however we'd have to stop using - to ignore failures in ugly: which would
+# make it less usefull 
+	$(MAKE) -s ugly | $(GREP) -v '^$(UGLY_RULE_HEADER)'; exit $$[1-$$?]
+
+diffemail: diffnotes $(DIFFEMAIL)
+
+diffclean: 
+	-$(RM) $(DIFF_DIR)/$(DIFF_NAME).{diff,notes,email,regr}*
+
+diffnotes: $(NOTES)
+	@(if test -e $(DIFF_NOTES); then $(ECHO) Already copied. Update DIFFNUM=$$[$(DIFFNUM)+1]; false; fi)
+	$(CP) $(NOTES) $(DIFF_NOTES)
+
+# no clean rule for $(NOTES), must remove manually
+diffskel: 
+	@(if test -e $(NOTES); then $(ECHO) "Will not clobber existing $(NOTES)"; false; fi)
+# note need to this for any new files or for deleted file that will not show up in p4 diff
+	$(P4) -s opened | $(GREP) -v exit: | $(GREP) -v " - edit" | $(AWK) '{printf "%s:\n %s\n.\n",$$2,$$4}' > $(NOTES)
+# FIXME: we should show to user anything other than info: from p4 -s output
+# FIXME: need to be converted to a cleaner format without //depot nor local path files
+# but we mostly care about actual diffs
+	$(P4) -s diff -sa | $(GREP) -v exit: | $(AWK) '{printf "%s:\n\n.\n",$$2}' >> $(NOTES)
+	$(PERL) -e 'printf "===========================================================================\ntoemail:\n<general notes here>\n"' >> $(NOTES)
+
+diffprep: diffnotes $(DIFF_DIFF) $(DIFFEMAIL)
+# FIXME: eventually require regression results before sending out diff?
+	@(r=`$(ECHO) ${REGRESSION}*`; if test -e $${r%% *}; then $(CHMOD) g+r ${REGRESSION}*; scp -q ${REGRESSION}* $(REMOTE_DIR); else $(ECHO) "WARNING: No regression results found!"; fi)
+	$(CHMOD) g+r $(DIFF_DIR)/$(DIFF_NAME).{diff,notes,email}
+# sanity check: don't copy diffs into the repo
+	@(if [ ${REMOTE_DIR} == ${CVSROOT} ]; then $(ECHO) "Bad review dir ${REMOTE_DIR}: check your CVSROOT" && false; fi)
+	scp -q $(DIFF_DIR)/$(DIFF_NAME).{diff,notes,email} $(REMOTE_DIR) 
+	@$(ECHO) [cr] $(DIFF_NAME)
+
+EMAIL_COMMAND=/usr/sbin/sendmail $(REVIEWER_MAIL) $(CARBON_COPY) < $(REVIEW_DIR)/$(DIFF_NAME).email
+diffsend: diffprep
+	@(if [ -z $(REMOTE_HOST) ]; then $(EMAIL_COMMAND); else ssh $(REMOTE_HOST) "$(EMAIL_COMMAND)"; fi)
+	@touch $(DIFF_DIR)/$(DIFF_NAME).$(REVIEWER_MAIL)
+	@$(ECHO) Mail sent
+
+
+# FIXME: make an official copy of the final diff, notes, and response comments
+# from previous diffs -- have <shareddir>/review/<user>/<tree>.{response,rebuttal}?
+commit: diffnotes diffcheck
+#FIXME: commitmsg.pl not supported
+# ${COMMIT_SCRIPT} ${DIFF_NOTES}
+	@$(ECHO) RTFM:  p4 change; p4 submit
+	$(P4) help simple
+
+
+##################################################
+# Support for running regression suite 
+#
+# We update the target's tools, libutil, and suite dirs from the local copy, allowing
+# testing of changes to other components than just core/.
+# We do assume that benchmarks/ is already set up and pointed at by a
+# DYNAMORIO_BENCHMARKS that is set in the shell rc file.
+#
+# To avoid the problem of a later checkout being newer than an older modified
+# file, we can either clobber every time or use rsync w/ separate targets --
+# we choose the latter, splitting up src targets, but assuming people rarely
+# send over multiple tools/libutil/suite targets.
+# By default we copy to a dedicated regression dir and don't clobber in-use
+# checkouts, etc. on the remote machine.
+#
+# WARNING: if you have serious clock skew rsync may copy the same files multiple times
+# in a row, thinking they're still new, or may not copy files that really are new.
+
+ifdef DYNAMORIO_REGRESSION
+  RUNREG_BASE := ${DYNAMORIO_REGRESSION}
+else
+  RUNREG_BASE := ~/dr/regression
+endif
+RUNREG_HOST=localhost
+RUNREG_TREE=${RUNREG_BASE}/trees/${CUR_TREE}
+RUNREG_SRC=${RUNREG_TREE}/${CORE_DIRNAME}
+RUNREG_MAKE=${RUNREG_BASE}/make
+RUNREG_TOOLS=${RUNREG_BASE}/tools
+RUNREG_LIBUTIL=${RUNREG_BASE}/libutil
+RUNREG_SUITE=${RUNREG_BASE}/suite
+RUNREG_BENCHMARKS=${RUNREG_BASE}/benchmarks
+RUNREG_API=${RUNREG_BASE}/api
+ifeq (${RUNREG_HOST}, localhost)
+RUNREG_LOC=
+RUNREG_CMD=$(SHELL) -c
+RUNREG_SCRIPT=$(PERL) runregression
+else
+RUNREG_LOC=${USER}@${RUNREG_HOST}:
+RUNREG_CMD=ssh ${USER}@${RUNREG_HOST}
+RUNREG_SCRIPT=perl runregression
+endif
+# DYNAMORIO_{TOOLS,LIBUTIL,MAKE,SUITE,BENCHMARKS,API} must be set
+# for a regression.  We'll supply default values relative to the
+# current directory if they're not explicitly provided.
+ifndef DYNAMORIO_TOOLS
+  DYNAMORIO_TOOLS := ../tools
+endif
+ifndef DYNAMORIO_LIBUTIL
+  DYNAMORIO_LIBUTIL := ../libutil
+endif
+ifndef DYNAMORIO_SUITE
+  DYNAMORIO_SUITE := ../suite
+endif
+ifndef DYNAMORIO_BENCHMARKS
+  DYNAMORIO_BENCHMARKS := ../benchmarks
+endif
+ifndef DYNAMORIO_API
+  DYNAMORIO_API := ../api
+endif
+ifeq ($(MACHINE), win32)
+DR_MAKE=$(shell $(CYGPATH) -ui ${DYNAMORIO_MAKE})
+DR_TOOLS=$(shell $(CYGPATH) -ui ${DYNAMORIO_TOOLS})
+DR_LIBUTIL=$(shell $(CYGPATH) -ui ${DYNAMORIO_LIBUTIL})
+DR_SUITE=$(shell $(CYGPATH) -ui ${DYNAMORIO_SUITE})
+DR_BENCHMARKS=$(shell $(CYGPATH) -ui ${DYNAMORIO_BENCHMARKS})
+DR_API=$(shell $(CYGPATH) -ui ${DYNAMORIO_API})
+else
+DR_MAKE=${DYNAMORIO_MAKE}
+DR_TOOLS=${DYNAMORIO_TOOLS}
+DR_LIBUTIL=${DYNAMORIO_LIBUTIL}
+DR_SUITE=${DYNAMORIO_SUITE}
+DR_BENCHMARKS=${DYNAMORIO_BENCHMARKS}
+DR_API=${DYNAMORIO_API}
+endif
+
+# To save time when rsync'ing the benchmarks directory, only include
+# the bare minimum needed for the regression:
+BENCHMARK_SRCS:=RunSuite			\
+		spec2000/Makefile*		\
+		spec2000/tools			\
+		spec2000/specperl		\
+		spec2000/int/crafty/RunSpec	\
+		spec2000/int/crafty/Makefile	\
+		spec2000/int/crafty/src		\
+		spec2000/int/crafty/data/test 	\
+		spec2000/int/gcc/RunSpec	\
+		spec2000/int/gcc/Makefile	\
+		spec2000/int/gcc/src		\
+		spec2000/int/gcc/data/test	\
+		spec2000/int/mcf/RunSpec	\
+		spec2000/int/mcf/Makefile	\
+		spec2000/int/mcf/src		\
+		spec2000/int/mcf/data/test	
+
+# Note below that we use rsync -R when transferting portions of the
+# benchmarks module so we can maintain the directory structure.  We
+# need the './' syntax to get it right.  See the description of -R in
+# the rsync man page.
+BENCHMARK_SRCS := $(patsubst %,${DR_BENCHMARKS}/./%,${BENCHMARK_SRCS})
+
+SHELL_RC=~/.bashrc
+
+# If you have problems rsync-ing (replacing an in-use file, etc.) can disable by
+# running with NORSYNC=1 (will still rsync the src tree)
+NORSYNC = 0
+
+# We use -C to avoid clobbering CVS files, so remote dirs can still be used as cvs co's
+# if so desired.
+RSYNC = rsync -auvzC --exclude=*.map --exclude=*.pdb
+
+RSYNC_TOOLS_FLAGS := --include=/external/whoami.exe --exclude=/external/*
+
+runregression:
+	@(if [ ! -d ${DR_MAKE} ]; then \
+		$(ECHO) "ERROR: ${DR_MAKE} does not exist!"; false; fi)
+	@(if [ ! -d ${DR_TOOLS} ]; then \
+		$(ECHO) "ERROR: ${DR_TOOLS} does not exist!"; false; fi)
+	@(if [ ! -d ${DR_LIBUTIL} ]; then \
+		$(ECHO) "ERROR: ${DR_LIBUTIL} does not exist!"; false; fi)
+	@(if [ ! -d ${DR_SUITE} ]; then \
+		$(ECHO) "ERROR: ${DR_SUITE} does not exist!"; false; fi)
+	@(if [ ! -d ${DR_BENCHMARKS} ]; then \
+		$(ECHO) "ERROR: ${DR_BENCHMARKS} does not exist!"; false; fi)
+	@(if [ ! -d ${DR_API} ]; then \
+		$(ECHO) "ERROR: ${DR_API} does not exist!"; false; fi)
+# Since there may be no subdirs already created, and we can't use rsync's --relative
+# w/o being able to break up the local path and w/o identical path tails, we use
+# mkdir -p to make the dir structure.
+	${RUNREG_CMD} "mkdir -p ${RUNREG_SRC}"
+	${RSYNC} . ${RUNREG_LOC}${RUNREG_SRC}
+ifeq (${NORSYNC}, 0)
+	${RSYNC} ${DR_MAKE}/ ${RUNREG_LOC}${RUNREG_MAKE}
+	${RSYNC} $(RSYNC_TOOLS_FLAGS) ${DR_TOOLS}/ ${RUNREG_LOC}${RUNREG_TOOLS}
+	${RSYNC} ${DR_LIBUTIL}/ ${RUNREG_LOC}${RUNREG_LIBUTIL}
+# FIXME: We no longer have the CVS/Entries file to easily get
+# a list of directories to rsync.  We may end up sending more
+# than necessary; for now just provide the long list of
+# exclusions to try to avoid sending generated files.
+	${RSYNC} \
+	    --exclude=*.exe --exclude=*.obj --exclude=*.pdb	\
+	    --exclude=*.ilk --exclude=*.clout --exclude=*.dll	\
+	    --exclude=*.last --exclude=*.lib --exclude=*.exp	\
+	    --exclude=results/ --exclude=TMPPREINJDIR/		\
+	    --exclude=TMPPREINJDIR/ --exclude=stress/run/	\
+	    --exclude=regression-*/ --exclude=cache/		\
+	    --exclude=logs/ --include=events*dummy?.so		\
+	    ${DR_SUITE}/ ${RUNREG_LOC}${RUNREG_SUITE}
+# -C auto-ignores *.exe so we explicitly include specperl.exe here
+	${RSYNC} -R --include=specperl.exe \
+	    ${BENCHMARK_SRCS} ${RUNREG_LOC}${RUNREG_BENCHMARKS}
+	${RSYNC} --exclude=/docs/html/ --exclude=/docs/latex/   \
+                 --include=VIPA_test.exe                        \
+	    ${DR_API}/ ${RUNREG_LOC}${RUNREG_API}
+endif
+# We must source .bashrc to get the compiler path, etc. set up
+# (cygwin bash non-interactive does not source it even if started by sshd).
+	${RUNREG_CMD} " \
+	    if [ -e ${SHELL_RC} ]; then source ${SHELL_RC}; fi; \
+	    cd ${RUNREG_SUITE}; \
+            export DYNAMORIO_HOME=${RUNREG_TREE}; \
+            export DYNAMORIO_TOOLS=${RUNREG_TOOLS}; \
+            export DYNAMORIO_LIBUTIL=${RUNREG_LIBUTIL}; \
+	    export DYNAMORIO_MAKE=${RUNREG_MAKE}; \
+	    export DYNAMORIO_BENCHMARKS=${RUNREG_BENCHMARKS}; \
+	    export DYNAMORIO_API=${RUNREG_API}; \
+            ${RUNREG_SCRIPT} ${RUNREG_OPTIONS}" 2>&1 | $(TEE) ${REGRESSION}-${RUNREG_HOST}
+	@$(GREP) "all builds succesful" ${REGRESSION}-${RUNREG_HOST}
+
+specupdate:
+	@if ! test ${DYNAMORIO_BENCHMARKS}; then $(ECHO) "ERROR: DYNAMORIO_BENCHMARKS not set"; exit -1; fi
+# Send just config files for spec2000
+# (need the dirs explicit so * doesn't squash them)
+# Make sure your symlinks are local, if possible -- FIXME: runstats.c is not
+	RUNREG_BENCHMARKS=`${RUNREG_CMD} "source ${SHELL_RC} > /dev/null; \
+            cygpath -u \\$${DYNAMORIO_BENCHMARKS} 2> /dev/null || $(ECHO) \\$${DYNAMORIO_BENCHMARKS}"`; \
+            ${RSYNC} --include=/spec2000/ --include=/spec2000/*/ \
+            --include=/spec2000/*/*/ --include=/RunSuite --include=/spec2000**/Makefile* \
+            --include=/spec2000**/RunSpec --include=/spec2000**/tools/* \
+            --exclude=* ${DR_BENCHMARKS}/ ${RUNREG_LOC}$${RUNREG_BENCHMARKS}
+
+up update: # (needed only since we don't have good make depends)
+# FIXME: not very sophisticated version for starters with P4, 
+# check if up to date, and update if not
+#	(cvs status | grep Status | grep -vE 'Up-to-date|Locally') && (cvs update; make clear) || $(ECHO) Up-to-date
+	$(P4) diff -sa  # new files not needed here
+	$(P4) sync
+	make clear
diff --git a/core/Makefile.custom_build b/core/Makefile.custom_build
new file mode 100644
index 000000000..964abe9c1
--- /dev/null
+++ b/core/Makefile.custom_build
@@ -0,0 +1,80 @@
+# **********************************************************
+# Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# Copyright (c) 2006-2007 Determina Corp.
+
+##################################################
+# pass build and version numbers
+# custom builds: 9XYYZ
+# X = developer, YY = tree, Z = diffnum
+# YY defaults to 1st 2 letters of CUR_TREE, unless CASENUM is defined,
+# in which case it is the last 2 letters of CASENUM (all % 10 of course)
+
+DEVELOPERS=dbruenin vkirians tgarnett slarsen bchandra
+ifeq ($(strip ${USER}),)
+# USER is set on unix, USERNAME on windows -- in this makefile we use USER
+USER=${USERNAME}
+endif
+
+ifndef BUILD_NUMBER
+# schemes like 1st letter, uid, etc. can be concocted but not easy to avoid
+# collisions so we just have a list
+DEVELOPER_NUMS=1 2 3 4 5 6 7 8 9
+DEVNUM=$(foreach num,${DEVELOPER_NUMS},\
+    $(if $(findstring ${USER},$(word ${num},${DEVELOPERS})), ${num}))
+ifeq (${strip ${DEVNUM}},)
+# unknown developer gets 0
+DEVNUM=0 
+endif
+
+CUR_FULL=$(shell cd ..$(CUR_FULL_ADD); $(ECHO) $$PWD)
+CUR_TREE=$(notdir ${CUR_FULL})
+ifeq ($(strip ${CASENUM}),)
+# convert 1st two chars in tree name to numbers where a=0, b=1, etc.
+# I had it as the more familiar a=1, b=2 but then numbers get 1 added to
+# them and it gets confusing.
+TREENUM=$(shell x=${CUR_TREE}; for ((i=0; i<2; i++)); do \
+          $(ECHO) $$((36\#$${x:$$i:1} % 10)); done) 
+else
+# take last 2 chars of CASENUM, converted to digits as above
+TREENUM=$(shell x=${CASENUM}; for ((i=2; i>0; i--)); do \
+          $(ECHO) $$((36\#$${x:$$((-$$i)):1} % 10)); done) 
+# FIXME: auto-create tree name for diffs by adding CASENUM to CUR_TREE?
+endif
+BUILD_NUMBER_SPACES=9${DEVNUM}${TREENUM}${DIFFNUM}
+# seems to be no make function for removing spaces so we invoke shell for sed
+BUILD_NUMBER=$(shell $(ECHO) ${BUILD_NUMBER_SPACES} | $(SED) 's/ //g')
+RESOURCE_DEFINES = $(D)PRIVATE_BUILD $(D)PRERELEASE_BUILD $(D)PRIVATE_BUILD_DESC="\"$(USER) DIFFNUM=$(DIFFNUM) $(CUR_FULL)\""
+endif # BUILD_NUMBER
+
+ifdef SPECIAL_BUILD
+  RESOURCE_DEFINES += $(D)SPECIAL_BUILD $(D)SPECIAL_BUILD_DESC="\"$(SPECIAL_BUILD)\""
+endif
+##################################################
\ No newline at end of file
diff --git a/core/buildmark.c b/core/buildmark.c
new file mode 100644
index 000000000..e233ec04b
--- /dev/null
+++ b/core/buildmark.c
@@ -0,0 +1,45 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+
+#include "globals.h"
+#include "defines.h"
+
+#if defined(__DATE__) && defined(__TIME__)
+const char dynamorio_buildmark[] = __DATE__ " " __TIME__;
+#else
+const char dynamorio_buildmark[] = "unknown";
+#endif
+
+const char dynamorio_version_string[] = PRODUCT_NAME" "VERSION_NUMBER_STRING", "\
+                                        BUILD_NUMBER_STRING", (c) "COMPANY_LONG_NAME;
diff --git a/core/dispatch.c b/core/dispatch.c
new file mode 100644
index 000000000..ab2e9446f
--- /dev/null
+++ b/core/dispatch.c
@@ -0,0 +1,1931 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * dispatch.c - central dynamo control manager
+ */
+
+#include "globals.h"
+#include "link.h"
+#include "fragment.h"
+#include "fcache.h"
+#include "monitor.h"
+#include "synch.h"
+#include <string.h> /* for strstr */
+
+#ifdef CLIENT_INTERFACE
+# include "emit.h"
+# include "arch.h"
+# include "instrument.h"
+#endif
+
+#ifdef DGC_DIAGNOSTICS
+# include "instr.h"
+# include "disassemble.h"
+#endif
+
+#ifdef RCT_IND_BRANCH
+#  include "rct.h"
+#endif
+
+#ifdef X64
+# include "decode.h" /* get_x86_mode */
+#endif
+
+/* forward declarations */
+static void
+dispatch_enter_dynamorio(dcontext_t *dcontext);
+
+static bool
+dispatch_enter_fcache(dcontext_t *dcontext, fragment_t *targetf);
+
+static void
+dispatch_enter_fcache_stats(dcontext_t *dcontext, fragment_t *targetf);
+
+static void
+enter_fcache(dcontext_t *dcontext, fcache_enter_func_t entry, cache_pc pc);
+
+static void
+dispatch_enter_native(dcontext_t *dcontext);
+
+static void
+dispatch_exit_fcache(dcontext_t *dcontext);
+
+static void
+dispatch_exit_fcache_stats(dcontext_t *dcontext);
+
+static void
+handle_post_system_call(dcontext_t *dcontext);
+
+#ifdef WINDOWS
+static void
+handle_callback_return(dcontext_t *dcontext);
+#endif
+
+#ifdef CLIENT_INTERFACE
+/* PR 356503: detect clients making syscalls via sysenter */
+static inline void
+found_client_sysenter(void)
+{
+    CLIENT_ASSERT(false, "Is your client invoking raw system calls via vdso sysenter? "
+                  "While such behavior is not recommended and can create problems, "
+                  "it may work with the -sysenter_is_int80 runtime option.");
+}
+#endif
+
+/* This is the central hub of control management in DynamoRIO.
+ * It is entered with a clean dstack at startup and after every cache
+ * exit, whether normal or kernel-mediated via a trampoline context switch.
+ * Having no stack state kept across cache executions avoids
+ * self-protection issues with the dstack.
+ */
+void
+dispatch(dcontext_t *dcontext)
+{
+    fragment_t *targetf;
+    fragment_t coarse_f;
+
+    ASSERT(dcontext == get_thread_private_dcontext());
+    dispatch_enter_dynamorio(dcontext);
+    LOG(THREAD, LOG_INTERP, 2, "\ndispatch: target = "PFX"\n", dcontext->next_tag);
+
+    /* This is really a 1-iter loop most of the time: we only iterate
+     * when we obtain a target fragment but then fail to enter the
+     * cache due to flushing before we get there.
+     */
+    do {
+        if (is_stopping_point(dcontext, dcontext->next_tag)) {
+            LOG(THREAD, LOG_INTERP, 1,
+                "\nFound DynamoRIO stopping point: thread %d returning to app @"PFX"\n",
+                get_thread_id(),  dcontext->next_tag);
+            dispatch_enter_native(dcontext);
+            ASSERT_NOT_REACHED();
+        }
+
+        /* Neither hotp_only nor thin_client should have any fragment 
+         * fcache related work to do.
+         */
+        ASSERT(!RUNNING_WITHOUT_CODE_CACHE());
+        targetf = fragment_lookup_fine_and_coarse(dcontext, dcontext->next_tag,
+                                                  &coarse_f, dcontext->last_exit);
+        do {
+            if (targetf != NULL) {
+                KSTART(monitor_enter);
+                /* invoke monitor to continue or start a trace 
+                 * may result in changing or nullifying targetf
+                 */
+                targetf = monitor_cache_enter(dcontext, targetf);
+                KSTOP_NOT_MATCHING(monitor_enter); /* or monitor_enter_thci */
+            }
+            if (targetf != NULL)
+                break;
+            if (USE_BB_BUILDING_LOCK()) {
+                mutex_lock(&bb_building_lock);
+                /* must re-lookup while holding lock and keep the lock until we've
+                 * built the bb and added it to the lookup table
+                 * FIXME: optimize away redundant lookup: flags to know why came out?
+                 */
+                targetf = fragment_lookup_fine_and_coarse(dcontext, dcontext->next_tag,
+                                                          &coarse_f, dcontext->last_exit);
+            }
+            if (targetf == NULL) {
+                SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+                targetf =
+                    build_basic_block_fragment(dcontext, dcontext->next_tag,
+                                               0, true/*link*/, true/*visible*/
+                                               _IF_CLIENT(false/*!for_trace*/)
+                                               _IF_CLIENT(NULL));
+                SELF_PROTECT_LOCAL(dcontext, READONLY);
+            }
+            ASSERT(targetf != NULL);
+            if (TEST(FRAG_COARSE_GRAIN, targetf->flags)) {
+                /* targetf is a static temp fragment protected by bb_building_lock,
+                 * so we must make a local copy to use before releasing the lock.
+                 * FIXME: best to pass local wrapper to build_basic_block_fragment
+                 * and all the way through emit and link?  Would need linkstubs
+                 * tailing the fragment_t.
+                 */
+                ASSERT(USE_BB_BUILDING_LOCK());
+                fragment_coarse_wrapper(&coarse_f, targetf->tag,
+                                        FCACHE_ENTRY_PC(targetf));
+                targetf = &coarse_f;
+            }
+            if (USE_BB_BUILDING_LOCK())
+                mutex_unlock(&bb_building_lock);
+            /* loop around and re-do monitor check */
+        } while (true);
+
+        if (dispatch_enter_fcache(dcontext, targetf)) {
+            /* won't reach here: will re-enter dispatch() with a clean stack */
+            ASSERT_NOT_REACHED();
+        } else
+            targetf = NULL; /* targetf was flushed */
+    } while (true);
+    ASSERT_NOT_REACHED();
+}
+
+/* returns true if pc is a point at which DynamoRIO should stop interpreting */
+bool
+is_stopping_point(dcontext_t *dcontext, app_pc pc)
+{
+    if ((pc == BACK_TO_NATIVE_AFTER_SYSCALL &&
+         /* case 6253: app may xfer to this "address" in which case pass
+          * exception to app
+          */
+         dcontext->native_exec_postsyscall != NULL)
+#ifdef DR_APP_EXPORTS
+        || (!automatic_startup && 
+            (pc == (app_pc)dynamorio_app_exit ||
+             /* FIXME: Is this a holdover from long ago? dymamo_thread_exit
+              * should not be called from the cache.
+              */
+             pc == (app_pc)dynamo_thread_exit ||
+             pc == (app_pc)dr_app_stop))
+#endif
+#ifdef WINDOWS
+        /* we go all the way to NtTerminateThread/NtTerminateProcess */
+#else /* LINUX */
+        /* we go all the way to SYS_exit or SYS_{,t,tg}kill(SIGABRT) */
+#endif
+        )
+        return true;
+
+    return false;
+}
+
+static void
+dispatch_enter_fcache_stats(dcontext_t *dcontext, fragment_t *targetf)
+{
+#ifdef DEBUG
+# ifdef DGC_DIAGNOSTICS
+    if (TEST(FRAG_DYNGEN, targetf->flags) && !is_dyngen_vsyscall(targetf->tag)) {
+        char buf[MAXIMUM_SYMBOL_LENGTH];
+        bool stack = is_address_on_stack(dcontext, targetf->tag);
+        LOG(THREAD, LOG_DISPATCH, 1, "Entry into dyngen F%d("PFX"%s%s) via:", 
+            targetf->id, targetf->tag,
+            stack ? " stack":"",
+            (targetf->flags & FRAG_DYNGEN_RESTRICTED) != 0 ? " BAD":"");
+        if (!LINKSTUB_FAKE(dcontext->last_exit)) {
+            app_pc translated_pc;
+            /* can't recreate if fragment is deleted -- but should be fake then */
+            ASSERT(!TEST(FRAG_WAS_DELETED, dcontext->last_fragment->flags));
+            translated_pc = recreate_app_pc(dcontext, EXIT_CTI_PC(dcontext->last_fragment,
+                                                                  dcontext->last_exit),
+                                            dcontext->last_fragment);
+            if (translated_pc != NULL) {
+                disassemble(dcontext, translated_pc, THREAD);
+                print_symbolic_address(translated_pc, buf, sizeof(buf), false);
+                LOG(THREAD, LOG_DISPATCH, 1, " %s\n", buf);
+            }
+            if (!stack &&
+                (strstr(buf, "user32.dll") != NULL || strstr(buf, "USER32.DLL") != NULL)) {
+                /* try to find who set up user32 callback */
+                dump_mcontext_callstack(dcontext);
+            }
+            DOLOG(stack ? 1U : 2U, LOG_DISPATCH, {
+                LOG(THREAD, LOG_DISPATCH, 1, "Originating bb:\n");
+                disassemble_app_bb(dcontext, dcontext->last_fragment->tag, THREAD);
+            });
+        } else {
+            /* FIXME: print type from last_exit */
+            LOG(THREAD, LOG_DISPATCH, 1, "\n");
+        }
+        if (stack) {
+            /* try to understand where code is on stack */
+            LOG(THREAD, LOG_DISPATCH, 1, "cur esp="PFX" ebp="PFX"\n",
+                get_mcontext(dcontext)->xsp, get_mcontext(dcontext)->xbp);
+            dump_mcontext_callstack(dcontext);
+        }
+    }
+# endif
+
+    if (stats->loglevel >= 2 && (stats->logmask & LOG_DISPATCH) != 0) {
+        /* FIXME: this should use a different mask - and get printed at level 2 when turned on */
+        DOLOG(4, LOG_DISPATCH, {
+            dump_mcontext(get_mcontext(dcontext), THREAD, DUMP_NOT_XML); });
+        DOLOG(6, LOG_DISPATCH, { dump_mcontext_callstack(dcontext); });
+        DOKSTATS({ DOLOG(6, LOG_DISPATCH, { kstats_dump_stack(dcontext); }); });
+# ifdef RETURN_STACK
+        DOLOG(3, LOG_DISPATCH, { print_return_stack(dcontext); });
+# endif
+# ifdef NATIVE_RETURN_CALLDEPTH
+        LOG(THREAD, LOG_DISPATCH, 3, "\tCall depth is %d\n",
+            dcontext->call_depth);
+# endif
+        LOG(THREAD, LOG_DISPATCH, 2, "Entry into F%d("PFX")."PFX" %s%s", 
+            targetf->id,
+            targetf->tag,
+            FCACHE_ENTRY_PC(targetf),
+            TEST(FRAG_COARSE_GRAIN, targetf->flags) ? "(coarse)" : "",
+            ((targetf->flags & FRAG_IS_TRACE_HEAD)!=0)? 
+            "(trace head)" : "",
+            ((targetf->flags & FRAG_IS_TRACE)!=0)? "(trace)" : "");
+        LOG(THREAD, LOG_DISPATCH, 2, "%s",
+            TEST(FRAG_SHARED, targetf->flags) ? "(shared)":"");
+# ifdef DGC_DIAGNOSTICS
+        LOG(THREAD, LOG_DISPATCH, 2, "%s",
+            TEST(FRAG_DYNGEN, targetf->flags) ? "(dyngen)":"");
+# endif
+        LOG(THREAD, LOG_DISPATCH, 2, "\n");
+
+        DOLOG(3, LOG_SYMBOLS, {
+            char symbuf[MAXIMUM_SYMBOL_LENGTH];
+            print_symbolic_address(targetf->tag, 
+                                   symbuf, sizeof(symbuf), true);
+            LOG(THREAD, LOG_SYMBOLS, 3, "\t%s\n", symbuf);
+        });
+    }
+#endif /* DEBUG */
+}
+
+/* Executes a target fragment in the fragment cache */
+static bool
+dispatch_enter_fcache(dcontext_t *dcontext, fragment_t *targetf)
+{
+    fcache_enter_func_t fcache_enter;
+    ASSERT(targetf != NULL);
+    /* ensure we don't take over when we should be going native */
+    ASSERT(dcontext->native_exec_postsyscall == NULL);
+
+    /* We wait until here, rather than at cache exit time, to do lazy
+     * linking so we can link to newly created fragments.
+     */
+    if (dcontext->last_exit == get_coarse_exit_linkstub() ||
+        /* We need to lazy link if either of src or tgt is coarse */
+        (LINKSTUB_DIRECT(dcontext->last_exit->flags) &&
+         TEST(FRAG_COARSE_GRAIN, targetf->flags))) {
+        coarse_lazy_link(dcontext, targetf);
+    }
+
+    if (!enter_nolinking(dcontext, targetf, true)) {
+        /* not actually entering cache, so back to couldbelinking */
+        enter_couldbelinking(dcontext, NULL, true);
+        LOG(THREAD, LOG_DISPATCH, 2, "Just flushed targetf, next_tag is "PFX"\n",
+            dcontext->next_tag);
+        STATS_INC(num_entrances_aborted);
+        /* shared entrance cannot-tell-if-deleted -> invalidate targetf
+         * but then may double-do the trace!
+         * FIXME: for now, we abort every time, ok to abort twice (first time
+         * b/c there was a real flush of targetf), but could be perf hit.
+         */
+        trace_abort(dcontext);
+        return false;
+    }
+
+    dispatch_enter_fcache_stats(dcontext, targetf);
+                    
+    /* FIXME: for now we do this before the synch point to avoid complexity of
+     * missing a KSTART(fcache_* for cases like NtSetContextThread where a thread
+     * appears back at dispatch() from the synch point w/o ever entering the cache.
+     * To truly fix we need to have the NtSetContextThread handler determine
+     * whether its suspended target is at this synch point or in the cache.
+     */
+    DOKSTATS({
+        /* stopped in dispatch_exit_fcache_stats */
+        if (TEST(FRAG_IS_TRACE, targetf->flags))
+            KSTART(fcache_trace_trace);
+        else
+            KSTART(fcache_default); /* fcache_bb_bb or fcache_bb_trace */
+        /* FIXME: overestimates fcache time by counting in
+         * fcache_enter/fcache_return for it - proper reading of this
+         * value should discount the minimal cost of
+         * fcache_enter/fcache_return for actual code cache times
+         */
+        /* FIXME: asynch events currently continue their current kstat
+         * until they get back to dispatch, so in-fcache kstats are counting
+         * the in-DR trampoline execution time!
+         */
+    });
+
+#ifdef WINDOWS
+    /* synch point for suspend, terminate, and detach */
+    /* assumes mcontext is valid including errno but not pc (which we fix here)
+     * assumes that thread is holding no locks
+     * also assumes past enter_nolinking, so could_be_linking is false 
+     * for safety with respect to flush */
+    /* a fast check before the heavy lifting */
+    if (should_wait_at_safe_spot(dcontext)) {
+        /* FIXME : we could put this synch point in enter_fcache but would need
+         * to use SYSCALL_PC for syscalls (see issues with that in win32/os.c) 
+         */
+        dr_mcontext_t *mcontext = get_mcontext(dcontext);
+        cache_pc save_pc = mcontext->pc;
+        /* FIXME : implementation choice, we could do recreate_app_pc 
+         * (fairly expensive but this is rare) instead of using the tag 
+         * which is a little hacky but should always be right */
+        mcontext->pc = targetf->tag;
+        /* could be targeting interception code or our dll main, would be 
+         * incorrect for GetContextThread and racy for detach, though we
+         * would expect it to be very rare */
+        if (!is_dynamo_address(mcontext->pc)) {
+            check_wait_at_safe_spot(dcontext, THREAD_SYNCH_VALID_MCONTEXT);
+            /* If we don't come back here synch-er is responsible for ensuring
+             * our kstat stack doesn't get off (have to do a KSTART here) -- we
+             * don't want to do the KSTART of fcache_* before this to avoid
+             * counting synch time.
+             */
+        } else {
+            LOG(THREAD, LOG_SYNCH, 1,
+                "wait_at_safe_spot - unable to wait, targeting dr addr "PFX,
+                mcontext->pc);
+            STATS_INC(no_wait_entries);
+        }
+        mcontext->pc = save_pc;
+    }
+#endif
+
+    IF_X64(ASSERT(get_x86_mode(dcontext) == TEST(FRAG_32_BIT, targetf->flags)));
+    if (TEST(FRAG_SHARED, targetf->flags))
+        fcache_enter = get_fcache_enter_shared_routine(dcontext);
+    else
+        fcache_enter = get_fcache_enter_private_routine(dcontext);
+
+    enter_fcache(dcontext, fcache_enter, FCACHE_ENTRY_PC(targetf));
+    ASSERT_NOT_REACHED();
+    return true;
+}
+
+/* Enters the cache at the specified entrance routine to execute the
+ * target pc.
+ * Does not return.
+ * Caller must do a KSTART to avoid kstats stack mismatches.
+ * FIXME: only allow access to fcache_enter routine through here?
+ * Indirect routine needs special treatment for handle_callback_return
+ */
+static void
+enter_fcache(dcontext_t *dcontext, fcache_enter_func_t entry, cache_pc pc)
+{
+    ASSERT(!is_couldbelinking(dcontext));
+    ASSERT(entry != NULL);
+    ASSERT(pc != NULL);
+    ASSERT(check_should_be_protected(DATASEC_RARELY_PROT));
+    /* CANNOT hold any locks across cache execution, as our thread synch
+     * assumes none are held
+     */
+    ASSERT_OWN_NO_LOCKS();
+    ASSERT(dcontext->try_except_state == NULL);
+
+    /* prepare to enter fcache */
+    LOG(THREAD, LOG_DISPATCH, 4, "fcache_enter = "PFX", target = "PFX"\n", entry, pc);
+    set_fcache_target(dcontext, pc);
+    ASSERT(pc != NULL);
+
+#ifdef PROFILE_RDTSC
+    if (dynamo_options.profile_times) {
+        /* prepare to enter fcache */
+        dcontext->prev_fragment = NULL;
+
+        /* top ten cache times */
+        dcontext->cache_frag_count = (uint64) 0;
+        dcontext->cache_enter_time = get_time();
+    }
+#endif
+
+    dcontext->whereami = WHERE_FCACHE;
+    (*entry)(dcontext);
+    ASSERT_NOT_REACHED();
+}
+
+#if defined(DR_APP_EXPORTS) || defined(LINUX)
+static void
+dispatch_at_stopping_point(dcontext_t *dcontext)
+{
+    /* start/stop interface */
+    KSTOP_NOT_MATCHING(dispatch_num_exits);
+    
+    /* if we stop in middle of tracing, thread-shared state may be messed
+     * up (e.g., monitor grabs fragment lock for unlinking),
+     * so abort the trace
+     */
+    if (is_building_trace(dcontext)) {
+        LOG(THREAD, LOG_INTERP, 1, "squashing trace-in-progress\n");
+        trace_abort(dcontext);
+    }
+    
+    LOG(THREAD, LOG_INTERP, 1, "\nappstart_cleanup: found stopping point\n");
+# ifdef DEBUG
+#  ifdef DR_APP_EXPORTS
+    if (dcontext->next_tag == (app_pc)dynamo_thread_exit)
+        LOG(THREAD, LOG_INTERP, 1, "\t==dynamo_thread_exit\n");
+    else if (dcontext->next_tag == (app_pc)dynamorio_app_exit)
+        LOG(THREAD, LOG_INTERP, 1, "\t==dynamorio_app_exit\n");
+    else if (dcontext->next_tag == (app_pc)dr_app_stop) {
+#   ifdef NATIVE_RETURN_TRY_TO_PUT_APP_RETURN_PC_ON_STACK
+        /* FIXME: restore app ret pc before we give control back */
+        *((int *)(get_mcontext(dcontext)->xsp)) = 
+#   endif
+        LOG(THREAD, LOG_INTERP, 1, "\t==dr_app_stop\n");
+    }
+#  endif
+# endif
+    
+# ifdef NATIVE_RETURN
+    /* HACK: end now and avoid having to get app return pc
+     * assumes single thread, entire-app run
+     */
+    {
+        byte *app_esp = get_mcontext(dcontext)->xsp;
+        asm("movl %0, %%esp" : : "m"(app_esp));
+        dynamorio_app_exit();
+        exit(0);
+    }
+# else
+    dynamo_thread_not_under_dynamo(dcontext);
+# endif
+}
+#endif
+
+/* Called when we reach an interpretation stopping point either for
+ * start/stop control of DR or for native_exec.  In both cases we give up
+ * control and "go native", but we do not clean up the current thread,
+ * assuming we will either take control back, or the app will explicitly
+ * request we clean up.
+ */     
+static void
+dispatch_enter_native(dcontext_t *dcontext)
+{
+    /* The new fcache_enter's clean dstack design makes it usable for
+     * entering native execution as well as the fcache.
+     */
+    fcache_enter_func_t go_native = get_fcache_enter_private_routine(dcontext);
+    ASSERT_OWN_NO_LOCKS();
+    if (dcontext->next_tag == BACK_TO_NATIVE_AFTER_SYSCALL) {
+        /* we're simply going native again after an intercepted syscall,
+         * not finalizing this thread or anything
+         */
+        IF_WINDOWS(DEBUG_DECLARE(extern dcontext_t *early_inject_load_helper_dcontext;))
+        ASSERT(DYNAMO_OPTION(native_exec_syscalls)); /* else wouldn't have intercepted */
+
+        /* Assert here we have a reason for going back to native (-native_exec and
+         * non-empty native_exec_areas, RUNNING_WITHOUT_CODE_CACHE, hotp nudge thread
+         * pretending to be native while loading a dll, or on win2k
+         * early_inject_init() pretending to be native to find the inject address). */
+        ASSERT((DYNAMO_OPTION(native_exec) && native_exec_areas != NULL &&
+                !vmvector_empty(native_exec_areas)) ||
+               IF_WINDOWS((DYNAMO_OPTION(early_inject) &&
+                         early_inject_load_helper_dcontext ==
+                         get_thread_private_dcontext()) ||)
+               IF_HOTP(dcontext->nudge_thread ||)
+               RUNNING_WITHOUT_CODE_CACHE());
+
+        ASSERT(dcontext->native_exec_postsyscall != NULL);
+        LOG(THREAD, LOG_ASYNCH, 1, "Returning to native "PFX" after a syscall\n",
+            dcontext->native_exec_postsyscall);
+        dcontext->next_tag = dcontext->native_exec_postsyscall;
+        dcontext->native_exec_postsyscall = NULL;
+        LOG(THREAD, LOG_DISPATCH, 2, "Entry into native_exec after intercepted syscall\n");
+        /* restore state as though never came out for syscall */
+        set_last_exit(dcontext, (linkstub_t *) get_native_exec_linkstub());
+        KSTART(fcache_default);
+        enter_nolinking(dcontext, NULL, true);
+    } 
+    else {
+#if defined(DR_APP_EXPORTS) || defined(LINUX)
+        dispatch_at_stopping_point(dcontext);
+        enter_nolinking(dcontext, NULL, false);
+#else
+        ASSERT_NOT_REACHED();
+#endif
+    }
+    set_fcache_target(dcontext, dcontext->next_tag);
+    dcontext->whereami = WHERE_APP;
+    (*go_native)(dcontext);
+    ASSERT_NOT_REACHED();
+}
+
+static void
+dispatch_enter_dynamorio(dcontext_t *dcontext)
+{
+    /* We're transitioning to DynamoRIO from somewhere: either the fcache,
+     * the kernel (WHERE_TRAMPOLINE), or the app itself via our start/stop API.
+     * N.B.: set whereami to WHERE_APP iff this is the first dispatch() entry
+     * for this thread!
+     */
+    where_am_i_t wherewasi = dcontext->whereami;
+#ifdef LINUX
+    if (!(wherewasi == WHERE_FCACHE || wherewasi == WHERE_TRAMPOLINE ||
+          wherewasi == WHERE_APP)) {
+        /* This is probably our own syscalls hitting our own sysenter
+         * hook (PR 212570), since we're not completely user library
+         * independent (PR 206369).
+         * The primary calls I'm worried about are dl{open,close}.
+         * Note that we can't go jump to vsyscall_syscall_end_pc here b/c
+         * fcache_return cleared the dstack, so we can't really recover.
+         * We could put in a custom exit stub and return routine and recover,
+         * but we need to get library independent anyway so it's not worth it.
+         */
+        ASSERT(get_syscall_method() == SYSCALL_METHOD_SYSENTER);
+        IF_X64(ASSERT_NOT_REACHED()); /* no sysenter support on x64 */
+        /* PR 356503: clients using libraries that make syscalls can end up here */
+        IF_CLIENT_INTERFACE(found_client_sysenter());
+        ASSERT_BUG_NUM(206369, false &&
+                       "DR's own syscall (via user library) hit the sysenter hook");
+    }
+#endif
+    ASSERT(wherewasi == WHERE_FCACHE || wherewasi == WHERE_TRAMPOLINE ||
+           wherewasi == WHERE_APP);
+    dcontext->whereami = WHERE_DISPATCH;
+    ASSERT_LOCAL_HEAP_UNPROTECTED(dcontext);
+    ASSERT(check_should_be_protected(DATASEC_RARELY_PROT));
+    /* CANNOT hold any locks across cache execution, as our thread synch
+     * assumes none are held
+     */
+    ASSERT_OWN_NO_LOCKS();
+
+    DOLOG(2, LOG_INTERP, { 
+        if (wherewasi == WHERE_APP) {
+            LOG(THREAD, LOG_INTERP, 2, "\ninitial dispatch: target = "PFX"\n",
+                dcontext->next_tag);
+            dump_mcontext_callstack(dcontext);
+            dump_mcontext(get_mcontext(dcontext), THREAD, DUMP_NOT_XML);
+        }
+    });
+
+    /* We have to perform some tasks with last_exit early, before we
+     * become couldbelinking -- the rest are done in dispatch_exit_fcache().
+     * It's ok to de-reference last_exit since even though deleter may assume
+     * noone has ptrs to it, cannot delete until we're officially out of the
+     * cache, which doesn't happen until enter_couldbelinking -- still kind of
+     * messy that we're violating assumption of no ptrs...
+     */
+
+    if (wherewasi == WHERE_APP) { /* first entrance */
+        ASSERT(dcontext->last_exit == get_starting_linkstub()
+               /* new thread */
+               || IF_WINDOWS_ELSE_0(dcontext->last_exit == get_asynch_linkstub()));
+    } else {
+        ASSERT(dcontext->last_exit != NULL); /* MUST be set, if only to a fake linkstub_t */
+        /* cache last_exit's fragment */
+        dcontext->last_fragment = linkstub_fragment(dcontext, dcontext->last_exit);
+
+        /* If we exited from an indirect branch then dcontext->next_tag
+         * already has the next tag value; otherwise we must set it here,
+         * before we might dive back into the cache for a system call.
+         */
+        if (LINKSTUB_DIRECT(dcontext->last_exit->flags)) {
+            if (INTERNAL_OPTION(cbr_single_stub)) {
+                linkstub_t *nxt =
+                    linkstub_shares_next_stub(dcontext, dcontext->last_fragment,
+                                              dcontext->last_exit);
+                if (nxt != NULL) {
+                    /* must distinguish the two based on eflags */
+                    dcontext->last_exit =
+                        linkstub_cbr_disambiguate(dcontext, dcontext->last_fragment,
+                                                  dcontext->last_exit, nxt);
+                    ASSERT(dcontext->last_fragment ==
+                           linkstub_fragment(dcontext, dcontext->last_exit));
+                    STATS_INC(cbr_disambiguations);
+                }
+            }
+            
+            dcontext->next_tag = EXIT_TARGET_TAG(dcontext, dcontext->last_fragment,
+                                                 dcontext->last_exit);
+        } else {
+            /* get src info from coarse ibl exit into the right place */
+            if (DYNAMO_OPTION(coarse_units) &&
+                is_ibl_sourceless_linkstub((const linkstub_t*)dcontext->last_exit))
+                set_coarse_ibl_exit(dcontext);
+        }
+
+        dispatch_exit_fcache_stats(dcontext);
+        KSTOP_NOT_MATCHING(dispatch_num_exits);
+    }
+    KSTART(dispatch_num_exits); /* KSWITCHed next time around for a better explanation */
+
+    if (wherewasi != WHERE_APP) { /* if not first entrance */
+        if (get_at_syscall(dcontext))
+            handle_post_system_call(dcontext);
+
+        /* A non-ignorable syscall or cb return ending a bb must be acted on
+         * We do it here to avoid becoming couldbelinking twice.
+         *
+         */
+        if (TESTANY(LINK_NI_SYSCALL_ALL, dcontext->last_exit->flags)
+            IF_CLIENT_INTERFACE(|| instrument_invoke_another_syscall(dcontext))) {
+            handle_system_call(dcontext);
+            /* will return here if decided to skip the syscall; else, back to dispatch() */
+        }
+#ifdef WINDOWS
+        else if (TEST(LINK_CALLBACK_RETURN, dcontext->last_exit->flags)) {
+            handle_callback_return(dcontext);
+            ASSERT_NOT_REACHED();
+        }
+#endif
+
+        /* Case 8177: If we have a flushed fragment hit a self-write, we cannot
+         * delete it in our self-write handler (b/c of case 3559's incoming links
+         * union).  But, our self-write handler needs to be nolinking and needs to
+         * check sandbox2ro_threshold.  So, we do our self-write check first, but we
+         * don't actually delete there for FRAG_WAS_DELETED fragments.
+         */
+        if (TEST(LINK_SELFMOD_EXIT, dcontext->last_exit->flags)) {
+            SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+            /* this fragment overwrote its original memory image */
+            fragment_self_write(dcontext);
+            /* FIXME: optimize this to stay writable if we're going to
+             * be exiting dispatch as well -- no very quick check though
+             */
+            SELF_PROTECT_LOCAL(dcontext, READONLY);
+        }
+    }
+
+    /* make sure to tell flushers that we are now going to be mucking
+     * with link info
+     */
+    if (!enter_couldbelinking(dcontext, dcontext->last_fragment, true)) {
+        LOG(THREAD, LOG_DISPATCH, 2, "Just flushed last_fragment\n");
+        /* last_fragment flushed, but cannot access here to copy it
+         * to fake linkstub_t, so assert that callee did (either when freeing or
+         * when noticing pending deletion flag)
+         */
+        ASSERT(LINKSTUB_FAKE(dcontext->last_exit));
+    }
+
+    if (wherewasi != WHERE_APP) { /* if not first entrance */
+        /* now fully process the last cache exit as couldbelinking */
+        dispatch_exit_fcache(dcontext);
+    }
+}
+
+/* Processing of the last exit from the cache.
+ * Invariant: dcontext->last_exit != NULL, though it may be a sentinel (see below).
+ *
+ * Note that the last exit and its owning fragment may be _fake_, i.e., just
+ * a copy of the key fields we typically check, for the following cases:
+ *   - last fragment was flushed: fully deleted at cache exit synch point
+ *   - last fragment was deleted since it overwrote itself (selfmod)
+ *   - last fragment was deleted since it was a private trace building copy
+ *   - last fragment was deleted for other reasons?!?
+ *   - briefly during trace emitting, nobody should care though
+ *   - coarse grain fragment exits, for which we have no linkstub_t or other
+ *     extraneous bookkeeping
+ *
+ * For some cases we do not currently keep the key fields at all:
+ *   - last fragment was flushed: detected at write fault
+ * And some times we are unable to keep the key fields:
+ *   - last fragment was flushed: targeted in ibl via target_deleted path
+ * These last two cases are the only exits from fragment for which we
+ * do not know the key fields.  For the former, we exit in the middle of
+ * a fragment that was already created, so not knowing does not affect
+ * security policies or other checks much.  The latter is the most problematic,
+ * as we have a number of checks depending on knowing the last exit when indirect.
+ *
+ * We have other types of exits from the cache that never involved a real
+ * fragment, for which we also use fake linkstubs:
+ *   - no real last fragment: system call
+ *   - no real last fragment: sigreturn
+ *   - no real last fragment: native_exec return
+ *   - callbacks clear last_exit, but should come out of the cache at a syscall
+ *       (bug 2464 was back when tried to carry last_exit through syscall)
+ *       so this will end up looking like the system call case
+ */
+static void
+dispatch_exit_fcache(dcontext_t *dcontext)
+{
+    /* case 7966: no distinction of islinking-ness for hotp_only & thin_client */
+    ASSERT(RUNNING_WITHOUT_CODE_CACHE() || is_couldbelinking(dcontext));
+
+#ifdef NATIVE_RETURN
+    if (in_fcache(dcontext->next_tag)) {
+        /* when we unlink a ret, it ends up coming back here
+         * with the code cache target as next_tag.
+         * we translate that to an app pc here.
+         */
+        app_pc tgt_app_pc = ret_tgt_cache_to_app(dcontext,
+                                                 (cache_pc)dcontext->next_tag);
+        LOG(THREAD, LOG_DISPATCH, 3, "next_tag is in cache: "PFX"\n",
+            dcontext->next_tag);
+        LOG(THREAD, LOG_DISPATCH, 3, "next_tag "PFX" -> app ret addr "PFX"\n",
+            dcontext->next_tag, tgt_app_pc);
+        /* set last_retaddr for fixup_last_cti, if building trace */
+        dcontext->last_retaddr = dcontext->next_tag;
+        dcontext->next_tag = tgt_app_pc;
+        ASSERT(dcontext->last_exit->ret_pc != NULL);
+    }
+#endif
+
+    if (LINKSTUB_INDIRECT(dcontext->last_exit->flags)) {
+        /* indirect branch exit processing */
+
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+        /* PR 204770: use trace component bb tag for RCT source address */
+        app_pc src_tag = dcontext->last_fragment->tag;
+        if (!LINKSTUB_FAKE(dcontext->last_exit) &&
+            TEST(FRAG_IS_TRACE, dcontext->last_fragment->flags)) {
+            /* FIXME: should we call this for direct exits as well, up front? */
+            src_tag = get_trace_exit_component_tag
+                (dcontext, dcontext->last_fragment, dcontext->last_exit);
+        }
+#endif
+
+#ifdef RETURN_AFTER_CALL
+        /* This is the permission check for any new return target, it
+         * also double checks the findings of the indirect lookup
+         * routine
+         */
+        if (dynamo_options.ret_after_call && TEST(LINK_RETURN, dcontext->last_exit->flags)) {
+            /* ret_after_call will raise a security violation on failure */
+            SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+            ret_after_call_check(dcontext, dcontext->next_tag, src_tag);
+            SELF_PROTECT_LOCAL(dcontext, READONLY);
+        }
+#endif /* RETURN_AFTER_CALL */
+
+#ifdef RCT_IND_BRANCH
+        /* permission check for any new indirect call or jump target */
+        /* we care to detect violations only if blocking or at least
+         * reporting the corresponding branch types
+         */
+        if (TESTANY(OPTION_REPORT|OPTION_BLOCK, DYNAMO_OPTION(rct_ind_call)) ||
+            TESTANY(OPTION_REPORT|OPTION_BLOCK, DYNAMO_OPTION(rct_ind_jump))) {
+            if ((TEST(LINK_CALL, dcontext->last_exit->flags)
+                 && TESTANY(OPTION_REPORT|OPTION_BLOCK, DYNAMO_OPTION(rct_ind_call))) || 
+                (TEST(LINK_JMP, dcontext->last_exit->flags) 
+                 && TESTANY(OPTION_REPORT|OPTION_BLOCK, DYNAMO_OPTION(rct_ind_jump)))
+                ) {
+                /* case 4995: current shared syscalls implementation
+                 * reuses the indirect jump table and marks its
+                 * fake linkstub as such.
+                 */
+                if (LINKSTUB_FAKE(dcontext->last_exit) /* quick check */ &&
+                    IS_SHARED_SYSCALLS_LINKSTUB(dcontext->last_exit)) {
+                    ASSERT(IF_WINDOWS_ELSE(DYNAMO_OPTION(shared_syscalls), false));
+                    ASSERT(TEST(LINK_JMP, dcontext->last_exit->flags)); 
+                } else {
+                    /* rct_ind_branch_check will raise a security violation on failure */
+                    rct_ind_branch_check(dcontext, dcontext->next_tag, src_tag);
+                }
+            }
+        }
+#endif /* RCT_IND_BRANCH */
+
+        /* update IBL target tables for any indirect branch exit */
+        SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+        /* update IBL target table if target is a valid IBT */
+        /* FIXME: This is good for modularity but adds
+         * extra lookups in the fragment table.  If it is
+         * performance problem can we do it better?
+         * Probably best to get bb2bb to work better and
+         * not worry about optimizing DR code.
+         */
+        fragment_add_ibl_target(dcontext, dcontext->next_tag, 
+                                extract_branchtype(dcontext->last_exit->flags));
+        /* FIXME: optimize this to stay writable if we're going to
+         * be building a bb as well -- no very quick check though
+         */
+        SELF_PROTECT_LOCAL(dcontext, READONLY);
+    } /* LINKSTUB_INDIRECT */
+
+    /* ref bug 2323, we need monitor to restore last fragment now, 
+     * before we break out of the loop to build a new fragment
+     * ASSUMPTION: all unusual cache exits (asynch events) abort the current
+     * trace, so this is the only place we need to restore anything.
+     * monitor_cache_enter() asserts that for us.
+     * NOTE : we wait till after the cache exit stats and logs to call 
+     * monitor_cache_exit since it might change the flags of the last 
+     * fragment and screw up the stats
+     */
+    monitor_cache_exit(dcontext);
+
+#ifdef SIDELINE
+    /* sideline synchronization */
+    if (dynamo_options.sideline) {
+        thread_id_t tid = get_thread_id();
+        if (pause_for_sideline == tid) {
+            mutex_lock(&sideline_lock);
+            if (pause_for_sideline == tid) {
+                LOG(THREAD, LOG_DISPATCH|LOG_THREADS|LOG_SIDELINE, 2,
+                    "Thread %d waiting for sideline thread\n", tid);
+                signal_event(paused_for_sideline_event);
+                STATS_INC(num_wait_sideline);
+                wait_for_event(resume_from_sideline_event);
+                mutex_unlock(&sideline_lock);
+                LOG(THREAD, LOG_DISPATCH|LOG_THREADS|LOG_SIDELINE, 2,
+                    "Thread %d resuming after sideline thread\n", tid);
+                sideline_cleanup_replacement(dcontext);
+            } else
+                mutex_unlock(&sideline_lock);
+        }
+    }
+#endif
+
+#ifdef LINUX
+    if (dcontext->signals_pending) {
+        if (TEST(FRAG_HAS_SYSCALL, dcontext->last_fragment->flags)) {
+            /* restore syscall (they're a barrier to signals, so signal
+             * handler has cur frag exit before it does a syscall)
+             */
+            mangle_syscall_code(dcontext, dcontext->last_fragment,
+                                EXIT_CTI_PC(dcontext->last_fragment, dcontext->last_exit),
+                                true/*skip exit cti*/);
+        }
+        /* FIXME: can overflow app stack if stack up too many signals
+         * by interrupting prev handlers -- exacerbated by RAC lack of
+         * caching (case 1858), which causes a cache exit prior to
+         * executing every single sigreturn!
+         */
+        receive_pending_signal(dcontext);
+    }
+#endif
+
+#ifdef CLIENT_INTERFACE
+    /* is ok to put the lock after the null check, this is only 
+     * place they can be deleted 
+     */
+    if (dcontext->client_data != NULL && dcontext->client_data->to_do != NULL) {
+        client_todo_list_t *todo;
+        /* FIXME PR 200409: we're removing all API routines that use this
+         * todo list so we should never get here
+         */
+        if (SHARED_FRAGMENTS_ENABLED()) {
+            USAGE_ERROR("CLIENT_INTERFACE incompatible with -shared_{bbs,traces}"
+                        " at this time");
+        }
+# ifdef CLIENT_SIDELINE
+        mutex_lock(&(dcontext->client_data->sideline_mutex));
+# endif
+        todo = dcontext->client_data->to_do;
+        while (todo != NULL) {
+            client_todo_list_t *next_todo = todo->next;
+            fragment_t *f = fragment_lookup(dcontext, todo->tag);
+            if (f != NULL) {
+                if (todo->ilist != NULL) {
+                    /* doing a replacement */
+                    fragment_t * new_f;
+                    uint orig_flags = f->flags;
+                    void *vmlist = NULL;
+                    DEBUG_DECLARE(bool ok;)
+                    LOG(THREAD, LOG_INTERP, 3,
+                        "Going to do a client fragment replacement at "PFX"  F%d\n",
+                        f->tag, f->id);
+                    /* prevent emit from deleting f, we still need it */
+                    /* FIXME: if f is shared we must hold change_linking_lock
+                     * for the flags and vm area operations here
+                     */
+                    ASSERT(!TEST(FRAG_SHARED, f->flags));
+                    f->flags |= FRAG_CANNOT_DELETE;
+                    DEBUG_DECLARE(ok =)
+                        vm_area_add_to_list(dcontext, f->tag, &vmlist, orig_flags, f,
+                                            false/*no locks*/);
+                    ASSERT(ok); /* should never fail for private fragments */
+                    mangle(dcontext, todo->ilist, f->flags, true, true);
+                    new_f = emit_invisible_fragment(dcontext, todo->tag, todo->ilist,
+                                                    orig_flags, vmlist);
+                    f->flags = orig_flags; /* FIXME: ditto about change_linking_lock */
+                    instrlist_clear_and_destroy(dcontext, todo->ilist);
+                    fragment_copy_data_fields(dcontext, f, new_f);
+                    shift_links_to_new_fragment(dcontext, f, new_f);
+                    fragment_replace(dcontext, f, new_f);
+                    DOLOG(2, LOG_INTERP, {
+                        LOG(THREAD, LOG_INTERP, 3,
+                            "Finished emitting replacement fragment %d\n", new_f->id);
+                        disassemble_fragment(dcontext, new_f, stats->loglevel < 3);
+                    });
+                }
+                /* delete [old] fragment */
+                if ((f->flags & FRAG_CANNOT_DELETE) == 0) {
+                    uint actions;
+                    LOG(THREAD, LOG_INTERP, 3, "Client deleteing old F%d\n", f->id);
+                    if (todo->ilist != NULL) {
+                        /* for the fragment replacement case, the fragment should
+                         * already be unlinked and removed from the hash table.
+                         */
+                        actions = FRAGDEL_NO_UNLINK | FRAGDEL_NO_HTABLE;
+                    }
+                    else {
+                        actions = FRAGDEL_ALL;
+                    }
+                    fragment_delete(dcontext, f, actions);
+
+                    STATS_INC(num_fragments_deleted_client);
+                } else {
+                    LOG(THREAD, LOG_INTERP, 2, "Couldn't let client delete F%d\n",
+                        f->id);
+                }
+            } else {
+                LOG(THREAD, LOG_INTERP, 2,
+                    "Failed to delete/replace fragment at tag "PFX" because was already deleted",
+                    todo->tag);
+            }
+
+            HEAP_TYPE_FREE(dcontext, todo, client_todo_list_t, ACCT_OTHER, UNPROTECTED);
+            todo = next_todo;
+        }
+        dcontext->client_data->to_do = NULL;
+# ifdef CLIENT_SIDELINE
+        mutex_unlock(&(dcontext->client_data->sideline_mutex));
+# endif
+    }
+#endif /* CLIENT_INTERFACE */
+}
+
+/* stats and logs on why we exited the code cache */
+static void
+dispatch_exit_fcache_stats(dcontext_t *dcontext)
+{
+#if defined(DEBUG) || defined(KSTATS)
+    fragment_t *next_f;
+    fragment_t *last_f;
+    fragment_t coarse_f;
+#endif
+
+#ifdef PROFILE_RDTSC
+    if (dynamo_options.profile_times) {
+        int i,j;
+        uint64 end_time, total_time;
+        profile_fragment_dispatch(dcontext);
+
+        /* top ten cache times */
+        end_time = get_time();
+        total_time = end_time - dcontext->cache_enter_time;
+        for (i=0; i<10; i++) {
+            if (total_time > dcontext->cache_time[i]) {
+                /* insert */
+                for (j=9; j>i; j--) {
+                    dcontext->cache_time[j] = dcontext->cache_time[j-1];
+                    dcontext->cache_count[j] = dcontext->cache_count[j-1];
+                }
+                dcontext->cache_time[i] = total_time;
+                dcontext->cache_count[i] = dcontext->cache_frag_count;
+                break;
+            }
+        }
+    }
+#endif
+
+#if defined(DEBUG) || defined(KSTATS)
+    STATS_INC(num_exits);
+    ASSERT(dcontext->last_exit != NULL);
+
+    /* special exits that aren't from real fragments */
+
+    if (dcontext->last_exit == get_syscall_linkstub()) {
+        LOG(THREAD, LOG_DISPATCH, 2, "Exit from system call\n");
+        STATS_INC(num_exits_syscalls);
+# ifdef CLIENT_INTERFACE
+        /* PR 356503: clients using libraries that make syscalls, invoked from
+         * a clean call, will not trigger the whereami check below: so we
+         * locate here via mismatching kstat top-of-stack.
+         */
+        KSTAT_THREAD(fcache_default, {
+            if (ks->node[ks->depth - 1].var == pv) {
+                found_client_sysenter();
+            }
+        });
+# endif
+        KSTOP_NOT_PROPAGATED(syscall_fcache);
+        return;
+    }
+    else if (dcontext->last_exit == get_selfmod_linkstub()) {
+        LOG(THREAD, LOG_DISPATCH, 2, "Exit from fragment that self-flushed via code mod\n");
+        STATS_INC(num_exits_code_mod_flush);
+        KSWITCH_STOP_NOT_PROPAGATED(fcache_default);
+        return;
+    }
+    else if (dcontext->last_exit == get_ibl_deleted_linkstub()) {
+        LOG(THREAD, LOG_DISPATCH, 2, "Exit from fragment deleted but hit in ibl\n");
+        STATS_INC(num_exits_ibl_deleted);
+        KSWITCH_STOP_NOT_PROPAGATED(fcache_default);
+        return;
+    }
+# ifdef LINUX
+    else if (dcontext->last_exit == get_sigreturn_linkstub()) {
+        LOG(THREAD, LOG_DISPATCH, 2, "Exit from sigreturn, or os_forge_exception\n");
+        STATS_INC(num_exits_sigreturn);
+        KSTOP_NOT_MATCHING_NOT_PROPAGATED(syscall_fcache);
+        return;
+    }
+# else /* WINDOWS */
+    else if (dcontext->last_exit == get_asynch_linkstub()) {
+        LOG(THREAD, LOG_DISPATCH, 2, "Exit from asynch event\n");
+        STATS_INC(num_exits_asynch);
+        /* w/ -shared_syscalls can also be a fragment kstart */
+        KSTOP_NOT_MATCHING_NOT_PROPAGATED(syscall_fcache);
+        return;
+    }
+# endif
+    else if (dcontext->last_exit == get_native_exec_linkstub()) {
+        LOG(THREAD, LOG_DISPATCH, 2, "Exit from native_exec execution\n");
+        STATS_INC(num_exits_native_exec);
+        /* may be a quite large kstat count */
+        KSWITCH_STOP_NOT_PROPAGATED(native_exec_fcache);
+        return;
+    }
+    else if (dcontext->last_exit == get_native_exec_syscall_linkstub()) {
+        LOG(THREAD, LOG_DISPATCH, 2, "Exit from native_exec syscall trampoline\n");
+        STATS_INC(num_exits_native_exec_syscall);
+        /* may be a quite large kstat count */
+        KSWITCH_STOP_NOT_PROPAGATED(native_exec_fcache);
+        return;
+    }
+    else if (dcontext->last_exit == get_reset_linkstub()) {
+        LOG(THREAD, LOG_DISPATCH, 2, "Exit due to proactive reset\n");
+        STATS_INC(num_exits_reset);
+        KSWITCH_STOP_NOT_PROPAGATED(fcache_default);
+        return;
+    }
+# ifdef WINDOWS
+    else if (IS_SHARED_SYSCALLS_UNLINKED_LINKSTUB(dcontext->last_exit)) {
+        LOG(THREAD, LOG_DISPATCH, 2,
+            "Exit from unlinked shared syscall\n");
+        STATS_INC(num_unlinked_shared_syscalls_exits);
+        KSWITCH_STOP_NOT_PROPAGATED(fcache_default);
+        return;
+    }
+    else if (IS_SHARED_SYSCALLS_LINKSTUB(dcontext->last_exit)) {
+        LOG(THREAD, LOG_DISPATCH, 2, "Exit from shared syscall (%s)\n",
+            IS_SHARED_SYSCALLS_TRACE_LINKSTUB(dcontext->last_exit) ?
+            "trace" : "bb");
+        DOSTATS({
+            if (IS_SHARED_SYSCALLS_TRACE_LINKSTUB(dcontext->last_exit))
+                STATS_INC(num_shared_syscalls_trace_exits);
+            else
+                STATS_INC(num_shared_syscalls_bb_exits);
+        });
+        KSWITCH_STOP_NOT_PROPAGATED(fcache_default);
+        return;
+    }
+# endif
+# ifdef HOT_PATCHING_INTERFACE
+    else if (dcontext->last_exit == get_hot_patch_linkstub()) {
+        LOG(THREAD, LOG_DISPATCH, 2, "Exit from hot patch routine\n");
+        STATS_INC(num_exits_hot_patch);
+        KSWITCH_STOP_NOT_PROPAGATED(fcache_default);
+        return;
+    }
+# endif
+# ifdef CLIENT_INTERFACE
+    else if (dcontext->last_exit == get_client_linkstub()) {
+        LOG(THREAD, LOG_DISPATCH, 2, "Exit from client redirection\n");
+        STATS_INC(num_exits_client_redirect);
+        KSWITCH_STOP_NOT_PROPAGATED(fcache_default);
+        return;
+    }
+# endif
+
+    /* normal exits from real fragments, though the last_fragment may
+     * be deleted and we are working off a copy of its important fields
+     */
+
+    /* FIXME: this lookup is needed for KSTATS and STATS_*. STATS_* are only
+     * printed at loglevel 1, but maintained at loglevel 0, and if
+     * we want an external agent to examine them at 0 we will want
+     * to keep this...leaving for now
+     */
+    next_f = fragment_lookup_fine_and_coarse(dcontext, dcontext->next_tag,
+                                             &coarse_f, dcontext->last_exit);
+    last_f = dcontext->last_fragment;
+
+    DOKSTATS({
+        /* FIXME (case 4988): read top of kstats stack to get src
+         * type, and then split by last_fragment type as well
+         */
+        KSWITCH_STOP_NOT_PROPAGATED(fcache_default);
+    });
+
+    if (is_ibl_sourceless_linkstub((const linkstub_t*)dcontext->last_exit)) {
+        if (DYNAMO_OPTION(coarse_units)) {
+            LOG(THREAD, LOG_DISPATCH, 2, "Exit from coarse ibl from tag "PFX": %s %s",
+                dcontext->coarse_exit.src_tag,
+                TEST(FRAG_IS_TRACE, last_f->flags) ? "trace" : "bb",
+                TEST(LINK_RETURN, dcontext->last_exit->flags) ? "ret" :
+                TEST(LINK_CALL, dcontext->last_exit->flags) ? "call*" : "jmp*");
+        } else {
+            ASSERT(!DYNAMO_OPTION(indirect_stubs));
+            LOG(THREAD, LOG_DISPATCH, 2, "Exit from sourceless ibl: %s %s",
+                TEST(FRAG_IS_TRACE, last_f->flags) ? "trace" : "bb",
+                TEST(LINK_RETURN, dcontext->last_exit->flags) ? "ret" :
+                TEST(LINK_CALL, dcontext->last_exit->flags) ? "call*" : "jmp*");
+        }
+    } else if (dcontext->last_exit == get_coarse_exit_linkstub()) {
+        DOLOG(2, LOG_DISPATCH, {
+            coarse_info_t *info = dcontext->coarse_exit.dir_exit;
+            cache_pc stub;
+            ASSERT(info != NULL); /* though not initialized to NULL... */
+            stub = coarse_stub_lookup_by_target(dcontext, info, dcontext->next_tag);
+            LOG(THREAD, LOG_DISPATCH, 2,
+                "Exit from sourceless coarse-grain fragment via stub "PFX"\n", stub);
+        });
+        /* FIXME: this stat is not mutually exclusive of reason-for-exit stats */
+        STATS_INC(num_exits_coarse);
+    }
+    else if (dcontext->last_exit == get_coarse_trace_head_exit_linkstub()) {
+        LOG(THREAD, LOG_DISPATCH, 2,
+            "Exit from sourceless coarse-grain fragment targeting trace head");
+        /* FIXME: this stat is not mutually exclusive of reason-for-exit stats */
+        STATS_INC(num_exits_coarse_trace_head);
+    } else {
+        LOG(THREAD, LOG_DISPATCH, 2, "Exit from F%d("PFX")."PFX, 
+            last_f->id, last_f->tag, EXIT_CTI_PC(dcontext->last_fragment,
+                                                 dcontext->last_exit));
+    }
+
+    DOSTATS({
+        if (TEST(FRAG_IS_TRACE, last_f->flags))
+            STATS_INC(num_trace_exits);
+        else
+            STATS_INC(num_bb_exits);
+    });
+
+    LOG(THREAD, LOG_DISPATCH, 2, "%s",
+        TEST(FRAG_SHARED, last_f->flags) ? " (shared)":"");
+    DOLOG(2, LOG_SYMBOLS, { 
+        char symbuf[MAXIMUM_SYMBOL_LENGTH];
+        print_symbolic_address(last_f->tag, symbuf, sizeof(symbuf), true);
+        LOG(THREAD, LOG_SYMBOLS, 2, "\t%s\n", symbuf);
+    });
+
+# if defined(DEBUG) && defined(DGC_DIAGNOSTICS)
+    if (TEST(FRAG_DYNGEN, last_f->flags) && !is_dyngen_vsyscall(last_f->tag)) {
+        char buf[MAXIMUM_SYMBOL_LENGTH];
+        bool stack = is_address_on_stack(dcontext, last_f->tag);
+        app_pc translated_pc;
+        print_symbolic_address(dcontext->next_tag, buf, sizeof(buf), false);
+        LOG(THREAD, LOG_DISPATCH, 1,
+            "Exit from dyngen F%d("PFX"%s%s) w/ %s targeting "PFX" %s:",
+            last_f->id, last_f->tag, stack ? " stack":"",
+            (last_f->flags & FRAG_DYNGEN_RESTRICTED) != 0 ? " BAD":"",
+            LINKSTUB_DIRECT(dcontext->last_exit->flags) ? "db":"ib",
+            dcontext->next_tag, buf);
+        /* FIXME: risky if last fragment is deleted -- should check for that
+         * here and instead just print type from last_exit, since recreate
+         * may fail
+         */
+        translated_pc = recreate_app_pc(dcontext, EXIT_CTI_PC(dcontext->last_fragment,
+                                                              dcontext->last_exit),
+                                        dcontext->last_fragment);
+        if (translated_pc != NULL) {
+            disassemble(dcontext, translated_pc, THREAD);
+            LOG(THREAD, LOG_DISPATCH, 1, "\n");
+        }
+        DOLOG(stack ? 1U : 2U, LOG_DISPATCH, {
+            LOG(THREAD, LOG_DISPATCH, 1, "DGC bb:\n");
+            disassemble_app_bb(dcontext, last_f->tag, THREAD);
+        });
+    }
+# endif /* defined(DEBUG) && defined(DGC_DIAGNOSTICS) */
+
+    if (LINKSTUB_INDIRECT(dcontext->last_exit->flags)) {
+#ifdef RETURN_AFTER_CALL
+        bool ok = false;
+#endif
+        STATS_INC(num_exits_ind_total);
+        if (next_f == NULL) {
+            LOG(THREAD, LOG_DISPATCH, 2, " (target "PFX" not in cache)",
+                dcontext->next_tag);
+            STATS_INC(num_exits_ind_good_miss);
+            KSWITCH(num_exits_ind_good_miss);
+        } else if (is_building_trace(dcontext) &&
+                   !TEST(LINK_LINKED, dcontext->last_exit->flags)) {
+            LOG(THREAD, LOG_DISPATCH, 2, " (in trace-building mode)");
+            STATS_INC(num_exits_ind_trace_build);
+        } else if (TEST(FRAG_WAS_DELETED, last_f->flags) ||
+                   !INTERNAL_OPTION(link_ibl)) {
+            LOG(THREAD, LOG_DISPATCH, 2, " (src unlinked)");
+            STATS_INC(num_exits_ind_src_unlinked);
+        } else {
+            LOG(THREAD, LOG_DISPATCH, 2, " (target "PFX" in cache but not lookup table)",
+                dcontext->next_tag);
+            STATS_INC(num_exits_ind_bad_miss);
+
+            if (TEST(FRAG_IS_TRACE, last_f->flags)) {
+                STATS_INC(num_exits_ind_bad_miss_trace);
+                if (next_f && TEST(FRAG_IS_TRACE, next_f->flags)) {
+                    STATS_INC(num_exits_ind_bad_miss_trace2trace);
+                    KSWITCH(num_exits_ind_bad_miss_trace2trace);
+                } else if (next_f &&
+                           !TEST(FRAG_IS_TRACE, next_f->flags)) {
+                    if (!TEST(FRAG_IS_TRACE_HEAD, next_f->flags)) {
+                        STATS_INC(num_exits_ind_bad_miss_trace2bb_nth);
+                        KSWITCH(num_exits_ind_bad_miss_trace2bb_nth);
+                    } else {
+                        STATS_INC(num_exits_ind_bad_miss_trace2bb_th);
+                        KSWITCH(num_exits_ind_bad_miss_trace2bb_th);
+                    }
+                }
+            }
+            else {
+                STATS_INC(num_exits_ind_bad_miss_bb);
+                if (next_f && TEST(FRAG_IS_TRACE, next_f->flags)) {
+                    STATS_INC(num_exits_ind_bad_miss_bb2trace);
+                    KSWITCH(num_exits_ind_bad_miss_bb2trace);
+                } else if (next_f &&
+                           !TEST(FRAG_IS_TRACE, next_f->flags)) {
+                    DOSTATS({
+                        if (TEST(FRAG_IS_TRACE_HEAD, next_f->flags))
+                            STATS_INC(num_exits_ind_bad_miss_bb2bb_th);
+                    });
+                    STATS_INC(num_exits_ind_bad_miss_bb2bb);
+                    KSWITCH(num_exits_ind_bad_miss_bb2bb);
+                }
+            }
+        }
+        DOSTATS({
+            if (!TEST(FRAG_IS_TRACE, last_f->flags))
+                STATS_INC(num_exits_ind_non_trace);
+        });
+# ifdef RETURN_AFTER_CALL
+        /* split by ind branch type */
+        if (TEST(LINK_RETURN, dcontext->last_exit->flags)) {
+            LOG(THREAD, LOG_DISPATCH, 2, " (return from "PFX" non-trace tgt "PFX")",
+                EXIT_CTI_PC(dcontext->last_fragment, dcontext->last_exit),
+                dcontext->next_tag);
+            STATS_INC(num_exits_ret);
+            DOSTATS({
+                if (TEST(FRAG_IS_TRACE, last_f->flags))
+                    STATS_INC(num_exits_ret_trace);
+            });
+        }
+        else if (TESTANY(LINK_CALL|LINK_JMP, dcontext->last_exit->flags)) {
+            LOG(THREAD, LOG_DISPATCH, 2, " (ind %s from "PFX" non-trace tgt "PFX")",
+                TEST(LINK_CALL, dcontext->last_exit->flags) ? "call" : "jmp",
+                EXIT_CTI_PC(dcontext->last_fragment, dcontext->last_exit),
+                dcontext->next_tag);
+            DOSTATS({
+                if (TEST(LINK_CALL, dcontext->last_exit->flags)) {
+                    STATS_INC(num_exits_ind_call);
+                } else if (TEST(LINK_JMP, dcontext->last_exit->flags)) {
+                    STATS_INC(num_exits_ind_jmp);
+                } else
+                    ASSERT_NOT_REACHED();
+            });
+        } else if (!ok) {
+            LOG(THREAD, LOG_DISPATCH, 2, 
+                "WARNING: unknown indirect exit from "PFX", in %s fragment "PFX,
+                EXIT_CTI_PC(dcontext->last_fragment, dcontext->last_exit),
+                (TEST(FRAG_IS_TRACE, last_f->flags)) ? "trace" : "bb",
+                last_f);
+            STATS_INC(num_exits_ind_unknown);
+            ASSERT_NOT_REACHED();
+        }
+# endif /* RETURN_AFTER_CALL */
+    } else { /* DIRECT LINK */
+        ASSERT(LINKSTUB_DIRECT(dcontext->last_exit->flags) ||
+               IS_COARSE_LINKSTUB(dcontext->last_exit));
+
+        if (TESTANY(LINK_NI_SYSCALL_ALL,
+                    dcontext->last_exit->flags)) {
+            LOG(THREAD, LOG_DISPATCH, 2, " (block ends with syscall)");
+            STATS_INC(num_exits_dir_syscall);
+            /* FIXME: it doesn't matter whether next_f exists or not we're still in a syscall  */
+            KSWITCH(num_exits_dir_syscall);
+        }
+# ifdef WINDOWS
+        else if (TEST(LINK_CALLBACK_RETURN, dcontext->last_exit->flags)) {
+            LOG(THREAD, LOG_DISPATCH, 2, " (block ends with callback return)");
+            STATS_INC(num_exits_dir_cbret);
+        }
+# endif
+        else if (next_f == NULL) {
+            LOG(THREAD, LOG_DISPATCH, 2, " (target "PFX" not in cache)",
+                dcontext->next_tag);
+            STATS_INC(num_exits_dir_miss);
+            KSWITCH(num_exits_dir_miss);
+        } 
+        /* for SHARED_FRAGMENTS_ENABLED(), we do not grab the change_linking_lock
+         * for our is_linkable call since that leads to a lot of
+         * contention (and we don't want to go to a read-write model
+         * when most uses, and all non-debug uses, are writes!).
+         * instead, since we don't want to change state, we have no synch
+         * at all, which is ok since the state could have changed anyway
+         * (see comment at end of cases below)
+         */
+# ifdef DEBUG
+        else if (IS_COARSE_LINKSTUB(dcontext->last_exit)) {
+            LOG(THREAD, LOG_DISPATCH, 2, " (not lazily linked yet)");
+        }
+        else if (!is_linkable(dcontext, dcontext->last_fragment,
+                              dcontext->last_exit, next_f,
+                              false/*don't own link lock*/,
+                              false/*do not change trace head state*/)) {
+            STATS_INC(num_exits_dir_nolink);
+            LOG(THREAD, LOG_DISPATCH, 2, " (cannot link F%d->F%d)",
+                last_f->id, next_f->id);
+            if (is_building_trace(dcontext) &&
+                !TEST(LINK_LINKED, dcontext->last_exit->flags)) {
+                LOG(THREAD, LOG_DISPATCH, 2, " (in trace-building mode)");
+                STATS_INC(num_exits_dir_trace_build);
+            }
+#  ifndef TRACE_HEAD_CACHE_INCR
+            else if (TEST(FRAG_IS_TRACE_HEAD, next_f->flags)) {
+                LOG(THREAD, LOG_DISPATCH, 2, " (target F%d is trace head)",
+                    next_f->id);
+                STATS_INC(num_exits_dir_trace_head);
+            }
+#  endif
+            else if ((last_f->flags & FRAG_SHARED) != (next_f->flags & FRAG_SHARED)) {
+                LOG(THREAD, LOG_DISPATCH, 2, " (cannot link shared to private)",
+                    last_f->id, next_f->id);
+                STATS_INC(num_exits_dir_nolink_sharing);
+            }
+#  ifdef DGC_DIAGNOSTICS
+            else if ((next_f->flags & FRAG_DYNGEN) != (last_f->flags & FRAG_DYNGEN)) {
+                LOG(THREAD, LOG_DISPATCH, 2, " (cannot link DGC to non-DGC)",
+                    last_f->id, next_f->id);
+            }
+#  endif
+            else if (INTERNAL_OPTION(nolink)) {
+                LOG(THREAD, LOG_DISPATCH, 2, " (nolink option is on)");
+            }
+            else if (!TEST(FRAG_LINKED_OUTGOING, last_f->flags)) {
+                LOG(THREAD, LOG_DISPATCH, 2, " (F%d is unlinked-out)", last_f->id);
+            }
+            else if (!TEST(FRAG_LINKED_INCOMING, next_f->flags)) {
+                LOG(THREAD, LOG_DISPATCH, 2, " (F%d is unlinked-in)", next_f->id);
+            }
+            else {
+                LOG(THREAD, LOG_DISPATCH, 2, " (unknown reason)");
+                /* link info could have changed after we exited cache so
+                 * this is probably not a problem, not much we can do
+                 * to distinguish race from real problem, so no assertion.
+                 * race can happen even w/ single_thread_in_DR.
+                 */
+                STATS_INC(num_exits_dir_race);
+            }
+        }
+#  ifdef TRACE_HEAD_CACHE_INCR
+        else if (TEST(FRAG_IS_TRACE_HEAD, next_f->flags)) {
+            LOG(THREAD, LOG_DISPATCH, 2, " (trace head F%d now hot!)", next_f->id);
+            STATS_INC(num_exits_dir_trace_hot);
+        }
+#  endif
+        else if (TEST(FRAG_IS_TRACE, next_f->flags) && TEST(FRAG_SHARED, last_f->flags)) {
+            LOG(THREAD, LOG_DISPATCH, 2,
+                " (shared trace head shadowed by private trace F%d)", next_f->id);
+            STATS_INC(num_exits_dir_nolink_sharing);
+        }
+        else if (dcontext->next_tag == last_f->tag && next_f != last_f) {
+            /* invisible emission and replacement */
+            LOG(THREAD, LOG_DISPATCH, 2, " (self-loop in F%d, replaced by F%d)",
+                last_f->id, next_f->id);
+            STATS_INC(num_exits_dir_self_replacement);
+        }
+#  ifdef LINUX
+        else if (dcontext->signals_pending) {
+            /* this may not always be the reason...the interrupted fragment
+             * field is modularly hidden in linux/signal.c though
+             */
+            LOG(THREAD, LOG_DISPATCH, 2, " (interrupted by delayable signal)");
+            STATS_INC(num_exits_dir_signal);
+        }
+#  endif
+#  ifdef NATIVE_RETURN
+        else if (TEST(FRAG_IS_TRACE_HEAD, next_f->flags)) {
+            LOG(THREAD, LOG_DISPATCH, 2, " (returned to old trace head)");
+        }
+#  endif
+        else if (TEST(FRAG_COARSE_GRAIN, next_f->flags) &&
+                 !TEST(FRAG_COARSE_GRAIN, last_f->flags)) {
+            LOG(THREAD, LOG_DISPATCH, 2, " (fine fragment targeting coarse trace head)");
+            /* FIXME: We would assert that FRAG_IS_TRACE_HEAD is set, but
+             * we have no way of setting that up for fine to coarse links
+             */
+            /* stats are done in monitor_cache_enter() */
+        }
+        else {
+            LOG(THREAD, LOG_DISPATCH, 2,
+                " (UNKNOWN DIRECT EXIT F%d."PFX"->F%d)",
+                last_f->id, EXIT_CTI_PC(dcontext->last_fragment, dcontext->last_exit),
+                next_f->id);
+            /* link info could have changed after we exited cache so
+             * this is probably not a problem, not much we can do
+             * to distinguish race from real problem, so no assertion.
+             * race can happen even w/ single_thread_in_DR.
+             */
+            STATS_INC(num_exits_dir_race);
+        }
+# endif /* DEBUG */
+    }
+    if (dcontext->last_exit == get_deleted_linkstub(dcontext)) {
+        LOG(THREAD, LOG_DISPATCH, 2, " (fragment was flushed)");
+    } 
+    LOG(THREAD, LOG_DISPATCH, 2, "\n");
+    DOLOG(5, LOG_DISPATCH, {
+        dump_mcontext(get_mcontext(dcontext), THREAD, DUMP_NOT_XML); });
+    DOLOG(6, LOG_DISPATCH, { dump_mcontext_callstack(dcontext); });
+    DOKSTATS({ DOLOG(6, LOG_DISPATCH, { kstats_dump_stack(dcontext); }); });
+# ifdef RETURN_STACK
+    if (stats->loglevel > 2 && (stats->logmask & LOG_DISPATCH) != 0)
+        print_return_stack(dcontext);
+# endif
+# ifdef NATIVE_RETURN_CALLDEPTH
+    LOG(THREAD, LOG_DISPATCH, 3, "\tCall depth is %d\n",
+        dcontext->call_depth);
+# endif
+#endif /* defined(DEBUG) || defined(KSTATS) */
+}
+
+
+/***************************************************************************
+ * SYSTEM CALLS
+ */
+
+#ifdef LINUX
+static void
+adjust_syscall_continuation(dcontext_t *dcontext)
+{
+    /* PR 212570: for linux sysenter, we hooked the sysenter return-to-user-mode
+     * point to go to post-do-vsyscall.  So we end up here w/o any extra
+     * work pre-syscall; and since we put the hook-displaced code in the nop
+     * space immediately after the sysenter instr, which is our normal
+     * continuation pc, we have no work to do here either!
+     */
+    if (get_syscall_method() == SYSCALL_METHOD_SYSENTER) {
+        /* we still see some int syscalls (for SYS_clone in particular) */
+        ASSERT(dcontext->sys_was_int ||
+               dcontext->asynch_target == vsyscall_syscall_end_pc);
+    } else if (vsyscall_syscall_end_pc != NULL &&
+               /* PR 341469: 32-bit apps (LOL64) on AMD hardware have
+                * OP_syscall in a vsyscall page
+                */
+               get_syscall_method() != SYSCALL_METHOD_SYSCALL) {
+        /* If we fail to hook we currently bail out to int; but we then
+         * need to manually jump to the sysenter return point.
+         * Once we have PR 288330 we can remove this.
+         */
+        if (dcontext->asynch_target == vsyscall_syscall_end_pc) {
+            ASSERT(vsyscall_sysenter_return_pc != NULL);
+            dcontext->asynch_target = vsyscall_sysenter_return_pc;
+        }
+    }
+}
+#endif
+
+/* used to execute a system call instruction in the code cache 
+ * dcontext->next_tag is store elsewhere and restored after the system call
+ * for resumption of execution post-syscall
+ */
+void
+handle_system_call(dcontext_t *dcontext)
+{
+    fcache_enter_func_t fcache_enter = get_fcache_enter_private_routine(dcontext);
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    app_pc do_syscall = (app_pc) get_do_syscall_entry(dcontext);
+#ifdef CLIENT_INTERFACE 
+    bool execute_syscall = true;
+#endif
+#ifdef WINDOWS
+    /* make sure to ask about syscall before pre_syscall, which will swap new mc in! */
+    bool use_prev_dcontext = is_cb_return_syscall(dcontext);
+#else
+    if (TEST(LINK_NI_SYSCALL_INT, dcontext->last_exit->flags)) {
+        LOG(THREAD, LOG_SYSCALLS, 2, "Using do_int_syscall\n");
+        do_syscall = (app_pc) get_do_int_syscall_entry(dcontext);
+        /* last_exit will be for the syscall so set a flag (could alternatively
+         * set up a separate exit stub but this is simpler) */
+        dcontext->sys_was_int = true;
+    } else
+        dcontext->sys_was_int = false;
+#endif
+
+#ifdef CLIENT_INTERFACE 
+    /* We invoke here rather than inside pre_syscall() primarily so we can
+     * set use_prev_dcontext(), but also b/c the windows and linux uses
+     * are identical.  We do want this prior to xbp-param changes for linux
+     * sysenter-to-int (PR 313715) since to client this should
+     * look like the original sysenter.  For Windows we could put this
+     * after sysenter handling but it's not clear which is better: we'll
+     * assert if client changes xsp/xdx but that's fine.
+     */
+    if (!instrument_pre_syscall(dcontext, (int) mc->xax)) {
+        /* we won't execute post-syscall so we do not need to store
+         * dcontext->sys_*
+         */
+        execute_syscall = false;
+        LOG(THREAD, LOG_SYSCALLS, 2, "skipping syscall %d on client request\n",
+            mc->xax);
+    }
+# ifdef WINDOWS
+    /* re-set in case client changed the number */
+    use_prev_dcontext = is_cb_return_syscall(dcontext);
+# endif
+#endif
+
+    /* some syscalls require modifying local memory 
+     * FIXME: move this unprot down to those syscalls to avoid unprot-prot-unprot-prot
+     * with the new clean dstack design -- though w/ shared_syscalls perhaps most
+     * syscalls coming through here will need this
+     */
+    SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+
+    KSWITCH(num_exits_dir_syscall); /* encapsulates syscall overhead */
+
+    LOG(THREAD, LOG_SYSCALLS, 2,
+        "Entry into do_syscall to execute a non-ignorable system call\n");
+#ifdef SIDELINE
+    /* clear cur-trace field so we don't think cur trace is still running */
+    sideline_trace = NULL;
+#endif
+
+    /* our flushing design assumes our syscall handlers are nolinking,
+     * to avoid multiple-flusher deadlocks
+     */
+    ASSERT(!is_couldbelinking(dcontext));
+
+    /* we need to store the next pc since entering the fcache will clobber it
+     * with the do_syscall entry point.
+     * we store in a dcontext slot since some syscalls need to view or modify it
+     * (the asynch ones: sigreturn, ntcontinue, etc., hence the name asynch_target).
+     * Yes this works with an NtContinue being interrupted in the kernel for an APC --
+     * we want to know the NtContinue target, there is no other target to remember.
+     * The only problem is if a syscall that modifies asynch_target fails -- then we
+     * want the old value, so we store it here.
+     */
+    dcontext->asynch_target = get_fcache_target(dcontext);
+
+#ifdef WINDOWS
+    if (get_syscall_method() == SYSCALL_METHOD_SYSENTER) {
+        /* kernel sends control directly to 0x7ffe0304 so we need
+         * to mangle the return address
+         */
+        /* Ref case 5461 - edx will become top of stack post-syscall */
+        ASSERT(get_mcontext(dcontext)->xsp == get_mcontext(dcontext)->xdx);
+#ifdef HOT_PATCHING_INTERFACE
+        /* For hotp_only, vsyscall_syscall_end_pc can be NULL as dr will never
+         * interp a system call.  Also, for hotp_only, control can came here 
+         * from native only to do a syscall that was hooked.
+         */
+        ASSERT(!DYNAMO_OPTION(hotp_only) ||
+               (DYNAMO_OPTION(hotp_only) &&
+                dcontext->next_tag == BACK_TO_NATIVE_AFTER_SYSCALL));
+#else
+        ASSERT(vsyscall_syscall_end_pc != NULL);
+#endif
+        /* NOTE - the stack mangling must match that of intercept_nt_continue()
+         * and shared_syscall as not all routines looking at the stack
+         * differentiate. */
+        if (dcontext->asynch_target == vsyscall_syscall_end_pc) {
+#ifdef HOT_PATCHING_INTERFACE
+            /* Don't expect to be here for -hotp_only */
+            ASSERT_CURIOSITY(!DYNAMO_OPTION(hotp_only));
+#endif
+            /* currently pc is the ret after sysenter, we need it to be the return point
+             * (the ret after the call to the vsyscall sysenter)
+             * we do not need to keep the old asynch_target -- if we decide not to do
+             * the syscall we just have to pop the retaddr
+             */
+            dcontext->asynch_target = *((app_pc *)get_mcontext(dcontext)->xsp);
+            ASSERT(dcontext->thread_record->under_dynamo_control);
+        } else {
+            /* else, special case like native_exec_syscall */
+            LOG(THREAD, LOG_ALL, 2, "post-sysenter target is non-vsyscall "PFX"\n",
+                dcontext->asynch_target);
+            ASSERT(DYNAMO_OPTION(native_exec_syscalls) && 
+                   !dcontext->thread_record->under_dynamo_control);
+        }
+        /* FIXME A lack of write access to %esp will generate an exception
+         * originating from DR though it's really an app problem (unless we
+         * screwed up wildly). Should we call is_writeable(%esp) and force
+         * a new UNWRITEABLE_MEMORY_EXECUTION_EXCEPTION so that we don't take
+         * the blame?
+         */
+        if (DYNAMO_OPTION(sygate_sysenter)) {
+            /* So stack looks like
+             * esp +0 app_ret_addr
+             *     +4 app_val1
+             * for the case 5441 Sygate hack the sysenter needs to have a ret
+             * address that's in ntdll.dll, but we also need to redirect
+             * control back to do_syscall. So we mangle to
+             * esp +0 sysenter_ret_address (ret in ntdll)
+             *     +4 after_do_syscall
+             * dc->sysenter_storage app_val1
+             * dc->asynch_target app_ret_addr
+             * After do_syscall we push app_val1 (since stack is popped twice)
+             * and send control to asynch_target (implicitly doing the
+             * post_sysenter ret instr).
+             */
+            dcontext->sysenter_storage = 
+                *((app_pc *)(get_mcontext(dcontext)->xsp+XSP_SZ));
+            *((app_pc *)get_mcontext(dcontext)->xsp) = sysenter_ret_address;
+            *((app_pc *)(get_mcontext(dcontext)->xsp+XSP_SZ)) =
+                after_do_syscall_code(dcontext);
+        } else {
+            *((app_pc *)get_mcontext(dcontext)->xsp) =
+                after_do_syscall_code(dcontext);
+        }
+    }
+#endif
+
+    /* first do the pre-system-call */
+    if (IF_CLIENT_INTERFACE(execute_syscall &&) pre_system_call(dcontext)) {
+        /* now do the actual syscall instruction */
+#ifdef LINUX
+        /* FIXME: move into some routine inside linux/?
+         * if so, move #include of sys/syscall.h too
+         */
+        if (is_clone_thread_syscall(dcontext)) {
+            /* code for after clone is in do_clone_syscall
+             * it expects the pc after the app's syscall instr to be in edi --
+             * the alternative is to modify the do_syscall code dynamically, here!
+             * FIXME PR 286194: currently we do not restore this in the child at all:
+             * better to keep a stack in parent, or use CLONE_SETTLS (PR 285898), or
+             * set up child's TLS ourselves and restore parent's post-syscall?
+             */
+            /* save register in sys_param1, which is not used already in pre/post */
+            dcontext->sys_param1 = mc->CLONE_SCRATCH_REG_MC;
+            mc->CLONE_SCRATCH_REG_MC = (reg_t) dcontext->asynch_target;
+            do_syscall = (app_pc) get_do_clone_syscall_entry(dcontext);
+        } else if (is_sigreturn_syscall(dcontext)) {
+            /* HACK: sigreturn goes straight to fcache_return, which expects
+             * app eax to already be in mcontext.  pre-syscall cannot do that since
+             * do_syscall needs the syscall num in eax!
+             * so we have to do it here (alternative is to be like NtContinue handling
+             * with a special entry point, ends up being same sort of thing as here)
+             */
+            /* pre-sigreturn handler put dest eax in next_tag
+             * save it in sys_param1, which is not used already in pre/post 
+             */
+            /* for CLIENT_INTERFACE, pre-sigreturn handler took eax after
+             * client had chance to change it, so we have the proper value here.
+             */
+            dcontext->sys_param1 = (reg_t) dcontext->next_tag;
+        }
+#else
+        if (use_prev_dcontext) {
+            /* get the current, but now swapped out, dcontext */
+            dcontext_t *tmp_dcontext = dcontext;
+            LOG(THREAD, LOG_SYSCALLS, 1, "handling a callback return\n");
+            dcontext = get_prev_swapped_dcontext(tmp_dcontext);
+            LOG(THREAD, LOG_SYSCALLS, 1, "swapped dcontext from "PFX" to "PFX"\n",
+                tmp_dcontext, dcontext);
+            /* we have special fcache_enter that uses different dcontext,
+             * FIXME: but what if syscall fails?  need to unswap dcontexts!
+             */
+            fcache_enter = get_fcache_enter_indirect_routine(dcontext);
+            /* avoid synch errors with dispatch -- since enter_fcache will set
+             * whereami for prev dcontext, not real one!
+             */
+            tmp_dcontext->whereami = WHERE_FCACHE;
+        }
+#endif
+
+        SELF_PROTECT_LOCAL(dcontext, READONLY);
+
+        set_at_syscall(dcontext, true);
+        KSTART(syscall_fcache); /* stopped in dispatch_exit_fcache_stats */
+        enter_fcache(dcontext, fcache_enter, do_syscall);
+        /* will handle post processing in handle_post_system_call */
+        ASSERT_NOT_REACHED();
+    }
+    else {
+#ifdef CLIENT_INTERFACE 
+        /* give the client its post-syscall event since we won't be calling
+         * post_system_call(), unless the client itself was the one who skipped.
+         */
+        if (execute_syscall) {
+            instrument_post_syscall(dcontext, dcontext->sys_num);
+        }
+#endif
+#ifdef WINDOWS
+        if (get_syscall_method() == SYSCALL_METHOD_SYSENTER) {
+            /* decided to skip syscall -- pop retaddr, restore sysenter storage
+             * (if applicable) and set next target */
+            get_mcontext(dcontext)->xsp += XSP_SZ;
+            if (DYNAMO_OPTION(sygate_sysenter)) {
+                *((app_pc *)get_mcontext(dcontext)->xsp) =
+                    dcontext->sysenter_storage;
+            }
+            set_fcache_target(dcontext, dcontext->asynch_target);
+        }
+#else
+        adjust_syscall_continuation(dcontext);
+        set_fcache_target(dcontext, dcontext->asynch_target);
+#endif
+    }
+    SELF_PROTECT_LOCAL(dcontext, READONLY);
+}
+
+static void
+handle_post_system_call(dcontext_t *dcontext)
+{
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+
+    ASSERT(!is_couldbelinking(dcontext));
+    ASSERT(get_at_syscall(dcontext));
+
+    set_at_syscall(dcontext, false);
+
+    /* some syscalls require modifying local memory */
+    SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+
+#ifdef LINUX
+    /* restore mcontext values prior to invoking instrument_post_syscall() */
+    if (was_clone_thread_syscall(dcontext)) {
+        /* Make sure syscall didn't modify xbp.  The parent comes here, but the child
+         * goes straight to dispatch from new_thread_dynamo_start().
+         */
+        ASSERT(mc->CLONE_SCRATCH_REG_MC == (reg_t)dcontext->asynch_target);
+        mc->CLONE_SCRATCH_REG_MC = dcontext->sys_param1;
+    } else if (was_sigreturn_syscall(dcontext)) {
+        /* restore app xax */
+        mc->xax = dcontext->sys_param1;
+    }
+#endif
+
+    post_system_call(dcontext);
+
+    /* restore state for continuation in instruction after syscall */
+    /* FIXME: need to handle syscall failure -- those that clobbered asynch_target
+     * need to restore it to its previous value, which has to be stored somewhere!
+     */
+#ifdef WINDOWS
+    if (DYNAMO_OPTION(sygate_sysenter) &&
+        get_syscall_method() == SYSCALL_METHOD_SYSENTER) {
+        /* restore sysenter_storage, note stack was popped twice for
+         * syscall so need to push the value */
+        get_mcontext(dcontext)->xsp -= XSP_SZ;
+        *((app_pc *)get_mcontext(dcontext)->xsp) = dcontext->sysenter_storage;
+    }
+#else
+    adjust_syscall_continuation(dcontext);
+#endif
+    set_fcache_target(dcontext, dcontext->asynch_target);
+#ifdef WINDOWS
+    /* We no longer need asynch_target so zero it out. Other pieces of DR
+     * -- callback & APC handling, detach -- test asynch_target to determine
+     * where the next app pc to execute is stored. If asynch_target != 0,
+     * it holds the value, else it's in the esi slot.
+     */
+    dcontext->asynch_target = 0;
+#endif
+
+    LOG(THREAD, LOG_SYSCALLS, 3, "finished handling system call\n");
+    
+    SELF_PROTECT_LOCAL(dcontext, READONLY);
+    /* caller will go back to couldbelinking status */
+}
+
+#ifdef WINDOWS
+/* in callback.c */
+extern void callback_start_return(int app_errno, dr_mcontext_t *mc);
+/* used to execute an int 2b instruction in code cache */
+static void
+handle_callback_return(dcontext_t *dcontext)
+{
+    dcontext_t *prev_dcontext;
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    fcache_enter_func_t fcache_enter = get_fcache_enter_indirect_routine(dcontext);
+    LOG(THREAD, LOG_ASYNCH, 3, "handling a callback return\n");
+    /* may have to abort trace -> local heap */
+    SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+    KSWITCH(num_exits_dir_cbret);
+    callback_start_return(dcontext->app_errno, mc);
+    /* get the current, but now swapped out, dcontext */
+    prev_dcontext = get_prev_swapped_dcontext(dcontext);
+    SELF_PROTECT_LOCAL(dcontext, READONLY);
+
+    /* obey flushing protocol, plus set whereami (both using real dcontext) */
+    dcontext->whereami = WHERE_FCACHE;
+    set_at_syscall(dcontext, true); /* will be set to false on other end's post-syscall */
+    ASSERT(!is_couldbelinking(dcontext));
+
+    /* if we get an APC it should be after returning to prev cxt, so don't need
+     * to worry about asynch_target
+     */
+
+    /* make sure set the next_tag of prev_dcontext, not dcontext! */
+    set_fcache_target(prev_dcontext, (app_pc) get_do_callback_return_entry(prev_dcontext));
+    DOLOG(4, LOG_ASYNCH, {
+        LOG(THREAD, LOG_ASYNCH, 3, "passing prev dcontext "PFX", next_tag "PFX":\n",
+            prev_dcontext, prev_dcontext->next_tag);
+        dump_mcontext(get_mcontext(prev_dcontext), THREAD, DUMP_NOT_XML);
+    });
+    /* make sure to pass prev_dcontext, this is a special fcache enter routine
+     * that indirects through the dcontext passed to it (so ignores the switch-to
+     * dcontext that callback_start_return swapped into the main dcontext)
+     */
+    KSTART(syscall_fcache);     /* continue the interrupted syscall handling */
+    (*fcache_enter)(prev_dcontext);
+    /* callback return does not return to here! */
+    DOLOG(1, LOG_ASYNCH, {
+        LOG(THREAD, LOG_SYSCALLS, 1, "ERROR: int 2b returned!\n");
+        dump_mcontext(get_mcontext(dcontext), THREAD, DUMP_NOT_XML);
+    });
+    ASSERT_NOT_REACHED();
+}
+#endif /* WINDOWS */
+
+/* used to execute a system call instruction in code cache 
+ * not expected to return
+ * caller must set up mcontext with proper system call number and arguments
+ */
+void
+issue_last_system_call_from_app(dcontext_t *dcontext)
+{
+    LOG(THREAD, LOG_SYSCALLS, 2, "issue_last_system_call_from_app("PIFX")\n",
+        get_mcontext(dcontext)->xax);
+
+    /* it's up to the caller to let go of the bb building lock if it was held
+     * on this path, since not all paths to here hold it
+     */
+
+    if (is_couldbelinking(dcontext))
+        enter_nolinking(dcontext, NULL, true);
+    KSTART(syscall_fcache); /* stopped in dispatch_exit_fcache_stats */
+    enter_fcache(dcontext, get_fcache_enter_private_routine(dcontext),
+                 (app_pc) get_global_do_syscall_entry());
+    ASSERT_NOT_REACHED();
+}
+
+/* Stores the register parameters into the mcontext and calls dispatch.
+ * Checks whether currently on initstack and if so clears the initstack_mutex.
+ * Does not return.
+ */
+void
+transfer_to_dispatch(dcontext_t *dcontext, int app_errno, dr_mcontext_t *mc)
+{
+    app_pc cur_xsp;
+    int using_initstack = 0;
+    copy_mcontext(mc, get_mcontext(dcontext));
+    dcontext->app_errno = app_errno;
+    GET_STACK_PTR(cur_xsp);
+    if (is_on_initstack(cur_xsp))
+        using_initstack = 1;
+    LOG(THREAD, LOG_ASYNCH, 2,
+        "transfer_to_dispatch: pc=0x%08x, xsp="PFX", initstack=%d\n",
+        dcontext->next_tag, mc->xsp, using_initstack);
+
+    /* next, want to switch to dstack, and if using initstack, free mutex.
+     * finally, call dispatch(dcontext).
+     * note that we switch to the base of dstack, deliberately squashing
+     * what may have been there before, for both new dcontext and reuse dcontext
+     * options.
+     */
+    call_switch_stack(dcontext, dcontext->dstack, dispatch, using_initstack,
+                      false/*do not return on error*/);
+    ASSERT_NOT_REACHED();
+}
+
diff --git a/core/dispatch.h b/core/dispatch.h
new file mode 100644
index 000000000..0a6734260
--- /dev/null
+++ b/core/dispatch.h
@@ -0,0 +1,80 @@
+/* **********************************************************
+ * Copyright (c) 2000-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/* file "dispatch.h" */
+
+#ifndef _DISPATCH_H_
+#define _DISPATCH_H_ 1
+
+/* returns true if pc is a point at which DynamoRIO should stop interpreting */
+bool
+is_stopping_point(dcontext_t *dcontext, app_pc pc);
+
+/* central hub of control flow in DynamoRIO */
+void
+dispatch(dcontext_t *dcontext);
+
+void
+issue_last_system_call_from_app(dcontext_t *dcontext);
+
+void
+transfer_to_dispatch(dcontext_t *dcontext, int app_errno, dr_mcontext_t *mc);
+
+/* hooks on entry/exit to/from DR */
+#define NO_HOOK ((void (*)(void)) NULL)
+
+#define HOOK_ENABLED_HELPER SELF_PROTECT_ON_CXT_SWITCH
+
+#define HOOK_ENABLED (HOOK_ENABLED_HELPER || INTERNAL_OPTION(single_thread_in_DR))
+
+#define ENTER_DR_HOOK (HOOK_ENABLED ? entering_dynamorio : NO_HOOK)
+#define EXIT_DR_HOOK  (HOOK_ENABLED ? exiting_dynamorio : NO_HOOK)
+
+#define ENTERING_DR() do {   \
+        if (HOOK_ENABLED)    \
+            ENTER_DR_HOOK(); \
+    } while (0);
+
+#define EXITING_DR() do {    \
+        if (HOOK_ENABLED)    \
+            EXIT_DR_HOOK();  \
+    } while (0);
+
+
+/* magic value to set next_tag to, to indicate a return to native_exec */
+#define BACK_TO_NATIVE_AFTER_SYSCALL ((app_pc)(ptr_uint_t) -1)
+
+#endif /* _DISPATCH_H_ */
diff --git a/core/dynamo.c b/core/dynamo.c
new file mode 100644
index 000000000..22c4d383d
--- /dev/null
+++ b/core/dynamo.c
@@ -0,0 +1,2755 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * dynamo.c -- initialization and cleanup routines for DynamoRIO
+ */
+
+#include "globals.h"
+#include "link.h"
+#include "fragment.h"
+#include "fcache.h"
+#include "emit.h"
+#include "dispatch.h"
+#include "utils.h"
+#include "monitor.h"
+#include "vmareas.h"
+#ifdef SIDELINE
+# include "sideline.h"
+#endif
+#ifdef PAPI
+# include "perfctr.h"
+#endif
+#ifdef CLIENT_INTERFACE
+# include "instrument.h"
+#endif
+#include "hotpatch.h"
+#include "moduledb.h"
+#include "module_shared.h"
+#include "defines.h"
+#include "synch.h"
+
+#include <string.h>
+
+#ifdef WINDOWS
+/* for close handle, duplicate handle, free memory and constants associated with them */
+#include "ntdll.h"
+#include "nudge.h" /* to get generic_nudge_target() address for an assert */
+#endif
+
+#ifdef RCT_IND_BRANCH
+#  include "rct.h"
+#endif
+
+#include "perscache.h"
+
+/* global thread-shared variables */
+bool    dynamo_initialized = false;
+bool    dynamo_heap_initialized = false;
+bool    automatic_startup = false;
+bool    control_all_threads = false;
+#ifdef WINDOWS
+bool    dr_early_injected = false;
+int     dr_early_injected_location = INJECT_LOCATION_Invalid;
+
+/* should be set if we are controlling the primary thread, either by
+ * injecting initially (!dr_injected_secondary_thread), or by retaking
+ * over (dr_late_injected_primary_thread).  Used only for debugging
+ * purposes, yet can't rely on !dr_injected_secondary_thread very
+ * early in the process
+ */
+bool    dr_injected_primary_thread = false;
+bool    dr_injected_secondary_thread = false;
+
+/* should be set once we retakeover the primary thread for -inject_primary */
+bool    dr_late_injected_primary_thread = false;
+#endif /* WINDOWS */
+#ifdef DR_APP_EXPORTS
+/* flags to indicate when DR is being initialized / exited using the API */
+bool    dr_api_entry = false;
+bool    dr_api_exit  = false;
+#endif
+#ifdef RETURN_AFTER_CALL
+bool    dr_preinjected = false;
+#endif /* RETURN_AFTER_CALL */
+bool    dynamo_exited = false;
+bool    dynamo_exited_and_cleaned = false;
+#ifdef DEBUG
+bool    dynamo_exited_log_and_stats = false;
+#endif
+/* Only used in release build to decide whether synch is needed, justifying
+ * its placement in .nspdata.  If we use it for more we should protect it.
+ */
+DECLARE_NEVERPROT_VAR(bool dynamo_all_threads_synched, false);
+bool    dynamo_resetting = false;
+#if defined(CLIENT_INTERFACE) || defined(STANDALONE_UNIT_TEST)
+bool    standalone_library = false;
+#endif
+#ifdef LINUX
+bool    post_execve = false;
+#endif
+/* initial stack so we don't have to use app's */
+byte *  initstack;
+
+#if defined(WINDOWS) && defined(STACK_GUARD_PAGE)
+/* PR203701: separate stack for error reporting when the dstack is exhausted */
+#define EXCEPTION_STACK_SIZE (2 * PAGE_SIZE)
+DECLARE_NEVERPROT_VAR(byte *exception_stack, NULL);
+#endif
+
+/*******************************************************/
+/* separate segment of Non-Self-Protected data to avoid data section
+ * protection issues -- we need to write to these vars in bootstrapping
+ * spots where we cannot unprotect first
+ */
+START_DATA_SECTION(NEVER_PROTECTED_SECTION, "w");
+
+/* spinlock used in assembly trampolines when we can't spare registers for more */
+mutex_t initstack_mutex VAR_IN_SECTION(NEVER_PROTECTED_SECTION)
+     = INIT_SPINLOCK_FREE(initstack_mutex);
+byte * initstack_app_xsp VAR_IN_SECTION(NEVER_PROTECTED_SECTION) = 0;
+/* keeps track of how many threads are in cleanup_and_terminate */
+int exiting_thread_count VAR_IN_SECTION(NEVER_PROTECTED_SECTION) = 0;
+
+/* This is unprotected to allow stats to be written while the data
+ * segment is still protected (right now the only ones are selfmod stats)
+ */
+static dr_statistics_t nonshared_stats VAR_IN_SECTION(NEVER_PROTECTED_SECTION)
+     = {{0},};
+
+/* Each lock protects its corresponding datasec_start, datasec_end, and
+ * datasec_writable variables.
+ */
+static mutex_t datasec_lock[DATASEC_NUM] VAR_IN_SECTION(NEVER_PROTECTED_SECTION) = {{0}};
+
+/* back to normal section */
+END_DATA_SECTION()
+/*******************************************************/
+
+/* Like a recursive lock: 0==readonly, 1+=writable.
+ * This would be a simple array, but we need each in its own protected
+ * section, as this could be exploited.
+ */
+const uint datasec_writable_neverprot = 1; /* always writable */
+uint datasec_writable_rareprot = 1;
+DECLARE_FREQPROT_VAR(uint datasec_writable_freqprot, 1);
+DECLARE_CXTSWPROT_VAR(uint datasec_writable_cxtswprot, 1);
+
+static app_pc datasec_start[DATASEC_NUM];
+static app_pc datasec_end[DATASEC_NUM];
+
+const uint DATASEC_SELFPROT[] = {
+    0,
+    SELFPROT_DATA_RARE,
+    SELFPROT_DATA_FREQ,
+    SELFPROT_DATA_CXTSW,
+};
+
+const char * const DATASEC_NAMES[] = {
+    NEVER_PROTECTED_SECTION,
+    RARELY_PROTECTED_SECTION,
+    FREQ_PROTECTED_SECTION,
+    CXTSW_PROTECTED_SECTION,
+};
+
+/* kept in unprotected heap to avoid issues w/ data segment being RO */
+typedef struct _protect_info_t {
+    /* FIXME: this needs to be a recursive lock to handle signals
+     * and exceptions!
+     */
+    mutex_t lock;
+    int num_threads_unprot; /* # threads in DR code */
+    int num_threads_suspended;
+} protect_info_t;
+static protect_info_t *protect_info;
+
+static void data_section_init(void);
+static void data_section_exit(void);
+
+#ifdef DEBUG /*************************/
+
+#include <time.h>
+
+/* FIXME: not all dynamo_options references are #ifdef DEBUG
+ * are we trying to hardcode the options for a release build?
+ */
+# ifdef LINUX
+   /* linux include files for mmap stuff*/
+#  include <sys/ipc.h>
+#  include <sys/types.h>
+#  include <unistd.h>
+# endif
+
+static uint starttime;
+
+file_t main_logfile = INVALID_FILE;
+
+#endif /* DEBUG ****************************/
+
+dr_statistics_t *stats = NULL;
+
+DECLARE_FREQPROT_VAR(static int num_known_threads, 0);
+DECLARE_FREQPROT_VAR(static uint threads_ever_count, 0);
+
+/* FIXME : not static so os.c can hand walk it for dump core */
+thread_record_t ** all_threads; /* ALL_THREADS_HASH_BITS-bit addressed hash table */
+
+/* these locks are used often enough that we put them in .cspdata: */
+
+/* not static so can be referenced in win32/os.c for SuspendThread handling, 
+ * FIXME : is almost completely redundant in usage with thread_initexit_lock
+ * maybe replace this lock with thread_initexit_lock? */
+DECLARE_CXTSWPROT_VAR(mutex_t all_threads_lock, INIT_LOCK_FREE(all_threads_lock));
+/* used for synch to prevent thread creation/deletion in critical periods 
+ * due to its use for flushing, this lock cannot be held while couldbelinking!
+ */
+DECLARE_CXTSWPROT_VAR(mutex_t thread_initexit_lock,
+                      INIT_LOCK_FREE(thread_initexit_lock));
+
+/* recursive to handle signals/exceptions while in DR code */
+DECLARE_CXTSWPROT_VAR(static recursive_lock_t thread_in_DR_exclusion,
+                      INIT_RECURSIVE_LOCK(thread_in_DR_exclusion));
+
+/****************************************************************************/
+#ifdef DEBUG
+
+static const char *
+main_logfile_name(void)
+{
+    return get_app_name_for_path();
+}
+
+static char *
+thread_logfile_name(void)
+{
+    return "log";
+}
+
+#endif /* DEBUG */
+/****************************************************************************/
+
+static void
+statistics_pre_init(void)
+{
+    /* until it's set up for real, point at static var
+     * really only logmask and loglevel are meaningful, so be careful!
+     * statistics_init and create_log_directory are the only routines that
+     * use stats before it's set up for real, currently
+     */
+    /* The indirection here is left over from when we used to allow alternative
+     * locations for stats (namely shared memory for the old MIT gui). */
+    stats = &nonshared_stats;
+    stats->process_id = get_process_id();
+    strncpy(stats->process_name, get_application_name(), MAXIMUM_PATH);
+    stats->process_name[MAXIMUM_PATH-1] = '\0';
+    ASSERT(strlen(stats->process_name) > 0);
+    stats->num_stats = 0;
+}
+
+static void
+statistics_init(void)
+{
+    /* should have called statistics_pre_init() first */
+    ASSERT(stats == &nonshared_stats);
+    ASSERT(stats->num_stats == 0);
+#ifndef DEBUG
+    if (!DYNAMO_OPTION(global_rstats)) {
+        /* references to stat values should return 0 (static var) */
+        return;
+    }
+#endif
+    stats->num_stats = 0
+#ifdef DEBUG
+# define STATS_DEF(desc, name) +1
+#else
+# define RSTATS_DEF(desc, name) +1
+#endif
+# include "statsx.h"
+#undef STATS_DEF
+#undef RSTATS_DEF
+        ;
+    /* We inline the stat description to make it easy for external processes
+     * to view our stats: they don't have to chase pointers, and we could put
+     * this in shared memory easily.  However, we do waste some memory, but
+     * not much in release build.
+     */
+#ifdef DEBUG
+# define STATS_DEF(desc, statname) \
+    strncpy(stats->statname##_pair.name, desc, \
+            BUFFER_SIZE_ELEMENTS(stats->statname##_pair.name)); \
+    NULL_TERMINATE_BUFFER(stats->statname##_pair.name);
+#else
+# define RSTATS_DEF(desc, statname) \
+    strncpy(stats->statname##_pair.name, desc, \
+            BUFFER_SIZE_ELEMENTS(stats->statname##_pair.name)); \
+    NULL_TERMINATE_BUFFER(stats->statname##_pair.name);
+#endif
+# include "statsx.h"
+#undef STATS_DEF
+#undef RSTATS_DEF
+}
+
+static void
+statistics_exit(void)
+{
+    stats = NULL;
+}
+
+dr_statistics_t *
+get_dr_stats(void)
+{
+    return stats;
+}
+
+/* initialize per-process dynamo state; this must be called before any
+ * threads are created and before any other API calls are made;
+ * returns zero on success, non-zero on failure 
+ */
+DYNAMORIO_EXPORT int
+dynamorio_app_init(void)
+{
+    int size;
+
+    if (!dynamo_initialized /* we do enter if nullcalls is on */) {
+
+#ifdef WINDOWS
+        /* MUST do this before making any system calls */
+        syscalls_init();
+#endif
+        /* avoid time() for libc independence */
+        DODEBUG(starttime = query_time_seconds(););
+
+#ifdef LINUX
+        if (getenv(DYNAMORIO_VAR_EXECVE) != NULL) {
+            post_execve = true;
+            /* important to remove it, don't want to propagate to forked children, etc. */
+            unsetenv(DYNAMORIO_VAR_EXECVE);
+            /* check that it's gone: we've had problems with unsetenv */
+            ASSERT(getenv(DYNAMORIO_VAR_EXECVE) == NULL);
+        } else
+            post_execve = false;
+#endif
+
+        /* default non-zero dynamo settings (options structure is
+         * initialized to 0 automatically)
+         */
+#ifdef DEBUG
+# ifndef INTERNAL
+        nonshared_stats.logmask = LOG_ALL_RELEASE;
+# else
+        nonshared_stats.logmask = LOG_ALL;
+# endif
+        statistics_pre_init();
+#endif
+
+        options_init();
+#ifdef WINDOWS
+        syscalls_init_options_read(); /* must be called after options_init
+                                       * but before init_syscall_trampolines */
+#endif
+        utils_init();
+        data_section_init();
+
+#ifdef DEBUG
+        /* decision: nullcalls WILL create a dynamorio.log file and
+         * fill it with perfctr stats!
+         */
+        if (stats->loglevel > 0) {
+            main_logfile = open_log_file(main_logfile_name(), NULL, 0);
+        } else {
+            /* loglevel 0 means we don't create a log file!
+             * if the loglevel is later raised, too bad!  it all goes to stderr!
+             * N.B.: when checking for no logdir, we check for empty string or
+             * first char '<'!
+             */
+            strncpy(stats->logdir, "<none (loglevel was 0 on startup)>", MAXIMUM_PATH-1);
+            stats->logdir[MAXIMUM_PATH-1]  = '\0'; /* if max no null */
+            main_logfile = INVALID_FILE;
+        }
+
+# ifdef PAPI
+        /* setup hardware performance counting */
+        hardware_perfctr_init();
+# endif
+
+        DOLOG(1, LOG_TOP, {
+            print_version_and_app_info(GLOBAL);
+        });
+
+        /* now exit if nullcalls, now that perfctrs are set up */
+        if (INTERNAL_OPTION(nullcalls)) {
+            print_file(main_logfile,
+                       "** nullcalls is set, NOT taking over execution **\n\n");
+            return SUCCESS;
+        }
+
+        LOG(GLOBAL, LOG_TOP, 1, PRODUCT_NAME"'s stack size: %d Kb\n",
+            DYNAMORIO_STACK_SIZE/1024);
+#endif /* !DEBUG */
+
+        /* set up exported statistics struct */
+#ifndef DEBUG
+        statistics_pre_init();
+#endif
+        statistics_init();
+
+#ifdef CLIENT_INTERFACE
+        /* PR 200207: load the client lib before callback_interception_init
+         * since the client library load would hit our own hooks (xref hotpatch
+         * cases about that). */
+        /* Must be before [vmm_]heap_init() so we can register the client lib as 
+         * reachable from the dr heap. Xref PR 215395. */
+        instrument_load_client_libs();
+#endif
+
+        /* initialize components (CAUTION: order is important here) */
+        vmm_heap_init(); /* must be called even if not using vmm heap */
+        heap_init();
+        dynamo_heap_initialized = true;
+
+        /* The process start event should be done after os_init() but before
+         * process_control_int() because the former initializes event logging
+         * and the latter can kill the process if a violation occurs.
+         */
+        SYSLOG(SYSLOG_INFORMATION,
+               IF_CLIENT_INTERFACE_ELSE(INFO_PROCESS_START_CLIENT, INFO_PROCESS_START),
+               IF_CLIENT_INTERFACE_ELSE(3, 2),
+               get_application_name(), get_application_pid()
+               _IF_NOT_CLIENT_INTERFACE(get_application_md5()));
+
+#ifdef PROCESS_CONTROL
+        if (IS_PROCESS_CONTROL_ON())    /* Case 8594. */
+            process_control_init();
+#endif
+                    
+        dynamo_vm_areas_init();
+        proc_init();
+        modules_init(); /* before vm_areas_init() */
+        os_init();
+        arch_init();
+        synch_init();
+
+#ifdef KSTATS
+        kstat_init();
+#endif
+        monitor_init();
+        fcache_init();
+        link_init();
+        fragment_init();
+        moduledb_init(); /* before vm_areas_init, after heap_init */
+        perscache_init(); /* before vm_areas_init */
+
+        if (!DYNAMO_OPTION(thin_client)) {
+#ifdef HOT_PATCHING_INTERFACE
+            /* must init hotp before vm_areas_init() calls find_executable_vm_areas() */
+            if (DYNAMO_OPTION(hot_patching))
+                hotp_init();
+#endif
+        }
+
+#ifdef INTERNAL
+        {
+            char initial_options[MAX_OPTIONS_STRING];
+            get_dynamo_options_string(&dynamo_options, 
+                                      initial_options, sizeof(initial_options), true);
+            SYSLOG_INTERNAL_INFO("Initial options = %s", initial_options);
+            DOLOG(1, LOG_TOP, {
+                get_pcache_dynamo_options_string(&dynamo_options, initial_options,
+                                                 sizeof(initial_options),
+                                                 OP_PCACHE_LOCAL);
+                LOG(GLOBAL, LOG_TOP, 1, "Initial pcache-affecting options = %s\n",
+                    initial_options);
+            });
+        }
+#endif /* INTERNAL */
+
+        /* initial stack so we don't have to use app's 
+         * N.B.: we never de-allocate initstack (see comments in app_exit)
+         */
+        initstack = (byte *) stack_alloc(DYNAMORIO_STACK_SIZE);
+
+#if defined(WINDOWS) && defined(STACK_GUARD_PAGE)
+        /* PR203701: separate stack for error reporting when the
+         * dstack is exhausted
+         */
+        exception_stack = (byte *) stack_alloc(EXCEPTION_STACK_SIZE);
+#endif
+        LOG(GLOBAL, LOG_TOP, 1, "\n");
+
+        /* initialize thread hashtable */
+        /* Note: for thin_client, this isn't needed if it is only going to
+         * look for spawned processes; however, if we plan to promote from 
+         * thin_client to hotp_only mode (highly likely), this would be needed.
+         * For now, leave it in there unless thin_client footprint becomes an 
+         * issue.
+         */
+        size = HASHTABLE_SIZE(ALL_THREADS_HASH_BITS) * sizeof(thread_record_t*);
+        all_threads = (thread_record_t**) global_heap_alloc(size HEAPACCT(ACCT_THREAD_MGT));
+        memset(all_threads, 0, size);
+
+#ifdef SIDELINE
+        /* initialize sideline thread after thread table is set up */
+        if (dynamo_options.sideline)
+            sideline_init();
+#endif
+
+        /* thread-specific initialization for the first thread we inject in
+         * (in a race with injected threads, sometimes it is not the primary thread)
+         */
+        dynamo_thread_init();
+
+        /* We move vm_areas_init() below dynamo_thread_init() so we can have
+         * two things: 1) a dcontext and 2) a SIGSEGV handler, for TRY/EXCEPT
+         * inside vm_areas_init() for PR 361594's probes and for safe_read().
+         * This means vm_areas_thread_init() runs before vm_areas_init().
+         */
+        if (!DYNAMO_OPTION(thin_client)) {
+            vm_areas_init();
+#ifdef RCT_IND_BRANCH
+            /* relies on is_in_dynamo_dll() which needs vm_areas_init */
+            rct_init();
+#endif
+        } else {
+            /* This is needed to handle exceptions in thin_client mode, mostly
+             * internal ones, but can be app ones too. */
+            dynamo_vm_areas_lock();
+            find_dynamo_library_vm_areas();
+            dynamo_vm_areas_unlock();
+        }
+
+#ifdef CLIENT_INTERFACE
+        /* client last, in case it depends on other inits: must be after
+         * dynamo_thread_init so the client can use a dcontext (PR 216936).
+         * Note that we *load* the client library before installing our hooks,
+         * but call the client's init routine afterward so that we correctly
+         * report crashes (PR 200207).
+         * Note: DllMain in client libraries can crash and we still won't
+         *       report; better document that client libraries shouldn't have
+         *       DllMain.
+         * TODO: doing client init after inserting dr hooks currently breaks
+         *       hot patch dll loading for probe api (only), so it temporarily
+         *       moved before callback_interception_init.
+         */
+        instrument_init();
+#endif
+
+#ifdef WINDOWS
+        if (!INTERNAL_OPTION(noasynch)) {
+            /* PR 216934: This is only this late b/c we originally didn't have
+             * GLOBAL_DCONTEXT and had to be post-thread-init.  Now we should
+             * move it up to report errors in client init routines: though we
+             * then have to handle client library loads w/ our hooks in place.
+             */
+            callback_interception_init();
+        }
+#endif /* WINDOWS */
+
+        if (SELF_PROTECT_ON_CXT_SWITCH) {
+            protect_info = (protect_info_t *)
+                global_unprotected_heap_alloc(sizeof(protect_info_t) HEAPACCT(ACCT_OTHER));
+            ASSIGN_INIT_LOCK_FREE(protect_info->lock, protect_info);
+            protect_info->num_threads_unprot = 0; /* ENTERING_DR() below will inc to 1 */
+            protect_info->num_threads_suspended = 0;
+            if (INTERNAL_OPTION(single_privileged_thread)) {
+                /* FIXME: thread_initexit_lock must be a recursive lock! */
+                ASSERT_NOT_IMPLEMENTED(false);
+                /* grab the lock now -- the thread that is in dynamo must be holding
+                 * the lock, and we are the initial thread in dynamo!
+                 */
+                mutex_lock(&thread_initexit_lock);
+            }
+            /* ENTERING_DR will increment, so decrement first
+             * FIXME: waste of protection change since will nop-unprotect!
+             */
+            if (TEST(SELFPROT_DATA_CXTSW, DYNAMO_OPTION(protect_mask)))
+                datasec_writable_cxtswprot = 0;
+            /* FIXME case 8073: remove once freqprot not every cxt sw */
+            if (TEST(SELFPROT_DATA_FREQ, DYNAMO_OPTION(protect_mask)))
+                datasec_writable_freqprot = 0;
+        }
+        /* this thread is now entering DR */
+        ENTERING_DR();
+
+#ifdef WINDOWS
+        if (DYNAMO_OPTION(early_inject)) {
+            /* AFTER callback_interception_init and self protect init and
+             * ENTERING_DR() */
+            early_inject_init();
+        }
+#endif
+    }
+
+    dynamo_initialized = true;
+
+    /* Protect .data, assuming all vars there have been initialized. */
+    SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+
+    /* internal-only options for testing run-once (case 3990) */
+    if (INTERNAL_OPTION(unsafe_crash_process)) {
+        SYSLOG_INTERNAL_ERROR("Crashing the process deliberately!");
+        *((int *)PTR_UINT_MINUS_1) = 0;
+    }
+    if (INTERNAL_OPTION(unsafe_hang_process)) {
+        event_t never_signaled = create_event();
+        SYSLOG_INTERNAL_ERROR("Hanging the process deliberately!");
+        wait_for_event(never_signaled);
+        destroy_event(never_signaled);
+    }
+
+    return SUCCESS;
+}
+
+#ifdef LINUX
+void
+dynamorio_fork_init(dcontext_t *dcontext)
+{
+    /* on a fork we want to re-initialize some data structures, especially
+     * log files, which we want a separate directory for
+     */
+    thread_record_t **threads;
+    int i, num_threads;
+# ifdef DEBUG
+    char parent_logdir[MAXIMUM_PATH];
+# endif
+
+    /* sanity check, plus need to set this for statistics_init:
+     * even if parent did an execve, env var should be reset by now
+     */
+    post_execve = (getenv(DYNAMORIO_VAR_EXECVE) != NULL);
+    ASSERT(!post_execve);
+
+# ifdef DEBUG
+    /* copy stats->logdir
+     * stats->logdir is static, so current copy is fine, don't need
+     * frozen copy
+     */
+    strncpy(parent_logdir, stats->logdir, MAXIMUM_PATH-1);
+    stats->logdir[MAXIMUM_PATH-1]  = '\0'; /* if max no null */
+# endif
+
+    if (get_log_dir(PROCESS_DIR, NULL, NULL)) {
+        /* we want brand new log dir  */
+        enable_new_log_dir();
+        create_log_dir(PROCESS_DIR);
+    }
+
+# ifdef DEBUG
+    /* just like dynamorio_app_init, create main_logfile before stats */
+    if (stats->loglevel > 0) {
+        /* we want brand new log files */
+        main_logfile = open_log_file(main_logfile_name(), NULL, 0);
+        print_file(main_logfile, "%s\n", dynamorio_version_string);
+        print_file(main_logfile, "New log file for child %d forked by parent %d\n",
+                   get_thread_id(), get_parent_id());
+        print_file(main_logfile, "Parent's log dir: %s\n", parent_logdir);
+    }
+
+    stats->process_id = get_process_id();
+
+    if (stats->loglevel > 0) {
+        /* FIXME: share these few lines of code w/ dynamorio_app_init? */
+        LOG(GLOBAL, LOG_TOP, 1, "Running: %s\n", stats->process_name);
+#  ifndef  _WIN32_WCE
+        LOG(GLOBAL, LOG_TOP, 1, "DYNAMORIO_OPTIONS: %s\n", option_string);
+#  endif
+    }
+# endif /* DEBUG */
+
+    /* must re-hash parent entry in threads table, plus no longer have any
+     * other threads (fork -> we're alone in address space), so clear
+     * out entire thread table, then add child
+     */
+    mutex_lock(&thread_initexit_lock);
+    get_list_of_threads(&threads, &num_threads);
+    for (i=0; i<num_threads; i++) {
+        remove_thread(threads[i]->id);
+    }
+    mutex_unlock(&thread_initexit_lock);
+    global_heap_free(threads, num_threads*sizeof(thread_record_t*)
+                     HEAPACCT(ACCT_THREAD_MGT));
+
+    add_thread(get_thread_id(), true/*under dynamo control*/, dcontext);
+
+    GLOBAL_STAT(num_threads) = 1;
+# ifdef DEBUG
+    if (stats->loglevel > 0) {
+        /* need a new thread-local logfile */
+        dcontext->logfile = open_log_file(thread_logfile_name(), NULL, 0);
+        print_file(dcontext->logfile, "%s\n", dynamorio_version_string);
+        print_file(dcontext->logfile, "New log file for child %d forked by parent %d\n",
+                   get_thread_id(), get_parent_id());
+        LOG(THREAD, LOG_TOP|LOG_THREADS, 1,
+            "THREAD %d (dcontext "PFX")\n\n", get_thread_id(), dcontext);
+    }
+# endif
+    num_threads = 1;
+
+    /* FIXME: maybe should have a callback list for who wants to be notified
+     * on a fork -- probably everyone who makes a log file on init.
+     */
+    os_fork_init(dcontext);
+    fragment_fork_init(dcontext);
+
+# ifdef CLIENT_INTERFACE
+    if (!IS_INTERNAL_STRING_OPTION_EMPTY(client_lib)) {
+        instrument_fork_init(dcontext);
+    }
+# endif
+}
+#endif /* LINUX */
+
+#if defined(CLIENT_INTERFACE) || defined(STANDALONE_UNIT_TEST)
+/* To make DynamoRIO useful as a library for a standalone client
+ * application (as opposed to a client library that works with
+ * DynamoRIO in executing a target application).  This makes DynamoRIO
+ * useful as an IA-32 disassembly library, etc.  
+ */
+dcontext_t *
+standalone_init(void)
+{
+    dcontext_t *dcontext;
+    standalone_library = true;
+    /* We have release-build stats now so this is not just DEBUG */
+    stats = &nonshared_stats;
+    /* FIXME: I believe we only need dcontext for the heap.
+     * Nothing else should fail, unless client calls something that's
+     * not initialized...for now we blame client if that happens.
+     */
+#ifdef WINDOWS
+    /* MUST do this before making any system calls */
+    syscalls_init();
+#endif
+    options_init();
+    vmm_heap_init();
+    heap_init();
+    dynamo_heap_initialized = true;
+    dynamo_vm_areas_init();
+    proc_init();
+    os_init();
+
+    os_tls_init();
+    dcontext = create_new_dynamo_context(true/*initial*/);
+    set_thread_private_dcontext(dcontext);
+    /* sanity check */
+    ASSERT(get_thread_private_dcontext() == dcontext);
+
+    heap_thread_init(dcontext);
+#ifdef STANDALONE_UNIT_TEST
+    /* FIXME: share code w/ main init routine? */
+    nonshared_stats.logmask = LOG_ALL;
+    options_init();
+    if (stats->loglevel > 0) {
+        char initial_options[MAX_OPTIONS_STRING];
+        main_logfile = open_log_file(main_logfile_name(), NULL, 0);
+        print_file(main_logfile, "%s\n", dynamorio_version_string);
+        print_file(main_logfile, "Log file for standalone unit test\n");
+        get_dynamo_options_string(&dynamo_options, 
+                                  initial_options, sizeof(initial_options), true);
+        SYSLOG_INTERNAL_INFO("Initial options = %s", initial_options);
+        print_file(main_logfile, "\n");
+    }
+#endif
+    dynamo_initialized = true;
+
+    return dcontext;
+}
+
+void
+standalone_exit(void)
+{
+    /* should clean up here */
+}
+#endif
+
+/* Perform exit tasks that require full thread data structs, which we have
+ * already cleaned up by the time we reach dynamo_shared_exit() for both
+ * debug and detach paths.
+ */
+void
+dynamo_process_exit_with_thread_info(void)
+{
+    perscache_fast_exit(); /* "fast" b/c called in release as well */
+}
+
+/* shared between app_exit and detach */
+int
+dynamo_shared_exit(IF_WINDOWS_ELSE_NP(bool detach_stacked_callbacks, void)) 
+{
+    DEBUG_DECLARE(uint endtime);
+    /* set this now, could already be set */
+    dynamo_exited = true;
+
+    /* now free the all_threads memory */
+    mutex_lock(&all_threads_lock);
+    global_heap_free(all_threads,
+                     HASHTABLE_SIZE(ALL_THREADS_HASH_BITS) * sizeof(thread_record_t*)
+                     HEAPACCT(ACCT_THREAD_MGT));
+    all_threads = NULL;
+    mutex_unlock(&all_threads_lock);
+    
+    /* avoid time() for libc independence */
+    DODEBUG(endtime = query_time_seconds(););
+    LOG(GLOBAL, LOG_STATS, 1, "\n#### Statistics for entire process:\n");
+    LOG(GLOBAL, LOG_STATS, 1, "Total running time: %d seconds\n",
+        endtime - starttime);
+    
+#ifdef PAPI
+    hardware_perfctr_exit();
+#endif
+#ifdef DEBUG
+#  ifdef INTERNAL
+    print_optimization_stats();
+#  endif
+    DOLOG(1, LOG_STATS, {
+        dump_global_stats(false);
+    });
+#endif
+    
+    if (SELF_PROTECT_ON_CXT_SWITCH) {
+        DELETE_LOCK(protect_info->lock);
+        global_unprotected_heap_free(protect_info, sizeof(protect_info_t) HEAPACCT(ACCT_OTHER));
+    }
+
+    dynamo_exited_and_cleaned = true;
+
+    /* call all component exit routines (CAUTION: order is important here) */
+
+    DELETE_RECURSIVE_LOCK(thread_in_DR_exclusion);
+    DOSTATS({
+        LOG(GLOBAL, LOG_TOP|LOG_THREADS, 1,
+            "fcache_stats_exit: before fragment cleanup\n");
+        DOLOG(1, LOG_CACHE, fcache_stats_exit(););
+    });
+#ifdef RCT_IND_BRANCH
+    if (!DYNAMO_OPTION(thin_client))
+        rct_exit();
+#endif
+    fragment_exit();
+#ifdef CLIENT_INTERFACE
+    /* We tell the client as soon as possible in case it wants to use services from other
+     * components.  Must be after fragment_exit() so that the client gets all the
+     * fragment_deleted() callbacks (xref PR 228156). FIXME - might be issues with the
+     * client trying to use api routines that depend on fragment state. FIXME - 
+     * (xref PR 200207) must be after callback_interception_unintercept() since we
+     * need to remove our LdrUnloadDll hook before we unload the client library, but
+     * this means we won't catch exceptions for the client here. */
+    instrument_exit();
+#endif
+#ifdef WINDOWS
+    /* callback_interception_exit must be after fragment exit for CLIENT_INTERFACE so
+     * that fragment_exit->frees fragments->instrument_fragment_deleted->
+     * hide_tag_from_fragment->is_intercepted_app_pc won't crash. xref PR 228156 */
+    if (!INTERNAL_OPTION(noasynch)) {
+        callback_interception_exit();
+    }
+#endif
+    link_exit();
+    fcache_exit();
+    monitor_exit();
+    synch_exit();
+    arch_exit(IF_WINDOWS(detach_stacked_callbacks));
+#ifdef CALL_PROFILE
+    /* above os_exit to avoid eventlog_mutex trigger if we're the first to
+     * create a log file
+     */
+    profile_callers_exit();
+#endif
+    os_fast_exit();
+    os_slow_exit();
+    modules_exit(); /* after aslr_exit() from os_slow_exit() */
+    vm_areas_exit();
+    perscache_slow_exit(); /* fast called in dynamo_process_exit_with_thread_info() */
+    moduledb_exit(); /* before heap_exit */
+#ifdef HOT_PATCHING_INTERFACE
+    if (DYNAMO_OPTION(hot_patching))
+        hotp_exit();
+#endif
+#if defined(WINDOWS) && defined(STACK_GUARD_PAGE) && defined(DEBUG)
+    /* Free exception stack before calling heap_exit */
+    stack_free(exception_stack, EXCEPTION_STACK_SIZE);
+    exception_stack = NULL;
+#endif
+    heap_exit();
+    vmm_heap_exit();
+    diagnost_exit();
+    data_section_exit();
+    /* funny dependences: options exit just frees lock, not destroying
+     * any options that are needed for other exits, so do it prior to
+     * checking locks in debug build
+     */
+    options_exit();
+    utils_exit();
+
+#ifdef KSTATS
+    kstat_exit();
+#endif
+
+    DELETE_LOCK(all_threads_lock);
+    DELETE_LOCK(thread_initexit_lock);
+
+    DOLOG(1, LOG_STATS, {
+        /* dump after cleaning up to make it easy to check if stats that
+         * are inc-ed and dec-ed actually come down to 0
+         */
+        dump_global_stats(false);
+    });
+
+    statistics_exit();
+#ifdef DEBUG
+# ifdef DEADLOCK_AVOIDANCE
+    ASSERT(locks_not_closed() == 0);
+# endif
+    dynamo_exited_log_and_stats = true;
+    if (main_logfile != STDERR) {
+        /* do it this way just in case someone tries to log to the global file
+         * right now */
+        file_t file_temp = main_logfile;
+        main_logfile = INVALID_FILE;
+        os_close(file_temp);
+    }
+#else
+# ifdef DEADLOCK_AVOIDANCE
+    ASSERT(locks_not_closed() == 0);
+# endif
+#endif /* DEBUG */
+
+    dynamo_initialized = false;
+    return SUCCESS;
+}
+
+int
+dynamorio_app_exit(void)
+{
+    return dynamo_process_exit();
+}
+
+#ifdef DEBUG
+/* cleanup after the application has exited */
+static int
+dynamo_process_exit_cleanup(void)
+{
+    /* CAUTION: this should only be invoked after all app threads have stopped */
+    if (!dynamo_exited && !INTERNAL_OPTION(nullcalls)) {
+        dcontext_t *dcontext;
+
+        APP_EXPORT_ASSERT(dynamo_initialized, "Improper DynamoRIO initialization");
+
+        dcontext = get_thread_private_dcontext();
+
+        /* this normally will already have been called but sometimes
+         * app_exit is called before app_stop.
+         * we want to call this now to stop timers, etc.
+         */
+        dynamo_thread_not_under_dynamo(dcontext);
+
+#ifdef LINUX
+        if (get_num_threads() > 1 ||
+            (get_num_threads() == 1 && dcontext == NULL)) {
+            /* FIXME: assumption: pthreads is being used, it keeps a monitor
+             * thread around.  We cannot exit dynamo before the monitor thread
+             * dies.  But, we can assume that the monitor will never call
+             * dr_app_stop, so we assume we'll see its SYS_exit.
+             * So, we just wait for that.
+             */
+            LOG(GLOBAL, LOG_TOP|LOG_THREADS, 1,
+                "dynamorio_app_exit: threads remain, so not exiting process\n");
+            dynamo_thread_exit();
+            return SUCCESS;
+        }
+#endif
+
+        /* we deliberately do NOT clean up initstack (which was
+         * allocated using a separate mmap and so is not part of some
+         * large unit that is de-allocated), as it is used in special
+         * circumstances to call us...FIXME: is this memory leak ok?
+         * is there a better solution besides assuming the app stack?
+         */
+
+#ifdef SIDELINE
+        if (dynamo_options.sideline) {
+            /* exit now to make thread cleanup simpler */
+            sideline_exit();
+        }
+#endif
+
+        /* perform exit tasks that require full thread data structs */
+        dynamo_process_exit_with_thread_info();
+    
+        if (INTERNAL_OPTION(single_privileged_thread)) {
+            mutex_unlock(&thread_initexit_lock);
+        }
+
+#ifdef WINDOWS
+        /* we don't check control_all_threads b/c we're just killing
+         * the threads we know about here
+         */
+        if (automatic_startup) {
+            /* if ExitProcess called before all threads terminated, they won't
+             * all have gone through dynamo_thread_exit, so clean them up now
+             * so we can get stats about them
+             */
+            int num_threads;
+            thread_record_t **threads;
+            DEBUG_DECLARE(bool ok;)
+            LOG(GLOBAL, LOG_TOP|LOG_THREADS, 1,
+                "\ndynamorio_app_exit: cleaning up %d un-terminated threads\n",
+                get_num_threads());
+
+#ifdef CLIENT_INTERFACE
+            /* make sure client nudges are finished */
+            wait_for_outstanding_nudges();
+#endif
+
+            /* xref case 8747, requesting suspended is preferable to terminated and it
+             * doesn't make a difference here which we use (since the process is about
+             * to die). */
+            DEBUG_DECLARE(ok =)
+                synch_with_all_threads(THREAD_SYNCH_SUSPENDED_AND_CLEANED, 
+                                       &threads, &num_threads, 
+                                       /* Case 6821: other synch-all-thread uses that
+                                        * only care about threads carrying fcache
+                                        * state can ignore us
+                                        */
+                                       THREAD_SYNCH_NO_LOCKS_NO_XFER,
+                                       /* if we fail to suspend a thread (e.g., privilege
+                                        * problems) ignore it. FIXME: retry instead? */
+                                       THREAD_SYNCH_SUSPEND_FAILURE_IGNORE);
+            ASSERT(ok);
+            ASSERT(threads == NULL && num_threads == 0); /* We asked for CLEANED */
+            /* the synch_with_all_threads function grabbed the 
+             * thread_initexit_lock for us! */
+            /* do this now after all threads we know about are killed and 
+             * while we hold the thread_initexit_lock so any new threads that 
+             * are waiting on it won't get in our way (see thread_init()) */
+            dynamo_exited = true;
+            end_synch_with_all_threads(threads, num_threads, false/*don't resume*/);
+        }
+        /* now that APC interception point is unpatched and 
+         * dynamorio_exited is set and we've killed all the theads we know
+         * about, assumption is that no other threads will be running in 
+         * dynamorio code from here on out (esp. when we get into shared exit)
+         * that will do anything that could be dangerous (could possibly be
+         * a thread in the APC interception code prior to reaching thread_init 
+         * but it will only global log and do thread_lookup which should be 
+         * safe throughout) */
+
+        if (thread_lookup(get_thread_id()) == NULL) {
+            LOG(GLOBAL, LOG_TOP|LOG_THREADS, 1,
+                "Current thread never under DynamoRIO control, not exiting it\n");
+        } else {
+            /* call thread_exit even if !under_dynamo_control, could have
+             * been at one time
+             */
+            /* exit this thread now */
+            dynamo_thread_exit();
+        }
+
+        /* FIXME : our call un-interception isn't atomic so (miniscule) chance
+         * of something going wrong if new thread is just hitting its init APC
+         */
+        if (!INTERNAL_OPTION(noasynch)) {
+            callback_interception_unintercept();
+        }
+
+#else /* LINUX */
+        /* avoid protecting DR data structs when clean up thread */
+        dynamo_exited = true;
+        dynamo_thread_exit();
+        unhook_vsyscall();
+#endif /* LINUX */
+        return dynamo_shared_exit(IF_WINDOWS(false /* not detaching */));
+    }
+    return SUCCESS;
+}
+#endif /* DEBUG */
+
+int
+dynamo_nullcalls_exit(void)
+{
+    /* this routine is used when nullcalls is turned on
+     * simply to get perfctr numbers in a log file
+     */
+    ASSERT(INTERNAL_OPTION(nullcalls));
+#ifdef PAPI
+    hardware_perfctr_exit();
+#endif
+
+#ifdef DEBUG
+    if (main_logfile != STDERR) {
+        os_close(main_logfile);
+        main_logfile = INVALID_FILE;
+    }
+#endif /* DEBUG */
+
+    dynamo_exited = true;
+    return SUCCESS;
+}
+
+/* called when we see that the process is about to exit */
+int
+dynamo_process_exit(void)
+{
+#ifndef DEBUG
+    bool each_thread;
+#endif
+    SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+    synchronize_dynamic_options();
+    SYSLOG(SYSLOG_INFORMATION, INFO_PROCESS_STOP, 
+           2, get_application_name(), get_application_pid());
+#ifdef DEBUG
+    if (!dynamo_exited) {
+        if (INTERNAL_OPTION(nullcalls)) {
+            /* if nullcalls is on we still do perfctr stats, and this is
+             * the only place we can print them out and exit
+             */
+            dynamo_nullcalls_exit();
+        }
+        else {
+            /* we don't check automatic_startup -- even if the app_
+             * interface is used, we are about to be gone from the process
+             * address space, so we clean up now
+             */
+            LOG(GLOBAL, LOG_TOP, 1,
+                "\ndynamo_process_exit from thread %d -- cleaning up dynamo\n",
+                get_thread_id());
+            dynamo_process_exit_cleanup();
+        }
+    }
+
+    return SUCCESS;
+
+#else
+    if (dynamo_exited)
+        return SUCCESS;
+
+    dynamo_exited = true;
+
+    /* don't need to do much!
+     * we didn't create any IPC objects or anything that might be persistent
+     * beyond our death, we're not holding any systemwide locks, etc.
+     */
+ 
+    /* It is not clear whether the Event Log service can handle well unterminated connections */
+
+    /* Do we need profile data for each thread?
+     * Note that windows prof_pcs duplicates the thread walk in os_exit()
+     * FIXME: should combine that thread walk with this one
+     */
+    each_thread = TRACEDUMP_ENABLED();
+# ifdef LINUX
+    each_thread = each_thread || INTERNAL_OPTION(profile_pcs);
+# endif
+# ifdef KSTATS
+    each_thread = each_thread || DYNAMO_OPTION(kstats);
+# endif
+# ifdef CLIENT_INTERFACE
+    each_thread = each_thread || !INTERNAL_OPTION(nullcalls);
+# endif
+    if (each_thread) {
+        thread_record_t **threads;
+        int num, i;
+        mutex_lock(&thread_initexit_lock);
+        get_list_of_threads(&threads, &num);
+        for (i = 0; i < num; i++) {
+            /* FIXME: separate trace dump from rest of fragment cleanup code */
+            if (TRACEDUMP_ENABLED() IF_CLIENT_INTERFACE(|| true))
+                /* We always want to call this for CI builds so we can get the
+                 * dr_fragment_deleted() callbacks.
+                 */
+                fragment_thread_exit(threads[i]->dcontext);
+# ifdef LINUX
+            if (INTERNAL_OPTION(profile_pcs))
+                pcprofile_thread_exit(threads[i]->dcontext);
+# endif
+# ifdef KSTATS
+            if (DYNAMO_OPTION(kstats))
+                kstat_thread_exit(threads[i]->dcontext);
+# endif
+# ifdef CLIENT_INTERFACE
+            /* Inform client of all thread exits */
+            if (!INTERNAL_OPTION(nullcalls))
+                instrument_thread_exit(threads[i]->dcontext);
+# endif
+        }
+        global_heap_free(threads, num*sizeof(thread_record_t*) 
+                         HEAPACCT(ACCT_THREAD_MGT));
+        mutex_unlock(&thread_initexit_lock);
+    }
+
+    /* FIXME: separate trace dump from rest of fragment cleanup code.  For client
+     * interface we need to call fragment_exit to get all the fragment deleted events. */
+    if (TRACEDUMP_ENABLED() IF_CLIENT_INTERFACE(|| dr_fragment_deleted_hook_exists()))
+        fragment_exit();
+
+    /* Inform client of process exit */
+#ifdef CLIENT_INTERFACE
+    if (!INTERNAL_OPTION(nullcalls)) {
+# ifdef WINDOWS
+        /* instrument_exit() unloads the client library, so make sure
+         * LdrUnloadDll isn't hooked.
+         */
+        if (!INTERNAL_OPTION(noasynch)) {
+            callback_interception_unintercept();
+        }
+# endif
+        /* Must be after fragment_exit() so that the client gets all the
+         * fragment_deleted() callbacks (xref PR 228156).  FIXME - might be issues
+         * with the client trying to use api routines that depend on fragment state.
+         * FIXME (xref PR 200207) must be after callback_interception_unintercept() since
+         * we need to remove our LdrUnloadDll hook before we unload the client library,
+         * but this means we won't catch exceptions for the client here. */
+        /* PR 327300: avoid crashes if client tries to access the now-freed
+         * dcontext->client_data (dr_get_tls_field(), etc. will crash) */
+        set_thread_private_dcontext(NULL);
+        instrument_exit();
+    }
+#endif
+ 
+#ifdef CALL_PROFILE
+    profile_callers_exit();
+#endif
+    dynamo_process_exit_with_thread_info();
+# ifdef KSTATS
+    if (DYNAMO_OPTION(kstats))
+        kstat_exit();
+# endif
+    /* so make sure eventlog connection is terminated (if present)  */
+    os_fast_exit();
+
+    return SUCCESS;
+#endif /* !DEBUG */
+}
+
+dcontext_t *
+create_new_dynamo_context(bool initial)
+{
+    dcontext_t *dcontext;
+    size_t alloc = sizeof(dcontext_t) + proc_get_cache_line_size();
+    void *alloc_start = (void *) 
+        ((TEST(SELFPROT_GLOBAL, dynamo_options.protect_mask) &&
+          !TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)) ?
+         /* if protecting global but not dcontext, put whole thing in unprot mem */
+         global_unprotected_heap_alloc(alloc HEAPACCT(ACCT_OTHER)) :
+         global_heap_alloc(alloc HEAPACCT(ACCT_OTHER)));
+    dcontext = (dcontext_t*) proc_bump_to_end_of_cache_line((ptr_uint_t)alloc_start);
+    ASSERT(proc_is_cache_aligned(dcontext));
+    /* 264138: ensure xmm0-5 are aligned so we can use movdqa */
+    ASSERT(ALIGNED(get_mcontext(dcontext)->xmm, 16));
+    /* also ensure we don't have extra padding beyond x86.asm defines */
+    ASSERT(sizeof(dr_mcontext_t) == IF_X64_ELSE(18,10)*sizeof(reg_t) + XMM_SLOTS_SIZE);
+
+    /* Put here all one-time dcontext field initialization 
+     * Make sure to update create_callback_dcontext to shared
+     * fields across callback dcontexts for the same thread.
+     */
+    /* must set to 0 so can tell if initialized for callbacks! */
+    memset(dcontext, 0x0, sizeof(dcontext_t));
+    dcontext->allocated_start = alloc_start;
+    /* we share a single dstack across all callbacks */
+    if (initial)
+        dcontext->dstack = (byte *) stack_alloc(DYNAMORIO_STACK_SIZE);
+#ifdef RETURN_STACK
+    dcontext->rstack = (byte *) stack_alloc(DYNAMORIO_STACK_SIZE);
+    /* Cannot have RETURN_STACK with inlined IBLs */
+    ASSERT_NOT_IMPLEMENTED(false == DYNAMO_OPTION(inline_trace_ibl));
+#endif
+    if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)) {
+        dcontext->upcontext.separate_upcontext =
+            global_unprotected_heap_alloc(sizeof(unprotected_context_t) HEAPACCT(ACCT_OTHER));
+        /* don't need to initialize upcontext */
+        LOG(GLOBAL, LOG_TOP, 2, "new dcontext="PFX", dcontext->upcontext="PFX"\n",
+            dcontext, dcontext->upcontext.separate_upcontext);
+        dcontext->upcontext_ptr = dcontext->upcontext.separate_upcontext;
+    } else
+        dcontext->upcontext_ptr = &(dcontext->upcontext.upcontext);
+#ifdef HOT_PATCHING_INTERFACE
+    /* Set the hot patch exception state to be empty/unused. */
+    DODEBUG(memset(&dcontext->hotp_excpt_state, -1, sizeof(dr_jmp_buf_t)););
+#endif
+    ASSERT(dcontext->try_except_state == NULL);
+
+    DODEBUG({dcontext->logfile = INVALID_FILE;});
+    dcontext->owning_thread = get_thread_id();
+#ifdef LINUX
+    dcontext->owning_process = get_process_id();
+#endif
+    /* thread_record is set in add_thread */
+    /* all of the thread-private fcache and hashtable fields are shared
+     * among all dcontext instances of a thread, so the caller must
+     * set those fields
+     */
+    /* rest of dcontext initialization happens in initialize_dynamo_context(),
+     * which is executed for each dr_app_start() and each
+     * callback start
+     */
+    return dcontext;
+}
+
+static void
+delete_dynamo_context(dcontext_t *dcontext, bool free_stack)
+{
+    if (free_stack) {
+        ASSERT(dcontext->dstack != NULL);
+        ASSERT(!is_currently_on_dstack(dcontext));
+        stack_free(dcontext->dstack, DYNAMORIO_STACK_SIZE);
+    } /* else will be cleaned up by caller */
+
+    ASSERT(dcontext->try_except_state == NULL);
+#ifdef WINDOWS
+    aslr_free_last_section_file_name(dcontext);
+#endif
+
+#ifdef RETURN_STACK
+    LOG(THREAD, LOG_TOP, 1, "Return stack still has %d pair(s) on it\n",
+        (dcontext->rstack - dcontext->top_of_rstack) / 8);
+    stack_free(dcontext->rstack, DYNAMORIO_STACK_SIZE);
+#endif
+
+    if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)) {
+        global_unprotected_heap_free(dcontext->upcontext.separate_upcontext,
+                                     sizeof(unprotected_context_t) HEAPACCT(ACCT_OTHER));
+    }
+    if (TEST(SELFPROT_GLOBAL, dynamo_options.protect_mask) &&
+        !TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)) {
+        /* if protecting global but not dcontext, we put whole thing in unprot mem */
+        global_unprotected_heap_free(dcontext->allocated_start,
+                                     sizeof(dcontext_t) + proc_get_cache_line_size()
+                                     HEAPACCT(ACCT_OTHER));
+    } else {
+        global_heap_free(dcontext->allocated_start,
+                         sizeof(dcontext_t) + proc_get_cache_line_size()
+                         HEAPACCT(ACCT_OTHER));
+    }
+}
+
+/* This routine is called not only at thread initialization,
+ * but for every callback, etc. that gets a fresh execution
+ * environment!
+ */
+void
+initialize_dynamo_context(dcontext_t *dcontext)
+{
+    /* we can't just zero out the whole thing b/c we have persistent state
+     * (fields kept across callbacks, like dstack, module-private fields, next & prev, etc.)
+     */
+    memset(dcontext->upcontext_ptr, 0, sizeof(unprotected_context_t));
+    dcontext->initialized = true;
+    dcontext->whereami = WHERE_APP;
+    dcontext->next_tag = NULL;
+    dcontext->native_exec_retval = NULL;
+    dcontext->native_exec_retloc = NULL;
+    dcontext->native_exec_postsyscall = NULL;
+#ifdef RETURN_STACK
+    dcontext->top_of_rstack = dcontext->rstack;
+#endif
+#ifdef X64
+    dcontext->x86_mode = false;
+#endif
+    dcontext->sys_num = 0;
+#ifdef WINDOWS
+    dcontext->app_errno = 0;
+    dcontext->sys_param_base = NULL;
+    /* always initialize aslr_context */
+    dcontext->aslr_context.sys_aslr_clobbered = 0;
+    dcontext->aslr_context.randomized_section_handle = INVALID_HANDLE_VALUE;
+    dcontext->aslr_context.original_image_section_handle = INVALID_HANDLE_VALUE;
+    dcontext->aslr_context.original_section_base = ASLR_INVALID_SECTION_BASE;
+    /* note that aslr_context.last_child_padded is preserved across callbacks */
+    aslr_free_last_section_file_name(dcontext);
+    dcontext->ignore_enterexit = false;
+#else
+    dcontext->sys_param0 = 0;
+    dcontext->sys_param1 = 0;
+    dcontext->sys_param2 = 0;
+#endif
+
+#ifdef LINUX
+    dcontext->signals_pending = false;
+#endif
+
+    /* all thread-private fields are initialized in dynamo_thread_init
+     * or in create_callback_dcontext because they must be initialized differently
+     * in those two cases
+     */
+
+    set_last_exit(dcontext, (linkstub_t *) get_starting_linkstub());
+
+#ifdef NATIVE_RETURN
+# ifdef NATIVE_RETURN_CALLDEPTH
+    dcontext->call_depth = 0;
+# endif
+    dcontext->last_retaddr = NULL;
+    dcontext->num_calls_in_trace = 0;
+#endif
+#ifdef PROFILE_RDTSC
+    dcontext->start_time = (uint64) 0;
+    dcontext->prev_fragment = NULL;
+    dcontext->cache_frag_count = (uint64) 0;
+    {
+        int i;
+        for (i=0; i<10; i++) {
+            dcontext->cache_time[i] = (uint64) 0;
+            dcontext->cache_count[i] = (uint64) 0;
+        }
+    }
+#endif
+#ifdef DEBUG
+    dcontext->in_opnd_disassemble = false;
+#endif
+#ifdef WINDOWS
+    /* Other pieces of DR -- callback & APC handling, detach -- test
+     * asynch_target to determine where the next app pc to execute is
+     * stored. Init it to 0 to indicate that this context's most recent
+     * syscall was not executed from handle_system_call().
+     */
+    dcontext->asynch_target = NULL;
+    /* next_saved and prev_unused are zeroed out when dcontext is
+     * created; we shouldn't zero them here, they may have valid data
+     */
+    dcontext->valid = true;
+#endif
+#ifdef HOT_PATCHING_INTERFACE
+    dcontext->nudge_thread = false;     /* Fix for case 5367. */
+#endif
+#ifdef CHECK_RETURNS_SSE2
+    /* initialize sse2 index with 0
+     * go ahead and use eax, it's dead (about to return)
+     */
+# ifdef LINUX
+    asm("movl $0, %eax");
+    asm("pinsrw $7,%eax,%xmm7");
+# else
+# error NYI
+# endif
+#endif
+    /* We don't need to initialize dcontext->coarse_exit as it is only
+     * read when last_exit indicates a coarse exit, which sets the fields.
+     */
+}
+
+#ifdef WINDOWS
+/* on windows we use a new dcontext for each callback context */
+dcontext_t *
+create_callback_dcontext(dcontext_t *old_dcontext)
+{
+    dcontext_t *new_dcontext = create_new_dynamo_context(false);
+    new_dcontext->valid = false;
+    /* all of these fields are shared among all dcontexts of a thread: */
+    new_dcontext->owning_thread = old_dcontext->owning_thread;
+#ifdef LINUX
+    new_dcontext->owning_process = old_dcontext->owning_process;
+#endif
+    new_dcontext->thread_record = old_dcontext->thread_record;
+    /* now that we have clean stack usage we can share a single stack */
+    ASSERT(old_dcontext->dstack != NULL);
+    new_dcontext->dstack = old_dcontext->dstack;
+#ifdef X64
+    new_dcontext->x86_mode = old_dcontext->x86_mode;
+#endif
+    new_dcontext->link_field = old_dcontext->link_field;
+    new_dcontext->monitor_field = old_dcontext->monitor_field;
+    new_dcontext->fcache_field = old_dcontext->fcache_field;
+    new_dcontext->fragment_field = old_dcontext->fragment_field;
+    new_dcontext->heap_field = old_dcontext->heap_field;
+    new_dcontext->vm_areas_field = old_dcontext->vm_areas_field;
+    new_dcontext->os_field = old_dcontext->os_field;
+    new_dcontext->synch_field = old_dcontext->synch_field;
+    /* case 8958: copy win32_start_addr in case we produce a forensics file
+     * from within a callback.
+     */
+    new_dcontext->win32_start_addr = old_dcontext->win32_start_addr;
+#ifdef LINUX
+    new_dcontext->signal_field = old_dcontext->signal_field;
+    new_dcontext->pcprofile_field = old_dcontext->pcprofile_field;
+#endif
+    new_dcontext->private_code = old_dcontext->private_code;
+#ifdef CLIENT_INTERFACE
+    new_dcontext->client_data = old_dcontext->client_data;
+#endif
+#ifdef DEBUG
+    new_dcontext->logfile = old_dcontext->logfile;
+    new_dcontext->thread_stats = old_dcontext->thread_stats;
+#endif
+#ifdef DEADLOCK_AVOIDANCE
+    new_dcontext->thread_owned_locks = old_dcontext->thread_owned_locks;
+#endif
+#ifdef KSTATS
+    new_dcontext->thread_kstats = old_dcontext->thread_kstats;
+#endif
+    /* at_syscall is real time based, not app context based, so shared 
+     *
+     * FIXME: Yes need to share when swapping at NtCallbackReturn, but
+     * want to keep old so when return from cb will do post-syscall for
+     * syscall that triggered cb in the first place!
+     * Plus, new cb calls initialize_dynamo_context(), which clears this field
+     * anyway!  This all works now b/c we don't have alertable syscalls
+     * that we do post-syscall processing on.
+     */
+    new_dcontext->upcontext_ptr->at_syscall = old_dcontext->upcontext_ptr->at_syscall;
+#ifdef HOT_PATCHING_INTERFACE   /* Fix for case 5367. */
+    /* hotp_excpt_state should be unused at this point.  If it is used, it can
+     * be only because a hot patch made a system call with a callback.  This is 
+     * a bug because hot patches can't do system calls, let alone one with 
+     * callbacks.
+     */
+    DODEBUG({
+        dr_jmp_buf_t empty;
+        memset(&empty, -1, sizeof(dr_jmp_buf_t));
+        ASSERT(memcmp(&old_dcontext->hotp_excpt_state, &empty,
+                      sizeof(dr_jmp_buf_t)) == 0);
+    });
+    new_dcontext->nudge_thread = old_dcontext->nudge_thread;
+#endif
+    /* our exceptions should be handled within one DR context switch */
+    ASSERT(old_dcontext->try_except_state == NULL);
+    new_dcontext->local_state = old_dcontext->local_state;
+#ifdef WINDOWS
+    new_dcontext->aslr_context.last_child_padded =
+        old_dcontext->aslr_context.last_child_padded;
+#endif
+
+    LOG(new_dcontext->logfile, LOG_TOP, 2,
+        "made new dcontext "PFX" (old="PFX")\n", new_dcontext, old_dcontext);
+    return new_dcontext;
+}
+#endif
+
+bool
+is_thread_initialized(void)
+{
+#if defined(LINUX) && defined(HAVE_TLS)
+    /* We don't want to pay the get_thread_id() cost on every
+     * get_thread_private_dcontext() when we only really need the
+     * check for this call here, so we explicitly check.
+     */
+    if (get_tls_thread_id() != get_thread_id())
+        return false;
+#endif
+    return (get_thread_private_dcontext() != NULL);
+}
+
+
+bool
+is_thread_known(thread_id_t tid)
+{
+    return (thread_lookup(tid) != NULL);
+}
+
+int
+get_num_threads(void)
+{
+    return num_known_threads;
+}
+
+/* This routine takes a snapshot of all the threads known to DR,
+ * NOT LIMITED to those currently under DR control!
+ * It returns an array of thread_record_t* and the length of the array
+ * The caller must free the array using global_heap_free
+ * The caller must hold the thread_initexit_lock to ensure that threads
+ * are not created or destroyed before the caller is done with the list
+ * The caller CANNOT be could_be_linking, else a deadlock with flushing
+ * can occur (unless the caller is the one flushing)
+ */
+void
+get_list_of_threads(thread_record_t ***list, int *num)
+{
+    int i, cur = 0, max_num;
+    thread_record_t *tr;
+    thread_record_t **mylist;
+
+    /* Only a flushing thread can get the thread snapshot while being
+     * couldbelinking -- else a deadlock w/ flush!
+     * FIXME: this assert should be on any acquisition of thread_initexit_lock!
+     */
+    ASSERT(is_self_flushing() || !is_self_couldbelinking());
+    ASSERT(all_threads != NULL);
+    ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
+
+    mutex_lock(&all_threads_lock);
+    max_num = get_num_threads();
+    mylist = (thread_record_t **) global_heap_alloc(max_num*sizeof(thread_record_t*)
+                                                 HEAPACCT(ACCT_THREAD_MGT));
+    for (i = 0; i < HASHTABLE_SIZE(ALL_THREADS_HASH_BITS); i++) {
+        for (tr = all_threads[i]; tr != NULL; tr = tr->next) {
+            /* include those for which !tr->under_dynamo_control */
+            mylist[cur] = tr;
+            cur++;
+        }
+    }
+    
+    ASSERT(cur == max_num);
+
+    *num = cur;
+    *list = mylist;
+    mutex_unlock(&all_threads_lock);
+}
+
+/* assumes caller can ensure that thread is either suspended or self to
+ * avoid races
+ */
+thread_record_t * 
+thread_lookup(thread_id_t tid)
+{
+    thread_record_t *tr;
+    uint hindex;
+
+    /* check that caller is self or has initexit_lock
+     * FIXME: no way to tell who has initexit_lock
+     */
+    ASSERT(mutex_testlock(&thread_initexit_lock) || tid == get_thread_id());
+
+    hindex = HASH_FUNC_BITS(tid, ALL_THREADS_HASH_BITS);
+    mutex_lock(&all_threads_lock);
+    if (all_threads == NULL) { 
+        tr = NULL;
+    } else { 
+        tr = all_threads[hindex];
+    }
+    while (tr != NULL) {
+        if (tr->id == tid) {
+            mutex_unlock(&all_threads_lock);
+            return tr;
+        }
+        tr = tr->next;
+    }
+    mutex_unlock(&all_threads_lock);
+    return NULL;
+}
+
+/* assumes caller can ensure that thread is either suspended or self to
+ * avoid races
+ */
+uint
+get_thread_num(thread_id_t tid)
+{
+    thread_record_t *tr = thread_lookup(tid);
+    if (tr != NULL)
+        return tr->num;
+    else
+        return 0; /* yes can't distinguish from 1st thread, who cares */
+}
+
+void
+add_thread(IF_WINDOWS_(HANDLE hthread) 
+           thread_id_t tid, bool under_dynamo_control,
+           dcontext_t *dcontext)
+{
+    thread_record_t *tr;
+    uint hindex;
+
+    ASSERT(all_threads != NULL);
+
+    /* add entry to thread hashtable */
+    tr = (thread_record_t *) global_heap_alloc(sizeof(thread_record_t)
+                                            HEAPACCT(ACCT_THREAD_MGT));
+#ifdef WINDOWS
+    /* we duplicate the thread psuedo-handle, this should give us full rights
+     * Note that instead asking explicitly for THREAD_ALL_ACCESS or just for
+     * THREAD_TERMINATE|THREAD_SUSPEND_RESUME|THREAD_GET_CONTEXT|THREAD_SET_CONTEXT
+     * does not seem able to acquire more rights than simply duplicating the
+     * app handle gives.
+     */
+    LOG(GLOBAL, LOG_THREADS, 1, "Thread %d app handle rights: "PFX"\n",
+        tid, nt_get_handle_access_rights(hthread));
+    duplicate_handle(NT_CURRENT_PROCESS, hthread, NT_CURRENT_PROCESS,
+                     &tr->handle, 0, 0, 
+                     DUPLICATE_SAME_ACCESS|DUPLICATE_SAME_ATTRIBUTES);
+    /* We prob. only need TERMINATE (for kill thread), SUSPEND/RESUME/GET_CONTEXT
+     * (for synchronizing), and SET_CONTEXT (+ synchronizing requirements, for 
+     * detach).  All access includes this and quite a bit more. */
+# if 0
+    /* eventually should be a real assert, but until we have a story for the
+     * injected detach threads, have to ifdef out even the ASSERT_CURIOSITY 
+     * (even a syslog internal warning is prob. to noisy for QA) */
+    ASSERT_CURIOSITY(TESTALL(THREAD_ALL_ACCESS, nt_get_handle_access_rights(tr->handle)));
+# endif
+    LOG(GLOBAL, LOG_THREADS, 1, "Thread %d our handle rights: "PFX"\n",
+        tid, nt_get_handle_access_rights(tr->handle));
+#endif
+    tr->id = tid;
+    ASSERT(tid != INVALID_THREAD_ID); /* ensure os never assigns invalid id to a thread */
+    tr->under_dynamo_control = under_dynamo_control;
+    tr->dcontext = dcontext;
+    if (dcontext != NULL) /* we allow NULL for dr_create_client_thread() */
+        dcontext->thread_record = tr;
+
+    mutex_lock(&all_threads_lock);
+    tr->num = threads_ever_count++;
+    hindex = HASH_FUNC_BITS(tr->id, ALL_THREADS_HASH_BITS);
+    tr->next = all_threads[hindex];
+    all_threads[hindex] = tr;
+    /* must be inside all_threads_lock to avoid race w/ get_list_of_threads */
+    RSTATS_ADD_PEAK(num_threads, 1);
+    RSTATS_INC(num_threads_created);
+    num_known_threads++;
+    mutex_unlock(&all_threads_lock);
+}
+
+/* return false if couldn't find the thread */
+bool
+remove_thread(IF_WINDOWS_(HANDLE hthread) thread_id_t tid)
+{
+    thread_record_t *tr = NULL, *prevtr;
+    uint hindex = HASH_FUNC_BITS(tid, ALL_THREADS_HASH_BITS);
+
+    ASSERT(all_threads != NULL);
+
+    mutex_lock(&all_threads_lock);
+    for (tr = all_threads[hindex], prevtr = NULL; tr; prevtr = tr, tr = tr->next) {
+        if (tr->id == tid) {
+            if (prevtr)
+                prevtr->next = tr->next;
+            else
+                all_threads[hindex] = tr->next;
+#ifdef WINDOWS
+            close_handle(tr->handle);
+#endif
+            global_heap_free(tr, sizeof(thread_record_t) HEAPACCT(ACCT_THREAD_MGT));
+            break;
+        }
+    }
+    /* must be inside all_threads_lock to avoid race w/ get_list_of_threads */
+    if (tr != NULL) {
+        RSTATS_DEC(num_threads);
+        num_known_threads--;
+    }
+    mutex_unlock(&all_threads_lock);
+    return (tr != NULL);
+}
+
+/* this bool is protected by reset_pending_lock */
+DECLARE_FREQPROT_VAR(static bool reset_at_nth_thread_triggered, false);
+
+/* thread-specific initialization 
+ * returns -1 if current thread has already been initialized
+ */
+int
+dynamo_thread_init(void)
+{
+    dcontext_t *dcontext;
+    /* due to lock issues (see below) we need another var */
+    bool reset_at_nth_thread_pending = false;
+    bool under_dynamo_control = true;
+    APP_EXPORT_ASSERT(dynamo_initialized || dynamo_exited ||
+                      get_num_threads() == 0,
+                      PRODUCT_NAME" not initialized");
+
+    if (INTERNAL_OPTION(nullcalls))
+        return SUCCESS;
+
+    /* note that ENTERING_DR is assumed to have already happened: in apc handler
+     * for win32, in new_thread_setup for linux, in main init for 1st thread
+     */
+
+    /* synch point so thread creation can be prevented for critical periods */
+    mutex_lock(&thread_initexit_lock);
+
+    /* The assumption is that if dynamo_exited, then we are about to exit and
+     * clean up, initializing this thread then would be dangerous, better to 
+     * wait here for the app to die.  Is safe with detach, since a thread 
+     * should never reach here when dynamo_exited is true during detach */
+    /* under current implementation of process exit, can happen only under
+     * debug build, or app_start app_exit interface */
+    while (dynamo_exited) {
+        /* logging should be safe, though might not actually result in log
+         * message */
+        DODEBUG_ONCE(LOG(GLOBAL, LOG_THREADS, 1, 
+                         "Thread %d reached initialization point while dynamo exiting, "
+                         "waiting for app to exit\n", get_thread_id());); 
+        mutex_unlock(&thread_initexit_lock);
+        thread_yield();
+        /* just in case we want to support exited and then restarted at some 
+         * point */
+        mutex_lock(&thread_initexit_lock);
+    }
+
+
+    if (is_thread_initialized()) {
+        mutex_unlock(&thread_initexit_lock);
+        return -1;
+    }
+
+    os_tls_init();
+    dcontext = create_new_dynamo_context(true/*initial*/);
+    initialize_dynamo_context(dcontext);
+    set_thread_private_dcontext(dcontext);
+    /* sanity check */
+    ASSERT(get_thread_private_dcontext() == dcontext);
+
+    /* set local state pointer for access from other threads */
+    dcontext->local_state = get_local_state();
+
+    /* For hotp_only, the thread should run native, not under dr.  However,
+     * the core should still get control of the thread at hook points to track 
+     * what the application is doing & at patched points to execute hot patches.
+     * It is the same for thin_client except that there are fewer hooks, only to
+     * follow children.
+     */
+    if (RUNNING_WITHOUT_CODE_CACHE())
+        under_dynamo_control = false;
+
+    /* add entry to thread hashtable before creating logdir so have thread num.
+     * otherwise we'd like to do this only after we'd fully initialized the thread, but we
+     * hold the thread_initexit_lock, so nobody should be listing us -- thread_lookup
+     * on other than self, or a thread list, should only be done while the initexit_lock
+     * is held.  CHECK: is this always correct?  thread_lookup does have an assert
+     * to try and enforce but cannot tell who has the lock.
+     */
+    add_thread(IF_WINDOWS_(NT_CURRENT_THREAD) get_thread_id(), 
+               under_dynamo_control, dcontext);
+
+    LOG(GLOBAL, LOG_TOP|LOG_THREADS, 1,
+        "\ndynamo_thread_init: %d thread(s) now, dcontext="PFX", #=%d, id=%d, pid=%d\n\n",
+        GLOBAL_STAT(num_threads), dcontext, get_thread_num(get_thread_id()),
+        get_thread_id(), get_process_id());
+
+    DOLOG(1, LOG_STATS, {
+        dump_global_stats(false);
+    });
+#ifdef DEBUG
+    if (stats->loglevel > 0) {
+        dcontext->logfile = open_log_file(thread_logfile_name(), NULL, 0);
+        print_file(dcontext->logfile, "%s\n", dynamorio_version_string);
+    } else {
+        dcontext->logfile = INVALID_FILE;
+    }
+    DOLOG(1, LOG_TOP|LOG_THREADS, {
+        LOG(THREAD, LOG_TOP|LOG_THREADS, 1, PRODUCT_NAME" built with: %s\n", DYNAMORIO_DEFINES);
+        LOG(THREAD, LOG_TOP|LOG_THREADS, 1, PRODUCT_NAME" built on: %s\n", dynamorio_buildmark);
+    });
+
+    LOG(THREAD, LOG_TOP|LOG_THREADS, 1,
+        "THREAD %d (dcontext "PFX")\n\n", get_thread_id(), dcontext);
+    LOG(THREAD, LOG_TOP|LOG_THREADS, 1,
+        "DR stack is "PFX"-"PFX"\n", dcontext->dstack - DYNAMORIO_STACK_SIZE, dcontext->dstack);
+#endif
+
+#ifdef DEADLOCK_AVOIDANCE
+    locks_thread_init(dcontext);
+#endif
+    heap_thread_init(dcontext);
+    DOSTATS({ stats_thread_init(dcontext); });
+#ifdef KSTATS
+    kstat_thread_init(dcontext);
+#endif
+    os_thread_init(dcontext);
+    arch_thread_init(dcontext);
+    synch_thread_init(dcontext);
+
+    if (!DYNAMO_OPTION(thin_client))
+        vm_areas_thread_init(dcontext);
+
+    monitor_thread_init(dcontext);
+    fcache_thread_init(dcontext);
+    link_thread_init(dcontext);
+    fragment_thread_init(dcontext);
+
+    if (!DYNAMO_OPTION(thin_client)) {
+#ifdef CLIENT_INTERFACE
+        /* put client last, may depend on other thread inits.
+         * Note that we are calling this prior to instrument_init()
+         * now (PR 216936), which is required to intialize
+         * the client dcontext field prior to instrument_init().
+         */
+        instrument_thread_init(dcontext);
+#endif
+
+#ifdef SIDELINE
+        if (dynamo_options.sideline) {
+            /* wake up sideline thread -- ok to call if thread already awake */
+            sideline_start();
+        }
+#endif
+    }
+
+    /* must check # threads while holding thread_initexit_lock, yet cannot
+     * call fcache_reset_all_caches_proactively while holding it due to
+     * rank order of reset_pending_lock which we must also hold -- so we
+     * set a local bool reset_at_nth_thread_pending
+     */
+    if (DYNAMO_OPTION(reset_at_nth_thread) != 0 && !reset_at_nth_thread_triggered
+        && (uint) get_num_threads() == DYNAMO_OPTION(reset_at_nth_thread)) {
+        mutex_lock(&reset_pending_lock);
+        if (!reset_at_nth_thread_triggered) {
+            reset_at_nth_thread_triggered = true;
+            reset_at_nth_thread_pending = true;
+        }
+        mutex_unlock(&reset_pending_lock);
+    }
+
+    mutex_unlock(&thread_initexit_lock);
+    DOLOG(1, LOG_STATS, {
+        dump_thread_stats(dcontext, false);
+    });
+
+    if (reset_at_nth_thread_pending) {
+        mutex_lock(&reset_pending_lock);
+        /* fcache_reset_all_caches_proactively() will unlock */
+        fcache_reset_all_caches_proactively(RESET_ALL);
+    }
+    return SUCCESS;
+}
+
+
+/* thread-specific cleanup */
+/* Note : if this routine is not called by thread id, then other_thread should
+ * be true and the calling thread should hold the thread_initexit_lock
+ */
+static int
+dynamo_thread_exit_common(dcontext_t *dcontext, thread_id_t id,
+                          IF_WINDOWS_(bool detach_stacked_callbacks) bool other_thread)
+{
+    dcontext_t *dcontext_tmp;
+#ifdef WINDOWS
+    dcontext_t *dcontext_next;
+    int num_dcontext;
+#endif
+    bool on_dstack = !other_thread && is_currently_on_dstack(dcontext);
+    /* cache this now for use after freeing dcontext */
+    local_state_t *local_state = dcontext->local_state;
+
+    if (INTERNAL_OPTION(nullcalls) || dcontext == NULL)
+        return SUCCESS;
+
+    /* make sure don't get into deadlock w/ flusher */
+    enter_threadexit(dcontext);
+
+    /* synch point so thread exiting can be prevented for critical periods */
+    /* see comment at start of method for other thread exit */
+    if (!other_thread) 
+        mutex_lock(&thread_initexit_lock);
+
+    ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
+#ifdef WINDOWS
+    /* need to clean up thread stack before clean up other thread data, but
+     * after we're made nolinking
+     */
+    os_thread_stack_exit(dcontext);
+    /* free the thread's application stack if requested */
+    if (dcontext->free_app_stack) {
+        byte *base;
+        /* only used for nudge threads currently */
+        ASSERT(dcontext->nudge_target == generic_nudge_target);
+        if (get_stack_bounds(dcontext, &base, NULL)) {
+            NTSTATUS res;
+            ASSERT(base != NULL);
+            res = nt_free_virtual_memory(base);
+            ASSERT(NT_SUCCESS(res));
+        } else {
+            /* stack should be available here */
+            ASSERT_NOT_REACHED();
+        }
+    }
+#endif
+
+#ifdef SIDELINE
+    /* N.B.: do not clean up any data structures while sideline thread
+     * is still running!  put it to sleep for duration of this routine!
+     */
+    if (!DYNAMO_OPTION(thin_client)) {
+        if (dynamo_options.sideline) {
+            /* put sideline thread to sleep */
+            sideline_stop();
+            /* sideline_stop will not return until sideline thread is asleep */
+        }
+    }
+#endif
+
+    LOG(GLOBAL, LOG_TOP|LOG_THREADS, 1,
+        "\ndynamo_thread_exit (thread #%d id=%d): %d thread(s) now\n\n",
+        get_thread_num(id), id, GLOBAL_STAT(num_threads)-1);
+
+    DOLOG(1, LOG_STATS, {
+        dump_global_stats(false);
+    });
+
+    LOG(THREAD, LOG_STATS|LOG_THREADS, 1, "\n## Statistics for this thread:\n");
+
+#ifdef PROFILE_RDTSC
+    if (dynamo_options.profile_times) {
+        int i;
+        ASSERT(dcontext);
+        LOG(THREAD, LOG_STATS|LOG_THREADS, 1, "\nTop ten cache times:\n");
+        for (i=0; i<10; i++) {
+            if (dcontext->cache_time[i] > (uint64) 0) {
+                uint top_part, bottom_part;
+                divide_int64_print(dcontext->cache_time[i], kilo_hertz, false, 
+                                   3, &top_part, &bottom_part);
+                LOG(THREAD, LOG_STATS|LOG_THREADS, 1, 
+                    "\t#%2d = %6u.%.3u ms, %9d hits\n", 
+                    i+1, top_part, bottom_part, (int)dcontext->cache_count[i]);
+            }
+        }
+        LOG(THREAD, LOG_STATS|LOG_THREADS, 1, "\n");
+    }
+#endif
+
+    /* fcache stats needs to examine fragment state, so run it before
+     * fragment exit, but real fcache exit needs to be after fragment exit
+     */
+#ifdef DEBUG
+    fcache_thread_exit_stats(dcontext);
+#endif
+    /* must abort now to avoid deleting possibly un-deletable fragments
+     * monitor_thread_exit remains later b/c of monitor_remove_fragment calls
+     */
+    trace_abort(dcontext);
+    fragment_thread_exit(dcontext);
+#ifdef CLIENT_INTERFACE
+    /* PR 243759: don't free client_data until after all fragment deletion events */
+    if (!DYNAMO_OPTION(thin_client))
+        instrument_thread_exit(dcontext);
+#endif
+    fcache_thread_exit(dcontext);
+    link_thread_exit(dcontext);
+    monitor_thread_exit(dcontext);
+    if (!DYNAMO_OPTION(thin_client))
+        vm_areas_thread_exit(dcontext);
+    synch_thread_exit(dcontext);
+    arch_thread_exit(dcontext _IF_WINDOWS(detach_stacked_callbacks));
+    os_thread_exit(dcontext);
+    DOLOG(1, LOG_STATS, {
+        dump_thread_stats(dcontext, false);
+    });
+#ifdef KSTATS
+    kstat_thread_exit(dcontext);
+#endif
+    DOSTATS({ stats_thread_exit(dcontext); });
+    heap_thread_exit(dcontext);
+#ifdef DEADLOCK_AVOIDANCE
+    locks_thread_exit(dcontext);
+#endif
+
+#ifdef DEBUG
+    if (dcontext->logfile != INVALID_FILE) {
+        os_flush(dcontext->logfile);
+        os_close(dcontext->logfile);
+    }
+#endif
+
+    /* remove thread from threads hashtable */
+    remove_thread(IF_WINDOWS_(NT_CURRENT_THREAD) id);
+
+    dcontext_tmp = dcontext;
+#ifdef WINDOWS
+    /* clean up all the dcs */
+    num_dcontext = 0;
+# ifdef DCONTEXT_IN_EDI
+    /* go to one end of list */
+    while (dcontext_tmp->next_saved)
+        dcontext_tmp = dcontext_tmp->next_saved;
+# else
+    /* already at one end of list */
+# endif
+    /* make sure we invalidate the dcontext before releasing the memory  */
+    /* when cleaning up other threads, we cannot set their dcs to null,
+     * but we only do this at dynamorio_app_exit so who cares
+     */
+    if (id == get_thread_id())
+        set_thread_private_dcontext(NULL);
+
+    /* delete through to other end */
+    while (dcontext_tmp) {
+        num_dcontext++;
+        dcontext_next = dcontext_tmp->prev_unused;
+        delete_dynamo_context(dcontext_tmp,
+                              dcontext_tmp == dcontext/*do not free dup cb stacks*/
+                              && !on_dstack/*do not free own stack*/);
+        dcontext_tmp = dcontext_next;
+    }
+    LOG(GLOBAL, LOG_STATS|LOG_THREADS, 1, "\tdynamo contexts used: %d\n",
+        num_dcontext);
+#else /* LINUX */
+    set_thread_private_dcontext(NULL);
+    delete_dynamo_context(dcontext_tmp, !on_dstack/*do not free own stack*/);
+#endif /* LINUX */
+    os_tls_exit(local_state);
+
+#ifdef SIDELINE
+    /* see notes above -- we can now wake up sideline thread */
+    if (dynamo_options.sideline && get_num_threads() > 0) {
+        sideline_start();
+    }
+#endif
+    if (!other_thread) {
+        mutex_unlock(&thread_initexit_lock);
+        /* FIXME: once thread_initexit_lock is released, we're not on
+         * thread list, and a terminate targeting us could kill us in the middle
+         * of this call -- but this can't come before the unlock b/c the lock's
+         * in the data segment!  (see case 3121)
+         * (note we do not re-protect for process exit, see !dynamo_exited check
+         * in exiting_dynamorio)
+         */
+        if (!on_dstack) {
+            EXITING_DR();
+            /* else, caller will clean up stack and then call EXITING_DR(),
+             * probably via dynamo_thread_stack_free_and_exit(), as the stack free
+             * must be done before the exit
+             */
+        } 
+    }
+
+    return SUCCESS;
+}
+
+int
+dynamo_thread_exit(void)
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    return dynamo_thread_exit_common(dcontext, get_thread_id(), IF_WINDOWS_(false) false);
+}
+
+/* NOTE : you must hold thread_initexit_lock to call this function! */
+int
+dynamo_other_thread_exit(thread_record_t *tr _IF_WINDOWS(bool detach_stacked_callbacks))
+{
+    /* FIXME: Usually a safe spot for cleaning other threads should be
+     * under num_exits_dir_syscall, but for now rewinding all the way
+     */
+    KSTOP_REWIND_DC(tr->dcontext, thread_measured);
+    KSTART_DC(tr->dcontext, thread_measured);
+    return dynamo_thread_exit_common(tr->dcontext, tr->id,
+                                     IF_WINDOWS_(detach_stacked_callbacks) true);
+}
+
+/* Called from another stack to finish cleaning up a thread.
+ * The final steps are to free the stack and perform the exit hook.
+ */
+void
+dynamo_thread_stack_free_and_exit(byte *stack)
+{
+    if (stack != NULL) {
+        stack_free(stack, DYNAMORIO_STACK_SIZE);
+        /* ASSUMPTION: if stack is NULL here, the exit was done earlier
+         * (fixes case 6967)
+         */
+        EXITING_DR();
+    }
+}
+
+/* defined in x86/x86_code.c */
+extern void dynamo_start(dr_mcontext_t *mc);
+
+#ifdef DR_APP_EXPORTS
+/* API routine to initialize DR */
+DR_APP_API int
+dr_app_setup(void)
+{
+    dr_api_entry = true;
+    return dynamorio_app_init();
+}
+
+/* API routine to exit DR */
+DR_APP_API int
+dr_app_cleanup(void)
+{
+    SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+    dr_api_exit = true;
+    SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT); /* to keep properly nested */
+    return dynamorio_app_exit();
+}
+
+/* Called by dr_app_start in arch-specific assembly file */
+void
+dr_app_start_helper(dr_mcontext_t *mc)
+{
+    apicheck(dynamo_initialized, PRODUCT_NAME" not initialized");
+    if (!INTERNAL_OPTION(nullcalls)) {
+        dynamo_start(mc);
+        /* the interpreter takes over from here */
+    }
+}
+
+/* dummy routine that returns control to the app if it is currently
+ * under dynamo control
+ */
+DR_APP_API void
+dr_app_stop(void)
+{
+    /* the application regains control in here */
+}
+#endif
+
+/* For use by threads that start and stop whether dynamo controls them
+ */
+void
+dynamo_thread_under_dynamo(dcontext_t *dcontext)
+{
+    ASSERT(dcontext != NULL);
+    /* FIXME: mark under_dynamo_control?
+     * see comments in not routine below
+     */
+    os_thread_under_dynamo(dcontext);
+#ifdef SIDELINE
+    if (dynamo_options.sideline) {
+        /* wake up sideline thread -- ok to call if thread already awake */
+        sideline_start();
+    }
+#endif
+}
+
+/* For use by threads that start and stop whether dynamo controls them
+ */
+void
+dynamo_thread_not_under_dynamo(dcontext_t *dcontext)
+{
+#ifdef LINUX
+    /* FIXME : on windows this only matters when using the app_start app_stop 
+     * interface, and it screws up the thread lookup and synch routines
+     * undefining for windows for SAV alpha, revisit later with better model
+     */
+
+    if (dcontext == NULL)
+        return;
+    /* FIXME: mark !under_dynamo_control?
+     * conflict with use in callback, also use at exit to decide what threads to kill
+     * FIXME: when does thread_exit get called?  if thread says "dr_app_stop"
+     * and we mark it no longer under our control when do we exit it?
+     */
+    os_thread_not_under_dynamo(dcontext);
+# ifdef SIDELINE
+    /* FIXME: if # active threads is 0, then put sideline thread to sleep! */
+    if (dynamo_options.sideline) {
+        /* put sideline thread to sleep */
+        sideline_stop();
+    }
+# endif
+# ifdef DEBUG
+    os_flush(dcontext->logfile);
+# endif
+#endif /* LINUX */
+}
+
+/* Called by dynamorio_app_take_over in arch-specific assembly file */
+void
+dynamorio_app_take_over_helper(dr_mcontext_t *mc)
+{
+    static bool have_taken_over = false; /* ASSUMPTION: not an actual write */
+    SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+    APP_EXPORT_ASSERT(dynamo_initialized, PRODUCT_NAME" not initialized");
+#ifdef RETURN_AFTER_CALL
+    /* FIXME : this is set after dynamo_initialized, so a slight race with
+     * an injected thread turning on .C protection before the main thread
+     * sets this. */
+    dr_preinjected = true;      /* currently only relevant on Win32 */
+#endif
+
+    if (!INTERNAL_OPTION(nullcalls) && !have_taken_over) {
+        have_taken_over = true;
+        LOG(GLOBAL, LOG_TOP, 1, "dynamorio_app_take_over\n");
+        /* set this flag to indicate that we should run until the program dies: */
+        automatic_startup = true;
+
+        if (DYNAMO_OPTION(inject_primary))
+            take_over_primary_thread();
+
+        /* who knows when this was called -- no guarantee we control all threads --
+         * unless we were auto-injected (preinject library calls this routine)
+         */
+        control_all_threads = automatic_startup;
+        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+
+        /* For hotp_only and thin_client, the app should run native, except 
+         * for our hooks. 
+         * This is where apps hooked using appinit key are let go native.
+         * Even though control is going to native app code, we want 
+         * automatic_startup and control_all_threads set.
+         */
+        if (!RUNNING_WITHOUT_CODE_CACHE())
+            dynamo_start(mc);
+        /* the interpreter takes over from here */
+    } else
+        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+}
+
+#ifdef WINDOWS
+extern app_pc parent_early_inject_address; /* from os.c */ 
+
+/* in arch-specific assembly file */
+void dynamorio_app_take_over(void);
+
+DYNAMORIO_EXPORT void
+dynamorio_app_init_and_early_takeover(uint inject_location, void *restore_code)
+{
+    bool res;
+    ASSERT(!dynamo_initialized && !dynamo_exited);
+    /* This routine combines dynamorio_app_init() and dynamrio_app_takeover into
+     * a single routine that also handles any early injection cleanup needed. */
+    /* FIXME for apc early hook, need special handling in callback.c to replace
+     * the early hook and then touch up the hook code to handle any queued
+     * up threads (and be finally early remote thread safe).
+     */
+    ASSERT_NOT_IMPLEMENTED(inject_location != INJECT_LOCATION_KiUserApc);
+    /* currently only Ldr* hook points are known to work */
+    ASSERT_CURIOSITY(INJECT_LOCATION_IS_LDR(inject_location));
+    /* See notes in os.c DLLMain. When early injected we are unable to find
+     * the address of LdrpLoadDll so we use the parent's value which is passed
+     * to us at the start of restore_code. FIXME - if we start using multiple
+     * inject locations we'll probably have to ensure we always pass this.
+     */
+    if (INJECT_LOCATION_IS_LDR(inject_location)) {
+        parent_early_inject_address = *(app_pc *)restore_code;
+    }
+    dr_early_injected = true;
+    dr_early_injected_location = inject_location;
+    res = dynamorio_app_init();
+    ASSERT(res == SUCCESS);
+    ASSERT(dynamo_initialized && !dynamo_exited);
+    LOG(GLOBAL, LOG_TOP, 1, "dynamorio_app_init_and_early_take_over\n");
+    /* FIXME - restore code needs to be freed, but we have to return through it
+     * first... could instead duplicate its tail here if we wrap this
+     * routine in asm or eqv. pass the continuation state in as args. */
+    /* FIXME - NOTE app_takeover sets preinjected for rct (should prob. rename)
+     * which needs to be done whenever we takeover not at the bottom of the
+     * callstack.  INJECT_LOCATION_KiUserApc won't need to set this if we takeover
+     * in such a way as to handle the return back to our hook code without a
+     * violation. */
+    ASSERT(inject_location != INJECT_LOCATION_KiUserApc);
+    dynamorio_app_take_over();
+}
+#endif
+
+/***************************************************************************
+ * SELF-PROTECTION
+ */
+
+/* FIXME: even with -single_privileged_thread, we aren't fully protected,
+ * because there's a window between us resuming the other threads and
+ * returning to our caller where another thread could clobber our return
+ * address or something.
+ */
+static void
+dynamorio_protect(void)
+{
+    ASSERT(SELF_PROTECT_ON_CXT_SWITCH);
+    LOG(GLOBAL, LOG_DISPATCH, 4, "dynamorio_protect thread=%d\n", get_thread_id());
+    /* we don't protect local heap here, that's done lazily */
+
+    mutex_lock(&protect_info->lock);
+    ASSERT(protect_info->num_threads_unprot > 0);
+    /* FIXME: nice to also catch double enters but would need to track more info */
+    if (protect_info->num_threads_unprot <= 0) {
+        /* Defensive code to prevent crashes from double exits (the theory
+         * for case 7631/8030).  However, this precludes an extra exit+enter
+         * pair from working properly (though an extra enter+exit will continue
+         * to work), though such a pair would have crashed if another thread
+         * had entered in the interim anyway.
+         */
+        protect_info->num_threads_unprot = 0;
+        mutex_unlock(&protect_info->lock);
+        return;
+    }
+    protect_info->num_threads_unprot--;
+    if (protect_info->num_threads_unprot > 0) {
+        /* other threads still in DR, cannot protect global memory */
+        LOG(GLOBAL, LOG_DISPATCH, 4, "dynamorio_protect: not last thread => nop\n");
+        mutex_unlock(&protect_info->lock);
+        return;
+    }
+
+    SELF_PROTECT_GLOBAL(READONLY);
+
+    if (INTERNAL_OPTION(single_privileged_thread)) {
+        /* FIXME: want to resume threads and allow thread creation only
+         * _after_ protect data segment, but lock is in data segment!
+         */
+        if (protect_info->num_threads_suspended > 0) {
+            thread_record_t *tr;
+            int i, num = 0;
+            /* we do not need to grab the all_threads_lock because
+             * no threads can be added or removed so who cares if we
+             * access the data structure simultaneously with another
+             * reader of it
+             */
+            for (i = 0; i < HASHTABLE_SIZE(ALL_THREADS_HASH_BITS); i++) {
+                for (tr = all_threads[i]; tr; tr = tr->next) {
+                    if (tr->under_dynamo_control) {
+                        thread_resume(all_threads[i]);
+                        num++;
+                    }
+                }
+            }
+            ASSERT(num == protect_info->num_threads_suspended);
+            protect_info->num_threads_suspended = 0;
+        }
+        
+        /* thread init/exit can proceed now */
+        mutex_unlock(&thread_initexit_lock);
+    }
+
+    /* FIXME case 8073: temporary until we put in unprots in the
+     * right places.  if we were to leave this here we'd want to combine
+     * .fspdata and .cspdata for more efficient prot changes.
+     */
+    SELF_PROTECT_DATASEC(DATASEC_FREQ_PROT);
+    SELF_PROTECT_DATASEC(DATASEC_CXTSW_PROT);
+
+    mutex_unlock(&protect_info->lock);
+}
+
+static void 
+dynamorio_unprotect(void)
+{
+    ASSERT(SELF_PROTECT_ON_CXT_SWITCH);
+
+    mutex_lock(&protect_info->lock); /* lock in unprot heap, not data segment, so safe! */
+    protect_info->num_threads_unprot++;
+    if (protect_info->num_threads_unprot == 1) {
+        /* was protected, so we need to do the unprotection */
+        SELF_UNPROTECT_DATASEC(DATASEC_CXTSW_PROT);
+        /* FIXME case 8073: temporary until we put in unprots in the
+         * right places.  if we were to leave this here we'd want to combine
+         * .fspdata and .cspdata for more efficient prot changes.
+         */
+        SELF_UNPROTECT_DATASEC(DATASEC_FREQ_PROT);
+
+        if (INTERNAL_OPTION(single_privileged_thread)) {
+            /* FIXME: want to suspend all other threads _before_ unprotecting anything,
+             * but need to guarantee no new threads while we're suspending them,
+             * and can't do that without setting a lock => need data segment!
+             */
+            mutex_lock(&thread_initexit_lock);
+        
+            if (get_num_threads() > 1) {
+                thread_record_t *tr;
+                int i;
+                /* current multiple-thread solution: suspend all other threads! */
+                ASSERT(protect_info->num_threads_suspended == 0);
+                /* we do not need to grab the all_threads_lock because
+                 * no threads can be added or removed so who cares if we
+                 * access the data structure simultaneously with another
+                 * reader of it
+                 */
+                for (i = 0; i < HASHTABLE_SIZE(ALL_THREADS_HASH_BITS); i++) {
+                    for (tr = all_threads[i]; tr; tr = tr->next) {
+                        if (tr->under_dynamo_control) {
+                            thread_suspend(all_threads[i]);
+                            protect_info->num_threads_suspended++;
+                        }
+                    }
+                }
+            }
+            /* we don't unlock or resume threads until we re-enter cache */
+        }
+
+        SELF_PROTECT_GLOBAL(WRITABLE);
+    }
+    /* we don't re-protect local heap here, that's done at points where
+     * it was protected lazily
+     */
+    mutex_unlock(&protect_info->lock);
+    LOG(GLOBAL, LOG_DISPATCH, 4, "dynamorio_unprotect thread=%d\n", get_thread_id());
+}
+
+#ifdef DEBUG
+const char *
+get_data_section_name(app_pc pc)
+{
+    uint i;
+    for (i=0; i<DATASEC_NUM; i++) {
+        if (pc >= datasec_start[i] && pc < datasec_end[i])
+            return DATASEC_NAMES[i];
+    }
+    return NULL;
+}
+
+bool
+check_should_be_protected(uint sec)
+{
+    /* Blindly asserting that a data section is protected is racy as
+     * another thread could be in an unprot window.  We use some
+     * heuristics to try and identify bugs where a section is left
+     * unprot, but it's not easy.
+     */
+    if (/* case 8107: for INJECT_LOCATION_LdrpLoadImportModule we
+         * load a helper library and end up in dispatch() for
+         * syscall_while_native before DR is initialized.
+         */
+        !dynamo_initialized ||
+#ifdef WINDOWS
+        /* case 8113: detach currently unprots .data prior to its
+         * thread synch, so don't count anything after that
+         */
+        doing_detach ||
+#endif
+        !TEST(DATASEC_SELFPROT[sec], DYNAMO_OPTION(protect_mask)) ||
+        DATASEC_PROTECTED(sec))
+        return true;
+    STATS_INC(datasec_not_prot);
+    /* FIXME: even checking get_num_threads()==1 is still racy as a thread could
+     * exit, and it's not worth grabbing thread_initexit_lock here..
+     */
+    if (threads_ever_count == 1)
+        return false;
+    /* FIXME: no count of threads in DR or anything so can't conclude much
+     * Just return true and hope developer looks at datasec_not_prot stats.
+     * We do have an ASSERT_CURIOSITY on the stat in data_section_exit().
+     */
+    return true;
+}
+
+# ifdef WINDOWS
+/* Assumed to only be called about DR dll writable regions */
+bool
+data_sections_enclose_region(app_pc start, app_pc end)
+{
+    /* rather than solve the general enclose problem, we check for 32-bit,
+     * where .data|.fspdata|.cspdata|.nspdata form the only writable region,
+     * and 64-bit, where .pdata is between .data and .fspdata.
+     */
+#  ifdef X64
+    return ((datasec_start[DATASEC_RARELY_PROT] == start &&
+             datasec_end[DATASEC_RARELY_PROT] == end) ||
+            (datasec_start[DATASEC_FREQ_PROT] == start &&
+             datasec_end[DATASEC_FREQ_PROT] == datasec_start[DATASEC_CXTSW_PROT] &&
+             datasec_end[DATASEC_CXTSW_PROT] == datasec_start[DATASEC_NEVER_PROT] &&
+             datasec_end[DATASEC_NEVER_PROT] == end));
+#  else
+    return (datasec_start[DATASEC_RARELY_PROT] == start &&
+            datasec_end[DATASEC_RARELY_PROT] == datasec_start[DATASEC_FREQ_PROT] &&
+            datasec_end[DATASEC_FREQ_PROT] == datasec_start[DATASEC_CXTSW_PROT] &&
+            datasec_end[DATASEC_CXTSW_PROT] == datasec_start[DATASEC_NEVER_PROT] &&
+            datasec_end[DATASEC_NEVER_PROT] == end);
+#  endif
+}
+# endif /* WINDOWS */
+#endif /* DEBUG */
+
+static void
+get_data_section_bounds(uint sec)
+{
+    /* FIXME: on linux we should include .got and .dynamic in one of our
+     * sections, requiring specifying the order of sections (case 3789)!
+     * Should use an ld script to ensure that .nspdata is last, or find a unique
+     * attribute to force separation (perhaps mark as rwx, then
+     * remove the x at init time?)  ld 2.15 puts it at the end, but
+     * ld 2.13 puts .got and .dynamic after it!  For now we simply
+     * don't protect subsequent guys.
+     * On win32 there are no other rw sections, fortunately.
+     */
+    ASSERT(sec >=0 && sec < DATASEC_NUM);
+    /* for DEBUG we use for data_sections_enclose_region() */
+    ASSERT(IF_WINDOWS(IF_DEBUG(true ||))
+           TEST(DATASEC_SELFPROT[sec], dynamo_options.protect_mask));
+    mutex_lock(&datasec_lock[sec]);
+    ASSERT(datasec_start[sec] == NULL);
+    get_named_section_bounds(get_dynamorio_dll_start(), DATASEC_NAMES[sec],
+                             &datasec_start[sec], &datasec_end[sec]);
+    mutex_unlock(&datasec_lock[sec]);
+    ASSERT(ALIGNED(datasec_start[sec], PAGE_SIZE));
+    ASSERT(ALIGNED(datasec_end[sec], PAGE_SIZE));
+    ASSERT(datasec_start[sec] < datasec_end[sec]);
+#ifdef WINDOWS
+    if (IF_DEBUG(true ||) TEST(DATASEC_SELFPROT[sec], dynamo_options.protect_mask))
+        merge_writecopy_pages(datasec_start[sec], datasec_end[sec]);
+#endif
+}
+
+#ifdef LINUX
+/* We get into problems if we keep a .section open across string literals, etc.
+ * (such as when wrapping a function to get its local-scope statics in that section),
+ * but the VAR_IN_SECTION does the real work for us, just so long as we have one
+ * .section decl somewhere.
+ */
+DECLARE_DATA_SECTION(RARELY_PROTECTED_SECTION, "w")
+DECLARE_DATA_SECTION(FREQ_PROTECTED_SECTION, "w")
+DECLARE_DATA_SECTION(NEVER_PROTECTED_SECTION, "w")
+END_DATA_SECTION_DECLARATIONS()
+#endif
+
+static void
+data_section_init(void)
+{
+    uint i;
+    for (i=0; i<DATASEC_NUM; i++) {
+        if (datasec_start[i] != NULL) {
+            /* We were called early due to an early syslog.
+             * We still retain our slightly later normal init position so we can
+             * log, etc. in normal runs.
+             */
+            return;
+        }
+        ASSIGN_INIT_LOCK_FREE(datasec_lock[i], datasec_selfprot_lock);
+        /* for DEBUG we use for data_sections_enclose_region() */
+        if (IF_WINDOWS(IF_DEBUG(true ||))
+            TEST(DATASEC_SELFPROT[i], dynamo_options.protect_mask)) {
+            get_data_section_bounds(i);
+        }
+    }
+    DODEBUG({
+        /* ensure no overlaps */
+        uint j;
+        for (i=0; i<DATASEC_NUM; i++) {
+            for (j=i+1; j<DATASEC_NUM; j++) {
+                ASSERT(datasec_start[i] >= datasec_end[j] ||
+                       datasec_start[j] >= datasec_end[i]);
+            }
+        }
+    });
+}
+
+static void
+data_section_exit(void)
+{
+    uint i;
+    DOSTATS({
+        /* There can't have been that many races.
+         * A failure to re-protect should result in a ton of dispatch
+         * entrances w/ .data unprot, so should show up here.
+         */
+        ASSERT_CURIOSITY(GLOBAL_STAT(datasec_not_prot) < 20);
+    });
+    for (i=0; i<DATASEC_NUM; i++)
+        DELETE_LOCK(datasec_lock[i]);
+}
+
+#define DATASEC_WRITABLE_MOD(which, op) \
+    ((which) == DATASEC_RARELY_PROT ? (datasec_writable_rareprot op) : \
+     ((which) == DATASEC_CXTSW_PROT ? (datasec_writable_cxtswprot op) : \
+      ((which) == DATASEC_FREQ_PROT ? (datasec_writable_freqprot op) : \
+       (ASSERT_NOT_REACHED(), datasec_writable_neverprot))))
+
+/* WARNING: any DO_ONCE will call this routine, so don't call anything here
+ * that has a DO_ONCE, to avoid deadlock!
+ */
+void
+protect_data_section(uint sec, bool writable)
+{
+    ASSERT(sec >=0 && sec < DATASEC_NUM);
+    ASSERT(TEST(DATASEC_SELFPROT[sec], dynamo_options.protect_mask));
+    /* We can be called very early before data_section_init() so init here
+     * (data_section_init() has no dependences).
+     */
+    if (datasec_start[sec] == NULL) {
+        /* should only happen early in init */
+        ASSERT(!dynamo_initialized);
+        data_section_init();
+    }
+    mutex_lock(&datasec_lock[sec]);
+    ASSERT(datasec_start[sec] != NULL);
+    /* if using libc, we cannot print while data segment is read-only! 
+     * thus, if making it writable, do that first, otherwise do it last.
+     * w/ ntdll this is not a problem.
+     */
+    /* Remember that multiple threads can be doing (unprotect,protect) pairs of
+     * calls simultaneously.  The datasec_lock makes each individual call atomic,
+     * and if all calls are properly nested, our use of counters should result in
+     * the proper protection only after the final protect call and not in the
+     * middle of some other thread's writes to the data section.
+     */
+    if (writable) {
+        /* On-context-switch protection has a separate mechanism for
+         * only protecting when the final thread leaves DR
+         */
+        ASSERT_CURIOSITY(DATASEC_WRITABLE(sec) <= 2); /* shouldn't nest too deep! */
+        if (DATASEC_WRITABLE(sec) == 0) {
+            make_writable(datasec_start[sec], datasec_end[sec] - datasec_start[sec]);
+            STATS_INC(datasec_prot_changes);
+        } else
+            STATS_INC(datasec_prot_wasted_calls);
+        DATASEC_WRITABLE_MOD(sec, ++);
+    }
+    LOG(TEST(DATASEC_SELFPROT[sec], SELFPROT_ON_CXT_SWITCH) ? THREAD_GET : GLOBAL,
+        LOG_VMAREAS, TEST(DATASEC_SELFPROT[sec], SELFPROT_ON_CXT_SWITCH) ? 3U : 2U,
+        "protect_data_section: %s %s %s %d\n",
+        DATASEC_WRITABLE(sec) == 1 ? "changing" : "nop",
+        DATASEC_NAMES[sec], writable ? "rw" : "r", DATASEC_WRITABLE(sec));
+    if (!writable) {
+        ASSERT(DATASEC_WRITABLE(sec) > 0);
+        DATASEC_WRITABLE_MOD(sec, --);
+        if (DATASEC_WRITABLE(sec) == 0) {
+            make_unwritable(datasec_start[sec], datasec_end[sec] - datasec_start[sec]);
+            STATS_INC(datasec_prot_changes);
+        } else
+            STATS_INC(datasec_prot_wasted_calls);
+    }
+    mutex_unlock(&datasec_lock[sec]);
+}
+
+/* enter/exit DR hooks */
+void
+entering_dynamorio(void)
+{
+    if (SELF_PROTECT_ON_CXT_SWITCH)
+        dynamorio_unprotect();
+    ASSERT(HOOK_ENABLED);
+    LOG(GLOBAL, LOG_DISPATCH, 3, "entering_dynamorio thread=%d\n", get_thread_id());
+    STATS_INC(num_entering_DR);
+    if (INTERNAL_OPTION(single_thread_in_DR)) {
+        acquire_recursive_lock(&thread_in_DR_exclusion);
+        LOG(GLOBAL, LOG_DISPATCH, 3, "entering_dynamorio thread=%d count=%d\n",
+            get_thread_id(), thread_in_DR_exclusion.count);
+    }
+}
+
+void 
+exiting_dynamorio(void)
+{
+    ASSERT(HOOK_ENABLED);
+    LOG(GLOBAL, LOG_DISPATCH, 3, "exiting_dynamorio thread=%d\n", get_thread_id());
+    STATS_INC(num_exiting_DR);
+    if (INTERNAL_OPTION(single_thread_in_DR)) {
+        /* thread init/exit can proceed now */
+        LOG(GLOBAL, LOG_DISPATCH, 3, "exiting_dynamorio thread=%d count=%d\n",
+            get_thread_id(), thread_in_DR_exclusion.count - 1);
+        release_recursive_lock(&thread_in_DR_exclusion);
+    }
+    if (SELF_PROTECT_ON_CXT_SWITCH && !dynamo_exited)
+        dynamorio_protect();
+}
+
+/* Note this includes any stack guard pages */
+bool
+is_on_initstack(byte *esp)
+{
+    return (esp <= initstack && esp > initstack - DYNAMORIO_STACK_SIZE);
+}
+
+/* Note this includes any stack guard pages */
+bool
+is_on_dstack(dcontext_t *dcontext, byte *esp)
+{
+    return (esp <= dcontext->dstack && 
+            esp > dcontext->dstack - DYNAMORIO_STACK_SIZE);
+}
+
+bool
+is_currently_on_dstack(dcontext_t *dcontext)
+{
+    byte *cur_esp;
+    GET_STACK_PTR(cur_esp);
+    return is_on_dstack(dcontext, cur_esp);
+}
diff --git a/core/emit.c b/core/emit.c
new file mode 100644
index 000000000..b4398137b
--- /dev/null
+++ b/core/emit.c
@@ -0,0 +1,1026 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * emit.c - fragment code generation routines
+ */
+
+#include "globals.h"
+#include "link.h"
+#include "fragment.h"
+#include "fcache.h"
+#include "proc.h"
+#include "instrlist.h"
+#include "emit.h"
+#include "instrlist.h"
+#include "instr.h"
+#include "monitor.h"
+#include <string.h> /* memcpy */
+
+#ifdef DEBUG
+# include "decode_fast.h" /* for decode_next_pc for stress_recreate_pc */
+#endif
+
+#define STATS_FCACHE_ADD(flags, stat, val) DOSTATS({   \
+    if (TEST(FRAG_SHARED, (flags))) {                  \
+        if (IN_TRACE_CACHE(flags))                     \
+            STATS_ADD(fcache_shared_trace_##stat, val);\
+        else                                           \
+            STATS_ADD(fcache_shared_bb_##stat, val);   \
+    }                                                  \
+    else if (IN_TRACE_CACHE(flags))                    \
+        STATS_ADD(fcache_trace_##stat, val);           \
+    else                                               \
+        STATS_ADD(fcache_bb_##stat, val);              \
+})
+
+#ifdef INTERNAL
+/* case 4344 - verify we can recreate app pc in fragment, returns the pc of
+ * the last instruction in the body of f */
+static cache_pc
+get_last_fragment_body_instr_pc(dcontext_t *dcontext, fragment_t *f)
+{
+    cache_pc body_last_inst_pc;
+    linkstub_t *l;
+    
+    /* Assumption : the last exit stub exit cti is the last instruction in the
+     * body.  PR 215217 enforces this for CLIENT_INTERFACE as well. */
+    l = FRAGMENT_EXIT_STUBS(f);
+    /* never called on future fragments, so a stub should exist */
+    while (!LINKSTUB_FINAL(l))
+        l = LINKSTUB_NEXT_EXIT(l);
+    
+    body_last_inst_pc = EXIT_CTI_PC(f, l);
+    return body_last_inst_pc;
+}
+
+void
+stress_test_recreate(dcontext_t *dcontext, fragment_t *f, 
+                     instrlist_t *ilist)
+{
+    cache_pc body_end_pc = get_last_fragment_body_instr_pc(dcontext, f);
+    app_pc recreated_pc;
+
+    LOG(THREAD, LOG_MONITOR, 2, "Testing recreating Fragment %d for tag "PFX" at "PFX"\n",
+        f->id, f->tag, f->start_pc);
+
+    DOLOG(3, LOG_INTERP, {
+        /* visualize translation info if it were to be recorded for every
+         * fragment, not just deleted ones -- for debugging only
+         */
+        translation_info_t *info = record_translation_info(dcontext, f, NULL);
+        translation_info_print(info, f->start_pc, THREAD);
+        translation_info_free(dcontext, info);
+        /* handy reference of app code and fragment -- only 1st part of trace though */
+        DOLOG(3, LOG_INTERP, { disassemble_app_bb(dcontext, f->tag, THREAD); });
+        DOLOG(3, LOG_INTERP, { disassemble_fragment(dcontext, f, false); });
+    });
+
+    DOLOG(2, LOG_MONITOR, {
+        /* Translate them all.
+         * Useful when verifying manually, o/w we just ensure no asserts or crashes.
+         */
+        cache_pc cpc = f->start_pc;
+        while (cpc <= body_end_pc) {
+            recreated_pc = recreate_app_pc(dcontext, cpc, NULL/*for full test*/);
+            LOG(THREAD, LOG_MONITOR, 2, "\ttranslated cache "PFX" => app "PFX"\n",
+                cpc, recreated_pc);
+            cpc = decode_next_pc(dcontext, cpc);
+        }
+    });
+ 
+    recreated_pc = recreate_app_pc(dcontext, body_end_pc, NULL/*for full test*/);
+    /* FIXME: we should figure out how to test each instruction, while knowing the app state */
+    LOG(THREAD, LOG_MONITOR, 2, "Testing recreating Fragment #%d recreated_pc="PFX"\n",
+        GLOBAL_STAT(num_fragments), recreated_pc);
+
+    ASSERT(recreate_app_pc != NULL);
+
+    if (INTERNAL_OPTION(stress_recreate_state) && ilist != NULL)
+        stress_test_recreate_state(dcontext, f, ilist);
+}
+#endif /* INTERNAL */
+
+/* here instead of link.c b/c link.c doesn't deal w/ Instrs */
+bool
+final_exit_shares_prev_stub(dcontext_t *dcontext, instrlist_t *ilist, uint frag_flags)
+{
+    /* if a cbr is final exit pair, should they share a stub? */
+    if (INTERNAL_OPTION(cbr_single_stub) && !TEST(FRAG_COARSE_GRAIN, frag_flags)) {
+        /* don't need to expand since is_exit_cti will rule out level 0 */
+        instr_t *inst = instrlist_last(ilist);
+        /* FIXME: we could support code after the last cti (this is ubr so
+         * would be out-of-line code) or between cbr and ubr but for
+         * simplicity of identifying exits for traces we don't
+         */
+        if (instr_is_exit_cti(inst) && instr_is_ubr(inst)) {
+            /* don't need to expand since is_exit_cti will rule out level 0 */
+            instr_t *prev_cti = instr_get_prev(inst);
+            if (prev_cti != NULL && instr_is_exit_cti(prev_cti)
+                /* cti_loop is fine since cti points to loop instr, enabling
+                 * our disambiguation to know which state to look at */
+                && instr_is_cbr(prev_cti)
+#ifdef CUSTOM_EXIT_STUBS
+                /* no custom code */
+                && instr_exit_stub_code(prev_cti) == NULL
+                && instr_exit_stub_code(inst) == NULL
+#endif
+#ifdef PROFILE_LINKCOUNT
+                /* no linkcount code, which is only present in traces */
+                && (!TEST(FRAG_IS_TRACE, frag_flags) || !INTERNAL_OPTION(profile_counts))
+#endif
+                /* no separate freeing */
+                && ((TEST(FRAG_SHARED, frag_flags) &&
+                     !DYNAMO_OPTION(unsafe_free_shared_stubs)) ||
+                    (!TEST(FRAG_SHARED, frag_flags) &&
+                     !DYNAMO_OPTION(free_private_stubs)))) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
+/* Walks ilist and f's linkstubs, setting each linkstub_t's fields appropriately
+ * for the corresponding exit cti in ilist.
+ * If emit is true, also encodes each instr in ilist to f's cache slot,
+ * increments stats for new fragments, and returns the final pc after all encodings.
+ */
+cache_pc
+set_linkstub_fields(dcontext_t *dcontext, fragment_t *f, instrlist_t *ilist, 
+                    uint num_direct_stubs, uint num_indirect_stubs, bool emit)
+{
+    uint i;
+    bool frag_offs_at_end;
+    linkstub_t *l;
+    cache_pc pc;
+    instr_t *inst;
+    app_pc target;
+    DEBUG_DECLARE(instr_t *prev_cti = NULL;)
+
+    pc = FCACHE_ENTRY_PC(f);
+    l = FRAGMENT_EXIT_STUBS(f);
+    i = 0;
+    frag_offs_at_end = linkstub_frag_offs_at_end(f->flags, num_direct_stubs,
+                                                 num_indirect_stubs);
+    for (inst = instrlist_first(ilist); inst; inst = instr_get_next(inst)) {
+#ifdef NATIVE_RETURN
+        if (instr_opcode_valid(inst) && instr_is_return(inst))
+            prev_ret = pc;
+#endif
+        if (instr_is_exit_cti(inst)) {
+            /* l is currently zeroed out but otherwise uninitialized 
+             * stub starts out as unlinked and never-been-linked 
+             */
+            ASSERT(l->flags == 0);
+            i++;
+            if (i == num_direct_stubs + num_indirect_stubs) {
+                /* set final flag */
+                l->flags |= LINK_END_OF_LIST;
+            }
+            if (frag_offs_at_end)
+                l->flags |= LINK_FRAG_OFFS_AT_END;
+
+            DODEBUG({
+                if (emit && is_exit_cti_patchable(dcontext, inst, f->flags)) {
+                    uint off = patchable_exit_cti_align_offs(dcontext, inst, pc);
+                    if (off > 0) {
+                        ASSERT(!PAD_FRAGMENT_JMPS(f->flags));
+                        STATS_PAD_JMPS_ADD(f->flags, unaligned_exits, 1);
+                        STATS_PAD_JMPS_ADD(f->flags, unaligned_exit_bytes, off);
+                    }
+                }
+            });
+            /* An alternative way of testing this is to match with 
+             * is_return_lookup_routine() whenever we get that */
+            /* FIXME: doing the above is much easier now and it is more reliable
+             * than expecting the branch type flags to propagate through 
+             */
+            l->flags |= instr_exit_branch_type(inst);
+
+            target = instr_get_branch_target_pc(inst);
+
+            if (is_indirect_branch_lookup_routine(dcontext, (cache_pc)target)) {
+                ASSERT(IF_WINDOWS_ELSE_0(is_shared_syscall_routine(dcontext, target)) ||
+                       is_ibl_routine_type(dcontext, (cache_pc)target,
+                                           extract_branchtype((ushort)instr_exit_branch_type(inst))));
+                /* this is a mangled form of an original indirect
+                 * branch or is a mangled form of an indirect branch
+                 * to a real native pc out of the fragment
+                 */
+                l->flags |= LINK_INDIRECT;
+                ASSERT(!LINKSTUB_DIRECT(l->flags));
+                ASSERT(!LINKSTUB_NORMAL_DIRECT(l->flags));
+                ASSERT(!LINKSTUB_CBR_FALLTHROUGH(l->flags));
+                ASSERT(LINKSTUB_INDIRECT(l->flags));
+#ifdef NATIVE_RETURN
+                /* to unlink we have to nop-out the ret, but it's hard to find,
+                 * so we store its pc
+                 */
+                if (prev_ret != NULL) {
+                    indirect_linkstub_t *il = (indirect_linkstub_t *) l;
+                    ASSERT_NOT_TESTED();
+                    il->ret_pc = prev_ret;
+                    prev_ret = NULL;
+                }
+#endif
+            }
+            else {
+                DOSTATS({
+                    if (emit) {
+                        if (PTR_UINT_ABS(target - f->tag) > SHRT_MAX) {
+                            if (num_indirect_stubs == 0 && num_direct_stubs == 2 &&
+                                i == 2)
+                                STATS_INC(num_bb_fallthru_far);
+                            STATS_INC(num_bb_exit_tgt_far);
+                        } else {
+                            if (num_indirect_stubs == 0 && num_direct_stubs == 2 &&
+                                i == 2)
+                                STATS_INC(num_bb_fallthru_near);
+                            STATS_INC(num_bb_exit_tgt_near);
+                        }
+                    }
+                });
+
+                if (LINKSTUB_FINAL(l) &&
+                    use_cbr_fallthrough_short(f->flags, num_direct_stubs,
+                                              num_indirect_stubs)) {
+                    /* this is how we mark a cbr fallthrough, w/ both
+                     * LINK_DIRECT and LINK_INDIRECT
+                     */
+                    l->flags |= LINK_DIRECT | LINK_INDIRECT;
+                    /* ensure our macros are in synch */
+                    ASSERT(LINKSTUB_DIRECT(l->flags));
+                    ASSERT(!LINKSTUB_NORMAL_DIRECT(l->flags));
+                    ASSERT(LINKSTUB_CBR_FALLTHROUGH(l->flags));
+                    ASSERT(!LINKSTUB_INDIRECT(l->flags));
+                    DOSTATS({
+                        if (emit)
+                            STATS_INC(num_bb_cbr_fallthru_shrink);
+                    });
+                    ASSERT(prev_cti != NULL && instr_is_cbr(prev_cti));
+                    /* should always qualify for single stub */
+                    ASSERT(!INTERNAL_OPTION(cbr_single_stub) ||
+                           /* FIXME: this duplicates calc of final_cbr_single_stub
+                            * bool cached in emit_fragment_common()
+                            */
+                           (inst == instrlist_last(ilist) && 
+                            final_exit_shares_prev_stub(dcontext, ilist, f->flags)));
+                } else {
+                    direct_linkstub_t *dl = (direct_linkstub_t *) l;
+                    l->flags |= LINK_DIRECT;
+                    /* ensure our macros are in synch */
+                    ASSERT(LINKSTUB_DIRECT(l->flags));
+                    ASSERT(LINKSTUB_NORMAL_DIRECT(l->flags));
+                    ASSERT(!LINKSTUB_CBR_FALLTHROUGH(l->flags));
+                    ASSERT(!LINKSTUB_INDIRECT(l->flags));
+                    dl->target_tag = target;
+                }
+#ifdef NATIVE_RETURN
+                ASSERT(prev_ret == NULL);
+#endif
+            }
+
+            if (should_separate_stub(dcontext, target, f->flags)
+#ifdef CUSTOM_EXIT_STUBS
+                /* we can't separate custom stubs */
+                && instr_exit_stub_code(inst) == NULL
+#endif
+                )
+                l->flags |= LINK_SEPARATE_STUB;
+
+            /* FIXME: we don't yet support !emit ctis: need to avoid patching
+             * the cti when emit the exit stub */
+            ASSERT_NOT_IMPLEMENTED(!emit || instr_ok_to_emit(inst));
+
+            if (LINKSTUB_CBR_FALLTHROUGH(l->flags)) {
+                /* target is indicated via cti_offset */
+                ASSERT_TRUNCATE(l->cti_offset, short, target - f->tag);
+                l->cti_offset = (ushort) /* really a short */ (target - f->tag);
+            } else {
+                ASSERT_TRUNCATE(l->cti_offset, ushort, pc - f->start_pc);
+                l->cti_offset = (ushort) (pc - f->start_pc);
+            }
+
+            DODEBUG({
+                /* ensure LINK_ flags were transferred via instr_exit_branch_type */
+                if (instr_branch_selfmod_exit(inst)) {
+                    ASSERT(!LINKSTUB_INDIRECT(l->flags) &&
+                           TEST(LINK_SELFMOD_EXIT, l->flags));
+                }
+            });
+#ifdef UNSUPPORTED_API
+            DODEBUG({
+                if (instr_branch_targets_prefix(inst))
+                    ASSERT(TEST(LINK_TARGET_PREFIX, l->flags));
+            });
+#endif
+
+            if (!EXIT_HAS_STUB(l->flags, f->flags)) {
+                /* exit cti points straight at ibl routine */
+                instr_set_branch_target_pc(inst, get_unlinked_entry(dcontext, target));
+            } else {
+                /* HACK: set the branch target pc in inst to be its own pc- this ensures
+                   that instr_encode will not fail due to address span problems- the
+                   correct target (to the exit stub) will get patched in when the
+                   exit stub corresponding to this exit branch is emitted later */
+                instr_set_branch_target_pc(inst, pc);
+            }
+            /* PR 267260/PR 214962: keep this exit cti marked */
+            instr_set_our_mangling(inst, true);
+
+            LOG(THREAD, LOG_EMIT, dcontext == GLOBAL_DCONTEXT ||
+                dcontext->in_opnd_disassemble ? 5U : 3U,
+                "exit_branch_type=0x%x target="PFX" l->flags=0x%x\n",
+                instr_exit_branch_type(inst), target, l->flags);
+
+            DODEBUG({
+                if (TEST(FRAG_COARSE_GRAIN, f->flags)) {
+                    ASSERT(!frag_offs_at_end);
+                    /* FIXME: indirect stubs should be separated
+                     * eventually, but right now no good place to put them
+                     * so keeping inline
+                     */
+                    ASSERT(LINKSTUB_INDIRECT(l->flags) ||
+                           TEST(LINK_SEPARATE_STUB, l->flags));
+                }
+            });
+
+            /* traversal depends on flags being set */
+            ASSERT(l->flags != 0);
+            ASSERT(i <= num_direct_stubs + num_indirect_stubs);
+            l = LINKSTUB_NEXT_EXIT(l);
+            DODEBUG({ prev_cti = inst; });
+        } /* exit cti */
+        if (instr_ok_to_emit(inst)) {
+            if (emit) {
+                pc = instr_encode(dcontext, inst, pc);
+                ASSERT(pc != NULL);
+            } else {
+                pc += instr_length(dcontext, inst);
+            }
+        }
+    }
+    return pc;
+}
+
+/* Emits code for ilist into the fcache, returns created fragment.
+ * It is not added to the fragment table, nor is it linked!
+ */
+static fragment_t *
+emit_fragment_common(dcontext_t *dcontext, app_pc tag,
+                     instrlist_t *ilist, uint flags, void *vmlist,
+                     bool link_fragment, bool add_to_htable,
+                     fragment_t *replace_fragment)
+{
+    fragment_t  *f;
+    instr_t     *inst;
+    cache_pc  pc = 0;
+    app_pc    target;
+    linkstub_t  *l;
+    uint      len;
+    uint      offset = 0;
+    uint      copy_sz = 0;
+    uint      extra_jmp_padding_body = 0;
+    uint      extra_jmp_padding_stubs = 0;
+    uint      last_pad_offset = 0;
+    uint      num_direct_stubs = 0;
+    uint      num_indirect_stubs = 0;
+    uint      stub_size_total = 0; /* those in fcache w/ fragment */
+#ifdef CUSTOM_EXIT_STUBS
+    bool      custom_stubs_present = false;
+#endif
+#ifdef NATIVE_RETURN
+    cache_pc  prev_ret = 0;
+#endif
+    bool      final_cbr_single_stub = false;
+    byte      *prev_stub_pc = NULL;
+    uint      stub_size = 0;
+    bool      no_stub = false;
+    IF_X64(bool x86_mode;)
+
+    KSTART(emit);
+    /* we do entire cache b/c links may touch many units
+     * FIXME: change to lazier version triggered by segfaults or something?
+     */
+    SELF_PROTECT_CACHE(dcontext, NULL, WRITABLE);
+
+    /* ensure some higher-level lock is held if f is shared */
+    ASSERT(!TEST(FRAG_SHARED, flags) || INTERNAL_OPTION(single_thread_in_DR) ||
+           OWN_MUTEX(&bb_building_lock) || OWN_MUTEX(&trace_building_lock));
+
+    /* 1st walk through instr list:
+     * -- determine body size and number of exit stubs required;
+     * -- if not padding jmps sets offsets as well
+     */
+#ifdef X64
+    ASSERT(instrlist_first(ilist) != NULL);
+    x86_mode = instr_get_x86_mode(instrlist_first(ilist));
+    if (x86_mode)
+        flags |= FRAG_32_BIT;
+#endif
+    for (inst = instrlist_first(ilist); inst; inst = instr_get_next(inst)) {
+        /* Since decode_fragment needs to be able to decode from the code
+         * cache, we require that each fragment has a single mode
+         * (xref PR 278329)
+         */
+        IF_X64(CLIENT_ASSERT(instr_get_x86_mode(inst) == x86_mode,
+                             "single fragment cannot mix x86 and x64 modes"));
+        if (!PAD_FRAGMENT_JMPS(flags)) {
+            /* we're going to skip the 2nd pass, save this instr's offset in 
+             * the note field (used by instr_encode) */
+            instr_set_note(inst, (void *)(ptr_uint_t)offset);
+        }
+        if (instr_ok_to_emit(inst))
+            offset += instr_length(dcontext, inst);
+        ASSERT_NOT_IMPLEMENTED(!TEST(INSTR_HOT_PATCHABLE, inst->flags));
+        if (instr_is_exit_cti(inst)) {
+            target = instr_get_branch_target_pc(inst);
+            len = exit_stub_size(dcontext, (cache_pc)target, flags);
+            if (PAD_FRAGMENT_JMPS(flags) && instr_ok_to_emit(inst)) {
+                /* Most exits have only a single patchable jmp (is difficult 
+                 * to handle all the races for more then one). Exceptions are 
+                 * usually where you have to patch the jmp in the body as well 
+                 * as in the stub and include inlined_indirect (without 
+                 * -atomic_inlined_linking), TRACE_HEAD_CACHE_INCR, or a 
+                 * custom exit stub with PROFILE_LINKCOUNT.  All of these
+                 * have issues with atomically linking/unlinking. Inlined 
+                 * indirect has special support for unlinking (but not linking
+                 * hence can't use inlined_ibl on shared frags without 
+                 * -atomic_inlined_linking, but is otherwise ok).  I suspect 
+                 * the other two exceptions are ok as well in practice (just
+                 * racy as to whether the trace head count gets incremented or
+                 * the custom code is executed or we exit cache unnecessarily). 
+                 */
+                if (is_exit_cti_patchable(dcontext, inst, flags)) {
+                    if (last_pad_offset == 0 || 
+                        !WITHIN_PAD_REGION(last_pad_offset, offset)) {
+                        last_pad_offset = offset - CTI_PATCH_OFFSET;
+                        extra_jmp_padding_body += MAX_PAD_SIZE;
+                    }
+                }
+                if (is_exit_cti_stub_patchable(dcontext, inst, flags)) {
+                    extra_jmp_padding_stubs += MAX_PAD_SIZE;
+                }
+            }
+            if (is_indirect_branch_lookup_routine(dcontext, target)) {
+                num_indirect_stubs++;
+                STATS_INC(num_indirect_exit_stubs);
+                LOG(THREAD, LOG_EMIT, 3, "emit_fragment: %s use ibl <"PFX">\n", 
+                    TEST(FRAG_IS_TRACE, flags) ? "trace" : "bb", target);
+                stub_size_total += len;
+                STATS_FCACHE_ADD(flags, indirect_stubs, len);
+            } else {
+                num_direct_stubs++;
+                STATS_INC(num_direct_exit_stubs);
+
+                /* if a cbr is final exit pair, should they share a stub? */
+                if (INTERNAL_OPTION(cbr_single_stub) &&
+                    inst == instrlist_last(ilist) && 
+                    final_exit_shares_prev_stub(dcontext, ilist, flags)) {
+                    final_cbr_single_stub = true;
+                    STATS_INC(num_cbr_single_stub);
+                } else if (!should_separate_stub(dcontext, target, flags)) {
+                    stub_size_total += len;
+                    STATS_FCACHE_ADD(flags, direct_stubs, len);
+                } else /* ensure have cti to jmp to separate stub! */
+                    ASSERT(instr_ok_to_emit(inst));
+            }
+#ifdef CUSTOM_EXIT_STUBS
+            if (!custom_stubs_present && instr_exit_stub_code(inst) != NULL)
+                custom_stubs_present = true;
+#endif
+        }
+    }
+
+    DOSTATS({
+        if (!TEST(FRAG_IS_TRACE, flags)) {
+            if (num_indirect_stubs > 0) {
+                if (num_indirect_stubs == 1 && num_direct_stubs == 0)
+                    STATS_INC(num_bb_one_indirect_exit);
+                else /* funny bb w/ mixture of ind and dir exits */
+                    STATS_INC(num_bb_indirect_extra_exits);
+            } else {
+                if (num_direct_stubs == 1)
+                    STATS_INC(num_bb_one_direct_exit);
+                else if (num_direct_stubs == 2)
+                    STATS_INC(num_bb_two_direct_exits);
+                else
+                    STATS_INC(num_bb_many_direct_exits);
+            }
+            if (TEST(FRAG_HAS_DIRECT_CTI, flags))
+                STATS_INC(num_bb_has_elided);
+            if (linkstub_frag_offs_at_end(flags, num_direct_stubs,
+                                          num_indirect_stubs))
+                STATS_INC(num_bb_fragment_offset);
+        }
+    });
+#ifndef CLIENT_INTERFACE
+    /* (can't have ifdef inside DOSTATS so we separate it from above stats)
+     * in a product build we only expect certain kinds of bbs
+     */
+    ASSERT_CURIOSITY(TEST(FRAG_IS_TRACE, flags) ||
+                     (num_indirect_stubs == 1 && num_direct_stubs == 0) ||
+                     (num_indirect_stubs == 0 && num_direct_stubs <= 2) ||
+                     (num_indirect_stubs <= 1 && num_direct_stubs >= 1 &&
+                      TEST(FRAG_SELFMOD_SANDBOXED, flags)));
+#endif
+            
+#ifdef CUSTOM_EXIT_STUBS
+    if (custom_stubs_present) {
+        LOG(THREAD, LOG_EMIT, 3, "emit_fragment: custom stubs present\n");
+        /* separate walk just for custom exit stubs -- need to get offsets correct
+         */
+        for (inst = instrlist_first(ilist); inst; inst = instr_get_next(inst)) {
+            if (instr_is_exit_cti(inst)) {
+                /* custom exit stub code */
+                instrlist_t *custom = instr_exit_stub_code(inst);
+                if (custom != NULL) {
+                    instr_t *in;
+                    for (in = instrlist_first(custom); in; in = instr_get_next(in)) {
+                        ASSERT_NOT_IMPLEMENTED(!TEST(INSTR_HOT_PATCHABLE, 
+                                                     inst->flags));
+                        if (!PAD_FRAGMENT_JMPS(flags)) {
+                            /* we're going to skip the 2nd pass, save this 
+                             * instr's offset in the note field for use by
+                             * instr_encode */
+                            instr_set_note(in, offset);
+                        }
+                        offset += instr_length(dcontext, in);
+                    }
+                }
+                target = instr_get_branch_target_pc(inst);
+                offset += exit_stub_size(dcontext, (cache_pc)target, flags);
+            }
+        }
+        offset -= stub_size_total;
+    }
+#endif
+
+    STATS_PAD_JMPS_ADD(flags, body_bytes, extra_jmp_padding_body);
+    STATS_PAD_JMPS_ADD(flags, stub_bytes, extra_jmp_padding_stubs);
+
+    STATS_FCACHE_ADD(flags, bodies, offset);
+    STATS_FCACHE_ADD(flags, prefixes, fragment_prefix_size(flags));
+
+    if (TEST(FRAG_SELFMOD_SANDBOXED, flags)) {
+        /* We need a copy of the original app code at bottom of
+         * fragment.  We count it as part of the fragment body size,
+         * and use a size field stored at the very end (whose storage
+         * is also included in the fragment body size) to distinguish
+         * the real body from the selfmod copy (storing it there
+         * rather than in fragment_t to save space in the common case).
+         */
+        /* assume contiguous bb */
+        app_pc end_bb_pc;
+        ASSERT((flags & FRAG_HAS_DIRECT_CTI) == 0);
+        /* FIXME PR 215217: a client may have truncated or otherwise changed
+         * the code, but we assume no new code has been added.  Thus, checking
+         * the original full range can only result in a false positive selfmod
+         * event, which is a performance issue only.
+         */
+        end_bb_pc = find_app_bb_end(dcontext, tag, flags);
+        ASSERT(end_bb_pc > tag);
+        ASSERT_TRUNCATE(copy_sz, uint, (ptr_uint_t)(end_bb_pc - tag) + sizeof(uint));
+        copy_sz = (uint)(end_bb_pc - tag) + sizeof(uint);
+        /* ensure this doesn't push fragment size over limit */
+        ASSERT(offset + copy_sz <= MAX_FRAGMENT_SIZE);
+        offset += copy_sz;
+        STATS_FCACHE_ADD(flags, selfmod_copy, copy_sz);
+    }
+    
+    /* FIXME on linux the signal fence exit before a syscall can trigger 
+     * these ASSERTS. We need some way to mark that exit always unlinked so
+     * we don't need to pad for it or figure out a better way to remove nops
+     * for tracing. Xref PR 215179, we allow additional pads for CLIENT_INTERFACE
+     * and LINUX by marking the bb untraceable and inserting nops. */
+#if !defined(LINUX) && !defined(CLIENT_INTERFACE)
+# if !defined(PROFILE_LINKCOUNT) && !defined(TRACE_HEAD_CACHE_INCR)
+    /* bbs shouldn't need more than a single pad */
+    ASSERT((PAD_FRAGMENT_JMPS(flags) && TEST(FRAG_IS_TRACE, flags)) ||
+           extra_jmp_padding_body+extra_jmp_padding_stubs == 
+           (PAD_FRAGMENT_JMPS(flags) ? MAX_PAD_SIZE : 0U));
+# else
+    /* no more than two pads should be needed for a bb with these defines */
+    ASSERT((PAD_FRAGMENT_JMPS(flags) && TEST(FRAG_IS_TRACE, flags)) ||
+           extra_jmp_padding_body+extra_jmp_padding_stubs <= 
+           (PAD_FRAGMENT_JMPS(flags) ? 2*MAX_PAD_SIZE : 0U));
+# endif
+#endif
+
+    /* create a new fragment_t, or fill in the emit wrapper for coarse-grain */
+    /* FIXME : don't worry too much about whether padding should be requested in
+     * the stub or body argument, fragment_create doesn't distinguish between
+     * the two */
+    f = fragment_create(dcontext, tag, offset+extra_jmp_padding_body,
+                        num_direct_stubs, num_indirect_stubs,
+                        stub_size_total+extra_jmp_padding_stubs, flags);
+    ASSERT(f != NULL);
+    DOSTATS({
+        /* PR 217008: avoid gcc warning from truncation assert in XSTATS_TRACK_MAX:
+         * "comparison is always true due to limited range of data type"
+         * to turn off have to turn off Wextra; hopefully future gcc
+         * will have patch out there for Walways-true. */
+        int tmp_size = f->size;
+        STATS_TRACK_MAX(max_fragment_requested_size, tmp_size);
+    });
+
+    if (PAD_FRAGMENT_JMPS(flags)) {
+        uint start_shift;
+        /* 2nd (pad_jmps) walk through instr list:
+         * -- record offset of each instr from start of fragment body. 
+         * -- insert any nops needed for patching alignment 
+         * recreate needs to do this too, so we use a shared routine */
+        start_shift = nop_pad_ilist(dcontext, f, ilist, true /* emitting, set offset */);
+        fcache_shift_start_pc(dcontext, f, start_shift);
+    }
+
+    /* emit prefix */
+    insert_fragment_prefix(dcontext, f);
+
+    /* 3rd walk through instr list: (2nd if -no_pad_jmps)
+     * -- initialize and set fields in link stub for each exit cti;
+     * -- emit each instr into the fragment. 
+     */
+    pc = set_linkstub_fields(dcontext, f, ilist, num_direct_stubs, num_indirect_stubs,
+                             true/*encode each instr*/);
+    /* pc should now be pointing to the beginning of the first exit stub */
+
+    /* emit the exit stub code */
+#ifdef CUSTOM_EXIT_STUBS
+    /* need to re-walk the instrlist to get the custom code
+     * if we had another linkstub_t field we could store it there (used to
+     * put it in stub_pc but that's not available in indirect_linkstub_t anymore)
+     */
+    inst = instrlist_first(ilist);
+#endif
+    for (l = FRAGMENT_EXIT_STUBS(f); l; l = LINKSTUB_NEXT_EXIT(l)) {
+#ifdef CUSTOM_EXIT_STUBS
+        byte *old_pc;
+        /* find inst corresponding to l */
+        while (!instr_is_exit_cti(inst)) {
+            inst = instr_get_next(inst);
+            ASSERT(inst != NULL);
+        }
+#endif
+       
+        if (TEST(FRAG_COARSE_GRAIN, flags) && LINKSTUB_DIRECT(l->flags)) {
+            /* Coarse-grain fragments do not have direct exit stubs.
+             * Instead they have entrance stubs, created when linking.
+             */
+            continue;
+        }
+
+        if (!EXIT_HAS_STUB(l->flags, flags)) {
+            /* there is no stub */
+            continue;
+        }
+
+        if (final_cbr_single_stub && LINKSTUB_FINAL(l)) {
+#ifdef CUSTOM_EXIT_STUBS
+            ASSERT(instr_exit_stub_code(inst) == NULL);
+#endif
+            no_stub = true;
+            if (!TEST(LINK_SEPARATE_STUB, l->flags)) {
+                /* still need to patch the cti, so set pc back to prev stub pc */
+                pc = prev_stub_pc;
+            }
+            LOG(THREAD, LOG_EMIT, 3,
+                "final exit sharing prev exit's stub @ "PFX"\n", prev_stub_pc);
+        }
+
+        if (TEST(LINK_SEPARATE_STUB, l->flags)) {
+            if (no_stub) {
+                if (LINKSTUB_NORMAL_DIRECT(l->flags)) {
+                    direct_linkstub_t *dl = (direct_linkstub_t *) l;
+                    dl->stub_pc = prev_stub_pc;
+                } else {
+                    ASSERT(LINKSTUB_CBR_FALLTHROUGH(l->flags));
+                    /* stub pc computation should return prev pc */
+                    ASSERT(EXIT_STUB_PC(dcontext, f, l) == prev_stub_pc);
+                } 
+            } else {
+                separate_stub_create(dcontext, f, l);
+            }
+            prev_stub_pc = EXIT_STUB_PC(dcontext, f, l);
+            ASSERT(prev_stub_pc != NULL);
+            /* pointing at start of stub is the unlink entry */
+            ASSERT(linkstub_unlink_entry_offset(dcontext, f, l) == 0);
+            patch_branch(EXIT_CTI_PC(f, l), EXIT_STUB_PC(dcontext, f, l), false);
+#ifdef CUSTOM_EXIT_STUBS
+            /* we don't currently support separate custom stubs */
+            ASSERT(instr_exit_stub_code(inst) == NULL);
+#endif
+            continue;
+        }
+
+        ASSERT(EXIT_HAS_LOCAL_STUB(l->flags, flags));
+
+        if (PAD_FRAGMENT_JMPS(flags)) {
+            uint custom_exit_length = 0;
+#ifdef CUSTOM_EXIT_STUBS
+            /* need to figure out size to get right offset */
+            if (custom_stubs_present) {
+                /* inst is pointing at l's exit inst */
+                instrlist_t *custom = (instrlist_t *) instr_exit_stub_code(inst);
+                if (custom != NULL) {
+                    instr_t *in;
+                    ASSERT(!no_stub);
+                    for (in = instrlist_first(custom); in; in = instr_get_next(in)) {
+                        custom_exit_length += instr_length(dcontext, in);
+                    }
+                }
+            }
+#endif
+            pc = pad_for_exitstub_alignment(dcontext, l, f, pc+custom_exit_length);
+        }
+
+#ifdef CUSTOM_EXIT_STUBS
+        old_pc = pc;
+        if (custom_stubs_present) {
+            /* inst is pointing at l's exit inst */
+            instrlist_t *custom = (instrlist_t *) instr_exit_stub_code(inst);
+            if (custom != NULL) {
+                instr_t *in;
+                ASSERT(!no_stub);
+                for (in = instrlist_first(custom); in; in = instr_get_next(in)) {
+                    pc = instr_encode(dcontext, in, (void*)pc);
+                    ASSERT(pc != NULL);
+                }
+            }
+        }
+        /* stub_pc is start of entire stub (== start of custom stub) for
+         * patching, fixed_stub_offset is start of fixed part of stub (beyond
+         * custom part) for things like linkcount that modify that
+         */
+        if (LINKSTUB_NORMAL_DIRECT(l->flags)) {
+            direct_linkstub_t *dl = (direct_linkstub_t *) l;
+            ASSERT(!TEST(LINK_SEPARATE_STUB, l->flags));
+            dl->stub_pc = old_pc;
+        }
+        ASSERT_TRUNCATE(l->fixed_stub_offset, ushort, (pc - old_pc));
+        l->fixed_stub_offset = (ushort) (pc - old_pc);
+        /* relocate the exit branch target so it takes to the stub */
+        patch_branch(EXIT_CTI_PC(f, l), old_pc, false);
+#else
+        if (LINKSTUB_NORMAL_DIRECT(l->flags)) {
+            direct_linkstub_t *dl = (direct_linkstub_t *) l;
+            dl->stub_pc = pc;
+        }
+        /* relocate the exit branch target so it takes to the unlink
+         * entry to the stub
+         */
+        patch_branch(EXIT_CTI_PC(f, l),
+                     pc + linkstub_unlink_entry_offset(dcontext, f, l), false);
+        LOG(THREAD, LOG_EMIT, 3,
+            "Exit cti "PFX" is targeting "PFX" + 0x%x => "PFX"\n",
+            EXIT_CTI_PC(f, l), pc, linkstub_unlink_entry_offset(dcontext, f, l),
+            pc + linkstub_unlink_entry_offset(dcontext, f, l));
+#endif
+
+/* FIXME : once bytes_for_exitstub_alignment is implemented for 
+ * PROFILE_LINKCOUNT remove this ifndef */
+#ifndef PROFILE_LINKCOUNT
+        DODEBUG({
+            uint shift = bytes_for_exitstub_alignment(dcontext, l, f, pc);
+            if (shift > 0) {
+                ASSERT(!PAD_FRAGMENT_JMPS(flags));
+                STATS_PAD_JMPS_ADD(flags, unaligned_stubs, 1);
+                STATS_PAD_JMPS_ADD(flags, unaligned_stubs_bytes, shift);
+            }
+        });
+#endif
+
+        /* insert an exit stub */
+        prev_stub_pc = pc;
+        if (!no_stub)
+            stub_size = insert_exit_stub(dcontext, f, l, pc);
+        /* note that we don't do proactive linking here since it may
+         * depend on whether this is a trace fragment, which is marked
+         * by the caller, who is responsible for calling link_new_fragment
+         */
+
+        /* if no_stub we assume stub_size is still what it was for prev stub,
+         * and yes we do need to adjust it back to the end of the single stub
+         */
+        pc += stub_size;
+    }
+
+    ASSERT(pc - f->start_pc <= f->size);
+
+    /* Give back extra space to fcache */
+    STATS_PAD_JMPS_ADD(flags, excess_bytes,  
+                       f->size - (pc - f->start_pc) - copy_sz);
+    if (PAD_FRAGMENT_JMPS(flags) &&
+        INTERNAL_OPTION(pad_jmps_return_excess_padding) &&
+        f->size - (pc - f->start_pc) - copy_sz > 0) {
+        /* will adjust size, must call before we copy the selfmod since we
+         * break abstraction by putting the copy space in the fcache
+         * extra field and fcache needs to read/modify the fields */
+        fcache_return_extra_space(dcontext, f, 
+                                  f->size - (pc - f->start_pc) - copy_sz);
+    }
+
+    if (TEST(FRAG_SELFMOD_SANDBOXED, flags)) {
+        /* put copy of the original app code at bottom of fragment */
+        cache_pc copy_pc;
+
+        ASSERT(f->size > copy_sz);
+        copy_pc = f->start_pc + f->size - copy_sz;
+        ASSERT(copy_pc == pc || 
+               (PAD_FRAGMENT_JMPS(flags) && 
+                !INTERNAL_OPTION(pad_jmps_return_excess_padding)));
+        /* size is stored at the end, but included in copy_sz */
+        memcpy(copy_pc, tag, copy_sz - sizeof(uint));
+        *((uint *)(copy_pc + copy_sz - sizeof(uint))) = copy_sz;
+        /* count copy as part of fragment */
+        pc = copy_pc + copy_sz;
+    }
+
+    ASSERT(pc - f->start_pc <= f->size);
+    STATS_TRACK_MAX(max_fragment_size, pc - f->start_pc);
+    STATS_PAD_JMPS_ADD(flags, sum_fragment_bytes_ever, pc - f->start_pc);
+
+    /* if we don't give the extra space back to fcache, need to nop out the
+     * rest of the memory to avoid problems with shifting fcache pointers */
+    if (PAD_FRAGMENT_JMPS(flags) && !INTERNAL_OPTION(pad_jmps_return_excess_padding)) {
+        /* these can never be reached, but will be decoded by shift 
+         * fcache pointers */
+        SET_TO_NOPS(pc, f->size - (pc - f->start_pc));
+    } else {
+        ASSERT(f->size - (pc - f->start_pc) == 0);
+    }
+
+    /* finalize the fragment
+     * that means filling in all offsets, etc. that weren't known at
+     * instrlist building time
+     */
+#ifdef PROFILE_RDTSC
+    if (dynamo_options.profile_times)
+        finalize_profile_call(dcontext, f);
+#endif
+#ifdef CHECK_RETURNS_SSE2
+    finalize_return_check(dcontext, f);
+#endif
+    if ((flags & FRAG_IS_TRACE) != 0) {
+        /* trace-only finalization */
+#ifdef SIDELINE
+        if (dynamo_options.sideline) {
+            finalize_sideline_prefix(dcontext, f);
+        }
+#endif
+    } else {
+        /* bb-only finalization */
+    }
+    if ((flags & FRAG_SELFMOD_SANDBOXED) != 0) {
+        finalize_selfmod_sandbox(dcontext, f);
+    }
+
+    /* add fragment to vm area lists */
+    vm_area_add_fragment(dcontext, f, vmlist);
+
+    /* store translation info, if requested */
+    if (TEST(FRAG_HAS_TRANSLATION_INFO, f->flags)) {
+        ASSERT(!TEST(FRAG_COARSE_GRAIN, f->flags));
+        fragment_record_translation_info(dcontext, f, ilist);
+    }
+
+    /* if necessary, i-cache sync */
+    machine_cache_sync((void*)f->start_pc, (void*)(f->start_pc+f->size), true);
+
+    /* Future removal and replacement w/ the real fragment must be atomic
+     * wrt linking, so we hold the change_linking_lock across both (xref
+     * case 5474).
+     * We must grab the change_linking_lock even for private fragments
+     * if we have any shared fragments in the picture, to make atomic
+     * our future fragment additions and removals and the associated
+     * fragment and future fragment lookups.
+     * Optimization: we could do away with this and try to only
+     * grab it when a private fragment needs to create a shared
+     * future, redoing our lookup with the lock held.
+     */
+    if (link_fragment || add_to_htable)
+        SHARED_RECURSIVE_LOCK(acquire, change_linking_lock);
+
+    if (link_fragment) {
+        /* link BEFORE adding to the hashtable, to reduce races, though we
+         * should be able to handle them :)
+         */
+        if (replace_fragment)
+            shift_links_to_new_fragment(dcontext, replace_fragment, f);
+        else
+            link_new_fragment(dcontext, f);
+    }
+
+    if (add_to_htable) {
+        if (TEST(FRAG_COARSE_GRAIN, f->flags)) {
+            /* added in link_new_fragment */
+        } else
+            fragment_add(dcontext, f);
+
+        DODEBUG({
+            if (TEST(FRAG_SHARED, flags))
+                ASSERT(fragment_lookup_future(dcontext, tag) == NULL);
+            else
+                ASSERT(fragment_lookup_private_future(dcontext, tag) == NULL);
+        });
+    }
+
+    if (link_fragment || add_to_htable)
+        SHARED_RECURSIVE_LOCK(release, change_linking_lock);
+
+    SELF_PROTECT_CACHE(dcontext, NULL, READONLY);
+
+    KSTOP(emit);
+    return f;
+}
+
+/* Emits code for ilist into the fcache, returns the created fragment.
+ * Does not add the fragment to the ftable, leaving it as an "invisible"
+ * fragment.  This means it is the caller's responsibility to ensure
+ * it is properly disposed of when done with.
+ * The fragment is also not linked, to give the caller more flexibility.
+ */
+fragment_t *
+emit_invisible_fragment(dcontext_t *dcontext, app_pc tag,
+                        instrlist_t *ilist, uint flags, void *vmlist)
+{
+    return emit_fragment_common(dcontext, tag, ilist, flags, vmlist,
+                                false /* don't link: up to caller */,
+                                false /* don't add: it's invisible! */,
+                                NULL /* not replacing */);
+}
+
+/* Emits code for ilist into the fcache, returns the created
+ * fragment.  Adds the fragment to the fragment hashtable and
+ * links it as a new fragment.
+ */
+fragment_t *
+emit_fragment(dcontext_t *dcontext, app_pc tag, instrlist_t *ilist, uint flags,
+              void *vmlist, bool link)
+{
+    return emit_fragment_common(dcontext, tag, ilist, flags, vmlist,
+                                link, 
+                                true /* add to htable */,
+                                NULL /* not replacing */);
+}
+
+/* Emits code for ilist into the fcache, returns the created
+ * fragment.  Adds the fragment to the fragment hashtable and
+ * links it as a new fragment.
+ */
+fragment_t *
+emit_fragment_ex(dcontext_t *dcontext, app_pc tag, instrlist_t *ilist, uint flags,
+                 void *vmlist, bool link, bool visible)
+{
+    return emit_fragment_common(dcontext, tag, ilist, flags, vmlist,
+                                link, visible, NULL /* not replacing */);
+}
+
+/* Emits code for ilist into the fcache, returns the created
+ * fragment.  Adds the fragment to the fragment hashtable and
+ * links it as a new fragment by subsuming replace's links.
+ */
+fragment_t *
+emit_fragment_as_replacement(dcontext_t *dcontext, app_pc tag, instrlist_t *ilist,
+                             uint flags, void *vmlist, fragment_t *replace)
+{
+    return emit_fragment_common(dcontext, tag, ilist, flags, vmlist,
+                                true /* link it up */,
+                                true /* add to htable */,
+                                replace /* replace this fragment */);
+}
+
diff --git a/core/emit.h b/core/emit.h
new file mode 100644
index 000000000..c31f7e84b
--- /dev/null
+++ b/core/emit.h
@@ -0,0 +1,89 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * emit.h - fragment code generation interfaces
+ */
+
+#ifndef _EMIT_H_
+#define _EMIT_H_ 1
+
+/* Emits code for ilist into the fcache, returns the created
+ * fragment.  Adds the fragment to the fragment hashtable and
+ * links it as a new fragment.
+ */
+fragment_t *
+emit_fragment(dcontext_t *dcontext, app_pc tag,
+              instrlist_t *ilist, uint flags, void *vmlist,
+              bool link);
+
+fragment_t *
+emit_fragment_ex(dcontext_t *dcontext, app_pc tag,
+                 instrlist_t *ilist, uint flags, void *vmlist,
+                 bool link, bool visible);
+
+/* Emits code for ilist into the fcache, returns the created fragment.
+ * Does not add the fragment to the ftable, leaving it as an "invisible"
+ * fragment.  This means it is the caller's responsibility to ensure
+ * it is properly disposed of when done with.
+ * The fragment is also not linked, to give the caller more flexibility.
+ */
+fragment_t *
+emit_invisible_fragment(dcontext_t *dcontext, app_pc tag,
+                        instrlist_t *ilist, uint flags, void *vmlist);
+
+fragment_t *
+emit_fragment_as_replacement(dcontext_t *dcontext, app_pc tag, instrlist_t *ilist,
+                             uint flags, void *vmlist, fragment_t *replace);
+
+# ifdef INTERNAL
+void stress_test_recreate(dcontext_t *dcontext, fragment_t *f, 
+                          instrlist_t *ilist);
+# endif
+
+bool
+final_exit_shares_prev_stub(dcontext_t *dcontext, instrlist_t *ilist, uint frag_flags);
+
+/* Walks ilist and f's linkstubs, setting each linkstub_t's fields appropriately
+ * for the corresponding exit cti in ilist.
+ * If emit is true, also encodes each instr in ilist to f's cache slot,
+ * increments stats for new fragments, and returns the final pc after all encodings.
+ */
+cache_pc
+set_linkstub_fields(dcontext_t *dcontext, fragment_t *f, instrlist_t *ilist, 
+                    uint num_direct_stubs, uint num_indirect_stubs, bool emit);
+
+#endif /* _EMIT_H_ */
diff --git a/core/fcache.c b/core/fcache.c
new file mode 100644
index 000000000..229cfa146
--- /dev/null
+++ b/core/fcache.c
@@ -0,0 +1,4588 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * fcache.c - fcache manager
+ */
+
+#include "globals.h"
+#include "link.h"
+#include "fragment.h"
+#include "fcache.h"
+#include "monitor.h"
+#ifdef HOT_PATCHING_INTERFACE
+# include "hotpatch.h"
+#endif
+#include <string.h> /* for memcpy */
+#include <stddef.h> /* for offsetof */
+#include <limits.h> /* for UCHAR_MAX */
+#include "perscache.h"
+#include "synch.h"
+
+/* A code cache is made up of multiple separate mmapped units
+ * We grow a unit by resizing, shifting, and relinking, up to a maximum size,
+ * at which point we create a separate unit if we need more space
+ * Cache is extremely flexible in allowing resizing (hard to support)
+ * and separate units of different sizes, in any combination.
+ * We will build a unit larger than cache_{bb,trace}_unit_max for a single large request,
+ * up to the max cache size.
+ * To save memory, we don't make units larger than 64KB.  Not much advantage
+ * to have huge units.
+ */
+
+#ifdef NATIVE_RETURN
+/* cannot handle resizing of cache, separate units only */
+# define DISALLOW_CACHE_RESIZING
+#endif
+
+/* to make it easy to switch to INTERNAL_OPTION */
+#define FCACHE_OPTION(o) dynamo_options.o
+
+/* 
+ * unit initial size is FCACHE_OPTION(cache_{bb,trace}_unit_init, default is 32*1024 
+ * it grows by 4X steps up to FCACHE_OPTION(cache_{bb,trace}_unit_quadruple, default is 32*1024
+ * unit max size is FCACHE_OPTION(cache_{bb,trace}_unit_max, default is 64*1024 
+ * once at max size, we make new units, all of max size
+ *
+ * thus default is to do no quadrupling, just a single doubling and then no more resizing
+ * FIXME: should we stop resizing altogether and just have variable-sized
+ * separate units?  it's not like a 32K unit is too small to keep around...
+ * OTOH, we want the flexibility of resizing, for servers apps with lots of threads
+ * we may move the initial unit size smaller
+ */
+
+/* Invariant: a cache unit is always at least this constant times the largest
+ * fragment inside it in size (this can make it larger than cache_{bb,trace}_unit_max)
+ */
+#define MAX_SINGLE_MULTIPLE 2
+
+/* adaptive working set reactive cache expansion default parameters:
+ * first expansion(s) free
+ *   FCACHE_OPTION(cache_{bb,trace}_unit_upgrade, default is 64KB so 32=>64 is free
+ * after that, only expand if regenerated/replaced ratio matches these
+ * numbers (no floating-point, so we use 2 ints):
+ *   dynamo_options.cache_{bb,trace}_regen default = 10
+ *   dynamo_options.cache_{bb,trace}_replace default = 50
+ * special cases:
+ *   if cache_{bb,trace}_regen == 0, never increases cache size after free upgrade
+ *   if cache_{bb,trace}_replace == 0, always increases (effectively 
+ *     disabling adaptive working set, although the nice way to disable
+ *     is to use -no_finite_{bb,trace}_cache)
+ */
+
+/*
+ * Maximum cache sizes are stored in these two options:
+ *   dynamo_options.cache_bb_max
+ *   dynamo_options.cache_trace_max
+ * A value of 0 means "infinite"
+ */
+
+/* Minimum size to leave as an empty hole, which will be prepended to FIFO
+ * for private cache and so will be filled even if it means bumping adjacent guys.
+ * For shared cache, we currently don't fill empty slots, but once we do we will
+ * only fill with a fragment that fits (case 4485).  Shared empties are not prepended
+ * but rather are placed in the same location in the FIFO as the deleted guy.
+ * Should be the minimum common fragment size.
+ */
+#define MIN_EMPTY_HOLE(cache) \
+  MAX(((cache)->is_trace ? 64U : \
+        ((!(cache)->is_shared && DYNAMO_OPTION(separate_private_stubs)) ? 20U : \
+         (((cache)->is_shared && DYNAMO_OPTION(separate_shared_stubs)) ? 20U : 64U))), \
+      MIN_FCACHE_SLOT_SIZE(cache))
+
+/* Minimum end-of-cache hole size -- anything smaller and the cache is "full"
+ * This is 2x the smallest fragment size
+ * FIXME: use larger size for trace cache?
+ */
+#define MIN_UNIT_END_HOLE(cache) ((uint) 2 * MIN_EMPTY_HOLE(cache))
+
+/* This is ignored for coarse fragments */
+#define START_PC_ALIGNMENT 4
+
+/* Alignment: we assume basic blocks don't care so much about alignment, we go
+ * to 4 to avoid sub-word fetches. NOTE we also need at least START_PC_ALIGNMENT
+ * byte alignment for the start_pc padding for -pad_jmps_shift_{bb,trace} support 
+ * (need at least that much even without the option since we back align the 
+ * start_pc to get the header)
+ */
+#define SLOT_ALIGNMENT(cache) \
+    ( (cache)->is_trace ? DYNAMO_OPTION(cache_trace_align) : \
+      ((cache)->is_coarse ? DYNAMO_OPTION(cache_coarse_align) : \
+        DYNAMO_OPTION(cache_bb_align)) )
+
+/* We use a header to have a backpointer to the fragment_t */
+ /* FIXME: currently this abstraction type is unused, we rather use
+  * *(fragment_t **) when working with the backpointer.  If are to add
+  * more fields should use this.  Although fragment_t may be a better
+  * place to keep such information.  */
+typedef struct _live_header_t {
+    /* FIXME: sizeof(live_header_t) should match HEADER_SIZE */
+    fragment_t *f;
+} live_header_t;
+
+
+/* NOTE this must be a multiple of START_PC_ALIGNMENT bytes so that the 
+ * start_pc is properly aligned (for -pad_jmps_shift_{bb,trace} support) */
+#define HEADER_SIZE(f) \
+    (TEST(FRAG_COARSE_GRAIN, (f)->flags) ? 0 : (sizeof(fragment_t*)))
+#define HEADER_SIZE_FROM_CACHE(cache) \
+    ((cache)->is_coarse ? 0 : (sizeof(fragment_t*)))
+
+/**************************************************
+ * We use a FIFO replacement strategy
+ * Empty slots in the cache are always at the front of the list, and they
+ * take the form of the empty_slot_t struct.
+ * Target fragments to delete are represented in the FIFO by their fragment_t
+ * struct, using the next_fcache field to chain them.
+ * We use a header for each fragment so we can delete adjacent-in-cache
+ * (different from indirected FIFO list) to make contiguous space available.
+ * Padding for alignment is always at the end of the fragment, so it can be
+ * combined with empty space eaten up when deleting a fragment but only needing
+ * some of its room.  We assume everything starts out aligned, and the same alignment
+ * on the end of each slot keeps it aligned.
+ * Thus:
+ *           ------
+ *           header
+ *           <up to START_PC_ALIGNMENT-1 bytes padding, stored in the alignment of start_pc>
+ * start_pc: prefix
+ *           body
+ *           stubs
+ *           <padding for alignment>
+ *           ------
+ *           header
+ *           <up to START_PC_ALIGNMENT-1 bytes padding, stored in the alignment of start_pc>
+ * start_pc: ...
+ */
+typedef struct _empty_slot_t {
+    cache_pc  start_pc;         /* very top of location in fcache */
+    /* flags MUST be at same location as fragment_t->flags
+     * we use flags==FRAG_IS_EMPTY_SLOT to indicate an empty slot
+     */
+    uint      flags;
+    fragment_t  *next_fcache;     /* for chaining fragments in fcache unit */
+    fragment_t  *prev_fcache;     /* for chaining fragments in fcache unit */
+    uint      fcache_size;      /* size rounded up to cache line boundaries;
+                                 * not just ushort so we can merge adjacents;
+                                 * not size_t since each unit assumed <4GB. */
+} empty_slot_t;
+
+
+/* macros to make dealing with both fragment_t and empty_slot_t easier */
+#define FRAG_EMPTY(f) (TEST(FRAG_IS_EMPTY_SLOT, (f)->flags))
+
+#define FRAG_START(f) (TEST(FRAG_IS_EMPTY_SLOT, (f)->flags) ? \
+    ((empty_slot_t *)(f))->start_pc : (f)->start_pc)
+
+/* N.B.: must hold cache lock across any set of a fragment's start_pc or size
+ * once that fragment is in a cache, as contig-cache-walkers need a consistent view!
+ * FIXME: we can't assert as we can't do a unit lookup at all use sites
+ */
+#define FRAG_START_ASSIGN(f, val) do { \
+    if (TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) \
+        ((empty_slot_t *)(f))->start_pc = (val); \
+    else \
+        (f)->start_pc = (val); \
+} while (0)
+
+/* for -pad_jmps_shift_{bb,trace} we may have shifted the start_pc forward by up
+ * to START_PC_ALIGNMENT-1 bytes, back align to get the right header pointer */
+#define FRAG_START_PADDING(f) \
+    ((TEST(FRAG_IS_EMPTY_SLOT, (f)->flags) || !PAD_JMPS_SHIFT_START((f)->flags)) ?      \
+     0 : (ASSERT(CHECK_TRUNCATE_TYPE_uint((ptr_uint_t)                                  \
+                 ((f)->start_pc - ALIGN_BACKWARD((f)->start_pc, START_PC_ALIGNMENT)))), \
+         (uint)((ptr_uint_t)(f)->start_pc -                                             \
+                ALIGN_BACKWARD((f)->start_pc, START_PC_ALIGNMENT))))
+
+#define FRAG_HDR_START(f) (FRAG_START(f) - HEADER_SIZE(f) - FRAG_START_PADDING(f))
+
+#define FRAG_SIZE(f) ((TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) ? \
+    ((empty_slot_t *)(f))->fcache_size : \
+    (f)->size + (uint)(f)->fcache_extra + FRAG_START_PADDING(f))
+
+/* N.B.: must hold cache lock across any set of a fragment's start_pc or size
+ * once that fragment is in a cache, as contig-cache-walkers need a consistent view!
+ * FIXME: we can't assert as we can't do a unit lookup at all use sites
+ */
+#define FRAG_SIZE_ASSIGN(f, val) do {                                                   \
+    if (TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) {                                             \
+        ASSERT_TRUNCATE(((empty_slot_t *)(f))->fcache_size, uint, val);                    \
+        ((empty_slot_t *)(f))->fcache_size = (val);                                        \
+    } else {                                                                            \
+        /* cl has string limit so need temp to get ASSERT to compile */                 \
+        uint extra_tmp = ((val) - ((f)->size + FRAG_START_PADDING(f)));                 \
+        ASSERT_TRUNCATE((f)->fcache_extra, byte, extra_tmp);                            \
+        (f)->fcache_extra = (byte) extra_tmp;                                           \
+    }                                                                                   \
+} while (0)
+
+#define FIFO_NEXT(f) ((TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) ? \
+    ((empty_slot_t *)(f))->next_fcache : (ASSERT(!TEST(FRAG_SHARED, (f)->flags)), \
+    ((private_fragment_t *)(f))->next_fcache))
+
+#define FIFO_NEXT_ASSIGN(f, val) do { \
+    if (TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) \
+        ((empty_slot_t *)(f))->next_fcache = (val); \
+    else { \
+        ASSERT(!TEST(FRAG_SHARED, (f)->flags)); \
+        ((private_fragment_t *)(f))->next_fcache = (val); \
+    } \
+} while (0)
+
+#define FIFO_PREV(f) ((TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) ? \
+    ((empty_slot_t *)(f))->prev_fcache : (ASSERT(!TEST(FRAG_SHARED, (f)->flags)), \
+    ((private_fragment_t *)(f))->prev_fcache))
+
+#define FIFO_PREV_ASSIGN(f, val) do { \
+    if (TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) \
+        ((empty_slot_t *)(f))->prev_fcache = (val); \
+    else { \
+        ASSERT(!TEST(FRAG_SHARED, (f)->flags)); \
+        ((private_fragment_t *)(f))->prev_fcache = (val); \
+    } \
+} while (0)
+
+#define FRAG_TAG(f) ((TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) ? \
+    ((empty_slot_t *)(f))->start_pc : (f)->tag)
+
+#ifdef DEBUG
+# define FRAG_ID(f) ((TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) ? -1 : (f)->id)
+#endif
+
+#define FIFO_UNIT(f) \
+    (fcache_lookup_unit((TEST(FRAG_IS_EMPTY_SLOT, (f)->flags)) ? \
+        (((empty_slot_t *)(f))->start_pc) : ((f)->start_pc)))
+
+/* Shared fragments do NOT use a FIFO as they cannot easily replace existing
+ * fragments.  Instead they use a free list (below).
+ */
+#define USE_FIFO(f) (!TEST(FRAG_SHARED, (f)->flags))
+#define USE_FREE_LIST(f) (TEST(FRAG_SHARED, (f)->flags) && \
+                          !TEST(FRAG_COARSE_GRAIN, (f)->flags))
+#define USE_FIFO_FOR_CACHE(c) (!(c)->is_shared)
+#define USE_FREE_LIST_FOR_CACHE(c) ((c)->is_shared && !(c)->is_coarse)
+
+/**************************************************
+ * Free empty slot lists of different sizes for shared caches, where we
+ * cannot easily delete victims to make room in too-small slots.
+ */
+static const uint FREE_LIST_SIZES[] = {
+    /* Since we have variable sizes, and we do not pad to bucket
+     * sizes, each bucket contains free slots that are in
+     * [SIZE[bucket], SIZE[bucket+1]) where the top one is infinite.  
+     * case 7318 about further extensions
+     *
+     * Free slots in the current unit are added only if in the middle
+     * of the unit (the last ones are turned back into unclaimed space).
+     *
+     * Tuned for bbs: smallest are 40 (with stubs), plurality are 56, distribution trails 
+     * off slowly beyond that.  Note that b/c of pad_jmps we request more than
+     * the final sizes.
+     * FIXME: these #s are for inlined stubs, should re-tune w/ separate stubs
+     * (case 7163)
+     */
+    0, 44, 52, 56, 64, 72, 80, 112, 172
+};
+#define FREE_LIST_SIZES_NUM (BUFFER_SIZE_ELEMENTS(FREE_LIST_SIZES))
+
+/* To support physical cache contiguity walking we store both a
+ * next-free and prev-free pointer and a size at the top of the empty slot, and
+ * don't waste memory with an empty_slot_t data structure.
+ * We also use flags field to allow distinguishing free list slots
+ * from live fragment_t's (see notes by flags field below).
+ *
+ * Our free list coalescing assumes that a fragment_t that follows a
+ * free list entry has the FRAG_FOLLOWS_FREE_ENTRY flag set.
+ *
+ * FIXME: could avoid heap w/ normal empty_slot_t scheme: if don't have
+ * separate stubs, empty_slot_t @ 20 bytes (no start_pc) should fit in
+ * any cache slot.  Could save a few MB of heap on large apps.
+ * (This is case 4937.)
+ *
+ * FIXME: If free lists work well we could try using for private
+ * caches as well, instead of the empty_slot_t-struct-on-FIFO scheme.
+ * 
+ * FIXME: unit pointer may be useful to avoid fcache_lookup_unit
+ */
+typedef struct _free_list_header_t {
+    struct _free_list_header_t *next;
+    /* We arrange these two so that the FRAG_FCACHE_FREE_LIST flag will be set
+     * at the proper bit as though these were a "uint flags" at the same offset
+     * in the struct as fragment_t.flags.  Since noone else examines a free list
+     * as though it might be a fragment_t, we don't care about the other flags.
+     * We have an ASSERT in fcache_init() to ensure the byte ordering is right.
+     *
+     * Since we compare a fragment_t* to this inlined struct, we're really
+     * comparing a fragment_t* to the first field, the next pointer.  So when we
+     * de-reference the flags we're looking at the flags of the next entry in
+     * the free list.  Thus to identify a free list entry we must check for
+     * either NULL or for the FRAG_FCACHE_FREE_LIST flag.
+     */
+    ushort flags;
+    ushort size;
+    struct _free_list_header_t *prev;
+} free_list_header_t;
+
+/* We also place a size field at the end of the free list slot.
+ * Combined w/ the FRAG_FOLLOWS_FREE_ENTRY flag this allows us to coalesce
+ * new free list entries with existing previous entries.
+ */
+typedef struct _free_list_footer_t {
+    ushort size;
+} free_list_footer_t;
+
+#define MAX_FREE_ENTRY_SIZE USHRT_MAX
+
+/* See notes above: since f is either fragment_t* or free_list_header_t.next,
+ * we're checking the next free list entry's flags by dereferencing, forcing a
+ * check for NULL as well (== end of list).
+ */
+#define FRAG_IS_FREE_LIST(f) ((f) == NULL || TEST(FRAG_FCACHE_FREE_LIST, (f)->flags))
+
+/* De-references the fragment header stored at the start of the next
+ * fcache slot, given a pc and a size for the current slot.
+ */
+#define FRAG_NEXT_SLOT(pc, size) (*((fragment_t **)((pc) + (size))))
+
+/* Caller must know that the next slot is a free slot! */
+#define FRAG_NEXT_FREE(pc, size) ((free_list_header_t *)((pc) + (size)))
+
+/* FIXME: for non-free-list-using caches we could shrink this.
+ * Current smallest bb is 5 bytes (single jmp) align-4 + header is 12,
+ * and we're at 16 here, so we are wasting some space, but very few
+ * fragments are under 16 bytes (see request_size_histogram[])
+ */
+#define MIN_FCACHE_SLOT_SIZE(cache) \
+    ((cache)->is_coarse ? 0 : (sizeof(free_list_header_t)+sizeof(free_list_footer_t)))
+
+/**************************************************
+ * single mmapped piece of cache
+ */
+struct _fcache;
+
+#define UNIT_RESERVED_SIZE(u) ((size_t)((u)->reserved_end_pc - (u)->start_pc))
+
+typedef struct _fcache_unit_t {
+    cache_pc start_pc;         /* start address of fcache storage */
+    cache_pc end_pc;           /* end address of committed storage, open-ended */
+    cache_pc cur_pc;           /* if not filled up yet, bottom of cache */
+    cache_pc reserved_end_pc;  /* reservation end address, open-ended */
+    size_t size;               /* committed size: equals (end_pc - start_pc) */
+    bool full;                 /* to tell whether cache is filled to end */
+    struct _fcache *cache;     /* up-pointer to parent cache */
+#if defined(SIDELINE) || defined(WINDOWS_PC_SAMPLE)
+    dcontext_t *dcontext;
+#endif
+    bool writable;             /* remember state of cache memory protection */
+#ifdef WINDOWS_PC_SAMPLE
+    /* We cache these values for units_to_{flush,free} units whose cache
+     * field has been invalidated
+     */
+    bool was_trace;
+    bool was_shared;
+    profile_t *profile;
+#endif
+    bool pending_free;         /* was entire unit flushed and slated for free? */
+#ifdef DEBUG
+    bool pending_flush;        /* indicates in-limbo unit pre-flush is still live */
+#endif    
+    uint flushtime;            /* free this unit when this flushtime is freed --
+                                * used only for units_to_free list, else 0 */
+    struct _fcache_unit_t *next_global; /* used to link all units */
+    struct _fcache_unit_t *prev_global; /* used to link all units */
+    struct _fcache_unit_t *next_local;  /* used to link an fcache_t's units */
+} fcache_unit_t;
+
+#ifdef DEBUG
+/* study request allocation sizes */
+enum {
+    HISTOGRAM_GRANULARITY = 4,
+    HISTOGRAM_MAX_SIZE = 256,
+};
+#endif /* DEBUG */
+
+/**************************************************
+ * one "code cache" of a single type of fragment, made up of potentially
+ * multiple FcacheUnits
+ */
+typedef struct _fcache {
+    /* FIXME: do we want space or perf here (bitfield vs full field)? */
+    bool is_trace:1;             /* for varying alignment, etc. */
+    bool is_shared:1;
+#ifdef DEBUG
+    /* A local cache's pointer has not escaped to any other thread.
+     * We only use this flag to get around lock ordering issues w/
+     * persistent caches and we don't bother to set it for all
+     * private caches.
+     */
+    bool is_local:1;           /* no lock needed since only known to this thread */
+#endif
+    /* Is this a dedicated coarse-grain cache unit */
+    bool is_coarse:1;
+    fragment_t *fifo;            /* the FIFO list of fragments to delete.
+                                * also includes empty slots as EmptySlots
+                                * (all empty slots are at front of FIFO) */
+    fcache_unit_t *units;         /* list of all units, also FIFO -- the front
+                                * of the list is the only potentially
+                                * non-full unit */
+    size_t size;               /* sum of sizes of all units */
+
+    /* can't rely on bb_building_lock b/c shared deletion doesn't hold it,
+     * and cleaner to have dedicated lock
+     */
+    mutex_t lock;
+
+#ifdef DEBUG
+    char *name;
+    bool consistent; /* used to avoid fcache_fragment_pclookup problems */
+#endif
+
+    /* backpointer for mapping cache pc to coarse info for inter-unit unlink */
+    coarse_info_t *coarse_info;
+
+    /* we cache parameter here so we don't have to dispatch on bb/trace
+     * type every time -- this also allows flexibility if we ever want
+     * to have different parameters per thread or something.
+     * not much of a space hit at all since there are 2 caches per thread
+     * and then 2 global caches.
+     */
+    uint max_size;             /* maximum sum of sizes */
+    uint max_unit_size;
+    uint max_quadrupled_unit_size;
+    uint free_upgrade_size;
+    uint init_unit_size;
+    bool finite_cache;
+    uint regen_param;
+    uint replace_param;
+
+    /* for adaptive working set: */
+    uint      num_regenerated;
+    uint      num_replaced; /* for shared cache, simply number created */
+    /* for fifo caches, wset_check is simply an optimization to avoid
+     * too many checks when parameters are such that regen<<replace
+     */
+    int      wset_check;
+    /* for non-fifo caches, this flag indicates we should start
+     * recording num_regenerated and num_replaced
+     */
+    bool     record_wset;
+
+    free_list_header_t *free_list[FREE_LIST_SIZES_NUM];
+#ifdef DEBUG
+    uint free_stats_freed[FREE_LIST_SIZES_NUM]; /* occurrences */
+    uint free_stats_reused[FREE_LIST_SIZES_NUM]; /* occurrences */
+    uint free_stats_coalesced[FREE_LIST_SIZES_NUM]; /* occurrences */
+    uint free_stats_split[FREE_LIST_SIZES_NUM]; /* entry split, occurrences */
+    uint free_stats_charge[FREE_LIST_SIZES_NUM]; /* bytes on free list */
+    /* sizes of real requests and frees */
+    uint request_size_histogram[HISTOGRAM_MAX_SIZE/HISTOGRAM_GRANULARITY];
+    uint free_size_histogram[HISTOGRAM_MAX_SIZE/HISTOGRAM_GRANULARITY];
+#endif
+} fcache_t;
+
+/**************************************************
+ * per-thread structure: 
+ * FIXME: give a better name to distinguish from heap.c's _thread_units_t
+ */
+typedef struct _thread_units_t {
+    fcache_t *bb;    /* basic block fcache */
+    fcache_t *trace; /* trace fcache */
+    /* we delay unmapping units, but only one at a time: */
+    cache_pc pending_unmap_pc;
+    size_t pending_unmap_size;
+    /* are there units waiting to be flushed at a safe spot? */
+    bool pending_flush;
+} thread_units_t;
+
+#define ALLOC_DC(dc, cache) ((cache)->is_shared ? GLOBAL_DCONTEXT : (dc))
+
+/* We cannot acquire shared_cache_lock while allsynch-flushing as we then hold
+ * the lower-ranked shared_vm_areas lock, but the allsynch makes it safe to not
+ * acquire it.
+ */
+#define PROTECT_CACHE(cache, op) do {                         \
+    if ((cache)->is_shared && !is_self_allsynch_flushing()) { \
+        mutex_##op(&(cache)->lock);                           \
+    }                                                         \
+} while (0);
+
+#define CACHE_PROTECTED(cache) \
+    (!(cache)->is_shared || (cache)->is_local || \
+     OWN_MUTEX(&(cache)->lock) || dynamo_all_threads_synched)
+
+/**************************************************
+ * global, unique thread-shared structure: 
+ */
+typedef struct _fcache_list_t {
+    /* These lists are protected by allunits_lock. */
+    fcache_unit_t *units;      /* list of all allocated fcache units */
+    fcache_unit_t *dead;       /* list of deleted units ready for re-allocation */
+    /* FIXME: num_dead duplicates stats->fcache_num_free, but we want num_dead
+     * for release build too, so it's separate...can we do better?
+     */
+    uint num_dead;
+
+    /* Global lists of cache units to flush and to free, chained by next_local
+     * and kept on the live units list.  Protected by unit_flush_lock, NOT by
+     * allunits_lock.
+     * We keep these list pointers on the heap for selfprot (case 8074).
+     */
+    /* units to be flushed once at a safe spot */
+    fcache_unit_t *units_to_flush;
+    /* units to be freed once their contents are, kept sorted in
+     * increasing flushtime, with a tail pointer to make appends easy
+     */
+    fcache_unit_t *units_to_free;
+    fcache_unit_t *units_to_free_tail;
+} fcache_list_t;
+
+/* Kept on the heap for selfprot (case 7957). */
+static fcache_list_t *allunits;
+/* FIXME: rename to fcache_unit_lock? */
+DECLARE_CXTSWPROT_VAR(static mutex_t allunits_lock, INIT_LOCK_FREE(allunits_lock));
+
+DECLARE_CXTSWPROT_VAR(static mutex_t unit_flush_lock, INIT_LOCK_FREE(unit_flush_lock));
+
+static fcache_t *shared_cache_bb;
+static fcache_t *shared_cache_trace;
+
+/* To locate the fcache_unit_t corresponding to a fragment or empty slot
+ * we use an interval data structure rather than waste space with a
+ * backpointer in each fragment.
+ * Non-static so that synch routines in os.c can check its lock before calling
+ * is_pc_recreatable which calls in_fcache.
+ * Kept on the heap for selfprot (case 7957).
+ */
+vm_area_vector_t *fcache_unit_areas;
+
+/* indicates a reset is in progress (whereas dynamo_resetting indicates that
+ * all threads are suspended and so no synch is needed)
+ */
+DECLARE_FREQPROT_VAR(bool reset_in_progress, false);
+/* protects reset triggers: reset_pending, reset_in_progress, reset_at_nth_thread 
+ * FIXME: use separate locks for separate triggers?
+ * reset_at_nth_thread is wholly inside dynamo.c, e.g.
+ */
+DECLARE_CXTSWPROT_VAR(mutex_t reset_pending_lock, INIT_LOCK_FREE(reset_pending_lock));
+/* indicates a call to fcache_reset_all_caches_proactively() is pending in dispatch */
+DECLARE_FREQPROT_VAR(uint reset_pending, 0);
+
+/* these cannot be per-cache since caches are reset so we have them act globally.
+ * protected by allunits_lock since only read during unit creation.
+ */
+enum {
+    CACHE_BB = 0,
+    CACHE_TRACE,
+    CACHE_NUM_TYPES,
+};
+DECLARE_FREQPROT_VAR(uint reset_at_nth_unit[CACHE_NUM_TYPES], {0,});
+DECLARE_FREQPROT_VAR(uint reset_every_nth_unit[CACHE_NUM_TYPES], {0,});
+
+#define STATS_FCACHE_ADD(cache, stat, val) DOSTATS({   \
+    if (cache->is_shared) {                            \
+        if (cache->is_trace)                           \
+            STATS_ADD(fcache_shared_trace_##stat, val);\
+        else                                           \
+            STATS_ADD(fcache_shared_bb_##stat, val);   \
+    }                                                  \
+    else if (cache->is_trace)                          \
+        STATS_ADD(fcache_trace_##stat, val);           \
+    else                                               \
+        STATS_ADD(fcache_bb_##stat, val);              \
+})
+
+/* convenience routine to avoid casts to signed types everywhere */
+#define STATS_FCACHE_SUB(cache, stat, val) \
+    STATS_FCACHE_ADD(cache, stat, -(stats_int_t)(val))
+
+#define STATS_FCACHE_MAX(cache, stat1, stat2) DOSTATS({          \
+    if (cache->is_shared) {                                      \
+        if (cache->is_trace)                                     \
+            STATS_MAX(fcache_shared_trace_##stat1, fcache_shared_trace_##stat2); \
+        else                                                     \
+            STATS_MAX(fcache_shared_bb_##stat1, fcache_shared_bb_##stat2); \
+    }                                                            \
+    else if (cache->is_trace)                                    \
+        STATS_MAX(fcache_trace_##stat1, fcache_trace_##stat2);   \
+    else                                                         \
+        STATS_MAX(fcache_bb_##stat1, fcache_bb_##stat2);         \
+})
+
+#ifdef DEBUG
+/* forward decl */
+# ifdef INTERNAL
+static void
+verify_fifo(dcontext_t *dcontext, fcache_t *cache);
+
+static void
+print_fifo(dcontext_t *dcontext, fcache_t *cache);
+# endif
+
+static void 
+fcache_cache_stats(dcontext_t *dcontext, fcache_t *cache);
+#endif
+
+static fcache_t *
+fcache_cache_init(dcontext_t *dcontext, uint flags, bool initial_unit);
+
+static void 
+fcache_cache_free(dcontext_t *dcontext, fcache_t *cache, bool free_units);
+
+static void
+add_to_free_list(dcontext_t *dcontext, fcache_t *cache, fcache_unit_t *unit,
+                 fragment_t *f, cache_pc start_pc, uint size);
+
+static void 
+fcache_free_unit(dcontext_t *dcontext, fcache_unit_t *unit, bool dealloc_or_reuse);
+
+#define CHECK_PARAMS(who, name, ret) do {                           \
+    /* make it easier to set max */                                 \
+    if (FCACHE_OPTION(cache_##who##_max) > 0 &&                     \
+        FCACHE_OPTION(cache_##who##_max) < FCACHE_OPTION(cache_##who##_unit_max)) {     \
+        FCACHE_OPTION(cache_##who##_unit_max) = FCACHE_OPTION(cache_##who##_max);       \
+        FCACHE_OPTION(cache_##who##_unit_init) = FCACHE_OPTION(cache_##who##_max);      \
+        FCACHE_OPTION(cache_##who##_unit_quadruple) = FCACHE_OPTION(cache_##who##_max); \
+        FCACHE_OPTION(cache_##who##_unit_upgrade) = FCACHE_OPTION(cache_##who##_max);   \
+    }                                                                     \
+    /* case 7626: don't short-circuit checks, as later ones may be needed */ \
+    ret = check_param_bounds(&FCACHE_OPTION(cache_##who##_max),           \
+                             PAGE_SIZE,                                   \
+                             0,                                           \
+                             name" cache max size")                       \
+        || ret;                                                           \
+    /* N.B.: we assume cache unit max fits in uint */                     \
+    ret = check_param_bounds(&FCACHE_OPTION(cache_##who##_unit_max),      \
+                             FCACHE_OPTION(cache_##who##_unit_init),      \
+                             FCACHE_OPTION(cache_##who##_max),            \
+                             name" cache unit max size")                  \
+        || ret;                                                           \
+    ret = check_param_bounds(&FCACHE_OPTION(cache_##who##_unit_quadruple),\
+                             FCACHE_OPTION(cache_##who##_unit_init),      \
+                             FCACHE_OPTION(cache_##who##_max),            \
+                             name" cache unit quadruple-to size")         \
+        || ret;                                                           \
+    ret = check_param_bounds(&FCACHE_OPTION(cache_##who##_unit_upgrade),  \
+                             FCACHE_OPTION(cache_##who##_unit_init),      \
+                             FCACHE_OPTION(cache_##who##_max),            \
+                             name" cache unit free upgrade size")         \
+        || ret;                                                           \
+    ret = check_param_bounds(&FCACHE_OPTION(cache_##who##_unit_init),     \
+                             /* x64 does not support resizing fcache units */ \
+                             IF_X64_ELSE(FCACHE_OPTION(cache_##who##_unit_max), \
+                                         PAGE_SIZE),                      \
+                             FCACHE_OPTION(cache_##who##_unit_max),       \
+                             name" cache unit init size")                 \
+        || ret;                                                           \
+    ret = check_param_bounds(&FCACHE_OPTION(cache_commit_increment),      \
+                             PAGE_SIZE,                                   \
+                             FCACHE_OPTION(cache_##who##_unit_init),      \
+                             name" cache_commit_increment")               \
+        || ret;                                                           \
+} while (0);
+
+#define CHECK_WSET_PARAM(param, ret) do {                                     \
+        if (dynamo_options.cache_##param##_regen < 0) {                       \
+            USAGE_ERROR("-cache_"#param"_regen must be >= 0, is %d, setting to 0", \
+                        dynamo_options.cache_##param##_regen);                \
+            dynamo_options.cache_##param##_regen = 0;                         \
+            ret = true;                                                       \
+        }                                                                     \
+        if (dynamo_options.cache_##param##_replace < 0) {                     \
+            USAGE_ERROR("-cache_"#param"_replace must be >= 0, id %d, setting to 0", \
+                        dynamo_options.cache_##param##_replace);              \
+            dynamo_options.cache_##param##_replace = 0;                       \
+            ret = true;                                                       \
+        }                                                                     \
+        if (dynamo_options.cache_##param##_replace != 0 &&                    \
+            dynamo_options.cache_##param##_regen > dynamo_options.cache_##param##_replace) { \
+            USAGE_ERROR("-cache_"#param"_regen (currently %d) must be <= "    \
+                        "-cache_"#param"_replace (currently %d) (if -cache_"  \
+                        #param"_replace > 0), setting regen to equal replace",\
+                        dynamo_options.cache_##param##_regen,                 \
+                        dynamo_options.cache_##param##_replace);              \
+            dynamo_options.cache_##param##_regen = dynamo_options.cache_##param##_replace; \
+            ret = true;                                                       \
+        }                                                                     \
+    } while (0);
+
+/* pulled out from fcache_init, checks for compatibility among the fcache 
+ * options, returns true if modified the value of any options to make them
+ * compatible.  This is called while the options are writable. */
+bool
+fcache_check_option_compatibility()
+{
+    bool ret = false;
+    uint i;
+    CHECK_PARAMS(bb, "Basic block", ret);
+    CHECK_PARAMS(trace, "Trace", ret);
+    CHECK_WSET_PARAM(bb, ret);
+    CHECK_WSET_PARAM(trace, ret);
+    if (DYNAMO_OPTION(shared_bbs)) {
+        if (DYNAMO_OPTION(cache_shared_bb_max) > 0) {
+            /* case 8203: NYI */
+            USAGE_ERROR("-cache_shared_bb_max != 0 not supported");
+            dynamo_options.cache_shared_bb_max = 0;
+            ret = true;
+        }
+        CHECK_PARAMS(shared_bb, "Shared bb", ret);
+        CHECK_WSET_PARAM(shared_bb, ret);    
+        /* FIXME: cannot handle resizing of cache, separate units only */
+        /* case 7626: don't short-circuit checks, as later ones may be needed */
+        ret = check_param_bounds(&FCACHE_OPTION(cache_shared_bb_unit_init),
+                                 FCACHE_OPTION(cache_shared_bb_unit_max),
+                                 FCACHE_OPTION(cache_shared_bb_unit_max),
+                                 "cache_shared_bb_unit_init should equal cache_shared_bb_unit_max")
+            || ret;
+    }
+    if (DYNAMO_OPTION(shared_traces)) {
+        if (DYNAMO_OPTION(cache_shared_trace_max) > 0) {
+            /* case 8203: NYI */
+            USAGE_ERROR("-cache_shared_trace_max != 0 not supported");
+            dynamo_options.cache_shared_trace_max = 0;
+            ret = true;
+        }
+        CHECK_PARAMS(shared_trace, "Shared trace", ret);
+        CHECK_WSET_PARAM(shared_trace, ret);    
+        /* FIXME: cannot handle resizing of cache, separate units only */
+        ret = check_param_bounds(&FCACHE_OPTION(cache_shared_trace_unit_init),
+                                 FCACHE_OPTION(cache_shared_trace_unit_max),
+                                 FCACHE_OPTION(cache_shared_trace_unit_max),
+                                 "cache_shared_trace_unit_init should equal cache_shared_trace_unit_max")
+            || ret;
+    }
+    if (INTERNAL_OPTION(pad_jmps_shift_bb) &&
+        DYNAMO_OPTION(cache_bb_align) < START_PC_ALIGNMENT) {
+        USAGE_ERROR("if -pad_jmps_shift_bb, -cache_bb_align must be >= %d",
+                    START_PC_ALIGNMENT);
+        dynamo_options.cache_bb_align = START_PC_ALIGNMENT;
+        ret = true;
+    }
+    /* (case 8647: cache_coarse_align can be anything as we don't pad jmps) */
+    if (INTERNAL_OPTION(pad_jmps_shift_trace) &&
+        DYNAMO_OPTION(cache_trace_align) < START_PC_ALIGNMENT) {
+        USAGE_ERROR("if -pad_jmps_shift_trace, -cache_trace_align must be >= %d",
+                    START_PC_ALIGNMENT);
+        dynamo_options.cache_trace_align = START_PC_ALIGNMENT;
+        ret = true;
+    }
+    reset_at_nth_unit[CACHE_BB] = DYNAMO_OPTION(reset_at_nth_bb_unit);
+    reset_every_nth_unit[CACHE_BB] = DYNAMO_OPTION(reset_every_nth_bb_unit);
+    reset_at_nth_unit[CACHE_TRACE] = DYNAMO_OPTION(reset_at_nth_trace_unit);
+    reset_every_nth_unit[CACHE_TRACE] = DYNAMO_OPTION(reset_every_nth_trace_unit);
+    /* yes can set both to different values -- but every won't kick in until
+     * after first at
+     */
+    for (i=0; i<CACHE_NUM_TYPES; i++) {
+        if (reset_every_nth_unit[i] > 0 && reset_at_nth_unit[i] == 0)
+            reset_at_nth_unit[i] = reset_every_nth_unit[i];
+    }
+    return ret;
+}
+
+/* thread-shared initialization that should be repeated after a reset */
+static void
+fcache_reset_init(void)
+{
+    /* case 7966: don't initialize at all for hotp_only & thin_client
+     * FIXME: could set initial sizes to 0 for all configurations, instead
+     */
+    if (RUNNING_WITHOUT_CODE_CACHE())
+        return;
+
+    if (DYNAMO_OPTION(shared_bbs)) {
+        shared_cache_bb = fcache_cache_init(GLOBAL_DCONTEXT, FRAG_SHARED, true);
+        ASSERT(shared_cache_bb != NULL);
+        LOG(GLOBAL, LOG_CACHE, 1, "Initial shared bb cache is %d KB\n",
+            shared_cache_bb->init_unit_size/1024);
+    }
+    if (DYNAMO_OPTION(shared_traces)) {
+        shared_cache_trace = fcache_cache_init(GLOBAL_DCONTEXT,
+                                               FRAG_SHARED|FRAG_IS_TRACE, true);
+        ASSERT(shared_cache_trace != NULL);
+        LOG(GLOBAL, LOG_CACHE, 1, "Initial shared trace cache is %d KB\n",
+            shared_cache_trace->init_unit_size/1024);
+    }
+}
+
+/* initialization -- needs no locks */
+void
+fcache_init()
+{
+    ASSERT(offsetof(fragment_t, flags) == offsetof(empty_slot_t, flags));
+    DODEBUG({
+        /* ensure flag in ushort is at same spot as in uint */
+        static free_list_header_t free;
+        free.flags = FRAG_FAKE | FRAG_FCACHE_FREE_LIST;
+        ASSERT(TEST(FRAG_FCACHE_FREE_LIST, ((fragment_t*)(&free))->flags));
+        /* ensure treating fragment_t* as next will work */
+        ASSERT(offsetof(free_list_header_t, next) == offsetof(live_header_t, f));
+    });
+        
+    /* we rely on this */
+    ASSERT(FREE_LIST_SIZES[0] == 0);
+
+    VMVECTOR_ALLOC_VECTOR(fcache_unit_areas, GLOBAL_DCONTEXT,
+                          VECTOR_SHARED | VECTOR_NEVER_MERGE,
+                          fcache_unit_areas);
+
+    allunits = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, fcache_list_t, ACCT_OTHER, PROTECTED);
+    allunits->units = NULL;
+    allunits->dead = NULL;
+    allunits->num_dead = 0;
+    allunits->units_to_flush = NULL;
+    allunits->units_to_free = NULL;
+    allunits->units_to_free_tail = NULL;
+
+    fcache_reset_init();
+}
+
+#ifdef WINDOWS_PC_SAMPLE
+static void
+fcache_unit_profile_stop(fcache_unit_t *u)
+{
+    int sum;
+
+    stop_profile(u->profile);
+    sum = sum_profile(u->profile);
+    if (sum > 0) {
+        bool shared, trace;
+        if (u->cache == NULL) {
+            shared = u->was_shared;
+            trace = u->was_trace;
+        } else {
+            shared = u->cache->is_shared;
+            trace = u->cache->is_trace;
+        }
+        mutex_lock(&profile_dump_lock);
+        if (shared) {
+            print_file(profile_file,
+                       "\nDumping fcache %s unit profile (Shared)\n%d hits\n",
+                       trace ? "trace" :  "bb", 
+                       sum);
+        } else {
+            print_file(profile_file, 
+                       "\nDumping fcache %s unit profile (Thread %d)\n%d hits\n",
+                       trace ? "trace" :  "bb", 
+                       u->dcontext->owning_thread, sum);
+        }
+        dump_profile(profile_file, u->profile);
+        mutex_unlock(&profile_dump_lock);
+    }
+}
+#endif
+
+static inline void
+remove_unit_from_cache(fcache_unit_t *u)
+{
+    ASSERT(u->cache != NULL);
+    u->cache->size -= u->size;
+    RSTATS_DEC(fcache_num_live);
+    STATS_FCACHE_SUB(u->cache, capacity, u->size);
+#ifdef WINDOWS_PC_SAMPLE
+    /* Units moved to units_to_{flush,free} can't have their profile stopped
+     * until they are really freed, so we must cache their type here.
+     * We do need to clear their cache field to support private or other
+     * deletable flushable unit types (though w/ default ops today no flushable
+     * unit will have its cache deleted).
+     */
+    u->was_trace = u->cache->is_trace;
+    u->was_shared = u->cache->is_shared;
+#endif
+    u->cache = NULL;
+}
+
+static void
+fcache_really_free_unit(fcache_unit_t *u, bool on_dead_list, bool dealloc_unit)
+{
+    if (TEST(SELFPROT_CACHE, dynamo_options.protect_mask) && !u->writable) {
+        change_protection((void*)u->start_pc, u->size, WRITABLE);
+    }
+#ifdef WINDOWS_PC_SAMPLE
+    if (u->profile != NULL) {
+        if (!on_dead_list)
+            fcache_unit_profile_stop(u);
+        free_profile(u->profile);
+        u->profile = NULL;
+    }
+#endif
+    if (u->cache != NULL)
+        remove_unit_from_cache(u);
+    if (on_dead_list) {
+        ASSERT(u->cache == NULL);
+        allunits->num_dead--;
+        RSTATS_DEC(fcache_num_free);
+        STATS_SUB(fcache_free_capacity, u->size);
+    }
+    STATS_SUB(fcache_combined_capacity, u->size);
+    /* remove from interval data struct first to avoid races w/ it
+     * being re-used and not showing up in in_fcache
+     */
+    vmvector_remove(fcache_unit_areas, u->start_pc, u->reserved_end_pc);
+    if (dealloc_unit)
+        heap_munmap((void*)u->start_pc, UNIT_RESERVED_SIZE(u));
+    /* always dealloc the metadata */
+    nonpersistent_heap_free(GLOBAL_DCONTEXT, u, sizeof(fcache_unit_t)
+                            HEAPACCT(ACCT_MEM_MGT));
+}
+
+#ifdef DEBUG
+/* needs to be called before fragment_exit */
+void
+fcache_stats_exit()
+{
+    if (DYNAMO_OPTION(shared_bbs)) {
+        fcache_t *cache = shared_cache_bb;
+        /* cache may be NULL, for stats called after fcache_exit() */
+        if (cache != NULL) {
+            /* FIXME: report_dynamorio_problem() calls
+             * dump_global_stats() which currently regularly calls
+             * this, so any ASSERTs on this path will deadlock
+             * (workaround is to be vigilant and use msgbox_mask).
+             */
+            ASSERT_DO_NOT_OWN_MUTEX(cache->is_shared, &cache->lock);
+            PROTECT_CACHE(cache, lock);
+            fcache_cache_stats(GLOBAL_DCONTEXT, cache);
+            PROTECT_CACHE(cache, unlock);
+        }
+    }
+    if (DYNAMO_OPTION(shared_traces)) {
+        fcache_t *cache = shared_cache_trace;
+        if (cache != NULL) {
+            ASSERT_DO_NOT_OWN_MUTEX(cache->is_shared, &cache->lock);
+            PROTECT_CACHE(cache, lock);
+            fcache_cache_stats(GLOBAL_DCONTEXT, cache);
+            PROTECT_CACHE(cache, unlock);
+        }
+    }
+}
+#endif
+
+/* Free all thread-shared state not critical to forward progress;
+ * fcache_reset_init() will be called before continuing.
+ */
+static void
+fcache_reset_free(void)
+{
+    fcache_unit_t *u, *next_u;
+
+    /* case 7966: don't initialize at all for hotp_only & thin_client
+     * FIXME: could set initial sizes to 0 for all configurations, instead
+     */
+    if (RUNNING_WITHOUT_CODE_CACHE())
+        return;
+
+    /* FIXME: for reset (not exit), optimize to avoid calling
+     * fcache_really_free_unit() to move units onto dead list only to delete
+     * here: should directly delete, but maintain fcache stats.
+     */
+
+    /* we do not acquire each shared cache's lock for reset, assuming no
+     * synch issues (plus the lock will be deleted)
+     */
+    if (DYNAMO_OPTION(shared_bbs)) {
+        fcache_cache_free(GLOBAL_DCONTEXT, shared_cache_bb, true);
+        shared_cache_bb = NULL;
+    }
+    if (DYNAMO_OPTION(shared_traces)) {
+        fcache_cache_free(GLOBAL_DCONTEXT, shared_cache_trace, true);
+        shared_cache_trace = NULL;
+    }
+
+    /* there may be units stranded on the to-flush list.
+     * we must free the units here as they are unreachable elsewhere.
+     * their fragments will be freed by the fragment htable walk.
+     */
+    mutex_lock(&unit_flush_lock);
+    u = allunits->units_to_flush;
+    while (u != NULL) {
+        next_u = u->next_local;
+        LOG(GLOBAL, LOG_CACHE, 2,
+            "@ reset-free freeing to-be-flushed unit "PFX"-"PFX"\n",
+            u->start_pc, u->end_pc);
+        fcache_free_unit(GLOBAL_DCONTEXT, u, true);
+        u = next_u;
+    }
+    allunits->units_to_flush = NULL;
+    mutex_unlock(&unit_flush_lock);
+
+    /* should be freed via vm_area_check_shared_pending() */
+    ASSERT(allunits->units_to_free == NULL);
+
+    mutex_lock(&allunits_lock);
+    u = allunits->dead;
+    while (u != NULL) {
+        next_u = u->next_global;
+        fcache_really_free_unit(u, true/*on dead list*/, true/*dealloc*/);
+        u = next_u;
+    }
+    /* clear fields for reset_init() */
+    allunits->dead = NULL;
+    allunits->num_dead = 0;
+    mutex_unlock(&allunits_lock);
+}
+
+/* atexit cleanup -- needs no locks */
+void
+fcache_exit()
+{
+    fcache_unit_t *u, *next_u;
+
+    DOSTATS({
+        LOG(GLOBAL, LOG_TOP|LOG_THREADS, 1,
+            "fcache_exit: before fcache cleanup\n");
+        DOLOG(1, LOG_CACHE, fcache_stats_exit(););
+    });
+
+    fcache_reset_free();
+
+    /* free heap for all live units (reset did dead ones) */
+    mutex_lock(&allunits_lock);
+    u = allunits->units;
+    while (u != NULL) {
+        next_u = u->next_global;
+        fcache_really_free_unit(u, false/*live*/, true/*dealloc*/);
+        u = next_u;
+    }
+    mutex_unlock(&allunits_lock);
+
+    ASSERT(vmvector_empty(fcache_unit_areas));
+    vmvector_delete_vector(GLOBAL_DCONTEXT, fcache_unit_areas);
+
+    HEAP_TYPE_FREE(GLOBAL_DCONTEXT, allunits, fcache_list_t, ACCT_OTHER, PROTECTED);
+
+    DELETE_LOCK(allunits_lock);
+    DELETE_LOCK(reset_pending_lock);
+    DELETE_LOCK(unit_flush_lock);
+}
+
+#if defined(WINDOWS_PC_SAMPLE) && !defined(DEBUG)
+/* for fast exit path only, normal path taken care of in free unit*/
+void
+fcache_profile_exit()
+{
+    fcache_unit_t *u;
+    mutex_lock(&allunits_lock); 
+    for (u = allunits->units; u != NULL; u = u->next_global) {
+        if (u->profile) {
+            fcache_unit_profile_stop(u);
+            free_profile(u->profile);
+            u->profile = NULL;
+        }
+    }
+    mutex_unlock(&allunits_lock);
+}
+#endif
+
+static fcache_unit_t *
+fcache_lookup_unit(cache_pc pc)
+{
+    /* let's see if this becomes frequent enough to be a perf hit */
+    STATS_INC(fcache_unit_lookups);
+    return (fcache_unit_t *) vmvector_lookup(fcache_unit_areas, pc);
+}
+
+/* Returns the fragment_t whose body (not cache slot) contains lookup_pc */  
+fragment_t *
+fcache_fragment_pclookup(dcontext_t *dcontext, cache_pc lookup_pc, fragment_t *wrapper)
+{
+    fragment_t *f, *found = NULL;
+    cache_pc pc;
+    fcache_unit_t *unit = fcache_lookup_unit(lookup_pc);
+    if (unit == NULL)
+        return NULL;
+    LOG(THREAD, LOG_CACHE, 5, "fcache_fragment_pclookup "PFX" -> "PFX"-"PFX"\n",
+        lookup_pc, unit->start_pc, unit->end_pc);
+    if (unit->cache->is_coarse) {
+        /* No metadata in cache so we must walk the htable.  We shouldn't need
+         * to lock the cache itself.
+         */
+        coarse_info_t *info = unit->cache->coarse_info;
+        cache_pc body;
+        app_pc tag;
+        ASSERT(info != NULL);
+        tag = fragment_coarse_pclookup(dcontext, info, lookup_pc, &body);
+        ASSERT(wrapper != NULL);
+        fragment_coarse_wrapper(wrapper, tag, body);
+        return wrapper;
+    }
+    PROTECT_CACHE(unit->cache, lock);
+    DODEBUG({
+        if (!unit->cache->consistent) {
+            /* We're in the middle of an fcache operation during which we cannot
+             * physically walk the cache.  ASSUMPTION: this only happens for
+             * debug builds when we pclookup on disassembly.
+             */
+            PROTECT_CACHE(unit->cache, unlock);
+            return fragment_pclookup_by_htable(dcontext, lookup_pc, wrapper);
+        }
+    });
+    pc = unit->start_pc;
+    while (pc < unit->cur_pc && pc < lookup_pc) {
+        f = *((fragment_t **)pc);
+        LOG(THREAD, LOG_CACHE, 6, "\treading "PFX" -> "PFX"\n", pc, f);
+        if (!USE_FIFO_FOR_CACHE(unit->cache)) {
+            if (FRAG_IS_FREE_LIST(f)) {
+                pc += ((free_list_header_t *) pc)->size;
+                continue;
+            }
+        }
+        ASSERT(f != NULL);
+        ASSERT(FIFO_UNIT(f) == unit);
+        ASSERT(FRAG_HDR_START(f) == pc);
+        if (!FRAG_EMPTY(f) &&
+            lookup_pc < (f->start_pc + f->size) && lookup_pc >= f->start_pc) {
+            found = f;
+            LOG(THREAD, LOG_CACHE, 5, "\tfound F%d ("PFX")."PFX"\n",
+                f->id, f->tag, f->start_pc);
+            break;
+        }
+        /* advance to contiguously-next fragment_t in cache */
+        pc += FRAG_SIZE(f);
+    }
+    PROTECT_CACHE(unit->cache, unlock);
+    return found;
+}
+
+#ifdef DEBUG
+static bool
+fcache_pc_in_live_unit(fcache_t *cache, cache_pc pc)
+{
+    fcache_unit_t *unit;
+    for (unit = cache->units; unit != NULL; unit = unit->next_local) {
+        if (pc >= unit->start_pc && pc < unit->end_pc)
+            return true;
+    }
+    unit = fcache_lookup_unit(pc);
+    /* pending flush is still considered live: removed from all lists
+     * just prior to flush synch
+     */
+    if (unit != NULL && unit->pending_flush)
+        return true;
+    return false;
+}
+#endif
+
+bool
+fcache_is_writable(fragment_t *f)
+{
+    fcache_unit_t *unit = fcache_lookup_unit(f->start_pc);
+    ASSERT(unit != NULL);
+    return unit->writable;
+}
+
+/* If f is null, changes protection of entire fcache
+ * Else, does the unit f is part of
+ */
+void 
+fcache_change_fragment_protection(dcontext_t *dcontext, fragment_t *f, bool writable)
+{
+    fcache_unit_t *u;
+    ASSERT(TEST(SELFPROT_CACHE, dynamo_options.protect_mask));
+    if (f != NULL) {
+        u = fcache_lookup_unit(f->start_pc);
+        ASSERT(u != NULL);
+        if (u->writable == writable)
+            return;
+        change_protection((void*)u->start_pc, u->size, writable);
+        u->writable = writable;
+    } else {
+        /* else, do entire fcache
+         * win32 does not allow single protection change call on units that
+         * were allocated with separate calls so we don't try to combine
+         * adjacent units here
+         */
+        /* FIXME: right now no synch here, so one thread could unprot, another prots,
+         * and the first segfaults
+         */
+        mutex_lock(&allunits_lock);
+        u = allunits->units;
+        while (u != NULL) {
+            if (u->writable != writable) {
+                change_protection((void*)u->start_pc, u->size, writable);
+                u->writable = writable;
+            }
+            u = u->next_global;
+        }
+        mutex_unlock(&allunits_lock);
+    }
+}
+
+
+/* return true if pc is in the fcache address space */
+/* This routine can be called with a thread suspended in an unknown state.
+ * Currently only the fcache_unit_areas write lock is checked, so if
+ * this routine is changed
+ * to grab any other locks, or call a routine that does, then the 
+ * at_safe_spot() routine in os.c must be updated */
+bool
+in_fcache(void * pc)
+{
+    return fcache_lookup_unit((cache_pc)pc) != NULL;
+}
+
+
+/* Pass NULL for pc if this routine should allocate the cache space.
+ * If pc is non-NULL, this routine assumes that size is fully
+ * committed and initializes accordingly.
+ */
+static fcache_unit_t *
+fcache_create_unit(dcontext_t *dcontext, fcache_t *cache, cache_pc pc, size_t size)
+{
+    fcache_unit_t *u = NULL;
+    uint cache_type;
+    ASSERT(CACHE_PROTECTED(cache));
+
+    /* currently we assume for FIFO empties that we can't have a single
+     * contiguous empty slot that overflows a uint.
+     * minor: all our stats only take ints, as well
+     */
+    ASSERT(CHECK_TRUNCATE_TYPE_uint(size));
+    ASSERT(ALIGNED(size, PAGE_SIZE));
+
+    if (pc == NULL) {
+        /* take from dead list if possible */
+        mutex_lock(&allunits_lock);
+        if (allunits->dead != NULL) {
+            fcache_unit_t *prev_u = NULL;
+            u = allunits->dead;
+            while (u != NULL) {
+                if (u->size >= size &&
+                    (cache->max_size == 0 || cache->size + u->size <= cache->max_size)) {
+                    /* remove from dead list */
+                    if (prev_u == NULL)
+                        allunits->dead = u->next_global;
+                    else
+                        prev_u->next_global = u->next_global;
+                    LOG(THREAD, LOG_CACHE, 1,
+                        "\tFound unit "PFX" of size %d (need %d) on dead list\n",
+                        u->start_pc, u->size/1024, size/1024);
+                    allunits->num_dead--;
+                    RSTATS_DEC(fcache_num_free);
+                    STATS_SUB(fcache_free_capacity, u->size);
+#ifdef WINDOWS_PC_SAMPLE
+                    if (u->profile) {
+                        reset_profile(u->profile);
+                        start_profile(u->profile);
+                    }
+#endif
+                    break;
+                }
+                prev_u = u;
+                u = u->next_global;
+            }
+        }
+        mutex_unlock(&allunits_lock);
+    }
+
+    if (u == NULL) {
+        size_t commit_size;
+        /* use global heap b/c this can be re-used by later threads */
+        u = (fcache_unit_t *)
+            nonpersistent_heap_alloc(GLOBAL_DCONTEXT, sizeof(fcache_unit_t)
+                                     HEAPACCT(ACCT_MEM_MGT));
+        if (pc != NULL) {
+            u->start_pc = pc;
+            commit_size = size;
+            STATS_FCACHE_ADD(cache, claimed, size);
+            STATS_ADD(fcache_combined_claimed, size);
+        } else {
+            /* allocate new unit */
+            commit_size = DYNAMO_OPTION(cache_commit_increment);
+            ASSERT(commit_size <= size);
+            u->start_pc = (cache_pc) heap_mmap_reserve(size, commit_size);
+        }
+        ASSERT(u->start_pc != NULL);
+        ASSERT(proc_is_cache_aligned((void *)u->start_pc));
+        LOG(THREAD, LOG_HEAP, 3, "fcache_create_unit -> "PFX"\n", u->start_pc);
+        u->size = commit_size;
+        u->end_pc = u->start_pc + commit_size;
+        u->reserved_end_pc = u->start_pc + size;
+        vmvector_add(fcache_unit_areas, u->start_pc, u->reserved_end_pc, (void *) u);
+        STATS_ADD(fcache_combined_capacity, u->size);
+        STATS_MAX(peak_fcache_combined_capacity, fcache_combined_capacity);
+        
+#ifdef WINDOWS_PC_SAMPLE
+        if (dynamo_options.profile_pcs && 
+            dynamo_options.prof_pcs_fcache >= 2 && 
+            dynamo_options.prof_pcs_fcache <= 32) {
+            u->profile = create_profile(u->start_pc, u->reserved_end_pc, 
+                                        dynamo_options.prof_pcs_fcache, NULL);
+            start_profile(u->profile);
+        } else 
+            u->profile = NULL;
+#endif
+    }
+
+    cache->size += u->size;
+
+    u->cur_pc = u->start_pc;
+    u->full = false;
+    u->cache = cache;
+#if defined(SIDELINE) || defined(WINDOWS_PC_SAMPLE)
+    u->dcontext = dcontext;
+#endif
+    u->writable = true;
+    u->pending_free = false;
+    DODEBUG({ u->pending_flush = false; });
+    u->flushtime = 0;
+
+    RSTATS_ADD_PEAK(fcache_num_live, 1);
+    STATS_FCACHE_ADD(u->cache, capacity, u->size);
+    STATS_FCACHE_MAX(u->cache, capacity_peak, capacity);
+
+    u->next_local = NULL; /* must be set by caller */
+    mutex_lock(&allunits_lock);
+
+    if (allunits->units != NULL)
+        allunits->units->prev_global = u;
+    u->next_global = allunits->units;
+    u->prev_global = NULL;
+    allunits->units = u;
+
+    cache_type = cache->is_trace ? CACHE_TRACE : CACHE_BB;
+    if (reset_at_nth_unit[cache_type] > 0) {
+        /* reset on nth NEW unit (ignoring total unit count, whether
+         * some were flushed, etc.)
+         */
+        reset_at_nth_unit[cache_type]--;
+        if (reset_at_nth_unit[cache_type] == 0) {
+            schedule_reset(RESET_ALL);
+            if (reset_every_nth_unit[cache_type] > 0)
+                reset_at_nth_unit[cache_type] = reset_every_nth_unit[cache_type];
+        }
+    }
+
+    mutex_unlock(&allunits_lock);
+
+    return u;
+}
+
+/* this routine does NOT remove the unit from its local cache list */
+static void 
+fcache_free_unit(dcontext_t *dcontext, fcache_unit_t *unit, bool dealloc_or_reuse)
+{
+    DODEBUG({
+        if (unit->flushtime > 0) {
+            ASSERT_OWN_MUTEX(true, &unit_flush_lock);
+            /* we set to 0 to avoid this assert on fcache_reset_exit()
+             * when freeing dead units -- not needed for release build
+             */
+            unit->flushtime = 0;
+        } else {
+            ASSERT(dynamo_exited || dynamo_resetting || CACHE_PROTECTED(unit->cache));
+        }
+    });
+    mutex_lock(&allunits_lock);
+    /* remove from live list */
+    if (unit->prev_global != NULL)
+        unit->prev_global->next_global = unit->next_global;
+    else
+        allunits->units = unit->next_global;
+    if (unit->next_global != NULL)
+        unit->next_global->prev_global = unit->prev_global;
+    STATS_FCACHE_SUB(unit->cache, claimed, (unit->cur_pc - unit->start_pc));
+    STATS_FCACHE_SUB(unit->cache, empty, (unit->cur_pc - unit->start_pc));
+    STATS_SUB(fcache_combined_claimed, (unit->cur_pc - unit->start_pc));
+
+    if (!dealloc_or_reuse) {
+        /* up to caller to dealloc */
+        mutex_unlock(&allunits_lock);
+        /* we do want to update cache->size and fcache_unit_areas: */
+        fcache_really_free_unit(unit, false/*live*/, false/*do not dealloc unit*/);
+    }
+    /* heuristic: don't keep around more dead units than max(5, 1/4 num threads) */
+    else if (allunits->num_dead < 5 ||
+             allunits->num_dead * 4U <= (uint) get_num_threads()) {
+        /* Keep dead list sorted small-to-large to avoid grabbing large
+         * when can take small and then needing to allocate when only
+         * have small left.  Helps out with lots of small threads.
+         */
+        fcache_unit_t *u, *prev_u;
+        for (u = allunits->dead, prev_u = NULL;
+             u != NULL && u->size < unit->size;
+             prev_u = u, u = u->next_global)
+            ;
+        /* prev_global and next_local are not used in the dead list */
+        unit->prev_global = NULL;
+        unit->next_local = NULL;
+        if (prev_u == NULL) {
+            unit->next_global = allunits->dead;
+            allunits->dead = unit;
+        } else {
+            unit->next_global = u;
+            prev_u->next_global = unit;
+        }
+        allunits->num_dead++;
+        RSTATS_ADD_PEAK(fcache_num_free, 1);
+        STATS_ADD(fcache_free_capacity, unit->size);
+#ifdef WINDOWS_PC_SAMPLE
+        if (unit->profile)
+            fcache_unit_profile_stop(unit);
+#endif
+        /* this is done by fcache_really_free_unit for else path */
+        remove_unit_from_cache(unit);
+        mutex_unlock(&allunits_lock);
+    } else {
+        mutex_unlock(&allunits_lock);
+        fcache_really_free_unit(unit, false/*live*/, true/*dealloc*/);
+    }
+}
+
+/* assuming size will either be aligned at VM_ALLOCATION_BOUNDARY or
+ * smaller where no adjustment is necessary 
+ */
+#define FCACHE_GUARDED(size)                                    \
+        ((size) -                                               \
+         ((DYNAMO_OPTION(guard_pages) &&                        \
+           ((size) >= VM_ALLOCATION_BOUNDARY - 2 * PAGE_SIZE))  \
+          ? (2 * PAGE_SIZE) : 0))
+
+#define SET_CACHE_PARAMS(cache, which) do {                             \
+    cache->max_size =                                                   \
+        FCACHE_GUARDED(FCACHE_OPTION(cache_##which##_max));             \
+    cache->max_unit_size =                                              \
+        FCACHE_GUARDED(FCACHE_OPTION(cache_##which##_unit_max));        \
+    cache->max_quadrupled_unit_size =                                   \
+        FCACHE_GUARDED(FCACHE_OPTION(cache_##which##_unit_quadruple));  \
+    cache->free_upgrade_size =                                          \
+        FCACHE_GUARDED(FCACHE_OPTION(cache_##which##_unit_upgrade));    \
+    cache->init_unit_size =                                             \
+        FCACHE_GUARDED(FCACHE_OPTION(cache_##which##_unit_init));       \
+    cache->finite_cache = dynamo_options.finite_##which##_cache;        \
+    cache->regen_param = dynamo_options.cache_##which##_regen;          \
+    cache->replace_param = dynamo_options.cache_##which##_replace;      \
+} while (0);
+
+static fcache_t *
+fcache_cache_init(dcontext_t *dcontext, uint flags, bool initial_unit)
+{
+    fcache_t *cache = (fcache_t *)
+        nonpersistent_heap_alloc(dcontext, sizeof(fcache_t) HEAPACCT(ACCT_MEM_MGT));
+    cache->fifo = NULL;
+    cache->size = 0;
+    cache->is_trace = TEST(FRAG_IS_TRACE, flags);
+    cache->is_shared = TEST(FRAG_SHARED, flags);
+    cache->is_coarse = TEST(FRAG_COARSE_GRAIN, flags);
+    DODEBUG({ cache->is_local = false; });
+    cache->coarse_info = NULL;
+    DODEBUG({ cache->consistent = true; });
+    if (cache->is_shared) {
+        ASSERT(dcontext == GLOBAL_DCONTEXT);
+        if (TEST(FRAG_IS_TRACE, flags)) {
+            DODEBUG({ cache->name = "Trace (shared)"; });
+            SET_CACHE_PARAMS(cache, shared_trace);
+        } else if (cache->is_coarse) {
+            DODEBUG({ cache->name = "Coarse basic block (shared)"; });
+            SET_CACHE_PARAMS(cache, coarse_bb);
+        } else {
+            DODEBUG({ cache->name = "Basic block (shared)"; });
+            SET_CACHE_PARAMS(cache, shared_bb);
+        }
+    } else {
+        ASSERT(dcontext != GLOBAL_DCONTEXT);
+        if (TEST(FRAG_IS_TRACE, flags)) {
+            DODEBUG({ cache->name = "Trace (private)"; });
+            SET_CACHE_PARAMS(cache, trace);
+        } else {
+            DODEBUG({ cache->name = "Basic block (private)"; });
+            SET_CACHE_PARAMS(cache, bb);
+        }
+    }
+#ifdef DISALLOW_CACHE_RESIZING
+    /* cannot handle resizing of cache, separate units only */
+    cache->init_unit_size = cache->max_unit_size;
+#endif
+    if (cache->is_shared)
+        ASSIGN_INIT_LOCK_FREE(cache->lock, shared_cache_lock);
+    if (initial_unit) {
+        PROTECT_CACHE(cache, lock);
+        cache->units = fcache_create_unit(dcontext, cache, NULL, cache->init_unit_size);
+        PROTECT_CACHE(cache, unlock);
+    } else
+        cache->units = NULL;
+    cache->num_regenerated = 0;
+    cache->num_replaced = 0;
+    cache->wset_check = 0;
+    cache->record_wset = false;
+    if (cache->is_shared) { /* else won't use free list */
+        memset(cache->free_list, 0, sizeof(cache->free_list));
+        DODEBUG({
+            memset(cache->free_stats_freed, 0, sizeof(cache->free_stats_freed));
+            memset(cache->free_stats_reused, 0, sizeof(cache->free_stats_reused));
+            memset(cache->free_stats_coalesced, 0, sizeof(cache->free_stats_coalesced));
+            memset(cache->free_stats_charge, 0, sizeof(cache->free_stats_charge));
+            memset(cache->free_stats_split, 0, sizeof(cache->free_stats_split));
+            memset(cache->request_size_histogram, 0, 
+                   sizeof(cache->request_size_histogram));
+            memset(cache->free_size_histogram, 0, 
+                   sizeof(cache->free_size_histogram));
+        });
+    }
+    return cache;
+}
+
+/* assumption: only called on thread exit path.
+ * If !free_units, we do not de-allocate or move the units to the dead list,
+ * but we still remove from the live list.
+ */
+static void 
+fcache_cache_free(dcontext_t *dcontext, fcache_t *cache, bool free_units)
+{
+    fcache_unit_t *u, *next_u;
+    dcontext_t *alloc_dc = ALLOC_DC(dcontext, cache);
+    DEBUG_DECLARE(size_t cache_size = cache->size;)
+    DEBUG_DECLARE(size_t size_check = 0;)
+
+    if (USE_FIFO_FOR_CACHE(cache)) {
+        fragment_t *f, *nextf;
+        ASSERT(cache->consistent);
+        for (f = cache->fifo; f != NULL; f = nextf) {
+            /* fragment exit may have already happened, but we won't deref
+             * freed memory here since fragments will have been removed from FIFO
+             */
+            nextf = FIFO_NEXT(f);
+            if (FRAG_EMPTY(f)) {
+                nonpersistent_heap_free(alloc_dc, f, sizeof(empty_slot_t)
+                                        HEAPACCT(ACCT_FCACHE_EMPTY));
+            }
+        }
+        cache->fifo = NULL;
+    }
+    ASSERT(cache->fifo == NULL);
+
+    u = cache->units;
+    while (u != NULL) {
+        DODEBUG({ size_check += u->size; });
+        next_u = u->next_local;
+        fcache_free_unit(dcontext, u, free_units);
+        u = next_u;
+    }
+    /* we must use pre-cached cache_size since fcache_free_unit decrements it */
+    ASSERT(size_check == cache_size);
+    ASSERT(cache->size == 0);
+
+    if (cache->is_shared)
+        DELETE_LOCK(cache->lock);
+
+    nonpersistent_heap_free(alloc_dc, cache, sizeof(fcache_t) HEAPACCT(ACCT_MEM_MGT));
+}
+
+#ifdef DEBUG
+void 
+fcache_free_list_consistency(dcontext_t *dcontext, fcache_t *cache, int bucket)
+{
+    uint live = 0;
+    uint charge = 0;
+    uint waste = 0;
+    free_list_header_t *header = cache->free_list[bucket];
+    uint prev_size = 0;
+    fcache_unit_t *unit;
+    
+    LOG(GLOBAL, LOG_CACHE, 3, "fcache_free_list_consistency %s bucket[%2d], %3d bytes\n", 
+        cache->name, bucket, FREE_LIST_SIZES[bucket]);
+    /* walk list, and verify counters */
+    while (header != NULL) {
+        cache_pc start_pc = (cache_pc) header;
+        uint size = header->size;
+
+        ASSERT(size >= FREE_LIST_SIZES[bucket] &&
+               size <= MAX_FREE_ENTRY_SIZE &&
+               (bucket == FREE_LIST_SIZES_NUM-1 || size < FREE_LIST_SIZES[bucket+1]));
+
+        /* FIXME: should ASSERT entries in a bucket are all sorted
+         * properly, when we start keeping them in order */
+        ASSERT(prev_size < size || true /* not sorted yet */);
+        prev_size = size;
+        ASSERT(header->next == NULL || header->next->prev == header);
+        ASSERT(header->prev == NULL || header->prev->next == header);
+        ASSERT(FRAG_IS_FREE_LIST((fragment_t *)header));
+
+        unit = fcache_lookup_unit(start_pc);
+        if (unit->cur_pc > start_pc + header->size) {
+            fragment_t *subseq = FRAG_NEXT_SLOT(start_pc, header->size);
+            /* should NOT be followed by a free list entry; should instead be
+             * followed by a marked fragment_t.
+             */
+            ASSERT((!FRAG_IS_FREE_LIST(subseq) &&
+                    TEST(FRAG_FOLLOWS_FREE_ENTRY, subseq->flags)) ||
+                    /* ok to have subsequent free entry if unable to coalesce due
+                     * to ushort size limits */
+                    size + FRAG_NEXT_FREE(start_pc, header->size)->size >
+                    MAX_FREE_ENTRY_SIZE);
+        }
+        /* Invariant: no free list entry at append point */
+        ASSERT(unit->full || unit->cur_pc != start_pc + header->size);
+
+        header = header->next;
+
+        /* maximum waste if this entry is used.  The scheme before
+         * case 7318 was really wasting memory, for comparison here */
+        LOG(GLOBAL, LOG_CACHE, 4, "\t  @"PFX": %3d bytes, %3d max waste\n",
+            start_pc, size, size - FREE_LIST_SIZES[bucket]);
+        live++;
+        charge += size;
+        waste += size - FREE_LIST_SIZES[bucket];
+    }
+
+    ASSERT(live == cache->free_stats_freed[bucket] -
+           (cache->free_stats_reused[bucket] + cache->free_stats_coalesced[bucket]));
+    ASSERT(charge == cache->free_stats_charge[bucket]);
+    ASSERT(waste >= (
+                     /* waste estimate = 
+                        charged bytes - live entries * bucket _minimal_ size */
+                     cache->free_stats_charge[bucket] - 
+                     (cache->free_stats_freed[bucket] - 
+                      (cache->free_stats_reused[bucket] +
+                       cache->free_stats_coalesced[bucket])) * 
+                     FREE_LIST_SIZES[bucket]));
+    LOG(GLOBAL, LOG_CACHE, 2, "\t#%2d %3d bytes == %d live, %8d charge, %8d waste\n",
+        bucket, 
+        FREE_LIST_SIZES[bucket],
+        live,
+        charge, 
+        waste);
+}
+
+/* FIXME: put w/ periodic stats dumps and not only at end? */
+static void 
+fcache_cache_stats(dcontext_t *dcontext, fcache_t *cache)
+{
+    fcache_unit_t *u;
+    int i = 0;
+    size_t capacity = 0;
+    size_t used = 0;
+    bool full = true;
+    ASSERT_OWN_MUTEX(!dynamo_exited && cache->is_shared, &cache->lock);
+    for (u = cache->units; u != NULL; u = u->next_local, i++) {
+        capacity += u->size;
+        used += u->cur_pc - u->start_pc;
+        full &= u->full;
+        LOG(THREAD, LOG_CACHE, 1,
+            "\t%s unit %d @"PFX": capacity %d KB, used %d KB, %s\n",
+            cache->name, i, u->start_pc, u->size/1024, (u->cur_pc - u->start_pc)/1024,
+            u->full ? "full" : "not full");
+    }
+    LOG(THREAD, LOG_CACHE, 1, "%s cache: capacity %d KB, used %d KB, %s\n",
+        cache->name, capacity/1024, used/1024, full ? "full" : "not full");
+    if (DYNAMO_OPTION(cache_shared_free_list) && 
+        cache->is_shared) { /* using free list */
+        int bucket;
+        LOG(GLOBAL, LOG_CACHE, 1, "fcache %s free list stats:\n", cache->name);
+        for (bucket = 0; bucket < FREE_LIST_SIZES_NUM; bucket++) {
+            LOG(GLOBAL, LOG_ALL, 1, 
+                "\t#%2d %3d bytes : %7d free, %7d reuse, %5d coalesce, %5d split\n"
+                "\t    %3d bytes : %5d live, %8d charge, %8d waste\n",
+                bucket, 
+                FREE_LIST_SIZES[bucket],
+                cache->free_stats_freed[bucket],
+                cache->free_stats_reused[bucket],
+                cache->free_stats_coalesced[bucket],
+                cache->free_stats_split[bucket],
+                FREE_LIST_SIZES[bucket],                
+                cache->free_stats_freed[bucket] -
+                (cache->free_stats_reused[bucket] + cache->free_stats_coalesced[bucket]),
+                cache->free_stats_charge[bucket],
+                /* waste = charged bytes - live entries * bucket _minimal_ size */
+                cache->free_stats_charge[bucket] - 
+                (cache->free_stats_freed[bucket] -
+                 (cache->free_stats_reused[bucket] +
+                  cache->free_stats_coalesced[bucket])) * 
+                FREE_LIST_SIZES[bucket]);
+        }
+
+        DOLOG(1, LOG_CACHE, {
+            /* FIXME: add in all debug runs, if not too slow */
+            for (bucket = 0; bucket < FREE_LIST_SIZES_NUM; bucket++) {
+                fcache_free_list_consistency(dcontext, cache, bucket);
+            }
+        });
+
+        LOG(GLOBAL, LOG_ALL, 1, "fcache %s requests and frees histogram:\n", cache->name);
+        for (bucket = 0; 
+             bucket < sizeof(cache->request_size_histogram)/
+                 sizeof(cache->request_size_histogram[0]); 
+             bucket++) {
+            if (cache->request_size_histogram[bucket] != 0 ||
+                cache->free_size_histogram[bucket] != 0) {
+                LOG(GLOBAL, LOG_ALL, 1, "\t# %3d bytes == %8d requests   %8d freed\n",
+                    bucket*HISTOGRAM_GRANULARITY, 
+                    cache->request_size_histogram[bucket],
+                    cache->free_size_histogram[bucket]);
+            }
+        }
+    }
+}
+
+static inline
+int
+get_histogram_bucket(int size)
+{
+    uint histogram_bucket = size/HISTOGRAM_GRANULARITY;
+    if (histogram_bucket >= HISTOGRAM_MAX_SIZE/HISTOGRAM_GRANULARITY)
+        histogram_bucket = (HISTOGRAM_MAX_SIZE/HISTOGRAM_GRANULARITY) - 1;
+    return histogram_bucket;
+}
+
+#endif /* DEBUG */
+
+static void
+fcache_shift_fragments(dcontext_t *dcontext, fcache_unit_t *unit, ssize_t shift,
+                       cache_pc start, cache_pc end, size_t old_size)
+{
+    fragment_t *f;
+    cache_pc pc, new_pc;
+
+    ASSERT_OWN_MUTEX(unit->cache->is_shared, &unit->cache->lock);
+
+    /* free list scheme can't be walked expecting FIFO headers */
+    ASSERT(!DYNAMO_OPTION(cache_shared_free_list) || !unit->cache->is_shared);
+
+    /* would need to re-relativize fcache exit prefix for coarse */
+    ASSERT(!unit->cache->is_coarse);
+
+    DODEBUG({ unit->cache->consistent = false; });
+    LOG(THREAD, LOG_CACHE, 2, "fcache_shift_fragments: first pass\n");
+    /* walk the physical cache and shift each fragment 
+     * fine to walk the old memory, we just need the fragment_t* pointers
+     */
+    pc = unit->start_pc;
+    LOG(THREAD, LOG_CACHE, 2, "  unit "PFX"-"PFX" [-"PFX"]\n",
+        pc, unit->end_pc, unit->reserved_end_pc);
+    while (pc < unit->cur_pc) {
+        f = *((fragment_t **)pc);
+        ASSERT(f != NULL);
+        ASSERT(FIFO_UNIT(f) == unit); /* sanity check */
+        if (FRAG_EMPTY(f))
+            FRAG_START_ASSIGN(f, FRAG_START(f) + shift);
+        else {
+            LOG(THREAD, LOG_CACHE, 5, "\treading "PFX" -> "PFX" = F%d\n", pc, f, f->id);
+            fragment_shift_fcache_pointers(dcontext, f, shift, start, end, old_size);
+        }
+        /* now that f->start_pc is updated, update the backpointer */
+        new_pc = FRAG_HDR_START(f);
+        LOG(THREAD, LOG_CACHE, 4, "resize: writing "PFX" to "PFX"\n", f, new_pc);
+        *((fragment_t **)new_pc) = f;
+        /* move to contiguously-next fragment_t in cache */
+        pc += FRAG_SIZE(f);
+    }
+
+    DOLOG(2, LOG_FRAGMENT, {
+        /* need to check for consistency all tables at this point */
+        study_all_hashtables(dcontext);
+    });
+
+
+    LOG(THREAD, LOG_CACHE, 2, "fcache_shift_fragments: second pass\n");
+    /* have to do a second pass to link them to each other */
+    pc = unit->start_pc;
+    LOG(THREAD, LOG_CACHE, 2, "  unit "PFX"-"PFX" [-"PFX"]\n",
+        pc, unit->end_pc, unit->reserved_end_pc);
+    while (pc < unit->cur_pc) {
+        f = *((fragment_t **)pc);
+        ASSERT(f != NULL);
+        /* can't repeat the FIFO_UNIT(f)==unit check b/c we've already adjusted
+         * f->start_pc, which is used to find the unit
+         */
+        if (!FRAG_EMPTY(f)) {
+            LOG(THREAD, LOG_CACHE, 5, "\treading "PFX" -> "PFX" = F%d\n", pc, f, f->id);
+        
+            /* inter-cache links must be redone:
+             * we have links from bb cache to trace cache, and sometimes links
+             * the other direction, for example from a trace to a bb that
+             * cannot be a trace head (e.g., is marked CANNOT_BE_TRACE).
+             * simplest to re-link every fragment in the shifted cache.
+             * N.B.: we do NOT need to re-link the outgoing, since
+             * fragment_shift_fcache_pointers re-relativized all outgoing
+             * ctis by the shifted amount.
+             */
+            if (TEST(FRAG_LINKED_INCOMING, f->flags)) {
+                unlink_fragment_incoming(dcontext, f);
+                link_fragment_incoming(dcontext, f, false/*not new*/);
+            }
+        }
+        
+        /* move to contiguously-next fragment_t in cache */
+        pc += FRAG_SIZE(f);
+    }
+    DODEBUG({ unit->cache->consistent = true; });
+}
+
+static void
+cache_extend_commitment(fcache_unit_t *unit, size_t commit_size)
+{
+    ASSERT(unit != NULL);
+    ASSERT(ALIGNED(commit_size, DYNAMO_OPTION(cache_commit_increment)));
+    heap_mmap_extend_commitment(unit->end_pc, commit_size);
+    unit->end_pc += commit_size;
+    unit->size += commit_size;
+    unit->cache->size += commit_size;
+    unit->full = false;
+    STATS_FCACHE_ADD(unit->cache, capacity, commit_size);
+    STATS_FCACHE_MAX(unit->cache, capacity_peak, capacity);
+    STATS_ADD(fcache_combined_capacity, commit_size);
+    ASSERT(unit->end_pc <= unit->reserved_end_pc);
+    ASSERT(unit->size <= UNIT_RESERVED_SIZE(unit));
+}
+
+/* FIXME case 8617: now that we have cache commit-on-demand we should
+ * make the private-configuration caches larger.  We could even get
+ * rid of the fcache shifting.
+ */
+static void
+fcache_increase_size(dcontext_t *dcontext, fcache_t *cache, fcache_unit_t *unit,
+                     uint slot_size)
+{
+    bool reallocated = false;
+    cache_pc new_memory = NULL;
+    ssize_t shift;
+    size_t new_size = unit->size;
+    size_t commit_size;
+    /* we shouldn't come here if we have reservation room */
+    ASSERT(unit->reserved_end_pc == unit->end_pc);
+    if (new_size*4 <= cache->max_quadrupled_unit_size)
+        new_size *= 4;
+    else
+        new_size *= 2;
+    if (new_size < slot_size * MAX_SINGLE_MULTIPLE)
+        new_size = ALIGN_FORWARD(slot_size * MAX_SINGLE_MULTIPLE, PAGE_SIZE);
+    /* unit limit */
+    if (new_size > cache->max_unit_size)
+        new_size = cache->max_unit_size;
+    /* total cache limit */
+    if (cache->max_size != 0 && cache->size - unit->size + new_size > cache->max_size) {
+        new_size = cache->max_size - cache->size + unit->size;
+    }
+    commit_size = new_size; /* should be re-set below, this makes compiler happy */
+    /* FIXME: shouldn't this routine return whether it
+     * allocated enough space for slot_size?
+     */
+    ASSERT(unit->size + slot_size <= new_size);
+    LOG(THREAD, LOG_CACHE, 2, "Increasing %s unit size from %d KB to %d KB\n",
+        cache->name, unit->size/1024, new_size/1024);
+#ifdef DISALLOW_CACHE_RESIZING
+    SYSLOG_INTERNAL_ERROR("This build cannot handle cache resizing");
+    ASSERT_NOT_REACHED();
+#endif
+    ASSERT(CACHE_PROTECTED(cache));
+
+    /* take from dead list if possible */
+    if (allunits->dead != NULL) {
+        fcache_unit_t *u, *prev_u;
+        mutex_lock(&allunits_lock);
+        u = allunits->dead;
+        prev_u = NULL;
+        while (u != NULL) {
+            if (UNIT_RESERVED_SIZE(u) >= new_size) {
+                thread_units_t *tu = (thread_units_t *) dcontext->fcache_field;
+                /* remove from dead list */
+                if (prev_u == NULL)
+                    allunits->dead = u->next_global;
+                else
+                    prev_u->next_global = u->next_global;
+                allunits->num_dead--;
+                RSTATS_DEC(fcache_num_free);
+                STATS_SUB(fcache_free_capacity, u->size);
+                LOG(THREAD, LOG_CACHE, 1, "\tFound unit of size %d on dead list\n",
+                    u->size/1024);
+                new_memory = u->start_pc;
+                ASSERT_TRUNCATE(new_size, uint, UNIT_RESERVED_SIZE(u));
+                new_size = UNIT_RESERVED_SIZE(u);
+                u->cache = cache;
+                /* Add to stats prior to extending commit as that will add the
+                 * extension amount itself.
+                 * Don't need to add to combined capacity: it includes free.
+                 */
+                STATS_FCACHE_ADD(cache, capacity, u->size);
+                STATS_FCACHE_MAX(cache, capacity_peak, capacity);
+                if (u->size < new_size) { /* case 8688: fill out to promised size */
+                    size_t new_commit =
+                        ALIGN_FORWARD(new_size - u->size,
+                                      DYNAMO_OPTION(cache_commit_increment));
+                    ASSERT(u->size + new_commit <= UNIT_RESERVED_SIZE(u));
+                    cache_extend_commitment(u, new_commit);
+                    /* we increase cache's size below so undo what
+                     * cache_extend_commitment did */
+                    u->cache->size -= new_commit;
+                    ASSERT(u->size >= new_size);
+                }
+                commit_size = u->size;
+                /* use unit's fcache_unit_t struct but u's mmap space */
+                ASSERT(tu->pending_unmap_pc == NULL);
+                tu->pending_unmap_pc = unit->start_pc;
+                tu->pending_unmap_size = UNIT_RESERVED_SIZE(unit);
+                STATS_FCACHE_SUB(cache, capacity, unit->size);
+                STATS_SUB(fcache_combined_capacity, unit->size);
+#ifdef WINDOWS_PC_SAMPLE
+                if (u->profile != NULL) {
+                    free_profile(u->profile);
+                    u->profile = NULL;
+                }
+#endif
+                nonpersistent_heap_free(GLOBAL_DCONTEXT, u, sizeof(fcache_unit_t)
+                                        HEAPACCT(ACCT_MEM_MGT));
+                break;
+            }
+            prev_u = u;
+            u = u->next_global;
+        }
+        mutex_unlock(&allunits_lock);
+    } 
+    if (new_memory == NULL) {
+        /* allocate new memory for unit */
+        reallocated = true;
+        commit_size = 0;
+        while (commit_size < slot_size && unit->size + commit_size < new_size) {
+            commit_size += DYNAMO_OPTION(cache_commit_increment);
+        }
+        /* FIXME: If not we have a problem -- this routine should return failure */
+        ASSERT(commit_size >= slot_size);
+        commit_size += unit->size;
+        ASSERT(commit_size <= new_size);
+        new_memory = (cache_pc) heap_mmap_reserve(new_size, commit_size);
+        STATS_FCACHE_SUB(cache, capacity, unit->size);
+        STATS_FCACHE_ADD(cache, capacity, commit_size);
+        STATS_FCACHE_MAX(cache, capacity_peak, capacity);
+        STATS_SUB(fcache_combined_capacity, unit->size);
+        STATS_ADD(fcache_combined_capacity, commit_size);
+        STATS_MAX(peak_fcache_combined_capacity, fcache_combined_capacity);
+        LOG(THREAD, LOG_HEAP, 3, "fcache_increase_size -> "PFX"\n", new_memory);
+        ASSERT(new_memory != NULL);
+        ASSERT(proc_is_cache_aligned((void *)new_memory));
+    }
+
+    /* while we can handle resizing any unit, we only expect to resize
+     * the initial unit in a cache until it reaches the max unit size.
+     */
+    ASSERT(unit == cache->units && unit->next_local == NULL);
+
+    /* copy old data over to new memory */
+    memcpy(new_memory, unit->start_pc, unit->size);
+
+    /* update pc-relative into-cache or out-of-cache pointers
+     * also update stored addresses like start pc
+     * assumption: all intra-cache links will still work!
+     * they're all relative, we copied entire cache!
+     */
+    shift = (new_memory - unit->start_pc);
+    /* make sure we don't screw up any alignment */
+    ASSERT(ALIGNED(shift, proc_get_cache_line_size()));
+    ASSERT(ALIGNED(shift, PAD_JMPS_ALIGNMENT));
+    fcache_shift_fragments(dcontext, unit, shift,
+                           new_memory, new_memory + new_size, unit->size);
+
+    /* now change unit fields */
+    if (reallocated) {
+        /* de-allocate old memory -- not now, but next time we're in
+         * fcache_add_fragment, b/c the current ilist for being-added fragment
+         * may reference memory in old cache
+         */
+        thread_units_t *tu = (thread_units_t *) dcontext->fcache_field;
+        ASSERT(tu->pending_unmap_pc == NULL);
+        tu->pending_unmap_pc = unit->start_pc;
+        tu->pending_unmap_size = UNIT_RESERVED_SIZE(unit);
+    }
+
+    /* whether newly allocated or taken from dead list, increase cache->size
+     * by the difference between new size and old size
+     */
+    cache->size -= unit->size;
+    cache->size += commit_size;
+    unit->cur_pc += shift;
+    unit->start_pc = new_memory;
+    unit->size = commit_size;
+    unit->end_pc = unit->start_pc + commit_size;
+    unit->reserved_end_pc = unit->start_pc + new_size;
+    vmvector_add(fcache_unit_areas, unit->start_pc,
+                 unit->reserved_end_pc, (void *) unit);
+    unit->full = false; /* reset */
+
+#ifdef WINDOWS_PC_SAMPLE
+    {
+        /* old unit was copied to start of enlarged unit, can copy old prof
+         * buffer to start of new buffer and maintain correspondence
+         */
+        profile_t *old_prof = unit->profile;
+        if (old_prof != NULL) {
+            unit->profile = create_profile(unit->start_pc, unit->reserved_end_pc, 
+                                           dynamo_options.prof_pcs_fcache, 
+                                           NULL);
+            stop_profile(old_prof);
+            ASSERT(unit->profile->buffer_size >= old_prof->buffer_size);
+            memcpy(unit->profile->buffer, old_prof->buffer,
+                   old_prof->buffer_size);
+            free_profile(old_prof);
+            start_profile(unit->profile);
+        }
+    }
+#endif
+
+    DOLOG(2, LOG_CACHE, { verify_fifo(dcontext, cache); });
+    LOG(THREAD, LOG_CACHE, 1, "\tDone increasing unit size\n");
+}
+
+static void
+fcache_thread_reset_init(dcontext_t *dcontext)
+{
+    /* nothing */
+}
+
+void
+fcache_thread_init(dcontext_t *dcontext)
+{
+    thread_units_t *tu = (thread_units_t *) heap_alloc(dcontext, sizeof(thread_units_t)
+                                                 HEAPACCT(ACCT_OTHER));
+    dcontext->fcache_field = (void *) tu;
+    /* don't build trace cache until we actually build a trace
+     * this saves memory for both DYNAMO_OPTION(disable_traces) and for
+     * idle threads that never do much
+     */
+    tu->trace = NULL;
+    /* in fact, let's delay both, cost is single conditional in fcache_add_fragment,
+     * once we have that conditional for traces it's no extra cost for bbs
+     */
+    tu->bb = NULL;
+    tu->pending_unmap_pc = NULL;
+    tu->pending_flush = false;
+
+    fcache_thread_reset_init(dcontext);
+}
+
+/* see if a fragment with that tag has existed, ever, in any cache */
+bool
+fragment_lookup_deleted(dcontext_t *dcontext, app_pc tag)
+{
+    future_fragment_t *fut;
+    if (SHARED_FRAGMENTS_ENABLED() && dcontext != GLOBAL_DCONTEXT) {
+        fut = fragment_lookup_private_future(dcontext, tag);
+        if (fut != NULL)
+            return TEST(FRAG_WAS_DELETED, fut->flags);
+        /* if no private, lookup shared */
+    }
+    fut = fragment_lookup_future(dcontext, tag);
+    return (fut != NULL && TEST(FRAG_WAS_DELETED, fut->flags));
+}
+
+/* find a fragment that existed in the same type of cache */
+static future_fragment_t *
+fragment_lookup_cache_deleted(dcontext_t *dcontext, fcache_t *cache, app_pc tag)
+{
+    future_fragment_t *fut;
+    if (!cache->is_shared) {
+        /* only look for private futures, since we only care about whether this
+         * cache needs to be resized, and thus only if we kicked tag out of
+         * this cache, not whether we kicked it out of the shared cache
+         */
+        fut = fragment_lookup_private_future(dcontext, tag);
+    } else
+        fut = fragment_lookup_future(dcontext, tag);
+    if (fut != NULL && TEST(FRAG_WAS_DELETED, fut->flags))
+        return fut;
+    else
+        return NULL;
+}
+
+#ifdef DEBUG
+/* this routine is separate from fcache_thread_exit because it needs to
+ * be run before fragment_thread_exit, whereas the real fcache cleanup
+ * needs to be done after fragment's cleanup
+ */
+void
+fcache_thread_exit_stats(dcontext_t *dcontext)
+{
+    DOELOG(1, LOG_CACHE, {
+        thread_units_t *tu = (thread_units_t *) dcontext->fcache_field;
+        if (tu->bb != NULL)
+            fcache_cache_stats(dcontext, tu->bb);
+        if (tu->trace != NULL)
+            fcache_cache_stats(dcontext, tu->trace);
+    });
+}
+#endif
+
+static void
+fcache_thread_reset_free(dcontext_t *dcontext)
+{
+    thread_units_t *tu = (thread_units_t *) dcontext->fcache_field;
+    if (tu->pending_unmap_pc != NULL) {
+        /* de-allocate old memory -- stats have already been taken care of */
+        /* remove from interval data struct first to avoid races w/ it
+         * being re-used and not showing up in in_fcache
+         */
+        vmvector_remove(fcache_unit_areas, tu->pending_unmap_pc,
+                        tu->pending_unmap_pc+tu->pending_unmap_size);
+        heap_munmap(tu->pending_unmap_pc, tu->pending_unmap_size);
+        tu->pending_unmap_pc = NULL;
+    }
+    if (tu->bb != NULL) {
+        fcache_cache_free(dcontext, tu->bb, true);
+        tu->bb = NULL;
+    }
+    if (tu->trace != NULL) {
+        fcache_cache_free(dcontext, tu->trace, true);
+        tu->trace = NULL;
+    }
+}
+
+void
+fcache_thread_exit(dcontext_t *dcontext)
+{
+    DEBUG_DECLARE(thread_units_t *tu = (thread_units_t *) dcontext->fcache_field;)
+    fcache_thread_reset_free(dcontext);
+    DODEBUG({
+        /* for non-debug we do fast exit path and don't free local heap */
+        heap_free(dcontext, tu, sizeof(thread_units_t) HEAPACCT(ACCT_OTHER));
+    });
+}
+
+#if defined(DEBUG) && defined(INTERNAL)
+static void
+print_fifo(dcontext_t *dcontext, fcache_t *cache)
+{
+    fragment_t *f = cache->fifo;
+    ASSERT(USE_FIFO_FOR_CACHE(cache));
+    ASSERT(CACHE_PROTECTED(cache));
+    while (f != NULL) {
+        /* caller sets loglevel */
+        LOG(THREAD, LOG_CACHE, 1, "\tF%d "PFX" = @"PFX" size %d\n",
+            FRAG_ID(f), FRAG_TAG(f), FRAG_HDR_START(f), FRAG_SIZE(f));
+        f = FIFO_NEXT(f);
+    }
+}
+
+static void
+verify_fifo(dcontext_t *dcontext, fcache_t *cache)
+{
+    fragment_t *f = cache->fifo;
+    fcache_unit_t *u;
+    cache_pc pc;
+    ASSERT(USE_FIFO_FOR_CACHE(cache));
+    ASSERT(CACHE_PROTECTED(cache));
+    while (f != NULL) {
+        LOG(THREAD, LOG_CACHE, 6, "\t*"PFX" F%d "PFX" = @"PFX" size %d\n",
+            f, FRAG_ID(f), FRAG_TAG(f), FRAG_HDR_START(f), FRAG_SIZE(f));
+        /* check that header is intact */
+        pc = FRAG_HDR_START(f);
+        ASSERT(*((fragment_t **)pc) == f);
+        /* check that no missing space */
+        pc += FRAG_SIZE(f);
+        u = FIFO_UNIT(f);
+        /* free list scheme can't be walked expecting FIFO headers */
+        if ((!DYNAMO_OPTION(cache_shared_free_list) || !cache->is_shared)) {
+            if (pc < u->cur_pc)
+                ASSERT(*((fragment_t **)pc) != NULL);
+        }
+        f = FIFO_NEXT(f);
+    }
+}
+#endif
+           
+static void
+fifo_append(fcache_t *cache, fragment_t *f)
+{
+    ASSERT(USE_FIFO(f));
+    ASSERT(CACHE_PROTECTED(cache));
+    /* start has prev to end, but end does NOT have next to start */
+    FIFO_NEXT_ASSIGN(f, NULL);
+    if (cache->fifo == NULL) {
+        cache->fifo = f;
+        FIFO_PREV_ASSIGN(f, f);
+    } else {
+        FIFO_PREV_ASSIGN(f, FIFO_PREV(cache->fifo));
+        FIFO_NEXT_ASSIGN(FIFO_PREV(cache->fifo), f);
+        FIFO_PREV_ASSIGN(cache->fifo, f);
+    }
+    FIFO_NEXT_ASSIGN(f, NULL);
+    LOG(THREAD_GET, LOG_CACHE, 5, "fifo_append F%d @"PFX"\n",
+        FRAG_ID(f), FRAG_HDR_START(f));
+    DOLOG(6, LOG_CACHE, { print_fifo(get_thread_private_dcontext(), cache); });
+}
+
+static void
+fifo_remove(dcontext_t *dcontext, fcache_t *cache, fragment_t *f)
+{
+    ASSERT(USE_FIFO(f));
+    ASSERT(CACHE_PROTECTED(cache));
+    ASSERT(cache->fifo != NULL);
+    /* start has prev to end, but end does NOT have next to start */
+    if (f == cache->fifo) {
+        cache->fifo = FIFO_NEXT(f);
+    } else {
+        FIFO_NEXT_ASSIGN(FIFO_PREV(f), FIFO_NEXT(f));
+    }
+    if (FIFO_NEXT(f) == NULL) {
+        if (cache->fifo != NULL)
+            FIFO_PREV_ASSIGN(cache->fifo, FIFO_PREV(f));
+    } else {
+        FIFO_PREV_ASSIGN(FIFO_NEXT(f), FIFO_PREV(f));
+    }
+    LOG(THREAD, LOG_CACHE, 5, "fifo_remove F%d @"PFX"\n",
+        FRAG_ID(f), FRAG_HDR_START(f));
+    DOLOG(6, LOG_CACHE, { print_fifo(dcontext, cache); });
+    if (FRAG_EMPTY(f)) {
+        STATS_FCACHE_SUB(cache, empty, FRAG_SIZE(f));
+        STATS_FCACHE_ADD(cache, used, FRAG_SIZE(f));
+        nonpersistent_heap_free(ALLOC_DC(dcontext, cache), f, sizeof(empty_slot_t)
+                                HEAPACCT(ACCT_FCACHE_EMPTY));
+    }
+}
+
+static void
+fifo_prepend_empty(dcontext_t *dcontext, fcache_t *cache, fcache_unit_t *unit,
+                   fragment_t *f, cache_pc start_pc, uint size)
+{
+    empty_slot_t *slot;
+    ASSERT(CACHE_PROTECTED(cache));
+
+    STATS_FCACHE_ADD(cache, empty, size);
+
+    if (DYNAMO_OPTION(cache_shared_free_list) && cache->is_shared && !cache->is_coarse) {
+        ASSERT(USE_FREE_LIST_FOR_CACHE(cache));
+        add_to_free_list(dcontext, cache, unit, f, start_pc, size);
+        return;
+    }
+    /* FIXME: make cache_shared_free_list always on and remove the option
+     * as there really is no alternative implemented -- we just waste the space.
+     * FIXME case 8714: anything we can do for coarse-grain?
+     */
+    if (!USE_FIFO_FOR_CACHE(cache))
+        return;
+
+    /* don't make two entries for adjacent empties
+     * for efficiency only check front of FIFO -- most common case anyway
+     */
+    if (cache->fifo != NULL && FRAG_EMPTY(cache->fifo)) {
+        if (FRAG_HDR_START(cache->fifo) == start_pc+size) {
+            LOG(THREAD, LOG_CACHE, 5, "prepend: just enlarging next empty\n");
+            FRAG_START_ASSIGN(cache->fifo, start_pc + HEADER_SIZE_FROM_CACHE(cache));
+            *((fragment_t **)start_pc) = cache->fifo;
+            FRAG_SIZE_ASSIGN(cache->fifo, FRAG_SIZE(cache->fifo) + size);
+            return;
+        } else if (FRAG_HDR_START(cache->fifo) + FRAG_SIZE(cache->fifo) == start_pc) {
+            LOG(THREAD, LOG_CACHE, 5, "prepend: just enlarging prev empty\n");
+            FRAG_SIZE_ASSIGN(cache->fifo, FRAG_SIZE(cache->fifo) + size);
+            return;
+        }
+    }
+
+    slot = (empty_slot_t *)
+        nonpersistent_heap_alloc(ALLOC_DC(dcontext, cache), sizeof(empty_slot_t)
+                                 HEAPACCT(ACCT_FCACHE_EMPTY));
+    slot->flags = FRAG_FAKE | FRAG_IS_EMPTY_SLOT;
+    LOG(THREAD, LOG_CACHE, 5, "prepend: writing "PFX" to "PFX"\n", slot, start_pc);
+    *((empty_slot_t **)start_pc) = slot;
+    slot->start_pc = start_pc + HEADER_SIZE_FROM_CACHE(cache);
+    slot->fcache_size = size;
+    /* stick on front */
+    slot->next_fcache = cache->fifo;
+    if (cache->fifo == NULL)
+        slot->prev_fcache = (fragment_t *) slot;
+    else {
+        slot->prev_fcache = FIFO_PREV(cache->fifo);
+        FIFO_PREV_ASSIGN(cache->fifo, (fragment_t *)slot);
+    }
+    /* start has prev to end, but end does NOT have next to start */
+    cache->fifo = (fragment_t *) slot;
+    LOG(THREAD, LOG_CACHE, 5, "fifo_prepend_empty F-1 @"PFX"\n", start_pc);
+    DOLOG(6, LOG_CACHE, { print_fifo(dcontext, cache); });
+}
+
+/* returns whether the cache should be allowed to grow */
+static bool
+check_regen_replace_ratio(dcontext_t *dcontext, fcache_t *cache, uint add_size)
+{
+    if (cache->max_size != 0 && cache->size + add_size > cache->max_size) {
+        /* if at max size, avoid regen/replace checks */
+        LOG(THREAD, LOG_CACHE, 4, "at max size 0x%x\n", cache->max_size);
+        return false;
+    } else if (!cache->finite_cache || cache->replace_param == 0) {
+        /* always upgrade -- adaptive working set is disabled */
+        LOG(THREAD, LOG_CACHE, 4, "upgrading since fcache_replace==0\n");
+        return true;
+    } else if (cache->regen_param == 0) {
+        /* never upgrade due to regen ratio */
+        LOG(THREAD, LOG_CACHE, 4, "will never upgrade since fcache_regen==0\n");
+        return false;
+    } else if (cache->wset_check > 0) {
+        /* wset_check is only used for fifo caches */
+        ASSERT(USE_FIFO_FOR_CACHE(cache));
+        cache->wset_check--;
+        LOG(THREAD, LOG_CACHE, 4, "dec wset_check -> %d\n", cache->wset_check);
+        return false;
+    } else if (cache->size < cache->free_upgrade_size) {
+        /* free upgrade, but set check for next time */
+        if (USE_FIFO_FOR_CACHE(cache)) {
+            ASSERT(cache->wset_check == 0);
+            cache->wset_check = cache->replace_param;
+        } else {
+            ASSERT(!cache->is_coarse); /* no individual fragment support */
+            /* if a new unit would put us over the free upgrade point,
+             * start keeping track of regen stats
+             */
+            if (cache->size + cache->max_unit_size >= cache->free_upgrade_size
+                /* could come here after having already set this if flush
+                 * a larger unit than last new unit and drop back below threshold
+                 */
+                && !cache->record_wset) {
+                cache->record_wset = true;
+            }
+        }
+        LOG(THREAD, LOG_CACHE, 3, "Free upgrade, no resize check\n");
+        return true;
+    } else {
+        if (USE_FIFO_FOR_CACHE(cache)) {
+            /* wait cache->replace_param frags before checking again,
+             * to avoid too many checks when regen<<replace
+             */
+            cache->wset_check = cache->replace_param;
+        } else {
+            ASSERT(!cache->is_coarse); /* no individual fragment support */
+            if (!cache->record_wset) {
+                /* now we are big enough that we need to keep track, though
+                 * ideally we should hit this prior to the free upgrade point,
+                 * as otherwise this is a 2nd free resize, but might not if
+                 * create a new unit larger than max unit size.
+                 */
+                cache->record_wset = true;
+                return true;
+            }
+        }
+        /* FIXME: for shared w/ replace==100 perhaps remove this if */
+        if (cache->num_replaced >= cache->replace_param &&
+            cache->num_regenerated >= cache->regen_param) {
+            /* minimum regen/replaced ratio, compute w/o using floating point ops
+             * and avoiding overflow (unless replace overflows before regen
+             * hits regen_param, which is very unlikely and who cares if it does)
+             */
+            /* this loop guaranteed to terminate b/c we check for 0 above */
+            ASSERT(cache->replace_param > 0 && cache->regen_param > 0);
+            while (cache->num_replaced >= cache->replace_param &&
+                   cache->num_regenerated >= cache->regen_param) {
+                cache->num_replaced -= cache->replace_param;
+                cache->num_regenerated -= cache->regen_param;
+            }
+            LOG(THREAD, LOG_CACHE, 3,
+                "Resize check: for %s unit: %d regenerated / %d replaced\n",
+                cache->name, cache->num_regenerated, cache->num_replaced);
+            if (cache->num_regenerated >= cache->regen_param) {
+                LOG(THREAD, LOG_CACHE, 1,
+                    "%s unit reached ratio with %d regenerated / %d replaced\n",
+                    cache->name, cache->num_regenerated, cache->num_replaced);
+                return true;
+            }
+        }
+        LOG(THREAD, LOG_CACHE, 4, "No resize allowed yet\n");
+        return false;
+    }
+    return true;
+}
+
+/* Adds size to the end of the non-empty unit unit
+ * If a small area is eaten and added to size, returns that amount
+ */
+static size_t
+extend_unit_end(dcontext_t *dcontext, fcache_t *cache,
+                fcache_unit_t *unit, size_t size, bool rest_empty)
+{
+    size_t left, extra = 0;
+    ASSERT(CACHE_PROTECTED(cache));
+    unit->cur_pc += size;
+    STATS_FCACHE_ADD(cache, claimed, size);
+    STATS_ADD(fcache_combined_claimed, size);
+    left = unit->end_pc - unit->cur_pc;
+    ASSERT(unit->end_pc >= unit->cur_pc);
+    if (left < MIN_UNIT_END_HOLE(cache)) {
+        LOG(THREAD_GET, LOG_CACHE, 3, "\tunit is now full\n");
+        unit->full = true;
+        if (left > 0) {
+            /* eat up too-small area at end */
+            extra = left;
+            unit->cur_pc += extra;
+            STATS_FCACHE_ADD(cache, claimed, extra);
+            STATS_ADD(fcache_combined_claimed, extra);
+        }
+    } else if (rest_empty) {
+        /* make entire rest of unit into an empty slot */
+        ASSERT(CHECK_TRUNCATE_TYPE_uint(left));
+        fifo_prepend_empty(dcontext, cache, unit, NULL, unit->cur_pc, (uint)left);
+        unit->cur_pc += left;
+        STATS_FCACHE_ADD(cache, claimed, left);
+        STATS_ADD(fcache_combined_claimed, left);
+        unit->full = true;
+    }
+    LOG(THREAD, LOG_CACHE, 5, "\t\textend_unit_end: %d + %d / %d => cur_pc = "PFX"\n",
+        size, extra, left, unit->cur_pc);
+    /* FIXME: if extended b/c need new unit (size==0), extra is empty space, but
+     * we cannot add it to stats b/c will never be removed!
+     */
+    STATS_FCACHE_ADD(cache, used, (size + extra));
+    STATS_FCACHE_MAX(cache, peak, used);
+    return extra;
+}
+
+/* Returns whether was able to either resize unit or create a new unit.
+ * For non-FIFO caches this routine cannot fail and must suspend the world
+ * and reset if necessary.
+ */
+static bool
+try_for_more_space(dcontext_t *dcontext, fcache_t *cache, fcache_unit_t *unit,
+                   uint slot_size)
+{
+    ASSERT(CACHE_PROTECTED(cache));
+
+    if (unit->end_pc < unit->reserved_end_pc) {
+        /* extend commitment if have more reserved */
+        uint commit_size = DYNAMO_OPTION(cache_commit_increment);
+        while (unit->cur_pc + slot_size > unit->end_pc + commit_size)
+            commit_size *= 2;
+        if (unit->end_pc + commit_size > unit->reserved_end_pc) {
+            ASSERT_TRUNCATE(commit_size, uint, unit->reserved_end_pc - unit->end_pc);
+            commit_size = (uint) (unit->reserved_end_pc - unit->end_pc);
+        }
+        cache_extend_commitment(unit, commit_size);
+        if (unit->cur_pc + slot_size > unit->end_pc) {
+            /* must be a huge trace or something
+             * still worth committing, we'll make an empty here 
+             */
+            extend_unit_end(dcontext, cache, unit, 0, true);
+            /* continue below and try to make more space */
+        } else
+            return true;
+    }
+
+    /* see if we have room to expand according to user-set maximum */
+    if (cache->max_size == 0 || cache->size + slot_size <= cache->max_size) {
+        LOG(THREAD, LOG_CACHE, 1, "max size = %d, cur size = %d\n",
+            cache->max_size/1024, cache->size/1024);
+        /* at larger sizes better to create separate units to avoid expensive
+         * re-linking when resize */
+        if (unit->size >= cache->max_unit_size) {
+            fcache_unit_t *newunit;
+            size_t newsize;
+            ASSERT(!USE_FIFO_FOR_CACHE(cache) ||
+                   cache->fifo != NULL); /* shouldn't be empty! */
+            /* fill out to end first -- turn remaining room into empty slot */
+            extend_unit_end(dcontext, cache, unit, 0, true);
+            ASSERT(unit->full);
+
+            /* before create a new unit, see if we should flush an old one */
+            if (!USE_FIFO_FOR_CACHE(cache) && cache->finite_cache && !cache->is_coarse) {
+                /* wset algorithm for shared caches: when request a new unit,
+                 * must grant the request, but if regen/replace ratio does not
+                 * exceed target, must flush an old unit.
+                 */
+                if (!check_regen_replace_ratio(dcontext, cache,
+                                               0 /*not adding a fragment*/)) {
+                    /* flush the oldest unit, at the end of the list */
+                    thread_units_t *tu = (thread_units_t *) dcontext->fcache_field;
+                    fcache_unit_t *oldest = cache->units, *prev = NULL;
+                    ASSERT(oldest != NULL);
+
+                    /* another place where a prev_local would be nice */
+                    for (; oldest->next_local != NULL;
+                         prev = oldest, oldest = oldest->next_local)
+                        ; /* nothing */
+
+                    /* Indicate unit is still live even though off live list.
+                     * Flag will be cleared once really flushed in
+                     * fcache_flush_pending_units()
+                     */
+                    DODEBUG({ oldest->pending_flush = true; });
+
+                    LOG(THREAD, LOG_CACHE, 2,
+                        "marking unit "PFX"-"PFX" for flushing\n",
+                        oldest->start_pc, oldest->end_pc);
+
+                    /* move to pending-flush list and set trigger */
+                    if (prev == NULL)
+                        cache->units = oldest->next_local;
+                    else
+                        prev->next_local = oldest->next_local;
+                    /* clear local just in case, should be no downstream use */
+                    if (unit == oldest)
+                        unit = NULL;
+
+                    mutex_lock(&unit_flush_lock);
+                    oldest->next_local = allunits->units_to_flush;
+                    allunits->units_to_flush = oldest;
+                    STATS_ADD_PEAK(cache_units_toflush, 1);
+                    /* FIXME case 8743: we should call remove_unit_from_cache() here,
+                     * but we need the cache field for chain_fragments_for_flush() --
+                     * so we assume for now that there are no deletable caches that
+                     * don't use fifos yet are finite, and let
+                     * append_units_to_free_list() remove from the cache later on.
+                     * This does mean that cache->size is too big from now until
+                     * then, so we don't really support hardcoded cache sizes.
+                     */
+                    mutex_unlock(&unit_flush_lock);
+
+                    tu->pending_flush = true;
+                    STATS_INC(cache_units_wset_flushed);
+                } else
+                    STATS_INC(cache_units_wset_allowed);
+            }
+
+            /* now make a new unit 
+             * if new frag is large, make unit large as well
+             */
+            newsize = cache->max_unit_size;
+            if (newsize < slot_size * MAX_SINGLE_MULTIPLE)
+                newsize = ALIGN_FORWARD(slot_size * MAX_SINGLE_MULTIPLE, PAGE_SIZE);
+            /* final adjustment: make sure don't go over max */
+            if (cache->max_size > 0 && cache->size + newsize > cache->max_size)
+                newsize = cache->max_size - cache->size;
+            newunit = fcache_create_unit(dcontext, cache, NULL, newsize);
+            LOG(THREAD, LOG_CACHE, 1, "Creating a new %s unit of %d KB @"PFX"\n",
+                cache->name, newunit->size/1024, newunit->start_pc);
+            newunit->next_local = cache->units;
+            cache->units = newunit;
+        } else {
+            ASSERT(!cache->is_coarse); /* no individual support so harder to resize */
+            LOG(THREAD, LOG_CACHE, 1, "Increasing size of %s unit of %d KB @"PFX"\n",
+                cache->name, unit->size/1024, unit->start_pc);
+            fcache_increase_size(dcontext, cache, unit, slot_size);
+            LOG(THREAD, LOG_CACHE, 1, "\tnow %d KB @"PFX"\n",
+                unit->size/1024, unit->start_pc);
+        }
+        /* reset counters, but not deleted table */
+        cache->num_replaced = 0;
+        cache->num_regenerated = 0;
+        DOLOG(2, LOG_CACHE, { fcache_cache_stats(dcontext, cache); });
+        return true;
+    } else {
+        if (!USE_FIFO_FOR_CACHE(cache)) {
+            /* options check up front shouldn't allow us to get here */
+            ASSERT_NOT_REACHED();
+            /* Case 8203: we need a new reset type that doesn't free anything,
+             * and aborts traces only of other threads (not this one, as this
+             * could be a trace we're emitting now).  Then we could free all
+             * fragments in a unit here.
+             * In order to do the reset we'd first need to release cache->lock,
+             * if !cache->is_trace release the bb_building_lock, and enter_nolinking().
+             * Note that for bb cache we could instead do a full reset and then
+             * transfer_to_dispatch() but in debug builds we won't
+             * free locals in prior stack frames: the fragment_t, the
+             * instrlist_t, etc.  For trace cache doing that would be a bigger
+             * step backward and take longer to get back here.
+             */
+        }
+        /* tell user if fragment bigger than max size
+         * FIXME: but if trace cache has small max size, should just
+         * not build traces that big!
+         */
+        if (cache->max_size > 0 && slot_size > (int) cache->max_size) {
+#ifdef INTERNAL
+            char *name = "";
+            DODEBUG(name = cache->name;);
+#endif
+            USAGE_ERROR("single %s fragment (%d bytes) > max cache size (%d bytes)",
+                        name, slot_size, cache->max_size);
+        }
+        return false;
+    }
+}
+
+static void
+place_fragment(dcontext_t *dcontext, fragment_t *f, fcache_unit_t *unit,
+               cache_pc header_pc)
+{
+    fcache_t *cache = unit->cache;
+    ASSERT_OWN_MUTEX(unit->cache->is_shared, &unit->cache->lock);
+    DOLOG(3, LOG_CACHE, { /* only to reduce perf hit */
+        fragment_t wrapper;
+        /* cannot call fragment_pclookup as it will grab the fcache lock */
+        ASSERT(fragment_pclookup_by_htable(dcontext, header_pc + HEADER_SIZE(f),
+                                           &wrapper) == NULL);
+    });
+    if (HEADER_SIZE(f) > 0) {
+        /* add header */
+        LOG(THREAD, LOG_CACHE, 5, "place: writing "PFX" to "PFX"\n", f, header_pc);
+        *((fragment_t **)header_pc) = f;
+    }
+    /* we assume alignment padding was added at end of prev fragment, so this
+     * guy needs no padding at start
+     */
+    FRAG_START_ASSIGN(f, header_pc + HEADER_SIZE(f));
+    ASSERT(ALIGNED(FRAG_HDR_START(f), SLOT_ALIGNMENT(cache)));
+    STATS_FCACHE_ADD(cache, headers, HEADER_SIZE(f));
+    STATS_FCACHE_ADD(cache, align, f->fcache_extra - (stats_int_t)HEADER_SIZE(f));
+
+    /* for shared caches we must track regen/replace on every placement */
+    if (cache->record_wset) {
+        /* FIXME: how is this supposed to work for traces where a bb
+         * may have replaced the future?  xref case 7151, though that
+         * should be a problem for private as well...
+         */
+        future_fragment_t *fut = fragment_lookup_cache_deleted(dcontext, cache, f->tag);
+        ASSERT(!USE_FIFO_FOR_CACHE(cache));
+        ASSERT(!cache->is_coarse);
+        cache->num_replaced++; /* simply number created past record_wset point */
+        if (fut != NULL) {
+            cache->num_regenerated++;
+            STATS_INC(num_fragments_regenerated);
+            SHARED_FLAGS_RECURSIVE_LOCK(fut->flags, acquire, change_linking_lock);
+            fut->flags &= ~FRAG_WAS_DELETED;
+            SHARED_FLAGS_RECURSIVE_LOCK(fut->flags, release, change_linking_lock);
+        }
+        LOG(THREAD, LOG_CACHE, 4, "For %s unit: %d regenerated / %d replaced\n",
+            cache->name, cache->num_regenerated, cache->num_replaced);
+    }
+}
+
+#ifdef DEBUG
+static void
+removed_fragment_stats(dcontext_t *dcontext, fcache_t *cache, fragment_t *f)
+{
+    int prefixes = fragment_prefix_size(f->flags);
+    linkstub_t *l = FRAGMENT_EXIT_STUBS(f);
+    int stubs = 0;
+    uint selfmod = 0;
+    /* N.B.: we cannot call EXIT_STUB_PC() here as with the -detect_dangling_fcache
+     * option the fragment will be obliterated at this point and we will not
+     * be able to locate the stub pc for an indirect exit.
+     * Instead we simply calculate what the stub sizes should be.
+     */
+    for (; l != NULL; l = LINKSTUB_NEXT_EXIT(l)) {
+        int sz;
+        if (!EXIT_HAS_LOCAL_STUB(l->flags, f->flags))
+            continue; /* it's kept elsewhere */
+        sz = linkstub_size(dcontext, f, l);
+#ifdef CUSTOM_EXIT_STUBS
+        sz += l->fixed_stub_offset;
+#endif
+        if (LINKSTUB_INDIRECT(l->flags))
+            STATS_FCACHE_ADD(cache, indirect_stubs, -sz);
+        else {
+            ASSERT(LINKSTUB_DIRECT(l->flags));
+            STATS_FCACHE_ADD(cache, direct_stubs, -sz);
+        }
+        stubs += sz;
+    }
+    if (TEST(FRAG_SELFMOD_SANDBOXED, f->flags)) {
+        /* we cannot go and re-decode the app bb now, since it may have been
+         * changed (selfmod doesn't make page RO!), so we use a stored size
+         * that's there just for stats
+         */
+        selfmod = FRAGMENT_SELFMOD_COPY_SIZE(f);
+        STATS_FCACHE_SUB(cache, selfmod_copy, selfmod);
+    }
+    STATS_FCACHE_SUB(cache, bodies, (f->size - (prefixes + stubs + selfmod)));
+    STATS_FCACHE_SUB(cache, prefixes, prefixes);
+    STATS_FCACHE_SUB(cache, headers, HEADER_SIZE(f));
+    STATS_FCACHE_SUB(cache, align, (f->fcache_extra - (stats_int_t)HEADER_SIZE(f)));
+}
+#endif /* DEBUG */
+
+static void
+force_fragment_from_cache(dcontext_t *dcontext, fcache_t *cache, fragment_t *victim)
+{
+    bool empty = FRAG_EMPTY(victim); /* fifo_remove will free empty slot */
+    ASSERT(CACHE_PROTECTED(cache));
+    if (USE_FIFO(victim))
+        fifo_remove(dcontext, cache, victim);
+    if (!empty) {
+        /* don't need to add deleted -- that's done by link.c for us,
+         * when it makes a future fragment it uses the FRAG_WAS_DELETED flag 
+         */
+        if (cache->finite_cache)
+            cache->num_replaced++;
+        DOSTATS({ removed_fragment_stats(dcontext, cache, victim); });
+        STATS_INC(num_fragments_replaced);
+        fragment_delete(dcontext, victim, FRAGDEL_NO_FCACHE);
+    }
+}
+
+static bool
+replace_fragments(dcontext_t *dcontext, fcache_t *cache, fcache_unit_t *unit,
+                  fragment_t *f, fragment_t *fifo, uint slot_size)
+{
+    fragment_t *victim;
+    uint slot_so_far;
+    cache_pc pc, header_pc;
+
+    ASSERT(CACHE_PROTECTED(cache));
+    /* free list scheme can't be walked expecting FIFO headers */
+    ASSERT(!DYNAMO_OPTION(cache_shared_free_list) || !cache->is_shared);
+    ASSERT(USE_FIFO_FOR_CACHE(cache));
+
+#ifdef NATIVE_RETURN
+    LOG(THREAD, 3, LOG_CACHE, "NATIVE_RETURN cannot handle deleting fragments\n");
+    return false;
+#endif
+
+    DODEBUG({ cache->consistent = false; });
+    /* first walk: make sure this is possible (look for un-deletable frags) */
+    slot_so_far = 0;
+    pc = FRAG_HDR_START(fifo);
+    victim = fifo;
+    while (true) {
+        if (TEST(FRAG_CANNOT_DELETE, victim->flags)) {
+            DODEBUG({ cache->consistent = true; });
+            return false;
+        }
+        slot_so_far += FRAG_SIZE(victim);
+        if (slot_so_far >= slot_size)
+            break;
+        /* look at contiguously-next fragment_t in cache */
+        pc += FRAG_SIZE(victim);
+        if (pc == unit->cur_pc) {
+            /* we can just take unallocated space */
+            break;
+        }
+        ASSERT(pc < unit->cur_pc);
+        victim = *((fragment_t **)pc);
+        LOG(THREAD, LOG_CACHE, 5, "\treading "PFX" -> "PFX"\n", pc, victim);
+        ASSERT(victim != NULL);
+        ASSERT(FIFO_UNIT(victim) == unit);
+    }
+
+    LOG(THREAD, LOG_CACHE, 4, "\treplacing fragment(s) in filled unit\n");
+
+    /* record stats that will be destroyed */
+    header_pc = FRAG_HDR_START(fifo);
+
+    /* second walk: do the deletion */
+    slot_so_far = 0;
+    pc = header_pc;
+    victim = fifo;
+    while (true) {
+        slot_so_far += FRAG_SIZE(victim);
+        pc += FRAG_SIZE(victim);
+        LOG(THREAD, LOG_CACHE, 4, "\t\tdeleting F%d => %d bytes\n",
+            FRAG_ID(victim), slot_so_far);
+        force_fragment_from_cache(dcontext, cache, victim);
+        if (slot_so_far >= slot_size)
+            break;
+        /* look at contiguously-next fragment_t in cache
+         * assumption: wouldn't be here if not enough victims below us, so
+         * don't need to check for end of cache
+         */
+        if (pc == unit->cur_pc) {
+            /* take unallocated space */
+            size_t extra = extend_unit_end(dcontext, cache, unit,
+                                           slot_size - slot_so_far, false);
+            LOG(THREAD, LOG_CACHE, 4, "\t\textending unit by %d => %d bytes\n",
+                slot_size - slot_so_far, slot_size);
+            ASSERT(CHECK_TRUNCATE_TYPE_uint(extra));
+            FRAG_SIZE_ASSIGN(f, slot_size+(uint)extra);
+            /* no splitting will be needed */
+            slot_so_far = slot_size;
+            break;
+        }
+        ASSERT(pc < unit->cur_pc);
+        victim = *((fragment_t **)pc);
+    }
+
+    if (slot_so_far > slot_size) {
+        uint diff = slot_so_far - slot_size;
+        /* if we were using free lists we would check for next slot being a
+         * free entry and if so coalescing any size space with it
+         */
+        if (diff < MIN_EMPTY_HOLE(cache)) {
+            FRAG_SIZE_ASSIGN(f, slot_so_far);
+            LOG(THREAD, LOG_CACHE, 4, "\t\teating extra %d bytes\n", diff);
+        } else {
+            /* add entry for diff */
+            fifo_prepend_empty(dcontext, cache, unit, NULL, header_pc + FRAG_SIZE(f), diff);
+            STATS_FCACHE_SUB(cache, used, diff);
+        }
+    }
+
+    place_fragment(dcontext, f, unit, header_pc);
+    fifo_append(cache, f);
+
+    if (cache->finite_cache && cache->num_replaced > 0) {
+        future_fragment_t *fut = fragment_lookup_cache_deleted(dcontext, cache, f->tag);
+        ASSERT(cache->finite_cache && cache->replace_param > 0);
+        if (fut != NULL) {
+            cache->num_regenerated++;
+            STATS_INC(num_fragments_regenerated);
+            SHARED_FLAGS_RECURSIVE_LOCK(fut->flags, acquire, change_linking_lock);
+            fut->flags &= ~FRAG_WAS_DELETED;
+            SHARED_FLAGS_RECURSIVE_LOCK(fut->flags, release, change_linking_lock);
+        }
+        LOG(THREAD, LOG_CACHE, 4, "For %s unit: %d regenerated / %d replaced\n",
+            cache->name, cache->num_regenerated, cache->num_replaced);
+    }
+    DODEBUG({ cache->consistent = true; });
+    return true;
+}
+
+static inline bool
+replace_fifo(dcontext_t *dcontext, fcache_t *cache, fragment_t *f, uint slot_size,
+             fragment_t *fifo)
+{
+    fcache_unit_t *unit;
+    ASSERT(USE_FIFO(f));
+    ASSERT(CACHE_PROTECTED(cache));
+    while (fifo != NULL) {
+        unit = FIFO_UNIT(fifo);
+        if ((ptr_uint_t)(unit->end_pc - FRAG_HDR_START(fifo)) >= slot_size) {
+            /* try to replace fifo and possibly subsequent frags with f
+             * could fail if un-deletable frags
+             */
+            DOLOG(4, LOG_CACHE, { verify_fifo(dcontext, cache); });
+            if (replace_fragments(dcontext, cache, unit, f, fifo, slot_size))
+                return true;
+        }
+        fifo = FIFO_NEXT(fifo);
+    }
+    return false;
+}
+
+static inline
+int
+find_free_list_bucket(uint size)
+{
+    int bucket;
+    /* Find maximum slot we are >= than, for size in [SIZE[bucket], SIZE[bucket+1]) */
+    for (bucket = FREE_LIST_SIZES_NUM-1; size < FREE_LIST_SIZES[bucket]; bucket--)
+        ; /* no body */
+    ASSERT(bucket >= 0);
+    return bucket;
+}
+
+static inline free_list_footer_t *
+free_list_footer_from_header(free_list_header_t *h)
+{
+    return (free_list_footer_t *) (((cache_pc)h) + h->size - sizeof(free_list_footer_t));
+}
+
+static inline free_list_header_t *
+free_list_header_from_footer(free_list_footer_t *h)
+{
+    return (free_list_header_t *) (((cache_pc)h) + sizeof(free_list_footer_t) - h->size);
+}
+
+static inline void
+remove_from_free_list(fcache_t *cache, uint bucket, free_list_header_t *header
+                      _IF_DEBUG(bool coalesce))
+{
+    ASSERT(CACHE_PROTECTED(cache));
+    ASSERT(DYNAMO_OPTION(cache_shared_free_list) && cache->is_shared);
+    LOG(GLOBAL, LOG_CACHE, 4, 
+        "remove_from_free_list: %s bucket[%d] %d bytes @"PFX"\n",
+        cache->name, bucket,header->size, header);
+    if (header->prev != NULL)
+        header->prev->next = header->next;
+    else
+        cache->free_list[bucket] = header->next;
+    if (header->next != NULL)
+        header->next->prev = header->prev;
+    /* it's up to the caller to adjust FRAG_FOLLOWS_FREE_ENTRY if a fragment_t
+     * follows header.
+     * no reason to remove FRAG_FCACHE_FREE_LIST flag here.
+     */
+    DOSTATS({
+        if (coalesce)
+            cache->free_stats_coalesced[bucket]++;
+        else
+            cache->free_stats_reused[bucket]++;
+        cache->free_stats_charge[bucket] -= header->size;
+    });
+}
+
+/* If freeing a fragment, must pass that as f; else, pass NULL as f */
+static void
+add_to_free_list(dcontext_t *dcontext, fcache_t *cache, fcache_unit_t *unit,
+                 fragment_t *f, cache_pc start_pc, uint size)
+{
+    int bucket;
+    free_list_header_t *header = (free_list_header_t *)start_pc;
+    free_list_footer_t *footer;
+
+    ASSERT(CACHE_PROTECTED(cache));
+    ASSERT(DYNAMO_OPTION(cache_shared_free_list) && cache->is_shared);
+    DODEBUG({
+        /* only count frees of actual fragments */
+        if (f != NULL)
+            cache->free_size_histogram[get_histogram_bucket(size)]++; 
+    });
+    ASSERT(dynamo_resetting || fcache_pc_in_live_unit(cache, start_pc));
+
+    if (size > MAX_FREE_ENTRY_SIZE) {
+        /* FIXME PR 203913: fifo_prepend_empty can handle larger sizes, but
+         * we can't: we would need to split into two empty slots.
+         * For now we bail and leak.
+         */
+        ASSERT_NOT_REACHED();
+        return;
+    }
+
+    /* check next slot first before we potentially shift back from coalescing */
+    if (unit->cur_pc > start_pc + size) {
+        fragment_t *subseq = FRAG_NEXT_SLOT(start_pc, size);
+        if (FRAG_IS_FREE_LIST(subseq)) {
+            /* this is a free list entry, coalesce with it */
+            free_list_header_t *next_header = FRAG_NEXT_FREE(start_pc, size);
+            /* only coalesce if not over size limit */
+            if ((uint)next_header->size + size <= MAX_FREE_ENTRY_SIZE) {
+                uint next_bucket = find_free_list_bucket(next_header->size);
+                LOG(GLOBAL, LOG_CACHE, 4, 
+                    "add_to_free_list: coalesce w/ next %s bucket[%d] %d bytes @"PFX"\n",
+                    cache->name, next_bucket, next_header->size, next_header);
+                size += next_header->size;
+                /* OPTIMIZATION: if still in same bucket can eliminate some work */
+                remove_from_free_list(cache, next_bucket, next_header
+                                      _IF_DEBUG(true/*coalesce*/));
+                /* fall-through and add to free list anew
+                 * (potentially coalesce with prev as well) 
+                 */
+                STATS_FCACHE_ADD(cache, free_coalesce_next, 1);
+            } else {
+                /* FIXME: if we have a few giant free entries we should
+                 * free the whole unit.
+                 */
+                STATS_FCACHE_ADD(cache, free_coalesce_too_big, 1);
+            }
+        } else {
+            /* A real fragment_t: mark it.
+             * This is the only place we need to mark, as we disallow free lists at
+             * the end of the current unit (so an appended fragment_t will never need
+             * this flag).
+             */
+            LOG(GLOBAL, LOG_CACHE, 4, 
+                "add_to_free_list: marking next F%d("PFX")."PFX" as after-free\n",
+                subseq->id, subseq->tag, subseq->start_pc);
+            ASSERT(FIFO_UNIT(subseq) == unit);
+            ASSERT(FRAG_HDR_START(subseq) == start_pc + size);
+            /* can already be marked if we're called due to a split */
+            if (!TEST(FRAG_FOLLOWS_FREE_ENTRY, subseq->flags)) {
+                if (TEST(FRAG_SHARED, subseq->flags))
+                    acquire_recursive_lock(&change_linking_lock);
+                subseq->flags |= FRAG_FOLLOWS_FREE_ENTRY;
+                if (TEST(FRAG_SHARED, subseq->flags))
+                    release_recursive_lock(&change_linking_lock);
+            }
+        }
+    }
+    /* If we're actually freeing a real fragment_t, we can coalesce with prev.
+     * Other reasons to come here should never have a free entry in the prev slot.
+     */
+    if (f != NULL && TEST(FRAG_FOLLOWS_FREE_ENTRY, f->flags)) {
+        /* coalesce with prev */
+        free_list_footer_t *prev_footer = (free_list_footer_t *)
+            (start_pc - sizeof(free_list_footer_t));
+        free_list_header_t *prev_header = free_list_header_from_footer(prev_footer);
+        uint prev_bucket = find_free_list_bucket(prev_footer->size);
+        /* only coalesce if not over size limit */
+        uint new_size = (uint)prev_header->size + size;
+        if (new_size <= MAX_FREE_ENTRY_SIZE) {
+            LOG(GLOBAL, LOG_CACHE, 4, 
+                "add_to_free_list: coalesce w/ prev %s bucket[%d] %d bytes @"PFX"\n",
+                cache->name, prev_bucket, prev_header->size, prev_header);
+            size += prev_header->size;
+            header = prev_header;
+            start_pc = (cache_pc) header;
+            /* OPTIMIZATION: if still in same bucket can eliminate some work */
+            remove_from_free_list(cache, prev_bucket, prev_header
+                                  _IF_DEBUG(true/*coalesce*/));
+            /* fall-through and add to free list anew */
+            STATS_FCACHE_ADD(cache, free_coalesce_prev, 1);
+        } else {
+            /* see FIXMEs for next-coalesce-too-large above */
+            STATS_FCACHE_ADD(cache, free_coalesce_too_big, 1);
+        }
+    }
+
+    /* Invariant: no free entry can end at the append point of the current unit.
+     * It we want to relax this we must mark an appended fragment as
+     * FRAG_FOLLOWS_FREE_ENTRY.  We do allow a free entry at the very end of the
+     * now-full cur unit.  FIXME: this code is fragile wrt extend_unit_end's
+     * fifo_prepend_empty(), which wants a free list at the end of the unit, and
+     * only avoids disaster here by not incrementing cur_pc until afterward, so
+     * our condition here is not triggered.  We could add another param to
+     * distinguish (f==NULL is not good enough as splits also call here).
+     */
+    if (unit == cache->units && unit->cur_pc == start_pc + size) {
+        /* free space at end of current unit: just adjust cur_pc */
+        ASSERT((unit->full && unit->cur_pc == unit->end_pc) ||
+               (!unit->full && unit->cur_pc < unit->end_pc));
+        unit->cur_pc = start_pc;
+        unit->full = false;
+        STATS_FCACHE_ADD(cache, return_last, 1);
+        STATS_FCACHE_SUB(cache, claimed, size);
+        STATS_SUB(fcache_combined_claimed, size);
+        return;
+    }
+
+    /* ok to call w/ small size if you know it will be coalesced or returned,
+     * but not if it's to be its own entry
+     */
+    ASSERT(size >= MIN_FCACHE_SLOT_SIZE(cache));
+
+    bucket = find_free_list_bucket(size);
+
+    header->next = cache->free_list[bucket];
+    header->prev = NULL;
+    ASSERT_TRUNCATE(header->size, ushort, size);
+    header->size = (ushort) size;
+    header->flags = FRAG_FAKE | FRAG_FCACHE_FREE_LIST;
+    footer = free_list_footer_from_header(header);
+    ASSERT_TRUNCATE(footer->size, ushort, size);
+    footer->size = (ushort) size;
+    if (cache->free_list[bucket] != NULL) {
+        ASSERT(cache->free_list[bucket]->prev == NULL);
+        cache->free_list[bucket]->prev = header;
+    }
+    cache->free_list[bucket] = header;
+    /* FIXME: case 7318 we should keep sorted */
+
+    DOSTATS({
+        /* FIXME: we could split freed into pure-freed, split-freed, and coalesce-freed */
+        cache->free_stats_freed[bucket]++;
+        cache->free_stats_charge[bucket] += size;
+        LOG(GLOBAL, LOG_CACHE, 4, 
+            "add_to_free_list: %s bucket[%d] %d bytes @"PFX"\n",
+            cache->name, bucket, size, start_pc);
+        /* assumption: caller has already adjusted cache's empty space stats */
+    });
+}
+
+static bool
+find_free_list_slot(dcontext_t *dcontext, fcache_t *cache, fragment_t *f, uint size)
+{
+    int bucket;
+    cache_pc start_pc;
+    uint free_size;
+    free_list_header_t *header = NULL;
+    fcache_unit_t *unit = NULL;
+    DEBUG_DECLARE(bool split_empty = false;)
+
+    ASSERT(!USE_FIFO(f) && USE_FREE_LIST(f));
+    ASSERT(CACHE_PROTECTED(cache));
+    ASSERT(DYNAMO_OPTION(cache_shared_free_list) && cache->is_shared);
+    LOG(THREAD, LOG_CACHE, 4, "find_free_list_slot: %d bytes\n", size);
+    DODEBUG({ cache->request_size_histogram[get_histogram_bucket(size)]++;});
+
+    if (size > MAX_FREE_ENTRY_SIZE) {
+        /* FIXME: we may have adjacent un-coalesced free slots we could use */
+        return false;
+    }
+
+    /* Strategy: search first in current bucket and if nothing is
+     * found, then upgrade.  Coalescing and splitting allows a blind
+     * always-upgrade strategy.
+     */
+    /* case 7318 for discussion on additional search strategies
+     * FIXME: for any bucket if we are
+     * too close to the top we should look in the next bucket?  (We
+     * used to have a scheme for FREE_LIST_BOTTOM_BUCKET_MARGIN)
+     */
+    for (bucket = find_free_list_bucket(size); bucket < FREE_LIST_SIZES_NUM; bucket++) {
+        if (cache->free_list[bucket] == NULL)
+            continue;
+
+        /* search for first entry larger than size in current bucket */
+        /* note that for a bucket of only one size, this search should
+         * finish immediately */
+        header = cache->free_list[bucket];
+
+        while (header != NULL && header->size < size) {
+            /* FIXME: if we keep the list sorted, we'd not waste too
+             * much space by picking the first large enough slot */
+            /* FIXME: if we want to coalesce here we can act on any
+             * fragment while walking list and make sure that it is
+             * coalesced */
+            header = header->next;
+        }
+        if (header != NULL)
+            break;
+    }
+    if (bucket >= FREE_LIST_SIZES_NUM)
+        return false;
+
+    DOSTATS({
+        if (bucket > find_free_list_bucket(size))
+            STATS_FCACHE_ADD(cache, free_use_larger, 1);
+    });
+    ASSERT(header != NULL);
+    /* found big enough free slot, extract from free list */
+    remove_from_free_list(cache, bucket, header _IF_DEBUG(false/*!coalesce*/));
+
+    start_pc = (cache_pc) header;
+    free_size = header->size;
+    ASSERT(free_size >= size);
+    ASSERT(free_size <= MAX_FREE_ENTRY_SIZE);
+
+    /* FIXME: if this vmarea lookup is expensive, we can also keep the
+     * unit ptr/tag in the free header.*/
+    unit = fcache_lookup_unit(start_pc);
+    ASSERT(unit != NULL);
+    ASSERT(fcache_pc_in_live_unit(cache, start_pc));
+
+    /* FIXME: if bucket sizes are spread apart further than
+     * MIN_EMPTY_HOLE() this will also kick in.  Currently an
+     * issue only for traces and the bucket [112, 172).
+     */
+    /* if enough room left over, split it off as its own free slot */
+    if (free_size - size > MIN_EMPTY_HOLE(cache)) {
+        DODEBUG({
+            cache->free_stats_split[bucket]++;
+            split_empty = true;
+        });
+        STATS_FCACHE_ADD(cache, free_split, 1);
+
+        add_to_free_list(dcontext, cache, unit, NULL, start_pc + size, free_size - size);
+        free_size = size;
+        /* next fragment_t remains marked as FRAG_FOLLOWS_FREE_ENTRY */
+    } else {
+        /* taking whole entry */
+        if (unit->cur_pc > start_pc + free_size) {
+            fragment_t *subseq = FRAG_NEXT_SLOT(start_pc, free_size);
+            /* remove FRAG_FOLLOWS_FREE_ENTRY flag from subsequent fragment_t, if it exists */
+            if (!FRAG_IS_FREE_LIST(subseq)) {
+                LOG(GLOBAL, LOG_CACHE, 4, 
+                    "find_free_list_slot: un-marking next F%d("PFX")."PFX" as after-free\n",
+                    subseq->id, subseq->tag, subseq->start_pc);
+                ASSERT(FIFO_UNIT(subseq) == unit);
+                ASSERT(FRAG_HDR_START(subseq) == start_pc + free_size);
+                if (TEST(FRAG_SHARED, subseq->flags))
+                    acquire_recursive_lock(&change_linking_lock);
+                ASSERT(TEST(FRAG_FOLLOWS_FREE_ENTRY, subseq->flags));
+                subseq->flags &= ~FRAG_FOLLOWS_FREE_ENTRY;
+                if (TEST(FRAG_SHARED, subseq->flags))
+                    release_recursive_lock(&change_linking_lock);
+            } else {
+                /* shouldn't be free list entry following this one, unless
+                 * unable to coalesce due to ushort size limits 
+                 */
+                ASSERT(free_size +
+                       FRAG_NEXT_FREE(start_pc, free_size)->size > MAX_FREE_ENTRY_SIZE);
+            }
+        }
+    }
+
+    place_fragment(dcontext, f, unit, start_pc);
+    FRAG_SIZE_ASSIGN(f, free_size);
+
+    DOSTATS({
+        LOG(GLOBAL, LOG_CACHE, 4, "find_free_list_slot: %s bucket[%d]%s"
+            " %d bytes @"PFX" requested %d, waste=%d\n", 
+            cache->name, bucket, split_empty ? "split" : "", 
+            free_size, start_pc, size, (free_size - size));
+
+    });
+    STATS_FCACHE_ADD(cache, align, free_size - size);
+    STATS_FCACHE_SUB(cache, empty, free_size);
+    return true;
+}
+
+/* separate routine b/c it may recurse */
+static void
+add_fragment_common(dcontext_t *dcontext, fcache_t *cache, fragment_t *f, uint slot_size)
+{
+    fragment_t *fifo = NULL;
+    fcache_unit_t *unit;
+    ASSERT(CACHE_PROTECTED(cache));
+
+    /* first, check fifo for empty slot 
+     * don't look for empty of appropriate size -- kick out the neighbors!
+     * we've found that works better, else empty list too long, and splitting
+     * it by size ruins the fifo
+     */
+    if (USE_FREE_LIST_FOR_CACHE(cache)) {
+        if (DYNAMO_OPTION(cache_shared_free_list) &&
+            find_free_list_slot(dcontext, cache, f, slot_size))
+            return;
+        /* If no free list, no way to insert fragment into middle of cache */
+    } else if (USE_FIFO_FOR_CACHE(cache)) {
+        fifo = cache->fifo;
+        while (fifo != NULL && FRAG_EMPTY(fifo)) {
+            unit = FIFO_UNIT(fifo);
+            if ((ptr_uint_t)(unit->end_pc - FRAG_HDR_START(fifo)) >= slot_size) {
+                /* try to replace fifo and possibly subsequent frags with f
+                 * could fail if un-deletable frags
+                 */
+                LOG(THREAD, LOG_CACHE, 4, "\ttrying to fit in empty slot\n");
+                DOLOG(4, LOG_CACHE, { verify_fifo(dcontext, cache); });
+                if (replace_fragments(dcontext, cache, unit, f, fifo, slot_size))
+                    return;
+            }
+            fifo = FIFO_NEXT(fifo);
+        }
+    }
+
+    /* second, look for room at end, if cache never filled up before */
+    unit = cache->units; /* most recent is only potentially non-full unit */
+    if (!unit->full && (ptr_uint_t)(unit->end_pc - unit->cur_pc) >= slot_size) {
+        /* just add to end */
+        size_t extra;
+        place_fragment(dcontext, f, unit, unit->cur_pc);
+        extra = extend_unit_end(dcontext, cache, unit, slot_size, false);
+        if (extra > 0) {
+            ASSERT(CHECK_TRUNCATE_TYPE_uint(extra));
+            FRAG_SIZE_ASSIGN(f, slot_size + (uint)extra);
+            STATS_FCACHE_ADD(cache, align, extra);
+        }
+        if (USE_FIFO_FOR_CACHE(cache))
+            fifo_append(cache, f);
+        LOG(THREAD, LOG_CACHE, 4,
+            "\tadded F%d to unfilled unit @"PFX" (%d [/%d] bytes left now)\n",
+            f->id, f->start_pc, unit->end_pc - unit->cur_pc, UNIT_RESERVED_SIZE(unit));
+        return;
+    }
+
+    /* third, resize and try again.
+     * for fifo caches, don't resize unless regen/replace ratio warrants it.
+     */
+    if (!USE_FIFO_FOR_CACHE(cache) ||
+        cache->is_coarse || 
+        check_regen_replace_ratio(dcontext, cache, slot_size)) {
+        LOG(THREAD, LOG_CACHE, 3, "\tcache is full, trying to acquire more space\n");
+        if (try_for_more_space(dcontext, cache, unit, slot_size)) {
+            add_fragment_common(dcontext, cache, f, slot_size);
+            return;
+        }
+    }
+
+    /* all our subsequent schemes require a FIFO (non-FIFO caches will have
+     * been resized in step 3)
+     */
+    ASSERT(USE_FIFO_FOR_CACHE(cache));
+
+    /* finally, boot somebody out -- in FIFO order, only go to next if
+     * not enough room from victim to end of cache.
+     * fifo should be pointing to first non-empty slot!
+     */
+    if (replace_fifo(dcontext, cache, f, slot_size, fifo))
+        return;
+    
+    /* if get here, no room, so must resize */
+    LOG(THREAD, LOG_CACHE, 3,
+        "\ttried to avoid resizing, but no way around it for fragment size %d\n",
+        slot_size);
+    if (try_for_more_space(dcontext, cache, unit, slot_size)) {
+        add_fragment_common(dcontext, cache, f, slot_size);
+        return;
+    }
+
+    /* if get here, must have a very large fragment relative to constrained cache size,
+     * plus some undeletable fragments right in the middle.
+     * abort the trace, hopefully making fragments re-deletable, and try again.
+     */
+    trace_abort(dcontext);
+    if (replace_fifo(dcontext, cache, f, slot_size, cache->fifo))
+        return;
+    
+    /* could still get here if undeletable fragments...but what can we do?
+     * current impl only makes trace-in-progress undeletable, right?
+     * so should never get here
+     */
+    ASSERT_NOT_REACHED();
+}
+
+void
+fcache_shift_start_pc(dcontext_t *dcontext, fragment_t *f, uint space)
+{
+    fcache_unit_t *unit;
+    fcache_t *cache = NULL;
+
+    if (space == 0)
+        return;
+
+    /* must hold cache lock across any set of a fragment's start_pc or size
+     * once that fragment is in a cache, as contig-cache-walkers need a consistent view!
+     */
+    if (TEST(FRAG_SHARED, f->flags)) {
+        /* optimization: avoid unit lookup for private fragments.
+         * this assumes that no cache lock is used for private fragments!
+         */
+        unit = FIFO_UNIT(f);
+        ASSERT(unit != NULL);
+        cache = unit->cache;
+        PROTECT_CACHE(cache, lock);
+    }
+
+    /* we back align to remove the padding */
+    ASSERT(ALIGNED(f->start_pc, START_PC_ALIGNMENT));
+
+    /* adjusting start_pc */
+    ASSERT(space <= START_PC_ALIGNMENT-1); /* most we can shift */
+    ASSERT(PAD_JMPS_SHIFT_START(f->flags));
+
+    /* FIXME : no need to set this memory to anything, but is easier to debug
+     * if it's valid instructions */
+    SET_TO_DEBUG(f->start_pc, space);
+
+    f->start_pc += space;
+    ASSERT_TRUNCATE(f->size, ushort, (f->size - space));
+    f->size = (ushort) (f->size - space);
+    DODEBUG({
+        if (space > 0) {
+            STATS_PAD_JMPS_ADD(f->flags, num_start_pc_shifted, 1);
+            STATS_PAD_JMPS_ADD(f->flags, start_pc_shifted_bytes, space);
+        }
+    });
+
+    if (TEST(FRAG_SHARED, f->flags))
+        PROTECT_CACHE(cache, unlock);
+}
+
+void 
+fcache_return_extra_space(dcontext_t *dcontext, fragment_t *f, size_t space_in)
+{
+    fcache_unit_t *unit = FIFO_UNIT(f);
+    fcache_t *cache = unit->cache;
+    uint returnable_space, slot_padding, space;
+    cache_pc end_addr;
+    uint min_padding = 0;
+    byte *returnable_start;
+    bool released = false;
+    PROTECT_CACHE(cache, lock);
+
+    /* truncate up front */
+    ASSERT_TRUNCATE(space, uint, space_in);
+    space = (uint) space_in;
+
+    DOSTATS({
+        if (ALIGN_FORWARD(f->size + HEADER_SIZE(f) - space,
+                          SLOT_ALIGNMENT(cache)) <
+            MIN_FCACHE_SLOT_SIZE(cache))
+            STATS_INC(num_final_fragment_too_small);
+    });
+
+    /* get the total amount of free space at the end of the fragment including
+     * any end padding (for slot alignment etc., stored in fcache->extra) */
+    returnable_space = space + f->fcache_extra - HEADER_SIZE(f);
+    if (FRAG_SIZE(f) - returnable_space < MIN_FCACHE_SLOT_SIZE(cache)) {
+        min_padding = returnable_space - (FRAG_SIZE(f) - MIN_FCACHE_SLOT_SIZE(cache));
+        returnable_space = FRAG_SIZE(f) - MIN_FCACHE_SLOT_SIZE(cache);
+    }
+    /* now adjust for slot alignment padding */
+    end_addr = FRAG_HDR_START(f)+FRAG_SIZE(f)-returnable_space;
+    ASSERT_TRUNCATE(slot_padding, uint, ALIGN_FORWARD(end_addr, SLOT_ALIGNMENT(cache)) -
+                    (ptr_uint_t)end_addr);
+    slot_padding = (uint) (ALIGN_FORWARD(end_addr, SLOT_ALIGNMENT(cache)) -
+        (ptr_uint_t)end_addr);
+    returnable_space -= slot_padding;
+    returnable_start = FRAG_HDR_START(f)+FRAG_SIZE(f)-returnable_space;
+    ASSERT(FRAG_SIZE(f) - returnable_space >= MIN_FCACHE_SLOT_SIZE(cache));
+    ASSERT(ALIGNED(returnable_space, SLOT_ALIGNMENT(cache)));
+    ASSERT(ALIGNED(returnable_start, SLOT_ALIGNMENT(cache)));
+
+    if (returnable_space > 0) {
+        STATS_INC(pad_jmps_fragments_overestimated);
+        /* first check if f is the last fragment in the unit */
+        if (FRAG_HDR_START(f)+FRAG_SIZE(f) == unit->end_pc) {
+            /* f is the last fragment in a full unit */
+            ASSERT(unit->full);
+            if (returnable_space >= MIN_UNIT_END_HOLE(cache)) {
+                if (unit == cache->units) {
+                    /* we just filled this unit, mark un-full and adjust cur_pc */
+                    ASSERT(unit->cur_pc == unit->end_pc);
+                    unit->full = false;
+                    unit->cur_pc -= returnable_space;
+                    STATS_FCACHE_SUB(cache, claimed, returnable_space);
+                    STATS_SUB(fcache_combined_claimed, returnable_space);
+                    released = true;
+                } else {
+                    /* create a new empty slot */
+                    fifo_prepend_empty(dcontext, cache, unit, NULL,
+                                       returnable_start, returnable_space);
+                    released = true;
+                }
+            }
+        } else if (FRAG_HDR_START(f)+FRAG_SIZE(f) == unit->cur_pc) {
+            /* f is the last fragment in a non-full unit */
+            ASSERT(!unit->full);
+            unit->cur_pc -= returnable_space;
+            STATS_FCACHE_SUB(cache, claimed, returnable_space);
+            STATS_SUB(fcache_combined_claimed, returnable_space);
+            released = true;
+        } else {
+            if (!(DYNAMO_OPTION(cache_shared_free_list) && cache->is_shared)) {
+                fragment_t *next_f;
+                /* since we aren't at the end of the used space in the unit there must
+                 * be a fragment or empty slot after us */
+                next_f = FRAG_NEXT_SLOT(FRAG_HDR_START(f), FRAG_SIZE(f));
+                ASSERT(next_f != NULL); /* sanity check, though not very good one */
+                if (FRAG_EMPTY(next_f)) {
+                    STATS_FCACHE_ADD(cache, empty, returnable_space);
+                    FRAG_START_ASSIGN(next_f, returnable_start + HEADER_SIZE(f));
+                    *((fragment_t **)returnable_start) = next_f;
+                    FRAG_SIZE_ASSIGN(next_f, FRAG_SIZE(next_f) + returnable_space);
+                    released = true;
+                }
+            }
+            if (!released) {
+                /* return excess if next slot is a free list (will be coalesced) or
+                 * if excess is large enough by itself
+                 */
+                fragment_t *subseq = FRAG_NEXT_SLOT(FRAG_HDR_START(f), FRAG_SIZE(f));
+                ASSERT(FRAG_HDR_START(f)+FRAG_SIZE(f) < unit->cur_pc);
+                if ((FRAG_IS_FREE_LIST(subseq) &&
+                     /* make sure will coalesce
+                      * FIXME: fragile if coalesce rules change -- perhaps have
+                      * free list routine return failure?
+                      */
+                     returnable_space + FRAG_NEXT_FREE(FRAG_HDR_START(f),
+                                                       FRAG_SIZE(f))->size <=
+                     MAX_FREE_ENTRY_SIZE) ||
+                    returnable_space >= MIN_EMPTY_HOLE(cache)) {
+                    fifo_prepend_empty(dcontext, cache, unit, NULL, returnable_start, 
+                                       returnable_space);
+                    released = true;
+                    DOSTATS({
+                        if (FRAG_IS_FREE_LIST(subseq))
+                            STATS_INC(pad_jmps_excess_next_free);
+                    });
+                }
+            }
+        }
+   }
+
+    /* even if returnable_space is 0, space is not, and we need to shift it
+     * from f->size to f->fcache_extra
+     */
+    /* update fragment values + sanity check */
+    ASSERT(f->fcache_extra + space == slot_padding + HEADER_SIZE(f) +
+           returnable_space + min_padding);
+    ASSERT_TRUNCATE(f->size, ushort, (f->size - space));
+    f->size = (ushort) (f->size - space);
+    FRAG_SIZE_ASSIGN(f, f->size + HEADER_SIZE(f) + FRAG_START_PADDING(f) + 
+                     slot_padding + (released ? 0 : returnable_space) + min_padding);
+    ASSERT(FRAG_SIZE(f) >= MIN_FCACHE_SLOT_SIZE(cache));
+    DOSTATS({
+        if (released)
+            STATS_FCACHE_SUB(cache, used, returnable_space);
+        else if (returnable_space > 0) {
+            STATS_INC(pad_jmps_excess_wasted);
+            STATS_ADD(pad_jmps_excess_wasted_bytes, returnable_space);
+        }
+    });
+
+    DOLOG(3, LOG_CACHE, {
+        if (USE_FIFO_FOR_CACHE(cache))
+            verify_fifo(dcontext, cache); 
+    });
+
+    PROTECT_CACHE(cache, unlock);
+
+    STATS_PAD_JMPS_ADD(f->flags, extra_space_released, 
+                       released ? returnable_space : 0);
+}
+
+static fcache_t *
+get_cache_for_new_fragment(dcontext_t *dcontext, fragment_t *f)
+{
+    thread_units_t *tu = (thread_units_t *) dcontext->fcache_field;
+    fcache_t *cache = NULL;
+    if (TEST(FRAG_SHARED, f->flags)) {
+        if (TEST(FRAG_COARSE_GRAIN, f->flags)) {
+            /* new fragment must be targeting the one non-frozen unit */
+            coarse_info_t *info = get_executable_area_coarse_info(f->tag);
+            ASSERT(info != NULL);
+            if (info != NULL && info->frozen)
+                info = info->non_frozen;
+            ASSERT(info != NULL);
+            if (info->cache == NULL) {
+                /* due to lock ordering problems we must create the cache
+                 * before acquiring the info->lock
+                 */
+                cache = fcache_cache_init(GLOBAL_DCONTEXT, f->flags, true);
+                mutex_lock(&info->lock);
+                if (info->cache == NULL) {
+                    cache->coarse_info = info;
+                    coarse_unit_init(info, cache);
+                    ASSERT(cache == info->cache);
+                    mutex_unlock(&info->lock);
+                } else {
+                    /* w/ bb_building_lock we shouldn't have a race here */
+                    ASSERT_CURIOSITY(false && "race in creating coarse cache");
+                    mutex_unlock(&info->lock);
+                    fcache_cache_free(GLOBAL_DCONTEXT, cache, true);
+                }
+            }
+            ASSERT(info->cache != NULL);
+            ASSERT(((fcache_t *)info->cache)->coarse_info == info);
+            return (fcache_t *) info->cache;
+        } else {
+            if (IN_TRACE_CACHE(f->flags))
+                return shared_cache_trace;
+            else
+                return shared_cache_bb;
+        }
+    } else {
+        /* thread-private caches are delayed */
+        if (IN_TRACE_CACHE(f->flags)) {
+            if (tu->trace == NULL) {
+                tu->trace = fcache_cache_init(dcontext, FRAG_IS_TRACE, true);
+                ASSERT(tu->trace != NULL);
+                LOG(THREAD, LOG_CACHE, 1, "Initial trace cache is %d KB\n",
+                    tu->trace->init_unit_size/1024);
+            }
+            return tu->trace;
+        } else {
+            if (tu->bb == NULL) {
+                tu->bb = fcache_cache_init(dcontext, 0/*private bb*/, true);
+                ASSERT(tu->bb != NULL);
+                LOG(THREAD, LOG_CACHE, 1, "Initial basic block cache is %d KB\n",
+                    tu->bb->init_unit_size/1024);
+            }
+            return tu->bb;
+        }
+    }
+    ASSERT_NOT_REACHED();
+    return NULL;
+}
+
+void 
+fcache_add_fragment(dcontext_t *dcontext, fragment_t *f)
+{
+    thread_units_t *tu = (thread_units_t *) dcontext->fcache_field;
+    uint slot_size;
+    fcache_t *cache = get_cache_for_new_fragment(dcontext, f);
+    ASSERT(cache != NULL);
+    PROTECT_CACHE(cache, lock);
+    /* set start_pc to START_PC_ALIGNMENT so will work with FRAG_START_PADDING */
+    f->start_pc = (cache_pc)(ptr_uint_t)START_PC_ALIGNMENT;
+
+    /* delayed unmap...presumably ok to do now! */
+    if (tu->pending_unmap_pc != NULL) {
+        /* de-allocate old memory */
+        /* remove from interval data struct first to avoid races w/ it
+         * being re-used and not showing up in in_fcache
+         */
+        vmvector_remove(fcache_unit_areas, tu->pending_unmap_pc,
+                        tu->pending_unmap_pc+tu->pending_unmap_size);
+        /* caller must dec stats since here we don't know type of cache */
+        heap_munmap(tu->pending_unmap_pc, tu->pending_unmap_size);
+        tu->pending_unmap_pc = NULL;
+    }
+
+    /* starting address of a fragment and its size should always
+     * be cache-line-aligned.  we use a 4-byte header as a backpointer
+     * to the fragment_t.
+     */
+    slot_size = f->size + HEADER_SIZE(f);
+    if (slot_size < MIN_FCACHE_SLOT_SIZE(cache)) {
+        STATS_INC(num_fragment_too_small);
+        slot_size = MIN_FCACHE_SLOT_SIZE(cache);
+    }
+    slot_size = (uint) ALIGN_FORWARD(slot_size, SLOT_ALIGNMENT(cache));
+    ASSERT(slot_size >= f->size + HEADER_SIZE(f));
+    ASSERT_TRUNCATE(f->fcache_extra, byte, (slot_size - f->size));
+    f->fcache_extra = (byte) (slot_size - f->size);
+    LOG(THREAD, LOG_CACHE, 4,
+        "fcache_add_fragment to %s cache (size %dKB): F%d w/ size %d (=> %d)\n",
+        cache->name, cache->units->size/1024, f->id, f->size, slot_size);
+    
+    add_fragment_common(dcontext, cache, f, slot_size);
+    ASSERT(!PAD_JMPS_SHIFT_START(f->flags) ||
+           ALIGNED(f->start_pc, START_PC_ALIGNMENT)); /* for start_pc padding to work */
+    DOLOG(3, LOG_CACHE, {
+        if (USE_FIFO_FOR_CACHE(cache))
+            verify_fifo(dcontext, cache); 
+    });
+    PROTECT_CACHE(cache, unlock);
+}
+
+void
+fcache_remove_fragment(dcontext_t *dcontext, fragment_t *f)
+{
+    fcache_unit_t *unit = FIFO_UNIT(f);
+    fcache_t *cache = unit->cache;
+
+    /* should only be deleted through proper synched channels */
+    ASSERT(dcontext != GLOBAL_DCONTEXT || dynamo_exited || dynamo_resetting ||
+           TEST(FRAG_WAS_DELETED, f->flags) || is_self_allsynch_flushing());
+    PROTECT_CACHE(cache, lock);
+
+    LOG(THREAD, LOG_CACHE, 4, "fcache_remove_fragment: F%d from %s unit\n",
+        f->id, cache->name);
+    
+    DOSTATS({ removed_fragment_stats(dcontext, cache, f); });
+    STATS_FCACHE_SUB(cache, used, FRAG_SIZE(f));
+
+#ifdef DEBUG_MEMORY
+    /* Catch stale execution by filling w/ int3.
+     * We do this before fifo_prepend_empty to avoid figuring whether
+     * to leave alone th1 st 4 bytes of fragment space or not (it's
+     * used to store the size for cache_shared_free_list).
+     * FIXME: put in the rest of the patterns and checks to make this
+     * parallel to heap DEBUG_MEMORY (==case 5657)
+     */
+    memset(f->start_pc, DEBUGGER_INTERRUPT_BYTE, f->size);
+#endif
+
+    /* if the entire unit is being freed, do not place individual fragments in 
+     * the unit on free lists or the FIFO
+     */
+    if (!unit->pending_free) {
+        /* empty slots always go on front */
+        fifo_prepend_empty(dcontext, cache, unit, f, FRAG_HDR_START(f), FRAG_SIZE(f));
+        if (USE_FIFO(f)) {
+            fifo_remove(dcontext, cache, f);
+            DOLOG(3, LOG_CACHE, { verify_fifo(dcontext, cache); });
+        }
+    }
+    PROTECT_CACHE(cache, unlock);
+}
+
+
+#ifdef SIDELINE
+dcontext_t * 
+get_dcontext_for_fragment(fragment_t *f)
+{
+    fcache_unit_t *unit = fcache_lookup_unit(f->start_pc);
+    ASSERT(unit != NULL);
+    return unit->dcontext;
+}
+#endif
+
+/***************************************************************************
+ * FLUSHING OF UNITS
+ *
+ * General strategy: first, put units to flush on the global units_to_flush
+ * list.  At a nolinking point, use flush synch to get other threads out of
+ * the target units, and then walk each unit using contiguous header walk.
+ * Remove free list and lazy entries, and chain up all fragments to pass to
+ * flush unlink_shared().  Move units to a global units_to_free list, recording
+ * their flushtimes.  We'll be notified when a pending delete entry is
+ * freed, and we can then check which units are safe to free.
+ */
+
+bool
+fcache_is_flush_pending(dcontext_t *dcontext)
+{
+    thread_units_t *tu = (thread_units_t *) dcontext->fcache_field;
+    return tu->pending_flush;
+}
+
+/* accepts a chain of units linked by next_local.
+ * caller must set pending_free and flushtime fields.
+ */
+static void
+append_units_to_free_list(fcache_unit_t *u)
+{
+    mutex_lock(&unit_flush_lock);
+
+    /* must append to keep in increasing flushtime order */
+    if (allunits->units_to_free_tail == NULL) {
+        ASSERT(allunits->units_to_free == NULL);
+        allunits->units_to_free = u;
+    } else {
+        ASSERT(allunits->units_to_free->flushtime <= u->flushtime);
+        ASSERT(allunits->units_to_free_tail->next_local == NULL);
+        ASSERT(allunits->units_to_free_tail != u);
+        allunits->units_to_free_tail->next_local = u;
+    }
+    STATS_ADD_PEAK(cache_units_tofree, 1);
+    /* support adding a chain */
+    ASSERT(u->flushtime > 0);
+    ASSERT(u->pending_free);
+    while (u->next_local != NULL) {
+        if (u->cache != NULL)
+            remove_unit_from_cache(u);
+        ASSERT(u->cache == NULL);
+        STATS_ADD_PEAK(cache_units_tofree, 1);
+        ASSERT(u->flushtime <= u->next_local->flushtime);
+        u = u->next_local;
+        ASSERT(u->flushtime > 0);
+        ASSERT(u->pending_free);
+    }
+    allunits->units_to_free_tail = u;
+    ASSERT(allunits->units_to_free_tail->next_local == NULL);
+
+    mutex_unlock(&unit_flush_lock);
+}
+
+/* It is up to the caller to ensure it's safe to string the fragments in
+ * unit into a list, by only calling us between stage1 and stage2 of
+ * flushing (there's no other synch that is safe).
+ */
+static fragment_t *
+chain_fragments_for_flush(dcontext_t *dcontext, fcache_unit_t *unit, fragment_t **tail)
+{
+    fragment_t *list = NULL, *f = NULL, *prev_f = NULL;
+    cache_pc pc;
+    ASSERT(is_self_flushing());
+    LOG(THREAD, LOG_CACHE, 4, "\tchaining fragments in unit "PFX"-"PFX"\n",
+        unit->start_pc, unit->end_pc);
+    /* FIXME: we walk all fragments here just to call
+     * vm_area_remove_fragment(), and do another complete walk in
+     * unlink_fragments_for_deletion() -- can we reduce to one walk?
+     */
+    pc = unit->start_pc;
+    while (pc < unit->cur_pc) {
+        bool add_to_list = false;
+        f = *((fragment_t **)pc);
+        LOG(THREAD, LOG_CACHE, 5, "\treading "PFX" -> "PFX"\n", pc, f);
+        if (USE_FREE_LIST_FOR_CACHE(unit->cache)) {
+            if (FRAG_IS_FREE_LIST(f)) {
+                /* we're going to free the whole unit so we have to
+                 * remove this entry from the free list
+                 */
+                /* while flush synch is enough, cleaner to hold official lock */
+                free_list_header_t *cur_free = ((free_list_header_t *) pc);
+                int bucket = find_free_list_bucket(cur_free->size);
+                LOG(THREAD, LOG_CACHE, 5,
+                    "\tremoving free list entry "PFX" size %d bucket [%d]\n",
+                    cur_free, cur_free->size, bucket);
+                /* we officially grab the lock for free list manip, though
+                 * flush synch is enough.  we can't hold cache lock for this
+                 * entire routine b/c it has lower rank than the lazy_delete lock.
+                 */
+                PROTECT_CACHE(unit->cache, lock);
+                remove_from_free_list(unit->cache, bucket, cur_free
+                                      _IF_DEBUG(false/*!coalesce*/));
+                PROTECT_CACHE(unit->cache, unlock);
+                pc += cur_free->size;
+                continue;
+            }
+        }
+        ASSERT(f != NULL);
+        ASSERT(FIFO_UNIT(f) == unit);
+        ASSERT(FRAG_HDR_START(f) == pc);
+        ASSERT(!TEST(FRAG_CANNOT_DELETE, f->flags));
+        if (TEST(FRAG_WAS_DELETED, f->flags)) {
+            /* must be a consistency-flushed or lazily deleted fragment.
+             * while typically it will be deleted before the other fragments
+             * in this unit (since flushtime now is < ours, and lazy are deleted
+             * before pending-delete), there is a case where it will not be:
+             * if the lazy list hits its threshold and a new pending-delete entry
+             * is created before we free our unit-flush pending entry, the
+             * lazy pending entry will be freed AFTER flushed unit's entries.
+             * we must remove from the lazy list, and go ahead and put into the
+             * pending delete list, to keep all our dependences together.
+             * fragment_unlink_for_deletion() will not re-do the unlink for
+             * the lazy fragment.
+             * FIXME: this is inefficient b/c no prev ptr in lazy list
+             */
+            add_to_list = remove_from_lazy_deletion_list(dcontext, f);
+            /* if not found, we assume the fragment has already been moved to
+             * a pending delete entry, which must have a lower timestamp
+             * than ours, so we're all set since vm_area_check_shared_pending()
+             * now walks all to-be-freed entries in increasing flushtime order.
+             */
+            LOG(THREAD, LOG_CACHE, 5,
+                "\tlazily-deleted fragment F%d."PFX" removed from lazy list\n",
+                f->id, f->start_pc);
+        } else {
+            LOG(THREAD, LOG_CACHE, 5,
+                "\tadding F%d."PFX" to to-flush list\n", f->id, f->start_pc);
+            vm_area_remove_fragment(dcontext, f);
+            add_to_list = true;
+        }
+        if (add_to_list) {
+            if (prev_f == NULL)
+                list = f;
+            else
+                prev_f->next_vmarea = f;
+            prev_f = f;
+        }
+        /* advance to contiguously-next fragment_t in cache */
+        pc += FRAG_SIZE(f);
+    }
+    ASSERT(pc == unit->cur_pc);
+    /* If entire unit is free list entries or lazy-dels that
+     * were moved to pending-delete list, then we'll have no list and
+     * prev_f==NULL.
+     */
+    if (list == NULL) {
+        ASSERT(prev_f == NULL);
+        /* we have to finish off the flush synch so we just pass no list
+         * or region to flush_fragments_unlink_shared() 
+         */
+        STATS_INC(cache_units_flushed_nolive);        
+    } else {
+        ASSERT(list != NULL);
+        ASSERT(prev_f != NULL);
+        prev_f->next_vmarea = NULL;
+    }
+
+    if (tail != NULL)
+        *tail = prev_f;
+    return list;
+}
+
+/* Flushes all fragments in the units in the units_to_flush list and
+ * moves those units to the units_to_free list.
+ * This routine can only be called when !is_self_couldbelinking() and when
+ * no locks are held.
+ */
+bool
+fcache_flush_pending_units(dcontext_t *dcontext, fragment_t *was_I_flushed)
+{
+    thread_units_t *tu = (thread_units_t *) dcontext->fcache_field;
+    fcache_unit_t *unit_flushed = NULL;
+    fcache_unit_t *u, *local_to_flush;
+    bool not_flushed = true;
+    fragment_t *list_head = NULL, *list_tail = NULL;
+    fragment_t *list_add_head, *list_add_tail;
+    uint flushtime;
+    DEBUG_DECLARE(bool flushed;)
+
+    ASSERT(!is_self_couldbelinking());
+    ASSERT_OWN_NO_LOCKS();
+
+    if (!tu->pending_flush)
+        return not_flushed;
+    tu->pending_flush = false;
+
+    /* we grab a local copy to deal w/ races to flush these units up front
+     * rather than getting into the flush synch and finding someone beat us
+     */
+    mutex_lock(&unit_flush_lock);
+    local_to_flush = allunits->units_to_flush;
+    allunits->units_to_flush = NULL;
+    mutex_unlock(&unit_flush_lock);
+    if (local_to_flush == NULL)
+        return not_flushed;
+
+    LOG(THREAD, LOG_CACHE, 2, "flushing fragments in all pending units\n");
+
+    /* flush flag is private, and shared caches are synch-ed within
+     * pending_flush_units_in_cache, so no lock needed here
+     */
+    if (was_I_flushed != NULL)
+        unit_flushed = FIFO_UNIT(was_I_flushed);
+
+    /* First we have to synch w/ all threads to avoid races w/ other threads
+     * manipulating fragments in these units at the same time that we are (e.g.,
+     * lazily deleting a trace head).  Sure, the unit is not on the live
+     * list anymore, but the fragments are reachable.
+     */
+    DEBUG_DECLARE(flushed =)
+        flush_fragments_synch_unlink_priv(dcontext, EMPTY_REGION_BASE,
+                                          EMPTY_REGION_SIZE,
+                                          false/*don't have thread_initexit_lock*/,
+                                          false/*not invalidating exec areas*/,
+                                          false/*don't force synchall*/
+                                          _IF_DGCDIAG(NULL));
+    ASSERT(flushed);
+
+    KSTART(cache_flush_unit_walk);
+    for (u = local_to_flush; u != NULL; u = u->next_local) {
+        /* not implemented for private units, since no reason to use flush there */
+        ASSERT(u->cache->is_shared);
+
+        /* unit is no longer to be considered live -- no other thread should
+         * use it as a live unit from this point on
+         */
+        DODEBUG({ u->pending_flush = false; });
+
+        /* Indicate that individual fragments in this unit must not be put
+         * on free lists/FIFO empty slots.
+         * FIXME: would be nice to set this earlier when move to
+         * units_to_flush list, but then end up w/ freed fragments in the
+         * middle of the cache that we can't identify in our walk -- so we
+         * inefficiently put on free list and then take off in the walk.
+         */
+        u->pending_free = true;
+
+        if (u == unit_flushed)
+            not_flushed = false;
+
+        list_add_head = chain_fragments_for_flush(dcontext, u, &list_add_tail);
+        if (list_head == NULL) {
+            list_head = list_add_head;
+            list_tail = list_add_tail;
+        } else if (list_add_head != NULL) {
+            list_tail->next_vmarea = list_add_head;
+            list_tail = list_add_tail;
+        }
+        ASSERT(list_tail == NULL /*list empty so far*/ ||
+               list_tail->next_vmarea == NULL);
+
+        STATS_INC(cache_units_flushed);
+        STATS_DEC(cache_units_toflush);
+    }
+    KSTOP(cache_flush_unit_walk);
+    /* it's ok if list_head is NULL */
+
+    /* do the rest of the unlink and move the fragments to pending del list */
+    flush_fragments_unlink_shared(dcontext, EMPTY_REGION_BASE, EMPTY_REGION_SIZE,
+                                  list_head _IF_DGCDIAG(NULL));
+    flushtime = flushtime_global;
+    flush_fragments_end_synch(dcontext, false/*don't keep initexit_lock*/);
+    
+    for (u = local_to_flush; u != NULL; u = u->next_local) {
+        u->flushtime = flushtime;
+        LOG(THREAD, LOG_CACHE, 2,
+            "flushed fragments in unit "PFX"-"PFX" @flushtime %u\n",
+            u->start_pc, u->end_pc, u->flushtime);
+    }
+    
+    append_units_to_free_list(local_to_flush);
+
+    return not_flushed;
+}
+
+void
+fcache_free_pending_units(dcontext_t *dcontext, uint flushtime)
+{
+    fcache_unit_t *u, *nxt;
+    mutex_lock(&unit_flush_lock);
+    for (u = allunits->units_to_free; u != NULL; u = nxt) {
+        nxt = u->next_local;
+        /* free list must be sorted in increasing flushtime */
+        ASSERT(nxt == NULL || u->flushtime <= nxt->flushtime);
+        if (u->flushtime <= flushtime) {
+            if (u == allunits->units_to_free_tail) {
+                ASSERT(u == allunits->units_to_free);
+                ASSERT(nxt == NULL);
+                allunits->units_to_free_tail = NULL;
+            }
+            allunits->units_to_free = nxt;
+            LOG(THREAD, LOG_CACHE, 2, "freeing flushed unit "PFX"-"PFX"\n",
+                u->start_pc, u->end_pc);
+            ASSERT(u->pending_free);
+            u->pending_free = false;
+            /* free_unit will set flushtime to 0 (needs it to assert locks) */
+            fcache_free_unit(dcontext, u, true);
+            STATS_INC(cache_units_flushed_freed);
+            STATS_DEC(cache_units_tofree);
+        } else
+            break; /* sorted! */
+    }
+    mutex_unlock(&unit_flush_lock);
+}
+
+/* Used to prevent shared units earmarked for freeing from being re-used.
+ * Caller must be at full synch for a flush.
+ */
+static void
+fcache_mark_units_for_free(dcontext_t *dcontext, fcache_t *cache)
+{
+    fcache_unit_t *u, *head;
+    ASSERT(is_self_flushing()); /* FIXME: want to assert in full synch */
+    PROTECT_CACHE(cache, lock);
+    /* Mark all units as pending_free to avoid fragment deletion from adding
+     * them to free lists.
+     * Also set flushtime and move them to the pending_free list.
+     */
+    u = cache->units;
+    ASSERT(u != NULL);
+    /* leave one unit */
+    head = u->next_local;
+    cache->units->next_local = NULL;
+    for (u = head; u != NULL; u = u->next_local) {
+        u->pending_free = true;
+        u->flushtime = flushtime_global;
+        remove_unit_from_cache(u);
+    }
+    PROTECT_CACHE(cache, unlock);
+    append_units_to_free_list(head);
+}
+
+/* Flush all fragments and mark as many cache units as possible for freeing
+ * (while invalidate_code_cache() only flushes all fragments and does not try
+ * to free any units -- it is meant for consistency purposes, while this is
+ * meant for capacity purposes).
+ * FIXME: currently only marks shared cache units for freeing.
+ * FIXME: should add -stress_flush_units N parameter
+ */
+void
+fcache_flush_all_caches()
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    ASSERT(dcontext != NULL);
+    ASSERT_NOT_TESTED();
+    /* FIXME: share parameters w/ invalidate_code_cache()?
+     * FIXME: efficiency of region-based vs unit-based flushing
+     */
+    flush_fragments_in_region_start(dcontext, UNIVERSAL_REGION_BASE,
+                                    UNIVERSAL_REGION_SIZE,
+                                    false /* don't own initexit_lock */,
+                                    true /* remove futures */,
+                                    false/*not invalidating exec areas*/,
+                                    false/*don't force synchall*/
+                                    _IF_DGCDIAG(NULL));
+    /* In presence of any shared fragments, all threads are stuck here at synch point,
+     * so we can mess w/ global cache units in an atomic manner wrt the flush.  
+     * We can't do private here since threads are let go if no shared
+     * fragments are enabled, but better to have each thread mark its
+     * own anyway.  FIXME -- put flag in delete-list entry
+     */
+    if (DYNAMO_OPTION(shared_bbs)) {
+        fcache_mark_units_for_free(dcontext, shared_cache_bb);
+    }
+    if (DYNAMO_OPTION(shared_traces)) {
+        fcache_mark_units_for_free(dcontext, shared_cache_trace);
+    }
+    /* FIXME: for thread-private units, should use a trigger in
+     * vm_area_flush_fragments() to call a routine here that frees all but
+     * one unit
+     */
+    flush_fragments_in_region_finish(dcontext, false /*don't keep initexit_lock*/);
+    STATS_INC(fcache_flush_all);
+}
+
+/***************************************************************************/
+
+/* Flush all fragments from all caches and free all of those caches, starting
+ * over completely, by suspending all other threads and freeing all fragments
+ * and cache units immediately.
+ * Can only be called while !couldbelinking.
+ * Assumes caller holds reset_pending_lock.
+ * Simultaneous resets are not queued up -- one wins and the rest are canceled.
+ * Use the schedule_reset() routine to queue up resets of different types, which
+ * will all be combined.
+ * FIXME: currently target is ignored and assumed to be RESET_ALL
+ */
+void
+fcache_reset_all_caches_proactively(uint target)
+{
+    thread_record_t **threads;
+    dcontext_t *my_dcontext = get_thread_private_dcontext();
+    int i, num_threads;
+    const thread_synch_state_t desired_state =
+        THREAD_SYNCH_SUSPENDED_VALID_MCONTEXT_OR_NO_XFER;
+
+    /* Reset is meaningless for hotp_only and thin_client modes (case 8389),
+     * though it can be used to release old tables; but old tables aren't
+     * needed for hotp_only so they should be stored anyway, which is NYI
+     * today.
+     */
+    ASSERT(DYNAMO_OPTION(enable_reset) && !RUNNING_WITHOUT_CODE_CACHE());
+    ASSERT(!is_self_couldbelinking());
+
+    /* synch with other threads also trying to call this routine */
+    /* FIXME: use a cleaner model than having callers grab this lock? */
+    ASSERT_OWN_MUTEX(true, &reset_pending_lock);
+    if (reset_in_progress) {
+        mutex_unlock(&reset_pending_lock);
+        return;
+    }
+    /* N.B.: we relax various synch checks if dynamo_resetting is true, since
+     * we will not be holding some locks we normally would need when
+     * deleting shared fragments, etc., assuming that we suspend all
+     * threads in DR while resetting -- if that ever changes we need to
+     * tighten up all those checks again!
+     */
+    reset_in_progress = true;
+    /* this lock is only for synchronizing resets and we do not give it the
+     * rank it would need to be held across the whole routine
+     */
+    mutex_unlock(&reset_pending_lock);
+
+    LOG(GLOBAL, LOG_CACHE, 2, 
+        "\nfcache_reset_all_caches_proactively: thread %d suspending all threads\n",
+        get_thread_id());
+
+    /* Suspend all DR-controlled threads at safe locations.
+     * Case 6821: other synch-all-thread uses can be ignored, as none of them carry
+     * any non-persistent state.
+     */
+    if (!synch_with_all_threads(desired_state, &threads, &num_threads,
+                                /* when called prior to entering the cache we could set
+                                 * mcontext->pc to next_tag and use
+                                 * THREAD_SYNCH_VALID_MCONTEXT, but some callers (like
+                                 * nudge) do not satisfy that */
+                                THREAD_SYNCH_NO_LOCKS_NO_XFER, /* Case 6821 */
+                                /* if we fail to suspend a thread (e.g., for privilege
+                                 * reasons) just abort */
+                                THREAD_SYNCH_SUSPEND_FAILURE_ABORT
+                                /* if we get in a race with detach, or are having
+                                 * synch issues for whatever reason, bail out sooner
+                                 * rather than later */
+                                | THREAD_SYNCH_SMALL_LOOP_MAX)) {
+        /* just give up */
+        reset_in_progress = false;
+        ASSERT(!OWN_MUTEX(&all_threads_synch_lock) && !OWN_MUTEX(&thread_initexit_lock));
+        ASSERT(threads == NULL);
+        ASSERT(!dynamo_all_threads_synched);
+        STATS_INC(fcache_reset_abort);
+        LOG(GLOBAL, LOG_CACHE, 2, 
+            "fcache_reset_all_caches_proactively: aborting due to thread synch failure\n");
+        /* FIXME: may need DO_ONCE but only if we do a LOT of resets combined with
+         * other nudges or sources of thread permission problems */
+        SYSLOG_INTERNAL_WARNING("proactive reset aborted due to thread synch failure");
+        return;
+    }
+
+    /* now we own the thread_initexit_lock */
+    ASSERT(OWN_MUTEX(&all_threads_synch_lock) && OWN_MUTEX(&thread_initexit_lock));
+    STATS_INC(fcache_reset_proactively);
+    DOSTATS({
+        if (TEST(RESET_PENDING_DELETION, target))
+            STATS_INC(fcache_reset_pending_del);
+    });
+
+    DOLOG(1, LOG_STATS, {
+        LOG(GLOBAL, LOG_STATS, 1, "\n**************************Stats BEFORE reset:\n");
+        dump_global_stats(false); 
+    });
+
+    LOG(GLOBAL, LOG_CACHE, 2, 
+        "fcache_reset_all_caches_proactively: walking the threads\n");
+    DOSTATS({
+        SYSLOG_INTERNAL_INFO("proactive reset @ %d fragments", GLOBAL_STAT(num_fragments));
+    });
+
+    /* reset_free and reset_init may write to .data.
+     * All threads are suspended so no security risk.
+     */
+    SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+
+    /* no lock needed */
+    dynamo_resetting = true;
+
+    /* We free everything before re-init so we can free all heap units.
+     * For everything to be freed, it must either be individually freed,
+     * or it must reside in non-persistent heap units,
+     * which will be thrown out wholesale in heap_reset_free().  The latter
+     * is preferable to not waste time on individual deletion.
+     * FIXME: add consistency check walks before and after for all modules
+     */
+    for (i = 0; i < num_threads; i++) {
+        dcontext_t *dcontext = threads[i]->dcontext;
+        if (dcontext != NULL) { /* include my_dcontext here */
+            LOG(GLOBAL, LOG_CACHE, 2, 
+                "\tconsidering thread #%d %d\n", i, threads[i]->id);
+            if (dcontext != my_dcontext) {
+                /* must translate BEFORE freeing any memory! */
+                if (is_thread_currently_native(threads[i])) {
+                    /* native_exec's regain-control point is in our DLL,
+                     * and lost-control threads are truly native, so no
+                     * state to worry about except for hooks -- and we're
+                     * not freeing the interception buffer.
+                     */
+                    LOG(GLOBAL, LOG_CACHE, 2,
+                        "\tcurrently native so no translation needed\n");
+                } else if (thread_synch_state_no_xfer(dcontext)) {
+                    /* Case 6821: do not translate other synch-all-thread users.
+                     * They have no persistent state, so leave alone.
+                     * Also xref case 7760.
+                     */
+                    LOG(GLOBAL, LOG_FRAGMENT, 2,
+                        "\tat THREAD_SYNCH_NO_LOCKS_NO_XFER so no translation needed\n");
+                } else {
+                    translate_from_synchall_to_dispatch(threads[i], desired_state);
+                }
+            }
+            last_exit_deleted(dcontext);
+            if (target == RESET_PENDING_DELETION) {
+                /* case 7394: need to abort other threads' trace building
+                 * since the reset xfer to dispatch will disrupt it
+                 */
+                if (is_building_trace(dcontext)) {
+                    LOG(THREAD, LOG_FRAGMENT, 2,
+                        "\tsquashing trace of thread %d\n", i);
+                    trace_abort(dcontext);
+                }
+            } else {
+                LOG(GLOBAL, LOG_CACHE, 2, "\tfreeing memory in thread %d\n", i);
+                LOG(THREAD, LOG_CACHE, 2, "------- reset for thread %d -------\n",
+                    threads[i]->id);
+                /* N.B.: none of these can assume the executing thread is the
+                 * dcontext owner, esp. wrt tls!
+                 */
+                /* FIXME: now we have {thread_,}init(), {thread_,}exit(), and
+                 * *_reset() -- can we systematically construct these lists of
+                 * module calls?  The list here, though, is a subset of the others.
+                 */
+                /* monitor must go first to remove any undeletable private fragments */
+                monitor_thread_reset_free(dcontext);
+                fragment_thread_reset_free(dcontext);
+                fcache_thread_reset_free(dcontext);
+                /* arch and os data is all persistent */
+                vm_areas_thread_reset_free(dcontext);
+                /* now we throw out all non-persistent private heap units */
+                heap_thread_reset_free(dcontext);
+            }
+        }
+    }
+    if (target == RESET_PENDING_DELETION) {
+        /* FIXME: optimization: suspend only those threads with low flushtimes */
+        LOG(GLOBAL, LOG_CACHE, 2, 
+            "fcache_reset_all_caches_proactively: clearing shared deletion list\n");
+        /* free entire shared deletion list */
+        vm_area_check_shared_pending(GLOBAL_DCONTEXT, NULL);
+    } else {
+        fragment_reset_free();
+        link_reset_free();
+        fcache_reset_free();
+        /* monitor only has thread-private data */
+        /* arch and os data is all persistent */
+        vm_areas_reset_free();
+# ifdef HOT_PATCHING_INTERFACE
+        hotp_reset_free();
+# endif
+        /* now we throw out all non-persistent global heap units */
+        heap_reset_free();
+        
+        LOG(GLOBAL, LOG_CACHE, 2, 
+            "fcache_reset_all_caches_proactively: re-initializing\n");
+
+        /* now set up state all over again */
+        heap_reset_init();
+# ifdef HOT_PATCHING_INTERFACE
+        hotp_reset_init();
+# endif
+        vm_areas_reset_init();
+        fcache_reset_init();
+        link_reset_init();
+        fragment_reset_init();
+
+        for (i = 0; i < num_threads; i++) {
+            dcontext_t *dcontext = threads[i]->dcontext;
+            if (dcontext != NULL) { /* include my_dcontext here */
+                LOG(GLOBAL, LOG_CACHE, 2, 
+                    "fcache_reset_all_caches_proactively: re-initializing thread %d\n", i);
+                /* now set up private state all over again -- generally, we can do
+                 * this before the global free/init since our private/global
+                 * free/init are completely separate (due to the presence of
+                 * persistent state we cannot do a global quick-free anyway).
+                 * But when using shared IBL tables, fragment_reset_init() must
+                 * be called before fragment_thread_reset_init(), since the
+                 * latter copies global state initialized by the former. To
+                 * simplify the code, we simply init all global state prior to
+                 * initing private state (xref case 8092).
+                 */
+                heap_thread_reset_init(dcontext);
+                vm_areas_thread_reset_init(dcontext);
+                monitor_thread_reset_init(dcontext);
+                fcache_thread_reset_init(dcontext);
+                fragment_thread_reset_init(dcontext);
+            }
+        }
+    }
+
+    /* we assume these are atomic and need no locks */
+    dynamo_resetting = false;
+    /* new resets will now queue up on all_threads_synch_lock */
+    reset_in_progress = false;
+
+    SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+
+    DOLOG(1, LOG_STATS, {
+        LOG(GLOBAL, LOG_STATS, 1, "\n**************************Stats AFTER reset:\n");
+        dump_global_stats(false); 
+    });
+
+    LOG(GLOBAL, LOG_CACHE, 2, 
+        "fcache_reset_all_caches_proactively: resuming all threads\n");
+    end_synch_with_all_threads(threads, num_threads, true/*resume*/);
+}
+
+/* returns true if specified target wasn't already scheduled for reset */
+bool
+schedule_reset(uint target)
+{
+    bool added_target;
+    ASSERT(target != 0);
+    mutex_lock(&reset_pending_lock);
+    added_target = !TESTALL(target, reset_pending);
+    reset_pending |= target;
+    mutex_unlock(&reset_pending_lock);
+    return added_target;
+}
+
+#if 0 /* currently not used see note in fcache_low_on_memory */
+static void
+fcache_reset_cache(dcontext_t *dcontext, fcache_t *cache)
+{
+    fcache_unit_t *u, *prevu;
+    fragment_t *f;
+    cache_pc pc, last_pc;
+    bool unit_empty;
+    uint num_units = 0;
+    uint sz;
+
+#ifdef NATIVE_RETURN
+    ASSERT_NOT_REACHED(); /* NATIVE_RETURN requires no fragment deletions */
+#endif
+    /* FIXME: this is called when low on memory: safe to grab lock? */
+    PROTECT_CACHE(cache, lock);
+
+    LOG(THREAD, LOG_CACHE, 2, "fcache_reset_cache %s\n", cache->name);
+    /* we need to free entire units, so don't walk FIFO */
+    for (u = cache->units; u != NULL; u = u->next_local) {
+        LOG(THREAD, LOG_CACHE, 3, "  unit %d: "PFX" -> cur "PFX"\n",
+            num_units, u->start_pc, u->cur_pc);
+        num_units++;
+        /* try to delete everybody we can */
+        unit_empty = true;
+        pc = u->start_pc;
+        last_pc = pc;
+        while (pc < u->cur_pc) {
+            LOG(THREAD, LOG_CACHE, 4, "  f @ "PFX"\n", pc);
+            f = *((fragment_t **)pc);
+            ASSERT(f != NULL);
+            ASSERT(FIFO_UNIT(f) == u);
+            /* go to contiguously-next fragment_t in cache */
+            sz = FRAG_SIZE(f);
+            /* FIXME: do we still need the notion of undeletable fragments?
+             * Should do an analysis and see if we ever use it anymore.
+             * It is a powerful feature to support, but also a limiting one...
+             */
+            if (TEST(FRAG_CANNOT_DELETE, f->flags)) {
+                unit_empty = false;
+                /* FIXME: this allocates memory for the empty_slot_t data struct! */
+                fifo_prepend_empty(dcontext, cache, u, NULL, last_pc, pc - last_pc);
+                STATS_FCACHE_SUB(cache, used, (pc - last_pc));
+                last_pc = pc + sz;
+            } else {
+                /* FIXME: in low-memory situation, will we have problem
+                 * with the future fragment that will be created?
+                 * Even worse, what if it triggers a resize of its hashtable?
+                 * Since we're deleting everyone, we should set some flag saying
+                 * "don't create any future fragment for this deleted fragment"
+                 */
+                force_fragment_from_cache(dcontext, cache, f);
+            }
+            pc += sz;
+        }
+        if (last_pc < u->cur_pc) {
+            fifo_prepend_empty(dcontext, cache, u, NULL, last_pc, pc - last_pc);
+            STATS_FCACHE_SUB(cache, used, (pc - last_pc));
+        }
+        if (unit_empty) {
+            /* hack to indicate empty -- we restore end_pc in next loop */
+            u->end_pc = NULL;
+        }
+    }
+
+    for (prevu = NULL, u = cache->units; u != NULL; prevu = u, u = u->next_local) {
+        if (u->end_pc == NULL) {
+            u->end_pc = u->start_pc + u->size;
+            /* have to leave one unit */
+            if (num_units > 1) {
+                num_units--;
+                fcache_free_unit(dcontext, u, true);
+                if (prevu == NULL)
+                    u = cache->units;
+                else
+                    u = prevu->next_local;
+            }
+        }
+    }
+
+    /* FIXME: try to shrink remaining unit(s)?  Would we do that by freeing
+     * just the tail of the unit?
+     */
+    PROTECT_CACHE(cache, unlock);
+}
+#endif /* #if 0 */
+
+/* This routine has to assume it cannot allocate memory.
+ * Always safe to free free list (using lock).
+ * Other allocations must be freed only in the current thread:
+ * cannot get list of all threads for a global approach b/c that requires memory.
+ * We let the other threads trip over the low memory trigger to flush their own
+ * caches.
+ */
+void
+fcache_low_on_memory()
+{
+    fcache_unit_t *u, *next_u;
+    size_t freed = 0;
+
+#if 0
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    thread_units_t *tu = (thread_units_t *) dcontext->fcache_field;
+    /* FIXME: we cannot reset the cache at arbitrary points -- and we can
+     * be called at any alloc point!  If in middle of fragment creation,
+     * we can't just go delete the fragment!
+     * STRATEGY:
+     * keep a reserved piece of heap per thread that's big enough to get to
+     * a safe point from any DR allocation site (perhaps it should use
+     * a stack allocator).   We keep going, using that for memory
+     * (have to work out shared vs private memory issues if building a
+     * shared bb), and then at a safe point we reset the cache.
+     */
+    /* free the current thread's entire cache, leaving one empty unit */
+    if (tu->bb != NULL)
+        fcache_reset_cache(dcontext, tu->bb);
+    if (tu->trace != NULL)
+        fcache_reset_cache(dcontext, tu->trace);
+#endif
+
+    /* now free the entire dead list (including thread units just moved here) */
+    LOG(GLOBAL, LOG_CACHE|LOG_STATS, 1,
+        "fcache_low_on_memory: about to free dead list units\n");
+    /* WARNING: this routine is called at arbitrary allocation failure points,
+     * so we have to be careful what locks we grab.
+     * No allocation site can hold a lock weaker in rank than heap_unit_lock,
+     * b/c it could deadlock on the allocation itself! So we only have to worry
+     * about the locks of rank between heap_alloc_lock and allunits_lock --
+     * currently dynamo_areas, fcache_unit_areas and global_alloc_lock. We
+     * check for those locks here.  FIXME we have no way to check if holding
+     * a readlock on the dynamo/fcache_unit_areas lock.  FIXME owning the
+     * dynamo_areas lock here is prob. not that uncommon, we may be able to
+     * release and re-grab it but would have to be sure that works in all the
+     * corner cases (if the failing alloc is for a dynamo_areas vector resize
+     * etc.).
+     */
+    if (lockwise_safe_to_allocate_memory() &&
+        !self_owns_dynamo_vm_area_lock() &&
+        !self_owns_write_lock(&fcache_unit_areas->lock)) {
+        mutex_lock(&allunits_lock);
+        u = allunits->dead;
+        while (u != NULL) {
+            next_u = u->next_global;
+            freed += u->size;
+            fcache_really_free_unit(u, true/*on dead list*/, true/*dealloc*/);
+            u = next_u;
+        }
+        allunits->dead = NULL;
+        mutex_unlock(&allunits_lock);
+        LOG(GLOBAL, LOG_CACHE|LOG_STATS, 1,
+            "fcache_low_on_memory: freed %d KB\n", freed/1024);
+    } else {
+        LOG(GLOBAL, LOG_CACHE|LOG_STATS, 1,
+            "fcache_low_on_memory: cannot walk units b/c of deadlock potential\n");
+    }
+
+    options_make_writable();
+    /* be more aggressive about not resizing cache
+     * FIXME: I just made this up -- have param to control?
+     * FIXME: restore params back to original values at some point?
+     */
+    if (dynamo_options.finite_bb_cache && dynamo_options.cache_bb_replace > 0) {
+        dynamo_options.cache_bb_regen *= 2;
+        if (dynamo_options.cache_bb_regen > dynamo_options.cache_bb_replace)
+            dynamo_options.cache_bb_regen = 4*dynamo_options.cache_bb_replace/5;
+    }
+    if (dynamo_options.finite_shared_bb_cache &&
+        dynamo_options.cache_shared_bb_replace > 0) {
+        dynamo_options.cache_shared_bb_regen *= 2;
+        if (dynamo_options.cache_shared_bb_regen >
+            dynamo_options.cache_shared_bb_replace) {
+            dynamo_options.cache_shared_bb_regen =
+                4*dynamo_options.cache_shared_bb_replace/5;
+        }
+    }
+    if (dynamo_options.finite_shared_trace_cache &&
+        dynamo_options.cache_shared_trace_replace > 0) {
+        dynamo_options.cache_shared_trace_regen *= 2;
+        if (dynamo_options.cache_shared_trace_regen >
+            dynamo_options.cache_shared_trace_replace) {
+            dynamo_options.cache_shared_trace_regen =
+                4*dynamo_options.cache_shared_trace_replace/5;
+        }
+    }
+    /* FIXME: be more or less agressive about traces than bbs?
+     * could get rid of trace cache altogether...
+     */
+    if (dynamo_options.finite_trace_cache && dynamo_options.cache_trace_replace > 0) {
+        dynamo_options.cache_trace_regen *= 2;
+        if (dynamo_options.cache_trace_regen > dynamo_options.cache_trace_replace)
+            dynamo_options.cache_trace_regen = 4*dynamo_options.cache_trace_replace/5;
+    }
+    options_restore_readonly();
+}
+
+
+/***************************************************************************
+ * COARSE-GRAIN UNITS
+ */
+
+/* Returns NULL if pc is not an address contained in a coarse fcache unit */
+coarse_info_t *
+get_fcache_coarse_info(cache_pc pc)
+{
+    fcache_unit_t *unit = fcache_lookup_unit(pc);
+    if (unit == NULL)
+        return NULL;
+    ASSERT((unit->cache->is_coarse && unit->cache->coarse_info != NULL) ||
+           (!unit->cache->is_coarse && unit->cache->coarse_info == NULL));
+    return unit->cache->coarse_info;
+}
+
+void
+fcache_coarse_cache_delete(dcontext_t *dcontext, coarse_info_t *info)
+{
+    fcache_t *cache;
+    ASSERT(info != NULL);
+    ASSERT_OWN_MUTEX(!info->is_local, &info->lock);
+    cache = (fcache_t *) info->cache;
+    if (cache == NULL) /* lazily initialized, so common to have empty units */
+        return;
+    /* We don't PROTECT_CACHE(cache, lock) to avoid rank order w/ coarse info lock.
+     * We assume that deletion can only happen for local cache or at reset/exit.
+     */
+    DODEBUG({ cache->is_local = true; });
+    fcache_cache_free(dcontext, cache, !info->frozen/*do not free frozen unit*/);
+    info->cache = NULL;
+    /* We unlink any outgoing links by walking the stubs, not walking the units,
+     * so nothing else to do here
+     */
+}
+
+/* Returns an upper bound on the size needed for the cache if info is frozen. */
+size_t
+coarse_frozen_cache_size(dcontext_t *dcontext, coarse_info_t *info)
+{
+    fcache_t *cache;
+    ASSERT(info != NULL);
+    ASSERT_OWN_MUTEX(true, &info->lock);
+    cache = (fcache_t *) info->cache;
+    if (cache == NULL)
+        return 0;
+    /* we ignore any shrinking from eliding fall-through ubrs
+     * or conversion to 8-bit-jmps
+     */
+    /* cache->size is simply committed size, so subtract unused committed
+     * at end of last unit; we ignore small unused space at end of each unit.
+     */
+    return cache->size - (cache->units->end_pc - cache->units->cur_pc);
+}
+
+/* Assumes that no cache lock is needed because info is newly created
+ * and unknown to all but this thread.
+ */
+void
+fcache_coarse_init_frozen(dcontext_t *dcontext, coarse_info_t *info, 
+                          cache_pc start_pc, size_t size)
+{
+    fcache_t *cache = fcache_cache_init(GLOBAL_DCONTEXT, FRAG_SHARED | FRAG_COARSE_GRAIN,
+                                      false/*no initial unit*/);
+    /* We don't PROTECT_CACHE(cache, lock) to avoid rank order w/ coarse info lock,
+     * assuming that info is newly created and unknown to all but this thread.
+     * (For freezing we also have dynamo_all_threads_synched, but we don't for
+     * loading in persisted caches.)
+     */
+    DODEBUG({ cache->is_local = true; });
+    cache->units = fcache_create_unit(dcontext, cache, start_pc, size);
+    DODEBUG({ cache->is_local = false; });
+    cache->units->cur_pc = cache->units->end_pc;
+    cache->units->full = true;
+    cache->coarse_info = info;
+    info->cache = cache;
+}
+
+/* Used when swapping info structs for in-place freezing */
+void
+fcache_coarse_set_info(dcontext_t *dcontext, coarse_info_t *info)
+{
+    fcache_t *cache;
+    ASSERT(info != NULL);
+    ASSERT_OWN_MUTEX(true, &info->lock);
+    cache = (fcache_t *) info->cache;
+    cache->coarse_info = info;
+}
diff --git a/core/fcache.h b/core/fcache.h
new file mode 100644
index 000000000..354ad0bb4
--- /dev/null
+++ b/core/fcache.h
@@ -0,0 +1,164 @@
+/* **********************************************************
+ * Copyright (c) 2000-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * fcache.h - fcache exports
+ */
+
+#ifndef _FCACHE_H_
+#define _FCACHE_H_ 1
+
+/* who's in the "trace" cache? 
+ * when we have private traces we put temp-private bbs there (to avoid
+ * perf hit in speccpu of having separate priv bb cache that's not
+ * normally used)
+ */
+#define IN_TRACE_CACHE(flags) \
+    (TEST(FRAG_IS_TRACE, (flags)) || \
+     (!DYNAMO_OPTION(shared_traces) && TEST(FRAG_TEMP_PRIVATE, (flags))))
+
+/* Case 8647: we don't need to pad jmps for coarse-grain bbs */
+#define PAD_FRAGMENT_JMPS(flags) \
+    (TEST(FRAG_COARSE_GRAIN, (flags)) ? false : DYNAMO_OPTION(pad_jmps))
+
+#define PAD_JMPS_SHIFT_START(flags) \
+    (PAD_FRAGMENT_JMPS(flags) ? \
+     (TEST(FRAG_IS_TRACE, (flags)) ? INTERNAL_OPTION(pad_jmps_shift_trace) : \
+      INTERNAL_OPTION(pad_jmps_shift_bb)) : false)
+
+/* control over what to reset */
+enum {
+    /* everything, separate since more than sum of others */
+    RESET_ALL              = 0x001,
+    /* NYI (case 6335): just bb caches + heap */
+    RESET_BASIC_BLOCKS     = 0x002,
+    /* NYI (case 6335): just trace caches + heap */
+    RESET_TRACES           = 0x004,
+    /* just pending deletion entries (-reset_every_nth_pending)
+     * TODO OPTIMIZATION (case 7147): we could avoid suspending
+     * everyone and only suspend those threads w/ low flushtimes.
+     */
+    RESET_PENDING_DELETION = 0x008,
+};
+
+/* protects all reset triggers: reset_pending and reset_at_nth_thread_triggered 
+ * resets are not queued up -- one wins and the rest are canceled
+ */
+extern mutex_t reset_pending_lock;
+/* indicates a call to fcache_reset_all_caches_proactively() is pending from dispatch */
+extern uint reset_pending;
+
+void fcache_init(void);
+void fcache_exit(void);
+
+bool fcache_check_option_compatibility(void);
+
+#if defined(WINDOWS_PC_SAMPLE) && !defined(DEBUG)
+/* for fast exit path */
+void fcache_profile_exit(void);
+#endif
+
+#ifdef DEBUG
+void fcache_stats_exit(void);
+#endif
+
+bool in_fcache(void *pc);
+
+void fcache_thread_init(dcontext_t *dcontext);
+#ifdef DEBUG
+void fcache_thread_exit_stats(dcontext_t *dcontext);
+#endif
+void fcache_thread_exit(dcontext_t *dcontext);
+
+void fcache_add_fragment(dcontext_t *dcontext, fragment_t *f);
+void fcache_shift_start_pc(dcontext_t *dcontext, fragment_t *f, uint space);
+void fcache_return_extra_space(dcontext_t *dcontext, fragment_t *f, size_t space);
+void fcache_remove_fragment(dcontext_t *dcontext, fragment_t *f);
+
+bool fcache_is_flush_pending(dcontext_t *dcontext);
+bool fcache_flush_pending_units(dcontext_t *dcontext, fragment_t *was_I_flushed);
+void fcache_free_pending_units(dcontext_t *dcontext, uint flushtime);
+void fcache_flush_all_caches(void);
+
+void fcache_reset_all_caches_proactively(uint target);
+bool schedule_reset(uint target);
+
+void fcache_low_on_memory(void);
+
+/* macros to put mask check outside of function, for efficiency */
+/* when NULL is passed for f then the entire fcache will be affected */
+#define SELF_PROTECT_CACHE(dc, f, w) do {                    \
+    if (TEST(SELFPROT_CACHE, dynamo_options.protect_mask) && \
+        ((f) == NULL || (fcache_is_writable(f) != (w))))     \
+        fcache_change_fragment_protection(dc, f, w);         \
+} while (0);
+
+bool fcache_is_writable(fragment_t *f);
+void fcache_change_fragment_protection(dcontext_t *dcontext,
+                                       fragment_t *f, bool writable);
+
+#ifdef SIDELINE
+dcontext_t * get_dcontext_for_fragment(fragment_t *f);
+#endif
+
+/* for adaptive working set */
+bool fragment_lookup_deleted(dcontext_t *dcontext, app_pc tag);
+
+/* Returns the fragment_t whose body (not cache slot) contains lookup_pc */  
+fragment_t *
+fcache_fragment_pclookup(dcontext_t *dcontext, cache_pc lookup_pc, fragment_t *wrapper);
+
+void
+fcache_coarse_cache_delete(dcontext_t *dcontext, coarse_info_t *info);
+
+/* Returns the coarse info for the cache unit containing pc; if that unit
+ * is not coarse, returns NULL.
+ */
+coarse_info_t *
+get_fcache_coarse_info(cache_pc pc);
+
+/* Returns the total size needed for the cache if info is frozen. */
+size_t
+coarse_frozen_cache_size(dcontext_t *dcontext, coarse_info_t *info);
+
+void
+fcache_coarse_init_frozen(dcontext_t *dcontext, coarse_info_t *info,
+                          cache_pc start_pc, size_t size);
+
+void
+fcache_coarse_set_info(dcontext_t *dcontext, coarse_info_t *info);
+
+#endif /* _FCACHE_H_ */
diff --git a/core/fragment.c b/core/fragment.c
new file mode 100644
index 000000000..0b1009e3c
--- /dev/null
+++ b/core/fragment.c
@@ -0,0 +1,8618 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * fragment.c - fragment related routines
+ */
+
+#include "globals.h"
+#include "link.h"
+#include "fragment.h"
+#include "fcache.h"
+#include "emit.h"
+#include "monitor.h"
+#include <string.h> /* for memset */
+#include "instrument.h"
+#include <stddef.h> /* for offsetof */
+#include <limits.h> /* UINT_MAX */
+#include "perscache.h"
+#include "synch.h"
+
+/* FIXME: make these runtime parameters */
+#define INIT_HTABLE_SIZE_SHARED_BB    (DYNAMO_OPTION(coarse_units) ? 5 : 10)
+#define INIT_HTABLE_SIZE_SHARED_TRACE 10
+/* the only private bbs will be selfmod, so start small */
+#define INIT_HTABLE_SIZE_BB (DYNAMO_OPTION(shared_bbs) ? 5 : 10)
+/* coarse-grain fragments do not use futures */
+#define INIT_HTABLE_SIZE_SHARED_FUTURE (DYNAMO_OPTION(coarse_units) ? 5 : 10)
+#ifdef RETURN_AFTER_CALL
+   /* we have small per-module hashtables */
+#  define INIT_HTABLE_SIZE_AFTER_CALL 5
+#endif
+/* private futures are only used when we have private fragments */
+#define INIT_HTABLE_SIZE_FUTURE  \
+    ((DYNAMO_OPTION(shared_bbs) && DYNAMO_OPTION(shared_traces)) ? 5 : 9)
+
+/* per-module htables */
+#define INIT_HTABLE_SIZE_COARSE 5
+#define INIT_HTABLE_SIZE_COARSE_TH 4
+
+#ifdef RCT_IND_BRANCH
+#  include "rct.h"
+   /* we have small per-module hashtables */
+#  define INIT_HTABLE_SIZE_RCT_IBT 7
+
+#  ifndef RETURN_AFTER_CALL
+#    error RCT_IND_BRANCH requires RETURN_AFTER_CALL since it reuses data types
+# endif
+#endif
+
+/* if shared traces, we currently have no private traces so make table tiny
+ * FIMXE: should start out w/ no table at all
+ */
+#define INIT_HTABLE_SIZE_TRACE     (DYNAMO_OPTION(shared_traces) ? 6 : 9)
+/* for small table sizes resize is not an expensive operation and we start smaller */
+
+#ifdef NATIVE_RETURN
+#  define INIT_HTABLE_SIZE_DELETED    9
+#endif
+
+/* Current flusher, protected by thread_initexit_lock. */
+DECLARE_FREQPROT_VAR(static dcontext_t *flusher, NULL);
+/* Current allsynch-flusher, protected by thread_initexit_lock. */
+DECLARE_FREQPROT_VAR(static dcontext_t *allsynch_flusher, NULL);
+
+#if defined(NATIVE_RETURN) && !defined(DEBUG)
+/* for release build, we use this to identify early code that is
+ * full of returns that we didn't see the call for
+ */
+int num_fragments;
+#endif
+
+/* These global tables are kept on the heap for selfprot (case 7957) */
+
+/* synchronization to these tables is accomplished via read-write locks,
+ * where the writers are removal and resizing -- addition is atomic to
+ * readers.
+ * for now none of these are read from ibl routines so we only have to
+ * synch with other DR routines
+ */
+static fragment_table_t *shared_bb;
+static fragment_table_t *shared_trace;
+
+/* if we have either shared bbs or shared traces we need this shared: */
+static fragment_table_t *shared_future;
+
+/* Thread-shared tables are allocated in a shared per_thread_t.
+ * The structure is also used if we're dumping shared traces.
+ * Kept on the heap for selfprot (case 7957)
+ */
+static per_thread_t *shared_pt;
+
+#define USE_SHARED_PT() (SHARED_IBT_TABLES_ENABLED() || \
+    (TRACEDUMP_ENABLED() && DYNAMO_OPTION(shared_traces)))
+
+/* We keep track of "old" IBT target tables in a linked list and
+ * deallocate them in fragment_exit(). */
+/* FIXME Deallocate tables more aggressively using a distributed, refcounting
+ * algo as is used for shared deletion. */
+typedef struct _dead_fragment_table_t {
+    fragment_entry_t *table_unaligned;
+    uint table_flags;
+    uint capacity;
+    uint ref_count;
+    struct _dead_fragment_table_t *next;
+} dead_fragment_table_t;
+
+/* We keep these list pointers on the heap for selfprot (case 8074). */
+typedef struct _dead_table_lists_t {
+    dead_fragment_table_t *dead_tables;
+    dead_fragment_table_t *dead_tables_tail;
+} dead_table_lists_t;
+
+static dead_table_lists_t *dead_lists;
+
+DECLARE_CXTSWPROT_VAR(static mutex_t dead_tables_lock, INIT_LOCK_FREE(dead_tables_lock));
+
+#ifdef RETURN_AFTER_CALL
+/* High level lock for an atomic lookup+add operation on the
+ * after call tables. */
+DECLARE_CXTSWPROT_VAR(static mutex_t after_call_lock, INIT_LOCK_FREE(after_call_lock));
+/* We use per-module tables and only need this table for non-module code;
+ * on Linux though this is the only table used, until we have a module list.
+ */
+static rct_module_table_t rac_non_module_table;
+#endif
+
+/* allows independent sequences of flushes and delayed deletions,
+ * though with -syscalls_synch_flush additions we now hold this
+ * throughout a flush.
+ */
+DECLARE_CXTSWPROT_VAR(mutex_t shared_cache_flush_lock,
+                      INIT_LOCK_FREE(shared_cache_flush_lock));
+/* Global count of flushes, used as a timestamp for shared deletion.
+ * Reads may be done w/o a lock, but writes can only be done
+ * via increment_global_flushtime() while holding shared_cache_flush_lock.
+ */
+DECLARE_FREQPROT_VAR(uint flushtime_global, 0);
+
+#ifdef CLIENT_INTERFACE
+DECLARE_CXTSWPROT_VAR(mutex_t client_flush_request_lock,
+                      INIT_LOCK_FREE(client_flush_request_lock));
+DECLARE_CXTSWPROT_VAR(client_flush_req_t *client_flush_requests, NULL);
+#endif
+
+#if defined(RCT_IND_BRANCH) && defined(LINUX)
+/* On Win32 we use per-module tables; on Linux we use a single global table,
+ * until we have a module list.
+ */
+rct_module_table_t rct_global_table;
+#endif
+
+#define NULL_TAG ((app_pc)PTR_UINT_0)
+/* FAKE_TAG is used as a deletion marker for unlinked entries */
+#define FAKE_TAG ((app_pc)PTR_UINT_MINUS_1)
+
+/* instead of an empty hashtable slot containing NULL, we fill it
+ * with a pointer to this constant fragment, which we give a tag
+ * of 0:
+ */
+static const fragment_t null_fragment = { NULL_TAG, 0, 0, 0, 0,
+                                        HASHLOOKUP_NULL_START_PC, };
+/* to avoid range check on fast path using an end of table sentinel fragment */
+static const fragment_t sentinel_fragment = { NULL_TAG, 0, 0, 0, 0,
+                                            HASHLOOKUP_SENTINEL_START_PC, };
+
+/* Shared fragment IBTs: We need to preserve the open addressing traversal
+ * in the hashtable while marking a table entry as unlinked.
+ * A null_fragment won't work since it terminates the traversal,
+ * so we use an unlinked marker. The lookup table entry for
+ * an unlinked entry *always* has its start_pc_fragment set to
+ * an IBL target_delete entry.
+ */
+static const fragment_t unlinked_fragment = { FAKE_TAG, };
+
+/* macro used in the code from time of deletion markers */
+/* Shared fragment IBTs: unlinked_fragment isn't a real fragment either. So they
+ * are naturally deleted during a table resize. */
+#define REAL_FRAGMENT(fragment)                 \
+     ((fragment) != &null_fragment &&           \
+      (fragment) != &unlinked_fragment &&       \
+      (fragment) != &sentinel_fragment)
+
+#define GET_PT(dc) ((dc) == GLOBAL_DCONTEXT ? (USE_SHARED_PT() ? shared_pt : NULL) :\
+    (per_thread_t *) (dc)->fragment_field)
+
+#define TABLE_PROTECTED(ptable) \
+    (!TABLE_NEEDS_LOCK(ptable) || READWRITE_LOCK_HELD(&(ptable)->rwlock))
+
+/* everything except the invisible table is in here */
+#define GET_FTABLE_HELPER(pt, flags, otherwise)                       \
+    (TEST(FRAG_IS_TRACE, (flags)) ?                                   \
+     (TEST(FRAG_SHARED, (flags)) ? shared_trace : &pt->trace) :      \
+     (TEST(FRAG_SHARED, (flags)) ?                                    \
+      (TEST(FRAG_IS_FUTURE, (flags)) ? shared_future : shared_bb) : \
+      (TEST(FRAG_IS_FUTURE, (flags)) ? &pt->future :                  \
+      (otherwise))))
+
+#define GET_FTABLE(pt, flags) GET_FTABLE_HELPER(pt, (flags), &pt->bb)
+
+/* indirect branch table per target type (bb vs trace) and indirect branch type */
+#define GET_IBT_TABLE(pt, flags, branch_type)                         \
+    (TEST(FRAG_IS_TRACE, (flags)) ?                                   \
+     (DYNAMO_OPTION(shared_trace_ibt_tables) ?                        \
+      &shared_pt->trace_ibt[(branch_type)] :                           \
+      &(pt)->trace_ibt[(branch_type)]) :                              \
+     (DYNAMO_OPTION(shared_bb_ibt_tables) ?                           \
+      &shared_pt->bb_ibt[(branch_type)] :                              \
+      &(pt)->bb_ibt[(branch_type)]))
+
+/********************************** STATICS ***********************************/
+static uint fragment_heap_size(uint flags, int direct_exits, int indirect_exits);
+
+static void fragment_free_future(dcontext_t *dcontext, future_fragment_t *fut);
+
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+static void
+coarse_persisted_fill_ibl(dcontext_t *dcontext, coarse_info_t *info,
+                          ibl_branch_type_t branch_type);
+#endif
+
+#ifdef CLIENT_INTERFACE
+static void
+process_client_flush_requests(dcontext_t *dcontext, dcontext_t *alloc_dcontext,
+                              client_flush_req_t *req, bool flush);
+#endif
+
+#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
+/* trace logging and synch for shared trace file: */
+DECLARE_CXTSWPROT_VAR(static mutex_t tracedump_mutex, INIT_LOCK_FREE(tracedump_mutex));
+DECLARE_FREQPROT_VAR(static stats_int_t tcount, 0); /* protected by tracedump_mutex */
+static void exit_trace_file(per_thread_t *pt);
+static void output_trace(dcontext_t *dcontext, per_thread_t *pt,
+                         fragment_t *f, stats_int_t deleted_at);
+static void init_trace_file(per_thread_t *pt);
+#endif
+
+#define SHOULD_OUTPUT_FRAGMENT(flags) \
+    (TEST(FRAG_IS_TRACE, (flags)) && \
+    !TEST(FRAG_TRACE_OUTPUT, (flags)) && \
+    TRACEDUMP_ENABLED())
+
+#define FRAGMENT_COARSE_WRAPPER_FLAGS \
+    FRAG_FAKE | FRAG_SHARED | FRAG_COARSE_GRAIN | \
+    FRAG_LINKED_OUTGOING | FRAG_LINKED_INCOMING
+
+/* We use temporary fragment_t + linkstub_t structs to more easily
+ * use existing code when emitting coarse-grain fragments.
+ * Only 1-ind-exit or 1 or 2 dir exit bbs can be coarse-grain.
+ * The bb_building_lock protects use of this.
+ */
+DECLARE_FREQPROT_VAR(
+                     static struct {
+                         fragment_t f;
+                         union {
+                             struct {
+                                 direct_linkstub_t dir_exit_1;
+                                 direct_linkstub_t dir_exit_2;
+                             } dir_exits;
+                             indirect_linkstub_t ind_exit;
+                         } exits;
+                     } coarse_emit_fragment, {{0}});
+
+#ifdef SHARING_STUDY
+/***************************************************************************
+ * fragment_t sharing study
+ * Only used with -fragment_sharing_study
+ * When the option is off we go ahead and waste the 4 static vars
+ * below so we don't have to have a define and separate build.
+ */
+typedef struct _thread_list_t {
+    uint thread_num;
+    uint count;
+    struct _thread_list_t *next;
+} thread_list_t;
+
+typedef struct _shared_entry_t {
+    app_pc tag;
+    uint num_threads;
+    thread_list_t *threads;
+    uint heap_size;
+    uint cache_size;
+    struct _shared_entry_t *next;
+} shared_entry_t;
+# define SHARED_HASH_BITS 16
+static shared_entry_t ** shared_blocks;
+DECLARE_CXTSWPROT_VAR(static mutex_t shared_blocks_lock,
+                      INIT_LOCK_FREE(shared_blocks_lock));
+static shared_entry_t ** shared_traces;
+DECLARE_CXTSWPROT_VAR(static mutex_t shared_traces_lock,
+                      INIT_LOCK_FREE(shared_traces_lock));
+
+/* assumes caller holds table's lock! */
+static shared_entry_t * 
+shared_block_lookup(shared_entry_t **table, fragment_t *f)
+{
+    shared_entry_t *e;
+    uint hindex;
+
+    hindex = HASH_FUNC_BITS((ptr_uint_t)f->tag, SHARED_HASH_BITS);
+    /* using collision chains */
+    for (e = table[hindex]; e != NULL; e = e->next) {
+        if (e->tag == f->tag) {
+            return e;
+        }
+    }
+    return NULL;
+}
+
+static void
+reset_shared_block_table(shared_entry_t **table, mutex_t *lock)
+{
+    shared_entry_t *e, *nxte;
+    uint i;
+    uint size = HASHTABLE_SIZE(SHARED_HASH_BITS);
+    mutex_lock(lock);
+    for (i = 0; i < size; i++) {
+        for (e = table[i]; e != NULL; e = nxte) {
+            thread_list_t *tl = e->threads;
+            thread_list_t *tlnxt;
+            nxte = e->next;
+            while (tl != NULL) {
+                tlnxt = tl->next;
+                global_heap_free(tl, sizeof(thread_list_t) HEAPACCT(ACCT_OTHER));
+                tl = tlnxt;
+            }
+            global_heap_free(e, sizeof(shared_entry_t) HEAPACCT(ACCT_OTHER));
+        }
+    }
+    global_heap_free(table, size*sizeof(shared_entry_t*) HEAPACCT(ACCT_OTHER));
+    mutex_unlock(lock);
+}
+
+static void
+add_shared_block(shared_entry_t **table, mutex_t *lock, fragment_t *f)
+{
+    shared_entry_t *e;
+    uint hindex;
+    int num_direct = 0, num_indirect = 0;
+    linkstub_t *l = FRAGMENT_EXIT_STUBS(f);
+    /* use num to avoid thread_id_t recycling problems */
+    uint tnum = get_thread_num(get_thread_id());
+
+    mutex_lock(lock);
+    e = shared_block_lookup(table, f);
+    if (e != NULL) {
+        thread_list_t *tl = e->threads;
+        for (; tl != NULL; tl = tl->next) {
+            if (tl->thread_num == tnum) {
+                tl->count++;
+                LOG(GLOBAL, LOG_ALL, 2,
+                    "add_shared_block: tag "PFX", but re-add #%d for thread #%d\n",
+                    e->tag, tl->count, tnum);
+                mutex_unlock(lock);
+                return;
+            }
+        }
+        tl = global_heap_alloc(sizeof(thread_list_t) HEAPACCT(ACCT_OTHER));
+        tl->thread_num = tnum;
+        tl->count = 1;
+        tl->next = e->threads;
+        e->threads = tl;
+        e->num_threads++;
+        LOG(GLOBAL, LOG_ALL, 2, "add_shared_block: tag "PFX" thread #%d => %d threads\n",
+            e->tag, tnum, e->num_threads);
+        mutex_unlock(lock);
+        return;
+    }
+
+    /* get num stubs to find heap size */
+    for (; l != NULL; l = LINKSTUB_NEXT_EXIT(l)) {
+        if (LINKSTUB_DIRECT(l->flags))
+            num_direct++;
+        else {
+            ASSERT(LINKSTUB_INDIRECT(l->flags));
+            num_indirect++;
+        }
+    }
+
+    /* add entry to thread hashtable */
+    e = (shared_entry_t *) global_heap_alloc(sizeof(shared_entry_t) HEAPACCT(ACCT_OTHER));
+    e->tag = f->tag;
+    e->num_threads = 1;
+    e->heap_size = fragment_heap_size(f->flags, num_direct, num_indirect);
+    e->cache_size = (f->size + f->fcache_extra);
+    e->threads = global_heap_alloc(sizeof(thread_list_t) HEAPACCT(ACCT_OTHER));
+    e->threads->thread_num = tnum;
+    e->threads->count = 1;
+    e->threads->next = NULL;
+    LOG(GLOBAL, LOG_ALL, 2, "add_shared_block: tag "PFX", heap %d, cache %d, thread #%d\n",
+        e->tag, e->heap_size, e->cache_size, e->threads->thread_num);
+
+    hindex = HASH_FUNC_BITS((ptr_uint_t)f->tag, SHARED_HASH_BITS);
+    e->next = table[hindex];
+    table[hindex] = e;
+    mutex_unlock(lock);
+}
+
+static void
+print_shared_table_stats(shared_entry_t **table, mutex_t *lock, char *name)
+{
+    uint i;
+    shared_entry_t *e;
+    uint size = HASHTABLE_SIZE(SHARED_HASH_BITS);
+    uint tot = 0, shared_tot = 0, shared = 0, heap = 0, cache = 0,
+        creation_count = 0;
+
+    mutex_lock(lock);
+    for (i = 0; i < size; i++) {
+        for (e = table[i]; e != NULL; e = e->next) {
+            thread_list_t *tl = e->threads;
+            tot++;
+            shared_tot += e->num_threads;
+            for (; tl != NULL; tl = tl->next)
+                creation_count += tl->count;
+            if (e->num_threads > 1) {
+                shared++;
+                /* assume similar size for each thread -- cache padding
+                 * only real difference
+                 */
+                heap += (e->heap_size * e->num_threads);
+                cache += (e->cache_size * e->num_threads);
+            }
+        }
+    }
+    mutex_unlock(lock);
+    LOG(GLOBAL, LOG_ALL, 1, "Shared %s statistics:\n", name);
+    LOG(GLOBAL, LOG_ALL, 1, "\ttotal blocks:   %10d\n", tot);
+    LOG(GLOBAL, LOG_ALL, 1, "\tcreation count: %10d\n", creation_count);
+    LOG(GLOBAL, LOG_ALL, 1, "\tshared count:   %10d\n", shared_tot);
+    LOG(GLOBAL, LOG_ALL, 1, "\tshared blocks:  %10d\n", shared);
+    LOG(GLOBAL, LOG_ALL, 1, "\tshared heap:    %10d\n", heap);
+    LOG(GLOBAL, LOG_ALL, 1, "\tshared cache:   %10d\n", cache);
+}
+
+void
+print_shared_stats()
+{
+    print_shared_table_stats(shared_blocks, &shared_blocks_lock, "basic block");
+    print_shared_table_stats(shared_traces, &shared_traces_lock, "trace");
+}
+#endif /* SHARING_STUDY ***************************************************/
+
+
+#ifdef FRAGMENT_SIZES_STUDY /*****************************************/
+#include <math.h>
+/* don't bother to synchronize these */
+static int bb_sizes[200000];
+static int trace_sizes[40000];
+static int num_bb = 0;
+static int num_traces = 0;
+
+void
+record_fragment_size(int size, bool is_trace)
+{
+    if (is_trace) {
+        trace_sizes[num_traces] = size;
+        num_traces++;
+        ASSERT(num_traces < 40000);
+    } else {
+        bb_sizes[num_bb] = size;
+        num_bb++;
+        ASSERT(num_bb < 200000);
+    }
+}
+
+void
+print_size_results()
+{
+    LOG(GLOBAL, LOG_ALL, 1, "Basic block sizes (bytes):\n");
+    print_statistics(bb_sizes, num_bb);
+    LOG(GLOBAL, LOG_ALL, 1, "Trace sizes (bytes):\n");
+    print_statistics(trace_sizes, num_traces);
+}
+#endif /* FRAGMENT_SIZES_STUDY */ /*****************************************/
+
+#define FRAGTABLE_WHICH_HEAP(flags)                                    \
+   (TESTALL(FRAG_TABLE_INCLUSIVE_HIERARCHY | FRAG_TABLE_IBL_TARGETED,  \
+            (flags)) ? ACCT_IBLTABLE : ACCT_FRAG_TABLE)
+
+#ifdef HASHTABLE_STATISTICS
+# define UNPROT_STAT(stats) unprot_stats->stats
+/* FIXME: either put in nonpersistent heap as appropriate, or
+ * preserve across resets
+ */
+# define ALLOC_UNPROT_STATS(dcontext, table) do {                          \
+        (table)->unprot_stats =                                            \
+           HEAP_TYPE_ALLOC((dcontext), unprot_ht_statistics_t,                 \
+                           FRAGTABLE_WHICH_HEAP((table)->table_flags),     \
+                           UNPROTECTED);                                   \
+         memset((table)->unprot_stats, 0, sizeof(unprot_ht_statistics_t));     \
+} while (0)
+# define DEALLOC_UNPROT_STATS(dcontext, table)                             \
+   HEAP_TYPE_FREE((dcontext), (table)->unprot_stats, unprot_ht_statistics_t,   \
+                  FRAGTABLE_WHICH_HEAP((table)->table_flags), UNPROTECTED)
+# define CHECK_UNPROT_STATS(table) ASSERT(table.unprot_stats != NULL)
+
+static void
+check_stay_on_trace_stats_overflow(dcontext_t *dcontext, ibl_branch_type_t branch_type)
+{
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+    hashtable_statistics_t *lookup_stats = &pt->trace_ibt[branch_type].unprot_stats->
+        trace_ibl_stats[branch_type];
+    if (lookup_stats->ib_stay_on_trace_stat < lookup_stats->ib_stay_on_trace_stat_last) {
+        lookup_stats->ib_stay_on_trace_stat_ovfl++;
+    }
+    lookup_stats->ib_stay_on_trace_stat_last = lookup_stats->ib_stay_on_trace_stat;
+    /* FIXME: ib_trace_last_ibl_exit should have an overflow check as well */
+}
+#endif /* HASHTABLE_STATISTICS */
+
+/* init/update the tls slots storing this table's mask and lookup base 
+ * N.B.: for thread-shared the caller must call for each thread
+ */
+/* currently we don't support a mixture */
+static inline void
+update_lookuptable_tls(dcontext_t *dcontext, ibl_table_t *table)
+{
+    /* use dcontext->local_state, rather than get_local_state(), to support
+     * being called from other threads!
+     */
+    local_state_extended_t *state =
+        (local_state_extended_t *) dcontext->local_state;
+
+    ASSERT(state != NULL);
+    ASSERT(DYNAMO_OPTION(ibl_table_in_tls));
+    /* We must hold at least the read lock here, else we could grab
+     * an inconsistent mask/lookuptable pair if another thread is in the middle
+     * of resizing the table (case 10405).
+     */
+    ASSERT_TABLE_SYNCHRONIZED(table, READWRITE);
+    /* case 10296: for shared tables we must update the table
+     * before the mask, as the ibl lookup code accesses the mask first,
+     * and old mask + new table is ok since it will de-ref within the
+     * new table (we never shrink tables) and be a miss, whereas
+     * new mask + old table can de-ref beyond the end of the table,
+     * crashing or worse.
+     */
+    state->table_space.table[table->branch_type].lookuptable =
+        table->table;
+    state->table_space.table[table->branch_type].hash_mask =
+        table->hash_mask;
+}
+
+#ifdef DEBUG
+static const char *ibl_bb_table_type_names[IBL_BRANCH_TYPE_END] = 
+    {"ret_bb", "indcall_bb", "indjmp_bb"};
+static const char *ibl_trace_table_type_names[IBL_BRANCH_TYPE_END] = 
+    {"ret_trace", "indcall_trace", "indjmp_trace"};
+#endif
+
+#ifdef DEBUG
+static inline void
+dump_lookuptable_tls(dcontext_t *dcontext)
+{
+    /* use dcontext->local_state, rather than get_local_state(), to support
+     * being called from other threads!
+     */
+    if (DYNAMO_OPTION(ibl_table_in_tls)) {
+
+        local_state_extended_t *state =
+            (local_state_extended_t *) dcontext->local_state;
+        ibl_branch_type_t branch_type;
+
+        ASSERT(state != NULL);
+        for (branch_type = IBL_BRANCH_TYPE_START; 
+             branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+            LOG(THREAD, LOG_FRAGMENT, 1,
+                "\t Table %s, table "PFX", mask "PFX"\n",
+                !SHARED_BB_ONLY_IB_TARGETS() ?
+                ibl_trace_table_type_names[branch_type] :
+                ibl_bb_table_type_names[branch_type],
+                state->table_space.table[branch_type].lookuptable,
+                state->table_space.table[branch_type].hash_mask);
+        }
+    }
+}
+#endif
+
+/*******************************************************************************
+ * IBL HASHTABLE INSTANTIATION
+ */
+#define FRAGENTRY_FROM_FRAGMENT(f) { (f)->tag, (f)->start_pc }
+
+/* macros w/ name and types are duplicated in fragment.h -- keep in sync */
+#define NAME_KEY ibl
+#define ENTRY_TYPE fragment_entry_t
+/* not defining HASHTABLE_USE_LOOKUPTABLE */
+/* compiler won't let me use null_fragment.tag here */
+static const fragment_entry_t fe_empty = { NULL_TAG, HASHLOOKUP_NULL_START_PC };
+static const fragment_entry_t fe_sentinel = { NULL_TAG, HASHLOOKUP_SENTINEL_START_PC };
+#define ENTRY_TAG(fe)         ((ptr_uint_t)(fe).tag_fragment)
+#define ENTRY_EMPTY           (fe_empty)
+#define ENTRY_SENTINEL        (fe_sentinel)
+#define IBL_ENTRY_IS_EMPTY(fe) \
+    ((fe).tag_fragment == fe_empty.tag_fragment && \
+     (fe).start_pc_fragment == fe_empty.start_pc_fragment)
+#define IBL_ENTRY_IS_INVALID(fe) ((fe).tag_fragment == FAKE_TAG)
+#define IBL_ENTRY_IS_SENTINEL(fe) \
+    ((fe).tag_fragment == fe_sentinel.tag_fragment && \
+     (fe).start_pc_fragment == fe_sentinel.start_pc_fragment)
+#define ENTRY_IS_EMPTY(fe)    IBL_ENTRY_IS_EMPTY(fe)
+#define ENTRY_IS_SENTINEL(fe) IBL_ENTRY_IS_SENTINEL(fe)
+#define ENTRY_IS_INVALID(fe)  IBL_ENTRY_IS_INVALID(fe)
+#define IBL_ENTRIES_ARE_EQUAL(fe1,fe2)  ((fe1).tag_fragment == (fe2).tag_fragment)
+#define ENTRIES_ARE_EQUAL(fe1,fe2)  IBL_ENTRIES_ARE_EQUAL(fe1,fe2)
+#define HASHTABLE_WHICH_HEAP(flags) FRAGTABLE_WHICH_HEAP(flags)
+#define HTLOCK_RANK               table_rwlock
+#define HASHTABLE_ENTRY_STATS 1
+
+#include "hashtablex.h"
+/* all defines are undef-ed at end of hashtablex.h */
+
+/* required routines for hashtable interface that we don't need for this instance */
+
+static void
+hashtable_ibl_free_entry(dcontext_t *dcontext, ibl_table_t *table,
+                         fragment_entry_t entry)
+{
+    /* nothing to do, data is inlined */
+}
+
+/*******************************************************************************
+ * FRAGMENT HASHTABLE INSTANTIATION
+ */
+
+/* macros w/ name and types are duplicated in fragment.h -- keep in sync */
+#define NAME_KEY fragment
+#define ENTRY_TYPE fragment_t *
+/* not defining HASHTABLE_USE_LOOKUPTABLE */
+
+#define ENTRY_TAG(f)              ((ptr_uint_t)(f)->tag)
+/* instead of setting to 0, point at null_fragment */
+#define ENTRY_EMPTY               ((fragment_t *)&null_fragment)
+#define ENTRY_SENTINEL            ((fragment_t *)&sentinel_fragment)
+#define ENTRY_IS_EMPTY(f)         ((f) == (fragment_t *)&null_fragment)
+#define ENTRY_IS_SENTINEL(f)      ((f) == (fragment_t *)&sentinel_fragment)
+#define ENTRY_IS_INVALID(f)       ((f) == (fragment_t *)&unlinked_fragment)
+#define ENTRIES_ARE_EQUAL(f,g)    ((f) == (g))
+#define HASHTABLE_WHICH_HEAP(flags) FRAGTABLE_WHICH_HEAP(flags)
+#define HTLOCK_RANK               table_rwlock
+
+#include "hashtablex.h"
+/* all defines are undef-ed at end of hashtablex.h */
+
+static void
+hashtable_fragment_resized_custom(dcontext_t *dcontext, fragment_table_t *table,
+                                  uint old_capacity, fragment_t **old_table,
+                                  fragment_t **old_table_unaligned,
+                                  uint old_ref_count, uint old_table_flags)
+{
+    /* nothing */
+}
+
+static void
+hashtable_fragment_init_internal_custom(dcontext_t *dcontext, fragment_table_t *table)
+{
+    /* nothing */
+}
+
+#ifdef DEBUG
+static void
+hashtable_fragment_study_custom(dcontext_t *dcontext, fragment_table_t *table, 
+                                uint entries_inc/*amnt table->entries was pre-inced*/)
+{
+    /* nothing */
+}
+#endif
+
+/* callers should use either hashtable_ibl_preinit or hashtable_resize instead */
+static void
+hashtable_ibl_init_internal_custom(dcontext_t *dcontext, ibl_table_t *table)
+{
+    ASSERT(null_fragment.tag == NULL_TAG);
+    ASSERT(null_fragment.start_pc == HASHLOOKUP_NULL_START_PC);
+    ASSERT(FAKE_TAG != NULL_TAG);
+
+    ASSERT(sentinel_fragment.tag == NULL_TAG);
+    ASSERT(sentinel_fragment.start_pc == HASHLOOKUP_SENTINEL_START_PC);
+    ASSERT(HASHLOOKUP_SENTINEL_START_PC != HASHLOOKUP_NULL_START_PC);
+
+    ASSERT(TEST(FRAG_TABLE_IBL_TARGETED, table->table_flags));
+    ASSERT(TEST(FRAG_TABLE_INCLUSIVE_HIERARCHY, table->table_flags));
+
+    /* every time we resize a table we reset the flush threshold,
+     * since it is cleared in place after one flush
+     */
+    table->groom_factor_percent = 
+        TEST(FRAG_TABLE_TRACE, table->table_flags) ?  
+        DYNAMO_OPTION(trace_ibt_groom) : DYNAMO_OPTION(bb_ibt_groom);
+    table->max_capacity_bits = 
+        TEST(FRAG_TABLE_TRACE, table->table_flags) ?  
+        DYNAMO_OPTION(private_trace_ibl_targets_max) : 
+        DYNAMO_OPTION(private_bb_ibl_targets_max);
+
+#ifdef HASHTABLE_STATISTICS
+    if (INTERNAL_OPTION(hashtable_ibl_stats)) {
+        if (table->unprot_stats == NULL) {
+            /* first time, not a resize */
+            ALLOC_UNPROT_STATS(dcontext, table);
+        } /* else, keep original */
+    }
+#endif /* HASHTABLE_STATISTICS */
+
+    if (SHARED_IB_TARGETS() &&
+        !TEST(FRAG_TABLE_SHARED, table->table_flags)) {
+        /* currently we don't support a mixture */
+        ASSERT(TEST(FRAG_TABLE_TARGET_SHARED, table->table_flags));
+        ASSERT(TEST(FRAG_TABLE_IBL_TARGETED, table->table_flags));
+        ASSERT(table->branch_type != IBL_NONE);
+        /* Only data for one set of tables is stored in TLS -- for the trace
+         * tables in the default config OR the BB tables in shared BBs
+         * only mode.
+         */
+        if ((TEST(FRAG_TABLE_TRACE, table->table_flags) ||
+             SHARED_BB_ONLY_IB_TARGETS()) &&
+            DYNAMO_OPTION(ibl_table_in_tls))
+            update_lookuptable_tls(dcontext, table);
+    }
+}
+
+/* We need our own routines to init/free our added fields */
+static void
+hashtable_ibl_myinit(dcontext_t *dcontext, ibl_table_t *table, uint bits, 
+                     uint load_factor_percent, hash_function_t func,
+                     uint hash_offset, ibl_branch_type_t branch_type,
+                     bool use_lookup, uint table_flags NAME(const char *table_name))
+{
+    uint flags = table_flags;
+    ASSERT(dcontext != GLOBAL_DCONTEXT || TEST(FRAG_TABLE_SHARED, flags));
+    /* flags shared by all ibl tables */
+    flags |= FRAG_TABLE_INCLUSIVE_HIERARCHY;
+    flags |= FRAG_TABLE_IBL_TARGETED;
+    flags |= HASHTABLE_ALIGN_TABLE;
+    /* use entry stats with all our ibl-targeted tables */
+    flags |= HASHTABLE_USE_ENTRY_STATS;
+#ifdef HASHTABLE_STATISTICS
+    /* indicate this is first time, not a resize */
+    table->unprot_stats = NULL;
+#endif
+    table->branch_type = branch_type;
+    hashtable_ibl_init(dcontext, table, bits, load_factor_percent,
+                       func, hash_offset, flags NAME(table_name));
+}
+
+static void
+hashtable_ibl_myfree(dcontext_t *dcontext, ibl_table_t *table)
+{
+#ifdef HASHTABLE_STATISTICS
+    if (INTERNAL_OPTION(hashtable_ibl_stats)) {
+        ASSERT(TEST(FRAG_TABLE_IBL_TARGETED, table->table_flags));
+        DEALLOC_UNPROT_STATS(dcontext, table);
+    }
+#endif /* HASHTABLE_STATISTICS */
+    hashtable_ibl_free(dcontext, table);
+}
+
+static void
+hashtable_fragment_free_entry(dcontext_t *dcontext, fragment_table_t *table,
+                              fragment_t *f)
+{
+    if (TEST(FRAG_TABLE_INCLUSIVE_HIERARCHY, table->table_flags)) {
+        ASSERT_NOT_REACHED(); /* case 7691 */
+    } else {
+        if (TEST(FRAG_IS_FUTURE, f->flags))
+            fragment_free_future(dcontext, (future_fragment_t *)f);
+        else
+            fragment_free(dcontext, f);
+    }
+}
+
+static inline bool
+fragment_add_to_hashtable(dcontext_t *dcontext, fragment_t *e, fragment_table_t *table)
+{
+    /* When using shared IBT tables w/trace building and BB2BB IBL, there is a
+     * race between adding a BB target to a table and having it marked by
+     * another thread as a trace head. The race exists because the two functions
+     * do not use a common lock.
+     * The race does NOT cause a correctness problem since a) the marking thread
+     * removes the trace head from the table and b) any subsequent add attempt
+     * is caught in add_ibl_target(). The table lock is used during add and
+     * remove operations and FRAG_IS_TRACE_HEAD is checked while holding
+     * the lock. So although a trace head may be present in a table temporarily --
+     * it's being marked while an add operation that has passed the frag flags
+     * check is in progress -- it will be subsequently removed by the marking
+     * thread.
+     * However, the existence of the race does mean that
+     * we cannot ASSERT(!(FRAG_IS_TRACE_HEAD,...)) at arbitrary spots along the
+     * add_ibl_target() path since such an assert could fire due to the race.
+     * What is likely a safe point to assert is when there is only a single
+     * thread in the process.
+     */
+    DODEBUG({
+        if (TEST(FRAG_TABLE_IBL_TARGETED, table->table_flags) &&
+            get_num_threads() == 1)
+            ASSERT(!TEST(FRAG_IS_TRACE_HEAD, e->flags));
+    });
+
+    return hashtable_fragment_add(dcontext, e, table);
+}
+
+/* updates all fragments in a given fragment table which may
+ * have IBL routine heads inlined in the indirect exit stubs
+ *
+ * FIXME: [perf] should add a filter of which branch types need updating if
+ * updating all is a noticable performance hit.
+ *
+ * FIXME: [perf] Also it maybe better to traverse all fragments in an fcache
+ * unit instead of entries in a half-empty hashtable
+ */
+static void
+update_indirect_exit_stubs_from_table(dcontext_t *dcontext, 
+                                      fragment_table_t *ftable)
+{
+    fragment_t *f;
+    linkstub_t *l;
+    uint i;
+
+    for (i = 0; i < ftable->capacity; i++) {
+        f = ftable->table[i];
+        if (!REAL_FRAGMENT(f))
+            continue;
+        for (l = FRAGMENT_EXIT_STUBS(f); l != NULL; l = LINKSTUB_NEXT_EXIT(l)) {
+            if (LINKSTUB_INDIRECT(l->flags)) {
+                /* FIXME: should add a filter of which branch types need updating */
+                update_indirect_exit_stub(dcontext, f, l); 
+                LOG(THREAD, LOG_FRAGMENT, 5,
+                    "\tIBL target table resizing: updating F%d\n", f->id);
+                STATS_INC(num_ibl_stub_resize_updates);
+            }
+        }
+    }
+}
+
+static void
+safely_nullify_tables(dcontext_t *dcontext, ibl_table_t *new_table,
+                      fragment_entry_t *table, uint capacity)
+{
+    uint i;
+    cache_pc target_delete = get_target_delete_entry_pc(dcontext, new_table);
+
+    ASSERT(target_delete != NULL);
+    ASSERT_TABLE_SYNCHRONIZED(new_table, WRITE);
+    for (i = 0; i < capacity; i++) {
+        if (IBL_ENTRY_IS_SENTINEL(table[i])) {
+            ASSERT(i == capacity - 1);
+            continue;
+        }
+        /* We need these writes to be atomic, so check that they're aligned. */
+        ASSERT(ALIGNED(&table[i].tag_fragment, 4));
+        ASSERT(ALIGNED(&table[i].start_pc_fragment, 4));
+        /* We update the tag first so that so that a thread that's skipping
+         * along a chain will exit ASAP. Breaking the chain is ok since we're
+         * nullifying the entire table.
+         */
+        table[i].tag_fragment = fe_empty.tag_fragment;
+        /* We set the payload to target_delete to induce a cache exit.
+         *
+         * The target_delete path leads to a loss of information -- we can't
+         * tell what the src fragment was (the one that transitioned to the
+         * IBL code) and this in principle could weaken our RCT checks (see case
+         * 5085). In practical terms, RCT checks are unaffected since they
+         * are not employed on in-cache transitions such as an IBL hit.
+         * (All transitions to target_delete are a race along the hit path.)
+         * If we still want to preserve the src info, we can leave the payload
+         * as-is, possibly pointing to a cache address. The effect is that
+         * any thread accessing the old table on the IBL hit path will not exit
+         * the cache as early. (We should leave the fragment_t* value in the
+         * table untouched also so that the fragment_table_t is in a consistent
+         * state.)
+         */
+        table[i].start_pc_fragment = target_delete;
+    }
+    STATS_INC(num_shared_ibt_table_flushes);
+}
+
+/* Add an item to the dead tables list */
+static inline void
+add_to_dead_table_list(dcontext_t *alloc_dc, ibl_table_t *ftable,
+                       uint old_capacity,
+                       fragment_entry_t *old_table_unaligned, uint old_ref_count,
+                       uint old_table_flags)
+{
+    dead_fragment_table_t *item =(dead_fragment_table_t*)
+        heap_alloc(GLOBAL_DCONTEXT, sizeof(dead_fragment_table_t)
+                   HEAPACCT(ACCT_IBLTABLE));
+
+    LOG(GLOBAL, LOG_FRAGMENT, 2,
+        "add_to_dead_table_list %s "PFX" capacity %d\n",
+        ftable->name, old_table_unaligned, old_capacity);
+    ASSERT(old_ref_count >= 1); /* someone other the caller must be holding a reference */
+    /* write lock must be held so that ref_count is copied accurately */
+    ASSERT_TABLE_SYNCHRONIZED(ftable, WRITE);
+    item->capacity = old_capacity;
+    item->table_unaligned = old_table_unaligned;
+    item->table_flags = old_table_flags;
+    item->ref_count = old_ref_count;
+    item->next = NULL;
+    /* Add to the end of list. We use a FIFO because generally we'll be
+     * decrementing ref-counts for older tables before we do so for
+     * younger tables. A FIFO will yield faster searches than, say, a
+     * stack.
+     */
+    mutex_lock(&dead_tables_lock);
+    if (dead_lists->dead_tables == NULL) {
+        ASSERT(dead_lists->dead_tables_tail == NULL);
+        dead_lists->dead_tables = item;
+    }
+    else {
+        ASSERT(dead_lists->dead_tables_tail != NULL);
+        ASSERT(dead_lists->dead_tables_tail->next == NULL);
+        dead_lists->dead_tables_tail->next = item;
+    }
+    dead_lists->dead_tables_tail = item;
+    mutex_unlock(&dead_tables_lock);
+    STATS_ADD_PEAK(num_dead_shared_ibt_tables, 1);
+    STATS_INC(num_total_dead_shared_ibt_tables);
+}
+
+/* forward decl */
+static inline void
+update_private_ptr_to_shared_ibt_table(dcontext_t *dcontext,
+                                       ibl_branch_type_t branch_type, bool trace,
+                                       bool adjust_old_ref_count, bool lock_table);
+static void
+hashtable_ibl_resized_custom(dcontext_t *dcontext, ibl_table_t *table,
+                             uint old_capacity, fragment_entry_t *old_table,
+                             fragment_entry_t *old_table_unaligned,
+                             uint old_ref_count, uint old_table_flags)
+{
+    dcontext_t *alloc_dc = FRAGMENT_TABLE_ALLOC_DC(dcontext, table->table_flags);
+    per_thread_t *pt = GET_PT(dcontext);
+    bool shared_ibt_table =
+        TESTALL(FRAG_TABLE_TARGET_SHARED | FRAG_TABLE_SHARED, table->table_flags);
+    ASSERT(TEST(FRAG_TABLE_IBL_TARGETED, table->table_flags));
+
+    /* If we change an ibl-targeted table, must patch up every
+     * inlined indirect exit stub that targets it.
+     * For our per-type ibl tables however we don't bother updating
+     * fragments _targeted_ by the resized table, instead we need to
+     * update all fragments that may be a source of an inlined IBL.
+     */
+
+    /* private inlined IBL heads targeting this table need to be updated */
+    if (DYNAMO_OPTION(inline_trace_ibl) && PRIVATE_TRACES_ENABLED()) {
+        /* We'll get here on a trace table resize, while we
+         * need to patch only when the trace_ibt tables are resized.
+         */
+        /* We assume we don't inline IBL lookup targeting tables of basic blocks
+         * and so shouldn't need to do this for now. */
+        ASSERT(dcontext != GLOBAL_DCONTEXT && pt != NULL); /* private traces */
+        if (TESTALL(FRAG_TABLE_INCLUSIVE_HIERARCHY | FRAG_TABLE_TRACE, 
+                    table->table_flags)) {
+            /* need to update all traces that could be targeting the
+             * currently resized table */
+            LOG(THREAD, LOG_FRAGMENT, 2,
+                "\tIBL target table resizing: updating all private trace fragments\n");
+            update_indirect_exit_stubs_from_table(dcontext, &pt->trace);
+        }
+    }
+
+    /* if we change the trace table (or an IBL target trace
+     * table), must patch up every inlined indirect exit stub
+     * in all bb fragments in case the inlined target is the
+     * resized table
+     */
+    if (DYNAMO_OPTION(inline_bb_ibl)) {
+        LOG(THREAD, LOG_FRAGMENT, 3,
+            "\tIBL target table resizing: updating bb fragments\n");
+        update_indirect_exit_stubs_from_table(dcontext, &pt->bb);
+    }
+#ifdef NATIVE_RETURN
+    if (DYNAMO_OPTION(inline_bb_ibl) ||
+        DYNAMO_OPTION(inline_trace_ibl)) {
+        uint i;
+        /* deleted table too */
+        for (i = 0; i < pt->deleted.capacity; i++) {
+            f = pt->deleted.table[i];
+            if (!REAL_FRAGMENT(f))
+                continue;
+            for (l = FRAGMENT_EXIT_STUBS(f); l != NULL; l = LINKSTUB_NEXT_EXIT(l)) {
+                if (LINKSTUB_INDIRECT(l->flags)
+                    ((TEST(FRAG_IS_TRACE, f->flags) &&
+                      DYNAMO_OPTION(inline_trace_ibl)) ||
+                     (!TEST(FRAG_IS_TRACE, f->flags) &&
+                      DYNAMO_OPTION(inline_bb_ibl)))) {
+                    update_indirect_exit_stub(dcontext, f, l);
+                    LOG(THREAD, LOG_FRAGMENT, 5,
+                        "\ttrace table resizing: updating deleted F%d\n",
+                        f->id);
+                }
+            }
+        }
+    }
+#endif
+    
+    /* don't need to update any inlined lookups in shared fragments */
+
+    if (shared_ibt_table) {
+        if (old_ref_count > 0) {
+            /* The old table should be nullified ASAP. Since threads update
+             * their table pointers on-demand only when they exit the cache
+             * after a failed IBL lookup, they could have IBL targets for
+             * stale entries. This would likely occur only when there's an
+             * app race but in the future could occur due to cache
+             * management.
+             */
+            safely_nullify_tables(dcontext, table, old_table, old_capacity);
+            add_to_dead_table_list(alloc_dc, table, old_capacity, 
+                                   old_table_unaligned,
+                                   old_ref_count, table->table_flags);
+        }
+        /* Update the resizing thread's private ptr. */
+        update_private_ptr_to_shared_ibt_table(dcontext, table->branch_type,
+                                               TEST(FRAG_TABLE_TRACE,
+                                                    table->table_flags),
+                                               false, /* no adjust
+                                                       * old ref-count */
+                                               false  /* already hold lock */);
+        ASSERT(table->ref_count == 1);
+   }
+    
+    /* CHECK: is it safe to update the table without holding the lock? */
+    /* Using the table flags to drive the update of generated code may
+     * err on the side of caution, but it's the best way to guarantee
+     * that all of the necessary code is updated.
+     * We may perform extra unnecessary updates when a table that's
+     * accessed off of the dcontext/per_thread_t is grown, but that doesn't
+     * cause correctness problems and likely doesn't hurt peformance.
+     */
+    STATS_INC(num_ibt_table_resizes);
+    update_generated_hashtable_access(dcontext);
+}
+
+#ifdef DEBUG
+static void
+hashtable_ibl_study_custom(dcontext_t *dcontext, ibl_table_t *table, 
+                           uint entries_inc/*amnt table->entries was pre-inced*/)
+{
+# ifdef HASHTABLE_STATISTICS
+    /* For trace table(s) only, use stats from emitted ibl routines */
+    if (TEST(FRAG_TABLE_IBL_TARGETED, table->table_flags) &&
+        INTERNAL_OPTION(hashtable_ibl_stats)) {
+        per_thread_t *pt = GET_PT(dcontext);
+        ibl_branch_type_t branch_type;
+
+        for (branch_type = IBL_BRANCH_TYPE_START; 
+             branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+            /* This is convoluted since given a table we have to
+             * recover its branch type.
+             * FIXME: should simplify these assumptions one day
+             */
+            /* Current table should be targeted only by one of the IBL routines */
+            if (!((!DYNAMO_OPTION(disable_traces) &&
+                   table == &pt->trace_ibt[branch_type]) ||
+                  (DYNAMO_OPTION(bb_ibl_targets) &&
+                   table == &pt->bb_ibt[branch_type])))
+                continue;
+            /* stats for lookup routines from bb's and trace's targeting the current table */
+            print_hashtable_stats(dcontext, entries_inc == 0 ? "Total" : "Current", 
+                                  table->name,
+                                  "trace ibl ", get_branch_type_name(branch_type),
+                                  &table->UNPROT_STAT(trace_ibl_stats[branch_type]));
+            print_hashtable_stats(dcontext, entries_inc == 0 ? "Total" : "Current", 
+                                  table->name,
+                                  "bb ibl ", 
+                                  get_branch_type_name(branch_type),
+                                  &table->UNPROT_STAT(bb_ibl_stats[branch_type]));
+        }
+    }
+# endif /* HASHTABLE_STATISTICS */
+}
+#endif /* DEBUG */
+
+#if defined(DEBUG) || defined(CLIENT_INTERFACE)
+/* filter specifies flags for fragments which are OK to be freed */
+/* NOTE - if this routine is ever used for non DEBUG purposes be aware that
+ * because of case 7697 we don't unlink when we free the hashtable elements.
+ * As such, if we aren't also freeing all fragments that could possibly link
+ * to fragments in this table at the same time (synchronously) we'll have
+ * problems (for ex. a trace only reset would need to unlink incoming, or
+ * allowing private->shared linking would need to ulink outgoing).
+ */
+static void
+hashtable_fragment_reset(dcontext_t *dcontext, fragment_table_t *table)
+{
+    int i;
+    fragment_t *f;
+
+    /* case 7691: we now use separate ibl table types */
+    ASSERT(!TEST(FRAG_TABLE_INCLUSIVE_HIERARCHY, table->table_flags));
+    LOG(THREAD, LOG_FRAGMENT, 2, "hashtable_fragment_reset\n");
+    DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
+        hashtable_fragment_load_statistics(dcontext, table);
+    });
+    if (TEST(FRAG_TABLE_SHARED, table->table_flags) &&
+        TEST(FRAG_TABLE_IBL_TARGETED, table->table_flags)) {
+        DOLOG(4, LOG_FRAGMENT, {
+            hashtable_fragment_dump_table(dcontext, table);
+        });
+    }
+    DODEBUG({
+        hashtable_fragment_study(dcontext, table, 0/*table consistent*/);
+        /* ensure write lock is held if the table is shared, unless exiting
+         * or resetting (N.B.: if change reset model to not suspend all in-DR
+         * threads, will have to change this and handle rank order issues)
+         */
+        if (!dynamo_exited && !dynamo_resetting)
+            ASSERT_TABLE_SYNCHRONIZED(table, WRITE);
+    });
+    /* Go in reverse order (for efficiency) since using
+     * hashtable_fragment_remove_helper to keep all reachable, which is required
+     * for dynamo_resetting where we unlink fragments here and need to be able to
+     * perform lookups.
+     */
+    i = table->capacity - 1 - 1 /* sentinel */;
+    while (i >= 0) {
+        f = table->table[i];
+        if (f == &null_fragment) {
+            i--;
+        } else { /* i stays put */
+            /* The shared BB table is reset at process reset or shutdown, so
+             * trace_abort() has already been called by (or for) every thread.
+             * If shared traces is true, by this point none of the shared BBs
+             * should have FRAG_TRACE_BUILDING set since the flag is cleared
+             * by trace_abort(). Of course, the flag shouldn't be present
+             * if shared traces is false so we don't need to conditionalize
+             * the assert.
+             */
+            ASSERT(!TEST(FRAG_TRACE_BUILDING, f->flags));
+            hashtable_fragment_remove_helper(table, i, &table->table[i]);
+            if (!REAL_FRAGMENT(f))
+                continue; 
+            /* make sure no other hashtable has shared fragments in it
+             * this routine is called on shared table, but only after dynamo_exited
+             * the per-thread IBL tables contain pointers to shared fragments
+             * and are OK
+             */
+            ASSERT(dynamo_exited || !TEST(FRAG_SHARED, f->flags) || dynamo_resetting);
+
+# if defined(SIDELINE) && defined(PROFILE_LINKCOUNT)
+            if ((f->flags & FRAG_DO_NOT_SIDELINE) != 0) {
+                /* print out total count of exit counters */
+                LOG(THREAD, LOG_SIDELINE, 2, "\tSidelined trace F%d total times executed: "
+                    LINKCOUNT_FORMAT_STRING "\n", f->id, get_total_linkcount(f));
+            }
+# endif
+            if (TEST(FRAG_IS_FUTURE, f->flags)) {
+                DODEBUG({ ((future_fragment_t *)f)->incoming_stubs = NULL; });
+                fragment_free_future(dcontext, (future_fragment_t *)f);
+            } else {
+                DOSTATS({
+                    if (dynamo_resetting)
+                        STATS_INC(num_fragments_deleted_reset);
+                    else
+                        STATS_INC(num_fragments_deleted_exit);
+                });
+                /* Xref 7697 - unlinking the fragments here can screw up the
+                 * future table as we are walking in hash order, so we don't
+                 * unlink.  See note at top of routine for issues with not
+                 * unlinking here if this code is ever used in non debug
+                 * builds. */
+                fragment_delete(dcontext, f,
+                                FRAGDEL_NO_HTABLE | FRAGDEL_NO_UNLINK |
+                                FRAGDEL_NEED_CHLINK_LOCK |
+                                (dynamo_resetting ? 0 : FRAGDEL_NO_OUTPUT));
+            }
+        }
+    }
+    table->entries = 0;
+    table->unlinked_entries = 0;
+}
+#endif /* DEBUG || CLIENT_INTERFACE */
+
+/*
+ *******************************************************************************/
+
+
+#if defined(RETURN_AFTER_CALL) || defined (RCT_IND_BRANCH)
+/*******************************************************************************
+ * APP_PC HASHTABLE INSTANTIATION
+ */
+/* FIXME: RCT tables no longer use future_fragment_t and can be moved out of fragment.c */
+
+/* The ENTRY_* defines are undef-ed at end of hashtablex.h so we make our own.
+ * Would be nice to re-use ENTRY_IS_EMPTY, etc., though w/ multiple htables
+ * in same file can't realistically get away w/o custom defines like these:
+ */
+#define APP_PC_EMPTY                 (NULL)
+/* assume 1 is always invalid address */
+#define APP_PC_SENTINEL              ((app_pc)PTR_UINT_1)
+#define APP_PC_ENTRY_IS_EMPTY(pc)    ((pc) == APP_PC_EMPTY)
+#define APP_PC_ENTRY_IS_SENTINEL(pc) ((pc) == APP_PC_SENTINEL)
+#define APP_PC_ENTRY_IS_REAL(pc) (!APP_PC_ENTRY_IS_EMPTY(pc) && \
+                                  !APP_PC_ENTRY_IS_SENTINEL(pc))
+/* 2 macros w/ name and types are duplicated in fragment.h -- keep in sync */
+#define NAME_KEY app_pc
+#define ENTRY_TYPE app_pc
+/* not defining HASHTABLE_USE_LOOKUPTABLE */
+#define ENTRY_TAG(f)              ((ptr_uint_t)(f))
+#define ENTRY_EMPTY               APP_PC_EMPTY
+#define ENTRY_SENTINEL            APP_PC_SENTINEL
+#define ENTRY_IS_EMPTY(f)         APP_PC_ENTRY_IS_EMPTY(f)
+#define ENTRY_IS_SENTINEL(f)      APP_PC_ENTRY_IS_SENTINEL(f)
+#define ENTRY_IS_INVALID(f)       (false) /* no invalid entries */
+#define ENTRIES_ARE_EQUAL(f,g)    ((f) == (g))
+#define HASHTABLE_WHICH_HEAP(flags) (ACCT_AFTER_CALL)
+#define HTLOCK_RANK               app_pc_table_rwlock
+#define HASHTABLE_SUPPORT_PERSISTENCE 1
+
+#include "hashtablex.h"
+/* all defines are undef-ed at end of hashtablex.h */
+
+/* required routines for hashtable interface that we don't need for this instance */
+
+static void
+hashtable_app_pc_init_internal_custom(dcontext_t *dcontext, app_pc_table_t *htable)
+{ /* nothing */
+}
+
+static void
+hashtable_app_pc_resized_custom(dcontext_t *dcontext, app_pc_table_t *htable,
+                                uint old_capacity, app_pc *old_table,
+                                app_pc *old_table_unaligned,
+                                uint old_ref_count, uint old_table_flags)
+{ /* nothing */ 
+}
+
+# ifdef DEBUG
+static void
+hashtable_app_pc_study_custom(dcontext_t *dcontext, app_pc_table_t *htable, 
+                              uint entries_inc/*amnt table->entries was pre-inced*/)
+{ /* nothing */ 
+}
+# endif
+
+static void
+hashtable_app_pc_free_entry(dcontext_t *dcontext, app_pc_table_t *htable,
+                            app_pc entry)
+{
+    /* nothing to do, data is inlined */
+}
+
+#endif /* defined(RETURN_AFTER_CALL) || defined (RCT_IND_BRANCH) */
+/*******************************************************************************/
+
+
+bool
+fragment_initialized(dcontext_t *dcontext)
+{
+    return (dcontext != GLOBAL_DCONTEXT && dcontext->fragment_field != NULL);
+}
+
+/* thread-shared initialization that should be repeated after a reset */
+void
+fragment_reset_init(void)
+{
+    /* case 7966: don't initialize at all for hotp_only & thin_client */
+    if (RUNNING_WITHOUT_CODE_CACHE())
+        return;
+
+    mutex_lock(&shared_cache_flush_lock);
+    /* ASSUMPTION: a reset frees all deletions that use flushtimes, so we can
+     * reset the global flushtime here
+     */
+    flushtime_global = 0;
+    mutex_unlock(&shared_cache_flush_lock);
+
+    if (SHARED_FRAGMENTS_ENABLED()) {
+        if (DYNAMO_OPTION(shared_bbs)) {
+            hashtable_fragment_init(GLOBAL_DCONTEXT, shared_bb,
+                                    INIT_HTABLE_SIZE_SHARED_BB,
+                                    INTERNAL_OPTION(shared_bb_load), 
+                                    (hash_function_t)INTERNAL_OPTION(alt_hash_func),
+                                    0 /* hash_mask_offset */,
+                                    FRAG_TABLE_SHARED | FRAG_TABLE_TARGET_SHARED
+                                    NAME("shared_bb"));
+        }
+        if (DYNAMO_OPTION(shared_traces)) {
+            hashtable_fragment_init(GLOBAL_DCONTEXT, shared_trace,
+                                    INIT_HTABLE_SIZE_SHARED_TRACE,
+                                    INTERNAL_OPTION(shared_trace_load), 
+                                    (hash_function_t)INTERNAL_OPTION(alt_hash_func),
+                                    0 /* hash_mask_offset */,
+                                    FRAG_TABLE_SHARED | FRAG_TABLE_TARGET_SHARED
+                                    NAME("shared_trace"));
+        }
+        /* init routine will work for future_fragment_t* same as for fragment_t* */
+        hashtable_fragment_init(GLOBAL_DCONTEXT, shared_future,
+                                INIT_HTABLE_SIZE_SHARED_FUTURE,
+                                INTERNAL_OPTION(shared_future_load),
+                                (hash_function_t)INTERNAL_OPTION(alt_hash_func),
+                                0 /* hash_mask_offset */,
+                                FRAG_TABLE_SHARED | FRAG_TABLE_TARGET_SHARED
+                                NAME("shared_future"));
+    }
+
+    if (SHARED_IBT_TABLES_ENABLED()) {
+
+        ibl_branch_type_t branch_type;
+
+        ASSERT(USE_SHARED_PT());
+
+        for (branch_type = IBL_BRANCH_TYPE_START; 
+             branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+            if (DYNAMO_OPTION(shared_trace_ibt_tables)) {
+                hashtable_ibl_myinit(GLOBAL_DCONTEXT, &shared_pt->trace_ibt[branch_type],
+                                     DYNAMO_OPTION(shared_ibt_table_trace_init),
+                                     DYNAMO_OPTION(shared_ibt_table_trace_load),
+                                     HASH_FUNCTION_NONE,
+                                     HASHTABLE_IBL_OFFSET(branch_type),
+                                     branch_type,
+                                     false, /* no lookup table */
+                                     FRAG_TABLE_SHARED |
+                                     FRAG_TABLE_TARGET_SHARED |
+                                     FRAG_TABLE_TRACE
+                                     NAME(ibl_trace_table_type_names[branch_type]));
+#ifdef HASHTABLE_STATISTICS
+                if (INTERNAL_OPTION(hashtable_ibl_stats)) {
+                    CHECK_UNPROT_STATS(&shared_pt->trace_ibt[branch_type]);
+                    /* for compatibility using an entry in the per-branch type stats */
+                    INIT_HASHTABLE_STATS(shared_pt->trace_ibt[branch_type].
+                                         UNPROT_STAT(trace_ibl_stats[branch_type]));
+                } else {
+                    shared_pt->trace_ibt[branch_type].unprot_stats = NULL;
+                }
+#endif /* HASHTABLE_STATISTICS */
+            }
+
+            if (DYNAMO_OPTION(shared_bb_ibt_tables)) {
+                hashtable_ibl_myinit(GLOBAL_DCONTEXT, &shared_pt->bb_ibt[branch_type],
+                                     DYNAMO_OPTION(shared_ibt_table_bb_init),
+                                     DYNAMO_OPTION(shared_ibt_table_bb_load),
+                                     HASH_FUNCTION_NONE,
+                                     HASHTABLE_IBL_OFFSET(branch_type),
+                                     branch_type,
+                                     false, /* no lookup table */
+                                     FRAG_TABLE_SHARED |
+                                     FRAG_TABLE_TARGET_SHARED
+                                     NAME(ibl_bb_table_type_names[branch_type]));
+                /* mark as inclusive table for bb's - we in fact currently
+                 * keep only frags that are not FRAG_IS_TRACE_HEAD */
+#ifdef HASHTABLE_STATISTICS
+                if (INTERNAL_OPTION(hashtable_ibl_stats)) {
+                    /* for compatibility using an entry in the per-branch type stats */
+                    CHECK_UNPROT_STATS(&shared_pt->bb_ibt[branch_type]);
+                    /* FIXME: we don't expect trace_ibl_stats yet */
+                    INIT_HASHTABLE_STATS(shared_pt->bb_ibt[branch_type].
+                                         UNPROT_STAT(bb_ibl_stats[branch_type]));
+                } else {
+                    shared_pt->bb_ibt[branch_type].unprot_stats = NULL;
+                }
+#endif /* HASHTABLE_STATISTICS */
+            }
+        }
+    }
+
+#ifdef SHARING_STUDY
+    if (INTERNAL_OPTION(fragment_sharing_study)) {
+        uint size = HASHTABLE_SIZE(SHARED_HASH_BITS) * sizeof(shared_entry_t*);
+        shared_blocks = (shared_entry_t**) global_heap_alloc(size HEAPACCT(ACCT_OTHER));
+        memset(shared_blocks, 0, size);
+        shared_traces = (shared_entry_t**) global_heap_alloc(size HEAPACCT(ACCT_OTHER));
+        memset(shared_traces, 0, size);
+    }
+#endif
+}
+
+/* thread-shared initialization */
+void
+fragment_init()
+{
+    /* case 7966: don't initialize at all for hotp_only & thin_client
+     * FIXME: could set initial sizes to 0 for all configurations, instead
+     */
+    if (RUNNING_WITHOUT_CODE_CACHE())
+        return;
+
+    /* make sure fields are at same place */
+    ASSERT(offsetof(fragment_t, flags) == offsetof(future_fragment_t, flags));
+    ASSERT(offsetof(fragment_t, tag) == offsetof(future_fragment_t, tag));
+
+    /* ensure we can read this w/o a lock: no cache line crossing, please */
+    ASSERT(ALIGNED(&flushtime_global, 4));
+
+    if (SHARED_FRAGMENTS_ENABLED()) {
+        /* tables are persistent across resets, only on heap for selfprot (case 7957) */
+        if (DYNAMO_OPTION(shared_bbs)) {
+            shared_bb = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, fragment_table_t,
+                                        ACCT_FRAG_TABLE, PROTECTED);
+        }
+        if (DYNAMO_OPTION(shared_traces)) {
+            shared_trace = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, fragment_table_t,
+                                           ACCT_FRAG_TABLE, PROTECTED);
+        }
+        shared_future = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, fragment_table_t,
+                                        ACCT_FRAG_TABLE, PROTECTED);
+    }
+
+    if (USE_SHARED_PT())
+        shared_pt = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, per_thread_t, ACCT_OTHER, PROTECTED);
+
+    if (SHARED_IBT_TABLES_ENABLED()) {
+        dead_lists =
+            HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT,  dead_table_lists_t, ACCT_OTHER, PROTECTED);
+        memset(dead_lists, 0, sizeof(*dead_lists));
+    }
+
+    fragment_reset_init();
+
+#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
+    if (TRACEDUMP_ENABLED() && DYNAMO_OPTION(shared_traces)) {
+        ASSERT(USE_SHARED_PT());
+        shared_pt->tracefile = open_log_file("traces-shared", NULL, 0);
+        ASSERT(shared_pt->tracefile != INVALID_FILE);
+        init_trace_file(shared_pt);
+    }
+#endif
+}
+
+/* Free all thread-shared state not critical to forward progress;
+ * fragment_reset_init() will be called before continuing.
+ */
+void
+fragment_reset_free(void)
+{
+    /* case 7966: don't initialize at all for hotp_only & thin_client */
+    if (RUNNING_WITHOUT_CODE_CACHE())
+        return;
+
+    /* We must study the ibl tables before the trace/bb tables so that we're
+     * not looking at freed entries
+     */
+    if (SHARED_IBT_TABLES_ENABLED()) {
+
+        ibl_branch_type_t branch_type;
+        dead_fragment_table_t *current, *next;
+        DEBUG_DECLARE(int table_count = 0;)
+        DEBUG_DECLARE(stats_int_t dead_tables = GLOBAL_STAT(num_dead_shared_ibt_tables);)
+
+        for (branch_type = IBL_BRANCH_TYPE_START; 
+             branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+            if (DYNAMO_OPTION(shared_trace_ibt_tables)) {
+                DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
+                    hashtable_ibl_load_statistics(GLOBAL_DCONTEXT,
+                                                  &shared_pt->trace_ibt[branch_type]);
+                });
+                hashtable_ibl_myfree(GLOBAL_DCONTEXT,
+                                     &shared_pt->trace_ibt[branch_type]);
+            }
+            if (DYNAMO_OPTION(shared_bb_ibt_tables)) {
+                DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
+                    hashtable_ibl_load_statistics(GLOBAL_DCONTEXT,
+                                                  &shared_pt->bb_ibt[branch_type]);
+                });
+                hashtable_ibl_myfree(GLOBAL_DCONTEXT,
+                                     &shared_pt->bb_ibt[branch_type]);
+            }
+        }
+
+        /* Delete dead tables. */
+        /* grab lock for consistency, although we expect a single thread */
+        mutex_lock(&dead_tables_lock);
+        current = dead_lists->dead_tables;
+        while (current != NULL) {
+            DODEBUG({table_count++;});
+            next = current->next;
+            LOG(GLOBAL, LOG_FRAGMENT, 2,
+                "fragment_reset_free: dead table "PFX" cap %d, freeing\n",
+                current->table_unaligned, current->capacity);
+            hashtable_ibl_free_table(GLOBAL_DCONTEXT, current->table_unaligned,
+                                     current->table_flags, current->capacity);
+            heap_free(GLOBAL_DCONTEXT, current, sizeof(dead_fragment_table_t)
+                      HEAPACCT(ACCT_IBLTABLE));
+            STATS_DEC(num_dead_shared_ibt_tables);
+            STATS_INC(num_dead_shared_ibt_tables_freed);
+            current = next;
+            DODEBUG({
+                if (dynamo_exited)
+                    STATS_INC(num_dead_shared_ibt_tables_freed_at_exit);
+            });
+        }
+        dead_lists->dead_tables = dead_lists->dead_tables_tail = NULL;
+        ASSERT(table_count == dead_tables);
+        mutex_unlock(&dead_tables_lock);
+    }
+
+    /* FIXME: Take in a flag "permanent" that controls whether exiting or
+     * resetting.  If resetting only, do not free unprot stats and entry stats
+     * (they're already in persistent heap, but we explicitly free them).
+     * This will be easy w/ unprot but will take work for entry stats
+     * since they resize themselves.
+     * Or, move them both to a new unprot and nonpersistent heap so we can
+     * actually free the memory back to the os, if we don't care to keep
+     * the stats across the reset.
+     */
+    /* N.B.: to avoid rank order issues w/ shared_vm_areas lock being acquired
+     * after table_rwlock we do NOT grab the write lock before calling
+     * reset on the shared tables!  We assume that reset involves suspending
+     * all other threads in DR and there will be no races.  If the reset model
+     * changes, the lock order will have to be addressed.
+     */
+    if (SHARED_FRAGMENTS_ENABLED()) {
+        /* clean up pending delayed deletion, if any */
+        vm_area_check_shared_pending(GLOBAL_DCONTEXT/*== safe to free all*/, NULL);
+
+        if (DYNAMO_OPTION(coarse_units)) {
+            /* We need to free coarse units earlier than vm_areas_exit() so we
+             * call it here.  Must call before we free fine fragments so coarse
+             * can clean up incoming pointers.
+             */
+            vm_area_coarse_units_reset_free();
+        }
+
+#if defined(DEBUG) || defined(CLIENT_INTERFACE)
+        /* We need for CLIENT_INTERFACE to get fragment deleted events. */
+# if !defined(DEBUG) && defined(CLIENT_INTERFACE)
+        if (dr_fragment_deleted_hook_exists()) {
+# endif
+            if (DYNAMO_OPTION(shared_bbs))
+                hashtable_fragment_reset(GLOBAL_DCONTEXT, shared_bb);
+            if (DYNAMO_OPTION(shared_traces))
+                hashtable_fragment_reset(GLOBAL_DCONTEXT, shared_trace);
+            DODEBUG({hashtable_fragment_reset(GLOBAL_DCONTEXT, shared_future);});
+# if !defined(DEBUG) && defined(CLIENT_INTERFACE)
+        }
+# endif
+#endif
+
+        if (DYNAMO_OPTION(shared_bbs))
+            hashtable_fragment_free(GLOBAL_DCONTEXT, shared_bb);
+        if (DYNAMO_OPTION(shared_traces))
+            hashtable_fragment_free(GLOBAL_DCONTEXT, shared_trace);
+        hashtable_fragment_free(GLOBAL_DCONTEXT, shared_future);
+        /* Do NOT free RAC table as its state cannot be rebuilt.
+         * We also do not free other RCT tables to avoid the time to rebuild them.
+         */
+    }
+
+#ifdef SHARING_STUDY
+    if (INTERNAL_OPTION(fragment_sharing_study)) {
+        print_shared_stats();
+        reset_shared_block_table(shared_blocks, &shared_blocks_lock);
+        reset_shared_block_table(shared_traces, &shared_traces_lock);
+    }
+#endif
+}
+
+/* free all state */
+void
+fragment_exit()
+{
+    /* case 7966: don't initialize at all for hotp_only & thin_client
+     * FIXME: could set initial sizes to 0 for all configurations, instead
+     */
+    if (RUNNING_WITHOUT_CODE_CACHE())
+        goto cleanup;
+
+#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
+    if (TRACEDUMP_ENABLED() && DYNAMO_OPTION(shared_traces)) {
+        /* write out all traces prior to deleting any, so links print nicely */
+        uint i;
+        fragment_t *f;
+        /* change_linking_lock is required for output_trace(), though there
+         * won't be any races at this point of exiting.
+         */
+        acquire_recursive_lock(&change_linking_lock);
+        TABLE_RWLOCK(shared_trace, read, lock);
+        for (i = 0; i < shared_trace->capacity; i++) {
+            f = shared_trace->table[i];
+            if (!REAL_FRAGMENT(f))
+                continue; 
+            if (SHOULD_OUTPUT_FRAGMENT(f->flags))
+                output_trace(GLOBAL_DCONTEXT, shared_pt, f, -1);
+        }
+        TABLE_RWLOCK(shared_trace, read, unlock);
+        release_recursive_lock(&change_linking_lock);
+        exit_trace_file(shared_pt);
+    }
+#endif
+
+#ifdef FRAGMENT_SIZES_STUDY
+    DOLOG(1, (LOG_FRAGMENT|LOG_STATS), {
+        print_size_results();
+    });
+#endif
+
+    fragment_reset_free();
+
+#ifdef RETURN_AFTER_CALL
+    if (dynamo_options.ret_after_call && rac_non_module_table.live_table != NULL) {
+        DODEBUG({
+            DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
+                hashtable_app_pc_load_statistics(GLOBAL_DCONTEXT,
+                                                 rac_non_module_table.live_table);
+            });
+            hashtable_app_pc_study(GLOBAL_DCONTEXT, rac_non_module_table.live_table,
+                                   0/*table consistent*/);
+        });
+        hashtable_app_pc_free(GLOBAL_DCONTEXT, rac_non_module_table.live_table);
+        HEAP_TYPE_FREE(GLOBAL_DCONTEXT, rac_non_module_table.live_table,
+                       app_pc_table_t, ACCT_AFTER_CALL, PROTECTED);
+        rac_non_module_table.live_table = NULL;
+    }
+    ASSERT(rac_non_module_table.persisted_table == NULL);
+    DELETE_LOCK(after_call_lock);
+#endif
+
+#if defined(RCT_IND_BRANCH) && defined(LINUX)
+    /* we do not free these tables in fragment_reset_free() b/c we
+     * would just have to build them all back up again in order to
+     * continue execution
+     */
+    if ((TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) ||
+         TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump))) &&
+        rct_global_table.live_table != NULL) {
+        DODEBUG({
+            DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
+                hashtable_app_pc_load_statistics(GLOBAL_DCONTEXT,
+                                                 rct_global_table.live_table);
+            });
+            hashtable_app_pc_study(GLOBAL_DCONTEXT, rct_global_table.live_table,
+                                   0/*table consistent*/);
+        });
+        hashtable_app_pc_free(GLOBAL_DCONTEXT, rct_global_table.live_table);
+        HEAP_TYPE_FREE(GLOBAL_DCONTEXT, rct_global_table.live_table, app_pc_table_t,
+                       ACCT_AFTER_CALL, PROTECTED);
+        rct_global_table.live_table = NULL;
+    } else
+        ASSERT(rct_global_table.live_table == NULL);
+    ASSERT(rct_global_table.persisted_table == NULL);
+#endif /* RCT_IND_BRANCH */
+
+    if (SHARED_FRAGMENTS_ENABLED()) {
+        /* tables are persistent across resets, only on heap for selfprot (case 7957) */
+        if (DYNAMO_OPTION(shared_bbs)) {
+            HEAP_TYPE_FREE(GLOBAL_DCONTEXT, shared_bb, fragment_table_t,
+                           ACCT_FRAG_TABLE, PROTECTED);
+            shared_bb = NULL;
+        } else
+            ASSERT(shared_bb == NULL);
+        if (DYNAMO_OPTION(shared_traces)) {
+            HEAP_TYPE_FREE(GLOBAL_DCONTEXT, shared_trace, fragment_table_t,
+                           ACCT_FRAG_TABLE, PROTECTED);
+            shared_trace = NULL;
+        } else
+            ASSERT(shared_trace == NULL);
+        HEAP_TYPE_FREE(GLOBAL_DCONTEXT, shared_future, fragment_table_t,
+                       ACCT_FRAG_TABLE, PROTECTED);
+        shared_future = NULL;
+    }
+
+    if (SHARED_IBT_TABLES_ENABLED()) {
+        HEAP_TYPE_FREE(GLOBAL_DCONTEXT, dead_lists, dead_table_lists_t,
+                       ACCT_OTHER, PROTECTED);
+        dead_lists = NULL;
+    } else
+        ASSERT(dead_lists == NULL);
+
+    if (USE_SHARED_PT()) {
+        ASSERT(shared_pt != NULL);
+        HEAP_TYPE_FREE(GLOBAL_DCONTEXT, shared_pt, per_thread_t, ACCT_OTHER, PROTECTED);
+        shared_pt = NULL;
+    } else
+        ASSERT(shared_pt == NULL);
+
+    if (SHARED_IBT_TABLES_ENABLED())
+        DELETE_LOCK(dead_tables_lock);
+#ifdef SHARING_STUDY
+    if (INTERNAL_OPTION(fragment_sharing_study)) {
+        DELETE_LOCK(shared_blocks_lock);
+        DELETE_LOCK(shared_traces_lock);
+    }
+#endif
+ cleanup:
+    /* FIXME: we shouldn't need these locks anyway for hotp_only & thin_client */
+#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
+    DELETE_LOCK(tracedump_mutex);
+#endif
+#ifdef CLIENT_INTERFACE
+    process_client_flush_requests(NULL, GLOBAL_DCONTEXT, client_flush_requests,
+                                  false /* no flush */);
+    DELETE_LOCK(client_flush_request_lock);
+#endif
+    DELETE_LOCK(shared_cache_flush_lock);
+}
+
+/* Decrement the ref-count for any reference to table that the
+ * per_thread_t contains.  If could_be_live is true, will acquire write
+ * locks for the currently live tables. */
+/* NOTE: Can't inline in release build -- too many call sites? */
+static /* inline */ void
+dec_table_ref_count(dcontext_t *dcontext, ibl_table_t *table, bool could_be_live)
+{
+    ibl_table_t *live_table = NULL;
+    ibl_branch_type_t branch_type;
+
+    /* Search live tables. A live table's ref-count is decremented
+     * during a thread exit. */
+    /* FIXME If the table is more likely to be dead, we can reverse the order
+     * and search dead tables first. */
+    if (!DYNAMO_OPTION(ref_count_shared_ibt_tables))
+        return;
+    ASSERT(TESTALL(FRAG_TABLE_SHARED | FRAG_TABLE_IBL_TARGETED,
+                   table->table_flags));
+    if (could_be_live) {
+        for (branch_type = IBL_BRANCH_TYPE_START; 
+             branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+            /* We match based on lookup table addresses. We need to lock the table
+             * during the compare and hold the lock during the ref-count dec to
+             * prevent a race with it being moved to the dead list.
+             */
+            ibl_table_t *sh_table_ptr = TEST(FRAG_TABLE_TRACE, table->table_flags) ?
+                &shared_pt->trace_ibt[branch_type] : &shared_pt->bb_ibt[branch_type];
+            TABLE_RWLOCK(sh_table_ptr, write, lock);
+            if (table->table == sh_table_ptr->table) {
+                live_table = sh_table_ptr;
+                break;
+            }
+            TABLE_RWLOCK(sh_table_ptr, write, unlock);
+        }
+    }
+    if (live_table != NULL) {
+        /* During shutdown, the ref-count can reach 0. The table is freed
+         * in the fragment_exit() path. */
+        ASSERT(live_table->ref_count >= 1);
+        live_table->ref_count--;
+        TABLE_RWLOCK(live_table, write, unlock);
+    }
+    else { /* Search the dead tables list. */
+        dead_fragment_table_t *current = dead_lists->dead_tables;
+        dead_fragment_table_t *prev = NULL;
+
+        ASSERT(dead_lists->dead_tables != NULL);
+        ASSERT(dead_lists->dead_tables_tail != NULL);
+        /* We expect to be removing from the head of the list but due to
+         * races could be removing from the middle, i.e., if a preceding
+         * entry is about to be removed by another thread but the
+         * dead_tables_lock hasn't been acquired yet by that thread.
+         */
+        mutex_lock(&dead_tables_lock);
+        for (current = dead_lists->dead_tables; current != NULL;
+             prev = current, current = current->next) {
+            if (current->table_unaligned == table->table_unaligned) {
+                ASSERT_CURIOSITY(current->ref_count >= 1);
+                current->ref_count--;
+                if (current->ref_count == 0) {
+                    LOG(GLOBAL, LOG_FRAGMENT, 2,
+                        "dec_table_ref_count: table "PFX" cap %d at ref 0, freeing\n",
+                        current->table_unaligned, current->capacity);
+                    /* Unlink this table from the list. */
+                    if (prev != NULL)
+                        prev->next = current->next;
+                    if (current == dead_lists->dead_tables) {
+                        /* remove from the front */
+                        ASSERT(prev == NULL);
+                        dead_lists->dead_tables = current->next;
+                    }
+                    if (current == dead_lists->dead_tables_tail)
+                        dead_lists->dead_tables_tail = prev;
+                    hashtable_ibl_free_table(GLOBAL_DCONTEXT, 
+                                             current->table_unaligned,
+                                             current->table_flags,
+                                             current->capacity);
+                    heap_free(GLOBAL_DCONTEXT, current, sizeof(dead_fragment_table_t)
+                              HEAPACCT(ACCT_IBLTABLE));
+                    STATS_DEC(num_dead_shared_ibt_tables);
+                    STATS_INC(num_dead_shared_ibt_tables_freed);
+                }
+                break;
+            }
+        }
+        mutex_unlock(&dead_tables_lock);
+        ASSERT(current != NULL);
+    }
+}
+
+/* Decrement the ref-count for every shared IBT table that the
+ * per_thread_t has a reference to. */
+static void
+dec_all_table_ref_counts(dcontext_t *dcontext, per_thread_t *pt)
+{
+    /* We can also decrement ref-count for dead shared tables here. */
+    if (SHARED_IBT_TABLES_ENABLED()) {
+        ibl_branch_type_t branch_type;
+        for (branch_type = IBL_BRANCH_TYPE_START; 
+             branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+            if (DYNAMO_OPTION(shared_trace_ibt_tables)) {
+                ASSERT(pt->trace_ibt[branch_type].table != NULL);
+                dec_table_ref_count(dcontext, &pt->trace_ibt[branch_type],
+                                    true/*check live*/);
+            }
+            if (DYNAMO_OPTION(shared_bb_ibt_tables)) {
+                ASSERT(pt->bb_ibt[branch_type].table != NULL);
+                dec_table_ref_count(dcontext, &pt->bb_ibt[branch_type],
+                                    true/*check live*/);
+            }
+        }
+    }
+}
+
+/* re-initializes non-persistent memory */
+void
+fragment_thread_reset_init(dcontext_t *dcontext)
+{
+    per_thread_t *pt;
+    ibl_branch_type_t branch_type;
+
+    /* case 7966: don't initialize at all for hotp_only & thin_client */
+    if (RUNNING_WITHOUT_CODE_CACHE())
+        return;
+
+    pt = (per_thread_t *) dcontext->fragment_field;
+
+    /* important to init w/ cur timestamp to avoid this thread dec-ing ref
+     * count when it wasn't included in ref count init value!
+     * assumption: don't need lock to read flushtime_global atomically.
+     * when resetting, though, thread free & re-init is done before global free,
+     * so we have to explicitly set to 0 for that case.
+     */
+    pt->flushtime_last_update = (dynamo_resetting) ? 0 : flushtime_global;
+
+    /* set initial hashtable sizes */
+    hashtable_fragment_init(dcontext, &pt->bb, INIT_HTABLE_SIZE_BB,
+                            INTERNAL_OPTION(private_bb_load),
+                            (hash_function_t)INTERNAL_OPTION(alt_hash_func),
+                            0, 0 NAME("bblock"));
+
+    /* init routine will work for future_fragment_t* same as for fragment_t* */
+    hashtable_fragment_init(dcontext, &pt->future, INIT_HTABLE_SIZE_FUTURE,
+                            INTERNAL_OPTION(private_future_load),
+                            (hash_function_t)INTERNAL_OPTION(alt_hash_func),
+                            0 /* hash_mask_offset */, 0
+                            NAME("future"));
+    
+    /* The trace table now is not used by IBL routines, and
+     * therefore doesn't need a lookup table, we can also use the
+     * alternative hash functions and use a higher load.
+     */
+    if (PRIVATE_TRACES_ENABLED()) {
+        hashtable_fragment_init(dcontext, &pt->trace, INIT_HTABLE_SIZE_TRACE,
+                                INTERNAL_OPTION(private_trace_load),
+                                (hash_function_t)INTERNAL_OPTION(alt_hash_func),
+                                0 /* hash_mask_offset */,
+                                FRAG_TABLE_TRACE
+                                NAME("trace"));
+    }
+
+    /* We'll now have more control over hashtables based on branch
+     * type.  The most important of all is of course the return
+     * target table.  These tables should be populated only when
+     * we know that the entry is a valid target, a trace is
+     * created, and it is indeed targeted by an IBL.
+     */
+    /* These tables are targeted by both bb and trace routines */
+    for (branch_type = IBL_BRANCH_TYPE_START; 
+         branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+        if (!DYNAMO_OPTION(disable_traces)
+            /* If no traces and no bb ibl targets we point ibl at
+             * an empty trace table */
+            || !DYNAMO_OPTION(bb_ibl_targets)) {
+            if (!DYNAMO_OPTION(shared_trace_ibt_tables)) {
+                hashtable_ibl_myinit(dcontext, &pt->trace_ibt[branch_type],
+                                     DYNAMO_OPTION(private_trace_ibl_targets_init),
+                                     DYNAMO_OPTION(private_ibl_targets_load),
+                                     HASH_FUNCTION_NONE,
+                                     HASHTABLE_IBL_OFFSET(branch_type),
+                                     branch_type,
+                                     false, /* no lookup table */
+                                     (DYNAMO_OPTION(shared_traces) ?
+                                      FRAG_TABLE_TARGET_SHARED : 0) |
+                                     FRAG_TABLE_TRACE
+                                     NAME(ibl_trace_table_type_names[branch_type]));
+#ifdef HASHTABLE_STATISTICS
+                if (INTERNAL_OPTION(hashtable_ibl_stats)) {
+                    CHECK_UNPROT_STATS(pt->trace_ibt[branch_type]);
+                    /* for compatibility using an entry in the per-branch type stats */
+                    INIT_HASHTABLE_STATS(pt->trace_ibt[branch_type].
+                                         UNPROT_STAT(trace_ibl_stats[branch_type]));
+                } else {
+                    pt->trace_ibt[branch_type].unprot_stats = NULL;
+                }
+#endif /* HASHTABLE_STATISTICS */
+            }
+            else {
+                /* ensure table from last time (if we had a reset) not still there */
+                memset(&pt->trace_ibt[branch_type], 0, sizeof(pt->trace_ibt[branch_type]));
+                update_private_ptr_to_shared_ibt_table(dcontext, branch_type,
+                                                       true,  /* trace = yes */
+                                                       false, /* no adjust old
+                                                               * ref-count */
+                                                       true   /* lock */);
+#ifdef HASHTABLE_STATISTICS
+                if (INTERNAL_OPTION(hashtable_ibl_stats)) {
+                    ALLOC_UNPROT_STATS(dcontext, &pt->trace_ibt[branch_type]);
+                    CHECK_UNPROT_STATS(pt->trace_ibt[branch_type]);
+                    INIT_HASHTABLE_STATS(pt->trace_ibt[branch_type].
+                                         UNPROT_STAT(trace_ibl_stats[branch_type]));
+                } else {
+                    pt->trace_ibt[branch_type].unprot_stats = NULL;
+                }
+#endif
+            }
+        }
+
+        /* When targetting BBs, currently the source is assumed to be only a
+         * bb since traces going to a bb for the first time should mark it
+         * as a trace head.  Therefore the tables are currently only
+         * targeted by bb IBL routines.  It will be possible to later
+         * deal with trace heads and allow a trace to target a BB with
+         * the intent of modifying its THCI.
+         *
+         * (FIXME: having another table for THCI IBLs seems better than
+         * adding a counter (starting at -1) to all blocks and
+         * trapping when 0 for marking a trace head and again at 50
+         * for creating a trace.  And that is all of course after proving
+         * that doing it in DR has significant impact.)
+         *
+         * Note that private bb2bb transitions are not captured when
+         * we run with -shared_bbs.
+         */
+
+        /* These tables should be populated only when we know that the
+         * entry is a valid target, and it is indeed targeted by an
+         * IBL.  They have to be per-type so that our security
+         * policies are properly checked.
+         */
+        if (DYNAMO_OPTION(bb_ibl_targets)) {
+            if (!DYNAMO_OPTION(shared_bb_ibt_tables)) {
+                hashtable_ibl_myinit(dcontext, &pt->bb_ibt[branch_type],
+                                     DYNAMO_OPTION(private_bb_ibl_targets_init),
+                                     DYNAMO_OPTION(private_bb_ibl_targets_load),
+                                     HASH_FUNCTION_NONE,
+                                     HASHTABLE_IBL_OFFSET(branch_type),
+                                     branch_type,
+                                     false, /* no lookup table */
+                                     (DYNAMO_OPTION(shared_bbs) ?
+                                      FRAG_TABLE_TARGET_SHARED : 0)
+                                     NAME(ibl_bb_table_type_names[branch_type]));
+                /* mark as inclusive table for bb's - we in fact currently
+                 * keep only frags that are not FRAG_IS_TRACE_HEAD */
+#ifdef HASHTABLE_STATISTICS
+                if (INTERNAL_OPTION(hashtable_ibl_stats)) {
+                    /* for compatibility using an entry in the per-branch type stats */
+                    CHECK_UNPROT_STATS(pt->bb_ibt[branch_type]);
+                    /* FIXME: we don't expect trace_ibl_stats yet */
+                    INIT_HASHTABLE_STATS(pt->bb_ibt[branch_type].
+                                         UNPROT_STAT(bb_ibl_stats[branch_type]));
+                } else {
+                    pt->bb_ibt[branch_type].unprot_stats = NULL;
+                }
+#endif /* HASHTABLE_STATISTICS */
+            }
+            else {
+                /* ensure table from last time (if we had a reset) not still there */
+                memset(&pt->bb_ibt[branch_type], 0, sizeof(pt->bb_ibt[branch_type]));
+                update_private_ptr_to_shared_ibt_table(dcontext, branch_type,
+                                                       false, /* trace = no */
+                                                       false, /* no adjust old
+                                                               * ref-count */
+                                                       true   /* lock */);
+#ifdef HASHTABLE_STATISTICS
+                if (INTERNAL_OPTION(hashtable_ibl_stats)) {
+                    ALLOC_UNPROT_STATS(dcontext, &pt->bb_ibt[branch_type]);
+                    CHECK_UNPROT_STATS(pt->bb_ibt[branch_type]);
+                    INIT_HASHTABLE_STATS(pt->bb_ibt[branch_type].
+                                         UNPROT_STAT(trace_ibl_stats[branch_type]));
+                } else {
+                    pt->bb_ibt[branch_type].unprot_stats = NULL;
+                }
+#endif
+            }
+        }
+    }
+    ASSERT(IBL_BRANCH_TYPE_END == 3);
+
+#ifdef NATIVE_RETURN
+    hashtable_fragment_init(dcontext, &pt->deleted, INIT_HTABLE_SIZE_DELETED, 75, 
+                            (hash_function_t)INTERNAL_OPTION(alt_hash_func), 
+                            0 /* hash_mask_offset */, 0 NAME("deleted"));
+#endif
+    update_generated_hashtable_access(dcontext);
+}
+
+void
+fragment_thread_init(dcontext_t *dcontext)
+{
+    /* we allocate per_thread_t in the global heap solely for self-protection,
+     * even when turned off, since even with a lot of threads this isn't a lot of
+     * pressure on the global heap
+     */
+    per_thread_t *pt;
+
+    /* case 7966: don't initialize un-needed data for hotp_only & thin_client.
+     * FIXME: could set htable initial sizes to 0 for all configurations, instead.
+     * per_thread_t is pretty big, so we avoid it, though it costs us checks for
+     * hotp_only in the islinking-related routines.
+     */
+    if (RUNNING_WITHOUT_CODE_CACHE())
+        return;
+
+    pt = (per_thread_t *) global_heap_alloc(sizeof(per_thread_t) HEAPACCT(ACCT_OTHER));
+    dcontext->fragment_field = (void *) pt;
+
+    fragment_thread_reset_init(dcontext);
+
+#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
+    if (TRACEDUMP_ENABLED() && PRIVATE_TRACES_ENABLED()) {
+        pt->tracefile = open_log_file("traces", NULL, 0);
+        ASSERT(pt->tracefile != INVALID_FILE);
+        init_trace_file(pt);
+    }
+#endif
+#if defined(CLIENT_INTERFACE) && defined(CLIENT_SIDELINE)
+    ASSIGN_INIT_LOCK_FREE(pt->fragment_delete_mutex, fragment_delete_mutex);
+#endif
+
+    pt->could_be_linking = false;
+    pt->wait_for_unlink = false;
+    pt->about_to_exit = false;
+    pt->flush_queue_nonempty = false;
+    pt->waiting_for_unlink = create_event();
+    pt->finished_with_unlink = create_event();
+    ASSIGN_INIT_LOCK_FREE(pt->linking_lock, linking_lock);
+    pt->finished_all_unlink = create_event();
+    pt->soon_to_be_linking = false;
+    pt->at_syscall_at_flush = false;
+
+#ifdef PROFILE_LINKCOUNT
+    pt->tracedump_num_below_threshold = 0;
+    pt->tracedump_count_below_threshold = (linkcount_type_t) 0;
+#endif
+}
+
+#if defined(DEBUG) && defined(NATIVE_RETURN)
+static void
+study_and_free_hashtable(dcontext_t *dcontext, fragment_table_t *table)
+{
+    DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
+        hashtable_fragment_load_statistics(dcontext, table);
+    });
+    hashtable_fragment_reset(dcontext, table);
+    hashtable_fragment_free(dcontext, table);
+}
+#endif
+
+static bool
+check_flush_queue(dcontext_t *dcontext, fragment_t *was_I_flushed);
+
+/* frees all non-persistent memory */
+void
+fragment_thread_reset_free(dcontext_t *dcontext)
+{
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+    DEBUG_DECLARE(ibl_branch_type_t branch_type;)
+
+    /* case 7966: don't initialize at all for hotp_only & thin_client */
+    if (RUNNING_WITHOUT_CODE_CACHE())
+        return;
+
+    /* Dec ref count on any shared tables that are pointed to. */
+    dec_all_table_ref_counts(dcontext, pt);
+
+#ifdef DEBUG
+    /* for non-debug we do fast exit path and don't free local heap */
+    SELF_PROTECT_CACHE(dcontext, NULL, WRITABLE);
+
+    /* we remove flushed fragments from the htable, and they can be
+     * flushed after enter_threadexit() due to os_thread_stack_exit(),
+     * so we need to check the flush queue here
+     */
+    mutex_lock(&pt->linking_lock);
+    check_flush_queue(dcontext, NULL);
+    mutex_unlock(&pt->linking_lock);
+
+    /* For consistency we remove entries from the IBL targets
+     * tables before we remove them from the trace table.  However,
+     * we cannot free any fragments because for sure all of them will 
+     * be present in the trace table.
+     */
+    for (branch_type = IBL_BRANCH_TYPE_START; 
+         branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+        if (!DYNAMO_OPTION(disable_traces)
+            /* If no traces and no bb ibl targets we point ibl at
+             * an empty trace table */
+            || !DYNAMO_OPTION(bb_ibl_targets)) {
+            if (!DYNAMO_OPTION(shared_trace_ibt_tables)) {
+                DOLOG(2, LOG_FRAGMENT, {
+                        hashtable_ibl_dump_table(dcontext, &pt->trace_ibt[branch_type]);
+                    });
+                DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
+                    hashtable_ibl_load_statistics(dcontext, &pt->trace_ibt[branch_type]);
+                });
+                hashtable_ibl_myfree(dcontext, &pt->trace_ibt[branch_type]);
+            } else {
+# ifdef HASHTABLE_STATISTICS
+                if (INTERNAL_OPTION(hashtable_ibl_stats)) {
+                    print_hashtable_stats(dcontext, "Total",
+                                          shared_pt->trace_ibt[branch_type].name,
+                                          "trace ibl ",
+                                          get_branch_type_name(branch_type),
+                                          &pt->trace_ibt[branch_type].
+                                          UNPROT_STAT(trace_ibl_stats[branch_type]));
+                    DEALLOC_UNPROT_STATS(dcontext, &pt->trace_ibt[branch_type]);
+                }
+# endif
+                memset(&pt->trace_ibt[branch_type], 0,
+                       sizeof(pt->trace_ibt[branch_type]));
+            }
+        }
+        if (DYNAMO_OPTION(bb_ibl_targets)) {
+            if (!DYNAMO_OPTION(shared_bb_ibt_tables)) {
+                DOLOG(2, LOG_FRAGMENT, {
+                        hashtable_ibl_dump_table(dcontext, &pt->bb_ibt[branch_type]);
+                    });
+                DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
+                    hashtable_ibl_load_statistics(dcontext, &pt->bb_ibt[branch_type]);
+                });
+                hashtable_ibl_myfree(dcontext, &pt->bb_ibt[branch_type]);
+            } else {
+# ifdef HASHTABLE_STATISTICS
+                if (INTERNAL_OPTION(hashtable_ibl_stats)) {
+                    print_hashtable_stats(dcontext, "Total",
+                                          shared_pt->bb_ibt[branch_type].name,
+                                          "bb ibl ",
+                                          get_branch_type_name(branch_type),
+                                          &pt->bb_ibt[branch_type].
+                                          UNPROT_STAT(bb_ibl_stats[branch_type]));
+                    DEALLOC_UNPROT_STATS(dcontext, &pt->bb_ibt[branch_type]);
+                }
+# endif
+                memset(&pt->bb_ibt[branch_type], 0, sizeof(pt->bb_ibt[branch_type]));
+            }
+        }
+    }
+
+    /* case 7653: we can't free the main tables prior to freeing the contents
+     * of all of them, as link freeing involves looking up in the other tables.
+     */
+    if (PRIVATE_TRACES_ENABLED()) {
+        DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
+            hashtable_fragment_load_statistics(dcontext, &pt->trace);
+        });
+        hashtable_fragment_reset(dcontext, &pt->trace);
+    }
+    DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
+        hashtable_fragment_load_statistics(dcontext, &pt->bb);
+    });
+    hashtable_fragment_reset(dcontext, &pt->bb);
+    DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
+        hashtable_fragment_load_statistics(dcontext, &pt->future);
+    });
+    hashtable_fragment_reset(dcontext, &pt->future);
+
+    if (PRIVATE_TRACES_ENABLED())
+        hashtable_fragment_free(dcontext, &pt->trace);
+    hashtable_fragment_free(dcontext, &pt->bb);
+    hashtable_fragment_free(dcontext, &pt->future);
+
+# ifdef NATIVE_RETURN
+    study_and_free_hashtable(dcontext, &pt->deleted);
+# endif
+
+    SELF_PROTECT_CACHE(dcontext, NULL, READONLY);
+
+#else
+    /* Case 10807: Clients need to be informed of fragment deletions
+     * so we'll reset the relevant hash tables for CI release builds.
+     */
+# ifdef CLIENT_INTERFACE
+    if (PRIVATE_TRACES_ENABLED())
+        hashtable_fragment_reset(dcontext, &pt->trace);
+    hashtable_fragment_reset(dcontext, &pt->bb);
+# endif
+
+#endif /* !DEBUG */
+}
+
+/* atexit cleanup */
+void
+fragment_thread_exit(dcontext_t *dcontext)
+{
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+
+    /* case 7966: don't initialize at all for hotp_only & thin_client */
+    if (RUNNING_WITHOUT_CODE_CACHE())
+        return;
+
+#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
+    if (TRACEDUMP_ENABLED() && PRIVATE_TRACES_ENABLED()) {
+        /* write out all traces prior to deleting any, so links print nicely */
+        uint i;
+        fragment_t *f;
+        for (i = 0; i < pt->trace.capacity; i++) {
+            f = pt->trace.table[i];
+            if (!REAL_FRAGMENT(f))
+                continue; 
+            if (SHOULD_OUTPUT_FRAGMENT(f->flags))
+                output_trace(dcontext, pt, f, -1);
+        }
+        exit_trace_file(pt);
+    }
+#endif
+
+    fragment_thread_reset_free(dcontext);
+
+    /* events are global */
+    destroy_event(pt->waiting_for_unlink);
+    destroy_event(pt->finished_with_unlink);
+    destroy_event(pt->finished_all_unlink);
+    DELETE_LOCK(pt->linking_lock);
+
+#if defined(CLIENT_INTERFACE) && defined(CLIENT_SIDELINE)
+    DELETE_LOCK(pt->fragment_delete_mutex);
+#endif
+
+    global_heap_free(pt, sizeof(per_thread_t) HEAPACCT(ACCT_OTHER));
+}
+
+#ifdef LINUX
+void
+fragment_fork_init(dcontext_t *dcontext)
+{
+    /* FIXME: what about global file? */
+# if defined(INTERNAL) || defined(CLIENT_INTERFACE)
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+    if (TRACEDUMP_ENABLED() && PRIVATE_TRACES_ENABLED()) {
+        /* new log dir has already been created, so just open a new log file */
+        pt->tracefile = open_log_file("traces", NULL, 0);
+        ASSERT(pt->tracefile != INVALID_FILE);
+        init_trace_file(pt);
+    }
+# endif
+}
+#endif
+
+/* fragment_t heap layout looks like this:
+ *
+ *   fragment_t/trace_t
+ *   translation_info_t*, if necessary
+ *   array composed of different sizes of linkstub_t subclasses:
+ *     direct_linkstub_t
+ *     cbr_fallthrough_linkstub_t
+ *     indirect_linkstub_t
+ *   post_linkstub_t, if necessary
+ */
+static uint
+fragment_heap_size(uint flags, int direct_exits, int indirect_exits)
+{
+    uint total_sz;
+    ASSERT((direct_exits + indirect_exits > 0) || TEST(FRAG_COARSE_GRAIN, flags));
+    total_sz = FRAGMENT_STRUCT_SIZE(flags) +
+        linkstubs_heap_size(flags, direct_exits, indirect_exits);
+    /* we rely on a small heap size for our ushort offset at the end */
+    ASSERT(total_sz <= USHRT_MAX);
+    return total_sz;
+}
+
+/* Allocates memory for a fragment_t and linkstubs and initializes them, but
+ * does not do any fcache-related initialization.
+ */
+static fragment_t *
+fragment_create_heap(dcontext_t *dcontext, 
+                     int direct_exits, int indirect_exits, uint flags)
+{
+    dcontext_t *alloc_dc = FRAGMENT_ALLOC_DC(dcontext, flags);
+    uint heapsz = fragment_heap_size(flags, direct_exits, indirect_exits);
+    /* linkstubs are in an array immediately after the fragment_t/trace_t struct */
+    fragment_t *f = (fragment_t *)
+        nonpersistent_heap_alloc(alloc_dc, heapsz
+                                 HEAPACCT(TEST(FRAG_IS_TRACE, flags) ?
+                                          ACCT_TRACE : ACCT_FRAGMENT));
+    LOG(THREAD, LOG_FRAGMENT, 5,
+        "fragment heap size for flags 0x%08x, exits %d %d, is %d => "PFX"\n",
+        flags, direct_exits, indirect_exits, heapsz, f);
+
+    return f;
+}
+
+static void
+fragment_init_heap(fragment_t *f, app_pc tag, int direct_exits, int indirect_exits,
+                   uint flags)
+{
+    ASSERT(f != NULL);
+    f->flags = flags; /* MUST set before calling fcache_add_fragment or
+                       * FRAGMENT_EXIT_STUBS */
+    f->tag = tag;
+    /* Let fragment_create() fill in; other users are building fake fragments */
+    DODEBUG({ f->id = -1; });
+    f->next_vmarea = NULL; /* must be set by caller */
+    f->prev_vmarea = NULL; /* must be set by caller */
+    f->also.also_vmarea = NULL; /* must be set by caller */
+
+    linkstubs_init(FRAGMENT_EXIT_STUBS(f), direct_exits, indirect_exits, f);
+
+    /* initialize non-ibt entry to top of fragment (caller responsible for
+     * setting up prefix)
+     */
+    f->prefix_size = 0;
+#ifdef FRAGMENT_SIZES_STUDY
+    record_fragment_size(f->size, (flags & FRAG_IS_TRACE) != 0);
+#endif
+
+    f->in_xlate.incoming_stubs = NULL;
+#ifdef CUSTOM_TRACES_RET_REMOVAL
+    f->num_calls = 0;
+    f->num_rets = 0;
+#endif
+
+    /* trace-only fields */
+    if (TEST(FRAG_IS_TRACE, flags)) {
+        trace_only_t *t = TRACE_FIELDS(f);
+        t->bbs = NULL;
+        /* real num_bbs won't be set until after the trace is emitted,
+         * but we need a non-zero value for linkstub_fragment() 
+         */
+        t->num_bbs = 1;
+#ifdef PROFILE_RDTSC
+        t->count = 0UL;
+        t->total_time = (uint64) 0;
+#endif
+#ifdef SIDELINE_COUNT_STUDY
+        t->count_old_pre = (linkcount_type_t) 0;
+        t->count_old_post = (linkcount_type_t) 0;
+#endif
+    } 
+}
+
+/* Create a new fragment_t with empty prefix and return it.
+ * The fragment_t is allocated on the global or local heap, depending on the flags,
+ * unless FRAG_COARSE_GRAIN is set, in which case the fragment_t is a unique
+ * temporary struct that is NOT heap allocated and is only safe to use
+ * so long as the bb_building_lock is held!
+ */
+fragment_t *
+fragment_create(dcontext_t *dcontext, app_pc tag, int body_size,
+                int direct_exits, int indirect_exits, int exits_size, uint flags)
+{
+    fragment_t *f;
+    DEBUG_DECLARE(stats_int_t next_id;)
+    DOSTATS({
+        /* should watch this stat and if it gets too high need to re-do
+         * who needs the post-linkstub offset
+         */
+        if (linkstub_frag_offs_at_end(flags, direct_exits, indirect_exits))
+            STATS_INC(num_fragment_post_linkstub);
+    });
+
+    /* ensure no races during a reset */
+    ASSERT(!dynamo_resetting);
+
+    if (TEST(FRAG_COARSE_GRAIN, flags)) {
+        ASSERT(DYNAMO_OPTION(coarse_units));
+        ASSERT_OWN_MUTEX(true, &bb_building_lock);
+        ASSERT(!TEST(FRAG_IS_TRACE, flags));
+        ASSERT(TEST(FRAG_SHARED, flags));
+        ASSERT(fragment_prefix_size(flags) == 0);
+        ASSERT((direct_exits == 0 && indirect_exits == 1) ||
+               (indirect_exits == 0 && (direct_exits == 1 || direct_exits == 2)));
+        /* FIXME: eliminate this temp fragment and linkstubs and
+         * have custom emit and link code that does not require such data
+         * structures?  It would certainly be faster code.
+         * But would still want to record each exit's target in a convenient
+         * data structure, for later linking, unless we try to link in
+         * the same pass in which we emit indirect stubs.
+         * We could also use fragment_create() and free the resulting struct
+         * somewhere and switch to a wrapper at that point.
+         */
+        memset(&coarse_emit_fragment, 0, sizeof(coarse_emit_fragment));
+        f = (fragment_t *) &coarse_emit_fragment;
+        /* We do not mark as FRAG_FAKE since this is pretty much a real
+         * fragment_t, and we do want to walk its linkstub_t structs, which
+         * are present.
+         */
+    } else {
+        f = fragment_create_heap(dcontext, direct_exits, indirect_exits, flags);
+    }
+
+    fragment_init_heap(f, tag, direct_exits, indirect_exits, flags);
+
+    /*  To make debugging easier we assign coarse-grain ids in the same namespace
+     * as fine-grain fragments, though we won't remember them at all.
+     */
+    STATS_INC_ASSIGN(num_fragments, next_id);
+    IF_X64(ASSERT_TRUNCATE(f->id, int, next_id));
+    DOSTATS({ f->id = (int) next_id; });
+    DO_GLOBAL_STATS({
+        if (!TEST(FRAG_IS_TRACE, f->flags))
+            RSTATS_INC(num_bbs);
+    });
+#if defined(NATIVE_RETURN) && !defined(DEBUG)
+    num_fragments++;
+#endif
+    DOSTATS({
+        /* avoid double-counting for adaptive working set */
+        if (!fragment_lookup_deleted(dcontext, tag) && !TEST(FRAG_COARSE_GRAIN, flags))
+            STATS_INC(num_unique_fragments);
+    });
+    /* FIXME: make fragment count a release-build stat so we can do this in
+     * release builds
+     */
+    DOSTATS({
+        if (stats != NULL &&
+            (uint) GLOBAL_STAT(num_fragments) == INTERNAL_OPTION(reset_at_fragment_count)) {
+            schedule_reset(RESET_ALL);
+        }
+    });
+
+    /* size is a ushort
+     * our offsets are ushorts as well: they assume body_size is small enough, not size
+     */
+#ifdef CLIENT_INTERFACE
+    if (body_size + exits_size + fragment_prefix_size(flags) > MAX_FRAGMENT_SIZE) {
+        FATAL_USAGE_ERROR(INSTRUMENTATION_TOO_LARGE, 2,
+                          get_application_name(), get_application_pid());
+    }
+#endif
+    ASSERT(body_size + exits_size + fragment_prefix_size(flags) <= MAX_FRAGMENT_SIZE);
+    /* currently MAX_FRAGMENT_SIZE is USHRT_MAX, but future proofing */
+    ASSERT_TRUNCATE(f->size, ushort,
+                    (body_size + exits_size + fragment_prefix_size(flags)));
+    f->size = (ushort) (body_size + exits_size + fragment_prefix_size(flags));
+
+    /* fcache_add will fill in start_pc, next_fcache, 
+     * prev_fcache, and fcache_extra
+     */
+    fcache_add_fragment(dcontext, f);
+
+    /* after fcache_add_fragment so we can call get_fragment_coarse_info */
+    DOSTATS({
+        if (TEST(FRAG_SHARED, flags)) {
+            STATS_INC(num_shared_fragments);
+            if (TEST(FRAG_IS_TRACE, flags))
+                STATS_INC(num_shared_traces);
+            else if (TEST(FRAG_COARSE_GRAIN, flags)) {
+                coarse_info_t *info = get_fragment_coarse_info(f);
+                if (get_executable_area_coarse_info(f->tag) != info)
+                    STATS_INC(num_coarse_secondary);
+                STATS_INC(num_coarse_fragments);
+            } else
+                STATS_INC(num_shared_bbs);
+        } else {
+            STATS_INC(num_private_fragments);
+            if (TEST(FRAG_IS_TRACE, flags))
+                STATS_INC(num_private_traces);
+            else
+                STATS_INC(num_private_bbs);
+        }
+    });
+
+    /* wait until initialized fragment completely before dumping any stats */
+    DOLOG(1, LOG_FRAGMENT|LOG_VMAREAS, {
+        if (INTERNAL_OPTION(global_stats_interval) &&
+            (f->id % INTERNAL_OPTION(global_stats_interval) == 0)) {
+            LOG(GLOBAL, LOG_FRAGMENT, 1, "Created %d fragments\n", f->id);
+            dump_global_stats(false);
+        }
+        if (INTERNAL_OPTION(thread_stats_interval) && 
+            INTERNAL_OPTION(thread_stats)) {
+            /* FIXME: why do we need a new dcontext? */
+            dcontext_t *dcontext = get_thread_private_dcontext();
+            if (THREAD_STATS_ON(dcontext) &&
+                THREAD_STAT(dcontext, num_fragments) % INTERNAL_OPTION(thread_stats_interval) == 0) {
+                dump_thread_stats(dcontext, false);
+            }
+        }
+    });
+
+#ifdef WINDOWS
+    DOLOG(1, LOG_FRAGMENT|LOG_VMAREAS, {
+        if (f->id % 50000 == 0) {
+            LOG(GLOBAL, LOG_VMAREAS, 1,
+                "50K fragment check point: here are the loaded modules:\n");
+            print_modules(GLOBAL, DUMP_NOT_XML);
+            LOG(GLOBAL, LOG_VMAREAS, 1,
+                "50K fragment check point: here are the executable areas:\n");
+            print_executable_areas(GLOBAL);
+        }
+    });
+#endif
+
+    return f;
+}
+
+/* Creates a new fragment_t+linkstubs from the passed-in fragment and
+ * fills in linkstub_t and fragment_t fields, copying the fcache-related fields
+ * from the passed-in fragment (so be careful how the fields are used).
+ * Meant to be used to create a full fragment from a coarse-grain fragment.
+ * Caller is responsible for freeing via fragment_free() w/ the same dcontext
+ * passed in here.
+ */
+fragment_t *
+fragment_recreate_with_linkstubs(dcontext_t *dcontext, fragment_t *f_src)
+{
+    uint num_dir, num_indir;
+    uint size;
+    fragment_t *f_tgt;
+    instrlist_t *ilist;
+    linkstub_t *l;
+    cache_pc body_end_pc;
+    /* Not FAKE since has linkstubs, but still fake in a sense since no fcache
+     * slot -- need to mark that?
+     */
+    uint flags = (f_src->flags & ~FRAG_FAKE);
+    ASSERT_CURIOSITY(TEST(FRAG_COARSE_GRAIN, f_src->flags)); /* only use so far */
+    /* FIXME case 9325: build from tag here?  Need to exactly re-mangle + re-instrument.
+     * We use _exact to get any elided final jmp not counted in size
+     */
+    ilist = decode_fragment_exact(dcontext, f_src, NULL, NULL, f_src->flags,
+                                  &num_dir, &num_indir);
+    f_tgt = fragment_create_heap(dcontext, num_dir, num_indir, flags);
+    fragment_init_heap(f_tgt, f_src->tag, num_dir, num_indir, flags);
+
+    f_tgt->start_pc = f_src->start_pc;
+    /* Can't call this until we have start_pc set */
+    body_end_pc = set_linkstub_fields(dcontext, f_tgt, ilist, num_dir, num_indir,
+                                      false/*do not emit*/);
+    /* Calculate total size */
+    IF_X64(ASSERT_TRUNCATE(size, uint, (body_end_pc - f_tgt->start_pc)));
+    size = (uint) (body_end_pc - f_tgt->start_pc);
+    for (l = FRAGMENT_EXIT_STUBS(f_tgt); l != NULL; l = LINKSTUB_NEXT_EXIT(l)) {
+        if (!EXIT_HAS_LOCAL_STUB(l->flags, f_tgt->flags))
+            continue; /* it's kept elsewhere */
+        size += linkstub_size(dcontext, f_tgt, l);
+#ifdef CUSTOM_EXIT_STUBS
+        size += l->fixed_stub_offset;
+#endif
+    }
+    ASSERT_TRUNCATE(f_tgt->size, ushort, size);
+    f_tgt->size = (ushort) size;
+    ASSERT(TEST(FRAG_FAKE, f_src->flags) || size == f_src->size);
+    ASSERT_TRUNCATE(f_tgt->prefix_size, byte, fragment_prefix_size(f_src->flags));
+    f_tgt->prefix_size = (byte) fragment_prefix_size(f_src->flags);
+    ASSERT(TEST(FRAG_FAKE, f_src->flags) || f_src->prefix_size == f_tgt->prefix_size);
+    f_tgt->fcache_extra = f_src->fcache_extra;
+
+    instrlist_clear_and_destroy(dcontext, ilist);
+
+    return f_tgt;
+}
+
+/* Frees the storage associated with f.
+ * Callers should use fragment_delete() instead of this routine, unless they
+ * obtained their fragment_t from fragment_recreate_with_linkstubs().
+ */
+void
+fragment_free(dcontext_t *dcontext, fragment_t *f)
+{
+    dcontext_t *alloc_dc = FRAGMENT_ALLOC_DC(dcontext, f->flags);
+    uint heapsz;
+    int direct_exits = 0;
+    int indirect_exits = 0;
+    linkstub_t *l = FRAGMENT_EXIT_STUBS(f);
+    for (; l != NULL; l = LINKSTUB_NEXT_EXIT(l)) {
+        if (LINKSTUB_DIRECT(l->flags))
+            direct_exits++;
+        else {
+            ASSERT(LINKSTUB_INDIRECT(l->flags));
+            indirect_exits++;
+        }
+    }
+    heapsz = fragment_heap_size(f->flags, direct_exits, indirect_exits);
+
+    STATS_INC(num_fragments_deleted);
+
+    if (HAS_STORED_TRANSLATION_INFO(f)) {
+        ASSERT(FRAGMENT_TRANSLATION_INFO(f) != NULL);
+        translation_info_free(dcontext, FRAGMENT_TRANSLATION_INFO(f));
+    } else
+        ASSERT(FRAGMENT_TRANSLATION_INFO(f) == NULL);
+
+    /* N.B.: monitor_remove_fragment() was called in fragment_delete,
+     * which is assumed to have been called prior to fragment_free
+     */
+
+    linkstub_free_exitstubs(dcontext, f);
+
+    if ((f->flags & FRAG_IS_TRACE) != 0) {
+        trace_only_t *t = TRACE_FIELDS(f);
+        if (t->bbs != NULL) {
+            nonpersistent_heap_free(alloc_dc, t->bbs, t->num_bbs*sizeof(trace_bb_info_t)
+                                    HEAPACCT(ACCT_TRACE));
+        }
+        nonpersistent_heap_free(alloc_dc, f, heapsz HEAPACCT(ACCT_TRACE));
+    } 
+    else {
+        nonpersistent_heap_free(alloc_dc, f, heapsz HEAPACCT(ACCT_FRAGMENT));
+    }
+}
+
+/* Returns the end of the fragment body + any local stubs (excluding selfmod copy) */
+cache_pc
+fragment_stubs_end_pc(fragment_t *f)
+{
+    if (TEST(FRAG_SELFMOD_SANDBOXED, f->flags))
+        return FRAGMENT_SELFMOD_COPY_PC(f);
+    else
+        return f->start_pc + f->size;
+}
+
+/* Returns the end of the fragment body (excluding exit stubs and selfmod copy) */
+cache_pc
+fragment_body_end_pc(dcontext_t *dcontext, fragment_t *f)
+{
+    linkstub_t *l;
+    for (l = FRAGMENT_EXIT_STUBS(f); l; l = LINKSTUB_NEXT_EXIT(l)) {
+        if (EXIT_HAS_LOCAL_STUB(l->flags, f->flags)) {
+            return EXIT_STUB_PC(dcontext, f, l);
+        }
+    }
+    /* must be no stubs after fragment body */
+    return fragment_stubs_end_pc(f);
+}
+
+#ifdef PROFILE_LINKCOUNT
+linkcount_type_t
+get_total_linkcount(fragment_t *f)
+{
+    /* return total count of exit counters */
+    linkstub_t *l;
+    linkcount_type_t total = (linkcount_type_t) 0;
+    for (l = FRAGMENT_EXIT_STUBS(f); l != NULL; l = LINKSTUB_NEXT_EXIT(l)) {
+        total += l->count;
+    }
+    return total;
+}
+#endif
+
+#if defined(CLIENT_INTERFACE) && defined(CLIENT_SIDELINE)
+/* synchronization routines needed for sideline threads so they don't get
+ * fragments they are referencing deleted */
+
+void
+fragment_get_fragment_delete_mutex(dcontext_t *dcontext)
+{
+    mutex_lock(&(((per_thread_t *) dcontext->fragment_field)->fragment_delete_mutex));
+}
+
+void
+fragment_release_fragment_delete_mutex(dcontext_t *dcontext)
+{
+    mutex_unlock(&(((per_thread_t *) dcontext->fragment_field)->fragment_delete_mutex));
+}
+#endif
+
+/* cleaner to have own flags since there are no negative versions
+ * of FRAG_SHARED and FRAG_IS_TRACE for distinguishing from "don't care"
+ */
+enum {
+    LOOKUP_TRACE   = 0x001,
+    LOOKUP_BB      = 0x002,
+    LOOKUP_PRIVATE = 0x004,
+    LOOKUP_SHARED  = 0x008,
+};
+
+/* A lookup constrained by bb/trace and/or shared/private */
+static inline fragment_t *
+fragment_lookup_type(dcontext_t *dcontext, app_pc tag, uint lookup_flags)
+{
+    fragment_t *f;
+
+    LOG(THREAD, LOG_MONITOR, 6, "fragment_lookup_type "PFX" 0x%x\n",
+        tag, lookup_flags);
+    if (dcontext != GLOBAL_DCONTEXT && TEST(LOOKUP_PRIVATE, lookup_flags)) {
+        /* FIXME: add a hashtablex.h wrapper that checks #entries and
+         * grabs lock for us for all lookups? 
+         */
+        /* look at private tables */
+        per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+        /* case 147: traces take precedence over bbs */
+        if (PRIVATE_TRACES_ENABLED() && TEST(LOOKUP_TRACE, lookup_flags)) {
+            /* now try trace table */
+            f = hashtable_fragment_lookup(dcontext, (ptr_uint_t)tag, &pt->trace);
+            if (f->tag != NULL) {
+                ASSERT(f->tag == tag);
+                DOLOG(2, LOG_FRAGMENT, {
+                    if (DYNAMO_OPTION(shared_traces)) {
+                        /* ensure private trace never shadows shared trace */
+                        fragment_t *sf;
+                        read_lock(&shared_trace->rwlock);
+                        sf = hashtable_fragment_lookup(dcontext, (ptr_uint_t)tag,
+                                                       shared_trace);
+                        read_unlock(&shared_trace->rwlock);
+                        ASSERT(sf->tag == NULL);
+                    }
+                });
+                ASSERT(!TESTANY(FRAG_FAKE|FRAG_COARSE_GRAIN, f->flags));
+                return f;
+            }
+        }
+        if (TEST(LOOKUP_BB, lookup_flags) && pt->bb.entries > 0) {
+            /* basic block table last */
+            f = hashtable_fragment_lookup(dcontext, (ptr_uint_t)tag, &pt->bb);
+            if (f->tag != NULL) {
+                ASSERT(f->tag == tag);
+                DOLOG(2, LOG_FRAGMENT, {
+                    if (DYNAMO_OPTION(shared_bbs)) {
+                        /* ensure private bb never shadows shared bb, except for
+                         * temp privates for trace building
+                         */
+                        fragment_t *sf;
+                        read_lock(&shared_bb->rwlock);
+                        sf = hashtable_fragment_lookup(dcontext, (ptr_uint_t)tag,
+                                                       shared_bb);
+                        read_unlock(&shared_bb->rwlock);
+                        ASSERT(sf->tag == NULL || TEST(FRAG_TEMP_PRIVATE, f->flags));
+                    }
+                });
+                ASSERT(!TESTANY(FRAG_FAKE|FRAG_COARSE_GRAIN, f->flags));
+                return f;
+            }
+        }
+    }
+
+    if (TEST(LOOKUP_SHARED, lookup_flags)) {
+        if (DYNAMO_OPTION(shared_traces) && TEST(LOOKUP_TRACE, lookup_flags)) {
+            /* MUST look at shared trace table before shared bb table,
+             * since a shared trace can shadow a shared trace head
+             */
+            read_lock(&shared_trace->rwlock);
+            f = hashtable_fragment_lookup(dcontext, (ptr_uint_t)tag, shared_trace);
+            read_unlock(&shared_trace->rwlock);
+            if (f->tag != NULL) {
+                ASSERT(f->tag == tag);
+                ASSERT(!TESTANY(FRAG_FAKE|FRAG_COARSE_GRAIN, f->flags));
+                return f;
+            }
+        }
+        
+        if (DYNAMO_OPTION(shared_bbs) && TEST(LOOKUP_BB, lookup_flags)) {
+            /* MUST look at private trace table before shared bb table,
+             * since a private trace can shadow a shared trace head
+             */
+            read_lock(&shared_bb->rwlock);
+            f = hashtable_fragment_lookup(dcontext, (ptr_uint_t)tag, shared_bb);
+            read_unlock(&shared_bb->rwlock);
+            if (f->tag != NULL) {
+                ASSERT(f->tag == tag);
+                ASSERT(!TESTANY(FRAG_FAKE|FRAG_COARSE_GRAIN, f->flags));
+                return f;
+            }
+        }
+    }
+
+    return NULL;
+}
+
+/* lookup a fragment tag */
+fragment_t *
+fragment_lookup(dcontext_t *dcontext, app_pc tag)
+{
+    return fragment_lookup_type(dcontext, tag,
+                                LOOKUP_TRACE|LOOKUP_BB|LOOKUP_PRIVATE|LOOKUP_SHARED);
+}
+
+/* lookup a fragment tag, but only look in trace tables
+ * N.B.: because of shadowing this may not return what fragment_lookup() returns!
+ */
+fragment_t *
+fragment_lookup_trace(dcontext_t *dcontext, app_pc tag)
+{
+    return fragment_lookup_type(dcontext, tag, LOOKUP_TRACE|LOOKUP_PRIVATE|LOOKUP_SHARED);
+}
+
+/* lookup a fragment tag, but only look in bb tables
+ * N.B.: because of shadowing this may not return what fragment_lookup() returns!
+ */
+fragment_t *
+fragment_lookup_bb(dcontext_t *dcontext, app_pc tag)
+{
+    return fragment_lookup_type(dcontext, tag, LOOKUP_BB|LOOKUP_PRIVATE|LOOKUP_SHARED);
+}
+
+/* lookup a fragment tag, but only look in shared bb table
+ * N.B.: because of shadowing this may not return what fragment_lookup() returns!
+ */
+fragment_t *
+fragment_lookup_shared_bb(dcontext_t *dcontext, app_pc tag)
+{
+    return fragment_lookup_type(dcontext, tag, LOOKUP_BB|LOOKUP_SHARED);
+}
+
+/* lookup a fragment tag, but only look in tables that are the same shared-ness
+ * as flags.
+ * N.B.: because of shadowing this may not return what fragment_lookup() returns!
+ */
+fragment_t *
+fragment_lookup_same_sharing(dcontext_t *dcontext, app_pc tag, uint flags)
+{
+    return fragment_lookup_type(dcontext, tag, LOOKUP_TRACE|LOOKUP_BB|
+                                (TEST(FRAG_SHARED, flags) ?
+                                 LOOKUP_SHARED : LOOKUP_PRIVATE));
+}
+
+#ifdef DEBUG /*currently only used for debugging */
+static fragment_t *
+hashtable_pclookup(dcontext_t *dcontext, fragment_table_t *table, cache_pc pc)
+{
+    uint i;
+    fragment_t *f;
+    ASSERT_TABLE_SYNCHRONIZED(table, READWRITE); /* lookup requires read (or write) lock */
+    for (i = 0; i < table->capacity; i++) {
+        f = table->table[i];
+        if (!REAL_FRAGMENT(f))
+            continue;
+        if (pc >= f->start_pc && pc < (f->start_pc + f->size)) {
+            return f;
+        }
+    }
+    return NULL;
+}
+
+/* lookup a fragment pc in the fcache by walking all hashtables.
+ * we have more efficient methods (fcache_fragment_pclookup) so this is only
+ * used for debugging.
+ */
+fragment_t *
+fragment_pclookup_by_htable(dcontext_t *dcontext, cache_pc pc, fragment_t *wrapper)
+{
+    /* if every fragment is guaranteed to end in 1+ stubs (which
+     * is not true for DYNAMO_OPTION(separate_private_stubs) we can
+     * simply decode forward until we hit the stub and recover
+     * the linkstub_t* from there -- much more efficient than walking
+     * all the hashtables, plus nicely handles invisible & removed frags!
+     * FIXME: measure perf hit of pclookup, implement this decode strategy.
+     * also we can miss invisible or removed fragments (case 122) so we
+     * may want this regardless of performance -- see also FIXME below.
+     */
+    fragment_t *f;
+    per_thread_t *pt = NULL;
+    if (dcontext != GLOBAL_DCONTEXT) {
+        pt = (per_thread_t *) dcontext->fragment_field;
+        /* look at private traces first */
+        if (PRIVATE_TRACES_ENABLED()) {
+            f = hashtable_pclookup(dcontext, &pt->trace, pc);
+            if (f != NULL)
+                return f;
+        }
+    }
+    if (DYNAMO_OPTION(shared_traces)) {
+        /* then shared traces */
+        read_lock(&shared_trace->rwlock);
+        f = hashtable_pclookup(dcontext, shared_trace, pc);
+        read_unlock(&shared_trace->rwlock);
+        if (f != NULL)
+            return f;
+    }
+    if (DYNAMO_OPTION(shared_bbs)) {
+        /* then shared basic blocks */
+        read_lock(&shared_bb->rwlock);
+        f = hashtable_pclookup(dcontext, shared_bb, pc);
+        read_unlock(&shared_bb->rwlock);
+        if (f != NULL)
+            return f;
+    }
+    if (dcontext != GLOBAL_DCONTEXT) {
+        /* now private basic blocks */
+        f = hashtable_pclookup(dcontext, &pt->bb, pc);
+        if (f != NULL)
+            return f;
+#ifdef NATIVE_RETURN
+        /* look at deleted fragments too */
+        f = hashtable_pclookup(dcontext, &pt->deleted, pc);
+        if (f != NULL)
+            return f;
+#endif
+    }
+    if (DYNAMO_OPTION(coarse_units)) {
+        coarse_info_t *info = get_executable_area_coarse_info(pc);
+        while (info != NULL) { /* loop over primary and secondary unit */
+            cache_pc body;
+            app_pc tag = fragment_coarse_pclookup(dcontext, info, pc, &body);
+            if (tag != NULL) {
+                ASSERT(wrapper != NULL);
+                fragment_coarse_wrapper(wrapper, tag, body);
+                return wrapper;
+            }
+            ASSERT(info->frozen || info->non_frozen == NULL);
+            info = info->non_frozen;
+            ASSERT(info == NULL || !info->frozen);
+        }
+    }
+    /* FIXME: shared fragment may have been removed from hashtable but
+     * still be in cache, and e.g. handle_modified_code still needs to know about it --
+     * should walk deletion vector
+     */
+    return NULL;
+}
+#endif /* DEBUG */
+
+/* lookup a fragment pc in the fcache */
+fragment_t *
+fragment_pclookup(dcontext_t *dcontext, cache_pc pc, fragment_t *wrapper)
+{
+    /* Rather than walk every single hashtable, including the invisible table,
+     * and the pending-deletion list (case 3567), we find the fcache unit
+     * and walk it.
+     * An even more efficient alternative would be to decode backward, but
+     * that's not doable in general.
+     *
+     * If every fragment is guaranteed to end in 1+ stubs (which
+     * is not true for DYNAMO_OPTION(separate_{private,shared}_stubs) we can
+     * simply decode forward until we hit the stub and recover
+     * the linkstub_t* from there.
+     * Or we can decode until we hit a jmp and if it's to a linked fragment_t,
+     * search its incoming list.
+     * Stub decoding is complicated by CLIENT_INTERFACE custom stubs and by
+     * PROFILE_LINKCOUNT stub variations.
+     */
+    return fcache_fragment_pclookup(dcontext, pc, wrapper);
+}
+
+/* Performs a pclookup and if the result is a coarse-grain fragment, allocates
+ * a new fragment_t+linkstubs.
+ * Returns in alloc whether the returned fragment_t was allocated and needs to be
+ * freed by the caller via fragment_free().
+ * If no result is found, alloc is set to false.
+ * FIXME: use FRAG_RECREATED flag to indicate allocated instead?
+ */
+fragment_t *
+fragment_pclookup_with_linkstubs(dcontext_t *dcontext, cache_pc pc,
+                                 /*OUT*/bool *alloc)
+{
+    fragment_t wrapper;
+    fragment_t *f = fragment_pclookup(dcontext, pc, &wrapper);
+    ASSERT(alloc != NULL);
+    if (f != NULL && TEST(FRAG_COARSE_GRAIN, f->flags)) {
+        ASSERT(f == &wrapper);
+        f = fragment_recreate_with_linkstubs(dcontext, f);
+        *alloc = true;
+    } else
+        *alloc = false;
+    return f;
+}
+
+/* add f to the ftable */
+void
+fragment_add(dcontext_t *dcontext, fragment_t *f)
+{
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+    fragment_table_t *table = GET_FTABLE(pt, f->flags);
+    bool resized;
+    /* no future frags! */
+    ASSERT(!TEST(FRAG_IS_FUTURE, f->flags));
+
+    DOCHECK(0, {
+        fragment_t *existing = fragment_lookup(dcontext, f->tag);
+        ASSERT(existing == NULL ||
+               IF_CUSTOM_TRACES(/* we create and persist shadowed trace heads */
+                                (TEST(FRAG_IS_TRACE_HEAD, f->flags) ||
+                                 TEST(FRAG_IS_TRACE_HEAD, existing->flags)) ||)
+               /* private trace or temp can shadow shared bb */
+               (TESTANY(FRAG_IS_TRACE | FRAG_TEMP_PRIVATE, f->flags) &&
+                TEST(FRAG_SHARED, f->flags) != TEST(FRAG_SHARED, existing->flags)) ||
+               /* shared trace can shadow shared trace head, even with
+                * -remove_shared_trace_heads */
+               (TESTALL(FRAG_IS_TRACE | FRAG_SHARED, f->flags) &&
+                !TEST(FRAG_IS_TRACE, existing->flags) &&
+                TESTALL(FRAG_SHARED | FRAG_IS_TRACE_HEAD, existing->flags)));
+    });
+
+    /* We'd like the shared fragment table synch to be independent of the
+     * bb building synch (which may become more fine-grained in the future),
+     * so an add needs to hold the write lock to prevent conflicts with
+     * other adds.
+     * We may be able to have a scheme where study() and remove() are writers
+     * but add() is a reader -- but that's confusing and prone to errors in
+     * the future.
+     * We assume that synchronizing addition of the same tag is done through
+     * other means -- we cannot grab this while performing the lookup
+     * w/o making our read locks check to see if we're the writer,
+     * which is a perf hit.  Only the actual hashtable add is a "write".
+     */
+    TABLE_RWLOCK(table, write, lock);
+    resized = fragment_add_to_hashtable(dcontext, f, table);
+    TABLE_RWLOCK(table, write, unlock);
+
+    /* After resizing a table that is targeted by inlined IBL heads
+     * the current fragment will need to be repatched; but, we don't have
+     * to update the stubs when using per-type trace tables since the
+     * trace table itself is not targeted therefore resizing it doesn't matter.
+     */
+
+#ifdef SHARING_STUDY
+    if (INTERNAL_OPTION(fragment_sharing_study)) {
+        if (TEST(FRAG_IS_TRACE, f->flags))
+            add_shared_block(shared_traces, &shared_traces_lock, f);
+        else
+            add_shared_block(shared_blocks, &shared_blocks_lock, f);
+    }
+#endif
+}
+
+#ifdef RETURN_STACK
+void 
+fragment_delete_from_return_stack(dcontext_t *dcontext, fragment_t *f)
+{
+    byte *rsp = dcontext->top_of_rstack;
+    app_pc ret_pc;
+    while (rsp < dcontext->rstack) {
+        /* get return pc in code cache */
+        ret_pc = *((app_pc *)rsp);
+        if (ret_pc >= f->start_pc && ret_pc < f->start_pc + f->size) {
+            /* clear both the app pc and our pc, easier to catch bugs */
+            *((int *)rsp) = 0;
+            *(((int *)rsp)+1) = 0;
+            LOG(THREAD, LOG_FRAGMENT, 4, "Clearing return stack entry for F%d\n", f->id);
+        }
+        rsp += 8;
+    }
+}
+#endif
+
+/* Many options, use macros in fragment.h for readability
+ * If output:
+ *   dumps f to trace file
+ * If remove:
+ *   removes f from ftable
+ * If unlink:
+ *   if f is linked, unlinks f
+ *   removes f from incoming link tables
+ * If fcache:
+ *   deletes f from fcache unit
+ */
+void
+fragment_delete(dcontext_t *dcontext, fragment_t *f, uint actions)
+{
+    LOG(THREAD, LOG_FRAGMENT, 3,
+        "fragment_delete: *"PFX" F%d("PFX")."PFX" %s 0x%x\n",
+        f, f->id, f->tag, f->start_pc,
+        TEST(FRAG_IS_TRACE, f->flags) ? "trace" : "bb", actions);
+    DOLOG(1, LOG_FRAGMENT, {
+        if ((f->flags & FRAG_CANNOT_DELETE) != 0) {
+            LOG(THREAD, LOG_FRAGMENT, 2,
+                "ERROR: trying to delete undeletable F%d("PFX") 0x%x\n",
+                f->id, f->tag, actions);
+        }
+    });
+    ASSERT((f->flags & FRAG_CANNOT_DELETE) == 0);
+    ASSERT((f->flags & FRAG_IS_FUTURE) == 0);
+
+    /* ensure the actual free of a shared fragment is done only
+     * after a multi-stage flush or a reset
+     */
+    ASSERT(!TEST(FRAG_SHARED, f->flags) || TEST(FRAG_WAS_DELETED, f->flags) ||
+           dynamo_exited || dynamo_resetting || is_self_allsynch_flushing());
+
+#if defined(CLIENT_INTERFACE) && defined(CLIENT_SIDELINE)
+    /* need to protect ability to reference frag fields and fcache space */
+    /* all other options are mostly notification */
+    if (!TEST(FRAGDEL_NO_HEAP, actions) || !TEST(FRAGDEL_NO_FCACHE, actions))
+        fragment_get_fragment_delete_mutex(dcontext);
+#endif
+
+#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
+    if (!TEST(FRAGDEL_NO_OUTPUT, actions)) {
+        if (TEST(FRAGDEL_NEED_CHLINK_LOCK, actions) && TEST(FRAG_SHARED, f->flags))
+            acquire_recursive_lock(&change_linking_lock);
+        else {
+            ASSERT(!TEST(FRAG_SHARED, f->flags) ||
+                   self_owns_recursive_lock(&change_linking_lock));
+        }
+        fragment_output(dcontext, f);
+        if (TEST(FRAGDEL_NEED_CHLINK_LOCK, actions) && TEST(FRAG_SHARED, f->flags))
+            release_recursive_lock(&change_linking_lock);
+    }
+#endif
+
+    if (!TEST(FRAGDEL_NO_MONITOR, actions))
+        monitor_remove_fragment(dcontext, f);
+
+    if (!TEST(FRAGDEL_NO_UNLINK, actions)) {
+        if (TEST(FRAGDEL_NEED_CHLINK_LOCK, actions) && TEST(FRAG_SHARED, f->flags))
+            acquire_recursive_lock(&change_linking_lock);
+        else {
+            ASSERT(!TEST(FRAG_SHARED, f->flags) ||
+                   self_owns_recursive_lock(&change_linking_lock));
+        }
+        if ((f->flags & FRAG_LINKED_INCOMING) != 0)
+            unlink_fragment_incoming(dcontext, f);
+        if ((f->flags & FRAG_LINKED_OUTGOING) != 0)
+            unlink_fragment_outgoing(dcontext, f);
+        incoming_remove_fragment(dcontext, f);
+        if (TEST(FRAGDEL_NEED_CHLINK_LOCK, actions) && TEST(FRAG_SHARED, f->flags))
+            release_recursive_lock(&change_linking_lock);
+    }
+
+    if (!TEST(FRAGDEL_NO_HTABLE, actions))
+        fragment_remove(dcontext, f);
+
+    if (!TEST(FRAGDEL_NO_VMAREA, actions))
+        vm_area_remove_fragment(dcontext, f);
+
+    if (!TEST(FRAGDEL_NO_FCACHE, actions)) {
+#ifdef NATIVE_RETURN
+        LOG(GLOBAL, LOG_FRAGMENT, 3,
+            "someone tried to delete F%d from the cache!\n", f->id);
+        /* HACK: just don't delete it!  count on removing it from the hashtable
+         * to be enough
+         */
+#else
+        fcache_remove_fragment(dcontext, f);
+#endif
+    }
+
+#ifdef RETURN_STACK
+    fragment_delete_from_return_stack(dcontext, f);
+#endif
+#ifdef SIDELINE
+    if (dynamo_options.sideline)
+        sideline_fragment_delete(f);
+#endif
+#ifdef CLIENT_INTERFACE
+    if (dr_fragment_deleted_hook_exists() &&
+        (!TEST(FRAGDEL_NO_HEAP, actions) || !TEST(FRAGDEL_NO_FCACHE, actions)))
+        instrument_fragment_deleted(dcontext, f->tag, f->flags);
+#endif
+#ifdef LINUX
+    if (INTERNAL_OPTION(profile_pcs))
+        pcprofile_fragment_deleted(dcontext, f);
+#endif
+    if (!TEST(FRAGDEL_NO_HEAP, actions)) {
+#ifdef NATIVE_RETURN
+        /* only free future frags */
+        if ((f->flags & FRAG_IS_FUTURE) != 0)
+#endif
+            fragment_free(dcontext, f);
+    }
+#if defined(CLIENT_INTERFACE) && defined(CLIENT_SIDELINE)
+    if (!TEST(FRAGDEL_NO_HEAP, actions) || !TEST(FRAGDEL_NO_FCACHE, actions))
+        fragment_release_fragment_delete_mutex(dcontext);
+#endif
+
+
+}
+
+/* Record translation info.  Typically used for pending-delete fragments
+ * whose original app code cannot be trusted as it has been modified (case
+ * 3559).
+ * Caller is required to take care of synch (typically this is called
+ * during a flush or during fragment emit)
+ */
+void
+fragment_record_translation_info(dcontext_t *dcontext, fragment_t *f, instrlist_t *ilist)
+{
+    ASSERT(!NEED_SHARED_LOCK(f->flags) ||
+           OWN_MUTEX(&bb_building_lock) ||
+           OWN_MUTEX(&trace_building_lock) ||
+           is_self_flushing());
+    /* We require that either the FRAG_WAS_DELETED flag is set, to
+     * indicate there is allocated memory in the live field that needs
+     * to be freed, or that the FRAG_HAS_TRANSLATION_INFO field is
+     * set, indicating that there is a special appended field pointing
+     * to the translation info.
+     */
+    if (TEST(FRAG_HAS_TRANSLATION_INFO, f->flags)) {
+        ASSERT(!TEST(FRAG_WAS_DELETED, f->flags));
+        *(FRAGMENT_TRANSLATION_INFO_ADDR(f)) =
+            record_translation_info(dcontext, f, ilist);
+        ASSERT(FRAGMENT_TRANSLATION_INFO(f) != NULL);
+        STATS_INC(num_fragment_translation_stored);
+    } else if (TEST(FRAG_WAS_DELETED, f->flags)) {
+        ASSERT(f->in_xlate.incoming_stubs == NULL);
+        if (INTERNAL_OPTION(safe_translate_flushed)) {
+            f->in_xlate.translation_info = record_translation_info(dcontext, f, ilist);
+            ASSERT(f->in_xlate.translation_info != NULL);
+            ASSERT(FRAGMENT_TRANSLATION_INFO(f) == f->in_xlate.translation_info);
+            STATS_INC(num_fragment_translation_stored);
+#ifdef INTERNAL
+            DODEBUG({
+                if (INTERNAL_OPTION(stress_recreate_pc)) {
+                    /* verify recreation */
+                    stress_test_recreate(dcontext, f, NULL);
+                }
+            });
+#endif
+        } else
+            f->in_xlate.translation_info = NULL;
+    } else
+        ASSERT_NOT_REACHED();
+}
+
+/* Removes the shared fragment f from all lookup tables in a safe
+ * manner that does not require a full flush synch.
+ * This routine can be called without synchronizing with other threads.
+ */
+void
+fragment_remove_shared_no_flush(dcontext_t *dcontext, fragment_t *f)
+{
+    DEBUG_DECLARE(bool shared_ibt_table_used = !TEST(FRAG_IS_TRACE, f->flags) ?
+                  DYNAMO_OPTION(shared_bb_ibt_tables) :
+                  DYNAMO_OPTION(shared_trace_ibt_tables);)
+
+    ASSERT_NOT_IMPLEMENTED(!TEST(FRAG_COARSE_GRAIN, f->flags));
+ 
+    /* Strategy: ensure no races in updating table or links by grabbing the high-level
+     * locks that are used to synchronize additions to the table itself.
+     * Then, simply remove directly from DR-only tables, and safely from ib tables.
+     * FIXME: There are still risks that the fragment's link state may change
+     */
+    LOG(THREAD, LOG_FRAGMENT, 3, "fragment_remove_shared_no_flush: F%d\n", f->id);
+    ASSERT(TEST(FRAG_SHARED, f->flags));
+    if (TEST(FRAG_IS_TRACE, f->flags)) {
+        mutex_lock(&trace_building_lock);
+    }
+    /* grab bb building lock even for traces to further prevent link changes */
+    mutex_lock(&bb_building_lock);
+
+    if (TEST(FRAG_WAS_DELETED, f->flags)) {
+        /* since caller can't grab locks, we can have a race where someone
+         * else deletes first -- in that case nothing to do
+         */
+        STATS_INC(shared_delete_noflush_race);
+        mutex_unlock(&bb_building_lock);
+        if (TEST(FRAG_IS_TRACE, f->flags))
+            mutex_unlock(&trace_building_lock);
+        return;
+    }
+
+    /* FIXME: try to share code w/ fragment_unlink_for_deletion() */
+
+    /* Make link changes atomic.  We also want vm_area_remove_fragment and
+     * marking as deleted to be atomic so we grab vm_areas lock up front.
+     */
+    acquire_recursive_lock(&change_linking_lock);
+    acquire_vm_areas_lock(dcontext, f->flags);
+
+    /* FIXME: share all this code w/ vm_area_unlink_fragments()
+     * The work there is just different enough to make that hard, though.
+     */
+    if (TEST(FRAG_LINKED_OUTGOING, f->flags))
+        unlink_fragment_outgoing(GLOBAL_DCONTEXT, f);
+    if (TEST(FRAG_LINKED_INCOMING, f->flags))
+        unlink_fragment_incoming(GLOBAL_DCONTEXT, f);
+    incoming_remove_fragment(GLOBAL_DCONTEXT, f);
+
+    /* remove from ib lookup tables in a safe manner. this removes the
+     * frag only from this thread's tables OR from shared tables.
+     */
+    fragment_prepare_for_removal(GLOBAL_DCONTEXT, f);
+    /* fragment_remove ignores the ibl tables for shared fragments */
+    fragment_remove(GLOBAL_DCONTEXT, f);
+    /* FIXME: we don't currently remove from thread-private ibl tables as that
+     * requires walking all of the threads. */
+    ASSERT_NOT_IMPLEMENTED(!IS_IBL_TARGET(f->flags) || shared_ibt_table_used);
+
+    vm_area_remove_fragment(dcontext, f);
+    /* case 8419: make marking as deleted atomic w/ fragment_t.also_vmarea field
+     * invalidation, so that users of vm_area_add_to_list() can rely on this
+     * flag to determine validity
+     */
+    f->flags |= FRAG_WAS_DELETED;
+
+    release_vm_areas_lock(dcontext, f->flags);
+    release_recursive_lock(&change_linking_lock);
+
+    /* if a flush occurs, this fragment will be ignored -- so we must store
+     * translation info now, just in case
+     */
+    if (!TEST(FRAG_HAS_TRANSLATION_INFO, f->flags))
+        fragment_record_translation_info(dcontext, f, NULL);
+
+    /* try to catch any potential races */
+    ASSERT(!TEST(FRAG_LINKED_OUTGOING, f->flags));
+    ASSERT(!TEST(FRAG_LINKED_INCOMING, f->flags));
+
+    mutex_unlock(&bb_building_lock);
+    if (TEST(FRAG_IS_TRACE, f->flags)) {
+        mutex_unlock(&trace_building_lock);
+    }
+
+    /* no locks can be held when calling this, but f is already unreachable,
+     * so can do this outside of locks
+     */
+    add_to_lazy_deletion_list(dcontext, f);
+}
+
+/* Prepares a fragment for delayed deletion by unlinking it.
+ * Caller is responsible for calling vm_area_remove_fragment().
+ * Caller must hold the change_linking_lock if f is shared.
+ */
+void
+fragment_unlink_for_deletion(dcontext_t *dcontext, fragment_t *f)
+{
+    ASSERT(!TEST(FRAG_SHARED, f->flags) ||
+           self_owns_recursive_lock(&change_linking_lock));
+    /* this is not an error since fcache unit flushing puts lazily-deleted
+     * fragments onto its list to ensure they are in the same pending
+     * delete entry as the normal fragments -- so this routine becomes
+     * a nop for them
+     */
+    if (TEST(FRAG_WAS_DELETED, f->flags)) {
+        LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 5,
+            "NOT unlinking F%d("PFX") for deletion\n", f->id, f->start_pc);
+        STATS_INC(deleted_frags_re_deleted);
+        return;
+    }
+    LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 5,
+        "unlinking F%d("PFX") for deletion\n", f->id, f->start_pc);
+#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
+    /* we output now to avoid problems reading component blocks
+     * of traces after source modules are unloaded
+     */
+    fragment_output(dcontext, f);
+#endif
+    if (TEST(FRAG_LINKED_OUTGOING, f->flags))
+        unlink_fragment_outgoing(dcontext, f);
+    if (TEST(FRAG_LINKED_INCOMING, f->flags))
+        unlink_fragment_incoming(dcontext, f);
+    /* need to remove outgoings from others' incoming and
+     * redirect others' outgoing to a future.  former must be
+     * done before we remove from hashtable, and latter must be
+     * done now to avoid other fragments jumping into stale code,
+     * so we do it here and not when we do the real fragment_delete().
+     * we don't need to do this for private fragments, but we do anyway
+     * so that we can use the fragment_t.incoming_stubs field as a union.
+     */
+    incoming_remove_fragment(dcontext, f);
+
+    if (TEST(FRAG_SHARED, f->flags)) {
+        /* we shouldn't need to worry about someone else changing the
+         * link status, since nobody else is allowed to be in DR now,
+         * and afterward they all must invalidate any ptrs they hold
+         * to flushed fragments, and flushed fragments are not
+         * reachable via hashtable or incoming lists!
+         */
+        /* ASSUMPTION: monitor_remove_fragment does NOT need to
+         * be called for all threads, since private trace head
+         * ctrs are cleared lazily (only relevant here for
+         * -shared_traces) and invalidating last_{exit,fragment}
+         * is done by the trace overlap and abort in the main
+         * flush loop.
+         */
+    }
+
+    /* need to remove from htable
+     * we used to only do fragment_prepare_for_removal() (xref case 1808)
+     * for private fragments, but for case 3559 we want to free up the
+     * incoming field at unlink time, and we must do all 3 of unlink,
+     * vmarea, and htable freeing at once.
+     */
+    fragment_remove(dcontext, f);
+
+    /* let recreate_fragment_ilist() know that this fragment is
+     * pending deletion and might no longer match the app's state.
+     * for shared fragments, also lets people know f is not in a normal
+     * vmarea anymore (though actually moving f is up to the caller).
+     * additionally the flag indicates that translation info was allocated
+     * for this fragment.
+     */
+    f->flags |= FRAG_WAS_DELETED;
+
+    /* the original app code cannot be used to recreate state, so we must
+     * store translation info now
+     */
+    if (!TEST(FRAG_HAS_TRANSLATION_INFO, f->flags))
+        fragment_record_translation_info(dcontext, f, NULL);
+
+    STATS_INC(fragments_unlinked_for_deletion);
+}
+
+/* When shared IBT tables are used, update thread-private state
+ * to reflect the current parameter values -- hash mask, table address --
+ * for the shared ftable.
+ */
+static bool
+update_private_ibt_table_ptrs(dcontext_t *dcontext, ibl_table_t *ftable
+                              _IF_DEBUG(fragment_entry_t **orig_table))
+{
+    bool table_change = false;
+
+    if (TEST(FRAG_TABLE_SHARED, ftable->table_flags)) {
+
+        per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+
+        if (TEST(FRAG_TABLE_TRACE, ftable->table_flags) &&
+            ftable->table != pt->trace_ibt[ftable->branch_type].table) {
+            DODEBUG({
+                if (orig_table != NULL)
+                    *orig_table =
+                        pt->trace_ibt[ftable->branch_type].table;
+            });
+            table_change = true;
+        }
+        else if (DYNAMO_OPTION(bb_ibl_targets) &&
+                 !TEST(FRAG_TABLE_TRACE, ftable->table_flags) &&
+                 ftable->table != pt->bb_ibt[ftable->branch_type].table) {
+            DODEBUG({
+                if (orig_table != NULL)
+                    *orig_table =
+                        pt->bb_ibt[ftable->branch_type].table;
+            });
+            table_change = true;
+        }
+        if (table_change) {
+            update_private_ptr_to_shared_ibt_table(dcontext, ftable->branch_type,
+                                                   TEST(FRAG_TABLE_TRACE,
+                                                        ftable->table_flags),
+                                                   true, /* adjust old
+                                                          * ref-count */
+                                                   true  /* lock */);
+            DODEBUG({
+                if (orig_table != NULL)
+                    ASSERT(ftable->table != *orig_table);
+            });
+        }
+#ifdef DEBUG
+        else if (orig_table != NULL)
+            *orig_table = NULL;
+#endif
+    }
+    return table_change;
+}
+
+/* Update the thread-private ptrs for the dcontext to point to the
+ * currently "live" shared IBT table for branch_type.
+ * When adjust_ref_count==true, adjust the ref-count for the old table
+ * that the dcontext currently points to.
+ * When lock_table==true, lock the shared table prior to manipulating
+ * it. If this is false, the caller must have locked the table already.
+ * NOTE: If adjust_ref_count=true, lock_table should be true also and
+ * the caller should NOT hold the table lock, since the underlying
+ * routines that manipulate the ref count lock the table.
+ */
+static inline void
+update_private_ptr_to_shared_ibt_table(dcontext_t *dcontext,
+                                       ibl_branch_type_t branch_type, bool trace,
+                                       bool adjust_old_ref_count, bool lock_table)
+{
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+    ibl_table_t *sh_table_ptr = trace ? &shared_pt->trace_ibt[branch_type] :
+        &shared_pt->bb_ibt[branch_type];
+    ibl_table_t *pvt_table_ptr = trace ? &pt->trace_ibt[branch_type] :
+        &pt->bb_ibt[branch_type];
+
+    /* Point to the new table. The shared table must be locked prior to
+     * accessing any of its fields. */
+    if (lock_table)
+        TABLE_RWLOCK(sh_table_ptr, write, lock);
+    ASSERT_OWN_WRITE_LOCK(true, &sh_table_ptr->rwlock);
+
+    /* We can get here multiple times due to callers being racy */
+    if (pvt_table_ptr->table == sh_table_ptr->table) {
+        SYSLOG_INTERNAL_WARNING_ONCE("racy private ptr to shared table update");
+        if (lock_table)
+            TABLE_RWLOCK(sh_table_ptr, write, unlock);
+        return;
+    }
+
+    /* Decrement the ref-count for any old table that is pointed to. */
+    if (adjust_old_ref_count) {
+        dec_table_ref_count(dcontext, pvt_table_ptr, false/*can't be live*/);
+    }
+
+    /* We must hold at least the read lock when writing, else we could grab
+     * an inconsistent mask/lookuptable pair if another thread is in the middle
+     * of resizing the table (case 10405).
+     */
+    /* Only data for one set of tables is stored in TLS -- for the trace
+     * tables in the default config OR the BB tables in shared BBs
+     * only mode.
+     */
+    if ((trace || SHARED_BB_ONLY_IB_TARGETS()) &&
+        DYNAMO_OPTION(ibl_table_in_tls))
+        update_lookuptable_tls(dcontext, sh_table_ptr);
+
+    ASSERT(pvt_table_ptr->table != sh_table_ptr->table);
+    pvt_table_ptr->table = sh_table_ptr->table;
+    pvt_table_ptr->hash_mask = sh_table_ptr->hash_mask;
+    /* We copy the unaligned value over also because it's used for matching
+     * in the dead table list. */
+    pvt_table_ptr->table_unaligned = sh_table_ptr->table_unaligned;
+    pvt_table_ptr->table_flags = sh_table_ptr->table_flags;
+    sh_table_ptr->ref_count++;
+    ASSERT(sh_table_ptr->ref_count > 0);
+
+    DODEBUG({
+        LOG(THREAD, LOG_FRAGMENT|LOG_STATS, 2,
+            "update_table_ptrs %s-%s table: addr "PFX", mask "PIFX"\n",
+            trace ? "trace" : "BB", sh_table_ptr->name,
+            sh_table_ptr->table, sh_table_ptr->hash_mask);
+        if ((trace || SHARED_BB_ONLY_IB_TARGETS()) &&
+            DYNAMO_OPTION(ibl_table_in_tls)) {
+            local_state_extended_t *state =
+                (local_state_extended_t *) dcontext->local_state;
+            LOG(THREAD, LOG_FRAGMENT|LOG_STATS, 2,
+                "TLS state %s-%s table: addr "PFX", mask "PIFX"\n",
+                trace ? "trace" : "BB", sh_table_ptr->name,
+                state->table_space.table[branch_type].lookuptable,
+                state->table_space.table[branch_type].hash_mask);
+        }
+    });
+#ifdef HASHTABLE_STATISTICS
+    if (INTERNAL_OPTION(hashtable_ibl_entry_stats)) {
+        pvt_table_ptr->entry_stats_to_lookup_table =
+            sh_table_ptr->entry_stats_to_lookup_table;
+    }
+    else
+        pvt_table_ptr->entry_stats_to_lookup_table = 0;
+#endif
+    if (lock_table)
+        TABLE_RWLOCK(sh_table_ptr, write, unlock);
+
+    /* We don't need the lock for this, and holding it will have rank order
+     * issues with disassembling in debug builds */
+    if (PRIVATE_TRACES_ENABLED() || DYNAMO_OPTION(bb_ibl_targets))
+        update_generated_hashtable_access(dcontext);
+
+    STATS_INC(num_shared_ibt_table_ptr_resets);
+}
+
+/* When shared IBT tables are used, update thread-private state
+ * to reflect the current parameter values -- hash mask, table address --
+ * for all tables.
+ */
+static bool
+update_all_private_ibt_table_ptrs(dcontext_t *dcontext, per_thread_t *pt)
+{
+    bool rc = false;
+
+    if (SHARED_IBT_TABLES_ENABLED()) {
+        ibl_branch_type_t branch_type;
+
+        for (branch_type = IBL_BRANCH_TYPE_START; 
+             branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+            if (DYNAMO_OPTION(shared_trace_ibt_tables)) {
+                if (update_private_ibt_table_ptrs(dcontext,
+                                                  &shared_pt->trace_ibt[branch_type]
+                                                  _IF_DEBUG(NULL)))
+                    rc = true;
+            }
+            if (DYNAMO_OPTION(shared_bb_ibt_tables)) {
+                if (update_private_ibt_table_ptrs(dcontext,
+                                                  &shared_pt->bb_ibt[branch_type]
+                                                  _IF_DEBUG(NULL)))
+                    rc = true;
+            }
+        }
+    }
+    return rc;
+}
+
+/* Prepares for removal of f from ftable (does not delete f) by pointing the
+ * fragment's lookup table entry to an entry point that leads to a cache exit.
+ * This routine is needed for safe removal of a fragment by a thread while
+ * another thread may be about to jump to it via an IBL. The lookuptable is
+ * left in a slightly inconsistent state but one that is accepted by the
+ * consistency check. See the note on hashtable_fragment_check_consistency
+ * in the routine.
+ *
+ * Returns true if the fragment was found & removed.
+ */
+static bool
+fragment_prepare_for_removal_from_table(dcontext_t *dcontext, fragment_t *f, 
+                                        ibl_table_t *ftable)
+{
+    uint hindex;
+    fragment_entry_t fe = FRAGENTRY_FROM_FRAGMENT(f);
+    fragment_entry_t *pg;
+
+    /* We need the write lock since the start_pc is modified (though we
+     * technically may be ok in all scenarios there) and to avoid problems
+     * with parallel prepares (shouldn't count on the bb building lock).
+     * Grab the lock after all private ptrs are updated since that
+     * operation might grab the same lock, if this remove is from a
+     * shared IBT table.
+     */
+    /* FIXME: why do we need to update here? */
+    update_private_ibt_table_ptrs(dcontext, ftable _IF_DEBUG(NULL));
+    TABLE_RWLOCK(ftable, write, lock);
+    pg = hashtable_ibl_lookup_for_removal(fe, ftable, &hindex);
+    if (pg != NULL) {
+        /* Note all IBL routines that could be looking up an entry
+         * in this table have to exit with equivalent register
+         * state.  It is possible to enter a private bb IBL
+         * lookup, shared bb IBL lookup or trace bb IBL lookup and
+         * if a delete race is hit then they would all go to the
+         * pending_delete_pc that we'll now supply. They HAVE to
+         * be all equivalent independent of the source fragment for this to work.
+         *
+         * On the other hand we can provide different start_pc
+         * values if we have different tables.  We currently don't
+         * take advantage of this but we'll leave the power in place.
+         */
+
+        /* FIXME: [perf] we could memoize this value in the table itself */
+        cache_pc pending_delete_pc =
+            get_target_delete_entry_pc(dcontext, ftable);
+
+        ASSERT(IBL_ENTRIES_ARE_EQUAL(*pg, fe));
+        ASSERT(pending_delete_pc != NULL);
+        LOG(THREAD, LOG_FRAGMENT, 3,
+            "fragment_prepare: remove F%d("PFX") from %s[%u] (table addr "PFX"), "
+            "set to "PFX"\n",
+            f->id, f->tag, ftable->name, hindex, ftable->table,
+            pending_delete_pc);
+
+        /* start_pc_fragment will not match start_pc for the table
+         * consistency checks. However, the hashtable_fragment_check_consistency
+         * routine verifies that either start_pc/start_pc_fragment match OR that
+         * the start_pc_fragment is set to the correct target_delete
+         * entry point.
+         *
+         * We change the tag to FAKE_TAG, which preserves linear probing.
+         * In a thread-shared table, this ensures that the same tag will never
+         * be present in more than one entry in a table (1 real entry &
+         * 1+ target_delete entries).
+         * This isn't needed in a thread-private table but doesn't hurt.
+         */
+        ftable->table[hindex].start_pc_fragment = pending_delete_pc;
+        ftable->table[hindex].tag_fragment = FAKE_TAG;
+        /* FIXME In a shared table, this means that the entry cannot
+         * be overwritten for a fragment with the same tag. */
+        ftable->unlinked_entries++;
+        ftable->entries--;
+        TABLE_RWLOCK(ftable, write, unlock);
+        ASSERT(!TEST(FRAG_CANNOT_DELETE, f->flags));
+        return true;
+    }
+    TABLE_RWLOCK(ftable, write, unlock);
+    return false;
+}
+
+/* Prepares fragment f for removal from all IBL routine targeted tables.
+ * Does not actually remove the entry from the table
+ * as that can only be done through proper cross-thread synchronization.
+ *
+ * Returns true if the fragment was found & removed.
+ */
+bool
+fragment_prepare_for_removal(dcontext_t *dcontext, fragment_t *f)
+{
+    per_thread_t *pt;
+    bool prepared = false;
+    ibl_branch_type_t branch_type;
+    if (!IS_IBL_TARGET(f->flags)) {
+        /* nothing to do */
+        return false;
+    }
+
+    ASSERT(TEST(FRAG_SHARED, f->flags) || dcontext != GLOBAL_DCONTEXT);
+    /* We need a real per_thread_t & context below so make sure we have one. */
+    if (dcontext == GLOBAL_DCONTEXT) {
+        dcontext = get_thread_private_dcontext();
+        ASSERT(dcontext != NULL);
+    }
+    pt = GET_PT(dcontext);
+    /* FIXME: as an optimization we could test if IS_IBL_TARGET() is
+     * set before looking it up
+     */
+
+    for (branch_type = IBL_BRANCH_TYPE_START; 
+         branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+        per_thread_t *local_pt = pt;
+        /* We put traces into the trace tables and BBs into the BB tables
+         * and sometimes put traces into BB tables also. We never put
+         * BBs into a trace table.
+         */
+        if (TEST(FRAG_IS_TRACE, f->flags)) {
+            if (DYNAMO_OPTION(shared_trace_ibt_tables))
+                local_pt = shared_pt;
+            if (fragment_prepare_for_removal_from_table(dcontext, f,
+                                                        &local_pt->
+                                                        trace_ibt[branch_type]))
+                prepared = true;
+        }
+        if (DYNAMO_OPTION(bb_ibl_targets) &&
+            (!TEST(FRAG_IS_TRACE, f->flags) ||
+             DYNAMO_OPTION(bb_ibt_table_includes_traces))) {
+            if (DYNAMO_OPTION(shared_bb_ibt_tables))
+                local_pt = shared_pt;
+            if (fragment_prepare_for_removal_from_table(dcontext, f,
+                                                        &local_pt->
+                                                        bb_ibt[branch_type])) {
+#ifdef DEBUG
+                ibl_table_t *ibl_table = GET_IBT_TABLE(pt, f->flags, branch_type);
+                fragment_entry_t current;
+
+                TABLE_RWLOCK(ibl_table, read, lock);
+                current = hashtable_ibl_lookup(dcontext, (ptr_uint_t)f->tag, ibl_table);
+                ASSERT(IBL_ENTRY_IS_EMPTY(current));
+                TABLE_RWLOCK(ibl_table, read, unlock);
+#endif
+                prepared = true;
+            }
+        }
+    }
+    return prepared;
+}
+
+/* FIXME: hashtable_fragment_reset() needs to walk the tables to get these
+ * stats, but then we'd need to subtract 1 from all smaller counts -
+ * e.g. if an entry is found in 3 tables we can add (1,-1,0) then
+ * we'll find it again and we should add (0,1,-1) and one more time
+ * when we should add (0,0,1).  In total all will be accounted for to
+ * (1,0,0) without messing much else.
+ */
+static inline void
+fragment_ibl_stat_account(uint flags, uint ibls_targeted)
+{
+    if (TEST(FRAG_IS_TRACE, flags)) {
+        switch (ibls_targeted) {
+        case 0: break; /* doesn't have to be a target of any IBL routine */
+        case 1: STATS_INC(num_traces_in_1_ibl_tables); break;
+        case 2: STATS_INC(num_traces_in_2_ibl_tables); break;
+        case 3: STATS_INC(num_traces_in_3_ibl_tables); break;
+        default: ASSERT_NOT_REACHED();
+        }
+    } else {
+        switch (ibls_targeted) {
+        case 0: break; /* doesn't have to be a target of any IBL routine */
+        case 1: STATS_INC(num_bbs_in_1_ibl_tables); break;
+        case 2: STATS_INC(num_bbs_in_2_ibl_tables); break;
+        case 3: STATS_INC(num_bbs_in_3_ibl_tables); break;
+        default: ASSERT_NOT_REACHED();
+        }
+    }                
+}
+
+/* Removes f from any IBT tables it is in.
+ * If f is in a shared table, only removes if from_shared is true, in
+ * which case dcontext must be GLOBAL_DCONTEXT and we must have
+ * dynamo_all_threads_synched (case 10137).
+ */
+void
+fragment_remove_from_ibt_tables(dcontext_t *dcontext, fragment_t *f,
+                                bool from_shared)
+{
+    bool shared_ibt_table =
+        (!TEST(FRAG_IS_TRACE, f->flags) && DYNAMO_OPTION(shared_bb_ibt_tables)) ||
+        (TEST(FRAG_IS_TRACE, f->flags) && DYNAMO_OPTION(shared_trace_ibt_tables));
+    fragment_entry_t fe = FRAGENTRY_FROM_FRAGMENT(f);
+
+    ASSERT(!from_shared || !shared_ibt_table || !IS_IBL_TARGET(f->flags) ||
+           (dcontext == GLOBAL_DCONTEXT && dynamo_all_threads_synched));
+    if (((!shared_ibt_table && dcontext != GLOBAL_DCONTEXT) ||
+         (from_shared && dcontext == GLOBAL_DCONTEXT && dynamo_all_threads_synched)) &&
+        IS_IBL_TARGET(f->flags)) {
+
+        /* trace_t tables should be all private and any deletions should follow strict
+         * two step deletion process, we don't need to be holding nested locks when
+         * removing any cached entries from the per-type IBL target tables.
+         */
+        /* FIXME: the stats on ibls_targeted are not quite correct - we need to
+         * gather these independently */
+        DEBUG_DECLARE(uint ibls_targeted = 0;)
+        ibl_branch_type_t branch_type;
+        per_thread_t *pt = GET_PT(dcontext);
+
+        ASSERT(TEST(FRAG_IS_TRACE, f->flags) || DYNAMO_OPTION(bb_ibl_targets));
+        for (branch_type = IBL_BRANCH_TYPE_START; 
+             branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+            /* assuming a single tag can't be both a trace and bb */
+            ibl_table_t *ibtable = GET_IBT_TABLE(pt, f->flags, branch_type);
+
+            ASSERT(!TEST(FRAG_TABLE_SHARED, ibtable->table_flags) ||
+                   dynamo_all_threads_synched);
+            TABLE_RWLOCK(ibtable, write, lock); /* satisfy asserts, even if allsynch */
+            if (hashtable_ibl_remove(fe, ibtable)) {
+                LOG(THREAD, LOG_FRAGMENT, 2,
+                    "  removed F%d("PFX") from IBT table %s\n",
+                    f->id, f->tag,
+                    TEST(FRAG_TABLE_TRACE, ibtable->table_flags) ?
+                    ibl_trace_table_type_names[branch_type] :
+                    ibl_bb_table_type_names[branch_type]);
+
+                DOSTATS({ibls_targeted++;});
+            }
+            TABLE_RWLOCK(ibtable, write, unlock);
+        }
+        DOSTATS({fragment_ibl_stat_account(f->flags, ibls_targeted);});
+    }
+}
+
+/* Removes ibl entries whose tags are in [start,end) */
+static uint
+fragment_remove_ibl_entries_in_region(dcontext_t *dcontext, app_pc start, app_pc end,
+                                      uint frag_flags)
+{
+    uint total_removed = 0;
+    per_thread_t *pt = GET_PT(dcontext);
+    ibl_branch_type_t branch_type;
+    ASSERT(pt != NULL);
+    ASSERT(TEST(FRAG_IS_TRACE, frag_flags) || DYNAMO_OPTION(bb_ibl_targets));
+    ASSERT(dcontext == get_thread_private_dcontext() || dynamo_all_threads_synched);
+    for (branch_type = IBL_BRANCH_TYPE_START; 
+         branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+        ibl_table_t *ibtable = GET_IBT_TABLE(pt, frag_flags, branch_type);
+        uint removed = 0;
+        TABLE_RWLOCK(ibtable, write, lock);
+        if (ibtable->entries > 0) {
+            removed = hashtable_ibl_range_remove(dcontext, ibtable,
+                                                 (ptr_uint_t)start, (ptr_uint_t)end);
+            /* Ensure a full remove gets everything */
+            ASSERT(start != UNIVERSAL_REGION_BASE || end != UNIVERSAL_REGION_END ||
+                   (ibtable->entries == 0 &&
+                    is_region_memset_to_char((app_pc)ibtable->table,
+                                             (ibtable->capacity-1)*sizeof(fragment_entry_t),
+                                             0)));
+        }
+        LOG(THREAD, LOG_FRAGMENT, 2,
+            "  removed %d entries (%d left) in "PFX"-"PFX" from IBT table %s\n",
+            removed, ibtable->entries, start, end, 
+            TEST(FRAG_TABLE_TRACE, ibtable->table_flags) ?
+            ibl_trace_table_type_names[branch_type] :
+            ibl_bb_table_type_names[branch_type]);
+        TABLE_RWLOCK(ibtable, write, unlock);
+        total_removed += removed;
+    }
+    return total_removed;
+}
+
+/* Removes shared (and incidentally private, but if no shared targets
+ * can exist, may remove nothing) ibl entries whose tags are in
+ * [start,end) from all tables associated w/ dcontext.  If
+ * dcontext==GLOBAL_DCONTEXT, uses the shared tables, if they exist;
+ * else, uses the private tables, if any.
+ */
+uint
+fragment_remove_all_ibl_in_region(dcontext_t *dcontext, app_pc start, app_pc end)
+{
+    uint removed = 0;
+    if (DYNAMO_OPTION(bb_ibl_targets) &&
+        ((dcontext == GLOBAL_DCONTEXT && DYNAMO_OPTION(shared_bb_ibt_tables)) ||
+         (dcontext != GLOBAL_DCONTEXT && !DYNAMO_OPTION(shared_bb_ibt_tables)))) {
+        removed +=
+            fragment_remove_ibl_entries_in_region(dcontext, start, end, 0/*bb table*/);
+    }
+    if (DYNAMO_OPTION(shared_traces) &&
+        ((dcontext == GLOBAL_DCONTEXT && DYNAMO_OPTION(shared_trace_ibt_tables)) ||
+         (dcontext != GLOBAL_DCONTEXT && !DYNAMO_OPTION(shared_trace_ibt_tables)))) {
+        removed +=
+            fragment_remove_ibl_entries_in_region(dcontext, start, end, FRAG_IS_TRACE);
+    }
+    return removed;
+}
+
+/* Removes f from any hashtables -- BB, trace, or future -- and IBT tables
+ * it is in, except for shared IBT tables. */
+void
+fragment_remove(dcontext_t *dcontext, fragment_t *f)
+{
+    per_thread_t *pt = GET_PT(dcontext);
+    fragment_table_t *table = GET_FTABLE(pt, f->flags);
+
+    ASSERT(TEST(FRAG_SHARED, f->flags) || dcontext != GLOBAL_DCONTEXT);
+    /* For consistency we remove entries from the IBT
+     * tables before we remove them from the trace table.
+     */
+    fragment_remove_from_ibt_tables(dcontext, f, false/*leave in shared*/);
+
+    /* We need the write lock since deleting shifts elements around (though we
+     * technically may be ok in all scenarios there) and to avoid problems with
+     * multiple removes at once (shouldn't count on the bb building lock)
+     */
+    TABLE_RWLOCK(table, write, lock);
+    if (hashtable_fragment_remove(f, table)) {
+        LOG(THREAD, LOG_FRAGMENT, 4,
+            "fragment_remove: removed F%d("PFX") from fcache lookup table\n",
+            f->id, f->tag);
+        TABLE_RWLOCK(table, write, unlock);
+        return;
+    }
+    TABLE_RWLOCK(table, write, unlock);
+
+    /* ok to not find a trace head used to start a trace -- fine to have deleted
+     * the trace head
+     */
+    ASSERT(cur_trace_tag(dcontext) == f->tag
+           /* PR 299808: we have invisible temp trace bbs */
+           IF_CLIENT_INTERFACE(|| TEST(FRAG_TEMP_PRIVATE, f->flags)));
+}
+
+/* Remove f from ftable, replacing it in the hashtable with new_f, 
+ * which has an identical tag.
+ * f's next field is left intact so this can be done while owner is in fcache
+ * f is NOT deleted in any other way!
+ * To delete later, caller must call fragment_delete w/ remove=false
+ */
+void
+fragment_replace(dcontext_t *dcontext, fragment_t *f, fragment_t *new_f)
+{
+    per_thread_t *pt = GET_PT(dcontext);
+    fragment_table_t *table = GET_FTABLE(pt, f->flags);
+    TABLE_RWLOCK(table, write, lock);
+    if (hashtable_fragment_replace(f, new_f, table)) {
+        fragment_entry_t fe = FRAGENTRY_FROM_FRAGMENT(f);
+        fragment_entry_t new_fe = FRAGENTRY_FROM_FRAGMENT(new_f);
+        LOG(THREAD, LOG_FRAGMENT, 4,
+            "removed F%d from fcache lookup table (replaced with F%d) "PFX"->~"PFX","PFX"\n",
+            f->id, new_f->id, f->tag, f->start_pc, new_f->start_pc);
+        /* Need to replace all entries from the IBL tables that may have this entry */
+        if (IS_IBL_TARGET(f->flags)) {
+            ibl_branch_type_t branch_type;
+            for (branch_type = IBL_BRANCH_TYPE_START;
+                 branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+                ibl_table_t *ibtable = GET_IBT_TABLE(pt, f->flags, branch_type);
+                /* currently we don't have shared ib target tables,
+                 * otherwise a write lock would come in the picture here
+                 */
+                ASSERT(!TEST(FRAG_TABLE_SHARED, ibtable->table_flags));
+                hashtable_ibl_replace(fe, new_fe, ibtable);
+            }
+        }
+    } else 
+        ASSERT_NOT_REACHED();
+    TABLE_RWLOCK(table, write, unlock);
+    
+    /* tell monitor f has disappeared, but do not delete from incoming table
+     * or from fcache, also do not dump to trace file
+     */
+    monitor_remove_fragment(dcontext, f);
+}
+
+void
+fragment_shift_fcache_pointers(dcontext_t *dcontext, fragment_t *f, ssize_t shift,
+                               cache_pc start, cache_pc end, size_t old_size)
+{
+    per_thread_t *pt = GET_PT(dcontext);
+
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false)); /* must re-relativize when copying! */
+
+    /* need to shift all stored cache_pcs.
+     * do not need to shift relative pcs pointing to other fragments -- they're
+     * all getting shifted too!
+     * just need to re-pc-relativize jmps to fixed locations, namely
+     * cti's in exit stubs, and call instructions inside fragments.
+     */
+
+    LOG(THREAD, LOG_FRAGMENT, 2, "fragment_shift_fcache_pointers: F%d + "SSZFMT"\n",
+        f->id, shift);
+
+    ASSERT(!TEST(FRAG_IS_FUTURE, f->flags)); /* only in-cache frags */
+#ifdef NATIVE_RETURN
+    /* should never get here, we disable resizing of fcache */
+    ASSERT_NOT_REACHED();
+#endif
+
+    f->start_pc += shift;
+
+    /* Should shift cached lookup entries in all IBL target tables, 
+     * order doesn't matter here: either way we'll be inconsistent, can't do this within the cache.
+     */
+    if (IS_IBL_TARGET(f->flags)) {
+        ibl_branch_type_t branch_type;
+        for (branch_type = IBL_BRANCH_TYPE_START; 
+             branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+            /* Of course, we need to shift only pointers into the cache that is getting shifted! */
+            ibl_table_t *ibtable = GET_IBT_TABLE(pt, f->flags, branch_type);
+            fragment_entry_t fe = FRAGENTRY_FROM_FRAGMENT(f);
+            fragment_entry_t *pg;
+            uint hindex;
+            TABLE_RWLOCK(ibtable, read, lock);
+            pg = hashtable_ibl_lookup_for_removal(fe, ibtable, &hindex);
+            if (pg != NULL)
+                pg->start_pc_fragment += shift;
+            TABLE_RWLOCK(ibtable, read, unlock);
+            LOG(THREAD, LOG_FRAGMENT, 2,
+                "fragment_table_shift_fcache_pointers: %s ibt %s shifted by %d\n",
+                TEST(FRAG_IS_TRACE, f->flags) ? "trace" : "BB", ibtable->name, shift);
+        }
+    }
+
+    linkstubs_shift(dcontext, f, shift);
+
+    DOLOG(6, LOG_FRAGMENT, { /* print after start_pc updated so get actual code */
+        LOG(THREAD, LOG_FRAGMENT, 6, "before shifting F%d ("PFX")\n", f->id, f->tag);
+        disassemble_fragment(dcontext, f, stats->loglevel < 3);
+    });
+
+    if (TEST(FRAG_SELFMOD_SANDBOXED, f->flags)) {
+        /* just re-finalize to update */
+        finalize_selfmod_sandbox(dcontext, f);
+    }
+
+    /* inter-cache links must be redone, but all fragment entry pcs must be
+     * fixed up first, so that's done separately
+     */
+
+    /* re-do pc-relative targets outside of cache */
+    shift_ctis_in_fragment(dcontext, f, shift, start, end, old_size);
+#ifdef CHECK_RETURNS_SSE2
+    finalize_return_check(dcontext, f);
+#endif
+
+    DOLOG(6, LOG_FRAGMENT, {
+        LOG(THREAD, LOG_FRAGMENT, 6, "after shifting F%d ("PFX")\n", f->id, f->tag);
+        disassemble_fragment(dcontext, f, stats->loglevel < 3);
+    });
+}
+
+/* this routine only copies data structures like bbs and statistics
+ */
+void 
+fragment_copy_data_fields(dcontext_t *dcontext, fragment_t *f_src, fragment_t *f_dst)
+{
+    if ((f_src->flags & FRAG_IS_TRACE) != 0) {
+        trace_only_t *t_src = TRACE_FIELDS(f_src);
+        trace_only_t *t_dst = TRACE_FIELDS(f_dst);
+        ASSERT((f_dst->flags & FRAG_IS_TRACE) != 0);
+        if (t_src->bbs != NULL) {
+            t_dst->bbs =
+                nonpersistent_heap_alloc(dcontext, t_src->num_bbs*sizeof(trace_bb_info_t)
+                                         HEAPACCT(ACCT_TRACE));
+            memcpy(t_dst->bbs, t_src->bbs, t_src->num_bbs*sizeof(trace_bb_info_t));
+            t_dst->num_bbs = t_src->num_bbs;
+        }
+
+#ifdef PROFILE_RDTSC
+        t_dst->count = t_src->count;
+        t_dst->total_time = t_src->total_time;
+#endif
+
+#if defined (PROFILE_LINKCOUNT) && defined(SIDELINE_COUNT_STUDY)
+        t_dst->count_old_pre = t_src->count_old_pre;
+        t_dst->count_old_post = t_src->count_old_post;
+#endif
+    }
+}
+
+#if defined(DEBUG) && defined(INTERNAL)
+static void
+dump_lookup_table(dcontext_t *dcontext, ibl_table_t *ftable)
+{
+    uint i;
+    cache_pc target_delete = get_target_delete_entry_pc(dcontext, ftable);
+
+    ASSERT(target_delete != NULL);
+    ASSERT(ftable->table != NULL);
+    LOG(THREAD, LOG_FRAGMENT, 1,
+        "%6s %10s %10s -- %s\n", "i", "tag", "target", ftable->name);
+    /* need read lock to traverse the table */
+    TABLE_RWLOCK(ftable, read, lock);
+    for (i = 0; i < ftable->capacity; i++) {
+        if (ftable->table[i].tag_fragment != 0) {
+            if (ftable->table[i].start_pc_fragment == target_delete) {
+                LOG(THREAD, LOG_FRAGMENT, 1,
+                    "%6x "PFX" target_delete\n",
+                    i, ftable->table[i].tag_fragment);
+                ASSERT(ftable->table[i].tag_fragment == FAKE_TAG);
+            }
+            else {
+                LOG(THREAD, LOG_FRAGMENT, 1,
+                    "%6x "PFX" "PFX"\n",
+                    i, ftable->table[i].tag_fragment,
+                    ftable->table[i].start_pc_fragment);
+            }
+        }
+        DODEBUG({ hashtable_ibl_check_consistency(dcontext, ftable, i); });
+    }
+    TABLE_RWLOCK(ftable, read, unlock);
+}
+#endif
+
+#ifdef DEBUG
+/* used only for debugging purposes, check if IBL routine leaks due to wraparound  */
+/* interesting only when not using INTERNAL_OPTION(ibl_sentinel_check) */
+static bool
+is_fragment_index_wraparound(dcontext_t *dcontext, ibl_table_t *ftable, fragment_t *f)
+{
+    uint hindex = HASH_FUNC((ptr_uint_t)f->tag, ftable);
+    uint found_at_hindex;
+    fragment_entry_t fe = FRAGENTRY_FROM_FRAGMENT(f);
+    fragment_entry_t *pg = hashtable_ibl_lookup_for_removal(fe, ftable, &found_at_hindex);
+    ASSERT(pg != NULL);
+    ASSERT(IBL_ENTRIES_ARE_EQUAL(*pg, fe));
+    LOG(THREAD, LOG_FRAGMENT, 3,
+        "is_fragment_index_wraparound F%d, tag "PFX", found_at_hindex 0x%x, preferred 0x%x\n",
+        f->id, f->tag, found_at_hindex, hindex);
+    return (found_at_hindex < hindex); /* wraparound */
+}
+#endif /* DEBUG */
+
+static void
+fragment_add_ibl_target_helper(dcontext_t *dcontext, fragment_t *f,
+                               ibl_table_t *ibl_table)
+{
+    fragment_entry_t current;
+    fragment_entry_t fe = FRAGENTRY_FROM_FRAGMENT(f);
+
+    /* Never add a BB to a trace table. */
+    ASSERT(!(!TEST(FRAG_IS_TRACE, f->flags) &&
+             TEST(FRAG_TABLE_TRACE, ibl_table->table_flags)));
+
+    /* adding is a write operation */
+    TABLE_RWLOCK(ibl_table, write, lock);
+    /* This is the last time the table lock is grabbed before adding the frag so
+     * check here to account for the race in the time between the
+     * FRAG_IS_TRACE_HEAD check in add_ibl_target() and now. We never add trace
+     * heads to an IBT target table.
+     */
+    if (TEST(FRAG_IS_TRACE_HEAD, f->flags)) {
+        TABLE_RWLOCK(ibl_table, write, unlock);
+        STATS_INC(num_th_bb_ibt_add_race);
+        return;
+    }
+    /* For shared tables, check again in case another thread snuck in
+     * before the preceding lock and added the target. */
+    if (TEST(FRAG_TABLE_SHARED, ibl_table->table_flags)) {
+        current = hashtable_ibl_lookup(dcontext, (ptr_uint_t)f->tag, ibl_table);
+        if (IBL_ENTRY_IS_EMPTY(current))
+            hashtable_ibl_add(dcontext, fe, ibl_table);
+        /* We don't ever expect to find a like-tagged fragment. A BB
+         * can be unlinked due to eviction or when it's marked as a trace
+         * head. Eviction (for example, due to cache consistency)
+         * sets start_pc_fragment to FAKE_TAG, so there can't be
+         * a tag match; &unlinked_fragment is returned, and this
+         * applies to traces also. For trace head marking, FAKE_TAG
+         * is also set so &unlinked_fragment is returned.
+         *
+         * If we didn't set FAKE_TAG for an unlinked entry, then it
+         * could be clobbered with a new fragment w/the same tag.
+         * In a shared table, unlinked entries cannot be clobbered
+         * except by fragments w/the same tags, so this could help
+         * limit the length of collision chains.
+         */
+    }
+    else {
+        hashtable_ibl_add(dcontext, fe, ibl_table);
+    }
+    TABLE_RWLOCK(ibl_table, write, unlock);
+    DOSTATS({
+        if (!TEST(FRAG_IS_TRACE, f->flags))
+            STATS_INC(num_bbs_ibl_targets);
+        /* We assume that traces can be added to trace and BB IBT tables but
+         * not just to BB tables. We count only traces added to trace tables
+         * so that we don't double increment.
+         */
+        else if (TEST(FRAG_IS_TRACE, f->flags) &&
+                 TEST(FRAG_TABLE_TRACE, ibl_table->table_flags))
+            STATS_INC(num_traces_ibl_targets);
+    });
+
+    /* Adding current exit to help calculate an estimated
+     * indirect branch fan-out, that is function fan-in for
+     * returns.  Note that other IBL hits to the same place
+     * will not have exits associated.
+     */
+    LOG(THREAD, LOG_FRAGMENT, 2,
+        "fragment_add_ibl_target added F%d("PFX"), branch %d, to %s, on exit from "PFX"\n",
+        f->id, f->tag, 
+        ibl_table->branch_type, ibl_table->name,
+        LINKSTUB_FAKE(dcontext->last_exit) ? 0 :
+        EXIT_CTI_PC(dcontext->last_fragment, dcontext->last_exit)
+        );
+    DOLOG(4, LOG_FRAGMENT, {
+        dump_lookuptable_tls(dcontext);
+        hashtable_ibl_dump_table(dcontext, ibl_table);
+        dump_lookup_table(dcontext, ibl_table);
+    });
+    DODEBUG({
+        if (TEST(FRAG_SHARED, f->flags) && !TEST(FRAG_IS_TRACE, f->flags))
+            LOG(THREAD, LOG_FRAGMENT, 2,
+                "add_ibl_target: shared BB F%d("PFX") added\n", f->id,
+                f->tag);
+    });
+}
+
+/* IBL targeted fragments per branch type */
+fragment_t *
+fragment_add_ibl_target(dcontext_t *dcontext, app_pc tag,
+                        ibl_branch_type_t branch_type)
+{
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+    fragment_t *f = NULL;
+
+    if (SHARED_BB_ONLY_IB_TARGETS()) {
+        fragment_t wrapper;
+        f = fragment_lookup_bb(dcontext, tag);
+        if (f == NULL) {
+            f = fragment_coarse_lookup_wrapper(dcontext, tag, &wrapper);
+            if (f != NULL) {
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+                if (TEST(COARSE_FILL_IBL_MASK(branch_type),
+                         DYNAMO_OPTION(coarse_fill_ibl))) {
+                    /* On-demand per-type ibl filling from the persisted RAC/RCT
+                     * table.  We limit to the first thread to ask for it by
+                     * clearing the coarse_info_t pending_table fields.
+                     */
+                    /* FIXME: combine w/ the coarse lookup to do this once only */
+                    coarse_info_t *coarse = get_fragment_coarse_info(f);
+                    ASSERT(coarse != NULL);
+                    if (coarse->persisted &&
+                        exists_coarse_ibl_pending_table(dcontext, coarse, branch_type)) {
+                        bool in_persisted_ibl = false;
+                        mutex_lock(&coarse->lock);
+                        if (exists_coarse_ibl_pending_table(dcontext,
+                                                            coarse, branch_type)) {
+                            ibl_table_t *ibl_table =
+                                GET_IBT_TABLE(pt, f->flags, branch_type);
+                            coarse_persisted_fill_ibl(dcontext, coarse, branch_type);
+                            TABLE_RWLOCK(ibl_table, read, lock);
+                            if (!IBL_ENTRY_IS_EMPTY(hashtable_ibl_lookup(dcontext,
+                                                                  (ptr_uint_t)tag, ibl_table)))
+                                in_persisted_ibl = true;
+                            TABLE_RWLOCK(ibl_table, read, unlock);
+                            if (in_persisted_ibl) {
+                                mutex_unlock(&coarse->lock);
+                                return f;
+                            }
+                        }
+                        mutex_unlock(&coarse->lock);
+                    }
+                }
+#endif /* defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH) */
+            }
+        }
+   } else {
+        f = fragment_lookup_trace(dcontext, tag);
+        if (f == NULL && DYNAMO_OPTION(bb_ibl_targets)) {
+            /* Populate with bb's that are not trace heads */
+            f = fragment_lookup_bb(dcontext, tag);
+            /* We don't add trace heads OR when a trace is targetting a BB. In the
+             * latter case, the BB will shortly be marked as a trace head and
+             * removed from the IBT table so we don't needlessly add it.
+             */
+            if (f != NULL &&
+                (TEST(FRAG_IS_TRACE_HEAD, f->flags) ||
+                 TEST(FRAG_IS_TRACE, dcontext->last_fragment->flags))) {
+                /* FIXME: should change the logic if trace headness becomes a private property */
+                f = NULL; /* ignore fragment */
+                STATS_INC(num_ib_th_target); /* counted in num_ibt_cold_misses */
+            }
+        }
+    }
+
+    LOG(THREAD, LOG_FRAGMENT, 3,
+        "fragment_add_ibl_target tag "PFX", branch %d, F%d %s\n",
+        tag, branch_type, f != NULL ? f->id : 0, 
+        (f != NULL && TEST(FRAG_IS_TRACE, f->flags)) ? "existing trace" : "");
+
+    /* a valid IBT fragment exists */
+    if (f != NULL) {
+        ibl_table_t *ibl_table = GET_IBT_TABLE(pt, f->flags, branch_type);
+        DEBUG_DECLARE(fragment_entry_t *orig_lookuptable = NULL;)
+        fragment_entry_t current;
+
+        /* Make sure this thread's local ptrs & state is current in case a
+         * shared table resize occured while it was in the cache. We update
+         * only during an IBL miss, since that's the first time that
+         * accessing the old table inflicted a cost (a context switch).
+         */
+        /* NOTE We could be more aggressive and update the private ptrs for
+         * all tables, not just the one being added to, by calling
+         * update_all_private_ibt_table_ptrs(). We could be even more
+         * aggressive by updating all ptrs on every cache exit in
+         * enter_couldbelinking() but that could also prove to be more
+         * expensive by invoking the update logic when an IBL miss didn't
+         * occur. However, more frequent updates could lead to old tables
+         * being freed earlier. We can revisit this if we see old tables
+         * piling up and not being freed in a timely manner.
+         */
+        update_private_ibt_table_ptrs(dcontext, ibl_table
+                                      _IF_DEBUG(&orig_lookuptable));
+
+        /* We can't place a private fragment into a thread-shared table.
+         * Nothing prevents a sandboxed or ignore syscalls frag from being
+         * the target of an IB. This is covered by case 5836.
+         *
+         * We don't need to re-check in the add_ibl_target_helper because
+         * the shared/private property applies to all IBT tables -- either
+         * all are shared or none are.
+         */
+        if (TEST(FRAG_TABLE_SHARED, ibl_table->table_flags) &&
+            !TEST(FRAG_SHARED, f->flags)) {
+            STATS_INC(num_ibt_shared_private_conflict);
+            return f;
+        }
+
+        ASSERT(TEST(FRAG_IS_TRACE, f->flags) ==
+               TEST(FRAG_TABLE_TRACE, ibl_table->table_flags));
+        /* We can't assert that an IBL target isn't a trace head due to a race
+         * between trace head marking and adding to a table. See the comments
+         * in fragment_add_to_hashtable().
+         */
+        TABLE_RWLOCK(ibl_table, read, lock);
+        current = hashtable_ibl_lookup(dcontext, (ptr_uint_t)tag, ibl_table);
+        TABLE_RWLOCK(ibl_table, read, unlock);
+        /* Now that we set the fragment_t* for any unlinked entry to
+         * &unlinked_fragment -- regardless of why it was unlinked --  and also
+         * set the lookup table tag to FAKE_TAG, we should never find a fragment
+         * with the same tag and should never have an unlinked marker returned
+         * here.
+         */
+        ASSERT(!IBL_ENTRY_IS_INVALID(current));
+        if (IBL_ENTRY_IS_EMPTY(current)) {
+            DOLOG(4, LOG_FRAGMENT, {
+                dump_lookuptable_tls(dcontext);
+                hashtable_ibl_dump_table(dcontext, ibl_table);
+                dump_lookup_table(dcontext, ibl_table);
+            });
+            fragment_add_ibl_target_helper(dcontext, f, ibl_table);
+            /* When using BB2BB IBL w/trace building, we add trace targets
+             * to the BB table. (We always add a trace target to the trace
+             * table.) We fool the helper routine into using the BB
+             * table by passing in a non-trace value for the flags argument.
+             */
+            if (TEST(FRAG_IS_TRACE, f->flags) && DYNAMO_OPTION(bb_ibl_targets) &&
+                DYNAMO_OPTION(bb_ibt_table_includes_traces)) {
+                ibl_table_t *ibl_table_too =
+                    GET_IBT_TABLE(pt, f->flags & ~FRAG_IS_TRACE, branch_type);
+
+                ASSERT(ibl_table_too != NULL);
+                ASSERT(!TEST(FRAG_TABLE_TRACE, ibl_table_too->table_flags));
+                /* Make sure this thread's local ptrs & state is up to
+                 * date in case a resize occured while it was in the cache. */
+                update_private_ibt_table_ptrs(dcontext, ibl_table_too
+                                              _IF_DEBUG(NULL));
+                fragment_add_ibl_target_helper(dcontext, f, ibl_table_too);
+            }
+        }
+        else {
+            DEBUG_DECLARE(const char *reason;)
+#ifdef DEBUG
+            if (is_building_trace(dcontext)) {
+                reason = "trace building";
+                STATS_INC(num_ibt_exit_trace_building);
+            } else if (TEST(FRAG_WAS_DELETED, dcontext->last_fragment->flags)) {
+                reason = "src unlinked (frag deleted)";
+                STATS_INC(num_ibt_exit_src_unlinked_frag_deleted);
+            } else if (!TEST(LINK_LINKED, dcontext->last_exit->flags) &&
+                       TESTALL(FRAG_SHARED | FRAG_IS_TRACE_HEAD,
+                           dcontext->last_fragment->flags) &&
+                       fragment_lookup_type(dcontext,
+                           dcontext->last_fragment->tag,
+                           LOOKUP_TRACE|LOOKUP_SHARED) != NULL) {
+                /* Another thread unlinked src as part of replacing it with
+                 * a new trace while this thread was in there (see case 5634
+                 * for details) */
+                reason = "src unlinked (shadowed)";
+                STATS_INC(num_ibt_exit_src_unlinked_shadowed);
+            } else if (!INTERNAL_OPTION(ibl_sentinel_check) &&
+                       is_fragment_index_wraparound(dcontext, ibl_table, f)) {
+                reason = "sentinel";
+                STATS_INC(num_ibt_leaks_likely_sentinel);
+            } else if (TEST(FRAG_SELFMOD_SANDBOXED, dcontext->last_fragment->flags)) {
+                reason = "src sandboxed";
+                STATS_INC(num_ibt_exit_src_sandboxed);
+            } else if (TEST(FRAG_TABLE_SHARED, ibl_table->table_flags) &&
+                       orig_lookuptable != ibl_table->table) {
+                /* A table resize could cause a miss when the target is
+                 * in the new table. */
+                reason = "shared IBT table resize";
+                STATS_INC(num_ibt_exit_shared_table_resize);
+            } else if (DYNAMO_OPTION(bb_ibl_targets) &&
+                       IS_SHARED_SYSCALLS_LINKSTUB(dcontext->last_exit) &&
+                       !DYNAMO_OPTION(disable_traces) &&
+                       !TEST(FRAG_IS_TRACE, f->flags)) {
+                reason = "shared syscall exit cannot target BBs";
+                STATS_INC(num_ibt_exit_src_trace_shared_syscall);
+            } else if (DYNAMO_OPTION(bb_ibl_targets) &&
+                       TEST(FRAG_IS_TRACE, f->flags) &&
+                       !DYNAMO_OPTION(bb_ibt_table_includes_traces)) {
+                reason = "BBs do not target traces";
+                STATS_INC(num_ibt_exit_src_trace_shared_syscall);
+            } else if (!INTERNAL_OPTION(link_ibl)) {
+                reason = "-no_link_ibl prevents ibl";
+                STATS_INC(num_ibt_exit_nolink);
+            } else {
+                reason = "BAD leak?";
+                DOLOG(3, LOG_FRAGMENT, {
+                    hashtable_ibl_dump_table(dcontext, ibl_table);
+                    hashtable_ibl_study(dcontext, ibl_table, 0/*table consistent*/);
+                });
+                STATS_INC(num_ibt_exit_unknown);
+                ASSERT_CURIOSITY(false && "fragment_add_ibl_target unknown reason");
+            }
+            /* nothing to do, just sanity checking */
+            LOG(THREAD, LOG_FRAGMENT, 2,
+                "fragment_add_ibl_target tag "PFX", F%d already added - %s\n",
+                tag, f->id, reason);
+#endif
+        }
+    } else {
+        STATS_INC(num_ibt_cold_misses);
+    }
+#ifdef HASHTABLE_STATISTICS
+    if (INTERNAL_OPTION(stay_on_trace_stats)) {
+        /* best effort: adjust for 32bit counter overflow occasionally
+         * we'll get a hashtable leak only when not INTERNAL_OPTION(ibl_sentinel_check)
+         */
+        check_stay_on_trace_stats_overflow(dcontext, branch_type);
+    }
+#endif /* HASHTABLE_STATISTICS */
+    DOLOG(4, LOG_FRAGMENT, {
+        dump_lookuptable_tls(dcontext);
+    });
+    return f;
+}
+
+/**********************************************************************/
+/* FUTURE FRAGMENTS */
+
+/* create a new fragment with empty prefix and return it
+ */
+static future_fragment_t *
+fragment_create_future(dcontext_t *dcontext, app_pc tag, uint flags)
+{
+    dcontext_t *alloc_dc = FRAGMENT_ALLOC_DC(dcontext, flags);
+    future_fragment_t *fut = (future_fragment_t*)
+        nonpersistent_heap_alloc(alloc_dc, sizeof(future_fragment_t)
+                                 HEAPACCT(ACCT_FRAG_FUTURE));
+    ASSERT(!NEED_SHARED_LOCK(flags) || self_owns_recursive_lock(&change_linking_lock));
+    LOG(THREAD, LOG_FRAGMENT, 4, "Created future fragment "PFX" w/ flags 0x%08x\n",
+        tag, flags|FRAG_FAKE|FRAG_IS_FUTURE);
+    STATS_INC(num_future_fragments);
+    DOSTATS({
+        if (TEST(FRAG_SHARED, flags))
+            STATS_INC(num_shared_future_fragments);
+    });
+    fut->tag = tag;
+    fut->flags = flags | FRAG_FAKE | FRAG_IS_FUTURE;
+    fut->incoming_stubs = NULL;
+    return fut;
+}
+
+static void
+fragment_free_future(dcontext_t *dcontext, future_fragment_t *fut)
+{
+    dcontext_t *alloc_dc = FRAGMENT_ALLOC_DC(dcontext, fut->flags);
+    LOG(THREAD, LOG_FRAGMENT, 4, "Freeing future fragment "PFX"\n", fut->tag);
+    ASSERT(fut->incoming_stubs == NULL);
+    nonpersistent_heap_free(alloc_dc, fut, sizeof(future_fragment_t)
+                            HEAPACCT(ACCT_FRAG_FUTURE));
+}
+
+future_fragment_t *
+fragment_create_and_add_future(dcontext_t *dcontext, app_pc tag, uint flags)
+{
+    per_thread_t *pt = GET_PT(dcontext);
+    future_fragment_t *fut = fragment_create_future(dcontext, tag, flags);
+    fragment_table_t *futtable = GET_FTABLE(pt, fut->flags);
+    ASSERT(!NEED_SHARED_LOCK(flags) || self_owns_recursive_lock(&change_linking_lock));
+    /* adding to the table is a write operation */
+    TABLE_RWLOCK(futtable, write, lock);
+    fragment_add_to_hashtable(dcontext, (fragment_t *)fut, futtable);
+    TABLE_RWLOCK(futtable, write, unlock);
+    return fut;
+}
+
+void
+fragment_delete_future(dcontext_t *dcontext, future_fragment_t *fut)
+{
+    per_thread_t *pt = GET_PT(dcontext);
+    fragment_table_t *futtable = GET_FTABLE(pt, fut->flags);
+    ASSERT(!NEED_SHARED_LOCK(fut->flags) ||
+           self_owns_recursive_lock(&change_linking_lock));
+    /* removing from the table is a write operation */
+    TABLE_RWLOCK(futtable, write, lock);
+    hashtable_fragment_remove((fragment_t *)fut, futtable);
+    TABLE_RWLOCK(futtable, write, unlock);
+    fragment_free_future(dcontext, fut);
+}
+
+static uint
+fragment_delete_futures_in_region(dcontext_t *dcontext, app_pc start, app_pc end)
+{
+    per_thread_t *pt = GET_PT(dcontext);
+    uint flags = FRAG_IS_FUTURE | (dcontext == GLOBAL_DCONTEXT ? FRAG_SHARED : 0);
+    fragment_table_t *futtable = GET_FTABLE(pt, flags);
+    uint removed;
+    /* Higher-level lock needed since we do lookup+add w/o holding table lock between */
+    ASSERT(!NEED_SHARED_LOCK(flags) || self_owns_recursive_lock(&change_linking_lock));
+    TABLE_RWLOCK(futtable, write, lock);
+    removed = hashtable_fragment_range_remove(dcontext, futtable,
+                                              (ptr_uint_t)start, (ptr_uint_t)end);
+    TABLE_RWLOCK(futtable, write, unlock);
+    return removed;
+}
+
+future_fragment_t *
+fragment_lookup_future(dcontext_t *dcontext, app_pc tag)
+{
+    /* default is to lookup shared, since private only sometimes exists,
+     * and often only care about trace head, for which always use shared
+     */
+    uint flags = SHARED_FRAGMENTS_ENABLED() ? FRAG_SHARED : 0;
+    per_thread_t *pt = GET_PT(dcontext);
+    fragment_table_t *futtable = GET_FTABLE(pt, FRAG_IS_FUTURE | flags);
+    fragment_t *f;
+    TABLE_RWLOCK(futtable, read, lock);
+    f = hashtable_fragment_lookup(dcontext, (ptr_uint_t)tag, futtable);
+    TABLE_RWLOCK(futtable, read, unlock);
+    if (f != &null_fragment)
+        return (future_fragment_t *) f;
+    return NULL;
+}
+
+future_fragment_t *
+fragment_lookup_private_future(dcontext_t *dcontext, app_pc tag)
+{
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+    fragment_table_t *futtable = GET_FTABLE(pt, FRAG_IS_FUTURE);
+    fragment_t *f = hashtable_fragment_lookup(dcontext, (ptr_uint_t)tag, futtable);
+    if (f != &null_fragment)
+        return (future_fragment_t *) f;
+    return NULL;
+}
+
+/* END FUTURE FRAGMENTS
+ **********************************************************************/
+
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+/* FIXME: move to rct.c when we move the whole app_pc table there */
+
+#define STATS_RCT_ADD(which, stat, val) DOSTATS({ \
+    if ((which) == RCT_RAC)                       \
+        STATS_ADD(rac_##stat, val);              \
+    else                                          \
+        STATS_ADD(rct_##stat, val);              \
+})
+
+static inline bool
+rct_is_global_table(rct_module_table_t *permod)
+{
+    return (permod == &rac_non_module_table ||
+            IF_LINUX_ELSE(permod == &rct_global_table, false));
+}
+
+static inline rct_module_table_t *
+rct_get_table(app_pc tag, rct_type_t which)
+{
+    rct_module_table_t *permod = os_module_get_rct_htable(tag, which);
+    if (permod == NULL) { /* not a module */
+        if (which == RCT_RAC)
+            permod = &rac_non_module_table;
+    }
+    return permod;
+}
+
+/* returns NULL if not found */
+static app_pc
+rct_table_lookup_internal(dcontext_t *dcontext, app_pc tag,
+                          rct_module_table_t *permod)
+{
+    app_pc actag = NULL;
+    ASSERT(os_get_module_info_locked());
+    if (permod != NULL) {
+        /* Check persisted table first as it's likely to be larger and
+         * it needs no read lock
+         */
+        if (permod->persisted_table != NULL) {
+            actag = hashtable_app_pc_rlookup(dcontext, (ptr_uint_t)tag,
+                                             permod->persisted_table);
+        }
+        if (actag == NULL && permod->live_table != NULL) {
+            actag = hashtable_app_pc_rlookup(dcontext, (ptr_uint_t)tag,
+                                             permod->live_table);
+        }
+    }
+    return actag;
+}
+
+/* returns NULL if not found */
+static app_pc
+rct_table_lookup(dcontext_t *dcontext, app_pc tag, rct_type_t which)
+{
+    app_pc actag = NULL;
+    rct_module_table_t *permod;
+    ASSERT(which >= 0 && which < RCT_NUM_TYPES);
+    os_get_module_info_lock();
+    permod = rct_get_table(tag, which);
+    actag = rct_table_lookup_internal(dcontext, tag, permod);
+    os_get_module_info_unlock();
+    return actag;
+}
+
+/* Caller must hold the higher-level lock.
+ * Returns whether added a new entry or not.
+ */
+static bool
+rct_table_add(dcontext_t *dcontext, app_pc tag, rct_type_t which)
+{
+    rct_module_table_t *permod;
+    /* we use a higher-level lock to synchronize the lookup + add
+     * combination with other simultaneous adds as well as with removals
+     */
+    /* FIXME We could use just the table lock for the lookup+add. This is cleaner
+     * than using another lock during the entire routine and acquiring & releasing
+     * the table lock in read mode for the lookup and then acquiring & releasing
+     * it again in write mode for the add. Also, any writes to the table outside
+     * of this routine would be blocked (as is desired). The down side is that
+     * reads would be blocked during the entire operation.
+     * The #ifdef DEBUG lookup would need to be moved to after the table lock
+     * is released to avoid a rank order violation (all table locks have the
+     * same rank). That's not problematic since it's only stat code.
+     */
+    /* If we no longer hold this high-level lock for adds+removes we need
+     * to hold the new add/remove lock across persist_size->persist
+     */
+    ASSERT_OWN_MUTEX(true, (which == RCT_RAC ? &after_call_lock : &rct_module_lock));
+    os_get_module_info_lock();
+    permod = rct_get_table(tag, which);
+    /* Xref case 9717, on a partial image mapping we may try to add locations
+     * (specifically the entry point) that our outside of any module. Technically this
+     * is also possible on a full mapping since we've seen entry points redirected
+     * (and there's nothing requiring that they be re-directed to another dll or, if
+     * at dr init, that we've already processed that target module, xref case 10693. */
+    ASSERT_CURIOSITY(permod != NULL || EXEMPT_TEST("partial_map.exe"));
+    if (permod == NULL || rct_table_lookup_internal(dcontext, tag, permod) != NULL) {
+        os_get_module_info_unlock();
+        return false;
+    }
+    if (permod->live_table == NULL) {
+        /* lazily initialized */
+        if (rct_is_global_table(permod))
+            SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+        permod->live_table = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, app_pc_table_t,
+                                             ACCT_AFTER_CALL, PROTECTED);
+        if (rct_is_global_table(permod)) {
+            /* For global tables we would have to move to heap, or
+             * else unprot every time, to maintain min and max: but
+             * the min-max optimization isn't going to help global
+             * tables so we just don't bother.
+             */
+            permod->live_min = NULL;
+            permod->live_max = (app_pc) POINTER_MAX;
+            SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+        }
+        hashtable_app_pc_init(GLOBAL_DCONTEXT, permod->live_table,
+                              which == RCT_RAC ? INIT_HTABLE_SIZE_AFTER_CALL :
+                              INIT_HTABLE_SIZE_RCT_IBT,
+                              which == RCT_RAC ? DYNAMO_OPTION(shared_after_call_load) :
+                              DYNAMO_OPTION(global_rct_ind_br_load),
+                              (hash_function_t)INTERNAL_OPTION(alt_hash_func),
+                              0 /* hash_mask_offset */,
+                              (SHARED_FRAGMENTS_ENABLED() ?
+                               HASHTABLE_ENTRY_SHARED : 0)
+                              | HASHTABLE_SHARED | HASHTABLE_PERSISTENT
+                              /* I'm seeing a number of high-ave-collision
+                               * cases on both rac and rct; there's no easy
+                               * way to estimate final size, so going to
+                               * relax a little as not perf-critical */
+                              | HASHTABLE_RELAX_CLUSTER_CHECKS
+                              NAME(which == RCT_RAC ? "after_call_targets" :
+                                   "rct_ind_targets"));
+        STATS_RCT_ADD(which, live_tables, 1);
+    }
+    ASSERT(permod->live_table != NULL);
+    /* adding is a write operation */
+    TABLE_RWLOCK(permod->live_table, write, lock);
+    hashtable_app_pc_add(dcontext, tag, permod->live_table);
+    TABLE_RWLOCK(permod->live_table, write, unlock);
+    /* case 7628: used for persistence optimization: but watch overhead */
+    if (!rct_is_global_table(permod)) {
+        /* See comments above */
+        if (permod->live_min == NULL || tag < permod->live_min)
+            permod->live_min = tag;
+        if (tag > permod->live_max)
+            permod->live_max = tag;
+    }
+    os_get_module_info_unlock();
+    STATS_RCT_ADD(which, live_entries, 1);
+    DOSTATS({
+        if (permod == &rac_non_module_table)
+            STATS_INC(rac_non_module_entries);
+    });
+    return true;
+}
+
+static void
+rct_table_flush_entry(dcontext_t *dcontext, app_pc tag, rct_type_t which)
+{
+    rct_module_table_t *permod;
+    /* need higher level lock to properly synchronize with lookup+add */
+    ASSERT_OWN_MUTEX(true, (which == RCT_RAC ? &after_call_lock : &rct_module_lock));
+    os_get_module_info_lock();
+    permod = rct_get_table(tag, which);
+    ASSERT(permod != NULL);
+    /* We should have removed any persist info before calling this routine */
+    ASSERT(permod->persisted_table == NULL);
+    ASSERT(permod->live_table != NULL);
+    if (permod->live_table != NULL) {
+        /* removing is a write operation */
+        TABLE_RWLOCK(permod->live_table, write, lock);
+        hashtable_app_pc_remove(tag, permod->live_table);
+        TABLE_RWLOCK(permod->live_table, write, unlock);
+    }
+    os_get_module_info_unlock();
+}
+
+/* Invalidates all after call or indirect branch targets from given
+ * range [text_start,text_end) which must be either completely
+ * contained in a single module or not touch any modules.
+ * Assuming any existing fragments that were added to IBL tables will
+ * be flushed independently: this routine only flushes the policy
+ * information.
+ * Note this needs to be called on app_memory_deallocation() for RAC
+ * (potentially from DGC), and rct_process_module_mmap() for other RCT
+ * entries which should be only in modules.
+ * 
+ * Returns entries flushed.
+ */
+static uint
+rct_table_invalidate_range(dcontext_t *dcontext, rct_type_t which,
+                           app_pc text_start, app_pc text_end)
+{
+    uint entries_removed = 0;
+    rct_module_table_t *permod;
+    /* need higher level lock to properly synchronize with lookup+add */
+    ASSERT_OWN_MUTEX(true, (which == RCT_RAC ? &after_call_lock : &rct_module_lock));
+    ASSERT(text_start < text_end);
+
+    if (DYNAMO_OPTION(rct_sticky)) {
+        /* case 5329 - leaving for bug-compatibility with previous releases */
+        /* trade-off is spurious RCT violations vs memory leak */
+        return 0;
+    }
+
+    /* We only support removing from within a single module or not touching
+     * any modules */
+    ASSERT(get_module_base(text_start) == get_module_base(text_end));
+
+    os_get_module_info_lock();
+    permod = rct_get_table(text_start, which);
+    ASSERT(permod != NULL);
+    /* We should have removed any persist info before calling this routine */
+    ASSERT(permod->persisted_table == NULL);
+    ASSERT(permod->live_table != NULL);
+    if (permod != NULL && permod->live_table != NULL) {
+        TABLE_RWLOCK(permod->live_table, write, lock);
+        entries_removed =
+            hashtable_app_pc_range_remove(dcontext, permod->live_table, 
+                                          (ptr_uint_t)text_start, (ptr_uint_t)text_end);
+
+        DODEBUG({
+            uint second_pass =
+                hashtable_app_pc_range_remove(dcontext, permod->live_table, 
+                                              (ptr_uint_t)text_start,
+                                              (ptr_uint_t)text_end);
+            ASSERT(second_pass == 0 && "nothing should be missed");
+            /* simplest sanity check that hashtable_app_pc_range_remove() works */
+        });
+        TABLE_RWLOCK(permod->live_table, write, unlock);
+    }
+    os_get_module_info_unlock();
+    return entries_removed;
+}
+
+static void
+rct_table_free_internal(dcontext_t *dcontext, app_pc_table_t *table)
+{
+    hashtable_app_pc_free(dcontext, table);
+    ASSERT(TEST(HASHTABLE_PERSISTENT, table->table_flags));
+    HEAP_TYPE_FREE(dcontext, table, app_pc_table_t, ACCT_AFTER_CALL, PROTECTED);
+}
+
+void
+rct_table_free(dcontext_t *dcontext, app_pc_table_t *table, bool free_data)
+{
+    DODEBUG({
+        DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
+            hashtable_app_pc_load_statistics(dcontext, table);
+        });
+        hashtable_app_pc_study(dcontext, table, 0/*table consistent*/);
+    });
+    if (!free_data) {
+        /* We don't need the free_data param anymore */
+        ASSERT(table->table_unaligned == NULL); /* don't try to free, part of mmap */
+    }
+    rct_table_free_internal(GLOBAL_DCONTEXT, table);
+}
+
+app_pc_table_t *
+rct_table_copy(dcontext_t *dcontext, app_pc_table_t *src)
+{
+    if (src == NULL)
+        return NULL;
+    else
+        return hashtable_app_pc_copy(GLOBAL_DCONTEXT, src);    
+}
+
+app_pc_table_t *
+rct_table_merge(dcontext_t *dcontext, app_pc_table_t *src1, app_pc_table_t *src2)
+{
+    if (src1 == NULL) {
+        if (src2 == NULL)
+            return NULL;
+        return hashtable_app_pc_copy(GLOBAL_DCONTEXT, src2);
+    } else if (src2 == NULL)
+        return hashtable_app_pc_copy(GLOBAL_DCONTEXT, src1);
+    else
+        return hashtable_app_pc_merge(GLOBAL_DCONTEXT, src1, src2);    
+}
+
+/* Up to caller to synchronize access to table. */
+uint
+rct_table_persist_size(dcontext_t *dcontext, app_pc_table_t *table)
+{
+    /* Don't persist zero-entry tables */
+    if (table == NULL || table->entries == 0)
+        return 0;
+    else
+        return hashtable_app_pc_persist_size(dcontext, table);
+}
+
+/* Up to caller to synchronize access to table.
+ * Returns true iff all writes succeeded.
+ */
+bool
+rct_table_persist(dcontext_t *dcontext, app_pc_table_t *table, file_t fd)
+{
+    bool success = true;
+    ASSERT(fd != INVALID_FILE);
+    ASSERT(table != NULL); /* caller shouldn't call us o/w */
+    if (table != NULL)
+        success = hashtable_app_pc_persist(dcontext, table, fd);
+    return success;
+}
+
+app_pc_table_t *
+rct_table_resurrect(dcontext_t *dcontext, byte *mapped_table, rct_type_t which)
+{
+    return hashtable_app_pc_resurrect(GLOBAL_DCONTEXT, mapped_table
+                                      NAME(which == RCT_RAC ? "after_call_targets" :
+                                           "rct_ind_targets"));
+}
+
+void
+rct_module_table_free(dcontext_t *dcontext, rct_module_table_t *permod, app_pc modpc)
+{
+    ASSERT(os_get_module_info_locked());
+    if (permod->live_table != NULL) {
+        rct_table_free(GLOBAL_DCONTEXT, permod->live_table, true);
+        permod->live_table = NULL;
+    }
+    if (permod->persisted_table != NULL) {
+        /* persisted table: table data is from disk, but header is on heap */
+        rct_table_free(GLOBAL_DCONTEXT, permod->persisted_table, false);
+        permod->persisted_table = NULL;
+        /* coarse_info_t has a duplicated pointer to the persisted table,
+         * but it should always be flushed before we get here
+         */
+        ASSERT(get_executable_area_coarse_info(modpc) == NULL);
+    }
+}
+
+void
+rct_module_table_persisted_invalidate(dcontext_t *dcontext, app_pc modpc)
+{
+    rct_module_table_t *permod;
+    uint i;
+    os_get_module_info_lock();
+    for (i = 0; i < RCT_NUM_TYPES; i++) {
+        permod = rct_get_table(modpc, i);
+        ASSERT(permod != NULL);
+        if (permod != NULL && permod->persisted_table != NULL) {
+            /* If the persisted table contains entries beyond what we will discover
+             * when we re-build its cache we must transfer those to the live table
+             * now.  At first I was only keeping entire-module RCT entries, but we
+             * must keep all RAC and RCT entries since we may not see the triggering
+             * code before we hit the check point (may reset at a ret so we won't see
+             * the call before the ret check; same for Borland SEH).  We can only not
+             * keep them on a module unload.
+             */
+            /* Optimization: don't transfer if about to unload.  This assumes
+             * there will be no persistence of a later coarse unit in a different
+             * region of the same module, which case 9651 primary_for_module
+             * currently ensures!  (We already have read lock; ok to grab again.)
+             */
+            if (!os_module_get_flag(modpc, MODULE_BEING_UNLOADED) && !dynamo_exited) {
+                /* FIXME case 10362: we could leave the file mapped in and
+                 * use the persisted RCT table independently of the cache
+                 */
+                /* Merging will remove any dups, though today we never re-load
+                 * pcaches so there shouldn't be any
+                 */
+                app_pc_table_t *merged =
+                    /* all modinfo RCT tables are on global heap */
+                    rct_table_merge(GLOBAL_DCONTEXT, permod->live_table,
+                                    permod->persisted_table);
+                if (permod->live_table != NULL)
+                    rct_table_free(GLOBAL_DCONTEXT, permod->live_table, true);
+                permod->live_table = merged;
+                LOG(THREAD, LOG_FRAGMENT, 2,
+                    "rct_module_table_persisted_invalidate "PFX": not unload, so "
+                    "moving persisted %d entries to live table\n",
+                    modpc, permod->persisted_table->entries);
+# ifdef WINDOWS
+                /* We leave the MODULE_RCT_LOADED flag */
+# endif
+                STATS_INC(rct_persisted_outlast_cache);
+            }
+            /* we rely on coarse_unit_reset_free() freeing the persisted table struct */
+            permod->persisted_table = NULL;
+        }
+    }
+    os_get_module_info_unlock();
+}
+
+/* Produces a new hashtable that contains all entries in the live and persisted
+ * tables for the module containing modpc that are within [limit_start, limit_end)
+ */
+app_pc_table_t *
+rct_module_table_copy(dcontext_t *dcontext, app_pc modpc, rct_type_t which,
+                      app_pc limit_start, app_pc limit_end)
+{
+    app_pc_table_t *merged = NULL;
+    rct_module_table_t *permod;
+    mutex_t *lock = (which == RCT_RAC) ? &after_call_lock : &rct_module_lock;
+    mutex_lock(lock);
+    if (which == RCT_RAC) {
+        ASSERT(DYNAMO_OPTION(ret_after_call));
+        if (!DYNAMO_OPTION(ret_after_call))
+            return NULL;
+    } else {
+        ASSERT(TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) ||
+               TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump)));
+        if (!TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) &&
+            !TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump)))
+            return NULL;
+    }
+    os_get_module_info_lock();
+    permod = rct_get_table(modpc, which);
+    ASSERT(permod != NULL);
+    if (permod != NULL) {
+        /* FIXME: we could pass the limit range down to hashtable_app_pc_{copy,merge}
+         * for more efficiency and to avoid over-sizing the table, but this
+         * should be rare w/ -persist_rct_entire and single-+x-section modules
+         */
+        merged = rct_table_merge(dcontext, permod->live_table, permod->persisted_table);
+        if (merged != NULL) {
+            DEBUG_DECLARE(uint removed = 0;)
+            TABLE_RWLOCK(merged, write, lock);
+            if (limit_start > permod->live_min) {
+                DEBUG_DECLARE(removed +=)
+                    hashtable_app_pc_range_remove(dcontext, merged,
+                                                  (ptr_uint_t)permod->live_min,
+                                                  (ptr_uint_t)limit_start);
+            }
+            if (limit_end <= permod->live_max) {
+                DEBUG_DECLARE(removed +=)
+                    hashtable_app_pc_range_remove(dcontext, merged,
+                                                  (ptr_uint_t)limit_end,
+                                                  (ptr_uint_t)permod->live_max+1);
+            }
+            TABLE_RWLOCK(merged, write, unlock);
+            STATS_RCT_ADD(which, module_persist_out_of_range, removed);
+        }
+    }
+    os_get_module_info_unlock();
+    mutex_unlock(lock);
+    return merged;
+}
+
+/* We return the persisted table so we can keep a pointer to it in the
+ * loaded coarse_info_t, but we must be careful to do a coordinated
+ * free of the duplicated pointer.
+ */
+bool
+rct_module_table_set(dcontext_t *dcontext, app_pc modpc, app_pc_table_t *table,
+                     rct_type_t which)
+{
+    rct_module_table_t *permod;
+    bool used = false;
+    mutex_t *lock = (which == RCT_RAC) ? &after_call_lock : &rct_module_lock;
+    mutex_lock(lock);
+    os_get_module_info_lock();
+    permod = rct_get_table(modpc, which);
+    ASSERT(permod != NULL);
+    ASSERT(permod->persisted_table == NULL); /* can't resurrect twice */
+    ASSERT(table != NULL);
+    /* Case 9834: avoid double-add from earlier entire-module resurrect */
+    ASSERT(which == RCT_RAC || !os_module_get_flag(modpc, MODULE_RCT_LOADED));
+    /* FIXME case 8648: we're loosening security by allowing ret to target any
+     * after-call executed in any prior run of this app or whatever
+     * app is producing, instead of just this run.
+     */
+    if (permod != NULL && permod->persisted_table == NULL) {
+        used = true;
+        /* There could be dups in persisted table that are already in
+         * live table (particularly from unloading and re-loading a pcache,
+         * though that never happens today).  Everything should work fine,
+         * and if we do unload the pcache prior to module unload the merge
+         * into the live entries will remove the dups.
+         */
+        permod->persisted_table = table;
+        ASSERT(permod->persisted_table->entries > 0);
+        /* FIXME: for case 9639 if we had the ibl table set up (say, for shared
+         * ibl tables) and stored the cache pc (though I guess coarse htable is
+         * set up) we could fill the ibl table here as well (but then we'd
+         * have to abort for hotp conflicts earlier in coarse_unit_load()).
+         * Instead we delay and do it in rac_persisted_fill_ibl().
+         */
+        LOG(THREAD, LOG_FRAGMENT, 2,
+            "rct_module_table_resurrect: added %d %s entries\n",
+            permod->persisted_table->entries, which == RCT_RAC ? "RAC" : "RCT");
+        STATS_RCT_ADD(which, persisted_tables, 1);
+        STATS_RCT_ADD(which, persisted_entries, permod->persisted_table->entries);
+    }
+    os_get_module_info_unlock();
+    mutex_unlock(lock);
+    return used;
+}
+
+bool
+rct_module_persisted_table_exists(dcontext_t *dcontext, app_pc modpc,
+                                  rct_type_t which)
+{
+    bool exists = false;
+    rct_module_table_t *permod;
+    os_get_module_info_lock();
+    permod = rct_get_table(modpc, which);
+    exists = (permod != NULL && permod->persisted_table != NULL);
+    os_get_module_info_unlock();
+    return exists;
+}
+
+uint
+rct_module_live_entries(dcontext_t *dcontext, app_pc modpc, rct_type_t which)
+{
+    uint entries = 0;
+    rct_module_table_t *permod;
+    os_get_module_info_lock();
+    permod = rct_get_table(modpc, which);
+    if (permod != NULL && permod->live_table != NULL)
+        entries = permod->live_table->entries;
+    os_get_module_info_unlock();
+    return entries;
+}
+
+static void
+coarse_persisted_fill_ibl_helper(dcontext_t *dcontext, ibl_table_t *ibl_table,
+                                 coarse_info_t *info, app_pc_table_t *ptable,
+                                 ibl_branch_type_t branch_type)
+{
+    uint i;
+    fragment_t wrapper;
+    app_pc tag;
+    cache_pc body_pc;
+    DEBUG_DECLARE(uint added = 0;)
+    ASSERT(ptable != NULL);
+    if (ptable == NULL)
+        return;
+    ASSERT(os_get_module_info_locked());
+
+    /* Make sure this thread's local ptrs & state are up to
+     * date in case a resize occured while it was in the cache. */
+    update_private_ibt_table_ptrs(dcontext, ibl_table _IF_DEBUG(NULL));
+
+    /* Avoid hash collision asserts while adding by sizing up front;
+     * FIXME: we may over-size for INDJMP table
+     */
+    TABLE_RWLOCK(ibl_table, write, lock);
+    hashtable_ibl_check_size(dcontext, ibl_table, 0, ptable->entries);
+    TABLE_RWLOCK(ibl_table, write, unlock);
+
+    /* FIXME: we should hold ptable's read lock but it's lower ranked
+     * than the fragment table's lock, so we rely on os module lock
+     */
+    for (i = 0; i < ptable->capacity; i++) {
+        tag = ptable->table[i];
+        if (APP_PC_ENTRY_IS_REAL(tag)) {
+            /* FIXME: should we persist the cache pcs to save time here?  That won't
+             * be ideal if we ever use the mmapped table directly (for per-module
+             * tables: case 9672).  We could support both tag-only and tag-cache
+             * pairs by having the 1st entry be a flags word.
+             */
+            fragment_coarse_lookup_in_unit(dcontext, info, tag, NULL, &body_pc);
+            /* may not be present, given no checks in rct_entries_in_region() */
+            if (body_pc != NULL && 
+                /* We can have same entry in both RAC and RCT table, and we use
+                 * both tables to fill the INDJMP table
+                 */
+                (branch_type != IBL_INDJMP ||
+                 !IBL_ENTRY_IS_EMPTY(hashtable_ibl_lookup(dcontext, (ptr_uint_t)tag,
+                                                          ibl_table)))) {
+                fragment_coarse_wrapper(&wrapper, tag, body_pc);
+                fragment_add_ibl_target_helper(dcontext, &wrapper, ibl_table);
+                DOSTATS({ added++; });
+            }
+        }
+    }
+    LOG(THREAD, LOG_FRAGMENT, 2,
+        "coarse_persisted_fill_ibl %s: added %d of %d entries\n",
+        get_branch_type_name(branch_type), added, ptable->entries);
+    STATS_ADD(perscache_ibl_prefill, added);
+}
+
+/* Case 9639: fill ibl table from persisted RAC/RCT table entries */
+static void
+coarse_persisted_fill_ibl(dcontext_t *dcontext, coarse_info_t *info,
+                          ibl_branch_type_t branch_type)
+{
+    per_thread_t *pt = GET_PT(dcontext);
+    ibl_table_t *ibl_table;
+    rct_module_table_t *permod;
+
+    /* Caller must hold info lock */
+    ASSERT_OWN_MUTEX(true, &info->lock);
+    ASSERT(exists_coarse_ibl_pending_table(dcontext, info, branch_type));
+    ASSERT(TEST(COARSE_FILL_IBL_MASK(branch_type),
+                DYNAMO_OPTION(coarse_fill_ibl)));
+    if (!exists_coarse_ibl_pending_table(dcontext, info, branch_type))
+        return;
+    
+    os_get_module_info_lock();
+    ibl_table = GET_IBT_TABLE(pt, FRAG_SHARED|FRAG_COARSE_GRAIN, branch_type);
+    if (branch_type == IBL_RETURN || branch_type == IBL_INDJMP) {
+        permod = rct_get_table(info->base_pc, RCT_RAC);
+        ASSERT(permod != NULL && permod->persisted_table != NULL);
+        if (permod != NULL && permod->persisted_table != NULL) {
+            LOG(THREAD, LOG_FRAGMENT, 2,
+                "coarse_persisted_fill_ibl %s: adding RAC %d entries\n",
+                get_branch_type_name(branch_type), permod->persisted_table->entries);
+            coarse_persisted_fill_ibl_helper(dcontext, ibl_table, info,
+                                             permod->persisted_table, branch_type);
+        }
+    }
+    if (branch_type == IBL_INDCALL || branch_type == IBL_INDJMP) {
+        permod = rct_get_table(info->base_pc, RCT_RCT);
+        ASSERT(permod != NULL && permod->persisted_table != NULL);
+        if (permod != NULL && permod->persisted_table != NULL) {
+            LOG(THREAD, LOG_FRAGMENT, 2,
+                "coarse_persisted_fill_ibl %s: adding RCT %d entries\n",
+                get_branch_type_name(branch_type), permod->persisted_table->entries);
+            coarse_persisted_fill_ibl_helper(dcontext, ibl_table, info,
+                                             permod->persisted_table, branch_type);
+        }
+    }
+    os_get_module_info_unlock();
+    /* We only fill for the 1st thread (if using per-thread ibl
+     * tables).  We'd need per-thread flags to do otherwise, and the
+     * goal is only to help startup performance.
+     * FIXME case 9639: later threads may do startup work in various apps,
+     * and we may want a better solution here.
+     */
+    info->ibl_pending_used |= COARSE_FILL_IBL_MASK(branch_type);
+}
+
+#endif /* defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH) */
+
+#ifdef RETURN_AFTER_CALL
+
+/* returns NULL if not found */
+app_pc
+fragment_after_call_lookup(dcontext_t *dcontext, app_pc tag)
+{
+    return rct_table_lookup(dcontext, tag, RCT_RAC);
+}
+
+void
+fragment_add_after_call(dcontext_t *dcontext, app_pc tag)
+{
+    mutex_lock(&after_call_lock);
+    if (!rct_table_add(dcontext, tag, RCT_RAC))
+        STATS_INC(num_existing_after_call);
+    else
+        STATS_INC(num_future_after_call);
+    mutex_unlock(&after_call_lock);
+}
+
+/* flushing a fragment invalidates the after call entry */
+void
+fragment_flush_after_call(dcontext_t *dcontext, app_pc tag)
+{
+    mutex_lock(&after_call_lock);
+    rct_table_flush_entry(dcontext, tag, RCT_RAC);
+    mutex_unlock(&after_call_lock);
+    STATS_INC(num_future_after_call_removed);
+    STATS_DEC(num_future_after_call);
+}
+
+/* see comments in rct_table_invalidate_range() */
+uint
+invalidate_after_call_target_range(dcontext_t *dcontext, 
+                                   app_pc text_start, app_pc text_end)
+{
+    uint entries_removed;
+    mutex_lock(&after_call_lock);
+    entries_removed =
+        rct_table_invalidate_range(dcontext, RCT_RAC, text_start, text_end);
+    mutex_unlock(&after_call_lock);
+
+    STATS_ADD(num_future_after_call_removed, entries_removed);
+    STATS_SUB(num_future_after_call, entries_removed);
+
+    LOG(THREAD, LOG_FRAGMENT, 2,
+        "invalidate_rct_target_range "PFX"-"PFX": removed %d entries\n",
+        text_start, text_end, entries_removed);
+
+    return entries_removed;
+}
+
+#endif /* RETURN_AFTER_CALL */
+
+/***********************************************************************/
+#ifdef RCT_IND_BRANCH
+/*
+ * RCT indirect branch policy bookkeeping.  Mostly a set of wrappers
+ * around the basic hashtable functionality.
+ *
+ * FIXME: all of these routines should be moved to rct.c after we move
+ * the hashtable primitives to fragment.h as static inline's
+ */
+
+/* returns NULL if not found */
+app_pc
+rct_ind_branch_target_lookup(dcontext_t *dcontext, app_pc tag)
+{
+    return rct_table_lookup(dcontext, tag, RCT_RCT);
+}
+
+/* returns true if a new entry for target was added, 
+ * or false if target was already known
+ */
+/* Note - entries are expected to be within MEM_IMAGE */
+bool
+rct_add_valid_ind_branch_target(dcontext_t *dcontext, app_pc tag)
+{
+    ASSERT_OWN_MUTEX(true, &rct_module_lock);
+    DOLOG(2, LOG_FRAGMENT, {
+        /* FIXME: would be nice to add a heavy weight check that we're
+         * really only a PE IMAGE via is_in_code_section()
+         */
+    });
+    if (!rct_table_add(dcontext, tag, RCT_RCT))
+        return false;
+    else {
+        STATS_INC(rct_ind_branch_entries);
+        return true; /* new entry */
+    }
+}
+
+/* invalidate an indirect branch target and free any associated memory */
+/* FIXME: note that this is currently not used and
+ * invalidate_ind_branch_target_range() will be the likely method to
+ * use for most cases when a whole module range is invalidated.
+ */
+void
+rct_flush_ind_branch_target_entry(dcontext_t *dcontext, app_pc tag)
+{
+    ASSERT_OWN_MUTEX(true, &rct_module_lock); /* synch with adding */
+    rct_table_flush_entry(dcontext, tag, RCT_RCT);
+    STATS_DEC(rct_ind_branch_entries);
+    STATS_INC(rct_ind_branch_entries_removed);
+}
+
+/* see comments in rct_table_invalidate_range() */
+uint
+invalidate_ind_branch_target_range(dcontext_t *dcontext, 
+                                   app_pc text_start, app_pc text_end)
+{
+    uint entries_removed;
+    ASSERT_OWN_MUTEX(true, &rct_module_lock); /* synch with adding */
+
+    entries_removed = 
+        rct_table_invalidate_range(dcontext, RCT_RCT, text_start, text_end);
+    STATS_ADD(rct_ind_branch_entries_removed, entries_removed);
+    STATS_SUB(rct_ind_branch_entries, entries_removed);
+
+    return entries_removed;
+}
+#endif /* RCT_IND_BRANCH */
+
+
+#ifdef NATIVE_RETURN
+/* add f to the deleted table
+ * we call this in non-sideline-safe manners!  FIXME
+ */
+void
+fragment_add_deleted(dcontext_t *dcontext, fragment_t *f)
+{
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+    fragment_table_t *deltable = &pt->deleted;
+    /* adding is a write operation */
+    TABLE_RWLOCK(deltable, write, lock);
+    fragment_add_to_hashtable(dcontext, f, deltable);
+    TABLE_RWLOCK(deltable, write, unlock);
+}
+#endif
+
+/****************************************************************************/
+/* CACHE CONSISTENCY */
+
+/* Handle exits from the cache from our self-modifying code sandboxing
+ * instrumentation.
+ */
+void
+fragment_self_write(dcontext_t *dcontext)
+{
+    ASSERT(!is_self_couldbelinking());
+    /* need to delete just this fragment, then start interpreting
+     * at the instr after the self-write instruction
+     */
+    dcontext->next_tag = EXIT_TARGET_TAG(dcontext, dcontext->last_fragment,
+                                         dcontext->last_exit);
+    LOG(THREAD, LOG_ALL, 2, "Sandboxing exit from fragment "PFX" @"PFX"\n",
+        dcontext->last_fragment->tag,
+        EXIT_CTI_PC(dcontext->last_fragment, dcontext->last_exit));
+    LOG(THREAD, LOG_ALL, 2, "\tset next_tag to "PFX"\n", dcontext->next_tag);
+    /* We come in here both for actual selfmod and for exec count thresholds,
+     * to avoid needing separate LINK_ flags.
+     */
+    if (DYNAMO_OPTION(sandbox2ro_threshold) > 0) {
+        if (vm_area_selfmod_check_clear_exec_count(dcontext, dcontext->last_fragment)) {
+            /* vm_area_* deleted this fragment by flushing so nothing more to do */
+            return;
+        }
+    }
+    LOG(THREAD, LOG_ALL, 1, "WARNING: fragment "PFX" @"PFX" overwrote its own code\n",
+        dcontext->last_fragment->tag,
+        EXIT_CTI_PC(dcontext->last_fragment, dcontext->last_exit));
+    STATS_INC(num_self_writes);
+    if (TEST(FRAG_WAS_DELETED, dcontext->last_fragment->flags)) {
+        /* Case 8177: case 3559 unionized fragment_t.in_xlate, so we cannot delete a
+         * fragment that has already been unlinked in the first stage of a flush.
+         * The flush queue check, which comes after this (b/c we want to be
+         * nolinking), will delete.
+         */
+        ASSERT(((per_thread_t *) dcontext->fragment_field)->flush_queue_nonempty);
+        STATS_INC(num_self_writes_after_flushes);
+    } else {
+#ifdef PROGRAM_SHEPHERDING
+        /* we can't call fragment_delete if vm_area_t deletes it by flushing */
+        if (!vm_area_fragment_self_write(dcontext, dcontext->last_fragment->tag))
+#endif
+            {
+                fragment_delete(dcontext, dcontext->last_fragment, FRAGDEL_ALL);
+                STATS_INC(num_fragments_deleted_selfmod);
+            }
+    }
+}
+
+/* coarse_grain says to use just the min_pc and max_pc of f.
+ * otherwise a full walk of the original code is done to see if
+ * any piece of the fragment really does overlap.
+ * returns the tag of the bb that actually overlaps (i.e., finds the
+ * component bb of a trace that does the overlapping).
+ */
+bool
+fragment_overlaps(dcontext_t *dcontext, fragment_t *f,
+                  byte *region_start, byte *region_end, bool coarse_grain,
+                  overlap_info_t *info_res, app_pc *bb_tag)
+{
+    overlap_info_t info;
+    info.overlap = false;
+    if ((f->flags & FRAG_IS_TRACE) != 0) {
+        uint i;
+        trace_only_t *t = TRACE_FIELDS(f);
+        /* look through all blocks making up the trace */
+        ASSERT(t->bbs != NULL);
+        /* trace should have at least one bb */
+        ASSERT(t->num_bbs > 0);
+        for (i=0; i<t->num_bbs; i++) {
+            if (app_bb_overlaps(dcontext, t->bbs[i].tag, f->flags,
+                                region_start, region_end, &info)) {
+                if (bb_tag != NULL)
+                    *bb_tag = t->bbs[i].tag;
+                break;
+            }
+        }
+    } else {
+        app_bb_overlaps(dcontext, f->tag, f->flags,
+                        region_start, region_end, &info);
+        if (info.overlap && bb_tag != NULL)
+            *bb_tag = f->tag;
+    }
+    if (info_res != NULL)
+        *info_res = info;
+    return info.overlap;
+}
+
+#ifdef DEBUG
+void
+study_all_hashtables(dcontext_t *dcontext)
+{
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+    ibl_branch_type_t branch_type;
+
+    for (branch_type = IBL_BRANCH_TYPE_START; 
+         branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+        if (!DYNAMO_OPTION(disable_traces)) {
+            per_thread_t *ibl_pt = pt;
+            if (DYNAMO_OPTION(shared_trace_ibt_tables))
+                ibl_pt = shared_pt;
+            hashtable_ibl_study(dcontext, &ibl_pt->trace_ibt[branch_type], 
+                                0/*table consistent*/);
+        }
+        if (DYNAMO_OPTION(bb_ibl_targets)) {
+            per_thread_t *ibl_pt = pt;
+            if (DYNAMO_OPTION(shared_bb_ibt_tables))
+                ibl_pt = shared_pt;
+            hashtable_ibl_study(dcontext, &ibl_pt->bb_ibt[branch_type], 
+                                0/*table consistent*/);
+        }
+    }
+    if (PRIVATE_TRACES_ENABLED())
+        hashtable_fragment_study(dcontext, &pt->trace, 0/*table consistent*/);
+    hashtable_fragment_study(dcontext, &pt->bb, 0/*table consistent*/);
+    hashtable_fragment_study(dcontext, &pt->future, 0/*table consistent*/);
+    if (DYNAMO_OPTION(shared_bbs))
+        hashtable_fragment_study(dcontext, shared_bb, 0/*table consistent*/);
+    if (DYNAMO_OPTION(shared_traces))
+        hashtable_fragment_study(dcontext, shared_trace, 0/*table consistent*/);
+    if (SHARED_FRAGMENTS_ENABLED())
+        hashtable_fragment_study(dcontext, shared_future, 0/*table consistent*/);
+# ifdef RETURN_AFTER_CALL
+    if (dynamo_options.ret_after_call && rac_non_module_table.live_table != NULL) {
+        hashtable_app_pc_study(dcontext, rac_non_module_table.live_table,
+                               0/*table consistent*/);
+    }
+# endif
+# if defined(RCT_IND_BRANCH) && defined(LINUX)
+    if ((TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) ||
+         TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump))) &&
+        rct_global_table.live_table != NULL) {
+        hashtable_app_pc_study(dcontext, rct_global_table.live_table,
+                               0/*table consistent*/);
+    }
+# endif /* RCT_IND_BRANCH */
+# if defined(WIN32) && (defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH))
+    {
+        module_iterator_t *mi = module_iterator_start();
+        uint i;
+        rct_module_table_t *permod;
+        while (module_iterator_hasnext(mi)) {
+            module_area_t *data = module_iterator_next(mi);
+            for (i = 0; i < RCT_NUM_TYPES; i++) {
+                permod = os_module_get_rct_htable(data->start, i);
+                ASSERT(permod != NULL);
+                if (permod->persisted_table != NULL) {
+                    LOG(THREAD, LOG_FRAGMENT, 2,
+                        "%s persisted hashtable for %s "PFX"-"PFX"\n",
+                        i == RCT_RAC ? "RAC" : "RCT",
+                        GET_MODULE_NAME(&data->names), data->start, data->end);
+                    hashtable_app_pc_study(dcontext, permod->persisted_table,
+                                           0/*table consistent*/);
+                }
+                if (permod->live_table != NULL) {
+                    LOG(THREAD, LOG_FRAGMENT, 2,
+                        "%s live hashtable for %s "PFX"-"PFX"\n",
+                        i == RCT_RAC ? "RAC" : "RCT",
+                        GET_MODULE_NAME(&data->names), data->start, data->end);
+                    hashtable_app_pc_study(dcontext, permod->live_table,
+                                           0/*table consistent*/);
+                }
+            }
+        }
+        module_iterator_stop(mi);
+    }
+# endif /* defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH) */
+}
+#endif /* DEBUG */
+
+/****************************************************************************
+ * FLUSHING
+ *
+ * Two-stage freeing of fragments via immediate unlinking followed by lazy 
+ * deletion.
+ */
+
+uint
+get_flushtime_last_update(dcontext_t *dcontext)
+{
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+    return pt->flushtime_last_update;
+}
+
+void
+set_flushtime_last_update(dcontext_t *dcontext, uint val)
+{
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+    pt->flushtime_last_update = val;
+}
+
+void
+set_at_syscall(dcontext_t *dcontext, bool val)
+{
+    ASSERT(dcontext != GLOBAL_DCONTEXT);
+    dcontext->upcontext_ptr->at_syscall = val;
+}
+
+bool
+get_at_syscall(dcontext_t *dcontext)
+{
+    ASSERT(dcontext != GLOBAL_DCONTEXT);
+    return dcontext->upcontext_ptr->at_syscall;
+}
+
+/* Assumes caller takes care of synchronization.
+ * Returns false iff was_I_flushed ends up being deleted right now from
+ * a private cache OR was_I_flushed has been flushed from a shared cache
+ * and is pending final deletion.
+ */
+static bool
+check_flush_queue(dcontext_t *dcontext, fragment_t *was_I_flushed)
+{
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+    bool not_flushed = true;
+    ASSERT_OWN_MUTEX(true, &pt->linking_lock);
+    /* first check private queue and act on pending deletions */
+    if (pt->flush_queue_nonempty) {
+        bool local_prot = local_heap_protected(dcontext);
+        if (local_prot)
+            SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+        /* remove local vm areas on t->Q queue and all frags in their lists */
+        not_flushed = not_flushed &&
+            vm_area_flush_fragments(dcontext, was_I_flushed);
+        pt->flush_queue_nonempty = false;
+        LOG(THREAD, LOG_FRAGMENT, 2, "Hashtable state after flushing the queue:\n");
+        DOLOG(2, LOG_FRAGMENT, {
+            study_all_hashtables(dcontext);
+        });
+        if (local_prot)
+            SELF_PROTECT_LOCAL(dcontext, READONLY);
+    }
+    /* now check shared queue to dec ref counts */
+    if (DYNAMO_OPTION(shared_deletion) &&
+        /* No lock needed: any racy incs to global are in safe direction, and our inc
+         * is atomic so we shouldn't see any partial-word-updated values here.  This
+         * check is our shared deletion algorithm's only perf hit when there's no
+         * actual shared flushing.
+         */
+        pt->flushtime_last_update < flushtime_global) {
+        /* dec ref count on any pending shared areas */
+        not_flushed = not_flushed &&
+            vm_area_check_shared_pending(dcontext, was_I_flushed);
+        /* Remove unlinked markers if called for.
+         * FIXME If a thread's flushtime is updated due to shared syscall sync,
+         * its tables won't be rehashed here -- the thread's flushtime will be
+         * equal to the global flushtime so the 'if' isn't entered. We have
+         * multiple options as to other points for rehashing -- a table add, a
+         * table delete, any entry into DR. For now, we've chosen a table add
+         * (see check_table_size()).
+         */
+        if (SHARED_IB_TARGETS() &&
+            (INTERNAL_OPTION(rehash_unlinked_threshold) < 100 ||
+             INTERNAL_OPTION(rehash_unlinked_always))) {
+
+            ibl_branch_type_t branch_type;
+
+            for (branch_type = IBL_BRANCH_TYPE_START; 
+                 branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+
+                ibl_table_t *table = &pt->bb_ibt[branch_type];
+
+                if (table->unlinked_entries > 0 &&
+                    (INTERNAL_OPTION(rehash_unlinked_threshold) <
+                     (100 * table->unlinked_entries /
+                      (table->unlinked_entries + table->entries)) ||
+                     INTERNAL_OPTION(rehash_unlinked_always))) {
+                    STATS_INC(num_ibt_table_rehashes);
+                    LOG(THREAD, LOG_FRAGMENT, 1,
+                        "Rehash table %s: linked %u, unlinked %u\n",
+                        table->name, table->entries, table->unlinked_entries);
+                    hashtable_ibl_unlinked_remove(dcontext, table);
+                }
+            }
+        }
+    }
+
+    /* FIXME This is the ideal location for inserting refcounting logic
+     * for freeing a resized shared IBT table, as is done for shared
+     * deletion above.
+     */
+
+    return not_flushed;
+}
+
+/* Note that an all-threads-synch flush does NOT set the self-flushing flag,
+ * so use is_self_allsynch_flushing() instead.
+ */
+bool
+is_self_flushing()
+{
+    /* race condition w/ flusher being updated -- but since only testing vs self,
+     * if flusher update is atomic, should be safe
+     */
+    return (get_thread_private_dcontext() == flusher);
+}
+
+bool
+is_self_allsynch_flushing()
+{
+    /* race condition w/ allsynch_flusher being updated -- but since only testing
+     * vs self, if flusher update is atomic, should be safe
+     */
+    return (allsynch_flusher != NULL &&
+            get_thread_private_dcontext() == allsynch_flusher);
+}
+
+/* N.B.: only accurate if called on self (else a race condition) */
+bool
+is_self_couldbelinking()
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    /* if no dcontext yet then can't be couldbelinking */
+    return (dcontext != NULL &&
+            !RUNNING_WITHOUT_CODE_CACHE() /*case 7966: has no pt*/ &&
+            is_couldbelinking(dcontext));
+}
+
+/* N.B.: can only call if target thread is self, suspended, or waiting for flush */
+bool
+is_couldbelinking(dcontext_t *dcontext)
+{
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+    /* the lock is only needed for self writing or another thread reading
+     * but if different thread we require thread is suspended or waiting for
+     * flush so is ok */
+    /* FIXME : add an assert that the thread that owns the dcontext is either
+     * the caller, at the flush sync wait, or is suspended by thread_synch 
+     * routines */
+    return (!RUNNING_WITHOUT_CODE_CACHE() /*case 7966: has no pt*/ &&
+            pt->could_be_linking);
+}
+
+static void
+wait_for_flusher_nolinking(dcontext_t *dcontext)
+{
+    /* FIXME: can have livelock w/ these types of synch loops,
+     * any way to work into deadlock-avoidance?
+     */
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+    ASSERT(!pt->could_be_linking);
+    while (pt->wait_for_unlink) {
+        LOG(THREAD, LOG_DISPATCH|LOG_THREADS, 2,
+            "Thread %d waiting for flush (flusher is %d @flushtime %d)\n",
+            /* safe to deref flusher since flusher is waiting for our signal */
+            dcontext->owning_thread, flusher->owning_thread, flushtime_global);
+        mutex_unlock(&pt->linking_lock);
+        STATS_INC(num_wait_flush);
+        wait_for_event(pt->finished_all_unlink);
+        LOG(THREAD, LOG_DISPATCH|LOG_THREADS, 2,
+            "Thread %d resuming after flush\n", dcontext->owning_thread);
+        mutex_lock(&pt->linking_lock);
+    }
+}
+
+static void
+wait_for_flusher_linking(dcontext_t *dcontext)
+{
+    /* FIXME: can have livelock w/ these types of synch loops,
+     * any way to work into deadlock-avoidance?
+     */
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+    ASSERT(pt->could_be_linking);
+    while (pt->wait_for_unlink) {
+        LOG(THREAD, LOG_DISPATCH|LOG_THREADS, 2,
+            "Thread %d waiting for flush (flusher is %d @flushtime %d)\n",
+            /* safe to deref flusher since flusher is waiting for our signal */
+            dcontext->owning_thread, flusher->owning_thread, flushtime_global);
+        mutex_unlock(&pt->linking_lock);
+        signal_event(pt->waiting_for_unlink);
+        STATS_INC(num_wait_flush);
+        wait_for_event(pt->finished_with_unlink);
+        LOG(THREAD, LOG_DISPATCH|LOG_THREADS, 2,
+            "Thread %d resuming after flush\n", dcontext->owning_thread);
+        mutex_lock(&pt->linking_lock);
+    }
+}
+
+#ifdef DEBUG
+static void
+check_safe_for_flush_synch(dcontext_t *dcontext)
+{
+    /* We cannot hold any locks at synch points that wait for flushers, as we
+     * could prevent forward progress of a couldbelinking thread that the
+     * flusher will wait for.
+     */
+    /* FIXME: will fail w/ -single_thread_in_DR, along w/ the other similar
+     * asserts for flushing and cache entering
+     */
+    ASSERT(thread_owns_no_locks(dcontext) ||
+           /* if thread exits while a trace is in progress (case 8055) we're
+            * holding thread_initexit_lock, which prevents any flusher from
+            * proceeding and hitting a deadlock point, so we should be safe
+            */
+           thread_owns_one_lock(dcontext, &thread_initexit_lock) ||
+           /* it is safe to be the all-thread-syncher and hit a flush synch
+            * point, as no flusher can be active (all threads should be suspended
+            * except this thread)
+            */
+           thread_owns_two_locks(dcontext, &thread_initexit_lock,
+                                 &all_threads_synch_lock));
+}
+#endif
+
+#ifdef CLIENT_INTERFACE
+static void
+process_client_flush_requests(dcontext_t *dcontext, dcontext_t *alloc_dcontext,
+                              client_flush_req_t *req, bool flush)
+{
+    client_flush_req_t *iter = req;
+    while (iter != NULL) {
+        client_flush_req_t *next = iter->next;
+        if (flush) {
+            /* Note that we don't free futures from potentially linked-to region b/c we
+             * don't have lazy linking (xref case 2236) */
+            /* FIXME - if there's more then one of these would be nice to batch them
+             * especially for the synch all ones. */
+            if (iter->flush_callback != NULL) {
+                /* FIXME - for implementation simplicity we do a synch-all flush so
+                 * that we can inform the client right away, it might be nice to use
+                 * the more performant regular flush when possible. */
+                flush_fragments_from_region(dcontext, iter->start, iter->size,
+                                            true/*force synchall*/);
+                (*iter->flush_callback)(iter->flush_id);
+            } else {
+                /* do a regular flush */
+                flush_fragments_from_region(dcontext, iter->start, iter->size,
+                                            false/*don't force synchall*/);
+            }
+        }
+        HEAP_TYPE_FREE(alloc_dcontext, iter, client_flush_req_t, ACCT_OTHER, UNPROTECTED);
+        iter = next;
+    }
+}
+#endif
+
+/* Returns false iff was_I_flushed ends up being deleted 
+ * if cache_transition is true, assumes entering the cache now.
+ */
+bool
+enter_nolinking(dcontext_t *dcontext, fragment_t *was_I_flushed, bool cache_transition)
+{
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+    bool not_flushed = true;
+#ifdef CLIENT_INTERFACE
+    client_flush_req_t *req;
+#endif
+
+    /*case 7966: has no pt, no flushing either */
+    if (RUNNING_WITHOUT_CODE_CACHE())
+        return true;
+
+    DODEBUG({ check_safe_for_flush_synch(dcontext); });
+
+    /* FIXME: once we have this working correctly, come up with scheme
+     * that avoids synch in common case
+     */
+    mutex_lock(&pt->linking_lock);
+    ASSERT(pt->could_be_linking);
+
+    wait_for_flusher_linking(dcontext);
+    not_flushed = not_flushed && check_flush_queue(dcontext, was_I_flushed);
+    pt->could_be_linking = false;
+    mutex_unlock(&pt->linking_lock);
+
+    if (!cache_transition)
+        return not_flushed;
+
+    /* now we act on pending actions that can only be done while nolinking
+     * FIXME: optimization if add more triggers here: use a single
+     * master trigger as a first test to avoid testing all
+     * conditionals every time
+     */
+
+    if (reset_pending != 0) {
+        mutex_lock(&reset_pending_lock);
+        if (reset_pending != 0) {
+            uint target = reset_pending;
+            reset_pending = 0;
+            /* fcache_reset_all_caches_proactively() will unlock */
+            fcache_reset_all_caches_proactively(target);
+            LOG(THREAD, LOG_DISPATCH, 2,
+                "Just reset all caches, next_tag is "PFX"\n", dcontext->next_tag);
+            /* fragment is gone for sure, so return false */
+            return false;
+        }
+        mutex_unlock(&reset_pending_lock);
+    }
+
+    /* FIXME: perf opt: make global flag can check w/ making a call,
+     * or at least inline the call
+     */
+    if (fcache_is_flush_pending(dcontext)) {
+        not_flushed = not_flushed &&
+            fcache_flush_pending_units(dcontext, was_I_flushed);
+    }
+
+#ifdef CLIENT_INTERFACE
+    /* Handle flush requests queued via dr_flush_fragments()/dr_delay_flush_region() */
+    /* thread private list */
+    process_client_flush_requests(dcontext, dcontext, dcontext->client_data->flush_list,
+                                  true/*flush*/);
+    dcontext->client_data->flush_list = NULL;
+    /* global list */
+    mutex_lock(&client_flush_request_lock);
+    req = client_flush_requests;
+    client_flush_requests = NULL;
+    mutex_unlock(&client_flush_request_lock);
+    /* NOTE - we must release the lock before doing the flush. */
+    process_client_flush_requests(dcontext, GLOBAL_DCONTEXT, req, true/*flush*/);
+    /* FIXME - this is an ugly, yet effective, hack.  The problem is there is no
+     * good way to tell currently if we flushed was_I_flushed.  Since it could be
+     * gone by now we pretend that it was flushed if we did any flushing at all.
+     * Dispatch should refind the fragment if it wasn't flushed. */
+    if (req != NULL)
+        return false;
+#endif
+
+    return not_flushed;
+}
+
+/* Returns false iff was_I_flushed ends up being deleted */
+bool
+enter_couldbelinking(dcontext_t *dcontext, fragment_t *was_I_flushed, bool cache_transition)
+{
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+    bool not_flushed;
+
+    /*case 7966: has no pt, no flushing either */
+    if (RUNNING_WITHOUT_CODE_CACHE())
+        return true;
+
+    DODEBUG({ check_safe_for_flush_synch(dcontext); });
+
+    mutex_lock(&pt->linking_lock);
+    ASSERT(!pt->could_be_linking);
+    /* ensure not still marked at_syscall */
+    ASSERT(!DYNAMO_OPTION(syscalls_synch_flush) || !get_at_syscall(dcontext));
+
+    /* for thread-shared flush and thread-private flush+execareas atomicity,
+     * to avoid non-properly-nested locks (could have flusher hold
+     * pt->linking_lock for the entire flush) we need an additional
+     * synch point here for shared flushing to synch w/ all threads.
+     * I suppose switching from non-nested locks to this loop isn't
+     * nec. helping deadlock avoidance -- can still hang -- but this
+     * should be better performance-wise (only a few writes and a
+     * conditional in the common case here, no extra locks) 
+     */
+    pt->soon_to_be_linking = true;
+    wait_for_flusher_nolinking(dcontext);
+    pt->soon_to_be_linking = false;
+
+    pt->could_be_linking = true;
+    not_flushed = check_flush_queue(dcontext, was_I_flushed);
+    mutex_unlock(&pt->linking_lock);
+
+    return not_flushed;
+}
+
+/* NOTE - this routine may be called more then one time for the same exiting thread,
+ * xref case 8047.  Any once only reference counting, cleanup, etc. should be done in
+ * fragment_thread_exit().  This routine is just a stripped down version of 
+ * enter_nolinking() to keep an exiting thread from deadlocking with flushing. */
+void
+enter_threadexit(dcontext_t *dcontext)
+{
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+
+    /*case 7966: has no pt, no flushing either */
+    if (RUNNING_WITHOUT_CODE_CACHE())
+        return;
+
+    mutex_lock(&pt->linking_lock);
+    /* must dec ref count on shared regions before we die */
+    check_flush_queue(dcontext, NULL);
+    pt->could_be_linking = false;
+    if (pt->wait_for_unlink) {
+        /* make sure don't get into deadlock w/ flusher */
+        pt->about_to_exit = true; /* let flusher know can ignore us */
+        signal_event(pt->waiting_for_unlink); /* wake flusher up */
+    }
+    mutex_unlock(&pt->linking_lock);
+}
+
+/* caller must hold shared_cache_flush_lock */
+void
+increment_global_flushtime()
+{
+    ASSERT_OWN_MUTEX(true, &shared_cache_flush_lock);
+    /* reset will turn flushtime_global back to 0, so we schedule one
+     * when we're approaching overflow
+     */
+    if (flushtime_global == UINT_MAX/2) {
+        ASSERT_NOT_TESTED(); /* FIXME: add -stress_flushtime_global_max */
+        SYSLOG_INTERNAL_WARNING("flushtime_global approaching UINT_MAX, resetting");
+        schedule_reset(RESET_ALL);
+    }
+    ASSERT(flushtime_global < UINT_MAX);
+
+    /* compiler should 4-byte-align so no cache line crossing
+     * (asserted in fragment_init()
+     */
+    flushtime_global++;
+    LOG(GLOBAL, LOG_VMAREAS, 2, "new flush timestamp: %u\n",
+        flushtime_global);
+}
+
+/* The master flusher routines are split into 3:
+ *   stage1) flush_fragments_synch_unlink_priv
+ *   stage2) flush_fragments_unlink_shared
+ *   stage3) flush_fragments_end_synch
+ * which MUST be used together.  They are split to allow the caller to
+ * perform custom operations at certain synchronization points in the
+ * middle of flushing without using callback routines.  The stages and
+ * points are:
+ *
+ *   stage1: head synch, priv flushee unlink
+ *     here caller can produce a custom list of fragments to flush while
+ *       all threads are fully synched
+ *   stage2: shared flushee unlink
+ *
+ *    between stage2 and stage3, a region flush performs additional actions:
+ *     region flush: exec area lock
+ *       here caller can do custom exec area manipulations
+ *     region flush: exec area unlock
+ *
+ *   stage3: tail synch
+ *
+ * When doing a region flush with no custom region removals (the default is
+ * removing [base,base+size)), use the routine flush_fragments_and_remove_region().
+ * When doing a region flush with custom removals, use
+ * flush_fragments_in_region_start() and flush_fragments_in_region_finish().
+ * When doing a flush with no region removals at all use flush_fragments_from_region().
+ *
+ * The thread requesting a flush must be !couldbelinking and must not
+ * be holding any locks that are ever grabbed while a thread is
+ * couldbelinking (pretty much all locks), as it must wait for other
+ * threads who are couldbelinking.  The thread_initexit_lock is held
+ * from stage 1 to stage 3 (if there are fragments to flush), and
+ * region flushing also holds the executable_areas lock between stage
+ * 2 and stage 3.  After stage 1 no thread is couldbelinking() until
+ * the synchronization release in stage 3.
+ *
+ * The general delayed-deletion flushing strategy involves first
+ * freezing the threads via the thread_initexit_lock.
+ * It then walks the thread list and marks all vm areas that overlap
+ * base...base+size as to-be-deleted, along with unlinking all fragments in those
+ * vm areas and unlinking shared_syscall for that thread.
+ * Fragments in the target area are not actually deleted until their owning thread
+ * checks its pending deletion queue, which it does prior to entering the fcache.
+ * Also flushes shared fragments, in a similar manner, with deletion delayed
+ * until all threads are out of the shared cache.
+ */
+
+/* Variables shared between the 3 flush stage routines
+ * flush_fragments_synch_unlink_priv(),
+ * flush_fragments_unlink_shared(), and flush_fragments_end_synch.
+ * As there can only be one flush at a time, we only need one copy,
+ * protected by the thread_initexit_lock.
+ */
+/* Not persistent across code cache execution, so unprotected */
+DECLARE_NEVERPROT_VAR(static thread_record_t **flush_threads, NULL);
+DECLARE_NEVERPROT_VAR(static int flush_num_threads, 0);
+DECLARE_NEVERPROT_VAR(static int pending_delete_threads, 0);
+DECLARE_NEVERPROT_VAR(static int shared_flushed, 0);
+DECLARE_NEVERPROT_VAR(static bool flush_synchall, false);
+#ifdef DEBUG
+DECLARE_NEVERPROT_VAR(static int num_flushed, 0);
+DECLARE_NEVERPROT_VAR(static int flush_last_stage, 0);
+#endif
+
+static void
+flush_fragments_free_futures(app_pc base, size_t size)
+{
+    int i;
+    dcontext_t *tgt_dcontext;
+    ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
+    ASSERT((allsynch_flusher == NULL && flusher == get_thread_private_dcontext()) ||
+           (flusher == NULL && allsynch_flusher == get_thread_private_dcontext()));
+    ASSERT(flush_num_threads > 0);
+    ASSERT(flush_threads != NULL);
+    if (DYNAMO_OPTION(free_unmapped_futures) && !RUNNING_WITHOUT_CODE_CACHE()) {
+        /* We need to free the futures after all fragments have been unlinked,
+         * as unlinking will create new futures
+         */
+        acquire_recursive_lock(&change_linking_lock);
+        for (i=0; i<flush_num_threads; i++) {
+            tgt_dcontext = flush_threads[i]->dcontext;
+            if (tgt_dcontext != NULL) {
+                fragment_delete_futures_in_region(tgt_dcontext, base, base + size);
+                thcounter_range_remove(tgt_dcontext, base, base + size);
+            }
+        }
+        if (SHARED_FRAGMENTS_ENABLED())
+            fragment_delete_futures_in_region(GLOBAL_DCONTEXT, base, base + size);
+        release_recursive_lock(&change_linking_lock);
+    } /* else we leak them */
+}
+
+/* This routine begins a flush that requires full thread synch: currently,
+ * it is used for flushing coarse-grain units and for dr_flush_region()
+ */
+static void
+flush_fragments_synchall_start(dcontext_t *ignored, app_pc base, size_t size,
+                               bool exec_invalid)
+{
+    dcontext_t *my_dcontext = get_thread_private_dcontext();
+    app_pc exec_start = NULL, exec_end = NULL;
+    bool all_synched = true;
+    int i;
+    const thread_synch_state_t desired_state =
+        THREAD_SYNCH_SUSPENDED_VALID_MCONTEXT_OR_NO_XFER;
+    DEBUG_DECLARE(bool ok;)
+    KSTART(synchall_flush);
+    LOG(GLOBAL, LOG_FRAGMENT, 2, 
+        "\nflush_fragments_synchall_start: thread %d suspending all threads\n",
+        get_thread_id());
+
+    STATS_INC(flush_synchall);
+    /* suspend all DR-controlled threads at safe locations */
+    DEBUG_DECLARE(ok =)
+        synch_with_all_threads(desired_state, &flush_threads, &flush_num_threads,
+                               THREAD_SYNCH_NO_LOCKS_NO_XFER,
+                               /* if we fail to suspend a thread (e.g., for
+                                * privilege reasons), ignore it since presumably
+                                * the failed thread is some injected thread not
+                                * running to-be-flushed code, so we continue
+                                * with the flush!
+                                */
+                               THREAD_SYNCH_SUSPEND_FAILURE_IGNORE);
+    ASSERT(ok);
+    /* now we own the thread_initexit_lock */
+    ASSERT(OWN_MUTEX(&all_threads_synch_lock) && OWN_MUTEX(&thread_initexit_lock));
+
+    /* We do NOT set flusher as is_self_flushing() is all about
+     * couldbelinking, while our synch is of a different nature.  The
+     * trace_abort() is_self_flushing() check is the only worrisome
+     * one: FIXME.
+     */
+    ASSERT(flusher == NULL);
+    ASSERT(allsynch_flusher == NULL);
+    allsynch_flusher = my_dcontext;
+    flush_synchall = true;
+    ASSERT(flush_last_stage == 0);
+    DODEBUG({ flush_last_stage = 1; });
+   
+    LOG(GLOBAL, LOG_FRAGMENT, 2, 
+        "flush_fragments_synchall_start: walking the threads\n");
+    /* We rely on coarse fragments not touching more than one vmarea region
+     * for our ibl invalidation.  It's
+     * ok to invalidate more than we need to so we don't care if there are
+     * multiple coarse units within this range.  We just need the exec areas
+     * bounds that overlap the flush region.
+     */
+    if (!executable_area_overlap_bounds(base, base+size, &exec_start, &exec_end,
+                                        0, true/*doesn't matter w/ 0*/)) {
+        /* caller checks for overlap but lock let go so can get here; go ahead
+         * and do synch per flushing contract.
+         */
+        exec_start = base;
+        exec_end = base+size;
+    }
+    LOG(GLOBAL, LOG_FRAGMENT, 2, 
+        "flush_fragments_synchall_start: from "PFX"-"PFX" => coarse "PFX"-"PFX"\n",
+        base, base+size, exec_start, exec_end);
+
+    /* FIXME: share some of this code that I duplicated from reset */
+    for (i = 0; i < flush_num_threads; i++) {
+        dcontext_t *dcontext = flush_threads[i]->dcontext;
+        if (dcontext != NULL) { /* include my_dcontext here */
+            DEBUG_DECLARE(uint removed;)
+            LOG(GLOBAL, LOG_FRAGMENT, 2, 
+                "\tconsidering thread #%d %d\n", i, flush_threads[i]->id);
+            if (dcontext != my_dcontext) {
+                /* must translate BEFORE freeing any memory! */
+                if (!thread_synch_successful(flush_threads[i])) {
+                    /* FIXME case 9480: if we do get here for low-privilege handles
+                     * or exceeding our synch try count, best to not move the thread
+                     * as we won't get a clean translation.  Chances are it is
+                     * not in the region being flushed.
+                     */
+                    SYSLOG_INTERNAL_ERROR_ONCE("failed to synch with thread during "
+                                               "synchall flush");
+                    LOG(THREAD, LOG_FRAGMENT|LOG_SYNCH, 2, 
+                        "failed to synch with thread %d\n", i);
+                    STATS_INC(flush_synchall_fail);
+                    all_synched = false;
+                } else if (is_thread_currently_native(flush_threads[i])) {
+                    /* native_exec's regain-control point is in our DLL,
+                     * and lost-control threads are truly native, so no
+                     * state to worry about except for hooks -- and we're
+                     * not freeing the interception buffer.
+                     */
+                    LOG(GLOBAL, LOG_FRAGMENT, 2,
+                        "\tcurrently native so no translation needed\n");
+                } else if (thread_synch_state_no_xfer(dcontext)) {
+                    /* Case 6821: do not translate other synch-all-thread users.
+                     * They have no fragment state, so leave alone.
+                     * All flush call points are in syscalls or from nudges.
+                     * Xref case 8901 on ensuring nudges don't try to return
+                     * to the code cache.
+                     */
+                    LOG(GLOBAL, LOG_FRAGMENT, 2,
+                        "\tat THREAD_SYNCH_NO_LOCKS_NO_XFER so no translation needed\n");
+                    STATS_INC(flush_synchall_races);
+                } else {
+                    translate_from_synchall_to_dispatch(flush_threads[i], desired_state);
+                }
+            }
+            if (dcontext == my_dcontext || thread_synch_successful(flush_threads[i])) {
+                last_exit_deleted(dcontext);
+                /* case 7394: need to abort other threads' trace building
+                 * since the reset xfer to dispatch will disrupt it.
+                 * also, with PR 299808, we now have thread-shared
+                 * "undeletable" trace-temp fragments, so we need to abort
+                 * all traces.
+                 */
+                if (is_building_trace(dcontext)) {
+                    LOG(THREAD, LOG_FRAGMENT, 2,
+                        "\tsquashing trace of thread %d\n", i);
+                    trace_abort(dcontext);
+                }
+            }
+            /* Since coarse fragments never cross coarse/non-coarse executable region
+             * bounds, we can bound their bodies by taking
+             * executable_area_distinct_bounds().  This lets us remove by walking the
+             * ibl tables and looking only at tags, rather than walking the htable of
+             * each coarse unit.  FIXME: not clear this is a perf win: I arbitrarily
+             * picked it under assumption that ibl tables are relatively small.  It
+             * would be a clearer win if we could do the fine fragments this way
+             * also, but fine fragments are not constrained and could be missed using
+             * only a tag-based range remove.
+             */
+            DEBUG_DECLARE(removed =)
+                fragment_remove_all_ibl_in_region(dcontext, exec_start, exec_end);
+            LOG(THREAD, LOG_FRAGMENT, 2,
+                "\tremoved %d ibl entries in "PFX"-"PFX"\n",
+                removed, exec_start, exec_end);
+            /* Free any fine private fragments in the region */
+            vm_area_allsynch_flush_fragments(dcontext, dcontext, base, base+size,
+                                             exec_invalid, all_synched/*ignored*/);
+            if (!SHARED_IBT_TABLES_ENABLED() && SHARED_FRAGMENTS_ENABLED()) {
+                /* Remove shared fine fragments from private ibl tables */
+                vm_area_allsynch_flush_fragments(dcontext, GLOBAL_DCONTEXT,
+                                                 base, base+size,
+                                                 exec_invalid, all_synched/*ignored*/);
+            }
+        }
+    }
+    /* Removed shared coarse fragments from ibl tables, before freeing any */
+    if (SHARED_IBT_TABLES_ENABLED() && SHARED_FRAGMENTS_ENABLED())
+        fragment_remove_all_ibl_in_region(GLOBAL_DCONTEXT, exec_start, exec_end);
+    /* Free coarse units and shared fine fragments, as well as removing shared fine
+     * entries in any shared ibl tables
+     */
+    if (SHARED_FRAGMENTS_ENABLED()) {
+        vm_area_allsynch_flush_fragments(GLOBAL_DCONTEXT, GLOBAL_DCONTEXT,
+                                         base, base+size, exec_invalid, all_synched);
+    }
+}
+
+static void
+flush_fragments_synchall_end(dcontext_t *ignored)
+{
+    thread_record_t **temp_threads = flush_threads;
+    DEBUG_DECLARE(dcontext_t *my_dcontext = get_thread_private_dcontext();)
+    LOG(GLOBAL, LOG_FRAGMENT, 2, 
+        "flush_fragments_synchall_end: resuming all threads\n");
+
+    /* We need to clear this before we release the locks.  We use a temp var
+     * so we can use end_synch_with_all_threads.
+     */
+    flush_threads = NULL;
+    ASSERT(flusher == NULL);
+    flush_synchall = false;
+    ASSERT(dynamo_all_threads_synched);
+    ASSERT(allsynch_flusher == my_dcontext);
+    allsynch_flusher = NULL;
+    end_synch_with_all_threads(temp_threads, flush_num_threads, true/*resume*/);
+    KSTOP(synchall_flush);
+}
+
+/* This routine begins a flush of the group of fragments in the memory
+ * region [base, base+size) by synchronizing with each thread and unlinking
+ * all private fragments in the region.
+ *
+ * The exec_invalid parameter must be set to indicate whether the
+ * executable area is being invalidated as well or this is just a capacity
+ * flush (or a flush to change instrumentation).
+ *
+ * If size==0 then no unlinking occurs; however, the full synch is
+ * performed (so the caller should check size if no action is desired on
+ * zero size).
+ *
+ * If size>0 and there is no executable area overlap, then no synch is
+ * performed and false is returned.  The caller must acquire the executable
+ * areas lock and re-check the overlap if exec area manipulation is to be
+ * performed.  Returns true otherwise.
+ */
+bool
+flush_fragments_synch_unlink_priv(dcontext_t *dcontext, app_pc base, size_t size,
+                                  /* WARNING: case 8572: the caller owning this lock
+                                   * is incompatible w/ suspend-the-world flushing!
+                                   */
+                                  bool own_initexit_lock, bool exec_invalid,
+                                  bool force_synchall _IF_DGCDIAG(app_pc written_pc))
+{
+    dcontext_t *tgt_dcontext;
+    per_thread_t *tgt_pt;
+    int i;
+
+    LOG(THREAD, LOG_FRAGMENT, 2,
+        "FLUSH STAGE 1: synch_unlink_priv(thread %d flushtime %d): "PFX"-"PFX"\n",
+        dcontext->owning_thread, flushtime_global, base, base + size);
+    /* Case 9750: to specify a region of size 0, do not pass in NULL as the base!
+     * Use EMPTY_REGION_{BASE,SIZE} instead.
+     */
+    ASSERT(base != NULL || size != 0);
+
+    /* our flushing design requires that flushers are NOT couldbelinking
+     * and are not holding any locks
+     */
+    ASSERT(!is_self_couldbelinking());
+#if defined(DEADLOCK_AVOIDANCE) && defined(DEBUG)
+    if (own_initexit_lock) {
+        /* We can get here while holding the all_threads_synch_lock
+         * for detach (& prob. TerminateProcess(0) with other threads
+         * still active) on XP and 2003 via os_thread_stack_exit()
+         * (for other thread exit) if we have executed off that stack.
+         * Can be seen with Araktest detach on violation using the stack
+         * attack button. */
+        ASSERT(thread_owns_first_or_both_locks_only(dcontext,
+                                                    &thread_initexit_lock,
+                                                    &all_threads_synch_lock));
+    } else {
+        ASSERT_OWN_NO_LOCKS();
+    }
+#endif
+    ASSERT(dcontext == get_thread_private_dcontext());
+ 
+    /* quick check for overlap first by using read lock and avoiding
+     * thread_initexit_lock:
+     * if no overlap, must hold lock through removal of region
+     * if overlap, ok to release for a bit
+     */
+    if (size > 0 && !executable_vm_area_executed_from(base, base+size)) {
+        /* Only a curiosity since we can have a race (not holding exec areas lock) */
+        ASSERT_CURIOSITY((!SHARED_FRAGMENTS_ENABLED() ||
+                          !thread_vm_area_overlap(GLOBAL_DCONTEXT, base, base+size)) &&
+                         !thread_vm_area_overlap(dcontext, base, base+size));
+        return false;
+    }
+    /* Only a curiosity since we can have a race (not holding exec areas lock) */
+    ASSERT_CURIOSITY(size == 0 ||
+                     executable_vm_area_overlap(base, base+size, false/*no lock*/));
+
+    STATS_INC(num_flushes);
+
+    if (force_synchall ||
+        (size > 0 && executable_vm_area_coarse_overlap(base, base+size))) {
+        /* Coarse units do not support individual unlinking (though prior to
+         * freezing they do, we ignore that) and instead require all-thread-synch
+         * in order to flush.  For that we cannot be already holding
+         * thread_initexit_lock!  FIXME case 8572: the only caller who does hold it
+         * now is os_thread_stack_exit().  For now relying on that stack not
+         * overlapping w/ any coarse regions.
+         */
+        ASSERT(!own_initexit_lock);
+        /* The synchall will flush fine as well as coarse so we'll be done */
+        flush_fragments_synchall_start(dcontext, base, size, exec_invalid);
+        return true;
+    }
+
+    /* Take a snapshot of the threads in the system.
+     * Grab the thread lock to prevent threads from being created or
+     * exited for the duration of this routine
+     * FIXME -- is that wise?  could be a relatively long time!
+     * It's unlikely a new thread will run code in a region being
+     * unmapped...but we would like to prevent threads from exiting
+     * while we're messing with their data.
+     * FIXME: need to special-case every instance where thread_initexit_lock
+     * is grabbed, to avoid deadlocks!  we already do for a thread exiting.
+     */
+    if (!own_initexit_lock)
+        mutex_lock(&thread_initexit_lock);
+    ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
+    flusher = dcontext;
+    get_list_of_threads(&flush_threads, &flush_num_threads);
+
+    ASSERT(flush_last_stage == 0);
+    DODEBUG({ flush_last_stage = 1; });
+
+    /* FIXME: we can optimize this even more to not grab thread_initexit_lock */
+    if (RUNNING_WITHOUT_CODE_CACHE()) /* case 7966: nothing to flush, ever */
+        return true;
+
+    /* Set the ref count of threads who may be using a deleted fragment.  We
+     * include ourselves in the ref count as we could be invoked with a cache
+     * return point before any synch (as done w/ hotpatches) and should NOT
+     * consider ourselves done w/ all cache fragments right now.  We assume we
+     * will soon hit a real synch point to dec the count for us, and don't try
+     * to specially handle the single-threaded case here.
+     */
+    pending_delete_threads = flush_num_threads;
+
+    DODEBUG({ num_flushed = 0; });
+
+#ifdef WINDOWS
+    /* Make sure exit fcache if currently inside syscall.  For thread-shared
+     * shared_syscall, we re-link after the shared fragments have all been
+     * unlinked and removed from the ibl tables.  All we lose is the bounded time
+     * on a thread checking its private flush queue, which in a thread-shared
+     * configuration doesn't seem so bad.  For thread-private this will work as
+     * well, with the same time bound lost, but we're not as worried about memory
+     * usage of thread-private configurations.
+     */
+    if (DYNAMO_OPTION(shared_syscalls) && IS_SHARED_SYSCALL_THREAD_SHARED)
+        unlink_shared_syscall(GLOBAL_DCONTEXT);
+#endif
+
+    for (i=0; i<flush_num_threads; i++) {
+        tgt_dcontext = flush_threads[i]->dcontext;
+        tgt_pt = (per_thread_t *) tgt_dcontext->fragment_field;
+        LOG(THREAD, LOG_FRAGMENT, 2,
+            "  considering thread %d/%d = %d\n", i+1, flush_num_threads,
+            flush_threads[i]->id);
+        ASSERT(is_thread_known(tgt_dcontext->owning_thread));
+
+        /* can't do anything, even check if thread has any vm areas overlapping flush
+         * region, until sure thread is in fcache or somewhere that won't change
+         * vm areas or linking state
+         */
+        mutex_lock(&tgt_pt->linking_lock);
+        /* Must explicitly check for self and avoid synch then, o/w will lock up
+         * if ever called from a could_be_linking location (currently only
+         * happens w/ app syscalls)
+         */
+        if (tgt_dcontext != dcontext && tgt_pt->could_be_linking) {
+            /* remember we have a global lock, thread_initexit_lock, so two threads
+             * cannot be here at the same time!
+             */
+            LOG(THREAD, LOG_FRAGMENT, 2,
+                "\twaiting for thread %d\n", tgt_dcontext->owning_thread);
+            tgt_pt->wait_for_unlink = true;
+            mutex_unlock(&tgt_pt->linking_lock);
+            wait_for_event(tgt_pt->waiting_for_unlink);
+            mutex_lock(&tgt_pt->linking_lock);
+            tgt_pt->wait_for_unlink = false;
+            LOG(THREAD, LOG_FRAGMENT, 2,
+                "\tdone waiting for thread %d\n", tgt_dcontext->owning_thread);
+        } else {
+            LOG(THREAD, LOG_FRAGMENT, 2,
+                "\tthread %d synch not required\n", tgt_dcontext->owning_thread);
+        }
+            
+        /* it is now safe to access link, vm, and trace info in tgt_dcontext
+         * => FIXME: rename, since not just accessing linking info?
+         * FIXME: if includes vm access, are syscalls now bad?
+         * what about handle_modified_code, considered nolinking since straight
+         * from cache via fault handler?  reading should be ok, but exec area splits
+         * and whatnot?
+         */
+
+        if (tgt_pt->about_to_exit) {
+            /* thread is about to exit, it's waiting for us to give up
+             * thread_initexit_lock -- we don't need to flush it
+             */
+            goto next_thread;
+        }
+
+        /* if a trace-in-progress crosses this region, must squash the trace
+         * (all traces are essentially frozen now since threads stop in dispatch)
+         */
+        if (size > 0 /* else, no region to cross */ &&
+            is_building_trace(tgt_dcontext)) {
+            void *trace_vmlist = cur_trace_vmlist(tgt_dcontext);
+            if (trace_vmlist != NULL &&
+                vm_list_overlaps(tgt_dcontext, trace_vmlist, base, base+size)) {
+                LOG(THREAD, LOG_FRAGMENT, 2,
+                    "\tsquashing trace of thread %d\n", tgt_dcontext->owning_thread);
+                trace_abort(tgt_dcontext);
+            }
+        }
+
+#ifdef WINDOWS
+        /* Make sure exit fcache if currently inside syscall.  If thread has no
+         * vm area overlap, will be re-linked down below before going to the
+         * next thread, unless we have shared fragments, in which case we force
+         * the thread to check_flush_queue() on its next synch point.  It will
+         * re-link in vm_area_flush_fragments().
+         */
+        if (DYNAMO_OPTION(shared_syscalls) && !IS_SHARED_SYSCALL_THREAD_SHARED)
+            unlink_shared_syscall(tgt_dcontext);
+#endif
+
+        /* Optimization for shared deletion strategy: perform flush work
+         * for a thread waiting at a system call on behalf of that thread
+         * (which we do before the flush tail synch, as if we did it now we
+         * would need to inc flushtime_global up front, and then we'd have synch
+         * issues trying to prevent thread we hadn't synched with from checking
+         * the pending list before we put flushed fragments on it).
+         * We do count these threads in the ref count as we check the pending
+         * queue on their behalf after adding the newly flushed fragments to
+         * the queue, so the ref count gets decremented right away.
+         *
+         * We must do this AFTER unlinking shared_syscall's post-syscall ibl, to
+         * avoid races -- the thread will hit a real synch point before accessing
+         * any fragments or link info.
+         * Do this BEFORE checking whether fragments in region to catch all threads.
+         */
+        ASSERT(!tgt_pt->at_syscall_at_flush);
+        if (DYNAMO_OPTION(syscalls_synch_flush) && get_at_syscall(tgt_dcontext)) {
+            /* we have to know exactly which threads were at_syscall here when
+             * we get to post-flush, so we cache in this special bool
+             */
+            DEBUG_DECLARE(bool tables_updated;)
+            tgt_pt->at_syscall_at_flush = true;
+#ifdef DEBUG
+            tables_updated =
+#endif
+                update_all_private_ibt_table_ptrs(tgt_dcontext, tgt_pt);
+            STATS_INC(num_shared_flush_atsyscall);
+            DODEBUG({
+                if (tables_updated)
+                    STATS_INC(num_shared_tables_updated_atsyscall);
+            });
+        }
+
+        /* don't need to go any further if thread has no frags in region */
+        if (size == 0 || !thread_vm_area_overlap(tgt_dcontext, base, base+size)) {
+            LOG(THREAD, LOG_FRAGMENT, 2,
+                "\tthread %d has no fragments in region to flush\n",
+                tgt_dcontext->owning_thread);
+#ifdef WINDOWS
+            /* restore, since won't be restored in vm_area_flush_fragments */
+            if (DYNAMO_OPTION(shared_syscalls)) {
+                if (SHARED_FRAGMENTS_ENABLED()) {
+                    /* we cannot re-link shared_syscall here as that would allow
+                     * the target thread to enter to-be-flushed fragments prior
+                     * to their being unlinked and removed from ibl tables -- so
+                     * we force this thread to re-link in check_flush_queue.
+                     * we could re-link after unlink/removal of fragments,
+                     * if it's worth optimizing == case 6194/PR 210655.
+                     */
+                    tgt_pt->flush_queue_nonempty = true;
+                    STATS_INC(num_flushq_relink_syscall);
+                } else if (!IS_SHARED_SYSCALL_THREAD_SHARED) {
+                    /* no shared fragments, and no private ones being flushed,
+                     * so this thread is all set
+                     */
+                    link_shared_syscall(tgt_dcontext);
+                }
+            }
+#endif
+            goto next_thread;
+        }
+
+        LOG(THREAD, LOG_FRAGMENT, 2, "\tflushing fragments for thread %d\n",
+            flush_threads[i]->id);
+        DOLOG(2, LOG_FRAGMENT, {
+            if (tgt_dcontext != dcontext) {
+                LOG(tgt_dcontext->logfile, LOG_FRAGMENT, 2,
+                    "thread %d is flushing our fragments\n",
+                    dcontext->owning_thread);
+            }
+        });
+
+        if (size > 0) {
+            /* unlink all frags in overlapping regions, and mark regions for deletion */
+            tgt_pt->flush_queue_nonempty = true;
+#ifdef DEBUG
+            num_flushed += 
+#endif
+                vm_area_unlink_fragments(tgt_dcontext, base, base + size, 0
+                                         _IF_DGCDIAG(written_pc));
+        }
+
+    next_thread:
+        /* for thread-shared, we CANNOT let any thread become could_be_linking, for normal
+         * flushing synch -- for thread-private, we can, but we CANNOT let any thread
+         * that we've already synched with for flushing go and change the exec areas
+         * vector!  the simplest solution is to have thread-private, like thread-shared,
+         * stop all threads at a cache exit point.
+         * FIXME: optimize thread-private by allowing already-synched threads to
+         * continue but not grab executable_areas lock, while letting
+         * to-be-synched threads grab the lock (otherwise could wait forever)
+         */
+        /* Since we must let go of lock for proper nesting, we use a new
+         * synch to stop threads at cache exit, since we need them all
+         * out of DR for duration of shared flush.
+         */
+        if (tgt_dcontext != dcontext && !tgt_pt->could_be_linking)
+            tgt_pt->wait_for_unlink = true; /* stop at cache exit */
+        mutex_unlock(&tgt_pt->linking_lock);
+    }
+
+    return true;
+}
+
+/* This routine continues a flush of one of two groups of fragments:
+ * 1) if list!=NULL, the list of shared fragments beginning at list and 
+ *    chained by next_vmarea (we assume that private fragments are
+ *    easily deleted by the owning thread and do not require a flush).
+ *    Caller should call vm_area_remove_fragment() for each target fragment,
+ *    building a custom list chained by next_vmarea, and pass that list to this
+ *    routine.
+ * 2) otherwise, all fragments in the memory region [base, base+size).
+ *
+ * This routine MUST be called after flush_fragments_synch_unlink_priv(), and
+ * must be followed with flush_fragments_end_synch().
+ */
+void
+flush_fragments_unlink_shared(dcontext_t *dcontext, app_pc base, size_t size,
+                              fragment_t *list _IF_DGCDIAG(app_pc written_pc))
+{
+    /* we would assert that size > 0 || list != NULL but we have to pass
+     * in 0 and NULL for unit flushing when no live fragments are in the unit
+     */
+
+    LOG(THREAD, LOG_FRAGMENT, 2,
+        "FLUSH STAGE 2: unlink_shared(thread %d): flusher is %d\n",
+        dcontext->owning_thread, (flusher == NULL) ? -1 : flusher->owning_thread);
+    ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
+    ASSERT(flush_threads != NULL);
+    ASSERT(flush_num_threads > 0);
+    ASSERT(flush_last_stage == 1);
+    DODEBUG({ flush_last_stage = 2; });
+
+    if (RUNNING_WITHOUT_CODE_CACHE()) /* case 7966: nothing to flush, ever */
+        return;
+    if (flush_synchall) /* no more flush work to do */
+        return;
+
+    if (SHARED_FRAGMENTS_ENABLED()) {
+        /* Flushing shared fragments: the strategy is again immediate
+         * unlinking (plus hashtable removal and vm area list removal)
+         * with delayed deletion.  Unlinking is atomic, and hashtable
+         * removal is for now since there are no shared bbs in ibls from
+         * the cache.  But deletion needs to ensure that every thread who may
+         * hold a pointer to a shared fragment or may be inside a shared
+         * fragment is ok with the fragment being deleted.  We use a reference count
+         * for each flushed region to ensure that every thread has reached a synch
+         * point before we actually free the fragments in the region.
+         *
+         * We do have pathological cases where a single thread at a wait syscall or
+         * an infinite loop in the cache will prevent all freeing.  One solution
+         * to the syscall problem is to store a flag saying that a thread is at
+         * the shared_syscall routine.  That routine is private and unlinked so
+         * we know the thread will hit a synch routine if it exits, so no race.
+         * We then wouldn't bother to inc the ref count for that thread.
+         *
+         * Our fallback proactive deletion is to do a full reset when the size
+         * of the pending deletion list gets too long, though we could be more
+         * efficient by only resetting the pending list, and by only suspending
+         * those threads that haven't dec-ed a ref count in a particular region.
+         */
+        LOG(THREAD, LOG_FRAGMENT, 2, "  flushing shared fragments\n");
+        if (DYNAMO_OPTION(shared_deletion)) {
+            /* We use shared_cache_flush_lock to make atomic the increment of
+             * flushtime_global and the adding of pending deletion fragments with
+             * that flushtime, wrt other threads checking the pending list.
+             */
+            mutex_lock(&shared_cache_flush_lock);
+        }
+        /* Increment flush count for shared deletion algorithm and for list-based
+         * flushing (such as for shared cache units).  We could wait
+         * and only do this if we actually flush shared fragments, but it's
+         * simpler when done in this routine, and nearly every flush does flush
+         * shared fragments.
+         */
+        increment_global_flushtime();
+        /* Both vm_area_unlink_fragments and unlink_fragments_for_deletion call
+         * back to flush_invalidate_ibl_shared_target to remove shared
+         * fragments from private/shared ibl tables
+         */
+        if (list == NULL) {
+            shared_flushed =
+                vm_area_unlink_fragments(GLOBAL_DCONTEXT, base, base + size,
+                                         pending_delete_threads _IF_DGCDIAG(written_pc));
+        } else {
+            shared_flushed = unlink_fragments_for_deletion(GLOBAL_DCONTEXT, list,
+                                                           pending_delete_threads);
+        }
+        if (DYNAMO_OPTION(shared_deletion))
+            mutex_unlock(&shared_cache_flush_lock);
+
+        DODEBUG({
+            num_flushed += shared_flushed;
+            if (shared_flushed > 0)
+                STATS_INC(num_shared_flushes);
+        });
+    }
+
+#ifdef WINDOWS
+    /* Re-link thread-shared shared_syscall */
+    if (DYNAMO_OPTION(shared_syscalls) && IS_SHARED_SYSCALL_THREAD_SHARED)
+        link_shared_syscall(GLOBAL_DCONTEXT);
+#endif
+
+    STATS_ADD(num_flushed_fragments, num_flushed);
+    DODEBUG({
+        if (num_flushed > 0) {
+            LOG(THREAD, LOG_FRAGMENT, 1, "Flushed %5d fragments from "PFX"-"PFX"\n",
+                num_flushed, base, ((char *)base)+size);
+        } else {
+            STATS_INC(num_empty_flushes);
+            LOG(THREAD, LOG_FRAGMENT, 2, "Flushed     0 fragments from "PFX"-"PFX"\n",
+                base, ((char *)base)+size);
+        }
+    });
+}
+
+/* Invalidates (does not remove) shared fragment f from the private/shared
+ * ibl tables.  Can only be called in flush stage 2.
+ */
+void
+flush_invalidate_ibl_shared_target(dcontext_t *dcontext, fragment_t *f)
+{
+    ASSERT(is_self_flushing());
+    ASSERT(!flush_synchall);
+    ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
+    ASSERT(flush_threads != NULL);
+    ASSERT(flush_num_threads > 0);
+    ASSERT(flush_last_stage == 2);
+    ASSERT(TEST(FRAG_SHARED, f->flags));
+    /*case 7966: has no pt, no flushing either */
+    if (RUNNING_WITHOUT_CODE_CACHE()) {
+        ASSERT_NOT_REACHED(); /* shouldn't get here */
+        return;
+    }
+    if (!SHARED_IB_TARGETS())
+        return;
+    if (SHARED_IBT_TABLES_ENABLED()) {
+        /* Remove from shared ibl tables.
+         * dcontext need not be GLOBAL_DCONTEXT.
+         */
+        fragment_prepare_for_removal(dcontext, f);
+    } else {
+        /* We can't tell afterward which entries to remove from the private
+         * ibl tables, as we no longer keep fragment_t* pointers (we used to
+         * look for FRAG_WAS_DELETED) and we can't do a range remove (tags
+         * don't map regularly to regions).  So we must invalidate each
+         * fragment as we process it.  It's ok to walk the thread list
+         * here since we're post-synch for all threads.
+         */
+        int i;
+        for (i=0; i<flush_num_threads; i++) {
+            fragment_prepare_for_removal(flush_threads[i]->dcontext, f);
+        }
+    }
+}
+
+/* Must ONLY be called as the third part of flushing (after
+ * flush_fragments_synch_unlink_priv() and flush_fragments_unlink_shared()).
+ * Uses the static shared variables flush_threads and flush_num_threads.
+ */
+void
+flush_fragments_end_synch(dcontext_t *dcontext, bool keep_initexit_lock)
+{
+    dcontext_t *tgt_dcontext;
+    per_thread_t *tgt_pt;
+    int i;
+
+    LOG(THREAD, LOG_FRAGMENT, 2,
+        "FLUSH STAGE 3: end_synch(thread %d): flusher is %d\n",
+        dcontext->owning_thread, (flusher == NULL) ? -1 : flusher->owning_thread);
+
+    if (!is_self_flushing() && !flush_synchall/* doesn't set flusher */) {
+        LOG(THREAD, LOG_FRAGMENT, 2, "\tnothing was flushed\n");
+        ASSERT_DO_NOT_OWN_MUTEX(!keep_initexit_lock, &thread_initexit_lock);
+        ASSERT_OWN_MUTEX(keep_initexit_lock, &thread_initexit_lock);
+        return;
+    }
+
+    ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
+
+    ASSERT(flush_threads != NULL);
+    ASSERT(flush_num_threads > 0);
+    ASSERT(flush_last_stage == 2);
+    DODEBUG({ flush_last_stage = 0; });
+
+    if (flush_synchall) {
+        flush_fragments_synchall_end(dcontext);
+        return;
+    }
+
+    /* now can let all threads at DR synch point go 
+     * FIXME: if implement thread-private optimization above, this would turn into
+     * re-setting exec areas lock to treat all threads uniformly
+     */
+    for (i=flush_num_threads-1; i>=0; i--) {
+        /*case 7966: has no pt, no flushing either */
+        if (RUNNING_WITHOUT_CODE_CACHE())
+            continue;
+
+        tgt_dcontext = flush_threads[i]->dcontext;
+        tgt_pt = (per_thread_t *) tgt_dcontext->fragment_field;
+        /* re-acquire lock */
+        mutex_lock(&tgt_pt->linking_lock);
+
+        /* Optimization for shared deletion strategy: perform flush work
+         * for a thread waiting at a system call, as we didn't add it to the
+         * ref count in the pre-flush synch.
+         * We assume that shared_syscall is still unlinked at this point and
+         * will not be relinked until we let the thread go.
+         */
+        if (DYNAMO_OPTION(syscalls_synch_flush) && tgt_pt->at_syscall_at_flush) {
+            /* Act on behalf of the thread as though it's at a synch point, but
+             * only wrt shared fragments (we don't free private fragments here,
+             * though we could -- should we?  may make flush time while holding
+             * lock take too long?  FIXME)
+             * Currently this works w/ syscalls from dispatch, and w/
+             * -shared_syscalls by using unprotected storage (thus a slight hole
+             * but very hard to exploit for security purposes: can get stale code
+             * executed, but we already have that window, or crash us).  
+             * FIXME: Does not work w/ -ignore_syscalls, but those are private
+             * for now.
+             */
+            DEBUG_DECLARE(uint pre_flushtime = flushtime_global;)
+            vm_area_check_shared_pending(tgt_dcontext, NULL);
+            /* lazy deletion may inc flushtime_global, so may have a higher
+             * value than our cached one, but should never be lower
+             */
+            ASSERT(tgt_pt->flushtime_last_update >= pre_flushtime);
+            tgt_pt->at_syscall_at_flush = false;
+        }
+
+        if (tgt_dcontext != dcontext) {
+            if (tgt_pt->could_be_linking) {
+                signal_event(tgt_pt->finished_with_unlink);
+            } else {
+                /* we don't need to wait on a !could_be_linking thread
+                 * so we use this bool to tell whether we should signal 
+                 * the event.
+                 * FIXME: really we want a pulse that wakes ALL waiting
+                 * threads and then resets the event!
+                 */
+                tgt_pt->wait_for_unlink = false;
+                if (tgt_pt->soon_to_be_linking)
+                    signal_event(tgt_pt->finished_all_unlink);
+            }
+        }
+        mutex_unlock(&tgt_pt->linking_lock);
+    }
+
+    /* thread init/exit can proceed now */
+    flusher = NULL;
+    global_heap_free(flush_threads, flush_num_threads*sizeof(thread_record_t*)
+                     HEAPACCT(ACCT_THREAD_MGT));
+    flush_threads = NULL;
+    if (!keep_initexit_lock)
+        mutex_unlock(&thread_initexit_lock);
+}
+
+/* This routine performs flush stages 1 and 2 (synch_unlink_priv()
+ * and unlink_shared()) and then returns after grabbing the
+ * executable_areas lock so that removal of this area from the global list
+ * is atomic w/ the flush and local removals, before letting the threads go
+ * -- we return while holding both the thread_initexit_lock and the exec
+ * areas lock
+ *
+ * FIXME: this only helps thread-shared: for thread-private, should let
+ * already-synched threads continue but not grab executable_areas lock, while
+ * letting to-be-synched threads grab the lock.
+ *
+ * returning while holding locks isn't ideal -- but other choices are worse:
+ * 1) grab exec areas lock and then let go of initexit lock -- bad nesting
+ * 2) do all work here, which has to include:
+ *    A) remove region (nearly all uses) == flush_fragments_and_remove_region
+ *    B) change region from rw to r (splitting if nec) (vmareas.c app rw->r)
+ *    C) restore writability and change to selfmod (splitting if nec)
+ *       (vmareas.c handle_modified_code)
+ *    D) remove region and then look up a certain pc and return whether
+ *       flushing overlapped w/ it (vmareas.c handle_modified_code)
+ */
+void
+flush_fragments_in_region_start(dcontext_t *dcontext, app_pc base, size_t size,
+                                bool own_initexit_lock, bool free_futures,
+                                bool exec_invalid, bool force_synchall
+                                _IF_DGCDIAG(app_pc written_pc))
+{
+    KSTART(flush_region);
+    while (true) {
+        if (flush_fragments_synch_unlink_priv(dcontext, base, size, own_initexit_lock,
+                                              exec_invalid, force_synchall
+                                              _IF_DGCDIAG(written_pc))) {
+            break;
+        } else {
+            /* grab lock and then re-check overlap */
+            executable_areas_lock();
+            if (!executable_vm_area_executed_from(base, base+size)) {
+                LOG(THREAD, LOG_FRAGMENT, 2,
+                    "\tregion not executable, so no fragments to flush\n");
+                /* caller will release lock! */
+                STATS_INC(num_noncode_flushes);
+                return;
+            }
+            /* unlock and try again */
+            executable_areas_unlock();
+        }
+    }
+
+    flush_fragments_unlink_shared(dcontext, base, size, NULL
+                                  _IF_DGCDIAG(written_pc));
+
+    /* We need to free the futures after all fragments have been unlinked */
+    if (free_futures) {
+        flush_fragments_free_futures(base, size);
+    }
+
+    executable_areas_lock();
+}
+
+/* must ONLY be called as the second half of flush_fragments_in_region_start().
+ * uses the shared variables flush_threads and flush_num_threads
+ */
+void
+flush_fragments_in_region_finish(dcontext_t *dcontext, bool keep_initexit_lock)
+{
+    /* done w/ exec areas lock; also free any non-executed coarse units */
+    free_nonexec_coarse_and_unlock();
+
+    flush_fragments_end_synch(dcontext, keep_initexit_lock);
+    KSTOP(flush_region);
+}
+
+/* flush and remove region from exec list, atomically */
+void
+flush_fragments_and_remove_region(dcontext_t *dcontext, app_pc base, size_t size,
+                                  bool own_initexit_lock, bool free_futures)
+{
+    flush_fragments_in_region_start(dcontext, base, size, own_initexit_lock,
+                                    free_futures, true/*exec invalid*/,
+                                    false/*don't force synchall*/ _IF_DGCDIAG(NULL));
+    /* ok to call on non-exec region, so don't need to test return value
+     * both flush routines will return quickly if nothing to flush/was flushed
+     */
+    remove_executable_region(base, size, true/*have lock*/);
+    flush_fragments_in_region_finish(dcontext, own_initexit_lock);
+
+    /* verify initexit lock is in the right state */
+    ASSERT_OWN_MUTEX(own_initexit_lock, &thread_initexit_lock);
+    ASSERT_DO_NOT_OWN_MUTEX(!own_initexit_lock, &thread_initexit_lock);
+}
+
+/* Flushes fragments from the region without any changes to the exec list.
+ * Does not free futures and caller can't be holding the initexit lock.
+ * FIXME - add argument paramters (free futures etc.) as needed. */
+void
+flush_fragments_from_region(dcontext_t *dcontext, app_pc base, size_t size,
+                            bool force_synchall)
+{
+    /* we pass false to flush_fragments_in_region_start() below for owning the initexit
+     * lock */
+    ASSERT_DO_NOT_OWN_MUTEX(true, &thread_initexit_lock);
+
+    /* ok to call on non-exec region, so don't need to test return value
+     * both flush routines will return quickly if nothing to flush/was flushed */
+    flush_fragments_in_region_start(dcontext, base, size, false /*don't own initexit*/,
+                                    false/*don't free futures*/, false/*exec valid*/,
+                                    force_synchall _IF_DGCDIAG(NULL));
+    flush_fragments_in_region_finish(dcontext, false);
+}
+
+/* Invalidate all fragments in all caches.  Currently executed
+ * fragments may be alive until they reach an exit.
+ *
+ * Note not necessarily immediately freeing memory like
+ * fcache_reset_all_caches_proactively().
+ */
+void
+invalidate_code_cache()
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    LOG(GLOBAL, LOG_FRAGMENT, 2, "invalidate_code_cache()\n");
+    flush_fragments_in_region_start(dcontext, UNIVERSAL_REGION_BASE,
+                                    UNIVERSAL_REGION_SIZE,
+                                    false, true /* remove futures */,
+                                    false /*exec valid*/, false /*don't force synchall*/
+                                    _IF_DGCDIAG(NULL));
+    flush_fragments_in_region_finish(dcontext, false);
+}
+
+/* Flushes all areas stored in the vector toflush.
+ * Synchronization of toflush is up to caller, but as locks cannot be
+ * held when flushing, toflush must be thread-private.
+ * Currently only used for pcache hotp interop (case 9970).
+ */
+void
+flush_vmvector_regions(dcontext_t *dcontext, vm_area_vector_t *toflush,
+                       bool free_futures, bool exec_invalid)
+{
+    vmvector_iterator_t vmvi;
+    app_pc start, end;
+    ASSERT(toflush != NULL && !TEST(VECTOR_SHARED, toflush->flags));
+    ASSERT(!RUNNING_WITHOUT_CODE_CACHE());
+    ASSERT(DYNAMO_OPTION(coarse_units) && DYNAMO_OPTION(use_persisted)
+           IF_HOTP(&& DYNAMO_OPTION(hot_patching)));
+    if (vmvector_empty(toflush))
+        return;
+    vmvector_iterator_start(toflush, &vmvi);
+    while (vmvector_iterator_hasnext(&vmvi)) {
+        vmvector_iterator_next(&vmvi, &start, &end);
+        /* FIXME case 10086: optimization: batch the flushes together so we only
+         * synch once.  Currently this is rarely used, and with few areas in the
+         * vector, so we don't bother.  To batch them we'd need to assume the
+         * worst and do a synchall up front, which could be more costly than 2
+         * or 3 separate flushes that do not involve coarse code.
+         */
+        ASSERT_OWN_NO_LOCKS();
+        flush_fragments_in_region_start(dcontext, start, end - start,
+                                        false/*no lock*/, free_futures, exec_invalid,
+                                        false/*don't force synchall*/ _IF_DGCDIAG(NULL));
+        flush_fragments_in_region_finish(dcontext, false/*no lock*/);
+        STATS_INC(num_flush_vmvector);
+    }
+    vmvector_iterator_stop(&vmvi);
+}
+
+/****************************************************************************/
+#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
+
+void
+fragment_output(dcontext_t *dcontext, fragment_t *f)
+{
+    ASSERT(!TEST(FRAG_SHARED, f->flags) ||
+           self_owns_recursive_lock(&change_linking_lock));
+    if (SHOULD_OUTPUT_FRAGMENT(f->flags)) {
+        per_thread_t *pt = (dcontext == GLOBAL_DCONTEXT) ? shared_pt :
+            (per_thread_t *) dcontext->fragment_field;
+        output_trace(dcontext, pt, f,
+                     IF_DEBUG_ELSE(GLOBAL_STAT(num_fragments),
+                                   GLOBAL_STAT(num_traces)+GLOBAL_STAT(num_bbs))+1);
+    }
+}
+
+void
+init_trace_file(per_thread_t *pt)
+{
+    if (INTERNAL_OPTION(tracedump_binary)) {
+        /* first 4 bytes in binary file gives size of linkcounts
+         * 0 if no linkcounts
+         */
+        tracedump_file_header_t hdr =
+            {CURRENT_API_VERSION, IF_X64_ELSE(true, false), 0 };
+#ifdef PROFILE_LINKCOUNT
+        if (dynamo_options.profile_counts) {
+            hdr.linkcount_size = sizeof(linkcount_type_t);
+            ASSERT_NOT_IMPLEMENTED(false == DYNAMO_OPTION(inline_trace_ibl));
+            /* Cannot use PROFILE_LINKCOUNT with inline_trace_ibl */
+        }
+#endif
+        os_write(pt->tracefile, &hdr, sizeof(hdr));
+    }
+}
+
+void
+exit_trace_file(per_thread_t *pt)
+{
+#ifdef PROFILE_LINKCOUNT
+    if (dynamo_options.tracedump_threshold > 0) {
+        if (INTERNAL_OPTION(tracedump_binary)) {
+            os_write(pt->tracefile, &pt->tracedump_num_below_threshold,
+                     sizeof(pt->tracedump_num_below_threshold));
+            os_write(pt->tracefile, &pt->tracedump_count_below_threshold,
+                     sizeof(pt->tracedump_count_below_threshold));
+        } else {
+            print_file(pt->tracefile, "\nTraces below dump threshold of %d: %d\n",
+                       dynamo_options.tracedump_threshold, pt->tracedump_num_below_threshold);
+            print_file(pt->tracefile, "Total count below dump threshold: "
+                       LINKCOUNT_FORMAT_STRING"\n", pt->tracedump_count_below_threshold);
+        }
+    }
+#endif
+    os_close(pt->tracefile);
+}
+
+/* Binary trace dump is used to save time and space.
+ * The format is in fragment.h.
+ * FIXME: add general symbol table support to disassembly?
+ *   We'd dump num_targets, then fcache_enter, ibl, trace_cache_incr addrs?
+ *   Reader would then add exit stubs & other traces to table?
+ *   But links will target cache pcs...
+ */
+
+#define TRACEBUF_SIZE 2048
+
+#define TRACEBUF_MAKE_ROOM(p, buf, sz) do {           \
+      if (p+sz >= &buf[TRACEBUF_SIZE]) {              \
+          os_write(pt->tracefile, buf, (p - buf));    \
+          p = buf;                                    \
+      }                                               \
+  } while (0)
+
+static void
+output_trace_binary(dcontext_t *dcontext, per_thread_t *pt, fragment_t *f,
+                    stats_int_t trace_num)
+{
+    /* FIXME:
+     * We do not support PROFILE_RDTSC or various small fields
+     */
+    /* FIXME: should allocate buffer elsewhere */
+    byte buf[TRACEBUF_SIZE];
+    byte *p = buf;
+    trace_only_t *t = TRACE_FIELDS(f);
+    linkstub_t *l;
+    tracedump_trace_header_t hdr = {
+        (int) trace_num, f->tag, f->start_pc, f->prefix_size, 0, f->size,
+        (INTERNAL_OPTION(tracedump_origins) ? t->num_bbs : 0),
+        IF_X64_ELSE(!TEST(FRAG_32_BIT, f->flags), false),
+    };
+    tracedump_stub_data_t stub;
+    /* Should we widen the identifier? */
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(trace_num)));
+
+    for (l = FRAGMENT_EXIT_STUBS(f); l != NULL; l = LINKSTUB_NEXT_EXIT(l))
+        hdr.num_exits++;
+
+    TRACEBUF_MAKE_ROOM(p, buf, sizeof(hdr));
+    *((tracedump_trace_header_t *)p) = hdr;
+    p += sizeof(hdr);
+
+    if (INTERNAL_OPTION(tracedump_origins)) {
+        uint i;
+        for (i=0; i<t->num_bbs; i++) {
+            instr_t *inst;
+            instrlist_t *ilist;
+            int size = 0;
+
+            TRACEBUF_MAKE_ROOM(p, buf, sizeof(app_pc));
+            *((app_pc *)p) = t->bbs[i].tag;
+            p += sizeof(app_pc);
+
+            /* we assume that the target is readable, since we dump prior
+             * to unloading of modules on flush events
+             */
+            ilist = build_app_bb_ilist(dcontext, t->bbs[i].tag, INVALID_FILE);
+            for (inst = instrlist_first(ilist); inst; inst = instr_get_next(inst)) {
+                size += instr_length(dcontext, inst);
+            }
+
+            TRACEBUF_MAKE_ROOM(p, buf, sizeof(int));
+            *((int *)p) = size;
+            p += sizeof(int);
+
+            for (inst = instrlist_first(ilist); inst; inst = instr_get_next(inst)) {
+                TRACEBUF_MAKE_ROOM(p, buf, instr_length(dcontext, inst));
+                /* PR 302353: we can't use instr_encode() as it will
+                 * try to re-relativize rip-rel instrs, which may fail
+                 */
+                ASSERT(instr_get_raw_bits(inst) != NULL);
+                memcpy(p, instr_get_raw_bits(inst), instr_length(dcontext, inst));
+                p += instr_length(dcontext, inst);
+            }
+            /* free the instrlist_t elements */
+            instrlist_clear_and_destroy(dcontext, ilist);
+        }
+    }
+
+    ASSERT(SEPARATE_STUB_MAX_SIZE == DIRECT_EXIT_STUB_SIZE(0));
+
+    for (l = FRAGMENT_EXIT_STUBS(f); l != NULL; l = LINKSTUB_NEXT_EXIT(l)) {
+        cache_pc stub_pc = EXIT_STUB_PC(dcontext, f, l);
+        stub.cti_offs = l->cti_offset;
+        stub.stub_pc = stub_pc;
+        stub.target = EXIT_TARGET_TAG(dcontext, f, l);
+        stub.linked = TEST(LINK_LINKED, l->flags);
+        stub.stub_size = DIRECT_EXIT_STUB_SIZE(f->flags);
+        ASSERT(DIRECT_EXIT_STUB_SIZE(f->flags) <= SEPARATE_STUB_MAX_SIZE);
+
+        TRACEBUF_MAKE_ROOM(p, buf, STUB_DATA_FIXED_SIZE);
+        memcpy(p, &stub, STUB_DATA_FIXED_SIZE);
+        p += STUB_DATA_FIXED_SIZE;
+
+#ifdef PROFILE_LINKCOUNT
+        if (dynamo_options.profile_counts) {
+            *((linkcount_type_t *)p) = l->count;
+            p += sizeof(linkcount_type_t);
+        }
+#endif
+        if (TEST(LINK_SEPARATE_STUB, l->flags)) {
+            TRACEBUF_MAKE_ROOM(p, buf, DIRECT_EXIT_STUB_SIZE(f->flags));
+            ASSERT(stub_pc < f->start_pc || stub_pc >= f->start_pc+f->size);
+            memcpy(p, stub_pc, DIRECT_EXIT_STUB_SIZE(f->flags));
+            p += DIRECT_EXIT_STUB_SIZE(f->flags);
+        } else { /* ensure client's method of identifying separate stubs works */
+            ASSERT(stub_pc == NULL /* no stub at all */ ||
+                   (stub_pc >= f->start_pc && stub_pc < f->start_pc+f->size));
+        }
+    }
+
+    if (f->size >= TRACEBUF_SIZE) {
+        os_write(pt->tracefile, buf, (p - buf));
+        p = buf;
+        os_write(pt->tracefile, f->start_pc, f->size);
+    } else {
+        /* single syscall for all but largest traces */
+        TRACEBUF_MAKE_ROOM(p, buf, f->size);
+        memcpy(p, f->start_pc, f->size);
+        p += f->size;
+        os_write(pt->tracefile, buf, (p - buf));
+        p = buf;
+    }
+}
+
+/* Output the contents of the specified trace.
+ * Does full disassembly of every instruction.
+ * If deleted_at != -1, it is taken to be the fragment id that
+ * caused the flushing of this fragment from the cache
+ * If f is shared, pt argument must be shared_pt, and caller must hold
+ * the change_linking_lock.
+ */
+static void
+output_trace(dcontext_t *dcontext, per_thread_t *pt, fragment_t *f,
+             stats_int_t deleted_at)
+{
+    trace_only_t *t = TRACE_FIELDS(f);
+#ifdef WINDOWS
+    char buf[MAXIMUM_PATH];
+#endif
+    stats_int_t trace_num;
+    IF_X64(bool old_mode;)
+    ASSERT(SHOULD_OUTPUT_FRAGMENT(f->flags));
+    ASSERT(TEST(FRAG_IS_TRACE, f->flags));
+    ASSERT(!TEST(FRAG_SELFMOD_SANDBOXED, f->flags)); /* no support for selfmod */
+    /* already been output?  caller should check this flag, just like trace flag */
+    ASSERT(!TEST(FRAG_TRACE_OUTPUT, f->flags));
+    ASSERT(!TEST(FRAG_SHARED, f->flags) ||
+           self_owns_recursive_lock(&change_linking_lock));
+    f->flags |= FRAG_TRACE_OUTPUT;
+
+#ifdef PROFILE_LINKCOUNT
+    if (dynamo_options.tracedump_threshold > 0) {
+        linkcount_type_t count = get_total_linkcount(f);
+        if (count < (linkcount_type_t) dynamo_options.tracedump_threshold) {
+            pt->tracedump_num_below_threshold++;
+            pt->tracedump_count_below_threshold += count;
+            return;
+        }
+    }
+#endif
+
+    LOG(THREAD, LOG_FRAGMENT, 4, "output_trace: F%d("PFX")\n", f->id, f->tag);
+    /* Recreate in same mode as original fragment */
+    IF_X64(old_mode = set_x86_mode(dcontext, FRAG_IS_32(f->flags)));
+
+    /* xref 8131/8202 if dynamo_resetting we don't need to grab the tracedump
+     * mutex to ensure we're the only writer and grabbing here on reset path
+     * can lead to a rank-order violation. */
+    if (!dynamo_resetting)
+        mutex_lock(&tracedump_mutex);
+    trace_num = tcount;
+    tcount++;
+    if (!TEST(FRAG_SHARED, f->flags)) {
+        /* No lock is needed because we use thread-private files.
+         * If dumping traces for a different thread (dynamo_other_thread_exit
+         * for ex.) caller is responsible for the neccesary synchronization. */
+        ASSERT(pt != shared_pt);
+        if (!dynamo_resetting)
+            mutex_unlock(&tracedump_mutex);
+    } else {
+        ASSERT(pt == shared_pt);
+    }
+
+    /* binary dump requested? */
+    if (INTERNAL_OPTION(tracedump_binary)) {
+        output_trace_binary(dcontext, pt, f, trace_num);
+        goto output_trace_done;
+    }
+
+    /* just origins => just bb tags in text */
+    if (INTERNAL_OPTION(tracedump_origins) &&
+        !INTERNAL_OPTION(tracedump_text)) {
+        uint i;
+        print_file(pt->tracefile, "Trace %d\n", tcount);
+#ifdef DEBUG
+        print_file(pt->tracefile, "Fragment %d\n", f->id);
+#endif
+        for (i = 0; i < t->num_bbs; i++) {
+            print_file(pt->tracefile, "\tbb %d = "PFX"\n", i, t->bbs[i].tag);
+        }
+        print_file(pt->tracefile, "\n");
+        goto output_trace_done;
+    }
+
+    /* full text dump */
+
+    print_file(pt->tracefile, "=============================================="
+               "=============================\n\n");
+    print_file(pt->tracefile, "TRACE # %d\n", tcount);
+#ifdef DEBUG
+    print_file(pt->tracefile, "Fragment # %d\n", f->id);
+#endif
+    print_file(pt->tracefile, "Tag = "PFX"\n", f->tag);
+    print_file(pt->tracefile, "Thread = %d\n", get_thread_id());
+    if (deleted_at > -1) {
+        print_file(pt->tracefile,
+                   "*** Flushed from cache when top fragment id was %d\n",
+                   deleted_at);
+    }
+    
+#ifdef WINDOWS
+    /* FIXME: for fragments flushed by unloaded modules, naturally we
+     * won't be able to get the module name b/c by now it is unloaded,
+     * the only fix is to keep a listing of mapped modules and only
+     * unmap in our listing at this point
+     */
+    get_module_name(f->tag, buf, sizeof(buf));
+    if (buf[0] != '\0')
+        print_file(pt->tracefile, "Module of basic block 0 = %s\n", buf);
+    else
+        print_file(pt->tracefile, "Module of basic block 0 = <unknown>\n");
+#endif
+
+#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
+    if (INTERNAL_OPTION(tracedump_origins)) {
+        uint i;
+        print_file(pt->tracefile, "\nORIGINAL CODE:\n");
+        for (i = 0; i < t->num_bbs; i++) {
+            /* we only care about the printing that the build routine does */
+            print_file(pt->tracefile, "basic block # %d: ", i);
+            /* we assume that the target is readable, since we dump prior
+             * to unloading of modules on flush events
+             */
+            ASSERT(is_readable_without_exception(t->bbs[i].tag, sizeof(t->bbs[i])));
+            disassemble_app_bb(dcontext, t->bbs[i].tag, pt->tracefile);
+        }
+        print_file(pt->tracefile, "END ORIGINAL CODE\n\n");
+    }
+#endif
+
+#ifdef PROFILE_RDTSC
+    if (dynamo_options.profile_times) {
+        /* Cycle counts are too high due to extra instructions needed
+         * to save regs and store time, so we subtract a constant for
+         * each fragment entry.
+         * Experiments using large-iter loops show that:
+         *   empty loop overhead = 2 cycles
+         *   2 pushes, 2 moves, 2 pops = 7 cycles - 2 cycles = 5 cycles
+         *   add in 1 rdtsc = 38 cycles - 2 cycles = 36 cycles
+         *   add in 2 rdtsc = 69 cycles - 2 cycles = 67 cycles
+         * If we assume the time transfer happens in the middle of the rdtsc, we
+         * should use 36 cycles.
+         */
+        trace_only_t *tr = TRACE_FIELDS(f);
+        const int adjustment = 37;
+        uint64 real_time = tr->total_time;
+        uint64 temp;
+        uint time_top, time_bottom;
+        print_file(pt->tracefile, "Size = %d (+ %d for profiling)\n",
+                   f->size-profile_call_size(), profile_call_size());
+        print_file(pt->tracefile, "Profiling:\n");
+        print_file(pt->tracefile, "\tcount = "UINT64_FORMAT_STRING"\n", tr->count);
+        print_file(pt->tracefile, "\tmeasured cycles = "PFX"\n", real_time);
+        temp = tr->count * (uint64)adjustment;
+        if (real_time < temp) {
+            print_file(pt->tracefile,
+            "\t  ERROR: adjustment too large, cutting off at 0, should use < %d\n",
+                     (int)(real_time / tr->count));
+            real_time = 0;
+        } else {
+            real_time -= temp;
+        }
+        print_file(pt->tracefile, "\tadjusted cycles = "PFX"\n", real_time);
+        divide_uint64_print(real_time, kilo_hertz, false, 6, 
+                            &time_top, &time_bottom);
+        print_file(pt->tracefile, "\ttime  = %u.%.6u ms\n",
+                   time_top, time_bottom);
+    } else {
+        print_file(pt->tracefile, "Size = %d\n", f->size);
+    }
+#else
+    print_file(pt->tracefile, "Size = %d\n", f->size);
+#endif /* PROFILE_RDTSC */
+
+#ifdef PROFILE_LINKCOUNT
+    if (dynamo_options.profile_counts) {
+        int num_exits = 0;
+        linkstub_t *ls;
+        print_file(pt->tracefile, "Exit stubs:\n");
+        for (ls = FRAGMENT_EXIT_STUBS(f); ls; ls = LINKSTUB_NEXT_EXIT(ls)) {
+            int id = -1;
+            app_pc target;
+            if ((ls->flags & LINK_INDIRECT) != 0) {
+                target = 0;
+                id = -1;
+            } else {
+# ifdef DEBUG
+                fragment_t *targetf = fragment_lookup(dcontext,
+                                                    EXIT_TARGET_TAG(dcontext, f, ls));
+                id = (targetf != NULL) ? targetf->id : -1;
+# else
+                id = -1;
+# endif
+                target = EXIT_TARGET_TAG(dcontext, f, ls);
+            }
+            print_file(pt->tracefile,
+                       "\t#%d: target = "PFX" (F#%d), count = "
+                       LINKCOUNT_FORMAT_STRING "%s\n",
+                       num_exits, target, id, ls->count,
+                       ((ls->flags & LINK_LINKED) != 0) ? ", linked" : "");
+            num_exits++;
+        }
+# ifdef SIDELINE_COUNT_STUDY
+        if (dynamo_options.sideline && t->count_old_pre > (linkcount_type_t)0) {
+            print_file(pt->tracefile, "Sideline: pre-opt count = "
+                       LINKCOUNT_FORMAT_STRING "\n",
+                       t->count_old_pre);
+            print_file(pt->tracefile, "\tpost-opt count = "
+                       LINKCOUNT_FORMAT_STRING "\n",
+                       t->count_old_post);
+            print_file(pt->tracefile, "\tnew trace count = "
+                       LINKCOUNT_FORMAT_STRING "\n",
+                       get_total_linkcount(f));
+        }
+# endif
+    }
+#endif /* PROFILE_LINKCOUNT */
+
+#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
+    print_file(pt->tracefile, "Body:\n");
+    disassemble_fragment_body(dcontext, f, pt->tracefile);
+
+    print_file(pt->tracefile, "END TRACE %d\n\n", tcount);
+#endif
+
+ output_trace_done:
+    IF_X64(set_x86_mode(dcontext, old_mode));
+    if (TEST(FRAG_SHARED, f->flags) && !dynamo_resetting) {
+        ASSERT_OWN_MUTEX(true, &tracedump_mutex);
+        mutex_unlock(&tracedump_mutex);
+    } else {
+        ASSERT_DO_NOT_OWN_MUTEX(true, &tracedump_mutex);
+    }
+}
+
+#endif /* defined(INTERNAL) || defined(CLIENT_INTERFACE) */
+
+/****************************************************************************/
+
+#ifdef PROFILE_RDTSC
+/* This routine is called at the start of every trace
+ * it assumes the caller has saved the caller-saved registers
+ * (eax, ecx, edx)
+ *
+ * See insert_profile_call() for the assembly code prefix that calls
+ * this routine.   The end time is computed in the assembly code and passed 
+ * in to have as accurate times as possible (don't want the profiling overhead
+ * added into the times).  Also, the assembly code computes the new start
+ * time after this routine returns.  
+ *
+ * We could generate a custom copy of this routine
+ * for every thread, but we don't do that now -- don't need to.
+ */
+void
+profile_fragment_enter(fragment_t *f, uint64 end_time)
+{
+    dcontext_t *dcontext;
+#ifdef WINDOWS
+    /* must get last error prior to getting dcontext */
+    int error_code = get_last_error();
+#endif
+    trace_only_t *t = TRACE_FIELDS(f);
+
+    /* this is done in assembly:   uint64 end_time = get_time() */
+    dcontext = get_thread_private_dcontext();
+
+    /* increment this fragment's execution count */
+    t->count++;
+
+    /***********************************************************************/
+    /* all-in-cache sequence profiling */
+
+    /* top ten cache times */
+    dcontext->cache_frag_count++;
+
+    /***********************************************************************/
+
+    /* we rely on dispatch being the only way to enter the fcache.
+     * dispatch sets prev_fragment to null prior to entry */
+    if (dcontext->prev_fragment != NULL) {
+        trace_only_t *last_t = TRACE_FIELDS(dcontext->prev_fragment);
+        ASSERT((dcontext->prev_fragment->flags & FRAG_IS_TRACE) != 0);
+        /* charge time to last fragment */
+        last_t->total_time += (end_time - dcontext->start_time);
+    }
+
+    /* set up for next fragment */
+    dcontext->prev_fragment = f;
+    /* this is done in assembly:   dcontext->start_time = get_time() */
+
+#ifdef WINDOWS
+    /* retore app's error code */
+    set_last_error(error_code);
+#endif
+}
+
+/* this routine is called from dispatch after exiting the fcache
+ * it finishes up the final fragment's time slot
+ */
+void
+profile_fragment_dispatch(dcontext_t *dcontext)
+{
+    /* end time slot ASAP, should we try to move this to assembly routines exiting
+     * the cache?  probably not worth it, in a long-running program shouldn't
+     * be exiting the cache that often */
+    uint64 end_time = get_time();
+    bool tagtable = LINKSTUB_INDIRECT(dcontext->last_exit->flags);
+    if (dcontext->prev_fragment != NULL && (dcontext->prev_fragment->flags & FRAG_IS_TRACE) != 0) {
+        /* end time slot, charge time to last fragment
+         * there's more overhead here than other time endings, so subtract
+         * some time off.  these numbers are pretty arbitrary:
+         */
+        trace_only_t *last_t = TRACE_FIELDS(dcontext->prev_fragment);
+        const uint64 adjust = (tagtable) ? 72 : 36;
+        uint64 add = end_time - dcontext->start_time;
+        if (add < adjust) {
+            SYSLOG_INTERNAL_ERROR("ERROR: profile_fragment_dispatch: add was %d, tagtable %d",
+                                  (int)add, tagtable);
+            add = 0;
+        } else
+            add -= adjust;
+        ASSERT((dcontext->prev_fragment->flags & FRAG_IS_TRACE) != 0);
+        last_t->total_time += add;
+    }
+}
+#endif /* PROFILE_RDTSC */
+
+
+/*******************************************************************************
+ * COARSE-GRAIN FRAGMENT HASHTABLE INSTANTIATION
+ */
+
+/* Synch model: the htable lock should be held during all
+ * app_to_cache_t manipulations, to be consistent (not strictly
+ * necessary as there are no removals from the htable except in
+ * all-thread-synch flushes).  Copies of a looked-up cache_pc are safe
+ * to use w/o the htable lock, or any lock -- they're pointers into
+ * cache units, and there are no deletions of cache units, except in
+ * all-thread-synch flushes.
+ */
+
+/* 2 macros w/ name and types are duplicated in fragment.h -- keep in sync */
+#define NAME_KEY coarse
+#define ENTRY_TYPE app_to_cache_t
+static app_to_cache_t a2c_empty = { NULL, NULL };
+static app_to_cache_t a2c_sentinel = { /*assume invalid*/(app_pc)PTR_UINT_MINUS_1,
+                                       NULL };
+/* FIXME: want to inline the app_to_cache_t struct just like lookuptable
+ * does and use it for main table -- no support for that right now
+ */
+/* not defining HASHTABLE_USE_LOOKUPTABLE */
+#define ENTRY_TAG(f)              ((ptr_uint_t)(f).app)
+#define ENTRY_EMPTY               (a2c_empty)
+#define ENTRY_SENTINEL            (a2c_sentinel)
+#define ENTRY_IS_EMPTY(f)         ((f).app == a2c_empty.app)
+#define ENTRY_IS_SENTINEL(f)      ((f).app == a2c_sentinel.app)
+#define ENTRY_IS_INVALID(f)       (false) /* no invalid entries */
+#define ENTRIES_ARE_EQUAL(f,g)    ((f).app == (g).app)
+#define HASHTABLE_WHICH_HEAP(flags) (ACCT_FRAG_TABLE)
+/* note that we give the th table a lower-ranked coarse_th_htable_rwlock */
+#define COARSE_HTLOCK_RANK        coarse_table_rwlock /* for use after hashtablex.h */
+#define HTLOCK_RANK               COARSE_HTLOCK_RANK
+#define HASHTABLE_SUPPORT_PERSISTENCE 1
+
+#include "hashtablex.h"
+/* All defines are undef-ed at end of hashtablex.h
+ * Would be nice to re-use ENTRY_IS_EMPTY, etc., though w/ multiple htables
+ * in same file can't realistically get away w/o custom defines like these:
+ */
+#define A2C_ENTRY_IS_EMPTY(a2c) ((a2c).app == NULL)
+#define A2C_ENTRY_IS_SENTINEL(a2c) ((a2c).app == a2c_sentinel.app)
+#define A2C_ENTRY_IS_REAL(a2c) (!A2C_ENTRY_IS_EMPTY(a2c) && !A2C_ENTRY_IS_SENTINEL(a2c))
+
+/* required routines for hashtable interface that we don't need for this instance */
+
+static void
+hashtable_coarse_init_internal_custom(dcontext_t *dcontext, coarse_table_t *htable)
+{ /* nothing */
+}
+
+static void
+hashtable_coarse_resized_custom(dcontext_t *dcontext, coarse_table_t *htable,
+                                uint old_capacity, app_to_cache_t *old_table,
+                                app_to_cache_t *old_table_unaligned,
+                                uint old_ref_count, uint old_table_flags)
+{ /* nothing */ 
+}
+
+# ifdef DEBUG
+static void
+hashtable_coarse_study_custom(dcontext_t *dcontext, coarse_table_t *htable, 
+                              uint entries_inc/*amnt table->entries was pre-inced*/)
+{ /* nothing */ 
+}
+# endif
+
+static void
+hashtable_coarse_free_entry(dcontext_t *dcontext, coarse_table_t *htable,
+                            app_to_cache_t entry)
+{
+    /* nothing to do, data is inlined */
+}
+
+/* Pass 0 for the initial capacity to use the default.
+ * Initial capacities are number of entires and NOT bits in mask or anything.
+ */
+void
+fragment_coarse_htable_create(coarse_info_t *info, uint init_capacity,
+                              uint init_th_capacity)
+{
+    coarse_table_t *th_htable;
+    coarse_table_t *htable;
+    uint init_size;
+    ASSERT(SHARED_FRAGMENTS_ENABLED());
+    
+    /* Case 9537: If we start the new table small and grow it we have large
+     * collision chains as we map the lower address space of a large table into
+     * the same lower fraction of a smaller table, so we create our table fully
+     * sized up front.
+     */
+    if (init_capacity != 0) {
+        init_size = hashtable_bits_given_entries(init_capacity,
+                                                 DYNAMO_OPTION(coarse_htable_load));
+    } else
+        init_size = INIT_HTABLE_SIZE_COARSE;
+    LOG(GLOBAL, LOG_FRAGMENT, 2, "Coarse %s htable %d capacity => %d bits\n",
+        info->module, init_capacity, init_size);
+    htable = NONPERSISTENT_HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, coarse_table_t,
+                                           ACCT_FRAG_TABLE);
+    hashtable_coarse_init(GLOBAL_DCONTEXT, htable, init_size,
+                          DYNAMO_OPTION(coarse_htable_load),
+                          (hash_function_t)INTERNAL_OPTION(alt_hash_func),
+                          0 /* hash_mask_offset */,
+                          HASHTABLE_ENTRY_SHARED | HASHTABLE_SHARED |
+                          HASHTABLE_RELAX_CLUSTER_CHECKS
+                          NAME("coarse htable"));
+    info->htable = (void *) htable;
+
+    /* We could create th_htable lazily independently of htable but not worth it */
+    if (init_th_capacity != 0) {
+        init_size = hashtable_bits_given_entries(init_th_capacity,
+                                                 DYNAMO_OPTION(coarse_th_htable_load));
+    } else
+        init_size = INIT_HTABLE_SIZE_COARSE_TH;
+    LOG(GLOBAL, LOG_FRAGMENT, 2, "Coarse %s th htable %d capacity => %d bits\n",
+        info->module, init_th_capacity, init_size);
+    th_htable = NONPERSISTENT_HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, coarse_table_t,
+                                              ACCT_FRAG_TABLE);
+    hashtable_coarse_init(GLOBAL_DCONTEXT, th_htable, init_size,
+                          DYNAMO_OPTION(coarse_th_htable_load),
+                          (hash_function_t)INTERNAL_OPTION(alt_hash_func),
+                          0 /* hash_mask_offset */,
+                          HASHTABLE_ENTRY_SHARED | HASHTABLE_SHARED |
+                          HASHTABLE_RELAX_CLUSTER_CHECKS
+                          NAME("coarse th htable"));
+    /* We give th table a lower lock rank for coarse_body_from_htable_entry().
+     * FIXME: add param to init() that takes in lock rank?
+     */
+    ASSIGN_INIT_READWRITE_LOCK_FREE(th_htable->rwlock, coarse_th_table_rwlock);
+    info->th_htable = (void *) th_htable;
+}
+
+/* Adds all entries from stable into dtable, offsetting by dst_cache_offset,
+ * which is the offset from dst->cache_start_pc at which
+ * the src cache has been placed.
+ */
+static void
+fragment_coarse_htable_merge_helper(dcontext_t *dcontext,
+                                    coarse_info_t *dst, coarse_table_t *dtable,
+                                    coarse_info_t *src, coarse_table_t *stable,
+                                    ssize_t dst_cache_offset)
+{
+    app_to_cache_t a2c, look_a2c;
+    uint i;
+    /* assumption: dtable is private to this thread and so does not need synch */
+    DODEBUG({ dtable->is_local = true; });
+    TABLE_RWLOCK(stable, read, lock);
+    for (i = 0; i < stable->capacity; i++) {
+        a2c = stable->table[i];
+        if (A2C_ENTRY_IS_REAL(a2c)) {
+            look_a2c = hashtable_coarse_lookup(dcontext, (ptr_uint_t)a2c.app, dtable);
+            if (A2C_ENTRY_IS_EMPTY(look_a2c)) {
+                a2c.cache += dst_cache_offset;
+                if (!dst->frozen) { /* adjust absolute value */
+                    ASSERT_NOT_TESTED();
+                    a2c.cache += (dst->cache_start_pc - src->cache_start_pc);
+                }
+                hashtable_coarse_add(dcontext, a2c, dtable);
+            } else {
+                /* Our merging-with-dups strategy requires that we not
+                 * merge them in this early
+                 */
+                ASSERT_NOT_REACHED();
+            }
+        }
+    }
+    TABLE_RWLOCK(stable, read, unlock);
+    DODEBUG({ dtable->is_local = false; });
+}
+
+/* Merges the main and th htables from info1 and info2 into new htables for dst.
+ * If !add_info2, makes room for but does not add entries from info2.
+ * If !add_th_htable, creates but does not add entries to dst->th_htable.
+ * FIXME: we can't use hashtable_coarse_merge() b/c we don't want to add
+ * entries from the 2nd table, and we do custom mangling of entries when adding.
+ * Is it worth parametrizing hashtable_coarse_merge() to share anything?
+ */
+void
+fragment_coarse_htable_merge(dcontext_t *dcontext, coarse_info_t *dst,
+                             coarse_info_t *info1, coarse_info_t *info2,
+                             bool add_info2, bool add_th_htable)
+{
+    coarse_table_t *thht_dst, *thht1, *thht2;
+    coarse_table_t *ht_dst, *ht1, *ht2;
+    uint merged_entries = 0;
+
+    ASSERT(SHARED_FRAGMENTS_ENABLED());
+    ASSERT(info1 != NULL && info2 != NULL);
+    ht1 = (coarse_table_t *) info1->htable;
+    ht2 = (coarse_table_t *) info2->htable;
+    thht1 = (coarse_table_t *) info1->th_htable;
+    thht2 = (coarse_table_t *) info2->th_htable;
+    ASSERT(dst != NULL && dst->htable == NULL && dst->th_htable == NULL);
+
+    /* We go to the trouble of determining non-dup total entries to
+     * avoid repeatedly increasing htable size on merges and hitting
+     * collision asserts.  FIXME: should we shrink afterward instead?
+     * Or just ignore collision asserts until at full size?
+     */
+    merged_entries = hashtable_coarse_num_unique_entries(dcontext, ht1, ht2);
+    STATS_ADD(coarse_merge_dups, ht1->entries + ht2->entries - merged_entries);
+    LOG(THREAD, LOG_FRAGMENT, 2, "Merging %s: %d + %d => %d (%d unique) entries\n",
+        info1->module, ht1->entries, ht2->entries, ht1->entries + ht2->entries,
+        merged_entries);
+
+    /* We could instead copy ht1 and then add ht2; for simplicity we re-use
+     * our create-empty routine and add both
+     */
+    fragment_coarse_htable_create(dst, merged_entries,
+                                  /* Heuristic to never over-size, yet try to avoid
+                                   * collision asserts while resizing the table;
+                                   * FIXME: if we shrink the main htable afterward
+                                   * could do the same here and start at sum of
+                                   * entries */
+                                  MAX(thht1->entries, thht2->entries));
+    ht_dst = (coarse_table_t *) dst->htable;
+    thht_dst = (coarse_table_t *) dst->th_htable;
+    ASSERT(ht_dst != NULL && thht_dst != NULL);
+
+    /* For now we only support frozen tables; else will have to change
+     * the offsets to be stubs for main table and cache for th table
+     */
+    ASSERT(info1->frozen && info2->frozen);
+    fragment_coarse_htable_merge_helper(dcontext, dst, ht_dst, info1, ht1, 0);
+    if (add_info2) {
+        fragment_coarse_htable_merge_helper(dcontext, dst, ht_dst, info2, ht2,
+                                            info1->cache_end_pc -
+                                            info1->cache_start_pc);
+    }
+    if (add_th_htable) {
+        ASSERT_NOT_TESTED();
+        fragment_coarse_htable_merge_helper(dcontext, dst, thht_dst, info1, thht1, 0);
+        fragment_coarse_htable_merge_helper(dcontext, dst, thht_dst, info2, thht2,
+                                            /* stubs_end_pc is not allocated end */
+                                            info1->mmap_pc + info1->mmap_size -
+                                            info1->stubs_start_pc);
+    }
+}
+
+#ifndef DEBUG
+/* not in header file unless debug, in which case it's static */
+static void
+coarse_body_from_htable_entry(dcontext_t *dcontext, coarse_info_t *info,
+                              app_pc tag, cache_pc res,
+                              cache_pc *stub_pc_out/*OUT*/,
+                              cache_pc *body_pc_out/*OUT*/);
+#endif
+
+static void
+study_and_free_coarse_htable(coarse_info_t *info, coarse_table_t *htable,
+                             bool never_persisted _IF_DEBUG(char *name))
+{
+    LOG(GLOBAL, LOG_FRAGMENT, 1, "Coarse %s %s hashtable stats:\n",
+        info->module, name);
+    DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
+        hashtable_coarse_load_statistics(GLOBAL_DCONTEXT, htable);
+    });
+    DODEBUG({
+        hashtable_coarse_study(GLOBAL_DCONTEXT, htable, 0/*table consistent*/);
+    });
+    DOLOG(3, LOG_FRAGMENT, {
+        hashtable_coarse_dump_table(GLOBAL_DCONTEXT, htable);
+    });
+#ifdef CLIENT_INTERFACE
+    /* Only raise deletion events if client saw creation events: so no persisted
+     * units
+     */
+    if (!info->persisted && htable == info->htable &&
+        dr_fragment_deleted_hook_exists()) {
+        app_to_cache_t a2c;
+        uint i;
+        dcontext_t *dcontext = get_thread_private_dcontext();
+        cache_pc body = NULL;
+        TABLE_RWLOCK(htable, read, lock);
+        for (i = 0; i < htable->capacity; i++) {
+            a2c = htable->table[i];
+            if (A2C_ENTRY_IS_REAL(a2c)) {
+                if (info->frozen)
+                    body = a2c.cache;
+                else { /* make sure not an entrance stub w/ no body */
+                    coarse_body_from_htable_entry(dcontext, info, a2c.app, a2c.cache,
+                                                  NULL, &body);
+                }
+                if (body != NULL) {
+                    instrument_fragment_deleted(get_thread_private_dcontext(), a2c.app,
+                                                FRAGMENT_COARSE_WRAPPER_FLAGS);
+                }
+            }
+        }
+        TABLE_RWLOCK(htable, read, unlock);
+    }
+#endif
+    /* entries are inlined so nothing external to free */
+    if (info->persisted && !never_persisted) {
+        /* ensure won't try to free (part of mmap) */
+        ASSERT(htable->table_unaligned == NULL);
+    }
+    hashtable_coarse_free(GLOBAL_DCONTEXT, htable);
+    NONPERSISTENT_HEAP_TYPE_FREE(GLOBAL_DCONTEXT, htable, coarse_table_t, ACCT_FRAG_TABLE);
+}
+
+void
+fragment_coarse_htable_free(coarse_info_t *info)
+{
+    ASSERT_OWN_MUTEX(!info->is_local, &info->lock);
+    if (info->htable == NULL) {
+        /* lazily initialized, so common to have empty units */
+        ASSERT(info->th_htable == NULL);
+        ASSERT(info->pclookup_htable == NULL);
+        return;
+    }
+    study_and_free_coarse_htable(info, (coarse_table_t *) info->htable, false
+                                 _IF_DEBUG("main"));
+    info->htable = NULL;
+    study_and_free_coarse_htable(info, (coarse_table_t *) info->th_htable, false
+                                 _IF_DEBUG("tracehead"));
+    info->th_htable = NULL;
+    if (info->pclookup_htable != NULL) {
+        ASSERT(DYNAMO_OPTION(coarse_pclookup_table));
+        study_and_free_coarse_htable(info, (coarse_table_t *) info->pclookup_htable,
+                                     true/*never persisted*/ _IF_DEBUG("pclookup"));
+        info->pclookup_htable = NULL;
+    }
+}
+
+uint
+fragment_coarse_num_entries(coarse_info_t *info)
+{
+    coarse_table_t *htable;
+    ASSERT(info != NULL);
+    htable = (coarse_table_t *) info->htable;
+    if (htable == NULL)
+        return 0;
+    return htable->entries;
+}
+
+/* Add coarse fragment represented by wrapper f to the hashtable
+ * for unit info.
+ */
+void
+fragment_coarse_add(dcontext_t *dcontext, coarse_info_t *info,
+                    app_pc tag, cache_pc cache)
+{
+    coarse_table_t *htable;
+    app_to_cache_t a2c = {tag, cache};
+
+    ASSERT(info != NULL && info->htable != NULL);
+    htable = (coarse_table_t *) info->htable;
+
+    DODEBUG({
+        /* We have lock rank order problems b/c this lookup may acquire
+         * the th table read lock, and we can't split its rank from
+         * the main table's.  So we live w/ a racy assert that could
+         * have false positives or negatives.
+         */
+        cache_pc stub;
+        cache_pc body;
+        /* OK to have dup entries for entrance stubs in other units,
+         * or a stub already present for a trace head */
+        fragment_coarse_lookup_in_unit(dcontext, info, tag, &stub, &body);
+        ASSERT(body == NULL);
+        ASSERT(stub == NULL ||
+               coarse_is_trace_head_in_own_unit(dcontext, tag, stub,
+                                                /* case 10876: pass what we're adding */
+                                                (ptr_uint_t)cache + info->cache_start_pc,
+                                                true, info));
+        /* There can only be one body.  But similarly to above,
+         * fragment_coarse_lookup may look in the secondary unit, so
+         * we can't do this lookup holding the write lock.
+         */
+        if (!coarse_is_entrance_stub(cache)) {
+            coarse_info_t *xinfo = get_executable_area_coarse_info(tag);
+            fragment_t *f;
+            ASSERT(xinfo != NULL);
+            /* If an official unit, tag should not exist in any other official unit */
+            ASSERT((info != xinfo && info != xinfo->non_frozen) ||
+                   fragment_coarse_lookup(dcontext, tag) == NULL);
+            f = fragment_lookup(dcontext, tag);
+            /* ok for trace to shadow coarse trace head */
+            ASSERT(f == NULL || TEST(FRAG_IS_TRACE, f->flags));
+        }
+    });
+    TABLE_RWLOCK(htable, write, lock);
+    /* Table is not an ibl table so we ignore resize return value */
+    hashtable_coarse_add(dcontext, a2c, htable);
+    TABLE_RWLOCK(htable, write, unlock);
+
+#ifdef SHARING_STUDY
+    if (INTERNAL_OPTION(fragment_sharing_study)) {
+        ASSERT_NOT_IMPLEMENTED(false && "need to pass f in to add_shared_block");
+    }
+#endif
+}
+
+/* Returns the body pc of the coarse trace head fragment corresponding to tag, or
+ * NULL if not found.  Caller must hold the th table's read or write lock!
+ */
+static cache_pc
+fragment_coarse_th_lookup(dcontext_t *dcontext, coarse_info_t *info, app_pc tag)
+{
+    cache_pc res = NULL;
+    app_to_cache_t a2c;
+    coarse_table_t *htable;
+    ASSERT(info != NULL);
+    ASSERT(info->htable != NULL);
+    htable = (coarse_table_t *) info->th_htable;
+    ASSERT(TABLE_PROTECTED(htable));
+    a2c = hashtable_coarse_lookup(dcontext, (ptr_uint_t)tag, htable);
+    if (!A2C_ENTRY_IS_EMPTY(a2c)) {
+        ASSERT(BOOLS_MATCH(info->frozen, info->stubs_start_pc != NULL));
+        /* for frozen, th_htable only holds stubs */
+        res = ((ptr_uint_t)a2c.cache) + info->stubs_start_pc;
+    }
+    return res;
+}
+
+/* Performs two actions while holding the trace head table's write lock,
+ * making them atomic (solving the race in case 8795):
+ * 1) unlinks the coarse fragment's entrance pc and points it at the
+ *    trace head exit routine;
+ * 2) adds the coarse fragment's body pc to the trace head hashtable.
+ *
+ * Case 8628: if we split the main htable into stubs and bodies then
+ * we can eliminate the th htable as it will be part of the body table.
+ *
+ * We could merge this info in to the thcounter table, and assume that
+ * most trace heads are coarse so we're not wasting much memory with
+ * the extra field.  But then we have to walk entire stub table on
+ * cache flush and individually clear body pcs from monitor table, so
+ * better to have own th table?
+ */
+void
+fragment_coarse_th_unlink_and_add(dcontext_t *dcontext, app_pc tag,
+                                  cache_pc stub_pc, cache_pc body_pc)
+{
+    ASSERT(stub_pc != NULL);
+    if (body_pc != NULL) {
+        /* trace head is in this unit, so we have to add it to our th htable */
+        coarse_info_t *info = get_fcache_coarse_info(body_pc);
+        coarse_table_t *th_htable;
+        app_to_cache_t a2c = {tag, body_pc};
+        ASSERT(info != NULL && info->th_htable != NULL);
+        ASSERT(!info->frozen);
+        th_htable = (coarse_table_t *) info->th_htable;
+        TABLE_RWLOCK(th_htable, write, lock);
+        ASSERT(fragment_coarse_th_lookup(dcontext, info, tag) == NULL);
+        unlink_entrance_stub(dcontext, stub_pc, FRAG_IS_TRACE_HEAD, info);
+        /* Table is not an ibl table so we ignore resize return value */
+        hashtable_coarse_add(dcontext, a2c, th_htable);
+        TABLE_RWLOCK(th_htable, write, unlock);
+        LOG(THREAD, LOG_FRAGMENT, 4,
+            "adding to coarse th table for %s: "PFX"->"PFX"\n",
+            info->module, tag, body_pc);
+    } else {
+        /* ensure lives in another unit */
+        ASSERT(fragment_coarse_lookup(dcontext, tag) != stub_pc);
+        unlink_entrance_stub(dcontext, stub_pc, FRAG_IS_TRACE_HEAD, NULL);
+    }
+}
+
+/* Only use when building up a brand-new table.  Otherwise use
+ * fragment_coarse_th_unlink_and_add().
+ */
+void
+fragment_coarse_th_add(dcontext_t *dcontext, coarse_info_t *info,
+                       app_pc tag, cache_pc cache)
+{
+    coarse_table_t *th_htable;
+    app_to_cache_t a2c = {tag, cache};
+    ASSERT(info != NULL && info->th_htable != NULL);
+    ASSERT(info->frozen); /* only used when merging units */
+    th_htable = (coarse_table_t *) info->th_htable;
+    TABLE_RWLOCK(th_htable, write, lock);
+    ASSERT(fragment_coarse_th_lookup(dcontext, info, tag) == NULL);
+    /* Table is not an ibl table so we ignore resize return value */
+    hashtable_coarse_add(dcontext, a2c, th_htable);
+    TABLE_RWLOCK(th_htable, write, unlock);
+}
+
+/* The input here is the result of a lookup in the main htable.
+ * For a frozen unit, this actually looks up the stub pc since res is
+ * always the body pc.
+ * For a non-frozen unit this determines where to obtain the body pc.
+ * FIXME: case 8628 will simplify this whole thing
+ */
+/* exported only for assert in push_pending_freeze() */
+IF_DEBUG_ELSE(,static) void
+coarse_body_from_htable_entry(dcontext_t *dcontext, coarse_info_t *info,
+                              app_pc tag, cache_pc res,
+                              cache_pc *stub_pc_out/*OUT*/,
+                              cache_pc *body_pc_out/*OUT*/)
+{
+    cache_pc stub_pc = NULL, body_pc = NULL;
+    /* Should be passing absolute pc, not offset */
+    ASSERT(!info->frozen || res == NULL || res >= info->cache_start_pc);
+    if (info->frozen) {
+        /* We still need the stub table for lazy linking and for
+           shifting links from a trace head to a trace.
+         */
+        body_pc = res;
+        if (stub_pc_out != NULL) {
+            TABLE_RWLOCK((coarse_table_t *) info->th_htable, read, lock);
+            stub_pc = fragment_coarse_th_lookup(dcontext, info, tag);
+            TABLE_RWLOCK((coarse_table_t *) info->th_htable, read, unlock);
+        }
+    } else {
+        /* In a non-frozen unit, htable entries are always stubs */
+        DOCHECK(1, { ASSERT(coarse_is_entrance_stub(res)); });
+        stub_pc = res;
+        if (body_pc_out != NULL) {
+            /* keep the th table entry and stub link status linked atomically */
+            TABLE_RWLOCK((coarse_table_t *) info->th_htable, read, lock);
+            body_pc = fragment_coarse_th_lookup(dcontext, info, tag);
+            if (body_pc != NULL) {
+                /* We can't just check coarse_is_trace_head(res) because a
+                 * shadowed trace head is directly linked to its trace.  The
+                 * trace has lookup order precedence, but the head must show up
+                 * if asked about here!
+                 */
+                /* FIXME: we could have a flags OUT param and set FRAG_IS_TRACE_HEAD
+                 * to help out fragment_lookup_fine_and_coarse*().
+                 */
+            } else {
+                if (entrance_stub_linked(res, info)) {
+                    /* We only want to set body_pc if it is present in this unit */
+                    cache_pc tgt = entrance_stub_jmp_target(res);
+                    if (get_fcache_coarse_info(tgt) == info)
+                        body_pc = tgt;
+                    else
+                        body_pc = NULL;
+                } else
+                    body_pc = NULL;
+                DOCHECK(1, {
+                    ASSERT(!coarse_is_trace_head(res) ||
+                           /* allow targeting trace head in another unit */
+                           body_pc == NULL);
+                });
+            }
+            TABLE_RWLOCK((coarse_table_t *) info->th_htable, read, unlock);
+        }
+    }
+    if (stub_pc_out != NULL)
+        *stub_pc_out = stub_pc;
+    if (body_pc_out != NULL)
+        *body_pc_out = body_pc;
+}
+
+/* Coarse fragments have two entrance points: the actual fragment
+ * body, and the entrance stub that is used for indirection and
+ * convenient incremental building for intra-unit non-frozen linking
+ * as well as always used at the source end for inter-unit linking.
+ * This routine returns both.  If the stub_pc returned is non-NULL, it
+ * only indicates that there is an outgoing link from a fragment
+ * present in this unit that targets the queried tag (and for
+ * intra-unit links in a frozen unit, such a link may exist but
+ * stub_pc will be NULL as the entrance stub indirection will have
+ * been replaced with a direct link).  The fragment corresponding to
+ * the tag is only present in this unit if the body_pc returned is
+ * non-NULL.
+ */
+void
+fragment_coarse_lookup_in_unit(dcontext_t *dcontext, coarse_info_t *info, app_pc tag,
+                               /* FIXME: have separate type for stub pc vs body pc? */
+                               cache_pc *stub_pc_out/*OUT*/,
+                               cache_pc *body_pc_out/*OUT*/)
+{
+    cache_pc res = NULL;
+    cache_pc stub_pc = NULL, body_pc = NULL;
+    app_to_cache_t a2c;
+    coarse_table_t *htable;
+    ASSERT(info != NULL);
+    if (info->htable == NULL) /* not initialized yet, so no code there */
+        goto coarse_lookup_return;
+    htable = (coarse_table_t *) info->htable;
+    TABLE_RWLOCK(htable, read, lock);
+    a2c = hashtable_coarse_lookup(dcontext, (ptr_uint_t)tag, htable);
+    if (!A2C_ENTRY_IS_EMPTY(a2c)) {
+        ASSERT(BOOLS_MATCH(info->frozen, info->cache_start_pc != NULL));
+        /* for frozen, htable only holds body pc */
+        res = ((ptr_uint_t)a2c.cache) + info->cache_start_pc;
+    }
+    if (res != NULL)
+        coarse_body_from_htable_entry(dcontext, info, tag, res, &stub_pc, &body_pc);
+    else if (info->frozen) /* need a separate lookup for stub */
+        coarse_body_from_htable_entry(dcontext, info, tag, res, &stub_pc, &body_pc);
+    TABLE_RWLOCK(htable, read, unlock);
+    /* cannot have both a shared coarse and shared fine bb for same tag
+     * (can have shared trace shadowing shared coarse trace head bb, or
+     * private bb shadowing shared coarse bb)
+     */
+    ASSERT(body_pc == NULL || fragment_lookup_shared_bb(dcontext, tag) == NULL);
+ coarse_lookup_return:
+    if (stub_pc_out != NULL)
+        *stub_pc_out = stub_pc;
+    if (body_pc_out != NULL)
+        *body_pc_out = body_pc;
+}
+
+/* Returns the body pc of the coarse fragment corresponding to tag, or
+ * NULL if not found
+ */
+cache_pc
+fragment_coarse_lookup(dcontext_t *dcontext, app_pc tag)
+{
+    cache_pc res = NULL;
+    coarse_info_t *info = get_executable_area_coarse_info(tag);
+    /* We must check each unit in turn */
+    while (info != NULL) { /* loop over primary and secondary unit */
+        fragment_coarse_lookup_in_unit(dcontext, info, tag, NULL, &res);
+        if (res != NULL)
+            return res;
+        ASSERT(info->frozen || info->non_frozen == NULL);
+        info = info->non_frozen;
+        ASSERT(info == NULL || !info->frozen);
+    }
+    return NULL;
+}
+
+/* It's up to the caller to hold locks preventing simultaneous writes to wrapper */
+void
+fragment_coarse_wrapper(fragment_t *wrapper, app_pc tag, cache_pc body_pc)
+{
+    ASSERT(wrapper != NULL);
+    if (wrapper == NULL)
+        return;
+    ASSERT(tag != NULL);
+    ASSERT(body_pc != NULL);
+    /* FIXME: fragile to rely on other routines not inspecting other
+     * fields of fragment_t -- but setting to 0 perhaps no better than garbage
+     * in that case.  We do need to set prefix_size, incoming_stubs, and
+     * {next,prev}_vmarea to NULL, for sure.
+     */
+    memset(wrapper, 0, sizeof(*wrapper));
+    wrapper->tag = tag;
+    wrapper->start_pc = body_pc;
+    wrapper->flags = FRAGMENT_COARSE_WRAPPER_FLAGS;
+    /* We don't have stub pc so can't fill in FRAG_IS_TRACE_HEAD -- so we rely
+     * on callers passing src info to fragment_lookup_fine_and_coarse()
+     */
+}
+
+/* If finds a coarse fragment for tag, returns wrapper; else returns NULL */
+fragment_t *
+fragment_coarse_lookup_wrapper(dcontext_t *dcontext, app_pc tag, fragment_t *wrapper)
+{
+    cache_pc coarse;
+    ASSERT(wrapper != NULL);
+    coarse = fragment_coarse_lookup(dcontext, tag);
+    if (coarse != NULL) {
+        fragment_coarse_wrapper(wrapper, tag, coarse);
+        return wrapper;
+    }
+    return NULL;
+}
+
+/* Takes in last_exit in order to mark trace headness.
+ * FIXME case 8600: should we replace all other lookups with this one?
+ * Fragile to only have select callers use this and everyone else
+ * ignore coarse fragments...
+ */
+fragment_t *
+fragment_lookup_fine_and_coarse(dcontext_t *dcontext, app_pc tag,
+                                fragment_t *wrapper, linkstub_t *last_exit)
+{
+    fragment_t *res = fragment_lookup(dcontext, tag);
+    if (DYNAMO_OPTION(coarse_units)) {
+        ASSERT(wrapper != NULL);
+        if (res == NULL) {
+            res = fragment_coarse_lookup_wrapper(dcontext, tag, wrapper);
+            /* FIXME: no way to know if source is a fine fragment! */
+            if (res != NULL && last_exit == get_coarse_trace_head_exit_linkstub())
+                res->flags |= FRAG_IS_TRACE_HEAD;
+        } else {
+            /* cannot have both coarse and fine shared bb for same tag
+             * (can have shared trace shadowing shared coarse trace head bb,
+             * or private bb with same tag)
+             */
+            ASSERT(TEST(FRAG_IS_TRACE, res->flags) ||
+                   !TEST(FRAG_SHARED, res->flags) ||
+                   fragment_coarse_lookup(dcontext, tag) == NULL);
+        }
+    }
+    return res;
+}
+
+/* Takes in last_exit in order to mark trace headness.
+ * FIXME: should we replace all other same_sharing lookups with this one?
+ * Fragile to only have select callers use this and everyone else
+ * ignore coarse fragments...
+ */
+fragment_t *
+fragment_lookup_fine_and_coarse_sharing(dcontext_t *dcontext, app_pc tag,
+                                        fragment_t *wrapper, linkstub_t *last_exit,
+                                        uint share_flags)
+{
+    fragment_t *res =
+        fragment_lookup_same_sharing(dcontext, tag, share_flags);
+    if (DYNAMO_OPTION(coarse_units) && TEST(FRAG_SHARED, share_flags)) {
+        ASSERT(wrapper != NULL);
+        if (res == NULL) {
+            res = fragment_coarse_lookup_wrapper(dcontext, tag, wrapper);
+            if (res != NULL && last_exit == get_coarse_trace_head_exit_linkstub())
+                res->flags |= FRAG_IS_TRACE_HEAD;
+        }
+    }
+    return res;
+}
+
+/* Returns the owning unit of f (versus get_executable_area_coarse_info(f->tag)
+ * which may return a frozen unit that must be checked for f along with its
+ * secondary unfrozen unit).
+ */
+coarse_info_t *
+get_fragment_coarse_info(fragment_t *f)
+{
+    /* We have multiple potential units per vmarea region, so we use the fcache
+     * pc to get the unambiguous owning unit
+     */
+    if (!TEST(FRAG_COARSE_GRAIN, f->flags))
+        return NULL;
+    ASSERT(FCACHE_ENTRY_PC(f) != NULL);
+    return get_fcache_coarse_info(FCACHE_ENTRY_PC(f));
+}
+
+/* Pass in info if you know it; else this routine will look it up.
+ * Checks for stub targeting a trace head, or targeting a trace thus
+ * indicating that this is a shadowed trace head.
+ * If body_in or info_in is NULL, this routine will look them up.
+ */
+bool
+coarse_is_trace_head_in_own_unit(dcontext_t *dcontext, app_pc tag, cache_pc stub,
+                                 cache_pc body_in, bool body_valid,
+                                 coarse_info_t *info_in)
+{
+    coarse_info_t *info = info_in;
+    ASSERT(stub != NULL);
+    if (coarse_is_trace_head(stub))
+        return true;
+    if (info == NULL)
+        info = get_stub_coarse_info(stub);
+    if (info == NULL)
+        return false;
+    /* If a coarse stub is linked to a fine fragment and there exists
+     * a body for that target tag in the same unit as the stub, we assume that
+     * we have a shadowed coarse trace head.
+     */
+    if (entrance_stub_linked(stub, info) &&
+        /* Target is fine if no info for it */
+        get_fcache_coarse_info(entrance_stub_jmp_target(stub)) == NULL) {
+        cache_pc body = body_in;
+        if (!body_valid) {
+            ASSERT(body == NULL);
+            fragment_coarse_lookup_in_unit(dcontext, info, tag, NULL, &body);
+        }
+        if (body != NULL)
+            return true;
+    }
+    return false;
+}
+
+/* Returns whether an entry exists. */
+bool
+fragment_coarse_replace(dcontext_t *dcontext, coarse_info_t *info, app_pc tag,
+                        cache_pc new_value)
+{
+    app_to_cache_t old_entry = {tag, NULL/*doesn't matter*/};
+    app_to_cache_t new_entry = {tag, new_value};
+    coarse_table_t *htable;
+    bool res = false;
+    ASSERT(info != NULL && info->htable != NULL);
+    htable = (coarse_table_t *) info->htable;
+    TABLE_RWLOCK(htable, read, lock);
+    res = hashtable_coarse_replace(old_entry, new_entry, info->htable);
+    TABLE_RWLOCK(htable, read, unlock);
+    return res;
+}
+
+/* Returns the tag for the coarse fragment whose body contains pc.
+ * If that returned tag != NULL, also returns the body pc in the optional OUT param.
+ * FIXME: verify not called too often: watch the kstat.
+ */
+app_pc
+fragment_coarse_pclookup(dcontext_t *dcontext, coarse_info_t *info, cache_pc pc,
+                         /*OUT*/cache_pc *body_out)
+{
+    app_to_cache_t a2c;
+    coarse_table_t *htable;
+    cache_pc body_pc;
+    ssize_t closest_distance = SSIZE_T_MAX;
+    app_pc closest = NULL;
+    uint i;
+    ASSERT(info != NULL);
+    if (info->htable == NULL)
+        return NULL;
+    KSTART(coarse_pclookup);
+    htable = (coarse_table_t *) info->htable;
+    TABLE_RWLOCK(htable, read, lock);
+    for (i = 0; i < htable->capacity; i++) {
+        a2c = htable->table[i];
+        /* must check for sentinel */
+        if (A2C_ENTRY_IS_REAL(a2c)) {
+            ASSERT(BOOLS_MATCH(info->frozen, info->cache_start_pc != NULL));
+            /* for frozen, htable only holds body pc */
+            a2c.cache += (ptr_uint_t) info->cache_start_pc;
+            /* We have no body length so we must walk entire table */
+            coarse_body_from_htable_entry(dcontext, info, a2c.app,
+                                          a2c.cache, NULL, &body_pc);
+            if (body_pc != NULL && body_pc <= pc && (pc - body_pc) < closest_distance) {
+                closest_distance = pc - body_pc;
+                closest = a2c.app;
+            }
+        }
+    }
+    TABLE_RWLOCK(htable, read, unlock);
+    if (body_out != NULL)
+        *body_out = pc - closest_distance;
+    KSTOP(coarse_pclookup);
+    return closest;
+}
+
+/* Returns the tag for the coarse fragment whose body _begins at_ pc */
+app_pc
+fragment_coarse_entry_pclookup(dcontext_t *dcontext, coarse_info_t *info, cache_pc pc)
+{
+    app_to_cache_t main_a2c;
+    app_to_cache_t pc_a2c;
+    coarse_table_t *main_htable;
+    coarse_table_t *pc_htable;
+    cache_pc body_pc;
+    app_pc res = NULL;
+    uint i;
+    ASSERT(info != NULL);
+    if (info->htable == NULL)
+        return NULL;
+    /* FIXME: we could use this table for non-frozen if we updated it
+     * when we add to main htable; for now we only support frozen use
+     */
+    if (!DYNAMO_OPTION(coarse_pclookup_table) || !info->frozen) {
+        res = fragment_coarse_pclookup(dcontext, info, pc, &body_pc);
+        if (body_pc == pc)
+            return res;
+        else
+            return NULL;
+    }
+    KSTART(coarse_pclookup);
+
+    if (info->pclookup_htable == NULL) {
+        mutex_lock(&info->lock);
+        if (info->pclookup_htable == NULL) {
+            /* set up reverse lookup table */
+            main_htable = (coarse_table_t *) info->htable;
+            pc_htable = NONPERSISTENT_HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, coarse_table_t,
+                                                      ACCT_FRAG_TABLE);
+            hashtable_coarse_init(GLOBAL_DCONTEXT, pc_htable, main_htable->hash_bits,
+                                  DYNAMO_OPTION(coarse_pclookup_htable_load),
+                                  (hash_function_t)INTERNAL_OPTION(alt_hash_func),
+                                  0 /* hash_mask_offset */,
+                                  HASHTABLE_ENTRY_SHARED | HASHTABLE_SHARED |
+                                  HASHTABLE_RELAX_CLUSTER_CHECKS
+                                  NAME("coarse pclookup htable"));
+            /* We give pc table a lower lock rank so we can add below
+             * while holding the lock, though the table is still local
+             * while we hold info->lock and do not write it out to its
+             * info-> field, so we could update the add() and
+             * deadlock-avoidance routines instead (xref case 9522).
+             * FIXME: add param to init() that takes in lock rank?
+             */
+            ASSIGN_INIT_READWRITE_LOCK_FREE(pc_htable->rwlock,
+                                            coarse_pclookup_table_rwlock);
+            TABLE_RWLOCK(main_htable, read, lock);
+            TABLE_RWLOCK(pc_htable, write, lock);
+            for (i = 0; i < main_htable->capacity; i++) {
+                main_a2c = main_htable->table[i];
+                /* must check for sentinel */
+                if (A2C_ENTRY_IS_REAL(main_a2c)) {
+                    ASSERT(BOOLS_MATCH(info->frozen, info->cache_start_pc != NULL));
+                    coarse_body_from_htable_entry(dcontext, info, main_a2c.app,
+                                                  main_a2c.cache
+                                                  /* frozen htable only holds body pc */
+                                                  + (ptr_uint_t) info->cache_start_pc,
+                                                  NULL, &body_pc);
+                    if (body_pc != NULL) {
+                        /* We can have two tags with the same cache pc, if
+                         * one is a single jmp that was elided (but we only
+                         * do that if -unsafe_freeze_elide_sole_ubr).
+                         * It's unsafe b/c of case 9677: when translating back
+                         * to app pc we will just take 1st one in linear walk of
+                         * htable, which may be the wrong one!
+                         */
+                        pc_a2c = hashtable_coarse_lookup(dcontext,
+                                                         (ptr_uint_t)body_pc, pc_htable);
+                        if (A2C_ENTRY_IS_EMPTY(pc_a2c)) {
+                            pc_a2c.app = body_pc;
+                            pc_a2c.cache = main_a2c.app;
+                            hashtable_coarse_add(dcontext, pc_a2c, pc_htable);
+                        } else {
+                            ASSERT(DYNAMO_OPTION(unsafe_freeze_elide_sole_ubr));
+                        }
+                    }
+                }
+            }
+            TABLE_RWLOCK(pc_htable, write, unlock);
+            TABLE_RWLOCK(main_htable, read, unlock);
+            /* Only when fully initialized can we set it, as we hold no lock for it */
+            info->pclookup_htable = (void *) pc_htable;
+        }
+        mutex_unlock(&info->lock);
+    }
+
+    pc_htable = (coarse_table_t *) info->pclookup_htable;
+    ASSERT(pc_htable != NULL);
+    TABLE_RWLOCK(pc_htable, read, lock);
+    pc_a2c = hashtable_coarse_lookup(dcontext, (ptr_uint_t)pc, pc_htable);
+    if (!A2C_ENTRY_IS_EMPTY(pc_a2c))
+        res = pc_a2c.cache;
+    TABLE_RWLOCK(pc_htable, read, unlock);
+    KSTOP(coarse_pclookup);
+    return res;
+}
+
+/* case 9900: must have dynamo_all_threads_synched since we haven't resolved
+ * lock rank ordering issues with the hashtable locks
+ */
+static void
+fragment_coarse_entry_freeze(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info,
+                             pending_freeze_t *pending)
+{
+    app_to_cache_t frozen_a2c;
+    app_to_cache_t looka2c = {0,0};
+    coarse_table_t *frozen_htable;
+    cache_pc tgt;
+    if (pending->entrance_stub) {
+        frozen_htable = (coarse_table_t *) freeze_info->dst_info->th_htable;
+        /* case 9900: rank order conflict with coarse_info_incoming_lock:
+         * do not grab frozen_htable write lock (only need read but we were
+         * grabbing write to match assert): mark is_local instead and rely
+         * on dynamo_all_threads_synched
+         */
+        DODEBUG({ frozen_htable->is_local = true; });
+        ASSERT_NOT_IMPLEMENTED(dynamo_all_threads_synched && "case 9900");
+    } else {
+        frozen_htable = (coarse_table_t *) freeze_info->dst_info->htable;
+    }
+    ASSERT_OWN_WRITE_LOCK(!frozen_htable->is_local, &frozen_htable->rwlock);
+    looka2c = hashtable_coarse_lookup(dcontext, (ptr_uint_t)pending->tag, frozen_htable);
+    if (A2C_ENTRY_IS_EMPTY(looka2c)) {
+        frozen_a2c.app = pending->tag;
+        if (pending->entrance_stub) {
+            /* add inter-unit/trace-head stub to htable */
+            LOG(THREAD, LOG_FRAGMENT, 4,
+                "  adding pending stub "PFX"."PFX" => "PFX"\n",
+                pending->tag, pending->cur_pc, freeze_info->stubs_cur_pc);
+            frozen_a2c.cache = (cache_pc)
+                (freeze_info->stubs_cur_pc - freeze_info->stubs_start_pc);
+            hashtable_coarse_add(dcontext, frozen_a2c, frozen_htable);
+            /* copy to new stubs area */
+            transfer_coarse_stub(dcontext, freeze_info, pending->cur_pc,
+                                 pending->trace_head, true/*replace*/);
+        } else {
+            /* fall-through optimization */
+            if (DYNAMO_OPTION(coarse_freeze_elide_ubr) &&
+                pending->link_cti_opnd != NULL &&
+                pending->link_cti_opnd + 4 == freeze_info->cache_cur_pc &&
+                pending->elide_ubr) {
+                ASSERT(!pending->trace_head);
+                LOG(THREAD, LOG_FRAGMENT, 4, "  fall-through opt from prev fragment\n");
+                freeze_info->cache_cur_pc -= JMP_LONG_LENGTH;
+                pending->link_cti_opnd = NULL;
+                STATS_INC(coarse_freeze_fallthrough);
+                DODEBUG({ freeze_info->num_elisions++; });
+            }
+            LOG(THREAD, LOG_FRAGMENT, 4,
+                "  adding pending %sfragment "PFX"."PFX" => "PFX"\n",
+                pending->trace_head ? "trace head " : "",
+                pending->tag, pending->cur_pc, freeze_info->cache_cur_pc);
+            /* add to htable the offset from start of cache */
+            frozen_a2c.cache = (cache_pc)
+                (freeze_info->cache_cur_pc - freeze_info->cache_start_pc);
+            hashtable_coarse_add(dcontext, frozen_a2c, frozen_htable);
+            /* copy to new cache */
+            transfer_coarse_fragment(dcontext, freeze_info, pending->cur_pc);
+        }
+        tgt = frozen_a2c.cache;
+    } else {
+        tgt = looka2c.cache;
+        /* Should not hit any links to TH, so should hit once, from htable walk */
+        ASSERT(!pending->trace_head || pending->entrance_stub);
+        /* May have added entrance stub for intra-unit TH as non-TH if it was
+         * linked to a trace, since didn't have body pc at the time, so we
+         * fix up here on the proactive add when adding its body
+         */
+        if (pending->entrance_stub && pending->trace_head && freeze_info->unlink) {
+            cache_pc abs_tgt = tgt + (ptr_uint_t)freeze_info->stubs_start_pc;
+            transfer_coarse_stub_fix_trace_head(dcontext, freeze_info, abs_tgt);
+        }
+    }
+    if (pending->link_cti_opnd != NULL) {
+        /* fix up incoming link */
+        cache_pc patch_tgt = (cache_pc)
+            (((ptr_uint_t)(pending->entrance_stub ? freeze_info->stubs_start_pc :
+                           freeze_info->cache_start_pc)) + tgt);
+        ASSERT(!pending->trace_head || pending->entrance_stub);
+        LOG(THREAD, LOG_FRAGMENT, 4, "  patch link "PFX" => "PFX"."PFX"%s\n",
+            pending->link_cti_opnd, pending->tag, patch_tgt,
+            pending->entrance_stub ? " stub" : "");
+        insert_relative_target(pending->link_cti_opnd, patch_tgt, NOT_HOT_PATCHABLE);
+    }
+    if (pending->entrance_stub) {
+        DODEBUG({ frozen_htable->is_local = false; });
+    }
+}
+
+/* There are several strategies for copying each fragment and non-inter-unit stub
+ * to new, compact storage.  Here we use a cache-driven approach, necessarily
+ * augmented with the htable as we cannot find tags for arbitrary fragments in
+ * the cache.  We use a pending-add stack to avoid a second pass.
+ * By adding ubr targets last, we can elide fall-through jmps.
+ * FIXME case 9428:
+ *   - Sorting entrance stubs for faster lazy linking lookup
+ *   - Use 8-bit-relative jmps when possible for compaction, though this
+ *     requires pc translation support and probably an extra pass when freezing.
+ *
+ * Tasks:
+ * Copy each new fragment and non-inter-unit stub to the new region
+ * Unlink all inter-unit entrance stubs, unless freezing and not
+ *   writing to disk.
+ * Re-target intra-unit jmps from old entrance stub to new fragment location
+ * Re-target jmp to stub
+ * Re-target indirect stubs to new prefix
+ * Re-target inter-unit and trace head stubs to new prefix
+ *
+ * case 9900: must have dynamo_all_threads_synched since we haven't resolved
+ * lock rank ordering issues with the hashtable locks
+ */
+void
+fragment_coarse_unit_freeze(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info)
+{
+    pending_freeze_t pending_local;
+    pending_freeze_t *pending;
+    app_to_cache_t a2c;
+    coarse_table_t *frozen_htable;
+    coarse_table_t *htable;
+    cache_pc body_pc;
+    uint i;
+    ASSERT(freeze_info != NULL && freeze_info->src_info != NULL);
+    if (freeze_info->src_info->htable == NULL)
+        return;
+    LOG(THREAD, LOG_FRAGMENT, 2,
+        "freezing fragments in %s\n", freeze_info->src_info->module);
+
+    /* we walk just the main htable and find trace heads by asking for the body pc */
+    htable = (coarse_table_t *) freeze_info->src_info->htable;
+    DOSTATS({
+        LOG(THREAD, LOG_ALL, 1,
+            "htable pre-freezing %s\n", freeze_info->src_info->module);
+        hashtable_coarse_study(dcontext, htable, 0/*clean state*/);
+    });
+    frozen_htable = (coarse_table_t *) freeze_info->dst_info->htable;
+    /* case 9900: rank order conflict with coarse_info_incoming_lock:
+     * do not grab frozen_htable write lock: mark is_local instead and rely
+     * on dynamo_all_threads_synched
+     */
+    DODEBUG({ frozen_htable->is_local = true; });
+    ASSERT_NOT_IMPLEMENTED(dynamo_all_threads_synched && "case 9900");
+    /* FIXME case 9522: not grabbing TABLE_RWLOCK(htable, read, lock) due to
+     * rank order violation with frozen_htable write lock!  We go w/ the write
+     * lock since lookup routines check for it.  To solve we'll need a way to
+     * tell deadlock checking that frozen_htable is private to this thread and
+     * that no other thread can hold its lock.  For now we only support
+     * all-synch, but if later we want !in_place that needs no synch we'll need
+     * a solution.
+     */
+    ASSERT_NOT_IMPLEMENTED(dynamo_all_threads_synched && "case 9522");
+    /* FIXME: we're doing htable order, better to use either tag (original app)
+     * or cache (execution) order?
+     */
+    for (i = 0; i < htable->capacity || freeze_info->pending != NULL; i++) {
+
+        /* Process pending entries first; then continue through htable */
+        while (freeze_info->pending != NULL) {
+            pending = freeze_info->pending;
+            freeze_info->pending = pending->next;
+            fragment_coarse_entry_freeze(dcontext, freeze_info, pending);
+            HEAP_TYPE_FREE(dcontext, pending, pending_freeze_t,
+                           ACCT_MEM_MGT/*appropriate?*/, UNPROTECTED);
+        }
+
+        if (i < htable->capacity) {
+            a2c = htable->table[i];
+            /* must check for sentinel */
+            if (!A2C_ENTRY_IS_REAL(a2c))
+                continue;
+        } else
+            continue;
+
+        LOG(THREAD, LOG_FRAGMENT, 4,
+            " %d app="PFX", cache="PFX"\n", i, a2c.app, a2c.cache);
+        coarse_body_from_htable_entry(dcontext, freeze_info->src_info, a2c.app,
+                                      a2c.cache, NULL, &body_pc);
+
+        if (body_pc == NULL) {
+            /* We add only when targeted by fragments, so we don't have to
+             * figure out multiple times whether intra-unit or not
+             */
+            LOG(THREAD, LOG_FRAGMENT, 4,
+                "  ignoring entrance stub "PFX"\n", a2c.cache);
+        } else {
+            pending_local.tag = a2c.app;
+            pending_local.cur_pc = body_pc;
+            pending_local.entrance_stub = false;
+            pending_local.link_cti_opnd = NULL;
+            pending_local.elide_ubr = true; /* doesn't matter since no link */
+            pending_local.trace_head =
+                coarse_is_trace_head_in_own_unit(dcontext, a2c.app, a2c.cache,
+                                                 body_pc, true, freeze_info->src_info);
+            pending_local.next = NULL;
+            fragment_coarse_entry_freeze(dcontext, freeze_info, &pending_local);
+
+            if (pending_local.trace_head) {
+                /* we do need to proactively add the entrance stub, in
+                 * case it is only targeted by an indirect branch
+                 */
+                LOG(THREAD, LOG_FRAGMENT, 4,
+                    "  adding trace head entrance stub "PFX"\n", a2c.cache);
+                pending_local.tag = a2c.app;
+                pending_local.cur_pc = a2c.cache;
+                pending_local.entrance_stub = true;
+                pending_local.link_cti_opnd = NULL;
+                pending_local.elide_ubr = true; /* doesn't matter since no link */
+                pending_local.trace_head = true;
+                fragment_coarse_entry_freeze(dcontext, freeze_info, &pending_local);
+            }
+        }
+    }
+    DODEBUG({ frozen_htable->is_local = false; });
+    DOSTATS({
+        /* The act of freezing tends to improve hashtable layout */
+        LOG(THREAD, LOG_ALL, 1,
+            "htable post-freezing %s\n", freeze_info->src_info->module);
+        hashtable_coarse_study(dcontext, frozen_htable, 0/*clean state*/);
+    });
+}
+
+uint
+fragment_coarse_htable_persist_size(dcontext_t *dcontext, coarse_info_t *info,
+                                    bool cache_table)
+{
+    coarse_table_t *htable = (coarse_table_t *)
+        (cache_table ? info->htable : info->th_htable);
+    return hashtable_coarse_persist_size(dcontext, htable);
+}
+
+/* Returns true iff all writes succeeded. */
+bool
+fragment_coarse_htable_persist(dcontext_t *dcontext,  coarse_info_t *info,
+                               bool cache_table, file_t fd)
+{
+    coarse_table_t *htable = (coarse_table_t *)
+        (cache_table ? info->htable : info->th_htable);
+    ASSERT(fd != INVALID_FILE);
+    return hashtable_coarse_persist(dcontext, htable, fd);
+}
+
+void
+fragment_coarse_htable_resurrect(dcontext_t *dcontext,  coarse_info_t *info,
+                                 bool cache_table, byte *mapped_table)
+{
+    coarse_table_t **htable = (coarse_table_t **)
+        (cache_table ? &info->htable : &info->th_htable);
+    ASSERT(info->frozen);
+    ASSERT(mapped_table != NULL);
+    ASSERT(*htable == NULL);
+    *htable = hashtable_coarse_resurrect(dcontext, mapped_table
+                                         NAME(cache_table ? "persisted cache htable" :
+                                              "persisted stub htable"));
+    /* generally want to keep basic alignment */
+    ASSERT_CURIOSITY(ALIGNED((*htable)->table, sizeof(app_pc)));
+}
+
+/*******************************************************************************/
diff --git a/core/fragment.h b/core/fragment.h
new file mode 100644
index 000000000..90809b2af
--- /dev/null
+++ b/core/fragment.h
@@ -0,0 +1,1215 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/* 
+ * fragment.h - fragment_t structures 
+ */
+
+#ifndef _FRAGMENT_H_
+#define _FRAGMENT_H_ 1
+
+#include "hashtable.h"
+
+/* Flags, stored in fragment_t->flags bitfield
+ */
+#define FRAG_IS_FUTURE              0x000001
+#define FRAG_TRACE_LINKS_SHIFTED    0x000002
+#define FRAG_IS_TRACE               0x000004
+#define FRAG_IS_TRACE_HEAD          0x000008
+#define FRAG_LINKED_OUTGOING        0x000010
+#define FRAG_LINKED_INCOMING        0x000020
+#define FRAG_CANNOT_DELETE          0x000040
+#define FRAG_CANNOT_BE_TRACE        0x000080
+
+/* Indicates an irregular fragment_t.  In particular, there are no
+ * trailing linkstubs after this fragment_t struct.  Note that other
+ * "fake fragment_t" flags should be set in combination with this one
+ * (FRAG_IS_{FUTURE,EXTRA_VMAREA*,EMPTY_SLOT}, FRAG_FCACHE_FREE_LIST).
+ */
+#define FRAG_FAKE                   0x000100
+
+/* this flag indicates fragment writes all 6 flags prior to reading */
+#define FRAG_WRITES_EFLAGS_6        0x000200
+/* this flag indicates fragment writes OF before reading it */
+#define FRAG_WRITES_EFLAGS_OF       0x000400
+
+/* This is not a fragment_t but an fcache free list entry.
+ * In current usage this is checked to see if the previous free list entry is
+ * a free list entry (see fcache.c's free_list_header_t.flags).
+ * This flag MUST be in the bottom 16 bits since free_list_header_t.flags
+ * is a ushort!
+ */
+#define FRAG_FCACHE_FREE_LIST       0x000800
+
+#define FRAG_HAS_SYSCALL            0x001000
+/* this flags indicates that a trace is being built from fragment_t->tag */
+#define FRAG_TRACE_BUILDING         0x002000
+
+/* used on future fragments, currently only read for adaptive working set 
+ * also used for fragments to know whether they are on the deleted list 
+ * (shared) or flush queue (private)
+ */
+#define FRAG_WAS_DELETED            0x004000
+/* indicates frag is from a non-protected page and may be self-modifying */
+#define FRAG_SELFMOD_SANDBOXED      0x008000
+/* indicates whether frag contains an elided direct cti */
+#define FRAG_HAS_DIRECT_CTI         0x010000
+/* used by fcache to distinguish fragment_t from its own empty slot struct */
+#define FRAG_IS_EMPTY_SLOT          0x020000
+/* used by vmarea to distinguish fragment_t from its own multi unit struct */
+#define FRAG_IS_EXTRA_VMAREA        0x040000
+#define FRAG_IS_EXTRA_VMAREA_INIT   0x080000
+
+#ifdef PROGRAM_SHEPHERDING
+/* indicates from memory that wasn't part of code from image on disk */
+# define FRAG_DYNGEN                0x100000
+# ifdef DGC_DIAGNOSTICS
+/* for now, only used to identify regions that fail our policies */
+#  define FRAG_DYNGEN_RESTRICTED    0x200000
+# endif
+#endif
+
+#ifdef X64
+/* this fragment contains 32-bit code */
+# define FRAG_32_BIT                0x400000
+# ifdef RETURN_STACK
+#  error RETURN_STACK not compatible with X64
+# endif
+#elif defined(RETURN_STACK)
+/* used for RETURN_STACK */
+# define FRAG_ENDS_WITH_RETURN      0x400000
+#endif
+
+#define FRAG_MUST_END_TRACE         0x800000
+
+#define FRAG_SHARED                0x1000000
+/* indicates a temporary private copy of a shared bb, used
+ * for trace building
+ */
+#define FRAG_TEMP_PRIVATE          0x2000000
+
+#define FRAG_TRACE_OUTPUT          0x4000000
+
+#define FRAG_CBR_FALLTHROUGH_SHORT 0x8000000
+
+/* Indicates coarse-grain cache management, i.e., batch units with
+ * no individual fragment_t.
+ */
+#define FRAG_COARSE_GRAIN         0x10000000
+
+/* Translation info was recorded at fragment emit time in a post-fragment_t field.
+ * This is NOT set for flushed fragments, which store their info in the in_xlate
+ * union instead and are marked FRAG_WAS_DELETED, though if both flags are set
+ * then the info is in the post-fragment_t field.
+ */
+#define FRAG_HAS_TRANSLATION_INFO 0x20000000
+
+#ifdef SIDELINE
+# define FRAG_DO_NOT_SIDELINE     0x40000000
+#endif
+
+/* This fragment immediately follows a free entry in the fcache */
+#define FRAG_FOLLOWS_FREE_ENTRY   0x80000000
+
+/* Flags that a future fragment can transfer to a real on taking its place:
+ * Naturally we don't want FRAG_IS_FUTURE or FRAG_WAS_DELETED.
+ * FRAG_SHARED has to already be on the real fragment.
+ * Do NOT take the FRAG_TEMP_PRIVATE flag that we put on futures as an
+ * optimization in case never used -- it will mess up which heap is used, etc.
+ * There aren't really any other flags added to future fragments.
+ * Even FRAG_IS_TRACE_HEAD is only used for marking shared secondary
+ * trace heads from private traces.
+ */
+#define FUTURE_FLAGS_TRANSFER (FRAG_IS_TRACE_HEAD)
+/* only used for debugging */
+#define FUTURE_FLAGS_ALLOWED (FUTURE_FLAGS_TRANSFER|FRAG_FAKE|FRAG_IS_FUTURE|\
+                              FRAG_WAS_DELETED|FRAG_SHARED|FRAG_TEMP_PRIVATE)
+
+/* to save space size field is a ushort => maximum fragment size */
+enum { MAX_FRAGMENT_SIZE = USHRT_MAX };
+
+/* fragment structure used for basic blocks and traces 
+ * this is the core structure shared by everything
+ * trace heads and traces extend it below
+ */
+struct _fragment_t {
+    /* WARNING: the tag offset is assumed to be 0 in x86/emit_utils.c
+     * Also, next and flags' offsets must match future_fragment_t's
+     * And flags' offset must match fcache.c's empty_slot_t as well as
+     * vmarea.c's multi_entry_t structs
+     */
+    app_pc    tag;              /* non-zero fragment tag used for lookups */
+
+    /* Contains FRAG_ flags.  Should only be modified for FRAG_SHARED fragments
+     * while holding the change_linking_lock.
+     */
+    uint      flags;
+
+    /* trace head counters are in separate hashtable since always private.
+     * FIXME: when all fragments are private, a separate table uses more memory
+     * than having a counter field for all fragments, including non-trace-heads
+     */
+
+    /* size in bytes of the fragment (includes body and stubs, and for
+     * selfmod fragments also includes selfmod app code copy and size field)
+     */    
+    ushort       size;
+
+    /* both of these fields are tiny -- padding shouldn't be more than a cache line
+     * size (32 P3, 64 P4), prefix should be even smaller.
+     * they combine with size to shrink fragment_t for us
+     * N.B.: byte is an unsigned char
+     */
+    byte      prefix_size;      /* size of prefix, after which is non-ind. br. entry */
+    byte      fcache_extra;     /* padding to fit in fcache slot, also includes the header */
+
+    cache_pc  start_pc;         /* very top of fragment's code, equals
+                                 * entry point when indirect branch target */
+
+    union {
+        /* For a live fragment, we store a list of other fragments' exits that target
+         * this fragment (outgoing exit stubs are all allocated with fragment_t struct,
+         * use FRAGMENT_EXIT_STUBS() to access).
+         */
+        linkstub_t  *incoming_stubs;
+        /* For a pending-deletion fragment (marked with FRAG_WAS_DELETED),
+         * we store translation info.
+         */
+        translation_info_t *translation_info;
+    } in_xlate;
+
+    fragment_t  *next_vmarea;     /* for chaining fragments in vmarea list */
+    fragment_t  *prev_vmarea;     /* for chaining fragments in vmarea list */
+    union {
+        fragment_t  *also_vmarea;     /* for chaining fragments across vmarea lists */
+        /* For lazily-deleted fragments, we store the flushtime here, as this
+         * field is no longer used once a fragment is not live.
+         */
+        uint      flushtime;
+    } also;
+
+#ifdef DEBUG
+    int       id;               /* thread-shared-unique fragment identifier */
+#endif
+
+#ifdef CUSTOM_TRACES_RET_REMOVAL
+    int num_calls;
+    int num_rets;
+#endif
+}; /* fragment_t */
+
+
+/* Shared fragments don't need some fields that private ones do, so we
+ * dynamically choose different structs.  fragment_t is for shared only.
+ * Here we again use C awkwardness to have a subclass.
+ */
+typedef struct _private_fragment_t {
+    fragment_t  f;
+    fragment_t  *next_fcache;     /* for chaining fragments in fcache unit */
+    fragment_t  *prev_fcache;     /* for chaining fragments in fcache unit */
+} private_fragment_t;
+
+
+/* Structure used for future fragments, separate to save memory.
+ * next and flags must be at same offset as for fragment_t, so that
+ * hashtable (next) and link.c (flags) can polymorphize fragment_t
+ * and future_fragment_t.  The rule is enforced in fragment_init.
+ */
+struct _future_fragment_t {
+    app_pc    tag;              /* non-zero fragment tag used for lookups */
+    uint      flags;            /* contains FRAG_ flags */
+    linkstub_t  *incoming_stubs;  /* list of other fragments' exits that target
+                                 * this fragment */
+};
+
+typedef struct _trace_bb_info_t {
+    app_pc tag;
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+    /* PR 204770: holds # exits in the trace corresponding to that bb.
+     * Used to obtain a better RCT source address.
+     * We could recreate to obtain this, except for flushed fragments: but
+     * we do this frequently enough that it's simpler to store for all
+     * fragments.
+     */
+    uint num_exits;
+#endif
+} trace_bb_info_t;
+
+/* N.B.: if you add fields to trace_t, make sure to add them
+ * to fragment_copy_data_fields as well as fragment_create and fragment_free
+ */
+typedef struct _trace_only_t {
+#ifdef PROFILE_RDTSC
+    uint64    count;            /* number of executions of this fragment */
+    uint64    total_time;       /* total time ever spent in this fragment */
+#endif
+
+#ifdef SIDELINE_COUNT_STUDY
+    linkcount_type_t count_old_pre;
+    linkcount_type_t count_old_post;
+#endif
+
+    /* holds the tags (and other info) for all constituent basic blocks */
+    trace_bb_info_t *bbs;
+    uint    num_bbs;
+} trace_only_t;
+
+/* trace extension of fragment_t */
+struct _trace_t {
+    fragment_t     f; /* shared fields */
+    trace_only_t t;
+}; /* trace_t */
+
+/* private version of trace_t */
+typedef struct _private_trace_t {
+    private_fragment_t  f;
+    trace_only_t        t;
+} private_trace_t;
+
+/* convenient way to deal w/ trace fields: this returns trace_only_t* */
+#define TRACE_FIELDS(f) (ASSERT(TEST(FRAG_IS_TRACE, (f)->flags)), \
+    (TEST(FRAG_SHARED, (f)->flags) ? \
+     &(((trace_t *)(f))->t) : &(((private_trace_t *)(f))->t)))
+
+/* FIXME: Can be used to determine if a frag should have a prefix since currently
+ * all IB targets have the same prefix. Use a different macro if different frags
+ * have different prefixes, i.e., BBs vs. traces.
+ */
+/* Historically traces were the only IBL targeted fragments, we'd want to have BBs
+ * targeted too, yet not all of them should indeed be targeted.
+ * See case 147 about possible extensions for bb non-tracehead
+ * fragments.
+ */
+/* FIXME: case 147: private bb's would have a different prefix
+ * therefore should be taken out of here.  Other than that there is no good reason
+ * not to be able to target them.  case 5836 covers targeting private fragments
+ * when using thread-shared ibl tables.
+ * A good example is FRAG_SELFMOD_SANDBOXED bb's which are always private.
+ * However it's not very useful to go after the short lived FRAG_TEMP_PRIVATE
+ */
+/* can a frag w/the given flags be an IBL target? */
+#define IS_IBL_TARGET(flags)                                         \
+    (TEST(FRAG_IS_TRACE, (flags)) ? \
+     (TEST(FRAG_SHARED, (flags)) || !DYNAMO_OPTION(shared_trace_ibt_tables)) : \
+     (DYNAMO_OPTION(bb_ibl_targets) && \
+      (TEST(FRAG_SHARED, (flags)) || !DYNAMO_OPTION(shared_bb_ibt_tables))))
+
+#define HASHTABLE_IBL_OFFSET(branch_type) \
+    (((branch_type) == IBL_INDCALL) ? DYNAMO_OPTION(ibl_indcall_hash_offset) : \
+     DYNAMO_OPTION(ibl_hash_func_offset))
+
+#ifdef DEBUG
+# define NAME(x) , x
+#else
+# define NAME(x)
+#endif
+
+#ifdef HASHTABLE_STATISTICS
+/* Statistics written from the cache that must be allocated separately */
+typedef struct _unprot_ht_statistics_t {
+    /* Statistics for App mode indirect branch lookups. Useful only for trace table. */
+    /* These should be accessible by indirect_branch_lookup emitted routines. */
+    /* Should have the form <ibl_routine>_stats, and are per hash table per routine (and per thread) */
+    /* FIXME: These are in the hash table itself for easier access when sharing IBL routines */
+
+    hashtable_statistics_t trace_ibl_stats[IBL_BRANCH_TYPE_END];
+    hashtable_statistics_t bb_ibl_stats[IBL_BRANCH_TYPE_END];
+
+    /* FIXME: this should really go to x86/arch.c instead of here */
+# ifdef WINDOWS
+    hashtable_statistics_t shared_syscall_hit_stats; /* miss path is shared with trace_ibl */
+#  endif
+} unprot_ht_statistics_t;
+#endif /* HASHTABLE_STATISTICS */
+
+/* Cached tag->start_pc table used by lookuptable in fragment_table_t as
+ * well as by ibl_table_t
+ */
+typedef struct _fragment_entry_t {
+    app_pc    tag_fragment;     /* non-zero fragment tag used for lookups */
+    cache_pc  start_pc_fragment;/* very top of fragment's code from fragment_t */
+} fragment_entry_t;
+
+#define HASHLOOKUP_NULL_START_PC ((cache_pc)PTR_UINT_0)
+#define HASHLOOKUP_SENTINEL_START_PC ((cache_pc)PTR_UINT_1)
+
+/* Flags stored in {fragment,ibl}_table_t->flags bitfield
+ */
+/* Indicates that fragment entries are shared between multiple tables in an
+ * inclusive hierarchical fashion, so only removal from the master table (which
+ * is not so marked) will result in fragment deletion. Used primarily for
+ * IBL targeted tables
+ */
+/* Updates to these flags should be reflected in
+ * x86/arch.c:table_flags_to_frag_flags() */
+#define FRAG_TABLE_INCLUSIVE_HIERARCHY   HASHTABLE_NOT_PRIMARY_STORAGE
+/* Set for IBL targeted tables, used in conjuction with FRAG_INCLUSIVE_HIERARCHY */
+#define FRAG_TABLE_IBL_TARGETED          HASHTABLE_LOCKLESS_ACCESS
+/* Set for IBL targeted tables, indicates that the table holds shared targets */
+#define FRAG_TABLE_TARGET_SHARED         HASHTABLE_ENTRY_SHARED
+/* Indicates that the table is shared */
+#define FRAG_TABLE_SHARED                HASHTABLE_SHARED
+/* is this table allocated in persistent memory? */
+#define FRAG_TABLE_PERSISTENT            HASHTABLE_PERSISTENT
+/* Indicates that the table targets traces */
+#define FRAG_TABLE_TRACE                 HASHTABLE_CUSTOM_FLAGS_START 
+
+/* hashtable of fragment_t* entries */
+/* macros w/ name and types are duplicated in fragment.c -- keep in sync */
+#define NAME_KEY fragment
+#define ENTRY_TYPE fragment_t *
+/* not defining HASHTABLE_USE_LOOKUPTABLE */
+#define HASHTABLEX_HEADER 1
+#define CUSTOM_FIELDS /* none */
+#include "hashtablex.h"
+#undef HASHTABLEX_HEADER
+
+
+/* hashtable of fragment_entry_t entries for per-type ibl tables */
+/* macros w/ name and types are duplicated in fragment.c -- keep in sync */
+#define NAME_KEY ibl
+#define ENTRY_TYPE fragment_entry_t
+/* not defining HASHTABLE_USE_LOOKUPTABLE */
+#ifdef HASHTABLE_STATISTICS
+# define HASHTABLE_ENTRY_STATS 1
+# define CUSTOM_FIELDS \
+    ibl_branch_type_t branch_type; \
+    /* stats written from the cache must be unprotected by allocating separately \
+     * FIXME: we could avoid this when protect_mask==0 by having a union here,   \
+     * like we have with mcontext in the dcontext, but not worth the complexity  \
+     * or space for debug-build-only stats \
+     */ \
+    unprot_ht_statistics_t *unprot_stats;
+#else
+# define CUSTOM_FIELDS \
+    ibl_branch_type_t branch_type;
+#endif /* HASHTABLE_STATISTICS */
+#define HASHTABLEX_HEADER 1
+#include "hashtablex.h"
+#undef HASHTABLEX_HEADER
+
+
+#if defined(RETURN_AFTER_CALL) || defined (RCT_IND_BRANCH)
+/* 3 macros w/ name and types are duplicated in fragment.c -- keep in sync */
+#define NAME_KEY app_pc
+#define ENTRY_TYPE app_pc
+/* not defining HASHTABLE_USE_LOOKUPTABLE */
+#define CUSTOM_FIELDS /* none */
+#define HASHTABLEX_HEADER 1
+#include "hashtablex.h"
+#undef HASHTABLEX_HEADER
+#endif /* defined(RETURN_AFTER_CALL) || defined (RCT_IND_BRANCH) */
+
+
+/* We keep basic blocks and traces in separate hashtables.  This is to
+ * speed up indirect_branch_lookup that looks for traces only, but it
+ * means our lookup function has to look in both hashtables.  This has
+ * no noticeable performance impact.  A strategy of having an
+ * all-fragment hashtable and a trace-only hashtable that mirrors just
+ * the traces of the all-fragment hashtable performs similarly but is
+ * more complicated since fragments need two different next fields,
+ * plus it uses more memory because traces are in two hashtables
+ * simultaneously.
+ *
+ * FIXME: Shared bb IBL routines indirectly access only a few fields
+ * from each fragment_table_t which will touch a separate cache line for
+ * each.  However, trace IBL routines don't indirect so I don't expect
+ * a performance hit of using the current struct layout.  
+ * FIXME: The bb IBL routines however are shared and therefore
+ * indirect, so splitting the fragment_table_t in two compactable
+ * structures may be worth trying.
+ */
+typedef struct _per_thread_t {
+    ibl_table_t trace_ibt[IBL_BRANCH_TYPE_END]; /* trace IB targets */
+    ibl_table_t bb_ibt[IBL_BRANCH_TYPE_END]; /* bb IB targets */
+    fragment_table_t bb;
+    fragment_table_t trace;
+    fragment_table_t future;
+#ifdef NATIVE_RETURN
+    fragment_table_t deleted;
+#endif
+#if defined(CLIENT_INTERFACE) && defined(CLIENT_SIDELINE)
+    mutex_t fragment_delete_mutex;
+#endif
+    file_t tracefile;
+
+    /* used for unlinking other threads' caches for flushing */
+    bool           could_be_linking;     /* accessing link data structs? */
+    bool           wait_for_unlink;      /* should this thread wait at synch point? */
+    bool           about_to_exit;        /* is this thread about to exit, so no need to flush? */
+    bool           flush_queue_nonempty; /* is this thread's deletion queue nonempty? */
+    event_t        waiting_for_unlink;   /* synch bet flusher and flushee */
+    event_t        finished_with_unlink; /* ditto */
+    event_t        finished_all_unlink;/* ditto */
+    /* this lock controls all 4 vars above, plus linking/unlinking shared_syscall,
+     * plus modifying queue of to-be-deleted thread-local vm regions
+     */
+    mutex_t          linking_lock;
+    bool           soon_to_be_linking; /* tells flusher thread is at cache exit synch */
+    /* for shared_deletion protocol */
+    uint           flushtime_last_update;
+    /* for syscalls_synch_flush, only used to cache whether a thread was at
+     * a syscall during early flushing stages for use in later stages.
+     * not used while not flushing.
+     */
+    bool           at_syscall_at_flush;
+
+#ifdef PROFILE_LINKCOUNT
+    uint tracedump_num_below_threshold;
+    linkcount_type_t tracedump_count_below_threshold;
+#endif
+} per_thread_t;
+
+
+#define FCACHE_ENTRY_PC(f) (f->start_pc + f->prefix_size)
+#define FCACHE_PREFIX_ENTRY_PC(f) \
+    (f->start_pc + f->prefix_size - FRAGMENT_BASE_PREFIX_SIZE(f->flags))
+#define FCACHE_IBT_ENTRY_PC(f) (f->start_pc)
+
+/* translation info pointer can be at end of any struct, so rather than have
+ * 8 different structs we keep it out of the formal struct definitions
+ */
+#define FRAGMENT_STRUCT_SIZE(flags) \
+    ((TEST(FRAG_IS_TRACE, (flags)) ? \
+      (TEST(FRAG_SHARED, (flags)) ? sizeof(trace_t) : sizeof(private_trace_t)) : \
+      (TEST(FRAG_SHARED, (flags)) ? sizeof(fragment_t) : sizeof(private_fragment_t))) \
+     + (TEST(FRAG_HAS_TRANSLATION_INFO, flags) ? sizeof(translation_info_t*) : 0))
+
+#define FRAGMENT_EXIT_STUBS(f) \
+   (TEST(FRAG_FAKE, (f)->flags) ? \
+    (ASSERT(false && "fake fragment_t has no exit stubs!"), (linkstub_t *)NULL) : \
+    ((linkstub_t *)(((byte*)(f)) + FRAGMENT_STRUCT_SIZE((f)->flags))))
+
+/* selfmod copy size is stored at very end of fragment space */
+#define FRAGMENT_SELFMOD_COPY_SIZE(f)                \
+  (ASSERT(TEST(FRAG_SELFMOD_SANDBOXED, (f)->flags)), \
+   (*((uint *)((f)->start_pc + (f)->size - sizeof(uint)))))
+#define FRAGMENT_SELFMOD_COPY_PC(f)                  \
+  (ASSERT(TEST(FRAG_SELFMOD_SANDBOXED, (f)->flags)), \
+   ((f)->start_pc + (f)->size - FRAGMENT_SELFMOD_COPY_SIZE(f)))
+
+#define FRAGMENT_TRANSLATION_INFO_ADDR(f) \
+  (TEST(FRAG_HAS_TRANSLATION_INFO, (f)->flags) ? \
+   ((translation_info_t **)(((byte*)(f)) + FRAGMENT_STRUCT_SIZE((f)->flags) \
+                            - sizeof(translation_info_t*))) : \
+    ((INTERNAL_OPTION(safe_translate_flushed) && TEST(FRAG_WAS_DELETED, (f)->flags)) ? \
+     &((f)->in_xlate.translation_info) : NULL))
+
+#define HAS_STORED_TRANSLATION_INFO(f) \
+  (TEST(FRAG_HAS_TRANSLATION_INFO, (f)->flags) || \
+   (INTERNAL_OPTION(safe_translate_flushed) && TEST(FRAG_WAS_DELETED, (f)->flags)))
+
+#define FRAGMENT_TRANSLATION_INFO(f) \
+  (HAS_STORED_TRANSLATION_INFO(f) ? (*(FRAGMENT_TRANSLATION_INFO_ADDR(f))) : NULL)
+
+/* Returns the end of the fragment body + any local stubs (excluding selfmod copy) */
+cache_pc
+fragment_stubs_end_pc(fragment_t *f);
+
+/* Returns the end of the fragment body (excluding exit stubs and selfmod copy) */
+cache_pc
+fragment_body_end_pc(dcontext_t *dcontext, fragment_t *f);
+
+bool
+fragment_initialized(dcontext_t *dcontext);
+
+void
+fragment_init(void);
+
+void
+fragment_exit(void);
+
+void
+fragment_reset_init(void);
+
+void
+fragment_reset_free(void);
+
+void
+fragment_thread_init(dcontext_t *dcontext);
+
+void
+fragment_thread_exit(dcontext_t *dcontext);
+
+/* re-initializes non-persistent memory */
+void
+fragment_thread_reset_init(dcontext_t *dcontext);
+
+/* frees all non-persistent memory */
+void
+fragment_thread_reset_free(dcontext_t *dcontext);
+
+#ifdef LINUX
+void
+fragment_fork_init(dcontext_t *dcontext);
+#endif
+
+#ifdef PROFILE_LINKCOUNT
+linkcount_type_t
+get_total_linkcount(fragment_t *f);
+#endif
+
+fragment_t *
+fragment_create(dcontext_t *dcontext, app_pc tag,
+                int body_size, int direct_exits, int indirect_exits,
+                int exits_size, uint flags);
+
+/* Creates a new fragment_t+linkstubs from the passed-in fragment and
+ * fills in linkstub_t and fragment_t fields, copying the fcache-related fields
+ * from the passed-in fragment (so be careful how the fields are used).
+ * Meant to be used to create a full fragment from a coarse-grain fragment.
+ * Caller is responsible for freeing via fragment_free().
+ */
+fragment_t *
+fragment_recreate_with_linkstubs(dcontext_t *dcontext, fragment_t *f_src);
+
+/* Frees the storage associated with f.
+ * Callers should use fragment_delete() instead of this routine, unless they
+ * obtained their fragment_t from fragment_recreate_with_linkstubs().
+ */
+void
+fragment_free(dcontext_t *dcontext, fragment_t *f);
+
+void
+fragment_add(dcontext_t *dcontext, fragment_t *f);
+
+void
+fragment_copy_data_fields(dcontext_t *dcontext, fragment_t *f_src, fragment_t *f_dst);
+
+/* options for fragment_delete actions param
+ * N.B.: these are NEGATIVE since callers care what's NOT done
+ */
+enum {
+    FRAGDEL_ALL               = 0x000,
+    FRAGDEL_NO_OUTPUT         = 0x001,
+    FRAGDEL_NO_UNLINK         = 0x002,
+    FRAGDEL_NO_HTABLE         = 0x004,
+    FRAGDEL_NO_FCACHE         = 0x008,
+    FRAGDEL_NO_HEAP           = 0x010,
+    FRAGDEL_NO_MONITOR        = 0x020,
+    FRAGDEL_NO_VMAREA         = 0x040,
+
+    FRAGDEL_NEED_CHLINK_LOCK  = 0x080,
+};
+void
+fragment_delete(dcontext_t *dcontext, fragment_t *f, uint actions);
+
+void
+fragment_record_translation_info(dcontext_t *dcontext, fragment_t *f, instrlist_t *ilist);
+
+void
+fragment_remove_shared_no_flush(dcontext_t *dcontext, fragment_t *f);
+
+void
+fragment_unlink_for_deletion(dcontext_t *dcontext, fragment_t *f);
+
+bool
+fragment_prepare_for_removal(dcontext_t *dcontext, fragment_t *f);
+
+/* Removes f from any IBT tables it is in. */
+void
+fragment_remove_from_ibt_tables(dcontext_t *dcontext, fragment_t *f, bool from_shared);
+
+uint
+fragment_remove_all_ibl_in_region(dcontext_t *dcontext, app_pc start, app_pc end);
+
+/* Removes f from any hashtables -- BB, trace, future -- and IBT tables
+ * it is in */
+void
+fragment_remove(dcontext_t *dcontext, fragment_t *f);
+
+void
+fragment_replace(dcontext_t *dcontext, fragment_t *f, fragment_t *new_f);
+
+#ifdef RETURN_STACK
+void
+fragment_delete_from_return_stack(dcontext_t *dcontext, fragment_t *f);
+#endif
+
+fragment_t *
+fragment_lookup(dcontext_t *dcontext, app_pc tag);
+
+fragment_t *
+fragment_lookup_bb(dcontext_t *dcontext, app_pc tag);
+
+fragment_t *
+fragment_lookup_shared_bb(dcontext_t *dcontext, app_pc tag);
+
+fragment_t *
+fragment_lookup_trace(dcontext_t *dcontext, app_pc tag);
+
+fragment_t *
+fragment_lookup_same_sharing(dcontext_t *dcontext, app_pc tag, uint flags);
+
+fragment_t *
+fragment_pclookup(dcontext_t *dcontext, cache_pc pc, fragment_t *wrapper);
+
+/* Performs a pclookup and if the result is a coarse-grain fragment, allocates
+ * a new fragment_t+linkstubs.
+ * Returns in alloc whether the returned fragment_t was allocated and needs to be
+ * freed by the caller via fragment_free().
+ * If no result is found, alloc is set to false.
+ */
+fragment_t *
+fragment_pclookup_with_linkstubs(dcontext_t *dcontext, cache_pc pc,
+                                 /*OUT*/bool *alloc);
+
+#ifdef DEBUG
+fragment_t *
+fragment_pclookup_by_htable(dcontext_t *dcontext, cache_pc pc, fragment_t *wrapper);
+#endif
+
+void
+fragment_shift_fcache_pointers(dcontext_t *dcontext, fragment_t *f, ssize_t shift,
+                               cache_pc start, cache_pc end, size_t old_size);
+
+fragment_t *
+fragment_add_ibl_target(dcontext_t *dcontext, app_pc tag, ibl_branch_type_t branch_type);
+
+/* future fragments */
+future_fragment_t *
+fragment_create_and_add_future(dcontext_t *dcontext, app_pc tag, uint flags);
+
+void
+fragment_delete_future(dcontext_t *dcontext, future_fragment_t *fut);
+
+future_fragment_t *
+fragment_lookup_future(dcontext_t *dcontext, app_pc tag);
+
+future_fragment_t *
+fragment_lookup_private_future(dcontext_t *dcontext, app_pc tag);
+
+#ifdef RETURN_AFTER_CALL
+app_pc
+fragment_after_call_lookup(dcontext_t *dcontext, app_pc tag);
+void
+fragment_add_after_call(dcontext_t *dcontext, app_pc tag);
+void
+fragment_flush_after_call(dcontext_t *dcontext, app_pc tag);
+uint
+invalidate_after_call_target_range(dcontext_t *dcontext, 
+                                   app_pc text_start, app_pc text_end);
+#endif /* RETURN_AFTER_CALL */
+
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+/* case 9672: we split our RCT and RAC targets into per-module tables.
+ * To support sharing, we separate the persisted from the live.
+ * Typedef is in globals.h
+ */
+struct _rct_module_table_t {
+    app_pc_table_t *persisted_table;
+    app_pc_table_t *live_table;
+    /* Optimization: to avoid walking the table to find entries in a
+     * coarse unit's region we track the max and min entries in the
+     * live table.  In the common case we can then do a straight copy
+     */
+    app_pc live_min;
+    app_pc live_max;
+};
+
+# ifdef LINUX
+extern rct_module_table_t rct_global_table;
+# endif
+
+void
+rct_module_table_free(dcontext_t *dcontext, rct_module_table_t *permod, app_pc modpc);
+
+bool
+rct_module_table_set(dcontext_t *dcontext, app_pc modpc, app_pc_table_t *table,
+                     rct_type_t which);
+
+void
+rct_module_table_persisted_invalidate(dcontext_t *dcontext, app_pc modpc);
+
+bool
+rct_module_persisted_table_exists(dcontext_t *dcontext, app_pc modpc,
+                                  rct_type_t which);
+
+uint
+rct_module_live_entries(dcontext_t *dcontext, app_pc modpc, rct_type_t which);
+
+app_pc_table_t *
+rct_module_table_copy(dcontext_t *dcontext, app_pc modpc, rct_type_t which,
+                      app_pc limit_start, app_pc limit_end);
+
+void
+rct_table_free(dcontext_t *dcontext, app_pc_table_t *table, bool free_data);
+
+app_pc_table_t *
+rct_table_copy(dcontext_t *dcontext, app_pc_table_t *src);
+
+app_pc_table_t *
+rct_table_merge(dcontext_t *dcontext, app_pc_table_t *src1, app_pc_table_t *src2);
+
+uint
+rct_table_persist_size(dcontext_t *dcontext, app_pc_table_t *table);
+
+bool
+rct_table_persist(dcontext_t *dcontext, app_pc_table_t *table, file_t fd);
+
+app_pc_table_t *
+rct_table_resurrect(dcontext_t *dcontext, byte *mapped_table, rct_type_t which);
+
+#endif /* RETURN_AFTER_CALL || RCT_IND_BRANCH */
+
+#ifdef NATIVE_RETURN
+void
+fragment_add_deleted(dcontext_t *dcontext, fragment_t *f);
+#endif
+
+#if defined(CLIENT_INTERFACE) && defined(CLIENT_SIDELINE)
+/* synchronization routine for sideline thread */
+void
+fragment_get_fragment_delete_mutex(dcontext_t *dcontext);
+
+void
+fragment_release_fragment_delete_mutex(dcontext_t *dcontext);
+#endif
+
+/*******************************************************************************
+ * COARSE-GRAIN FRAGMENT HASHTABLE
+ */
+
+/* N.B.: if you change the coarse_table_t struct you must increase
+ * PERSISTENT_CACHE_VERSION!
+ */
+typedef struct _app_to_cache_t {
+    app_pc app;
+    /* cache is absolute pc for non-frozen units, but a relative offset for frozen.
+     * PR 203915: should we split into separate instantation to
+     * use a 32-bit offset for frozen on x64?  For now not worth it.
+     */
+    cache_pc cache;
+} app_to_cache_t;
+
+/* 3 macros w/ name and types are duplicated in fragment.c -- keep in sync */
+#define NAME_KEY coarse
+#define ENTRY_TYPE app_to_cache_t
+/* not defining HASHTABLE_USE_LOOKUPTABLE */
+#define CUSTOM_FIELDS /* none */
+#define HASHTABLEX_HEADER 1
+#include "hashtablex.h"
+#undef HASHTABLEX_HEADER
+
+void
+fragment_coarse_htable_create(coarse_info_t *info, uint init_capacity,
+                              uint init_th_capacity);
+
+void
+fragment_coarse_htable_free(coarse_info_t *info);
+
+/* Merges the main and th htables from info1 and info2 into new htables for dst.
+ * If !add_info2, makes room for but does not add entries from info2.
+ * If !add_th_htable, creates but does not add entries to dst->th_htable.
+ */
+void
+fragment_coarse_htable_merge(dcontext_t *dcontext, coarse_info_t *dst,
+                             coarse_info_t *info1, coarse_info_t *info2,
+                             bool add_info2, bool add_th_htable);
+
+uint
+fragment_coarse_num_entries(coarse_info_t *info);
+
+/* Add coarse fragment represented by wrapper f to its hashtable */
+void
+fragment_coarse_add(dcontext_t *dcontext, coarse_info_t *info,
+                    app_pc tag, cache_pc cache);
+
+void
+fragment_coarse_th_add(dcontext_t *dcontext, coarse_info_t *info,
+                       app_pc tag, cache_pc cache);
+
+void
+fragment_coarse_th_unlink_and_add(dcontext_t *dcontext, app_pc tag,
+                                  cache_pc stub_pc, cache_pc body_pc);
+
+#ifdef DEBUG
+/* only exported to allow an assert to avoid rank order issues in
+ * push_pending_freeze() */
+void
+coarse_body_from_htable_entry(dcontext_t *dcontext, coarse_info_t *info,
+                              app_pc tag, cache_pc res,
+                              cache_pc *stub_pc_out/*OUT*/,
+                              cache_pc *body_pc_out/*OUT*/);
+#endif
+
+void
+fragment_coarse_lookup_in_unit(dcontext_t *dcontext, coarse_info_t *info, app_pc tag,
+                               cache_pc *stub_pc/*OUT*/, cache_pc *body_pc/*OUT*/);
+
+cache_pc
+fragment_coarse_lookup(dcontext_t *dcontext, app_pc tag);
+
+void
+fragment_coarse_wrapper(fragment_t *wrapper, app_pc tag, cache_pc body_pc);
+
+fragment_t *
+fragment_coarse_lookup_wrapper(dcontext_t *dcontext, app_pc tag, fragment_t *wrapper);
+
+fragment_t *
+fragment_lookup_fine_and_coarse(dcontext_t *dcontext, app_pc tag,
+                                fragment_t *wrapper, linkstub_t *last_exit);
+
+fragment_t *
+fragment_lookup_fine_and_coarse_sharing(dcontext_t *dcontext, app_pc tag,
+                                        fragment_t *wrapper, linkstub_t *last_exit,
+                                        uint share_flags);
+
+coarse_info_t *
+get_fragment_coarse_info(fragment_t *f);
+
+bool
+coarse_is_trace_head_in_own_unit(dcontext_t *dcontext, app_pc tag, cache_pc stub,
+                                 cache_pc body_in, bool body_valid,
+                                 coarse_info_t *info_in);
+
+bool
+fragment_coarse_replace(dcontext_t *dcontext, coarse_info_t *info, app_pc tag,
+                        cache_pc new_value);
+
+/* Returns the tag for the coarse fragment whose body contains pc, as well as the
+ * body pc in the optional OUT param.
+ */
+app_pc
+fragment_coarse_pclookup(dcontext_t *dcontext, coarse_info_t *info, cache_pc pc,
+                         /*OUT*/cache_pc *body);
+
+/* Returns the tag for the coarse fragment whose body _begins at_ pc */
+app_pc
+fragment_coarse_entry_pclookup(dcontext_t *dcontext, coarse_info_t *info,
+                               cache_pc pc);
+
+void
+fragment_coarse_unit_freeze(dcontext_t *dcontext,
+                            coarse_freeze_info_t *freeze_info);
+
+uint
+fragment_coarse_htable_persist_size(dcontext_t *dcontext, coarse_info_t *info,
+                                    bool cache_table);
+
+bool
+fragment_coarse_htable_persist(dcontext_t *dcontext,  coarse_info_t *info,
+                               bool cache_table, file_t fd);
+
+void
+fragment_coarse_htable_resurrect(dcontext_t *dcontext,  coarse_info_t *info,
+                                 bool cache_table, byte *mapped_table);
+
+/*******************************************************************************/
+
+void
+fragment_output(dcontext_t *dcontext, fragment_t *f);
+
+bool
+fragment_overlaps(dcontext_t *dcontext, fragment_t *f,
+                  byte *region_start, byte *region_end, bool page_only,
+                  overlap_info_t *info_res, app_pc *bb_tag);
+
+bool
+is_self_flushing(void);
+
+bool
+is_self_allsynch_flushing(void);
+
+bool
+is_self_couldbelinking(void);
+
+/* N.B.: can only call if target thread is suspended or waiting for flush */
+bool
+is_couldbelinking(dcontext_t *dcontext);
+
+/* Returns false iff watch ends up being deleted */
+bool
+enter_nolinking(dcontext_t *dcontext, fragment_t *watch, bool temporary);
+
+/* Returns false iff watch ends up being deleted */
+bool
+enter_couldbelinking(dcontext_t *dcontext, fragment_t *watch, bool temporary);
+
+void
+enter_threadexit(dcontext_t *dcontext);
+
+uint
+get_flushtime_last_update(dcontext_t *dcontext);
+
+void
+set_flushtime_last_update(dcontext_t *dcontext, uint val);
+
+/* caller must hold shared_cache_flush_lock */
+void
+increment_global_flushtime(void);
+
+void
+set_at_syscall(dcontext_t *dcontext, bool val);
+
+bool
+get_at_syscall(dcontext_t *dcontext);
+
+/****************************************************************************
+ * FLUSHING
+ *
+ * Typical use is to flush a memory region in
+ * flush_fragments_and_remove_region().  If custom executable areas editing
+ * is required, the memory-region-flushing pair
+ * flush_fragments_in_region_start() and flush_fragments_in_region_finish()
+ * should be used (they MUST be used together). If no executable area removal
+ * is needed use flush_fragments_from_region().
+ *
+ * Alternatively, the trio (also not usable individually) 
+ *   1) flush_fragments_synch_unlink_priv
+ *   2) flush_fragments_unlink_shared
+ *   3) flush_fragments_end_synch
+ * provide flushing for non-memory regions via a list of fragments.  The
+ * list should be chained by next_vmarea and vm_area_remove_fragment()
+ * should already have been called on each fragment in the list.  This
+ * usage does not involve the executable_areas lock at all.
+ *
+ * The exec_invalid parameter must be set to indicate whether the
+ * executable area is being invalidated as well or this is just a capacity
+ * flush (or a flush to change instrumentation).
+ *
+ * Possibly the thread_initexit_lock (if there are fragments to
+ * flush), and always executable_areas lock for region flushing, ARE
+ * HELD IN BETWEEN the routines, and no thread is could_be_linking()
+ * in between.
+ * WARNING: case 8572: the caller owning the thread_initexit_lock
+ * is incompatible w/ suspend-the-world flushing!
+ */
+/* If size>0, returns whether there is an overlap; if not, no synch is done.
+ * If size==0, synch is always performed and true is always returned.
+ */
+bool
+flush_fragments_synch_unlink_priv(dcontext_t *dcontext, app_pc base, size_t size,
+                                  bool own_initexit_lock, bool exec_invalid,
+                                  bool force_synchall _IF_DGCDIAG(app_pc written_pc));
+
+void
+flush_fragments_unlink_shared(dcontext_t *dcontext, app_pc base, size_t size,
+                              fragment_t *list _IF_DGCDIAG(app_pc written_pc));
+
+/* Invalidates (does not remove) shared fragment f from the private/shared
+ * ibl tables.  Can only be called in flush stage 2.
+ */
+void
+flush_invalidate_ibl_shared_target(dcontext_t *dcontext, fragment_t *f);
+
+void
+flush_fragments_end_synch(dcontext_t *dcontext, bool keep_initexit_lock);
+
+void
+flush_fragments_in_region_start(dcontext_t *dcontext, app_pc base, size_t size,
+                                bool own_initexit_lock, bool free_futures,
+                                bool exec_invalid, bool force_synchall
+                                _IF_DGCDIAG(app_pc written_pc));
+
+void
+flush_fragments_in_region_finish(dcontext_t *dcontext, bool keep_initexit_lock);
+
+void
+flush_fragments_and_remove_region(dcontext_t *dcontext, app_pc base, size_t size,
+                                  bool own_initexit_lock, bool free_futures);
+
+void
+flush_fragments_from_region(dcontext_t *dcontext, app_pc base, size_t size,
+                            bool force_synchall);
+
+void
+flush_fragments_custom_list(dcontext_t *dcontext, fragment_t *list,
+                            bool own_initexit_lock, bool exec_invalid);
+
+/* Invalidate all fragments in all caches.  Currently executed
+ * fragments may be alive until they reach an exit.  It should be used
+ * for correctness when finishing existing fragment execution is
+ * acceptable. There are no guarantees on actual deletion from this
+ * interface.
+ */
+void
+invalidate_code_cache(void);
+
+/* Flushes all areas stored in the vector toflush.
+ * Synch is up to caller, but as locks cannot be held when flushing,
+ * toflush needs to be thread-private.
+ */
+void
+flush_vmvector_regions(dcontext_t *dcontext, vm_area_vector_t *toflush,
+                       bool free_futures, bool exec_invalid);
+
+/*
+ ****************************************************************************/
+
+/* locks must be exported for DEADLOCK_AVOIDANCE */
+/* allows independent sequences of flushes and delayed deletions */
+extern mutex_t shared_cache_flush_lock;
+/* Global count of flushes, used as a timestamp for shared deletion.
+ * Writes to it are protected by shared_cache_flush_lock.
+ */
+extern uint flushtime_global;
+
+#ifdef CLIENT_INTERFACE
+extern mutex_t client_flush_request_lock;
+extern client_flush_req_t *client_flush_requests;
+#endif
+
+#ifdef PROFILE_RDTSC
+void
+profile_fragment_enter(fragment_t *f, uint64 end_time);
+
+void
+profile_fragment_dispatch(dcontext_t *dcontext);
+#endif
+
+void
+fragment_self_write(dcontext_t *dcontext);
+
+#ifdef SHARING_STUDY
+void
+print_shared_stats(void);
+#endif
+
+#define PROTECT_FRAGMENT_ENABLED(flags) \
+    (TEST(FRAG_SHARED, (flags)) ?                         \
+     TEST(SELFPROT_GLOBAL, dynamo_options.protect_mask) : \
+     TEST(SELFPROT_LOCAL, dynamo_options.protect_mask))
+
+#ifdef DEBUG
+void study_all_hashtables(dcontext_t *dcontext);
+#endif /* DEBUG */
+
+#ifdef HASHTABLE_STATISTICS
+/* in x86/interp.c */
+int
+append_ib_trace_last_ibl_exit_stat(dcontext_t *dcontext, instrlist_t *trace,
+                                   app_pc speculate_next_tag);
+
+static INLINE_ONCE 
+hashtable_statistics_t*
+get_ibl_per_type_statistics(dcontext_t *dcontext, ibl_branch_type_t branch_type)
+{
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+    return &pt->trace_ibt[branch_type].unprot_stats->
+        trace_ibl_stats[branch_type];
+}
+#endif /* HASHTABLE_STATISTICS */
+
+
+/* DR_API EXPORT TOFILE dr_tools.h */
+/* DR_API EXPORT BEGIN */
+
+/****************************************************************************
+ * BINARY TRACE DUMP FORMAT
+ */
+
+/**<pre>
+ * Binary trace dump format:
+ * the file starts with a tracedump_file_header_t
+ * then, for each trace:
+     struct _tracedump_trace_header
+     if num_bbs > 0 # tracedump_origins
+       foreach bb:
+           app_pc tag;
+           int bb_code_size;
+           byte code[bb_code_size];
+     endif
+     foreach exit:
+       struct _tracedump_stub_data
+       if linkcount_size > 0
+         linkcount_type_t count; # sizeof == linkcount_size
+       endif
+       if separate from body
+       (i.e., exit_stub < cache_start_pc || exit_stub >= cache_start_pc+code_size):
+           byte stub_code[15]; # all separate stubs are 15
+       endif
+     endfor
+     byte code[code_size];
+ * if the -tracedump_threshold option (deprecated) was specified:
+     int num_below_treshold
+     linkcount_type_t count_below_threshold
+   endif
+</pre>
+ */
+typedef struct _tracedump_file_header_t {
+    int version;           /**< The DynamoRIO version that created the file. */
+    bool x64;              /**< Whether a 64-bit DynamoRIO library created the file. */
+    int linkcount_size;    /**< Size of the linkcount (linkcounts are deprecated). */
+} tracedump_file_header_t;
+
+/** Header for an individual trace in a binary trace dump file. */
+typedef struct _tracedump_trace_header_t {
+    int frag_id;           /**< Identifier for the trace. */
+    app_pc tag;            /**< Application address for start of trace. */
+    app_pc cache_start_pc; /**< Code cache address of start of trace. */
+    int entry_offs;        /**< Offset into trace of normal entry. */
+    int num_exits;         /**< Number of exits from the trace. */
+    int code_size;         /**< Length of the trace in the code cache. */
+    uint num_bbs;          /**< Number of constituent basic blocks making up the trace. */
+    bool x64;              /**< Whether the trace contains 64-bit code. */
+} tracedump_trace_header_t;
+
+/** Size of tag + bb_code_size fields for each bb. */
+#define BB_ORIGIN_HEADER_SIZE (sizeof(app_pc)+sizeof(int))
+
+/**< tracedump_stub_data_t.stub_size will not exceed this value. */
+#define SEPARATE_STUB_MAX_SIZE IF_X64_ELSE(23, 15)
+
+/** The format of a stub in a trace dump file. */
+typedef struct _tracedump_stub_data {
+    int cti_offs;   /**< Offset from the start of the fragment. */
+    /* stub_pc is an absolute address, since can be separate from body. */
+    app_pc stub_pc; /**< Code cache address of the stub. */
+    app_pc target;  /**< Target of the stub. */
+    bool linked;    /**< Whether the stub is linked to its target. */
+    int stub_size;  /**< Length of stub_code array */
+    /****** the rest of the fields are optional and may not be present! ******/
+    union {
+        uint count32; /**< 32-bit exit execution count. */
+        uint64 count64; /**< 64-bit exit execution count. */
+    } count; /**< Which field is present depends on the first entry in
+              * the file, which indicates the linkcount size. */
+    /** Code for exit stubs.  Only present if:
+     *   stub_pc < cache_start_pc ||
+     *   stub_pc >= cache_start_pc+code_size). 
+     * The actual size of the array varies and is indicated by the stub_size field.
+     */
+    byte stub_code[1/*variable-sized*/];
+} tracedump_stub_data_t;
+
+/** The last offset into tracedump_stub_data_t of always-present fields. */
+#define STUB_DATA_FIXED_SIZE (offsetof(tracedump_stub_data_t, count))
+
+/****************************************************************************/
+/* DR_API EXPORT END */
+
+#endif /* _FRAGMENT_H_ */
diff --git a/core/gen_event_strings.pl b/core/gen_event_strings.pl
new file mode 100755
index 000000000..2b3a84b43
--- /dev/null
+++ b/core/gen_event_strings.pl
@@ -0,0 +1,90 @@
+#!/usr/local/bin/perl
+
+# **********************************************************
+# Copyright (c) 2004-2009 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# Copyright (c) 2004-2007 Determina Corp.
+
+### gen_event_strings.pl
+### reads in a .mc file and creates a header file with fmt string constants
+### matching those in the .mc file (in printf style format)
+
+$usage = "Usage : gen_event_strings <mc file> <target file>";
+
+#print stderr "$#ARGV\n";
+#print stderr "$ARGV[0]\n";
+#print stderr "$ARGV[1]\n";
+
+if ($#ARGV != 1) {
+    print "$usage";
+    exit 0;
+}
+
+open(IN, "<$ARGV[0]") || die("Error : gen_event_strings : couldn't open .mc file ","$ARGV[0]","\n");
+
+open(OUT, ">$ARGV[1]") || die("Error : gen_event_strings : couldn't open output file ","$ARGV[0]","\n");
+
+while ($line = <IN>) {
+    $line =~ s/\r?\n$//; # chomp not sufficient for cross-plaform perl
+    if ($line =~ /^SymbolicName *= *(\S+)\s*$/) {
+        $name = $1 . "_STRING";
+        $val = "";
+        if ($line = <IN>) {
+            $line =~ s/\r?\n$//; # chomp not sufficient for cross-plaform perl
+            if ($line !~ /^Language *= *English\s*$/) {
+                # may need to remove once have multilingual support? FIXME
+                die("Error : gen_event_strings : pattern match for Language=English failed for ",$name,"\n");
+            }
+            $success = 0;
+            while ($line = <IN>) {
+                $line =~ s/\r?\n$//; # chomp not sufficient for cross-plaform perl
+                if ($line =~ /^\.\s*$/) {
+                    $sucess = 1;
+                    last;
+                }
+                $val .= $line;
+                $val .= "\\n";
+            }
+            # remove final newline
+            chop $val;
+            chop $val;
+            if ($sucess == 0) {
+                die("Error : file ended unexpectedly");
+            }
+            # FIXME we only need %s for now, but following should be general
+            # as long as the letter codes match (seems to be the case), what 
+            # about 1, 2, 3 can the .mc fmt string reorder the arguments?
+            $val =~ s/%.!(.)!/%\1/g;
+            print OUT "#define ",$name," \"",$val,"\"\n";
+        } else {
+            die("Error : file ended unexpectedly");
+        }
+    }
+}
diff --git a/core/globals.h b/core/globals.h
new file mode 100644
index 000000000..3fbc16a6a
--- /dev/null
+++ b/core/globals.h
@@ -0,0 +1,904 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * globals.h - global defines and typedefs, included in all files
+ */
+
+#ifndef _GLOBALS_H_
+#define _GLOBALS_H_ 1
+
+#ifdef WINDOWS
+/* Vista SDK compiler default is to set NTDDI_VERSION to NTDDI_LONGHORN, causing
+ * problems w/ new Vista-only flags like in PROCESS_ALL_ACCESS in win32/injector.c.
+ * The problematic flag there, PROCESS_QUERY_LIMITED_INFORMATION, is a subset of
+ * PROCESS_QUERY_INFORMATION, so we don't lose anything by not asking for it on Vista.
+ * But if they add new non-subset flags in the future we'd need dynamic dispatch,
+ * as earlier Windows versions give access denied on unknown flags!
+ */
+#define _WIN32_WINNT _WIN32_WINNT_NT4 /* ==0x0400; NTDDI_VERSION is set from this */
+
+#  define WIN32_LEAN_AND_MEAN
+/* Exclude rarely-used stuff from Windows headers */
+
+/* Case 1167 - bumping up warning level to 4 - work in progress FIXME: */
+#pragma warning( disable : 4054) //from function pointer 'void (__cdecl *)(void )' to data pointer 'unsigned char *'
+#pragma warning( disable : 4100) //'envp' : unreferenced formal parameter
+#pragma warning( disable : 4127) //conditional expression is constant (majority of warnings - 2078)
+#pragma warning( disable : 4189) //'start_pc' : local variable is initialized but not referenced
+#pragma warning( disable : 4204) //nonstandard extension used : non-constant aggregate initializer
+#pragma warning( disable : 4210) //nonstandard extension used : function given file scope
+#pragma warning( disable : 4505) //unreferenced local function has been removed
+#pragma warning( disable : 4702) //unreachable code (should be disabled DEBUG=0, e.g. for INTERNAL_OPTION test)
+
+/**************************************************/
+/* warnings on compiling with VC 8.0, all on VC or PlatformSDK header files */
+
+/* shows up in buildtools/VC/8.0/dist/VC/include/vadefs.h
+ * supposed to include identifier in the pop pragma and they don't
+ */
+#pragma warning( disable : 4159) // #pragma pack has popped previously pushed identifier
+
+/* FIXME case 191729: this is coming from our own code.  We could
+ * switch to the _s versions when on Windows.
+ */
+#pragma warning( disable : 4996) //'sscanf' was declared deprecated
+
+/**************************************************/
+
+#endif
+
+#include "globals_shared.h"
+
+/* currently we always export statistics structure */
+#define DYNAMORIO_STATS_EXPORTS 1
+
+/* we only export IR interface if CLIENT_INTERFACE is defined */
+#ifdef CLIENT_INTERFACE
+/* in Makefile, we define these (so that genapi.pl, etc. get them):
+ *    define DYNAMORIO_IR_EXPORTS 1
+ *    define CUSTOM_EXIT_STUBS 1
+ *    define CUSTOM_TRACES 1
+ * CUSTOM_TRACES_RET_REMOVAL is aggressive -- assumes calling convention kept
+ * Only useful if custom traces are doing inlining => do not define for external
+ * release, or even by default for internal since it's always on even if
+ * not building custom traces!
+ */
+#endif /* CLIENT_INTERFACE */
+
+#  ifdef WINDOWS
+#    define DYNAMORIO_EXPORT __declspec(dllexport)
+#  elif defined(USE_VISIBILITY_ATTRIBUTES)
+/* PR 262804: we use "protected" instead of "default" to ensure our
+ * own uses won't be preempted.  Note that for DR_APP_API in
+ * lib/dr_app.h we get a link error trying to use "protected": but we
+ * don't use dr_app_* internally anyway, so leaving as default.
+ */
+#    define DYNAMORIO_EXPORT __attribute__ ((visibility ("protected")))
+#  else
+/* visibility attribute not available in gcc < 3.3 (we use linker script) */
+#    define DYNAMORIO_EXPORT 
+#  endif
+
+#ifdef DYNAMORIO_IR_EXPORTS
+#  define DR_API DYNAMORIO_EXPORT
+#else
+#  define DR_API 
+#endif
+#ifdef UNSUPPORTED_API
+#  define DR_UNS_API DR_API
+#else
+#  define DR_UNS_API /* nothing */
+#endif
+
+#define INLINE_ONCE inline
+
+#include <stdlib.h>
+#include <stdio.h>
+
+/* N.B.: some of these typedefs and defines are duplicated in
+ * lib/globals_shared.h!
+ */
+
+#ifdef WINDOWS
+#include <windows.h>
+typedef unsigned long ulong;
+typedef unsigned short ushort;
+/* We can't put this in globals_shared.h b/c it needs windows.h and
+ * not all users of globals_shared.h want that included, so we
+ * duplicate it here (we do have Linux file_t stuff in globals_shared.h)
+ */
+/* since a FILE cannot be used outside of the DLL it was created in,
+ * we have to use HANDLE on Windows
+ * we hide the distinction behind the file_t type
+ */
+typedef HANDLE file_t;
+#define INVALID_FILE INVALID_HANDLE_VALUE
+#define STDOUT (file_t)get_stdout_handle()
+#define STDERR (file_t)get_stderr_handle()
+#define DIRSEP '\\'
+#define ALT_DIRSEP '/'
+
+#else /* LINUX */
+/* uint, ushort, and ulong are in types.h */
+#include <sys/types.h> /* for wait */
+#define DIRSEP '/'
+#define ALT_DIRSEP DIRSEP
+#endif
+
+/* FIXME: what is range of thread_id_t on linux and on win32?
+ * linux routines use -1 as sentinel, right?
+ * on win32, are ids only 16 bits?
+ * if so, change thread_id_t to be a signed int and use -1?
+ * For now, based on observation, no process on linux and no thread on windows
+ * has id 0 (on windows even a new thread in its init apc has a non-0 id)
+ */
+#define INVALID_THREAD_ID  0
+
+typedef unsigned char uchar;
+typedef byte * cache_pc;  /* fragment cache pc */
+typedef size_t app_rva_t;   /* application offset from module base; PE32+ modules are
+                           * limited to 2GB, but not elf x64 med/large code model. */
+
+#define SUCCESS 0
+#define FAILURE 1
+
+/* macros to make conditional compilation look prettier */
+#ifdef DGC_DIAGNOSTICS
+# define _IF_DGCDIAG(x) , x
+# define IF_DGCDIAG_ELSE(x, y) x
+#else
+# define _IF_DGCDIAG(x)
+# define IF_DGCDIAG_ELSE(x, y) y
+#endif
+
+/* make sure defines are consistent */
+#ifndef X86
+#error Must define X86, no other platforms are supported
+#endif
+
+#if defined(PAPI) && defined(WINDOWS)
+# error PAPI does not work on WINDOWS
+#endif
+
+#if defined(DCONTEXT_IN_EDI) && !defined(STEAL_REGISTER)
+# error Must steal register to keep dcontext in edi
+#endif
+
+#if defined(SIDELINE_COUNT_STUDY)
+# if !defined(PROFILE_LINKCOUNT) || !defined(SIDELINE)
+#  error SIDELINE_COUNT_STUDY requires PROFILE_LINKCOUNT and defined(SIDELINE)
+# endif
+#endif
+
+#ifdef NATIVE_RETURN
+# ifndef CUSTOM_EXIT_STUBS
+#  error NATIVE_RETURN requires CUSTOM_EXIT_STUBS
+# endif
+/* NOTE NATIVE_RETURN requires -unsafe_ignore_eflags */
+#endif
+
+#ifdef DGC_DIAGNOSTICS
+# ifndef PROGRAM_SHEPHERDING
+#  error DGC_DIAGNOSTICS requires PROGRAM_SHEPHERDING
+# endif
+# ifndef DEBUG
+#  error DGC_DIAGNOSTICS requires DEBUG
+# endif
+#endif
+
+#ifndef PROGRAM_SHEPHERDING
+# ifdef SIMULATE_ATTACK
+#  error SIMULATE_ATTACK requires PROGRAM_SHEPHERDING
+# endif
+#endif
+
+struct _opnd_t;
+typedef struct _opnd_t opnd_t;
+struct _instr_t;
+typedef struct _instr_t instr_t;
+struct _instr_list_t;
+struct _fragment_t;
+typedef struct _fragment_t fragment_t;
+struct _future_fragment_t;
+typedef struct _future_fragment_t future_fragment_t;
+struct _trace_t;
+typedef struct _trace_t trace_t;
+struct _linkstub_t;
+typedef struct _linkstub_t linkstub_t;
+struct _dcontext_t;
+typedef struct _dcontext_t dcontext_t;
+struct vm_area_vector_t;
+typedef struct vm_area_vector_t vm_area_vector_t;
+struct _coarse_info_t;
+typedef struct _coarse_info_t coarse_info_t;
+struct _coarse_freeze_info_t;
+typedef struct _coarse_freeze_info_t coarse_freeze_info_t;
+struct _module_data_t;
+/* DR_API EXPORT TOFILE dr_defines.h */
+/* DR_API EXPORT BEGIN */
+typedef struct _instr_list_t instrlist_t;
+typedef struct _module_data_t module_data_t;
+/* DR_API EXPORT END */
+
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+struct _rct_module_table_t;
+typedef struct _rct_module_table_t rct_module_table_t;
+
+typedef enum { 
+    RCT_RAC = 0,
+    RCT_RCT,
+    RCT_NUM_TYPES,
+} rct_type_t;
+#endif
+
+#ifdef CLIENT_INTERFACE
+typedef struct _client_to_do_list_t {
+    /* used to make a list of fragments to delete/replace */
+    /* deletes frag at tag if ilist is null else replaces it with ilist */
+    instrlist_t *ilist;
+    app_pc tag;
+    struct _client_to_do_list_t *next;
+} client_todo_list_t;
+
+/* Clients need a separate list to queue up flush requests from dr_flush() */
+typedef struct _client_flush_req_t {
+    app_pc start;
+    size_t size;
+    uint   flush_id; /* client supplied identifier for this flush */
+    void (*flush_callback)(int);
+    struct _client_flush_req_t *next;
+} client_flush_req_t;
+
+/* for -thin_client we don't allocate client_data currently, also client_data could be
+ * NULL during thread startup or teardown (i.e. mutex_wait_contended_lock() usage) */
+#define IS_CLIENT_THREAD(dcontext) \
+    ((dcontext) != NULL && (dcontext)->client_data != NULL && \
+     (dcontext)->client_data->is_client_thread)
+
+/* Client interface-specific data for dcontexts */
+typedef struct _client_data_t {
+    /* field for use by user via exported API */
+    void *         user_field;
+    client_todo_list_t * to_do;
+    client_flush_req_t *flush_list;
+# ifdef CLIENT_SIDELINE
+    mutex_t            sideline_mutex;
+    mutex_t            sideline_heap_lock;
+# endif
+    /* fields for doing release and debug build checks against erroneous API usage */
+    module_data_t      *no_delete_mod_data;
+
+    /* Client-owned threads, such as a client nudge thread, require special
+     * synchronization support. is_client_thread means that the thread is currently
+     * completely owned by the client.  client_thread_safe_for_sync is use to mark
+     * client-owned threads that are safe for synch_with_all_threads synchronization but
+     * are in dynamo/native code (such as in dr_thread_yield(), dr_sleep(),
+     * dr_mutex_lock() and dr_messagebox()).  Note it does not need to be set when
+     * the client is in client library code.  For dr_mutex_lock() we set client_grab_mutex
+     * to the client mutex that is being locked so that we can set
+     * client_thread_safe_for_sync only around the actual wait.
+     * FIXME - PR 231301, we may need a way for clients that call ntdll directly to
+     * mark client_thread_safe_for_synch for client-owned threads when calling out to
+     * ntdll. Especially if they're calling system calls that wait or take a long time
+     * to finish etc. Applies to generated code and other libraries called by the client
+     * lib as well.
+     */
+    bool           is_client_thread; /* NOTE - use IS_CLIENT_THREAD() */
+    bool           client_thread_safe_for_synch;
+    void           *client_grab_mutex;
+
+    /* flags for asserts on linux and for getting param base right on windows */
+    bool           in_pre_syscall;
+    bool           in_post_syscall;
+    /* flag for dr_syscall_invoke_another() */
+    bool           invoke_another_syscall;
+} client_data_t;
+#endif
+
+typedef struct _thread_record_t {
+    thread_id_t id;     /* linux: pid, win32: thread id */
+#ifdef WINDOWS
+    HANDLE handle;    /* win32 thread handle */
+#endif
+    uint num;         /* creation ordinal */
+    bool under_dynamo_control; /* used for deciding whether to intercept events */
+    dcontext_t *dcontext; /* allows other threads to see this thread's context */
+    struct _thread_record_t * next;
+} thread_record_t;
+
+/* we don't include dr_api.h, that's for external use, we only need _app
+ * (everything in dr_defines.h is duplicated in our own header files)
+ */
+#ifdef DR_APP_EXPORTS
+/* we only export app interface if DR_APP_EXPORTS is defined */
+# include "dr_app.h"
+/* a few always-exported routines are part of the app interface */
+# undef DYNAMORIO_EXPORT
+# define DYNAMORIO_EXPORT DR_APP_API
+#endif
+
+#ifdef PROFILE_LINKCOUNT
+#ifndef LINKCOUNT_64_BITS
+typedef uint linkcount_type_t;
+#else
+typedef uint64 linkcount_type_t;
+#endif
+#endif
+
+#include "heap.h"
+#include "utils.h"
+#include "options.h"
+#include "os_exports.h"
+#include "arch_exports.h"
+#include "vmareas.h"
+#include "instrlist.h"
+#include "dispatch.h"
+
+#include "dr_stats.h"
+
+/* size of each Dynamo thread-private stack */
+#define DYNAMORIO_STACK_SIZE dynamo_options.stack_size
+
+/* global flags */
+extern bool automatic_startup;   /* ignore start/stop api, run entire program */
+extern bool control_all_threads; /* ok for "weird" things to happen -- not all
+                                    threads are under our control */
+extern bool dynamo_heap_initialized;  /* has dynamo_heap been initialized? */
+extern bool dynamo_initialized;  /* has dynamo been initialized? */
+extern bool dynamo_exited;       /* has dynamo exited? */
+extern bool dynamo_exited_and_cleaned; /* has dynamo component cleanup started? */
+#ifdef DEBUG
+extern bool dynamo_exited_log_and_stats; /* are stats and logfile shut down? */
+#endif
+extern bool dynamo_resetting;    /* in middle of global reset? */
+extern bool dynamo_all_threads_synched; /* are all other threads suspended safely? */
+
+#if defined(CLIENT_INTERFACE) || defined(STANDALONE_UNIT_TEST)
+extern bool standalone_library;  /* used as standalone library */
+#endif
+#ifdef LINUX
+extern bool post_execve;         /* have we performed an execve? */
+#endif
+
+/* global instance of statistics struct */
+extern dr_statistics_t *stats;
+
+/* the process-wide logfile */
+extern file_t main_logfile;
+
+/* initial stack so we don't have to use app's */
+extern byte *  initstack;
+extern mutex_t   initstack_mutex;
+extern byte *  initstack_app_xsp;
+
+#if defined(WINDOWS) && defined(STACK_GUARD_PAGE)
+/* PR203701: separate stack for error reporting when the dstack is exhausted */
+extern byte *  exception_stack;
+#endif
+
+/* keeps track of how many threads are in cleanup_and_terminate so that we know
+ * if any threads could still be using shared resources even if they aren't on
+ * the all_threads list */
+extern int exiting_thread_count;
+
+bool is_on_initstack(byte *esp);
+
+bool is_on_dstack(dcontext_t *dcontext, byte *esp);
+bool is_currently_on_dstack(dcontext_t *dcontext);
+
+#ifdef WINDOWS
+extern bool    dr_early_injected;
+extern int     dr_early_injected_location;
+extern bool    dr_injected_primary_thread;
+extern bool    dr_injected_secondary_thread;
+extern bool    dr_late_injected_primary_thread;
+
+#endif
+#ifdef RETURN_AFTER_CALL
+extern bool    dr_preinjected;
+#endif
+#ifdef DR_APP_EXPORTS
+/* flags to indicate when DR is being initialized / exited using the API */
+extern bool    dr_api_entry;
+extern bool    dr_api_exit;
+#endif
+
+/* in dynamo.c */
+/* 9-bit addressed hash table takes up 2K, has capacity of 512
+ * we never resize, assuming won't be seeing more than a few hundred threads
+ */
+#define ALL_THREADS_HASH_BITS 9
+extern thread_record_t **all_threads;
+extern mutex_t all_threads_lock;
+DYNAMORIO_EXPORT int dynamorio_app_init(void);
+int dynamorio_app_exit(void);
+#if defined(CLIENT_INTERFACE) || defined(STANDALONE_UNIT_TEST)
+dcontext_t * standalone_init(void);
+void standalone_exit(void);
+#endif
+thread_record_t * thread_lookup(thread_id_t tid);
+void add_thread(IF_WINDOWS_(HANDLE hthread) 
+                thread_id_t tid, bool under_dynamo_control, dcontext_t *dcontext);
+bool remove_thread(IF_WINDOWS_(HANDLE hthread) thread_id_t tid);
+uint get_thread_num(thread_id_t tid);
+int get_num_threads(void);
+void get_list_of_threads(thread_record_t ***list, int *num);
+bool is_thread_known(thread_id_t tid);
+bool is_thread_initialized(void);
+int dynamo_thread_init(void);
+int dynamo_thread_exit(void);
+void dynamo_thread_stack_free_and_exit(byte *stack);
+int dynamo_other_thread_exit(thread_record_t *tr
+                             _IF_WINDOWS(bool detach_stacked_callbacks));
+void dynamo_thread_under_dynamo(dcontext_t *dcontext);
+void dynamo_thread_not_under_dynamo(dcontext_t *dcontext);
+/* used for synch to prevent thread creation/deletion in critical periods */
+extern mutex_t thread_initexit_lock;
+dcontext_t * create_new_dynamo_context(bool initial);
+void initialize_dynamo_context(dcontext_t *dcontext);
+dcontext_t * create_callback_dcontext(dcontext_t *old_dcontext);
+int dynamo_nullcalls_exit(void);
+int dynamo_process_exit(void);
+#ifdef LINUX
+void dynamorio_fork_init(dcontext_t *dcontext);
+#endif
+dr_statistics_t * get_dr_stats(void);
+
+/* functions needed by detach */
+int dynamo_shared_exit(IF_WINDOWS_ELSE_NP(bool detach_stacked_callbacks, void));
+/* perform exit tasks that require full thread data structs */
+void dynamo_process_exit_with_thread_info(void);
+
+/* enter/exit DR hooks */
+void entering_dynamorio(void);
+void exiting_dynamorio(void);
+
+void handle_system_call(dcontext_t *dcontext);
+
+/* self-protection */
+void protect_data_section(uint sec, bool writable);
+#ifdef DEBUG
+const char *get_data_section_name(app_pc pc);
+bool check_should_be_protected(uint sec);
+# ifdef WINDOWS
+bool data_sections_enclose_region(app_pc start, app_pc end);
+# endif
+#endif /* DEBUG */
+
+/* all the locks used to protect shared data structures during multi-operation
+ * sequences, exported so that micro-operations can assert that one is held 
+ */
+extern mutex_t bb_building_lock;
+extern recursive_lock_t change_linking_lock;
+
+/* where the current app thread's control is */
+typedef enum {
+    WHERE_APP=0, 
+    WHERE_INTERP,
+    WHERE_DISPATCH,
+    WHERE_MONITOR,
+    WHERE_SYSCALL_HANDLER,
+    WHERE_SIGNAL_HANDLER,
+    WHERE_TRAMPOLINE,
+    WHERE_CONTEXT_SWITCH,
+    WHERE_IBL,
+    WHERE_FCACHE,
+    WHERE_UNKNOWN,
+#ifdef HOT_PATCHING_INTERFACE
+    WHERE_HOTPATCH,
+#endif
+    WHERE_LAST
+} where_am_i_t;
+
+/* make args easier to read for protection change calls 
+ * since only two possibilities not using new type
+ */
+enum {
+    READONLY=false, 
+    WRITABLE=true
+};
+
+/* To handle TRY/EXCEPT/FINALLY setjmp */
+typedef struct try_except_context_t {
+    /* FIXME: we are using a local dr_jmp_buf which is relatively
+     * small so minimal risk of dstack pressure.  Alternatively, we
+     * can disallow nesting and have a single buffer per dcontext.
+     */
+    dr_jmp_buf_t context;
+
+    struct try_except_context_t *prev_context;
+} try_except_context_t;
+
+typedef struct {
+    /* WARNING: if you change the offsets of any of these fields, 
+     * you must also change the offsets in <arch>/<arch.s> 
+     */
+    dr_mcontext_t mcontext;        /* real machine context (in arch_exports.h) */
+#ifdef LINUX
+    int            errno;           /* errno used for app and dynamo */
+#endif
+    bool at_syscall;                /* for shared deletion syscalls_synch_flush,
+                                     * as well as syscalls handled from dispatch,
+                                     * and for reset to identify when at syscalls
+                                     */
+#ifdef X64
+    /* For Windows, 4 bytes of padding to fill out the 4-byte at_syscall,
+     * since dr_mcontext_t is 8-byte aligned */
+#endif
+} unprotected_context_t;
+
+/* dynamo-specific context associated with each active app thread 
+ * N.B.: make sure to update these routines as necessary if
+ * you add or remove fields:
+ *   create_new_dynamo_context
+ *   create_callback_dcontext
+ *   initialize_dynamo_context
+ *   swap_dcontexts
+ * if you add any pointers to data structures, make sure callback_setup()
+ *   clears them to prevent stale pointers on callback return
+ */
+struct _dcontext_t {
+    /* NOTE: For any field to survive across callback stack switches it must either
+     * be indirected through a modular field or explicitly copied in
+     * create_callback_dcontext() (like the modular fields are).
+     */
+
+    /* WARNING: if you change the offsets of any of these fields, up through
+     * ignore_enterexit, you must also change the offsets in <arch>/<arch.s> 
+     */
+
+    /* if SELFPROT_DCONTEXT, must split dcontext into unprotected and
+     * protected fields depending on whether they must be read-only
+     * when in the code cache.
+     * we waste sizeof(unprotected_context_t) bytes to provide runtime flexibility:
+     */
+    union {
+        /* we use separate_upcontext if 
+         *    (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask))
+         * else we use the inlined upcontext
+         */
+        unprotected_context_t *separate_upcontext;
+        unprotected_context_t upcontext;
+    } upcontext;
+    /* HACK for x86.asm lack of runtime param access: this is either
+     * a self-ptr (to inlined upcontext) or if we separate upcontext it points there.
+     */
+    unprotected_context_t *upcontext_ptr;
+    
+    /* The next application pc to execute.
+     * Also used to store the cache pc to execute when entering the code cache,
+     * and set to the sentinel value BACK_TO_NATIVE_AFTER_SYSCALL for native_exec.
+     * FIXME: change to a union?
+     */
+    app_pc         next_tag;
+
+    linkstub_t *     last_exit;       /* last exit from cache */
+    byte *         dstack;          /* thread-private dynamo stack */
+
+    /* These are expected to retain their values across an entire native execution.
+     * We assume that callbacks and native_exec executions are properly nested, and
+     * we don't share these across callbacks, using the original value on returning.
+     */
+    app_pc         native_exec_retval; /* native_exec return address */
+    app_pc         native_exec_retloc; /* native_exec return address app stack location */
+
+#ifdef RETURN_STACK
+    byte *         rstack;          /* bottom of return stack */
+    byte *         top_of_rstack;   /* top of return stack */
+#endif
+    int            app_errno;       /* storage for app's errno while in dynamo */
+    bool           is_exiting;      /* flag for exiting thread */
+#ifdef WINDOWS
+    /* storage for an extra app value around sysenter system calls for the
+     * case 5441 Sygate interoperability hack */
+    /* FIXME - this needs to be moved into the upcontext as is written to
+     * in cache by ignore/shared_syscall, ramifications? */
+    app_pc         sysenter_storage;
+
+    /* used to avoid enter/exit hooks for certain system calls (see case 4942) */
+    bool           ignore_enterexit;
+#endif
+
+    /* Coarse-grain cache exits require extra state storage as they do not
+     * use per-exit separate data structures
+     */
+    union {
+        /* Indirect branches store on exit the source tag (with the type of
+         * branch coming from fake linkstubs), while direct store the source unit
+         */
+        app_pc src_tag;
+        coarse_info_t *dir_exit;
+    } coarse_exit;
+
+    /************* end of offset-crucial fields *********************/
+
+    /* FIXME: now that we initialize a new thread's dcontext right away, and
+     * a new callback's as well, we should be able to get rid of this
+     */
+    bool           initialized;     /* has this context been used yet? */
+    thread_id_t      owning_thread;
+#ifdef LINUX
+    process_id_t     owning_process; /* handle shared address space w/o shared pid */
+#endif
+    thread_record_t   *thread_record;  /* so don't have to do a thread_lookup */
+    where_am_i_t       whereami;        /* where control is at the moment */
+    void *         allocated_start; /* used for cache alignment */
+    fragment_t *     last_fragment;   /* cached value of linkstub_fragment(last_exit) */
+
+    int            sys_num;         /* used for post_system_call */
+#ifdef WINDOWS
+    reg_t *        sys_param_base;  /* used for post_system_call */
+#endif
+#if defined(LINUX) || defined(X64)
+    reg_t          sys_param0;      /* used for post_system_call */
+    reg_t          sys_param1;      /* used for post_system_call */
+    reg_t          sys_param2;      /* used for post_system_call */
+    reg_t          sys_param3;      /* used for post_system_call */
+#endif
+#ifdef LINUX
+    bool           sys_was_int;     /* was the last system call via do_int_syscall? */
+    bool           sys_xbp;         /* PR 313715: store orig xbp */
+#endif
+
+#ifdef X64
+    /* Is this thread in 32-bit (x86) or 64-bit (x64) mode?
+     * Default is to decode as base platform, but we want runtime ability to
+     * decode/encode 32-bit from 64-bit dynamorio.dll (we don't support the
+     * other way around: PR 236203).
+     */
+    bool           x86_mode;
+#endif
+
+    /* to make things more modular these are void*: */
+    void *         link_field;
+    void *         monitor_field;
+    void *         fcache_field;
+    void *         fragment_field;
+    void *         heap_field;
+    void *         vm_areas_field;
+    void *         os_field;
+    void *         synch_field;
+#ifdef LINUX
+    void *         signal_field;
+    void *         pcprofile_field;
+    bool           signals_pending;
+#endif
+    void *         private_code;          /* various thread-private routines */
+
+#ifdef TRACE_HEAD_CACHE_INCR
+    cache_pc       trace_head_pc; /* HACK to jmp to trace head w/o a prefix */
+#endif
+
+#ifdef WINDOWS
+    /* these fields used for "stack" of contexts for callbacks */
+# ifdef DCONTEXT_IN_EDI
+    dcontext_t *next_saved;
+# endif
+    dcontext_t *prev_unused;
+    /* need to be able to tell which dcontexts in callback stack are valid */
+    bool           valid;
+    /* special slot used to deal with callback returns, where we want to
+     * restore state of prior guy on callback stack, yet need current state
+     * to restore native state prior to callback return interrupt/syscall.
+     * Also used as temp next_tag slot for fcache_enter_indirect and cb ret.
+     */
+    reg_t          nonswapped_scratch;
+#endif
+
+    /* next_tag holds the do_syscall entry point, so we need another
+     * slot to hold asynch targets for APCs to know next target and
+     * for NtContinue and sigreturn to set next target
+     */
+    app_pc         asynch_target;
+
+    /* must store post-intercepted-syscall target to allow using normal
+     * dispatch() for native_exec syscalls
+     */
+    app_pc         native_exec_postsyscall;
+
+#ifdef PROGRAM_SHEPHERDING
+    bool           alloc_no_reserve; /* to implement executable_if_alloc policy */
+#endif
+
+#ifdef CUSTOM_TRACES_RET_REMOVAL
+    int num_calls;
+    int num_rets;
+    int call_depth;
+#endif
+#ifdef NATIVE_RETURN
+# ifdef NATIVE_RETURN_CALLDEPTH
+    uint           call_depth;
+# endif
+    cache_pc       last_retaddr; /* used by fixup_last_cti */
+    int            num_calls_in_trace;
+#endif
+
+#ifdef CHECK_RETURNS_SSE2
+    int            call_depth;
+    void *         call_stack;
+#endif
+
+#ifdef DEBUG
+    file_t                        logfile;
+    thread_local_statistics_t *     thread_stats;
+    bool                        expect_last_syscall_to_fail;
+    /* HACK to avoid recursion on pclookup for target invoking disassembly
+     * during decode_fragment() for a coarse target
+     */
+    bool                        in_opnd_disassemble;
+#endif /* DEBUG */
+#ifdef DEADLOCK_AVOIDANCE
+    thread_locks_t *               thread_owned_locks;
+#endif
+#ifdef KSTATS
+    thread_kstats_t *              thread_kstats;
+#endif
+
+#ifdef PROFILE_RDTSC
+    uint64         cache_enter_time;
+    uint64         start_time;     /* records start time for profiling */
+    fragment_t *     prev_fragment;
+    /* top ten times spent in cache */
+    uint64         cache_frag_count; /* num frag execs in single cache period */
+    uint64         cache_time[10];   /* top ten times spent in cache */
+    uint64         cache_count[10];  /* top ten cache_frag_counts */
+#endif
+
+#ifdef CLIENT_INTERFACE
+    /* client interface-specific data */
+    client_data_t *client_data;
+#endif
+
+    /* FIXME trace_sysenter_exit is used to capture an exit from a trace that
+     * ends in a SYSENTER and to enable trace head marking. So it's really a
+     * monitor-centric variable. It's placed in the context for now so that
+     * it's not shared across contexts (as the monitor data is). Cross-context
+     * sharing of this field could cause inadvertent trace head marking,
+     * for example, during a callback. A longer-term fix may be to move it to
+     * a context-private non-shared monitor struct.
+     */
+    bool           trace_sysenter_exit;
+
+    /* DR sets this field to indicate that it's forging an exception that
+     * may appear to originate in DR but should be passed on to the app. */
+    app_pc         forged_exception_addr;
+#ifdef HOT_PATCHING_INTERFACE
+    /* Fix for case 5367. */
+    bool           nudge_thread;    /* True only if this is a nudge thread. */
+    dr_jmp_buf_t   hotp_excpt_state;    /* To handle hot patch exceptions. */
+#endif
+    try_except_context_t *try_except_state; /* for TRY/EXCEPT/FINALLY */
+    bool unwinding_exception;   /* NYI support for TRY/FINALLY - 
+                                 * marks exception until an EXCEPT handles */
+    
+#ifdef WINDOWS                    
+    /* for ASLR_SHARED_CONTENT, note per callback, not per thread to
+     * track properties of a syscall or a syscall pair.  Even we don't
+     * expect to get APCs or callbacks while processing normal DLLs
+     * this should be per dcontext.
+     */
+    aslr_syscall_context_t aslr_context;
+
+    /* Initially memset to 0 in create dcontext.  If this is a nudge (or
+     * internal detach) thread (as detected by start address in intercept_apc),
+     * nudge_target is set to the corresponding nudge routine (currently always
+     * generic_nudge_target).  We can then check this
+     * value in those routines after we come out of the cache as a security
+     * measure (xref case 552).  This gives us some protection against an
+     * attacker leveraging our own detach routines and the like.  FIXME - if
+     * the attacker is able to specify the start address for a newly created
+     * thread then they can fake this. */
+    void           *nudge_target;
+
+    /* If free_app_stack is set, we free the application stack during thread exit
+     * cleanup.  Used for nudge threads. */
+    bool           free_app_stack;
+#endif /* WINDOWS */
+
+    /* we keep an absolute address pointer to our tls state so that we
+     * can access it from other threads
+     */
+    local_state_t *local_state;
+
+#ifdef WINDOWS
+    /* case 8721: saving the win32 start address so we can print it in the
+     * ldmp.  An alternative solution is to call NtQueryInformationThread
+     * with ThreadQuerySetWin32StartAddress at dump time.  According to 
+     * Nebbet, however, a thread calling ZwReplyWaitReplyPort or 
+     * ZwReplyWaitRecievePort will clobber the start address.
+     */
+    app_pc win32_start_addr;
+#endif
+
+    /* Used to abort bb building on decode faults.  Not persistent across cache. */
+    void *bb_build_info;
+};
+
+/* sentinel value for dcontext_t* used to indicate
+ * "global rather than a particular thread" 
+ */
+#define GLOBAL_DCONTEXT  ((dcontext_t *)PTR_UINT_MINUS_1)
+
+/* FIXME: why do we need to force the inline for this simple function? */
+static INLINE_FORCED dr_mcontext_t *
+get_mcontext(dcontext_t *dcontext)
+{
+    if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask))
+        return &(dcontext->upcontext.separate_upcontext->mcontext);
+    else
+        return &(dcontext->upcontext.upcontext.mcontext);
+}
+
+/* A number of routines (dump_mbi*, dump_mcontext, dump_callstack, 
+ * print_modules) have an argument on whether to dump in an xml friendly format
+ * (for xml diagnostics files). We use these defines for readability. */
+enum {
+    DUMP_XML=true,
+    DUMP_NOT_XML=false
+};
+
+/* Code cleanliness rules */
+#ifdef WINDOWS
+#  define snprintf   _snprintf
+#  define snwprintf  _snwprintf
+#  define vsnprintf  _vsnprintf
+#  define strcasecmp _stricmp
+#  define strncasecmp _strnicmp
+#  define wcscasecmp _wcsicmp
+#endif
+
+#define printf   printf_forbidden_function
+#define sprintf  sprintf_forbidden_function
+#define swprintf swprintf_forbidden_function
+#define vsprintf vsprintf_forbidden_function
+#define __try    __try_forbidden_construct /* see case 4461 */
+
+/* libc independence */
+#define mprotect     mprotect_forbidden_function
+#define mmap         mmap_forbidden_function
+#define munmap       munmap_forbidden_function
+#define getppid      getppid_forbidden_function
+#define sched_yield  sched_yield_forbidden_function
+#define dup          dup_forbidden_function
+#define sigaltstack  sigaltstack_forbidden_function
+#define setitimer    setitimer_forbidden_function
+#define _exit        _exit_forbidden_function
+#define gettimeofday gettimeofday_forbidden_function
+#define time         time_forbidden_function
+#define modify_ldt   modify_ldt_forbidden_function
+
+#endif /* _GLOBALS_H_ */
diff --git a/core/hashtable.c b/core/hashtable.c
new file mode 100644
index 000000000..4ef4c5eac
--- /dev/null
+++ b/core/hashtable.c
@@ -0,0 +1,194 @@
+/* **********************************************************
+ * Copyright (c) 2006-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2006-2007 Determina Corp. */
+
+#include "globals.h"
+#include "hashtable.h"
+
+/* Returns the proper number of hash bits to have a capacity with the
+ * given load for the given number of entries
+ */
+uint
+hashtable_bits_given_entries(uint entries, uint load)
+{
+    /* Add 1 for the sentinel */
+    return hashtable_num_bits(((entries + 1) * 100) / load);
+}
+
+#ifdef HASHTABLE_STATISTICS
+
+/* caller is responsible for any needed synchronization */
+void
+print_hashtable_stats(dcontext_t *dcontext, 
+                      const char *is_final_str, const char *is_trace_str,
+                      const char *lookup_routine_str, 
+                      const char *brtype_str,
+                      hashtable_statistics_t *lookup_stats) 
+{
+    uint64 hits_stat = lookup_stats->hit_stat;
+    uint64 total_lookups;
+    if (lookup_stats->hit_stat < lookup_stats->collision_hit_stat) {
+        /* HACK OVERFLOW: we special case here the case of a single overflow */
+        /* assuming only one overflow, which is the only case on spec (GAP)  */
+        hits_stat = (uint64)lookup_stats->hit_stat + UINT_MAX + 1;
+    }
+    total_lookups= hits_stat + lookup_stats->miss_stat + lookup_stats->collision_hit_stat;
+
+    DOLOG(1, LOG_FRAGMENT|LOG_STATS, {
+        uint miss_top=0; uint miss_bottom=0; 
+        uint hit_top=0; uint hit_bottom=0; 
+        uint col_top=0; uint col_bottom=0;
+        if (total_lookups > 0) {
+            divide_uint64_print(lookup_stats->miss_stat, total_lookups, false,
+                                4, &miss_top, &miss_bottom);
+        }
+        if (hits_stat > 0) {
+            divide_uint64_print(lookup_stats->collision_hit_stat, hits_stat,
+                                false, 4, &hit_top, &hit_bottom);
+            divide_uint64_print(hits_stat + lookup_stats->collision_stat, 
+                                hits_stat, false, 4, &col_top, &col_bottom);
+        }
+        LOG(THREAD, LOG_FRAGMENT|LOG_STATS, 1,
+            "%s %s table %s%s lookup hits%s: "UINT64_FORMAT_STRING", "
+            "misses: %u, total: "UINT64_FORMAT_STRING", miss%%=%u.%.4u\n",
+            is_final_str, is_trace_str, lookup_routine_str, brtype_str,
+            (lookup_stats->hit_stat < lookup_stats->collision_hit_stat) ?
+            "[OVFL]" : "",
+            hits_stat,
+            lookup_stats->miss_stat,
+            total_lookups,
+            miss_top, miss_bottom);
+        LOG(THREAD, LOG_FRAGMENT|LOG_STATS, 1,
+            "%s %s table %s%s "
+            "collisions: %u, collision hits: %u, "
+            ">2_or_miss: %u, overwrap: %u\n",
+            is_final_str, is_trace_str, lookup_routine_str, brtype_str,
+            lookup_stats->collision_stat,
+            /* FIXME: collision hit stats are updated only when inlining IBL head */
+            lookup_stats->collision_hit_stat,
+            lookup_stats->collision_stat - lookup_stats->collision_hit_stat,
+            lookup_stats->overwrap_stat);
+        LOG(THREAD, LOG_FRAGMENT|LOG_STATS, 1,
+            "%s %s table %s%s lookup "
+            " coll%%=%u.%.4u, dyn.avgcoll=%u.%.4u\n",
+            is_final_str, is_trace_str, lookup_routine_str, brtype_str,
+            hit_top, hit_bottom,
+            col_top, col_bottom);
+        if (lookup_stats->race_condition_stat || lookup_stats->unlinked_count_stat) {
+            LOG(THREAD, LOG_FRAGMENT|LOG_STATS, 1,
+                "%s %s table %s%s inlined ibl unlinking races: %d, unlinked: %d\n",
+                is_final_str, is_trace_str, lookup_routine_str, brtype_str,
+                lookup_stats->race_condition_stat,
+                lookup_stats->unlinked_count_stat);
+        }
+        if (lookup_stats->ib_stay_on_trace_stat) {
+            uint ontrace_top=0; uint ontrace_bottom=0; 
+            uint lastexit_top=0; uint lastexit_bottom=0; 
+            uint speculate_lastexit_top=0; uint speculate_lastexit_bottom=0; 
+            /* indirect branch lookups */
+            uint64 total_dynamic_ibs = 
+                total_lookups + lookup_stats->ib_stay_on_trace_stat + 
+                (DYNAMO_OPTION(speculate_last_exit) ? lookup_stats->ib_trace_last_ibl_speculate_success : 0);
+
+            /* FIXME: add ib_stay_on_trace_stat_ovfl here */
+
+            if (total_dynamic_ibs > 0) {
+                divide_uint64_print(lookup_stats->ib_stay_on_trace_stat, 
+                                    total_dynamic_ibs, false,
+                                    4, &ontrace_top, &ontrace_bottom);
+                divide_uint64_print(lookup_stats->ib_trace_last_ibl_exit, 
+                                    total_dynamic_ibs, false,
+                                    4, &lastexit_top, &lastexit_bottom);
+                divide_uint64_print(lookup_stats->ib_trace_last_ibl_speculate_success, 
+                                    total_dynamic_ibs, false,
+                                    4, &speculate_lastexit_top, &speculate_lastexit_bottom);
+            }
+
+            /* all percentages here relative to IB lookups */
+            /* ontrace is how often we stay on trace with respect to all indirect branches */
+            /* ontrace_fail is assuming IBL's that are not the last exit are due to ontrace failures */
+            /* lastexit is how many are on last IBL when it is not speculated */
+            /* lastexit_speculate is how many are speculatively hit (as of all IBs) */
+            LOG(THREAD, LOG_FRAGMENT|LOG_STATS, 1,
+                "%s %s table %s%s stay on trace hit:%s %u, last_ibl: %u, ontrace%%=%u.%.4u, lastexit%%=%u.%.4u\n",
+                is_final_str, is_trace_str, lookup_routine_str, brtype_str,
+                lookup_stats->ib_stay_on_trace_stat_ovfl ? " OVFL" : "",
+                lookup_stats->ib_stay_on_trace_stat,
+                lookup_stats->ib_trace_last_ibl_exit,
+                ontrace_top, ontrace_bottom,
+                lastexit_top, lastexit_bottom
+                );
+            LOG(THREAD, LOG_FRAGMENT|LOG_STATS, 1,
+                "%s %s table %s%s last trace exit speculation hit: %u, lastexit_ontrace%%=%u.%.4u(%%IB)\n",
+                is_final_str, is_trace_str, lookup_routine_str, brtype_str,
+                lookup_stats->ib_trace_last_ibl_speculate_success,
+                speculate_lastexit_top, speculate_lastexit_bottom);
+        }
+
+        if (lookup_stats->ib_trace_last_ibl_exit > 0) {
+            /* ignoring indirect branches that stayed on trace */
+            /* FIXME: add ib_stay_on_trace_stat_ovfl here */
+            uint speculate_only_lastexit_top=0; uint speculate_only_lastexit_bottom=0; 
+            uint lastexit_ibl_top=0; uint lastexit_ibl_bottom=0; 
+            uint speculate_lastexit_ibl_top=0; uint speculate_lastexit_ibl_bottom=0; 
+
+            uint64 total_dynamic_ibl_no_trace = total_lookups + lookup_stats->ib_trace_last_ibl_exit;
+            if (total_dynamic_ibl_no_trace > 0) {
+                divide_uint64_print(lookup_stats->ib_trace_last_ibl_exit, 
+                                    total_dynamic_ibl_no_trace, false,
+                                    4, &lastexit_ibl_top, &lastexit_ibl_bottom);
+                divide_uint64_print(lookup_stats->ib_trace_last_ibl_speculate_success, 
+                                    total_dynamic_ibl_no_trace, false,
+                                    4, &speculate_lastexit_ibl_top, &speculate_lastexit_ibl_bottom);
+            }
+
+            /* ib_trace_last_ibl_exit includes all ib_trace_last_ibl_speculate_success */
+            divide_uint64_print(lookup_stats->ib_trace_last_ibl_speculate_success, 
+                                lookup_stats->ib_trace_last_ibl_exit, false,
+                                4, &speculate_only_lastexit_top, &speculate_only_lastexit_bottom);
+
+            
+            LOG(THREAD, LOG_FRAGMENT|LOG_STATS, 1,
+                "%s %s table %s%s last trace exit speculation hit: %u, speculation miss: %u, lastexit%%=%u.%.4u(%%IBL), lastexit_succ%%=%u.%.4u(%%IBL), spec hit%%=%u.%.4u(%%last exit)\n",
+                is_final_str, is_trace_str, lookup_routine_str, brtype_str,
+                lookup_stats->ib_trace_last_ibl_speculate_success,
+                lookup_stats->ib_trace_last_ibl_exit - lookup_stats->ib_trace_last_ibl_speculate_success,
+                lastexit_ibl_top, lastexit_ibl_bottom,
+                speculate_lastexit_ibl_top, speculate_lastexit_ibl_bottom,
+                speculate_only_lastexit_top, speculate_only_lastexit_bottom
+                );
+        }
+    });
+}
+
+#endif /* HASHTABLE_STATISTICS */
diff --git a/core/hashtable.h b/core/hashtable.h
new file mode 100644
index 000000000..2738caf1d
--- /dev/null
+++ b/core/hashtable.h
@@ -0,0 +1,187 @@
+/* **********************************************************
+ * Copyright (c) 2006-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2006-2007 Determina Corp. */
+
+/*
+ * hashtable.h
+ *
+ * Hashtable support common among all hashtables
+ *
+ */
+
+#ifndef _HASHTABLE_H_
+#define _HASHTABLE_H_ 1
+
+
+/* Flags stored in the table_flags bitfield that are common to
+ * all hashtables
+ */
+/* Is the table itself shared? */
+#define HASHTABLE_SHARED                0x00000001
+/* Are the entries in the table shared? 
+ * FIXME: right now this is never used independently from LOCKLESS_ACCESS,
+ * perhaps get rid of it
+ */
+#define HASHTABLE_ENTRY_SHARED          0x00000002
+/* Is the table accessed for reads concurrently without a lock? */
+#define HASHTABLE_LOCKLESS_ACCESS       0x00000004
+/* Is the table the primary, master list of its contents? */
+#define HASHTABLE_NOT_PRIMARY_STORAGE   0x00000004
+/* Is the table allocated in persistent memory? */
+#define HASHTABLE_PERSISTENT            0x00000008
+/* For debug builds: use per-entry statistics? */
+#define HASHTABLE_USE_ENTRY_STATS       0x00000010
+/* For non-performance-critical tables where we'd rather save memory */
+#define HASHTABLE_RELAX_CLUSTER_CHECKS  0x00000020
+/* Disallow adding or removing from the table, and acquire no locks on lookups
+ * Only used for persisted coarse units.
+ */
+#define HASHTABLE_READ_ONLY             0x00000040
+/* Align the main table to the cache line */
+#define HASHTABLE_ALIGN_TABLE           0x00000080
+
+/* Specific tables can add their own flags starting with this value
+ * FIXME: any better way? how know when hit limit with <<?
+ */
+#define HASHTABLE_CUSTOM_FLAGS_START    0x00010000
+
+/* Flags to NOT propagate to copies */
+#define HASHTABLE_COPY_IGNORE_FLAGS     (HASHTABLE_READ_ONLY)
+
+
+/* single_thread_in_DR does not need rw locks since we do not access shared
+ * tables from ibl while holding locks (we do so in a lockless manner)
+ */
+#define TABLE_NEEDS_LOCK(ptable)                          \
+    (TEST(HASHTABLE_SHARED, (ptable)->table_flags) &&     \
+     !TEST(HASHTABLE_READ_ONLY, (ptable)->table_flags))   \
+
+/* is_local is currently debug-only and only affects asserts */
+#define ASSERT_TABLE_SYNCHRONIZED(ptable, RW)                                \
+    ASSERT_OWN_##RW##_LOCK(TABLE_NEEDS_LOCK(ptable) &&                       \
+                           !(ptable)->is_local &&                            \
+                           !INTERNAL_OPTION(single_thread_in_DR), &(ptable)->rwlock)
+
+#define TABLE_RWLOCK(ptable,rw,lock) do {   \
+    if (TABLE_NEEDS_LOCK(ptable))           \
+        rw##_##lock(&(ptable)->rwlock);     \
+} while (0)
+
+#define TABLE_MEMOP(table_flags, op) \
+    (TEST((table_flags), HASHTABLE_PERSISTENT) ? \
+     heap_##op : nonpersistent_heap_##op)
+
+#define TABLE_TYPE_MEMOP(table_flags, op, dc, type, which, protected) \
+    (TEST((table_flags), HASHTABLE_PERSISTENT) ? \
+     HEAP_TYPE_##op(dc, type, which, protected): \
+     NONPERSISTENT_HEAP_TYPE_##op(dc, type, which))
+
+/* table capacity includes a sentinel so this is equivalent to
+ * hash_index % (ftable->capacity - 1) 
+ */
+#define HASH_INDEX_WRAPAROUND(hash_index,ftable) \
+    ((hash_index) & (uint)(ftable->hash_mask >> ftable->hash_mask_offset))
+
+#ifdef HASHTABLE_STATISTICS
+/* Just a typechecking memset() wrapper */
+# define INIT_HASHTABLE_STATS(lookup_stats) do {                \
+        memset(&lookup_stats, 0, sizeof(hashtable_statistics_t));  \
+        ASSERT(lookup_stats.hit_stat == 0);                     \
+} while (0)
+# define HTABLE_STAT_INC(ftable,event) ftable->drlookup_stats.event##_stat++
+#else
+# define HTABLE_STAT_INC(ftable,event) ((void)0)
+#endif
+
+
+#ifdef HASHTABLE_STATISTICS
+typedef struct _hashtable_statistics_t {
+    uint hit_stat;    /* hit count of hash table lookups */
+    uint collision_hit_stat; /* lookups which collided but were hits */
+    uint collision_stat; /* total collision lookups (including misses) */
+    uint miss_stat; /* lookups which still ended up with a context switch */
+    uint overwrap_stat; /* lookups whose collision chain crosses overwrap sentinel */
+    uint race_condition_stat; /* number of inilned ibl unlinking race condition cases */
+    uint unlinked_count_stat; /* number of times unlinked */
+
+    /* Stay on trace check combined with hashtable lookups gives the
+     * dynamic count of indirect branches 
+     * FIXME: case 4817 where bb's could also cache one target
+     * or see the CGO03 paper for multiple cached locations with similar stats
+     */
+    uint ib_stay_on_trace_stat; /* hash table lookup avoided - cached single target location */
+    /* This is likely to overflow in many cases, yet 64-bit in cache
+     * increment is too heavy weight, should instead save last value
+     * assuming that some rare, yet common enough event happens
+     * between every 4 billion increments
+     */
+    uint ib_stay_on_trace_stat_last; /* last value */
+    uint ib_stay_on_trace_stat_ovfl; /* number of times last was bigger than current */
+
+    uint ib_trace_last_ibl_exit; /* hash table lookup for last exit - not cached */
+    /* FIXME: add last and ovfl flag here as well */
+
+    /* case 4817: add counter on extra check for converting IB into conditional 
+     * and add its success rate
+     */
+    uint ib_trace_last_ibl_speculate_success; /* stay-on-trace check success on last exit */
+    /* FIXME: add last and ovfl flag here as well */
+} hashtable_statistics_t;
+
+/* A table used by IBL parallel to the lookuptable to collect statistics */
+typedef struct _fragment_stat_entry_t {
+    /* hits * static collision length should equal */
+    uint    hits; /* hits in IBL */
+    /* we need a filler for in-cache lookups */
+    uint    age; /* used to measure for how long an entry hasn't been used */
+} fragment_stat_entry_t;
+#endif /* HASHTABLE_STATISTICS */
+
+/* Returns the proper number of hash bits to have a capacity with the
+ * given load for the given number of entries
+ */
+uint
+hashtable_bits_given_entries(uint entries, uint load);
+
+#ifdef HASHTABLE_STATISTICS
+/* caller is responsible for any needed synchronization */
+void
+print_hashtable_stats(dcontext_t *dcontext, 
+                      const char *is_final_str, const char *is_trace_str,
+                      const char *lookup_routine_str, 
+                      const char *brtype_str,
+                      hashtable_statistics_t *lookup_stats);
+#endif
+
+
+#endif /* _HASHTABLE_H_ */
diff --git a/core/hashtablex.h b/core/hashtablex.h
new file mode 100644
index 000000000..11c06ab55
--- /dev/null
+++ b/core/hashtablex.h
@@ -0,0 +1,2229 @@
+/* **********************************************************
+ * Copyright (c) 2006-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2006-2007 Determina Corp. */
+
+/*
+ * hashtablex.h
+ *
+ * Generic hashtable support.
+ *
+ * Implementation is an open-address hashtable with sentinel.
+ * Supports invalid slots for concurrent-access usage, in particular
+ * lockless read concurrent access (such as for indirect branch lookup tables).
+ * Also supports a synchronized lookup table.
+ *
+ * Usage: define the following macros, and then include this file.
+ * It is meant to be included multiple times.
+ * Define HASHTABLEX_HEADER to get the data struct and no implementation;
+ * else, you'll get implementation but NOT the data struct.
+ *
+ * set HASHTABLE_USE_LOOKUPTABLE if using lookuptable
+ * set HASHTABLE_ENTRY_STATS if want entry stats
+ *
+ * for struct:
+ *   token NAME_KEY
+ *   type ENTRY_TYPE
+ *  if HASHTABLE_USE_LOOKUPTABLE is defined:
+ *   type AUX_ENTRY_TYPE
+ *  endif
+ *   fields CUSTOM_FIELDS
+ *
+ * for main table:
+ *   ptr_uint_t ENTRY_TAG(entry)
+ *   bool ENTRY_IS_EMPTY(entry)
+ *   bool ENTRY_IS_SENTINEL(entry)
+ *   bool ENTRY_IS_INVALID(entry)
+ *     assumption: invalid entries are only used with
+ *     HASHTABLE_LOCKLESS_ACCESS tables
+ *   bool ENTRIES_ARE_EQUAL(entry1, entry2)
+ *     if using pointers, pointer equality is fine
+ *   ENTRY_TYPE ENTRY_EMPTY
+ *   ENTRY_TYPE ENTRY_SENTINEL
+ *     FIXME: to support structs we'll need lhs like AUX_ENTRY_SET_TO_SENTINEL
+ *   (up to caller to ever set an entry to invalid)
+ *   HTLOCK_RANK
+ *     table_rwlock will work for most users.  
+ *     Needs higher rank than memory alloc locks.
+ *
+ * for lookuptable:
+ *  if HASHTABLE_USE_LOOKUPTABLE is defined:
+ *   ptr_uint_t AUX_ENTRY_TAG(aux_entry)
+ *   bool AUX_ENTRY_IS_EMPTY(aux_entry)
+ *     empty entry is assumed to be all zeros!
+ *   bool AUX_ENTRY_IS_SENTINEL(aux_entry)
+ *   bool AUX_ENTRY_IS_INVALID(aux_entry)
+ *   bool AUX_PAYLOAD_IS_INVALID(dcontext, table, aux_entry)
+ *   AUX_ENTRY_SET_TO_SENTINEL(aux_entry)
+ *     takes lhs since may be struct and C only allows setting to {} in decl
+ *   AUX_ENTRY_IS_SET_TO_ENTRY(aux_entry, entry)
+ *   AUX_ENTRY_SET_TO_ENTRY(aux_entry, entry)
+ *     takes lhs since may be struct and C only allows setting to {} in decl
+ *   string AUX_ENTRY_FORMAT_STR
+ *   printf-args AUX_ENTRY_FORMAT_ARGS(aux_entry)
+ *  endif
+ * we assume value semantics, i.e., we can copy entries via the = operator
+ *
+ * for HEAP_ACCOUNTING:
+ *   heapacct-enum HASHTABLE_WHICH_HEAP(table_flags)
+ *
+ * to obtain persistence routines, define
+ *   HASHTABLE_SUPPORT_PERSISTENCE
+ *
+ * for custom behavior we assume that these routines exist:
+ *
+ *    static void
+ *    HTNAME(hashtable_,NAME_KEY,_init_internal_custom)
+ *         (dcontext_t *dcontext,
+ *          HTNAME(,NAME_KEY,_table_t) *htable
+ *          _IFLOOKUP(bool use_lookup));
+ *
+ *    static void
+ *    HTNAME(hashtable_,NAME_KEY,_resized_custom)
+ *         (dcontext_t *dcontext,
+ *          HTNAME(,NAME_KEY,_table_t) *htable,
+ *          uint old_capacity, ENTRY_TYPE *old_table,
+ *          ENTRY_TYPE *old_table_unaligned
+ *          _IFLOOKUP(AUX_ENTRY_TYPE* lookuptable)
+ *          _IFLOOKUP(byte *old_lookup_table_unaligned),
+ *          uint old_ref_count,
+ *          uint old_table_flags);
+ *
+ *   #ifdef DEBUG
+ *    static void
+ *    HTNAME(hashtable_,NAME_KEY,_study_custom)
+ *         (dcontext_t *dcontext,
+ *          HTNAME(,NAME_KEY,_table_t) *htable, 
+ *          uint entries_inc);
+ *   #endif
+ *
+ *    static void
+ *    HTNAME(hashtable_,NAME_KEY,_free_entry)
+ *         (dcontext_t *dcontext, HTNAME(,NAME_KEY,_table_t) *table,
+ *          ENTRY_TYPE entry);
+ *
+ * FIXME:
+ * - use defines to weed out unused static routines
+ * - rename unlinked_entries to invalid_entries
+ * - remove references to "ibl"/"ibt" and make them general comments
+ * - same with references to "fragment_t" or "f"
+ * - caller uses expanded names "hashtable_fragment_init" -- should we
+ *   provide a macro HASHTABLE_INIT(NAME_KEY, args...) or ok to mix
+ *   the generated names here w/ hardcoded names in instantiations?
+ * - provide wrapper routines for lookup, etc. that lock and unlock?
+ *   see app_pc table usage for after call and rct target tables.
+ * - eliminate the HASHTABLE_ENTRY_SHARED flag?
+ */
+
+
+#define EXPANDKEY(pre, key, post) pre##key##post
+#define HTNAME(pre, key, post) EXPANDKEY(pre, key, post)
+#define KEY_STRING STRINGIFY(NAME_KEY)
+
+#define ENTRY_IS_REAL(e) \
+    (!ENTRY_IS_EMPTY(e) && !ENTRY_IS_SENTINEL(e) && !ENTRY_IS_INVALID(e))
+
+#ifdef HASHTABLE_USE_LOOKUPTABLE
+# define _IFLOOKUP(x) , x
+# define IFLOOKUP_ELSE(x,y) (x)
+#else
+# define _IFLOOKUP(x) /* nothing */
+# define IFLOOKUP_ELSE(x,y) (y)
+#endif
+
+/****************************************************************************/
+#ifdef HASHTABLEX_HEADER
+
+/* N.B.: if you change any fields here you must increase
+ * PERSISTENT_CACHE_VERSION!
+ */
+typedef struct HTNAME(_,NAME_KEY,_table_t) {
+    /* entries used from shared private IBL routines copy come first:
+     * used to be lookuptable, now table for case 7691 */
+    /* preferred location of a given tag is then at
+     * lookuptable[(hash_func(tag) & hash_mask) >> hash_offset] 
+     */
+    ptr_uint_t      hash_mask;   /* mask selects the index bits of hash value */
+#ifdef HASHTABLE_USE_LOOKUPTABLE 
+    AUX_ENTRY_TYPE *lookuptable; /* allocation aligned within lookuptable_unaligned */
+#endif
+    ENTRY_TYPE     *table; /* hash_bits-bit addressed hash table */
+
+    uint           ref_count;   /* when table is shared -- HASHTABLE_SHARED --
+                                 * # threads with active ptrs to the table */
+    uint           hash_bits;
+    hash_function_t  hash_func;   /* selects hash function */
+    uint           hash_mask_offset; /* ignores given number of LSB bits */
+    uint           capacity;    /* = 2^hash_bits + 1 sentinel */
+    uint           entries;
+    /* FIXME: rename to invalid_entries to be more general */
+    uint           unlinked_entries;
+
+    uint load_factor_percent; /* \alpha = load_factor_percent/100 */
+    uint resize_threshold;    /*  = capacity * load_factor */
+
+    uint groom_factor_percent; /* \gamma = groom_factor_percent/100 */
+    uint groom_threshold;    /*  = capacity * groom_factor_percent */
+
+    uint max_capacity_bits;    /* log_2 of maximum size to grow table */
+
+#ifdef HASHTABLE_STATISTICS
+    /* These refer only to accesses from DR lookups */
+    hashtable_statistics_t drlookup_stats;
+
+# ifdef HASHTABLE_ENTRY_STATS
+    /* Accesses from ibl */
+    fragment_stat_entry_t *entry_stats;
+    /* precomputed (entry_stats - lookup_table) for easy parallel
+     * table access from IBL routines
+     */
+    uint entry_stats_to_lookup_table;
+    uint added_since_dumped; /* clock handle - usually the same as
+                              * delta in entries, unless we have removed
+                              */
+# endif
+#endif /* HASHTABLE_STATISTICS */
+    uint table_flags; /* the HASHTABLE_* values are used here */
+    read_write_lock_t rwlock;       /* shared tables should use a read/write lock */
+    ENTRY_TYPE *table_unaligned;  /* real alloc for table if HASHTABLE_ALIGN_TABLE */
+#ifdef HASHTABLE_USE_LOOKUPTABLE
+    byte *lookup_table_unaligned; /* real allocation unit for lookuptable */
+#endif
+#ifdef DEBUG
+    const char *name;
+    bool is_local;           /* no lock needed since only known to this thread */
+#endif
+    CUSTOM_FIELDS
+} HTNAME(,NAME_KEY,_table_t);
+
+
+
+/****************************************************************************/
+#else /* implementation */
+
+/* forward decls */
+static bool
+HTNAME(hashtable_,NAME_KEY,_check_size)(dcontext_t *dcontext,
+                                        HTNAME(,NAME_KEY,_table_t) *htable,
+                                        uint add_now, uint add_later);
+
+static void
+HTNAME(hashtable_,NAME_KEY,_init_internal_custom)(dcontext_t *dcontext,
+                                                  HTNAME(,NAME_KEY,_table_t) *htable
+                                                  _IFLOOKUP(bool use_lookup));
+
+static void
+HTNAME(hashtable_,NAME_KEY,_resized_custom)(dcontext_t *dcontext,
+                                            HTNAME(,NAME_KEY,_table_t) *htable,
+                                            uint old_capacity, ENTRY_TYPE *old_table,
+                                            ENTRY_TYPE *old_table_unaligned
+                                            _IFLOOKUP(AUX_ENTRY_TYPE* lookuptable)
+                                            _IFLOOKUP(byte *old_lookup_table_unaligned),
+                                            uint old_ref_count,
+                                            uint old_table_flags);
+
+static uint
+HTNAME(hashtable_,NAME_KEY,_unlinked_remove)(dcontext_t *dcontext,
+                                             HTNAME(,NAME_KEY,_table_t) *table);
+
+static void
+HTNAME(hashtable_,NAME_KEY,_groom_table)(dcontext_t *dcontext,
+                                         HTNAME(,NAME_KEY,_table_t) *table);
+
+static inline void
+HTNAME(hashtable_,NAME_KEY,_groom_helper)(dcontext_t *dcontext,
+                                          HTNAME(,NAME_KEY,_table_t) *table);
+
+static void
+HTNAME(hashtable_,NAME_KEY,_free_entry)
+    (dcontext_t *dcontext, HTNAME(,NAME_KEY,_table_t) *table, ENTRY_TYPE entry);
+
+#ifdef DEBUG
+static void
+HTNAME(hashtable_,NAME_KEY,_study)(dcontext_t *dcontext,
+                                   HTNAME(,NAME_KEY,_table_t) *htable, 
+                                   uint entries_inc);
+
+static void
+HTNAME(hashtable_,NAME_KEY,_study_custom)(dcontext_t *dcontext,
+                                          HTNAME(,NAME_KEY,_table_t) *htable, 
+                                          uint entries_inc);
+
+# ifdef HASHTABLE_ENTRY_STATS
+void
+HTNAME(hashtable_,NAME_KEY,_dump_entry_stats)(dcontext_t *dcontext,
+                                              HTNAME(,NAME_KEY,_table_t) *htable);
+# endif
+
+void
+HTNAME(hashtable_,NAME_KEY,_dump_table)(dcontext_t *dcontext,
+                                        HTNAME(,NAME_KEY,_table_t) *htable);
+#endif /* DEBUG */
+
+#if defined(DEBUG) && defined(INTERNAL)
+static void
+HTNAME(hashtable_,NAME_KEY,_load_statistics)(dcontext_t *dcontext,
+                                             HTNAME(,NAME_KEY,_table_t) *htable);
+#endif
+
+
+/* get size in bytes padded for later cache alignment */
+static inline
+size_t
+HTNAME(hashtable_,NAME_KEY,_table_aligned_size)(uint table_capacity, uint flags)
+{
+    size_t size;
+    /* we assume table size allows 32-bit indices */
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(table_capacity*sizeof(ENTRY_TYPE))));
+    size = table_capacity*sizeof(ENTRY_TYPE);
+    if (TEST(HASHTABLE_ALIGN_TABLE, flags)) {
+        /* aligned at least at 4, and may be aligned */
+        size += proc_get_cache_line_size() - 4;
+    }
+    return size;
+}
+
+#ifdef HASHTABLE_USE_LOOKUPTABLE 
+/* get size in bytes padded for later cache alignment */
+static inline
+size_t
+HTNAME(hashtable_,NAME_KEY,_lookuptable_aligned_size)(uint table_capacity)
+{
+
+    return table_capacity*sizeof(AUX_ENTRY_TYPE)
+        + proc_get_cache_line_size() - 4; /* aligned at least at 4, and may be aligned */
+}
+#endif
+
+/* callers should use either hashtable_init or hashtable_resize instead */
+static void
+HTNAME(hashtable_,NAME_KEY,_init_internal)
+     (dcontext_t *dcontext, HTNAME(,NAME_KEY,_table_t) *table,
+      uint bits, uint load_factor_percent, hash_function_t func, 
+      uint hash_mask_offset _IFLOOKUP(bool use_lookup))
+{
+    uint i;
+    uint sentinel_index;
+    size_t alloc_size;
+
+    table->hash_bits = bits;
+    table->hash_func = func;
+    table->hash_mask_offset = hash_mask_offset;
+    table->hash_mask = HASH_MASK(table->hash_bits) << hash_mask_offset;
+    table->capacity = HASHTABLE_SIZE(table->hash_bits);
+
+    /* 
+     * add an extra null_fragment at end to allow critical collision path
+     * not to worry about table overwrap
+     * FIXME: case 2147 to stay at power of 2 should use last element instead
+     */
+    table->capacity++;
+    sentinel_index = table->capacity-1;    
+
+    table->entries = 0;
+    table->unlinked_entries = 0;
+
+    /* STUDY: try different ratios for the load factor \alpha
+     * It may save memory and it will not necessarily hurt performance:
+     * better cache utilization may help us even if touch more entries
+     * on a cache line.
+     */
+    table->load_factor_percent = load_factor_percent;
+    /* be careful with integer overflows if ever let this in non-debug versions
+     * ASSERT is not enough if this is a user controlled release option - needs
+     * sanity check always
+     */
+    ASSERT(table->load_factor_percent > 0 && table->load_factor_percent < 100);
+    table->resize_threshold = table->capacity * table->load_factor_percent / 100;
+
+    table->groom_factor_percent = 0; /* grooming disabled */
+    table->max_capacity_bits = 0; /* unlimited */
+
+    ASSERT(table->groom_factor_percent >= 0 /* 0 == disabled */);
+    ASSERT(table->groom_factor_percent < 100);
+    ASSERT(table->groom_factor_percent <= table->load_factor_percent);
+    /* when groom factor < load/2 then we'd have to groom during table rehashing */
+    ASSERT(table->groom_factor_percent == 0
+           || table->groom_factor_percent*2 > table->load_factor_percent);
+
+    table->groom_threshold = table->capacity * table->groom_factor_percent / 100;
+
+    alloc_size = HTNAME(hashtable_,NAME_KEY,_table_aligned_size)
+        (table->capacity, table->table_flags);
+    table->table_unaligned = (ENTRY_TYPE *) TABLE_MEMOP(table->table_flags, alloc)
+        (dcontext, alloc_size HEAPACCT(HASHTABLE_WHICH_HEAP(table->table_flags)));
+    if (TEST(HASHTABLE_ALIGN_TABLE, table->table_flags)) {
+        ASSERT(ALIGNED(table->table_unaligned, 4)); /* guaranteed by heap_alloc */
+        table->table = (ENTRY_TYPE *) 
+            ALIGN_FORWARD(table->table_unaligned, proc_get_cache_line_size());
+        ASSERT(ALIGNED(table->table, proc_get_cache_line_size()));
+        ASSERT(ALIGNED(table->table, sizeof(ENTRY_TYPE)));
+    } else
+        table->table = table->table_unaligned;
+    for (i=0; i<table->capacity; i++)
+        table->table[i] = ENTRY_EMPTY;
+    /* overwrite last element to be a sentinel, reached only by assembly routines */
+    table->table[sentinel_index] = ENTRY_SENTINEL;
+
+    table->ref_count = 0;
+#ifdef HASHTABLE_USE_LOOKUPTABLE 
+    if (use_lookup) {
+        /* We need to allocate aligned size, yet there is no point in
+         * calling heap_mmap for small sizes.  Instead we use normal
+         * heap and guarantee alignment manually by padding.  For
+         * larger sizes, we're already wasting a page beyond the table
+         * size, so this will not waste more memory.
+         */
+        size_t lookup_table_allocation =
+            HTNAME(hashtable_,NAME_KEY,_lookuptable_aligned_size)(table->capacity);
+
+        table->lookup_table_unaligned = 
+            TABLE_MEMOP(table->table_flags, alloc)
+            (dcontext, lookup_table_allocation
+             HEAPACCT(HASHTABLE_WHICH_HEAP(table->table_flags)));
+        ASSERT(ALIGNED(table->lookup_table_unaligned, 4)); /* guaranteed by heap_alloc */
+
+        table->lookuptable = (AUX_ENTRY_TYPE*) 
+            ALIGN_FORWARD(table->lookup_table_unaligned, 
+                          proc_get_cache_line_size());
+                                           
+        /* When the table is used for IBL, for correctness we need to
+         * make sure it's allocated at a 4 byte aligned address. This
+         * is required so that writes to the start_pc field during a
+         * flush are atomic. If the start address is 4-byte aligned
+         * then each 8 byte entry -- and the entry's two 4 byte fields
+         * -- will also be aligned.  This should be guaranteed by
+         * heap_alloc and we even align the table start address at
+         * cache line.
+         */
+        ASSERT(ALIGNED(table->lookuptable, 4));
+
+        /* make sure an entry doesn't cross a cache line boundary for performance */
+        ASSERT(ALIGNED(table->lookuptable, sizeof(AUX_ENTRY_TYPE)));
+
+        /* A minor point: save one extra cache line on the whole table
+         * by making sure first entry is aligned to a cache line
+         * boundary, otherwise if straddling we would need
+         * 1 + table_size/cache_line_size to fit the whole table in d-cache
+         */
+        ASSERT(ALIGNED(table->lookuptable, proc_get_cache_line_size()));
+
+        LOG(THREAD, LOG_HTABLE, 2,
+            "hashtable_"KEY_STRING"_init %s lookup unaligned="PFX" "
+            "aligned="PFX" allocated=%d\n",
+            table->name, table->lookup_table_unaligned, 
+            table->lookuptable, lookup_table_allocation);
+
+        /* set all to null_fragment {tag : 0, start_pc : 0} */
+        memset(table->lookuptable, 0, table->capacity*sizeof(AUX_ENTRY_TYPE));
+        ASSERT(AUX_ENTRY_IS_EMPTY(table->lookuptable[0]));
+        /* set last to sentinel_fragment {tag : 0, start_pc : 1} */
+        AUX_ENTRY_SET_TO_SENTINEL(table->lookuptable[sentinel_index]);
+    } else {
+        /* TODO: emit_utils assumes lookuptable will exist, but we can't match
+         * the initializations.
+         */
+        table->lookup_table_unaligned = NULL;
+        table->lookuptable = NULL;
+    }
+#endif
+
+    ASSERT(ENTRY_IS_EMPTY(ENTRY_EMPTY));
+    ASSERT(ENTRY_IS_SENTINEL(ENTRY_SENTINEL));
+
+    LOG(THREAD, LOG_HTABLE, 1,
+        "hashtable_"KEY_STRING"_init %s htable="PFX" bits=%d size=%d "
+        "mask="PFX" offset=%d load=%d%% resize=%d\n"
+        "               %s %s "PFX" %s "PFX"  groom=%d%% groom_at=%d\n",
+        table->name, table, bits, table->capacity,
+        table->hash_mask,
+        table->hash_mask_offset,
+        table->load_factor_percent, table->resize_threshold,
+        table->name, 
+        "table", table->table,
+        IFLOOKUP_ELSE(use_lookup ? "lookup":"", ""),
+        IFLOOKUP_ELSE(table->lookuptable, NULL),
+        table->groom_factor_percent, table->groom_threshold);
+
+#ifdef HASHTABLE_STATISTICS
+# ifdef HASHTABLE_ENTRY_STATS
+    if (INTERNAL_OPTION(hashtable_ibl_entry_stats)) {
+        if (TEST(HASHTABLE_USE_ENTRY_STATS, table->table_flags)) {
+#  ifdef HASHTABLE_USE_LOOKUPTABLE
+            ASSERT(sizeof(fragment_stat_entry_t) == sizeof(AUX_ENTRY_TYPE));
+#  else
+            ASSERT(sizeof(fragment_stat_entry_t) == sizeof(ENTRY_TYPE));
+#  endif
+            if (table->entry_stats != NULL) { /* resize */
+                /* assuming resize is always doubling the table */
+                /* FIXME: too error prone we should pass old capacity
+                 * somewhere if case 2147 changes the table size 
+                 */
+                uint old_capacity = HASHTABLE_SIZE(table->hash_bits-1) + 1 /* sentinel */;
+                /* make sure we've printed the old stats, now losing them */
+                HEAP_ARRAY_FREE(dcontext, table->entry_stats, fragment_stat_entry_t,
+                                old_capacity,
+                                HASHTABLE_WHICH_HEAP(table->table_flags),
+                                UNPROTECTED);
+            }
+            /* FIXME: either put in nonpersistent heap as appropriate, or
+             * preserve across resets
+             */
+            table->entry_stats =
+                HEAP_ARRAY_ALLOC(dcontext, fragment_stat_entry_t, table->capacity,
+                                 HASHTABLE_WHICH_HEAP(table->table_flags),
+                                 UNPROTECTED);
+            ASSERT_TRUNCATE(table->entry_stats_to_lookup_table, uint,
+                            ((byte*)table->entry_stats - (byte*)table->table));
+            table->entry_stats_to_lookup_table = (uint)
+#  ifdef HASHTABLE_USE_LOOKUPTABLE
+                ((byte*)table->entry_stats - (byte*)table->lookuptable);
+#  else
+                ((byte*)table->entry_stats - (byte*)table->table);
+#  endif
+            memset(table->entry_stats, 0, table->capacity*sizeof(fragment_stat_entry_t));
+        }
+    }
+# endif
+#endif
+
+    HTNAME(hashtable_,NAME_KEY,_init_internal_custom)(dcontext, table
+                                                      _IFLOOKUP(use_lookup));
+}
+
+static void
+HTNAME(hashtable_,NAME_KEY,_init)(dcontext_t *dcontext,
+                                  HTNAME(,NAME_KEY,_table_t) *table,
+                                  uint bits, 
+                                  uint load_factor_percent, hash_function_t func,
+                                  uint hash_offset
+                                  /* FIXME: turn this bool into a HASHTABLE_ flag */
+                                  _IFLOOKUP(bool use_lookup),
+                                  uint table_flags NAME(const char *table_name))
+{
+#ifdef DEBUG
+    table->name = table_name;
+    table->is_local = false;
+#endif
+    table->table_flags = table_flags;
+#ifdef HASHTABLE_STATISTICS
+    /* indicate this is first time, not a resize */
+# ifdef HASHTABLE_ENTRY_STATS
+    table->entry_stats = NULL;
+    table->added_since_dumped = 0;
+# endif
+#endif
+    ASSERT(dcontext != GLOBAL_DCONTEXT || TEST(HASHTABLE_SHARED, table_flags));
+    HTNAME(hashtable_,NAME_KEY,_init_internal)(dcontext, table, bits,
+                                               load_factor_percent, func,
+                                               hash_offset _IFLOOKUP(use_lookup));
+    ASSIGN_INIT_READWRITE_LOCK_FREE(table->rwlock, HTLOCK_RANK);
+#ifdef HASHTABLE_STATISTICS
+    INIT_HASHTABLE_STATS(table->drlookup_stats);
+#endif
+}
+
+/* caller is responsible for any needed synchronization */
+static void
+HTNAME(hashtable_,NAME_KEY,_resize)(dcontext_t *dcontext,
+                                    HTNAME(,NAME_KEY,_table_t) *table)
+{
+    HTNAME(hashtable_,NAME_KEY,_init_internal)
+        (dcontext, table, table->hash_bits, 
+         table->load_factor_percent, table->hash_func, 
+         table->hash_mask_offset
+         /* keep using lookup if used so far */
+         _IFLOOKUP(table->lookuptable != 0));
+}
+
+static inline void
+HTNAME(hashtable_,NAME_KEY,_free_table)(dcontext_t *alloc_dc,
+                                        ENTRY_TYPE *table_unaligned
+                                        _IFLOOKUP(byte *lookup_table_unaligned),
+                                        uint flags, uint capacity)
+{
+    if (table_unaligned != NULL) {
+        TABLE_MEMOP(flags, free)
+            (alloc_dc, table_unaligned,
+             HTNAME(hashtable_,NAME_KEY,_table_aligned_size)(capacity, flags)
+             HEAPACCT(HASHTABLE_WHICH_HEAP(flags)));
+    }
+#ifdef HASHTABLE_USE_LOOKUPTABLE
+    if (lookup_table_unaligned != NULL) {
+        TABLE_MEMOP(flags, free)
+            (alloc_dc, lookup_table_unaligned,
+             HTNAME(hashtable_,NAME_KEY,_lookuptable_aligned_size)(capacity)
+             HEAPACCT(HASHTABLE_WHICH_HEAP(flags)));
+    }
+#endif
+}
+
+static void
+HTNAME(hashtable_,NAME_KEY,_free)(dcontext_t *dcontext,
+                                  HTNAME(,NAME_KEY,_table_t) *table)
+{
+    LOG(THREAD, LOG_HTABLE, 1,
+        "hashtable_"KEY_STRING"_free %s table="PFX" bits=%d size=%d "
+        "load=%d%% resize=%d %s groom=%d%% groom_at=%d\n",
+        table->name, table->table, table->hash_bits, table->capacity,
+        table->load_factor_percent, table->resize_threshold,
+        IFLOOKUP_ELSE((table->lookuptable != NULL) ? "use lookup":"", ""),
+        table->groom_factor_percent, table->groom_threshold);
+
+#ifdef HASHTABLE_STATISTICS
+# ifdef HASHTABLE_ENTRY_STATS
+    if (INTERNAL_OPTION(hashtable_ibl_entry_stats)) {
+        if (TEST(HASHTABLE_USE_ENTRY_STATS, table->table_flags)) {
+            HEAP_ARRAY_FREE(dcontext, table->entry_stats, fragment_stat_entry_t,
+                            table->capacity,
+                            HASHTABLE_WHICH_HEAP(table->table_flags),
+                            UNPROTECTED);
+        } else
+            ASSERT(table->entry_stats == NULL);
+    }
+# endif
+#endif /* HASHTABLE_STATISTICS */
+
+    HTNAME(hashtable_,NAME_KEY,_free_table)(dcontext, table->table_unaligned
+                                            _IFLOOKUP(table->lookup_table_unaligned),
+                                            table->table_flags, table->capacity);
+    table->table = NULL;
+    table->table_unaligned = NULL;
+#ifdef HASHTABLE_USE_LOOKUPTABLE
+    table->lookuptable = NULL;
+    table->lookup_table_unaligned = NULL;
+#endif
+    DELETE_READWRITE_LOCK(table->rwlock);
+}
+
+/* Need to keep the cached start_pc_fragment consistent between lookuptable and
+ * the htable.
+ * Shared fragment IBTs: Unlinked lookup table entries are marked with
+ * unlinked_fragment and
+ * are expected to target a target_delete_entry. 
+ */
+#ifdef DEBUG
+static inline void
+HTNAME(hashtable_,NAME_KEY,_check_consistency)(dcontext_t *dcontext,
+                                               HTNAME(,NAME_KEY,_table_t) *htable,
+                                               uint hindex)
+{
+# ifdef HASHTABLE_USE_LOOKUPTABLE
+    if (htable->lookuptable == NULL) {
+        LOG(THREAD, LOG_HTABLE, 6,
+            "[%u] tag="PFX")\n",
+            hindex, ENTRY_TAG(htable->table[hindex]));
+    } else {
+        LOG(THREAD, LOG_HTABLE, 6,
+            "[%u] "AUX_ENTRY_FORMAT_STR" tag="PFX")\n",
+            hindex,
+            AUX_ENTRY_FORMAT_ARGS(htable->lookuptable[hindex]),
+            ENTRY_IS_REAL(htable->table[hindex]) ?
+            ENTRY_TAG(htable->table[hindex]) : 0);
+# else
+        LOG(THREAD, LOG_HTABLE, 6,
+            "[%u] tag="PFX")\n", hindex,
+            ENTRY_IS_REAL(htable->table[hindex]) ?
+            ENTRY_TAG(htable->table[hindex]) : 0);
+# endif
+        /* We can't assert that an IBL target isn't a trace head due to a race
+         * between trace head marking and adding to a table. See the comments
+         * in fragment_add_to_hashtable().
+         */
+        if (ENTRY_IS_INVALID(htable->table[hindex])) {
+            ASSERT(TEST(HASHTABLE_NOT_PRIMARY_STORAGE, htable->table_flags));
+            ASSERT(TEST(HASHTABLE_LOCKLESS_ACCESS, htable->table_flags));
+# ifdef HASHTABLE_USE_LOOKUPTABLE
+            ASSERT(AUX_ENTRY_IS_INVALID(htable->lookuptable[hindex]));
+# endif
+        }
+# ifdef HASHTABLE_USE_LOOKUPTABLE
+        /* "Inclusive hierarachy" lookup tables -- per-type tables not attached
+         * to a table such as the BB table -- are simpler to reason about
+         * since we have more latitude setting fragment_t ptrs and so can ensure
+         * that the entry is always sync-ed with the corresponding fragment_t*.
+         * For a non-"inclusive hierarachy" table, only when the entry has not
+         * been unlinked (and so doesn't point to target_delete) can we expect
+         * the lookup table and fragment_t* to be in-sync.
+         */
+        else if (TEST(HASHTABLE_NOT_PRIMARY_STORAGE, htable->table_flags) ||
+                 AUX_PAYLOAD_IS_INVALID(dcontext, htable,
+                                        htable->lookuptable[hindex])) {
+            ASSERT(AUX_ENTRY_IS_SET_TO_ENTRY(htable->lookuptable[hindex],
+                                             htable->table[hindex]));
+        }
+        /* Shouldn't be needed but could catch errors so leaving in. */
+        else {
+            ASSERT(AUX_PAYLOAD_IS_INVALID(dcontext, htable,
+                                          htable->lookuptable[hindex]));
+        }
+    }
+# endif
+}
+#endif
+
+/* Returns entry if tag found;
+ *  Otherwise returns an "empty" entry -- does NOT return NULL!
+ * Shared tables: It can be the case that an "invalid" entry is returned,
+ *     which also has a tag of 0. This can occur when htable has a lookup table
+ *     and 'tag' is for a fragment that was unlinked due to shared deletion.
+ */
+/* CHECK this routine is partially specialized, otherwise turn it into a macro */
+static /* want to inline but > limit in fragment_lookup */ ENTRY_TYPE
+HTNAME(hashtable_,NAME_KEY,_lookup)(dcontext_t *dcontext, ptr_uint_t tag,
+                                    HTNAME(,NAME_KEY,_table_t) *htable)
+{
+    ENTRY_TYPE e;
+    ptr_uint_t ftag;
+    uint hindex = HASH_FUNC(tag, htable);
+#ifdef HASHTABLE_STATISTICS
+    uint collision_len = 0;
+#endif
+    ASSERT_TABLE_SYNCHRONIZED(htable, READWRITE); /* requires read (or write) lock */
+    e = htable->table[hindex];
+
+    DODEBUG({
+        HTNAME(hashtable_,NAME_KEY,_check_consistency)(dcontext,htable,hindex);
+    });
+    while (!ENTRY_IS_EMPTY(e)) {
+        DODEBUG({
+            HTNAME(hashtable_,NAME_KEY,_check_consistency)(dcontext,htable,hindex);
+        });
+        /* If a FAKE_TAG is present in lookuptable as an unlinked marker, use the
+         * table entry. */
+#ifdef HASHTABLE_USE_LOOKUPTABLE
+        ftag = (htable->lookuptable != NULL &&
+                !AUX_ENTRY_IS_INVALID(htable->lookuptable[hindex])) ?
+                AUX_ENTRY_TAG(htable->lookuptable[hindex]) : ENTRY_TAG(e);
+#else
+        ftag = ENTRY_TAG(e);
+#endif
+        /* FIXME future frags have a 0 tag and that's why we have
+         * to compare with null_fragment for end of chain in table[]
+         * whenever future frags go to their own table, this code should
+         * be reworked to touch lookuptable only -- i.e. become while
+         * (ftag != NULL_TAG).
+         */
+        if (ftag == tag) {
+#ifdef HASHTABLE_STATISTICS
+            if (collision_len > 0) 
+                HTABLE_STAT_INC(htable,collision_hit);
+            HTABLE_STAT_INC(htable,hit);
+#endif
+            return e;
+        }
+        /* collision */
+#ifdef HASHTABLE_STATISTICS
+        LOG(THREAD_GET, LOG_HTABLE, 6,
+            "(hashtable_"KEY_STRING"_lookup: collision sequence "PFX" "
+            "%u [len=%u])\n", tag, hindex, collision_len);
+        collision_len++;
+        HTABLE_STAT_INC(htable,collision);
+#endif
+        hindex = HASH_INDEX_WRAPAROUND(hindex + 1, htable);
+        e = htable->table[hindex];
+        DODEBUG({
+            HTNAME(hashtable_,NAME_KEY,_check_consistency)(dcontext,htable,hindex);
+        });
+    }
+    HTABLE_STAT_INC(htable,miss);
+    return e;
+}
+
+/* Convenience routine that grabs the read lock and does a lookup */
+static inline ENTRY_TYPE
+HTNAME(hashtable_,NAME_KEY,_rlookup)(dcontext_t *dcontext, ptr_uint_t tag,
+                                     HTNAME(,NAME_KEY,_table_t) *htable)
+{
+    ENTRY_TYPE e;
+    TABLE_RWLOCK(htable, read, lock);
+    e = HTNAME(hashtable_,NAME_KEY,_lookup)(dcontext, tag, htable);
+    TABLE_RWLOCK(htable, read, unlock);
+    return e;
+}
+
+/* add f to a fragment table
+ * returns whether resized the table or not
+ * N.B.: this routine will recursively call itself via check_table_size if the
+ * table is resized
+ */
+static inline bool
+HTNAME(hashtable_,NAME_KEY,_add)(dcontext_t *dcontext, ENTRY_TYPE e,
+                                 HTNAME(,NAME_KEY,_table_t) *table)
+{
+    uint hindex;
+    bool resized;
+    DEBUG_DECLARE(uint cluster_len = 0;)
+
+    ASSERT_TABLE_SYNCHRONIZED(table, WRITE); /* add requires write lock */
+
+    ASSERT(!TEST(HASHTABLE_READ_ONLY, table->table_flags));
+    if (TEST(HASHTABLE_READ_ONLY, table->table_flags))
+        return false;
+
+    /* ensure higher-level synch not allowing any races between lookup and add */
+    /* Shared fragment IBTs: Tolerate unlinked markers. */
+    ASSERT(!ENTRY_IS_REAL(HTNAME(hashtable_,NAME_KEY,_lookup)
+                          (dcontext, ENTRY_TAG(e), table)));
+
+    /* check_table_size increments table->entries for us and ensures
+     * we have enough space.  don't grab any properties of table prior to this
+     * call, like hindex, as it will change if resized.
+     */
+    resized = !HTNAME(hashtable_,NAME_KEY,_check_size)(dcontext, table, 1, 0);
+
+    hindex = HASH_FUNC(ENTRY_TAG(e), table);
+    /* find an empty null slot */
+    do {
+        LOG(THREAD_GET, LOG_HTABLE, 4,
+            "hashtable_"KEY_STRING"_add("PFX") mask="PFX" offset=%d trying "
+            PFX":\n", ENTRY_TAG(e), table->hash_mask, table->hash_mask_offset, hindex);
+
+        /* We could use !ENTRY_IS_REAL() here but it could make what we're doing
+         * more confusing since the &unlinked_fragment is handled a little
+         * differently. So we enumerate the cases to make things more
+         * apparent.
+         */
+        if (ENTRY_IS_EMPTY(table->table[hindex]))
+            break;
+        /* Replace pending deletion entries in a private table but not
+         * in a shared table. */
+        if (ENTRY_IS_INVALID(table->table[hindex])) {
+            /* FIXME We cannot blindly overwrite an unlinked entry for a shared
+             * table. We overwrite an entry only when we know that since the
+             * unlink, every thread has exited the cache at least once (just as
+             * with shared deletion). Refcounting on a per-entry basis is
+             * very likely not worth the potential gain.
+             *
+             * A broader approach that may be effective enough piggybacks
+             * on shared deletion. When a flusher unlinks entries in a
+             * shared table, the table is marked FRAG_NO_CLOBBER_UNLINK --
+             * this prevents unlinked entries from being replaced. After a
+             * thread decrements a shared deletion refcount via
+             * check_flush_queue(), it checks if the shared deletion queuee
+             * is empty. If the queue is empty, then we know that since the
+             * last flush (and table unlinks) all threads have
+             * exited the cache at least once, so any shared tables marked
+             * as FRAG_NO_CLOBBER_UNLINK can have that flag cleared.
+             */
+            if (!TEST(HASHTABLE_SHARED, table->table_flags)) {
+                ASSERT(table->unlinked_entries > 0);
+                ASSERT(TEST(HASHTABLE_LOCKLESS_ACCESS, table->table_flags));
+#ifdef HASHTABLE_USE_LOOKUPTABLE
+                ASSERT(AUX_PAYLOAD_IS_INVALID(dcontext, table,
+                                              table->lookuptable[hindex]));
+                ASSERT(AUX_ENTRY_IS_INVALID(table->lookuptable[hindex]));
+                LOG(THREAD_GET, LOG_HTABLE, 4, "   replace target_delete "
+                    "entry "AUX_ENTRY_FORMAT_STR"\n",
+                    AUX_ENTRY_FORMAT_ARGS(table->lookuptable[hindex]));
+#endif
+                STATS_INC(num_ibt_replace_unlinked_fragments);
+                break;
+            }
+        }
+        DODEBUG({++cluster_len;});
+        hindex = HASH_INDEX_WRAPAROUND(hindex + 1, table);
+    } while (1); 
+
+    /* FIXME: case 4814 we may want to flush the table if we are running into a too long
+     * collision cluster
+     */
+    DOLOG(1, LOG_HTABLE, {
+        if (cluster_len > HASHTABLE_SIZE((1+table->hash_bits)/2))
+            LOG(THREAD_GET, LOG_HTABLE, 
+                cluster_len > HASHTABLE_SIZE((1+table->hash_bits)/2 + 1) ? 
+                    1U : 2U,
+                "hashtable_"KEY_STRING"_add: long collision sequence len=%u"
+                "for "PFX" %s table[%u] capacity=%d entries=%d)\n", 
+                cluster_len, ENTRY_TAG(e), table->name, hindex,
+                table->capacity, table->entries);
+    });
+
+    /* If we had uniformly distributed hash functions, expected max length is 
+     * sqrt(capacity.\pi/8) see Knuth vol.3, FIXME we double below because this
+     * sometimes ASSERTS for the shared_future_table at the 10 to 11 bits
+     * transition (seems to be fine at larger sizes)
+     * bug2241 we add an additional 64 to handle problems in private future 
+     * tables at small sizes, for bug2271 we disable for tables using the
+     * _NONE hash function (currently private bb and trace) when we have no
+     * shared fragments
+     */
+#ifdef DEBUG
+    if (!TEST(HASHTABLE_RELAX_CLUSTER_CHECKS, table->table_flags) &&
+        (table->hash_func != HASH_FUNCTION_NONE || SHARED_FRAGMENTS_ENABLED())) {
+        uint max_cluster_len =
+            HASHTABLE_SIZE((1+table->hash_bits)/2 + 1 /* double */) + 64;
+        if (!(cluster_len <= max_cluster_len)) { 
+            DO_ONCE({ /* once reach this may fire many times in a row */
+                /* always want to know which table this is */
+                SYSLOG_INTERNAL_WARNING("cluster length assert: %s cluster=%d vs %d,"
+                                        " cap=%d, entries=%d",
+                                        table->name, cluster_len, max_cluster_len,
+                                        table->capacity, table->entries);
+                DOLOG(3, LOG_HTABLE, {
+                    HTNAME(hashtable_,NAME_KEY,_dump_table)(dcontext, table);
+                });
+                ASSERT_CURIOSITY(false && "table collision cluster is too large");
+            });
+        }
+    }
+#endif
+    /* actually add the entry
+     * add to lookuptable first to avoid race spoiling ASSERT_CONSISTENCY -- better
+     * for other thread to miss in lookup
+     */
+#ifdef HASHTABLE_USE_LOOKUPTABLE
+    if (table->lookuptable != NULL) {
+        AUX_ENTRY_SET_TO_ENTRY(table->lookuptable[hindex], e);
+        ASSERT(!AUX_ENTRY_IS_INVALID(table->lookuptable[hindex]));
+    }
+#endif
+    if (ENTRY_IS_INVALID(table->table[hindex]))
+        table->unlinked_entries--;
+    table->table[hindex] = e;
+    ASSERT(!ENTRY_IS_INVALID(table->table[hindex]));
+    LOG(THREAD_GET, LOG_HTABLE, 4,
+        "hashtable_"KEY_STRING"_add: added "PFX" to %s at table[%u]\n", 
+        ENTRY_TAG(e), table->name, hindex);
+
+    return resized;
+}
+
+/* Ensures that the table has enough room for add_now+add_later new entries.
+ * If not, the table is resized and rehashed into its new space.
+ * The table's entry count is incremented by add_now but nothing changes
+ * with respect to add_later other than making space.
+ * Returns false if it had to re-size the table.
+ * Caller must hold the write lock, if this is a shared table!
+ * N.B.: this routine will recursively be called when resizing, b/c it
+ * calls fragment_add_to_hashtable, which calls back, but should never
+ * trigger another resize.
+ */
+static bool
+HTNAME(hashtable_,NAME_KEY,_check_size)(dcontext_t *dcontext,
+                                        HTNAME(,NAME_KEY,_table_t) *table,
+                                        uint add_now, uint add_later)
+{
+    dcontext_t *alloc_dc = FRAGMENT_TABLE_ALLOC_DC(dcontext, table->table_flags);
+    uint entries;
+    bool shared_lockless = TESTALL(HASHTABLE_ENTRY_SHARED | HASHTABLE_SHARED |
+                                   HASHTABLE_LOCKLESS_ACCESS, table->table_flags);
+    /* FIXME: too many assumptions here that if lockless, then a lookuptable
+     * is always used, etc.
+     */
+    bool lockless = TESTALL(HASHTABLE_LOCKLESS_ACCESS | HASHTABLE_ENTRY_SHARED,
+                            table->table_flags);
+
+    /* write lock must be held */
+    ASSERT_TABLE_SYNCHRONIZED(table, WRITE);
+
+    ASSERT(!TEST(HASHTABLE_READ_ONLY, table->table_flags));
+    if (TEST(HASHTABLE_READ_ONLY, table->table_flags))
+        return false;
+
+    /* check flush threshold to see if we'd want to flush hashtable */
+    if (table->entries > table->groom_threshold && 
+        table->groom_threshold > 0) {
+        HTNAME(hashtable_,NAME_KEY,_groom_table)(dcontext, table);
+        /* FIXME Grooming a table in-place doesn't work for a shared IBT table.
+         * To make it work, we should a) realloc a same-sized table
+         * b) re-add all entries in the old table, c) add the old
+         * to the dead list, and d) then call groom_table(). (b) is
+         * needed because we can't assume that groom_table() will
+         * always flush the entire table.
+         *
+         * Or we could groom the old table -- making sure that it's done
+         * thread-safely -- and then copy into the new table, requiring
+         * fewer re-adds.
+         *
+         * This is covered by case 5838.
+         */
+    }
+    /* FIXME: case 4814 we should move clock handles here
+     */
+#ifdef HASHTABLE_STATISTICS
+# ifdef HASHTABLE_ENTRY_STATS
+    /* dump per entry hit statistics regularly to see working set */
+    if (table->entry_stats != NULL) {
+        table->added_since_dumped += add_now;
+        if (table->added_since_dumped >= INTERNAL_OPTION(hashtable_ibl_study_interval)) {
+            LOG(THREAD, LOG_HTABLE, 2,
+                "dump_and_clean_entry_statistics %s added %d\n",
+                table->name, table->added_since_dumped);
+            HTNAME(hashtable_,NAME_KEY,_dump_entry_stats)(dcontext, table);
+            table->added_since_dumped = 0;
+        }
+    }
+# endif
+#endif /* HASHTABLE_STATISTICS */
+
+    /* Pretend we had full # entries; we'll lower later */
+    table->entries += add_now + add_later;
+
+    /* check resize threshold to see if a larger hashtable is needed 
+     * or that we may want to reset the table.
+     *
+     * For an IBT table, the # unlinked entries needs to be checked also.
+     * For a shared table, they cannot be replaced so they are effectively
+     * real entries. For a private table, they can be replaced but those
+     * that remain present can lengthen collision chain lengths.
+     *
+     * NOTE: unlinked_entries is used only when shared targets are
+     * stored in IBT tables, since unlinking in those tables is NOT
+     * followed up by a full removal of the unlinked entries.
+     */
+    entries = lockless ? table->entries + table->unlinked_entries : table->entries;
+    if (entries > table->resize_threshold) {
+        ENTRY_TYPE *old_table = table->table;
+        ENTRY_TYPE *old_table_unaligned = table->table_unaligned;
+        uint old_capacity = table->capacity;
+#ifdef HASHTABLE_USE_LOOKUPTABLE
+        AUX_ENTRY_TYPE *old_lookuptable_to_nullify = table->lookuptable;
+        byte *old_lookup_table_unaligned = table->lookup_table_unaligned;
+#endif
+        uint old_ref_count = table->ref_count - 1; /* remove this thread's
+                                                    * reference to the table */
+        uint i;
+        DEBUG_DECLARE(uint old_entries  = table->entries - add_later;)
+
+        DODEBUG({
+            /* study before resizing */
+            HTNAME(hashtable_,NAME_KEY,_study)(dcontext, table, add_now+add_later);
+        });
+ 
+#ifdef HASHTABLE_STATISTICS
+# ifdef HASHTABLE_ENTRY_STATS
+        if (table->entry_stats != NULL) {
+            LOG(THREAD, LOG_HTABLE, 2,
+                "dump_and_clean_entry_statistics %s resized\n", table->name);
+            HTNAME(hashtable_,NAME_KEY,_dump_entry_stats)(dcontext, table);
+        }
+# endif
+#endif /* HASHTABLE_STATISTICS */
+
+        /* For a shared IBT table, the flushing/grooming is done after
+         * a resize -- we can't groom in-place. */
+        if (table->hash_bits == table->max_capacity_bits && !shared_lockless) {
+            STATS_INC(num_ibt_max_capacity);
+            HTNAME(hashtable_,NAME_KEY,_groom_helper)(dcontext, table);
+            return true; /* == did not resize the table */
+        }
+
+        /* For an IBT table, if the # unlinked entries is what kicked
+         * in the resize then check the # actual entries and do
+         * a same-size realloc to eliminate the unlinked entries. Also,
+         * if we've reached the max size, do a same-size realloc.
+         *
+         * Otherwise, double the table size. 
+         */
+        if (lockless &&
+            (table->entries <= table->resize_threshold ||
+             table->hash_bits == table->max_capacity_bits)) {
+            STATS_INC(num_same_size_ibt_table_resizes);
+        }
+        else {
+            DEBUG_DECLARE(uint old_bits = table->hash_bits;)
+            ASSERT(table->resize_threshold ==
+                   (HASHTABLE_SIZE(table->hash_bits)+1/*sentinel*/)
+                   * table->load_factor_percent / 100);
+            while (entries >
+                   (HASHTABLE_SIZE(table->hash_bits)+1/*sentinel*/)
+                   * table->load_factor_percent / 100
+                   && table->hash_bits != table->max_capacity_bits) {
+                table->hash_bits++;     /* double the size */
+            }
+            ASSERT(table->hash_bits > old_bits);
+        }
+
+        HTNAME(hashtable_,NAME_KEY,_resize)(alloc_dc, table);
+        /* will be incremented by rehashing below -- in fact, by
+         * recursive calls to this routine from
+         * fragment_add_to_hashtable -- see warning below
+         */
+        table->entries = 0; 
+        table->unlinked_entries = 0; 
+        ASSERT(table->ref_count == 0);
+
+        /* can't just memcpy, must rehash */
+        /* for open address table rehash should first find an empty slot and start
+           from there so that we make sure that entries that used to find a hit on the first lookup
+           continue to do so instead of creating even longer collision parking lots 
+           FIXME: can we do better?
+        */
+        for (i = 0; i < old_capacity; i++) {
+            ENTRY_TYPE e = old_table[i];
+            if (!ENTRY_IS_REAL(e))
+                continue;
+
+            /* Don't carry over frags that point to target_delete. This can
+             * happen in any IBT table, shared or private, that targets
+             * shared fragments.
+             */
+            if (lockless && ENTRY_IS_INVALID(old_table[i])) {
+                LOG(GLOBAL, LOG_HTABLE, 1,
+                    "Don't copy tag "PFX" in %s[%u] across a resize\n",
+                    ENTRY_TAG(e), table->name, i);
+                DODEBUG({old_entries--;});
+                STATS_INC(num_ibt_unlinked_entries_not_moved);
+                continue;
+            }
+#ifdef HASHTABLE_USE_LOOKUPTABLE
+            if (lockless && old_lookuptable_to_nullify != NULL &&
+                AUX_ENTRY_IS_INVALID(old_lookuptable_to_nullify[i])) {
+                LOG(GLOBAL, LOG_HTABLE, 1,
+                    "Don't copy tag "PFX" in %s[%u] across a resize\n",
+                    ENTRY_TAG(e), table->name, i);
+                ASSERT(AUX_PAYLOAD_IS_INVALID(dcontext, table,
+                                              old_lookuptable_to_nullify[i]));
+                DODEBUG({old_entries--;});
+                STATS_INC(num_ibt_unlinked_entries_not_moved);
+                continue;
+            }
+#endif
+
+            /* N.B.: this routine will call us again, but we assume the
+             * resize will NEVER be triggered, since we hold the write lock
+             * and set table->entries to 0.
+             * We could have a special add routine that doesn't call us,
+             * I suppose.
+             */
+            HTNAME(hashtable_,NAME_KEY,_add)(dcontext, e, table);
+        }
+        table->entries += add_now; /* for about-to-be-added fragment(s) */
+        /* (add_later will be added later though calls to this same routine) */
+        /* For a shared IBT table, the flushing/grooming is done after a
+         * same-size realloc -- we can't groom the original table in-place.
+         * Groom when we know that the table was just bumped up in size
+         * earlier in the routine -- compare the old and new sizes to
+         * determine.
+         */
+        if (table->hash_bits == table->max_capacity_bits &&
+            old_capacity == table->capacity) {
+            STATS_INC(num_ibt_max_capacity);
+            ASSERT(shared_lockless);
+            HTNAME(hashtable_,NAME_KEY,_groom_helper)(dcontext, table);
+        }
+        else {
+            /* should have rehashed all old entries into new table */
+            ASSERT(table->entries == old_entries);
+        }
+
+        LOG(THREAD, LOG_HTABLE, 2,
+            "%s hashtable resized at %d entries from capacity %d to %d\n",
+            table->name, table->entries, old_capacity, table->capacity);
+
+        /* Since readers now synchronize with writers of shared htables, 
+         * we can now delete old htable even when sharing.
+         */
+
+        DODEBUG({
+            LOG(THREAD, LOG_HTABLE, 1, "Rehashed %s table\n", table->name);
+            /* study after rehashing 
+             * ok to become reader for study while a writer
+             */
+            HTNAME(hashtable_,NAME_KEY,_study)(dcontext, table, add_now);
+
+            DOLOG(3, LOG_HTABLE, {
+                HTNAME(hashtable_,NAME_KEY,_dump_table)(dcontext, table);
+            });
+        });
+
+        /* Shared IBT tables are resized at safe points, not here, since
+         * they are accessed while in-cache, unlike other shared tables
+         * such as the shared BB or shared trace table.
+         */
+        if (!shared_lockless) {
+            HTNAME(hashtable_,NAME_KEY,_free_table)
+                (alloc_dc, old_table_unaligned _IFLOOKUP(old_lookup_table_unaligned),
+                 table->table_flags, old_capacity);
+        }
+        else {
+            if (old_ref_count == 0) {
+                /* Note that a write lock is held on the table, so no danger of
+                 * a double free. */
+                HTNAME(hashtable_,NAME_KEY,_free_table)
+                    (GLOBAL_DCONTEXT, old_table_unaligned
+                     _IFLOOKUP(old_lookup_table_unaligned),
+                     table->table_flags, old_capacity);
+                STATS_INC(num_shared_ibt_tables_freed_immediately);
+            }
+        }
+
+        HTNAME(hashtable_,NAME_KEY,_resized_custom)
+            (dcontext, table, old_capacity, old_table, old_table_unaligned
+             _IFLOOKUP(old_lookuptable_to_nullify)
+             _IFLOOKUP(old_lookup_table_unaligned),
+             old_ref_count, table->table_flags);
+
+        return false; /* == resized the table */
+    }
+    /* If there are too many unlinked markers cluttering the table, remove them. */
+    else if (table->unlinked_entries > 0 &&
+             ((INTERNAL_OPTION(rehash_unlinked_threshold) < 100 &&
+               INTERNAL_OPTION(rehash_unlinked_threshold) <
+               (100 * table->unlinked_entries /
+                (table->entries  + table->unlinked_entries)))
+              || INTERNAL_OPTION(rehash_unlinked_always))) {
+        /* Currently, such markers should be present only when shared BBs are IB
+         * targets or traces are shared. */
+        ASSERT(SHARED_IB_TARGETS());
+        ASSERT(TESTALL(HASHTABLE_ENTRY_SHARED | HASHTABLE_LOCKLESS_ACCESS,
+                       table->table_flags));
+        STATS_INC(num_ibt_table_rehashes);
+        LOG(THREAD, LOG_HTABLE, 1, "Rehash table %s: linked %u, unlinked %u\n",
+            table->name, table->entries, table->unlinked_entries);
+        /* we will inc for these at a "later" time */
+        table->entries -= add_later;
+        /* entries was incremented earlier but the new entry hasn't been
+         * added yet. We decrement before and re-inc after so that any calls
+         * to study_hashtable in the remove routine succeed.
+         */
+        DODEBUG({ table->entries -= add_now; });
+        HTNAME(hashtable_,NAME_KEY,_unlinked_remove)(dcontext, table);
+        DODEBUG({ table->entries += add_now; });
+    } else {
+        /* we will inc for these at a "later" time */
+        table->entries -= add_later;
+    }
+    return true; /* == did not resize the table */
+}
+
+/* Return pointer to the record for fragment in its collision chain (table or next field)
+   or NULL if not found */
+static inline ENTRY_TYPE *
+HTNAME(hashtable_,NAME_KEY,_lookup_for_removal)(ENTRY_TYPE fr,
+                                                HTNAME(,NAME_KEY,_table_t) *htable,
+                                                uint *rhindex)
+{
+    uint hindex = HASH_FUNC(ENTRY_TAG(fr), htable);
+    ENTRY_TYPE *pg = &htable->table[hindex];
+    while (!ENTRY_IS_EMPTY(*pg)) {
+        if (ENTRIES_ARE_EQUAL(fr, *pg)) {
+            *rhindex = hindex;
+            return pg;
+        }
+        /* collision */
+        LOG(THREAD_GET, LOG_HTABLE, 6, "(hashtable_"KEY_STRING"_lookup_for_"
+            "removal: collision sequence "PFX" %u)\n", ENTRY_TAG(fr), hindex);
+        hindex = HASH_INDEX_WRAPAROUND(hindex + 1, htable);
+        pg = &htable->table[hindex];
+    }
+    *rhindex = 0; /* to make sure always initialized */
+    return NULL;
+}
+    
+#ifdef HASHTABLE_USE_LOOKUPTABLE
+/* FIXME: figure out what weight function I tipped off so now this is too much to inline */
+static INLINE_FORCED void
+HTNAME(hashtable_,NAME_KEY,_update_lookup)(HTNAME(,NAME_KEY,_table_t) *htable,
+                                           uint hindex)
+{
+    if (htable->lookuptable != NULL) {
+        AUX_ENTRY_SET_TO_ENTRY(htable->lookuptable[hindex], htable->table[hindex]);
+        LOG(THREAD_GET, LOG_HTABLE, 4, "hashtable_"KEY_STRING"_update_lookup:"
+            "updated "PFX" at table[%u]\n", 
+            AUX_ENTRY_TAG(htable->lookuptable[hindex]), hindex);
+    }
+}
+#endif
+
+/* This is based on Algorithm R from Knuth's Vol.3, section 6.4 */
+/* Deletion markers tend to form clusters ("parking lot effect"),  */
+/* and in steady state the hashtable will always be full.   */
+/* This slightly more complicated deletion scheme solves these undesired effects, */
+/* with the final arrangement being as if the elements were never inserted. */
+/* Returns whether it copied from the start of the table to the end (wraparound).
+ * FIXME: if callers need more info, could return the final hindex, or the final
+ * hole.
+ */
+static uint
+HTNAME(hashtable_,NAME_KEY,_remove_helper_open_address)
+     (HTNAME(,NAME_KEY,_table_t) *htable, uint hindex, ENTRY_TYPE *prevg)
+{
+    bool wrapped = false;
+    /* Assumptions:
+       We have to move the htable->table and lookuptable elements.
+       It is OK to do so since the address of these structures is never passed back to clients, 
+       instead, all clients can only hold onto a fragment_t* itself, not to the indirection here.
+    */
+    LOG(THREAD_GET, LOG_HTABLE, 4,
+        "hashtable_"KEY_STRING"_remove_helper_open_address(table="PFX", "
+        "hindex=%u)\n", htable, hindex);
+
+    for (;;) {
+        uint preferred;
+        uint hole = hindex;
+
+        /* first go ahead and set entry to null */
+        htable->table[hole] = ENTRY_EMPTY;
+#ifdef HASHTABLE_USE_LOOKUPTABLE
+        HTNAME(hashtable_,NAME_KEY,_update_lookup)(htable, hole);
+#endif
+        do {
+            /* positive probing to get the rest in the same cache line
+               also gains from +1 unit stride HW prefetching 
+            */
+            hindex = HASH_INDEX_WRAPAROUND(hindex + 1, htable);
+
+            /* no orphaned elements, we're done */
+            /* Note that an &unlinked_fragment will get moved since it
+             * and its lookup table entry are designed to preserve linear
+             * probing. See the comment after fragment_update_lookup()
+             * for the implications.
+             */
+            if (ENTRY_IS_EMPTY(htable->table[hindex])) 
+                return wrapped;
+
+            preferred = HASH_FUNC(ENTRY_TAG(htable->table[hindex]), htable);
+
+            /* Verify if it will be lost if we leave a hole behind its preferred address */
+            /* [preferred] <= [hole] < [hindex]  : BAD */
+            /* [hindex] < [preferred] <= [hole]  : BAD [after wraparound]  */
+            /* [hole] < [hindex] < [preferred]   : BAD [after wraparound]  */
+            /* Note the <='s:  hole != hindex, but it is possible that preferred == hole */
+        } while (!(((preferred <= hole) && (hole < hindex))      ||
+                   ((hindex < preferred) && (preferred <= hole)) ||
+                   ((hole < hindex) && (hindex < preferred))));
+
+        LOG(THREAD_GET, LOG_HTABLE, 3,
+            "hashtable_"KEY_STRING"_remove_helper_open_address: moving "PFX" "
+            "from table[%u] into table[%u], preferred=%u\n", 
+            ENTRY_TAG(htable->table[hindex]), hindex, hole, preferred);
+
+        /* need to move current entry into the hole */
+        htable->table[hole] = htable->table[hindex];
+        if (hindex < hole)
+            wrapped = true;
+#ifdef HASHTABLE_USE_LOOKUPTABLE
+        HTNAME(hashtable_,NAME_KEY,_update_lookup)(htable, hole);
+        /* Since an &unlinked entry can be moved into a hole, we take
+         * special care to sync the lookup table to preserve the assert-able
+         * conditions that an unlinked entry has a lookup table entry
+         * w/a FAKE_TAG tag and a target_delete start_pc. FAKE_TAG is
+         * copied via the preceding update_lookup() but that also sets
+         * start_pc to 0. We manually set start_pc to the start_pc value of
+         * the old entry, to carry over the target_delete value.
+         *
+         * We also block the call to fragment_update_lookup() in the
+         * caller.
+         */
+        /* FIXME We can remove this specialized code and simplify the
+         * ASSERT_CONSISTENCY by using a dedicated unlinked fragment per
+         * table. Each unlinked fragment can have its start_pc set to
+         * the corresponding target_delete value for the table.
+         */
+        if (ENTRY_IS_INVALID(htable->table[hole])) {
+            htable->lookuptable[hole] = htable->lookuptable[hindex];
+            ASSERT(!AUX_ENTRY_IS_EMPTY(htable->lookuptable[hole]));
+            LOG(THREAD_GET, LOG_HTABLE, 4, "   re-set "PFX" at table[%u]\n", 
+                AUX_ENTRY_TAG(htable->lookuptable[hindex]), hole);
+        }
+#endif
+    }
+    return wrapped;
+}
+
+/* Returns whether it copied from the start of the table to the end (wraparound). */
+static inline bool
+HTNAME(hashtable_,NAME_KEY,_remove_helper)(HTNAME(,NAME_KEY,_table_t) *htable,
+                                           uint hindex, ENTRY_TYPE *prevg)
+{
+    /* Non-trivial for open addressed scheme */
+    /* just setting elements to null will make unreachable any following elements */
+    /* better solution is to move entries that would become unreachable  */
+    bool wrapped = 
+        HTNAME(hashtable_,NAME_KEY,_remove_helper_open_address)(htable, hindex, prevg);
+
+#ifdef HASHTABLE_USE_LOOKUPTABLE
+    /* Don't sync the lookup table for unlinked fragments -- see the comments
+     * above in fragment_remove_helper_open_address(). */
+    if (!ENTRY_IS_INVALID(htable->table[hindex]))
+        HTNAME(hashtable_,NAME_KEY,_update_lookup)(htable, hindex);
+#endif
+    htable->entries--;
+    return wrapped;
+}
+
+/* Removes f from htable (does not delete f)
+ * returns true if fragment found and removed
+ */
+static inline bool
+HTNAME(hashtable_,NAME_KEY,_remove)(ENTRY_TYPE fr, HTNAME(,NAME_KEY,_table_t) *htable)
+{
+    uint hindex;
+    ENTRY_TYPE *pg;
+    ASSERT_TABLE_SYNCHRONIZED(htable, WRITE); /* remove requires write lock */
+    ASSERT(!TEST(HASHTABLE_READ_ONLY, htable->table_flags));
+    if (TEST(HASHTABLE_READ_ONLY, htable->table_flags))
+        return false;
+    pg = HTNAME(hashtable_,NAME_KEY,_lookup_for_removal)(fr, htable, &hindex);
+    if (pg != NULL) {
+        HTNAME(hashtable_,NAME_KEY,_remove_helper)(htable, hindex, pg);
+        return true;
+    }
+    return false;
+}
+
+
+/* Replaces a fragment in hashtable assuming tag is preserved
+ * returns true if fragment found and removed
+ */
+static inline bool
+HTNAME(hashtable_,NAME_KEY,_replace)
+     (ENTRY_TYPE old_e, ENTRY_TYPE new_e, HTNAME(,NAME_KEY,_table_t) *htable)
+{
+    uint hindex;
+    ENTRY_TYPE *pg =
+        HTNAME(hashtable_,NAME_KEY,_lookup_for_removal)(old_e, htable, &hindex);
+    /* replace requires write lock only because we have readers who
+     * need global consistency and replace requires two local writes!
+     */
+    ASSERT_TABLE_SYNCHRONIZED(htable, WRITE); 
+    ASSERT(!TEST(HASHTABLE_READ_ONLY, htable->table_flags));
+    if (TEST(HASHTABLE_READ_ONLY, htable->table_flags))
+        return false;
+    if (pg != NULL) {
+        ASSERT(ENTRY_TAG(old_e) == ENTRY_TAG(new_e));
+        ASSERT(ENTRIES_ARE_EQUAL(*pg, old_e));
+        *pg = new_e;
+
+#ifdef HASHTABLE_USE_LOOKUPTABLE
+        if (htable->lookuptable != NULL) {
+            /* TODO: update
+             * tag doesn't change, only start_pc may change */
+            ASSERT(AUX_ENTRY_TAG(htable->lookuptable[hindex]) ==
+                   ENTRY_TAG(htable->table[hindex]));
+            AUX_ENTRY_SET_TO_ENTRY(htable->lookuptable[hindex], htable->table[hindex]);
+        }
+#endif
+        return true;
+    }
+
+    return false;
+}
+
+/* removes all entries within a specified range of tags
+ * 
+ * should generalize hashtable_reset to do this in all cases,
+ * yet we haven't had an instance where this is necessary
+ *
+ * FIXME: note that we don't do a full type dispatch here, while
+ * hashtable_reset is not properly moving elements hence can't be used
+ * for removing subsets, and is inefficient!  
+ */
+static uint
+HTNAME(hashtable_,NAME_KEY,_range_remove)(dcontext_t *dcontext,
+                                          HTNAME(,NAME_KEY,_table_t) *table,
+                                          ptr_uint_t tag_start, ptr_uint_t tag_end)
+{
+    int i;
+    ENTRY_TYPE e;
+    uint entries_removed = 0;
+    DEBUG_DECLARE(uint entries_initial = 0;)
+
+    ASSERT(!TEST(HASHTABLE_READ_ONLY, table->table_flags));
+    if (TEST(HASHTABLE_READ_ONLY, table->table_flags))
+        return 0;
+    LOG(THREAD, LOG_HTABLE, 2, "hashtable_"KEY_STRING"_range_remove\n");
+    DOLOG(2, LOG_HTABLE|LOG_STATS, {
+        HTNAME(hashtable_,NAME_KEY,_load_statistics)(dcontext, table);
+    });
+    DODEBUG({
+        HTNAME(hashtable_,NAME_KEY,_study)(dcontext, table, 0/*table consistent*/);
+        /* ensure write lock is held if the table is shared, unless exiting */
+        if (!dynamo_exited)
+            ASSERT_TABLE_SYNCHRONIZED(table, WRITE);  /* remove requires write lock */
+        entries_initial = table->entries;
+    });
+
+    /* Deletion in hashtable_NAME_KEY_remove_helper has to move entries
+     * in order to keep all reachable.  We go in reverse order to
+     * efficiently delete all entries */
+    i = table->capacity - 1 - 1 /* sentinel */; 
+    while (i >= 0) {
+        e = table->table[i];
+        if (!ENTRY_IS_EMPTY(e)
+            && ENTRY_TAG(e) >= tag_start && ENTRY_TAG(e) < tag_end) {
+            if (HTNAME(hashtable_,NAME_KEY,_remove_helper)(table, i, &table->table[i])) {
+                /* Pulled a chain across the wraparound, so we must start over
+                 * at the end as otherwise we will never look at that last
+                 * element (case 10384).
+                 */
+                i = table->capacity - 1 - 1 /* sentinel */;
+            } else {
+                /* we can assume deletion doesn't move any entries ahead of i
+                 * to smaller values, so i stays here
+                 */
+            }
+            entries_removed++;
+            /* de-allocate payload */
+            HTNAME(hashtable_,NAME_KEY,_free_entry)(dcontext, table, e);
+        } else {
+            i--;
+        }
+    }
+    ASSERT(table->entries + entries_removed == entries_initial);
+    return entries_removed;
+}
+
+/* removes all unlinked entries from table's lookup table */
+static uint
+HTNAME(hashtable_,NAME_KEY,_unlinked_remove)(dcontext_t *dcontext,
+                                             HTNAME(,NAME_KEY,_table_t) *table)
+{
+    uint i;
+    ENTRY_TYPE e;
+    uint entries_removed = 0;
+
+    ASSERT(!TEST(HASHTABLE_READ_ONLY, table->table_flags));
+    if (TEST(HASHTABLE_READ_ONLY, table->table_flags))
+        return 0;
+    LOG(THREAD, LOG_HTABLE, 2, "hashtable_"KEY_STRING"_unlinked_remove\n");
+    /* body based on hashtable_range_remove() */
+
+    ASSERT(TEST(HASHTABLE_LOCKLESS_ACCESS, table->table_flags));
+    DOLOG(2, LOG_HTABLE|LOG_STATS, {
+        HTNAME(hashtable_,NAME_KEY,_load_statistics)(dcontext, table);
+    });
+    DODEBUG({
+        /* ensure write lock is held if the table is shared, unless exiting */
+        if (!dynamo_exited)
+            ASSERT_TABLE_SYNCHRONIZED(table, WRITE);  /* remove requires write lock */
+    });
+    /* Deletion in fragment_remove_fragment_helper has to move entries
+     * in order to keep all reachable.  We go in reverse order to
+     * efficiently delete all entries
+     */
+    i = table->capacity - 1 - 1 /* sentinel */;
+    while (i >= 0) {
+        e = table->table[i];
+        if (ENTRY_IS_INVALID(e)) {
+            LOG(THREAD, LOG_HTABLE|LOG_STATS, 2,
+                "unlinked_remove(%s) -- %u\n", table->name, i);
+            if (HTNAME(hashtable_,NAME_KEY,_remove_helper)(table, i, &table->table[i])) {
+                /* Pulled a chain across the wraparound, so we must start over
+                 * at the end as otherwise we will never look at that last
+                 * element (case 10384).
+                 */
+                i = table->capacity - 1 - 1 /* sentinel */;
+            } else {
+                /* we can assume deletion doesn't move any entries ahead of i
+                 * to smaller values, so i stays here
+                 */
+            }
+            entries_removed++;
+        } else {
+            i--;
+        }
+    }
+    /* fragment_remove_fragment_helper decrements table->entries but we
+     * want only unlinked_entries decremented, so adjust entries. */
+    table->entries += entries_removed;
+    LOG(THREAD, LOG_HTABLE|LOG_STATS, 1,
+        "unlinked_remove(%s) -- %d deletions\n", table->name, entries_removed);
+    ASSERT(entries_removed == table->unlinked_entries);
+#ifdef HASHTABLE_USE_LOOKUPTABLE
+    DODEBUG({
+        /* Check that there are no remnants of unlinked fragments in the table. */
+        for (i = 0; i < table->capacity; i++) {
+            ASSERT(!ENTRY_IS_INVALID(table->table[i]));
+            ASSERT(!AUX_ENTRY_IS_INVALID(table->lookuptable[i]));
+            ASSERT(!AUX_PAYLOAD_IS_INVALID(dcontext, table, table->lookuptable[i]));
+        }
+    });
+#else
+    DODEBUG({
+        /* Check that there are no remnants of unlinked fragments in the table. */
+        for (i = 0; i < table->capacity; i++) {
+            ASSERT(!ENTRY_IS_INVALID(table->table[i]));
+        }
+    });
+#endif
+    table->unlinked_entries = 0;
+    DOLOG(3, LOG_HTABLE, {
+        HTNAME(hashtable_,NAME_KEY,_dump_table)(dcontext, table);
+    });
+    DODEBUG({
+        HTNAME(hashtable_,NAME_KEY,_study)(dcontext, table, 0/*table consistent*/);
+    });
+    return entries_removed;
+}
+
+/* we should clean the table from entries that are not frequently used
+ */
+static void
+HTNAME(hashtable_,NAME_KEY,_groom_table)(dcontext_t *dcontext,
+                                         HTNAME(,NAME_KEY,_table_t) *table)
+{
+    DOLOG(1, LOG_STATS, { print_timestamp(THREAD); });
+    LOG(THREAD, LOG_HTABLE, 1,
+        "hashtable_"KEY_STRING"_groom_table %s\n", table->name);
+
+    /* flush only tables caching data persistent in another table */
+    ASSERT(TEST(HASHTABLE_NOT_PRIMARY_STORAGE, table->table_flags));
+
+    DOLOG(3, LOG_HTABLE, {
+        HTNAME(hashtable_,NAME_KEY,_dump_table)(dcontext, table);
+    });
+
+    LOG(THREAD, LOG_HTABLE, 2,
+        "%s hashtable flushing at %d entries capacity %d\n",
+        table->name, table->entries, table->capacity);
+
+    /* most simple grooming technique - just flush everyone */
+    HTNAME(hashtable_,NAME_KEY,_range_remove)(dcontext, table, 0, UINT_MAX);
+    ASSERT(table->entries == 0);
+
+    /* FIXME: we should do better - we can tag fragments that have
+     * been readded after getting flushed, so that they are not
+     * flushed next time we do this Some kind of aging that can be
+     * used also if we do a real clock working set.
+     */
+
+    /* will not flush again until table gets resized */
+    table->groom_threshold = 0;
+
+    /* FIXME: we may want to do this more often - so that we can catch
+     * phases and that we don't even have to resize if working set
+     * does in fact fit here.  In that case we may want to 
+     * do have a step function, 
+     * e.g. (groom 50, resize 80, groom step 10) translating into
+     * 50 - groom, 60 - groom, 70 - groom, 80 - resize 
+     */
+
+    STATS_INC(num_ibt_groomed);
+    LOG(THREAD, LOG_HTABLE, 1,
+        "hashtable_"KEY_STRING"_groom_table %s totally groomed - "
+        "should be empty\n", table->name);
+    DOLOG(3, LOG_HTABLE, {
+        HTNAME(hashtable_,NAME_KEY,_dump_table)(dcontext, table);
+    });
+}
+
+static inline void
+HTNAME(hashtable_,NAME_KEY,_groom_helper)(dcontext_t *dcontext,
+                                          HTNAME(,NAME_KEY,_table_t) *table)
+{
+    /* flush only tables caching data persistent in another table */
+    ASSERT(TEST(HASHTABLE_NOT_PRIMARY_STORAGE, table->table_flags));
+
+    ASSERT(table->hash_bits != 0);
+    /* can't double size, and there is no point in resizing
+     * we have to flush it.
+     */
+    LOG(THREAD, LOG_HTABLE, 1,
+        "hashtable_"KEY_STRING"_check_size reached maximum size %s\n",
+        table->name);
+    /* Currently groom_table() resets the whole table, but if
+     * it gets smarter, we may want to invoke the reset all
+     * logic here.
+     */
+    table->entries--;   /* entry not added yet */
+    HTNAME(hashtable_,NAME_KEY,_groom_table)(dcontext, table);
+    table->entries++;
+    /* can't make forward progress if groom_table() doesn't
+     * remove at least one entry
+     */
+    ASSERT(table->entries <= table->resize_threshold);
+}
+
+
+#ifdef DEBUG
+/* The above dyn.avg.coll is a little off: we can't show average
+ * succesfull search time, since some collisions are for misses but
+ * indcalls_miss_stat includes misses both with and without
+ * collisions.
+ */
+static void
+HTNAME(hashtable_,NAME_KEY,_study)(dcontext_t *dcontext,
+                                   HTNAME(,NAME_KEY,_table_t) *table, 
+                                   uint entries_inc/*amnt table->entries was pre-inced*/)
+{
+    /* hashtable sparseness study */
+    uint i;
+    uint max = 0;
+    uint num = 0, len = 0, num_collisions = 0;
+    uint colliding_frags = 0;
+    uint total_len = 0;
+    uint hindex;
+    const char *name = table->name;
+    uint ave_len_threshold;
+
+    uint overwraps = 0;
+    ENTRY_TYPE e;
+    bool lockless_access = TEST(HASHTABLE_LOCKLESS_ACCESS, table->table_flags);
+
+    /* studying needs the entire table to be in a consistent state.
+     * we considered having read mean local read/write and write mean global
+     * read/write, thus making study() a writer and add() a reader,
+     * but we do want add() and remove() to be exclusive w/o relying on the
+     * bb building lock, so we have them as writers and study() as a reader.
+     */
+    TABLE_RWLOCK(table, read, lock);
+
+    for (i = 0; i < table->capacity; i++) {
+        e = table->table[i];
+        DODEBUG({ HTNAME(hashtable_,NAME_KEY,_check_consistency)(dcontext,table,i); });
+
+        if (ENTRY_IS_EMPTY(e))
+            continue;
+
+        if (ENTRY_IS_SENTINEL(e)) {
+            ASSERT(i == table->capacity - 1);
+            /* don't count in collision length - not a real fragment */
+            continue;
+        }
+
+        hindex = HASH_FUNC(ENTRY_TAG(e), table);
+        if (i < hindex) {
+            len = i + (table->capacity - hindex - 1) + 1; /* counting the sentinel */
+            overwraps++;
+            LOG(THREAD, LOG_HTABLE|LOG_STATS, 2, 
+                "WARNING: hashtable_"KEY_STRING"_study: overwrap[%d] of "
+                "len=%d, F tag="PFX",i=%d,hindex=%d\n", 
+                overwraps, len, ENTRY_TAG(e), i, hindex);
+        }
+        else {
+            len = i - hindex + 1;
+        }
+        
+        if (ENTRY_IS_INVALID(e)) {
+            ASSERT(lockless_access);
+            LOG(THREAD, LOG_HTABLE|LOG_STATS, 2,
+                "hashtable_"KEY_STRING"_study: entry not found in %s[%u]\n",
+                table->name, i);
+            len = 0;
+            continue;
+        }
+        /* check for unique entries - the entry found by
+         * fragment_lookup_in_hashtable should be us! */
+        else {
+            uint found_at_hindex;
+            ENTRY_TYPE *pg =
+                HTNAME(hashtable_,NAME_KEY,_lookup_for_removal)
+                (e, table, &found_at_hindex);
+            ASSERT(pg != NULL);
+            ASSERT(ENTRIES_ARE_EQUAL(*pg, e));
+            ASSERT(found_at_hindex == i && "duplicate entry found");
+        }
+        if (len > 0) {
+            if (len > max)
+                max = len;
+            total_len += len;
+            num++;
+            if (len > 1) {
+                num_collisions++;
+                colliding_frags += len;
+            }
+        }
+    }
+
+    DOLOG(1, LOG_HTABLE|LOG_STATS, {
+        uint st_top=0; uint st_bottom=0;
+        if (num != 0)
+            divide_uint64_print(total_len, num, false, 2, &st_top, &st_bottom);
+        LOG(THREAD, LOG_HTABLE|LOG_STATS, 1,
+            "%s %s hashtable statistics: num=%d, max=%d, #>1=%d, st.avg=%u.%.2u\n",
+            entries_inc == 0 ? "Total" : "Current", name,
+            num, max, num_collisions, st_top, st_bottom);
+    });
+
+    /* static average length is supposed to be under 5 even up
+     * to load factors of 90% see Knuth vol.3 or in CLR (p.238-9 in
+     * first edition) but of course only if we had uniformly
+     * distributed hash functions 
+     */
+    /* for bug 2271 we make more lenient for non trace tables using the
+     * _NONE hash function (i.e. private bb) when we are !SHARED_FRAGMENTS_ENABLED().
+     */
+    if (!TEST(HASHTABLE_RELAX_CLUSTER_CHECKS, table->table_flags) &&
+        (lockless_access || table->hash_func != HASH_FUNCTION_NONE ||
+         SHARED_FRAGMENTS_ENABLED()))
+        ave_len_threshold = 5;
+    else
+        ave_len_threshold = 10;
+    DOLOG(1, LOG_HTABLE|LOG_STATS, {
+        /* This happens enough that it's good to get some info on it */
+        if (!((total_len <= ave_len_threshold * num
+               || (TEST(HASHTABLE_RELAX_CLUSTER_CHECKS, table->table_flags) &&
+                   table->capacity <= 513))
+              && "hash table high average collision length")) {
+            LOG(THREAD, LOG_HTABLE|LOG_STATS, 1,
+                "htable %s ave len: tot=%d <= %d, cap=%d entr=%d fac=%d\n",
+                name, total_len, ave_len_threshold*num, table->capacity, table->entries,
+                table->load_factor_percent);
+            HTNAME(hashtable_,NAME_KEY,_dump_table)(dcontext, table);
+        }
+    });
+    ASSERT_CURIOSITY((total_len <= ave_len_threshold * num
+                      || (TEST(HASHTABLE_RELAX_CLUSTER_CHECKS, table->table_flags) &&
+                          table->capacity <= 513))
+                     && "hash table high average collision length");
+    DOLOG(3, LOG_HTABLE, {
+        HTNAME(hashtable_,NAME_KEY,_dump_table)(dcontext, table);
+    });
+
+    {
+        uint doublecheck = num + entries_inc;
+
+        LOG(THREAD, LOG_HTABLE|LOG_STATS, 2,
+            "\t%s doublecheck %u (unlinked %u) == %u %d\n",
+            name, table->entries, table->unlinked_entries, doublecheck, entries_inc);
+        ASSERT(table->entries == doublecheck);
+    }
+
+#ifdef HASHTABLE_STATISTICS
+    print_hashtable_stats(dcontext, entries_inc == 0 ? "Total" : "Current", 
+                          name,
+                          "fragment_lookup", "", &table->drlookup_stats);
+#endif
+
+    HTNAME(hashtable_,NAME_KEY,_study_custom)(dcontext, table, entries_inc);
+
+    TABLE_RWLOCK(table, read, unlock);
+}
+
+void
+HTNAME(hashtable_,NAME_KEY,_dump_table)(dcontext_t *dcontext,
+                                        HTNAME(,NAME_KEY,_table_t) *htable)
+{
+    uint i;
+    bool track_cache_lines;
+    bool entry_size;
+    size_t cache_line_size = proc_get_cache_line_size();
+    uint cache_lines_used = 0;
+    bool cache_line_in_use = false;
+
+#ifdef HASHTABLE_USE_LOOKUPTABLE
+    track_cache_lines = true;
+    entry_size = sizeof(AUX_ENTRY_TYPE);
+#else
+    track_cache_lines = TEST(HASHTABLE_ALIGN_TABLE, htable->table_flags);
+    entry_size = sizeof(ENTRY_TYPE);
+#endif
+
+    DOLOG(1, LOG_HTABLE|LOG_STATS, {
+        HTNAME(hashtable_,NAME_KEY,_load_statistics)(dcontext, htable);
+    });
+    LOG(THREAD, LOG_HTABLE, 1,
+        "  i      tag     coll     hits  age %s dump\n", htable->name);
+    /* need read lock to traverse the table */
+    TABLE_RWLOCK(htable, read, lock);
+    for (i = 0; i < htable->capacity; i++) {
+        if (track_cache_lines && (i*entry_size % cache_line_size == 0)) {
+            if (cache_line_in_use)
+                cache_lines_used++;
+            cache_line_in_use = false;
+        }
+
+        if (ENTRY_IS_INVALID(htable->table[i])) {
+            LOG(THREAD, LOG_HTABLE, 1, "%6x *** unlinked marker ***\n", i);
+        }
+        else if (!ENTRY_IS_EMPTY(htable->table[i])) {
+            uint preferred = HASH_FUNC(ENTRY_TAG(htable->table[i]), htable);
+            uint collision_len = (preferred <= i) ? i - preferred /* collision */
+                : (htable->capacity + i - preferred)    /* with overwrap */;
+            /* overwrap count should include the sentinel to total length */
+# if defined(HASHTABLE_STATISTICS) && defined(HASHTABLE_ENTRY_STATS)
+            LOG(THREAD, LOG_HTABLE, 1,
+                "%6x "PFX" %3d %c  %8d  %3d\n",
+                i, ENTRY_TAG(htable->table[i]), 
+                collision_len + 1, (preferred <= i) ? ' ' : 'O' /* overwrap */,
+                htable->entry_stats ? htable->entry_stats[i].hits : 0,
+                htable->entry_stats ? htable->entry_stats[i].age : 0
+                );
+# else /* !(HASHTABLE_STATISTICS && HASHTABLE_ENTRY_STATS) */
+            LOG(THREAD, LOG_HTABLE, 1,
+                "%6x "PFX" %3d %c \n",
+                i, ENTRY_TAG(htable->table[i]), 
+                collision_len + 1, (preferred <= i) ? ' ' : 'O' /* overwrap */
+                );
+# endif /* (HASHTABLE_STATISTICS && HASHTABLE_ENTRY_STATS) */
+            if (track_cache_lines)
+                cache_line_in_use = true;
+        } else {
+            /* skip null_fragment entries */
+        }
+
+        DOLOG(2, LOG_HTABLE, {
+            /* print full table */
+            if (ENTRY_IS_EMPTY(htable->table[i]))
+                LOG(THREAD, LOG_HTABLE, 2,
+                    "%6x "PFX"\n", i, 0);
+        });
+        DOLOG(2, LOG_HTABLE, {
+            if (track_cache_lines &&
+                (i+1)*entry_size % cache_line_size == 0 && cache_line_in_use)
+                LOG(THREAD, LOG_HTABLE, 1,
+                    "----cache line----\n");
+        });
+        DODEBUG({ HTNAME(hashtable_,NAME_KEY,_check_consistency)(dcontext, htable, i); });
+    }
+    if (track_cache_lines) {
+        if (cache_line_in_use)
+            cache_lines_used++;
+        if (cache_lines_used) {
+            LOG(THREAD, LOG_HTABLE, 1, "%s %d%% cache density, cache_lines_used=%d "
+                "(%dKB), minimum needed %d (%dKB)\n", 
+                htable->name,
+                100 * htable->entries * entry_size /
+                (cache_lines_used * cache_line_size),
+                cache_lines_used, cache_lines_used*cache_line_size/1024,
+                htable->entries*entry_size/cache_line_size, 
+                htable->entries*entry_size/1024);
+        }
+    }
+    TABLE_RWLOCK(htable, read, unlock);
+}
+#endif /* DEBUG */
+
+#if defined(DEBUG) && defined(INTERNAL)
+static void
+HTNAME(hashtable_,NAME_KEY,_load_statistics)(dcontext_t *dcontext,
+                                             HTNAME(,NAME_KEY,_table_t) *table)
+{
+    LOG(THREAD, LOG_HTABLE|LOG_STATS, 1,
+        "%s hashtable: %d entries, %d unlinked entries, %d capacity,"
+        " %d%% load\n",
+        table->name,
+        table->entries, table->unlinked_entries, table->capacity, 
+        (100 * table->entries) / table->capacity);
+}
+#endif
+
+#ifdef HASHTABLE_STATISTICS
+# ifdef HASHTABLE_ENTRY_STATS
+void
+HTNAME(hashtable_,NAME_KEY,_dump_entry_stats)(dcontext_t *dcontext,
+                                              HTNAME(,NAME_KEY,_table_t) *htable)
+{
+    /* mostly a copy of dump_table but printing only entries with non 0 stats */
+    uint i;
+    uint max_age = 0;
+    DOLOG(1, LOG_STATS, { print_timestamp(THREAD); });
+    LOG(THREAD, LOG_HTABLE, 1, "dump_and_clean_entry_statistics: %s\n", htable->name);
+
+    DOLOG(1, LOG_HTABLE|LOG_STATS, {
+        HTNAME(hashtable_,NAME_KEY,_load_statistics)(dcontext, htable);
+
+        /* TODO: should preserve a copy of the old hashtable_statistics_t
+         * so that the difference between the two can be matched with
+         * the per-entry hits and collisions.  though a Perl script will do.
+         */
+    });
+    LOG(THREAD, LOG_HTABLE, 1,
+        "  i      tag     coll       hits   age %s\n", htable->name);
+    /* need read lock to traverse the table */
+    TABLE_RWLOCK(htable, read, lock);
+    for (i = 0; i < htable->capacity; i++) {
+        if (!ENTRY_IS_EMPTY(htable->table[i]) &&
+            !ENTRY_IS_INVALID(htable->table[i])) {
+            uint preferred = HASH_FUNC(ENTRY_TAG(htable->table[i]), htable);
+            uint collision_len = (preferred <= i) ? i - preferred /* collision */
+                : (htable->capacity + i - preferred)    /* with overwrap */;
+
+            if (htable->entry_stats[i].hits == 0) {
+                /* no hits in a row */
+                htable->entry_stats[i].age++;
+                if (max_age < htable->entry_stats[i].age)
+                    max_age = htable->entry_stats[i].age;
+            }
+
+            LOG(THREAD, LOG_HTABLE, htable->entry_stats[i].hits ? 1U: 2U, /* only hits */
+                "%6x "PFX" %3d %c %8d   %3d\n",
+                i, ENTRY_TAG(htable->table[i]), 
+                collision_len + 1, (preferred <= i) ? ' ' : 'O' /* overwrap */,
+                htable->entry_stats[i].hits, htable->entry_stats[i].age
+                );
+
+        } else {
+            /* skip null_fragment entries */
+        }
+        DODEBUG({ HTNAME(hashtable_,NAME_KEY,_check_consistency)(dcontext, htable, i); });
+    }
+    if (max_age > 0) {
+        LOG(THREAD, LOG_HTABLE, 1,
+            "hashtable_"KEY_STRING"dump_entry_stats: %s max_age:%d\n",
+            htable->name, max_age);
+    }
+
+    TABLE_RWLOCK(htable, read, unlock);
+}
+# endif
+#endif /* HASHTABLE_STATISTICS */
+
+#ifdef HASHTABLE_SUPPORT_PERSISTENCE
+uint
+HTNAME(hashtable_,NAME_KEY,_persist_size)(dcontext_t *dcontext,
+                                          HTNAME(,NAME_KEY,_table_t) *htable)
+{
+    if (htable == NULL)
+        return 0;
+    return (sizeof(*htable) + (htable->capacity * sizeof(ENTRY_TYPE)));
+}
+
+/* We need enough of the htable struct fields that we simply persist
+ * the entire struct to avoid defining a separate subset struct.
+ * The version field of the entire file suffices to version the htable struct.
+ * The table pointer (and stats in debug) need to be writable,
+ * so at load time we do not directly use the mmapped copy, but we could
+ * with a little work (case 10349: see comments below as well) (not worth
+ * making COW and having the whole page private: case 9582).
+ * Returns true iff all writes succeeded.
+ */
+bool
+HTNAME(hashtable_,NAME_KEY,_persist)(dcontext_t *dcontext,
+                                     HTNAME(,NAME_KEY,_table_t) *htable,
+                                     file_t fd)
+{
+    uint size;
+    ASSERT(htable != NULL);
+    if (htable == NULL)
+        return false;
+    ASSERT(fd != INVALID_FILE);
+    size = sizeof(*htable);
+    if (os_write(fd, htable, size) != (int)size)
+        return false;
+    /* We don't bother to align */
+    size = htable->capacity * sizeof(ENTRY_TYPE);
+    if (os_write(fd, htable->table, size) != (int)size)
+        return false;
+    return true;
+}
+
+static uint
+HTNAME(hashtable_,NAME_KEY,_num_unique_entries)(dcontext_t *dcontext,
+                                                HTNAME(,NAME_KEY,_table_t) *src1,
+                                                HTNAME(,NAME_KEY,_table_t) *src2)
+{
+    HTNAME(,NAME_KEY,_table_t) *big, *small;
+    uint unique, i;
+    ENTRY_TYPE e;
+    ASSERT(src1 != NULL && src2 != NULL);
+    if (src1->entries >= src2->entries) {
+        big = src1;
+        small = src2;
+    } else {
+        big = src2;
+        small = src1;
+    }
+    unique = big->entries;
+    /* Deadlock-avoidance won't let us grab both locks; for now we only
+     * use this when all-synched so we let that solve the problem.
+     * N.B.: we assume that on suspend failure for flush synchall
+     * (which ignores such failures) we do not come here as we abort
+     * coarse freezing/merging/persist.  FIXME: should we export
+     * another variable, or not set dynamo_all_threads_synched?
+     */
+    ASSERT(dynamo_all_threads_synched);
+    DODEBUG({ big->is_local = true; });
+    TABLE_RWLOCK(small, read, lock);
+    for (i = 0; i < small->capacity; i++) {
+        e = small->table[i];
+        if (ENTRY_IS_REAL(e)) {
+            if (ENTRY_IS_EMPTY(HTNAME(hashtable_,NAME_KEY,_lookup(dcontext,
+                                                                  ENTRY_TAG(e), big))))
+                unique++;
+        }
+    }
+    TABLE_RWLOCK(small, read, unlock);
+    DODEBUG({ big->is_local = false; });
+    return unique;
+}
+
+/* Adds all entries from src to dst */
+static void
+HTNAME(hashtable_,NAME_KEY,_add_all)(dcontext_t *dcontext,
+                                     HTNAME(,NAME_KEY,_table_t) *dst,
+                                     HTNAME(,NAME_KEY,_table_t) *src,
+                                     bool check_dups)
+{
+    ENTRY_TYPE e;
+    uint i;
+    ASSERT(dst != NULL && src != NULL);
+    /* assumption: dst is private to this thread and so does not need synch */
+    DODEBUG({ dst->is_local = true; });
+    TABLE_RWLOCK(src, read, lock);
+    for (i = 0; i < src->capacity; i++) {
+        e = src->table[i];
+        if (ENTRY_IS_REAL(e)) {
+            if (!check_dups ||
+                ENTRY_IS_EMPTY(HTNAME(hashtable_,NAME_KEY,_lookup(dcontext,
+                                                                  ENTRY_TAG(e), dst)))) {
+                /* add will also add lookuptable entry, if any */
+                HTNAME(hashtable_,NAME_KEY,_add)(dcontext, e, dst);
+            }
+        }
+    }
+    TABLE_RWLOCK(src, read, unlock);
+    DODEBUG({ dst->is_local = false; });
+}
+
+/* Creates a new hashtable that contains the union of src1 and src2 (removing
+ * the duplicates)
+ */
+HTNAME(,NAME_KEY,_table_t) *
+HTNAME(hashtable_,NAME_KEY,_merge)(dcontext_t *dcontext,
+                                   HTNAME(,NAME_KEY,_table_t) *src1,
+                                   HTNAME(,NAME_KEY,_table_t) *src2)
+{
+    HTNAME(,NAME_KEY,_table_t) *dst;
+    uint merged_entries, init_bits;
+    ASSERT(src1 != NULL && src2 != NULL);
+    /* We only support merging the same type of table */
+    ASSERT((src1->table_flags & ~(HASHTABLE_COPY_IGNORE_FLAGS)) ==
+           (src2->table_flags & ~(HASHTABLE_COPY_IGNORE_FLAGS)));
+    ASSERT(src1->load_factor_percent == src2->load_factor_percent);
+    ASSERT(src1->hash_func == src2->hash_func);
+    dst = (HTNAME(,NAME_KEY,_table_t) *)
+        TABLE_TYPE_MEMOP(src1->table_flags, ALLOC, dcontext, HTNAME(,NAME_KEY,_table_t),
+                         HASHTABLE_WHICH_HEAP(src1->table_flags), PROTECTED);
+    merged_entries =
+        HTNAME(hashtable_,NAME_KEY,_num_unique_entries)(dcontext, src1, src2);
+    LOG(THREAD, LOG_HTABLE, 2,
+        "hashtable_"KEY_STRING"_merge: %d + %d => %d unique entries\n",
+        src1->entries, src2->entries, merged_entries);
+    init_bits =
+        hashtable_bits_given_entries(merged_entries, src1->load_factor_percent);
+    HTNAME(hashtable_,NAME_KEY,_init)(dcontext, dst, init_bits,
+                                      src1->load_factor_percent, src1->hash_func,
+                                      src1->hash_mask_offset _IFLOOKUP(use_lookup),
+                                      src1->table_flags & ~(HASHTABLE_COPY_IGNORE_FLAGS)
+                                      NAME(src1->name));
+    HTNAME(hashtable_,NAME_KEY,_add_all)(dcontext, dst, src1, false/*don't check dups*/);
+    HTNAME(hashtable_,NAME_KEY,_add_all)(dcontext, dst, src2, true/*check dups*/);
+    return dst;
+}
+
+/* Performs a deep copy (struct plus table) of src */
+HTNAME(,NAME_KEY,_table_t) *
+HTNAME(hashtable_,NAME_KEY,_copy)(dcontext_t *dcontext,
+                                  HTNAME(,NAME_KEY,_table_t) *src)
+{
+    HTNAME(,NAME_KEY,_table_t) *dst;
+    ASSERT(src != NULL);
+    dst = (HTNAME(,NAME_KEY,_table_t) *)
+        TABLE_TYPE_MEMOP(src->table_flags, ALLOC, dcontext, HTNAME(,NAME_KEY,_table_t),
+                         HASHTABLE_WHICH_HEAP(src->table_flags), PROTECTED);
+    /* use init() rather than memcpy of header, so we get table and/or lookuptable
+     * allocated to proper alignment
+     */
+    HTNAME(hashtable_,NAME_KEY,_init)(dcontext, dst, src->hash_bits,
+                                      src->load_factor_percent, src->hash_func,
+                                      src->hash_mask_offset _IFLOOKUP(use_lookup),
+                                      src->table_flags & ~(HASHTABLE_COPY_IGNORE_FLAGS)
+                                      NAME(src->name));
+    dst->entries = src->entries;
+    dst->unlinked_entries = src->unlinked_entries;
+    if (dst->table != NULL)
+        memcpy(dst->table, src->table, dst->capacity*sizeof(ENTRY_TYPE));
+#ifdef HASHTABLE_USE_LOOKUPTABLE
+    if (dst->lookuptable != NULL)
+        memcpy(dst->lookuptable, src->lookuptable, dst->capacity*sizeof(AUX_ENTRY_TYPE));
+#endif
+    return dst;
+}
+
+/* See comments in HTNAME(hashtable_,NAME_KEY,_persist)
+ * Returns a newly allocated struct on the heap.
+ */
+HTNAME(,NAME_KEY,_table_t) *
+HTNAME(hashtable_,NAME_KEY,_resurrect)(dcontext_t *dcontext, byte *mapped_table
+                                       NAME(const char *table_name))
+{
+    HTNAME(,NAME_KEY,_table_t) *htable;
+    uint flags;
+    ASSERT(mapped_table != NULL);
+    flags = ((HTNAME(,NAME_KEY,_table_t) *)mapped_table)->table_flags;
+    /* FIXME: the free, and the init alloc, are in client code: would be better
+     * to have all in same file using more-easily-kept-consistent alloc routines
+     */
+    htable = (HTNAME(,NAME_KEY,_table_t) *)
+        TABLE_TYPE_MEMOP(flags, ALLOC, dcontext, HTNAME(,NAME_KEY,_table_t),
+                         HASHTABLE_WHICH_HEAP(table->table_flags), PROTECTED);
+    /* FIXME case 10349: we could directly use the mmapped struct when
+     * !HASHTABLE_STATISTICS if we supported calculating where the table lies
+     * in all the htable routines, and set HASHTABLE_READ_ONLY when persisting
+     */
+    memcpy(htable, mapped_table, sizeof(*htable));
+    htable->table_flags |= HASHTABLE_READ_ONLY;
+    htable->table = (ENTRY_TYPE *) (mapped_table + sizeof(*htable));
+    htable->table_unaligned = NULL;
+    ASSIGN_INIT_READWRITE_LOCK_FREE(htable->rwlock, HTLOCK_RANK);
+    DODEBUG({
+        htable->name = table_name;
+    });
+#ifdef HASHTABLE_STATISTICS
+    INIT_HASHTABLE_STATS(htable->drlookup_stats);
+#endif
+    return htable;
+}
+#endif /* HASHTABLE_SUPPORT_PERSISTENCE */
+
+#endif
+/****************************************************************************/
+
+/* make it easier to have multiple instantiations in a file
+ * user must re-specify name and types for struct and impl, though
+ */
+#undef NAME_KEY
+#undef ENTRY_TYPE
+#undef AUX_ENTRY_TYPE
+#undef CUSTOM_FIELDS
+
+#undef ENTRY_TAG
+#undef ENTRY_IS_EMPTY
+#undef ENTRY_IS_SENTINEL
+#undef ENTRY_IS_INVALID
+#undef ENTRIES_ARE_EQUAL
+#undef ENTRY_EMPTY
+#undef ENTRY_SENTINEL
+
+#undef AUX_ENTRY_TAG
+#undef AUX_ENTRY_IS_EMPTY
+#undef AUX_ENTRY_IS_SENTINEL
+#undef AUX_ENTRY_IS_INVALID
+#undef AUX_PAYLOAD_IS_INVALID
+#undef AUX_ENTRY_SET_TO_SENTINEL
+#undef AUX_ENTRY_IS_SET_TO_ENTRY
+#undef AUX_ENTRY_SET_TO_ENTRY
+#undef AUX_ENTRY_FORMAT_STR
+#undef AUX_ENTRY_FORMAT_ARGS
+
+#undef HASHTABLE_WHICH_HEAP
+#undef HASHTABLE_USE_LOOKUPTABLE
+#undef HASHTABLE_ENTRY_STATS
+#undef HASHTABLE_SUPPORT_PERSISTENCE
+#undef HTLOCK_RANK
+
+#undef _IFLOOKUP
+#undef IFLOOKUP_ELSE
diff --git a/core/heap.c b/core/heap.c
new file mode 100644
index 000000000..c5f1f78b8
--- /dev/null
+++ b/core/heap.c
@@ -0,0 +1,4580 @@
+/* **********************************************************
+ * Copyright (c) 2001-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2001 Hewlett-Packard Company */
+
+/*
+ * heap.c - heap manager
+ */
+
+#include "globals.h"
+#include <string.h>             /* for memcpy */
+#include <limits.h>
+
+#include "fragment.h"  /* for struct sizes */
+#include "link.h"      /* for struct sizes */
+#include "instr.h"     /* for struct sizes */
+#include "fcache.h"    /* fcache_low_on_memory */
+#ifdef DEBUG
+# include "hotpatch.h" /* To handle leak for case 9593. */
+#endif
+
+#ifdef HEAP_ACCOUNTING
+# ifndef DEBUG
+# error HEAP_ACCOUNTING requires DEBUG
+# endif
+#endif
+
+extern bool vm_areas_exited;
+
+/***************************************************************************
+ * we cannot use malloc in the middle of interpreting the client program
+ * because we could be in the middle of interpreting malloc, which is not
+ * always reentrant
+ * 
+ * We have a virtual memory manager which makes sure memory is
+ * reserved within the application address space so that we don't have
+ * to fight with the application.  We call os_heap_reserve to allocate
+ * virtual space in a single consecutive region.  We later use
+ * os_heap_commit to get committed memory in large chunks and manage
+ * the chunks using a simple scheme of free lists of different sizes.
+ * The virtual memory manager has to store out of band information about 
+ * used and free blocks, since of course there is no real memory to use.
+ * The chunks (heap units) store in band extra information both for
+ * used and free.  However, in the allocated blocks within a unit we
+ * don't need to store any information since heap_free passes in the
+ * size; we store the next pointers for the free lists at the start of
+ * the free blocks themselves.  We have one large reservation for most of 
+ * our allocations, and yet another for allocations that we do not
+ * plan on ever freeing up on detach - the only unavoidable tombstones
+ * are those for thread private code system calls that may be stuck on
+ * callbacks.  In case we run out of reserved memory we do fall back
+ * on requests from the OS, but any of these may fail if we are
+ * competing with the application.
+ * 
+ * looking at dynamo behavior as of Jan 2001, most heap_alloc requests are
+ * for < 128 bytes, very few for larger, so we have a bunch of fixed-size
+ * blocks of small sizes
+ *
+ * the UINT_MAX size is a variable-length block, we keep one byte to store
+ * the size (again storing the next pointer when free at the start of
+ * what we pass to the user)
+ */
+
+static const uint BLOCK_SIZES[] = {
+    8, /* for instr bits */
+#ifndef X64
+    /* for x64 future_fragment_t is 24 bytes (could be 20 if we could put flags last) */
+    sizeof(future_fragment_t), /* 12 (24 x64) */
+#endif
+    /* we have a lot of size 16 requests for IR but they are transient */
+    24, /* fcache empties and vm_area_t are now 20, vm area extras still 24 */
+    ALIGN_FORWARD(sizeof(fragment_t) + sizeof(indirect_linkstub_t), HEAP_ALIGNMENT), /* 40 dbg / 36 rel */
+#if defined(X64) || defined(PROFILE_LINKCOUNT) || defined(CUSTOM_EXIT_STUBS)
+    sizeof(instr_t), /* 64 (104 x64) */
+    sizeof(fragment_t) + sizeof(direct_linkstub_t)
+        + sizeof(cbr_fallthrough_linkstub_t), /* 68 dbg / 64 rel, 112 x64 */
+    /* all other bb/trace buckets are 8 larger but in same order */
+#else
+    sizeof(fragment_t) + sizeof(direct_linkstub_t)
+        + sizeof(cbr_fallthrough_linkstub_t), /* 60 dbg / 56 rel */
+    sizeof(instr_t), /* 64 */
+#endif
+    /* we keep this bucket even though only 10% or so of normal bbs
+     * hit this.
+     * FIXME: release == instr_t here so a small waste when walking buckets 
+     */
+    ALIGN_FORWARD(sizeof(fragment_t) + 2*sizeof(direct_linkstub_t),
+                  HEAP_ALIGNMENT), /* 68 dbg / 64 rel (128 x64) */
+    ALIGN_FORWARD(sizeof(trace_t) + 2*sizeof(direct_linkstub_t) + sizeof(uint),
+                  HEAP_ALIGNMENT), /* 80 dbg / 76 rel (148 x64 => 152) */
+    /* FIXME: measure whether should put in indirect mixes as well */
+    ALIGN_FORWARD(sizeof(trace_t) + 3*sizeof(direct_linkstub_t) + sizeof(uint),
+                  HEAP_ALIGNMENT), /* 96 dbg / 92 rel (180 x64 => 184) */
+    ALIGN_FORWARD(sizeof(trace_t) + 5*sizeof(direct_linkstub_t) + sizeof(uint),
+                  HEAP_ALIGNMENT), /* 128 dbg / 124 rel (244 x64 => 248) */
+    256,
+    512,
+    UINT_MAX /* variable-length */
+};
+#define BLOCK_TYPES (sizeof(BLOCK_SIZES)/sizeof(uint))
+
+#ifdef DEBUG
+/* FIXME: would be nice to have these stats per HEAPACCT category */
+/* These are ints only b/c we used to do non-atomic adds and wanted to
+ * gracefully handle underflow to negative values
+ */
+DECLARE_NEVERPROT_VAR(static int block_total_count[BLOCK_TYPES], {0});
+DECLARE_NEVERPROT_VAR(static int block_count[BLOCK_TYPES], {0});
+DECLARE_NEVERPROT_VAR(static int block_peak_count[BLOCK_TYPES], {0});
+DECLARE_NEVERPROT_VAR(static int block_wasted[BLOCK_TYPES], {0});
+DECLARE_NEVERPROT_VAR(static int block_peak_wasted[BLOCK_TYPES], {0});
+DECLARE_NEVERPROT_VAR(static int block_align_pad[BLOCK_TYPES], {0});
+DECLARE_NEVERPROT_VAR(static int block_peak_align_pad[BLOCK_TYPES], {0});
+DECLARE_NEVERPROT_VAR(static bool out_of_vmheap_once, false);
+#endif
+
+/* variable-length: we steal one int for the size */
+#define HEADER_SIZE  (sizeof(size_t))
+/* VARIABLE_SIZE is assignable */
+#define VARIABLE_SIZE(p) (*(size_t *)((p)-HEADER_SIZE))
+#define MEMSET_HEADER(p, value) VARIABLE_SIZE(p) = HEAP_TO_PTR_UINT(value)
+#define GET_VARIABLE_ALLOCATION_SIZE(p)  (VARIABLE_SIZE(p) + HEADER_SIZE)
+
+/* heap is allocated in units
+ * we start out with a small unit, then each additional unit we
+ * need doubles in size, up to a maximum, we default to 32kb initial size
+ * (24kb useful with guard pages), max size defaults to 64kb (56kb useful with
+ * guard pages), we keep the max small to save memory, it doesn't seem to be 
+ * perf hit! Though with guard pages we are wasting quite a bit of reserved 
+ * (though not commited) space */
+/* the only big things global heap is used for are pc sampling
+ * hash table and sideline sampling hash table -- if none of those
+ * are in use, 16KB should be plenty, we default to 32kb since guard
+ * pages are on by default (gives 24kb useful) max size is same as for 
+ * normal heap units.
+ */
+/* the old defaults were 32kb (usable) for initial thread private and 16kb 
+ * (usable) for initial global, changed to simplify the logic for allocating
+ * in multiples of the os allocation granularity.  The new defaults prob.
+ * make more sense with the shared cache then the old did anyways.
+ */
+/* restrictions -
+ * any guard pages are included in the size, size must be > UNITOVERHEAD
+ * for best performance sizes should be of the form
+ * 2^n * page_size (where n is a positve integer) and max should be a multiple
+ * of the os allocation granularity so that with enough doublings we are 
+ * reserving memory in multiples of the allocation granularity and not wasting
+ * any virtual address space (beyond our guard pages)
+ */ 
+#define HEAP_UNIT_MIN_SIZE INTERNAL_OPTION(initial_heap_unit_size)
+#define HEAP_UNIT_MAX_SIZE INTERNAL_OPTION(max_heap_unit_size)
+#define GLOBAL_UNIT_MIN_SIZE INTERNAL_OPTION(initial_global_heap_unit_size)
+
+#define GUARD_PAGE_ADJUSTMENT (dynamo_options.guard_pages ? 2 * PAGE_SIZE : 0)
+
+/* gets usable space in the unit */
+#define UNITROOM(u) ((size_t) (u->end_pc - u->start_pc))
+#define UNIT_RESERVED_ROOM(u) (u->reserved_end_pc - u->start_pc)
+/* we keep the heap_unit_t header at top of the unit, this macro calculates
+ * the commited size of the unit by adding header size to available size
+ */
+#define UNIT_COMMIT_SIZE(u) (UNITROOM(u) + sizeof(heap_unit_t))
+#define UNIT_RESERVED_SIZE(u) (UNIT_RESERVED_ROOM(u) + sizeof(heap_unit_t))
+#define UNIT_ALLOC_START(u) (u->start_pc - sizeof(heap_unit_t))
+#define UNIT_GET_START_PC(u) (byte*)(((ptr_uint_t)u) + sizeof(heap_unit_t))
+#define UNIT_COMMIT_END(u) (u->end_pc)
+#define UNIT_RESERVED_END(u) (u->reserved_end_pc)
+
+/* gets the allocated size of the unit (reserved size + guard pages) */
+#define UNITALLOC(u) (UNIT_RESERVED_SIZE(u) + GUARD_PAGE_ADJUSTMENT)
+/* gets unit overhead, includes the reserved (guard pages) and commited 
+ * (sizeof(heap_unit_t)) portions
+ */ 
+#define UNITOVERHEAD (sizeof(heap_unit_t) + GUARD_PAGE_ADJUSTMENT)
+
+/* any alloc request larger than this needs a special unit */
+#define MAXROOM (HEAP_UNIT_MAX_SIZE - UNITOVERHEAD)
+
+/* maximum valid allocation (to guard against internal integer overflows) */
+#define MAX_VALID_HEAP_ALLOCATION INT_MAX
+
+/* thread-local heap structure
+ * this struct is kept at top of unit itself, not in separate allocation
+ */
+typedef struct _heap_unit_t {
+    heap_pc start_pc;         /* start address of heap storage */
+    heap_pc end_pc;           /* open-ended end address of heap storage */
+    heap_pc cur_pc;           /* open-ended current end of allocated storage */
+    heap_pc reserved_end_pc;  /* open-ended end of reserved (not nec committed) memory */
+    bool in_vmarea_list;      /* perf opt for delayed batch vmarea updating */
+#ifdef DEBUG
+    int      id;              /* # of this unit */
+#endif
+    struct _heap_unit_t *next_local;  /* used to link thread's units */
+    struct _heap_unit_t *next_global; /* used to link all units */
+    struct _heap_unit_t *prev_global; /* used to link all units */
+} heap_unit_t;
+
+#ifdef HEAP_ACCOUNTING
+typedef struct _heap_acct_t {
+    size_t alloc_reuse[ACCT_LAST];
+    size_t alloc_new[ACCT_LAST];
+    size_t cur_usage[ACCT_LAST];
+    size_t max_usage[ACCT_LAST];
+    size_t max_single[ACCT_LAST];
+    uint num_alloc[ACCT_LAST];
+} heap_acct_t;
+#endif
+
+/* FIXME (case 6336): rename to heap_t:
+ *   a heap_t is a collection of units with the same properties
+ * to reflect that this is used for more than just thread-private memory.
+ * Also rename the "tu" vars to "h"
+ */
+typedef struct _thread_units_t {
+    heap_unit_t *top_unit; /* start of linked list of heap units */
+    heap_unit_t *cur_unit; /* current unit in heap list */
+    heap_pc free_list[BLOCK_TYPES];
+#ifdef DEBUG
+    int num_units;   /* total # of heap units */
+#endif
+    dcontext_t *dcontext;  /* back pointer to owner */
+    bool writable;       /* remember state of heap protection */
+#ifdef HEAP_ACCOUNTING
+    heap_acct_t acct;
+#endif
+} thread_units_t;
+
+/* per-thread structure: */
+typedef struct _thread_heap_t {
+    thread_units_t *local_heap;
+    thread_units_t *nonpersistent_heap;
+} thread_heap_t;
+
+/* global, unique thread-shared structure: 
+ * FIXME: give this name to thread_units_t, and name this AllHeapUnits
+ */
+typedef struct _heap_t {
+    heap_unit_t *units;     /* list of all allocated units */
+    heap_unit_t *dead;      /* list of deleted units ready for re-allocation */
+    /* FIXME: num_dead duplicates stats->heap_num_free, but we want num_dead
+     * for release build too, so it's separate...can we do better?
+     */
+    uint num_dead;
+} heap_t;
+
+/* no synch needed since only written once */
+static bool heap_exiting = false;
+
+/* Lock used only for managing heap units, not for normal thread-local alloc.
+ * Must be recursive due to circular dependencies between vmareas and global heap.
+ * Furthermore, always grab dynamo_vm_areas_lock() before grabbing this lock,
+ * to make DR areas update and heap alloc/free atomic!
+ */
+DECLARE_CXTSWPROT_VAR(static recursive_lock_t heap_unit_lock,
+                      INIT_RECURSIVE_LOCK(heap_unit_lock));
+/* N.B.: if these two locks are ever owned at the same time, the convention is
+ * that global_alloc_lock MUST be grabbed first, to avoid deadlocks
+ */
+/* separate lock for global heap access to avoid contention between local unit
+ * creation and global heap alloc
+ * must be recursive so that heap_vmareas_synch_units can hold it and heap_unit_lock
+ * up front to avoid deadlocks, and still allow vmareas to global_alloc --
+ * BUT we do NOT want global_alloc() to be able to recurse!
+ * FIXME: either find a better solution to the heap_vmareas_synch_units deadlock
+ * that is as efficient, or find a way to assert that the only recursion is
+ * from heap_vmareas_synch_units to global_alloc
+ */
+DECLARE_CXTSWPROT_VAR(static recursive_lock_t global_alloc_lock,
+                      INIT_RECURSIVE_LOCK(global_alloc_lock));
+
+#if defined(DEBUG) && defined(HEAP_ACCOUNTING) && defined(HOT_PATCHING_INTERFACE)
+static int get_special_heap_header_size(void);
+#endif
+
+vm_area_vector_t *landing_pad_areas;    /* PR 250294 */
+#ifdef WINDOWS
+static void release_landing_pad_mem(void);
+#endif
+
+/* Indicates whether should back out of a global alloc/free and grab the
+ * DR areas lock first, to retry
+ */
+static bool
+safe_to_allocate_or_free_heap_units()
+{
+    return ((!self_owns_recursive_lock(&global_alloc_lock) &&
+             !self_owns_recursive_lock(&heap_unit_lock)) ||
+            self_owns_dynamo_vm_area_lock());
+}
+
+/* indicates a dynamo vm area remove was delayed 
+ * protected by the heap_unit_lock
+ */
+DECLARE_FREQPROT_VAR(static bool dynamo_areas_pending_remove, false);
+
+#ifdef HEAP_ACCOUNTING
+const char * whichheap_name[] = {
+    /* max length for aligned output is length of "BB Fragments" */
+    "BB Fragments",
+    "Coarse Links",
+    "Future Frag",
+    "Frag Tables",
+    "IBL Tables",
+    "Traces",
+    "FC Empties",
+    "Vm Multis",
+    "IR",
+    "RCT Tables",
+    "VM Areas",
+    "Symbols",
+# ifdef SIDELINE
+    "Sideline",
+# endif
+    "TH Counter",
+    "Tombstone",
+    "Hot Patching",
+    "Thread Mgt",
+    "Memory Mgt",
+    "Stats",
+    "SpecialHeap",
+    /* NOTE: Add your heap name here */
+    "Other",
+};
+
+/* Since using a lock for these stats adds a lot of contention, we
+ * follow a two-pronged strategy:
+ * 1) For accurate stats we add a thread's final stats to the global only
+ * when it is cleaned up.  But, this prevents global stats from being
+ * available in the middle of a run or if a run is not cleaned up nicely.
+ * 2) We have a set of heap_accounting stats for incremental global stats
+ * that are available at any time, yet racy and so may be off a little.
+ */
+/* all set to 0 is only initialization we need */
+DECLARE_NEVERPROT_VAR(static thread_units_t global_racy_units, {0});
+
+/* macro to get the type abstracted */
+# define ACCOUNT_FOR_ALLOC_HELPER(type, tu, which, alloc_sz, ask_sz) do { \
+    (tu)->acct.type[which] += alloc_sz;                                \
+    (tu)->acct.num_alloc[which]++;                                     \
+    (tu)->acct.cur_usage[which] += alloc_sz;                           \
+    if ((tu)->acct.cur_usage[which] > (tu)->acct.max_usage[which])     \
+        (tu)->acct.max_usage[which] = (tu)->acct.cur_usage[which];     \
+    if (ask_sz > (tu)->acct.max_single[which])                         \
+        (tu)->acct.max_single[which] = ask_sz;                         \
+} while (0)
+
+# define ACCOUNT_FOR_ALLOC(type, tu, which, alloc_sz, ask_sz) do { \
+    STATS_ADD_PEAK(heap_claimed, alloc_sz);                        \
+    ACCOUNT_FOR_ALLOC_HELPER(type, tu, which, alloc_sz, ask_sz);   \
+    ACCOUNT_FOR_ALLOC_HELPER(type, &global_racy_units, which,      \
+                             alloc_sz, ask_sz);                    \
+} while (0)
+
+# define ACCOUNT_FOR_FREE(tu, which, size) do {    \
+    STATS_SUB(heap_claimed, (size));         \
+    (tu)->acct.cur_usage[which] -= size;           \
+    global_racy_units.acct.cur_usage[which] -= size;    \
+} while (0)
+
+#else
+# define ACCOUNT_FOR_ALLOC(type, tu, which, alloc_sz, ask_sz)
+# define ACCOUNT_FOR_FREE(tu, which, size)
+#endif
+
+typedef byte *vm_addr_t;
+
+#ifdef X64
+/* designates the boundaries within which we must allocate DR heap space */
+static byte *heap_allowable_region_start = (byte *)PTR_UINT_0;
+static byte *heap_allowable_region_end = (byte *)POINTER_MAX;
+
+/* used only to protect read/write access to the must_reach_* static variables in
+ * request_region_be_heap_reachable() */
+DECLARE_CXTSWPROT_VAR(static mutex_t request_region_be_heap_reachable_lock,
+                      INIT_LOCK_FREE(request_region_be_heap_reachable_lock));
+
+/* Request that the supplied region be 32bit offset reachable from the DR heap.  Should
+ * be called before vmm_heap_init() so we can place the DR heap to meet these constraints.
+ * Can also be called post vmm_heap_init() but at that point acts as an assert that the
+ * supplied region is reachable since the heap is already reserved. */
+void
+request_region_be_heap_reachable(byte *start, size_t size)
+{
+    /* initialize so will be overridden on first call; protected by the
+     * request_region_be_heap_reachable_lock */
+    static byte *must_reach_region_start = (byte *)POINTER_MAX;
+    static byte *must_reach_region_end = (byte *)PTR_UINT_0;
+
+    LOG(GLOBAL, LOG_HEAP, 2,
+        "Adding must-be-reachable-from-heap region "PFX"-"PFX"\n"
+        "Existing must-be-reachable region "PFX"-"PFX"\n"
+        "Existing allowed range "PFX"-"PFX"\n",
+        start, start+size, must_reach_region_start, must_reach_region_end,
+        heap_allowable_region_start, heap_allowable_region_end);
+    ASSERT(start + size >= start); /* overflow check */
+
+    mutex_lock(&request_region_be_heap_reachable_lock);
+    if (start < must_reach_region_start) {
+        byte *allowable_end_tmp;
+        SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+        must_reach_region_start = start;
+        allowable_end_tmp = REACHABLE_32BIT_END(must_reach_region_start,
+                                                must_reach_region_end);
+        /* PR 215395 - add in absolute address reachability */
+        if (DYNAMO_OPTION(heap_in_lower_4GB) &&
+            allowable_end_tmp > ( byte *)POINTER_MAX_32BIT) {
+            allowable_end_tmp = (byte *)POINTER_MAX_32BIT;
+        }
+        /* Write assumed to be atomic so we don't have to hold a lock to use
+         * heap_allowable_region_end. */
+        heap_allowable_region_end = allowable_end_tmp;
+        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+    } 
+    if (start + size > must_reach_region_end) {
+        SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+        must_reach_region_end = start + size;
+        /* Write assumed to be atomic so we don't have to hold a lock to use
+         * heap_allowable_region_start. */
+        heap_allowable_region_start = REACHABLE_32BIT_START(must_reach_region_start,
+                                                            must_reach_region_end);
+        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+    }
+    ASSERT(must_reach_region_start <= must_reach_region_end); /* correctness check */
+    /* verify can be addressed absolutely (if required), correctness check */
+    ASSERT(!DYNAMO_OPTION(heap_in_lower_4GB) ||
+           heap_allowable_region_end <= (byte *)POINTER_MAX_32BIT);
+    mutex_unlock(&request_region_be_heap_reachable_lock);
+
+    LOG(GLOBAL, LOG_HEAP, 1,
+        "Added must-be-reachable-from-heap region "PFX"-"PFX"\n"
+        "New must-be-reachable region "PFX"-"PFX"\n"
+        "New allowed range "PFX"-"PFX"\n",
+        start, start+size, must_reach_region_start, must_reach_region_end,
+        heap_allowable_region_start, heap_allowable_region_end);
+
+    /* Reachability checks (xref PR 215395, note since we currently can't directly
+     * control where DR/client dlls are loaded these could fire if rebased). */
+    ASSERT(heap_allowable_region_start <= must_reach_region_start && 
+           "PR 215395 reachability contraints not satisfiable");
+    ASSERT(must_reach_region_end <= heap_allowable_region_end && 
+           "PR 215395 reachability contraints not satisfiable");
+
+    /* Handle release build failure. */
+    if (heap_allowable_region_start > must_reach_region_start ||
+        must_reach_region_end > heap_allowable_region_end) {
+        /* FIXME - in a released product we may want to detach or something else less
+         * drastic than triggering a FATAL_USAGE_ERROR. */
+        FATAL_USAGE_ERROR(HEAP_CONTRAINTS_UNSATISFIABLE, 2,
+                          get_application_name(), get_application_pid());
+    }
+}
+#endif
+
+/* forward declarations of static functions */
+static void threadunits_init(dcontext_t *dcontext, thread_units_t *tu, size_t size);
+/* dcontext only used for debugging */
+static void threadunits_exit(thread_units_t *tu, dcontext_t *dcontext);
+static void *common_heap_alloc(thread_units_t *tu, size_t size
+                               HEAPACCT(which_heap_t which));
+static bool common_heap_free(thread_units_t *tu, void *p, size_t size
+                             HEAPACCT(which_heap_t which));
+static void release_real_memory(void *p, size_t size, bool remove_vm);
+static void release_guarded_real_memory(vm_addr_t p, size_t size, bool remove_vm,
+                                        bool guarded);
+
+typedef enum {
+    /* I - Init, Interop - first allocation failed
+     *    check for incompatible kernel drivers 
+     */
+    OOM_INIT    = 0x1,
+    /* R - Reserve - out of virtual reservation *
+     *    increase -vm_size to reserve more memory
+     */
+    OOM_RESERVE = 0x2,
+    /* C - Commit - systemwide page file limit, or current process job limit hit
+     * Increase pagefile size, check for memory leak in any application.
+     *
+     * FIXME: possible automatic actions
+     *    if systemwide failure we may want to wait if transient 
+     *    FIXME: if in a job latter we want to detect and just die 
+     *    (though after freeing as much memory as we can)
+     */
+    OOM_COMMIT  = 0x4,
+    /* E - Extending Commit - same reasons as Commit 
+     *    as a possible workaround increasing -heap_commit_increment
+     *    may make expose us to commit-ing less frequently,
+     *    On the other hand committing smaller chunks has a higher 
+     *    chance of getting through when there is very little memory.
+     *
+     *    FIXME: not much more informative than OOM_COMMIT
+     */
+    OOM_EXTEND  = 0x8,
+} oom_source_t;
+
+static void report_low_on_memory(oom_source_t source, 
+                                 heap_error_code_t os_error_code);
+
+/* virtual memory manager */
+enum {VMM_BLOCK_SIZE = 64*1024}; /* 64KB */
+/* Our current allocation unit matches the allocation granularity on
+   windows, to avoid worrying about external fragmentation 
+   Since most of our allocations fall within this range this makes the
+   common operation be finding a single empty block.
+*/
+
+enum {
+    /* maximum 512MB virtual memory units */
+    MAX_VMM_HEAP_UNIT_SIZE = 512*1024*1024, 
+    /* We should normally have only one large unit, so this is in fact
+     * the maximum we should count on in one process 
+     */
+    /* minimum will be used only if an invalid option is set */
+    MIN_VMM_HEAP_UNIT_SIZE = VMM_BLOCK_SIZE 
+};
+
+typedef struct {
+    vm_addr_t start_addr;         /* base virtual address */
+    vm_addr_t end_addr;           /* noninclusive virtual memory range [start,end) */
+    vm_addr_t alloc_start;        /* base allocation virtual address */
+    size_t    alloc_size;         /* allocation size */
+    /* for 64-bit do we want to shift to size_t to allow a larger region? 
+     * if so must update the bitmap_t routines
+     */
+    uint    num_blocks;         /* total number of blocks in virtual allocation */
+
+    mutex_t   lock;               /* write access to the rest of the fields is protected */
+    /* We make an assumption about the bitmap_t implementation being
+       static therefore we don't grab locks on read accesses.  Anyways,
+       currently the bitmap_t is used with no write intent only for ASSERTs. */
+    uint    num_free_blocks;    /* currently free blocks */
+    /* Bitmap uses 1KB static data for granularity 64KB and static maximum 512MB */
+    /* Since we expect only two of these, for now it is ok for users
+       to have static max rather than dynamically allocating with
+       exact size - however this field is left last in the structure
+       in case we do want to save some memory 
+    */
+    bitmap_element_t blocks[BITMAP_INDEX(MAX_VMM_HEAP_UNIT_SIZE/VMM_BLOCK_SIZE)]; 
+} vm_heap_t;
+
+/* We keep our heap management structs on the heap for selfprot (case 8074).
+ * Note that we do have static structs for bootstrapping and we later move
+ * the data here.
+ */
+typedef struct _heap_management_t {
+    /* high-level management */
+    /* we reserve only a single vm_heap_t for guaranteed allocation, 
+     * we fall back to OS when run out of reservation space */
+    vm_heap_t vmheap;
+    heap_t heap;
+    /* thread-shared heaps: */
+    thread_units_t global_units;
+    thread_units_t global_nonpersistent_units;
+    bool global_heap_writable;
+    thread_units_t global_unprotected_units;
+} heap_management_t;
+
+/* For bootstrapping until we can allocate our real heapmgt (case 8074).
+ * temp_heapmgt.lock is initialized in vmm_heap_unit_init().
+ */
+static heap_management_t temp_heapmgt;
+static heap_management_t *heapmgt = &temp_heapmgt; /* initial value until alloced */
+
+static bool vmm_heap_exited = false; /* FIXME: used only to thwart stack_free from trying, 
+                                        should change the interface for the last stack
+                                     */
+
+static inline
+uint
+vmm_addr_to_block(vm_heap_t *vmh, vm_addr_t p)
+{
+    ASSERT(CHECK_TRUNCATE_TYPE_uint((p - vmh->start_addr) / VMM_BLOCK_SIZE));
+    return (uint) ((p - vmh->start_addr) / VMM_BLOCK_SIZE);
+}
+
+static inline
+vm_addr_t
+vmm_block_to_addr(vm_heap_t *vmh, uint block)
+{
+    ASSERT(block >=0 && block < vmh->num_blocks);
+    return (vm_addr_t)(vmh->start_addr + block*VMM_BLOCK_SIZE);
+}
+
+static bool
+vmm_in_same_block(vm_addr_t p1, vm_addr_t p2)
+{
+    return vmm_addr_to_block(&heapmgt->vmheap, p1) ==
+        vmm_addr_to_block(&heapmgt->vmheap, p2);
+}
+
+#if defined(DEBUG) && defined(INTERNAL)
+static void
+vmm_dump_map(vm_heap_t *vmh)
+{
+    uint i;
+    bitmap_element_t *b = vmh->blocks;
+    uint bitmap_size = vmh->num_blocks;
+    uint last_i = 0;
+    bool is_used = bitmap_test(b, 0) == 0;
+
+    LOG(GLOBAL, LOG_HEAP, 3, "vmm_dump_map("PFX")\n", vmh);
+    /* raw dump first - if you really want binary dump use windbg's dyd */
+    DOLOG(3, LOG_HEAP, {
+        dump_buffer_as_bytes(GLOBAL, b, 
+                             BITMAP_INDEX(bitmap_size)*sizeof(bitmap_element_t), 
+                             DUMP_RAW|DUMP_ADDRESS);
+    });
+
+    LOG(GLOBAL, LOG_HEAP, 1, "\nvmm_dump_map("PFX") virtual regions\n", vmh);
+#define VMM_DUMP_MAP_LOG(i, last_i)                                                     \
+    LOG(GLOBAL, LOG_HEAP, 1, PFX"-"PFX" size=%d %s\n", vmm_block_to_addr(vmh, last_i), \
+        vmm_block_to_addr(vmh, i-1) + VMM_BLOCK_SIZE - 1,                               \
+        (i-last_i)*VMM_BLOCK_SIZE,                                                      \
+        is_used ? "reserved" : "free");
+
+    for (i=0; i < bitmap_size; i++) {
+        /* start counting at free/used boundaries */
+        if (is_used != (bitmap_test(b, i) == 0)) {
+            VMM_DUMP_MAP_LOG(i, last_i);
+            is_used = (bitmap_test(b, i) == 0);
+            last_i = i;
+        }
+    }
+    VMM_DUMP_MAP_LOG(bitmap_size, last_i);
+}
+#endif /* DEBUG */
+
+void
+print_vmm_heap_data(file_t outf)
+{
+    mutex_lock(&heapmgt->vmheap.lock);
+    print_file(outf, "VM heap: addr range "PFX"--"PFX", # free blocks %d\n",
+               heapmgt->vmheap.start_addr, heapmgt->vmheap.end_addr,
+               heapmgt->vmheap.num_free_blocks);
+    mutex_unlock(&heapmgt->vmheap.lock);
+}
+
+static inline
+void
+vmm_heap_initialize_unusable(vm_heap_t *vmh)
+{
+    vmh->start_addr = vmh->end_addr = NULL;
+    vmh->num_free_blocks = vmh->num_blocks = 0;
+}
+
+static
+void
+vmm_heap_unit_init(vm_heap_t *vmh, size_t size)
+{
+    ptr_uint_t preferred;
+    heap_error_code_t error_code;
+    ASSIGN_INIT_LOCK_FREE(vmh->lock, vmh_lock);
+    
+    size = ALIGN_FORWARD(size, VMM_BLOCK_SIZE);
+    ASSERT(size <= MAX_VMM_HEAP_UNIT_SIZE);
+    vmh->alloc_size = size;
+
+    if (size == 0) {
+        vmm_heap_initialize_unusable(&heapmgt->vmheap);
+        return;
+    }
+
+    /* Out of 32 bits = 12 bits are page offset, windows wastes 4 more
+     * since its allocation base is 64KB, and if we want to stay
+     * safely in say 0x20000000-0x2fffffff we're left with only 12
+     * bits of randomness - which may be too little.  On the other
+     * hand changing any of the lower 16 bits will make our bugs
+     * non-deterministic. */
+    /* Make sure we don't waste the lower bits from our random number */
+    preferred = (
+#if defined(WINDOWS) || !defined(X64)
+                 DYNAMO_OPTION(vm_base)
+#else
+                 /* For 64-bit linux we can't use a fixed base since our library
+                  * might be loaded anywhere.  For now we just hope to find
+                  * enough free space at this random offset from it. 
+                  * Probably the best long-term solution is to make our heap
+                  * and library not need to reach each other, which should only
+                  * require indirect jumps in fcache_{enter,return}.  Selfmod
+                  * s2ro isn't perf-critical.  Client library callees and data
+                  * should ideally be close to the heap though.
+                  * Xref PR 253624.
+                  */
+                 (ptr_uint_t) get_dynamorio_dll_start()
+#endif
+                 + get_random_offset(DYNAMO_OPTION(vm_max_offset)/VMM_BLOCK_SIZE)
+                 *VMM_BLOCK_SIZE);
+    preferred = ALIGN_FORWARD(preferred, VMM_BLOCK_SIZE);
+
+    /* let's assume a single chunk is sufficient to reserve */
+    vmh->start_addr = NULL;
+#ifdef X64
+    if ((byte *)preferred < heap_allowable_region_start ||
+        (byte *)preferred + size > heap_allowable_region_end) {
+        ASSERT(false && "-vm_base [plus rand(-vm_max_offset)] doesn't satisfy "
+               "heap reachability constraints");
+        error_code = HEAP_ERROR_NOT_AT_PREFERRED;
+    } else {
+#endif
+        vmh->start_addr = os_heap_reserve((void*)preferred, size, &error_code);
+        LOG(GLOBAL, LOG_HEAP, 1,
+            "vmm_heap_unit_init preferred="PFX" got start_addr="PFX"\n",
+            preferred, vmh->start_addr);
+#ifdef X64
+    }
+#endif
+    if (vmh->start_addr == NULL && DYNAMO_OPTION(vm_allow_not_at_base)) {
+        SYSLOG_INTERNAL_WARNING("Preferred vmm heap allocation failed");
+        /* need extra size to ensure alignment */
+        vmh->alloc_size = size + VMM_BLOCK_SIZE;
+#ifdef X64
+        /* PR 215395, make sure allocation satisfies heap reachability contraints */
+        vmh->alloc_start = os_heap_reserve_in_region(heap_allowable_region_start,
+                                                     heap_allowable_region_end,
+                                                     size + VMM_BLOCK_SIZE, &error_code);
+#else
+        vmh->alloc_start = (heap_pc)
+            os_heap_reserve(NULL, size + VMM_BLOCK_SIZE, &error_code);
+#endif
+        vmh->start_addr = (heap_pc) ALIGN_FORWARD(vmh->alloc_start, VMM_BLOCK_SIZE);
+        LOG(GLOBAL, LOG_HEAP, 1, "vmm_heap_unit_init unable to allocate at preferred="
+            PFX" letting OS place got start_addr="PFX" \n", preferred, vmh->start_addr);
+    }
+#ifdef X64
+    ASSERT(vmh->start_addr >= heap_allowable_region_start &&
+           vmh->start_addr + size <= heap_allowable_region_end);
+    /* ensure future out-of-block heap allocations are reachable from this allocation */
+    if (vmh->start_addr != NULL)
+        request_region_be_heap_reachable(vmh->start_addr, size);
+#endif
+    if (vmh->start_addr == 0) {
+        vmm_heap_initialize_unusable(vmh);
+        /* we couldn't even reserve initial virtual memory - we're out of luck */
+        /* FIXME: case 7373 make sure we tag as a potential
+         * interoperability issue, in staging mode we should probably
+         * get out from the process since we haven't really started yet
+         */
+        report_low_on_memory(OOM_INIT, error_code);
+    }
+    vmh->end_addr = vmh->start_addr + size;
+    ASSERT_TRUNCATE(vmh->num_blocks, uint, size / VMM_BLOCK_SIZE);
+    vmh->num_blocks = (uint) (size / VMM_BLOCK_SIZE);
+    vmh->num_free_blocks = vmh->num_blocks;
+    LOG(GLOBAL, LOG_HEAP, 2, "vmm_heap_unit_init ["PFX","PFX") total=%d free=%d\n",
+        vmh->start_addr, vmh->end_addr, vmh->num_blocks, vmh->num_free_blocks);
+
+    /* make sure static bitmap_t size is properly aligned on block boundaries */
+    ASSERT(ALIGNED(MAX_VMM_HEAP_UNIT_SIZE, VMM_BLOCK_SIZE));
+    bitmap_initialize_free(vmh->blocks, vmh->num_blocks);
+    DOLOG(1, LOG_HEAP, {
+        vmm_dump_map(vmh);
+    });
+    ASSERT(bitmap_check_consistency(vmh->blocks, vmh->num_blocks, vmh->num_free_blocks));
+}
+
+static
+void
+vmm_heap_unit_exit(vm_heap_t *vmh)
+{
+    LOG(GLOBAL, LOG_HEAP, 1, "vmm_heap_unit_exit ["PFX","PFX") total=%d free=%d\n",
+        vmh->start_addr, vmh->end_addr, vmh->num_blocks, vmh->num_free_blocks);
+    /* we assume single thread in DR at this point */
+    DELETE_LOCK(vmh->lock);
+
+    if (vmh->start_addr == NULL)
+        return;
+
+    DOLOG(1, LOG_HEAP, { vmm_dump_map(vmh); });
+    ASSERT(bitmap_check_consistency(vmh->blocks, 
+                                    vmh->num_blocks, vmh->num_free_blocks));
+    ASSERT(vmh->num_blocks * VMM_BLOCK_SIZE ==
+           (ptr_uint_t)(vmh->end_addr - vmh->start_addr));
+
+    /* In case there are no tombstones we can just free the unit and
+     * that is what we'll do, otherwise it will stay up forever.
+     */
+    if (vmh->num_free_blocks == vmh->num_blocks) {
+        heap_error_code_t error_code;
+        os_heap_free(vmh->alloc_start, vmh->alloc_size, &error_code);
+        ASSERT(error_code == HEAP_ERROR_SUCCESS);
+    } else {
+        /* FIXME: doing nothing for now - we only care about this in
+         * detach scenarios where we should try to clean up from the
+         * virtual address space
+         */
+    }
+    vmm_heap_initialize_unusable(vmh);
+}
+
+/* Returns whether within the region we reserved from the OS for doling
+ * out internally via our vm_heap_t; asserts that the address was also
+ * logically reserved within the vm_heap_t.
+ */
+static
+bool
+vmm_is_reserved_unit(vm_heap_t *vmh, vm_addr_t p, size_t size)
+{
+    size = ALIGN_FORWARD(size, VMM_BLOCK_SIZE);
+    if (p < vmh->start_addr || vmh->end_addr < p/*overflow*/ ||
+        vmh->end_addr < (p + size))
+        return false;
+    ASSERT(CHECK_TRUNCATE_TYPE_uint(size/VMM_BLOCK_SIZE));
+    ASSERT(bitmap_are_reserved_blocks(vmh->blocks, vmh->num_blocks, 
+                                      vmm_addr_to_block(vmh, p),
+                                      (uint)size/VMM_BLOCK_SIZE));
+    return true;
+}
+
+/* Returns whether entirely within the region we reserved from the OS for doling
+ * out internally via our vm_heap_t
+ */
+bool
+is_vmm_reserved_address(byte *pc, size_t size)
+{
+    ASSERT(heapmgt != NULL);
+    /* Case 10293: we don't call vmm_is_reserved_unit to avoid its
+     * assert, which we want to maintain for callers only dealing with
+     * DR-allocated addresses, while this routine is called w/ random
+     * addresses
+     */
+    return (heapmgt != NULL && heapmgt->vmheap.start_addr != NULL &&
+            pc >= heapmgt->vmheap.start_addr &&
+            (pc + size > pc) /*overflow*/ &&
+            (pc + size) <= heapmgt->vmheap.end_addr);
+}
+
+void
+get_vmm_heap_bounds(byte **heap_start/*OUT*/, byte **heap_end/*OUT*/)
+{
+    ASSERT(heapmgt != NULL);
+    ASSERT(heap_start != NULL && heap_end != NULL);
+    *heap_start = heapmgt->vmheap.start_addr;
+    *heap_end = heapmgt->vmheap.end_addr;
+}
+
+bool
+rel32_reachable_from_heap(byte *tgt)
+{
+    byte *heap_start, *heap_end;
+    ptr_int_t new_offs;
+    /* FIXME: also check if we're now allocating memory beyond the vmm heap */
+    get_vmm_heap_bounds(&heap_start, &heap_end);
+    new_offs = (tgt > heap_start) ? (tgt - heap_end) : (heap_start - tgt);
+    return REL32_REACHABLE_OFFS(new_offs);
+}
+
+/* Reservations here are done with VMM_BLOCK_SIZE alignment
+ * (e.g. 64KB) but the caller is not forced to request at that
+ * alignment.  We explicitly synchronize reservations and decommits
+ * within the vm_heap_t.
+
+ * Returns NULL if the VMMHeap is full or too fragmented to satisfy
+ * the request.
+ */
+static vm_addr_t
+vmm_heap_reserve_blocks(vm_heap_t *vmh, size_t size)
+{
+    vm_addr_t p;
+    uint request;
+    uint first_block;
+
+    size = ALIGN_FORWARD(size, VMM_BLOCK_SIZE);
+    ASSERT_TRUNCATE(request, uint, size/VMM_BLOCK_SIZE);
+    request = (uint) size/VMM_BLOCK_SIZE;
+
+    LOG(GLOBAL, LOG_HEAP, 2, "vmm_heap_reserve_blocks: size=%d in blocks=%d free_blocks~=%d\n",
+        size, request, vmh->num_free_blocks);
+
+    mutex_lock(&vmh->lock);
+    if (vmh->num_free_blocks < request) {
+        mutex_unlock(&vmh->lock);
+        return NULL;
+    }
+    first_block = bitmap_allocate_blocks(vmh->blocks, vmh->num_blocks, request);
+    if (first_block != BITMAP_NOT_FOUND) {
+        vmh->num_free_blocks -= request;
+    }
+    mutex_unlock(&vmh->lock);
+
+    if (first_block != BITMAP_NOT_FOUND) {
+        p = vmm_block_to_addr(vmh, first_block);
+        /* FIXME: add a stat vmm_vsize_wasted because of alignment */
+        STATS_ADD_PEAK(vmm_vsize_used, size);
+        STATS_ADD_PEAK(vmm_vsize_blocks_used, request);
+    } else {
+        p = NULL;
+    }
+    LOG(GLOBAL, LOG_HEAP, 2, "vmm_heap_reserve_blocks: size=%d blocks=%d p="PFX"\n",
+        size, request, p);
+    DOLOG(5, LOG_HEAP, { vmm_dump_map(vmh); });
+    return p;
+}
+
+/* We explicitly synchronize reservations and decommits within the vm_heap_t.
+ * Update bookkeeping information about the freed region.
+ */
+static void
+vmm_heap_free_blocks(vm_heap_t *vmh, vm_addr_t p, size_t size)
+{
+    uint first_block = vmm_addr_to_block(vmh, p);
+    uint request;
+
+    size = ALIGN_FORWARD(size, VMM_BLOCK_SIZE);
+    ASSERT_TRUNCATE(request, uint, size/VMM_BLOCK_SIZE);
+    request = (uint) size/VMM_BLOCK_SIZE;
+
+    LOG(GLOBAL, LOG_HEAP, 2, "vmm_heap_free_blocks: size=%d blocks=%d p="PFX"\n",
+        size, request, p);
+
+    mutex_lock(&vmh->lock);
+    bitmap_free_blocks(vmh->blocks, vmh->num_blocks, first_block, request);
+    vmh->num_free_blocks += request;
+    mutex_unlock(&vmh->lock);
+
+    ASSERT(vmh->num_free_blocks <= vmh->num_blocks);
+    STATS_SUB(vmm_vsize_used, size);
+    STATS_SUB(vmm_vsize_blocks_used, request);
+}
+
+/* This is the proper interface for the rest of heap.c to the os_heap_* functions */
+
+
+/* place all the local-scope static vars (from DO_THRESHOLD) into .fspdata to avoid
+ * protection changes */
+START_DATA_SECTION(FREQ_PROTECTED_SECTION, "w");
+
+static bool
+at_reset_at_vmm_limit()
+{
+    return
+        (DYNAMO_OPTION(reset_at_vmm_percent_free_limit) != 0 &&
+         100 * heapmgt->vmheap.num_free_blocks < 
+         DYNAMO_OPTION(reset_at_vmm_percent_free_limit) * heapmgt->vmheap.num_blocks) ||
+        (DYNAMO_OPTION(reset_at_vmm_free_limit) != 0 &&
+         heapmgt->vmheap.num_free_blocks * VMM_BLOCK_SIZE <
+         DYNAMO_OPTION(reset_at_vmm_free_limit));
+}
+
+/* Reserve virtual address space without committing swap space for it */
+static vm_addr_t
+vmm_heap_reserve(size_t size, heap_error_code_t *error_code)
+{
+    vm_addr_t p;
+    /* should only be used on sizable aligned pieces */
+    ASSERT(size > 0 && ALIGNED(size, PAGE_SIZE));
+    ASSERT(!OWN_MUTEX(&reset_pending_lock));
+
+    if (DYNAMO_OPTION(vm_reserve)) {
+        /* FIXME: should we make this an external option? */
+        if (INTERNAL_OPTION(vm_use_last) ||
+            (DYNAMO_OPTION(switch_to_os_at_vmm_reset_limit) && at_reset_at_vmm_limit())) {
+            DO_ONCE({
+                if (DYNAMO_OPTION(reset_at_switch_to_os_at_vmm_limit))
+                    schedule_reset(RESET_ALL);
+                DODEBUG({
+                    if (!INTERNAL_OPTION(vm_use_last)) {
+                        ASSERT_CURIOSITY(false && "running low on vm reserve");
+                    }
+                });
+                /* FIXME - for our testing would be nice to have some release build
+                 * notification of this ... */
+            });     
+#ifdef X64
+            /* PR 215395, make sure allocation satisfies heap reachability contraints */
+            p = os_heap_reserve_in_region(heap_allowable_region_start,
+                                          heap_allowable_region_end,
+                                          size, error_code);
+            /* ensure future heap allocations are reachable from this allocation */
+            if (p != NULL)
+                request_region_be_heap_reachable(p, size);
+#else
+            p = os_heap_reserve(NULL, size, error_code);
+#endif
+            if (p != NULL)
+                return p;
+            LOG(GLOBAL, LOG_HEAP, 1, "vmm_heap_reserve: failed "PFX"\n", 
+                *error_code);
+        }
+
+        if (at_reset_at_vmm_limit()) {
+            /* We're running low on our reservation, trigger a reset */
+            if (schedule_reset(RESET_ALL)) {
+                STATS_INC(reset_low_vmm_count);
+                DO_THRESHOLD_SAFE(DYNAMO_OPTION(report_reset_vmm_threshold),
+                                  FREQ_PROTECTED_SECTION,
+                {/* < max - nothing */},
+                {/* >= max */
+                    /* FIXME - do we want to report more then once to give some idea of 
+                     * how much thrashing there is? */
+                    DO_ONCE({
+                        SYSLOG_CUSTOM_NOTIFY(SYSLOG_WARNING, MSG_LOW_ON_VMM_MEMORY, 2,
+                                             "Potentially thrashing on low virtual "
+                                             "memory resetting.", get_application_name(),
+                                             get_application_pid());
+                        /* want QA to notice */
+                        ASSERT_CURIOSITY(false && "vmm heap limit reset thrashing");
+                    });
+                });
+            }
+        }
+
+        p = vmm_heap_reserve_blocks(&heapmgt->vmheap, size);
+        LOG(GLOBAL, LOG_HEAP, 2, "vmm_heap_reserve: size=%d p="PFX"\n", 
+            size, p);
+
+        if (p)
+            return p;
+        DO_ONCE({
+            DODEBUG({ out_of_vmheap_once = true; });
+            if (!INTERNAL_OPTION(skip_out_of_vm_reserve_curiosity)) {
+                /* this maybe unsafe for early services w.r.t. case 666 */
+                SYSLOG_INTERNAL_WARNING("Out of vmheap reservation - reserving %dKB."
+                                        "Falling back onto OS allocation", size/1024);
+                ASSERT_CURIOSITY(false && "Out of vmheap reservation");
+            }
+            /* This actually-out trigger is only trying to help issues like a
+             * thread-private configuration being a memory hog (and thus we use up
+             * our reserve). Reset needs memory, and this is asynchronous, so no
+             * guarantees here anyway (the app may have already reserved all memory
+             * beyond our reservation, see sqlsrvr.exe and cisvc.exe for ex.) which is
+             * why we have -reset_at_vmm_threshold to make reset success more likely. */
+            if (DYNAMO_OPTION(reset_at_vmm_full)) {
+                schedule_reset(RESET_ALL);
+            }
+        });
+    }
+    /* if we fail to allocate from our reservation we fall back to the OS */
+#ifdef X64
+    /* PR 215395, make sure allocation satisfies heap reachability contraints */
+    p = os_heap_reserve_in_region(heap_allowable_region_start, heap_allowable_region_end,
+                                  size, error_code);
+    /* ensure future heap allocations are reachable from this allocation */
+    if (p != NULL)
+        request_region_be_heap_reachable(p, size);
+#else
+    p = os_heap_reserve(NULL, size, error_code);
+#endif
+    return p;
+}
+
+/* Commit previously reserved pages, returns false when out of memory
+ * This is here just to complement the vmm interface, in fact it is
+ * almost an alias for os_heap_commit.  (If we had strict types then
+ * here we'd convert a vm_addr_t into a heap_pc.)
+ */
+static inline bool 
+vmm_heap_commit(vm_addr_t p, size_t size, uint prot, heap_error_code_t *error_code)
+{
+    bool res =  os_heap_commit(p, size, prot, error_code);
+    size_t commit_used, commit_limit;
+    ASSERT(!OWN_MUTEX(&reset_pending_lock));
+    if ((DYNAMO_OPTION(reset_at_commit_percent_free_limit) != 0 ||
+         DYNAMO_OPTION(reset_at_commit_free_limit) != 0) &&
+        os_heap_get_commit_limit(&commit_used, &commit_limit)) {
+        size_t commit_left = commit_limit - commit_used;
+        ASSERT(commit_used <= commit_limit);
+        /* FIXME - worry about overflow in the multiplies below? With 4kb pages isn't
+         * an issue till 160GB of committable memory. */
+        if ((DYNAMO_OPTION(reset_at_commit_free_limit) != 0 &&
+             commit_left < DYNAMO_OPTION(reset_at_commit_free_limit) / PAGE_SIZE) ||
+            (DYNAMO_OPTION(reset_at_commit_percent_free_limit) != 0 && 
+             100 * commit_left <
+             DYNAMO_OPTION(reset_at_commit_percent_free_limit) * commit_limit)) {
+            /* Machine is getting low on memory, trigger a reset */
+            /* FIXME - if we aren't the ones hogging committed memory (rougue app) then
+             * do we want a version of reset that doesn't de-commit our already grabbed 
+             * memory to avoid someone else stealing it (or perhaps keep just a minimal
+             * level to ensure we make some progress)? */
+            /* FIXME - the commit limit is for the whole system; we have no good way of
+             * telling if we're running in a job and if so what the commit limit for the
+             * job is. */
+            /* FIXME - if a new process is started under dr while the machine is already
+             * past the threshold we will just spin resetting here and not make any
+             * progress, may be better to only reset when we have a reasonable amount of
+             * non-persistent memory to free (so that we can at least make some progress
+             * before resetting again). */
+            /* FIXME - the threshold is calculated at the current page file size, but
+             * it's possible that the pagefile is expandable (dependent on disk space of
+             * course) and thus we're preventing a potentially beneficial (to us)
+             * upsizing of the pagefile here.  See "HKLM\SYSTEM\CCS\ControlSession /
+             * Manager\Memory Management" for the initial/max size of the various page
+             * files (query SystemPafefileInformation only gets you the current size). */
+            /* xref case 345 on fixmes (and link to wiki discussion) */
+            if (schedule_reset(RESET_ALL)) {
+                STATS_INC(reset_low_commit_count);
+                DO_THRESHOLD_SAFE(DYNAMO_OPTION(report_reset_commit_threshold),
+                                  FREQ_PROTECTED_SECTION,
+                {/* < max - nothing */},
+                {/* >= max */
+                    /* FIXME - do we want to report more then once to give some idea of 
+                     * how much thrashing there is? */
+                    DO_ONCE({
+                        SYSLOG_CUSTOM_NOTIFY(SYSLOG_WARNING,
+                                             MSG_LOW_ON_COMMITTABLE_MEMORY, 2,
+                                             "Potentially thrashing on low committable "
+                                             "memory resetting.", get_application_name(),
+                                             get_application_pid());
+                        /* want QA to notice */
+                        ASSERT_CURIOSITY(false && "commit limit reset thrashing");
+                    });
+                });
+            }
+        }
+    }
+    if (!res && 
+        DYNAMO_OPTION(oom_timeout) != 0) {
+        DEBUG_DECLARE(heap_error_code_t old_error_code = *error_code;)
+        ASSERT(old_error_code != HEAP_ERROR_SUCCESS);
+
+        /* check whether worth retrying */
+        if (!os_heap_systemwide_overcommit(*error_code)) {
+            /* FIXME: we should check whether current process is the hog */
+            /* unless we have used the memory, there is still a
+             * miniscule chance another thread will free up some or
+             * will attempt suicide, so could retry even if current
+             * process has a leak */
+            ASSERT_NOT_IMPLEMENTED(false);
+            /* retry */
+        }
+
+        SYSLOG_INTERNAL_WARNING("vmm_heap_commit oom: timeout and retry");
+        /* let's hope a memory hog dies in the mean time */
+        os_timeout(DYNAMO_OPTION(oom_timeout));
+
+        res = os_heap_commit(p, size, prot, error_code);
+        DODEBUG({
+            if (res) {
+                SYSLOG_INTERNAL_WARNING("vmm_heap_commit retried, got away!  old="PFX" new="PFX"\n", 
+                                        old_error_code, *error_code);
+            } else {
+                SYSLOG_INTERNAL_WARNING("vmm_heap_commit retrying, no luck.  old="PFX" new="PFX"\n", 
+                                        old_error_code, *error_code);
+            }
+        });
+    }
+
+    return res;
+}
+
+/* back to normal section */
+END_DATA_SECTION()
+
+/* Free previously reserved and possibly committed memory.  Check if
+ * it is within the memory managed by the virtual memory manager we
+ * only decommit back to the OS, and we remove the vmm reservation.
+ * Keep in mind that this can be called on units that are not fully
+ * committed, e.g. guard pages are added to this - as long as the
+ * os_heap_decommit interface can handle this we're OK
+ */
+static void
+vmm_heap_free(vm_addr_t p, size_t size, heap_error_code_t *error_code)
+{
+    LOG(GLOBAL, LOG_HEAP, 2, "vmm_heap_free: size=%d p="PFX" is_reserved=%d\n", 
+        size, p, vmm_is_reserved_unit(&heapmgt->vmheap, p, size));
+
+    /* the memory doesn't have to be within our VM reserve if it
+       was allocated as an extra OS call when if we ran out */
+    if (DYNAMO_OPTION(vm_reserve)) {
+        if (vmm_is_reserved_unit(&heapmgt->vmheap, p, size)) {
+            os_heap_decommit(p, size, error_code);
+            vmm_heap_free_blocks(&heapmgt->vmheap, p, size);
+            LOG(GLOBAL, LOG_HEAP, 2, "vmm_heap_free: freed size=%d p="PFX"\n", 
+                size, p);
+            return;
+        } else {
+            /* FIXME: check if this is stack_free getting in the way, then ignore it */
+            /* FIXME: could do this by overriding the meaning of the vmheap fields
+               after cleanup to a different combination that start_pc = end_pc = NULL
+             */
+            /* FIXME: see vmm_heap_unit_exit for the current stack_free problem */
+            if (vmm_heap_exited) {
+                *error_code = HEAP_ERROR_SUCCESS;
+                return;
+            }
+        }
+    }
+    os_heap_free(p, size, error_code);
+}
+
+static void
+vmm_heap_decommit(vm_addr_t p, size_t size, heap_error_code_t *error_code)
+{
+    LOG(GLOBAL, LOG_HEAP, 2, "vmm_heap_decommit: size=%d p="PFX" is_reserved=%d\n", 
+        size, p, vmm_is_reserved_unit(&heapmgt->vmheap, p, size));
+    os_heap_decommit(p, size, error_code);
+    /* nothing to be done to vmm blocks */
+}
+
+/* Caller is required to handle thread synchronization and to update dynamo vm areas.
+ * size must be PAGE_SIZE-aligned.
+ * Returns NULL if fails to allocate memory!
+ */
+static void *
+vmm_heap_alloc(size_t size, uint prot, heap_error_code_t *error_code)
+{
+    vm_addr_t p = vmm_heap_reserve(size, error_code);
+    if (!p)
+        return NULL;               /* out of reserved memory */
+
+    if (!vmm_heap_commit(p, size, prot, error_code))
+        return NULL;               /* out of committed memory */
+    return p;
+}
+
+/* virtual memory manager initialization */
+void
+vmm_heap_init()
+{
+#ifdef X64
+    /* add reachable regions before we allocate the heap, xref PR 215395 */
+# ifdef WINDOWS
+    request_region_be_heap_reachable(get_ntdll_base(),
+                                     get_allocation_size(get_ntdll_base(), NULL));
+    request_region_be_heap_reachable(get_dynamorio_dll_start(),
+                                     get_allocation_size(get_dynamorio_dll_start(), NULL));
+# else /* LINUX */
+    /* FIXME - On Linux we compile core with -fpic and samples Makefile uses it as
+     * well (comments suggest problems if we don't).  But without our own loader
+     * that means we can't control where the libraries are loaded.  For now we count
+     * on a good choice of vm_base.  See PR 253624. */
+    request_region_be_heap_reachable(get_dynamorio_dll_start(),
+                                     get_dynamorio_dll_end() - get_dynamorio_dll_start());
+# endif
+#endif /* X64 */
+    
+    IF_WINDOWS(ASSERT(VMM_BLOCK_SIZE == OS_ALLOC_GRANULARITY));
+    if (DYNAMO_OPTION(vm_reserve)) {
+        vmm_heap_unit_init(&heapmgt->vmheap, DYNAMO_OPTION(vm_size));
+    }
+}
+
+void
+vmm_heap_exit()
+{
+    /* virtual memory manager exit */
+    if (DYNAMO_OPTION(vm_reserve)) {
+        /* FIXME: we have three regions that are not explicitly
+         * deallocated current stack, init stack, global_do_syscall
+         */
+        DEBUG_DECLARE(uint perstack =
+                      ALIGN_FORWARD_UINT(dynamo_options.stack_size +
+                                         (dynamo_options.guard_pages ? (2*PAGE_SIZE) : 0),
+                                         VMM_BLOCK_SIZE) /
+                      VMM_BLOCK_SIZE;)
+        /* FIXME: on detach arch_thread_exit should explicitly mark as
+           left behind all TPCs needed so then we can assert even for
+           detach
+         */
+            ASSERT(IF_WINDOWS(doing_detach || )  /* not deterministic when detaching */
+               heapmgt->vmheap.num_free_blocks == heapmgt->vmheap.num_blocks 
+               - perstack * 1 /* initstack */
+               /* current stack */
+                   - perstack * ((IF_WINDOWS_ELSE(doing_detach, false)
+                                  IF_APP_EXPORTS(|| dr_api_exit)) ? 0 : 1));
+
+        /* FIXME: On process exit we are currently executing off a
+         *  stack in this region so we cannot free the whole allocation.
+
+         * FIXME: Any tombstone allocations will have to use a
+         * different interface than the generic heap_mmap() which is
+         * sometimes used to leave things behind.  FIXME: Currently
+         * we'll leave behind the whole vm unit if any tombstones are
+         * left - which in fact is always the case, no matter whether
+         * thread private code needs to be left or not.
+         
+         * global_do_syscall 32 byte allocation should be part of our
+         * dll and won't have to be left.
+
+         * The current stack is the main problem because it is later
+         * cleaned up in cleanup_and_terminate by calling stack_free which
+         * in turn gets all the way to vmm_heap_free.  Therefore we add an
+         * explicit test for vmm_heap_exited, so that we can otherwise free
+         * bookkeeping information and delete the lock now.
+       
+         * Potential solution to most of these problems is to have
+         * cleanup_and_terminate call vmm_heap_exit when cleaning up
+         * the process, or to just leave the vm mapping behind and
+         * simply pass a different argument to stack_free.
+         */
+        vmm_heap_unit_exit(&heapmgt->vmheap);
+
+        vmm_heap_exited = true;
+    }
+}
+
+/* checks for compatibility among heap options, returns true if
+ * modified the value of any options to make them compatible 
+ */
+bool
+heap_check_option_compatibility()
+{
+    bool ret = false;
+
+    ret = check_param_bounds(&dynamo_options.vm_size, MIN_VMM_HEAP_UNIT_SIZE,
+                             MAX_VMM_HEAP_UNIT_SIZE, "vm_size")
+        || ret;
+#ifdef INTERNAL
+    /* if max_heap_unit_size is too small you may get a funny message
+     * "initial_heap_unit_size must be >= 8229 and <= 4096" but in
+     * release build we will take the min and then complain about
+     * max_heap_unit_size and set it to the min also, so it all works
+     * out w/o needing an extra check() call.
+     */
+    /* case 7626: don't short-circuit checks, as later ones may be needed */
+    ret = check_param_bounds(&dynamo_options.initial_heap_unit_size,
+                             /* if have units smaller than a page we end up
+                              * allocating 64KB chunks for "oversized" units
+                              * for just about every alloc!  so round up to
+                              * at least a page.
+                              */
+                             ALIGN_FORWARD(UNITOVERHEAD + 1, PAGE_SIZE),
+                             HEAP_UNIT_MAX_SIZE, "initial_heap_unit_size")
+        || ret;
+    ret = check_param_bounds(&dynamo_options.initial_global_heap_unit_size,
+                             ALIGN_FORWARD(UNITOVERHEAD + 1, PAGE_SIZE),
+                             HEAP_UNIT_MAX_SIZE, "initial_global_heap_unit_size")
+        || ret;
+    ret = check_param_bounds(&dynamo_options.max_heap_unit_size,
+                             MAX(HEAP_UNIT_MIN_SIZE, GLOBAL_UNIT_MIN_SIZE),
+                             INT_MAX, "max_heap_unit_size")
+        || ret;
+#endif
+    return ret;
+}
+
+/* thread-shared initialization that should be repeated after a reset */
+void
+heap_reset_init()
+{
+    if (DYNAMO_OPTION(enable_reset)) {
+        threadunits_init(GLOBAL_DCONTEXT, &heapmgt->global_nonpersistent_units,
+                         GLOBAL_UNIT_MIN_SIZE);
+    }
+}
+
+/* initialization */
+void
+heap_init()
+{
+    int i;
+    uint prev_sz = 0;
+
+    LOG(GLOBAL, LOG_TOP|LOG_HEAP, 2, "Heap bucket sizes are:\n");
+    /* make sure we'll preserve alignment */
+    ASSERT(ALIGNED(HEADER_SIZE, HEAP_ALIGNMENT));
+    /* make sure free list pointers will fit */
+    ASSERT(BLOCK_SIZES[0] >= sizeof(heap_pc*));
+    /* since sizes depend on size of structs, make sure they're in order */
+    for (i = 0; i < BLOCK_TYPES; i++) {
+        ASSERT(BLOCK_SIZES[i] > prev_sz);
+        /* we assume all of our heap allocs are aligned */
+        ASSERT(i == BLOCK_TYPES-1 || ALIGNED(BLOCK_SIZES[i], HEAP_ALIGNMENT));
+        prev_sz = BLOCK_SIZES[i];
+        LOG(GLOBAL, LOG_TOP|LOG_HEAP, 2, "\t%d bytes\n", BLOCK_SIZES[i]);
+    }
+
+    /* we assume writes to some static vars are atomic,
+     * i.e., the vars don't cross cache lines.  they shouldn't since
+     * they should all be 4-byte-aligned in the data segment.
+     * FIXME: ensure that release build aligns ok?
+     * I would be quite surprised if static vars were not 4-byte-aligned!
+     */
+    ASSERT(ALIGN_BACKWARD(&heap_exiting, CACHE_LINE_SIZE()) ==
+           ALIGN_BACKWARD(&heap_exiting + 1, CACHE_LINE_SIZE()));
+    ASSERT(ALIGN_BACKWARD(&heap_unit_lock.owner, CACHE_LINE_SIZE()) ==
+           ALIGN_BACKWARD(&heap_unit_lock.owner + 1, CACHE_LINE_SIZE()));
+
+    /* For simplicity we go through our normal heap mechanism to allocate
+     * our post-init heapmgt struct
+     */
+    ASSERT(heapmgt == &temp_heapmgt);
+    heapmgt->global_heap_writable = true;
+    threadunits_init(GLOBAL_DCONTEXT, &heapmgt->global_units, GLOBAL_UNIT_MIN_SIZE);
+
+    heapmgt = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, heap_management_t, ACCT_MEM_MGT, PROTECTED);
+    memset(heapmgt, 0, sizeof(*heapmgt));
+    ASSERT(sizeof(temp_heapmgt) == sizeof(*heapmgt));
+    memcpy(heapmgt, &temp_heapmgt, sizeof(temp_heapmgt));
+
+    threadunits_init(GLOBAL_DCONTEXT, &heapmgt->global_unprotected_units,
+                     GLOBAL_UNIT_MIN_SIZE);
+    heap_reset_init();
+
+#ifdef WINDOWS
+    /* PR 250294: As part of 64-bit hook work, hook reachability was addressed
+     * using landing pads (see win32/callback.c for more explanation).  Landing
+     * pad areas are a type of special heap, so they should be initialized
+     * during heap init.
+     * Each landing pad area has its own allocation pointer, so they shouldn't
+     * be merged automatically. 
+     */
+    VMVECTOR_ALLOC_VECTOR(landing_pad_areas, GLOBAL_DCONTEXT,
+                          VECTOR_SHARED | VECTOR_NEVER_MERGE,
+                          landing_pad_areas_lock);
+#endif
+}
+
+/* need to not remove from vmareas on process exit -- vmareas has already exited! */
+static void
+really_free_unit(heap_unit_t *u)
+{
+    STATS_SUB(heap_capacity, UNIT_COMMIT_SIZE(u));
+    STATS_ADD(heap_reserved_only, (stats_int_t)(UNIT_COMMIT_SIZE(u) - UNIT_RESERVED_SIZE(u)));
+    /* remember that u itself is inside unit, not separately allocated */
+    release_guarded_real_memory((vm_addr_t)u, UNIT_RESERVED_SIZE(u),
+                                false/*do not update DR areas now*/, true);
+}
+
+/* Free all thread-shared state not critical to forward progress;
+ * heap_reset_init() will be called before continuing.
+ */
+void
+heap_reset_free()
+{
+    heap_unit_t *u, *next_u;
+    /* FIXME: share some code w/ heap_exit -- currently only called by reset */
+    ASSERT(DYNAMO_OPTION(enable_reset));
+
+    /* we must grab this lock before heap_unit_lock to avoid rank 
+     * order violations when freeing
+     */
+    dynamo_vm_areas_lock();
+
+    /* for combining stats into global_units we need this lock
+     * FIXME: remove if we go to separate stats sum location
+     */
+    DODEBUG({ acquire_recursive_lock(&global_alloc_lock); });
+
+    acquire_recursive_lock(&heap_unit_lock);
+
+    LOG(GLOBAL, LOG_HEAP, 1, "Pre-reset, global heap unit stats:\n");
+    /* FIXME: free directly rather than putting on dead list first */
+    threadunits_exit(&heapmgt->global_nonpersistent_units, GLOBAL_DCONTEXT);
+
+    /* free all dead units */
+    u = heapmgt->heap.dead;
+    while (u != NULL) {
+        next_u = u->next_global;
+        LOG(GLOBAL, LOG_HEAP, 1, "\tfreeing dead unit "PFX"-"PFX" [-"PFX"]\n",
+            u, UNIT_COMMIT_END(u), UNIT_RESERVED_END(u));
+        RSTATS_DEC(heap_num_free);
+        really_free_unit(u);
+        u = next_u;
+    }
+    heapmgt->heap.dead = NULL;
+    heapmgt->heap.num_dead = 0;
+    release_recursive_lock(&heap_unit_lock);
+    DODEBUG({ release_recursive_lock(&global_alloc_lock); });
+    dynamo_vm_areas_unlock();
+}
+
+/* atexit cleanup */
+void
+heap_exit()
+{
+    heap_unit_t *u, *next_u;
+    heap_management_t *temp;
+
+    heap_exiting = true;
+    /* FIXME: we shouldn't need either lock if executed last */
+    dynamo_vm_areas_lock();
+    acquire_recursive_lock(&heap_unit_lock);
+
+#ifdef WINDOWS
+    release_landing_pad_mem();  /* PR 250294 */
+#endif
+
+    LOG(GLOBAL, LOG_HEAP, 1, "Global unprotected heap unit stats:\n");
+    threadunits_exit(&heapmgt->global_unprotected_units, GLOBAL_DCONTEXT);
+    if (DYNAMO_OPTION(enable_reset)) {
+        LOG(GLOBAL, LOG_HEAP, 1, "Global nonpersistent heap unit stats:\n");
+        threadunits_exit(&heapmgt->global_nonpersistent_units, GLOBAL_DCONTEXT);
+    }
+
+    /* Now we need to go back to the static struct to clean up */
+    ASSERT(heapmgt != &temp_heapmgt);
+    memcpy(&temp_heapmgt, heapmgt, sizeof(temp_heapmgt));
+    temp = heapmgt;
+    heapmgt = &temp_heapmgt;
+    HEAP_TYPE_FREE(GLOBAL_DCONTEXT, temp, heap_management_t, ACCT_MEM_MGT, PROTECTED);
+
+    LOG(GLOBAL, LOG_HEAP, 1, "Global heap unit stats:\n");
+    threadunits_exit(&heapmgt->global_units, GLOBAL_DCONTEXT);
+
+    /* free heap for all unfreed units */
+    LOG(GLOBAL, LOG_HEAP, 1, "Unfreed units:\n");
+    u = heapmgt->heap.units;
+    while (u != NULL) {
+        next_u = u->next_global;
+        LOG(GLOBAL, LOG_HEAP, 1, "\tfreeing live unit "PFX"-"PFX" [-"PFX"]\n",
+            u, UNIT_COMMIT_END(u), UNIT_RESERVED_END(u));
+        RSTATS_DEC(heap_num_live);
+        really_free_unit(u);
+        u = next_u;
+    }
+    heapmgt->heap.units = NULL;
+    u = heapmgt->heap.dead;
+    while (u != NULL) {
+        next_u = u->next_global;
+        LOG(GLOBAL, LOG_HEAP, 1, "\tfreeing dead unit "PFX"-"PFX" [-"PFX"]\n",
+            u, UNIT_COMMIT_END(u), UNIT_RESERVED_END(u));
+        RSTATS_DEC(heap_num_free);
+        really_free_unit(u);
+        u = next_u;
+    }
+    heapmgt->heap.dead = NULL;
+    release_recursive_lock(&heap_unit_lock);
+    dynamo_vm_areas_unlock();
+
+    DELETE_RECURSIVE_LOCK(heap_unit_lock);
+    DELETE_RECURSIVE_LOCK(global_alloc_lock);
+#ifdef X64
+    DELETE_LOCK(request_region_be_heap_reachable_lock);
+#endif
+}
+
+/* FIXME:
+ * detect if the app is who we're fighting for memory, if so, don't
+ * free memory, else the app will just keep grabbing more.
+ * need a test for hitting 2GB (or 3GB!) user mode limit.
+ */
+static void
+heap_low_on_memory()
+{
+    /* free some memory! */
+    heap_unit_t *u, *next_u;
+    size_t freed = 0;
+    LOG(GLOBAL, LOG_CACHE|LOG_STATS, 1,
+        "heap_low_on_memory: about to free dead list units\n");
+    /* WARNING: this routine is called at arbitrary allocation failure points,
+     * so we have to be careful what locks we grab
+     * However, no allocation site can hold a lock weaker in rank than
+     * heap_unit_lock, b/c it could deadlock on the allocation itself!
+     * So we're safe.
+     */
+    /* must grab this lock prior to heap_unit_lock if affecting DR vm areas 
+     * this is recursive so ok if we ran out of memory while holding DR vm area lock
+     */
+    ASSERT(safe_to_allocate_or_free_heap_units());
+    dynamo_vm_areas_lock();
+    acquire_recursive_lock(&heap_unit_lock);
+    u = heapmgt->heap.dead;
+    while (u != NULL) {
+        next_u = u->next_global;
+        freed += UNIT_COMMIT_SIZE(u);
+        /* FIXME: if out of committed pages only, could keep our reservations */
+        LOG(GLOBAL, LOG_HEAP, 1, "\tfreeing dead unit "PFX"-"PFX" [-"PFX"]\n",
+            u, UNIT_COMMIT_END(u), UNIT_RESERVED_END(u));
+        RSTATS_DEC(heap_num_free);
+        really_free_unit(u);
+        u = next_u;
+        heapmgt->heap.num_dead--;
+    }
+    heapmgt->heap.dead = NULL;
+    release_recursive_lock(&heap_unit_lock);
+    dynamo_vm_areas_unlock();
+    LOG(GLOBAL, LOG_CACHE|LOG_STATS, 1,
+        "heap_low_on_memory: freed %d KB\n", freed/1024);
+    /* FIXME: we don't keep a list of guard pages, which we may decide to throw
+     * out or compact at this time. 
+     */
+    /* FIXME: should also fix up the allocator to look in other free lists
+     * of sizes larger than asked for, we may have plenty of memory available
+     * in other lists!  see comments in common_heap_alloc
+     */
+}
+
+static char*
+get_oom_source_name(oom_source_t source)
+{
+    /* currently only single character codenames, 
+     * (still as a string though) 
+     */
+    char *code_name = "?";
+
+    switch (source) {
+    case OOM_INIT     : code_name = "I"; break;
+    case OOM_RESERVE  : code_name = "R"; break;
+    case OOM_COMMIT   : code_name = "C"; break;
+    case OOM_EXTEND   : code_name = "E"; break;
+    default:
+        ASSERT_NOT_REACHED();
+    }
+    return code_name;
+}
+
+static bool
+silent_oom_for_process(oom_source_t source)
+{
+    if (TESTANY(OOM_COMMIT|OOM_EXTEND, source) && 
+        !IS_STRING_OPTION_EMPTY(silent_commit_oom_list)) {
+        bool onlist;
+        const char *process_name = get_short_name(get_application_name());
+        string_option_read_lock();
+        onlist = check_filter_with_wildcards(DYNAMO_OPTION(silent_commit_oom_list), 
+                                             process_name);
+        string_option_read_unlock();
+
+        if (onlist) {
+            SYSLOG_INTERNAL_WARNING("not reporting last words of executable %s", 
+                                    process_name);
+            return true;
+        }
+    }
+    return false;
+}
+
+/* oom_source_t identifies the action we were taking, os_error_code is
+ * the returned value from the last system call - opaque at this OS
+ * independent layer.
+ */
+static void 
+report_low_on_memory(oom_source_t source, heap_error_code_t os_error_code)
+{
+    if (TESTANY(DYNAMO_OPTION(silent_oom_mask), source)
+        || silent_oom_for_process(source)) {
+        SYSLOG_INTERNAL_WARNING("Mostly silent OOM: %s "PFX".\n",
+                                get_oom_source_name(source), os_error_code);
+        /* still produce an ldmp for internal use */
+        if (TEST(DUMPCORE_OUT_OF_MEM_SILENT, DYNAMO_OPTION(dumpcore_mask)))
+            os_dump_core("Out of memory, silently aborting program.");
+    } else {
+        char *oom_source_code = get_oom_source_name(source);
+        char status_hex[19]; /* FIXME: for 64bit hex need 16+NULL */
+        /* note 0x prefix added by the syslog */
+        snprintf(status_hex, 
+                 BUFFER_SIZE_ELEMENTS(status_hex), PFX, /* FIXME: 32bit */
+                 os_error_code);
+        NULL_TERMINATE_BUFFER(status_hex);
+        /* SYSLOG first */
+        SYSLOG_CUSTOM_NOTIFY(SYSLOG_CRITICAL, MSG_OUT_OF_MEMORY, 4,
+                             "Out of memory.  Program aborted.",
+                             get_application_name(), get_application_pid(),
+                             oom_source_code, status_hex
+                             );
+
+        /* FIXME: case 7306 can't specify arguments in SYSLOG_CUSTOM_NOTIFY */
+        SYSLOG_INTERNAL_WARNING("OOM Status: %s %s\n", oom_source_code, status_hex); 
+
+        /* FIXME: case 7296 - ldmp even if we have decided not to produce an event above */
+        if (TEST(DUMPCORE_OUT_OF_MEM, DYNAMO_OPTION(dumpcore_mask)))
+            os_dump_core("Out of memory, aborting program.");
+
+        /* passing only status code to XML where we should have a stack dump and callstack */
+        report_diagnostics("Out of memory", status_hex, NO_VIOLATION_BAD_INTERNAL_STATE);
+    } 
+    os_terminate(NULL, TERMINATE_PROCESS);
+    ASSERT_NOT_REACHED();
+}
+
+/* update statistics for committed memory, and add to vm_areas */
+static inline void
+account_for_memory(void *p, size_t size, uint prot, bool add_vm _IF_DEBUG(char *comment))
+{
+    STATS_ADD_PEAK(memory_capacity, size);
+
+    /* case 3045: areas inside the vmheap reservation are not added to the list */
+    if (vmm_is_reserved_unit(&heapmgt->vmheap, p, size))
+        return;
+
+    if (add_vm) {
+        add_dynamo_vm_area(p, ((app_pc)p) + size, prot, false _IF_DEBUG(comment));
+    } else {
+        /* due to circular dependencies bet vmareas and global heap we do not call
+         * add_dynamo_vm_area here, instead we indicate that something has changed
+         */
+        mark_dynamo_vm_areas_stale();
+        /* NOTE: 'prot' info is lost about this region, but is needed in
+         * heap_vmareas_synch_units to update all_memory_areas.  Currently
+         * heap_create_unit is the only place that passes 'false' with prot rw-.
+         */
+        ASSERT(TESTALL(MEMPROT_READ | MEMPROT_WRITE, prot));
+    }
+}
+
+/* remove_vm MUST be false iff this is heap memory, which is updated separately */
+static void
+update_dynamo_areas_on_release(app_pc start, app_pc end, bool remove_vm)
+{
+    if (!vm_areas_exited && !heap_exiting) { /* avoid problems when exiting */
+        /* case 3045: areas inside the vmheap reservation are not added to the list */
+        if (vmm_is_reserved_unit(&heapmgt->vmheap, start, end - start))
+            return;
+        if (remove_vm) {
+            remove_dynamo_vm_area(start, end);
+        } else {
+            /* due to cyclic dependencies bet heap and vmareas we cannot remove incrementally.
+             * the pending set is protected by the same lock needed to synch the vm areas,
+             * so we will never mis-identify free memory as DR memory.
+             */
+            mark_dynamo_vm_areas_stale();
+            dynamo_areas_pending_remove = true;
+        }
+    }
+}
+
+bool
+lockwise_safe_to_allocate_memory()
+{
+    /* check whether it's safe to hold a lock that normally can be held
+     * for memory allocation -- i.e., check whether we hold the
+     * global_alloc_lock
+     */
+    return !self_owns_recursive_lock(&global_alloc_lock);
+}
+
+/* we indirect all os memory requests through here so we have a central place
+ * to handle the out-of-memory condition.
+ * add_vm MUST be false iff this is heap memory, which is updated separately.
+ */
+static void *
+get_real_memory(size_t size, uint prot, bool add_vm _IF_DEBUG(char *comment))
+{
+    void *p;
+    heap_error_code_t error_code;
+    /* must round up to page sizes, else vmm_heap_alloc assert triggers */
+    size = ALIGN_FORWARD(size, PAGE_SIZE);
+
+    /* memory alloc/dealloc and updating DR list must be atomic */
+    dynamo_vm_areas_lock(); /* if already hold lock this is a nop */
+
+    p = vmm_heap_alloc(size, prot, &error_code);
+    if (p == NULL) {
+        SYSLOG_INTERNAL_WARNING_ONCE("Out of memory -- cannot reserve or "
+                                     "commit %dKB.  Trying to recover.", size/1024);
+        /* we should be ok here, shouldn't come in here holding global_alloc_lock
+         * or heap_unit_lock w/o first having grabbed DR areas lock
+         */
+        ASSERT(safe_to_allocate_or_free_heap_units());
+        heap_low_on_memory();
+        fcache_low_on_memory();
+        /* try again
+         * FIXME: have more sophisticated strategy of freeing a little, then getting
+         * more drastic with each subsequent failure
+         * FIXME: can only free live fcache units for current thread w/ current
+         * impl...should we wait a while and try again if out of memory, hoping
+         * other threads have freed some?!?!
+         */
+        p = vmm_heap_alloc(size, prot, &error_code);
+        if (p == NULL) {
+            report_low_on_memory(OOM_RESERVE, error_code);
+        }
+        SYSLOG_INTERNAL_WARNING_ONCE("Out of memory -- but still alive after "
+                                     "emergency free.");
+    }
+
+    account_for_memory(p, size, prot, add_vm _IF_DEBUG(comment));
+    dynamo_vm_areas_unlock();
+
+    return p;
+}
+
+static void
+release_memory_and_update_areas(app_pc p, size_t size, bool decommit, bool remove_vm)
+{
+    heap_error_code_t error_code;
+    /* these two operations need to be atomic wrt DR area updates */
+    dynamo_vm_areas_lock(); /* if already hold lock this is a nop */
+    /* ref case 3035, we must remove from dynamo_areas before we free in case
+     * we end up allocating memory in the process of removing the area 
+     * (we don't want to end up getting the memory we just freed since that
+     *  would lead to errors in the list when we finally did remove it)
+     */
+    update_dynamo_areas_on_release(p, p + size, remove_vm);
+    if (decommit)
+        vmm_heap_decommit(p, size, &error_code);
+    else
+        vmm_heap_free(p, size, &error_code);
+    ASSERT(error_code == HEAP_ERROR_SUCCESS);
+    dynamo_vm_areas_unlock();
+}
+
+/* remove_vm MUST be false iff this is heap memory, which is updated separately */
+static void
+release_real_memory(void *p, size_t size, bool remove_vm)
+{
+    /* must round up to page sizes for vmm_heap_free */
+    size = ALIGN_FORWARD(size, PAGE_SIZE);
+
+    release_memory_and_update_areas((app_pc)p, size, false/*free*/, remove_vm);
+
+    /* avoid problem w/ being called by cleanup_and_terminate after dynamo_process_exit */
+    DOSTATS({
+        if (!dynamo_exited_log_and_stats)
+            STATS_SUB(memory_capacity, size);
+    });
+}
+
+static void
+extend_commitment(vm_addr_t p, size_t size, uint prot, 
+                  bool initial_commit)
+{
+    heap_error_code_t error_code;
+    ASSERT(ALIGNED(p, PAGE_SIZE));
+    size = ALIGN_FORWARD(size, PAGE_SIZE);
+    if (!vmm_heap_commit(p, size, prot, &error_code)) {
+        SYSLOG_INTERNAL_WARNING_ONCE("Out of memory - cannot extend commit "
+                                     "%dKB. Trying to recover.", size/1024);
+        heap_low_on_memory();
+        fcache_low_on_memory();
+        /* see low-memory ideas in get_real_memory */
+        if (!vmm_heap_commit(p, size, prot, &error_code)) {
+            report_low_on_memory(initial_commit ? OOM_COMMIT : OOM_EXTEND, 
+                                 error_code);
+        }
+
+        SYSLOG_INTERNAL_WARNING_ONCE("Out of memory in extend - still alive "
+                                     "after emergency free.");
+    }
+}
+
+/* a wrapper around get_real_memory that adds a guard page on each side of the requested unit.
+ * These should consume only uncommitted virtual address and should not use any physical memory.
+ * add_vm MUST be false iff this is heap memory, which is updated separately.
+ */
+static vm_addr_t
+get_guarded_real_memory(size_t reserve_size, size_t commit_size, uint prot,
+                        bool add_vm, bool guarded _IF_DEBUG(char *comment))
+{
+    vm_addr_t p;
+    uint guard_size = PAGE_SIZE;
+    heap_error_code_t error_code;
+    ASSERT(reserve_size >= commit_size);
+    if (!guarded || !dynamo_options.guard_pages) {
+        if (reserve_size == commit_size)
+            return get_real_memory(reserve_size, prot, add_vm _IF_DEBUG(comment));
+        guard_size = 0;
+    }
+
+    reserve_size = ALIGN_FORWARD(reserve_size, PAGE_SIZE);
+    commit_size = ALIGN_FORWARD(commit_size, PAGE_SIZE);
+
+    reserve_size += 2* guard_size;  /* add top and bottom guards */
+
+    /* memory alloc/dealloc and updating DR list must be atomic */
+    dynamo_vm_areas_lock(); /* if already hold lock this is a nop */
+    p = vmm_heap_reserve(reserve_size, &error_code);
+    if (p == NULL) {    
+        /* This is very unlikely to happen - we have to reach at least 2GB reserved memory. */
+        SYSLOG_INTERNAL_WARNING_ONCE("Out of memory - cannot reserve %dKB. "
+                                     "Trying to recover.", reserve_size/1024);
+        heap_low_on_memory();
+        fcache_low_on_memory();
+
+        p = vmm_heap_reserve(reserve_size, &error_code);
+        if (p == NULL) {
+            report_low_on_memory(OOM_RESERVE, error_code);
+        }
+
+        SYSLOG_INTERNAL_WARNING_ONCE("Out of memory on reserve - but still "
+                                     "alive after emergency free.");
+    }
+    /* includes guard pages if add_vm -- else, heap_vmareas_synch_units() will
+     * add guard pages in by assuming one page on each side of every heap unit
+     * if dynamo_options.guard_pages
+     */
+    account_for_memory((void *)p, reserve_size, prot, add_vm _IF_DEBUG(comment));
+    dynamo_vm_areas_unlock();
+
+    STATS_ADD_PEAK(reserved_memory_capacity, reserve_size);
+    STATS_ADD_PEAK(guard_pages, 2);
+
+    p += guard_size;
+    extend_commitment(p, commit_size, prot, true /* initial commit */);
+
+    return p;
+}
+
+/* a wrapper around get_release_memory that also frees the guard pages on each side of the requested unit. 
+ * remove_vm MUST be false iff this is heap memory, which is updated separately
+ */
+static void
+release_guarded_real_memory(vm_addr_t p, size_t size, bool remove_vm, bool guarded)
+{
+    if (!guarded || !dynamo_options.guard_pages) {
+        release_real_memory(p, size, remove_vm);
+        return;
+    }
+
+    size = ALIGN_FORWARD(size, PAGE_SIZE);
+    size += PAGE_SIZE * 2;  /* add top and bottom guards */
+    p -= PAGE_SIZE;
+
+    release_memory_and_update_areas((app_pc)p, size, false/*free*/, remove_vm);
+
+    /* avoid problem w/ being called by cleanup_and_terminate after dynamo_process_exit */
+    DOSTATS({
+        if (!dynamo_exited_log_and_stats) {
+            STATS_SUB(memory_capacity, size);
+            STATS_SUB(reserved_memory_capacity, size);
+            STATS_ADD(guard_pages, -2);
+        }
+    });
+}
+
+/* use heap_mmap to allocate large chunks of executable memory
+ * it's mainly used to allocate our fcache units
+ */
+void *
+heap_mmap_ex(size_t reserve_size, size_t commit_size, uint prot, bool guarded)
+{
+    void *p = get_guarded_real_memory(reserve_size, commit_size, prot, true, guarded
+                                      _IF_DEBUG("heap_mmap"));
+#ifdef DEBUG_MEMORY
+    memset(p, HEAP_ALLOCATED_BYTE, commit_size);
+#endif
+    /* We rely on this for freeing _post_stack in absence of dcontext */
+    ASSERT(!DYNAMO_OPTION(vm_reserve) ||
+           !DYNAMO_OPTION(stack_shares_gencode) ||
+           (ptr_uint_t)p - (guarded ? (GUARD_PAGE_ADJUSTMENT/2) : 0) ==
+           ALIGN_BACKWARD(p, VMM_BLOCK_SIZE));
+    LOG(GLOBAL, LOG_HEAP, 2, "heap_mmap: %d bytes [/ %d] @ "PFX"\n",
+        commit_size, reserve_size, p);
+    STATS_ADD_PEAK(mmap_capacity, commit_size);
+    STATS_ADD_PEAK(mmap_reserved_only, (reserve_size - commit_size));
+    return p;
+}
+
+/* use heap_mmap to allocate large chunks of executable memory
+ * it's mainly used to allocate our fcache units
+ */
+void *
+heap_mmap_reserve(size_t reserve_size, size_t commit_size)
+{
+    /* heap_mmap always marks as executable */
+    return heap_mmap_ex(reserve_size, commit_size,
+                        MEMPROT_EXEC|MEMPROT_READ|MEMPROT_WRITE, true);
+}
+
+/* It is up to the caller to ensure commit_size is a page size multiple,
+ * and that it does not extend beyond the initial reservation.
+ */
+void
+heap_mmap_extend_commitment(void *p, size_t commit_size)
+{
+    extend_commitment(p, commit_size, MEMPROT_EXEC|MEMPROT_READ|MEMPROT_WRITE,
+                      false /*not initial commit*/);
+    STATS_SUB(mmap_reserved_only, commit_size);
+    STATS_ADD_PEAK(mmap_capacity, commit_size);
+#ifdef DEBUG_MEMORY
+    memset(p, HEAP_ALLOCATED_BYTE, commit_size);
+#endif
+}
+
+/* De-commits from a committed region. */
+void
+heap_mmap_retract_commitment(void *retract_start, size_t decommit_size)
+{
+    heap_error_code_t error_code;
+    ASSERT(ALIGNED(decommit_size, PAGE_SIZE));
+    vmm_heap_decommit(retract_start, decommit_size, &error_code);
+    STATS_ADD(mmap_reserved_only, decommit_size);
+    STATS_ADD_PEAK(mmap_capacity, -(stats_int_t)decommit_size);
+}
+
+/* Allocates executable memory in the same allocation region as this thread's
+ * stack, to save address space (case 9474).
+ */
+void *
+heap_mmap_reserve_post_stack(dcontext_t *dcontext,
+                             size_t reserve_size, size_t commit_size)
+{
+    void *p;
+    byte *stack_reserve_end = NULL;
+    heap_error_code_t error_code;
+    size_t available = 0;
+    uint prot;
+    bool known_stack = false;
+    ASSERT(reserve_size > 0 && commit_size < reserve_size);
+    /* 1.5 * guard page adjustment since we'll share the middle one */ 
+    if (DYNAMO_OPTION(stack_size) + reserve_size +
+        GUARD_PAGE_ADJUSTMENT + GUARD_PAGE_ADJUSTMENT / 2 > VMM_BLOCK_SIZE) {
+        /* there's not enough room to share the allocation block, stack is too big */
+        LOG(GLOBAL, LOG_HEAP, 1, "Not enough room to allocate 0x%08x bytes post stack "
+            "of size 0x%08x\n", reserve_size, DYNAMO_OPTION(stack_size));
+        return heap_mmap_reserve(reserve_size, commit_size);
+    }
+    if (DYNAMO_OPTION(stack_shares_gencode) &&
+        /* FIXME: we could support this w/o vm_reserve, or when beyond
+         * the reservation, but we don't bother */
+        DYNAMO_OPTION(vm_reserve) &&
+        dcontext != GLOBAL_DCONTEXT && dcontext != NULL) {
+        stack_reserve_end = dcontext->dstack + GUARD_PAGE_ADJUSTMENT/2;
+#if defined(LINUX) && !defined(HAVE_PROC_MAPS)
+        if (!dynamo_initialized) {
+            /* memory info is not yet set up.  since so early we only support
+             * post-stack if inside vmm (won't be true only for pathological
+             * tiny vmm sizes)
+             */
+            if (vmm_is_reserved_unit(&heapmgt->vmheap, stack_reserve_end, reserve_size)) {
+                known_stack = true;
+                available = reserve_size;
+                prot = 0;
+            } else
+                known_stack = false;
+        } else
+#else
+            known_stack = get_memory_info(stack_reserve_end, NULL, &available, &prot);
+#endif
+        /* If ever out of vmheap, then may have free space beyond stack,
+         * which we could support but don't (see FIXME above) */
+        ASSERT(out_of_vmheap_once ||
+               (known_stack && available >= reserve_size && prot == 0));
+    }
+    if (!known_stack ||
+        /* if -no_vm_reserve will short-circuit so no vmh deref danger */
+        !vmm_in_same_block(dcontext->dstack,
+                           /* we do want a guard page at the end */
+                           stack_reserve_end + reserve_size) ||
+        available < reserve_size) {
+        ASSERT(!DYNAMO_OPTION(stack_shares_gencode) ||
+               !DYNAMO_OPTION(vm_reserve) || out_of_vmheap_once);
+        DOLOG(1, LOG_HEAP, {
+            if (known_stack && available < reserve_size) {
+                LOG(GLOBAL, LOG_HEAP, 1,
+                    "heap_mmap_reserve_post_stack: avail %d < needed %d\n",
+                    available, reserve_size);
+            }
+        });
+        STATS_INC(mmap_no_share_stack_region);
+        return heap_mmap_reserve(reserve_size, commit_size);
+    }
+    ASSERT(DYNAMO_OPTION(vm_reserve));
+    ASSERT(stack_reserve_end != NULL);
+    prot = MEMPROT_EXEC|MEMPROT_READ|MEMPROT_WRITE;
+    /* memory alloc/dealloc and updating DR list must be atomic */
+    dynamo_vm_areas_lock(); /* if already hold lock this is a nop */
+    /* We share the stack's end guard page as our start guard page */
+    if (vmm_is_reserved_unit(&heapmgt->vmheap, stack_reserve_end, reserve_size)) {
+        /* Memory is already reserved with OS */
+        p = stack_reserve_end;
+    } else {
+        p = os_heap_reserve(stack_reserve_end, reserve_size, &error_code);
+#ifdef X64
+        /* ensure future heap allocations are reachable from this allocation
+         * (this will also verify that this region meets reachability requirements) */
+        if (p != NULL)
+            request_region_be_heap_reachable(p, reserve_size);
+#endif
+        if (p == NULL) {
+            ASSERT_NOT_REACHED();
+            LOG(GLOBAL, LOG_HEAP, 1,
+                "heap_mmap_reserve_post_stack: reserve failed "PFX"\n", error_code);
+            dynamo_vm_areas_unlock();
+            STATS_INC(mmap_no_share_stack_region);
+            return heap_mmap_reserve(reserve_size, commit_size);
+        }
+        ASSERT(error_code == HEAP_ERROR_SUCCESS);
+    }
+    if (!vmm_heap_commit(p, commit_size, prot, &error_code)) {
+        ASSERT_NOT_REACHED();
+        LOG(GLOBAL, LOG_HEAP, 1, "heap_mmap_reserve_post_stack: commit failed "PFX"\n",
+            error_code);
+        if (!vmm_is_reserved_unit(&heapmgt->vmheap, stack_reserve_end, reserve_size)) {
+            os_heap_free(p, reserve_size, &error_code);
+            ASSERT(error_code == HEAP_ERROR_SUCCESS);
+        }
+        dynamo_vm_areas_unlock();
+        STATS_INC(mmap_no_share_stack_region);
+        return heap_mmap_reserve(reserve_size, commit_size);
+    }
+    account_for_memory(p, reserve_size, prot, true/*add now*/
+                       _IF_DEBUG("heap_mmap_reserve_post_stack"));
+    dynamo_vm_areas_unlock();
+    /* We rely on this for freeing in absence of dcontext */
+    ASSERT((ptr_uint_t)p - GUARD_PAGE_ADJUSTMENT/2 !=
+           ALIGN_BACKWARD(p, VMM_BLOCK_SIZE));
+#ifdef DEBUG_MEMORY
+    memset(p, HEAP_ALLOCATED_BYTE, commit_size);
+#endif
+    LOG(GLOBAL, LOG_HEAP, 2, "heap_mmap w/ stack: %d bytes [/ %d] @ "PFX"\n",
+        commit_size, reserve_size, p);
+    STATS_ADD_PEAK(mmap_capacity, commit_size);
+    STATS_ADD_PEAK(mmap_reserved_only, (reserve_size - commit_size));
+    STATS_INC(mmap_share_stack_region);
+    return p;
+}
+
+/* De-commits memory that was allocated in the same allocation region as this
+ * thread's stack (case 9474).
+ */
+void
+heap_munmap_post_stack(dcontext_t *dcontext, void *p, size_t reserve_size)
+{
+    /* We would require a valid dcontext and compare to the stack reserve end,
+     * but on detach we have no dcontext, so we instead use block alignment.
+     */
+    DODEBUG({
+        if (dcontext != NULL && dcontext != GLOBAL_DCONTEXT &&
+            DYNAMO_OPTION(vm_reserve) && DYNAMO_OPTION(stack_shares_gencode)) {
+            bool at_stack_end = (p == dcontext->dstack + GUARD_PAGE_ADJUSTMENT/2);
+            bool at_block_start = ((ptr_uint_t)p - GUARD_PAGE_ADJUSTMENT/2 ==
+                                   ALIGN_BACKWARD(p, VMM_BLOCK_SIZE));
+            ASSERT((at_stack_end && !at_block_start) ||
+                   (!at_stack_end && at_block_start));
+        }
+    });
+    if (!DYNAMO_OPTION(vm_reserve) ||
+        !DYNAMO_OPTION(stack_shares_gencode) ||
+        (ptr_uint_t)p - GUARD_PAGE_ADJUSTMENT/2 == ALIGN_BACKWARD(p, VMM_BLOCK_SIZE)) {
+        heap_munmap(p, reserve_size);
+    } else {
+        /* Detach makes it a pain to pass in the commit size so
+         * we use the reserve size, which works fine.
+         */
+        release_memory_and_update_areas((app_pc)p, reserve_size, true/*decommit*/,
+                                        true/*update now*/);
+        LOG(GLOBAL, LOG_HEAP, 2, "heap_munmap_post_stack: %d bytes @ "PFX"\n",
+            reserve_size, p);
+        STATS_SUB(mmap_capacity, reserve_size);
+        STATS_SUB(mmap_reserved_only, reserve_size);
+    }
+}
+
+/* use heap_mmap to allocate large chunks of executable memory
+ * it's mainly used to allocate our fcache units
+ */
+void *
+heap_mmap(size_t size)
+{
+    return heap_mmap_reserve(size, size);
+}
+
+/* free memory-mapped storage */
+void
+heap_munmap_ex(void *p, size_t size, bool guarded)
+{
+#ifdef DEBUG_MEMORY
+    /* can't set to HEAP_UNALLOCATED_BYTE since really not in our address 
+     * space anymore */
+#endif
+    release_guarded_real_memory((vm_addr_t)p, size, true/*update DR areas immediately*/,
+                                guarded);
+
+    DOSTATS({
+        /* avoid problem w/ being called by cleanup_and_terminate after dynamo_process_exit */
+        if (!dynamo_exited_log_and_stats) {
+            LOG(GLOBAL, LOG_HEAP, 2, "heap_munmap: %d bytes @ "PFX"\n", size, p);
+            STATS_SUB(mmap_capacity, size);
+            STATS_SUB(mmap_reserved_only, size);
+        }
+    });
+}
+
+/* free memory-mapped storage */
+void
+heap_munmap(void *p, size_t size)
+{
+    heap_munmap_ex(p, size, true/*guarded*/);
+}
+
+#ifdef STACK_GUARD_PAGE
+# define STACK_GUARD_PAGES 1    
+#endif
+
+/* use stack_alloc to build a stack -- it returns TOS
+ * For STACK_GUARD_PAGE, it also marks the bottom STACK_GUARD_PAGES==1 
+ * to detect overflows when used.
+ */
+void *
+stack_alloc(size_t size)
+{
+    void *p;
+
+    /* we reserve and commit at once for now
+     * FIXME case 2330: commit-on-demand could allow larger max sizes w/o
+     * hurting us in the common case
+     */
+    p = get_guarded_real_memory(size, size, MEMPROT_READ|MEMPROT_WRITE, true, true
+                                _IF_DEBUG("stack_alloc"));
+#ifdef DEBUG_MEMORY
+    memset(p, HEAP_ALLOCATED_BYTE, size);
+#endif
+
+#ifdef STACK_GUARD_PAGE
+    /* mark the bottom page non-accessible to trap stack overflow */
+    /* NOTE: the guard page should be included in the total memory requested */
+# ifdef WINDOWS
+    mark_page_as_guard((byte *)p + ((STACK_GUARD_PAGES - 1) * PAGE_SIZE));
+# else
+    /* FIXME: make no access, not just no write -- and update signal.c to
+     * look at reads and not just writes -- though unwritable is nearly as good
+     */
+#  if defined(CLIENT_INTERFACE) || defined(STANDALONE_UNIT_TEST)
+    if (!standalone_library)
+#  endif
+        make_unwritable(p, STACK_GUARD_PAGES * PAGE_SIZE);
+# endif
+#endif
+
+    STATS_ADD(stack_capacity, size);
+    STATS_MAX(peak_stack_capacity, stack_capacity);
+    /* stack grows from high to low */
+    return (void *) ((ptr_uint_t)p + size);
+}
+
+/* free stack storage */
+void
+stack_free(void *p, size_t size)
+{
+    if (size == 0)
+        size = DYNAMORIO_STACK_SIZE;
+    p = (void *) ((vm_addr_t)p - size);
+    release_guarded_real_memory((vm_addr_t)p, size, true/*update DR areas immediately*/,
+                                true);
+    DOSTATS({
+        if (!dynamo_exited_log_and_stats)
+            STATS_SUB(stack_capacity, size);
+    });
+}
+
+#ifdef STACK_GUARD_PAGE
+/* only checks initstack and current dcontext 
+ * does not check any dstacks on the callback stack (win32) */
+bool
+is_stack_overflow(dcontext_t *dcontext, byte *sp)
+{
+    /* ASSUMPTION: size of stack is DYNAMORIO_STACK_SIZE = dynamo_options.stack_size
+     * currently sideline violates that for a thread stack
+     * but all dstacks and initstack should be this size
+     */
+    byte *bottom = dcontext->dstack - DYNAMORIO_STACK_SIZE;
+    /* see if in bottom guard page of dstack */
+    if (sp >= bottom && sp < bottom + (STACK_GUARD_PAGES * PAGE_SIZE))
+        return true;
+    /* now check the initstack */
+    bottom = initstack - DYNAMORIO_STACK_SIZE;
+    if (sp >= bottom && sp < bottom + (STACK_GUARD_PAGES * PAGE_SIZE))
+        return true;
+    return false;
+}
+#endif
+
+byte *
+map_file(file_t f, size_t size, size_t offs, app_pc addr, uint prot, bool copy_on_write)
+{
+    byte *view;
+    /* memory alloc/dealloc and updating DR list must be atomic */
+    dynamo_vm_areas_lock(); /* if already hold lock this is a nop */
+    view = os_map_file(f, size, offs, addr, prot, copy_on_write);
+    if (view != NULL) {
+        STATS_ADD_PEAK(file_map_capacity, size);
+        account_for_memory((void *)view, size, prot, true/*add now*/
+                           _IF_DEBUG("map_file"));
+    }
+    dynamo_vm_areas_unlock();
+    return view;
+}
+
+bool
+unmap_file(byte *map, size_t size)
+{
+    bool success;
+    ASSERT(map != NULL);
+    /* memory alloc/dealloc and updating DR list must be atomic */
+    dynamo_vm_areas_lock(); /* if already hold lock this is a nop */
+    update_dynamo_areas_on_release(map, map+size, true/*remove now*/);
+    STATS_SUB(file_map_capacity, size);
+    success = os_unmap_file(map, size);
+    dynamo_vm_areas_unlock();
+    return success;
+}
+
+
+/* We cannot incrementally keep dynamo vm area list up to date due to
+ * circular dependencies bet vmareas and global heap (trust me, I've tried
+ * to support it with reentrant routines and recursive locks, the hard part
+ * is getting add_vm_area to be reentrant or to queue up adding areas,
+ * I think this solution is much more elegant, plus it avoids race conditions
+ * between DR memory allocation and the vmareas list by ensuring the list
+ * is up to date at the exact time of each query).
+ * Instead we on-demand walk the units.
+ * Freed units can usually be removed incrementally, except when we
+ * hold the heap_unit_lock when we run out of memory -- when we set
+ * a flag telling the caller of this routine to remove all heap areas
+ * from the vm list prior to calling us to add the real ones back in.
+ * Re-adding everyone is the simplest policy, so we don't have to keep
+ * track of who's been added.
+ * The caller is assumed to hold the dynamo vm areas write lock.
+ */
+void
+heap_vmareas_synch_units()
+{
+    heap_unit_t *u, *next;
+    /* make sure to add guard page on each side, as well */
+    uint offs = (dynamo_options.guard_pages) ? PAGE_SIZE : 0;
+    /* we again have circular dependence w/ vmareas if it happens to need a
+     * new unit in the course of adding these areas, so we use a recursive lock!
+     * furthermore, we need to own the global lock now, to avoid deadlock with
+     * another thread who does global_alloc and then needs a new unit!
+     * which means that the global_alloc lock must be recursive since vmareas
+     * may need to global_alloc...
+     */
+    /* if chance could own both locks, must grab both now
+     * always grab global_alloc first, then we won't have deadlocks
+     */
+    acquire_recursive_lock(&global_alloc_lock);
+    acquire_recursive_lock(&heap_unit_lock);
+    if (dynamo_areas_pending_remove) {
+        dynamo_areas_pending_remove = false;
+        remove_dynamo_heap_areas();
+
+        /* When heap units are removed from the dynamo_area, they should be
+         * marked so.  See case 4196.
+         */
+        for (u = heapmgt->heap.units; u != NULL; u = u->next_global) 
+            u->in_vmarea_list = false;
+        for (u = heapmgt->heap.dead; u != NULL; u = u->next_global) 
+            u->in_vmarea_list = false;
+    }
+    for (u = heapmgt->heap.units; u != NULL; u = next) {
+        app_pc start = (app_pc)u - offs;
+        app_pc end = UNIT_RESERVED_END(u) + offs;
+        /* u can be moved to dead list, so cache the next link; case 4196. */
+        next = u->next_global;
+        /* case 3045: areas inside the vmheap reservation are not added to the list */
+        if (!u->in_vmarea_list && !vmm_is_reserved_unit(&heapmgt->vmheap,
+                                                        start, end - start)) {
+            /* case 4196 if next is used by dynamo_vmareas then next
+             * may become dead if vector is resized, then u should be
+             * alive and u->next_global should be reset AFTER add  */
+            bool next_may_die = 
+                is_dynamo_area_buffer(UNIT_GET_START_PC(next)); /* keep breaking abstractions */
+            /* dynamo_areas.buf vector may get resized and u can either
+             * go to the dead unit list, or it can be released back to
+             * the OS.  We'll mark it as being in vmarea list to avoid
+             * re-adding when going through dead one's, and we'll mark
+             * _before_ the potential free.  If dynamo_areas.buf is
+             * freed back to the OS we'll have another iteration in
+             * update_dynamo_vm_areas() until we get fully
+             * synchronized, so we don't need to worry about the
+             * inconsistency.
+             */
+            u->in_vmarea_list = true;
+            add_dynamo_heap_vm_area(start, end, true, false _IF_DEBUG("heap unit"));
+            /* NOTE: Since we could mark_dynamo_vm_areas_stale instead of adding to
+             * it, we may lose prot info about this unit.
+             * FIXME: Currently, this is done only at one place, which allocates unit
+             * as MEMPROT_READ | MEMPROT_WRITE.  If other places are added, then this
+             * needs to change.
+             */
+            update_all_memory_areas((app_pc)u, UNIT_RESERVED_END(u),
+                                    MEMPROT_READ | MEMPROT_WRITE); /* unit */
+            if (offs != 0) {
+                /* guard pages */
+                update_all_memory_areas((app_pc)u - offs, (app_pc)u, MEMPROT_NONE);
+                update_all_memory_areas(UNIT_RESERVED_END(u),
+                                        UNIT_RESERVED_END(u) + offs, MEMPROT_NONE);
+            }
+            if (next_may_die) {
+                STATS_INC(num_vmareas_resize_synch);
+                /* if next was potentially on dead row, then current
+                 * should still be live and point to the next live
+                 */
+                next = u->next_global;
+            }
+        }
+    }
+    for (u = heapmgt->heap.dead; u != NULL; u = next) {
+        app_pc start = (app_pc)u - offs;
+        app_pc end = UNIT_RESERVED_END(u) + offs;
+        /* u can be moved to live list, so cache the next link; case 4196. */
+        next = u->next_global;
+        /* case 3045: areas inside the vmheap reservation are not added to the list */
+        if (!u->in_vmarea_list && !vmm_is_reserved_unit(&heapmgt->vmheap,
+                                                        start, end - start)) {
+            u->in_vmarea_list = true;
+            add_dynamo_heap_vm_area(start, end, true, false _IF_DEBUG("dead heap unit"));
+            update_all_memory_areas((app_pc)u, UNIT_RESERVED_END(u),
+                                    MEMPROT_READ | MEMPROT_WRITE); /* unit */
+            if (offs != 0) {
+                /* guard pages */
+                update_all_memory_areas(start, (app_pc)u, MEMPROT_NONE);
+                update_all_memory_areas(UNIT_RESERVED_END(u), end, MEMPROT_NONE);
+            }
+            /* case 4196 if next was put back on live list for
+             * dynamo_areas.buf vector, then next will no longer be a
+             * valid iterator over dead list
+             */
+            if (is_dynamo_area_buffer(UNIT_GET_START_PC(next))) { /* keep breaking abstractions */
+                STATS_INC(num_vmareas_resize_synch);
+                ASSERT_NOT_TESTED();
+                next = u->next_global;
+            }
+        }
+    }
+    release_recursive_lock(&heap_unit_lock);
+    release_recursive_lock(&global_alloc_lock);
+}
+
+/* shared between global and global_unprotected */
+static void *
+common_global_heap_alloc(thread_units_t *tu, size_t size HEAPACCT(which_heap_t which))
+{
+    void *p;
+    acquire_recursive_lock(&global_alloc_lock);
+    p = common_heap_alloc(tu, size HEAPACCT(which));
+    release_recursive_lock(&global_alloc_lock);
+    if (p == NULL) {
+        /* circular dependence solution: we need to hold DR lock before
+         * global alloc lock -- so we back out, grab it, and retry
+         */
+        dynamo_vm_areas_lock();
+        acquire_recursive_lock(&global_alloc_lock);
+        p = common_heap_alloc(tu, size HEAPACCT(which));
+        release_recursive_lock(&global_alloc_lock);
+        dynamo_vm_areas_unlock();
+    }
+    ASSERT(p != NULL);
+    return p;
+}
+
+/* shared between global and global_unprotected */
+static void
+common_global_heap_free(thread_units_t *tu, void *p, size_t size HEAPACCT(which_heap_t which))
+{
+    bool ok;
+    if (p == NULL) {
+        ASSERT(false && "attempt to free NULL");
+        return;
+    }
+
+    acquire_recursive_lock(&global_alloc_lock);
+    ok = common_heap_free(tu, p, size HEAPACCT(which));
+    release_recursive_lock(&global_alloc_lock);
+    if (!ok) {
+        /* circular dependence solution: we need to hold DR lock before
+         * global alloc lock -- so we back out, grab it, and retry
+         */
+        dynamo_vm_areas_lock();
+        acquire_recursive_lock(&global_alloc_lock);
+        ok = common_heap_free(tu, p, size HEAPACCT(which));
+        release_recursive_lock(&global_alloc_lock);
+        dynamo_vm_areas_unlock();
+    }
+    ASSERT(ok);
+}
+
+/* these functions use the global heap instead of a thread's heap: */
+void *
+global_heap_alloc(size_t size HEAPACCT(which_heap_t which))
+{
+    void *p = common_global_heap_alloc(&heapmgt->global_units, size HEAPACCT(which));
+    ASSERT(p != NULL);
+    LOG(GLOBAL, LOG_HEAP, 6, "\nglobal alloc: "PFX" (%d bytes)\n", p, size);
+    return p;
+}
+
+void
+global_heap_free(void *p, size_t size HEAPACCT(which_heap_t which))
+{
+    common_global_heap_free(&heapmgt->global_units, p, size HEAPACCT(which));
+    LOG(GLOBAL, LOG_HEAP, 6, "\nglobal free: "PFX" (%d bytes)\n", p, size);
+}
+
+
+/* reallocate area 
+   allocates new_num elements of element_size 
+   if ptr is NULL acts like global_heap_alloc,
+   copies an old_num elements of given size in the new area */
+/* FIXME: do a heap_realloc and a special_heap_realloc too */
+void *
+global_heap_realloc(void *ptr, size_t old_num, size_t new_num, size_t element_size
+                    HEAPACCT(which_heap_t which))
+{
+    void *new_area = global_heap_alloc(new_num * element_size HEAPACCT(which));
+    if (ptr) {
+        ASSERT(old_num <= new_num);
+        memcpy(new_area, ptr, old_num * element_size);
+        global_heap_free(ptr, old_num * element_size HEAPACCT(which));
+    }
+    return new_area;
+}
+
+/* size does not include guard pages (if any) and is reserved, but only
+ * DYNAMO_OPTION(heap_commit_increment) is committed up front
+ */
+static heap_unit_t *
+heap_create_unit(thread_units_t *tu, size_t size, bool must_be_new)
+{
+    heap_unit_t *u = NULL, *dead = NULL, *prev_dead = NULL;
+    bool new_unit = false;
+
+    /* we do not restrict size to unit max as we have to make larger-than-max
+     * units for oversized requests
+     */
+
+    /* modifying heap list and DR areas must be atomic, and must grab
+     * DR area lock before heap_unit_lock
+     */
+    ASSERT(safe_to_allocate_or_free_heap_units());
+    dynamo_vm_areas_lock();
+    /* take from dead list if possible */
+    acquire_recursive_lock(&heap_unit_lock);
+
+    /* FIXME: need to unprotect units that we're going to perform
+     * {next,prev}_global assignments too -- but need to know whether
+     * to re-protect -- do all at once, or each we need?  add a writable
+     * flag to heap_unit_t?
+     */
+
+    if (!must_be_new) {
+        for (dead = heapmgt->heap.dead;
+             dead != NULL && UNIT_RESERVED_SIZE(heapmgt->heap.dead) < size;
+             prev_dead = dead, dead = dead->next_global)
+            ;
+    }
+    if (dead != NULL) {
+        if (prev_dead == NULL)
+            heapmgt->heap.dead = dead->next_global;
+        else
+            prev_dead->next_global = dead->next_global;
+        u = dead;
+        heapmgt->heap.num_dead--;
+        RSTATS_DEC(heap_num_free);
+        release_recursive_lock(&heap_unit_lock);
+        LOG(GLOBAL, LOG_HEAP, 2,
+            "Re-using dead heap unit: "PFX"-"PFX" %d KB (need %d KB)\n",
+            u, ((byte*)u)+size, UNIT_RESERVED_SIZE(u)/1024, size/1024);
+    } else {
+        size_t commit_size = DYNAMO_OPTION(heap_commit_increment);
+        release_recursive_lock(&heap_unit_lock); /* do not hold while asking for memory */
+        /* create new unit */
+        ASSERT(commit_size <= size);
+        u = (heap_unit_t *)
+            get_guarded_real_memory(size, commit_size, MEMPROT_READ|MEMPROT_WRITE,
+                                    false, true _IF_DEBUG(""));
+        new_unit = true;
+        /* FIXME: handle low memory conditions by freeing units, + fcache units? */
+        ASSERT(u);
+        LOG(GLOBAL, LOG_HEAP, 2, "New heap unit: "PFX"-"PFX"\n", u, ((byte*)u)+size);
+        /* u is kept at top of unit itself, so displace start pc */
+        u->start_pc = (heap_pc) (((ptr_uint_t)u) + sizeof(heap_unit_t));
+        u->end_pc = ((heap_pc)u) + commit_size;
+        u->reserved_end_pc = ((heap_pc)u) + size;
+        u->in_vmarea_list = false;
+        STATS_ADD(heap_capacity, commit_size);
+        STATS_MAX(peak_heap_capacity, heap_capacity);
+        /* FIXME: heap sizes are not always page-aligned so stats will be off */
+        STATS_ADD_PEAK(heap_reserved_only, (u->reserved_end_pc - u->end_pc));
+    }
+    RSTATS_ADD_PEAK(heap_num_live, 1);
+
+    u->cur_pc = u->start_pc;
+    u->next_local = NULL;
+    DODEBUG({
+        u->id = tu->num_units;
+        tu->num_units++;
+    });
+
+    acquire_recursive_lock(&heap_unit_lock);
+    u->next_global = heapmgt->heap.units;
+    if (heapmgt->heap.units != NULL)
+        heapmgt->heap.units->prev_global = u;
+    u->prev_global = NULL;
+    heapmgt->heap.units = u;
+    release_recursive_lock(&heap_unit_lock);
+    dynamo_vm_areas_unlock();
+
+#ifdef DEBUG_MEMORY
+    memset(u->start_pc, HEAP_UNALLOCATED_BYTE, u->end_pc - u->start_pc);
+#endif
+    return u;
+}
+
+/* dcontext only used to determine whether a global unit or not */
+static void 
+heap_free_unit(heap_unit_t *unit, dcontext_t *dcontext)
+{
+    heap_unit_t *u, *prev_u;
+#ifdef DEBUG_MEMORY
+    /* Unit should already be set to all HEAP_UNALLOCATED by the individual 
+     * frees and the free list cleanup, verify. */
+    /* NOTE - this assert fires if any memory in the unit wasn't freed. This
+     * would include memory allocated ACCT_TOMBSTONE (which we don't currently 
+     * use). Using ACCT_TOMBSTONE is dangerous since we will still free the 
+     * unit here (say at proc or thread exit) even if there are ACCT_TOMBSTONE 
+     * allocations in it. */
+    /* Note, this memset check is done only on the special heap unit header,
+     * not on the unit itself - FIXME: case 10434.  Maybe we should embed the
+     * special heap unit header in the first special heap unit itself. */
+    /* The hotp_only leak relaxation below is for case 9588 & 9593.  */
+    CLIENT_ASSERT(IF_HOTP(hotp_only_contains_leaked_trampoline
+                          (unit->start_pc, unit->end_pc - unit->start_pc) ||)
+                  is_region_memset_to_char(unit->start_pc, 
+                                           unit->end_pc - unit->start_pc,
+                                           HEAP_UNALLOCATED_BYTE),
+                  "memory leak detected");
+#endif
+    /* modifying heap list and DR areas must be atomic, and must grab
+     * DR area lock before heap_unit_lock
+     */
+    ASSERT(safe_to_allocate_or_free_heap_units());
+    dynamo_vm_areas_lock();
+    acquire_recursive_lock(&heap_unit_lock);
+
+    /* FIXME: need to unprotect units that we're going to perform
+     * {next,prev}_global assignments too -- but need to know whether
+     * to re-protect -- do all at once, or each we need?  add a writable
+     * flag to heap_unit_t?
+     */
+
+    /* remove from live list */
+    if (unit->prev_global != NULL) {
+        unit->prev_global->next_global = unit->next_global;
+    } else
+        heapmgt->heap.units = unit->next_global;
+    if (unit->next_global != NULL) {
+        unit->next_global->prev_global = unit->prev_global;
+    }
+    /* prev_global is not used in the dead list */
+    unit->prev_global = NULL;
+    RSTATS_DEC(heap_num_live);
+
+    /* heuristic: don't keep around more dead units than max(5, 1/4 num threads)
+     * FIXME: share the policy with the fcache dead unit policy
+     * also, don't put special larger-than-max units on free list
+     */
+    if (UNITALLOC(unit) <= HEAP_UNIT_MAX_SIZE &&
+        (heapmgt->heap.num_dead < 5 ||
+         heapmgt->heap.num_dead * 4U <= (uint) get_num_threads())) {
+        /* Keep dead list sorted small-to-large to avoid grabbing large
+         * when can take small and then needing to allocate when only
+         * have small left.  Helps out with lots of small threads.
+         */
+        for (u = heapmgt->heap.dead, prev_u = NULL;
+             u != NULL && UNIT_RESERVED_SIZE(u) < UNIT_RESERVED_SIZE(unit);
+             prev_u = u, u = u->next_global)
+            ;
+        if (prev_u == NULL) {
+            unit->next_global = heapmgt->heap.dead;
+            heapmgt->heap.dead = unit;
+        } else {
+            unit->next_global = u;
+            prev_u->next_global = unit;
+        }
+        heapmgt->heap.num_dead++;
+        release_recursive_lock(&heap_unit_lock);
+        RSTATS_ADD_PEAK(heap_num_free, 1);
+    } else {
+        /* don't need to hold this while freeing since still hold DR areas lock */
+        release_recursive_lock(&heap_unit_lock);
+        LOG(GLOBAL, LOG_HEAP, 1, "\tfreeing excess dead unit "PFX"-"PFX" [-"PFX"]\n",
+            unit, UNIT_COMMIT_END(unit), UNIT_RESERVED_END(unit));
+        really_free_unit(unit);
+    }
+    /* FIXME: shrink lock-held path if we see contention */
+    dynamo_vm_areas_unlock();
+}
+
+#ifdef DEBUG_MEMORY
+static heap_unit_t *
+find_heap_unit(thread_units_t *tu, heap_pc p, size_t size)
+{
+    /* FIXME (case 6198): this is a perf hit in debug builds.  But, we can't use
+     * a new vmvector b/c of circular dependences.  Proposal: use custom data
+     * field of vm_area_t in dynamo_areas list for heap entries to store a pointer
+     * to the heap_unit_t struct, and add a backpointer to the owning thread_units_t
+     * in heap_unit_t.  Then have to make sure it's ok lock-wise to query the
+     * dynamo_areas in the middle of an alloc or a free.  It should be but for
+     * global alloc and free we will have to grab the dynamo_areas lock up front
+     * every time instead of the very rare times now when we need a new unit.
+     */
+    heap_unit_t *unit;
+    for (unit = tu->top_unit; 
+         unit != NULL && (p < unit->start_pc || p+size > unit->end_pc); 
+         unit = unit->next_local);
+    return unit;
+}
+#endif
+
+static void
+threadunits_init(dcontext_t *dcontext, thread_units_t *tu, size_t size)
+{
+    int i;
+    DODEBUG({
+        tu->num_units = 0;
+    });
+    tu->top_unit = heap_create_unit(tu, size - GUARD_PAGE_ADJUSTMENT,
+                                    false/*can reuse*/);
+    tu->cur_unit = tu->top_unit;
+    tu->dcontext = dcontext;
+    tu->writable = true;
+#ifdef HEAP_ACCOUNTING
+    memset(&tu->acct, 0, sizeof(tu->acct));
+#endif
+    for (i=0; i<BLOCK_TYPES; i++)
+        tu->free_list[i] = NULL;
+}
+
+#ifdef HEAP_ACCOUNTING
+#define MAX_5_DIGIT 99999
+static void
+print_tu_heap_statistics(thread_units_t *tu, file_t logfile, char *prefix)
+{
+    int i;
+    size_t total = 0, cur = 0;
+    LOG(logfile, LOG_HEAP|LOG_STATS, 1, "%s heap breakdown:\n", prefix);
+    for (i = 0; i < ACCT_LAST; i++) {
+        /* print out cur since this is done periodically, not just at end */
+        LOG(logfile, LOG_HEAP|LOG_STATS, 1,
+            "%12s: cur=%5"SZFC"K, max=%5"SZFC"K, #=%7d, 1=",
+            whichheap_name[i], tu->acct.cur_usage[i]/1024,
+            tu->acct.max_usage[i]/1024, tu->acct.num_alloc[i]);
+        if (tu->acct.max_single[i] <= MAX_5_DIGIT)
+            LOG(logfile, LOG_HEAP|LOG_STATS, 1, "%5"SZFC, tu->acct.max_single[i]);
+        else {
+            LOG(logfile, LOG_HEAP|LOG_STATS, 1, "%4"SZFC"K",
+                tu->acct.max_single[i]/1024);
+        }
+        LOG(logfile, LOG_HEAP|LOG_STATS, 1,
+            ", new=%5"SZFC"K, re=%5"SZFC"K\n",
+            tu->acct.alloc_new[i]/1024, tu->acct.alloc_reuse[i]/1024);
+        total += tu->acct.max_usage[i];
+        cur += tu->acct.cur_usage[i];
+    }
+    LOG(logfile, LOG_HEAP|LOG_STATS, 1,
+        "Total cur usage: %6"SZFC" KB\n", cur/1024);
+    LOG(logfile, LOG_HEAP|LOG_STATS, 1,
+        "Total max (not nec. all used simult.): %6"SZFC" KB\n", total/1024);
+}
+
+void
+print_heap_statistics()
+{
+    /* just do cur thread, don't try to walk all threads */
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    DOSTATS({
+        uint i;
+        LOG(GLOBAL, LOG_STATS, 1, "Heap bucket usage counts and wasted memory:\n");
+        for (i=0; i<BLOCK_TYPES; i++) {
+            LOG(GLOBAL, LOG_STATS|LOG_HEAP, 1,
+                "%2d %3d count=%9u peak_count=%9u peak_wasted=%9u peak_align=%9u\n",
+                i, BLOCK_SIZES[i], block_total_count[i], block_peak_count[i],
+                block_peak_wasted[i], block_peak_align_pad[i]);
+        }
+    });
+    if (dcontext != NULL) {
+        thread_heap_t *th = (thread_heap_t *) dcontext->heap_field;
+        if (th != NULL) { /* may not be initialized yet */
+            print_tu_heap_statistics(th->local_heap, THREAD, "Thread");
+            if (DYNAMO_OPTION(enable_reset)) {
+                ASSERT(th->nonpersistent_heap != NULL);
+                print_tu_heap_statistics(th->nonpersistent_heap, THREAD,
+                                         "Thread non-persistent");
+            }
+        }
+    }
+    if (DYNAMO_OPTION(enable_reset)) {
+        print_tu_heap_statistics(&heapmgt->global_nonpersistent_units, GLOBAL,
+                                 "Non-persistent global units");
+    }
+    print_tu_heap_statistics(&global_racy_units, GLOBAL, "Racy Up-to-date Process");
+    print_tu_heap_statistics(&heapmgt->global_units, GLOBAL,
+                             "Updated-at-end Process (max is total of maxes)");
+}
+
+static void
+add_heapacct_to_global_stats(heap_acct_t *acct)
+{
+    /* add this thread's stats to the accurate (non-racy) global stats 
+     * FIXME: this gives a nice in-one-place total, but loses the
+     * global-heap-only stats -- perhaps should add a total_units stats
+     * to capture total and leave global alone here?
+     */
+    uint i;
+    acquire_recursive_lock(&global_alloc_lock);
+    for (i = 0; i < ACCT_LAST; i++) {
+        heapmgt->global_units.acct.alloc_reuse[i] += acct->alloc_reuse[i];
+        heapmgt->global_units.acct.alloc_new[i] += acct->alloc_new[i];
+        heapmgt->global_units.acct.cur_usage[i] += acct->cur_usage[i];
+        /* FIXME: these maxes are now not simultaneous max but sum-of-maxes */
+        heapmgt->global_units.acct.max_usage[i] += acct->max_usage[i];
+        heapmgt->global_units.acct.max_single[i] += acct->max_single[i];
+        heapmgt->global_units.acct.num_alloc[i] += acct->num_alloc[i];
+    }
+    release_recursive_lock(&global_alloc_lock);
+}
+#endif
+
+/* dcontext only used for debugging */
+static void
+threadunits_exit(thread_units_t *tu, dcontext_t *dcontext)
+{
+    heap_unit_t *u, *next_u;
+#ifdef DEBUG
+    size_t total_heap_used = 0;
+# ifdef HEAP_ACCOUNTING
+    int j;
+# endif
+#endif
+#ifdef DEBUG_MEMORY
+    /* verify and clear (for later asserts) the free list */
+    uint i;
+    for (i = 0; i < BLOCK_TYPES; i++) {
+        heap_pc p, next_p;
+        for (p = tu->free_list[i]; p != NULL; p = next_p) {
+            next_p = *(heap_pc *)p;
+            /* clear the pointer to the next free for later asserts */
+            *(heap_pc *)p = (heap_pc) HEAP_UNALLOCATED_PTR_UINT;
+            if (i < BLOCK_TYPES-1) {
+                CLIENT_ASSERT(is_region_memset_to_char(p, BLOCK_SIZES[i], 
+                                                       HEAP_UNALLOCATED_BYTE),
+                              "memory corruption detected");
+            } else {
+                /* variable sized blocks */
+                CLIENT_ASSERT(is_region_memset_to_char(p, VARIABLE_SIZE(p),
+                                                       HEAP_UNALLOCATED_BYTE),
+                              "memory corruption detected");
+                /* clear the header for later asserts */
+                MEMSET_HEADER(p, HEAP_UNALLOCATED);
+            }
+        }
+        tu->free_list[i] = NULL;
+    }
+#endif
+    u = tu->top_unit;
+    while (u != NULL) {
+        DOLOG(1, LOG_HEAP|LOG_STATS, {
+            size_t num_used = u->cur_pc - u->start_pc;
+            total_heap_used += num_used;
+            LOG(THREAD,
+                LOG_HEAP|LOG_STATS, 1,
+                "Heap unit %d @"PFX"-"PFX" [-"PFX"] ("SZFMT" [/"SZFMT"] KB): used "
+                SZFMT" KB\n",
+                u->id, u, UNIT_COMMIT_END(u),
+                UNIT_RESERVED_END(u), (UNIT_COMMIT_SIZE(u))/1024,
+                (UNIT_RESERVED_SIZE(u))/1024, num_used/1024);
+        });
+        next_u = u->next_local;
+        heap_free_unit(u, dcontext);
+        u = next_u;
+    }
+    LOG(THREAD, LOG_HEAP|LOG_STATS, 1,
+        "\tTotal heap used: "SZFMT" KB\n", total_heap_used/1024);
+#if defined(DEBUG) && defined(HEAP_ACCOUNTING)
+    /* FIXME: separate scopes - in smaller functions for each of DEBUG_MEMORY and HEAP_ACCOUNTING */
+    for (j = 0; j < ACCT_LAST; j++) {
+        size_t usage = tu->acct.cur_usage[j];
+        if (usage > 0) {
+            LOG(THREAD, LOG_HEAP|LOG_STATS, 1,
+                "WARNING: %s "SZFMT" bytes not freed!\n",
+                whichheap_name[j], tu->acct.cur_usage[j]);
+            
+# ifdef HOT_PATCHING_INTERFACE      /* known leaks for case 9593 */
+            if (DYNAMO_OPTION(hotp_only) &&
+                ((j == ACCT_SPECIAL && usage == (size_t)hotp_only_tramp_bytes_leaked) ||
+                 /* +4 is for the allocation's header; internal to heap mgt. */
+                 (j == ACCT_MEM_MGT &&
+                  usage == (size_t)(get_special_heap_header_size() + 4) &&
+                  hotp_only_tramp_bytes_leaked > 0)))
+                    continue;
+# endif
+            if (j != ACCT_TOMBSTONE /* known leak */ &&
+                INTERNAL_OPTION(heap_accounting_assert)) {
+                SYSLOG_INTERNAL_ERROR("memory leak: %s "SZFMT" bytes not freed",
+                                      whichheap_name[j], tu->acct.cur_usage[j]);
+                CLIENT_ASSERT(false && "potential memory leak",
+                              "memory leak detected");
+            }
+        }
+    }
+    if (tu != &heapmgt->global_units)
+        add_heapacct_to_global_stats(&tu->acct);
+
+    DOLOG(1, LOG_HEAP|LOG_STATS, {        
+        print_tu_heap_statistics(tu, THREAD,
+                                 dcontext == GLOBAL_DCONTEXT ? "Process" : "Thread");
+    });
+#endif  /* defined(DEBUG) && defined(HEAP_ACCOUNTING) */
+}
+
+void
+heap_thread_reset_init(dcontext_t *dcontext)
+{
+    thread_heap_t *th = (thread_heap_t *) dcontext->heap_field;
+    if (DYNAMO_OPTION(enable_reset)) {
+        ASSERT(th->nonpersistent_heap != NULL);
+        threadunits_init(dcontext, th->nonpersistent_heap, HEAP_UNIT_MIN_SIZE);
+    }
+}
+
+void
+heap_thread_init(dcontext_t *dcontext)
+{
+    thread_heap_t *th = (thread_heap_t *)
+        global_heap_alloc(sizeof(thread_heap_t) HEAPACCT(ACCT_MEM_MGT));
+    dcontext->heap_field = (void *) th;
+    th->local_heap = (thread_units_t *) global_heap_alloc(sizeof(thread_units_t)
+                                                       HEAPACCT(ACCT_MEM_MGT));
+    threadunits_init(dcontext, th->local_heap, HEAP_UNIT_MIN_SIZE);
+    if (DYNAMO_OPTION(enable_reset)) {
+        th->nonpersistent_heap = (thread_units_t *)
+            global_heap_alloc(sizeof(thread_units_t) HEAPACCT(ACCT_MEM_MGT));
+    } else
+        th->nonpersistent_heap = NULL;
+    heap_thread_reset_init(dcontext);
+}
+
+void
+heap_thread_reset_free(dcontext_t *dcontext)
+{
+    thread_heap_t *th = (thread_heap_t *) dcontext->heap_field;
+    if (DYNAMO_OPTION(enable_reset)) {
+        ASSERT(th->nonpersistent_heap != NULL);
+        /* FIXME: free directly rather than sending to dead list for
+         * heap_reset_free() to free!
+         * FIXME: for reset, don't free last unit so don't have to
+         * recreate in reset_init()
+         */
+        threadunits_exit(th->nonpersistent_heap, dcontext);
+    }
+}
+
+void
+heap_thread_exit(dcontext_t *dcontext)
+{
+    thread_heap_t *th = (thread_heap_t *) dcontext->heap_field;
+    threadunits_exit(th->local_heap, dcontext);
+    heap_thread_reset_free(dcontext);
+    global_heap_free(th->local_heap, sizeof(thread_units_t) HEAPACCT(ACCT_MEM_MGT));
+    if (DYNAMO_OPTION(enable_reset)) {
+        ASSERT(th->nonpersistent_heap != NULL);
+        global_heap_free(th->nonpersistent_heap, sizeof(thread_units_t)
+                         HEAPACCT(ACCT_MEM_MGT));
+    }
+    global_heap_free(th, sizeof(thread_heap_t) HEAPACCT(ACCT_MEM_MGT));
+}
+
+#if defined(DEBUG_MEMORY) && defined(DEBUG)
+void
+print_free_list(thread_units_t *tu, int i)
+{
+    void *p;
+    int len = 0;
+    dcontext_t *dcontext = tu->dcontext;
+    LOG(THREAD, LOG_HEAP, 1,
+        "Free list for size %d (== %d bytes):\n", i, BLOCK_SIZES[i]);
+    p = (void *) tu->free_list[i];
+    while (p != NULL) {
+        LOG(THREAD, LOG_HEAP, 1, "\tp = "PFX"\n", p);
+        len++;
+        p = *((char **)p);
+    }
+    LOG(THREAD, LOG_HEAP, 1, "Total length is %d\n", len);
+}
+#endif
+
+/* Used for both heap_unit_t and special_heap_unit_t.
+ * Returns the amount it increased the unit by, so caller should increment
+ * end_pc.
+ * Both end_pc and reserved_end_pc are assumed to be open-ended!
+ */
+static size_t
+common_heap_extend_commitment(heap_pc cur_pc, heap_pc end_pc, heap_pc reserved_end_pc,
+                              size_t size_need, uint prot)
+{
+    if (end_pc < reserved_end_pc) {
+        /* extend commitment if have more reserved */
+        size_t commit_size = DYNAMO_OPTION(heap_commit_increment);
+        if (cur_pc + size_need > end_pc + commit_size) {
+            commit_size = ALIGN_FORWARD(cur_pc + size_need - (ptr_uint_t)end_pc,
+                                        PAGE_SIZE);
+        }
+        if (end_pc + commit_size > reserved_end_pc) {
+            /* commit anyway before caller moves on to new unit so that
+             * we keep an invariant that all units but the current one
+             * are fully committed, so our algorithm for looking at the end
+             * of prior units holds
+             */
+            commit_size = reserved_end_pc - end_pc;
+        }
+        extend_commitment(end_pc, commit_size, prot, false /* extension */);
+#ifdef DEBUG_MEMORY
+        memset(end_pc, HEAP_UNALLOCATED_BYTE, commit_size);
+#endif
+        /* caller should do end_pc += commit_size */
+        STATS_ADD_PEAK(heap_capacity, commit_size);
+        /* FIXME: heap sizes are not always page-aligned so stats will be off */
+        STATS_SUB(heap_reserved_only, commit_size);
+        ASSERT(end_pc <= reserved_end_pc);
+        return commit_size;
+    } else
+        return 0;
+}
+
+static void
+heap_unit_extend_commitment(heap_unit_t *u, size_t size_need, uint prot)
+{
+    u->end_pc +=
+        common_heap_extend_commitment(u->cur_pc, u->end_pc, u->reserved_end_pc,
+                                      size_need, prot);
+}
+
+/* allocate storage on the DR heap
+ * returns NULL iff caller needs to grab dynamo_vm_areas_lock() and retry
+ */
+static void*
+common_heap_alloc(thread_units_t *tu, size_t size HEAPACCT(which_heap_t which))
+{
+    heap_unit_t *u = tu->cur_unit;
+    heap_pc p = NULL;
+    int bucket = 0;
+    size_t alloc_size, aligned_size;
+#if defined(DEBUG_MEMORY) && defined(DEBUG)
+    size_t check_alloc_size;
+    dcontext_t *dcontext = tu->dcontext;
+    ASSERT_CURIOSITY(which != ACCT_TOMBSTONE && 
+                     "Do you really need to use ACCT_TOMBSTONE? (potentially dangerous)");
+#endif
+    ASSERT(size > 0); /* we don't want to pay check cost in release */
+    ASSERT(size < MAX_VALID_HEAP_ALLOCATION && "potential integer overflow");
+    /* we prefer to crash than having heap overflows */
+    if (size > MAX_VALID_HEAP_ALLOCATION) {
+        /* This routine can currently accomodate without integer
+         * overflows sizes up to UINT_MAX - sizeof(heap_unit_t), but
+         * INT_MAX should be more than enough.
+         *
+         * Caller will likely crash, but that is better than a heap
+         * overflow, where a crash would be the best we can hope for.
+         */
+        return NULL;
+    }
+
+    /* NOTE - all of our buckets are sized to preserve alignment, so this can't change
+     * which bucket is used. */
+    aligned_size = ALIGN_FORWARD(size, HEAP_ALIGNMENT);
+    while (aligned_size > BLOCK_SIZES[bucket])
+        bucket++;
+    if (bucket == BLOCK_TYPES-1)
+        alloc_size = aligned_size + HEADER_SIZE;
+    else
+        alloc_size = BLOCK_SIZES[bucket];
+    ASSERT(size <= alloc_size);
+#ifdef DEBUG_MEMORY
+    /* case 10292: use original calculated size for later check */
+    check_alloc_size = alloc_size;
+#endif
+    if (alloc_size > MAXROOM) {
+        /* too big for normal unit, build a special unit just for this allocation */
+        /* don't need alloc_size or even aligned_size, just need size */
+        /* must create new since on free we assume exactly this size */
+        heap_unit_t *new_unit, *prev;
+
+        if (!safe_to_allocate_or_free_heap_units()) {
+            /* circular dependence solution: we need to hold DR lock before
+             * global alloc lock -- so we back out, grab it, and then come back
+             */
+            return NULL;
+        }
+            
+        ASSERT(size < size + sizeof(heap_unit_t) && "overflow");
+        new_unit = heap_create_unit(tu, size + sizeof(heap_unit_t), true/*must be new*/);
+        /* we want to commit the whole thing right away */
+        heap_unit_extend_commitment(new_unit, size, MEMPROT_READ|MEMPROT_WRITE);
+        prev = tu->top_unit;
+        alloc_size = size;
+        /* insert prior to cur unit (new unit will be full, so keep cur unit
+         * where it is)
+         */
+        while (prev != u && prev->next_local != u) {
+            ASSERT(prev != NULL && prev->next_local != NULL);
+            prev = prev->next_local;
+        }
+        if (prev == u) {
+            ASSERT(prev == tu->top_unit);
+            tu->top_unit = new_unit;
+        } else
+            prev->next_local = new_unit;
+        new_unit->next_local = u;
+#ifdef DEBUG_MEMORY
+        LOG(THREAD, LOG_HEAP, 3,
+            "\tCreating new oversized heap unit %d (%d [/%d] KB)\n",
+            new_unit->id, UNIT_COMMIT_SIZE(new_unit)/1024,
+            UNIT_RESERVED_SIZE(new_unit)/1024);
+#endif
+        p = new_unit->start_pc;
+        new_unit->cur_pc += size;
+        ACCOUNT_FOR_ALLOC(alloc_new, tu, which, size, size);
+        goto done_allocating;
+    }
+    if (tu->free_list[bucket] != NULL) {
+        if (bucket == BLOCK_TYPES-1) {
+            /* variable-length blocks, try to find one big enough */
+            size_t sz;
+            heap_pc next = tu->free_list[bucket];
+            heap_pc prev;
+            do {
+                prev = p;
+                p = next;
+                /* aligned_size is written right _before_ next pointer */
+                sz = VARIABLE_SIZE(next);
+                next = *((heap_pc*)p);
+            } while (aligned_size > sz && next != NULL);
+            if (aligned_size <= sz) {
+                ASSERT(ALIGNED(next, HEAP_ALIGNMENT));
+                /* found one, extract from free list */
+                if (p == tu->free_list[bucket])
+                    tu->free_list[bucket] = next;
+                else
+                    *((heap_pc *)prev) = next;
+#ifdef DEBUG_MEMORY
+                LOG(THREAD, LOG_HEAP, 2,
+                    "Variable-size block: allocating "PFX" (%d bytes [%d aligned] in "
+                    "%d block)\n", p, size, aligned_size, sz);
+                /* ensure memory we got from the free list is in a heap unit */
+                ASSERT(find_heap_unit(tu, p, sz) != NULL);
+#endif
+                ASSERT(ALIGNED(sz, HEAP_ALIGNMENT));
+                alloc_size = sz + HEADER_SIZE;
+                ACCOUNT_FOR_ALLOC(alloc_reuse, tu, which, alloc_size, aligned_size);
+            } else {
+                /* no free block big enough available */
+                p = NULL;
+            }
+        } else {
+            /* fixed-length free block available */
+            p = tu->free_list[bucket];
+            tu->free_list[bucket] = *((heap_pc *)p);
+            ASSERT(ALIGNED(tu->free_list[bucket], HEAP_ALIGNMENT));
+#ifdef DEBUG_MEMORY
+            /* ensure memory we got from the free list is in a heap unit */
+            ASSERT(find_heap_unit(tu, p, alloc_size) != NULL);
+#endif
+            ACCOUNT_FOR_ALLOC(alloc_reuse, tu, which, alloc_size, aligned_size);
+        }
+    }
+    if (p == NULL) {
+        /* no free blocks, grab a new one */
+        /* FIXME: if no more heap but lots of larger blocks available,
+         * should use the larger blocks instead of failing! */
+
+        /* see if room for allocation size */
+        ASSERT(ALIGNED(u->cur_pc, HEAP_ALIGNMENT));
+        ASSERT(ALIGNED(alloc_size, HEAP_ALIGNMENT));
+        if (u->cur_pc + alloc_size > u->end_pc) {
+            /* We either have to extend the current unit or, failing that,
+             * allocate a new unit. */
+            if (!safe_to_allocate_or_free_heap_units()) {
+                /* circular dependence solution: we need to hold dynamo areas
+                 * lock before global alloc lock in case we end up adding a new
+                 * unit or we hit oom (which may free units) while extending
+                 * the commmitment -- so we back out, grab it, and then come
+                 * back. */
+                return NULL;
+            }
+            /* try to extend if possible */
+            heap_unit_extend_commitment(u, alloc_size, MEMPROT_READ|MEMPROT_WRITE);
+            /* check again after extending commit */
+            if (u->cur_pc + alloc_size > u->end_pc) {
+                /* no room, look for room at end of previous units
+                 * FIXME: instead should put end of unit space on free list!
+                 */
+                heap_unit_t *prev = tu->top_unit;
+                while (1) {
+                    if (prev->cur_pc + alloc_size <= prev->end_pc) {
+                        tu->cur_unit = prev;
+                        u = prev;
+                        break;
+                    }
+                    if (prev->next_local == NULL) {
+                        /* no room anywhere, so create new unit 
+                         * double size of old unit (until hit max size)
+                         */
+                        heap_unit_t *new_unit;
+                        size_t unit_size;
+
+                        unit_size = UNITALLOC(u) * 2;
+                        while (unit_size < alloc_size + UNITOVERHEAD)
+                            unit_size *= 2;
+                        if (unit_size > HEAP_UNIT_MAX_SIZE)
+                            unit_size = HEAP_UNIT_MAX_SIZE;
+                        /* size for heap_create_unit doesn't include any guard 
+                         * pages */
+                        ASSERT(unit_size > UNITOVERHEAD);
+                        unit_size -= GUARD_PAGE_ADJUSTMENT;
+                        new_unit = heap_create_unit(tu, unit_size, false/*can reuse*/);
+                        prev->next_local = new_unit;
+#ifdef DEBUG_MEMORY
+                        LOG(THREAD, LOG_HEAP, 2,
+                            "\tCreating new heap unit %d (%d [/%d] KB)\n",
+                            new_unit->id, UNIT_COMMIT_SIZE(new_unit)/1024,
+                            UNIT_RESERVED_SIZE(new_unit)/1024);
+#endif
+                        /* use new unit for all future non-free-list allocations 
+                         * we'll try to use the free room at the end of the old unit(s)
+                         * only when we next run out of room
+                         */
+                        tu->cur_unit = new_unit;
+                        u = new_unit;
+                        /* may need to extend now if alloc_size is large */
+                        heap_unit_extend_commitment(u, alloc_size,
+                                                    MEMPROT_READ|MEMPROT_WRITE);
+                        /* otherwise would have been bigger than MAXROOM */
+                        ASSERT(alloc_size <= (ptr_uint_t) (u->end_pc - u->cur_pc));
+                        break;
+                    }
+                    prev = prev->next_local;
+                }
+            }
+        }
+
+        p = u->cur_pc;
+        if (bucket == BLOCK_TYPES-1) {
+            /* we keep HEADER_SIZE bytes to store the size */
+            p += HEADER_SIZE;
+            VARIABLE_SIZE(p) = aligned_size;
+        }
+        u->cur_pc += alloc_size;
+
+        ACCOUNT_FOR_ALLOC(alloc_new, tu, which, alloc_size, aligned_size);
+    }
+    DOSTATS({
+        /* do this before done_allocating: want to ignore special-unit allocs */
+        ATOMIC_ADD(int, block_count[bucket], 1);
+        ATOMIC_ADD(int, block_total_count[bucket], 1);
+        /* FIXME: should atomically store inc-ed val in temp to avoid races w/ max */
+        ATOMIC_MAX(int, block_peak_count[bucket], block_count[bucket]);
+        ASSERT(CHECK_TRUNCATE_TYPE_uint(alloc_size - aligned_size));
+        ATOMIC_ADD(int, block_wasted[bucket], (int) (alloc_size - aligned_size));
+        /* FIXME: should atomically store val in temp to avoid races w/ max */
+        ATOMIC_MAX(int, block_peak_wasted[bucket], block_wasted[bucket]);
+        if (aligned_size > size) {
+            ASSERT(CHECK_TRUNCATE_TYPE_uint(aligned_size - size));
+            ATOMIC_ADD(int, block_align_pad[bucket], (int) (aligned_size - size));
+            /* FIXME: should atomically store val in temp to avoid races w/ max */
+            ATOMIC_MAX(int, block_peak_align_pad[bucket], block_align_pad[bucket]);
+            STATS_ADD_PEAK(heap_align, aligned_size - size);
+            LOG(GLOBAL, LOG_STATS, 5,
+                "alignment mismatch: %s ask %d, aligned is %d -> %d pad\n",
+                IF_HEAPACCT_ELSE(whichheap_name[which], ""),
+                size, aligned_size, aligned_size-size);
+        }
+        if (bucket == BLOCK_TYPES-1) {
+            STATS_ADD(heap_headers, HEADER_SIZE);
+            STATS_INC(heap_allocs_variable);
+        } else {
+            STATS_INC(heap_allocs_buckets);
+            if (alloc_size > aligned_size) {
+                STATS_ADD_PEAK(heap_bucket_pad, alloc_size - aligned_size);
+                LOG(GLOBAL, LOG_STATS, 5,
+                    "bucket mismatch: %s ask (aligned) %d, got %d, -> %d\n",
+                    IF_HEAPACCT_ELSE(whichheap_name[which], ""),
+                    aligned_size, alloc_size, alloc_size-aligned_size);
+            }
+        }
+    });
+ done_allocating:
+#ifdef DEBUG_MEMORY
+    if (bucket == BLOCK_TYPES-1 && check_alloc_size <= MAXROOM) {
+        /* verify is unallocated memory, skip possible free list next pointer */
+        CLIENT_ASSERT(is_region_memset_to_char
+                      (p+sizeof(heap_pc *), (alloc_size-HEADER_SIZE)-sizeof(heap_pc *),
+                       HEAP_UNALLOCATED_BYTE), "memory corruption detected");
+        LOG(THREAD, LOG_HEAP, 6,
+            "\nalloc var "PFX"-"PFX" %d bytes, ret "PFX"-"PFX" %d bytes\n",
+            p-HEADER_SIZE, p-HEADER_SIZE+alloc_size, alloc_size, p, p+size, size);
+        /* there can only be extra padding if we took off of the free list */
+        memset(p+size, HEAP_PAD_BYTE, (alloc_size-HEADER_SIZE)-size);
+    } else {
+        /* verify is unallocated memory, skip possible free list next pointer */
+        CLIENT_ASSERT(is_region_memset_to_char
+                      (p+sizeof(heap_pc *), alloc_size-sizeof(heap_pc *), 
+                       HEAP_UNALLOCATED_BYTE), "memory corruption detected");
+        LOG(THREAD, LOG_HEAP, 6,
+            "\nalloc fix or oversize "PFX"-"PFX" %d bytes, ret "PFX"-"PFX" %d bytes\n",
+            p, p+alloc_size, alloc_size, p, p+size, size);
+        memset(p+size, HEAP_PAD_BYTE, alloc_size-size);
+    }
+    memset(p, HEAP_ALLOCATED_BYTE, size);
+# ifdef HEAP_ACCOUNTING
+    LOG(THREAD, LOG_HEAP, 6, "\t%s\n", whichheap_name[which]);
+# endif
+#endif
+    return (void*)p;
+}
+
+
+/* allocate storage on the app's heap */
+void*
+heap_alloc(dcontext_t *dcontext, size_t size HEAPACCT(which_heap_t which))
+{
+    thread_units_t *tu;
+    void *ret_val;
+    if (dcontext == GLOBAL_DCONTEXT)
+        return global_heap_alloc(size HEAPACCT(which));
+    tu = ((thread_heap_t *) dcontext->heap_field)->local_heap;
+#if defined(CLIENT_SIDELINE) && defined(CLIENT_INTERFACE)
+    mutex_lock(&dcontext->sideline_heap_lock);
+#endif
+    ret_val = common_heap_alloc(tu, size HEAPACCT(which));
+#if defined(CLIENT_SIDELINE) && defined(CLIENT_INTERFACE)
+    mutex_unlock(&dcontext->sideline_heap_lock);
+#endif
+    ASSERT(ret_val != NULL);
+    return ret_val;
+}
+
+/* free heap storage
+ * returns false if caller needs to grab dynamo_vm_areas_lock() and retry
+ */
+static bool
+common_heap_free(thread_units_t *tu, void *p_void, size_t size HEAPACCT(which_heap_t which))
+{
+    int bucket = 0;
+    heap_pc p = (heap_pc) p_void;
+#if defined(DEBUG_MEMORY) || defined(HEAP_ACCOUNTING)
+    DEBUG_DECLARE(dcontext_t *dcontext = tu->dcontext;)
+#endif
+    size_t alloc_size, aligned_size = ALIGN_FORWARD(size, HEAP_ALIGNMENT);
+    ASSERT(size > 0); /* we don't want to pay check cost in release */
+    ASSERT(p != NULL);
+#ifdef DEBUG_MEMORY
+    /* FIXME: this is risky...will we ever see data that's first or last 
+     * word == HEAP_UNALLOCATED_UINT?
+     * In fact this does happen in case 8802: app's eax is 0xcdcdcdcd (from an
+     * app dbg memset) and we have dcontext->allocated_start==dcontext.  We
+     * relax to allow one of the first two words to match.
+     */
+    ASSERT((*(uint *)p != HEAP_UNALLOCATED_UINT ||
+            (size >= 2*sizeof(uint) && *(((uint *)p)+1) != HEAP_UNALLOCATED_UINT)) && 
+           *(uint *)(p+size-sizeof(int)) != HEAP_UNALLOCATED_UINT);
+#endif
+
+    while (aligned_size > BLOCK_SIZES[bucket])
+        bucket++;
+    if (bucket == BLOCK_TYPES-1)
+        alloc_size = aligned_size + HEADER_SIZE;
+    else
+        alloc_size = BLOCK_SIZES[bucket];
+
+    if (alloc_size > MAXROOM) {
+        /* we must have used a special unit just for this allocation */
+        heap_unit_t *u = tu->top_unit, *prev = NULL;
+
+#ifdef DEBUG_MEMORY
+        /* ensure we are freeing memory in a proper unit */
+        ASSERT(find_heap_unit(tu, p, size) != NULL);
+#endif
+
+        if (!safe_to_allocate_or_free_heap_units()) {
+            /* circular dependence solution: we need to hold DR lock before
+             * global alloc lock -- so we back out, grab it, and then come back
+             */
+            return false;
+        }
+
+        while (u != NULL && u->start_pc != p) {
+            prev = u;
+            u = u->next_local;
+        }
+        ASSERT(u != NULL);
+        /* remove this unit from this thread's list, move to dead list
+         * for future use -- no problems will be caused by it being
+         * larger than normal
+         */
+        if (prev == NULL)
+            tu->top_unit = u->next_local;
+        else
+            prev->next_local = u->next_local;
+        /* just retire the unit # */
+#ifdef DEBUG_MEMORY
+        LOG(THREAD, LOG_HEAP, 3, "\tFreeing oversized heap unit %d (%d KB)\n",
+            u->id, size);
+        /* go ahead and set unallocated, even though we are just going to free
+         * the unit, is needed for an assert in heap_free_unit anyways */
+        memset(p, HEAP_UNALLOCATED_BYTE, size);
+#endif
+        ASSERT(size == UNITROOM(u));
+        heap_free_unit(u, tu->dcontext);
+        ACCOUNT_FOR_FREE(tu, which, size);
+        return true;
+    } else if (bucket == BLOCK_TYPES-1) {
+        ASSERT(GET_VARIABLE_ALLOCATION_SIZE(p) >= alloc_size);
+        alloc_size = GET_VARIABLE_ALLOCATION_SIZE(p);
+        ASSERT(alloc_size - HEADER_SIZE >= aligned_size);
+    }
+
+#if defined(DEBUG) || defined(DEBUG_MEMORY) || defined(HEAP_ACCOUNTING)
+    if (bucket == BLOCK_TYPES-1) {
+# ifdef DEBUG_MEMORY
+        LOG(THREAD, LOG_HEAP, 6,
+            "\nfree var "PFX"-"PFX" %d bytes, asked "PFX"-"PFX" %d bytes\n",
+            p-HEADER_SIZE, p-HEADER_SIZE+alloc_size, alloc_size, p, p+size, size);
+        ASSERT(is_region_memset_to_char(p+size, (alloc_size-HEADER_SIZE)-size,
+                                        HEAP_PAD_BYTE));
+        /* ensure we are freeing memory in a proper unit */
+        ASSERT(find_heap_unit(tu, p, alloc_size - HEADER_SIZE) != NULL);
+        /* set used and padding memory back to unallocated */
+        memset(p, HEAP_UNALLOCATED_BYTE, alloc_size-HEADER_SIZE);
+# endif
+        STATS_SUB(heap_headers, HEADER_SIZE);
+    } else {
+# ifdef DEBUG_MEMORY
+        LOG(THREAD, LOG_HEAP, 6,
+            "\nfree fix "PFX"-"PFX" %d bytes, asked "PFX"-"PFX" %d bytes\n",
+            p, p+alloc_size, alloc_size, p, p+size, size);
+        ASSERT(is_region_memset_to_char(p+size, alloc_size-size,
+                                        HEAP_PAD_BYTE));
+        /* ensure we are freeing memory in a proper unit */
+        ASSERT(find_heap_unit(tu, p, alloc_size) != NULL);
+        /* set used and padding memory back to unallocated */
+        memset(p, HEAP_UNALLOCATED_BYTE, alloc_size);
+# endif
+        STATS_SUB(heap_bucket_pad, (alloc_size - aligned_size));
+    }
+    STATS_SUB(heap_align, (aligned_size - size));
+    DOSTATS({
+        ATOMIC_ADD(int, block_count[bucket], -1);
+        ATOMIC_ADD(int, block_wasted[bucket], -(int)(alloc_size - aligned_size));
+        ATOMIC_ADD(int, block_align_pad[bucket], -(int)(aligned_size - size));
+    });
+# ifdef HEAP_ACCOUNTING
+    LOG(THREAD, LOG_HEAP, 6, "\t%s\n", whichheap_name[which]);
+    ACCOUNT_FOR_FREE(tu, which, alloc_size);
+# endif
+#endif
+
+    /* write next pointer */
+    *((heap_pc*)p) = tu->free_list[bucket];
+    ASSERT(ALIGNED(tu->free_list[bucket], HEAP_ALIGNMENT));
+    tu->free_list[bucket] = p;
+    ASSERT(ALIGNED(tu->free_list[bucket], HEAP_ALIGNMENT));
+    return true;
+}
+
+/* free heap storage */
+void
+heap_free(dcontext_t *dcontext, void *p, size_t size HEAPACCT(which_heap_t which))
+{
+    thread_units_t *tu;
+    DEBUG_DECLARE(bool ok;)
+    if (dcontext == GLOBAL_DCONTEXT) {
+        global_heap_free(p, size HEAPACCT(which));
+        return;
+    }
+    tu = ((thread_heap_t *) dcontext->heap_field)->local_heap;
+#if defined(CLIENT_SIDELINE) && defined(CLIENT_INTERFACE)
+    mutex_lock(&dcontext->sideline_heap_lock);
+#endif
+    DEBUG_DECLARE(ok = ) common_heap_free(tu, p, size HEAPACCT(which));
+    ASSERT(ok);
+#if defined(CLIENT_SIDELINE) && defined(CLIENT_INTERFACE)
+    mutex_unlock(&dcontext->sideline_heap_lock);
+#endif
+}
+
+bool local_heap_protected(dcontext_t *dcontext)
+{
+    thread_heap_t *th = (thread_heap_t *) dcontext->heap_field;
+    return (!th->local_heap->writable ||
+            (th->nonpersistent_heap != NULL && !th->nonpersistent_heap->writable));
+}
+
+static inline void
+protect_local_units_helper(heap_unit_t *u, bool writable)
+{
+    /* win32 does not allow single protection change call on units that
+     * were allocated with separate calls so we don't try to combine
+     * adjacent units here
+     */
+    while (u != NULL) {
+        change_protection(UNIT_ALLOC_START(u), UNIT_COMMIT_SIZE(u), writable);
+        u = u->next_local;
+    }
+}
+
+static void
+protect_threadunits(thread_units_t *tu, bool writable)
+{
+    ASSERT(TEST(SELFPROT_LOCAL, dynamo_options.protect_mask));
+    if (tu->writable == writable)
+        return;
+    protect_local_units_helper(tu->top_unit, writable);
+    tu->writable = writable;
+}
+
+void
+protect_local_heap(dcontext_t *dcontext, bool writable)
+{
+    thread_heap_t *th = (thread_heap_t *) dcontext->heap_field;
+    protect_threadunits(th->local_heap, writable);
+    if (DYNAMO_OPTION(enable_reset))
+        protect_threadunits(th->nonpersistent_heap, writable);
+}
+
+/* assumption: vmm_heap_alloc only gets called for HeapUnits themselves, which
+ * are protected by us here, so ignore os heap
+ */
+void
+protect_global_heap(bool writable)
+{
+    ASSERT(TEST(SELFPROT_GLOBAL, dynamo_options.protect_mask));
+
+    acquire_recursive_lock(&global_alloc_lock);
+
+    if (heapmgt->global_heap_writable == writable) {
+        release_recursive_lock(&global_alloc_lock);
+        return;
+    }
+    /* win32 does not allow single protection change call on units that
+     * were allocated with separate calls so we don't try to combine
+     * adjacent units here
+
+     * FIXME: That may no longer be true for our virtual memory manager that 
+     * will in fact be allocated as a single unit.  It is only in case
+     * we have run out of that initial allocation that we may have to 
+     * keep a separate list of allocations.
+     */
+
+    if (!writable) {
+        ASSERT(heapmgt->global_heap_writable);
+        heapmgt->global_heap_writable = writable;
+    }
+
+    protect_local_units_helper(heapmgt->global_units.top_unit, writable);
+    if (DYNAMO_OPTION(enable_reset)) {
+        protect_local_units_helper(heapmgt->global_nonpersistent_units.top_unit,
+                                   writable);
+    }
+
+    if (writable) {
+        ASSERT(!heapmgt->global_heap_writable);
+        heapmgt->global_heap_writable = writable;
+    }
+
+    release_recursive_lock(&global_alloc_lock);
+}
+
+/* FIXME: share some code...right now these are identical to protected
+ * versions except the unit used
+ */
+void * 
+global_unprotected_heap_alloc(size_t size HEAPACCT(which_heap_t which))
+{
+    void *p = common_global_heap_alloc(&heapmgt->global_unprotected_units,
+                                       size HEAPACCT(which));
+    ASSERT(p != NULL);
+    LOG(GLOBAL, LOG_HEAP, 6, "\nglobal unprotected alloc: "PFX" (%d bytes)\n", p, size);
+    return p;
+}
+
+void
+global_unprotected_heap_free(void *p, size_t size HEAPACCT(which_heap_t which))
+{
+    common_global_heap_free(&heapmgt->global_unprotected_units,
+                            p, size HEAPACCT(which));
+    LOG(GLOBAL, LOG_HEAP, 6, "\nglobal unprotected free: "PFX" (%d bytes)\n", p, size);
+}
+
+void *
+nonpersistent_heap_alloc(dcontext_t *dcontext, size_t size HEAPACCT(which_heap_t which))
+{
+    void *p;
+    if (DYNAMO_OPTION(enable_reset)) {
+        if (dcontext == GLOBAL_DCONTEXT) {
+            p = common_global_heap_alloc(&heapmgt->global_nonpersistent_units,
+                                         size HEAPACCT(which));
+            LOG(GLOBAL, LOG_HEAP, 6,
+                "\nglobal nonpersistent alloc: "PFX" (%d bytes)\n", p, size);
+        } else {
+            thread_units_t *nph = ((thread_heap_t *) dcontext->heap_field)->nonpersistent_heap;
+            p = common_heap_alloc(nph, size HEAPACCT(which));
+        }
+    } else {
+        p = heap_alloc(dcontext, size HEAPACCT(which));
+    }
+    ASSERT(p != NULL);
+    return p;
+}
+
+void
+nonpersistent_heap_free(dcontext_t *dcontext, void *p, size_t size
+                        HEAPACCT(which_heap_t which))
+{
+    if (DYNAMO_OPTION(enable_reset)) {
+        if (dcontext == GLOBAL_DCONTEXT) {
+            common_global_heap_free(&heapmgt->global_nonpersistent_units,
+                                    p, size HEAPACCT(which));
+            LOG(GLOBAL, LOG_HEAP, 6,
+                "\nglobal nonpersistent free: "PFX" (%d bytes)\n", p, size);
+        } else {
+            thread_units_t *nph = ((thread_heap_t *) dcontext->heap_field)->nonpersistent_heap;
+            DEBUG_DECLARE(bool ok =) common_heap_free(nph, p, size HEAPACCT(which));
+            ASSERT(ok);
+        }
+    } else {
+        heap_free(dcontext, p, size HEAPACCT(which));
+    }
+}
+
+/****************************************************************************
+ * SPECIAL SINGLE-ALLOC-SIZE HEAP SERVICE
+ */
+
+/* Assumptions:
+ *   All allocations are of a single block size
+ *   If use_lock is false, no synchronization is needed or even safe
+ */
+
+/* We use our own unit struct to give us flexibility.
+ * 1) We don't always allocate the header inline.
+ * 2) We are sometimes executed from and so need pc prof support.
+ * 3) We don't need all the fields of heap_unit_t.
+ */
+typedef struct _special_heap_unit_t {
+    heap_pc alloc_pc;         /* start of allocation region */
+    heap_pc start_pc;         /* first address we'll give out for storage */
+    heap_pc end_pc;           /* open-ended address of heap storage */
+    heap_pc cur_pc;           /* current end (open) of allocated storage */
+    heap_pc reserved_end_pc;  /* (open) end of reserved (not nec committed) memory */
+#ifdef WINDOWS_PC_SAMPLE
+    profile_t *profile;
+#endif
+#ifdef DEBUG
+    int      id;              /* # of this unit */
+#endif
+    struct _special_heap_unit_t *next;
+} special_heap_unit_t;
+
+#define SPECIAL_UNIT_COMMIT_SIZE(u) ((u)->end_pc - (u)->alloc_pc)
+#define SPECIAL_UNIT_RESERVED_SIZE(u) ((u)->reserved_end_pc - (u)->alloc_pc)
+#define SPECIAL_UNIT_HEADER_INLINE(u) ((u)->alloc_pc != (u)->start_pc)
+
+/* the cfree list stores a next ptr and a count */
+typedef struct _cfree_header {
+    struct _cfree_header *next_cfree;
+    uint count;
+} cfree_header_t;
+
+typedef struct _special_units_t {
+    special_heap_unit_t *top_unit; /* start of linked list of heap units */
+    special_heap_unit_t *cur_unit; /* current unit in heap list */
+    uint block_size;    /* all blocks are this size */
+    heap_pc free_list;
+    cfree_header_t *cfree_list;
+#ifdef DEBUG
+    int num_units;      /* total # of heap units */
+#endif
+    bool writable:1;      /* remember state of heap protection */
+    bool executable:1;
+    /* if use_lock is false, grabbing _any_ lock may be hazardous!
+     * (this isn't just an optimization, it's for correctness)
+     */
+    bool use_lock:1;
+    bool in_iterator:1;
+    bool persistent:1;
+    mutex_t lock;
+
+    /* Yet another feature added: pclookup, but across multiple heaps,
+     * so it's via a passed-in vector and passed-in data
+     */
+    vm_area_vector_t *heap_areas;
+    void *lookup_retval;
+
+#ifdef WINDOWS_PC_SAMPLE
+    struct _special_units_t *next;
+#endif
+#ifdef HEAP_ACCOUNTING
+    /* we only need one bucket for SpecialHeap but to re-use code we waste space */
+    heap_acct_t acct;
+#endif
+} special_units_t;
+
+#if defined(WINDOWS_PC_SAMPLE) && !defined(DEBUG)
+/* For fast exit path we need a quick way to walk all the units */
+DECLARE_CXTSWPROT_VAR(static mutex_t special_units_list_lock,
+                      INIT_LOCK_FREE(special_units_list_lock));
+/* This is only used for profiling so we don't bother to protect it */
+DECLARE_CXTSWPROT_VAR(static special_units_t *special_units_list, NULL);
+#endif
+
+#if defined(DEBUG) && defined(HEAP_ACCOUNTING) && defined(HOT_PATCHING_INTERFACE)
+/* To get around the problem of the special_units_t "module" being defined after
+ * the heap module in the same file.  Part of fix for case 9593 that required
+ * leaking memory.
+ */
+static int
+get_special_heap_header_size(void)
+{
+    return sizeof(special_units_t);
+}
+#endif
+
+#ifdef WINDOWS_PC_SAMPLE
+static inline bool special_heap_profile_enabled()
+{
+    return (dynamo_options.profile_pcs && 
+            dynamo_options.prof_pcs_stubs >= 2 && 
+            dynamo_options.prof_pcs_stubs <= 32);
+}
+#endif
+
+static inline uint
+get_prot(special_units_t *su)
+{
+    return (su->executable ?
+            MEMPROT_READ|MEMPROT_WRITE|MEMPROT_EXEC :
+            MEMPROT_READ|MEMPROT_WRITE);
+}
+
+static void
+special_unit_extend_commitment(special_heap_unit_t *u, size_t size_need, uint prot)
+{
+    u->end_pc +=
+        common_heap_extend_commitment(u->cur_pc, u->end_pc, u->reserved_end_pc,
+                                      size_need, prot);
+}
+
+/* If pc is NULL, allocates memory and stores the header inside it;
+ * If pc is non-NULL, allocates separate memory for the header, and
+ * uses pc for the heap region (assuming size is fully committed).
+ * unit_full only applies to the non-NULL case, indicating whether
+ * to continue to allocate from this unit.
+ */
+static special_heap_unit_t *
+special_heap_create_unit(special_units_t *su, byte *pc, size_t size, bool unit_full)
+{
+    special_heap_unit_t *u;
+    size_t commit_size;
+    uint prot = get_prot(su);
+    ASSERT_OWN_MUTEX(su->use_lock, &su->lock);
+
+    if (pc != NULL) {
+        u = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, special_heap_unit_t, ACCT_MEM_MGT, PROTECTED);
+        ASSERT(u != NULL);
+        u->start_pc = pc;
+        u->alloc_pc = pc;
+        commit_size = size;
+        /* caller should arrange alignment */
+        ASSERT(!IS_POWER_OF_2(su->block_size) || ALIGNED(u->start_pc, su->block_size));
+    } else {
+        commit_size = DYNAMO_OPTION(heap_commit_increment);
+        ASSERT(commit_size <= size);
+        /* create new unit */
+        u = (special_heap_unit_t *) get_guarded_real_memory(size, commit_size, prot,
+                                                            true, true
+                                                            _IF_DEBUG("special_heap"));
+        ASSERT(u != NULL);
+        u->alloc_pc = (heap_pc) u;
+        /* u is kept at top of unit itself, so displace start pc */
+        u->start_pc = (heap_pc) (((ptr_uint_t)u) + sizeof(special_heap_unit_t));
+        /* some users expect alignment; not much of a space loss for those who don't */
+        if (IS_POWER_OF_2(su->block_size)) {
+            STATS_ADD(heap_special_align,
+                      ALIGN_FORWARD(u->start_pc, su->block_size) -
+                      (ptr_uint_t)u->start_pc);
+            u->start_pc = (heap_pc) ALIGN_FORWARD(u->start_pc, su->block_size);
+        }
+    }
+    u->end_pc = u->alloc_pc + commit_size;
+    u->reserved_end_pc = u->alloc_pc + size;
+    if (pc != NULL && unit_full) {
+        ASSERT(u->reserved_end_pc == u->end_pc);
+        u->cur_pc = u->end_pc;
+    } else
+        u->cur_pc = u->start_pc;
+    u->next = NULL;
+    DODEBUG({
+        u->id = su->num_units;
+        su->num_units++;
+    });
+
+#ifdef WINDOWS_PC_SAMPLE
+    if (special_heap_profile_enabled()) {
+        u->profile = create_profile((app_pc)PAGE_START(u->start_pc), u->reserved_end_pc, 
+                                    dynamo_options.prof_pcs_stubs, NULL);
+        start_profile(u->profile);
+    } else 
+        u->profile = NULL;
+#endif
+
+    /* N.B.: if STATS macros ever change to grab a mutex, we could deadlock
+     * if !su->use_lock!
+     */
+    STATS_ADD_PEAK(heap_capacity, commit_size);
+    STATS_ADD_PEAK(heap_special_capacity, commit_size);
+    STATS_ADD_PEAK(heap_special_units, 1);
+    STATS_ADD_PEAK(heap_reserved_only, (u->reserved_end_pc - u->end_pc));
+
+    if (su->heap_areas != NULL) {
+        vmvector_add(su->heap_areas, u->alloc_pc, u->reserved_end_pc, su->lookup_retval);
+    }
+
+#ifdef DEBUG_MEMORY
+    /* Don't clobber already-allocated memory */
+    if (pc == NULL)
+        memset(u->start_pc, HEAP_UNALLOCATED_BYTE, u->end_pc - u->start_pc);
+#endif
+    return u;
+}
+
+/* caller must store the special_units_t *, which is opaque */
+static void *
+special_heap_init_internal(uint block_size, bool use_lock, bool executable,
+                           bool persistent, vm_area_vector_t *vector, void *vector_data,
+                           byte *heap_region, size_t heap_size, bool unit_full)
+{
+    special_units_t *su;
+    size_t unit_size = (block_size * 16 > HEAP_UNIT_MIN_SIZE) ?
+        (block_size * 16) : HEAP_UNIT_MIN_SIZE;
+    su = (special_units_t *)
+        (persistent ? global_heap_alloc(sizeof(special_units_t) HEAPACCT(ACCT_MEM_MGT)) :
+         nonpersistent_heap_alloc(GLOBAL_DCONTEXT, sizeof(special_units_t)
+                                  HEAPACCT(ACCT_MEM_MGT)));
+    ASSERT(block_size >= sizeof(heap_pc *) && "need room for free list ptrs");
+    ASSERT(block_size >= sizeof(heap_pc *) + sizeof(uint) &&
+           "need room for cfree list ptrs");
+    su->block_size = block_size;
+    su->executable = executable;
+    su->persistent = persistent;
+    su->writable = true;
+    su->free_list = NULL;
+    su->cfree_list = NULL;
+    DODEBUG({ 
+        su->num_units = 0;
+    });
+    ASSERT((vector == NULL) == (vector_data == NULL));
+    su->heap_areas = vector;
+    su->lookup_retval = vector_data;
+    su->in_iterator = false;
+    if (use_lock)
+        ASSIGN_INIT_LOCK_FREE(su->lock, special_heap_lock);
+    /* For persistent cache loading we hold executable_areas lock and so
+     * cannot acquire special_heap_lock -- so we do not acquire
+     * for the initial unit creation, which is safe since su is still
+     * private to this routine.
+     */
+    su->use_lock = false; /* we set to real value below */
+    su->top_unit = special_heap_create_unit(su, heap_region,
+                                            heap_size == 0 ? unit_size : heap_size,
+                                            unit_full);
+    su->use_lock = use_lock;
+#ifdef HEAP_ACCOUNTING
+    memset(&su->acct, 0, sizeof(su->acct));
+#endif
+    su->cur_unit = su->top_unit;
+
+#if defined(WINDOWS_PC_SAMPLE) && !defined(DEBUG)
+    if (special_heap_profile_enabled()) {
+        /* Add to the global master list, which requires a lock */
+        mutex_lock(&special_units_list_lock); 
+        su->next = special_units_list;
+        special_units_list = su;
+        mutex_unlock(&special_units_list_lock); 
+    }
+#endif
+
+    return su;
+}
+
+/* Typical usage */
+void *
+special_heap_init(uint block_size, bool use_lock, bool executable, bool persistent)
+{
+    return special_heap_init_internal(block_size, use_lock, executable,
+                                      persistent, NULL, NULL, NULL, 0, false);
+}
+
+/* Special heap w/ a vector for lookups.  Also supports a pre-created heap region
+ * (heap_region, heap_region+heap_size) whose fullness is unit_full. */
+void *
+special_heap_pclookup_init(uint block_size, bool use_lock, bool executable,
+                           bool persistent, vm_area_vector_t *vector, void *vector_data,
+                           byte *heap_region, size_t heap_size, bool unit_full)
+{
+    return special_heap_init_internal(block_size, use_lock, executable, persistent,
+                                      vector, vector_data, heap_region, heap_size,
+                                      unit_full);
+}
+
+/* Sets the vector data for the lookup vector used by the special heap */
+void
+special_heap_set_vector_data(void *special, void *vector_data)
+{
+    special_units_t *su = (special_units_t *) special;
+    special_heap_unit_t *u;
+    ASSERT(su->heap_areas != NULL);
+    /* FIXME: more efficient to walk the vector, but no interface
+     * to set the data: we'd need to expose the iterator index or
+     * the vmarea struct rather than than the clean copy we have now
+     */
+    for (u = su->top_unit; u != NULL; u = u->next) {
+        vmvector_modify_data(su->heap_areas, u->alloc_pc,
+                             u->reserved_end_pc, vector_data);
+    }
+}
+
+/* Returns false if the special heap has more than one unit or has a
+ * non-externally-allocated unit.
+ * Sets the cur pc for the only unit to end_pc.
+ */
+bool
+special_heap_set_unit_end(void *special, byte *end_pc)
+{
+    special_units_t *su = (special_units_t *) special;
+    if (su->top_unit->next != NULL ||
+        SPECIAL_UNIT_HEADER_INLINE(su->top_unit) ||
+        end_pc < su->top_unit->start_pc ||
+        end_pc > su->top_unit->end_pc)
+        return false;
+    su->top_unit->cur_pc = end_pc;
+    return true;
+}
+
+#ifdef WINDOWS_PC_SAMPLE
+static void
+special_heap_profile_stop(special_heap_unit_t *u)
+{
+    int sum;
+    ASSERT(special_heap_profile_enabled());
+    stop_profile(u->profile);
+    sum = sum_profile(u->profile);
+    if (sum > 0) {
+        mutex_lock(&profile_dump_lock);
+        print_file(profile_file,
+                   "\nDumping special heap unit profile\n%d hits\n", sum);
+        dump_profile(profile_file, u->profile);
+        mutex_unlock(&profile_dump_lock);
+    }
+}
+#endif
+
+#if defined(WINDOWS_PC_SAMPLE) && !defined(DEBUG)
+/* for fast exit path only, normal path taken care of */
+void
+special_heap_profile_exit()
+{
+    special_heap_unit_t *u;
+    special_units_t *su;
+    ASSERT(special_heap_profile_enabled()); /* will never be compiled in I guess :) */
+    mutex_lock(&special_units_list_lock); 
+    for (su = special_units_list; su != NULL; su = su->next) {
+        if (su->use_lock)
+            mutex_lock(&su->lock);
+        for (u = su->top_unit; u != NULL; u = u->next) {
+            if (u->profile != NULL)
+                special_heap_profile_stop(u);
+            /* fast exit path: do not bother to free */
+        }
+        if (su->use_lock)
+            mutex_unlock(&su->lock);
+    }
+    mutex_unlock(&special_units_list_lock);
+}
+#endif
+
+void
+special_heap_exit(void *special)
+{
+    special_units_t *su = (special_units_t *) special;
+    special_heap_unit_t *u, *next_u;
+#ifdef DEBUG
+    size_t total_heap_used = 0;
+#endif
+    u = su->top_unit;
+    while (u != NULL) {
+        /* Assumption: it's ok to use print_lock even if !su->use_lock */
+        DOLOG(1, LOG_HEAP|LOG_STATS, {
+            size_t num_used = u->cur_pc - u->start_pc;
+            total_heap_used += num_used;
+            LOG(THREAD_GET, LOG_HEAP|LOG_STATS, 1,
+                "Heap unit "SZFMT" (size "SZFMT" [/"SZFMT"] KB): used "SZFMT" KB\n",
+                u->id, (SPECIAL_UNIT_COMMIT_SIZE(u))/1024,
+                SPECIAL_UNIT_RESERVED_SIZE(u)/1024, num_used/1024);
+        });
+        next_u = u->next;
+#ifdef WINDOWS_PC_SAMPLE
+        if (u->profile != NULL) {
+            ASSERT(special_heap_profile_enabled());
+            special_heap_profile_stop(u);
+            free_profile(u->profile);
+            u->profile = NULL;
+        }
+#endif
+        STATS_ADD(heap_special_units, -1);
+        STATS_SUB(heap_special_capacity, SPECIAL_UNIT_COMMIT_SIZE(u));
+        if (su->heap_areas != NULL) {
+            vmvector_remove(su->heap_areas, u->alloc_pc, u->reserved_end_pc);
+        }
+        if (!SPECIAL_UNIT_HEADER_INLINE(u)) {
+            HEAP_TYPE_FREE(GLOBAL_DCONTEXT, u, special_heap_unit_t,
+                           ACCT_MEM_MGT, PROTECTED);
+            /* up to creator to free the heap region */
+        } else {
+            release_guarded_real_memory((vm_addr_t)u, SPECIAL_UNIT_RESERVED_SIZE(u),
+                                        true/*update DR areas immediately*/, true);
+        }
+        u = next_u;
+    }
+#ifdef HEAP_ACCOUNTING
+    add_heapacct_to_global_stats(&su->acct);
+#endif
+    LOG(THREAD_GET, LOG_HEAP|LOG_STATS, 1, "\tTotal heap used: "SZFMT" KB\n",
+        total_heap_used/1024);
+#if defined(WINDOWS_PC_SAMPLE) && !defined(DEBUG)
+    if (special_heap_profile_enabled()) {
+        /* Removed this special_units_t from the master list */
+        mutex_lock(&special_units_list_lock); 
+        if (special_units_list == su)
+            special_units_list = su->next;
+        else {
+            special_units_t *prev = special_units_list;
+            ASSERT(prev != NULL);
+            for (; prev->next != NULL && prev->next != su; prev = prev->next)
+                ;/*nothing*/
+            ASSERT(prev->next == su);
+            prev->next = su->next;
+        }
+        mutex_unlock(&special_units_list_lock); 
+    }
+#endif
+    if (su->use_lock)
+        DELETE_LOCK(su->lock);
+    /* up to caller to free the vector, which is typically multi-heap */
+    if (su->persistent) {
+        global_heap_free(su, sizeof(special_units_t) HEAPACCT(ACCT_MEM_MGT));
+    } else {
+        nonpersistent_heap_free(GLOBAL_DCONTEXT, su, sizeof(special_units_t)
+                                HEAPACCT(ACCT_MEM_MGT));
+    }
+}
+
+void *
+special_heap_calloc(void *special, uint num)
+{
+#ifdef DEBUG
+    dcontext_t *dcontext = get_thread_private_dcontext();
+#endif
+    special_units_t *su = (special_units_t *) special;
+    special_heap_unit_t *u;
+    void *p = NULL;
+    bool took_free = false;
+    ASSERT(num > 0);
+    if (su->use_lock)
+        mutex_lock(&su->lock);
+    u = su->cur_unit;
+    if (su->free_list != NULL && num == 1) {
+        p = (void *) su->free_list;
+        su->free_list = *((heap_pc *)p);
+        took_free = true;
+    } else if (su->cfree_list != NULL && num > 1) {
+        /* FIXME: take a piece of cfree if num == 1?
+         * seems better to save the bigger pieces
+         */
+        cfree_header_t *cfree = su->cfree_list, *prev = NULL;
+        while (cfree != NULL && cfree->count < num) {
+            prev = cfree;
+            cfree = cfree->next_cfree;
+        }
+        if (cfree != NULL) {
+            ASSERT(cfree->count >= num);
+            took_free = true;
+            if (cfree->count == num) {
+                /* take it out of list altogether */
+                if (prev == NULL)
+                    su->cfree_list = cfree->next_cfree;
+                else
+                    prev->next_cfree = cfree->next_cfree;
+                p = (void *) cfree;
+            } else if (cfree->count == num+1) {
+                /* add single-size piece to normal free list */
+                heap_pc tail = ((heap_pc) cfree) + num * su->block_size;
+                *((heap_pc *)tail) = su->free_list;
+                su->free_list = tail;
+                p = (void *) cfree;
+            } else {
+                /* if take tail don't have to change free list ptrs at all */
+                cfree->count -= num;
+                p = (void *) (((heap_pc) cfree) + (cfree->count - num) * su->block_size);
+            }
+        }
+    }
+    if (!took_free) {
+        /* no free blocks, grab a new one */
+        if (u->cur_pc + su->block_size*num > u->end_pc) {
+            /* simply extend commitment, if possible */
+            DEBUG_DECLARE(size_t pre_commit_size = SPECIAL_UNIT_COMMIT_SIZE(u);)
+            special_unit_extend_commitment(u, su->block_size*num, get_prot(su));
+            STATS_ADD_PEAK(heap_special_capacity,
+                           SPECIAL_UNIT_COMMIT_SIZE(u) - pre_commit_size);
+            /* check again after extending commit */
+            if (u->cur_pc + su->block_size*num > u->end_pc) {
+                /* no room, need new unit */
+                special_heap_unit_t *new_unit;
+                special_heap_unit_t *prev = su->top_unit;
+                size_t size = UNITALLOC(u);
+                while (prev->next != NULL)
+                    prev = prev->next;
+                /* create new unit double size of old unit (until hit max size) */
+                if (size*2 <= HEAP_UNIT_MAX_SIZE)
+                    size *= 2;
+                /* we don't support arbitrarily long sequences */
+                ASSERT(su->block_size * num < size);
+                new_unit = special_heap_create_unit(su, NULL, size, false/*empty*/);
+                prev->next = new_unit;
+                if (su->use_lock) {
+                    /* if synch bad so is printing */
+                    LOG(THREAD, LOG_HEAP, 3, "\tCreating new heap unit %d\n",
+                        new_unit->id);
+                }
+                su->cur_unit = new_unit;
+                u = new_unit;
+                ASSERT(u->cur_pc + su->block_size*num <= u->end_pc);
+            }
+        }
+
+        p = (void *) u->cur_pc;
+        u->cur_pc += su->block_size*num;
+        ACCOUNT_FOR_ALLOC(alloc_new, su, ACCT_SPECIAL,
+                          su->block_size*num, su->block_size*num);
+    } else {
+        ACCOUNT_FOR_ALLOC(alloc_reuse, su, ACCT_SPECIAL,
+                          su->block_size*num, su->block_size*num);
+    }
+    if (su->use_lock)
+        mutex_unlock(&su->lock);
+
+#ifdef DEBUG_MEMORY
+    memset(p, HEAP_ALLOCATED_BYTE, su->block_size*num);
+#endif
+    ASSERT(p != NULL);
+    return (void*)p;
+}
+
+void *
+special_heap_alloc(void *special)
+{
+    return special_heap_calloc(special, 1);
+}
+
+void
+special_heap_cfree(void *special, void *p, uint num)
+{
+    special_units_t *su = (special_units_t *) special;
+    ASSERT(num > 0);
+    ASSERT(p != NULL);
+    /* Allow freeing while iterating w/o deadlock (iterator holds lock) */
+    ASSERT(!su->in_iterator || OWN_MUTEX(&su->lock));
+    if (su->use_lock && !su->in_iterator)
+        mutex_lock(&su->lock);
+#ifdef DEBUG_MEMORY
+    /* FIXME: ensure that p is in allocated state */
+    memset(p, HEAP_UNALLOCATED_BYTE, su->block_size*num);
+#endif
+    if (num == 1) {
+        /* write next pointer */
+        *((heap_pc *)p) = su->free_list;
+        su->free_list = (heap_pc)p;
+    } else {
+        cfree_header_t *cfree = (cfree_header_t *) p;
+        cfree->next_cfree = su->cfree_list;
+        cfree->count = num;
+        su->cfree_list = cfree;
+    }
+#ifdef HEAP_ACCOUNTING
+    ACCOUNT_FOR_FREE(su, ACCT_SPECIAL, su->block_size*num);
+#endif
+    if (su->use_lock && !su->in_iterator)
+        mutex_unlock(&su->lock);
+}
+
+void
+special_heap_free(void *special, void *p)
+{
+    special_heap_cfree(special, p, 1);
+}
+
+/* Special heap iterator.  Initialized with special_heap_iterator_start(), which
+ * grabs the heap lock (regardless of whether synch is used for allocs), and
+ * destroyed with special_heap_iterator_stop() to release the lock.
+ * If the special heap uses no lock for alloc, it is up to the caller
+ * to prevent race conditions causing problems.
+ * Accessor special_heap_iterator_next() should be called only when
+ * predicate special_heap_iterator_hasnext() is true.
+ * Any mutation of the heap while iterating will result in a deadlock
+ * for heaps that use locks for alloc, except for individual freeing,
+ * which will proceed w/o trying to grab the lock a second time.
+ * FIXME: could generalize to regular heaps if a use arises.
+ */
+void
+special_heap_iterator_start(void *heap, special_heap_iterator_t *shi)
+{
+    special_units_t *su = (special_units_t *) heap;
+    ASSERT(heap != NULL);
+    ASSERT(shi != NULL);
+    mutex_lock(&su->lock);
+    shi->heap = heap;
+    shi->next_unit = (void *) su->top_unit;
+    su->in_iterator = true;
+}
+
+bool
+special_heap_iterator_hasnext(special_heap_iterator_t *shi)
+{
+    ASSERT(shi != NULL);
+    DODEBUG({
+        special_units_t *su = (special_units_t *) shi->heap;
+        ASSERT(su != NULL);
+        ASSERT_OWN_MUTEX(true, &su->lock);
+    });
+    return (shi->next_unit != NULL);
+}
+
+/* Iterator accessor:
+ * Has to be initialized with special_heap_iterator_start, and should be
+ * called only when special_heap_iterator_hasnext() is true.
+ * Sets the area boundaries in area_start and area_end.
+ */
+void
+special_heap_iterator_next(special_heap_iterator_t *shi /* IN/OUT */,
+                           app_pc *heap_start /* OUT */, app_pc *heap_end /* OUT */)
+{
+    special_units_t *su;
+    special_heap_unit_t *u;
+    ASSERT(shi != NULL);
+    su = (special_units_t *) shi->heap;
+    ASSERT(su != NULL);
+    ASSERT_OWN_MUTEX(true, &su->lock);
+    u = (special_heap_unit_t *) shi->next_unit;
+    ASSERT(u != NULL);
+    if (u != NULL) { /* caller error, but paranoid */
+        if (heap_start != NULL)
+            *heap_start = u->start_pc;
+        ASSERT(u->cur_pc <= u->end_pc);
+        if (heap_end != NULL)
+            *heap_end = u->cur_pc;
+        shi->next_unit = (void *) u->next;
+    }
+}
+
+void
+special_heap_iterator_stop(special_heap_iterator_t *shi)
+{
+    special_units_t *su;
+    ASSERT(shi != NULL);
+    su = (special_units_t *) shi->heap;
+    ASSERT(su != NULL);
+    ASSERT_OWN_MUTEX(true, &su->lock);
+    su->in_iterator = false;
+    mutex_unlock(&su->lock);
+    DODEBUG({
+        shi->heap = NULL;
+        shi->next_unit = NULL;
+    });
+}
+
+#if defined(DEBUG) && defined(HOT_PATCHING_INTERFACE)
+/* We leak hotp trampolines as part of fix for case 9593; so, during a detach
+ * we can't delete the trampoline heap.  However if that heap's lock isn't
+ * deleted, we'll assert.  This routine is used only for that.  Normally, we
+ * should call special_heap_exit() which deletes the lock. */
+void
+special_heap_delete_lock(void *special)
+{
+    special_units_t *su = (special_units_t *)special;
+
+    /* No one calls this routine unless they have a lock to delete. */
+    ASSERT(su != NULL);
+    if (su == NULL)
+        return;
+
+    ASSERT(su->use_lock);
+    if (su->use_lock)
+        DELETE_LOCK(su->lock);
+}
+#endif
+
+/*----------------------------------------------------------------------------*/
+#ifdef WINDOWS  /* currently not used on linux */
+/* Landing pads (introduced as part of work for PR 250294). */
+
+/* landing_pad_areas is a vmvector made up of regions of memory called 
+ * landing pad areas, each of which contains multiple landing pads.  Landing
+ * pads are small trampolines used to jump from the hook point to the main 
+ * trampoline.  This is used in both 32-bit and 64-bit DR.  In both cases it
+ * will handle the problem of hook chaining by 3rd party software and us having
+ * to release our hooks (we'll nop the landing pad and free the trampoline).
+ * In 64-bit it also solves the problem of reachability of the 5-byte rel jmp
+ * we use for hooking, i.e., that 5-byte rel jmp may not reach the main
+ * trampoline in DR heap.  We have to maintain the hook a 5-byte jmp because
+ * hotp_only assumes it (see PR 250294).
+ *
+ * A landing pad will have nothing more than a jump (5-byte rel for 32-bit DR
+ * and 64-bit abs ind jmp for 64-bit DR) to the trampoline and a 5-byte rel jmp
+ * back to the next instruction after the hook.
+ *
+ * To handle hook chaining landing pads won't be released till process exit
+ * (not on a detach), their first jump will just be nop'ed.  As landing pads
+ * aren't released till exit, all landing pads are just incrementally allocated
+ * in a landing pad area.
+ *
+ * Note: Landing pad areas don't necessarily have to fall within the vm_reserve
+ * region or capacity, so aren't accounted by our vmm.
+ *
+ * Note: If in future other needs for such region specific allocation should
+ * arise, then we should convert this into special_heap_alloc_in_region().  For
+ * now, landing pads are the only consumers, so was decided to be acceptable.
+ *
+ * See win32/callback.c for emit_landing_pad_code() and landing pad usage.
+ */
+
+typedef struct {
+    byte *start;        /* start of reserved region */
+    byte *end;          /* end of reserved region */
+    byte *commit_end;   /* end of committed memory in the reserved region */
+    byte *cur_ptr;      /* pointer to next allocatable landing pad memory */
+} landing_pad_area_t;
+
+/* Allocates a landing pad so that a hook inserted at addr_to_hook can reach
+ * its trampoline via the landing pad.  The landing pad will reachable by a
+ * 32-bit relative jmp from addr_to_hook.
+ * Note: we may want to generalize this at some point such that the size of the
+ *       landing pad is passed as an argument.
+ */
+byte *
+alloc_landing_pad(app_pc addr_to_hook)
+{
+    app_pc hook_region_start, hook_region_end;
+    app_pc alloc_region_start, alloc_region_end;
+    app_pc lpad_area_start = NULL, lpad_area_end;
+    app_pc lpad = NULL;
+    landing_pad_area_t *lpad_area = NULL;
+
+    /* Allocate the landing pad area such that any hook from within the module
+     * or memory region containing addr_to_hook can use the same area for a
+     * landing pad.  Makes it more efficient. */
+    hook_region_start = get_allocation_base(addr_to_hook);
+    if (hook_region_start == NULL) {  /* to support raw virtual address hooks */
+        ASSERT_CURIOSITY("trying to hook raw or unallocated memory?");
+        hook_region_start = addr_to_hook;
+        hook_region_end = addr_to_hook;
+    } else {
+        hook_region_end = hook_region_start +
+                          get_allocation_size(hook_region_start, NULL);
+        ASSERT(hook_region_end > hook_region_start);    /* check overflow */
+        /* If region size is > 2 GB, then it isn't an image; PE32{,+} restrict
+         * images to 2 GB.  Also, if region is > 2 GB the reachability macros
+         * called below will return a region smaller (and with start and end
+         * inverted) than the region from which the reachability is desired,
+         * i.e., some of the areas in [hook_region_start, hook_region_end)
+         * won't be able to reach the region computed.
+         *
+         * A better choice is to pick something smaller (100 MB) because if the
+         * region is close to 2 GB in size then we might not be able to
+         * allocate memory for a landing pad that is reachable.
+         */
+        if (hook_region_end - hook_region_start > 100 * 1024 * 1024) {
+            /* Try a smaller region of 100 MB around the address to hook. */
+            ASSERT_CURIOSITY(false && "seeing patch region > 100 MB - DGC?");
+            hook_region_start = MIN(addr_to_hook, MAX(hook_region_start,
+                                                      addr_to_hook - 50 * 1024 * 1024));
+            hook_region_end = MAX(addr_to_hook, MIN(hook_region_end,
+                                                    addr_to_hook + 50 * 1024 * 1024));
+        }
+    }
+
+    /* Define the region that can be reached from anywhere within the
+     * hook region with a 32-bit rel jmp.
+     */
+    alloc_region_start = REACHABLE_32BIT_START(hook_region_start,
+                                               hook_region_end);
+    alloc_region_end = REACHABLE_32BIT_END(hook_region_start, hook_region_end);
+    ASSERT(alloc_region_start < alloc_region_end);
+
+    /* Check if there is an existing landing pad area within the reachable
+     * region for the hook location.  If so use it, else allocate one.
+     */
+    write_lock(&landing_pad_areas->lock);
+    if (vmvector_overlap(landing_pad_areas, alloc_region_start,
+                         alloc_region_end)) {
+        /* Now we have to get that landing pad area that is FULLY contained
+         * within alloc_region_start and alloc_region_end.  If a landing pad
+         * area is only partially within the alloc region, then a landing pad
+         * created there won't be able to reach the addr_to_hook.  BTW, that
+         * landing pad area should have enough space to allocate a landing pad!
+         * If these conditions are met allocate a landing pad.
+         */
+        vmvector_iterator_t lpad_area_iter;
+        vmvector_iterator_start(landing_pad_areas, &lpad_area_iter);
+        while (vmvector_iterator_hasnext(&lpad_area_iter)) {
+            lpad_area = vmvector_iterator_next(&lpad_area_iter,
+                                               &lpad_area_start,
+                                               &lpad_area_end);
+            if (lpad_area_start < alloc_region_end &&
+                lpad_area_end > alloc_region_start &&
+                (lpad_area->cur_ptr + LANDING_PAD_SIZE) < lpad_area_end) {
+                 /* See if enough memory in this landing pad area has been
+                  * committed, if not commit more memory.
+                  */
+                 if ((lpad_area->cur_ptr + LANDING_PAD_SIZE) >=
+                     lpad_area->commit_end) {
+                     extend_commitment(lpad_area->commit_end, PAGE_SIZE,
+                                       MEMPROT_READ|MEMPROT_EXEC,
+                                       false /* not initial commit */);
+                     lpad_area->commit_end += PAGE_SIZE;
+                 }
+
+                /* Update the current pointer for the landing pad area, i.e.,
+                 * allocate the landing pad.
+                 */
+                lpad = lpad_area->cur_ptr;
+                lpad_area->cur_ptr += LANDING_PAD_SIZE;
+                break;
+            }
+        }
+        vmvector_iterator_stop(&lpad_area_iter);
+    } 
+    
+    /* If a landing pad area wasn't found because there wasn't any in the
+     * allocation region or none fully contained within the allocation region,
+     * then create a new one within the allocation region.  Then allocate a
+     * landing pad in it.
+     */
+    if (lpad == NULL) {
+        heap_error_code_t heap_error;
+        lpad_area_start = os_heap_reserve_in_region(alloc_region_start,
+                                                    alloc_region_end,
+                                                    LANDING_PAD_AREA_SIZE,
+                                                    &heap_error);
+        if (lpad_area_start == NULL ||
+            heap_error == HEAP_ERROR_CANT_RESERVE_IN_REGION) {
+            /* Should retry with using just the aligned target address - we may
+             * have made the region so large that there's nothing nearby to
+             * reserve.
+             */
+            lpad_area_start = os_heap_reserve(
+                                  (void *)ALIGN_FORWARD(addr_to_hook,
+                                                        LANDING_PAD_AREA_SIZE),
+                                  LANDING_PAD_AREA_SIZE, &heap_error);
+            if (lpad_area_start == NULL) {
+                /* Even at startup when there will be enough memory,
+                 * theoritically 2 GB of dlls might get packed together before
+                 * we get control (very unlikely), so we can fail.  If it does,
+                 * then say 'oom' and exit.
+                 */ 
+                SYSLOG_INTERNAL_WARNING("unable to reserve memory for landing pads");
+                report_low_on_memory(OOM_RESERVE, heap_error);
+            }
+        }
+
+        /* Allocate the landing pad area as rx, allocate a landing pad in it
+         * and add it to landing_pad_areas vector.  Note, we only commit 4k
+         * initially even though we reserve 64k (LANDING_PAD_AREA_SIZE), to
+         * avoid wastage.
+         */
+        extend_commitment(lpad_area_start, PAGE_SIZE, MEMPROT_READ|MEMPROT_EXEC,
+                          true /* initial commit */);
+
+        lpad_area = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, landing_pad_area_t,
+                                    ACCT_VMAREAS, PROTECTED);
+        lpad_area->start = lpad_area_start;
+        lpad_area->end = lpad_area_start + LANDING_PAD_AREA_SIZE;
+        lpad_area->commit_end = lpad_area_start + PAGE_SIZE;
+        lpad_area->cur_ptr = lpad_area_start;
+        lpad = lpad_area->cur_ptr;
+        lpad_area->cur_ptr += LANDING_PAD_SIZE;
+
+        vmvector_add(landing_pad_areas, lpad_area->start,
+                     lpad_area->end, lpad_area);
+        STATS_INC(num_landing_pad_areas);
+    }
+
+    /* Landing pads aren't added to executable_areas here because not all
+     * landing pads should be added.  Only the ones used for DR hooks should be
+     * added to executable areas (which is done using
+     * landing_pads_to_executable_areas() at the end of inserting DR hooks).
+     * hotp_only related landing pads shouldn't be added to executable areas as
+     * their trampolines aren't added to executable_areas.  This is why landing
+     * pads aren't added to executable_areas here, at the point of allocation.
+     */
+
+    /* Boundary check to make sure the allocation is within the landing pad area. */
+    ASSERT(lpad_area->cur_ptr <= lpad_area->start + LANDING_PAD_AREA_SIZE);
+    write_unlock(&landing_pad_areas->lock);
+    return lpad;
+}
+
+/* Adds or removes all landing pads from executable_areas by adding whole
+ * landing pad areas.  This is done to prevent bb building from considering
+ * landing pads to be selfmod code; as such, these don't have to be
+ * {add,remov}ed from executable_areas for hotp_only or for thin_client mode.
+ */
+void
+landing_pads_to_executable_areas(bool add)
+{
+    vmvector_iterator_t lpad_area_iter;
+    app_pc lpad_area_start, lpad_area_end;
+    DEBUG_DECLARE(landing_pad_area_t *lpad_area;)
+    uint lpad_area_size;
+
+    if (RUNNING_WITHOUT_CODE_CACHE())
+        return;
+
+    /* With code cache, there should be only one landing pad area, just for
+     * dr hooks in ntdll.  For 64-bit, the image entry hook will result in a
+     * new landing pad.
+     */
+    IF_X64_ELSE(,ASSERT(landing_pad_areas->length == 1);)
+
+    /* Just to be safe, walk through all areas in release build. */
+    vmvector_iterator_start(landing_pad_areas, &lpad_area_iter);
+    while (vmvector_iterator_hasnext(&lpad_area_iter)) {
+
+        DEBUG_DECLARE(lpad_area = )
+            vmvector_iterator_next(&lpad_area_iter, &lpad_area_start,
+                                   &lpad_area_end);
+        lpad_area_size = (uint)(lpad_area_end - lpad_area_start);
+        ASSERT(lpad_area_size == LANDING_PAD_AREA_SIZE);
+        /* Current ptr should be within area. */
+        ASSERT(lpad_area->cur_ptr < lpad_area_end);
+        if (add) {
+            add_executable_region(lpad_area_start, lpad_area_size
+                _IF_DEBUG("add landing pad areas after inserting dr hooks"));
+        } else {
+            remove_executable_region(lpad_area_start, lpad_area_size,
+                                     false /* no lock */);
+        }
+    }
+    vmvector_iterator_stop(&lpad_area_iter);
+}
+
+/* Delete landing_pad_areas and the landing_pad_area_t allocated for each
+ * landing pad area.  However, release all landing pads only on process exit;
+ * for detach leave the landing pads in (in case some one hooks after us they
+ * shouldn't crash if they chain correctly).
+ */
+static void
+release_landing_pad_mem(void)
+{
+    vmvector_iterator_t lpad_area_iter;
+    app_pc lpad_area_start, lpad_area_end;
+    landing_pad_area_t *lpad_area;
+    heap_error_code_t heap_error;
+
+    vmvector_iterator_start(landing_pad_areas, &lpad_area_iter);
+    while (vmvector_iterator_hasnext(&lpad_area_iter)) {
+        lpad_area = vmvector_iterator_next(&lpad_area_iter, &lpad_area_start,
+                                           &lpad_area_end);
+        HEAP_TYPE_FREE(GLOBAL_DCONTEXT, lpad_area, landing_pad_area_t,
+                       ACCT_VMAREAS, PROTECTED); 
+        if (!doing_detach)  /* On normal exit release the landing pads. */
+            os_heap_free(lpad_area_start, LANDING_PAD_AREA_SIZE, &heap_error);
+    }
+    vmvector_iterator_stop(&lpad_area_iter);
+    vmvector_delete_vector(GLOBAL_DCONTEXT, landing_pad_areas);
+}
+#endif  /* WINDOWS */
+/*----------------------------------------------------------------------------*/
diff --git a/core/heap.h b/core/heap.h
new file mode 100644
index 000000000..1a33322d9
--- /dev/null
+++ b/core/heap.h
@@ -0,0 +1,319 @@
+/* **********************************************************
+ * Copyright (c) 2001-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2001 Hewlett-Packard Company */
+
+/*
+ * heap.c - heap manager
+ */
+
+#ifndef _HEAP_H_
+#define _HEAP_H_ 1
+
+#ifdef HEAP_ACCOUNTING
+typedef enum {
+    ACCT_FRAGMENT=0,
+    ACCT_COARSE_LINK,
+    ACCT_FRAG_FUTURE,
+    ACCT_FRAG_TABLE,
+    ACCT_IBLTABLE,
+    ACCT_TRACE,
+    ACCT_FCACHE_EMPTY,
+    ACCT_VMAREA_MULTI,
+    ACCT_IR,
+    ACCT_AFTER_CALL,
+    ACCT_VMAREAS,
+    ACCT_SYMBOLS,
+# ifdef SIDELINE
+    ACCT_SIDELINE,
+# endif
+    ACCT_THCOUNTER,
+    ACCT_TOMBSTONE, /* N.B.: leaks in this category are not reported;
+                     * not currently used */
+
+    ACCT_HOT_PATCHING,
+    ACCT_THREAD_MGT,
+    ACCT_MEM_MGT,
+    ACCT_STATS,
+    ACCT_SPECIAL,
+    /* NOTE: Also update the whichheap_name in heap.c when adding here */
+    ACCT_OTHER,
+    ACCT_LAST
+} which_heap_t;
+
+# define HEAPACCT(x) , x
+# define IF_HEAPACCT_ELSE(x, y) x
+#else
+# define HEAPACCT(x)
+# define IF_HEAPACCT_ELSE(x, y) y
+#endif
+
+typedef byte * heap_pc;
+#define HEAP_ALIGNMENT sizeof(heap_pc*)
+extern vm_area_vector_t *landing_pad_areas;
+
+/* Request that the supplied region be 32bit offset reachable from the DR heap.  Should
+ * be called before vmm_heap_init() so we can place the DR heap to meet these constraints.
+ * Can also be called post vmm_heap_init() but at that point acts as an assert that the
+ * supplied region is reachable since the heap is already reserved. */
+void
+request_region_be_heap_reachable(byte *start, size_t size);
+
+/* virtual heap manager */
+void vmm_heap_init(void);
+void vmm_heap_exit(void);
+void print_vmm_heap_data(file_t outf);
+void get_vmm_heap_bounds(byte **heap_start/*OUT*/, byte **heap_end/*OUT*/);
+
+bool heap_check_option_compatibility(void);
+
+bool is_vmm_reserved_address(byte *pc, size_t size);
+bool rel32_reachable_from_heap(byte *target);
+
+/* heap management */
+void heap_init(void);
+void heap_exit(void);
+void heap_reset_init(void);
+void heap_reset_free(void);
+void heap_thread_init(dcontext_t *dcontext);
+void heap_thread_exit(dcontext_t *dcontext);
+
+/* re-initializes non-persistent memory */
+void heap_thread_reset_init(dcontext_t *dcontext);
+/* frees all non-persistent memory */
+void heap_thread_reset_free(dcontext_t *dcontext);
+
+/* these functions use the global heap instead of a thread's heap: */
+void *global_heap_alloc(size_t size HEAPACCT(which_heap_t which));
+void global_heap_free(void *p, size_t size HEAPACCT(which_heap_t which));
+void *global_heap_realloc(void *ptr, size_t old_num, size_t new_num,
+                          size_t element_size HEAPACCT(which_heap_t which));
+
+bool lockwise_safe_to_allocate_memory(void);
+
+/* use heap_mmap to allocate large chunks of executable memory */
+void *heap_mmap(size_t size);
+void heap_munmap(void *p, size_t size);
+void *heap_mmap_reserve(size_t reserve_size, size_t commit_size);
+
+void *heap_mmap_ex(size_t reserve_size, size_t commit_size, uint prot, bool guarded);
+void heap_munmap_ex(void *p, size_t size, bool guarded);
+
+/* updates dynamo_areas and calls the os_ versions */
+byte *
+map_file(file_t f, size_t size, size_t offs, app_pc addr, uint prot,
+         bool copy_on_write);
+bool
+unmap_file(byte *map, size_t size);
+
+/* Allocates executable memory in the same allocation region as this thread's
+ * stack, to save address space (case 9474).
+ */
+void *
+heap_mmap_reserve_post_stack(dcontext_t *dcontext,
+                             size_t reserve_size, size_t commit_size);
+void
+heap_munmap_post_stack(dcontext_t *dcontext, void *p, size_t reserve_size);
+
+/* It is up to the caller to ensure commit_size is a page size multiple,
+ * and that it does not extend beyond the initial reservation.
+ */
+void
+heap_mmap_extend_commitment(void *p, size_t commit_size);
+
+void
+heap_mmap_retract_commitment(void *retract_start, size_t decommit_size);
+
+/* use stack_alloc to build a stack -- it returns TOS */
+void *stack_alloc(size_t size);
+void stack_free(void *p, size_t size);
+
+#ifdef STACK_GUARD_PAGE
+/* checks if pc is in guard page on stack */
+bool is_stack_overflow(dcontext_t *dcontext, byte *sp);
+#endif
+
+/* these are for thread-local allocs
+ * passing dcontext == GLOBAL_DCONTEXT will end up calling global_heap_{alloc,free}
+ */
+void *heap_alloc(dcontext_t *dcontext, size_t size HEAPACCT(which_heap_t which));
+void heap_free(dcontext_t *dcontext, void *p, size_t size HEAPACCT(which_heap_t which));
+
+#ifdef HEAP_ACCOUNTING
+void print_heap_statistics(void);
+#endif
+
+/* FIXME: persistence is yet another dimension here
+ * let's clean all these up and have a single alloc routine?
+ */
+void *nonpersistent_heap_alloc(dcontext_t *dcontext, size_t size
+                               HEAPACCT(which_heap_t which));
+void nonpersistent_heap_free(dcontext_t *dcontext, void *p, size_t size
+                             HEAPACCT(which_heap_t which));
+
+bool local_heap_protected(dcontext_t *dcontext);
+void protect_local_heap(dcontext_t *dcontext, bool writable);
+void protect_global_heap(bool writable);
+void *global_unprotected_heap_alloc(size_t size HEAPACCT(which_heap_t which));
+void global_unprotected_heap_free(void *p, size_t size HEAPACCT(which_heap_t which));
+
+#define UNPROTECTED_LOCAL_ALLOC(dc, ...) global_unprotected_heap_alloc(__VA_ARGS__)
+#define UNPROTECTED_LOCAL_FREE(dc, ...) global_unprotected_heap_free(__VA_ARGS__)
+#define UNPROTECTED_GLOBAL_ALLOC global_unprotected_heap_alloc
+#define UNPROTECTED_GLOBAL_FREE global_unprotected_heap_free
+
+/* use global heap for shared fragments and their related data structures */
+#define FRAGMENT_ALLOC_DC(dc, flags) (TEST(FRAG_SHARED, (flags)) ? GLOBAL_DCONTEXT : (dc))
+#define FRAGMENT_TABLE_ALLOC_DC(dc, flags) (TEST(FRAG_TABLE_SHARED, (flags)) ? GLOBAL_DCONTEXT : (dc))
+
+/* convenience for allocating a single type: does cast, sizeof, and HEAPACCT
+ * for you, and takes param for whether protected or not
+ */
+#define PROTECTED true
+#define UNPROTECTED false
+#define HEAP_ARRAY_ALLOC(dc, type, num, which, protected) \
+    ((protected) ? \
+        (type *) heap_alloc(dc, sizeof(type)*(num) HEAPACCT(which)) : \
+        (type *) UNPROTECTED_LOCAL_ALLOC(dc, sizeof(type)*(num) HEAPACCT(which)))
+#define HEAP_TYPE_ALLOC(dc, type, which, protected) \
+    HEAP_ARRAY_ALLOC(dc, type, 1, which, protected)
+#define HEAP_ARRAY_ALLOC_MEMSET(dc, type, num, which, protected, val)   \
+    memset(HEAP_ARRAY_ALLOC(dc, type, num, which, protected), (val),    \
+           sizeof(type)*(num));
+
+#define HEAP_ARRAY_FREE(dc, p, type, num, which, protected) \
+    ((protected) ? \
+        heap_free(dc, (type*)p, sizeof(type)*(num) HEAPACCT(which)) : \
+        UNPROTECTED_LOCAL_FREE(dc, (type*)p, sizeof(type)*(num) HEAPACCT(which)))
+#define HEAP_TYPE_FREE(dc, p, type, which, protected) \
+    HEAP_ARRAY_FREE(dc, p, type, 1, which, protected)
+
+/* nonpersistent heap is assumed to be protected */
+#define NONPERSISTENT_HEAP_ARRAY_ALLOC(dc, type, num, which) \
+    (type *) nonpersistent_heap_alloc(dc, sizeof(type)*(num) HEAPACCT(which))
+#define NONPERSISTENT_HEAP_TYPE_ALLOC(dc, type, which) \
+    NONPERSISTENT_HEAP_ARRAY_ALLOC(dc, type, 1, which)
+#define NONPERSISTENT_HEAP_ARRAY_FREE(dc, p, type, num, which) \
+    nonpersistent_heap_free(dc, (type*)p, sizeof(type)*(num) HEAPACCT(which))
+#define NONPERSISTENT_HEAP_TYPE_FREE(dc, p, type, which) \
+    NONPERSISTENT_HEAP_ARRAY_FREE(dc, p, type, 1, which)
+
+/* special heap of same-sized blocks that avoids global locks */
+void *special_heap_init(uint block_size, bool use_lock, bool executable,
+                        bool persistent);
+void special_heap_exit(void *special);
+void *special_heap_alloc(void *special);
+void *special_heap_calloc(void *special, uint num);
+void special_heap_free(void *special, void *p);
+void special_heap_cfree(void *special, void *p, uint num);
+#if defined(WINDOWS_PC_SAMPLE) && !defined(DEBUG)
+void special_heap_profile_exit(void);
+#endif
+
+/* iterator over units in a special heap */
+typedef struct _special_heap_iterator {
+    /* opaque types, exposed to support stack-alloc iterator */
+    void *heap;
+    void *next_unit;
+} special_heap_iterator_t;
+
+/* Special heap iterator.  Initialized with special_heap_iterator_start(), and
+ * destroyed with special_heap_iterator_stop() to release the lock.
+ * If the special heap uses no lock for alloc, it is up to the caller
+ * to prevent race conditions causing problems.
+ * Accessor special_heap_iterator_next() should be called only when
+ * predicate special_heap_iterator_hasnext() is true.
+ * Any mutation of the heap while iterating will result in a deadlock
+ * for heaps that use locks for alloc.
+ */
+void
+special_heap_iterator_start(void *heap, special_heap_iterator_t *shi);
+bool
+special_heap_iterator_hasnext(special_heap_iterator_t *shi);
+void
+special_heap_iterator_next(special_heap_iterator_t *shi /* IN/OUT */,
+                           app_pc *heap_start /* OUT */, app_pc *heap_end /* OUT */);
+void
+special_heap_iterator_stop(special_heap_iterator_t *shi);
+
+/* Special heap w/ a vector for lookups */
+void *
+special_heap_pclookup_init(uint block_size, bool use_lock, bool executable,
+                           bool persistent, vm_area_vector_t *vector, void *vector_data,
+                           byte *heap_region, size_t heap_size, bool unit_full);
+
+void
+special_heap_set_vector_data(void *special, void *vector_data);
+
+/* Returns false if the special heap has more than one unit or has a
+ * non-externally-allocated unit.
+ * Sets the cur pc for the only unit to end_pc.
+ */
+bool
+special_heap_set_unit_end(void *special, byte *end_pc);
+
+void heap_vmareas_synch_units(void);
+
+/* Utility routine to delete special heap locks; needed for case 9593. */
+void special_heap_delete_lock(void *special);
+
+#ifdef DEBUG_MEMORY
+# define HEAP_TO_BYTE_EX(hex) 0x##hex
+# define HEAP_TO_BYTE(hex) HEAP_TO_BYTE_EX(hex)
+# define HEAP_TO_UINT_EX(hex) 0x##hex##hex##hex##hex
+# define HEAP_TO_UINT(hex) HEAP_TO_UINT_EX(hex)
+# ifdef X64
+#  define HEAP_TO_PTR_UINT_EX(hex) 0x##hex##hex##hex##hex##hex##hex##hex##hex
+# else
+#  define HEAP_TO_PTR_UINT_EX(hex) 0x##hex##hex##hex##hex
+# endif
+# define HEAP_TO_PTR_UINT(hex) HEAP_TO_PTR_UINT_EX(hex)
+
+# define HEAP_UNALLOCATED cd
+# define HEAP_UNALLOCATED_BYTE HEAP_TO_BYTE(HEAP_UNALLOCATED)
+# define HEAP_UNALLOCATED_UINT HEAP_TO_UINT(HEAP_UNALLOCATED)
+# define HEAP_UNALLOCATED_PTR_UINT HEAP_TO_PTR_UINT(HEAP_UNALLOCATED)
+
+# define HEAP_ALLOCATED ab
+# define HEAP_ALLOCATED_BYTE HEAP_TO_BYTE(HEAP_ALLOCATED)
+# define HEAP_ALLOCATED_UINT HEAP_TO_UINT(HEAP_ALLOCATED)
+# define HEAP_ALLOCATED_PTR_UINT HEAP_TO_PTR_UINT(HEAP_ALLOCATED)
+
+# define HEAP_PAD bc
+# define HEAP_PAD_BYTE HEAP_TO_BYTE(HEAP_PAD)
+# define HEAP_PAD_UINT HEAP_TO_UINT(HEAP_PAD)
+# define HEAP_PAD_PTR_UINT HEAP_TO_PTR_UINT(HEAP_PAD)
+#endif
+
+#endif /* _HEAP_H_ */
diff --git a/core/hotpatch.c b/core/hotpatch.c
new file mode 100644
index 000000000..1c96f44ad
--- /dev/null
+++ b/core/hotpatch.c
@@ -0,0 +1,6375 @@
+/* **********************************************************
+ * Copyright (c) 2005-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2005-2007 Determina Corp. */
+
+/*
+ * hotpatch.c - Hot patching mechanism
+ */
+
+#include "globals.h"
+#include "fragment.h"
+#include "arch.h"
+#include "instr.h"
+#include "instr_create.h"
+#include "decode.h"
+#include "instrument.h"
+#include "hotpatch.h"
+#include "hotpatch_interface.h"
+#include "moduledb.h"   /* macros for nudge; can be moved with nudge to os.c */
+
+#ifndef WINDOWS
+# include <string.h>
+#endif
+
+#include <limits.h>  /* for ULLONG_MAX */
+
+#include "fcache.h"  /* for fcache_reset_all_caches_proactively */
+#ifdef GBOP
+# include "aslr.h"
+#endif
+
+#include "perscache.h"
+#include "synch.h"
+
+/* Note: HOT_PATCHING_INTERFACE refers to the mechanism for injecting code at
+ *       arbitrary points in the application text.  It was formerly known as
+ *       constraint injection.  It has nothing to do with the DR mechanism that 
+ *       allows for dynamically changing existing instructions, as indicated
+ *       by INSTR_HOT_PATCHABLE in instr.h
+ */
+#ifdef HOT_PATCHING_INTERFACE  /* Around the whole file */
+/*----------------------------------------------------------------------------*/
+/* Local typed constants. */
+
+/* Local untyped constants. */
+
+/* defined for non-GBOP as well since used for -probe_api */   
+#define HOTP_ONLY_GBOP_PRECEDENCE 10   
+
+/* Limits for vulnerability data provided by the constraint writer. */
+enum {
+    MIN_HOTP_INTERFACE_VERSION = HOTP_INTERFACE_VERSION,
+    MAX_HOTP_INTERFACE_VERSION = HOTP_INTERFACE_VERSION,
+    MIN_NUM_VULNERABILITIES = 1,
+    MAX_NUM_VULNERABILITIES = 10000,
+    MIN_VULNERABILITY_ID_LENGTH = 1,
+    MAX_VULNERABILITY_ID_LENGTH = 32,
+    MIN_POLICY_ID_LENGTH = HOTP_POLICY_ID_LENGTH,
+    MAX_POLICY_ID_LENGTH = HOTP_POLICY_ID_LENGTH,
+    MIN_POLICY_VERSION = 1,
+    MAX_POLICY_VERSION = 10000,
+    MIN_NUM_SETS = 1,
+    MAX_NUM_SETS = 10000,
+    MIN_NUM_MODULES = 1,
+    MAX_NUM_MODULES = 10000,
+
+    /* We don't expect PE files to be more than 1 GB in size. */
+    MAX_MODULE_SIZE = 1024*1024*1024,
+
+    /* Can have PEs with time stamp as zero, though fairly unlikely; zero
+     * checksum is more likely, zero file version is somewhat likely and zero
+     * {image,code} size is extremely unlikely.  The max values though are 
+     * unlikely to exist in reality, we use these limits as the don't care 
+     * values for timestamp, checksum, {image,code} size & file version.
+     */
+    MIN_PE_TIMESTAMP = 0,
+    MAX_PE_TIMESTAMP = UINT_MAX,
+    PE_TIMESTAMP_IGNORE = UINT_MAX,
+    PE_TIMESTAMP_UNAVAILABLE = PE_TIMESTAMP_IGNORE - 1,
+
+    MIN_PE_CHECKSUM = 0,
+    MAX_PE_CHECKSUM = UINT_MAX,
+    PE_CHECKSUM_IGNORE = UINT_MAX,
+    PE_CHECKSUM_UNAVAILABLE = PE_CHECKSUM_IGNORE - 1,
+
+    MIN_PE_IMAGE_SIZE = 0,
+    MAX_PE_IMAGE_SIZE = UINT_MAX,
+    PE_IMAGE_SIZE_IGNORE = UINT_MAX,
+    PE_IMAGE_SIZE_UNAVAILABLE = PE_IMAGE_SIZE_IGNORE - 1,
+
+    MIN_PE_CODE_SIZE = 0,       /* kbdus.dll has only data in it */
+    MAX_PE_CODE_SIZE = UINT_MAX,
+    PE_CODE_SIZE_IGNORE = UINT_MAX,
+    PE_CODE_SIZE_UNAVAILABLE = PE_CODE_SIZE_IGNORE - 1,
+
+    MIN_PE_FILE_VERSION = 0,
+    MAX_PE_FILE_VERSION = ULLONG_MAX,
+    PE_FILE_VERSION_IGNORE = IF_WINDOWS_ELSE(MODULE_FILE_VERSION_INVALID, ULLONG_MAX),
+    PE_FILE_VERSION_UNAVAILABLE = PE_FILE_VERSION_IGNORE - 1,
+
+    MIN_NUM_PATCH_POINT_HASHES = 1,
+    MAX_NUM_PATCH_POINT_HASHES = 10000,
+    MIN_HASH_START_OFFSET = 1,
+    MAX_HASH_START_OFFSET = MAX_MODULE_SIZE,    /* Can't exceed module size. */
+    MIN_HASH_LENGTH = 1,
+    MAX_HASH_LENGTH = MAX_MODULE_SIZE,          /* Can't exceed module size. */
+    MIN_HASH_VALUE = 0,
+    MAX_HASH_VALUE = UINT_MAX,
+
+    MIN_NUM_PATCH_POINTS = MIN_NUM_PATCH_POINT_HASHES,
+    MAX_NUM_PATCH_POINTS = MAX_NUM_PATCH_POINT_HASHES,
+    MIN_PATCH_OFFSET = 1,
+    MAX_PATCH_OFFSET = MAX_MODULE_SIZE,         /* Can't exceed module size. */
+    MIN_PATCH_PRECEDENCE = 1,
+    MAX_PATCH_PRECEDENCE = 10000,
+    MIN_DETECTOR_OFFSET = 1,
+    /* Hot patch dlls shouldn't be anywhere near 10 MB in size;
+     * this check is just to catch some wrong file being loaded by accident.
+     * Today a typical hot patch is far less than 1k in size, so to hit 10 MB
+     * we would need a mininum of 10000 constraints of 1k each - unlikely.
+     */
+    MAX_DETECTOR_OFFSET = 10*1024*1024,
+    /* Protectors should exist for all hot patches; even if it does nothing. */
+    MIN_PROTECTOR_OFFSET = 1,
+    MAX_PROTECTOR_OFFSET = MAX_DETECTOR_OFFSET,
+    /* Zero offset either means there is no protector or no control flow 
+     * change is requested by the protector. 
+     */
+    MIN_RETURN_ADDR = 0,
+    /* We don't expect return addresses to be across modules; given that we 
+     * don't expect a module to be more than 1 GB in size, the return address 
+     * offset shouldn't be more than 1 GB too.
+     */
+    MAX_RETURN_ADDR = MAX_PATCH_OFFSET,
+    MIN_MODE = HOTP_MODE_OFF,
+    MAX_MODE = HOTP_MODE_PROTECT,
+
+    /* case 8051: > 256KB per-process means we should start thinking about 
+     * sharing.  24-Apr-07: sharing is in plan for 4.3 or 4.4; upping to 384k.
+     * Note: this is used only in debug builds; release builds can handle all
+     * sizes as long as we don't run out of memory. */
+    MAX_POLICY_FILE_SIZE = 384 * 1024
+
+};
+#define PE_NAME_IGNORE  "*"         /* Can't have strings in the enum. */
+#define PE_NAME_UNAVAILABLE  '\0'
+/*----------------------------------------------------------------------------*/
+/* Local type definitions. */
+
+/* Module signature is used to uniquely describe a module, in our case, a Win32
+ * PE module.
+ */
+typedef struct {            /* xref case 4688 */
+    const char *pe_name;
+
+    /* Don't-care values for pe_{checksum,timestamp,{image,code}_size,
+     * file_version} will be their respective MAX values.  See enum above.
+     */
+    uint pe_checksum;
+    uint pe_timestamp;
+    size_t pe_image_size;
+
+    /* Refers to the sum of the unpadded sizes of all executable sections in the
+     * PE image.  The section size used is from get_image_section_unpadded_size()
+     * which equals VirtualSize (unless that is 0 in which case it equals SizeOfRawData).
+     *
+     * As an aside note that VirtualSize usually has no alignment padding while 
+     * SizeOfRawData is typically padded to FileAlignment (the image loader pads
+     * VirtualSize to SectionAligment), so SizeOfRawData is often larger than
+     * VirtualSize for fully initialized sections (- this is the opposite of how it is
+     * in unix/elf, i.e., raw/file size is usually smaller than virtual/mem size because
+     * the latter does the alignment; also in unix, there are usually two different
+     * mmaps as opposed to one on windows to load the image).  Though xref case 5355,
+     * what is actually accepted (and generated by some compilers) differs from what is
+     * typical/legal in pe specifications.
+     *
+     * Using _code_ rather than _text_ in the name because text usually refers
+     * only to the .text section.
+     */
+    size_t pe_code_size;
+
+    /* Found in the resource section, some PE file may not have it, in which
+     * case it will be set to its don't-care value.
+     */
+    uint64 pe_file_version;
+} hotp_module_sig_t;
+
+/* A patch point describes what application address to patch and the address of
+ * the hot patches that will be used for patching.  If a hot patch (only a
+ * protector) intends to change the flow of application's execution, then the
+ * address to which control should go to after the hot patch is executed is
+ * also specified.  A precedence attribute defines the order (rank) in which a
+ * particular patch is to be applied if more than one need to be applied at the
+ * same application offset.  All addresses are relative to the base of the 
+ * module.
+ */
+/* TODO: typedef uint app_rva_t to define offsets; app_pc is actually an address, 
+ *       not an offset, so don't use it for defining offsets
+ */
+/* app_pc is a pointer, not an offset; using it to a compute pointer with
+ * base address gives a compiler error about adding two pointers.  Hence, a new 
+ * type to define module offsets.
+ */
+typedef struct {
+    app_rva_t offset;         /* offset relative to the base of the module where 
+                             * thepatch is to be applied */
+
+    /* TODO: clearly split each structure into read only and runtime data 
+     * because things are tending to go out of synch again; can create a 
+     * parallel tree later on. 
+     */
+    app_rva_t detector_fn;    /* Offset of the detector function from the base
+                             * of the hot patch dll */
+    app_rva_t protector_fn;
+    app_rva_t return_addr;
+
+    /* NYI (was never needed in practice, though at design time I thought this
+     * was needed for supporting multiple patches at the same address); lower
+     * numbers mean high precedence.
+     */
+    uint precedence; 
+
+    /*------------------------------------------------------------------------*/
+    /* The following fields are part of runtime policy/vulnerability data, not 
+     * part of vulnerability definitions, i.e., shouldn't be shared across 
+     * processes.
+     */
+    /* TODO: num_injected at the vulnerability level; relevant here? */
+
+    /* Buffer to hold the trampoline with which a patch point was hooked in
+     * order to execute a hot patch in hotp_only mode.  Should be NULL for
+     * regular hot patching, i.e., with fcache.  
+     */
+    byte *trampoline;
+
+    /* Pointer to the copy of app code that resides inside the trampoline, 
+     * that gets executed at the end of trampoline execution; this is the app 
+     * code that existed at the injection point.  Used only by hotp_only.
+     */
+    byte *app_code_copy;
+
+    /* Pointer to the cti inside the trampoline (the one that is used to 
+     * implement AFTER_INTERCEPT_LET_GO_ALT_DYN) that is used to 
+     * change control flow.  Used only in hotp_only mode for a patch point 
+     * that requests a control flow change, i.e., has non-zero return_addr.
+     */
+    byte *tramp_exit_cti;
+} hotp_patch_point_t;
+
+/* Experiments showed that the maximum size of a single interception 
+ * trampoline/hook is about 400 to 450 bytes, so 512 should be adequate.
+ */
+#define HOTP_ONLY_TRAMPOLINE_SIZE 512
+#define HOTP_ONLY_NUM_THREADS_AT_INIT -1
+
+/* A patch region size of 5 is used for hotp_only mode.  This is done so
+ * that the same vm_area_t vector (hotp_patch_point_areas) can be used for
+ * patch point overlap checks and address lookup.  Note: 5 is the minimum 
+ * bytes needed to encode/insert a direct jmp with 32-bit displacement, i.e.,
+ * a hook.
+ * For hotp in code cache, all patch regions are points, so patch region size
+ * 1 is used.  In this mode it is used only for patch address lookup.
+ *
+ * NOTE: Investigate issues when implementing hotp_only for native_exec dlls as
+ *       we would have to have regions with different sizes - might trigger a
+ *       few hotp asserts.
+ *
+ * Use -1 as an error catching value if this macro is used without -hot_patching.
+ */
+#define HOTP_ONLY_PATCH_REGION_SIZE     (5)
+#define HOTP_CACHE_PATCH_REGION_SIZE    (1)
+#define HOTP_BAD_PATCH_REGION_SIZE      (-1)
+#define HOTP_PATCH_REGION_SIZE              \
+    (DYNAMO_OPTION(hot_patching) ?          \
+        (DYNAMO_OPTION(hotp_only) ?         \
+            HOTP_ONLY_PATCH_REGION_SIZE :   \
+            HOTP_CACHE_PATCH_REGION_SIZE) : \
+        HOTP_BAD_PATCH_REGION_SIZE)
+
+/* This structure is used to define a hash value for a specified region
+ * around a patch point as decided by the hot patch writer.  This hash, which
+ * is provided by the hot patch writer will be used at run time as part of the
+ * mechanism to identify a given PE module for injecting hot patches.
+ */
+typedef struct {
+    /* Offset, relative to the base of the module, that should be used as the 
+     * starting point of hash computation string; for the module to be patched.
+     */
+    app_rva_t start;
+    uint len;           /* number of bytes to be used for hash computation */
+    uint hash_value;
+} hotp_patch_point_hash_t;
+
+typedef struct {
+    hotp_module_sig_t sig;
+    uint num_patch_points;
+    hotp_patch_point_t *patch_points;
+    uint num_patch_point_hashes;
+    hotp_patch_point_hash_t *hashes;
+
+    /* Data computed at run time; should be zeroed out at read time */
+    bool matched;           /* True if current module is loaded & matched. */
+    app_pc base_address;
+} hotp_module_t;
+
+typedef struct {
+    uint num_modules;
+    hotp_module_t *modules;
+} hotp_set_t;
+
+/* Note: status and statistics are kept in a separate structure to allow for
+ *       easy output, either via a file or via read only memory.
+ * Note: whole struct is runtime data; hence separated out.
+ */
+typedef struct {
+    hotp_exec_status_t exec_status;
+
+    /* Points to the one in hotp_policy_status_t to avoid duplication. */
+    hotp_inject_status_t *inject_status;
+
+    /* TODO: num_injected at the vulnerability level */
+    /* TODO: decide on the size of stats (uint or uint64) before finalizing 
+     *       interface with jim
+     */
+    uint64 num_detected;
+    uint64 num_not_detected;
+    uint64 num_detector_error;
+    uint64 num_protected;
+    uint64 num_not_protected;
+    uint64 num_kill_thread;
+    uint64 num_kill_process;
+    uint64 num_raise_exception;
+    uint64 num_change_control_flow;
+    uint64 num_protector_error;
+    uint64 num_aborted;
+} hotp_vul_info_t;
+
+/* The types are defined as unique bit flags because it may be possible in 
+ * future that we have a case that is more than one type.  
+ * For example a hot patch with a symbolic offset may be 
+ * HOTP_TYPE_SYMBOLIC_TYPE | HOTP_TYPE_HOT_PATCH, whereas a gbop hook may be 
+ * HOTP_TYPE_SYMBOLIC_TYPE | HOTP_TYPE_GBOP_HOOK.
+ */
+typedef enum {
+    /* This represents the patches that fix vulnerabilities, as described by
+     * the hot patch injection design.
+     */
+    HOTP_TYPE_HOT_PATCH = 0x1,
+
+    /* This represents all gbop hook points.  This type is different in that it:
+     * 1. Isn't specified by a config file; well not as of now (FIXME?),
+     * 2. Is specified by gbop_hooks and/or gbop_include_list (FIXME: NYI),
+     * 3. Can't be turned off by modes file; will not as of now (FIXME?),
+     * 4. Can be turned of by gbop_exclude_list (FIXME: NYI),
+     * 5. Uses a symbolic name rather than identifying the PE uniquely,
+     * 6. Has a generic detector and protector which is part of the core, and,
+     * 7. Uses the core defaults for events, actions, dumps & forensics 
+     *      (FIXME: NYI).
+     */
+    HOTP_TYPE_GBOP_HOOK = 0x2,
+
+    /* Currently will be exclusive with HOTP_TYPE_{HOT_PATCH,GBOP_HOOK},
+     * eventually will co-exist.
+     */
+    HOTP_TYPE_PROBE = 0x4
+} hotp_type_t;
+
+/* hotp_vul_t defines a vulnerability */
+/* the entire expanded structure of hotp_vul_t consists of constant data,
+ *  except for a couple of runtime data; this is so that policies can be
+ *  easily read in from file/memory in a binary format, thus eliminating the
+ *  need to do any data formatting/processing inside the core.
+ */
+typedef struct {
+    const char *vul_id;
+
+    /* policy_id is of the format XXXX.XXXX so that it can be used to
+     * generate the corresponding threat_id; so use
+     *      char policy_id[MAX_POLICY_ID_LENGTH + 1];
+     * to be consistent with hotp_policy_status_t;  TODO
+     * not done now because SET_STR doesn't handle arrays.
+     */
+    const char *policy_id;
+    uint policy_version;
+    const char *hotp_dll;
+    const char *hotp_dll_hash;
+    hotp_policy_mode_t mode;
+
+    uint num_sets;
+    hotp_set_t *sets;
+
+    /* Data computed at run time; should be zeroed out at read time */
+    hotp_vul_info_t *info;
+    app_pc  hotp_dll_base;
+    /* TODO: if policy data is going to be shared across multiple processes,
+     *       info (i.e., runtime data) can't be part of this; a parallel runtime
+     *       structure must be created;  not a big issue till hot patches reach
+     *       thousands in number
+     */
+
+    /* FIXME: right now this isn't specified by the config file because 
+     * config files are assumed to define only hotpatches.  Also, gbop_hooks 
+     * are added to the table by a different routine, so there is no room 
+     * for ambiguity.  If we decide to use the config file for all, then 
+     * this type should come from there - that would involve revving up 
+     * the hotp interface, i.e., engine version.
+     * Note: probe types are provided by client libraries directly via
+     *       dr_register_probes.
+     */
+    hotp_type_t type;
+
+    /* The following fields were introduced for probe api. */
+
+    /* Unique ID for each probe; must be unique across different clients in the
+     * same process to avoid one client from controlling another's probes. */
+    unsigned int id;
+} hotp_vul_t;
+
+/* Maintain a list of vulnerability tables so that they can be freed at exit
+ * time.  Nudge for policy reading creates new tables.  The old ones should
+ * be left alone so that races between hot patch execution and table freeing
+ * are avoided (case 5521).  All such tables are freed during DR exit. 
+ * FIXME: Release tables using a ref_count in case there are many & memory usage
+ *        is high.  It is highly unlikely that a given process will get more 
+ *        than a couple of policy read nudges during its lifetime.  
+ *        memory usage issue not correctness one, work on it after beta.
+ */
+typedef struct hotp_vul_tab_t {
+    hotp_vul_t *vul_tab;
+    uint num_vuls;
+    struct hotp_vul_tab_t *next;
+} hotp_vul_tab_t;
+
+/* TODO: for now this just has debug information; later on move all hot patch
+ * related globals into this structure.  The debug variable listed below needed
+ * to be updated during loader activity and that conflicts with our data segment
+ * protection.
+ */
+#ifdef DEBUG
+typedef struct hotp_globals_t {
+    /* The variables below help catch removing the same patch twice and 
+     * injecting it twice, which is ok only for loader safety.  Technically 
+     * each patch point should have this variable, but given that the loader 
+     * loads/relocates one dll at a time, this should be ok.
+     */
+    bool ldr_safe_hook_removal;     /* used only in -hotp_only mode */
+    bool ldr_safe_hook_injection;   /* used only in -hotp_only mode */
+} hotp_globals_t;
+#endif
+/*----------------------------------------------------------------------------*/
+/* Macro definitions. */
+
+/* These macros serve two purposes.  Firstly they provide a clean interface 
+ * to access the global hotp_vul_table, so that direct use of the global 
+ * variable can be avoided.  Secondly they improve readability; given that 
+ * these structures are nested, accessing a member directly would result in 
+ * long lines of code, which aren't very readable.
+ */
+/* TODO: Derek feels that these macros obfuscate the code rather than making 
+ *       them readable, which is opposite to what I thought.  Try using local
+ *       variables and if that looks good, remove these macros.
+ */
+#define VUL(vul_table, i)              (vul_table[i])
+#define SET(vul_table, v, i)           (VUL(vul_table, v).sets[i])
+#define MODULE(vul_table, v, s, i)     (SET(vul_table, v, s).modules[i])
+#define SIG(vul_table, v, s, m)        (MODULE(vul_table, v, s, m).sig)
+#define PPOINT(vul_table, v, s, m, i)  (MODULE(vul_table, v, s, m).patch_points[i])
+#define PPOINT_HASH(vul_table, v, s, m, i)  \
+    (MODULE(vul_table, v, s, m).hashes[i])
+
+#define NUM_GLOBAL_VULS                 (hotp_num_vuls)
+#define GLOBAL_VUL_TABLE                (hotp_vul_table)
+#define GLOBAL_VUL(i)                   VUL(GLOBAL_VUL_TABLE, i)
+#define GLOBAL_SET(v, i)                SET(GLOBAL_VUL_TABLE, v, i)
+#define GLOBAL_MODULE(v, s, i)          MODULE(GLOBAL_VUL_TABLE, v, s,i)
+#define GLOBAL_SIG(v, s, m)             SIG(GLOBAL_VUL_TABLE, v, s, m)
+#define GLOBAL_PPOINT(v, s, m, i)       PPOINT(GLOBAL_VUL_TABLE, v, s, m, i)
+#define GLOBAL_HASH(v, s, m, i)         PPOINT_HASH(GLOBAL_VUL_TABLE, v, s, m, i)
+
+/* TODO: change it to model ATOMIC_ADD; can't use ATOMIC_ADD directly because
+ *       it wants only uint, not uint64 which is what all vulnerability stats
+ *       are;  maybe the easy way is to make the vul stat uint, but don't know
+ *       if that will result in overflows fairly quickly, esp. for long running
+ *       apps.  either way, make this increment non racy, the users of this 
+ *       macro assume atomic increments.
+ */
+#define VUL_STAT_INC(x)     ((x)++);
+
+#define SET_NUM(var, type, limit_str, input_ptr)                            \
+{                                                                           \
+    char *str = hotp_get_next_str(&(input_ptr));                            \
+    char *hex_fmt, *dec_fmt;                                                \
+    type temp;                                                              \
+                                                                            \
+    ASSERT(sizeof(type) == sizeof(uint) || sizeof(type) == sizeof(uint64)); \
+    hex_fmt = (sizeof(type) == sizeof(uint)) ? "0x%x" : "0x" HEX64_FORMAT_STRING; \
+    dec_fmt = (sizeof(type) == sizeof(uint)) ? "%d" : INT64_FORMAT_STRING;  \
+                                                                            \
+    if (sscanf(str, hex_fmt, &temp) == 1 || sscanf(str, dec_fmt, &temp) == 1) { \
+        if (temp < (type)(MIN_##limit_str) ||                               \
+            temp > (type)(MAX_##limit_str))                                 \
+            goto error_reading_policy;  /* Range error */                   \
+        (var) = temp;                                                       \
+    }                                                                       \
+    else                                                                    \
+        goto error_reading_policy;      /* Parse error. */                  \
+}
+
+/* FIXME: range check strs for min & max length; null check already done. */
+#define SET_STR_DUP(var, input_ptr)                                    \
+{                                                                      \
+    char *str = hotp_get_next_str(&(input_ptr));                       \
+                                                                       \
+    if (str == NULL)                                                   \
+        goto error_reading_policy;                                     \
+    (var) = dr_strdup(str HEAPACCT(ACCT_HOT_PATCHING));                  \
+}
+
+#define SET_STR_PTR(var, input_ptr)                                    \
+{                                                                      \
+    char *str = hotp_get_next_str(&(input_ptr));                       \
+                                                                       \
+    if (str == NULL)                                                   \
+        goto error_reading_policy;                                     \
+    (var) = str;                                                       \
+}
+
+#define SET_STR(var, input_ptr) SET_STR_DUP(var, input_ptr)
+
+#define HOTP_IS_IN_REGION(region_start, region_size, addr) \
+    (((addr) >= (region_start)) && ((addr) < ((region_start) + (region_size))))
+
+/* This checks addresses. */
+#define HOTP_ONLY_IS_IN_TRAMPOLINE(ppoint, addr)               \
+    (((ppoint)->trampoline == NULL || (addr) == NULL) ? false : \
+        HOTP_IS_IN_REGION((ppoint)->trampoline, HOTP_ONLY_TRAMPOLINE_SIZE, addr))
+
+/* This checks offsets/RVAs. */
+#define HOTP_ONLY_IS_IN_PATCH_REGION(ppoint, addr)   \
+    (((ppoint)->offset <= 0 || (addr) <= 0) ? false : \
+        HOTP_IS_IN_REGION((ppoint)->offset, HOTP_PATCH_REGION_SIZE, addr))
+
+/* TODO: PR 225550 - make this a better function so that each probe is 
+ * identified uniquely so as to prevent clients from modifying each others' 
+ * probes - make it a function of the client name, probe def & this counter.
+ * Note: probe id is generated outside hotp_vul_table_lock because of having
+ *       to load probe/callback dlls without hitting dr hooks, so updates to
+ *       probe_id_counter haver to be atomic.
+ */
+#define GENERATE_PROBE_ID() (atomic_add_exchange_int((int*)&probe_id_counter, 4))
+/*----------------------------------------------------------------------------*/
+/* Local function prototypes. */
+static void hotp_change_control_flow(const hotp_context_t *app_reg_ptr, 
+                                     const app_pc target);
+
+static after_intercept_action_t hotp_gateway(const hotp_vul_t *vul_tab,
+                                           const uint num_vuls, 
+                                           const uint vul_index, 
+                                           const uint set_index, 
+                                           const uint module_index, 
+                                           const uint ppoint_index, 
+                                           hotp_context_t *app_reg_ptr, 
+                                           const bool own_hot_patch_lock);
+
+static void hotp_free_vul_table(hotp_vul_t *tab, uint num_vuls_alloc);
+static hotp_exec_status_t hotp_execute_patch(hotp_func_t hotp_fn_ptr, 
+                                             hotp_context_t *hotp_cxt,
+                                             hotp_policy_mode_t mode,
+                                             bool dump_excpt_info,
+                                             bool dump_error_info);
+static void hotp_update_vul_stats(const hotp_exec_status_t exec_status, 
+                                  const uint vul_index);
+static void hotp_event_notify(hotp_exec_status_t exec_status, bool protected,
+                              const hotp_offset_match_t *inject_point, 
+                              const app_pc bad_addr,
+                              const hotp_context_t *hotp_cxt);
+#if defined(DEBUG) && defined(INTERNAL)
+static void hotp_dump_reg_state(const hotp_context_t *reg_state, 
+                                const app_pc eip, const uint loglevel);
+#endif
+static void hotp_only_inject_patch(const hotp_offset_match_t *ppoint_desc, 
+                                   const thread_record_t **all_threads, 
+                                   const int num_threads);
+static void hotp_only_remove_patch(const hotp_module_t *module, 
+                                   hotp_patch_point_t *cur_ppoint);
+after_intercept_action_t hotp_only_gateway(app_state_at_intercept_t *state);
+static uint hotp_compute_hash(app_pc base, hotp_patch_point_hash_t *hash);
+#ifdef GBOP
+static void
+hotp_only_read_gbop_policy_defs(hotp_vul_t *tab, uint *num_vuls);
+#endif
+
+/* TODO: add function protoypes for all functions in this file */
+/*----------------------------------------------------------------------------*/
+/* Local data. */
+
+hotp_policy_status_table_t *hotp_policy_status_table;
+
+/* FIXME: create hotp_vul_table_t and put these three into it */
+static hotp_vul_t *hotp_vul_table; 
+static uint hotp_num_vuls; 
+static hotp_vul_tab_t *hotp_old_vul_tabs;
+
+DECLARE_CXTSWPROT_VAR(static read_write_lock_t hotp_vul_table_lock, {{0}});
+/* Special heap for hotp_only trampolines; heap is executable. */
+static void *hotp_only_tramp_heap;
+/* Leak to handle case 9593.  This should go if we find a cleaner solution. */
+#if defined(DEBUG) && defined(HEAP_ACCOUNTING)
+DECLARE_NEVERPROT_VAR(int hotp_only_tramp_bytes_leaked, 0);
+#endif
+/* This is used to cache hotp_only_tramp_heap for handling leak asserts during
+ * detach and to track whether or not any hotp_only patch was removed.  Case
+ * 9593 & PR 215520. */
+static void *hotp_only_tramp_heap_cache;
+
+/* Trampoline area vector; currently used only to identify if a thread is in
+ * the middle of hot patch execution during suspension - for multiprocessor
+ * safe hot patch removal in hotp_only mode.
+ * Kept on the heap for selfprot (case 7957).
+ */
+static vm_area_vector_t *hotp_only_tramp_areas;
+
+/* This has all the matched patch points, i.e., patch points that have been 
+ * determined by hotp_process_image() to be ready to be injected.  Only that 
+ * function adds or removes from this vector because only that function does
+ * module matching. 
+ * The custom data stored is a hotp_offset_match_t structure which describes
+ * the patch point precisely in the GLOBAL_VUL_TABLE.
+ * For hotp_only this refers to all injected patches because they get injected 
+ * during match/dll load time.  For fcache based hot patches, this may or may 
+ * not specify patch injection, but will specify matches.  This is because for
+ * hotp_only matching & injection are done in one shot, whereas they are split
+ * for fcache based hot patches.
+ * This vector is not static, it is on the heap because of selfprot; case 8074.
+ * Uses:
+ *  1. for hotp_only to solve the overlapping hashes problem (case 7279).
+ *  2. for offset lookup for both hotp and hotp_only (case 8132).
+ *  3. NYI - all patch removal & injection; perscache stuff (case 10728).
+ */
+static vm_area_vector_t *hotp_patch_point_areas;
+
+#ifdef DEBUG
+static hotp_globals_t *hotp_globals;
+#endif
+
+#ifdef CLIENT_INTERFACE
+/* Global counter used to generate unique ids for probes.  This is updated
+ * atomically and isn't guarded by any lock.  See GENERATE_PROBE_ID() for
+ * details.
+ */
+static unsigned int probe_id_counter;
+#endif
+/*----------------------------------------------------------------------------*/
+/* Function definitions. */
+
+/* Don't expose the hot patch lock directly outside this module. */
+read_write_lock_t *
+hotp_get_lock(void)
+{
+    ASSERT(DYNAMO_OPTION(hot_patching));
+    return &hotp_vul_table_lock;
+}
+
+static inline app_pc
+hotp_ppoint_addr(const hotp_module_t *module, const hotp_patch_point_t *ppoint)
+{
+    app_pc ppoint_offset;
+    ASSERT(module != NULL && ppoint != NULL);
+    ASSERT(module->base_address != NULL && ppoint->offset != 0);
+
+    ppoint_offset = module->base_address + ppoint->offset;
+
+    /* The patch point should be inside the code section of a loaded module. */
+    ASSERT(is_in_code_section(module->base_address, ppoint_offset, NULL, NULL));
+
+    return ppoint_offset;
+}
+
+static void 
+hotp_ppoint_areas_add(hotp_offset_match_t *ppoint_desc)
+{
+    hotp_module_t *module;
+    hotp_patch_point_t *ppoint;
+    hotp_offset_match_t *copy;
+    app_pc ppoint_start, ppoint_end;
+
+    ASSERT(ppoint_desc != NULL);
+    ASSERT(GLOBAL_VUL_TABLE != NULL && hotp_patch_point_areas != NULL);
+    ASSERT_OWN_READWRITE_LOCK(true, &hotp_vul_table_lock);
+
+    module = &GLOBAL_MODULE(ppoint_desc->vul_index, ppoint_desc->set_index,
+                            ppoint_desc->module_index);
+    ppoint = &module->patch_points[ppoint_desc->ppoint_index];
+
+    /* Shouldn't be adding to hotp_patch_point_areas if the module hasn't been 
+     * matched.
+     */
+    ASSERT(module->matched);
+    ppoint_start = hotp_ppoint_addr(module, ppoint);
+    ppoint_end = ppoint_start + HOTP_PATCH_REGION_SIZE;
+
+    /* Each matched (or injected) patch point should be added only 
+     * once and removed only once, so before adding, make sure that it 
+     * is not already in there. 
+     */
+    ASSERT(!vmvector_overlap(hotp_patch_point_areas, ppoint_start, ppoint_end));
+
+    copy = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, hotp_offset_match_t,
+                           ACCT_HOT_PATCHING, PROTECTED);
+    *copy = *ppoint_desc;
+    vmvector_add(hotp_patch_point_areas, ppoint_start, ppoint_end, (void *)copy);
+}
+
+static void
+hotp_ppoint_areas_remove(app_pc pc)
+{
+    hotp_offset_match_t *ppoint_desc;
+    DEBUG_DECLARE(bool ok;)
+
+    ASSERT(pc != NULL);
+    ASSERT(GLOBAL_VUL_TABLE != NULL && hotp_patch_point_areas != NULL);
+    ASSERT_OWN_READWRITE_LOCK(true, &hotp_vul_table_lock);
+
+    ppoint_desc = (hotp_offset_match_t *)
+                  vmvector_lookup(hotp_patch_point_areas, pc); 
+
+    DODEBUG({
+        hotp_module_t *module;
+        hotp_patch_point_t *ppoint;
+
+        /* Shouldn't be trying to remove something that wasn't added. */
+        ASSERT(ppoint_desc != NULL);
+
+        /* Verify that the ppoint_desc in the vmvector corresponds to pc. */
+        module = &GLOBAL_MODULE(ppoint_desc->vul_index, ppoint_desc->set_index,
+                                ppoint_desc->module_index);
+        ppoint = &module->patch_points[ppoint_desc->ppoint_index];
+        ASSERT(pc == hotp_ppoint_addr(module, ppoint));
+    });
+
+    HEAP_TYPE_FREE(GLOBAL_DCONTEXT, ppoint_desc, hotp_offset_match_t,
+                   ACCT_HOT_PATCHING, PROTECTED);
+
+    DEBUG_DECLARE(ok = )
+        vmvector_remove(hotp_patch_point_areas, pc,
+                        pc + HOTP_PATCH_REGION_SIZE); 
+    ASSERT(ok);
+}
+
+static void 
+hotp_ppoint_areas_release(void)
+{
+    app_pc vm_start, vm_end;
+    hotp_offset_match_t *ppoint_desc;
+    vmvector_iterator_t iterator;
+
+    ASSERT_OWN_READWRITE_LOCK(true, &hotp_vul_table_lock);
+
+    /* Release all patch point descriptors. */
+    vmvector_iterator_start(hotp_patch_point_areas, &iterator);
+    while (vmvector_iterator_hasnext(&iterator)) {
+        ppoint_desc = vmvector_iterator_next(&iterator, &vm_start, &vm_end);
+        ASSERT(ppoint_desc != NULL);
+        HEAP_TYPE_FREE(GLOBAL_DCONTEXT, ppoint_desc, hotp_offset_match_t,
+                       ACCT_HOT_PATCHING, PROTECTED);
+    }
+    vmvector_iterator_stop(&iterator);
+
+    /* Remove all vm_areas in the vmvector. */
+    vmvector_remove(hotp_patch_point_areas, UNIVERSAL_REGION_BASE, UNIVERSAL_REGION_END);
+    ASSERT(vmvector_empty(hotp_patch_point_areas));
+}
+
+/* Used to read in vulnerability definitions from file */
+static char*
+hotp_get_next_str(char **start)
+{
+    char *end = *start, *temp = *start;
+    bool dos_line_terminator = false;
+
+    if (start == NULL || *start == NULL)
+        return NULL;
+
+    while (*end != '\n' && *end != '\r' && *end != '\0')
+        end++;
+
+    if (*end != '\0') {
+        if (*end == '\r') {
+            if (*(end + 1) == '\n')
+                dos_line_terminator = true;
+            else
+                SYSLOG_INTERNAL_WARNING("Mac OS style line separator!");
+        }
+
+        *end++ = '\0';
+        if (dos_line_terminator)
+            end++;
+    }
+    *start = end;
+    return temp;
+}
+
+/* Used to read either the policy file or the modes file. */  
+enum {
+    POLICY_FILE = 1,
+    MODES_FILE
+};
+
+static char *
+hotp_read_data_file(uint type, size_t *buf_len /* OUT */)
+{
+    int retval;
+    char file[MAXIMUM_PATH];
+
+    ASSERT(type == POLICY_FILE || type == MODES_FILE);
+    ASSERT(buf_len != NULL);
+
+    *buf_len = 0;
+
+    retval = get_parameter(type == POLICY_FILE ?
+                               L_IF_WIN(DYNAMORIO_VAR_HOT_PATCH_POLICIES) :
+                               L_IF_WIN(DYNAMORIO_VAR_HOT_PATCH_MODES), 
+                           file, BUFFER_SIZE_ELEMENTS(file));
+    if (IS_GET_PARAMETER_FAILURE(retval)) {
+        SYSLOG_INTERNAL_WARNING("Can't find %s definition directory name.",
+                                (type == POLICY_FILE) ? "policy" : "mode");
+        return NULL;
+    }
+
+    /* The {defs,modes} file is
+     * $DYNAMORIO_HOT_PATCH_POLICIES/<engine>/HOTP_{POLICIES,MODES}_FILENAME
+     */
+    CAT_AND_TERMINATE(file, "\\" STRINGIFY(HOTP_INTERFACE_VERSION) "\\");
+    CAT_AND_TERMINATE(file, type == POLICY_FILE ? HOTP_POLICIES_FILENAME : 
+                                                  HOTP_MODES_FILENAME);
+    
+    LOG(GLOBAL, LOG_HOT_PATCHING, 1, "Hot patch %s definition file: %s\n",
+        (type == POLICY_FILE) ? "policy" : "mode", file);
+
+    return read_entire_file(file, buf_len HEAPACCT(ACCT_HOT_PATCHING));
+}
+
+/* On a successful read, this should return a valid pointer to a vulnerability
+ * table and modify the size argument passed to it.  If it fails, it should 
+ * dump a log event, return NULL & not modify the size.
+ *
+ * The caller should release the old table & make the return value the new
+ * table; the reason for doing this table swap outside this function is to
+ * allow (future work) identification of vulnerabilities that have actually
+ * changed; from this set of changed vulnerabilities, identify those that have 
+ * been injected and flush only those (an optimization issue).
+ *
+ * Policy file format: (indentations don't appear in the actual file, they exist
+ * here to illustrate the format & to show where multiple data can occur; also 
+ * format is close to binary as it is now)
+ * All integers/hex_numbers are 32-bits unless explicitly stated otherwise.
+
+<engine_version-str>
+<num_vulnerabilities-decimal_integer>
+  <vulnerability_id-str>
+  <policy_id-str>
+  <version-decimal_integer>
+  <hotpatch_dll-str>
+  <hotpatch_dll_hash-str>
+  <num_sets-decimal_integer>
+    <num_modules-decimal_integer>
+    <pe_name-str>
+    <pe_timestamp-hex_number>
+    <pe_checksum-hex_number>
+    <pe_image_size-hex_number>
+    <pe_code_size-hex_number>
+    <pe_file_version-hex_number-64_bits>
+    <num_hashes-decimal_integer>
+      <start-hex_number>
+      <length-hex_number>
+      <hash-decimal_integer>
+    <num_patch_points-decimal_integer>
+      <offset-hex_number>
+      <precedence-decimal_integer>
+      <detector_offset-hex_number>
+      <protector_offset-hex_number>
+      <return_addr-hex_number>
+
+ * TODO: all unused fields, i.e., runtime fields in the data structures should
+ *       be set to NULL/0 to avoid any assumption violations down stream.
+ * TODO: after reading in the vulnerability data, that region should be write 
+ *       protected
+ */
+static hotp_vul_t *
+hotp_read_policy_defs(uint *num_vuls_read)
+{   
+    hotp_vul_t *tab = NULL;
+    uint hotp_interface_version;
+    uint vul = 0, set, module, hash, ppoint;
+    uint num_vuls = 0, num_vuls_alloc = 0;
+    char *buf = NULL;  /* TODO: for now only; will go after file mapping */
+    size_t buf_len = 0;
+    char *start = NULL;
+    DEBUG_DECLARE(bool started_parsing = false;)
+
+    /* Read the config file only if -liveshields is turned on.  If it isn't
+     * turned on, read gbop hooks if -gbop is specified.
+     */
+    if (!DYNAMO_OPTION(liveshields)) {
+#ifdef GBOP
+        if (DYNAMO_OPTION(gbop))
+            goto read_gbop_only;
+#endif
+        return NULL;
+    }
+
+    buf = hotp_read_data_file(POLICY_FILE, &buf_len);
+    if (buf == NULL) {
+        ASSERT(buf_len == 0);
+        goto error_reading_policy;
+    } else {
+        ASSERT(buf_len > 0);
+        ASSERT_CURIOSITY(buf_len < MAX_POLICY_FILE_SIZE);
+    }
+
+    start = buf;
+    DEBUG_DECLARE(started_parsing = true;)
+    SET_NUM(hotp_interface_version, uint, HOTP_INTERFACE_VERSION, start);
+    SET_NUM(num_vuls, uint, NUM_VULNERABILITIES, start);
+#ifdef GBOP
+    if (DYNAMO_OPTION(gbop))
+        num_vuls_alloc = gbop_get_num_hooks();
+#endif
+    num_vuls_alloc += num_vuls;
+    ASSERT(num_vuls_alloc > 0 && num_vuls_alloc <= MAX_NUM_VULNERABILITIES);
+
+    /* Zero out all dynamically allocated hotpatch table structures to avoid
+     * leaks when there is a parse error.  See case 8272, 9045.
+     */
+    tab = HEAP_ARRAY_ALLOC_MEMSET(GLOBAL_DCONTEXT, hotp_vul_t, num_vuls_alloc,
+                                  ACCT_HOT_PATCHING, PROTECTED, 0); 
+
+    for (vul = 0; vul < num_vuls; vul++) {
+        /* FIXME: bounds checking; length should be > 2 && < 32; not null */
+        SET_STR(VUL(tab, vul).vul_id, start);
+        SET_STR(VUL(tab, vul).policy_id, start);
+        SET_NUM(VUL(tab, vul).policy_version, uint, POLICY_VERSION, start);
+
+        /* FIXME: strdup strings because the buffer/mapped file will be deleted 
+         *        after processing; don't use strdup though!
+         *        works right now till the next time I read in a policy file
+         *        into buf[]; if that read fails the old data will be corrupt!
+         *        remember, if not strdup'ed, all strings are in writable memory
+         */
+        SET_STR(VUL(tab, vul).hotp_dll, start);
+        SET_STR(VUL(tab, vul).hotp_dll_hash, start);
+        SET_NUM(VUL(tab, vul).num_sets, uint, NUM_SETS, start);
+
+        /* Initialize all runtime values in the structure. */
+        VUL(tab, vul).mode = HOTP_MODE_OFF;     /* Fix for case 5326. */
+        VUL(tab, vul).type = HOTP_TYPE_HOT_PATCH;
+
+        VUL(tab, vul).sets = HEAP_ARRAY_ALLOC_MEMSET(GLOBAL_DCONTEXT, 
+                                 hotp_set_t, VUL(tab, vul).num_sets,
+                                 ACCT_HOT_PATCHING, PROTECTED, 0);
+        VUL(tab, vul).info = HEAP_ARRAY_ALLOC_MEMSET(GLOBAL_DCONTEXT, 
+                                 hotp_vul_info_t, 1, ACCT_HOT_PATCHING,
+                                 PROTECTED, 0);
+
+        for (set = 0; set < VUL(tab, vul).num_sets; set++) {
+            SET_NUM(SET(tab, vul, set).num_modules, uint, NUM_MODULES, start);
+            SET(tab, vul, set).modules = HEAP_ARRAY_ALLOC_MEMSET(GLOBAL_DCONTEXT,
+                                             hotp_module_t,
+                                             SET(tab, vul, set).num_modules, 
+                                             ACCT_HOT_PATCHING, PROTECTED, 0);
+            for (module = 0; module < SET(tab, vul, set).num_modules;
+                 module++) {
+                SET_STR(SIG(tab, vul, set, module).pe_name, start);
+                SET_NUM(SIG(tab, vul, set, module).pe_timestamp, uint,
+                        PE_TIMESTAMP, start);
+                SET_NUM(SIG(tab, vul, set, module).pe_checksum, uint,
+                        PE_CHECKSUM, start);
+                SET_NUM(SIG(tab, vul, set, module).pe_image_size, uint,
+                        PE_IMAGE_SIZE, start);
+                SET_NUM(SIG(tab, vul, set, module).pe_code_size, uint,
+                        PE_CODE_SIZE, start);
+                SET_NUM(SIG(tab, vul, set, module).pe_file_version, uint64,
+                        PE_FILE_VERSION, start);
+
+                /* Initialize all runtime values in the structure. */
+                MODULE(tab, vul, set, module).matched = false;
+                MODULE(tab, vul, set, module).base_address = NULL;
+
+                SET_NUM(MODULE(tab, vul, set, module).num_patch_point_hashes,
+                        uint, NUM_PATCH_POINT_HASHES, start);
+                MODULE(tab, vul, set, module).hashes =
+                    HEAP_ARRAY_ALLOC_MEMSET(GLOBAL_DCONTEXT,
+                        hotp_patch_point_hash_t,
+                        MODULE(tab, vul, set, module).num_patch_point_hashes,
+                        ACCT_HOT_PATCHING, PROTECTED, 0);
+
+                for (hash = 0;
+                     hash < MODULE(tab, vul, set, module).num_patch_point_hashes;
+                     hash++) {
+                    SET_NUM(PPOINT_HASH(tab, vul, set, module, hash).start,
+                            app_rva_t, HASH_START_OFFSET, start);
+                    SET_NUM(PPOINT_HASH(tab, vul, set, module, hash).len, 
+                            uint, HASH_LENGTH, start);
+                    SET_NUM(PPOINT_HASH(tab, vul, set, module, hash).hash_value,
+                            uint, HASH_VALUE, start);
+                }
+
+                SET_NUM(MODULE(tab, vul, set, module).num_patch_points, uint,
+                        NUM_PATCH_POINTS, start);
+                MODULE(tab, vul, set, module).patch_points =
+                    HEAP_ARRAY_ALLOC_MEMSET(GLOBAL_DCONTEXT, hotp_patch_point_t,
+                        MODULE(tab, vul, set, module).num_patch_points, 
+                        ACCT_HOT_PATCHING, PROTECTED, 0);
+
+                for (ppoint = 0;
+                     ppoint < MODULE(tab, vul, set, module).num_patch_points;
+                     ppoint++) {
+                    SET_NUM(PPOINT(tab, vul, set, module, ppoint).offset,
+                            app_rva_t, PATCH_OFFSET, start);
+                    SET_NUM(PPOINT(tab, vul, set, module, ppoint).precedence, 
+                            uint, PATCH_PRECEDENCE, start);
+                    SET_NUM(PPOINT(tab, vul, set, module, ppoint).detector_fn, 
+                            app_rva_t, DETECTOR_OFFSET, start);
+
+                    /* Both protector and return address can be NULL */
+                    SET_NUM(PPOINT(tab, vul, set, module, ppoint).protector_fn, 
+                            app_rva_t, PROTECTOR_OFFSET, start);
+                    SET_NUM(PPOINT(tab, vul, set, module, ppoint).return_addr, 
+                            app_rva_t, RETURN_ADDR, start);
+                    PPOINT(tab, vul, set, module, ppoint).trampoline = NULL;
+                    PPOINT(tab, vul, set, module, ppoint).app_code_copy = NULL;
+                    PPOINT(tab, vul, set, module, ppoint).tramp_exit_cti = NULL;
+                }
+            }
+        }
+    }
+
+#ifdef GBOP
+    if (DYNAMO_OPTION(gbop)) {
+        hotp_only_read_gbop_policy_defs(tab, &num_vuls /* IN OUT arg */);
+        ASSERT(num_vuls_alloc == num_vuls);
+    }
+#endif
+    *num_vuls_read = num_vuls;
+    LOG(GLOBAL, LOG_HOT_PATCHING, 1,
+        "read %d vulnerability definitions\n", num_vuls);
+    heap_free (GLOBAL_DCONTEXT, buf, buf_len HEAPACCT(ACCT_HOT_PATCHING));
+    return tab;
+
+ error_reading_policy:
+    /* TODO: log error, free stuff, set tab to null, leave size intact & exit
+     *       for now just assert to make sure that bugs don't escape.
+     */
+    /* TODO: provide line #, not offset; alex couldn't use the offset */
+    SYSLOG_INTERNAL_WARNING("Error reading or parsing hot patch definitions");
+    /* Need this curiosity to make qa notice; the warning is handy for
+     * development testing only.  No hot patching on Linux, so don't assert.
+     * FIXME: Convert to assert after case 9066 has been fixed & tested.
+     * Note: Warn for missing file, but assert for parsing error; latter is 
+     * bug, former may just be a hotpatch-less installation - mostly coredev.
+     */
+    IF_WINDOWS(ASSERT_CURIOSITY(!started_parsing &&
+                              "Error parsing hot patch definitions");)
+    *num_vuls_read = 0;
+    if (tab != NULL) {
+        ASSERT(num_vuls_alloc > 0);
+        /* If gbop is on, then during a parse error num_vuls (parsed) must be
+         * less than num_vuls_alloc because if table as been allocated space
+         * has been allocated for gbop entries as well which wouldn't have been
+         * read on a parse error.  It is read after this point; see below.
+         */
+        IF_GBOP(ASSERT(!DYNAMO_OPTION(gbop) || num_vuls < num_vuls_alloc);)
+        /* On error free the whole table, not just what was read; case 9044. */
+        hotp_free_vul_table(tab, num_vuls_alloc);
+        tab = NULL;
+    }
+
+    /* buf can be allocated even if vulnerability table hasn't been allocated.
+     * See case 8332.
+     */
+    if (buf != NULL) {
+        ASSERT(buf_len > 0);
+        heap_free(GLOBAL_DCONTEXT, buf, buf_len HEAPACCT(ACCT_HOT_PATCHING));
+        LOG(GLOBAL, LOG_HOT_PATCHING, 1,
+            "error reading vulnerability file at offset "SZFMT"\n",
+            (ptr_uint_t)(start - buf));
+    }
+
+#ifdef GBOP
+    /* Even if we couldn't read the hot patch policies, we should still 
+     * allocate a new table and read in the gbop hooks.
+     */
+ read_gbop_only:
+    if (DYNAMO_OPTION(gbop)) {
+        num_vuls_alloc = gbop_get_num_hooks();
+        ASSERT(num_vuls_alloc > 0 && num_vuls_alloc <= MAX_NUM_VULNERABILITIES);
+        num_vuls = 0;
+
+        tab = HEAP_ARRAY_ALLOC(GLOBAL_DCONTEXT, hotp_vul_t, num_vuls_alloc,
+                               ACCT_HOT_PATCHING, PROTECTED);
+        hotp_only_read_gbop_policy_defs(tab, &num_vuls /* IN OUT arg */);
+
+        ASSERT(num_vuls_alloc == num_vuls);
+        *num_vuls_read = num_vuls;
+    }
+#endif
+
+    return tab;
+}
+
+/* TODO: An efficiency issue: don't load all hot patch dlls unless the mode
+ *       for at least one corresponding policy is detect or protect; this will
+ *       avoid loading all hot patch dlls whether they are used or not.  Note:
+ *       this is still eager loading as per the design.
+ */
+static void
+hotp_load_hotp_dlls(hotp_vul_t *vul_tab, uint num_vuls)
+{
+    uint vul;
+    int retval;
+    /* TODO: these arrays are large so make them static with a lock to avoid
+     *        a potential runtime stack overflow.
+     */
+    char hotp_dll_path[MAXIMUM_PATH];
+    char hotp_dll_cache[MAXIMUM_PATH];
+
+    /* Only liveshields need to know DYNAMORIO_HOME, probes give full paths. */
+    if (DYNAMO_OPTION(liveshields)) {
+        /* If null or non-existent hotp_dll_cache directory raise error log, 
+         * disable all associated vuls?   We are going to assert/log if we can't
+         * find the dll (below) anyway.
+         */
+        retval = get_parameter(L_IF_WIN(DYNAMORIO_VAR_HOME), hotp_dll_cache, 
+                               BUFFER_SIZE_ELEMENTS(hotp_dll_cache));
+        if (IS_GET_PARAMETER_FAILURE(retval)) {
+            SYSLOG_INTERNAL_WARNING("Can't read %s.  Hot patch dll loading "
+                                    "failed; hot patching won't work.",
+                                    DYNAMORIO_VAR_HOME);
+            return;
+        }
+    } else {
+#ifdef CLIENT_INTERFACE
+        ASSERT(DYNAMO_OPTION(probe_api));
+#endif
+    }
+
+    /* Compute dll cache path, i.e., $DYNAMORIO_HOME/lib/hotp/<engine>/ */
+    NULL_TERMINATE_BUFFER(hotp_dll_cache);
+    CAT_AND_TERMINATE(hotp_dll_cache, HOT_PATCHING_DLL_CACHE_PATH);
+    CAT_AND_TERMINATE(hotp_dll_cache, STRINGIFY(HOTP_INTERFACE_VERSION) "\\");
+
+    for (vul = 0; vul < num_vuls; vul++) {
+        /* Hot patch dlls exist only for the type hot_patch and probe, not for
+         * gbop hooks; well, not at least for now.
+         */
+        if (!TESTANY(HOTP_TYPE_HOT_PATCH | HOTP_TYPE_PROBE,
+                     VUL(vul_tab, vul).type)) {
+            ASSERT(TESTALL(HOTP_TYPE_GBOP_HOOK, VUL(vul_tab, vul).type));
+            /* TODO: also assert that the base is dynamorio.dll & remediator
+             * offsets are what they should be - use a DODEBUG
+             */
+            continue;
+        }
+
+        if (VUL(vul_tab, vul).hotp_dll_base == NULL) {    /* Not loaded yet. */
+            ASSERT(TESTANY(HOTP_TYPE_HOT_PATCH | HOTP_TYPE_PROBE,
+                           VUL(vul_tab, vul).type));
+            ASSERT(VUL(vul_tab, vul).hotp_dll != NULL);
+
+            /* Liveshields give just the base name which is used to compute
+             * full path, i.e., DYNAMORIO_HOME/lib/hotp/hotp_dll. */
+            if (TEST(HOTP_TYPE_HOT_PATCH, VUL(vul_tab, vul).type)) { 
+                strncpy(hotp_dll_path, hotp_dll_cache, 
+                        BUFFER_SIZE_ELEMENTS(hotp_dll_path) - 1);
+                NULL_TERMINATE_BUFFER(hotp_dll_path);
+
+                /* Hot patch dll names should just be base names; with no / or \. */
+                ASSERT(strchr(VUL(vul_tab, vul).hotp_dll, '\\') == NULL && 
+                       strchr(VUL(vul_tab, vul).hotp_dll, '/') == NULL);
+                strncat(hotp_dll_path, VUL(vul_tab, vul).hotp_dll,
+                        BUFFER_SIZE_ELEMENTS(hotp_dll_path) -
+                        strlen(hotp_dll_path) - 1);
+            } else {
+                /* Probe api calls provide full path to hotp dlls. */
+                strncpy(hotp_dll_path, VUL(vul_tab, vul).hotp_dll,
+                        BUFFER_SIZE_ELEMENTS(hotp_dll_path) - 1);
+            }
+            NULL_TERMINATE_BUFFER(hotp_dll_path);
+            ASSERT(strlen(hotp_dll_path) < BUFFER_SIZE_ELEMENTS(hotp_dll_path));
+
+            /* TODO: check if file exists; if not log, turn off associated
+             *       vulnerabilities & bail out; need to think through the
+             *       error exit mechanism while reading polcy-{defs,modes}.
+             */
+
+            /* FIXME: currently our loadlibrary hits our own syscall_while_native
+             * hook and goes to dispatch, which expects protected data sections.
+             * Once we have our own loader we can remove this.
+             */
+            VUL(vul_tab, vul).hotp_dll_base = 
+                load_shared_library(hotp_dll_path);
+        
+            /* TODO: if module base is null, raise a log event, mark vul as not
+             *       usable (probably a new status) and move on; for now just
+             *       assert.
+             */
+            /* TODO: assert that all detector_fn & protector_fn offsets 
+             *       associated with this hotp_dll actually lie within its
+             *       text space.
+             */
+            if (VUL(vul_tab, vul).hotp_dll_base == NULL) {
+                LOG(GLOBAL, LOG_HOT_PATCHING, 1,
+                    "unable to load hotp_dll: %s\n", hotp_dll_path);
+                ASSERT(VUL(vul_tab, vul).hotp_dll_base != NULL);
+            }
+            LOG(GLOBAL, LOG_HOT_PATCHING, 1, "loaded hotp_dll: %s at "PFX"\n",
+                hotp_dll_path, VUL(vul_tab, vul).hotp_dll_base);
+
+            /* TODO: this one must be done asap; add the hot patch dll's text
+             *       either to a new vm_area_vector_t or executable_vm_areas;
+             *       check with the team first. case 5381
+             *  add_executable_vm_area(hotp_dll_text_base, size_of_text,
+             *                         VM_UNMOD_IMAGE, false
+                                       _IF_DEBUG("hot patch dll loading"));
+             */
+            /* TODO: assert that hotp_dll's dllmain is null to prevent control
+             *       flow from going there during the thread creation due to
+             *       nudge; but how?
+             */
+        }
+    }
+}
+
+/* TODO: need a lot more LOG, ASSERT and SYSLOG statements */
+/*----------------------------------------------------------------------------*/
+
+/* TODO: for now just read from a flat file; change it in next phase to 
+ *       file/shmem depending upon what we decide; same goes for binary vs.
+ *       text format;  either way, the format of communication has to be defined
+ *       so that nodemanager & core know what to write & read - key items 
+ *       include number of mode changes transmitted & the structure of each.
+ *
+ * Mode file format:
+ * <num_mode_update_entries>
+ * <policy_id-str>:<mode-decimal_integer>
+ * ...
+ * mode 0 - off, 1 - detect, 2 - protect;
+ *
+ * TODO: eventually, modes will be strings (good idea?, not binary; might be
+ *       better to leave it as it is today.
+ */
+static void 
+hotp_read_policy_modes(hotp_policy_mode_t **old_modes)
+{
+    /* TODO: for the next phase decide whether to use registry key or option
+     *        string; for use a registry key.
+     */
+    uint mode = 0, vul, num_mode_update_entries, i;
+    char *buf = NULL;
+    size_t buf_len = 0;
+    char *start = NULL;
+
+    /* Old modes are needed only by regular hotp for flushing patches;
+     * hotp_only shouldn't use them.
+     */
+    ASSERT(!DYNAMO_OPTION(hotp_only) || old_modes == NULL);
+    if (old_modes != NULL)  /* init to NULL because there are error exits */
+        *old_modes = NULL;
+
+    /* Can be called only during hotp_init() or during a nudge. */
+    ASSERT_OWN_WRITE_LOCK(true, &hotp_vul_table_lock);
+
+    /* This function shouldn't be called before policies are read. 
+     * Sometimes, the node manager can nudge for a mode read without specifying 
+     * policies first!  This may happen during startup.  Case 5448.
+     */
+    if (GLOBAL_VUL_TABLE == NULL) {
+        LOG(GLOBAL, LOG_HOT_PATCHING, 1, "Modes can't be set without "
+            "policy definitions.  Probably caused due to a nudge by the node "
+            "manager to read modes when there were no policies.");
+        return;
+    }
+
+    buf = hotp_read_data_file(MODES_FILE, &buf_len);
+    if (buf == NULL) {
+        ASSERT(buf_len == 0);
+        return;
+    }
+    ASSERT(buf_len > 0);
+
+    /* Allocate space to save the old modes if they were requested for. */
+    if (old_modes != NULL) {
+        *old_modes = HEAP_ARRAY_ALLOC(GLOBAL_DCONTEXT, hotp_policy_mode_t, 
+                                      NUM_GLOBAL_VULS, ACCT_HOT_PATCHING, PROTECTED);
+        ASSERT(*old_modes != NULL);     /* make sure that space was allocated */
+    }
+
+    /* Turn off all vulnerabilities before reading modes.  Only those for which
+     * a mode is specified should be on.  Fix for case 5565.  As the write lock
+     * is held, there is no danger of any lookup providing a no-match when there
+     * is one.
+     */
+    for (vul = 0; vul < NUM_GLOBAL_VULS; vul++) {
+        if (old_modes != NULL)
+            (*old_modes)[vul] = GLOBAL_VUL(vul).mode;
+
+        /* Only hot patch types can be turned off by mode files.  Other types
+         * like gbop hooks can't be.
+         */
+        if (TESTALL(HOTP_TYPE_HOT_PATCH, GLOBAL_VUL(vul).type))
+            GLOBAL_VUL(vul).mode = HOTP_MODE_OFF;
+    }
+
+    start = buf;
+    SET_NUM(num_mode_update_entries, uint, NUM_VULNERABILITIES, start);
+
+    /* TODO: what if num_mode_update_entries is more than the entries in the 
+     * file?
+     */
+    for (i = 0; i < num_mode_update_entries; i++) {
+        bool matched = false;
+        char *temp, *policy_id;
+
+        SET_STR_PTR(policy_id, start);
+        temp = strchr(policy_id, ':');
+        if (temp == NULL)
+            goto error_reading_policy;
+        *temp++ = '\0';     /* TODO: during file mapping, this won't work */
+
+        SET_NUM(mode, uint, MODE, temp);
+
+        /* Must set mode for all vulnerabilities with a matching policy_id, not
+         * just the first one.
+         */
+        for (vul = 0; vul < NUM_GLOBAL_VULS; vul++) {
+            if (strncmp(GLOBAL_VUL(vul).policy_id, policy_id, 
+                        MAX_POLICY_ID_LENGTH) == 0) {
+                GLOBAL_VUL(vul).mode = mode;
+                matched = true;
+            }
+        }
+
+        /* If during mode update policy_id from a mode file doesn't have
+         * a corresponding vul_t, log a warning.  When the node manager is 
+         * starting up, modes file can be inconsistent, so this may happen 
+         * (cases 5500 & 5526).  However this could be a bug somewhere in the 
+         * pipe line (EV, nm, policy package, etc) too.
+         */
+        if (!matched)
+            SYSLOG_INTERNAL_WARNING("While reading modes, found a mode "
+                "definition for a policy (%s) that didn't exist", policy_id);
+    }
+
+/* TODO: make the macro take this as an argument or find a neutral name */
+ error_reading_policy:
+    ASSERT(buf != NULL);
+    heap_free(GLOBAL_DCONTEXT, buf, buf_len HEAPACCT(ACCT_HOT_PATCHING));
+    return;
+}
+
+static void
+hotp_set_policy_status(const uint vul_index, const hotp_inject_status_t status)
+{
+    uint crc_buf_size;
+
+    ASSERT_OWN_WRITE_LOCK(true, &hotp_vul_table_lock);
+
+    ASSERT(hotp_policy_status_table != NULL);
+    ASSERT(status == HOTP_INJECT_NO_MATCH || status == HOTP_INJECT_PENDING ||
+           status == HOTP_INJECT_IN_PROGRESS || status == HOTP_INJECT_DETECT ||
+           status == HOTP_INJECT_PROTECT || status == HOTP_INJECT_ERROR);
+
+    /* Given that no other thread, app or nudge, will change this without the
+     * hot patch lock, this can be done without an atomic write.
+     */
+    ASSERT(GLOBAL_VUL(vul_index).info->inject_status != NULL);
+    *(GLOBAL_VUL(vul_index).info->inject_status) = status;
+    
+    /* Compute CRC after this status update and put it in the
+     * policy status table so that the node manager is protected from
+     * reading invalid status due to policy status table being reset/
+     * relllocated due to hotp_init or nudge or detach taking place.
+     *
+     * Note: The CRC write to the table doesn't need to be atomic too.  Also,
+     *       the CRC value is for all bytes of the policy status table except
+     *       the CRC itself.  Otherwise we would have to do the CRC computation
+     *       twice; wastefully expensive.
+     */
+    crc_buf_size = hotp_policy_status_table->size -
+                   sizeof(hotp_policy_status_table->crc);
+    hotp_policy_status_table->crc = crc32((char*)&hotp_policy_status_table->size, 
+                                          crc_buf_size);
+}
+
+/* The status of hot patches is directly read by the node manager from the
+ * memory address specified in the drmarker; no nudge is needed.  While the 
+ * table is being created, the drmarker pointer will be null and set only 
+ * after the table is fully initialized.  Also, updates to the table entries 
+ * are made with the hot patch lock, as with creation.  The only way the node 
+ * manager can get invalid data is after it reads drmarker, this routine
+ * releases the old policy status table before the node manager can read it.
+ * That is guarded by the table CRC, which is likely to be wrong.  If drmarker 
+ * points to memory released to the os or NULL, node manager will get a memory 
+ * read error and it should be able to reattempt within which the new table will
+ * be ready.
+ *
+ * Format of policy status table in memory:
+ * <CRC32-uint> - CRC of size_in_bytes - sizeof(CRC32, i.e, uint).
+ * <size_in_bytes-uint>
+ * <num_policy_entries-uint>
+ * <hotp_policy_status_t>*
+ */
+static void
+hotp_init_policy_status_table(void)
+{
+    uint i, num_policies = NUM_GLOBAL_VULS, size_in_bytes, crc_buf_size;
+    hotp_policy_status_table_t *temp;
+
+    /* Can be called only during hotp_init() or during a nudge. */
+    ASSERT_OWN_WRITE_LOCK(true, &hotp_vul_table_lock);
+    ASSERT(!DATASEC_PROTECTED(DATASEC_RARELY_PROT));
+
+    /* This function shouldn't be called before policies and/or modes are read. 
+     * Sometimes, the node manager can nudge for a mode read without specifying 
+     * policies first!  This may happen during startup.  Case 5448.
+     */
+    if (GLOBAL_VUL_TABLE == NULL) {
+        LOG(GLOBAL, LOG_HOT_PATCHING, 1, "Policy status table can't be created "
+            "without policy definitions.  Probably caused due to a nudge by the "
+            "node manager to read modes when there were no policies.  Or "
+            "because all probes registered using the probe api were invalid.");
+        return;
+    }
+
+    /* This function is called each time a new policies and/or modes are read
+     * in.  Each such time all existing injected hot patches are removed, so
+     * the policy status table associated with the old global vulnerability 
+     * table must be released or resized to fit only the new set of 
+     * hot patches turned on.  The former is simpler to do.
+     *
+     * Note: if the optimization of flushing only those policies that
+     * have changed is implemented, which is not the case today, then just
+     * releasing policy status table will result in incorrect inject status.  
+     * It should be released after the new table is created and filled with
+     * old values.
+     */
+    if (hotp_policy_status_table != NULL) {
+        temp = hotp_policy_status_table;
+        hotp_policy_status_table = NULL;
+
+        /* If dr_marker_t isn't initialized, this won't be set.  In that case,
+         * the dr_marker_t initialization code will set up the policy status table.
+         * This can happen at init time because hotp_init() is called before 
+         * callback_interception_init().
+         */
+        set_drmarker_hotp_policy_status_table(NULL);
+
+        heap_free(GLOBAL_DCONTEXT, temp, temp->size HEAPACCT(ACCT_HOT_PATCHING));
+    }
+
+    /* Right now, the status table contains as many elements as
+     * vulnerabilities.  The original idea was to have only policies which
+     * are turned on in the table.  This caused failures in the core
+     * because we need to maintain status internally for vulnerabilities
+     * that are turned off too.  Case 5326.
+     */
+    size_in_bytes = sizeof(hotp_policy_status_table_t) +
+                    sizeof(hotp_policy_status_t) * num_policies;
+    temp = heap_alloc(GLOBAL_DCONTEXT, size_in_bytes HEAPACCT(ACCT_HOT_PATCHING));
+    temp->size = size_in_bytes;
+    temp->policy_status_array = (hotp_policy_status_t*)((char*)temp +
+                                    sizeof(hotp_policy_status_table_t));
+
+    /* Init status buffer elements & set up global vul table pointers */
+    /* TODO: two vulnerabilities can belong to the same policy; need to check
+     *       for that and avoid duplication in the table;  not needed now 
+     *       because we don't have such policies yet.
+     */
+    for (i = 0; i < NUM_GLOBAL_VULS; i++) {
+        strncpy(temp->policy_status_array[i].policy_id,
+                GLOBAL_VUL(i).policy_id, MAX_POLICY_ID_LENGTH);
+        NULL_TERMINATE_BUFFER(temp->policy_status_array[i].policy_id);
+        temp->policy_status_array[i].inject_status = HOTP_INJECT_NO_MATCH;
+
+        /* Fix for case 5484, where the node manager wasn't able to tell if an 
+         * inject status was for a policy that was turned on or off.
+         */
+        temp->policy_status_array[i].mode = GLOBAL_VUL(i).mode;
+
+        /* The inject status in global vulnerability table should point 
+         * to the corresponding element in this table.
+         */
+        GLOBAL_VUL(i).info->inject_status =
+            &temp->policy_status_array[i].inject_status;
+    }
+    temp->num_policies = i;
+
+    /* Set the table CRC now that the table has been initialized. */
+    crc_buf_size = temp->size - sizeof(temp->crc);
+    temp->crc = crc32((char*)&temp->size, crc_buf_size);
+
+    /* Make the policy status table live.  If the dr_marker_t isn't initialized
+     * this won't be set.  In that case, the dr_marker_t initialization code will 
+     * set up the policy status table; happens during startup/initialization.
+     */
+    hotp_policy_status_table = temp;
+
+    set_drmarker_hotp_policy_status_table((void*)temp);
+}
+
+/* Frees all the dynamically allocated members of vul (strings, info, sets,
+ * modules and patch points).  NOTE: It doesn't free the vul itself.
+ */
+static void hotp_free_one_vul(hotp_vul_t *vul)
+{
+    uint set_idx, module_idx, ppoint_idx;
+
+    /* If this routine is called with a NULL for argument then there is a bug
+     * somewhere.*/
+    ASSERT(vul != NULL);
+    if (vul == NULL)
+        return;
+
+    if (vul->vul_id != NULL)
+        dr_strfree(vul->vul_id HEAPACCT(ACCT_HOT_PATCHING));
+    if (vul->policy_id != NULL)
+        dr_strfree(vul->policy_id HEAPACCT(ACCT_HOT_PATCHING));
+    if (vul->hotp_dll != NULL)
+        dr_strfree(vul->hotp_dll HEAPACCT(ACCT_HOT_PATCHING));
+    if (vul->hotp_dll_hash != NULL)
+        dr_strfree(vul->hotp_dll_hash HEAPACCT(ACCT_HOT_PATCHING));
+    if (vul->info != NULL) {
+        HEAP_TYPE_FREE(GLOBAL_DCONTEXT, vul->info, hotp_vul_info_t,
+                       ACCT_HOT_PATCHING, PROTECTED);
+    }
+
+    if (vul->sets == NULL)
+        return;
+
+    /* If a set's array isn't null, then the number of sets can't be zero. */
+    ASSERT(vul->num_sets > 0);
+    for (set_idx = 0; set_idx < vul->num_sets; set_idx++) {
+        hotp_set_t *set = &vul->sets[set_idx];
+
+        if (set->modules == NULL)
+            continue;
+
+        /* If a modules array isn't null, then the number of modules can't 
+         * be zero. 
+         */
+        ASSERT(set->num_modules > 0);
+        for (module_idx = 0; module_idx < set->num_modules; module_idx++) {
+            hotp_module_t *module = &set->modules[module_idx];
+            if (module->sig.pe_name != NULL) {
+                dr_strfree(module->sig.pe_name HEAPACCT(ACCT_HOT_PATCHING));
+            }
+
+            if (module->hashes != NULL) {
+                ASSERT(module->num_patch_point_hashes > 0);
+                HEAP_ARRAY_FREE(GLOBAL_DCONTEXT, module->hashes,
+                                hotp_patch_point_hash_t,
+                                module->num_patch_point_hashes,
+                                ACCT_HOT_PATCHING, PROTECTED);
+            }
+
+            if (module->patch_points != NULL) {
+                ASSERT(module->num_patch_points > 0);
+                for (ppoint_idx = 0; ppoint_idx < module->num_patch_points;
+                     ppoint_idx++) {
+                    hotp_patch_point_t *ppoint = &module->patch_points[ppoint_idx];
+                    if (ppoint->trampoline != NULL) {
+                        ASSERT(DYNAMO_OPTION(hotp_only));
+                        ASSERT(ppoint->app_code_copy != NULL);
+                        special_heap_free(hotp_only_tramp_heap,
+                                          (void *) ppoint->trampoline);
+                    }
+                }
+                HEAP_ARRAY_FREE(GLOBAL_DCONTEXT, module->patch_points,
+                                hotp_patch_point_t, module->num_patch_points,
+                                ACCT_HOT_PATCHING, PROTECTED);
+            }
+        }
+        HEAP_ARRAY_FREE(GLOBAL_DCONTEXT, set->modules, hotp_module_t,
+                        set->num_modules, ACCT_HOT_PATCHING, PROTECTED);
+    }
+    HEAP_ARRAY_FREE(GLOBAL_DCONTEXT, vul->sets, hotp_set_t,
+                    vul->num_sets, ACCT_HOT_PATCHING, PROTECTED);
+}
+
+/* Release all memory used by the hot patch vulnerability table, tab.
+ * num_vuls_alloc is number of vulnerability defs. the table has space for.
+ * The table may not always contain num_vuls_alloc policy defs.  Where there
+ * is an error during policy defs file parsing they can be fewer in number with
+ * the last one (one where the error happened) being partial.  Cases 8272, 9045.
+ */
+static void
+hotp_free_vul_table(hotp_vul_t *tab, uint num_vuls_alloc)
+{
+    uint vul_idx;
+
+    if (tab == NULL) {
+        ASSERT(num_vuls_alloc == 0);
+        return;
+    }
+
+    /* If the table isn't NULL, the number of vulnerabilities can't be zero. */
+    ASSERT(num_vuls_alloc > 0);
+
+    for (vul_idx = 0; vul_idx < num_vuls_alloc; vul_idx++) {
+        hotp_free_one_vul(&tab[vul_idx]);
+    }
+    HEAP_ARRAY_FREE(GLOBAL_DCONTEXT, tab, hotp_vul_t, num_vuls_alloc,
+                    ACCT_HOT_PATCHING, PROTECTED);
+}
+
+/* This routine flushes all fragments in fcache that have been injected with a 
+ * hot patch, i.e., restoring an app text to its pre-hot-patch state.
+ * 
+ * Note: hot patch removal is not optimized, i.e., changes to existing
+ * policy definitions, modes or actual injection status aren't used to limit
+ * flushing.  Not a performance issue for now.
+ * TODO: flush only those vulnerabilities that have actually changed, not 
+ *       every thing that is active or has been injected. 
+ * TODO: make this use loaded_module_areas & get rid off the 4-level nested
+ *          loops.
+ */
+static void
+hotp_remove_patches_from_module(const hotp_vul_t *vul_tab, const uint num_vuls, 
+                                const bool hotp_only, const app_pc mod_base,
+                                const hotp_policy_mode_t *old_modes)
+{
+    uint vul_idx, set_idx, module_idx, ppoint_idx;
+    hotp_module_t *module;
+    hotp_patch_point_t *ppoint;
+    dcontext_t *dcontext = get_thread_private_dcontext();
+
+    ASSERT(dcontext != NULL && dcontext != GLOBAL_DCONTEXT);
+    /* For hotp_only patch removal, we should be running in hotp_only mode. */
+    ASSERT(!hotp_only || DYNAMO_OPTION(hotp_only));
+    /* Old vulnerability modes shouldn't be used with hotp_only. */
+    ASSERT(!DYNAMO_OPTION(hotp_only) || old_modes == NULL);
+    /* Alternate modes shouldn't be used during module specific removal also. */
+    ASSERT(mod_base == NULL || old_modes == NULL);
+
+    /* Though trying to flush a NULL vul table is a bug, this can happen 
+     * because the node manager can nudge the core to read modes when it hasn't
+     * provided the policies!  See case 5448.  Hence just a warning & no assert.
+     */
+    if (vul_tab == NULL) {
+        LOG(GLOBAL, LOG_HOT_PATCHING, 1, "Hot patch flushing has been invoked "
+            "with a NULL table");
+        return;
+    }
+
+    LOG(GLOBAL, LOG_HOT_PATCHING, 1, "flushing as a result of nudge\n");
+    for (vul_idx = 0; vul_idx < num_vuls; vul_idx++) {
+        bool set_processed = false;
+        const hotp_vul_t *vul = &VUL(vul_tab, vul_idx);
+
+        /* Nothing to remove or flush if the mode is off, i.e., nothing would
+         * have been injected.  
+         * Note: Both vul's current mode & its old mode should be off to skip 
+         * removal; even if one is not, then that vulnerability's patches need 
+         * to be removed.  In otherwords, if patch was previously on (injected)
+         * or is now on (to be injected), corresponding bbs must be flushed;
+         * this is for regular hotp, not for hotp_only which has no flushing.
+         */
+        if (vul->mode == HOTP_MODE_OFF) {
+            if (old_modes == NULL) {
+                /* If there is no old_mode, skip right here. */
+                continue;
+            } else if (old_modes[vul_idx] == HOTP_MODE_OFF) {
+                /* If old_mode exists, that must be off too in order to skip. */
+                continue;
+            }
+        }
+        ASSERT(vul->mode == HOTP_MODE_DETECT ||
+               vul->mode == HOTP_MODE_PROTECT ||
+               (old_modes != NULL && (old_modes[vul_idx] == HOTP_MODE_DETECT ||
+                                      old_modes[vul_idx] == HOTP_MODE_PROTECT)));
+
+        for (set_idx = 0; set_idx < VUL(vul_tab, vul_idx).num_sets; set_idx++) {
+            /* Only the first matching set should be used; case 10248. */
+            if (set_processed)
+                break;
+
+            for (module_idx = 0; module_idx < SET(vul_tab, vul_idx, set_idx).num_modules;
+                 module_idx++) {
+                module = &MODULE(vul_tab, vul_idx, set_idx, module_idx);
+                if (module->matched) {
+                    /* If a specific module is mentioned remove patches from 
+                     * just that.
+                     */
+                    if (mod_base != NULL && mod_base != module->base_address)
+                        continue;
+
+                    set_processed = true;
+                    /* Otherwise, flush all patch points in any module that
+                     * matches.  Nothing to flush in unmatched modules. 
+                     */
+                    for (ppoint_idx = 0; ppoint_idx < module->num_patch_points;
+                         ppoint_idx++) {
+                        ppoint = &module->patch_points[ppoint_idx];
+                        if (hotp_only) {
+                            /* For a hotp_only patch, we can only remove that 
+                             * which has been injected, unlike the hotp mode 
+                             * where we might just be flushing out uninjected 
+                             * fragments or don't know which particular patch 
+                             * point has been injected (in hotp_only mode all 
+                             * of them should be injected if one is injected). 
+                             */
+                            if (ppoint->trampoline != NULL)
+                                hotp_only_remove_patch(module, ppoint);
+                            else
+                                /* If module is matched and mode is on, then 
+                                 * hotp_only patch targeting the current 
+                                 * ppoint must be injected unless it has
+                                 * been removed to handle loader-safety issues.
+                                 */
+                                ASSERT(hotp_globals->ldr_safe_hook_removal &&
+                                       "hotp_only - double patch removal");
+                        } else {
+                            app_pc flush_addr = hotp_ppoint_addr(module, ppoint);
+
+                            ASSERT_OWN_NO_LOCKS();
+                            LOG(GLOBAL, LOG_HOT_PATCHING, 4,
+                                "flushing "PFX" due to a nudge\n", flush_addr);
+                            flush_fragments_in_region_start(dcontext, flush_addr,
+                                                            1, false /* no lock */,
+                                                            false /* keep futures */,
+                                                            false/*exec still valid*/,
+                                                            false/*don't force synchall*/
+                                                            _IF_DGCDIAG(NULL));
+                            flush_fragments_in_region_finish(dcontext, false);
+                            /* TODO: ASSERT (flushed fragments have really been)
+                             *       flushed but how, using a vm_areas_overlap()
+                             *       or fragment_lookup() check?
+                             */
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+/* TODO: make this use hotp_patch_point_areas & get rid off the 4-level nested
+ *          loops which is used in hotp_remove_patches_from_module.
+ */
+static void
+hotp_remove_hot_patches(const hotp_vul_t *vul_tab, const uint num_vuls, 
+                        const bool hotp_only,
+                        const hotp_policy_mode_t *old_modes)
+{
+    /* Old vulnerability modes shouldn't be used with hotp_only. */
+    ASSERT(!DYNAMO_OPTION(hotp_only) || old_modes == NULL);
+    hotp_remove_patches_from_module(vul_tab, num_vuls, hotp_only, NULL,
+                                    old_modes);
+}
+
+/* TODO: vlad wanted the ability to ignore some attributes during checking;
+ *       this is not for constraints, but if he wants an ad-hoc patch to fix
+ *       something other than a vulnerability, say, broken code that is not
+ *       a vulnerability;  for hot patches/constraints all attributes must be
+ *       checked, no ignoring stuff.
+ */
+static bool 
+hotp_module_match(const hotp_module_t *module, const app_pc base,
+                  const uint checksum, const uint timestamp,
+                  const size_t image_size, const size_t code_size,
+                  const uint64 file_version, const char *name, hotp_type_t type)
+{
+    uint hash_index, computed_hash;
+    hotp_patch_point_hash_t *hash;
+    bool matched;
+
+    ASSERT(module != NULL && base != NULL);
+    ASSERT(TESTANY(HOTP_TYPE_HOT_PATCH | HOTP_TYPE_GBOP_HOOK | HOTP_TYPE_PROBE,
+                   type));
+
+    LOG(GLOBAL, LOG_HOT_PATCHING, 1, "Matching module base "PFX" %s\n", base, name);
+
+    /* For library offset or export function based patch points, the probe will
+     * define a library by name (if needed we expand it to include the
+     * liveshield type matching, but the client can do it outside) */
+    /* gbop type patches provide a symbolic name to hook, so there is nothing
+     * to match it with other than the pe name.
+     */
+    if (TESTALL(HOTP_TYPE_PROBE, type)
+        IF_GBOP(|| (TESTALL(HOTP_TYPE_GBOP_HOOK, type)))) {
+        ASSERT(module->sig.pe_checksum == 0 && module->sig.pe_timestamp == 0 &&
+               module->sig.pe_image_size == 0 && module->sig.pe_code_size == 0 &&
+               module->sig.pe_file_version == 0 &&
+               module->num_patch_points == 1 && module->patch_points != NULL &&
+               module->num_patch_point_hashes == 0 && module->hashes == NULL);
+        if (name == NULL) {
+            /* if the only check is the module name, then a NULL name means
+             * the module wasn't matched;  otherwise this check would be bogus.
+             */
+            return false;
+        } else if (strncasecmp(module->sig.pe_name, name, MAXIMUM_PATH) == 0) {
+            /* FIXME: strcmp() is faster than the ignore case version,
+             * but we shouldn't rely on the PE name case to be the
+             * same in all versions of Windows.
+             */
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    /* These checks are for hot patch types, i.e., ones that have offset rvas 
+     * specified for each known version.
+     * First stage check: PE timestamp, PE checksum, PE code_size, PE file 
+     * version & PE name, i.e., signature match.
+     *
+     * FIXME: Today error handling of PE parsing is not done by the core, so
+     * unavailability of an attribute isn't recorded.  Thus IGNORE and
+     * UNAVAILABLE are treated the same for module matching.  When the core can
+     * handle it the UNAVAILABLE part should be removed from the checks, and
+     * checks for unavailability should be done.  Case 9215 tracks the core not
+     * handling PE parsing for malformed files and their impact on hot patching.
+     */
+    ASSERT(TESTALL(HOTP_TYPE_HOT_PATCH, type));
+
+    matched = module->sig.pe_timestamp == timestamp ||
+              module->sig.pe_timestamp == PE_TIMESTAMP_IGNORE ||
+              module->sig.pe_timestamp == PE_TIMESTAMP_UNAVAILABLE;
+
+    matched = matched && (module->sig.pe_checksum == checksum ||
+                          module->sig.pe_checksum == PE_CHECKSUM_IGNORE ||
+                          module->sig.pe_checksum == PE_CHECKSUM_UNAVAILABLE);
+
+    matched = matched && (module->sig.pe_image_size == image_size ||
+                          module->sig.pe_image_size == PE_IMAGE_SIZE_IGNORE ||
+                          module->sig.pe_image_size == PE_IMAGE_SIZE_UNAVAILABLE);
+
+    matched = matched && (module->sig.pe_code_size == code_size ||
+                          module->sig.pe_code_size == PE_CODE_SIZE_IGNORE ||
+                          module->sig.pe_code_size == PE_CODE_SIZE_UNAVAILABLE);
+
+    matched = matched && (module->sig.pe_file_version == file_version ||
+                          module->sig.pe_file_version == PE_FILE_VERSION_IGNORE ||
+                          module->sig.pe_file_version == PE_FILE_VERSION_UNAVAILABLE);
+
+    matched = matched && ((strncmp(module->sig.pe_name, PE_NAME_IGNORE,
+                                   sizeof(PE_NAME_IGNORE)) == 0) ||
+                          (name == NULL && /* no name case */
+                           module->sig.pe_name[0] == PE_NAME_UNAVAILABLE) ||
+                          (name != NULL &&
+                           (strncmp(module->sig.pe_name, name, MAXIMUM_PATH) == 0)));
+
+    if (matched) {
+        LOG(GLOBAL, LOG_HOT_PATCHING, 1, "Module signature check passed\n");
+
+        /* First stage check was true, now let us do the second stage check,
+         * i.e., check the hashes of patch points in the module.
+         */
+        ASSERT(module->num_patch_point_hashes > 0 && 
+               module->hashes != NULL);
+        for (hash_index = 0; hash_index < module->num_patch_point_hashes;
+             hash_index++) {
+            hash = &module->hashes[hash_index];
+            computed_hash = hotp_compute_hash(base, hash);
+            if (computed_hash != hash->hash_value)
+                return false;
+        }
+        LOG(GLOBAL, LOG_HOT_PATCHING, 1, "Patch point hash check passed\n");
+        return true;
+    }
+    return false;
+}
+
+/* Used to compute the hash of a patch point hash region.  In hotp_only mode,
+ * if there is an overlap between a hash region and a patch region, the 
+ * image bytes, stored at the top of the trampoline, are used to create copy 
+ * of the image on which crc32 is computed.  In regular hotp mode, crc32 is 
+ * called directly.
+ */
+static uint
+hotp_compute_hash(app_pc base, hotp_patch_point_hash_t *hash)
+{
+    uint crc, copy_size;
+    char *hash_buf, *copy;
+    app_pc hash_start, hash_end, vm_start, vm_end, src, dst, trampoline;
+    vmvector_iterator_t iterator;
+    hotp_offset_match_t *ppoint_desc;
+
+    ASSERT(base != NULL && hash != NULL);
+    ASSERT(hash->start > 0 && hash->len > 0);
+
+    hash_start = base + hash->start;
+    hash_end = hash_start + hash->len;
+
+    /* If the hash region overlaps with any patch point region, then use the
+     * original image bytes to compute the crc32.  Valid for hotp_only because
+     * in hotp mode, i.e., with a code cache, we don't modify the original code.
+     */
+    if (DYNAMO_OPTION(hotp_only) &&
+        vmvector_overlap(hotp_patch_point_areas, hash_start, hash_end)) {
+
+        /* Make sure that the patch region size for hotp_only is correct. */
+        ASSERT(HOTP_PATCH_REGION_SIZE == HOTP_ONLY_PATCH_REGION_SIZE);
+
+        /* Allocate a buffer & copy the image bytes represented by the hash.
+         * This will include bytes modified by a prior hotp_only patch.
+         * Note: an extra 2 x HOTP_PATCH_REGION_SIZE is allocated to be used
+         * as overflow buffers at the front & back of the copy; makes handling
+         * the overlap scenarios (4 different ones) easy.
+         */
+        copy_size = hash->len + (2 * HOTP_PATCH_REGION_SIZE);
+        copy = HEAP_ARRAY_ALLOC(GLOBAL_DCONTEXT, char, copy_size,
+                                ACCT_HOT_PATCHING, PROTECTED);
+        hash_buf = copy + HOTP_PATCH_REGION_SIZE;
+        memcpy(hash_buf, hash_start, hash->len);
+
+        /* Now, for each vmarea that overlaps, copy the original image bytes
+         * into the buffer.
+         * FIXME: we do a linear walk as opposed to walking over only those
+         *  regions that overlap, ineffcient; see case 8211 about a new 
+         *  vmvector iterator that walks over only overlapping regions.
+         */
+        vmvector_iterator_start(hotp_patch_point_areas, &iterator);
+        while (vmvector_iterator_hasnext(&iterator)) {
+            ppoint_desc = vmvector_iterator_next(&iterator, &vm_start, &vm_end);
+            trampoline = GLOBAL_PPOINT(ppoint_desc->vul_index,
+                                       ppoint_desc->set_index, 
+                                       ppoint_desc->module_index,
+                                       ppoint_desc->ppoint_index).trampoline;
+
+            /* If the patch isn't injected, overlap doesn't matter because
+             * the image hasn't been changed.  Overlap with an uninjected patch
+             * region can only happen when loader safety is in progress during 
+             * which a patch point is removed (only from the image, not 
+             * hotp_patch_point_areas) and it is re-injected; the re-injection 
+             * of the patch point will overlap with itself.  See case 8222.
+             */
+            if (trampoline == NULL) {
+                /* If hash belongs ppoint_desc, i.e., overlaps with self, then
+                 * base and module's base must match.
+                 */
+                ASSERT(base == GLOBAL_MODULE(ppoint_desc->vul_index,
+                                   ppoint_desc->set_index,
+                                   ppoint_desc->module_index).base_address);
+                continue;
+            }
+
+            /* If the trampoline exists, it better be a valid one, i.e., the 
+             * patch corresponding to this vmarea must be injected.
+             */
+            ASSERT(vmvector_overlap(hotp_only_tramp_areas, trampoline, 
+                                    trampoline + HOTP_ONLY_TRAMPOLINE_SIZE));
+
+            /* The size of each vmarea in hotp_patch_point_areas must be 
+             * equal to that of the patch region.
+             */
+            ASSERT(vm_end - vm_start == HOTP_PATCH_REGION_SIZE);
+
+            /* The module corresponding to this vm area (patch point) should
+             * have been matched by a vul. def. (in hotp_process_image).
+             */
+            ASSERT(GLOBAL_MODULE(ppoint_desc->vul_index, ppoint_desc->set_index,
+                                 ppoint_desc->module_index).matched);
+
+            /* There are a few scenarios for a hash & patch point to overlap, 
+             * vmarea fully within the hash area, vice versa, partial below, 
+             * partial above, and exact on either side or both
+             * Using an extra buffer the size of a patch region at the front 
+             * and back allows all the scenarios to be handled with a single 
+             * equation - eliminates messy code; worth allocating 10 bytes 
+             * extra. 
+             * Note: the extra buffer can be 1 byte shorter on either side, but
+             *  leaving it at patch point region size, just to be safe.
+             */
+            if (vm_start < hash_end && vm_end > hash_start) {
+                src = trampoline;
+                dst = (app_pc) hash_buf + (vm_start - hash_start);
+
+                /* Just make sure that we don't trash anything when copying the 
+                 * original image over the bytes in hash_buf.
+                 */
+                ASSERT((dst >= (app_pc)copy) &&
+                       ((dst + HOTP_PATCH_REGION_SIZE) <=
+                        ((app_pc)copy + copy_size)));
+
+                /* If the hash overlaps with a patch point region, then the 
+                 * current image & the copy should be different, i.e., a patch
+                 * must exist at that point.
+                 */
+                ASSERT(memcmp(dst, src, HOTP_PATCH_REGION_SIZE) != 0);
+
+                /* CAUTION: this memcpy assumes the location & size of 
+                 * app code copy in the trampoline, i.e., the first 5 bytes of
+                 * trampoline contain the orignal app code; so any changes 
+                 * should be kept in sync.
+                 */
+                memcpy(dst, src, HOTP_PATCH_REGION_SIZE);
+            }
+            /* FIXME: if the iterator guaranteed order, we can break out after
+             *  the first non-match - optimization.
+             */
+        }
+        vmvector_iterator_stop(&iterator);
+        crc = crc32(hash_buf, hash->len);
+        HEAP_ARRAY_FREE(GLOBAL_DCONTEXT, copy, char, copy_size, 
+                        ACCT_HOT_PATCHING, PROTECTED);
+    } else {
+        /* No overlap; image is unmodified, so image's crc32 should be valid. */
+        crc = crc32((char *)hash_start, hash->len);
+    }
+    return crc;
+}
+
+/* TODO: this function should be used for startup & nudge dll list walking, 
+ *          dll loading and unloading.
+ * TODO: assert somehow that every time this function is invoked there must
+ *          be a flush preceding or succeeding it, except at startup
+ * TODO: os specific routine; move to win32/ 
+ * TODO: this function is called when vm_areas_init() is invoked, but 
+ *          hotp_init() is called after vm_areas_init()!  bogus - check 
+ *          other start up scenarios like retakeover to see if policy reading 
+ *          & activation get out of order;  this is the same issue that vlad
+ *          pointed out: make sure that process_image() is called after 
+ *          hotp_init()
+ * TODO: process_{image,mmap}() should never be called on hot patch dlls
+ *       because dr is loading them;  assert for this somewhere to prevent
+ *       assumption violation bugs.
+ */
+static void 
+hotp_process_image_helper(const app_pc base, const bool loaded,
+                          const bool own_hot_patch_lock, const bool just_check, 
+                          bool *needs_processing, const thread_record_t **all_threads, 
+                          const int num_threads, const bool ldr_safety,
+                          vm_area_vector_t *toflush);
+void 
+hotp_process_image(const app_pc base, const bool loaded,
+                   const bool own_hot_patch_lock, const bool just_check, 
+                   bool *needs_processing, const thread_record_t **all_threads, 
+                   const int num_threads)
+{
+    hotp_process_image_helper(base, loaded, own_hot_patch_lock, just_check, 
+                   needs_processing, all_threads, num_threads, false, NULL);
+}
+
+/* Helper routine for seeing if point is in hotp_ppoint_vec */
+bool
+hotp_ppoint_on_list(app_rva_t ppoint,
+                   app_rva_t *hotp_ppoint_vec, uint hotp_ppoint_vec_num)
+{
+    bool on_list = false;
+    uint i;
+    /* We assume there are at most a handful of these so we don't sort.
+     * If we add GBOP hooks we may want to do that. */
+#ifdef GBOP
+    ASSERT(DYNAMO_OPTION(gbop) == 0);
+#endif
+    ASSERT(ppoint != 0);
+    ASSERT(hotp_ppoint_vec != NULL && hotp_ppoint_vec_num > 0);
+    if (hotp_ppoint_vec == NULL)
+        return false;
+    for (i = 0; i < hotp_ppoint_vec_num; i++) {
+        if (ppoint == hotp_ppoint_vec[i]) {
+            on_list = true;
+            break;
+        }
+    }
+    return on_list;
+}
+
+/* Returns true if there is a persistent cache in [base,base+image_size) that
+ * may contain code for any of the patch points of module
+ */
+static bool
+hotp_perscache_overlap(uint vul, uint set, uint module, app_pc base, size_t image_size)
+{
+    vmvector_iterator_t vmvi;
+    coarse_info_t *info;
+    uint pp;
+    bool flush_perscache = false;
+    ASSERT(DYNAMO_OPTION(use_persisted_hotp));
+    ASSERT(!DYNAMO_OPTION(hotp_only));
+    vm_area_coarse_iter_start(&vmvi, base);
+    /* We have a lot of nested linear walks here, esp. when called from
+     * hotp_process_image_helper inside nested loops, but typically the coarse
+     * iterator involves one binary search and only one match, and
+     * hotp_ppoint_on_list and the pp for loop here only a few entries each;
+     * so this routine shouldn't be a perf bottleneck by itself.
+     */
+    while (!flush_perscache &&
+           vm_area_coarse_iter_hasnext(&vmvi, base+image_size)) {
+        info = vm_area_coarse_iter_next(&vmvi, base+image_size);
+        ASSERT(info != NULL);
+        if (info == NULL) /* be paranoid */
+            continue;
+        if (info->hotp_ppoint_vec == NULL)
+            flush_perscache = true;
+        else {
+            ASSERT(info->persisted);
+            for (pp = 0; pp < GLOBAL_MODULE(vul, set, module).num_patch_points; pp++) {
+                if (!hotp_ppoint_on_list(GLOBAL_PPOINT(vul, set, module, pp).offset,
+                                         info->hotp_ppoint_vec,
+                                         info->hotp_ppoint_vec_num)) {
+                    flush_perscache = true;
+                    break;
+                }
+            }
+        }
+        /* Should be able to ignore 2ndary unit */
+        ASSERT(info->non_frozen == NULL || info->non_frozen->hotp_ppoint_vec == NULL);
+    }
+    vm_area_coarse_iter_stop(&vmvi);
+    return flush_perscache;
+}
+
+/* This helper exists mainly to handle the loader safety case for adding 
+ * ppoint areas.  vm_areas should be added to ppoint_areas only during module
+ * load/unload (including the initial stack walk) and during policy read 
+ * nudge, not during a reinjection during loader safety.
+ * The same holds good for removal, but today that isn't an
+ * issue because loader safety uses hotp_remove_patches_from_module() to do
+ * it, which doesn't modify ppoint areas.
+ * FIXME: once hotp_inject_patches_into_module() is implemented
+ * based on loaded_module_areas and used in hotp_only_mem_prot_change() instead
+ * of hotp_process_image_helper, this can go.
+ */
+static void 
+hotp_process_image_helper(const app_pc base, const bool loaded,
+                          const bool own_hot_patch_lock, const bool just_check, 
+                          bool *needs_processing, const thread_record_t **all_threads, 
+                          const int num_threads_arg, const bool ldr_safety,
+                          vm_area_vector_t *toflush)
+{
+    uint vul_idx, set_idx, module_idx, ppoint_idx;
+    hotp_module_t *module;
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    uint checksum, timestamp;
+    size_t image_size = 0, code_size;
+    uint64 file_version;
+    module_names_t *names = NULL;
+    const char *name = NULL, *pe_name = NULL, *mod_name = NULL;
+    int num_threads = num_threads_arg;
+    bool any_matched = false;
+    bool flush_perscache = false;
+    bool perscache_range_overlap = false;
+
+    ASSERT(base != NULL);   /* Is it a valid dll in loaded memory? */
+
+    LOG(GLOBAL, LOG_HOT_PATCHING, 2, "hotp_process_image "PFX" %s w/ %d vuls\n",
+        base, loaded ? "load" : "unload", NUM_GLOBAL_VULS);
+
+    ASSERT(dcontext != GLOBAL_DCONTEXT);
+    /* note that during startup processing due to
+     * find_executable_vm_areas() dcontext can in fact be NULL
+     */
+    if (dcontext != NULL && dcontext->nudge_thread)     /* Fix for case 5367. */
+        return;
+#ifdef WINDOWS
+    if (num_threads == 0 && !just_check && DYNAMO_OPTION(hotp_only)) {
+        ASSERT(!dynamo_initialized);
+        num_threads = HOTP_ONLY_NUM_THREADS_AT_INIT;
+        /* For hotp_only, all threads should be suspended before patch injection.  
+         * However, at this point in startup, callback hooks aren't in place and 
+         * we don't know if any other thread is running around that the core 
+         * doesn't know about.  This would be rare and with early injection, rarer.
+         * However, if that thread is executing in a region being patched we can 
+         * fail spectacularly.  Curiosity in the meanwhile.
+         * Also, to be on the safe side grab the synchronization locks.
+         */
+        ASSERT_CURIOSITY(check_sole_thread());
+        ASSERT(!own_hot_patch_lock);    /* can't get hotp lock before sync locks */
+        mutex_lock(&all_threads_synch_lock);
+        mutex_lock(&thread_initexit_lock);
+    }
+#endif
+
+    if (!own_hot_patch_lock)
+        write_lock(&hotp_vul_table_lock);
+    ASSERT_OWN_READWRITE_LOCK(true, &hotp_vul_table_lock);
+
+    /* Caller doesn't want to process the image, but to know if it matches. */
+    if (just_check) {
+        /* Only hotp_only needs this; not regular hot patching. */
+        ASSERT(DYNAMO_OPTION(hotp_only));
+        ASSERT(needs_processing != NULL);
+        *needs_processing = false;  /* will be set it to true, if needed */
+    } else
+        ASSERT(needs_processing == NULL);
+        
+    /* get module information from PE once (case 7990) */
+    /* FIXME: once all pe information is available in loaded_module_areas, use
+     *        that here
+     * FIXME: file_version is obtained by walking the resouce section which is expensive;
+     *        the same is true for code_size to some extent, i.e., expensive but not 
+     *        that much.  So we may be better off by computing them in separate routines
+     *        predicated by the first check - and put all these into hotp_get_module_sig()
+     */
+    os_get_module_info_lock();
+    if (!os_get_module_info_all_names(base, &checksum, &timestamp, &image_size,
+                                      &names, &code_size, &file_version)) {
+        /* FIXME: case 9778 - module info is now obtained from 
+         * loaded_module_areas vector, which doesn't seem to have hotp dll 
+         * info, so we hit this.  As a first step this is converted to a log 
+         * to make tests work;  will have to read it from pe directly (using 
+         * try/except) if it isn't a hotp dll - if that doesn't work then be 
+         * curious.  Also, need to find out if it was triggered only for hotp
+         * dlls. */
+        LOG(GLOBAL, LOG_HOT_PATCHING, 2, "unreadable PE base ("PFX")?\n", base);
+        os_get_module_info_unlock();
+        goto hotp_process_image_exit;
+    } else {
+        /* Make our own copy of both the pe name and the general module name.
+         * This is because pe name can be null for executables, which is fine
+         * for liveshields, but not for gbop or probe api - they just specify a
+         * module name, so we have to use any name that is available.  Note: as
+         * of today, gbop hasn't been done on executables, which is why it
+         * worked - it is broken for hooks in exes - a FIXME, but gbop is going
+         * away anyway. */
+        pe_name = dr_strdup(names->module_name HEAPACCT(ACCT_HOT_PATCHING));
+        mod_name = dr_strdup(GET_MODULE_NAME(names) HEAPACCT(ACCT_HOT_PATCHING));
+        os_get_module_info_unlock();
+        /* These values can't be read in from a module, they are used by the
+         * patch writer to hint to the core to ignore the corresponding checks.
+         */
+        ASSERT_CURIOSITY(timestamp != PE_TIMESTAMP_IGNORE &&
+                         checksum != PE_CHECKSUM_IGNORE &&
+                         image_size != PE_IMAGE_SIZE_IGNORE);
+    }
+#ifdef WINDOWS
+    DODEBUG({
+        if (TEST(ASLR_DLL, DYNAMO_OPTION(aslr)) &&
+            TEST(ASLR_SHARED_CONTENTS, DYNAMO_OPTION(aslr_cache))) {
+            /* case 8507 - the timestamp and possibly checksum of the current mapping,
+               possibly ASLRed, may not be the same as the application DLL */
+            uint pe_timestamp;
+            uint pe_checksum;
+            bool ok = os_get_module_info(base, &pe_checksum, &pe_timestamp, NULL, NULL,
+                                         NULL, NULL);
+            ASSERT_CURIOSITY(timestamp != 0);
+            /* Note that if we don't find the DLL in the module list,
+             * we'll keep using the previously found checksum and
+             * timestamp.  Although normally all DLLs are expected to be
+             * listed, currently that is done only with ASLR_TRACK_AREAS.
+             */
+            /* case 5381: we don't ASSERT(ok) b/c hotpatch DLLs aren't listed in our
+             * own module areas, so we don't always find all modules */
+            /* with the current scheme the checksum is still the original DLLs checksum
+             * though it won't check, and the timestamp is bumped by one second
+             */
+            ASSERT(!ok || pe_checksum == checksum);
+            ASSERT_CURIOSITY(!ok || 
+                             pe_timestamp == timestamp || 
+                             pe_timestamp == timestamp + 1);
+        }
+    });
+#endif /* WINDOWS */
+
+    if (!DYNAMO_OPTION(hotp_only)) {
+        perscache_range_overlap =
+            executable_vm_area_persisted_overlap(base, base+image_size);
+    }
+
+    /* TODO: assert that 'base' is the module's base address, 
+     *       get_dll_short_name() expects this; will be used for sig check
+     *       use the fn() that gets only what is in the PE
+     * FIXME: eliminate this n^4 loop for each module {load,unload}; case 10683
+     */
+    for (vul_idx = 0; vul_idx < NUM_GLOBAL_VULS; vul_idx++) {
+        bool set_matched = false;
+
+        if (TESTALL(HOTP_TYPE_PROBE, GLOBAL_VUL(vul_idx).type)
+            IF_GBOP(|| (TESTALL(HOTP_TYPE_GBOP_HOOK, GLOBAL_VUL(vul_idx).type)))) {
+            name = mod_name;
+        } else {    
+            ASSERT(TESTALL(HOTP_TYPE_HOT_PATCH, GLOBAL_VUL(vul_idx).type));
+            name = pe_name;
+        } 
+
+        for (set_idx = 0; set_idx < GLOBAL_VUL(vul_idx).num_sets; set_idx++) {
+            /* Case 10248 - multiple sets can match, but only the first such set
+             * should be used, the rest discarded.  In the old model only one
+             * set matched, but it was changed to let the patch writer to
+             * progressively relax the matching criteria. */
+            if (set_matched)
+                break;
+
+            for (module_idx = 0;
+                 module_idx < GLOBAL_SET(vul_idx, set_idx).num_modules;
+                 module_idx++) {
+                module = &GLOBAL_MODULE(vul_idx, set_idx, module_idx);
+
+                /* When unloading a matched dll in hotp_only mode, all injected 
+                 * patches must be removed before proceeding any further.  
+                 * Otherwise hotp_module_match() will fail in the id hash 
+                 * computation part due to a changed image, due to injection.
+                 */
+                if (base == module->base_address && !loaded) {
+                    if (just_check) {   /* caller doesn't want processing */
+                        *needs_processing = true;
+                        goto hotp_process_image_exit;
+                    }
+
+                    /* For hotp_only if a module matches all patch points
+                     * in it must be removed in one shot;  just as they are
+                     * injected in one shot.
+                     */
+                    if (GLOBAL_VUL(vul_idx).mode == HOTP_MODE_DETECT ||
+                        GLOBAL_VUL(vul_idx).mode == HOTP_MODE_PROTECT) {
+                        for (ppoint_idx = 0; ppoint_idx < module->num_patch_points;
+                             ppoint_idx++) {
+                            hotp_patch_point_t *ppoint;
+                            ppoint = &module->patch_points[ppoint_idx];
+
+                            if (DYNAMO_OPTION(hotp_only)) {
+                                if (ppoint->trampoline != NULL) {
+                                     hotp_only_remove_patch(module, ppoint);
+                                } else {
+                                    /* If module is matched & mode is on, then the 
+                                     * patch must be injected unless it has been
+                                     * removed to handle loader-safety issues.
+                                     */
+                                    ASSERT(hotp_globals->ldr_safe_hook_removal &&
+                                           "hotp_only - double patch removal");
+                                }
+                            }
+                            /* xref case 10736.
+                             * For hotp_only, module load & inject, and 
+                             * similarly, module unload and remove are done
+                             * together, so hot_patch_point_areas won't be out
+                             * of synch.  However, for hotp with fcache, a
+                             * module unload can remove the patches from
+                             * hotp_patch_point_areas before flushing them.
+                             * This can prevent the flush from happening if
+                             * hotp_patch_point_areas is used for it (which
+                             * isn't done today; case 10728).  It can also
+                             * result in voiding a patch injection for a new bb
+                             * in that module, i.e., module can be without a
+                             * patch for a brief period till it is unloaded.
+                             */
+                            hotp_ppoint_areas_remove(
+                                hotp_ppoint_addr(module, ppoint));
+                        }
+                    }
+
+                    /* Once hotp_only patches are removed, the module must
+                     * match at this point.
+                     * TODO: multiple vulnerabilities targeting the same module
+                     *       & whose hashes overlap, won't be {inject,remove}d 
+                     *       because the image gets modified with the injection
+                     *       of the first one and the hash check for the second
+                     *       one will fail.
+                     */
+                    ASSERT_CURIOSITY(hotp_module_match(module, base, checksum,
+                                                       timestamp, image_size,
+                                                       code_size, file_version,
+                                                       name,
+                                                       GLOBAL_VUL(vul_idx).type));
+                }
+
+                /* FIXME: there's no reason to compute whether an OFF patch
+                 * matches; just wasted cycles, as we come back here on
+                 * any path that later turns the patch on, and no external
+                 * stats rely on knowing whether an off patch matches.
+                 */
+                if (hotp_module_match(module, base, checksum, timestamp, 
+                                      image_size, code_size, file_version, name,
+                                      GLOBAL_VUL(vul_idx).type)) {
+                    set_matched = true;
+                    if (just_check) {   /* caller doesn't want processing */
+                        *needs_processing = true;
+                        goto hotp_process_image_exit;
+                    }
+
+                    if (loaded) {  /* loading dll */
+                        bool patch_enabled =
+                            (GLOBAL_VUL(vul_idx).mode == HOTP_MODE_DETECT ||
+                             GLOBAL_VUL(vul_idx).mode == HOTP_MODE_PROTECT);
+                        LOG(GLOBAL, LOG_HOT_PATCHING, 1, 
+                            "activating vulnerability %s while loading %s\n", 
+                            GLOBAL_VUL(vul_idx).vul_id, module->sig.pe_name);
+
+                        any_matched = true;
+                        /* Case 9970: See if we need to flush any
+                         * perscaches in the region.  Once we decide to flush
+                         * we're going to flush everything.  We avoid the later
+                         * flush on a nudge in vm_area_allsynch_flush_fragments().
+                         * We currently come here for OFF patches, so we explicitly
+                         * check for that before flushing.
+                         */
+                        if (patch_enabled &&
+                            perscache_range_overlap && !flush_perscache &&
+                            DYNAMO_OPTION(use_persisted_hotp)) {
+                            flush_perscache =
+                                hotp_perscache_overlap(vul_idx, set_idx, module_idx,
+                                                       base, image_size);
+                        }
+
+                        /* TODO: check if all modules in the current 
+                         *        vulnerability are active; if so activate the 
+                         *        policy
+                         *        also, add patch points to lookup structures
+                         *        only if entire vulnerability is active;
+                         *        needed to enforce atomicity of patch injection
+                         */
+                        /* the base is used to find the runtime address of 
+                         * patch offset in the current lookup routine; till a 
+                         * offset lookup hash is constructed the base address 
+                         * is needed because the offset in the patchpoint 
+                         * structure is read only data that should be fixed to 
+                         * point to the runtime address.  even then, the flush 
+                         * routine would need to know which offset, i.e., 
+                         * runtime offset, to flush; so this base_address is 
+                         * needed or a runtime data field must be created.
+                         */
+                        module->base_address = base;
+                        module->matched = true;
+                        hotp_set_policy_status(vul_idx, HOTP_INJECT_PENDING);
+
+                        /* gbop type hooks don't have patch offsets defined, 
+                         * as they use function names; must set them otherwise 
+                         * patching will blow up.
+                         */
+                        if (TESTALL(HOTP_TYPE_GBOP_HOOK, GLOBAL_VUL(vul_idx).type)) {
+                            /* FIXME: assert on all patch point fields being 0,
+                             * except precedence.
+                             * also, ASSERT on func_addr & func_name;
+                             */
+                            app_pc func_addr;
+                            app_rva_t offset;
+
+                            /* gbop is only in -client mode, i.e., hotp_only */
+                            ASSERT(DYNAMO_OPTION(hotp_only));
+
+                            func_addr = (app_pc) get_proc_address((module_handle_t)base,
+                                                         GLOBAL_VUL(vul_idx).vul_id);
+                            if (func_addr != NULL) {    /* fix for case 7969 */
+                                ASSERT(func_addr > base);
+                                offset = func_addr - base;
+                                module->patch_points[0].offset = offset;
+                            } else {
+                                /* Some windows versions won't have some gbop
+                                 * hook funcs or get_proc_address might just 
+                                 * fail; either way just ignore such hooks.
+                                 * TODO: think about this - design issue.
+                                 */
+                                module->base_address = NULL;
+                                module->matched = false;
+                                continue;
+                            }
+                        }
+
+                        /* For hotp_only if a module matches all patch points
+                         * in it must be injected in one shot.  
+                         */
+                        if (patch_enabled) {
+                            hotp_offset_match_t ppoint_desc;
+
+                            ppoint_desc.vul_index = vul_idx;
+                            ppoint_desc.set_index = set_idx;
+                            ppoint_desc.module_index = module_idx;
+                            for (ppoint_idx = 0;
+                                 ppoint_idx < module->num_patch_points; 
+                                 ppoint_idx++) {
+                                ppoint_desc.ppoint_index = ppoint_idx;
+
+                                /* ldr_safety can happen only for hotp_only. */
+                                ASSERT(DYNAMO_OPTION(hotp_only) || !ldr_safety);
+
+                                /* Don't re-add a patch point to the vector
+                                 * during patch injection while handling 
+                                 * loader safe injection.
+                                 */
+                                if (!ldr_safety)
+                                    hotp_ppoint_areas_add(&ppoint_desc);
+
+                                if (DYNAMO_OPTION(hotp_only)) {
+                                    hotp_only_inject_patch(&ppoint_desc,
+                                                           all_threads,
+                                                           num_threads);
+                                }
+                            }
+                        }
+                    } else {    /* unloading dll */
+                        /* TODO: same issues as in the 'if' block above, but
+                         *        reverse.
+                         */
+                        module->base_address = NULL;
+                        module->matched = false;
+                        hotp_set_policy_status(vul_idx, HOTP_INJECT_NO_MATCH);
+                    }
+                }
+            }
+        }
+    }
+    
+    if (!DYNAMO_OPTION(use_persisted_hotp)) /* else we check in loop above */
+        flush_perscache = any_matched && perscache_range_overlap;
+    if (flush_perscache) {
+        ASSERT(any_matched && perscache_range_overlap);
+        ASSERT(!DYNAMO_OPTION(hotp_only));
+        /* During startup we process hotp before we add exec areas, so we
+         * should only get a match in a later nudge, when we pass in toflush.
+         */
+        ASSERT(dynamo_initialized);
+        ASSERT(toflush != NULL);
+#ifdef WINDOWS
+        ASSERT(dcontext->nudge_target != NULL);
+#else
+        ASSERT_NOT_REACHED(); /* No nudge on Linux, should only be here for nudge. */
+#endif
+        if (toflush != NULL) { /* be paranoid (we fail otherwise though) */
+            LOG(GLOBAL, LOG_HOT_PATCHING, 2,
+                "Hotp for "PFX"-"PFX" %s overlaps perscache, flushing\n",
+                base, base+image_size, name);
+            /* As we hold the hotp_vul_table_lock we cannot flush here;
+             * instead we add to a pending-flush vmvector.
+             */
+            vmvector_add(toflush, base, base+image_size, NULL);
+            STATS_INC(hotp_persist_flush);
+            /* FIXME: we could eliminate this and rely on our later flush of the
+             * patch area, as we're only coming here for nudges; we technically
+             * only need an explicit check when loading a perscache, as long as
+             * hotp defs are set up first.
+             */
+        }
+    }
+
+ hotp_process_image_exit:
+    if (pe_name != NULL)
+        dr_strfree(pe_name HEAPACCT(ACCT_HOT_PATCHING));
+    if (mod_name != NULL)
+        dr_strfree(mod_name HEAPACCT(ACCT_HOT_PATCHING));
+    /* Don't unlock in case the lock was already obtained before reaching this
+     * function.  Only in that case lock_acquired will be false.
+     */
+    /* TODO: or does this go after flush? */
+    if (!own_hot_patch_lock)
+        write_unlock(&hotp_vul_table_lock);
+    /* TODO: also there are some race conditions with nudging & policy lookup/
+     *       injection; sort those out; flushing before or after reading the 
+     *       policy plays a role too.
+     */
+#ifdef WINDOWS
+    if (num_threads == HOTP_ONLY_NUM_THREADS_AT_INIT) {
+        ASSERT(DYNAMO_OPTION(hotp_only));
+        ASSERT(!just_check);
+        ASSERT_CURIOSITY(check_sole_thread());
+        mutex_unlock(&thread_initexit_lock);
+        mutex_unlock(&all_threads_synch_lock);
+    }
+#endif
+}
+
+/* If vec==NULL, returns the number of patch points for the
+ *   matched vuls in [start,end).
+ * Else, stores in vec the offsets for all the matched patch points in [start,end).
+ * Returns -1 if vec!=NULL and vec_num is too small (still fills it up).
+ * For now this routine assumes that [start,end) is contained in a single module.
+ * The caller must own the hotp_vul_table_lock (as a read lock).
+ */
+static int
+hotp_patch_point_persist_helper(const app_pc start, const app_pc end,
+                                app_rva_t *vec, uint vec_num)
+{
+    uint num_ppoints = 0;
+    uint vul, set, module, pp;
+    /* FIXME: check [start,end) instead of module */
+    app_pc modbase = get_module_base(start);
+    ASSERT(modbase == get_module_base(end));
+    ASSERT(start != NULL); /* only support single module for now */
+    ASSERT_OWN_READ_LOCK(true, &hotp_vul_table_lock);
+    if (GLOBAL_VUL_TABLE == NULL)
+        return 0;
+    /* FIXME: once hotp_patch_point_areas is not just hotp_only, use it here */
+    for (vul = 0; vul < NUM_GLOBAL_VULS; vul++) {
+        bool set_processed = false;
+
+        /* Ignore if off or dll wasn't loaded */
+        if (GLOBAL_VUL(vul).mode == HOTP_MODE_OFF ||
+            GLOBAL_VUL(vul).hotp_dll_base == NULL)
+            continue;
+        for (set = 0; set < GLOBAL_VUL(vul).num_sets; set++) {
+            /* Only the first matching set should be used; case 10248. */
+            if (set_processed)
+                break;
+
+            for (module = 0; module < GLOBAL_SET(vul, set).num_modules; module++) {
+                if (!GLOBAL_MODULE(vul, set, module).matched ||
+                    modbase != GLOBAL_MODULE(vul, set, module).base_address)
+                    continue;
+                set_processed = true;
+                if (vec == NULL) {
+                    num_ppoints += GLOBAL_MODULE(vul, set, module).num_patch_points;
+                } else {
+                    for (pp = 0; pp < GLOBAL_MODULE(vul, set, module).num_patch_points;
+                         pp++) {
+                        if (num_ppoints >= vec_num) {
+                            /* It's ok to get here, just currently no callers do */
+                            ASSERT_NOT_TESTED();
+                            return -1;
+                        }
+                        vec[num_ppoints++] = GLOBAL_PPOINT(vul, set, module, pp).offset;
+                    }
+                }
+            }
+        }
+    }
+    return num_ppoints;
+}
+
+/* Returns the number of patch points for the matched vuls in [start,end).
+ * For now this routine assumes that [start,end) is contained in a single module.
+ * The caller must own the hotp_vul_table_lock (as a read lock).
+ */
+int
+hotp_num_matched_patch_points(const app_pc start, const app_pc end)
+{
+    return hotp_patch_point_persist_helper(start, end, NULL, 0);
+}
+
+/* Stores in vec the offsets for all the matched patch points in [start,end).
+ * Returns -1 if vec_num is too small (still fills it up).
+ * For now this routine assumes that [start,end) is contained in a single module.
+ * The caller must own the hotp_vul_table_lock (as a read lock).
+ */
+int
+hotp_get_matched_patch_points(const app_pc start, const app_pc end,
+                              app_rva_t *vec, uint vec_num)
+{
+    return hotp_patch_point_persist_helper(start, end, vec, vec_num);
+}
+
+/* Checks whether any matched patch point in [start, end) is not listed on
+ * hotp_ppoint_vec.  If hotp_ppoint_vec is NULL just checks whether any patch
+ * point is matched in the region.  For now this routine assumes that
+ * [start,end) is contained in a single module.
+ */
+bool
+hotp_point_not_on_list(const app_pc start, const app_pc end, bool own_hot_patch_lock,
+                       app_rva_t *hotp_ppoint_vec, uint hotp_ppoint_vec_num)
+{
+    /* We could use hotp_process_image_helper()'s just_check but would have
+     * to add hotp_ppoint_vec arg; plus we don't care about module matching.
+     */
+    bool not_on_list = false;
+    uint vul, set, module, pp;
+    /* FIXME: check [start,end) instead of module */
+    app_pc modbase = get_module_base(start);
+    DEBUG_DECLARE(bool matched = false;)
+    ASSERT(modbase == get_module_base(end));
+    if (!own_hot_patch_lock)
+        read_lock(&hotp_vul_table_lock);
+    ASSERT_OWN_READWRITE_LOCK(true, &hotp_vul_table_lock);
+    if (GLOBAL_VUL_TABLE == NULL)
+        goto hotp_policy_list_exit;
+    /* FIXME: I would make an iterator to share w/ patch_point_persist_helper but
+     * this many-nested loop lookup should go away in general ASAP and be
+     * replaced w/ hotp_patch_point_areas which is currently only hotp_only.
+     */
+    for (vul = 0; vul < NUM_GLOBAL_VULS; vul++) {
+        bool set_processed = false;
+
+        /* Ignore if off or dll wasn't loaded */
+        if (GLOBAL_VUL(vul).mode == HOTP_MODE_OFF ||
+            GLOBAL_VUL(vul).hotp_dll_base == NULL)
+            continue;
+        for (set = 0; set < GLOBAL_VUL(vul).num_sets; set++) {
+            /* Only the first matching set should be used; case 10248. */
+            if (set_processed)
+                break;
+
+            for (module = 0; module < GLOBAL_SET(vul, set).num_modules; module++) {
+                if (!GLOBAL_MODULE(vul, set, module).matched ||
+                    modbase != GLOBAL_MODULE(vul, set, module).base_address)
+                    continue;
+                /* We have a match; only ok if on the list */
+                DODEBUG({ matched = true; });
+                set_processed = true;
+                ASSERT(!not_on_list); /* should have exited if not on list */
+                not_on_list = true;
+                if (hotp_ppoint_vec != NULL && DYNAMO_OPTION(use_persisted_hotp)) {
+                    for (pp = 0; pp < GLOBAL_MODULE(vul, set, module).num_patch_points;
+                         pp++) {
+                        if (!hotp_ppoint_on_list(GLOBAL_PPOINT(vul, set, module, pp).
+                                                 offset, hotp_ppoint_vec,
+                                                 hotp_ppoint_vec_num))
+                            goto hotp_policy_list_exit;
+                    }
+                    not_on_list = false;
+                } else
+                    goto hotp_policy_list_exit;
+            }
+        }
+    }
+ hotp_policy_list_exit:
+    if (!own_hot_patch_lock)
+        read_unlock(&hotp_vul_table_lock);
+    DOSTATS({
+        if (matched && !not_on_list) {
+            ASSERT(hotp_ppoint_vec != NULL && DYNAMO_OPTION(use_persisted_hotp));
+            STATS_INC(perscache_hotp_conflict_avoided);
+        }
+    });
+    return not_on_list;
+}
+
+/* TODO: change this to walk the new PE list (not for now though); needed only
+ *          during nudge; start up walk is already done by the core, piggyback
+ *          on that and call hotp_process_image() there; basically, get rid
+ *          of the need to walk the loader list 
+ *          Note: for -probe_api, we walk the module list at start up
+ *                because client init is done after vmareas_init, i.e., after
+ *                scanning for modules in memory and processing them.
+ *                
+ */
+static void
+hotp_walk_loader_list(thread_record_t **all_threads, const int num_threads,
+                      vm_area_vector_t *toflush, bool probe_init)
+{
+/* This routine will go away; till then need to compile on linux.  Not walking
+ * the module list on linux means that no vulnerability will get activated
+ * for injection; that is ok as we aren't trying to build a linux version now.
+ */
+#ifdef WINDOWS
+    /* TODO: this routine uses PEB, etc, this has to be os specific */
+    PEB *peb = get_own_peb();
+    PEB_LDR_DATA *ldr = peb->LoaderData;
+    LIST_ENTRY *e, *start;
+    LDR_MODULE *mod;
+
+    /* For hotp_only, all_threads can be valid, i.e., all known threads may be 
+     * suspended.  Even if they are not, all synch locks will be held, so that 
+     * module processing can happen without races.  Check for that. 
+     * Note: for -probe_api, this routine can be called during dr init time,
+     * i.e., synch locks won't be held, so we shouldn't assert.
+     */
+    if (!probe_init) {
+        ASSERT_OWN_MUTEX(DYNAMO_OPTION(hotp_only), &all_threads_synch_lock);
+        ASSERT_OWN_MUTEX(DYNAMO_OPTION(hotp_only), &thread_initexit_lock);
+    }
+
+    /* Flushing of pcaches conflicting with hot patches is handled for dll
+     * loads by the pcache loads.  Conflicts at hotp_init time can't happen as
+     * pcaches won't be loaded then (they are loaded in vm_areas_init which
+     * comes afterwards).  However for nudging and client init
+     * (dr_register_probes) this is needed because pcaches can be loaded by
+     * then.  Note even though client init happens during startup, it happens
+     * after vm_areas_init, hence pcaches can be loaded.  PR 226578 tracks
+     * implementing pcache flushes for probe api - till then this assert is
+     * relaxed.
+     */
+#ifdef CLIENT_INTERFACE
+    ASSERT(toflush != NULL || DYNAMO_OPTION(hotp_only) ||
+           (DYNAMO_OPTION(probe_api) && !DYNAMO_OPTION(use_persisted)));
+#else
+    ASSERT(toflush != NULL || DYNAMO_OPTION(hotp_only));
+#endif
+
+    start = &ldr->InLoadOrderModuleList;
+    for (e = start->Flink; e != start; e = e->Flink) {
+        mod = (LDR_MODULE*) e;
+
+        /* TODO: ASSERT that the module is loaded? */
+        hotp_process_image_helper(mod->BaseAddress, true, 
+                                  probe_init ? false : true, false, NULL, 
+                                  all_threads, num_threads, false/*!ldr*/,
+                                  toflush);
+
+        /* TODO: make hotp_process_image() emit different log messages
+         *          depending upon which path it is invoked from.
+         */
+    }
+#endif  /* WINDOWS */
+}
+
+void 
+hotp_init(void)
+{
+    ASSIGN_INIT_READWRITE_LOCK_FREE(hotp_vul_table_lock, hotp_vul_table_lock);
+
+    /* Assuming no locks are held on while initializing hot patching. */
+    ASSERT_OWN_NO_LOCKS();
+    ASSERT(DYNAMO_OPTION(hot_patching));
+#ifdef GBOP
+    /* gbop can't be turned on without hotp_only. */
+    ASSERT(DYNAMO_OPTION(hotp_only) || !DYNAMO_OPTION(gbop));
+#endif
+
+    if (DYNAMO_OPTION(hotp_only)) {
+        VMVECTOR_ALLOC_VECTOR(hotp_only_tramp_areas, GLOBAL_DCONTEXT,
+                              VECTOR_SHARED | VECTOR_NEVER_MERGE,
+                              hotp_only_tramp_areas_lock);
+    }
+
+    write_lock(&hotp_vul_table_lock);
+
+#ifdef DEBUG
+    hotp_globals = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, hotp_globals_t,
+                                    ACCT_HOT_PATCHING, PROTECTED);
+    hotp_globals->ldr_safe_hook_removal = false;
+    hotp_globals->ldr_safe_hook_injection = false;
+#endif
+    /* Currently hotp_patch_point_areas is used for hotp_only to do module
+     * matching (case 7279) and offset lookup (case 8132), and offset lookup
+     * only for hotp with fcache (case 10075).  Later on it will be 
+     * used for patch injection, removal, perscache stuff, etc; case 10728.
+     */
+    VMVECTOR_ALLOC_VECTOR(hotp_patch_point_areas, GLOBAL_DCONTEXT,
+                          VECTOR_SHARED | VECTOR_NEVER_MERGE,
+                          hotp_patch_point_areas_lock);
+
+    /* hotp_only trampolines should be allocated on a special heap that allows
+     * code to be executed in it.
+     */
+    if (DYNAMO_OPTION(hotp_only))
+        hotp_only_tramp_heap = special_heap_init(HOTP_ONLY_TRAMPOLINE_SIZE,
+                                                 true, /* yes, use a lock */
+                                                 true, /* make it executable */
+                                                 true /* it is persistent */);
+    ASSERT(GLOBAL_VUL_TABLE == NULL && NUM_GLOBAL_VULS == 0);
+    GLOBAL_VUL_TABLE = hotp_read_policy_defs(&NUM_GLOBAL_VULS);
+    if (GLOBAL_VUL_TABLE != NULL) {
+        hotp_load_hotp_dlls(GLOBAL_VUL_TABLE, NUM_GLOBAL_VULS);
+        hotp_read_policy_modes(NULL);
+        /* Policy status table must be initialized after the global 
+         * vulnerability table and modes are read, but before module list 
+         * is iterated over. 
+         */
+        hotp_init_policy_status_table();
+
+        /* We don't need to call hotp_walk_loader_list() here as
+         * find_executable_vm_areas() will call hotp_process_image() for us.
+         */
+    }
+    else {
+        LOG(GLOBAL, LOG_HOT_PATCHING, 2, "No hot patch definitions to read\n");
+    }
+
+    /* Release locks. */
+    write_unlock(&hotp_vul_table_lock);
+
+    /* Can't hold any locks at the end of hot patch initializations. */
+    ASSERT_OWN_NO_LOCKS();
+}
+
+/* thread-shared initialization that should be repeated after a reset */
+void
+hotp_reset_init(void)
+{
+    /* nothing to do */
+}
+
+/* Free all thread-shared state not critical to forward progress;
+ * hotp_reset_init() will be called before continuing.
+ */
+void
+hotp_reset_free(void)
+{
+    /* Free old tables.  Hot patch code must ensure that no old table
+     * pointer is kept across a synch-all point, which is also a reset
+     * point (case 7760 & 8921).
+     */
+    hotp_vul_tab_t *current_tab, *temp_tab;
+    if (!DYNAMO_OPTION(hot_patching))
+        return;
+    write_lock(&hotp_vul_table_lock);
+    temp_tab = hotp_old_vul_tabs;
+    while (temp_tab != NULL) {
+        current_tab = temp_tab;
+        temp_tab = temp_tab->next;
+        hotp_free_vul_table(current_tab->vul_tab, current_tab->num_vuls);
+        heap_free(GLOBAL_DCONTEXT, current_tab, sizeof(hotp_vul_tab_t)
+                  HEAPACCT(ACCT_HOT_PATCHING));
+    }
+    hotp_old_vul_tabs = NULL;
+    write_unlock(&hotp_vul_table_lock);
+}
+
+/* Free up all allocated memory and delete hot patching lock. */
+void
+hotp_exit(void)
+{
+    /* This assert will ensure that there is only one thread in the process
+     * during exit.  Grab the hot patch lock all the same because a nudge
+     * can come in at this point; freeing things without the lock is bad.
+     */
+    ASSERT(dynamo_exited);
+    ASSERT(DYNAMO_OPTION(hot_patching));
+    write_lock(&hotp_vul_table_lock);
+
+    /* Release the hot patch policy status table if allocated.  This table
+     * may not be allocated till the end if there were no hot patch definitions
+     * but -hot_patching was turned on.
+     */
+    if (hotp_policy_status_table != NULL) {
+        heap_free(GLOBAL_DCONTEXT, hotp_policy_status_table, 
+                  hotp_policy_status_table->size HEAPACCT(ACCT_HOT_PATCHING));
+        hotp_policy_status_table = NULL;
+    }
+
+    /* Release the patch point areas vector before the table. */
+    hotp_ppoint_areas_release();
+    vmvector_delete_vector(GLOBAL_DCONTEXT, hotp_patch_point_areas);
+    hotp_patch_point_areas = NULL;
+
+    /* Release the global vulnerability table and old tables if any. */
+    hotp_free_vul_table(GLOBAL_VUL_TABLE, NUM_GLOBAL_VULS);
+    /* case 8118: set to NULL since referenced in hotp_print_diagnostics() */
+    GLOBAL_VUL_TABLE = NULL;
+
+#ifdef DEBUG
+    HEAP_TYPE_FREE(GLOBAL_DCONTEXT, hotp_globals, hotp_globals_t,
+                   ACCT_HOT_PATCHING, PROTECTED);
+#endif
+    write_unlock(&hotp_vul_table_lock);
+
+    hotp_reset_free();
+
+    if (DYNAMO_OPTION(hotp_only)) {
+#ifdef WINDOWS
+        /* Don't free the heap upon detach - app may have hooked with our
+         * trampoline code; case 9593.  Make this memory efficient, i.e., delete
+         * the heap if no collisions were detected; part of bookkeepping needed
+         * to not leak all removed hotp trampolines, but only those that have a
+         * potential collision; a minor TODO - - would save a max of 50kb.
+         * Note: heap lock should be deleted even if heap isn't! */
+        /* If hotp_only_tramp_heap_cache is NULL, it means that no patches
+         * were removed (either because they weren't injected or just not
+         * removed).  This means we don't have to leak the trampolines even
+         * for detach (PR 215520). */
+        if (!doing_detach || hotp_only_tramp_heap_cache == NULL)
+            special_heap_exit(hotp_only_tramp_heap);
+# ifdef DEBUG
+        else
+            special_heap_delete_lock(hotp_only_tramp_heap);
+# endif
+#else
+        special_heap_exit(hotp_only_tramp_heap);
+#endif
+
+        hotp_only_tramp_heap = NULL;
+        vmvector_delete_vector(GLOBAL_DCONTEXT, hotp_only_tramp_areas);
+        hotp_only_tramp_areas = NULL;
+    }
+
+    DELETE_READWRITE_LOCK(hotp_vul_table_lock);
+}
+
+/* Hot patch policy update action handler */
+bool
+nudge_action_read_policies(void)
+{
+    hotp_vul_t *old_vul_table = NULL, *new_vul_table = NULL;
+    uint num_old_vuls = 0, num_new_vuls;
+    int num_threads = 0;
+    thread_record_t **all_threads = NULL;
+
+    STATS_INC(hotp_num_policy_nudge);
+    /* Fix for case 6090;  TODO: remove when -hotp_policy_size is removed */
+    synchronize_dynamic_options();
+    new_vul_table = hotp_read_policy_defs(&num_new_vuls);
+    if (new_vul_table != NULL) {
+        bool old_value; 
+        hotp_vul_tab_t *temp;
+        where_am_i_t wherewasi;
+        dcontext_t *dcontext = get_thread_private_dcontext();
+        vm_area_vector_t toflush; /* never initialized for hotp_only */
+
+        /* If dynamo_exited was false till the check in this routine, then
+         * this thread would have been intercepted by the core, i.e., it 
+         * would have got a dcontext.  The assert is to catch
+         * bugs; the if is to make sure that the release build doesn't 
+         * crash in case this happens.
+         */
+        ASSERT(dcontext != NULL && dcontext != GLOBAL_DCONTEXT);
+        if (dcontext == NULL) {
+            return false;       /* skip further processing */
+        }
+
+        /* When the nudge thread starts up, the core takes control and 
+         * lets it go once it is identified as nudge.  However, 
+         * under_dynamo_control is still true because we come here from the cache.
+         * We need to set under_dynamo_control to false during hot patch dll loading,
+         * otherwise the core will take over again at the dll loading interception point.
+         * Once hot patch dlls are loaded we restore under_dynamo_control in case it's
+         * relied on elsewhere. Note - this isn't needed for 
+         * loading hot patch dlls at startup because thread init comes 
+         * after hotp_init(), so under_dynamo_control isn't set.  Only 
+         * hot patch dll loading during nudge needs this.
+         * TODO: under_dynamo_control needs cleanup - see case 529, 5183.
+         */
+        old_value = dcontext->thread_record->under_dynamo_control;
+        dcontext->thread_record->under_dynamo_control = false;
+
+        /* Fix for case 5367.  TODO: undo fix after writing own loader. */
+        wherewasi = dcontext->whereami;
+        dcontext->whereami = WHERE_APP;     /* WHERE_APP?  more like WHERE_DR */
+        dcontext->nudge_thread = true;
+
+        /* Fix for case 5376.  There can be a deadlock if a nudge happened
+         * to result in hot patch dlls being loaded when at the same time
+         * an app dll was being loaded; hotp_vul_table_lock & LoaderLock
+         * would create a deadlock.  So while loading the hot patch dlls
+         * the hotp_vul_table_lock shouldn't be held.
+         * To avoid this the table is read, stored in a temporary variable
+         * and hot patch dlls are loaded using that temp. table - all this
+         * is now done without the hotp_vul_table_lock.  Then the vul table
+         * lock is grabbed (see below) and the global table is setup.
+         *
+         * FIXME: The longer term solution is to have our own loader to 
+         * load hot patch dlls.
+         */
+        hotp_load_hotp_dlls(new_vul_table, num_new_vuls);
+
+        /* Must be set to false, otherwise the subsequent module list 
+         * walking will be useless, i.e., won't be able to identify 
+         * modules for hot patching because hotp_process_image() won't work.
+         */
+        dcontext->nudge_thread = false;
+
+        /* If whereami changed, that means, the probably was a callback,
+         * which can lead to other bugs.  So, let us make sure it doesn't.
+         */
+        ASSERT(dcontext->whereami == WHERE_APP);
+        dcontext->whereami = wherewasi;
+        dcontext->thread_record->under_dynamo_control = old_value;
+
+        /* Suspend all threads (for hotp_only) and grab locks. */
+        if (DYNAMO_OPTION(hotp_only)) {
+#ifdef WINDOWS
+            DEBUG_DECLARE(bool ok =)
+                synch_with_all_threads(THREAD_SYNCH_SUSPENDED, &all_threads, 
+                                       /* Case 6821: other synch-all-thread uses that
+                                        * only care about threads carrying fcache
+                                        * state can ignore us
+                                        */
+                                       &num_threads, THREAD_SYNCH_NO_LOCKS_NO_XFER,
+                                       /* if we fail to suspend a thread (e.g., privilege
+                                        * problems) ignore it. FIXME: retry instead? */
+                                       THREAD_SYNCH_SUSPEND_FAILURE_IGNORE);
+            ASSERT(ok);
+#endif
+        }
+        /* CAUTION: Setting up the global table, reading modes, setting up
+         * policy status table and module list walking MUST all be
+         * done in that order with the table lock held as all of them
+         * update the global table.
+         */
+        write_lock(&hotp_vul_table_lock);
+
+        /* For hotp_only, all patches have to be removed before doing 
+         * anything with new vulnerability data, and nothing after that,
+         * which is unlike hotp, where removal has to be done before & after.
+         */
+        if (DYNAMO_OPTION(hotp_only)) {
+            hotp_remove_hot_patches(GLOBAL_VUL_TABLE, NUM_GLOBAL_VULS,
+                                    true, NULL);
+        }
+        /* Save the old table for flushing & launch the new table. */
+        old_vul_table = GLOBAL_VUL_TABLE;
+        num_old_vuls = NUM_GLOBAL_VULS;
+        hotp_ppoint_areas_release();    /* throw out the old patch points */
+        SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+        GLOBAL_VUL_TABLE = new_vul_table;
+        NUM_GLOBAL_VULS = num_new_vuls;
+        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+            
+        hotp_read_policy_modes(NULL);
+
+        /* Policy status table must be initialized after the global 
+         * vulnerability table and modes are read, but before module list 
+         * is iterated over. 
+         */
+        SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+        hotp_init_policy_status_table();
+        if (!DYNAMO_OPTION(hotp_only))
+            vmvector_init_vector(&toflush, 0); /* no lock init needed since not used */
+        hotp_walk_loader_list(all_threads, num_threads,
+                              DYNAMO_OPTION(hotp_only) ? NULL : &toflush,
+                              false /* !probe_init */);
+        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+
+        /* Release all locks. */
+        write_unlock(&hotp_vul_table_lock);
+#ifdef WINDOWS
+        if (DYNAMO_OPTION(hotp_only)) {
+            end_synch_with_all_threads(all_threads, num_threads, true/*resume*/);
+        }
+#endif
+
+        /* If a new vulnerability table was created, then flush the bbs 
+         * with hot patches from the old table and then free that table.  
+         * Note, the old table has to be freed outside the scope of the 
+         * hotp_vul_table_lock because bbs corresponding to that table 
+         * can't be flushed inside it.  See flushing comments below.
+         */
+        ASSERT(old_vul_table != GLOBAL_VUL_TABLE);
+
+        if (!DYNAMO_OPTION(hotp_only)) {
+            if (!vmvector_empty(&toflush)) {
+                ASSERT(DYNAMO_OPTION(coarse_units) && DYNAMO_OPTION(use_persisted));
+                /* case 9970: we must flush the perscache and ibl tables.
+                 * FIXME optimization: don't flush the fine-grained fragments
+                 * or non-persisted unit(s) (there can be multiple).
+                 */
+                flush_vmvector_regions(get_thread_private_dcontext(), &toflush,
+                                       false/*keep futures*/, false/*exec still valid*/);
+            }
+            /* FIXME: don't need to flush non-persisted coarse units since
+             * patch points are fine-grained: would have to widen
+             * flush interface.  Note that we do avoid flushing perscaches
+             * that do not contain the old patch points.
+             */
+            hotp_remove_hot_patches(old_vul_table, num_old_vuls, false, NULL);
+            vmvector_reset_vector(GLOBAL_DCONTEXT, &toflush);
+        } /* else toflush is uninitialized */
+
+        /* Freeing the old vulnerability table immediately causes a race 
+         * with hot patch execution (see case 5521), so it is put on a free
+         * list and freed at a reset or dr exit. hotp_vul_table_lock must be held here;
+         * though this list is a new structure, a new lock is unnecessary.
+         * Also, don't chain empty tables; a NULL table can occur when
+         * no hot patches are loaded during startup, but are nudged in.
+         *
+         * Case 8921: We can't add to the old list prior to removing
+         * hot patches since the synch-all for -coarse_unit or
+         * -hotp_only flushing is a reset point and the table can then
+         * be freed underneath us.  Thus we pay the cost of re-acquiring the lock.
+         * This can also end up with tables on the old list in a different order
+         * than their nudges, but that's not a problem.
+         * FIXME case 8921: -hotp_only should free the table up front
+         * FIXME: we should synch-all once, up front, and then avoid this ugliness
+         * as well as multiple flush synchs.
+         * FIXME: hotp could indirect the table like hotp_only to allow 
+         * earlier freeing.
+         */
+        /* FIXME: don't add to old table list if in hotp_only mode, there is no
+         * eed because there is a lookup before execution and there is no lazy
+         * flush going on.
+         */
+        if (old_vul_table != NULL) {
+            temp = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, hotp_vul_tab_t, 
+                                   ACCT_HOT_PATCHING, PROTECTED);
+            temp->vul_tab = old_vul_table;
+            temp->num_vuls = num_old_vuls;
+            write_lock(&hotp_vul_table_lock);
+            SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+            temp->next = hotp_old_vul_tabs;
+            hotp_old_vul_tabs = temp;
+            SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+            write_unlock(&hotp_vul_table_lock);
+        }
+    }
+    else {
+        /* Note, if the new table wasn't read in successfully, then the old
+         * table isn't touched, i.e., status quo is maintained.
+         */
+        LOG(GLOBAL, LOG_HOT_PATCHING, 2, "No hot patch policies to read\n");
+    }
+    return true;
+}
+
+#ifdef WINDOWS /* no nudging yet on Linux */
+/* This routine handles hot patch nudges. */
+void
+hotp_nudge_handler(uint nudge_action_mask)
+{
+   /* Note, multiple nudges will be synchronized by the hotp_vul_table_lock.    
+    * It is irrelevant if nudge threads change order between reading and 
+    * flushing. */
+
+    ASSERT(DYNAMO_OPTION(liveshields) && DYNAMO_OPTION(hot_patching));
+    ASSERT(nudge_action_mask != 0); /* else shouldn't be called */
+
+    if (TEST(NUDGE_GENERIC(lstats), nudge_action_mask)) {
+        SYSLOG_INTERNAL_WARNING("Stat dumping for hot patches not done yet.");
+    }
+
+    if (TEST(NUDGE_GENERIC(policy), nudge_action_mask)) {
+        LOG(GLOBAL, LOG_HOT_PATCHING, 1, "\n nudged to read in policy\n");
+        nudge_action_read_policies();
+    }
+
+    if (TEST(NUDGE_GENERIC(mode), nudge_action_mask)) {
+        thread_record_t **all_threads = NULL;
+        int num_threads = 0;
+        hotp_policy_mode_t *old_modes = NULL;
+        vm_area_vector_t toflush; /* never initialized for hotp_only */
+
+        LOG(GLOBAL, LOG_HOT_PATCHING, 1, "\n nudged to read in policy\n");
+
+        STATS_INC(hotp_num_mode_nudge);
+
+        /* If -liveshields isn't on, then modes nudges are irrelevant. */
+        if (!DYNAMO_OPTION(liveshields))
+            return;
+
+        /* Suspend all threads (for hotp_only) and grab locks. */
+        if (DYNAMO_OPTION(hotp_only)) {
+            DEBUG_DECLARE(bool ok =)
+                synch_with_all_threads(THREAD_SYNCH_SUSPENDED, &all_threads, 
+                                       /* Case 6821: other synch-all-thread uses that
+                                        * only care about threads carrying fcache
+                                        * state can ignore us
+                                        */
+                                       &num_threads, THREAD_SYNCH_NO_LOCKS_NO_XFER,
+                                       /* if we fail to suspend a thread (e.g., privilege
+                                        * problems) ignore it. FIXME: retry instead? */
+                                       THREAD_SYNCH_SUSPEND_FAILURE_IGNORE);
+            ASSERT(ok);
+        }
+        write_lock(&hotp_vul_table_lock);
+
+        /* For hotp_only, all patches have to be removed before doing anything 
+         * with new mode data; loader list walking will inject new ones.
+         */
+        if (DYNAMO_OPTION(hotp_only)) {
+            hotp_remove_hot_patches(GLOBAL_VUL_TABLE, NUM_GLOBAL_VULS, 
+                                    true, NULL);
+        }
+        hotp_ppoint_areas_release();    /* throw out the old patch points */
+        /* Old modes are for regular hot patching, not for hotp_only. */
+        hotp_read_policy_modes(DYNAMO_OPTION(hotp_only) ? NULL : &old_modes);
+
+        /* Policy status table must be initialized after the global 
+         * vulnerability table and modes are read, but before module list 
+         * is iterated over. 
+         */
+        SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+        hotp_init_policy_status_table();
+        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+
+        if (!DYNAMO_OPTION(hotp_only))
+            vmvector_init_vector(&toflush, 0); /* no lock init needed since not used */
+        hotp_walk_loader_list(all_threads, num_threads,
+                              DYNAMO_OPTION(hotp_only) ? NULL : &toflush,
+                              false /* !probe_init */);
+
+        /* Release all locks. */
+        write_unlock(&hotp_vul_table_lock);
+        if (DYNAMO_OPTION(hotp_only)) {
+            end_synch_with_all_threads(all_threads, num_threads, true/*resume*/);
+        }
+
+        /* If modes did change, then we need to flush out patches that were 
+         * injected because their old modes were on (detect or protect).  
+         * Fix for case 6619; resulted in using old_modes for patch removal.
+         * Note: Just like policy reading, flushing has to be done outside the
+         * scope of the hotp_vul_table_lock & ONLY after reading the new modes.
+         */
+        if (!DYNAMO_OPTION(hotp_only)) {
+            if (!vmvector_empty(&toflush)) {
+                /* case 9970: we must flush the perscache and ibl tables.
+                 * FIXME optimization: don't flush the fine-grained fragments
+                 * or non-persisted unit(s) (there can be multiple).
+                 */
+                flush_vmvector_regions(get_thread_private_dcontext(), &toflush,
+                                       false/*keep futures*/, false/*exec still valid*/);
+            }
+            hotp_remove_hot_patches(GLOBAL_VUL_TABLE, NUM_GLOBAL_VULS, false, 
+                                    old_modes);
+            if (old_modes != NULL) {
+                HEAP_ARRAY_FREE(GLOBAL_DCONTEXT, old_modes, hotp_policy_mode_t,
+                                NUM_GLOBAL_VULS, ACCT_HOT_PATCHING, PROTECTED);
+            }
+            vmvector_reset_vector(GLOBAL_DCONTEXT, &toflush);
+        } /* else toflush is uninitialized */
+    }
+
+    /* Flushing injected bbs must be outside the scope of hotp_vul_table_lock.
+     * Otherwise, flushing will deadlock.  See case 5415.
+     * Though it happens so, it is safe.  The side effect of this is that
+     * bbs with hot patches that have been turned off would still be active
+     * till the flush below, which is ok as they were already active.  Similarly
+     * hot patches that have been turned on will not work until the flush 
+     * happens.
+     * 
+     * There are two flushes for hotp mode per nudge (policy or mode read) and
+     * one for hotp_only mode.  For hotp, the first flush is to clean out bbs
+     * with old/injected patches and is done above (nudge_action_read_policies -
+     * in the case of policy nudge).
+     *
+     * The second flush is to remove bbs corresponding to new policies/modes,
+     * i.e., bbs that were already translated but weren't injected based on 
+     * any old policies/modes, but are by new ones.  This is applicable to 
+     * both policy & mode reading.
+     * Note that for case 9995 we avoided flushing perscaches that do not contain
+     * the new patch points at match time, and we avoid flushing here with
+     * checks in vm_area_allsynch_flush_fragments.
+     */
+    if (TEST(NUDGE_GENERIC(mode), nudge_action_mask) ||
+        TEST(NUDGE_GENERIC(policy), nudge_action_mask)) {
+        if (!DYNAMO_OPTION(hotp_only))
+            hotp_remove_hot_patches(GLOBAL_VUL_TABLE, NUM_GLOBAL_VULS, false, NULL);
+    }
+}
+#endif /* WINDOWS */
+
+/* This is a faster lookup of hotp_vul_table, see case 8132.
+ * FIXME: try to see if this can be merged with hotp_lookup_patch_addr.
+ */
+static bool
+hotp_only_lookup_patch_addr(const app_pc pc, hotp_offset_match_t *match)
+{
+    hotp_offset_match_t *ppoint_desc;
+
+    ASSERT(pc != NULL && match != NULL);
+    ASSERT(DYNAMO_OPTION(hotp_only));
+
+    /* This is always initialized at startup, so can't be NULL at this point. */
+    ASSERT(hotp_patch_point_areas != NULL);
+
+    /* Table read & injection as done together, if a module matches; even if it
+     * doesn't no patching will take place when table is NULL.  Similarly, no
+     * patch is left when table is emptied/cleared for update.  Thus, hotp
+     * won't execute if the global table is null, which is where this lookup is
+     * done. */
+    ASSERT(GLOBAL_VUL_TABLE != NULL);
+
+    ASSERT_OWN_READ_LOCK(true, &hotp_vul_table_lock);
+
+    ppoint_desc = (hotp_offset_match_t *)
+                      vmvector_lookup(hotp_patch_point_areas, pc); 
+    if (ppoint_desc == NULL) {
+        /* Custom data for this vector can't be NULL, so NULL means failure. */
+        return false;
+    } else {
+        *match = *ppoint_desc;
+        return true;
+    }
+}
+
+/* TODO: need to use the concept of a policy activation in addition with pc
+ *       to ensure that the library of the patch point is actually loaded! 
+ *       i think this is best if done at the time of adding/removing patch 
+ *       points to lookup structures
+ * TODO: start using the source (i.e., pc's) dll name to verify 
+ *       patch point/policy with the policy's dll; similar issue as above.
+ * TODO: take an argument for lock; in the bb stage call it with no lock; in
+ *       the injecting stage call it with lock; 
+ * TODO: split up lookup into two, a vm area lookup in the outer decode loop in
+ *       build_bb_ilist() & a pc lookup inside hotp_inject(); the former
+ *       will be racy and will serve as a first level check; the latter is to
+ *       be used only for injection purposes and won't be racy (because it will 
+ *       be called within the scope of GLOBAL_VUL_TABLE or pc hash lock), 
+ *       and won't be visible outside the hotpatch module;
+ *       see is_executable_address() for sample.
+ *       may need new locks for lookup data structures.
+ *
+ * pc lookup should match only if pc matches, dll matches, mode is not 
+ * off and all dlls are available (i.e., vulnerability is active).
+ */
+static bool
+hotp_lookup_patch_addr(const app_pc pc, hotp_offset_match_t *match, 
+                       bool own_hot_patch_lock)
+{
+    bool res = false;
+    hotp_offset_match_t *ppoint_desc = NULL;
+
+    ASSERT(pc != NULL && match != NULL);
+    if (pc == NULL)     /* Defensively exit. */
+        return false;
+
+    /* This is called only during patch clean call injection into fcache, hence
+     * not applicable to hotp_only.
+     */
+    ASSERT(!DYNAMO_OPTION(hotp_only));
+
+    /* There is a remote possibility that GLOBAL_VUL_TABLE can become NULL
+     * between the time the hotp lookup in bb building succeeded and the time
+     * actual patch injection takes place.  This can be caused by a nudge
+     * with an empty or faulty policy config file.  So, can't assert on
+     * GLOBAL_VUL_TABLE not being NULL.
+     */
+    if (GLOBAL_VUL_TABLE == NULL)           /* Nothing to lookup. */
+        return res;
+
+    /* This is always initialized at startup, so can't be NULL at this point. */
+    ASSERT(hotp_patch_point_areas != NULL);
+
+    if (!own_hot_patch_lock)                /* Fix for case 5323. */
+        read_lock(&hotp_vul_table_lock);
+
+    /* Can come here with either the read lock (during instruction matching) or
+     * with the write lock (during injection).
+     */
+    ASSERT_OWN_READWRITE_LOCK(true, &hotp_vul_table_lock);
+
+    ppoint_desc = (hotp_offset_match_t *)
+                      vmvector_lookup(hotp_patch_point_areas, pc);
+    if (ppoint_desc != NULL) {
+
+        /* If the hot patch dll for this vulnerability wasn't loaded for any 
+         * reason, don't even bother with pc matching; we can't execute the 
+         * corresponding patch as it hasn't been loaded.  Fix for case 6032. 
+         * TODO: when splitting up the pc lookup, this should be taken care
+         * of too.
+         * Assert as it is a LiveShield product bug, not dr bug; but handle it.
+         */
+        ASSERT(GLOBAL_VUL(ppoint_desc->vul_index).hotp_dll_base != NULL &&
+               "hot patch dll loaded");
+        if (GLOBAL_VUL(ppoint_desc->vul_index).hotp_dll_base == NULL)
+            goto hotp_lookup_patch_addr_exit;   /* lookup failed */
+
+        /* TODO: check if vul. is ready, i.e., all modules match */
+        ASSERT(GLOBAL_MODULE(ppoint_desc->vul_index, ppoint_desc->set_index, 
+                             ppoint_desc->module_index).matched);
+        ASSERT(GLOBAL_VUL(ppoint_desc->vul_index).mode == HOTP_MODE_DETECT ||
+               GLOBAL_VUL(ppoint_desc->vul_index).mode == HOTP_MODE_PROTECT);
+
+        /* TODO: assert that the indices are within limits */
+        /* TODO: vulnerability is returned without a lock for it,
+         * definitely a problem because it can be updated while being used.
+         * TODO: Also, need to figure out a way to return multiple matches.
+         */
+
+        res = true;             /* vmvector lookup succeeded */
+        if (match != NULL)      /* match can't be NULL, but be cautious. */
+            *match = *ppoint_desc;
+
+        LOG(GLOBAL, LOG_HOT_PATCHING, 1,
+            "lookup for "PFX" succeeded with vulnerability #%s\n",
+            pc, GLOBAL_VUL(ppoint_desc->vul_index).vul_id);
+    }
+
+ hotp_lookup_patch_addr_exit:
+    if (!own_hot_patch_lock)
+        read_unlock(&hotp_vul_table_lock);
+    return res;
+}
+
+/* Returns true if the region passed in should be patched and the module is
+ * ready, i.e., loaded & matched.
+ * Note: start and end define a region that is looked up a vmvector,
+ * hotp_patch_point_ares.  Though our vmvector can accept end being NULL,
+ * signifying no upper ceiling, it doesn't make sense of hot patch lookup - at
+ * best it signifies an error somewhere.  So a NULL for end will be treated as a
+ * lookup failure.
+ */
+bool
+hotp_does_region_need_patch(const app_pc start, const app_pc end,
+                            bool own_hot_patch_lock)
+{
+    bool res = false;
+    ASSERT(start != NULL && end != NULL);
+
+    if (start == NULL || end == NULL)
+        return false;
+
+    /* This is called only for finding out if a bb needs a hot patch, so can't
+     * be used for hotp_only.
+     */
+    ASSERT(!DYNAMO_OPTION(hotp_only));
+
+    /* Called during bb building even when there is no hot patch info available. */
+    if (GLOBAL_VUL_TABLE == NULL)
+        return false;
+
+    /* This is always initialized at startup, so can't be NULL at this point. */
+    ASSERT(hotp_patch_point_areas != NULL);
+
+    if (!own_hot_patch_lock)                /* Fix for case 5323. */
+        read_lock(&hotp_vul_table_lock);
+
+    /* Caller must come in with lock - that is the use today.  However, this
+     * doesn't need the caller to hold the hotp_vul_table_lock; can do so by
+     * itself.  Imposed by fix for case 8780 - excessive holding of hotp lock.
+     * need to find a better solution (FIXME).
+     */
+    ASSERT_OWN_READWRITE_LOCK(true, &hotp_vul_table_lock);
+
+    res = vmvector_overlap(hotp_patch_point_areas, start, end);
+
+    if (!own_hot_patch_lock)
+        read_unlock(&hotp_vul_table_lock);
+
+    return res;
+}
+
+/* for the given ilist, it will insert the call to hot patch gateway before 
+ * instruction 'where' such that the hot patch corresponding to the given 
+ * 'policy' will be invoked with the mode specified by the policy.
+ *
+ * The disassembly of what is injected at each patch point is shown below.
+ * Note: the disassembly below may change based on the new design.
+ *
+    Clean call preparation
+        Note: this clean call prep shows accessing dcontext directly, i.e., as
+        in thread private case.  In the shared fragments case dcontext will
+        first be loaded from the TLS.  See prepare_for_clean_call() for details.
+
+        mov    %esp -> dcontext.mcontext.mcontext.xsp
+        mov    dcontext.dstack -> %esp 
+        pushf  
+        pusha  ; save app reg. state
+        push   $0x00000000 %esp -> %esp (%esp) 
+        popf   %esp (%esp) -> %esp 
+        addr16 mov    %fs:0x34 -> %eax  ; last error value
+        push   %eax %esp -> %esp (%esp) 
+
+    Save app state & make call to hotp_gateway
+        if (SHARED_FRAGMENTS_ENABLED()) {
+            mov    %fs:TLS_DCONTEXT_SLOT -> %eax
+            mov    %eax(DSTACK_OFFSET) -> %eax
+        } else {
+            mov    dcontext.dstack -> %eax   ; locate app reg state on stack
+        }
+        sub    HOTP_CONTEXT_OFFSET_ON_DSTACK,%eax -> %eax 
+
+    pusha was done on dr stack, so esp is dr's; get and spill the app's esp
+        if (SHARED_FRAGMENTS_ENABLED()) {
+            mov    %fs:TLS_DCONTEXT_SLOT -> %ecx
+            mov    %ecx(XSP_OFFSET) -> %ecx
+        } else {
+            mov    dcontext.mcontext.mcontext.xsp -> %ecx
+        }
+        mov    %ecx -> [%eax + 0xc] ; 0xc == offsetof(hotp_context_t, xsp)
+
+ Note: Don't send func_ptr; security hazard; use indices into hotp_vul_table -
+       one for vul, set, mod & ppt; this way the gateway can pick out the exact
+       hotpatch offset from the table which is in read only memory; this also
+       avoids the need to maintain a hash for hot patch offsets which can be
+       looked up by hotp_gateway() before doing the hot patch call.
+
+        push   false;   don't have the hotp_vul_table_lock
+        push   $eax ;   app_reg_ptr
+        push   ppoint_index
+        push   module_index
+        push   set_index
+        push   vul_index
+        push   num_vuls
+        push   vul_table_ptr
+        call   hotp_gateway()
+
+    The hot patch could have changed esp we sent to it via app_reg_ptr.  As we 
+    restore esp from dcontext, save app_reg_ptr->xsp in the dcontext.
+    Fix for case 5594. 
+        app_reg_ptr = dstack - HOTP_CONTEXT_OFFSET_ON_DSTACK
+        app_esp_p = app_reg_ptr + offsetof(hotp_context_t, xsp)
+    See clean call above.
+
+        if (SHARED_FRAGMENTS_ENABLED()) {
+            mov    %fs:TLS_DCONTEXT_SLOT -> %eax
+            mov    %eax(DSTACK_OFFSET) -> %eax
+        } else {
+            mov    dcontext.dstack -> %eax
+        }
+        mov    (%eax-$0x14) -> %eax   ; eax = [app_esp_p]
+        if (SHARED_FRAGMENTS_ENABLED()) {
+            mov    %fs:TLS_DCONTEXT_SLOT -> %ecx
+            mov    $eax -> %ecx(XSP_OFFSET)
+        } else {
+            mov    %eax -> dcontext.mcontext.mcontext.xsp
+        }
+
+    Clean call cleanup
+        add    $0x1c %esp -> %esp   ; pop off the 7 args to hotp_gateway()
+        pop    %esp (%esp) -> %eax %esp 
+        addr16 mov    %eax -> %fs:0x34  ; last error value 
+        popa   ; restore app reg. state
+        popf   
+        mov    dcontext.mcontext.mcontext.xsp -> %esp 
+
+ * CAUTION: Any change to this function will affect hotp_change_control_flow().
+ *          What is stored in the app/dr stack by the code generated by this 
+ *          routine is used and modified by hotp_change_control_flow().
+ */
+/* TODO: PR 226888 - make hotp bbs shared - they are enabled to be shared, but
+ *          actually aren't shared yet.
+ */
+static int 
+hotp_inject_gateway_call(dcontext_t *dcontext, instrlist_t *ilist,
+                         instr_t *where, const hotp_offset_match_t *match)
+{
+    /* TODO: use a separate stack later on; don't pollute the dr stack;
+     * for now use dr stack for executing the hot patch code.
+     */
+    /* NOTE: app_reg pointer computation assumes certain behavior from 
+     * dr_prepare_for_call, i.e., first thing is all app registers are 
+     * pushed on to DR stack; eax is scratch at this point.  
+     * TODO: Add asserts here for these.
+     */
+#define HOTP_CONTEXT_OFFSET_ON_DSTACK   sizeof(hotp_context_t)
+
+/* Loads contents of dcontext at offset to reg.  For shared fragments it is 
+ * loaded via dc_reg; load dc into dc_reg if it isn't available (!have_dc).
+ */
+#define GET_FROM_DC_OFFS_TO_REG(offset, reg, have_dc, dc_reg)                  \
+    do {                                                                       \
+        if (SHARED_FRAGMENTS_ENABLED()) {                                      \
+            if (!have_dc)                                                      \
+                insert_get_mcontext_base(dcontext, ilist, where, dc_reg);      \
+            MINSERT(ilist, where, instr_create_restore_from_dc_via_reg         \
+                    (dcontext, dc_reg, reg, offset));                          \
+        } else {                                                               \
+            MINSERT(ilist, where, instr_create_restore_from_dcontext           \
+                    (dcontext, reg, offset));                                  \
+        }                                                                      \
+    } while (0)
+
+    /* Using client api to avoid duplicating code. */
+    /* FIXME PR 226036: set hotp_context_t pc field?  left as 0 by dr_prepare_for_call */
+    dr_prepare_for_call(dcontext, ilist, where);
+
+    /* DSTACK_OFFSET isn't within the upcontext so if it's separate our use of
+     * insert_get_mcontext_base() above is incorrect. */
+    ASSERT_NOT_IMPLEMENTED(!TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask));
+
+    /* We push eax as a parameter to the call */
+    GET_FROM_DC_OFFS_TO_REG(DSTACK_OFFSET, REG_XAX, false /* !have_dc */, REG_XBX);
+
+    /* app reg ptr is put in eax */
+    MINSERT(ilist, where, INSTR_CREATE_sub
+            (dcontext, opnd_create_reg(REG_XAX),
+             OPND_CREATE_INT8(HOTP_CONTEXT_OFFSET_ON_DSTACK)));
+
+    /* Get the app esp stored in dcontext.mcontext & spill it in the right
+     * location for the hot patch code. 
+     */
+    GET_FROM_DC_OFFS_TO_REG(XSP_OFFSET, REG_XCX, true /* have_dc */, REG_XBX);
+    MINSERT(ilist, where, INSTR_CREATE_mov_st
+            (dcontext, OPND_CREATE_MEM32(REG_XAX, offsetof(hotp_context_t, xsp)),
+             opnd_create_reg(REG_XCX)));
+
+    dr_insert_call(dcontext, ilist, where, (app_pc)&hotp_gateway, 8,
+                   OPND_CREATE_INTPTR(GLOBAL_VUL_TABLE),
+                   OPND_CREATE_INT32(NUM_GLOBAL_VULS),
+                   OPND_CREATE_INT32(match->vul_index),
+                   OPND_CREATE_INT32(match->set_index),
+                   OPND_CREATE_INT32(match->module_index),
+                   OPND_CREATE_INT32(match->ppoint_index),
+                   /* app reg ptr put in eax above */
+                   opnd_create_reg(REG_XAX),
+                   OPND_CREATE_INT32(false));
+
+    /* TODO: also, for multiple patch points for one offset, gateway will have
+     *       to take variable arguments, i.e., one set per patch.
+     */
+
+    /* Copy app esp from context passed to hot patch into mcontext to set up
+     * for restore.  Fix for case 5594.
+     */
+    GET_FROM_DC_OFFS_TO_REG(DSTACK_OFFSET, REG_XAX, false /* !have_dc */, REG_XBX);
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(offsetof(hotp_context_t, xsp))));
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(HOTP_CONTEXT_OFFSET_ON_DSTACK)));
+    MINSERT(ilist, where, INSTR_CREATE_mov_ld
+            (dcontext, opnd_create_reg(REG_XAX),
+             OPND_CREATE_MEM32(REG_XAX, (int)offsetof(hotp_context_t, xsp) -
+                               (int)HOTP_CONTEXT_OFFSET_ON_DSTACK)));
+
+    if (SHARED_FRAGMENTS_ENABLED()) {
+        MINSERT(ilist, where, instr_create_save_to_dc_via_reg
+                (dcontext, REG_XBX, REG_XAX, XSP_OFFSET));
+    } else {
+        MINSERT(ilist, where, instr_create_save_to_dcontext
+                (dcontext, REG_XAX, XSP_OFFSET));
+    }
+
+    dr_cleanup_after_call(dcontext, ilist, where, 0);
+
+    return 1;   /* TODO: why return anything here? */
+}
+
+/* If the given ilist has instructions that are targeted by any vulnerabilities,
+ * this routine will identify those policies and insert code into the basic block to
+ * call the hot patch code corresponding to the matching vulnerabilities.
+ *
+ * Note: Expand the ilist corresponding to the bb only if a hot patch needs to
+ * be injected into it; taken care of by the boolean that predicates the
+ * call to this function. 
+ */
+bool 
+hotp_inject(dcontext_t *dcontext, instrlist_t *ilist)
+{
+    bool injected_hot_patch = false;
+    instr_t *instr, *next;
+    hotp_offset_match_t match = {-1, -1, -1, -1};
+    app_pc translation_target = NULL;           /* Fix for case 5981. */
+    bool caller_owns_hotp_lock = self_owns_write_lock(hotp_get_lock());
+
+    /* This routine is for injecting hot patches into an ilist, i.e., into the
+     * fcache.  Shouldn't be here for -hotp_only which patches the image.
+     */
+    ASSERT(!DYNAMO_OPTION(hotp_only));
+
+    if (!caller_owns_hotp_lock)
+        write_lock(&hotp_vul_table_lock);   /* Fix for case 5323. */
+
+    /* Expand the ilist corresponding to the basic block and for each 
+     * instruction in the ilist, check if one or more injections to 
+     * the gateway should be made and then do so.
+     */
+    instr = instrlist_first_expanded(dcontext, ilist);
+    while (instr != NULL) {
+        next = instr_get_next_expanded(dcontext, ilist, instr);
+
+        /* TODO: must have way to ensure that all offsets matched for this 
+         * basic block are patched (not missed) and correctly too. but how?
+         */
+
+        /* TODO: hotp_lookup_patch_addr(), i.e., the second/internal lookup should
+         *       be able to return multiple matching vulnerabilities/ppoints -
+         *       need a new data structure for it. 
+         */
+        /* TODO: for now this is just one vul, so no loop is used inside 
+         * this if; must change to handle multiple matching policies, i.e, 
+         * multiple injections;  that should handle precedences if offsets are
+         * the same.
+         */
+        if (hotp_lookup_patch_addr(instr_get_raw_bits(instr), &match,
+                                   true /* own hotp_vul_table_lock */)) {
+            /* The mode better be either protect or detect at this point! */
+            hotp_policy_mode_t mode = GLOBAL_VUL(match.vul_index).mode;
+            ASSERT (mode == HOTP_MODE_DETECT || mode == HOTP_MODE_PROTECT);
+
+            LOG(GLOBAL, LOG_HOT_PATCHING, 1, "injecting into %s at "PIFX"\n", 
+                GLOBAL_MODULE(match.vul_index, match.set_index, 
+                              match.module_index).sig.pe_name,
+                GLOBAL_PPOINT(match.vul_index, match.set_index,
+                              match.module_index, match.ppoint_index).offset);
+            /* TODO: assert somewhere that a given vul can't patch the same 
+             * offset twice in a given module.  guess this can be done at vul
+             * table creation time, i.e., during startup or nudge from nodemgr.
+             */
+            /* The translation target for the inserted instructions is set to 
+             * the address of the instruction preceding the one to be patched. 
+             * Otherwise if an app exception happens in this bb 
+             * recreate_app_state_from_ilist() would fail.  
+             * Note: the only problem is if the first instruction in a bb is 
+             *       the patchee; in that case we use that address itself 
+             *       though the exception handler will complain about not 
+             *       being able to create app state.  However, it will get the
+             *       right state, so we are fine in release builds.  FIXME.
+             * Part of fix for case 5981. 
+             */
+            if (translation_target == NULL)
+                translation_target = instr_get_raw_bits(instr);
+            instrlist_set_translation_target(ilist, translation_target);
+            instrlist_set_our_mangling(ilist, true); /* PR 267260 */
+            hotp_inject_gateway_call(dcontext, ilist, instr, &match);
+            instrlist_set_translation_target(ilist, NULL);
+            instrlist_set_our_mangling(ilist, false); /* PR 267260 */
+            STATS_INC(hotp_num_inject);
+            injected_hot_patch = true;
+            if (mode == HOTP_MODE_DETECT)
+                hotp_set_policy_status(match.vul_index, HOTP_INJECT_DETECT);
+            else
+                hotp_set_policy_status(match.vul_index, HOTP_INJECT_PROTECT);
+        }
+        translation_target = instr_get_raw_bits(instr);
+        instr = next;
+    }
+    if (!caller_owns_hotp_lock)
+        write_unlock(&hotp_vul_table_lock);
+    return injected_hot_patch;
+}
+
+/* For hotp_only, a patch region shouldn't contain any jmp, call, ret or int
+ * instructions that start and end within it; it is ok if a jmp, a call, 
+ * a ret or a int spans the entire patch region or beyond it.  This is to
+ * ensure that no control flow can come into the middle of a patch region.
+ * Those valid calls/jmps that can exist in the patch region should only target
+ * some image address that belongs to the app, not stack or heap ==>
+ * ==> FIXME case 7657: need to relax that to allow 3rd party hookers (and, app
+ * itself could be targeting heap).
+ * Also, the patch region shouldn't be already hooked by the core's hooks, i.e.,
+ * non hotp_only core hooks.
+ * TODO: strengthen this function; today it checks for what is not allowed and
+ *       allows all else; make it check for what is allow too, i.e., be precise
+ *       because assumptions can break with instruction extensions.
+ */
+static bool
+hotp_only_patch_region_valid(const app_pc addr_to_hook)
+{
+    instr_t *inst;
+    app_pc pc = addr_to_hook;
+    app_pc start_pc = addr_to_hook;
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    bool res = true;
+
+    /* As of today hot patches can only target the .text section in a module. */
+    DODEBUG({
+        if (!is_in_code_section(get_module_base(addr_to_hook), addr_to_hook, NULL, NULL))
+            return false;
+    });
+
+    /* Happens during hotp_init(); thread init happens afterwards so dcontext 
+     * isn't set up.
+     */
+    if (dcontext == NULL)
+        dcontext = GLOBAL_DCONTEXT;
+
+    inst = instr_create(dcontext);
+    while (pc < addr_to_hook + HOTP_PATCH_REGION_SIZE) {
+        instr_reset(dcontext, inst);
+        pc = decode(dcontext, pc, inst);
+        if (instr_is_cti(inst) || instr_is_interrupt(inst)) {
+            /* cti is in patch region followed by other instructions in it. 
+             * Shouldn't patch this as control can come into the middle of the 
+             * patch region.
+             */
+            if ((start_pc + instr_length(dcontext, inst)) <
+                 (addr_to_hook + HOTP_PATCH_REGION_SIZE)) {
+                LOG(GLOBAL, LOG_HOT_PATCHING, 2, "invalid hotp_only patch point"
+                    " at "PFX"; there is cti inside it\n", start_pc);
+                res = false;
+                goto hotp_only_patch_region_valid_exit;
+            } else {
+                /* cti is in the patch region & spans till or beyond the end
+                 * of the patch region, i.e., this region is valid.
+                 */
+                if (instr_is_call(inst)) {
+                    /* FIXME: Mangling calls in patch regions hasn't been done 
+                     * yet.  See case 6839.
+                     */
+                    LOG(GLOBAL, LOG_HOT_PATCHING, 1, "Warning: not mangling "
+                        "valid call in hotp_only patch region; not supported "
+                        "yet, see case 6839.");
+                }
+                if (instr_is_call_direct(inst) || instr_is_ubr(inst) ||
+                    instr_is_cbr(inst)) {
+                    app_pc target = instr_get_branch_target_pc(inst);
+                    /* FIXME: core doesn't handle far ctis today, see case 6962;
+                     * when far ctis are handled, this assert can go.
+                     */
+                    ASSERT(!instr_is_far_abs_cti(inst));
+#ifdef WINDOWS
+                    /* Does it overlap with any of the core's hooks?  
+                     * Note: native_exec_syscalls don't use the landing pad as
+                     * of now, so we still have to look at the
+                     * interception_buffer.  Also, the vmvector_overlap may
+                     * trigger for hotp_only hooks too.  Once native_exec
+                     * hooking uses landing pads change this so that the target
+                     * of the landing pad is checked to see if it is in the
+                     * interception buffer.  Not a big deal as both result in
+                     * the hooking being aborted - just the log message
+                     * changes.
+                     */
+                    if (vmvector_overlap(landing_pad_areas, target, target + 1) ||
+                        is_in_interception_buffer(target)) {
+                        LOG(GLOBAL, LOG_HOT_PATCHING, 2, "invalid hotp_only "
+                            "patch point at "PFX"; it collides with a core hook\n",
+                            start_pc);
+                        res = false;
+                        goto hotp_only_patch_region_valid_exit;
+                    }
+#endif /* WINDOWS */
+                    /* Overlaps with any injected hot patch?  Case 7998.
+                     * This is not infrequent, some dlls like urlmon or rpcrt4
+                     * have a .orpc section which results in an unmatched page
+                     * protection change, like rw-, r-x, r-x; the last one
+                     * results in double injection, which should be ignored.
+                     * See case 9588 and 9906 where this causes a crash.
+                     * Note: as all hotp_only hooks go through landing pads we
+                     * don't have to check hotp_only_tramp_areas.
+                     */
+                    if (vmvector_overlap(landing_pad_areas, target, target + 1)) {
+#ifdef WINDOWS    /* WINDOWS_VERSION_2003 doesn't exist on linux. */
+                        DODEBUG({
+                            const char *reason = "unknown";
+                            if (hotp_globals->ldr_safe_hook_injection) {
+                                reason = "due to loader safety";
+                            } else if (get_os_version() >= WINDOWS_VERSION_2003){
+                                /* On 2k3 loader lock isn't held during dll
+                                 * loading before executing image entry, so we 
+                                 * can't tell for sure. */
+                                /* FIXME case 10636: what about vista? */
+                                reason = "2003; may be due to loader safety";
+                            } else {
+                                ASSERT_NOT_REACHED();   /* Unknown reason. */
+                            }
+                            LOG(GLOBAL, LOG_HOT_PATCHING, 2, "Blocking double "
+                                "injection at "PFX" in module at "PFX" - %s\n",
+                                start_pc, get_module_base(start_pc), reason);
+                        });
+#endif
+                        res = false;
+                        goto hotp_only_patch_region_valid_exit;
+                    }
+
+                    /* This check concludes it is a 3rd party hook if
+                     * target is not in current image; target may be in another
+                     * image, mapped read-only file, or heap.  The first two
+                     * may not be 3rd party hook conflicts (rare).  For now, we
+                     * conservatively conclude these to be hook conflicts.
+                     * Note: Whether a hook targets image or heap has no
+                     * bearing on how easily we can interop with it.
+                     * FIXME: track patch point from mmap to point of hooking
+                     * to see if it is hooked before concluding hook conflict;
+                     * case 10433.
+                     */
+                    DODEBUG({
+                        if (!is_in_any_section(get_module_base(start_pc), target,
+                                               NULL, NULL)) {
+                            LOG(GLOBAL, LOG_HOT_PATCHING, 2, "cti in patch region "
+                                PFX"; cti target "PFX" isn't inside image "
+                                "- potential 3rd-party hooker", start_pc, target);
+                            SYSLOG_INTERNAL_WARNING("Potential 3rd party hook "
+                                                    "conflict at "PFX, start_pc);
+                        }
+                    });
+
+                    /* No app jump should be targeting the core. */
+                    if (is_in_dynamo_dll(target)) {
+                        LOG(GLOBAL, LOG_HOT_PATCHING, 2, "invalid hotp_only "
+                            "patch point at "PFX"; cti targets dynamorio.dll!\n", 
+                            start_pc);
+                        ASSERT_NOT_REACHED();
+                        res = false;
+                        goto hotp_only_patch_region_valid_exit;
+                    }
+                    SYSLOG_INTERNAL_WARNING_ONCE("cti found at hotp point, will chain");
+                    LOG(GLOBAL, LOG_HOT_PATCHING, 2, "found chainable cti at "
+                        "patch point at "PFX"\n", start_pc);
+                }
+            }
+        }
+        start_pc = pc;
+    }
+ hotp_only_patch_region_valid_exit:
+    instr_destroy(dcontext, inst);
+    return res;
+}
+
+/* Injects one hotp_only patch, i.e., inserts trampoline to execute a hot patch.
+ *
+ * FIXME: multi-thread safe injection hasn't been implemented; when that is 
+ *        implemented this routine will have to assert that all threads in this 
+ *        process have stopped.  see case 6662.
+ *        Note: injections are done per module, not for the whole policy table,
+ *              so there might be performance issues with stopping and resuming
+ *              all threads for each module to be patched.
+ *             
+ * FIXME: injection currently doesn't check if loader is finished with a module
+ *        before injecting; needs to be done.  also, while injecting the loader
+ *        shouldn't be allowed to modify the module.  see case 6662.
+ *
+ * FIXME: patch removal hasn't been implemented yet for hotp_only; when doing so
+ *        trampoline code must be released, hook removed & image processed to
+ *        set it to unmatched.  see case 6663.
+ */
+static void
+hotp_only_inject_patch(const hotp_offset_match_t *ppoint_desc, 
+                       const thread_record_t **all_threads, const int num_threads)
+{
+    hotp_vul_t *vul;
+    hotp_set_t *set;
+    hotp_module_t *module;
+    hotp_patch_point_t *ppoint, *cur_ppoint;
+    uint ppoint_idx;
+    app_pc addr_to_hook, cflow_target;
+    byte *end;
+    bool patched = false;
+
+    ASSERT(DYNAMO_OPTION(hotp_only));
+
+    /* At startup there should be no other thread than this, so all_threads 
+     * won't be valid.
+     */
+    if (num_threads != HOTP_ONLY_NUM_THREADS_AT_INIT) {
+        ASSERT(ppoint_desc != NULL && all_threads != NULL);
+    } else {
+        ASSERT(ppoint_desc != NULL && all_threads == NULL);
+    }
+
+    /* Check if it is safe to patch, i.e., no known threads should be running
+     * around (of course for the unknown thread this won't help; see 
+     * hotp_init() for the comment about that corner case).
+     */
+    ASSERT_OWN_MUTEX(true, &all_threads_synch_lock);
+    ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
+    ASSERT_OWN_READWRITE_LOCK(true, &hotp_vul_table_lock);
+
+    vul = &GLOBAL_VUL(ppoint_desc->vul_index);
+    set = &vul->sets[ppoint_desc->set_index];
+    module = &set->modules[ppoint_desc->module_index];
+    cur_ppoint = &module->patch_points[ppoint_desc->ppoint_index];
+    addr_to_hook = hotp_ppoint_addr(module, cur_ppoint);
+
+    /* Can't inject a hot patch if its container dll isn't loaded.  This can
+     * happen if there is a bug in the policy def file or while core was loading
+     * the dll.
+     */
+    if (vul->hotp_dll_base == NULL) {
+        SYSLOG_INTERNAL_WARNING("Hot patch dll (%s) hasn't been loaded; "
+                                "aborting hotp_only injection", vul->hotp_dll);
+        ASSERT(false);
+        return;
+    }
+
+    /* If addr_to_hook doesn't conform to the patch region definition, then 
+     * don't inject the patch.
+     */
+    if (!hotp_only_patch_region_valid(addr_to_hook)) {
+        STATS_INC(hotp_only_aborted_injects);
+        return;
+    }
+
+    if (cur_ppoint->trampoline != NULL) {
+        /* FIXME case 9148/7657: we can have hookers who chain off our old
+         * trampoline via a +rwx prot change followed by a +rx change that
+         * triggers us adding new hooks without removing the old.  We go ahead
+         * and leave that bug in and live with the leak for now since it works
+         * out better in terms of chaining (o/w we will re-use old trampoline
+         * buffers that are pointed to by the hooker's chaining, causing
+         * infinite recursion, incorrect API calls, or worse).  Our hook code is
+         * then called twice, but this can only happen for GBOP (o/w the hash
+         * wouldn't match) which can handle duplicate checks.  We need a
+         * comprehensive hooker + loader compatibility policy that minimizes
+         * these types of problems (case 7657).
+         */
+        SYSLOG_INTERNAL_WARNING("patch point "PFX" in module %s being re-patched; "
+                                "old patch leaked", addr_to_hook, module->sig.pe_name);
+        ASSERT(cur_ppoint->app_code_copy != NULL);
+    } else {
+        ASSERT(cur_ppoint->app_code_copy == NULL);
+        ASSERT(cur_ppoint->tramp_exit_cti == NULL);
+    }
+
+    /* Shouldn't be injecting anything that is isn't turned on. */
+    ASSERT(vul->mode == HOTP_MODE_DETECT || vul->mode == HOTP_MODE_PROTECT);
+
+    /* Make sure that patch region size isn't messed up. */
+    ASSERT(HOTP_PATCH_REGION_SIZE == HOTP_ONLY_PATCH_REGION_SIZE);
+
+    cur_ppoint->trampoline = (byte *) special_heap_alloc(hotp_only_tramp_heap); 
+
+    /* The patch region has been checked for validity by now, so if there are
+     * other hooks in there smash them.  Also, control flow change is 
+     * implemented using AFTER_INTERCEPT_DYNAMIC_DECISION hooking model and 
+     * using AFTER_INTERCEPT_LET_GO_ALT_DYN; the only difference being that the 
+     * alternate target is not provided at hook time because it is unknown till 
+     * hooking is completed.  The alternate target is provided after hooking; 
+     * see below in the hook conflict resolution code.
+     */
+    end = hook_text(cur_ppoint->trampoline, addr_to_hook, hotp_only_gateway, 
+                    (void *) addr_to_hook,
+                    cur_ppoint->return_addr != 0 ?
+                        AFTER_INTERCEPT_DYNAMIC_DECISION : 
+                        AFTER_INTERCEPT_LET_GO, 
+                    false, /* don't abort if hooked, smash it */
+                    true,  /* ignore ctis; they have been checked for already */
+                    &cur_ppoint->app_code_copy,
+                    cur_ppoint->return_addr != 0 ? &cur_ppoint->tramp_exit_cti :
+                                                   NULL);
+
+    /* Did we hook it successfully? */
+    ASSERT(*addr_to_hook == JMP_REL32_OPCODE);
+
+    /* Trampoline code shouldn't overflow the trampoline buffer here.  By now
+     * the damage is already done.  In a debug build it is ok, but in a release
+     * build?  FIXME: need to make intercept_call() take a buffer length.
+     */
+    ASSERT((end - cur_ppoint->trampoline) <= HOTP_ONLY_TRAMPOLINE_SIZE);
+
+    /* The copy of the hooked app code should be within the trampoline. */
+    ASSERT(HOTP_ONLY_IS_IN_TRAMPOLINE(cur_ppoint, cur_ppoint->app_code_copy));
+
+    /* If the current hot patch has a control flow change address then the
+     * cti that does the control flow change should be inside the trampoline.
+     */
+    ASSERT(cur_ppoint->return_addr == 0 ||
+           HOTP_ONLY_IS_IN_TRAMPOLINE(cur_ppoint, cur_ppoint->tramp_exit_cti));
+    
+    /* Now that the trampoline has been created to our satisfaction, add it to
+     * the trampoline vector.  Note, all thread synch locks & hot patch locks
+     * must be held before adding anything to the vector.
+     */
+    vmvector_add(hotp_only_tramp_areas, cur_ppoint->trampoline,
+                 cur_ppoint->trampoline + HOTP_ONLY_TRAMPOLINE_SIZE, 
+                 (void *) cur_ppoint);
+
+    if (cur_ppoint->return_addr != 0) {
+        /* As of today our trampolines only use rel32 jmps. */
+        ASSERT(*cur_ppoint->tramp_exit_cti == JMP_REL32_OPCODE);
+
+        /* A hot patch can't change control flow to go to the point where it 
+         * is injected; would lead to an infinite loop. 
+         */
+        ASSERT(cur_ppoint->return_addr != cur_ppoint->offset);
+
+        /* Go through all the patch points in this module, including the 
+         * current one, to see if the current patch point's 
+         * control-flow-change-target is in the middle of any patch region 
+         * that has been hooked by the core; this is to make sure that we 
+         * end up jumping to the copy of the app code in the trampoline as 
+         * opposed to jumping to the hook itself!
+         */
+        for (ppoint_idx = 0; ppoint_idx < module->num_patch_points;
+             ppoint_idx++) {
+            ppoint = &module->patch_points[ppoint_idx];
+
+            /* If a ppoint hasn't been patched yet, don't do try to a resolve 
+             * control flow change conflict targeting it!  If a ppoint has 
+             * been patched, is cur_ppoint inside it?  If so, resolve conflict.
+             */
+            if (ppoint->trampoline != NULL &&
+                HOTP_ONLY_IS_IN_PATCH_REGION(ppoint, cur_ppoint->return_addr)) {
+
+                /* If ppoint has been injected, then its app_code_copy must
+                 * point to the copy of the app code that was overwritten by 
+                 * the hook.
+                 */
+                ASSERT(HOTP_ONLY_IS_IN_TRAMPOLINE(ppoint, ppoint->app_code_copy));
+
+                /* Without multiple patch points at the same offset, a control
+                 * flow change target can collide with only one patch region.
+                 */
+                ASSERT(!patched);
+
+                /* Control flow transfer is going to the middle of another
+                 * hot patch's patch region; one which has been injected.
+                 * So fix the cur_ppoint trampoline's exit cti to target the 
+                 * app code copy stored in the target hot patch's trampoline as
+                 * opposed to actual image.
+                 */
+                cflow_target = ppoint->app_code_copy +
+                               (cur_ppoint->return_addr - ppoint->offset);
+                ASSERT(HOTP_ONLY_IS_IN_TRAMPOLINE(ppoint, cflow_target));
+                patch_branch(cur_ppoint->tramp_exit_cti, cflow_target, false);
+                patched = true;
+
+#ifndef DEBUG
+                /* Cycle through all patches even if patched for debug builds;
+                 * it helps to catch multiple ppoints in the same offset.  In
+                 * release builds, this is an inefficiency, so just break.
+                 */
+                break;
+#endif
+                STATS_INC(hotp_only_cflow_collision);
+            }
+        }
+
+        /* Control flow change is to a point inside the module which isn't a
+         * patch point.
+         */
+        if (!patched) {
+            cflow_target = module->base_address + cur_ppoint->return_addr;
+            patch_branch(cur_ppoint->tramp_exit_cti, cflow_target, false);
+        }
+    }
+
+    /* Now, check in the current module, if any other injected patch point's 
+     * control-flow-change target is the current patch point's region; if so 
+     * make it jump to the app_code_copy in the trampoline buffer of the 
+     * current ppoint.
+     */
+    for (ppoint_idx = 0; ppoint_idx < module->num_patch_points; ppoint_idx++) {
+         ppoint = &module->patch_points[ppoint_idx];
+        /* No point in checking the current patch point with itself; of course
+         * there will be a collision.
+         */
+        if (ppoint != cur_ppoint) {
+            /* If ppoint hasn't been injected, nothing to do.  If it has been &
+             * its return_addr collides with cur_ppoint's patch region, 
+             * then resolve conflict, i.e., change control flow to the copy 
+             * of app code inside cur_ppoint's trampoline.
+             */
+            if (ppoint->trampoline != NULL &&
+                HOTP_ONLY_IS_IN_PATCH_REGION(cur_ppoint, ppoint->return_addr)) {
+
+                ASSERT(HOTP_ONLY_IS_IN_TRAMPOLINE(ppoint, ppoint->app_code_copy));
+                ASSERT(HOTP_ONLY_IS_IN_TRAMPOLINE(ppoint, ppoint->tramp_exit_cti));
+
+                ASSERT(*(ppoint->tramp_exit_cti) == JMP_REL32_OPCODE);
+                cflow_target = cur_ppoint->app_code_copy + 
+                               (ppoint->return_addr - cur_ppoint->offset);
+                ASSERT(HOTP_ONLY_IS_IN_TRAMPOLINE(cur_ppoint, cflow_target));
+                patch_branch(ppoint->tramp_exit_cti, cflow_target, false);
+                STATS_INC(hotp_only_cflow_collision);
+            }
+        }
+    }
+
+#ifdef WINDOWS
+    /* If any suspended app thread is in the middle of the current patch point
+     * then it needs to be relocated, i.e., its eip needs to be changed to point
+     * to the correct offset in the app_code_copy in the trampoline.
+     */
+    if (num_threads != HOTP_ONLY_NUM_THREADS_AT_INIT) {
+        int i;
+        bool res;
+        app_pc eip;
+        CONTEXT cxt;
+        thread_id_t my_tid = get_thread_id();
+
+        for (i = 0; i < num_threads; i++) {
+            /* Skip the current thread; nudge thread's Eip isn't relevant. */
+            if (my_tid == all_threads[i]->id)
+                continue;
+
+            /* App thread can't be in the core holding a lock when suspended. */
+            ASSERT(thread_owns_no_locks(all_threads[i]->dcontext));
+
+            cxt.ContextFlags = CONTEXT_FULL; /* PR 264138: don't need xmm regs */
+            res = thread_get_context((thread_record_t *)all_threads[i], &cxt);
+            ASSERT(res);
+            eip = (app_pc)cxt.CXT_XIP;
+
+            /* 3 conditions have to be met to relocate an app thread during
+             * hotp_only patching.
+             * 1. thread's eip should be greater than the module base of the
+             *      current ppoint; if not, negative offsets will result which
+             *      can cause wrap arounds in the HOTP_ONLY_IS_IN_PATCH_REGION
+             *      check which uses app_rva_t (size_t).
+             * 2. if eip is at the start of the patch region, don't relocate
+             *      it; just let it go to the trampoline.  Fixes a security 
+             *      issue: a live process which is blocked on a system call can 
+             *      be patched right after the syscall so that a vulnerability 
+             *      in the results can be caught;  if relocated, the first time,
+             *      the hotpatch won't execute, just the app code copy, thereby 
+             *      letting the attack slip.  Rare & theoritical (because we 
+             *      don't allow returns inside the ppoint & because it is hard 
+             *      the attack has to be timed to be after the patch but 
+             *      before it is executed) hole.
+             * 3. eip should be inside the patch region defined by cur_ppoint.
+             */
+            if (eip > module->base_address && eip != addr_to_hook &&
+                HOTP_ONLY_IS_IN_PATCH_REGION(cur_ppoint,
+                    (app_rva_t)(eip - module->base_address))) {
+                /* FIXME: this is one place that may need work if we mangle
+                 *       cti_short in the patch region;  see case 6839.
+                 */
+                cxt.CXT_XIP = (ptr_uint_t)(cur_ppoint->app_code_copy +
+                                           (eip - (module->base_address +
+                                                   cur_ppoint->offset)));
+                res = thread_set_context((thread_record_t *)all_threads[i], &cxt);
+                ASSERT(res);
+            }
+        }
+    }
+#endif  /* WINDOWS */
+
+    STATS_INC(hotp_only_num_inject);
+
+    if (vul->mode == HOTP_MODE_DETECT)
+        hotp_set_policy_status(ppoint_desc->vul_index, HOTP_INJECT_DETECT);
+    else
+        hotp_set_policy_status(ppoint_desc->vul_index, HOTP_INJECT_PROTECT);
+}
+
+/* Does mp safe removal of one hotp_only patch.  At the point of suspension, 
+ * each thread shouldn't be in all of the following: dr, hotp_dll and dr_stack. 
+ */
+static void
+hotp_only_remove_patch(const hotp_module_t *module, 
+                       hotp_patch_point_t *cur_ppoint)
+{
+    bool res;
+    app_pc addr_to_unhook;
+    DEBUG_DECLARE(char ppoint_content[HOTP_ONLY_PATCH_REGION_SIZE]);
+
+    ASSERT(DYNAMO_OPTION(hotp_only));
+
+    /* Are we at a mp-safe spot to remove the patches? */
+    ASSERT_OWN_MUTEX(true, &all_threads_synch_lock);
+    ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
+    ASSERT_OWN_READWRITE_LOCK(true, &hotp_vul_table_lock);
+
+    addr_to_unhook = hotp_ppoint_addr(module, cur_ppoint);
+
+    /* Is there a hook at this place? */
+    ASSERT(*addr_to_unhook == JMP_REL32_OPCODE);
+
+    /* Is there a valid trampoline? */
+    ASSERT(cur_ppoint->trampoline != NULL);
+    ASSERT(cur_ppoint->app_code_copy != NULL);
+
+    /* Save the 5 original app code bytes by getting it from the trampoline 
+     * (today we store those at the start of the trampoline).  Check that 
+     * those bytes match after unhooking.
+     */
+    ASSERT(HOTP_PATCH_REGION_SIZE == HOTP_ONLY_PATCH_REGION_SIZE);
+    DODEBUG({
+        memcpy(ppoint_content, cur_ppoint->trampoline, 
+               HOTP_PATCH_REGION_SIZE);
+    });
+    unhook_text(cur_ppoint->trampoline, addr_to_unhook);
+    ASSERT(!memcmp(ppoint_content, addr_to_unhook, HOTP_PATCH_REGION_SIZE));
+    /* Don't release the trampoline, just leak it, i.e., don't call
+     * special_heap_free.  This is how we handle the interop and detach
+     * problems created by hotp & 3rd-party hooks colliding.  Not elegant or
+     * memory efficient, but will handle the cases of the 3rd party reading our
+     * hook before hooking and/or leaving the page marked rwx.   See cases
+     * 9906, 9588, 9593, 9148 & 9157.
+     * FIXME: have a better mechanism to resolve hook conflict issues; currently
+     *          only a minimalist solution is in place; case 7657, case 10433.
+     * FIXME: make leaking selective, i.e., don't leak all trampoline, leak
+     *          only the ones that collide with 3rd party hooks - need to do
+     *          some bookkeepping; not a big issue, but about 20k to 50k can be
+     *          lost for each process otherwise, case 10433.
+     */
+    /* The ifdef mess in the next few lines is to handle the leak for case 9593. */
+#ifdef DEBUG
+# ifdef HEAP_ACCOUNTING
+    hotp_only_tramp_bytes_leaked += HOTP_ONLY_TRAMPOLINE_SIZE;
+# endif
+#endif
+    /* Tramp heap is freed before memory leak is checked, so cache the value.
+     * hotp_only_tramp_heap_cache also tracks if there was patch removal.  */
+    if (hotp_only_tramp_heap_cache == NULL) {
+        SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+        hotp_only_tramp_heap_cache = hotp_only_tramp_heap;  
+        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+    }
+
+    /* Note: as we aren't freeing the trampoline, we have to nop it else we can
+     * have bad consequences like a conflicting 3rd party hook jumping to our 
+     * trampoline after we detach!  See case 9593. 
+     * This is done by bypassing the whole trampoline and jumping to the part 
+     * that executes the original app code and returns to the address after 
+     * the hook point.  */
+    insert_jmp_at_tramp_entry(cur_ppoint->trampoline, cur_ppoint->app_code_copy);
+
+    /* Note, all thread synch locks & hot patch locks must be held before 
+     * removing anything from the vector.
+     */
+    res = vmvector_remove(hotp_only_tramp_areas, cur_ppoint->trampoline, 
+                    cur_ppoint->trampoline + HOTP_ONLY_TRAMPOLINE_SIZE);
+    ASSERT(res);
+
+    /* Today for hotp_only all patches in a module are applied and removed in 
+     * one shot, and control flow change doesn't go across modules, so there 
+     * is no need to patch no tramp_exit_cti (to make sure that control flow 
+     * change requested is not affected) as a result of patch removal 
+     * (remember that all threads are suspended at outside of any hot patches 
+     * during the patch removal process).  If in future we allow control flow 
+     * change to go across modules, then we will need to go through all 
+     * modules & their patch points to fix the tramp_exit_cti. 
+     */
+
+    cur_ppoint->trampoline = NULL;
+    cur_ppoint->tramp_exit_cti = NULL;
+    cur_ppoint->app_code_copy = NULL;
+}
+
+/* Returns true if the eip is inside any hotp_only trampoline. */
+bool 
+hotp_only_in_tramp(const app_pc pc)
+{
+    /* Only after successfully stopping all threads will hotp_only_tramp_areas
+     * vector will be written to.  This means that during synching when each 
+     * thread is suspended, where this function is called, there should be no 
+     * one updating the hotp_only_tramp_areas vector.
+     */
+    if (DYNAMO_OPTION(hotp_only)) {
+        ASSERT(!WRITE_LOCK_HELD(&hotp_only_tramp_areas->lock));
+        return vmvector_overlap(hotp_only_tramp_areas, pc, pc + 1);
+    }
+    else
+        return false;   /* check is moot if there is no trampoline */
+}
+
+/* This routine is used to remove hotp_only patches on a detach.  
+ * Note: Though hotp_exit gets called by detach, the removal of patches can't 
+ * be done there because the synch locks and thread data structures won't be 
+ * available at that point.  Hence the patch removal has to be done earlier 
+ * inside detach. 
+ */
+void 
+hotp_only_detach_helper(void)
+{
+    /* Can't be removing hotp_only patches when hotp_only mode isn't on.  Though
+     * we assert, it is safe to doing nothing in release builds and just return.
+     */
+    ASSERT(DYNAMO_OPTION(hotp_only));
+    if (!DYNAMO_OPTION(hotp_only))
+        return;
+
+    /* Thread synch locks must be held before removing. */
+    ASSERT_OWN_MUTEX(true, &all_threads_synch_lock);
+    ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
+
+    write_lock(&hotp_vul_table_lock);
+    hotp_remove_hot_patches(GLOBAL_VUL_TABLE, NUM_GLOBAL_VULS, true, NULL);
+    write_unlock(&hotp_vul_table_lock);
+}
+
+/* This function is used to handle loader safe injection for hotp_only mode.  
+ * This is done by removing patches in regions the loader wants to write to 
+ * and reinjecting them afterwards; done by monitoring memory protection 
+ * changes made.  
+ *
+ * FIXME case 9148: this causes problems with hookers who read prior to marking
+ * +w and thus chain with our old trampoline that we are about to remove here!
+ * That's why we don't call here for +rwx changes, where we live with a leak on
+ * the +rx change (case 9148), which is better than hookers who mark +rw and can
+ * end up with infinite recursions or wrong API calls.  We need a better
+ * approach to handling both loader and hooker interop (case 7657).
+ *
+ * Note: all patches in a module come out, not just the page in question because
+ * if a non-loader-agent changes the image and messes up our hash checks, then
+ * we wouldn't be able to reinsert any into that page, leaving a multiple ppoint
+ * policy in an inconsistent state or a ppoint in protect mode unprotected. 
+ * Another reason for pulling out all is module atomicity; set atomicity will
+ * involve removing patches from other modules too!
+ * 
+ * Note: we don't handle with some one trying to change memory protection 
+ * across two modules in with a single syscall; don't think it is allowed.
+ */
+void
+hotp_only_mem_prot_change(const app_pc start, const size_t size, 
+                          const bool remove, const bool inject)
+{
+    app_pc base;
+    bool needs_processing = false;
+    int num_threads = 0;
+    thread_record_t **all_threads = NULL;
+#ifdef WINDOWS
+    DEBUG_DECLARE(bool ok;)
+#endif
+
+    /* For hotp_only, for regular mode, vmarea tracking will flush the
+     * necessary fragments.
+     */
+    ASSERT(DYNAMO_OPTION(hotp_only));
+    ASSERT(start != NULL && size > 0);
+    ASSERT((remove == true || remove == false) &&
+           (inject == true || inject == false));
+
+    ASSERT(remove != inject);    /* One and only one must be true. */
+    if (remove == inject)       /* Defensively just ignore. */
+        return;
+
+    base = get_module_base(start);
+
+    /* If base doesn't belong to any module; ignore.  We don't hot patch DGC. */
+    if (base == NULL)
+        return;
+    /* The end of the region better be in the image! */
+    ASSERT(base == get_module_base(base + size));
+
+#ifdef WINDOWS
+    DODEBUG({
+        if (get_loader_lock_owner() != get_thread_id()) {
+            LOG(GLOBAL, LOG_HOT_PATCHING, 1, "Warning: Loader lock not held "
+                "during image memory protection change; possible incompatible "
+                "hooker or w2k3 loader.");
+        }
+    });
+#endif
+
+    /* Inefficient check to see if this module has been matched for hot
+     * patching.  hotp_process_image() is needed only when loading or 
+     * unloading a dll, not here, which is post module loading.
+     * FIXME: Use vmvector_overlap check on loaded_module_areas after 
+     * integrating it with hotp.  Optimization.
+     */
+    hotp_process_image(base, inject ? true : false, false, true, 
+                       &needs_processing, NULL, 0);
+    if (!needs_processing) {     /* Ignore if it isn't a vulnerable module. */
+        LOG(THREAD_GET, LOG_HOT_PATCHING, 2,
+            "hotp_only_mem_prot_change: no work to be done for base "PFX"\n", base);
+        return;
+    }
+
+#ifdef WINDOWS
+    /* Ok, let's suspend all threads and do the injection/removal. */
+    DEBUG_DECLARE(ok =)
+        synch_with_all_threads(THREAD_SYNCH_SUSPENDED, &all_threads, &num_threads,
+                               /* Case 6821: other synch-all-thread uses that
+                                * only care about threads carrying fcache
+                                * state can ignore us
+                                */
+                               THREAD_SYNCH_NO_LOCKS_NO_XFER,
+                               /* if we fail to suspend a thread (e.g., privilege
+                                * problems) ignore it. FIXME: retry instead? */
+                               THREAD_SYNCH_SUSPEND_FAILURE_IGNORE);
+    ASSERT(ok);
+#endif
+    write_lock(&hotp_vul_table_lock);
+
+    /* Using hotp_process_image to inject is inefficient because it goes 
+     * through the whole vul table.
+     * FIXME: Optimization: write hotp_only_inject_patches() which should use
+     *      hotp_patch_point_areas; use that to do the injection here.
+     */
+    if (inject) {
+        LOG(THREAD_GET, LOG_HOT_PATCHING, 1,
+            "hotp_only_mem_prot_change: injecting for base "PFX"\n", base);
+        DODEBUG(hotp_globals->ldr_safe_hook_injection = true;); /* Case 7998. */
+        DODEBUG(hotp_globals->ldr_safe_hook_removal = false;);  /* Case 7832. */
+        hotp_process_image_helper(base, true, true, false, NULL,
+                                  (const thread_record_t **)all_threads,
+                                  num_threads, true, NULL);
+        DODEBUG(hotp_globals->ldr_safe_hook_injection = false;);
+    /* Similarly, hotp_remove_patches_from_module() is inefficient too.  
+     * FIXME: using loaded_module_areas in that routine.
+     */
+    } else if (remove) {
+        LOG(THREAD_GET, LOG_HOT_PATCHING, 1,
+            "hotp_only_mem_prot_change: removing for base "PFX"\n", base);
+        hotp_remove_patches_from_module(GLOBAL_VUL_TABLE, NUM_GLOBAL_VULS,
+                                        true, base, NULL);
+        /* Used to detect double removal while handling loader-safety. */
+        DODEBUG(hotp_globals->ldr_safe_hook_removal = true;);   /* Case 7832. */
+    }
+    write_unlock(&hotp_vul_table_lock);
+#ifdef WINDOWS
+    end_synch_with_all_threads(all_threads, num_threads, true/*resume*/);
+#endif
+}
+
+/* This is the routine that will serve as the entry point into the core for
+ * executing hot patches in the -hotp_only mode.
+ * FIXME: for now, dr stack is used to execute the hot patch; later on a 
+ *        separate stack should be used.
+ */
+after_intercept_action_t 
+hotp_only_gateway(app_state_at_intercept_t *state)
+{
+    hotp_offset_match_t match;
+    hotp_context_t cxt;
+    app_pc hook_addr = (app_pc) state->callee_arg;
+    after_intercept_action_t res = AFTER_INTERCEPT_LET_GO;
+
+    read_lock(&hotp_vul_table_lock);
+    ASSERT(DYNAMO_OPTION(hotp_only));
+
+    /* Callee_arg contains the application eip to be patched.  It better be
+     * inside a code region.
+     */
+    ASSERT(is_in_code_section(get_module_base(hook_addr), hook_addr, NULL, NULL));
+
+    /* Note: for -hotp_only vulnerability table access during hot patch 
+     * execution is indirect, i.e., we do a lookup.  For hot patches in the
+     * code cache, this information is embedded in the injected code.
+     */
+    if (hotp_only_lookup_patch_addr(hook_addr, &match)) {
+        cxt = state->mc;
+        res = hotp_gateway(GLOBAL_VUL_TABLE, NUM_GLOBAL_VULS, match.vul_index,
+                           match.set_index, match.module_index,
+                           match.ppoint_index, &cxt, true /* have lock */);
+        /* The hot patch could have modified app state as part of the fix, so
+         * copy it back.
+         */
+        state->mc = cxt;
+    } else {
+        /* If we reached here, there was a hot patch that was injected that no
+         * longer matches, i.e., there is no matching definition.  Could be
+         * because the mode was changed, the module got unloaded or new defs
+         * came in, etc.  With mp-safe hotp_only patch injection, vulnerability/
+         * policy data changes are preceeded by removal of all injected patches;
+         * and patch removal guarantees that no patch will be executing.
+         * This means that offset lookup should always succeed in this routine.
+         */
+        ASSERT_NOT_REACHED();
+    }
+    read_unlock(&hotp_vul_table_lock);
+    return res;
+}
+
+
+/* TODO: for multiple patch points, need to pass the number of patch points;
+ *       preferably as the first argument.
+ * TODO: this routine calls dr routines, i.e., switches to dr from the fcache.
+ *        the switching involves protections changes (ENTER_DR_HOOK); however, 
+ *        the clean call mechanism used to reach here doesn't call the hook!
+ *       also, there are assumptions in dr about locks being held across fcache
+ *        - Derek raised these issues as some that came up during client 
+ *          interface design; he also raised some interesing points about
+ *          generating control flow change code rather than doing it in C.
+ *        - Derek also mentioned if the gateway was called from C code within
+ *          dr, then things should be fine; in other words, bail out of the
+ *          code cache for bb that need hot patching and execute the gateway
+ *          from within dr - this is the model we will be switching to in 
+ *          the immediate future.
+ */
+static after_intercept_action_t 
+hotp_gateway(const hotp_vul_t *vul_tab, const uint num_vuls,
+             const uint vul_index, const uint set_index, 
+             const uint module_index, const uint ppoint_index,
+             hotp_context_t *app_reg_ptr, const bool own_hot_patch_lock)
+{
+    /* FIXME: racy access here; getting lock may be expensive;  even if that is
+     *        ok, must make sure that no one will come in here and wait on the
+     *        lock while a hot patch flush happens due to a nudge - deadlock.
+     *        also, before executing each hot patch, it must be verfied
+     *        that it still is valid because it could have been changed by a 
+     *        nudge;  see case 5052.
+     *        looks like the whole hot patch execution should be covered by a
+     *        lock - same hotp_vul_table lock or a new one?  new one I think.
+     *      Derek: let flush worry about invalidation; just grab locks for 
+     *        table lookup or stats update.
+     *      see case 5521.
+     */
+    hotp_policy_mode_t mode;
+    hotp_type_t hotp_type;
+    app_rva_t detector_offset, protector_offset;
+    hotp_func_t detector_fn, protector_fn = NULL;
+    hotp_exec_status_t exec_status, temp;
+    hotp_offset_match_t ppoint = {vul_index, set_index, module_index, ppoint_index};
+    bool dump_excpt_info, dump_error_info;
+    after_intercept_action_t res = AFTER_INTERCEPT_LET_GO;
+
+    /* FIXME: till hotp interface is expanded to send arguments to detectors
+     *  and protectors, this spill is the simplest way to send/receive args.
+     *  xref case 6804.
+     */
+    reg_t gbop_eax_spill = 0, gbop_edx_spill = 0;
+    app_pc gbop_bad_addr = NULL;
+    app_pc ppoint_addr;     /* Fix for case 6054.  Exposed for gbop. */
+
+    DODEBUG({
+        dcontext_t *dcontext = get_thread_private_dcontext();
+        ASSERT_CURIOSITY(dcontext != NULL && dcontext != GLOBAL_DCONTEXT 
+                         && "unknown thread");
+
+        /* App esp should neither be on dr stack nor on initstack; see case 7058.
+         * TODO: when the hot patch interface expands to having eip, assert that
+         *       the eip isn't inside dr.
+         */
+        ASSERT(!is_on_dstack(dcontext, (byte *)app_reg_ptr->xsp) &&
+               !is_on_initstack((byte *)app_reg_ptr->xsp));
+    });
+    ppoint_addr = hotp_ppoint_addr(
+                    &MODULE(vul_tab, vul_index, set_index, module_index), 
+                    &PPOINT(vul_tab, vul_index, set_index, module_index,
+                            ppoint_index));
+
+    /* If we change this to be invoked from dispatch, should remove this.
+     * Note that we assume that hotp_only, which is invoked from
+     * interception code that has its own enter hook embedded, will
+     * not call any of these hooks -- else we do a double-enter here and
+     * the exit via hotp_change_control_flow() results in unprotected .data!
+     */
+    ENTERING_DR();
+
+    if (!own_hot_patch_lock)
+        /* Note: for regular hot patches (!hotp_only) vulnerability table
+         * access during execution isn't via a lookup and all the old tables
+         * are alive, so we don't need to grab the lock here; if we do an 
+         * indirect access then we need it.  It is left in there for safety.
+         */
+        read_lock(&hotp_vul_table_lock);        /* Part of fix for case 5521. */
+    else
+        ASSERT_OWN_READ_LOCK(true, &hotp_vul_table_lock);
+
+    /* Check the validity of the input indices before using them.  The injection
+     * routine should be generating code to send the right values.  These
+     * asserts will trigger if either the injected code is messed up or a nudge
+     * resulted in a vulnerability change that didn't have a corresponding
+     * flush of injected bbs/traces.  One other possibility is that while in
+     * this function the vulnerability table changed due to a nudge - this can't
+     * happen because a nudge would result in a flush, which would wait for all
+     * threads to come out of the cache, thus out of this function before
+     * modifying the table.
+     */
+    ASSERT(vul_index >= 0 && vul_index < num_vuls);
+    ASSERT(set_index >= 0 && set_index < VUL(vul_tab, vul_index).num_sets);
+    ASSERT(module_index >= 0 &&
+           module_index < SET(vul_tab, vul_index, set_index).num_modules);
+    ASSERT(ppoint_index >= 0 &&
+           ppoint_index < MODULE(vul_tab, vul_index, set_index,
+                                 module_index).num_patch_points);
+
+    mode = VUL(vul_tab, vul_index).mode;  /* Racy; see assert comments above. */
+    hotp_type = VUL(vul_tab, vul_index).type;
+
+    /* For hotp_only mode control can't reach here if the mode is off because
+     * in order to change modes all patches are removed first.  However, for 
+     * regular hot patching, patch removal (flushing) is done after mode change
+     * and outside the scope of the hotp_vul_table_lock, so control can be in
+     * the gateway with the mode set to off, but only for one execution per 
+     * thread because the fragment has been unlinked by the flush and 
+     * scheduled for deletion, so there is no entry to it.
+     */
+    DODEBUG({
+        if (mode == HOTP_MODE_OFF) {
+            ASSERT(!DYNAMO_OPTION(hotp_only));
+            STATS_INC(hotp_exec_mode_off);
+        } else {
+            ASSERT(mode == HOTP_MODE_DETECT || mode == HOTP_MODE_PROTECT);
+        }
+    });
+
+    /* The hot patch dll specified by the vulnerability better be loaded by
+     * this point.  Unloaded hot patch dlls will result in the vulnerability
+     * being deactivated, so we should never reach this point for such 
+     * vulnerabilities.
+     */
+    ASSERT(VUL(vul_tab, vul_index).hotp_dll_base != NULL);
+
+    detector_offset = PPOINT(vul_tab, vul_index, set_index, module_index, 
+                             ppoint_index).detector_fn;
+    /* TODO: make the assertion range tigher by using the actual size of the
+     *       text section of the hot patch dll.
+     */
+    ASSERT((detector_offset >= MIN_DETECTOR_OFFSET &&
+            detector_offset <= MAX_DETECTOR_OFFSET) ||
+           TESTALL(HOTP_TYPE_PROBE, hotp_type));    /* no detector for probes */
+
+    protector_offset = PPOINT(vul_tab, vul_index, set_index, module_index, 
+                              ppoint_index).protector_fn;
+    ASSERT(protector_offset >= MIN_PROTECTOR_OFFSET &&
+           protector_offset <= MAX_PROTECTOR_OFFSET);
+
+    /* Compute the hot patch function addresses with the hot patch dll base. */
+    detector_fn = (hotp_func_t)((ptr_uint_t)VUL(vul_tab, vul_index).hotp_dll_base + 
+                                detector_offset);
+
+    protector_fn = (hotp_func_t)((ptr_uint_t)VUL(vul_tab, vul_index).hotp_dll_base +
+                                 protector_offset);
+    ASSERT(detector_fn != protector_fn);    /* can't be the same code! */
+
+    LOG(GLOBAL, LOG_HOT_PATCHING, 2, "Invoking detector for vulnerability %s\n",
+        VUL(vul_tab, vul_index).vul_id);
+    LOG(GLOBAL, LOG_HOT_PATCHING, 4, "Register state sent to detector\n");
+    DOLOG(4, LOG_HOT_PATCHING, {
+        hotp_dump_reg_state(app_reg_ptr, ppoint_addr, 4); 
+    });
+
+    /* gbop hooks need to know current pc & will return the bad return address 
+     * if it is faulty.  As hotp_context_t doesn't have eip as of now, we pass
+     * it via edx.  eax is used to get the return value.
+     * TODO: make this a function & move it to the gbop section
+     * FIXME PR 226036: hotp_context_t does have eip now, use it!
+     */
+    if (TESTALL(HOTP_TYPE_GBOP_HOOK, hotp_type)) {
+#ifdef GBOP
+        ASSERT(DYNAMO_OPTION(gbop) && DYNAMO_OPTION(hotp_only));
+#endif
+        gbop_eax_spill = APP_XAX(app_reg_ptr);
+        gbop_edx_spill = APP_XDX(app_reg_ptr);
+        APP_XDX(app_reg_ptr) = (reg_t) ppoint_addr;
+    } else {
+        /* A hot patch can be only one type. */
+        ASSERT(TEST(HOTP_TYPE_HOT_PATCH, hotp_type) ^
+               TEST(HOTP_TYPE_PROBE, hotp_type));
+    }
+    /* Under the current design, a detector will always be called; a protector
+     * will be called only if the mode says so.
+     * 
+     * Forensics dumped for hot patch exceptions and errors are done so once
+     * for each vulnerability; otherwise we could flood the machine.  Events 
+     * are logged every time; for errors, this is predicated by the patch 
+     * returning HOTP_EXEC_LOG_EVENT.  Cores are dumped only if the 
+     * dumpcore mask is set;  for exceptions, it is done every time and
+     * for errors it is done once, if the mask is set (because for exceptions
+     * it hard to convey the "once only" information to the exception handler).
+     * We use num_{aborted,detector_error,protector_error} as booleans to 
+     * control this.
+     * TODO: area to revisit when we work on information throttling.
+     */
+    dump_excpt_info = (VUL(vul_tab, vul_index).info->num_aborted == 0);
+    dump_error_info = (VUL(vul_tab, vul_index).info->num_detector_error == 0);
+
+    if (TESTALL(HOTP_TYPE_PROBE, hotp_type)) {
+        /* No detectors for probes.  This status means execute the protector. */
+        exec_status = HOTP_EXEC_EXPLOIT_DETECTED;
+    } else {
+        /* A hot patch can be only one type. */
+        ASSERT(TEST(HOTP_TYPE_HOT_PATCH, hotp_type) ^
+               TEST(HOTP_TYPE_GBOP_HOOK, hotp_type));
+
+        exec_status = hotp_execute_patch(detector_fn, app_reg_ptr, mode,
+                                         dump_excpt_info, dump_error_info);
+
+        LOG(GLOBAL, LOG_HOT_PATCHING, 3, "Detector finished for vulnerability %s\n",
+            VUL(vul_tab, vul_index).vul_id);
+        STATS_INC(hotp_num_det_exec);
+        hotp_update_vul_stats(exec_status, vul_index);
+    }
+    temp = exec_status & ~HOTP_EXEC_LOG_EVENT;
+    ASSERT(temp == HOTP_EXEC_EXPLOIT_DETECTED ||
+           temp == HOTP_EXEC_EXPLOIT_NOT_DETECTED ||
+           temp == HOTP_EXEC_DETECTOR_ERROR ||
+           temp == HOTP_EXEC_ABORTED);
+
+    /* Restore eax & edx spilled for executing gbop remediators.
+     * TODO: make this a function & move it to the gbop section
+     */
+    if (TESTALL(HOTP_TYPE_GBOP_HOOK, hotp_type)) {
+#ifdef GBOP
+        ASSERT(DYNAMO_OPTION(gbop) && DYNAMO_OPTION(hotp_only));
+#endif
+        if (temp == HOTP_EXEC_EXPLOIT_DETECTED)
+            gbop_bad_addr = (app_pc) APP_XAX(app_reg_ptr);
+        APP_XAX(app_reg_ptr) = gbop_eax_spill;
+        APP_XDX(app_reg_ptr) = gbop_edx_spill;
+    }
+
+    if (temp == HOTP_EXEC_ABORTED || temp == HOTP_EXEC_DETECTOR_ERROR)
+        goto hotp_gateway_ret;
+
+    /* If the patch asked for violation notification, do so only if its mode 
+     * is set to detect.  For protect mode, the protector will report the 
+     * violation if asked, so don't worry about it.  The exception here is for 
+     * gbop hooks, which currently always run in protect mode, which need to 
+     * honor -detect_mode, in which case we report the violation right here.  
+     * Note: In this case, the gbop protector won't get executed even though 
+     * its mode is set to protect.  See below.
+     */
+    if (TEST(exec_status, HOTP_EXEC_LOG_EVENT) &&
+        ((mode == HOTP_MODE_DETECT) ||
+         (TESTALL(HOTP_TYPE_GBOP_HOOK, hotp_type)
+#ifdef PROGRAM_SHEPHERDING
+          && DYNAMO_OPTION(detect_mode)
+#endif
+         ))) {
+        hotp_event_notify(exec_status, false, &ppoint, gbop_bad_addr,
+                          app_reg_ptr);
+    }
+
+    /* The protector should be invoked only if an exploit was detected and the
+     * mode was set to protect.
+     * In the case of gbop hooks, -detect_mode shouldn't invoke the protector. 
+     * Note: Unlike hot patches, gbop hooks must conform to core reporting and
+     * remediation options.  As of today hot patch actions are specified by the
+     * patch {writer}.  There are plans to have an override, case 8095.
+     */
+    if (TESTALL(HOTP_TYPE_GBOP_HOOK, hotp_type)
+#ifdef PROGRAM_SHEPHERDING
+        && DYNAMO_OPTION(detect_mode)
+#endif
+       )
+        goto hotp_gateway_ret;
+
+    if (mode == HOTP_MODE_PROTECT &&
+        temp == HOTP_EXEC_EXPLOIT_DETECTED) {
+
+        LOG(GLOBAL, LOG_HOT_PATCHING, 2, "Invoking protector for vulnerability %s\n",
+            VUL(vul_tab, vul_index).vul_id);
+        LOG(GLOBAL, LOG_HOT_PATCHING, 6, "Register state sent to protector\n");
+        DOLOG(6, LOG_HOT_PATCHING, { 
+            hotp_dump_reg_state(app_reg_ptr, ppoint_addr, 6); 
+        });
+
+        /* See detector execution comments above for details about hot patch
+         * error & exception handling.
+         * TODO: area to revisit when we work on information throttling.
+         */
+        dump_error_info = (VUL(vul_tab, vul_index).info->num_protector_error == 0);
+        exec_status = hotp_execute_patch(protector_fn, app_reg_ptr, mode,
+                                         dump_excpt_info, dump_error_info);
+
+        /* TODO: probes have no return codes defined. PR 229881. */
+        if (TESTALL(HOTP_TYPE_PROBE, hotp_type)) {
+            exec_status = HOTP_EXEC_EXPLOIT_PROTECTED;
+        }
+
+        temp = exec_status & ~HOTP_EXEC_LOG_EVENT;
+        ASSERT(temp == HOTP_EXEC_EXPLOIT_PROTECTED ||
+               temp == HOTP_EXEC_EXPLOIT_NOT_PROTECTED ||
+               temp == HOTP_EXEC_EXPLOIT_KILL_THREAD ||
+               temp == HOTP_EXEC_EXPLOIT_KILL_PROCESS ||
+               temp == HOTP_EXEC_EXPLOIT_RAISE_EXCEPTION ||
+               temp == HOTP_EXEC_CHANGE_CONTROL_FLOW ||
+               temp == HOTP_EXEC_PROTECTOR_ERROR ||
+               temp == HOTP_EXEC_ABORTED);
+
+        LOG(GLOBAL, LOG_HOT_PATCHING, 4, "Register state after protector\n");
+        DOLOG(4, LOG_HOT_PATCHING, { 
+            hotp_dump_reg_state(app_reg_ptr, ppoint_addr, 4);
+        });
+        LOG(GLOBAL, LOG_HOT_PATCHING, 3, "Protector finished for vulnerability %s\n",
+            VUL(vul_tab, vul_index).vul_id);
+        STATS_INC(hotp_num_prot_exec);
+        hotp_update_vul_stats(exec_status, vul_index);
+
+        if (temp == HOTP_EXEC_ABORTED || temp == HOTP_EXEC_PROTECTOR_ERROR)
+            goto hotp_gateway_ret;
+
+        /* Which one comes first, esp with kill/raise & cflow change? */
+        /* Raise an event only if requested by the protector. */
+        if (TEST(exec_status, HOTP_EXEC_LOG_EVENT))
+            hotp_event_notify(exec_status, true, &ppoint, gbop_bad_addr, 
+                              app_reg_ptr);
+
+        if (TEST(exec_status, HOTP_EXEC_CHANGE_CONTROL_FLOW)) {
+
+            app_rva_t return_rva = PPOINT(vul_tab, vul_index, set_index,
+                                        module_index, ppoint_index).return_addr;
+            app_pc module_base = MODULE(vul_tab, vul_index, set_index, 
+                                        module_index).base_address;
+
+            /* gbop hooks shouldn't be changing control flow. */
+            ASSERT(!TESTALL(HOTP_TYPE_GBOP_HOOK, hotp_type));
+
+            /* hotp_only control flow change is implemented using the alt exit
+             * feature in our intercept_call() mechanism.
+             */
+            if (DYNAMO_OPTION(hotp_only)) {
+                res = AFTER_INTERCEPT_LET_GO_ALT_DYN;
+                goto hotp_gateway_ret;
+            }
+
+            /* If control flow change is requested by a protector, then the
+             * offset to which the control should go to shouldn't be zero
+             * and the dll should be in memory!
+             */
+            ASSERT(return_rva != 0 && module_base != NULL); 
+            LOG(GLOBAL, LOG_HOT_PATCHING, 1, "Control flow change requested by "
+                "vulnerability %s\n", VUL(vul_tab, vul_index).vul_id);
+
+            /* Release the lock because control flow change won't return. */
+            read_unlock(&hotp_vul_table_lock);  /* Part of fix for case 5521. */
+            hotp_change_control_flow(app_reg_ptr, module_base + return_rva);
+            ASSERT_NOT_REACHED();
+        }
+    }
+
+ hotp_gateway_ret:
+    if (!own_hot_patch_lock)
+        read_unlock(&hotp_vul_table_lock);  /* Part of fix for case 5521. */
+
+    /* if we change this to be invoked from dispatch, should remove this */
+    EXITING_DR();
+    return res;
+}
+
+/* This routine will execute the given hot patch (either detector or protector)
+ * and return the appropriate execution status.  If the hot patch causes an
+ * exception, it will be terminated and the exception handler will automatically
+ * return to this function and this function will return with status 
+ * HOTP_EXEC_ABORTED.
+ * If a hot patch exception occurs
+ *     - it dumps a forenscis file if asked for (using dump_excpt_info)
+ *     - the exception handler dumps a core if the mask is set and logs an event
+ * If a hot patch returns HOTP_{DETECTOR,PROTECTOR}_ERROR,
+ *     - it dumps a forenscis file if asked for (using dump_error_info)
+ *     - it dumps a core if asked for and the mask is set
+ *     - it logs an event if the patch returns HOTP_EXEC_LOG_EVENT too.
+ * Hot patch exceptions and errors are treated similarly because they both 
+ * point to a faulty hot patch.  The only differences are in the string of the
+ * event logged and the cause-string of the forensics file.
+ *
+ * Note: this routine uses a shadow app reg state to recover from a hot patch
+ *       exception cleanly.
+ * FIXME: using setjmp & longjmp can cause problems if the compiler decides
+ *        to reuse unused args/locals; should probably use volatile for those.
+ */
+static hotp_exec_status_t
+hotp_execute_patch(hotp_func_t hotp_fn_ptr, hotp_context_t *hotp_cxt, 
+                   hotp_policy_mode_t mode, bool dump_excpt_info,
+                   bool dump_error_info)
+{
+    hotp_exec_status_t exec_status, exec_status_only;
+    hotp_context_t local_cxt;
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    where_am_i_t wherewasi;
+
+    ASSERT(dcontext != NULL && dcontext != GLOBAL_DCONTEXT);
+    if (dcontext == NULL ||     /* case 9385: unknown thread */
+        dcontext == GLOBAL_DCONTEXT /* just bug */) {
+        SYSLOG_INTERNAL_WARNING("hotp_execute_patch: unknown thread");
+        return HOTP_EXEC_ABORTED;
+    }
+
+    /* For hot patching with fcache, today, hot patches are executed only from 
+     * within the fcache.  For hotp_only, there is no fcache; hot patches are
+     * executed directly when they are WHERE_APP.
+
+Question for reviewer: for hotp_only, when control comes to the gateway, should
+    whereami be changed to something other than WHERE_APP because we are technically
+    in the core now; if so, would it be WHERE_TRAMPOLINE?
+     */
+    ASSERT(dcontext->whereami == WHERE_FCACHE ||
+           (dcontext->whereami == WHERE_APP && DYNAMO_OPTION(hotp_only)));
+    wherewasi = dcontext->whereami;
+
+    /* In case the hot patch causes an exception, the context may be in an 
+     * inconsistent state.  To prevent that make a copy of the app's context 
+     * and pass the copy to the hot patch.
+     * Note: nothing is done for partial memory writes.  TODO: how to fix this?
+     */
+    memcpy(&local_cxt, hotp_cxt, sizeof(hotp_context_t));
+    dcontext->whereami = WHERE_HOTPATCH;
+
+    if (DR_SETJMP(&dcontext->hotp_excpt_state) == 0) {  /* TRY block */
+        exec_status = (*hotp_fn_ptr)(&local_cxt);
+        exec_status_only =  exec_status & ~HOTP_EXEC_LOG_EVENT;
+
+        /* Successful execution can't return exception code. */
+        ASSERT(exec_status_only != HOTP_EXEC_ABORTED);
+
+        if (mode == HOTP_MODE_DETECT) {
+            /* The detector shouldn't have modified register state.
+             * Note: currently there is no way to find out if the memory state
+             *       was modified.
+             */
+            ASSERT(memcmp(hotp_cxt, &local_cxt, sizeof(hotp_context_t)) == 0);
+        } else if (mode == HOTP_MODE_PROTECT) {
+            /* Copy back local context which may have been modified by the 
+             * protector back to the context passed in, i.e., apply the 
+             * changes enforced by the hot patch.  Note: this is applicable 
+             * only for registers not memory
+             */
+            memcpy(hotp_cxt, &local_cxt, sizeof(hotp_context_t));
+        }
+
+        if ((TEST(HOTP_EXEC_DETECTOR_ERROR, exec_status_only) ||
+            (TEST(HOTP_EXEC_PROTECTOR_ERROR, exec_status_only)))) {
+            char *msg = TEST(HOTP_EXEC_DETECTOR_ERROR, exec_status_only) ?
+                "Hot patch detector error" : 
+                "Hot patch protector error";
+            if (dump_error_info) {
+                if (TEST(DUMPCORE_HOTP_FAILURE, DYNAMO_OPTION(dumpcore_mask)))
+                    os_dump_core(msg);
+            }
+            if (TEST(HOTP_EXEC_LOG_EVENT, exec_status))
+                SYSLOG_CUSTOM_NOTIFY(SYSLOG_ERROR, MSG_HOT_PATCH_FAILURE, 3,
+                                     "Hot patch error, continuing.", 
+                                     get_application_name(), 
+                                     get_application_pid(), "<none>");
+#ifdef PROGRAM_SHEPHERDING
+            if (dump_error_info) {
+                report_diagnostics(msg, NULL, HOT_PATCH_FAILURE);
+            }
+#endif
+        }
+    } else {                                            /* EXCEPT block */
+        /* Hot patch crashed! */
+        if (dump_excpt_info) {
+            /* Usually logging the event, dumping core and forensics are done
+             * together.  In this case the first two are done in the exception
+             * handler because that is where the exception specific information
+             * is available.  Forensics are dumped here because this is where
+             * the failing vulnerability's information is available.  Trying to
+             * do all in one place would require too many pieces of information
+             * being passed around.
+             */
+            /* TODO: no hot patch exception specific information is dumped 
+             *       in the forensics files today;  need to do so.
+             */
+            /* TODO: title should say detector or protector exception. */
+#ifdef PROGRAM_SHEPHERDING
+            report_diagnostics("Hot patch exception", NULL, HOT_PATCH_FAILURE);
+#endif
+        }
+        exec_status = HOTP_EXEC_ABORTED;
+    }
+    dcontext->whereami = wherewasi;
+
+    /* Reset hotp_excpt_state to unused.  This will be used in 
+     * create_callback_dcontext() to catch potential callbacks, which might 
+     * lead to nested hot patch exceptions, that result due to system calls 
+     * mades from a hot patch.  Hot patches shouldn't make system calls.
+     */
+    DODEBUG(memset(&dcontext->hotp_excpt_state, -1, sizeof(dr_jmp_buf_t)););
+
+    return exec_status;
+}
+
+/* This routine plugs the hot patch violation event into the core's existing
+ * reporting mechanism.  A new threat id (.H) will be generated for hot patch
+ * violations and the event log will mention whether the violation was detected
+ * or protected.  -report_max will apply these violations.  However, 
+ * -detect_mode and other termination options like -kill_thread, etc. won't be. 
+ *
+ * TODO: currently, -kill_thread and such will apply to hot patches.  They 
+ * must be decoupled.  Not done currently because a clean way of doing it is 
+ * out of the scope of blowfish beta.
+ */
+static void
+hotp_event_notify(hotp_exec_status_t exec_status, bool protected, 
+                  const hotp_offset_match_t *inject_point,
+                  const app_pc bad_addr, const hotp_context_t *hotp_cxt)
+{
+#ifdef PROGRAM_SHEPHERDING
+    const char *threat_id = NULL;
+    hotp_type_t hotp_type = GLOBAL_VUL(inject_point->vul_index).type;
+    action_type_t action;
+    app_pc faulting_addr = NULL, inject_addr, old_next_tag;
+    security_violation_t violation_type = INVALID_VIOLATION, res;
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    fragment_t src_frag = {0}, *old_last_frag;
+    dr_mcontext_t old_mc;
+
+    ASSERT(dcontext != NULL && dcontext != GLOBAL_DCONTEXT);
+    ASSERT(inject_point != NULL && hotp_cxt != NULL);
+
+    /* action mapping */
+    switch (exec_status & ~HOTP_EXEC_LOG_EVENT) {
+    case HOTP_EXEC_EXPLOIT_KILL_THREAD:
+        action = ACTION_TERMINATE_THREAD;
+        break;
+    case HOTP_EXEC_EXPLOIT_KILL_PROCESS:
+        action = ACTION_TERMINATE_PROCESS;
+        break;
+    case HOTP_EXEC_EXPLOIT_RAISE_EXCEPTION:
+        action = ACTION_THROW_EXCEPTION;
+        break;
+    default:
+        action = ACTION_CONTINUE;
+    };
+    inject_addr = GLOBAL_MODULE(inject_point->vul_index,
+                                inject_point->set_index,
+                                inject_point->module_index).base_address +    
+                  GLOBAL_PPOINT(inject_point->vul_index,
+                                inject_point->set_index,
+                                inject_point->module_index,
+                                inject_point->ppoint_index).offset;
+
+    /* Determine the faulting address, violation type and threat id. */
+    if (TESTALL(HOTP_TYPE_GBOP_HOOK, hotp_type)) {    /* gbop hook type */
+#ifdef GBOP
+        /* FIXME: share reporting code with gbop_validate_and_act() - have
+         *  one reporting interface for gbop.  Case 8096.  Changes here or
+         *  in gbop_validate_and_act() should be kept in sync.
+         */
+        /* Even though many gbop hooks are implemented with hotp_only interface 
+         * gbop violations are treated separately.
+         */
+        ASSERT(DYNAMO_OPTION(hotp_only));   /* no gbopping in code cache mode */
+
+        /* NOTE: For gbop, the source is actually the hook address and the 
+         * target is the failing address, not vice versa.
+         */
+        faulting_addr = bad_addr;
+        violation_type = GBOP_SOURCE_VIOLATION;
+
+        /* gbop remediations are decided in security_violation(), so set it to
+         * the expected default here otherwise security_violation would assert.
+         */
+        action = ACTION_TERMINATE_PROCESS;
+#endif
+    } else {    /* hot patch type */
+        ASSERT(TESTALL(HOTP_TYPE_HOT_PATCH, hotp_type));
+        ASSERT(bad_addr == NULL);
+        faulting_addr = inject_addr;
+        if (protected) {
+            violation_type = HOT_PATCH_PROTECTOR_VIOLATION;
+        } else {
+            violation_type = HOT_PATCH_DETECTOR_VIOLATION;
+        }
+        threat_id = GLOBAL_VUL(inject_point->vul_index).policy_id;
+        ASSERT(threat_id != NULL);
+    }
+
+    /* Save the last fragment, next tag & register state, set them up with the
+     * right values, report and restore afterwards.
+     * Note: for hotp_only, src & tgt are the same; for gbop see comment above.
+     */
+    hotp_spill_before_notify(dcontext, &old_last_frag, &src_frag, inject_addr,
+                             &old_next_tag, faulting_addr, &old_mc, 
+                             (void *)hotp_cxt, CXT_TYPE_HOT_PATCH);
+
+    res = security_violation_internal(dcontext, faulting_addr, violation_type, 
+                                      OPTION_REPORT|OPTION_BLOCK, threat_id,
+                                      action, &hotp_vul_table_lock);
+
+    /* Some sanity checks before we go on our merry way.  BTW, couldn't use 
+     * DODEBUG because gcc complained about using the #ifdef GBOP inside a
+     * macro; not an issue; the compiler should optimize out the whole if-block.
+     */
+    if (res == ALLOWING_BAD) {
+        /* Threat exemptions are only for gbop hooks, they don't make sense 
+         * for hot patches - if you don't want a hot patch's event, just
+         * turn it off.
+         */
+        ASSERT(TESTALL(HOTP_TYPE_GBOP_HOOK, hotp_type));
+        ASSERT(!TESTALL(HOTP_TYPE_HOT_PATCH, hotp_type));
+    } else if (res == HOT_PATCH_DETECTOR_VIOLATION ||
+               res == HOT_PATCH_PROTECTOR_VIOLATION) {
+        /* Can return only to continue. */
+        ASSERT(action == ACTION_CONTINUE);
+    } else {
+#ifdef GBOP
+        ASSERT(res == GBOP_SOURCE_VIOLATION);
+        ASSERT(action == ACTION_CONTINUE || DYNAMO_OPTION(detect_mode));
+#endif
+    }
+
+    hotp_restore_after_notify(dcontext, old_last_frag, old_next_tag, &old_mc);
+
+    /* Can't leave this function without holding the hotp lock! */
+    ASSERT_OWN_READ_LOCK(true, &hotp_vul_table_lock);
+#endif
+}
+
+/* This is a hack to make hotp use our existing security violation reporting 
+ * mechanism, which relies on fragments & tags to report violations & generate 
+ * forensics.  Case 8079 talks about cleaning up the reporting interface.
+ */
+void
+hotp_spill_before_notify(dcontext_t *dcontext,
+                         fragment_t **frag_spill /* OUT */,
+                         fragment_t *new_frag, const app_pc new_frag_tag,
+                         app_pc *new_tag_spill /* OUT */,
+                         const app_pc new_next_tag,
+                         dr_mcontext_t *cxt_spill /* OUT */,
+                         const void *new_cxt, cxt_type_t cxt_type)
+{
+    dr_mcontext_t *mc;
+    ASSERT(dcontext != NULL && dcontext != GLOBAL_DCONTEXT);
+    ASSERT(frag_spill != NULL && new_frag != NULL && new_frag_tag != NULL);
+    ASSERT(new_tag_spill != NULL && new_next_tag != NULL);
+    ASSERT(cxt_spill != NULL && new_cxt != NULL);
+    ASSERT(cxt_type == CXT_TYPE_HOT_PATCH || cxt_type == CXT_TYPE_CORE_HOOK);
+
+    *frag_spill = dcontext->last_fragment;
+    *new_tag_spill = dcontext->next_tag;
+
+    new_frag->tag = new_frag_tag;
+    dcontext->last_fragment = (fragment_t *) new_frag;
+    dcontext->next_tag = new_next_tag;
+
+    /* For hotp_only the last_fragment should be linkstub_empty_fragment,
+     * which is static in link.c
+     *
+     * next_tag can be set to BACK_TO_NATIVE_AFTER_SYSCALL, so can't
+     * easily assert on that.
+     */
+    ASSERT(!DYNAMO_OPTION(hotp_only) || 
+           ((*frag_spill)->tag == NULL &&
+            (*frag_spill)->flags == FRAG_FAKE));
+
+    /* Saving & swapping contexts - this is needed to produce the 
+     * correct machine context for forensics; there can be two types, viz., 
+     * hotp_context_t if called from hotp_event_notify() and 
+     * app_state_at_intercept_t if called from gbop_validate_and_act().  
+     */
+    mc = get_mcontext(dcontext);
+    ASSERT(mc != NULL);
+    memcpy(cxt_spill, mc, sizeof(*cxt_spill));
+    if (cxt_type == CXT_TYPE_HOT_PATCH) {
+        hotp_context_t *new = (hotp_context_t *) new_cxt;
+        *mc = *new;
+        /* FIXME PR 226036: use hotp_context_t.xip */
+        mc->pc = NULL;      /* pc reported in source, so NULL here is ok. */
+    } else if (cxt_type == CXT_TYPE_CORE_HOOK) {
+        app_state_at_intercept_t *new = (app_state_at_intercept_t *) new_cxt;
+        *mc = new->mc;
+        /* FIXME PR 226036: use hotp_context_t.xip */
+        mc->pc = NULL;      /* pc reported in source, so NULL here is ok. */
+    } else 
+        ASSERT_NOT_REACHED();
+}
+
+/* Restore dcontext last_fragment & next_tag after reporting the violation. */
+/* FIXME: old_cxt is unused; see case 8099 about dumping context. */
+void
+hotp_restore_after_notify(dcontext_t *dcontext, const fragment_t *old_frag, 
+                         const app_pc old_next_tag,
+                         const dr_mcontext_t *old_cxt)
+{
+    dr_mcontext_t *mc;
+    ASSERT(dcontext != NULL && dcontext != GLOBAL_DCONTEXT);
+    ASSERT (old_cxt != NULL);
+
+    dcontext->last_fragment = (fragment_t *) old_frag;
+    dcontext->next_tag = old_next_tag;
+
+    mc = get_mcontext(dcontext);
+    ASSERT(mc != NULL);
+    memcpy(mc, old_cxt, sizeof(*old_cxt));
+}
+
+#if defined(DEBUG) && defined(INTERNAL)
+/* FIXME PR 226036: eip is now part of hotp_context_t */
+static void
+hotp_dump_reg_state(const hotp_context_t *reg_state, const app_pc eip, 
+                    const uint loglevel)
+{
+    LOG(GLOBAL, LOG_HOT_PATCHING, loglevel, "eax: "PFX"\n", APP_XAX(reg_state));
+    LOG(GLOBAL, LOG_HOT_PATCHING, loglevel, "ecx: "PFX"\n", APP_XCX(reg_state));
+    LOG(GLOBAL, LOG_HOT_PATCHING, loglevel, "edx: "PFX"\n", APP_XDX(reg_state));
+    LOG(GLOBAL, LOG_HOT_PATCHING, loglevel, "ebx: "PFX"\n", APP_XBX(reg_state));
+    LOG(GLOBAL, LOG_HOT_PATCHING, loglevel, "esp: "PFX"\n", APP_XSP(reg_state));
+    LOG(GLOBAL, LOG_HOT_PATCHING, loglevel, "ebp: "PFX"\n", APP_XBP(reg_state));
+    LOG(GLOBAL, LOG_HOT_PATCHING, loglevel, "esi: "PFX"\n", APP_XSI(reg_state));
+    LOG(GLOBAL, LOG_HOT_PATCHING, loglevel, "edi: "PFX"\n", APP_XDI(reg_state));
+    LOG(GLOBAL, LOG_HOT_PATCHING, loglevel, "eip: "PFX"\n", eip);
+}
+#endif
+
+static void
+hotp_update_vul_stats(const hotp_exec_status_t exec_status, 
+                      const uint vul_index)
+{
+    hotp_exec_status_t temp = exec_status & ~HOTP_EXEC_LOG_EVENT;
+
+    ASSERT(temp == HOTP_EXEC_EXPLOIT_DETECTED ||
+           temp == HOTP_EXEC_EXPLOIT_NOT_DETECTED ||
+           temp == HOTP_EXEC_DETECTOR_ERROR ||
+           temp == HOTP_EXEC_EXPLOIT_PROTECTED ||
+           temp == HOTP_EXEC_EXPLOIT_NOT_PROTECTED ||
+           temp == HOTP_EXEC_EXPLOIT_KILL_THREAD ||
+           temp == HOTP_EXEC_EXPLOIT_KILL_PROCESS ||
+           temp == HOTP_EXEC_EXPLOIT_RAISE_EXCEPTION ||
+           temp == HOTP_EXEC_CHANGE_CONTROL_FLOW ||
+           temp == HOTP_EXEC_PROTECTOR_ERROR ||
+           temp == HOTP_EXEC_ABORTED);
+
+    /* FIXME: Grabbing the hot patch lock here to update stats will deadlock
+     *        if a nudge is waiting for this thread to get out.  If a lock 
+     *        isn't grabbed, then the stats may be slightly inaccurate if 2 
+     *        threads update the same stat for a given vulnerability at
+     *        the same time; odds are low and inaccurate stats aren't a problem.
+     *        We aren't trying to provide accurate stats; besides stats for a
+     *        vulnerability for all process in all nodes using it is vague
+     *        data anyway.
+     *        if VUL_STAT_INC becomes atomic, we won't need a lock here.
+     * FIXME: Vlad suggested creating a stats lock; good idea;
+     */
+    switch (temp) {
+    case HOTP_EXEC_EXPLOIT_DETECTED: {
+        VUL_STAT_INC(GLOBAL_VUL(vul_index).info->num_detected);
+        break;
+    }
+    case HOTP_EXEC_EXPLOIT_NOT_DETECTED: {
+        VUL_STAT_INC(GLOBAL_VUL(vul_index).info->num_not_detected);
+        break;
+    }
+    case HOTP_EXEC_DETECTOR_ERROR: {
+        VUL_STAT_INC(GLOBAL_VUL(vul_index).info->num_detector_error);
+        break;
+    }
+    case HOTP_EXEC_EXPLOIT_PROTECTED: {
+        VUL_STAT_INC(GLOBAL_VUL(vul_index).info->num_protected);
+        break;
+    }
+    case HOTP_EXEC_EXPLOIT_NOT_PROTECTED: {
+        VUL_STAT_INC(GLOBAL_VUL(vul_index).info->num_not_protected);
+        break;
+    }
+    case HOTP_EXEC_EXPLOIT_KILL_THREAD: {
+        VUL_STAT_INC(GLOBAL_VUL(vul_index).info->num_kill_thread);
+        break;
+    }
+    case HOTP_EXEC_EXPLOIT_KILL_PROCESS: {
+        VUL_STAT_INC(GLOBAL_VUL(vul_index).info->num_kill_process);
+        break;
+    }
+    case HOTP_EXEC_EXPLOIT_RAISE_EXCEPTION: {
+        VUL_STAT_INC(GLOBAL_VUL(vul_index).info->num_raise_exception);
+        break;
+    }
+    case HOTP_EXEC_CHANGE_CONTROL_FLOW: {
+        VUL_STAT_INC(GLOBAL_VUL(vul_index).info->num_change_control_flow);
+        break;
+    }
+    case HOTP_EXEC_PROTECTOR_ERROR: {
+        VUL_STAT_INC(GLOBAL_VUL(vul_index).info->num_protector_error);
+        break;
+    }
+    case HOTP_EXEC_ABORTED: {
+        VUL_STAT_INC(GLOBAL_VUL(vul_index).info->num_aborted);
+        break;
+    }
+    default: ASSERT_NOT_REACHED();
+    }
+}
+
+/* Note: 1. This function will not return, unless there is an error.
+ *       2. The number of patch points must be passed to this function to handle
+ *          control flow changes with multiple patches at the same offset.
+ *
+ * CAUTION: Any change to the code generated by hotp_inject_gateway_call() (and,
+ *          thus, prepare_for_clean_call()), will affect how the app. state
+ *          is spilled on the dr stack.  This function uses that app. state,
+ *          hence, relies on that order being constant.
+ *          TODO: How to link an assert to these two, so that any change is
+ *                caught immediately?
+ *
+ * TODO: show stack diagrams otherwise it is going to be messy
+ */
+/* These constants refer to the offset of eflags and errno that are saved on
+ * the stack as part of the clean call.  The offsets are relative to the 
+ * location of the pushed register state, i.e., esp after pusha in the clean 
+ * call sequence.  Any change to prepare_for_clean_call() will affect this.
+ */
+#define CLEAN_CALL_XFLAGS_OFFSET    1
+#define CLEAN_CALL_ERRNO_OFFSET     2
+static void
+hotp_change_control_flow(const hotp_context_t *app_reg_ptr, const app_pc target)
+{
+    dcontext_t *dcontext;
+    int app_errno;
+    dr_mcontext_t mc = *app_reg_ptr;
+
+    /* TODO: Eventually, must assert that target is in some module. */
+    ASSERT(app_reg_ptr != NULL && target != NULL);
+
+    dcontext = get_thread_private_dcontext();
+    ASSERT(dcontext != NULL && dcontext != GLOBAL_DCONTEXT);
+
+    /* Get errno from the state that was spilled during clean call. 
+     * This is the only reliable place to get it from.  The ones in the
+     * mcontext might be stale if the control flow was inside the fcache for
+     * a while.  FIXME: should we use app_state_at_intercept_t, which
+     * has app_errno, for hotp_context_t?
+     */
+    app_errno = *(((uint*)app_reg_ptr) - CLEAN_CALL_ERRNO_OFFSET);
+
+    dcontext->next_tag = target;    /* Set up actual control flow change. */
+    dcontext->whereami = WHERE_FCACHE;
+    /* FIXME: should determine the actual fragment exiting from */
+    set_last_exit(dcontext, (linkstub_t *) get_hot_patch_linkstub());
+
+    STATS_INC(hotp_num_cflow_change);
+    LOG(GLOBAL, LOG_HOT_PATCHING, 1, "Changing control flow to "PFX"\n", target);
+    transfer_to_dispatch(dcontext, app_errno, &mc);
+    ASSERT_NOT_REACHED();
+}
+
+/* Prints hotpatch vulnerability table information to forensics file in xml
+ * format.
+ */
+void
+hotp_print_diagnostics(file_t diagnostics_file)
+{
+    uint vul, set, module, pp, hash;
+
+    if (GLOBAL_VUL_TABLE == NULL) {
+        print_file(diagnostics_file,
+            "<hotpatching-information>\n"
+            "Hotpatch vulnerability table is NULL\n"
+            "</hotpatching-information>\n");
+        return;
+    }
+
+    print_file(diagnostics_file,
+        "<hotpatching-information>\n"
+        "<vulnerability-table num-vulnerabilities=\"%u\">\n", NUM_GLOBAL_VULS);
+    for (vul = 0; vul < NUM_GLOBAL_VULS; vul++) {
+        hotp_inject_status_t inject_status;
+
+        print_file(diagnostics_file,
+            "  <vulnerability id=\"%s\" num-sets=\"%u\">\n", 
+                GLOBAL_VUL(vul).vul_id,
+                GLOBAL_VUL(vul).num_sets);
+        print_file(diagnostics_file, 
+            "    <policy id=\"%s\" mode=\"%d\" version=\"%u\"/>\n",
+                GLOBAL_VUL(vul).policy_id,
+                GLOBAL_VUL(vul).mode,
+                GLOBAL_VUL(vul).policy_version);
+
+        /* For vulnerabilities that haven't been used, print only the
+         * {vul,policy}_id and policy_version; helps to prevent clustter in the
+         * forensics file.  Using the ids & version we can get the hot patch 
+         * definition from our packages/code in house, so not dumping them 
+         * doesn't hamper diagnosis.  Case 8549.
+         */
+        inject_status = *(GLOBAL_VUL(vul).info->inject_status);
+        if (inject_status == HOTP_INJECT_NO_MATCH ||
+            inject_status == HOTP_INJECT_OFF) {
+            print_file(diagnostics_file, "  </vulnerability>\n");
+            continue;
+        }
+
+        print_file(diagnostics_file,
+            "    <hotpatch-dll name=\"%s\" base=\""PFX"\" hash=\"%s\"/>\n",
+                GLOBAL_VUL(vul).hotp_dll,
+                GLOBAL_VUL(vul).hotp_dll_base,
+                GLOBAL_VUL(vul).hotp_dll_hash);
+        for (set = 0; set < GLOBAL_VUL(vul).num_sets; set++) {
+            bool print_sets = false;
+            for (module = 0; module < GLOBAL_SET(vul, set).num_modules;
+                 module++) {
+                /* If a module isn't matched, then it hasn't been used, don't
+                 * dump it; case 8549.
+                 */
+                if (!GLOBAL_MODULE(vul, set, module).matched)
+                    continue;
+
+                if (!print_sets) {  /* Print set title if needed, case 8549. */
+                    print_file(diagnostics_file, 
+                        "    <set num-modules=\"%u\">\n",
+                            GLOBAL_SET(vul, set).num_modules);
+                    print_sets = true;
+                }
+                 
+                print_file(diagnostics_file,
+                    "      <module pe_name=\"%s\" pe_checksum=\"0x%x\" "
+                    "pe_timestamp=\"0x%x\" pe_image_size=\""PIFX"\" "
+                    "pe_code_size=\""PIFX"\" "
+                    "pe_file_version=\"0x"HEX64_FORMAT_STRING"\" "
+                    "num-hashes=\"%u\" num-patch-points=\"%u\">\n",
+                        GLOBAL_SIG(vul, set, module).pe_name,
+                        GLOBAL_SIG(vul, set, module).pe_checksum,
+                        GLOBAL_SIG(vul, set, module).pe_timestamp,
+                        GLOBAL_SIG(vul, set, module).pe_image_size,
+                        GLOBAL_SIG(vul, set, module).pe_code_size,
+                        GLOBAL_SIG(vul, set, module).pe_file_version,
+                        GLOBAL_MODULE(vul, set, module).num_patch_point_hashes,
+                        GLOBAL_MODULE(vul, set, module).num_patch_points);
+                for (hash = 0; 
+                     hash < GLOBAL_MODULE(vul, set, module).num_patch_point_hashes;
+                     hash++) {
+                    print_file(diagnostics_file,
+                        "        <hash start=\""PFX"\" length=\"0x%x\" "
+                        "hash=\"%u\"/>\n",
+                            GLOBAL_HASH(vul, set, module, hash).start,
+                            GLOBAL_HASH(vul, set, module, hash).len,
+                            GLOBAL_HASH(vul, set, module, hash).hash_value);
+                }
+                for (pp = 0; 
+                     pp < GLOBAL_MODULE(vul, set, module).num_patch_points;
+                     pp++) {
+                    print_file(diagnostics_file,
+                        "        <hotpatch precedence=\"%u\" offset=\""PIFX"\">\n",
+                            GLOBAL_PPOINT(vul, set, module, pp).precedence,
+                            GLOBAL_PPOINT(vul, set, module, pp).offset);
+                    print_file(diagnostics_file, 
+                        "          <function type=\"detector\" "
+                        "offset=\""PIFX"\"/>\n",
+                            GLOBAL_PPOINT(vul, set, module, pp).detector_fn);
+                    print_file(diagnostics_file, 
+                        "          <function type=\"protector\" "
+                        "offset=\""PIFX"\" return=\""PFX"\"/>\n",
+                            GLOBAL_PPOINT(vul, set, module, pp).protector_fn,
+                            GLOBAL_PPOINT(vul, set, module, pp).return_addr);
+                    print_file(diagnostics_file,
+                        "        </hotpatch>\n");
+                }
+                print_file(diagnostics_file,
+                    "      </module>\n");
+            }
+            if (print_sets) {   /* xref case 8549 */
+                print_file(diagnostics_file,
+                    "    </set>\n");
+            }
+        }
+        print_file(diagnostics_file,
+            "    <stats "
+            "num-detected=\""UINT64_FORMAT_STRING"\" "
+            "num-not-detected=\""UINT64_FORMAT_STRING"\" "
+            "num-detector-error=\""UINT64_FORMAT_STRING"\" "
+            "num-protected=\""UINT64_FORMAT_STRING"\" "
+            "num-not-protected=\""UINT64_FORMAT_STRING"\" "
+            "num-kill-thread=\""UINT64_FORMAT_STRING"\" "
+            "num-kill-process=\""UINT64_FORMAT_STRING"\" "
+            "num-raise-exception=\""UINT64_FORMAT_STRING"\" "
+            "num-change-control-flow=\""UINT64_FORMAT_STRING"\" "
+            "num-protector-error=\""UINT64_FORMAT_STRING"\" "
+            "num-aborted=\"" UINT64_FORMAT_STRING"\">\n",
+                   GLOBAL_VUL(vul).info->num_detected,
+                   GLOBAL_VUL(vul).info->num_not_detected,
+                   GLOBAL_VUL(vul).info->num_detector_error,
+                   GLOBAL_VUL(vul).info->num_protected,
+                   GLOBAL_VUL(vul).info->num_not_protected,
+                   GLOBAL_VUL(vul).info->num_kill_thread,
+                   GLOBAL_VUL(vul).info->num_kill_process,
+                   GLOBAL_VUL(vul).info->num_raise_exception,
+                   GLOBAL_VUL(vul).info->num_change_control_flow,
+                   GLOBAL_VUL(vul).info->num_protector_error,
+                   GLOBAL_VUL(vul).info->num_aborted);
+        print_file(diagnostics_file,
+            "      <status type=\"execution\">%d</status>\n"
+            "      <status type=\"injection\">%d</status>\n",
+                   GLOBAL_VUL(vul).info->exec_status,
+                   *(GLOBAL_VUL(vul).info->inject_status));
+        print_file(diagnostics_file,
+            "    </stats>\n"
+            "  </vulnerability>\n");
+    }
+    print_file(diagnostics_file,
+        "</vulnerability-table>\n"
+        "</hotpatching-information>\n");
+}
+
+#if defined(DEBUG) && defined(DEBUG_MEMORY)
+/* Part of bug fix for case 9593 which required leaking trampolines. */
+bool
+hotp_only_contains_leaked_trampoline(byte *pc, size_t size)
+{
+    if (!DYNAMO_OPTION(hotp_only) IF_WINDOWS(|| !doing_detach))
+        return false;
+
+    /* Today memory debug checks for special heap units only do heap accouting,
+     * but not memcmp, both of which are done for regular heaps.  Special heaps
+     * are where the leaked trampolines are located.  If we do implement that
+     * check then this code in #if 0 would be needed.  Case 10434. */
+#if 0
+    for (i = 0; i < hotp_only_num_tramps_leaked; i++)
+        if (hotp_only_tramps_leaked[i] >= pc &&
+            hotp_only_tramps_leaked[i] < (pc + size)) {
+            /* Make sure we don't have trampolines across heap units! */
+            ASSERT((hotp_only_tramps_leaked[i] + HOTP_ONLY_TRAMPOLINE_SIZE) <=
+                   (pc + size));
+            return true; 
+        }
+#endif
+
+    /* The actual special_units_t structure (pointed to by
+     * hotp_only_tramp_heap) is also leaked.
+     * Note: hotp_only_tramp_heap_cache can be NULL if no hotp_only type
+     * patches were ever removed either because they were never injected or
+     * just weren't removed.
+     */
+    if ((byte *) hotp_only_tramp_heap_cache >= pc &&
+        (byte *) hotp_only_tramp_heap_cache < (pc + size))
+        return true;
+
+    return false;
+}
+#endif
+/*----------------------------------------------------------------------------*/
+#ifdef GBOP
+
+/* This section contains most of the functionality needed to treat gbop hooks 
+ * as hotp_only patches, thus giving gbop hooks access to all hotp_only patch 
+ * functionality.  See case 7949 & 7127.
+ */
+
+/* Note: Both the gbop detector and protector request for log events.  However, 
+ *  the detector events are reported only in -detect_mode and protector ones 
+ *  in !-detect_mode.
+ * Note: The app eax & edx are spilled by the gateway and used as scratch;
+ *  eax to get the faulting address & edx to send in the current pc (which is
+ *  also set by the gateway). xref case 6804 about hotp interface expansion.
+ */
+static hotp_exec_status_t
+hotp_only_gbop_detector(hotp_context_t *cxt)
+{
+    if (gbop_check_valid_caller((app_pc) APP_XBP(cxt),
+                                (app_pc) APP_XSP(cxt),
+                                (app_pc) APP_XDX(cxt), 
+                                (app_pc *) &APP_XAX(cxt))) {
+        return HOTP_EXEC_EXPLOIT_NOT_DETECTED;
+    } else {
+        /* Ask for event; needed to log event if -detect_mode is specified. */
+        return HOTP_EXEC_EXPLOIT_DETECTED | HOTP_EXEC_LOG_EVENT;
+    }
+}
+
+static hotp_exec_status_t
+hotp_only_gbop_protector(hotp_context_t *cxt)
+{
+#ifdef PROGRAM_SHEPHERDING
+    ASSERT(!DYNAMO_OPTION(detect_mode));    /* No protection in detect_mode. */
+#endif
+
+    /* Just log the event; the remediation action for gbop is determined by
+     * security_violation() using core options like -kill_thread.
+     */
+    return HOTP_EXEC_EXPLOIT_PROTECTED | HOTP_EXEC_LOG_EVENT;
+}
+
+/* Note: num_vuls is an IN OUT argument; is specifies the current table size and
+ * is updated to the new size after reading gbop hooks.  The IN value is used
+ * as the append index into the table.
+ */
+static void
+hotp_only_read_gbop_policy_defs(hotp_vul_t *tab, uint *num_vuls)
+{
+    uint vul_idx;
+    hotp_vul_t *vul;
+    hotp_set_t *set;
+    hotp_module_t *module;
+    hotp_patch_point_t *patch_point;
+    uint gbop_num_hooks = gbop_get_num_hooks();
+    app_pc dr_base = NULL;
+
+    ASSERT(tab != NULL && num_vuls != NULL);
+    ASSERT(gbop_num_hooks > 0);
+    /* No gbopping for regular hotp, at least not until hotp_only and regular 
+     * hotp coexist, i.e., hotp_only for native_exec dlls (case 6892)
+     */
+    ASSERT(DYNAMO_OPTION(hotp_only) && DYNAMO_OPTION(gbop));
+
+    dr_base = get_module_base((app_pc)hotp_only_read_gbop_policy_defs);
+    ASSERT(dr_base != NULL);
+
+    for (vul_idx = *num_vuls; vul_idx < (*num_vuls + gbop_num_hooks); vul_idx++) {
+        uint gbop_hook_idx = vul_idx - *num_vuls;
+        const gbop_hook_desc_t *gbop_hook = gbop_get_hook(gbop_hook_idx);
+        ASSERT(gbop_hook != NULL);
+        vul = &tab[vul_idx];
+        vul->vul_id = dr_strdup(gbop_hook->func_name
+                                HEAPACCT(ACCT_HOT_PATCHING));
+        /* FIXME: construct this from a combination of {mod,func}_name, or 
+         * func_name and bad_ret_address.  For now, just hard code it.
+         * strdup because hotp_free() thinks this is allocated.
+         */
+        vul->policy_id = dr_strdup("GBOP.VIOL" HEAPACCT(ACCT_HOT_PATCHING));
+
+        /* FIXME: should this be used to track changes to the gbop detector 
+         * and protector?  does it matter, after all these patches will only 
+         * go as part of the core?
+         */
+        vul->policy_version = 1;
+
+        vul->hotp_dll = NULL;
+        
+        vul->hotp_dll_hash = NULL;
+        /* There is no notion of only detecting and doing nothing for gbop, so
+         * the mode is always protect.
+         */
+        /* gbop_execlude_filter handles any os specific gbop set removals, xref 9772 */
+        if (gbop_exclude_filter(gbop_hook)) {
+            vul->mode = HOTP_MODE_OFF;
+            LOG(GLOBAL, LOG_HOT_PATCHING, 1, "Excluding %s!%s\n", 
+                gbop_hook->mod_name, gbop_hook->func_name);
+        } else {
+            vul->mode = HOTP_MODE_PROTECT;
+        }
+
+        vul->num_sets = 1;
+        set = HEAP_ARRAY_ALLOC(GLOBAL_DCONTEXT, hotp_set_t, 1, 
+                               ACCT_HOT_PATCHING, PROTECTED);
+        vul->sets = set;
+        vul->info = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, hotp_vul_info_t,
+                                    ACCT_HOT_PATCHING, PROTECTED);
+        memset(vul->info, 0, sizeof(hotp_vul_info_t));  /* Initialize stats. */
+
+        vul->hotp_dll_base = dr_base;
+        vul->type = HOTP_TYPE_GBOP_HOOK;
+
+        set->num_modules = 1;
+        module = HEAP_ARRAY_ALLOC(GLOBAL_DCONTEXT, hotp_module_t, 1, 
+                                  ACCT_HOT_PATCHING, PROTECTED);
+        set->modules = module;
+
+        module->sig.pe_name = dr_strdup(gbop_hook->mod_name
+                                        HEAPACCT(ACCT_HOT_PATCHING));
+        module->sig.pe_checksum = 0;
+        module->sig.pe_timestamp = 0;
+        module->sig.pe_image_size = 0;
+        module->sig.pe_code_size = 0;
+        module->sig.pe_file_version = 0;
+        module->num_patch_points = 1;
+        patch_point = HEAP_ARRAY_ALLOC(GLOBAL_DCONTEXT, hotp_patch_point_t, 1,
+                                       ACCT_HOT_PATCHING, PROTECTED);
+        module->patch_points = patch_point;
+        module->num_patch_point_hashes = 0;
+        module->hashes = NULL;
+        module->matched = false;
+        module->base_address = NULL;
+
+        /* The actual patch offset will be computed if the module matches. 
+         * vul_id holds the function name, which will be used to compute the
+         * offset.  See hotp_process_image().
+         * FIXME: we could use a union for offset to hold offset or func_name;
+         *  that would be elegant, but would require changing too many things
+         *  in hotp - not a good idea, not at least for the first implementation
+         */
+        patch_point->offset = 0;
+        patch_point->detector_fn = (app_pc)hotp_only_gbop_detector - dr_base;
+        patch_point->protector_fn = (app_pc)hotp_only_gbop_protector - dr_base;
+
+        /* Longer term issue: do we want to have the notion of changing 
+         * control flow for gbop hooks?
+         */
+        patch_point->return_addr = 0;
+        /* Precedence hasn't been implemented yet, however, if it had been,
+         * then we don't want gbop hooks to interfere with other patches.
+         */
+        patch_point->precedence = HOTP_ONLY_GBOP_PRECEDENCE;
+        patch_point->trampoline = NULL;
+        patch_point->app_code_copy = NULL;
+        patch_point->tramp_exit_cti = NULL;
+    }
+
+    *num_vuls += gbop_num_hooks; 
+    ASSERT(*num_vuls == vul_idx);
+}
+
+#endif  /* GBOP */
+#ifdef CLIENT_INTERFACE
+/*----------------------------------------------------------------------------*/
+/* TODO: move all probe api into a separate file - cleaner */
+/* Adding a few data types to see if doxygen works.  Will update later.
+*/
+/* DR_API EXPORT TOFILE dr_probe.h */
+/* DR_API EXPORT BEGIN */
+/**
+ * @file dr_probe.h
+ * @brief Support for the Probe API.
+ *
+ *  Describes all the data types and functions associated with the \ref
+ * API_probe.
+ */
+
+/** Specifies what type of computation to use when computing the address of a
+ * probe insertion point or callback function.
+ */
+typedef enum {
+    /** Use the raw virtual address specified. */
+    DR_PROBE_ADDR_VIRTUAL,
+
+    /** Compute address by adding the offset specified to the base of the
+     * library specified.
+     *
+     * For probe insertion, if library isn't loaded, the insertion will be
+     * aborted.  For computing callback function address if the library isn't
+     * loaded, it will be loaded and then the address computation will be done;
+     * if it can't be loaded, probe request is aborted.
+     */
+    DR_PROBE_ADDR_LIB_OFFS,
+
+    /** Compute address by getting the address of the specified exported
+     * function from the specified library.
+     *
+     * If the exported function specified isn't available either for the probe
+     * insertion point or for the callback function, the probe insertion is
+     * aborted.  For computing callback function address if the library isn't
+     * loaded, it will be loaded and then the address computation will be done;
+     * if it can't be loaded, probe request is aborted. */
+    DR_PROBE_ADDR_EXP_FUNC,
+} dr_probe_addr_t;
+
+/** Defines the location where a probe is to be inserted or callback function
+ * defined as an offset within a library. */
+typedef struct {
+    /** IN - Full name of the library. */
+    char *library;
+
+    /** IN - Offset to use inside library.  If out of bounds of library, probe
+     * request is aborted.  Offset can point to non text location as long as it
+     * is marked executable (i.e., ..x). */
+    app_rva_t offset;
+} dr_probe_lib_offs_t;
+
+/** Defines the location where a probe is to be inserted or callback function
+ * defined as an exported function within a library. */
+typedef struct {
+    /** IN - Full name of the library. */
+    char *library;
+
+    /** IN - Name of exported function inside library.  If the function can't
+     * be found in the library, then this probe request is aborted. */
+    char *func;
+} dr_probe_exp_func_t;
+
+
+/** Defines a memory location where a probe is to be inserted or where a
+ * callback function exists.  One of three types of address computation as
+ * described by dr_probe_addr_t is used identify the location.
+ */
+typedef struct {
+    /** IN - Specifies the type of address computation to use. */
+    dr_probe_addr_t type;
+
+    union {
+        /** IN - Raw virtual address in the process space. */
+        app_pc vaddr;
+
+        /** IN - Library offset in the process space. */
+        dr_probe_lib_offs_t lib_offs;
+
+        /** IN - Exported function in the process space. */
+        dr_probe_exp_func_t exp_func;
+    };
+} dr_probe_location_t;
+
+/** probe descriptor
+ * This structure describes the characteristics of a probe.  It is used to add,
+ * update and remove probes using dr_register_probes().
+ */
+typedef struct {
+    /** IN - Pointer to the name of the probe.*/
+    char *name;
+
+    /** IN - Location where the probe is to be inserted.  If inserting inside a
+     * library, the insertion will be done only if the library is loaded, the
+     * location falls within its bounds and the location is marked executable.
+     * If inserting outside a library the memory location should be allocated
+     * and marked executable.  If neither, the probe will be put in a pending
+     * state where its id will be NULL and its status reflecting the state. */
+    dr_probe_location_t insert_loc;
+
+    /** IN - Location of the callback function.  If callback is inside a
+     * library, the library location will be used if it is within its bounds
+     * and is marked executable; if library isn't loaded, an attempt will be
+     * made to load it.  If using a raw virtual address, then that location
+     * should be mapped and marked executable.  If neither is true, the probe
+     * insertion or update will be aborted and status updated accordingly. */
+    dr_probe_location_t callback_func;
+
+    /** OUT - Upon successful probe insertion a unique id will be created. */
+    unsigned int id;
+
+    /** OUT - Specifies the current status of the probe. */
+    dr_probe_status_t status;
+} dr_probe_desc_t;
+
+/******************************************************************************/
+DR_API
+/**
+ * Allows the caller to insert probes into specified executable memory
+ * locations in the process address space.  Upon subsequent execution of
+ * instructions at these memory locations the appropriate probes will be
+ * triggered and the corresponding callback functions will be invoked.
+ * Subsequent calls to dr_register_probes() will allow the caller to remove,
+ * update and add more probes.
+ *
+ * @param       probes  Pointer to an array of probe descriptors of type 
+ *                      dr_probe_desc_t.  Each descriptor describes the 
+ *                      location of the probe, the callback function and other 
+ *                      data.  This is an in/out parameter, see dr_probe_desc_t 
+ *                      for details.  If probes isn't NULL, points to valid
+ *                      memory and num_probes is greater than 0, id and status
+ *                      for each probe are set in the corresponding
+ *                      dr_probe_desc_t.  If probes is NULL or num_probes is 0,
+ *                      nothing is set in probes.  Invalid memory may trigger
+ *                      an exception.
+ *
+ * @param [in]  num_probes  Specifies the number of probe descriptors in 
+ *                          the array pointed to by probes.
+ *
+ * \remarks
+ * If a probe definition is invalid, it will not be registered, i.e. DynamoRIO 
+ *  will not retain its definition.  The error code will be returned in the
+ *  status field in of that probe's dr_probe_desc_t element and the
+ *  corresponding id field in dr_probe_desc_t is set to NULL.
+ *
+ * \remarks
+ * When a client calls dr_register_probes() from dr_init() (which is the 
+ *  earliest dr_register_probes() can be called), not all valid probes are
+ *  guaranteed to be inserted upon retrun from dr_register_probes().  Some
+ *  valid probes may not be inserted if the target module has not been loaded,
+ *  the insertion point is not executable, or the memory is otherwise
+ *  inaccessible.  In such cases, DynamoRIO retains all valid probe
+ *  information and inserts these probes when the memory locations become
+ *  available.
+ *
+ * \remarks
+ * When dr_register_probes() is called after dr_init(), the behavior is 
+ *  identical to being called from dr_init() with one caveat: even valid probes
+ *  aren't guaranteed to be registered when dr_register_probes() returns.
+ *  However, valid probes will be registered with a few milliseconds usually.
+ *  To prevent performance and potential deadlock issues, it is recommended
+ *  that a client shouldn't wait in a loop till the probe status changes.
+ *  Instead, clients should check probe status at a later callback.
+ *
+ * \remarks
+ * A client can determine the status of a registered probe in one of two ways.
+ *  1) Read it from the status field in the associated dr_probe_desc_t element
+ *  when dr_register_probes() returns, or 2) call dr_get_probe_status() with
+ *  the id of the probe.
+ *
+ * \remarks
+ * To add, remove or update currently registered probes dr_register_probes() 
+ *  should be called again with a new set of probe definitions.  The new list
+ *  of probes completely replaces the existing probes.  That is, existing
+ *  probes not specified in the new list are removed from the process.
+ *
+ * \remarks
+ * The probe insertion address has limitations.  5 bytes begining at the start
+ *  of a probe insertion address should have specific characteristics.  No
+ *  instruction should straddle this start of this region, i.e., the probe
+ *  insertion address should be the beginning of an instruction.  Also, no flow
+ *  of control should jump into the middle of this 5 byte region beginning at
+ *  the probe insertion address.  Further, there should be no int instructions
+ *  in this region.  call and jump instructions are allowed in this region as
+ *  long as the don't terminate before the end of the region, i.e., no other
+ *  instruction should start after them in this region (as it will allow
+ *  control flow to return to the middle of this region).  If these rules are
+ *  not adhered to the results are be unspecified; may result in process crash.
+ *  The above mentioned restrictions hold only when using \ref API_probe not
+ *  when using API_BT or both simultaneously.
+ *
+ * \remarks
+ * If only the \ref API_probe is used 5 bytes starting at the probe insertion
+ *  address will be modified.  If the process will read this memory then the
+ *  probe should be moved to another location or removed to avoid unknown
+ *  changes in process behavior.  If the client will read this memory, then it
+ *  has to do so before requesting probe insertion.
+ *
+ * \see dr_get_probe_status().
+ */
+void
+dr_register_probes(
+    dr_probe_desc_t *probes,
+    unsigned int num_probes
+);
+
+DR_API
+/** Used to get the current status of a probe.
+ *
+ * \param[in]   id  Unique identifier of the probe for which status is desired.
+ *
+ * \param[out]  status  Pointer to user allocated data of type dr_probe_status_t
+ *                      in which the status of the probe specified by id is
+ *                      returned.  If id matches and status isn't NULL, the
+ *                      status for the matching probe is returned.  If id
+ *                      doesn't match or if status is NULL, nothing is
+ *                      returned in status. 
+ *
+ * \return      If id matches and status is copied successfully, 1 is returned,
+ *              else 0 is returned.
+ */
+int
+dr_get_probe_status(unsigned int id, dr_probe_status_t *status);
+/* DR_API EXPORT END */
+
+/* Both dr_{insert,update}_probes() will be replaced by dr_register_probes() -
+ * PR 225547.  The user will call the same routine to insert or update probes.
+ * Subsequent calls will result in old probes being removed and new ones
+ * inserted.  By manipulating the input array the user can do inserts (adding
+ * new defs. to the array), updates (modifying existing defs) or removes (just
+ * removing unwanted defs from the array).
+ * Depending upon the context (init or other place) an internal nudge will be
+ * created.  
+ * NOTE: for beta, there is no update, i.e., this routine can be called only
+ * once.
+ * NOTE: The input is an array of probes because allowing the user to do
+ * individual probe registration will result in a nudge for each one, which is
+ * very expensive.  Also of note is that it isn't uncommon for API to request
+ * arrays; WIN32 native API does it many places.
+ * TODO: change hotp vul table to be a list - better for clients, esp. multiple
+ * ones; also good if probes are used with LS - PR 225673.
+ */
+void
+dr_register_probes(
+    dr_probe_desc_t *probes,
+    unsigned int num_probes)
+{
+    unsigned int i, valid_probes;
+    hotp_vul_t *tab, *vul;
+    hotp_set_t *set;
+    hotp_module_t *module;
+    hotp_patch_point_t *ppoint;
+    static bool probes_initialized = false;
+
+    /* For now, probes are supported iff probe api is explicity turned on.
+     * Also, liveshields shouldn't be on when probe api is on.
+     */
+    CLIENT_ASSERT(DYNAMO_OPTION(hot_patching) && DYNAMO_OPTION(probe_api) &&
+                  !DYNAMO_OPTION(liveshields), "To use Probe API, "
+                  "-hot_patching, -probe_api and -no_liveshields options "
+                  "should be used.");
+
+    if (!DYNAMO_OPTION(probe_api))
+        return; /* be safe */
+
+    /* Hot patching subsystem should be initialized by now. */
+    ASSERT(hotp_patch_point_areas != NULL);
+    ASSERT(!DYNAMO_OPTION(hotp_only) || hotp_only_tramp_areas != NULL);
+
+    if (num_probes < MIN_NUM_VULNERABILITIES ||
+        num_probes > MAX_NUM_VULNERABILITIES || probes == NULL) {
+        return;
+    }
+
+    /* For beta probe registration can be done only once.  However, multiple
+     * calls to this routine should be allowed during
+     * 1. dr init time - which doesn't need a nudge but needs remove & reinsert
+     *      - PR 225580
+     * 2. any other point in dr - which requires an internal nudge - PR 225578
+     * Once both these are implemented the probes_initialized bool can go.
+     *
+     * Note: as we don't have multiple clients and at startup as we are single
+     *       threaded here, there is no need for a lock for this temp. bool.
+     */
+    if (probes_initialized)
+        return;
+    probes_initialized = true;
+
+    /* Zero out all dynamically allocated hotpatch table structures to avoid
+     * leaks when there is a parse error.  See PR 212707, 213480.
+     */
+    tab = HEAP_ARRAY_ALLOC_MEMSET(GLOBAL_DCONTEXT, hotp_vul_t, num_probes,
+                                  ACCT_HOT_PATCHING, PROTECTED, 0); 
+
+    for (i = 0, valid_probes = 0; i < num_probes; i++) {
+        char *temp;
+
+        vul = &tab[valid_probes];
+        /* memset vul to 0 here because parse errors can leave freed pointers */
+        memset(vul, sizeof(hotp_vul_t), 0);
+
+        /* TODO: remove this once support is added for exported functions
+         * (PR 225654) & raw addresses (PR 225658); for now just prevent 
+         * needless user errors.
+         */
+        if (probes[i].insert_loc.type != DR_PROBE_ADDR_LIB_OFFS ||
+            probes[i].callback_func.type != DR_PROBE_ADDR_LIB_OFFS) {
+            probes[i].status = DR_PROBE_STATUS_UNSUPPORTED;
+            goto dr_probe_parse_error;
+        }
+
+        /* TODO: validate probe def PR 225663 */
+        vul->vul_id = dr_strdup(probes[i].name HEAPACCT(ACCT_HOT_PATCHING));
+
+        /* For probe api policy_id isn't needed, but it can't be set it to NULL
+         * because policy status table (used by drview) set up will crash. */
+        temp = HEAP_ARRAY_ALLOC(GLOBAL_DCONTEXT, char, MAX_POLICY_ID_LENGTH + 1,
+                                ACCT_HOT_PATCHING, PROTECTED);
+        strncpy(temp, probes[i].name, MAX_POLICY_ID_LENGTH);
+        temp[MAX_POLICY_ID_LENGTH] = '\0';
+        /* TODO: validate probe def PR 225663 */
+        vul->policy_id = temp;
+
+        /* Note: if there is a need (highly doubt it) we can expand the probe
+         * api to support versioning; for now just set it to 1. */
+        vul->policy_version = 1;
+
+        vul->hotp_dll = NULL;
+        if (probes[i].callback_func.type == DR_PROBE_ADDR_LIB_OFFS) {
+            if (probes[i].callback_func.lib_offs.library != NULL) {
+                /* TODO: validate probe def PR 225663 */
+                vul->hotp_dll = dr_strdup(probes[i].callback_func.lib_offs.library
+                                          HEAPACCT(ACCT_HOT_PATCHING));
+            } else {
+                probes[i].status = DR_PROBE_STATUS_INVALID_LIB;
+                goto dr_probe_parse_error;
+            }
+        } else if (probes[i].callback_func.type == DR_PROBE_ADDR_EXP_FUNC) {
+            /* TODO: NYI - support for exported functions (PR 225654) */
+            probes[i].status = DR_PROBE_STATUS_UNSUPPORTED;
+            goto dr_probe_parse_error;
+        } else if (probes[i].callback_func.type != DR_PROBE_ADDR_VIRTUAL) {
+            /* TODO: NYI - support for exported functions (PR 225654) */
+            probes[i].status = DR_PROBE_STATUS_UNSUPPORTED;
+            goto dr_probe_parse_error;
+        }
+
+        vul->mode = HOTP_MODE_PROTECT;
+        vul->num_sets = 1;
+        set = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, hotp_set_t, ACCT_HOT_PATCHING,
+                              PROTECTED);
+        vul->sets = set;
+        vul->info = HEAP_ARRAY_ALLOC_MEMSET(GLOBAL_DCONTEXT, hotp_vul_info_t, 1,
+                                            ACCT_HOT_PATCHING, PROTECTED, 0);
+
+        /* Note: if probe is inside client dll, then client dll SHOULD be in
+         * our module_areas - how to assert on this?
+         * Update: I found that neither the client dll nor any dll loaded
+         * during client init is in our loaded_module_areas because these dlls
+         * are loaded after the vm scan in vm_areas_init() but before dr hooks
+         * are inserted - I got pop ups in os_get_module_info() because of
+         * this.  PR 225670.
+         */
+        vul->hotp_dll_base = NULL;
+        vul->type = HOTP_TYPE_PROBE;
+
+        set->num_modules = 1;
+        module = HEAP_ARRAY_ALLOC_MEMSET(GLOBAL_DCONTEXT, hotp_module_t, 1, 
+                                         ACCT_HOT_PATCHING, PROTECTED, 0);
+        set->modules = module;
+
+        if (probes[i].insert_loc.type == DR_PROBE_ADDR_LIB_OFFS) {
+            if (probes[i].insert_loc.lib_offs.library != NULL) {
+                module->sig.pe_name = dr_strdup(probes[i].insert_loc.lib_offs.library
+                                                HEAPACCT(ACCT_HOT_PATCHING));
+            } else {
+                probes[i].status = DR_PROBE_STATUS_INVALID_LIB;
+                goto dr_probe_parse_error;
+            }
+        } else if (probes[i].insert_loc.type == DR_PROBE_ADDR_EXP_FUNC) {
+            /* TODO: NYI - support for exported functions (PR 225654) */
+            probes[i].status = DR_PROBE_STATUS_UNSUPPORTED;
+        } else if (probes[i].insert_loc.type != DR_PROBE_ADDR_VIRTUAL) {
+            /* TODO: NYI - support for exported functions (PR 225654) */
+            probes[i].status = DR_PROBE_STATUS_UNSUPPORTED;
+            goto dr_probe_parse_error;
+        }
+
+        ppoint = HEAP_ARRAY_ALLOC_MEMSET(GLOBAL_DCONTEXT, hotp_patch_point_t, 1,
+                                         ACCT_HOT_PATCHING, PROTECTED, 0);
+        module->num_patch_points = 1;
+        module->patch_points = ppoint;
+
+        /* The actual patch address will be computed if the module matches. 
+         * vul_id holds the function name, which will be used to compute the
+         * offset.  See hotp_process_image().
+         */
+        /* TODO: validate probe & callback addr here if possible; PR 225663 */
+        ppoint->offset = probes[i].insert_loc.lib_offs.offset;
+        ppoint->detector_fn = 0;    /* No detector for probes. */
+        ppoint->protector_fn = probes[i].callback_func.lib_offs.offset;
+
+        /* Precedence hasn't been implemented yet, however, if it had been,
+         * then we don't want gbop hooks to interfere with client probes.
+         */
+#define HOTP_PROBE_PRECEDENCE (HOTP_ONLY_GBOP_PRECEDENCE - 1)   
+        ppoint->precedence = HOTP_PROBE_PRECEDENCE;
+
+        /* id generation should be the last step because parsing of a
+         * probe can be aborted before that and we don't want an id being
+         * returned for a probe that is rejected.
+         */
+        vul->id = GENERATE_PROBE_ID();
+
+        /* If we parsed a probe definition to this point then it is valid. */
+        valid_probes++;
+        continue;
+
+ dr_probe_parse_error:
+        /* Invalid probes are not kept inside dr, but discarded, so a 0 id
+         * should be returned for them
+         */
+        probes[i].id = 0;
+        hotp_free_one_vul(vul);
+    }
+
+    /* If there were some invalid probes then free extra memory in the initial
+     * table allocation.
+     */
+    if (valid_probes < num_probes) {
+        hotp_vul_t *old_tab = tab;
+        if (valid_probes > 0) {
+            tab = HEAP_ARRAY_ALLOC(GLOBAL_DCONTEXT, hotp_vul_t, valid_probes,
+                                   ACCT_HOT_PATCHING, PROTECTED); 
+            memcpy(tab, old_tab, sizeof(hotp_vul_t) * valid_probes);
+        } else {
+            tab = NULL;
+        }
+        HEAP_ARRAY_FREE(GLOBAL_DCONTEXT, old_tab, hotp_vul_t, num_probes,
+                        ACCT_HOT_PATCHING, PROTECTED);
+    }
+    hotp_load_hotp_dlls(tab, valid_probes);
+
+    /* Can't load dlls with hotp lock held - can deadlock if app is loading a
+     * dll too (see hotp nudge for details).  We solve this by setting up the
+     * hotp table in a temp var, doing the load on it and then grabbing the
+     * hotp lock and setting the global hotp table.  If we have our own loader
+     * (PR 209430) we won't need to do this.
+     */
+    write_lock(&hotp_vul_table_lock);
+
+    GLOBAL_VUL_TABLE = tab;
+    NUM_GLOBAL_VULS = valid_probes;
+
+    if (GLOBAL_VUL_TABLE != NULL) {
+        ASSERT(NUM_GLOBAL_VULS > 0);
+        /* Policy status table must be initialized after the global
+         * vulnerability table is setup, but before module list is iterated
+         * over because it uses the former and the latter will set status. 
+         */
+        hotp_init_policy_status_table();
+    }
+    write_unlock(&hotp_vul_table_lock);
+
+    /* Unlike hotp_init(), client init happens after vmareas_init(), i.e.,
+     * after module processing, so we have to walk the module list again.  It
+     * is ok to do the walk without the hotp lock because 
+     *  1. that is what is done between hotp_init() and vmareas_init() as no
+     *      change can happen to the hotp table at that time (nudges
+     *      are nop'ed during dr init).
+     *  2. dr_register_probes()'s execution at init time can't overlap with
+     *      another dr_register_probes() because
+     *      a. a second instance can't be called in dr_init() before the first
+     *          one returns.
+     *      b. if a second one is called via a custom nudge or from a callback 
+     *          it is nop'ed by this routine (at least for this release - for 
+     *          next release will have to figure the callback part); btw, nudges
+     *          during dr init are nop'ed anyway.
+     *  3. dr_register_probes()'s execution at init time also can't overlap 
+     *      with a liveshield nudge (even if we support them both 
+     *      simultaneously) because during dr init all nudges are nop'ed.
+     *
+     * NOTE: for probe/hot patch related nudges after dr init (whether custom
+     * nudge, liveshield nudge or internal nudge triggered by calling
+     * dr_register_probes() after init), loader walking has to be done with the
+     * hotp lock held otherwise two nudges can mess up each other (one common
+     * problem would be double injection/removal for hotp_only).  This is done
+     * in nudge_action_read_policies() and hotp_nudge_handler() for
+     * liveshields.
+     *
+     * For probes, nudge (custom or internally triggered) isn't supported today
+     * - a TODO.  When we do that this routine can't be shared as is for both
+     * probe registration at init and probe registration after init.
+     */
+    if (GLOBAL_VUL_TABLE != NULL) {
+        /* TODO: opt: if it is safe move client_init() between hotp_init() &
+         * vm_areas_init() then this loader-list-walk can be eliminated.
+         */
+        hotp_walk_loader_list(NULL, 0, NULL, true /* probe_init */);
+    }
+}
+
+/* TODO: currently no status is set, probe status & LS status codes needed to
+ * be merged, status code groups have to be defined (invalid, wating to be
+ * injected, etc.) so nothing is returned.  PR 225548.
+ */
+int
+dr_get_probe_status(unsigned int id, dr_probe_status_t *status)
+{
+    unsigned int i;
+    bool res = false;
+
+    /* For now, probes are supported iff probe api is explicity turned on.
+     * Also, liveshields shouldn't be on when probe api is on.
+     */
+    CLIENT_ASSERT(DYNAMO_OPTION(hot_patching) && DYNAMO_OPTION(probe_api) &&
+                  !DYNAMO_OPTION(liveshields), "To use Probe API, "
+                  "-hot_patching, -probe_api and -no_liveshields options "
+                  "should be used.");
+
+    if (!DYNAMO_OPTION(probe_api))
+        return res; /* be safe */
+
+    if (status == NULL)
+        return res;
+
+    *status = DR_PROBE_STATUS_INVALID_ID;
+    read_lock(&hotp_vul_table_lock);
+    for (i = 0; i < NUM_GLOBAL_VULS; i++) {
+        if (id == GLOBAL_VUL(i).id) {
+            *status = *(GLOBAL_VUL(i).info->inject_status);
+            res = true;
+            break;
+        }
+    }
+
+    read_unlock(&hotp_vul_table_lock);
+    return res;
+}
+#endif  /* CLIENT_INTERFACE */
+#endif  /* HOT_PATCHING_INTERFACE */
+
+/* Got hotp_read_policy_defs() working, so this can be used for testing now. */
+#ifdef HOT_PATCHING_INTERFACE_UNIT_TESTS
+static void 
+hotp_read_policies(void)
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+
+    /* TODO: once this function starts reading from file/memory, the data below
+     *       can be used as part of unit-hotpatch.c 
+     */
+    static hotp_patch_point_t pp1 = {(app_rva_t)0x673e, (app_rva_t) 0x1010, 
+                                     (app_rva_t) 0x1010, (app_rva_t) 0x6741, 0};
+    static hotp_module_t mod1 = {{"ci_loop_test.exe", 0, 0x4241c037, 0}, 
+                                 1, &pp1, false, NULL};
+    static hotp_set_t set1 = {1, &mod1};
+    static hotp_vul_info_t info1 = {0}, info2 = {0};
+
+    static hotp_patch_point_t pp2 = {(app_rva_t)0x440f, (app_rva_t) &hotp_nimda, 
+                                     (app_rva_t) &hotp_nimda, (app_rva_t) 0, 0};
+    static hotp_module_t mod2 = {{"iisrtl.dll", 0x20190, 0x384399bc, 0x21000},
+                                 1, &pp2, false, NULL};
+    static hotp_set_t set2 = {1, &mod2};
+
+    NUM_GLOBAL_VULS = 2;
+    hotp_vul_table = (hotp_vul_t*) heap_alloc(GLOBAL_DCONTEXT,
+                                              sizeof(hotp_vul_t) * NUM_GLOBAL_VULS
+                                              HEAPACCT(ACCT_HOT_PATCHING));
+
+    /* ci_loop_text.exe vulnerability */
+    GLOBAL_VUL(0).vul_id = "ci_loop_test-vul"; 
+    GLOBAL_VUL(0).policy_id = "ci_loop_test-policy"; 
+    GLOBAL_VUL(0).hotp_dll = "c:\\cygwin\\home\\bharath\\ci\\hotp_2.5.dll";
+    GLOBAL_VUL(0).hotp_dll_hash = NULL;
+    GLOBAL_VUL(0).hotp_dll_base = NULL;     /* Runtime data! just for now */
+    GLOBAL_VUL(0).mode = HOTP_MODE_OFF;
+    GLOBAL_VUL(0).num_sets = 1;
+    GLOBAL_VUL(0).sets = &set1;
+    GLOBAL_VUL(0).info = &info1;
+
+    /* nimda vulnerability */
+    GLOBAL_VUL(1).vul_id = "nimda-vul"; 
+    GLOBAL_VUL(1).policy_id = "nimda-policy"; 
+    GLOBAL_VUL(1).hotp_dll = "hotp_2_5.dll";
+    GLOBAL_VUL(1).hotp_dll_hash = NULL;
+    GLOBAL_VUL(0).hotp_dll_base = NULL;     /* Runtime data! just for now */
+    GLOBAL_VUL(1).mode = HOTP_MODE_OFF;
+    GLOBAL_VUL(1).num_sets = 1;
+    GLOBAL_VUL(1).sets = &set2;
+    GLOBAL_VUL(1).info = &info2;
+}
+#endif  /* HOT_PATCHING_INTERFACE_UNIT_TESTS */
diff --git a/core/hotpatch.h b/core/hotpatch.h
new file mode 100644
index 000000000..d6efb8d30
--- /dev/null
+++ b/core/hotpatch.h
@@ -0,0 +1,150 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2005-2007 Determina Corp. */
+
+/*
+ * hotpatch.h - Exported interfaces from the hot patching module
+ */
+
+#ifndef _HOTPATCH_H_
+#define _HOTPATCH_H_ 1
+
+#ifdef HOT_PATCHING_INTERFACE  /* Around the whole file */
+
+/*----------------------------------------------------------------------------*/
+/* Exported data types */
+
+/* This type identifies a patch point that was matched during offset lookup. 
+ * It describes the module, set and vulnerability the matching patch point
+ * belongs to. 
+ */
+typedef struct {
+    /* Array index of the matching vulnerability in hotp_vul_table. */
+    int vul_index;
+    
+    /* Array index of the matching set in hotp_vul_table[vul_index]. */
+    int set_index;
+    int module_index;  /* Matching module index in the matching set. */
+    int ppoint_index;  /* Matching patch point index in the matching module. */
+} hotp_offset_match_t;
+
+/*----------------------------------------------------------------------------*/
+/* Exported data */
+typedef enum {
+    /* Definitions of context types that are to be dumped into forensics files.
+     * Used by hotp, hotp_only & gbop to dump the right app context; case 8099.
+     * This is to overcome the limitations in using our current interface
+     * to report violations; planned to be fixed for Marlin - case 8079.
+     */
+    CXT_TYPE_HOT_PATCH,
+    CXT_TYPE_CORE_HOOK
+} cxt_type_t;
+
+/* Leak to handle case 9593.  This should go if we find a cleaner solution. */
+#ifdef HEAP_ACCOUNTING
+DEBUG_DECLARE(extern int hotp_only_tramp_bytes_leaked;)
+#endif
+#ifdef DEBUG_MEMORY
+DEBUG_DECLARE(bool hotp_only_contains_leaked_trampoline(byte *pc, size_t size);)
+#endif
+/*----------------------------------------------------------------------------*/
+/* Exported function prototypes */
+bool hotp_does_region_need_patch(const app_pc start, const app_pc end,
+                                 bool own_hot_patch_lock);
+bool hotp_inject(dcontext_t *dcontext, instrlist_t *ilist);
+
+/* hotp_process_image() can be called in two different ways, one to process
+ * the image and the other one to just check it (bool just_check).  just_check
+ * is used only in hotp_only mode to find out if a dll being loaded needs
+ * patching.
+ * FIXME: have another wrapper for just_check case; keeps it clean.
+ */
+void hotp_process_image(const app_pc base, const bool loaded, 
+                        const bool own_hot_patch_lock, const bool just_check,
+                        bool *needs_processing, const thread_record_t **all_threads, 
+                        const int num_threads);
+
+bool
+hotp_ppoint_on_list(app_rva_t ppoint,
+                    app_rva_t *hotp_ppoint_vec, uint hotp_ppoint_vec_num);
+
+/* Returns the number of patch points for the matched vuls in [start,end).
+ * For now this routine assumes that [start,end) is contained in a single module.
+ * The caller must own the hotp_vul_table_lock (as a read lock).
+ */
+int
+hotp_num_matched_patch_points(const app_pc start, const app_pc end);
+
+/* Stores in vec the offsets for all the matched patch points in [start,end).
+ * Returns -1 if vec_num is too small (still fills it up).
+ * For now this routine assumes that [start,end) is contained in a single module.
+ * The caller must own the hotp_vul_table_lock (as a read lock).
+ */
+int
+hotp_get_matched_patch_points(const app_pc start, const app_pc end,
+                              app_rva_t *vec, uint vec_num);
+
+/* Checks whether any matched patch point in [start, end) is not listed on
+ * hotp_ppoint_vec.  If hotp_ppoint_vec is NULL just checks whether any patch
+ * point is matched in the region.  For now this routine assumes that
+ * [start,end) is contained in a single module.
+ */
+bool
+hotp_point_not_on_list(const app_pc start, const app_pc end, bool own_hot_patch_lock,
+                       app_rva_t *hotp_ppoint_vec, uint hotp_ppoint_vec_num);
+
+void hotp_nudge_handler(uint nudge_action_mask);
+void hotp_init(void);
+void hotp_exit(void);
+void hotp_reset_init(void);
+void hotp_reset_free(void);
+read_write_lock_t *hotp_get_lock(void);
+void hotp_print_diagnostics(file_t diagnostics_file);
+bool hotp_only_in_tramp(const app_pc eip);
+void hotp_only_detach_helper(void);
+void hotp_only_mem_prot_change(const app_pc start, const size_t size, 
+                               const bool remove, const bool inject);
+void hotp_spill_before_notify(dcontext_t *dcontext,
+                              fragment_t **frag_spill /* OUT */,
+                              fragment_t *new_frag, const app_pc new_frag_tag,
+                              app_pc *new_tag_spill /* OUT */, 
+                              const app_pc new_tag,
+                              dr_mcontext_t *cxt_spill /* OUT */,
+                              const void *new_cxt, cxt_type_t cxt_type);
+void hotp_restore_after_notify(dcontext_t *dcontext,
+                               const fragment_t *old_frag, 
+                               const app_pc old_next_tag,
+                               const dr_mcontext_t *old_cxt);
+
+#endif /* HOT_PATCHING_INTERFACE  Around the whole file */
+#endif /* _HOTPATCH_H_ 1 */
diff --git a/core/instrlist.c b/core/instrlist.c
new file mode 100644
index 000000000..ad82604d7
--- /dev/null
+++ b/core/instrlist.c
@@ -0,0 +1,387 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/* file "instr_list.c" */
+
+#include "globals.h"
+#include "instrlist.h"
+#include "instr.h"
+#include <string.h>
+
+#ifdef DEBUG
+/* case 10450: give messages to clients */
+# undef ASSERT /* N.B.: if have issues w/ DYNAMO_OPTION, re-instate */
+# undef ASSERT_TRUNCATE
+# undef ASSERT_BITFIELD_TRUNCATE
+# undef ASSERT_NOT_REACHED
+# define ASSERT DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_TRUNCATE DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_BITFIELD_TRUNCATE DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_NOT_REACHED DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+#endif
+
+/* returns an empty instrlist_t object */
+instrlist_t*
+instrlist_create(dcontext_t *dcontext)
+{
+    instrlist_t *ilist = (instrlist_t*) heap_alloc(dcontext, sizeof(instrlist_t)
+                                               HEAPACCT(ACCT_IR));
+    CLIENT_ASSERT(ilist != NULL, "instrlist_create: allocation error");
+    instrlist_init(ilist);
+    return ilist;
+}
+
+/* initializes an instrlist_t object */
+void
+instrlist_init(instrlist_t *ilist)
+{
+    CLIENT_ASSERT(ilist != NULL, "instrlist_create: NULL parameter");
+    ilist->first = ilist->last = NULL;
+    ilist->flags = 0;   /* no flags set */
+    ilist->translation_target = NULL;
+}
+
+/* frees the instrlist_t object */
+void
+instrlist_destroy(dcontext_t *dcontext, instrlist_t *ilist)
+{
+    CLIENT_ASSERT(ilist->first == NULL && ilist->last == NULL,
+                  "instrlist_destroy: list not empty");
+    heap_free(dcontext, ilist, sizeof(instrlist_t) HEAPACCT(ACCT_IR));
+}
+
+/* frees the Instrs in the instrlist_t */
+void
+instrlist_clear(dcontext_t *dcontext, instrlist_t *ilist)
+{
+    instr_t *instr;
+    while (NULL != (instr = instrlist_first(ilist))) {
+        instrlist_remove(ilist, instr);
+        instr_destroy(dcontext, instr);
+    }
+}
+
+/* frees the Instrs in the instrlist_t and the instrlist_t object itself */
+void
+instrlist_clear_and_destroy(dcontext_t *dcontext, instrlist_t *ilist)
+{
+    instrlist_clear(dcontext,ilist);
+    instrlist_destroy(dcontext,ilist);
+}
+
+/* All future Instrs inserted into ilist that do not have raw bits
+ * will have instr_set_translation called with pc as the target
+ */
+void
+instrlist_set_translation_target(instrlist_t *ilist, app_pc pc)
+{
+    ilist->translation_target = pc;
+}
+
+app_pc
+instrlist_get_translation_target(instrlist_t *ilist)
+{
+    return ilist->translation_target;
+}
+
+void
+instrlist_set_our_mangling(instrlist_t *ilist, bool ours)
+{
+    if (ours)
+        ilist->flags |= INSTR_OUR_MANGLING;
+    else
+        ilist->flags &= ~INSTR_OUR_MANGLING;
+}
+
+bool
+instrlist_get_our_mangling(instrlist_t *ilist)
+{
+    return TEST(INSTR_OUR_MANGLING, ilist->flags);
+}
+
+/* returns the first inst in the list */
+instr_t*
+instrlist_first(instrlist_t *ilist)
+{
+    return ilist->first;
+}
+
+/* returns the last inst in the list */
+instr_t*
+instrlist_last(instrlist_t *ilist)
+{
+    return ilist->last;
+}
+
+static inline void
+check_translation(instrlist_t *ilist, instr_t *inst)
+{
+    if (ilist->translation_target != NULL && instr_get_translation(inst) == NULL) {
+        instr_set_translation(inst, ilist->translation_target);
+    }
+    if (instrlist_get_our_mangling(ilist))
+        instr_set_our_mangling(inst, true);
+}
+
+/* appends inst to the list ("inst" can be a chain of insts) */
+void
+instrlist_append(instrlist_t *ilist, instr_t *inst)
+{
+    instr_t *top = inst;
+    instr_t *bot;
+
+    CLIENT_ASSERT(instr_get_prev(inst) == NULL,
+                  "instrlist_append: cannot add middle of list");
+    check_translation(ilist, inst);
+    while (instr_get_next(inst)) {
+        inst = instr_get_next(inst);
+        check_translation(ilist, inst);
+    }
+    bot = inst;
+    if (ilist->last) {
+        instr_set_next(ilist->last, top);
+        instr_set_prev(top, ilist->last);
+        ilist->last = bot;
+    } 
+    else {
+        ilist->first = top;
+        ilist->last = bot;
+    }
+}
+
+/* prepends inst to the list ("inst" can be a chain of insts) */
+void
+instrlist_prepend(instrlist_t *ilist, instr_t *inst)
+{
+    instr_t *top = inst;
+    instr_t *bot;
+
+    CLIENT_ASSERT(instr_get_prev(inst) == NULL,
+                  "instrlist_prepend: cannot add middle of list");
+    check_translation(ilist, inst);
+    while (instr_get_next(inst)) {
+        inst = instr_get_next(inst);
+        check_translation(ilist, inst);
+    }
+    bot = inst;
+    if (ilist->first) {
+        instr_set_next(bot, ilist->first);
+        instr_set_prev(ilist->first, bot);
+        ilist->first = top;
+    } 
+    else {
+        ilist->first = top;
+        ilist->last = bot;
+    }
+}
+
+
+/* inserts "inst" before "where" ("inst" can be a chain of insts) */
+void
+instrlist_preinsert(instrlist_t *ilist, instr_t *where, instr_t *inst)
+{
+    instr_t *whereprev;
+    instr_t *top = inst;
+    instr_t *bot;
+
+    if (where == NULL) {
+        /* if where is NULL there is no inst to send for a "before" */
+        instrlist_append(ilist, inst);
+        return;
+    }
+
+    CLIENT_ASSERT(where != NULL, "instrlist_preinsert: where cannot be NULL");
+    CLIENT_ASSERT(instr_get_prev(inst) == NULL,
+                  "instrlist_preinsert: cannot add middle of list");
+    whereprev = instr_get_prev(where);
+    check_translation(ilist, inst);
+    while (instr_get_next(inst)) {
+        inst = instr_get_next(inst);
+        check_translation(ilist, inst);
+    }
+    bot = inst;
+    if (whereprev) {
+        instr_set_next(whereprev, top);
+        instr_set_prev(top, whereprev);
+    }
+    else {
+        ilist->first = top;
+    }
+    instr_set_next(bot, where);
+    instr_set_prev(where, bot);
+}
+
+/* inserts "inst" after "where" ("inst" can be a chain of insts) */
+void
+instrlist_postinsert(instrlist_t *ilist, instr_t *where, instr_t *inst)
+{
+    instr_t *wherenext;
+    instr_t *top = inst;
+    instr_t *bot;
+
+    if (where == NULL) {
+        /* if where is NULL there is no inst to send for an "after" */
+        instrlist_prepend(ilist, inst);
+        return;
+    }
+
+    CLIENT_ASSERT(where != NULL, "instrlist_postinsert: where cannot be NULL");
+    CLIENT_ASSERT(instr_get_prev(inst) == NULL,
+                  "instrlist_postinsert: cannot add middle of list");
+
+    wherenext = instr_get_next(where);
+    check_translation(ilist, inst);
+    while (instr_get_next(inst)) {
+        inst = instr_get_next(inst);
+        check_translation(ilist, inst);
+    }
+    bot = inst;
+    instr_set_next(where, top);
+    instr_set_prev(top, where);
+    if (wherenext) {
+        instr_set_next(bot, wherenext);
+        instr_set_prev(wherenext, bot);
+    }
+    else {
+        ilist->last = bot;
+    }
+}
+
+/* replace oldinst with newinst, remove oldinst from ilist, and return oldinst 
+   (newinst can be a chain of insts) */
+instr_t*
+instrlist_replace(instrlist_t *ilist, instr_t *oldinst, instr_t *newinst)
+{
+    instr_t *where;
+
+    CLIENT_ASSERT(oldinst != NULL, "instrlist_replace: oldinst cannot be NULL");
+    CLIENT_ASSERT(instr_get_prev(newinst) == NULL,
+                  "instrlist_replace: cannot add middle of list");
+    where = instr_get_prev(oldinst);
+    instrlist_remove(ilist, oldinst);
+    if (where)
+        instrlist_postinsert(ilist, where, newinst);
+    else
+        instrlist_prepend(ilist, newinst);
+
+    return oldinst;
+}
+
+/* removes "inst" from the instrlist_t it currently belongs to */
+void
+instrlist_remove(instrlist_t *ilist, instr_t *inst)
+{
+    if (instr_get_prev(inst))
+        instr_set_next(instr_get_prev(inst), instr_get_next(inst));
+    else
+        ilist->first = instr_get_next(inst);
+
+    if (instr_get_next(inst))
+        instr_set_prev(instr_get_next(inst), instr_get_prev(inst));
+    else
+        ilist->last = instr_get_prev(inst);
+
+    instr_set_prev(inst, NULL);
+    instr_set_next(inst, NULL);
+}
+
+instrlist_t*
+instrlist_clone(dcontext_t *dcontext, instrlist_t *old)
+{
+    instr_t *inst, *copy;
+    instrlist_t *newlist = instrlist_create(dcontext);
+
+    inst = instrlist_first(old);
+    while (inst != NULL) {
+        copy = instr_clone(dcontext, inst);
+        /* to copy instr targets we temporarily clobber note field */
+        instr_set_note(inst, (void *)copy);
+        instrlist_append(newlist, copy);
+        inst = instr_get_next(inst);
+    }
+
+    /* Fix up instr targets and restore note field */
+    for (inst = instrlist_first(old), copy = instrlist_first(newlist);
+         inst != NULL && copy != NULL;
+         inst = instr_get_next(inst), copy = instr_get_next(copy)) {
+        if (instr_is_cti(copy) && opnd_is_instr(instr_get_target(copy))) {
+            opnd_t op = instr_get_target(copy);
+            instr_t *tgt;
+            CLIENT_ASSERT(opnd_get_instr(op) != NULL,
+                          "instrlist_clone: NULL instr target");
+            /* would be nice to assert the instr tgt is in this instrlist */
+            tgt = (instr_t *) instr_get_note(opnd_get_instr(op));
+            if (opnd_is_far_instr(op)) {
+                instr_set_target(copy, opnd_create_far_instr
+                                 (opnd_get_segment_selector(op), tgt));
+            } else
+                instr_set_target(copy, opnd_create_instr(tgt));
+        }
+        /* restore note field */
+        instr_set_note(inst, instr_get_note(copy));
+    }
+
+    return newlist;
+}
+
+
+/*
+ * puts a whole list (prependee) onto the front of ilist
+ * frees prependee when done because it will contain nothing useful
+ */
+  
+void
+instrlist_prepend_instrlist(dcontext_t *dcontext,instrlist_t *ilist,
+                            instrlist_t *prependee)
+{
+    instr_t *first=instrlist_first(prependee);
+    if (!first)
+        return;
+    instrlist_prepend(ilist,first);
+    instrlist_init(prependee);
+    instrlist_destroy(dcontext,prependee);
+}
+
+void instrlist_append_instrlist(dcontext_t *dcontext,instrlist_t *ilist,
+                                instrlist_t *appendee)
+{
+    instr_t *first=instrlist_first(appendee);
+    if (!first)
+        return;
+    instrlist_append(ilist,first);
+    instrlist_init(appendee);
+    instrlist_destroy(dcontext,appendee);
+}
diff --git a/core/instrlist.h b/core/instrlist.h
new file mode 100644
index 000000000..7de33f99a
--- /dev/null
+++ b/core/instrlist.h
@@ -0,0 +1,188 @@
+/* **********************************************************
+ * Copyright (c) 2000-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/* file "instrlist.h" */
+
+#ifndef _INSTRLIST_H_
+#define _INSTRLIST_H_ 1
+
+struct _instr_list_t {
+    instr_t *first;
+    instr_t *last;
+    int flags;
+    app_pc translation_target;
+}; /* instrlist_t */
+
+/* DR_API EXPORT TOFILE dr_ir_instrlist.h */
+/* DR_API EXPORT BEGIN */
+/****************************************************************************
+ * INSTRLIST ROUTINES
+ */
+/**
+ * @file dr_ir_instrlist.h
+ * @brief Functions to create and manipulate lists of instructions.
+ */
+/* DR_API EXPORT END */
+
+/* basic instrlist_t functions */
+
+DR_API
+/** Returns an initialized instrlist_t allocated on the thread-local heap. */
+instrlist_t*
+instrlist_create(dcontext_t *dcontext);
+
+DR_API
+/** Initializes \p ilist. */
+void 
+instrlist_init(instrlist_t *ilist);
+
+DR_API
+/** Deallocates the thread-local heap storage for \p ilist. */
+void 
+instrlist_destroy(dcontext_t *dcontext, instrlist_t *ilist);
+
+DR_API
+/** Frees the instructions in \p ilist. */
+void
+instrlist_clear(dcontext_t *dcontext, instrlist_t *ilist);
+
+DR_API
+/** Destroys the instructions in \p ilist and destroys the instrlist_t object itself. */
+void
+instrlist_clear_and_destroy(dcontext_t *dcontext, instrlist_t *ilist);
+
+DR_API
+/** 
+ * All future instructions inserted into \p ilist that do not have raw bits
+ * will have instr_set_translation() called with \p pc as the target.
+ * This is a convenience routine to make it easy to have the same
+ * code generate non-translation and translation instructions, and it does
+ * not try to enforce that all instructions have translations (e.g.,
+ * some could be inserted via instr_set_next()).
+ */
+void
+instrlist_set_translation_target(instrlist_t *ilist, app_pc pc);
+
+DR_API
+/** Returns the translation target, or NULL if none is set. */
+app_pc
+instrlist_get_translation_target(instrlist_t *ilist);
+
+/* not exported: for PR 267260 */
+void
+instrlist_set_our_mangling(instrlist_t *ilist, bool ours);
+
+bool
+instrlist_get_our_mangling(instrlist_t *ilist);
+
+DR_API
+/** Returns the first instr_t in \p ilist. */
+instr_t*
+instrlist_first(instrlist_t *ilist);
+
+DR_API
+/** Returns the last instr_t in \p ilist. */
+instr_t*
+instrlist_last(instrlist_t *ilist);
+
+DR_API
+/** Adds \p instr to the end of \p ilist. */
+void 
+instrlist_append(instrlist_t *ilist, instr_t *instr);
+
+DR_API
+/** Adds \p instr to the front of \p ilist. */
+void 
+instrlist_prepend(instrlist_t *ilist, instr_t *instr);
+
+
+DR_API
+/** 
+ * Allocates a new instrlist_t and for each instr_t in \p old allocates
+ * a new instr_t using instr_clone to produce a complete copy of \p old.
+ * Each operand that is opnd_is_instr() has its target updated
+ * to point to the corresponding instr_t in the new instrlist_t
+ * (this routine assumes that all such targets are contained within \p old,
+ * and may fault otherwise).
+ */
+instrlist_t*
+instrlist_clone(dcontext_t *dcontext, instrlist_t *old);
+
+
+/* Adds every instr_t in prependee to the front of ilist (maintaining
+ * the original order).
+ * Then calls instrlist_destroy on prependee.
+ * FIXME: get rid of this?
+ */
+void 
+instrlist_prepend_instrlist(dcontext_t *dcontext, instrlist_t *ilist,
+                            instrlist_t *prependee);
+
+/* Adds every instr_t in appendee to the end of ilist (maintaining
+ * the original order).
+ * Then calls instrlist_destroy on appendee.
+ * FIXME: get rid of this?
+ */
+void 
+instrlist_append_instrlist(dcontext_t *dcontext, instrlist_t *ilist,
+                           instrlist_t *appendee);
+
+/* instrlist_t functions focusing on a paricular instr_t
+ * For insert_before and insert_after, if the instr is NULL, either 
+ * instrlist_append or instrlist_prepend will be called
+ */
+
+DR_API
+/** Inserts \p instr into \p ilist prior to \p where. */
+void   
+instrlist_preinsert(instrlist_t *ilist, instr_t *where, instr_t *instr);
+
+DR_API
+/** Inserts \p instr into \p ilist after \p where. */
+void   
+instrlist_postinsert(instrlist_t *ilist, instr_t *where, instr_t *instr);
+
+DR_API
+/** Replaces \p oldinst with \p newinst in \p ilist (does not destroy \p oldinst). */
+instr_t*
+instrlist_replace(instrlist_t *ilist, instr_t *oldinst, instr_t *newinst);
+
+DR_API
+/** Removes (does not destroy) \p instr from \p ilist. */
+void   
+instrlist_remove(instrlist_t *ilist, instr_t *instr);
+
+#endif /* _INSTRLIST_H_ */
diff --git a/core/io.c b/core/io.c
new file mode 100644
index 000000000..76b4e59e1
--- /dev/null
+++ b/core/io.c
@@ -0,0 +1,618 @@
+/* **********************************************************
+ * Copyright (c) 2002-2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+
+/*
+ * io.c - Linux specific routines for i/o that will not
+ * trigger problematic pthread calls.
+ * For PR 219380 we also export this routine to clients on Windows.
+ */
+
+/* FIXME: failure modes should be more gracefull then failing asserts in most places */
+
+#include "globals.h"
+#include <string.h>
+#include <stdarg.h> /* for varargs */
+
+#define VA_ARG_CHAR2INT
+#define BUF_SIZE 64
+
+const static char base_letters[] = {
+    '0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'
+};
+const static char base_letters_cap[] = {
+    '0','1','2','3','4','5','6','7','8','9','A','B','C','D','E','F'
+};
+
+#ifdef LINUX
+const static double pos_inf = 1.0/0.0;
+const static double neg_inf = -1.0/0.0;
+#else
+/* Windows says "error C2099: initializer is not a constant", or
+ * "error C2124: divide or mod by zero", for the above.
+ */
+# pragma warning(disable : 4723) /* warning C4723: potential divide by 0 */
+const static double zerof = 0.0;
+# define pos_inf (1.0/zerof)
+# define neg_inf (-1.0/zerof)
+#endif
+
+/* convert uint64 to a string */
+static char *
+uint64_to_str(uint64 num, int base, char * buf, int decimal, bool caps)
+{
+    int    cnt;
+    char   *p = buf;
+    int    end = (43 > decimal ? 43 : decimal);
+    ASSERT(decimal < BUF_SIZE - 1); /* so don't overflow buf */
+
+    buf[end] = '\0';
+    for (cnt = end - 1; cnt >= 0; cnt--) {
+        buf[cnt] = caps ? base_letters_cap[(num %base)] : base_letters[(num % base)];
+        num /= base;
+    }
+
+    while (*p && *p == '0' && end - decimal > 0) {
+        p++;
+        decimal++;
+    }
+    
+    return p;
+}
+
+/* convert ulong to a string */
+static char *
+ulong_to_str(ulong num, int base, char *buf, int decimal, bool caps)
+{
+    int    cnt;
+    char   *p = buf;
+    int    end = (22 > decimal ? 22 : decimal); /* room for 64 bits octal */
+    ASSERT(decimal < BUF_SIZE - 1); /* so don't overflow buf */
+
+    buf[end] = '\0';
+    for (cnt = end - 1; cnt >= 0; cnt--) {
+        buf[cnt] = caps ? base_letters_cap[(num %base)] : base_letters[(num % base)];
+        num /= base;
+    }
+
+    while (*p && *p == '0' && end - decimal> 0) {
+        p++;
+        decimal++;
+    }
+
+    return p;
+}
+
+/* assumes that d > 0 */
+static long
+double2int(double d)
+{
+    long i = (long)d;
+    if ((d - (double)i) >= 0.5)
+        return i+1;
+    else
+        return i;
+}
+
+static char *
+double_to_str(double d, int decimal, char *buf, bool force_dot, bool suppress_zeros)
+{
+    /* support really big numbers with %f? */
+    char tmpbuf[BUF_SIZE];
+    char *pre, *post, *c;
+    long predot, postdot, sub, i;
+
+    /* get pre and post dot sections as integers */
+    if (d < 0)
+        d = -d;
+    if (decimal > 0)
+        predot = (long)d;
+    else
+        predot = double2int(d);
+    sub = 1;
+    for (i=0; i<decimal; i++)
+        sub *= 10;
+    postdot = double2int((d - (long)d) * (double)sub);
+    if (postdot == sub) {
+        /* we had a .9* that rounded up! */
+        postdot = 0;
+        predot++;
+    }
+
+    pre = ulong_to_str((ulong)predot, 10, tmpbuf, 1, false);
+    for (i=0, c = pre; *c; c++)
+        buf[i++] = *c;
+    if (force_dot || !(decimal == 0 || (suppress_zeros && postdot == 0))) {
+        buf[i++] = '.';
+        post = ulong_to_str((ulong)postdot, 10, tmpbuf, decimal, false);
+        for (c = post; *c; c++)
+            buf[i++] = *c;
+        /* remove trailing zeros */
+        if (suppress_zeros) {
+            while (buf[i-1] == '0')
+                i--; 
+        }
+    }
+
+    buf[i] = '\0';
+    ASSERT(i<BUF_SIZE);  /* make sure don't overflow buffer */
+    return buf;
+}
+
+static char *
+double_to_exp_str(double d, int exp, int decimal, char * buf, bool force_dot, bool suppress_zeros, bool caps)
+{
+    char tmp_buf[BUF_SIZE];
+    char *tc;
+    int i = 0;
+    uint abval;
+    
+    tc = double_to_str(d, decimal, tmp_buf, force_dot, suppress_zeros);
+    while (*tc) {
+        buf[i++] = *tc++;
+    }
+    if (caps)
+        buf[i++] = 'E';
+    else
+        buf[i++] = 'e';
+    if (exp < 0) {
+        buf[i++] = '-';
+        abval = -exp;
+    } else {
+        buf[i++] = '+';
+        abval = exp;
+    }
+    /* exp value always printed as at least 2 characters */
+    tc = ulong_to_str((ulong)abval, 10, tmp_buf, 2, false);
+    while (*tc) {
+        buf[i++] = *tc++;
+    }
+    buf[i] = '\0';
+    ASSERT(i < BUF_SIZE); /* make sure don't overflow buffer */
+    return buf;
+}
+
+
+int
+our_vsnprintf(char *s, size_t max, const char *fmt, va_list ap)
+{
+    const char *c;
+    char     *str = NULL;
+    char     *start = s;
+    char     buf[BUF_SIZE];
+    
+    if (fmt == NULL)
+        return 0;
+    
+    c = fmt;
+    while (*c) {
+        if (*c == '%') {
+            int  fill = 0;
+            char filler = ' ';
+            int decimal = -1; /* codes defaults (6 int, 1 float, all string) */
+            char charbuf[2] = {'\0','\0'};
+            char prefixbuf[3] = {'\0','\0','\0'};
+            char *prefix;
+            bool minus_flag = false;
+            bool plus_flag = false;
+            bool pound_flag = false;
+            bool space_flag = false;
+            bool h_type = false;
+            bool l_type = false;
+            bool ll_type = false;
+            prefix = prefixbuf;
+            c++;
+            ASSERT(*c);
+
+            /* Collect flags -, +, #, 0,  */
+            while (*c == '0' || *c == '-' || *c == '#' || *c == '+' || *c == ' ') {
+                if (*c == '0') 
+                    filler = '0';
+                if (*c == '-') 
+                    minus_flag = true;
+                if (*c == '+') 
+                    plus_flag = true;
+                if (*c == '#') 
+                    pound_flag = true;
+                if (*c == ' ') 
+                    space_flag = true;
+                c++;
+                ASSERT(*c);
+            }
+            if (minus_flag) 
+                filler = ' ';
+            if (plus_flag) 
+                space_flag = false;
+        
+            /* get field width */
+            if (*c == '*') {
+                fill = va_arg(ap, int);
+                if (fill < 0) {
+                    minus_flag = true;
+                    fill = -fill;
+                }
+                c++;
+                ASSERT(*c);
+            } else {
+                while (*c >= '0' && *c <= '9') {
+                    fill *= 10;
+                    fill += *c - '0';
+                    c++;
+                    ASSERT(*c);
+                }
+            }
+
+            /* get precision */
+            if (*c == '.') {
+                c++;
+                ASSERT(*c);
+                decimal = 0;
+                if (*c == '*') {
+                    decimal = va_arg(ap, int);
+                    c++;
+                    ASSERT(*c);
+                } else {
+                    while (*c >= '0' && *c <= '9') {
+                        decimal *= 10;
+                        decimal += *c - '0';
+                        c++;
+                        ASSERT(*c);
+                    }
+                }
+            }
+
+            /* get size modifiers l, h, ll/L */
+            if (*c == 'l' || *c == 'L' || *c == 'h') {
+                if (*c == 'L') 
+                    ll_type = true;
+                if (*c == 'h') 
+                    h_type = true;
+                if (*c == 'l') {
+                    c++;
+                    ASSERT(*c);
+                    if (*c == 'l') {
+                        ll_type = true;
+                        c++;
+                        ASSERT(*c);
+                    } else {
+                        l_type = true;
+                    }
+                } else {
+                    c++;
+                    ASSERT(*c);
+                }
+            }
+
+            /* dispatch */
+            switch (*c) {
+            case '%':
+                charbuf[0] = '%';
+                str = charbuf;
+                break;
+            case 'd':
+            case 'i':
+                {
+                    long val;
+                    ulong abval;
+                    if (decimal == -1) 
+                        decimal = 1;  /* defaults */
+                    else
+                        filler = ' ';
+                    if (l_type) 
+                        val = va_arg(ap, long);  /* get arg */
+                    else if (h_type) 
+                        val = (long)va_arg(ap, int); /* short is promoted to int */
+                    else
+                        val = (long)va_arg(ap, int);
+                    if (val >= 0 && space_flag) 
+                        prefixbuf[0] = ' ';  /* set prefix */
+                    if (val >= 0 && plus_flag) 
+                        prefixbuf[0] = '+';
+                    if (val < 0) {
+                        prefixbuf[0] = '-';
+                        abval = -val;
+                    } else {
+                        abval = val;
+                    }
+                    str = ulong_to_str(abval, 10, buf, decimal, false); /* generate string */
+                    break;
+                }
+            case 'I': 
+                { /* %I64, to match Win32 */
+                    uint64 val;
+                    /* must be %I64u */
+                    ASSERT(*(c+1)=='6' && *(c+2)=='4' && *(c+3)=='u');
+                    if (decimal == -1)
+                        decimal = 1;
+                    else
+                        filler = ' ';
+                    c += 3;
+                    val = (uint64)va_arg(ap, uint64);
+                    str = uint64_to_str(val, 10, buf, decimal, false);
+                    break;
+                }
+            case 'u':
+                /* handle long long u type */
+                if (decimal == -1)
+                    decimal = 1;
+                else
+                    filler = ' ';
+                if (ll_type) {
+                    str = uint64_to_str((uint64)va_arg(ap, uint64), 10, buf, decimal, false);
+                    break;
+                }
+                /* note no break */
+            case 'x':
+            case 'X':
+            case 'o':
+            case 'p':
+                {
+                    ptr_uint_t val;
+                    bool caps = *c == 'X';
+                    int base = 10;
+                    if (decimal == -1)
+                        decimal = 1; /* defaults */
+                    else
+                        filler = ' ';
+                    if (*c == 'p')
+                        decimal = 2 * sizeof(void *); /* pointer precision */
+                    if ((pound_flag && *c != 'u') || (*c == 'p')) {  /* generate prefix */
+                        prefixbuf[0] = '0';
+                        if (*c == 'x' || *c == 'p')
+                            prefixbuf[1] = 'x';
+                        if (*c == 'X')
+                            prefixbuf[1] = 'X';
+                    }
+                    ASSERT(sizeof(void *) == sizeof(val));
+                    if (*c == 'p')
+                        val = (ptr_uint_t) va_arg(ap, void *);  /* get val */
+                    else if (l_type)
+                        val = (ptr_uint_t) va_arg(ap, ulong);
+                    else if (h_type)
+                        val = (ptr_uint_t) va_arg(ap, uint); /* ushort promoted */
+                    else
+                        val = (ptr_uint_t) va_arg(ap, uint);
+                    if (*c == 'o')
+                        base = 8;    /* determine base */
+                    if (*c == 'x' || *c == 'X' || *c == 'p')
+                        base = 16;
+                    ASSERT(sizeof(val) == sizeof(ulong));
+                    str = ulong_to_str((ulong)val, base, buf, decimal, caps);  /* generate string */
+                    break;
+                }
+            case 'c':
+                /* FIXME: using int instead of char seems to work for RH7.2 as
+                 * well as 8.0, but using char crashes 8.0 but not 7.2
+                 */
+#ifdef VA_ARG_CHAR2INT
+                charbuf[0] = (char) va_arg(ap, int); /* char -> int in va_list */
+#else
+                charbuf[0] = va_arg(ap, char);
+#endif
+                str = charbuf;
+                break;
+            case 's':
+                str = va_arg(ap, char*);
+                break;
+            case 'g':
+            case 'G':
+                if (decimal == 0 || decimal == -1)
+                    decimal = 1; /* default */
+                /* no break */
+            case 'e':
+            case 'E':
+            case 'f':
+                {
+                    bool caps = (*c == 'E') || (*c == 'G');
+                    /* pretty sure will always be promoted to a double in arg list */
+                    double val = va_arg(ap, double);
+                    double d = val;
+                    int exp = 0;
+                    bool is_g = (*c == 'g' || *c == 'G'); 
+                    /* check for NaN */
+                    if (val != val) {
+                        if (caps)
+                            str = "NAN";
+                        else
+                            str = "nan";
+                        if (space_flag)
+                            prefixbuf[0] = ' ';
+                        break;
+                    }
+                    if (decimal == -1)
+                        decimal = 6; /* default */
+                    if (val >= 0 && space_flag)
+                        prefixbuf[0] = ' '; /* get prefix */
+                    if (val >= 0 && plus_flag)
+                        prefixbuf[0] = '+';
+                    if (val < 0)
+                        prefixbuf[0] = '-';
+                    /* check for inf */
+                    if (val == pos_inf || val == neg_inf) {
+                        if (caps)
+                            str = "INF";
+                        else
+                            str = "inf";
+                        break;
+                    }
+                    if (*c == 'f') { /* ready to generate string now for f */
+                        str = double_to_str(val, decimal, buf, pound_flag, false);
+                        break;
+                    }
+                    /* get exponent value */
+                    while (d >= 10.0 || d <= -10.0) {
+                        exp++;
+                        d = d / 10.0;
+                    }
+                    while (d < 1.0 && d > -1.0 && d != 0.0) {
+                        exp--;
+                        d = d * 10.0;
+                    }
+
+                    if (is_g)
+                        decimal--; /* g/G precision is number of signifigant digits */
+                    if (is_g && exp >= -4 && exp <= decimal) {
+                        /* exp is small enought for f, print without exponent */
+                        str = double_to_str(val, decimal, buf, pound_flag, !pound_flag);
+                        break;
+                    }
+                    /* print with exponent */
+                    str = double_to_exp_str(d, exp, decimal, buf, pound_flag, is_g && !pound_flag, caps);
+                    break;
+                }
+            case 'n':
+                {
+                    /* save num of chars printed so far in address specified */
+                    uint num_char = (uint) (s - start);
+                    /* yes, snprintf on all platforms returns int, not ssize_t */
+                    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(s - start)));
+                    if (l_type) {
+                        long * val = va_arg(ap, long *);
+                        *val = (long) num_char;
+                    } else if (h_type) {
+                        short * val = va_arg(ap, short *);
+                        *val = (short) num_char;
+                    } else {
+                        int * val = va_arg(ap, int *);
+                        *val = num_char;
+                    }
+                    buf[0] = '\0';
+                    str = buf;
+                    break;
+                }
+                /* FIXME : support the following? */
+            case 'a':
+            case 'A':
+            default:
+                ASSERT_NOT_REACHED();
+            }
+
+            /* if filler is 0 fill after prefix, else fill before prefix */
+            /* if - flag then fill after str and ignore filler type */
+
+            /* calculate number of fill characters */
+            if (fill > 0) {
+                IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(strlen(str) + strlen(prefix))));
+                fill -= (uint) (strlen(str) + strlen(prefix));
+            }
+            /* insert prefix if filler is 0, filler won't be 0 if - flag is set */
+            if (filler == '0') {
+                while (*prefix) {
+                    if (max > 0 && (size_t)(s - start) >= max)
+                        goto max_reached;
+                    *s = *prefix;
+                    s++;
+                    prefix++;
+                }
+            }
+            /* fill now if not left justified */
+            if (fill > 0 && !minus_flag) {
+                int i;
+                for (i = 0; i < fill; i++) {
+                    if (max > 0 && (size_t)(s - start) >= max)
+                        goto max_reached;
+                    *s = filler;
+                    s++;
+                }
+            }
+            /* insert prefix if not 0 filling */
+            if (filler != '0') {
+                while (*prefix) {
+                    if (max > 0 && (size_t)(s - start) >= max)
+                        goto max_reached;
+                    *s = *prefix;
+                    s++;
+                    prefix++;
+                }
+            }
+            /* insert the actual str representation */
+            while (*str) {
+                if (max > 0 && (size_t)(s - start) >= max)
+                    goto max_reached;
+                if (*c == 's' && decimal == 0)
+                    break;  /* check string precision */
+                decimal--;
+                *s = *str;
+                s++;
+                str++;
+            }
+            /* if left justified do the fill now after the actual string */
+            if (fill > 0 && minus_flag) {
+                int i;
+                for (i = 0; i < fill; i++) {
+                    if (max > 0 && (size_t)(s - start) >= max)
+                        goto max_reached;
+                    *s = filler;
+                    s++;
+                }
+            }
+            c++;
+        }
+        else {
+            const char *cstart = c;
+            int  nbytes = 0;
+            while (*c && *c != '%') {
+                nbytes++;
+                c++;
+            }
+            while (cstart < c) {
+                if (max > 0 && (size_t)(s - start) >= max)
+                    goto max_reached;
+                *s = *cstart;
+                s++;
+                cstart++;
+            }
+        }
+    }
+
+    if (max == 0 || (size_t)(s - start) < max)
+      *s = '\0';
+
+ max_reached:
+    /* yes, snprintf on all platforms returns int, not ssize_t */
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(s - start)));
+    return (int) (s - start);
+}
+
+int
+our_snprintf(char *s, size_t max, const char *fmt, ...)
+{
+    int res;
+    va_list ap;
+    ASSERT(s);
+    va_start(ap, fmt);
+    res = our_vsnprintf(s, max, fmt, ap);
+    va_end(ap);
+    return res;
+}
diff --git a/core/lib/dr_api.h b/core/lib/dr_api.h
new file mode 100644
index 000000000..03efc6095
--- /dev/null
+++ b/core/lib/dr_api.h
@@ -0,0 +1,101 @@
+/* **********************************************************
+ * Copyright (c) 2002-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifndef _DR_API_H_
+#define _DR_API_H_ 1
+
+/**
+ * @file dr_api.h
+ * @brief Top-level include file for DynamoRIO API.  
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Defines and type definitions */
+#include "dr_defines.h"
+
+/* Event registration routines */
+#include "dr_events.h"
+
+/* Application start/stop interface */
+#include "dr_app.h"
+
+/* High-level routines: memory allocation, mutex support, file 
+ * support, printing, thread support, adaptive optimization, 
+ * custom traces, processor-specific utilities, trace dumping, 
+ * and module information.
+ */
+#include "dr_tools.h"
+
+/* Utility routines for identifying features of the processor. */
+#include "dr_proc.h"
+
+/* opnd_t (operand) routines */
+#include "dr_ir_opnd.h"
+
+/* Instruction convenience & decode/disassemble routines */
+#include "dr_ir_utils.h"
+
+/* instrlist_t routines */
+#include "dr_ir_instrlist.h"
+
+/* instr_t routines */
+#include "dr_ir_instr.h"
+
+/* opcode OP_ constants and opcode-only routines */
+#include "dr_ir_opcodes.h"
+
+/* CREATE_INSTR_ macros */
+#include "dr_ir_macros.h"
+
+/**
+ * When registering a process, users must provide a list of paths to
+ * client libraries and their associated client-specific options.  DR
+ * looks up "dr_init" in each client library and calls that function
+ * when the process starts.  Clients can register to receive callbacks
+ * for the various events within dr_init.  Note that client paths and
+ * options cannot include semicolons. \p client_id is the ID supplied
+ * at registration and is used to identify the client for
+ * dr_get_options(), dr_get_client_path(), and external nudges.
+ */
+DR_EXPORT void dr_init(client_id_t client_id);
+
+/* Version checking */
+DR_EXPORT LINK_ONCE int _USES_DR_VERSION_ = 103; /* 1.3.x */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _DR_API_H_ */
diff --git a/core/lib/dr_app.h b/core/lib/dr_app.h
new file mode 100644
index 000000000..5a044330f
--- /dev/null
+++ b/core/lib/dr_app.h
@@ -0,0 +1,92 @@
+/* **********************************************************
+ * Copyright (c) 2002-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/**
+ * @file dr_app.h
+ * @brief DR's application interface for running portions
+ * of a program under its control.
+ */
+
+#ifndef _DR_APP_H_
+#define _DR_APP_H_ 1
+
+#ifdef WINDOWS
+#  ifdef DR_APP_EXPORTS
+#    define DR_APP_API __declspec(dllexport)
+#  else
+#    define DR_APP_API __declspec(dllimport)
+#  endif
+#else /* LINUX */
+#  if defined(DR_APP_EXPORTS) && defined(USE_VISIBILITY_ATTRIBUTES)
+#    define DR_APP_API __attribute__ ((visibility ("default")))
+#  else
+#    define DR_APP_API
+#  endif
+#endif
+
+/****************************************************************************
+ * DR Application Interface
+ */
+
+/**
+ * Application-wide initialization. Must be called before any other
+ * API function, and before the application creates any threads. Returns
+ * zero on success.
+ */
+DR_APP_API int dr_app_setup(void);
+
+/**
+ * Application-wide cleanup.  Prints statistics. Returns zero on success.
+ */
+DR_APP_API int dr_app_cleanup(void);
+
+/**
+ * Causes application to run under DR control upon return 
+ * from this call.
+ */
+DR_APP_API void dr_app_start(void);
+
+/**
+ * Causes application to run directly on the machine upon return 
+ * from this call; no effect if application is not currently
+ * running under DR control.
+ */
+DR_APP_API void dr_app_stop(void);
+
+/**
+ * Causes application to run under DR control upon return from 
+ * this call.  DR never releases control. Useful for overriding 
+ * dr_app_start/dr_app_stop calls in the rest of a program.
+ */
+DR_APP_API void dr_app_take_over(void);
+
+#endif /* _DR_APP_H_ */
diff --git a/core/lib/dr_config.h b/core/lib/dr_config.h
new file mode 100644
index 000000000..aadcf96d2
--- /dev/null
+++ b/core/lib/dr_config.h
@@ -0,0 +1,668 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifndef _DR_CONFIG_H_
+#define _DR_CONFIG_H_ 1
+
+#ifdef WINDOWS
+
+/* DR_API EXPORT TOFILE dr_config.h */
+/* DR_API EXPORT BEGIN */
+#ifdef WINDOWS
+/****************************************************************************
+ * Deployment API
+ */
+/**
+ * @file dr_config.h
+ * @brief Deployment API for Windows.  Use these functions to register
+ * processes to run under DynamoRIO, unregister processes, obtain existing 
+ * registration information, and nudge running processes.
+ * \note The dr_config library is currently not multi-thread safe. Users of
+ * the library should ensure that no more then one thread accesses the library
+ * at a time.  This limitation will be addressed in future releases.
+ */
+
+/** Maximum length of a registered process's options string */
+#define DR_MAX_OPTIONS_LENGTH 512
+
+/** Specifies DynamoRIO's operation mode. */
+typedef enum {
+
+    /** 
+     * No mode.  Clients should not attempt to register a process in
+     * this mode.
+     */
+    DR_MODE_NONE = 0,
+
+    /** Run DynamoRIO in Code Manipulation mode. */
+#ifdef HOT_PATCHING_INTERFACE
+    /** Note that this mode also supports the Probe API. */
+#endif
+    DR_MODE_CODE_MANIPULATION = 1,
+
+#ifdef HOT_PATCHING_INTERFACE
+    /** Run DynamoRIO in Probe mode.  This mode has no code cache. */
+    DR_MODE_PROBE = 2,
+
+#endif
+
+#ifdef PROGRAM_SHEPHERDING
+    /** Run DynamoRIO in Memory Firewall mode. */
+    DR_MODE_MEMORY_FIREWALL = 3,
+
+#endif
+
+} dr_operation_mode_t;
+
+/** Return status codes for process registration, unregistration and nudging. */
+typedef enum {
+    /** Operation succeeded. */
+    DR_SUCCESS,
+
+    /** Process registration failed due to an existing registration. */
+    DR_PROC_REG_EXISTS,
+
+    /** Operation failed because the supplied process is not registered. */
+    DR_PROC_REG_INVALID,
+
+    /** Client registration failed due to an invalid priority value. */
+    DR_PRIORITY_INVALID,
+
+    /** Client registration failed due to a conflicting ID. */
+    DR_ID_CONFLICTING,
+
+    /** Client operation failed due to an invalid client ID. */
+    DR_ID_INVALID,
+
+    /** Unknown failure. Check that caller has adequate permissions for this operation. */
+    DR_FAILURE,
+
+    /** Nudge operation failed because the specified process id is not under DR. */
+    DR_NUDGE_PID_NOT_INJECTED,
+
+    /** Nudge operation timed out waiting for target process to finish handling a nudge.*/
+    DR_NUDGE_TIMEOUT,
+
+} dr_config_status_t;
+
+/** Allow targeting both WOW64 and native 64-bit processes separately. */
+typedef enum {
+    DR_PLATFORM_DEFAULT, /**< The platform this tool is compiled for. */
+    DR_PLATFORM_32BIT,   /**< 32-bit settings (for 32-bit or WOW64 processes). */
+    DR_PLATFORM_64BIT,   /**< 64-bit settings (for native 64-bit processes). */
+} dr_platform_t;
+
+/**
+ * Register a process to run under DynamoRIO.  This requires administrative
+ * privileges.  Note that this routine only sets the base options to run
+ * a process under DynamoRIO.  To register one or more clients, call
+ * dr_register_client() subsequently.
+ *
+ * \param[in]   process_name    A NULL-terminated string specifying the name 
+ *                              of the target process.  The string should 
+ *                              identify the base name of the process, not the 
+ *                              full path of the executable (e.g., calc.exe).
+ *
+ * \param[in]   dr_root_dir     A NULL-terminated string specifying the full
+ *                              path to a valid DynamoRIO root directory.
+ *                              The string length cannot exceed MAX_PATH.
+ *
+ * \param[in]   dr_mode         Specifies the mode under which DynamoRIO should 
+ *                              operate.  See dr_operation_mode_t.
+ *
+ * \param[in]   debug           If true, a DynamoRIO debug build will be used;
+ *                              otherwise, a release build will be used.
+ *
+ * \param[in]   dr_platform     Configurations are kept separate on 64-bit Windows
+ *                              for 32-bit (WOW64) processes and 64-bit processes.
+ *                              This parameter allows selecting which of those
+ *                              configurations to set.
+ *
+ * \param[in]   dr_options      A NULL-terminated string controlling
+ *                              DynamoRIO's behavior.  Most users should
+ *                              not need to specify options.  The total
+ *                              string length cannot exceed #DR_MAX_OPTIONS_LENGTH.
+ *
+ * \return      A dr_config_status_t code indicating the result of 
+ *              registration.  Note that registration fails if the requested
+ *              process is already registered.  To modify a process's 
+ *              registration, first call dr_unregister_process() to remove an
+ *              existing registration.
+ *
+ * \remarks
+ * After registration, a process will run under DynamoRIO the next time
+ * it launches.  Note that some processes may require a system reboot to 
+ * restart.  Process registration persists across reboots until explicitly 
+ * unregistered.
+ *
+ * \note On Windows NT, a reboot is required after the initial
+ * dr_register_process() in order for DynamoRIO to take control of any
+ * application.
+ *
+ * \note An application that does not link with user32.dll will not be
+ * run under control of DynamoRIO unless it's launched by the \c drinject.exe 
+ * tool (in bin/bin32 or bin/bin64) or the parent process (typically
+ * explorer.exe, for manually launched applications) is already under
+ * DynamoRIO control (the parent can be in any mode, but 32-bit parent
+ * cannot inject into a 64-bit child). Only some small non-graphical
+ * applications do not link with user32.dll.
+ */
+dr_config_status_t 
+dr_register_process(const char *process_name,
+                    const char *dr_root_dir,
+                    dr_operation_mode_t dr_mode,
+                    bool debug,
+                    dr_platform_t dr_platform,
+                    const char *dr_options);
+
+/**
+ * Unregister a process from running under DynamoRIO.  This requires administrative
+ * privileges.
+ *
+ * \param[in]   process_name    A NULL-terminated string specifying the name 
+ *                              of the target process.  The string should 
+ *                              identify the base name of the process, not the 
+ *                              full path of the executable (e.g., calc.exe).
+ *
+ * \param[in]   dr_platform     Configurations are kept separate on 64-bit Windows
+ *                              for 32-bit (WOW64) processes and 64-bit processes.
+ *                              This parameter allows selecting which of those
+ *                              configurations to unset.
+ *
+ * \return      A dr_config_status_t code indicating the result of 
+ *              unregistration.  Note that unregistration fails if the process 
+ *              is not currently registered to run under DynamoRIO.
+ */
+dr_config_status_t
+dr_unregister_process(const char *process_name,
+                      dr_platform_t dr_platform);
+
+/**
+ * Check if a process is registered to run under DynamoRIO.  To obtain client
+ * information, use dr_get_client_info().
+ *
+ * \param[in]   process_name    A NULL-terminated string specifying the name 
+ *                              of the target process.  The string should 
+ *                              identify the base name of the process, not the 
+ *                              full path of the executable (e.g., calc.exe).
+ *
+ * \param[in]   dr_platform     Configurations are kept separate on 64-bit Windows
+ *                              for 32-bit (WOW64) processes and 64-bit processes.
+ *                              This parameter allows selecting which of those
+ *                              configurations to check.
+ *
+ * \param[out]  dr_root_dir     If the process is registered, the root DynamoRIO
+ *                              directory provided at registration.  Callers can
+ *                              pass NULL if this value is not needed.  Otherwise, 
+ *                              the parameter must be a caller-allocated array of 
+ *                              length MAX_PATH.
+ *
+ * \param[out]  dr_mode         If the process is registered, the mode provided
+ *                              at registration.  Callers can pass NULL if this
+ *                              value is not needed.
+ *
+ * \param[out]  debug           If the process is registered, the debug flag
+ *                              provided at registration.  Callers can pass NULL
+ *                              if this value is not needed.
+ *
+ * \param[out]  dr_options      If the process is registered, the extra DynamoRIO
+ *                              parameters provided at registration.  Callers can
+ *                              pass NULL if this value is not needed.  Otherwise,
+ *                              the parameter must be a caller-allocated array of
+ *                              length #DR_MAX_OPTIONS_LENGTH.
+ *
+ * \return      true if the process is registered for the given platform.
+ */
+bool
+dr_process_is_registered(const char *process_name,
+                         dr_platform_t dr_platform,
+                         char *dr_root_dir              /* OUT */,
+                         dr_operation_mode_t *dr_mode   /* OUT */,
+                         bool *debug                    /* OUT */,
+                         char *dr_options               /* OUT */);
+
+typedef struct _dr_registered_process_iterator_t dr_registered_process_iterator_t;
+
+/**
+ * Creates and starts an iterator for iterating over all processes registered for
+ * the given platform. 
+ *
+ * \param[in]   dr_platform     Configurations are kept separate on 64-bit Windows
+ *                              for 32-bit (WOW64) processes and 64-bit processes.
+ *                              This parameter allows selecting which of those
+ *                              configurations to check.
+ *
+ * \return      iterator for use with dr_registered_process_iterator_hasnext()and 
+ *              dr_registered_process_iterator_next().  Must be freed
+ *              with dr_registered_process_iterator_stop()
+ */
+dr_registered_process_iterator_t *
+dr_registered_process_iterator_start(dr_platform_t dr_platform);
+
+/**
+ * \param[in]    iter           A registered process iterator created with
+ *                              dr_registered_process_iterator_start()
+ *
+ * \return      true if there are more registered processes to iterate over
+ */
+bool
+dr_registered_process_iterator_hasnext(dr_registered_process_iterator_t *iter);
+
+/**
+ * Return information about a registered process
+ *
+ * \param[in]    iter           A registered process iterator created with
+ *                              dr_registered_process_iterator_start().
+ *
+ * \param[out]   process_name   The name of the registered process. Callers can
+ *                              pass NULL if this value is not needed. Otherwise
+ *                              the parameter must be a caller-allocated array
+ *                              of length MAX_PATH.
+ *
+ * \param[out]  dr_root_dir     The root DynamoRIO directory provided at registration.
+ *                              Callers can pass NULL if this value is not needed.
+ *                              Otherwise, the parameter must be a caller-allocated
+ *                              array of length MAX_PATH.
+ *
+ * \param[out]  dr_mode         If the process is registered, the mode provided
+ *                              at registration.  Callers can pass NULL if this
+ *                              value is not needed.
+ *
+ * \param[out]  debug           If the process is registered, the debug flag
+ *                              provided at registration.  Callers can pass NULL
+ *                              if this value is not needed.
+ *
+ * \param[out]  dr_options      If the process is registered, the extra DynamoRIO
+ *                              parameters provided at registration.  Callers can
+ *                              pass NULL if this value is not needed.  Otherwise,
+ *                              the parameter must be a caller-allocated array of
+ *                              length #DR_MAX_OPTIONS_LENGTH.
+ *
+ * \return      true if the process is registered for the given platform.
+ */
+void
+dr_registered_process_iterator_next(dr_registered_process_iterator_t *iter,
+                                    char *process_name /* OUT */,
+                                    char *dr_root_dir /* OUT */,
+                                    dr_operation_mode_t *dr_mode /* OUT */,
+                                    bool *debug /* OUT */,
+                                    char *dr_options /* OUT */);
+
+/**
+ * Stops and frees a registered process iterator.
+ *
+ * \param[in]    iter           A registered process iterator created with
+ *                              dr_registered_process_iterator_start()
+ */
+void
+dr_registered_process_iterator_stop(dr_registered_process_iterator_t *iter);
+
+/**
+ * Register a client for a particular process.  Note that the process must
+ * first be registered via dr_register_process() before calling this routine.
+ *
+ * \param[in]   process_name    A NULL-terminated string specifying the name 
+ *                              of the target process.  The string should 
+ *                              identify the base name of the process, not the 
+ *                              full path of the executable (e.g., calc.exe).
+ *
+ * \param[in]   dr_platform     Configurations are kept separate on 64-bit Windows
+ *                              for 32-bit (WOW64) processes and 64-bit processes.
+ *                              This parameter allows selecting which of those
+ *                              configurations to unset.
+ *
+ * \param[in]   client_id       A client_id_t uniquely identifying the client.
+ *                              DynamoRIO provides the client ID as a parameter
+ *                              to dr_init().  Clients use this ID to retrieve
+ *                              client-specific path and option information.
+ *                              Outside entities also identify the target of
+ *                              a nudge via this ID.
+ *
+ * \param[in]   client_pri      The client number, or priority.  Client registration
+ *                              includes a value indicating the priority of a client
+ *                              relative to other clients.  In multi-client settings,
+ *                              a client's priority influences event callback 
+ *                              ordering.  That is, higher priority clients can 
+ *                              register their callbacks first; DynamoRIO then calls
+ *                              these routines last.  Client priorities range
+ *                              consecutively from 0 to N-1, where N is the number
+ *                              of registered clients.  Specify priority
+ *                              0 to register a client with highest priority.
+ *
+ * \param[in]   client_path     A NULL-terminated string specifying the full path
+ *                              to a valid client library.  The string length
+ *                              cannot exceed MAX_PATH.  The client path may not
+ *                              include any semicolons.
+ *
+ * \param[in]   client_options  A NULL-terminated string specifying options that
+ *                              are available to the client via dr_get_options().
+ *                              The string length cannot exceed #DR_MAX_OPTIONS_LENGTH.
+ *                              The client options may not include any semicolons.
+ *
+ * \return      A dr_config_status_t code indicating the result of registration.
+ */
+dr_config_status_t
+dr_register_client(const char *process_name,
+                   dr_platform_t dr_platform,
+                   client_id_t client_id,
+                   size_t client_pri,
+                   const char *client_path,
+                   const char *client_options);
+
+/**
+ * \param[in]   process_name    A NULL-terminated string specifying the name 
+ *                              of the target process.  The string should 
+ *                              identify the base name of the process, not the 
+ *                              full path of the executable (e.g., calc.exe).
+ *
+ * \param[in]   dr_platform     Configurations are kept separate on 64-bit Windows
+ *                              for 32-bit (WOW64) processes and 64-bit processes.
+ *                              This parameter allows selecting which of those
+ *                              configurations to unset.
+ *
+ * \param[in]   client_id       The unique client ID provided at client
+ *                              registration.
+ *
+ * \return      A dr_config_status_t code indicating the result of unregistration.
+ */
+dr_config_status_t
+dr_unregister_client(const char *process_name,
+                     dr_platform_t dr_platform,
+                     client_id_t client_id);
+
+/**
+ * Retrieve the number of clients registered for a particular process.
+ *
+ * \param[in]   process_name    A NULL-terminated string specifying the name 
+ *                              of the target process.  The string should 
+ *                              identify the base name of the process, not the 
+ *                              full path of the executable (e.g., calc.exe).
+ *
+ * \param[in]   dr_platform     Configurations are kept separate on 64-bit Windows
+ *                              for 32-bit (WOW64) processes and 64-bit processes.
+ *                              This parameter allows selecting which of those
+ *                              configurations to unset.
+ *
+ * \return      The number of clients registered for the given process and platform.
+ */
+size_t
+dr_num_registered_clients(const char *process_name,
+                          dr_platform_t dr_platform);
+
+/**
+ * Retrieve client registration information for a particular process.
+ *
+ * \param[in]   process_name    A NULL-terminated string specifying the name 
+ *                              of the target process.  The string should 
+ *                              identify the base name of the process, not the 
+ *                              full path of the executable (e.g., calc.exe).
+ *
+ * \param[in]   dr_platform     Configurations are kept separate on 64-bit Windows
+ *                              for 32-bit (WOW64) processes and 64-bit processes.
+ *                              This parameter allows selecting which of those
+ *                              configurations to unset.
+ *
+ * \param[in]   client_id       The unique client ID provided at client
+ *                              registration.
+ *
+ * \param[out]  client_pri      The client's priority.
+ *
+ * \param[out]  client_path     The client's path provided at registration.
+ *                              Callers can pass NULL if this value is not needed.
+ *                              Otherwise, the parameter must be a caller-allocated
+ *                              array of length MAX_PATH.
+ *
+ * \param[out]  client_options  The client options provided at registration.
+ *                              Callers can pass NULL if this value is not needed.
+ *                              Otherwise, the parameter must be a caller-allocated
+ *                              array of length #DR_MAX_OPTIONS_LENGTH.
+ *
+ * \return      A dr_config_status_t code indicating the result of the call.
+ */
+dr_config_status_t
+dr_get_client_info(const char *process_name,
+                   dr_platform_t dr_platform,
+                   client_id_t client_id,
+                   size_t *client_pri,  /* OUT */
+                   char *client_path,   /* OUT */
+                   char *client_options /* OUT */);
+
+typedef struct _dr_client_iterator_t dr_client_iterator_t;
+
+/**
+ * Creates and starts an iterator for iterating over all clients registered for
+ * the given process.
+ *
+ * \param[in]   process_name    A NULL-terminated string specifying the name 
+ *                              of the target process.  The string should 
+ *                              identify the base name of the process, not the 
+ *                              full path of the executable (e.g., calc.exe).
+ *
+ * \param[in]   dr_platform     Configurations are kept separate on 64-bit Windows
+ *                              for 32-bit (WOW64) processes and 64-bit processes.
+ *                              This parameter allows selecting which of those
+ *                              configurations to check.
+ *
+ * \return      iterator for use with dr_client_iterator_hasnext() and
+ *              dr_client_iterator_next().  Must be freed with
+ *              dr_client_iterator_stop()
+ */
+dr_client_iterator_t *
+dr_client_iterator_start(const char *process_name,
+                         dr_platform_t dr_platform);
+
+/**
+ * \param[in]   iter            A client iterator created with dr_client_iterator_start()
+ *
+ * \return      true if there are more clients to iterate over
+ */
+bool
+dr_client_iterator_hasnext(dr_client_iterator_t *iter);
+
+/**
+ * Return information about a client.
+ *
+ * \param[in]   iter            A client iterator created with dr_client_iterator_start()
+ *
+ * \param[out]  client_id       The unique client ID provided at client registration.
+ *
+ * \param[out]  client_pri      The client's priority.
+ *
+ * \param[out]  client_path     The client's path provided at registration.
+ *                              Callers can pass NULL if this value is not needed.
+ *                              Otherwise, the parameter must be a caller-allocated
+ *                              array of length MAX_PATH.
+ *
+ * \param[out]  client_options  The client options provided at registration.
+ *                              Callers can pass NULL if this value is not needed.
+ *                              Otherwise, the parameter must be a caller-allocated
+ *                              array of length #DR_MAX_OPTIONS_LENGTH.
+ */
+void
+dr_client_iterator_next(dr_client_iterator_t *iter,
+                        client_id_t *client_id, /* OUT */
+                        size_t *client_pri,     /* OUT */
+                        char *client_path,      /* OUT */
+                        char *client_options    /* OUT */);
+
+/**
+ * Stops and frees a client iterator.
+ *
+ * \param[in]   iter            A client iterator created with dr_client_iterator_start()
+ */
+void
+dr_client_iterator_stop(dr_client_iterator_t *iter);
+
+
+/**
+ * Provides a mechanism for an external entity on the guest OS to
+ * communicate with a client.  Requires administrative privileges.  A
+ * process 'nudge' causes a client event handler to be invoked (use
+ * dr_register_nudge_event() to register the handler function).  A
+ * nudge is ignored if the process is not running under DynamoRIO,
+ * the specified client is not loaded, or if the client does not 
+ * provide a handler.
+ *
+ * \param[in]   process_name    A NULL-terminated string specifying the name 
+ *                              of the target process.  The string should 
+ *                              identify the base name of the process, not the 
+ *                              full path of the executable.
+ *
+ * \param[in]   client_id       The unique client ID provided at client
+ *                              registration.
+ *
+ * \param[in]   arg             An argument passed to the client's nudge
+ *                              handler.
+ *
+ * \param[in]   timeout_ms      The number of milliseconds to wait for each
+ *                              nudge to complete before continuing. If INFINITE
+ *                              is supplied then the wait is unbounded. If 0
+ *                              is supplied the no wait is performed.  If a
+ *                              wait times out DR_NUDGE_TIMEOUT will be returned.
+ *
+ * \param[out]  nudge_count     Returns the number of processes nudged.
+ *                              Client can supply NULL if this value is
+ *                              not needed.
+ *
+ * \return      A dr_config_status_t code indicating the result of the nudge.
+ *
+ * \remarks
+ * If there are multiple processes executing with the same name, all
+ * of them are nudged.
+ *
+ * \remarks
+ * A process nudge is a one way asynchronous communication from an
+ * external entity to a client.  It does not allow a client to
+ * return information back to the nudge originator.  To communicate
+ * from a client to another process, use the channels specified
+ * in \ref sec_comm.
+ *
+ * \note Nudging 64-bit processes is not yet supported.
+ */
+dr_config_status_t
+dr_nudge_process(const char *process_name,
+                 client_id_t client_id,
+                 uint64 arg,
+                 uint timeout_ms,
+                 int *nudge_count /*OUT */);
+
+/**
+ * Provides a mechanism for an external entity on the guest OS to
+ * communicate with a client.  Requires administrative privileges.  A
+ * process 'nudge' causes a client event handler to be invoked (use
+ * dr_register_nudge_event() to register the handler function).  A
+ * nudge is ignored if the process is not running under DynamoRIO,
+ * the specified client is not loaded, or if the client does not 
+ * provide a handler.
+ *
+ * \param[in]   process_id      The system id of the process to nudge
+ *                              (see dr_get_process_id())
+ *
+ * \param[in]   client_id       The unique client ID provided at client
+ *                              registration.
+ *
+ * \param[in]   arg             An argument passed to the client's nudge
+ *                              handler.
+ *
+ * \param[in]   timeout_ms      The number of milliseconds to wait for the
+ *                              nudge to complete before returning. If INFINITE
+ *                              is supplied then the wait is unbounded. If 0
+ *                              is supplied the no wait is performed.  If the
+ *                              wait times out DR_NUDGE_TIMEOUT will be returned.
+ *
+ * \return      A dr_config_status_t code indicating the result of the nudge.
+ *
+ * \remarks
+ * A process nudge is a one way asynchronous communication from an
+ * external entity to a client.  It does not allow a client to
+ * return information back to the nudge originator.  To communicate
+ * from a client to another process, use the channels specified
+ * in \ref sec_comm.
+ *
+ * \note Nudging 64-bit processes is not yet supported.
+ */
+dr_config_status_t
+dr_nudge_pid(process_id_t process_id,
+             client_id_t client_id,
+             uint64 arg,
+             uint timeout_ms);
+
+/**
+ * Provides a mechanism for an external entity on the guest OS to
+ * communicate with a client.  Requires administrative privileges.  A
+ * process 'nudge' causes a client event handler to be invoked (use
+ * dr_register_nudge_event() to register the handler function).  A
+ * nudge is ignored if the process is not running under DynamoRIO,
+ * the specified client is not loaded, or if the client does not 
+ * provide a handler.  Nudges are attempted to all processes running
+ * on the system.
+ *
+ * \param[in]   client_id       The unique client ID provided at client
+ *                              registration.
+ *
+ * \param[in]   arg             An argument passed to the client's nudge
+ *                              handler.
+ *
+ * \param[in]   timeout_ms      The number of milliseconds to wait for each
+ *                              nudge to complete before continuing. If INFINITE
+ *                              is supplied then the wait is unbounded. If 0
+ *                              is supplied the no wait is performed.  If a
+ *                              wait times out DR_NUDGE_TIMEOUT will be returned.
+ *
+ * \param[out]  nudge_count     Returns the number of processes nudged.
+ *                              Client can supply NULL if this value is
+ *                              not needed.
+ *
+ * \return      A dr_config_status_t code indicating the result of the nudge.
+ *
+ * \remarks
+ * A process nudge is a one way asynchronous communication from an
+ * external entity to a client.  It does not allow a client to
+ * return information back to the nudge originator.  To communicate
+ * from a client to another process, use the channels specified
+ * in \ref sec_comm.
+ *
+ * \note Nudging 64-bit processes is not yet supported.
+ */
+dr_config_status_t
+dr_nudge_all(client_id_t client_id,
+             uint64 arg,
+             uint timeout_ms,
+             int *nudge_count /*OUT */);
+
+#endif /* WINDOWS */
+
+/* DR_API EXPORT END */
+
+#endif /* WINDOWS */
+
+#endif /* _DR_CONFIG_H_ */
diff --git a/core/lib/dr_ntdll.def b/core/lib/dr_ntdll.def
new file mode 100644
index 000000000..46ab8a2d8
--- /dev/null
+++ b/core/lib/dr_ntdll.def
@@ -0,0 +1,161 @@
+;; **********************************************************
+;; Copyright (c) 2008 VMware, Inc.  All rights reserved.
+;; **********************************************************
+
+;; Redistribution and use in source and binary forms, with or without
+;; modification, are permitted provided that the following conditions are met:
+;; 
+;; * Redistributions of source code must retain the above copyright notice,
+;;   this list of conditions and the following disclaimer.
+;; 
+;; * Redistributions in binary form must reproduce the above copyright notice,
+;;   this list of conditions and the following disclaimer in the documentation
+;;   and/or other materials provided with the distribution.
+;; 
+;; * Neither the name of VMware, Inc. nor the names of its contributors may be
+;;   used to endorse or promote products derived from this software without
+;;   specific prior written permission.
+;; 
+;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+;; ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+;; LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+;; OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+;; DAMAGE.
+
+;; PR 219380: provide ntdll.dll link support to clients
+
+LIBRARY dynamorio
+EXPORTS
+        wcstoul = NTDLL.wcstoul
+        wcstombs = NTDLL.wcstombs
+        wcstol = NTDLL.wcstol
+        wcsstr = NTDLL.wcsstr
+        wcsspn = NTDLL.wcsspn
+        wcsrchr = NTDLL.wcsrchr
+        wcspbrk = NTDLL.wcspbrk
+        wcsncpy = NTDLL.wcsncpy
+        wcsncmp = NTDLL.wcsncmp
+        wcsncat = NTDLL.wcsncat
+        wcslen = NTDLL.wcslen
+        wcscspn = NTDLL.wcscspn
+        wcscpy = NTDLL.wcscpy
+        wcscmp = NTDLL.wcscmp
+        wcschr = NTDLL.wcschr
+        wcscat = NTDLL.wcscat
+        towupper = NTDLL.towupper
+        towlower = NTDLL.towlower
+        toupper = NTDLL.toupper
+        tolower = NTDLL.tolower
+        tan = NTDLL.tan
+        strtoul = NTDLL.strtoul
+        strtol = NTDLL.strtol
+        strstr = NTDLL.strstr
+        strspn = NTDLL.strspn
+        strrchr = NTDLL.strrchr
+        strpbrk = NTDLL.strpbrk
+        strncpy = NTDLL.strncpy
+        strncmp = NTDLL.strncmp
+        strncat = NTDLL.strncat
+        strlen = NTDLL.strlen
+        strcspn = NTDLL.strcspn
+        strcmp = NTDLL.strcmp
+        strchr = NTDLL.strchr
+        sscanf = NTDLL.sscanf
+        sqrt = NTDLL.sqrt
+        sprintf = NTDLL.sprintf
+        sin = NTDLL.sin
+        qsort = NTDLL.qsort
+        pow = NTDLL.pow
+        memset = NTDLL.memset
+        memmove = NTDLL.memmove
+        memcpy = NTDLL.memcpy
+        memcmp = NTDLL.memcmp
+        memchr = NTDLL.memchr
+        mbstowcs = NTDLL.mbstowcs
+        log = NTDLL.log
+        labs = NTDLL.labs
+        isxdigit = NTDLL.isxdigit
+        iswxdigit = NTDLL.iswxdigit
+        iswspace = NTDLL.iswspace
+        iswlower = NTDLL.iswlower
+        iswdigit = NTDLL.iswdigit
+        iswctype = NTDLL.iswctype
+        iswalpha = NTDLL.iswalpha
+        isupper = NTDLL.isupper
+        isspace = NTDLL.isspace
+        ispunct = NTDLL.ispunct
+        isprint = NTDLL.isprint
+        islower = NTDLL.islower
+        isgraph = NTDLL.isgraph
+        isdigit = NTDLL.isdigit
+        iscntrl = NTDLL.iscntrl
+        isalpha = NTDLL.isalpha
+        isalnum = NTDLL.isalnum
+        floor = NTDLL.floor
+        fabs = NTDLL.fabs
+        cos = NTDLL.cos
+        ceil = NTDLL.ceil
+        atol = NTDLL.atol
+        atoi = NTDLL.atoi
+        atan = NTDLL.atan
+        abs = NTDLL.abs
+        _wtol = NTDLL._wtol
+        _wtoi64 = NTDLL._wtoi64
+        _wtoi = NTDLL._wtoi
+        _wcsupr = NTDLL._wcsupr
+        _wcsnicmp = NTDLL._wcsnicmp
+        _wcslwr = NTDLL._wcslwr
+        _wcsicmp = NTDLL._wcsicmp
+        _vsnprintf = NTDLL._vsnprintf
+        _ultow = NTDLL._ultow
+        _ultoa = NTDLL._ultoa
+        _ui64toa = NTDLL._ui64toa
+        _toupper = NTDLL._toupper
+        _tolower = NTDLL._tolower
+        _strupr = NTDLL._strupr
+        _strnicmp = NTDLL._strnicmp
+        _strlwr = NTDLL._strlwr
+        _stricmp = NTDLL._stricmp
+        _strcmpi = NTDLL._strcmpi
+        _snwprintf = NTDLL._snwprintf
+;; we get "instrument.obj : warning LNK4197: export '_snprintf' specified 
+;;   multiple times; using first specification", so we have to explicitly
+;; re-list our own export (order doesn't matter):
+        snprintf
+        _snprintf = NTDLL._snprintf
+        _memicmp = NTDLL._memicmp
+        _memccpy = NTDLL._memccpy
+        _ltow = NTDLL._ltow
+        _ltoa = NTDLL._ltoa
+        _itow = NTDLL._itow
+        _itoa = NTDLL._itoa
+        _i64tow = NTDLL._i64tow
+        _i64toa = NTDLL._i64toa
+        _ftol = NTDLL._ftol
+        _fltused = NTDLL._fltused
+        _chkstk = NTDLL._chkstk
+        _aullshr = NTDLL._aullshr
+        _aullrem = NTDLL._aullrem
+        _aulldiv = NTDLL._aulldiv
+        _atoi64 = NTDLL._atoi64
+        _allshr = NTDLL._allshr
+        _allshl = NTDLL._allshl
+        _allrem = NTDLL._allrem
+        _allmul = NTDLL._allmul
+        _alldiv = NTDLL._alldiv
+        __toascii = NTDLL.__toascii
+        __iscsymf = NTDLL.__iscsymf
+        __iscsym = NTDLL.__iscsym
+        __isascii = NTDLL.__isascii
+;; these are needed for samples/stats.c
+        NtCreateSection = NTDLL.NtCreateSection
+        NtMapViewOfSection = NTDLL.NtMapViewOfSection
+        NtUnmapViewOfSection = NTDLL.NtUnmapViewOfSection
+        NtOpenDirectoryObject = NTDLL.NtOpenDirectoryObject
+        RtlInitUnicodeString = NTDLL.RtlInitUnicodeString
diff --git a/core/lib/dr_ntdll_x64.def b/core/lib/dr_ntdll_x64.def
new file mode 100644
index 000000000..29415c003
--- /dev/null
+++ b/core/lib/dr_ntdll_x64.def
@@ -0,0 +1,151 @@
+;; **********************************************************
+;; Copyright (c) 2008 VMware, Inc.  All rights reserved.
+;; **********************************************************
+
+;; Redistribution and use in source and binary forms, with or without
+;; modification, are permitted provided that the following conditions are met:
+;; 
+;; * Redistributions of source code must retain the above copyright notice,
+;;   this list of conditions and the following disclaimer.
+;; 
+;; * Redistributions in binary form must reproduce the above copyright notice,
+;;   this list of conditions and the following disclaimer in the documentation
+;;   and/or other materials provided with the distribution.
+;; 
+;; * Neither the name of VMware, Inc. nor the names of its contributors may be
+;;   used to endorse or promote products derived from this software without
+;;   specific prior written permission.
+;; 
+;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+;; ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+;; LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+;; OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+;; DAMAGE.
+
+;; PR 219380: provide ntdll.dll link support to clients
+
+LIBRARY dynamorio
+EXPORTS
+        wcstoul = NTDLL.wcstoul
+        wcstombs = NTDLL.wcstombs
+        wcstol = NTDLL.wcstol
+        wcsstr = NTDLL.wcsstr
+        wcsspn = NTDLL.wcsspn
+        wcsrchr = NTDLL.wcsrchr
+        wcspbrk = NTDLL.wcspbrk
+        wcsncpy = NTDLL.wcsncpy
+        wcsncmp = NTDLL.wcsncmp
+        wcsncat = NTDLL.wcsncat
+        wcslen = NTDLL.wcslen
+        wcscspn = NTDLL.wcscspn
+        wcscpy = NTDLL.wcscpy
+        wcscmp = NTDLL.wcscmp
+        wcschr = NTDLL.wcschr
+        wcscat = NTDLL.wcscat
+        towupper = NTDLL.towupper
+        towlower = NTDLL.towlower
+        toupper = NTDLL.toupper
+        tolower = NTDLL.tolower
+        tan = NTDLL.tan
+        strtoul = NTDLL.strtoul
+        strtol = NTDLL.strtol
+        strstr = NTDLL.strstr
+        strspn = NTDLL.strspn
+        strrchr = NTDLL.strrchr
+        strpbrk = NTDLL.strpbrk
+        strncpy = NTDLL.strncpy
+        strncmp = NTDLL.strncmp
+        strncat = NTDLL.strncat
+        strlen = NTDLL.strlen
+        strcspn = NTDLL.strcspn
+        strcmp = NTDLL.strcmp
+        strchr = NTDLL.strchr
+        sscanf = NTDLL.sscanf
+        sqrt = NTDLL.sqrt
+        sprintf = NTDLL.sprintf
+        sin = NTDLL.sin
+        qsort = NTDLL.qsort
+        pow = NTDLL.pow
+        memset = NTDLL.memset
+        memmove = NTDLL.memmove
+        memcpy = NTDLL.memcpy
+        memcmp = NTDLL.memcmp
+        memchr = NTDLL.memchr
+        mbstowcs = NTDLL.mbstowcs
+        log = NTDLL.log
+        labs = NTDLL.labs
+        isxdigit = NTDLL.isxdigit
+        iswxdigit = NTDLL.iswxdigit
+        iswspace = NTDLL.iswspace
+        iswlower = NTDLL.iswlower
+        iswdigit = NTDLL.iswdigit
+        iswctype = NTDLL.iswctype
+        iswalpha = NTDLL.iswalpha
+        isupper = NTDLL.isupper
+        isspace = NTDLL.isspace
+        ispunct = NTDLL.ispunct
+        isprint = NTDLL.isprint
+        islower = NTDLL.islower
+        isgraph = NTDLL.isgraph
+        isdigit = NTDLL.isdigit
+        iscntrl = NTDLL.iscntrl
+        isalpha = NTDLL.isalpha
+        isalnum = NTDLL.isalnum
+        floor = NTDLL.floor
+        fabs = NTDLL.fabs
+        cos = NTDLL.cos
+        ceil = NTDLL.ceil
+        atol = NTDLL.atol
+        atoi = NTDLL.atoi
+        atan = NTDLL.atan
+        abs = NTDLL.abs
+        _wtol = NTDLL._wtol
+        _wtoi64 = NTDLL._wtoi64
+        _wtoi = NTDLL._wtoi
+        _wcsupr = NTDLL._wcsupr
+        _wcsnicmp = NTDLL._wcsnicmp
+        _wcslwr = NTDLL._wcslwr
+        _wcsicmp = NTDLL._wcsicmp
+        _vsnprintf = NTDLL._vsnprintf
+        _ultow = NTDLL._ultow
+        _ultoa = NTDLL._ultoa
+        _ui64toa = NTDLL._ui64toa
+        _toupper = NTDLL._toupper
+        _tolower = NTDLL._tolower
+        _strupr = NTDLL._strupr
+        _strnicmp = NTDLL._strnicmp
+        _strlwr = NTDLL._strlwr
+        _stricmp = NTDLL._stricmp
+        _strcmpi = NTDLL._strcmpi
+        _snwprintf = NTDLL._snwprintf
+;; we get "instrument.obj : warning LNK4197: export '_snprintf' specified 
+;;   multiple times; using first specification", so we have to explicitly
+;; re-list our own export (order doesn't matter):
+        snprintf
+        _snprintf = NTDLL._snprintf
+        _memicmp = NTDLL._memicmp
+        _memccpy = NTDLL._memccpy
+        _ltow = NTDLL._ltow
+        _ltoa = NTDLL._ltoa
+        _itow = NTDLL._itow
+        _itoa = NTDLL._itoa
+        _i64tow = NTDLL._i64tow
+        _i64toa = NTDLL._i64toa
+        _fltused = NTDLL._fltused
+        _atoi64 = NTDLL._atoi64
+        __toascii = NTDLL.__toascii
+        __iscsymf = NTDLL.__iscsymf
+        __iscsym = NTDLL.__iscsym
+        __isascii = NTDLL.__isascii
+;; these are needed for samples/stats.c
+        NtCreateSection = NTDLL.NtCreateSection
+        NtMapViewOfSection = NTDLL.NtMapViewOfSection
+        NtUnmapViewOfSection = NTDLL.NtUnmapViewOfSection
+        NtOpenDirectoryObject = NTDLL.NtOpenDirectoryObject
+        RtlInitUnicodeString = NTDLL.RtlInitUnicodeString
diff --git a/core/lib/dr_stats.h b/core/lib/dr_stats.h
new file mode 100644
index 000000000..b6235c4b9
--- /dev/null
+++ b/core/lib/dr_stats.h
@@ -0,0 +1,126 @@
+/* **********************************************************
+ * Copyright (c) 2002-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* dr_stats.h: DR's shared memory interface */
+
+#ifndef _DR_STATS_H_
+#define _DR_STATS_H_ 1
+
+#include "globals_shared.h"
+
+#ifdef WINDOWS
+  /* No registry for stats (b/c it requires advapi32.dll which can't be used
+   * when injected via user32.dll registry key)
+   * Instead, a piece of shared memory with the key base name below holds the
+   * total number of DR instances.
+   */
+# define DR_SHMEM_KEY "DynamoRIOStatistics" 
+#elif defined(LINUX)
+# define DYNAMORIO_MAGIC_STRING "DYNAMORIO_MAGIC_STRING"
+# define DYNAMORIO_MAGIC_STRING_LEN 16 /*include trailing \0*/
+#endif
+
+#define STAT_NAME_MAX_LEN 50
+typedef struct _single_stat_t {
+    /* We inline the stat description to make it easy for external processes
+     * to view our stats: they don't have to chase pointers, and we could put
+     * this in shared memory easily.
+     */
+    char name[STAT_NAME_MAX_LEN]; /* the description of the stat */
+    /* FIXME PR 216209: we'll want 64-bit stats for x64 address regions; we can
+     * either add per-stat types, or just widen them all.
+     */
+    stats_int_t value;
+} single_stat_t;
+
+/* Parameters and statistics exported by DR via drmarker.
+ * These should be treated as read-only except for the log_ fields.
+ * Unless otherwise mentioned, these stats are all process-wide.
+ */
+#define NUM_EVENTS 27
+typedef struct _dr_statistics_t {
+#ifdef LINUX
+    char magicstring[DYNAMORIO_MAGIC_STRING_LEN];
+#endif
+    process_id_t process_id;    /* process id */
+    char process_name[MAXIMUM_PATH]; /* process name */
+    uint logmask;               /* what to log */
+    uint loglevel;              /* how much detail to log */
+    char logdir[MAXIMUM_PATH];  /* full path of logging directory */ 
+    uint64 perfctr_vals[NUM_EVENTS];
+    uint num_stats;
+#ifdef NOT_DYNAMORIO_CORE
+    /* variable-length to avoid tying to specific DR version */
+    single_stat_t stats[1];
+#else
+# ifdef DEBUG
+#  define STATS_DEF(desc, name) single_stat_t name##_pair;
+# else
+#  define RSTATS_DEF(desc, name) single_stat_t name##_pair;
+# endif
+#  include "statsx.h"
+# undef STATS_DEF
+# undef RSTATS_DEF
+#endif
+} dr_statistics_t;
+
+#ifndef NOT_DYNAMORIO_CORE
+/* Thread local statistics */
+typedef struct {
+    thread_id_t thread_id;
+    mutex_t thread_stats_lock;    /* transactional stats, for multiple stats invariants to hold */
+    /* TODO: We may also want to print another threads's stats without necessarily halting it,
+     * TODO: add stat name##_delta, which should be applied as a batch to the safe to read values.
+     * The basic idea of transactional stats is that uncommitted changes are not visible to readers.
+     * Some invariants between statistics, i.e. A=B+C should hold at the dump/committed points.
+     *
+     * The plan is: 
+     *   1) delta accessed w/o lock only by the owning thread,
+     *   2) on dump any other thread which only reads the committed values while holding the commit lock,
+     *   3) The owning thread is the single writer to the committed values to apply the deltas,
+     *      while holding the commit lock.
+     *
+     * Used for other threads to be able to request thread local stats, 
+     * and also for the not fully explained self-interruption on linux?
+     */
+#ifdef DEBUG
+# define STATS_DEF(desc, name) stats_int_t name##_thread;  
+#else
+# define RSTATS_DEF(desc, name) stats_int_t name##_thread;  
+#endif
+# include "statsx.h"
+} thread_local_statistics_t;
+#undef STATS_DEF
+#undef RSTATS_DEF
+#endif /* !NOT_DYNAMORIO_CORE */
+
+#endif /* _DR_STATS_H_ */
diff --git a/core/lib/fortran.c b/core/lib/fortran.c
new file mode 100644
index 000000000..6489b92be
--- /dev/null
+++ b/core/lib/fortran.c
@@ -0,0 +1,66 @@
+/* **********************************************************
+ * Copyright (c) 2001-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+
+/* Fortran wrappers for dynamo
+ * Fortran has restrictions on function names so can't just call dynamo_*!
+ * Call dr*() instead, and link with this .o file.  
+ * For example:
+ *    CALL DRSETUP()
+ *    CALL DRSTART()
+ */
+
+#include "dr_app.h"
+
+#ifdef DR_APP_EXPORTS
+void drsetup_()
+{
+    dr_app_setup();
+}
+
+void drcleanup_()
+{
+    dr_app_cleanup();
+}
+
+void drstart_()
+{
+    dr_app_start();
+}
+
+void drstop_()
+{
+    dr_app_stop();
+}
+#endif
diff --git a/core/lib/genapi.pl b/core/lib/genapi.pl
new file mode 100755
index 000000000..c81b38d64
--- /dev/null
+++ b/core/lib/genapi.pl
@@ -0,0 +1,442 @@
+#!/usr/local/bin/perl
+
+# **********************************************************
+# Copyright (c) 2002-2009 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# Copyright (c) 2003-2007 Determina Corp.
+# Copyright (c) 2002-2003 Massachusetts Institute of Technology
+# Copyright (c) 2002 Hewlett Packard Company
+
+### genapi.pl
+###
+### generates header files and filter files for our exported data and routines
+### N.B.: this script assumes several things about the layout of declarations:
+###   1) comments are not sprinkled in weird places
+###   2) multiple declarations do not share the same line
+###
+
+$usage = "Usage: $0 -header <destinationdir> <defines> | -filter <defines> | -debug\n";
+
+$header = 0;
+$debug = 0;
+$filter = 0;
+
+if ($#ARGV < 0) {
+    print "$usage";
+    exit 0;
+}
+
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq '-header') {
+        $header = 1;
+        shift;
+        if ($#ARGV != 1) {
+            print "$usage";
+            exit 0;
+        }
+        $dir = $ARGV[0];
+        if (! -d $dir) {
+            print "$dir is not a directory\n";
+            print "$usage";
+            exit 0;
+        }
+        shift;
+        $define_str = $ARGV[0];
+        $define_str =~ s/^[-\/]D//;
+        $define_str =~ s/ [-\/]D/ /g;
+        @define_els = split(' ', $define_str);
+        foreach $i (@define_els) {
+            $defines{$i} = 1;
+        }
+    } elsif ($ARGV[0] eq '-filter') {
+        $filter = 1;
+        shift;
+        $define_str = $ARGV[0];
+        $define_str =~ s/^[-\/]D//;
+        $define_str =~ s/ [-\/]D/ /g;
+        @define_els = split(' ', $define_str);
+        foreach $i (@define_els) {
+            $defines{$i} = 1;
+        }
+    } elsif ($ARGV[0] eq '-debug') {
+        $debug = 1;
+    } else {
+        print "$usage";
+        exit 0;
+    }
+    shift;
+}
+
+# I used to just do 
+#   open(FIND, "find . -name \\*.h |")
+# but there are dependencies between header files, and certain orders
+# look nicer than others, so we have an explicit list here:
+# Also, we now send output to multiple files, and their names are indicated
+# by comments in the header files.
+@headers = 
+    (
+     './instrlist.h',
+     './lib/globals_shared.h', # defs
+     './globals.h',
+     './x86/arch_exports.h', # encode routines
+     './x86/proc.h',
+     './module_shared.h',
+     './x86/instrument.h',
+     './x86/instr.h',
+     './x86/instr_create.h',
+     './x86/decode.h',       # OPSZ_ consts, decode routines
+     './x86/decode_fast.h',  # decode routines
+     './x86/disassemble.h',  # disassemble routines
+     './fragment.h',         # binary tracedump format
+     './win32/os_private.h', # rsrc section walking
+     './hotpatch.c',         # probe api
+     './lib/dr_config.h',
+     );
+
+# PR 214947: VMware retroactively holds the copyright.
+$copyright = q+/* **********************************************************
+ * Copyright (c) 2002-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
++;
+
+if ($header) {
+    $outfile_init = 0;
+} else {
+    open(OUT, ">-"); # stdout
+    if ($filter) {
+        # linker version script w/ no versions, just global and local symbol lists
+        # except old ld requires version -- ok, we'll put in one
+        print OUT "DYNAMORIO_0.9.5 {\nglobal:\n";
+    }
+}
+
+foreach $file (@headers) {
+    open(IN, "< $file") || die "Error: Couldn't open $file for input\n";
+    if ($debug) {
+        print stderr "Working on $file\n";
+        print OUT "\n/* from $file ----------------------------- */\n";
+    }
+    $output_routine = 0;
+    $output_directly = 0;
+    $output_verbatim = 0;
+    $did_output_something = 0;
+    $prev_define = 0;
+    $in_define = 0;
+    while (<IN>) {
+        chop;
+        # handle DOS end-of-line:
+        if ($_ =~ /\r$/) { chop; };
+        $l = $_;
+
+        if ($l =~ /^DR_API/) {
+            $output_routine = 1;
+            $did_output_something = 1;
+            $type = "";
+        } elsif ($header && $l =~ /DR_API EXPORT TOFILE ([A-Za-z_0-9\.]+)/) {
+            $outfile_init = 1;
+            $outfile = $1;
+            if (!defined($files{$outfile})) {
+                $files{$outfile} = $outfile;
+                $wrapdef{$outfile} = $outfile;
+                $wrapdef{$outfile} =~ s/\./_/g;
+                $wrapdef{$outfile} = "_" . uc($wrapdef{$outfile}) . "_";
+                if (! -e "$dir/$outfile") {
+                    open(OUT, "> $dir/$outfile") ||
+                        die "Error: Couldn't open $dir/$outfile for output\n";
+                    print OUT "$copyright";
+                    print OUT "#ifndef $wrapdef{$outfile}\n#define $wrapdef{$outfile} 1\n\n";
+                } else {
+                    open(OUT, ">> $dir/$outfile") ||
+                        die "Error: Couldn't open $dir/$outfile for append\n";
+                }
+            } else {
+                # FIXME: should store original filehandle and re-use it
+                open(OUT, ">> $dir/$outfile") ||
+                    die "Error: Couldn't open $dir/$outfile for append\n";
+            }
+        } elsif ($header && $l =~ /DR_API EXPORT BEGIN/) {
+            $output_directly = 1;
+            $did_output_something = 1;
+        } elsif ($header && $l =~ /DR_API EXPORT VERBATIM/) {
+            $output_verbatim = 1;
+            $did_output_something = 1;
+        }
+        # if outputting verbatim, just output and grab the next line
+        # we can skip the following logic that interprets the #ifdefs,
+        # #ifndefs, etc.
+        elsif ($output_verbatim) {
+            if ($l =~ /DR_API EXPORT END/) {
+                $output_verbatim = 0;
+            }
+            else {
+                print OUT "$l\n";
+            }
+            next;
+        } 
+
+        # to handle defines (yes a hack...need a better solution):
+        # only a define on immediately prior line to DR_API or to
+        # EXPORT BEGIN will kill following output, until the very next
+        # endif
+        # assumption: only those defines passed to us count,
+        # and they're only used in simple "#ifdef" statements
+        # (can't use cpp b/c it also removes #if's around entire .h file,
+        # plus we want to keep #ifdef LINUX and #ifdef WINDOWS stuff)
+        if ($prev_define && ($output_routine || $output_directly)) {
+            $in_undefined = 1;
+            if ($debug) {
+                print stderr "\tSkipping!\n";
+            }
+        }
+        if ($in_undefined) {
+            if ($l =~ /^\#\s*endif/ || $l =~ /^\#\s*else/) {
+                $in_undefined = 0;
+                if ($l =~ /^\#\s*else/) {
+                    $kill_endif = 1;
+                }
+            }
+            # assumption: nothing else on endif line
+            $skip = 1;
+            # have to check for closings here
+            if ($output_routine && $l =~ /;/) {
+                $output_routine = 0;
+            }
+            if ($output_directly && $l =~ /DR_API EXPORT END/) {
+                $output_directly = 0;
+            }
+        } else {
+            $skip = 0;
+        }
+        if ($kill_endif && $l =~ /^\#\s*endif/) {
+            $kill_endif = 0;
+            $skip = 1;
+        }
+        if ($in_define || $in_define_keep) {
+            if ($l =~ /^\#\s*endif/) {
+                # we only support keep-defines being innermost
+                if ($in_define_keep) {
+                    $in_define_keep = 0;
+                } else {
+                    $in_define = 0;
+                    $skip = 1;
+                }
+            }
+        }
+        if ($l =~ /^\#\s*ifdef (\S+)/ || $l =~ /^\#\s*if defined\((\S+)\)/) {
+            # we want to keep all WINDOWS, LINUX, and X64 defines, so ignore those.
+            if ($1 eq "WINDOWS" || $1 eq "LINUX" || $1 eq "X64" || $1 eq "X86_64") {
+                $in_define_keep = 1;
+            } else {
+                if ($debug) {
+                    print stderr "Found ifdef $1 => $defines{$1}\n";
+                }
+                if (!defined($defines{$1})) {
+                    $prev_define = 1;
+                } elsif ($output_routine || $output_directly) {
+                    $in_define = 1;
+                }
+                $skip = 1;
+            }
+        } elsif ($l =~ /^\#\s*ifndef (\S+)/) {
+            # we want to keep all WINDOWS, LINUX, and X64 defines, so ignore those.
+            if ($1 eq "WINDOWS" || $1 eq "LINUX" || $1 eq "X64" || $1 eq "X86_64") {
+                $in_define_keep = 1;
+            } else {
+                if ($debug) {
+                    print stderr "Found ifndef $1 => $defines{$1}\n";
+                }
+                if (defined($defines{$1})) {
+                    $prev_define = 1;
+                } elsif ($output_routine || $output_directly) {
+                    $in_define = 1;
+                }
+                $skip = 1;
+            }
+        } else {
+            $prev_define = 0;
+        }
+        if ($skip) {
+            next;
+        }
+
+        if ($output_routine) {
+            if ($filter) {
+                # only export these guys for DYNAMORIO_IR_EXPORTS
+                if (defined($defines{DYNAMORIO_IR_EXPORTS})) {
+                    # symbol export list (-filter option)
+                    if ($l =~ /^([A-Za-z0-9_]+)\s*\(/ ||
+                        # order is important: 2nd line might pick up _IF_X64
+                        # inside param list
+                        $l =~ /^[a-zA-Z_].*\s+([A-Za-z0-9_]+)\s*\(/) {
+                        print OUT "$1;\n";
+                    }
+                }
+                if ($l =~ /;\s*$/ &&
+                    $l !~ /^\s*\*/ && $l !~ /^\s*\/\*/) { # ignore ; inside comment
+                    $output_routine = 0;
+                }
+            } else {
+                if ($header && !$outfile_init) {
+                    die "Error: no initial output file specified\n";
+                }
+                unless ($debug) {
+                    $l =~ s/^DR_API\s*//;
+                }
+                if ($type eq "") {
+                    if ($l =~ /^\s*[A-Za-z_]+/) {
+                        $l =~ /^\s*([A-Za-z0-9_]+\s*\*?)/;
+                        $type = $1;
+                        $type =~ s/\s*$//;
+                    }
+                }
+                $l =~ s/dcontext_t *\*dcontext/void *drcontext/;
+                if ($l =~ /\);\s*$/) {
+                    $output_routine = 0;
+                    if ($debug || $header) {
+                        print OUT "$l\n";
+                    } else {
+                        $l =~ s/;//;
+                        print OUT "$l\n";
+                        if ($type =~ /\*/) {
+                            $ret = "\treturn NULL;\n";
+                        } elsif ($type eq "void") {
+                            $ret = "";
+                        } elsif ($type eq "bool") {
+                            $ret = "\treturn false;\n";
+                        } elsif ($type eq "float") {
+                            $ret = "\treturn 0.f;\n";
+                        } elsif ($type eq "int" || $type eq "uint") {
+                            $ret = "\treturn 0;\n";
+                        } else {
+                            # static to avoid "uninitialized var" compiler warnings
+                            $ret = "\tstatic $type bogus;\n\treturn bogus;\n";
+                        }
+                        print OUT "{\n$ret}\n";
+                    }
+                } else {
+                    print OUT "$l\n";
+                }
+            }
+        } elsif ($output_directly) {
+            # instr_create.h needs this in its raw output for x64, but keep var name
+            $l =~ s/dcontext_t *\*dcontext/void *dcontext/;
+            # let's keep blank lines if we didn't create them by stripping
+            # everything out
+            if ($l =~ /^\s*$/) {
+                print OUT "\n";
+            }
+            elsif ($l !~ /DR_API/) {
+                if ($l =~ /OP_/) {
+                    # remove pointers into decode tables
+                    $l =~ s/(OP_[a-zA-Z0-9_]*,) *\/\*[^\*]*\*\/(.*)/\1\2/;
+                }
+                # PR 227381: auto-insert doxygen comments for REG_ enum lines without any
+                if ($file =~ "/instr.h" &&
+                    $l =~ /^ *[RS]EG_/ && $l !~ /\/\*\*</) {
+                    $l =~ s|([RS]EG_)(\w+), *|\1\2, /**< The "\L\2" register. */\n    |g;
+                }
+                # Strip out FIXME comments
+                $l =~ s/\/\* FIXME.*\*\///;
+                if ($l !~ /^\s*$/) {
+                    print OUT "$l\n";
+                }
+            }
+            if ($l =~ /DR_API EXPORT END/) {
+                $output_directly = 0;
+                print OUT "\n";
+            }
+        }
+    }
+    if ($did_output_something && !$filter) {
+        print OUT "\n";
+    }
+    close(IN);
+}
+
+if ($header) {
+    foreach $fname (keys(%files)) {
+        # FIXME: should store original filehandle, and close it here
+        open(OUT, ">> $dir/$files{$fname}") ||
+            die "Error: Couldn't open $dir/$files{$fname} for append\n";
+        print OUT "\n#endif /* $wrapdef{$fname} */\n";
+        close(OUT);
+    }
+} else {
+    if ($filter) {
+        if (defined($defines{DR_APP_EXPORTS})) {
+            # now add in the APP_API routines -- ok to list symbol not found,
+            # so we need no hacks to remove the LOGPC ones based on the defines:
+            $app = `grep 'DR_APP_API.*;' ./lib/dr_app.h`;
+            $app =~ s/DR_APP_API [a-z]+ ([A-Za-z_]+)\(void\);/\1;/g;
+            print OUT $app;
+        }
+        # dynamorio_app_init and dynamorio_app_take_over are used 
+        # by the linux injector
+        print OUT "dynamorio_app_init;\n";
+        print OUT "dynamorio_app_take_over;\n";
+        # add in our_std*, which are hard to match w/ DR_API and I'm in a hurry
+        # (should switch to visibility attributes now that we upgraded gcc)
+        print OUT "our_stdout;\n";
+        print OUT "our_stderr;\n";
+        # linker version script w/ no versions, just global and local symbol lists
+        print OUT "\nlocal: *;\n};\n";
+    }
+    close(OUT);
+}
+
diff --git a/core/lib/globals_shared.h b/core/lib/globals_shared.h
new file mode 100644
index 000000000..a6c312465
--- /dev/null
+++ b/core/lib/globals_shared.h
@@ -0,0 +1,1568 @@
+/* **********************************************************
+ * Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+
+/*
+ * globals_shared.h
+ *
+ * definitions to be shared with core-external modules
+ *
+ */
+
+/* FIXME: dynamorio references should be removed throughout */
+
+#ifndef _GLOBALS_SHARED_H_
+#define _GLOBALS_SHARED_H_
+
+/* if not Linux, we assume Windows */
+#ifndef LINUX
+#  ifndef WINDOWS
+#    define WINDOWS
+#  endif
+#endif
+
+/* DR_API EXPORT TOFILE dr_defines.h */
+/* DR_API EXPORT BEGIN */
+/****************************************************************************
+ * GENERAL TYPEDEFS AND DEFINES
+ */
+
+/**
+ * @file dr_defines.h
+ * @brief Basic defines and type definitions.
+ */
+
+#ifdef API_EXPORT_ONLY
+/* A client's target operating system and architecture must be specified. */
+#if (!defined(LINUX) && !defined(WINDOWS)) || (defined(LINUX) && defined(WINDOWS))
+# error Target operating system unspecified: must define either WINDOWS xor LINUX
+#endif
+#endif
+
+#ifdef API_EXPORT_ONLY
+#if (!defined(X86_64) && !defined(X86_32)) || (defined(X86_64) && defined(X86_32))
+# error Target architecture unspecified: must define either X86_64 xor X86_32
+#endif
+#endif
+
+#if defined(X86_64) && !defined(X64)
+# define X64
+#endif
+
+#ifdef API_EXPORT_ONLY
+#ifdef WINDOWS
+#  define WIN32_LEAN_AND_MEAN
+#  include <windows.h>
+#  include <winbase.h>
+#else
+#  include <stdio.h>
+#  include <stdlib.h>
+#endif
+#endif
+/* DR_API EXPORT END */
+#include <limits.h>  /* for USHRT_MAX */
+#ifdef LINUX
+#  include <sys/types.h>        /* Fix for case 5341. */
+#  include <signal.h>
+#endif
+/* DR_API EXPORT BEGIN */
+#ifdef WINDOWS
+/* allow nameless struct/union */
+#  pragma warning(disable: 4201)
+#endif
+
+#ifdef WINDOWS
+#  define DR_EXPORT __declspec(dllexport)
+#  define LINK_ONCE __declspec(selectany)
+#  define ALIGN_VAR(x) __declspec(align(x))
+#  define inline __inline
+#  define INLINE_FORCED __forceinline
+#else
+#  define DR_EXPORT
+/* we assume gcc is being used */
+#  define LINK_ONCE __attribute__ ((weak))
+#  define ALIGN_VAR(x) __attribute__ ((aligned (x)))
+#  define inline __inline__
+#  define INLINE_FORCED inline
+#endif
+
+#ifdef AVOID_API_EXPORT
+/* We want a consistent size so we stay away from MAX_PATH.
+ * MAX_PATH is 260 on Windows, but 4096 on Linux, should up this.
+ * FIXME: should undef MAX_PATH and define it to an error-producing value
+ * and clean up all uses of it
+ */
+#endif
+/** Cross-platform maximum file path length. */
+#define MAXIMUM_PATH      260 
+
+/* DR_API EXPORT END */
+/* DR_API EXPORT VERBATIM */
+#ifndef NULL
+#  define NULL (0)
+#endif
+
+#ifndef __cplusplus
+#  ifndef DR_DO_NOT_DEFINE_bool
+typedef int bool;
+#  endif
+#  define true  (1)
+#  define false (0)
+#endif
+
+#ifndef DR_DO_NOT_DEFINE_uint
+typedef unsigned int uint;
+#endif
+#ifndef DR_DO_NOT_DEFINE_ushort
+typedef unsigned short ushort;
+#endif
+#ifndef DR_DO_NOT_DEFINE_byte
+typedef unsigned char byte;
+#endif
+#ifndef DR_DO_NOT_DEFINE_sbyte
+typedef signed char sbyte;
+#endif
+typedef byte * app_pc;
+
+typedef void (*generic_func_t) ();
+
+#ifdef WINDOWS
+#  ifndef DR_DO_NOT_DEFINE_uint64
+typedef unsigned __int64 uint64;
+#  endif
+#  ifndef DR_DO_NOT_DEFINE_int64
+typedef __int64 int64;
+#  endif
+#  ifdef X64
+typedef __int64 ssize_t;
+#  else
+typedef int ssize_t;
+#  endif
+#  define INT64_FORMAT "I64"
+#else /* Linux */
+#  ifdef X64
+#    ifndef DR_DO_NOT_DEFINE_uint64
+typedef unsigned long int uint64;
+#    endif
+#    ifndef DR_DO_NOT_DEFINE_int64
+typedef long int int64;
+#    endif
+#    define INT64_FORMAT "l"
+#  else
+#    ifndef DR_DO_NOT_DEFINE_uint64
+typedef unsigned long long int uint64;
+#    endif
+#    ifndef DR_DO_NOT_DEFINE_int64
+typedef long long int int64;
+#    endif
+#    define INT64_FORMAT "ll"
+#  endif
+#endif
+/* DR_API EXPORT END */
+/* DR_API EXPORT BEGIN */
+
+/* a register value: could be of any type; size is what matters. */
+#ifdef X64
+typedef uint64 reg_t;
+#else
+typedef uint reg_t;
+#endif
+/* integer whose size is based on pointers: ptr diff, mask, etc. */
+typedef reg_t ptr_uint_t;
+#ifdef X64
+typedef int64 ptr_int_t;
+#else
+typedef int ptr_int_t;
+#endif
+/* for memory region sizes, use size_t */
+
+#ifdef WINDOWS
+typedef ptr_uint_t thread_id_t;
+typedef ptr_uint_t process_id_t;
+#else /* Linux */
+typedef pid_t thread_id_t;
+typedef pid_t process_id_t;
+#endif
+
+#ifdef API_EXPORT_ONLY
+#ifdef WINDOWS
+/* since a FILE cannot be used outside of the DLL it was created in,
+ * we have to use HANDLE on Windows
+ * we hide the distinction behind the file_t type
+ */
+typedef HANDLE file_t;
+/** The sentinel value for an invalid file_t. */
+#  define INVALID_FILE INVALID_HANDLE_VALUE
+/* dr_get_stdout_file and dr_get_stderr_file return errors as 
+ * INVALID_HANDLE_VALUE.  We leave INVALID_HANDLE_VALUE as is, 
+ * since it equals INVALID_FILE
+ */
+/** The file_t value for standard output. */
+#  define STDOUT (dr_get_stdout_file())
+/** The file_t value for standard error. */
+#  define STDERR (dr_get_stderr_file())
+#endif
+#endif
+
+#ifdef LINUX
+typedef int file_t;
+/** The sentinel value for an invalid file_t. */
+#  define INVALID_FILE -1
+/** Allow use of stdout after the application closes it. */
+extern file_t our_stdout;
+/** Allow use of stderr after the application closes it. */
+extern file_t our_stderr;
+/** The file_t value for standard output. */
+#  define STDOUT (our_stdout == INVALID_FILE ? stdout->_fileno : our_stdout)
+/** The file_t value for standard error. */
+#  define STDERR (our_stderr == INVALID_FILE ? stderr->_fileno : our_stderr)
+#endif
+
+#ifdef AVOID_API_EXPORT
+/* Note that we considered using a 128-bit GUID for the client ID,
+ * but decided it was unnecessary since the client registration
+ * routine will complain about conflicting IDs.  Also, we're storing
+ * this value in the registry, so no reason to make it any longer
+ * than we have to.
+ */
+#endif
+/**
+ * ID used to uniquely identify a client.  This value is set at
+ * client registration and passed to the client in dr_init().
+ */
+typedef uint client_id_t;
+
+#ifdef API_EXPORT_ONLY
+/**
+ * Internal structure of opnd_t is below abstraction layer.
+ * But compiler needs to know field sizes to copy it around
+ */
+typedef struct {
+#ifdef X64
+    uint black_box_uint;
+    uint64 black_box_uint64;
+#else
+    uint black_box_uint[3];
+#endif
+} opnd_t;
+
+/**
+ * Internal structure of instr_t is below abstraction layer, but we
+ * provide its size so that it can be used in stack variables
+ * instead of always allocated on the heap.
+ */
+typedef struct {
+#ifdef X64
+    uint black_box_uint[26];
+#else
+    uint black_box_uint[16];
+#endif
+} instr_t;
+#endif /* API_EXPORT_ONLY */
+
+/* DR_API EXPORT END */
+
+#ifndef IN
+# define IN /* marks input param */
+#endif
+#ifndef OUT
+# define OUT /* marks output param */
+#endif
+#ifndef INOUT
+# define INOUT /* marks input+output param */
+#endif
+
+#ifdef X64
+# define POINTER_MAX ULLONG_MAX
+# define SSIZE_T_MAX LLONG_MAX
+# define POINTER_MAX_32BIT ((ptr_uint_t)UINT_MAX) 
+#else
+# define POINTER_MAX UINT_MAX
+# define SSIZE_T_MAX INT_MAX
+#endif
+
+#define PTR_UINT_0       ((ptr_uint_t)0U)
+#define PTR_UINT_1       ((ptr_uint_t)1U)
+#define PTR_UINT_MINUS_1 ((ptr_uint_t)-1)
+
+#define MAX_CLIENT_LIBS 16
+
+/* FIXME: these macros double-evaluate their args -- if we can't trust
+ * compiler to do CSE, should we replace w/ inline functions?  would need
+ * separate signed and unsigned versions
+ */
+#ifndef DR_DO_NOT_DEFINE_MAX_MIN
+# define MAX(x,y) ((x) >= (y) ? (x) : (y))
+# define MIN(x,y) ((x) <= (y) ? (x) : (y))
+#endif
+#define PTR_UINT_ABS(x) ((x) < 0 ? -(x) : (x))
+
+/* check if all bits in mask are set in var */
+#define TESTALL(mask, var) (((mask) & (var)) == (mask))
+/* check if any bit in mask is set in var */
+#define TESTANY(mask, var) (((mask) & (var)) != 0)
+/* check if a single bit is set in var */
+#define TEST TESTANY
+
+#define BOOLS_MATCH(a, b) (((a) && (b)) || (!(a) && !(b)))
+
+/* macros to make conditional compilation look prettier */
+#ifdef DEBUG
+# define IF_DEBUG(x) x
+# define _IF_DEBUG(x) , x
+# define IF_DEBUG_ELSE(x, y) x
+# define IF_DEBUG_ELSE_0(x) (x)
+# define IF_DEBUG_ELSE_NULL(x) (x)
+#else
+# define IF_DEBUG(x)
+# define _IF_DEBUG(x)
+# define IF_DEBUG_ELSE(x, y) y
+# define IF_DEBUG_ELSE_0(x) 0
+# define IF_DEBUG_ELSE_NULL(x) (NULL)
+#endif
+
+#ifdef INTERNAL
+# define IF_INTERNAL(x) x
+# define IF_INTERNAL_ELSE(x,y) x
+#else
+# define IF_INTERNAL(x)
+# define IF_INTERNAL_ELSE(x,y) y
+#endif
+
+#ifdef WINDOWS
+# define IF_WINDOWS(x) x
+# define IF_WINDOWS_(x) x, 
+# define _IF_WINDOWS(x) , x
+# define IF_WINDOWS_ELSE_0(x) (x)
+# define IF_WINDOWS_ELSE(x,y) (x)
+# define IF_WINDOWS_ELSE_NP(x,y) x
+# define IF_LINUX(x)
+# define IF_LINUX_ELSE(x,y) y
+# define IF_LINUX_(x)
+# define _IF_LINUX(x)
+#else
+# define IF_WINDOWS(x)
+# define IF_WINDOWS_(x)
+# define _IF_WINDOWS(x)
+# define IF_WINDOWS_ELSE_0(x) (0)
+# define IF_WINDOWS_ELSE(x,y) (y)
+# define IF_WINDOWS_ELSE_NP(x,y) y
+# define IF_LINUX(x) x
+# define IF_LINUX_ELSE(x,y) x
+# define IF_LINUX_(x) x,
+# define _IF_LINUX(x) , x
+#endif
+
+#ifdef VMX86_SERVER
+# define IF_VMX86(x) x
+# define IF_VMX86_ELSE(x,y) x
+#else
+# define IF_VMX86(x)
+# define IF_VMX86_ELSE(x,y) y
+#endif
+
+#ifdef LINUX
+# ifdef HAVE_TLS
+#  define IF_HAVE_TLS_ELSE(x, y) x
+#  define IF_NOT_HAVE_TLS(x)
+# else
+#  define IF_HAVE_TLS_ELSE(x, y) y
+#  define IF_NOT_HAVE_TLS(x) x
+# endif
+#else
+/* Windows always has TLS.  Better to generally define HAVE_TLS instead? */
+# define IF_HAVE_TLS_ELSE(x, y) x
+# define IF_NOT_HAVE_TLS(x)
+#endif
+
+#if defined(WINDOWS) && !defined(NOT_DYNAMORIO_CORE)
+# define IF_WINDOWS_AND_CORE(x) x
+#else
+# define IF_WINDOWS_AND_CORE(x)
+#endif
+
+#ifdef PROGRAM_SHEPHERDING
+# define IF_PROG_SHEP(x) x
+#else
+# define IF_PROG_SHEP(x)
+#endif
+
+#if defined(PROGRAM_SHEPHERDING) && defined(RCT_IND_BRANCH)
+# define IF_RCT_IND_BRANCH(x) x
+#else
+# define IF_RCT_IND_BRANCH(x)
+#endif
+
+#if defined(PROGRAM_SHEPHERDING) && defined(RETURN_AFTER_CALL)
+# define IF_RETURN_AFTER_CALL(x) x
+# define IF_RETURN_AFTER_CALL_ELSE(x, y) x
+#else
+# define IF_RETURN_AFTER_CALL(x)
+# define IF_RETURN_AFTER_CALL_ELSE(x, y) y
+#endif
+
+#ifdef HOT_PATCHING_INTERFACE
+# define IF_HOTP(x) x
+#else
+# define IF_HOTP(x)
+#endif
+
+#ifdef CLIENT_INTERFACE
+# define IF_CLIENT_INTERFACE(x) x
+# define IF_CLIENT_INTERFACE_ELSE(x, y) x
+# define _IF_CLIENT_INTERFACE(x) , x
+# define _IF_NOT_CLIENT_INTERFACE(x)
+/* _IF_CLIENT_INTERFACE is too long */
+# define _IF_CLIENT(x) , x
+#else
+# define IF_CLIENT_INTERFACE(x)
+# define IF_CLIENT_INTERFACE_ELSE(x, y) y
+# define _IF_CLIENT_INTERFACE(x)
+# define _IF_NOT_CLIENT_INTERFACE(x) , x
+# define _IF_CLIENT(x)
+#endif
+
+#ifdef CUSTOM_TRACES
+# define IF_CUSTOM_TRACES(x) x
+#else
+# define IF_CUSTOM_TRACES(x)
+#endif
+
+#ifdef DR_APP_EXPORTS
+# define IF_APP_EXPORTS(x) x
+#else
+# define IF_APP_EXPORTS(x)
+#endif
+
+#ifdef GBOP
+# define IF_GBOP(x) x
+#else
+# define IF_GBOP(x)
+#endif
+
+#ifdef PROCESS_CONTROL
+# define IF_PROC_CTL(x) x
+#else
+# define IF_PROC_CTL(x)
+#endif
+
+#ifdef KSTATS
+# define IF_KSTATS(x) x
+#else
+# define IF_KSTATS(x)
+#endif
+
+/* DR_API EXPORT TOFILE dr_defines.h */
+/* DR_API EXPORT BEGIN */
+#ifdef X64
+# define IF_X64(x) x
+# define IF_X64_ELSE(x, y) x
+# define IF_X64_(x) x,
+# define _IF_X64(x) , x
+# define IF_NOT_X64(x)
+# define _IF_NOT_X64(x)
+#else
+# define IF_X64(x)
+# define IF_X64_ELSE(x, y) y
+# define IF_X64_(x)
+# define _IF_X64(x)
+# define IF_NOT_X64(x) x
+# define _IF_NOT_X64(x) , x
+#endif
+/* DR_API EXPORT END */
+
+typedef enum {
+    SYSLOG_INFORMATION = 0x1,
+    SYSLOG_WARNING     = 0x2,
+    SYSLOG_ERROR       = 0x4,
+    SYSLOG_CRITICAL    = 0x8,
+    SYSLOG_NONE        = 0x0,
+    SYSLOG_ALL         = SYSLOG_INFORMATION | SYSLOG_WARNING | SYSLOG_ERROR | SYSLOG_CRITICAL
+} syslog_event_type_t;
+
+#define DYNAMO_OPTION(opt) (ASSERT_OWN_READWRITE_LOCK(IS_OPTION_STRING(opt), \
+                            &options_lock), dynamo_options.opt)
+/* For use where we don't want ASSERT defines. Currently only used in FATAL_USAGE_ERROR
+ * so it can be used in files that require all asserts to be client asserts. */
+#define DYNAMO_OPTION_NOT_STRING(opt) (dynamo_options.opt)
+
+#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
+#define EXPOSE_INTERNAL_OPTIONS
+#endif
+
+#ifdef EXPOSE_INTERNAL_OPTIONS
+  /* Use only for experimental non-release options */
+  /* Internal option value can be set on the command line only in INTERNAL builds */
+  /* We should remove the ASSERT if we convert an internal option into non-internal */
+# define INTERNAL_OPTION(opt) ((IS_OPTION_INTERNAL(opt)) ? (DYNAMO_OPTION(opt)) : \
+                               (ASSERT_MESSAGE("non-internal option argument "#opt, false), \
+                                DYNAMO_OPTION(opt)))
+#else
+  /* Use only for experimental non-release options, 
+     default value is assumed and command line options are ignored */
+  /* We could use IS_OPTION_INTERNAL(opt) ? to determine whether an option is defined as INTERNAL in
+     optionsx.h and have that be the only place to modify to transition between internal and external options.
+     The compiler should be able to eliminate the inappropriate part of the constant expression,
+     if the specific option is no longer defined as internal so we don't have to modify the code.
+     Still I'd rather have explicit uses of DYNAMO_OPTION or INTERNAL_OPTION for now.
+  */
+# define INTERNAL_OPTION(opt) DEFAULT_INTERNAL_OPTION_VALUE(opt)
+#endif /* EXPOSE_INTERNAL_OPTIONS */
+
+#ifdef LINUX
+#ifndef DR_DO_NOT_DEFINE_uint32
+ typedef unsigned int uint32;
+#endif
+/* Note: Linux paths are longer than the 260 limit on Windows, 
+   yet we can't afford the 4K of PATH_MAX from <limits.h> */
+typedef uint64 timestamp_t;
+#  define NAKED
+#  define ZHEX32_FORMAT_STRING "%08x"
+#  define HEX32_FORMAT_STRING "%x"
+
+#else /* WINDOWS */
+/* We no longer include windows.h here, in order to more easily include
+ * this file for types like reg_t in hotpatch_interface.h and not force
+ * hotpatch compilation to point at windows include directories.
+ * By not including it here we must define MAX_PATH and include windows.h
+ * for NOT_DYNAMORIO_CORE in options.c and drmarker.c for share/ compilation.
+ */
+#  define MAX_PATH 260 /* from winbase.h */
+#ifndef DR_DO_NOT_DEFINE_uint
+ typedef unsigned int uint;
+#endif
+#ifndef DR_DO_NOT_DEFINE_uint32
+ typedef unsigned int uint32;
+#endif
+/* NOTE : unsigned __int64 not convertible to double */
+typedef unsigned __int64 uint64;
+typedef __int64 int64;
+#  ifdef X64
+typedef int64 ssize_t;
+#  else
+typedef int ssize_t;
+#  endif
+/* PR 232882: %I32x printf format code not supported on NT's ntdll */
+#  ifdef X64
+#    define ZHEX32_FORMAT_STRING "%08I32x"
+#    define HEX32_FORMAT_STRING "%I32x"
+#  else
+#    define ZHEX32_FORMAT_STRING "%08x"
+#    define HEX32_FORMAT_STRING "%x"
+#  endif
+/* VC6 doesn't define the standard ULLONG_MAX */
+#  if _MSC_VER <= 1200 && !defined(ULLONG_MAX)
+#    define ULLONG_MAX _UI64_MAX 
+#  endif
+typedef uint64 timestamp_t;
+#  define NAKED __declspec( naked )
+#endif
+
+#define FIXED_TIMESTAMP_FORMAT "%10"INT64_FORMAT"u"
+
+/* DR_API EXPORT TOFILE dr_defines.h */
+/* DR_API EXPORT BEGIN */
+#define UINT64_FORMAT_CODE INT64_FORMAT"u"
+#define INT64_FORMAT_CODE INT64_FORMAT"d"
+#define UINT64_FORMAT_STRING "%"UINT64_FORMAT_CODE
+#define INT64_FORMAT_STRING "%"INT64_FORMAT_CODE
+#define HEX64_FORMAT_STRING "%"INT64_FORMAT"x"
+#define ZHEX64_FORMAT_STRING "%016"INT64_FORMAT"x"
+#ifdef API_EXPORT_ONLY
+#define ZHEX32_FORMAT_STRING "%08x"
+#define HEX32_FORMAT_STRING "%x"
+/* Convenience defines for cross-platform printing */
+#ifdef X64
+# define PFMT ZHEX64_FORMAT_STRING
+# define SZFMT INT64_FORMAT_STRING
+#else
+# define PFMT ZHEX32_FORMAT_STRING
+# define SZFMT "%d"
+#endif
+#endif
+/* DR_API EXPORT END */
+/* workaround for lack of defines stack and assumption in genapi.pl
+ * that ignored defines are outermost */
+/* DR_API EXPORT BEGIN */
+#ifdef API_EXPORT_ONLY
+#define PFX "0x"PFMT
+#endif
+/* DR_API EXPORT END */
+
+/* Statistics are 64-bit for x64, 32-bit for x86, so we don't have overflow
+ * for pointer-sized stats.
+ * TODO PR 227994: use per-statistic types for variable bitwidths.
+ */
+#ifdef X64
+typedef int64 stats_int_t;
+#else
+typedef int stats_int_t;
+#endif
+
+/* For printing pointers: we do not use %p as gcc prepends 0x and uses
+ * lowercase while cl does not prepend, puts leading 0's, and uses uppercase.
+ * Also, the C standard does not allow min width for %p.
+ * Two macros:
+ * - PFMT == Pointer Format == with leading zeros
+ * - PIFMT == Pointer Integer Format == no leading zeros
+ * Convenience macros to shrink long lines:
+ * - PFX == Pointer Format with leading 0x
+ * - PIFX == Pointer Integer Format with leading 0x
+ * For printing memory region sizes:
+ * - SZFMT == Size Format
+ * - SSZFMT == Signed Size Format
+ * For printing 32-bit integers as hex we use %x.  We could use a macro
+ * there and then disallow %x, to try and avoid 64-bit printing bugs,
+ * but it wouldn't be a panacea.
+ */
+#define L_UINT64_FORMAT_STRING L"%"L_EXPAND_LEVEL(UINT64_FORMAT_CODE)
+#ifdef X64
+#  define PFMT ZHEX64_FORMAT_STRING
+#  define PIFMT HEX64_FORMAT_STRING
+#  define SZFMT UINT64_FORMAT_STRING
+#  define SSZFMT INT64_FORMAT_STRING
+#  define SZFC UINT64_FORMAT_CODE
+#  define SSZFC INT64_FORMAT_CODE
+#else
+#  define PFMT ZHEX32_FORMAT_STRING
+#  define PIFMT HEX32_FORMAT_STRING
+#  define SZFMT "%u"
+#  define SSZFMT "%d"
+#  define SZFC "u"
+#  define SSZFC "d"
+#endif
+#define L_PFMT L"%016"L_EXPAND_LEVEL(INT64_FORMAT)L"x"
+#define PFX "0x"PFMT
+#define PIFX "0x"PIFMT
+
+/* printf code for {thread,process}_id_t */
+#ifdef WINDOWS
+# define IDFMT SZFMT
+#else
+# define IDFMT "%d"
+#endif
+
+/* Maximum length of any registry parameter. Note that some are further
+ * restricted to MAXIMUM_PATH from their usage. */
+#define MAX_REGISTRY_PARAMETER 512
+/* Maximum length of option string in the registry */
+#define MAX_OPTIONS_STRING     512
+/* Maximum length of any individual list option's string */
+#define MAX_LIST_OPTION_LENGTH 512
+/* Maximum length of the path secified by a path option */
+#define MAX_PATH_OPTION_LENGTH MAXIMUM_PATH
+/* Maximum length of any individual option. */
+#define MAX_OPTION_LENGTH      512
+
+#define MAX_PARAMNAME_LENGTH 64
+
+/* Arbitrary debugging-only module name length maximum */
+#define MAX_MODNAME_INTERNAL 64
+
+/* Maximum string representation of a DWORD:
+ * 0x80000000 = -2147483648 -> 11 characters + 1 for null termination
+ */
+#define MAX_DWORD_STRING_LENGTH 12
+
+typedef char pathstring_t[MAX_PATH_OPTION_LENGTH];
+/* The liststring_t type is assumed to contain ;-separated values that are
+ * appended to if multiple option instances are specified
+ */
+typedef char liststring_t[MAX_LIST_OPTION_LENGTH];
+
+/* convenience macros for secure string buffer operations */
+#define BUFFER_SIZE_BYTES(buf)      sizeof(buf)
+#define BUFFER_SIZE_ELEMENTS(buf)   (BUFFER_SIZE_BYTES(buf) / sizeof(buf[0]))
+#define BUFFER_LAST_ELEMENT(buf)    buf[BUFFER_SIZE_ELEMENTS(buf) - 1]
+#define NULL_TERMINATE_BUFFER(buf)  BUFFER_LAST_ELEMENT(buf) = 0
+
+#define BUFFER_ROOM_LEFT_W(wbuf) (BUFFER_SIZE_ELEMENTS(wbuf) - wcslen(wbuf) - 1)
+#define BUFFER_ROOM_LEFT(abuf) (BUFFER_SIZE_ELEMENTS(abuf) - strlen(abuf) - 1)
+
+/* strncat() calls require termination after each to be safe, especially if
+ * cating repeatedly.  For example see hotpatch.c:hotp_load_hotp_dlls().
+ */
+#define CAT_AND_TERMINATE(buf, str) do {        \
+      strncat(buf, str, BUFFER_ROOM_LEFT(buf)); \
+      NULL_TERMINATE_BUFFER(buf);               \
+  } while(0)
+
+
+/* platform-independent */
+
+#define EXPANDSTR(x) #x
+#define STRINGIFY(x) EXPANDSTR(x)
+
+/* Allow custom builds to specify the product name */
+#ifdef CUSTOM_PRODUCT_NAME
+#  define PRODUCT_NAME STRINGIFY(CUSTOM_PRODUCT_NAME)
+#else
+#  define PRODUCT_NAME "DynamoRIO"
+#endif
+/* I initially was going to use "VMware, Inc." for the registry and eventlog,
+ * since other VMware products use the longer name in the registry,
+ * but both Jim's original comment here about "alphanum only" and memories
+ * of issues w/ eventlog Araksha => Determin[a] have made us cautious
+ * (though in my tests on win2k the longer name worked fine for eventlog).
+ * Update:
+ * PR 323321: split eventlog from reg name so we can use longer name in reg.
+ */
+#define COMPANY_NAME "VMware, Inc." /* used for reg key */
+#define COMPANY_NAME_EVENTLOG "VMware" /* used for event log */
+#define COMPANY_LONG_NAME "VMware, Inc."
+
+#ifdef BUILD_NUMBER
+#  define BUILD_NUMBER_STRING "build "STRINGIFY(BUILD_NUMBER)
+#else
+#  define BUILD_NUMBER_STRING "custom build"
+#  define BUILD_NUMBER (0)
+#endif
+
+#ifdef VERSION_NUMBER
+#  define VERSION_NUMBER_STRING "version "STRINGIFY(VERSION_NUMBER)
+#else
+#  define VERSION_NUMBER_STRING "internal version"
+#  define VERSION_NUMBER (0.0)
+#endif
+
+#ifdef HOT_PATCHING_INTERFACE
+#  define HOT_PATCHING_DLL_CACHE_PATH "\\lib\\hotp\\"
+#  define HOTP_MODES_FILENAME "ls-modes.cfg"
+#  define HOTP_POLICIES_FILENAME "ls-defs.cfg"
+#endif
+
+/* FIXME: the environment vars need to be renamed - it will be a pain */
+#define DYNAMORIO_VAR_HOME_ID       DYNAMORIO_HOME
+#define DYNAMORIO_VAR_LOGDIR_ID     DYNAMORIO_LOGDIR
+#define DYNAMORIO_VAR_OPTIONS_ID    DYNAMORIO_OPTIONS
+#define DYNAMORIO_VAR_AUTOINJECT_ID DYNAMORIO_AUTOINJECT
+#define DYNAMORIO_VAR_UNSUPPORTED_ID  DYNAMORIO_UNSUPPORTED
+#define DYNAMORIO_VAR_RUNUNDER_ID   DYNAMORIO_RUNUNDER
+#define DYNAMORIO_VAR_CMDLINE_ID    DYNAMORIO_CMDLINE
+#define DYNAMORIO_VAR_ONCRASH_ID    DYNAMORIO_ONCRASH
+#define DYNAMORIO_VAR_SAFEMARKER_ID DYNAMORIO_SAFEMARKER 
+/* NT only, value should be all CAPS and specifies a boot option to match */
+
+#define DYNAMORIO_VAR_CACHE_ROOT_ID   DYNAMORIO_CACHE_ROOT
+/* we have to create our own properly secured directory, that allows
+ * only trusted producers to create DLLs, and all publishers to read
+ * them.  Note that per-user directories may also be created by the trusted component
+ * allowing users to safely use their own private caches.
+ */
+#define DYNAMORIO_VAR_CACHE_SHARED_ID   DYNAMORIO_CACHE_SHARED
+/* a directory giving full write privileges to Everyone.  Therefore
+ * none of its contents can be trusted without explicit verification.
+ * Expected to be a subdirectory of DYNAMORIO_CACHE_ROOT.
+ */
+
+/* Location for persisted caches; FIXME: currently the same as the ASLR sharing dir */
+#define DYNAMORIO_VAR_PERSCACHE_ROOT_ID   DYNAMORIO_CACHE_ROOT
+/* FIXME case 9651: security model, etc. */
+#define DYNAMORIO_VAR_PERSCACHE_SHARED_ID   DYNAMORIO_CACHE_SHARED
+/* case 10255: use a suffix to distinguish from ASLR files in same dir
+ * DR persisted cache => "dpc"
+ */
+#define PERSCACHE_FILE_SUFFIX "dpc"
+
+#ifdef HOT_PATCHING_INTERFACE
+# define DYNAMORIO_VAR_HOT_PATCH_POLICES_ID  DYNAMORIO_HOT_PATCH_POLICIES
+# define DYNAMORIO_VAR_HOT_PATCH_MODES_ID  DYNAMORIO_HOT_PATCH_MODES
+#endif
+#ifdef PROCESS_CONTROL
+# define DYNAMORIO_VAR_APP_PROCESS_WHITELIST_ID  DYNAMORIO_APP_PROCESS_WHITELIST
+# define DYNAMORIO_VAR_ANON_PROCESS_WHITELIST_ID  DYNAMORIO_ANON_PROCESS_WHITELIST
+
+# define DYNAMORIO_VAR_APP_PROCESS_BLACKLIST_ID  DYNAMORIO_APP_PROCESS_BLACKLIST
+# define DYNAMORIO_VAR_ANON_PROCESS_BLACKLIST_ID  DYNAMORIO_ANON_PROCESS_BLACKLIST
+#endif
+
+#define DYNAMORIO_VAR_HOME       STRINGIFY(DYNAMORIO_VAR_HOME_ID)
+#define DYNAMORIO_VAR_LOGDIR     STRINGIFY(DYNAMORIO_VAR_LOGDIR_ID)
+#define DYNAMORIO_VAR_OPTIONS    STRINGIFY(DYNAMORIO_VAR_OPTIONS_ID)
+#define DYNAMORIO_VAR_AUTOINJECT STRINGIFY(DYNAMORIO_VAR_AUTOINJECT_ID)
+#define DYNAMORIO_VAR_UNSUPPORTED  STRINGIFY(DYNAMORIO_VAR_UNSUPPORTED_ID)
+#define DYNAMORIO_VAR_RUNUNDER   STRINGIFY(DYNAMORIO_VAR_RUNUNDER_ID)
+#define DYNAMORIO_VAR_CMDLINE    STRINGIFY(DYNAMORIO_VAR_CMDLINE_ID)
+#define DYNAMORIO_VAR_ONCRASH    STRINGIFY(DYNAMORIO_VAR_ONCRASH_ID)
+#define DYNAMORIO_VAR_SAFEMARKER STRINGIFY(DYNAMORIO_VAR_SAFEMARKER_ID)
+#define DYNAMORIO_VAR_CACHE_ROOT  STRINGIFY(DYNAMORIO_VAR_CACHE_ROOT_ID)
+#define DYNAMORIO_VAR_CACHE_SHARED  STRINGIFY(DYNAMORIO_VAR_CACHE_SHARED_ID)
+#define DYNAMORIO_VAR_PERSCACHE_ROOT  STRINGIFY(DYNAMORIO_VAR_PERSCACHE_ROOT_ID)
+#define DYNAMORIO_VAR_PERSCACHE_SHARED  STRINGIFY(DYNAMORIO_VAR_PERSCACHE_SHARED_ID)
+#ifdef HOT_PATCHING_INTERFACE
+# define DYNAMORIO_VAR_HOT_PATCH_POLICIES  STRINGIFY(DYNAMORIO_VAR_HOT_PATCH_POLICES_ID)
+# define DYNAMORIO_VAR_HOT_PATCH_MODES  STRINGIFY(DYNAMORIO_VAR_HOT_PATCH_MODES_ID)
+#endif
+#ifdef PROCESS_CONTROL
+# define DYNAMORIO_VAR_APP_PROCESS_WHITELIST \
+    STRINGIFY(DYNAMORIO_VAR_APP_PROCESS_WHITELIST_ID)
+# define DYNAMORIO_VAR_ANON_PROCESS_WHITELIST \
+    STRINGIFY(DYNAMORIO_VAR_ANON_PROCESS_WHITELIST_ID)
+
+# define DYNAMORIO_VAR_APP_PROCESS_BLACKLIST \
+    STRINGIFY(DYNAMORIO_VAR_APP_PROCESS_BLACKLIST_ID)
+# define DYNAMORIO_VAR_ANON_PROCESS_BLACKLIST \
+    STRINGIFY(DYNAMORIO_VAR_ANON_PROCESS_BLACKLIST_ID)
+#endif
+
+#ifdef LINUX
+
+#  define DYNAMORIO_VAR_EXECVE  "DYNAMORIO_POST_EXECVE"
+#  define DYNAMORIO_VAR_EXECVE_LOGDIR  "DYNAMORIO_EXECVE_LOGDIR"
+#  define L_IF_WIN(x) x
+
+#else /* WINDOWS */
+
+#  define EXPAND_LEVEL(str) str              /* expand one level */
+#  define L_EXPAND_LEVEL(str) L(str)
+#  define L(str) L ## str
+#  define LCONCAT(wstr,str) EXPAND_LEVEL(wstr)L("\\")L(str)
+#  define L_IF_WIN(x) L_EXPAND_LEVEL(x)
+
+/* unicode versions of shared names*/
+#  define L_DYNAMORIO_VAR_HOME        L_EXPAND_LEVEL(DYNAMORIO_VAR_HOME)
+#  define L_DYNAMORIO_VAR_LOGDIR      L_EXPAND_LEVEL(DYNAMORIO_VAR_LOGDIR)
+#  define L_DYNAMORIO_VAR_OPTIONS     L_EXPAND_LEVEL(DYNAMORIO_VAR_OPTIONS)
+#  define L_DYNAMORIO_VAR_AUTOINJECT  L_EXPAND_LEVEL(DYNAMORIO_VAR_AUTOINJECT)
+#  define L_DYNAMORIO_VAR_UNSUPPORTED L_EXPAND_LEVEL(DYNAMORIO_VAR_UNSUPPORTED)
+#  define L_DYNAMORIO_VAR_RUNUNDER    L_EXPAND_LEVEL(DYNAMORIO_VAR_RUNUNDER)
+#  define L_DYNAMORIO_VAR_CMDLINE     L_EXPAND_LEVEL(DYNAMORIO_VAR_CMDLINE)
+#  define L_DYNAMORIO_VAR_ONCRASH     L_EXPAND_LEVEL(DYNAMORIO_VAR_ONCRASH)
+#  define L_DYNAMORIO_VAR_SAFEMARKER  L_EXPAND_LEVEL(DYNAMORIO_VAR_SAFEMARKER)
+# ifdef HOT_PATCHING_INTERFACE
+#  define L_DYNAMORIO_VAR_HOT_PATCH_POLICIES  L_EXPAND_LEVEL(DYNAMORIO_VAR_HOT_PATCH_POLICIES)
+#  define L_DYNAMORIO_VAR_HOT_PATCH_MODES  L_EXPAND_LEVEL(DYNAMORIO_VAR_HOT_PATCH_MODES)
+# endif
+# ifdef PROCESS_CONTROL
+#  define L_DYNAMORIO_VAR_APP_PROCESS_WHITELIST \
+    L_EXPAND_LEVEL(DYNAMORIO_VAR_APP_PROCESS_WHITELIST)
+#  define L_DYNAMORIO_VAR_ANON_PROCESS_WHITELIST \
+    L_EXPAND_LEVEL(DYNAMORIO_VAR_ANON_PROCESS_WHITELIST)
+
+#  define L_DYNAMORIO_VAR_APP_PROCESS_BLACKLIST \
+    L_EXPAND_LEVEL(DYNAMORIO_VAR_APP_PROCESS_BLACKLIST)
+#  define L_DYNAMORIO_VAR_ANON_PROCESS_BLACKLIST \
+    L_EXPAND_LEVEL(DYNAMORIO_VAR_ANON_PROCESS_BLACKLIST)
+# endif
+
+#  define L_PRODUCT_NAME              L_EXPAND_LEVEL(PRODUCT_NAME)
+#  define L_COMPANY_NAME              L_EXPAND_LEVEL(COMPANY_NAME)
+#  define L_COMPANY_LONG_NAME         L_EXPAND_LEVEL(COMPANY_LONG_NAME)
+
+/* event log registry keys */
+#  define EVENTLOG_HIVE HKEY_LOCAL_MACHINE
+#  define EVENTLOG_NAME COMPANY_NAME_EVENTLOG
+#  define EVENTSOURCE_NAME PRODUCT_NAME /* should be different than logfile name */
+
+#  define EVENTLOG_REGISTRY_SUBKEY "System\\CurrentControlSet\\Services\\EventLog"
+#  define L_EVENTLOG_REGISTRY_SUBKEY L_EXPAND_LEVEL(EVENTLOG_REGISTRY_SUBKEY)
+#  define L_EVENTLOG_REGISTRY_KEY L"\\Registry\\Machine\\"L_EXPAND_LEVEL(EVENTLOG_REGISTRY_SUBKEY)
+#  define L_EVENT_LOG_KEY LCONCAT(L_EVENTLOG_REGISTRY_KEY,EVENTLOG_NAME)
+#  define L_EVENT_LOG_SUBKEY LCONCAT(L_EVENTLOG_REGISTRY_SUBKEY,EVENTLOG_NAME)
+#  define L_EVENT_LOG_NAME L_EXPAND_LEVEL(EVENTLOG_NAME)      
+#  define L_EVENT_SOURCE_NAME L_EXPAND_LEVEL(EVENTSOURCE_NAME)
+#  define L_EVENT_SOURCE_KEY LCONCAT(L_EVENT_LOG_KEY, EVENTSOURCE_NAME)
+#  define L_EVENT_SOURCE_SUBKEY LCONCAT(L_EVENT_LOG_SUBKEY, EVENTSOURCE_NAME)
+
+#  define EVENT_LOG_KEY LCONCAT(L_EXPAND_LEVEL(EVENTLOG_REGISTRY_SUBKEY),EVENTLOG_NAME)
+#  define EVENT_SOURCE_KEY LCONCAT(EVENT_LOG_KEY,EVENTSOURCE_NAME)
+/* Log key values (NOTE the values here are the values our installer uses,
+ * not sure what all of them mean).  FIXME would be nice if these were
+ * shared with the installer config file. Only used by DRcontrol (via 
+ * share/config.c) to set up new eventlogs (mainly for vista where our
+ * installer doesn't work yet xref case 8482).*/
+#  define L_EVENT_FILE_VALUE_NAME L"File"
+#  define L_EVENT_FILE_NAME_PRE_VISTA \
+       L"%SystemRoot%\\system32\\config\\"L_EXPAND_LEVEL(EVENTLOG_NAME)L".evt"
+#  define L_EVENT_FILE_NAME_VISTA \
+       L"%SystemRoot%\\system32\\winevt\\logs\\"L_EXPAND_LEVEL(EVENTLOG_NAME)L".elf"
+#  define L_EVENT_MAX_SIZE_NAME L"MaxSize"
+#  define EVENT_MAX_SIZE 0x500000
+#  define L_EVENT_RETENTION_NAME L"Retention"
+#  define EVENT_RETENTION 0
+/* Src key values */
+#  define L_EVENT_TYPES_SUPPORTED_NAME L"TypesSupported"
+#  define EVENT_TYPES_SUPPORTED 0x7 /* info | warning | error */
+#  define L_EVENT_CATEGORY_COUNT_NAME L"CategoryCount"
+#  define EVENT_CATEGORY_COUNT 0
+#  define L_EVENT_CATEGORY_FILE_NAME L"CategoryMessageFile"
+#  define L_EVENT_MESSAGE_FILE L"EventMessageFile"
+
+/* shared object directory base */
+/* base root in global object namespace, not in BaseNamedObjects or Sessions */
+#  define DYNAMORIO_SHARED_OBJECT_BASE L("\\")L_EXPAND_LEVEL(COMPANY_NAME)
+/* shared object directory for shared DLL cache */
+#  define DYNAMORIO_SHARED_OBJECT_DIRECTORY LCONCAT(DYNAMORIO_SHARED_OBJECT_BASE, "SharedCache")
+
+/* registry */
+#  define DYNAMORIO_REGISTRY_BASE_SUBKEY "Software\\"COMPANY_NAME"\\"PRODUCT_NAME
+#  define DYNAMORIO_REGISTRY_BASE L"\\Registry\\Machine\\Software\\"L_EXPAND_LEVEL(COMPANY_NAME)L("\\")L_EXPAND_LEVEL(PRODUCT_NAME)
+#  define DYNAMORIO_REGISTRY_HIVE HKEY_LOCAL_MACHINE
+#  define DYNAMORIO_REGISTRY_KEY_SET COMPANY_NAME
+#  define DYNAMORIO_REGISTRY_KEY    DYNAMORIO_REGISTRY_BASE_SUBKEY
+#  define L_DYNAMORIO_REGISTRY_KEY L"Software\\"L_EXPAND_LEVEL(COMPANY_NAME)L"\\"L_EXPAND_LEVEL(PRODUCT_NAME)
+/* this is just here to keep things in one place. probably should combine 
+   with other keys. */
+#  define DYNAMORIO_REGISTRY_CONFIG_SUBKEY L"Software\\"L_EXPAND_LEVEL(COMPANY_NAME)L"\\Configuration\\"L_EXPAND_LEVEL(PRODUCT_NAME)
+
+#  define INJECT_ALL_HIVE    HKEY_LOCAL_MACHINE
+#  define INJECT_ALL_KEY     "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows"
+#  define INJECT_ALL_SUBKEY  "AppInit_DLLs"
+#  define INJECT_ALL_LOAD_SUBKEY        "LoadAppInit_DLLs"
+
+#  define INJECT_ALL_HIVE_L    L"\\Registry\\Machine\\"
+#  define INJECT_ALL_KEY_L     L_EXPAND_LEVEL(INJECT_ALL_KEY)
+#  define INJECT_ALL_SUBKEY_L  L_EXPAND_LEVEL(INJECT_ALL_SUBKEY)
+#  define INJECT_ALL_LOAD_SUBKEY_L      L_EXPAND_LEVEL(INJECT_ALL_LOAD_SUBKEY)
+
+#  define INJECT_DLL_NAME      "drpreinject.dll"
+#  define INJECT_DLL_8_3_NAME  "DRPREI~1.DLL"
+
+#  define INJECT_HELPER_DLL1_NAME      "drearlyhelp1.dll"
+#  define INJECT_HELPER_DLL2_NAME      "drearlyhelp2.dll"
+
+#  define DEBUGGER_INJECTION_HIVE       HKEY_LOCAL_MACHINE
+#  define DEBUGGER_INJECTION_KEY        "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Image File Execution Options"
+#  define DEBUGGER_INJECTION_VALUE_NAME "Debugger"
+
+#  define DEBUGGER_INJECTION_HIVE_L       L"\\Registry\\Machine\\"
+#  define DEBUGGER_INJECTION_KEY_L        L_EXPAND_LEVEL(DEBUGGER_INJECTION_KEY)
+#  define DEBUGGER_INJECTION_VALUE_NAME_L L_EXPAND_LEVEL(DEBUGGER_INJECTION_VALUE_NAME)
+
+#  define DRINJECT_NAME                 "drinject.exe"
+
+/* shared module uses SVCHOST_NAME and EXE_SUFFIX separately */
+#  define SVCHOST_NAME         "svchost"
+#  define EXE_SUFFIX           ".exe"
+#  define L_EXE_SUFFIX         L_EXPAND_LEVEL(EXE_SUFFIX)
+#  define SVCHOST_EXE_NAME     SVCHOST_NAME EXE_SUFFIX
+#  define L_SVCHOST_EXE_NAME   L_EXPAND_LEVEL(SVCHOST_NAME) L_EXPAND_LEVEL(EXE_SUFFIX)
+
+/* for processview, etc */
+#  define DYNAMORIO_LIBRARY_NAME "dynamorio.dll"
+#  define DLLPATH_RELEASE    "\\lib\\release\\"DYNAMORIO_LIBRARY_NAME
+#  define DLLPATH_DEBUG      "\\lib\\debug\\"DYNAMORIO_LIBRARY_NAME
+#  define DLLPATH_PROFILE    "\\lib\\profile\\"DYNAMORIO_LIBRARY_NAME
+
+#  define L_DYNAMORIO_LIBRARY_NAME L_EXPAND_LEVEL(DYNAMORIO_LIBRARY_NAME)
+#  define L_DLLPATH_RELEASE    L"\\lib\\release\\"L_DYNAMORIO_LIBRARY_NAME
+#  define L_DLLPATH_DEBUG      L"\\lib\\debug\\"L_DYNAMORIO_LIBRARY_NAME
+#  define L_DLLPATH_PROFILE    L"\\lib\\profile\\"L_DYNAMORIO_LIBRARY_NAME
+
+#  define INJECT_ALL_DLL_SUBPATH   "\\lib\\"INJECT_DLL_8_3_NAME
+#  define L_INJECT_ALL_DLL_SUBPATH   L"\\lib\\"L_EXPAND_LEVEL(INJECT_DLL_8_3_NAME)
+
+enum DLL_TYPE {
+        DLL_NONE,
+        DLL_UNKNOWN,
+        DLL_RELEASE,
+        DLL_DEBUG,
+        DLL_PROFILE,
+        DLL_CUSTOM,
+        DLL_PATHHAS
+};
+
+/* DYNAMORIO_RUNUNDER controls the injection technique and process naming, 
+ *  it is a bitmask of the values below:
+ * RUNUNDER_ON: 
+ *  take over the app indicated by the corresponding app-specific
+ *  subkey; when this is a global param, it only acts as a default for
+ *  subkeys which don't explicitly set RUNUNDER. indicates current
+ *  default takeover method via the preinjector / AppInit_DLLs
+ * RUNUNDER_ALL:
+ *  use as a global parameter only, for doing "run all". exclude with
+ *  app-specific RUNUNDER_OFF.  You need to use in conjuction with ON
+ *
+ *
+ * RUNUNDER_EXPLICIT:
+ *  indicates that the app will use the alternate injection technique
+ *  currently via -follow_explicit_children, but might change to drinject 
+ *  in the per-executable debugger registry key at some point
+ *
+ *
+ * RUNUNDER_COMMANDLINE_MATCH:
+ *  indicates that the process command line must exactly match the
+ *  value in the DYNAMORIO_CMDLINE app-specific subkey, or else no
+ *  takeover will be done.  Note that only a single instance of 
+ *  a given executable name can be controlled this way.
+ *
+ * RUNUNDER_COMMANDLINE_DISPATCH: 
+ *  marks that the processes with this executable name should be
+ *  differentiated by their canonicalized commandline.  For example,
+ *  different dllhost instances will get their own different subkeys,
+ *  just the way the svchost.exe instances have full sets of option
+ *  keys.
+ *
+ * RUNUNDER_COMMANDLINE_NO_STRIP:
+ *  this flag is used only in conjunction with
+ *  RUNUNDER_COMMANDLINE_DISPATCH.  Our default canonicalization rule
+ *  strips the first argument on the commandline for historic reasons.
+ *  Since that was the way we had previously hardcoded the names for
+ *  say "svchost.exe-netsvcs".  In case we actually want to dispatch
+ *  on a single argument - which would be the case for say msiexec.exe
+ *  /v then this flag is needed.
+ *
+ * RUNUNDER_ONCE:
+ *  is used by staging mode to specify that the executable corresponding 
+ *  to the current process shouldn't run under DR the next time, i.e., 
+ *  turn off its RUNUNDER_ON flag after checking it for the current process.
+ *  This is needed to prevent perpetual crash-&-boot cycles due to DR failure.  
+ *  See case 3702.
+ **/
+enum {
+    /* FIXME: keep in mind that we only read decimal values */
+    RUNUNDER_OFF                  = 0x00,   /* 0 */
+    RUNUNDER_ON                   = 0x01,   /* 1 */
+    RUNUNDER_ALL                  = 0x02,   /* 2 */
+
+    /* Dummy field to keep track of which processes were RUNUNDER_EXPLICIT
+     * before we moved to -follow_systemwide by default (for -early_injection)
+     * (note this was the old RUNUNDER_EXPLICIT value) */
+    RUNUNDER_FORMERLY_EXPLICIT    = 0x04,   /* 4 */
+
+    RUNUNDER_COMMANDLINE_MATCH    = 0x08,   /* 8 */
+    RUNUNDER_COMMANDLINE_DISPATCH = 0x10,  /* 16 */
+    RUNUNDER_COMMANDLINE_NO_STRIP = 0x20,  /* 32 */
+    RUNUNDER_ONCE                 = 0x40,  /* 64 */
+
+    RUNUNDER_EXPLICIT             = 0x80,  /* 128 */
+};
+
+/* A bitmask of possible actions to take on a nudge.  Accessed via
+ * NUDGE_GENERIC(name) Recommended to always pass -nudge opt so to
+ * synchronize options first.  For many state transition nudge's
+ * option change will trigger any other actions: e.g. start
+ * protecting, simulate attack, etc..  Only pulse-like events that
+ * should be acted on only once need separate handling here.  Not all
+ * combinations are meaningful, and the order of execution is not
+ * determined from the order in the definitions.
+ */
+/* To use as an iterator define NUDGE_DEF(name, comment) */
+
+/* CAUTION: DO NOT change ordering of the nudge definitions for non-NYI, i.e.,
+ *          implemented nudges.  These numbers correspond to specific masks 
+ *          that are used by the nodemanager/drcontrol (thus QA).  Will lead to
+ *          a lot of unwanted confusion.
+ */
+#define NUDGE_DEFINITIONS()                                                     \
+    /* Control nudges */                                                        \
+    NUDGE_DEF(opt, "Synchronize dynamic options")                               \
+    NUDGE_DEF(reset, "Reset code caches") /* flush & delete */                  \
+    NUDGE_DEF(detach, "Detach")                                                 \
+    NUDGE_DEF(mode, "Liveshield mode update")                                   \
+    NUDGE_DEF(policy, "Liveshield policy update")                               \
+    NUDGE_DEF(lstats, "Liveshield statistics NYI")                              \
+    NUDGE_DEF(process_control, "Process control nudge") /* Case 8594. */        \
+    NUDGE_DEF(upgrade, "DR upgrade NYI case 4179")                              \
+    NUDGE_DEF(kstats, "Dump kstats in log or kstat file NYI")                   \
+    /* internal options */                                                      \
+    NUDGE_DEF(stats, "Dump internal stats in logfiles NYI")                     \
+    NUDGE_DEF(invalidate, "Invalidate code caches NYI") /* flush */             \
+    /* stress testing */                                                        \
+    NUDGE_DEF(recreate_pc, "Recreate PC NYI")                                   \
+    NUDGE_DEF(recreate_state, "Recreate state NYI")                             \
+    NUDGE_DEF(reattach, "Reattach - almost detach, NYI")                        \
+    /* diagnostics */                                                           \
+    NUDGE_DEF(diagnose, "Request diagnostic file NYI")                          \
+    NUDGE_DEF(ldmp, "Dump core")                                                \
+    NUDGE_DEF(freeze, "Freeze coarse units")                                    \
+    NUDGE_DEF(persist, "Persist coarse units")                                  \
+    /* client nudge */                                                          \
+    NUDGE_DEF(client, "Client nudge")                                           \
+    /* security testing */                                                      \
+    NUDGE_DEF(violation, "Simulate a security violation")                       \
+    /* ADD NEW NUDGE_DEFs only immediately above this line  */                  \
+    /* since these are used as a bitmask only 30 types can be supported */
+
+typedef enum {
+    NUDGE_DR_PARAMETRIZED_START,
+#define NUDGE_DEF(name, comment) NUDGE_DR_##name, 
+    NUDGE_DEFINITIONS()
+#undef NUDGE_DEF
+    NUDGE_DR_PARAMETRIZED_END
+} nudge_generic_type_t;
+
+/* note that these are bitmask values */
+#define NUDGE_GENERIC(name) (1 << (NUDGE_DR_##name))
+
+#define NUDGE_ARG_VERSION_1 1
+#define NUDGE_ARG_CURRENT_VERSION NUDGE_ARG_VERSION_1
+
+/* nudge_arg_t flags */
+enum {
+    NUDGE_IS_INTERNAL       = 0x01, /* nudge is internally generated */
+    NUDGE_NUDGER_FREE_STACK = 0x02, /* nudger will free the nudge thread's stack so the
+                                     * nudge thread itself shouldn't */
+    NUDGE_FREE_ARG          = 0x04, /* nudge arg is in a separate allocation and should
+                                     * be freed by the nudge thread */
+};
+
+typedef struct {
+    uint version; /* version number for future proofing */
+    uint nudge_action_mask; /* drawn from NUDGE_DEFS above */
+    uint flags; /* flags drawn from above enum */
+    client_id_t client_id; /* unique ID identifying client */
+    uint64 client_arg; /* argument for a client nudge */
+    /* Add future arguments for nudge actions here. */
+} nudge_arg_t;
+
+#endif /* WINDOWS */
+
+#ifdef HOT_PATCHING_INTERFACE
+/* These type defintions define the hot patch interface between the core &
+ * the node manager.
+ * CAUTION: Any changes to this can break the hot patch interface between the
+ *          core and the node manager.
+ */
+
+/* All hot patch policy IDs must be of the form XXXX.XXXX; this ID is used to
+ * generate the threat ID for a given hot patch violation.
+ */
+#define HOTP_POLICY_ID_LENGTH 9
+
+/* Define AVOID_API_EXPORT here and undef it after the ifdef-endif using it.
+ * This way it will just be used for compling dr code and not for
+ * genapi.pl which generates client header files.  In otherwords, this allows
+ * having code that isn't visible in the client headers but is visible for dr
+ * builds.  This helps sharing types and code between dr and client, but with
+ * some hidden extras for dr builds.
+ */
+#define AVOID_API_EXPORT 1
+
+/* DR_API EXPORT TOFILE dr_probe.h */
+/* DR_API EXPORT BEGIN */
+
+/** Describes the status of a probe at any given point.  The status is returned
+ * by dr_register_probes() in the dr_probe_desc_t structure for each probe
+ * specified.  It can be obtained for individual probes by calling
+ * dr_get_probe_status().
+ */
+typedef enum {
+    /* Error codes. */
+
+    /** Exceptions during callback execution and other unknown errors. */
+    DR_PROBE_STATUS_ERROR = 1,
+
+    /** An invalid or unknown probe ID was specified with dr_get_probe_status(). */
+    DR_PROBE_STATUS_INVALID_ID = 2,
+
+    /* All the invalid states listed below may arise statically (at the
+     * time of parsing the probes, i.e., inside dr_register_probes() or
+     * dynamically (i.e., when modules are loaded or unloaded)).
+     */ 
+
+    /** The numeric virtual address specified for the probe insertion location
+     * or the callback function is invalid.
+     */
+    DR_PROBE_STATUS_INVALID_VADDR = 3,
+
+    /** The pointer to the name of the library containing the probe insertion
+     * location or the callback function is invalid or the library containing
+     * the callback function can't be loaded.
+     */
+    DR_PROBE_STATUS_INVALID_LIB = 4,
+
+    /** The library offset for either the probe insertion location or the
+     * callback function is invalid; for ex., if the offset is out of bounds.
+     */
+    DR_PROBE_STATUS_INVALID_LIB_OFFS = 5,
+
+    /** The pointer to the name of the exported function, where the probe is to
+     * be inserted or which is the callback function, is invalid or the exported
+     * function doesn't exist.
+     */
+    DR_PROBE_STATUS_INVALID_FUNC = 6,
+
+    /* Codes for pending cases, i.e., valid probes haven't been inserted
+     * because certain events haven't transpired.
+     */
+
+    /** The numeric virtual address specified for the probe insertion location
+     * or the callback function isn't executable.  This may be so at the time
+     * of probe registration or latter if the memory protections are changed.
+     * An inserted probe might reach this state if the probe insertion point or
+     * the callback function is made non-executable after being executable.
+     */
+    DR_PROBE_STATUS_VADDR_NOT_EXEC = 7,
+
+    /** The library where the probe is to be inserted isn't in the process. */
+    DR_PROBE_STATUS_LIB_NOT_SEEN = 8,
+
+    /** Execution hasn't reached the probe insertion point yet.  This is valid
+     * for Code Manipulation mode only because in that mode probes are inserted
+     * only in the dynamic instruction stream.
+     */
+    DR_PROBE_STATUS_WAITING_FOR_EXEC = 9,
+
+    /** Either the library where the probe is to be inserted has been unloaded
+     * or the library containing the callback function has been unloaded.
+     */
+    DR_PROBE_STATUS_LIB_UNLOADED = 10,
+
+    /* Miscellaneous status codes. */
+
+    /** Probe was successfully inserted. */
+    DR_PROBE_STATUS_INJECTED = 11,
+
+    /** One or more aspects of the probe aren't supported as of now. */
+    DR_PROBE_STATUS_UNSUPPORTED = 12,
+
+# ifdef AVOID_API_EXPORT
+    /* DON'T CHANGE THE VALUES OF THE DR_* CONSTANTS DEFINED ABOVE.  They are
+     * exported to clients, whereas constants in this ifdef aren't.  Any change
+     * to those values will likely break old clients with newer versions of DR
+     * (backward compatibility).  New status codes should go after
+     * DR_PROBE_STATUS_UNSUPPORTED.
+     */
+    /* Note: constants are numbered to prevent the compiler from resetting the
+     * sequence based on the symbolic assignments below.  HOTP_INJECT_DETECT
+     * ended up getting the same number as one the values above!  Ditto with
+     * HOTP_INJECT_OFF.  Though these duplications only broke the tools build
+     * they can cause subtle runtime errors, so forcing numbers.
+     */
+
+    /* The constants below are used only for LiveShields.  */
+
+    /* Defines the current injection status of a policy, i.e., was it injected
+     * or not, why and why not?  Today we don't distinguish the reasons for
+     * error cases, i.e., all of them are rolled into one.
+     *
+     * Constants list from most important status to least, from a reporting
+     * point of view; don't change this arbitrarily.
+     *
+     * CAUTION: Any changes to this will break drview, so they must be kept in
+     * sync.
+     */
+
+    /* Deactivation, exception, error, etc. */
+    HOTP_INJECT_ERROR = DR_PROBE_STATUS_ERROR,
+
+    /* Completely injected in protect mode. */
+    HOTP_INJECT_PROTECT = DR_PROBE_STATUS_INJECTED,
+
+    /* Completely injected in detect mode.  Not applicable to probes as they
+     * don't have detectors.  Restart numbering at 100 to give enough room for
+     * future probe status constants.
+     */
+    HOTP_INJECT_DETECT = 100,
+
+    /* One or more patch points in a vulnerability have been patched, but not 
+     * all, yet.  N/A to probes as they can't group multiple patch points.
+     */
+    HOTP_INJECT_IN_PROGRESS = 101,
+
+    /* Vulnerability was matched and is ready for injection, but no patch point
+     * has been seen yet.
+     */
+    HOTP_INJECT_PENDING = DR_PROBE_STATUS_WAITING_FOR_EXEC,
+
+    /* Vulnerability hasn't been matched yet.  May mean that it is not yet
+     * vulnerable (because all dlls haven't been loaded) or just not vulnerable
+     * at all;  there is no way to distinguish between the two.
+     */
+    HOTP_INJECT_NO_MATCH = DR_PROBE_STATUS_LIB_NOT_SEEN,
+
+    /*
+    TODO: must distinguish between no match & vulnerable vs. no match & not 
+          vulnerable; future work if needed.
+    HOTP_INJECT_NO_MATCH_VULNERABLE,
+    HOTP_INJECT_NO_MATCH_NOT_VULNERABLE,
+    */
+
+    HOTP_INJECT_OFF = 102   /* Policy has been turned off, so no injection. */
+# endif /* AVOID_API_EXPORT */
+} dr_probe_status_t;
+/* DR_API EXPORT END */
+#undef AVOID_API_EXPORT
+
+typedef dr_probe_status_t hotp_inject_status_t;
+
+
+/* Modes are at a policy level, not a vulnerability level, even though the 
+ * core organizes things at the vulnerability level.
+ */
+typedef enum {
+    HOTP_MODE_OFF = 0,
+    HOTP_MODE_DETECT  = 1,
+    HOTP_MODE_PROTECT = 2
+} hotp_policy_mode_t;
+
+/* This structure is used to form a table that contains the status of all
+ * active policies.  This is separated out into a separate table, as opposed
+ * to being part of the hotp_vul_info_t and thus part of the global 
+ * vulnerability table, because the node manager will be directly reading 
+ * this information from the core's memory.  Thus, this structure serves 
+ * as a container to expose only that data which will be needed by the node 
+ * manager from the core regarding hot patches.
+ */
+typedef struct {
+    /* polciy_id is the same as the one in hotp_vul_t.  Duplicated because can't
+     * have this pointing to the hopt_vul_t structure because the node manager 
+     * will have to chase the pointer for each element to read it rather than a
+     * single block of memory.
+     */
+    char policy_id[HOTP_POLICY_ID_LENGTH + 1];
+    hotp_inject_status_t inject_status;
+
+    /* This is the same as the one in hotp_vul_t.  Duplicated for the same
+     * reason policy_id (see above) was.  Can't have the hotp_vul_t structure
+     * pointing here too because that struct/table needs to be initialized for 
+     * the policy status table to be created; catch-22.  
+     * Fix for case 5484, where the node manager wasn't able to tell if an 
+     * inject status was for a policy that was turned on or off.
+     */
+    hotp_policy_mode_t mode;
+} hotp_policy_status_t;
+
+/* Node manager should read either this struct first or the first two elements
+ * in it.  Then it should read the rest.  Note: 'size' refers to the size of
+ * both this struct and the size of the policy_status_array in bytes.
+ */
+typedef struct {
+    /* This is the crc for the whole table, except itself, i.e., the crc
+     * computation starts from 'size'.  This is needed otherwise writing the
+     * crc value itself will change the crc of the table!  Can get past it
+     * by doing crc twice, but it is needlessly expensive.
+     *
+     * CAUTION: don't not move crc and size around as it can cause both the
+     * node manager & the core to break.
+     */
+    uint crc;
+    uint size; /* Size of this struct plus the table. */
+    uint num_policies;
+    hotp_policy_status_t *policy_status_array;
+} hotp_policy_status_table_t;
+
+#endif /* ifdef HOT_PATCHING_INTERFACE */
+
+/* These constants & macros are used by core, share and preinject, so this is
+ * the only place they will build for win32 and linux! */
+/* return codes for [gs]et_parameter style functions
+ * failure == 0 for compatibility with get_parameter()
+ * if GET_PARAMETER_NOAPPSPECIFIC is returned, that means the
+ *  parameter returned is from the global options, because there
+ *  was no app-specific key present.
+ * Note: constants shouldn't be added to this enum without checking whether
+ *       the macros below will work; they should if errors are all negative and
+ *       valid codes are all positive.
+ */
+enum {
+    GET_PARAMETER_BUF_TOO_SMALL = -1,
+    GET_PARAMETER_FAILURE = 0,
+    GET_PARAMETER_SUCCESS = 1,
+    GET_PARAMETER_NOAPPSPECIFIC = 2,
+    SET_PARAMETER_FAILURE = GET_PARAMETER_FAILURE,
+    SET_PARAMETER_SUCCESS = GET_PARAMETER_SUCCESS
+};
+#define IS_GET_PARAMETER_FAILURE(x) ((x) <= 0 )
+#define IS_GET_PARAMETER_SUCCESS(x) ((x) > 0 )
+
+/* X86 only but no ifdef since hotp builds don't set that define */
+/* User-mode machine context.
+ * We have it here instead of in arch_exports.h so that we
+ * can expose it to hotpatch_interface.h.
+ *
+ * If any field offsets are changed, or fields are added, update
+ * the following:
+ *   - OFFSET defines in x86/arch.h and their uses in fcache_{enter,return}
+ *   - DR_MCONTEXT_SIZE and PUSH_DR_MCONTEXT in x86/x86.asm
+ *   - clean call layout of dr_mcontext_t on the stack in x86/mangle.c
+ *   - interception layout of dr_mcontext_t on the stack in win32/callback.c
+ *   - context_to_mcontext (and vice versa) in win32/ntdll.c
+ *   - dump_mcontext in x86/arch.c
+ *   - inject_into_thread in win32/inject.c
+ * Also, hotp_context_t exposes the dr_mcontext_t struct to hot patches,
+ * so be careful when changing any field offsets.
+ *
+ * FIXME: remove eax-ebx-ecx-edx and use the local_state_t
+ * version from within DR as well as from the ibl (case 3701).
+ *
+ * PR 264138: for xmm fields, we do NOT specify 16-byte alignment for
+ * our own, to avoid the 8-byte padding at the end for 32-bit mode
+ * that we don't want to manually insert for our hand-rolled structs
+ * on the stack.
+ * It's ok for the client version to have padding as dr_get_mcontext copies,
+ * but we don't currently assume alignment of client-declared dr_mcontext_t,
+ * so we have no alignment declarations there at the moment either.
+ */
+
+/* PR 306394: for 32-bit xmm0-7 are caller-saved, yet we don't expect
+ * them to be touched by DR, client, kernel, or libraries DR/client
+ * call: so for now we just leave space for future use.
+ * In particular for LOL64 we should be careful about kernel use.
+ */
+#ifdef WINDOWS
+# define NUM_XMM_SAVED 6 /* xmm0-5; for 32-bit we have space for xmm0-7 */
+#else
+# ifdef X64
+#  define NUM_XMM_SAVED 16 /* xmm0-15 */
+# else
+#  define NUM_XMM_SAVED 0 /* we have space for xmm0-7, but not saved right now */
+# endif
+#endif
+
+/* DR_API EXPORT TOFILE dr_defines.h */
+/* DR_API EXPORT BEGIN */
+/** 128-bit XMM register. */
+typedef union _dr_xmm_t {
+#ifdef AVOID_API_EXPORT
+    /* We avoid having 8-byte-aligned fields here for 32-bit: they cause
+     * cl to add padding in app_state_at_intercept_t and unprotected_context_t,
+     * which messes up our interception stack layout and our x86.asm offsets.
+     * We don't access these very often, so we just omit this field.
+     */
+#endif
+#ifdef API_EXPORT_ONLY
+    uint64 u64[2]; /**< Representation as 2 64-bit integers. */
+#endif
+    uint   u32[4]; /**< Representation as 4 32-bit integers. */
+    byte   u8[16]; /**< Representation as 8 8-bit integers. */
+    reg_t  reg[IF_X64_ELSE(2,4)]; /**< Representation as 2 or 4 registers. */
+} dr_xmm_t;
+
+#ifdef AVOID_API_EXPORT
+/* If this is increased, you'll probably need to increase the size of
+ * inject_into_thread's buf and INTERCEPTION_CODE_SIZE (for Windows).
+ * Also, update NUM_XMM_SLOTS in x86.asm and get_xmm_caller_saved.
+ */
+#endif
+#ifdef X64
+# ifdef WINDOWS
+#  define NUM_XMM_SLOTS 6 /** Number of xmm reg slots in dr_mcontext_t */ /* xmm0-5 */
+# else
+#  define NUM_XMM_SLOTS 16 /** Number of xmm reg slots in dr_mcontext_t */ /* xmm0-15 */
+# endif
+#else
+# define NUM_XMM_SLOTS 8 /** Number of xmm reg slots in dr_mcontext_t */ /* xmm0-7 */
+#endif
+
+/**
+ * Machine context structure.
+ */
+typedef struct _dr_mcontext_t {
+#ifdef AVOID_API_EXPORT
+    /* FIXME: have special comment syntax instead of bogus ifdef to
+     * get genapi to strip out internal-only comments? */
+    /* Our inlined ibl uses eax-edx, so we place them together to fit
+     * on the same 32-byte cache line; yet we also want to simplify
+     * things by keeping this in pusha order.  Whether on a 32-bit or
+     * 64-bit machine, or a 32-byte or 64-byte cache line, they will
+     * still be on the same line, assuming this struct is
+     * cache-line-aligned (which it is if in dcontext).
+     * Any changes in order here must be mirrored in x86/x86.asm offsets.
+     */
+#endif
+    union {
+        reg_t xdi; /**< platform-independent name for full rdi/edi register */
+        reg_t IF_X64_ELSE(rdi, edi); /**< platform-dependent name for rdi/edi register */
+    }; /* anonymous union of alternative names for rdi/edi register */
+    union {
+        reg_t xsi; /**< platform-independent name for full rsi/esi register */
+        reg_t IF_X64_ELSE(rsi, esi); /**< platform-dependent name for rsi/esi register */
+    }; /* anonymous union of alternative names for rsi/esi register */
+    union {
+        reg_t xbp; /**< platform-independent name for full rbp/ebp register */
+        reg_t IF_X64_ELSE(rbp, ebp); /**< platform-dependent name for rbp/ebp register */
+    }; /* anonymous union of alternative names for rbp/ebp register */
+    union {
+        reg_t xsp; /**< platform-independent name for full rsp/esp register */
+        reg_t IF_X64_ELSE(rsp, esp); /**< platform-dependent name for rsp/esp register */
+    }; /* anonymous union of alternative names for rsp/esp register */
+    union {
+        reg_t xbx; /**< platform-independent name for full rbx/ebx register */
+        reg_t IF_X64_ELSE(rbx, ebx); /**< platform-dependent name for rbx/ebx register */
+    }; /* anonymous union of alternative names for rbx/ebx register */
+    union {
+        reg_t xdx; /**< platform-independent name for full rdx/edx register */
+        reg_t IF_X64_ELSE(rdx, edx); /**< platform-dependent name for rdx/edx register */
+    }; /* anonymous union of alternative names for rdx/edx register */
+    union {
+        reg_t xcx; /**< platform-independent name for full rcx/ecx register */
+        reg_t IF_X64_ELSE(rcx, ecx); /**< platform-dependent name for rcx/ecx register */
+    }; /* anonymous union of alternative names for rcx/ecx register */
+    union {
+        reg_t xax; /**< platform-independent name for full rax/eax register */
+        reg_t IF_X64_ELSE(rax, eax); /**< platform-dependent name for rax/eax register */
+    }; /* anonymous union of alternative names for rax/eax register */
+#ifdef X64
+    reg_t r8;  /**< r8 register. \note For 64-bit DR builds only. */
+    reg_t r9;  /**< r9 register. \note For 64-bit DR builds only. */
+    reg_t r10; /**< r10 register. \note For 64-bit DR builds only. */
+    reg_t r11; /**< r11 register. \note For 64-bit DR builds only. */
+    reg_t r12; /**< r12 register. \note For 64-bit DR builds only. */
+    reg_t r13; /**< r13 register. \note For 64-bit DR builds only. */
+    reg_t r14; /**< r14 register. \note For 64-bit DR builds only. */
+    reg_t r15; /**< r15 register. \note For 64-bit DR builds only. */
+#endif
+    /**
+     * The SSE registers xmm0-xmm5 (-xmm15 on Linux) are volatile
+     * (caller-saved) for 64-bit and WOW64, and are actually zeroed out on
+     * Windows system calls.  These fields are ignored for 32-bit processes
+     * that are not WOW64, or if the underlying processor does not support
+     * SSE.  Use dr_mcontext_xmm_fields_valid() to determine whether the
+     * fields are valid.
+     */
+#ifdef AVOID_API_EXPORT
+    /* PR 264138: we must preserve xmm0-5 if on a 64-bit Windows kernel,
+     * and xmm0-15 if in a 64-bit Linux app (PR 302107).  (Note that
+     * mmx0-7 are also caller-saved on linux but we assume they're not
+     * going to be used by DR, libc, or client routines: overlap w/
+     * floating point).  For Windows we assume that none of our routines
+     * (or libc routines that we call, except the floating-point ones,
+     * where we explicitly save state) clobber beyond xmm0-5.  Rather than
+     * have a separate WOW64 build, we have them in the struct but ignored
+     * for normal 32-bit.
+     * PR 306394: for now we leave space for preserving xmm0-7 for 32-bit,
+     * but we do not currently save them.
+     */
+#endif
+    dr_xmm_t xmm[NUM_XMM_SLOTS];
+    union {
+        reg_t xflags; /**< platform-independent name for full rflags/eflags register */
+        reg_t IF_X64_ELSE(rflags, eflags); /**< platform-dependent name for
+                                                rflags/eflags register */
+    }; /* anonymous union of alternative names for rflags/eflags register */
+    /*
+     * Anonymous union of alternative names for the program counter / 
+     * instruction pointer (eip/rip).  This field is not always set or 
+     * read by all API routines.
+     */ 
+    union {
+        byte *xip; /**< platform-independent name for full rip/eip register */
+        byte *pc; /**< platform-independent alt name for full rip/eip register */
+        byte *IF_X64_ELSE(rip, eip); /**< platform-dependent name for rip/eip register */
+    };
+} dr_mcontext_t;
+/* DR_API EXPORT END */
+
+#endif /* ifndef _GLOBALS_SHARED_H_ */
diff --git a/core/lib/hotpatch_interface.h b/core/lib/hotpatch_interface.h
new file mode 100644
index 000000000..33609b5d7
--- /dev/null
+++ b/core/lib/hotpatch_interface.h
@@ -0,0 +1,123 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2005-2007 Determina Corp. */
+
+#ifndef _HOTPATCH_INTERFACE_H_
+#define _HOTPATCH_INTERFACE_H_ 1
+
+/* This file defines the interface between the core and the hot patches.  Any
+ * changes to this file will most likely require changes to both.
+ */
+
+#include "globals_shared.h" /* for reg_t */
+
+/* CAUTION: The hot patch interface (i.e., engine) version should be kept in
+ * synch with the hot patches (hotpatch module) any time a new engine version 
+ * is defined.
+ */
+#define HOTP_INTERFACE_VERSION 42000
+
+/* This enum specifies the possible status codes that a hot patch routine can
+ * return to convey how its execution proceeded.  This has to be a bit flag 
+ * because event logging requests by a protector can be combined with other 
+ * status codes.
+ *
+ * CAUTION: Changes to this enum will break hot patch code; all hot patch code
+ *          has to be recompiled and hot patching engine version probably has
+ *          to be upgraded.  Be sure to talk with Alex.
+ */
+typedef enum {
+    /* Detector status codes. */
+    HOTP_EXEC_EXPLOIT_DETECTED          = 0x00000001,
+    HOTP_EXEC_EXPLOIT_NOT_DETECTED      = 0x00000002,
+    HOTP_EXEC_DETECTOR_ERROR            = 0x00000004,
+
+    /* All codes below can only be returned by a protector. */
+    HOTP_EXEC_EXPLOIT_PROTECTED         = 0x00000008,
+    HOTP_EXEC_EXPLOIT_NOT_PROTECTED     = 0x00000010,
+    HOTP_EXEC_PROTECTOR_ERROR           = 0x00000020,
+
+    /* These codes return both a status & request for a particular action. */
+    HOTP_EXEC_EXPLOIT_KILL_THREAD       = 0x00000040,
+    HOTP_EXEC_EXPLOIT_KILL_PROCESS      = 0x00000080,
+    HOTP_EXEC_EXPLOIT_RAISE_EXCEPTION   = 0x00000100,
+    HOTP_EXEC_CHANGE_CONTROL_FLOW       = 0x00000200,
+
+    /* This flag is orthogonal to the ones above and can be specified with 
+     * any of those.  This can be used both by detectors and protectors.
+     */
+    HOTP_EXEC_LOG_EVENT                 = 0x00000400,
+
+    /* This status shouldn't be returned by a hot patch code.  It is used to 
+     * identify unexpected aborts of hot patch code, mostly due to exceptions.
+     */
+    HOTP_EXEC_ABORTED                   = 0x00000800
+} hotp_exec_status_t;
+
+typedef dr_mcontext_t hotp_context_t;
+
+/* TODO: may have to define one for detector & one for protector because their 
+ *       interface types will be different once the protector logging is done.
+ */
+typedef hotp_exec_status_t (*hotp_func_t)(hotp_context_t *app_reg_ptr);
+
+#define APP_XDI(x) (((hotp_context_t*)(x))->xdi)
+#define APP_XSI(x) (((hotp_context_t*)(x))->xsi)
+#define APP_XBP(x) (((hotp_context_t*)(x))->xbp)
+#define APP_XSP(x) (((hotp_context_t*)(x))->xsp)
+#define APP_XBX(x) (((hotp_context_t*)(x))->xbx)
+#define APP_XDX(x) (((hotp_context_t*)(x))->xdx)
+#define APP_XCX(x) (((hotp_context_t*)(x))->xcx)
+#define APP_XAX(x) (((hotp_context_t*)(x))->xax)
+#define APP_R8(x)  (((hotp_context_t*)(x))->r8)
+#define APP_R9(x)  (((hotp_context_t*)(x))->r9)
+#define APP_R10(x) (((hotp_context_t*)(x))->r10)
+#define APP_R11(x) (((hotp_context_t*)(x))->r11)
+#define APP_R12(x) (((hotp_context_t*)(x))->r12)
+#define APP_R13(x) (((hotp_context_t*)(x))->r13)
+#define APP_R14(x) (((hotp_context_t*)(x))->r14)
+#define APP_R15(x) (((hotp_context_t*)(x))->r15)
+#define APP_XFLAGS(x) (((hotp_context_t*)(x))->xflags)
+
+#ifndef X64 /* legacy support */
+#define APP_EDI APP_XDI
+#define APP_ESI APP_XSI
+#define APP_EBP APP_XBP
+#define APP_ESP APP_XSP
+#define APP_EBX APP_XBX
+#define APP_EDX APP_XDX
+#define APP_ECX APP_XCX
+#define APP_EAX APP_XAX
+#endif
+
+#endif /* _HOTPATCH_INTERFACE_H_ */
diff --git a/core/lib/kstatsx.h b/core/lib/kstatsx.h
new file mode 100644
index 000000000..a810a7b19
--- /dev/null
+++ b/core/lib/kstatsx.h
@@ -0,0 +1,143 @@
+/* **********************************************************
+ * Copyright (c) 2004-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2004-2007 Determina Corp. */
+
+/*
+ * kstatsx.h
+ *
+ * Timing statistics definitions and descriptions
+ *
+ */
+
+/* This file is included multiple times 
+   - in stats.h once for structure definition, 
+   - in stats.c
+*/
+
+/* Client files that include this header should define the following macros
+#define KSTAT_DEF(description, name)
+#define KSTAT_SUM(description, name, var1, var2) name = var1 + var2
+*/
+
+/* Keep descriptions and names up to a reasonable length -
+ * respectively 50 and 15 characters */
+
+KSTAT_DEF("total measured and propagated in thread", thread_measured)
+
+KSTAT_DEF("in bb building", bb_building)
+KSTAT_DEF("in bb decoding", bb_decoding) /* sub-node of bb_building */
+KSTAT_DEF("in emitting BB", bb_emit) /* sub-node of bb_building */
+KSTAT_DEF("in mangling", mangling)
+KSTAT_DEF("in emit", emit)
+KSTAT_DEF("in hotpatch lookup", hotp_lookup)
+KSTAT_DEF("in trace building", trace_building)
+KSTAT_DEF("making temp private bb for trace building", temp_private_bb)
+KSTAT_DEF("in trace monitor ", monitor_enter)
+KSTAT_DEF("in trace monitor, thci ", monitor_enter_thci)
+KSTAT_DEF("cache flush unit walk ", cache_flush_unit_walk)
+KSTAT_DEF("flush_region", flush_region)
+KSTAT_DEF("synchall flush ", synchall_flush)
+KSTAT_DEF("coarse pclookup", coarse_pclookup)
+KSTAT_DEF("coarse freeze all", coarse_freeze_all)
+KSTAT_DEF("persisted cache generation", persisted_generation)
+KSTAT_DEF("persisted cache load", persisted_load)
+
+KSTAT_DEF("in dispatch exit, default", dispatch_num_exits)
+/* preserving STATS names for num_exits_* responsible for time in DR */
+KSTAT_DEF("in dispatch exit, ind target not in cache", num_exits_ind_good_miss)
+KSTAT_DEF("in dispatch exit, dir target not in cache", num_exits_dir_miss)
+KSTAT_SUM("in dispatch exit, all target not in cache", num_exits_not_in_cache,
+          num_exits_ind_good_miss, num_exits_dir_miss)
+
+KSTAT_DEF("in dispatch exit, BB2BB, ind target ...", num_exits_ind_bad_miss_bb2bb)
+KSTAT_DEF("in dispatch exit, BB2trace, ind target ...", num_exits_ind_bad_miss_bb2trace)
+KSTAT_SUM("in dispatch exit, from BB", num_exits_ind_bad_miss_bb, 
+          num_exits_ind_bad_miss_bb2bb, num_exits_ind_bad_miss_bb2trace)
+
+KSTAT_DEF("in dispatch exit, trace2trace, ind target ...", num_exits_ind_bad_miss_trace2trace)
+
+KSTAT_DEF("in dispatch exit, trace2BB not trace head, ind target", num_exits_ind_bad_miss_trace2bb_nth)
+KSTAT_DEF("in dispatch exit, trace2BB trace head, ind target", num_exits_ind_bad_miss_trace2bb_th)
+KSTAT_SUM("in dispatch exit, trace2BB, ind target ", num_exits_ind_bad_miss_trace2bb,
+          num_exits_ind_bad_miss_trace2bb_nth, num_exits_ind_bad_miss_trace2bb_th)
+
+KSTAT_SUM("in dispatch exit, from trace", num_exits_ind_bad_miss_trace,
+          num_exits_ind_bad_miss_trace2trace, num_exits_ind_bad_miss_trace2bb)
+
+KSTAT_SUM("in dispatch exit, ind target in cache but not table", num_exits_ind_bad_miss,
+          num_exits_ind_bad_miss_trace, num_exits_ind_bad_miss_bb)
+KSTAT_DEF("in dispatch exit, syscall handling", num_exits_dir_syscall)
+KSTAT_DEF("in dispatch exit, callback return", num_exits_dir_cbret)
+
+KSTAT_DEF("in LOG", logging)
+
+KSTAT_DEF("empty block overhead", overhead_empty)
+KSTAT_DEF("nested block overhead", overhead_nested)
+
+KSTAT_DEF("in syscalls [not propagated]", syscall_fcache)
+KSTAT_DEF("pre-syscall handling", pre_syscall)
+KSTAT_DEF("post-syscall handling", post_syscall)
+KSTAT_DEF("pre-syscall FreeVM handling", pre_syscall_free)
+KSTAT_DEF("pre-syscall ProtectVM handling", pre_syscall_protect)
+KSTAT_DEF("pre-syscall Unmap handling", pre_syscall_unmap)
+KSTAT_DEF("post-syscall AllocVM handling", post_syscall_alloc)
+KSTAT_DEF("post-syscall Map handling", post_syscall_map)
+
+KSTAT_DEF("native_exec [not propagated]", native_exec_fcache)
+
+KSTAT_DEF("in fcache, default", fcache_default)
+KSTAT_DEF("in bb cache, [not propagated]", fcache_bb_bb)
+KSTAT_DEF("in trace cache, [not propagated]", fcache_trace_trace)
+/* hard to SUM it up against either bb or trace only */
+KSTAT_DEF("in bb cache out from trace cache, [not propagated]", fcache_bb_trace)
+
+/* assuming we'll deal with lock contention separately we don't propagate this time to callers */
+KSTAT_DEF("wait event (+context switch) [not propagated]", wait_event)
+
+/* FIXME: we should add all critical section bodies as suggested in
+ * the mutex_t definition - if only we can share LOCK_RANK definitions
+ * in lockx.h and it will just work
+ */
+
+KSTAT_DEF("in rct analysis no relocations", rct_no_reloc)
+KSTAT_DEF("in rct analysis using relocations [outer loop]", rct_reloc)
+KSTAT_DEF("in rct analysis using relocations [per page loop]", rct_reloc_per_page)
+KSTAT_DEF("in aslr_generate_relocated_section for validation", aslr_validate_relocate)
+KSTAT_DEF("in module_contents_compare or aslr_compare_in_place", aslr_compare)
+
+#ifdef KSTAT_UNIT_TEST
+KSTAT_DEF("empty block overhead", empty)
+KSTAT_DEF("total measured", measured)
+KSTAT_DEF("in outer loop", iloop)
+KSTAT_DEF("in inner loop", jloop)
+#endif /* KSTAT_UNIT_TEST */
diff --git a/core/lib/statsx.h b/core/lib/statsx.h
new file mode 100644
index 000000000..f5e85d2a7
--- /dev/null
+++ b/core/lib/statsx.h
@@ -0,0 +1,1164 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+
+/*
+ * statsx.h
+ *
+ * Statistics definitions and descriptions
+ *
+ */
+
+/* This file is included multiple times 
+   - in dynamorio_stats.h once for structure definition, 
+   - in utils.c - dump_global_stats, dump_thread_stats
+   - TODO: should also be included in the GUI
+*/
+
+/* Client files that include this header should define the following macros:
+#ifdef DEBUG
+# define STATS_DEF(description, name) // def here
+#else
+# define RSTATS_DEF(description, name) // def here
+#endif
+#undef STATS_DEF
+#undef RSTATS_DEF
+*/
+
+/* Add new statistics at the end so that older GUIs can still show old statistics */
+/* Try to avoid preprocessor directives in STATS - 
+   shouldn't matter, and makes old GUIs compatible */
+/* Keep descriptions up to a reasonable length (column 60) */
+
+/* use RSTATS_DEF for statistics that are available in release build as well
+ * as debug (you must use RSTATS_INC, etc. as well with such stats)
+ */
+#ifdef DEBUG
+# define RSTATS_DEF STATS_DEF
+#else
+# define STATS_DEF(desc, name) /* nothing */
+#endif
+
+    STATS_DEF("Application exited", exited)
+    RSTATS_DEF("Threads under DynamoRIO control", num_threads)
+    RSTATS_DEF("Peak threads under DynamoRIO control", peak_num_threads)
+    RSTATS_DEF("Threads ever created", num_threads_created)
+    STATS_DEF("Threads killed", num_threads_killed)
+    STATS_DEF("Threads killed cleanly", num_threads_killed_cleanly)
+#ifdef WINDOWS
+    RSTATS_DEF("Callbacks", num_callbacks)
+    STATS_DEF("Callbacks in no-asynch threads", num_callbacks_noasynch)
+    RSTATS_DEF("APCs", num_APCs)
+    STATS_DEF("APCs in no-asynch threads", num_APCs_noasynch)
+    RSTATS_DEF("Exceptions", num_exceptions)
+    STATS_DEF("Exceptions in no-asynch threads", num_exceptions_noasynch)
+    STATS_DEF("KiRaiseUserException instances", num_raise_exceptions)
+    STATS_DEF("KiRaiseUserException in no-asynch threads", num_raise_exceptions_noasynch)
+    STATS_DEF("Asynch event while lost control", num_asynch_while_lost)
+    STATS_DEF("Exceptions while lost control", num_except_while_lost)
+    STATS_DEF("Exceptions at SetContext points for reset", num_reset_setcontext_at_fault)
+    STATS_DEF("Exceptions at SetContext points for app", num_app_setcontext_at_fault)
+    STATS_DEF("Exceptions in decoding app memory", num_exceptions_decode)
+    STATS_DEF("GetContextThread w/o CONTEXT_CONTROL", num_app_getcontext_no_control)
+    STATS_DEF("SetContextThread w/o CONTEXT_CONTROL", num_app_setcontext_no_control)
+    STATS_DEF("Re-takeovers after native", num_retakeover_after_native)
+#else
+    RSTATS_DEF("Total signals delivered", num_signals)
+#endif
+    RSTATS_DEF("System calls, pre", pre_syscall)
+    RSTATS_DEF("System calls, post", post_syscall)
+    STATS_DEF("System calls, pre, ignorable", pre_syscall_ignorable)
+    STATS_DEF("System calls, post, ignorable", post_syscall_ignorable)
+    STATS_DEF("Ignorable system calls", ignorable_syscalls)
+#ifdef WINDOWS
+    STATS_DEF("Ignorable system calls not matching sysenter pattern",
+              ignorable_syscalls_failed_sysenter_pattern)
+#endif
+    STATS_DEF("Optimizable system calls", optimizable_syscalls)
+    STATS_DEF("Non-ignorable system calls", non_ignorable_syscalls)
+#ifdef WINDOWS
+    STATS_DEF("Instances of interrupt 2B", num_int2b)
+#endif
+    RSTATS_DEF("Native modules present", num_native_module_loads)
+    STATS_DEF("Native module entrance checks", num_native_entrance_checks)
+    STATS_DEF("Native module entrance TOS checks", num_native_entrance_TOS_checks)
+    STATS_DEF("Native module entrance TOS decodes", num_native_entrance_TOS_decodes)
+    STATS_DEF("Native module entrance potential misses", num_native_entrance_miss)
+    STATS_DEF("Native module entrance blocks", num_native_module_entrances)
+    STATS_DEF("Native module entrance blocks, call", num_native_module_entrances_call)
+    STATS_DEF("Native module entrance blocks, call*", num_native_module_entrances_indcall)
+    STATS_DEF("Native module entrance blocks, call+jmp*", num_native_module_entrances_plt)
+    STATS_DEF("Native module entrance blocks, TOS unknown", num_native_module_entrances_TOS_unknown)
+    STATS_DEF("Native module entrance blocks, TOS jmp*", num_native_module_entrances_TOS_jmp)
+    STATS_DEF("Native module entrance blocks, call* jmp/call", num_native_module_entrances_callcall)
+    STATS_DEF("Native module execution entrances", num_native_module_enter)
+    STATS_DEF("Native module execution exits", num_native_module_exit)
+    STATS_DEF("Native our-fault write faults", num_native_cachecons_faults)
+    STATS_DEF("System call trampolines, total", num_syscall_trampolines)
+    STATS_DEF("System call trampolines, native", num_syscall_trampolines_native)
+    STATS_DEF("System call trampolines, cache", num_syscall_trampolines_cache)
+    STATS_DEF("System call trampolines, DR", num_syscall_trampolines_DR)
+    STATS_DEF("System call trampolines, retakeover", num_syscall_trampolines_retakeover)
+    RSTATS_DEF("Application mmaps", num_app_mmaps)
+    RSTATS_DEF("Application munmaps", num_app_munmaps)
+    STATS_DEF("Module rebindings", num_app_rebinds)
+#ifdef WINDOWS
+    STATS_DEF("Application map mismatches with sections", map_section_mismatch)
+#endif
+    STATS_DEF("Application modules with long names", app_modname_too_long)
+    STATS_DEF("Application modules with code", num_app_code_modules)
+    STATS_DEF("Application code seen (bytes)", app_code_seen)
+    STATS_DEF("Interpreted calls, direct and indirect", num_all_calls)
+    STATS_DEF("Interpreted indirect calls", num_indirect_calls)
+    STATS_DEF("Interpreted convertible indirect calls", num_convertible_indcalls)
+    STATS_DEF("Interpreted to-sysenter indirect calls", num_sysenter_indcalls)
+    STATS_DEF("Interpreted indirect jmps", num_indirect_jumps)
+    STATS_DEF("Interpreted rets", num_returns)
+    STATS_DEF("Interpreted far direct calls", num_far_dir_calls)
+    STATS_DEF("Interpreted far direct jmps", num_far_dir_jmps)
+    STATS_DEF("Interpreted far indirect calls", num_far_ind_calls)
+    STATS_DEF("Interpreted far indirect jmps", num_far_ind_jmps)
+    STATS_DEF("Interpreted far rets", num_far_rets)
+    STATS_DEF("Interpreted irets", num_irets)
+    STATS_DEF("Decoded jcc branch hints", num_branch_hints)
+
+    STATS_DEF("Dynamic option synchronizations", option_synchronizations)
+    STATS_DEF("Dynamic option synchronizations, no change", option_synchronizations_nop)
+    STATS_DEF("Security violations, heap", num_heap_violations)
+    STATS_DEF("Security violations, stack", num_stack_violations)
+#ifdef WINDOWS
+    STATS_DEF("Exempted regions, NtFlush, +w", num_NT_flush_w)
+    STATS_DEF("Exempted regions, NtFlush, -w", num_NT_flush_r)
+    STATS_DEF("Exempted regions, NtFlush, +w & on exec list", num_NT_flush_w2r)
+    STATS_DEF("Non-exempt regions b/c NtFlush was by loader", num_NT_flush_loader)
+    STATS_DEF("Exempted regions, NtFlush, heap", num_NT_flush_heap)
+    STATS_DEF("Exempted regions, NtFlush, stack", num_NT_flush_stack)
+    STATS_DEF("Exempted regions, loader-marked", num_mark_after_load)
+#endif
+    STATS_DEF("Exempted regions, ->rx text", num_2rx_text)
+    STATS_DEF("Exempted regions, any text", num_text)
+    STATS_DEF("Exempted regions, rwx text", num_hook)
+    STATS_DEF("Non-exempted regions, rwx text, native module", num_hook_native_ignored)
+    STATS_DEF("Exempted regions, on text list", num_text_list)
+    STATS_DEF("Exempted regions, on IMAGE text list", num_image_text_list)
+    STATS_DEF("Exempted regions, .data section", num_dot_data)
+    STATS_DEF("Exempted regions, on .data section list", num_dot_data_list)
+    STATS_DEF("Exempted regions, .data x section", num_dot_data_x)
+    STATS_DEF("Exempted regions, on .data x section list", num_dot_data_x_list)
+    STATS_DEF("Exempted regions, ..x", num_mark_if_x)
+    STATS_DEF("Exempted regions, .-x", num_mark_if_rx)
+    STATS_DEF("Exempted regions, loaded executable or library image", num_exempt_image)
+    STATS_DEF("Exempted regions, on loaded executable or library image list", num_exempt_image_list)
+    STATS_DEF("Exempted regions, on moduledb exempt image list", num_moduledb_exempt_image)
+    STATS_DEF("Exempted threats", num_exempt_threat)
+    STATS_DEF("Silently blocked threats", num_silently_blocked_threat)
+    STATS_DEF("Exempted regions from dll2heap", num_exempt_dll2heap)
+    STATS_DEF("Exempted regions from moduledb exempt dll2heap", num_moduledb_exempt_dll2heap)
+    STATS_DEF("Exempted regions from dll2stack", num_exempt_dll2stack)
+    STATS_DEF("Exempted regions from moduledb exempt dll2stack", num_moduledb_exempt_dll2stack)
+    STATS_DEF("Exempted regions, alloc", num_alloc_exec)
+    STATS_DEF("Exempted future executions, heap", num_exec_future_heap)
+    STATS_DEF("Exempted future executions, stack", num_exec_future_stack)
+    STATS_DEF("Exempted future executions, once-only", num_exec_future_once)
+#ifdef WINDOWS
+    STATS_DEF("Exempted loader-marked executions", num_exec_after_load)
+#endif
+    STATS_DEF("Exempted trampoline executions, heap", num_trampolines_heap)
+    STATS_DEF("Exempted trampoline executions, stack", num_trampolines_stack)
+    STATS_DEF("Exempted trampoline direct calls", trampolines_direct_calls)
+    STATS_DEF("Exempted trampoline direct jmps", trampolines_direct_jmps)
+    STATS_DEF("Exempted .NET COM method table returns", trampolines_com_rets)
+    STATS_DEF("Exempted displaced hook code", trampolines_displaced_code)
+    STATS_DEF("Exempted assumed driver address", num_driver_areas)
+
+    STATS_DEF("Code origin addresses checked", checked_addresses)
+    STATS_DEF("Code origin addresses in last area", looked_up_in_last_area)
+
+    STATS_DEF("Writable code regions", num_writable_code_regions)
+    STATS_DEF("Writable code regions we made read-only", num_rw2r_code_regions)
+    STATS_DEF("Writable executable regions we made read-only", num_delayed_rw2r)
+    STATS_DEF("Memory regions marked as sandboxed", num_selfmod_vm_areas)
+    STATS_DEF("Code regions not switched due to other sub-page", num_ro2sandbox_other_sub)
+    STATS_DEF("Code regions switched read-only to sandbox", num_ro2sandbox)
+    STATS_DEF("Code regions sandboxed before being read-only", num_sandbox_before_ro)
+    STATS_DEF("Code regions switched sandbox to read-only", num_sandbox2ro)
+    STATS_DEF("Code regions race in sandbox2ro attempt", num_sandbox2ro_race)
+    STATS_DEF("Code regions flush race in sandbox2ro attempt", num_sandbox2ro_flush_race)
+    STATS_DEF("Code regions on stack with sandbox counters", num_sandbox2ro_onstack)
+    STATS_DEF("Sandboxed fragments", num_sandboxed_fragments)
+    STATS_DEF("Sandboxed fragment executions", num_sandbox_execs)
+    STATS_DEF("Blocks ended early due to sandboxing limitations", num_bb_end_early)
+    STATS_DEF("Self-writes detected by sandboxing", num_self_writes)
+    STATS_DEF("Self-writes overruled by flushes", num_self_writes_after_flushes)
+    STATS_DEF("Write faults on read-only code regions", num_write_faults)
+    STATS_DEF("Write fault races", num_write_fault_races)
+    STATS_DEF("Write fault races, one selfmod", num_write_fault_races_selfmod)
+    STATS_DEF("Flushes racy, no exec removal since selfmod", flush_selfmod_race_no_remove)
+    STATS_DEF("Cache consistency flushes", num_flushes)
+    STATS_DEF("Cache consistency flushes that flushed nothing", num_empty_flushes)
+    STATS_DEF("Cache consistency flushes via synchall", flush_synchall)
+    STATS_DEF("Thread not translated in synchall flush (flush race)", flush_synchall_races)
+    STATS_DEF("Thread not synched with in synchall flush", flush_synchall_fail)
+    STATS_DEF("Cache consistency coarse units flushed", flush_coarse_units)
+    STATS_DEF("Cache consistency persisted units flushed", flush_persisted_units)
+    STATS_DEF("Cache consistency persisted flushed at unload", flush_persisted_unload)
+    STATS_DEF("Pending-delete fragments re-deleted", deleted_frags_re_deleted)
+    STATS_DEF("Cache consistency non-code nop flushes", num_noncode_flushes)
+    STATS_DEF("Flushes that flushed >=1 shared fragment", num_shared_flushes)
+    STATS_DEF("Flushes of entire cache", fcache_flush_all)
+    STATS_DEF("Fcache units flushed", cache_units_flushed)
+    STATS_DEF("Fcache units flushed and freed", cache_units_flushed_freed)
+    STATS_DEF("Fcache units on to-flush list", cache_units_toflush)
+    STATS_DEF("Peak fcache units on to-flush list", peak_cache_units_toflush)
+    STATS_DEF("Fcache units on to-free list", cache_units_tofree)
+    STATS_DEF("Peak fcache units on to-free list", peak_cache_units_tofree)
+    STATS_DEF("Fcache units flushed for wset", cache_units_wset_flushed)
+    STATS_DEF("Fcache units allowed w/o a flush for wset", cache_units_wset_allowed)
+    STATS_DEF("Fcache units flushed w/ no live fragments", cache_units_flushed_nolive)
+    STATS_DEF("Flushes of vmvector areas", num_flush_vmvector)
+    STATS_DEF("Shared deletion regions unlinked", num_shared_flush_regions)
+    STATS_DEF("Shared deletion region walks", num_shared_flush_walks)
+    STATS_DEF("Shared deletion region at-syscall walks", num_shared_flush_atsyscall)
+    STATS_DEF("Shared deletion ref count decrements", num_shared_flush_refdec)
+    STATS_DEF("Shared deletion flushtime diff > #threads", num_shared_flush_diffthreads)
+    STATS_DEF("Shared deletion max flushtime diff", num_shared_flush_maxdiff)
+    STATS_DEF("Shared deletion max pending", num_shared_flush_maxpending)
+    STATS_DEF("Shared deletion region removals: ref 0", num_shared_flush_refzero)
+    STATS_DEF("Shared deletion region removals: at exit", num_shared_flush_atexit)
+    STATS_DEF("Shared deletion region removals: at reset", num_shared_flush_atreset)
+    STATS_DEF("Shared deletion: shared IBT tables unlinked",
+              num_shared_ibt_table_unlinks)
+    STATS_DEF("Shared deletion: empty shared IBT tables skipped",
+              num_shared_empty_ibt_tables_skipped)
+    STATS_DEF("Shared deletion: shared IBTs unlinked", num_shared_ibt_unlinks)
+    STATS_DEF("Shared deletion: shared IBT entries skipped",
+              num_shared_ibt_entries_skipped)
+    STATS_DEF("Shared deletion: shared IBT entries examined",
+              num_shared_ibt_entries_examined)
+    STATS_DEF("Flush queue marked nonempty to relink shared_syscall",
+              num_flushq_relink_syscall)
+    STATS_DEF("Flush queue marked nonempty, yet empty", num_flushq_actually_empty)
+    STATS_DEF("Fragments added to lazy deletion list", num_lazy_deletion_appends)
+    STATS_DEF("Fragments freed from lazy deletion list at exit",
+              num_lazy_deletion_frees_atexit)
+    STATS_DEF("Fragments freed from lazy deletion list", num_lazy_deletion_frees)
+    STATS_DEF("Lazy list instances moved to pending list", num_lazy_del_to_pending)
+    STATS_DEF("Lazy list fragments moved to pending list", num_lazy_del_frags_to_pending)
+    STATS_DEF("Translation info computed", translations_computed)
+    STATS_DEF("Fragments with translation info stored", num_fragment_translation_stored)
+    STATS_DEF("Resets of entire fcache, proactively", fcache_reset_proactively)
+    STATS_DEF("Resets due to too many pending deletions", fcache_reset_pending_del)
+    STATS_DEF("Resets aborted due to thread synch problems", fcache_reset_abort)
+    STATS_DEF("Resets due to running low on vmm_heap", reset_low_vmm_count)
+    STATS_DEF("Resets due to running low on committable memory", reset_low_commit_count)
+    STATS_DEF("Number of queries of the commit limit", commit_limit_queries)
+    STATS_DEF("Data section protection changes", datasec_prot_changes)
+    STATS_DEF("Data section protection wasted calls", datasec_prot_wasted_calls)
+    STATS_DEF("Data section checks: unprotected (racy)", datasec_not_prot)
+    STATS_DEF("Generated code protection changes", gencode_prot_changes)
+    STATS_DEF("Protection change calls", protection_change_calls)
+    STATS_DEF("Protection change pages", protection_change_pages)
+    STATS_DEF("Unload race detected", num_unloaded_race)
+    STATS_DEF("Unload race prevented spurious .B", num_unloaded_race_code_origins)
+
+#ifdef RETURN_AFTER_CALL
+    STATS_DEF("After call instructions, total added", ret_after_call_added)
+    STATS_DEF("After call instructions, existing", num_existing_after_call)
+    STATS_DEF("After call instructions, current", num_future_after_call)
+    STATS_DEF("After call instructions, removed", num_future_after_call_removed)
+    STATS_DEF("RAC persisted tables created", rac_persisted_tables)
+    STATS_DEF("RAC persisted table entries", rac_persisted_entries)
+    STATS_DEF("RAC live tables created", rac_live_tables)
+    STATS_DEF("RAC live table entries", rac_live_entries)
+    STATS_DEF("RAC non-module table entries", rac_non_module_entries)
+    STATS_DEF("Return after call total validations", ret_after_call_validations)
+    STATS_DEF("Return after call good returns", ret_after_call_good)
+    STATS_DEF("Return after call, before start ok", ret_after_call_before_start)
+    STATS_DEF("Return after call, signal restorers ok", ret_after_call_signal_restorer)
+    STATS_DEF("Return after call, known exceptions ok", ret_after_call_known_exceptions)
+    STATS_DEF("Return after call, SEH exemptions ok", ret_after_call_SEH)
+    STATS_DEF("Return after call, push reg; ret", ret_after_call_pushregret)
+    STATS_DEF("Return after call, .xdata call* target", ret_after_call_xdata)
+    STATS_DEF("Return after call, known exempt exceptions ok", ret_after_call_exempt_exceptions)
+    STATS_DEF("Return after call, during unload prevented spurious .C", num_unloaded_race_during)
+    STATS_DEF("Return after call, after unload, silenced .C", num_unloaded_race_after)
+    STATS_DEF("RCT, moduledb exemptions ok", num_rct_moduledb_exempt)
+    STATS_DEF("RCT, driver address ok", num_rct_driver_address)
+    STATS_DEF("Return after call, exempt locations, added", ret_after_call_exempt_added)
+    STATS_DEF("Return after call violations, BAD", ret_after_call_violations)
+#endif /* RETURN_AFTER_CALL */
+#ifdef RCT_IND_BRANCH
+    STATS_DEF("RCT persisted tables created", rct_persisted_tables)
+    STATS_DEF("RCT persisted table entries", rct_persisted_entries)
+    STATS_DEF("RCT live tables created", rct_live_tables)
+    STATS_DEF("RCT live table entries", rct_live_entries)
+    STATS_DEF("RCT indirect branch targets, total", rct_ind_branch_valid_targets)
+    STATS_DEF("RCT indirect branch target entries, current", rct_ind_branch_entries)
+    STATS_DEF("RCT indirect branch target entries, removed", rct_ind_branch_entries_removed)
+    STATS_DEF("RCT indirect branch targets existing", rct_ind_branch_existing_targets)
+    STATS_DEF("RCT indirect branch modules added", rct_ind_branch_modules_analyzed)
+    STATS_DEF("RCT indirect branch sections added", rct_ind_branch_sections_analyzed)
+    STATS_DEF("RCT indirect branch not code section", rct_ind_branch_not_code_section)
+    STATS_DEF("RCT modules loaded from persisted data",
+              rct_ind_branch_modules_persist_loaded)
+    STATS_DEF("RCT modules with partial persisted data",
+              rct_ind_branch_modules_persist_partial)
+    STATS_DEF("RCT indirect branch unload race", rct_ind_branch_unload_race)
+    STATS_DEF("RCT indirect branch ref outside module", rct_ind_branch_ref_outside_module)
+    STATS_DEF("RCT indirect branch reloc sec w/o valid targets", rct_ind_branch_no_valid_targets)
+    STATS_DEF("RCT indirect branch exports added", rct_ind_added_exports)
+    STATS_DEF("RCT indirect branch exports already added", rct_ind_already_added_exports)
+    STATS_DEF("RCT indirect branch ok after analyze-at-violation", rct_ok_at_vio)
+    STATS_DEF("RCT indirect branch modules scanned at init", rct_scan_at_init)
+    STATS_DEF("RCT indirect branch modules scanned at violation", rct_scan_at_vio)
+    STATS_DEF("RCT indirect branch rip-rel new targets added", rct_ind_rip_rel_new)
+    STATS_DEF("RCT indirect branch rip-rel targets already added", rct_ind_rip_rel_old)
+    STATS_DEF("RCT indirect branch rip-rel scan added", rct_ind_rip_rel_scan_new)
+    STATS_DEF("RCT indirect branch rip-rel scan already added", rct_ind_rip_rel_scan_old)
+    STATS_DEF("RCT indirect branch rip-rel scan out of range", rct_ind_rip_rel_scan_data)
+    STATS_DEF("RCT indirect branch SEH64 new targets added", rct_ind_seh64_new)
+    STATS_DEF("RCT indirect branch SEH64 targets already added", rct_ind_seh64_old)
+    STATS_DEF("RCT indirect branch SEH64 plus-1 entries", rct_ind_seh64_plus1)
+    STATS_DEF("RCT indirect branch total validations", rct_ind_branch_validations)
+    STATS_DEF("RCT indirect call good", rct_ind_call_good)
+    STATS_DEF("RCT indirect jmp good", rct_ind_jmp_good)
+    STATS_DEF("RCT indirect call exports", rct_ind_call_exports)
+    STATS_DEF("RCT indirect jmp exports", rct_ind_jmp_exports)
+    STATS_DEF("RCT indirect jmp instead of ret to after call", rct_ind_jmp_allowed_to_ac)
+    STATS_DEF("RCT indirect call exemptions", rct_ind_call_exemptions)
+    STATS_DEF("RCT indirect jmp exemptions", rct_ind_jmp_exemptions)
+    STATS_DEF("RCT indirect jmp x64-switch exemptions", rct_ind_jmp_x64switch)
+
+    STATS_DEF("RCT indirect call violations, BAD", rct_ind_call_violations)
+    STATS_DEF("RCT indirect jmp violations, BAD", rct_ind_jmp_violations)
+#endif /* RCT_IND_BRANCH */
+    STATS_DEF("App protection changes of DR memory", app_modify_DR_prot)
+    STATS_DEF("App protection changes of ntdll code", app_modify_ntdll_prot)
+    STATS_DEF("App protection changes of patch proof module", app_modify_module_prot)
+    STATS_DEF("App loads of patch proof modules", num_patch_proof_module_loads)
+    STATS_DEF("App writes to pretend-writable DR memory", app_modify_DR_writes)
+    STATS_DEF("App writes to pretend-writable ntdll memory", app_modify_ntdll_writes)
+    STATS_DEF("App writes to pretend-writable module memory", app_modify_patch_module_writes)
+    STATS_DEF("App writes emulated successfully", num_emulated_writes)
+    STATS_DEF("App writes emulated un-successfully", num_emulated_write_failures)
+
+    STATS_DEF("Fragments generated, bb and trace", num_fragments)
+    RSTATS_DEF("Basic block fragments generated", num_bbs)
+    RSTATS_DEF("Trace fragments generated", num_traces)
+    STATS_DEF("Trace fragments aborted for any reason", num_aborted_traces)
+    STATS_DEF("Trace fragments aborted: shared race", num_aborted_traces_race)
+    STATS_DEF("Trace fragments aborted: client bad mod", num_aborted_traces_client)
+    STATS_DEF("Trace building aborted: shared race", num_trace_building_race)
+    STATS_DEF("Trace building truncated: next bb deleted", num_trace_next_bb_deleted)
+    STATS_DEF("Trace building reset: no trace head",
+              num_reset_trace_no_trace_head)
+    STATS_DEF("Trace building reset: in-progress flag cleared",
+              num_trace_building_ip_cleared)
+    STATS_DEF("Number of bbs in all emitted traces", num_bbs_in_all_traces)
+    STATS_DEF("Number of bbs in all aborted traces", num_bbs_in_all_aborted_traces)
+    STATS_DEF("Maximum number of bbs in a trace", max_bbs_in_a_trace)
+    STATS_DEF("Traces truncated due to cache size limits", num_max_trace_size_enforced)
+    STATS_DEF("Number of times max_trace_bbs was enforced", num_max_trace_bbs_enforced)
+    STATS_DEF("Trace wannabes prevented from being traces", num_wannabe_traces)
+    STATS_DEF("Trace head too large to be a trace", num_huge_fragments)
+    STATS_DEF("Shared trace links shifted back to trace head", links_shared_trace_to_head)
+    STATS_DEF("Shadowed trace head deleted", shadowed_trace_head_deleted)
+    STATS_DEF("Trace head counters reset on trace deletion", th_counter_reset)
+    STATS_DEF("Trace heads re-marked", trace_head_remark)
+    STATS_DEF("Future fragments generated", num_future_fragments)
+    STATS_DEF("Shared fragments generated", num_shared_fragments)
+    STATS_DEF("Shared bbs generated", num_shared_bbs)
+    STATS_DEF("Shared traces generated", num_shared_traces)
+    STATS_DEF("Private fragments generated", num_private_fragments)
+    STATS_DEF("Private bbs generated", num_private_bbs)
+    STATS_DEF("Private traces generated", num_private_traces)
+    STATS_DEF("Shared future fragments generated", num_shared_future_fragments)
+    STATS_DEF("Unique fragments generated", num_unique_fragments)
+    STATS_DEF("Maximum fragment requested size in bytes", max_fragment_requested_size)
+    STATS_DEF("Maximum fragment size in bytes", max_fragment_size)
+    STATS_DEF("Maximum instrs in a bb", max_instrs_in_a_bb)
+    STATS_DEF("BBs truncated due to instr limits", num_max_bb_instrs_enforced)
+    STATS_DEF("BBs truncated due to selfmod write limits", num_max_selfmod_writes_enforced)
+    STATS_DEF("Maximum elided jmps in a bb", max_elided_jmps)
+    STATS_DEF("Maximum elided calls in a bb", max_elided_calls)
+    STATS_DEF("Total elided jmps in all bbs", total_elided_jmps)
+    STATS_DEF("Total elided calls in all bbs", total_elided_calls)
+    STATS_DEF("BB elisions prevented for coarse-grain", elisions_prevented_for_coarse)
+    STATS_DEF("Trace heads unlinked in IBTs", num_th_bb_ibt_unlinked)
+    STATS_DEF("Trace head IBT add race", num_th_bb_ibt_add_race)
+    STATS_DEF("Direct exit stubs created", num_direct_exit_stubs)
+    STATS_DEF("Indirect exit stubs created", num_indirect_exit_stubs)
+    STATS_DEF("Separate stubs created", num_separate_stubs)
+    STATS_DEF("Entrance stubs created", num_entrance_stubs)
+#ifdef X64
+    STATS_DEF("Rip-relative instrs mangled", rip_rel_instrs)
+    STATS_DEF("Rip-relative leas mangled", rip_rel_lea)
+    STATS_DEF("Rip-relative instrs w/ un-reachable targets", rip_rel_unreachable)
+    STATS_DEF("Rip-relative unreachable spill avoided", rip_rel_unreachable_nospill)
+#endif
+    STATS_DEF("BBs with one indirect exit", num_bb_one_indirect_exit)
+    STATS_DEF("BBs with indirect exit + other exits", num_bb_indirect_extra_exits)
+    STATS_DEF("BBs with one direct exit", num_bb_one_direct_exit)
+    STATS_DEF("BBs with two direct exits", num_bb_two_direct_exits)
+    STATS_DEF("BBs with >2 direct exits", num_bb_many_direct_exits)
+    STATS_DEF("BBs with an elided cti", num_bb_has_elided)
+    STATS_DEF("BBs with an also_vmarea", num_bb_also_vmarea)
+    STATS_DEF("BB direct exits >SHRT_MAX from fragment tag", num_bb_exit_tgt_far)
+    STATS_DEF("BB direct exits <=SHRT_MAX from fragment tag", num_bb_exit_tgt_near)
+    STATS_DEF("BB cbr fall-through >SHRT_MAX from fragment tag", num_bb_fallthru_far)
+    STATS_DEF("BB cbr fall-through <=SHRT_MAX from fragment tag", num_bb_fallthru_near)
+    STATS_DEF("BBs using post-linkstub fragment offset", num_bb_fragment_offset)
+    STATS_DEF("BBs using cbr fall-through smaller linkstub", num_bb_cbr_fallthru_shrink)
+    STATS_DEF("BBs that write OF but no other arithmetic flags", bbs_eflags_writes_of)
+    STATS_DEF("BBs that write all 6 arithmetic flags", bbs_eflags_writes_6)
+    STATS_DEF("BBs that read a flag before writing any", bbs_eflags_reads)
+    STATS_DEF("BBs that write no arithmetic flags", bbs_eflags_writes_none)
+    STATS_DEF("BBs that write no arithmetic flags, end in ib", bbs_eflags_writes_none_ind)
+    STATS_DEF("Cbrs sharing a single exit stub", num_cbr_single_stub)
+    STATS_DEF("Fragments requiring post_linkstub offs", num_fragment_post_linkstub)
+    STATS_DEF("Fragments smaller than minimum fcache slot size", num_fragment_too_small)
+    STATS_DEF("Fragments final size < minimum fcache slot size", num_final_fragment_too_small)
+    STATS_DEF("Fragments unlinked for flushing", num_flushed_fragments)
+    STATS_DEF("Fragments deleted for any reason", num_fragments_deleted)
+    STATS_DEF("Fragments unlinked for deletion", fragments_unlinked_for_deletion)
+    STATS_DEF("Fragments unlinked for deletion from list", 
+              list_entries_unlinked_for_deletion)
+    STATS_DEF("Shared fragments deleted no-flush, race", shared_delete_noflush_race)
+    STATS_DEF("Trace component fragments deleted", trace_components_deleted)
+    STATS_DEF("Fragments deleted due to capacity conflicts", num_fragments_replaced)
+    STATS_DEF("Fragments deleted on thread/process death", num_fragments_deleted_exit)
+    STATS_DEF("Fragments deleted on thread/process reset", num_fragments_deleted_reset)
+    STATS_DEF("Trace heads marked", num_trace_heads_marked)
+    STATS_DEF("Fragments deleted and replaced with traces", num_fragments_deleted_trace_heads)
+    STATS_DEF("Fragments deleted after selfmod", num_fragments_deleted_selfmod)
+    STATS_DEF("Fragments deleted for munmap or RO consistency", num_fragments_deleted_consistency)
+    STATS_DEF("Fragments deleted for copy & replace", num_fragments_deleted_copy_and_replace)
+#ifdef CLIENT_INTERFACE
+    STATS_DEF("Fragments deleted by client interface", num_fragments_deleted_client)
+#endif
+#ifdef SIDELINE
+    STATS_DEF("Fragments deleted by sideline replacement", num_fragments_deleted_sideline)
+#endif
+    STATS_DEF("Trace fragments in 3 IBL tables", num_traces_in_3_ibl_tables)
+    STATS_DEF("Trace fragments in 2 IBL tables", num_traces_in_2_ibl_tables)
+    STATS_DEF("Trace fragments in 1 IBL tables", num_traces_in_1_ibl_tables)
+    STATS_DEF("Trace fragments targeted by IBL", num_traces_ibl_targets)
+    STATS_DEF("IBTs replaced previous fragments", num_ibt_replace_previous_fragments)
+    STATS_DEF("IBTs replaced unlinked fragments",
+              num_ibt_replace_unlinked_fragments)
+    STATS_DEF("IBT linked/unlinked table rehashes", num_ibt_table_rehashes)
+    STATS_DEF("IBT resizes", num_ibt_table_resizes)
+    STATS_DEF("Same-size IBT table resizes", num_same_size_ibt_table_resizes)
+    STATS_DEF("Shared IBT table flushes", num_shared_ibt_table_flushes)
+    STATS_DEF("Shared IBT table ptr resets", num_shared_ibt_table_ptr_resets)
+    STATS_DEF("IBT adds disallowed shared table, private fragment",
+              num_ibt_shared_private_conflict)
+    STATS_DEF("Dead shared IBT tables", num_dead_shared_ibt_tables)
+    STATS_DEF("Peak # dead shared IBT tables", peak_num_dead_shared_ibt_tables)
+    STATS_DEF("Total dead shared IBT tables", num_total_dead_shared_ibt_tables)
+    STATS_DEF("Dead shared IBT tables freed", num_dead_shared_ibt_tables_freed)
+    STATS_DEF("Dead shared IBT tables freed: at exit",
+              num_dead_shared_ibt_tables_freed_at_exit)
+    STATS_DEF("Shared IBT tables freed: immediately",
+              num_shared_ibt_tables_freed_immediately)
+    STATS_DEF("Pvt ptrs to shared tables updated at-syscall walks",
+              num_shared_tables_updated_atsyscall)
+    STATS_DEF("IBT unlinked entries NOT moved on resize",
+              num_ibt_unlinked_entries_not_moved)
+    STATS_DEF("BB fragments in 3 IBL tables", num_bbs_in_3_ibl_tables)
+    STATS_DEF("BB fragments in 2 IBL tables", num_bbs_in_2_ibl_tables)
+    STATS_DEF("BB fragments in 1 IBL tables", num_bbs_in_1_ibl_tables)
+    STATS_DEF("BB fragments targeted by IBL", num_bbs_ibl_targets)
+    STATS_DEF("Exits due to IBL cold misses", num_ibt_cold_misses)
+    STATS_DEF("Exits due to IB targeting TH", num_ib_th_target)
+    STATS_DEF("Exits preventable if IB targeted bbs", num_ibt_bb_preventable)
+    STATS_DEF("Extra exits due to trace building", num_ibt_exit_trace_building)
+    STATS_DEF("Extra exits due to IBL sentinel leaks, BAD", num_ibt_leaks_likely_sentinel)
+    STATS_DEF("Extra exits due to selfmod exits", num_ibt_exit_src_sandboxed)
+    STATS_DEF("Extra exits due to shared table resizing",
+              num_ibt_exit_shared_table_resize)
+    STATS_DEF("Extra exits due to src unlinked (frag deleted)",
+              num_ibt_exit_src_unlinked_frag_deleted)
+    STATS_DEF("Extra exits due to src unlinked (shadowed)",
+              num_ibt_exit_src_unlinked_shadowed)
+    STATS_DEF("Extra exits due to shared syscall targetting BB",
+              num_ibt_exit_src_trace_shared_syscall)
+    STATS_DEF("Extra IBT exits due to -no_link_ibl", num_ibt_exit_nolink)
+    STATS_DEF("Extra IBT exits due to unknown reasons", num_ibt_exit_unknown)
+    STATS_DEF("Fragments regenerated, in-cache replacement", num_fragments_regenerated)
+    STATS_DEF("Fragments regenerated or duplicated", num_fragments_deja_vu)
+    STATS_DEF("Trace fragments extended", num_traces_extended)
+    STATS_DEF("Trace building private copies created", num_trace_private_copies)
+    STATS_DEF("Trace building private copies deleted", num_trace_private_deletions)
+    STATS_DEF("Trace building private copies flushed b4 deleted", num_trace_private_deletions_flushed)
+    STATS_DEF("Trace building private copies aborting traces", num_trace_private_copy_abort)
+    STATS_DEF("Trace building private copies futures deleted", num_trace_private_fut_del)
+    STATS_DEF("Trace building private copies futures avoided", num_trace_private_fut_avoid)
+    STATS_DEF("Trace inline-ib comparisons", trace_ib_cmp)
+#ifdef X64
+    STATS_DEF("Trace inline-ib no eflag restore needed", trace_ib_no_flag_restore)
+#endif
+    STATS_DEF("Trace fragments extended, ibl exits updated", num_traces_ibl_extended)
+#ifdef WINDOWS
+    STATS_DEF("Trace fragments extended w/shared syscall block",
+              num_traces_shared_syscall_extended)
+#endif
+#ifdef CUSTOM_TRACES
+    STATS_DEF("Custom traces extended beyond normal stop", custom_traces_stop_late)
+    STATS_DEF("Custom traces stopped early", custom_traces_stop_early)
+    STATS_DEF("Shadowed bbs built for custom traces", custom_traces_bbs_built)
+#endif
+    STATS_DEF("Recreated fragments, total", num_recreated_fragments)
+    STATS_DEF("Recreated fragments, traces", num_recreated_traces)
+    STATS_DEF("Recreations via app re-decode", recreate_via_app_ilist)
+    STATS_DEF("Recreations via stored info", recreate_via_stored_info)
+    STATS_DEF("Recreation spill value restores", recreate_spill_restores)
+    STATS_DEF("IBL stubs updated on table resize", num_ibl_stub_resize_updates)
+
+    STATS_DEF("Patched fragments", emit_patched_fragments)
+    STATS_DEF("Patched relocation slots", emit_patched_relocations)
+    STATS_DEF("Branches linked, direct", num_direct_links)
+    STATS_DEF("Branches linked, indirect", num_indirect_links)
+    STATS_DEF("Branches already linked, indirect", num_indirect_already_linked)
+
+    RSTATS_DEF("Coarse-grain units", num_coarse_units)
+    RSTATS_DEF("Peak coarse-grain units", peak_num_coarse_units)
+    STATS_DEF("Coarse-grain fragments generated", num_coarse_fragments)
+    STATS_DEF("Coarse-grain fragments in secondary units", num_coarse_secondary)
+    STATS_DEF("Fine-grain fragments in coarse-grain regions", num_fine_in_coarse)
+    STATS_DEF("Coarse prevented by overlap w/ fine region", coarse_overlap_with_fine)
+    STATS_DEF("Coarse prevented by non-exit cti", coarse_prevent_cti)
+    STATS_DEF("Coarse prevented by privateness", coarse_prevent_private)
+    STATS_DEF("Coarse prevented by ind call conversion", coarse_prevent_indcall)
+    STATS_DEF("Coarse prevented by system call", coarse_prevent_syscall)
+    STATS_DEF("Coarse prevented by must-end-trace", coarse_prevent_end_trace)
+    STATS_DEF("Coarse prevented by cannot-be-trace", coarse_prevent_no_trace)
+    STATS_DEF("Coarse prevented by selfmod", coarse_prevent_selfmod)
+    STATS_DEF("Coarse prevented by translation store", coarse_prevent_translation)
+    STATS_DEF("Coarse prevented by cb return", coarse_prevent_cbret)
+    STATS_DEF("Coarse prevented by native_exec", coarse_prevent_native_exec)
+    STATS_DEF("Coarse prevented by client", coarse_prevent_client)
+    STATS_DEF("Lazy links from coarse-grain", lazy_links_from_coarse)
+    STATS_DEF("Lazy links from persisted units", lazy_links_from_persisted)
+    STATS_DEF("Lazy links from fine-grain", lazy_links_from_fine)
+    STATS_DEF("Lazy links that failed to link", lazy_links_failed)
+    STATS_DEF("Coarse-grain direct re-links", coarse_relinks)
+    STATS_DEF("Coarse-grain trace head path-dependent", coarse_th_path_dependent)
+    STATS_DEF("Coarse-grain trace heads from fine", coarse_th_from_fine)
+    STATS_DEF("Coarse grain freezes", coarse_freezes)
+    STATS_DEF("Coarse grain freezes aborted", coarse_freeze_abort)
+    STATS_DEF("Coarse grain at-unload not persisted: synch failure",
+              persist_unload_suspend_failure)
+    STATS_DEF("Coarse grain units frozen", coarse_units_frozen)
+    STATS_DEF("Coarse grain fall-throughs during freezing", coarse_freeze_fallthrough)
+    STATS_DEF("Coarse grain trace heads found on shift", coarse_th_on_shift)
+    STATS_DEF("Coarse grain units persisted: attempts", coarse_units_persist_try)
+    STATS_DEF("Coarse grain units persist-at-unload attempts", persist_unload_try)
+    STATS_DEF("Coarse grain units persisted: success", coarse_units_persist)
+    STATS_DEF("Coarse grain fragments frozen", coarse_fragments_frozen)
+    STATS_DEF("Coarse grain code persisted (bytes)", coarse_code_persisted)
+    STATS_DEF("Coarse grain units not persisted: too small", persist_too_small)
+    STATS_DEF("Coarse grain code small, persisted anyway", persist_code_small)
+    STATS_DEF("Coarse grain units not persisted: no space", coarse_units_persist_nospace)
+    STATS_DEF("Coarse grain units not persisted: duplicate", coarse_units_persist_dup)
+    STATS_DEF("Coarse grain units not persisted: file/name error",
+              coarse_units_persist_error)
+    STATS_DEF("Coarse grain units not persisted: excluded",
+              coarse_units_persist_excluded)
+    STATS_DEF("Persisted cache files renamed", persist_rename_success)
+    STATS_DEF("Coarse grain units not persisted: rename tmp fail",
+              persist_rename_tmp_fail)
+    STATS_DEF("Coarse grain units not persisted: rename fail", persist_rename_fail)
+    STATS_DEF("Coarse grain units not persisted: rename race", persist_rename_race)
+    STATS_DEF("Persisted cache files renamed & deleted", persist_delete_success)
+    STATS_DEF("Persisted cache tmp file delete error on failure",
+              persist_delete_tmp_fail)
+    STATS_DEF("Coarse grain units merged", coarse_units_merged)
+    STATS_DEF("Coarse grain merges with on-disk files", coarse_merge_disk)
+    STATS_DEF("Coarse grain region mismatches w/ on-disk files",
+              coarse_merge_disk_mismatch)
+    STATS_DEF("Coarse grain failed merges w/ on-disk files",
+              coarse_merge_disk_fail)
+    STATS_DEF("Coarse grain dups when merging", coarse_merge_dups)
+    STATS_DEF("RAC entries out of coarse range when persisting",
+              rac_module_persist_out_of_range)
+    STATS_DEF("RCT entries out of coarse range when persisting",
+              rct_module_persist_out_of_range)
+    STATS_DEF("RCT entries that outlasted persisted cache",
+              rct_persisted_outlast_cache)
+    STATS_DEF("Coarse units marked invalid", coarse_marked_invalid)
+    STATS_DEF("Coarse units marked valid", coarse_marked_valid)
+    STATS_DEF("Coarse units executed while invalid", coarse_executed_invalid)
+    STATS_DEF("Coarse region merged with IAT post-rebind", coarse_merge_IAT)
+
+    STATS_DEF("Hotpatch match requiring persisted cache flush", hotp_persist_flush)
+
+    STATS_DEF("Persisted cache exec loads attempted", perscache_load_attempt)
+    STATS_DEF("Persisted cache post-rebind re-loads attempted", perscache_rebind_load)
+    STATS_DEF("Persisted cache non-exec loads attempted", perscache_load_nox_attempt)
+    RSTATS_DEF("Persisted caches successfully loaded", perscache_loaded)
+    STATS_DEF("Persisted cache load error: name error/excluded",
+              perscache_load_noname)
+    STATS_DEF("Persisted cache load error: excluded", perscache_load_excluded)
+    STATS_DEF("Persisted cache load error: file not found", perscache_load_nofile)
+    STATS_DEF("Persisted cache load error: bad file", perscache_bad_file)
+    STATS_DEF("Persisted cache load error: version mismatch", perscache_version_mismatch)
+    STATS_DEF("Persisted cache load error: md5 mismatch", perscache_md5_mismatch)
+    STATS_DEF("Persisted cache load error: modinfo mismatch", perscache_modinfo_mismatch)
+    STATS_DEF("Persisted cache load error: modbase mismatch", perscache_base_mismatch)
+    STATS_DEF("Persisted cache load error: region mismatch", perscache_region_mismatch)
+    STATS_DEF("Persisted cache load error: tls offs mismatch", perscache_tls_mismatch)
+    STATS_DEF("Persisted cache load error: no trace support", perscache_trace_mismatch)
+    STATS_DEF("Persisted cache load error: no RAC/RCT support", perscache_rct_mismatch)
+    STATS_DEF("Persisted cache load error: option mismatch", perscache_options_mismatch)
+    STATS_DEF("Persisted cache load error: maps not adjacent",
+              perscache_maps_not_adjacent)
+#ifdef HOT_PATCHING_INTERFACE
+    STATS_DEF("Persisted cache load error: hotp conflict", perscache_hotp_conflict)
+    STATS_DEF("Persisted cache hotp conflict avoided", perscache_hotp_conflict_avoided)
+    STATS_DEF("Persisted cache hotp nudge flush avoided", perscache_hotp_flush_avoided)
+#endif
+    STATS_DEF("Persisted cache ibl entries prefilled", perscache_ibl_prefill)
+    STATS_DEF("Persisted cache stub unprot for link", pcache_unprot_link)
+    STATS_DEF("Persisted cache stub unprot for unlink", pcache_unprot_unlink)
+    STATS_DEF("Persisted cache stub writes over limit", pcache_unprot_over_limit)
+    STATS_DEF("Persisted cache stub pages touched", pcache_stub_touched)
+
+#ifdef SIDELINE
+    STATS_DEF("Waits due to sideline", num_wait_sideline)
+#endif
+    STATS_DEF("Waits due to flushing", num_wait_flush)
+    STATS_DEF("Waits due to shared cache barrier", num_wait_shared_barrier)
+
+    STATS_DEF("Entrance hooks to DR", num_entering_DR)
+    STATS_DEF("Exit hooks from DR", num_exiting_DR)
+    STATS_DEF("Fcache exits, total", num_exits)
+    STATS_DEF("Fcache exits, coarse-grain fragments", num_exits_coarse)
+    STATS_DEF("Fcache exits, coarse-grain targeting trace head", num_exits_coarse_trace_head)
+    STATS_DEF("Fcache exits, fine targeting th coarse", num_exits_fine2th_coarse)
+    STATS_DEF("Fcache exits, fine targeting non-th coarse", num_exits_fine2non_th_coarse)
+    STATS_DEF("Fcache exits, system call executions", num_exits_syscalls)
+    STATS_DEF("Fcache exits, flushed due to code mod", num_exits_code_mod_flush)
+    STATS_DEF("Fcache exits, deleted but hit in ibl", num_exits_ibl_deleted)
+#ifdef LINUX
+    STATS_DEF("Fcache exits, sigreturn", num_exits_sigreturn)
+#else /* WINDOWS */
+    STATS_DEF("Fcache exits, asynch", num_exits_asynch)
+#endif
+    STATS_DEF("Fcache exits, native_exec executions", num_exits_native_exec)
+    STATS_DEF("Fcache exits, native_exec syscalls", num_exits_native_exec_syscall)
+    STATS_DEF("Fcache exits, proactive reset", num_exits_reset)
+#ifdef HOT_PATCHING_INTERFACE
+    STATS_DEF("Fcache exits, hot patching control flow change", num_exits_hot_patch)
+#endif
+#ifdef CLIENT_INTERFACE
+    STATS_DEF("Fcache exits, client redirecting", num_exits_client_redirect)
+#endif
+    STATS_DEF("Fcache entrances aborted b/c target missing", num_entrances_aborted)
+    STATS_DEF("Fcache exits, from traces", num_trace_exits)
+    STATS_DEF("Fcache exits, from BBs", num_bb_exits)
+    STATS_DEF("Fcache exits, total indirect branches", num_exits_ind_total)
+    STATS_DEF("Fcache exits, non-trace indirect branches", num_exits_ind_non_trace)
+    STATS_DEF("Fcache exits, ind target not in cache", num_exits_ind_good_miss)
+    STATS_DEF("Fcache exits, ind target extending a trace, BAD", num_exits_ind_trace_build)
+    STATS_DEF("Fcache exits, ind target in cache but not table", num_exits_ind_bad_miss)
+    STATS_DEF("Fcache exits, src fragment unlinked", num_exits_ind_src_unlinked)
+
+    STATS_DEF("Fcache exits, from BB, ind target ...", num_exits_ind_bad_miss_bb)
+    STATS_DEF("Fcache exits, BB->BB, ind target ...", num_exits_ind_bad_miss_bb2bb)
+    STATS_DEF("Fcache exits, BB->BB trace head, ind target ...", num_exits_ind_bad_miss_bb2bb_th)
+    STATS_DEF("Fcache exits, BB->trace, ind target ...", num_exits_ind_bad_miss_bb2trace)
+
+    STATS_DEF("Fcache exits, from trace, ind target ...", num_exits_ind_bad_miss_trace)
+    STATS_DEF("Fcache exits, trace->trace, ind target ...", num_exits_ind_bad_miss_trace2trace)
+    STATS_DEF("Fcache exits, trace->BB not trace head, ind target", num_exits_ind_bad_miss_trace2bb_nth)
+    STATS_DEF("Fcache exits, trace->BB trace head, ind target", num_exits_ind_bad_miss_trace2bb_th)
+
+#ifdef RETURN_AFTER_CALL
+    /* These statistics available only under RAC, should make default  */
+    STATS_DEF("Fcache exits, indirect calls", num_exits_ind_call)
+    STATS_DEF("Fcache exits, indirect jmps", num_exits_ind_jmp)
+    STATS_DEF("Fcache exits, returns", num_exits_ret)
+    STATS_DEF("Fcache exits, returns from traces", num_exits_ret_trace)
+#endif /* RETURN_AFTER_CALL */
+    STATS_DEF("Fcache exits, unknown indirect, BAD", num_exits_ind_unknown)
+    STATS_DEF("Fcache exits, dir target not in cache", num_exits_dir_miss)
+    STATS_DEF("Fcache exits, link not allowed", num_exits_dir_nolink)
+#ifndef TRACE_HEAD_CACHE_INCR
+    STATS_DEF("Fcache exits, target trace head", num_exits_dir_trace_head)
+#endif
+#ifdef TRACE_HEAD_CACHE_INCR
+    STATS_DEF("Fcache exits, target trace head now hot", num_exits_dir_trace_hot)
+#endif
+    STATS_DEF("Fcache exits, extending a trace", num_exits_dir_trace_build)
+    STATS_DEF("Fcache exits, non-ignorable system call", num_exits_dir_syscall)
+#ifdef WINDOWS
+    STATS_DEF("Fcache exits, callback return", num_exits_dir_cbret)
+    STATS_DEF("Fcache exits, from unlinked shared syscall",
+              num_unlinked_shared_syscalls_exits)
+    STATS_DEF("Fcache exits, from BB->shared syscall",
+              num_shared_syscalls_bb_exits)
+    STATS_DEF("Fcache exits, from trace->shared syscall",
+              num_shared_syscalls_trace_exits)
+#endif
+    STATS_DEF("Fcache exits, no link shared <-> private", num_exits_dir_nolink_sharing)
+    STATS_DEF("Fcache exits, presumed link race condition", num_exits_dir_race)
+    STATS_DEF("Fcache exits, self-replacement", num_exits_dir_self_replacement)
+#ifdef LINUX
+    STATS_DEF("Fcache exits, signal delivery", num_exits_dir_signal)
+#endif
+    STATS_DEF("Fcache exits needing cbr disambiguation", cbr_disambiguations)
+
+    STATS_DEF("Fragments with OF restore prefix", num_oflag_prefix_restore)
+    STATS_DEF("Fcache free capacity (bytes)", fcache_free_capacity)
+
+    STATS_DEF("Fcache trace capacity (bytes)", fcache_trace_capacity)
+    STATS_DEF("Fcache trace peak capacity (bytes)", fcache_trace_capacity_peak)
+    STATS_DEF("Fcache trace space claimed (bytes)", fcache_trace_claimed)
+    STATS_DEF("Fcache trace space used (bytes)", fcache_trace_used)
+    STATS_DEF("Fcache trace peak used (bytes)", fcache_trace_peak)
+    STATS_DEF("Fcache trace headers (bytes)", fcache_trace_headers)
+    STATS_DEF("Fcache trace fragment bodies (bytes)", fcache_trace_bodies)
+    STATS_DEF("Fcache trace direct exit stubs (bytes)", fcache_trace_direct_stubs)
+    STATS_DEF("Fcache trace indirect exit stubs (bytes)", fcache_trace_indirect_stubs)
+    STATS_DEF("Fcache trace fragment prefixes (bytes)", fcache_trace_prefixes)
+    STATS_DEF("Fcache trace align space (bytes)", fcache_trace_align)
+    STATS_DEF("Fcache trace empty space (bytes)", fcache_trace_empty)
+    STATS_DEF("Fcache trace selfmod copy (bytes)", fcache_trace_selfmod_copy)
+    STATS_DEF("Fcache trace free coalesce prev", fcache_trace_free_coalesce_prev)
+    STATS_DEF("Fcache trace free coalesce next", fcache_trace_free_coalesce_next)
+    STATS_DEF("Fcache trace free coalesce too big", fcache_trace_free_coalesce_too_big)
+    STATS_DEF("Fcache trace return last", fcache_trace_return_last)
+    STATS_DEF("Fcache trace free use larger bucket", fcache_trace_free_use_larger)
+    STATS_DEF("Fcache trace free split", fcache_trace_free_split)
+
+    STATS_DEF("Fcache bb capacity (bytes)", fcache_bb_capacity)
+    STATS_DEF("Fcache bb peak capacity (bytes)", fcache_bb_capacity_peak)
+    STATS_DEF("Fcache bb space claimed (bytes)", fcache_bb_claimed)
+    STATS_DEF("Fcache bb space used (bytes)", fcache_bb_used)
+    STATS_DEF("Fcache bb peak used (bytes)", fcache_bb_peak)
+    STATS_DEF("Fcache bb headers (bytes)", fcache_bb_headers)
+    STATS_DEF("Fcache bb fragment bodies (bytes)", fcache_bb_bodies)
+    STATS_DEF("Fcache bb direct exit stubs (bytes)", fcache_bb_direct_stubs)
+    STATS_DEF("Fcache bb indirect exit stubs (bytes)", fcache_bb_indirect_stubs)
+    STATS_DEF("Fcache bb fragment prefixes (bytes)", fcache_bb_prefixes)
+    STATS_DEF("Fcache bb align space (bytes)", fcache_bb_align)
+    STATS_DEF("Fcache bb empty space (bytes)", fcache_bb_empty)
+    STATS_DEF("Fcache bb selfmod copy (bytes)", fcache_bb_selfmod_copy)
+    STATS_DEF("Fcache bb free coalesce prev", fcache_bb_free_coalesce_prev)
+    STATS_DEF("Fcache bb free coalesce next", fcache_bb_free_coalesce_next)
+    STATS_DEF("Fcache bb free coalesce too big", fcache_bb_free_coalesce_too_big)
+    STATS_DEF("Fcache bb return last", fcache_bb_return_last)
+    STATS_DEF("Fcache bb free use larger bucket", fcache_bb_free_use_larger)
+    STATS_DEF("Fcache bb free split", fcache_bb_free_split)
+
+    STATS_DEF("Fcache shared bb capacity (bytes)", fcache_shared_bb_capacity)
+    STATS_DEF("Fcache shared bb peak capacity (bytes)", fcache_shared_bb_capacity_peak)
+    STATS_DEF("Fcache shared bb space claimed (bytes)", fcache_shared_bb_claimed)
+    STATS_DEF("Fcache shared bb space used (bytes)", fcache_shared_bb_used)
+    STATS_DEF("Fcache shared bb peak used (bytes)", fcache_shared_bb_peak)
+    STATS_DEF("Fcache shared bb headers (bytes)", fcache_shared_bb_headers)
+    STATS_DEF("Fcache shared bb fragment bodies (bytes)", fcache_shared_bb_bodies)
+    STATS_DEF("Fcache shared bb direct exit stubs (bytes)", fcache_shared_bb_direct_stubs)
+    STATS_DEF("Fcache shared bb indirect exit stubs (bytes)", fcache_shared_bb_indirect_stubs)
+    STATS_DEF("Fcache shared bb fragment prefixes (bytes)", fcache_shared_bb_prefixes)
+    STATS_DEF("Fcache shared bb align space (bytes)", fcache_shared_bb_align)
+    STATS_DEF("Fcache shared bb empty space (bytes)", fcache_shared_bb_empty)
+    STATS_DEF("Fcache shared bb selfmod copy (bytes)", fcache_shared_bb_selfmod_copy)
+    STATS_DEF("Fcache shared bb free coalesce prev", fcache_shared_bb_free_coalesce_prev)
+    STATS_DEF("Fcache shared bb free coalesce next", fcache_shared_bb_free_coalesce_next)
+    STATS_DEF("Fcache shared bb free coalesce too big", fcache_shared_bb_free_coalesce_too_big)
+    STATS_DEF("Fcache shared bb return last", fcache_shared_bb_return_last)
+    STATS_DEF("Fcache shared bb free use larger bucket", fcache_shared_bb_free_use_larger)
+    STATS_DEF("Fcache shared bb free split", fcache_shared_bb_free_split)
+
+    STATS_DEF("Fcache shared trace capacity (bytes)", fcache_shared_trace_capacity)
+    STATS_DEF("Fcache shared trace peak capacity (bytes)", fcache_shared_trace_capacity_peak)
+    STATS_DEF("Fcache shared trace space claimed (bytes)", fcache_shared_trace_claimed)
+    STATS_DEF("Fcache shared trace space used (bytes)", fcache_shared_trace_used)
+    STATS_DEF("Fcache shared trace peak used (bytes)", fcache_shared_trace_peak)
+    STATS_DEF("Fcache shared trace headers (bytes)", fcache_shared_trace_headers)
+    STATS_DEF("Fcache shared trace fragment bodies (bytes)", fcache_shared_trace_bodies)
+    STATS_DEF("Fcache shared trace direct exit stubs (bytes)", fcache_shared_trace_direct_stubs)
+    STATS_DEF("Fcache shared trace indirect exit stubs (bytes)", fcache_shared_trace_indirect_stubs)
+    STATS_DEF("Fcache shared trace fragment prefixes (bytes)", fcache_shared_trace_prefixes)
+    STATS_DEF("Fcache shared trace align space (bytes)", fcache_shared_trace_align)
+    STATS_DEF("Fcache shared trace empty space (bytes)", fcache_shared_trace_empty)
+    STATS_DEF("Fcache shared trace selfmod copy (bytes)", fcache_shared_trace_selfmod_copy)
+    STATS_DEF("Fcache shared trace free coalesce prev", fcache_shared_trace_free_coalesce_prev)
+    STATS_DEF("Fcache shared trace free coalesce next", fcache_shared_trace_free_coalesce_next)
+    STATS_DEF("Fcache shared trace free coalesce too big", fcache_shared_trace_free_coalesce_too_big)
+    STATS_DEF("Fcache shared trace return last", fcache_shared_trace_return_last)
+    STATS_DEF("Fcache shared trace free use larger bucket", fcache_shared_trace_free_use_larger)
+    STATS_DEF("Fcache shared trace free split", fcache_shared_trace_free_split)
+
+    STATS_DEF("Fcache coarse bb capacity (bytes)", fcache_coarse_bb_capacity)
+    STATS_DEF("Fcache coarse bb peak capacity (bytes)", fcache_coarse_bb_capacity_peak)
+    STATS_DEF("Fcache coarse bb space claimed (bytes)", fcache_coarse_bb_claimed)
+    STATS_DEF("Fcache coarse bb space used (bytes)", fcache_coarse_bb_used)
+    STATS_DEF("Fcache coarse bb peak used (bytes)", fcache_coarse_bb_peak)
+    STATS_DEF("Fcache coarse bb headers (bytes)", fcache_coarse_bb_headers)
+    STATS_DEF("Fcache coarse bb fragment bodies (bytes)", fcache_coarse_bb_bodies)
+    STATS_DEF("Fcache coarse bb direct exit stubs (bytes)", fcache_coarse_bb_direct_stubs)
+    STATS_DEF("Fcache coarse bb indirect exit stubs (bytes)", fcache_coarse_bb_indirect_stubs)
+    STATS_DEF("Fcache coarse bb fragment prefixes (bytes)", fcache_coarse_bb_prefixes)
+    STATS_DEF("Fcache coarse bb align space (bytes)", fcache_coarse_bb_align)
+    STATS_DEF("Fcache coarse bb empty space (bytes)", fcache_coarse_bb_empty)
+    STATS_DEF("Fcache coarse bb selfmod copy (bytes)", fcache_coarse_bb_selfmod_copy)
+    STATS_DEF("Fcache coarse bb free coalesce prev", fcache_coarse_bb_free_coalesce_prev)
+    STATS_DEF("Fcache coarse bb free coalesce next", fcache_coarse_bb_free_coalesce_next)
+    STATS_DEF("Fcache coarse bb free coalesce too big", fcache_coarse_bb_free_coalesce_too_big)
+    STATS_DEF("Fcache coarse bb return last", fcache_coarse_bb_return_last)
+    STATS_DEF("Fcache coarse bb free use larger bucket", fcache_coarse_bb_free_use_larger)
+    STATS_DEF("Fcache coarse bb free split", fcache_coarse_bb_free_split)
+
+    STATS_DEF("Fcache combined claimed (bytes)", fcache_combined_claimed)
+    STATS_DEF("Fcache combined capacity (bytes)", fcache_combined_capacity)
+    STATS_DEF("Peak fcache combined capacity (bytes)", peak_fcache_combined_capacity)
+    RSTATS_DEF("Fcache units on live list", fcache_num_live)
+    RSTATS_DEF("Peak fcache units on live list", peak_fcache_num_live)
+    RSTATS_DEF("Fcache units on free list", fcache_num_free)
+    RSTATS_DEF("Peak fcache units on free list", peak_fcache_num_free)
+    STATS_DEF("Fcache unit lookups", fcache_unit_lookups)
+
+    STATS_DEF("Separate shared trace direct exit stubs (bytes)",
+              separate_shared_trace_direct_stubs)
+    STATS_DEF("Separate shared bb direct exit stubs (bytes)",
+              separate_shared_bb_direct_stubs)
+    STATS_DEF("Separate shared bb direct entrance stubs (bytes)",
+              separate_shared_bb_entrance_stubs)
+    STATS_DEF("Separate trace direct exit stubs (bytes)", separate_trace_direct_stubs)
+    STATS_DEF("Separate bb direct exit stubs (bytes)", separate_bb_direct_stubs)
+    STATS_DEF("Special heap units", heap_special_units)
+    STATS_DEF("Peak special heap units", peak_heap_special_units)
+    STATS_DEF("Special heap align space (bytes)", heap_special_align)
+    STATS_DEF("Special heap capacity (bytes)", heap_special_capacity)
+    STATS_DEF("Peak special heap capacity (bytes)", peak_heap_special_capacity)
+
+    RSTATS_DEF("Heap units on live list", heap_num_live)
+    RSTATS_DEF("Peak heap units on live list", peak_heap_num_live)
+    RSTATS_DEF("Heap units on free list", heap_num_free)
+    RSTATS_DEF("Peak heap units on free list", peak_heap_num_free)
+    STATS_DEF("Heap headers (bytes)", heap_headers)
+    STATS_DEF("Heap align space (bytes)", heap_align)
+    STATS_DEF("Peak heap align space (bytes)", peak_heap_align)
+    STATS_DEF("Heap bucket pad space (bytes)", heap_bucket_pad)
+    STATS_DEF("Peak heap bucket pad space (bytes)", peak_heap_bucket_pad)
+    STATS_DEF("Heap allocs in buckets", heap_allocs_buckets)
+    STATS_DEF("Heap allocs variable-sized", heap_allocs_variable)
+    STATS_DEF("Total reserved memory", reserved_memory_capacity)
+    STATS_DEF("Peak total reserved memory", peak_reserved_memory_capacity)
+    STATS_DEF("Guard pages, reserved virtual pages", guard_pages)
+    STATS_DEF("Peak guard pages, reserved virtual pages", peak_guard_pages)
+
+    STATS_DEF("Stack capacity (bytes)", stack_capacity)
+    STATS_DEF("Peak stack capacity (bytes)", peak_stack_capacity)
+    STATS_DEF("Mmaps sharing stack alloc region", mmap_share_stack_region)
+    STATS_DEF("Mmaps unable to share stack alloc region", mmap_no_share_stack_region)
+    STATS_DEF("Mmap capacity (bytes)", mmap_capacity)
+    STATS_DEF("Peak mmap capacity (bytes)", peak_mmap_capacity)
+    STATS_DEF("Mmap reserved but not committed (bytes)", mmap_reserved_only)
+    STATS_DEF("Peak mmap reserved but not committed (bytes)", peak_mmap_reserved_only)
+    STATS_DEF("Heap claimed (bytes)", heap_claimed)
+    STATS_DEF("Peak heap claimed (bytes)", peak_heap_claimed)
+    STATS_DEF("Heap capacity (bytes)", heap_capacity)
+    STATS_DEF("Peak heap capacity (bytes)", peak_heap_capacity)
+    STATS_DEF("Heap reserved but not committed (bytes)", heap_reserved_only)
+    STATS_DEF("Peak heap reserved but not committed (bytes)", peak_heap_reserved_only)
+    STATS_DEF("File map capacity (bytes)", file_map_capacity)
+    STATS_DEF("Peak file map capacity (bytes)", peak_file_map_capacity)
+    STATS_DEF("Total memory from OS", memory_capacity)
+    STATS_DEF("Peak total memory from OS", peak_memory_capacity)
+    STATS_DEF("Our library space (bytes)", dr_library_space)
+
+#ifdef WINDOWS
+    /* reserved means reserved but not commited */
+    STATS_DEF("Application reserved-only capacity (bytes)", app_reserved_capacity)
+    STATS_DEF("Peak application reserved-only capacity (bytes)", peak_app_reserved_capacity)
+    STATS_DEF("Application committed capacity (bytes)", app_committed_capacity)
+    STATS_DEF("Peak application committed capacity (bytes)", peak_app_committed_capacity)
+#endif
+    STATS_DEF("Application stack capacity (bytes)", app_stack_capacity)
+    STATS_DEF("Peak application stack capacity (bytes)", peak_app_stack_capacity)
+    STATS_DEF("Application heap capacity (bytes)", app_heap_capacity)
+    STATS_DEF("Peak application heap capacity (bytes)", peak_app_heap_capacity)
+#ifdef WINDOWS
+    STATS_DEF("Application image capacity (bytes)", app_image_capacity)
+    STATS_DEF("Peak application image capacity (bytes)", peak_app_image_capacity)
+    /* for win32, image is separated from rest of mmap */
+#endif
+    STATS_DEF("Application mmap capacity (bytes)", app_mmap_capacity)
+    STATS_DEF("Peak application mmap capacity (bytes)", peak_app_mmap_capacity)
+    STATS_DEF("Application executable capacity (bytes)", app_exec_capacity)
+    STATS_DEF("Peak application executable capacity (bytes)", peak_app_exec_capacity)
+    STATS_DEF("Application read-only capacity (bytes)", app_ro_capacity)
+    STATS_DEF("Peak application read-only capacity (bytes)", peak_app_ro_capacity)
+    STATS_DEF("Application writable capacity (bytes)", app_rw_capacity)
+    STATS_DEF("Peak application writable capacity (bytes)", peak_app_rw_capacity)
+    STATS_DEF("Total (app + us) virtual size (bytes)", total_vsize)
+    STATS_DEF("Peak total (app + us) virtual size (bytes)", peak_total_vsize)
+    STATS_DEF("Application virtual size (bytes)", app_vsize)
+    STATS_DEF("Peak application virtual size (bytes)", peak_app_vsize)
+    STATS_DEF("Our additional virtual size (bytes)", dr_vsize)
+    STATS_DEF("Peak our additional virtual size (bytes)", peak_dr_vsize)
+#ifdef WINDOWS
+    STATS_DEF("Our commited capacity (bytes)", dr_commited_capacity)
+    STATS_DEF("Our peak commited capacity (bytes)", peak_dr_commited_capacity)
+    STATS_DEF("Our reserved capacity (bytes)", dr_reserved_capacity)
+    STATS_DEF("Our peak reserved capacity (bytes)", peak_dr_reserved_capacity)
+    STATS_DEF("App unallocatable free space", app_wasted_vsize)
+    STATS_DEF("Peak app unallocatable free space", peak_app_wasted_vsize)
+    STATS_DEF("Our unallocatable free space", dr_wasted_vsize)
+    STATS_DEF("Our peak unallocatable free space", peak_dr_wasted_vsize)
+    STATS_DEF("Total unallocatable free space", total_wasted_vsize)
+    STATS_DEF("Peak total unallocatable free space", peak_total_wasted_vsize)
+    STATS_DEF("Number of unaligned allocations (TEB's etc.)", unaligned_allocations)
+    STATS_DEF("Peak unaligned allocations", peak_unaligned_allocations)
+#endif
+    STATS_DEF("Our virtual memory blocks in use", vmm_vsize_blocks_used)
+    STATS_DEF("Our peak virtual memory blocks in use", peak_vmm_vsize_blocks_used)
+    STATS_DEF("Our virtual memory in use (bytes)", vmm_vsize_used)
+    STATS_DEF("Our peak virtual memory in use (bytes)", peak_vmm_vsize_used)
+    STATS_DEF("Number of landing pad areas allocated", num_landing_pad_areas)
+    STATS_DEF("Total times mutexes acquired", total_acquired)
+    STATS_DEF("Total times mutexes contended", total_contended)
+#ifdef WINDOWS
+    STATS_DEF("Unable to wait_at_safe_spot cache entries", no_wait_entries)
+#endif
+    STATS_DEF("Interoperability fixups for apc_policy", num_used_apc_policy)
+    STATS_DEF("Interoperability fixups for thread_policy", num_used_thread_policy)
+    STATS_DEF("Number of safe reads", num_safe_reads)
+    STATS_DEF("Number of safe writes", num_safe_writes)
+    STATS_DEF("Number of vmarea vector resize reallocations", num_vmareas_resized)
+    STATS_DEF("Number of vmarea vector resize synch fixups", num_vmareas_resize_synch)
+    STATS_DEF("Peak vmarea vector length", max_vmareas_length)
+    STATS_DEF("Peak dynamo areas vector length", max_DRareas_length)
+    STATS_DEF("Peak executable areas vector length", max_execareas_length)
+    STATS_DEF("Peak module areas vector length", max_modareas_length)
+
+     /* probably more -pad_jmps stats then needed, remove some of the less important ones
+      * once we've better characterized the behavior */
+    STATS_DEF("-pad_jmps fragments size overestimated", pad_jmps_fragments_overestimated)
+    STATS_DEF("-pad_jmps excess instances coalesced with next free", pad_jmps_excess_next_free)
+    STATS_DEF("-pad_jmps excess instances failed to be returned", pad_jmps_excess_wasted)
+    STATS_DEF("-pad_jmps excess bytes failed to be returned", pad_jmps_excess_wasted_bytes)
+    STATS_DEF("-pad_jmps body bytes shared bb", pad_jmps_shared_bb_body_bytes)
+    STATS_DEF("-pad_jmps stub bytes shared bb", pad_jmps_shared_bb_stub_bytes)
+    STATS_DEF("Emitted unaligned patch exits shared bb", pad_jmps_shared_bb_unaligned_exits)
+    STATS_DEF("Emitted unaligned exits bytes shared bb", pad_jmps_shared_bb_unaligned_exit_bytes)
+    STATS_DEF("Emitted unaligned stubs shared bb", pad_jmps_shared_bb_unaligned_stubs)
+    STATS_DEF("Emitted unaligned stubs bytes shared bb", pad_jmps_shared_bb_unaligned_stubs_bytes)
+    STATS_DEF("-pad_jmps excess bytes shared bb", pad_jmps_shared_bb_excess_bytes)
+    STATS_DEF("Bytes shared frags ever", pad_jmps_shared_bb_sum_fragment_bytes_ever)
+    STATS_DEF("-pad_jmps start_pcs shifted shared bb", pad_jmps_shared_bb_num_start_pc_shifted)
+    STATS_DEF("-pad_jmps start_pcs shifted bytes shared bb", pad_jmps_shared_bb_start_pc_shifted_bytes)
+    STATS_DEF("-pad_jmps excess bytes released shared bb", pad_jmps_shared_bb_extra_space_released)
+    STATS_DEF("-pad_jmps shifted stubs shared bb", pad_jmps_shared_bb_num_shifted_stubs)
+    STATS_DEF("-pad_jmps shifted stub bytes shared bb", pad_jmps_shared_bb_shifted_stub_bytes)
+    STATS_DEF("-pad_jmps no shift stubs shared bb", pad_jmps_shared_bb_num_stubs_no_shift)
+    STATS_DEF("-pad_jmps inserted nops shared bb", pad_jmps_shared_bb_num_nops)
+    STATS_DEF("-pad_jmps inserted nop bytes shared bb", pad_jmps_shared_bb_nop_bytes)
+    STATS_DEF("-pad_jmps no pad exits shared bb", pad_jmps_shared_bb_num_no_pad_exits)
+    STATS_DEF("-pad_jmps_mark_no_trace untraceable bbs", pad_jmps_mark_no_trace)
+    STATS_DEF("-pad_jmps_mark_no_trace block trace on reemit", pad_jmps_block_trace_reemit)
+
+    /* shtrace to avoid going over 60 columns */
+    STATS_DEF("-pad_jmps body bytes shtrace", pad_jmps_shared_trace_body_bytes)
+    STATS_DEF("-pad_jmps stub bytes shtrace", pad_jmps_shared_trace_stub_bytes)
+    STATS_DEF("Emitted unaligned patch exits shtrace", pad_jmps_shared_trace_unaligned_exits)
+    STATS_DEF("Emitted unaligned exits bytes shtrace", pad_jmps_shared_trace_unaligned_exit_bytes)
+    STATS_DEF("Emitted unaligned stubs shtrace", pad_jmps_shared_trace_unaligned_stubs)
+    STATS_DEF("Emitted unaligned stubs bytes shtrace", pad_jmps_shared_trace_unaligned_stubs_bytes)
+    STATS_DEF("-pad_jmps excess bytes shtrace", pad_jmps_shared_trace_excess_bytes)
+    STATS_DEF("Bytes shared frags ever", pad_jmps_shared_trace_sum_fragment_bytes_ever)
+    STATS_DEF("-pad_jmps start_pcs shifted shtrace", pad_jmps_shared_trace_num_start_pc_shifted)
+    STATS_DEF("-pad_jmps start_pcs shifted bytes shtrace", pad_jmps_shared_trace_start_pc_shifted_bytes)
+    STATS_DEF("-pad_jmps excess bytes released shtrace", pad_jmps_shared_trace_extra_space_released)
+    STATS_DEF("-pad_jmps shifted stubs shtrace", pad_jmps_shared_trace_num_shifted_stubs)
+    STATS_DEF("-pad_jmps shifted stub bytes shtrace", pad_jmps_shared_trace_shifted_stub_bytes)
+    STATS_DEF("-pad_jmps no shift stubs shtrace", pad_jmps_shared_trace_num_stubs_no_shift)
+    STATS_DEF("-pad_jmps inserted nops shtrace", pad_jmps_shared_trace_num_nops)
+    STATS_DEF("-pad_jmps inserted nop bytes shtrace", pad_jmps_shared_trace_nop_bytes)
+    STATS_DEF("-pad_jmps no pad exits shtrace", pad_jmps_shared_trace_num_no_pad_exits)
+
+    STATS_DEF("-pad_jmps body bytes trace", pad_jmps_trace_body_bytes)
+    STATS_DEF("-pad_jmps stub bytes trace", pad_jmps_trace_stub_bytes)
+    STATS_DEF("Emitted unaligned exits trace", pad_jmps_trace_unaligned_exits)
+    STATS_DEF("Emitted unaligned exits bytes trace", pad_jmps_trace_unaligned_exit_bytes)
+    STATS_DEF("Emitted unaligned stubs trace", pad_jmps_trace_unaligned_stubs)
+    STATS_DEF("Emitted unaligned stubs bytes trace", pad_jmps_trace_unaligned_stubs_bytes)
+    STATS_DEF("-pad_jmps excess bytes trace", pad_jmps_trace_excess_bytes)
+    STATS_DEF("Bytes trace frags ever", pad_jmps_trace_sum_fragment_bytes_ever)
+    STATS_DEF("-pad_jmps start_pcs shifted trace", pad_jmps_trace_num_start_pc_shifted)
+    STATS_DEF("-pad_jmps start_pcs shifted bytes trace", pad_jmps_trace_start_pc_shifted_bytes)
+    STATS_DEF("-pad_jmps excess bytes released trace", pad_jmps_trace_extra_space_released)
+    STATS_DEF("-pad_jmps shifted stubs trace", pad_jmps_trace_num_shifted_stubs)
+    STATS_DEF("-pad_jmps shifted stub bytes trace", pad_jmps_trace_shifted_stub_bytes)
+    STATS_DEF("-pad_jmps no shift stubs trace", pad_jmps_trace_num_stubs_no_shift)
+    STATS_DEF("-pad_jmps inserted nops trace", pad_jmps_trace_num_nops)
+    STATS_DEF("-pad_jmps inserted nop bytes trace", pad_jmps_trace_nop_bytes)
+    STATS_DEF("-pad_jmps no pad exits trace", pad_jmps_trace_num_no_pad_exits)
+
+    STATS_DEF("-pad_jmps body bytes temp", pad_jmps_temp_body_bytes)
+    STATS_DEF("-pad_jmps stub bytes temp", pad_jmps_temp_stub_bytes)
+    STATS_DEF("Emitted unaligned exits temp", pad_jmps_temp_unaligned_exits)
+    STATS_DEF("Emitted unaligned exits bytes temp", pad_jmps_temp_unaligned_exit_bytes)
+    STATS_DEF("Emitted unaligned stubs temp", pad_jmps_temp_unaligned_stubs)
+    STATS_DEF("Emitted unaligned stubs bytes temp", pad_jmps_temp_unaligned_stubs_bytes)
+    STATS_DEF("-pad_jmps excess bytes temp", pad_jmps_temp_excess_bytes)
+    STATS_DEF("Bytes temp frags ever", pad_jmps_temp_sum_fragment_bytes_ever)
+    STATS_DEF("-pad_jmps start_pcs shifted temp", pad_jmps_temp_num_start_pc_shifted)
+    STATS_DEF("-pad_jmps start_pcs shifted bytes temp", pad_jmps_temp_start_pc_shifted_bytes)
+    STATS_DEF("-pad_jmps excess bytes released temp", pad_jmps_temp_extra_space_released)
+    STATS_DEF("-pad_jmps shifted stubs temp", pad_jmps_temp_num_shifted_stubs)
+    STATS_DEF("-pad_jmps shifted stub bytes temp", pad_jmps_temp_shifted_stub_bytes)
+    STATS_DEF("-pad_jmps no shift stubs temp", pad_jmps_temp_num_stubs_no_shift)
+    STATS_DEF("-pad_jmps inserted nops temp", pad_jmps_temp_num_nops)
+    STATS_DEF("-pad_jmps inserted nop bytes temp", pad_jmps_temp_nop_bytes)
+    STATS_DEF("-pad_jmps no pad exits temp", pad_jmps_temp_num_no_pad_exits)
+
+    STATS_DEF("-pad_jmps body bytes bb", pad_jmps_bb_body_bytes)
+    STATS_DEF("-pad_jmps stub bytes bb", pad_jmps_bb_stub_bytes)
+    STATS_DEF("Emitted unaligned exits bb", pad_jmps_bb_unaligned_exits)
+    STATS_DEF("Emitted unaligned exits bytes bb", pad_jmps_bb_unaligned_exit_bytes)
+    STATS_DEF("Emitted unaligned stubs bb", pad_jmps_bb_unaligned_stubs)
+    STATS_DEF("Emitted unaligned stubs bytes bb", pad_jmps_bb_unaligned_stubs_bytes)
+    STATS_DEF("-pad_jmps excess bytes bb", pad_jmps_bb_excess_bytes)
+    STATS_DEF("Bytes bb frags ever", pad_jmps_bb_sum_fragment_bytes_ever)
+    STATS_DEF("-pad_jmps start_pcs shifted bb", pad_jmps_bb_num_start_pc_shifted)
+    STATS_DEF("-pad_jmps start_pcs shifted bytes bb", pad_jmps_bb_start_pc_shifted_bytes)
+    STATS_DEF("-pad_jmps excess bytes released bb", pad_jmps_bb_extra_space_released)
+    STATS_DEF("-pad_jmps shifted stubs bb", pad_jmps_bb_num_shifted_stubs)
+    STATS_DEF("-pad_jmps shifted stub bytes bb", pad_jmps_bb_shifted_stub_bytes)
+    STATS_DEF("-pad_jmps no shift stubs bb", pad_jmps_bb_num_stubs_no_shift)
+    STATS_DEF("-pad_jmps inserted nops bb", pad_jmps_bb_num_nops)
+    STATS_DEF("-pad_jmps inserted nop bytes bb", pad_jmps_bb_nop_bytes)
+    STATS_DEF("-pad_jmps no pad exits bb", pad_jmps_bb_num_no_pad_exits)
+
+    STATS_DEF("Num unsafe hot patches", unaligned_patches)
+    STATS_DEF("Num nops removed for tracing", num_nops_removed)
+    STATS_DEF("Num bytes nops removed for tracing", num_nop_bytes_removed)
+    STATS_DEF("Num synch yields for exiting threads", synch_yields_for_exiting_thread)
+    STATS_DEF("Num synch yields", synch_yields)
+    STATS_DEF("Num synch loops in wait_at_safe_spot", synch_loops_wait_safe)
+    STATS_DEF("Multiple setcontexts while in wait_at_safe_spot", wait_multiple_setcxt)
+
+#ifdef WINDOWS
+    /* Borland SEH releated stats, xref case 5752 */
+    STATS_DEF("Num build bb loop breaks due to fs instrs", num_bb_build_fs)
+    STATS_DEF("Num bbs terminated early for process SEH", num_process_SEH_bb_early_terminate)
+    STATS_DEF("Num bbs terminated early for debug only process SEH checks", num_process_SEH_bb_early_terminate_debug)
+    STATS_DEF("Num pops to fs0 ignored by process SEH", num_process_SEH_pop_fs0)
+    STATS_DEF("Num observed writes of a NULL/EndList SEH frame", num_endlist_SEH_write)
+    STATS_DEF("Num SEH frame push/pops processed", num_SEH_pushes_processed)
+    STATS_DEF("Num fs instructions processed that aren't SEH", num_fs_movs_not_SEH)
+# ifdef RETURN_AFTER_CALL
+    STATS_DEF("Num borland SEH try constructs initially matched", num_borland_SEH_initial_match)
+    STATS_DEF("Num borland SEH duplicate frames processed", num_borland_SEH_dup_frame)
+    STATS_DEF("Num borland SEH try constructs fully matched", num_borland_SEH_try_match)
+    STATS_DEF("Num borland SEH finally constructs fully matched", num_borland_SEH_finally_match)
+    STATS_DEF("Num borland SEH modified except exemptions", num_borland_SEH_modified)
+# endif
+#endif
+    STATS_DEF("Trace fragment ending with an IBL", num_traces_end_at_ibl)
+    STATS_DEF("Trace fragment ending with an IBL, return", num_traces_end_at_ibl_return)
+    STATS_DEF("Trace fragment ending with an IBL, ind call", num_traces_end_at_ibl_ind_call)
+    STATS_DEF("Trace fragment ending with an IBL, ind jump", num_traces_end_at_ibl_ind_jump)
+    STATS_DEF("Trace fragment ending with an IBL, syscall", num_traces_end_at_ibl_syscall)
+    STATS_DEF("Trace fragment ending at MUST_END_TRACE", num_traces_at_must_end_trace)
+    STATS_DEF("Trace fragment ending with an IBL, speculative", num_traces_end_at_ibl_speculative_link)
+    STATS_DEF("Yields in intercept_apc waiting for dynamo_initialized", apc_yields_while_initializing)
+    STATS_DEF("IBL Tables groomed", num_ibt_groomed)
+    STATS_DEF("IBL Tables reached maximum capacity", num_ibt_max_capacity)
+    STATS_DEF("IAT areas in current modules", num_IAT_areas)
+    STATS_DEF("Indirect calls through IAT", num_indirect_calls_IAT)
+    STATS_DEF("Indirect calls through IAT elided", num_indirect_calls_IAT_elided)
+    STATS_DEF("Indirect calls through IAT converted", num_indirect_calls_IAT_converted)
+    STATS_DEF("Indirect calls through IAT to native module, skipped", num_indirect_calls_IAT_native)
+    STATS_DEF("Indirect jumps through IAT", num_indirect_jumps_IAT)
+    STATS_DEF("Indirect jumps through IAT elided", num_indirect_jumps_IAT_elided)
+    STATS_DEF("Indirect jumps through IAT converted", num_indirect_jumps_IAT_converted)
+    STATS_DEF("Indirect jumps through IAT non PLT, curiosity", num_indirect_jumps_IAT_not_PLT)
+    STATS_DEF("Indirect jumps likely PLT not IAT, curiosity", num_indirect_jumps_PLT_not_IAT)
+    STATS_DEF("Indirect jumps likely PLT", num_indirect_jumps_likely_PLT)
+    STATS_DEF("Indirect jumps through IAT to native module, skipped", num_indirect_jumps_IAT_native)
+    STATS_DEF("Invalidating cache due to IAT hooker", num_invalidate_IAT_hooker)
+    STATS_DEF("Num flushes in app mem prot chg -> rwx we could've skipped", num_possible_app_to_rwx_skip_flush)
+#ifdef HOT_PATCHING_INTERFACE
+    STATS_DEF("Protector executions", hotp_num_prot_exec)
+    STATS_DEF("Detector executions", hotp_num_det_exec)
+    STATS_DEF("Hot patch control flow changes", hotp_num_cflow_change)
+    STATS_DEF("Policy read nudges", hotp_num_policy_nudge)
+    STATS_DEF("Mode read nudges", hotp_num_mode_nudge)
+    STATS_DEF("Hot patches injected", hotp_num_inject)
+    STATS_DEF("Fragments that have direct cti that should be hot patched", 
+              hotp_num_frag_direct_cti)
+    STATS_DEF("Hot patches executed after mode was set to OFF; for hotp mode",
+              hotp_exec_mode_off)
+    STATS_DEF("Hot patches injected in hotp_only mode", hotp_only_num_inject)
+    STATS_DEF("Hot patch control flow collisions in hotp_only mode", hotp_only_cflow_collision)
+    STATS_DEF("hotp_only injections aborted", hotp_only_aborted_injects)
+#endif
+    STATS_DEF("ASLR DLLs randomized", aslr_dlls_bumped)
+    STATS_DEF("ASLR DLLs randomized, retry", aslr_error_retry)
+    STATS_DEF("ASLR DLLs conflicted, giving up", aslr_dll_conflict_giveup)
+    STATS_DEF("ASLR DLLs conflicted, better fit retry", aslr_dll_conflict_fit_retry)
+    STATS_DEF("ASLR DLLs randomized, retry: at base", aslr_retry_at_base)
+    STATS_DEF("ASLR DLLs randomized, retry: not at base", aslr_retry_not_at_base)
+    STATS_DEF("ASLR DLLs randomized, retry: app error", aslr_error_on_retry)
+    STATS_DEF("ASLR DLLs randomized, exempted: reverted", aslr_dlls_exempted)
+    STATS_DEF("mmap --x mappings loader, no fixup", app_mmap_section_x)
+    STATS_DEF("mmap rw- mappings loader, fixup", app_mmap_section_rw)
+    STATS_DEF("mmap r-- mappings non-loader", app_mmap_section_r)
+    STATS_DEF("mmap requested base mappings", app_mmap_requested_base)
+    STATS_DEF("mmap PE as MAPPED not IMAGE", app_mmap_PE_as_MAPPED)
+    STATS_DEF("ASLR randomized mmap of not PE", app_mmap_not_PE_rebased)
+    STATS_DEF("ASLR sharing: not using shared", aslr_dlls_not_shared)
+    STATS_DEF("ASLR sharing: shared map attempted", aslr_dlls_shared_mapped)
+    STATS_DEF("ASLR sharing: shared mapped good", aslr_dlls_shared_mapped_good)
+    STATS_DEF("ASLR sharing: shared map rebased, not shared", aslr_dlls_shared_map_rebased)
+    STATS_DEF("ASLR sharing: shared map failed, BAD", aslr_dlls_shared_map_failed)
+    STATS_DEF("ASLR sharing: compare handle with KnownDlls", aslr_dlls_known_dlls_query)
+    STATS_DEF("ASLR wouldbe RCT ind call/jmp", aslr_rct_ind_wouldbe)
+    STATS_DEF("ASLR wouldbe RCT ret", aslr_rct_ret_wouldbe)
+    STATS_DEF("ASLR wouldbe target, native execution fault", aslr_wouldbe_exec)
+    STATS_DEF("ASLR heap pads, current", aslr_heap_pads)
+    STATS_DEF("ASLR heap pads, peak", peak_aslr_heap_pads)
+    STATS_DEF("ASLR heap pads, ever", ever_aslr_heap_pads)
+    STATS_DEF("ASLR initial padding, KB", aslr_heap_initial_reservation)
+    STATS_DEF("ASLR current total random padding, KB", aslr_heap_total_reservation)
+    STATS_DEF("ASLR peak random padding, KB", peak_aslr_heap_total_reservation)
+    STATS_DEF("ASLR HEAP_FILL gave up filling", aslr_heap_giveup_filling)
+    /* GBOP */
+    STATS_DEF("GBOP validations", gbop_validations)
+    STATS_DEF("GBOP violations, BAD", gbop_violations)
diff --git a/core/link.c b/core/link.c
new file mode 100644
index 000000000..fabe667c1
--- /dev/null
+++ b/core/link.c
@@ -0,0 +1,3799 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * link.c - fragment linker routines
+ */
+
+#include "globals.h"
+#include "link.h"
+#include "fragment.h"
+#include "fcache.h"
+#include "emit.h"
+#include "monitor.h"
+#include <string.h> /* for memset */
+#include "perscache.h"
+#include "instr.h" /* PC_RELATIVE_TARGET */
+
+/* fragment_t and future_fragment_t are guaranteed to have flags field at same offset,
+ * so we use it to find incoming_stubs offset
+ */
+#define FRAG_INCOMING_ADDR(f)                                         \
+    ((common_direct_linkstub_t **)                                    \
+     (!TEST(FRAG_IS_FUTURE, f->flags) ? &f->in_xlate.incoming_stubs : \
+      &((future_fragment_t *)f)->incoming_stubs))
+
+
+/* forward decl */
+static void
+coarse_stubs_init(void);
+
+static void
+coarse_stubs_free(void);
+
+static fragment_t *
+fragment_link_lookup_same_sharing(dcontext_t *dcontext, app_pc tag,
+                                  linkstub_t *last_exit, uint flags);
+
+static void
+link_new_coarse_grain_fragment(dcontext_t *dcontext, fragment_t *f);
+
+static void
+coarse_link_to_fine(dcontext_t *dcontext, cache_pc src_stub, fragment_t *src_f,
+                    fragment_t *target_f, bool do_link/*else just add incoming*/);
+
+static coarse_incoming_t *
+prepend_new_coarse_incoming(coarse_info_t *info, cache_pc coarse, linkstub_t *fine);
+
+static fragment_t *
+fragment_coarse_link_wrapper(dcontext_t *dcontext, app_pc target_tag,
+                             cache_pc know_coarse);
+
+static void
+set_fake_direct_linkstub(direct_linkstub_t *proxy, app_pc target, cache_pc stub);
+
+/* Removes an incoming entry from a fine fragment to a coarse unit */
+static void
+coarse_remove_incoming(dcontext_t *dcontext, fragment_t *src_f, linkstub_t *src_l,
+                       fragment_t *targetf);
+
+/* We use temporary structs to pass targets to is_linkable().
+ * change_linking_lock protects use of these.
+ * FIXME: change is_linkable to take in field values directly?
+ */
+DECLARE_FREQPROT_VAR(static fragment_t temp_targetf, {0});
+DECLARE_FREQPROT_VAR(static direct_linkstub_t temp_linkstub, {{{0}}});
+
+/* This lock makes changes in a shared fragment's link state consistent --
+ * the flags indicating whether it is linked and the link state of each of
+ * its exits.  Since future fragments are driven by linking, this lock
+ * also synchronizes creation and deletion of future fragments.
+ * Exported so micro routines can assert whether held.
+ */
+DECLARE_CXTSWPROT_VAR(recursive_lock_t change_linking_lock,
+                      INIT_RECURSIVE_LOCK(change_linking_lock));
+
+/* Special executable heap for separate stubs
+ * To avoid wasting capacity space we use a shared heap for all stubs
+ */
+void *stub_heap;
+#ifdef X64
+void *stub32_heap;
+#endif
+
+#ifdef X64
+# define SEPARATE_STUB_HEAP(flags) (FRAG_IS_32(flags) ? stub32_heap : stub_heap)
+#else
+# define SEPARATE_STUB_HEAP(flags) stub_heap
+#endif
+
+/* We save 1 byte per stub by not aligning to 16/24 bytes, since
+ * infrequently executed and infrequently accessed (heap free list
+ * adds to start so doesn't walk list).
+ */
+#define SEPARATE_STUB_ALLOC_SIZE(flags) (DIRECT_EXIT_STUB_SIZE(flags)) /* 15x23 */
+
+/* Coarse stubs must be hot-patchable, so we avoid having their last
+ * 4 bytes cross cache lines.
+ * For x64, the stub is 29 bytes long, so the last 4 bytes are fine
+ * for a 16-byte cache line.
+ */
+#define COARSE_STUB_ALLOC_SIZE(flags) \
+    (ALIGN_FORWARD(STUB_COARSE_DIRECT_SIZE(flags), 4)) /* 16x32 */
+
+/* Static linkstubs to give information on special exits from the cache.
+ * We make them const to get them in read-only memory even if we have to cast a lot.
+ * Our accessor routines do NOT cast since equality tests can use const and they
+ * are the most common use.
+ * FIXME: we're accumulating a lot of these -- but we don't have room in our flags
+ * space to distinguish any other way nicely, so we carry on w/ a bunch of
+ * identical static linkstubs.
+ */
+/* linkstub_fragment() returns a static fragment_t for these fake linkstubs: */
+static const fragment_t linkstub_empty_fragment = { NULL, FRAG_FAKE, };
+#ifdef X64
+static const fragment_t linkstub_empty_fragment_x86 = { NULL, FRAG_FAKE | FRAG_32_BIT, };
+#endif
+static const linkstub_t linkstub_starting    = { LINK_FAKE, 0 };
+static const linkstub_t linkstub_reset       = { LINK_FAKE, 0 };
+static const linkstub_t linkstub_syscall     = { LINK_FAKE, 0 };
+static const linkstub_t linkstub_selfmod     = { LINK_FAKE, 0 };
+static const linkstub_t linkstub_ibl_deleted = { LINK_FAKE, 0 };
+#ifdef LINUX
+static const linkstub_t linkstub_sigreturn   = { LINK_FAKE, 0 };
+#else /* WINDOWS */
+static const linkstub_t linkstub_asynch      = { LINK_FAKE, 0 };
+#endif
+static const linkstub_t linkstub_native_exec = { LINK_FAKE, 0 };
+/* this one we give the flag LINK_NI_SYSCALL for executing a syscall in dispatch() */
+static const linkstub_t linkstub_native_exec_syscall =
+    { LINK_FAKE| LINK_NI_SYSCALL, 0 };
+
+#ifdef WINDOWS
+/* used for shared_syscall exits */
+static const fragment_t linkstub_shared_syscall_trace_fragment =
+    { NULL, FRAG_FAKE | FRAG_HAS_SYSCALL | FRAG_IS_TRACE, };
+static const fragment_t linkstub_shared_syscall_bb_fragment =
+    { NULL, FRAG_FAKE | FRAG_HAS_SYSCALL, };
+# ifdef PROFILE_LINKCOUNT
+/* For PROFILE_LINKCOUNT we need to be able to write to these from the cache
+ * to update their linkcounts, so we can't have them const or in .data.
+ * Since this is not a product config we don't care if we don't protect their
+ * flags (if we did we could put these on the heap).
+ */
+DECLARE_NEVERPROT_VAR(static linkstub_t linkstub_shared_syscall_trace,
+    { LINK_FAKE| LINK_INDIRECT | LINK_JMP, 0 });
+DECLARE_NEVERPROT_VAR(static linkstub_t linkstub_shared_syscall_bb,
+    { LINK_FAKE| LINK_INDIRECT | LINK_JMP, 0 });
+DECLARE_NEVERPROT_VAR(static linkstub_t linkstub_shared_syscall_unlinked,
+    { LINK_FAKE, 0 });
+# else
+static const linkstub_t linkstub_shared_syscall_trace =
+    { LINK_FAKE| LINK_INDIRECT | LINK_JMP, 0 };
+static const linkstub_t linkstub_shared_syscall_bb =
+    { LINK_FAKE| LINK_INDIRECT | LINK_JMP, 0 };
+/* NOT marked as LINK_INDIRECT|LINK_JMP since we don't bother updating the ibl table
+ * on the unlink path */
+static const linkstub_t linkstub_shared_syscall_unlinked = { LINK_FAKE, 0 };
+# endif
+#endif
+
+/* A unique fragment_t for use when the details don't matter */
+static const fragment_t coarse_fragment =
+    { NULL, FRAG_FAKE | FRAG_COARSE_GRAIN | FRAG_SHARED |
+      /* We mark as linked so is_linkable() won't reject it on any side of a link */
+      FRAG_LINKED_OUTGOING | FRAG_LINKED_INCOMING, };
+
+/* We don't mark as direct since not everything checks for being fake */
+static const linkstub_t linkstub_coarse_exit = { LINK_FAKE, 0 };
+static const linkstub_t linkstub_coarse_trace_head_exit = { LINK_FAKE, 0 };
+
+#ifdef HOT_PATCHING_INTERFACE
+/* used to change control flow in a hot patch routine */
+static const linkstub_t linkstub_hot_patch = { LINK_FAKE, 0 };
+#endif
+
+#ifdef CLIENT_INTERFACE
+/* used for dr_redirect_exection() call to transfer_to_dispatch() */
+static const linkstub_t linkstub_client = { LINK_FAKE, 0 };
+#endif
+
+/* for !DYNAMO_OPTION(indirect_stubs)
+ * FIXME: these are used for shared_syscall as well, yet not marked as
+ * FRAG_HAS_SYSCALL, but nobody checks for that currently
+ */
+static const fragment_t linkstub_ibl_trace_fragment =
+    { NULL, FRAG_FAKE | FRAG_IS_TRACE, };
+static const fragment_t linkstub_ibl_bb_fragment =
+    { NULL, FRAG_FAKE, };
+
+# ifdef PROFILE_LINKCOUNT
+DECLARE_NEVERPROT_VAR(static const linkstub_t linkstub_ibl_trace_ret,
+    { LINK_FAKE | LINK_INDIRECT | LINK_RETURN, 0 });
+DECLARE_NEVERPROT_VAR(static const linkstub_t linkstub_ibl_trace_jmp,
+    { LINK_FAKE | LINK_INDIRECT | LINK_JMP, 0 });
+DECLARE_NEVERPROT_VAR(static const linkstub_t linkstub_ibl_trace_call,
+    { LINK_FAKE | LINK_INDIRECT | LINK_CALL, 0 });
+DECLARE_NEVERPROT_VAR(static const linkstub_t linkstub_ibl_bb_ret,
+    { LINK_FAKE | LINK_INDIRECT | LINK_RETURN, 0 });
+DECLARE_NEVERPROT_VAR(static const linkstub_t linkstub_ibl_bb_jmp,
+    { LINK_FAKE | LINK_INDIRECT | LINK_JMP, 0 });
+DECLARE_NEVERPROT_VAR(static const linkstub_t linkstub_ibl_bb_call,
+    { LINK_FAKE | LINK_INDIRECT | LINK_CALL, 0 });
+# else /* !PROFILE_LINKCOUNT */
+static const linkstub_t linkstub_ibl_trace_ret =
+    { LINK_FAKE | LINK_INDIRECT | LINK_RETURN, 0 };
+static const linkstub_t linkstub_ibl_trace_jmp =
+    { LINK_FAKE | LINK_INDIRECT | LINK_JMP, 0 };
+static const linkstub_t linkstub_ibl_trace_call =
+    { LINK_FAKE | LINK_INDIRECT | LINK_CALL, 0 };
+static const linkstub_t linkstub_ibl_bb_ret =
+    { LINK_FAKE | LINK_INDIRECT | LINK_RETURN, 0 };
+static const linkstub_t linkstub_ibl_bb_jmp =
+    { LINK_FAKE | LINK_INDIRECT | LINK_JMP, 0 };
+static const linkstub_t linkstub_ibl_bb_call =
+    { LINK_FAKE | LINK_INDIRECT | LINK_CALL, 0 };
+# endif
+
+static inline bool
+is_empty_fragment(fragment_t *f)
+{
+    return (f == (fragment_t *)&linkstub_empty_fragment
+            IF_X64(|| f == (fragment_t *)&linkstub_empty_fragment_x86));
+}
+
+#ifdef X64
+fragment_t *
+empty_fragment_mark_x86(fragment_t *f)
+{
+    ASSERT(f == (fragment_t *)&linkstub_empty_fragment);
+    return (fragment_t *)&linkstub_empty_fragment_x86;
+}
+#endif
+
+/* used to hold important fields for last_exits that are flushed */
+typedef struct thread_link_data_t {
+    linkstub_t linkstub_deleted;
+    fragment_t linkstub_deleted_fragment;
+    /* The ordinal is the count from the end.
+     * -1 means invalid.
+     * The value corresponds to the get_deleted_linkstub() linkstub_t only and
+     * may be stale wrt dcontext->last_exit.
+     */
+    int linkstub_deleted_ordinal;
+} thread_link_data_t;
+
+/* thread-shared initialization that should be repeated after a reset */
+void
+link_reset_init(void)
+{
+    if (DYNAMO_OPTION(separate_private_stubs) ||
+        DYNAMO_OPTION(separate_shared_stubs)) {
+        stub_heap = special_heap_init(SEPARATE_STUB_ALLOC_SIZE(0/*default*/),
+                                      true /* must synch */, true /* +x */,
+                                      false /* not persistent */);
+#ifdef X64
+        stub32_heap = special_heap_init(SEPARATE_STUB_ALLOC_SIZE(FRAG_32_BIT),
+                                        true /* must synch */, true /* +x */,
+                                        false /* not persistent */);
+#endif
+    }
+}
+
+/* Free all thread-shared state not critical to forward progress;
+ * link_reset_init() will be called before continuing.
+ */
+void
+link_reset_free(void)
+{
+    if (DYNAMO_OPTION(separate_private_stubs) ||
+        DYNAMO_OPTION(separate_shared_stubs)) {
+        special_heap_exit(stub_heap);
+#ifdef X64
+        special_heap_exit(stub32_heap);
+#endif
+    }
+}
+
+void
+link_init()
+{
+    link_reset_init();
+    coarse_stubs_init();
+}
+
+void
+link_exit()
+{
+    coarse_stubs_free();
+    link_reset_free();
+    DELETE_RECURSIVE_LOCK(change_linking_lock);
+}
+
+void
+link_thread_init(dcontext_t *dcontext)
+{
+    thread_link_data_t *ldata =
+        HEAP_TYPE_ALLOC(dcontext, thread_link_data_t, ACCT_OTHER, PROTECTED);
+    dcontext->link_field = (void *) ldata;
+    memset(&ldata->linkstub_deleted, 0, sizeof(ldata->linkstub_deleted));
+    memset(&ldata->linkstub_deleted_fragment, 0,
+           sizeof(ldata->linkstub_deleted_fragment));
+    ldata->linkstub_deleted_ordinal = -1;
+    /* Mark as fake */
+    ldata->linkstub_deleted_fragment.flags = FRAG_FAKE;
+    ldata->linkstub_deleted.flags = LINK_FAKE;
+}
+
+void
+link_thread_exit(dcontext_t *dcontext)
+{
+    thread_link_data_t *ldata = (thread_link_data_t *) dcontext->link_field;
+    HEAP_TYPE_FREE(dcontext, ldata, thread_link_data_t, ACCT_OTHER, PROTECTED);
+}
+
+/* Initializes an array of linkstubs beginning with first */
+void
+linkstubs_init(linkstub_t *first, int num_direct, int num_indirect, fragment_t *f)
+{
+    /* we don't know the sequencing of direct and indirect so all we do
+     * here is zero everything out
+     */
+    uint size = linkstubs_heap_size(f->flags, num_direct, num_indirect);
+    ASSERT(num_direct + num_indirect > 0);
+    memset(first, 0, size);
+    /* place the offset to the fragment_t*, if necessary */
+    if (linkstub_frag_offs_at_end(f->flags, num_direct, num_indirect)) {
+        ushort offs;
+        /* size includes post_linkstub_t so we have to subtract it off */
+        post_linkstub_t *post = (post_linkstub_t *)
+            (((byte *)first) + size - sizeof(post_linkstub_t));
+        ASSERT_TRUNCATE(offs, ushort, (((byte *)post) - ((byte *)f)));
+        offs = (ushort) (((byte *)post) - ((byte *)f));
+        post->fragment_offset = offs;
+    }
+}
+
+uint
+linkstub_size(dcontext_t *dcontext, fragment_t *f, linkstub_t *l)
+{
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    /* FIXME: optimization: now for ind exits we compute a pc from
+     * flags and then exit_stub_size looks up the pc to find whether
+     * indirect -- should pass that flag.
+     * There are a number of uses of this idiom in emit_utils.c as well.
+     */
+    return exit_stub_size(dcontext, EXIT_TARGET_TAG(dcontext, f, l), f->flags);
+}
+
+/* we don't want to propagate our cbr hack during trace building, etc. */
+uint
+linkstub_propagatable_flags(uint flags)
+{
+    if (LINKSTUB_CBR_FALLTHROUGH(flags))
+        flags &= ~LINK_INDIRECT;
+    return flags;
+}
+
+/* is a post_linkstub_t structure required to store the fragment_t offset? */
+bool
+linkstub_frag_offs_at_end(uint flags, int direct_exits, int indirect_exits)
+{
+    ASSERT((direct_exits + indirect_exits > 0) || TEST(FRAG_COARSE_GRAIN, flags));
+    /* common bb types do not have an offset at the end:
+     * 1) single indirect exit
+     * 2) two direct exits
+     * 3) coarse-grain, which of course have no fragment_t
+     * see linkstub_fragment() for how we find their owning fragment_t's.
+     * Since we have to assume sizeof(fragment_t) we can only allow
+     * shared fragments to not have an offset, unless we add a LINK_SHARED
+     * flag, which we can do if we end up having a product config w/ private bbs.
+     * FIXME: make the sizeof calculation dynamic such that the dominant
+     * type of bb fragment is the one w/o the post_linkstub_t.
+     */
+    return !(!TEST(FRAG_IS_TRACE, flags) &&
+             TEST(FRAG_SHARED, flags) &&
+             /* We can't tell from the linkstub_t whether there is a
+              * translation field.  FIXME: we could avoid this problem
+              * by storing the translation field after the linkstubs.
+              */
+             !TEST(FRAG_HAS_TRANSLATION_INFO, flags) &&
+             ((direct_exits == 2 && indirect_exits == 0) ||
+              (direct_exits == 0 && indirect_exits == 1)))
+        && !TEST(FRAG_COARSE_GRAIN, flags);
+}
+
+bool
+use_cbr_fallthrough_short(uint flags, int direct_exits, int indirect_exits)
+{
+    ASSERT((direct_exits + indirect_exits > 0) || TEST(FRAG_COARSE_GRAIN, flags));
+    if (direct_exits != 2 || indirect_exits != 0)
+        return false;
+#ifdef CLIENT_INTERFACE
+    /* cannot handle instrs inserted between cbr and fall-through jmp */
+    return false;
+#else
+    /* we can only use the cbr_fallthrough_linkstub_t if these conditions
+     * apply:
+     *   1) separate stubs will not be individually freed -- we could
+     *      have a fancy scheme that frees both at once, but we simply
+     *      disallow the struct if any freeing will occur.
+     *   2) the fallthrough target is close to the start pc
+     *   3) the fallthrough exit immediately follows the cbr exit
+     *      (we assume this is true if !CLIENT_INTERFACE)
+     * conditions 2 and 3 are asserted in emit.c
+     */
+    return (TEST(FRAG_CBR_FALLTHROUGH_SHORT, flags) &&
+            !TEST(FRAG_COARSE_GRAIN, flags) &&
+            ((TEST(FRAG_SHARED, flags) && !DYNAMO_OPTION(unsafe_free_shared_stubs)) ||
+             (!TEST(FRAG_SHARED, flags) && !DYNAMO_OPTION(free_private_stubs))));
+#endif
+}
+
+/* includes the post_linkstub_t offset struct size */
+uint
+linkstubs_heap_size(uint flags, int direct_exits, int indirect_exits)
+{
+    uint offset_sz, linkstub_size;
+    ASSERT((direct_exits + indirect_exits > 0) || TEST(FRAG_COARSE_GRAIN, flags));
+    if (use_cbr_fallthrough_short(flags, direct_exits, indirect_exits)) {
+        linkstub_size = sizeof(direct_linkstub_t) +
+            sizeof(cbr_fallthrough_linkstub_t);
+    } else {
+        linkstub_size = direct_exits * sizeof(direct_linkstub_t) +
+            indirect_exits * sizeof(indirect_linkstub_t);
+    }
+    if (linkstub_frag_offs_at_end(flags, direct_exits, indirect_exits))
+        offset_sz = sizeof(post_linkstub_t);
+    else
+        offset_sz = 0;
+    return linkstub_size + offset_sz;
+}
+
+/* Locate the owning fragment_t for a linkstub_t */
+fragment_t *
+linkstub_fragment(dcontext_t *dcontext, linkstub_t *l)
+{
+    if (LINKSTUB_FAKE(l)) {
+#ifdef WINDOWS
+        if (l == &linkstub_shared_syscall_trace)
+            return (fragment_t *)&linkstub_shared_syscall_trace_fragment;
+        else if (l == &linkstub_shared_syscall_bb)
+            return (fragment_t *)&linkstub_shared_syscall_bb_fragment;
+#endif
+        if (l == &linkstub_ibl_trace_ret ||
+            l == &linkstub_ibl_trace_jmp ||
+            l == &linkstub_ibl_trace_call)
+            return (fragment_t *)&linkstub_ibl_trace_fragment;
+        else if (l == &linkstub_ibl_bb_ret ||
+                 l == &linkstub_ibl_bb_jmp ||
+                 l == &linkstub_ibl_bb_call)
+            return (fragment_t *)&linkstub_ibl_bb_fragment;
+        if (dcontext != NULL && dcontext != GLOBAL_DCONTEXT) {
+            thread_link_data_t *ldata = (thread_link_data_t *) dcontext->link_field;
+            if (l == &ldata->linkstub_deleted)
+                return &ldata->linkstub_deleted_fragment;
+        }
+        /* For coarse proxies, we need a fake FRAG_SHARED fragment_t for is_linkable */
+        if (LINKSTUB_COARSE_PROXY(l->flags))
+            return (fragment_t *)&coarse_fragment;
+        return (fragment_t *)&linkstub_empty_fragment;
+    }
+    /* To save space we no longer store a backpointer in the linkstub_t.
+     * Instead we use several schemes based on the type of owning
+     * fragment_t.  We could walk backward but that gets complicated with
+     * hacks to distinguish types of structs by their final fields w/o
+     * adding secondary flags fields.
+     */
+    if (TEST(LINK_FRAG_OFFS_AT_END, l->flags)) {
+        /* For traces and unusual bbs, we store an offset to the 
+         * fragment_t at the end of the linkstub_t list.
+         */
+        post_linkstub_t *post;
+        for (; !LINKSTUB_FINAL(l); l = LINKSTUB_NEXT_EXIT(l))
+            ; /* nothing */
+        ASSERT(l != NULL && TEST(LINK_END_OF_LIST, l->flags));
+        post = (post_linkstub_t *) (((byte*)l) + LINKSTUB_SIZE(l));
+        return (fragment_t *) (((byte *)post) - post->fragment_offset);
+    } else {
+        /* Otherwise, we assume this is one of 2 types of basic block! */
+        if (LINKSTUB_INDIRECT(l->flags)) {
+            /* Option 1: a single indirect exit */
+            ASSERT(TEST(LINK_END_OF_LIST, l->flags));
+            return (fragment_t *) ( ((byte *)l) - sizeof(fragment_t) );
+        } else {
+            ASSERT(LINKSTUB_DIRECT(l->flags));
+            /* Option 2: two direct exits (doesn't matter if 2nd uses
+             * cbr_fallthrough_linkstub_t or not)
+             * (single direct exit is very rare but if later becomes common
+             * could add a LINK_ flag to distinguish)
+             */
+            if (TEST(LINK_END_OF_LIST, l->flags)) {
+                return (fragment_t *) ( ((byte *)l) - sizeof(direct_linkstub_t) -
+                                      sizeof(fragment_t) );
+            } else {
+                return (fragment_t *) ( ((byte *)l) - sizeof(fragment_t) );
+            }
+        }
+    }
+    ASSERT_NOT_REACHED();
+    return NULL;
+}
+
+#ifdef DEBUG
+bool
+linkstub_owned_by_fragment(dcontext_t *dcontext, fragment_t *f, linkstub_t *l)
+{
+    linkstub_t *ls;
+    /* handle fake linkstubs first since FRAGMENT_EXIT_STUBS shouldn't be called
+     * on fake fragments!
+     */
+    if (LINKSTUB_FAKE(l)) {
+        ASSERT(TEST(FRAG_FAKE, f->flags) ||
+               /* during emit, coarse-grain has real fragment_t but sometimes fake
+                * linkstubs during linking
+                */
+               TEST(FRAG_COARSE_GRAIN, f->flags));
+        /* Coarse-grain temp fragment_t's also have temp fake linkstub_t's */ 
+        if (TEST(FRAG_COARSE_GRAIN, f->flags))
+            return true;
+        else {
+            return (linkstub_fragment(dcontext, l) == f
+                    /* For reset exit stub we need a fake empty fragment marked as x86 */
+                    IF_X64(|| (is_empty_fragment(linkstub_fragment(dcontext, l)) &&
+                               f == (fragment_t *)&linkstub_empty_fragment_x86)));
+        }
+    }
+    for (ls = FRAGMENT_EXIT_STUBS(f); ls != NULL; ls = LINKSTUB_NEXT_EXIT(ls)) {
+        if (ls == l)
+            return true;
+    }
+    return false;
+}
+#endif
+
+void
+set_last_exit(dcontext_t *dcontext, linkstub_t *l)
+{
+    /* We try to set last_fragment every time we set last_exit, rather than
+     * leaving it as a dangling pointer until the next dispatch entrance,
+     * to avoid cases like bug 7534.  However, fcache_return only sets
+     * last_exit, though it hits the dispatch point that sets last_fragment
+     * soon after, and everyone who frees fragments that other threads can
+     * see should call last_exit_deleted() on the other threads.
+     */
+    ASSERT(l != NULL);
+    dcontext->last_exit = l;
+    dcontext->last_fragment = linkstub_fragment(dcontext, dcontext->last_exit);
+    ASSERT(dcontext->last_fragment != NULL);
+    /* We also cannot leave dir_exit as a dangling pointer */
+    dcontext->coarse_exit.dir_exit = NULL;
+}
+
+void
+last_exit_deleted(dcontext_t *dcontext)
+{
+    thread_link_data_t *ldata = (thread_link_data_t *) dcontext->link_field;
+    linkstub_t *l;
+    /* if this gets called twice, second is a nop
+     * FIXME: measure how often, reduce dup calls
+     */
+    if (LINKSTUB_FAKE(dcontext->last_exit) &&
+        /* be defensive (remember case 7534!): re-do in case someone
+         * sets last_exit to FAKE but leaves last_fragment dangling
+         */
+        dcontext->last_fragment == &ldata->linkstub_deleted_fragment)
+        return;
+    ASSERT(LINKSTUB_FAKE(&ldata->linkstub_deleted));
+    ASSERT(linkstub_fragment(dcontext, &ldata->linkstub_deleted) ==
+           &ldata->linkstub_deleted_fragment);
+    DODEBUG({
+        /* easier to clear these and ensure whole thing is 0 for the fragment */
+        ldata->linkstub_deleted_fragment.tag = 0;
+        ldata->linkstub_deleted_fragment.flags = 0;
+        ldata->linkstub_deleted_fragment.id = 0;
+        ASSERT(is_region_memset_to_char((byte *)&ldata->linkstub_deleted_fragment,
+                                        sizeof(ldata->linkstub_deleted_fragment), 0));
+    });
+    /* FIME: should we have special dcontext fields last_exit_flags &
+     * last_fragment_{flags,tag} and make everyone use those to ensure
+     * nobody treats these as the real thing?
+     * But trace building and some others need the real thing, so would
+     * have to check whether valid anyway, so just as costly to check
+     * whether these are valid.  FRAG_FAKE and LINK_FAKE tags help us out here.
+     */
+    ldata->linkstub_deleted.flags = dcontext->last_exit->flags;
+    ldata->linkstub_deleted.flags |= LINK_FAKE;
+    ldata->linkstub_deleted_fragment.tag = dcontext->last_fragment->tag;
+    ldata->linkstub_deleted_fragment.flags |= FRAG_FAKE;
+#ifdef DEBUG_MEMORY
+    ASSERT(dcontext->last_fragment->flags != HEAP_UNALLOCATED_UINT);
+    ASSERT(dcontext->last_fragment->id != HEAP_UNALLOCATED_UINT);
+#endif
+    ldata->linkstub_deleted_fragment.flags = dcontext->last_fragment->flags;
+    DODEBUG({
+        ldata->linkstub_deleted_fragment.id = dcontext->last_fragment->id;
+    });
+
+    /* Store which exit this is, for trace building.
+     * Our ordinal is the count from the end.
+     */
+    if (TEST(FRAG_FAKE, dcontext->last_fragment->flags))
+        ldata->linkstub_deleted_ordinal = -1; /* invalid */
+    else {
+        ldata->linkstub_deleted_ordinal = 0;
+        for (l = FRAGMENT_EXIT_STUBS(dcontext->last_fragment);
+             l != NULL && l != dcontext->last_exit;
+             l = LINKSTUB_NEXT_EXIT(l)) {
+            /* nothing */
+        }
+        if (l == dcontext->last_exit) {
+            /* 0 means the last one */
+            for (l = LINKSTUB_NEXT_EXIT(l); l != NULL; l = LINKSTUB_NEXT_EXIT(l))
+                ldata->linkstub_deleted_ordinal++;
+        } else {
+            ASSERT_NOT_REACHED();
+            ldata->linkstub_deleted_ordinal = -1; /* invalid */
+        }
+    }
+
+    /* now install the copy as the last exit, but don't overwrite an existing
+     * fake exit (like a native_exec exit: case 8033)
+     */
+    if (!LINKSTUB_FAKE(dcontext->last_exit))
+        dcontext->last_exit = &ldata->linkstub_deleted;
+    dcontext->last_fragment = &ldata->linkstub_deleted_fragment;
+
+    /* We also cannot leave dir_exit as a dangling pointer */
+    dcontext->coarse_exit.dir_exit = NULL;
+}
+
+void
+set_coarse_ibl_exit(dcontext_t *dcontext)
+{
+    thread_link_data_t *ldata = (thread_link_data_t *) dcontext->link_field;
+    app_pc src_tag = dcontext->coarse_exit.src_tag;
+    ASSERT(src_tag != NULL);
+
+    if (!DYNAMO_OPTION(coarse_units) ||
+        !is_ibl_sourceless_linkstub((const linkstub_t*)dcontext->last_exit))
+        return;
+
+    /* We use the sourceless linkstubs for the ibl, but we do have source info
+     * in dcontext->coarse_exit.src_tag, so once back in DR we switch to our
+     * deleted structs to fill in the src info
+     */
+    /* Re-use the deletion routine to fill in the fields for us.
+     * It won't replace last_exit itself as it's already fake, but it will
+     * replace last_fragment.  FIXME: now we have last_fragment != result of
+     * linkstub_fragment()!
+     */
+    last_exit_deleted(dcontext);
+    ldata->linkstub_deleted_fragment.tag = src_tag;
+}
+
+/* The ordinal is the count from the end.
+ * -1 means invalid.
+ * The value corresponds to the get_deleted_linkstub() linkstub_t only and
+ * may be stale wrt dcontext->last_exit.
+ */
+int
+get_last_linkstub_ordinal(dcontext_t *dcontext)
+{
+    thread_link_data_t *ldata;
+    ASSERT(dcontext != GLOBAL_DCONTEXT);
+    ldata = (thread_link_data_t *) dcontext->link_field;
+    return ldata->linkstub_deleted_ordinal;
+}
+
+linkstub_t *
+get_deleted_linkstub(dcontext_t *dcontext)
+{
+    thread_link_data_t *ldata;
+    ASSERT(dcontext != GLOBAL_DCONTEXT);
+    ldata = (thread_link_data_t *) dcontext->link_field;
+    return &ldata->linkstub_deleted;
+}
+
+const linkstub_t *
+get_starting_linkstub()
+{
+    return &linkstub_starting;
+}
+
+const linkstub_t *
+get_reset_linkstub()
+{
+    return &linkstub_reset;
+}
+
+const linkstub_t *
+get_syscall_linkstub()
+{
+    return &linkstub_syscall;
+}
+
+const linkstub_t *
+get_selfmod_linkstub()
+{
+    return &linkstub_selfmod;
+}
+
+const linkstub_t *
+get_ibl_deleted_linkstub()
+{
+    return &linkstub_ibl_deleted;
+}
+
+#ifdef LINUX
+const linkstub_t *
+get_sigreturn_linkstub()
+{
+    return &linkstub_sigreturn;
+}
+#else /* WINDOWS */
+const linkstub_t *
+get_asynch_linkstub()
+{
+    return &linkstub_asynch;
+}
+#endif
+
+const linkstub_t *
+get_native_exec_linkstub()
+{
+    return &linkstub_native_exec;
+}
+
+const linkstub_t *
+get_native_exec_syscall_linkstub()
+{
+    return &linkstub_native_exec_syscall;
+}
+
+#ifdef WINDOWS
+const linkstub_t *
+get_shared_syscalls_unlinked_linkstub()
+{
+    return &linkstub_shared_syscall_unlinked;
+}
+
+const linkstub_t *
+get_shared_syscalls_trace_linkstub()
+{
+    return &linkstub_shared_syscall_trace;
+}
+
+const linkstub_t *
+get_shared_syscalls_bb_linkstub()
+{
+    return &linkstub_shared_syscall_bb;
+}
+#endif /* WINDOWS */
+
+#ifdef HOT_PATCHING_INTERFACE
+const linkstub_t *
+get_hot_patch_linkstub()
+{
+    return &linkstub_hot_patch;
+}
+#endif
+
+#ifdef CLIENT_INTERFACE
+const linkstub_t *
+get_client_linkstub()
+{
+    return &linkstub_client;
+}
+#endif
+
+bool
+is_ibl_sourceless_linkstub(const linkstub_t *l)
+{
+    return (l == &linkstub_ibl_trace_ret ||
+            l == &linkstub_ibl_trace_jmp ||
+            l == &linkstub_ibl_trace_call ||
+            l == &linkstub_ibl_bb_ret ||
+            l == &linkstub_ibl_bb_jmp ||
+            l == &linkstub_ibl_bb_call);
+}
+
+const linkstub_t *
+get_ibl_sourceless_linkstub(uint link_flags, uint frag_flags)
+{
+    if (TEST(FRAG_IS_TRACE, frag_flags)) {
+        if (TEST(LINK_RETURN, link_flags))
+            return &linkstub_ibl_trace_ret;
+        if (TEST(LINK_JMP, link_flags))
+            return &linkstub_ibl_trace_jmp;
+        if (TEST(LINK_CALL, link_flags))
+            return &linkstub_ibl_trace_call;
+    } else {
+        if (TEST(LINK_RETURN, link_flags))
+            return &linkstub_ibl_bb_ret;
+        if (TEST(LINK_JMP, link_flags))
+            return &linkstub_ibl_bb_jmp;
+        if (TEST(LINK_CALL, link_flags))
+            return &linkstub_ibl_bb_call;
+    }
+    ASSERT_NOT_REACHED();
+    return NULL;
+}
+
+/* Direct exit not targeting a trace head */
+const linkstub_t *
+get_coarse_exit_linkstub()
+{
+    return &linkstub_coarse_exit;
+}
+
+/* Direct exit targeting a trace head */
+const linkstub_t *
+get_coarse_trace_head_exit_linkstub()
+{
+    return &linkstub_coarse_trace_head_exit;
+}
+
+bool
+should_separate_stub(dcontext_t *dcontext, app_pc target, uint fragment_flags)
+{
+    return (local_exit_stub_size(dcontext, target, fragment_flags) == 0);
+}
+
+int
+local_exit_stub_size(dcontext_t *dcontext, app_pc target, uint fragment_flags)
+{
+    /* linking shared separate stubs is not yet atomic so we only support
+     * separate private stubs
+     * FIXME: optimization: some callers have a linkstub_t so we could provide
+     * a separate routine for that to avoid the now-costly computation of
+     * target for indirect exits.
+     */
+    int sz = exit_stub_size(dcontext, target, fragment_flags);
+    if (((DYNAMO_OPTION(separate_private_stubs) &&
+          !TEST(FRAG_COARSE_GRAIN, fragment_flags) &&
+          !TEST(FRAG_SHARED, fragment_flags)) ||
+         (DYNAMO_OPTION(separate_shared_stubs) &&
+          !TEST(FRAG_COARSE_GRAIN, fragment_flags) &&
+          TEST(FRAG_SHARED, fragment_flags)) ||
+         /* entrance stubs are always separated */
+         (TEST(FRAG_COARSE_GRAIN, fragment_flags)
+          /* FIXME: for now we inline ind stubs but eventually we want to separate.
+           * We need this check only for coarse since its stubs are the same size
+           * as the direct stubs.
+           */
+          && !is_indirect_branch_lookup_routine(dcontext, (cache_pc)target))) &&
+        /* we only separate stubs of the regular type, which we determine
+         * by letting exit_stub_size dispatch on flags and return its
+         * results in the stub size
+         */
+        sz == (TEST(FRAG_COARSE_GRAIN, fragment_flags) ?
+               STUB_COARSE_DIRECT_SIZE(fragment_flags) :
+               DIRECT_EXIT_STUB_SIZE(fragment_flags)))
+        return 0;
+    else
+        return sz;
+}
+
+static inline bool
+is_cbr_of_cbr_fallthrough(linkstub_t *l)
+{
+    linkstub_t *nxt = LINKSTUB_NEXT_EXIT(l);
+    bool yes = (nxt != NULL && LINKSTUB_CBR_FALLTHROUGH(nxt->flags));
+    ASSERT(!yes || LINKSTUB_FINAL(nxt));
+    return yes;
+}
+
+void
+separate_stub_create(dcontext_t *dcontext, fragment_t *f, linkstub_t *l)
+{
+    int emit_sz;
+    cache_pc stub_pc;
+    ASSERT(LINKSTUB_DIRECT(l->flags));
+    ASSERT(DYNAMO_OPTION(separate_private_stubs) ||
+           DYNAMO_OPTION(separate_shared_stubs));
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    ASSERT(TEST(LINK_SEPARATE_STUB, l->flags));
+    if (LINKSTUB_CBR_FALLTHROUGH(l->flags)) {
+        /* there is no field for the fallthrough of a short cbr -- it assumes
+         * the cbr and fallthrough stubs are contiguous, and calculates its
+         * stub pc from the cbr stub pc, which we assume here has already
+         * been created since we create them in forward order
+         */
+        stub_pc = EXIT_STUB_PC(dcontext, f, l);
+    } else {
+        direct_linkstub_t *dl = (direct_linkstub_t *) l;
+        ASSERT(LINKSTUB_NORMAL_DIRECT(l->flags));
+        ASSERT(dl->stub_pc == NULL);
+        /* if -cbr_single_stub, LINKSTUB_CBR_FALLTHROUGH _always_ shares a stub,
+         * as its requirements are a superset of cbr stub sharing, so we don't
+         * need a separate flag.  (if we do need one, we could use the 00 combo
+         * for both cbr linkstubs (except if 2nd is cbr-fallthrough, but then
+         * just -cbr_single_stub is enough), testing !FAKE.)
+         */
+        if (is_cbr_of_cbr_fallthrough(l) && !INTERNAL_OPTION(cbr_single_stub)) {
+            /* we have to allocate a pair together */
+            dl->stub_pc = (cache_pc) special_heap_calloc(SEPARATE_STUB_HEAP(f->flags), 2);
+        } else
+            dl->stub_pc = (cache_pc) special_heap_alloc(SEPARATE_STUB_HEAP(f->flags));
+        ASSERT(dl->stub_pc == EXIT_STUB_PC(dcontext, f, l));
+        stub_pc = dl->stub_pc;
+    }
+    emit_sz = insert_exit_stub(dcontext, f, l, stub_pc);
+    ASSERT(emit_sz <= SEPARATE_STUB_ALLOC_SIZE(f->flags));
+    DOSTATS({
+        size_t alloc_size = SEPARATE_STUB_ALLOC_SIZE(f->flags);
+        STATS_INC(num_separate_stubs);
+        if (TEST(FRAG_SHARED, f->flags)) {
+            if (TEST(FRAG_IS_TRACE, f->flags))
+                STATS_ADD(separate_shared_trace_direct_stubs, alloc_size);
+            else
+                STATS_ADD(separate_shared_bb_direct_stubs, alloc_size);
+        } else if (TEST(FRAG_IS_TRACE, f->flags))
+            STATS_ADD(separate_trace_direct_stubs, alloc_size);
+        else
+            STATS_ADD(separate_bb_direct_stubs, alloc_size);
+    });
+}
+
+/* deletion says are we freeing b/c we're freeing the whole fragment,
+ * or just freeing b/c we're linking and don't need this stub anymore?
+ */
+static void
+separate_stub_free(dcontext_t *dcontext, fragment_t *f, linkstub_t *l,
+                   bool deletion)
+{
+    ASSERT(LINKSTUB_DIRECT(l->flags));
+    ASSERT(DYNAMO_OPTION(separate_private_stubs) ||
+           DYNAMO_OPTION(separate_shared_stubs));
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    ASSERT(TEST(LINK_SEPARATE_STUB, l->flags));
+    ASSERT(exit_stub_size(dcontext, EXIT_TARGET_TAG(dcontext, f, l), f->flags) <=
+           SEPARATE_STUB_ALLOC_SIZE(f->flags));
+    if (LINKSTUB_CBR_FALLTHROUGH(l->flags)) {
+        ASSERT(deletion);
+        /* was already freed by forward walk hitting 1st exit */
+    } else {
+        direct_linkstub_t *dl = (direct_linkstub_t *) l;
+        ASSERT(LINKSTUB_NORMAL_DIRECT(l->flags));
+        /* for -cbr_single_stub, non-deletion-freeing is disallowed, and
+         * for deletion freeing, up to caller to not call us twice.
+         * FIXME: we could support freeing when both stubs are linked if
+         * we either added an identifying flag or re-calculated whether should
+         * share (won't be able to use stub_pc equality anymore if can be NULL)
+         */
+        ASSERT(dl->stub_pc == EXIT_STUB_PC(dcontext, f, l));
+        ASSERT(dl->stub_pc != NULL);
+        if (is_cbr_of_cbr_fallthrough(l) && !INTERNAL_OPTION(cbr_single_stub)) {
+            /* we allocated a pair */
+            special_heap_cfree(SEPARATE_STUB_HEAP(f->flags), dl->stub_pc, 2);
+        } else {
+            special_heap_free(SEPARATE_STUB_HEAP(f->flags), dl->stub_pc);
+        }
+        dl->stub_pc = NULL;
+    }
+    DOSTATS({
+        size_t alloc_size = SEPARATE_STUB_ALLOC_SIZE(f->flags);
+        if (TEST(FRAG_SHARED, f->flags)) {
+            if (TEST(FRAG_IS_TRACE, f->flags)) {
+                STATS_ADD(separate_shared_trace_direct_stubs, -(int)alloc_size);
+            } else {
+                STATS_ADD(separate_shared_bb_direct_stubs, -(int)alloc_size);
+            }
+        } else if (TEST(FRAG_IS_TRACE, f->flags))
+            STATS_SUB(separate_trace_direct_stubs, alloc_size);
+        else
+            STATS_SUB(separate_bb_direct_stubs, alloc_size);
+    });
+}
+
+/* if this linkstub shares the stub with the next linkstub, returns the
+ * next linkstub; else returns NULL
+ */
+linkstub_t *
+linkstub_shares_next_stub(dcontext_t *dcontext, fragment_t *f, linkstub_t *l)
+{
+    linkstub_t *nxt;
+    if (!INTERNAL_OPTION(cbr_single_stub))
+        return NULL;
+    nxt = LINKSTUB_NEXT_EXIT(l);
+    if (nxt != NULL && 
+        /* avoid stub computation for indirect, which fails if fcache is freed */
+        LINKSTUB_DIRECT(nxt->flags) && LINKSTUB_DIRECT(l->flags) &&
+        EXIT_STUB_PC(dcontext, f, nxt) == EXIT_STUB_PC(dcontext, f, l)) {
+        ASSERT(LINKSTUB_FINAL(nxt));
+        return nxt;
+    } else
+        return NULL;
+}
+
+void
+linkstub_free_exitstubs(dcontext_t *dcontext, fragment_t *f)
+{
+    linkstub_t *l;
+    for (l = FRAGMENT_EXIT_STUBS(f); l != NULL; l = LINKSTUB_NEXT_EXIT(l)) {
+        if (TEST(LINK_SEPARATE_STUB, l->flags) &&
+            EXIT_STUB_PC(dcontext, f, l) != NULL) {
+            linkstub_t *nxt = linkstub_shares_next_stub(dcontext, f, l);
+            if (nxt != NULL && !LINKSTUB_CBR_FALLTHROUGH(nxt->flags)) {
+                /* next linkstub shares our stub, so clear his now to avoid
+                 * double free
+                 */
+                direct_linkstub_t *dl = (direct_linkstub_t *) nxt;
+                dl->stub_pc = NULL;
+            }
+            separate_stub_free(dcontext, f, l, true);
+        }
+    }
+}
+
+void
+linkstubs_shift(dcontext_t *dcontext, fragment_t *f, ssize_t shift)
+{
+    linkstub_t *l;
+    for (l = FRAGMENT_EXIT_STUBS(f); l != NULL; l = LINKSTUB_NEXT_EXIT(l)) {
+        /* don't need to shift separated b/c shift_ctis_in_fragment will detect
+         * as an out-of-cache target and shift for us.
+         * l->stub_pc does not change since it's an absolute pc pointing outside
+         * of the resized cache.
+         * we also don't need to shift indirect stubs as they do not store
+         * an absolute pointer to their stub pc.
+         */
+        if (!TEST(LINK_SEPARATE_STUB, l->flags) && LINKSTUB_NORMAL_DIRECT(l->flags)) {
+            direct_linkstub_t *dl = (direct_linkstub_t *) l;
+            dl->stub_pc += shift;
+        } /* else, no change */
+    }
+}
+
+/* returns true if the exit l can be linked to the fragment f 
+ * if mark_new_trace_head is false, this routine does not change any state.
+ */
+bool
+is_linkable(dcontext_t *dcontext, fragment_t *from_f, linkstub_t *from_l, fragment_t *to_f,
+            bool have_link_lock, bool mark_new_trace_head)
+{
+    /* monitor_is_linkable is what marks trace heads, so must
+     * call it no matter the result
+     */
+    if (!monitor_is_linkable(dcontext, from_f, from_l, to_f,
+                             have_link_lock, mark_new_trace_head))
+        return false;
+    /* cannot link between shared and private caches 
+     * N.B.: we assume this in other places, like our use of fragment_lookup_same_sharing()
+     * for linking, so if we change this we need to change more than this routine here
+     */
+    if ((from_f->flags & FRAG_SHARED) != (to_f->flags & FRAG_SHARED))
+        return false;
+#ifdef DGC_DIAGNOSTICS
+    /* restrict linking so we can study entry/exit behavior */
+    if ((from_f->flags & FRAG_DYNGEN) != (to_f->flags & FRAG_DYNGEN))
+        return false;
+#endif
+    /* do not link exit from non-ignorable syscall ending a frag */
+    if (TESTANY(LINK_NI_SYSCALL_ALL, from_l->flags))
+        return false;
+#ifdef WINDOWS
+    if (TEST(LINK_CALLBACK_RETURN, from_l->flags))
+        return false;
+#endif
+    /* never link a selfmod exit branch */
+    if (TEST(LINK_SELFMOD_EXIT, from_l->flags))
+        return false;
+    /* don't link from a non-outgoing-linked fragment, or to a
+     * non-incoming-linked fragment, except for self-loops
+     */
+    if ((!TEST(FRAG_LINKED_OUTGOING, from_f->flags) ||
+         !TEST(FRAG_LINKED_INCOMING, to_f->flags)) &&
+        from_f != to_f)
+        return false;
+    /* rarely set so we test it last */
+    if (INTERNAL_OPTION(nolink))
+        return false;
+    return true;
+}
+
+/* links the branch at EXIT_CTI_PC(f, l) to jump directly to the entry point of
+ * fragment targetf
+ * Assumes that is_linkable has already been called!
+ * does not modify targetf's incoming branch list (because that should be done
+ * once while linking and unlinking may happen multiple times)
+ */
+static void
+link_branch(dcontext_t *dcontext, fragment_t *f, linkstub_t *l, fragment_t *targetf,
+            bool hot_patch)
+{
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    /* ASSUMPTION: always unlink before linking to somewhere else, so nop if linked */
+    if (INTERNAL_OPTION(nolink) || TEST(LINK_LINKED, l->flags))
+        return;
+    if (LINKSTUB_DIRECT(l->flags)) {
+        LOG(THREAD, LOG_LINKS, 4, "    linking F%d("PFX")."PFX" -> F%d("PFX")="PFX"\n", 
+            f->id, f->tag, EXIT_CTI_PC(f, l),
+            targetf->id, targetf->tag, FCACHE_ENTRY_PC(targetf));
+#ifdef TRACE_HEAD_CACHE_INCR
+        {
+            common_direct_linkstub_t *cdl = (common_direct_linkstub_t *) l;
+            if (TEST(FRAG_IS_TRACE_HEAD, targetf->flags))
+                cdl->target_fragment = targetf;
+            else
+                cdl->target_fragment = NULL;
+        }
+#endif
+        if (LINKSTUB_COARSE_PROXY(l->flags)) {
+            link_entrance_stub(dcontext, EXIT_STUB_PC(dcontext, f, l),
+                               FCACHE_ENTRY_PC(targetf), HOT_PATCHABLE, NULL);
+        } else {
+            bool do_not_need_stub =
+                link_direct_exit(dcontext, f, l, targetf, hot_patch) &&
+                TEST(LINK_SEPARATE_STUB, l->flags) &&
+                ((DYNAMO_OPTION(free_private_stubs) &&
+                  !TEST(FRAG_SHARED, f->flags)) ||
+                 (DYNAMO_OPTION(unsafe_free_shared_stubs) &&
+                  TEST(FRAG_SHARED, f->flags)));
+            ASSERT(!TEST(FRAG_FAKE, f->flags) && !LINKSTUB_FAKE(l));
+            if (do_not_need_stub)
+                separate_stub_free(dcontext, f, l, false);
+        }
+    } else if (LINKSTUB_INDIRECT(l->flags)) {
+        if (INTERNAL_OPTION(link_ibl))
+            link_indirect_exit(dcontext, f, l, hot_patch);
+    } else
+        ASSERT_NOT_REACHED();
+    
+    l->flags |= LINK_LINKED;
+}
+
+/* Unlinks one linked branch.  Returns false if the linkstub_t should be removed
+ * from the target's incoming list; else returns true.
+ */
+static bool
+unlink_branch(dcontext_t *dcontext, fragment_t *f, linkstub_t *l)
+{
+    bool keep = true;
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    if ((l->flags & LINK_LINKED) == 0)
+        return keep;
+    if (LINKSTUB_DIRECT(l->flags)) {
+        LOG(THREAD, LOG_LINKS, 4, "    unlinking branch F%d."PFX"\n",
+            f->id, EXIT_CTI_PC(f, l));
+        if (LINKSTUB_COARSE_PROXY(l->flags)) {
+            uint flags = 0;
+            cache_pc stub = EXIT_STUB_PC(dcontext, f, l);
+            /* When we create a trace from a coarse head, we point the head's
+             * entrance stub at the trace.  If we later remove the trace we need
+             * to re-instate the link to the head (well, to the head inc
+             * routine).
+             */
+            if (coarse_is_trace_head_in_own_unit(dcontext, f->tag, stub, NULL,
+                                                 false, NULL))
+                flags = FRAG_IS_TRACE_HEAD;
+            unlink_entrance_stub(dcontext, stub, flags, NULL);
+            /* The caller is now supposed to remove the incoming entry and
+             * free the heap space for this proxy linkstub_t
+             */
+            keep = false;
+        } else {
+            ASSERT(!LINKSTUB_FAKE(l));
+            /* stub may already exist for TRACE_HEAD_CACHE_INCR,
+             * linkcount profiling, etc. */
+            if (EXIT_STUB_PC(dcontext, f, l) == NULL &&
+                TEST(LINK_SEPARATE_STUB, l->flags))
+                separate_stub_create(dcontext, f, l);
+            unlink_direct_exit(dcontext, f, l);
+        }
+    } else if (LINKSTUB_INDIRECT(l->flags)) {
+        if (INTERNAL_OPTION(link_ibl))
+            unlink_indirect_exit(dcontext, f, l);
+    } else
+        ASSERT_NOT_REACHED();
+
+    l->flags &= ~LINK_LINKED;
+    return keep;
+}
+
+static inline bool
+incoming_direct_linkstubs_match(common_direct_linkstub_t *dl1,
+                                common_direct_linkstub_t *dl2)
+{
+    return ((dl1 == dl2 && !LINKSTUB_FAKE(&dl1->l) && !LINKSTUB_FAKE(&dl2->l)) ||
+            /* For coarse-grain we must match by value */
+            (LINKSTUB_FAKE(&dl1->l) && LINKSTUB_FAKE(&dl2->l) &&
+             LINKSTUB_NORMAL_DIRECT(dl1->l.flags) &&
+             LINKSTUB_NORMAL_DIRECT(dl2->l.flags) &&
+             ((direct_linkstub_t *)dl1)->stub_pc == ((direct_linkstub_t *)dl2)->stub_pc));
+}
+
+/* Returns the linkstub_t * for the link from f's exit l if it exists in
+ * targetf's incoming list.  N.B.: if target is coarse, its unit lock
+ * is released prior to returning the pointer!
+ */
+static linkstub_t *
+incoming_find_link(dcontext_t *dcontext, fragment_t *f, linkstub_t *l,
+                   fragment_t *targetf)
+{
+    common_direct_linkstub_t *s;
+    common_direct_linkstub_t **inlist = FRAG_INCOMING_ADDR(targetf);
+    common_direct_linkstub_t *dl = (common_direct_linkstub_t *) l;
+    ASSERT(LINKSTUB_DIRECT(l->flags));
+    ASSERT(!LINKSTUB_FAKE(l) ||
+           (LINKSTUB_COARSE_PROXY(l->flags) &&
+            TEST(FRAG_COARSE_GRAIN, f->flags) && LINKSTUB_NORMAL_DIRECT(l->flags)));
+    if (TEST(FRAG_COARSE_GRAIN, targetf->flags)) {
+        coarse_incoming_t *e;
+        coarse_info_t *info = get_fragment_coarse_info(targetf);
+        ASSERT(info != NULL);
+        mutex_lock(&info->incoming_lock);
+        for (e = info->incoming; e != NULL; e = e->next) {
+            if (!e->coarse) {
+                linkstub_t *ls;
+                for (ls = e->in.fine_l; ls != NULL; ls = LINKSTUB_NEXT_INCOMING(ls)) {
+                    if (incoming_direct_linkstubs_match(((common_direct_linkstub_t *)ls),
+                                                        dl)) {
+                        mutex_unlock(&info->incoming_lock);
+                        return ls;
+                    }
+                }
+            }
+        }
+        mutex_unlock(&info->incoming_lock);
+    } else {
+        for (s = *inlist; s != NULL; s = (common_direct_linkstub_t *) s->next_incoming) {
+            ASSERT(LINKSTUB_DIRECT(s->l.flags));
+            if (incoming_direct_linkstubs_match(s, dl))
+                return (linkstub_t *)s;
+        }
+    }
+    return NULL;
+}
+
+/* Returns true if the link from f's exit l exists in targetf's incoming list */
+static bool
+incoming_link_exists(dcontext_t *dcontext, fragment_t *f, linkstub_t *l,
+                     fragment_t *targetf)
+{
+    return incoming_find_link(dcontext, f, l, targetf) != NULL;
+}
+
+/* Removes the link from l to targetf from the incoming table 
+ * N.B.: may end up deleting targetf!
+ * If l is a fake linkstub_t, then f must be coarse-grain, and this
+ * routine searches targetf's incoming links for a match (since the
+ * exact stored linkstub_t != l).
+ */
+static void
+incoming_remove_link_nosearch(dcontext_t *dcontext, fragment_t *f, linkstub_t *l,
+                              fragment_t *targetf, linkstub_t *prevl,
+                              common_direct_linkstub_t **inlist)
+{
+    common_direct_linkstub_t *dl = (common_direct_linkstub_t *) l;
+    common_direct_linkstub_t *dprev = (common_direct_linkstub_t *) prevl;
+    ASSERT(LINKSTUB_DIRECT(l->flags));
+    ASSERT(prevl == NULL || LINKSTUB_DIRECT(prevl->flags));
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    ASSERT(!LINKSTUB_FAKE(l) ||
+           (LINKSTUB_COARSE_PROXY(l->flags) &&
+            TEST(FRAG_COARSE_GRAIN, f->flags) && LINKSTUB_NORMAL_DIRECT(l->flags)));
+    /* no links across caches so only checking f's sharedness is enough */
+    ASSERT(!NEED_SHARED_LOCK(f->flags) ||
+           self_owns_recursive_lock(&change_linking_lock));
+
+    if (dprev != NULL)
+        dprev->next_incoming = dl->next_incoming;
+    else {
+        /* if no incoming links left, and targetf is future, we could
+         * delete it here, but adaptive wset and trace head
+         * counters wanting persistent info means we want to always
+         * have either the future or the real thing -- unless
+         * it was NEVER executed: then could remove it, which we do for
+         * shared cache temporary private futures (see below)
+         * FIXME: do this for other futures as well?
+         */
+        if (dl->next_incoming == NULL) {
+            if (TESTALL(FRAG_TEMP_PRIVATE | FRAG_IS_FUTURE, targetf->flags) &&
+                !TEST(FRAG_WAS_DELETED, targetf->flags)) {
+                /* this is a future created only as an outgoing link of a private
+                 * bb created solely for trace creation.
+                 * since it never had a real fragment that was executed, we can
+                 * toss it, and in fact if we don't we waste a bunch of memory.
+                 */
+                LOG(THREAD, LOG_LINKS, 3,
+                    "incoming_remove_link: temp future "PFX" has no incoming, removing\n",
+                    targetf->tag);
+                ASSERT(!LINKSTUB_FAKE(l));
+                DODEBUG({ ((future_fragment_t *)targetf)->incoming_stubs = NULL; });
+                fragment_delete_future(dcontext, (future_fragment_t *) targetf);
+                /* WARNING: do not reference targetf after this */
+                STATS_INC(num_trace_private_fut_del);
+                return;
+            }
+        } 
+        *inlist = (common_direct_linkstub_t *) dl->next_incoming;
+    }
+    dl->next_incoming = NULL;
+    if (LINKSTUB_COARSE_PROXY(l->flags)) {
+        /* We don't have any place to keep l so we free now.  We'll
+         * re-alloc if we lazily re-link.
+         */
+        LOG(THREAD, LOG_LINKS, 4,
+            "freeing proxy incoming "PFX" from coarse "PFX" to fine tag "PFX"\n",
+            l, EXIT_STUB_PC(dcontext, f, l), EXIT_TARGET_TAG(dcontext, f, l));
+        NONPERSISTENT_HEAP_TYPE_FREE(GLOBAL_DCONTEXT, l, direct_linkstub_t,
+                                     ACCT_COARSE_LINK);
+    } else
+        ASSERT(!LINKSTUB_FAKE(l));
+}
+
+static bool
+incoming_remove_link_search(dcontext_t *dcontext, fragment_t *f, linkstub_t *l,
+                            fragment_t *targetf, common_direct_linkstub_t **inlist)
+{
+    common_direct_linkstub_t *s, *prevs, *dl;
+    dl = (common_direct_linkstub_t *) l;
+    ASSERT(LINKSTUB_DIRECT(l->flags));
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    ASSERT(!LINKSTUB_FAKE(l) ||
+           (LINKSTUB_COARSE_PROXY(l->flags) &&
+            TEST(FRAG_COARSE_GRAIN, f->flags) && LINKSTUB_NORMAL_DIRECT(l->flags)));
+    for (s = *inlist, prevs = NULL; s;
+         prevs = s, s = (common_direct_linkstub_t *) s->next_incoming) {
+        ASSERT(LINKSTUB_DIRECT(s->l.flags));
+        if (incoming_direct_linkstubs_match(s, dl)) {
+            /* We must remove s and NOT the passed-in l as coarse_remove_outgoing()
+             * passes in a new proxy that we use only to match and find the
+             * entry in the list to remove.
+             */
+            incoming_remove_link_nosearch(dcontext, f, (linkstub_t *)s,
+                                          targetf, (linkstub_t *)prevs, inlist);
+            return true;
+        }
+    }
+    return false;
+}
+
+/* Removes the link from l to targetf from the incoming table 
+ * N.B.: may end up deleting targetf!
+ * If l is a fake linkstub_t, then f must be coarse-grain, and this
+ * routine searches targetf's incoming links for a match (since the
+ * exact stored linkstub_t != l).
+ */
+static void
+incoming_remove_link(dcontext_t *dcontext, fragment_t *f, linkstub_t *l,
+                     fragment_t *targetf)
+{
+    /* no links across caches so only checking f's sharedness is enough */
+    ASSERT(!NEED_SHARED_LOCK(f->flags) ||
+           self_owns_recursive_lock(&change_linking_lock));
+    ASSERT(!LINKSTUB_COARSE_PROXY(l->flags) || TEST(FRAG_COARSE_GRAIN, f->flags));
+    ASSERT(!TEST(FRAG_COARSE_GRAIN, f->flags) ||
+           !TEST(FRAG_COARSE_GRAIN, targetf->flags));
+    if (TEST(FRAG_COARSE_GRAIN, targetf->flags)) {
+        coarse_remove_incoming(dcontext, f, l, targetf);
+    } else {
+        if (incoming_remove_link_search(dcontext, f, l, targetf,
+                                        FRAG_INCOMING_ADDR(targetf)))
+            return;
+        DODEBUG({
+            ASSERT(targetf != NULL && targetf->tag == EXIT_TARGET_TAG(dcontext, f, l));
+            LOG(THREAD, LOG_LINKS, 1, 
+                "incoming_remove_link: no link from F%d("PFX")."PFX" -> "
+                "F%d("PFX")\n",
+                f->id, f->tag, EXIT_CTI_PC(f, l), 
+                targetf->id, EXIT_TARGET_TAG(dcontext, f, l));
+        });
+        ASSERT_NOT_REACHED();
+    }
+}
+ 
+/* Adds the link from l to targetf to the list of incoming links
+ * for targetf.
+ */
+static inline void
+add_to_incoming_list(dcontext_t *dcontext, fragment_t *f, linkstub_t *l,
+                     fragment_t *targetf, bool linked)
+{
+    ASSERT(LINKSTUB_DIRECT(l->flags));
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    ASSERT(!incoming_link_exists(dcontext, f, l, targetf));
+    /* no inter-cache incoming entries */
+    ASSERT((f->flags & FRAG_SHARED) == (targetf->flags & FRAG_SHARED));
+    if (TEST(FRAG_COARSE_GRAIN, targetf->flags)) {
+        coarse_info_t *info = get_fragment_coarse_info(targetf);
+        prepend_new_coarse_incoming(info, NULL, l);
+    } else {
+        common_direct_linkstub_t **inlist =
+            (common_direct_linkstub_t **) FRAG_INCOMING_ADDR(targetf);
+        common_direct_linkstub_t *dl = (common_direct_linkstub_t *) l;
+        /* ensure not added twice b/c future not unlinked, etc. */
+        ASSERT(*inlist != dl);
+        ASSERT(!LINKSTUB_FAKE(l) || LINKSTUB_COARSE_PROXY(l->flags));
+        ASSERT(!is_empty_fragment(linkstub_fragment(dcontext, l)));
+        dl->next_incoming = (linkstub_t *) *inlist;
+        *inlist = dl;
+    }
+}
+
+/* Private fragment outgoing links require extra processing
+ * First, a private fragment link produces both shared (for trace head
+ * marking) and private (for incoming) futures -- we ensure the shared
+ * exists here.
+ * Second, we need to mark shared future frags as secondary trace heads
+ * NOW since we won't put private traces on shared incoming lists.
+ * We can do primary trace heads now, too, not a problem.
+ */
+static void
+add_private_check_shared(dcontext_t *dcontext, fragment_t *f, linkstub_t *l)
+{
+    fragment_t *targetf;
+    app_pc target_tag = EXIT_TARGET_TAG(dcontext, f, l);
+    ASSERT(LINKSTUB_DIRECT(l->flags));
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    if (!SHARED_FRAGMENTS_ENABLED())
+        return;
+    ASSERT(!TEST(FRAG_SHARED, f->flags));
+    ASSERT(!SHARED_FRAGMENTS_ENABLED() || dynamo_exited ||
+           self_owns_recursive_lock(&change_linking_lock));
+    targetf = fragment_link_lookup_same_sharing(dcontext, target_tag, l, FRAG_SHARED);
+    if (targetf == NULL)
+        targetf = (fragment_t *) fragment_lookup_future(dcontext, target_tag);
+    if (targetf == NULL) {
+        targetf = (fragment_t *)
+            fragment_create_and_add_future(dcontext, target_tag, FRAG_SHARED);
+    }
+    
+    if (!TEST(FRAG_IS_TRACE_HEAD, targetf->flags)) {
+        uint th = should_be_trace_head(dcontext, f, l, target_tag, targetf->flags,
+                                       true/* have linking lock*/);
+        if (TEST(TRACE_HEAD_YES, th)) {
+            if (TEST(FRAG_IS_FUTURE, targetf->flags)) {
+                LOG(THREAD, LOG_LINKS, 4,
+                    "    marking future ("PFX") as trace head\n", 
+                    target_tag);
+                targetf->flags |= FRAG_IS_TRACE_HEAD;
+            } else {
+                mark_trace_head(dcontext, targetf, f, l);
+            }
+            ASSERT(!TEST(TRACE_HEAD_OBTAINED_LOCK, th));
+        }
+    }
+}
+
+/* Adds the link l to the list of incoming future links
+ * for l's target.
+ */
+static void
+add_future_incoming(dcontext_t *dcontext, fragment_t *f, linkstub_t *l)
+{
+    future_fragment_t *targetf;
+    app_pc target_tag = EXIT_TARGET_TAG(dcontext, f, l);
+    ASSERT(LINKSTUB_DIRECT(l->flags));
+    ASSERT(!SHARED_FRAGMENTS_ENABLED() || !dynamo_exited ||
+           self_owns_recursive_lock(&change_linking_lock));
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    if (!TEST(FRAG_SHARED, f->flags))
+        targetf = fragment_lookup_private_future(dcontext, target_tag);
+    else
+        targetf = fragment_lookup_future(dcontext, target_tag);
+    if (targetf == NULL) {
+        /* if private, lookup-and-add being atomic is not an issue;
+         * for shared, the change_linking_lock atomicizes for us
+         */
+        targetf = fragment_create_and_add_future(dcontext, target_tag,
+                                                 /* take temp flag if present, so we
+                                                  * know to remove this future later
+                                                  * if never used
+                                                  */
+                                                 (f->flags &
+                                                  (FRAG_SHARED | FRAG_TEMP_PRIVATE)));
+    }
+    add_to_incoming_list(dcontext, f, l, (fragment_t*)targetf, false);
+
+    if (!TEST(FRAG_SHARED, f->flags)) {
+        /* private fragments need to ensure a shared fragment/future exists,
+         * and need to perform secondary trace head marking on it here!
+         */
+        add_private_check_shared(dcontext, f, l);
+    }
+}
+
+/* Adds the link from l to targetf to the list of incoming links
+ * for targetf.
+ */
+static void
+add_incoming(dcontext_t *dcontext, fragment_t *f, linkstub_t *l,
+             fragment_t *targetf, bool linked)
+{
+    ASSERT(TEST(FRAG_SHARED, f->flags) == TEST(FRAG_SHARED, targetf->flags));
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    LOG(THREAD, LOG_LINKS, 4,
+        "    add incoming F%d("PFX")."PFX" -> F%d("PFX")\n",
+        f->id, f->tag, EXIT_CTI_PC(f, l), targetf->id, targetf->tag);
+    add_to_incoming_list(dcontext, f, l, targetf, linked);
+
+    if (!TEST(FRAG_SHARED, f->flags)) {
+        /* private fragments need to ensure a shared fragment/future exists,
+         * and need to perform secondary trace head marking on it here!
+         */
+        add_private_check_shared(dcontext, f, l);
+    }
+}
+
+/* fragment_t f is being removed */
+future_fragment_t *
+incoming_remove_fragment(dcontext_t *dcontext, fragment_t *f)
+{
+    future_fragment_t *future = NULL;
+    linkstub_t *l;
+
+    /* pendel-del frags use fragment_t.in_xlate differently: they should
+     * never call this routine once they're marked for deletion
+     */
+    ASSERT(!TEST(FRAG_WAS_DELETED, f->flags));
+
+    /* link data struct change in shared fragment must be synchronized
+     * no links across caches so only checking f's sharedness is enough
+     */
+    ASSERT(!NEED_SHARED_LOCK(f->flags) ||
+           self_owns_recursive_lock(&change_linking_lock));
+
+    /* if removing shared trace, move its links back to the shadowed shared trace head */
+    /* flags not preserved for coarse so we have to check all coarse bbs */
+    if (TESTANY(FRAG_TRACE_LINKS_SHIFTED | FRAG_COARSE_GRAIN, f->flags)) {
+        if (TEST(FRAG_IS_TRACE, f->flags)) {
+            fragment_t wrapper;
+            /* FIXME case 8600: provide single lookup routine here */
+            fragment_t *bb = fragment_lookup_bb(dcontext, f->tag);
+            if (bb == NULL ||
+                (bb->flags & FRAG_SHARED) != (f->flags & FRAG_SHARED)) {
+                /* Can't use lookup_fine_and_coarse since trace will shadow coarse */
+                bb = fragment_coarse_lookup_wrapper(dcontext, f->tag, &wrapper);
+                LOG(THREAD, LOG_LINKS, 4,
+                    "incoming_remove_fragment shared trace "PFX": %s coarse thead\n",
+                    f->tag, bb == NULL ? "did not find" : "found");
+            }
+            if (bb != NULL && (TESTANY(FRAG_TRACE_LINKS_SHIFTED | FRAG_COARSE_GRAIN,
+                                       bb->flags))) {
+                ASSERT(TEST(FRAG_IS_TRACE_HEAD, bb->flags) ||
+                       TEST(FRAG_COARSE_GRAIN, bb->flags));
+                /* FIXME: this will mark trace head as FRAG_LINKED_INCOMING --
+                 * but then same thing for a new bb marked as a trace head
+                 * before linking via its previous future, so not a new problem.
+                 * won't actually link incoming since !linkable.
+                 */
+                if (TEST(FRAG_COARSE_GRAIN, bb->flags)) {
+                    /* We assume the coarse-grain bb is a trace head -- our method
+                     * of lookup is unable to mark it so we mark it here
+                     */
+                    bb->flags |= FRAG_IS_TRACE_HEAD;
+                }
+                shift_links_to_new_fragment(dcontext, f, bb);
+                STATS_INC(links_shared_trace_to_head);
+                ASSERT(f->in_xlate.incoming_stubs == NULL);
+                return NULL;
+            }
+        } else if (TEST(FRAG_IS_TRACE_HEAD, f->flags)) {
+            fragment_t *trace = fragment_lookup_trace(dcontext, f->tag);
+            /* regardless of -remove_shared_trace_heads, a shared trace will
+             * at least briefly shadow and shift links from a shared trace head.
+             * FIXME: add a FRAG_LINKS_SHIFTED flag to know for sure?
+             */
+            if (trace != NULL &&
+                (trace->flags & FRAG_SHARED) == (f->flags & FRAG_SHARED)) {
+                /* nothing to do -- links already shifted, no future needed */
+                STATS_INC(shadowed_trace_head_deleted);
+                ASSERT(f->in_xlate.incoming_stubs == NULL);
+                return NULL;
+            }
+        }
+    }
+
+    /* remove outgoing links from others' incoming lists */
+    for (l = FRAGMENT_EXIT_STUBS(f); l; l = LINKSTUB_NEXT_EXIT(l)) {
+        if (LINKSTUB_DIRECT(l->flags)) {
+            /* must explicitly check for self -- may not be in table, if
+             * flushing due to munmap
+             */
+            fragment_t *targetf;
+            app_pc target_tag = EXIT_TARGET_TAG(dcontext, f, l);
+            if (target_tag == f->tag)
+                targetf = f;
+            else {              
+                /* only want fragments in same shared/private cache */
+                targetf = fragment_link_lookup_same_sharing(dcontext, target_tag,
+                                                            NULL, f->flags);
+                if (targetf == NULL) {
+                    /* don't worry, all routines can handle fragment_t* that is really
+                     * future_fragment_t*
+                     */
+                    /* make sure future is in proper shared/private table */
+                    if (!TEST(FRAG_SHARED, f->flags)) {
+                        targetf = (fragment_t *)
+                            fragment_lookup_private_future(dcontext, target_tag);
+                    } else {
+                        targetf = (fragment_t *)
+                            fragment_lookup_future(dcontext, target_tag);
+                    }
+                }
+            }
+            LOG(THREAD, LOG_LINKS, 4,
+                "    removed F%d("PFX")."PFX" -> ("PFX") from incoming list\n",
+                f->id, f->tag, EXIT_CTI_PC(f, l), target_tag);
+            ASSERT(targetf != NULL);
+            if (targetf != NULL) /* play it safe */
+                incoming_remove_link(dcontext, f, l, targetf);
+        }
+    }
+
+    if (TEST(FRAG_TEMP_PRIVATE, f->flags) && f->in_xlate.incoming_stubs == NULL) {
+        /* if there are incoming stubs, the private displaced a prior future, which
+         * does need to be replaced -- if not, we don't need a future
+         */
+        LOG(THREAD, LOG_LINKS, 4,
+            "    not bothering with future for temp private F%d("PFX")\n", f->id, f->tag);
+        STATS_INC(num_trace_private_fut_avoid);
+        return NULL;
+    }
+    /* add future fragment
+     * FIXME: optimization is to convert f to future, that requires
+     * coordinating with fragment_remove, fragment_delete, etc.
+     */
+    LOG(THREAD, LOG_LINKS, 4,
+        "    adding future fragment for deleted F%d("PFX")\n",
+        f->id, f->tag);
+    DODEBUG({
+        if (TEST(FRAG_SHARED, f->flags))
+            ASSERT(fragment_lookup_future(dcontext, f->tag) == NULL);
+        else
+            ASSERT(fragment_lookup_private_future(dcontext, f->tag) == NULL);
+    });
+    future = fragment_create_and_add_future(dcontext, f->tag, 
+                                /* make sure future is in proper shared/private table.
+                                 * Do not keep FRAG_TEMP_PRIVATE as getting here means
+                                 * there was a real future here before the private, so our
+                                 * future removal optimization does not apply.
+                                 */
+                                (FRAG_SHARED & f->flags) |
+                                FRAG_WAS_DELETED);
+
+    future->incoming_stubs = f->in_xlate.incoming_stubs;
+    DODEBUG({ f->in_xlate.incoming_stubs = NULL; });
+
+    return future;
+}
+
+#ifdef DEBUG
+static void inline
+debug_after_link_change(dcontext_t *dcontext, fragment_t *f, char *msg)
+{
+    DOLOG(5, LOG_LINKS, {
+        LOG(THREAD, LOG_LINKS, 5, "%s\n", msg);
+        disassemble_fragment(dcontext, f, stats->loglevel < 3);
+    });
+}
+#endif
+
+/* Link all incoming links from other fragments to f
+ */
+void
+link_fragment_incoming(dcontext_t *dcontext, fragment_t *f, bool new_fragment)
+{
+    linkstub_t *l;
+    LOG(THREAD, LOG_LINKS, 4, "  linking incoming links for F%d("PFX")\n",
+        f->id, f->tag);
+    /* ensure some higher-level lock is held if f is shared
+     * no links across caches so only checking f's sharedness is enough
+     */
+    ASSERT_OWN_MUTEX(NEED_SHARED_LOCK(f->flags) ||
+                     (new_fragment && SHARED_FRAGMENTS_ENABLED()), &change_linking_lock);
+    ASSERT(!TEST(FRAG_LINKED_INCOMING, f->flags));
+    f->flags |= FRAG_LINKED_INCOMING;
+
+    /* link incoming links */
+    for (l = f->in_xlate.incoming_stubs; l != NULL; l = LINKSTUB_NEXT_INCOMING(l)) {
+        bool local_trace_head = false;
+        fragment_t *in_f = linkstub_fragment(dcontext, l);
+        if (TEST(FRAG_COARSE_GRAIN, f->flags)) {
+            /* Case 8907: remove trace headness markings, as each fine source
+             * should only consider this a trace head considering its own
+             * path to it.
+             */
+            local_trace_head = TEST(FRAG_IS_TRACE_HEAD, f->flags);
+            f->flags &= ~FRAG_IS_TRACE_HEAD;
+        }
+        /* only direct branches are marked on targets' incoming */
+        ASSERT(LINKSTUB_DIRECT(l->flags));
+        if (is_linkable(dcontext, in_f, l, f,
+                        NEED_SHARED_LOCK(f->flags) ||
+                        (new_fragment && SHARED_FRAGMENTS_ENABLED()),
+                        true/*mark new trace heads*/)) {
+            /* unprotect on demand, caller will re-protect */
+            SELF_PROTECT_CACHE(dcontext, in_f, WRITABLE);
+            link_branch(dcontext, in_f, l, f, HOT_PATCHABLE);
+        } else {
+            LOG(THREAD, LOG_LINKS, 4, "    not linking F%d("PFX")."PFX" -> F%d("PFX")"
+                " is not linkable!\n", 
+                in_f->id, in_f->tag, EXIT_CTI_PC(in_f, l),
+                f->id, f->tag);
+        }
+        /* Restore trace headness, if present before and not set in is_linkable */
+        if (local_trace_head && !TEST(FRAG_IS_TRACE_HEAD, f->flags))
+            f->flags |= FRAG_IS_TRACE_HEAD;
+    }
+}
+
+/* Link outgoing links of f to other fragments in the fcache (and
+ * itself if it has self loops).  If new_fragment is true, all of f's
+ * outgoing links are recorded in the incoming link lists of their
+ * targets.
+ */
+void
+link_fragment_outgoing(dcontext_t *dcontext, fragment_t *f, bool new_fragment)
+{
+    linkstub_t *l;
+    LOG(THREAD, LOG_LINKS, 4, "  linking outgoing links for F%d("PFX")\n",
+        f->id, f->tag);
+    /* ensure some higher-level lock is held if f is shared
+     * no links across caches so only checking f's sharedness is enough
+     */
+    ASSERT_OWN_MUTEX(NEED_SHARED_LOCK(f->flags) ||
+                     (new_fragment && SHARED_FRAGMENTS_ENABLED()), &change_linking_lock);
+    ASSERT(!TEST(FRAG_LINKED_OUTGOING, f->flags));
+    f->flags |= FRAG_LINKED_OUTGOING;
+
+    /* new_fragment: already protected
+     * unprotect on demand, caller will re-protect
+     */
+    if (!new_fragment)
+        SELF_PROTECT_CACHE(dcontext, f, WRITABLE);
+
+    /* link outgoing exits */
+    for (l = FRAGMENT_EXIT_STUBS(f); l; l = LINKSTUB_NEXT_EXIT(l)) {
+        if (LINKSTUB_DIRECT(l->flags)) {
+            /* found a linkable direct branch */
+            app_pc target_tag = EXIT_TARGET_TAG(dcontext, f, l);
+            fragment_t *g;
+            /* f may be invisible, so explicitly check for self-loops */
+            if (target_tag == f->tag)
+                g = f;
+            else /* primarily interested in fragment of same sharing */
+                g = fragment_link_lookup_same_sharing(dcontext, target_tag, l, f->flags);
+            if (g != NULL) {
+                if (is_linkable(dcontext, f, l, g,
+                                NEED_SHARED_LOCK(f->flags) ||
+                                (new_fragment && SHARED_FRAGMENTS_ENABLED()),
+                                true/*mark new trace heads*/)) {
+                    link_branch(dcontext, f, l, g, HOT_PATCHABLE);
+                    if (new_fragment)
+                        add_incoming(dcontext, f, l, g, true);
+                } else {
+                    if (new_fragment)
+                        add_incoming(dcontext, f, l, g, false);
+                    LOG(THREAD, LOG_LINKS, 4, "    not linking F%d("PFX")."PFX" -> "
+                        "F%d("PFX" == "PFX")%s%s%s\n", 
+                        f->id, f->tag, EXIT_CTI_PC(f, l),
+                        g->id, g->tag, target_tag,
+                        ((g->flags & FRAG_IS_TRACE_HEAD)!=0)?
+                        " (trace head)":"",
+                        ((l->flags & LINK_LINKED) != 0)?" (linked)":"",
+                        ((l->flags & LINK_SELFMOD_EXIT) != 0)?" (selfmod)":"");
+                }
+            } else {
+                if (new_fragment)
+                    add_future_incoming(dcontext, f, l);
+                LOG(THREAD, LOG_LINKS, 4,
+                    "    future-linking F%d("PFX")."PFX" -> ("PFX")\n", 
+                    f->id, f->tag, EXIT_CTI_PC(f, l),
+                    target_tag);
+            }
+        } else {
+            ASSERT(LINKSTUB_INDIRECT(l->flags));
+            /* indirect branches: just let link_branch handle the
+             * exit stub target
+             */
+#ifdef DGC_DIAGNOSTICS
+            /* do not link outgoing indirect so we see where it's going to go */
+            if ((f->flags & FRAG_DYNGEN) == 0)
+#endif
+                link_branch(dcontext, f, l, NULL, HOT_PATCHABLE);
+        }
+    }
+
+#ifdef DEBUG
+    debug_after_link_change(dcontext, f, "Fragment after linking outgoing");
+#endif
+}
+
+/* Unlinks all incoming branches into fragment f
+ */
+void
+unlink_fragment_incoming(dcontext_t *dcontext, fragment_t *f)
+{
+    linkstub_t *l, *nextl, *prevl;
+    LOG(THREAD, LOG_LINKS, 4, "unlinking incoming to frag F%d("PFX")\n",
+        f->id, f->tag);
+    /* ensure some higher-level lock is held if f is shared
+     * no links across caches so only checking f's sharedness is enough
+     */
+    ASSERT(!NEED_SHARED_LOCK(f->flags) ||
+           self_owns_recursive_lock(&change_linking_lock));
+    /* allow for trace head to be unlinked in middle of being unlinked --
+     * see comments in mark_trace_head in monitor.c
+     */
+    ASSERT(TESTANY(FRAG_LINKED_INCOMING|FRAG_IS_TRACE_HEAD, f->flags));
+    /* unlink incoming branches */
+    ASSERT(!TEST(FRAG_COARSE_GRAIN, f->flags));
+    for (prevl = NULL, l = f->in_xlate.incoming_stubs; l != NULL; l = nextl) {
+        fragment_t *in_f = linkstub_fragment(dcontext, l);
+        bool keep = true;
+        nextl = LINKSTUB_NEXT_INCOMING(l);
+        /* not all are linked (e.g., to trace head) */
+        if (TEST(LINK_LINKED, l->flags)) {
+            /* unprotect on demand, caller will re-protect */
+            SELF_PROTECT_CACHE(dcontext, in_f, WRITABLE);
+            keep = unlink_branch(dcontext, in_f, l);
+        } else
+            ASSERT(!LINKSTUB_FAKE(l));
+        if (!keep) {
+            incoming_remove_link_nosearch(dcontext, in_f, l, f, prevl,
+                                          FRAG_INCOMING_ADDR(f));
+        } else
+            prevl = l;
+    }
+    f->flags &= ~FRAG_LINKED_INCOMING;
+}
+
+/* Unlinks all outgoing branches from f
+ */
+void
+unlink_fragment_outgoing(dcontext_t *dcontext, fragment_t *f)
+{
+    linkstub_t *l;
+    DEBUG_DECLARE(bool keep;)
+    LOG(THREAD, LOG_LINKS, 4, "unlinking outgoing from frag F%d("PFX")\n",
+        f->id, f->tag);
+    /* ensure some higher-level lock is held if f is shared
+     * no links across caches so only checking f's sharedness is enough
+     */
+    ASSERT(!NEED_SHARED_LOCK(f->flags) ||
+           self_owns_recursive_lock(&change_linking_lock));
+    ASSERT(TEST(FRAG_LINKED_OUTGOING, f->flags));
+    /* unprotect on demand, caller will re-protect */
+    SELF_PROTECT_CACHE(dcontext, f, WRITABLE);
+    /* unlink outgoing direct & indirect branches */
+    for (l = FRAGMENT_EXIT_STUBS(f); l; l = LINKSTUB_NEXT_EXIT(l)) {
+        if ((l->flags & LINK_LINKED) != 0) {
+            DEBUG_DECLARE(keep =)
+                unlink_branch(dcontext, f, l); /* works for fine and coarse targets */
+            ASSERT(keep);
+        }
+    }
+    f->flags &= ~FRAG_LINKED_OUTGOING;
+
+#ifdef DEBUG
+    debug_after_link_change(dcontext, f, "Fragment after unlinking outgoing");
+#endif
+}
+
+/* Performs proactive linking, and inserts all links into appropriate
+ * incoming links lists
+ * f may be visible or invisible
+ */
+void
+link_new_fragment(dcontext_t *dcontext, fragment_t *f)
+{
+    future_fragment_t *future;
+    if (TEST(FRAG_COARSE_GRAIN, f->flags)) {
+        link_new_coarse_grain_fragment(dcontext, f);
+        return;
+    }
+    LOG(THREAD, LOG_LINKS, 4, "linking new fragment F%d("PFX")\n", f->id, f->tag);
+    /* ensure some higher-level lock is held if f is shared
+     * no links across caches so only checking f's sharedness is enough
+     */
+    ASSERT(!NEED_SHARED_LOCK(f->flags) ||
+           self_owns_recursive_lock(&change_linking_lock));
+
+    /* transfer existing future incoming links to this fragment */
+    if (!TEST(FRAG_SHARED, f->flags))
+        future = fragment_lookup_private_future(dcontext, f->tag);
+    else
+        future = fragment_lookup_future(dcontext, f->tag);
+    if (future != NULL) {
+        uint futflags = future->flags;
+        LOG(THREAD, LOG_LINKS, 4,
+            "  transferring incoming links from existing future frag, flags=0x%08x\n",
+            futflags);
+        f->in_xlate.incoming_stubs = future->incoming_stubs;
+        DODEBUG({ future->incoming_stubs = NULL; });
+        /* also transfer any flags that were stored in future
+         * N.B.: these flags must not be anything that is required when creating
+         * a fragment, they may only be things like TRACE_HEAD if they are
+         * to work properly
+         */
+        /* we only expect certain flags on future fragments */
+        ASSERT_CURIOSITY((futflags & ~(FUTURE_FLAGS_ALLOWED)) == 0);
+        /* sharedness must match */
+        ASSERT(TEST(FRAG_SHARED, f->flags) == TEST(FRAG_SHARED, futflags));
+        f->flags |= (futflags & (FUTURE_FLAGS_TRANSFER));
+        /* make sure existing flags and flags from build are compatible */
+        /* trace head and frag cannot be trace head incompatible */
+        if ((f->flags & FRAG_CANNOT_BE_TRACE) != 0 &&
+            (f->flags & FRAG_IS_TRACE_HEAD) != 0) { 
+            f->flags &= ~FRAG_IS_TRACE_HEAD;
+            LOG(THREAD, LOG_MONITOR, 2,
+                "fragment marked as trace head before being built, but now "
+                "cannot be trace head, unmarking trace head : address "PFX"\n", f->tag);
+        }
+        if (TESTALL(FRAG_IS_TRACE|FRAG_IS_TRACE_HEAD, f->flags)) {
+            /* we put the trace head flag on futures to mark secondary
+             * shared trace heads from private traces -- but it can end up
+             * marking traces.  remove it in that case as it WILL mess up
+             * linking (case 7465).
+             */
+            ASSERT(SHARED_FRAGMENTS_ENABLED());
+            LOG(THREAD, LOG_MONITOR, 2,
+                "trace F%d("PFX") inheriting trace head from future: discarding\n",
+                f->id, f->tag);
+            f->flags &= ~FRAG_IS_TRACE_HEAD;
+        }
+        fragment_delete_future(dcontext, future);
+    }
+    DODEBUG({
+        if (TEST(FRAG_SHARED, f->flags))
+            ASSERT(fragment_lookup_future(dcontext, f->tag) == NULL);
+        else
+            ASSERT(fragment_lookup_private_future(dcontext, f->tag) == NULL);
+    });
+
+    /* link incoming branches first, so no conflicts w/ self-loops
+     * that were just linked being added to future unlinked list
+     */
+    link_fragment_incoming(dcontext, f, true/*new*/);
+    link_fragment_outgoing(dcontext, f, true/*new*/);
+}
+
+/* Changes all incoming links to old_f to point to new_f
+ * old_f and new_f must have the same tag
+ * Links up all new_f's outgoing links
+ * (regardless of whether old_f is linked or not)
+ * These changes are all atomic, so this routine can be run by another
+ * thread while the owning thread is in the code cache (but not while it
+ * is in dynamo code!)
+ */
+void
+shift_links_to_new_fragment(dcontext_t *dcontext,
+                            fragment_t *old_f, fragment_t *new_f)
+{
+    linkstub_t *l;
+    bool have_link_lock = 
+        (TEST(FRAG_SHARED, old_f->flags) || TEST(FRAG_SHARED, new_f->flags)) &&
+        !INTERNAL_OPTION(single_thread_in_DR);
+    cache_pc old_stub = NULL;
+    cache_pc old_body = NULL;
+    coarse_info_t *info = NULL;
+    DEBUG_DECLARE(bool keep;)
+    ASSERT(old_f->tag == new_f->tag);
+    /* usually f's flags are just copied to new, so don't assert
+     * that new_f's LINKED_ flags are 0, but we do assume it has
+     * never been linked!
+     */
+
+    LOG(THREAD, LOG_LINKS, 4, "shifting links from F%d("PFX") to F%d("PFX")\n",
+        old_f->id, old_f->tag, new_f->id, new_f->tag);
+    /* ensure some higher-level lock is held if f is shared
+     * no links across caches so only checking f's sharedness is enough
+     */
+    ASSERT(!have_link_lock || self_owns_recursive_lock(&change_linking_lock));
+
+    /* if the new fragment had the exact same sequence of exits as the old,
+     * we could walk in lockstep, calling incoming_table_change_linkstub,
+     * but the new fragment could have completely different exits, so we'd
+     * better remove all the old from others' incoming lists, and then add
+     * the new -- don't worry about synchronization, owning thread can
+     * only be in fcache, not in dynamo code
+     */
+    /* remove old outgoing links from others' incoming lists */
+    LOG(THREAD, LOG_LINKS, 4, "  removing outgoing links for F%d("PFX")\n",
+        old_f->id, old_f->tag);\
+    if (TEST(FRAG_COARSE_GRAIN, old_f->flags)) {
+        /* FIXME: we could implement full non-fake fragment_t recovery, and
+         * engineer the normal link paths to do the right thing for coarse
+         * fragments, to avoid all the coarse checks in this routine
+         */
+        info = get_fragment_coarse_info(old_f);
+        ASSERT(info != NULL);
+        fragment_coarse_lookup_in_unit(dcontext, info, old_f->tag, &old_stub, &old_body);
+        ASSERT(old_stub != NULL || info->frozen);
+        ASSERT(old_body == FCACHE_ENTRY_PC(old_f));
+
+        /* We should only call this when emitting a trace */
+        ASSERT(TESTALL(FRAG_IS_TRACE | FRAG_SHARED, new_f->flags));
+
+        /* Case 8627: Since we can't really remove the fragment, we may as well
+         * leave the links intact.  Besides, it's difficult to walk the outgoing
+         * links: the best way is to decode_fragment() and then walk the exit
+         * ctis (xref case 8571 on decoding up to linkstub_t level).  There are no
+         * uniqueness requirements on incoming entries so long as there aren't
+         * two from the same coarse unit, so it's fine for the trace to also be
+         * in target incoming lists.  Plus, this means we don't have to do
+         * anything if we later delete the trace.
+         */
+        ASSERT(!TEST(FRAG_COARSE_GRAIN, new_f->flags)); /* ensure distinct incomings */
+    } else {
+        for (l = FRAGMENT_EXIT_STUBS(old_f); l; l = LINKSTUB_NEXT_EXIT(l)) {
+            if (LINKSTUB_DIRECT(l->flags)) {
+                /* incoming links do not cross sharedness boundaries */
+                app_pc target_tag = EXIT_TARGET_TAG(dcontext, old_f, l);
+                fragment_t *targetf =
+                    fragment_link_lookup_same_sharing(dcontext, target_tag,
+                                                      l, old_f->flags);
+                if (targetf == NULL) {
+                    /* don't worry, all routines can handle fragment_t* that is really
+                     * future_fragment_t*
+                     */
+                    /* make sure future is in proper shared/private table */
+                    if (!TEST(FRAG_SHARED, old_f->flags)) {
+                        targetf = (fragment_t *)
+                            fragment_lookup_private_future(dcontext, target_tag);
+                    } else {
+                        targetf = (fragment_t *)
+                            fragment_lookup_future(dcontext, target_tag);
+                    }
+                }
+                ASSERT(targetf != NULL);
+                /* if targetf == old_f, must remove self-link, b/c it won't be
+                 * redirected below as it won't appear on incoming list (new
+                 * self-link will).
+                 *
+                 * to avoid dangling links to other fragments (that may be deleted
+                 * and leave the link there since not in incoming) we also unlink
+                 * everything else.
+                 */
+                if (TEST(LINK_LINKED, l->flags)) {
+                    DEBUG_DECLARE(keep =) unlink_branch(dcontext, old_f, l);
+                    ASSERT(keep);
+                }
+                LOG(THREAD, LOG_LINKS, 4,
+                    "    removed F%d("PFX")."PFX" -> ("PFX") from incoming lists\n",
+                    old_f->id, old_f->tag, EXIT_CTI_PC(old_f, l), target_tag);
+                incoming_remove_link(dcontext, old_f, l, targetf);
+            } else {
+                ASSERT(LINKSTUB_INDIRECT(l->flags));
+                if (TEST(LINK_LINKED, l->flags)) {
+                    DEBUG_DECLARE(keep =) unlink_branch(dcontext, old_f, l);
+                    ASSERT(keep);
+                }
+            }
+        }
+    }
+    old_f->flags &= ~FRAG_LINKED_OUTGOING;
+
+    /* We copy prior to link-outgoing as that can add to incoming for self-links */
+    ASSERT(new_f->in_xlate.incoming_stubs == NULL);
+    ASSERT(!TEST(FRAG_COARSE_GRAIN, old_f->flags) ||
+           old_f->in_xlate.incoming_stubs == NULL);
+    new_f->in_xlate.incoming_stubs = old_f->in_xlate.incoming_stubs;
+    old_f->in_xlate.incoming_stubs = NULL;
+    if (TEST(FRAG_COARSE_GRAIN, new_f->flags) &&
+        new_f->in_xlate.incoming_stubs != NULL) {
+        if (info == NULL)
+            info = get_fragment_coarse_info(new_f);
+        prepend_new_coarse_incoming(info, NULL, new_f->in_xlate.incoming_stubs);
+    }
+
+    /* N.B.: unlike linking a new fragment, the owning thread could be in
+     * the fcache, and the 1st link is NOT guaranteed to be atomic,
+     * so we have to link the outgoing links for the new fragment before
+     * we ever link anybody up to it
+     */
+    /* link outgoing exits
+     * linking an already-linked branch is a nop, but as an optimization,
+     * rather than unlinking we mark them as not linked and put the proper
+     * link in with one cache write rather than two
+     */
+    if (!TEST(FRAG_COARSE_GRAIN, new_f->flags)) {
+        if (TEST(FRAG_LINKED_OUTGOING, new_f->flags)) {
+            for (l = FRAGMENT_EXIT_STUBS(new_f); l; l = LINKSTUB_NEXT_EXIT(l)) {
+                if (LINKSTUB_DIRECT(l->flags) && TEST(LINK_LINKED, l->flags)) {
+                    l->flags &= ~LINK_LINKED; /* leave inconsistent briefly */
+                }
+            }
+        }
+        new_f->flags &= ~FRAG_LINKED_OUTGOING; /* avoid assertion failure */
+        link_fragment_outgoing(dcontext, new_f, true/*add incoming*/);
+    } else {
+        ASSERT(TESTALL(FRAG_IS_TRACE | FRAG_SHARED, old_f->flags));
+        /* We assume this is re-instating a coarse trace head upon deleting
+         * a shared trace -- and that we never did unlink the trace head's outgoing
+         */
+    }
+    ASSERT(TEST(FRAG_LINKED_OUTGOING, new_f->flags));
+
+    /* now shift incoming links from old fragment to new one */
+    LOG(THREAD, LOG_LINKS, 4, "  transferring incoming links from F%d to F%d\n",
+        old_f->id, new_f->id);
+    old_f->flags &= ~FRAG_LINKED_INCOMING;
+    LOG(THREAD, LOG_LINKS, 4, "  linking incoming links for F%d("PFX")\n",
+        new_f->id, new_f->tag);
+    if (TEST(FRAG_COARSE_GRAIN, old_f->flags)) {
+        coarse_incoming_t *e, *prev_e = NULL, *next_e;
+        bool remove_entry;
+        /* We don't yet support coarse to coarse shifts (see above for one reason) */
+        ASSERT(!TEST(FRAG_COARSE_GRAIN, new_f->flags));
+        /* Change the entrance stub to point to the trace, which redirects
+         * all incoming from inside the unit.
+         */
+        if (old_stub != NULL) {
+            ASSERT(!entrance_stub_linked(old_stub, info) ||
+                   entrance_stub_jmp_target(old_stub) == FCACHE_ENTRY_PC(old_f));
+            if (entrance_stub_linked(old_stub, info)) {
+                /* If was never marked as trace head we must mark now,
+                 * else we will lose track of the body pc!
+                 */
+                coarse_mark_trace_head(dcontext, old_f, info, old_stub,
+                                       FCACHE_ENTRY_PC(old_f));
+                STATS_INC(coarse_th_on_shift);
+            }
+            coarse_link_to_fine(dcontext, old_stub, old_f, new_f, true/*do link*/);
+        } else {
+            /* FIXME: patch the frozen trace head to redirect?
+             * Else once in the unit will not go to trace.
+             * But by case 8151 this is only a trace head for paths coming
+             * from outside, which will go to the trace, so it should be ok.
+             */
+            ASSERT(info->frozen);
+        }
+        new_f->flags |= FRAG_LINKED_INCOMING;
+
+        /* Now re-route incoming from outside the unit */
+        mutex_lock(&info->incoming_lock);
+        for (e = info->incoming; e != NULL; e = next_e) {
+            next_e = e->next;
+            remove_entry = false;
+            if (!e->coarse) {
+                app_pc tgt = NULL;
+                linkstub_t *next_l;
+                for (l = e->in.fine_l; l != NULL; l = next_l) {
+                    fragment_t *in_f = linkstub_fragment(dcontext, l);
+                    next_l = LINKSTUB_NEXT_INCOMING(l);
+                    ASSERT(!TEST(LINK_FAKE, l->flags));
+                    if (tgt == NULL) {
+                        tgt = EXIT_TARGET_TAG(dcontext, in_f, l);
+                    } else {
+                        /* Every fine incoming in a single coarse-list entry should
+                         * target the same tag
+                         */
+                        ASSERT(EXIT_TARGET_TAG(dcontext, in_f, l) == tgt);
+                    }
+                    if (tgt == old_f->tag) {
+                        /* unprotect on demand (caller will re-protect)
+                         * FIXME: perhaps that's true for other link routines,
+                         * but shift_links_to_new_fragment() is called from
+                         * places where we need to double-check
+                         */
+                        SELF_PROTECT_CACHE(dcontext, in_f, WRITABLE);
+                        DEBUG_DECLARE(keep =) unlink_branch(dcontext, in_f, l);
+                        ASSERT(keep);
+                        if (is_linkable(dcontext, in_f, l, new_f, have_link_lock,
+                                        true/*mark trace heads*/)) {
+                            link_branch(dcontext, in_f, l, new_f, HOT_PATCHABLE);
+                            add_incoming(dcontext, in_f, l, new_f, true/*linked*/);
+                        } else {
+                            LOG(THREAD, LOG_LINKS, 4,
+                                "    not linking F%d("PFX")."PFX" -> F%d("PFX")"
+                                " is not linkable!\n", 
+                                in_f->id, in_f->tag, EXIT_CTI_PC(in_f, l),
+                                new_f->id, new_f->tag);
+                            add_incoming(dcontext, in_f, l, new_f, false/*!linked*/);
+                        }
+                        remove_entry = true;
+                   }
+                }
+            } else if (entrance_stub_jmp_target(e->in.stub_pc) ==
+                       FCACHE_ENTRY_PC(old_f)) {
+                /* FIXME: we don't know the tag of the src (and cannot find it
+                 * (case 8565))!  We'll use old_f's tag to avoid triggering any
+                 * new trace head rules.  Presumably they would have already
+                 * been triggered unless they vary based on coarse or fine.
+                 */
+                fragment_t *src_f = fragment_coarse_link_wrapper(dcontext, old_f->tag,
+                                                               e->in.stub_pc);
+                set_fake_direct_linkstub(&temp_linkstub, old_f->tag, e->in.stub_pc);
+                LOG(THREAD, LOG_LINKS, 4,
+                    "    shifting coarse link "PFX" -> %s "PFX" to F%d("PFX")\n",
+                    e->in.stub_pc, info->module, old_f->tag, new_f->id, new_f->tag);
+                if (is_linkable(dcontext, src_f, (linkstub_t *) &temp_linkstub,
+                                new_f, have_link_lock, true/*mark trace heads*/)) {
+                    DODEBUG({
+                        /* avoid assert about being already linked */
+                        unlink_entrance_stub(dcontext, e->in.stub_pc, new_f->flags,
+                                             NULL);
+                    });
+                    coarse_link_to_fine(dcontext, e->in.stub_pc, src_f, new_f,
+                                        true/*do link*/);
+                }
+                remove_entry = true;
+            }
+            if (remove_entry) {
+                if (prev_e == NULL)
+                    info->incoming = e->next;
+                else
+                    prev_e->next = e->next;
+                LOG(THREAD, LOG_LINKS, 4,
+                    "freeing coarse_incoming_t "PFX"\n", e);
+                NONPERSISTENT_HEAP_TYPE_FREE(GLOBAL_DCONTEXT, e, coarse_incoming_t,
+                                             ACCT_COARSE_LINK);
+            } else
+                prev_e = e;
+        }
+        mutex_unlock(&info->incoming_lock);
+
+    } else if (TEST(FRAG_COARSE_GRAIN, new_f->flags)) {
+        /* Change the entrance stub to point to the trace head routine again
+         * (we only shift to coarse trace heads).
+         */
+        coarse_info_t *info = get_fragment_coarse_info(new_f);
+        cache_pc new_stub, new_body;
+        ASSERT(info != NULL);
+        fragment_coarse_lookup_in_unit(dcontext, info, old_f->tag, &new_stub, &new_body);
+        ASSERT(new_body == FCACHE_ENTRY_PC(new_f));
+        if (new_stub != NULL) {
+            unlink_entrance_stub(dcontext, new_stub, FRAG_IS_TRACE_HEAD, info);
+            ASSERT(coarse_is_trace_head_in_own_unit(dcontext, new_f->tag,
+                                                    new_stub, new_body, true, info));
+            /* If we ever support shifting to non-trace-heads we'll want to point the
+             * stub at the fragment and not at the head incr routine
+             */
+        } else
+            ASSERT(info->frozen);
+        /* We can re-use link_fragment_incoming, but be careful of any future
+         * changes that require splitting out the coarse-and-fine-shared part.
+         */
+        new_f->flags &= ~FRAG_LINKED_INCOMING; /* wrapper is marked as linked */
+        link_fragment_incoming(dcontext, new_f, true/*new*/);
+        ASSERT(TEST(FRAG_LINKED_INCOMING, new_f->flags));
+    } else {
+        new_f->flags |= FRAG_LINKED_INCOMING;
+        for (l = new_f->in_xlate.incoming_stubs; l != NULL;
+             l = LINKSTUB_NEXT_INCOMING(l)) {
+            fragment_t *in_f = linkstub_fragment(dcontext, l);
+            ASSERT(!is_empty_fragment(in_f));
+            if (is_linkable(dcontext, in_f, l, new_f, have_link_lock,
+                            true/*mark new trace heads*/)) {
+                /* used to check to make sure was linked but no reason to? */
+                /* already did self-links */
+                if (in_f != new_f) {
+                    if (TEST(LINK_LINKED, l->flags))
+                        l->flags &= ~LINK_LINKED; /* else, link_branch is a nop */
+                    link_branch(dcontext, in_f, l, new_f, HOT_PATCHABLE);
+                } else {
+                    LOG(THREAD, LOG_LINKS, 4, "    not linking F%d("PFX")."PFX" -> "
+                        "F%d("PFX" == "PFX") (self-link already linked)\n", 
+                        in_f->id, in_f->tag, EXIT_CTI_PC(in_f, l),
+                        new_f->id, new_f->tag);
+                }
+            } else if ((l->flags & LINK_LINKED) != 0) {
+                DEBUG_DECLARE(keep =) unlink_branch(dcontext, in_f, l);
+                ASSERT(keep);
+                LOG(THREAD, LOG_LINKS, 4,
+                    "    not linking F%d("PFX")."PFX" -> F%d("PFX")"
+                    "(src not outgoing-linked)\n", 
+                    in_f->id, in_f->tag, EXIT_CTI_PC(in_f, l),
+                    new_f->id, new_f->tag);
+            } else {
+                LOG(THREAD, LOG_LINKS, 4, "    not linking F%d("PFX")."PFX" -> "
+                    "F%d("PFX" == "PFX")\n", 
+                    in_f->id, in_f->tag, EXIT_CTI_PC(in_f, l),
+                    new_f->id, new_f->tag);
+            }
+        }
+    }
+
+    /* For the common case of a trace shadowing a trace head
+     * (happens w/ shared traces, and with CUSTOM_TRACES),
+     * ensure that when we delete the trace we shift back and
+     * when we delete the head we don't complain that we're
+     * missing links.
+     */
+    if (TEST(FRAG_IS_TRACE, new_f->flags) &&
+        TEST(FRAG_IS_TRACE_HEAD, old_f->flags)) {
+        ASSERT((!NEED_SHARED_LOCK(new_f->flags) &&
+                !NEED_SHARED_LOCK(old_f->flags)) ||
+               self_owns_recursive_lock(&change_linking_lock));
+        LOG(THREAD, LOG_LINKS, 4, "Marking old and new as FRAG_TRACE_LINKS_SHIFTED\n");
+        new_f->flags |= FRAG_TRACE_LINKS_SHIFTED;
+        old_f->flags |= FRAG_TRACE_LINKS_SHIFTED;
+    }
+
+    DOLOG(4, LOG_LINKS, {
+        LOG(THREAD, LOG_LINKS, 4, "Old fragment after shift:\n");
+        disassemble_fragment(dcontext, old_f, stats->loglevel < 4);
+        LOG(THREAD, LOG_LINKS, 4, "New fragment after shift:\n");
+        disassemble_fragment(dcontext, new_f, stats->loglevel < 4);
+    });
+}
+
+
+/***************************************************************************
+ * COARSE-GRAIN UNITS
+ */
+
+static vm_area_vector_t *coarse_stub_areas;
+
+static void
+coarse_stubs_init()
+{
+    VMVECTOR_ALLOC_VECTOR(coarse_stub_areas, GLOBAL_DCONTEXT,
+                          VECTOR_SHARED | VECTOR_NEVER_MERGE,
+                          coarse_stub_areas);
+}
+
+static void
+coarse_stubs_free()
+{
+    ASSERT(coarse_stub_areas != NULL);
+    /* should be empty from special_heap_exit(), from vm_area_coarse_units_reset_free() */
+    ASSERT(vmvector_empty(coarse_stub_areas));
+    vmvector_delete_vector(GLOBAL_DCONTEXT, coarse_stub_areas);
+}
+
+/* current prefix size is 37 bytes, so need 3 stub slots */
+static inline uint
+num_coarse_stubs_for_prefix(coarse_info_t *info)
+{
+    uint prefix_size = coarse_exit_prefix_size(COARSE_32_FLAG(info));
+    size_t stub_size = COARSE_STUB_ALLOC_SIZE(COARSE_32_FLAG(info));
+    ASSERT(CHECK_TRUNCATE_TYPE_uint(ALIGN_FORWARD(prefix_size, stub_size)));
+    return (uint) (ALIGN_FORWARD(prefix_size, stub_size) / stub_size);
+}
+
+uint
+coarse_stub_alignment(coarse_info_t *info)
+{
+    return (uint) COARSE_STUB_ALLOC_SIZE(COARSE_32_FLAG(info));
+}
+
+/* Pass in NULL for pc to have stubs created.
+ * Size must include room for prefixes as well as stubs.
+ */
+cache_pc
+coarse_stubs_create(coarse_info_t *info, cache_pc pc, size_t size)
+{
+    byte *end_pc;
+    ASSERT(coarse_stub_areas != NULL);
+    info->stubs = special_heap_pclookup_init((uint)
+                                             COARSE_STUB_ALLOC_SIZE(COARSE_32_FLAG(info)),
+                                             true /* must synch */, true /* +x */,
+                                             false /* not persistent */,
+                                             coarse_stub_areas,
+                                             info /* support pclookup w/ info retval */,
+                                             pc, size, pc != NULL/*full if pre-alloc*/);
+    /* Create the fcache_return_coarse prefix for this unit.
+     * We keep it here rather than at the top of the fcache unit because:
+     * 1) stubs are writable while fcache should be read-only, and we may want
+     *    to patch the prefix when persistent to point to the current fcache_return
+     * 2) we need to find the prefix given just a stub and no info on the
+     *    src body in fcache
+     * We have to make sure stub walks skip over the prefix.
+     */
+    if (pc != NULL) {
+        /* Header is separate, so we know we can start at the top */
+        info->fcache_return_prefix = pc;
+    } else {
+        info->fcache_return_prefix = (cache_pc)
+            special_heap_calloc(info->stubs, num_coarse_stubs_for_prefix(info));
+    }
+    end_pc = emit_coarse_exit_prefix(GLOBAL_DCONTEXT, info->fcache_return_prefix, info);
+    /* we have to align for pc != NULL;
+     * caller should be using calloc if pc==NULL but we align just in case
+     */
+    end_pc = (cache_pc) ALIGN_FORWARD(end_pc, coarse_stub_alignment(info));
+    ASSERT(pc == NULL || end_pc <= pc+size);
+    ASSERT(info->trace_head_return_prefix != NULL);
+    ASSERT(info->ibl_ret_prefix != NULL);
+    ASSERT(info->ibl_call_prefix != NULL);
+    ASSERT(info->ibl_jmp_prefix != NULL);
+    ASSERT(end_pc - info->fcache_return_prefix <=
+           (int)(COARSE_STUB_ALLOC_SIZE(COARSE_32_FLAG(info))*
+                 num_coarse_stubs_for_prefix(info)));
+    DODEBUG({
+        SET_TO_NOPS(end_pc, info->fcache_return_prefix +
+                    COARSE_STUB_ALLOC_SIZE(COARSE_32_FLAG(info))*
+                    num_coarse_stubs_for_prefix(info) - end_pc);
+    });
+    return end_pc;
+}
+
+/* Iteration support over coarse entrance stubs */
+
+typedef struct {
+    special_heap_iterator_t shi;
+    cache_pc start, end, pc;
+    coarse_info_t *info;
+} coarse_stubs_iterator_t;
+
+static void
+coarse_stubs_iterator_start(coarse_info_t *info, coarse_stubs_iterator_t *csi)
+{
+    special_heap_iterator_start(info->stubs, &csi->shi);
+    csi->info = info;
+    if (special_heap_iterator_hasnext(&csi->shi)) {
+        special_heap_iterator_next(&csi->shi, &csi->start, &csi->end);
+        /* skip the prefix kept at the top of the first unit */
+        ASSERT(csi->start == info->fcache_return_prefix);
+        /* coarse_stubs_iterator_next() will add 1 */
+        csi->pc = csi->start + COARSE_STUB_ALLOC_SIZE(COARSE_32_FLAG(info))*
+            (num_coarse_stubs_for_prefix(info)-1);
+    } else {
+        csi->start = NULL;
+        csi->end = NULL;
+        csi->pc = NULL;
+    }
+}
+
+static bool inline
+coarse_stubs_iterator_hasnext(coarse_stubs_iterator_t *csi)
+{
+    return (csi->pc < csi->end || special_heap_iterator_hasnext(&csi->shi));
+}
+
+static cache_pc inline
+coarse_stubs_iterator_next(coarse_stubs_iterator_t *csi)
+{
+    if (csi->pc == NULL)
+        return NULL;
+    ASSERT(csi->pc < csi->end);
+    csi->pc += COARSE_STUB_ALLOC_SIZE(COARSE_32_FLAG(csi->info));
+    if (csi->pc >= csi->end) {
+        if (special_heap_iterator_hasnext(&csi->shi)) {
+            special_heap_iterator_next(&csi->shi, &csi->start, &csi->end);
+            csi->pc = csi->start;
+        } else
+            return NULL;
+    }
+    /* N.B.: once we free entrance stubs we'll need to identify a
+     * freed pattern here.  For now we assume everything is
+     * a stub.
+     */
+    ASSERT(coarse_is_entrance_stub(csi->pc));
+    return csi->pc;
+}
+
+static void inline
+coarse_stubs_iterator_stop(coarse_stubs_iterator_t *csi)
+{
+    special_heap_iterator_stop(&csi->shi);
+}
+
+void
+coarse_stubs_delete(coarse_info_t *info)
+{
+    DEBUG_DECLARE(coarse_stubs_iterator_t csi;)
+    DEBUG_DECLARE(cache_pc pc;)
+    ASSERT_OWN_MUTEX(!dynamo_all_threads_synched && !info->is_local, &info->lock);
+    if (info->stubs == NULL) {
+        /* lazily initialized, so common to have empty units */
+        ASSERT(info->fcache_return_prefix == NULL);
+        return;
+    }
+#ifdef DEBUG
+    if (info->frozen) {
+        /* We allocated the stub unit ourselves so nothing to free here */
+    } else {
+        special_heap_cfree(info->stubs, info->fcache_return_prefix,
+                           num_coarse_stubs_for_prefix(info));
+        /* We can't free while using the iterator (lock issues) so we
+         * free all at once
+         */
+        coarse_stubs_iterator_start(info, &csi);
+        for (pc = coarse_stubs_iterator_next(&csi); pc != NULL;
+             pc = coarse_stubs_iterator_next(&csi)) {
+            special_heap_free(info->stubs, pc);
+        }
+        coarse_stubs_iterator_stop(&csi);
+    }
+#endif
+    LOG(THREAD_GET, LOG_LINKS|LOG_HEAP, 1,
+        "Coarse special heap exit %s\n", info->module);
+    special_heap_exit(info->stubs);
+    info->stubs = NULL;
+    info->fcache_return_prefix = NULL;
+    info->trace_head_return_prefix = NULL;
+    info->ibl_ret_prefix = NULL;
+    info->ibl_call_prefix = NULL;
+    info->ibl_jmp_prefix = NULL;
+}
+
+/* N.B.: once we start freeing entrance stubs, we need to fill the space,
+ * so that our stub walk can identify live stubs
+ */
+static cache_pc
+entrance_stub_create(dcontext_t *dcontext, coarse_info_t *info,
+                     fragment_t *f, linkstub_t *l)
+{
+    uint emit_sz;
+    cache_pc stub_pc;
+    DEBUG_DECLARE(size_t stub_size = COARSE_STUB_ALLOC_SIZE(COARSE_32_FLAG(info));)
+    ASSERT(DYNAMO_OPTION(coarse_units));
+    ASSERT(info != NULL && info->stubs != NULL);
+    ASSERT(LINKSTUB_DIRECT(l->flags));
+    ASSERT(TEST(LINK_SEPARATE_STUB, l->flags));
+    ASSERT(exit_stub_size(dcontext, EXIT_TARGET_TAG(dcontext, f, l), f->flags) <=
+           (ssize_t) stub_size);
+    /* We hot-patch our stubs and we assume that aligning them to 16 is enough */
+    ASSERT((cache_line_size % stub_size) == 0);
+    stub_pc = (cache_pc) special_heap_alloc(info->stubs);
+    ASSERT(ALIGNED(stub_pc, coarse_stub_alignment(info)));
+    emit_sz = insert_exit_stub(dcontext, f, l, stub_pc);
+    LOG(THREAD, LOG_LINKS, 4,
+        "created new entrance stub @"PFX" for "PFX" w/ source F%d("PFX")."PFX"\n",
+        stub_pc, EXIT_TARGET_TAG(dcontext, f, l), f->id, f->tag, FCACHE_ENTRY_PC(f));
+    ASSERT(emit_sz <= stub_size);
+    DODEBUG({
+        SET_TO_NOPS(stub_pc + emit_sz, stub_size - emit_sz);
+    });
+    STATS_ADD(separate_shared_bb_entrance_stubs, stub_size);
+    STATS_INC(num_entrance_stubs);
+    return stub_pc;
+}
+
+/* Sets flags for a fake linkstub_t for an incoming list entry for a coarse source */
+static void
+set_fake_direct_linkstub(direct_linkstub_t *proxy, app_pc target, cache_pc stub)
+{
+    /* ASSUMPTION: this combination is unique to coarse-grain proxy stubs.
+     * The LINKSTUB_COARSE_PROXY() macro tests these (except LINK_LINKED).
+     */
+    proxy->cdl.l.flags = LINK_FAKE | LINK_DIRECT | LINK_LINKED | LINK_SEPARATE_STUB;
+    proxy->cdl.l.cti_offset = 0;
+    proxy->target_tag = target;
+    proxy->stub_pc = stub;
+}
+
+#if defined(DEBUG) && defined(INTERNAL)
+static void
+print_coarse_incoming(dcontext_t *dcontext, coarse_info_t *info)
+{
+    coarse_incoming_t *e;
+    ASSERT_OWN_MUTEX(true, &info->incoming_lock);
+    LOG(THREAD, LOG_LINKS, 1, "coarse incoming entries for %s\n", info->module);
+    for (e = info->incoming; e != NULL; e = e->next) {
+        LOG(THREAD, LOG_LINKS, 1, "\t"PFX" %d ",
+            e, e->coarse);
+        if (e->coarse)
+            LOG(THREAD, LOG_LINKS, 1, PFX"\n", e->in.stub_pc);
+        else {
+            fragment_t *f = linkstub_fragment(dcontext, e->in.fine_l);
+            LOG(THREAD, LOG_LINKS, 1, "F%d("PFX")\n", f->id, f->tag);
+        }
+    }
+}
+#endif
+
+/* Must pass NULL for exactly one of coarse or fine */
+static coarse_incoming_t *
+prepend_new_coarse_incoming(coarse_info_t *info, cache_pc coarse, linkstub_t *fine)
+{
+    coarse_incoming_t *entry =
+        NONPERSISTENT_HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, coarse_incoming_t,
+                                      ACCT_COARSE_LINK);
+    /* entries are freed in coarse_remove_outgoing()/coarse_unit_unlink() */
+    if (fine == NULL) {
+        /* FIXME: rather than separate entries per stub pc, to save memory
+         * we could have a single one for the whole unit (and we'd search
+         * here to see if it already exists) and search when unlinking to
+         * find the individual stubs
+         */
+        ASSERT(coarse != NULL);
+        entry->coarse = true;
+        entry->in.stub_pc = coarse;
+        LOG(THREAD_GET, LOG_LINKS, 4,
+            "created new coarse_incoming_t "PFX" coarse from "PFX"\n",
+            entry, entry->in.stub_pc);
+    } else {
+        ASSERT(coarse == NULL);
+        entry->coarse = false;
+        /* We put the whole linkstub_t list as one entry */
+        entry->in.fine_l = fine;
+        LOG(THREAD_GET, LOG_LINKS, 4,
+            "created new coarse_incoming_t "PFX" fine from "PFX"\n",
+            entry, entry->in.fine_l);
+        DODEBUG({
+            linkstub_t *l;
+            for (l = fine; l != NULL; l = LINKSTUB_NEXT_INCOMING(l))
+                ASSERT(!TEST(LINK_FAKE, l->flags));
+        });
+    }
+    ASSERT(info != NULL);
+    mutex_lock(&info->incoming_lock);
+    entry->next = info->incoming;
+    info->incoming = entry;
+    DOLOG(5, LOG_LINKS, {
+        LOG(GLOBAL, LOG_LINKS, 4, "after adding new incoming "PFX":\n", entry);
+        print_coarse_incoming(GLOBAL_DCONTEXT, info);
+    });
+    mutex_unlock(&info->incoming_lock);
+    return entry;
+}
+
+/* Pass in coarse if you already know it; else, this routine will look it up */
+static fragment_t *
+fragment_coarse_link_wrapper(dcontext_t *dcontext, app_pc target_tag,
+                             cache_pc know_coarse)
+{
+    ASSERT(self_owns_recursive_lock(&change_linking_lock));
+    if (know_coarse == NULL) {
+        return fragment_coarse_lookup_wrapper(dcontext, target_tag, &temp_targetf);
+    } else {
+        fragment_coarse_wrapper(&temp_targetf, target_tag, know_coarse);
+        return &temp_targetf;
+    }
+    return NULL;
+}
+
+static fragment_t *
+fragment_link_lookup_same_sharing(dcontext_t *dcontext, app_pc tag,
+                                  linkstub_t *last_exit, uint flags)
+{
+    /* Assumption: if asking for private won't use temp_targetf.
+     * Else need to grab the lock when linking private fragments
+     * (in particular, temps for trace building)
+     */
+    ASSERT(!TEST(FRAG_SHARED, flags) || self_owns_recursive_lock(&change_linking_lock));
+    return fragment_lookup_fine_and_coarse_sharing(dcontext, tag,
+                                                   &temp_targetf, last_exit, flags);
+}
+
+static void
+coarse_link_to_fine(dcontext_t *dcontext, cache_pc src_stub, fragment_t *src_f,
+                    fragment_t *target_f, bool do_link/*else just add incoming*/)
+{
+    /* We may call this multiple times for multiple sources going through the
+     * same entrance stub
+     */
+    ASSERT(self_owns_recursive_lock(&change_linking_lock));
+    set_fake_direct_linkstub(&temp_linkstub, target_f->tag, src_stub);
+    if (incoming_link_exists(dcontext, src_f, (linkstub_t *)&temp_linkstub, target_f)) {
+        ASSERT(entrance_stub_jmp_target(src_stub) == FCACHE_ENTRY_PC(target_f));
+        LOG(THREAD, LOG_LINKS, 4,
+            "    already linked coarse "PFX"."PFX"->F%d("PFX")\n",
+            src_f->tag, FCACHE_ENTRY_PC(src_f), target_f->id, target_f->tag);
+    } else {
+        direct_linkstub_t *proxy =
+            NONPERSISTENT_HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, direct_linkstub_t,
+                                          ACCT_COARSE_LINK);
+        LOG(THREAD, LOG_LINKS, 4,
+            "created new proxy incoming "PFX" from coarse "PFX" to fine F%d\n",
+            proxy, src_stub, target_f->id);
+        LOG(THREAD, LOG_LINKS, 4,
+            "    linking coarse stub "PFX"->F%d("PFX")\n",
+            src_stub, target_f->id, target_f->tag);
+        /* Freed in incoming_remove_link() called from unlink_fragment_incoming()
+         * or from coarse_unit_unlink() calling coarse_remove_outgoing().
+         * Note that we do not unlink fine fragments on reset/exit (case 7697)
+         * so we can't rely solely on unlink_fragment_incoming() to free these
+         * for us.
+         */
+        set_fake_direct_linkstub(proxy, target_f->tag, src_stub);
+        add_incoming(dcontext, src_f, (linkstub_t *) proxy, target_f, true/*linked*/);
+        if (do_link) {
+            /* Should not be linked somewhere else */
+            ASSERT(!entrance_stub_linked(src_stub, NULL) ||
+                   entrance_stub_jmp_target(src_stub) == FCACHE_ENTRY_PC(target_f));
+            proxy->cdl.l.flags &= ~LINK_LINKED; /* so link_branch isn't a nop */
+            link_branch(dcontext, src_f, (linkstub_t *) proxy, target_f, HOT_PATCHABLE);
+        } else {
+            /* Already linked (we're freezing or shifting) */
+            ASSERT(entrance_stub_linked(src_stub, NULL) &&
+                   entrance_stub_jmp_target(src_stub) == FCACHE_ENTRY_PC(target_f));
+        }
+    }
+}
+
+/* Links up an entrance stub to its target, whether that is a local coarse,
+ * remote coarse, or remote fine fragment.
+ * Takes in f and l since for a new coarse fragment those already exist;
+ * other callers will have to fabricate.
+ * Returns true if the link was enacted.
+ */
+static bool
+coarse_link_direct(dcontext_t *dcontext, fragment_t *f, linkstub_t *l,
+                   coarse_info_t *src_info, cache_pc stub, app_pc target_tag,
+                   cache_pc local_tgt_in, bool new_stub)
+{
+    bool linked = false;
+    /* Targets are always body pcs.  stub is the stub pc we'll link through. */
+    cache_pc local_tgt = NULL;
+    cache_pc remote_tgt = NULL;
+    cache_pc coarse_tgt = NULL;
+    fragment_t *target_f = NULL;
+    ASSERT(self_owns_recursive_lock(&change_linking_lock));
+    ASSERT(entrance_stub_target_tag(stub) == target_tag);
+    /* Note that it is common for stub to already be linked (b/c we have
+     * entrance stubs shared by multiple sources), yet we still need to call
+     * is_linkable
+     */
+    DOSTATS({
+        if (entrance_stub_linked(stub, src_info))
+            STATS_INC(coarse_relinks);
+    });
+    /* Since we leave shadowed trace heads visible we must first look in the
+     * fine tables
+     */
+    ASSERT(TEST(FRAG_SHARED, f->flags));
+    target_f = fragment_lookup_same_sharing(dcontext, target_tag, FRAG_SHARED);
+    if (target_f == NULL) {
+        if (local_tgt_in == NULL) {
+            /* Use src_info if available -- else look up by tag */
+            coarse_info_t *info = src_info;
+            if (info == NULL) {
+                ASSERT(f->tag != NULL);
+                info = get_fragment_coarse_info(f);
+            }
+            if (info != NULL) {
+                fragment_coarse_lookup_in_unit(dcontext, info, target_tag, NULL,
+                                               &local_tgt);
+            }
+        } else
+            local_tgt = local_tgt_in;
+        if (local_tgt == NULL) {
+            remote_tgt = fragment_coarse_lookup(dcontext, target_tag);
+            coarse_tgt = remote_tgt;
+        } else
+            coarse_tgt = local_tgt;
+    } else
+        ASSERT(!TEST(FRAG_COARSE_GRAIN, target_f->flags));
+    if (coarse_tgt != NULL || target_f != NULL) {
+        if (target_f == NULL) {
+            /* No fine-grain fragment_t so make a fake one to use for the
+             * is_linkable() and mark_trace_head() paths.
+             * We can only recover certain flags, and we assume that others
+             * cannot be represented in a coarse unit anyway.
+             */
+            target_f = fragment_coarse_link_wrapper(dcontext, target_tag, coarse_tgt);
+            if (stub != NULL && coarse_is_trace_head(stub))
+                target_f->flags |= FRAG_IS_TRACE_HEAD;
+        }
+        if (is_linkable(dcontext, f, l, target_f,
+                        NEED_SHARED_LOCK(f->flags) || SHARED_FRAGMENTS_ENABLED(),
+                        true/*mark new trace heads*/)) {
+            linked = true;
+            if (local_tgt == NULL) {
+                /* Target is outside this unit, either a fine fragment_t
+                 * or another unit's coarse fragment
+                 */
+                if (!TEST(FRAG_COARSE_GRAIN, target_f->flags)) {
+                    if (!entrance_stub_linked(stub, src_info))
+                        coarse_link_to_fine(dcontext, stub, f, target_f, true/*link*/);
+                    else {
+                        ASSERT(entrance_stub_jmp_target(stub) ==
+                               FCACHE_ENTRY_PC(target_f));
+                    }
+                } else {
+                    if (!entrance_stub_linked(stub, src_info)) {
+                        coarse_info_t *target_info = get_fcache_coarse_info(coarse_tgt);
+                        ASSERT(stub != remote_tgt);
+                        LOG(THREAD, LOG_LINKS, 4,
+                            "    linking coarse "PFX"."PFX"->%s "PFX"\n",
+                            f->tag, FCACHE_ENTRY_PC(f), target_info->module,
+                            coarse_tgt);
+                        link_entrance_stub(dcontext, stub, coarse_tgt,
+                                           HOT_PATCHABLE, src_info);
+                        /* add to incoming list */
+                        ASSERT(target_info != NULL);
+                        ASSERT(coarse_tgt != NULL);
+                        prepend_new_coarse_incoming(target_info, stub, NULL);
+                    } else
+                        ASSERT(entrance_stub_jmp_target(stub) == coarse_tgt);
+                }
+            } else {
+                coarse_info_t *target_info = get_fcache_coarse_info(coarse_tgt);
+                ASSERT(target_info != NULL);
+                if (new_stub || target_info->persisted) {
+                    if (!entrance_stub_linked(stub, src_info)) {
+                        LOG(THREAD, LOG_LINKS, 4,
+                            "    linking coarse "PFX"."PFX"->"PFX" intra-unit\n",
+                            f->tag, FCACHE_ENTRY_PC(f), coarse_tgt);
+                        link_entrance_stub(dcontext, stub, coarse_tgt,
+                                           HOT_PATCHABLE, src_info);
+                    } else
+                        ASSERT(entrance_stub_jmp_target(stub) == coarse_tgt);
+                } else {
+                    /* Should only need to link if target is created,
+                     * when it should be linked by link_new_coarse_grain_fragment(),
+                     * or for persisted trace heads
+                     */
+                    ASSERT(entrance_stub_linked(stub, src_info));
+                }
+            }
+        } else {
+            /* No incoming needed since linking lazily */
+            /* Currently we do not support non-linkable coarse exits except
+             * for trace head targets (as they can all share an entrance).
+             * For others we'd need per-exit entrance stubs and custom fcache
+             * return paths: FIXME.
+             */
+            /* We should have converted the entrance stub to a trace head
+             * stub in mark_trace_head()
+             */
+            ASSERT((!TEST(FRAG_COARSE_GRAIN, target_f->flags) &&
+                    (TEST(FRAG_IS_TRACE_HEAD, target_f->flags) ||
+                     !TEST(FRAG_SHARED, target_f->flags))) ||
+                   coarse_is_trace_head(stub));
+            ASSERT(!entrance_stub_linked(stub, src_info));
+            LOG(THREAD, LOG_LINKS, 4,
+                "    NOT linking coarse "PFX"."PFX"->F%d("PFX") (!linkable th)\n",
+                f->tag, FCACHE_ENTRY_PC(f), target_f->id, target_f->tag);
+        }
+    } else {
+        /* We use our entrance stub as a future fragment placeholder */
+        ASSERT(!entrance_stub_linked(stub, src_info));
+        LOG(THREAD, LOG_LINKS, 4,
+            "    NOT linking coarse "PFX"."PFX"->"PFX" (doesn't exist)\n",
+            f->tag, FCACHE_ENTRY_PC(f), target_tag);
+    }
+    return linked;
+}
+
+static void
+link_new_coarse_grain_fragment(dcontext_t *dcontext, fragment_t *f)
+{
+    coarse_info_t *info = get_fragment_coarse_info(f);
+    linkstub_t *l;
+    cache_pc orig_stub, self_stub;
+    cache_pc local_stub, local_body;
+    future_fragment_t *future;
+    bool th_unlink = false;
+
+    ASSERT(info != NULL);
+    /* ensure some higher-level lock is held if f is shared
+     * no links across caches so only checking f's sharedness is enough
+     */
+    ASSERT(!NEED_SHARED_LOCK(f->flags) ||
+           self_owns_recursive_lock(&change_linking_lock));
+    LOG(THREAD, LOG_LINKS, 4, "linking coarse-grain fragment F%d("PFX")\n",
+        f->id, f->tag);
+    ASSERT(TESTALL(FRAG_COARSE_GRAIN|FRAG_SHARED, f->flags));
+
+    /* Transfer existing fine-grained future incoming links to this fragment */
+    future = fragment_lookup_future(dcontext, f->tag); /* shared only */
+    if (future != NULL) {
+        uint futflags = future->flags;
+        LOG(THREAD, LOG_LINKS, 4,
+            "  transferring flags 0x%08x from existing future frag\n", futflags);
+        /* we only expect certain flags on future fragments */
+        ASSERT_CURIOSITY((futflags & ~(FUTURE_FLAGS_ALLOWED)) == 0);
+        /* sharedness must match */
+        ASSERT(TEST(FRAG_SHARED, f->flags) == TEST(FRAG_SHARED, futflags));
+        /* FIXME: we will discard all of these flags, which right now only include
+         * secondary shared trace heads from private traces, fortunately, and
+         * we'll use that by converting to a trace head below.
+         */
+        f->flags |= (futflags & (FUTURE_FLAGS_TRANSFER));
+        /* we shouldn't have any of the incompatibilites a new fine fragment does */
+        ASSERT(!TESTANY(FRAG_CANNOT_BE_TRACE|FRAG_IS_TRACE, f->flags));
+        if (TEST(FRAG_IS_TRACE_HEAD, f->flags))
+            mark_trace_head(dcontext, f, f, NULL);
+
+        if (future->incoming_stubs != NULL) {
+            LOG(THREAD, LOG_LINKS, 4,
+                "  transferring incoming links from existing future frag\n");
+            prepend_new_coarse_incoming(info, NULL, future->incoming_stubs);
+            /* We can re-use link_fragment_incoming, but be careful of any future
+             * changes that require splitting out the coarse-and-fine-shared part.
+             */
+            f->in_xlate.incoming_stubs = future->incoming_stubs;
+            DODEBUG({ future->incoming_stubs = NULL; });
+            link_fragment_incoming(dcontext, f, true/*new*/);
+        }
+
+        fragment_delete_future(dcontext, future);
+    }
+    ASSERT(fragment_lookup_future(dcontext, f->tag) == NULL);
+
+    /* There is no proactive linking from coarse-grain fragments: it's
+     * all done lazily, so there are no records of who wanted to link
+     * to this fragment from other coarse units.
+     * For sources inside this unit, they all point at the entrance stub, so
+     * our only incoming link action is to link the entrance stub to us.
+     * FIXME: we assume that is_linkable() conditions haven't changed -- that
+     * it only needs to be called once for coarse-grain intra-unit links.
+     * This means that trace head definitions, etc., will not see any changed
+     * data about this target.
+     */
+    fragment_coarse_lookup_in_unit(dcontext, info, f->tag, &orig_stub, NULL);
+    if (orig_stub == NULL) {
+        /* To enable easy trace head unlinking we want an entrance stub for
+         * every intra-linked-to fragment.  Currently we proactively create an
+         * entrance stub for ourselves at creation time.
+         *
+         * Case 8628: An alternative is to delay the stub creation while
+         * maintaining the easy unlink: put body pc in htable now, and only
+         * create stub if we see a second (creation implies a first :)) link to
+         * here.  That means we have mixed body and stub pc's in the htable for
+         * non-frozen units, requiring a vector binary search (for
+         * coarse_is_entrance_stub()) on every lookup.  For now we avoid that,
+         * but perhaps the waste of memory is less efficient, since once-only
+         * code won't ever use this entrance stub.  Plus, we need that vector
+         * search on every lookup for frozen units anyway, requiring us to eat
+         * the cost or split the htable.  If we switch to the alternative we can
+         * move the htable-add back to emit().
+         */
+        set_fake_direct_linkstub(&temp_linkstub, f->tag, NULL);
+        temp_linkstub.cdl.l.flags &= ~LINK_LINKED;
+        self_stub = entrance_stub_create(dcontext, info, f, (linkstub_t *)&temp_linkstub);
+    } else
+        self_stub = orig_stub;
+    if (self_stub != NULL) {
+        ASSERT(coarse_is_entrance_stub(self_stub));
+        if (TEST(FRAG_IS_TRACE_HEAD, f->flags)) { /* from future or incoming */
+            if (orig_stub == NULL) {
+                /* mark_trace_head didn't know where stub is -- so we must unlink: */
+                ASSERT(!entrance_stub_linked(self_stub, info) ||
+                       entrance_stub_jmp_target(self_stub) == FCACHE_ENTRY_PC(f));
+                /* unlink the stub and add the body pc to the th htable -- but
+                 * do it after the linking below, so that noone uses the body
+                 * pc table entry before the fragment is fully linked
+                 */
+                th_unlink = true;
+            } else {
+                if (!coarse_is_trace_head(self_stub)) {
+                    /* mark_trace_head() was called for incoming and marked it for a
+                     * different source coarse unit -- so we don't consider it a
+                     * trace head here.  FIXME: should we consider it one, and keep
+                     * path-dependent trace heads to a minimum?
+                     */
+                    f->flags &= ~FRAG_IS_TRACE_HEAD;
+                    STATS_INC(coarse_th_path_dependent);
+                    LOG(THREAD, LOG_LINKS, 4,
+                        "    trace head via inter-unit path, not intra-unit\n");
+                }
+            }
+        }
+    } else
+        ASSERT_NOT_REACHED(); /* won't happen w/ current proactive stub creation */
+
+    f->flags |= FRAG_LINKED_OUTGOING;
+    for (l = FRAGMENT_EXIT_STUBS(f); l; l = LINKSTUB_NEXT_EXIT(l)) {
+        if (LINKSTUB_DIRECT(l->flags)) {
+            bool new_stub = false;
+            app_pc target_tag = EXIT_TARGET_TAG(dcontext, f, l);
+            direct_linkstub_t *dl = (direct_linkstub_t *) l;
+            fragment_coarse_lookup_in_unit(dcontext, info, target_tag,
+                                           &local_stub, &local_body);
+            if (local_stub == NULL) {
+                /* We need a new entrance stub, even for intra-unit links where
+                 * the target exists (alternative of directly linking intra-unit
+                 * now requires unlink if mark target as trace head later).  If
+                 * the target doesn't exist we need one regardless.
+                 */
+                new_stub = true;
+                if (target_tag == f->tag) {
+                    local_stub = self_stub;
+                } else
+                    local_stub = entrance_stub_create(dcontext, info, f, l);
+            }
+            ASSERT(dl->stub_pc == NULL); /* not set in emit */
+            /* Case 9708: must set stub so we can mark extra-unit trace head
+             * if local stub is new.
+             */
+            dl->stub_pc = local_stub;
+            LOG(THREAD, LOG_LINKS, 4,
+                "  linking coarse "PFX"."PFX"->"PFX" to entrance stub\n",
+                f->tag, FCACHE_ENTRY_PC(f), local_stub);
+            patch_branch(EXIT_CTI_PC(f, l), local_stub,
+                         /*new, unreachable*/NOT_HOT_PATCHABLE);
+            /* case 9009: can't link stub to self until self fully linked;
+             * no incoming needed so fine to rely on self link below.
+             */
+            if (local_stub != self_stub) {
+                coarse_link_direct(dcontext, f, l, info, local_stub, target_tag,
+                                   local_body, new_stub);
+            }
+            /* case 9009: can't add to htable prior to linking, though
+             * here change_linking_lock should prevent anyone from
+             * using this stub for the first time.  Still, can't be
+             * too safe.
+             */
+            if (new_stub) {
+                if (local_body != NULL) {
+                    /* If we don't proactively create self entrance stubs,
+                     * here we would need to replace the htable entry of the
+                     * target's body with its new entrance stub -- but w/
+                     * current design we shouldn't enter this if.
+                     */
+                    ASSERT_NOT_REACHED();
+                    /* This is an entrance stub for a target in our own unit.
+                     * Point htable at stub instead of fragment.
+                     */
+                    fragment_coarse_replace(dcontext, info, f->tag, local_stub);
+                } else {
+                    /* We add to htable so others targeting this tag will use
+                     * the same entrance stub.
+                     * Case 9009: we cannot add self yet before we link our
+                     * other exits!  We'll do the add below.
+                     */
+                    if (local_stub != self_stub)
+                        fragment_coarse_add(dcontext, info, target_tag, local_stub);
+                }
+            }
+        } else {
+            ASSERT(LINKSTUB_INDIRECT(l->flags));
+            /* indirect branches: just let link_branch handle the
+             * exit stub target
+             */
+#ifdef DGC_DIAGNOSTICS
+            /* We don't support unlinked indirect branches.
+             * FIXME: should turn on -no_link_ibl
+             */
+            ASSERT_NOT_IMPLEMENTED(false);
+#endif
+            l->flags |= LINK_LINKED;
+        }
+    }
+
+    /* Perform self-stub-linking and htable adding only when fragment is fully
+     * linked (case 9009).  Outgoing linking shouldn't change incoming link or
+     * trace head status, so we shouldn't get asserts about not being a trace head
+     * yet not being linked.
+     */
+    if (self_stub != NULL) {
+        if (th_unlink) {
+            fragment_coarse_th_unlink_and_add(dcontext, f->tag,
+                                              self_stub, FCACHE_ENTRY_PC(f));
+        }
+        if (!TEST(FRAG_IS_TRACE_HEAD, f->flags)) {
+            LOG(THREAD, LOG_LINKS, 4,
+                "    linking coarse entrance stub to self "PFX"->"PFX"."PFX"\n",
+                self_stub, f->tag, FCACHE_ENTRY_PC(f));
+            link_entrance_stub(dcontext, self_stub, FCACHE_ENTRY_PC(f),
+                               /*new fragment: no races*/NOT_HOT_PATCHABLE, NULL);
+            ASSERT(entrance_stub_jmp_target(self_stub) == FCACHE_ENTRY_PC(f));
+        } else {
+            LOG(THREAD, LOG_LINKS, 4,
+                "    NOT linking coarse entrance stub "PFX"->"PFX" since trace head\n",
+                self_stub, f->tag);
+        }
+    }
+        
+    /* we add here rather than in emit() caller since we have self_stub ptr */
+    if (orig_stub == NULL)
+        fragment_coarse_add(dcontext, info, f->tag, self_stub);
+    else {
+        DODEBUG({
+            /* Stub was added by earlier target of this tag */
+            fragment_coarse_lookup_in_unit(dcontext, info, f->tag,
+                                           &local_stub, &local_body);
+            ASSERT(local_stub == self_stub);
+            ASSERT(local_body == FCACHE_ENTRY_PC(f));
+        });
+    }
+}
+
+/* Removes an incoming entry from a fine fragment to a coarse unit */
+static void
+coarse_remove_incoming(dcontext_t *dcontext, fragment_t *src_f, linkstub_t *src_l,
+                       fragment_t *targetf)
+{
+    coarse_info_t *info = get_fragment_coarse_info(targetf);
+    coarse_incoming_t *e, *prev_e;
+    ASSERT(!TEST(FRAG_COARSE_GRAIN, src_f->flags));
+    ASSERT(!LINKSTUB_FAKE(src_l));
+    ASSERT(TEST(FRAG_COARSE_GRAIN, targetf->flags));
+    LOG(THREAD, LOG_LINKS, 4, "coarse_remove_incoming %s "PFX" to "PFX"\n",
+        info->module, src_f->tag, targetf->tag);
+
+    mutex_lock(&info->incoming_lock);
+    for (prev_e = NULL, e = info->incoming; e != NULL; prev_e = e, e = e->next) {
+        if (!e->coarse) {
+            if (incoming_remove_link_search(dcontext, src_f, src_l, targetf,
+                                            (common_direct_linkstub_t **)
+                                            &e->in.fine_l)) {
+                if (e->in.fine_l == NULL) {
+                    /* If no fine entries left, remove the incoming entry wrapper */
+                    if (prev_e == NULL)
+                        info->incoming = e->next;
+                    else
+                        prev_e->next = e->next;
+                    LOG(THREAD, LOG_LINKS, 4,
+                        "freeing coarse_incoming_t "PFX"\n", e);
+                    NONPERSISTENT_HEAP_TYPE_FREE(GLOBAL_DCONTEXT, e, coarse_incoming_t,
+                                                 ACCT_COARSE_LINK);
+                }
+                break;
+            }
+        }
+    }
+    DOLOG(5, LOG_LINKS, {
+        LOG(THREAD, LOG_LINKS, 4, "after removing incoming:\n");
+        print_coarse_incoming(dcontext, info);
+    });
+    mutex_unlock(&info->incoming_lock);
+}
+
+/* Removes any incoming data recording the outgoing link from stub */
+void
+coarse_remove_outgoing(dcontext_t *dcontext, cache_pc stub, coarse_info_t *src_info)
+{
+    cache_pc target_tag = entrance_stub_target_tag(stub);
+    /* ASSUMPTION: coarse-grain are always shared and cannot target private */
+    fragment_t *targetf = fragment_lookup_same_sharing(dcontext, target_tag,
+                                                     FRAG_SHARED);
+    ASSERT(entrance_stub_linked(stub, src_info));
+    if (targetf != NULL) {
+        /* targeting a real fragment_t */
+        direct_linkstub_t proxy;
+        set_fake_direct_linkstub(&proxy, target_tag, stub);
+        LOG(THREAD, LOG_LINKS, 4,
+            "    removing coarse link "PFX" -> F%d("PFX")."PFX"\n",
+            stub, targetf->id, targetf->tag, FCACHE_ENTRY_PC(targetf));
+        incoming_remove_link(dcontext, (fragment_t *) &coarse_fragment,
+                             (linkstub_t *) &proxy, targetf);
+        ASSERT(entrance_stub_linked(stub, src_info));
+        /* case 9635: pass flags */
+        unlink_entrance_stub(dcontext, stub, targetf->flags, src_info);
+    } else {
+        cache_pc target_pc = entrance_stub_jmp_target(stub);
+        coarse_info_t *target_info = get_fcache_coarse_info(target_pc);
+        /* can only be NULL if targeting fine fragment, which is covered above! */
+        ASSERT(target_info != NULL);
+        if (target_info != src_info) {
+            coarse_incoming_t *e, *prev_e = NULL;
+            bool found = false;
+            unlink_entrance_stub(dcontext, stub, 0, src_info);
+            LOG(THREAD, LOG_LINKS, 4,
+                "    removing coarse link "PFX" -> %s "PFX"\n",
+                stub, target_info->module, target_tag);
+            mutex_lock(&target_info->incoming_lock);
+            for (e = target_info->incoming; e != NULL; e = e->next) {
+                if (e->coarse && e->in.stub_pc == stub) {
+                    if (prev_e == NULL)
+                        target_info->incoming = e->next;
+                    else
+                        prev_e->next = e->next;
+                    LOG(THREAD, LOG_LINKS, 4,
+                        "freeing coarse_incoming_t "PFX"\n", e);
+                    NONPERSISTENT_HEAP_TYPE_FREE(GLOBAL_DCONTEXT, e, coarse_incoming_t,
+                                                 ACCT_COARSE_LINK);
+                    found = true;
+                    break;
+                } else
+                    prev_e = e;
+            }
+            DOLOG(5, LOG_LINKS, {
+                LOG(THREAD, LOG_LINKS, 4, "after removing outgoing, target:\n");
+                print_coarse_incoming(dcontext, target_info);
+            });
+            mutex_unlock(&target_info->incoming_lock);
+            ASSERT(found);
+        } else {
+            /* an intra-unit link, so no incoming entry */
+            LOG(THREAD, LOG_LINKS, 4,
+                "    not removing coarse link to self "PFX" -> %s "PFX"\n",
+                stub, target_info->module, target_tag);
+        }
+    }
+}
+
+void
+coarse_mark_trace_head(dcontext_t *dcontext, fragment_t *f,
+                       coarse_info_t *info, cache_pc stub, cache_pc body)
+{
+    /* We do NOT keep incoming info for extra-unit trace head targets,
+     * as we do not need it (though we could unlink in coarse_unit_unlink).
+     */
+    if (entrance_stub_linked(stub, info)) {
+        LOG(THREAD, LOG_LINKS, 4, "  removing outgoing %s @"PFX"\n",
+            info->module, stub);
+        coarse_remove_outgoing(dcontext, stub, info);
+    }
+
+    LOG(THREAD, LOG_LINKS, 4,
+        "\tunlinking entrance stub "PFX", pointing at th routine\n", stub);
+    /* Convert to a trace head stub.  If body is in this unit (i.e.,
+     * not an extra-unit trace head targets), since we can no longer
+     * find the body pc from the stub, we must also store it in a
+     * separate htable.
+     */
+    fragment_coarse_th_unlink_and_add(dcontext, f->tag, stub, body);
+}
+
+/* Unlinks both incoming and outgoing links.
+ * Coarse units cannot be re-linked, so unlinking will remove all data
+ * structs in incoming lists.
+ * Due to lock rank order, caller must hold change_linking_lock in addition
+ * to info->lock.
+ */
+void
+coarse_unit_unlink(dcontext_t *dcontext, coarse_info_t *info)
+{
+    /* Unlink incoming */
+    coarse_incoming_t *e, *next_e;
+    DEBUG_DECLARE(bool keep;)
+    ASSERT(info != NULL);
+    ASSERT(self_owns_recursive_lock(&change_linking_lock));
+    ASSERT_OWN_MUTEX(true, &info->lock);
+    if (info->stubs == NULL) /* lazily initialized, so common to have empty units */
+        return;
+    mutex_lock(&info->incoming_lock);
+#ifndef DEBUG
+    /* case 8599: fast exit/reset path: all incoming links are in
+     * nonpersistent memory
+     */
+    if (dynamo_exited || dynamo_resetting) {
+        info->incoming = NULL;
+        mutex_unlock(&info->incoming_lock);
+        return;
+    }
+#endif
+    LOG(THREAD, LOG_LINKS, 4, "coarse_unit_unlink %s\n", info->module);
+
+    DOLOG(5, LOG_LINKS, {
+        LOG(THREAD, LOG_LINKS, 4, "about to remove all incoming:\n");
+        print_coarse_incoming(dcontext, info);
+    });
+    for (e = info->incoming; e != NULL; e = next_e) {
+        next_e = e->next;
+        if (e->coarse) {
+            unlink_entrance_stub(dcontext, e->in.stub_pc, 0, NULL);
+        } else {
+            linkstub_t *l, *last_l = NULL;
+            future_fragment_t *future;
+            app_pc tgt = NULL;
+            for (l = e->in.fine_l; l != NULL; last_l = l, l = LINKSTUB_NEXT_INCOMING(l)) {
+                fragment_t *in_f = linkstub_fragment(dcontext, l);
+                if (tgt == NULL) {
+                    /* unprotect on demand (caller will re-protect) */
+                    SELF_PROTECT_CACHE(dcontext, in_f, WRITABLE);
+                    tgt = EXIT_TARGET_TAG(dcontext, in_f, l);
+                } else {
+                    /* Every fine incoming in a single coarse-list entry should
+                     * target the same tag
+                     */
+                    ASSERT(EXIT_TARGET_TAG(dcontext, in_f, l) == tgt);
+                }
+                DEBUG_DECLARE(keep =) unlink_branch(dcontext, in_f, l);
+                ASSERT(keep);
+            }
+            /* Ensure we shifted links properly to traces replacing coarse heads */
+            ASSERT(fragment_lookup_trace(dcontext, tgt) == NULL);
+            future = fragment_lookup_future(dcontext, tgt);
+            if (future == NULL) {
+                LOG(THREAD, LOG_LINKS, 4,
+                    "    adding future fragment for removed coarse target "PFX"\n", tgt);
+                future = fragment_create_and_add_future(dcontext, tgt,
+                                                        FRAG_SHARED|FRAG_WAS_DELETED);
+                future->incoming_stubs = e->in.fine_l;
+            } else {
+                /* It's possible to have multiple incoming entries here for later-linked
+                 * fine sources, and thus for this routine to have created a future
+                 * from an earlier entry.
+                 */
+                common_direct_linkstub_t *dl = (common_direct_linkstub_t *) last_l;
+                ASSERT(last_l != NULL);
+                dl->next_incoming = future->incoming_stubs;
+                future->incoming_stubs = e->in.fine_l;
+            }
+        }
+        LOG(THREAD, LOG_LINKS, 4, "freeing coarse_incoming_t "PFX"\n", e);
+        NONPERSISTENT_HEAP_TYPE_FREE(GLOBAL_DCONTEXT, e, coarse_incoming_t,
+                                     ACCT_COARSE_LINK);
+    }
+    info->incoming = NULL;
+    mutex_unlock(&info->incoming_lock);
+
+    coarse_unit_unlink_outgoing(dcontext, info);
+}
+
+void
+coarse_unit_unlink_outgoing(dcontext_t *dcontext, coarse_info_t *info)
+{
+    coarse_stubs_iterator_t csi;
+    cache_pc pc;
+    ASSERT(info != NULL);
+    ASSERT(self_owns_recursive_lock(&change_linking_lock));
+    ASSERT_OWN_MUTEX(true, &info->lock);
+
+    /* Unlink outgoing by walking the stubs */
+    coarse_stubs_iterator_start(info, &csi);
+    for (pc = coarse_stubs_iterator_next(&csi); pc != NULL;
+         pc = coarse_stubs_iterator_next(&csi)) {
+        if (entrance_stub_linked(pc, info)) {
+            LOG(THREAD, LOG_LINKS, 4, "  removing outgoing %s @"PFX"\n",
+                info->module, pc);
+            coarse_remove_outgoing(dcontext, pc, info);
+        } else {
+            /* Lazy linking, and when we unlink (for trace head) we remove
+             * incoming then, so no incoming entries to remove here.
+             */
+            LOG(THREAD, LOG_LINKS, 4,
+                "  not removing unlinked outgoing %s @"PFX"\n", info->module, pc);
+        }
+    }
+    coarse_stubs_iterator_stop(&csi);
+}
+
+#ifdef DEBUG
+bool
+coarse_unit_outgoing_linked(dcontext_t *dcontext, coarse_info_t *info)
+{
+    coarse_stubs_iterator_t csi;
+    cache_pc pc;
+    bool linked = false;
+    ASSERT(info != NULL);
+    mutex_lock(&info->lock);
+
+    /* Check outgoing links by walking the stubs */
+    coarse_stubs_iterator_start(info, &csi);
+    for (pc = coarse_stubs_iterator_next(&csi); pc != NULL;
+         pc = coarse_stubs_iterator_next(&csi)) {
+        if (entrance_stub_linked(pc, info)) {
+            linked = true;
+            goto coarse_unit_outgoing_linked_done;
+        }
+    }
+coarse_unit_outgoing_linked_done:
+    coarse_stubs_iterator_stop(&csi);
+    mutex_unlock(&info->lock);
+    return linked;
+}
+#endif
+
+/* Returns the entrance stub that targets target_pc.
+ * We cannot find the unique source tag, only the stub (case 8565).
+ */
+cache_pc
+coarse_stub_lookup_by_target(dcontext_t *dcontext, coarse_info_t *info,
+                             cache_pc target_tag)
+{
+    cache_pc stub;
+    ASSERT(info != NULL);
+    fragment_coarse_lookup_in_unit(dcontext, info, target_tag, &stub, NULL);
+    return stub;
+}
+
+/* Coarse-grain lazy linking: if either source or target is coarse.
+ * 
+ * We don't keep the bookkeeping around (viz., future fragments) to support
+ * proactive linking.  Lazy linking does mean that we need some source info,
+ * unfortunately.  We use a per-unit fcache_return miss path to identify source
+ * unit, and then search that unit's stubs for the target recorded by the
+ * entrance stub.  We cannot find the unique source tag, only the entrance stub,
+ * which we assume is enough for linking (case 8565).
+ */
+void
+coarse_lazy_link(dcontext_t *dcontext, fragment_t *targetf)
+{
+    DEBUG_DECLARE(bool linked = false;)
+    DEBUG_DECLARE(bool already_linked = false;)
+    ASSERT(dcontext->next_tag == targetf->tag);
+    if (dcontext->last_exit == get_coarse_exit_linkstub() &&
+        /* rule out !is_linkable targets now to avoid work and assert below */
+        TEST(FRAG_SHARED, targetf->flags) &&
+        (!TEST(FRAG_IS_TRACE_HEAD, targetf->flags) ||
+         TEST(FRAG_COARSE_GRAIN, targetf->flags) ||
+         DYNAMO_OPTION(disable_traces))) {
+        /* Source is coarse */
+        fragment_t temp_sourcef;
+        cache_pc stub;
+        coarse_info_t *info = dcontext->coarse_exit.dir_exit;
+        if (info != NULL) {
+            ASSERT(is_dynamo_address((byte *)info)); /* ensure union used as expected */
+            stub = coarse_stub_lookup_by_target(dcontext, info, dcontext->next_tag);
+            ASSERT(stub != NULL);
+            /* May already be linked (we may have just built its target)
+             * Case 8825: we must hold the change_linking_lock when we check.
+             * Cheaper to just grab lock than to test first, unless high contention.
+             */
+            acquire_recursive_lock(&change_linking_lock);
+            if (!entrance_stub_linked(stub, info) &&
+                (!coarse_is_trace_head(stub) || DYNAMO_OPTION(disable_traces))) {
+                /* We don't know the specific tag of the source, as multiple may
+                 * share a single entrance stub.  ASSUMPTION (case 8565): the
+                 * only way in which is_linkable depends on tags is for trace
+                 * head identification, which should happen at
+                 * coarse-fragment-to-entrance-stub link time.
+                 */
+                memset(&temp_sourcef, 0, sizeof(temp_sourcef));
+                temp_sourcef.tag = NULL; /* thus no trace head trigger */
+                DODEBUG({ temp_sourcef.start_pc = stub; });
+                temp_sourcef.flags = FRAG_SHARED | FRAG_COARSE_GRAIN |
+                    FRAG_LINKED_OUTGOING | FRAG_LINKED_INCOMING;
+                
+                set_fake_direct_linkstub(&temp_linkstub, dcontext->next_tag, NULL);
+                temp_linkstub.cdl.l.flags &= ~LINK_LINKED;
+                
+                /* FIXME: we have targetf, we should use it here
+                 * FIXME: Our source f has no tag, so we pass in info -- fragile!
+                 */
+                if (coarse_link_direct(dcontext, &temp_sourcef,
+                                       (linkstub_t *) &temp_linkstub,
+                                       info, stub, dcontext->next_tag, NULL,
+                                       false/*stub should already exist*/)) {
+                    LOG(THREAD, LOG_LINKS, 4,
+                        "lazy linked "PFX" -> F%d("PFX")."PFX"\n",
+                        stub, targetf->id, targetf->tag, FCACHE_ENTRY_PC(targetf));
+                    STATS_INC(lazy_links_from_coarse);
+                    DOSTATS({
+                        if (info->persisted)
+                            STATS_INC(lazy_links_from_persisted);
+                    });
+                    DODEBUG({ linked = true; });
+                }
+            } else {
+                DODEBUG({ already_linked = true; });
+            }
+            release_recursive_lock(&change_linking_lock);
+        }
+    } else if (TEST(FRAG_COARSE_GRAIN, targetf->flags)) {
+        /* Target is coarse */
+        linkstub_t *l = dcontext->last_exit;
+        fragment_t *f = dcontext->last_fragment;
+        if (dcontext->next_tag != EXIT_TARGET_TAG(dcontext, f, l) ||
+            !TEST(FRAG_SHARED, f->flags)) {
+            /* Rule out syscall-skip and other cases where we cannot link.
+             * is_linkable() should catch, but this way we avoid work and avoid
+             * the incoming_link_exists() assert below.
+             */
+            goto lazy_link_done;
+        }
+        /* May already be linked (we may have just built its target) */
+        if (LINKSTUB_DIRECT(l->flags) && !LINKSTUB_FAKE(l) &&
+            !TEST(FRAG_FAKE, f->flags) && !TEST(LINK_LINKED, l->flags)) {
+            acquire_recursive_lock(&change_linking_lock);
+            /* FIXME: provide common routine for this: dup of link_fragment_outgoing */
+            if (!TEST(LINK_LINKED, l->flags) /* case 8825: test w/ lock! */ &&
+                is_linkable(dcontext, f, l, targetf, true/*have change_linking_lock*/,
+                            true/*mark new trace heads*/)) {
+                LOG(THREAD, LOG_LINKS, 4,
+                    "lazy linking F%d("PFX") -> F%d("PFX")."PFX"\n",
+                    f->id, f->tag, targetf->id, targetf->tag, FCACHE_ENTRY_PC(targetf));
+                link_branch(dcontext, f, l, targetf, HOT_PATCHABLE);
+                add_incoming(dcontext, f, l, targetf, true);
+                STATS_INC(lazy_links_from_fine);
+                DODEBUG({ linked = true; });
+            } else {
+                DOCHECK(1, { /* PR 307698: perf hit */
+                    ASSERT(incoming_link_exists(dcontext, f, l, targetf) ||
+                           /* case 8786: another thread could have built a
+                            * shared trace to replace this coarse fragment and
+                            * already shifted incoming links to the new trace */
+                           (DYNAMO_OPTION(shared_traces) &&
+                            fragment_lookup_trace(dcontext, dcontext->next_tag) != NULL));
+                });
+            }
+            release_recursive_lock(&change_linking_lock);
+        } else {
+            /* We'll treat indirect xfer as already-linked */
+            DODEBUG({ already_linked = true; });
+        }
+    }
+ lazy_link_done:
+    DODEBUG({
+        if (!linked) {
+            LOG(THREAD, LOG_LINKS, 4,
+                "NOT lazy linking F%d("PFX") -> F%d("PFX")."PFX"%s\n",
+                dcontext->last_fragment->id, dcontext->last_fragment->tag,
+                targetf->id, targetf->tag, FCACHE_ENTRY_PC(targetf),
+                already_linked ? " already linked" : "");
+            if (!already_linked) {
+                STATS_INC(lazy_links_failed);
+            }
+        }
+    });
+}
+
+/* Passing in stub_pc's info avoids a vmvector lookup */
+cache_pc
+fcache_return_coarse_prefix(cache_pc stub_pc, coarse_info_t *info /*OPTIONAL*/)
+{
+    if (info == NULL)
+        info = (coarse_info_t *) vmvector_lookup(coarse_stub_areas, stub_pc);
+    else {
+        DOCHECK(1, { /* PR 307698: perf hit */
+            ASSERT(info == vmvector_lookup(coarse_stub_areas, stub_pc));
+        });
+    }
+    if (info != NULL) {
+        return info->fcache_return_prefix;
+    }
+    return NULL;
+}
+
+/* Passing in stub_pc's info avoids a vmvector lookup */
+cache_pc
+trace_head_return_coarse_prefix(cache_pc stub_pc, coarse_info_t *info /*OPTIONAL*/)
+{
+    if (info == NULL)
+        info = (coarse_info_t *) vmvector_lookup(coarse_stub_areas, stub_pc);
+    else {
+        DOCHECK(1, { /* PR 307698: perf hit */
+            ASSERT(info == vmvector_lookup(coarse_stub_areas, stub_pc));
+        });
+    }
+    if (info != NULL) {
+        return info->trace_head_return_prefix;
+    }
+    return NULL;
+}
+
+/* Either the stub pc or the cti pc will work, as indirect stubs are inlined */
+cache_pc
+get_coarse_ibl_prefix(dcontext_t *dcontext, cache_pc stub_pc,
+                      ibl_branch_type_t branch_type)
+{
+    /* Indirect stubs are inlined in the cache */
+    coarse_info_t *info = (coarse_info_t *) get_fcache_coarse_info(stub_pc);
+    if (info != NULL) {
+        if (branch_type == IBL_RETURN)
+            return info->ibl_ret_prefix;
+        else if (branch_type == IBL_INDCALL)
+            return info->ibl_call_prefix;
+        else if (branch_type == IBL_INDJMP)
+            return info->ibl_jmp_prefix;
+        else
+            ASSERT_NOT_REACHED();
+    }
+    return NULL;
+}
+
+bool
+in_coarse_stubs(cache_pc pc)
+{
+    return (vmvector_lookup(coarse_stub_areas, pc) != NULL);
+}
+
+bool
+in_coarse_stub_prefixes(cache_pc pc)
+{
+    coarse_info_t *info = get_stub_coarse_info(pc);
+    if (info != NULL) {
+        return (pc >= info->fcache_return_prefix &&
+                pc < info->fcache_return_prefix +
+                COARSE_STUB_ALLOC_SIZE(COARSE_32_FLAG(info))*
+                num_coarse_stubs_for_prefix(info));
+    }
+    return false;
+}
+
+/* If target is a coarse ibl prefix, returns its target (i.e., the final
+ * ibl routine target); else, returns the original target passed in.
+ */
+cache_pc
+coarse_deref_ibl_prefix(dcontext_t *dcontext, cache_pc target)
+{
+    coarse_info_t *info = get_stub_coarse_info(target);
+    if (info != NULL) {
+        if (target >= info->ibl_ret_prefix &&
+            target <= info->ibl_jmp_prefix) {
+            ASSERT(*target == JMP_OPCODE);
+            return (cache_pc) PC_RELATIVE_TARGET(target+1);
+        }
+    }
+    return target;
+}
+
+coarse_info_t *
+get_stub_coarse_info(cache_pc pc)
+{
+    return (coarse_info_t *) vmvector_lookup(coarse_stub_areas, pc);
+}
+
+/* Returns the total size needed for stubs (including prefixes) if info is frozen. */
+size_t
+coarse_frozen_stub_size(dcontext_t *dcontext, coarse_info_t *info,
+                        uint *num_fragments, uint *num_stubs)
+{
+    coarse_stubs_iterator_t csi;
+    cache_pc pc;
+    size_t size = 0;
+    size_t stub_size = COARSE_STUB_ALLOC_SIZE(COARSE_32_FLAG(info));
+    uint num_unlinked = 0;
+    uint num_inter = 0;
+    uint num_intra = 0;
+    ASSERT(info != NULL);
+    ASSERT(self_owns_recursive_lock(&change_linking_lock));
+    ASSERT_OWN_MUTEX(true, &info->lock);
+    if (info->stubs == NULL) /* lazily initialized, so common to have empty units */
+        return size;
+    LOG(THREAD, LOG_LINKS, 4, "coarse_frozen_stub_size %s\n", info->module);
+    coarse_stubs_iterator_start(info, &csi);
+    /* we do include the prefix size here */
+    size += stub_size*num_coarse_stubs_for_prefix(info);
+    for (pc = coarse_stubs_iterator_next(&csi); pc != NULL;
+         pc = coarse_stubs_iterator_next(&csi)) {
+        if (entrance_stub_linked(pc, info)) {
+            cache_pc target = entrance_stub_jmp_target(pc);
+            /* When frozen we keep trace head stubs and only eliminate
+             * stubs that are linked and pointing at this unit
+             */
+            if (get_fcache_coarse_info(target) != info) {
+                size += stub_size;
+                num_inter++;
+            } else {
+                num_intra++;
+            }
+        } else {
+            /* Lazy linking is only for extra-unit targets so an unlinked
+             * stub must have no intra-unit target
+             */
+            size += stub_size;
+            num_unlinked++; /* trace head, or just no target hit yet */
+        }
+    }
+    coarse_stubs_iterator_stop(&csi);
+    LOG(THREAD, LOG_LINKS, 2,
+        "coarse_frozen_stub_size %s: %d intra, %d inter, %d unlinked => %d bytes\n",
+        info->module, num_intra, num_inter, num_unlinked, size);
+    if (num_fragments != NULL)
+        *num_fragments = num_intra;
+    if (num_stubs != NULL)
+        *num_stubs = num_inter + num_unlinked;
+    ASSERT(size == stub_size*(num_inter + num_unlinked +
+                              num_coarse_stubs_for_prefix(info)));
+    return size;
+}
+
+/* Intended to be called after freezing shifts a unit's fragments around.
+ * Updates the stub stored in the incoming entry for this outoing link.
+ * src_info should be the old, currently-in-vmareas info.
+ * Does not dereference old_stub (so ok if now invalid memory), but assumes
+ * that new_stub is already linked properly.
+ */
+void
+coarse_update_outgoing(dcontext_t *dcontext, cache_pc old_stub, cache_pc new_stub,
+                       coarse_info_t *src_info, bool replace)
+{
+    cache_pc target_tag = entrance_stub_target_tag(new_stub);
+    /* ASSUMPTION: coarse-grain are always shared and cannot target private */
+    fragment_t *targetf = fragment_lookup_same_sharing(dcontext, target_tag,
+                                                     FRAG_SHARED);
+    DOCHECK(1, { /* PR 307698: perf hit */
+        ASSERT(entrance_stub_linked(new_stub, NULL/*may not point to src_info*/));
+    });
+    if (targetf != NULL) {
+        /* targeting a real fragment_t */
+        LOG(THREAD, LOG_LINKS, 4,
+            "    %s coarse link ["PFX"=>"PFX"] -> F%d("PFX")."PFX"\n",
+            replace ? "updating" : "adding",
+            old_stub, new_stub, targetf->id, targetf->tag, FCACHE_ENTRY_PC(targetf));
+        ASSERT(entrance_stub_jmp_target(new_stub) == FCACHE_ENTRY_PC(targetf));
+        if (replace) {
+            direct_linkstub_t *l;
+            direct_linkstub_t proxy;
+            set_fake_direct_linkstub(&proxy, target_tag, old_stub);
+            l = (direct_linkstub_t *)
+                incoming_find_link(dcontext, (fragment_t *) &coarse_fragment,
+                                   (linkstub_t *) &proxy, targetf);
+            ASSERT(l != NULL);
+            ASSERT(LINKSTUB_NORMAL_DIRECT(l->cdl.l.flags));
+            l->stub_pc = new_stub;
+        } else {
+            coarse_link_to_fine(dcontext, new_stub, (fragment_t *) &coarse_fragment,
+                                targetf, false/*just add incoming*/);
+        }
+    } else {
+        cache_pc target_pc = entrance_stub_jmp_target(new_stub);
+        coarse_info_t *target_info = get_fcache_coarse_info(target_pc);
+        /* can only be NULL if targeting fine fragment, which is covered above! */
+        ASSERT(target_info != NULL);
+        if (target_info != src_info) {
+            coarse_incoming_t *e;
+            bool found = false;
+            LOG(THREAD, LOG_LINKS, 4,
+                "    %s coarse link ["PFX"=>"PFX"] -> %s "PFX"\n",
+                replace ? "updating" : "adding",
+                old_stub, new_stub, target_info->module, target_tag);
+            if (replace) {
+                mutex_lock(&target_info->incoming_lock);
+                for (e = target_info->incoming; e != NULL; e = e->next) {
+                    if (e->coarse && e->in.stub_pc == old_stub) {
+                        e->in.stub_pc = new_stub;
+                        found = true;
+                        break;
+                    }
+                }
+                DOLOG(5, LOG_LINKS, {
+                    LOG(THREAD, LOG_LINKS, 4, "after updating outgoing, target:\n");
+                    print_coarse_incoming(dcontext, target_info);
+                });
+                mutex_unlock(&target_info->incoming_lock);
+                ASSERT(found);
+            } else {
+                prepend_new_coarse_incoming(target_info, new_stub, NULL);
+            }
+        } else {
+            ASSERT_NOT_REACHED(); /* currently caller checks for intra */
+            /* an intra-unit link, so no incoming entry */
+            LOG(THREAD, LOG_LINKS, 4,
+                "    not updating coarse link to self "PFX" -> %s "PFX"\n",
+                old_stub, target_info->module, target_tag);
+        }
+    }
+}
+
+/* Intended to be called after freezing has shifted a unit's fragments around.
+ * Re-links incoming links to point at their targets' new locations.
+ * Does NOT update outgoing links (which should be done incrementally
+ * at freeze time when both old and new stub pcs are known).
+ * Due to lock rank order, caller must hold change_linking_lock in addition
+ * to info->lock.  Caller is also assumed to have stopped the world.
+ */
+void
+coarse_unit_shift_links(dcontext_t *dcontext, coarse_info_t *info)
+{
+    /* Re-link incoming */
+    coarse_incoming_t *e;
+    cache_pc new_tgt;
+    bool hot_patch = (dynamo_all_threads_synched ? NOT_HOT_PATCHABLE : HOT_PATCHABLE);
+    ASSERT(info != NULL);
+    ASSERT(self_owns_recursive_lock(&change_linking_lock));
+    ASSERT_OWN_MUTEX(!dynamo_all_threads_synched, &info->lock);
+    if (info->stubs == NULL) /* lazily initialized, so common to have empty units */
+        return;
+    LOG(THREAD, LOG_LINKS, 4, "coarse_unit_shift_links %s\n", info->module);
+
+    mutex_lock(&info->incoming_lock);
+    DOLOG(5, LOG_LINKS, {
+        LOG(THREAD, LOG_LINKS, 4, "about to patch all incoming links:\n");
+        print_coarse_incoming(dcontext, info);
+    });
+    for (e = info->incoming; e != NULL; e = e->next) {
+        new_tgt = NULL;
+        if (e->coarse) {
+            /* coarse never have incoming structs for unlinked links */
+            app_pc tag = entrance_stub_target_tag(e->in.stub_pc);
+            fragment_coarse_lookup_in_unit(dcontext, info, tag, NULL, &new_tgt);
+            ASSERT(new_tgt != NULL);
+            link_entrance_stub(dcontext, e->in.stub_pc, new_tgt, hot_patch, NULL);
+        } else {
+            linkstub_t *l;
+            app_pc tag = NULL;
+            for (l = e->in.fine_l; l != NULL; l = LINKSTUB_NEXT_INCOMING(l)) {
+                /* fine can have incoming structs yet not be linked (trace head, etc.) */
+                if (TEST(LINK_LINKED, l->flags)) {
+                    fragment_t *in_f = linkstub_fragment(dcontext, l);
+                    if (tag == NULL) {
+                        /* unprotect on demand (caller will re-protect) */
+                        SELF_PROTECT_CACHE(dcontext, in_f, WRITABLE);
+                        tag = EXIT_TARGET_TAG(dcontext, in_f, l);
+                        fragment_coarse_lookup_in_unit(dcontext, info, tag,
+                                                       NULL, &new_tgt);
+                    } else
+                        ASSERT(EXIT_TARGET_TAG(dcontext, in_f, l) == tag);
+                    ASSERT(new_tgt != NULL);
+                    patch_branch(EXIT_CTI_PC(in_f, l), new_tgt, hot_patch);
+                }
+            }
+        }
+    }
+    mutex_unlock(&info->incoming_lock);
+}
+
+/* Updates the info pointers embedded in the coarse_stub_areas vector */
+void
+coarse_stubs_set_info(coarse_info_t *info)
+{
+    special_heap_set_vector_data(info->stubs, info);
+}
+
+void
+coarse_stubs_set_end_pc(coarse_info_t *info, byte *end_pc)
+{
+    DEBUG_DECLARE(bool ok =) special_heap_set_unit_end(info->stubs, end_pc);
+    ASSERT(ok);
+    info->stubs_end_pc = end_pc;
+}
diff --git a/core/link.h b/core/link.h
new file mode 100644
index 000000000..15ac32082
--- /dev/null
+++ b/core/link.h
@@ -0,0 +1,573 @@
+/* **********************************************************
+ * Copyright (c) 2000-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * link.h - link exports
+ */
+
+#ifndef _LINK_H_
+#define _LINK_H_ 1
+
+
+/* linkstub_t flags
+ * WARNING: flags field is a ushort, so max flag is 0x8000!
+ */
+enum {
+    /***************************************************/
+    /* these first flags are also used on the instr_t data structure */
+
+    /* Type of branch and thus which struct is used for this exit.
+     * Due to a tight namespace (flags is a ushort field), we pack our
+     * 3 types into these 2 bits, so the LINKSTUB_ macros are used to
+     * distinguish, rather than raw bit tests:
+     *
+     *   name          LINK_DIRECT LINK_INDIRECT  struct
+     *   ---------------  ---         ---         --------------------------
+     *   (subset of fake)  0           0          linkstub_t          
+     *   normal direct     1           0          direct_linkstub_t
+     *   normal indirect   0           1          indirect_linkstub_t
+     *   cbr fallthrough   1           1          cbr_fallthrough_linkstub_t
+     *
+     * Note that we can have fake linkstubs that should be treated as
+     * direct or indirect, so LINK_FAKE is a separate flag.
+     */
+    LINK_DIRECT          = 0x0001,
+    LINK_INDIRECT        = 0x0002,
+    /* more specifics on type of branch
+     * must check LINK_DIRECT vs LINK_INDIRECT for JMP and CALL.
+     * absence of all of these is relied on as an indicator of shared_syscall
+     * in indirect_linkstub_target(), so we can't get rid of LINK_RETURN
+     * and use absence of CALL & JMP to indicate it.
+     */
+    LINK_RETURN          = 0x0004,
+    LINK_CALL            = 0x0008,
+    LINK_JMP             = 0x0010,
+    /* if need another flag, could use JMP|CALL to indicate JMP_PLT,
+     * and make sure all testers for CALL also test for !JMP
+     */
+    LINK_IND_JMP_PLT     = 0x0020,
+
+    LINK_SELFMOD_EXIT    = 0x0040,
+#ifdef UNSUPPORTED_API
+    LINK_TARGET_PREFIX   = 0x0080,
+#endif
+#ifdef X64
+    /* PR 257963: since we don't store targets of ind branches, we need a flag
+     * so we know whether this is a trace cmp exit, which has its own ibl entry
+     */
+    LINK_TRACE_CMP       = 0x0100,
+#endif
+#ifdef WINDOWS
+    LINK_CALLBACK_RETURN = 0x0200,
+#else
+    /* PR 286922: we support both OP_sys{call,enter}- and OP_int-based system calls */
+    LINK_NI_SYSCALL_INT  = 0x0200,
+#endif
+    /* indicates whether exit is before a non-ignorable syscall */
+    LINK_NI_SYSCALL      = 0x0400,
+    LINK_FINAL_INSTR_SHARED_FLAG = LINK_NI_SYSCALL,
+    /* end of instr_t-shared flags  */
+    /***************************************************/
+
+    LINK_FRAG_OFFS_AT_END= 0x0800,
+
+    LINK_END_OF_LIST     = 0x1000,
+
+    LINK_FAKE            = 0x2000,
+
+    LINK_LINKED          = 0x4000,
+
+    LINK_SEPARATE_STUB   = 0x8000,
+
+    /* WARNING: flags field is a ushort, so max flag is 0x8000! */
+};
+
+#ifdef LINUX
+# define LINK_NI_SYSCALL_ALL (LINK_NI_SYSCALL | LINK_NI_SYSCALL_INT)
+#else
+# define LINK_NI_SYSCALL_ALL LINK_NI_SYSCALL
+#endif
+
+/* for easy printing */
+#ifndef LINKCOUNT_64_BITS
+#  define LINKCOUNT_FORMAT_STRING "%lu"
+#else
+#  define LINKCOUNT_FORMAT_STRING UINT64_FORMAT_STRING
+#endif
+
+/* linkstub_t heap layout is now quite variable.  linkstubs are laid out
+ * after the fragment_t structure, which is itself variable.
+ * The indirect_linkstub_t split was case 6468
+ * The cbr_fallthrough_linkstub_t split was case 6528
+ *
+ *   fragment_t/trace_t
+ *   array composed of three different sizes of linkstub_t subclasses:
+ *     direct_linkstub_t
+ *     cbr_fallthrough_linkstub_t
+ *     indirect_linkstub_t
+ *   post_linkstub_t
+ *   
+ * There are three types of specially-supported basic blocks that
+ * have no post_linkstub_t:
+ *   
+ *   fragment_t
+ *   indirect_linkstub_t
+ *   
+ *   fragment_t
+ *   direct_linkstub_t
+ *   direct_linkstub_t
+ *
+ *   fragment_t
+ *   direct_linkstub_t
+ *   cbr_fallthrough_linkstub_t
+ */
+
+/* To save memory, we have three types of linkstub structs: direct,
+ * cbr_fallthrough, and indirect.  They each contain the flags field
+ * at the same offset, so this struct can be used as a superclass.
+ */
+struct _linkstub_t {
+    ushort         flags;          /* contains LINK_ flags above */
+
+    /* We limit all fragment bodies to USHRT_MAX size
+     * thus we can save space by making the branch ptr a ushort offset.
+     * Do not directly access this field -- use EXIT_CTI_PC()
+     */
+    ushort         cti_offset;     /* offset from fragment start_pc of this cti */
+
+#ifdef CUSTOM_EXIT_STUBS
+    ushort         fixed_stub_offset;  /* offset in bytes of fixed part of exit stub from
+                                          stub_pc, which points to custom prefix of stub */
+#endif
+
+#ifdef PROFILE_LINKCOUNT
+    linkcount_type_t count;
+#endif
+};
+
+
+/* linkage info common to all direct fragment exits */
+typedef struct _common_direct_linkstub_t {
+    linkstub_t       l;
+
+    /* outgoing stubs of a fragment never change, and are allocated in
+     * an array, but we walk them like a linked list since we don't want
+     * to waste space storing the count and all of our access patterns
+     * want to touch them all anyway.
+     * use LINKSTUB_NEXT_EXIT() to access the next, and LINKSTUB_FINAL()
+     * to test if the current guy is the final.
+     *
+     * Incoming stubs do change and we use this field to chain them:
+     */
+    linkstub_t       *next_incoming;
+
+#ifdef TRACE_HEAD_CACHE_INCR
+    /* for linking to trace head, we store the actual fragment_t target.
+     * if the target is deleted the link will be unlinked, preventing
+     * a stale fragment_t* pointer from sitting around
+     */
+    fragment_t       *target_fragment;
+#endif
+} common_direct_linkstub_t;
+
+
+/* linkage info for each direct fragment exit */
+typedef struct _direct_linkstub_t {
+    common_direct_linkstub_t cdl;
+
+    /* tag identifying the intended app target of the exit branch 
+     * Do not directly access this field -- use EXIT_TARGET_TAG(), which
+     * works for all linkstub types.
+     */
+    app_pc         target_tag;
+
+    /* Must be absolute pc b/c we relocate some stubs away from fragment body.
+     * Do not directly access this field -- use EXIT_STUB_PC(), which
+     * works for all linkstub types.
+     */
+    cache_pc       stub_pc;
+} direct_linkstub_t;
+
+
+/* linkage info for cbr fallthrough exits that satisfy two conditions:
+ *   1) separate stubs will not be individually freed -- we could
+ *      have a fancy scheme that frees both at once, but we simply
+ *      disallow the struct if any freeing will occur.
+ *   2) the fallthrough target is within range of a signed short from the
+ *      owning fragment's start pc (this is typically true even w/ eliding).
+ *   3) the fallthrough exit immediately follows the cbr exit.
+ * this direct stub split was case 6528.
+ */
+typedef struct _cbr_fallthrough_linkstub_t {
+    /* We have no cti_offset as we assume this exit's cti immediately follows
+     * the preceding cbr (see conditions above).
+     *
+     * Our target_tag uses the cti_offset field instead.  Since this struct is
+     * only used for the 2nd (fallthrough) exit of a cbr whose target
+     * is within range for a signed short from the owning fragment_t's tag,
+     * we re-use the cti_offset field.
+     *
+     * This struct is also only used when its exit stub is adjacent to the
+     * prior exit's, so we don't need to store stub_pc here.
+     */
+    common_direct_linkstub_t cdl;
+} cbr_fallthrough_linkstub_t;
+
+
+/* linkage info for each indirect fragment exit (slit for case 6468) */
+typedef struct _indirect_linkstub_t {
+    linkstub_t       l;
+
+#ifdef NATIVE_RETURN
+    cache_pc       ret_pc;         /* if this exit is a ret, cache pc of the ret */
+#endif
+} indirect_linkstub_t;
+
+
+/* Data shared among all linkstubs for a particular fragment_t.
+ * Kept at the end of the array of linkstubs, and not present for
+ * certain common types of fragment_t.
+ */
+typedef struct _post_linkstub_t {
+    /* fragment_t + linkstub_t heap size cannot exceed the maximum fragment
+     * body size since max sizeof linkstub_t-subclass (16) < min sizeof
+     * exit stub (15) + corresponding cti (at least 2).  Thus we can
+     * use a ushort here, paving the way for an additional field in
+     * the future (though right now the other 2 bytes are wasted as
+     * this struct is aligned to 4 bytes ).
+     */
+    ushort         fragment_offset;
+    /* We force the compiler to maintain our 4-byte-alignment for heap allocations.
+     * FIXME: should we support un-aligned in non-special heaps?
+     * or could use special heap if we really wanted to save this 2 bytes.
+     * also see case 6518 for a way to use this for Traces.
+     */
+    ushort         padding;
+} post_linkstub_t;
+
+/* For chaining together a list of inter-coarse-unit incoming stubs.  To
+ * eliminate the need for wrappers for a series of fine-grained linkstubs, we
+ * directly chain those -- so when walking, walk a fine entry's linkstubs
+ * completely before going to the next coarse_incoming_t entry.
+ */
+typedef struct _coarse_incoming_t {
+    union {
+        cache_pc stub_pc;
+        linkstub_t *fine_l;
+    } in;
+    bool coarse;
+    struct _coarse_incoming_t *next;
+} coarse_incoming_t;
+
+/* this one does not take in flags b/c historically we used other fields */
+#define LINKSTUB_FAKE(l) (TEST(LINK_FAKE, (l)->flags))
+/* direct includes normal direct and cbr fallthrough */
+#define LINKSTUB_DIRECT(flags) (TEST(LINK_DIRECT, (flags)))
+#define LINKSTUB_NORMAL_DIRECT(flags) \
+    (TEST(LINK_DIRECT, (flags)) && !TEST(LINK_INDIRECT, (flags)))
+#define LINKSTUB_INDIRECT(flags) \
+    (!TEST(LINK_DIRECT, (flags)) && TEST(LINK_INDIRECT, (flags)))
+#define LINKSTUB_CBR_FALLTHROUGH(flags) \
+    (TEST(LINK_DIRECT, (flags)) && TEST(LINK_INDIRECT, (flags)))
+
+#define LINKSTUB_FINAL(l)  (TEST(LINK_END_OF_LIST, (l)->flags))
+
+/* We assume this combination of flags is unique for coarse proxies. */
+#define LINKSTUB_COARSE_PROXY(flags) \
+    (TESTALL(LINK_FAKE|LINK_DIRECT|LINK_SEPARATE_STUB, (flags)))
+
+/* could optimize to avoid redundant tests */
+#define LINKSTUB_SIZE(l) \
+    (LINKSTUB_NORMAL_DIRECT((l)->flags) ? sizeof(direct_linkstub_t) : \
+     (LINKSTUB_INDIRECT((l)->flags) ? sizeof(indirect_linkstub_t) : \
+      (LINKSTUB_CBR_FALLTHROUGH((l)->flags) ? sizeof(cbr_fallthrough_linkstub_t) : \
+       sizeof(linkstub_t))))
+
+#define LINKSTUB_NEXT_EXIT(l) \
+    ((LINKSTUB_FINAL((l))) ? NULL : \
+        ((linkstub_t *) (((byte*)(l)) + LINKSTUB_SIZE(l))))
+
+/* we pay the cost of the check in release builds to have the
+ * safety return value of NULL
+ */
+#define LINKSTUB_NEXT_INCOMING(l) \
+    (LINKSTUB_NORMAL_DIRECT((l)->flags) ? \
+     (((direct_linkstub_t *)(l))->cdl.next_incoming) : \
+     (LINKSTUB_CBR_FALLTHROUGH((l)->flags) ? \
+      (((cbr_fallthrough_linkstub_t *)(l))->cdl.next_incoming) : \
+      (ASSERT(false && "indirect linkstub has no next_incoming"), NULL)))
+
+/* if sharing a stub then no offs, else offs to get to subsequent stub */
+#define CBR_FALLTHROUGH_STUB_OFFS(f) \
+    (INTERNAL_OPTION(cbr_single_stub) ? 0 : DIRECT_EXIT_STUB_SIZE((f)->flags))
+
+#define EXIT_CTI_PC_HELPER(f, l) \
+    (ASSERT(LINKSTUB_NORMAL_DIRECT((l)->flags)), \
+     ((f)->start_pc + (l)->cti_offset))
+
+#define EXIT_CTI_PC(f, l) \
+    (LINKSTUB_CBR_FALLTHROUGH((l)->flags) ? \
+     (cbr_fallthrough_exit_cti(EXIT_CTI_PC_HELPER(f, FRAGMENT_EXIT_STUBS(f)))) : \
+     ((f)->start_pc + (l)->cti_offset))
+
+#define EXIT_STUB_PC_HELPER(dc, f, l) \
+    (ASSERT(LINKSTUB_NORMAL_DIRECT((l)->flags)), \
+     (((direct_linkstub_t *)(l))->stub_pc))
+
+#define EXIT_STUB_PC(dc, f, l) \
+    (LINKSTUB_NORMAL_DIRECT((l)->flags) ? (((direct_linkstub_t *)(l))->stub_pc) : \
+     (LINKSTUB_CBR_FALLTHROUGH((l)->flags) ? \
+      (EXIT_STUB_PC_HELPER(dc, f, FRAGMENT_EXIT_STUBS(f)) \
+       + CBR_FALLTHROUGH_STUB_OFFS(f)) :                  \
+       indirect_linkstub_stub_pc(dc, f, l)))
+
+#define EXIT_TARGET_TAG(dc, f, l) \
+    (LINKSTUB_NORMAL_DIRECT((l)->flags) ? (((direct_linkstub_t *)(l))->target_tag) : \
+     (LINKSTUB_CBR_FALLTHROUGH((l)->flags) ? \
+      ((f)->tag + ((short)(l)->cti_offset)) : \
+       indirect_linkstub_target(dc, f, l)))
+
+#ifdef CUSTOM_EXIT_STUBS
+# define EXIT_FIXED_STUB_PC(dc, f, l) (EXIT_STUB_PC(dc, f, l) + (l)->fixed_stub_offset)
+#endif
+
+#ifdef WINDOWS
+# define EXIT_TARGETS_SHARED_SYSCALL(flags) \
+    (DYNAMO_OPTION(shared_syscalls) &&     \
+     !TESTANY(LINK_RETURN|LINK_CALL|LINK_JMP|LINK_IND_JMP_PLT, (flags)))
+#else
+# define EXIT_TARGETS_SHARED_SYSCALL(flags) (false)
+#endif
+
+/* indirect exits w/o inlining have no stub at all for -no_indirect_stubs */
+#define EXIT_HAS_STUB(l_flags, f_flags)                                       \
+    (DYNAMO_OPTION(indirect_stubs) || !LINKSTUB_INDIRECT((l_flags)) ||      \
+     (!EXIT_TARGETS_SHARED_SYSCALL(l_flags) &&                                \
+      ((DYNAMO_OPTION(inline_trace_ibl) && TEST(FRAG_IS_TRACE, (f_flags))) || \
+       (DYNAMO_OPTION(inline_bb_ibl) && !TEST(FRAG_IS_TRACE, (f_flags))))))
+
+/* two cases with no local stub: a separate stub or no stub at all */
+#define EXIT_HAS_LOCAL_STUB(l_flags, f_flags) \
+    (EXIT_HAS_STUB(l_flags, f_flags) && !TEST(LINK_SEPARATE_STUB, (l_flags)))
+
+void link_init(void);
+void link_exit(void);
+void link_reset_init(void);
+void link_reset_free(void);
+void link_thread_init(dcontext_t *dcontext);
+void link_thread_exit(dcontext_t *dcontext);
+
+/* coarse-grain support */
+
+cache_pc
+coarse_stubs_create(coarse_info_t *info, cache_pc pc, size_t size);
+
+void
+coarse_stubs_delete(coarse_info_t *info);
+
+uint
+coarse_stub_alignment(coarse_info_t *info);
+
+void
+coarse_mark_trace_head(dcontext_t *dcontext, fragment_t *f,
+                       coarse_info_t *info, cache_pc stub, cache_pc body);
+
+void
+coarse_unit_unlink(dcontext_t *dcontext, coarse_info_t *info);
+
+void
+coarse_unit_unlink_outgoing(dcontext_t *dcontext, coarse_info_t *info);
+
+#ifdef DEBUG
+bool
+coarse_unit_outgoing_linked(dcontext_t *dcontext, coarse_info_t *info);
+#endif
+
+cache_pc
+coarse_stub_lookup_by_target(dcontext_t *dcontext, coarse_info_t *info,
+                             cache_pc target_tag);
+
+void
+coarse_lazy_link(dcontext_t *dcontext, fragment_t *targetf);
+
+cache_pc
+fcache_return_coarse_prefix(cache_pc stub_pc, coarse_info_t *info /*OPTIONAL*/);
+
+cache_pc
+trace_head_return_coarse_prefix(cache_pc stub_pc, coarse_info_t *info /*OPTIONAL*/);
+
+cache_pc
+get_coarse_ibl_prefix(dcontext_t *dcontext, cache_pc stub_pc,
+                      ibl_branch_type_t branch_type);
+
+bool
+in_coarse_stubs(cache_pc pc);
+
+bool
+in_coarse_stub_prefixes(cache_pc pc);
+
+cache_pc
+coarse_deref_ibl_prefix(dcontext_t *dcontext, cache_pc target);
+
+coarse_info_t *
+get_stub_coarse_info(cache_pc pc);
+
+/* Initializes an array of linkstubs beginning with first */
+void linkstubs_init(linkstub_t *first, int num_direct, int num_indirect, fragment_t *f);
+
+bool is_linkable(dcontext_t *dcontext, fragment_t *from_f, linkstub_t *l,
+                 fragment_t *targetf, bool have_link_lock, bool mark_new_trace_head);
+
+uint linkstub_size(dcontext_t *dcontext, fragment_t *f, linkstub_t *l);
+
+uint linkstub_propagatable_flags(uint flags);
+
+bool linkstub_frag_offs_at_end(uint flags, int direct_exits, int indirect_exits);
+
+bool use_cbr_fallthrough_short(uint flags, int direct_exits, int indirect_exits);
+
+uint linkstubs_heap_size(uint flags, int direct_exits, int indirect_exits);
+
+fragment_t *linkstub_fragment(dcontext_t *dcontext, linkstub_t *l);
+
+#ifdef X64
+/* Converts the canonical empty fragment to an empty fragment marked FRAG_32_BIT */
+fragment_t *empty_fragment_mark_x86(fragment_t *f);
+#endif
+
+#ifdef DEBUG
+bool linkstub_owned_by_fragment(dcontext_t *dcontext, fragment_t *f, linkstub_t *l);
+#endif
+
+void set_last_exit(dcontext_t *dcontext, linkstub_t *l);
+void last_exit_deleted(dcontext_t *dcontext);
+void set_coarse_ibl_exit(dcontext_t *dcontext);
+int get_last_linkstub_ordinal(dcontext_t *dcontext);
+
+linkstub_t * get_deleted_linkstub(dcontext_t *dcontext);
+const linkstub_t * get_starting_linkstub(void);
+const linkstub_t * get_reset_linkstub(void);
+const linkstub_t * get_syscall_linkstub(void);
+const linkstub_t * get_selfmod_linkstub(void);
+const linkstub_t * get_ibl_deleted_linkstub(void);
+#ifdef LINUX
+const linkstub_t * get_sigreturn_linkstub(void);
+#else /* WINDOWS */
+const linkstub_t * get_asynch_linkstub(void);
+#endif
+const linkstub_t * get_native_exec_linkstub(void);
+const linkstub_t * get_native_exec_syscall_linkstub(void);
+#ifdef HOT_PATCHING_INTERFACE
+const linkstub_t * get_hot_patch_linkstub(void);
+#endif
+#ifdef CLIENT_INTERFACE
+const linkstub_t * get_client_linkstub(void);
+#endif
+const linkstub_t * get_ibl_sourceless_linkstub(uint link_flags, uint frag_flags);
+bool is_ibl_sourceless_linkstub(const linkstub_t *l);
+const linkstub_t * get_coarse_exit_linkstub(void);
+const linkstub_t * get_coarse_trace_head_exit_linkstub(void);
+
+# define IS_COARSE_LINKSTUB(l) ((l) == get_coarse_exit_linkstub() || \
+    (l) == get_coarse_trace_head_exit_linkstub())
+
+#ifdef WINDOWS
+const linkstub_t * get_shared_syscalls_unlinked_linkstub(void);
+# define IS_SHARED_SYSCALLS_UNLINKED_LINKSTUB(l)       \
+    ((l) == get_shared_syscalls_unlinked_linkstub())
+
+const linkstub_t * get_shared_syscalls_trace_linkstub(void);
+const linkstub_t * get_shared_syscalls_bb_linkstub(void);
+# define IS_SHARED_SYSCALLS_LINKSTUB(l)               \
+      ((l) == get_shared_syscalls_trace_linkstub() || \
+       (l) == get_shared_syscalls_bb_linkstub() ||    \
+       (l) == get_shared_syscalls_unlinked_linkstub())
+# define IS_SHARED_SYSCALLS_TRACE_LINKSTUB(l)         \
+      ((l) == get_shared_syscalls_trace_linkstub())
+#else
+# define IS_SHARED_SYSCALLS_UNLINKED_LINKSTUB(l) false
+# define IS_SHARED_SYSCALLS_LINKSTUB(l) false
+# define IS_SHARED_SYSCALLS_TRACE_LINKSTUB(l) false
+#endif
+
+bool should_separate_stub(dcontext_t *dcontext, app_pc target, uint fragment_flags);
+int local_exit_stub_size(dcontext_t *dcontext, app_pc target, uint fragment_flags);
+void separate_stub_create(dcontext_t *dcontext, fragment_t *f, linkstub_t *l);
+void linkstub_free_exitstubs(dcontext_t *dcontext, fragment_t *f);
+
+void linkstubs_shift(dcontext_t *dcontext, fragment_t *f, ssize_t shift);
+
+void link_new_fragment(dcontext_t *dcontext, fragment_t *f);
+void link_fragment_outgoing(dcontext_t *dcontext, fragment_t *f, bool new_fragment);
+void unlink_fragment_outgoing(dcontext_t *dcontext, fragment_t *f);
+void link_fragment_incoming(dcontext_t *dcontext, fragment_t *f, bool new_fragment);
+void unlink_fragment_incoming(dcontext_t *dcontext, fragment_t *f);
+
+void shift_links_to_new_fragment(dcontext_t *dcontext, fragment_t *old_f,
+                                 fragment_t *new_f);
+future_fragment_t * incoming_remove_fragment(dcontext_t *dcontext, fragment_t *f);
+
+/* if this linkstub shares the stub with the next linkstub, returns the
+ * next linkstub; else returns NULL
+ */
+linkstub_t *
+linkstub_shares_next_stub(dcontext_t *dcontext, fragment_t *f, linkstub_t *l);
+
+/* Returns the total size needed for stubs if info is frozen. */
+size_t
+coarse_frozen_stub_size(dcontext_t *dcontext, coarse_info_t *info,
+                        uint *num_fragments, uint *num_stubs);
+
+/* Removes any incoming data recording the outgoing link from stub */
+void
+coarse_remove_outgoing(dcontext_t *dcontext, cache_pc stub, coarse_info_t *src_info);
+
+void
+coarse_update_outgoing(dcontext_t *dcontext, cache_pc old_stub, cache_pc new_stub,
+                       coarse_info_t *src_info, bool replace);
+
+void
+coarse_unit_shift_links(dcontext_t *dcontext, coarse_info_t *info);
+
+/* Updates the info pointers embedded in the coarse_stub_areas vector */
+void
+coarse_stubs_set_info(coarse_info_t *info);
+
+/* Sets the final used pc in a frozen stub region */
+void
+coarse_stubs_set_end_pc(coarse_info_t *info, byte *end_pc);
+
+#endif /* _LINK_H_ */
diff --git a/core/linux/Makefile b/core/linux/Makefile
new file mode 100644
index 000000000..9c650bfb6
--- /dev/null
+++ b/core/linux/Makefile
@@ -0,0 +1,45 @@
+# **********************************************************
+# Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# Copyright (c) 2003-2007 Determina Corp.
+
+# local submodule Makefile wrapper
+#   keep in sync the copies in win32 and linux
+# execute any rule using the top level makefile
+
+# You can actually override any rule here, 
+# but currently there is no reason to do so
+
+all:
+
+# for any action we go to the master Makefile
+%: force
+	@$(MAKE) -C .. $@
+force: ;
diff --git a/core/linux/diagnost.c b/core/linux/diagnost.c
new file mode 100644
index 000000000..d5573c2d7
--- /dev/null
+++ b/core/linux/diagnost.c
@@ -0,0 +1,63 @@
+/* **********************************************************
+ * Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+
+#include "globals.h"
+
+/* Collects and displays diagnostic information.
+ * FIXME: Stub for future development; diagnostics not supported on linux (yet)
+ */
+void 
+report_diagnostics(IN const char *message,
+                   IN const char *name,
+                   security_violation_t violation_type)
+{
+    /* FIXME: Diagnostics not implemented for Linux */
+}
+
+void
+append_diagnostics(file_t diagnostics_file, const char *message, const char *name,
+                   security_violation_t violation_type)
+{
+}
+
+void
+diagnost_exit()
+{
+}
+
+bool
+check_for_unsupported_modules()
+{
+    return false;
+}
diff --git a/core/linux/module.c b/core/linux/module.c
new file mode 100644
index 000000000..a1e671772
--- /dev/null
+++ b/core/linux/module.c
@@ -0,0 +1,591 @@
+/* **********************************************************
+ * Copyright (c) 2008-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "globals.h"
+#include "module_shared.h"
+#include "os_private.h"
+#include <elf.h>    /* for ELF types */
+#include <string.h>
+
+
+/* Question : how is the size of the initial map determined?  There seems to be no better
+ * way than to walk the program headers and find the largest virtual offset.  You'd think
+ * there would be a field in the header or something easier then that...
+ *
+ * Generally the section headers will be unavailable to us unless we go to disk 
+ * (investigate, pursuant to the answer to the above question being large enough to
+ * always include the section table, might they be visible briefly during the
+ * first map before the program headers are processed and re-map/bss overwrites? Probably
+ * would depend on the .bss being large enough, grr....), but at least the elf header and
+ * program headers should be in memory.
+ * 
+ * So to determine individual sections probably have to go to disk, but could try to 
+ * backtrack some of them out from program headers which need to point to plt relocs
+ * etc.
+ */
+
+
+/* Is there an ELF header for a shared object at address 'base'? 
+ * If size == 0 then checks for header readability else assumes that size bytes from
+ * base are readable (unmap races are then callers responsibility). */
+bool
+is_elf_so_header(app_pc base, size_t size)
+{
+    /* FIXME We could check more fields in the header just as the
+     * dlopen() does. */
+    static const unsigned char ei_expected[SELFMAG] = {
+        [EI_MAG0] = ELFMAG0,
+        [EI_MAG1] = ELFMAG1,
+        [EI_MAG2] = ELFMAG2,
+        [EI_MAG3] = ELFMAG3
+    };
+    ELF_HEADER_TYPE elf_header;
+
+    if (base == NULL) {
+        ASSERT(false && "is_elf_so_header(): NULL base");
+        return false;
+    }
+
+    /* read the header */
+    if (size >= sizeof(ELF_HEADER_TYPE)) {
+        elf_header = *(ELF_HEADER_TYPE *)base;
+    } else if (size == 0) {
+        if (!safe_read(base, sizeof(ELF_HEADER_TYPE), &elf_header))
+            return false;
+    } else {
+        return false;
+    }
+
+    /* We check the first 4 bytes which is the magic number. */
+    if ((size == 0 || size >= sizeof(ELF_HEADER_TYPE)) &&
+        elf_header.e_ident[EI_MAG0] == ei_expected[EI_MAG0] &&
+        elf_header.e_ident[EI_MAG1] == ei_expected[EI_MAG1] &&
+        elf_header.e_ident[EI_MAG2] == ei_expected[EI_MAG2] &&
+        elf_header.e_ident[EI_MAG3] == ei_expected[EI_MAG3]) {
+        /* FIXME - should we add any of these to the check? For real 
+         * modules all of these should hold. */
+        ASSERT_CURIOSITY(elf_header.e_version == 1);
+        ASSERT_CURIOSITY(elf_header.e_ehsize == sizeof(ELF_HEADER_TYPE));
+        ASSERT_CURIOSITY(elf_header.e_ident[EI_OSABI] == ELFOSABI_SYSV);
+        ASSERT_CURIOSITY(elf_header.e_type == ET_DYN ||
+                         elf_header.e_type == ET_EXEC);
+#ifdef X64
+        ASSERT_CURIOSITY(elf_header.e_machine == EM_X86_64);
+#else
+        ASSERT_CURIOSITY(elf_header.e_machine == EM_386);
+#endif
+        return true;
+    }
+    return false;
+}
+
+void
+os_modules_init(void)
+{
+    /* nothing */
+}
+
+void
+os_modules_exit(void)
+{
+    /* nothing */
+}
+
+/* Returned addresses out_base and out_end are relative to the actual
+ * loaded module base, so the "base" param should be added to produce
+ * absolute addresses.
+ */
+bool
+module_walk_program_headers(app_pc base, size_t view_size, bool at_map,
+                            OUT app_pc *out_base /* relative pc */,
+                            OUT app_pc *out_end /* relative pc */,
+                            OUT char **out_soname,
+                            OUT size_t *out_align)
+{
+    app_pc mod_base = (app_pc) POINTER_MAX, mod_end = (app_pc)0;
+    char *soname = NULL;
+    bool found_load = false;
+    ELF_HEADER_TYPE *elf_hdr = (ELF_HEADER_TYPE *) base;
+    ASSERT(is_elf_so_header(base, view_size));    
+
+    /* On adjusting virtual address in the elf headers -
+     * To compute the base address, one determines the memory address associated with
+     * the lowest p_vaddr value for a PT_LOAD segment. One then obtains the base
+     * address by truncating the memory address to the nearest multiple of the
+     * maximum page size and subtracting the truncated lowest p_vaddr value.
+     * All virtual addresses are assuming the module is loaded at its
+     * base address. */
+    ASSERT_CURIOSITY(elf_hdr->e_phoff != 0 &&
+                     elf_hdr->e_phoff + elf_hdr->e_phnum * elf_hdr->e_phentsize <=
+                     view_size);
+    if (elf_hdr->e_phoff != 0 &&
+        elf_hdr->e_phoff + elf_hdr->e_phnum * elf_hdr->e_phentsize <= view_size) {
+        /* walk the program headers */
+        uint i;
+        ASSERT_CURIOSITY(elf_hdr->e_phentsize == sizeof(ELF_PROGRAM_HEADER_TYPE));
+        /* we need mod_base and mod_end to be fully computed for use in reading
+         * out_soname, so we do a full segment walk up front
+         */
+        mod_base = module_vaddr_from_prog_header(base + elf_hdr->e_phoff,
+                                                 elf_hdr->e_phnum, &mod_end);
+        /* now we do our own walk */
+        for (i = 0; i < elf_hdr->e_phnum; i++) {
+            ELF_PROGRAM_HEADER_TYPE *prog_hdr = (ELF_PROGRAM_HEADER_TYPE *)
+                (base + elf_hdr->e_phoff + i * elf_hdr->e_phentsize);
+            if (prog_hdr->p_type == PT_LOAD) {
+                if (out_align != NULL) {
+                    if (!found_load) {
+                        *out_align = prog_hdr->p_align;
+                    } else {
+                        /* We expect all segments to have the same alignment */
+                        ASSERT_CURIOSITY(*out_align == prog_hdr->p_align);
+                    }
+                }
+                found_load = true;
+            }
+            if (out_soname != NULL && prog_hdr->p_type == PT_DYNAMIC) {
+                /* if at_map use file offset as segments haven't been remapped yet and
+                 * the dynamic section isn't usually in the first segment (FIXME in
+                 * theory it's possible to construct a file where the dynamic section
+                 * isn't mapped in as part of the initial map because large parts of the
+                 * initial portion of the file aren't part of the in memory image which
+                 * is fixed up with a PT_LOAD).
+                 * 
+                 * If not at_map use virtual address adjusted for possible loading not
+                 * at base. */
+                /* FIXME: somehow dl_iterate_phdr() implies that this offset can
+                 * be unsigned, but I don't trust that: probably only works b/c
+                 * of wraparound arithmetic
+                 */
+                ssize_t offset = base - mod_base; /* offset loaded at relative to base */
+                ELF_DYNAMIC_ENTRY_TYPE *dyn = (ELF_DYNAMIC_ENTRY_TYPE *)
+                    (at_map ? base + prog_hdr->p_offset :
+                     (app_pc)prog_hdr->p_vaddr + offset);
+                dcontext_t *dcontext = get_thread_private_dcontext();
+
+                TRY_EXCEPT(dcontext, {
+                    int soname_index = -1;
+                    char *strtab = NULL;
+                    while (dyn->d_tag != DT_NULL) {
+                        if (dyn->d_tag == DT_SONAME) {
+                            soname_index = dyn->d_un.d_val;
+                            if (strtab != NULL)
+                                break;
+                        }
+                        if (dyn->d_tag == DT_STRTAB) {
+                            /* FIXME - if at_map this needs to be adjusted if not
+                             * in the first segment since we haven't re-mapped later
+                             * ones yet. Since it's read only I've never seen it
+                             * not be in the first segment, but should fix or at
+                             * least check. PR 307610. */
+                            
+                            /* Xref PR 307687 - on some machines (for already loaded
+                             * modules) someone (presumably the loader?) has relocated
+                             * this address.  The Elf spec is adamant that dynamic
+                             * entry addresses shouldn't have relocations (for
+                             * consistency) so must be the loader doing it on its
+                             * own (same .so on different machines will be different
+                             * here).  FIXME - how can we reliably tell if it has
+                             * been relocated or not?  We can check against the
+                             * module bounds, but if it is loaded at a small offset
+                             * that's potentially ambiguous. No other real option
+                             * though so we'll stick to that and default to already
+                             * relocated (which seems to be the case for the newer
+                             * ld versions). */
+                            strtab = (char *)dyn->d_un.d_ptr;
+                            if (at_map || (app_pc)strtab < base ||
+                                (app_pc)strtab > base + (mod_end - mod_base)) {
+                                /* not relocated, adjust by offset */
+                                strtab = (char *)dyn->d_un.d_ptr + offset;
+                            }
+                            
+                            /* sanity check strtab location */
+                            if ((app_pc)strtab < base ||
+                                (app_pc)strtab > base + (mod_end - mod_base)) {
+                                ASSERT_CURIOSITY(false && "strtab not in module");
+                                strtab = NULL;
+                            } else if (at_map && (app_pc)strtab > base + view_size) {
+                                ASSERT_CURIOSITY(false && "strtab not in inital map");
+                                strtab = NULL;
+                            }
+                            
+                            if (soname_index != -1)
+                                break;
+                        }
+                        dyn++;
+                    }
+                    if (soname_index != -1 && strtab != NULL) {
+                        soname = strtab + soname_index;
+
+                        /* sanity check soname location */
+                        if ((app_pc)soname < base ||
+                            (app_pc)soname > base + (mod_end - mod_base)) {
+                            ASSERT_CURIOSITY(false && "soname not in module");
+                            soname = NULL;
+                        } else if (at_map && (app_pc)soname < base + view_size) {
+                            ASSERT_CURIOSITY(false && "soname not in inital map");
+                            soname = NULL;
+                        }
+
+                        /* test string readability while still in try/except
+                         * in case we screwed up somewhere or module is
+                         * malformed/only partially mapped */
+                        if (soname != NULL && strlen(soname) == -1) {
+                            ASSERT_NOT_REACHED();
+                        }
+                    }
+                } , { /* EXCEPT */
+                    ASSERT_CURIOSITY(false && "crashed while walking dynamic header");
+                    soname = NULL;
+                });
+            }
+        }
+    }
+    ASSERT_CURIOSITY(found_load && mod_base != (app_pc)POINTER_MAX &&
+                     mod_end != (app_pc)0);
+    ASSERT_CURIOSITY(mod_end > mod_base);
+    if (out_base != NULL)
+        *out_base = mod_base;
+    if (out_end != NULL)
+        *out_end = mod_end;
+    if (out_soname != NULL)
+        *out_soname = soname;
+    return found_load;
+}
+
+uint
+module_num_program_headers(app_pc base)
+{
+    ELF_HEADER_TYPE *elf_hdr = (ELF_HEADER_TYPE *) base;
+    ASSERT(is_elf_so_header(base, 0));    
+    return elf_hdr->e_phnum;
+}
+
+/* Returns the minimum p_vaddr field, aligned to page boundaries, in
+ * the loadable segments in the prog_header array, or POINTER_MAX if
+ * there are no loadable segments.
+ */
+app_pc
+module_vaddr_from_prog_header(app_pc prog_header, uint num_segments,
+                              OUT app_pc *out_end)
+{
+    uint i;
+    app_pc min_vaddr = (app_pc) POINTER_MAX;
+    app_pc mod_end = (app_pc) PTR_UINT_0;
+    for (i = 0; i < num_segments; i++) {
+        /* Without the ELF header we use sizeof instead of elf_hdr->e_phentsize
+         * which must be a reliable assumption as dl_iterate_phdr() doesn't
+         * bother to deliver the entry size.
+         */
+        ELF_PROGRAM_HEADER_TYPE *prog_hdr = (ELF_PROGRAM_HEADER_TYPE *)
+            (prog_header + i * sizeof(ELF_PROGRAM_HEADER_TYPE));
+        if (prog_hdr->p_type == PT_LOAD) {
+            /* ELF requires p_vaddr to already be aligned to p_align */
+            min_vaddr =
+                MIN(min_vaddr, (app_pc) ALIGN_BACKWARD(prog_hdr->p_vaddr, PAGE_SIZE));
+            mod_end =
+                MAX(mod_end, (app_pc)
+                    ALIGN_FORWARD(prog_hdr->p_vaddr + prog_hdr->p_memsz, PAGE_SIZE));
+        }
+    }
+    if (out_end != NULL)
+        *out_end = mod_end;
+    return min_vaddr;
+}
+
+bool
+module_read_program_header(app_pc base,
+                           uint segment_num,
+                           OUT app_pc *segment_base /* relative pc */,
+                           OUT app_pc *segment_end /* relative pc */,
+                           OUT uint *segment_prot,
+                           OUT size_t *segment_align)
+{
+    ELF_HEADER_TYPE *elf_hdr = (ELF_HEADER_TYPE *) base;
+    ELF_PROGRAM_HEADER_TYPE *prog_hdr;
+    ASSERT(is_elf_so_header(base, 0));    
+    if (elf_hdr->e_phoff != 0) {
+        ASSERT_CURIOSITY(elf_hdr->e_phentsize == sizeof(ELF_PROGRAM_HEADER_TYPE));
+        prog_hdr = (ELF_PROGRAM_HEADER_TYPE *)
+            (base + elf_hdr->e_phoff + segment_num * elf_hdr->e_phentsize);
+        if (prog_hdr->p_type == PT_LOAD) {
+            /* ELF requires p_vaddr to already be aligned to p_align */
+            if (segment_base != NULL)
+                *segment_base = (app_pc) prog_hdr->p_vaddr;
+            /* up to caller to align end if desired */
+            if (segment_end != NULL) {
+                *segment_end = (app_pc) (prog_hdr->p_vaddr + prog_hdr->p_memsz);
+            }
+            if (segment_prot != NULL) {
+                *segment_prot = 0;
+                if (TEST(PF_X, prog_hdr->p_flags))
+                    *segment_prot |= MEMPROT_EXEC;
+                if (TEST(PF_W, prog_hdr->p_flags))
+                    *segment_prot |= MEMPROT_WRITE;
+                if (TEST(PF_R, prog_hdr->p_flags))
+                    *segment_prot |= MEMPROT_READ;
+            }
+            if (segment_align != NULL)
+                *segment_align = prog_hdr->p_align;
+            return true;
+        }
+    }
+    return false;
+}
+
+void
+os_module_area_init(module_area_t *ma, app_pc base, size_t view_size,
+                    bool at_map, uint64 inode, const char *filename
+                    HEAPACCT(which_heap_t which))
+{
+    app_pc mod_base, mod_end;
+    int offset;
+    char *soname = NULL;
+    ELF_HEADER_TYPE *elf_hdr = (ELF_HEADER_TYPE *) base;
+    ASSERT(is_elf_so_header(base, view_size));  
+
+    module_walk_program_headers(base, view_size, at_map,
+                                &mod_base, &mod_end, &soname, &ma->os_data.alignment);
+
+    /* expect to map whole module */
+    /* XREF 307599 on rounding module end to the next PAGE boundary */
+    ASSERT_CURIOSITY(mod_end - mod_base == at_map ?
+                     ALIGN_FORWARD(view_size, PAGE_SIZE) : view_size);
+
+    ma->os_data.base_address = mod_base;
+    offset = base - mod_base;
+
+    ma->entry_point = (app_pc)elf_hdr->e_entry + offset;
+
+    /* names - note os.c callers don't distinguish between no filename and an empty
+     * filename, we treat both as NULL, but leave the distinction for SONAME. */
+    if (filename == NULL || filename[0] == '\0')
+        ma->names.file_name = NULL;
+    else
+        ma->names.file_name = dr_strdup(get_short_name(filename) HEAPACCT(which));
+    ma->names.inode = inode;
+    if (soname == NULL)
+        ma->names.module_name = NULL;
+    else
+        ma->names.module_name = dr_strdup(soname HEAPACCT(which));
+}
+
+void
+free_module_names(module_names_t *mod_names HEAPACCT(which_heap_t which))
+{
+    ASSERT(mod_names != NULL);
+
+    if (mod_names->module_name != NULL)
+        dr_strfree(mod_names->module_name HEAPACCT(which));
+    if (mod_names->file_name != NULL)
+        dr_strfree(mod_names->file_name HEAPACCT(which));
+}
+
+void
+print_modules(file_t f, bool dump_xml)
+{
+    module_iterator_t *mi;
+
+    /* we walk our own module list that is populated on an initial walk through memory,
+     * and further kept consistent on memory mappings of likely modules */
+    print_file(f, dump_xml ? "<loaded-modules>\n" : "\nLoaded modules:\n");
+
+    mi = module_iterator_start();
+    while (module_iterator_hasnext(mi)) {
+        module_area_t *ma = module_iterator_next(mi);
+        print_file(f, dump_xml ? 
+                   "\t<so range=\""PFX"-"PFX"\" "
+                   "entry=\""PFX"\" base_address="PFX"\n"
+                   "\tname=\"%s\" />\n" :
+                   "  "PFX"-"PFX" entry="PFX" base_address="PFX"\n"
+                   "\tname=\"%s\" \n",
+                   ma->start, ma->end - 1, /* inclusive */
+                   ma->entry_point, ma->os_data.base_address,
+                   GET_MODULE_NAME(&ma->names) == NULL ?
+                       "(null)" : GET_MODULE_NAME(&ma->names));
+    }
+    module_iterator_stop(mi);
+
+    if (dump_xml)
+        print_file(f, "</loaded-modules>\n");
+    else
+        print_file(f, "\n");
+}
+
+void
+os_module_area_reset(module_area_t *ma HEAPACCT(which_heap_t which))
+{
+    /* nothing */
+}
+
+generic_func_t
+get_proc_address(module_handle_t lib, const char *name)
+{
+    ASSERT(is_elf_so_header((app_pc)lib, 0));
+    ASSERT_NOT_IMPLEMENTED(false && "PR 307607");
+    return NULL;
+}
+
+/* Returns the bounds of the first section with matching name. */
+bool
+get_named_section_bounds(app_pc module_base, const char *name,
+                         app_pc *start/*OUT*/, app_pc *end/*OUT*/)
+{
+    ASSERT(is_elf_so_header(module_base, 0));
+    if (start != NULL)
+        *start = NULL;
+    if (end != NULL)
+        *end = NULL;
+    return false;
+}
+
+bool
+rct_is_exported_function(app_pc tag)
+{
+    /* FIXME: not implemented */
+    return false;
+}
+
+
+/* FIXME PR 295529: NYI, here so code origins policies aren't all ifdef WINDOWS */
+const char *
+get_module_short_name(app_pc pc HEAPACCT(which_heap_t which))
+{
+    ASSERT_NOT_IMPLEMENTED(false);
+    return NULL;
+}
+
+/* FIXME PR 295529: NYI, here so moduledb code isn't all ifdef WINDOWS */
+bool
+get_module_company_name(app_pc mod_base, char *out_buf, size_t out_buf_size)
+{
+    ASSERT_NOT_IMPLEMENTED(false);
+    return false;
+}
+
+/* FIXME PR 295529: NYI, here so code origins policies aren't all ifdef WINDOWS */
+app_pc
+get_module_base(app_pc pc)
+{
+    return NULL;
+    ASSERT_NOT_IMPLEMENTED(false);
+}
+
+/* FIXME PR 212458: NYI, here so code origins policies aren't all ifdef WINDOWS */
+bool
+is_range_in_code_section(app_pc module_base, app_pc start_pc, app_pc end_pc,
+                         app_pc *sec_start /* OUT */, app_pc *sec_end /* OUT */)
+{
+    ASSERT_NOT_IMPLEMENTED(false);
+    return false;
+}
+
+/* FIXME PR 212458: NYI, here so code origins policies aren't all ifdef WINDOWS */
+bool
+is_in_code_section(app_pc module_base, app_pc addr,
+                   app_pc *sec_start /* OUT */, app_pc *sec_end /* OUT */)
+{
+    ASSERT_NOT_IMPLEMENTED(false);
+    return false;
+}
+
+/* FIXME PR 212458: NYI, here so code origins policies aren't all ifdef WINDOWS */
+bool
+is_in_dot_data_section(app_pc module_base, app_pc addr,
+                       app_pc *sec_start /* OUT */, app_pc *sec_end /* OUT */)
+{
+    return false;
+    ASSERT_NOT_IMPLEMENTED(false);
+}
+
+/* FIXME PR 212458: NYI, here so code origins policies aren't all ifdef WINDOWS */
+bool
+is_in_any_section(app_pc module_base, app_pc addr,
+                  app_pc *sec_start /* OUT */, app_pc *sec_end /* OUT */)
+{
+    ASSERT_NOT_IMPLEMENTED(false);
+    return false;
+}
+
+/* FIXME PR 295529: NYI, here so code origins policies aren't all ifdef WINDOWS */
+bool
+is_mapped_as_image(app_pc module_base)
+{
+    ASSERT_NOT_IMPLEMENTED(false);
+    return false;
+}
+
+
+/* FIXME PR 295529: NYI, present so that all hot patching code aren't all ifdef WINDOWS */
+bool
+os_get_module_info(const app_pc pc, uint *checksum, uint *timestamp,
+                   size_t *size, const char **pe_name, size_t *code_size,
+                   uint64 *file_version)
+{
+    return false;
+    ASSERT_NOT_IMPLEMENTED(false); /* yes I know it's dead: leave it here (case 10703) */
+}
+
+bool
+os_get_module_info_all_names(const app_pc pc, uint *checksum, uint *timestamp,
+                             size_t *size, module_names_t **names,
+                             size_t *code_size, uint64 *file_version)
+{
+    ASSERT_NOT_IMPLEMENTED(false);
+    return false;
+}
+
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+extern rct_module_table_t rct_global_table;
+
+/* Caller must hold module_data_lock */
+rct_module_table_t *
+os_module_get_rct_htable(app_pc pc, rct_type_t which)
+{
+    /* FIXME: until we have a module list we use global rct and rac tables */
+    if (which == RCT_RCT)
+        return &rct_global_table;
+    return NULL; /* we use rac_non_module_table */
+}
+#endif
+
+void
+module_calculate_digest(/*OUT*/ module_digest_t *digest,
+                        app_pc module_base, 
+                        size_t module_size,
+                        bool full_digest,
+                        bool short_digest,
+                        uint short_digest_size,
+                        uint sec_characteristics)
+{
+    ASSERT_NOT_IMPLEMENTED(false); /* FIXME PR 295534: NYI */
+}
diff --git a/core/linux/module.h b/core/linux/module.h
new file mode 100644
index 000000000..ac0faac6b
--- /dev/null
+++ b/core/linux/module.h
@@ -0,0 +1,65 @@
+/* **********************************************************
+ * Copyright (c) 2008-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifndef MODULE_H
+#define MODULE_H
+
+/* FIXME - can we have 32bit and 64bit elf files in the same process like we see in
+ * windows WOW?  What if anything should we do to accomodate that? */
+#ifdef X64
+# define ELF_HEADER_TYPE Elf64_Ehdr
+# define ELF_PROGRAM_HEADER_TYPE Elf64_Phdr
+# define ELF_DYNAMIC_ENTRY_TYPE Elf64_Dyn
+#else
+# define ELF_HEADER_TYPE Elf32_Ehdr
+# define ELF_PROGRAM_HEADER_TYPE Elf32_Phdr
+# define ELF_DYNAMIC_ENTRY_TYPE Elf32_Dyn
+#endif
+
+/* FIXME NYI */
+#define OS_IMAGE_READ    (ASSERT_NOT_IMPLEMENTED(false), 0)
+#define OS_IMAGE_WRITE   (ASSERT_NOT_IMPLEMENTED(false), 0)
+#define OS_IMAGE_EXECUTE (ASSERT_NOT_IMPLEMENTED(false), 0)
+
+typedef struct _os_module_data_t {
+    /* To compute the base address, one determines the memory address associated with
+     * the lowest p_vaddr value for a PT_LOAD segment. One then obtains the base
+     * address by truncating the memory load address to the nearest multiple of the
+     * maximum page size and subtracting the truncated lowest p_vaddr value. 
+     * Thus, this is better thought of as the "base_offset", the relocation
+     * offset.
+     */
+    app_pc base_address; 
+    size_t alignment; /* the alignment between segments */
+} os_module_data_t;
+
+#endif /* MODULE_H */
diff --git a/core/linux/os.c b/core/linux/os.c
new file mode 100644
index 000000000..0ce7af53a
--- /dev/null
+++ b/core/linux/os.c
@@ -0,0 +1,5407 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * os.c - Linux specific routines
+ */
+
+/* for mmap-related #defines */
+#include <sys/types.h>
+#include <sys/mman.h>
+/* for open */
+#include <sys/stat.h>
+#include <fcntl.h>
+/* for pthread thread-local-storage */
+#include <pthread.h>
+#include "globals.h"
+#include <string.h>
+#include <unistd.h> /* for write and usleep and _exit */
+#include <limits.h>
+#include <sys/sysinfo.h>        /* for get_nprocs_conf */
+#include <dlfcn.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <signal.h>
+#include <syslog.h>             /* vsyslog */
+#include "vmareas.h"
+#ifdef RCT_IND_BRANCH
+#  include "rct.h"
+#endif
+#include "syscall.h"            /* our own local copy */
+#include "os_private.h"
+#include "module_shared.h"
+
+#ifdef X86
+# include "instr.h" /* for get_segment_base() */
+#endif
+
+#ifdef CLIENT_INTERFACE
+# include "instrument.h"
+#endif
+
+#ifndef HAVE_PROC_MAPS
+# define _GNU_SOURCE 1
+/* despite man page I had to define this in addition to prev line */
+# define __USE_GNU 1
+/* this relies on using -iquote instead of -I to avoid conflicts with
+ * core/link.h
+ */
+# include <link.h> /* struct dl_phdr_info */
+#endif
+
+#include <asm/ldt.h>
+/* The ldt struct in ldt.h used to be just "struct modify_ldt_ldt_s"; then that
+ * was also typdef-ed as modify_ldt_t; then it was just user_desc.
+ * To compile on old and new we inline our own copy of the struct:
+ */
+typedef struct _our_modify_ldt_t {
+    unsigned int  entry_number;
+    unsigned int  base_addr;
+    unsigned int  limit;
+    unsigned int  seg_32bit:1;
+    unsigned int  contents:2;
+    unsigned int  read_exec_only:1;
+    unsigned int  limit_in_pages:1;
+    unsigned int  seg_not_present:1;
+    unsigned int  useable:1;
+} our_modify_ldt_t;
+
+#ifndef HAVE_TLS
+/* We use a table lookup to find a thread's dcontext */
+/* Our only current no-TLS target, VMKernel (VMX86_SERVER), doesn't have apps with
+ * tons of threads anyway
+ */
+#define MAX_THREADS 512
+typedef struct _tls_slot_t {
+    thread_id_t tid;
+    dcontext_t *dcontext;
+} tls_slot_t;
+/* Stored in heap for self-prot */
+static tls_slot_t *tls_table;
+/* not static so deadlock_avoidance_unlock() can look for it */
+DECLARE_CXTSWPROT_VAR(mutex_t tls_lock, INIT_LOCK_FREE(tls_lock));
+#endif
+
+#include <stddef.h> /* for offsetof */
+
+#include <sys/utsname.h> /* for struct utsname */
+
+/* full path to our own library, used for execve */
+static char dynamorio_library_path[MAXIMUM_PATH];
+
+/* pc values delimiting dynamo dll image */
+static app_pc dynamo_dll_start = NULL;
+static app_pc dynamo_dll_end = NULL; /* open-ended */
+
+/* does the kernel provide tids that must be used to distinguish threads in a group? */
+static bool kernel_thread_groups;
+
+static bool kernel_64bit;
+
+static pid_t pid_cached;
+
+#ifdef PROFILE_RDTSC
+uint kilo_hertz; /* cpu clock speed */
+#endif
+
+/* lock to guard reads from /proc/self/maps in get_memory_info_from_os(). */
+static mutex_t memory_info_buf_lock = INIT_LOCK_FREE(memory_info_buf_lock);
+/* lock for iterator where user needs to allocate memory */
+static mutex_t maps_iter_buf_lock = INIT_LOCK_FREE(maps_iter_buf_lock);
+
+/* Xref PR 258731, dup of STDOUT/STDERR in case app wants to close them. */
+DR_API file_t our_stdout = INVALID_FILE;
+DR_API file_t our_stderr = INVALID_FILE;
+
+/* Track all memory regions seen by DR. We track these ourselves to prevent
+ * repeated reads of /proc/self/maps (case 3771). The MEMPROT_* value on a
+ * region is stored in the custom field.
+ * all_memory_areas is updated along with dynamo_areas, due to cyclic
+ * dependencies.
+ */
+static vm_area_vector_t *all_memory_areas;
+
+/* HACK to make all_memory_areas->lock recursive 
+ * protected for both read and write by all_memory_areas->lock
+ * FIXME: provide general rwlock w/ write portion recursive
+ * FIXME: eliminate duplicate code (see dynamo_areas_recursion)
+ */
+DECLARE_CXTSWPROT_VAR(uint all_memory_areas_recursion, 0);
+
+static void
+process_mmap(dcontext_t *dcontext, app_pc base, size_t size, uint prot
+             _IF_DEBUG(char *map_type));
+
+/* Iterator over /proc/self/maps
+ * Called at arbitrary places, so cannot use fopen.
+ * The whole thing is serialized, so entries cannot be referenced
+ * once the iteration ends.
+ */
+typedef struct _maps_iter_t {
+    app_pc vm_start;
+    app_pc vm_end;
+    uint prot;
+    size_t offset; /* offset into the file being mapped */
+    uint64 inode; /* FIXME - use ino_t?  We need to know what size code to use for the
+                   * scanf and I don't trust that we, the maps file, and any clients
+                   * will all agree on it's size since it seems to be defined differently
+                   * depending on whether large file support is compiled in etc.  Just
+                   * using uint64 might be safer (see also inode in module_names_t). */
+    const char *comment; /* usually file name with path, but can also be [vsdo] for the
+                          * vsyscall page */
+    /* for internal use by the iterator.  we could make these static,
+     * since we only support one concurrent iterator, but then we have
+     * .data protection complexities.
+     */
+    bool may_alloc;
+    file_t maps;
+    char *newline;
+    int bufread;
+    int bufwant;
+    char *buf;
+    char *comment_buffer;
+} maps_iter_t;
+
+static bool maps_iterator_start(maps_iter_t *iter, bool may_alloc);
+static void maps_iterator_stop(maps_iter_t *iter);
+static bool maps_iterator_next(maps_iter_t *iter);
+
+static int
+get_library_bounds(const char *name, app_pc *start/*IN/OUT*/, app_pc *end/*OUT*/,
+                   char *fullpath/*OPTIONAL OUT*/, size_t path_size);
+
+/* vsyscall page.  hardcoded at 0xffffe000 in earlier kernels, but
+ * randomly placed since fedora2.
+ * marked rx then: FIXME: should disallow this guy when that's the case!
+ * random vsyscall page is identified in maps files as "[vdso]"
+ * (kernel-provided fake shared library or Virt Dyn Shared Object)
+ */
+app_pc vsyscall_page_start = NULL;
+/* pc of the end of the syscall instr itself */
+app_pc vsyscall_syscall_end_pc = NULL;
+/* pc where kernel returns control after sysenter vsyscall */
+app_pc vsyscall_sysenter_return_pc = NULL;
+#define VSYSCALL_PAGE_START_HARDCODED ((app_pc)(ptr_uint_t) 0xffffe000)
+
+/* The pthreads library keeps errno in its pthread_descr data structure,
+ * which it looks up by dispatching on the stack pointer.  This doesn't work
+ * when within dynamo.  Thus, we define our own __errno_location() for use both
+ * by us and the app, to prevent pthreads looking at the stack pointer when
+ * out of the code cache.
+ */
+
+ /* FIXME: maybe we should create 1st dcontext earlier so we don't need init_errno?
+  * any problems with init_errno being set and then dcontext->errno being read?
+  * FIXME: if a thread issues a dr_app_stop, then we don't want to use
+  * this errno slot?  But it may later do a start...probably ok to keep using
+  * the slot.  But, when threads die, they'll all use the same init_errno!
+  */
+static int init_errno; /* errno until 1st dcontext created */
+
+/* With KEEP_SYMBOLS_FOR_LIBC_BACKTRACE our __errno_location gets interpreted
+ * in the code cache and on x64 we hit an assert on our own segment use
+ * in get_thread_private_dcontext():
+ *   "no support yet for application using non-NPTL segment"
+ * FIXME: though we used to use this for the app too: presumably we don't
+ * need that anymore, since having our own private copy is just as good
+ * to isolated DR errno changes from app errno uses.
+ */
+__attribute__ ((visibility ("hidden")))
+int *
+__errno_location() {
+    /* Each dynamo thread should have a separate errno */
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    if (dcontext == NULL)
+        return &init_errno;
+    else {
+        /* WARNING: init_errno is in data segment so can be RO! */
+        return &(dcontext->upcontext_ptr->errno);
+    }
+}
+
+/* N.B.: pthreads has two other locations it keeps on a per-thread basis:
+ * h_errno and res_state.  See glibc-2.2.4/linuxthreads/errno.c.
+ * If dynamo ever modifies those we'll need to do to them what we now do to
+ * errno.
+ */
+
+
+/* The environment vars exhibit totally messed up behavior when someone
+ * does an execve of /bin/sh -- not sure what's going on, but using our
+ * own implementation of unsetenv fixes all our problems.  If we use
+ * libc's, unsetenv either does nothing or ends up having getenv return
+ * NULL for other vars that are obviously set (by iterating through environ).
+ * FIXME: find out the real story here.
+ */
+extern char **__environ;
+#include <errno.h> /* for EINVAL */
+/* avoid problems with use of errno as var name in rest of file */
+#undef errno
+#define __set_errno(val) (*__errno_location ()) = (val)
+#define __get_errno() (*__errno_location())
+int
+our_unsetenv(const char *name)
+{
+    /* This is pretty much straight from the glibc sources */
+    size_t len;
+    char **ep;
+    
+    if (name == NULL || *name == '\0' || strchr (name, '=') != NULL) {
+        __set_errno (EINVAL);
+        return -1;
+    }
+    
+    len = strlen (name);
+    
+    /* FIXME: glibc code grabs a lock here, we don't have access to that lock
+     * LOCK;
+     */
+    
+    ep = __environ;
+    while (*ep != NULL)
+        if (!strncmp (*ep, name, len) && (*ep)[len] == '=') {
+            /* Found it.  Remove this pointer by moving later ones back.  */
+            char **dp = ep;
+            
+            do {
+                dp[0] = dp[1];
+            } while (*dp++);
+            /* Continue the loop in case NAME appears again.  */
+        } else
+            ++ep;
+
+    /* FIXME: glibc code unlocks here, we don't have access to that lock
+     * UNLOCK;
+     */
+    
+    return 0;
+}
+
+
+#if !defined(STATIC_LIBRARY) && !defined(STANDALONE_UNIT_TEST)
+/* shared library init and exit */
+int
+_init()
+{
+# ifdef INIT_TAKE_OVER
+    /* if do not want to use drpreload.so, we can take over here */
+    extern void dynamorio_app_take_over(void);
+    if (dynamorio_app_init() == 0 /* success */) {
+        dynamorio_app_take_over();
+    }
+# endif
+    return 0;
+}
+
+int
+_fini()
+{
+    return 0;
+}
+#endif
+
+bool
+kernel_is_64bit(void)
+{
+    return kernel_64bit;
+}
+
+static void
+get_uname(void)
+{
+    /* assumption: only called at init, so we don't need any synch
+     * or .data unprot
+     */
+    static struct utsname uinfo; /* can be large, avoid stack overflow */
+    DEBUG_DECLARE(int res =)
+        dynamorio_syscall(SYS_uname, 1, (ptr_uint_t)&uinfo);
+    ASSERT(res >= 0);
+    LOG(GLOBAL, LOG_TOP, 1, "uname:\n\tsysname: %s\n", uinfo.sysname);
+    LOG(GLOBAL, LOG_TOP, 1, "\tnodename: %s\n", uinfo.nodename);
+    LOG(GLOBAL, LOG_TOP, 1, "\trelease: %s\n", uinfo.release);
+    LOG(GLOBAL, LOG_TOP, 1, "\tversion: %s\n", uinfo.version);
+    LOG(GLOBAL, LOG_TOP, 1, "\tmachine: %s\n", uinfo.machine);
+    if (strncmp(uinfo.machine, "x86_64", sizeof("x86_64")) == 0)
+        kernel_64bit = true;
+}
+
+/* os-specific initializations */
+void
+os_init(void)
+{
+    get_uname();
+
+    /* determine whether gettid is provided and needed for threads,
+     * or whether getpid suffices.  even 2.4 kernels have gettid
+     * (maps to getpid), don't have an old enough target to test this.
+     */
+    kernel_thread_groups = (dynamorio_syscall(SYS_gettid, 0) >= 0);
+    LOG(GLOBAL, LOG_TOP|LOG_STATS, 1, "thread id is from %s\n",
+        kernel_thread_groups ? "gettid" : "getpid");
+    ASSERT_CURIOSITY(kernel_thread_groups);
+
+    pid_cached = get_process_id();
+
+#ifdef VMX86_SERVER
+    vmk_init();
+#endif
+    
+    signal_init();
+
+#ifdef PROFILE_RDTSC
+    if (dynamo_options.profile_times) {
+        ASSERT_NOT_TESTED();
+        kilo_hertz = get_timer_frequency();
+        LOG(GLOBAL, LOG_TOP|LOG_STATS, 1, "CPU MHz is %d\n", kilo_hertz/1000);
+    }
+#endif /* PROFILE_RDTSC */
+    /* Need to be after heap_init */
+    VMVECTOR_ALLOC_VECTOR(all_memory_areas, GLOBAL_DCONTEXT,
+                          VECTOR_SHARED | VECTOR_NEVER_MERGE, all_memory_areas);
+}
+
+/* we need to re-cache after a fork */
+static char *
+get_application_pid_helper(bool ignore_cache)
+{
+    static char pidstr[16];
+
+    if (!pidstr[0] || ignore_cache) {
+        int pid = get_process_id();
+        snprintf(pidstr, sizeof(pidstr)-1, "%d", pid);
+    }
+    return pidstr;
+}
+
+/* get application pid, (cached), used for event logging */
+char*
+get_application_pid()
+{
+    return get_application_pid_helper(false);
+}
+
+/* get application name, (cached), used for event logging */
+/* FIXME : Not yet implemented */
+char*
+get_application_name()
+{
+    static char name_buf[MAXIMUM_PATH];
+    
+    if (!name_buf[0]) {
+        /* FIXME PR 363063: move getnamefrompid() here and replace /proc reliance
+         * ideally w/ os-independent method, but could use VSI for ESX
+         */
+        extern void getnamefrompid(int pid, char *name, uint maxlen);
+        getnamefrompid(get_process_id(), name_buf, BUFFER_SIZE_ELEMENTS(name_buf));
+    }
+    return name_buf;
+}
+
+const char *
+get_application_short_name()
+{
+    /* FIXME: NYI: need get_application_name() to have full path, here not */
+    return get_application_name();
+}
+
+/* Processor information provided by kernel */
+#define PROC_CPUINFO "/proc/cpuinfo"
+#define CPUMHZ_LINE_LENGTH  64
+#define CPUMHZ_LINE_FORMAT "cpu MHz\t\t: %lu.%03lu\n"
+/* printed in /usr/src/linux-2.4/arch/i386/kernel/setup.c calibrated in time.c */
+/* seq_printf(m, "cpu MHz\t\t: %lu.%03lu\n", cpu_khz / 1000, (cpu_khz % 1000)) */
+/* e.g. cpu MHz           : 1594.851 */
+static timestamp_t
+get_timer_frequency_cpuinfo(void)
+{
+    FILE *cpuinfo;
+    char line[CPUMHZ_LINE_LENGTH];
+    ulong cpu_mhz = 1, cpu_khz = 0; 
+    
+    /* FIXME: use os_open; for now only called for init, so ok to use fopen */
+    cpuinfo=fopen(PROC_CPUINFO,"r");
+    ASSERT(cpuinfo != NULL);
+
+    while (!feof(cpuinfo)) {
+        if (fgets(line, sizeof(line), cpuinfo) == NULL)
+            break;
+        if (sscanf(line, CPUMHZ_LINE_FORMAT, &cpu_mhz, &cpu_khz) == 2) {
+            LOG(GLOBAL, LOG_ALL, 2, "Processor speed exactly %lu.%03luMHz\n", cpu_mhz, cpu_khz);
+            break;
+        }
+    }
+    fclose(cpuinfo);
+    return cpu_mhz*1000 + cpu_khz;
+}
+
+timestamp_t
+get_timer_frequency()
+{
+#ifdef VMX86_SERVER
+    if (os_in_vmkernel_userworld()) {
+        return vmk_get_timer_frequency();
+    }
+#endif
+   return get_timer_frequency_cpuinfo();
+}
+
+/* seconds since 1970 */
+uint
+query_time_seconds(void)
+{
+    return (uint) dynamorio_syscall(SYS_time, 1, NULL);
+}
+
+#ifdef RETURN_AFTER_CALL
+/* Finds the bottom of the call stack, presumably at program startup. */
+/* This routine is a copycat of internal_dump_callstack and makes assumptions about program state, 
+   i.e. that frame pointers are valid and should be used only in well known points for release build.
+*/
+static app_pc
+find_stack_bottom()
+{
+    app_pc retaddr = 0;
+    int depth = 0;
+    reg_t *fp;
+    /* from dump_dr_callstack() */
+    asm("mov  %%"ASM_XBP", %0" : "=m"(fp));
+
+    LOG(THREAD_GET, LOG_ALL, 3, "Find stack bottom:\n");
+    while (fp != NULL && is_readable_without_exception((byte *)fp, sizeof(reg_t)*2)) {
+        retaddr = (app_pc)*(fp+1);      /* presumably also readable */
+        LOG(THREAD_GET, LOG_ALL, 3,
+            "\tframe ptr "PFX" => parent "PFX", ret = "PFX"\n", fp, *fp, retaddr);
+        depth++;
+        /* yes I've seen weird recursive cases before */
+        if (fp == (reg_t *) *fp || depth > 100)
+            break;
+        fp = (reg_t *) *fp;
+    }
+    return retaddr;
+}
+#endif /* RETURN_AFTER_CALL */
+
+/* os-specific atexit cleanup */
+void
+os_slow_exit(void)
+{
+    signal_exit();
+    DELETE_LOCK(memory_info_buf_lock);
+    DELETE_LOCK(maps_iter_buf_lock);
+    vmvector_delete_vector(GLOBAL_DCONTEXT, all_memory_areas);
+    all_memory_areas = NULL;
+}
+
+/* os-specific atexit cleanup */
+void
+os_fast_exit(void)
+{
+    /* nothing */
+}
+
+/* clean up may be impossible - just terminate */
+void
+os_terminate(dcontext_t *dcontext, terminate_flags_t flags)
+{
+    /* FIXME: terminate type is ignored */
+    ASSERT_NOT_IMPLEMENTED(flags == TERMINATE_PROCESS);
+    /* FIXME: kill entire thread group if linux kernel supports groups; else,
+     * if tid==pid and no groups, try to hack it?
+     */
+    exit_syscall(1);
+}
+
+int 
+os_timeout(int time_in_milliseconds)
+{
+    ASSERT_NOT_IMPLEMENTED(false);
+    return 0;
+}
+
+/************************************************************************
+ * SEGMENT STEALING
+ *
+ * Not easy to make truly transparent -- but the alternative of dispatch
+ * by thread id on global memory has performance implications.
+ * Pull the non-STEAL_SEGMENT code out of the cvs attic for a base if
+ * transparency becomes more of a problem.
+ */
+
+#if defined(X64) && !defined(ARCH_SET_GS)
+# define ARCH_SET_GS 0x1001
+# define ARCH_SET_FS 0x1002
+# define ARCH_GET_FS 0x1003
+# define ARCH_GET_GS 0x1004
+#endif
+
+/* We support 3 different methods of creating a segment (see os_tls_init()) */
+typedef enum {
+    TLS_TYPE_NONE,
+    TLS_TYPE_LDT,
+    TLS_TYPE_GDT,
+#ifdef X64
+    TLS_TYPE_ARCH_PRCTL,
+#endif
+} tls_type_t;
+
+#ifdef HAVE_TLS
+/* GDT slot we use for set_thread_area.
+ * This depends on the kernel, not on the app!
+ */
+#define GDT_32BIT  8 /*  6=NPTL, 7=wine */
+#define GDT_64BIT 14 /* 12=NPTL, 13=wine */
+
+static int
+modify_ldt_syscall(int func, void *ptr, unsigned long bytecount)
+{
+    return dynamorio_syscall(SYS_modify_ldt, 3, func, ptr, bytecount);
+}
+
+/* reading ldt entries gives us the raw format, not struct modify_ldt_ldt_s */
+typedef struct {
+    unsigned int limit1500:16;
+    unsigned int base1500:16;
+    unsigned int base2316:8;
+    unsigned int type:4;
+    unsigned int not_system:1;
+    unsigned int privilege_level:2;
+    unsigned int seg_present:1;
+    unsigned int limit1916:4;
+    unsigned int custom:1;
+    unsigned int zero:1;
+    unsigned int seg_32bit:1;
+    unsigned int limit_in_pages:1;
+    unsigned int base3124:8;
+} raw_ldt_entry_t;
+
+enum {
+    LDT_TYPE_CODE      = 0x8,
+    LDT_TYPE_DOWN      = 0x4,
+    LDT_TYPE_WRITE     = 0x2,
+    LDT_TYPE_ACCESSED  = 0x1,
+};
+
+#ifdef DEBUG
+# if 0 /* not used */
+#  ifdef X64
+    /* Intel docs confusingly say that app descriptors are 8 bytes while
+     * system descriptors are 16; more likely all are 16, as linux kernel
+     * docs seem to assume. */
+#   error NYI
+#  endif
+static void
+print_raw_ldt(raw_ldt_entry_t *ldt)
+{
+    LOG(GLOBAL, LOG_ALL, 1,
+        "ldt @"PFX":\n", ldt);
+    LOG(GLOBAL, LOG_ALL, 1,
+        "\traw: "PFX" "PFX"\n", *((unsigned int *)ldt), *(1+(unsigned int *)ldt));
+    LOG(GLOBAL, LOG_ALL, 1,
+        "\tbase: 0x%x\n", (ldt->base3124<<24) | (ldt->base2316<<16) | ldt->base1500);
+    LOG(GLOBAL, LOG_ALL, 1,
+        "\tlimit: 0x%x\n", (ldt->limit1916<<16) | ldt->limit1500);
+    LOG(GLOBAL, LOG_ALL, 1,
+        "\tlimit_in_pages: 0x%x\n", ldt->limit_in_pages);
+    LOG(GLOBAL, LOG_ALL, 1,
+        "\tseg_32bit: 0x%x\n", ldt->seg_32bit);
+    LOG(GLOBAL, LOG_ALL, 1,
+        "\tseg_present: 0x%x\n", ldt->seg_present);
+    LOG(GLOBAL, LOG_ALL, 1,
+        "\tprivilege_level: 0x%x\n", ldt->privilege_level);
+    LOG(GLOBAL, LOG_ALL, 1,
+        "\tnot_system: 0x%x\n", ldt->not_system);
+    LOG(GLOBAL, LOG_ALL, 1,
+        "\ttype: 0x%x == %s%s%s%s\n", ldt->type,
+           (ldt->type & LDT_TYPE_CODE)  ? "code":"data",
+           (ldt->type & LDT_TYPE_DOWN)  ? " top-down":"",
+           (ldt->type & LDT_TYPE_WRITE) ? " W":" ",
+           (ldt->type & LDT_TYPE_ACCESSED)  ? " accessed":"");
+}
+
+static void
+print_all_ldt(void)
+{
+    int i, bytes;
+    /* can't fit 64K on our stack */
+    raw_ldt_entry_t *ldt = global_heap_alloc(sizeof(raw_ldt_entry_t) * LDT_ENTRIES
+                                           HEAPACCT(ACCT_OTHER));
+    /* make sure our struct size jives w/ ldt.h */
+    ASSERT(sizeof(raw_ldt_entry_t) == LDT_ENTRY_SIZE);
+    memset(ldt, 0, sizeof(ldt));
+    bytes = modify_ldt_syscall(0, (void *)ldt, sizeof(raw_ldt_entry_t) * LDT_ENTRIES);
+    LOG(GLOBAL, LOG_ALL, 3, "read %d bytes, should == %d * %d\n",
+        bytes, sizeof(raw_ldt_entry_t), LDT_ENTRIES);
+    ASSERT(bytes == 0 /* no ldt entries */ ||
+           bytes == sizeof(raw_ldt_entry_t) * LDT_ENTRIES);
+    for (i = 0; i < bytes/sizeof(raw_ldt_entry_t); i++) {
+        if (((ldt[i].base3124<<24) | (ldt[i].base2316<<16) | ldt[i].base1500) != 0) {
+            LOG(GLOBAL, LOG_ALL, 1, "ldt at index %d:\n", i);
+            print_raw_ldt(&ldt[i]);
+        }
+    }
+    global_heap_free(ldt, sizeof(raw_ldt_entry_t) * LDT_ENTRIES HEAPACCT(ACCT_OTHER));
+}
+# endif /* #if 0 */
+#endif /* DEBUG */
+
+#define LDT_ENTRIES_TO_CHECK 128
+
+/* returns -1 if all indices are in use */
+static int
+find_unused_ldt_index()
+{
+    int i, bytes;
+    /* N.B.: we don't have 64K of stack for the full LDT_ENTRIES
+     * array, and I don't want to allocate a big array on the heap
+     * when it's very doubtful any more than a handful of these
+     * descriptors are actually in use
+     */
+    raw_ldt_entry_t ldt[LDT_ENTRIES_TO_CHECK];
+    ASSERT(LDT_ENTRIES_TO_CHECK < LDT_ENTRIES);
+    /* make sure our struct size jives w/ ldt.h */
+    ASSERT(sizeof(raw_ldt_entry_t) == LDT_ENTRY_SIZE);
+    memset(ldt, 0, sizeof(ldt));
+    bytes = modify_ldt_syscall(0, (void *)ldt, sizeof(ldt));
+    if (bytes == 0) {
+        /* no indices are taken yet */
+        return 0;
+    }
+    ASSERT(bytes == sizeof(ldt));
+    for (i = 0; i < bytes/sizeof(raw_ldt_entry_t); i++) {
+        if (((ldt[i].base3124<<24) | (ldt[i].base2316<<16) | ldt[i].base1500) == 0) {
+            return i;
+        }
+    }
+    return -1;
+}
+
+static void
+initialize_ldt_struct(our_modify_ldt_t *ldt, void *base, size_t size, uint index)
+{
+    ASSERT(ldt != NULL);
+    ldt->entry_number = index;
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint((ptr_uint_t)base)));
+    ldt->base_addr = (int)(ptr_int_t) base;
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(size)));
+    ldt->limit = size;
+    ldt->seg_32bit = IF_X64_ELSE(0, 1);
+    ldt->contents = MODIFY_LDT_CONTENTS_DATA;
+    ldt->read_exec_only = 0;
+    ldt->limit_in_pages = 0;
+    ldt->seg_not_present = 0;
+    ldt->useable = 0; /* becomes custom bit */
+}
+
+static void
+clear_ldt_struct(our_modify_ldt_t *ldt, uint index)
+{
+    /* set fields to match LDT_empty() macro from linux kernel */
+    memset(ldt, 0, sizeof(*ldt));
+    ldt->seg_not_present = 1;
+    ldt->read_exec_only = 1;
+    ldt->entry_number = index;
+}
+
+static void
+create_ldt_entry(void *base, size_t size, uint index)
+{
+    our_modify_ldt_t array;
+    int ret;
+    initialize_ldt_struct(&array, base, size, index);
+    ret = modify_ldt_syscall(1, (void *)&array, sizeof(array));
+    ASSERT(ret >= 0);
+}
+
+static void
+clear_ldt_entry(uint index)
+{
+    our_modify_ldt_t array;
+    int ret;
+    clear_ldt_struct(&array, index);
+    ret = modify_ldt_syscall(1, (void *)&array, sizeof(array));
+    ASSERT(ret >= 0);
+}
+#endif /* HAVE_TLS */
+
+/* segment selector format:
+ * 15..............3      2          1..0
+ *        index      0=GDT,1=LDT  Requested Privilege Level
+ */
+#define USER_PRIVILEGE  3
+#define LDT_NOT_GDT     1
+#define GDT_NOT_LDT     0
+#define LDT_SELECTOR(idx) ((idx) << 3 | ((LDT_NOT_GDT) << 2) | (USER_PRIVILEGE))
+#define GDT_SELECTOR(idx) ((idx) << 3 | ((GDT_NOT_LDT) << 2) | (USER_PRIVILEGE))
+
+#define WRITE_FS(val) \
+    ASSERT(sizeof(val) == sizeof(void*));                           \
+    asm volatile("mov %0,%%"ASM_XAX"; mov %%"ASM_XAX", %"ASM_SEG";" \
+                 : : "m" ((val)) : ASM_XAX);
+
+/* FIXME: assumes that fs/gs is not already in use by app */
+static bool
+is_segment_register_initialized(void)
+{
+    uint sel;
+    /* pre-P6 family leaves upper 2 bytes undefined, so we clear them
+     * we don't clear and then use movw b/c that takes an extra clock cycle
+     */
+    asm volatile("movl %"ASM_SEG", %%eax; andl $0x0000ffff, %%eax; "\
+                 "movl %%eax, %0" : "=m"(sel) : : "eax");
+    return sel != 0;
+}
+
+/* layout of our TLS */
+typedef struct _os_local_state_t {
+    /* put state first to ensure that it is cache-line-aligned */
+    /* On Linux, we always use the extended structure. */
+    local_state_extended_t state;
+    /* linear address of tls page */
+    struct _os_local_state_t *self;
+    /* store what type of TLS this is so we can clean up properly */
+    tls_type_t tls_type;
+    /* For pre-SYS_set_thread_area kernels (pre-2.5.32, pre-NPTL), each
+     * thread needs its own ldt entry */
+    int ldt_index;
+    /* tid needed to ensure children are set up properly */
+    thread_id_t tid;
+} os_local_state_t;
+
+#define TLS_LOCAL_STATE_OFFSET (offsetof(os_local_state_t, state))
+
+/* offset from top of page */
+#define TLS_OS_LOCAL_STATE     0x00
+
+#define TLS_SELF_OFFSET        (TLS_OS_LOCAL_STATE + offsetof(os_local_state_t, self))
+#define TLS_THREAD_ID_OFFSET   (TLS_OS_LOCAL_STATE + offsetof(os_local_state_t, tid))
+#define TLS_DCONTEXT_OFFSET    (TLS_OS_LOCAL_STATE + TLS_DCONTEXT_SLOT)
+
+/* N.B.: imm and idx are ushorts!
+ * imm must be a preprocessor constant
+ * cannot find a gcc asm constraint that will put an immed w/o a $ in, so
+ * we use the preprocessor to paste the constant in.
+ * Also, var needs to match the pointer size, or else we'll get stack corruption.
+ */
+#define WRITE_TLS_SLOT_IMM(imm, var)                                  \
+    IF_NOT_HAVE_TLS(ASSERT_NOT_REACHED());                            \
+    ASSERT(sizeof(var) == sizeof(void*));                             \
+    asm("mov %0, %%"ASM_XAX : : "m"((var)) : ASM_XAX);                \
+    asm("mov %"ASM_XAX", "ASM_SEG":"STRINGIFY(imm) : : "m"((var)) : ASM_XAX);
+
+#define READ_TLS_SLOT_IMM(imm, var)                  \
+    IF_NOT_HAVE_TLS(ASSERT_NOT_REACHED());           \
+    ASSERT(sizeof(var) == sizeof(void*));            \
+    asm("mov "ASM_SEG":"STRINGIFY(imm)", %"ASM_XAX); \
+    asm("mov %%"ASM_XAX", %0" : "=m"((var)) : : ASM_XAX);
+
+#define WRITE_TLS_SLOT(idx, var)                            \
+    IF_NOT_HAVE_TLS(ASSERT_NOT_REACHED());                  \
+    ASSERT(sizeof(var) == sizeof(void*));                   \
+    asm("mov %0, %%"ASM_XAX : : "m"((var)) : ASM_XAX);      \
+    asm("movzx %0, %%"ASM_XDX"" : : "m"((idx)) : ASM_XDX);  \
+    asm("mov %%"ASM_XAX", %"ASM_SEG":(%%"ASM_XDX")" : : : ASM_XAX, ASM_XDX);
+
+/* FIXME: get_thread_private_dcontext() is a bottleneck, so it would be
+ * good to figure out how to easily change this to use an immediate since it is
+ * known at compile time -- see comments above for the _IMM versions
+ */
+#define READ_TLS_SLOT(idx, var)                                    \
+    ASSERT(sizeof(var) == sizeof(void*));                          \
+    asm("movzx %0, %%"ASM_XAX : : "m"((idx)) : ASM_XAX);           \
+    asm("mov %"ASM_SEG":(%%"ASM_XAX"), %%"ASM_XAX : : : ASM_XAX);  \
+    asm("mov %%"ASM_XAX", %0" : "=m"((var)) : : ASM_XAX);
+
+/* converts a local_state_t offset to a segment offset */
+ushort
+os_tls_offset(ushort tls_offs)
+{
+    /* no ushort truncation issues b/c TLS_LOCAL_STATE_OFFSET is 0 */
+    IF_NOT_HAVE_TLS(ASSERT_NOT_REACHED());
+    ASSERT(TLS_LOCAL_STATE_OFFSET == 0);
+    return (TLS_LOCAL_STATE_OFFSET + tls_offs);
+}
+
+void *
+get_tls(ushort tls_offs)
+{
+    void *val;
+    READ_TLS_SLOT(tls_offs, val);
+    return val;
+}
+
+void
+set_tls(ushort tls_offs, void *value)
+{
+    WRITE_TLS_SLOT(tls_offs, value);
+}
+
+
+#ifdef X86
+/* Returns POINTER_MAX on failure.
+ * Assumes that cs, ss, ds, and es are flat.
+ */
+byte *
+get_segment_base(uint seg)
+{
+    if (seg == SEG_CS || seg == SEG_SS || seg == SEG_DS || seg == SEG_ES)
+        return NULL;
+    else /* FIXME: NYI */
+        return (byte *) POINTER_MAX;
+}
+#endif
+
+local_state_extended_t *
+get_local_state_extended()
+{
+    os_local_state_t *os_tls;
+    ushort offs = TLS_SELF_OFFSET;
+    ASSERT(is_segment_register_initialized());
+    READ_TLS_SLOT(offs, os_tls);
+    return &(os_tls->state);
+}
+
+local_state_t *
+get_local_state()
+{
+#ifdef HAVE_TLS
+    return (local_state_t *) get_local_state_extended();
+#else
+    return NULL;
+#endif
+}
+
+void
+os_tls_init()
+{
+#ifdef HAVE_TLS
+    /* We create a 1-page segment with an LDT entry for each thread and load its
+     * selector into fs/gs.
+     * FIXME PR 205276: this whole scheme currently does not check if app is using
+     * segments need to watch modify_ldt syscall
+     */
+    /* FIXME: heap_mmap marks as exec, we just want RW */
+    byte *segment = heap_mmap(PAGE_SIZE);
+    os_local_state_t *os_tls = (os_local_state_t *) segment;
+    int index = -1;
+    uint selector;
+    int res;
+# ifdef X64
+    byte *cur_gs;
+# endif
+
+    LOG(GLOBAL, LOG_THREADS, 1, "os_tls_init for thread "IDFMT"\n", get_thread_id());
+
+    /* MUST zero out dcontext slot so uninit access gets NULL */
+    memset(segment, 0, PAGE_SIZE);
+    /* store key data in the tls itself */
+    os_tls->self = os_tls;
+    os_tls->tid = get_thread_id();
+    os_tls->tls_type = TLS_TYPE_NONE;
+    ASSERT(proc_is_cache_aligned(os_tls->self + TLS_LOCAL_STATE_OFFSET));
+    /* Verify that local_state_extended_t should indeed be used. */
+    ASSERT(DYNAMO_OPTION(ibl_table_in_tls));
+
+    /* We have four different ways to obtain TLS, each with its own limitations:
+     *
+     * 1) Piggyback on the threading system (like we do on Windows): here that would
+     *    be pthreads, which uses a segment since at least RH9, and uses gdt-based
+     *    segments for NPTL.  The advantage is we won't run out of ldt or gdt entries
+     *    (except when the app itself would).  The disadvantage is we're stealing
+     *    application slots and we rely on user mode interfaces.
+     *
+     * 2) Steal an ldt entry via SYS_modify_ldt.  This suffers from the 8K ldt entry
+     *    limit and requires that we update manually on a new thread.  For 64-bit
+     *    we're limited here to a 32-bit base.  (Strangely, the kernel's
+     *    include/asm-x86_64/ldt.h implies that the base is ignored: but it doesn't
+     *    seem to be.)
+     * 
+     * 3) Steal a gdt entry via SYS_set_thread_area.  There is a 3rd unused entry
+     *    (after pthreads and wine) we could use.  The kernel swaps for us, and with
+     *    CLONE_TLS the kernel will set up the entry for a new thread for us.  Xref
+     *    PR 192231 and PR 285898.  This system call is disabled on 64-bit 2.6
+     *    kernels (though the man page for arch_prctl implies it isn't for 2.5
+     *    kernels?!?)
+     *
+     * 4) Use SYS_arch_prctl.  This is only implemented on 64-bit kernels, and can
+     *    only be used to set the gdt entries that fs and gs select for.  Faster to
+     *    use <4GB base (obtain with mmap MAP_32BIT) since can use gdt; else have to
+     *    use wrmsr.  The man pages say "ARCH_SET_GS is disabled in some kernels".
+     */
+
+# ifdef X64
+    /* First choice is gdt, which means arch_prctl.  Since this may fail
+     * on some kernels, we require -heap_in_lower_4GB so we can fall back
+     * on modify_ldt.
+     */
+    res = dynamorio_syscall(SYS_arch_prctl, 2, ARCH_GET_GS, &cur_gs);
+    if (res >= 0) {
+        LOG(GLOBAL, LOG_THREADS, 1, "os_tls_init: cur gs base is "PFX"\n", cur_gs);
+        /* If we're a non-initial thread, gs will be set to the parent thread's value */
+        if (cur_gs == NULL || is_dynamo_address(cur_gs)) {
+            res = dynamorio_syscall(SYS_arch_prctl, 2, ARCH_SET_GS, segment);
+            if (res >= 0) {
+                os_tls->tls_type = TLS_TYPE_ARCH_PRCTL;
+                LOG(GLOBAL, LOG_THREADS, 1,
+                    "os_tls_init: arch_prctl successful for base "PFX"\n", segment);
+            } else {
+                /* we've found a kernel where ARCH_SET_GS is disabled */
+                ASSERT_CURIOSITY(false && "arch_prctl failed on set but not get");
+                LOG(GLOBAL, LOG_THREADS, 1,
+                    "os_tls_init: arch_prctl failed: error %d\n", res);
+            }
+        } else {
+            /* FIXME PR 205276: we don't currently handle it: fall back on ldt, but
+             * we'll have the same conflict w/ the selector...
+             */
+            ASSERT_BUG_NUM(205276, cur_gs == NULL);
+        }
+    }
+# endif
+    if (os_tls->tls_type == TLS_TYPE_NONE) {
+        /* Second choice is set_thread_area */
+        /* PR 285898: if we added CLONE_SETTLS to all clone calls (and emulated vfork
+         * with clone) we could avoid having to set tls up for each thread (as well
+         * as solve race PR 207903), at least for kernel 2.5.32+.  For now we stick
+         * w/ manual setup.
+         */
+        our_modify_ldt_t desc;
+        /* Base here must be 32-bit */
+        IF_X64(ASSERT(DYNAMO_OPTION(heap_in_lower_4GB) && segment <= (byte*)UINT_MAX));
+        /* We could pass -1 and ask for the next available slot, but I'm
+         * afraid of being earlier than pthreads or wine and breaking
+         * their assumptions, so we hardcode the 3rd slot.
+         */
+        initialize_ldt_struct(&desc, segment, PAGE_SIZE,
+# ifndef HAVE_TLS_AREA_MULTI
+                              /* for ESX4.0 there is only one slot: taking it for now,
+                               * though we'll conflict w/ pthreads
+                               */
+                              -1
+# else
+                              kernel_is_64bit() ? GDT_64BIT : GDT_32BIT
+# endif
+                              );
+        res = dynamorio_syscall(SYS_set_thread_area, 1, &desc);
+        if (res >= 0) {
+            LOG(GLOBAL, LOG_THREADS, 1,
+                "os_tls_init: set_thread_area successful for base "PFX"\n", segment);
+            os_tls->tls_type = TLS_TYPE_GDT;
+            index = desc.entry_number;
+            selector = GDT_SELECTOR(index);
+            WRITE_FS(selector); /* macro needs lvalue! */
+        } else {
+            LOG(GLOBAL, LOG_THREADS, 1,
+                "os_tls_init: set_thread_area failed: error %d\n", res);
+        }
+    }
+    if (os_tls->tls_type == TLS_TYPE_NONE) {
+        /* Third choice: modify_ldt, which should be available on kernel 2.3.99+ */
+        /* Base here must be 32-bit */
+        IF_X64(ASSERT(DYNAMO_OPTION(heap_in_lower_4GB) && segment <= (byte*)UINT_MAX));
+        /* we have the thread_initexit_lock so no race here */
+        index = find_unused_ldt_index();
+        selector = LDT_SELECTOR(index);
+        ASSERT(index != -1);
+        create_ldt_entry((void *)segment, PAGE_SIZE, index);
+        os_tls->tls_type = TLS_TYPE_LDT;
+        WRITE_FS(selector); /* macro needs lvalue! */
+        LOG(GLOBAL, LOG_THREADS, 1,
+            "os_tls_init: modify_ldt successful for base "PFX" w/ index %d\n",
+            segment, index);
+    }
+    os_tls->ldt_index = index;
+    ASSERT(os_tls->tls_type != TLS_TYPE_NONE);
+#else
+    tls_table = (tls_slot_t *)
+        global_heap_alloc(MAX_THREADS*sizeof(tls_slot_t) HEAPACCT(ACCT_OTHER));
+    memset(tls_table, 0, MAX_THREADS*sizeof(tls_slot_t));
+#endif
+}
+
+/* Can be called from other threads, so uses the parameter to locate tls */
+void
+os_tls_exit(local_state_t *local_state)
+{
+#ifdef HAVE_TLS
+    int res;
+    static const ptr_uint_t zero = 0;
+    /* We can't read from fs: as we can be called from other threads */
+    /* ASSUMPTION: local_state_t is laid out at same start as local_state_extended_t */
+    os_local_state_t *os_tls = (os_local_state_t *)
+        ((byte*)local_state) - offsetof(os_local_state_t, state);
+    tls_type_t tls_type = os_tls->tls_type;
+    int index = os_tls->ldt_index;
+
+    /* We assume that if called from another thread, that thread has had
+     * its own segment register cleared already: FIXME: is this safe? */
+    WRITE_FS(zero); /* macro needs lvalue! */
+    heap_munmap(os_tls->self, PAGE_SIZE);
+
+    if (tls_type == TLS_TYPE_LDT)
+        clear_ldt_entry(index);
+    else if (tls_type == TLS_TYPE_GDT) {
+        our_modify_ldt_t desc;
+        clear_ldt_struct(&desc, index);
+        res = dynamorio_syscall(SYS_set_thread_area, 1, &desc);
+        ASSERT(res >= 0);        
+    }
+# ifdef X64
+    else if (tls_type == TLS_TYPE_ARCH_PRCTL) {
+        res = dynamorio_syscall(SYS_arch_prctl, 2, ARCH_SET_GS, NULL);
+        ASSERT(res >= 0);
+        /* syscall re-sets gs register so re-clear it */
+        WRITE_FS(zero); /* macro needs lvalue! */
+    }
+# endif
+#else
+    global_heap_free(tls_table, MAX_THREADS*sizeof(tls_slot_t) HEAPACCT(ACCT_OTHER));
+    DELETE_LOCK(tls_lock);
+#endif
+}
+
+void
+os_thread_init(dcontext_t *dcontext)
+{
+    os_thread_data_t *ostd = (os_thread_data_t *)
+        heap_alloc(dcontext, sizeof(os_thread_data_t) HEAPACCT(ACCT_OTHER));
+    dcontext->os_field = (void *) ostd;
+    /* make sure stack fields, etc. are 0 now so they can be initialized on demand
+     * (don't have app esp register handy here to init now)
+     */
+    memset(ostd, 0, sizeof(*ostd));
+
+#ifdef RETURN_AFTER_CALL
+    if (!dynamo_initialized) {
+        /* Find the bottom of the stack of the initial (native) entry */
+        ostd->stack_bottom_pc = find_stack_bottom();
+        LOG(THREAD, LOG_ALL, 1, "Stack bottom pc = "PFX"\n", ostd->stack_bottom_pc);
+    } else {
+        /* We only need the stack bottom for the initial thread */
+        ostd->stack_bottom_pc = NULL;
+    }
+#endif
+
+    ASSIGN_INIT_LOCK_FREE(ostd->suspend_lock, suspend_lock);
+
+    signal_thread_init(dcontext);
+    if (INTERNAL_OPTION(profile_pcs)) {
+        pcprofile_thread_init(dcontext);
+    }
+}
+
+void
+os_thread_exit(dcontext_t *dcontext)
+{
+    os_thread_data_t *ostd = (os_thread_data_t *) dcontext->os_field;
+
+    DELETE_LOCK(ostd->suspend_lock);
+
+    signal_thread_exit(dcontext);
+    if (INTERNAL_OPTION(profile_pcs)) {
+        pcprofile_thread_exit(dcontext);
+    }
+
+    /* for non-debug we do fast exit path and don't free local heap */
+    DODEBUG({
+        heap_free(dcontext, ostd, sizeof(os_thread_data_t) HEAPACCT(ACCT_OTHER));
+    });
+}
+
+void
+os_fork_init(dcontext_t *dcontext)
+{
+    pid_cached = get_process_id();
+    get_application_pid_helper(true);
+    if (INTERNAL_OPTION(profile_pcs)) {
+        pcprofile_fork_init(dcontext);
+    }
+}
+
+void
+os_thread_under_dynamo(dcontext_t *dcontext)
+{
+    /* FIXME: what about signals? */
+    if (INTERNAL_OPTION(profile_pcs)) {
+        start_itimer(dcontext);
+    }
+}
+
+void
+os_thread_not_under_dynamo(dcontext_t *dcontext)
+{
+    /* FIXME: what about signals? */
+    if (INTERNAL_OPTION(profile_pcs)) {
+        stop_itimer();
+    }
+}
+
+static pid_t
+get_process_group_id()
+{
+    return dynamorio_syscall(SYS_getpgid, 0);
+}
+
+process_id_t
+get_process_id()
+{
+    return dynamorio_syscall(SYS_getpid, 0);
+}
+
+process_id_t
+get_parent_id(void)
+{
+    return dynamorio_syscall(SYS_getppid, 0);
+}
+
+thread_id_t 
+get_thread_id()
+{
+    if (kernel_thread_groups) {
+        return dynamorio_syscall(SYS_gettid, 0);
+    } else {
+        return dynamorio_syscall(SYS_getpid, 0);
+    }
+}
+
+thread_id_t
+get_tls_thread_id(void)
+{
+    ptr_int_t tid; /* can't use thread_id_t since it's 32-bits */
+    /* need dedicated-storage var for _TLS_SLOT macros, can't use expr */
+    ushort offs;
+    if (!is_segment_register_initialized())
+        return INVALID_THREAD_ID;
+    offs = TLS_THREAD_ID_OFFSET;
+    READ_TLS_SLOT(offs, tid);
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(tid)));
+    return (thread_id_t) tid;
+}
+
+/* returns the thread-private dcontext pointer for the calling thread */
+dcontext_t*
+get_thread_private_dcontext(void)
+{
+#ifdef HAVE_TLS
+    dcontext_t *dcontext;
+    /* FIXME: need dedicated-storage var for _TLS_SLOT macros, can't use expr */
+    ushort offs;
+    /* We have to check this b/c this is called from __errno_location prior
+     * to os_tls_init, as well as after os_tls_exit, and early in a new
+     * thread's initialization (see comments below on that).
+     */
+    if (!is_segment_register_initialized())
+        return NULL;
+    /* We used to check tid and return NULL to distinguish parent from child, but
+     * that was affecting performance (xref PR 207366: but I'm leaving the assert in
+     * for now so debug build will still incur it).  So we fixed the cases that
+     * needed that:
+     *
+     * - dynamo_thread_init() calling is_thread_initialized() for a new thread 
+     *   created via clone or the start/stop interface: so we have 
+     *   is_thread_initialized() pay the get_thread_id() cost.
+     * - new_thread_setup()'s ENTER_DR_HOOK kstats, or a crash and the signal
+     *   handler asking about dcontext: we have new_thread_dynamo_start()
+     *   clear the segment register for us early on.
+     * - child of fork (ASSERT_OWN_NO_LOCKS, etc. on re-entering DR):
+     *   here we just suppress the assert: we'll use this same dcontext.
+     *   xref PR 209518 where w/o this fix we used to need an extra KSTOP.
+     *
+     * An alternative would be to have the parent thread clear the segment
+     * register, or even set up the child's TLS ahead of time ourselves
+     * (and special-case so that we know if at clone syscall the app state is not
+     * quite correct: but we're already stealing a register there: PR 286194).
+     * We could also have the kernel set up TLS for us (PR 285898).
+     *
+     * For hotp_only or non-full-control (native_exec, e.g.) (PR 212012), this
+     * routine is not the only issue: we have to catch all new threads since
+     * hotp_only gateways assume tls is set up.
+     * Xref PR 192231.
+     */
+    /* PR 307698: this assert causes large slowdowns (also xref PR 207366) */ 
+    DOCHECK(1, {
+        ASSERT(get_tls_thread_id() == get_thread_id() ||
+               /* ok for fork as mentioned above */
+               pid_cached != get_process_id());
+    });
+    offs = TLS_DCONTEXT_OFFSET;
+    READ_TLS_SLOT(offs, dcontext);
+    return dcontext;
+#else
+    /* Assumption: no lock needed on a read => no race conditions between
+     * reading and writing same tid!  Since both get and set are only for
+     * the current thread, they cannot both execute simultaneously for the
+     * same tid, right?
+     */
+    thread_id_t tid = get_thread_id();
+    int i;
+    if (tls_table != NULL) {
+        for (i=0; i<MAX_THREADS; i++) {
+            if (tls_table[i].tid == tid) {
+                return tls_table[i].dcontext;
+            }
+        }
+    }
+    return NULL;
+#endif
+}
+
+/* sets the thread-private dcontext pointer for the calling thread */
+void
+set_thread_private_dcontext(dcontext_t *dcontext)
+{
+#ifdef HAVE_TLS
+    /* FIXME: need dedicated-storage var for _TLS_SLOT macros, can't use expr */
+    ushort offs = TLS_DCONTEXT_OFFSET;
+    ASSERT(is_segment_register_initialized());
+    WRITE_TLS_SLOT(offs, dcontext);
+#else
+    thread_id_t tid = get_thread_id();
+    int i;
+    bool found = false;
+    ASSERT(tls_table != NULL);
+    mutex_lock(&tls_lock);
+    for (i=0; i<MAX_THREADS; i++) {
+        if (tls_table[i].tid == tid) {
+            if (dcontext == NULL) {
+                /* if setting to NULL, clear the entire slot for reuse */
+                tls_table[i].tid = 0;
+            }
+            tls_table[i].dcontext = dcontext;
+            found = true;
+            break;
+        }
+    }
+    if (!found) {
+        if (dcontext == NULL) {
+            /* don't do anything...but why would this happen? */
+        } else {
+            /* look for an empty slot */
+            for (i=0; i<MAX_THREADS; i++) {
+                if (tls_table[i].tid == 0) {
+                    tls_table[i].tid = tid;
+                    tls_table[i].dcontext = dcontext;
+                    found = true;
+                    break;
+                }
+            }
+        }
+    }
+    mutex_unlock(&tls_lock);
+    ASSERT(found);
+#endif
+}
+
+/* replaces old with new
+ * use for forking: child should replace parent's id with its own
+ */
+static void
+replace_thread_id(thread_id_t old, thread_id_t new)
+{
+#ifdef HAVE_TLS
+    /* FIXME: need dedicated-storage var for _TLS_SLOT macros, can't use expr */
+    ushort offs = TLS_THREAD_ID_OFFSET;
+    ptr_int_t new_tid = new; /* can't use thread_id_t since it's 32-bits */
+    ASSERT(is_segment_register_initialized());
+    DODEBUG({
+        ptr_int_t old_tid; /* can't use thread_id_t since it's 32-bits */
+        READ_TLS_SLOT(offs, old_tid);
+        IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(old_tid)));
+        ASSERT(old_tid == old);
+    });
+    WRITE_TLS_SLOT(offs, new_tid);
+#else
+    int i;
+    mutex_lock(&tls_lock);
+    for (i=0; i<MAX_THREADS; i++) {
+        if (tls_table[i].tid == old) {
+            tls_table[i].tid = new;
+            break;
+        }
+    }
+    mutex_unlock(&tls_lock);
+#endif
+}
+
+/* translate permission string to platform independent protection bits */
+static inline uint
+permstr_to_memprot(const char * const perm)
+{
+    uint mem_prot = 0;
+    if (perm == NULL || *perm == '\0')
+        return mem_prot;
+    if (perm[2]=='x')
+        mem_prot |= MEMPROT_EXEC;
+    if (perm[1]=='w')
+        mem_prot |= MEMPROT_WRITE;
+    if (perm[0]=='r')
+        mem_prot |= MEMPROT_READ;
+    return mem_prot;
+}
+
+/* translate platform independent protection bits to native flags */
+static inline uint
+memprot_to_osprot(uint prot)
+{
+    uint mmap_prot = 0;
+    if (TEST(MEMPROT_EXEC, prot))
+        mmap_prot |= PROT_EXEC;
+    if (TEST(MEMPROT_READ, prot))
+        mmap_prot |= PROT_READ;
+    if (TEST(MEMPROT_WRITE, prot))
+        mmap_prot |= PROT_WRITE;
+    return mmap_prot;
+}
+
+/* translate native flags to platform independent protection bits */
+static inline uint
+osprot_to_memprot(uint prot)
+{
+    uint mem_prot = 0;
+    if (TEST(PROT_EXEC, prot))
+        mem_prot |= MEMPROT_EXEC;
+    if (TEST(PROT_READ, prot))
+        mem_prot |= MEMPROT_READ;
+    if (TEST(PROT_WRITE, prot))
+        mem_prot |= MEMPROT_WRITE;
+    return mem_prot;
+}
+
+/* returns osprot flags preserving all native protection flags except
+ * for RWX, which are replaced according to memprot */
+uint
+osprot_replace_memprot(uint old_osprot, uint memprot)
+{
+    /* Note only protection flags PROT_ are relevant to mprotect() 
+     * and they are separate from any other MAP_ flags passed to mmap()
+     */
+    uint new_osprot = memprot_to_osprot(memprot);
+    return new_osprot;
+}
+
+/* libc independence */
+static inline long
+mprotect_syscall(byte *p, size_t size, uint prot)
+{
+    return dynamorio_syscall(SYS_mprotect, 3, p, size, prot);
+}
+
+static bool
+mmap_syscall_succeeded(byte *retval)
+{
+    ptr_int_t result = (ptr_int_t) retval;
+    /* libc interprets up to -PAGE_SIZE as an error, and you never know if
+     * some weird errno will be used by say vmkernel (xref PR 365331) 
+     */
+    bool fail = (result < 0 && result >= -PAGE_SIZE);
+    ASSERT_CURIOSITY(!fail ||
+#ifdef VMX86_SERVER
+                     result == -ENOENT  ||
+#endif
+                     result == -EBADF   ||
+                     result == -EACCES  ||
+                     result == -EINVAL  ||
+                     result == -ETXTBSY ||
+                     result == -EAGAIN  ||
+                     result == -ENOMEM  ||
+                     result == -ENODEV  ||
+                     result == -EFAULT);
+    return !fail;
+}
+
+static inline byte *
+mmap_syscall(byte *addr, size_t len, ulong prot, ulong flags, ulong fd, ulong pgoff)
+{
+    return (byte *) dynamorio_syscall(IF_X64_ELSE(SYS_mmap, SYS_mmap2), 6,
+                                      addr, len, prot, flags, fd, pgoff);
+}
+
+static inline long
+munmap_syscall(byte *addr, size_t len)
+{
+    return dynamorio_syscall(SYS_munmap, 2, addr, len);
+}
+
+/* caller is required to handle thread synchronization and to update dynamo vm areas */
+void
+os_heap_free(void *p, size_t size, heap_error_code_t *error_code)
+{
+    long rc;
+    ASSERT(error_code != NULL);
+    if (!dynamo_exited)
+        LOG(GLOBAL, LOG_HEAP, 4, "os_heap_free: %d bytes @ "PFX"\n", size, p);
+    rc = munmap_syscall(p, size);
+    if (rc != 0) {
+        *error_code = -rc;
+    } else {
+        *error_code = HEAP_ERROR_SUCCESS;
+    }
+    ASSERT(rc == 0);    
+}
+
+/* reserve virtual address space without committing swap space for it, 
+   and of course no physical pages since it will never be touched */
+/* to be transparent, we do not use sbrk, and are
+ * instead using mmap, and asserting that all os_heap requests are for
+ * reasonably large pieces of memory */
+void *
+os_heap_reserve(void *preferred, size_t size, heap_error_code_t *error_code)
+{
+    void *p;
+    uint prot = PROT_NONE;
+#ifdef VMX86_SERVER
+    /* PR 365331: we need to be in the mmap_text region for code cache and
+     * gencode.  Ideally we should have a parameter indicating whether this
+     * reservation will be used for text or data: then we could avoid
+     * filling up the mmap_text region (xref PR 365331).
+     */
+    ASSERT(preferred == NULL ||
+           ((byte *)preferred >= os_vmk_mmap_text_start() &&
+            ((byte *)preferred)+size <= os_vmk_mmap_text_end()));
+    /* Note that a preferred address overrides PROT_EXEC and a mmap_data
+     * address will be honored, even though any execution there will fault.
+     */
+    /* FIXME: note that PROT_EXEC => read access, so our guard pages and other
+     * non-committed memory, while not writable, is readable.
+     * Plus, we can't later clear all prot bits for userworld mmap due to PR 107872
+     * (PR 365748 covers fixing this for us).
+     * But in most uses we should get our preferred vmheap and shouldn't run
+     * out of vmheap, so this should be a corner-case issue.
+     */
+    if (preferred == NULL)
+        prot = PROT_EXEC;
+#endif
+    /* should only be used on aligned pieces */
+    ASSERT(size > 0 && ALIGNED(size, PAGE_SIZE));
+    ASSERT(error_code != NULL);
+
+    /* FIXME: note that this memory is in fact still committed - see man mmap */
+    /* FIXME: case 2347 on Linux or -vm_reserve should be set to false */
+    /* FIXME: Need to actually get a mmap-ing with |MAP_NORESERVE */
+    p = mmap_syscall(preferred, size, prot, MAP_PRIVATE|MAP_ANONYMOUS
+                     IF_X64(| (DYNAMO_OPTION(heap_in_lower_4GB) ? MAP_32BIT : 0)),
+                     -1, 0);
+    if (!mmap_syscall_succeeded(p)) {
+        *error_code = -(heap_error_code_t)(ptr_int_t)p;
+        LOG(GLOBAL, LOG_HEAP, 4,
+            "os_heap_reserve %d bytes failed "PFX"\n", size, p);
+        return NULL;
+    } else if (preferred != NULL && p != preferred) {
+        /* We didn't get the preferred address.  To harmonize with windows behavior and
+         * give greater control we fail the reservation. */
+        heap_error_code_t dummy;
+        *error_code = HEAP_ERROR_NOT_AT_PREFERRED;
+        os_heap_free(p, size, &dummy);
+        ASSERT(dummy == HEAP_ERROR_SUCCESS);
+        LOG(GLOBAL, LOG_HEAP, 4,
+            "os_heap_reserve %d bytes at "PFX" not preferred "PFX"\n",
+            size, preferred, p);
+        return NULL;
+    } else {
+        *error_code = HEAP_ERROR_SUCCESS;
+    }
+    LOG(GLOBAL, LOG_HEAP, 2, "os_heap_reserve: %d bytes @ "PFX"\n", size, p);
+#ifdef VMX86_SERVER
+    /* PR 365331: ensure our memory is all in the mmap_text region */
+    ASSERT((byte *)p >= os_vmk_mmap_text_start() &&
+           ((byte *)p) + size <= os_vmk_mmap_text_end());
+#endif
+    return p;
+}
+
+void *
+os_heap_reserve_in_region(void *start, void *end, size_t size,
+                          heap_error_code_t *error_code)
+{
+    byte *p = NULL;
+    byte *try_start;
+
+    LOG(GLOBAL, LOG_HEAP, 3,
+        "os_heap_reserve_in_region: "SZFMT" bytes in "PFX"-"PFX"\n", size, start, end);
+
+    /* if no restriction on location use regular os_heap_reserve() */
+    if (start == (void *)PTR_UINT_0 && end == (void *)POINTER_MAX)
+        return os_heap_reserve(NULL, size, error_code);
+
+    /* FIXME: be smarter and use a memory query instead of trying every single page 
+     * FIXME: if result is not at preferred it may still be in range: so
+     * make syscall ourselves and avoid this loop.
+     */
+    for (try_start = (byte *) start; try_start < (byte *)end - size;
+         try_start += PAGE_SIZE) {
+        p = os_heap_reserve(try_start, size, error_code);
+        if (*error_code == HEAP_ERROR_SUCCESS && p != NULL &&
+            p >= (byte *)start && p + size <= (byte *)end)
+            break;
+    }
+    if (p == NULL)
+        *error_code = HEAP_ERROR_CANT_RESERVE_IN_REGION;
+    else
+        *error_code = HEAP_ERROR_SUCCESS;
+
+    LOG(GLOBAL, LOG_HEAP, 2,
+        "os_heap_reserve_in_region: reserved "SZFMT" bytes @ "PFX" in "PFX"-"PFX"\n",
+        size, p, start, end);
+    return p;
+}
+
+/* commit previously reserved with os_heap_reserve pages */
+/* returns false when out of memory */
+/* A replacement of os_heap_alloc can be constructed by using os_heap_reserve 
+   and os_heap_commit on a subset of the reserved pages. */
+/* caller is required to handle thread synchronization */
+bool
+os_heap_commit(void *p, size_t size, uint prot, heap_error_code_t *error_code)
+{
+    uint os_prot = memprot_to_osprot(prot);
+    long res;
+    /* should only be used on aligned pieces */
+    ASSERT(size > 0 && ALIGNED(size, PAGE_SIZE));
+    ASSERT(p);
+    ASSERT(error_code != NULL);
+
+    /* FIXME: note that the memory would not be not truly committed if we have */
+    /* not actually marked a mmap-ing without MAP_NORESERVE */
+    res = mprotect_syscall(p, size, os_prot);
+    if (res != 0) {
+        *error_code = -res;
+        return false;
+    } else {
+        *error_code = HEAP_ERROR_SUCCESS;
+    }
+
+    LOG(GLOBAL, LOG_HEAP, 2, "os_heap_commit: %d bytes @ "PFX"\n", size, p);
+    return true;
+}
+
+/* caller is required to handle thread synchronization and to update dynamo vm areas */
+void
+os_heap_decommit(void *p, size_t size, heap_error_code_t *error_code)
+{
+    int rc;
+    ASSERT(error_code != NULL);
+
+    if (!dynamo_exited)
+        LOG(GLOBAL, LOG_HEAP, 4, "os_heap_decommit: %d bytes @ "PFX"\n", size, p);
+
+    *error_code = HEAP_ERROR_SUCCESS;
+    /* FIXME: for now do nothing since os_heap_reserve has in fact committed the memory */
+    rc = 0;
+    /* TODO: 
+           p = mmap_syscall(p, size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+       we should either do a mremap() 
+       or we can do a munmap() followed 'quickly' by a mmap() - 
+       also see above the comment that os_heap_reserve() in fact is not so lightweight
+    */
+    ASSERT(rc == 0);    
+}
+
+bool
+os_heap_systemwide_overcommit(heap_error_code_t last_error_code)
+{
+    /* FIXME: conservative answer yes */
+    return true;
+}
+
+bool
+os_heap_get_commit_limit(size_t *commit_used, size_t *commit_limit)
+{
+    /* FIXME - NYI */
+    return false;
+}
+
+/* yield the current thread */
+void
+thread_yield()
+{
+    dynamorio_syscall(SYS_sched_yield, 0);
+}
+
+static bool
+thread_signal(thread_record_t *tr, int signum)
+{
+    /* FIXME: for non-NPTL use SYS_kill */
+    /* Note that the pid is equivalent to the thread group id.
+     * However, we can have threads sharing address space but not pid
+     * (if created via CLONE_VM but not CLONE_THREAD), so make sure to
+     * use the pid of the target thread, not our pid.
+     */
+    return (dynamorio_syscall(SYS_tgkill, 3, tr->dcontext->owning_process,
+                              tr->id, signum) == 0);
+}
+
+void
+thread_sleep(uint64 milliseconds)
+{
+    struct timespec req;
+    struct timespec remain;
+    int count = 0;
+    req.tv_sec = (milliseconds / 1000);
+    /* docs say can go up to 1000000000, but doesn't work on FC9 */
+    req.tv_nsec = (milliseconds % 1000) * 1000;
+    while (dynamorio_syscall(SYS_nanosleep, 2, &req, &remain) == -1) {
+        /* interrupted by signal or something: finish the interval */
+        ASSERT(remain.tv_sec < (milliseconds / 1000) &&
+               remain.tv_nsec < (milliseconds % 1000) * 1000);
+        if (count++ > 3) {
+            ASSERT_NOT_REACHED();
+            break; /* paranoid */
+        }
+        req = remain;
+    }
+}
+
+bool
+thread_suspend(thread_record_t *tr)
+{
+    os_thread_data_t *ostd = (os_thread_data_t *) tr->dcontext->os_field;
+    ASSERT(ostd != NULL);
+    /* See synch comments in thread_resume: the mutex held there
+     * prevents prematurely sending a re-suspend signal.
+     */
+    mutex_lock(&ostd->suspend_lock);
+    ostd->suspend_count++;
+    ASSERT(ostd->suspend_count > 0);
+    /* If already suspended, do not send another signal.  However, we do
+     * need to ensure the target is suspended in case of a race, so we can't
+     * just return.
+     */
+    if (ostd->suspend_count == 1) {
+        /* PR 212090: we use a custom signal handler to suspend.  We wait
+         * here until the target reaches the suspend point, and leave it
+         * up to the caller to check whether it is a safe suspend point,
+         * to match Windows behavior.
+         */
+        ASSERT(!ostd->suspended);
+        if (!thread_signal(tr, SUSPEND_SIGNAL)) {
+            ostd->suspend_count--;
+            mutex_unlock(&ostd->suspend_lock);
+            return false;
+        }
+    }
+    /* we can unlock before the wait loop b/c we're using a separate "resumed"
+     * bool and thread_resume holds the lock across its wait.  this way a resume
+     * can proceed as soon as the suspended thread is suspended, before the
+     * suspending thread gets scheduled again.
+     */
+    mutex_unlock(&ostd->suspend_lock);
+    /* FIXME PR 295561: use futex */
+    while (!ostd->suspended)
+        thread_yield();
+    return true;
+}
+
+bool
+thread_resume(thread_record_t *tr)
+{
+    os_thread_data_t *ostd = (os_thread_data_t *) tr->dcontext->os_field;
+    ASSERT(ostd != NULL);
+    /* This mutex prevents sending a re-suspend signal before the target
+     * reaches a safe post-resume point from a first suspend signal.
+     * Given that race, we can't just use atomic_add_exchange_int +
+     * atomic_dec_becomes_zero on suspend_count.
+     */
+    mutex_lock(&ostd->suspend_lock);
+    ASSERT(ostd->suspend_count > 0);
+    ostd->suspend_count--;
+    if (ostd->suspend_count > 0) {
+        mutex_unlock(&ostd->suspend_lock);
+        return true; /* still suspended */
+    }
+    ostd->wakeup = true;
+    /* FIXME PR 295561: use futex */
+    while (!ostd->resumed)
+        thread_yield();
+    ostd->wakeup = false;
+    ostd->resumed = false;
+    mutex_unlock(&ostd->suspend_lock);
+    return true;
+}
+
+bool
+thread_terminate(thread_record_t *tr)
+{
+    /* FIXME PR 297902: for NPTL this will take down the whole group:
+     * should instead send SIGUSR2 and have a flag set telling
+     * target thread to execute SYS_exit
+     */
+    return thread_signal(tr, SIGKILL);
+}
+
+bool
+thread_get_mcontext(thread_record_t *tr, dr_mcontext_t *mc)
+{
+    /* PR 212090: only works when target is suspended by us, and
+     * we then take the signal context
+     */
+    os_thread_data_t *ostd = (os_thread_data_t *) tr->dcontext->os_field;
+    ASSERT(ostd != NULL);
+    ASSERT(ostd->suspend_count > 0);
+    if (ostd->suspend_count == 0)
+        return false;
+    ASSERT(ostd->suspended_sigcxt != NULL);
+    sigcontext_to_mcontext(mc, ostd->suspended_sigcxt);
+    return true;
+}
+
+bool
+thread_set_mcontext(thread_record_t *tr, dr_mcontext_t *mc)
+{
+    /* PR 212090: only works when target is suspended by us, and
+     * we then replace the signal context
+     */
+    os_thread_data_t *ostd = (os_thread_data_t *) tr->dcontext->os_field;
+    ASSERT(ostd != NULL);
+    ASSERT(ostd->suspend_count > 0);
+    if (ostd->suspend_count == 0)
+        return false;
+    ASSERT(ostd->suspended_sigcxt != NULL);
+    mcontext_to_sigcontext(ostd->suspended_sigcxt, mc);
+    return true;
+}
+
+bool
+is_thread_currently_native(thread_record_t *tr)
+{
+    return (!tr->under_dynamo_control);
+}
+
+int
+get_num_processors()
+{
+    static uint num_cpu = 0;         /* cached value */
+    if (!num_cpu) {
+        num_cpu = get_nprocs_conf();
+        ASSERT(num_cpu);
+    }
+    return num_cpu;
+}
+
+
+#if defined(CLIENT_INTERFACE) || defined(HOT_PATCHING_INTERFACE)
+shlib_handle_t 
+load_shared_library(char *name)
+{
+    return dlopen(name, RTLD_LAZY);
+}
+#endif
+
+#if defined(CLIENT_INTERFACE)
+shlib_routine_ptr_t
+lookup_library_routine(shlib_handle_t lib, char *name)
+{
+    return dlsym(lib, name);
+}
+
+void
+unload_shared_library(shlib_handle_t lib)
+{
+    if (!DYNAMO_OPTION(avoid_dlclose))
+        dlclose(lib);
+}
+
+void
+shared_library_error(char *buf, int maxlen)
+{
+    char *err = dlerror();
+    strncpy(buf, err, maxlen-1);
+    buf[maxlen-1] = '\0'; /* strncpy won't put on trailing null if maxes out */
+}
+
+/* addr is any pointer known to lie within the library */
+bool
+shared_library_bounds(IN shlib_handle_t lib, IN byte *addr,
+                      OUT byte **start, OUT byte **end)
+{
+    /* PR 366195: dlopen() handle truly is opaque, so we have to use addr */
+    ASSERT(start != NULL && end != NULL);
+    *start = addr;
+    return (get_library_bounds(NULL, start, end, NULL, 0) > 0);
+}
+#endif /* defined(CLIENT_INTERFACE) */
+
+/* FIXME - not available in 2.0 or earlier kernels, not really an issue since no one
+ * should be running anything that old. */
+int
+llseek_syscall(int fd, int64 offset, int origin, int64 *result)
+{
+#ifdef X64
+    *result = dynamorio_syscall(SYS_lseek, 3, fd, offset, origin);
+    return (int) *result;
+#else
+    return dynamorio_syscall(SYS__llseek, 5, fd, (uint)((offset >> 32) & 0xFFFFFFFF),
+                             (uint)(offset & 0xFFFFFFFF), result, origin); 
+#endif
+}
+
+/* FIXME : implement? is easy for files, but is only used for dirs on linux
+ * right now which don't know how to do without sideeffects */
+bool
+os_file_exists(const char *fname, bool is_dir)
+{
+#if 0 /* compile out to prevent warning */
+    ASSERT_NOT_IMPLEMENTED(true);
+#endif
+    return true;
+}
+
+bool
+os_get_file_size(const char *file, uint64 *size)
+{
+    ASSERT_NOT_IMPLEMENTED(false);
+    return false;
+}
+
+bool
+os_get_file_size_by_handle(file_t fd, uint64 *size)
+{
+    int64 cur_pos = os_tell(fd);
+    int64 file_size = -1;
+    int result;
+    bool ret = true;
+
+    if (cur_pos == -1)
+        return false;
+
+    /* FIXME - multithread safety? Presumably caller is synchronizing file access. */
+    ASSERT_NOT_TESTED();
+    result = llseek_syscall(fd, 0, SEEK_END, &file_size);
+
+    if (result != 0 || file_size == -1)
+        ret = false;
+    else if (size != NULL)
+        *size = file_size;
+
+    result = llseek_syscall(fd, cur_pos, SEEK_SET, &file_size);
+    ASSERT(result == 0);
+
+    return ret;
+}
+
+/* created directory will be owned by effective uid,
+ * Note a symbolic link will never be followed.
+ */
+bool
+os_create_dir(const char *fname, create_directory_flags_t create_dir_flags)
+{
+    bool require_new = TEST(CREATE_DIR_REQUIRE_NEW, create_dir_flags);
+    int rc = dynamorio_syscall(SYS_mkdir, 2, fname, S_IRWXU|S_IRWXG);
+    ASSERT(create_dir_flags == CREATE_DIR_REQUIRE_NEW || 
+           create_dir_flags == CREATE_DIR_ALLOW_EXISTING);
+    return (rc == 0 || (!require_new && rc == -EEXIST));
+}
+
+int
+open_syscall(const char *file, int flags, int mode)
+{
+    ASSERT(file != NULL);
+    return dynamorio_syscall(SYS_open, 3, file, flags, mode);
+}
+
+int
+close_syscall(int fd)
+{
+    return dynamorio_syscall(SYS_close, 1, fd);
+}
+
+int
+dup_syscall(int fd)
+{
+    return dynamorio_syscall(SYS_dup, 1, fd);
+}
+
+ssize_t
+read_syscall(int fd, void *buf, size_t nbytes)
+{
+    return dynamorio_syscall(SYS_read, 3, fd, buf, nbytes);
+}
+
+ssize_t
+write_syscall(int fd, const void *buf, size_t nbytes)
+{
+    return dynamorio_syscall(SYS_write, 3, fd, buf, nbytes);
+}
+
+/* not easily accessible in header files */
+#ifdef X64
+/* not needed */
+# define O_LARGEFILE    0
+#else
+# define O_LARGEFILE    0100000
+#endif
+
+/* we assume that opening for writing wants to create file */
+file_t
+os_open(const char *fname, int os_open_flags)
+{
+    int res;
+    int flags = 0;
+    if (TEST(OS_OPEN_ALLOW_LARGE, os_open_flags))
+        flags |= O_LARGEFILE;
+    if (!TEST(OS_OPEN_WRITE, os_open_flags)) 
+        res = open_syscall(fname, flags|O_RDONLY, 0);
+    else {    
+        res = open_syscall(fname, flags|O_RDWR|O_CREAT|
+                           (TEST(OS_OPEN_APPEND, os_open_flags) ? 
+                            O_APPEND : 0)|
+                           (TEST(OS_OPEN_REQUIRE_NEW, os_open_flags) ? 
+                            O_EXCL : 0), 
+                           S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP);
+    }
+    if (res < 0)
+        return INVALID_FILE;
+    return res;
+}
+
+file_t
+os_open_directory(const char *fname, int os_open_flags)
+{
+    /* no special handling */
+    return os_open(fname, os_open_flags);
+}
+
+void
+os_close(file_t f)
+{
+    close_syscall(f);
+}
+
+ssize_t
+os_write(file_t f, const void *buf, size_t count)
+{
+    return write_syscall(f, buf, count);
+}
+
+ssize_t 
+os_read(file_t f, void *buf, size_t count)
+{
+    return read_syscall(f, buf, count);
+}
+
+void
+os_flush(file_t f)
+{
+    /* we're not using FILE*, so there is no buffering */
+}
+
+/* seek the current file position to offset bytes from origin, return true if succesful */
+bool
+os_seek(file_t f, int64 offset, int origin)
+{
+    int64 result;
+    int ret = 0;
+
+    ret = llseek_syscall(f, offset, origin, &result);
+
+    return (ret == 0);
+}
+
+/* return the current file position, -1 on failure */
+int64
+os_tell(file_t f)
+{
+    int64 result = -1;
+    int ret = 0;
+
+    ret = llseek_syscall(f, 0, SEEK_CUR, &result);
+
+    if (ret != 0)
+        return -1;
+
+    return result;
+}
+
+bool
+os_rename_file(const char *orig_name, const char *new_name, bool replace)
+{
+    ASSERT_NOT_IMPLEMENTED(false); /* FIXME PR 295534: NYI */ 
+    return false;
+}
+
+bool
+os_delete_mapped_file(const char *filename)
+{
+    ASSERT_NOT_IMPLEMENTED(false); /* FIXME PR 295534: NYI */ 
+    return false;
+}
+
+byte *
+os_map_file(file_t f, size_t size, uint64 offs, app_pc addr, uint prot,
+            bool copy_on_write)
+{
+    /* Linux mmap has 32-bit off_t; mmap2 considers it a page-size multiple;
+     * we stick w/ mmap and assume no need for larger offsets for now.
+     */
+    ASSERT(CHECK_TRUNCATE_TYPE_uint(offs));
+    byte *map = mmap_syscall(addr, size, memprot_to_osprot(prot),
+                             copy_on_write ? MAP_PRIVATE : MAP_SHARED,
+                             f, offs);
+    ASSERT_NOT_TESTED();
+    if (!mmap_syscall_succeeded(map))
+        map = NULL;
+    return map;
+}
+
+bool
+os_unmap_file(byte *map, size_t size)
+{
+    long res = munmap_syscall(map, size);
+    ASSERT_NOT_TESTED();
+    return (res == 0);
+}
+
+bool
+os_get_disk_free_space(/*IN*/ file_t file_handle,
+                       /*OUT*/ uint64 *AvailableQuotaBytes /*OPTIONAL*/,
+                       /*OUT*/ uint64 *TotalQuotaBytes /*OPTIONAL*/,
+                       /*OUT*/ uint64 *TotalVolumeBytes /*OPTIONAL*/)
+{
+    ASSERT_NOT_IMPLEMENTED(false); /* FIXME PR 295534: NYI */ 
+    return false;
+}
+
+void
+exit_syscall(long status)
+{
+    dynamorio_syscall(SYS_exit, 1, status);
+}
+
+/* FIXME: this one will not be easily internationalizable 
+   yet it is easier to have a syslog based Unix implementation with real strings.
+ */
+
+void 
+os_syslog(syslog_event_type_t priority, uint message_id, 
+          uint substitutions_num, va_list args) {
+    int native_priority;
+    switch (priority) {
+    case SYSLOG_INFORMATION: native_priority = LOG_INFO; break;
+    case SYSLOG_WARNING:     native_priority = LOG_WARNING; break;
+    case SYSLOG_CRITICAL:    native_priority = LOG_CRIT; break;
+    case SYSLOG_ERROR:       native_priority = LOG_ERR; break;
+    default:
+        ASSERT_NOT_REACHED();
+    }
+    /* can amount to passing a format string (careful here) to vsyslog */
+
+    /* Never let user controlled data in the format string! */
+    ASSERT_NOT_IMPLEMENTED(false);
+}
+
+static bool
+all_memory_areas_initialized(void)
+{
+    return (all_memory_areas != NULL && !vmvector_empty(all_memory_areas));
+}
+
+#if defined(DEBUG) && defined(INTERNAL)
+static void
+print_all_memory_areas(file_t outf)
+{
+    vmvector_iterator_t vmvi;
+    if (all_memory_areas == NULL || vmvector_empty(all_memory_areas))
+        return;
+
+    vmvector_iterator_start(all_memory_areas, &vmvi);
+    while (vmvector_iterator_hasnext(&vmvi)) {
+        app_pc start, end;
+        void *nxt = vmvector_iterator_next(&vmvi, &start, &end);
+        uint prot = (uint)(ptr_uint_t) nxt;
+        ASSERT(CHECK_TRUNCATE_TYPE_uint((ptr_uint_t)nxt));
+        print_file(outf, PFX"-"PFX" prot=%s\n", start, end,
+                   memprot_string(prot));
+    }
+    vmvector_iterator_stop(&vmvi);
+}
+#endif
+
+bool
+safe_read_ex(const void *base, size_t size, void *out_buf, size_t *bytes_read)
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    bool res = false;
+    STATS_INC(num_safe_reads);
+    if (dcontext != NULL) {
+        TRY_EXCEPT(dcontext, {
+            memcpy(out_buf, base, size);
+            res = true;
+         } , { /* EXCEPT */
+            /* nothing: res is already false */
+         });
+    } else {
+        /* this is subject to races, but should only happen at init/attach when
+         * there should only be one live thread.
+         */
+        if (is_readable_without_exception(base, size)) {
+            memcpy(out_buf, base, size);
+            res = true;
+        }
+    }
+    if (res) {
+        if (bytes_read != NULL)
+            *bytes_read = size;
+        return true;
+    } else {
+        if (bytes_read != NULL)
+            *bytes_read = 0;
+        return false;
+    }
+}
+
+/* FIXME - fold this together with safe_read_ex() (is a lot of places to update) */
+bool
+safe_read(const void *base, size_t size, void *out_buf)
+{
+    return safe_read_ex(base, size, out_buf, NULL);
+}
+
+bool
+safe_write_ex(void *base, size_t size, const void *in_buf, size_t *bytes_written)
+{
+    uint prot;
+    byte *region_base;
+    size_t region_size;
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    bool res = false;
+    if (bytes_written != NULL)
+        *bytes_written = 0;
+
+    if (dcontext != NULL) {
+        TRY_EXCEPT(dcontext, {
+            /* We abort on the 1st fault, just like safe_read */
+            memcpy(base, in_buf, size);
+            res = true;
+         } , { /* EXCEPT */
+            /* nothing: res is already false */
+         });
+    } else {
+        /* this is subject to races, but should only happen at init/attach when
+         * there should only be one live thread.
+         */
+        /* on x86 must be readable to be writable so start with that */
+        if (is_readable_without_exception(base, size) &&
+            get_memory_info_from_os(base, &region_base, &region_size, &prot) &&
+            TEST(MEMPROT_WRITE, prot)) {
+            size_t bytes_checked = region_size - ((byte *)base - region_base);
+            while (bytes_checked < size) {
+                if (!get_memory_info_from_os(region_base + region_size, &region_base,
+                                             &region_size, &prot) ||
+                    !TEST(MEMPROT_WRITE, prot))
+                    return false;
+                bytes_checked += region_size;
+            }
+        } else {
+            return false;
+        }
+        /* ok, checks passed do the copy, FIXME - because of races this isn't safe! */
+        memcpy(base, in_buf, size);
+        res = true;
+    }
+
+    if (res) {
+        if (bytes_written != NULL)
+            *bytes_written = size;
+    }
+    return res;
+}
+
+/* is_readable_without_exception checks to see that all bytes with addresses
+ * from pc to pc+size-1 are readable and that reading from there won't
+ * generate an exception. if 'from_os' is true, check what the os thinks
+ * the prot bits are instead of using the all memory list.
+ */
+static bool
+is_readable_without_exception_internal(const byte *pc, size_t size, bool query_os)
+{
+    uint prot;
+    byte *check_pc = (byte *) ALIGN_BACKWARD(pc, PAGE_SIZE);
+    if (size > ((byte *)POINTER_MAX - pc))
+        size = (byte *)POINTER_MAX - pc;
+    do {
+        bool rc = query_os ?
+            get_memory_info_from_os(check_pc, NULL, NULL, &prot) :
+            get_memory_info(check_pc, NULL, NULL, &prot);
+        if (!rc || !TESTANY(MEMPROT_READ|MEMPROT_EXEC, prot))
+            return false;
+        check_pc += PAGE_SIZE;
+    } while (check_pc != 0/*overflow*/ && check_pc < pc+size);
+    return true;
+}
+
+bool
+is_readable_without_exception(const byte *pc, size_t size)
+{
+    /* FIXME case 9745: disabling all_memory_areas here since causing a ton
+     * of asserts; re-enable once we're sure it truly matches the real
+     * world
+     */
+    return is_readable_without_exception_internal(pc, size, 
+#ifdef HAVE_PROC_MAPS
+                                                  true /* do query os */
+#else
+                                                  false
+#endif
+                                                  );
+}
+
+/* Identical to is_readable_without_exception except that the os is queried
+ * for info on the indicated region */
+bool
+is_readable_without_exception_query_os(byte *pc, size_t size)
+{
+    return is_readable_without_exception_internal(pc, size, true);
+}
+
+bool
+is_user_address(byte *pc)
+{
+    /* FIXME: NYI */
+    /* note returning true will always skip the case 9022 logic on Linux */
+    return true;
+}
+
+/* change protections on memory region starting at pc of length length */
+bool
+set_protection(byte *pc, size_t length, uint prot/*MEMPROT_*/)
+{
+    app_pc start_page = (app_pc) PAGE_START(pc);
+    uint num_bytes = ALIGN_FORWARD(length + (pc - start_page), PAGE_SIZE);
+    long res = 0;
+    uint flags = memprot_to_osprot(prot);
+#ifdef IA32_ON_IA64
+    LOG(THREAD_GET, LOG_VMAREAS, 1, "protection change not supported on IA64\n"); 
+    LOG(THREAD_GET, LOG_VMAREAS, 1, " attempted change_prot("PFX", "PIFX", %s) => "
+        "mprotect("PFX", "PIFX")==%d pages\n",
+        pc, length, memprot_string(prot), start_page, num_bytes,
+        num_bytes / PAGE_SIZE);
+#else
+    DOSTATS({
+        /* once on each side of prot, to get on right side of writability */
+        if (!TEST(PROT_WRITE, flags)) {
+            STATS_INC(protection_change_calls);
+            STATS_ADD(protection_change_pages, num_bytes / PAGE_SIZE);
+        }
+    });
+    res = mprotect_syscall((void *) start_page, num_bytes, flags);
+    if (res != 0) 
+        return false;
+    LOG(THREAD_GET, LOG_VMAREAS, 3, "change_prot("PFX", "PIFX", %s) => "
+        "mprotect("PFX", "PIFX", %d)==%d pages\n",
+        pc, length, memprot_string(prot), start_page, num_bytes, flags,
+        num_bytes / PAGE_SIZE);
+#endif
+    DOSTATS({
+        /* once on each side of prot, to get on right side of writability */
+        if (TEST(PROT_WRITE, flags)) {
+            STATS_INC(protection_change_calls);
+            STATS_ADD(protection_change_pages, num_bytes / PAGE_SIZE);
+        }
+    });
+    all_memory_areas_lock();
+    ASSERT(vmvector_overlap(all_memory_areas, start_page, start_page + num_bytes));
+    LOG(GLOBAL, LOG_VMAREAS, 3, "\tupdating all_memory_areas "PFX"-"PFX" prot->%d\n",
+        start_page, start_page + num_bytes, osprot_to_memprot(flags));
+    update_all_memory_areas(start_page, start_page + num_bytes,
+                            osprot_to_memprot(flags));
+    all_memory_areas_unlock();
+    return true;
+}
+
+/* change protections on memory region starting at pc of length length */
+bool
+change_protection(byte *pc, size_t length, bool writable)
+{
+    uint flags = (writable) ? (MEMPROT_READ|MEMPROT_WRITE) : (MEMPROT_READ);
+    return set_protection(pc, length, flags);
+}
+
+/* make pc's page writable
+ * FIXME: how get current protection?  would like to keep old read/exec flags
+ */
+bool
+make_writable(byte *pc, size_t size)
+{
+    long res;
+    app_pc start_page = (app_pc) PAGE_START(pc);
+    size_t prot_size = (size == 0) ? PAGE_SIZE : size;
+    uint prot = PROT_EXEC|PROT_READ|PROT_WRITE;
+
+    ASSERT(start_page == pc && ALIGN_FORWARD(size, PAGE_SIZE) == size);
+#ifdef IA32_ON_IA64
+    LOG(THREAD_GET, LOG_VMAREAS, 1, "protection change not supported on IA64\n"); 
+    LOG(THREAD_GET, LOG_VMAREAS, 3,
+        "attempted make_writable: pc "PFX" -> "PFX"-"PFX"\n",
+        pc, start_page, start_page + prot_size);
+#else
+    res = mprotect_syscall((void *) start_page, prot_size, prot);
+    LOG(THREAD_GET, LOG_VMAREAS, 3, "make_writable: pc "PFX" -> "PFX"-"PFX" %d\n",
+        pc, start_page, start_page + prot_size, res);
+    ASSERT(res == 0);
+    if (res != 0) 
+        return false;
+#endif
+    STATS_INC(protection_change_calls);
+    STATS_ADD(protection_change_pages, size / PAGE_SIZE);
+
+    /* update all_memory_areas list with the protection change */
+    all_memory_areas_lock();
+    ASSERT(vmvector_overlap(all_memory_areas, start_page, start_page + prot_size));
+    LOG(GLOBAL, LOG_VMAREAS, 3, "\tupdating all_memory_areas "PFX"-"PFX" prot->%d\n",
+        start_page, start_page + prot_size, osprot_to_memprot(prot));
+    update_all_memory_areas(start_page, start_page + prot_size,
+                            osprot_to_memprot(prot));
+    all_memory_areas_unlock();
+    return true;
+}
+
+/* like make_writable but adds COW */
+bool make_copy_on_writable(byte *pc, size_t size)
+{
+    /* FIXME: for current usage this should be fine */
+    return make_writable(pc, size);
+}
+
+/* make pc's page unwritable 
+ * FIXME: how get current protection?  would like to keep old read/exec flags
+ */
+void
+make_unwritable(byte *pc, size_t size)
+{
+    long res;
+    app_pc start_page = (app_pc) PAGE_START(pc);
+    size_t prot_size = (size == 0) ? PAGE_SIZE : size;
+    uint prot = PROT_EXEC|PROT_READ;
+
+    ASSERT(start_page == pc && ALIGN_FORWARD(size, PAGE_SIZE) == size);
+    /* inc stats before making unwritable, in case messing w/ data segment */
+    STATS_INC(protection_change_calls);
+    STATS_ADD(protection_change_pages, size / PAGE_SIZE);
+#ifdef IA32_ON_IA64
+    LOG(THREAD_GET, LOG_VMAREAS, 1, "protection change not supported on IA64\n"); 
+    LOG(THREAD_GET, LOG_VMAREAS, 3,
+        "attempted make_writable: pc "PFX" -> "PFX"-"PFX"\n",
+        pc, start_page, start_page + prot_size);
+#else
+    res = mprotect_syscall((void *) start_page, prot_size, prot);
+    LOG(THREAD_GET, LOG_VMAREAS, 3, "make_unwritable: pc "PFX" -> "PFX"-"PFX"\n",
+        pc, start_page, start_page + prot_size);
+    ASSERT(res == 0);
+
+    /* update all_memory_areas list with the protection change */
+    if (all_memory_areas_initialized()) {
+        all_memory_areas_lock();
+        ASSERT(vmvector_overlap(all_memory_areas, start_page, start_page + prot_size));
+        LOG(GLOBAL, LOG_VMAREAS, 3, "\tupdating all_memory_areas "PFX"-"PFX" prot->%d\n",
+            start_page, start_page + prot_size, osprot_to_memprot(prot));
+        update_all_memory_areas(start_page, start_page + prot_size,
+                                osprot_to_memprot(prot));
+        all_memory_areas_unlock();
+    }
+#endif
+}
+
+/****************************************************************************/
+/* SYSTEM CALLS */
+
+/* SYS_ defines are in /usr/include/bits/syscall.h
+ * numbers used by libc are in /usr/include/asm/unistd.h
+ * kernel defines are in /usr/src/linux-2.4/include/asm-i386/unistd.h
+ * kernel function names are in /usr/src/linux/arch/i386/kernel/entry.S
+ *
+ * For now, we've copied the SYS/NR defines from syscall.h and unistd.h
+ * and put them in our own local syscall.h.
+ */
+
+bool
+ignorable_system_call(int num)
+{
+    switch (num) {
+#if defined(SYS_exit_group)
+    case SYS_exit_group:
+#endif
+    case SYS_exit:
+    case SYS_mmap:
+#ifndef X64
+    case SYS_mmap2:
+#endif
+    case SYS_munmap:
+    case SYS_mremap:
+    case SYS_mprotect:
+    case SYS_execve:
+    case SYS_clone:
+    case SYS_fork:
+    case SYS_vfork:
+    case SYS_kill:
+#if defined(SYS_tkill)
+    case SYS_tkill:
+#endif
+#if defined(SYS_tgkill)
+    case SYS_tgkill:
+#endif
+#ifndef X64
+    case SYS_signal:
+    case SYS_sigaction:
+    case SYS_sigsuspend:
+    case SYS_sigpending:
+    case SYS_sigreturn:
+    case SYS_sigprocmask:
+#endif
+    case SYS_rt_sigreturn:
+    case SYS_rt_sigaction:
+    case SYS_rt_sigprocmask:
+    case SYS_rt_sigpending:
+    case SYS_rt_sigtimedwait:
+    case SYS_rt_sigqueueinfo:
+    case SYS_rt_sigsuspend:
+    case SYS_sigaltstack:
+#ifndef X64
+    case SYS_sgetmask:
+    case SYS_ssetmask:
+#endif
+    case SYS_setitimer:
+    case SYS_getitimer:
+    case SYS_close:
+        return false;
+    default:
+        return true;
+    }
+}
+
+typedef struct {
+        unsigned long addr;
+        unsigned long len;
+        unsigned long prot;
+        unsigned long flags;
+        unsigned long fd;
+        unsigned long offset;
+} mmap_arg_struct_t;
+
+static inline reg_t *
+sys_param_addr(dcontext_t *dcontext, int num)
+{
+    /* we force-inline get_mcontext() and so don't take it as a param */
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+#ifdef X64
+    switch (num) {
+    case 0: return &mc->xdi;
+    case 1: return &mc->xsi;
+    case 2: return &mc->xdx;
+    case 3: return &mc->r10; /* since rcx holds retaddr for syscall instr */
+    case 4: return &mc->r8;
+    case 5: return &mc->r9;
+    default: CLIENT_ASSERT(false, "invalid system call parameter number");
+    }
+#else
+    /* even for vsyscall where ecx (syscall) or esp (sysenter) are saved into
+     * ebp, the original parameter registers are not yet changed pre-syscall,
+     * except for ebp, which is pushed on the stack:
+     *     0xffffe400  55                   push   %ebp %esp -> %esp (%esp)
+     *     0xffffe401  89 cd                mov    %ecx -> %ebp
+     *     0xffffe403  0f 05                syscall -> %ecx
+     *   
+     *     0xffffe400  51                   push   %ecx %esp -> %esp (%esp) 
+     *     0xffffe401  52                   push   %edx %esp -> %esp (%esp) 
+     *     0xffffe402  55                   push   %ebp %esp -> %esp (%esp) 
+     *     0xffffe403  89 e5                mov    %esp -> %ebp 
+     *     0xffffe405  0f 34                sysenter -> %esp 
+     */
+    switch (num) {
+    case 0: return &mc->xbx;
+    case 1: return &mc->xcx;
+    case 2: return &mc->xdx;
+    case 3: return &mc->xsi;
+    case 4: return &mc->xdi;
+    /* FIXME: do a safe_read: but what about performance?
+     * See the #if 0 below, as well. */
+    case 5: return (dcontext->sys_was_int ? &mc->xbp : ((reg_t*)mc->xsp));
+    default: CLIENT_ASSERT(false, "invalid system call parameter number");
+    }
+#endif
+    return 0;
+}
+
+static inline reg_t
+sys_param(dcontext_t *dcontext, int num)
+{
+    return *sys_param_addr(dcontext, num);
+}
+
+/* since always coming from dispatch now, only need to set mcontext */
+#define SET_RETURN_VAL(dc, val)  get_mcontext(dc)->xax = (val);
+
+#ifdef CLIENT_INTERFACE
+DR_API
+reg_t
+dr_syscall_get_param(void *drcontext, int param_num)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    dcontext->client_data->in_post_syscall = true;
+    CLIENT_ASSERT(dcontext->client_data->in_pre_syscall,
+                  "dr_syscall_get_param() can only be called from pre-syscall event");
+    return sys_param(dcontext, param_num);
+}
+
+DR_API
+void
+dr_syscall_set_param(void *drcontext, int param_num, reg_t new_value)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(dcontext->client_data->in_pre_syscall ||
+                  dcontext->client_data->in_post_syscall,
+                  "dr_syscall_set_param() can only be called from a syscall event");
+    *sys_param_addr(dcontext, param_num) = new_value;
+}
+
+DR_API
+reg_t
+dr_syscall_get_result(void *drcontext)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(dcontext->client_data->in_post_syscall,
+                  "dr_syscall_get_param() can only be called from post-syscall event");
+    return get_mcontext(dcontext)->xax;
+}
+
+DR_API
+void
+dr_syscall_set_result(void *drcontext, reg_t value)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(dcontext->client_data->in_pre_syscall ||
+                  dcontext->client_data->in_post_syscall,
+                  "dr_syscall_set_result() can only be called from a syscall event");
+    SET_RETURN_VAL(dcontext, value);
+}
+
+DR_API
+void
+dr_syscall_set_sysnum(void *drcontext, int new_num)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    CLIENT_ASSERT(dcontext->client_data->in_pre_syscall ||
+                  dcontext->client_data->in_post_syscall,
+                  "dr_syscall_set_sysnum() can only be called from a syscall event");
+    mc->xax = new_num;
+}
+
+DR_API
+void
+dr_syscall_invoke_another(void *drcontext)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(dcontext->client_data->in_post_syscall,
+                  "dr_syscall_invoke_another() can only be called from post-syscall event");
+    LOG(THREAD, LOG_SYSCALLS, 2, "invoking additional syscall on client request\n");
+    dcontext->client_data->invoke_another_syscall = true;
+    if (get_syscall_method() == SYSCALL_METHOD_SYSENTER) {
+        dr_mcontext_t *mc = get_mcontext(dcontext);
+        /* restore xbp to xsp */
+        mc->xbp = mc->xsp;
+    }
+    /* for x64 we don't need to copy xcx into r10 b/c we use r10 as our param */
+}
+#endif /* CLIENT_INTERFACE */
+
+static inline bool
+is_clone_thread_syscall_helper(ptr_uint_t sysnum, ptr_uint_t flags)
+{
+    return (sysnum == SYS_vfork ||
+            (sysnum == SYS_clone && TEST(CLONE_VM, flags)));
+}
+
+
+bool
+is_clone_thread_syscall(dcontext_t *dcontext)
+{
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    return is_clone_thread_syscall_helper(mc->xax, sys_param(dcontext, 0));
+}
+
+bool
+was_clone_thread_syscall(dcontext_t *dcontext)
+{
+    return is_clone_thread_syscall_helper(dcontext->sys_num,
+                                          /* flags in param0 */
+                                          dcontext->sys_param0);
+}
+
+static inline bool
+is_sigreturn_syscall_helper(int sysnum)
+{
+    return (IF_NOT_X64(sysnum == SYS_sigreturn ||) sysnum == SYS_rt_sigreturn);
+}
+
+bool
+is_sigreturn_syscall(dcontext_t *dcontext)
+{
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    return is_sigreturn_syscall_helper(mc->xax);
+}
+
+bool
+was_sigreturn_syscall(dcontext_t *dcontext)
+{
+    return is_sigreturn_syscall_helper(dcontext->sys_num);
+}
+
+/* process a signal this process/thread is sending to itself */
+static void
+handle_self_signal(dcontext_t *dcontext, uint sig)
+{
+    /* FIXME PR 297903: watch for all DEFAULT_TERMINATE signals,
+     * and for any thread in the group, not just self.
+     *
+     * FIXME PR 297033: watch for SIGSTOP and SIGCONT.
+     *
+     * With -intercept_all_signals, we only need to watch for SIGKILL
+     * and SIGSTOP here, and we avoid the FIXMEs below.  If it's fine
+     * for DR not to clean up on a SIGKILL, then SIGSTOP is all that's
+     * left (at least once we have PR 297033 and are intercepting the
+     * various STOP variations and CONT).
+     */
+    if (sig == SIGABRT && !DYNAMO_OPTION(intercept_all_signals)) {
+        LOG(GLOBAL, LOG_TOP|LOG_SYSCALLS, 1,
+            "thread %d sending itself a SIGABRT\n", get_thread_id());
+        KSTOP(num_exits_dir_syscall);
+        /* FIXME: need to check whether app has a handler for SIGABRT! */
+        /* FIXME PR 211180/6723: this will do SYS_exit rather than the SIGABRT.
+         * Should do set_default_signal_action(SIGABRT) (and set a flag so
+         * no races w/ another thread re-installing?) and then SYS_kill.
+         */
+        cleanup_and_terminate(dcontext, SYS_exit, -1, 0,
+                              (get_num_threads() == 1 && !dynamo_exited));
+        ASSERT_NOT_REACHED();
+    }
+}
+
+/* System call interception: put any special handling here
+ * Arguments come from the pusha right before the call
+ */
+
+/* WARNING: flush_fragments_and_remove_region assumes that pre and post system
+ * call handlers do not examine or modify fcache or its fragments in any
+ * way except for calling flush_fragments_and_remove_region!
+ */
+
+/* WARNING: All registers are IN values, but NOT OUT values --
+ * must set mcontext's register for that.
+ */
+/* Returns false if system call should NOT be executed
+ * Returns true if system call should go ahead
+ */
+/* FIXME: split out specific handlers into separate routines
+ */
+bool
+pre_system_call(dcontext_t *dcontext)
+{
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    bool execute_syscall = true;
+    where_am_i_t old_whereami = dcontext->whereami;
+    dcontext->whereami = WHERE_SYSCALL_HANDLER;
+    /* FIXME We haven't yet done the work to detect which syscalls we
+     * can determine a priori will fail. Once we do, we will set the
+     * expect_last_syscall_to_fail to true for those case, and can
+     * confirm in post_system_call() that the syscall failed as
+     * expected.
+     */
+    DODEBUG(dcontext->expect_last_syscall_to_fail = false;);
+
+    /* save key register values for post_system_call (they get clobbered
+     * in syscall itself)
+     */
+    dcontext->sys_num = mc->xax;
+
+    RSTATS_INC(pre_syscall);
+    DOSTATS({
+        if (ignorable_system_call(mc->xax))
+            STATS_INC(pre_syscall_ignorable);
+    });
+    LOG(THREAD, LOG_SYSCALLS, 2, "system call %d\n", mc->xax);
+
+    /* PR 313715: If we fail to hook the vsyscall page (xref PR 212570, PR 288330)
+     * we fall back on int, but we have to tweak syscall param #5 (ebp)
+     * Once we have PR 288330 we can remove this.
+     */
+    if (should_syscall_method_be_sysenter() && !dcontext->sys_was_int) {
+        dcontext->sys_xbp = mc->xbp;
+        /* not using SAFE_READ due to performance concerns (we do this for
+         * every single system call on systems where we can't hook vsyscall!)
+         */
+        TRY_EXCEPT(dcontext, /* try */ {
+            mc->xbp = *(reg_t*)mc->xsp;
+        }, /* except */ {
+            ASSERT_NOT_REACHED();
+            mc->xbp = 0;
+        });
+    }
+
+    switch (mc->xax) {
+
+#if defined(SYS_exit_group)
+    case SYS_exit_group:
+# ifdef VMX86_SERVER
+        if (os_in_vmkernel_32bit()) {
+            /* on esx 3.5 => ENOSYS, so wait for SYS_exit */
+            DODEBUG({ dcontext->expect_last_syscall_to_fail = true; });
+            break;
+        }
+# endif
+        /* fall-through */
+#endif
+    case SYS_exit: {
+        LOG(GLOBAL, LOG_TOP|LOG_SYSCALLS, 1,
+            "SYS_exit(%d) encountered in thread %d\n", mc->xax, get_thread_id());
+        if (get_num_threads() == 1 && !dynamo_exited) {
+            LOG(THREAD, LOG_TOP|LOG_SYSCALLS, 1,
+                "SYS_exit(%d) encountered in final thread %d, exiting DynamoRIO\n",
+                mc->xax, get_thread_id());
+            /* we want to clean up even if not automatic startup! */
+            automatic_startup = true;
+        } else {
+            LOG(THREAD, LOG_TOP|LOG_THREADS|LOG_SYSCALLS, 1,
+                "SYS_exit(%d) encountered, cleaning up thread %d\n", mc->xax,
+                get_thread_id());
+        }
+        KSTOP(num_exits_dir_syscall);
+
+        /* N.B.: in all other cases, we just let interp exit,
+         * and then either _fini is executed on app's stack,
+         * or dynamorio_app_exit is called on app's stack...
+         * here, we must switch back to app's stack, and hope
+         * it's valid, because once we exit our stack is gone!
+         * Can't access local vars once kill dstack, so we grab
+         * the exit val and push it on the new stack right now
+         * as an argument for the later call to _exit
+         */
+        cleanup_and_terminate(dcontext, mc->xax, sys_param(dcontext, 0),
+                              sys_param(dcontext, 1),
+                              (get_num_threads() == 1 && !dynamo_exited));
+        break;
+    }
+
+    /****************************************************************************/
+    /* MEMORY REGIONS */
+
+#ifndef X64
+    case SYS_mmap: {
+        /* in /usr/src/linux/arch/i386/kernel/sys_i386.c:
+           asmlinkage int old_mmap(struct mmap_arg_struct_t *arg)
+         */
+        mmap_arg_struct_t *arg = (mmap_arg_struct_t *) sys_param(dcontext, 0);
+        DOLOG(2, LOG_SYSCALLS, {
+            mmap_arg_struct_t arg_buf;
+            if (safe_read(arg, sizeof(mmap_arg_struct_t), &arg_buf)) {
+                void *addr = (void *) arg->addr;
+                size_t len = (size_t) arg->len;
+                uint prot = (uint) arg->prot;
+                LOG(THREAD, LOG_SYSCALLS, 2,
+                    "syscall: mmap addr="PFX" size="PIFX" prot=0x%x"
+                    " flags="PIFX" offset="PIFX" fd=%d\n",
+                    addr, len, prot, arg->flags, arg->offset, arg->fd);
+            }
+        });
+        /* post_system_call does the work */
+        dcontext->sys_param0 = (reg_t) arg;
+        break;
+    }
+#endif
+    case IF_X64_ELSE(SYS_mmap,SYS_mmap2): {
+        /* in /usr/src/linux/arch/i386/kernel/sys_i386.c:
+           asmlinkage long sys_mmap2(unsigned long addr, unsigned long len,
+             unsigned long prot, unsigned long flags,
+             unsigned long fd, unsigned long pgoff)
+         */
+        void *addr = (void *) sys_param(dcontext, 0);
+        size_t len = (size_t) sys_param(dcontext, 1);
+        uint prot = (uint) sys_param(dcontext, 2);
+        LOG(THREAD, LOG_SYSCALLS, 2,
+            "syscall: mmap2 addr="PFX" size="PIFX" prot=0x%x"
+            " flags="PIFX" offset="PIFX" fd=%d\n",
+            addr, len, prot, sys_param(dcontext, 3),
+            sys_param(dcontext, 5), sys_param(dcontext, 4));
+        /* post_system_call does the work */
+        dcontext->sys_param0 = (reg_t) addr;
+        dcontext->sys_param1 = len;
+        dcontext->sys_param2 = prot;
+        break;
+    }
+    /* must flush stale fragments when we see munmap/mremap */
+    case SYS_munmap: {
+        /* in /usr/src/linux/mm/mmap.c:
+           asmlinkage long sys_munmap(unsigned long addr, uint len)
+         */
+        app_pc addr = (void *) sys_param(dcontext, 0);
+        size_t len = (size_t) sys_param(dcontext, 1);
+        LOG(THREAD, LOG_SYSCALLS, 2, "syscall: munmap addr="PFX" size="PFX"\n",
+            addr, len);
+        RSTATS_INC(num_app_munmaps);
+        /* FIXME addr is supposed to be on a page boundary so we
+         * could detect that condition here and set
+         * expect_last_syscall_to_fail.
+         */
+        /* save params in case an undo is needed in post_system_call */
+        dcontext->sys_param0 = (reg_t) addr;
+        dcontext->sys_param1 = len;
+        /* We assume that the unmap will succeed and so are conservative
+         * and remove the region from exec areas and flush all fragments
+         * prior to issuing the syscall. If the unmap fails, we try to
+         * recover in post_system_call() by re-adding the region. This
+         * approach has its shortcomings -- see comments below in
+         * post_system_call().
+         */
+        /* Check for unmapping a module. */
+        os_get_module_info_lock();
+        if (module_overlaps(addr, len)) {
+            /* FIXME - handle unmapping more then one module at once, or only unmapping
+             * part of a module (for which case should adjust view size? or treat as full
+             * unmap?). Theoretical for now as we haven't seen this. */
+            module_area_t *ma = module_pc_lookup(addr);
+            ASSERT_CURIOSITY(ma != NULL);
+            ASSERT_CURIOSITY(addr == ma->start);
+            /* XREF 307599 on rounding module end to the next PAGE boundary */
+            ASSERT_CURIOSITY((app_pc)ALIGN_FORWARD(addr+len, PAGE_SIZE) == ma->end);
+            os_get_module_info_unlock();
+            if (ma != NULL)
+                module_list_remove(addr, ALIGN_FORWARD(len, PAGE_SIZE));
+        } else 
+            os_get_module_info_unlock();
+        app_memory_deallocation(dcontext, (app_pc)addr, len, 
+                                false /* don't own thread_initexit_lock */,
+                                true /* image, FIXME: though not necessarily */);
+        /* FIXME: case 4983 use is_elf_so_header() */
+        remove_from_all_memory_areas(addr, addr + len);
+        break;
+    }
+    case SYS_mremap: {
+        /* in /usr/src/linux/mm/mmap.c:
+           asmlinkage unsigned long sys_mremap(unsigned long addr,
+             unsigned long old_len, unsigned long new_len,
+             unsigned long flags, unsigned long new_addr)
+        */
+        app_pc addr = (void *) sys_param(dcontext, 0);
+        size_t old_len = (size_t) sys_param(dcontext, 1);
+        size_t new_len = (size_t) sys_param(dcontext, 2);
+        LOG(THREAD, LOG_SYSCALLS, 2, "syscall: mremap addr="PFX" size="PFX"\n",
+            addr, old_len);
+        /* post_system_call does the work */
+        dcontext->sys_param0 = (reg_t) addr;
+        dcontext->sys_param1 = old_len;
+        dcontext->sys_param2 = new_len;
+        DODEBUG({
+            /* we don't expect to see remappings of modules */
+            os_get_module_info_lock();
+            ASSERT_CURIOSITY(!module_overlaps(addr, old_len));
+            os_get_module_info_unlock();
+        });
+        break;
+    }
+    case SYS_mprotect: {
+        /* in /usr/src/linux/mm/mprotect.c:
+           asmlinkage long sys_mprotect(unsigned long start, uint len,
+           unsigned long prot)
+        */
+        uint res;
+        app_pc addr  = (void *) sys_param(dcontext, 0);
+        size_t len  = (size_t) sys_param(dcontext, 1);
+        uint prot = (uint) sys_param(dcontext, 2);
+        uint new_memprot;
+        /* save params in case an undo is needed in post_system_call */
+        dcontext->sys_param0 = (reg_t) addr;
+        dcontext->sys_param1 = len;
+        dcontext->sys_param2 = prot;
+        LOG(THREAD, LOG_SYSCALLS, 2,
+            "syscall: mprotect addr="PFX" size="PFX" prot=%s\n",
+            addr, len, memprot_string(osprot_to_memprot(prot)));
+        res = app_memory_protection_change(dcontext, addr, len, 
+                                           osprot_to_memprot(prot),
+                                           &new_memprot, NULL);
+        if (res != DO_APP_MEM_PROT_CHANGE) {
+            if (res == FAIL_APP_MEM_PROT_CHANGE) {
+                ASSERT_NOT_IMPLEMENTED(false); /* return code? */
+            } else {
+                ASSERT_NOT_IMPLEMENTED(res != SUBSET_APP_MEM_PROT_CHANGE);
+                ASSERT_NOT_REACHED();
+            }
+            execute_syscall = false;
+        }
+        else {
+            /* FIXME Store state for undo if the syscall fails. */
+            /* update all_memory_areas list */
+            all_memory_areas_lock();
+            ASSERT(vmvector_overlap(all_memory_areas, addr, addr + len));
+            LOG(GLOBAL, LOG_VMAREAS|LOG_SYSCALLS, 3,
+                "\tupdating all_memory_areas "PFX"-"PFX" prot->%d\n",
+                addr, addr+len, osprot_to_memprot(prot));
+            update_all_memory_areas(addr, addr+len, osprot_to_memprot(prot));
+            all_memory_areas_unlock();
+        }
+        break;
+    }
+    case SYS_uselib: {
+        /* Used to get the kernel to load a share library (legacy system call).
+         * Was primarily used when statically linking to dynamically loaded shared
+         * libraries that were loaded at known locations.  Shouldn't be used by
+         * applications using the dynamic loader (ld) which is currently the only
+         * way we can inject so we don't expect to see this.  PR 307621. */
+        ASSERT_NOT_IMPLEMENTED(false);
+        break;
+    }
+
+    /****************************************************************************/
+    /* SPAWNING */
+
+    case SYS_clone: {
+        /* in /usr/src/linux/arch/i386/kernel/process.c 
+         * 32-bit params: flags, newsp, ptid, tls, ctid
+         * 64-bit params: should be the same yet tls (for ARCH_SET_FS) is in r8?!?
+         *   I don't see how sys_clone gets its special args: shouldn't it
+         *   just get pt_regs as a "special system call"?
+         *   sys_clone(unsigned long clone_flags, unsigned long newsp,
+         *     void __user *parent_tid, void __user *child_tid, struct pt_regs *regs)
+         */
+        uint flags = (uint) sys_param(dcontext, 0);
+        LOG(THREAD, LOG_SYSCALLS, 2, "syscall: clone with flags = "PFX"\n", flags);
+        LOG(THREAD, LOG_SYSCALLS, 2, "args: "PFX", "PFX", "PFX", "PFX", "PFX"\n",
+            sys_param(dcontext, 0), sys_param(dcontext, 1), sys_param(dcontext, 2),
+            sys_param(dcontext, 3), sys_param(dcontext, 4));
+        handle_clone(dcontext, flags);
+        if ((flags & CLONE_VM) == 0) {
+            LOG(THREAD, LOG_SYSCALLS, 1, "\tWARNING: CLONE_VM not set!\n");
+        }
+        /* save for post_system_call */
+        dcontext->sys_param0 = (reg_t) flags;
+        break;
+    }
+
+    case SYS_vfork: {
+        /* treat as if sys_clone with flags just as sys_vfork does */
+        /* in /usr/src/linux/arch/i386/kernel/process.c */
+        uint flags = CLONE_VFORK | CLONE_VM | SIGCHLD;
+        LOG(THREAD, LOG_SYSCALLS, 2, "syscall: vfork\n");
+        handle_clone(dcontext, flags);
+
+        /* save for post_system_call, treated as if SYS_clone */
+        dcontext->sys_param0 = (reg_t) flags;
+        break;
+    }
+
+    case SYS_fork: {
+        LOG(THREAD, LOG_SYSCALLS, 2, "syscall: fork\n");
+        break;
+    }
+
+    case SYS_execve: {
+        /* in /usr/src/linux/arch/i386/kernel/process.c:
+         asmlinkage int sys_execve(struct pt_regs regs) { ...
+         error = do_execve(filename, (char **) regs.xcx, (char **) regs.xdx, &regs);
+         * in fs/exec.c:
+         int do_execve(char * filename, char ** argv, char ** envp, struct pt_regs * regs)
+         */
+        /* We need to make sure we get injected into the new image:
+         * we simply make sure LD_PRELOAD contains us, and that our directory
+         * is on LD_LIBRARY_PATH (seems not to work to put absolute paths in
+         * LD_PRELOAD).
+         * FIXME: this doesn't work for setuid programs
+         * We also pass the current DYNAMORIO_OPTIONS to the new image.
+         * FIXME: security hole here -- have some other way to pass the options?
+         */
+        DEBUG_DECLARE(char *fname = (char *)  sys_param(dcontext, 0);)
+        DEBUG_DECLARE(char **argv = (char **) sys_param(dcontext, 1);)
+        char **envp = (char **) sys_param(dcontext, 2);
+        int i, preload = -1, ldpath = -1, ops = -1;
+        bool preload_us = false, ldpath_us = false;
+        LOG(GLOBAL, LOG_ALL, 1, "\n---------------------------------------------------------------------------\n");
+        LOG(THREAD, LOG_ALL, 1, "\n---------------------------------------------------------------------------\n");
+        DODEBUG({
+            SYSLOG_INTERNAL_INFO("-- execve %s --", fname);
+            LOG(THREAD, LOG_SYSCALLS, 1, "syscall: execve %s\n", fname);
+            LOG(GLOBAL, LOG_TOP|LOG_SYSCALLS, 1, "execve %s\n", fname);
+            if (stats->loglevel >= 3) {
+                if (argv == NULL) {
+                    LOG(THREAD, LOG_SYSCALLS, 3, "\targs are NULL\n");
+                } else {
+                    for (i = 0; argv[i] != NULL; i++) {
+                        LOG(THREAD, LOG_SYSCALLS, 2, "\targ %d: len=%d\n",
+                            i, strlen(argv[i]));
+                        LOG(THREAD, LOG_SYSCALLS, 3, "\targ %d: %s\n",
+                            i, argv[i]);
+                    }
+                }
+            }
+        });
+        if (envp == NULL) {
+            LOG(THREAD, LOG_SYSCALLS, 3, "\tenv is NULL\n");
+            i = 0;
+        } else {
+            for (i = 0; envp[i] != NULL; i++) {
+                /* execve env vars should never be set here */
+                ASSERT(strstr(envp[i], DYNAMORIO_VAR_EXECVE) != envp[i]);
+                if (strstr(envp[i], DYNAMORIO_VAR_OPTIONS) == envp[i]) {
+                    ops = i;
+                }
+                if (strstr(envp[i], "LD_LIBRARY_PATH=") == envp[i]) {
+                    ldpath = i;
+                    if (strstr(envp[i], dynamorio_library_path) != NULL)
+                        ldpath_us = true;
+                }
+                if (strstr(envp[i], "LD_PRELOAD=") == envp[i]) {
+                    preload = i;
+                    if (strstr(envp[i], DYNAMORIO_PRELOAD_NAME) != NULL &&
+                        strstr(envp[i], DYNAMORIO_LIBRARY_NAME) != NULL) {
+                        preload_us = true;
+                    }
+                }
+                LOG(THREAD, LOG_SYSCALLS, 3, "\tenv %d: %s\n", i, envp[i]);
+            }
+        }
+        dcontext->sys_param0 = (reg_t) envp;
+
+#ifdef STATIC_LIBRARY
+        /* no way we can inject, we just lose control */
+        SYSLOG_INTERNAL_WARNING("WARNING: static DynamoRIO library, losing control on execve");
+        break;
+#endif
+
+        {
+            /* We want to add new env vars, so we create a new envp
+             * array.  We have to deallocate them and restore the old
+             * envp if execve fails; if execve succeeds, the address
+             * space is reset so we don't need to do anything.  
+             */
+            int num_old = i;
+            uint sz;
+            char *var, *old;
+            int idx_preload = preload, idx_ldpath = ldpath, idx_ops = ops;
+            char *options = option_string; /* global var */
+            int num_new;
+            char **new_envp;
+            num_new = 
+                (get_log_dir(PROCESS_DIR, NULL, NULL) ? 1 : 0) + /* logdir */
+                2 + /* execve indicator var plus final NULL */
+                ((preload<0) ? 1 : 0) +
+                ((ldpath<0) ? 1 : 0) +
+                ((ops < 0 && options != NULL) ? 1 : 0);
+            new_envp = heap_alloc(dcontext, sizeof(char*)*(num_old+num_new)
+                                  HEAPACCT(ACCT_OTHER));
+            memcpy(new_envp, envp, sizeof(char*)*num_old);
+
+            *sys_param_addr(dcontext, 2) = (reg_t) new_envp;  /* OUT */
+            /* store for reset in case execve fails */
+            dcontext->sys_param1 = (reg_t) new_envp;
+            dcontext->sys_param2 = (reg_t) num_old;
+            /* if no var to overwrite, need new var at end */
+            if (preload < 0)
+                idx_preload = i++;
+            if (ldpath < 0)
+                idx_ldpath = i++;
+            if (ops < 0 && options != NULL)
+                idx_ops = i++;
+            if (!preload_us) {
+                LOG(THREAD, LOG_SYSCALLS, 1,
+                    "WARNING: execve env does NOT preload DynamoRIO, forcing it!\n");
+                if (preload >= 0) {
+                    sz = strlen(envp[preload]) + strlen(DYNAMORIO_PRELOAD_NAME)+
+                        strlen(DYNAMORIO_LIBRARY_NAME) + 3;
+                    var = heap_alloc(dcontext, sizeof(char)*sz HEAPACCT(ACCT_OTHER));
+                    old = envp[preload] + strlen("LD_PRELOAD=");
+                    snprintf(var, sz, "LD_PRELOAD=%s %s %s",
+                            DYNAMORIO_PRELOAD_NAME, DYNAMORIO_LIBRARY_NAME, old);
+                } else {
+                    sz = strlen("LD_PRELOAD=") + strlen(DYNAMORIO_PRELOAD_NAME) + 
+                        strlen(DYNAMORIO_LIBRARY_NAME) + 2;
+                    var = heap_alloc(dcontext, sizeof(char)*sz HEAPACCT(ACCT_OTHER));
+                    snprintf(var, sz, "LD_PRELOAD=%s %s",
+                            DYNAMORIO_PRELOAD_NAME, DYNAMORIO_LIBRARY_NAME);
+                }
+                *(var+sz-1) = '\0'; /* null terminate */
+                new_envp[idx_preload] = var;
+                LOG(THREAD, LOG_SYSCALLS, 2, "\tnew env %d: %s\n",
+                    idx_preload, new_envp[idx_preload]);
+            }
+            if (!ldpath_us) {
+                if (ldpath >= 0) {
+                    sz = strlen(envp[ldpath]) + strlen(dynamorio_library_path) + 2;
+                    var = heap_alloc(dcontext, sizeof(char)*sz HEAPACCT(ACCT_OTHER));
+                    old = envp[ldpath] + strlen("LD_LIBRARY_PATH=");
+                    snprintf(var, sz, "LD_LIBRARY_PATH=%s:%s", dynamorio_library_path, old);
+                } else {
+                    sz = strlen("LD_LIBRARY_PATH=") + strlen(dynamorio_library_path) + 1;
+                    var = heap_alloc(dcontext, sizeof(char)*sz HEAPACCT(ACCT_OTHER));
+                    snprintf(var, sz, "LD_LIBRARY_PATH=%s", dynamorio_library_path);
+                }
+                *(var+sz-1) = '\0'; /* null terminate */
+                new_envp[idx_ldpath] = var;
+                LOG(THREAD, LOG_SYSCALLS, 2, "\tnew env %d: %s\n",
+                    idx_ldpath, new_envp[idx_ldpath]);
+            }
+            if (ops < 0 && options != NULL) {
+                sz = strlen(DYNAMORIO_VAR_OPTIONS) + strlen(options) + 2;
+                var = heap_alloc(dcontext, sizeof(char)*sz HEAPACCT(ACCT_OTHER));
+                snprintf(var, sz, "%s=%s", DYNAMORIO_VAR_OPTIONS, options);
+                *(var+sz-1) = '\0'; /* null terminate */
+                new_envp[idx_ops] = var;
+                LOG(THREAD, LOG_SYSCALLS, 2, "\tnew env %d: %s\n",
+                    idx_ops, new_envp[idx_ops]);
+            }
+            {
+                uint logdir_length;
+                if (get_log_dir(PROCESS_DIR, NULL, &logdir_length)) {
+                    sz = strlen(DYNAMORIO_VAR_EXECVE_LOGDIR) + 1 +
+                         logdir_length;
+                    var = heap_alloc(dcontext, 
+                                     sizeof(char)*sz HEAPACCT(ACCT_OTHER));
+                    snprintf(var, sz, "%s=", DYNAMORIO_VAR_EXECVE_LOGDIR);
+                    get_log_dir(PROCESS_DIR, var+strlen(var), &logdir_length);
+                    *(var+sz-1) = '\0'; /* null terminate */
+                    new_envp[i++] = var;
+                    LOG(THREAD, LOG_SYSCALLS, 2, "\tnew env %d: %s\n", i-1, new_envp[i-1]);
+                }
+            }
+            sz = strlen(DYNAMORIO_VAR_EXECVE) + 3;
+            /* we always pass this var to indicate "post-execve" */
+            var = heap_alloc(dcontext, sizeof(char)*sz HEAPACCT(ACCT_OTHER));
+            snprintf(var, sz, "%s=1", DYNAMORIO_VAR_EXECVE);
+            *(var+sz-1) = '\0'; /* null terminate */
+            new_envp[i++] = var;
+            LOG(THREAD, LOG_SYSCALLS, 2, "\tnew env %d: %s\n", i-1, new_envp[i-1]);
+            /* must end with NULL */
+            new_envp[i] = NULL;
+        }
+
+        break;
+    }
+
+    /****************************************************************************/
+    /* SIGNALS */
+ 
+    case SYS_rt_sigaction: {   /* 174 */
+        /* in /usr/src/linux/kernel/signal.c:
+           asmlinkage long
+           sys_rt_sigaction(int sig, const struct sigaction *act, 
+             struct sigaction *oact, size_t sigsetsize)
+         */
+        int sig  = (int) sys_param(dcontext, 0);
+        const kernel_sigaction_t *act = (const kernel_sigaction_t *) sys_param(dcontext, 1);
+        kernel_sigaction_t *oact = (kernel_sigaction_t *) sys_param(dcontext, 2);
+        size_t sigsetsize = (size_t) sys_param(dcontext, 3);
+        /* post_syscall does some work as well */
+        dcontext->sys_param0 = (reg_t) sig;
+        dcontext->sys_param1 = (reg_t) act;
+        dcontext->sys_param2 = (reg_t) oact;
+        dcontext->sys_param3 = (reg_t) sigsetsize;
+        execute_syscall = handle_sigaction(dcontext, sig, act, oact, sigsetsize);
+        if (!execute_syscall) {
+            SET_RETURN_VAL(dcontext, 0);
+        }
+        break;
+    }
+#ifndef X64
+    case SYS_sigreturn: {      /* 119 */
+        /* in /usr/src/linux/arch/i386/kernel/signal.c:
+           asmlinkage int sys_sigreturn(unsigned long __unused)
+         */
+        handle_sigreturn(dcontext, false);
+        break;
+    }
+#endif
+    case SYS_rt_sigreturn: {   /* 173 */
+        /* in /usr/src/linux/arch/i386/kernel/signal.c:
+           asmlinkage int sys_rt_sigreturn(unsigned long __unused)
+         */
+        handle_sigreturn(dcontext, true);
+        break;
+    }
+    case SYS_sigaltstack: {    /* 186 */
+        /* in /usr/src/linux/arch/i386/kernel/signal.c:
+           asmlinkage int
+           sys_sigaltstack(const stack_t *uss, stack_t *uoss)
+        */
+        const stack_t *uss = (const stack_t *) sys_param(dcontext, 0);
+        stack_t *uoss = (stack_t *) sys_param(dcontext, 1);
+        execute_syscall =
+            handle_sigaltstack(dcontext, uss, uoss);
+        if (!execute_syscall) {
+            SET_RETURN_VAL(dcontext, 0);
+        }
+        break;
+    }
+    case SYS_rt_sigprocmask: { /* 175 */
+        /* in /usr/src/linux/kernel/signal.c:
+           asmlinkage long
+           sys_rt_sigprocmask(int how, sigset_t *set, sigset_t *oset, 
+             size_t sigsetsize)
+         */
+        handle_sigprocmask(dcontext, (int) sys_param(dcontext, 0),
+                           (kernel_sigset_t *) sys_param(dcontext, 1),
+                           (kernel_sigset_t *) sys_param(dcontext, 2),
+                           (size_t) sys_param(dcontext, 3));
+        break;
+    }
+    case SYS_rt_sigsuspend: { /* 179 */
+        /* in /usr/src/linux/kernel/signal.c:
+           asmlinkage int
+           sys_rt_sigsuspend(sigset_t *unewset, size_t sigsetsize)
+         */
+        handle_sigsuspend(dcontext, (kernel_sigset_t *) sys_param(dcontext, 0),
+                          (size_t) sys_param(dcontext, 1));
+        break;
+    }
+    case SYS_kill: {           /* 37 */
+        /* in /usr/src/linux/kernel/signal.c:
+         * asmlinkage long sys_kill(int pid, int sig)
+         */
+        pid_t pid = (pid_t) sys_param(dcontext, 0);
+        uint sig = (uint) sys_param(dcontext, 1);
+        /* We check whether targeting this process or this process group */
+        if (pid == get_process_id() || pid == 0 || pid == -get_process_group_id()) {
+            handle_self_signal(dcontext, sig);
+        }
+        break;
+    }
+#if defined(SYS_tkill)
+    case SYS_tkill: {          /* 238 */
+        /* in /usr/src/linux/kernel/signal.c:
+         * asmlinkage long sys_tkill(int pid, int sig)
+         */
+        pid_t tid = (pid_t) sys_param(dcontext, 0);
+        uint sig = (uint) sys_param(dcontext, 1);
+        if (tid == get_thread_id()) {
+            handle_self_signal(dcontext, sig);
+        }
+        break;
+    }
+#endif
+#if defined(SYS_tgkill)
+    case SYS_tgkill: {          /* 270 */
+        /* in /usr/src/linux/kernel/signal.c:
+         * asmlinkage long sys_tgkill(int tgid, int pid, int sig)
+         */
+        pid_t tgid = (pid_t) sys_param(dcontext, 0);
+        pid_t tid = (pid_t) sys_param(dcontext, 1);
+        uint sig = (uint) sys_param(dcontext, 2);
+        /* some kernels support -1 values:
+         +   tgkill(-1, tid, sig)  == tkill(tid, sig)
+         *   tgkill(tgid, -1, sig) == kill(tgid, sig)
+         * the 2nd was proposed but is not in 2.6.20 so I'm ignoring it, since
+         * I don't want to kill the thread when the signal is never sent!
+         * FIXME: the 1st is in my tkill manpage, but not my 2.6.20 kernel sources!
+         */
+        if ((tgid == -1 || tgid == get_process_id()) &&
+            tid == get_thread_id()) {
+            handle_self_signal(dcontext, sig);
+        }
+        break;
+    }
+#endif
+#if 0
+# ifndef X64
+    case SYS_signal: {         /* 48 */
+        /* in /usr/src/linux/kernel/signal.c:
+           asmlinkage unsigned long
+           sys_signal(int sig, __sighandler_t handler)
+         */
+        break;
+    }
+    case SYS_sigaction: {      /* 67 */
+        /* in /usr/src/linux/arch/i386/kernel/signal.c:
+           asmlinkage int 
+           sys_sigaction(int sig, const struct old_sigaction *act,
+                         struct old_sigaction *oact)
+         */
+        break;
+    }
+    case SYS_sigsuspend: {     /* 72 */
+        /* in /usr/src/linux/arch/i386/kernel/signal.c:
+           asmlinkage int
+           sys_sigsuspend(int history0, int history1, old_sigset_t mask)
+         */
+        break;
+    }
+    case SYS_sigprocmask: {    /* 126 */
+        /* in /usr/src/linux/kernel/signal.c:
+           asmlinkage long
+           sys_sigprocmask(int how, old_sigset_t *set, old_sigset_t *oset)
+         */
+        break;
+    }
+# endif
+#else
+    /* until we've implemented them, keep down here to get warning: */
+# ifndef X64
+    case SYS_signal:
+    case SYS_sigaction:
+    case SYS_sigsuspend:
+    case SYS_sigprocmask:
+# endif
+#endif
+
+#ifndef X64
+    case SYS_sigpending:      /* 73 */
+    case SYS_sgetmask:        /* 68 */
+    case SYS_ssetmask:        /* 69 */
+#endif
+    case SYS_rt_sigpending:   /* 176 */
+    case SYS_rt_sigtimedwait: /* 177 */
+    case SYS_rt_sigqueueinfo: /* 178 */
+    case SYS_setitimer:       /* 104 */
+    case SYS_getitimer: {      /* 105 */
+        /* FIXME: handle all of these syscalls! */
+        LOG(THREAD, LOG_ASYNCH|LOG_SYSCALLS, 1,
+            "WARNING: unhandled signal system call %d\n", mc->xax);
+        break;
+    }
+
+    case SYS_close: {
+        /* in fs/open.c: asmlinkage long sys_close(unsigned int fd) */
+        uint fd = (uint) sys_param(dcontext, 0);
+        LOG(THREAD, LOG_SYSCALLS, 3, "syscall: close fd %d\n", fd);
+#ifdef DEBUG
+        if (stats->loglevel > 0) {
+            /* I assume other threads' files, and for that matter forked children's
+             * files, will not be completely closed just by us closing them
+             */
+            if (fd == main_logfile || fd == dcontext->logfile) {
+                LOG(THREAD, LOG_TOP|LOG_SYSCALLS, 1,
+                    "\tWARNING: app trying to close DynamoRIO file!  Not allowing it.\n");
+                return false;
+            }
+        }
+#endif
+        /* Xref PR 258731 - duplicate STDOUT/STDERR when app closes them so we (or
+         * a client) can continue to use them for logging. */
+        if (DYNAMO_OPTION(dup_stdout_on_close) && fd == STDOUT) {
+            our_stdout = dup_syscall(fd);
+            LOG(THREAD, LOG_TOP|LOG_SYSCALLS, 1,
+                "WARNING: app is closing stdout=%d - duplicating descriptor for "
+                "DynamoRIO usage got %d.\n", fd, our_stdout);
+        }
+        if (DYNAMO_OPTION(dup_stderr_on_close) && fd == STDERR) {
+            our_stderr = dup_syscall(fd);
+            LOG(THREAD, LOG_TOP|LOG_SYSCALLS, 1,
+                "WARNING: app is closing stderr=%d - duplicating descriptor for "
+                "DynamoRIO usage got %d.\n", fd, our_stderr);
+        }
+        break;
+    }
+
+    } /* end switch */
+
+    dcontext->whereami = old_whereami;
+    return execute_syscall;
+}
+
+void
+update_all_memory_areas(app_pc start, app_pc end, uint prot)
+{
+    /* all_memory_areas lock is held higher up the call chain to avoid rank
+     * order violation with heap_unit_lock */
+    ASSERT_OWN_WRITE_LOCK(true, &all_memory_areas->lock);
+
+    if (vmvector_overlap(all_memory_areas, start, end)) {
+        bool removed;
+        LOG(THREAD_GET, LOG_VMAREAS|LOG_SYSCALLS, 4,
+            "update_all_memory_areas: overlap found, removing and adding: "
+            PFX"-"PFX" prot=%d\n", start, end, prot);
+        /* New region to be added overlaps with one or more existing
+         * regions.  Split already existing region(s) accordingly and add
+         * the new region */
+        removed = vmvector_remove(all_memory_areas, start, end);
+        ASSERT(removed);
+    }
+    LOG(THREAD_GET, LOG_VMAREAS|LOG_SYSCALLS, 3,
+        "update_all_memory_areas: adding: "PFX"-"PFX" prot=%d\n", start, end, prot);
+    vmvector_add(all_memory_areas, start, end, (void *)(ptr_uint_t)prot);
+}
+
+bool
+remove_from_all_memory_areas(app_pc start, app_pc end)
+{
+    bool ok;
+    DEBUG_DECLARE(dcontext_t *dcontext = get_thread_private_dcontext());
+
+    ok = vmvector_remove(all_memory_areas, start, end);
+    ASSERT(ok);
+    LOG(THREAD, LOG_VMAREAS|LOG_SYSCALLS, 3,
+        "remove_from_all_memory_areas: removed: "PFX"-"PFX"\n", start, end);
+    return ok;
+}
+
+/* We consider a module load to happen at the first mmap, so we check on later
+ * overmaps to ensure things look consistent. */
+static bool
+mmap_check_for_module_overlap(app_pc base, size_t size, bool readable, uint64 inode,
+                              bool at_map)
+{
+    module_area_t *ma;
+
+    os_get_module_info_lock();
+    ma = module_pc_lookup(base);
+    if (ma != NULL) {
+        /* FIXME - how can we distinguish between the loader mapping the segments
+         * over the initial map from someone just mapping over part of a module? If
+         * is the latter case need to adjust the view size or remove from module list. */
+        LOG(GLOBAL, LOG_VMAREAS, 2, "%s mmap overlapping module area : \n"
+            "\tmap : base="PFX" base+size="PFX" inode="UINT64_FORMAT_STRING"\n"
+            "\tmod : start="PFX" end="PFX" inode="UINT64_FORMAT_STRING"\n",
+            at_map ? "new" : "existing", base, base+size, inode,
+            ma->start, ma->end, ma->names.inode);
+        ASSERT_CURIOSITY(base >= ma->start);
+        if (at_map) {
+            ASSERT_CURIOSITY(base+size <= ma->end);
+        } else {
+            /* FIXME - I'm having problems with this check for existing maps.  I
+             * haven't been able to get gdb to break in early enough to really get a good
+             * look at the early loader behavior.  Two issues:  One case is with our .so
+             * for which the anonymous .bss mapping is one page larger than expected
+             * (which might be some loader bug in the size calculation? or something? if
+             * so should see it trigger the at_map curiosity on some dll and can address
+             * then) and the other is that for a few executables the .bss mapping is much
+             * larger (~0x20000 larger) then expected when running under DR (but not
+             * running natively where it is instead the expected size).  Both could just
+             * be the loader merging adjacent identically protected regions though I
+             * can't explain the discrepancy between DR and native given that our vmmheap
+             * is elsewhere in the address space (so who and how allocated that adjacent
+             * memory). I've yet to see any issue with dynamically loaded modules so
+             * it's probably the loader merging regions.  Still worth investigating. */
+            ASSERT_CURIOSITY(inode == 0 /*see above comment*/|| base+size <= ma->end);
+        }
+        ASSERT_CURIOSITY(ma->names.inode == inode || inode == 0 /* for .bss */);
+        DODEBUG({
+            if (readable && is_elf_so_header(base, size)) {
+                /* Case 8879: For really small modules, to save disk space, the same
+                 * disk page could hold both RO and .data, occupying just 1 page of
+                 * disk space, e.g. /usr/lib/httpd/modules/mod_auth_anon.so.  When
+                 * such a module is mapped in, the os maps the same disk page twice,
+                 * one readonly and one copy-on-write (see pg.  96, Sec 4.4 from
+                 * Linkers and Loaders by John R.  Levine).  This makes the data
+                 * section also satisfy the elf_header check above.  So, if the new
+                 * mmap overlaps an elf_area and it is also a header, then make sure
+                 * the previous page (correcting for alignment) is also a elf_header.
+                 * Note, if it is a header of a different module, then we'll not have
+                 * an overlap, so we will not hit this case.
+                 */
+                ASSERT_CURIOSITY(ma->start + ma->os_data.alignment == base);
+            }
+        });
+    }
+    os_get_module_info_unlock();
+    return ma != NULL;
+}
+
+/* All processing for mmap and mmap2. */
+static void
+process_mmap(dcontext_t *dcontext, app_pc base, size_t size, uint prot
+            _IF_DEBUG(char *map_type))
+{
+    bool image = false;
+    uint memprot = osprot_to_memprot(prot);
+    app_pc area_start;
+    app_pc area_end;
+    /* PR 314391: if we declare as uint, vmvector_lookup_data clobbers
+     * the 32 bits next to it on the stack!
+     */
+    ptr_uint_t area_prot;
+    bool overlaps_image;
+
+    LOG(THREAD, LOG_SYSCALLS, 4, "process_mmap("PFX","PFX",%s,%s)\n",
+        base, size, memprot_string(memprot), map_type);
+    /* Notes on how ELF SOs are mapped in.
+     *
+     * o The initial mmap for an ELF file specifies enough space for
+     *   all segments (and their constituent sections) in the file.
+     *   The protection bits for that section are used for the entire
+     *   region, and subsequent mmaps for subsequent segments within
+     *   the region modify their portion's protection bits as needed.
+     *   So if the prot bits for the first segment are +x, the entire
+     *   region is +x. ** Note that our primary concern is adjusting
+     *   exec areas to reflect the prot bits of subsequent
+     *   segments. ** The region is added to the all-memory areas
+     *   and also to exec areas (as determined by app_memory_allocation()).
+     *
+     * o Any subsequent segment sub-mappings specify their own protection
+     *   bits and therefore are added to the exec areas via normal
+     *   processing. They are also "naturally" added to the all-mems list.
+     *   We do a little extra processing when mapping into a previously
+     *   mapped region and the prot bits mismatch; if the new mapping is
+     *   not +x, flushing needs to occur.
+     */
+    /* process_mmap can be called with PROT_NONE, so we need to check if we
+     * can read the memory to see if it is a elf_header
+     */
+    /* FIXME - get inode for check */
+    overlaps_image = mmap_check_for_module_overlap(base, size,
+                                                   TEST(MEMPROT_READ, memprot), 0, true);
+    if (overlaps_image) {
+        /* FIXME - how can we distinguish between the loader mapping the segments
+         * over the initial map from someone just mapping over part of a module? If
+         * is the latter case need to adjust the view size or remove from module list. */
+        image = true;
+        DODEBUG({ map_type = "ELF SO"; });
+    } else if (TEST(MEMPROT_READ, memprot) && is_elf_so_header(base, size)) {
+        maps_iter_t iter;
+        bool found_map = false;;
+        uint64 inode = 0;
+        char *filename = "";
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2, "dlopen "PFX"-"PFX"%s\n",
+            base, base+size, TEST(MEMPROT_EXEC, memprot) ? " +x": "");
+        image = true;
+        DODEBUG({ map_type = "ELF SO"; });
+        /* Mapping in a new module.  From what we've observed of the loader's
+         * behavior, it first maps the file in with size equal to the final
+         * memory image size (I'm not sure how it gets that size without reading
+         * in the elf header and then walking through all the program headers to
+         * get the largest virtual offset).  This is neccesary to reserve all the
+         * space that will be needed.  It then walks through the program headers
+         * mapping over the the previously mapped space with the appropriate
+         * permissions and offsets.  Note that the .bss portion is mapped over
+         * as anonymous.  It may also, depending on the program headers, make some
+         * areas read-only after fixing up their relocations etc. NOTE - at
+         * no point are the section headers guaranteed to be mapped in so we can't
+         * reliably walk sections (only segments) without looking to disk.
+         */
+        /* FIXME - when should we add the module to our list?  At the first map
+         * seems to be the best choice as we know the bounds and it's difficult to
+         * tell when the loader is finished.  The downside is that at the intial map
+         * the memory layout isn't finalized (memory beyond the first segment will
+         * be shifted for page alignment reasons), so we have to be careful and
+         * make adjustments to read anything beyond the first segment until the 
+         * loader finishes. This goes for the client too as it gets notified when we
+         * add to the list.  FIXME we could try to track the expected segment overmaps
+         * and only notify the client after the last one (though that's still before
+         * linking and relocation, but that's true on Windows too). */
+        /* Get filename & inode for the list. */
+        maps_iterator_start(&iter, true /* plan to alloc a module_area_t */);
+        while (maps_iterator_next(&iter)) {
+            if (iter.vm_start == base) {
+                ASSERT_CURIOSITY(iter.inode != 0);
+                ASSERT_CURIOSITY(iter.offset == 0); /* first map shouldn't have offset */
+                /* XREF 307599 on rounding module end to the next PAGE boundary */
+                ASSERT_CURIOSITY(iter.vm_end - iter.vm_start ==
+                                 ALIGN_FORWARD(size, PAGE_SIZE));
+                inode = iter.inode;
+                filename = dr_strdup(iter.comment HEAPACCT(ACCT_OTHER));
+                found_map = true;
+                break;
+            }
+        }
+        maps_iterator_stop(&iter);
+        ASSERT_CURIOSITY(found_map); /* barring weird races we should find this map */
+        /* XREF 307599 on rounding module end to the next PAGE boundary */
+        module_list_add(base, ALIGN_FORWARD(size, PAGE_SIZE), true, inode, filename);
+        if (found_map)
+            dr_strfree(filename HEAPACCT(ACCT_OTHER));
+    }
+
+    if (vmvector_lookup_data(all_memory_areas, base, &area_start, &area_end,
+                             (void **) &area_prot)) {
+        uint new_memprot;
+        LOG(THREAD, LOG_SYSCALLS, 4, "\tprocess overlap w/"PFX"-"PFX" prot=%d\n",
+            area_start, area_end, (uint)area_prot);
+        if (area_prot != memprot) {
+            DEBUG_DECLARE(uint res =)
+                app_memory_protection_change(dcontext, base, size, memprot, 
+                                             &new_memprot, NULL);
+            ASSERT_NOT_IMPLEMENTED(res != PRETEND_APP_MEM_PROT_CHANGE &&
+                                   res != SUBSET_APP_MEM_PROT_CHANGE);
+                   
+        }
+    }
+    all_memory_areas_lock();
+    update_all_memory_areas(base, base + size, memprot);
+    all_memory_areas_unlock();
+
+    /* app_memory_allocation() expects to not see an overlap -- exec areas
+     * doesn't expect one. We have yet to see a +x mmap into a previously
+     * mapped +x region so this may not be an issue.
+     */
+    LOG(THREAD, LOG_SYSCALLS, 4, "\t try app_mem_alloc\n");
+    if (app_memory_allocation(dcontext, base, size, memprot, image _IF_DEBUG(map_type)))
+        STATS_INC(num_app_code_modules);
+    LOG(THREAD, LOG_SYSCALLS, 4, "\t app_mem_alloc -- DONE\n");
+}
+
+/* Returns false if system call should NOT be executed
+ * Returns true if system call should go ahead
+ */
+/* FIXME: split out specific handlers into separate routines
+ */
+void
+post_system_call(dcontext_t *dcontext)
+{
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    /* registers have been clobbered, so sysnum is kept in dcontext */
+    int sysnum = dcontext->sys_num;
+    /* We expect most syscall failures to return < 0, so >= 0 is success.
+     * Some syscall return addresses that have the sign bit set and so
+     * appear to be failures but are not. They are handled on a
+     * case-by-case basis in the switch statement below.
+     */
+    ptr_int_t result = (ptr_int_t) mc->xax; /* signed */
+    bool success = (result >= 0);
+    app_pc base;
+    size_t size;
+    uint prot;
+    where_am_i_t old_whereami;
+    DEBUG_DECLARE(bool ok;)
+#ifdef CLIENT_INTERFACE 
+    DEBUG_DECLARE(bool client_failed = false;)
+#endif
+
+    RSTATS_INC(post_syscall);
+
+    old_whereami = dcontext->whereami;
+    dcontext->whereami = WHERE_SYSCALL_HANDLER;
+
+    /* PR 313715: restore xbp since for some vsyscall sequences that use
+     * the syscall instruction its value is needed:
+     *   0xffffe400 <__kernel_vsyscall+0>:       push   %ebp
+     *   0xffffe401 <__kernel_vsyscall+1>:       mov    %ecx,%ebp
+     *   0xffffe403 <__kernel_vsyscall+3>:       syscall
+     *   0xffffe405 <__kernel_vsyscall+5>:       mov    $0x2b,%ecx
+     *   0xffffe40a <__kernel_vsyscall+10>:      movl   %ecx,%ss
+     *   0xffffe40c <__kernel_vsyscall+12>:      mov    %ebp,%ecx
+     *   0xffffe40e <__kernel_vsyscall+14>:      pop    %ebp
+     *   0xffffe40f <__kernel_vsyscall+15>:      ret
+     */
+    if (should_syscall_method_be_sysenter() && !dcontext->sys_was_int) {
+        mc->xbp = dcontext->sys_xbp;
+    }
+
+#ifdef CLIENT_INTERFACE 
+    /* after restore of xbp so client sees it as though was sysenter */
+    instrument_post_syscall(dcontext, sysnum);
+    /* re-set these vars */
+    result = (ptr_int_t) mc->xax; /* signed */
+    DODEBUG({
+        if (success && result < 0)
+            client_failed = true;
+    });
+    success = (result >= 0);
+#endif
+
+    /* handle fork, try to do it early before too much logging occurs */
+    if (sysnum == SYS_fork ||
+        (sysnum == SYS_clone && !TEST(CLONE_VM, dcontext->sys_param0))) {
+        if (result == 0) {
+            /* we're the child */
+            thread_id_t child = get_thread_id();
+#ifdef DEBUG
+            thread_id_t parent = get_parent_id();
+            SYSLOG_INTERNAL_INFO("-- parent %d forked child %d --", parent, child);
+#endif
+            /* first, fix TLS of dcontext */
+            ASSERT(parent != 0);
+            /* change parent pid to our pid */
+            replace_thread_id(dcontext->owning_thread, child);
+            dcontext->owning_thread = child;
+
+            /* now let dynamo initialize new shared memory, logfiles, etc. 
+             * need access to static vars in dynamo.c, that's why we don't do it. */
+            /* FIXME - xref PR 246902 - dispatch runs a lot of code before
+             * getting to post_system_call() is any of that going to be messed up
+             * by waiting till here to fixup the child logfolder/file and tid?
+             */
+            dynamorio_fork_init(dcontext);
+
+            LOG(THREAD, LOG_SYSCALLS, 1,
+                "after fork-like syscall: parent is %d, child is %d\n", parent, child);
+        } else {
+            /* we're the parent */
+        }
+    }
+
+
+    LOG(THREAD, LOG_SYSCALLS, 2,
+        "post syscall: sysnum="PFX", result="PFX" (%d)\n",
+        sysnum, mc->xax, (int)mc->xax);
+
+    switch (sysnum) {
+
+    /****************************************************************************/
+    /* MEMORY REGIONS */
+
+#ifndef X64
+    case SYS_mmap2:
+#endif
+    case SYS_mmap: {
+        DEBUG_DECLARE(char *map_type;)
+        RSTATS_INC(num_app_mmaps);
+        base = (app_pc) mc->xax; /* For mmap, it's NOT arg->addr! */
+        /* mmap isn't simply a user-space wrapper for mmap2. It's called
+         * directly when dynamically loading an SO, i.e., dlopen(). */
+        success = mmap_syscall_succeeded((app_pc)result);
+        /* The syscall either failed OR the retcode is less than the
+         * largest uint value of any errno and the addr returned is
+         * page-aligned.
+         */
+        ASSERT_CURIOSITY(!success ||
+                         ((app_pc)result < (app_pc)(ptr_int_t)-0x1000 &&
+                          ALIGNED(base, PAGE_SIZE)));
+        if (!success)
+            goto exit_post_system_call;
+#ifndef X64
+        if (sysnum == SYS_mmap) {
+            /* The syscall succeeded so the read of 'arg' should be
+             * safe. */
+            mmap_arg_struct_t *arg = (mmap_arg_struct_t *) dcontext->sys_param0;
+            size = (size_t) arg->len;
+            prot = (uint) arg->prot;
+            DEBUG_DECLARE(map_type = "mmap";)
+        }
+        else {
+#endif
+            size = (size_t) dcontext->sys_param1;
+            prot = (uint) dcontext->sys_param2;
+            DEBUG_DECLARE(map_type = IF_X64_ELSE("mmap2","mmap");)
+#ifndef X64
+        }
+#endif
+        process_mmap(dcontext, base, size, prot _IF_DEBUG(map_type));
+        break;
+    }
+    case SYS_munmap: {
+        app_pc addr = (app_pc) dcontext->sys_param0;
+        size_t len = (size_t) dcontext->sys_param1;
+        /* We assumed in pre_system_call() that the unmap would succeed
+         * and flushed fragments and removed the region from exec areas.
+         * If the unmap failed, we re-add the region to exec areas.
+         *
+         * The same logic can be used on Windows (but isn't yet).
+         */
+        /* FIXME There are shortcomings to the approach. If another thread
+         * executes in the region after our pre_system_call processing
+         * but before the re-add below, it will get a security violation.
+         * That's less than ideal but at least isn't a security hole.
+         * The overall shortcoming is that we lose the state from our
+         * stateful security policies -- future exec list, tables used
+         * for RCT (.C/.E/.F) -- which can't be easily restored. Also,
+         * the re-add could add a region that wasn't on the exec list
+         * previously.
+         *
+         * See case 7559 for a better approach.
+         */
+        if (!success) {
+            uint memprot;
+            DEBUG_DECLARE(ok =)
+                get_memory_info_from_os(addr, NULL, NULL, &memprot);
+            ASSERT(ok);
+            app_memory_allocation(dcontext, addr, len, memprot,
+                                  false/*not image -- FIXME: look in maps to find out*/
+                                  _IF_DEBUG("failed munmap"));
+            ASSERT(!vmvector_overlap(all_memory_areas, addr, addr + len));
+            /* Re-add to the all-mems list. */
+            vmvector_add(all_memory_areas, addr, addr + len, (void *)(ptr_uint_t)memprot);
+        }
+        break;
+    }
+    case SYS_mremap: {
+        app_pc old_base = (app_pc) dcontext->sys_param0;
+        size_t old_size = (size_t) dcontext->sys_param1;
+        uint prot = 0;
+        base = (app_pc) mc->xax;
+        size = (size_t) dcontext->sys_param2;
+        /* even if no shift, count as munmap plus mmap */
+        RSTATS_INC(num_app_munmaps);
+        RSTATS_INC(num_app_mmaps);
+        success = !(result == -EINVAL ||
+                    result == -EAGAIN ||
+                    result == -ENOMEM ||
+                    result == -EFAULT);
+        /* The syscall either failed OR the retcode is less than the
+         * largest uint value of any errno and the addr returned is
+         * is page-aligned.
+         */
+        ASSERT_CURIOSITY(!success ||
+                         ((app_pc)result < (app_pc)(ptr_int_t)-0x1000 &&
+                          ALIGNED(base, PAGE_SIZE)));
+        if (!success)
+             goto exit_post_system_call;
+
+        if (base != old_base || size < old_size) { /* take action only if
+                                                    * there was a change */
+            /* fragments were shifted...don't try to fix them, just flush */
+            app_memory_deallocation(dcontext, (app_pc)old_base, old_size,
+                                    false /* don't own thread_initexit_lock */,
+                                    false /* not image, FIXME: somewhat arbitrary */);
+            DEBUG_DECLARE(ok =) get_memory_info(old_base, NULL, NULL, &prot);
+            ASSERT(ok);
+            DODEBUG({
+                    /* we don't expect to see remappings of modules */
+                    os_get_module_info_lock();
+                    ASSERT_CURIOSITY(!module_overlaps(base, size));
+                    os_get_module_info_unlock();
+                });
+            /* Verify that the current prot on the new region (according to
+             * the os) is the same as what the prot used to be for the old
+             * region.
+             */
+            DODEBUG({
+                uint memprot;
+                ok = get_memory_info_from_os(base, NULL, NULL, &memprot);
+                ASSERT(memprot == prot);
+            });
+            app_memory_allocation(dcontext, base, size, prot,
+                                  false/*not image -- FIXME: look in maps to find out*/
+                                  _IF_DEBUG("mremap"));
+            /* Now modify the all-mems list. */
+            /* We don't expect an existing entry for the new region. */
+            all_memory_areas_lock();
+            ASSERT(!vmvector_overlap(all_memory_areas, base, base + size));
+            DEBUG_DECLARE(ok =)
+                vmvector_remove_containing_area(all_memory_areas, old_base,
+                                                NULL, NULL);
+            ASSERT(ok);
+            vmvector_add(all_memory_areas, base, base + size, (void *)(ptr_uint_t)prot);
+            all_memory_areas_unlock();
+        }
+        break;
+    }
+    case SYS_mprotect: {
+        base = (app_pc) dcontext->sys_param0;
+        size = dcontext->sys_param1;
+        prot = dcontext->sys_param2;
+        if (!success) {
+            uint memprot;
+            /* Revert the prot bits if needed. */
+            DEBUG_DECLARE(ok =)
+                get_memory_info_from_os(base, NULL, NULL, &memprot);
+            ASSERT(ok);
+            LOG(THREAD, LOG_SYSCALLS, 3,
+                "syscall: mprotect failed: "PFX"-"PFX" prot->%d\n",
+                base, base+size, osprot_to_memprot(prot));
+            if (prot != memprot_to_osprot(memprot)) {
+                /* We're trying to reverse the prot change, assuming that
+                 * this action doesn't have any unexpected side effects
+                 * when doing so (such as not reversing some bit of internal
+                 * state).
+                 */
+                uint new_memprot;
+                DEBUG_DECLARE(uint res =)
+                    app_memory_protection_change(dcontext, base, size,
+                                                 osprot_to_memprot(prot),
+                                                 &new_memprot,
+                                                 NULL);
+                ASSERT_NOT_IMPLEMENTED(res != SUBSET_APP_MEM_PROT_CHANGE);
+                ASSERT(res == DO_APP_MEM_PROT_CHANGE ||
+                       res == PRETEND_APP_MEM_PROT_CHANGE);
+                /* NYI Store state to revert the all-mems list. */
+                ASSERT_NOT_IMPLEMENTED(false);
+            }
+        }
+        break;
+    }
+
+    /****************************************************************************/
+    /* SPAWNING -- fork mostly handled above */
+
+    case SYS_clone: {
+        /* in /usr/src/linux/arch/i386/kernel/process.c */
+        LOG(THREAD, LOG_SYSCALLS, 2, "syscall: clone returned "PFX"\n", mc->xax);
+        break;
+    }
+
+    case SYS_fork: {
+        LOG(THREAD, LOG_SYSCALLS, 2, "syscall: fork returned "PFX"\n", mc->xax);
+        break;
+    }
+
+    case SYS_vfork: {
+        LOG(THREAD, LOG_SYSCALLS, 2, "syscall: vfork returned "PFX"\n", mc->xax);
+        break;
+    }
+
+    case SYS_execve: {
+        /* if we get here it means execve failed (doesn't return on success)
+         * we have to restore env to how it was
+         */
+        char **old_envp = (char **) dcontext->sys_param0;
+        char **new_envp = (char **) dcontext->sys_param1;
+        int num_old = (int) dcontext->sys_param2;
+        success = false;
+        ASSERT(result < 0);
+        LOG(THREAD, LOG_SYSCALLS, 2, "syscall: execve failed\n");
+#ifdef STATIC_LIBRARY
+        /* nothing to clean up */
+        break;
+#endif
+        if (new_envp != NULL) {
+            int i;
+            LOG(THREAD, LOG_SYSCALLS, 2, "\tcleaning up our env vars\n");
+            /* we replaced existing ones and/or added new ones */
+            for (i=0; i<num_old; i++) {
+                if (old_envp[i] != new_envp[i]) {
+                    heap_free(dcontext, new_envp[i],
+                              sizeof(char)*(strlen(new_envp[i])+1)
+                              HEAPACCT(ACCT_OTHER));
+                }
+            }
+            for (; new_envp[i] != NULL; i++) {
+                heap_free(dcontext, new_envp[i],
+                          sizeof(char)*(strlen(new_envp[i])+1)
+                          HEAPACCT(ACCT_OTHER));
+            }
+            i++; /* need to de-allocate final null slot too */
+            heap_free(dcontext, new_envp, sizeof(char*)*i HEAPACCT(ACCT_OTHER));
+            /* restore prev envp */
+            *sys_param_addr(dcontext, 2) = (reg_t) old_envp;
+        }
+        /* Don't 'break' as we have an ASSERT(success) just below
+         * the switch(). */
+        goto exit_post_system_call;
+        break; /* unnecessary but good form so keep it */
+    }
+
+    /****************************************************************************/
+    /* SIGNALS */
+
+    case SYS_rt_sigaction: {   /* 174 */
+        /* in /usr/src/linux/kernel/signal.c:
+           asmlinkage long
+           sys_rt_sigaction(int sig, const struct sigaction *act, 
+             struct sigaction *oact, size_t sigsetsize)
+         */
+        /* FIXME Handle syscall failure. */
+        int sig  = (int) dcontext->sys_param0;
+        const kernel_sigaction_t *act =
+            (const kernel_sigaction_t *) dcontext->sys_param1;
+        kernel_sigaction_t *oact = (kernel_sigaction_t *) dcontext->sys_param2;
+        size_t sigsetsize = (size_t) dcontext->sys_param3;
+        if (!success)
+            goto exit_post_system_call;
+
+        handle_post_sigaction(dcontext, sig, act, oact, sigsetsize);
+        break;
+    }
+    case SYS_rt_sigprocmask: { /* 175 */
+        /* in /usr/src/linux/kernel/signal.c:
+           asmlinkage long
+           sys_rt_sigprocmask(int how, sigset_t *set, sigset_t *oset, 
+             size_t sigsetsize)
+         */
+        /* FIXME Handle syscall failure. */
+        handle_post_sigprocmask(dcontext, (int) sys_param(dcontext, 0),
+                                (kernel_sigset_t *) sys_param(dcontext, 1),
+                                (kernel_sigset_t *) sys_param(dcontext, 2),
+                                (size_t) sys_param(dcontext, 3));
+        break;
+    }
+
+#ifndef X64
+    case SYS_sigreturn:        /* 119 */
+#endif
+    case SYS_rt_sigreturn:     /* 173 */
+        /* there is no return value: it's just the value of eax, so avoid
+         * assert below
+         */
+        success = true;
+        break;
+
+    } /* switch */
+    DODEBUG({
+        if (ignorable_system_call(sysnum))
+            STATS_INC(post_syscall_ignorable);
+        else
+            ASSERT(success || dcontext->expect_last_syscall_to_fail
+                   IF_CLIENT_INTERFACE(|| client_failed));
+    });
+
+
+ exit_post_system_call:
+    dcontext->whereami = old_whereami;
+}
+
+/***************************************************************************/
+/* Memory areas provided by kernel in /proc */
+/* On all supported linux kernels /proc/self/maps -> /proc/$pid/maps */
+
+#define PROC_SELF_MAPS "/proc/self/maps"
+
+/* these are defined in /usr/src/linux/fs/proc/array.c */
+#define MAPS_LINE_LENGTH        4096
+/* for systems with sizeof(void*) == 4: */
+#define MAPS_LINE_FORMAT4         "%08lx-%08lx %s %08lx %*s "UINT64_FORMAT_STRING" %4096s"
+#define MAPS_LINE_MAX4  49 /* sum of 8  1  8  1 4 1 8 1 5 1 10 1 */
+/* for systems with sizeof(void*) == 8: */
+#define MAPS_LINE_FORMAT8         "%016lx-%016lx %s %016lx %*s "UINT64_FORMAT_STRING" %4096s"
+#define MAPS_LINE_MAX8  73 /* sum of 16  1  16  1 4 1 16 1 5 1 10 1 */
+
+#define MAPS_LINE_MAX   MAPS_LINE_MAX8
+
+/* can't use fopen -- strategy: read into buffer, look for newlines.
+ * fail if single line too large for buffer -- so size it appropriately:
+ */
+/* since we're called from signal handler, etc., keep our stack usage
+ * low by using static bufs (it's over 4K after all).
+ * FIXME: now we're using 16K right here: should we shrink?
+ */
+#define BUFSIZE (MAPS_LINE_LENGTH+8)
+static char buf_scratch[BUFSIZE];
+static char comment_buf_scratch[BUFSIZE];
+/* To satisfy our two uses (inner use with memory_info_buf_lock versus
+ * outer use with maps_iter_buf_lock), we have two different locks and
+ * two different sets of static buffers.  This is to avoid lock
+ * ordering issues: we need an inner lock for use in places like signal
+ * handlers, but an outer lock when the iterator user allocates memory.
+ */
+static char buf_iter[BUFSIZE];
+static char comment_buf_iter[BUFSIZE];
+
+static bool
+maps_iterator_start(maps_iter_t *iter, bool may_alloc)
+{
+    /* Don't assign the local ptrs until the lock is grabbed to make
+     * their relationship clear. */
+    if (may_alloc) {
+        mutex_lock(&maps_iter_buf_lock);
+        iter->buf = (char *) &buf_iter;
+        iter->comment_buffer = (char *) &comment_buf_iter;
+    } else {
+        mutex_lock(&memory_info_buf_lock);
+        iter->buf = (char *) &buf_scratch;
+        iter->comment_buffer = (char *) &comment_buf_scratch;
+    }
+
+    iter->maps = os_open(PROC_SELF_MAPS, OS_OPEN_READ);
+    ASSERT(iter->maps != INVALID_FILE);
+    iter->buf[BUFSIZE-1] = '\0'; /* permanently */
+
+    iter->may_alloc = may_alloc;
+    iter->newline = NULL;
+    iter->bufread = 0;
+    iter->vm_start = NULL;
+    iter->comment = iter->comment_buffer;
+    return true;
+}
+
+static void
+maps_iterator_stop(maps_iter_t *iter)
+{
+    ASSERT((iter->may_alloc && OWN_MUTEX(&maps_iter_buf_lock)) ||
+           (!iter->may_alloc && OWN_MUTEX(&memory_info_buf_lock)));
+    os_close(iter->maps);
+    if (iter->may_alloc)
+        mutex_unlock(&maps_iter_buf_lock);
+    else
+        mutex_unlock(&memory_info_buf_lock);
+}
+
+static bool
+maps_iterator_next(maps_iter_t *iter)
+{
+    char perm[16];
+    char *line;
+    int len;
+    app_pc prev_start = iter->vm_start;
+    ASSERT((iter->may_alloc && OWN_MUTEX(&maps_iter_buf_lock)) ||
+           (!iter->may_alloc && OWN_MUTEX(&memory_info_buf_lock)));
+    if (iter->newline == NULL) {
+        iter->bufwant = BUFSIZE-1;
+        iter->bufread = os_read(iter->maps, iter->buf, iter->bufwant);
+        ASSERT(iter->bufread <= iter->bufwant);
+        LOG(GLOBAL, LOG_VMAREAS, 6,
+            "get_memory_info_from_os: bytes read %d/want %d\n",
+            iter->bufread, iter->bufwant);
+        if (iter->bufread <= 0)
+            return false;
+        iter->buf[iter->bufread] = '\0';
+        iter->newline = strchr(iter->buf, '\n');
+        line = iter->buf;
+    } else {
+        line = iter->newline + 1;
+        iter->newline = strchr(line, '\n');
+        if (iter->newline == NULL) {
+            /* FIXME clean up: factor out repetitive code */
+            /* shift 1st part of line to start of buf, then read in rest */
+            /* the memory for the processed part can be reused  */
+            iter->bufwant = line - iter->buf;
+            ASSERT(iter->bufwant <= iter->bufread);
+            len = iter->bufread - iter->bufwant; /* what is left from last time */
+            /* since strings may overlap, should use memmove, not strncpy */
+            /* FIXME corner case: if len == 0, nothing to move */
+            memmove(iter->buf, line, len);
+            iter->bufread = os_read(iter->maps, iter->buf+len, iter->bufwant);
+            ASSERT(iter->bufread <= iter->bufwant);
+            if (iter->bufread <= 0)
+                return false;
+            iter->bufread += len; /* bufread is total in buf */
+            iter->buf[iter->bufread] = '\0';
+            iter->newline = strchr(iter->buf, '\n');
+            line = iter->buf;
+        }
+    }
+    LOG(GLOBAL, LOG_VMAREAS, 6, 
+        "\nget_memory_info_from_os: newline=[%s]\n",
+        iter->newline ? iter->newline : "(null)");
+
+    /* buffer is big enough to hold at least one line */
+    ASSERT(iter->newline != NULL);
+    *iter->newline = '\0';
+    LOG(GLOBAL, LOG_VMAREAS, 6, 
+        "\nget_memory_info_from_os: line=[%s]\n", line);
+    iter->comment_buffer[0]='\0';
+    len = sscanf(line,
+#ifdef IA32_ON_IA64
+                 MAPS_LINE_FORMAT8, /* cross-compiling! */
+#else              
+                 sizeof(void*) == 4 ? MAPS_LINE_FORMAT4 : MAPS_LINE_FORMAT8,
+#endif
+                 (unsigned long*)&iter->vm_start, (unsigned long*)&iter->vm_end,
+                 perm, (unsigned long*)&iter->offset, &iter->inode,
+                 iter->comment_buffer);
+    if (iter->vm_start <= prev_start) {
+        /* the maps file has expanded underneath us (presumably due to our
+         * own committing while iterating): skip ahead */
+        LOG(GLOBAL, LOG_VMAREAS, 2, 
+            "maps_iterator_next: maps file changed: skipping 0x%08x\n", prev_start);
+        iter->vm_start = prev_start;
+        return maps_iterator_next(iter);
+    }
+    if (len<6)
+        iter->comment_buffer[0]='\0';
+    iter->prot = permstr_to_memprot(perm);
+    return true;
+}
+
+#ifndef HAVE_PROC_MAPS
+/* PR 361594: os-independent alternative to /proc/maps, though this
+ * relies on user libraries
+ */
+typedef struct _dl_iterate_data_t {
+    app_pc target_base;
+    char *path_out;
+    size_t path_size;
+} dl_iterate_data_t;
+
+static int
+dl_iterate_get_path_cb(struct dl_phdr_info *info, size_t size, void *data)
+{
+    dl_iterate_data_t *iter_data = (dl_iterate_data_t *) data;
+    /* info->dlpi->addr is offset from preferred so we need to calculate the
+     * absolute address of the base.
+     * we can calculate the absolute address of the first segment, but ELF
+     * doesn't seem to guarantee that either the elf header (base of
+     * file) or the program headers (info->dlpi_phdr) are later than
+     * the min_vaddr, so it's a little confusing as to what would be
+     * in the maps file or whathever and would thus be the base we're looking
+     * to match: for now we assume the page with min_vaddr is that base.
+     * If elf header, program header, and 1st segment could all be on
+     * separate pages, I don't see any way to find the elf header in
+     * such cases short of walking backward and looking for the magic #s.
+     */
+    app_pc min_vaddr =
+        module_vaddr_from_prog_header((app_pc)info->dlpi_phdr, info->dlpi_phnum, NULL);
+    app_pc base = info->dlpi_addr + min_vaddr;
+    LOG(GLOBAL, LOG_VMAREAS, 2,
+        "dl_iterate_get_path_cb: addr="PFX" hdrs="PFX" base="PFX" name=%s\n",
+        info->dlpi_addr, info->dlpi_phdr, base, info->dlpi_name);
+    if (iter_data->target_base == base) {
+        /* We want just the path, not the filename */
+        char *slash = rindex(info->dlpi_name, '/');
+        ASSERT_CURIOSITY(slash != NULL);
+        ASSERT_CURIOSITY((slash - info->dlpi_name) < iter_data->path_size);
+        strncpy(iter_data->path_out, info->dlpi_name,
+                MIN(iter_data->path_size, (slash - info->dlpi_name)));
+        iter_data->path_out[iter_data->path_size] = '\0';
+        return 1; /* done iterating */
+    }
+    return 0; /* keep looking */
+}
+#endif
+
+/* Finds the bounds of the library with name "name".  If "name" is NULL,
+ * "start" must be non-NULL and must be an address within the library.
+ * Note that we can't just walk backward and look for is_elf_so_header() b/c
+ * some ELF files are mapped twice and it's not clear how to know if
+ * one has hit the original header or a later header: this is why we allow
+ * any address in the library.
+ * The resulting start and end are the bounds of the library.
+ * Return value is the number of distinct memory regions that comprise the library.
+ */
+/* Gets the library bounds from walking the map file (as opposed to using our cached
+ * module_list) since is only used for dr and client libraries which aren't on the list.
+ */
+static int
+get_library_bounds(const char *name, app_pc *start/*IN/OUT*/, app_pc *end/*OUT*/,
+                   char *fullpath/*OPTIONAL OUT*/, size_t path_size)
+{
+    int count = 0;
+#ifdef HAVE_PROC_MAPS
+    bool found_library = false;
+    char libname[MAXIMUM_PATH];
+    const char *name_cmp = name;
+    maps_iter_t iter;
+    app_pc last_base = NULL;
+    app_pc last_end = NULL;
+#endif
+    size_t image_size = 0;
+    app_pc cur_end = NULL;
+    app_pc mod_start = NULL;
+    ASSERT(name != NULL || start != NULL);
+
+#ifndef HAVE_PROC_MAPS
+    /* PR 361594: os-independent alternative to /proc/maps */
+    ASSERT(start != NULL);
+    ASSERT(is_elf_so_header(*start, 0));
+    /* Get size by walking the program headers. Size includes the .bss section. */
+    if (module_walk_program_headers(*start,
+                                    /* FIXME: don't have view size: but
+                                     * anything larger than header sizes works
+                                     */
+                                    PAGE_SIZE, 
+                                    false, &mod_start, &cur_end, NULL, NULL)) {
+        /* we're passed back start,end of preferred base */
+        image_size = cur_end - mod_start;
+        mod_start = *start;
+        cur_end = mod_start + image_size;
+        count = 1; /* nobody uses this anyway */
+    } else {
+        ASSERT_NOT_REACHED();
+    }
+    if (fullpath != NULL) {
+        /* We need the path for dynamorio_library_path for execve.
+         * For now we use dl_iterate_phdr() from glibc 2.2.4+.
+         */
+        dl_iterate_data_t iter_data = { mod_start, fullpath, path_size };
+        DEBUG_DECLARE(int res = )
+           dl_iterate_phdr(dl_iterate_get_path_cb, &iter_data);
+        ASSERT(res == 1);
+    }
+    LOG(GLOBAL, LOG_VMAREAS, 2,
+        "get_library_bounds %s => "PFX"-"PFX" %s\n",
+        name == NULL ? "<null>" : name, mod_start, cur_end,
+        fullpath == NULL ? "<no path requested>" : fullpath);
+#else
+    maps_iterator_start(&iter, false/*won't alloc*/);
+    libname[0] = '\0';
+    while (maps_iterator_next(&iter)) {
+        LOG(GLOBAL, LOG_VMAREAS, 5,"start="PFX" end="PFX" prot=%x comment=%s\n",
+            iter.vm_start, iter.vm_end, iter.prot, iter.comment);
+
+        /* Record the base of each differently-named set of entries up until
+         * we find our target, when we'll clobber libpath
+         */
+        if (!found_library &&
+            strncmp(libname, iter.comment, BUFFER_SIZE_ELEMENTS(libname)) != 0) {
+            last_base = iter.vm_start;
+            /* last_end is used to know what's readable beyond last_base */
+            if (TEST(PROT_READ, iter.prot))
+                last_end = iter.vm_end;
+            else
+                last_end = last_base;
+            /* remember name so we can find the base of a multiply-mapped so */
+            strncpy(libname, iter.comment, BUFFER_SIZE_ELEMENTS(libname));
+            NULL_TERMINATE_BUFFER(libname);
+        }
+
+        if ((name_cmp != NULL && strstr(iter.comment, name_cmp) != NULL) ||
+            (start != NULL && *start >= iter.vm_start && *start < iter.vm_end)) {
+            if (!found_library) {
+                size_t mod_readable_sz;
+                char *dst = (fullpath != NULL) ? fullpath : libname;
+                size_t dstsz = (fullpath != NULL) ? path_size :
+                    BUFFER_SIZE_ELEMENTS(libname);
+                char *slash = rindex(iter.comment, '/');
+                ASSERT_CURIOSITY(slash != NULL);
+                ASSERT_CURIOSITY((slash - iter.comment) < dstsz);
+                strncpy(dst, iter.comment, MIN(dstsz, (slash - iter.comment)));
+                /* if max no null */
+                dst[dstsz - 1] = '\0';
+                if (name == NULL)
+                    name_cmp = dst;
+                found_library = true;
+                /* Most library have multiple segments, and some have the
+                 * ELF header repeated in a later mapping, so we can't rely
+                 * on is_elf_so_header() and header walking.
+                 * We use the name tracking to remember the first entry
+                 * that had this name.
+                 */
+                if (last_base == NULL) {
+                    mod_start = iter.vm_start;
+                    mod_readable_sz = iter.vm_end - iter.vm_start;
+                } else {
+                    mod_start = last_base;
+                    mod_readable_sz = last_end - last_base;
+                }
+                if (is_elf_so_header(mod_start, mod_readable_sz)) {
+                    app_pc mod_base, mod_end;
+                    if (module_walk_program_headers(mod_start, mod_readable_sz, false,
+                                                    &mod_base, &mod_end, NULL, NULL)) {
+                        image_size = mod_end - mod_base;
+                        ASSERT_CURIOSITY(image_size != 0);
+                    } else {
+                        ASSERT_NOT_REACHED();
+                    }
+                } else {
+                    ASSERT(false && "expected elf header");
+                }
+            }
+            count++;
+            cur_end = iter.vm_end;
+#if 0
+            /* FIXME: this is for libdynamorio.so only: */
+            if (TESTANY(SELFPROT_ANY_DATA_SECTION, dynamo_options.protect_mask) &&
+                TESTALL(MEMPROT_READ|MEMPROT_WRITE, iter.prot)) {
+                /* FIXME: PR 212458/212459: we could get entire rw region here and
+                 * subtract out .data, .fspdata, .cspdata, and .nspdata,
+                 * but better to explicitly add .got, etc., since we should
+                 * examine each rw section and decide on frequency of unprotection.
+                 */
+            }
+#endif
+        } else if (found_library) {
+            /* hit non-matching, we expect module segments to be adjacent */
+            break;
+        }
+    }
+
+    /* Xref PR 208443: .bss sections are anonymous (no file name listed in
+     * maps file), but not every library has one.  We have to parse the ELF
+     * header to know since we can't assume that a subsequent anonymous
+     * region is .bss. */
+    if (image_size != 0 && cur_end - mod_start < image_size) {
+        /* Found a .bss section. Check current mapping (note might only be
+         * part of the mapping (due to os region merging? FIXME investigate). */
+        ASSERT_CURIOSITY(iter.vm_start == cur_end /* no gaps, FIXME might there be
+                                                   * a gap if the file has large
+                                                   * alignment and no data section?
+                                                   * curiosity for now*/);
+        ASSERT_CURIOSITY(iter.inode == 0); /* .bss is anonymous */
+        ASSERT_CURIOSITY(iter.vm_end - mod_start >= image_size);/* should be big enough */
+        count++;
+        cur_end = mod_start + image_size;
+    } else {
+        /* Shouldn't have more mapped then the size of the module, unless it's a
+         * second adjacent separate map of the same file.  Curiosity for now. */
+        ASSERT_CURIOSITY(image_size == 0 || cur_end - mod_start == image_size);
+    }
+    maps_iterator_stop(&iter);
+#endif
+
+    if (start != NULL)
+        *start = mod_start;
+    if (end != NULL)
+        *end = cur_end;
+    return count;
+}
+
+/* initializes dynamorio library bounds.
+ * does not use any heap.
+ * assumed to be called prior to find_executable_vm_areas.
+ */
+static int
+get_dynamo_library_bounds(void)
+{
+    /* Note that we're not counting DYNAMORIO_PRELOAD_NAME as a DR area, to match
+     * Windows, so we should unload it like we do there. The other reason not to
+     * count it is so is_in_dynamo_dll() can be the only exception to the
+     * never-execute-from-DR-areas list rule
+     */
+    int res;
+    /* PR 361594: we get our bounds from linker-provided symbols.
+     * Note that referencing the value of these symbols will crash:
+     * always use the address only.
+     */
+    extern int dynamorio_so_start, dynamorio_so_end;
+    app_pc check_start, check_end;
+    dynamo_dll_start = (app_pc) &dynamorio_so_start;
+    dynamo_dll_end = (app_pc) ALIGN_FORWARD(&dynamorio_so_end, PAGE_SIZE);
+#ifndef HAVE_PROC_MAPS
+    check_start = dynamo_dll_start;
+#endif
+    res = get_library_bounds(DYNAMORIO_LIBRARY_NAME,
+                             &check_start, &check_end,
+                             dynamorio_library_path,
+                             BUFFER_SIZE_ELEMENTS(dynamorio_library_path));
+    LOG(GLOBAL, LOG_VMAREAS, 1, PRODUCT_NAME" library path: %s\n",
+        dynamorio_library_path);
+    ASSERT(check_start == dynamo_dll_start && check_end == dynamo_dll_end);
+    LOG(GLOBAL, LOG_VMAREAS, 1, "DR library bounds: "PFX" to "PFX"\n",
+        dynamo_dll_start, dynamo_dll_end);
+#ifdef STATIC_LIBRARY
+    dynamorio_library_path[0] = '\0';
+#else
+    ASSERT(res > 0);
+#endif
+    return res;
+}
+
+#ifdef DEBUG
+void
+mem_stats_snapshot()
+{
+    /* FIXME: NYI */
+}
+#endif
+
+bool
+is_in_dynamo_dll(app_pc pc)
+{
+    ASSERT(dynamo_dll_start != NULL);
+    return (pc >= dynamo_dll_start && pc < dynamo_dll_end);
+}
+
+app_pc
+get_dynamorio_dll_start()
+{
+    if (dynamo_dll_start == NULL)
+        get_dynamo_library_bounds();
+    ASSERT(dynamo_dll_start != NULL);
+    return dynamo_dll_start;
+}
+
+app_pc
+get_dynamorio_dll_end()
+{
+    if (dynamo_dll_end == NULL)
+        get_dynamo_library_bounds();
+    ASSERT(dynamo_dll_end != NULL);
+    return dynamo_dll_end;
+}
+
+app_pc
+get_dynamorio_dll_preferred_base()
+{
+    /* on Linux there is no preferred base if we're PIC, 
+     * therefore is always equal to dynamo_dll_start  */
+    return get_dynamorio_dll_start();
+}
+
+
+/***************************************************************************/
+#ifndef HAVE_PROC_MAPS
+/* PR 361594: os-independent alternative to /proc/maps */
+
+# define VSYSCALL_PAGE_SO_NAME "linux-gate.so"
+/* For now we assume no OS config has user addresses above this value 
+ * We just go to max 32-bit (64-bit not supported yet: want lazy probing)
+ */
+# ifdef X64
+#  error X64 requires HAVE_PROC_MAPS: PR 364552
+# endif
+# define USER_MAX 0xfffff000
+
+/* callback for dl_iterate_phdr() for adding existing modules to our lists */
+static int
+dl_iterate_get_areas_cb(struct dl_phdr_info *info, size_t size, void *data)
+{
+    int *count = (int *) data;
+    uint i;
+    /* see comments in dl_iterate_get_path_cb() */
+    app_pc modend;
+    app_pc min_vaddr = module_vaddr_from_prog_header((app_pc)info->dlpi_phdr,
+                                                     info->dlpi_phnum, &modend);
+    app_pc modbase = info->dlpi_addr + min_vaddr;
+    size_t modsize = modend - min_vaddr;
+    LOG(GLOBAL, LOG_VMAREAS, 2,
+        "dl_iterate_get_areas_cb: addr="PFX" hdrs="PFX" base="PFX" name=%s\n",
+        info->dlpi_addr, info->dlpi_phdr, modbase, info->dlpi_name);
+    ASSERT(info->dlpi_phnum == module_num_program_headers(modbase));
+
+# ifndef X64
+    if (modsize == PAGE_SIZE && info->dlpi_name[0] == '\0') {
+        /* Candidate for VDSO.  Xref PR 289138 on using AT_SYSINFO to locate. */
+        /* Xref VSYSCALL_PAGE_START_HARDCODED but later linuxes randomize */
+        char *soname;
+        if (module_walk_program_headers(modbase, modsize, false,
+                                        NULL, NULL, &soname, NULL) &&
+            strncmp(soname, VSYSCALL_PAGE_SO_NAME,
+                    strlen(VSYSCALL_PAGE_SO_NAME)) == 0) {
+            ASSERT(!dynamo_initialized); /* .data should be +w */
+            ASSERT(vsyscall_page_start == NULL);
+            vsyscall_page_start = modbase;
+            LOG(GLOBAL, LOG_VMAREAS, 1, "found vsyscall page @ "PFX"\n",
+                vsyscall_page_start);
+        }
+    }
+# endif
+    if (modbase != vsyscall_page_start)
+        module_list_add(modbase, modsize, false, 0/*don't have inode*/, info->dlpi_name);
+
+    for (i = 0; i < info->dlpi_phnum; i++) {
+        app_pc start, end;
+        uint prot;
+        size_t align;
+        if (module_read_program_header(modbase, i, &start, &end, &prot, &align)) {
+            start += info->dlpi_addr;
+            end += info->dlpi_addr;
+            LOG(GLOBAL, LOG_VMAREAS, 2,
+                "\tsegment %d: "PFX"-"PFX" %s align=%d\n",
+                i, start, end, memprot_string(prot), align);
+            start = (app_pc) ALIGN_BACKWARD(start, PAGE_SIZE);
+            end = (app_pc) ALIGN_FORWARD(end, PAGE_SIZE);
+            LOG(GLOBAL, LOG_VMAREAS, 4,
+                "find_executable_vm_areas: adding: "PFX"-"PFX" prot=%d\n",
+                start, end, prot);
+            all_memory_areas_lock();
+            update_all_memory_areas(start, end, prot);
+            all_memory_areas_unlock();
+            if (app_memory_allocation(NULL, start, end - start, prot, true/*image*/
+                                      _IF_DEBUG("ELF SO")))
+                (*count)++;
+        }
+    }
+    return 0; /* keep iterating */
+}
+
+/* helper for find_vm_areas_via_probe() and get_memory_info_from_os() 
+ * returns the passed-in pc if the probe was successful; else, returns
+ * where the next probe should be (to skip DR memory).
+ * if the probe was successful, returns in prot the results.
+ */
+static app_pc
+probe_address(dcontext_t *dcontext, app_pc pc,
+              byte *our_heap_start, byte *our_heap_end, OUT uint *prot)
+{
+    app_pc base;
+    size_t size;
+    ASSERT(ALIGNED(pc, PAGE_SIZE));
+    ASSERT(prot != NULL);
+    *prot = MEMPROT_NONE;
+
+    /* skip our own vmheap */
+    if (pc >= our_heap_start && pc < our_heap_end)
+        return our_heap_end;
+# ifdef STACK_GUARD_PAGE
+    /* if no vmheap and we probe our own stack, the SIGSEGV handler will
+     * report stack overflow as it checks that prior to handling TRY
+     */
+    if (is_stack_overflow(dcontext, pc))
+        return pc + PAGE_SIZE;
+# endif
+    /* for find_vm_areas_via_probe(), skip modules added by dl_iterate_get_areas_cb.
+     * for get_memory_info_from_os() we may want to be able to disable this if
+     * we think all_memory_areas() is wrong?
+     */
+    if (get_memory_info(pc, &base, &size, prot))
+        return base + size;
+
+    TRY_EXCEPT(dcontext, /* try */ {
+        PROBE_READ_PC(pc);
+        *prot |= MEMPROT_READ;
+    }, /* except */ {
+        /* nothing: just continue */
+    });
+    /* x86 can't be writable w/o being readable.  avoiding nested TRY though. */
+    if (TEST(MEMPROT_READ, *prot)) {
+        TRY_EXCEPT(dcontext, /* try */ {
+            PROBE_WRITE_PC(pc);
+            *prot |= MEMPROT_WRITE;
+        }, /* except */ {
+            /* nothing: just continue */
+        });
+    }
+
+    LOG(GLOBAL, LOG_VMAREAS, 5, "%s: probe "PFX" => %s\n",
+        __func__, pc, memprot_string(*prot));
+
+    return pc;
+}
+
+/* helper for find_vm_areas_via_probe() */
+static inline int
+probe_add_region(app_pc *last_start, uint *last_prot,
+                 app_pc pc, uint prot, bool force)
+{
+    int count = 0;
+    if (force || prot != *last_prot) {
+        /* We record unreadable regions as the absence of an entry */
+        if (*last_prot != MEMPROT_NONE) {
+            all_memory_areas_lock();
+            update_all_memory_areas(*last_start, pc, *last_prot);
+            all_memory_areas_unlock();
+            if (app_memory_allocation(NULL, *last_start, pc - *last_start,
+                                      *last_prot, false/*!image*/ _IF_DEBUG("")))
+                count++;
+        }
+        *last_prot = prot;
+        *last_start = pc;
+    }
+    return count;
+}
+
+/* non-/proc/maps version of find_executable_vm_areas() */
+static int
+find_vm_areas_via_probe(void)
+{
+    /* PR 361594: now that sigsegv handler is set up, loop&probe.
+     * First, dl_iterate_phdr() to get modules, and walk their
+     * segments to get internal regions: then can avoid
+     * wasting time probing modules.
+     *
+     * TODO PR 364552:
+     * Would be nice to probe lazily to avoid touching all non-module
+     * pages and avoid wasting our time on faults in large empty areas
+     * of the address space.  This is especially important for 64-bit.
+     * If done lazily, to avoid races, should do "lock add *page,0".
+     * If that faults, then just try to read.  However, we'll need
+     * nested SIGSEGV support to handle probing while inside a SIGSEGV
+     * handler: that's PR 287309.
+     *
+     * Note that we have no good way (at least that's not racy, or
+     * that'll work if there's no NX) to check for +x, and as such we
+     * require HAVE_PROC_MAPS for PROGRAM_SHEPHERDING (also xref PR
+     * 210383: NX transparency).
+     *
+     * Note also that we assume a "normal" segment setup: no funny
+     * +x but -rw segments.
+     */
+    int count = 0;
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    app_pc pc, last_start = NULL;
+    uint prot, last_prot = MEMPROT_NONE;
+    byte *our_heap_start, *our_heap_end;
+    get_vmm_heap_bounds(&our_heap_start, &our_heap_end);
+
+    DEBUG_DECLARE(int res = )
+       dl_iterate_phdr(dl_iterate_get_areas_cb, &count);
+    ASSERT(res == 0);
+
+    ASSERT(dcontext != NULL);
+
+    ASSERT(ALIGNED(USER_MAX, PAGE_SIZE));
+    for (pc = (app_pc) PAGE_SIZE; pc < (app_pc) USER_MAX; ) {
+        prot = MEMPROT_NONE;
+        app_pc next_pc = probe_address(dcontext, pc, our_heap_start, our_heap_end, &prot);
+        count += probe_add_region(&last_start, &last_prot, pc, prot, next_pc != pc);
+        if (next_pc != pc) {
+            pc = next_pc;
+            last_prot = MEMPROT_NONE; /* ensure we add adjacent region */
+            last_start = pc;
+        } else
+            pc += PAGE_SIZE;
+    }
+    count += probe_add_region(&last_start, &last_prot, pc, prot, true);
+    return count;
+}
+
+#endif /* !HAVE_PROC_MAPS */
+/***************************************************************************/
+
+
+/* assumed to be called after find_dynamo_library_vm_areas() */
+int
+find_executable_vm_areas(void)
+{
+    int count = 0;
+
+#ifndef HAVE_PROC_MAPS
+    count = find_vm_areas_via_probe();
+#else
+    maps_iter_t iter;
+    maps_iterator_start(&iter, true/*may alloc*/);
+    while (maps_iterator_next(&iter)) {
+        bool overlaps_image;
+        bool image = false;
+        size_t size = iter.vm_end - iter.vm_start;
+        DEBUG_DECLARE(char *map_type = "Private");
+        /* we can't really tell what's a stack and what's not, but we rely on
+         * our passing NULL preventing rwx regions from being added to executable
+         * or future list, even w/ -executable_if_alloc
+         */
+
+        LOG(GLOBAL, LOG_VMAREAS, 2,
+            "start="PFX" end="PFX" prot=%x comment=%s\n",
+            iter.vm_start, iter.vm_end, iter.prot, iter.comment);
+
+        /* find modules for module list */
+        overlaps_image = mmap_check_for_module_overlap(iter.vm_start, size,
+                                                       TEST(MEMPROT_READ, iter.prot),
+                                                       iter.inode, false);
+
+        if (strncmp(iter.comment, VSYSCALL_PAGE_MAPS_NAME,
+                    strlen(VSYSCALL_PAGE_MAPS_NAME)) == 0
+            /* Older kernels do not label it as "[vdso]", but it is hardcoded there */
+            IF_NOT_X64(|| iter.vm_start == VSYSCALL_PAGE_START_HARDCODED)) {
+# ifndef X64
+            /* We assume no vsyscall page for x64; thus, checking the
+             * hardcoded address shouldn't have any false positives.
+             */
+            ASSERT(iter.vm_end - iter.vm_start == PAGE_SIZE);
+            ASSERT(!dynamo_initialized); /* .data should be +w */
+            ASSERT(vsyscall_page_start == NULL);
+            vsyscall_page_start = iter.vm_start;
+            LOG(GLOBAL, LOG_VMAREAS, 1, "found vsyscall page @ "PFX" %s\n",
+                vsyscall_page_start, iter.comment);
+# endif
+        } else if (overlaps_image) {
+            /* we already added the whole image region when we hit the first map for it */
+            image = true;
+            DODEBUG({ map_type = "ELF SO"; });
+        } else if (TEST(MEMPROT_READ, iter.prot) &&
+                   is_elf_so_header(iter.vm_start, size)) {
+            size_t image_size = size;
+            app_pc mod_base, mod_end;
+            image = true;
+            DODEBUG({ map_type = "ELF SO"; });
+            LOG(GLOBAL, LOG_VMAREAS, 2,
+                "Found already mapped module first segment :\n"
+                "\t"PFX"-"PFX"%s inode="UINT64_FORMAT_STRING" name=%s\n",
+                iter.vm_start, iter.vm_end, TEST(MEMPROT_EXEC, iter.prot) ? " +x": "",
+                iter.inode, iter.comment);
+            ASSERT_CURIOSITY(iter.inode != 0); /* mapped images should have inodes */
+            ASSERT_CURIOSITY(iter.offset == 0); /* first map shouldn't have offset */
+            /* Get size by walking the program headers. Size includes the .bss section. */
+            if (module_walk_program_headers(iter.vm_start, size, false,
+                                            &mod_base, &mod_end, NULL, NULL)) {
+                image_size = mod_end - mod_base;
+            } else {
+                ASSERT_NOT_REACHED();
+            }
+            LOG(GLOBAL, LOG_VMAREAS, 2,
+                "Found already mapped module total module :\n"
+                "\t"PFX"-"PFX" inode="UINT64_FORMAT_STRING" name=%s\n",
+                iter.vm_start, iter.vm_start+image_size, iter.inode, iter.comment);
+            module_list_add(iter.vm_start, image_size, false, iter.inode, iter.comment);
+        } else if (iter.inode != 0) {
+            DODEBUG({ map_type = "Mapped File"; });
+        }
+
+        /* add all regions (incl. dynamo_areas and stack) to all_memory_areas */
+        LOG(GLOBAL, LOG_VMAREAS, 4,
+            "find_executable_vm_areas: adding: "PFX"-"PFX" prot=%d\n",
+            iter.vm_start, iter.vm_end, iter.prot);
+        all_memory_areas_lock();
+        update_all_memory_areas(iter.vm_start, iter.vm_end, iter.prot);
+        all_memory_areas_unlock();
+    
+        /* FIXME: best if we could pass every region to vmareas, but
+         * it has no way of determining if this is a stack b/c we don't have
+         * a dcontext at this point -- so we just don't pass the stack
+         */
+        if (app_memory_allocation(NULL, iter.vm_start, (iter.vm_end - iter.vm_start),
+                                  iter.prot, image _IF_DEBUG(map_type))) {
+            count++;
+        }
+
+    }
+    maps_iterator_stop(&iter);
+#endif /* HAVE_PROC_MAPS */
+
+    DODEBUG({
+        LOG(GLOBAL, LOG_VMAREAS, 4, "init: all memory areas:\n");
+        DOLOG(4, LOG_VMAREAS, print_all_memory_areas(GLOBAL););
+    });
+
+    /* now that we've walked memory print all modules */
+    LOG(GLOBAL, LOG_VMAREAS, 2, "Module list after memory walk\n");
+    DOLOG(1, LOG_VMAREAS, { print_modules(GLOBAL, DUMP_NOT_XML); });
+
+    STATS_ADD(num_app_code_modules, count);
+    return count;
+}
+
+/* initializes dynamorio library bounds.
+ * does not use any heap.
+ * assumed to be called prior to find_executable_vm_areas.
+ */
+int
+find_dynamo_library_vm_areas(void)
+{
+    /* We didn't add inside get_dynamo_library_bounds b/c it was called pre-alloc.
+     * We don't bother to break down the sub-regions.
+     * Assumption: we don't need to have the protection flags for DR sub-regions.
+     */
+    add_dynamo_vm_area(get_dynamorio_dll_start(), get_dynamorio_dll_end(),
+                       MEMPROT_READ|MEMPROT_WRITE|MEMPROT_EXEC,
+                       true /* from image */ _IF_DEBUG(dynamorio_library_path));
+    return 1;
+}
+
+bool
+get_stack_bounds(dcontext_t *dcontext, byte **base, byte **top)
+{
+    os_thread_data_t *ostd = (os_thread_data_t *) dcontext->os_field;
+    if (ostd->stack_base == NULL) {
+        /* initialize on-demand since don't have app esp handy in os_thread_init()
+         * FIXME: the comment here -- ignoring it for now, if hit cases confirming
+         * it the right thing will be to merge adjacent rwx regions and assume
+         * their union is the stack -- otherwise have to have special stack init
+         * routine called from x86.asm new_thread_dynamo_start and internal_dynamo_start,
+         * and the latter is not a do-once...
+         */
+         size_t size;
+         bool ok;
+         /* store stack info at thread startup, since stack can get fragmented in
+          * /proc/self/maps w/ later mprotects and it can be hard to piece together later
+          */
+         /* FIXME: once our mem list is robust use it */
+         ok = get_memory_info_from_os((app_pc)get_mcontext(dcontext)->xsp,
+                                      &ostd->stack_base, &size, NULL);
+         ASSERT(ok);
+         ostd->stack_top = ostd->stack_base + size;
+         LOG(THREAD, LOG_THREADS, 1, "App stack is "PFX"-"PFX"\n",
+             ostd->stack_base, ostd->stack_top);
+    }
+    if (base != NULL)
+        *base = ostd->stack_base;
+    if (top != NULL)
+        *top = ostd->stack_top;
+    return true;
+}
+
+#ifdef RETURN_AFTER_CALL
+initial_call_stack_status_t
+at_initial_stack_bottom(dcontext_t *dcontext, app_pc target_pc)
+{
+    os_thread_data_t *ostd = (os_thread_data_t *) dcontext->os_field;
+    if (target_pc == ostd->stack_bottom_pc) {
+        return INITIAL_STACK_BOTTOM_REACHED;
+    } else {
+        return INITIAL_STACK_BOTTOM_NOT_REACHED;
+    }
+    /* we never start with an INITIAL_STACK_EMPTY */
+}
+#endif /* RETURN_AFTER_CALL */
+
+/* Use our cached data structures to retrieve memory info */
+bool
+get_memory_info(const byte *pc, byte **base_pc, size_t *size,
+                uint *prot /* OUT optional, returns MEMPROT_* value */)
+{
+    app_pc start;
+    app_pc end;
+    /* PR 314391: if we declare as uint, vmvector_lookup_data clobbers
+     * the 32 bits next to it on the stack!
+     */
+    ptr_uint_t cached_prot;
+    bool found;
+
+    /* Check all-regions for an overlap and return the pertinent info
+     * if it exists. */
+    found = vmvector_lookup_data(all_memory_areas, (app_pc)pc,
+                                 &start, &end, (void **) &cached_prot);
+    if (found) {
+        if (base_pc != NULL)
+            *base_pc = start;
+        if (size != NULL)
+            *size = end - start;
+        if (prot != NULL) {
+            IF_X64(ASSERT_TRUNCATE(*prot, uint, cached_prot));
+            *prot = (uint) cached_prot;
+        }
+#ifdef HAVE_PROC_MAPS
+        DODEBUG({
+            byte *from_os_base_pc;
+            size_t from_os_size;
+            uint from_os_prot;
+            bool from_os_rc;
+            from_os_rc = get_memory_info_from_os(pc, &from_os_base_pc, &from_os_size,
+                                                 &from_os_prot);
+            ASSERT(from_os_rc == true);
+            /* /proc/maps could break/combine regions listed so region bounds as
+             * listed by all_memory_areas and /proc/maps won't agree.
+             * FIXME: Have seen instances where all_memory_areas lists the region as
+             * r--, where /proc/maps lists it as r-x.  Infact, all regions listed in
+             * /proc/maps are executable, even guard pages --x (see case 8821)
+             */
+            if (from_os_base_pc != start ||
+                from_os_size != end - start ||
+                from_os_prot != cached_prot) {
+                LOG(THREAD_GET, LOG_VMAREAS, 3, "get_memory_info mismatch! "
+                    "(can happen if os combines entries in /proc/pid/maps)\n"
+                    "\tos says: "PFX"-"PFX" prot="PFX"\n"
+                    "\tcache says: "PFX"-"PFX" prot="PFX"\n",
+                    from_os_base_pc, from_os_base_pc + from_os_size,
+                    from_os_prot, start, end, cached_prot);
+            }
+        });
+#endif
+    } else {
+        /* Could be a bad access by app (like common/segfault.c), or it could
+         * be memory added by a client without our knowledge.  We can end up in
+         * an infinite loop trying to forge a SIGSEGV in that situation if
+         * executing from what we think is unreadable memory, so best to check
+         * with the OS (xref PR 363811).
+         */
+#ifdef HAVE_PROC_MAPS
+        LOG(THREAD_GET, LOG_ALL, 1, "get_memory_info: pc="PFX" not found\n", pc);
+        DOLOG(4, LOG_VMAREAS, print_all_memory_areas(THREAD_GET););
+        found = get_memory_info_from_os(pc, base_pc, size, (void *) prot);
+#else
+        /* FIXME: This is dangerous since we need to probe and we do
+         * not yet support nested SIGSEGV (PR 287309).
+         */
+#endif
+    }
+
+    return found;
+}
+
+/* called at arbitrary places, so cannot use fopen */
+bool
+get_memory_info_from_os(const byte *pc, byte **base_pc, size_t *size,
+                        uint *prot /* OUT optional, returns MEMPROT_* value */)
+{
+    bool found = false;
+
+#ifndef HAVE_PROC_MAPS
+    app_pc probe_pc, next_pc;
+    app_pc start_pc = (app_pc) pc, end_pc = (app_pc) pc + PAGE_SIZE;
+    byte *our_heap_start, *our_heap_end;
+    uint cur_prot = MEMPROT_NONE;
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    ASSERT(dcontext != NULL);
+    get_vmm_heap_bounds(&our_heap_start, &our_heap_end);
+    /* silly to loop over all x64 pages */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+
+    next_pc = probe_address(dcontext, (app_pc) pc, our_heap_start,
+                            our_heap_end, &cur_prot);
+    if (next_pc != pc) {
+        /* FIXME: should iterate the cases: if vmheap, return its bounds */
+        return get_memory_info(pc, base_pc, size, prot);
+    } else
+        found = true;
+    if (base_pc != NULL || size != NULL) {
+        for (probe_pc = (app_pc) ALIGN_BACKWARD(pc, PAGE_SIZE) - PAGE_SIZE;
+             probe_pc > (app_pc) NULL; probe_pc -= PAGE_SIZE) {
+            uint prot = MEMPROT_NONE;
+            next_pc = probe_address(dcontext, probe_pc, our_heap_start,
+                                    our_heap_end, &prot);
+            if (next_pc != pc || prot != cur_prot)
+                break;
+        }
+        start_pc = probe_pc + PAGE_SIZE;
+        ASSERT(ALIGNED(USER_MAX, PAGE_SIZE));
+        for (probe_pc = (app_pc) ALIGN_FORWARD(pc, PAGE_SIZE);
+             probe_pc < (app_pc) USER_MAX; probe_pc += PAGE_SIZE) {
+            uint prot = MEMPROT_NONE;
+            next_pc = probe_address(dcontext, probe_pc, our_heap_start,
+                                    our_heap_end, &prot);
+            if (next_pc != pc || prot != cur_prot)
+                break;
+        }
+        end_pc = probe_pc;
+    }
+    all_memory_areas_lock();
+    update_all_memory_areas(start_pc, end_pc, cur_prot);
+    all_memory_areas_unlock();
+    app_memory_allocation(NULL, start_pc, end_pc - start_pc, cur_prot,
+                          false/*!image*/ _IF_DEBUG("unknown"));
+    if (prot != NULL)
+        *prot = cur_prot;
+    if (base_pc != NULL)
+        *base_pc = start_pc;
+    if (size != NULL)
+        *size = end_pc - start_pc;
+#else
+    maps_iter_t iter;
+    maps_iterator_start(&iter, false/*won't alloc*/);
+    while (maps_iterator_next(&iter)) {
+        if (pc >= iter.vm_start && pc < iter.vm_end) {
+            if (base_pc != NULL)
+                *base_pc = iter.vm_start;
+            if (size != NULL)
+                *size = (iter.vm_end - iter.vm_start);
+            if (prot != NULL) {
+                *prot = iter.prot;
+            }
+            /* On early (pre-Fedora 2) kernels the vsyscall page is listed
+             * with no permissions at all in the maps file.  Here's RHEL4:
+             *   ffffe000-fffff000 ---p 00000000 00:00 0
+             * We return "rx" as the permissions in that case.
+             */
+            if (vsyscall_page_start != NULL &&
+                pc >= vsyscall_page_start && pc < vsyscall_page_start+PAGE_SIZE) {
+                ASSERT(iter.vm_start == vsyscall_page_start);
+                ASSERT(iter.vm_end - iter.vm_start == PAGE_SIZE);
+                if (prot != NULL && iter.prot == MEMPROT_NONE) {
+                    *prot = (MEMPROT_READ|MEMPROT_EXEC);
+                }
+            }
+            found = true;
+            break;
+        }
+    }
+    maps_iterator_stop(&iter);
+#endif
+    return found;
+}
+
+/* HACK to get recursive write lock for internal and external use
+ * FIXME: code blatantly copied from dynamo_vm_areas_{un}lock(); eliminate duplication!
+ */
+void
+all_memory_areas_lock()
+{
+    /* ok to ask for locks or mark stale before all_memory_areas is allocated,
+     * during heap init and before we can allocate it.  no lock needed then.
+     */
+    ASSERT(all_memory_areas != NULL ||
+           get_num_threads() <= 1 /* must be only DR thread */);
+    if (all_memory_areas == NULL)
+        return;
+    if (self_owns_write_lock(&all_memory_areas->lock)) {
+        all_memory_areas_recursion++;
+        /* we have a 5-deep path:
+         *   global_heap_alloc | heap_create_unit | get_guarded_real_memory | 
+         *   heap_low_on_memory | release_guarded_real_memory
+         */
+        ASSERT_CURIOSITY(all_memory_areas_recursion <= 4);
+    } else
+        write_lock(&all_memory_areas->lock);
+}
+
+void
+all_memory_areas_unlock()
+{
+    /* ok to ask for locks or mark stale before all_memory_areas is allocated,
+     * during heap init and before we can allocate it.  no lock needed then.
+     */
+    ASSERT(all_memory_areas != NULL ||
+           get_num_threads() <= 1 /*must be only DR thread*/);
+    if (all_memory_areas == NULL)
+        return;
+    if (all_memory_areas_recursion > 0) {
+        ASSERT_OWN_WRITE_LOCK(true, &all_memory_areas->lock);
+        all_memory_areas_recursion--;
+    } else
+        write_unlock(&all_memory_areas->lock);
+}
+
+
+/* in utils.c, exported only for our hack! */
+extern void deadlock_avoidance_unlock(mutex_t *lock, bool ownable);
+
+void
+mutex_wait_contended_lock(mutex_t *lock)
+{
+    IF_CLIENT_INTERFACE(dcontext_t *dcontext = get_thread_private_dcontext();)
+    /* FIXME: we don't actually use system calls to synchronize on Linux,
+     * one day we would use futex(2) on this path (PR 295561). 
+     * For now we use a busy-wait lock.
+     * If we do use a true wait need to set client_thread_safe_for_synch around it */
+
+    /* we now have to undo our earlier request */
+    atomic_dec_and_test(&lock->lock_requests);
+
+    while (!mutex_trylock(lock)) {
+#ifdef CLIENT_INTERFACE
+        if (dcontext != NULL && IS_CLIENT_THREAD(dcontext) &&
+            (mutex_t *)dcontext->client_data->client_grab_mutex == lock)
+            dcontext->client_data->client_thread_safe_for_synch = true;
+#endif
+        thread_yield();
+#ifdef CLIENT_INTERFACE
+        if (dcontext != NULL && IS_CLIENT_THREAD(dcontext) &&
+            (mutex_t *)dcontext->client_data->client_grab_mutex == lock)
+            dcontext->client_data->client_thread_safe_for_synch = false;
+#endif
+    }
+
+#ifdef DEADLOCK_AVOIDANCE
+    /* HACK: trylock's success causes it to do DEADLOCK_AVOIDANCE_LOCK, so to
+     * avoid two in a row (causes assertion on owner) we unlock here
+     * In the future we will remove the trylock here and this will go away.
+     */
+    deadlock_avoidance_unlock(lock, true);
+#endif
+
+    return;
+    
+}
+
+void
+mutex_notify_released_lock(mutex_t *lock)
+{
+    /* nothing to do here */
+}
+
+/* read_write_lock_t implementation doesn't expect the contention path
+   helpers to guarantee the lock is held (unlike mutexes) so simple
+   yields are still acceptable.
+*/
+void
+rwlock_wait_contended_writer(read_write_lock_t *rwlock)
+{
+    thread_yield();
+}
+
+void
+rwlock_notify_writer(read_write_lock_t *rwlock)
+{
+    /* nothing to do here */
+}
+
+void
+rwlock_wait_contended_reader(read_write_lock_t *rwlock)
+{
+    thread_yield();
+}
+
+void
+rwlock_notify_readers(read_write_lock_t *rwlock)
+{
+    /* nothing to do here */
+}
+
+/***************************************************************************/
+
+/* events are un-signaled when successfully waited upon. */
+typedef struct linux_event_t {
+    volatile bool signaled;
+    mutex_t lock;
+} linux_event_t;
+
+
+/* FIXME: this routine will need to have a macro wrapper to let us assign different ranks to 
+ * all events for DEADLOCK_AVOIDANCE.  Currently a single rank seems to work.
+ */
+event_t
+create_event()
+{
+    event_t e = (event_t) global_heap_alloc(sizeof(linux_event_t) HEAPACCT(ACCT_OTHER));
+    e->signaled = false;
+    ASSIGN_INIT_LOCK_FREE(e->lock, event_lock); /* FIXME: we'll need to pass the event name here */
+    return e;
+}
+
+void
+destroy_event(event_t e)
+{
+    DELETE_LOCK(e->lock);
+    global_heap_free(e, sizeof(linux_event_t) HEAPACCT(ACCT_OTHER));
+}
+
+/* FIXME PR 295561: use futex */
+void
+signal_event(event_t e)
+{
+    mutex_lock(&e->lock);
+    e->signaled = true;
+    LOG(THREAD_GET, LOG_THREADS, 3,"thread %d signalling event "PFX"\n",get_thread_id(),e);
+    mutex_unlock(&e->lock);
+}
+
+void
+reset_event(event_t e)
+{
+    mutex_lock(&e->lock);
+    e->signaled = false;
+    LOG(THREAD_GET, LOG_THREADS, 3,"thread %d resetting event "PFX"\n",get_thread_id(),e);
+    mutex_unlock(&e->lock);
+}
+
+/* FIXME: compare use and implementation with  man pthread_cond_wait */
+/* FIXME PR 295561: use futex */
+void
+wait_for_event(event_t e)
+{
+#ifdef DEBUG
+    dcontext_t *dcontext = get_thread_private_dcontext();
+#endif
+    /* Use a user-space event on Linux, a kernel event on Windows. */
+    LOG(THREAD, LOG_THREADS, 3, "thread %d waiting for event "PFX"\n",get_thread_id(),e);
+    while (true) {
+        if (e->signaled) {
+            mutex_lock(&e->lock);
+            if (!e->signaled) {
+                /* some other thread beat us to it */
+                LOG(THREAD, LOG_THREADS, 3, "thread %d was beaten to event "PFX"\n",
+                    get_thread_id(),e);
+                mutex_unlock(&e->lock);
+            } else {
+                /* reset the event */
+                e->signaled = false;
+                mutex_unlock(&e->lock);
+                LOG(THREAD, LOG_THREADS, 3,
+                    "thread %d finished waiting for event "PFX"\n", get_thread_id(),e);
+                return;
+            }
+        }
+        thread_yield();
+    }
+}
+
+uint
+os_random_seed()
+{
+    uint seed;
+    /* reading from /dev/urandom for a non-blocking random */
+    int urand = os_open("/dev/urandom", OS_OPEN_READ);
+    DEBUG_DECLARE(int read = )os_read(urand, &seed, sizeof(seed));
+    ASSERT(read == sizeof(seed));
+    os_close(urand);
+    
+    return seed;
+}
+
+#ifdef RCT_IND_BRANCH
+/* Analyze a range in a possibly new module
+ * return false if not a code section in a module
+ * otherwise returns true and adds all valid targets for rct_ind_branch_check
+ */
+bool
+rct_analyze_module_at_violation(dcontext_t *dcontext, app_pc target_pc)
+{
+    /* FIXME: note that this will NOT find the data section corresponding to the given PC
+     * we don't yet have a corresponding get_allocation_size or an ELF header walk routine
+     * on linux
+     */
+    app_pc code_start;
+    size_t code_size;
+    uint prot;
+     
+    if (!get_memory_info(target_pc, &code_start, &code_size, &prot))
+        return false;
+    /* TODO: in almost all cases expect the region at module_base+module_size to be 
+     * the corresponding data section.  
+     * Writable yet initialized data indeed needs to be processed.
+     */
+
+    if (code_size > 0) {
+        app_pc code_end = code_start + code_size;
+        
+        app_pc data_start;
+        size_t data_size;
+
+        ASSERT(TESTALL(MEMPROT_READ|MEMPROT_EXEC, prot)); /* code */
+
+        if (!get_memory_info(code_end, &data_start, &data_size, &prot))
+            return false;
+
+        ASSERT(data_start == code_end);
+        ASSERT(TESTALL(MEMPROT_READ|MEMPROT_WRITE, prot)); /* data */
+
+        app_pc text_start = code_start;
+        app_pc text_end = data_start + data_size;
+
+        /* TODO: performance: should do this only in case relocation info is not present */
+        DEBUG_DECLARE(uint found = )
+            find_address_references(dcontext, text_start, text_end, 
+                                    code_start, code_end);
+        LOG(GLOBAL, LOG_RCT, 2, PFX"-"PFX" : %d ind targets of %d code size", 
+            text_start, text_end,
+            found, code_size);
+        return true;
+    }
+    return false;
+}
+
+#ifdef X64
+bool
+rct_add_rip_rel_addr(dcontext_t *dcontext, app_pc tgt _IF_DEBUG(app_pc src))
+{
+    /* FIXME PR 276762: not implemented */
+    return false;
+}
+#endif
+
+#ifdef HOT_PATCHING_INTERFACE
+void*
+get_drmarker_hotp_policy_status_table()
+{
+    ASSERT_NOT_IMPLEMENTED(false);
+    return NULL;
+}
+
+void
+set_drmarker_hotp_policy_status_table(void *new_table)
+{
+    ASSERT_NOT_IMPLEMENTED(false);
+}
+
+byte *
+hook_text(byte *hook_code_buf, const app_pc image_addr, 
+          intercept_function_t hook_func, const void *callee_arg,
+          const after_intercept_action_t action_after,
+          const bool abort_if_hooked, const bool ignore_cti,
+          byte **app_code_copy_p, byte **alt_exit_cti_p)
+{
+    ASSERT_NOT_IMPLEMENTED(false);
+    return NULL;
+}
+
+void
+unhook_text(byte *hook_code_buf, app_pc image_addr)
+{
+    ASSERT_NOT_IMPLEMENTED(false);
+}
+
+void
+insert_jmp_at_tramp_entry(byte *trampoline, byte *target)
+{
+    ASSERT_NOT_IMPLEMENTED(false);
+}
+#endif  /* HOT_PATCHING_INTERFACE */
+#endif /* RCT_IND_BRANCH */
+
+bool
+aslr_is_possible_attack(app_pc target)
+{
+    /* FIXME: ASLR not implemented */
+    return false;
+}
+
+app_pc
+aslr_possible_preferred_address(app_pc target_addr)
+{
+    /* FIXME: ASLR not implemented */
+    return NULL;
+}
+
+void
+take_over_primary_thread()
+{
+    /* nothing to do here */
+}
+
+bool
+os_current_user_directory(char *directory_prefix /* INOUT */,
+                          uint directory_len,
+                          bool create)
+{
+#if 0
+    /* FIXME: case 9922 */
+    ASSERT_NOT_IMPLEMENTED(true);
+#endif
+    return false;
+}
+
+bool
+os_validate_user_owned(file_t file_or_directory_handle)
+{
+    /* note on Linux this scheme should never be used */
+    ASSERT(false && "chown Alice evilfile");
+    return false;
+}
+
+bool
+os_check_option_compatibility(void)
+{
+    /* no options are Linux OS version dependent */
+    return false;
+}
diff --git a/core/linux/os_exports.h b/core/linux/os_exports.h
new file mode 100644
index 000000000..87fbfb5da
--- /dev/null
+++ b/core/linux/os_exports.h
@@ -0,0 +1,231 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * os_exports.h - Linux specific exported declarations
+ */
+
+#ifndef _OS_EXPORTS_H_
+#define _OS_EXPORTS_H_ 1
+
+#include <stdarg.h>
+#include "os_shared.h"
+
+#define getpid getpid_forbidden_use_get_process_id
+
+#define DYNAMORIO_LIBRARY_NAME "libdynamorio.so"
+#define DYNAMORIO_PRELOAD_NAME "libdrpreload.so"
+
+/* FIXME: I made these up */
+#define OS_ALLOC_GRANULARITY     (ASSERT_NOT_IMPLEMENTED(false), 4*1024)
+#define MAP_FILE_VIEW_ALIGNMENT  (ASSERT_NOT_IMPLEMENTED(false), 4*1024)
+
+/* We steal a segment register, and so use fs for x86 (where pthreads
+ * uses gs) and gs for x64 (where pthreads uses fs) (presumably to
+ * avoid conflicts w/ wine).
+ * Keep this consistent w/ the TLS_SEG_OPCODE define in x86/instr.h
+ * and TLS_SEG in x86/asm_defines.asm
+ *
+ * PR 205276 covers transparently stealing our segment selector.
+ */
+#ifdef X64
+# define SEG_TLS SEG_GS
+# define ASM_SEG "%gs"
+#else
+# define SEG_TLS SEG_FS
+# define ASM_SEG "%fs"
+#endif
+
+void *get_tls(ushort tls_offs);
+void set_tls(ushort tls_offs, void *value);
+
+/* in os.c */
+void os_fork_init(dcontext_t *dcontext);
+void os_thread_stack_store(dcontext_t *dcontext);
+app_pc get_dynamorio_dll_end(void);
+thread_id_t get_tls_thread_id(void);
+
+/* We do NOT want our libc routines wrapped by pthreads, so we use
+ * our own syscall wrappers.
+ */
+int open_syscall(const char *file, int flags, int mode);
+int close_syscall(int fd);
+int dup_syscall(int fd);
+ssize_t read_syscall(int fd, void *buf, size_t nbytes);
+ssize_t write_syscall(int fd, const void *buf, size_t nbytes);
+void exit_syscall(long status);
+process_id_t get_parent_id(void);
+
+/* to avoid pthreads problems we must have our own vnsprintf */
+#include <stdarg.h> /* for va_list */
+int our_snprintf(char *s, size_t max, const char *fmt, ...);
+int our_vsnprintf(char *s, size_t max, const char *fmt, va_list ap);
+#define snprintf our_snprintf
+#define vsnprintf our_vsnprintf
+
+/* to avoid unsetenv problems we have our own unsetenv */
+#define unsetenv our_unsetenv
+int our_unsetenv(const char *name);
+
+
+/* new segment support
+ * name is a string
+ * wx should contain one of the strings "w", "wx", "x", or ""
+ */
+/* FIXME: also want control over where in rw region or ro region this
+ * section goes -- for cl, order linked seems to do it, but for linux 
+ * will need a linker script (see linux/os.c for the nspdata problem)
+ */
+#define DECLARE_DATA_SECTION(name, wx) \
+     asm(".section "name", \"a"wx"\", @progbits"); \
+     asm(".align 0x1000");
+
+#define END_DATA_SECTION_DECLARATIONS() \
+     asm(".section .data"); \
+     asm(".align 0x1000");
+
+/* the VAR_IN_SECTION macro change where each var goes */
+#define START_DATA_SECTION(name, wx) /* nothing */
+#define END_DATA_SECTION() /* nothing */
+
+/* Any assignment, even to 0, puts vars in current .data and not .bss for cl,
+ * but for gcc we need to explicitly declare which section.  We still need
+ * the .section asm above to give section attributes and alignment.
+ */
+#define VAR_IN_SECTION(name) __attribute__ ((section (name)))
+
+/* location of vsyscall "vdso" page */
+extern app_pc vsyscall_page_start;
+/* pc of the end of the syscall instr itself */
+extern app_pc vsyscall_syscall_end_pc;
+/* pc where kernel returns control after sysenter vsyscall */
+extern app_pc vsyscall_sysenter_return_pc;
+#define VSYSCALL_PAGE_MAPS_NAME "[vdso]"
+
+/* We need a place to store the continuation pc for the child thread.
+ * We pick a register not used for SYS_clone parameters.
+ * FIXME PR 286194: currently we do not restore this in the child at all: better to
+ * keep a stack in parent, or use CLONE_SETTLS (PR 285898)?
+ */
+#define CLONE_SCRATCH_REG_MC  xbp
+#define CLONE_SCRATCH_REG_REG REG_XBP
+
+bool is_clone_thread_syscall(dcontext_t *dcontext);
+bool was_clone_thread_syscall(dcontext_t *dcontext);
+bool is_sigreturn_syscall(dcontext_t *dcontext);
+bool was_sigreturn_syscall(dcontext_t *dcontext);
+bool ignorable_system_call(int num);
+
+bool kernel_is_64bit(void);
+
+/* in arch.c */
+bool unhook_vsyscall(void);
+
+/***************************************************************************/
+/* in signal.c */
+
+/* defines and typedefs exported for dr_jmp_buf_t */
+
+#define NUM_NONRT   32 /* include 0 to make offsets simple */
+#define NUM_RT      32
+#define OFFS_RT     32
+/* FIXME: PR 362835: actually 64, not 63, is the highest valid signum */
+#define MAX_SIGNUM  ((OFFS_RT) + (NUM_RT))
+
+/* size of long */
+#ifdef X64
+# define _NSIG_BPW 64
+#else
+# define _NSIG_BPW 32
+#endif
+
+#define _NSIG_WORDS (MAX_SIGNUM / _NSIG_BPW)
+
+/* kernel's sigset_t packs info into bits, while glibc's uses a short for
+ * each (-> 8 bytes vs. 128 bytes)
+ */
+typedef struct _kernel_sigset_t {
+    unsigned long sig[_NSIG_WORDS];
+} kernel_sigset_t;
+
+void receive_pending_signal(dcontext_t *dcontext);
+void start_itimer(dcontext_t *dcontext);
+void stop_itimer(void);
+void start_PAPI_timer(void);
+void stop_PAPI_timer(void);
+bool is_signal_restorer_code(byte *pc, size_t *len);
+
+#define CONTEXT_HEAP_SIZE(sc) (sizeof(sc))
+#define CONTEXT_HEAP_SIZE_OPAQUE (CONTEXT_HEAP_SIZE(struct sigcontext))
+
+/* struct sigcontext field name changes */
+#ifdef X64
+# define SC_XIP rip
+# define SC_XAX rax
+# define SC_XCX rcx
+# define SC_XDX rdx
+# define SC_XBX rbx
+# define SC_XSP rsp
+# define SC_XBP rbp
+# define SC_XSI rsi
+# define SC_XDI rdi
+# define SC_XFLAGS eflags
+#else
+# define SC_XIP eip
+# define SC_XAX eax
+# define SC_XCX ecx
+# define SC_XDX edx
+# define SC_XBX ebx
+# define SC_XSP esp
+# define SC_XBP ebp
+# define SC_XSI esi
+# define SC_XDI edi
+# define SC_XFLAGS eflags
+#endif
+
+/***************************************************************************/
+
+/* in pcprofile.c */
+void pcprofile_fragment_deleted(dcontext_t *dcontext, fragment_t *f);
+void pcprofile_thread_exit(dcontext_t *dcontext);
+
+/* in stackdump.c */
+/* fork, dump core, and use gdb for complete stack trace */
+void stackdump(void);
+/* use backtrace feature of glibc for quick but sometimes incomplete trace */
+void glibc_stackdump(int fd);
+
+#endif /* _OS_EXPORTS_H_ */
diff --git a/core/linux/os_private.h b/core/linux/os_private.h
new file mode 100644
index 000000000..c9d26e18d
--- /dev/null
+++ b/core/linux/os_private.h
@@ -0,0 +1,150 @@
+/* **********************************************************
+ * Copyright (c) 2008-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * os_private.h - declarations shared among os-specific files, but not
+ * exported to the rest of the code
+ */
+
+#ifndef _OS_PRIVATE_H_
+#define _OS_PRIVATE_H_ 1
+
+#include <signal.h> /* for stack_t */
+
+/* for inline asm */
+#ifdef X64
+# define ASM_XAX "rax"
+# define ASM_XDX "rdx"
+# define ASM_XBP "rbp"
+# define ASM_XSP "rsp"
+#else
+# define ASM_XAX "eax"
+# define ASM_XDX "edx"
+# define ASM_XBP "ebp"
+# define ASM_XSP "esp"
+#endif
+
+/* PR 212090: the signal we use to suspend threads */
+#define SUSPEND_SIGNAL SIGUSR2
+
+/* thread-local data that's os-private, for modularity */
+typedef struct _os_thread_data_t {
+    /* store stack info at thread startup, since stack can get fragmented in
+     * /proc/self/maps w/ later mprotects and it can be hard to piece together later
+     */
+    app_pc stack_base;
+    app_pc stack_top;
+
+#ifdef RETURN_AFTER_CALL
+    app_pc stack_bottom_pc;     /* return target in the loader at program startup */
+#endif
+
+    /* PR 2120990: for thread suspension */
+    /* This lock synchronizes suspension and resumption and controls
+     * access to suspend_count and the bools below in thread_suspend
+     * and suspend_resume.  handle_suspend_signal() does not use the
+     * mutex as it is not safe to do so, but our suspend and resume
+     * synch avoids any need for it there.
+     */
+    mutex_t suspend_lock;
+    int suspend_count;
+    /* We would use event_t's here except we can't use mutexes in
+     * our signal handler 
+     */
+    bool suspended;
+    bool wakeup;
+    bool resumed;
+    struct sigcontext *suspended_sigcxt;
+} os_thread_data_t;
+
+/* in signal.c */
+struct _kernel_sigaction_t;
+typedef struct _kernel_sigaction_t kernel_sigaction_t;
+
+void signal_init(void);
+void signal_exit(void);
+void signal_thread_init(dcontext_t *dcontext);
+void signal_thread_exit(dcontext_t *dcontext);
+void handle_clone(dcontext_t *dcontext, uint flags);
+bool handle_sigaction(dcontext_t *dcontext, int sig,
+                      const kernel_sigaction_t *act, 
+                      kernel_sigaction_t *oact, size_t sigsetsize);
+void handle_post_sigaction(dcontext_t *dcontext, int sig,
+                           const kernel_sigaction_t *act, 
+                           kernel_sigaction_t *oact, size_t sigsetsize);
+void handle_sigreturn(dcontext_t *dcontext, bool rt);
+bool handle_sigaltstack(dcontext_t *dcontext, const stack_t *stack,
+                        stack_t *old_stack);
+void handle_sigprocmask(dcontext_t *dcontext, int how, kernel_sigset_t *set,
+                        kernel_sigset_t *oset, size_t sigsetsize);
+void handle_post_sigprocmask(dcontext_t *dcontext, int how, kernel_sigset_t *set,
+                             kernel_sigset_t *oset, size_t sigsetsize);
+void handle_sigsuspend(dcontext_t *dcontext, kernel_sigset_t *set,
+                       size_t sigsetsize);
+void
+sigcontext_to_mcontext(dr_mcontext_t *mc, struct sigcontext *sc);
+
+void
+mcontext_to_sigcontext(struct sigcontext *sc, dr_mcontext_t *mc);
+
+bool
+set_default_signal_action(int sig);
+
+/* in pcprofile.c */
+void pcprofile_thread_init(dcontext_t *dcontext);
+void pcprofile_fork_init(dcontext_t *dcontext);
+
+/* in module.c */
+bool is_elf_so_header(app_pc base, size_t size);
+bool module_walk_program_headers(app_pc base, size_t view_size, bool at_map,
+                                 app_pc *out_base, app_pc *out_end, char **out_soname,
+                                 size_t *out_align);
+
+uint module_num_program_headers(app_pc base);
+
+app_pc module_vaddr_from_prog_header(app_pc prog_header, uint num_segments,
+                                     OUT app_pc *mod_end);
+
+bool module_read_program_header(app_pc base,
+                                uint segment_num,
+                                OUT app_pc *segment_base,
+                                OUT app_pc *segment_end,
+                                OUT uint *segment_prot,
+                                OUT size_t *segment_align);
+
+void os_request_live_coredump(const char *msg);
+
+#ifdef VMX86_SERVER
+#  include "vmkuw.h"
+#endif
+
+#endif /* _OS_PRIVATE_H_ */
diff --git a/core/linux/pcprofile.c b/core/linux/pcprofile.c
new file mode 100644
index 000000000..97a14bc75
--- /dev/null
+++ b/core/linux/pcprofile.c
@@ -0,0 +1,539 @@
+/* **********************************************************
+ * Copyright (c) 2001-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2001 Hewlett-Packard Company */
+
+/*
+ * pcprofile.c - pc sampling profiler for dynamo
+ */
+
+#include "globals.h"
+#include "utils.h"
+#include "fragment.h"
+#include "fcache.h"
+#include <string.h> /* for memset */
+
+/* Don't use symtab, it doesn't give us anything that addr2line or
+ * other post-execution tools can't (it doesn't see into shared libraries),
+ * plus will want a script to process the raw data we output anyway!
+ * If you put symtab back in, add dynamo option "-profexecname"
+ */
+#define USE_SYMTAB 0
+#if USE_SYMTAB
+#include "symtab.h"
+static bool valid_symtab;
+#endif
+
+/* Profiling via pc sampling
+ * We store the pc for dynamo and app pc's, for fragments we store the
+ * tag and offset
+ * In order to handle tags that aren't original program pc's we can't
+ * use tag+offset as the hashtable key -- so we use fcache pc as the
+ * key.  To handle deleted fragments and later fragments re-using the same
+ * fcache pcs, we have a "retired" flag that we set when a fragment is
+ * deleted.  Thus we can end up printing out a separate count for
+ * the same pc.
+ */
+typedef struct _pc_profile_entry_t {
+    void *                  pc;      /* the pc */
+    where_am_i_t          whereami;      /* location of pc */
+#ifdef DEBUG
+    int                     id;      /* if in fragment, id */
+#endif
+    app_pc                 tag;      /* if in fragment, tag */
+    uint                offset;      /* if in fragment, offset from start pc */
+    bool                 trace;      /* if in fragment, is it a trace? */
+    int                counter;      /* execution counter */
+    bool               retired;      /* owning fragment was deleted */
+    struct _pc_profile_entry_t *next;  /* for chaining entries */
+} pc_profile_entry_t;
+
+#define HASH_BITS 14
+
+/* The timer and all data are per-thread */
+typedef struct _thread_pc_info_t {
+    pc_profile_entry_t **htable; /* HASH_BITS-bit addressed hash table, key is pc */
+    void *special_heap;
+    file_t file;
+    int where[WHERE_LAST];
+} thread_pc_info_t;
+
+/* forward declarations for static functions */
+static pc_profile_entry_t *pcprofile_add_entry(thread_pc_info_t *info, void *pc, int whereami);
+static pc_profile_entry_t *pcprofile_lookup(thread_pc_info_t *info, void *pc);
+static void pcprofile_reset(thread_pc_info_t *info);
+static void pcprofile_results(thread_pc_info_t *info);
+
+/* initialization */
+void
+pcprofile_thread_init(dcontext_t *dcontext)
+{
+    int i;
+    int size = HASHTABLE_SIZE(HASH_BITS) * sizeof(pc_profile_entry_t*);
+
+    thread_pc_info_t *info = heap_alloc(dcontext, sizeof(thread_pc_info_t)
+                                    HEAPACCT(ACCT_OTHER));
+    dcontext->pcprofile_field = (void *) info;
+    memset(info, 0, sizeof(thread_pc_info_t));
+
+    info->htable = (pc_profile_entry_t**) heap_alloc(dcontext, size
+                                                 HEAPACCT(ACCT_OTHER));
+    memset(info->htable, 0, size);
+#if USE_SYMTAB
+    valid_symtab = symtab_init();
+#endif
+    for (i = 0; i < WHERE_LAST; i++)
+        info->where[i] = 0;
+    info->file = open_log_file("pcsamples", NULL, 0);
+    info->special_heap = special_heap_init(sizeof(pc_profile_entry_t), false /* no locks */,
+                                           false /* -x */, true /* persistent */);
+
+    start_itimer(dcontext);
+}
+
+/* cleanup */
+void
+pcprofile_thread_exit(dcontext_t *dcontext)
+{
+    int size;
+    thread_pc_info_t *info = (thread_pc_info_t *) dcontext->pcprofile_field;
+
+    /* don't want any alarms while holding lock for printing results
+     * (see notes under pcprofile_cache_flush below)
+     */
+    stop_itimer();
+
+    pcprofile_results(info);
+    size = HASHTABLE_SIZE(HASH_BITS) * sizeof(pc_profile_entry_t*);
+    pcprofile_reset(info); /* special heap so no fast path */
+#ifdef DEBUG
+    /* for non-debug we do fast exit path and don't free local heap */
+    heap_free(dcontext, info->htable, size HEAPACCT(ACCT_OTHER));
+#endif
+#if USE_SYMTAB
+    if (valid_symtab)
+        symtab_exit();
+#endif
+    os_close(info->file);
+    special_heap_exit(info->special_heap);
+#ifdef DEBUG
+    /* for non-debug we do fast exit path and don't free local heap */
+    heap_free(dcontext, info, sizeof(thread_pc_info_t) HEAPACCT(ACCT_OTHER));
+#endif
+}
+
+void
+pcprofile_fork_init(dcontext_t *dcontext)
+{
+    /* FIXME: hmmm...I guess a forked child will just start from scratch?
+     */
+    thread_pc_info_t *info = (thread_pc_info_t *) dcontext->pcprofile_field;
+    pcprofile_reset(info);
+    info->file = open_log_file("pcsamples", NULL, 0);
+}
+
+#if 0
+static void
+pcprof_dump_callstack(dcontext_t *dcontext, app_pc cur_pc, app_pc ebp, file_t outfile)
+{
+    uint *pc = (uint *) ebp;
+    int num = 0;
+    if (cur_pc != NULL) {
+        print_file(outfile, "\tcurrent pc = "PFX"\n", cur_pc);
+    }
+    /* only DR routines 
+     * cannot call is_readable b/c that asks for memory -> livelock
+     */
+    while (pc != NULL && (pc >= dcontext->dstack - DYNAMORIO_STACK_SIZE) &&
+           pc < dcontext->dstack) {
+        print_file(outfile, 
+            "\tframe ptr "PFX" => parent "PFX", ret = "PFX"\n", pc, *pc, *(pc+1));
+        num++;
+        /* yes I've seen weird recursive cases before */
+        if (pc == (uint *) *pc || num > 100)
+            break;
+        pc = (uint *) *pc;
+    }
+}
+#endif
+
+/* Handle a pc sample
+ *
+ * WARNING: this function could interrupt any part of dynamo!
+ * Make sure nothing is done that could cause deadlock or
+ * data structure mishaps.
+ * Right now interrupting heap_alloc or interrupting pcprofile_results are
+ * the only bad things that could happen, both are dealt with.
+ */
+void
+pcprofile_alarm(dcontext_t *dcontext, void *pc, app_pc ebp)
+{               
+    thread_pc_info_t *info = (thread_pc_info_t *) dcontext->pcprofile_field;
+    pc_profile_entry_t *entry;
+    fragment_t *fragment;
+    fragment_t wrapper;
+
+    entry = pcprofile_lookup(info, pc);
+
+#if 0
+    print_file(info->file, "\nalarm "PFX"\n", pc);
+    pcprof_dump_callstack(dcontext, pc, ebp, info->file);
+#endif
+
+    if (entry != NULL) {
+        entry->counter++;
+    } else {
+        /* the process is suspended, so we don't care about data races, but
+         * we can't have anyone else holding locks we need.
+         * special_heap routines do not use any locks.
+         */
+        entry = pcprofile_add_entry(info, pc, dcontext->whereami);
+        /* if in a fragment, get fragment tag & offset now */
+        if (entry->whereami == WHERE_FCACHE) {
+            fragment = fragment_pclookup(dcontext, pc, &wrapper);
+            if (fragment == NULL) {
+                /* FIXME: should check fcache_unit_areas.lock before
+                 * calling (case 1317) and at least assert on it, if counting
+                 * on whereami to ensure no DR locks are held
+                 */
+                if (in_fcache(pc)) {
+                    entry->whereami = WHERE_UNKNOWN;
+                } else {
+                    /* identify parts of our assembly code now.
+                     * it's all generated and post-process can't identify.
+                     * assume code order is as follows:
+                     */
+                    if (in_context_switch_code(dcontext, (cache_pc)pc)) {
+                        entry->whereami = WHERE_CONTEXT_SWITCH;
+                    } else if (in_indirect_branch_lookup_code(dcontext,
+                                                              (cache_pc)pc)) {
+                        entry->whereami = WHERE_IBL;
+                    } else {
+                        entry->whereami = WHERE_UNKNOWN;
+                    }
+                }
+            } else {
+#ifdef DEBUG
+                entry->id = fragment->id;
+#endif
+                entry->tag = fragment->tag;
+                ASSERT(CHECK_TRUNCATE_TYPE_int((byte *)pc - fragment->start_pc));
+                entry->offset = (int) ((byte *)pc - fragment->start_pc);
+                entry->trace = (fragment->flags & FRAG_IS_TRACE) != 0;
+            }
+        }
+    }
+
+    /* update whereami counters */
+    info->where[entry->whereami]++;
+}
+
+/* create a new, initialized profile pc entry */
+static pc_profile_entry_t *
+pcprofile_add_entry(thread_pc_info_t *info, void *pc, int whereami)
+{
+    uint hindex;
+    pc_profile_entry_t *e;
+
+    /* special_heap is hardcoded for sizeof(pc_profile_entry_t) */
+    e = (pc_profile_entry_t*) special_heap_alloc(info->special_heap);
+    e->pc = pc;
+    e->counter = 1;
+    e->whereami = whereami;
+    e->next = NULL;
+#if DEBUG
+    e->id = 0;
+#endif
+    e->tag = 0;
+    e->offset = 0;
+    e->trace = false;
+    e->retired = false;
+
+    /* add e to the htable */
+    hindex = HASH_FUNC_BITS((ptr_uint_t)pc, HASH_BITS);
+    e->next = info->htable[hindex];
+    info->htable[hindex] = e;
+
+    return e;
+}
+
+/* Lookup an entry by pc and return a pointer to the corresponding entry 
+ * Returns NULL if no such entry exists */
+static pc_profile_entry_t *
+pcprofile_lookup(thread_pc_info_t *info, void *pc)
+{
+    uint hindex;
+    pc_profile_entry_t *e;
+    /* find the entry corresponding to tag */
+    hindex = HASH_FUNC_BITS((ptr_uint_t)pc, HASH_BITS);
+    for (e = info->htable[hindex]; e; e = e->next) {
+        if (e->pc == pc && !e->retired)
+            return e;
+    }
+    return NULL;
+}
+
+/* When a fragment is deleted we "retire" all its entries
+ * Thus we end up with multiple entires with the same pc
+ */
+void
+pcprofile_fragment_deleted(dcontext_t *dcontext, fragment_t *f)
+{
+    thread_pc_info_t *info;
+    int i;
+    pc_profile_entry_t *e;
+    if (dcontext == GLOBAL_DCONTEXT) {
+        dcontext = get_thread_private_dcontext();
+    }
+    if (dcontext == NULL) {
+        ASSERT(dynamo_exited);
+        return;
+    }
+    info = (thread_pc_info_t *) dcontext->pcprofile_field;
+    for (i = 0; i < HASHTABLE_SIZE(HASH_BITS); i++) {
+        for (e = info->htable[i]; e; e = e->next) {
+            if (e->tag == f->tag) {
+                e->retired = true;
+            }
+        }
+    }
+}
+
+#if 0 /* not used */
+/* delete the trace head entry corresponding to tag if it exists */
+static void
+pcprofile_destroy_entry(thread_pc_info_t *info, void *pc)
+{
+    uint hindex;
+    pc_profile_entry_t *e, *prev_e = NULL;
+
+    /* find the trace head entry corresponding to tag and delete it */
+    hindex = HASH_FUNC_BITS((ptr_uint_t)pc, HASH_BITS);
+    for (e = info->htable[hindex]; e; prev_e = e, e = e->next) {
+        if (e->pc == pc) {
+            if (prev_e)
+                prev_e->next = e->next;
+            else
+                info->htable[hindex] = e->next;
+            break;
+        }
+    }
+    if (e)
+        special_heap_free(info->special_heap, e);
+}
+#endif
+
+/* reset profile structures */
+static void
+pcprofile_reset(thread_pc_info_t *info)
+{
+    int i;
+    for (i = 0; i < HASHTABLE_SIZE(HASH_BITS); i++) {
+        pc_profile_entry_t *e = info->htable[i];
+        while (e) {
+            pc_profile_entry_t *nexte = e->next;
+            special_heap_free(info->special_heap, e);
+            e = nexte;
+        }
+        info->htable[i] = NULL;
+    }
+    for (i = 0; i < WHERE_LAST; i++)
+        info->where[i] = 0;
+}
+
+/* Print the profile results 
+ * FIXME: It would be nice to print counts integrated with fragment listings
+ * That would require re-ordering *_exit() sequence (fragments are deleted first)
+ * Instead of doing that, can use a script to combine these tag+offsets with
+ * previously printed fragments
+ * FIXME: this routine uses floating point ops, if ever called not at thread exit
+ * must preserve fp state around the whole routine!
+ */
+static void
+pcprofile_results(thread_pc_info_t *info)
+{
+    int i, total = 0;
+    pc_profile_entry_t *e;
+
+    for (i = 0; i < WHERE_LAST; i++) 
+        total += info->where[i];
+    
+    print_file(info->file, "DynamoRIO library base: "PFX"\n",
+               get_dynamorio_dll_start());
+    print_file(info->file, "ITIMER distribution (%d):\n", total);
+    if (info->where[WHERE_APP] > 0) {
+        print_file(info->file, "  %5.1f%% of time in APPLICATION (%d)\n",
+                   (float)info->where[WHERE_APP]/(float)total * 100.0,
+                   info->where[WHERE_APP]);
+    }
+    if (info->where[WHERE_INTERP] > 0) {
+        print_file(info->file, "  %5.1f%% of time in INTERPRETER (%d)\n",
+                   (float)info->where[WHERE_INTERP]/(float)total * 100.0,
+                   info->where[WHERE_INTERP]);
+    }
+    if (info->where[WHERE_DISPATCH] > 0) {
+        print_file(info->file, "  %5.1f%% of time in DISPATCH (%d)\n",
+                   (float)info->where[WHERE_DISPATCH]/(float)total * 100.0,
+                   info->where[WHERE_DISPATCH]);
+    }
+    if (info->where[WHERE_MONITOR] > 0) {
+        print_file(info->file, "  %5.1f%% of time in MONITOR (%d)\n",
+                   (float)info->where[WHERE_MONITOR]/(float)total * 100.0,
+                   info->where[WHERE_MONITOR]);
+    }
+    if (info->where[WHERE_SYSCALL_HANDLER] > 0) {
+        print_file(info->file, "  %5.1f%% of time in SYSCALL HANDLER (%d)\n",
+                   (float)info->where[WHERE_SYSCALL_HANDLER]/(float)total * 100.0,
+                   info->where[WHERE_SYSCALL_HANDLER]);
+    }
+    if (info->where[WHERE_SIGNAL_HANDLER] > 0) {
+        print_file(info->file, "  %5.1f%% of time in SIGNAL HANDLER (%d)\n",
+                   (float)info->where[WHERE_SIGNAL_HANDLER]/(float)total * 100.0,
+                   info->where[WHERE_SIGNAL_HANDLER]);
+    }
+    if (info->where[WHERE_TRAMPOLINE] > 0) {
+        print_file(info->file, "  %5.1f%% of time in TRAMPOLINES (%d)\n",
+                   (float)info->where[WHERE_TRAMPOLINE]/(float)total * 100.0,
+                   info->where[WHERE_TRAMPOLINE]);
+    }
+    if (info->where[WHERE_CONTEXT_SWITCH] > 0) {
+        print_file(info->file, "  %5.1f%% of time in CONTEXT SWITCH (%d)\n",
+                   (float)info->where[WHERE_CONTEXT_SWITCH]/(float)total * 100.0,
+                   info->where[WHERE_CONTEXT_SWITCH]);
+    }
+    if (info->where[WHERE_IBL] > 0) {
+        print_file(info->file, "  %5.1f%% of time in INDIRECT BRANCH LOOKUP (%d)\n",
+                   (float)info->where[WHERE_IBL]/(float)total * 100.0,
+                   info->where[WHERE_IBL]);
+    }
+    if (info->where[WHERE_FCACHE] > 0) {
+        print_file(info->file, "  %5.1f%% of time in FRAGMENT CACHE (%d)\n",
+                   (float)info->where[WHERE_FCACHE]/(float)total * 100.0,
+                   info->where[WHERE_FCACHE]);
+    }
+    if (info->where[WHERE_UNKNOWN] > 0) {
+        print_file(info->file, "  %5.1f%% of time in UNKNOWN (%d)\n",
+                   (float)info->where[WHERE_UNKNOWN]/(float)total * 100.0,
+                   info->where[WHERE_UNKNOWN]);
+    }
+
+    print_file(info->file, "\nPC PROFILING RESULTS\n");
+
+    for (i = 0; i < HASHTABLE_SIZE(HASH_BITS); i++) {
+        e = info->htable[i];
+        while (e) {
+            if (e->whereami == WHERE_FCACHE) {
+                const char *type;
+                if (e->trace)
+                    type = "trace";
+                else
+                    type = "fragment";
+                print_file(info->file,
+#ifdef DEBUG
+                        "pc="PFX"\t#=%d\tin %s #%6d @"PFX" w/ offs "PFX"\n",
+                        e->pc, e->counter, type, e->id, e->tag, e->offset);
+#else
+                        "pc="PFX"\t#=%d\tin %s @"PFX" w/ offs "PFX"\n",
+                        e->pc, e->counter, type, e->tag, e->offset);
+#endif
+#if USE_SYMTAB
+                /* FIXME: this only works for fragments whose tags are app pc's! */
+                if (valid_symtab) {
+                    print_file(info->file,
+                            "\tin app = %s\n",
+                            symtab_lookup_pc((void *)(e->tag+e->offset)));
+                }
+#endif
+            } else if (e->whereami == WHERE_APP) {
+#if USE_SYMTAB
+                if (valid_symtab) {
+                    print_file(info->file, "pc="PFX"\t#=%d\tin the app = %s\n",
+                               e->pc, e->counter, symtab_lookup_pc(e->pc));
+                } else {
+#else
+                    print_file(info->file, "pc="PFX"\t#=%d\tin the app\n",
+                               e->pc, e->counter);
+#endif
+#if USE_SYMTAB
+                }
+#endif
+            } else if (e->whereami == WHERE_UNKNOWN) {
+                if (is_dynamo_address(e->pc)) {
+                    print_file(info->file, "pc="PFX"\t#=%d\tin DynamoRIO <SOMEWHERE>\n",
+                               e->pc, e->counter);
+                } else {
+                    char *comment = NULL;
+                    DODEBUG({ comment = get_address_comment(e->pc); });
+                    print_file(info->file, "pc="PFX"\t#=%d\tin uncategorized: %s\n",
+                               e->pc, e->counter,
+                               (comment==NULL)?"<UNKNOWN>":comment);
+                }
+            } else {
+#if USE_SYMTAB
+                if (valid_symtab) {
+                    print_file(info->file,
+                               "pc="PFX"\t#=%d\tin DynamoRIO = %s\n",
+                               e->pc, e->counter, symtab_lookup_pc(e->pc));
+                } else {
+#else
+                    print_file(info->file, "pc="PFX"\t#=%d\tin DynamoRIO",
+                               e->pc, e->counter);
+                    if (e->whereami == WHERE_INTERP) {
+                        print_file(info->file, " interpreter\n");
+                    } else if (e->whereami == WHERE_DISPATCH) {
+                        print_file(info->file, " dispatch\n");
+                    } else if (e->whereami == WHERE_MONITOR) {
+                        print_file(info->file, " monitor\n");
+                    } else if (e->whereami == WHERE_SIGNAL_HANDLER) {
+                        print_file(info->file, " signal handler\n");
+                    } else if (e->whereami == WHERE_SYSCALL_HANDLER) {
+                        print_file(info->file, " syscall handler\n");
+                    } else if (e->whereami == WHERE_CONTEXT_SWITCH) {
+                        print_file(info->file, " context switch\n");
+                    } else if (e->whereami == WHERE_IBL) {
+                        print_file(info->file, " indirect_branch_lookup\n");
+                    } else {
+                        print_file(STDERR, "ERROR: unknown whereAmI %d\n",
+                                e->whereami);
+                        ASSERT_NOT_REACHED();
+                    }
+#endif
+#if USE_SYMTAB
+                }
+#endif
+            }
+            e = e->next;
+        }
+    }    
+}
diff --git a/core/linux/preload.c b/core/linux/preload.c
new file mode 100644
index 000000000..85fc17b15
--- /dev/null
+++ b/core/linux/preload.c
@@ -0,0 +1,165 @@
+/* **********************************************************
+ * Copyright (c) 2001-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+
+/* preload.c - preload library,
+ *    used to launch dynamo on legacy binaries
+ *
+ * Note: preload is called last, so all threads started by other
+ *    libraries will NOT be caught, which is essentially the same
+ *    behavior as calling dynamorio_app_init() in main()
+ */
+
+#define START_DYNAMO 1          /* start dynamo on preload */
+#define VERBOSE_INIT_FINI 0     /* notification for _init and _fini  */
+#define VERBOSE 0
+#define INIT_BEFORE_LIBC 0
+
+#include <stdio.h>
+/* for open */
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+/* for getpid */
+#include <unistd.h>
+/* for rindex */
+#include <string.h>
+
+#if VERBOSE
+#define pf(fmt, args...) printf(fmt , ## args)
+#else 
+#define pf(...) /* C99 requires ... to mean at least one argument */
+#endif /* VERBOSE */
+
+#if START_DYNAMO
+void dynamorio_app_take_over(void);
+int dynamorio_app_init(void);
+#endif /* START_DYNAMO */
+
+#define MAX_COMMAND_LENGTH 1024
+
+/* Give a global variable definition. */
+int nothing = 0;
+
+/* this routine is straight from utils.c 
+ * except it uses libc routines open, read, and close, to avoid
+ * having to do syscall wrappers here, and since usage is prior
+ * to DR taking over and so we don't care about pthreads interference
+ */
+static void 
+getnamefrompid(int pid, char *name, uint maxlen)
+{
+    int fd, n;
+    char tempstring[200+1],*lastpart;
+    /*this is a shitty way of getting the process name,
+    but i can't think of anything better... */
+
+    snprintf(tempstring, 200+1, "/proc/%d/cmdline", pid);
+    fd = open(tempstring, O_RDONLY);
+    n = read(fd, tempstring, 200);
+    tempstring[n] = '\0';
+    lastpart = rindex(tempstring, '/');
+
+    if (lastpart == NULL)
+      lastpart = tempstring;
+    else
+      lastpart++; /*don't include last '/' in name*/ 
+
+    strncpy(name, lastpart, maxlen-1);
+    name[maxlen-1]  = '\0'; /* if max no null */
+
+    close(fd);
+}
+
+int
+#if INIT_BEFORE_LIBC
+_init(int argc, char *arg0, ...)
+{
+  char **argv = &arg0, **envp = &argv[argc + 1];
+#else
+_init ()
+{
+#endif
+    int init;
+    char name[MAX_COMMAND_LENGTH];
+#if VERBOSE_INIT_FINI
+    fprintf (stderr, "preload initialized\n");
+#endif /* VERBOSE_INIT_FINI */
+
+#if INIT_BEFORE_LIBC
+  {
+      int i;
+      for (i=0; i<argc; i++)
+          fprintf(stderr, "\targ %d = %s\n", i, argv[i]);
+      fprintf(stderr, "env 0 is %s\n", envp[0]);
+      fprintf(stderr, "env 1 is %s\n", envp[1]);
+      fprintf(stderr, "env 2 is %s\n", envp[2]);
+  }
+#endif
+
+#if START_DYNAMO
+    pf("ready to start dynamo\n");
+    getnamefrompid(getpid(), name, MAX_COMMAND_LENGTH);
+    pf("preload _init: running %s\n", name);
+# ifdef INTERNAL
+    /* HACK just for our benchmark scripts: 
+     * do not take over a process whose executable is named "texec"
+     */
+    if (strcmp(name, "texec") == 0) {
+        pf("running texec, NOT taking over!\n");
+        return 0;
+    }
+# endif
+    init = dynamorio_app_init();
+    pf("dynamorio_app_init() returned %d\n", init);
+    dynamorio_app_take_over();
+    pf("dynamo started\n");
+#endif /* START_DYNAMO */
+
+    return 0;
+}
+
+int
+_fini ()
+{
+#if VERBOSE_INIT_FINI
+    fprintf (stderr, "preload finalized\n");
+#endif /* VERBOSE_INIT_FINI */
+
+    /* since we're using dynamorio_app_take_over we do not need to call dr_app_stop
+     * or dynamorio_app_exit
+     */
+
+    return 0;
+}
diff --git a/core/linux/signal.c b/core/linux/signal.c
new file mode 100644
index 000000000..d63fe36fa
--- /dev/null
+++ b/core/linux/signal.c
@@ -0,0 +1,3843 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * signal.c - dynamorio signal handler
+ */
+
+#include <unistd.h>
+
+/* For FC8:
+ * We do not want bits/sigcontext.h from signal.h, since we want X86_FXSR_MAGIC
+ * from asm/sigcontext.h, which duplicates many struct defs from bits/sigcontext.h
+ */
+#define _BITS_SIGCONTEXT_H  1
+#include <signal.h>
+#include <asm/sigcontext.h> /* for X86_FXSR_MAGIC */
+
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <ucontext.h>
+#include <sys/syscall.h>
+#include <sched.h>
+#include <string.h> /* for memcpy and memset */
+#include "globals.h"
+#include "os_private.h"
+#include "fragment.h"
+#include "fcache.h"
+#include "perfctr.h"
+#include "arch.h"
+#include "monitor.h" /* for trace_abort */
+#include "link.h" /* for linking interrupted fragment_t */
+#include "instr.h" /* to find target of SIGSEGV */
+#include "decode.h" /* to find target of SIGSEGV */
+#include "decode_fast.h" /* to handle self-mod code */
+#include "synch.h"
+
+#ifdef CLIENT_INTERFACE
+# include "instrument.h"
+#endif
+
+#ifdef VMX86_SERVER
+# include <errno.h>
+#endif
+
+/* important reference files:
+ *   /usr/include/asm/signal.h
+ *   /usr/include/asm/siginfo.h
+ *   /usr/include/asm/ucontext.h
+ *   /usr/include/asm/sigcontext.h
+ *   /usr/include/sys/ucontext.h (see notes below...asm one is more useful)
+ *   /usr/include/bits/sigaction.h
+ *   /usr/src/linux/kernel/signal.c
+ *   /usr/src/linux/arch/i386/kernel/signal.c
+ *   /usr/src/linux/include/asm-i386/signal.h
+ *   /usr/src/linux/include/asm-i386/sigcontext.h
+ *   /usr/src/linux/include/asm-i386/ucontext.h
+ *   /usr/src/linux/include/linux/signal.h
+ *   /usr/src/linux/include/linux/sched.h (sas_ss_flags, on_sig_stack)
+ * ditto with x86_64, plus:
+ *   /usr/src/linux/arch/x86_64/ia32/ia32_signal.c
+ */
+
+
+/**** data structures ***************************************************/
+
+/* handler with SA_SIGINFO flag set gets three arguments: */
+typedef void (*handler_t)(int, struct siginfo *, void *);
+
+/* default actions */
+enum {
+    DEFAULT_TERMINATE,
+    DEFAULT_TERMINATE_CORE,
+    DEFAULT_IGNORE,
+    DEFAULT_STOP,
+    DEFAULT_CONTINUE,
+};
+
+int default_action[] = {
+    /* nothing    0 */   DEFAULT_IGNORE,
+    /* SIGHUP     1 */   DEFAULT_TERMINATE,
+    /* SIGINT     2 */   DEFAULT_TERMINATE,
+    /* SIGQUIT    3 */   DEFAULT_TERMINATE_CORE,
+    /* SIGILL     4 */   DEFAULT_TERMINATE_CORE,
+    /* SIGTRAP    5 */   DEFAULT_TERMINATE_CORE,
+    /* SIGABRT/SIGIOT 6 */   DEFAULT_TERMINATE_CORE,
+    /* SIGBUS     7 */   DEFAULT_TERMINATE, /* should be CORE */
+    /* SIGFPE     8 */   DEFAULT_TERMINATE_CORE,
+    /* SIGKILL    9 */   DEFAULT_TERMINATE,
+    /* SIGUSR1   10 */   DEFAULT_TERMINATE,
+    /* SIGSEGV   11 */   DEFAULT_TERMINATE_CORE,
+    /* SIGUSR2   12 */   DEFAULT_TERMINATE,
+    /* SIGPIPE   13 */   DEFAULT_TERMINATE,
+    /* SIGALRM   14 */   DEFAULT_TERMINATE,
+    /* SIGTERM   15 */   DEFAULT_TERMINATE,
+    /* SIGSTKFLT 16 */   DEFAULT_TERMINATE,
+    /* SIGCHLD   17 */   DEFAULT_IGNORE,
+    /* SIGCONT   18 */   DEFAULT_CONTINUE,
+    /* SIGSTOP   19 */   DEFAULT_STOP,
+    /* SIGTSTP   20 */   DEFAULT_STOP,
+    /* SIGTTIN   21 */   DEFAULT_STOP,
+    /* SIGTTOU   22 */   DEFAULT_STOP,
+    /* SIGURG    23 */   DEFAULT_IGNORE,
+    /* SIGXCPU   24 */   DEFAULT_TERMINATE,
+    /* SIGXFSZ   25 */   DEFAULT_TERMINATE,
+    /* SIGVTALRM 26 */   DEFAULT_TERMINATE,
+    /* SIGPROF   27 */   DEFAULT_TERMINATE,
+    /* SIGWINCH  28 */   DEFAULT_IGNORE,
+    /* SIGIO/SIGPOLL/SIGLOST 29 */ DEFAULT_TERMINATE,
+    /* SIGPWR    30 */   DEFAULT_TERMINATE,
+    /* SIGSYS/SIGUNUSED 31 */ DEFAULT_TERMINATE,
+
+    /* ASSUMPTION: all real-time have default of terminate...FIXME: ok? */
+    /* 32 */ DEFAULT_TERMINATE,
+    /* 33 */ DEFAULT_TERMINATE,
+    /* 34 */ DEFAULT_TERMINATE,
+    /* 35 */ DEFAULT_TERMINATE,
+    /* 36 */ DEFAULT_TERMINATE,
+    /* 37 */ DEFAULT_TERMINATE,
+    /* 38 */ DEFAULT_TERMINATE,
+    /* 39 */ DEFAULT_TERMINATE,
+    /* 40 */ DEFAULT_TERMINATE,
+    /* 41 */ DEFAULT_TERMINATE,
+    /* 42 */ DEFAULT_TERMINATE,
+    /* 43 */ DEFAULT_TERMINATE,
+    /* 44 */ DEFAULT_TERMINATE,
+    /* 45 */ DEFAULT_TERMINATE,
+    /* 46 */ DEFAULT_TERMINATE,
+    /* 47 */ DEFAULT_TERMINATE,
+    /* 48 */ DEFAULT_TERMINATE,
+    /* 49 */ DEFAULT_TERMINATE,
+    /* 50 */ DEFAULT_TERMINATE,
+    /* 51 */ DEFAULT_TERMINATE,
+    /* 52 */ DEFAULT_TERMINATE,
+    /* 53 */ DEFAULT_TERMINATE,
+    /* 54 */ DEFAULT_TERMINATE,
+    /* 55 */ DEFAULT_TERMINATE,
+    /* 56 */ DEFAULT_TERMINATE,
+    /* 57 */ DEFAULT_TERMINATE,
+    /* 58 */ DEFAULT_TERMINATE,
+    /* 59 */ DEFAULT_TERMINATE,
+    /* 60 */ DEFAULT_TERMINATE,
+    /* 61 */ DEFAULT_TERMINATE,
+    /* 62 */ DEFAULT_TERMINATE,
+    /* 63 */ DEFAULT_TERMINATE,
+};
+
+/* We know that many signals are always asynchronous.
+ * Others, however, may be synchronous or may not -- e.g., another process
+ * could send us a SIGSEGV, and there is no way we can tell whether it
+ * was generated by a real memory fault or not.  Thus we have to assume
+ * that we must not delay any SIGSEGV deliveries.
+ */
+bool can_always_delay[] = {
+    /* nothing    0 */             true,
+    /* SIGHUP     1 */             true,
+    /* SIGINT     2 */             true,
+    /* SIGQUIT    3 */             true,
+    /* SIGILL     4 */            false,
+    /* SIGTRAP    5 */            false,
+    /* SIGABRT/SIGIOT 6 */        false,
+    /* SIGBUS     7 */            false, 
+    /* SIGFPE     8 */            false,
+    /* SIGKILL    9 */             true,
+    /* SIGUSR1   10 */             true,
+    /* SIGSEGV   11 */            false,
+    /* SIGUSR2   12 */             true,
+    /* SIGPIPE   13 */            false,
+    /* SIGALRM   14 */             true,
+    /* SIGTERM   15 */             true,
+    /* SIGSTKFLT 16 */            false,
+    /* SIGCHLD   17 */             true,
+    /* SIGCONT   18 */             true,
+    /* SIGSTOP   19 */             true,
+    /* SIGTSTP   20 */             true,
+    /* SIGTTIN   21 */             true,
+    /* SIGTTOU   22 */             true,
+    /* SIGURG    23 */             true,
+    /* SIGXCPU   24 */            false,
+    /* SIGXFSZ   25 */             true,
+    /* SIGVTALRM 26 */             true,
+    /* SIGPROF   27 */             true,
+    /* SIGWINCH  28 */             true,
+    /* SIGIO/SIGPOLL/SIGLOST 29 */ true,
+    /* SIGPWR    30 */             true,
+    /* SIGSYS/SIGUNUSED 31 */     false,
+
+    /* ASSUMPTION: all real-time can be delayed */
+    /* 32 */                       true,
+    /* 33 */                       true,
+    /* 34 */                       true,
+    /* 35 */                       true,
+    /* 36 */                       true,
+    /* 37 */                       true,
+    /* 38 */                       true,
+    /* 39 */                       true,
+    /* 40 */                       true,
+    /* 41 */                       true,
+    /* 42 */                       true,
+    /* 43 */                       true,
+    /* 44 */                       true,
+    /* 45 */                       true,
+    /* 46 */                       true,
+    /* 47 */                       true,
+    /* 48 */                       true,
+    /* 49 */                       true,
+    /* 50 */                       true,
+    /* 51 */                       true,
+    /* 52 */                       true,
+    /* 53 */                       true,
+    /* 54 */                       true,
+    /* 55 */                       true,
+    /* 56 */                       true,
+    /* 57 */                       true,
+    /* 58 */                       true,
+    /* 59 */                       true,
+    /* 60 */                       true,
+    /* 61 */                       true,
+    /* 62 */                       true,
+    /* 63 */                       true,
+};
+
+/* we do not use SIGSTKSZ b/c for things like code modification
+ * we end up calling many core routines and so want more space
+ * (though currently non-debug stack size == SIGSTKSZ (8KB))
+ */
+#define SIGSTACK_SIZE DYNAMORIO_STACK_SIZE
+
+/* this flag not defined in our headers */
+#define SA_RESTORER 0x04000000
+
+/**** kernel_sigset_t ***************************************************/
+
+/* defines and typedefs are exported in os_exports.h for siglongjmp */
+
+/* most of these are from /usr/src/linux/include/linux/signal.h */
+static inline 
+void kernel_sigemptyset(kernel_sigset_t *set)
+{
+    memset(set, 0, sizeof(kernel_sigset_t));
+}
+
+static inline 
+void kernel_sigfillset(kernel_sigset_t *set)
+{
+    memset(set, -1, sizeof(kernel_sigset_t));
+}
+
+static inline 
+void kernel_sigaddset(kernel_sigset_t *set, int _sig)
+{
+    uint sig = _sig - 1;
+    if (_NSIG_WORDS == 1)
+        set->sig[0] |= 1UL << sig;
+    else
+        set->sig[sig / _NSIG_BPW] |= 1UL << (sig % _NSIG_BPW);
+}
+
+static inline 
+void kernel_sigdelset(kernel_sigset_t *set, int _sig)
+{
+    uint sig = _sig - 1;
+    if (_NSIG_WORDS == 1)
+        set->sig[0] &= ~(1UL << sig);
+    else
+        set->sig[sig / _NSIG_BPW] &= ~(1UL << (sig % _NSIG_BPW));
+}
+
+static inline 
+bool kernel_sigismember(kernel_sigset_t *set, int _sig)
+{
+    int sig = _sig - 1; /* go to 0-based */
+    if (_NSIG_WORDS == 1)
+        return 1 & (set->sig[0] >> sig);
+    else
+        return 1 & (set->sig[sig / _NSIG_BPW] >> (sig % _NSIG_BPW));
+}
+
+/* FIXME: how does libc do this? */
+static inline
+void copy_kernel_sigset_to_sigset(kernel_sigset_t *kset, sigset_t *uset)
+{
+    int sig;
+#ifdef DEBUG
+    int rc =
+#endif 
+        sigemptyset(uset);
+    ASSERT(rc == 0);
+    /* do this the slow way...I don't want to make assumptions about
+     * structure of user sigset_t
+     */
+    for (sig=1; sig<MAX_SIGNUM; sig++) {
+        if (kernel_sigismember(kset, sig))
+            sigaddset(uset, sig);
+    }
+}
+
+/* FIXME: how does libc do this? */
+static inline
+void copy_sigset_to_kernel_sigset(sigset_t *uset, kernel_sigset_t *kset)
+{
+    int sig;
+    kernel_sigemptyset(kset);
+    /* do this the slow way...I don't want to make assumptions about
+     * structure of user sigset_t
+     */
+    for (sig=1; sig<MAX_SIGNUM; sig++) {
+        if (sigismember(uset, sig))
+            kernel_sigaddset(kset, sig);
+    }
+}
+
+/**** frames *********************************************************/
+
+/* kernel's notion of sigaction has fields in different order from that
+ * used in glibc (I looked at glibc-2.2.4/sysdeps/unix/sysv/linux/i386/sigaction.c)
+ * also, /usr/include/asm/signal.h lists both versions
+ * I deliberately give kernel_sigaction_t's fields different names to help
+ * avoid confusion.
+ * (2.1.20 kernel has mask as 2nd field instead, and is expected to be passed
+ * in to the non-rt sigaction() call, which we do not yet support)
+ */
+struct _kernel_sigaction_t {
+    handler_t handler;
+    unsigned long flags;
+    void (*restorer)(void);
+    kernel_sigset_t mask;
+}; /* typedef in os_private.h */
+
+/* kernel's notion of ucontext is different from glibc's!
+ * this is adapted from asm/ucontext.h:
+ */
+typedef struct {
+    unsigned long     uc_flags;
+    struct ucontext  *uc_link;
+    stack_t           uc_stack;
+    struct sigcontext uc_mcontext;
+    kernel_sigset_t   uc_sigmask; /* mask last for extensibility */
+} kernel_ucontext_t;
+
+/* we assume frames look like this, with rt_sigframe used if SA_SIGINFO is set
+ * (these are from /usr/src/linux/arch/i386/kernel/signal.c for kernel 2.4.17)
+ */
+
+#define RETCODE_SIZE 8
+
+typedef struct sigframe {
+    char *pretcode;
+    int sig;
+    struct sigcontext sc;
+    struct _fpstate fpstate;
+    unsigned long extramask[_NSIG_WORDS-1];
+    char retcode[RETCODE_SIZE];
+    /* FIXME: this is a field I added, so our frame looks different from
+     * the kernel's...but where else can I store sig where the app won't
+     * clobber it?
+     * WARNING: our handler receives only rt frames, and we construct
+     * plain frames but never pass them to the kernel (on sigreturn() we
+     * just go to new context and interpret from there), so the only
+     * transparency problem here is if app tries to build its own plain
+     * frame and call sigreturn() unrelated to signal delivery.
+     */
+    int sig_noclobber;
+} sigframe_plain_t;
+
+/* the rt frame is used for SA_SIGINFO signals */
+typedef struct rt_sigframe {
+    char *pretcode;
+#ifdef X64
+    kernel_ucontext_t uc;
+    struct siginfo info;
+#else
+    int sig;
+    struct siginfo *pinfo;
+    void *puc;
+    struct siginfo info;
+    kernel_ucontext_t uc;
+    struct _fpstate fpstate;
+    char retcode[RETCODE_SIZE];
+#endif
+} sigframe_rt_t;
+
+
+/* if no app sigaction, it's RT, since that's our handler */
+#define IS_RT_FOR_APP(info, sig) \
+  IF_X64_ELSE(true, ((info)->app_sigaction[(sig)] == NULL ? true : \
+                     (TEST(SA_SIGINFO, (info)->app_sigaction[(sig)]->flags))))
+
+/* kernel sets size and sp to 0 for SS_DISABLE
+ * when asked, will hand back SS_ONSTACK only if current xsp is inside the
+ * alt stack; otherwise, if an alt stack is registered, it will give flags of 0
+ * We do not support the "legacy stack switching" that uses the restorer field
+ * as seen in kernel sources.
+ */
+#define APP_HAS_SIGSTACK(info) \
+  ((info)->app_sigstack.ss_sp != NULL && (info)->app_sigstack.ss_flags != SS_DISABLE)
+
+/* we have to queue up both rt and non-rt signals because we delay
+ * their delivery.
+ * PR 304708: we now leave in rt form right up until we copy to the
+ * app stack, so that we can deliver to a client at a safe spot
+ * in rt form.
+ */
+typedef struct _sigpending_t {
+    sigframe_rt_t rt_frame;
+#ifdef X64
+    /* fpstate is no longer kept inside the frame, and is not always present.
+     * if we delay we need to ensure we have room for it.
+     */
+    struct _fpstate __attribute__ ((aligned (16))) fpstate;
+#endif
+    struct _sigpending_t *next;
+} sigpending_t;
+
+#ifdef X64
+/* Extra space needed to put the signal frame on the app stack.
+ * We assume the stack pointer is 8-aligned already, so at most we
+ * need another 8 to align to 16.
+ */
+# define X64_FRAME_EXTRA (sizeof(struct _fpstate) + 8)
+#endif
+
+typedef struct _thread_sig_info_t {
+    /* we use kernel_sigaction_t so we don't have to translate back and forth
+     * between it and libc version.
+     * have to dynamically allocate app_sigaction array so we can share it.
+     */
+    kernel_sigaction_t **app_sigaction;
+
+    /* with CLONE_SIGHAND we may have to share app_sigaction */
+    bool shared_app_sigaction;
+    mutex_t *shared_lock;
+    int *shared_refcount;
+    /* signals we intercept must also be sharable */
+    bool *we_intercept;
+
+    /* cache restorer validity.  not shared: inheriter will re-populate. */
+    int restorer_valid[MAX_SIGNUM];
+
+    /* rest of app state */
+    stack_t app_sigstack;
+    sigpending_t *sigpending[MAX_SIGNUM];
+    kernel_sigset_t app_sigblocked;
+
+    /* to handle sigsuspend we have to save blocked set */
+    bool in_sigsuspend;
+    kernel_sigset_t app_sigblocked_save;
+
+    /* to inherit in children must not modify until they're scheduled */
+    volatile int num_unstarted_children;
+    mutex_t child_lock;
+
+    /* our own structures */
+    stack_t sigstack;
+    void *sigheap; /* special heap */
+    fragment_t *interrupted; /* frag we unlinked for delaying signal */
+
+#ifdef RETURN_AFTER_CALL
+    app_pc signal_restorer_retaddr;     /* last signal restorer, known ret exception */
+#endif
+} thread_sig_info_t;
+
+/**** function prototypes ***********************************************/
+
+/* in x86.asm for x64 */
+IF_NOT_X64(static) void
+master_signal_handler(int sig, siginfo_t *siginfo, kernel_ucontext_t *ucxt);
+
+static void
+intercept_signal(dcontext_t *dcontext, thread_sig_info_t *info, int sig);
+
+static void
+execute_handler_from_cache(dcontext_t *dcontext, int sig, sigframe_rt_t *our_frame,
+                           struct sigcontext *sc_orig);
+
+static bool
+execute_handler_from_dispatch(dcontext_t *dcontext, int sig);
+
+static void
+execute_default_from_cache(dcontext_t *dcontext, int sig, sigframe_rt_t *frame);
+
+static void
+execute_default_from_dispatch(dcontext_t *dcontext, int sig, sigframe_rt_t *frame);
+
+static bool
+handle_suspend_signal(dcontext_t *dcontext, kernel_ucontext_t *ucxt);
+
+static struct sigcontext *
+get_sigcontext_from_rt_frame(sigframe_rt_t *frame);
+
+/* in pcprofile.c */
+extern void pcprofile_alarm(dcontext_t *dcontext, void *pc, app_pc ebp);
+
+#ifdef DEBUG
+static void
+dump_sigset(dcontext_t *dcontext, kernel_sigset_t *set);
+#endif
+
+static inline int
+sigaction_syscall(int sig, kernel_sigaction_t *act, kernel_sigaction_t *oact)
+{
+#ifdef X64
+    /* PR 305020: must have SA_RESTORER for x64 */
+    if (act != NULL && !TEST(SA_RESTORER, act->flags)) {
+        act->flags |= SA_RESTORER;
+        act->restorer = (void (*)(void)) dynamorio_sigreturn;
+    }
+#endif
+    return dynamorio_syscall(SYS_rt_sigaction, 4, sig, act, oact,
+                             sizeof(kernel_sigset_t));
+}
+
+static inline int
+sigaltstack_syscall(const stack_t *newstack, stack_t *oldstack)
+{
+    return dynamorio_syscall(SYS_sigaltstack, 2, newstack, oldstack);
+}
+
+static inline int
+setitimer_syscall(int which, struct itimerval *val, struct itimerval *old)
+{
+    return dynamorio_syscall(SYS_setitimer, 3, which, val, old);
+}
+
+#ifdef CLIENT_INTERFACE
+/* PR 368737: exported but not documented yet until we're more comfortable
+ * that DR handles timer signals robustly (xref PR 205795).
+ */
+DR_API
+int
+dr_setitimer(int which, struct itimerval *val, struct itimerval *old)
+{
+    return setitimer_syscall(which, val, old);
+}
+#endif
+
+static inline int
+sigprocmask_syscall(int how, kernel_sigset_t *set, kernel_sigset_t *oset,
+                    size_t sigsetsize)
+{
+    return dynamorio_syscall(SYS_rt_sigprocmask, 4, how, set, oset, sigsetsize);
+}
+
+/* exported for stackdump.c */
+bool
+set_default_signal_action(int sig)
+{
+    kernel_sigaction_t act;
+    int rc;
+    act.handler = (handler_t) SIG_DFL;
+    /* arm the signal */
+    rc = sigaction_syscall(sig, &act, NULL);
+    return (rc == 0);
+}
+
+/* We assume that signal handlers will be shared most of the time
+ * (pthreads shares them)
+ * Rather than start out with the handler table in local memory and then
+ * having to transfer to global, we just always use global
+ */
+static void
+handler_free(dcontext_t *dcontext, void *p, size_t size)
+{
+    global_heap_free(p, size HEAPACCT(ACCT_OTHER));
+}
+
+static void *
+handler_alloc(dcontext_t *dcontext, size_t size)
+{
+    return global_heap_alloc(size HEAPACCT(ACCT_OTHER));
+}
+
+/**** floating point support ********************************************/
+
+/* The following code is based on routines in
+ *   /usr/src/linux/arch/i386/kernel/i387.c
+ * and definitions in
+ *   /usr/src/linux/include/asm-i386/processor.h
+ *   /usr/src/linux/include/asm-i386/i387.h
+ */
+
+struct i387_fsave_struct {
+    long        cwd;
+    long        swd;
+    long        twd;
+    long        fip;
+    long        fcs;
+    long        foo;
+    long        fos;
+    long        st_space[20];   /* 8*10 bytes for each FP-reg = 80 bytes */
+    long        status;         /* software status information */
+};
+
+/* note that fxsave requires that i387_fxsave_struct be aligned on
+ * a 16-byte boundary
+ */
+struct i387_fxsave_struct {
+    unsigned short      cwd;
+    unsigned short      swd;
+    unsigned short      twd;
+    unsigned short      fop;
+#ifdef X64
+    long        rip;
+    long        rdp;
+    int         mxcsr;
+    int         mxcsr_mask;
+    int         st_space[32];   /* 8*16 bytes for each FP-reg = 128 bytes */
+    int         xmm_space[64];  /* 16*16 bytes for each XMM-reg = 256 bytes */
+    int         padding[24];
+#else
+    long        fip;
+    long        fcs;
+    long        foo;
+    long        fos;
+    long        mxcsr;
+    long        reserved;
+    long        st_space[32];   /* 8*16 bytes for each FP-reg = 128 bytes */
+    long        xmm_space[32];  /* 8*16 bytes for each XMM-reg = 128 bytes */
+    long        padding[56];
+#endif
+} __attribute__ ((aligned (16)));
+
+union i387_union {
+    struct i387_fsave_struct    fsave;
+    struct i387_fxsave_struct   fxsave;
+};
+
+#ifndef X64
+/* For 32-bit if we use fxsave we need to convert it to the kernel's struct.
+ * For 64-bit the kernel's struct is identical to the fxsave format.
+ */
+static uint
+twd_fxsr_to_i387(struct i387_fxsave_struct *fxsave)
+{
+    struct _fpxreg *st = NULL;
+    uint twd = (uint) fxsave->twd;
+    uint tag;
+    uint ret = 0xffff0000;
+    int i;
+    for (i = 0 ; i < 8 ; i++) {
+        if (TEST(0x1, twd)) {
+            st = (struct _fpxreg *) &fxsave->st_space[i*4];
+
+            switch (st->exponent & 0x7fff) {
+            case 0x7fff:
+                tag = 2;        /* Special */
+                break;
+            case 0x0000:
+                if (st->significand[0] == 0 &&
+                    st->significand[1] == 0 &&
+                    st->significand[2] == 0 &&
+                    st->significand[3] == 0) {
+                    tag = 1;    /* Zero */
+                } else {
+                    tag = 2;    /* Special */
+                }
+                break;
+            default:
+                if (TEST(0x8000, st->significand[3])) {
+                    tag = 0;    /* Valid */
+                } else {
+                    tag = 2;    /* Special */
+                }
+                break;
+            }
+        } else {
+            tag = 3;            /* Empty */
+        }
+        ret |= (tag << (2 * i));
+        twd = twd >> 1;
+    }
+    return ret;
+}
+
+static void
+convert_fxsave_to_fpstate(struct _fpstate *fpstate,
+                          struct i387_fxsave_struct *fxsave)
+{
+    int i;
+
+    fpstate->cw = (uint)fxsave->cwd | 0xffff0000;
+    fpstate->sw = (uint)fxsave->swd | 0xffff0000;
+    fpstate->tag = twd_fxsr_to_i387(fxsave);
+    fpstate->ipoff = fxsave->fip;
+    fpstate->cssel = fxsave->fcs | ((uint)fxsave->fop << 16);
+    fpstate->dataoff = fxsave->foo;
+    fpstate->datasel = fxsave->fos;
+
+    for (i = 0; i < 8; i++) {
+        memcpy(&fpstate->_st[i], &fxsave->st_space[i*4], sizeof(fpstate->_st[i]));
+    }
+
+    fpstate->status = fxsave->swd;
+    fpstate->magic = X86_FXSR_MAGIC;
+
+    memcpy(&fpstate->_fxsr_env[0], fxsave,
+           sizeof(struct i387_fxsave_struct));
+}
+#endif /* !X64 */
+
+static void
+save_xmm(dcontext_t *dcontext, sigframe_rt_t *frame)
+{
+    int i;
+    struct sigcontext *sc = get_sigcontext_from_rt_frame(frame);
+    if (!preserve_xmm_caller_saved())
+        return;
+    for (i=0; i<NUM_XMM_SAVED; i++) {
+        /* we assume no padding */
+#ifdef X64
+        memcpy(sc->fpstate->xmm_space, get_mcontext(dcontext)->xmm,
+               NUM_XMM_SLOTS*XMM_REG_SIZE);
+#else
+        memcpy(sc->fpstate->_xmm, get_mcontext(dcontext)->xmm,
+               NUM_XMM_SLOTS*XMM_REG_SIZE);
+#endif
+    }
+}
+
+/* We can't tell whether the app has used fpstate yet so we preserve every time */
+static void
+save_fpstate(dcontext_t *dcontext, sigframe_rt_t *frame)
+{
+    /* FIXME: is there a better way to align this thing?
+     * the __attribute__ on the struct above doesn't do it
+     */
+    char align[sizeof(union i387_union) + 16];
+    union i387_union *temp = (union i387_union *)
+        ( (((ptr_uint_t)align) + 16) & ((ptr_uint_t)-16) );
+    struct sigcontext *sc = get_sigcontext_from_rt_frame(frame);
+    LOG(THREAD, LOG_ASYNCH, 3, "save_fpstate\n");
+    if (sc->fpstate == NULL) {
+#ifdef X64
+        /* fpstate is not inlined, so before getting here someone (copy_frame_to_*,
+         * or thread_set_self_context) is supposed to lay it out and point at it
+         */
+        ASSERT_NOT_REACHED();
+        return; /* just continue w/o saving */
+#else
+        /* may be NULL due to lazy fp state saving by kernel */
+        sc->fpstate = &frame->fpstate;
+#endif
+    } else {
+        LOG(THREAD, LOG_ASYNCH, 3,
+            "ptr="PFX", struct="PFX"\n",
+            sc->fpstate, IF_X64_ELSE(NULL, &frame->fpstate));
+    }
+    if (proc_has_feature(FEATURE_FXSR)) {
+        LOG(THREAD, LOG_ASYNCH, 3, "\ttemp="PFX"\n", temp);
+#ifdef X64
+        /* this is "unlazy_fpu" */
+        /* fxsaveq is only supported with gas >= 2.16 but we have that */
+        asm volatile( "fxsaveq %0 ; fnclex"
+                      : "=m" (temp->fxsave) );
+        /* now convert into struct _fpstate form */
+        ASSERT(sizeof(struct _fpstate) == sizeof(struct i387_fxsave_struct));
+        memcpy(sc->fpstate, &temp->fxsave, sizeof(struct i387_fxsave_struct));
+#else
+        /* this is "unlazy_fpu" */
+        asm volatile( "fxsave %0 ; fnclex"
+                      : "=m" (temp->fxsave) );
+        /* now convert into struct _fpstate form */
+        convert_fxsave_to_fpstate(sc->fpstate, &temp->fxsave);
+#endif
+    } else {
+        /* FIXME NYI: need to convert to fxsave format for sc->fpstate */
+        IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+        /* this is "unlazy_fpu" */
+        asm volatile( "fnsave %0 ; fwait"
+                      : "=m" (temp->fsave) );
+        /* now convert into struct _fpstate form */
+        temp->fsave.status = temp->fsave.swd;
+        memcpy(sc->fpstate, &temp->fsave, sizeof(struct i387_fsave_struct));
+    }
+
+    /* the app's xmm registers may be saved away in dr_mcontext_t, in which
+     * case we need to copy those values instead of using what was in
+     * the physical xmm registers
+     */
+    save_xmm(dcontext, frame);
+}
+
+/**** top-level routines ***********************************************/
+
+
+void
+signal_init()
+{
+    IF_X64(ASSERT(ALIGNED(offsetof(sigpending_t, fpstate), 16)));
+}
+
+void
+signal_exit()
+{
+#ifdef DEBUG
+    if (stats->loglevel > 0 && (stats->logmask & (LOG_ASYNCH|LOG_STATS)) != 0) {
+        LOG(GLOBAL, LOG_ASYNCH|LOG_STATS, 1,
+            "Total signals delivered: %d\n", GLOBAL_STAT(num_signals));
+    }
+#endif
+}
+
+void
+signal_thread_init(dcontext_t *dcontext)
+{
+    int i, rc;
+    kernel_sigaction_t oldact;
+    thread_id_t p_tid;
+    thread_record_t *tr;
+    thread_sig_info_t *info = HEAP_TYPE_ALLOC(dcontext, thread_sig_info_t,
+                                              ACCT_OTHER, PROTECTED);
+    dcontext->signal_field = (void *) info;
+
+    /* all fields want to be initialized to 0 */
+    memset(info, 0, sizeof(thread_sig_info_t));
+
+    /* our special heap to avoid reentrancy problems
+     * composed entirely of sigpending_t units
+     */
+    info->sigheap = special_heap_init(sizeof(sigpending_t),
+                                      false /* cannot have any locking */,
+                                      false /* -x */,
+                                      true /* persistent */);
+
+#ifdef HAVE_SIGALTSTACK
+    /* set up alternate stack */
+    info->sigstack.ss_sp = (char *) heap_alloc(dcontext, SIGSTACK_SIZE
+                                               HEAPACCT(ACCT_OTHER));
+    info->sigstack.ss_size = SIGSTACK_SIZE;
+    /* kernel will set xsp to sp+size to grow down from there, we don't have to */
+    info->sigstack.ss_flags = SS_ONSTACK;
+    rc = sigaltstack_syscall(&info->sigstack, &info->app_sigstack);
+    ASSERT(rc == 0);
+    LOG(THREAD, LOG_ASYNCH, 1, "signal stack is "PFX" - "PFX"\n",
+        info->sigstack.ss_sp, info->sigstack.ss_sp + info->sigstack.ss_size);
+    /* app_sigstack dealt with below, based on parentage */
+#endif
+
+    kernel_sigemptyset(&info->app_sigblocked);
+
+    ASSIGN_INIT_LOCK_FREE(info->child_lock, child_lock);
+
+    /* inherit and potentially share signal handler state from parent? */
+    p_tid = get_parent_id();
+    /* we're in a thread_init call so we already hold thread_initexit_lock! */
+    tr = thread_lookup(p_tid);
+    if (tr != NULL && tr->under_dynamo_control) {
+        /* initialize handler state in signal_thread_inherit() call,
+         * from here we don't know if should share or inherit
+         */
+        if (APP_HAS_SIGSTACK(info)) {
+            /* parent was under our control, so the real sigstack we see is just
+             * the parent's being inherited -- clear it now
+             */
+            memset(&info->app_sigstack, 0, sizeof(stack_t));
+        }
+    } else {
+        /* initialize in isolation */
+
+        if (APP_HAS_SIGSTACK(info)) {
+            /* parent was NOT under our control, so the real sigstack we see is
+             * a real sigstack that was present before we took control
+             */
+            LOG(THREAD, LOG_ASYNCH, 1, "app already has signal stack "PFX" - "PFX"\n",
+                info->app_sigstack.ss_sp,
+                info->app_sigstack.ss_sp + info->app_sigstack.ss_size);
+        }
+
+        info->app_sigaction = (kernel_sigaction_t **)
+            handler_alloc(dcontext, MAX_SIGNUM * sizeof(kernel_sigaction_t *));
+        memset(info->app_sigaction, 0, MAX_SIGNUM * sizeof(kernel_sigaction_t *));
+        memset(&info->restorer_valid, -1, MAX_SIGNUM * sizeof(info->restorer_valid[0]));
+        info->we_intercept = (bool *) handler_alloc(dcontext, MAX_SIGNUM * sizeof(bool));
+        memset(info->we_intercept, 0, MAX_SIGNUM * sizeof(bool));
+                
+        if (DYNAMO_OPTION(intercept_all_signals)) {
+            /* PR 304708: to support client signal handlers without
+             * the complexity of per-thread and per-signal callbacks
+             * we always intercept all signals.  We also check here
+             * for handlers the app registered before our init.
+             */
+            for (i=1; i<MAX_SIGNUM; i++) {
+                /* cannot intercept KILL or STOP */
+                if (i != SIGKILL && i != SIGSTOP &&
+                    /* FIXME PR 297033: we don't support intercepting DEFAULT_STOP /
+                     * DEFAULT_CONTINUE signals.  Once add support, update
+                     * dr_register_signal_event() comments.
+                     */
+                    default_action[i] != DEFAULT_STOP &&
+                    default_action[i] != DEFAULT_CONTINUE)
+                    intercept_signal(dcontext, info, i);
+            }
+        } else {
+            /* we intercept the following signals ourselves: */
+            intercept_signal(dcontext, info, SIGSEGV);
+            /* PR 313665: look for DR crashes on unaligned memory or mmap bounds */
+            intercept_signal(dcontext, info, SIGBUS);
+            /* PR 212090: the signal we use to suspend threads */
+            intercept_signal(dcontext, info, SUSPEND_SIGNAL);
+#ifdef PAPI
+            /* use SIGPROF for updating gui so it can be distinguished from SIGVTALRM */
+            intercept_signal(dcontext, info, SIGPROF);
+#endif
+            /* vtalarm only used with pc profiling.  it interferes w/ PAPI
+             * so arm this signal only if necessary
+             */
+            if (INTERNAL_OPTION(profile_pcs)) {
+                intercept_signal(dcontext, info, SIGVTALRM);
+            }
+#ifdef SIDELINE
+            intercept_signal(dcontext, info, SIGCHLD);
+#endif
+        
+            /* process any handlers app registered before our init */
+            for (i=1; i<MAX_SIGNUM; i++) {
+                if (info->we_intercept[i]) {
+                    /* intercept_signal already stored pre-existing handler */
+                    continue;
+                }
+                rc = sigaction_syscall(i, NULL, &oldact);
+                ASSERT(rc == 0
+#ifdef VMX86_SERVER
+                       /* Workaround for PR 223720, which was fixed in ESX4.0 but
+                        * is present in ESX3.5 and earlier: vmkernel treats
+                        * 63 and 64 as invalid signal numbers.
+                        */
+                       || (i >= 63 && rc == -EINVAL)
+#endif
+                       );
+                if (rc == 0 &&
+                    oldact.handler != (handler_t) SIG_DFL &&
+                    oldact.handler != (handler_t) master_signal_handler) {
+                    /* could be master_ if inherited */
+                    /* FIXME: if app removes handler, we'll never remove ours */
+                    intercept_signal(dcontext, info, i);
+                    info->we_intercept[i] = false;
+                }
+            }
+        }
+    }
+}
+
+/* called once a new thread's dcontext is created.
+ * Inherited and shared fields are set up here.
+ */
+void
+signal_thread_inherit(dcontext_t *dcontext)
+{
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    thread_record_t *tr;
+    int i;
+    thread_id_t p_tid = get_parent_id();
+    mutex_lock(&thread_initexit_lock); /* need lock to hold thread_record_t */
+    tr = thread_lookup(p_tid);
+    if (tr != NULL && tr->under_dynamo_control) {
+        thread_sig_info_t *p_info = (thread_sig_info_t *) tr->dcontext->signal_field;
+        uint clone_flags = tr->dcontext->sys_param0;
+        LOG(THREAD, LOG_ASYNCH, 1,
+            "parent pid is %d, parent sysnum is %d(%s), clone flags="PIFX"\n", 
+            p_tid, 
+            tr->dcontext->sys_num, 
+            tr->dcontext->sys_num == SYS_vfork ? "vfork" : 
+            tr->dcontext->sys_num == SYS_clone ? "clone" : "unexpected",
+            clone_flags);
+        if (tr->dcontext->sys_num == SYS_vfork) {
+            /* The above clone_flags argument is bogus.
+               SYS_vfork doesn't have a free register to keep the hardcoded value
+               see /usr/src/linux/arch/i386/kernel/process.c */
+            /* CHECK: is this the only place real clone flags are needed? */
+            clone_flags = CLONE_VFORK | CLONE_VM | SIGCHLD;
+        }
+        mutex_unlock(&thread_initexit_lock); /* done with tr */
+
+        /* handlers are either inherited or shared */
+        if ((clone_flags & CLONE_SIGHAND) != 0) {
+            /* need to share table of handlers! */
+            LOG(THREAD, LOG_ASYNCH, 2, "sharing signal handlers with parent\n");
+            info->shared_app_sigaction = true;
+            info->shared_refcount = p_info->shared_refcount;
+            info->shared_lock = p_info->shared_lock;
+            info->app_sigaction = p_info->app_sigaction;
+            info->we_intercept = p_info->we_intercept;
+            mutex_lock(info->shared_lock);
+            (*info->shared_refcount)++;
+#ifdef DEBUG
+            for (i = 0; i < MAX_SIGNUM; i++) {
+                if (info->app_sigaction[i] != NULL) {
+                    LOG(THREAD, LOG_ASYNCH, 2, "\thandler for signal %d is "PFX"\n",
+                        i, info->app_sigaction[i]->handler);
+                }
+            }
+#endif
+            mutex_unlock(info->shared_lock);
+        } else {
+            /* copy handlers */
+            LOG(THREAD, LOG_ASYNCH, 2, "inheriting signal handlers from parent\n");
+            info->app_sigaction = (kernel_sigaction_t **)
+                handler_alloc(dcontext, MAX_SIGNUM * sizeof(kernel_sigaction_t *));
+            memset(info->app_sigaction, 0, MAX_SIGNUM * sizeof(kernel_sigaction_t *));
+            for (i = 0; i < MAX_SIGNUM; i++) {
+                ASSERT(p_info->restorer_valid[i] == -1);
+                if (p_info->app_sigaction[i] != NULL) {
+                    info->app_sigaction[i] = (kernel_sigaction_t *)
+                        handler_alloc(dcontext, sizeof(kernel_sigaction_t));
+                    memcpy(info->app_sigaction[i], p_info->app_sigaction[i],
+                           sizeof(kernel_sigaction_t));
+                    LOG(THREAD, LOG_ASYNCH, 2, "\thandler for signal %d is "PFX"\n",
+                        i, info->app_sigaction[i]->handler);
+                }
+            }
+            info->we_intercept = (bool *) handler_alloc(dcontext, MAX_SIGNUM * sizeof(bool));
+            memcpy(info->we_intercept, p_info->we_intercept,
+                   MAX_SIGNUM * sizeof(bool));
+            mutex_lock(&p_info->child_lock);
+            p_info->num_unstarted_children--;
+            mutex_unlock(&p_info->child_lock);
+        }
+        /* rest of state is never shared.
+         * app_sigstack should already be in place, when we set up our sigstack
+         * we asked for old sigstack.
+         * FIXME: are current pending or blocked inherited?
+         */
+    } else {
+        mutex_unlock(&thread_initexit_lock);
+        LOG(THREAD, LOG_ASYNCH, 1, "WARNING: cannot find parent of thread %d\n",
+            get_thread_id());
+    }
+}
+
+void
+signal_thread_exit(dcontext_t *dcontext)
+{
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    int i;
+    kernel_sigaction_t act;
+    act.handler = (handler_t) SIG_DFL;
+    kernel_sigemptyset(&act.mask); /* does mask matter for SIG_DFL? */
+
+    while (info->num_unstarted_children > 0) {
+        /* must wait for children to start and copy our state
+         * before we destroy it!
+         */
+        thread_yield();
+    }
+
+    /* FIXME: w/ shared handlers, if parent (the owner here) dies,
+     * can children keep living w/ a copy of the handlers?
+     */
+    if (info->shared_app_sigaction) {
+        mutex_lock(info->shared_lock);
+        (*info->shared_refcount)--;
+        mutex_unlock(info->shared_lock);
+    }
+    if (!info->shared_app_sigaction || *info->shared_refcount == 0) {
+        LOG(THREAD, LOG_ASYNCH, 2, "Signal handler cleanup:\n");
+        for (i = 0; i < MAX_SIGNUM; i++) {
+            if (info->app_sigaction[i] != NULL) {
+                /* restore to old handler */
+                LOG(THREAD, LOG_ASYNCH, 2, "\trestoring "PFX" as handler for %d\n",
+                    info->app_sigaction[i]->handler, i);
+                sigaction_syscall(i, info->app_sigaction[i], NULL);
+                handler_free(dcontext, info->app_sigaction[i], sizeof(kernel_sigaction_t));
+            } else if (info->we_intercept[i]) {
+                /* restore to default */
+                LOG(THREAD, LOG_ASYNCH, 2, "\trestoring SIG_DFL as handler for %d\n", i);
+                sigaction_syscall(i, &act, NULL);
+            }
+            while (info->sigpending[i] != NULL) {
+                sigpending_t *temp = info->sigpending[i];
+                info->sigpending[i] = temp->next;
+                special_heap_free(info->sigheap, temp);
+            }
+        }
+        handler_free(dcontext, info->app_sigaction, MAX_SIGNUM * sizeof(kernel_sigaction_t *));
+        handler_free(dcontext, info->we_intercept, MAX_SIGNUM * sizeof(bool));
+        if (info->shared_lock != NULL) {
+            DELETE_LOCK(*info->shared_lock);
+            global_heap_free(info->shared_lock, sizeof(mutex_t) HEAPACCT(ACCT_OTHER));
+        }
+        if (info->shared_refcount != NULL)
+            global_heap_free(info->shared_refcount, sizeof(int) HEAPACCT(ACCT_OTHER));
+    }
+    special_heap_exit(info->sigheap);
+    DELETE_LOCK(info->child_lock);
+#ifdef DEBUG
+    /* for non-debug we do fast exit path and don't free local heap */
+# ifdef HAVE_SIGALTSTACK
+    heap_free(dcontext, info->sigstack.ss_sp, info->sigstack.ss_size
+              HEAPACCT(ACCT_OTHER));
+# endif
+    HEAP_TYPE_FREE(dcontext, info, thread_sig_info_t, ACCT_OTHER, PROTECTED);
+#endif
+#ifdef PAPI
+    stop_itimer();
+#endif
+}
+
+/* Set up master_signal_handler as the handler for signal "sig",
+ * for the current thread.  Since we deal with kernel data structures
+ * in our interception of system calls, we use them here as well,
+ * to avoid having to translate to/from libc data structures.
+ */
+static void
+intercept_signal(dcontext_t *dcontext, thread_sig_info_t *info, int sig)
+{
+    int rc;
+    kernel_sigaction_t act;
+    kernel_sigaction_t oldact;
+    ASSERT(sig < MAX_SIGNUM);
+
+    act.handler = (handler_t) master_signal_handler;
+    /* FIXME PR 287309: we need to NOT suppress further SIGSEGV */
+    kernel_sigfillset(&act.mask); /* block all signals within handler */
+    act.flags =
+        SA_SIGINFO | /* send 3 args to handler */
+        SA_ONSTACK; /* use our sigstack */
+    /* arm the signal */
+    rc = sigaction_syscall(sig, &act, &oldact);
+    ASSERT(rc == 0
+#ifdef VMX86_SERVER
+           /* Workaround for PR 223720, which was fixed in ESX4.0 but
+            * is present in ESX3.5 and earlier: vmkernel treats
+            * 63 and 64 as invalid signal numbers.
+            */
+           || (i >= 63 && rc == -EINVAL)
+#endif
+           );
+    if (rc != 0) /* be defensive: app will probably still work */
+        return;
+
+    if (oldact.handler != (handler_t) SIG_DFL &&
+        oldact.handler != (handler_t) master_signal_handler) {
+        /* save the app's action for sig */
+        if (info->shared_app_sigaction) {
+            /* app_sigaction structure is shared */
+            mutex_lock(info->shared_lock);
+        }
+        if (info->app_sigaction[sig] != NULL) {
+            /* go ahead and toss the old one, it's up to the app to store
+             * and then restore later if it wants to
+             */
+            handler_free(dcontext, info->app_sigaction[sig], sizeof(kernel_sigaction_t));
+        }
+        info->app_sigaction[sig] = (kernel_sigaction_t *)
+            handler_alloc(dcontext, sizeof(kernel_sigaction_t));
+        memcpy(info->app_sigaction[sig], &oldact, sizeof(kernel_sigaction_t));
+        /* clear cache */
+        info->restorer_valid[sig] = -1;
+        if (info->shared_app_sigaction)
+            mutex_unlock(info->shared_lock);
+#ifdef DEBUG
+        if (oldact.handler == (handler_t) SIG_IGN) {
+            LOG(THREAD, LOG_ASYNCH, 2,
+                "app already installed SIG_IGN as sigaction for signal %d\n", sig);
+        } else {
+            LOG(THREAD, LOG_ASYNCH, 2,
+                "app already installed "PFX" as sigaction for signal %d\n",
+                oldact.handler, sig);
+        }
+#endif
+    }
+    
+    LOG(THREAD, LOG_ASYNCH, 3, "\twe intercept signal %d\n", sig);
+    info->we_intercept[sig] = true;
+}
+
+/**** system call handlers ***********************************************/
+
+/* FIXME: invalid pointer passed to kernel will currently show up
+ * probably as a segfault in our handlers below...need to make them
+ * look like kernel, and pass error code back to os.c
+ */
+
+void
+handle_clone(dcontext_t *dcontext, uint flags)
+{
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    if ((flags & CLONE_VM) == 0) {
+        /* separate process not sharing memory */
+        if ((flags & CLONE_SIGHAND) != 0) {
+            /* FIXME: how deal with this?
+             * "man clone" says: "Since Linux 2.6.0-test6, flags must also 
+             * include CLONE_VM if CLONE_SIGHAND is specified"
+             */
+            LOG(THREAD, LOG_ASYNCH, 1, "WARNING: !CLONE_VM but CLONE_SIGHAND!\n");
+            ASSERT_NOT_IMPLEMENTED(false);
+        }
+        return;
+    }
+
+    if ((flags & CLONE_SIGHAND) != 0) {
+        /* need to share table of handlers! */
+        LOG(THREAD, LOG_ASYNCH, 2, "handle_clone: CLONE_SIGHAND set!\n");
+        if (!info->shared_app_sigaction) {
+            /* this is the start of a chain of sharing
+             * no synch needed here, child not created yet
+             */
+            info->shared_app_sigaction = true;
+            info->shared_refcount = (int *) global_heap_alloc(sizeof(int)
+                                                              HEAPACCT(ACCT_OTHER));
+            *info->shared_refcount = 1;
+            info->shared_lock = (mutex_t *) global_heap_alloc(sizeof(mutex_t)
+                                                            HEAPACCT(ACCT_OTHER));
+            ASSIGN_INIT_LOCK_FREE(*info->shared_lock, shared_lock);
+        } /* else, some ancestor is already owner */
+   } else {
+        /* child will inherit copy of current table -> cannot modify it
+         * until child is scheduled!  FIXME: any other way?
+         */
+        mutex_lock(&info->child_lock);
+        info->num_unstarted_children++;
+        mutex_unlock(&info->child_lock);
+    }
+}
+
+/* Returns false if should NOT issue syscall.
+ */
+bool
+handle_sigaction(dcontext_t *dcontext, int sig, const kernel_sigaction_t *act,
+                 kernel_sigaction_t *oact, size_t sigsetsize)
+{
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    kernel_sigaction_t *save;
+    kernel_sigaction_t *non_const_act = (kernel_sigaction_t *) act;
+    ASSERT(sig < MAX_SIGNUM);
+
+    if (act != NULL) {
+        /* app is installing a new action */
+        
+        while (info->num_unstarted_children > 0) {
+            /* must wait for children to start and copy our state
+             * before we modify it!
+             */
+            thread_yield();
+        }
+
+        if (info->shared_app_sigaction) {
+            /* app_sigaction structure is shared */
+            mutex_lock(info->shared_lock);
+        }
+
+        if (act->handler == (handler_t) SIG_IGN ||
+            act->handler == (handler_t) SIG_DFL) {
+            LOG(THREAD, LOG_ASYNCH, 2,
+                "app installed %s as sigaction for signal %d\n",
+                (act->handler == (handler_t) SIG_IGN) ? "SIG_IGN" : "SIG_DFL", sig);
+            if (!info->we_intercept[sig]) {
+                /* let the SIG_IGN/SIG_DFL go through, we want to remove our
+                 * handler.  we delete the stored app_sigaction in post_
+                 */
+                if (info->shared_app_sigaction)
+                    mutex_unlock(info->shared_lock);
+                return true;
+            }
+        } else {
+            LOG(THREAD, LOG_ASYNCH, 2,
+                "app installed "PFX" as sigaction for signal %d\n",
+                act->handler, sig);
+        }
+
+        /* save app's entire sigaction struct */
+        save = (kernel_sigaction_t *) handler_alloc(dcontext, sizeof(kernel_sigaction_t));
+        memcpy(save, act, sizeof(kernel_sigaction_t));
+        if (info->app_sigaction[sig] != NULL) {
+            /* go ahead and toss the old one, it's up to the app to store
+             * and then restore later if it wants to
+             */
+            handler_free(dcontext, info->app_sigaction[sig], sizeof(kernel_sigaction_t));
+        }
+        info->app_sigaction[sig] = save;
+        LOG(THREAD, LOG_ASYNCH, 3, "\tflags = "PFX", restorer = "PFX"\n",
+            act->flags, act->restorer);
+        /* clear cache */
+        info->restorer_valid[sig] = -1;
+        if (info->shared_app_sigaction)
+            mutex_unlock(info->shared_lock);
+
+        if (info->we_intercept[sig]) {
+            /* cancel the syscall */
+            return false;
+        }
+        /* now hand kernel our master handler instead of app's 
+         * FIXME: double-check we're dealing w/ all posible mask, flag
+         * differences bet app & our handler
+         */
+        non_const_act->handler = (handler_t) master_signal_handler;
+        /* block all signals within handler */
+        kernel_sigfillset(&(non_const_act->mask)); 
+        non_const_act->flags =
+            SA_SIGINFO | /* send 3 args to handler */
+            SA_ONSTACK; /* use our sigstack */
+#ifdef X64
+        /* PR 305020: must have SA_RESTORER for x64 */
+        non_const_act->flags |= SA_RESTORER;
+        non_const_act->restorer = (void (*)(void)) dynamorio_sigreturn;
+#endif
+
+        /* FIXME PR 297033: we don't support intercepting DEFAULT_STOP /
+         * DEFAULT_CONTINUE signals b/c we can't generate the default
+         * action: if the app registers a handler, though, we should work
+         * properly if we never see SIG_DFL.
+         */
+    }
+
+    /* oact is handled post-syscall */
+
+    return true;
+}
+
+/* os.c thinks it's passing us struct_sigaction, really it's kernel_sigaction_t,
+ * which has fields in different order.
+ */
+void
+handle_post_sigaction(dcontext_t *dcontext, int sig, const kernel_sigaction_t *act,
+                      kernel_sigaction_t *oact, size_t sigsetsize)
+{
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    ASSERT(sig < MAX_SIGNUM);
+    if (oact != NULL) {
+        /* FIXME: hold lock across the syscall?!?
+         * else could be modified and get wrong old action?
+         */
+        /* FIXME: make sure oact is readable & writable before accessing! */
+        if (info->shared_app_sigaction)
+            mutex_lock(info->shared_lock);
+        if (info->app_sigaction[sig] == NULL) {
+            if (info->we_intercept[sig]) {
+                /* need to pretend there is no handler */
+                memset(oact, 0, sizeof(*oact));
+                oact->handler = (handler_t) SIG_DFL;
+            } else {
+                ASSERT(oact->handler == (handler_t) SIG_IGN ||
+                       oact->handler == (handler_t) SIG_DFL);
+            }
+        } else {
+            memcpy(oact, info->app_sigaction[sig], sizeof(kernel_sigaction_t));
+
+            /* if installing IGN or DFL, delete ours */
+            if (act && ((act->handler == (handler_t) SIG_IGN ||
+                         act->handler == (handler_t) SIG_DFL) &&
+                        !info->we_intercept[sig])) {
+                /* remove old stored app action */
+                handler_free(dcontext, info->app_sigaction[sig],
+                             sizeof(kernel_sigaction_t));
+                info->app_sigaction[sig] = NULL;
+            }
+        }
+        if (info->shared_app_sigaction)
+            mutex_unlock(info->shared_lock);
+    }
+}
+
+/* Returns false if should NOT issue syscall */
+bool
+handle_sigaltstack(dcontext_t *dcontext, const stack_t *stack,
+                   stack_t *old_stack)
+{
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    if (old_stack != NULL) {
+        *old_stack = info->app_sigstack;
+    }
+    if (stack != NULL) {
+        info->app_sigstack = *stack;
+        LOG(THREAD, LOG_ASYNCH, 2, "app set up signal stack "PFX" - "PFX" %s\n",
+            stack->ss_sp, stack->ss_sp + stack->ss_size - 1,
+            (APP_HAS_SIGSTACK(info)) ? "enabled" : "disabled");
+        return false; /* always cancel syscall */
+    }
+    return true;
+}
+
+/* Blocked signals:
+ * In general, we don't need to keep track of blocked signals.
+ * We only need to do so for those signals we intercept ourselves.
+ * Thus, info->app_sigblocked ONLY contains entries for signals
+ * we intercept ourselves.
+ * PR 304708: we now intercept all signals.
+ */
+
+static void
+set_blocked(dcontext_t *dcontext, kernel_sigset_t *set)
+{
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    int i;
+    /* discard current blocked signals, re-set from new mask */
+    kernel_sigemptyset(&info->app_sigblocked);
+    for (i=0; i<MAX_SIGNUM; i++) {
+        if (info->we_intercept[i] && kernel_sigismember(set, i)) {
+            kernel_sigaddset(&info->app_sigblocked, i);
+        }
+    }
+#ifdef DEBUG
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_ASYNCH) != 0) {
+        LOG(THREAD, LOG_ASYNCH, 3, "blocked signals are now:\n");
+        dump_sigset(dcontext, &info->app_sigblocked);
+    }
+#endif
+}
+
+void
+handle_sigprocmask(dcontext_t *dcontext, int how, kernel_sigset_t *set,
+                   kernel_sigset_t *oset, size_t sigsetsize)
+{
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    int i;
+    LOG(THREAD, LOG_ASYNCH, 2, "handle_sigprocmask\n");
+    if (set != NULL) {
+        if (how == SIG_BLOCK) {
+            /* The set of blocked signals is the union of the current
+             * set and the set argument.
+             */
+            for (i=0; i<MAX_SIGNUM; i++) {
+                if (info->we_intercept[i] && kernel_sigismember(set, i)) {
+                    kernel_sigaddset(&info->app_sigblocked, i);
+                    kernel_sigdelset(set, i);
+                }
+            }
+        } else if (how == SIG_UNBLOCK) {
+            /* The signals in set are removed from the current set of
+             *  blocked signals.
+             */
+            for (i=0; i<MAX_SIGNUM; i++) {
+                if (info->we_intercept[i] && kernel_sigismember(set, i)) {
+                    kernel_sigdelset(&info->app_sigblocked, i);
+                    kernel_sigdelset(set, i);
+                }
+            }
+        } else if (how == SIG_SETMASK) {
+            /* The set of blocked signals is set to the argument set. */
+            kernel_sigemptyset(&info->app_sigblocked);
+            for (i=0; i<MAX_SIGNUM; i++) {
+                if (info->we_intercept[i] && kernel_sigismember(set, i)) {
+                    kernel_sigaddset(&info->app_sigblocked, i);
+                    kernel_sigdelset(set, i);
+                }
+            }
+        }
+#ifdef DEBUG
+        if (stats->loglevel >= 3 && (stats->logmask & LOG_ASYNCH) != 0) {
+            LOG(THREAD, LOG_ASYNCH, 3, "blocked signals are now:\n");
+            dump_sigset(dcontext, &info->app_sigblocked);
+        }
+#endif
+        /* make sure we deliver pending signals that are now unblocked
+         * FIXME: consider signal #S, which we intercept ourselves.
+         * If S arrives, then app blocks it prior to our delivering it,
+         * we then won't deliver it until app unblocks it...is this a
+         * problem?  Could have arrived a little later and then we would
+         * do same thing, but this way kernel may send one more than would
+         * get w/o dynamo?  This goes away if we deliver signals
+         * prior to letting app do a syscall.
+         */
+        if (!dcontext->signals_pending) {
+            for (i=0; i<MAX_SIGNUM; i++) {
+                if (info->sigpending[i] != NULL &&
+                    !kernel_sigismember(&info->app_sigblocked, i)) {
+                    /* since we're now in DR syscall handler, we know we'll
+                     * go back to dispatch and see this flag right away
+                     */
+                    dcontext->signals_pending = true;
+                    break;
+                }
+            }
+        }
+    }
+}
+
+/* need to add in our signals that the app thinks are blocked */
+void
+handle_post_sigprocmask(dcontext_t *dcontext, int how, kernel_sigset_t *set,
+                        kernel_sigset_t *oset, size_t sigsetsize)
+{
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    int i;
+    if (oset != NULL) {
+        for (i=0; i<MAX_SIGNUM; i++) {
+            if (info->we_intercept[i] &&
+                kernel_sigismember(&info->app_sigblocked, i)) {
+                kernel_sigaddset(oset, i);
+            }
+        }
+    }
+}
+
+void
+handle_sigsuspend(dcontext_t *dcontext, kernel_sigset_t *set,
+                  size_t sigsetsize)
+{
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    int i;
+    ASSERT(set != NULL);
+    LOG(THREAD, LOG_ASYNCH, 2, "handle_sigsuspend\n");
+    info->in_sigsuspend = true;
+    info->app_sigblocked_save = info->app_sigblocked;
+    kernel_sigemptyset(&info->app_sigblocked);
+    for (i=0; i<MAX_SIGNUM; i++) {
+        if (info->we_intercept[i] && kernel_sigismember(set, i)) {
+            kernel_sigaddset(&info->app_sigblocked, i);
+            kernel_sigdelset(set, i);
+        }
+    }
+#ifdef DEBUG
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_ASYNCH) != 0) {
+        LOG(THREAD, LOG_ASYNCH, 3, "in sigsuspend, blocked signals are now:\n");
+        dump_sigset(dcontext, &info->app_sigblocked);
+    }
+#endif
+}
+
+/**** utility routines ***********************************************/
+#ifdef DEBUG
+
+static void
+dump_fpstate(dcontext_t *dcontext, struct _fpstate *fp)
+{
+    int i,j;
+#ifdef X64
+    LOG(THREAD, LOG_ASYNCH, 1, "\tcwd="PFX"\n", fp->cwd);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tswd="PFX"\n", fp->swd);
+    LOG(THREAD, LOG_ASYNCH, 1, "\ttwd="PFX"\n", fp->twd);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tfop="PFX"\n", fp->fop);
+    LOG(THREAD, LOG_ASYNCH, 1, "\trip="PFX"\n", fp->rip);
+    LOG(THREAD, LOG_ASYNCH, 1, "\trdp="PFX"\n", fp->rdp);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tmxcsr="PFX"\n", fp->mxcsr);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tmxcsr_mask="PFX"\n", fp->mxcsr_mask);
+    for (i=0; i<8; i++) {
+        LOG(THREAD, LOG_ASYNCH, 1, "\tst%d = 0x", i);
+        for (j=0; j<4; j++) {
+            LOG(THREAD, LOG_ASYNCH, 1, "%08x", fp->st_space[i*4+j]);
+        }
+        LOG(THREAD, LOG_ASYNCH, 1, "\n");
+    }
+    for (i=0; i<16; i++) {
+        LOG(THREAD, LOG_ASYNCH, 1, "\txmm%d = 0x", i);
+        for (j=0; j<4; j++) {
+            LOG(THREAD, LOG_ASYNCH, 1, "%08x", fp->xmm_space[i*4+j]);
+        }
+        LOG(THREAD, LOG_ASYNCH, 1, "\n");
+    }
+#else
+    LOG(THREAD, LOG_ASYNCH, 1, "\tcw="PFX"\n", fp->cw);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tsw="PFX"\n", fp->sw);
+    LOG(THREAD, LOG_ASYNCH, 1, "\ttag="PFX"\n", fp->tag);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tipoff="PFX"\n", fp->ipoff);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tcssel="PFX"\n", fp->cssel);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tdataoff="PFX"\n", fp->dataoff);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tdatasel="PFX"\n", fp->datasel);
+    for (i=0; i<8; i++) {
+        LOG(THREAD, LOG_ASYNCH, 1, "\tst%d = ", i);
+        for (j=0; j<4; j++)
+            LOG(THREAD, LOG_ASYNCH, 1, "%04x ", fp->_st[i].significand[j]);
+        LOG(THREAD, LOG_ASYNCH, 1, "^ %04x\n", fp->_st[i].exponent);
+    }
+    LOG(THREAD, LOG_ASYNCH, 1, "\tstatus=0x%04x\n", fp->status);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tmagic=0x%04x\n", fp->magic);  
+
+    /* FXSR FPU environment */
+    for (i=0; i<6; i++) 
+        LOG(THREAD, LOG_ASYNCH, 1, "\tfxsr_env[%d] = "PFX"\n", i, fp->_fxsr_env[i]);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tmxcsr="PFX"\n", fp->mxcsr);
+    LOG(THREAD, LOG_ASYNCH, 1, "\treserved="PFX"\n", fp->reserved);
+    for (i=0; i<8; i++) {
+        LOG(THREAD, LOG_ASYNCH, 1, "\tfxsr_st%d = ", i);
+        for (j=0; j<4; j++)
+            LOG(THREAD, LOG_ASYNCH, 1, "%04x ", fp->_fxsr_st[i].significand[j]);
+        LOG(THREAD, LOG_ASYNCH, 1, "^ %04x\n", fp->_fxsr_st[i].exponent);
+        /* ignore padding */
+    }
+    for (i=0; i<8; i++) {
+        LOG(THREAD, LOG_ASYNCH, 1, "\txmm%d = ", i);
+        for (j=0; j<4; j++)
+            LOG(THREAD, LOG_ASYNCH, 1, "%04x ", fp->_xmm[i].element[j]);
+        LOG(THREAD, LOG_ASYNCH, 1, "\n");
+    }
+#endif
+    /* ignore padding */
+}
+
+static void
+dump_sigcontext(dcontext_t *dcontext, struct sigcontext *sc)
+{
+    LOG(THREAD, LOG_ASYNCH, 1, "\tgs=0x%04x"IF_NOT_X64(", __gsh=0x%04x")"\n",
+        sc->gs _IF_NOT_X64(sc->__gsh));
+    LOG(THREAD, LOG_ASYNCH, 1, "\tfs=0x%04x"IF_NOT_X64(", __fsh=0x%04x")"\n",
+        sc->fs _IF_NOT_X64(sc->__fsh));
+#ifndef X64
+    LOG(THREAD, LOG_ASYNCH, 1, "\tes=0x%04x, __esh=0x%04x\n", sc->es, sc->__esh);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tds=0x%04x, __dsh=0x%04x\n", sc->ds, sc->__dsh);
+#endif
+    LOG(THREAD, LOG_ASYNCH, 1, "\txdi="PFX"\n", sc->SC_XDI);
+    LOG(THREAD, LOG_ASYNCH, 1, "\txsi="PFX"\n", sc->SC_XSI);
+    LOG(THREAD, LOG_ASYNCH, 1, "\txbp="PFX"\n", sc->SC_XBP);
+    LOG(THREAD, LOG_ASYNCH, 1, "\txsp="PFX"\n", sc->SC_XSP);
+    LOG(THREAD, LOG_ASYNCH, 1, "\txbx="PFX"\n", sc->SC_XBX);
+    LOG(THREAD, LOG_ASYNCH, 1, "\txdx="PFX"\n", sc->SC_XDX);
+    LOG(THREAD, LOG_ASYNCH, 1, "\txcx="PFX"\n", sc->SC_XCX);
+    LOG(THREAD, LOG_ASYNCH, 1, "\txax="PFX"\n", sc->SC_XAX);
+#ifdef X64
+    LOG(THREAD, LOG_ASYNCH, 1, "\t r8="PFX"\n", sc->r8);
+    LOG(THREAD, LOG_ASYNCH, 1, "\t r9="PFX"\n", sc->r8);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tr10="PFX"\n", sc->r10);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tr11="PFX"\n", sc->r11);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tr12="PFX"\n", sc->r12);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tr13="PFX"\n", sc->r13);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tr14="PFX"\n", sc->r14);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tr15="PFX"\n", sc->r15);
+#endif
+    LOG(THREAD, LOG_ASYNCH, 1, "\ttrapno="PFX"\n", sc->trapno);
+    LOG(THREAD, LOG_ASYNCH, 1, "\terr="PFX"\n", sc->err);
+    LOG(THREAD, LOG_ASYNCH, 1, "\txip="PFX"\n", sc->SC_XIP);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tcs=0x%04x"IF_NOT_X64(", __esh=0x%04x")"\n",
+        sc->cs _IF_NOT_X64(sc->__csh));
+    LOG(THREAD, LOG_ASYNCH, 1, "\teflags="PFX"\n", sc->SC_XFLAGS);
+#ifndef X64
+    LOG(THREAD, LOG_ASYNCH, 1, "\tesp_at_signal="PFX"\n", sc->esp_at_signal);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tss=0x%04x, __ssh=0x%04x\n", sc->ss, sc->__ssh);
+#endif
+    if (sc->fpstate == NULL)
+        LOG(THREAD, LOG_ASYNCH, 1, "\tfpstate=<NULL>\n");
+    else
+        dump_fpstate(dcontext, sc->fpstate);
+    LOG(THREAD, LOG_ASYNCH, 1, "\toldmask="PFX"\n", sc->oldmask);
+    LOG(THREAD, LOG_ASYNCH, 1, "\tcr2="PFX"\n", sc->cr2);
+}
+
+static void
+dump_sigset(dcontext_t *dcontext, kernel_sigset_t *set)
+{
+    int sig;
+    for (sig=1; sig<MAX_SIGNUM; sig++) {
+        if (kernel_sigismember(set, sig))
+            LOG(THREAD, LOG_ASYNCH, 1, "\t%d = blocked\n", sig);
+    }
+}
+#endif /* DEBUG */
+
+/* FIXME: should copy xmm here too for client access; xref save_xmm() */
+void
+sigcontext_to_mcontext(dr_mcontext_t *mc, struct sigcontext *sc)
+{
+    ASSERT(mc != NULL && sc != NULL);
+    mc->xax = sc->SC_XAX;
+    mc->xbx = sc->SC_XBX;
+    mc->xcx = sc->SC_XCX;
+    mc->xdx = sc->SC_XDX;
+    mc->xsi = sc->SC_XSI;
+    mc->xdi = sc->SC_XDI;
+    mc->xbp = sc->SC_XBP;
+    mc->xsp = sc->SC_XSP;
+    mc->xflags = sc->SC_XFLAGS;
+    mc->pc = (app_pc) sc->SC_XIP;
+#ifdef X64
+    mc->r8  = sc->r8;
+    mc->r9  = sc->r9;
+    mc->r10 = sc->r10;
+    mc->r11 = sc->r11;
+    mc->r12 = sc->r12;
+    mc->r13 = sc->r13;
+    mc->r14 = sc->r14;
+    mc->r15 = sc->r15;
+#endif
+}
+
+/* FIXME: should copy xmm here too for client access; xref save_xmm() */
+void
+mcontext_to_sigcontext(struct sigcontext *sc, dr_mcontext_t *mc)
+{
+    sc->SC_XAX = mc->xax;
+    sc->SC_XBX = mc->xbx;
+    sc->SC_XCX = mc->xcx;
+    sc->SC_XDX = mc->xdx;
+    sc->SC_XSI = mc->xsi;
+    sc->SC_XDI = mc->xdi;
+    sc->SC_XBP = mc->xbp;
+    sc->SC_XSP = mc->xsp;
+    sc->SC_XFLAGS = mc->xflags;
+    sc->SC_XIP = (ptr_uint_t) mc->pc;
+#ifdef X64
+    sc->r8  = mc->r8;
+    sc->r9  = mc->r9;
+    sc->r10 = mc->r10;
+    sc->r11 = mc->r11;
+    sc->r12 = mc->r12;
+    sc->r13 = mc->r13;
+    sc->r14 = mc->r14;
+    sc->r15 = mc->r15;
+#endif
+}
+
+static void
+translate_sigcontext(dcontext_t *dcontext,  struct sigcontext *sc)
+{
+    dr_mcontext_t mcontext;
+ 
+    /* FIXME: what about floating-point state?  mmx regs? */
+    sigcontext_to_mcontext(&mcontext, sc);
+    /* FIXME: if cannot find exact match, we're in trouble!
+     * probably ok to delay, since that indicates not a synchronous
+     * signal. 
+     */
+    /* FIXME : in_fcache() (called by recreate_app_state) grabs fcache 
+     * fcache_unit_areas.lock, we could deadlock! Also on initexit_lock 
+     * == PR 205795/1317
+     */
+    /* For safe recreation we need to either be couldbelinking or hold the
+     * initexit lock (to keep someone from flushing current fragment), the
+     * initexit lock is easier
+     */
+    mutex_lock(&thread_initexit_lock);
+    /* PR 214962: we assume we're going to relocate to this stored context,
+     * so we restore memory now 
+     */
+    if (translate_mcontext(dcontext->thread_record, &mcontext, true/*restore memory*/)) {
+        mcontext_to_sigcontext(sc, &mcontext);
+    } else {
+        ASSERT_NOT_REACHED(); /* is ok to break things, is LINUX :) */
+        /* FIXME : what to do? reg state might be wrong at least get pc */
+        if (in_fcache((cache_pc)sc->SC_XIP)) {
+            sc->SC_XIP = (ptr_uint_t)recreate_app_pc(dcontext, mcontext.pc, NULL);
+            ASSERT(sc->SC_XIP != (ptr_uint_t)NULL);
+        } else {
+            /* FIXME : can't even get pc right, what do we do here? */
+            sc->SC_XIP = 0;
+        }
+    }
+    mutex_unlock(&thread_initexit_lock);
+    LOG(THREAD, LOG_ASYNCH, 3,
+        "\ttranslate_sigcontext: just set frame's eip to "PFX"\n", sc->SC_XIP);
+}
+
+/* Takes an os-specific context */
+void
+thread_set_self_context(void *cxt)
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    /* Unlike Windows we can't say "only set this subset of the
+     * full machine state", so we need to get the rest of the state,
+     */
+    sigframe_rt_t frame; /* for x64, 440 bytes */
+    struct sigcontext *sc = (struct sigcontext *) cxt;
+    app_pc xsp_for_sigreturn;
+#ifdef X64
+    struct _fpstate __attribute__ ((aligned (16))) fpstate; /* 512 bytes */
+    frame.uc.uc_mcontext.fpstate = &fpstate;
+#endif
+    memset(&frame, 0, sizeof(frame));
+    frame.uc.uc_mcontext = *sc;
+    save_fpstate(dcontext, &frame);
+    /* The kernel calls do_sigaltstack on sys_rt_sigreturn primarily to ensure
+     * the frame is ok, but the side effect is we can mess up our own altstack
+     * settings if we're not careful.  Having invalid ss_size looks good for
+     * kernel 2.6.23.9 at least so we leave frame.uc.uc_stack as all zeros.
+     */
+    /* make sure sigreturn's mask setting doesn't change anything */
+    sigprocmask_syscall(SIG_SETMASK, NULL, &frame.uc.uc_sigmask,
+                        sizeof(frame.uc.uc_sigmask));
+    LOG(THREAD_GET, LOG_ASYNCH, 2, "thread_set_self_context: pc="PFX"\n", sc->SC_XIP);
+    /* set up xsp to point at &frame + sizeof(char*) */
+    xsp_for_sigreturn = ((app_pc)&frame) + sizeof(char*);
+    asm("mov  %0, %%"ASM_XSP : : "m"(xsp_for_sigreturn));
+    asm("jmp dynamorio_sigreturn");
+    ASSERT_NOT_REACHED();
+}
+
+/* Takes a dr_mcontext_t */
+void
+thread_set_self_mcontext(dr_mcontext_t *mc)
+{
+    struct sigcontext sc;
+    mcontext_to_sigcontext(&sc, mc);
+    thread_set_self_context((void *)&sc);
+    ASSERT_NOT_REACHED();
+}
+
+static bool
+sig_has_restorer(thread_sig_info_t *info, int sig)
+{
+    if (info->app_sigaction[sig] == NULL)
+        return false;
+    if (TEST(SA_RESTORER, info->app_sigaction[sig]->flags))
+        return true;
+    if (info->app_sigaction[sig]->restorer == NULL)
+        return false;
+    /* we cache the result due to the safe_read cost */
+    if (info->restorer_valid[sig] == -1) {
+        /* With older kernels, don't seem to need flag: if sa_restorer !=
+         * NULL kernel will use it.  But with newer kernels that's not
+         * true, and sometimes libc does pass non-NULL.
+         */
+        /* Signal restorer code for Ubuntu 7.04:
+         *   0xffffe420 <__kernel_sigreturn+0>:      pop    %eax
+         *   0xffffe421 <__kernel_sigreturn+1>:      mov    $0x77,%eax
+         *   0xffffe426 <__kernel_sigreturn+6>:      int    $0x80
+         *   
+         *   0xffffe440 <__kernel_rt_sigreturn+0>:   mov    $0xad,%eax
+         *   0xffffe445 <__kernel_rt_sigreturn+5>:   int    $0x80
+         */
+        static const byte SIGRET_NONRT[8] =
+          {0x58, 0xb8, 0x77, 0x00, 0x00, 0x00, 0xcd, 0x80};
+        static const byte SIGRET_RT[8] = 
+          {0xb8, 0xad, 0x00, 0x00, 0x00, 0xcd, 0x80};
+        byte buf[MAX(sizeof(SIGRET_NONRT), sizeof(SIGRET_RT))]= {0};
+        if (safe_read(info->app_sigaction[sig]->restorer, sizeof(buf), buf) &&
+            ((IS_RT_FOR_APP(info, sig) &&
+              memcmp(buf, SIGRET_RT, sizeof(SIGRET_RT)) == 0) ||
+             (!IS_RT_FOR_APP(info, sig) &&
+              memcmp(buf, SIGRET_NONRT, sizeof(SIGRET_NONRT)) == 0))) {
+            LOG(THREAD_GET, LOG_ASYNCH, 2,
+                "sig_has_restorer %d: "PFX" looks like restorer, using w/o flag\n",
+                sig, info->app_sigaction[sig]->restorer);
+            info->restorer_valid[sig] = 1;
+        } else
+            info->restorer_valid[sig] = 0;
+    }
+    return (info->restorer_valid[sig] == 1);
+}
+
+/* Returns the size of the frame for delivering to the app.
+ * For x64 this does NOT include struct _fpstate.
+ */
+static uint
+get_app_frame_size(thread_sig_info_t *info, int sig)
+{
+    if (IS_RT_FOR_APP(info, sig))
+        return sizeof(sigframe_rt_t);
+    else
+        return sizeof(sigframe_plain_t);
+}
+
+static struct sigcontext *
+get_sigcontext_from_rt_frame(sigframe_rt_t *frame)
+{
+    return (struct sigcontext *) &(frame->uc.uc_mcontext);
+}
+
+static struct sigcontext *
+get_sigcontext_from_app_frame(thread_sig_info_t *info, int sig, void *frame)
+{
+    struct sigcontext *sc;
+    bool rtframe = IS_RT_FOR_APP(info, sig);
+    if (rtframe) {
+        sc = get_sigcontext_from_rt_frame((sigframe_rt_t *)frame);
+    } else {
+        sc = (struct sigcontext *) &(((sigframe_plain_t *)frame)->sc);
+    }
+    return sc;
+}
+
+static struct sigcontext *
+get_sigcontext_from_pending(thread_sig_info_t *info, int sig)
+{
+    ASSERT(info->sigpending[sig] != NULL);
+    return (struct sigcontext *) &(info->sigpending[sig]->rt_frame.uc.uc_mcontext);
+}
+
+/* Returns the address on the appropriate signal stack where we should copy
+ * the frame.  Includes spaces for fpstate for x64.
+ * If frame is NULL, assumes signal happened while in DR.
+ */
+static byte *
+get_sigstack_frame_ptr(dcontext_t *dcontext, int sig, sigframe_rt_t *frame)
+{
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    struct sigcontext *sc = (frame == NULL) ?
+        get_sigcontext_from_pending(info, sig) :
+        get_sigcontext_from_rt_frame(frame);
+    byte *sp;
+
+    if (frame != NULL) {
+        /* signal happened while in cache, grab interrupted xsp */
+        sp = (byte *) sc->SC_XSP;
+        LOG(THREAD, LOG_ASYNCH, 3,
+            "get_sigstack_frame_ptr: using frame's xsp "PFX"\n", sp);
+    } else {
+        /* signal happened while in DR, use stored xsp */
+        sp = (byte *) get_mcontext(dcontext)->xsp;
+        LOG(THREAD, LOG_ASYNCH, 3, "get_sigstack_frame_ptr: using app xsp "PFX"\n", sp);
+    }
+
+    if (APP_HAS_SIGSTACK(info)) {
+        /* app has own signal stack */
+        LOG(THREAD, LOG_ASYNCH, 3,
+            "get_sigstack_frame_ptr: app has own stack "PFX"\n",
+            info->app_sigstack.ss_sp);
+        LOG(THREAD, LOG_ASYNCH, 3,
+            "\tcur sp="PFX" vs app stack "PFX"-"PFX"\n",
+            sp, info->app_sigstack.ss_sp,
+            info->app_sigstack.ss_sp + info->app_sigstack.ss_size);
+        if (sp > (byte *)info->app_sigstack.ss_sp &&
+            sp - (byte *)info->app_sigstack.ss_sp < info->app_sigstack.ss_size) {
+            /* we're currently in the alt stack, so use current xsp */
+            LOG(THREAD, LOG_ASYNCH, 3,
+                "\tinside alt stack, so using current xsp "PFX"\n", sp);
+        } else {
+            /* need to go to top, stack grows down */
+            sp = info->app_sigstack.ss_sp + info->app_sigstack.ss_size - 1;
+            LOG(THREAD, LOG_ASYNCH, 3,
+                "\tnot inside alt stack, so using base xsp "PFX"\n", sp);
+        }
+    } 
+    /* now get frame pointer: need to go down to first field of frame */
+    sp = (byte *) (((ptr_uint_t)(sp - get_app_frame_size(info, sig)))
+                   & IF_X64_ELSE(-16ul, -8ul));
+#ifdef X64
+    sp -= X64_FRAME_EXTRA;
+#endif
+    /* PR 369907: don't forget the redzone */
+    sp -= REDZONE_SIZE;
+    return sp;
+}
+
+#ifndef X64
+static void
+convert_frame_to_nonrt(dcontext_t *dcontext, int sig, sigframe_rt_t *f_old,
+                       sigframe_plain_t *f_new)
+{
+    f_new->pretcode = f_old->pretcode;
+    f_new->sig = f_old->sig;
+    memcpy(&f_new->sc, &f_old->uc.uc_mcontext, sizeof(struct sigcontext));
+    memcpy(&f_new->fpstate, &f_old->fpstate, sizeof(struct _fpstate));
+    f_new->sc.oldmask = f_old->uc.uc_sigmask.sig[0];
+    memcpy(&f_new->extramask, &f_old->uc.uc_sigmask.sig[1],
+           (_NSIG_WORDS-1) * sizeof(uint));
+    memcpy(&f_new->retcode, &f_old->retcode, RETCODE_SIZE);
+    LOG(THREAD, LOG_ASYNCH, 3, "\tconverted rt frame to non-rt frame\n");
+    /* now fill in our extra field */
+    f_new->sig_noclobber = f_new->sig;
+}
+
+/* separated out to avoid the stack size cost on the common path */
+static void
+convert_frame_to_nonrt_partial(dcontext_t *dcontext, int sig, sigframe_rt_t *f_old,
+                               sigframe_plain_t *f_new, size_t size)
+{
+    sigframe_plain_t f_plain;
+    convert_frame_to_nonrt(dcontext, sig, f_old, &f_plain);
+    memcpy(f_new, &f_plain, size);
+}
+#endif
+
+/* Copies frame to sp.
+ * PR 304708: we now leave in rt form right up until we copy to the
+ * app stack, so that we can deliver to a client at a safe spot
+ * in rt form, so this routine now converts to a plain frame if necessary.
+ * If no restorer, touches up pretcode
+ * (and if rt_frame, touches up pinfo and puc)
+ * Also touches up fpstate pointer
+ */
+static void
+copy_frame_to_stack(dcontext_t *dcontext, int sig, sigframe_rt_t *frame, byte *sp)
+{
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    bool rtframe = IS_RT_FOR_APP(info, sig);
+    uint frame_size = get_app_frame_size(info, sig);
+    bool has_restorer = sig_has_restorer(info, sig);
+    byte *check_pc;
+    uint size = frame_size;
+#ifdef X64
+    size += X64_FRAME_EXTRA;
+    ASSERT(rtframe);
+#endif
+
+    LOG(THREAD, LOG_ASYNCH, 3, "copy_frame_to_stack: rt=%d, sp="PFX"\n", rtframe, sp);
+
+    /* before we write to the app's stack we need to see if it's writable */
+    check_pc = (byte *) ALIGN_BACKWARD(sp, PAGE_SIZE);
+    while (check_pc < (byte *)sp + size) {
+        uint prot;
+        DEBUG_DECLARE(bool ok = )
+            get_memory_info(check_pc, NULL, NULL, &prot);
+        ASSERT(ok);
+        if (!TEST(MEMPROT_WRITE, prot)) {
+            size_t rest = (byte *)sp + size - check_pc;
+            if (is_executable_area_writable(check_pc)) {
+                LOG(THREAD, LOG_ASYNCH, 2,
+                    "\tcopy_frame_to_stack: part of stack is unwritable-by-us @"PFX"\n",
+                    check_pc);
+                flush_fragments_and_remove_region(dcontext, check_pc, rest,
+                                                  false /* don't own initexit_lock */,
+                                                  false /* keep futures */);
+            } else {
+                LOG(THREAD, LOG_ASYNCH, 2,
+                    "\tcopy_frame_to_stack: part of stack is unwritable @"PFX"\n",
+                    check_pc);
+                /* copy what we can */
+                if (rtframe)
+                    memcpy(sp, frame, rest);
+#ifndef X64
+                else {
+                    convert_frame_to_nonrt_partial(dcontext, sig, frame,
+                                                   (sigframe_plain_t *) sp, rest);
+                }
+#endif
+                /* now throw exception
+                 * FIXME: what give as address?  what does kernel use?
+                 * If the app intercepts SIGSEGV then we'll come right back
+                 * here, so we terminate explicitly instead.  FIXME: set exit
+                 * code properly: xref PR 205310.
+                 */
+                if (info->app_sigaction[SIGSEGV] == NULL)
+                    os_forge_exception(0, UNREADABLE_MEMORY_EXECUTION_EXCEPTION);
+                else
+                    os_terminate(dcontext, TERMINATE_PROCESS);
+                ASSERT_NOT_REACHED();
+            }
+        }
+        check_pc += PAGE_SIZE;
+    }
+    if (rtframe)
+        memcpy(sp, frame, frame_size);
+#ifndef X64
+    else
+        convert_frame_to_nonrt(dcontext, sig, frame, (sigframe_plain_t *) sp);
+#endif
+
+    /* if !has_restorer we do NOT add the restorer code to the exec list here,
+     * to avoid removal problems (if handler never returns) and consistency problems
+     * (would have to mark as selfmod right now if on stack).
+     * for PROGRAM_SHEPHERDING we recognize as a pattern, and for consistency we
+     * allow entire region once try to execute -- not a performance worry since should
+     * very rarely be on the stack: should either be libc restorer code or with recent
+     * OS in rx vsyscall page.
+     */
+
+    /* fix up pretcode, pinfo, puc, fpstate */
+    if (rtframe) {
+        sigframe_rt_t *f_new = (sigframe_rt_t *) sp;
+        sigframe_rt_t *f_old = frame;
+        if (has_restorer)
+            f_new->pretcode = (char *) info->app_sigaction[sig]->restorer;
+        else {
+#ifdef X64
+            ASSERT_NOT_REACHED();
+#else
+            /* only point at retcode if old one was -- with newer OS, points at
+             * vsyscall page and there is no restorer, yet stack restorer code left
+             * there for gdb compatibility
+             */
+            if (f_old->pretcode == f_old->retcode)
+                f_new->pretcode = f_new->retcode;
+            else /* we set it to dynamorio_sigreturn in master_signal_handler */
+                ASSERT(f_new->pretcode = (char *) dynamorio_sigreturn);
+            LOG(THREAD, LOG_ASYNCH, 3, "\tleaving pretcode with old value\n");
+#endif
+        }
+#ifndef X64
+        f_new->pinfo = &(f_new->info);
+        f_new->puc = &(f_new->uc);
+        /* if fpstate ptr is not NULL, update it to new frame's fpstate struct */
+        if (f_new->uc.uc_mcontext.fpstate != NULL)
+            f_new->uc.uc_mcontext.fpstate = &f_new->fpstate;
+#else
+        if (f_old->uc.uc_mcontext.fpstate != NULL) {
+            byte *tgt = (byte *) ALIGN_FORWARD(sp + frame_size, 16);
+            ASSERT(tgt - (sp + frame_size) <= X64_FRAME_EXTRA);
+            memcpy(tgt, f_old->uc.uc_mcontext.fpstate, sizeof(struct _fpstate));
+            f_new->uc.uc_mcontext.fpstate = (struct _fpstate *) tgt;
+            LOG(THREAD, LOG_ASYNCH, 4, "\tfpstate old="PFX" new="PFX"\n",
+                f_old->uc.uc_mcontext.fpstate, f_new->uc.uc_mcontext.fpstate);
+        } else {
+            /* if fpstate is not set up, we're delivering signal immediately,
+             * and we shouldn't need an fpstate since DR code won't modify it;
+             * only if we delayed will we need it, and when delaying we make
+             * room and set up the pointer in copy_frame_to_pending
+             */
+            LOG(THREAD, LOG_ASYNCH, 4, "\tno fpstate needed\n");
+        }
+#endif
+        LOG(THREAD, LOG_ASYNCH, 3, "\tretaddr = "PFX"\n", f_new->pretcode);
+#ifdef RETURN_AFTER_CALL
+        info->signal_restorer_retaddr = (app_pc) f_new->pretcode;
+#endif
+        /* 32-bit kernel copies to aligned buf first */
+        IF_X64(ASSERT(ALIGNED(f_new->uc.uc_mcontext.fpstate, 16)));
+    } else {
+#ifdef X64
+        ASSERT_NOT_REACHED();
+#else
+        sigframe_plain_t *f_new = (sigframe_plain_t *) sp;
+        sigframe_plain_t *f_old = (sigframe_plain_t *) frame;
+        if (has_restorer)
+            f_new->pretcode = (char *) info->app_sigaction[sig]->restorer;
+        else {
+            /* see comments in rt case above */
+            if (f_old->pretcode == f_old->retcode)
+                f_new->pretcode = f_new->retcode;
+            else /* we set it to dynamorio_sigreturn in master_signal_handler */
+                f_new->pretcode = (char *) dynamorio_nonrt_sigreturn;
+            LOG(THREAD, LOG_ASYNCH, 3, "\tleaving pretcode with old value\n");
+        }
+        /* if fpstate ptr is not NULL, update it to new frame's fpstate struct */
+        if (f_new->sc.fpstate != NULL)
+            f_new->sc.fpstate = &f_new->fpstate;
+        LOG(THREAD, LOG_ASYNCH, 3, "\tretaddr = "PFX"\n", f_new->pretcode);
+# ifdef RETURN_AFTER_CALL
+        info->signal_restorer_retaddr = (app_pc) f_new->pretcode;
+# endif
+        /* 32-bit kernel copies to aligned buf so no assert on fpstate alignment */
+#endif
+    }
+}
+
+/* Copies frame to pending slot.
+ * PR 304708: we now leave in rt form right up until we copy to the
+ * app stack, so that we can deliver to a client at a safe spot
+ * in rt form.
+ */
+static void
+copy_frame_to_pending(dcontext_t *dcontext, int sig, sigframe_rt_t *frame)
+{
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    sigframe_rt_t *dst = &(info->sigpending[sig]->rt_frame);
+    LOG(THREAD, LOG_ASYNCH, 3, "copy_frame_to_pending\n");
+#ifdef DEBUG
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_ASYNCH) != 0) {
+        LOG(THREAD, LOG_ASYNCH, 3, "sigcontext:\n");
+        dump_sigcontext(dcontext, get_sigcontext_from_rt_frame(frame));
+    }
+#endif
+    memcpy(dst, frame, sizeof(*dst));
+#ifdef X64
+    /* we'll fill in updated fpstate at delivery time, but we go ahead and
+     * copy now in case our own retrieval somehow misses some fields
+     */
+    if (frame->uc.uc_mcontext.fpstate != NULL) {
+        memcpy(&info->sigpending[sig]->fpstate, frame->uc.uc_mcontext.fpstate,
+               sizeof(info->sigpending[sig]->fpstate));
+    }
+    /* we must set the pointer now so that later save_fpstate, etc. work */
+    dst->uc.uc_mcontext.fpstate = &info->sigpending[sig]->fpstate;        
+#else
+    dst->uc.uc_mcontext.fpstate = &dst->fpstate;
+#endif
+}
+
+/**** real work ***********************************************/
+
+#ifdef CLIENT_INTERFACE
+static dr_signal_action_t
+send_signal_to_client(dcontext_t *dcontext, int sig, sigframe_rt_t *frame)
+{
+    struct sigcontext *sc = (struct sigcontext *) &(frame->uc.uc_mcontext);
+    dr_siginfo_t si;
+    dr_signal_action_t action;
+    if (!dr_signal_hook_exists())
+        return DR_SIGNAL_DELIVER;
+    si.sig = sig;
+    si.drcontext = (void *) dcontext;
+    sigcontext_to_mcontext(&si.mcontext, sc);
+    action = instrument_signal(dcontext, &si);
+    if (action == DR_SIGNAL_DELIVER ||
+        action == DR_SIGNAL_REDIRECT) {
+        /* propagate client changes */
+        mcontext_to_sigcontext(sc, &si.mcontext);
+    }
+    return action;
+}
+#endif
+
+static void
+record_pending_signal(dcontext_t *dcontext, int sig, kernel_ucontext_t *ucxt,
+                      sigframe_rt_t *frame, bool forged)
+{
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    struct sigcontext *sc = (struct sigcontext *) &(ucxt->uc_mcontext);
+    byte *pc = (byte *) sc->SC_XIP;
+    bool receive_now = false;
+    bool blocked = false;
+    sigpending_t *pend;
+
+    if (info->in_sigsuspend) {
+        /* sigsuspend ends when a signal is received, so restore the
+         * old blocked set
+         */
+        info->app_sigblocked = info->app_sigblocked_save;
+        info->in_sigsuspend = false;
+#ifdef DEBUG
+        if (stats->loglevel >= 3 && (stats->logmask & LOG_ASYNCH) != 0) {
+            LOG(THREAD, LOG_ASYNCH, 3, "after sigsuspend, blocked signals are now:\n");
+            dump_sigset(dcontext, &info->app_sigblocked);
+        }
+#endif
+    }
+
+    if (info->app_sigaction[sig] != NULL &&
+        info->app_sigaction[sig]->handler == (handler_t)SIG_IGN
+        /* If a client registered a handler, put this in the queue.
+         * Races between registering, queueing, and delivering are fine.
+         */
+        IF_CLIENT_INTERFACE(&& !dr_signal_hook_exists())) {
+        LOG(THREAD, LOG_ASYNCH, 3,
+            "record_pending_signal (%d at pc "PFX"): action is SIG_IGN!\n",
+            sig, pc);
+        return;
+    } else if (kernel_sigismember(&info->app_sigblocked, sig)) {
+        /* signal is blocked by app, so just record it, don't receive now */
+        LOG(THREAD, LOG_ASYNCH, 2,
+            "record_pending_signal(%d at pc "PFX"): signal is currently blocked\n",
+            sig, pc);
+        blocked = true;
+    } else if (pc == 0 || /* special case -- see master_signal_handler FIXME */
+               /* PR 205795: to avoid lock problems w/ in_fcache (it grabs a lock, we
+                * could have interrupted someone holding that), we first check
+                * whereami --- if whereami is WHERE_FCACHE we still check the pc
+                * to distinguish generated routines, but at least we're certain
+                * it's not in DR where it could own a lock
+                */
+               (dcontext->whereami == WHERE_FCACHE && in_fcache(pc))) {
+        LOG(THREAD, LOG_ASYNCH, 2,
+            "record_pending_signal(%d) from cache pc "PFX"\n", sig, pc);
+        if (forged || can_always_delay[sig]) {
+            /* to make translation easier, want to delay if can until dispatch
+             * unlink cur frag, wait for dispatch
+             */
+            fragment_t wrapper;
+            fragment_t *f = fragment_pclookup(dcontext, pc, &wrapper);
+            ASSERT(f != NULL);
+            LOG(THREAD, LOG_ASYNCH, 2, "\tdelaying until exit F%d\n", f->id);
+            /* may not be linked if trace_relink or something */
+            if ((f->flags & FRAG_LINKED_OUTGOING) != 0) {
+                LOG(THREAD, LOG_ASYNCH, 3,
+                    "\tunlinking outgoing for interrupted F%d\n", f->id);
+                /* FIXME: this is same lock problem as mangle_syscall below...
+                 * and to fix this in same way would require not sharing
+                 * any bbs at all!!!
+                 * FIXME FIXME FIXME!!!
+                 */
+                SHARED_FLAGS_RECURSIVE_LOCK(f->flags, acquire, change_linking_lock);
+                unlink_fragment_outgoing(dcontext, f);
+                SHARED_FLAGS_RECURSIVE_LOCK(f->flags, release, change_linking_lock);
+                info->interrupted = f;
+            } else {
+                LOG(THREAD, LOG_ASYNCH, 3,
+                    "\toutgoing already unlinked for interrupted F%d\n", f->id);
+                /* either was unlinked for trace creation, or we got another
+                 * signal before exiting cache to handle 1st
+                 */
+                ASSERT(info->interrupted == NULL ||
+                       info->interrupted == f);
+            }
+            if ((f->flags & FRAG_HAS_SYSCALL) != 0) {
+                /* Syscalls are signal barriers!
+                 * Make sure the next syscall (if any) in f is not executed!
+                 * instead go back to dispatch right before the syscall
+                 */
+                /* syscall mangling does a bunch of decodes but only one write,
+                 * changing the target of a jmp, which should be atomic
+                 * except for cache lines -- FIXME, make the instr_encode use
+                 * a locked write?
+                 * Anyway, if that write is atomic, we just want to prevent
+                 * two people in here at same time, so we have to use linking lock --
+                 * but we CANNOT grab a lock like that on this path!
+                 * Possibility of livelock or deadlock.
+                 * Our solution is to not share bbs that contain syscalls.
+                 * They can still become traces, they just need to be private.
+                 */
+                ASSERT(!TEST(FRAG_SHARED, f->flags));
+                mangle_syscall_code(dcontext, f, pc, false/*do not skip exit cti*/);
+            }
+        } else {
+            /* the signal interrupted code cache => run handler now! */
+            receive_now = true;
+            LOG(THREAD, LOG_ASYNCH, 2, "\tnot certain can delay so handling now\n");
+        }
+    } else if (in_generated_routine(dcontext, pc)) {
+        /* Assumption: dynamo errors have been caught already inside
+         * the master_signal_handler, thus any error in a generated routine
+         * is an asynch signal that can be delayed
+         */
+        /* FIXME: dispatch on routine:
+         * if fcache_return, treat as dynamo
+         * if fcache_enter, unlink next frag, treat as dynamo
+         *   what if next frag has syscall in it?
+         * if indirect_branch_lookup prior to getting target...?!?
+         */
+        LOG(THREAD, LOG_ASYNCH, 2,
+            "record_pending_signal(%d) from gen routine "PFX"\n", sig, pc);
+        LOG(THREAD, LOG_ASYNCH, 1,
+            "WARNING: signal in gen routine: may cause problems!\n");
+    } else {
+        /* FIXME: what about routines called from code cache, like pre_system_call?
+         * want to unlink fragment and get back to dispatch...but syscall could
+         * take a while...and have to walk stack to find which fragment?
+         */
+        /* the signal interrupted dynamo => do not run handler now! */
+#ifdef DEBUG
+        /* FIXME: is_dynamo_address grabs too many locks, livelock can happen
+         * easily if any are interrupted, so we do not call it to distinguish
+         * DR addresses from libc or other libraries we use
+         */
+        LOG(THREAD, LOG_ASYNCH, 2,
+            "record_pending_signal(%d) from dynamo or lib at pc "PFX"\n", sig, pc);
+#endif
+    }
+
+    LOG(THREAD, LOG_ASYNCH, 3, "\taction is not SIG_IGN\n");
+    LOG(THREAD, LOG_ASYNCH, 3, "\tretaddr = "PFX"\n",
+        frame->pretcode); /* pretcode has same offs for plain */
+
+    if (receive_now) {
+        /* we need to translate sc before we know whether client wants to
+         * suppress, so we need a backup copy
+         */
+        struct sigcontext sc_orig = *sc;
+
+        ASSERT(!forged);
+        translate_sigcontext(dcontext, sc);
+
+        /* N.B.: since we abandon the old context for synchronous signals,
+         * we do not need to mark this fragment as FRAG_CANNOT_DELETE
+         */
+#ifdef DEBUG
+        if (stats->loglevel >= 2 && (stats->logmask & LOG_ASYNCH) != 0 &&
+            in_fcache(pc)) {
+            fragment_t wrapper;
+            fragment_t *f = fragment_pclookup(dcontext, pc, &wrapper);
+            ASSERT(f != NULL);
+            LOG(THREAD, LOG_ASYNCH, 2,
+                "Got signal at pc "PFX" in this fragment:\n", pc);
+            disassemble_fragment(dcontext, f, false);
+        }
+#endif
+
+        LOG(THREAD, LOG_ASYNCH, 2, "Going to receive signal now\n");
+        /* If we end up executing the default action, we'll go native
+         * since we translated the context.  If there's a handler,
+         * we'll copy the context to the app stack and then adjust the
+         * original on our stack so we take over.
+         */
+        execute_handler_from_cache(dcontext, sig, frame, &sc_orig);
+
+    } else {
+        /* Happened in DR, do not translate context.  Record for later processing
+         * at a safe point with a clean app state.
+         */
+        if (!blocked || sig >= OFFS_RT ||
+            (blocked && info->sigpending[sig] == NULL)) {
+            /* only have 1 pending for non-rt signals */
+
+            /* special heap alloc always uses sizeof(sigpending_t) blocks */
+            pend = special_heap_alloc(info->sigheap);
+            ASSERT(sig > 0 && sig < MAX_SIGNUM);
+            pend->next = info->sigpending[sig];
+            info->sigpending[sig] = pend;
+
+            /* FIXME: note that for asynchronous signals we don't need to
+             *  bother to record exact machine context, even entire frame,
+             *  since don't want to pass dynamo pc context to app handler.
+             *  only copy frame for synchronous signals?  those only
+             *  happen while in cache?  but for asynch, we would have to
+             *  construct our own frame...kind of a pain.  
+             */
+            copy_frame_to_pending(dcontext, sig, frame);
+        } else {
+            /* For clients, we document that we do not pass to them
+             * unless we're prepared to deliver to app.  We would have
+             * to change our model to pass them non-final-translated
+             * contexts in order to give them signals as soon as they
+             * come in.
+             */
+            LOG(THREAD, LOG_ASYNCH, 3,
+                "\tnon-rt signal already in queue, ignoring this one!\n");
+        }
+
+        if (!blocked)
+            dcontext->signals_pending = true;
+    }
+}
+
+#ifdef STACK_GUARD_PAGE
+static bool
+is_dstack_overflow(dcontext_t *dcontext, cache_pc instr_cache_pc,
+                   struct sigcontext *sc)
+{
+    instr_t instr;
+    /* initial sanity check though we don't know how long instr is */
+    if (!is_readable_without_exception(instr_cache_pc, 1))
+        return false;
+    instr_init(dcontext, &instr);
+    decode(dcontext, instr_cache_pc, &instr);
+    if (!instr_valid(&instr)) {
+        LOG(THREAD, LOG_ALL, 2,
+            "WARNING: got SIGSEGV for invalid instr at cache pc "PFX"\n", instr_cache_pc);
+        ASSERT_NOT_REACHED();
+        return false;
+    }
+    if (instr_writes_memory(&instr)) {
+        dr_mcontext_t mc;
+        app_pc target;
+        sigcontext_to_mcontext(&mc, sc);
+        target = instr_compute_address(&instr, &mc);
+        DOLOG(2, LOG_ALL, {
+            LOG(THREAD, LOG_ALL, 2,
+                "For SIGSEGV at cache pc "PFX", computed target "PFX"\n",
+                instr_cache_pc, target);
+            loginst(dcontext, 2, &instr, "\tfaulting write");
+        });
+        instr_free(dcontext, &instr);
+        return (is_stack_overflow(dcontext, target));
+    }
+    instr_free(dcontext, &instr);
+    return false;
+}
+#endif /* STACK_GUARD_PAGE */
+
+/* Distinguish SYS_kill-generated from instruction-generated signals.
+ * If sent from another process we can't tell, but if sent from this
+ * thread the interruption point should be our own post-syscall.
+ * FIXME PR 368277: for other threads in same process we should set a flag
+ * and identify them as well.
+ */
+static bool
+is_sys_kill(dcontext_t *dcontext, byte *pc, byte *xsp)
+{
+    return (is_at_do_syscall(dcontext, pc, xsp) &&
+            dcontext->sys_num == SYS_kill);
+}
+
+static bool
+check_for_modified_code(dcontext_t *dcontext, cache_pc instr_cache_pc,
+                        struct sigcontext *sc)
+{
+    /* special case: we expect a seg fault for executable regions
+     * that were writable and marked read-only by us.
+     * have to figure out the target address!
+     * unfortunately the OS doesn't tell us, nor whether it's a write.
+     * FIXME: if sent from SYS_kill(SIGSEGV), the pc will be post-syscall,
+     * and if that post-syscall instr is a write that could have faulted,
+     * how can we tell the difference?
+     */
+    instr_t instr;
+    instr_init(dcontext, &instr);
+    decode(dcontext, instr_cache_pc, &instr);
+    if (!instr_valid(&instr)) {
+        LOG(THREAD, LOG_ALL, 2,
+            "WARNING: got SIGSEGV for invalid instr at cache pc "PFX"\n", instr_cache_pc);
+        ASSERT_NOT_REACHED();
+        return false;
+    }
+    if (instr_writes_memory(&instr)) {
+        dr_mcontext_t mc;
+        app_pc target;
+        sigcontext_to_mcontext(&mc, sc);
+        target = instr_compute_address(&instr, &mc);
+        DOLOG(2, LOG_ALL, {
+            LOG(THREAD, LOG_ALL, 2,
+                "For SIGSEGV at cache pc "PFX", computed target "PFX"\n",
+                instr_cache_pc, target);
+            loginst(dcontext, 2, &instr, "\tfaulting write");
+        });
+        instr_free(dcontext, &instr);
+        if (was_executable_area_writable(target)) {
+            /* translate instr_cache_pc to original app pc
+             * DO NOT use translate_sigcontext, don't want to change the
+             * signal frame or else we'll lose control when we try to
+             * return to signal pc!
+             */
+            app_pc next_pc, translated_pc;
+            ASSERT((cache_pc)sc->SC_XIP == instr_cache_pc);
+            /* For safe recreation we need to either be couldbelinking or hold 
+             * the initexit lock (to keep someone from flushing current 
+             * fragment), the initexit lock is easier
+             */
+            mutex_lock(&thread_initexit_lock);
+            translated_pc = recreate_app_pc(dcontext, instr_cache_pc, NULL);
+            ASSERT(translated_pc != NULL);
+            mutex_unlock(&thread_initexit_lock);
+            next_pc =
+                handle_modified_code(dcontext, instr_cache_pc, translated_pc,
+                                     target);
+
+            /* going to exit from middle of fragment (at the write) so will mess up
+             * trace building
+             */
+            if (is_building_trace(dcontext)) {
+                LOG(THREAD, LOG_ASYNCH, 3, "\tsquashing trace-in-progress\n");
+                trace_abort(dcontext);
+            }
+
+            if (next_pc == NULL) {
+                /* re-execute the write -- just have master_signal_handler return */
+                return true;
+            } else {
+                /* Do not resume execution in cache, go back to dispatch.
+                 * Set our sigreturn context to point to fcache_return!
+                 * Then we'll go back through kernel, appear in fcache_return,
+                 * and go through dispatch & interp, without messing up dynamo stack.
+                 * Note that even if this is a write in the shared cache, we
+                 * still go to the private fcache_return for simplicity.
+                 */
+                sc->SC_XIP = (ptr_uint_t) fcache_return_routine(dcontext);
+                get_mcontext(dcontext)->xax = sc->SC_XAX;
+                sc->SC_XAX = (ptr_uint_t) get_selfmod_linkstub();
+                /* fcache_return will save rest of state */
+                dcontext->next_tag = next_pc;
+                LOG(THREAD, LOG_ASYNCH, 2,
+                    "\tset next_tag to "PFX", resuming in fcache_return\n",
+                    next_pc);
+                /* now have master_signal_handler return */
+                return true;
+            }
+        }
+    } else
+        instr_free(dcontext, &instr);
+    return false;
+}
+
+/* the master signal handler
+ * WARNING: behavior varies with different versions of the kernel!
+ * sigaction support was only added with 2.2
+ */
+#ifdef X64
+/* stub in x86.asm passes our xsp to us */
+void
+master_signal_handler_C(int sig, siginfo_t *siginfo, kernel_ucontext_t *ucxt,
+                        byte *xsp)
+#else
+static void
+master_signal_handler(int sig, siginfo_t *siginfo, kernel_ucontext_t *ucxt)
+#endif
+{
+#ifndef X64
+    /* get our frame base from the 1st arg, which is on the stack */
+    byte *xsp = (byte *) (&sig - 1);
+#endif
+    sigframe_rt_t *frame = (sigframe_rt_t *) xsp;
+#ifdef DEBUG
+    uint level = 2;
+# ifndef HAVE_PROC_MAPS
+    /* avoid logging every single TRY probe fault */
+    if (!dynamo_initialized)
+        level = 5;
+# endif
+#endif
+    bool local;
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    /* FIXME: ensure the path for recording a pending signal does not grab any DR locks
+     * that could have been interrupted
+     * e.g., synchronize_dynamic_options grabs the stats_lock!
+     */
+    if (dcontext == NULL) { /* FIXME: || !intercept_asynch, or maybe !under_our_control */
+        SYSLOG_INTERNAL_ERROR("ERROR: master_signal_handler w/ NULL dcontext: tid=%d, sig=%d",
+                              get_thread_id(), sig);
+        exit_syscall(1);
+    }
+
+    /* we may be entering dynamo from code cache! */
+    ENTERING_DR();
+    local = local_heap_protected(dcontext);
+    if (local)
+        SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+
+    LOG(THREAD, LOG_ASYNCH, level, "\nmaster_signal_handler: sig=%d, retaddr="PFX"\n",
+        sig, *((byte **)xsp));
+#ifndef X64
+    /* FIXME case 6700: 2.6.9 (FC3) kernel sets up our frame with a pretcode
+     * of 0x440.  This happens if our restorer is unspecified (though 2.6.9
+     * src code shows setting the restorer to a default value in that case...)
+     * or if we explicitly point at dynamorio_sigreturn.  I couldn't figure
+     * out why it kept putting 0x440 there.  So we fix the issue w/ this
+     * hardcoded return.
+     */
+    *((byte **)xsp) = (byte *) dynamorio_sigreturn;
+#endif
+
+    /* N.B.:
+     * ucontext_t is defined in two different places.  The one we get
+     * included is /usr/include/sys/ucontext.h, which would have us
+     * doing this:
+     *     void *pc = (void *) ucxt->uc_mcontext.gregs[EIP];
+     * However, EIP is not defined for us (used to be in older
+     * RedHat version) unless we define __USE_GNU, which we don't want to do
+     * for other reasons, so we'd have to also say:
+     *     #define EIP 14
+     * Instead we go by the ucontext_t definition in
+     * /usr/include/asm/ucontext.h, which has it containing a sigcontext struct,
+     * defined in /usr/include/asm/sigcontext.h.  This is the definition used
+     * by the kernel.  The two definitions are field-for-field
+     * identical except that the sys one has an fpstate struct at the end --
+     * but the next field in the frame is an fpstate.  The only mystery
+     * is why the rt frame is declared as ucontext instead of sigcontext.
+     * The kernel's version of ucontext must be the asm one!
+     * And the sys one grabs the next field of the frame.
+     * Also note that mcontext_t.fpregs == sigcontext.fpstate is NULL if
+     * floating point operations have not been used (lazy fp state saving).
+     * Also, sigset_t has different sizes according to kernel (8 bytes) vs.
+     * glibc (128 bytes?).
+     */
+
+    switch (sig) {
+
+    case SIGBUS: /* PR 313665: look for DR crashes on unaligned memory or mmap bounds */
+    case SIGSEGV: {
+        /* The kernel does NOT fill out the signal-specific fields of siginfo,
+         * except for SIGCHLD, thus we cannot do this:
+         *     void *pc = (void*) siginfo->si_addr;
+         * Thus we must use the third argument, which is a ucontext_t (see above)
+         */
+        /* FIXME PR 287309: we need to NOT suppress further SIGSEGV while in this
+         * handler, since we call decode() and need to handle faults due to app races
+         * or i/o paging.
+         */
+        struct sigcontext *sc = (struct sigcontext *) &(ucxt->uc_mcontext);
+        void *pc = (void *) sc->SC_XIP;
+        bool syscall_signal = false; /* signal came from syscall? */
+
+#ifdef SIDELINE
+        if (dcontext == NULL) {
+            SYSLOG_INTERNAL_ERROR("seg fault in sideline thread -- NULL dcontext!");
+            ASSERT_NOT_REACHED();
+        }
+#endif
+#ifdef STACK_GUARD_PAGE
+        if (sig == SIGSEGV && is_dstack_overflow(dcontext, pc, sc)) {
+            SYSLOG_INTERNAL_CRITICAL(PRODUCT_NAME" stack overflow at pc "PFX, pc);
+            /* options are already synchronized by the SYSLOG */
+            if (TEST(DUMPCORE_INTERNAL_EXCEPTION, dynamo_options.dumpcore_mask))
+                os_dump_core("stack overflow");
+            os_terminate(dcontext, TERMINATE_PROCESS);
+        }
+#endif /* STACK_GUARD_PAGE */
+        if (dcontext->try_except_state != NULL) {
+            /* handle our own TRY/EXCEPT */
+#ifdef HAVE_PROC_MAPS
+            /* our probe produces many of these every run */
+            SYSLOG_INTERNAL_WARNING("Handling our fault in a TRY at "PFX, pc);
+#endif
+            LOG(THREAD, LOG_ALL, level, "TRY fault at "PFX"\n", pc);
+            if (TEST(DUMPCORE_TRY_EXCEPT, DYNAMO_OPTION(dumpcore_mask)))
+                os_dump_core("try/except fault");
+
+            /* The exception interception code did an ENTER so we must EXIT here */
+            EXITING_DR();
+            /* Since we have no sigreturn we have to restore the mask manually,
+             * just like siglongjmp() 
+             */
+            sigprocmask_syscall(SIG_SETMASK,
+                                &dcontext->try_except_state->context.sigmask, NULL,
+                                sizeof(dcontext->try_except_state->context.sigmask));
+            DR_LONGJMP(&dcontext->try_except_state->context, LONGJMP_EXCEPTION);
+            ASSERT_NOT_REACHED();
+        }
+
+#ifdef CLIENT_INTERFACE
+        if (!IS_INTERNAL_STRING_OPTION_EMPTY(client_lib) && is_in_client_lib(pc)) {
+            char excpt_addr[IF_X64_ELSE(20,12)];
+            snprintf(excpt_addr, BUFFER_SIZE_ELEMENTS(excpt_addr), PFX, pc);
+            NULL_TERMINATE_BUFFER(excpt_addr);
+            SYSLOG_CUSTOM_NOTIFY(SYSLOG_ERROR, MSG_CLIENT_EXCEPTION, 3,
+                                 "Exception in client library.", 
+                                 get_application_name(), 
+                                 get_application_pid(),
+                                 excpt_addr);
+            /* kill process on a crash in client code */
+            os_terminate(dcontext, TERMINATE_PROCESS);
+        }
+#endif
+
+        /* FIXME: libc! 
+         * FIXME PR 205795: in_fcache and is_dynamo_address do grab locks!
+         */
+        if (!in_fcache(pc) && (is_dynamo_address(pc) ||
+                               in_generated_routine(dcontext, pc) ||
+                               is_at_do_syscall(dcontext, pc, (byte*)sc->SC_XSP))) {
+            /* kill(getpid(), SIGSEGV) looks just like a SIGSEGV in the store of eax
+             * to mcontext after the syscall instr in do_syscall -- try to distinguish:
+             */
+            if (is_sys_kill(dcontext, pc, (byte*)sc->SC_XSP)) {
+                LOG(THREAD, LOG_ALL, 2,
+                    "assuming SIGSEGV at post-do-syscall is kill, not our write fault\n");
+                syscall_signal = true;
+            }
+            if (!syscall_signal) {
+                char *where = in_generated_routine(dcontext, pc) ? " generated" : "";
+                SYSLOG(SYSLOG_CRITICAL, SIGSEGV_IN_SECURE_CORE, 7, 
+                       get_application_name(), get_application_pid(),
+                       (sig == SIGSEGV) ? "SEGV" : "BUS", where, 
+                       PRODUCT_NAME, pc, get_thread_id());
+                /* options are already synchronized by the SYSLOG */
+                if (TEST(DUMPCORE_INTERNAL_EXCEPTION, dynamo_options.dumpcore_mask))
+                    os_dump_core("sigsegv in secure core");
+                os_terminate(dcontext, TERMINATE_PROCESS);
+            }
+        }
+        /* if get here, pass the signal to the app */
+
+        /* FIXME: jmp to pc 0 => matches null_fragment => we jmp to 0!
+         * special case here, but is this best way to handle it, to have it
+         * match null_fragment?
+         */
+        if (sig == SIGSEGV && pc != 0 && !syscall_signal/*only for in-cache signals*/) {
+            /* special case: we expect a seg fault for executable regions
+             * that were writable and marked read-only by us.
+             */
+            if (check_for_modified_code(dcontext, pc, sc)) {
+                /* it was our signal, so don't pass to app -- return now */
+                break;
+            }
+        }
+        /* pass it to the application */
+        LOG(THREAD, LOG_ALL, 1,
+            "** Received SIG%s at cache pc "PFX" in thread %d\n",
+            (sig == SIGSEGV) ? "SEGV" : "BUS", pc, get_thread_id());
+        ASSERT(pc == 0 || syscall_signal || in_fcache(pc));
+        /* if we were building a trace, kill it */
+        if (is_building_trace(dcontext)) {
+            LOG(THREAD, LOG_ASYNCH, 3, "\tsquashing trace-in-progress\n");
+            trace_abort(dcontext);
+        }
+        record_pending_signal(dcontext, sig, ucxt, frame, false);
+        break;
+    }
+
+    /* PR 212090: the signal we use to suspend threads */
+    case SUSPEND_SIGNAL:
+        if (handle_suspend_signal(dcontext, ucxt))
+            record_pending_signal(dcontext, sig, ucxt, frame, false);
+        /* else, don't deliver to app */
+        break;
+
+#ifdef PAPI
+    /* use SIGPROF for updating gui so it can be distinguished from SIGVTALRM */
+    case SIGPROF: {
+        perfctr_update_gui();
+        break;
+    }
+#endif
+        
+    case SIGVTALRM: {
+        struct sigcontext *sc = (struct sigcontext *) &(ucxt->uc_mcontext);
+        void *pc = (void *) sc->SC_XIP;
+        /* FIXME: how tell if for us or user?
+         * we have to intercept setitimer, and have us give timer interrupts
+         * to app!
+         */
+        if (INTERNAL_OPTION(profile_pcs)) {
+            /* vtalarm only used with pc profiling.  it interferes w/ PAPI
+             * so arm this signal only if necessary
+             */
+            pcprofile_alarm(dcontext, pc, (app_pc) sc->SC_XBP);
+        } else
+            record_pending_signal(dcontext, sig, ucxt, frame, false);
+        break;
+    }
+
+#ifdef SIDELINE
+    case SIGCHLD: {
+        int status = siginfo->si_status;
+        if (siginfo->si_pid == 0) {
+            /* FIXME: with older versions of linux the sigchld fields of
+             * siginfo are not filled in properly!
+             * This is my attempt to handle that, pid seems to be 0
+             */
+            break;
+        }
+        if (status != 0) {
+            LOG(THREAD, LOG_ALL, 0, "*** Child thread died with error %d\n",
+                status);
+            ASSERT_NOT_REACHED();
+        }
+        break;
+    }
+#endif
+    
+    default: {
+        record_pending_signal(dcontext, sig, ucxt, frame, false);
+        break;
+    }
+    } /* end switch */
+    
+    LOG(THREAD, LOG_ASYNCH, 3, "\tmaster_signal_handler returning now\n\n");
+
+    /* restore protections */
+    if (local)
+        SELF_PROTECT_LOCAL(dcontext, READONLY);
+    EXITING_DR();
+}
+
+static void
+execute_handler_from_cache(dcontext_t *dcontext, int sig, sigframe_rt_t *our_frame,
+                           struct sigcontext *sc_orig)
+{
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    /* we want to modify the sc in DR's frame */
+    struct sigcontext *sc = get_sigcontext_from_rt_frame(our_frame);
+    kernel_sigset_t blocked;
+    /* Need to get xsp now before get new dcontext.
+     * This is the translated xsp, so we avoid PR 306410 (cleancall arg fault
+     * on dstack => handler run on dstack) that Windows hit.
+     */
+    byte *xsp = get_sigstack_frame_ptr(dcontext, sig,
+                                       our_frame/* take xsp from (translated)
+                                                 * interruption point */);
+
+#ifdef CLIENT_INTERFACE
+    dr_signal_action_t action = send_signal_to_client(dcontext, sig, our_frame);
+    /* in order to pass to the client, we come all the way here for signals
+     * the app has no handler for
+     */
+    if (action == DR_SIGNAL_REDIRECT) {
+        /* send_signal_to_client copied mcontext into our
+         * master_signal_handler frame, so we set up for fcache_return w/
+         * the mcontext state and this as next_tag
+         */
+        sigcontext_to_mcontext(get_mcontext(dcontext), sc);
+        dcontext->next_tag = (app_pc) sc->SC_XIP;
+        sc->SC_XIP = (ptr_uint_t) fcache_return_routine(dcontext);
+        sc->SC_XAX = (ptr_uint_t) get_sigreturn_linkstub();
+        if (is_building_trace(dcontext)) {
+            LOG(THREAD, LOG_ASYNCH, 3, "\tsquashing trace-in-progress\n");
+            trace_abort(dcontext);
+        }
+        return;
+    }
+    else if (action == DR_SIGNAL_SUPPRESS ||
+             (info->app_sigaction[sig] != NULL &&
+              info->app_sigaction[sig]->handler == (handler_t)SIG_IGN)) {
+        LOG(THREAD, LOG_ASYNCH, 2, "%s: not delivering!\n",
+            (action == DR_SIGNAL_SUPPRESS) ?
+            "client suppressing signal" :
+            "app signal handler is SIG_IGN");
+        /* restore original (untranslated) sc */
+        our_frame->uc.uc_mcontext = *sc_orig;
+        return;
+    }
+    else if (action == DR_SIGNAL_BYPASS ||
+             (info->app_sigaction[sig] == NULL ||
+              info->app_sigaction[sig]->handler == (handler_t)SIG_DFL)) {
+        LOG(THREAD, LOG_ASYNCH, 2, "%s: executing default action\n",
+            (action == DR_SIGNAL_BYPASS) ?
+            "client forcing default" :
+            "app signal handler is SIG_DFL");
+        execute_default_from_cache(dcontext, sig, our_frame);
+        /* if we haven't terminated, restore original (untranslated) sc */
+        our_frame->uc.uc_mcontext = *sc_orig;
+        return;
+    }
+    CLIENT_ASSERT(action == DR_SIGNAL_DELIVER, "invalid signal event return value");
+#else
+    if (info->app_sigaction[sig] == NULL ||
+        info->app_sigaction[sig]->handler == (handler_t)SIG_DFL) {
+        LOG(THREAD, LOG_ASYNCH, 3, "\taction is SIG_DFL\n");
+        execute_default_from_cache(dcontext, sig, our_frame);
+        /* if we haven't terminated, restore original (untranslated) sc */
+        our_frame->uc.uc_mcontext = *sc_orig;
+        return;
+    }
+    ASSERT(info->app_sigaction[sig] != NULL &&
+           info->app_sigaction[sig]->handler != (handler_t)SIG_IGN &&
+           info->app_sigaction[sig]->handler != (handler_t)SIG_DFL);
+#endif
+
+    LOG(THREAD, LOG_ASYNCH, 2, "execute_handler_from_cache for signal %d\n", sig);
+    RSTATS_INC(num_signals);
+
+    LOG(THREAD, LOG_ASYNCH, 3, "\txsp is "PFX"\n", xsp);
+
+    /* copy frame to appropriate stack and convert to non-rt if necessary */
+    copy_frame_to_stack(dcontext, sig, our_frame, (void *)xsp);
+    LOG(THREAD, LOG_ASYNCH, 3, "\tcopied frame from "PFX" to "PFX"\n", our_frame, xsp);
+
+    /* Because of difficulties determining when/if a signal handler
+     * returns, we do what the kernel does: abandon all of our current
+     * state, copy what we might need to the handler frame if we come back,
+     * and then it's ok if the handler doesn't return.
+     * If it does, we start interpreting afresh when we see sigreturn().
+     * This routine assumes anything needed to return has been put in the
+     * frame (only needed for signals queued up while in dynamo), and goes
+     * ahead and trashes the current dcontext.
+     */
+
+    /* if we were building a trace, kill it */
+    if (is_building_trace(dcontext)) {
+        LOG(THREAD, LOG_ASYNCH, 3, "\tsquashing trace-in-progress\n");
+        trace_abort(dcontext);
+    }
+
+    /* discard current blocked signals, re-set from sigaction mask */
+    blocked = info->app_sigaction[sig]->mask;
+    /* SA_NOMASK says whether to block sig itself or not */
+    if ((info->app_sigaction[sig]->flags & SA_NOMASK) == 0)
+        kernel_sigaddset(&blocked, sig);
+    set_blocked(dcontext, &blocked);
+
+    /* Set our sigreturn context (NOT for the app: we already copied the
+     * translated context to the app stack) to point to fcache_return!
+     * Then we'll go back through kernel, appear in fcache_return,
+     * and go through dispatch & interp, without messing up DR stack.
+     */
+    sc->SC_XIP = (ptr_uint_t) fcache_return_routine(dcontext);
+    sc->SC_XAX = (ptr_uint_t) get_sigreturn_linkstub();
+    /* Doesn't matter what most app registers are, signal handler doesn't
+     * expect anything except the frame on the stack.  We do need to set xsp,
+     * only because if app wants special signal stack we need to point xsp
+     * there.  (If no special signal stack, this is a nop.)
+     */
+    sc->SC_XSP = (ptr_uint_t) xsp;
+#ifdef X64
+    /* Set up args to handler: int sig, siginfo_t *siginfo, kernel_ucontext_t *ucxt */
+    sc->SC_XDI = sig;
+    sc->SC_XSI = (reg_t) &((sigframe_rt_t *)xsp)->info;
+    sc->SC_XDX = (reg_t) &((sigframe_rt_t *)xsp)->uc;
+#endif
+    /* Make sure handler is next thing we execute */       
+    dcontext->next_tag = (app_pc) info->app_sigaction[sig]->handler;
+
+    if ((info->app_sigaction[sig]->flags & SA_ONESHOT) != 0) {
+        /* clear handler now -- can't delete memory since sigreturn,
+         * others may look at sigaction struct, so we just set to default
+         */
+        info->app_sigaction[sig]->handler = (handler_t) SIG_DFL;
+    }
+
+    LOG(THREAD, LOG_ASYNCH, 3, "\tset next_tag to handler "PFX", xsp to "PFX"\n",
+        info->app_sigaction[sig]->handler, xsp);
+}
+
+static bool
+execute_handler_from_dispatch(dcontext_t *dcontext, int sig)
+{
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    handler_t handler;
+    byte *xsp = get_sigstack_frame_ptr(dcontext, sig, NULL);
+    sigframe_rt_t *frame = &(info->sigpending[sig]->rt_frame);
+    dr_mcontext_t *mcontext = get_mcontext(dcontext);
+    struct sigcontext *sc;
+    kernel_sigset_t blocked;
+
+#ifdef CLIENT_INTERFACE
+    dr_signal_action_t action;
+#else
+    if (info->app_sigaction[sig] == NULL ||
+        info->app_sigaction[sig]->handler == (handler_t)SIG_DFL) {
+        LOG(THREAD, LOG_ASYNCH, 3, "\taction is SIG_DFL\n");
+        execute_default_from_dispatch(dcontext, sig, frame);
+        return true;
+    }
+    ASSERT(info->app_sigaction[sig] != NULL &&
+           info->app_sigaction[sig]->handler != (handler_t)SIG_IGN &&
+           info->app_sigaction[sig]->handler != (handler_t)SIG_DFL);
+#endif
+
+    LOG(THREAD, LOG_ASYNCH, 2, "execute_handler_from_dispatch for signal %d\n", sig);
+    RSTATS_INC(num_signals);
+
+    /* modify the rtframe before copying to stack so we can pass final
+     * version to client, and propagate its mods
+     */
+    sc = get_sigcontext_from_rt_frame(frame);
+
+    /* Because of difficulties determining when/if a signal handler
+     * returns, we do what the kernel does: abandon all of our current
+     * state, copy what we might need to the handler frame if we come back,
+     * and then it's ok if the handler doesn't return.
+     * If it does, we start interpreting afresh when we see sigreturn().
+     */
+
+#ifdef DEBUG
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_ASYNCH) != 0) {
+        LOG(THREAD, LOG_ASYNCH, 3, "original sigcontext:\n");
+        dump_sigcontext(dcontext, sc);
+    }
+#endif
+    /* copy currently-interrupted-context to frame's context, so we can
+     * abandon the currently-interrupted context.
+     */
+    mcontext_to_sigcontext(sc, mcontext);
+    /* mcontext does not contain fp or mmx or xmm state, which may have
+     * changed since the frame was created (while finishing up interrupted
+     * fragment prior to returning to dispatch).  Since DR does not touch
+     * this state except for xmm on x64, we go ahead and copy the
+     * current state into the frame, and then touch up xmm for x64.
+     */
+    /* FIXME: should this be done for all pending as soon as reach
+     * dispatch?  what if get two asynch inside same frag prior to exiting
+     * cache?  have issues with fpstate, but also prob with next_tag? FIXME
+     */
+    /* FIXME: we should clear fpstate for app handler itself as that's
+     * how our own handler is executed.
+     */
+    save_fpstate(dcontext, frame);
+#ifdef DEBUG
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_ASYNCH) != 0) {
+        LOG(THREAD, LOG_ASYNCH, 3, "new sigcontext:\n");
+        dump_sigcontext(dcontext, sc);
+        LOG(THREAD, LOG_ASYNCH, 3, "\n");
+    }
+#endif
+    /* FIXME: other state?  debug regs?
+     * if no syscall allowed between master_ (when frame created) and
+     * receiving, then don't have to worry about debug regs, etc.
+     * check for syscall when record pending, if it exists, try to
+     * receive in pre_system_call or something? what if ignorable?  FIXME!
+     */
+
+    /* for the pc we want the app pc not the cache pc */
+    sc->SC_XIP = (ptr_uint_t) dcontext->next_tag;
+    LOG(THREAD, LOG_ASYNCH, 3, "\tset frame's eip to "PFX"\n", sc->SC_XIP);
+
+#ifdef CLIENT_INTERFACE
+    action = send_signal_to_client(dcontext, sig, frame);
+    /* in order to pass to the client, we come all the way here for signals
+     * the app has no handler for
+     */
+    if (action == DR_SIGNAL_REDIRECT) {
+        /* send_signal_to_client copied mcontext into frame's sc */
+        sigcontext_to_mcontext(get_mcontext(dcontext), sc);
+        dcontext->next_tag = (app_pc) sc->SC_XIP;
+        if (is_building_trace(dcontext)) {
+            LOG(THREAD, LOG_ASYNCH, 3, "\tsquashing trace-in-progress\n");
+            trace_abort(dcontext);
+        }
+        return true; /* don't try another signal */
+    }
+    else if (action == DR_SIGNAL_SUPPRESS ||
+        (info->app_sigaction[sig] != NULL &&
+         info->app_sigaction[sig]->handler == (handler_t)SIG_IGN)) {
+        LOG(THREAD, LOG_ASYNCH, 2, "%s: not delivering!\n",
+            (action == DR_SIGNAL_SUPPRESS) ?
+            "client suppressing signal" :
+            "app signal handler is SIG_IGN");
+        return false;
+    }
+    else if (action == DR_SIGNAL_BYPASS ||
+        (info->app_sigaction[sig] == NULL ||
+         info->app_sigaction[sig]->handler == (handler_t)SIG_DFL)) {
+        LOG(THREAD, LOG_ASYNCH, 2, "%s: executing default action\n",
+            (action == DR_SIGNAL_BYPASS) ?
+            "client forcing default" :
+            "app signal handler is SIG_DFL");
+        execute_default_from_dispatch(dcontext, sig, frame);
+        return true;
+    }
+    CLIENT_ASSERT(action == DR_SIGNAL_DELIVER, "invalid signal event return value");
+#endif
+
+    /* now that we've made all our changes and given the client a
+     * chance to make changes, copy the frame to the appropriate stack
+     * location and convert to non-rt if necessary 
+     */
+    copy_frame_to_stack(dcontext, sig, frame, xsp);
+    /* now point at the app's frame */
+    sc = get_sigcontext_from_app_frame(info, sig, (void *) xsp);
+
+    ASSERT(info->app_sigaction[sig] != NULL);
+    handler = info->app_sigaction[sig]->handler;
+
+    /* discard current blocked signals, re-set from sigaction mask */
+    blocked = info->app_sigaction[sig]->mask;
+    /* SA_NOMASK says whether to block sig itself or not */
+    if ((info->app_sigaction[sig]->flags & SA_NOMASK) == 0)
+        kernel_sigaddset(&blocked, sig);
+    set_blocked(dcontext, &blocked);
+
+    /* if we were building a trace, kill it */
+    if (is_building_trace(dcontext)) {
+        LOG(THREAD, LOG_ASYNCH, 3, "\tsquashing trace-in-progress\n");
+        trace_abort(dcontext);
+    }
+
+    /* Doesn't matter what most app registers are, signal handler doesn't
+     * expect anything except the frame on the stack.  We do need to set xsp.
+     */
+    mcontext->xsp = (ptr_uint_t) xsp;
+#ifdef X64
+    /* Set up args to handler: int sig, siginfo_t *siginfo, kernel_ucontext_t *ucxt */
+    mcontext->xdi = sig;
+    mcontext->xsi = (reg_t) &((sigframe_rt_t *)xsp)->info;
+    mcontext->xdx = (reg_t) &((sigframe_rt_t *)xsp)->uc;
+#endif
+    /* Make sure handler is next thing we execute */       
+    dcontext->next_tag = (app_pc) handler;
+
+    if ((info->app_sigaction[sig]->flags & SA_ONESHOT) != 0) {
+        /* clear handler now -- can't delete memory since sigreturn,
+         * others may look at sigaction struct, so we just set to default
+         */
+        info->app_sigaction[sig]->handler = (handler_t) SIG_DFL;
+    }
+
+    LOG(THREAD, LOG_ASYNCH, 3, "\tset xsp to "PFX"\n", xsp);
+    return true;
+}
+
+static void
+execute_default_action(dcontext_t *dcontext, int sig, sigframe_rt_t *frame,
+                       bool from_dispatch)
+{
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    struct sigcontext *sc = get_sigcontext_from_rt_frame(frame);
+    byte *pc = (byte *) sc->SC_XIP;
+
+    LOG(THREAD, LOG_ASYNCH, 3, "execute_default_action for signal %d\n", sig);
+
+    /* should only come here for signals we catch, or signal with ONESHOT
+     * that didn't sigreturn
+     */
+    ASSERT(info->we_intercept[sig] ||
+           (info->app_sigaction[sig]->flags & SA_ONESHOT) != 0);
+
+    if (info->app_sigaction[sig] != NULL &&
+        (info->app_sigaction[sig]->flags & SA_ONESHOT) != 0) {
+        if (!info->we_intercept[sig]) {
+            heap_free(dcontext, info->app_sigaction[sig], sizeof(kernel_sigaction_t)
+                      HEAPACCT(ACCT_OTHER));
+            info->app_sigaction[sig] = NULL;
+        }
+    }
+
+    /* FIXME PR 205310: we can't always perfectly emulate the default
+     * behavior.  To execute the default action, we have to un-register our
+     * handler, if we have one, for signals whose default action is not
+     * ignore or that will just be re-raised upon returning to the
+     * interrupted context -- FIXME: are any of the ignores repeated?
+     * SIGURG?
+     *
+     * If called from execute_handler_from_cache(), our master_signal_handler()
+     * is going to return directly to the translated context: which means we
+     * go native to re-execute the instr, which if it does in fact generate
+     * the signal again means we have a nice transparent core dump.
+     *
+     * If called from execute_handler_from_dispatch(), we need to generate
+     * the signal ourselves.
+     */
+    if (default_action[sig] != DEFAULT_IGNORE) {
+        DEBUG_DECLARE(bool ok =)
+            set_default_signal_action(sig);
+        ASSERT(ok);
+        /* FIXME: to avoid races w/ shared handlers should set a flag to
+         * prevent another thread from re-enabling.
+         * Perhaps worse: what if this signal arrives for another thread
+         * in the meantime (and the default is not terminate)?
+         */
+        if (info->shared_app_sigaction) {
+            LOG(THREAD, LOG_ASYNCH, 1,
+                "WARNING: having to install SIG_DFL for thread %d, but will be shared!\n",
+                get_thread_id());
+        }
+        if (default_action[sig] == DEFAULT_TERMINATE ||
+            default_action[sig] == DEFAULT_TERMINATE_CORE) {
+            /* N.B.: we don't have to restore our handler because the
+             * default action is for the process to die!
+             */
+            if (from_dispatch ||
+                can_always_delay[sig] ||
+                is_sys_kill(dcontext, pc, (byte*)sc->SC_XSP)) {
+                /* This must have come from SYS_kill rather than raised by
+                 * a faulting instruction.  Thus we can't go re-execute the
+                 * instr in order to re-raise the signal (if from_dispatch,
+                 * we delayed and can't re-execute anyway).  Instead we
+                 * re-generate via SYS_kill.  An alternative, if we don't
+                 * care about generating a core dump, is to use SYS_exit
+                 * and pass the right exit code to indicate the signal
+                 * number: that would avoid races w/ the sigaction.
+                 *
+                 * FIXME: should have app make the syscall to get a more
+                 * transparent core dump!
+                 */
+                KSTOP_NOT_MATCHING_NOT_PROPAGATED(dispatch_num_exits);
+                cleanup_and_terminate(dcontext, SYS_kill, get_process_id(), sig, true);
+                ASSERT_NOT_REACHED();
+            } else {
+                /* We assume that re-executing the interrupted instr will
+                 * re-raise the fault.  We could easily be wrong:
+                 * xref PR 363811 infinite loop due to memory we
+                 * thought was unreadable and thus thought would raise
+                 * a signal; xref PR 368277 to improve is_sys_kill().
+                 * FIXME PR 205310: we should check whether we come out of
+                 * the cache when we expected to terminate!
+                 *
+                 * An alternative is to abandon transparent core dumps and
+                 * do the same explicit SYS_kill we do for from_dispatch.
+                 * That would let us clean up DR as well.
+                 * FIXME: currently we do not clean up DR for a synchronous
+                 * signal death, but we do for asynch.
+                 */
+            }
+        } else {
+            /* FIXME PR 297033: in order to intercept DEFAULT_STOP /
+             * DEFAULT_CONTINUE signals, we need to set sigcontext to point
+             * to some kind of regain-control routine, so that when our
+             * thread gets to run again we can reset our handler.  So far
+             * we have no signals that fall here that we intercept.
+             */
+            CLIENT_ASSERT(false, "STOP/CONT signals not supported");
+        }
+#if defined(DEBUG) && defined(INTERNAL)
+        if (sig == SIGSEGV && !dynamo_exited) {
+            /* pc should be an app pc at this point (it was translated) --
+             * check for bad cases here
+             */
+            if (in_fcache(pc)) {
+                fragment_t wrapper;
+                fragment_t *f;
+                LOG(THREAD, LOG_ALL, 1,
+                    "Received SIGSEGV at pc "PFX" in thread %d\n", pc, get_thread_id());
+                f = fragment_pclookup(dcontext, pc, &wrapper);
+                if (f)
+                    disassemble_fragment(dcontext, f, false);
+                ASSERT_NOT_REACHED();
+            } else if (in_generated_routine(dcontext, pc)) {
+                LOG(THREAD, LOG_ALL, 1,
+                    "Received SIGSEGV at generated non-code-cache pc "PFX"\n", pc);
+                ASSERT_NOT_REACHED();
+            }
+        }
+#endif
+    }
+
+    /* now continue at the interruption point and re-raise the signal */
+}
+
+static void
+execute_default_from_cache(dcontext_t *dcontext, int sig, sigframe_rt_t *frame)
+{
+    execute_default_action(dcontext, sig, frame, false);
+}
+
+static void
+execute_default_from_dispatch(dcontext_t *dcontext, int sig, sigframe_rt_t *frame)
+{
+    execute_default_action(dcontext, sig, frame, true);
+}
+
+void
+receive_pending_signal(dcontext_t *dcontext)
+{
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    sigpending_t *temp;
+    int sig;
+    LOG(THREAD, LOG_ASYNCH, 3, "receive_pending_signal\n");
+    if (info->interrupted != NULL) {
+        LOG(THREAD, LOG_ASYNCH, 3, "\tre-linking outgoing for interrupted F%d\n",
+            info->interrupted->id);
+        SHARED_FLAGS_RECURSIVE_LOCK(info->interrupted->flags, acquire,
+                                    change_linking_lock);
+        link_fragment_outgoing(dcontext, info->interrupted, false);
+        SHARED_FLAGS_RECURSIVE_LOCK(info->interrupted->flags, release,
+                                    change_linking_lock);
+        info->interrupted = NULL;
+    }
+    /* grab first pending signal
+     * FIXME: start with real-time ones?
+     */
+    for (sig = 0; sig < MAX_SIGNUM; sig++) {
+        if (info->sigpending[sig] != NULL) {
+            bool executing = true;
+            if (kernel_sigismember(&info->app_sigblocked, sig)) {
+                LOG(THREAD, LOG_ASYNCH, 3, "\tsignal %d is blocked!\n", sig);
+                continue;
+            }
+            LOG(THREAD, LOG_ASYNCH, 3, "\treceiving signal %d\n", sig);
+            executing = execute_handler_from_dispatch(dcontext, sig);
+            temp = info->sigpending[sig];
+            info->sigpending[sig] = temp->next;
+            special_heap_free(info->sigheap, temp);
+
+            /* only one signal at a time! */
+            if (executing)
+                break;
+        }
+    }
+
+    /* we only clear this on a call to us where we find NO pending signals */
+    if (sig == MAX_SIGNUM) {
+        LOG(THREAD, LOG_ASYNCH, 3, "\tclearing signals_pending flag\n");
+        dcontext->signals_pending = false;
+    }
+}
+
+void
+handle_sigreturn(dcontext_t *dcontext, bool rt)
+{
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    struct sigcontext *sc;
+    int sig = 0;
+    app_pc next_pc;
+    /* xsp was put in mcontext prior to pre_system_call() */
+    reg_t xsp = get_mcontext(dcontext)->xsp;
+
+    LOG(THREAD, LOG_ASYNCH, 3, "%ssigreturn()\n", rt?"rt_":"");
+    LOG(THREAD, LOG_ASYNCH, 3, "\txsp is "PFX"\n", xsp);
+
+#ifdef PROGRAM_SHEPHERDING
+    /* if (!sig_has_restorer, region was never added to exec list,
+     * allowed as pattern only and kicked off at first write via
+     * selfmod detection or otherwise if vsyscall, so no worries
+     * about having to remove it here
+     */
+#endif
+
+    /* get sigframe: it's the top thing on the stack, except the ret
+     * popped off pretcode.
+     * WARNING: handler for tcsh's window_change (SIGWINCH) clobbers its
+     * signal # arg, so don't use frame->sig!  (kernel doesn't look at sig
+     * so app can get away with it)
+     */
+    if (rt) {
+        sigframe_rt_t *frame = (sigframe_rt_t *) (xsp - sizeof(char*));
+        /* use si_signo instead of sig, less likely to be clobbered by app */
+        sig = frame->info.si_signo;
+#ifndef X64
+        LOG(THREAD, LOG_ASYNCH, 3, "\tsignal was %d (did == param %d)\n", 
+            sig, frame->sig);
+        if (frame->sig != sig)
+            LOG(THREAD, LOG_ASYNCH, 1, "WARNING: app sig handler clobbered sig param\n");
+#endif
+        ASSERT(sig > 0 && sig < MAX_SIGNUM && IS_RT_FOR_APP(info, sig));
+        /* FIXME: what if handler called sigaction and requested rt
+         * when itself was non-rt?
+         */
+        sc = get_sigcontext_from_app_frame(info, sig, (void *) frame);
+        /* discard blocked signals, re-set from prev mask stored in frame */
+        set_blocked(dcontext, &frame->uc.uc_sigmask);
+    } else {
+        /* FIXME: libc's restorer pops prior to calling sigreturn, I have
+         * no idea why, but kernel asks for xsp-8 not xsp-4...weird!
+         */
+        kernel_sigset_t prevset;
+        sigframe_plain_t *frame = (sigframe_plain_t *) (xsp-8);
+        /* We don't trust frame->sig (app sometimes clobbers it), and for
+         * plain frame there's no other place that sig is stored,
+         * so as a hack we added a new frame!
+         * FIXME: this means we won't support nonstandard use of SYS_sigreturn,
+         * e.g., as NtContinue, if frame didn't come from a real signal and so
+         * wasn't copied to stack by us.
+         */
+        sig = frame->sig_noclobber;
+        LOG(THREAD, LOG_ASYNCH, 3, "\tsignal was %d (did == param %d)\n", 
+            sig, frame->sig);
+        if (frame->sig != sig)
+            LOG(THREAD, LOG_ASYNCH, 1, "WARNING: app sig handler clobbered sig param\n");
+        ASSERT(sig > 0 && sig < MAX_SIGNUM && !IS_RT_FOR_APP(info, sig));
+        sc = get_sigcontext_from_app_frame(info, sig, (void *) frame);
+        /* discard blocked signals, re-set from prev mask stored in frame */
+        prevset.sig[0] = frame->sc.oldmask;
+        if (_NSIG_WORDS > 1)
+            memcpy(&prevset.sig[1], &frame->extramask, sizeof(frame->extramask));
+        set_blocked(dcontext, &prevset);
+    }
+
+    /* We abandoned the previous context, so we need to start
+     * interpreting anew.  Regardless of whether we handled the signal
+     * from dispatch or the fcache, we want to go to the context
+     * stored in the frame.  So we have the kernel send us to
+     * fcache_return and set up for dispatch to use the frame's
+     * context.
+     */
+
+    /* if we were building a trace, kill it */
+    if (is_building_trace(dcontext)) {
+        LOG(THREAD, LOG_ASYNCH, 3, "\tsquashing trace-in-progress\n");
+        trace_abort(dcontext);
+    }
+
+    if ((info->app_sigaction[sig]->flags & SA_ONESHOT) != 0) {
+        ASSERT(info->app_sigaction[sig]->handler == (handler_t) SIG_DFL);
+        if (!info->we_intercept[sig]) {
+            /* let kernel do default independent of us */
+            heap_free(dcontext, info->app_sigaction[sig], sizeof(kernel_sigaction_t)
+                      HEAPACCT(ACCT_OTHER));
+            info->app_sigaction[sig] = NULL;
+        }
+    }
+
+    ASSERT(!in_fcache((app_pc) sc->SC_XIP));
+
+#ifdef DEBUG
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_ASYNCH) != 0) {
+        LOG(THREAD, LOG_ASYNCH, 3, "returning-to sigcontext:\n");
+        dump_sigcontext(dcontext, sc);
+    }
+#endif
+
+    /* set up for dispatch */
+    /* we have to use a different slot since next_tag ends up holding the do_syscall
+     * entry when entered from dispatch (we're called from pre_syscall, prior to entering cache)
+     */
+    dcontext->asynch_target = (app_pc) sc->SC_XIP;
+    next_pc = dcontext->asynch_target;
+    /* HACK to get eax put into mcontext AFTER do_syscall */
+    dcontext->next_tag = (app_pc) sc->SC_XAX;
+    /* use special linkstub so we know why we came out of the cache */
+    sc->SC_XAX = (ptr_uint_t) get_sigreturn_linkstub();
+
+    /* set our sigreturn context to point to fcache_return */
+    sc->SC_XIP = (ptr_uint_t) fcache_return_routine(dcontext);
+
+    /* if we overlaid inner frame on nested signal, will end up with this
+     * error -- disable in release build since this is often app's fault (stack
+     * too small)
+     * FIXME: how make this transparent?  what ends up happening is that we
+     * get a segfault when we start interpreting dispatch, we want to make it
+     * look like whatever would happen to the app...
+     */
+    ASSERT((app_pc)sc->SC_XIP != next_pc);
+
+    LOG(THREAD, LOG_ASYNCH, 3, "\tset next tag to "PFX", sc->SC_XIP to "PFX"\n",
+        next_pc, sc->SC_XIP);
+}
+
+bool
+is_signal_restorer_code(byte *pc, size_t *len)
+{
+    /* is this a sigreturn pattern placed by kernel on the stack or vsyscall page?
+     * for non-rt frame:
+     *    0x58           popl %eax
+     *    0xb8 <sysnum>  movl SYS_sigreturn, %eax
+     *    0xcd 0x80      int 0x80
+     * for rt frame:
+     *    0xb8 <sysnum>  movl SYS_rt_sigreturn, %eax
+     *    0xcd 0x80      int 0x80
+     */
+    /* optimized we only need two uint reads, but we have to do
+     * some little-endian byte-order reverses to get the right result
+     */
+#   define reverse(x) ((((x) & 0xff) << 24) | (((x) & 0xff00) << 8) | \
+                       (((x) & 0xff0000) >> 8) | (((x) & 0xff000000) >> 24))
+#ifndef X64
+    /* 58 b8 s4 s3 s2 s1 cd 80 */
+    static const uint non_rt_1w =  reverse(0x58b80000 | (reverse(SYS_sigreturn) >> 16));
+    static const uint non_rt_2w = reverse((reverse(SYS_sigreturn) << 16) | 0xcd80);
+#endif
+    /* b8 s4 s3 s2 s1 cd 80 XX */
+    static const uint rt_1w = reverse(0xb8000000 | (reverse(SYS_rt_sigreturn) >> 8));
+    static const uint rt_2w = reverse((reverse(SYS_rt_sigreturn) << 24) | 0x00cd8000);
+    /* test rt first as it's the most common 
+     * only 7 bytes here so we ignore the last one (becomes msb since little-endian)
+     */
+    if (*((uint *)pc) == rt_1w && (*((uint *)(pc+4)) & 0x00ffffff) == rt_2w) {
+        if (len != NULL)
+            *len = 7;
+        return true;
+    }
+#ifndef X64
+    if (*((uint *)pc) == non_rt_1w && *((uint *)(pc+4)) == non_rt_2w) {
+        if (len != NULL)
+            *len = 8;
+        return true;
+    }
+#endif
+    return false;
+}
+
+
+void
+os_forge_exception(app_pc target_pc, exception_type_t type)
+{
+    /* PR 205136:
+     * We want to deliver now, and the caller expects us not to return.
+     * We have two alternatives:
+     * 1) Emulate stack frame, and call transfer_to_dispatch() for delivery.  We
+     *    may not know how to fill out every field of the frame (cr2, etc.).  Plus,
+     *    we have problems w/ default actions (PR 205310) but we have to solve
+     *    those long-term anyway.  We also have to create different frames based on
+     *    whether app intercepts via rt or not.
+     * 2) Call SYS_tgkill from a special location that our handler can
+     *    recognize and know it's a signal meant for the app and that the
+     *    interrupted DR can be discarded.  We'd then essentially repeat 1,
+     *    but modifying the kernel-generated frame.  We'd have to always
+     *    intercept SIGILL.
+     * I'm going with #1 for now b/c the common case is simpler.
+     */
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+    sigframe_rt_t frame;
+    int sig;
+    switch (type) {
+    case ILLEGAL_INSTRUCTION_EXCEPTION: sig = SIGILL; break;
+    case UNREADABLE_MEMORY_EXECUTION_EXCEPTION: sig = SIGSEGV; break;
+    case IN_PAGE_ERROR_EXCEPTION: /* fall-through: Windows only */
+    default: ASSERT_NOT_REACHED(); sig = SIGSEGV; break;
+    }
+
+    /* since we always delay delivery, we always want an rt frame.  we'll convert
+     * to a plain frame on delivery.
+     */
+    memset(&frame, 0, sizeof(frame));
+    frame.info.si_signo = sig;
+#ifndef X64
+    frame.sig = sig;
+    frame.pinfo = &frame.info;
+    frame.puc = (void *) &frame.uc;
+    frame.uc.uc_mcontext.fpstate = &frame.fpstate;
+#endif
+    mcontext_to_sigcontext(&frame.uc.uc_mcontext, get_mcontext(dcontext));
+    frame.uc.uc_mcontext.SC_XIP = (reg_t) target_pc;
+    /* we'll fill in fpstate at delivery time
+     * FIXME: it seems to work w/o filling in the other state:
+     * I'm leaving segments, cr2, etc. all zero.
+     * Note that x64 kernel restore_sigcontext() only restores cs: it
+     * claims onus is on app's signal handler for other segments.
+     * We should try to share part of the GET_OWN_CONTEXT macro used for
+     * Windows.  Or we can switch to approach #2.
+     */
+    if (sig_has_restorer(info, sig))
+        frame.pretcode = (char *) info->app_sigaction[sig]->restorer;
+    else
+        frame.pretcode = (char *) dynamorio_sigreturn;
+
+    /* We assume that we do not need to translate the context when forged.
+     * If we did, we'd move this below enter_nolinking() (and update
+     * record_pending_signal() to do the translation).
+     */
+    record_pending_signal(dcontext, sig, &frame.uc, &frame, true/*forged*/);
+
+    /* For most callers this is not necessary and we only do it to match
+     * the Windows usage model: but for forging from our own handler,
+     * this is good b/c it resets us to the base of dstack.
+     */
+    /* tell dispatch() why we're coming there */
+    dcontext->whereami = WHERE_TRAMPOLINE;
+    KSTART(dispatch_num_exits);
+    /* we overload the meaning of the sigreturn linkstub */
+    set_last_exit(dcontext, (linkstub_t *) get_sigreturn_linkstub());
+    if (is_couldbelinking(dcontext))
+        enter_nolinking(dcontext, NULL, false);
+    transfer_to_dispatch(dcontext, dcontext->app_errno, get_mcontext(dcontext));
+    ASSERT_NOT_REACHED();
+}
+
+void
+os_request_fatal_coredump(const char *msg)
+{
+    set_default_signal_action(SIGSEGV);
+    SYSLOG_INTERNAL_ERROR("Crashing the process deliberately for a core dump!");
+    /* We try both the SIGKILL and the immediate crash since on some platforms
+     * the SIGKILL is delayed and on others the *-1 is hanging(?): should investigate
+     */
+    dynamorio_syscall(SYS_kill, 2, get_process_id(), SIGSEGV);
+    *((int *)PTR_UINT_MINUS_1) = 0;
+    /* To enable getting a coredump just make sure that rlimits are
+     * not preventing getting one, e.g. ulimit -c unlimited
+     */
+    return;
+}
+
+void
+os_request_live_coredump(const char *msg)
+{
+#ifdef VMX86_SERVER
+   if (os_in_vmkernel_userworld()) {
+      vmk_request_live_coredump(msg);
+      return;
+   }
+#endif
+   LOG(GLOBAL, LOG_ASYNCH, 1, "LiveCoreDump unsupported (PR 365105).  "
+       "Continuing execution without a core.\n");
+   return;
+}
+
+void
+os_dump_core(const char *msg)
+{
+    /* FIXME Case 3408: fork stack dump crashes on 2.6 kernel, so moving the getchar
+     * ahead to aid in debugging */
+    if (TEST(DUMPCORE_WAIT_FOR_DEBUGGER, dynamo_options.dumpcore_mask)) {
+        SYSLOG_INTERNAL_ERROR("looping so you can use gdb to attach to pid %s",
+                              get_application_pid());
+        IF_CLIENT_INTERFACE(SYSLOG(SYSLOG_CRITICAL, WAITING_FOR_DEBUGGER, 2,
+                                   get_application_name(), get_application_pid()));
+        /* getchar() can hit our own vsyscall hook (from PR 212570); typically we
+         * want to attach and not continue anyway, so doing an infinite loop:
+         */
+        while (true)
+            thread_yield();
+    }
+
+    if (DYNAMO_OPTION(live_dump)) {
+        os_request_live_coredump(msg);
+    }
+
+    /* fork, dump core, then use gdb to get a stack dump
+     * we can get into an infinite loop if there's a seg fault
+     * in the process of doing this -- so we have a do-once test,
+     * and if it failed we do the no-symbols dr callstack dump
+     */
+    static bool tried_stackdump = false;
+    if (!tried_stackdump) {
+        tried_stackdump = true;
+        stackdump();
+    } else {
+        static bool tried_calldump = false;
+        if  (!tried_calldump) {
+            tried_calldump = true;
+            dump_dr_callstack(STDERR);
+        }
+    }
+    if (!DYNAMO_OPTION(live_dump)) {
+        os_request_fatal_coredump(msg);
+        ASSERT_NOT_REACHED();
+    }
+}
+
+#ifdef RETURN_AFTER_CALL
+bool
+at_known_exception(dcontext_t *dcontext, app_pc target_pc, app_pc source_fragment)
+{
+    /* There is a known exception in signal restorers and the Linux dynamic symbol resoulution */
+    /* The latter we assume it is the only other recurring known exception, 
+       so the first time we pattern match to help make sure it is indeed _dl_runtime_resolve
+       (since with LD_BIND_NOW it will never be called).  After that we compare with the known value. */
+
+    static app_pc known_exception = 0;
+    thread_sig_info_t *info = (thread_sig_info_t *) dcontext->signal_field;
+
+    LOG(THREAD, LOG_INTERP, 1, "RCT: testing for KNOWN exception "PFX" "PFX"\n",
+        target_pc, source_fragment);
+
+    /* Check if this is a signal return.   
+       FIXME: we should really get that from the frame itself.
+       Since currently grabbing restorer only when copying a frame,
+       this will work with nested signals only if they all have same restorer
+       (I haven't seen restorers other than the one in libc)
+    */
+    if (target_pc == info->signal_restorer_retaddr) {
+        LOG(THREAD, LOG_INTERP, 1, "RCT: KNOWN exception this is a signal restorer --ok \n");
+        STATS_INC(ret_after_call_signal_restorer);
+        return true;
+    }
+
+    if (source_fragment == known_exception) {
+        LOG(THREAD, LOG_INTERP, 1, "RCT: KNOWN exception again _dl_runtime_resolve --ok\n");
+        return true;
+    }
+
+    if (known_exception == 0) {
+        /* It works for the LINUX loader hack in  _dl_runtime_resolve */
+        /* The offending sequence in ld-linux.so is
+           <_dl_runtime_resolve>:
+           c270: 5a                      pop    %edx
+           c271: 59                      pop    %ecx
+           c272: 87 04 24                xchg   %eax,(%esp)
+           c275: c2 08 00                ret    $0x8
+        */
+        /* The same code also is in 0000c280 <_dl_runtime_profile>
+           It maybe that either one or the other is ever used. 
+           Although performancewise this pattern matching is very cheap, 
+           for stricter security we assume only one is used in a session.
+        */
+        /* FIXME: This may change with future versions of libc, tested on
+         * RH8 and RH9 only.  Also works for whatever libc was in ubuntu 7.10.
+         */
+        /* However it does not work for ubuntu 8.04 where the code sequence has
+         * changed to the still similar :
+         * 2c50:  5a                   pop    %edx 
+         * 2c51:  8b 0c 24             mov    (%esp) -> %ecx 
+         * 2c54:  89 04 24             mov    %eax -> (%esp) 
+         * 2c57:  8b 44 24 04          mov    0x04(%esp) -> %eax 
+         * 2c5b:  c2 0c 00             ret    $0xc
+         * So we check for that sequence too.
+         */
+        static const byte DL_RUNTIME_RESOLVE_MAGIC_1[8] =
+          /* pop edx, pop ecx; xchg eax, (esp) ret 8 */
+          {0x5a, 0x59, 0x87, 0x04, 0x24, 0xc2, 0x08, 0x00};
+        static const byte DL_RUNTIME_RESOLVE_MAGIC_2[14] = 
+          /* pop edx, mov (esp)->ecx, mov eax->(esp), mov 4(esp)->eax, ret 12 */
+          {0x5a, 0x8b, 0x0c, 0x24, 0x89, 0x04, 0x24, 0x8b, 0x44, 0x24,
+           0x04, 0xc2, 0x0c, 0x00};
+        byte buf[MAX(sizeof(DL_RUNTIME_RESOLVE_MAGIC_1),
+                     sizeof(DL_RUNTIME_RESOLVE_MAGIC_2))]= {0};
+
+        if ((safe_read(source_fragment, sizeof(DL_RUNTIME_RESOLVE_MAGIC_1), buf)
+             && memcmp(buf, DL_RUNTIME_RESOLVE_MAGIC_1,
+                       sizeof(DL_RUNTIME_RESOLVE_MAGIC_1)) == 0) ||
+            (safe_read(source_fragment, sizeof(DL_RUNTIME_RESOLVE_MAGIC_2), buf)
+             && memcmp(buf, DL_RUNTIME_RESOLVE_MAGIC_2,
+                       sizeof(DL_RUNTIME_RESOLVE_MAGIC_2)) == 0)) {
+            LOG(THREAD, LOG_INTERP, 1, "RCT: KNOWN exception this is "
+                "_dl_runtime_resolve --ok \n");
+            known_exception = source_fragment;
+            return true;
+        } else {
+            return false;
+        }
+    }
+    return false;
+}
+#endif /* RETURN_AFTER_CALL */
+
+void
+start_itimer(dcontext_t *dcontext)
+{
+    int rc;
+
+    struct itimerval t;
+    t.it_interval.tv_sec = 0;
+    t.it_interval.tv_usec = 10000;
+    t.it_value.tv_sec = 0;
+    t.it_value.tv_usec = 10000;
+    rc = setitimer_syscall(ITIMER_VIRTUAL, &t, NULL);
+    ASSERT(rc == SUCCESS);
+}
+
+void
+stop_itimer()
+{
+    int rc;
+    struct itimerval t;
+    t.it_interval.tv_sec = 0;
+    t.it_interval.tv_usec = 0;
+    t.it_value.tv_sec = 0;
+    t.it_value.tv_usec = 0;
+    rc = setitimer_syscall(ITIMER_VIRTUAL, &t, NULL);
+    ASSERT(rc == SUCCESS);
+}
+
+#ifdef PAPI
+void start_PAPI_timer()
+{
+    int rc;
+    struct itimerval t;
+    t.it_interval.tv_sec = 0;
+    t.it_interval.tv_usec = 500000;
+    t.it_value.tv_sec = 0;
+    t.it_value.tv_usec = 500000;
+
+    /* use realtime timer for papi updates */
+    rc = setitimer_syscall(ITIMER_PROF, &t, NULL);
+    ASSERT(rc == SUCCESS);
+}
+
+void stop_PAPI_timer()
+{
+    int rc;
+    struct itimerval t;
+    t.it_interval.tv_sec = 0;
+    t.it_interval.tv_usec = 0;
+    t.it_value.tv_sec = 0;
+    t.it_value.tv_usec = 0;
+
+    /* use realtime timer for papi updates */
+    rc = setitimer_syscall(ITIMER_PROF, &t, NULL);
+    ASSERT(rc == SUCCESS);
+}
+#endif /* PAPI */
+
+/* Returns whether to pass on to app */
+static bool
+handle_suspend_signal(dcontext_t *dcontext, kernel_ucontext_t *ucxt)
+{
+    os_thread_data_t *ostd = (os_thread_data_t *) dcontext->os_field;
+    struct sigcontext *sc = (struct sigcontext *) &(ucxt->uc_mcontext);
+    kernel_sigset_t prevmask;
+    ASSERT(ostd != NULL);
+    /* If suspend_count is 0, we are not trying to suspend this thread
+     * (thread_resume() may have already decremented suspend_count to 0, but
+     * thread_suspend() will not send a signal until this thread unsets
+     * ostd->suspended, so not having a lock around the suspend_count read is
+     * ok), so pass signal to app.
+     * If we are trying or have already suspended this thread, our own
+     * thread_suspend() will not send a 2nd suspend signal until we are
+     * completely resumed, so we can distinguish app uses of SUSPEND_SIGNAL.  We
+     * can't have a race between the read and write of suspended_sigcxt b/c
+     * signals are blocked.  It's fine to have a race and reorder the app's
+     * signal w/ DR's.
+     */
+    if (ostd->suspend_count == 0 || ostd->suspended_sigcxt != NULL)
+        return true; /* pass to app */
+
+    ostd->suspended_sigcxt = sc;
+
+    /* We're sitting on our sigaltstack w/ all signals blocked.  We're
+     * going to stay here but unblock all signals so we don't lose any
+     * delivered while we're waiting.  We're at a safe enough point to
+     * re-enter master_signal_handler().  We use a mutex in
+     * thread_{suspend,resume} to prevent our own re-suspension signal
+     * from arriving before we've re-blocked on the resume.
+     */
+    set_blocked(dcontext, &ucxt->uc_sigmask);
+    sigprocmask_syscall(SIG_SETMASK, &ucxt->uc_sigmask, &prevmask,
+                        sizeof(ucxt->uc_sigmask));
+
+    LOG(THREAD, LOG_ASYNCH, 2, "handle_suspend_signal: suspended now\n");
+    /* We cannot use mutexes here as we have interrupted DR at an
+     * arbitrary point!  Thus we can't use the event_t routines.
+     * However, the existing synch and check above prevent any
+     * re-entrance here, and our cond vars target just a single thread,
+     * so we can get away w/o a mutex.
+     */
+    /* Notify thread_suspend that it can now return, as this thread is
+     * officially suspended now and is ready for thread_{get,set}_mcontext.
+     */
+    ASSERT(!ostd->suspended);
+    ostd->suspended = true;
+    /* FIXME PR 295561: use futex */
+    while (!ostd->wakeup)
+        thread_yield();
+    LOG(THREAD, LOG_ASYNCH, 2, "handle_suspend_signal: awake now\n");
+
+    /* re-block so our exit from master_signal_handler is not interrupted */
+    set_blocked(dcontext, &prevmask);
+    sigprocmask_syscall(SIG_SETMASK, &prevmask, NULL, sizeof(prevmask));
+    ostd->suspended_sigcxt = NULL;
+
+    /* Notify thread_resume that it can return now, which (assuming
+     * suspend_count is back to 0) means it's then safe to re-suspend. 
+     */
+    ostd->suspended = false; /* reset prior to signalling thread_resume */
+    ostd->resumed = true;
+
+    return false; /* do not pass to app */
+}
+
+/* PR 206278: for try/except we need to save the signal mask */
+void
+dr_setjmp_sigmask(dr_jmp_buf_t *buf)
+{
+    sigprocmask_syscall(SIG_SETMASK, NULL, &buf->sigmask, sizeof(buf->sigmask));
+}
diff --git a/core/linux/stackdump.c b/core/linux/stackdump.c
new file mode 100644
index 000000000..e65ae8214
--- /dev/null
+++ b/core/linux/stackdump.c
@@ -0,0 +1,314 @@
+/* **********************************************************
+ * Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+
+/*
+ * stackdump.c - support for dumping stack trace of DynamoRIO
+ * 
+ * based on code from David Spuler and idea from Castor Fu
+ * basic idea:
+ *   fork off a process which core dumps then run a debugger on it.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/file.h>
+#include <sys/wait.h>
+#include <string.h>
+#include <sys/syscall.h>
+
+/* for glibc_stackdump */
+#include <execinfo.h>
+#include <stdio.h>
+
+#include "globals.h"
+#include "os_private.h"
+
+/* use backtrace feature of glibc 
+ * only gives symbol names, and sometimes misses them
+ */
+void glibc_stackdump(int fd)
+{
+    void *rets[128];
+    int depth = backtrace(rets, 128);
+    backtrace_symbols_fd(rets, depth, fd);
+}
+
+
+#define DEBUGGER "gdb"
+/* add -q to suppress gdb copyright notice */
+#define QUIET_MODE "-q"
+#define TEMPORARY_FILENAME "/tmp/dynamorio.stackdump"
+#define CORE_NAME "core"
+
+#define VERBOSE 0
+
+/* use gdb -x <file> -batch, or redirect stdin?
+ * redirecting stdin has issues when where is multi-page, but
+ * batch will wait for stdin on multi-page, so we solve that
+ * by adding extra <enter>s in our temp file
+ */
+#define BATCH_MODE 0
+#if BATCH_MODE
+#define DEBUGGER_COMMAND "where\nquit\n"
+#else
+/* FIXME: want to have some <enter>s to handle multi-page,
+ * but don't want to repeat where cmd, so I use pwd, which is
+ * useful.  Hopefully two pages is enough.
+ */
+#define DEBUGGER_COMMAND "where\npwd\nquit\n"
+#endif
+
+pid_t
+wait_syscall(int *status)
+{
+    return dynamorio_syscall(SYS_wait4, 4, WAIT_ANY, status, 0, NULL);
+}
+
+int
+fork_syscall()
+{
+#if FORK_BROKEN_CASE_4967
+    /* FIXME: SYS_fork on dereksha is creating a child whose pid is
+     * same as parent but has a different tid, and the abort() to dump
+     * core kills the parent process -- looks just like a separate
+     * thread, not a separate process!  
+     *
+     * When I use glibc fork() I get the proper behavior.
+     * glibc 2.3.3 fork() calls clone() with flags = 0x01200011
+     * == CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID | SIGCHLD, which matches
+     * glibc-2.3.3-200405070341/nptl/sysdeps/unix/sysv/linux/i386/fork.c's
+     * ARCH_FORK, which is what glibc uses, so why doesn't SYS_fork do
+     * same thing?  Instead it simply sets SIGCHLD and not the CLONE_* flags.
+     * (see /usr/src/linux/arch/i386/kernel/process.c).  But, trying the CLONE_* flags
+     * doesn't do the trick -- libc fork() is doing something extra, and
+     * glibc-2.3.3-200405070341/nptl/sysdeps/unix/sysv/linux/fork.c's version of it
+     * shows it's doing some funny tricks with the pid!
+     *
+     * Once figure it out, need to have dynamic check for threading version to know
+     * what to do.
+     */
+# if 0
+    /* from /usr/include/bits/sched.h ifdef __USE_MISC: */
+#   define CLONE_CHILD_CLEARTID 0x00200000 /* Register exit futex and memory
+                                            * location to clear.  */
+#   define CLONE_CHILD_SETTID 0x01000000   /* Store TID in userlevel buffer in
+                                            * the child.  */
+    /* i386/fork.c pass a 5th arg, &THREAD_SELF->tid, is it needed for SETTID? */
+    static uint tid; 
+    return dynamorio_syscall(SYS_clone, 4/* 5 */,
+                             /* flags, newsp (if 0 -> cur esp), parent_tidptr, child_tidptr,
+                              * something! */
+                             CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID | SIGCHLD,
+                             0, NULL, NULL /*, &tid*/);
+# else
+    /* my workaround for now: just use libc -- binaries won't be back-compatible though */
+    return fork();
+# endif
+#else
+    return dynamorio_syscall(SYS_fork, 0);
+#endif
+}
+
+/* FIXME: get this all to work with pthreads linked in
+ * pthreads prevents creation of core file (can't call abort()) and prevents
+ * exec* from executing -- exec* just hangs
+ */
+bool
+pthreads_linked()
+{
+    return false; /* can't check open==__libc_open on all libcs -- give up for now */
+}
+
+/*-----------------------------------------------------------------------*/
+/* Procedure:
+   1. Fork off a child process that dumps core; creates the "core" file
+   2. Fork off a 2nd child process which:
+   2a. Creates a temporary file of input commands for the debugger
+   2b. Redirects stdin from this temporary file
+   2c. Executes the debugger using the redirected input commands
+*/
+/*-----------------------------------------------------------------------*/
+
+void stackdump()
+{
+    int pid, core_pid;
+    /* get name now -- will be same for children */
+    char *exec_name = get_application_name();
+    char core_name[128];
+    char tmp_name[128];
+    snprintf(tmp_name, 128, "%s.%d", TEMPORARY_FILENAME, get_process_id());
+
+    SYSLOG_INTERNAL_WARNING("stackdump: glibc backtrace:");
+    glibc_stackdump(STDERR);
+
+#ifdef VMX86_SERVER
+    if (os_in_vmkernel_userworld()) {
+       return;                     /* no local gdb, no multithreaded fork */
+    }
+#endif
+
+    if (pthreads_linked()) {
+        SYSLOG_INTERNAL_ERROR("stackdump: pthreads not supported, no core dump");
+        return;
+    }
+
+#if VERBOSE
+    SYSLOG_INTERNAL_ERROR("about to fork parent %d to dump core", get_process_id());
+#endif
+
+    /* Fork a child to dump core */
+    pid = fork_syscall();
+    if (pid == 0) { /* child */
+#if VERBOSE
+        SYSLOG_INTERNAL_ERROR("about to dump core in process %d parent %d thread %d",
+                              get_process_id(), get_parent_id(), get_thread_id());
+#endif
+        /* FIXME: on later linuxes signal seems to be propagated to parent,
+         * so even if get a core dump, it's of the parent and nobody's alive
+         * to call gdb!
+         */        
+        if (!pthreads_linked()) {
+            /* no pthreads */
+            abort();
+        } else {
+            /* pthreads has issues with abort(), this works but
+             * no core is dumped: 
+             */
+            if (!set_default_signal_action(SIGABRT)) {
+                SYSLOG_INTERNAL_ERROR("ERROR in setting handler");
+                exit_syscall(1);
+            }
+            dynamorio_syscall(SYS_kill, 2, get_process_id(), SIGABRT);
+        }
+#if VERBOSE
+        SYSLOG_INTERNAL_ERROR("about to exit process %d", get_process_id());
+#endif
+        exit_syscall(0);
+    }
+    else if (pid == -1) {
+        SYSLOG_INTERNAL_ERROR("ERROR: could not fork to dump core");
+        exit_syscall(1);
+    }
+#if VERBOSE
+    SYSLOG_INTERNAL_ERROR("parent %d %d waiting for child %d", get_process_id(),
+                          get_thread_id(), pid);
+#endif
+    /* Parent continues */
+    core_pid = pid;
+    while (wait_syscall(NULL) != pid) {
+        /* wait for core to be dumped */
+    }
+
+#if VERBOSE
+    SYSLOG_INTERNAL_ERROR("about to fork 2nd child to run gdb");
+#endif
+    /* Fork a 2nd child to run gdb */
+    pid = fork_syscall();
+    if (pid == 0) { /* child */
+        file_t fp;
+        int fd;
+    
+        /* Open a temporary file for the input: the "where" command */
+        fp = os_open(tmp_name, OS_OPEN_REQUIRE_NEW|OS_OPEN_WRITE);
+        os_write(fp, DEBUGGER_COMMAND, strlen(DEBUGGER_COMMAND));
+        os_close(fp);
+
+        fd = open_syscall(tmp_name, O_RDONLY, 0);
+        if (fd < 0) {
+            SYSLOG_INTERNAL_ERROR("ERROR: open failed on temporary file");
+            exit_syscall(1);
+        }
+#if !BATCH_MODE
+        /* Redirect stdin from the temporary file */
+        close_syscall(0); /* close stdin */
+        dup_syscall(fd); /* replace file descriptor 0 with reference to temp file */
+#endif
+        close_syscall(fd); /* close the other reference to temporary file */
+
+        /* Find the core file */
+        strncpy(core_name, CORE_NAME, 127);
+        core_name[127]  = '\0'; /* if max no null */
+        fd = open_syscall(core_name, O_RDONLY, 0);
+        if (fd < 0) {
+            snprintf(core_name, 128, "%s.%d", CORE_NAME, core_pid);
+            SYSLOG_INTERNAL_ERROR("core not found, trying %s", core_name);
+            fd = open_syscall(core_name, O_RDONLY, 0);
+            if (fd < 0) {
+                SYSLOG_INTERNAL_ERROR("ERROR: no core file found!");
+                exit_syscall(1);
+            }
+        }
+        close_syscall(fd);
+
+        /* avoid running the debugger under us! 
+         * FIXME: just remove our libraries, instead of entire env var?
+         */
+        setenv("LD_PRELOAD", "", true);
+
+        SYSLOG_INTERNAL_ERROR("-------------------------------------------");
+        SYSLOG_INTERNAL_ERROR("stackdump: --- now running the debugger ---");
+        SYSLOG_INTERNAL_ERROR("%s %s %s %s",
+                              DEBUGGER, QUIET_MODE, exec_name, core_name);
+        SYSLOG_INTERNAL_ERROR("-------------------------------------------");
+#if BATCH_MODE
+        execlp(DEBUGGER, DEBUGGER, QUIET_MODE, "-x", tmp_name, "-batch",
+               exec_name, core_name, NULL);
+#else
+        execlp(DEBUGGER, DEBUGGER, QUIET_MODE, exec_name, core_name, NULL);
+#endif
+        SYSLOG_INTERNAL_ERROR("ERROR: execlp failed for debugger");
+        exit_syscall(1);
+    }
+    else if (pid == -1) {
+        SYSLOG_INTERNAL_ERROR("ERROR: could not fork to run debugger");
+        exit_syscall(1);
+    }
+    /* Parent continues */
+    /* while(wait(NULL)>0) waits for all children, and could hang, so: */
+    while(wait_syscall(NULL) != pid) {
+        /* empty loop */
+    }
+
+    /* Wait for these children to complete before returning */
+
+    while (wait_syscall(NULL) > 0) {
+        /* empty loop */
+    }
+
+    remove(tmp_name); /* clean up the temporary file */
+    SYSLOG_INTERNAL_ERROR("-------------------------------------------");
+}
diff --git a/core/linux/symtab.c b/core/linux/symtab.c
new file mode 100644
index 000000000..1657995e9
--- /dev/null
+++ b/core/linux/symtab.c
@@ -0,0 +1,244 @@
+/* **********************************************************
+ * Copyright (c) 2001-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2001 Hewlett-Packard Company */
+
+/* support for looking up function containing a pc
+ * for pc-based profiling
+ * based on code from Giuseppe Desoli
+ * modified by bruening, Jan 2001
+ * need to add to build command "-lbfd -liberty"
+ * FIXME: assumes ELF executable compiled w/ -static
+ */
+
+#include <string.h>
+#include <errno.h>
+#include <bfd.h>
+/* globals.h gives us stdio, stdlib, and assert */
+#include "globals.h"
+
+static uint bfd_symcount, nonnull_symcount;
+static asymbol **bfd_syms = NULL;
+static bfd *infile = (bfd *) NULL;
+
+static int
+compare_symbols (const void *ap, const void *bp)
+{
+    const asymbol *a = *(const asymbol **) ap;
+    const asymbol *b = *(const asymbol **) bp;
+    
+    /* check for null pointers -- these are discarded syms   */
+    /* they end up at the end of the sort                    */
+    if (a == NULL)
+        return 1;
+    if (b == NULL)
+        return -1;
+    
+    if (bfd_asymbol_value(a) > bfd_asymbol_value(b))
+        return 1;
+    if (bfd_asymbol_value(a) < bfd_asymbol_value(b))
+        return -1;
+    
+    return strcmp (a->name, b->name);
+}
+
+/* sort the symbol table by section and by offset from the start of the section */
+static void
+sort_symtab()
+{
+    long i;
+    
+    qsort (bfd_syms, bfd_symcount, sizeof (asymbol *), compare_symbols);
+    
+    if (stats->loglevel > 2) {
+        LOG(GLOBAL, LOG_ALL, 3, "\n\nSYMBOL TABLE\n");
+        for (i = 0; i < bfd_symcount; i++) {
+            if (bfd_syms[i]) {
+                LOG(GLOBAL, LOG_ALL, 3, "[%5d] "PFX" 0x%x %5s %s\n", i,
+                        bfd_asymbol_value(bfd_syms[i]),
+                        bfd_syms[i]->flags,
+                        bfd_syms[i]->section->name,
+                        bfd_syms[i]->name);
+            } else {
+                LOG(GLOBAL, LOG_ALL, 3, "(null symbol)\n");
+            }
+        }
+        LOG(GLOBAL, LOG_ALL, 3, "\n\n");
+    }
+}
+
+int
+lookup_symbol_address(ptr_uint_t addr)
+{
+    uint low_idx, high_idx, middle_idx;
+    uint middle_addr;
+    
+    /* the idea is to use a binary search on the sorted symbols table */
+    /* the table is sorted by sections and addresses                  */
+    
+    low_idx = 0;
+    high_idx = nonnull_symcount;
+
+    /* first try to find the two symbols containing it */
+    for (;;) {
+        middle_idx = low_idx + ((high_idx - low_idx) >> 1);
+        if (middle_idx == low_idx)
+            break;
+        
+        /* we need real addresses so use bfd macros for the value here */
+        /* basically it's the symbols' value + their sections' vma     */
+        
+        middle_addr = bfd_asymbol_value (bfd_syms[middle_idx]);
+        
+        if (middle_addr > addr)
+            high_idx = middle_idx;
+        else if (middle_addr < addr)
+            low_idx = middle_idx;
+        else
+            break;
+    }
+    return middle_idx;
+}
+
+/* sets to null uninteresting symbols, sets nonnull_symcount to the number of
+ * nonnull symbols left
+ */
+void
+prepare_symtab()
+{
+    int i;
+    nonnull_symcount = bfd_symcount;
+    for (i = 0; i < bfd_symcount; i++) {
+        if (bfd_syms[i]) {
+            /* FIXME: the BSF_FUNCTION flag is only for ELF
+             * other ideas: remove all non-text-section symbols
+             */
+            if (! (bfd_syms[i]->flags & BSF_FUNCTION) ) {
+                /* remove from table by marking as null */
+                bfd_syms[i] = NULL;
+                nonnull_symcount--;
+            }
+        }
+    }
+}
+
+static asymbol **
+get_symtab(bfd *abfd)
+{
+    asymbol **sy = (asymbol **) NULL;
+    long storage;
+    
+    if (!(bfd_get_file_flags (abfd) & HAS_SYMS)) {
+        print_file(STDERR, "No symbols in \"%s\".\n", bfd_get_filename (abfd));
+        bfd_symcount = 0;
+        return NULL;
+    }
+    
+    storage = bfd_get_symtab_upper_bound (abfd);
+    if (storage < 0) {
+        print_file(STDERR, "BFD fatal error bfd_get_symtab_upper_bound\n");
+        return NULL;
+    }
+    
+    if (storage) {
+        sy = (asymbol **) xmalloc (storage);
+    }
+    bfd_symcount = bfd_canonicalize_symtab (abfd, sy);
+    if (bfd_symcount < 0) {
+        print_file(STDERR, "BFD fatal error bfd_canonicalize_symtab\n");
+        return NULL;
+    }
+    if (bfd_symcount == 0)
+        print_file(STDERR, "%s: No symbols\n", bfd_get_filename (abfd));
+    return sy;
+}
+
+/* this is currently called by pcprofile_init(), so nobody else can use
+ * symtab stuff until that is changed
+ */
+bool
+symtab_init()
+{
+    char *filein = dynamo_options.profexecname;
+    if (filein == NULL) {
+        return false;
+    }
+
+    infile = bfd_openr(filein, NULL);
+    if (infile == NULL) {
+        USAGE_ERROR("can't find file \"%s\" named in -profexecname", filein);
+        return false;
+    }
+
+    /* check the bfd format, this is required */
+    if (!bfd_check_format(infile, bfd_object)) {
+        bfd_close(infile);
+        USAGE_ERROR("-profexecname \"%s\" : not a bfd_object!", filein);
+        return false;
+    }
+
+    if ((bfd_syms = get_symtab(infile)) == NULL) {
+        if (bfd_syms) {
+            free(bfd_syms);
+        }
+        if (infile)
+            bfd_close(infile);
+        USAGE_ERROR("-profexecname \"%s\" : error getting symbol table", 
+                    filein);
+        return false;
+    }
+
+    prepare_symtab();
+    sort_symtab();
+    LOG(GLOBAL, LOG_ALL, 1, "SYMBOL TABLE FOR %s SUCCESFULLY LOADED\n\n", filein);
+    return true;
+}
+
+void
+symtab_exit()
+{
+    bfd_close(infile);
+}
+
+const char *
+symtab_lookup_pc(void * pc)
+{
+    int idx = lookup_symbol_address((ptr_uint_t)pc);
+    if (bfd_syms[idx] == NULL)
+        return "(null)";
+    else {
+        /* FIXME: try to find line number? */
+        return bfd_syms[idx]->name;
+    }
+}
diff --git a/core/linux/symtab.h b/core/linux/symtab.h
new file mode 100644
index 000000000..011ee5d46
--- /dev/null
+++ b/core/linux/symtab.h
@@ -0,0 +1,54 @@
+/* **********************************************************
+ * Copyright (c) 2001-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2001 Hewlett-Packard Company */
+
+/*
+ * symtab.h - support for looking up function a pc is in
+ * for pc-based profiling
+ */
+
+#ifndef _SYMTAB_H_
+#define _SYMTAB_H_ 1
+
+/* initialization */
+bool symtab_init(void);
+
+/* atexit cleanup */
+void symtab_exit(void);
+
+/* handle a pc sample */
+const char * symtab_lookup_pc(void *pc);
+
+#endif /* _SYMTAB_H_ */
diff --git a/core/linux/syscall.h b/core/linux/syscall.h
new file mode 100644
index 000000000..fb30cf5ff
--- /dev/null
+++ b/core/linux/syscall.h
@@ -0,0 +1,1052 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * syscall.h - SYS and NR defines
+ */
+
+#ifndef _SYSCALL_H_
+#define _SYSCALL_H_ 1
+
+#ifdef X64
+
+/* Copied from FC 8's /usr/include/asm/unistd_64.h 
+ * I stripped out the __SYSCALL__ lines and blank lines
+ */
+# define __NR_read                              0
+# define __NR_write                             1
+# define __NR_open                              2
+# define __NR_close                             3
+# define __NR_stat                              4
+# define __NR_fstat                             5
+# define __NR_lstat                             6
+# define __NR_poll                              7
+# define __NR_lseek                             8
+# define __NR_mmap                              9
+# define __NR_mprotect                          10
+# define __NR_munmap                            11
+# define __NR_brk                               12
+# define __NR_rt_sigaction                      13
+# define __NR_rt_sigprocmask                    14
+# define __NR_rt_sigreturn                      15
+# define __NR_ioctl                             16
+# define __NR_pread64                           17
+# define __NR_pwrite64                          18
+# define __NR_readv                             19
+# define __NR_writev                            20
+# define __NR_access                            21
+# define __NR_pipe                              22
+# define __NR_select                            23
+# define __NR_sched_yield                       24
+# define __NR_mremap                            25
+# define __NR_msync                             26
+# define __NR_mincore                           27
+# define __NR_madvise                           28
+# define __NR_shmget                            29
+# define __NR_shmat                             30
+# define __NR_shmctl                            31
+# define __NR_dup                               32
+# define __NR_dup2                              33
+# define __NR_pause                             34
+# define __NR_nanosleep                         35
+# define __NR_getitimer                         36
+# define __NR_alarm                             37
+# define __NR_setitimer                         38
+# define __NR_getpid                            39
+# define __NR_sendfile                          40
+# define __NR_socket                            41
+# define __NR_connect                           42
+# define __NR_accept                            43
+# define __NR_sendto                            44
+# define __NR_recvfrom                          45
+# define __NR_sendmsg                           46
+# define __NR_recvmsg                           47
+# define __NR_shutdown                          48
+# define __NR_bind                              49
+# define __NR_listen                            50
+# define __NR_getsockname                       51
+# define __NR_getpeername                       52
+# define __NR_socketpair                        53
+# define __NR_setsockopt                        54
+# define __NR_getsockopt                        55
+# define __NR_clone                             56
+# define __NR_fork                              57
+# define __NR_vfork                             58
+# define __NR_execve                            59
+# define __NR_exit                              60
+# define __NR_wait4                             61
+# define __NR_kill                              62
+# define __NR_uname                             63
+# define __NR_semget                            64
+# define __NR_semop                             65
+# define __NR_semctl                            66
+# define __NR_shmdt                             67
+# define __NR_msgget                            68
+# define __NR_msgsnd                            69
+# define __NR_msgrcv                            70
+# define __NR_msgctl                            71
+# define __NR_fcntl                             72
+# define __NR_flock                             73
+# define __NR_fsync                             74
+# define __NR_fdatasync                         75
+# define __NR_truncate                          76
+# define __NR_ftruncate                         77
+# define __NR_getdents                          78
+# define __NR_getcwd                            79
+# define __NR_chdir                             80
+# define __NR_fchdir                            81
+# define __NR_rename                            82
+# define __NR_mkdir                             83
+# define __NR_rmdir                             84
+# define __NR_creat                             85
+# define __NR_link                              86
+# define __NR_unlink                            87
+# define __NR_symlink                           88
+# define __NR_readlink                          89
+# define __NR_chmod                             90
+# define __NR_fchmod                            91
+# define __NR_chown                             92
+# define __NR_fchown                            93
+# define __NR_lchown                            94
+# define __NR_umask                             95
+# define __NR_gettimeofday                      96
+# define __NR_getrlimit                         97
+# define __NR_getrusage                         98
+# define __NR_sysinfo                           99
+# define __NR_times                             100
+# define __NR_ptrace                            101
+# define __NR_getuid                            102
+# define __NR_syslog                            103
+/* at the very end the stuff that never runs during the benchmarks */
+# define __NR_getgid                            104
+# define __NR_setuid                            105
+# define __NR_setgid                            106
+# define __NR_geteuid                           107
+# define __NR_getegid                           108
+# define __NR_setpgid                           109
+# define __NR_getppid                           110
+# define __NR_getpgrp                           111
+# define __NR_setsid                            112
+# define __NR_setreuid                          113
+# define __NR_setregid                          114
+# define __NR_getgroups                         115
+# define __NR_setgroups                         116
+# define __NR_setresuid                         117
+# define __NR_getresuid                         118
+# define __NR_setresgid                         119
+# define __NR_getresgid                         120
+# define __NR_getpgid                           121
+# define __NR_setfsuid                          122
+# define __NR_setfsgid                          123
+# define __NR_getsid                            124
+# define __NR_capget                            125
+# define __NR_capset                            126
+# define __NR_rt_sigpending                     127
+# define __NR_rt_sigtimedwait                   128
+# define __NR_rt_sigqueueinfo                   129
+# define __NR_rt_sigsuspend                     130
+# define __NR_sigaltstack                       131
+# define __NR_utime                             132
+# define __NR_mknod                             133
+/* Only needed for a.out */
+# define __NR_uselib                            134
+# define __NR_personality                       135
+# define __NR_ustat                             136
+# define __NR_statfs                            137
+# define __NR_fstatfs                           138
+# define __NR_sysfs                             139
+# define __NR_getpriority                       140
+# define __NR_setpriority                       141
+# define __NR_sched_setparam                    142
+# define __NR_sched_getparam                    143
+# define __NR_sched_setscheduler                144
+# define __NR_sched_getscheduler                145
+# define __NR_sched_get_priority_max            146
+# define __NR_sched_get_priority_min            147
+# define __NR_sched_rr_get_interval             148
+# define __NR_mlock                             149
+# define __NR_munlock                           150
+# define __NR_mlockall                          151
+# define __NR_munlockall                        152
+# define __NR_vhangup                           153
+# define __NR_modify_ldt                        154
+# define __NR_pivot_root                        155
+# define __NR__sysctl                           156
+# define __NR_prctl                             157
+# define __NR_arch_prctl                        158
+# define __NR_adjtimex                          159
+# define __NR_setrlimit                         160
+# define __NR_chroot                            161
+# define __NR_sync                              162
+# define __NR_acct                              163
+# define __NR_settimeofday                      164
+# define __NR_mount                             165
+# define __NR_umount2                           166
+# define __NR_swapon                            167
+# define __NR_swapoff                           168
+# define __NR_reboot                            169
+# define __NR_sethostname                       170
+# define __NR_setdomainname                     171
+# define __NR_iopl                              172
+# define __NR_ioperm                            173
+# define __NR_create_module                     174
+# define __NR_init_module                       175
+# define __NR_delete_module                     176
+# define __NR_get_kernel_syms                   177
+# define __NR_query_module                      178
+# define __NR_quotactl                          179
+# define __NR_nfsservctl                        180
+/* reserved for LiS/STREAMS */
+# define __NR_getpmsg                           181
+# define __NR_putpmsg                           182
+/* reserved for AFS */
+# define __NR_afs_syscall                       183
+/* reserved for tux */
+# define __NR_tuxcall                           184
+# define __NR_security                          185
+# define __NR_gettid                            186
+# define __NR_readahead                         187
+# define __NR_setxattr                          188
+# define __NR_lsetxattr                         189
+# define __NR_fsetxattr                         190
+# define __NR_getxattr                          191
+# define __NR_lgetxattr                         192
+# define __NR_fgetxattr                         193
+# define __NR_listxattr                         194
+# define __NR_llistxattr                        195
+# define __NR_flistxattr                        196
+# define __NR_removexattr                       197
+# define __NR_lremovexattr                      198
+# define __NR_fremovexattr                      199
+# define __NR_tkill                             200
+# define __NR_time                              201
+# define __NR_futex                             202
+# define __NR_sched_setaffinity                 203
+# define __NR_sched_getaffinity                 204
+# define __NR_set_thread_area                   205
+# define __NR_io_setup                          206
+# define __NR_io_destroy                        207
+# define __NR_io_getevents                      208
+# define __NR_io_submit                         209
+# define __NR_io_cancel                         210
+# define __NR_get_thread_area                   211
+# define __NR_lookup_dcookie                    212
+# define __NR_epoll_create                      213
+# define __NR_epoll_ctl_old                     214
+# define __NR_epoll_wait_old                    215
+# define __NR_remap_file_pages                  216
+# define __NR_getdents64                        217
+# define __NR_set_tid_address                   218
+# define __NR_restart_syscall                   219
+# define __NR_semtimedop                        220
+# define __NR_fadvise64                         221
+# define __NR_timer_create                      222
+# define __NR_timer_settime                     223
+# define __NR_timer_gettime                     224
+# define __NR_timer_getoverrun                  225
+# define __NR_timer_delete                      226
+# define __NR_clock_settime                     227
+# define __NR_clock_gettime                     228
+# define __NR_clock_getres                      229
+# define __NR_clock_nanosleep                   230
+# define __NR_exit_group                        231
+# define __NR_epoll_wait                        232
+# define __NR_epoll_ctl                         233
+# define __NR_tgkill                            234
+# define __NR_utimes                            235
+# define __NR_vserver                           236
+# define __NR_mbind                             237
+# define __NR_set_mempolicy                     238
+# define __NR_get_mempolicy                     239
+# define __NR_mq_open                           240
+# define __NR_mq_unlink                         241
+# define __NR_mq_timedsend                      242
+# define __NR_mq_timedreceive                   243
+# define __NR_mq_notify                         244
+# define __NR_mq_getsetattr                     245
+# define __NR_kexec_load                        246
+# define __NR_waitid                            247
+# define __NR_add_key                           248
+# define __NR_request_key                       249
+# define __NR_keyctl                            250
+# define __NR_ioprio_set                        251
+# define __NR_ioprio_get                        252
+# define __NR_inotify_init                      253
+# define __NR_inotify_add_watch                 254
+# define __NR_inotify_rm_watch                  255
+# define __NR_migrate_pages                     256
+# define __NR_openat                            257
+# define __NR_mkdirat                           258
+# define __NR_mknodat                           259
+# define __NR_fchownat                          260
+# define __NR_futimesat                         261
+# define __NR_newfstatat                        262
+# define __NR_unlinkat                          263
+# define __NR_renameat                          264
+# define __NR_linkat                            265
+# define __NR_symlinkat                         266
+# define __NR_readlinkat                        267
+# define __NR_fchmodat                          268
+# define __NR_faccessat                         269
+# define __NR_pselect6                          270
+# define __NR_ppoll                             271
+# define __NR_unshare                           272
+# define __NR_set_robust_list                   273
+# define __NR_get_robust_list                   274
+# define __NR_splice                            275
+# define __NR_tee                               276
+# define __NR_sync_file_range                   277
+# define __NR_vmsplice                          278
+# define __NR_move_pages                        279
+# define __NR_utimensat                         280
+# define __IGNORE_getcpu                /* implemented as a vsyscall */
+# define __NR_epoll_pwait                       281
+# define __NR_signalfd                          282
+# define __NR_timerfd                           283
+# define __NR_eventfd                           284
+# define __NR_fallocate                         285
+# ifndef __NO_STUBS
+#  define __ARCH_WANT_OLD_READDIR
+#  define __ARCH_WANT_OLD_STAT
+#  define __ARCH_WANT_SYS_ALARM
+#  define __ARCH_WANT_SYS_GETHOSTNAME
+#  define __ARCH_WANT_SYS_PAUSE
+#  define __ARCH_WANT_SYS_SGETMASK
+#  define __ARCH_WANT_SYS_SIGNAL
+#  define __ARCH_WANT_SYS_UTIME
+#  define __ARCH_WANT_SYS_WAITPID
+#  define __ARCH_WANT_SYS_SOCKETCALL
+#  define __ARCH_WANT_SYS_FADVISE64
+#  define __ARCH_WANT_SYS_GETPGRP
+#  define __ARCH_WANT_SYS_LLSEEK
+#  define __ARCH_WANT_SYS_NICE
+#  define __ARCH_WANT_SYS_OLD_GETRLIMIT
+#  define __ARCH_WANT_SYS_OLDUMOUNT
+#  define __ARCH_WANT_SYS_SIGPENDING
+#  define __ARCH_WANT_SYS_SIGPROCMASK
+#  define __ARCH_WANT_SYS_RT_SIGACTION
+#  define __ARCH_WANT_SYS_RT_SIGSUSPEND
+#  define __ARCH_WANT_SYS_TIME
+#  define __ARCH_WANT_COMPAT_SYS_TIME
+# endif /* __NO_STUBS */
+
+#else
+
+/* Copied from FC 8's /usr/include/asm/unistd_32.h */
+# define __NR_restart_syscall      0
+# define __NR_exit                 1
+# define __NR_fork                 2
+# define __NR_read                 3
+# define __NR_write                4
+# define __NR_open                 5
+# define __NR_close                6
+# define __NR_waitpid              7
+# define __NR_creat                8
+# define __NR_link                 9
+# define __NR_unlink              10
+# define __NR_execve              11
+# define __NR_chdir               12
+# define __NR_time                13
+# define __NR_mknod               14
+# define __NR_chmod               15
+# define __NR_lchown              16
+# define __NR_break               17
+# define __NR_oldstat             18
+# define __NR_lseek               19
+# define __NR_getpid              20
+# define __NR_mount               21
+# define __NR_umount              22
+# define __NR_setuid              23
+# define __NR_getuid              24
+# define __NR_stime               25
+# define __NR_ptrace              26
+# define __NR_alarm               27
+# define __NR_oldfstat            28
+# define __NR_pause               29
+# define __NR_utime               30
+# define __NR_stty                31
+# define __NR_gtty                32
+# define __NR_access              33
+# define __NR_nice                34
+# define __NR_ftime               35
+# define __NR_sync                36
+# define __NR_kill                37
+# define __NR_rename              38
+# define __NR_mkdir               39
+# define __NR_rmdir               40
+# define __NR_dup                 41
+# define __NR_pipe                42
+# define __NR_times               43
+# define __NR_prof                44
+# define __NR_brk                 45
+# define __NR_setgid              46
+# define __NR_getgid              47
+# define __NR_signal              48
+# define __NR_geteuid             49
+# define __NR_getegid             50
+# define __NR_acct                51
+# define __NR_umount2             52
+# define __NR_lock                53
+# define __NR_ioctl               54
+# define __NR_fcntl               55
+# define __NR_mpx                 56
+# define __NR_setpgid             57
+# define __NR_ulimit              58
+# define __NR_oldolduname         59
+# define __NR_umask               60
+# define __NR_chroot              61
+# define __NR_ustat               62
+# define __NR_dup2                63
+# define __NR_getppid             64
+# define __NR_getpgrp             65
+# define __NR_setsid              66
+# define __NR_sigaction           67
+# define __NR_sgetmask            68
+# define __NR_ssetmask            69
+# define __NR_setreuid            70
+# define __NR_setregid            71
+# define __NR_sigsuspend          72
+# define __NR_sigpending          73
+# define __NR_sethostname         74
+# define __NR_setrlimit           75
+# define __NR_getrlimit           76     /* Back compatible 2Gig limited rlimit */
+# define __NR_getrusage           77
+# define __NR_gettimeofday        78
+# define __NR_settimeofday        79
+# define __NR_getgroups           80
+# define __NR_setgroups           81
+# define __NR_select              82
+# define __NR_symlink             83
+# define __NR_oldlstat            84
+# define __NR_readlink            85
+# define __NR_uselib              86
+# define __NR_swapon              87
+# define __NR_reboot              88
+# define __NR_readdir             89
+# define __NR_mmap                90
+# define __NR_munmap              91
+# define __NR_truncate            92
+# define __NR_ftruncate           93
+# define __NR_fchmod              94
+# define __NR_fchown              95
+# define __NR_getpriority         96
+# define __NR_setpriority         97
+# define __NR_profil              98
+# define __NR_statfs              99
+# define __NR_fstatfs            100
+# define __NR_ioperm             101
+# define __NR_socketcall         102
+# define __NR_syslog             103
+# define __NR_setitimer          104
+# define __NR_getitimer          105
+# define __NR_stat               106
+# define __NR_lstat              107
+# define __NR_fstat              108
+# define __NR_olduname           109
+# define __NR_iopl               110
+# define __NR_vhangup            111
+# define __NR_idle               112
+# define __NR_vm86old            113
+# define __NR_wait4              114
+# define __NR_swapoff            115
+# define __NR_sysinfo            116
+# define __NR_ipc                117
+# define __NR_fsync              118
+# define __NR_sigreturn          119
+# define __NR_clone              120
+# define __NR_setdomainname      121
+# define __NR_uname              122
+# define __NR_modify_ldt         123
+# define __NR_adjtimex           124
+# define __NR_mprotect           125
+# define __NR_sigprocmask        126
+# define __NR_create_module      127
+# define __NR_init_module        128
+# define __NR_delete_module      129
+# define __NR_get_kernel_syms    130
+# define __NR_quotactl           131
+# define __NR_getpgid            132
+# define __NR_fchdir             133
+# define __NR_bdflush            134
+# define __NR_sysfs              135
+# define __NR_personality        136
+# define __NR_afs_syscall        137 /* Syscall for Andrew File System */
+# define __NR_setfsuid           138
+# define __NR_setfsgid           139
+# define __NR__llseek            140
+# define __NR_getdents           141
+# define __NR__newselect         142
+# define __NR_flock              143
+# define __NR_msync              144
+# define __NR_readv              145
+# define __NR_writev             146
+# define __NR_getsid             147
+# define __NR_fdatasync          148
+# define __NR__sysctl            149
+# define __NR_mlock              150
+# define __NR_munlock            151
+# define __NR_mlockall           152
+# define __NR_munlockall         153
+# define __NR_sched_setparam             154
+# define __NR_sched_getparam             155
+# define __NR_sched_setscheduler         156
+# define __NR_sched_getscheduler         157
+# define __NR_sched_yield                158
+# define __NR_sched_get_priority_max     159
+# define __NR_sched_get_priority_min     160
+# define __NR_sched_rr_get_interval      161
+# define __NR_nanosleep          162
+# define __NR_mremap             163
+# define __NR_setresuid          164
+# define __NR_getresuid          165
+# define __NR_vm86               166
+# define __NR_query_module       167
+# define __NR_poll               168
+# define __NR_nfsservctl         169
+# define __NR_setresgid          170
+# define __NR_getresgid          171
+# define __NR_prctl              172
+# define __NR_rt_sigreturn       173
+# define __NR_rt_sigaction       174
+# define __NR_rt_sigprocmask     175
+# define __NR_rt_sigpending      176
+# define __NR_rt_sigtimedwait    177
+# define __NR_rt_sigqueueinfo    178
+# define __NR_rt_sigsuspend      179
+# define __NR_pread64            180
+# define __NR_pwrite64           181
+# define __NR_chown              182
+# define __NR_getcwd             183
+# define __NR_capget             184
+# define __NR_capset             185
+# define __NR_sigaltstack        186
+# define __NR_sendfile           187
+# define __NR_getpmsg            188     /* some people actually want streams */
+# define __NR_putpmsg            189     /* some people actually want streams */
+# define __NR_vfork              190
+# define __NR_ugetrlimit         191     /* SuS compliant getrlimit */
+# define __NR_mmap2              192
+# define __NR_truncate64         193
+# define __NR_ftruncate64        194
+# define __NR_stat64             195
+# define __NR_lstat64            196
+# define __NR_fstat64            197
+# define __NR_lchown32           198
+# define __NR_getuid32           199
+# define __NR_getgid32           200
+# define __NR_geteuid32          201
+# define __NR_getegid32          202
+# define __NR_setreuid32         203
+# define __NR_setregid32         204
+# define __NR_getgroups32        205
+# define __NR_setgroups32        206
+# define __NR_fchown32           207
+# define __NR_setresuid32        208
+# define __NR_getresuid32        209
+# define __NR_setresgid32        210
+# define __NR_getresgid32        211
+# define __NR_chown32            212
+# define __NR_setuid32           213
+# define __NR_setgid32           214
+# define __NR_setfsuid32         215
+# define __NR_setfsgid32         216
+# define __NR_pivot_root         217
+# define __NR_mincore            218
+# define __NR_madvise            219
+# define __NR_madvise1           219     /* delete when C lib stub is removed */
+# define __NR_getdents64         220
+# define __NR_fcntl64            221
+/* 223 is unused */
+# define __NR_gettid             224
+# define __NR_readahead          225
+# define __NR_setxattr           226
+# define __NR_lsetxattr          227
+# define __NR_fsetxattr          228
+# define __NR_getxattr           229
+# define __NR_lgetxattr          230
+# define __NR_fgetxattr          231
+# define __NR_listxattr          232
+# define __NR_llistxattr         233
+# define __NR_flistxattr         234
+# define __NR_removexattr        235
+# define __NR_lremovexattr       236
+# define __NR_fremovexattr       237
+# define __NR_tkill              238
+# define __NR_sendfile64         239
+# define __NR_futex              240
+# define __NR_sched_setaffinity  241
+# define __NR_sched_getaffinity  242
+# define __NR_set_thread_area    243
+# define __NR_get_thread_area    244
+# define __NR_io_setup           245
+# define __NR_io_destroy         246
+# define __NR_io_getevents       247
+# define __NR_io_submit          248
+# define __NR_io_cancel          249
+# define __NR_fadvise64          250
+/* 251 is available for reuse (was briefly sys_set_zone_reclaim) */
+# define __NR_exit_group         252
+# define __NR_lookup_dcookie     253
+# define __NR_epoll_create       254
+# define __NR_epoll_ctl          255
+# define __NR_epoll_wait         256
+# define __NR_remap_file_pages   257
+# define __NR_set_tid_address    258
+# define __NR_timer_create       259
+# define __NR_timer_settime      (__NR_timer_create+1)
+# define __NR_timer_gettime      (__NR_timer_create+2)
+# define __NR_timer_getoverrun   (__NR_timer_create+3)
+# define __NR_timer_delete       (__NR_timer_create+4)
+# define __NR_clock_settime      (__NR_timer_create+5)
+# define __NR_clock_gettime      (__NR_timer_create+6)
+# define __NR_clock_getres       (__NR_timer_create+7)
+# define __NR_clock_nanosleep    (__NR_timer_create+8)
+# define __NR_statfs64           268
+# define __NR_fstatfs64          269
+# define __NR_tgkill             270
+# define __NR_utimes             271
+# define __NR_fadvise64_64       272
+# define __NR_vserver            273
+# define __NR_mbind              274
+# define __NR_get_mempolicy      275
+# define __NR_set_mempolicy      276
+# define __NR_mq_open            277
+# define __NR_mq_unlink          (__NR_mq_open+1)
+# define __NR_mq_timedsend       (__NR_mq_open+2)
+# define __NR_mq_timedreceive    (__NR_mq_open+3)
+# define __NR_mq_notify          (__NR_mq_open+4)
+# define __NR_mq_getsetattr      (__NR_mq_open+5)
+# define __NR_kexec_load         283
+# define __NR_waitid             284
+/* #define __NR_sys_setaltroot  285 */
+# define __NR_add_key            286
+# define __NR_request_key        287
+# define __NR_keyctl             288
+# define __NR_ioprio_set         289
+# define __NR_ioprio_get         290
+# define __NR_inotify_init       291
+# define __NR_inotify_add_watch  292
+# define __NR_inotify_rm_watch   293
+# define __NR_migrate_pages      294
+# define __NR_openat             295
+# define __NR_mkdirat            296
+# define __NR_mknodat            297
+# define __NR_fchownat           298
+# define __NR_futimesat          299
+# define __NR_fstatat64          300
+# define __NR_unlinkat           301
+# define __NR_renameat           302
+# define __NR_linkat             303
+# define __NR_symlinkat          304
+# define __NR_readlinkat         305
+# define __NR_fchmodat           306
+# define __NR_faccessat          307
+# define __NR_pselect6           308
+# define __NR_ppoll              309
+# define __NR_unshare            310
+# define __NR_set_robust_list    311
+# define __NR_get_robust_list    312
+# define __NR_splice             313
+# define __NR_sync_file_range    314
+# define __NR_tee                315
+# define __NR_vmsplice           316
+# define __NR_move_pages         317
+# define __NR_getcpu             318
+# define __NR_epoll_pwait        319
+# define __NR_utimensat          320
+# define __NR_signalfd           321
+# define __NR_timerfd            322
+# define __NR_eventfd            323
+# define __NR_fallocate          324
+#endif
+
+/* Copied from FC 8's /usr/include/bits/syscall.h */
+#define SYS__sysctl __NR__sysctl
+#define SYS_access __NR_access
+#define SYS_acct __NR_acct
+#define SYS_add_key __NR_add_key
+#define SYS_adjtimex __NR_adjtimex
+#define SYS_afs_syscall __NR_afs_syscall
+#define SYS_alarm __NR_alarm
+#define SYS_brk __NR_brk
+#define SYS_capget __NR_capget
+#define SYS_capset __NR_capset
+#define SYS_chdir __NR_chdir
+#define SYS_chmod __NR_chmod
+#define SYS_chown __NR_chown
+#define SYS_chroot __NR_chroot
+#define SYS_clock_getres __NR_clock_getres
+#define SYS_clock_gettime __NR_clock_gettime
+#define SYS_clock_nanosleep __NR_clock_nanosleep
+#define SYS_clock_settime __NR_clock_settime
+#define SYS_clone __NR_clone
+#define SYS_close __NR_close
+#define SYS_creat __NR_creat
+#define SYS_create_module __NR_create_module
+#define SYS_delete_module __NR_delete_module
+#define SYS_dup __NR_dup
+#define SYS_dup2 __NR_dup2
+#define SYS_epoll_create __NR_epoll_create
+#define SYS_epoll_ctl __NR_epoll_ctl
+#define SYS_epoll_pwait __NR_epoll_pwait
+#define SYS_epoll_wait __NR_epoll_wait
+#define SYS_eventfd __NR_eventfd
+#define SYS_execve __NR_execve
+#define SYS_exit __NR_exit
+#define SYS_exit_group __NR_exit_group
+#define SYS_faccessat __NR_faccessat
+#define SYS_fadvise64 __NR_fadvise64
+#define SYS_fallocate __NR_fallocate
+#define SYS_fchdir __NR_fchdir
+#define SYS_fchmod __NR_fchmod
+#define SYS_fchmodat __NR_fchmodat
+#define SYS_fchown __NR_fchown
+#define SYS_fchownat __NR_fchownat
+#define SYS_fcntl __NR_fcntl
+#define SYS_fdatasync __NR_fdatasync
+#define SYS_fgetxattr __NR_fgetxattr
+#define SYS_flistxattr __NR_flistxattr
+#define SYS_flock __NR_flock
+#define SYS_fork __NR_fork
+#define SYS_fremovexattr __NR_fremovexattr
+#define SYS_fsetxattr __NR_fsetxattr
+#define SYS_fstat __NR_fstat
+#define SYS_fstatfs __NR_fstatfs
+#define SYS_fsync __NR_fsync
+#define SYS_ftruncate __NR_ftruncate
+#define SYS_futex __NR_futex
+#define SYS_futimesat __NR_futimesat
+#define SYS_get_kernel_syms __NR_get_kernel_syms
+#define SYS_get_mempolicy __NR_get_mempolicy
+#define SYS_get_robust_list __NR_get_robust_list
+#define SYS_get_thread_area __NR_get_thread_area
+#define SYS_getcwd __NR_getcwd
+#define SYS_getdents __NR_getdents
+#define SYS_getdents64 __NR_getdents64
+#define SYS_getegid __NR_getegid
+#define SYS_geteuid __NR_geteuid
+#define SYS_getgid __NR_getgid
+#define SYS_getgroups __NR_getgroups
+#define SYS_getitimer __NR_getitimer
+#define SYS_getpgid __NR_getpgid
+#define SYS_getpgrp __NR_getpgrp
+#define SYS_getpid __NR_getpid
+#define SYS_getpmsg __NR_getpmsg
+#define SYS_getppid __NR_getppid
+#define SYS_getpriority __NR_getpriority
+#define SYS_getresgid __NR_getresgid
+#define SYS_getresuid __NR_getresuid
+#define SYS_getrlimit __NR_getrlimit
+#define SYS_getrusage __NR_getrusage
+#define SYS_getsid __NR_getsid
+#define SYS_gettid __NR_gettid
+#define SYS_gettimeofday __NR_gettimeofday
+#define SYS_getuid __NR_getuid
+#define SYS_getxattr __NR_getxattr
+#define SYS_init_module __NR_init_module
+#define SYS_inotify_add_watch __NR_inotify_add_watch
+#define SYS_inotify_init __NR_inotify_init
+#define SYS_inotify_rm_watch __NR_inotify_rm_watch
+#define SYS_io_cancel __NR_io_cancel
+#define SYS_io_destroy __NR_io_destroy
+#define SYS_io_getevents __NR_io_getevents
+#define SYS_io_setup __NR_io_setup
+#define SYS_io_submit __NR_io_submit
+#define SYS_ioctl __NR_ioctl
+#define SYS_ioperm __NR_ioperm
+#define SYS_iopl __NR_iopl
+#define SYS_ioprio_get __NR_ioprio_get
+#define SYS_ioprio_set __NR_ioprio_set
+#define SYS_kexec_load __NR_kexec_load
+#define SYS_keyctl __NR_keyctl
+#define SYS_kill __NR_kill
+#define SYS_lchown __NR_lchown
+#define SYS_lgetxattr __NR_lgetxattr
+#define SYS_link __NR_link
+#define SYS_linkat __NR_linkat
+#define SYS_listxattr __NR_listxattr
+#define SYS_llistxattr __NR_llistxattr
+#define SYS_lookup_dcookie __NR_lookup_dcookie
+#define SYS_lremovexattr __NR_lremovexattr
+#define SYS_lseek __NR_lseek
+#define SYS_lsetxattr __NR_lsetxattr
+#define SYS_lstat __NR_lstat
+#define SYS_madvise __NR_madvise
+#define SYS_mbind __NR_mbind
+#define SYS_migrate_pages __NR_migrate_pages
+#define SYS_mincore __NR_mincore
+#define SYS_mkdir __NR_mkdir
+#define SYS_mkdirat __NR_mkdirat
+#define SYS_mknod __NR_mknod
+#define SYS_mknodat __NR_mknodat
+#define SYS_mlock __NR_mlock
+#define SYS_mlockall __NR_mlockall
+#define SYS_mmap __NR_mmap
+#define SYS_modify_ldt __NR_modify_ldt
+#define SYS_mount __NR_mount
+#define SYS_move_pages __NR_move_pages
+#define SYS_mprotect __NR_mprotect
+#define SYS_mq_getsetattr __NR_mq_getsetattr
+#define SYS_mq_notify __NR_mq_notify
+#define SYS_mq_open __NR_mq_open
+#define SYS_mq_timedreceive __NR_mq_timedreceive
+#define SYS_mq_timedsend __NR_mq_timedsend
+#define SYS_mq_unlink __NR_mq_unlink
+#define SYS_mremap __NR_mremap
+#define SYS_msync __NR_msync
+#define SYS_munlock __NR_munlock
+#define SYS_munlockall __NR_munlockall
+#define SYS_munmap __NR_munmap
+#define SYS_nanosleep __NR_nanosleep
+#define SYS_nfsservctl __NR_nfsservctl
+#define SYS_open __NR_open
+#define SYS_openat __NR_openat
+#define SYS_pause __NR_pause
+#define SYS_personality __NR_personality
+#define SYS_pipe __NR_pipe
+#define SYS_pivot_root __NR_pivot_root
+#define SYS_poll __NR_poll
+#define SYS_ppoll __NR_ppoll
+#define SYS_prctl __NR_prctl
+#define SYS_pread64 __NR_pread64
+#define SYS_pselect6 __NR_pselect6
+#define SYS_ptrace __NR_ptrace
+#define SYS_putpmsg __NR_putpmsg
+#define SYS_pwrite64 __NR_pwrite64
+#define SYS_query_module __NR_query_module
+#define SYS_quotactl __NR_quotactl
+#define SYS_read __NR_read
+#define SYS_readahead __NR_readahead
+#define SYS_readlink __NR_readlink
+#define SYS_readlinkat __NR_readlinkat
+#define SYS_readv __NR_readv
+#define SYS_reboot __NR_reboot
+#define SYS_remap_file_pages __NR_remap_file_pages
+#define SYS_removexattr __NR_removexattr
+#define SYS_rename __NR_rename
+#define SYS_renameat __NR_renameat
+#define SYS_request_key __NR_request_key
+#define SYS_restart_syscall __NR_restart_syscall
+#define SYS_rmdir __NR_rmdir
+#define SYS_rt_sigaction __NR_rt_sigaction
+#define SYS_rt_sigpending __NR_rt_sigpending
+#define SYS_rt_sigprocmask __NR_rt_sigprocmask
+#define SYS_rt_sigqueueinfo __NR_rt_sigqueueinfo
+#define SYS_rt_sigreturn __NR_rt_sigreturn
+#define SYS_rt_sigsuspend __NR_rt_sigsuspend
+#define SYS_rt_sigtimedwait __NR_rt_sigtimedwait
+#define SYS_sched_get_priority_max __NR_sched_get_priority_max
+#define SYS_sched_get_priority_min __NR_sched_get_priority_min
+#define SYS_sched_getaffinity __NR_sched_getaffinity
+#define SYS_sched_getparam __NR_sched_getparam
+#define SYS_sched_getscheduler __NR_sched_getscheduler
+#define SYS_sched_rr_get_interval __NR_sched_rr_get_interval
+#define SYS_sched_setaffinity __NR_sched_setaffinity
+#define SYS_sched_setparam __NR_sched_setparam
+#define SYS_sched_setscheduler __NR_sched_setscheduler
+#define SYS_sched_yield __NR_sched_yield
+#define SYS_select __NR_select
+#define SYS_sendfile __NR_sendfile
+#define SYS_set_mempolicy __NR_set_mempolicy
+#define SYS_set_robust_list __NR_set_robust_list
+#define SYS_set_thread_area __NR_set_thread_area
+#define SYS_set_tid_address __NR_set_tid_address
+#define SYS_setdomainname __NR_setdomainname
+#define SYS_setfsgid __NR_setfsgid
+#define SYS_setfsuid __NR_setfsuid
+#define SYS_setgid __NR_setgid
+#define SYS_setgroups __NR_setgroups
+#define SYS_sethostname __NR_sethostname
+#define SYS_setitimer __NR_setitimer
+#define SYS_setpgid __NR_setpgid
+#define SYS_setpriority __NR_setpriority
+#define SYS_setregid __NR_setregid
+#define SYS_setresgid __NR_setresgid
+#define SYS_setresuid __NR_setresuid
+#define SYS_setreuid __NR_setreuid
+#define SYS_setrlimit __NR_setrlimit
+#define SYS_setsid __NR_setsid
+#define SYS_settimeofday __NR_settimeofday
+#define SYS_setuid __NR_setuid
+#define SYS_setxattr __NR_setxattr
+#define SYS_sigaltstack __NR_sigaltstack
+#define SYS_signalfd __NR_signalfd
+#define SYS_splice __NR_splice
+#define SYS_stat __NR_stat
+#define SYS_statfs __NR_statfs
+#define SYS_swapoff __NR_swapoff
+#define SYS_swapon __NR_swapon
+#define SYS_symlink __NR_symlink
+#define SYS_symlinkat __NR_symlinkat
+#define SYS_sync __NR_sync
+#define SYS_sync_file_range __NR_sync_file_range
+#define SYS_sysfs __NR_sysfs
+#define SYS_sysinfo __NR_sysinfo
+#define SYS_syslog __NR_syslog
+#define SYS_tee __NR_tee
+#define SYS_tgkill __NR_tgkill
+#define SYS_time __NR_time
+#define SYS_timer_create __NR_timer_create
+#define SYS_timer_delete __NR_timer_delete
+#define SYS_timer_getoverrun __NR_timer_getoverrun
+#define SYS_timer_gettime __NR_timer_gettime
+#define SYS_timer_settime __NR_timer_settime
+#define SYS_timerfd __NR_timerfd
+#define SYS_times __NR_times
+#define SYS_tkill __NR_tkill
+#define SYS_truncate __NR_truncate
+#define SYS_umask __NR_umask
+#define SYS_umount2 __NR_umount2
+#define SYS_uname __NR_uname
+#define SYS_unlink __NR_unlink
+#define SYS_unlinkat __NR_unlinkat
+#define SYS_unshare __NR_unshare
+#define SYS_uselib __NR_uselib
+#define SYS_ustat __NR_ustat
+#define SYS_utime __NR_utime
+#define SYS_utimensat __NR_utimensat
+#define SYS_utimes __NR_utimes
+#define SYS_vfork __NR_vfork
+#define SYS_vhangup __NR_vhangup
+#define SYS_vmsplice __NR_vmsplice
+#define SYS_vserver __NR_vserver
+#define SYS_wait4 __NR_wait4
+#define SYS_waitid __NR_waitid
+#define SYS_write __NR_write
+#define SYS_writev __NR_writev
+#ifdef X64
+# define SYS_accept __NR_accept
+# define SYS_arch_prctl __NR_arch_prctl
+# define SYS_bind __NR_bind
+# define SYS_connect __NR_connect
+# define SYS_epoll_ctl_old __NR_epoll_ctl_old
+# define SYS_epoll_wait_old __NR_epoll_wait_old
+# define SYS_getpeername __NR_getpeername
+# define SYS_getsockname __NR_getsockname
+# define SYS_getsockopt __NR_getsockopt
+# define SYS_listen __NR_listen
+# define SYS_msgctl __NR_msgctl
+# define SYS_msgget __NR_msgget
+# define SYS_msgrcv __NR_msgrcv
+# define SYS_msgsnd __NR_msgsnd
+# define SYS_newfstatat __NR_newfstatat
+# define SYS_recvfrom __NR_recvfrom
+# define SYS_recvmsg __NR_recvmsg
+# define SYS_security __NR_security
+# define SYS_semctl __NR_semctl
+# define SYS_semget __NR_semget
+# define SYS_semop __NR_semop
+# define SYS_semtimedop __NR_semtimedop
+# define SYS_sendmsg __NR_sendmsg
+# define SYS_sendto __NR_sendto
+# define SYS_setsockopt __NR_setsockopt
+# define SYS_shmat __NR_shmat
+# define SYS_shmctl __NR_shmctl
+# define SYS_shmdt __NR_shmdt
+# define SYS_shmget __NR_shmget
+# define SYS_shutdown __NR_shutdown
+# define SYS_socket __NR_socket
+# define SYS_socketpair __NR_socketpair
+# define SYS_tuxcall __NR_tuxcall
+#else
+# define SYS__llseek __NR__llseek
+# define SYS__newselect __NR__newselect
+# define SYS_bdflush __NR_bdflush
+# define SYS_break __NR_break
+# define SYS_chown32 __NR_chown32
+# define SYS_fadvise64_64 __NR_fadvise64_64
+# define SYS_fchown32 __NR_fchown32
+# define SYS_fcntl64 __NR_fcntl64
+# define SYS_fstat64 __NR_fstat64
+# define SYS_fstatat64 __NR_fstatat64
+# define SYS_fstatfs64 __NR_fstatfs64
+# define SYS_ftime __NR_ftime
+# define SYS_ftruncate64 __NR_ftruncate64
+# define SYS_getcpu __NR_getcpu
+# define SYS_getegid32 __NR_getegid32
+# define SYS_geteuid32 __NR_geteuid32
+# define SYS_getgid32 __NR_getgid32
+# define SYS_getgroups32 __NR_getgroups32
+# define SYS_getresgid32 __NR_getresgid32
+# define SYS_getresuid32 __NR_getresuid32
+# define SYS_getuid32 __NR_getuid32
+# define SYS_gtty __NR_gtty
+# define SYS_idle __NR_idle
+# define SYS_ipc __NR_ipc
+# define SYS_lchown32 __NR_lchown32
+# define SYS_lock __NR_lock
+# define SYS_lstat64 __NR_lstat64
+# define SYS_madvise1 __NR_madvise1
+# define SYS_mmap2 __NR_mmap2
+# define SYS_mpx __NR_mpx
+# define SYS_nice __NR_nice
+# define SYS_oldfstat __NR_oldfstat
+# define SYS_oldlstat __NR_oldlstat
+# define SYS_oldolduname __NR_oldolduname
+# define SYS_oldstat __NR_oldstat
+# define SYS_olduname __NR_olduname
+# define SYS_prof __NR_prof
+# define SYS_profil __NR_profil
+# define SYS_readdir __NR_readdir
+# define SYS_sendfile64 __NR_sendfile64
+# define SYS_setfsgid32 __NR_setfsgid32
+# define SYS_setfsuid32 __NR_setfsuid32
+# define SYS_setgid32 __NR_setgid32
+# define SYS_setgroups32 __NR_setgroups32
+# define SYS_setregid32 __NR_setregid32
+# define SYS_setresgid32 __NR_setresgid32
+# define SYS_setresuid32 __NR_setresuid32
+# define SYS_setreuid32 __NR_setreuid32
+# define SYS_setuid32 __NR_setuid32
+# define SYS_sgetmask __NR_sgetmask
+# define SYS_sigaction __NR_sigaction
+# define SYS_signal __NR_signal
+# define SYS_sigpending __NR_sigpending
+# define SYS_sigprocmask __NR_sigprocmask
+# define SYS_sigreturn __NR_sigreturn
+# define SYS_sigsuspend __NR_sigsuspend
+# define SYS_socketcall __NR_socketcall
+# define SYS_ssetmask __NR_ssetmask
+# define SYS_stat64 __NR_stat64
+# define SYS_statfs64 __NR_statfs64
+# define SYS_stime __NR_stime
+# define SYS_stty __NR_stty
+# define SYS_truncate64 __NR_truncate64
+# define SYS_ugetrlimit __NR_ugetrlimit
+# define SYS_ulimit __NR_ulimit
+# define SYS_umount __NR_umount
+# define SYS_vm86 __NR_vm86
+# define SYS_vm86old __NR_vm86old
+# define SYS_waitpid __NR_waitpid
+#endif
+
+#endif /* _SYSCALL_H_ */
diff --git a/core/linux/vmkuw.c b/core/linux/vmkuw.c
new file mode 100644
index 000000000..5c4150391
--- /dev/null
+++ b/core/linux/vmkuw.c
@@ -0,0 +1,295 @@
+/* **********************************************************
+ * Copyright (c) 2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* vmkuw.c - VMKernel UserWorld support functions */
+
+#ifndef VMX86_SERVER
+#error This is only needed for supporting ESX/ESXi
+#endif
+
+
+/*
+ * Our goal is to use unmodified VMware header files as much as
+ * possible, and hopefully we can #define or rename any conflicting C
+ * or CPP identifiers before including our headers.  If that is
+ * insufficient we'll preferably generate clean copies with regexp
+ * replacements, and hopefully we should not need to resort to
+ * copy&paste of relevant pieces and try to maintain our own header
+ * copies.
+ *
+ */
+
+#include "vmware.h"
+#include "vm_basic_types.h"
+
+#include "uwvmk.h"
+#include "uwvmkAPI.h"
+
+#include "user_layout.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+/* Avoid problems with use of errno as var name in rest of file */
+#undef errno
+
+#include <sys/types.h>
+#include <sys/sysctl.h>
+
+/* Avoid multiple definitions of typedefs with VMware headers */
+#define DR_DO_NOT_DEFINE_uint64
+#define DR_DO_NOT_DEFINE_uint32
+#define DR_DO_NOT_DEFINE_int32
+#define DR_DO_NOT_DEFINE_int64
+#define DR_DO_NOT_DEFINE_MAX_MIN
+
+/* Allow our macro definitions to override ones defined in VMware headers */
+
+#undef ASSERT
+#undef ALIGNED
+#undef ASSERT_NOT_TESTED
+#undef ASSERT_NOT_IMPLEMENTED
+#undef PAGE_SIZE
+
+#include "globals_shared.h"
+#include "globals.h"
+#include "utils.h"
+
+#include "vmkuw.h"
+
+enum {VMKUW_TYPE_NONE,
+      VMKUW_TYPE_KERN,
+      VMKUW_TYPE_VISOR,
+      VMKUW_TYPE_KERN64,
+      VMKUW_TYPE_VISOR64};
+
+/* Determine if we are running on VMKernel and if yes, what type. */
+static int
+os_get_vmk_type(void)
+{
+    /* should get initialized early on before self-protection is turned on */
+    static int vmkernel_type = -1;
+
+    /* Note that using get_uname() might be more portable than sysctl */
+    /* but for now matching with bora/lib/misc/hostType.c */
+
+    if (vmkernel_type == -1) {
+        char osname[128];
+        size_t osname_len;
+        int kern_ostype_ctl[] = { CTL_KERN, KERN_OSTYPE };
+        int rc;
+      
+        /* Only if the KERN_OSTYPE sysctl returns one of
+         * USERWORLD_SYSCTL_KERN_OSTYPE, USERWORLD_SYSCTL_VISOR_OSTYPE
+         * or USERWORLD_SYSCTL_VISOR64_OSTYPE
+         */
+
+        osname_len = sizeof(osname);
+        rc = sysctl(kern_ostype_ctl, ARRAYSIZE(kern_ostype_ctl),
+                    osname, &osname_len,
+                    0, 0);
+        if (rc == 0) {
+            osname_len = MAX(sizeof (osname), osname_len);
+            if (strncmp(osname, USERWORLD_SYSCTL_VISOR64_OSTYPE, osname_len) == 0) {
+                vmkernel_type = VMKUW_TYPE_VISOR64;
+            } else if (strncmp(osname, USERWORLD_SYSCTL_KERN64_OSTYPE, osname_len) == 0) {
+                vmkernel_type = VMKUW_TYPE_KERN64;
+            } else if (strncmp(osname, USERWORLD_SYSCTL_VISOR_OSTYPE, osname_len) == 0) {
+                vmkernel_type = VMKUW_TYPE_VISOR;
+            } else if (strncmp(osname, USERWORLD_SYSCTL_KERN_OSTYPE, osname_len) == 0) {
+                vmkernel_type = VMKUW_TYPE_KERN;
+            } else {
+                vmkernel_type = VMKUW_TYPE_NONE;
+            }
+            LOG(GLOBAL, LOG_ALL, 1, "%s: vmkernel_type = %d\n", __func__, vmkernel_type);
+        } else {
+            LOG(GLOBAL, LOG_ALL, 1,
+                "sysctl([ CTL_KERN, KERN_OSTYPE ]) failed\n");
+            vmkernel_type = 0;
+        }
+    }
+    return (vmkernel_type);
+}
+
+bool
+os_in_vmkernel_userworld(void)
+{
+    /*
+     * We're running in a userworld if 'userworld' (ESX COS) or 'vmkernel' (ESXi) show up
+     * in the sysctl ostype field.  (Vanilla linux and COS applications return 'Linux'
+     * for this field.)
+     */
+    return os_get_vmk_type() > 0;
+}
+
+bool
+os_in_vmkernel_32bit(void)
+{
+    /* FIXME PR 363075: we can distinguish visor from classic but not 32-bit
+     * (3.5) from 64-bit (4.0); we allow running on 3.5 via this option.
+     * os_get_vmk_type() on esx 4.0 classic returns VMKUW_TYPE_KERN
+     * and 4.0 visor returns VMKUW_TYPE_VISOR instead of the 64
+     * versions.  We only use this as a runtime check for whether
+     * SYS_exit_group is implemented, so if a user on 3.5 or 3.5i does
+     * not set the runtime option we will simply hang on exit in debug
+     * and probably crash in release.
+     */
+    return (DYNAMO_OPTION(esx_32bit));
+}
+
+/* Exported symbol also allows scanning symbol table to statically
+ * verify if a library matches a running kernel
+ */
+const uint32 vmkuw_syscall = UWVMKSYSCALL_CHECKSUM; /* generated in uwvmk_dist.h */
+
+/* Verify that the UWVMK kernel and our userspace versions match. */
+static bool
+vmk_verify_syscall_version(void)
+{
+    uint32 uversion = vmkuw_syscall;
+    uint32 kversion = 0xbad0beef;
+    int rc;
+
+    if (DYNAMO_OPTION(vmkuw_version) != 0)
+        uversion = DYNAMO_OPTION(vmkuw_version);
+
+    ASSERT(os_in_vmkernel_userworld());
+
+    /* VMKernel_GetSyscallVersion has a constant syscall number across builds */
+    rc = VMKernel_GetSyscallVersion(&kversion);
+    if (rc != 0) {
+        LOG(GLOBAL, LOG_ALL, 1,
+            "vmk_verify_syscall_version: "
+            "Error getting vmkernel syscall version: %d\n", rc);
+        return false;
+    }
+
+    LOG(GLOBAL, LOG_ALL, 2,
+        "%s: Kernel UWVMK version = %#x, "
+        "User UWVMK version = %#x\n",
+        __func__, 
+        kversion, uversion);
+
+    if (kversion == uversion) {
+        return true;
+    }
+
+    LOG(GLOBAL, LOG_ALL, 1,
+        "%s: FAIL: kernel UWVMK version (%#x) and user-mode UWVMK "
+        "version (%#x) do not match.\n",
+        __func__, kversion, uversion);
+   
+    LOG(GLOBAL, LOG_ALL, 1,
+        "** Please verify that the DynamoRIO build corresponds to "
+        "the ESX/Visor build\n"
+        "(or set your VMTREE)**\n");
+
+    return false;
+}
+
+/* vmk-specific initializations */
+void
+vmk_init(void)
+{
+    if (!os_in_vmkernel_userworld()) {
+        LOG(GLOBAL, LOG_ALL, 1,
+            "** You are running a DynamoRIO version that supports ESX on Linux or COS.\n"
+            "  Did you want to run this application as a userworld?\n");
+        return;
+    }
+
+    if (!vmk_verify_syscall_version()) {
+        /* we should just abort here, no point in letting application run */
+        SYSLOG(SYSLOG_CRITICAL, ESX_VERSION_MISMATCH,
+               2, get_application_name(), get_application_pid());
+        os_terminate(NULL, TERMINATE_PROCESS);
+    }
+
+    ASSERT((byte *) DYNAMO_OPTION(vm_base) >= os_vmk_mmap_text_start() &&
+           ((byte *) DYNAMO_OPTION(vm_base)) + 
+           DYNAMO_OPTION(vm_max_offset) +
+           DYNAMO_OPTION(vm_size) <= os_vmk_mmap_text_end());
+}
+
+
+/*
+ * Actual coredump location may vary -
+ * in release builds check tail of /var/log/messages for generated coredump
+ */
+void
+vmk_request_live_coredump(const char *msg)
+{
+    char coreDumpPath[1024];
+    VMK_ReturnStatus status;
+
+    if (!os_in_vmkernel_userworld())
+        return;
+
+    status = VMKernel_LiveCoreDump(coreDumpPath, sizeof coreDumpPath);
+
+    if (status != VMK_OK) {
+        LOG(THREAD_GET, LOG_ALL, 1, "LiveCoreDump %s returned error code %#x\n",
+               msg, status);
+    } else {
+        LOG(THREAD_GET, LOG_ALL, 1, "LiveCoreDump %s created core at %s\n",
+            msg, coreDumpPath);
+    }
+}
+
+timestamp_t
+vmk_get_timer_frequency(void)
+{
+    VMK_ReturnStatus status;
+    uint32 tsc_kHz_estimate;
+
+    status = VMKernel_GetTSCkhzEstimate(&tsc_kHz_estimate);
+    if (status != VMK_OK) {
+        ASSERT_NOT_TESTED();
+        LOG(THREAD_GET, LOG_ALL, 1, "%s: failed, error code %#x\n", __func__, status);
+        return 3*1000*1000;     /* 3GHz as a reasonable estimate for the next decade */
+    }
+
+    LOG(THREAD_GET, LOG_ALL, 1, "%s: TSC estimate in KHz: %d\n", __func__, tsc_kHz_estimate);
+    return tsc_kHz_estimate;
+}
+
+byte *
+os_vmk_mmap_text_start(void)
+{
+    return (byte *) VMK_USER_FIRST_MMAP_TEXT_VADDR;
+}
+
+byte *
+os_vmk_mmap_text_end(void)
+{
+    return (byte *) ALIGN_FORWARD(VMK_USER_LAST_MMAP_TEXT_VADDR, PAGE_SIZE);
+}
diff --git a/core/linux/vmkuw.h b/core/linux/vmkuw.h
new file mode 100644
index 000000000..42b492cbb
--- /dev/null
+++ b/core/linux/vmkuw.h
@@ -0,0 +1,46 @@
+/* **********************************************************
+ * Copyright (c) 2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifndef VMKUW_H
+#define VMKUW_H
+
+void vmk_init(void);
+bool os_in_vmkernel_userworld(void);
+bool os_in_vmkernel_32bit(void);
+
+void vmk_request_live_coredump(const char *msg);
+timestamp_t vmk_get_timer_frequency(void);
+
+byte * os_vmk_mmap_text_start(void);
+byte * os_vmk_mmap_text_end(void);
+
+#endif
diff --git a/core/mkrelfile.pl b/core/mkrelfile.pl
new file mode 100755
index 000000000..d0cff7892
--- /dev/null
+++ b/core/mkrelfile.pl
@@ -0,0 +1,94 @@
+#!/usr/local/bin/perl
+
+# **********************************************************
+# Copyright (c) 2002-2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# Copyright (c) 2003-2007 Determina Corp.
+# Copyright (c) 2002-2003 Massachusetts Institute of Technology
+# Copyright (c) 2002 Hewlett Packard Company
+
+### mkrelfile.pl
+###
+### strips vararg ifdef DEBUG macros from files
+### major assumption: a line with LOG on it has nothing else after
+### the LOG!
+###
+
+$usage = "Usage: $0 <infile>\n";
+if ($#ARGV != 0) {
+    print "$usage";
+    exit 0;
+}
+$infile = $ARGV[0];
+shift;
+
+$inmacro = 0;
+open(IN, "< $infile") || die "Error: Couldn't open $infile for input\n";
+while (<IN>) {
+    $l = $_;
+    if ($inmacro) {
+	# try to find ; not inside a string
+	if ($l =~ /;[^\"]*$/) {
+	    $inmacro = 0;
+	    $l =~ s/[^;]*;//;
+	} else {
+            # print line so that compiler will have right line numbers
+            # use backslash so will work inside of multiline macros
+            print "\\\n";
+            next;
+	}
+    }
+    elsif ($l =~ /\WLOG\W/) {
+	if (! ($l =~ /;[^\"]*$/) ) {
+	    $inmacro = 1;
+	}
+	if ($l =~ /\\\s*$/) {
+	    $l =~ s/(\W)LOG.*(\s*\\\s*)$/\1;\2/;
+	} else {
+	    $l =~ s/(\W)LOG.*$/\1;/;
+	}
+    }
+    print "$l";
+}
+close(IN);
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/core/module_list.c b/core/module_list.c
new file mode 100644
index 000000000..529393cff
--- /dev/null
+++ b/core/module_list.c
@@ -0,0 +1,511 @@
+/* **********************************************************
+ * Copyright (c) 2008-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "globals.h"
+#include "instrument.h"
+#include <string.h> /* for memset */
+
+/* Used for maintaining our module list.  Custom field points to
+ * further module information from PE/ELF headers.
+ * module_data_lock needs to be held when accessing the custom data fields.
+ * Kept on the heap for selfprot (case 7957).
+ */
+vm_area_vector_t *loaded_module_areas;
+
+/* To avoid breaking further the abstraction of vm_area_vector_t's
+ * currently grabbing a separate lock.  In addition to protecting each
+ * entry's data, this lock also makes atomic a lookup & remove or a
+ * lookup & add sequences.  LOOKUP is read and user can use any
+ * fields, REMOVE is a write and nobody should be able to lookup a
+ * custom data that is going to get removed, ADD is a write only to
+ * avoid a memory leak of readding a module.
+ */
+DECLARE_CXTSWPROT_VAR(read_write_lock_t module_data_lock,
+                      INIT_READWRITE_LOCK(module_data_lock));
+
+
+/**************** module_data_lock routines *****************/
+
+void
+os_get_module_info_lock(void)
+{
+    if (loaded_module_areas != NULL)
+        read_lock(&module_data_lock);
+    /* else we assume past exit: FIXME: best to have exited bool */
+}
+
+void
+os_get_module_info_unlock(void)
+{
+    if (loaded_module_areas != NULL) {
+        ASSERT_OWN_READ_LOCK(true, &module_data_lock);
+        read_unlock(&module_data_lock);
+    }
+}
+
+void
+os_get_module_info_write_lock(void)
+{
+    if (loaded_module_areas != NULL)
+        write_lock(&module_data_lock);
+    /* else we assume past exit: FIXME: best to have exited bool */
+}
+
+void
+os_get_module_info_write_unlock(void)
+{
+    if (loaded_module_areas != NULL)
+        write_unlock(&module_data_lock);
+    /* else we assume past exit: FIXME: best to have exited bool */
+}
+
+bool
+os_get_module_info_locked(void)
+{
+    if (loaded_module_areas != NULL)
+        return READWRITE_LOCK_HELD(&module_data_lock);
+    return false;
+}
+
+bool
+os_get_module_info_write_locked(void)
+{
+    if (loaded_module_areas != NULL)
+        return self_owns_write_lock(&module_data_lock);
+    return false;
+}
+
+
+/**************** module_area routines *****************/
+
+static module_area_t *
+module_area_create(app_pc base, size_t view_size, bool at_map _IF_LINUX(uint64 inode)
+                   _IF_LINUX(const char *filename))
+{
+    module_area_t *ma =
+        HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, module_area_t, ACCT_VMAREAS, PROTECTED);
+    memset(ma, 0, sizeof(*ma));
+    ma->start = base;
+    ma->end = base + view_size;
+    os_module_area_init(ma, base, view_size, at_map _IF_LINUX(inode)
+                        _IF_LINUX(filename) HEAPACCT(ACCT_VMAREAS));
+    return ma;
+}
+
+static void
+module_area_delete(module_area_t *ma)
+{
+    os_module_area_reset(ma HEAPACCT(ACCT_VMAREAS));
+    free_module_names(&ma->names HEAPACCT(ACCT_VMAREAS));
+    HEAP_TYPE_FREE(GLOBAL_DCONTEXT, ma, module_area_t, ACCT_VMAREAS, PROTECTED);
+}
+
+
+/**************** init/exit routines *****************/
+
+void
+modules_init(void)
+{
+    VMVECTOR_ALLOC_VECTOR(loaded_module_areas, GLOBAL_DCONTEXT,
+                          VECTOR_SHARED | VECTOR_NEVER_MERGE
+                          /* case 10335: we always use module_data_lock */
+                          | VECTOR_NO_LOCK,
+                          loaded_module_areas);
+    os_modules_init();
+}
+
+bool
+is_module_list_initialized(void)
+{
+    return loaded_module_areas != NULL;
+}
+
+void
+modules_reset_list(void)
+{
+    vmvector_iterator_t vmvi;
+    /* need to free each entry */
+    os_get_module_info_write_lock();
+    /* note our iterator doesn't support remove, 
+     * anyways we need to free all entries here */
+    vmvector_iterator_start(loaded_module_areas, &vmvi);
+    while (vmvector_iterator_hasnext(&vmvi)) {
+        app_pc start, end;
+        module_area_t *ma = (module_area_t*)vmvector_iterator_next(&vmvi, &start, &end);
+        ASSERT(ma != NULL);
+        ASSERT(ma->start == start && ma->end == end);
+        ma->flags |= MODULE_BEING_UNLOADED;
+        module_area_delete(ma);
+    }
+    vmvector_iterator_stop(&vmvi);
+    vmvector_reset_vector(GLOBAL_DCONTEXT, loaded_module_areas);
+    os_get_module_info_write_unlock();
+}
+
+void
+modules_exit(void)
+{
+    LOG(GLOBAL, LOG_VMAREAS, 2, "Module list at exit\n");
+    DOLOG(2, LOG_VMAREAS, { print_modules(GLOBAL, DUMP_NOT_XML); });
+
+    os_modules_exit();
+
+    modules_reset_list();
+    vmvector_delete_vector(GLOBAL_DCONTEXT, loaded_module_areas);
+    loaded_module_areas = NULL;
+    DELETE_READWRITE_LOCK(module_data_lock);
+}
+
+
+/**************** module_list updating routines *****************/
+
+void
+module_list_add(app_pc base, size_t view_size, bool at_map _IF_LINUX(uint64 inode)
+                _IF_LINUX(const char *filename))
+{
+#ifdef CLIENT_INTERFACE
+    module_data_t *client_data = NULL;
+    bool inform_client = false;
+#endif
+
+    ASSERT(loaded_module_areas != NULL);
+    ASSERT(!vmvector_overlap(loaded_module_areas, base, base+view_size));
+    os_get_module_info_write_lock();
+    /* defensively checking */
+    if (!vmvector_overlap(loaded_module_areas, base, base+view_size)) {
+        module_area_t *ma = module_area_create(base, view_size, at_map _IF_LINUX(inode)
+                                               _IF_LINUX(filename));
+        ASSERT(ma != NULL);
+        
+        LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 1, "module %s ["PFX","PFX"] added\n",
+            (GET_MODULE_NAME(&ma->names) == NULL) ? "<no name>" :
+            GET_MODULE_NAME(&ma->names), base, base+view_size);
+        
+        /* note that there is normally no need to hold even a
+         * read_lock to make sure that nobody is about to remove
+         * this entry.  While next to impossible that the
+         * currently added module will get unloaded by another
+         * thread, we do grab a full write lock around this safe
+         * lookup/add.
+         */
+        vmvector_add(loaded_module_areas, base, base+view_size, ma);
+        
+#ifdef CLIENT_INTERFACE
+        /* inform clients of module loads, we copy the data now and wait to
+         * call the client till after we've released the module areas lock */
+        if (!IS_STRING_OPTION_EMPTY(client_lib)) {
+            client_data = copy_module_area_to_module_data(ma);
+            inform_client = true;
+        }
+#endif
+    } else {
+        /* already added! */
+        /* only possible for manual NtMapViewOfSection, loader 
+         * can't be doing this to us */
+        ASSERT_CURIOSITY(false && "image load race");
+        /* do nothing */
+    }
+    os_get_module_info_write_unlock();
+#ifdef CLIENT_INTERFACE
+    if (inform_client) {
+        instrument_module_load(client_data, false /*loading now*/);
+        dr_free_module_data(client_data);
+    }
+#endif
+}
+
+void
+module_list_remove(app_pc base, size_t view_size)
+{
+    /* lookup and free module */
+#ifdef CLIENT_INTERFACE
+    module_data_t *client_data = NULL;
+    bool inform_client = false;
+#endif
+    module_area_t *ma;
+    
+    /* note that vmvector_lookup doesn't protect the custom data,
+     * and we need to bracket a lookup and remove in an unlikely
+     * application race (note we pre-process unmap)
+     */
+    ASSERT(loaded_module_areas != NULL);
+    os_get_module_info_write_lock();
+    ASSERT(vmvector_overlap(loaded_module_areas, base, base+view_size));
+    ma = (module_area_t*)vmvector_lookup(loaded_module_areas, base);
+    ASSERT_CURIOSITY(ma != NULL); /* loader can't have a race */
+    
+#ifdef CLIENT_INTERFACE
+    /* inform clients of module unloads, we copy the data now and wait to
+     * call the client till after we've released the module areas lock */
+    if (!IS_STRING_OPTION_EMPTY(client_lib)) {
+        client_data = copy_module_area_to_module_data(ma);
+        inform_client = true;
+    }
+#endif
+    /* defensively checking */
+    if (ma != NULL) {
+        vmvector_remove(loaded_module_areas, base, base+view_size);
+        /* extra nice, free only after removing from vector */
+        module_area_delete(ma);
+    }
+    ASSERT(!vmvector_overlap(loaded_module_areas, base, base+view_size));
+    os_get_module_info_write_unlock();
+#ifdef CLIENT_INTERFACE
+    if (inform_client) {
+        instrument_module_unload(client_data);
+        dr_free_module_data(client_data);
+    }
+#endif
+}
+
+
+/**************** module flag routines *****************/
+
+static bool
+os_module_set_flag_value(app_pc module_base, uint flag, bool set)
+{
+    module_area_t *ma;
+    bool found = false;
+    bool own_lock = os_get_module_info_write_locked();
+    if (!own_lock)
+        os_get_module_info_write_lock();
+    ma = module_pc_lookup(module_base);
+    if (ma != NULL) {
+        if (set)
+            ma->flags |= flag;
+        else
+            ma->flags &= ~flag;
+        found = true;
+    } 
+    if (!own_lock)
+        os_get_module_info_write_unlock();
+    return found;
+}
+
+bool
+os_module_set_flag(app_pc module_base, uint flag)
+{
+    return os_module_set_flag_value(module_base, flag, true);
+}
+
+bool
+os_module_clear_flag(app_pc module_base, uint flag)
+{
+    return os_module_set_flag_value(module_base, flag, false);
+}
+
+bool
+os_module_get_flag(app_pc module_base, uint flag)
+{
+    module_area_t *ma;
+    bool has_flag = false;
+    os_get_module_info_lock();
+    ma = module_pc_lookup(module_base);
+    if (ma != NULL) {
+        /* interface is for just one flag so no documentation of ANY vs ALL */
+        has_flag = TESTANY(flag, ma->flags);
+    } 
+    os_get_module_info_unlock();
+    return has_flag;
+}
+
+
+/**************** module_area accessor routines (os shared) *****************/
+
+/* Returns the module_area_ for the module containing pc (NULL if no such module is found)
+ */
+module_area_t *
+module_pc_lookup(byte *pc)
+{
+    ASSERT(loaded_module_areas != NULL);
+    ASSERT(os_get_module_info_locked());
+    return (module_area_t *)vmvector_lookup(loaded_module_areas, pc);
+}
+
+/* Returns true if the region overlaps any module areas. */
+bool
+module_overlaps(byte *pc, size_t len)
+{
+    ASSERT(loaded_module_areas != NULL);
+    ASSERT(os_get_module_info_locked());
+    return vmvector_overlap(loaded_module_areas, pc, pc+len);
+}
+
+/* Some callers want strdup, some want a passed-in buffer, and some want
+ * a buffer but if it's too small they then want strdup.
+ */
+static const char *
+os_get_module_name_internal(const app_pc pc, char *buf, size_t buf_len, bool truncate,
+                            size_t *copied HEAPACCT(which_heap_t which))
+{
+    const char *name = NULL;
+    size_t num = 0;
+    os_get_module_info_lock();
+    if (os_get_module_name(pc, &name) && name != NULL) {
+        if (buf == NULL || (!truncate && strlen(name) >= buf_len)) {
+            DOSTATS({
+                if (buf != NULL)
+                    STATS_INC(app_modname_too_long);
+            });
+            name = dr_strdup(name HEAPACCT(which));
+        } else {
+            strncpy(buf, name, buf_len);
+            buf[buf_len - 1] = '\0';
+            num = MIN(buf_len - 1, strlen(name));
+            name = buf;
+        }
+    } else if (buf != NULL)
+        buf[0] = '\0';
+    os_get_module_info_unlock();
+    if (copied != NULL)
+        *copied = num;
+    return name;
+}
+
+/* Convenience wrapper so we don't have to remember arg position of name
+ * in os_get_module_info().  Caller must hold module_data_lock.
+ * Unlike os_get_module_info(), sets *name to NULL if return value is false;
+ */
+bool
+os_get_module_name(const app_pc pc, /* OUT */ const char **name)
+{
+    module_area_t *ma;
+    
+    ASSERT(os_get_module_info_locked());
+    ma = module_pc_lookup(pc);
+    
+    if (ma != NULL)
+        *name = GET_MODULE_NAME(&ma->names);
+    else
+        *name = NULL;
+    return ma != NULL;
+}
+
+const char *
+os_get_module_name_strdup(const app_pc pc HEAPACCT(which_heap_t which))
+{
+    return os_get_module_name_internal(pc, NULL, 0, false/*no truncate*/,
+                                       NULL HEAPACCT(which));
+}
+
+/* Returns the number of characters copied (maximum is buf_len -1).
+ * If there is no module at pc, or no module name available, 0 is
+ * returned and the buffer set to "".
+ */
+size_t
+os_get_module_name_buf(const app_pc pc, char *buf, size_t buf_len)
+{
+    size_t copied;
+    os_get_module_name_internal(pc, buf, buf_len, true/*truncate*/,
+                                &copied HEAPACCT(ACCT_OTHER));
+    return copied;
+}
+
+/* Copies the module name into buf and returns a pointer to buf,
+ * unless buf is too small, in which case the module name is strdup-ed
+ * and a pointer to it returned (which the caller must strfree).
+ * If there is no module name, returns NULL.
+ */
+const char *
+os_get_module_name_buf_strdup(const app_pc pc, char *buf, size_t buf_len
+                              HEAPACCT(which_heap_t which))
+{
+    return os_get_module_name_internal(pc, buf, buf_len, false/*no truncate*/,
+                                       NULL HEAPACCT(which));
+}
+
+size_t
+os_module_get_view_size(app_pc mod_base)
+{
+    module_area_t *ma;
+    size_t view_size = 0;
+    os_get_module_info_lock();
+    ma = module_pc_lookup(mod_base);
+    if (ma != NULL) {
+        view_size = ma->end - ma->start;
+    } 
+    os_get_module_info_unlock();
+    return view_size;
+}
+
+
+/**************** module iterator routines *****************/
+
+struct _module_iterator_t {
+    vmvector_iterator_t vmvi;
+};
+
+/* Initialize a new module_iterator */
+module_iterator_t *
+module_iterator_start(void)
+{
+    module_iterator_t *mi = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, module_iterator_t,
+                                            ACCT_OTHER, UNPROTECTED);
+    ASSERT(loaded_module_areas != NULL);
+    /* loaded_module_areas doesn't use the vector lock */
+    os_get_module_info_lock();
+    vmvector_iterator_start(loaded_module_areas, &mi->vmvi);
+    return mi;
+}
+
+/* Returns true if there is another module in the list */
+bool
+module_iterator_hasnext(module_iterator_t *mi)
+{
+    ASSERT(os_get_module_info_locked());
+    return vmvector_iterator_hasnext(&mi->vmvi);
+}
+
+/* Retrieves the module_data_t for a loaded module */
+module_area_t *
+module_iterator_next(module_iterator_t *mi)
+{
+    app_pc start, end;
+    module_area_t *ma = (module_area_t *)
+        vmvector_iterator_next(&mi->vmvi, &start, &end);
+    ASSERT(os_get_module_info_locked());
+    ASSERT(ma != NULL);
+    ASSERT(ma->start == start && ma->end == end);
+    return ma;
+}
+
+/* User should call this routine to free the iterator */
+void
+module_iterator_stop(module_iterator_t *mi)
+{
+    vmvector_iterator_stop(&mi->vmvi);
+    /* loaded_module_areas doesn't use the vector lock */
+    ASSERT(os_get_module_info_locked());
+    os_get_module_info_unlock();
+    HEAP_TYPE_FREE(GLOBAL_DCONTEXT, mi, module_iterator_t, ACCT_OTHER, UNPROTECTED);
+}
+
diff --git a/core/module_shared.h b/core/module_shared.h
new file mode 100644
index 000000000..e96df73ec
--- /dev/null
+++ b/core/module_shared.h
@@ -0,0 +1,348 @@
+/* **********************************************************
+ * Copyright (c) 2008-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifndef MODULE_LIST_H
+#define MODULE_LIST_H
+
+/* DR_API EXPORT TOFILE dr_tools.h */
+/* DR_API EXPORT BEGIN */
+/**************************************************
+ * MODULE INFORMATION TYPES
+ */
+
+/**
+ * Type used for dr_get_proc_address().  On windows this is the same as the base
+ * address of the module.
+ */
+typedef void * module_handle_t;
+
+#ifdef WINDOWS
+
+#define MODULE_FILE_VERSION_INVALID ULLONG_MAX
+
+/**
+ * Used to hold .rsrc section version number information. This number is usually
+ * presented as p1.p2.p3.p4 by PE parsing tools.
+ */
+typedef union _version_number_t {
+    uint64 version;  /**< Representation as a 64-bit integer. */
+    struct {
+        uint ms;     /**< */
+        uint ls;     /**< */
+    } version_uint;  /**< Representation as 2 32-bit integers. */
+    struct {
+        ushort p2;   /**< */
+        ushort p1;   /**< */
+        ushort p4;   /**< */
+        ushort p3;   /**< */
+    } version_parts; /**< Representation as 4 16-bit integers. */
+} version_number_t;
+
+#endif
+
+/* DR_API EXPORT END */
+
+#include "module.h" /* include os_specific header */
+
+/* DR_API EXPORT TOFILE dr_tools.h */
+/* DR_API EXPORT BEGIN */
+
+/**
+ * Holds the names of a module.  This structure contains multiple
+ * fields corresponding to different sources of a module name.  Note
+ * that some of these names may not exist for certain modules.  It is
+ * highly likely, however, that at least one name is available.  Use
+ * dr_module_preferred_name() on the parent _module_data_t to get the
+ * preferred name of the module.
+ */
+typedef struct _module_names_t {
+    const char *module_name; /**< On windows this name comes from the PE header exports
+                              * section (NULL if the module has no exports section).  On
+                              * Linux the name will come from the ELF DYNAMIC program 
+                              * header (NULL if the module has no SONAME entry). */
+    const char *file_name; /**< The file name used to load this module. Note - on Windows
+                            * this is not always available. */
+#ifdef WINDOWS
+    const char *exe_name; /**< If this module is the main executable of this process then
+                           * this is the executable name used to launch the process (NULL
+                           * for all other modules). */
+    const char *rsrc_name; /**< The internal name given to the module in its resource
+                            * section. Will be NULL if the module has no resource section
+                            * or doesn't set this field within it. */
+#else /* LINUX */
+    uint64 inode; /**< The inode of the module file mapped in. */
+#endif
+} module_names_t;
+
+/* DR_API EXPORT END */
+
+#ifdef WINDOWS
+/* Introduced as part of case 9842; see module_names_t above.  Shouldn't
+ * put case number in exported parts (CLIENT api), so adding it here.
+ *
+ * The name precedence order is as follows,
+ *      Choice #1: PE exports name.
+ *      Choice #2: executable qualified name (This would be the last choice 
+ *                  except historically it's been #2 so we'll stick with that).
+ *      Choice #3: .rsrc original filename, already strduped.
+ *      Choice #4: file name. 
+ */
+#define GET_MODULE_NAME(mod_names)                                          \
+    (((mod_names)->module_name != NULL) ? (mod_names)->module_name :        \
+     (((mod_names)->exe_name != NULL) ? (mod_names)->exe_name :             \
+      (((mod_names)->rsrc_name != NULL) ? (mod_names)->rsrc_name :          \
+       (((mod_names)->file_name != NULL) ? (mod_names)->file_name : NULL))))
+#else
+/* For Linux the precedence order is as follows,
+ * Choice #1: the module_name (SONAME entry from the DYNAMIC program_header, if it exists)
+ * Choice #2: the filename of the file mapped in (from the maps file)
+ */
+#define GET_MODULE_NAME(mod_names)                                          \
+    (((mod_names)->module_name != NULL) ? (mod_names)->module_name :        \
+     (((mod_names)->file_name != NULL) ? (mod_names)->file_name : NULL))
+#endif
+
+/* Used to augments the basic vm_area_vector_t interval, all fields that
+ * we get from the loader or PE/ELF header should be maintained here. This is what
+ * we store in the loaded_modules_areas vector. */
+typedef struct _module_area_t {
+    /* vm_area_t has start and end fields, but we're duplicating them in module_data_t so
+     * the information is available to the module iterator. */
+    /* On windows start-end bound the view of the module that was mapped. This view size
+     * is almost always the same as the internal_size == pe size (the full size of the
+     * module).  On Vista we've seen drivers mapped into user processes (leading to
+     * view size = PAGE_ALIGN(pe_size) > pe size) and partial mappings of child
+     * executables (leading to view size < pe_size). */
+    app_pc start;
+    app_pc end;
+
+    app_pc entry_point;
+
+    uint flags;
+
+    module_names_t names;
+
+    os_module_data_t os_data; /* os specific data for this module */
+} module_area_t;
+
+/* Flags used in module_area_t.flags */
+enum {
+    /* case 9653: only the 1st coarse unit in a module's +x region(s) is persisted */
+    MODULE_HAS_PRIMARY_COARSE = 0x00000001,
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+    /* case 8648: did we load persisted RCT data for the whole module? */
+    MODULE_RCT_LOADED      = 0x00000002,
+#endif
+#ifdef RETURN_AFTER_CALL /* should include RCT_IND_BRANCH but matching other code */
+    MODULE_HAS_BORLAND_SEH = 0x00000004,
+#endif
+    /* case 9672: used to detect whether to preserve persisted RCT on a flush */
+    MODULE_BEING_UNLOADED  = 0x00000008,
+    /* case 9799: used to ensure persistent caches are safe to use */
+    MODULE_WAS_EXEMPTED    = 0x00000010,
+#if defined(X64) && (defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH))
+    /* PR 277064/277044: have we scanned the module yet? */
+    MODULE_RCT_SCANNED     = 0x00000020,
+#endif
+};
+
+/**************** init/exit routines *****************/
+void modules_init(void);
+bool is_module_list_initialized(void);
+void modules_exit(void);
+void modules_reset_list(void);
+
+/**************** module_list updating routines *****************/
+void module_list_add(app_pc base, size_t view_size, bool at_map _IF_LINUX(uint64 inode)
+                     _IF_LINUX(const char *filename));
+void module_list_remove(app_pc base, size_t view_size);
+
+/**************** module_data_lock routines *****************/
+void os_get_module_info_lock(void);
+void os_get_module_info_unlock(void);
+void os_get_module_info_write_lock(void);
+void os_get_module_info_write_unlock(void);
+bool os_get_module_info_locked(void);
+bool os_get_module_info_write_locked(void);
+
+/**************** module flag routines *****************/
+bool os_module_set_flag(app_pc module_base, uint flag);
+bool os_module_clear_flag(app_pc module_base, uint flag);
+bool os_module_get_flag(app_pc module_base, uint flag);
+
+
+/**************** module_area accessor routines (os shared) *****************/
+
+module_area_t * module_pc_lookup(byte *pc);
+
+bool module_overlaps(byte *pc, size_t len);
+
+/* Unlike os_get_module_info(), sets *name to NULL if return value is false */
+bool os_get_module_name(const app_pc pc, /*OUT*/ const char **name);
+
+const char * os_get_module_name_strdup(const app_pc pc HEAPACCT(which_heap_t which));
+
+/* Returns the number of characters copied (maximum is buf_len -1).
+ * If there is no module at pc, or no module name available, 0 is
+ * returned and the buffer set to "".
+ */
+size_t os_get_module_name_buf(const app_pc pc, char *buf, size_t buf_len);
+
+/* Copies the module name into buf and returns a pointer to buf,
+ * unless buf is too small, in which case the module name is strdup-ed
+ * and a pointer to it returned (which the caller must strfree).
+ * If there is no module name, returns NULL.
+ */
+const char * os_get_module_name_buf_strdup(const app_pc pc, char *buf, size_t buf_len
+                                           HEAPACCT(which_heap_t which));
+
+size_t os_module_get_view_size(app_pc module_base);
+
+
+/**************** module iterator routines *****************/
+/* module iterator fields */
+typedef struct _module_iterator_t module_iterator_t;
+/* If plan to write to module_area fields must call os_get_module_info_write_[un]lock
+ * around entire usage of the iterator. */
+module_iterator_t * module_iterator_start(void);
+bool module_iterator_hasnext(module_iterator_t *mi);
+module_area_t * module_iterator_next(module_iterator_t *mi);
+void module_iterator_stop(module_iterator_t *mi);
+
+/***************** in os specific module.c *****************/
+void os_modules_init(void);
+void os_modules_exit(void);
+void os_module_area_init(module_area_t *ma, app_pc base, size_t view_size,
+                         bool at_map _IF_LINUX(uint64 inode)
+                         _IF_LINUX(const char *filename) HEAPACCT(which_heap_t which));
+void os_module_area_reset(module_area_t *ma HEAPACCT(which_heap_t which));
+void free_module_names(module_names_t *mod_names HEAPACCT(which_heap_t which));
+
+
+
+/**************************************************************************************/
+/* Moved from os_shared.h to use typedefs here, in <os>/module.c 
+ * Should clean up and see if these can be shared/obsoleted by the os shared mod list. */
+bool os_get_module_info_all_names(const app_pc pc,
+                                  /* FIXME PR 215890: does ELF64 use 64-bit timestamp
+                                   * or checksum?
+                                   */
+                                  uint *checksum, uint *timestamp,
+                                  size_t *size, module_names_t **names,
+                                  size_t *code_size, uint64 *file_version);
+
+generic_func_t
+get_proc_address(module_handle_t lib, const char *name);
+
+void print_modules(file_t f, bool dump_xml);
+
+const char *get_module_short_name(app_pc pc HEAPACCT(which_heap_t which));
+
+app_pc get_module_base(app_pc pc);
+
+/* Returns true if [start_pc, end_pc) is within a single code section.
+ * Returns the bounds of the enclosing section in sec_start and sec_end.
+ * Note that unlike is_in_*_section routines, does NOT merge adjacent sections. */
+bool is_range_in_code_section(app_pc module_base, app_pc start_pc, app_pc end_pc,
+                              app_pc *sec_start /* OPTIONAL OUT */,
+                              app_pc *sec_end /* OPTIONAL OUT */);
+/* Returns true if addr is in a code section and if so returns in sec_start and sec_end
+ * the bounds of the section containing addr (MERGED with adjacent code sections). */
+bool is_in_code_section(app_pc module_base, app_pc addr,
+                        app_pc *sec_start /* OPTIONAL OUT */,
+                        app_pc *sec_end /* OPTIONAL OUT */);
+/* Same as above only for initalized data sections instead of code. */
+bool is_in_dot_data_section(app_pc module_base, app_pc addr,
+                            app_pc *sec_start /* OPTIONAL OUT */,
+                            app_pc *sec_end /* OPTIONAL OUT */);
+/* Same as above only for any section instead of code. */
+bool is_in_any_section(app_pc module_base, app_pc addr,
+                       app_pc *sec_start /* OPTIONAL OUT */,
+                       app_pc *sec_end /* OPTIONAL OUT */);
+
+bool is_mapped_as_image(app_pc module_base);
+
+
+bool os_get_module_info(const app_pc pc, uint *checksum,
+                        uint *timestamp, size_t *size, const char **name,
+                        size_t *code_size, uint64 *file_version);
+static inline bool
+module_info_exists(const app_pc pc)
+{
+    return os_get_module_info(pc, NULL, NULL, NULL, NULL, NULL, NULL);
+}
+
+bool get_named_section_bounds(app_pc module_base, const char *name,
+                              app_pc *start/*OUT*/, app_pc *end/*OUT*/);
+bool get_module_company_name(app_pc mod_base, char *out_buf, size_t out_buf_size);
+
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+rct_module_table_t *
+os_module_get_rct_htable(app_pc pc, rct_type_t which);
+#endif
+
+typedef struct {
+    /* MD5 of portions of files mapped as image sections */
+    byte full_MD5[MD5_RAW_BYTES];
+    /* a full digest uses all readable raw bytes (as also present in
+     * the file) in sections loaded in memory */
+    /* Note that for most files this should result in the same value
+     * as md5sum, except for files with digital signatures or
+     * debugging information that is not loaded in memory */
+
+    /* an MD5 digest of only header and footer of file with lengths
+     * specified by aslr_short_digest */
+    byte short_MD5[MD5_RAW_BYTES];
+
+    /* FIXME: case 4678 about possible NYI subregion checksums
+     * amenable to lazy evaluation.  Such digests which will be of
+     * dynamic size and file offset which will have to be specified
+     * here. */
+} module_digest_t;
+/* FIXME: rename since being used for module-independent purposes? */
+
+void
+module_calculate_digest(/*OUT*/ module_digest_t *digest,
+                        app_pc module_base, 
+                        size_t module_size,
+                        bool full_digest,
+                        bool short_digest,
+                        uint short_digest_size,
+                        uint sec_characteristics);
+
+/* actually in utils.c since os-independent */
+bool
+module_digests_equal(const module_digest_t *calculated_digest, 
+                     const module_digest_t *matching_digest,
+                     bool check_short, bool check_full);
+
+#endif /* MODULE_LIST_H */
diff --git a/core/moduledb.c b/core/moduledb.c
new file mode 100644
index 000000000..01e0c8bfa
--- /dev/null
+++ b/core/moduledb.c
@@ -0,0 +1,660 @@
+/* **********************************************************
+ * Copyright (c) 2006-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2006-2007 Determina Corp. */
+
+#include "globals.h"
+#include "utils.h"
+#include "moduledb.h"
+#include "module_shared.h"
+#include <string.h>  /* for memset, strlen */
+
+/* An array of pointers to the various exempt lists indexed by the 
+ * moduledb_exempt_list_t enum.  We have the ugliness of an extra indirection
+ * and dynamic sizing to move the array into the heap and avoid self 
+ * protection changes. */
+static char **exemption_lists = NULL;
+#define GET_EXEMPT_LIST(list) (exemption_lists[(list)])
+
+/* protects access to the above lists, we assume it is safe to check for NULL
+ * without holding the lock */
+DECLARE_CXTSWPROT_VAR(read_write_lock_t moduledb_lock,
+                      INIT_READWRITE_LOCK(moduledb_lock));
+
+static char *
+moduledb_add_to_exempt_list(char *list, const char *name)
+{
+    size_t cur_size_less_null = (list == NULL) ? 0 : strlen(list);
+    size_t needed_size = cur_size_less_null +
+        ((cur_size_less_null > 0) ? 1 /* ; */ : 0) +
+        strlen(name) + 1 /* NULL */;
+    char *new_list = (char *)
+        global_heap_realloc(list, cur_size_less_null + 1 /* NULL */,
+                            needed_size, sizeof(*list) HEAPACCT(ACCT_OTHER));
+    if (cur_size_less_null == 0)
+        *new_list = '\0';
+    else
+        ASSERT(strlen(new_list) == cur_size_less_null);
+
+    if (cur_size_less_null > 0) {
+        /* add separator */
+        strncat(new_list, ";", needed_size - cur_size_less_null - 1 /* null */);
+    }
+    strncat(new_list, name, needed_size - strlen(new_list) - 1 /* NULL */);
+
+    return new_list;
+}
+
+static char *
+moduledb_remove_from_exempt_list(char *list, const char *name)
+{
+    /* FIXME - does nothing, worry is if enough unique names are added to the
+     * list it could use a substantial amt of memory and take a while to walk
+     * (though note we won't get duplicate entries in module churn situations)
+     * or we could an get an accidental name collision with a later module. */
+    return list;
+}
+
+static void
+moduledb_update_exempt_list(char **list, const char *name, bool add)
+{
+    LOG(GLOBAL, LOG_MODULEDB, 2,
+        "\tlist before update \"%s\"\n", (*list == NULL) ? "" : *list);
+    write_lock(&moduledb_lock);
+    if (add && (*list == NULL || !check_filter(*list, name))) {
+        *list = moduledb_add_to_exempt_list(*list, name);
+    } else if (!add && *list != NULL && check_filter(*list, name)) {
+        *list = moduledb_remove_from_exempt_list(*list, name);
+    }
+    write_unlock(&moduledb_lock);
+    LOG(GLOBAL, LOG_MODULEDB, 2,
+        "\tlist after update \"%s\"\n", (*list == NULL) ? "" : *list);
+}
+
+static void
+moduledb_process_relaxation_flags(uint flags, const char *name, bool add)
+{
+    ASSERT_NOT_IMPLEMENTED(!TESTANY(~(MODULEDB_ALL_SECTIONS_BITS|MODULEDB_RCT_EXEMPT_TO|
+                                      MODULEDB_REPORT_ON_LOAD|MODULEDB_DLL2HEAP|
+                                      MODULEDB_DLL2STACK), flags));
+    if (TEST(MODULEDB_RCT_EXEMPT_TO, flags)) {
+        LOG(GLOBAL, LOG_MODULEDB, 1,
+            "%s module %s to moduledb exempt rct\n",
+            add ? "Adding" : "Removing", name);
+        moduledb_update_exempt_list(&GET_EXEMPT_LIST(MODULEDB_EXEMPT_RCT),
+                                    name, add);
+    }
+    if (TEST(MODULEDB_DLL2HEAP, flags)) {
+        LOG(GLOBAL, LOG_MODULEDB, 1,
+            "%s module %s to moduledb exempt dll2heap\n",
+            add ? "Adding" : "Removing", name);
+        moduledb_update_exempt_list(&GET_EXEMPT_LIST(MODULEDB_EXEMPT_DLL2HEAP),
+                                    name, add);
+    }
+    if (TEST(MODULEDB_DLL2STACK, flags)) {
+        LOG(GLOBAL, LOG_MODULEDB, 1,
+            "%s module %s to moduledb exempt dll2stack\n",
+            add ? "Adding" : "Removing", name);
+        moduledb_update_exempt_list(&GET_EXEMPT_LIST(MODULEDB_EXEMPT_DLL2STACK),
+                                    name, add);
+    }
+    if (TESTANY(MODULEDB_ALL_SECTIONS_BITS, flags)) {
+        uint all_sections = (flags & MODULEDB_ALL_SECTIONS_BITS) >>
+            MODULEDB_ALL_SECTIONS_SHIFT;
+        ASSERT_NOT_IMPLEMENTED(all_sections == SECTION_ALLOW);
+        if (all_sections == SECTION_ALLOW) {
+            LOG(GLOBAL, LOG_MODULEDB, 1,
+                "%s module %s to moduledb exec if image\n",
+                add ? "Adding" : "Removing", name);
+            moduledb_update_exempt_list(&GET_EXEMPT_LIST(MODULEDB_EXEMPT_IMAGE),
+                                        name, add);
+        }
+    }
+}
+
+/* for DO_THRESHOLD_SAFE statics */
+START_DATA_SECTION(FREQ_PROTECTED_SECTION, "w")
+
+void
+moduledb_report_exemption(char *fmt, app_pc addr1, app_pc addr2, const char *name)
+{
+    ASSERT(exemption_lists != NULL);
+    /* FIXME - need a release version of this */
+    /* FIXME - should these respect some MODULEDB_REPORT flag? */
+    DODEBUG({
+        /* FIXME - would be nice to only report unique module names per type. */
+        DO_THRESHOLD_SAFE(DYNAMO_OPTION(moduledb_exemptions_report_max),
+                          FREQ_PROTECTED_SECTION, {
+            /* < max */
+            SYSLOG_INTERNAL_WARNING(fmt, addr1, addr2, name);
+        },  { /* > max -> nothing */ });
+    });
+}
+
+/* somewhat arbitrary, but more than long enough for current usage */
+#define MAX_COMPANY_NAME 256
+
+void
+moduledb_process_image(const char *name, app_pc base, bool add)
+{
+    char company_name[MAX_COMPANY_NAME];
+    bool got_company_name;
+    bool relax = true;
+    ASSERT(exemption_lists != NULL);
+
+    if (!DYNAMO_OPTION(use_moduledb))
+        return;
+
+    /* caller has already verified this is a pe module */
+    got_company_name =
+        get_module_company_name(base, company_name,
+                                BUFFER_SIZE_ELEMENTS(company_name));
+    if (!got_company_name)
+        company_name[0] = '\0';
+    
+    if (got_company_name && company_name[0] != '\0' &&
+        (!IS_STRING_OPTION_EMPTY(whitelist_company_names_default) ||
+         !IS_STRING_OPTION_EMPTY(whitelist_company_names)) &&
+        check_list_default_and_append(dynamo_options.whitelist_company_names_default,
+                                      dynamo_options.whitelist_company_names,
+                                      company_name)) {
+        relax = false;
+        LOG(GLOBAL, LOG_MODULEDB, 1,
+            "Found module \"%s\" from whitelisted company \"%s\"\n",
+            name == NULL ? "no-name" : name, company_name);
+        /* FIXME - not all of our modules have the Company name
+         * field set (drpreinject & liveshields don't), need to avoid
+         * relaxing for those. Should add version info and also check
+         * nodgemgr and our other tools etc.  xref case 8723 */ 
+    }
+    if (relax) {
+        if (name == NULL || *name == '\0') {
+            if (add) {
+                /* FIXME - not able to relax for these nameless dlls, there shouldn't be
+                 * too many of these once we also fall back to the version original
+                 * filename for modules with no exports and we'll eventually exempt by
+                 * area in the modules list instead of by name anyways. */
+                /* FIXME - would be nice to use get_module_name to get the filename of
+                 * the module at least, but this a bad time w/respect to the loader to
+                 * be walking the lists. */
+                SYSLOG_INTERNAL_WARNING("Unable to relax for nameless unknown module "
+                                        "from \"%s\" @"PFX, company_name, base);
+            }
+        } else {
+            LOG(GLOBAL, LOG_MODULEDB, 1, "Loaded unknown module %s\n", name);
+            /* process the relaxations */
+            moduledb_process_relaxation_flags(DYNAMO_OPTION(unknown_module_policy),
+                                              name, add);
+            /* FIXME - prob. too noisy, on my machine there are
+             * usually 5 of these per process, two for logitech mouse hook
+             * dlls, 1 for drpreinject and 2 for Norton av. */
+            if (add && TEST(MODULEDB_REPORT_ON_LOAD,
+                            DYNAMO_OPTION(unknown_module_policy))) {
+                /* FIXME - will prob. need a release version of this */
+                DODEBUG({
+                    DO_THRESHOLD_SAFE(DYNAMO_OPTION(unknown_module_load_report_max),
+                                      FREQ_PROTECTED_SECTION, {
+                        /* < max */
+                        SYSLOG_INTERNAL_INFO("Relaxing protections for unknown module "
+                                             "%s @"PFX" from \"%s\"",
+                                             name, base, company_name);
+                    }, { /* > max -> nothing */ });
+                });
+            }
+        }
+    }
+}
+
+/* back to normal section */
+END_DATA_SECTION()
+
+static const char *const
+exempt_list_names[MODULEDB_EXEMPT_NUM_LISTS] = { "rct", "image", "dll2heap", "dll2stack" };
+
+void
+moduledb_init()
+{
+    uint exempt_array_size = (MODULEDB_EXEMPT_NUM_LISTS) * sizeof(char *);
+    ASSERT(exemption_lists == NULL);
+    exemption_lists = (char **) 
+        global_heap_alloc(exempt_array_size HEAPACCT(ACCT_OTHER));
+    ASSERT(exemption_lists != NULL);
+    memset(exemption_lists, 0, exempt_array_size);
+    DODEBUG({
+        int i;
+        for (i = 0; i < MODULEDB_EXEMPT_NUM_LISTS; i++)
+            ASSERT(exempt_list_names[i] != NULL);
+    });
+}
+
+void
+moduledb_exit()
+{
+    int i;
+    ASSERT(exemption_lists != NULL);
+    DOLOG(1, LOG_MODULEDB, {
+        LOG(GLOBAL, LOG_MODULEDB, 1, "Moduledb exit:\n");
+        print_moduledb_exempt_lists(GLOBAL);
+    });
+    for (i = 0; i < MODULEDB_EXEMPT_NUM_LISTS; i++) {
+        char *list = GET_EXEMPT_LIST(i);
+        if (list != NULL) {
+            global_heap_free(list, strlen(list) + 1 /* NULL */
+                             HEAPACCT(ACCT_OTHER));
+        }
+    }
+    global_heap_free(exemption_lists, (MODULEDB_EXEMPT_NUM_LISTS) * sizeof(char *)
+                     HEAPACCT(ACCT_OTHER));
+    DELETE_READWRITE_LOCK(moduledb_lock);
+}
+
+bool
+moduledb_exempt_list_empty(moduledb_exempt_list_t list)
+{
+    return GET_EXEMPT_LIST(list) == NULL;
+}
+
+/* checks to see if module name is on the moduledb exempt list pointed
+ * to by list */
+bool
+moduledb_check_exempt_list(moduledb_exempt_list_t list, const char *name)
+{
+    bool found = false;
+    ASSERT(exemption_lists != NULL);
+    read_lock(&moduledb_lock);
+    if (GET_EXEMPT_LIST(list) != NULL) {
+        LOG(GLOBAL, LOG_MODULEDB, 2,
+            "Moduledb checking %s exempt list =\"%s\" for module \"%s\"\n",
+            exempt_list_names[list],
+            GET_EXEMPT_LIST(list) == NULL ? "" : GET_EXEMPT_LIST(list), name);
+        found = check_filter(GET_EXEMPT_LIST(list), name);
+    }
+    read_unlock(&moduledb_lock);
+    return found;
+}
+
+void
+print_moduledb_exempt_lists(file_t file)
+{
+    int i;
+    ASSERT(exemption_lists != NULL);
+    read_lock(&moduledb_lock);
+    for (i = 0; i < MODULEDB_EXEMPT_NUM_LISTS; i++) {
+        ASSERT(exempt_list_names[i] != NULL);
+        print_file(file, "moduledb %s exemption list =\"%s\"\n",
+                   exempt_list_names[i],
+                   GET_EXEMPT_LIST(i) == NULL ? "" : GET_EXEMPT_LIST(i));
+    }
+    read_unlock(&moduledb_lock);
+}
+
+#ifdef PROCESS_CONTROL
+/***************************************************************************/
+/* Process control */
+
+/* Note: if the process control/lockdown feature increases in size, then 
+ *       create a separate file; for now let it be here.
+ */
+
+/* As of today, either the blacklist or the whitelist can be used, not both,
+ * according to the PRD.  In that case there is no need to distinguish between
+ * "not matched" and "no hashlist".  However, if we decide that both need to
+ * co-exist (Windows sw restriction policies do a mix), then this is needed. 
+ * The core is designed to be flexible to handle both.
+ */
+enum {
+    /* NOTE: this isn't the same as "empty" hashlist.  This just means that
+     * the hashlist registry key doesn't exist.
+     */
+    PROCESS_CONTROL_NO_HASHLIST,
+    PROCESS_CONTROL_HASHLIST_EMPTY,
+
+    /* Hash list is too big to fit in our buffer so no list was obtained, which
+     * means we can't use this to make any decision about process control,
+     * i.e., don't do any process control.
+     */
+    PROCESS_CONTROL_LONG_LIST,
+
+    PROCESS_CONTROL_NOT_MATCHED,
+    PROCESS_CONTROL_MATCHED
+};
+
+/* This macro defines "matched" to either being on the hashlist or the hashlist
+ * being empty; empty hashlist is a wildcard match for white and black list
+ * modes, but not for whitelist integrity mode.  Note, this macro is used only
+ * for white and black list modes, not whitelist integrity mode.
+ */
+# define IS_PROCESS_CONTROL_MATCHED(x) \
+     ((x) == PROCESS_CONTROL_MATCHED || (x) == PROCESS_CONTROL_HASHLIST_EMPTY)
+
+/* Records an event static that the hash list in reg_key is too long.  
+ * Note: the reg_key here is (char *) as opposed to (wchar_t *) because our
+ *       syslogging can only handle regular chars!
+ */
+static void
+process_control_report_long_list(char *reg_key)
+{
+    char num_hash_str[6];   /* Won't need more than 5 digits for num hashes! */
+
+    snprintf(num_hash_str, BUFFER_SIZE_ELEMENTS(num_hash_str), "%d",
+             DYNAMO_OPTION(pc_num_hashes));
+
+    /* Be safe, in case some one uses more than 99999 hashes! */
+    NULL_TERMINATE_BUFFER(num_hash_str);
+
+    SYSLOG(SYSLOG_WARNING, PROC_CTL_HASH_LIST_TOO_LONG, 4,
+           get_application_name(), get_application_pid(),
+           reg_key, num_hash_str);
+}
+
+/* This routine reads the hash keys from either the app-specific or the anonymous
+ * reg_key for the app and checks if md5_hash is on that list of keys.
+ * If so it returns true, if not false.
+ */
+static int
+process_control_match(const char *md5_hash, 
+                      const IF_WINDOWS_ELSE_NP(wchar_t, char) *reg_key)
+{
+    int ret_val, res;
+    char *hash_list;
+
+    /* The default size of the hash list is set to 100 hashes.  It is unlikely
+     * that anyone will exceed it even for anonymous hashes.  If they do then
+     * process control will be disabled, i.e., no enforcement will be done.
+     * Case 9252.  BTW, the +1 is for the delimiting ';'.
+     * FIXME: when we read md5 from a file this should go.
+     */
+    uint list_size = DYNAMO_OPTION(pc_num_hashes) * (MD5_STRING_LENGTH + 1);
+
+    /* Read the hash_list from the registry.  The hash_list is nothing but a 
+     * semicolon separated string, just like all core option string.
+     */
+    hash_list = heap_alloc(GLOBAL_DCONTEXT, list_size HEAPACCT(ACCT_OTHER));
+
+    hash_list[0] = '\0';    /* be safe, in case there is no list */
+    /* xref case 9119, we want the value from the unqualified key since our usage of
+     * this only depends on the exe (not its cmdline). */ 
+    res = get_unqualified_parameter(reg_key, hash_list, list_size);
+    hash_list[list_size - 1] = '\0';    /* be safe */
+
+    if (IS_GET_PARAMETER_SUCCESS(res)) {
+        if (hash_list[0] == '\0') {
+            /* empty hash is a wildcard match only for white and black list
+             * modes, not for whitelist integrity mode. */
+            ret_val = PROCESS_CONTROL_HASHLIST_EMPTY;
+        } else if (check_filter(hash_list, md5_hash)) {     /* hash matched */
+            ret_val = PROCESS_CONTROL_MATCHED;
+        } else {
+            ret_val = PROCESS_CONTROL_NOT_MATCHED;
+        }
+
+        /* FIXME: Anonymous hashes should be in a global registry key and app
+         * specific hashes in app specific keys.  Though there is the facility
+         * to do otherwise, we should restrict them because combinations
+         * like app specific anonymous hashes don't make sense.  But how to?
+         */
+    } else if (res == GET_PARAMETER_BUF_TOO_SMALL) {   /* Case 9252. */
+        ret_val = PROCESS_CONTROL_LONG_LIST;
+    } else {
+        ASSERT(res == GET_PARAMETER_FAILURE);
+        /* Couldn't read the key, so assuming there is no hashlist. */
+        ret_val = PROCESS_CONTROL_NO_HASHLIST;
+    }
+
+    heap_free(GLOBAL_DCONTEXT, hash_list, list_size HEAPACCT(ACCT_OTHER));
+    return ret_val;
+}
+
+/* In the regular whitelist mode, the process will be allowed to run if its
+ * executable's hash matches a hash either on the app specific list or the
+ * anonymous list, or if any of those lists are empty.
+ *
+ * In the whitelist integrity mode, the process will be allowed to run if its
+ * executable's hash matches a hash on its app specific hashlist or there is no
+ * app specific hashlist at all.  The idea is to ascertain that an executable
+ * hasn't changed.  If there is no need to track the change, then those exe's
+ * won't have a hashlist.
+ *      Though the regular whitelist mode can be used to do the same, there are
+ * holes in it like the ones below that have to be manually fixed.
+ *  1. support for anonymous hashes; would have to be disabled or not used.
+ *  2. support exe names without hashes; same resolution as point #1. 
+ *  3. apps would be killed if there is no hashlist; would have to add empty
+ *      global hashlists.
+ *
+ * Note: xref case 10969 for whitelist integrity mode.
+ */
+static void
+process_control_whitelist(const char *md5_hash)
+{
+    security_option_t type_handling;
+    action_type_t desired_action;
+
+    char *threat_id = NULL;
+    int anonymous = PROCESS_CONTROL_NOT_MATCHED;
+    int app_specific =
+        process_control_match(md5_hash, L_IF_WIN(DYNAMORIO_VAR_APP_PROCESS_WHITELIST));
+
+    /* Do the pure whitelist mode check in case both modes were specified
+     * accidentally; a matter of precedence.
+     */
+    if (IS_PROCESS_CONTROL_MODE_WHITELIST()) {
+        /* Allow the process if md5_hash matched a hash on either the app
+         * specific or anonymous hash list.
+         */
+        if (IS_PROCESS_CONTROL_MATCHED(app_specific))
+            return;
+        anonymous = process_control_match(md5_hash,
+                        L_IF_WIN(DYNAMORIO_VAR_ANON_PROCESS_WHITELIST));
+        if (IS_PROCESS_CONTROL_MATCHED(anonymous))
+            return;
+
+        threat_id = "WHIT.NOMA"; /* WHIT.NOMA = WHITe list NOt MAtched. */
+
+        /* If there was no match on the anonymous list and if it was too
+         * long, then we can't decide to kill the process because we didn't 
+         * search the full list.  Do no harm and ignore process control.  Case 9252.
+         */
+        if (anonymous == PROCESS_CONTROL_LONG_LIST) {
+            process_control_report_long_list(DYNAMORIO_VAR_ANON_PROCESS_WHITELIST);
+            return;
+        }
+    } else if (IS_PROCESS_CONTROL_MODE_WHITELIST_INTEGRITY()) {
+        /* Allow the process only if md5_hash matched a hash on the app
+         * specific hash list; this is the integrity tracking part.  If there
+         * is no hash list, it means that this process's exe wasn't added to
+         * the integrity mode, so let it run.
+         */
+        if (app_specific == PROCESS_CONTROL_MATCHED ||
+            app_specific == PROCESS_CONTROL_NO_HASHLIST)
+            return;
+
+        /* There is a bug in the controller if app specific hashlist is empty
+         * for the integrity mode.
+         */
+        ASSERT(app_specific != PROCESS_CONTROL_HASHLIST_EMPTY);
+
+        threat_id = "WHIT.INTG";    /* WHIT.NOMA = WHITe list INTeGrity mode. */
+    } else {
+        ASSERT_NOT_REACHED();
+        return;     /* play it safe */
+    }
+
+    /* If there was no match on the app specific list and if it was too
+     * long, then we can't decide to kill the process because we didn't search 
+     * the full list.  Do no harm and ignore process control.  Case 9252.
+     */
+    if (app_specific == PROCESS_CONTROL_LONG_LIST) {
+        process_control_report_long_list(DYNAMORIO_VAR_APP_PROCESS_WHITELIST);
+        return;
+    }
+
+    /* At this point, it should either be not-matched or no-hashlist. 
+     * Note: No hashlist is equivalent to no match; for white list, this means
+     * kill.
+     */
+    ASSERT((app_specific == PROCESS_CONTROL_NOT_MATCHED || 
+            app_specific == PROCESS_CONTROL_NO_HASHLIST));
+    /* Anonymous lists aren't applicable for integrity mode. */
+    if (IS_PROCESS_CONTROL_MODE_WHITELIST()) {
+        ASSERT((anonymous == PROCESS_CONTROL_NOT_MATCHED || 
+                anonymous == PROCESS_CONTROL_NO_HASHLIST));
+    }
+
+#ifdef PROGRAM_SHEPHERDING
+    /* Process control has its own detect_mode; case 10610. */
+    if (DYNAMO_OPTION(pc_detect_mode)) {
+        type_handling = OPTION_REPORT;
+        desired_action = ACTION_CONTINUE;
+    } else {
+        type_handling = OPTION_REPORT|OPTION_BLOCK_IGNORE_DETECT;
+        desired_action = ACTION_TERMINATE_PROCESS;
+    }
+
+    /* All process control violations will be .K.  As the exe name and
+     * pid are already in the event, the threat ID has nothing else to
+     * convey, hence a constant string is used.
+     */
+    security_violation_internal(GLOBAL_DCONTEXT, NULL,
+                                PROCESS_CONTROL_VIOLATION, 
+                                type_handling, threat_id,
+                                desired_action, NULL);
+
+    /* Can reach here only if process control is in detect mode. */
+    ASSERT(DYNAMO_OPTION(pc_detect_mode));
+#endif
+}
+
+static void
+process_control_blacklist(const char *md5_hash)
+{
+    security_option_t type_handling;
+    action_type_t desired_action;
+    int app_specific =
+        process_control_match(md5_hash, L_IF_WIN(DYNAMORIO_VAR_APP_PROCESS_BLACKLIST));
+
+    int anonymous =
+        process_control_match(md5_hash, L_IF_WIN(DYNAMORIO_VAR_ANON_PROCESS_BLACKLIST));
+
+    ASSERT(IS_PROCESS_CONTROL_MODE_BLACKLIST());
+
+    if (IS_PROCESS_CONTROL_MATCHED(app_specific) || 
+        IS_PROCESS_CONTROL_MATCHED(anonymous)) {
+#ifdef PROGRAM_SHEPHERDING
+        /* Process control has its own detect_mode; case 10610. */
+        if (DYNAMO_OPTION(pc_detect_mode)) {
+            type_handling = OPTION_REPORT;
+            desired_action = ACTION_CONTINUE;
+        } else {
+            type_handling = OPTION_REPORT|OPTION_BLOCK_IGNORE_DETECT;
+            desired_action = ACTION_TERMINATE_PROCESS;
+        }
+
+        /* All process control violations will be .K; for blacklist we kill
+         * if the current executable's process is either on the anonymous
+         * or on the app specific whitelist.  As the exe name and
+         * pid are already in the event, the threat ID will only convey what
+         * list was used, anonymous ("ANON.BLAC") or app-specific ("ANON.BLAC").
+         */
+        security_violation_internal(GLOBAL_DCONTEXT, NULL,
+                                    PROCESS_CONTROL_VIOLATION, type_handling,
+                                    app_specific == PROCESS_CONTROL_MATCHED ?
+                                        "BLAC.APPS" : "BLAC.ANON",
+                                    desired_action, NULL);
+
+        /* Can reach here only if process control is in detect mode. */
+        ASSERT(DYNAMO_OPTION(pc_detect_mode));
+#endif
+    } else if (app_specific == PROCESS_CONTROL_LONG_LIST) {     /* Case 9252. */
+        process_control_report_long_list(DYNAMORIO_VAR_APP_PROCESS_BLACKLIST);
+    } else if (anonymous == PROCESS_CONTROL_LONG_LIST){         /* Case 9252. */
+        process_control_report_long_list(DYNAMORIO_VAR_ANON_PROCESS_BLACKLIST);
+    } else {
+        /* At this point, it should either be not-matched or no-hashlist. 
+         * Note: No hashlist is equivalent to no match; for black list, this 
+         * means don't kill.
+         */
+        ASSERT((app_specific == PROCESS_CONTROL_NOT_MATCHED || 
+                app_specific == PROCESS_CONTROL_NO_HASHLIST) &&
+               (anonymous == PROCESS_CONTROL_NOT_MATCHED || 
+                anonymous == PROCESS_CONTROL_NO_HASHLIST));
+    }
+}
+
+/* This routine does all the process control work.  This work is the same for
+ * both the startup scenario and the nudge scenario.  The bool is to identify
+ * whether or not we are in the startup phase because only in that phase we
+ * dump a process-start-event.
+ */
+void
+process_control(void)
+{
+    const char *md5_hash = get_application_md5();
+    bool is_black, is_white, is_white_intg;
+
+    if (strlen(md5_hash) != MD5_STRING_LENGTH) {
+        /* FIXME: what to do if we couldn't get md5 for the current process?
+         * today the default is to ignore and keep going, but is that ok?
+         * should ev be notified?
+         */
+        ASSERT_NOT_REACHED();
+        return;
+    }
+
+    is_black = IS_PROCESS_CONTROL_MODE_BLACKLIST();
+    is_white = IS_PROCESS_CONTROL_MODE_WHITELIST();
+    is_white_intg = IS_PROCESS_CONTROL_MODE_WHITELIST_INTEGRITY();
+
+    /* Currently only one mode can be used; this is how our product is sold.
+     * However, there is nothing preventing us from using all of them (just
+     * make each process control mode below a separate if) with a precedence 
+     * order.  That was how it was originally; however if there is a bug in EV,
+     * we would accidentally run multiple modes at once.  So to safeguard
+     * against that the if-else construct below is used.
+     */
+    ASSERT((is_black && !is_white && !is_white_intg) ||
+           (!is_black && is_white && !is_white_intg) ||
+           (!is_black && !is_white && is_white_intg));
+
+    if (is_black)
+        process_control_blacklist(md5_hash);
+    else if (is_white || is_white_intg)
+        process_control_whitelist(md5_hash);
+    else
+        ASSERT_NOT_REACHED();
+}
+
+void process_control_init(void)
+{
+    process_control();
+}
+/***************************************************************************/
+#endif /* PROCESS_CONTROL */
diff --git a/core/moduledb.h b/core/moduledb.h
new file mode 100644
index 000000000..ca6caa350
--- /dev/null
+++ b/core/moduledb.h
@@ -0,0 +1,112 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2006-2007 Determina Corp. */
+
+/* Module database section flags - 
+ * These control the module section specific relaxations and are used in 2-bit
+ * fields in the module policy flags. */
+enum {
+    SECTION_NO_CHG = 0,
+    SECTION_IF_RX  = 1,
+    SECTION_IF_X   = 2,
+    SECTION_ALLOW  = 3,
+};
+
+/* Module database policy flags -
+ * these specify the module specific actions to take when a module is loaded */
+enum {
+    MODULEDB_ALL_SECTIONS_BITS   = 0x00000003, /* value from SECTION enum above */
+    MODULEDB_ALL_SECTIONS_SHIFT  = 0, /* >> neccesary to read ALL_SECTIONS_BITS */
+    MODULEDB_RCT_EXEMPT_TO       = 0x00000004,
+    MODULEDB_REPORT_ON_LOAD      = 0x00000008,
+    MODULEDB_DLL2HEAP            = 0x00000010,
+    MODULEDB_DLL2STACK           = 0x00000020,
+};
+
+/* Used to specify an exemption list for moduledb_check_exempt_list */
+typedef enum {
+    MODULEDB_EXEMPT_RCT          = 0,
+    MODULEDB_EXEMPT_IMAGE        = 1,
+    MODULEDB_EXEMPT_DLL2HEAP     = 2,
+    MODULEDB_EXEMPT_DLL2STACK    = 3,
+    MODULEDB_EXEMPT_NUM_LISTS    = 4,
+} moduledb_exempt_list_t;
+
+void
+moduledb_init(void);
+
+void
+moduledb_exit(void);
+
+void
+moduledb_process_image(const char *name, app_pc base, bool adding);
+
+void
+moduledb_report_exemption(char *fmt, app_pc addr1, app_pc addr2, const char *name);
+
+/* faster then check_exempt_list below as doesn't require grabbing a lock */
+bool
+moduledb_exempt_list_empty(moduledb_exempt_list_t list);
+
+/* checks to see if module name is on the moduledb exempt list pointed
+ * to by list */
+bool
+moduledb_check_exempt_list(moduledb_exempt_list_t list, const char *name);
+
+void
+print_moduledb_exempt_lists(file_t file);
+
+#ifdef PROCESS_CONTROL
+# define PROCESS_CONTROL_MODE_OFF                   0x0
+# define PROCESS_CONTROL_MODE_WHITELIST             0x1
+# define PROCESS_CONTROL_MODE_BLACKLIST             0x2
+
+/* This mode is identical to whitelist mode, but requires that the user specify
+ * an exe by name and its hashes; no anonymous hashes or exe names with no
+ * hashes.  Case 10969. */
+# define PROCESS_CONTROL_MODE_WHITELIST_INTEGRITY   0x4
+
+# define IS_PROCESS_CONTROL_MODE_WHITELIST() \
+     (TEST(PROCESS_CONTROL_MODE_WHITELIST, DYNAMO_OPTION(process_control)))
+# define IS_PROCESS_CONTROL_MODE_BLACKLIST() \
+     (TEST(PROCESS_CONTROL_MODE_BLACKLIST, DYNAMO_OPTION(process_control)))
+# define IS_PROCESS_CONTROL_MODE_WHITELIST_INTEGRITY() \
+     (TEST(PROCESS_CONTROL_MODE_WHITELIST_INTEGRITY, DYNAMO_OPTION(process_control)))
+# define IS_PROCESS_CONTROL_ON()                \
+     (IS_PROCESS_CONTROL_MODE_WHITELIST() ||    \
+      IS_PROCESS_CONTROL_MODE_BLACKLIST() ||    \
+      IS_PROCESS_CONTROL_MODE_WHITELIST_INTEGRITY())
+
+void process_control(void);
+void process_control_init(void);
+#endif
diff --git a/core/monitor.c b/core/monitor.c
new file mode 100644
index 000000000..2fbbbe29d
--- /dev/null
+++ b/core/monitor.c
@@ -0,0 +1,2499 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/* file "monitor.c"
+ */
+
+#include "globals.h"
+#include "fragment.h"
+#include "link.h"
+#include "utils.h"
+#include "emit.h"
+#include "fcache.h"
+#include "monitor.h"
+#ifdef CUSTOM_TRACES
+#  include "instrument.h"
+#endif
+#include <string.h> /* for memset */
+#include "instr.h"
+#include "perscache.h"
+
+#ifdef CLIENT_INTERFACE
+/* in interp.c.  not declared in arch_exports.h to avoid having to go
+ * make monitor_data_t opaque in globals.h.
+ */
+extern bool mangle_trace(dcontext_t *dcontext, instrlist_t *ilist, monitor_data_t *md);
+#endif
+
+/* SPEC2000 applu has a trace head entry fragment of size 40K! */
+/* streamit's fft had a 944KB bb (ridiculous unrolling) */
+/* no reason to have giant traces, second half will become 2ndary trace */
+#define MAX_TRACE_BUFFER_SIZE  16*1024 /* in bytes */
+/* most traces are under 1024 bytes.
+ * for x64, the rip-rel instrs prevent a memcpy on a resize
+ */
+#ifdef X64
+# define INITIAL_TRACE_BUFFER_SIZE MAX_TRACE_BUFFER_SIZE
+#else
+# define INITIAL_TRACE_BUFFER_SIZE 1024 /* in bytes */
+#endif
+
+#define INITIAL_NUM_BLKS 8
+
+#define INIT_COUNTER_TABLE_SIZE 9
+#define COUNTER_TABLE_LOAD 75
+/* counters must be in unprotected memory
+ * we don't support local unprotected so we use global
+ */
+/* cannot use HEAPACCT here so we use ... */
+#define COUNTER_ALLOC(dc, ...) \
+        (TEST(SELFPROT_LOCAL, dynamo_options.protect_mask) ?   \
+         global_unprotected_heap_alloc(__VA_ARGS__) :          \
+         heap_alloc(dc, __VA_ARGS__))
+#define COUNTER_FREE(dc, p, ...) \
+        (TEST(SELFPROT_LOCAL, dynamo_options.protect_mask) ?   \
+         global_unprotected_heap_free(p, __VA_ARGS__) :        \
+         heap_free(dc, p, __VA_ARGS__))
+
+static void reset_trace_state(dcontext_t *dcontext, bool grab_link_lock);
+
+/* synchronization of shared traces */
+DECLARE_CXTSWPROT_VAR(mutex_t trace_building_lock, INIT_LOCK_FREE(trace_building_lock));
+
+/* For clearing counters on trace deletion we follow a lazy strategy
+ * using a sentinel value to determine whether we've built a trace or not
+ */
+#define TH_COUNTER_CREATED_TRACE_VALUE() (INTERNAL_OPTION(trace_threshold) + 1U)
+
+static void
+delete_private_copy(dcontext_t *dcontext)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    if (md->last_copy != NULL) {
+        LOG(THREAD, LOG_MONITOR, 4,
+            "Deleting previous private copy F%d ("PFX")\n",
+            md->last_copy->id, md->last_copy->tag);
+        /* cannot have monitor_remove_fragment called, since that would abort trace
+         * if last_copy is last_fragment
+         */
+        if (md->last_copy == md->last_fragment) {
+            /* don't have to do internal_restore_last since deleting the thing */
+            md->last_fragment = NULL;
+        }
+        if (md->last_copy == dcontext->last_fragment)
+            last_exit_deleted(dcontext);
+        if (TEST(FRAG_WAS_DELETED, md->last_copy->flags)) {
+            /* case 8083: private copy can't be deleted in trace_abort() since
+             * needed for pc translation (at least until -safe_translate_flushed
+             * is on by default), so if we come here later we must check
+             * for an intervening flush to avoid double-deletion.
+             */
+            LOG(THREAD, LOG_MONITOR, 4,
+                "\tprivate copy was flushed so not re-deleting\n");
+            STATS_INC(num_trace_private_deletions_flushed);
+        } else {
+            fragment_delete(dcontext, md->last_copy, FRAGDEL_NO_MONITOR
+                            /* private fragments are invisible */
+                            IF_CLIENT_INTERFACE(|FRAGDEL_NO_HTABLE));
+        }
+        md->last_copy = NULL;
+        STATS_INC(num_trace_private_deletions);
+    }
+}
+
+/* Returns false if f has been deleted and no copy can be made. FIXME - also returns
+ * false if the emitted private fragment can't be added to a trace (for pad_jmps
+ * insertion reasons xref PR 215179). */
+static bool
+create_private_copy(dcontext_t *dcontext, fragment_t *f)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    /* trying to share the tail of the trace ilist is a bad idea --
+     * violates instrlist_t abstraction, have to deal w/ changes for
+     * bb->trace (like ibl target) and worry about encoding process
+     * changing instr_t state in a way that will affect the trace...
+     *
+     * instead we re-decode the thing, for simplicity
+     */
+
+    KSTART(temp_private_bb);
+    LOG(THREAD, LOG_MONITOR, 4,
+        "Creating private copy of F%d ("PFX") for trace creation\n", f->id, f->tag);
+
+    IF_X64(ASSERT((get_x86_mode(dcontext) && FRAG_IS_32(f->flags)) ||
+                  (!get_x86_mode(dcontext) && !FRAG_IS_32(f->flags))));
+
+    /* only keep one private copy around at a time
+     * we delete here, when we add a new copy, and not in internal_restore_last
+     * since if we do it there we'll clobber last_exit too early
+     */
+    if (md->last_copy != NULL)
+        delete_private_copy(dcontext);
+
+    /* PR 213760/PR 299808: rather than decode_fragment(), which is expensive for
+     * frozen coarse fragments, we re-build from app code (which also simplifies
+     * our client trace model).  If the existing f was flushed/deleted, that won't
+     * stop us from creating a new one for our trace.
+     */
+    /* emitting could clobber last_fragment, which will abort this trace,
+     * unfortunately -- need to be transactional so we finish building this guy,
+     * and then just stop (will delete on next trace build)
+     */
+    md->last_fragment =
+        build_basic_block_fragment(dcontext, f->tag, FRAG_TEMP_PRIVATE,
+                                   true/*link*/,
+                                   /* for clients we make temp-private even when
+                                    * thread-private versions already exist, so
+                                    * we have to make them invisible */
+                                   IF_CLIENT_INTERFACE_ELSE(false, true)
+                                   _IF_CLIENT(true/*for_trace*/)
+                                   _IF_CLIENT(md->pass_to_client ?
+                                              &md->unmangled_bb_ilist : NULL));
+    md->last_copy = md->last_fragment;
+
+    STATS_INC(num_trace_private_copies);
+    LOG(THREAD, LOG_MONITOR, 4,
+        "Created private copy F%d of original F%d ("PFX") for trace creation\n",
+        md->last_fragment->id, f->id, f->tag);
+    DOLOG(2, LOG_INTERP, { disassemble_fragment(dcontext, md->last_fragment,
+                                                stats->loglevel <= 3); });
+    KSTOP(temp_private_bb);
+    /* FIXME - PR 215179, with current hack pad_jmps sometimes marks fragments as
+     * CANNOT_BE_TRACE during emit (since we don't yet have a good way to handle
+     * inserted nops during tracing). This can happen for LINUX syscall fence exits and
+     * CLIENT_INTERFACE added exits.  However, it depends on the starting alignment of
+     * the fragment! So it's possible to have a fragment that is ok turn into a 
+     * fragment that isn't when re-emitted here.  We could keep the ilist around and
+     * use that to extend the trace later (instead of decoding the TEMP bb), but better
+     * to come up with a general fix for tracing through nops.
+     */
+    /* PR 299808: this can happen even more easily now that we re-pass the bb
+     * to the client.  FRAG_MUST_END_TRACE is handled in internal_extend_trace. 
+     */
+    if (TEST(FRAG_CANNOT_BE_TRACE, md->last_fragment->flags)) {
+        delete_private_copy(dcontext);
+        md->last_fragment = NULL;
+        md->last_copy = NULL;
+        STATS_INC(pad_jmps_block_trace_reemit);
+        return false;
+    }
+    return true;
+}
+
+#ifdef CLIENT_INTERFACE
+static void
+extend_unmangled_ilist(dcontext_t *dcontext)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    if (md->pass_to_client) {
+        instr_t *inst;
+        /* FIXME: pass out exit_type from build_basic_block_fragment instead
+         * of walking exit stubs here?
+         * FIXME: remove once we have PR 307284.
+         */
+        linkstub_t *l;
+        ASSERT(md->last_copy != NULL);
+        ASSERT(!TEST(FRAG_COARSE_GRAIN, md->last_copy->flags));
+        for (l = FRAGMENT_EXIT_STUBS(md->last_copy);
+             LINKSTUB_NEXT_EXIT(l) != NULL; l = LINKSTUB_NEXT_EXIT(l))
+            ; /* nothing */
+        md->final_exit_flags = l->flags;
+        LOG(THREAD, LOG_MONITOR, 2, "final exit flags: %x\n", md->final_exit_flags);
+
+        /* PR 299808: we need to keep a list of pre-mangled code */
+        ASSERT(md->unmangled_bb_ilist != NULL);
+        if (instrlist_first(md->unmangled_bb_ilist) != NULL) {
+            instrlist_append(&md->unmangled_ilist,
+                             instrlist_first(md->unmangled_bb_ilist));
+        }
+        DOLOG(4, LOG_INTERP, {
+            LOG(THREAD, LOG_INTERP, 4, "unmangled ilist with F%d("PFX"):\n",
+                md->last_copy->id, md->last_copy->tag);
+            instrlist_disassemble(dcontext, md->trace_tag, &md->unmangled_ilist, THREAD);
+        });
+
+        /* PR 299808: we need the end pc for boundary finding later */
+        ASSERT(md->num_blks < md->blk_info_length);
+        inst = instrlist_last(md->unmangled_bb_ilist);
+        if (inst == NULL) {
+            /* PR 366232: handle empty bbs */
+            md->blk_info[md->num_blks].bounds.end_pc =
+                EXIT_TARGET_TAG(dcontext, md->last_copy, l);
+        } else {
+            md->blk_info[md->num_blks].bounds.end_pc =
+                instr_get_translation(inst) + instr_length(dcontext, inst);
+        }
+        
+        instrlist_init(md->unmangled_bb_ilist); /* clear fields to make destroy happy */
+        instrlist_destroy(dcontext, md->unmangled_bb_ilist);
+        md->unmangled_bb_ilist = NULL;
+    }
+    /* If any constituent block wants to store (or the final trace hook wants to),
+     * then store for the trace.
+     */
+    if (md->last_copy != NULL && TEST(FRAG_HAS_TRANSLATION_INFO, md->last_copy->flags))
+        md->trace_flags |= FRAG_HAS_TRANSLATION_INFO;
+}
+#endif
+
+/* Initialization */
+/* thread-shared init does nothing, thread-private init does it all */
+void
+monitor_init()
+{
+    /* to reduce memory, we use ushorts for some offsets in fragment bodies,
+     * so we have to stop a trace at that size
+     * this does not include exit stubs
+     */
+    ASSERT(MAX_TRACE_BUFFER_SIZE <= MAX_FRAGMENT_SIZE);
+}
+
+/* re-initializes non-persistent memory */
+void
+monitor_thread_reset_init(dcontext_t *dcontext)
+{
+}
+
+/* frees all non-persistent memory */
+void
+monitor_thread_reset_free(dcontext_t *dcontext)
+{
+    /* remove any MultiEntries */
+    trace_abort(dcontext);
+    /* case 8083: we have to explicitly remove last copy since it can't be
+     * removed in trace_abort (at least until -safe_translate_flushed is on)
+     */
+    delete_private_copy(dcontext);    
+}
+
+void
+monitor_exit()
+{
+    LOG(GLOBAL, LOG_MONITOR|LOG_STATS, 1,
+        "Trace fragments generated: %d\n", GLOBAL_STAT(num_traces));
+    DELETE_LOCK(trace_building_lock);
+}
+
+void
+monitor_thread_init(dcontext_t *dcontext)
+{
+    monitor_data_t *md;
+
+    md = (monitor_data_t *)
+        heap_alloc(dcontext, sizeof(monitor_data_t) HEAPACCT(ACCT_TRACE));
+    dcontext->monitor_field = (void *) md;
+    memset(md, 0, sizeof(monitor_data_t));
+    reset_trace_state(dcontext, false /* link lock not needed */);
+
+    /* case 7966: don't initialize un-needed things for hotp_only & thin_client
+     * FIXME: could set initial sizes to 0 for all configurations, instead
+     * FIXME: we can optimize even more to not allocate md at all, but would need
+     * to have hotp_only checks in monitor_cache_exit(), etc.
+     */
+    if (RUNNING_WITHOUT_CODE_CACHE())
+        return;
+
+    /* FIXME : we should gather statistics on the hash table */
+    /* trace head counters are thread-private and must be kept in a
+     * separate table and not in the fragment_t structure.
+     */
+    md->thead_table.hash_bits = INIT_COUNTER_TABLE_SIZE;
+    md->thead_table.hash_mask = HASH_MASK(md->thead_table.hash_bits);
+    md->thead_table.hash_mask_offset = 0;
+    md->thead_table.hash_func = (hash_function_t)INTERNAL_OPTION(alt_hash_func);
+    md->thead_table.capacity = HASHTABLE_SIZE(md->thead_table.hash_bits);
+    md->thead_table.entries = 0;
+    md->thead_table.load_factor_percent = COUNTER_TABLE_LOAD;
+    md->thead_table.resize_threshold =
+        md->thead_table.capacity * md->thead_table.load_factor_percent / 100;
+    md->thead_table.counter_table = (trace_head_counter_t **)
+        COUNTER_ALLOC(dcontext, md->thead_table.capacity*sizeof(trace_head_counter_t*)
+                      HEAPACCT(ACCT_THCOUNTER));
+    memset(md->thead_table.counter_table, 0, md->thead_table.capacity*
+           sizeof(trace_head_counter_t*));
+}
+
+/* atexit cleanup */
+void
+monitor_thread_exit(dcontext_t *dcontext)
+{
+    DEBUG_DECLARE(uint i;)
+    DEBUG_DECLARE(monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;)
+
+    /* For non-debug we do fast exit path and don't free local heap.
+     * We call trace_abort so that in case the thread is terminated in
+     * the middle of trace building from a shared trace head, it has a
+     * chance to clear the FRAG_TRACE_BUILDING flag. Otherwise, a trace
+     * can never be built from that particular trace head.
+     */
+    trace_abort(dcontext);
+#ifdef DEBUG
+    if (md->trace_buf != NULL)
+        heap_free(dcontext, md->trace_buf, md->trace_buf_size HEAPACCT(ACCT_TRACE));
+    if (md->blk_info != NULL) {
+        heap_free(dcontext, md->blk_info, md->blk_info_length*sizeof(trace_bb_build_t)
+                  HEAPACCT(ACCT_TRACE));
+    }
+    if (md->last_copy != NULL)
+        delete_private_copy(dcontext);
+
+    /* case 7966: don't initialize at all for hotp_only
+     * FIXME: could set initial sizes to 0 for all configurations, instead
+     */
+    if (!RUNNING_WITHOUT_CODE_CACHE()) {
+        for (i = 0; i < md->thead_table.capacity; i++) {
+            trace_head_counter_t *e = md->thead_table.counter_table[i];
+            while (e) {
+                trace_head_counter_t *nexte = e->next;
+                COUNTER_FREE(dcontext, e, sizeof(trace_head_counter_t)
+                             HEAPACCT(ACCT_THCOUNTER));
+                e = nexte;
+            }
+            md->thead_table.counter_table[i] = NULL;
+        }
+        COUNTER_FREE(dcontext, md->thead_table.counter_table,
+                     md->thead_table.capacity*sizeof(trace_head_counter_t*)
+                     HEAPACCT(ACCT_THCOUNTER));
+    }
+    heap_free(dcontext, md, sizeof(monitor_data_t) HEAPACCT(ACCT_TRACE));
+#endif
+}
+
+static trace_head_counter_t *
+thcounter_lookup(dcontext_t *dcontext, app_pc tag)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    trace_head_counter_t *e;
+    uint hindex = HASH_FUNC((ptr_uint_t)tag, &md->thead_table);
+    for (e = md->thead_table.counter_table[hindex]; e; e = e->next) {
+        if (e->tag == tag)
+            return e;
+    }
+    return NULL;
+}
+
+static trace_head_counter_t *
+thcounter_add(dcontext_t *dcontext, app_pc tag)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    uint hindex;
+    /* counters may be persistent while fragment comes and goes from cache */
+    trace_head_counter_t *e = thcounter_lookup(dcontext, tag);
+    if (e)
+        return e;
+    e = (trace_head_counter_t *)
+        COUNTER_ALLOC(dcontext, sizeof(trace_head_counter_t) HEAPACCT(ACCT_THCOUNTER));
+    e->tag = tag;
+    e->counter = 0;
+    hindex = HASH_FUNC((ptr_uint_t)e->tag, &md->thead_table);
+    e->next = md->thead_table.counter_table[hindex];
+    md->thead_table.counter_table[hindex] = e;
+    return e;
+}
+
+#if 0 /* not used */
+/* delete the trace head entry corresponding to tag if it exists */
+static void
+thcounter_remove(dcontext_t *dcontext, app_pc tag)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    trace_head_counter_t *e, *prev_e = NULL;
+    uint hindex = HASH_FUNC((ptr_uint_t)tag, &md->thead_table);
+    for (e = md->thead_table.counter_table[hindex]; e; prev_e = e, e = e->next) {
+        if (e->tag == tag) {
+            if (prev_e)
+                prev_e->next = e->next;
+            else
+                md->thead_table.counter_table[hindex] = e->next;
+            COUNTER_FREE(dcontext, e, sizeof(trace_head_counter_t) HEAPACCT(ACCT_THCOUNTER));
+            break;
+        }
+    }
+}
+#endif
+
+/* Deletes all trace head entries in [start,end) */
+void
+thcounter_range_remove(dcontext_t *dcontext, app_pc start, app_pc end)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    trace_head_counter_t *e, *prev_e = NULL, *next_e;
+    uint i;
+    /* ensure no synch needed */
+    ASSERT(dcontext == get_thread_private_dcontext() ||
+           is_self_flushing() || is_self_allsynch_flushing());
+    for (i = 0; i < md->thead_table.capacity; i++) {
+        for (e = md->thead_table.counter_table[i], prev_e = NULL;
+             e != NULL; e = next_e) {
+            next_e = e->next;
+            if (e->tag >= start && e->tag < end) {
+                if (prev_e != NULL)
+                    prev_e->next = next_e;
+                else
+                    md->thead_table.counter_table[i] = next_e;
+                COUNTER_FREE(dcontext, e, sizeof(trace_head_counter_t)
+                             HEAPACCT(ACCT_THCOUNTER));
+            } else
+                prev_e = e;
+        }
+    }
+}
+
+bool
+is_building_trace(dcontext_t *dcontext)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    return (md->trace_tag != NULL);
+}
+
+app_pc
+cur_trace_tag(dcontext_t *dcontext)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    return md->trace_tag;
+}
+
+void *
+cur_trace_vmlist(dcontext_t *dcontext)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    return md->trace_vmlist;
+}
+
+static void
+reset_trace_state(dcontext_t *dcontext, bool grab_link_lock)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    /* reset the trace buffer */
+    instrlist_init(&(md->trace));
+#ifdef CLIENT_INTERFACE
+    if (instrlist_first(&md->unmangled_ilist) != NULL)
+        instrlist_clear(dcontext, &md->unmangled_ilist);
+    instrlist_init(&md->unmangled_ilist);
+    if (md->unmangled_bb_ilist != NULL)
+        instrlist_clear_and_destroy(dcontext, md->unmangled_bb_ilist);
+    md->unmangled_bb_ilist = NULL;
+#endif
+    md->trace_buf_top = 0;
+    ASSERT(md->trace_vmlist == NULL);
+    md->num_blks = 0;
+
+    /* If shared BBs are being used to build a shared trace, we may have
+     * FRAG_TRACE_BUILDING set on a shared BB w/the same tag (if there is a
+     * BB present -- it could've been deleted for cache management or cache
+     * consistency). Unset the flag so that a trace can be built from it
+     * in the future.
+     */
+    if (TEST(FRAG_SHARED, md->trace_flags) && DYNAMO_OPTION(shared_bbs)) {
+        /* Look in the shared BB table only since we're only interested 
+         * if a shared BB is present. */
+        fragment_t *bb = fragment_lookup_shared_bb(dcontext, md->trace_tag);
+        /* FRAG_TRACE_BUILDING may not be set if the BB was regenerated, so
+         * we can't expect it to be set simply because the BB is shared. Check
+         * just for the trace bulding flag.
+         */
+        if (grab_link_lock)
+            acquire_recursive_lock(&change_linking_lock);
+        if (bb != NULL && TEST(FRAG_TRACE_BUILDING, bb->flags)) {
+            /* The regenerate scenario is still racy w/respect to clearing the
+             * flag. The regenerated fragment could have another thread building
+             * a trace from it so the clear would be for the the wrong thread
+             * here. It doesn't cause a correctness problem because the
+             * emit-time race detection logic will catch it. (In testing w/IIS,
+             * we've seen very, very few emit-time aborts -- < 1% of all races.)
+             */
+            ASSERT(TESTALL(FRAG_SHARED | FRAG_IS_TRACE_HEAD, bb->flags));
+            STATS_INC(num_trace_building_ip_cleared);
+            bb->flags &= ~FRAG_TRACE_BUILDING;
+        }
+#ifdef DEBUG
+        /* As noted above, the trace head BB may no longer be present. This
+         * should be rare in most apps but we'll track it w/a counter in case
+         * we see lots of emit-time aborts.
+         */
+        else {
+            STATS_INC(num_reset_trace_no_trace_head);
+            /* The shared BB may been evicted during trace building and subsequently
+             * re-genned and so wouldn't be marked as FRAG_TRACE_BUILDING. It might
+             * be marked as a trace head, though, so we don't assert anything about
+             * that trait.
+             * FIXME We could add a strong ASSERT about the regen case if we added
+             * a trace_head_id field to monitor_data_t. The field would store the id
+             * of the shared BB trace head that caused trace building to begin. If
+             * a shared trace head isn't found but a shared BB is, the shared BB
+             * id should be greater than trace_head_id.
+             */
+        }
+#endif
+        if (grab_link_lock)
+            release_recursive_lock(&change_linking_lock);
+    }
+    md->trace_tag = NULL;  /* indicate return to search mode */
+    md->trace_flags = 0;
+    md->emitted_size = 0;
+    /* flags may not match, e.g., if frag was marked as trace head */
+    ASSERT(md->last_fragment == NULL ||
+           (md->last_fragment_flags & (FRAG_CANNOT_DELETE|FRAG_LINKED_OUTGOING)) ==
+           (md->last_fragment->flags & (FRAG_CANNOT_DELETE|FRAG_LINKED_OUTGOING)));
+    md->last_fragment_flags = 0;
+    /* we don't delete last_copy here to avoid issues w/ trace_abort deleting
+     * a fragment we're examining (seg fault, etc.)
+     */
+    md->last_fragment = NULL;
+    /* note that we don't delete last_copy here as it's needed for pc translation
+     * (at least until -safe_translate_flushed is on) (xref case 8083)
+     */
+#ifdef CUSTOM_TRACES_RET_REMOVAL
+    dcontext->call_depth = 0;
+#endif
+#ifdef NATIVE_RETURN
+    dcontext->last_retaddr = NULL;
+    dcontext->num_calls_in_trace = 0;
+#endif
+}
+
+/* called when a fragment is deleted */
+void 
+monitor_remove_fragment(dcontext_t *dcontext, fragment_t *f)
+{
+    monitor_data_t *md;
+    /* may be a global fragment -- but we still want our local trace data */
+    if (dcontext == GLOBAL_DCONTEXT) {
+        ASSERT(TEST(FRAG_SHARED, f->flags));
+        dcontext = get_thread_private_dcontext();
+        /* may still be null if exiting process -- in which case a nop for us */
+        if (dcontext == NULL) {
+            if (dynamo_exited)
+                return;
+            ASSERT_NOT_REACHED();
+            return;             /* safe default */
+        }
+    }    
+    md = (monitor_data_t *) dcontext->monitor_field;
+    if (md->last_copy == f) {
+        md->last_copy = NULL; /* no other action required */
+        STATS_INC(num_trace_private_deletions);
+    }
+    /* Must check to see if the last fragment, which was added to the
+     * trace, is being deleted before we're done with it.
+     * This can happen due to a flush from self-modifying code,
+     * or an munmap.
+     * Must check both last_fragment and last_exit.
+     * May come here before last_exit is set, or may come here after
+     * last_fragment is restored but before last_exit is used.
+     * FIXME: if we do manage to remove the check for last_fragment
+     * here, remove the last_exit clear in end_and_emit_trace
+     */
+    /* FIXME: case 5593 we may also unnecessarily abort a trace that
+     * starts at the next_tag and last_fragment is really not
+     * related.
+     */
+    if (md->last_fragment == f || dcontext->last_fragment == f) {
+        if (md->trace_tag > 0) {
+            LOG(THREAD, LOG_MONITOR, 2,
+                "Aborting current trace since F%d was deleted\n",
+                f->id);
+            /* abort current trace, we've lost a link */
+            trace_abort(dcontext);
+        }
+        /* trace_abort clears last_fragment -- and if not in trace-building
+         * mode, it should not be set!
+         */
+        ASSERT(md->last_fragment == NULL);
+        if (dcontext->last_fragment == f)
+            last_exit_deleted(dcontext);
+    }
+}
+
+/* Unlink the trace head fragment from any IBT tables in which it is in */
+static inline void
+unlink_ibt_trace_head(dcontext_t *dcontext, fragment_t *f)
+{
+    ASSERT(TEST(FRAG_IS_TRACE_HEAD, f->flags));
+    if (DYNAMO_OPTION(shared_bb_ibt_tables)) {
+        ASSERT(TEST(FRAG_SHARED, f->flags));
+        if (fragment_prepare_for_removal(GLOBAL_DCONTEXT, f)) {
+            LOG(THREAD, LOG_FRAGMENT, 3,
+                "  F%d("PFX") removed as trace head IBT\n", f->id, f->tag);
+            STATS_INC(num_th_bb_ibt_unlinked);
+        }
+    }
+    else {
+
+        /* To preserve the current paradigm of trace head-ness as a shared
+         * property, we must unlink the fragment from every thread's IBT tables.
+         * This is a heavyweight operation compared to the use of a shared table
+         * and requires additional changes -- for example, get_list_of_threads()
+         * can't currently be called from here. If we change trace head-ness
+         * to a private property, this becomes very easy and more performant
+         * than the use of a shared table. (Case 3530 discusses private vs shared
+         * trace head-ness.)
+         */
+        thread_record_t **threads;
+        int num_threads;
+        int i;
+ 
+        ASSERT_NOT_IMPLEMENTED(false);
+        /* fragment_prepare_for_removal will unlink from shared ibt; we cannot
+         * remove completely here */
+        fragment_remove_from_ibt_tables(dcontext, f, false/*leave in shared ibt*/);
+        /* Remove the fragment from other thread's tables. */
+        mutex_lock(&thread_initexit_lock);
+        get_list_of_threads(&threads, &num_threads);
+        mutex_unlock(&thread_initexit_lock);
+        for (i = 0; i < num_threads; i++) {
+            dcontext_t *tgt_dcontext = threads[i]->dcontext;
+            LOG(THREAD, LOG_FRAGMENT, 2,
+                "  considering thread %d/%d = %d\n", i+1, num_threads,
+                threads[i]->id);
+            ASSERT(is_thread_known(tgt_dcontext->owning_thread));
+            fragment_prepare_for_removal(TEST(FRAG_SHARED, f->flags) ?
+                                         GLOBAL_DCONTEXT : tgt_dcontext, f);
+        }
+        global_heap_free(threads, num_threads*sizeof(thread_record_t*)
+                         HEAPACCT(ACCT_THREAD_MGT));
+    }
+}
+
+/* if f is shared, caller MUST hold the change_linking_lock */
+void
+mark_trace_head(dcontext_t *dcontext_in, fragment_t *f, fragment_t *src_f,
+                linkstub_t *src_l)
+{
+    bool protected = false;
+    cache_pc coarse_stub = NULL, coarse_body = NULL;
+    coarse_info_t *info = NULL;
+    /* Case 9021: handle GLOBAL_DCONTEXT coming in via flush_fragments_synchall
+     * removing a fine trace that triggers a shift to its shadowed coarse trace
+     * head and a link_fragment_incoming on that head.
+     * Using the flushing thread's dcontext for the trace head counter is fine
+     * and shouldn't limit its becoming a new trace again.
+     */
+    dcontext_t *dcontext =
+        (dcontext_in == GLOBAL_DCONTEXT) ? get_thread_private_dcontext() : dcontext_in;
+    ASSERT(dcontext != NULL);
+
+    LOG(THREAD, LOG_MONITOR, 4, "marking F%d ("PFX") as trace head\n", f->id, f->tag);
+    ASSERT(!TEST(FRAG_IS_TRACE, f->flags));
+    ASSERT(!NEED_SHARED_LOCK(f->flags) ||
+           self_owns_recursive_lock(&change_linking_lock));
+
+    if (thcounter_lookup(dcontext, f->tag) == NULL) {
+        protected = local_heap_protected(dcontext);
+        if (protected) {
+            /* unprotect local heap */
+            protect_local_heap(dcontext, WRITABLE);
+        }
+        /* FIXME: private counter tables are used even for !shared_bbs since the
+         * counter field is not in fragment_t...
+         * Move counters to Future for all uses, giving us persistent counters too!
+         */
+        thcounter_add(dcontext, f->tag);
+    } else {
+        /* This does happen for resurrected fragments and coarse-grain fragments */
+        STATS_INC(trace_head_remark);
+    }
+    LOG(THREAD, LOG_MONITOR, 4, "mark_trace_head: flags 0x%08x\n", f->flags);
+    f->flags |= FRAG_IS_TRACE_HEAD;
+    LOG(THREAD, LOG_MONITOR, 4, "\tnow, flags 0x%08x\n", f->flags);
+    /* must unlink incoming links so that the counter will increment */
+    LOG(THREAD, LOG_MONITOR, 4, "unlinking incoming for new trace head F%d ("PFX")\n",
+        f->id, f->tag);
+    
+    if (TEST(FRAG_COARSE_GRAIN, f->flags)) {
+        /* For coarse trace heads, trace headness depends on the path taken
+         * (more specifically, on the entrance stub taken).  If we don't have
+         * any info on src_f we use f's unit.
+         */
+        info = get_fragment_coarse_info(src_f == NULL ? f : src_f);
+        if (info == NULL) {
+            /* case 8632: A fine source may not be in a coarse region,
+             * so there is nothing to unlink.
+             */
+        } else {
+            /* See if there is an entrance stub for this target in the source unit */
+            fragment_coarse_lookup_in_unit(dcontext, info, f->tag,
+                                           &coarse_stub, &coarse_body);
+            /* FIXME: don't allow marking for frozen units w/ no src info:
+             * shouldn't happen, except perhaps CLIENT_INTERFACE
+             */
+            ASSERT(src_f != NULL || !info->frozen);
+            if (src_f != NULL && TEST(FRAG_COARSE_GRAIN, src_f->flags) &&
+                src_l != NULL && LINKSTUB_NORMAL_DIRECT(src_l->flags)) {
+                direct_linkstub_t *dl = (direct_linkstub_t *) src_l;
+                if (dl->stub_pc != NULL && coarse_is_entrance_stub(dl->stub_pc)) {
+                    if (coarse_stub == NULL) {
+                        /* Case 9708: For a new fragment whose target exists but
+                         * is another unit and does not yet have an entrance
+                         * stub in the new fragment's unit, we will come here
+                         * w/o that entrance stub being in the htable.  We rely
+                         * on dl->stub_pc being set to that entrance stub.
+                         */
+                        coarse_stub = dl->stub_pc;
+                    } else
+                        ASSERT(dl->stub_pc == NULL || dl->stub_pc == coarse_stub);
+                }
+            }
+            if (coarse_stub != NULL) {
+                ASSERT(coarse_is_entrance_stub(coarse_stub));
+                /* FIXME: our coarse lookups do not always mark trace headness
+                 * (in particular, fragment_coarse_link_wrapper() calling
+                 * fragment_coarse_lookup_wrapper() does not), and we
+                 * un-mark as trace heads when linking incoming (case 8907),
+                 * so we may get here for an existing trace head.
+                 */
+                if (!coarse_is_trace_head_in_own_unit(dcontext, f->tag, coarse_stub,
+                                                      coarse_body, true,
+                                                      (src_f == NULL) ? info : NULL)) {
+                    ASSERT(coarse_body == NULL /* new fragment, or in other unit */ ||
+                           entrance_stub_jmp_target(coarse_stub) == coarse_body);
+                    if (coarse_body == NULL &&
+                        /* if stub is from tag's own unit */
+                        (src_f == NULL || get_fragment_coarse_info(f) == info)) {
+                        /* if marking new fragment, not in htable yet */
+                        coarse_body = FCACHE_ENTRY_PC(f);
+                    }
+                    coarse_mark_trace_head(dcontext, f, info, coarse_stub, coarse_body);
+                }
+            } else {
+                LOG(THREAD, LOG_MONITOR, 4, "\tno local stub, deferring th unlink\n");
+                /* Could be that this is a new fragment, in which case its entrance
+                 * stub will be unlinked and its body pc added to the th table in
+                 * link_new_coarse_grain_fragment(); or the source is a fine
+                 * fragment corresponding to another unit and thus no entrance stub
+                 * or htable changes are necessary.
+                 */
+                STATS_INC(coarse_th_from_fine);
+                /* id comparison could have a race w/ private frag gen so a curiosity */
+                ASSERT_CURIOSITY(GLOBAL_STAT(num_fragments) == f->id ||
+                                 (src_f != NULL &&
+                                  !TEST(FRAG_COARSE_GRAIN, src_f->flags)));
+            }
+        }
+    } else
+        unlink_fragment_incoming(dcontext, f);
+
+    if (DYNAMO_OPTION(bb_ibl_targets))
+        unlink_ibt_trace_head(dcontext, f);
+#ifdef TRACE_HEAD_CACHE_INCR
+    /* we deliberately link to THCI in two steps (unlink and then
+     * re-link), since combined they aren't atomic, separate atomic
+     * steps w/ ok intermediate (go back to DR) is fine
+     */
+    /* must re-link incoming links to point to trace_head_incr routine 
+     * FIXME: we get called in the middle of linking new fragments, so
+     * we end up linking some incoming links twice (no harm done except
+     * a waste of time) -- how fix it?
+     * When fix it, change link_branch to assert that !already linked
+     */
+    link_fragment_incoming(dcontext, f, false/*not new*/);
+#endif
+    STATS_INC(num_trace_heads_marked);
+    /* caller is either dispatch or inside emit_fragment, they take care of
+     * re-protecting fcache
+     */
+    if (protected) {
+        /* re-protect local heap */
+        protect_local_heap(dcontext, READONLY);
+    }
+}
+
+/* can ONLY be called by should_be_trace_head_internal, separated out
+ * to avoid recursion when re-verifying with change_linking_lock held
+ */
+static bool
+should_be_trace_head_internal_unsafe(dcontext_t *dcontext, fragment_t *from_f,
+                                     linkstub_t *from_l, app_pc to_tag, uint to_flags,
+                                     bool trace_sysenter_exit)
+{
+    app_pc from_tag;
+    uint from_flags;
+
+    if (DYNAMO_OPTION(disable_traces) ||
+        TEST(FRAG_IS_TRACE, to_flags) ||
+        TEST(FRAG_IS_TRACE_HEAD, to_flags) ||
+        TEST(FRAG_CANNOT_BE_TRACE, to_flags))
+        return false;
+
+    /* We know that the to_flags pass the test. */
+    if (trace_sysenter_exit)
+        return true;
+
+    from_tag = from_f->tag;
+    from_flags = from_f->flags;
+
+    /* A trace head is either
+     *   1) a link from a trace, or 
+     *   2) a backward direct branch
+     * Watch out -- since we stop building traces at trace heads,
+     * too many can hurt performance, especially if bbs do not follow
+     * direct ctis.  We can use shadowed bbs to go through trace
+     * head and trace boundaries (CUSTOM_TRACES).
+     */
+    /* trace heads can be created across private/shared cache bounds */
+    if (TEST(FRAG_IS_TRACE, from_flags) ||
+        (to_tag <= from_tag && LINKSTUB_DIRECT(from_l->flags)))
+        return true;
+
+    DOSTATS({
+        if (!DYNAMO_OPTION(disable_traces) &&
+            !TEST(FRAG_IS_TRACE, to_flags) &&
+            !TEST(FRAG_IS_TRACE_HEAD, to_flags) &&
+            !TEST(FRAG_CANNOT_BE_TRACE, to_flags)) {
+            STATS_INC(num_wannabe_traces);
+        }
+    });
+    return false;
+}
+
+
+/* Returns TRACE_HEAD_* flags indicating whether to_tag should be a
+ * trace head based on fragment traits and/or control flow between the
+ * link stub and the to_tag/to_flags.
+ *
+ * For -shared_bbs, will return TRACE_HEAD_OBTAINED_LOCK if the
+ * change_linking_lock is not already held (meaning from_l->fragment is
+ * private) and the to_tag is FRAG_SHARED and TRACE_HEAD_YES is being returned,
+ * since the change_linking_lock must be held and the TRACE_HEAD_YES result 
+ * re-verified.  In that case the caller must free the change_linking_lock.
+ * If trace_sysenter_exit = true, control flow rules are not checked, i.e., the
+ * from_l and to_tag params are not checked. This is provided to capture
+ * the case where the most recent cache exit was prior to a non-ignorable
+ * syscall via a SYSENTER instruction. See comments in monitor_cache_exit for
+ * details. This is the exception, not the norm.
+ *
+ * If the link stub is non-NULL, trace_sysenter_exit does NOT need to
+ * be set.
+ *
+ * FIXME This is a stopgap soln. The long-term fix is to not count on
+ * a link stub being passed in but rather pass in the most recent fragment's
+ * flags & tag explicitly. The flags & tag can be stored in a dcontext-private
+ * monitor structure, one that is not shared across call backs.
+ */
+static uint
+should_be_trace_head_internal(dcontext_t *dcontext, fragment_t *from_f, linkstub_t *from_l,
+                              app_pc to_tag, uint to_flags, bool have_link_lock,
+                              bool trace_sysenter_exit)
+{
+    uint result = 0;
+    if (should_be_trace_head_internal_unsafe(dcontext, from_f, from_l, to_tag, to_flags,
+                                             trace_sysenter_exit)) {
+        result |= TRACE_HEAD_YES;
+        ASSERT(!have_link_lock || self_owns_recursive_lock(&change_linking_lock));
+        if (!have_link_lock) {
+            /* If the target is shared, we must obtain the change_linking_lock and
+             * re-verify that it hasn't already been marked.
+             * If source is also shared then lock should already be held .
+             */
+            ASSERT(from_l == NULL || !NEED_SHARED_LOCK(from_f->flags));
+            if (NEED_SHARED_LOCK(to_flags)) {
+                acquire_recursive_lock(&change_linking_lock);
+                if (should_be_trace_head_internal_unsafe(dcontext, from_f, from_l,
+                                                         to_tag, to_flags,
+                                                         trace_sysenter_exit)) {
+                    result |= TRACE_HEAD_OBTAINED_LOCK;
+                } else {
+                    result &= ~TRACE_HEAD_YES;
+                    release_recursive_lock(&change_linking_lock);
+                }
+            }
+        }
+    }
+    return result;
+}
+
+/* Returns TRACE_HEAD_* flags indicating whether to_tag should be a
+ * trace head based on fragment traits and/or control flow between the
+ * link stub and the to_tag/to_flags.
+ *
+ * For -shared_bbs, will return TRACE_HEAD_OBTAINED_LOCK if the
+ * change_linking_lock is not already held (meaning from_l->fragment is
+ * private) and the to_tag is FRAG_SHARED and TRACE_HEAD_YES is being returned,
+ * since the change_linking_lock must be held and the TRACE_HEAD_YES result 
+ * re-verified.  In that case the caller must free the change_linking_lock.
+ */
+uint
+should_be_trace_head(dcontext_t *dcontext, fragment_t *from_f, linkstub_t *from_l,
+                     app_pc to_tag, uint to_flags, bool have_link_lock)
+{
+    return should_be_trace_head_internal(dcontext, from_f, from_l, to_tag, to_flags,
+                                         have_link_lock, false);
+}
+
+/* If upgrades to_f to a trace head, returns true, else returns false.
+ */
+static bool
+check_for_trace_head(dcontext_t *dcontext, fragment_t *from_f, linkstub_t *from_l,
+                     fragment_t *to_f, bool have_link_lock, bool trace_sysenter_exit)
+{
+    if (!DYNAMO_OPTION(disable_traces)) {
+        uint th = should_be_trace_head_internal(dcontext, from_f, from_l, to_f->tag,
+                                                to_f->flags, have_link_lock,
+                                                trace_sysenter_exit);
+        if (TEST(TRACE_HEAD_YES, th)) {
+            mark_trace_head(dcontext, to_f, from_f, from_l);
+            if (TEST(TRACE_HEAD_OBTAINED_LOCK, th))
+                release_recursive_lock(&change_linking_lock);
+            return true;
+        }
+    }
+    return false;
+}
+
+/* Linkability rules involving traces and trace heads.
+ * This routines also marks new traces heads if mark_new_trace_head is true.
+ * The current implementation of this routine assumes that we don't
+ * want to link potential trace heads.  A potential trace head is any
+ * block fragment that is reached by a backward (direct) branch.
+ */
+bool
+monitor_is_linkable(dcontext_t *dcontext, fragment_t *from_f, linkstub_t *from_l,
+                    fragment_t *to_f, bool have_link_lock, bool mark_new_trace_head)
+{
+    /* common case: both traces */
+    if (TEST(FRAG_IS_TRACE, from_f->flags) && TEST(FRAG_IS_TRACE, to_f->flags))
+        return true;
+    if (DYNAMO_OPTION(disable_traces))
+        return true;
+#ifndef TRACE_HEAD_CACHE_INCR
+    /* no link case -- block is a trace head */
+    if (TEST(FRAG_IS_TRACE_HEAD, to_f->flags) && !DYNAMO_OPTION(disable_traces))
+        return false;
+#endif
+    if (mark_new_trace_head) {
+        uint th = should_be_trace_head(dcontext, from_f, from_l, to_f->tag, to_f->flags,
+                                       have_link_lock);
+        if (TEST(TRACE_HEAD_YES, th)) {
+            mark_trace_head(dcontext, to_f, from_f, from_l);
+            if (TEST(TRACE_HEAD_OBTAINED_LOCK, th))
+                release_recursive_lock(&change_linking_lock);
+#ifdef TRACE_HEAD_CACHE_INCR
+            /* fine to link to trace head
+             * link will end up pointing not to fcache_return but to trace_head_incr
+             */
+            return true;
+#else
+            return false;
+#endif
+        }
+    }
+    return true;        /* otherwise */
+}
+
+/* If necessary, re-allocates the trace buffer to a larger size to
+ * hold add_size more bytes.
+ * If the resulting size will exceed the maximum trace
+ * buffer size, returns false, else returns true.
+ * FIXME: now that we have a real max limit on emitted trace size,
+ * should we have an unbounded trace buffer size?
+ * Also increases the size of the block array if necessary.
+ */
+static bool
+make_room_in_trace_buffer(dcontext_t *dcontext, uint add_size, fragment_t *f)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    uint size;
+    uint new_blks;
+    ssize_t realloc_shift;
+    instr_t *instr;
+    instrlist_t *trace = &md->trace;
+
+    size = md->trace_buf_size;
+    if (add_size > (size - md->trace_buf_top)) {
+        byte *new_tbuf;
+
+        if (size == 0)
+            size = INITIAL_TRACE_BUFFER_SIZE;
+        while (add_size > (size - md->trace_buf_top))
+            size *= 2;
+        if (size > MAX_TRACE_BUFFER_SIZE) {
+            LOG(THREAD, LOG_MONITOR, 2,
+                "Not letting trace buffer grow to %d bytes\n", size);
+            return false;
+        }
+        /* re-allocate trace buf */
+        LOG(THREAD, LOG_MONITOR, 3,
+            "\nRe-allocating trace buffer from %d to %d bytes\n",
+            md->trace_buf_size, size);
+        new_tbuf = heap_alloc(dcontext, size HEAPACCT(ACCT_TRACE));
+        if (md->trace_buf != NULL) {
+            /* copy entire thing, just in case */
+            IF_X64(ASSERT_NOT_REACHED()); /* can't copy w/o re-relativizing! */
+            memcpy(new_tbuf, md->trace_buf, md->trace_buf_size);
+            heap_free(dcontext, md->trace_buf, md->trace_buf_size HEAPACCT(ACCT_TRACE));
+            realloc_shift = new_tbuf - md->trace_buf;
+            /* need to walk through trace instr_t list and update addresses */
+            instr = instrlist_first(trace);
+            while (instr != NULL) {
+                byte *b = instr_get_raw_bits(instr);
+                if (b >= md->trace_buf &&
+                    b < md->trace_buf+md->trace_buf_size)
+                    instr_shift_raw_bits(instr, realloc_shift);
+                instr = instr_get_next(instr);
+            }
+        }
+        md->trace_buf = new_tbuf;
+        md->trace_buf_size = size;
+    }
+    if ((f->flags & FRAG_IS_TRACE) != 0) {
+        trace_only_t *t = TRACE_FIELDS(f);
+        new_blks = t->num_bbs;
+    } else
+        new_blks = 1;
+    if (md->num_blks + new_blks >= md->blk_info_length) {
+        trace_bb_build_t *new_buf;
+        uint new_len = md->blk_info_length;
+        if (new_len == 0)
+            new_len = INITIAL_NUM_BLKS;
+        do {
+            new_len *= 2;
+        } while (md->num_blks + new_blks >= new_len);
+        new_buf = (trace_bb_build_t *)
+            HEAP_ARRAY_ALLOC(dcontext, trace_bb_build_t, new_len, ACCT_TRACE, true);
+        /* PR 306761 relies on being zeroed */
+        if (new_len == INITIAL_NUM_BLKS)
+            memset(md->blk_info, 0, sizeof(trace_bb_build_t)*new_len);
+        LOG(THREAD, LOG_MONITOR, 3, "\nRe-allocating trace blks from %d to %d\n",
+            md->blk_info_length, new_len);
+        if (md->blk_info != NULL) {
+            memcpy(new_buf, md->blk_info, md->blk_info_length*sizeof(trace_bb_build_t));
+            HEAP_ARRAY_FREE(dcontext, md->blk_info, trace_bb_build_t, md->blk_info_length,
+                            ACCT_TRACE, true);
+        }
+        md->blk_info = new_buf;
+        md->blk_info_length = new_len;
+    }
+    return true;
+}
+
+static int
+trace_exit_stub_size_diff(dcontext_t *dcontext, fragment_t *f)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    int size = 0;
+    linkstub_t *l;
+    for (l = FRAGMENT_EXIT_STUBS(f); l != NULL; l = LINKSTUB_NEXT_EXIT(l)) {
+        if (linkstub_shares_next_stub(dcontext, f, l)) {
+            /* add stub size back in since we don't know if trace will also
+             * share (if client adds custom code, etc.)
+             * this also makes fixup_last_cti() code simpler since it can
+             * blindly remove and ignore sharing.
+             * if the trace does share for a final bb, we remove in
+             * end_and_emit_trace().
+             */
+            size += local_exit_stub_size(dcontext, EXIT_TARGET_TAG(dcontext, f, l),
+                                         md->trace_flags);
+        } else {
+            /* f's stub size will be considered as part of f->size so we need
+             * the difference here, not the absolute new size
+             */
+            size += local_exit_stub_size(dcontext, EXIT_TARGET_TAG(dcontext, f, l),
+                                         md->trace_flags) -
+                local_exit_stub_size(dcontext, EXIT_TARGET_TAG(dcontext, f, l),
+                                     f->flags);
+        }
+    }
+    return size;
+}
+
+/* don't build a single trace more than 1/8 of max trace cache size */
+enum { MAX_TRACE_FRACTION_OF_CACHE = 8 };
+
+/* Estimates the increase in the emitted size of the current trace if f were
+ * to be added to it.
+ * If that size exceeds the maximum fragment size, or a fraction of the maximum
+ * trace cache size, returns false.
+ * Returns the size calculations in two different parts:
+ * res_add_size is the accurate value of the body and exit stubs addition, while
+ * res_prev_mangle_size is an upper bound estimate of the change in size when
+ * the prior block in the trace is mangled to connect to f.
+ */
+static bool
+get_and_check_add_size(dcontext_t *dcontext, fragment_t *f, uint *res_add_size,
+                       uint *res_prev_mangle_size)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    uint add_size = f->size - fragment_prefix_size(f->flags) +
+        trace_exit_stub_size_diff(dcontext, f) + 
+        (PAD_FRAGMENT_JMPS(md->trace_flags) ? extend_trace_pad_bytes(f) : 0);
+    /* we estimate the size change from mangling the previous block to
+     * connect to this block if we were to add it
+     */
+    uint prev_mangle_size = TRACE_CTI_MANGLE_SIZE_UPPER_BOUND;
+    uint total_size = md->emitted_size + add_size + prev_mangle_size;
+    /* check whether adding f will push the trace over the edge */
+    bool ok = (total_size <= MAX_FRAGMENT_SIZE);
+    ASSERT(!TEST(FRAG_SELFMOD_SANDBOXED, f->flags)); /* no support for selfmod */
+    ASSERT(!TEST(FRAG_IS_TRACE, f->flags)); /* no support for traces */
+    LOG(THREAD, LOG_MONITOR, 4, "checking trace size: currently %d, add estimate %d\n"
+        "\t(body: %d, stubs: %d, pad: %d, mangle est: %d)\n"
+        "\t=> %d vs %d, %d vs %d\n",
+        md->emitted_size, add_size + prev_mangle_size,
+        f->size - fragment_prefix_size(f->flags),
+        trace_exit_stub_size_diff(dcontext, f),
+        (PAD_FRAGMENT_JMPS(md->trace_flags) ? extend_trace_pad_bytes(f) : 0),
+        prev_mangle_size, total_size, MAX_FRAGMENT_SIZE,
+        total_size*MAX_TRACE_FRACTION_OF_CACHE, DYNAMO_OPTION(cache_trace_max));
+    /* don't create traces anywhere near max trace cache size */
+    if (ok && DYNAMO_OPTION(cache_trace_max) > 0 &&
+        total_size*MAX_TRACE_FRACTION_OF_CACHE > DYNAMO_OPTION(cache_trace_max))
+        ok = false;
+    if (res_add_size != NULL)
+        *res_add_size = add_size;
+    if (res_prev_mangle_size != NULL)
+        *res_prev_mangle_size = prev_mangle_size;
+    return ok;
+}        
+
+/* propagate flags from a non-head bb component of a trace to the trace itself */
+static inline uint
+trace_flags_from_component_flags(uint flags)
+{
+    return (flags & (FRAG_HAS_SYSCALL | FRAG_HAS_DIRECT_CTI));
+}
+
+static inline uint
+trace_flags_from_trace_head_flags(uint head_flags)
+{
+    uint trace_flags = 0;
+    if (!INTERNAL_OPTION(unsafe_ignore_eflags_prefix)) {
+        trace_flags |= (head_flags & FRAG_WRITES_EFLAGS_6);
+        trace_flags |= (head_flags & FRAG_WRITES_EFLAGS_OF);
+    }
+    trace_flags |= FRAG_IS_TRACE;
+    trace_flags |= trace_flags_from_component_flags(head_flags);
+    if (DYNAMO_OPTION(shared_traces)) {
+        /* for now, all traces are shared */
+        trace_flags |= FRAG_SHARED;
+    }
+    return trace_flags;
+}
+
+/* Be careful with the case where the current fragment f to be executed
+ * has the same tag as the one we're emitting as a trace. 
+ */
+static fragment_t *
+end_and_emit_trace(dcontext_t *dcontext, fragment_t *cur_f)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    fragment_t *trace_head_f = NULL;
+    app_pc tag = md->trace_tag;
+    app_pc cur_f_tag = cur_f->tag; /* grab now before potential cur_f deletion */
+    instrlist_t *trace = &md->trace;
+    fragment_t *trace_f;
+    trace_only_t *trace_tr;
+    bool replace_trace_head = false;
+    fragment_t wrapper;
+    uint i;
+#if defined(DEBUG) || defined(INTERNAL) || defined(CLIENT_INTERFACE)
+    /* was the trace passed through optimizations or the client interface? */
+    bool externally_mangled = false;
+#endif
+    /* we cannot simply upgrade a basic block fragment
+     * to a trace b/c traces have prefixes that basic blocks don't!
+     */
+
+    DOSTATS({
+        /* static count last_exit statistics case 4817 */
+        if (LINKSTUB_INDIRECT(dcontext->last_exit->flags)) {
+            STATS_INC(num_traces_end_at_ibl);
+            if (TEST(LINK_CALL, dcontext->last_exit->flags)) {
+                STATS_INC(num_traces_end_at_ibl_ind_call);
+            } else if (TEST(LINK_JMP, dcontext->last_exit->flags)) {
+                /* shared system call (case 4995) */
+                if (IS_SHARED_SYSCALLS_LINKSTUB(dcontext->last_exit))
+                    STATS_INC(num_traces_end_at_ibl_syscall);
+                else
+                    STATS_INC(num_traces_end_at_ibl_ind_jump);
+            } else if (TEST(LINK_RETURN, dcontext->last_exit->flags)) {
+                STATS_INC(num_traces_end_at_ibl_return);
+            };
+        }
+    });
+
+#ifdef CLIENT_INTERFACE
+    if (md->pass_to_client) {
+        /* PR 299808: we pass the unmangled ilist we've been maintaining to the
+         * client, and we have to then re-mangle and re-connect.
+         */
+        dr_emit_flags_t emitflags = instrument_trace(dcontext, tag, &md->unmangled_ilist,
+                                                     false/*!recreating*/);
+        externally_mangled = true;
+        if (TEST(DR_EMIT_STORE_TRANSLATIONS, emitflags)) {
+            /* PR 214962: let client request storage instead of recreation */
+            md->trace_flags |= FRAG_HAS_TRANSLATION_INFO;
+        } /* else, leave translation flag if any bb requested it */
+
+        /* We now have to re-mangle and re-chain */
+        if (!mangle_trace(dcontext, &md->unmangled_ilist, md)) {
+            trace_abort(dcontext);
+            STATS_INC(num_aborted_traces_client);
+            trace_f = NULL;
+            goto end_and_emit_trace_return;
+        }
+        instrlist_clear(dcontext, &md->trace);
+        md->trace = md->unmangled_ilist;
+        instrlist_init(&md->unmangled_ilist);
+    }
+#endif
+
+    if (INTERNAL_OPTION(cbr_single_stub) &&
+        final_exit_shares_prev_stub(dcontext, trace, md->trace_flags)) {
+        /* while building, we re-add shared stub since not sure if
+         * trace will also share -- here we find out and adjust
+         */
+        instr_t *last = instrlist_last(trace);
+        app_pc target;
+        ASSERT(last != NULL && instr_is_exit_cti(last));
+        target = opnd_get_pc(instr_get_target(last));
+        md->emitted_size -= local_exit_stub_size(dcontext, target, md->trace_flags);
+    }
+
+    if (DYNAMO_OPTION(speculate_last_exit)
+#ifdef HASHTABLE_STATISTICS
+        || INTERNAL_OPTION(speculate_last_exit_stats) || INTERNAL_OPTION(stay_on_trace_stats)
+#endif
+        ) {
+        /* FIXME: speculation of last exit (case 4817) is currently
+         * only implemented for traces.  If we have a sharable version
+         * of fixup_last_cti() to pass that information based on instr
+         * list information about last exit we can use in
+         * emit_fragment_common().  That way both bb's and traces may
+         * have speculation added.
+         */
+        if (TEST(FRAG_MUST_END_TRACE, cur_f->flags)) {
+            /* This routine may be also reached on MUST_END_TRACE
+             * and in that case we haven't executed yet the last
+             * bb, so don't really know how to fix the last IBL
+             */
+            /* FIXME: add a stat when such are ending at an IBL */
+
+            ASSERT_CURIOSITY(dcontext->next_tag == cur_f->tag);
+            STATS_INC(num_traces_at_must_end_trace);
+        } else {
+            /* otherwise last_exit is the last trace BB and next_tag
+             * is the current IBL target that we'll always speculate */
+            if (LINKSTUB_INDIRECT(dcontext->last_exit->flags)) {
+                LOG(THREAD, LOG_MONITOR, 2, 
+                    "Last trace IBL exit (trace "PFX", next_tag "PFX")\n", 
+                    tag, dcontext->next_tag);
+                ASSERT_CURIOSITY(dcontext->next_tag != NULL);
+                if (DYNAMO_OPTION(speculate_last_exit)) {
+                    app_pc speculate_next_tag = dcontext->next_tag;
+#ifdef SPECULATE_LAST_EXIT_STUDY 
+                    /* for a performance study: add overhead on
+                     * all IBLs that never hit by comparing to a 0xbad tag */
+                    speculate_next_tag = 0xbad;
+#endif
+                    md->emitted_size += 
+                        append_trace_speculate_last_ibl(dcontext, trace, 
+                                                        speculate_next_tag, 
+                                                        false);
+                } else {
+#ifdef HASHTABLE_STATISTICS
+                    ASSERT(INTERNAL_OPTION(stay_on_trace_stats) || 
+                           INTERNAL_OPTION(speculate_last_exit_stats));
+                    DOSTATS({
+                        md->emitted_size += 
+                            append_ib_trace_last_ibl_exit_stat(dcontext, trace, 
+                                INTERNAL_OPTION(speculate_last_exit_stats) ? 
+                                dcontext->next_tag : NULL);
+                    });
+#endif
+                }
+            }
+        }
+    }
+
+    DOLOG(2, LOG_MONITOR, {
+        uint i;
+        LOG(THREAD, LOG_MONITOR, 2, "Ending and emitting hot trace (tag "PFX")\n", tag);
+        if (stats->loglevel >= 4) {
+            instrlist_disassemble(dcontext, md->trace_tag, trace, THREAD);
+            LOG(THREAD, LOG_MONITOR, 4, "\n");
+        }
+        LOG(THREAD, LOG_MONITOR, 2, "Trace blocks are:\n");
+        for (i=0; i<md->num_blks; i++) {
+            LOG(THREAD, LOG_MONITOR, 2, "\tblock %3d == "PFX" (%d exit(s))\n",
+                i, md->blk_info[i].info.tag,
+                IF_RETURN_AFTER_CALL_ELSE(md->blk_info[i].info.num_exits, 0));
+        }
+    });
+
+    /* WARNING: if you change how optimizations are performed, you
+     * must change recreate_app_state in x86/arch.c as well
+     */
+    
+#ifdef INTERNAL
+    if (dynamo_options.optimize
+#  ifdef SIDELINE
+        && !dynamo_options.sideline
+#  endif
+        ) {
+        optimize_trace(dcontext, tag, trace);
+        externally_mangled = true;
+    }
+#endif /* INTERNAL */
+
+#ifdef PROFILE_RDTSC
+    if (dynamo_options.profile_times) {
+        /* space was already reserved in buffer and in md->emitted_size */
+        add_profile_call(dcontext);
+    }
+#endif
+
+#ifdef SIDELINE
+    if (dynamo_options.sideline) {
+        /* FIXME: add size to emitted_size when start building trace to
+         * ensure room in buffer and in cache
+         */
+        add_sideline_prefix(dcontext, trace);
+    }
+#endif
+
+    /* delete any private copy now and use its space for this trace
+     * for private traces:
+     *   this way we use the head of FIFO for all our private copies, and
+     *   then replace w/ the trace, avoiding any fragmentation from the copies.
+     * for shared traces: FIXME: case 5137: move temps to private bb cache?
+     */
+    if (md->last_copy != NULL) {
+        if (cur_f == md->last_copy)
+            cur_f = NULL;
+        delete_private_copy(dcontext);
+    }
+
+    /* Shared trace synchronization model:
+     * We can't hold locks across cache executions, and we wouldn't want to have a
+     * massive trace building lock anyway, so we only grab a lock at the final emit
+     * moment and if there's a conflict the loser tosses his trace.
+     * We hold the lock across the trace head removal as well to avoid races there.
+     */
+    if (TEST(FRAG_SHARED, md->trace_flags)) {
+        ASSERT(DYNAMO_OPTION(shared_traces));
+        mutex_lock(&trace_building_lock);
+        /* we left the bb there, so we rely on any shared trace shadowing it */
+        trace_f = fragment_lookup_trace(dcontext, tag);
+        if (trace_f != NULL) {
+            /* someone beat us to it!  tough luck -- throw it all away */
+            ASSERT(TEST(FRAG_IS_TRACE, trace_f->flags));
+            mutex_unlock(&trace_building_lock);
+            trace_abort(dcontext);
+            STATS_INC(num_aborted_traces_race);
+#ifdef DEBUG
+            /* We expect to see this very rarely since we expect to detect
+             * practically all races (w/shared BBs anyway) much earlier.
+             * FIXME case 8769: we may need another way to prevent races w/
+             * -coarse_units!
+             */
+            if (DYNAMO_OPTION(shared_bbs) && !DYNAMO_OPTION(coarse_units))
+                ASSERT_CURIOSITY(false);
+#endif
+            /* deliberately leave trace_f as it is */
+            goto end_and_emit_trace_return;
+        }
+    }
+
+    /* Delete existing fragment(s) with tag value.
+     *
+     * For shared traces, if -no_remove_shared_trace_heads, we do not remove
+     * shared trace heads and only transfer their links
+     * over to the new trace (and if the trace is deleted we transfer the
+     * links back).  We leave them alone otherwise, shadowed in both the DR
+     * lookup tables and ibl tables.
+     * FIXME: trace head left w/ no incoming -- will this break assumptions?
+     * What if someone who held ptr before trace emit, or does a different
+     * lookup, tries to mess w/ trace head's links?
+     */
+    if (cur_f != NULL && cur_f->tag == tag) {
+        /* Optimization: could repeat for shared as well but we don't bother */
+        if (!TEST(FRAG_SHARED, cur_f->flags))
+            trace_head_f = cur_f;
+        /* Yipes, we're deleting the fragment we're supposed to execute next.
+         * Set cur_f to NULL even if not deleted, since we want to
+         * execute the trace in preference to the trace head.
+         */
+        cur_f = NULL;
+    }
+    /* remove private trace head fragment, if any */
+    if (trace_head_f == NULL) /* from cur_f */
+        trace_head_f = fragment_lookup_same_sharing(dcontext, tag, 0/*FRAG_PRIVATE*/);
+    /* We do not go through other threads and delete their private trace heads,
+     * presuming that they have them for a reason and don't want this shared trace
+     */
+    if (trace_head_f != NULL) {
+        LOG(THREAD, LOG_MONITOR, 4, "deleting private trace head fragment\n");
+        /* we have to manually check last_exit -- can't have fragment_delete()
+         * call monitor_remove_fragment() to avoid aborting our trace
+         */
+        if (trace_head_f == dcontext->last_fragment)
+            last_exit_deleted(dcontext);
+#ifdef NATIVE_RETURN
+        /* just remove from hashtable, can't delete from fcache b/c could ret there */
+        fragment_delete(dcontext, trace_head_f, FRAGDEL_NO_OUTPUT | FRAGDEL_NO_FCACHE |
+                        FRAGDEL_NO_VMAREA | FRAGDEL_NO_HEAP | FRAGDEL_NO_MONITOR);
+#else
+# ifdef CUSTOM_TRACES
+        /* If the trace is private, don't delete the head: the trace will simply
+         * shadow it.  If the trace is shared, we have to delete it.  We'll re-create
+         * the head as a shared bb if we ever do build a custom trace through it.
+         */
+        if (!TEST(FRAG_SHARED, md->trace_flags)) {
+            replace_trace_head = true;
+            /* we can't have our trace_head_f clobbered below */
+            CLIENT_ASSERT(!DYNAMO_OPTION(shared_bbs), "invalid private trace head and "
+                          "private traces but -shared_bbs for CUSTOM_TRACES");
+        } else {
+# endif
+            fragment_delete(dcontext, trace_head_f,
+                            FRAGDEL_NO_OUTPUT | FRAGDEL_NO_MONITOR);
+# ifdef CUSTOM_TRACES
+        }
+# endif
+#endif
+        if (!replace_trace_head) {
+            trace_head_f = NULL;
+            STATS_INC(num_fragments_deleted_trace_heads);
+        }
+    }
+    /* find shared trace head fragment, if any */
+    if (DYNAMO_OPTION(shared_bbs)) {
+        trace_head_f =
+            fragment_lookup_fine_and_coarse_sharing(dcontext, tag, &wrapper,
+                                                    NULL, FRAG_SHARED);
+        if (!TEST(FRAG_SHARED, md->trace_flags)) {
+            /* trace is private, so we can emit as a shadow of trace head */
+        } else if (trace_head_f != NULL) {
+            /* we don't remove until after emitting a shared trace to avoid races
+             * with trace head being re-created before the trace is visible
+             */
+            replace_trace_head = true;
+            if (!TEST(FRAG_IS_TRACE_HEAD, trace_head_f->flags)) {
+                ASSERT(TEST(FRAG_COARSE_GRAIN, trace_head_f->flags));
+                /* local wrapper so change_linking_lock not needed to change flags */
+                trace_head_f->flags |= FRAG_IS_TRACE_HEAD;
+            }
+        }
+    }
+            
+    /* Prevent deletion of last_fragment, which may be in the same
+     * cache as our trace (esp. w/ a MUST_END_TRACE trace head, since then the
+     * last_fragment can be another trace) from clobbering our trace!
+     * FIXME: would be cleaner to remove the need to abort the trace if
+     * last_fragment is deleted, but tricky to do that (see
+     * monitor_remove_fragment).  Could also use a special monitor_data_t field
+     * saying "ignore last_exit, I'm emitting now."
+     */
+    if (!LINKSTUB_FAKE(dcontext->last_exit)) /* head delete may have already done this */
+        last_exit_deleted(dcontext);
+    ASSERT(md->last_fragment == NULL);
+    ASSERT(md->last_copy == NULL);
+    /* ensure trace was NOT aborted */
+    ASSERT(md->trace_tag == tag);
+
+    /* emit trace fragment into fcache with tag value */
+    if (replace_trace_head) {
+#ifndef CUSTOM_TRACES
+        ASSERT(TEST(FRAG_SHARED, md->trace_flags));
+#endif
+        trace_f = emit_fragment_as_replacement(dcontext, tag, trace, md->trace_flags,
+                                               md->trace_vmlist, trace_head_f);
+    } else {
+        trace_f = emit_fragment(dcontext, tag, trace, md->trace_flags, md->trace_vmlist,
+                                true/*link*/);
+    }
+    ASSERT(trace_f != NULL);
+    /* our estimate should be conservative 
+     * if externally mangled, all bets are off for now --
+     * FIXME: would be nice to gracefully handle opt or client
+     * making the trace too big, and pass back an error msg?
+     * Perhaps have lower size bounds when optimization or client
+     * interface are on.
+     */
+    LOG(THREAD, LOG_MONITOR, 3, "Trace estimated size %d vs actual size %d\n",
+        md->emitted_size, trace_f->size);
+    ASSERT(trace_f->size <= md->emitted_size || externally_mangled);
+    /* our calculations should be exact, actually */
+    /* with -pad_jmps not exact anymore, we should be able to figure out 
+     * by how much though FIXME */
+    ASSERT_CURIOSITY(trace_f->size == md->emitted_size || externally_mangled ||
+                     PAD_FRAGMENT_JMPS(trace_f->flags));
+    trace_tr = TRACE_FIELDS(trace_f);
+    trace_tr->num_bbs = md->num_blks;
+    trace_tr->bbs = (trace_bb_info_t *)
+        nonpersistent_heap_alloc(FRAGMENT_ALLOC_DC(dcontext, trace_f->flags),
+                                 md->num_blks*sizeof(trace_bb_info_t)
+                                 HEAPACCT(ACCT_TRACE));
+    for (i = 0; i < md->num_blks; i++)
+        trace_tr->bbs[i] = md->blk_info[i].info;
+
+    if (TEST(FRAG_SHARED, md->trace_flags))
+        mutex_unlock(&trace_building_lock);
+
+    RSTATS_INC(num_traces);
+    STATS_ADD(num_bbs_in_all_traces, md->num_blks);
+    STATS_TRACK_MAX(max_bbs_in_a_trace, md->num_blks);
+    DOLOG(2, LOG_MONITOR, {
+        LOG(THREAD, LOG_MONITOR, 1, "Generated trace fragment #%d for tag "PFX"\n",
+            GLOBAL_STAT(num_traces), tag);
+        disassemble_fragment(dcontext, trace_f, stats->loglevel < 3);
+    });
+
+#ifdef INTERNAL
+    DODEBUG({
+        if (INTERNAL_OPTION(stress_recreate_pc)) {
+            /* verify trace recreation - done here after bb_tag[] is in place */
+            stress_test_recreate(dcontext, trace_f, trace);
+        }
+    });
+#endif
+
+    /* we can't call reset_trace_state() until after -remove_trace_components,
+     * but we must clear these two before enter_nolinking so that a flusher
+     * doesn't access them in an inconsistent state (trace_vmlist is invalid
+     * once also pointers are transfered to real fragment)
+     */
+    md->trace_vmlist = NULL;
+    md->trace_tag = NULL;
+
+    /* these calls to fragment_remove_shared_no_flush may become
+     * nolinking, meaning we need to hold no locks here, and that when
+     * we get back our local fragment_t pointers may be invalid.
+     */
+    /* remove shared trace head fragment */
+    if (trace_head_f != NULL && DYNAMO_OPTION(shared_bbs) &&
+        TEST(FRAG_SHARED, md->trace_flags) &&
+        /* We leave the head in the coarse table and let the trace shadow it.
+         * If we were to remove it we would need a solution to finding it for
+         * pc translation, which currently walks the htable.
+         */
+        !TEST(FRAG_COARSE_GRAIN, trace_head_f->flags) &&
+        /* if both shared only remove if option on, and no custom tracing */
+        IF_CUSTOM_TRACES(!dr_end_trace_hook_exists() &&)
+        INTERNAL_OPTION(remove_shared_trace_heads)) {
+        fragment_remove_shared_no_flush(dcontext, trace_head_f);
+        trace_head_f = NULL;
+    }
+
+    if (DYNAMO_OPTION(remove_trace_components)) {
+        uint i;
+        fragment_t *f;
+        /* use private md values, don't trust trace_tr */
+        for (i = 1/*skip trace head*/; i < md->num_blks; i++) {
+            f = fragment_lookup_bb(dcontext, md->blk_info[i].info.tag);
+            if (f != NULL) {
+                if (TEST(FRAG_SHARED, f->flags) && !TEST(FRAG_COARSE_GRAIN, f->flags)) {
+                    /* FIXME: grab locks up front instead of on each delete */
+                    fragment_remove_shared_no_flush(dcontext, f);
+                    trace_head_f = NULL; /* be safe */
+                } else
+                    fragment_delete(dcontext, f, FRAGDEL_NO_OUTPUT | FRAGDEL_NO_MONITOR);
+                STATS_INC(trace_components_deleted);
+            }
+        }
+    }
+
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, trace);
+
+    ASSERT_DO_NOT_OWN_MUTEX(true, &change_linking_lock);
+    md->trace_tag = tag; /* reinstate for reset */
+    reset_trace_state(dcontext, true /* might need change_linking_lock */);
+
+#ifdef DEBUG
+    /* If we're building shared traces and using shared BBs, FRAG_TRACE_BUILDING
+     * shouldn't be set on the trace head fragment. If we're not using shared
+     * BBs or are not building shared traces, the flag shouldn't be set then
+     * either. Basically, it should never be set at this point, after the call
+     * to reset_trace_state() just above.
+     */
+    if (trace_head_f != NULL)
+        ASSERT(!TEST(FRAG_TRACE_BUILDING, trace_head_f->flags));
+#endif
+
+ end_and_emit_trace_return:
+    if (cur_f == NULL && cur_f_tag == tag)
+        return trace_f;
+    else {
+        /* emitting the new trace may have deleted the next fragment to execute
+         * best way to find out is to re-look-up the next fragment (this only
+         * happens when emitting trace, so rare enough)
+         */
+        cur_f = fragment_lookup(dcontext, cur_f_tag);
+        return cur_f;
+    }
+}
+
+/* Note: The trace being built currently can be emitted in 
+ * internal_extend_trace() rather than the next time into monitor_cache_enter() 
+ * if fragment results in a system call (sysenter) or callback (int 2b), i.e.,
+ * is marked FRAG_MUST_END_TRACE.
+ */
+static fragment_t *
+internal_extend_trace(dcontext_t *dcontext, fragment_t *f, linkstub_t *prev_l,
+                      uint add_size)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    bool have_locks = false;
+    DEBUG_DECLARE(uint pre_emitted_size = md->emitted_size;)
+
+#ifdef CLIENT_INTERFACE
+    extend_unmangled_ilist(dcontext);
+#endif
+
+    /* if prev_l is fake, NULL it out */
+    if (is_ibl_sourceless_linkstub((const linkstub_t *)prev_l)) {
+        ASSERT(!DYNAMO_OPTION(indirect_stubs));
+        prev_l = NULL;
+    }
+    ASSERT(prev_l == NULL || !LINKSTUB_FAKE(prev_l) ||
+           /* we track the ordinal of the del linkstub so it's ok */
+           prev_l == get_deleted_linkstub(dcontext));
+
+    if (TEST(FRAG_SHARED, f->flags)) {
+        /* Case 8419: we must hold a lock to ensure f is not
+         * fragment_remove_shared_no_flush()-ed underneath us, eliminating its
+         * also fields needed for vm_area_add_to_list() (plus w/ the also field
+         * re-used for case 3559 we have crash potential).
+         */
+        have_locks = true;
+        /* lock rank order requires cll before shared_vm_areas */
+        SHARED_FLAGS_RECURSIVE_LOCK(f->flags, acquire, change_linking_lock);
+        acquire_vm_areas_lock(dcontext, f->flags);
+    }
+    if (TEST(FRAG_WAS_DELETED, f->flags)) {
+        /* We cannot continue if f is FRAG_WAS_DELETED (case 8419) since
+         * fragment_t.also is now invalid!
+         */
+        STATS_INC(num_trace_next_bb_deleted);
+        ASSERT(have_locks);
+        if (have_locks) {
+            release_vm_areas_lock(dcontext, f->flags);
+            SHARED_FLAGS_RECURSIVE_LOCK(f->flags, release, change_linking_lock);
+        }
+        return end_and_emit_trace(dcontext, f);
+    }
+
+    /* We have to calculate the added size before we extend, so we
+     * have that passed in, though without the estimate for the mangling
+     * of the previous block (thus including only f->size and the exit stub
+     * size changes), which we calculate in extend_trace.
+     * Existing custom stub code should already be in f->size.
+     * FIXME: if we ever have decode_fragment() convert, say, dcontext
+     * save/restore to tls, then we'll have to add in its size increases
+     * as well.
+     */
+    md->emitted_size += add_size;
+
+    md->trace_flags |= trace_flags_from_component_flags(f->flags);
+
+    /* call routine in interp.c */
+    md->emitted_size += extend_trace(dcontext, f, prev_l);
+
+    LOG(THREAD, LOG_MONITOR, 3, "extending added %d to size of trace => %d total\n",
+        md->emitted_size - pre_emitted_size, md->emitted_size);
+
+    vm_area_add_to_list(dcontext, md->trace_tag, &(md->trace_vmlist), md->trace_flags,
+                        f, have_locks);
+    if (have_locks) {
+        /* We must give up change_linking_lock in order to execute
+         * create_private_copy (it calls emit()) but we're at a stable state
+         * now.
+         */
+        release_vm_areas_lock(dcontext, f->flags);
+        SHARED_FLAGS_RECURSIVE_LOCK(f->flags, release, change_linking_lock);
+    }
+
+    DOLOG(3, LOG_MONITOR, {
+        LOG(THREAD, LOG_MONITOR, 4, "After extending, trace looks like this:\n");
+        instrlist_disassemble(dcontext, md->trace_tag, &md->trace, THREAD);
+    });
+    /* trace_t extended;  prepare bb for execution to find where to go next. */
+
+    /* For FRAG_MUST_END_TRACE fragments emit trace immediately to prevent 
+     * trace aborts due to syscalls and callbacks.  See case 3541.
+     */
+    if (TEST(FRAG_MUST_END_TRACE, f->flags)) {
+        /* We don't need to unlink f, but we would need to set FRAG_CANNOT_DELETE to
+         * prevent its deletion during emitting from clobbering the trace in the case
+         * that last_fragment==f (requires that f targets itself, and f is
+         * private like traces -- not possible w/ today's syscall-only MUST_END_TRACE
+         * fragments but could happen in the future) -- except that that's a general
+         * problem handled by clearing last_exit in end_and_emit_trace, so we do
+         * nothing here.
+         */
+        return end_and_emit_trace(dcontext, f);
+    }
+
+    if (TEST(FRAG_SHARED, f->flags) && !TEST(FRAG_TEMP_PRIVATE, f->flags)) {
+        /* strategy: make a private copy of f to avoid synch issues w/ others
+         * modifying its linking before we get back here
+         */
+        if (!TEST(FRAG_COARSE_GRAIN, f->flags)) {
+            if (!create_private_copy(dcontext, f)) {
+                return end_and_emit_trace(dcontext, f);
+            }
+        }
+        /* else, we should have made the private copy earlier */
+        ASSERT(md->last_copy != NULL);
+        ASSERT(md->last_copy->tag == f->tag);
+        ASSERT(md->last_fragment == md->last_copy);
+        if (md->trace_tag == NULL) {
+            /* trace was aborted b/c our new fragment clobbered someone (see
+             * comments in create_private_copy) --
+             * when emitting our private bb we can kill the last_fragment):
+             * just exit now
+             */
+            LOG(THREAD, LOG_MONITOR, 4, "Private copy ended up aborting trace!\n");
+            STATS_INC(num_trace_private_copy_abort);
+            /* trace abort happened in emit_fragment, so we went and undid the
+             * clearing of last_fragment by assigning it to last_copy, must re-clear!
+             */
+            md->last_fragment = NULL;
+            return f;
+        }
+
+        /* operate on new f from here on */
+        f = md->last_fragment;
+    } else {
+        /* must store this fragment, and also duplicate its flags so know what
+         * to restore.  can't rely on last_exit for restoring since could end up
+         * not coming out of cache from last_fragment (e.g., if hit sigreturn)
+         */
+        md->last_fragment = f;
+    }
+
+    /* hold lock across cannot delete changes too, and store of flags */
+    SHARED_FLAGS_RECURSIVE_LOCK(f->flags, acquire, change_linking_lock);
+
+    md->last_fragment_flags = f->flags;
+    if ((f->flags & FRAG_CANNOT_DELETE) == 0) {
+        /* don't let this fragment be deleted, we'll need it as
+         * dcontext->last_exit for extend_trace
+         */
+        f->flags |= FRAG_CANNOT_DELETE;
+        LOG(THREAD, LOG_MONITOR, 4,
+            "monitor marked F%d ("PFX") as un-deletable\n", f->id, f->tag);
+    }
+
+#ifdef CUSTOM_TRACES
+    /* may end up going through trace head, etc. that isn't linked */
+    if ((f->flags & FRAG_LINKED_OUTGOING) != 0) {
+#endif
+        /* unlink so monitor invoked on fragment exit */
+        unlink_fragment_outgoing(dcontext, f);
+        LOG(THREAD, LOG_MONITOR|LOG_LINKS, 4,
+            "monitor unlinked F%d ("PFX")\n", f->id, f->tag);
+#ifdef CUSTOM_TRACES
+    }
+#endif
+
+    SHARED_FLAGS_RECURSIVE_LOCK(f->flags, release, change_linking_lock);
+
+    return f;
+}
+
+/* we use last_fragment to hold bb that needs to be restored.
+ * it's a field used only by us.
+ */
+static void
+internal_restore_last(dcontext_t *dcontext)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    if (md->last_fragment == NULL)
+        return;
+    /* must restore fragment used to extend trace to pre-trace-building state.
+     * sometimes we come in here from trace_abort and we've already restored
+     * the last exit, so check before linking.
+     */
+    /* need to hold lock for any shared link modification */
+    SHARED_FLAGS_RECURSIVE_LOCK(md->last_fragment->flags, acquire, change_linking_lock);
+    if ((md->last_fragment_flags & FRAG_LINKED_OUTGOING) != 0 &&
+        (md->last_fragment->flags & FRAG_LINKED_OUTGOING) == 0) {
+        LOG(THREAD, LOG_MONITOR, 4,
+            "internal monitor: relinking last fragment F%d\n",
+            md->last_fragment->id);
+        link_fragment_outgoing(dcontext, md->last_fragment, false);
+    }
+    if ((md->last_fragment_flags & FRAG_CANNOT_DELETE) == 0 &&
+        (md->last_fragment->flags & FRAG_CANNOT_DELETE) != 0) {
+        LOG(THREAD, LOG_MONITOR, 4,
+            "internal monitor: re-marking last fragment F%d as deletable\n",
+            md->last_fragment->id);
+        md->last_fragment->flags &= ~FRAG_CANNOT_DELETE;
+    }
+    /* flags may not match, e.g., if frag was marked as trace head */
+    ASSERT((md->last_fragment_flags &
+            (FRAG_CANNOT_DELETE|FRAG_LINKED_OUTGOING)) ==
+           (md->last_fragment->flags &
+            (FRAG_CANNOT_DELETE|FRAG_LINKED_OUTGOING)));
+    /* hold lock across FRAG_CANNOT_DELETE changes and all other flag checks, too */
+    SHARED_FLAGS_RECURSIVE_LOCK(md->last_fragment->flags, release, change_linking_lock);
+
+    /* last_fragment is ONLY used for restoring, so kill now, else our own
+     * deletion of trace head will cause use to abort single-bb trace
+     * (see monitor_remove_fragment)
+     *
+     * Do NOT reset last_fragment_flags as that field is needed prior to the
+     * cache entry and is referenced in monitor_cache_enter().
+     */
+    md->last_fragment = NULL;
+}
+
+/* if we are building a trace, unfreezes and relinks the last_fragment */
+void
+monitor_cache_exit(dcontext_t *dcontext)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    /* where processing */
+    ASSERT(dcontext->whereami == WHERE_DISPATCH);
+    dcontext->whereami = WHERE_MONITOR;
+    if (md->trace_tag > 0 && md->last_fragment != NULL) {
+        /* unprotect local heap */
+        SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+        /* must restore fragment to pre-trace-building state */
+        internal_restore_last(dcontext);
+        /* re-protect local heap */
+        SELF_PROTECT_LOCAL(dcontext, READONLY);
+    }
+    else if (md->trace_tag == NULL) {
+        /* Capture the case where the most recent cache exit was prior to a
+         * non-ignorable syscall that used the SYSENTER instruction, which
+         * we've seen on XP and 2003. The 'ret' after the SYSENTER executes
+         * natively, and this piece of control flow isn't captured during
+         * linking so link-time trace head marking doesn't work. (The exit
+         * stub is marked as a direct exit.)  The exit stub is reset during
+         * syscall handling so indirect-exit trace head marking isn't
+         * possible either, so we have to use a dedicated var to capture
+         * this case.
+         *
+         * We need to set trace_sysenter_exit to true or false to prevent a
+         * stale value from reaching a later read of the flag.
+         *
+         * FIXME Rework this to store the last (pre-syscall) exit's fragment flags & tag
+         * in a dcontext-private place such as non-shared monitor data.
+         * Such a general mechanism will permit us to capture all
+         * trace head marking within should_be_trace_head().
+         */
+        dcontext->trace_sysenter_exit =
+            (TEST(FRAG_IS_TRACE, dcontext->last_fragment->flags) &&
+             TEST(LINK_NI_SYSCALL, dcontext->last_exit->flags));
+    }
+    dcontext->whereami = WHERE_DISPATCH;
+}
+
+static void
+check_fine_to_coarse_trace_head(dcontext_t *dcontext, fragment_t *f)
+{
+    /* Case 8632: When a fine fragment targets a coarse trace head, we have
+     * no way to indicate that (there is no entrance stub for the fine
+     * fragments, as once the coarse unit is frozen we can't use its
+     * entrance stub).  So we assume that an exit is due to trace headness
+     * discovered at link time iff it would now be considered a trace head.
+     * FIXME: any cleaner way?
+     */
+    if (TEST(FRAG_COARSE_GRAIN, f->flags) &&
+        !TEST(FRAG_IS_TRACE_HEAD, f->flags) &&
+        /* FIXME: We rule out empty fragments -- but in so doing we rule out deleted
+         * fragments.  Oh well.
+         */
+        !TESTANY(FRAG_COARSE_GRAIN|FRAG_FAKE, dcontext->last_fragment->flags)) {
+        /* we lock up front since check_for_trace_head() expects it for shared2shared */
+        acquire_recursive_lock(&change_linking_lock);
+        if (check_for_trace_head(dcontext, dcontext->last_fragment,
+                                 dcontext->last_exit, f, true/*have lock*/,
+                                 false/*not sysenter exit*/)) {
+            STATS_INC(num_exits_fine2th_coarse);
+        } else {
+            /* This does happen: e.g., if we abort a trace, we came from a private fine
+             * bb and may target a coarse bb
+             */
+            STATS_INC(num_exits_fine2non_th_coarse);
+        }
+        release_recursive_lock(&change_linking_lock);
+    }
+}
+
+/* This routine maintains the statistics that identify hot code
+ * regions, and it controls the building and installation of trace
+ * fragments.
+ */
+fragment_t *
+monitor_cache_enter(dcontext_t *dcontext, fragment_t *f)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    bool start_trace = false;
+    bool end_trace = false;
+#ifdef CUSTOM_TRACES
+    dr_custom_trace_action_t client = CUSTOM_TRACE_DR_DECIDES;
+#endif
+    trace_head_counter_t *ctr;
+    uint add_size = 0, prev_mangle_size = 0; /* NOTE these aren't set if end_trace */
+
+    if (DYNAMO_OPTION(disable_traces) || f == NULL) {
+        /* nothing to do */
+        ASSERT(md->trace_tag == NULL);
+        return f;
+    }
+
+    /* where processing */
+    ASSERT(dcontext->whereami == WHERE_DISPATCH);
+    dcontext->whereami = WHERE_MONITOR;
+
+    /* default internal routine */
+
+    /* Ensure we know whether f is a trace head, before we do anything else
+     * (xref bug 8637 on not terminating traces b/c we marked as head too late)
+     */
+    check_fine_to_coarse_trace_head(dcontext, f);
+
+    if (md->trace_tag > 0) {      /* in trace selection mode */
+        KSTART(trace_building);
+
+        /* unprotect local heap */
+        SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+        /* should have restored last fragment on cache exit */
+        ASSERT(md->last_fragment == NULL);
+#ifdef RETURN_STACK
+        if (md->num_blks > 0) {
+            if (TEST(FRAG_ENDS_WITH_RETURN, dcontext->last_fragment->flags))
+                end_trace = true;
+        }
+#endif
+
+        /* check for trace ending conditions that can be overridden by client */
+        end_trace = (end_trace ||
+                     TEST(FRAG_IS_TRACE, f->flags ) ||
+                     TEST(FRAG_IS_TRACE_HEAD, f->flags));
+#ifdef CUSTOM_TRACES
+        if (dr_end_trace_hook_exists()) {
+            client = instrument_end_trace(dcontext, md->trace_tag, f->tag);
+            /* Return values:
+             *   CUSTOM_TRACE_DR_DECIDES = use standard termination criteria
+             *   CUSTOM_TRACE_END_NOW    = end trace
+             *   CUSTOM_TRACE_CONTINUE   = do not end trace
+             */
+            if (client == CUSTOM_TRACE_END_NOW) {
+                DOSTATS({
+                    if (!end_trace) {
+                        LOG(THREAD, LOG_MONITOR, 3,
+                            "Client ending 0x%08x trace early @0x%08x\n",
+                            md->trace_tag, f->tag);
+                        STATS_INC(custom_traces_stop_early);
+                    }
+                });
+                end_trace = true;
+            } else if (client == CUSTOM_TRACE_CONTINUE) {
+                DOSTATS({
+                    if (end_trace) {
+                        LOG(THREAD, LOG_MONITOR, 3,
+                            "Client not ending 0x%08x trace @ normal stop @0x%08x\n",
+                            md->trace_tag, f->tag);
+                        STATS_INC(custom_traces_stop_late);
+                    }
+                });
+                end_trace = false;
+            }
+            LOG(THREAD, LOG_MONITOR, 4, "Client instrument_end_trace returned %d\n",
+                client);
+        }
+#endif
+        /* check for conditions signaling end of trace regardless of client */
+        end_trace = end_trace || TEST(FRAG_CANNOT_BE_TRACE, f->flags);
+
+#ifdef X64
+        /* no traces that mix 32 and 64: decode_fragment not set up for it */
+        if (TEST(FRAG_32_BIT, f->flags) != TEST(FRAG_32_BIT, md->trace_flags))
+            end_trace = true;
+#endif
+
+        if (!end_trace) {
+#ifdef CUSTOM_TRACES
+            /* we need a regular bb here, not a trace */
+            if (TEST(FRAG_IS_TRACE, f->flags)) {
+                /* We create an official, shared bb (we DO want to call the client bb
+                 * hook, right?).  We do not link the new, shadowed bb.
+                 */
+                fragment_t *head = NULL;
+                if (USE_BB_BUILDING_LOCK())
+                    mutex_lock(&bb_building_lock);
+                if (DYNAMO_OPTION(coarse_units)) {
+                    /* the existing lookup routines will shadow a coarse bb so we do
+                     * a custom lookup
+                     */
+                    head = fragment_coarse_lookup_wrapper(dcontext, f->tag,
+                                                          &md->wrapper);
+                }
+                if (head == NULL)
+                    head = fragment_lookup_bb(dcontext, f->tag);
+                if (head == NULL) {
+                    LOG(THREAD, LOG_MONITOR, 3,
+                        "Client custom trace 0x%08x requiring shadow bb 0x%08x\n",
+                        md->trace_tag, f->tag);
+                    SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+                    /* We need to mark as trace head to hit the shadowing checks
+                     * and asserts when adding to fragment htable and unlinking
+                     * on delete.
+                     */
+                    head = build_basic_block_fragment
+                        (dcontext, f->tag, FRAG_IS_TRACE_HEAD, false/*do not link*/,
+                         true/*visible*/ _IF_CLIENT(true/*for trace*/) _IF_CLIENT(NULL));
+                    SELF_PROTECT_LOCAL(dcontext, READONLY);
+                    STATS_INC(custom_traces_bbs_built);
+                    ASSERT(head != NULL);
+                    /* If it's not shadowing we should have linked before htable add.
+                     * We shouldn't end up w/ a bb of different sharing than the
+                     * trace: CUSTOM_TRACES rules out private traces and shared bbs,
+                     * and if circumstances changed since the original trace head bb
+                     * was made then the trace should have been flushed.
+                     */
+                    ASSERT((head->flags & FRAG_SHARED) == (f->flags & FRAG_SHARED));
+                    if (TEST(FRAG_COARSE_GRAIN, head->flags)) {
+                        /* we need a local copy before releasing the lock.
+                         * FIXME: share this code sequence w/ dispatch().
+                         */
+                        ASSERT(USE_BB_BUILDING_LOCK());
+                        fragment_coarse_wrapper(&md->wrapper, f->tag,
+                                                FCACHE_ENTRY_PC(head));
+                        md->wrapper.flags |= FRAG_IS_TRACE_HEAD;
+                        head = &md->wrapper;
+                    }
+                }
+                if (USE_BB_BUILDING_LOCK())
+                    mutex_unlock(&bb_building_lock);
+                /* use the bb from here on out */
+                f = head;
+            }
+#endif
+            if (TEST(FRAG_COARSE_GRAIN, f->flags)
+                IF_CLIENT_INTERFACE(|| md->pass_to_client)) {
+                /* We need linkstub_t info for trace_exit_stub_size_diff() so we go
+                 * ahead and make a private copy here.
+                 */
+                if (create_private_copy(dcontext, f)) {
+                    /* operate on new f from here on */
+                    f = md->last_fragment;
+                } else {
+                    end_trace = true;
+                }
+            }
+
+            if (!end_trace &&
+                !get_and_check_add_size(dcontext, f, &add_size, &prev_mangle_size)) {
+                STATS_INC(num_max_trace_size_enforced);
+                end_trace = true;
+            }
+        }
+        if (DYNAMO_OPTION(max_trace_bbs) > 0 &&
+            md->num_blks >= DYNAMO_OPTION(max_trace_bbs) && !end_trace) {
+            end_trace = true;
+            STATS_INC(num_max_trace_bbs_enforced);
+        }
+        end_trace = (end_trace ||
+                     /* mangling may never use trace buffer memory but just in case */
+                     !make_room_in_trace_buffer(dcontext, add_size + prev_mangle_size, f));
+
+#ifdef CUSTOM_TRACES
+        if (end_trace && client == CUSTOM_TRACE_CONTINUE) {
+            /* had to overide client, log */
+            LOG(THREAD, LOG_MONITOR, 2, PRODUCT_NAME" ignoring Client's decision to continue trace (cannot trace through next fragment), ending trace now\n");
+        }
+#endif
+
+        if (end_trace) {
+            LOG(THREAD, LOG_MONITOR, 3,
+                "NOT extending hot trace (tag "PFX") with F%d ("PFX")\n",
+                md->trace_tag, f->id, f->tag);
+
+            f = end_and_emit_trace(dcontext, f);
+            LOG(THREAD, LOG_MONITOR, 3, "Returning to search mode\n");
+        } else {
+            LOG(THREAD, LOG_MONITOR, 3,
+                "Extending hot trace (tag "PFX") with F%d ("PFX")\n",
+                md->trace_tag, f->id, f->tag);
+            /* add_size is set when !end_trace */
+            f = internal_extend_trace(dcontext, f, dcontext->last_exit, add_size);
+        }
+        dcontext->whereami = WHERE_DISPATCH;
+        /* re-protect local heap */
+        SELF_PROTECT_LOCAL(dcontext, READONLY);
+        KSTOP(trace_building);
+        return f;
+    }
+
+    /* if got here, md->trace_tag == NULL */
+
+    /* searching for a hot trace head */
+
+    if (TEST(FRAG_IS_TRACE, f->flags)) {
+        /* nothing to do */
+        dcontext->whereami = WHERE_DISPATCH;
+        return f;
+    }
+
+    if (!TEST(FRAG_IS_TRACE_HEAD, f->flags)) {
+
+        bool trace_head;
+
+        /* Dynamic marking of trace heads for:
+         * - indirect exits
+         * - an exit from a trace that ends just before a SYSENTER.
+         * - private secondary trace heads targeted by shared traces
+         *
+         * FIXME Rework this to use the last exit's fragment flags & tag that were
+         * stored in a dcontext-private place such as non-shared monitor data.
+         */
+        if (LINKSTUB_INDIRECT(dcontext->last_exit->flags) ||
+            dcontext->trace_sysenter_exit ||
+            /* mark private secondary trace heads from shared traces */
+            (TESTALL(FRAG_SHARED|FRAG_IS_TRACE, dcontext->last_fragment->flags) &&
+             !TESTANY(FRAG_SHARED|FRAG_IS_TRACE, f->flags))) {
+
+            bool need_lock = NEED_SHARED_LOCK(dcontext->last_fragment->flags);
+            if (need_lock)
+                acquire_recursive_lock(&change_linking_lock);
+
+            /* The exit stub is fake if trace_sysenter_exit is true, but the
+             * path thru check_for_trace_head() accounts for that.
+             */
+            trace_head =
+                check_for_trace_head(dcontext, dcontext->last_fragment,
+                                     dcontext->last_exit, f,
+                                     need_lock, dcontext->trace_sysenter_exit);
+
+            if (need_lock)
+                release_recursive_lock(&change_linking_lock);
+
+            /* link routines will unprotect as necessary, we then re-protect
+             * entire fcache
+             */
+            SELF_PROTECT_CACHE(dcontext, NULL, READONLY);
+        }
+        else {
+            /* whether direct or fake, not marking a trace head */
+            trace_head = false;
+        }
+
+        if (!trace_head) {
+            dcontext->whereami = WHERE_DISPATCH;
+            return f;
+        }
+    }
+
+    /* Found a trace head, increment its counter */
+    ctr = thcounter_lookup(dcontext, f->tag);
+    /* May not have been added for this thread yet */
+    if (ctr == NULL)
+        ctr = thcounter_add(dcontext, f->tag);
+    ASSERT(ctr != NULL);
+
+    if (ctr->counter == TH_COUNTER_CREATED_TRACE_VALUE()) {
+        /* trace_t head counter values are persistent, so we do not remove them on
+         * deletion.  However, when a trace is deleted we clear the counter, to
+         * prevent the new bb from immediately being considered hot, to help
+         * with phased execution (trace may no longer be hot).  To avoid having
+         * walk every thread for every trace deleted we use a lazy strategy,
+         * recognizing a counter that has already reached the threshold with a
+         * sentinel value.
+         */
+        ctr->counter = INTERNAL_OPTION(trace_counter_on_delete);
+        STATS_INC(th_counter_reset);
+    }
+
+    ctr->counter++;
+    /* Should never be > here (assert is down below) but we check just in case */
+    if (ctr->counter >= INTERNAL_OPTION(trace_threshold)) {
+        /* if cannot delete fragment, do not start trace -- wait until
+         * can delete it (w/ exceptions, deletion status changes). */
+        if (!TEST(FRAG_CANNOT_DELETE, f->flags)) {
+            if (!DYNAMO_OPTION(shared_traces))
+                start_trace = true;
+            /* FIXME To detect a trace building race w/private BBs at this point,
+             * we need a presence table to mark that a tag is being used for trace
+             * building. Generic hashtables can help with this (case 6206).
+             */
+            else if (!DYNAMO_OPTION(shared_bbs) || !TEST(FRAG_SHARED,f->flags))
+                start_trace = true;
+            else {
+                /* Check if trace building is in progress and act accordingly. */
+                ASSERT(TEST(FRAG_SHARED, f->flags));
+                /* Hold the change linking lock for flags changes. */
+                acquire_recursive_lock(&change_linking_lock);
+                if (TEST(FRAG_TRACE_BUILDING, f->flags)) {
+                    /* trace_t building w/this tag is already in-progress. */
+                    STATS_INC(num_trace_building_race);
+                }
+                else {
+                    f->flags |= FRAG_TRACE_BUILDING;
+                    start_trace = true;
+                }
+                release_recursive_lock(&change_linking_lock);
+            }
+        }
+        if (!start_trace) {
+            /* Back up the counter by one. This ensures that the
+             * counter will be == trace_threshold if this thread is later
+             * able to start building a trace w/this tag and ensures
+             * that our one-up sentinel works for lazy clearing.
+             */
+            ctr->counter--;
+            ASSERT(ctr->counter < INTERNAL_OPTION(trace_threshold));
+        }
+    }
+
+#ifdef CLIENT_INTERFACE
+    if (start_trace) {
+        /* We need to set pass_to_client before cloning */
+        /* PR 299808: cache whether we need to re-build bbs for clients up front,
+         * to be consistent across whole trace.  If client later unregisters bb
+         * hook then it will miss our call on constituent bbs: that's its problem.
+         */
+        md->pass_to_client = dr_bb_hook_exists() || dr_trace_hook_exists();
+        /* should already be initialized */
+        ASSERT(instrlist_first(&md->unmangled_ilist) == NULL);
+    }
+#endif
+    if (start_trace &&
+        (TEST(FRAG_COARSE_GRAIN, f->flags) IF_CLIENT_INTERFACE(|| md->pass_to_client))) {
+        ASSERT(TEST(FRAG_IS_TRACE_HEAD, f->flags));
+        /* We need linkstub_t info for trace_exit_stub_size_diff() so we go
+         * ahead and make a private copy here.
+         */
+        if (create_private_copy(dcontext, f)) {
+            /* operate on new f from here on */
+            f = md->last_fragment;
+        } else {
+            start_trace = false;
+        }
+    }
+    if (start_trace) {
+        KSTART(trace_building);
+        /* ensure our sentinel counter value for counter clearing will work */
+        ASSERT(ctr->counter == INTERNAL_OPTION(trace_threshold));
+        ctr->counter = TH_COUNTER_CREATED_TRACE_VALUE();
+        /* Found a hot trace head.  Switch this thread into trace
+           selection mode, and initialize the instrlist_t for the new
+           trace fragment with this block fragment.  Leave the
+           trace head entry locked so no one else tries to build
+           a trace from it.  Assume that a trace would never 
+           contain just one block, and thus we don't have to check
+           for end of trace condition here. */
+        /* unprotect local heap */
+        SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+#ifdef TRACE_HEAD_CACHE_INCR
+        /* we don't have to worry about skipping the cache incr routine link
+         * in the future since we can only encounter the trace head in our
+         * no-link trace-building mode, then we will delete it
+         */
+#endif
+        md->trace_tag = f->tag;
+        md->trace_flags = trace_flags_from_trace_head_flags(f->flags);
+        md->emitted_size = fragment_prefix_size(md->trace_flags);
+#ifdef PROFILE_RDTSC
+        if (dynamo_options.profile_times)
+            md->emitted_size += profile_call_size();
+#endif
+        LOG(THREAD, LOG_MONITOR, 2,
+            "Found hot trace head F%d (tag "PFX")\n", f->id, f->tag);
+        LOG(THREAD, LOG_MONITOR, 3, "Entering trace selection mode\n");
+        /* allocate trace buffer space */
+#ifdef CUSTOM_TRACES
+        /* we should have a bb here, since if a trace can't also be a trace head */
+        ASSERT(!TEST(FRAG_IS_TRACE, f->flags));
+#endif
+        if (!get_and_check_add_size(dcontext, f, &add_size, &prev_mangle_size) ||
+            /* mangling may never use trace buffer memory but just in case */
+            !make_room_in_trace_buffer(dcontext, md->emitted_size + add_size +
+                                       prev_mangle_size, f)) {
+            LOG(THREAD, LOG_MONITOR, 1,
+                "bb %d ("PFX") too big (%d) %s\n",
+                f->id, f->tag, f->size,
+                get_and_check_add_size(dcontext, f, NULL, NULL) ?
+                "trace buffer" : "trace body limit / trace cache size");
+            /* turn back into a non-trace head */
+            SHARED_FLAGS_RECURSIVE_LOCK(f->flags, acquire, change_linking_lock);
+            f->flags &= ~FRAG_IS_TRACE_HEAD;
+            /* make sure not marked as trace head again */
+            f->flags |= FRAG_CANNOT_BE_TRACE;
+            STATS_INC(num_huge_fragments);
+            /* have to relink incoming frags */
+            link_fragment_incoming(dcontext, f, false/*not new*/);
+            /* call reset_trace_state while holding the lock since it
+             * may manipulate frag flags */
+            reset_trace_state(dcontext,
+                              false /* already own change_linking_lock */);
+            SHARED_FLAGS_RECURSIVE_LOCK(f->flags, release, change_linking_lock);
+            /* FIXME: set CANNOT_BE_TRACE when first create a too-big fragment?
+             * export the size expansion factors considered?
+             */
+            /* now return */
+            dcontext->whereami = WHERE_DISPATCH;
+            /* link unprotects on demand, we then re-protect all */
+            SELF_PROTECT_CACHE(dcontext, NULL, READONLY);
+            /* re-protect local heap */
+            SELF_PROTECT_LOCAL(dcontext, READONLY);
+            KSTOP(trace_building);
+            return f;
+        }
+        f = internal_extend_trace(dcontext, f, NULL, add_size);
+        
+#ifdef RETURN_STACK
+        if (TEST(FRAG_ENDS_WITH_RETURN, f->flags) != 0 &&
+            !TEST(FRAG_IS_TRACE, f->flags)) {
+            f = end_and_emit_trace(dcontext, f);
+            LOG(THREAD, LOG_MONITOR, 3, "Returning to search mode\n");
+        }
+#endif /* RETURN_STACK */
+        
+        /* re-protect local heap */
+        SELF_PROTECT_LOCAL(dcontext, READONLY);
+        KSTOP(trace_building);
+    } else {
+        /* Not yet hot */
+        KSWITCH(monitor_enter_thci);
+    }
+    
+    /* release rest of state */
+    dcontext->whereami = WHERE_DISPATCH;
+    return f;
+}
+
+
+/* This routine internally calls enter_couldbelinking, thus it is safe
+ * to call from any linking state. Restores linking to previous state at exit.
+ * If calling on another thread, caller should be synchronized with that thread
+ * (either via flushing synch or thread_synch methods) FIXME : verify all users
+ * on other threads are properly synchronized
+ */
+void
+trace_abort(dcontext_t *dcontext)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    instrlist_t *trace;
+    bool prevlinking = true;
+
+    if (md->trace_tag == NULL)
+        return; /* NOT in trace selection mode */
+
+    /* we're changing linking state -- and we're often called from
+     * non-could-be-linking locations, so we synch w/ flusher here.
+     * additionally we are changing trace state that the flusher
+     * reads, and we could have a race condition, so we consider
+     * that to be a linking change as well. If we are the flusher
+     * than the synch is unnecessary and could even cause a livelock.
+     */
+    if (!is_self_flushing()) {
+        if (!is_couldbelinking(dcontext)) {
+            prevlinking = false;
+            enter_couldbelinking(dcontext, NULL, 
+                                 false/*not a cache transition*/);
+        }
+    }
+
+    /* must relink unlinked trace-extending fragment 
+     * cannot use last_exit, must use our own last_fragment just for this
+     * purpose, b/c may not exit cache from last_fragment
+     * (e.g., if hit sigreturn!)
+     */
+    if (md->last_fragment != NULL) {
+        internal_restore_last(dcontext);
+    }
+
+    /* free the instrlist_t elements */
+    trace = &md->trace;
+    instrlist_clear(dcontext, trace);
+
+    if (md->trace_vmlist != NULL) {
+        vm_area_destroy_list(dcontext, md->trace_vmlist);     
+        md->trace_vmlist = NULL;
+    }
+    STATS_INC(num_aborted_traces);
+    STATS_ADD(num_bbs_in_all_aborted_traces, md->num_blks);
+    ASSERT_DO_NOT_OWN_MUTEX(true, &change_linking_lock);
+    reset_trace_state(dcontext, true /* might need change_linking_lock */);
+
+    if (!prevlinking)
+        enter_nolinking(dcontext, NULL, false/*not a cache transition*/);
+}
+
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+/* PR 204770: use trace component bb tag for RCT source address */
+app_pc
+get_trace_exit_component_tag(dcontext_t *dcontext, fragment_t *f, linkstub_t *l)
+{
+    linkstub_t *stub;
+    uint exitnum = 0;
+    uint i, num;
+    app_pc tag = f->tag;
+    bool found = false;
+    trace_only_t *t = TRACE_FIELDS(f);
+    ASSERT(TEST(FRAG_IS_TRACE, f->flags));
+    ASSERT(linkstub_fragment(dcontext, l) == f);
+    for (stub = FRAGMENT_EXIT_STUBS(f); stub != NULL; stub = LINKSTUB_NEXT_EXIT(stub)) {
+        if (stub == l) {
+            found = true;
+            break;
+        }
+        exitnum++;
+    }
+    ASSERT(found);
+    if (!found) {
+        LOG(THREAD, LOG_MONITOR, 2,
+            "get_trace_exit_component_tag F%d("PFX"): can't find exit!\n",
+            f->id, f->tag);
+        return f->tag;
+    }
+    ASSERT(exitnum < t->num_bbs);
+    /* If we have coarse bbs, or max_elide_* is 0, we won't elide during bb building
+     * but we will during trace building.  Rather than recreate each bb and figure
+     * out how many exits it contributed, we store that information.
+     */
+    found = false;
+    for (i = 0, num = 0; i < t->num_bbs; i++) {
+        if (exitnum < num + t->bbs[i].num_exits) {
+            found = true;
+            tag = t->bbs[i].tag;
+            break;
+        }
+        num += t->bbs[i].num_exits;
+    }
+    ASSERT(found);
+    LOG(THREAD, LOG_MONITOR, 4,
+        "get_trace_exit_component_tag F%d("PFX") => bb #%d (exit #%d): "PFX"\n",
+        f->id, f->tag, i, exitnum, tag);
+    return tag;
+}
+#endif /* defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH) */
diff --git a/core/monitor.h b/core/monitor.h
new file mode 100644
index 000000000..b8750c74f
--- /dev/null
+++ b/core/monitor.h
@@ -0,0 +1,186 @@
+/* **********************************************************
+ * Copyright (c) 2000-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/* file "monitor.h"
+ */
+
+#ifndef _MONITOR_H_
+#define _MONITOR_H_ 1
+
+#include "fragment.h" /* for trace_bb_info_t, and for CUSTOM_TRACES
+                       * the "fragment_t wrapper" struct */
+
+/* synchronization of shared traces */
+extern mutex_t trace_building_lock;
+
+void monitor_init(void);
+void monitor_exit(void);
+void monitor_thread_init(dcontext_t *dcontext);
+void monitor_thread_exit(dcontext_t *dcontext);
+
+/* re-initializes non-persistent memory */
+void monitor_thread_reset_init(dcontext_t *dcontext);
+/* frees all non-persistent memory */
+void monitor_thread_reset_free(dcontext_t *dcontext);
+
+void monitor_remove_fragment(dcontext_t *dcontext, fragment_t *f);
+bool monitor_is_linkable(dcontext_t *dcontext, fragment_t *from_f,
+                         linkstub_t *from_l, fragment_t *to_f,
+                         bool have_link_lock, bool mark_new_trace_head);
+void mark_trace_head(dcontext_t *dcontext, fragment_t *f, fragment_t *src_f,
+                     linkstub_t *src_l);
+
+enum {
+    TRACE_HEAD_YES           = 0x01,
+    TRACE_HEAD_OBTAINED_LOCK = 0x02,
+};
+
+/* Returns TRACE_HEAD_* flags indicating whether to_tag should be
+ * a trace head.
+ * For -shared_bbs, will return TRACE_HEAD_OBTAINED_LOCK if the
+ * change_linking_lock is not already held (meaning from_l->fragment is
+ * private) and the to_tag is FRAG_SHARED and TRACE_HEAD_YES is being returned,
+ * since the change_linking_lock must be held and the TRACE_HEAD_YES result 
+ * re-verified.  In that case the caller must free the change_linking_lock.
+ */
+uint should_be_trace_head(dcontext_t *dcontext, fragment_t *from_f, linkstub_t *from_l,
+                          app_pc to_tag, uint to_flags, bool have_link_lock);
+
+fragment_t *monitor_cache_enter(dcontext_t *dcontext, fragment_t *f);
+void monitor_cache_exit(dcontext_t *dcontext);
+bool is_building_trace(dcontext_t *dcontext);
+app_pc cur_trace_tag(dcontext_t *dcontext);
+void * cur_trace_vmlist(dcontext_t *dcontext);
+
+/* This routine internally calls enter_couldbelinking, thus it is safe
+ * to call from any linking state. Restores linking to previous state at exit.
+ * If calling on another thread, caller should be synchornized with that thread
+ * (either via flushing synch or thread_synch methods) FIXME : verify all users
+ * on other threads are properly synchronized
+ */
+void trace_abort(dcontext_t *dcontext);
+
+void
+thcounter_range_remove(dcontext_t *dcontext, app_pc start, app_pc end);
+
+/* trace head counters are thread-private and must be kept in a
+ * separate table and not in the fragment_t structure.
+ * FIXME: may want to do this for non-shared-cache, since persistent counters
+ * may mitigate the performance hit of a small bb cache -- but for that
+ * we could keep the counters in future_fragment_t when a bb dies and
+ * re-initialize to that value when it comes back.
+ */
+typedef struct _trace_head_counter_t {
+    app_pc tag;
+    uint   counter;
+    /* FIXME: use open-address to save memory, and share code
+     * w/ fragment.c?
+     */
+    struct _trace_head_counter_t *next;
+} trace_head_counter_t;
+
+typedef struct _trace_head_table_t {
+    trace_head_counter_t **counter_table;
+    uint  hash_bits;
+    ptr_uint_t  hash_mask;
+    uint  hash_mask_offset;
+    hash_function_t hash_func;
+    uint  capacity;           /* = 2^bits */
+    uint  entries;
+    uint  load_factor_percent; /* \alpha = load_factor_percent/100 */
+    uint  resize_threshold;    /*  = capacity * load_factor */
+} trace_head_table_t;
+
+typedef struct _trace_bb_build_t {
+    trace_bb_info_t info;
+    /* PR 299808: we need to check bb bounds at emit time.  Also used
+     * for trace state translation.
+     */
+    union {
+        app_pc end_pc;
+        instr_t *end_instr;
+    } bounds;
+} trace_bb_build_t;
+
+typedef struct _monitor_data_t {
+    /* Fields used by monitor, but also by arch-specific code, so it's
+     * exported in the header file.
+     * Needs to be in separate struct to share across callbacks.
+     */
+    app_pc         trace_tag;       /* tag of trace head */
+    uint           trace_flags;     /* FRAGMENT_ flags for trace */
+    instrlist_t      trace;           /* place to build the instruction trace */
+    byte           *trace_buf;      /* place to temporarily store instr bytes */
+    uint           trace_buf_size;  /* length of trace_buf in bytes */
+    uint           trace_buf_top;   /* index of next free location in trace_buf */
+    void *         trace_vmlist;    /* list of vmareas trace touches */
+    uint           num_blks;        /* count of the number of blocks in trace */
+    trace_bb_build_t *blk_info;     /* info for all basic blocks making up trace */
+    uint           blk_info_length; /* length of blk_info array */
+    uint           emitted_size;    /* calculated final trace size once emitted */
+    fragment_t *     last_copy;     /* private copy of shared bb for trace building only 
+                                     * equals the previous last_fragment that was shared
+                                     */
+    fragment_t *     last_fragment;   /* for restoring (can't just use last_exit) */
+    uint           last_fragment_flags; /* for restoring */
+
+    /* trace head counters are thread-private and must be kept in a
+     * separate table and not in the fragment_t structure.
+     */
+    trace_head_table_t thead_table;
+
+#ifdef CLIENT_INTERFACE
+    /* PR 299808: we re-build each bb and pass to the client */
+    instrlist_t    unmangled_ilist;
+    instrlist_t    *unmangled_bb_ilist; /* next bb */
+    /* cache at start of trace building whether we're going to pass to client */
+    bool           pass_to_client;
+#endif
+    /* Record whether final block ends in syscall or int.
+     * FIXME: remove once we have PR 307284.
+     */
+    uint           final_exit_flags;
+
+#ifdef CUSTOM_TRACES
+    fragment_t     wrapper; /* for creating new shadowed trace heads */
+#endif
+} monitor_data_t;
+
+/* PR 204770: use trace component bb tag for RCT source address */
+app_pc
+get_trace_exit_component_tag(dcontext_t *dcontext, fragment_t *f, linkstub_t *l);
+
+#endif /* _MONITOR_H_ */
diff --git a/core/nudge.c b/core/nudge.c
new file mode 100644
index 000000000..b07808b1e
--- /dev/null
+++ b/core/nudge.c
@@ -0,0 +1,387 @@
+/* **********************************************************
+ * Copyright (c) 2008-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifdef WINDOWS /* Nudge only supported on Windows for now. */
+
+#include "globals.h"
+#include "nudge.h"
+#include "ntdll.h" /* for create_thread(), nt_free_virtual_memory() */
+#include "os_exports.h" /* for detach_helper(), get_stack_bounds() */
+#ifdef HOT_PATCHING_INTERFACE
+# include "hotpatch.h" /* for hotp_nudge_update() */
+#endif
+#ifdef PROCESS_CONTROL
+# include "moduledb.h" /* for process_control() */
+#endif
+#include "fragment.h" /* needed for perscache.h */
+#include "perscache.h" /* for coarse_units_freeze_all() */
+#ifdef CLIENT_INTERFACE
+# include "instrument.h" /* for instrement_nudge() */
+#endif
+#include "fcache.h" /* for reset routines */
+
+static void
+nudge_terminate_on_dstack(dcontext_t *dcontext)
+{
+    ASSERT(dcontext == get_thread_private_dcontext());
+    os_terminate(dcontext, TERMINATE_THREAD|TERMINATE_CLEANUP);
+    ASSERT_NOT_REACHED();
+}
+
+/* This is the target for all nudge threads.
+ * CAUTION: generic_nudge_target is added to global_rct_ind_targets table.  If this
+ * function is renamed or cloned, update rct_known_targets_init accordingly.
+ */
+void 
+generic_nudge_target(nudge_arg_t *arg)
+{
+    /* Fix for case 5130; volatile forces a 'call' instruction to be generated
+     * rather than 'jmp' during optimization.  FIXME: need a standardized & 
+     * better way of stopping core from emulating itself.
+     */
+    volatile bool nudge_result;
+
+    /* needed to make sure dr has a specific target to lookup and avoid 
+     * interpreting when taking over new threads; see must_not_be_inlined().
+     */
+    nudge_result = generic_nudge_handler(arg);
+
+    /* Should never return. */
+    ASSERT_NOT_REACHED();
+    os_terminate(NULL, TERMINATE_THREAD); /* just in case */
+}
+
+
+/* This is the actual nudge handler
+ * Notes: This function returns a boolean mainly to fix case 5130; it is not
+ *        really necessary.
+ */
+bool 
+generic_nudge_handler(nudge_arg_t *arg_dont_use)
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    nudge_arg_t safe_arg = {0};
+    uint nudge_action_mask = 0;
+
+    /* To be extra safe we use safe_read() to access the nudge argument, though once
+     * we get past the checks below we are trusting its content. */
+    ASSERT(arg_dont_use != NULL && "invalid nudge argument");
+    if (!safe_read(arg_dont_use, sizeof(nudge_arg_t), &safe_arg)) {
+        ASSERT(false && "invalid nudge argument");
+        goto nudge_finished;
+    }
+    nudge_action_mask = safe_arg.nudge_action_mask;
+
+    /* Future version checks would go here. */
+    ASSERT_CURIOSITY(safe_arg.version == NUDGE_ARG_CURRENT_VERSION);
+
+    /* FIXME - would be nice to inform nudge creator if we need to nop the nudge. */
+
+    /* Fix for case 5702.  If a nudge thread comes in during process exit, 
+     * don't process it, i.e., nop it. FIXME - this leaks the app stack and nudge arg
+     * if the nudge was supposed to free them. */
+    if (dynamo_exited)
+        goto nudge_finished;
+
+    /* Node manager will not be able to nudge before reading the drmarker and
+     * the dr_marker isn't available before callback_interception_init().
+     * Since after callback_interception_init() new threads won't be allowed
+     * to progress till dynamo_initialized is set, by the time a nudge thread
+     * reaches here dynamo_initialized should be set. */
+    ASSERT(dynamo_initialized);
+    if (!dynamo_initialized)
+        goto nudge_finished;
+
+    /* We should always have a dcontext. */
+    ASSERT(dcontext != NULL);
+    if (dcontext == NULL)
+        goto nudge_finished;
+
+    ENTERING_DR();
+
+    /* Nudge shouldn't start with any locks held.  Do this assert after the 
+     * dynamo_exited check, other wise the locks may be deleted. */
+    ASSERT_OWN_NO_LOCKS();
+
+    /* Xref case 552, the nudge_target value provides a reasonable measure
+     * of security against an attacker leveraging this routine. */
+    if (dcontext->nudge_target != (void *)generic_nudge_target) {
+        /* FIXME - should we report this likely attempt to attack us? need
+         * a unit test for this (though will then have to tone this down). */
+        ASSERT(false && "unauthorized thread tried to nudge");
+        /* If we really are under attack we should terminate immediately and
+         * proceed no further. Note we are leaking the app stack and nudge arg if we
+         * were supposed to free them. */
+        os_terminate(dcontext, TERMINATE_THREAD);
+        ASSERT_NOT_REACHED();
+    }
+
+    /* Free the arg if requested. */
+    if (TEST(NUDGE_FREE_ARG, safe_arg.flags)) {
+        ASSERT_NOT_TESTED();
+        nt_free_virtual_memory(arg_dont_use);
+    }
+
+    if (nudge_action_mask == 0) {
+        ASSERT_CURIOSITY(false && "Nudge: no action specified");
+        goto nudge_finished;
+    } else if (nudge_action_mask >= NUDGE_GENERIC(PARAMETRIZED_END)) {
+        ASSERT(false && "Nudge: unknown nudge action");
+        goto nudge_finished;
+    }
+
+    /* In -thin_client mode only detach and process_control nudges are allowed;
+     * case 8888. */
+#define VALID_THIN_CLIENT_NUDGES (NUDGE_GENERIC(process_control)|NUDGE_GENERIC(detach))
+    if (DYNAMO_OPTION(thin_client)) {
+        if (TEST(VALID_THIN_CLIENT_NUDGES, nudge_action_mask)) {
+             /* If it is a valid thin client nudge, then disable all others. */
+             nudge_action_mask &= VALID_THIN_CLIENT_NUDGES;
+        } else {
+            goto nudge_finished;   /* invalid nudge for thin_client, so mute it */
+        }
+    }
+
+    /* FIXME: NYI action handlers. As implemented move to desired order. */
+    if (TEST(NUDGE_GENERIC(upgrade), nudge_action_mask)) {
+        /* FIXME: watch out for flushed clean-call fragment */
+        nudge_action_mask &= ~NUDGE_GENERIC(upgrade);
+        ASSERT_NOT_IMPLEMENTED(false && "case 4179");
+    }
+    if (TEST(NUDGE_GENERIC(kstats), nudge_action_mask)) {
+        nudge_action_mask &= ~NUDGE_GENERIC(kstats);
+        ASSERT_NOT_IMPLEMENTED(false);
+    }
+#ifdef INTERNAL
+    if (TEST(NUDGE_GENERIC(stats), nudge_action_mask)) {
+        nudge_action_mask &= ~NUDGE_GENERIC(stats);
+        ASSERT_NOT_IMPLEMENTED(false);
+    }
+    if (TEST(NUDGE_GENERIC(invalidate), nudge_action_mask)) {
+        /* FIXME: watch out for flushed clean-call fragment  */
+        nudge_action_mask &= ~NUDGE_GENERIC(invalidate);
+        ASSERT_NOT_IMPLEMENTED(false);
+    }
+    if (TEST(NUDGE_GENERIC(recreate_pc), nudge_action_mask)) {
+        nudge_action_mask &= ~NUDGE_GENERIC(recreate_pc);
+        ASSERT_NOT_IMPLEMENTED(false);
+    }
+    if (TEST(NUDGE_GENERIC(recreate_state), nudge_action_mask)) {
+        nudge_action_mask &= ~NUDGE_GENERIC(recreate_state);
+        ASSERT_NOT_IMPLEMENTED(false);
+    }
+    if (TEST(NUDGE_GENERIC(reattach), nudge_action_mask)) {
+        /* FIXME: watch out for flushed clean-call fragment */
+        nudge_action_mask &= ~NUDGE_GENERIC(reattach);
+        ASSERT_NOT_IMPLEMENTED(false);
+    }
+#endif /* INTERNAL */
+    if (TEST(NUDGE_GENERIC(diagnose), nudge_action_mask)) {
+        nudge_action_mask &= ~NUDGE_GENERIC(diagnose);
+        ASSERT_NOT_IMPLEMENTED(false);
+    }
+
+    /* Implemented action handlers */
+    if (TEST(NUDGE_GENERIC(opt), nudge_action_mask)) {
+        nudge_action_mask &= ~NUDGE_GENERIC(opt);
+        synchronize_dynamic_options();
+    }
+    if (TEST(NUDGE_GENERIC(ldmp), nudge_action_mask)) {
+        nudge_action_mask &= ~NUDGE_GENERIC(ldmp);
+        os_dump_core("Nudge triggered ldmp.");
+    }
+    if (TEST(NUDGE_GENERIC(freeze), nudge_action_mask)) {
+        nudge_action_mask &= ~NUDGE_GENERIC(freeze);
+        coarse_units_freeze_all(true/*in-place: FIXME: separate nudge for non?*/);
+    }
+    if (TEST(NUDGE_GENERIC(persist), nudge_action_mask)) {
+        nudge_action_mask &= ~NUDGE_GENERIC(persist);
+        coarse_units_freeze_all(false/*!in-place==persist*/);
+    }
+#ifdef CLIENT_INTERFACE
+    if (TEST(NUDGE_GENERIC(client), nudge_action_mask)) {
+        nudge_action_mask &= ~NUDGE_GENERIC(client);
+        instrument_nudge(dcontext, safe_arg.client_id, safe_arg.client_arg);
+    }
+#endif
+#ifdef PROCESS_CONTROL
+    if (TEST(NUDGE_GENERIC(process_control), nudge_action_mask)) {  /* Case 8594 */
+        nudge_action_mask &= ~NUDGE_GENERIC(process_control);
+        /* Need to synchronize because process control can be switched between
+         * on (white or black list) & off.  FIXME - the nudge mask should specify this,
+         * but doesn't hurt to do it again. */
+        synchronize_dynamic_options();
+        if (IS_PROCESS_CONTROL_ON()) 
+            process_control();
+        
+        /* If process control is enforced then control won't come back.  If
+         * either -detect_mode is on or if there was nothing to enforce, control
+         * comes back in which case it is safe to let remaining nudges be
+         * processed because no core state would have been changed. */
+    }
+#endif
+#ifdef HOTPATCHING
+    if (DYNAMO_OPTION(hot_patching) && DYNAMO_OPTION(liveshields) &&
+        TEST_ANY(NUDGE_GENERIC(policy)|NUDGE_GENERIC(mode)|NUDGE_GENERIC(lstats),
+                 nudge_action_mask)) {
+        hotp_nudge_update(nudge_action_mask &
+                          (NUDGE_GENERIC(policy)|NUDGE_GENERIC(mode)|NUDGE_GENERIC(lstats)));
+        nudge_action_mask &= ~(NUDGE_GENERIC(policy)|NUDGE_GENERIC(mode)|NUDGE_GENERIC(lstats));
+    }
+#endif
+#ifdef PROGRAM_SHEPHERDING
+    if (TEST(NUDGE_GENERIC(violation), nudge_action_mask)) {
+        nudge_action_mask &= ~NUDGE_GENERIC(violation);
+        /* Use nudge mechanism to trigger a security violation at an
+         * arbitrary time. Note - is only useful for testing kill process attack 
+         * handling as this is not an app thread (we injected it). */
+        /* see bug 652 for planned improvements */
+        security_violation(dcontext, dcontext->next_tag, 
+                           ATTACK_SIM_NUDGE_VIOLATION, OPTION_BLOCK|OPTION_REPORT);
+    }
+#endif
+    if (TEST(NUDGE_GENERIC(reset), nudge_action_mask)) {
+        nudge_action_mask &= ~NUDGE_GENERIC(reset);
+        if (DYNAMO_OPTION(enable_reset)) {
+            mutex_lock(&reset_pending_lock);
+            /* fcache_reset_all_caches_proactively() will unlock */
+            fcache_reset_all_caches_proactively(RESET_ALL);
+            /* NOTE - reset is safe since we won't return to the code cache below (we
+             * will in fact not return at all). */
+        } else {
+            SYSLOG_INTERNAL_WARNING("nudge reset ignored since resets are disabled");
+        }
+    }
+    
+    /* The detach handler is last since in the common case it doesn't return. */
+    if (TEST(NUDGE_GENERIC(detach), nudge_action_mask)) {
+        nudge_action_mask &= ~NUDGE_GENERIC(detach);
+        detach_helper(DETACH_NORMAL_TYPE);
+    }
+
+ nudge_finished:
+
+    /* Note - for supporting detach with CLIENT_INTERFACE and nudge threads we need that
+     * no lock grabbing or other actions that would interfere with the detaching process
+     * occur in the cleanup path here. */
+
+    /* Case 8901: this routine is currently called from the code cache, which may have
+     * been reset underneath us, so we can't just blindly return.  This also gives us
+     * consistent behavior for handling stack freeing. */
+
+    /* Case 9020: no EXITING_DR() as os_terminate will do that for us */
+
+    /* FIXME - these nudge threads do hit dll mains for thread attach so app may have
+     * allocated some TLS memory which won't end up being freed since this won't go
+     * through dll main thread detach. The app may also object to unbalanced attach to
+     * detach ratio though we haven't seen that in practice. Long term we should take
+     * over and redirect the thread at the init apc so it doesn't go through the
+     * DllMains to start with. */
+
+    /* We have a general problem on how to free the application stack for nudges.
+     * Currently the app/os will never free a nudge thread's app stack:
+     *  On NT and 2k ExitThread would normally free the app stack, but we always
+     *  terminate nudge threads instead of allowing them to return and exit normally.
+     *  On XP and 2k3 none of our nudge creation routines inform csrss of the new thread
+     *  (which is who typically frees the stacks).
+     *  On Vista we don't use NtCreateThreadEx to create the nudge threads so the kernel
+     *  doesn't free the stack.
+     * As such we are left with two options: free the app stack here (nudgee free) or
+     * have the nudge thread creator free the app stack (nudger free).  Going with 
+     * nudgee free means we leak exit race nudge stacks whereas if we go with nudger free
+     * for external nudges then we'll leak timed out nudge stacks (for internal nudges
+     * we pretty much have to do nudgee free).  A nudge_arg_t flag is used to specify
+     * which model we use, but currently we always nudgee free.
+     *
+     * dynamo_thread_exit_common() is where the app stack is actually freed, not here.
+     */
+
+    if (dynamo_exited || !dynamo_initialized || dcontext == NULL) {
+        /* FIXME - no cleanup so we'll leak any memory allocated for this thread
+         * including the application's stack and arg if we were supposed to free them.
+         * We only expect to get here in rare races where the nudge thread was created
+         * before dr exited (i.e. before drmarker was freed) but didn't end up getting
+         * scheduled till after dr exited. */
+        os_terminate(dcontext, TERMINATE_THREAD);
+    } else {
+        /* Nudge threads should exit without holding any locks. */
+        ASSERT_OWN_NO_LOCKS();
+
+        /* if needed tell thread exit to free the application stack */
+        if (!TEST(NUDGE_NUDGER_FREE_STACK, safe_arg.flags)) {
+            dcontext->free_app_stack = true;
+        } else {
+            ASSERT_NOT_TESTED();
+        }
+
+        /* if freeing the app stack we must be on the dstack when we cleanup */
+        if (dcontext->free_app_stack && !is_currently_on_dstack(dcontext)) {
+            call_switch_stack(dcontext, dcontext->dstack, nudge_terminate_on_dstack,
+                              false /* not on initstack */, false /* don't return */);
+        } else {
+            /* Already on dstack or nudge creator will free app stack. */
+            os_terminate(dcontext, TERMINATE_THREAD|TERMINATE_CLEANUP);
+        }
+    }
+    ASSERT_NOT_REACHED(); /* we should never return */
+    return true;
+}
+
+/* nudge the current process */
+bool
+nudge_internal(uint nudge_action_mask, uint64 client_arg, client_id_t client_id)
+{
+    HANDLE hthread;
+    bool res;
+    nudge_arg_t nudge_arg;
+
+    nudge_arg.version = NUDGE_ARG_CURRENT_VERSION;
+    nudge_arg.nudge_action_mask = nudge_action_mask;
+    nudge_arg.flags = NUDGE_IS_INTERNAL;
+    nudge_arg.client_arg = client_arg;
+    nudge_arg.client_id = client_id;
+
+    LOG(GLOBAL, LOG_ALL, 1, "Creating internal nudge thread with action_mask 0x%08x\n",
+        nudge_action_mask);
+    hthread = create_thread(NT_CURRENT_PROCESS, IF_X64_ELSE(true, false),
+                            (void *)generic_nudge_target,
+                            NULL, &nudge_arg, sizeof(nudge_arg_t),
+                            15*PAGE_SIZE, 12*PAGE_SIZE, false, NULL);
+    ASSERT(hthread != INVALID_HANDLE_VALUE);
+    res = close_handle(hthread);
+    ASSERT(res);
+    LOG(GLOBAL, LOG_ALL, 1, "Finished creating internal nudge thread\n");
+
+    return (hthread != INVALID_HANDLE_VALUE);
+}
+
+#endif /* WINDOWS */
diff --git a/core/nudge.h b/core/nudge.h
new file mode 100644
index 000000000..171bdb18f
--- /dev/null
+++ b/core/nudge.h
@@ -0,0 +1,49 @@
+/* **********************************************************
+ * Copyright (c) 2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifndef _NUDGE_H_
+#define _NUDGE_H_
+
+#ifdef WINDOWS /* Nudge only supported on windows currently */
+
+/* Tiggers a nudge targeting this process.  nudge_action_mask should be drawn from the
+ * NUDGE_GENERIC(***) values.  client_id is only relevant for client nudges. */
+bool nudge_internal(uint nudge_action_mask, uint64 client_arg, client_id_t client_id);
+
+/* The following are exported only so other routines can check their addresses for
+ * nudge threads.  They are not meant to be called internally. */
+void generic_nudge_target(nudge_arg_t *arg);
+bool generic_nudge_handler(nudge_arg_t *arg);
+
+#endif
+
+#endif /* _NUDGE_H_ */
diff --git a/core/options.c b/core/options.c
new file mode 100644
index 000000000..7f09e0c7e
--- /dev/null
+++ b/core/options.c
@@ -0,0 +1,2116 @@
+/* **********************************************************
+ * Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+
+/*
+ * options.c
+ * 
+ * Options definition and handling of corresponding command line options
+ *
+ */
+
+#ifndef NOT_DYNAMORIO_CORE
+#  include "globals.h"
+#  include "fcache.h"
+#  include "monitor.h"
+#  include "defines.h"
+#  include "moduledb.h"     /* for the process control defines */
+#  include <string.h>
+
+#else  /* NOT_DYNAMORIO_CORE */
+#  include <stdio.h>            /* snprintf, sscanf */
+
+# ifdef WINDOWS
+#  define inline __inline
+#  define snprintf _snprintf
+#  define WIN32_LEAN_AND_MEAN
+#  include <windows.h> /* no longer included in globals_shared.h */
+# endif
+
+#include "globals_shared.h"
+
+typedef unsigned int uint;
+
+# define print_file(f,s,x)
+
+# ifndef bool
+typedef int bool;
+# endif
+# ifndef true
+#  define true  (1)
+#  define false (0)
+# endif
+# ifndef NULL
+#  define NULL  (0)
+# endif
+
+ struct stats_type {
+     int logmask;
+     int loglevel;
+ } thestats;
+
+ struct stats_type *stats = &thestats;
+
+/* define away core only features, depending on use outside the core
+ * may want to use some of these, NOTE build environments outside of the
+ * core can't handle VARARG macros! */
+# define ASSERT(x)
+/* right now all OPTION_PARSE_ERROR's have 6 arguments 
+ * once libutil, etc. are all using vc8 we can use ...
+ */
+# define OPTION_PARSE_ERROR(a, b, c, d, e, f) 
+
+static void ignore_varargs_function(char *format, ...) { }
+/* right now all FATAL_USAGE_ERROR's have 4 arguments */
+# define FATAL_USAGE_ERROR(a, b, c, d)
+/* USAGE_ERROR unfortunately needs a quick fix now */
+# define USAGE_ERROR 1 ? (void)0 : ignore_varargs_function
+# define DOLOG(a, b, c)
+
+#include "options.h"
+
+#endif /* NOT_DYNAMORIO_CORE */
+
+
+#ifndef EXPOSE_INTERNAL_OPTIONS
+/* default values for internal options are kept in a separate struct */
+#  define OPTION_COMMAND_INTERNAL(type, name, default_value, command_line_option, statement, description, flag, pcache) default_value, 
+#  define OPTION_COMMAND(type, name, default_value, command_line_option, \
+                         statement, description, flag, pcache) /* nothing */
+/* read only source for default internal option values and names
+ * no lock needed since never written
+ */
+const internal_options_t default_internal_options = {
+#  include "optionsx.h"
+};
+#undef OPTION_COMMAND_INTERNAL
+#undef OPTION_COMMAND
+#endif
+
+
+#ifdef EXPOSE_INTERNAL_OPTIONS
+#  define OPTION_COMMAND_INTERNAL OPTION_COMMAND
+#else 
+/* DON'T FIXME: In order to support easy switching of an internal
+   option into user accessible one we could waste some memory by
+   allocating fields in options_t even for INTERNAL options.  That would
+   let us have INTERNAL_OPTION be a more flexible macro that can use
+   either the constant value, or the dynamically set variable
+   depending on the option declaration in optionsx.h.  However, 
+   it increases the code size of this file (there is also a minor increase
+   in other object files since more option fields need longer than 8-bit offsets)
+   For now we can live without this.
+*/
+#  define OPTION_COMMAND_INTERNAL(type, name, default_value, command_line_option, statement, description, flag, pcache) /* nothing, */
+#endif
+
+
+/* all to default values */
+#define OPTION_COMMAND(type, name, default_value, command_line_option, \
+                       statement, description, flag, pcache) default_value,
+/* read only source for default option values and names
+ * no lock needed since never written
+ */
+const options_t default_options = {
+#  include "optionsx.h"
+};
+
+#ifndef NOT_DYNAMORIO_CORE /*****************************************/
+
+# define SELF_PROTECT_OPTIONS() SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT)
+# define SELF_UNPROTECT_OPTIONS() SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT)
+/* WARNING: testing positive direction is racy (other threads unprot .data
+ * for brief windows), negative is reliable
+ */
+# define OPTIONS_PROTECTED() (DATASEC_PROTECTED(DATASEC_RARELY_PROT))
+
+/* Holds a copy of the last read option string from the registry, NOT a
+ * canonical option string.
+ */
+char option_string[MAX_OPTIONS_STRING] = {0,};
+# define ASSERT_OWN_OPTIONS_LOCK(b, l) ASSERT_OWN_WRITE_LOCK(b, l)
+# define CORE_STATIC static
+/* official options */
+options_t dynamo_options = {
+# include "optionsx.h"
+};
+/* Temporary string, static to save stack space in synchronize_dynamic_options().
+ * FIXME case 8074: should protect this better as w/o DR dll randomization attacker
+ * can repeatedly try to clobber.  Move to heap?  Or shrink stack space elsewhere
+ * and put back as synchronize_dynamic_options() local var.
+ */
+DECLARE_FREQPROT_VAR(char new_option_string[MAX_OPTIONS_STRING], {0,});
+/* temporary structure */
+options_t temp_options = {
+# include "optionsx.h"
+};
+
+/* dynamo_options and temp_options, the two writable option structures,
+ * are both protected by this lock, which is kept outside of the protected 
+ * section to ease bootstrapping issues
+ */
+DECLARE_CXTSWPROT_VAR(read_write_lock_t options_lock, INIT_READWRITE_LOCK(options_lock));
+
+#else /* !NOT_DYNAMORIO_CORE ****************************************/
+# define ASSERT_OWN_OPTIONS_LOCK(b, l)
+# define CORE_STATIC
+#endif /* !NOT_DYNAMORIO_CORE ****************************************/
+
+#undef OPTION_COMMAND
+
+
+/* INITIALIZATION */
+
+/* sets defaults just like the above initialization */
+CORE_STATIC void
+set_dynamo_options_defaults(options_t *options)
+{
+    ASSERT_OWN_OPTIONS_LOCK(options==&dynamo_options || options==&temp_options, &options_lock);
+    *options = default_options;
+}
+#undef OPTION_COMMAND_INTERNAL
+
+/* For all other purposes OPTION_COMMAND_INTERNAL should be equivalent to either OPTION_COMMAND or nothing */
+#ifdef EXPOSE_INTERNAL_OPTIONS
+#  define OPTION_COMMAND_INTERNAL OPTION_COMMAND
+#else 
+#  define OPTION_COMMAND_INTERNAL(type, name, default_value, command_line_option, statement, description, flag, pcache) /* nothing */
+#endif
+
+/* PARSING HANDLER */
+/* returns the space delimited or quote-delimited word
+ * starting at strpos in the string str, or NULL if none
+ * wordbuf is a caller allocated buffer of size wordbuflen that will hold 
+     the copied word from the original string, since it assumes it cannot modify it
+ */
+static char*
+getword(const char *str, const char **strpos, char *wordbuf, uint wordbuflen)
+{
+    uint i = 0;
+    const char *pos = *strpos;
+    char quote = '\0';
+
+    if (pos < str || pos >= str + strlen(str))
+        return NULL;
+
+    if (*pos == '\0')
+        return NULL; /* no more words */
+
+    /* eat leading spaces */
+    while (*pos == ' ' || *pos == '\t') {
+        pos++;
+    }
+
+    /* extract the word */
+    if (*pos == '\'' || *pos == '\"' || *pos == '`') {
+        quote = *pos;
+        pos++; /* don't include surrounding quotes in word */
+    }
+    while (*pos) {
+        if (quote != '\0') {
+            /* if quoted, only end on matching quote */
+            if (*pos == quote) {
+                pos++; /* include the quote */
+                break;
+            }
+        } else {
+            /* if not quoted, end on whitespace */
+            if (*pos == ' ' || *pos == '\t' || 
+                *pos == '\n' || *pos == '\r')
+                break;
+        }
+        if (i<wordbuflen-1) {
+            wordbuf[i++] = *pos;
+            pos++;
+        } else {
+            OPTION_PARSE_ERROR(ERROR_OPTION_TOO_LONG_TO_PARSE, 4, 
+                               get_application_name(), get_application_pid(), 
+                               strpos, IF_DEBUG_ELSE("Terminating", "Continuing"));
+            /* just return truncated form */
+            break;
+        }
+    }
+    if (i == 0)
+        return NULL; /* no more words */
+
+    ASSERT(i<wordbuflen);
+    wordbuf[i] = '\0';
+    *strpos = pos;
+
+    return wordbuf;
+}
+
+#define ISBOOL_bool 1
+#define ISBOOL_uint 0
+#define ISBOOL_uint_size 0
+#define ISBOOL_uint_time 0
+#define ISBOOL_uint_addr 0
+#define ISBOOL_pathstring_t 0
+#define ISBOOL_liststring_t 0
+
+static void
+parse_bool(bool *var, void *value)
+{ 
+    *var = *(bool*)value;
+}
+
+static void
+parse_uint(uint *var, void *value)
+{
+    uint num;
+    char *opt = (char*)value;
+
+    if ((sscanf(opt, "0x%x", &num) == 1) ||
+        (sscanf(opt, "%d", &num) == 1)) { /* atoi(opt) */
+        *var = num;
+    } else {
+        /* var should be pre-initialized to default */
+        OPTION_PARSE_ERROR(ERROR_OPTION_BAD_NUMBER_FORMAT, 4, 
+                           get_application_name(), get_application_pid(), opt,
+                           IF_DEBUG_ELSE("Terminating", "Continuing"));
+    }
+}
+
+static void
+parse_uint_size(uint *var, void *value)
+{ 
+    char unit;
+    int num;
+    int factor = 0;
+
+    if (sscanf((char*)value, "%d%c", &num, &unit) == 1) {
+        /* no unit specifier, default unit is Kilo for compatibility */
+        factor = 1024;
+    } else {
+        switch (unit) {
+        case 'B': // plain bytes
+        case 'b': factor = 1; break;
+        case 'K': // Kilo (bytes)
+        case 'k': factor = 1024; break;
+        case 'M': // Mega (bytes)
+        case 'm': factor = 1024*1024; break;
+        default: 
+            /* var should be pre-initialized to default */
+            OPTION_PARSE_ERROR(ERROR_OPTION_UNKNOWN_SIZE_SPECIFIER, 4, 
+                               get_application_name(), get_application_pid(),
+                               (char *)value, IF_DEBUG_ELSE("Terminating", "Continuing"));
+            return;
+        }
+    }
+    *var = num * factor;
+}
+
+static void
+parse_uint_time(uint *var, void *value)
+{ 
+    char unit;
+    int num;
+    int factor = 0;
+
+    if (sscanf((char*)value, "%d%c", &num, &unit) == 1) {
+        /* no unit specifier, default unit is milliseconds */
+        factor = 1;
+    } else {
+        switch (unit) {
+        case 's': factor = 1000; break; // seconds in milliseconds
+        case 'm': factor = 1000*60; break; // minutes in milliseconds
+        default: 
+            /* var should be pre-initialized to default */
+            OPTION_PARSE_ERROR(ERROR_OPTION_UNKNOWN_TIME_SPECIFIER, 4, 
+                               get_application_name(), get_application_pid(), 
+                               (char *)value, IF_DEBUG_ELSE("Terminating", "Continuing")); 
+            return;
+        }
+    }
+    *var = num * factor;
+}
+
+static void
+parse_uint_addr(ptr_uint_t *var, void *value)
+{
+    ptr_uint_t num;
+    char *opt = (char*)value;
+
+    if (sscanf(opt, PIFX, &num) == 1) { /* atoi(opt) */
+        *var = num;
+    } else {
+        /* var should be pre-initialized to default */
+        OPTION_PARSE_ERROR(ERROR_OPTION_BAD_NUMBER_FORMAT, 4, 
+                           get_application_name(), get_application_pid(), opt,
+                           IF_DEBUG_ELSE("Terminating", "Continuing"));
+    }
+}
+
+static inline void
+parse_pathstring_t(pathstring_t *var, void *value) { 
+    strncpy(*var, (char*)value, MAX_PATH_OPTION_LENGTH-1);
+    if (strlen((char *)value) > MAX_PATH_OPTION_LENGTH) {
+        OPTION_PARSE_ERROR(ERROR_OPTION_TOO_LONG_TO_PARSE, 4,
+                           get_application_name(), get_application_pid(), 
+                           (char *)value, IF_DEBUG_ELSE("Terminating", "Continuing"));
+    }
+    /* truncate if max (strncpy doesn't put NULL) */
+    (*var)[MAX_PATH_OPTION_LENGTH-1] = '\0';
+}
+
+static void
+parse_liststring_t(liststring_t *var, void *value) { 
+    /* Case 5727: append by default (separating via ';') for liststring_t, as
+     * opposed to what we do for all other option types where the final
+     * specifier overwrites all previous.  The special prefix '#' can be used to
+     * indicate overwrite.
+     */
+    size_t len;
+    if (*((char *)value) == '#') {
+        strncpy(*var, (((char*)value)+1), MAX_LIST_OPTION_LENGTH-1);
+        len = strlen(((char *)value)+1);
+    } else {
+        len = strlen(*var) + strlen((char *)value);
+        if (*var[0] != '\0') {
+            len++; /*;*/
+            strncat(*var, ";", MAX_LIST_OPTION_LENGTH-1 - strlen(*var));
+        }
+        strncat(*var, (char*)value, MAX_LIST_OPTION_LENGTH-1 - strlen(*var));
+    }
+    if (len >= MAX_LIST_OPTION_LENGTH) {
+        /* FIXME: no longer is value always the single too-long factor
+         * (could be appending a short option to a very long string), so
+         * should we change the message to "option is too long, truncating"?
+         */
+        OPTION_PARSE_ERROR(ERROR_OPTION_TOO_LONG_TO_PARSE, 4,
+                           get_application_name(), get_application_pid(), 
+                           *var, IF_DEBUG_ELSE("Terminating", "Continuing"));
+    }
+    /* truncate if max (strncpy doesn't put NULL, even though strncat does) */
+    (*var)[MAX_LIST_OPTION_LENGTH-1] = '\0';
+}
+
+
+#define OPTION_COMMAND(type, name, default_value, command_line_option,  \
+                       statement, description, flag, pcache) {          \
+    value = NULL;                                                       \
+    if (ISBOOL_##type) {                                                \
+        if (!strcmp(opt+1, command_line_option)) {                      \
+            value = &value_true;                                        \
+        } else if (!strncmp(opt+1, "no_", 3)                            \
+                   && !strcmp(opt+4, command_line_option)) {            \
+            value = &value_false;                                       \
+        }                                                               \
+    } else {                                                            \
+        if (!strcmp(opt+1, command_line_option)) {                      \
+            value = getword(optstr, &pos, wordbuffer, sizeof(wordbuffer));\
+            /* FIXME: check argument */                                 \
+        }                                                               \
+    }                                                                   \
+    if (value) {                                                        \
+        parse_##type(&options->name, value);                            \
+        statement;                                                      \
+        continue; /* match found */                                     \
+    }                                                                   \
+}
+
+/* PR 330860: the for_this_process bool is read in OPTION_COMMAND statements */
+static int
+set_dynamo_options_common(options_t *options, char *optstr, bool for_this_process)
+{
+    char *opt;
+    const char *pos = optstr;
+    const char *prev_pos;
+    bool got_badopt = false;
+    char badopt[MAX_OPTION_LENGTH];
+
+    char wordbuffer[MAX_OPTION_LENGTH];
+
+    /* used in the OPTION_COMMAND define above, declared here to save stack
+     * space FIXME : value_true and value_false could be static const if
+     * we mark the value arguments to the parse functions const */
+    void *value = NULL;
+    bool value_true = true, value_false = false;
+
+    if (optstr == NULL)
+        return 0;
+
+    ASSERT_OWN_OPTIONS_LOCK(options==&dynamo_options || options==&temp_options, &options_lock);
+    ASSERT(!OPTIONS_PROTECTED());
+    prev_pos = pos;
+    while ((opt = getword(optstr, &pos, wordbuffer, sizeof(wordbuffer))) != NULL) {
+        if (opt[0]=='-') {
+            /* N.B.: case 7853,7863: this expansion results in excessive stack
+             * usage under gcc.  If strcmp is intrinsic, gcc 4.1 uses 3 local
+             * slots per strcmp, and doesn't overlap them at all.  With -O2 it
+             * reduces it to 1 slot per strcmp, though it should be 0!  But
+             * gcc 3.4 only makes it intrinsic at -O1+, so -O2 fixes the
+             * problem for 4.1 but makes it appear in 3.4!  For now we
+             * explicitly use -fno-builtin-strcmp to handle all gcc versions.
+             * Long-term we may need our own strcmp inline just for this
+             * expansion that knows it needs no per-cmp slot.
+             */
+#include "optionsx.h"             // will continue if a match is found
+        }
+
+        /* no matching option found */
+        if (!got_badopt) {
+            snprintf(badopt, BUFFER_SIZE_ELEMENTS(badopt), "%s", opt);
+            NULL_TERMINATE_BUFFER(badopt);
+        }
+        got_badopt = true;
+
+        prev_pos = pos;
+    }
+
+    /* we only report the first bad option */
+    if (got_badopt) {
+        OPTION_PARSE_ERROR(ERROR_OPTION_UNKNOWN, 4, get_application_name(), 
+                           get_application_pid(), badopt,
+                           IF_DEBUG_ELSE("Terminating", "Continuing"));
+    }
+
+    return (int)got_badopt;
+}
+#undef OPTION_COMMAND
+
+CORE_STATIC int
+set_dynamo_options(options_t *options, char *optstr)
+{
+    return set_dynamo_options_common(options, optstr, true);
+}
+
+#if !defined(NOT_DYNAMORIO_CORE) && defined(WINDOWS)
+static int
+set_dynamo_options_other_process(options_t *options, char *optstr)
+{
+    return set_dynamo_options_common(options, optstr, false);
+}
+#endif
+
+/* max==0 means no max and 0 is an ok value */
+/* if option is incompatible, will try to touch up the option
+ * by assigning min to make it valid returns true if changed the 
+ * option value 
+ */
+bool
+check_param_bounds(uint *val, uint min, uint max, char *name)
+{
+    bool ret = false;
+    uint new_val;
+    if ((max == 0 && *val != 0 && *val < min) ||
+        (max > 0 && (*val < min || *val > max))) {
+        if (max == 0) {
+            new_val = min;
+            USAGE_ERROR("%s must be >= %d, resetting from %d to %d",
+                        name, min, *val, new_val);
+        } else {
+            new_val = min;
+            USAGE_ERROR("%s must be >= %d and <= %d, resetting from %d to %d", 
+                        name, min, max, *val, new_val);
+        }
+        *val = new_val;
+        ret = true;
+    }
+    DOLOG(1, LOG_CACHE, {
+        if (*val == 0) {
+            LOG(GLOBAL, LOG_CACHE, 1, "%s: <unlimited>\n", name);
+        } else {
+            LOG(GLOBAL, LOG_CACHE, 1, "%s: %d KB\n", name, *val/1024);
+        }
+    });
+    return ret;
+}
+
+/* Print an option string from an option struct.
+ * Xref case 7939, in DEBUG builds the ? : creates an unshared implicit local
+ * that led to huge stack usage for update_dynamic_options() so we use
+ * methods instead of macros for those.  While we could make these inline
+ * (which DEBUG will ignore and in release will result in the same code as
+ * MACRO version) these are way off the hot path and this way cuts the
+ * size of the release build dll by ~7kb.
+ */
+static void
+PRINT_STRING_bool(char *optionbuff, bool value, const char *option) 
+{
+    snprintf(optionbuff, MAX_OPTION_LENGTH, "-%s%s ",
+             value ? "" : "no_", option);
+}
+static void
+PRINT_STRING_uint(char *optionbuff, uint value, const char *option)
+{
+    /* FIXME: 0x100 hack to get logmask printed in hex,
+     * loglevel etc in decimal */ 
+    snprintf(optionbuff, MAX_OPTION_LENGTH,
+             (value > 0x100 ? "-%s 0x%x " : "-%s %d "), option, value);
+}
+#define PRINT_STRING_uint_size(optionbuff,value,option) \
+    snprintf(optionbuff, MAX_OPTION_LENGTH, "-%s %d%s ", option, \
+             ((value) % 1024 == 0 ? (value)/1024 : (value)), \
+             ((value) % 1024 == 0 ? "K" : "B"))
+#define PRINT_STRING_uint_time(optionbuff,value,option) \
+    snprintf(optionbuff, MAX_OPTION_LENGTH, "-%s %d ", option, (value))
+#define PRINT_STRING_uint_addr(optionbuff,value,option) \
+    snprintf(optionbuff, MAX_OPTION_LENGTH, "-%s "PFX" ", option, (value));
+#define PRINT_STRING_pathstring_t(optionbuff,value,option) snprintf(optionbuff, MAX_OPTION_LENGTH, "-%s '%s' ", option, (value));
+#define PRINT_STRING_liststring_t(optionbuff,value,option) snprintf(optionbuff, MAX_OPTION_LENGTH, "-%s '%s' ", option, (value));
+
+
+#define DIFF_bool(value1,value2) ( value1 != value2 )
+#define DIFF_uint(value1,value2) ( value1 != value2 )
+#define DIFF_uint_size(value1,value2) ( value1 != value2 )
+#define DIFF_uint_time(value1,value2) ( value1 != value2 )
+#define DIFF_uint_addr(value1,value2) ( value1 != value2 )
+#define DIFF_pathstring_t(value1,value2) (strcmp(value1,value2))
+#define DIFF_liststring_t(value1,value2) (strcmp(value1,value2))
+
+#define COPY_bool(value1,value2) ( value1 = value2 )
+#define COPY_uint(value1,value2) ( value1 = value2 )
+#define COPY_uint_size(value1,value2) ( value1 = value2 )
+#define COPY_uint_time(value1,value2) ( value1 = value2 )
+#define COPY_uint_addr(value1,value2) ( value1 = value2 )
+#define COPY_pathstring_t(value1,value2) (strncpy(value1,value2,sizeof(value1)))
+#define COPY_liststring_t(value1,value2) (strncpy(value1,value2,sizeof(value1)))
+
+/* Keep in synch with get_pcache_dynamo_options_string */
+void
+get_dynamo_options_string(options_t *options, char *opstr, int len, bool minimal)
+{
+    char optionbuff[MAX_OPTION_LENGTH];
+    opstr[0] = 0;
+
+#define OPTION_COMMAND(type, name, default_value,                       \
+                       command_line_option, statement,                  \
+                       description, flag, pcache) {                     \
+        if (command_line_option[0] != ' ' && /* not synthethic */       \
+            (!minimal ||                                                \
+             DIFF_##type(options->name,default_options.name))) {        \
+            PRINT_STRING_##type(optionbuff, options->name,              \
+                                command_line_option);                   \
+            NULL_TERMINATE_BUFFER(optionbuff);                          \
+            strncat(opstr, optionbuff, len-strlen(opstr)-1);            \
+        }                                                               \
+    }   
+#include "optionsx.h"
+#undef OPTION_COMMAND
+
+
+    opstr[len-1] = 0;
+}
+
+/* Fills opstr with a minimal string of only
+ * persistent-cache-affecting options whose effect is >= pcache_effect
+ * and that are different from the defaults.
+ * Keep in synch with get_dynamo_options_string.
+ */
+void
+get_pcache_dynamo_options_string(options_t *options, char *opstr, int len,
+                                 op_pcache_t pcache_effect)
+{
+    char optionbuff[MAX_OPTION_LENGTH];
+    opstr[0] = 0;
+
+#define OPTION_COMMAND(type, name, default_value,                       \
+                       command_line_option, statement,                  \
+                       description, flag, pcache) {                     \
+        if (command_line_option[0] != ' ' && /* not synthethic */       \
+            OPTION_AFFECTS_PCACHE_##name >= pcache_effect &&              \
+            DIFF_##type(options->name,default_options.name)) {          \
+            PRINT_STRING_##type(optionbuff, options->name,              \
+                                command_line_option);                   \
+            NULL_TERMINATE_BUFFER(optionbuff);                          \
+            strncat(opstr, optionbuff, len-strlen(opstr)-1);            \
+        }                                                               \
+    }   
+#include "optionsx.h"
+#undef OPTION_COMMAND
+
+    opstr[len-1] = 0;
+}
+
+/* Returns whether any persistent-cache-affecting options whose effect
+ * is == pcache_effect were passed in that are different from the defaults.
+ */
+bool
+has_pcache_dynamo_options(options_t *options, op_pcache_t pcache_effect)
+{
+#define OPTION_COMMAND(type, name, default_value,                       \
+                       command_line_option, statement,                  \
+                       description, flag, pcache) {                     \
+        if (command_line_option[0] != ' ' && /* not synthethic */       \
+            OPTION_AFFECTS_PCACHE_##name == pcache_effect &&              \
+            DIFF_##type(options->name,default_options.name)) {          \
+            return true;                                                \
+        }                                                               \
+    }   
+#include "optionsx.h"
+#undef OPTION_COMMAND
+    return false;
+}
+
+enum {OPTION_TYPE_STATIC, OPTION_TYPE_DYNAMIC};
+
+#if defined(DEBUG) && defined(INTERNAL)
+/* Used in update_dynamic_options() below. Usage is thread-safe as potential
+ * accesses are protected by the options_lock. */
+static char optionbuff[MAX_OPTION_LENGTH];
+static char new_optionbuff[MAX_OPTION_LENGTH];
+#endif
+
+/* need to see if any dynamic options have changed and copy them over */
+/* returns number of updated dynamic options */
+static int
+update_dynamic_options(options_t *options, options_t *new_options)
+{
+    int updated = 0;
+
+    ASSERT_OWN_OPTIONS_LOCK(options==&dynamo_options || options==&temp_options, &options_lock);
+    ASSERT(!OPTIONS_PROTECTED());
+
+#define OPTION_COMMAND(type, name, default_value,                       \
+                       command_line_option, statement,                  \
+                       description, flag, pcache) {                     \
+        if (OPTION_TYPE_DYNAMIC == OPTION_TYPE_##flag) {                \
+            if (DIFF_##type(options->name, new_options->name)) {        \
+                COPY_##type(options->name, new_options->name);          \
+                updated++;                                              \
+            }                                                           \
+        } else {                                                        \
+            DOLOG(2, LOG_TOP, {                                         \
+                if (DIFF_##type(options->name, new_options->name)) {    \
+                    PRINT_STRING_##type(optionbuff, options->name,      \
+                                        command_line_option);           \
+                    NULL_TERMINATE_BUFFER(optionbuff);                  \
+                    PRINT_STRING_##type(new_optionbuff, new_options->name,\
+                                        command_line_option);           \
+                    NULL_TERMINATE_BUFFER(new_optionbuff);              \
+                    LOG(GLOBAL, LOG_TOP, 2,                             \
+                        "Updating dynamic options : Ignoring static option change (%.*s changed to %.*s)\n",                                              \
+                        MAX_LOG_LENGTH/2-80, optionbuff,                \
+                        MAX_LOG_LENGTH/2-80, new_optionbuff);           \
+                }                                                       \
+            });                                                         \
+        }                                                               \
+    }
+
+#include "optionsx.h"
+#undef OPTION_COMMAND
+
+     return updated;
+}
+
+/****************************************************************************/
+#ifndef NOT_DYNAMORIO_CORE
+/* compare short_name, usually module name, against a list option of the combined
+ * default option (that could be overridden) and append list that is usually used
+ */
+list_default_or_append_t
+check_list_default_and_append(liststring_t default_list,
+                              liststring_t append_list,
+                              const char *short_name)
+{
+    list_default_or_append_t onlist = LIST_NO_MATCH;
+    ASSERT(short_name != NULL);
+    /* The wildcard '*' is currently expected to be tested by callers
+     * to allow modules without a PE name.  FIXME: Alternatively could
+     * check whether either list is * and also handle NULL name.
+     *
+     * FIXME: case 3858 about providing a substitute PE name
+     */
+    /* inlined IS_STRING_OPTION_EMPTY */
+    if (!(default_list[0] == '\0')) {
+        string_option_read_lock();
+        LOG(THREAD_GET, LOG_INTERP|LOG_VMAREAS, 2,
+            "check_list_default_and_append: module %s vs default list %s\n",
+            short_name, default_list);
+        if (check_filter(default_list, short_name))
+            onlist = LIST_ON_DEFAULT;
+        string_option_read_unlock();
+    }
+    if (!onlist &&
+        !(append_list[0] == '\0')) {
+        string_option_read_lock();
+        LOG(THREAD_GET, LOG_INTERP|LOG_VMAREAS, 2,
+            "check_list_default_and_append: module %s vs append list %s\n",
+            short_name, append_list);
+        if (check_filter(append_list, short_name))
+            onlist = LIST_ON_APPEND;
+        string_option_read_unlock();
+    }
+    return onlist;
+}
+
+/* security options have to be enabled to be blocking or reporting */
+#define SECURITY_OPTION_CONSISTENT(security_option) do {                        \
+    if (!TEST(OPTION_ENABLED, DYNAMO_OPTION(security_option)) &&                \
+        TESTANY(OPTION_BLOCK|OPTION_REPORT, DYNAMO_OPTION(security_option))) {  \
+        USAGE_ERROR("Incompatible settings for %s", #security_option);          \
+        dynamo_options.security_option = OPTION_DISABLED;                       \
+        changed_options = true;                                                 \
+    }                                                                           \
+} while (0)
+
+/* check for incompatible options */
+static bool
+check_option_compatibility_helper(int recurse_count)
+{
+    bool changed_options = false;
+#ifdef EXPOSE_INTERNAL_OPTIONS
+    if (!INTERNAL_OPTION(inline_calls) && !DYNAMO_OPTION(disable_traces)) {
+        /* cannot disable inlining of calls and build traces (currently) */
+        USAGE_ERROR("-no_inline_calls not compatible with -disable_traces, setting to default");
+        SET_DEFAULT_VALUE(inline_calls);
+        SET_DEFAULT_VALUE(disable_traces);
+        changed_options = true;
+    }
+    if (INTERNAL_OPTION(tracedump_binary) && INTERNAL_OPTION(tracedump_text)) {
+        USAGE_ERROR("Cannot set both -tracedump_binary and -tracedump_text, setting to default");
+        SET_DEFAULT_VALUE(tracedump_binary);
+        SET_DEFAULT_VALUE(tracedump_text);
+        changed_options = true;
+    }
+    if (INTERNAL_OPTION(trace_threshold) > USHRT_MAX) {
+        USAGE_ERROR("trace threshold (%d) must be <= USHRT_MAX (%d), setting to max",
+                    INTERNAL_OPTION(trace_threshold), USHRT_MAX, USHRT_MAX);
+        /* user was probably trying to make the threshold very high, set it to
+         * max
+         */
+        /* from derek : this could wreak havoc w/ trace building fencepost 
+         * cases... in the case where if head gets hot but somebody
+         * else is building trace w/ it you wait, and end up inc-ing counter
+         * again, in which case would wrap around and not be hot!
+         * (THCI already has problem w/ that b/c it only checks for == not >=
+         * (to avoid eflags))
+         */
+        /* FIXME : we may want to set to USHRT_MAX-10 or some such, same with 
+         * check above */
+        dynamo_options.trace_threshold = USHRT_MAX;
+        changed_options = true;
+    }
+    if (INTERNAL_OPTION(trace_counter_on_delete) > INTERNAL_OPTION(trace_threshold)) {
+        USAGE_ERROR("trace_counter_on_delete cannot be > trace_threshold");
+        SET_DEFAULT_VALUE(trace_counter_on_delete);
+        changed_options = true;
+    }
+    if (INTERNAL_OPTION(alt_hash_func) >= HASH_FUNCTION_ENUM_MAX) {
+        USAGE_ERROR("Invalid selection (%d) for shared cache hash func, must be < %d", 
+                    INTERNAL_OPTION(alt_hash_func), 
+                    HASH_FUNCTION_ENUM_MAX);
+        SET_DEFAULT_VALUE(alt_hash_func);
+        changed_options = true;
+    }
+    if (DYNAMO_OPTION(inline_bb_ibl) && 
+        DYNAMO_OPTION(shared_bbs) &&
+        !DYNAMO_OPTION(atomic_inlined_linking)) {
+        USAGE_ERROR("-inline_bb_ibl requires -atomic_inlined_linking when -shared_bbs");
+        dynamo_options.atomic_inlined_linking = true;
+        changed_options = true;
+    }
+# ifdef SHARING_STUDY
+    if (INTERNAL_OPTION(fragment_sharing_study) && SHARED_FRAGMENTS_ENABLED()) {
+        USAGE_ERROR("-fragment_sharing_study requires only private fragments");
+        dynamo_options.fragment_sharing_study = false;
+        changed_options = true;
+    }
+# endif
+#endif /* EXPOSE_INTERNAL_OPTIONS */
+
+    if (!ALIGNED(DYNAMO_OPTION(stack_size), PAGE_SIZE)) {
+        USAGE_ERROR("-stack_size must be at least 12K and a multiple of the page size (4k)");
+        SET_DEFAULT_VALUE(stack_size);
+        changed_options = true;
+    }
+
+#if defined(PROFILE_LINKCOUNT) || defined(TRACE_HEAD_CACHE_INCR) || defined(CUSTOM_EXIT_STUBS)
+    if (DYNAMO_OPTION(pad_jmps)) {
+        USAGE_ERROR("-pad_jmps not supported in this build yet");
+    }
+#endif
+
+#if defined(LINUX) || defined(CLIENT_INTERFACE)
+    if (DYNAMO_OPTION(pad_jmps) && !DYNAMO_OPTION(pad_jmps_mark_no_trace)
+        && DYNAMO_OPTION(enable_traces)
+# ifndef LINUX
+        && INTERNAL_OPTION(code_api)
+# endif
+        ) {
+        USAGE_ERROR("-pad_jmps isn't safe with code_api or on Linux without -pad_jmps_mark_no_trace when traces are enabled");
+    }
+#endif
+
+#ifdef PROFILE_LINKCOUNT
+    if (DYNAMO_OPTION(profile_counts) && 
+        TESTANY(SELFPROT_LOCAL|SELFPROT_GLOBAL, DYNAMO_OPTION(protect_mask))) {
+        /* May not be able to write to some linkstubs! */
+        USAGE_ERROR("-prof_counts incompatible with heap write protection");
+    }
+#endif
+
+    /****************************************************************************
+     * warn of unfinished and untested self-protection options 
+     * FIXME: update once these features are complete
+     */
+#ifdef LINUX
+    if (dynamo_options.protect_mask != 0) {
+        USAGE_ERROR("selfprot not supported on linux: case 8023");
+        dynamo_options.protect_mask = 0;
+        changed_options = true;
+    }
+#endif
+#if defined(WINDOWS) && !defined(NOLIBC)
+    if (TEST(SELFPROT_ANY_DATA_SECTION, dynamo_options.protect_mask)) {
+        /* since libc routines are embedded in our dll and we run
+         * them from the code cache we hit write faults
+         */
+        USAGE_ERROR("Cannot protect data segment w/ a non-NOLIBC build");
+        dynamo_options.protect_mask &= ~SELFPROT_ANY_DATA_SECTION;
+        changed_options = true;
+    }
+#endif
+    if (
+#ifdef WINDOWS
+        /* FIXME: CACHE isn't multithread safe yet */
+        TEST(SELFPROT_CACHE, dynamo_options.protect_mask) ||
+#endif
+        /* FIXME: LOCAL has some unresolved issues w/ new heap units, etc. */
+        TEST(SELFPROT_LOCAL, dynamo_options.protect_mask) ||
+        TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)) {
+        ASSERT_NOT_TESTED();
+    }
+    /* warn of incompatible options */
+    if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask) &&
+        !TEST(SELFPROT_GLOBAL, dynamo_options.protect_mask)) {
+        USAGE_ERROR("dcontext is only actually protected if global is as well");
+        /* FIXME: turn off dcontext?  or let upcontext be split anyway? */
+    }
+    /* FIXME: better way to enforce these incompatibilities w/ certain builds
+     * than by turning off protection?  Should we halt instead?
+     */
+    if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask) && SHARED_FRAGMENTS_ENABLED()) {
+        /* FIXME: get all shared gen routines to properly handle unprotected_context_t */
+        USAGE_ERROR("Shared cache does not support protecting dcontext yet");
+        dynamo_options.protect_mask &= ~SELFPROT_DCONTEXT;
+        changed_options = true;
+    }
+#ifdef TRACE_HEAD_CACHE_INCR
+    if (TESTANY(SELFPROT_LOCAL|SELFPROT_GLOBAL, dynamo_options.protect_mask)) {
+        USAGE_ERROR("Cannot protect heap in a TRACE_HEAD_CACHE_INCR build");
+        dynamo_options.protect_mask &= ~(SELFPROT_LOCAL|SELFPROT_GLOBAL);
+        changed_options = true;
+    }
+#endif
+#ifdef CLIENT_INTERFACE
+    /* case 9714: The client interface is compatible with the current default
+     * protect_mask of 0x101, but is incompatible with the following:
+     */
+    if (TESTANY(SELFPROT_DATA_CXTSW | SELFPROT_GLOBAL | SELFPROT_DCONTEXT |
+                SELFPROT_LOCAL | SELFPROT_CACHE | SELFPROT_STACK,
+                dynamo_options.protect_mask)) {
+        USAGE_ERROR("CLIENT_INTERFACE incompatible with protect_mask %x at this time",
+                    dynamo_options.protect_mask);
+        dynamo_options.protect_mask &= ~(SELFPROT_DATA_CXTSW | SELFPROT_GLOBAL |
+                                         SELFPROT_DCONTEXT | SELFPROT_LOCAL |
+                                         SELFPROT_CACHE | SELFPROT_STACK);
+        changed_options = true;
+    }
+#endif
+#ifdef CUSTOM_TRACES
+    if (PRIVATE_TRACES_ENABLED() && DYNAMO_OPTION(shared_bbs)) {
+        /* Due to complications with shadowing, we do not support
+         * private traces and shared bbs
+         */
+        USAGE_ERROR("CUSTOM_TRACES incompatible with private traces and shared bbs");
+        dynamo_options.shared_bbs = false;
+        changed_options = true;
+    }
+#endif
+#ifdef RETURN_STACK
+    if (dynamo_options.protect_mask != 0) {
+        USAGE_ERROR("RETURN_STACK incompatible with self-protection at this time");
+        dynamo_options.protect_mask = 0;
+        changed_options = true;
+    }
+#endif
+#ifdef IA32_ON_IA64
+    if (dynamo_options.protect_mask != 0) {
+        USAGE_ERROR("IA32_ON_IA64 incompatible with self-protection at this time");
+        dynamo_options.protect_mask = 0;
+        changed_options = true;
+    }
+#endif
+    /****************************************************************************/
+
+#if defined(PROFILE_RDTSC) && defined(SIDELINE)
+    if (dynamo_options.profile_times && dynamo_options.sideline) {
+        USAGE_ERROR("-profile_times incompatible with -sideline, setting to defaults");
+        SET_DEFAULT_VALUE(profile_times);
+        SET_DEFAULT_VALUE(sideline);
+        changed_options = true;
+    }
+#endif 
+
+#ifdef LINUX
+# ifndef HAVE_TLS
+    if (SHARED_FRAGMENTS_ENABLED()) {
+        USAGE_ERROR("shared fragments not supported on this OS");
+        dynamo_options.shared_bbs = false;
+        dynamo_options.shared_traces = false;
+        changed_options = true;
+    }
+#  ifdef X64
+    /* PR 361894: we do not support x64 without TLS (xref PR 244737) */
+#   error X64 requires HAVE_TLS
+#  endif
+# endif
+
+# if !defined(HAVE_PROC_MAPS) && defined(PROGRAM_SHEPHERDING)
+    /* PR 361594: without +x info we cannot support code origins */
+    if (DYNAMO_OPTION(code_origins)) {
+        USAGE_ERROR("-code_origins not supported on this OS");
+        dynamo_options.code_origins = false;
+        changed_options = true;
+    }
+    /* FIXME: We can't support certain GBOP policies, either.  Anything else? */
+# endif
+#endif
+
+    /* Manipulate all of the options needed for -shared_traces. */
+    if (DYNAMO_OPTION(shared_traces)) {
+        if (!DYNAMO_OPTION(private_ib_in_tls)) {
+            SYSLOG_INTERNAL_INFO("-shared_traces requires -private_ib_in_tls, enabling");
+            dynamo_options.private_ib_in_tls = true;
+            changed_options = true;
+        }
+        if (!DYNAMO_OPTION(shared_trace_ibl_routine)) {
+            SYSLOG_INTERNAL_INFO("-shared_traces requires -shared_trace_ibl_routine, enabling");
+            dynamo_options.shared_trace_ibl_routine = true;
+            changed_options = true;
+        }
+        if (!DYNAMO_OPTION(atomic_inlined_linking)) {
+            SYSLOG_INTERNAL_INFO("-shared_traces requires -atomic_inlined_linking, enabling");
+            dynamo_options.atomic_inlined_linking = true;
+            changed_options = true;
+        }
+    }
+#ifdef EXPOSE_INTERNAL_OPTIONS
+# ifdef DEADLOCK_AVOIDANCE
+    if (INTERNAL_OPTION(mutex_callstack) > MAX_MUTEX_CALLSTACK) {
+        USAGE_ERROR("-mutex_callstack is compiled with MAX_MUTEX_CALLSTACK=%d", MAX_MUTEX_CALLSTACK);
+        dynamo_options.mutex_callstack = MAX_MUTEX_CALLSTACK;
+        changed_options = true;
+    }
+# endif
+    if (INTERNAL_OPTION(unsafe_ignore_eflags_ibl) &&
+        !INTERNAL_OPTION(unsafe_ignore_eflags_prefix)) {
+        USAGE_ERROR("-unsafe_ignore_eflags_ibl requires -unsafe_ignore_eflags_prefix, enabling");
+        dynamo_options.unsafe_ignore_eflags_prefix = true;
+        changed_options = true;
+    }
+# ifdef X64
+    /* saving in the trace and restoring in ibl means that
+     * -unsafe_ignore_eflags_{trace,ibl} must be equivalent
+     */
+    if ((INTERNAL_OPTION(unsafe_ignore_eflags_ibl) &&
+         !INTERNAL_OPTION(unsafe_ignore_eflags_trace)) ||
+        (!INTERNAL_OPTION(unsafe_ignore_eflags_ibl) &&
+         INTERNAL_OPTION(unsafe_ignore_eflags_trace))) {
+        USAGE_ERROR("-unsafe_ignore_eflags_ibl must match -unsafe_ignore_eflags_trace "
+                    "for x64: enabling both");
+        dynamo_options.unsafe_ignore_eflags_trace = true;
+        dynamo_options.unsafe_ignore_eflags_ibl = true;
+        changed_options = true;
+    }
+# endif
+#endif /* EXPOSE_INTERNAL_OPTIONS */
+    if (RUNNING_WITHOUT_CODE_CACHE() && DYNAMO_OPTION(enable_reset)) {
+        /* No reset for hotp_only and thin_client modes; case 8389. */
+        USAGE_ERROR("-enable_reset can't be used with -hotp_only or -thin_client");
+        DISABLE_RESET(&dynamo_options);
+    }
+    if (DYNAMO_OPTION(reset_at_vmm_percent_free_limit) > 100) {
+        USAGE_ERROR("-reset_at_vmm_percent_free_limit is percentage value, "
+                    "can't be > 100");
+        dynamo_options.reset_at_vmm_percent_free_limit = 100;
+        changed_options = true;
+    }
+    if (DYNAMO_OPTION(reset_at_commit_percent_free_limit) > 100) {
+        USAGE_ERROR("-reset_at_commit_percent_free_limit is percentage value, can't be > 100");
+        dynamo_options.reset_at_commit_percent_free_limit = 100;
+        changed_options = true;
+    }
+    if (!DYNAMO_OPTION(enable_reset)) {
+        if (DYNAMO_OPTION(reset_at_nth_thread)) {
+            USAGE_ERROR("-reset_at_nth_thread requires -enable_reset, enabling");
+            dynamo_options.enable_reset = true;
+            changed_options = true;
+        }
+#ifdef EXPOSE_INTERNAL_OPTIONS
+        else if (INTERNAL_OPTION(reset_at_fragment_count)) {
+            USAGE_ERROR("-reset_at_fragment_count requires -enable_reset, enabling");
+            dynamo_options.enable_reset = true;
+            changed_options = true;
+        }
+#endif /* EXPOSE_INTERNAL_OPTIONS */
+        else if (DYNAMO_OPTION(reset_at_switch_to_os_at_vmm_limit)) {
+            USAGE_ERROR("-reset_at_switch_to_os_at_vmm_limit requires -enable_reset, "
+                        " enabling");
+            dynamo_options.enable_reset = true;
+            changed_options = true;
+        }
+        else if (DYNAMO_OPTION(reset_at_vmm_percent_free_limit) != 0) {
+            USAGE_ERROR("-reset_at_vmm_percent_free_limit requires -enable_reset, "
+                        " enabling");
+            dynamo_options.enable_reset = true;
+            changed_options = true;
+        }
+        else if (DYNAMO_OPTION(reset_at_vmm_free_limit) != 0) {
+            USAGE_ERROR("-reset_at_vmm_free_limit requires -enable_reset, enabling");
+            dynamo_options.enable_reset = true;
+            changed_options = true;
+        }
+        else if (DYNAMO_OPTION(reset_at_vmm_full)) {
+            USAGE_ERROR("-reset_at_vmm_full requires -enable_reset, enabling");
+            dynamo_options.enable_reset = true;
+            changed_options = true;
+        }
+        else if (DYNAMO_OPTION(reset_at_commit_percent_free_limit) != 0) {
+            USAGE_ERROR("-reset_at_commit_percent_free_limit requires -enable_reset,"
+                        " enabling");
+            dynamo_options.enable_reset = true;
+            changed_options = true;
+        }
+        else if (DYNAMO_OPTION(reset_at_commit_free_limit) != 0) {
+            USAGE_ERROR("-reset_at_commit_free_limit requires -enable_reset, enabling");
+            dynamo_options.enable_reset = true;
+            changed_options = true;
+        }
+        else if (DYNAMO_OPTION(reset_every_nth_pending) > 0) {
+            USAGE_ERROR("-reset_every_nth_pending requires -enable_reset, enabling");
+            dynamo_options.enable_reset = true;
+            changed_options = true;
+        }
+        else if (DYNAMO_OPTION(reset_at_nth_bb_unit) > 0) {
+            USAGE_ERROR("-reset_at_nth_bb_unit requires -enable_reset, enabling");
+            dynamo_options.enable_reset = true;
+            changed_options = true;
+        }
+        else if (DYNAMO_OPTION(reset_at_nth_trace_unit) > 0) {
+            USAGE_ERROR("-reset_at_nth_trace_unit requires -enable_reset, enabling");
+            dynamo_options.enable_reset = true;
+            changed_options = true;
+        }
+        else if (DYNAMO_OPTION(reset_every_nth_bb_unit) > 0) {
+            USAGE_ERROR("-reset_every_nth_bb_unit requires -enable_reset, enabling");
+            dynamo_options.enable_reset = true;
+            changed_options = true;
+        }
+        else if (DYNAMO_OPTION(reset_every_nth_trace_unit) > 0) {
+            USAGE_ERROR("-reset_every_nth_trace_unit requires -enable_reset, enabling");
+            dynamo_options.enable_reset = true;
+            changed_options = true;
+        }
+    }
+
+#ifdef TRACE_HEAD_CACHE_INCR
+    if (dynamo_options.shared_traces) {
+        USAGE_ERROR("Cannot share traces in a TRACE_HEAD_CACHE_INCR build");
+        dynamo_options.shared_traces = false;
+        changed_options = true;
+    }
+#endif
+    /* FIXME We support only shared BBs as IBTs when trace building is on. */
+    if (DYNAMO_OPTION(bb_ibl_targets) && !DYNAMO_OPTION(disable_traces) &&
+        !DYNAMO_OPTION(shared_bbs)) {
+        USAGE_ERROR("-bb_ibl_targets w/traces not supported w/-no_shared_bbs, disabling");
+        dynamo_options.bb_ibl_targets = false;
+        changed_options = true;
+    }
+    /* We need private_ib_in_tls for shared BB IBTs. */
+    if (DYNAMO_OPTION(bb_ibl_targets) && DYNAMO_OPTION(shared_bbs) &&
+        !DYNAMO_OPTION(private_ib_in_tls)) {
+        SYSLOG_INTERNAL_INFO("-bb_ibl_targets w/traces requires -private_ib_in_tls, enabling");
+        dynamo_options.private_ib_in_tls = true;
+        changed_options = true;
+    }
+    /* We need shared tables for shared BB IBTs when trace building is on. */
+    if (DYNAMO_OPTION(bb_ibl_targets) && DYNAMO_OPTION(shared_bbs) &&
+        !DYNAMO_OPTION(disable_traces) && !DYNAMO_OPTION(shared_bb_ibt_tables)) {
+        SYSLOG_INTERNAL_INFO("-bb_ibl_targets -shared_bbs w/traces requires -shared_bb_ibt_tables, enabling");
+        dynamo_options.shared_bb_ibt_tables = true;
+        changed_options = true;
+    }
+    /* If we're still using BBs as IBTs when trace building is on and want to
+     * add traces to the BB IBT tables, don't let private traces get added to
+     * a shared table.
+     */
+    if (DYNAMO_OPTION(bb_ibl_targets) && !DYNAMO_OPTION(disable_traces) &&
+        DYNAMO_OPTION(bb_ibt_table_includes_traces) &&
+        DYNAMO_OPTION(shared_bb_ibt_tables) && !DYNAMO_OPTION(shared_traces)) {
+        SYSLOG_INTERNAL_INFO("-bb_ibt_table_includes_traces -shared_bb_ibt_tables requires -shared_traces, disabling -bb_ibt_table_includes_traces");
+        dynamo_options.bb_ibt_table_includes_traces = false;
+        changed_options = true;
+    }
+    /* When using BBs as IBTs when trace building is on and adding traces to
+     * the BB IBT table, BBs & traces must use the same type of prefix. */
+    if (DYNAMO_OPTION(bb_ibl_targets) && !DYNAMO_OPTION(disable_traces) &&
+        DYNAMO_OPTION(bb_ibt_table_includes_traces) &&
+        (DYNAMO_OPTION(trace_single_restore_prefix) != DYNAMO_OPTION(bb_single_restore_prefix))) {
+        SYSLOG_INTERNAL_INFO("For -bb_ibl_targets -bb_ibt_table_includes_traces, traces & BBs must use identical prefixes");
+        /* FIXME We could either set trace_single_restore_prefix and
+         * bb_single_restore_prefix to the same value or use
+         * -no_bb_ibt_table_includes_traces. For now, we do the latter as
+         * it's less disruptive overall -- the trace prefix setting isn't
+         * modified and full prefixes are not used on BBs, limiting the
+         * cache/memory size increase. We need to measure to determine
+         * the proper thing to do.
+         */
+        SYSLOG_INTERNAL_INFO("Disabling -bb_ibt_table_includes_traces");
+        dynamo_options.bb_ibt_table_includes_traces = false;
+        changed_options = true;
+    }
+    if (DYNAMO_OPTION(syscalls_synch_flush) && !DYNAMO_OPTION(shared_deletion)) {
+        /* right now syscalls_synch_flush only affects shared_deletion, so we want
+         * to disable it when shared_deletion is off -- but don't yell at user so
+         * not a USAGE_ERROR, simply an info event
+         */
+        SYSLOG_INTERNAL_INFO("-syscalls_synch_flush requires -shared_deletion, disabling");
+        dynamo_options.syscalls_synch_flush = false;
+        changed_options = true;
+    }
+    if (DYNAMO_OPTION(free_private_stubs) && !DYNAMO_OPTION(separate_private_stubs)) {
+        USAGE_ERROR("-free_private_stubs requires -separate_private_stubs, disabling");
+        dynamo_options.free_private_stubs = false;
+        changed_options = true;
+    }
+    if (DYNAMO_OPTION(unsafe_free_shared_stubs) && !DYNAMO_OPTION(separate_shared_stubs)) {
+        USAGE_ERROR("-unsafe_free_shared_stubs requires -separate_shared_stubs, disabling");
+        dynamo_options.unsafe_free_shared_stubs = false;
+        changed_options = true;
+    }
+#ifdef EXPOSE_INTERNAL_OPTIONS
+    if (!DYNAMO_OPTION(indirect_stubs)) {
+# ifdef PROGRAM_SHEPHERDING
+        if (DYNAMO_OPTION(ret_after_call) ||
+            DYNAMO_OPTION(rct_ind_call) != OPTION_DISABLED ||
+            DYNAMO_OPTION(rct_ind_jump) != OPTION_DISABLED) {
+            USAGE_ERROR("C, E, and F policies require -indirect_stubs, enabling");
+            dynamo_options.indirect_stubs = true;
+            changed_options = true;
+        }
+# endif
+# ifdef PROFILE_LINKCOUNT
+        if (INTERNAL_OPTION(profile_counts)) {
+            USAGE_ERROR("-prof_counts require -indirect_stubs, enabling");
+            dynamo_options.indirect_stubs = true;
+            changed_options = true;
+        }
+# endif
+# ifdef CUSTOM_EXIT_STUBS
+        USAGE_ERROR("CUSTOM_EXIT_STUBS requires -indirect_stubs, enabling");
+        dynamo_options.indirect_stubs = true;
+        changed_options = true;
+# endif        
+#ifdef HASHTABLE_STATISTICS
+        if ((!DYNAMO_OPTION(shared_traces) && DYNAMO_OPTION(inline_trace_ibl)) ||
+            (!DYNAMO_OPTION(shared_bbs) && DYNAMO_OPTION(inline_bb_ibl))) {
+            USAGE_ERROR("private inlined ibl requires -indirect_stubs, enabling");
+            dynamo_options.indirect_stubs = true;
+            changed_options = true;
+        }
+#endif
+    }
+#endif
+    if ((DYNAMO_OPTION(finite_shared_bb_cache) ||
+         DYNAMO_OPTION(finite_shared_trace_cache)) &&
+        !DYNAMO_OPTION(cache_shared_free_list)) {
+        USAGE_ERROR("-finite_shared_{bb,trace}_cache requires -cache_shared_free_list, enabling");
+        dynamo_options.cache_shared_free_list = true;
+        changed_options = true;
+    }
+#ifdef WINDOWS
+    if (DYNAMO_OPTION(shared_fragment_shared_syscalls) &&
+        !DYNAMO_OPTION(shared_syscalls)) {
+        SYSLOG_INTERNAL_INFO("-shared_fragment_shared_syscalls requires -shared_syscalls, disabling");
+        dynamo_options.shared_fragment_shared_syscalls = false;
+        changed_options = true;
+    }
+# ifdef X64
+    if (!DYNAMO_OPTION(shared_fragment_shared_syscalls)) {
+        /* we use tls for the continuation pc, and the shared gencode, always */
+        USAGE_ERROR("-shared_fragment_shared_syscalls is required for x64");
+        dynamo_options.shared_fragment_shared_syscalls = true;
+        changed_options = true;
+    }
+    if (!DYNAMO_OPTION(private_ib_in_tls)) {
+        USAGE_ERROR("-private_ib_in_tls is required for x64");
+        dynamo_options.private_ib_in_tls = true;
+        changed_options = true;
+    }
+# endif
+    /* We retain shared_fragment_shared_syscalls as a separate option since
+     * it can be used -- but isn't required -- for shared BBs only mode. */
+    if (SHARED_FRAGMENTS_ENABLED() &&
+        DYNAMO_OPTION(shared_syscalls) &&
+        !DYNAMO_OPTION(shared_fragment_shared_syscalls)) {
+        SYSLOG_INTERNAL_INFO("-shared_{bbs|traces} w/-shared_syscalls requires -shared_fragment_shared_syscalls, enabling");
+        dynamo_options.shared_fragment_shared_syscalls = true;
+        changed_options = true;
+    }
+    if (SHARED_IBT_TABLES_ENABLED() && DYNAMO_OPTION(shared_syscalls) &&
+        !DYNAMO_OPTION(shared_fragment_shared_syscalls)) {
+        SYSLOG_INTERNAL_INFO("-shared_{bb|trace}_ibt_tables requires -shared_fragment_shared_syscalls, enabling");
+        dynamo_options.shared_fragment_shared_syscalls = true;
+        changed_options = true;
+    }
+    /* Don't leave -shared_fragment_shared_syscalls on if we're not using shared
+     * fragments: case 8027. */
+    /* FIXME We could try to eliminate the info msg by pulling this logic and
+     * associated processing into an OPTION_COMMAND (but OPTION_COMMAND has
+     * its own imperfections).
+     */
+    if (DYNAMO_OPTION(shared_fragment_shared_syscalls) &&
+        /* x64 uses -shared_fragment_shared_syscalls always */
+        IF_X64_ELSE(false, !SHARED_FRAGMENTS_ENABLED())) {
+        SYSLOG_INTERNAL_INFO("-shared_fragment_shared_syscalls requires -shared_{bbs|traces}, disabling");
+        dynamo_options.shared_fragment_shared_syscalls = false;
+        changed_options = true;
+    }
+    /* We don't yet support shared BBs and private traces targeting shared syscall
+     * simultaneously: case 5436. */
+    if (DYNAMO_OPTION(shared_syscalls) && DYNAMO_OPTION(shared_bbs) &&
+        !DYNAMO_OPTION(shared_traces) && !DYNAMO_OPTION(disable_traces)) {
+        SYSLOG_INTERNAL_INFO("-shared_syscalls not supported with -shared_bbs -no_shared_traces, disabling");
+        dynamo_options.shared_syscalls = false;
+        changed_options = true;
+    }
+# ifdef EXPOSE_INTERNAL_OPTIONS
+    if (INTERNAL_OPTION(shared_syscalls_fastpath) &&
+        !DYNAMO_OPTION(shared_syscalls)) {
+        SYSLOG_INTERNAL_INFO("-shared_syscalls_fastpath requires -shared_syscalls, disabling");
+        dynamo_options.shared_syscalls_fastpath = false;
+        changed_options = true;
+    }
+    if (INTERNAL_OPTION(shared_syscalls_fastpath) &&
+        !DYNAMO_OPTION(disable_traces)) {
+        SYSLOG_INTERNAL_INFO("-shared_syscalls_fastpath requires -disable_traces, disabling");
+        dynamo_options.shared_syscalls_fastpath = false;
+        changed_options = true;
+    }
+# endif
+#endif
+    if (DYNAMO_OPTION(shared_bb_ibt_tables) && !DYNAMO_OPTION(shared_bbs)) {
+        SYSLOG_INTERNAL_INFO("-shared_bb_ibt_tables requires -shared_bbs, disabling");
+        dynamo_options.shared_bb_ibt_tables = false;
+        changed_options = true;
+    }
+    if (DYNAMO_OPTION(shared_bb_ibt_tables) && !DYNAMO_OPTION(bb_ibl_targets)) {
+        SYSLOG_INTERNAL_INFO("-shared_bb_ibt_tables requires -bb_ibl_targets, disabling");
+        dynamo_options.shared_bb_ibt_tables = false;
+        changed_options = true;
+    }
+    if (DYNAMO_OPTION(shared_trace_ibt_tables) && !DYNAMO_OPTION(shared_traces)) {
+        SYSLOG_INTERNAL_INFO("-shared_bb_ibt_tables requires -shared_traces, disabling");
+        dynamo_options.shared_trace_ibt_tables = false;
+        changed_options = true;
+    }
+    if (DYNAMO_OPTION(shared_trace_ibt_tables) &&
+        DYNAMO_OPTION(trace_ibt_groom) > 0) {
+        USAGE_ERROR("-trace_ibt_groom incompatible -shared_trace_ibt_tables, disabling");
+        dynamo_options.trace_ibt_groom = 0;
+        changed_options = true;
+    }
+    if (DYNAMO_OPTION(shared_bb_ibt_tables) && DYNAMO_OPTION(bb_ibt_groom) > 0) {
+        USAGE_ERROR("-bb_ibt_groom incompatible -shared_bb_ibt_tables, disabling");
+        dynamo_options.bb_ibt_groom = 0;
+        changed_options = true;
+    }
+    if (DYNAMO_OPTION(bb_ibt_groom) != 0
+        && DYNAMO_OPTION(bb_ibt_groom) > DYNAMO_OPTION(private_bb_ibl_targets_load)) {
+        SYSLOG_INTERNAL_INFO("-bb_ibt_groom > private_bb_ibl_targets_load, disabling");
+        dynamo_options.bb_ibt_groom = 0;
+        changed_options = true;
+    }
+    if (DYNAMO_OPTION(trace_ibt_groom) != 0
+        && DYNAMO_OPTION(trace_ibt_groom) > DYNAMO_OPTION(private_ibl_targets_load)) {
+        SYSLOG_INTERNAL_INFO("-trace_ibt_groom > private_ibl_targets_load, disabling");
+        dynamo_options.trace_ibt_groom = 0;
+        changed_options = true;
+    }
+#if defined(LINUX) && defined(HAVE_TLS)
+    if (!DYNAMO_OPTION(ibl_table_in_tls)) {
+        /* xref PR 211147 */
+        SYSLOG_INTERNAL_INFO("-no_ibl_table_in_tls invalid on Linux, disabling");
+        dynamo_options.ibl_table_in_tls = true;
+        changed_options = true;
+    }
+#endif
+    if (DYNAMO_OPTION(IAT_elide) && !DYNAMO_OPTION(IAT_convert)) {
+        USAGE_ERROR("-IAT_elide requires -IAT_convert, enabling");
+        dynamo_options.IAT_convert = true;
+        changed_options = true;
+    }
+
+    if (DYNAMO_OPTION(sandbox_writable) && DYNAMO_OPTION(sandbox_non_text)) {
+        USAGE_ERROR("-sandbox_writable and -sandbox_non_text are mutually exclusive, using -sandbox_non_text");
+        dynamo_options.sandbox_writable = false;
+        dynamo_options.sandbox_non_text = true;
+        changed_options = true;
+    }
+    if (DYNAMO_OPTION(sandbox2ro_threshold) == 1U) {
+        /* since we inc the counter before executing a selfmod fragment,
+         * a threshold of 1 can result in no progress
+         */
+        USAGE_ERROR("-sandbox2ro_threshold cannot be 1, changing to 2");
+        dynamo_options.sandbox2ro_threshold = 2U;
+        changed_options = true;
+    }
+
+#ifdef WINDOWS
+# ifdef PROGRAM_SHEPHERDING   
+    if (DYNAMO_OPTION(IAT_convert) && !DYNAMO_OPTION(emulate_IAT_writes)) {
+        /* FIXME: case 1948 we should in fact depend on emulate_IAT_read */
+        USAGE_ERROR("-IAT_convert requires -emulate_IAT_writes, enabling");
+        dynamo_options.emulate_IAT_writes = true;
+        changed_options = true;
+    }
+# else
+    if (DYNAMO_OPTION(IAT_convert)) {
+        /* FIXME: case 1948 we should in fact depend on emulate_IAT_read */
+        USAGE_ERROR("-IAT_convert requires unavailable -emulate_IAT_writes, disabling IAT_convert");
+        dynamo_options.IAT_convert = false;
+        changed_options = true;
+    }
+# endif /* PROGRAM_SHEPHERDING */
+#endif /* WINDOWS */
+
+#ifdef WINDOWS
+    /* In theory ignore syscalls should work for int system calls, and also for
+     * sysenter system calls when Sygate SPA is not installed [though haven't
+     * tested].  However, ignored sysenter system calls when SPA is installed
+     * may lead to them reporting/blocking violations on certain platforms as
+     * the necessary mangling is too much at this point (FIXME). */
+    if (DYNAMO_OPTION(ignore_syscalls) && DYNAMO_OPTION(sygate_sysenter)) {
+        USAGE_ERROR("-ignore_syscalls can't be used with -sygate_sysenter");
+        dynamo_options.ignore_syscalls = false;
+        changed_options = true;
+    }
+    /* shared/ignore syscall writes to sysenter_storage dcontext field which
+     * should be in upcontext or something FIXME */
+    if (DYNAMO_OPTION(sygate_sysenter) &&
+        TEST(SELFPROT_DCONTEXT, DYNAMO_OPTION(protect_mask))) {
+        USAGE_ERROR("-sygate_sysenter incompatbile with -protect_mask dc");
+        dynamo_options.protect_mask &= ~SELFPROT_DCONTEXT;
+        changed_options = true;
+    }
+    if (DYNAMO_OPTION(hook_conflict) > HOOKED_TRAMPOLINE_MAX ||
+        DYNAMO_OPTION(hook_conflict) == HOOKED_TRAMPOLINE_HOOK_DEEPER) {
+        USAGE_ERROR("-hook_conflict invalid or unsupported value");
+        SET_DEFAULT_VALUE(hook_conflict);
+        changed_options = true;
+    }
+    if (DYNAMO_OPTION(native_exec_hook_conflict) > HOOKED_TRAMPOLINE_MAX ||
+        DYNAMO_OPTION(native_exec_hook_conflict) == HOOKED_TRAMPOLINE_CHAIN) {
+        USAGE_ERROR("-native_exec_hook_conflict invalid or unsupported value");
+        SET_DEFAULT_VALUE(native_exec_hook_conflict);
+        changed_options = true;
+    }    
+#endif
+
+#ifdef WINDOWS
+    SECURITY_OPTION_CONSISTENT(apc_policy);
+    SECURITY_OPTION_CONSISTENT(thread_policy);
+#endif
+#ifdef RETURN_AFTER_CALL
+    SECURITY_OPTION_CONSISTENT(rct_ret_unreadable);
+#endif
+#ifdef RCT_IND_BRANCH
+    SECURITY_OPTION_CONSISTENT(rct_ind_call);
+    SECURITY_OPTION_CONSISTENT(rct_ind_jump);
+    if (!DYNAMO_OPTION(ret_after_call)
+        && TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump))) {
+        SYSLOG_INTERNAL_INFO(".F depends on .C after calls, disabling .F");
+        dynamo_options.rct_ind_jump = OPTION_DISABLED;
+        changed_options = true;
+    }
+#endif
+
+    if (DYNAMO_OPTION(ibl_hash_func_offset) > IBL_HASH_FUNC_OFFSET_MAX) {
+        USAGE_ERROR("-ibl_hash_func_offset currently can only be 0, 1, 2, or 3"
+                    IF_X64(" or 4"));
+        dynamo_options.ibl_hash_func_offset = IBL_HASH_FUNC_OFFSET_MAX;
+        changed_options = true;
+    }
+
+#ifdef HOT_PATCHING_INTERFACE
+    /* -hot_patching controls all code relating to reading policies, modes,
+     * loading dlls, nudging, etc.  can't do -hotp_only without those.
+     */
+    if (DYNAMO_OPTION(hotp_only) && !DYNAMO_OPTION(hot_patching)) {
+        USAGE_ERROR("-hotp_only depends on -hot_patching, enabling -hot_patching");
+        dynamo_options.hot_patching = true;
+        changed_options = true;
+    }
+    /* -hotp_only can't rely on interp to identify/trap system calls as app 
+     * image will be patched directly, i.e., no interp.  -native_exec_syscalls 
+     * is needed to gain control for important app system calls.
+     */
+    if (DYNAMO_OPTION(hotp_only) && !DYNAMO_OPTION(native_exec_syscalls)) {
+        USAGE_ERROR("-hotp_only depends on -native_exec_syscalls, "
+                    "enabling -native_exec_syscalls");
+        dynamo_options.native_exec_syscalls = true;
+        changed_options = true;
+    }
+# ifdef RCT_IND_BRANCH
+    if (DYNAMO_OPTION(hotp_only) &&
+        (DYNAMO_OPTION(rct_ind_call) != OPTION_DISABLED ||
+         DYNAMO_OPTION(rct_ind_jump) != OPTION_DISABLED)) {
+        USAGE_ERROR("-rct_ind_{call,jump} incompatible w/ -hotp_only, disabling");
+        dynamo_options.rct_ind_call = OPTION_DISABLED;
+        dynamo_options.rct_ind_jump = OPTION_DISABLED;
+        changed_options = true;
+    }
+# endif
+# ifdef RETURN_AFTER_CALL
+    if (DYNAMO_OPTION(hotp_only) && DYNAMO_OPTION(ret_after_call)) {
+        USAGE_ERROR("-ret_after_call incompatible w/ -hotp_only, disabling");
+        dynamo_options.ret_after_call = false;
+        changed_options = true;
+    }
+    if (DYNAMO_OPTION(borland_SEH_rct) && !DYNAMO_OPTION(process_SEH_push)) {
+        USAGE_ERROR("-borland_SEH_rct requires -process_SEH_push, enabling");
+        dynamo_options.process_SEH_push = true;
+        changed_options = true;
+    }
+# endif
+# ifdef KSTATS
+    /* case 6837: FIXME: remove once -hotp_only -kstats work */
+    if (DYNAMO_OPTION(hotp_only) && DYNAMO_OPTION(kstats)) {
+        USAGE_ERROR("-hotp_only doesn't support -kstats");
+        dynamo_options.kstats = false;
+        changed_options = true;
+    }
+# endif /* KSTATS */
+# ifdef CLIENT_INTERFACE
+    /* Probe API needs hot_patching.  Also, for the time being at least, 
+     * liveshields shouldn't be on when probe api is on.
+     */
+    if (DYNAMO_OPTION(probe_api)) {
+        if (!DYNAMO_OPTION(hot_patching)) {
+            USAGE_ERROR("-probe_api needs -hot_patching");
+            dynamo_options.hot_patching = true;
+            changed_options = true;
+        }
+        if (DYNAMO_OPTION(liveshields)) {
+            USAGE_ERROR("-probe_api and -liveshields aren't compatible");
+            dynamo_options.liveshields = false;
+            changed_options = true;
+        }
+    }
+# endif /* CLIENT_INTERFACE */
+#endif /* HOT_PATCHING_INTERFACE */
+#ifdef CLIENT_INTERFACE
+    /* PR 226578 - Probe API doesn't flush pcaches conflicting with hotpatches. */
+    if (DYNAMO_OPTION(probe_api) && DYNAMO_OPTION(use_persisted)) {
+        USAGE_ERROR("-probe_api and -use_persisted aren't compatible");
+        dynamo_options.use_persisted = false;
+        changed_options = true;
+    }
+# ifdef LINUX
+    /* PR 304708: we intercept all signals for a better client interface */
+    if (DYNAMO_OPTION(code_api) && !DYNAMO_OPTION(intercept_all_signals)) {
+        USAGE_ERROR("-code_api requires -intercept_all_signals");
+        dynamo_options.intercept_all_signals = true;
+        changed_options = true;
+    }
+# endif
+#endif /* CLIENT_INTERFACE */
+#ifdef CALL_PROFILE
+    if (DYNAMO_OPTION(prof_caller) >  MAX_CALL_PROFILE_DEPTH) {
+        USAGE_ERROR("-prof_caller must be <= %d",  MAX_CALL_PROFILE_DEPTH);
+        dynamo_options.prof_caller =  MAX_CALL_PROFILE_DEPTH;
+        changed_options = true;
+    }
+#endif
+
+#ifdef WINDOWS_PC_SAMPLE
+    if (DYNAMO_OPTION(prof_pcs_global) < 8 || DYNAMO_OPTION(prof_pcs_global) > 32) {
+        USAGE_ERROR("-prof_pcs_global must be >=8 and <= 32, setting to default");
+        SET_DEFAULT_VALUE(prof_pcs_global);
+        changed_options = true;
+    }
+    if (DYNAMO_OPTION(prof_pcs_stubs) < 2 || DYNAMO_OPTION(prof_pcs_stubs) > 32) {
+        USAGE_ERROR("-prof_pcs_stubs must be >= 2 and <= 32, setting to default");
+        /* maybe better to clamp to closest bound */
+        SET_DEFAULT_VALUE(prof_pcs_stubs);
+        changed_options = true;
+    }
+#endif
+
+#ifdef WINDOWS
+    if (DYNAMO_OPTION(inject_at_create_process) && !DYNAMO_OPTION(early_inject)) {
+        USAGE_ERROR("-inject_at_create_process requires -early_inject, setting to defaults");
+        SET_DEFAULT_VALUE(inject_at_create_process);
+        SET_DEFAULT_VALUE(early_inject);
+        changed_options = true;
+    }
+    if (DYNAMO_OPTION(follow_systemwide) && !DYNAMO_OPTION(early_inject) &&
+        !IS_STRING_OPTION_EMPTY(block_mod_load_list_default) &&
+        !check_filter(DYNAMO_OPTION(block_mod_load_list_default),
+                      "dynamorio.dll")) {
+        USAGE_ERROR("follow_systemwide is dangerous without -early_inject unless -block_mod_load_list[_default] includes dynamorio.dll");
+        dynamo_options.follow_systemwide = false;
+        changed_options = true;
+    }
+# ifndef NOT_DYNAMORIO_CORE /* can't check without get_os_version etc. */
+    if (DYNAMO_OPTION(early_inject)) {
+        /* using early inject */
+        if (DYNAMO_OPTION(early_inject_location) ==
+            INJECT_LOCATION_LdrpLoadImportModule ||
+            (DYNAMO_OPTION(early_inject_location) ==
+             INJECT_LOCATION_LdrDefault &&
+             (get_os_version() == WINDOWS_VERSION_NT ||
+              get_os_version() == WINDOWS_VERSION_2000))) {
+            /* we will be using INJECT_LOCATION_LdrpLoadImportModule for 
+             * child processes */
+            if (!dr_early_injected ||
+                dr_early_injected_location !=
+                INJECT_LOCATION_LdrpLoadImportModule) {
+                /* can't get address from parent */
+                /* our method of finding the address relies on
+                 * -native_exec_syscalls */
+                if (!DYNAMO_OPTION(native_exec_syscalls)) {
+                    USAGE_ERROR("early_inject_location LdrpLoadImportModule requires -native_exec_syscalls for first process in chain");
+                    /* FIXME is this the best remediation choice? */
+                    dynamo_options.native_exec_syscalls = true;
+                    changed_options = true;
+                    /* FIXME - check that helper dlls exist, need a way of
+                     * finding systemroot for that. */
+                }
+            }
+        }
+    }
+    if (DYNAMO_OPTION(early_inject_location) > INJECT_LOCATION_MAX) {
+        USAGE_ERROR("invalid value for -early_inject_location, setting default");
+        SET_DEFAULT_VALUE(early_inject_location);
+        changed_options = true;
+    }
+    if (DYNAMO_OPTION(early_inject_location) == INJECT_LOCATION_LdrCustom &&
+        DYNAMO_OPTION(early_inject_address) == 0) {
+        USAGE_ERROR("early_inject_location LdrCustom requires setting -early_inject_address");
+        SET_DEFAULT_VALUE(early_inject_location);
+        changed_options = true;
+    }
+    if ((DYNAMO_OPTION(follow_children) || DYNAMO_OPTION(follow_systemwide) ||
+         DYNAMO_OPTION(follow_explicit_children)) &&
+        get_os_version() == WINDOWS_VERSION_VISTA &&
+        (!DYNAMO_OPTION(early_inject) || 
+         (!DYNAMO_OPTION(inject_at_create_process) &&
+          !DYNAMO_OPTION(vista_inject_at_create_process)))) {
+        /* We won't follow into child processes.  Won't affect the current
+         * proccess so only a warning. */
+        SYSLOG_INTERNAL_WARNING("Vista requires -early_inject and -vista_"
+                                "inject_at_create_process options to follow "
+                                "into child processes"); 
+    }
+#ifdef PROCESS_CONTROL
+    if (IS_PROCESS_CONTROL_ON() && !DYNAMO_OPTION(follow_systemwide)) {
+        /* Process control can happen even in slisted processes, so 
+         * thin_client need not be true.  To reliably control all processes, 
+         * dr must exist in all of them, so follow_systemwide and runall must 
+         * be true.
+         */
+        USAGE_ERROR("-process_controls needs -follow_systemwide");
+        dynamo_options.follow_systemwide = true;
+        changed_options = true;
+
+        /* FIXME: assert that the global rununder registry key is set to
+         * rununder_all, but how?
+         */
+    }
+    if (IS_PROCESS_CONTROL_ON() && DYNAMO_OPTION(pc_num_hashes) < 100) {
+        USAGE_ERROR("-pc_num_hashes must be at least 100 to minimize auto "
+                    "shut off of process control");
+        dynamo_options.pc_num_hashes = 100;
+        changed_options = true;
+    }
+#endif /* PROCESS_CONTROL */
+    if (DYNAMO_OPTION(thin_client)) {
+        /* Note: can't change all these options here because the recursion
+         * exceeds the limit, so leaving it to the user to fix it.
+
+         * If thin_client is specified, it will override client, low, and all
+         * the options shown below.  The check for client/low check will only 
+         * fix those options that won't be fixed by the subsequent if, i.e.,
+         * vm* options, that is why there is no else if
+         */
+        /* Is there any option for high/server? */
+        if (DYNAMO_OPTION(client) || DYNAMO_OPTION(low)) {
+            USAGE_ERROR("-thin_client won't work with -client or -low");
+            dynamo_options.client = false;
+            dynamo_options.low = false;
+            dynamo_options.vm_size = 2*1024*1024;
+            dynamo_options.vm_base = 0;
+            dynamo_options.vm_max_offset = 0;
+            changed_options = true;
+        }
+#ifdef HOT_PATCHING_INTERFACE
+        if (DYNAMO_OPTION(hot_patching) || DYNAMO_OPTION(hotp_only)) {
+            USAGE_ERROR("-thin_client doesn't support hot patching");
+            dynamo_options.hot_patching = false;
+            dynamo_options.hotp_only = false;
+            changed_options = true;
+        }
+#endif
+#ifdef GBOP
+        if (DYNAMO_OPTION(gbop)) {
+            USAGE_ERROR("-thin_client doesn't support gbop");
+            dynamo_options.gbop = 0;
+            changed_options = true;
+        }
+#endif
+#ifdef WINDOWS
+        if (DYNAMO_OPTION(aslr)) {
+            USAGE_ERROR("-thin_client doesn't support aslr ");
+            dynamo_options.aslr = 0;
+            changed_options = true;
+        }
+#endif
+        if (!DYNAMO_OPTION(native_exec_syscalls)) {
+            USAGE_ERROR("-thin_client needs -native_exec_syscalls");
+            dynamo_options.native_exec_syscalls = true;
+            changed_options = true;
+        }
+#ifdef KSTATS
+        /* Same issue as hotp_only; case 6837. */
+        if (DYNAMO_OPTION(kstats)) {
+            USAGE_ERROR("-thin_client doesn't support -kstats");
+            dynamo_options.kstats = false;
+            changed_options = true;
+        }
+#endif
+
+        /* FIXME: not tested on vista where ldr_init_thunk is hooked first
+         *        and has a different process creation mechanism; case 8576.
+         */
+    }
+# endif /* !NOT_DYNAMORIO_CORE */
+#endif /* WINDOWS */
+
+#ifdef CLIENT_INTERFACE
+    if (!IS_INTERNAL_STRING_OPTION_EMPTY(client_lib) &&
+        !(INTERNAL_OPTION(code_api) || INTERNAL_OPTION(probe_api) 
+          IF_PROG_SHEP(|| DYNAMO_OPTION(security_api)))) {
+        USAGE_ERROR("-client_lib requires at least one API flag");
+    }
+#endif
+
+    if (DYNAMO_OPTION(coarse_units)) {
+#ifdef CUSTOM_EXIT_STUBS
+        USAGE_ERROR("-coarse_units incompatible with CUSTOM_EXIT_STUBS: disabling");
+        dynamo_options.coarse_units = false;
+        changed_options = true;
+#endif
+#ifdef CLIENT_INTERFACE
+        if (DYNAMO_OPTION(bb_prefixes)) {
+            /* coarse_units doesn't support prefixes in general.
+             * the variation by addr prefix according to processor type
+             * is also not stored in pcaches.
+             */
+            USAGE_ERROR("-coarse_units incompatible with -bb_prefixes: disabling");
+            dynamo_options.coarse_units = false;
+            changed_options = true;
+        }
+#endif
+        if (!DYNAMO_OPTION(shared_bbs)) {
+            USAGE_ERROR("-coarse_units requires -shared_bbs, enabling");
+            dynamo_options.shared_bbs = true;
+            changed_options = true;
+        }
+        if (DYNAMO_OPTION(inline_bb_ibl)) {
+            USAGE_ERROR("-coarse_units not compatible with -inline_bb_ibl, disabling");
+            dynamo_options.inline_bb_ibl = false;
+            changed_options = true;
+        }
+        if (DYNAMO_OPTION(bb_ibl_targets) && !DYNAMO_OPTION(disable_traces)) {
+            /* case 147/9636: NYI */
+            USAGE_ERROR("-coarse_units not compatible with -bb_ibl_targets in "
+                        "presence of traces, disabling");
+            dynamo_options.bb_ibl_targets = false;
+            changed_options = true;
+        }
+#ifdef EXPOSE_INTERNAL_OPTIONS
+        if (!DYNAMO_OPTION(indirect_stubs)) {
+            /* FIXME case 8827: wouldn't be hard to support, just need to ensure the
+             * shared use of the ibl fake stubs is properly separated in dispatch
+             */
+            USAGE_ERROR("case 8827: -coarse_units requires -indirect_stubs, enabling");
+            dynamo_options.indirect_stubs = true;
+            changed_options = true;
+        }
+        if (INTERNAL_OPTION(store_translations)) {
+            /* FIXME case 9707: NYI */
+            USAGE_ERROR("case 9707: -coarse_units does not support -store_translations, "
+                        "disabling");
+            dynamo_options.store_translations = false;
+            changed_options = true;
+        }
+#endif
+        if (DYNAMO_OPTION(IAT_elide)) {
+            /* FIXME case 9710: NYI */
+            USAGE_ERROR("case 9710: -coarse_units does not support -IAT_elide, "
+                        "disabling");
+            dynamo_options.IAT_elide = false;
+            changed_options = true;
+        }
+        if (DYNAMO_OPTION(unsafe_freeze_elide_sole_ubr) &&
+            !DYNAMO_OPTION(coarse_freeze_elide_ubr)) {
+            USAGE_ERROR("-unsafe_freeze_elide_sole_ubr requires "
+                        "-coarse_freeze_elide_ubr, enabling");
+            dynamo_options.coarse_freeze_elide_ubr = true;
+            changed_options = true;
+        }
+# ifdef PROGRAM_SHEPHERDING
+        if (DYNAMO_OPTION(coarse_merge_iat) &&
+            !DYNAMO_OPTION(executable_if_rx_text)
+            IF_WINDOWS(&& !DYNAMO_OPTION(executable_after_load))) {
+            /* case 8640: relies on -executable_{if_rx_text,after_load} */
+            USAGE_ERROR("-coarse_merge_iat requires "
+                        "-executable_{if_rx_text,after_load}; disabling");
+            dynamo_options.coarse_merge_iat = false;
+            changed_options = true;
+        }
+# endif
+    }
+
+    if (!DYNAMO_OPTION(persist_per_user) &&
+        (DYNAMO_OPTION(validate_owner_dir) ||
+         DYNAMO_OPTION(validate_owner_file))) {
+        USAGE_ERROR("-no_persist_per_user is insecure\n"
+                    "disabling validation, you are on your own!");
+        dynamo_options.validate_owner_file = false;
+        dynamo_options.validate_owner_dir = false;
+        changed_options = true;
+    }
+    
+#ifndef NOT_DYNAMORIO_CORE
+    /* fcache param checks rather involved, leave them in fcache.c */
+    /* case 7626: don't short-circuit checks, as later ones may be needed */
+    changed_options = fcache_check_option_compatibility() || changed_options;
+    changed_options = heap_check_option_compatibility() || changed_options;
+
+    changed_options = os_check_option_compatibility() || changed_options;
+#endif
+
+    if (changed_options) {
+        if (recurse_count > 5) {
+            /* prevent infinite loop, should never recurse this many times */
+            FATAL_USAGE_ERROR(OPTION_VERIFICATION_RECURSION, 2, get_application_name(), get_application_pid());
+        } else {
+            check_option_compatibility_helper(recurse_count+1);
+        }
+    }
+    return !changed_options;
+}
+
+/* returns true if changed any options */
+static bool
+check_option_compatibility()
+{
+    ASSERT_OWN_OPTIONS_LOCK(true, &options_lock);
+    ASSERT(!OPTIONS_PROTECTED());
+    return check_option_compatibility_helper(0);
+}
+
+/* returns true if changed any options */
+static bool
+check_dynamic_option_compatibility()
+{ 
+    ASSERT_OWN_OPTIONS_LOCK(true, &options_lock);
+    /* NOTE : use non-synch form of USAGE_ERROR  in here to avoid 
+     * infinite recursion */
+    return false; /* nothing to check for yet */
+} 
+
+/* initialize dynamo options */
+int
+options_init()
+{
+    int ret = 0, retval;
+
+    /* .lspdata pages start out writable so no unprotect needed here */
+    write_lock(&options_lock);
+    ASSERT(sizeof(dynamo_options) == sizeof(options_t));
+    /* get dynamo options */
+    retval = get_parameter(L_IF_WIN(DYNAMORIO_VAR_OPTIONS), option_string,
+                           sizeof(option_string));
+    if (IS_GET_PARAMETER_SUCCESS(retval))
+        ret = set_dynamo_options(&dynamo_options, option_string);
+    check_option_compatibility();
+    /* options will be protected when DR init is completed */
+    write_unlock(&options_lock);
+    return ret;
+}
+
+/* clean up dynamo options state */
+void
+options_exit()
+{
+    DELETE_READWRITE_LOCK(options_lock);
+}
+
+/* this function returns holding the options lock */
+void
+options_make_writable()
+{
+    ASSERT_DO_NOT_OWN_WRITE_LOCK(true, &options_lock);
+    write_lock(&options_lock);
+    SELF_UNPROTECT_OPTIONS();
+}
+
+/* assumes caller holds the options lock -- typically by calling
+ * options_make_writable() beforehand
+ */
+void
+options_restore_readonly()
+{
+    ASSERT_OWN_WRITE_LOCK(true, &options_lock);
+    SELF_PROTECT_OPTIONS();
+    write_unlock(&options_lock);
+}
+
+/* updates dynamic options and returns if any were changed */
+int
+synchronize_dynamic_options()
+{
+    int updated, retval;
+    bool compatibility_fixup = false;
+
+    if (!dynamo_options.dynamic_options)
+        return 0;
+
+    /* dynamic options */
+    STATS_INC(option_synchronizations);
+
+    /* make entire sequence atomic, esp since we're using a shared temp structure
+     * to save stack space.
+     * if we already have the lock, we must be in the middle of an update,
+     * so this becomes a nop.
+     */
+    if (self_owns_write_lock(&options_lock)) {
+        STATS_INC(option_synchronizations_nop);
+        return 0;
+    }
+    write_lock(&options_lock);
+
+    /* check again now that we hold write lock in case was modified */
+    if (!dynamo_options.dynamic_options) {
+        STATS_INC(option_synchronizations_nop);
+        write_unlock(&options_lock);
+        return 0;
+    }
+
+    /* get options */
+    retval = get_parameter(L_IF_WIN(DYNAMORIO_VAR_OPTIONS), new_option_string,
+                           sizeof(new_option_string));
+    if (IS_GET_PARAMETER_FAILURE(retval)) {
+        STATS_INC(option_synchronizations_nop);
+        write_unlock(&options_lock);
+        return 0;
+    }
+
+    if (strcmp(option_string, new_option_string) == 0) {
+        STATS_INC(option_synchronizations_nop);
+        write_unlock(&options_lock);
+        return 0;
+    }
+
+    SELF_UNPROTECT_OPTIONS();
+    set_dynamo_options_defaults(&temp_options);
+    set_dynamo_options(&temp_options, new_option_string);
+    updated = update_dynamic_options(&dynamo_options, &temp_options);
+    compatibility_fixup = check_dynamic_option_compatibility();
+    /* option_string holds a copy of the last read registry value */
+    strncpy(option_string, new_option_string, BUFFER_SIZE_ELEMENTS(option_string));
+    NULL_TERMINATE_BUFFER(option_string);
+    SELF_PROTECT_OPTIONS();
+
+    LOG(GLOBAL, LOG_ALL, 2 , "synchronize_dynamic_options: %s, updated = %d\n", new_option_string, updated);
+
+#ifdef EXPOSE_INTERNAL_OPTIONS
+    if (updated) {
+        get_dynamo_options_string(&dynamo_options, 
+                                  new_option_string, sizeof(new_option_string), true);
+        SYSLOG_INTERNAL_NO_OPTION_SYNCH(SYSLOG_INFORMATION, 
+                                        "Updated options = \"%s\"%s", 
+                                        new_option_string, 
+                                        compatibility_fixup ?
+                                        " after required compatibility fixups!" : "");
+        write_unlock(&options_lock);
+    } else
+#endif /* EXPOSE_INTERNAL_OPTIONS */
+        write_unlock(&options_lock);
+
+    return updated;
+}
+
+# ifdef WINDOWS
+/* Currently used to get child options to prevent aslr_dr for thin_client
+ * processes.  Assumes other processes, though there is nothing wrong with
+ * using this to read the current process's options; we just guard against it
+ * because it is already done in other places like init and dynamic option
+ * update.
+ * Return value : Pointer to the global temp_options struct; so don't try to 
+ *                  free it!
+ * Note         : The CALLER IS RESPONSIBLE for unlocking the options_lock 
+ *                  (write lock held) and shouldn't rely on the returned 
+ *                  pointer after that.
+ */
+const options_t *
+get_process_options(HANDLE process_handle)
+{
+    uint err;
+
+    /* Shouldn't be using this for the current process. */
+    ASSERT(process_handle != NT_CURRENT_PROCESS && 
+           process_handle != NT_CURRENT_THREAD && process_handle != NULL);
+    ASSERT(!READWRITE_LOCK_HELD(&options_lock));
+
+    write_lock(&options_lock);
+    SELF_UNPROTECT_OPTIONS();
+
+    /* Making an assumption that the core will be the same for the parent and
+     * child if set_dynamo_options_default is to work correctly.  I think it is
+     * reasonable.  FIXME: match parent & child cores & then use set default, 
+     * what otherwise?
+     */
+    set_dynamo_options_defaults(&temp_options);
+    err = get_process_parameter(process_handle, L_DYNAMORIO_VAR_OPTIONS,
+                                new_option_string, sizeof(new_option_string));
+    /* PR 330860: be sure not to set for this process */
+    if (IS_GET_PARAMETER_SUCCESS(err))
+        set_dynamo_options_other_process(&temp_options, new_option_string);
+
+    /* FIXME: options_t compatibility check isn't done because that function
+     * operates directly on dynamo_options!  As this is currently used only to
+     * detect if child is in thin_client we don't have to fix it because no
+     * option turns on thin_client and because fixing the compatibility checker
+     * so close to 4.2 release isn't a good idea.  Case 9193 tracks this.
+     */
+
+    SELF_PROTECT_OPTIONS();
+
+    /* Note: we are delibrately not unlocking options_lock; caller will do it.
+     * This is done so as to not expose a lot of the options module
+     * functionality outside when having to access another process's options
+     * temporarily. */
+    return &temp_options;
+}
+# endif /* WINDOWS */
+
+#endif /* NOT_DYNAMORIO_CORE */
+
+
+#ifdef OPTIONS_UNIT_TEST
+
+/* FIXME: not supported yet, try it out in UNIT_TEST */
+static void
+show_dynamo_options(bool minimal)
+{
+    char opstring[MAX_OPTIONS_STRING];
+
+    get_dynamo_options_string(&dynamo_options, opstring,
+                              sizeof(opstring), minimal);
+    print_file(STDERR, "%s\n", opstring);
+}
+
+/* USAGE Show descriptions of all available options */
+static void
+show_dynamo_option_descriptions()
+{
+#define OPTION_COMMAND(type, name, default_value,                       \
+                       command_line_option, statement,                  \
+                       description, flag)                               \
+    if (command_line_option[0] != ' ') { /* not synthethic */           \
+        print_file(STDERR, "-%-20s %s\n", command_line_option, description);        \
+    }
+#include "optionsx.h"
+#undef OPTION_COMMAND
+}
+
+int
+main() {
+    char buf[MAX_OPTIONS_STRING];
+    options_t new_options;
+    int updated;
+
+    standalone_init();
+    write_lock(&options_lock); /* simplicity: just grab whole time */
+    SELF_UNPROTECT_OPTIONS();
+
+    /* FIXME: actually use asserts for automated testing that does not require
+     * visual inspection
+     */
+
+    /* FIXME: test invalid options -- w/o dying! */
+
+    print_file(STDERR, "default---\n");
+    show_dynamo_options(false);
+    print_file(STDERR, "after---\n");
+    set_dynamo_options(&dynamo_options,
+                       "-stats -loglevel 1 -logmask 0x10 -exempt_text_list 'mylib.dll;evilbad.dll;really_long_name_for_a_dll.dll' -stderr_mask 12");
+    show_dynamo_options(true);
+
+    set_dynamo_options(&dynamo_options,
+                       "-logmask 17 -cache_bb_max 20 -cache_trace_max 20M -timeout 3m -diagnostics");
+    show_dynamo_options(true);
+
+    set_dynamo_options_defaults(&new_options);
+    set_dynamo_options(&new_options,
+                       "-logmask 7 -cache_bb_max 20 -cache_trace_max 20M -timeout 3m -detect_mode");
+    updated = update_dynamic_options(&dynamo_options, &new_options);
+    print_file(STDERR, "updated %d\n", updated);
+    show_dynamo_options(true);
+
+    show_dynamo_option_descriptions();
+
+    get_dynamo_options_string(&dynamo_options, buf, MAXIMUM_PATH, 1);
+    print_file(STDERR, "options string: %s\n", buf);
+
+    get_dynamo_options_string(&dynamo_options, buf, MAXIMUM_PATH, 0);
+    print_file(STDERR, "full options string: %s\n", buf);
+
+    set_dynamo_options_defaults(&dynamo_options);
+    get_dynamo_options_string(&dynamo_options, buf, MAXIMUM_PATH, 1);
+    print_file(STDERR, "default ops string: %s\n", buf);
+    SELF_PROTECT_OPTIONS();
+    write_unlock(&options_lock);
+
+    return 0;
+}
+
+#endif /* OPTIONS_UNIT_TEST */
diff --git a/core/options.h b/core/options.h
new file mode 100644
index 000000000..30f794d66
--- /dev/null
+++ b/core/options.h
@@ -0,0 +1,396 @@
+/* **********************************************************
+ * Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+
+/*
+ * options.h
+ *
+ * options struct and prototypes from options.c
+ * for use with core and gui/nm
+ */
+
+#ifndef _OPTIONS_H_
+#define _OPTIONS_H_
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+/* Security policy flags.  Note that if the off flag is not explicitly
+ * used in an option default value definition, then the option doesn't
+ * support that flag.
+*/
+typedef enum {
+    /* security mechanism needed for detection can be completely disabled */
+    OPTION_ENABLED     = 0x1,   /* if set, security mechanism is on */
+    OPTION_DISABLED    = 0x0,   /* if not set, security mechanism is off  */
+
+    /* security policy can be built on top.  For stateless policies
+     * all flags can be dynamic options.  For stateful security mechanisms as
+     * long as OPTION_ENABLED stays on for gathering information, the
+     * security policies OPTION_BLOCK|OPTION_REPORT can be dynamically
+     * set.  If no detection action is set, a detection check may be skipped.
+     */
+
+    /* associated security policy can be either blocking or not */
+    /* If set, app action (i.e., bad behavior) is disallowed and remediation
+     * action (kill thread, kill process, throw exception) is performed.
+     * Note: detect_mode will override this flag.
+     * FIXME: app_thread_policy_helper seems to be the one place where 
+     * detect_mode doesn't override this; case 9088 tracks this; xref case 8451
+     * for an explanation of why this seemed to leave as such.
+     */
+    OPTION_BLOCK       = 0x2,
+    OPTION_NO_BLOCK    = 0x0,
+
+    /* policies that lend themselves to standard attack handling may use this flag */
+    OPTION_HANDLING    = 0x4,   /* if set, overrides default attack handling */
+    OPTION_NO_HANDLING = 0x0,   /* when not set, default attack handling is used */
+
+    /* report or stay silent */
+    OPTION_REPORT      = 0x8,   /* if set, report action is being taken */
+    OPTION_NO_REPORT   = 0x0,   /* if not set, action is taken silently */
+
+    /* Block ignoring detect_mode; to handle case 10610. */
+    OPTION_BLOCK_IGNORE_DETECT  = 0x20,
+
+    /* modifications in security policy or detection mechanism are controlled with */
+    OPTION_CUSTOM      = 0x100, /* alternative policy bit - custom meaning per option */
+    OPTION_NO_CUSTOM   = 0x0,   /* alternative policy bit - custom meaning per option */
+} security_option_t;
+
+/* values taken by the option hook_conflict */
+enum {
+    /* these are mutually exclusive */
+    HOOKED_TRAMPOLINE_DIE    = 0, /* throw a fatal error if chained */
+    HOOKED_TRAMPOLINE_SQUASH = 1, /* modify any existing chains with a good guess */
+    HOOKED_TRAMPOLINE_CHAIN  = 2, /* rerelativize and mangle to support chaining */
+    HOOKED_TRAMPOLINE_HOOK_DEEPER = 3, /* move our hook deeper into the function */
+    HOOKED_TRAMPOLINE_MAX = 3,
+};
+
+/* for all option uses */
+/* N.B.: for 64-bit should we make the uint_size options be size_t?
+ * For now we're ok with all such options maxing out at 4GB, and in fact
+ * some like the fcache options are assumed to not be larger.
+ */
+#define uint_size uint
+#define uint_time uint
+/* So far all addr_t are external so we don't have a 64-bit problem */
+#define uint_addr ptr_uint_t
+
+/* to dispatch on string default values, kept in struct not enum */
+#define ISSTRING_bool 0
+#define ISSTRING_uint 0
+#define ISSTRING_uint_size 0
+#define ISSTRING_uint_time 0
+#define ISSTRING_ptr_uint_t 0
+#define ISSTRING_pathstring_t 1
+#define ISSTRING_liststring_t 1
+
+/* Does this option affect persistent cache formation? */
+typedef enum {
+    OP_PCACHE_NOP    = 0, /* No effect on pcaches */
+    OP_PCACHE_LOCAL  = 1, /* Can only relax (not tighten), and when it relaxes any
+                           * module that module is marked via
+                           * os_module_set_flag(MODULE_WAS_EXEMPTED).
+                           */
+    OP_PCACHE_GLOBAL = 2, /* Affects pcaches but not called out as local. */
+} op_pcache_t;
+
+/* We use an enum for default values of non-string options so that the compiler
+ * will fold them away (cl, at least, won't do it even for struct fields
+ * declared const), and also to determine whether options are internal or take
+ * string values or affect peristent caches.
+ * In order for this to work optionsx.h strings must either be EMPTY_STRING or
+ * a string in the OPTION_STRING() macro.
+ * N.B.: for 64-bit presumably every enum value is 64-bit-wide b/c some
+ * of them are.  FIXME: could split up the enums to save space: or are we
+ * never actually storing these values?
+ */
+#define OPTION_STRING(x) 0 /* no string in enum */
+#define EMPTY_STRING     0 /* no string in enum */
+#define OPTION_COMMAND(type, name, default_value, command_line_option, statement, \
+                       description, flag, pcache)                                 \
+    OPTION_DEFAULT_VALUE_##name = (ptr_uint_t) default_value, \
+    OPTION_IS_INTERNAL_##name = false, \
+    OPTION_IS_STRING_##name = ISSTRING_##type, \
+    OPTION_AFFECTS_PCACHE_##name = pcache,
+#define OPTION_COMMAND_INTERNAL(type, name, default_value, command_line_option, \
+                                statement, description, flag, pcache)           \
+    OPTION_DEFAULT_VALUE_##name = (ptr_uint_t) default_value, \
+    OPTION_IS_INTERNAL_##name = true, \
+    OPTION_IS_STRING_##name = ISSTRING_##type, \
+    OPTION_AFFECTS_PCACHE_##name = pcache,
+enum option_is_internal {
+#   include "optionsx.h"
+};
+#undef OPTION_COMMAND
+#undef OPTION_COMMAND_INTERNAL
+#undef OPTION_STRING
+#undef EMPTY_STRING
+
+/* for all other option uses */
+#define OPTION_STRING(x) x
+#define EMPTY_STRING {0}        /* fills the constant char structure with zeroes */
+
+/* the Option struct typedef */
+#ifdef EXPOSE_INTERNAL_OPTIONS
+#  define OPTION_COMMAND_INTERNAL(type, name, default_value, command_line_option, \
+                                  statement, description, flag, pcache)           \
+    type name;
+#else 
+#  define OPTION_COMMAND_INTERNAL(type, name, default_value, command_line_option, \
+                                  statement, description, flag, pcache) /* nothing */
+#endif
+
+#define OPTION_COMMAND(type, name, default_value, command_line_option, statement, \
+                       description, flag, pcache)                                 \
+    type name;
+typedef struct _options_t {
+#   include "optionsx.h"
+} options_t;
+
+#undef OPTION_COMMAND
+#undef OPTION_COMMAND_INTERNAL
+
+#ifndef EXPOSE_INTERNAL_OPTIONS
+/* special struct for internal option default values */
+#  define OPTION_COMMAND(type, name, default_value, command_line_option, statement, \
+                         description, flag, pcache) /* nothing */
+#  define OPTION_COMMAND_INTERNAL(type, name, default_value, command_line_option, \
+                                  statement, description, flag, pcache)           \
+    const type name;
+typedef struct _internal_options_t {
+#   include "optionsx.h"
+} internal_options_t;
+#  undef OPTION_COMMAND
+#  undef OPTION_COMMAND_INTERNAL
+#endif
+
+#undef uint_size
+#undef uint_time
+#undef uint_addr
+
+/* For default integer values we use an enum, while for default string values we
+ * use the default_options struct instance that is already being used
+ * for option parsing, and a second one for internal options when !INTERNAL
+ */
+extern const options_t default_options;
+
+#ifndef EXPOSE_INTERNAL_OPTIONS
+/* only needs to contain string options, but no compile-time way to
+ * do that without having OPTION_COMMAND_INTERNAL_STRING()!
+ */
+extern const internal_options_t default_internal_options;
+#endif
+
+#define IS_OPTION_INTERNAL(name) (OPTION_IS_INTERNAL_##name)
+#define IS_OPTION_STRING(name) (OPTION_IS_STRING_##name)
+/* FIXME : figure out a way to handle types here so that we don't have to cast
+ * strings to ints to avoid compiler warnings */
+#define DEFAULT_OPTION_VALUE(name) (IS_OPTION_STRING(name) ? (int)default_options.name : \
+                                    OPTION_DEFAULT_VALUE_##name)
+#ifdef EXPOSE_INTERNAL_OPTIONS
+#  define DEFAULT_INTERNAL_OPTION_VALUE DEFAULT_OPTION_VALUE
+#else
+#  define DEFAULT_INTERNAL_OPTION_VALUE(name) (IS_OPTION_STRING(name) ? \
+                                               default_internal_options.name : \
+                                               OPTION_DEFAULT_VALUE_##name)
+#endif
+/* for non-internal builds we don't support setting the default value
+ * since they are all constants
+ */
+#define SET_DEFAULT_VALUE(name) (dynamo_options.name = DEFAULT_OPTION_VALUE(name))
+
+
+/* checks for incompatible option values */
+/* max==0 means no max and 0 is an ok value */
+/* if option is incompatible, will try to touch up the option
+ * by assigning min to make it valid returns true if changed the 
+ * option value 
+ */
+bool
+check_param_bounds(uint *val, uint min, uint max, char *name);
+
+int 
+options_init(void);
+
+/* only frees a lock, does not destroy any options info other exit
+ * routines might need
+ */
+void
+options_exit(void);
+
+int
+synchronize_dynamic_options(void);
+
+#ifdef WINDOWS
+const options_t *
+get_process_options(HANDLE process_handle);
+#endif
+
+#define SYNCHRONIZE_DYNAMIC_OPTION(x) (                                         \
+        (dynamo_options.dynamic_options && synchronize_dynamic_options()),      \
+         dynamo_options.x)
+
+/* 
+ * if minimal then the options string only contains values different than
+ *  the defaults, otherwise it explicitly lists all options being used
+*/
+void
+get_dynamo_options_string(options_t *options, char *opstr, int len, bool minimal);
+
+/* Fills opstr with a minimal string of only
+ * persistent-cache-affecting options whose effect is >= pcache_effect
+ * and that are different from the defaults.
+ */
+void
+get_pcache_dynamo_options_string(options_t *options, char *opstr, int len,
+                                 op_pcache_t pcache_effect);
+
+bool
+has_pcache_dynamo_options(options_t *options, op_pcache_t pcache_effect);
+
+/****************************************************************************/
+#ifdef NOT_DYNAMORIO_CORE
+void
+set_dynamo_options_defaults(options_t *options);
+int
+set_dynamo_options(options_t *options, char *optstr);
+#else /* !NOT_DYNAMORIO_CORE */
+
+#include "utils.h"
+
+/* are any fragments (potentially) shared? */
+#define SHARED_FRAGMENTS_ENABLED() \
+    (DYNAMO_OPTION(shared_bbs) || DYNAMO_OPTION(shared_traces))
+
+/* are any traces (potentially) private? */
+/* FIXME Fix this if we permit private & shared traces to co-exist */
+#define PRIVATE_TRACES_ENABLED() \
+    (!DYNAMO_OPTION(disable_traces) && !DYNAMO_OPTION(shared_traces))
+
+/* are shared BBs ibl targets? */
+#define SHARED_BB_IB_TARGETS() \
+     (DYNAMO_OPTION(shared_bbs) && DYNAMO_OPTION(bb_ibl_targets))
+
+/* What this answers is
+ * "Are only shared BBs being created and are they valid IB targets?"
+ * It's used when a client wants to know if shared BBs are the only
+ * possible IB targets. So, for example, this is false whenever trace
+ * building is active, even if traces are not being added to the
+ * lookup tables.
+ */
+#define SHARED_BB_ONLY_IB_TARGETS() \
+    (SHARED_BB_IB_TARGETS() && DYNAMO_OPTION(disable_traces))
+
+/* are any shared fragments ibl targets? */
+#define SHARED_IB_TARGETS() \
+    (DYNAMO_OPTION(shared_traces) || SHARED_BB_IB_TARGETS())
+
+/* are any IBT tables (potentially) shared? */
+#define SHARED_IBT_TABLES_ENABLED() \
+    (DYNAMO_OPTION(shared_bb_ibt_tables) || DYNAMO_OPTION(shared_trace_ibt_tables))
+ 
+#define TRACEDUMP_ENABLED()                 \
+     (!DYNAMO_OPTION(disable_traces) &&     \
+      (INTERNAL_OPTION(tracedump_text) ||   \
+       INTERNAL_OPTION(tracedump_binary) || \
+       INTERNAL_OPTION(tracedump_origins)))
+
+#define RUNNING_WITHOUT_CODE_CACHE()        \
+    (IF_HOTP(DYNAMO_OPTION(hotp_only) ||) DYNAMO_OPTION(thin_client))
+
+extern char option_string[];
+extern options_t dynamo_options;
+extern read_write_lock_t options_lock;
+
+/* check for empty is considered safe w/o the read lock
+ * this takes the field name only, and not {DYNAMO,INTERNAL}_OPTION macro,
+ * since those macros will ASSERT_OWN_READWRITE_LOCK(<is_stringtype>, &options_lock)
+ */
+#define IS_STRING_OPTION_EMPTY(op) ((dynamo_options.op)[0] == '\0')
+
+/* single character check for ALL is considered safe w/o the read lock
+ * similarly to IS_STRING_OPTION_EMPTY see above
+ */
+#define IS_LISTSTRING_OPTION_FORALL(op) ((dynamo_options.op)[0] == '*')
+
+#ifdef EXPOSE_INTERNAL_OPTIONS
+#  define IS_INTERNAL_STRING_OPTION_EMPTY(op) IS_STRING_OPTION_EMPTY(op)
+#else
+#  define IS_INTERNAL_STRING_OPTION_EMPTY(op) ((default_internal_options.op)[0] == '\0')
+#endif
+
+/* 0=ret => 1, 1=call* => 2, 2=jmp* => 4 */
+#define COARSE_FILL_IBL_MASK(branch_type) (1 << (branch_type))
+
+/* full access to string requires read lock */
+static inline void
+string_option_read_lock() { 
+    read_lock(&options_lock); 
+}
+static inline void
+string_option_read_unlock() { 
+    read_unlock(&options_lock); 
+}
+
+typedef enum {
+    LIST_NO_MATCH   = 0, /* ensure can use as bool */
+    LIST_ON_DEFAULT = 1,
+    LIST_ON_APPEND  = 2,
+} list_default_or_append_t;
+
+/* compare short_name, usually module name, against a list option of the combined
+ * default option (that could be overridden) and append list that is usually used
+ */
+list_default_or_append_t
+check_list_default_and_append(liststring_t default_list, liststring_t append_list,
+                              const char *short_name);
+
+
+void
+options_make_writable(void);
+
+void
+options_restore_readonly(void);
+
+#endif /* !NOT_DYNAMORIO_CORE */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _OPTIONS_H_ */
diff --git a/core/optionsx.h b/core/optionsx.h
new file mode 100644
index 000000000..58eaf84c4
--- /dev/null
+++ b/core/optionsx.h
@@ -0,0 +1,2606 @@
+/* **********************************************************
+ * Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+
+/*
+ * optionsx.h
+ *
+ * Options and corresponding command line options
+ *
+ */
+
+/* This file is included multiple times 
+   - in globals.h once for structure definition, 
+   - in options.c, four times: for initialization, option parsing and enumeration
+   - TODO: should also be included in the GUI
+*/
+
+/* Client files that include this header should define the following macro
+#define OPTION_COMMAND(type, name, default_value, command_line_option, \
+                       statement, description, dynamic_flag, pcache)
+#include "optionsx.h"
+#undef OPTION_COMMAND
+   dynamic_flag takes the values
+     DYNAMIC - option can be modified externally
+     STATIC  - option cannot be modified externally and changes are ignored
+   pcache takes values from op_pcache_t
+*/
+
+/* PR 330860: the statement of an OPTION_COMMAND is allowed to read
+ * and write to the "options" struct.  It can also change other state
+ * but only inside of a check for the bool "for_this_process".
+ */
+
+/* The liststring_t type is assumed to contain ;-separated values that are
+ * appended to if multiple option instances are specified
+ */
+
+/* Shortcuts for the common cases */
+#define OPTION_DEFAULT(type, name, value, description)                  \
+    OPTION_COMMAND(type, name, value,       #name, {}, description,     \
+                   STATIC, OP_PCACHE_NOP)
+
+#define OPTION_NAME_DEFAULT(type, name, value, option_name, description)\
+    OPTION_COMMAND(type, name, value, option_name, {}, description, \
+                   STATIC, OP_PCACHE_NOP)
+
+#define OPTION_NAME(type, name, option_name, description)               \
+    OPTION_NAME_DEFAULT(type, name,     0, option_name, description)
+
+#define OPTION(type, name, description)                                 \
+    OPTION_DEFAULT(type, name, 0, description)
+
+#define PC_OPTION_DEFAULT(type, name, value, description)                  \
+    OPTION_COMMAND(type, name, value,       #name, {}, description, \
+                   STATIC, OP_PCACHE_GLOBAL)
+
+#define PC_OPTION(type, name, description)                               \
+    PC_OPTION_DEFAULT(type, name, 0, description)
+
+#define PCL_OPTION_DEFAULT(type, name, value, description)                 \
+    OPTION_COMMAND(type, name, value,       #name, {}, description, \
+                   STATIC, OP_PCACHE_LOCAL)
+
+#define PCL_OPTION(type, name, description)                              \
+    PCL_OPTION_DEFAULT(type, name, 0, description)
+
+/* Dynamic option shortcut definitions, 
+   CHECK: if most options end up being dynamic just add a flag to the above */
+#define DYNAMIC_OPTION_DEFAULT(type, name, value, description)          \
+    OPTION_COMMAND(type, name, value, #name, {}, description, DYNAMIC, OP_PCACHE_NOP)
+
+#define DYNAMIC_OPTION(type, name, description)                         \
+    DYNAMIC_OPTION_DEFAULT(type, name, 0, description)
+
+#define DYNAMIC_PCL_OPTION_DEFAULT(type, name, value, description)      \
+    OPTION_COMMAND(type, name, value, #name, {}, description, DYNAMIC, OP_PCACHE_LOCAL)
+
+#define DYNAMIC_PCL_OPTION(type, name, description)                     \
+    DYNAMIC_PCL_OPTION_DEFAULT(type, name, 0, description)
+
+/* Shortcut for defining alias command line options to set another option to a constant value, 
+   Note we can't read the real type/description, and aliases are not transitive */
+#define OPTION_ALIAS(new_alias, real_internal_name, real_value, dynamic_flag, pcache)   \
+    OPTION_COMMAND(bool, new_alias##_aka_##real_internal_name, false, #new_alias,       \
+    { if (options->new_alias##_aka_##real_internal_name)                                \
+        options->real_internal_name = real_value;                                       \
+    },                                                                                  \
+    #new_alias " is an alias for " #real_internal_name, dynamic_flag, pcache) 
+
+/* OPTION_COMMAND_INTERNAL is parsed separately in options.h to 
+   define constants with default values that are used by INTERNAL_OPTION */
+   /* FIXME: dynamic internal options not yet supported */
+#define OPTION_DEFAULT_INTERNAL(type, name, value, description) \
+    OPTION_COMMAND_INTERNAL(type, name, value,       #name, {}, description, \
+                            STATIC, OP_PCACHE_NOP)
+#define OPTION_NAME_INTERNAL(type, name, option_name, description) \
+    OPTION_COMMAND_INTERNAL(type, name,     0, option_name, {}, description, \
+                            STATIC, OP_PCACHE_NOP)
+#define OPTION_INTERNAL(type, name, description) \
+    OPTION_DEFAULT_INTERNAL(type, name,     0, description)
+
+#define PC_OPTION_DEFAULT_INTERNAL(type, name, value, description) \
+    OPTION_COMMAND_INTERNAL(type, name, value,       #name, {}, description, \
+                            STATIC, OP_PCACHE_GLOBAL)
+#define PC_OPTION_INTERNAL(type, name, description) \
+    PC_OPTION_DEFAULT_INTERNAL(type, name,     0, description)
+
+/* option helper macros */
+#define DISABLE_RESET(prefix)                           \
+{                                                       \
+    (prefix)->enable_reset = false;                     \
+    IF_INTERNAL((prefix)->reset_at_fragment_count = 0;) \
+    (prefix)->reset_at_nth_thread = 0;                  \
+    (prefix)->reset_at_switch_to_os_at_vmm_limit = false;\
+    (prefix)->reset_at_vmm_percent_free_limit = 0;      \
+    (prefix)->reset_at_vmm_free_limit = 0;              \
+    (prefix)->reset_at_vmm_full = false;                \
+    (prefix)->reset_at_commit_percent_free_limit = 0;   \
+    (prefix)->reset_at_commit_free_limit = 0;           \
+    (prefix)->reset_every_nth_pending = 0;              \
+    (prefix)->reset_at_nth_bb_unit = 0;                 \
+    (prefix)->reset_at_nth_trace_unit = 0;              \
+    (prefix)->reset_every_nth_bb_unit = 0;              \
+    (prefix)->reset_every_nth_trace_unit = 0;           \
+}
+#define REENABLE_RESET(prefix)                                              \
+{                                                                           \
+    (prefix)->enable_reset = true;                                          \
+    (prefix)->reset_at_vmm_full =                                           \
+                 DEFAULT_OPTION_VALUE(reset_at_vmm_full);                   \
+    (prefix)->reset_every_nth_pending =                                     \
+                 DEFAULT_OPTION_VALUE(reset_every_nth_pending);             \
+    (prefix)->reset_at_switch_to_os_at_vmm_limit =                          \
+                 DEFAULT_OPTION_VALUE(reset_at_switch_to_os_at_vmm_limit);  \
+    (prefix)->reset_at_vmm_percent_free_limit =                             \
+                 DEFAULT_OPTION_VALUE(reset_at_vmm_percent_free_limit);     \
+    (prefix)->reset_at_vmm_free_limit =                                     \
+                 DEFAULT_OPTION_VALUE(reset_at_vmm_free_limit);             \
+    (prefix)->reset_at_commit_percent_free_limit =                          \
+                 DEFAULT_OPTION_VALUE(reset_at_commit_percent_free_limit);  \
+    (prefix)->reset_at_commit_free_limit =                                  \
+                 DEFAULT_OPTION_VALUE(reset_at_commit_free_limit);          \
+}
+/* FIXME: case 9014 reenabling more resets or maybe
+ * reset_at_vmm_threshold reset_at_commit_threshold shouldn't be
+ * disabled in DISABLE_RESET but left under the master option
+ */
+
+#define DISABLE_TRACES(prefix)                          \
+{                                                       \
+    (prefix)->disable_traces = true;                    \
+    (prefix)->enable_traces = false;                    \
+    (prefix)->shared_traces = false;                    \
+    (prefix)->shared_trace_ibl_routine = false;         \
+    (prefix)->bb_ibl_targets = true;                    \
+}
+#define REENABLE_TRACES(prefix)                         \
+{                                                       \
+    (prefix)->disable_traces = false;                   \
+    (prefix)->enable_traces = true;                     \
+    (prefix)->shared_traces = true;                     \
+    (prefix)->shared_trace_ibl_routine = true;          \
+    (prefix)->bb_ibl_targets = false;                   \
+}
+
+#define ENABLE_SECURITY(prefix)                                          \
+{                                                                        \
+    options->native_exec = true;                                         \
+    options->code_origins = true;                                        \
+    options->ret_after_call = true;                                      \
+                                                                         \
+    /* we need to know ib sources */                                     \
+    options->indirect_stubs = true;                                      \
+                                                                         \
+    IF_RCT_IND_BRANCH(options->rct_ind_call = IF_WINDOWS_ELSE            \
+                      (OPTION_ENABLED|OPTION_BLOCK|OPTION_REPORT,        \
+                       OPTION_DISABLED);)                                \
+    IF_RCT_IND_BRANCH(options->rct_ind_jump = IF_WINDOWS_ELSE            \
+                      (OPTION_ENABLED|OPTION_BLOCK|OPTION_REPORT,        \
+                       OPTION_DISABLED);)                                \
+    IF_WINDOWS(options->apc_policy =                                     \
+               OPTION_ENABLED|OPTION_BLOCK|OPTION_REPORT|OPTION_CUSTOM;) \
+}
+
+/* options start here */
+    DYNAMIC_OPTION_DEFAULT(bool, dynamic_options, true, "dynamically update options")
+
+#ifdef EXPOSE_INTERNAL_OPTIONS
+    OPTION_COMMAND_INTERNAL(bool, dummy_version, 0, "version", {
+        if (for_this_process)
+            print_file(STDERR, "<%s>\n", dynamorio_version_string);
+    }, "print version number", STATIC, OP_PCACHE_NOP)
+#endif /* EXPOSE_INTERNAL_OPTIONS */
+
+    PC_OPTION_INTERNAL(bool, nolink, "disable linking")
+    PC_OPTION_DEFAULT_INTERNAL(bool, link_ibl, true, "link indirect branches")
+    OPTION_INTERNAL(bool, tracedump_binary, "binary dump of traces (after optimization)")
+    OPTION_INTERNAL(bool, tracedump_text, "text dump of traces (after optimization)")
+    OPTION_INTERNAL(bool, tracedump_origins, "write out original instructions for each trace")
+    OPTION(bool, syntax_intel, "use Intel disassembly syntax")
+#ifdef EXPOSE_INTERNAL_OPTIONS
+# ifdef PROFILE_LINKCOUNT
+    OPTION(uint, tracedump_threshold, "profile_counts threshold for dumping a trace")
+
+      /* threshold is limited to 32-bit integer even if linkcounts
+       * are 64 bits -- should be fine, few counts reach that high,
+       * purpose of threshold is to weed out the multitudes of
+       * wimpy traces to reduce the size of the trace file
+       */
+# endif
+#endif /* EXPOSE_INTERNAL_OPTIONS */
+    OPTION_INTERNAL(bool, bbdump_tags, "dump tags, sizes, and sharedness of all bbs")
+    OPTION_INTERNAL(bool, gendump, "dump generated code")
+    OPTION_DEFAULT(bool, global_rstats, true, "enable global release-build statistics")
+
+#ifdef DEBUG /* options that only work for debug build */
+    /* we do allow logging for customers for forensics/diagnostics that requires
+     * debug build for more information.
+     * FIXME: somehow restrict info we dump out to prevent IP leakage for INTERNAL=0?
+     * restrict loglevel to <= 2 for INTERNAL=0?
+     */
+    /* log control fields will be kept in dr_statistics_t structure so they can be updated,
+     yet we'll also have the initial value in options_t at the cost of 8 bytes */
+    OPTION_COMMAND(uint, stats_logmask, 0, "logmask", {
+        if (stats != NULL && for_this_process) 
+            stats->logmask = options->stats_logmask; 
+    }, "set mask for logging from specified modules", DYNAMIC, OP_PCACHE_NOP)
+
+    OPTION_COMMAND(uint, stats_loglevel, 0, "loglevel", {
+        if (stats != NULL && for_this_process)
+            stats->loglevel = options->stats_loglevel; 
+    },"set level of detail for logging", DYNAMIC, OP_PCACHE_NOP)
+    /* Note that these are not truly DYNAMIC, and they don't get synchronized before each LOG */
+    OPTION_DEFAULT(uint, checklevel, 0, "level of extra consistency checks (PR 211887)")
+
+    OPTION_DEFAULT_INTERNAL(bool, thread_stats, true, "enable thread local statistics")
+    OPTION_DEFAULT_INTERNAL(bool, global_stats, true, "enable global statistics")
+    OPTION_DEFAULT_INTERNAL(uint, thread_stats_interval, 10000,
+                   "per-thread statistics dump interval in fragments, 0 to disable periodic dump")
+    OPTION_DEFAULT_INTERNAL(uint, global_stats_interval, 5000,
+                   "global statistics dump interval in fragments, 0 to disable periodic dump")
+#  ifdef HASHTABLE_STATISTICS
+    OPTION_DEFAULT_INTERNAL(bool, hashtable_ibl_stats, true, "enable hashtable statistics for IBL routines")
+    /* off by default until non-sharing bug 5846 fixed */
+    OPTION_DEFAULT_INTERNAL(bool, hashtable_ibl_entry_stats, false, "enable hashtable statistics per IBL entry")
+    OPTION_DEFAULT_INTERNAL(uint, hashtable_ibl_study_interval, 50, "dump stats after some IBL entry additions")
+    OPTION_DEFAULT_INTERNAL(bool, stay_on_trace_stats, false, "enable stay on trace statistics")
+    OPTION_DEFAULT_INTERNAL(bool, speculate_last_exit_stats, false, "enable speculative last stay_on_trace_stats")
+#  endif
+#endif /* DEBUG */
+#ifdef KSTATS
+     /* turn on kstats by default for debug builds */
+    OPTION_DEFAULT(bool, kstats, IF_DEBUG_ELSE_0(true), "enable path timing statistics")
+#endif
+
+#ifdef DEADLOCK_AVOIDANCE
+    OPTION_DEFAULT_INTERNAL(bool, deadlock_avoidance, true, "enable deadlock avoidance checks")
+    OPTION_DEFAULT_INTERNAL(uint, mutex_callstack, 0 /* 0 to disable, 4 recommended, MAX_MUTEX_CALLSTACK */,
+                            "collect a callstack up to specified depth when a mutex is locked")
+#endif
+
+#ifdef CALL_PROFILE
+    OPTION(uint, prof_caller, /* 0 to disable, 3-5 recommended */
+           "collect caller data for instrumented routines to this depth")
+#endif
+
+#ifdef HEAP_ACCOUNTING
+    OPTION_DEFAULT_INTERNAL(bool, heap_accounting_assert, true, "enable heap accounting assert")
+#endif
+
+#if defined(LINUX)
+    OPTION_NAME_INTERNAL(bool, profile_pcs, "prof_pcs", "pc-sampling profiling")
+#else
+# ifdef WINDOWS_PC_SAMPLE
+     OPTION_NAME(bool, profile_pcs, "prof_pcs", "pc-sampling profiling")
+# endif
+#endif
+
+#ifdef EXPOSE_INTERNAL_OPTIONS
+# ifdef PROFILE_RDTSC
+    OPTION_NAME_INTERNAL(bool, profile_times, "prof_times", "profiling via measuring time"))
+# endif
+
+# ifdef PROFILE_LINKCOUNT
+    OPTION_NAME_INTERNAL(bool, profile_counts, "prof_counts", "profiling via counters")
+# endif
+
+# ifdef CLIENT_INTERFACE
+    /* FIXME (xref PR 215082): make these external now that our product is our API? */
+    OPTION_DEFAULT_INTERNAL(liststring_t, client_lib, EMPTY_STRING, 
+                            ";-separated string containing client "
+                            "lib paths, IDs, and options")
+
+    /* PR 200418: Code Manipulation API.  This option enables the code
+     * manipulation events and sets some default options.  We can't
+     * afford to check for this in our exported routines, so we allow
+     * ourselves to be used as a utility or standalone library
+     * regardless of this option.
+     */
+    OPTION_COMMAND_INTERNAL(bool, code_api, false, "code_api", {
+        if (options->code_api) {
+            /* PR 202669: larger stack size since we're saving a 512-byte
+             * buffer on the stack when saving fp state.
+             */
+            options->stack_size = 20*1024;
+
+            /* For CI builds we'll disable elision by default since we
+             * expect most CI users will prefer a view of the
+             * instruction stream that's as unmodified as possible.
+             * Also xref PR 214169: eliding calls presents a confusing
+             * view of basic blocks since clients see both the call
+             * and the called function in the same block.  TODO PR
+             * 214169: pass both sides to the client and merge
+             * internally to get the best of both worlds.
+             */
+            options->max_elide_jmp = 0;
+            options->max_elide_call = 0;
+
+            /* indcall2direct causes problems with the code manip API,
+             * so disable by default (xref PR 214051 & PR 214169).
+             * Even if we address those issues, we may want to keep
+             * disabled if we expect users will be confused by this
+             * optimization.
+             */
+            options->indcall2direct = false;
+
+            /* To support clients changing syscall numbers we need to
+             * be able to swap ignored for non-ignored (xref PR 307284)
+             */
+            options->inline_ignored_syscalls = false;
+
+            /* We want to see all code, performance and bugs be darned! 
+             * However the .pexe section thing on Vista is too dangerous so we
+             * leave that on. */
+            options->native_exec = false;
+            IF_WINDOWS_AND_CORE({
+                if (get_os_version() == WINDOWS_VERSION_VISTA) {
+                    /* turn native exec on, but limit it to just the pexe option */
+                    options->native_exec = true;
+                    options->native_exec_managed_code = false;
+                    memset(options->native_exec_default_list, 0,
+                           sizeof(options->native_exec_default_list));
+                }})
+
+            /* Don't randomize dynamorio.dll */
+            IF_WINDOWS(options->aslr_dr = false;)
+
+            /* FIXME PR 215179 on getting rid of this tracing restriction. */
+            options->pad_jmps_mark_no_trace = true;
+        }
+     }, "enable Code Manipulation API", STATIC, OP_PCACHE_NOP)
+
+    /* PR 200418: Probe API.  Note that the code manip API is off by 
+     * default, so -probe_api by itself will give users the lightweight
+     * and more restricted version of the probe API.  If users want the
+     * flexibility to place a probe anywhere, or if they also want code 
+     * manipulation ability, they'll have to enable the code manip API 
+     * as well.  For simplicity, we assume they're willing to accept 
+     * any associated performance hit (e.g., turning off elision), when
+     * enabling the code manip API.
+     */
+    OPTION_COMMAND_INTERNAL(bool, probe_api, false, "probe_api", {
+        if (options->probe_api) { /* else leave alone */
+            IF_HOTP(options->hot_patching = true);
+            IF_HOTP(options->liveshields = false;)
+            IF_GBOP(options->gbop = 0;)
+        }
+     }, "enable Probe API", STATIC, OP_PCACHE_NOP)
+
+    /* PR 326610: provide -opt_speed option.  In future we may want to
+     * expose these separately (PR 225139), and fully support their
+     * robustness esp in presence of clients: for now we consider this
+     * less robust.
+     * FIXME: actually we can't turn elision on b/c it violates our
+     * translation and client trace building assumptions: plus Tim
+     * hit crashes w/ max_elide_call and code_api on in 32-bit linux.
+     * So leaving option here for now but not documenting it.
+     */
+    OPTION_COMMAND(bool, opt_speed, false, "opt_speed", {
+        if (options->opt_speed) {
+            /* We now have -coarse_units and -indirect_stubs off by default,
+             * so elision is the only thing left here, but -indcall2direct
+             * is significant on windows server apps.
+             */
+            /* See comments under -code_api about why these cause problems
+             * with clients: but we risk it here.
+             */
+            options->max_elide_jmp = 16;
+            options->max_elide_call = 16;
+            options->indcall2direct = true;
+        }
+     }, "enable high performance at potential loss in client fidelity",
+        STATIC, OP_PCACHE_NOP)
+
+    /* We turned -coarse_units off by default due to PR 326815 */
+    OPTION_COMMAND(bool, opt_memory, false, "opt_memory", {
+        if (options->opt_memory) {
+            options->coarse_units = true;
+            options->indirect_stubs = true; /* required until we fix PR 213262 */
+        }
+     }, "enable memory savings at potential loss in performance", STATIC, OP_PCACHE_NOP)
+
+    PC_OPTION_INTERNAL(bool, bb_prefixes, "give all bbs a prefix")
+    /* If a client registers a bb hook, we force a full decode.  This option
+     * requests a full decode regardless of whether there is a bb hook. 
+     * Note that there is no way to make this available for non-CI builds
+     * yet be exposed in non-internal CI builds, so we make it CI-only.
+     */
+    OPTION_INTERNAL(bool, full_decode, "decode all instrs to level 3 during bb building")
+# endif
+#endif /* EXPOSE_INTERNAL_OPTIONS */
+
+#ifdef WINDOWS_PC_SAMPLE
+     OPTION_DEFAULT(uint, prof_pcs_DR, 2, "PC profile dynamorio.dll, value is bit shift to use, < 2 or > 32 disables, requires -prof_pcs")
+     OPTION_DEFAULT(uint, prof_pcs_gencode, 2, "PC profile generated code, value is bit shift to use, < 2 or > 32 disables, requires -prof_pcs")
+     OPTION_DEFAULT(uint, prof_pcs_fcache, 30, "PC profile fcache units, value is bit shift to use, < 2 or > 32 disables, requires -prof_pcs")
+     OPTION_DEFAULT(uint, prof_pcs_stubs, 30,
+         "PC profile separate stub units.  Value is bit shift to use: < 2 or > 32 "
+         "disables.  Requires -prof_pcs.")
+     OPTION_DEFAULT(uint, prof_pcs_ntdll, 30, "PC profile ntdll.dll, value is bit shift to use, < 2 or > 32 disables, requires -prof_pcs")
+     OPTION_DEFAULT(uint, prof_pcs_global, 30, "PC profile global, value is bit shift to use, < 8 or > 32 sets to default, requires -prof_pcs")
+     OPTION_DEFAULT(uint, prof_pcs_freq, 10000, "Profiling sample frequency in 100's of nanoseconds, requires -prof_pcs")
+#endif
+
+#ifdef WINDOWS
+    DYNAMIC_OPTION_DEFAULT(uint, msgbox_mask,
+         /* Enable for client debug bulids so DR ASSERTS are visible (xref PR 232783) */
+         IF_CLIENT_INTERFACE_ELSE(IF_DEBUG_ELSE(0xC, 0), 0),
+         "show a messagebox for events, use only for demonstration purposes")
+    OPTION_DEFAULT(uint_time, eventlog_timeout, 10000, "gives the timeout (in ms) to use for an eventlog transaction")
+#endif /* WINDOWS */
+    DYNAMIC_OPTION_DEFAULT(uint, syslog_mask, 0, /* PR 232126: re-enable: SYSLOG_ALL */
+          "log only specified message types")
+          /* Example: -syslog_mask 0x4 - error messages
+           *          -syslog_mask 0x6 - error and warning messages
+           */
+    OPTION_DEFAULT_INTERNAL(uint, syslog_internal_mask, 0, /* PR 232126: re-enable: SYSLOG_ALL */
+          "log only specified internal message types")
+     
+    OPTION_DEFAULT(bool, syslog_init, false, "initialize syslog, unnecessary if correctly installed") /* PR 232126: re-enable for product: true */
+#ifdef WINDOWS
+    DYNAMIC_OPTION(uint, internal_detach_mask, "indicates what events the core should detach from the app on")
+#endif
+    /* Turn on dumpcore by default, even for debug builds, until
+     * we decide to provide our ldmp tools externally.
+     */
+    DYNAMIC_OPTION_DEFAULT(uint, dumpcore_mask,
+                           IF_INTERNAL_ELSE(IF_DEBUG_ELSE_0(0x3ff), 0),
+                           "indicate events to dump core on")
+    IF_LINUX(OPTION_ALIAS(pause_on_error, dumpcore_mask, DUMPCORE_OPTION_PAUSE,
+                          STATIC, OP_PCACHE_NOP))
+    /* Note that you also won't get more then -report_max violation core dumps */
+    DYNAMIC_OPTION_DEFAULT(uint, dumpcore_violation_threshold, 3, "maximum number of violations to core dump on")
+
+    DYNAMIC_OPTION_DEFAULT(bool, live_dump, IF_WINDOWS_ELSE(true, false), "do a live core dump (no outside dependencies) when warranted by the dumpcore_mask")
+#ifdef WINDOWS
+    /* FIXME: make a dynamic option */
+    OPTION_INTERNAL(bool, external_dump, "do a core dump using an external debugger (specified in the ONCRASH registry value) when warranted by the dumpcore_mask (kills process on win2k or w/ drwtsn32)")
+#endif
+
+    /* FIXME: this is really needed in release builds only for our unit tests */
+    /* WARNING: on win32 GUI programs can cause problems */
+    OPTION_DEFAULT(uint, stderr_mask,
+                   /* Enable for client linux debug so ASSERTS are visible (PR 232783) */
+                   IF_CLIENT_INTERFACE_ELSE(IF_LINUX_ELSE(IF_DEBUG_ELSE(
+                       SYSLOG_ALL, SYSLOG_CRITICAL|SYSLOG_ERROR|SYSLOG_WARNING),
+                       SYSLOG_NONE), SYSLOG_NONE),
+                   "show messages onto stderr")
+
+#ifdef LINUX
+    OPTION_DEFAULT(bool, open_tcsh_fds, true, "at startup open several file descriptors "
+                   "to avoid conflicts with hardcoded tcsh assumptions.")
+    /* Xref PR 258731 - options to duplicate stdout/stderr for our or client logging if
+     * application tries to close them. */
+    OPTION_DEFAULT(bool, dup_stdout_on_close, true, "Duplicate stdout for DynamoRIO "
+                   "or client usage if app tries to close it.")
+    OPTION_DEFAULT(bool, dup_stderr_on_close, true, "Duplicate stderr for DynamoRIO "
+                   "or client usage if app tries to close it.")
+
+    /* Xref PR 308654 where calling dlclose on the client lib at exit time can lead
+     * to an app crash. */
+    OPTION_DEFAULT(bool, avoid_dlclose, true, "Avoid calling dlclose from DynamoRIO.")
+
+    /* PR 304708: we intercept all signals for a better client interface */
+    OPTION_DEFAULT(bool, intercept_all_signals, true, "intercept all signals")
+#endif /* LINUX */
+
+    /* Disable diagnostics by default. -security turns it on */
+    DYNAMIC_OPTION_DEFAULT(bool, diagnostics, false, "enable diagnostic reporting")
+
+    OPTION_DEFAULT(uint, max_supported_os_version, 60, 
+        /* case 447, defaults to supporting NT, 2000, XP, 2003, and Vista */
+        "Warn on unsupported (but workable) operating system versions greater than max_supported_os_version")
+
+    OPTION_DEFAULT(uint, os_aslr, 
+        /* case 8225 - for now we disable our own ASLR and persistent caches */
+         0x3 /* OS_ASLR_DISABLE_ASLR_ALL | OS_ASLR_DISABLE_PCACHE_ALL */,
+         "disable selectively pcache or our ASLR when OS provides ASLR on most modules")
+    OPTION_DEFAULT_INTERNAL(uint, os_aslr_version, 60, /* WINDOWS_VERSION_VISTA, Vista RTM+ */
+        "minimal OS version to assume ASLR may be provided by OS")
+
+    /* case 10509: we only use this on <= win2k as it significantly impacts boot time */
+    OPTION_DEFAULT(uint_time, svchost_timeout, 1000, "timeout (in ms) on an untimely unloaded library on Windows NT or Windows 2000") /* case 374 */
+
+    OPTION_DEFAULT(uint_time, deadlock_timeout, 
+        IF_DEBUG_ELSE_0(60)*3*1000, /* disabled in release */
+        "timeout (in ms) before assuming a deadlock had occurred (0 to disable)")
+
+    OPTION_DEFAULT(uint_size, stack_size, IF_X64_ELSE(20*1024,12*1024),
+                   "size of thread-private stacks, in KB")
+    OPTION_DEFAULT(bool, stack_shares_gencode, true,
+        "stack and thread-private generated code share an allocation region")
+
+    OPTION_DEFAULT(uint, spinlock_count_on_SMP, 1000U, "spinlock loop cycles on SMP")
+        /* This is a true spinlock where on an SMP we check for availability in a tight loop instead 
+           of yielding and waiting on a kernel object. 
+           See Robbins for a discussion on a good value for the above - 
+           it should be based on expectations on how long does it take to finish a task. */
+
+     /* INTERNAL options */
+     /* These options should be used with a wrapper INTERNAL_OPTION(opt) which in external */
+     /* builds is turned into the default value, hence all non-default code is dead.  */
+     /* This should be used for experimental internal options */
+    PC_OPTION_INTERNAL(bool, nullcalls, "do not take over")
+    /* replace dynamorio_app_init & _start w/ empty functions */
+
+    OPTION_COMMAND_INTERNAL(uint, trace_threshold, 50U, "trace_threshold",
+          { options->disable_traces = options->trace_threshold == 0; }, 
+          "hot threshold value for trace creation", STATIC, OP_PCACHE_NOP)
+            /* Override the default threshold for hot trace selection. */
+    /* PR 200418: Traces are off by default for the BT API.  We now have
+     * -enable_traces to turn them on; plus, -probe and -security
+     * turn them on.
+     * We mark as pcache-affecting though we have other explicit checks
+     */
+    OPTION_COMMAND(bool, disable_traces, false, "disable_traces", {
+        if (options->disable_traces) { /* else leave alone */
+            DISABLE_TRACES(options);
+        }
+     }, "disable trace creation (block fragments only)", STATIC, OP_PCACHE_GLOBAL)
+    OPTION_COMMAND(bool, enable_traces, true, "enable_traces", {
+        if (options->enable_traces) { /* else leave alone */
+            REENABLE_TRACES(options);
+        }
+     }, "enable trace creation", STATIC, OP_PCACHE_GLOBAL)
+    OPTION_DEFAULT_INTERNAL(uint, trace_counter_on_delete, 0U,
+        "trace head counter will be reset to this value upon trace deletion")
+
+    OPTION_DEFAULT(uint, max_elide_jmp,  16,
+        "maximum direct jumps to elide in a basic block")
+    OPTION_DEFAULT(uint, max_elide_call, 16,
+        "maximum direct calls to elide in a basic block")
+    OPTION_DEFAULT(bool, elide_back_jmps, true, "elide backward unconditional jumps in basic blocks?")
+    OPTION_DEFAULT(bool, elide_back_calls, true, "elide backward direct calls in basic blocks?")
+    /* Xref case 8163, if selfmod_max_writes is too small may cause problems
+     * with pattern reverify from case 4020. Though if too big can cause issues
+     * with max bb size (even 64 may be too big), xref case 7893. */
+    OPTION_DEFAULT(uint, selfmod_max_writes, 5,
+        "maximum write instrs per selfmod fragment")
+    OPTION_DEFAULT(uint, max_bb_instrs, 1024,
+        "maximum instrs per basic block")
+    PC_OPTION_DEFAULT(bool, process_SEH_push,
+        IF_RETURN_AFTER_CALL_ELSE(true, false),
+        "break bb's at an SEH push so we can see the frame pushed on in "
+        "interp, required for -borland_SEH_rct")
+    OPTION_DEFAULT_INTERNAL(bool, check_for_SEH_push, true,
+        "extra debug build checking to ensure -process_SEH_push is catching "
+        "all SEH frame pushes")
+
+    /* PR 361894: if no TLS available, we fall back to thread-private */
+    PC_OPTION_DEFAULT(bool, shared_bbs, IF_HAVE_TLS_ELSE(true, false),
+                      "use thread-shared basic blocks")
+    /* Note that if we want traces off by default we would have to turn
+     * off -shared_traces to avoid tripping over un-initialized ibl tables
+     * PR 361894: if no TLS available, we fall back to thread-private
+     */
+    OPTION_COMMAND(bool, shared_traces, IF_HAVE_TLS_ELSE(true, false),
+                   "shared_traces", {
+        /* for -no_shared_traces, set options back to defaults for private traces: */
+        IF_NOT_X64(options->private_ib_in_tls = options->shared_traces;)
+        options->atomic_inlined_linking = options->shared_traces;
+        options->shared_trace_ibl_routine = options->shared_traces;
+        /* private on by default, shared off until proven stable FIXME */
+        /* we prefer -no_indirect_stubs to inlining, though should actually
+         * measure it: FIXME */
+        if (!options->shared_traces && options->indirect_stubs)
+            options->inline_trace_ibl = true;
+        IF_NOT_X64(IF_WINDOWS(options->shared_fragment_shared_syscalls =
+                              (options->shared_traces && options->shared_syscalls);))
+     }, "use thread-shared traces", STATIC, OP_PCACHE_GLOBAL)
+
+    /* PR 361894: if no TLS available, we fall back to thread-private */
+    OPTION_COMMAND(bool, thread_private, IF_HAVE_TLS_ELSE(false, true),
+                   "thread_private", {
+        options->shared_bbs = !options->thread_private;
+        options->shared_traces = !options->thread_private;
+        if (options->thread_private && options->indirect_stubs)
+            options->coarse_units = true;
+        IF_NOT_X64(options->private_ib_in_tls = !options->thread_private;)
+        options->atomic_inlined_linking = !options->thread_private;
+        options->shared_trace_ibl_routine = !options->thread_private;
+        /* we prefer -no_indirect_stubs to inlining, though should actually
+         * measure it: FIXME */
+        if (options->thread_private && options->indirect_stubs)
+            options->inline_trace_ibl = true;
+        IF_NOT_X64(IF_WINDOWS(options->shared_fragment_shared_syscalls =
+                              (!options->thread_private && options->shared_syscalls);))
+     }, "use thread-private code caches", STATIC, OP_PCACHE_GLOBAL)
+
+    OPTION_DEFAULT_INTERNAL(bool, remove_shared_trace_heads, true,
+        "remove a shared trace head replaced with a trace")
+
+    OPTION_DEFAULT(bool, remove_trace_components, false,
+        "remove bb components of new traces")
+
+    OPTION_DEFAULT(bool, shared_deletion, true, "enable shared fragment deletion")
+    OPTION_DEFAULT(bool, syscalls_synch_flush, true, "syscalls are flush synch points (currently for shared_deletion only)")
+    OPTION_DEFAULT(uint, lazy_deletion_max_pending, 128,
+        "maximum size of lazy shared deletion list before moving to normal list")
+
+    OPTION_DEFAULT(bool, free_unmapped_futures, true,
+        "free futures on app mem dealloc (potential perf hit)")
+
+    /* Default TRUE as it's needed for shared_traces (which is on by default)
+     * and for x64 (PR 244737, PR 215396)
+     * PR 361894: if no TLS available, we fall back to thread-private
+     */
+    /* FIXME: private_ib_in_tls option should go away once case 3701 has all
+     * ibl using tls when any fragments are shared
+     */
+    PC_OPTION_DEFAULT(bool, private_ib_in_tls,
+                      IF_HAVE_TLS_ELSE(true, IF_X64_ELSE(true, false)),
+        "use tls for indirect branch slot in private caches")
+
+    OPTION_INTERNAL(bool, single_thread_in_DR, "only one thread in DR at a time")
+     /* deprecated: we have finer-grained synch that works now */
+
+    OPTION_DEFAULT(bool, separate_private_stubs, true,
+        "place private direct exit stubs in a separate area from the code cache")
+
+    OPTION_DEFAULT(bool, separate_shared_stubs, true,
+        "place shared direct exit stubs in a separate area from the code cache")
+
+    OPTION_DEFAULT(bool, free_private_stubs, true,
+        "free separated private direct exit stubs when not pointed at")
+
+    /* FIXME Freeing shared stubs is currently an unsafe option due to a lack of
+     * linking atomicity. (case 2081). */
+    OPTION_DEFAULT(bool, unsafe_free_shared_stubs, false,
+        "free separated shared direct exit stubs when not pointed at")
+
+    OPTION_DEFAULT_INTERNAL(bool, cbr_single_stub, true,
+        "both sides of a cbr share a single stub")
+
+    /* PR 210990: Improvement is in the noise for spec2k on P4, but is noticeable on
+     * Core2, and on IIS on P4.  Note that this gets disabled if
+     * coarse_units is on (PR 213262 covers supporting it there).
+     */
+    OPTION_COMMAND(bool, indirect_stubs, false, "indirect_stubs", {
+        /* we put inlining back in place if we have stubs, for private,
+         * though should re-measure whether inlining is worthwhile */
+        if (options->thread_private && options->indirect_stubs) {
+            options->inline_trace_ibl = true;
+            /* we also turn coarse on (xref PR 213262) */
+            options->coarse_units = true;
+        }
+     }, "use indirect stubs to keep source information", STATIC, OP_PCACHE_GLOBAL)
+
+    /* control inlining of fast path of indirect branch lookup routines */
+    /* NOTE : Since linking inline_indirect branches is not atomic (see bug 751)
+     * don't turn this on (need atomic linking for trace building in a shared
+     * cache) without turning on atomic_inlined_linking, 
+     * should be ok for traces since ?think? only need atomic unlinking 
+     * there (for flushing), reconsider esp. if we go to a shared trace cache
+     */ 
+    OPTION_DEFAULT(bool, inline_bb_ibl, false, "inline head of ibl routine in basic blocks")
+    /* Default TRUE as it's needed for shared_traces (which is on by default) */
+    /* PR 361894: if no TLS available, we fall back to thread-private */
+    OPTION_DEFAULT(bool, atomic_inlined_linking, IF_HAVE_TLS_ELSE(true, false),
+                   "make linking of inlined_ibls atomic with respect to thread in the cache, required for inline_{bb,traces}_ibl with {bb,traces} being shared, cost is an extra 7 bytes per inlined stub (77 bytes instead of 70)")
+    /* Default FALSE since not supported for shared_traces (which is on by default) */
+    OPTION_DEFAULT(bool, inline_trace_ibl, false, "inline head of ibl routine in traces")
+
+    OPTION_DEFAULT(bool, shared_bb_ibt_tables, false,
+        "use thread-shared BB IBT tables")
+
+    OPTION_DEFAULT(bool, shared_trace_ibt_tables, false,
+        "use thread-shared trace IBT tables")
+
+    OPTION_DEFAULT(bool, ref_count_shared_ibt_tables, true,
+        "use ref-counting to free thread-shared IBT tables prior to process exit")
+
+    /* PR 361894: if no TLS available, we fall back to thread-private */
+    OPTION_DEFAULT(bool, ibl_table_in_tls, IF_HAVE_TLS_ELSE(true, false),
+        "use TLS to hold IBL table addresses & masks")
+
+    OPTION_DEFAULT(bool, bb_ibl_targets, false, "enable BB to BB IBL")
+
+     /* IBL code cannot target both single restore prefix and full prefix frags
+      * simultaneously since the restore of %eax in the former case means that the
+      * 2nd flags restore in the full prefix would be wrong. So if the BB table
+      * is including trace targets, bb_single_restore_prefix and
+      * trace_single_restore_prefix must be the same value. See options.c for
+      * what is currently done for options compatability.
+      */
+    OPTION(bool, bb_ibt_table_includes_traces,
+        "BB IBT tables holds trace targets also")
+
+    PC_OPTION(bool, bb_single_restore_prefix, "BBs use single restore prefixes")
+
+    OPTION(bool, trace_single_restore_prefix, "Traces use single restore prefixes")
+
+    OPTION_DEFAULT_INTERNAL(uint, rehash_unlinked_threshold, 100,
+        "%-age of #unlinked entries to trigger a rehash of a shared BB IBT table")
+
+    OPTION_DEFAULT_INTERNAL(bool, rehash_unlinked_always, false,
+        "always rehash a shared BB IBT table when # unlinked entries > 0")
+
+#ifdef SHARING_STUDY
+    OPTION_COMMAND_INTERNAL(bool, fragment_sharing_study, false,
+        "fragment_sharing_study", {
+        if (options->fragment_sharing_study) { /* else leave alone */
+            options->shared_bbs = false;
+            options->shared_traces = false;
+            /* undo things that the default-on shared_traces turns on */
+            IF_NOT_X64(IF_WINDOWS(options->shared_fragment_shared_syscalls = false;))
+            IF_NOT_X64(options->private_ib_in_tls = false;)
+            options->shared_trace_ibl_routine = false;
+            /* will work w/ wset but let's not clutter creation count stats */
+            options->finite_bb_cache = false;
+            options->finite_trace_cache = false;
+        }
+    }, "counts duplication of bbs and traces among threads (requires all-private fragments)", STATIC, OP_PCACHE_NOP)
+#endif
+
+    OPTION_COMMAND(bool, shared_bbs_only, false, "shared_bbs_only", {
+        if (options->shared_bbs_only) { /* else leave alone */
+            DISABLE_TRACES(options);
+            options->shared_bbs = true;
+            options->private_ib_in_tls = true;
+        }
+    }, "Run in shared BBs, no traces mode", STATIC, OP_PCACHE_NOP)
+
+    /* control sharing of indirect branch lookup routines */
+    /* Default TRUE as it's needed for shared_traces (which is on by default) */
+    /* PR 361894: if no TLS available, we fall back to thread-private */
+    OPTION_DEFAULT(bool, shared_trace_ibl_routine, IF_HAVE_TLS_ELSE(true, false),
+                   "share ibl routine for traces")
+    OPTION_DEFAULT(bool, speculate_last_exit, false, 
+        "enable speculative linking of trace last IB exit")
+
+    OPTION_DEFAULT(uint, max_trace_bbs, 128, "maximum number of basic blocks in a trace")
+
+    /* FIXME: case 8023 covers re-enabling on linux */
+    OPTION_DEFAULT(uint, protect_mask,
+        IF_WINDOWS_ELSE(0x101 /* SELFPROT_DATA_RARE | SELFPROT_GENCODE */, 0/*NYI*/),
+        "which memory regions to protect")
+    OPTION_INTERNAL(bool, single_privileged_thread, "suspend all other threads when one is out of cache")
+
+     /*  1 == HASH_FUNCTION_MULTIPLY_PHI */
+    OPTION_DEFAULT_INTERNAL(uint, alt_hash_func, 1, "use to select alternate hashing functions for all fragment tables except those that have in cache lookups")
+
+    OPTION_DEFAULT(uint, ibl_hash_func_offset, 0, 
+        /* Ignore LSB bits for ret and indjmp hashtables (use ibl_indcall_hash_offset
+         * for indcall hashtables).
+         * This may change the hash function distribution and for offsets
+         * larger than 3 (4 on x64) will add an extra instruction to the IBL hit path.
+         */
+        "mask out lower bits in IBL table hash function")
+
+    /* PR 263331: call* targets on x64 are often 16-byte aligned so ignore LSB 4 */
+    OPTION_DEFAULT(uint, ibl_indcall_hash_offset, IF_X64_ELSE(4, 0), 
+        /* Ignore LSB bits for indcall hashtables. */
+        "mask out lower bits in indcall IBL table hash function")
+
+    OPTION_DEFAULT_INTERNAL(uint, shared_bb_load,
+        /* FIXME: since resizing is costly (no delete) this used to be up to 65 but that 
+         * hurt us lot (case 1677) when we hit a bad hash function distribution - 
+         * My current theory is that since modules addresses are 64KB
+         * aligned we are doing bad on the 16-bit capacity.
+         */
+        55, "load factor percent for shared bb hashtable")
+    OPTION_DEFAULT_INTERNAL(uint, shared_trace_load,
+        /* not used for in-cache lookups */
+        55, "load factor percent for shared trace hashtable")
+    OPTION_DEFAULT_INTERNAL(uint, shared_future_load,
+        /* performance not critical, save some memory */
+        60, "load factor percent for shared future hashtable")
+
+    OPTION_DEFAULT(uint, shared_after_call_load,
+        /* performance not critical */
+        /* we use per-module tables despite the name of the option */
+        80, "load factor percent for after call hashtables")
+
+    OPTION_DEFAULT(uint, global_rct_ind_br_load,
+        /* performance not critical */
+        /* we use per-module tables despite the name of the option */
+        80, "load factor percent for global rct ind branch hashtable")
+
+    OPTION_DEFAULT_INTERNAL(uint, private_trace_load,
+        /* trace table is no longer used for in-cache lookups
+         * we can probably use higher than \alpha = 1/2
+         */
+        55, "load factor percent for private trace hashtables")
+
+    OPTION_DEFAULT(uint, private_ibl_targets_load,
+        /* IBL tables are performance critical, so we use a smaller load.
+         * we started with \alpha = 1/2,
+         * 40 seemed to be the best tradeoff of memory & perf for
+         * crafty, which is of course our primary nemesis.
+         * Increasing to accommodate IIS for private tables.
+         * FIXME: case 4902 this doesn't really control the effective load.
+         */
+        50, "load factor percent for private ibl target trace hashtables")
+
+    OPTION_DEFAULT(uint, private_bb_ibl_targets_load,
+        /* BB IBL tables are not proven to be performance critical trying a 1/2
+         */
+        60, "load factor percent for private ibl hashtables targeting shared bbs")
+
+    OPTION_DEFAULT(uint, shared_ibt_table_trace_init,
+        7, "Shared trace shared IBT tables initial size, log_2 (in bits)")
+
+    OPTION_DEFAULT(uint, shared_ibt_table_bb_init,
+        7, "Shared BB shared IBT tables initial size, log_2 (in bits)")
+
+    OPTION_DEFAULT(uint, shared_ibt_table_trace_load,
+        50, "load factor percent for shared ibl hashtables targeting shared traces")
+
+    OPTION_DEFAULT(uint, shared_ibt_table_bb_load,
+        70, "load factor percent for shared ibl hashtables targeting shared bbs")
+
+    OPTION_DEFAULT(uint, coarse_htable_load,
+        /* there is a separate table per module so we keep the load high */
+        80, "load factor percent for all coarse module hashtables")
+
+    OPTION_DEFAULT(uint, coarse_th_htable_load,
+        /* there is a separate table per module so we keep the load high */
+        80, "load factor percent for all coarse module trace head hashtables")
+
+    OPTION_DEFAULT(uint, coarse_pclookup_htable_load,
+        /* there is a separate table per module so we keep the load high */
+        80, "load factor percent for all coarse module trace head hashtables")
+
+     /* FIXME: case 4814 currently disabled */
+    OPTION_DEFAULT(uint, bb_ibt_groom,
+        /* Should either be 0 to disable grooming or be <= private_bb_ibl_targets_load */
+        0, "groom factor percent for ibl hashtables targeting bb's")
+
+    OPTION_DEFAULT(uint, trace_ibt_groom,
+        /* Should either be 0 to disable grooming or be <= private_ibl_targets_load
+         * since traces are considered hot already, grooming the table may not work as well here
+         */
+        0, "groom factor percent for ibl hashtables targeting traces")
+
+     /* for small table sizes resize is not an expensive operation and we start smaller */
+    OPTION_DEFAULT(uint, private_trace_ibl_targets_init,
+        7, "Trace IBL tables initial size, log_2 (in bits)")
+
+    OPTION_DEFAULT(uint, private_bb_ibl_targets_init,
+        6, "BB IBL tables initial size, log_2 (in bits)")
+
+     /* maximum size of IBL table - table is reset instead of resized when reaching load factor */
+    OPTION_DEFAULT(uint, private_trace_ibl_targets_max,
+        0, /* 0 for unlimited */
+        "Trace IBL tables maximum size, log_2 (in bits)")
+
+    OPTION_DEFAULT(uint, private_bb_ibl_targets_max,
+        0, /* 0 for unlimited */
+        "BB IBL tables maximum size, log_2 (in bits)")
+
+    OPTION_DEFAULT_INTERNAL(uint, private_bb_load,
+        /* Note there are usually no private bbs when using shared_bbs */
+        60, "load factor percent for private bb hashtables")
+    OPTION_DEFAULT_INTERNAL(uint, private_future_load,
+        /* performance not critical, save memory */
+        /* This table is suffering from the worst collisions */
+        65, "load factor percent for private future hashtables")
+
+     /* FIXME: remove this once we are happy with new mutexes */
+    OPTION_DEFAULT_INTERNAL(bool, spin_yield_mutex, false, "use old spin-yield mutex implementation")
+
+     /* FIXME: remove this once we are happy with new rwlocks */
+    OPTION_DEFAULT_INTERNAL(bool, spin_yield_rwlock, false, "use old spin-yield rwlock implementation")
+
+    OPTION_INTERNAL(bool, simulate_contention, "simulate lock contention for testing purposes only")
+
+    OPTION_DEFAULT_INTERNAL(uint_size, initial_heap_unit_size, 32*1024, "initial private heap unit size")
+    OPTION_DEFAULT_INTERNAL(uint_size, initial_global_heap_unit_size, 32*1024, "initial global heap unit size")
+    OPTION_DEFAULT_INTERNAL(uint_size, max_heap_unit_size, 64*1024, "maximum heap unit size")
+    OPTION_DEFAULT(uint_size, heap_commit_increment, 4*1024, "heap commit increment")
+    OPTION_DEFAULT(uint, cache_commit_increment, 4*1024, "cache commit increment")
+
+    /* cache capacity control
+     * FIXME: these are external for now while we study the right way to
+     * tune them for server apps as well as desktop apps
+     * Should we make then internal again once we're happy with the defaults?
+     *
+     * FIXME: unit params aren't that user-friendly -- there's an ordering required:
+     * init < quadruple < max && init < upgrade < max
+     * We complain if that's violated, but if you just set max we make you go and
+     * set the others instead of forcing the others to be ==max.
+     *
+     * FIXME: now that we have cache commit-on-demand we should make the
+     * private-configuration caches larger.  We could even get rid of the
+     * fcache shifting.
+     */
+     /* Note that with guard pages any size of 64KB or bigger will get 8KB less than specified
+      * to make sure we don't waste virtual memory in the address space
+      */
+    OPTION(uint_size, cache_bb_max, "max size of bb cache, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+     /* for default configuration of all-shared we want a tiny bb cache for
+      * our temp private bbs
+      */
+    /* x64 does not support resizing individual cache units so start at 64 */
+    OPTION_DEFAULT(uint_size, cache_bb_unit_init, (IF_X64_ELSE(64,4)*1024),
+        "initial bb cache unit size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+    OPTION_DEFAULT(uint_size, cache_bb_unit_max, (64*1024),
+        "maximum bb cache unit size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+     /* w/ init at 4, we quadruple to 16 and then to 64 */
+    OPTION_DEFAULT(uint_size, cache_bb_unit_quadruple, (64*1024),
+        "bb cache units are grown by 4X until this size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+
+    OPTION(uint_size, cache_trace_max, "max size of trace cache, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+    /* x64 does not support resizing individual cache units so start at 64 */
+    OPTION_DEFAULT(uint_size, cache_trace_unit_init, (IF_X64_ELSE(64,8)*1024),
+        "initial trace cache unit size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+    OPTION_DEFAULT(uint_size, cache_trace_unit_max, (64*1024),
+        "maximum trace cache unit size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+    OPTION_DEFAULT(uint_size, cache_trace_unit_quadruple, (IF_X64_ELSE(64,32)*1024),
+        "trace cache units are grown by 4X until this size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+
+    OPTION(uint_size, cache_shared_bb_max, "max size of shared bb cache, in KB or MB")
+            /* override the default shared bb fragment cache size */
+            /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+    OPTION_DEFAULT(uint_size, cache_shared_bb_unit_init, (64*1024),
+        /* FIXME: cannot handle resizing of cache setting to unit_max, FIXME: should be 32*1024 */
+        "initial shared bb cache unit size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+    OPTION_DEFAULT(uint_size, cache_shared_bb_unit_max, (64*1024),
+        "maximum shared bb cache unit size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+    OPTION_DEFAULT(uint_size, cache_shared_bb_unit_quadruple, (64*1024), /* FIXME: should be 32*1024 */
+        "shared bb cache units are grown by 4X until this size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+
+    OPTION(uint_size, cache_shared_trace_max, "max size of shared trace cache, in KB or MB")
+            /* override the default shared trace fragment cache size */
+            /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+    OPTION_DEFAULT(uint_size, cache_shared_trace_unit_init, (64*1024),
+        /* FIXME: cannot handle resizing of cache setting to unit_max, FIXME: should be 32*1024 */
+        "initial shared trace cache unit size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+    OPTION_DEFAULT(uint_size, cache_shared_trace_unit_max, (64*1024),
+        "maximum shared trace cache unit size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+    OPTION_DEFAULT(uint_size, cache_shared_trace_unit_quadruple, (64*1024), /* FIXME: should be 32*1024 */
+        "shared trace cache units are grown by 4X until this size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+
+    OPTION(uint_size, cache_coarse_bb_max, "max size of coarse bb cache, in KB or MB")
+            /* override the default coarse bb fragment cache size */
+            /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+    OPTION_DEFAULT(uint_size, cache_coarse_bb_unit_init, (64*1024),
+        /* FIXME: cannot handle resizing of cache setting to unit_max, FIXME: should be 32*1024 */
+        "initial coarse bb cache unit size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+    OPTION_DEFAULT(uint_size, cache_coarse_bb_unit_max, (64*1024),
+        "maximum coarse bb cache unit size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+    OPTION_DEFAULT(uint_size, cache_coarse_bb_unit_quadruple, (64*1024), /* FIXME: should be 32*1024 */
+        "coarse bb cache units are grown by 4X until this size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+
+    /* adaptive working set */
+    OPTION_DEFAULT(bool, finite_bb_cache, true, "adaptive working set bb cache management")
+    OPTION_DEFAULT(bool, finite_trace_cache, true, "adaptive working set trace cache management")
+    OPTION_DEFAULT(bool, finite_shared_bb_cache, false, 
+        "adaptive working set shared bb cache management")
+    OPTION_DEFAULT(bool, finite_shared_trace_cache, false, 
+        "adaptive working set shared trace cache management")
+    OPTION_DEFAULT(bool, finite_coarse_bb_cache, false, 
+        "adaptive working set shared bb cache management")
+    OPTION_DEFAULT(uint_size, cache_bb_unit_upgrade, (64*1024),
+        "bb cache units are always upgraded to this size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+    OPTION_DEFAULT(uint_size, cache_trace_unit_upgrade, (64*1024),
+        "trace cache units are always upgraded to this size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+    OPTION_DEFAULT(uint_size, cache_shared_bb_unit_upgrade, (64*1024),
+        "shared bb cache units are always upgraded to this size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+    OPTION_DEFAULT(uint_size, cache_shared_trace_unit_upgrade, (64*1024),
+        "shared trace cache units are always upgraded to this size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+    OPTION_DEFAULT(uint_size, cache_coarse_bb_unit_upgrade, (64*1024),
+        "shared coarse cache units are always upgraded to this size, in KB or MB")
+        /* default size is in Kilobytes, Examples: 4, 4k, 4m, or 0 for unlimited */
+
+    OPTION_DEFAULT(uint, cache_bb_regen, 10,
+        "#regen per #replaced ratio for sizing bb cache")
+    OPTION_DEFAULT(uint, cache_bb_replace, 50,
+        "#regen per #replaced ratio for sizing bb cache")
+    OPTION_DEFAULT(uint, cache_trace_regen, 10,
+        "#regen per #replaced ratio for sizing trace cache")
+    OPTION_DEFAULT(uint, cache_trace_replace, 50,
+        "#regen per #replaced ratio for sizing trace cache")
+    OPTION_DEFAULT(uint, cache_shared_bb_regen, 20,
+        "#regen per #replaced ratio for sizing shared bb cache")
+    OPTION_DEFAULT(uint, cache_shared_bb_replace, 100,
+        /* doesn't mean much for shared sizing, so default 100 makes
+         * regen param a percentage */
+        "#regen per #replaced ratio for sizing shared bb cache")
+    OPTION_DEFAULT(uint, cache_shared_trace_regen, 10, 
+        "#regen per #replaced ratio for sizing shared trace cache")
+    OPTION_DEFAULT(uint, cache_shared_trace_replace, 100,
+        /* doesn't mean much for shared sizing, so default 100 makes
+         * regen param a percentage */
+        "#regen per #replaced ratio for sizing shared trace cache")
+    OPTION_DEFAULT(uint, cache_coarse_bb_regen, 20,
+        "#regen per #replaced ratio for sizing shared coarse cache")
+    OPTION_DEFAULT(uint, cache_coarse_bb_replace, 100,
+        /* doesn't mean much for shared sizing, so default 100 makes
+         * regen param a percentage */
+        "#regen per #replaced ratio for sizing shared coarse cache")
+
+    OPTION_DEFAULT(uint, cache_trace_align, 8, "alignment of trace cache slots")
+    OPTION_DEFAULT(uint, cache_bb_align, 4, "alignment of bb cache slots")
+    OPTION_DEFAULT(uint, cache_coarse_align, 1, "alignment of coarse bb cache slots")
+
+    OPTION_DEFAULT(uint, ro2sandbox_threshold, 10,
+        "#write faults in a region before switching to sandboxing, 0 to disable")
+    OPTION_DEFAULT(uint, sandbox2ro_threshold, 20,
+        "#executions in a sandboxed region before switching to page prot, 0 to disable")
+
+    OPTION_COMMAND(bool, sandbox_writable, false, "sandbox_writable", {
+        if (options->sandbox_writable) {
+            options->sandbox2ro_threshold = 0;
+        }
+    }, "always sandbox writable regions", STATIC, OP_PCACHE_GLOBAL)
+
+    /* FIXME: Do we want to turn this on by default?  If we do end up
+     * making this the default for a single process only, we might
+     * want to make it OP_PCACHE_NOP.
+     */
+    OPTION_COMMAND(bool, sandbox_non_text, false, "sandbox_non_text", {
+        if (options->sandbox_non_text) {
+            options->sandbox2ro_threshold = 0;
+        }
+    }, "always sandbox non-text writable regions", STATIC, OP_PCACHE_GLOBAL)
+
+    /* FIXME: separate for bb and trace shared caches? */
+    OPTION_DEFAULT(bool, cache_shared_free_list, true,
+        "use size-separated free lists to manage empty shared cache slots")
+
+    OPTION_COMMAND(bool, enable_reset, true, "enable_reset", {
+        if (!options->enable_reset) {
+            DISABLE_RESET(options);
+        }
+    }, "separate persistent memory from non-persistent for resets", STATIC,
+    OP_PCACHE_NOP)
+
+    OPTION_DEFAULT_INTERNAL(uint, reset_at_fragment_count, 0,
+        "reset all caches at a certain fragment count")
+    OPTION(uint, reset_at_nth_thread,
+        "reset all caches when the nth thread is explicitly created")
+    /* FIXME - is potentially using up all the os allocation leaving nothing for the
+     * app, however that's prob. better then us spinning (xref 9145). It would be nice
+     * to switch back to resetting when the address space is getting low, but there's
+     * no easy way to figure that out. */
+    OPTION_DEFAULT(bool, switch_to_os_at_vmm_reset_limit, true,
+        "if we hit the reset_at_vmm_*_limit switch to requesting from the os (so we'll "
+        "only actually reset once the os is out and we're at the limit)")
+    OPTION_DEFAULT(bool, reset_at_switch_to_os_at_vmm_limit, true, 
+        "schedule a reset the first (and only the first) time we switch to the os "
+        "allocations from -switch_to_os_at_vmm_reset_limit above")
+    OPTION_DEFAULT(uint, reset_at_vmm_percent_free_limit, 10,
+        "reset all when vmm heap % free is < reset_at_vmm_percent_free (0 disables)")
+    OPTION_DEFAULT(uint_size, reset_at_vmm_free_limit, 0,
+        "reset all when vmm heap has less then reset_at_vmm_free free memory remaining")
+    OPTION_DEFAULT(uint, report_reset_vmm_threshold, 3,
+        "syslog one thrash warning message after this many resets at low vmm heap free")
+    OPTION_DEFAULT(bool, reset_at_vmm_full, true,
+        "reset all caches the first time vmm heap runs out of space")
+    OPTION_DEFAULT(uint, reset_at_commit_percent_free_limit, 0,
+        "reset all less then this % of the commit limit remains free (0 disables)") 
+    OPTION_DEFAULT(uint_size, reset_at_commit_free_limit, (32 * 1024 * 1024),
+         "reset all when less then this much free committable memory remains")
+    OPTION_DEFAULT(uint, report_reset_commit_threshold, 3,
+        "syslog one thrash warning message after this many resets at low commit")
+    OPTION_DEFAULT(uint, reset_every_nth_pending, 35,
+        "reset all caches when pending deletion has this many entries")
+    /* the reset-by-unit options focus on filled units and not created units
+     * to avoid being triggered by new, empty, private units for new threads
+     */
+    OPTION(uint, reset_at_nth_bb_unit,
+        "reset all caches once, when the nth new bb cache unit is created/reused")
+    OPTION(uint, reset_at_nth_trace_unit,
+        "reset all caches once, when the nth new trace cache unit is created/reused")
+    /* these options essentially put a ceiling on the size of the cache */
+    OPTION(uint, reset_every_nth_bb_unit,
+        "reset all caches every nth bb cache unit that is created/reused")
+    OPTION(uint, reset_every_nth_trace_unit,
+        "reset all caches every nth trace cache unit that is created/reused")
+
+    /* virtual memory management */
+    /* See case 1990 for examples where we have to fight for virtual address space with the app */
+    /* FIXME: due to incomplete implementation for detaching we will leave memory behind */
+    OPTION_DEFAULT_INTERNAL(bool, skip_out_of_vm_reserve_curiosity, false,
+        "skip the assert curiosity on out of vm_reserve (for regression tests)")
+    OPTION_DEFAULT(bool, vm_reserve, true, "reserve virtual memory") 
+    /* FIXME - on 64bit probably will need more space */
+    /* For VMX86_SERVER we're in the mmap_text region, which is only 128MB
+     * total, so we scale down our reservation.  We could split up our data
+     * and text into two separate vmheap regions if we do need more room
+     * (that's PR 365331).
+     */
+    OPTION_DEFAULT(uint_size, vm_size, IF_VMX86_ELSE(32*1024*1024, 128*1024*1024),
+        "maximum virtual memory reserved, in KB or MB")
+        /* default size is in Kilobytes, Examples: 262144, 1024k, 256m, up to maximum of 512M */
+     /* FIXME: default value is currently not good enough for sqlserver, for which we need more than 256MB */
+
+    /* We hardcode an address in the mmap_text region here, but verify via
+     * the actual bora/vmkernel/public/user_layout.h in vmk_init().
+     */
+    OPTION_DEFAULT(uint_addr, vm_base, IF_VMX86_ELSE(0x10800000,0x16000000),
+                   "preferred base address hint (ignored for 64-bit linux)")
+     /* FIXME: we need to find a good location with no conflict with DLLs or apps allocations */
+    OPTION_DEFAULT(uint_addr, vm_max_offset, IF_VMX86_ELSE(0x05800000,0x10000000),
+                   "base address maximum random jitter")
+    OPTION_DEFAULT(bool, vm_allow_not_at_base, true, "if we can't allocate vm heap at "
+                   "preferred base (plus random jitter) allow the os to choose where to "
+                   "place it instead of dying")
+#ifdef X64
+    /* For linux, we assume our preferred address gets us <4GB as well: if that
+     * has issues we should make them independent (PR 253624).
+     */
+    OPTION_DEFAULT(bool, heap_in_lower_4GB, true,
+                   "on 64bit request that the dr heap "
+                   "be allocated entirely within the lower 4GB of address space so that "
+                   "it can be accessed directly as a 32bit address. See PR 215395.")
+#endif
+     /* FIXME: the lower 16 bits are ignored - so this here gives us
+      * 12bits of randomness.  Could make it larger if we verify as
+      * collision free the whole range [vm_base, * vm_base+vm_size+vm_max_offset) 
+      */
+    OPTION_INTERNAL(bool, vm_use_last, "use the vm reservation only as a last resort")
+
+    OPTION(uint, silent_oom_mask,
+        /* a mask of the oom_source_t constants, usually 12 ==
+         * (OOM_COMMIT | OOM_EXTEND) on commit limit either when
+         * system running out of pagefile or process hitting job limit
+         */
+        "silently die when out of memory")
+
+     /* FIXME: case 6919 forcing a hardcoded name in the core, this
+      * should rather go into a configuration file.  Since
+      * per-executable no real need for an append list. */
+    OPTION_DEFAULT(liststring_t, silent_commit_oom_list,
+        OPTION_STRING("wmiprvse.exe"),
+        "silently die on reachinig commit limit in these ;-separated executables")
+
+    OPTION_DEFAULT(uint_time, oom_timeout, 
+        5*1000, /* 5s */
+        /* 5 second x 2 -> adds at least 10 seconds before we terminate
+         * when out of memory, but gives us a chance to not die */
+        /* case 2294 pagefile resize, or case 7032 where we hope that
+         * a memory hog on the machine would die by the time we retry.
+         * Applies only to committed memory, for reservations sleeping
+         * is futile (other than artificial ballooning).
+         * 
+         * Note: Two of these timeouts on the same thread's request
+         * and we'll terminate - we will also try freeing up our own
+         * memory after the first timeout.
+         */
+        "short sleep (in ms) and retry after a commit failure")
+
+    /* The follow children options control when we inject into a child. We inject if 
+     * any one of the three says we should, see their descriptions for more details. */
+    DYNAMIC_OPTION(bool, follow_children, "inject into all spawned processes unless preinjector is set up to inject into them or they have app-specific RUNUNDER_OFF")
+    /* not dynamic do to interactions with -early_inject */
+    OPTION_DEFAULT(bool, follow_systemwide, true, "inject into all spawned processes that are configured to run under dr (app specific RUNUNDER_ON, or no app specific and RUNUNDER_ALL in the global key), dangerous without either -early_inject or -block_mod_load_list_default preventing double injection")
+    DYNAMIC_OPTION_DEFAULT(bool, follow_explicit_children, true, "inject into all spawned processes that have app-specific RUNUNDER_EXPLICIT")
+
+    /* FIXME - do we want to make any of the -early_inject* options dynamic?
+     * if so be sure we update os.c:early_inject_location on change etc. */
+    OPTION_DEFAULT(bool, early_inject, true, "inject early")
+    OPTION_DEFAULT(bool, early_inject_map, false, "inject early via map NYI")
+    /* see enum definition is os_shared.h for notes on what works with which
+     * os version */
+    OPTION_DEFAULT(uint, early_inject_location, 4 /* INJECT_LOCATION_LdrDefault */,
+        "where to hook for early_injection, LdrDefault picks the best location for this OS version, see enum for rest")
+    OPTION_DEFAULT(uint_addr, early_inject_address, 0,
+        "specify the address to hook at for INJECT_LOCATION_LdrCustom")
+#ifdef WINDOWS /* probably the surrounding options should also be under this ifdef */
+    OPTION_DEFAULT(pathstring_t, early_inject_helper_dll,
+        OPTION_STRING(INJECT_HELPER_DLL1_NAME),
+        "path to 1st early inject helper dll that is used to auto find LdrpLoadImportModule")
+    OPTION_DEFAULT(pathstring_t, early_inject_helper_name,
+        OPTION_STRING(INJECT_HELPER_DLL2_NAME),
+        "PE name of 2nd early inject helper dll that is used to auto find LdrpLoadImportModule")
+#endif
+    OPTION_DEFAULT_INTERNAL(bool, early_inject_stress_helpers, false,
+        "When early injected and using early_inject_location LdprLoadImportModule, don't use parent's address, instead always use helper dlls to find it")
+    /* FIXME - won't work till we figure out how to get the process parameters
+     * in maybe_inject_into_proccess() in os.c (see notes there) */
+    OPTION_DEFAULT(bool, inject_at_create_process, false,
+        "inject at post create process instead of create first thread, requires early injection")
+     /* Separated from above option since on Vista we have to inject at create
+      * process (there is no separate create first thread).  On vista proc
+      * params are available at create process so everything works out. */
+    OPTION_DEFAULT(bool, vista_inject_at_create_process, true,
+        "if os version is vista inject at post create, requires early injection")
+
+    OPTION_DEFAULT(bool, inject_primary, false,
+        /* case 9347 - we may leave early threads as unknown */
+        "check and wait for injection in the primary thread")
+
+    /* options for controlling the synch_with_* routines */
+    OPTION_DEFAULT(uint, synch_thread_max_loops, 10000, "max number of wait loops in "
+        "synch_with_thread before we give up (UINT_MAX loops forever)")
+    OPTION_DEFAULT(uint, synch_all_threads_max_loops, 10000, "max number of wait loops "
+        "in synch_with_all_threads before we give up (UINT_MAX loops forever)")
+    OPTION_DEFAULT(bool, synch_thread_sleep_UP, true, "for uni-proc machines : if true "
+        "use sleep in synch_with_* wait loops instead of yield")
+    OPTION_DEFAULT(bool, synch_thread_sleep_MP, true, "for multi-proc machines : if "
+        "true use sleep in synch_with_* wait loops instead of yield")
+    OPTION_DEFAULT(uint_time, synch_with_sleep_time, 5, "time in ms to sleep for each "
+        "wait loop in synch_with_* routines")
+#ifdef WINDOWS
+    /* FIXME - only an option since late in the release cycle - should always be on */
+    OPTION_DEFAULT(bool, suspend_on_synch_failure_for_app_suspend, true, "if we fail "
+        "to synch with a thread for an app suspend, suspend anyways to preserved the "
+        "apps suspend count")
+#endif
+
+    /* see in case 2520 why this is off by default for Windows */
+    OPTION_DEFAULT(bool, ignore_syscalls, IF_WINDOWS_ELSE(false, true),
+                   "ignore system calls that do not need to be intercepted")
+    /* Whether we inline ignoreable syscalls inside of bbs (xref PR 307284) */
+    OPTION_DEFAULT(bool, inline_ignored_syscalls, true,
+                   "inline ignored system calls in the middle of bbs")
+#ifdef LINUX
+    OPTION_DEFAULT(bool, hook_vsyscall, true, "hook vdso vsyscall if possible")
+    /* PR 356503: workaround to allow clients to make syscalls */
+    OPTION_ALIAS(sysenter_is_int80, hook_vsyscall, false, STATIC, OP_PCACHE_GLOBAL)
+#endif
+
+    /* These should be made internal when sufficiently tested */
+#ifdef WINDOWS
+    /* FIXME There's gotta be a better name for this. */
+    OPTION_DEFAULT(bool, ignore_syscalls_follow_sysenter, true,
+        "for ignore_syscalls, continue interp after the sysenter")
+    /* Optimize syscall handling for syscalls that don't need to be intercepted
+     * by DR by executing them using shared syscall. */
+    OPTION_DEFAULT(bool, shared_syscalls, true, "syscalls that do not need to be intercepted are executed by shared syscall")
+
+    /* Default TRUE as it's needed for shared_traces (which is on by default) */
+    /* PR 361894: if no TLS available, we fall back to thread-private */
+    OPTION_DEFAULT(bool, shared_fragment_shared_syscalls, IF_HAVE_TLS_ELSE(true, false),
+        "enable fragments that use shared syscall to be share-able")
+
+    /* Optimize shared syscall handling by using a faster code sequence if
+     * possible. This currently works only w/-disable_traces. */
+    OPTION_INTERNAL(bool, shared_syscalls_fastpath,
+        "use a faster version of shared syscall")
+    /* This option only applies when shared_syscalls is 'true'. 'true' means that
+     * only ignorable syscalls can be optimized. 'false' means that a much broader
+     * set of syscalls can be optimized -- any syscall that DR doesn't need to
+     * intercept, as identified in the syscall_requires_action array in win32/os.c.
+     */
+    OPTION_INTERNAL(bool, shared_eq_ignore,
+        "use ignorable syscall classification for shared_syscalls")
+
+    /* We mark as pcache-affecting though we have other explicit checks */
+    PC_OPTION_DEFAULT(uint, tls_align, 1, /* case 6770: for disabling alignment */
+        /* 0 - use processor cache line */
+        /* 1, 2, 4 - no alignment
+         * 32 - Pentium III, Pentium M cache line
+         * 64 - Pentium 4 cache line
+         */
+        /* FIXME: if we ever change our -tls_align default from 1 we should
+         * consider implications on platform-independence of persisted caches
+         */
+        "TLS slots preferred alignment")
+    /* We mark as pcache-affecting though we have other explicit checks */
+    PC_OPTION_DEFAULT(uint, tls_flags, 1|2 /* TLS_FLAG_BITMAP_TOP_DOWN | 
+                                          * TLS_FLAG_CACHE_LINE_START */, 
+         "TLS allocation choices")
+#endif /* WINDOWS */
+    OPTION_DEFAULT(bool, guard_pages, true, "add guard pages to our heap units")
+
+#ifdef PROGRAM_SHEPHERDING
+    /* PR 200418: -security_api just turns on the bits of -security needed for the
+     * Memory Firewall API */
+    OPTION_COMMAND_INTERNAL(bool, security_api, false, "security_api", {
+        if (options->security_api) {
+            ENABLE_SECURITY(options);
+        }
+     }, "enable Security API", STATIC, OP_PCACHE_NOP)
+
+    /* PR 200418: program shepherding is now runtime-option-controlled. Note - not
+     * option_command_internal so that we can use for VPS release builds. */
+    OPTION_COMMAND(bool, security, false, "security", {
+        if (options->security) {
+
+            options->diagnostics = true;
+
+            /* xref PR 232126 */
+            options->syslog_mask = SYSLOG_ALL;
+            options->syslog_init = true;
+            IF_INTERNAL(options->syslog_internal_mask = SYSLOG_ALL;)
+
+            /* VPS had -use_moduledb by default (and then disabled with -staged) */
+
+            ENABLE_SECURITY(options);
+
+            /* memory wins over gcc/gap perf issues (PR 326815)
+             * (ENABLE_SECURITY turns on -indirect_stubs for us)
+             */
+            options->coarse_units = true;
+        }
+     }, "enable Memory Firewall security checking", STATIC, OP_PCACHE_NOP)
+
+    /* attack handling options */
+    DYNAMIC_PCL_OPTION(bool, detect_mode, "only report security violations - will execute attackers code!")
+    OPTION_COMMAND(uint, detect_mode_max, 0, "detect_mode_max", 
+                   { options->detect_mode = true; }, 
+                  "max number of security violations to allow in detect_mode - will revert to "
+                  "next higher-priority handling option after the max", DYNAMIC, 
+                  OP_PCACHE_NOP)
+    DYNAMIC_OPTION(bool, diagnose_violation_mode, "on a security violations, report whether a trampoline")
+    DYNAMIC_OPTION_DEFAULT(uint, report_max, 20, "max number of security violations to report, (0 is infinitely many)")
+
+    /* alternatives to kill application */
+    DYNAMIC_OPTION(bool, kill_thread, "kill offending thread only, WARNING: application may hang")
+    OPTION_COMMAND(uint, kill_thread_max, 10, "kill_thread_max", 
+                   { options->kill_thread = true; }, 
+                   "max number of threads to kill before killing process", DYNAMIC, 
+                   OP_PCACHE_NOP)
+    DYNAMIC_OPTION(bool, throw_exception, "throw exception on security violations, WARNING: application may die")
+    OPTION_COMMAND(uint, throw_exception_max, 10, "throw_exception_max",
+                   { options->throw_exception = true; }, 
+                   "max number of exceptions before killing thread or process", DYNAMIC,
+                   OP_PCACHE_NOP)
+    DYNAMIC_OPTION_DEFAULT(uint, throw_exception_max_per_thread, 10, "max number of exceptions per single thread")
+    DYNAMIC_OPTION(uint_time, timeout, "timeout value to throttle down an attack")
+    /* FIXME: should this apply to the whole process or the attacked thread only? */
+#  ifdef SIMULATE_ATTACK
+    DYNAMIC_OPTION_DEFAULT(pathstring_t, simulate_at, EMPTY_STRING, "fragment count list for simulated attacks")
+#  endif
+
+    /* case 280: remove futureexec areas for selfmod regions.
+     * Disabled for now since -sandbox2ro_threshold invalidates its assumptions
+     * (case 8167)
+     */
+    OPTION_DEFAULT(bool, selfmod_futureexec, true,
+        "leave selfmod areas on the future-exec list")
+
+    /* our default policies --- true by default, so you'd use no_ to turn them off
+     * example: -no_executable_if_flush
+     * note that if_flush and if_alloc use the future list, and so their regions
+     * are considered executable until de-allocated -- even if written to!
+     * N.B.: any changes in defaults must also change the -X command below!
+     */
+    /* N.B.: case 9799: any changes in policy or exemption default values may
+     * require changing from PC_ to PCL_, and changing whether we mark modules as
+     * exempted!
+     */
+    OPTION(bool, code_origins, "check that code origins meet security policies")
+#  ifdef WINDOWS
+    PC_OPTION_DEFAULT(bool, executable_if_flush, true, "allow execution after a region has been NtFlushInstructionCache-d")
+    PC_OPTION(bool, executable_after_load, "allow execution from region marked x but modified during load time (normal behavior for relocation or rebinding)")
+    PC_OPTION_DEFAULT(bool, emulate_IAT_writes, true, "keep IAT non-w, emulate writes there")
+    PC_OPTION_DEFAULT(bool, unsafe_ignore_IAT_writes, false,
+       "ignore IAT writes by the loader, assuming nothing else writes at the same time")
+#  endif
+    PC_OPTION_DEFAULT(bool, executable_if_rx_text, true, "allow execution from any rx portion of the text section, subsumes -executable_after_load")
+    PC_OPTION_DEFAULT(bool, executable_if_alloc, true, "allow execution from certain regions marked x at allocation time")
+    PC_OPTION_DEFAULT(bool, executable_if_trampoline, true, "allow execution from pattern-matched trampoline blocks")
+    PC_OPTION_DEFAULT(bool, executable_if_hook, true, "allow execution from text section briefly marked rwx")
+
+    /* specific trampoline exemptions */
+    PC_OPTION_DEFAULT(bool, trampoline_dirjmp, true, "allow direct jmp trampoline")
+    PC_OPTION_DEFAULT(bool, trampoline_dircall, true, "allow direct call trampoline")
+    /* not needed w/ native_exec but may be needed in the future */
+    PC_OPTION(bool, trampoline_com_ret, "allow .NET COM method table ret trampoline")
+    /* allow simple hook displacement of original code */
+    PC_OPTION_DEFAULT(bool, trampoline_displaced_code, true,
+        "allow hook-displaced code trampoline")
+    PC_OPTION_DEFAULT(bool, executable_if_driver, true, 
+        "allow execution from a kernel-mode address (case 9022)")
+    PC_OPTION_DEFAULT(bool, driver_rct, true, 
+       /* marked as VM_DRIVER_ADDRESS */
+        "allow any RCT if source is from a kernel-mode address (case 9022/9096)")
+
+    /* methods to loosen policies */
+    PCL_OPTION(bool, executable_if_text,  "allow execution from text sections of modules, subsumes -executable_if_rx_text")
+    PCL_OPTION(bool, executable_if_dot_data,  "allow execution from .data sections of modules")
+    PCL_OPTION(bool, executable_if_dot_data_x,  "allow execution from .data sections of modulesif marked x")
+    /* ..x means care about execute permission, but not read or write; 
+     * .-x means care about execute with no write permission. See case 3287.
+     */
+    PCL_OPTION(bool, executable_if_x, "allow execution from regions marked ..x")
+    PCL_OPTION(bool, executable_if_rx, "allow execution from regions marked .-x")
+    PCL_OPTION(bool, executable_if_image, "allow execution from any mapped section from an executable or library image")
+    OPTION(bool, executable_stack, "allow execution from the stack")
+    OPTION(bool, executable_heap, "allow execution from the heap")
+    /* Obfuscated options how to suppress security violations of types A and B */
+    OPTION_ALIAS(A, executable_stack, true, STATIC, OP_PCACHE_NOP/*no effect on images*/)
+    OPTION_ALIAS(B, executable_heap, true, STATIC, OP_PCACHE_NOP/*no effect on images*/)
+    /* each exempt list has a corresponding boolean for easy disabling */
+    PCL_OPTION_DEFAULT(bool, exempt_text, true, "allow execution from exempt text sections")
+    PCL_OPTION_DEFAULT(liststring_t, exempt_text_list, EMPTY_STRING, "allow execution from text sections of these ;-separated modules")
+    PC_OPTION_DEFAULT(liststring_t, exempt_mapped_image_text_default_list, 
+        /* case 9385 - loaded in unknown thread  */
+        OPTION_STRING("LVPrcInj.dll"), 
+        "allow execution from text sections in MEM_IMAGE mappings"
+        " of these ;-separated modules, default")
+    PCL_OPTION_DEFAULT(liststring_t, exempt_mapped_image_text_list, EMPTY_STRING, 
+        "allow execution from text sections in MEM_IMAGE mappings"
+        " of these ;-separated modules, append")
+
+    PCL_OPTION_DEFAULT(bool, exempt_dot_data, true, "allow execution from exempt .data sections")
+    /* xref case 4244 on SM2USER.dll */ 
+    /* FIXME case 9799: since default not split out, anything on this list
+     * by default will not have shared pcaches for any process w/ ANY non-default
+     * exemption lists */
+    PCL_OPTION_DEFAULT(liststring_t, exempt_dot_data_list, OPTION_STRING("SM2USER.dll"), "allow execution from .data sections of these ;-separated modules")
+    PCL_OPTION_DEFAULT(bool, exempt_dot_data_x, true, "allow execution from exempt .data sections if marked x")
+    /* Case 7345: allow all kdb*.dlls for shark; later on (porpoise?) implement
+     * -executable_if_dot_data_rx, turn it on by default and set this exempt 
+     * list to empty.
+     */
+    /* FIXME case 9799: since default not split out, anything on this list
+     * by default will not have shared pcaches for any process w/ ANY non-default
+     * exemption lists */
+    PCL_OPTION_DEFAULT(liststring_t, exempt_dot_data_x_list, OPTION_STRING("kbd??.dll;kbd???.dll;kbd????.dll;kbd?????.dll"), "allow execution from .data sections of these ;-separated modules if marked x")
+    PCL_OPTION_DEFAULT(bool, exempt_image, true, "allow execution from exempt image modules")
+    PCL_OPTION_DEFAULT(liststring_t, exempt_image_list, EMPTY_STRING, "allow execution from anywhere in the image of these ;-separated modules")
+    OPTION_DEFAULT(bool, exempt_dll2heap, true, "allow execution in heap first targeted by exempt modules")
+    OPTION_DEFAULT(liststring_t, exempt_dll2heap_list, EMPTY_STRING, "allow execution in heap first targeted by these ;-separated modules")
+    OPTION_DEFAULT(bool, exempt_dll2stack, true, "allow execution in stack first targeted by exempt modules")
+    OPTION_DEFAULT(liststring_t, exempt_dll2stack_list, EMPTY_STRING, "allow execution in stack first targeted by these ;-separated modules")
+    DYNAMIC_PCL_OPTION_DEFAULT(bool, exempt_threat, true, "allow exempt threat ids")
+    DYNAMIC_PCL_OPTION_DEFAULT(liststring_t, exempt_threat_list, EMPTY_STRING, "silently allow these ;-separated threat ids, ? wildcards allowed")
+    DYNAMIC_OPTION_DEFAULT(liststring_t, silent_block_threat_list, EMPTY_STRING, "silently block these ;-separated threat ids, ? wildcards allowed")
+    /* note that exempt_threat_list takes precedence over silent_block_threat_list */
+
+#  ifdef RETURN_AFTER_CALL
+    PC_OPTION_DEFAULT(bool, ret_after_call, false, "return after previous call instructions only")
+
+    /* Obfuscated option how to suppress security violations of type C */
+    OPTION_ALIAS(C, ret_after_call, false, STATIC, OP_PCACHE_GLOBAL)
+
+    PC_OPTION_DEFAULT(bool, vbjmp_allowed, true, "allow execution of VB direct jmp via ret")
+    PC_OPTION_DEFAULT(bool, vbpop_rct, true, "allow execution of VB pop via ret")
+    PC_OPTION_DEFAULT(bool, fiber_rct, true, "allow execution of fiber initialization")
+    PC_OPTION_DEFAULT(bool, mso_rct, true, "allow execution of MSO continuations") /* FIXME: no clue what it is */
+    PC_OPTION_DEFAULT(bool, licdll_rct, true, "allow execution of licdll obfuscated call") /* FIXME: no clue what it is */
+    PC_OPTION_DEFAULT(bool, seh_rct, true, "allow execution of SEH ret constructs")
+    /* xref case 5752 */
+    PC_OPTION_DEFAULT(bool, borland_SEH_rct, true,
+        "allow execution of borland SEH constructs")
+    /* case 7317, from SQL2005 case 6534
+     * off by default as case 7266 has us currently running these dlls natively
+     */
+    PC_OPTION(bool, pushregret_rct, "allow execution of push;ret constructs")
+    /* PR 276529: ntdll64!RtlRestoreContext uses iret as a general jmp* */
+    PC_OPTION_DEFAULT(bool, iret_rct, IF_X64_ELSE(true, false),
+        "allow ntdll64!RtlRestoreContext iret")
+    /* case 7319, from SQL2005 cases 6541 and 6534 
+     * off by default as case 7266 has us currently running these dlls natively
+     */
+    PC_OPTION(bool, xdata_rct, "allow ret to .xdata NtFlush targets")
+    PC_OPTION_DEFAULT(bool, exempt_rct, true, "allow rct in exempt modules")
+    /* case 9725 slsvc.exe->heap .C (software licensing service on Vista) 
+     * FIXME slsvc.exe is also in exempt_rct_to_default_list for slsvc.exe -> slsvc.exe
+     * .C violations which would be covered here if it weren't for case 285. */
+    PC_OPTION_DEFAULT(liststring_t, exempt_rct_default_list,
+        OPTION_STRING("dpcdll.dll;licdll.dll;mso.dll;winlogon.exe;sysfer.dll;slsvc.exe"),
+        "allow rct within these ;-separated modules or to DGC")
+    PCL_OPTION_DEFAULT(liststring_t, exempt_rct_list, EMPTY_STRING,
+        "allow rct within these ;-separated modules or to DGC, append")
+
+    /* exempt_rct_from_{default,}_list are less strict than exempt_rct_list */
+    PCL_OPTION_DEFAULT(liststring_t, exempt_rct_from_default_list, 
+        EMPTY_STRING, "allow rct from these ;-separated modules")
+    PCL_OPTION_DEFAULT(liststring_t, exempt_rct_from_list, 
+        EMPTY_STRING, "allow rct from these ;-separated modules, append")
+
+    /* exempt_rct_to_{default,}_list are less strict than exempt_rct_list */
+     /* case 1690 dpcdll.dll, licdll.dll; case 1158 mso.dll */
+     /* case 1214 winlogon.exe */
+     /* case 5912 .F sysfer.dll, 
+      * FIXME: should be in exempt_rct_default_list, case 285 */
+     /* case 6076 blackd.exe: .F iss-pam1.dll ---> iss-pam1.dll, 
+      * FIXME: should be in exempt_rct_default_list, case 285 */
+     /* case 5051 w3wp.exe: .C jmail.dll ---> jmail.dll, 
+      * FIXME: should be in exempt_rct_default_list, case 285 */
+     /* case 6412, 7659: .E msvbvm50.dll;msvbvm60.dll;vbe6.dll */
+     /* case 9385 LVPrcInj.dll loaded by unknown thread */
+     /* case 9716 slc.dll (software licensing dll) on Vista .C */
+     /* case 9724 slsvc.exe->slsvc.exe (software licensing service) on Vista .C
+      * FIXME: should be only in exempt_rct_default_list, case 285 */
+    PC_OPTION_DEFAULT(liststring_t, exempt_rct_to_default_list, 
+        OPTION_STRING("dpcdll.dll;licdll.dll;mso.dll;winlogon.exe;sysfer.dll;iss-pam1.dll;jmail.dll;msvbvm50.dll;msvbvm60.dll;vbe6.dll;LVPrcInj.dll;slc.dll;slsvc.exe"),
+        "allow rct to these ;-separated modules")
+    PCL_OPTION_DEFAULT(liststring_t, exempt_rct_to_list, 
+        EMPTY_STRING, "allow rct to these ;-separated modules, append")
+
+     /* case 2144 - FIXME: should we have a general exemption for .A and .B as well? */
+     /* note we want to silently handle a .C - and to preserve compatibility with previous
+      * releases we use the default attack handling (e.g. kill thread for services)
+      */
+    OPTION_DEFAULT(uint, rct_ret_unreadable, OPTION_ENABLED|OPTION_BLOCK|OPTION_NO_HANDLING|OPTION_NO_REPORT,
+        "alternative handling of return targets in unreadable memory")
+    /* note indirect call and indirect jump will always just throw an exception */
+
+     /* case 5329 - leaving for bug-compatibility with previous releases */
+    PC_OPTION(bool, rct_sticky, 
+         "leaves all RCT tables on unmap, potential memory leak")
+     /* case 9331 - FIXME: still leaking on DGC */
+    PC_OPTION_DEFAULT(bool, rac_dgc_sticky, true,
+         "leaves all RAC tables from DGC, potential memory leak")
+
+    PC_OPTION_DEFAULT(uint, rct_cache_exempt, 1, /* RCT_CACHE_EXEMPT_MODULES */
+        "whether to cache exempted addresses, 0 never, 1 only in modules, 2 always")
+
+    PC_OPTION_DEFAULT(uint, rct_section_type, 0x20, /* IMAGE_SCN_CNT_CODE */
+        "bitflag to enable RCT checks on module code 0x20,data 0x40, or uninitialized sections 0x80")
+    PC_OPTION_DEFAULT(uint, rct_section_type_exclude,
+        0x80000020, /* IMAGE_SCN_MEM_WRITE|IMAGE_SCN_CNT_CODE, xref case 8360 */
+        "TESTALL bitflag to disable RCT checks for specific module sections sections that are matched by rct_section_type")
+
+    PC_OPTION_DEFAULT(bool, rct_modified_entry, true,
+        "if not within module, lookup image entry point"
+        "in LDR list for already mapped modules, and at MapViewOfSection for late") 
+     /* expected to be overwritten by mscoree.dll */
+
+#   ifdef RCT_IND_BRANCH
+    /* case 286 */
+    /* FIXME: currently OPTION_ENABLED doesn't do much yet, see rct_process_module_mmap() for details */
+    /* FIXME: not yet supported on Linux (case 4983) */
+    PC_OPTION_DEFAULT(uint, rct_ind_call, OPTION_DISABLED,
+        "indirect call policy: address taken instructions only")
+    PC_OPTION_DEFAULT(uint, rct_ind_jump, OPTION_DISABLED,
+        "indirect jump policy: address taken or return targets")
+
+    PC_OPTION_DEFAULT(bool, rct_analyze_at_load, true,
+        "analyze modules for ind branch targets at module load time")
+
+    PC_OPTION_DEFAULT(bool, rct_reloc, true,
+        "use relocation information to find references")
+
+    /* PR 215408: even when we have reloc info, we need to scan for rip-rel lea,
+     * but only in modules that executed code we didn't see.
+     * This option controls whether we proactively scan or only rely on our
+     * scan-on-violation (PR 277044/277064).
+     */
+    PC_OPTION_DEFAULT(bool, rct_scan_at_init, IF_X64_ELSE(true, false),
+        "scan modules present at inject time for rip-rel lea even when relocs are present")
+    /* PR 275723: RVA-table-based switch statements.  Not on for Linux b/c we
+     * don't have per-module RCT tables there.
+     */
+    PC_OPTION_DEFAULT(bool, rct_exempt_intra_jmp,
+        IF_X64_ELSE(IF_WINDOWS_ELSE(true, false), false),
+        "allow jmps to target any intra-module address")
+
+    /* Obfuscated options to suppress security violations of types E and F */
+    OPTION_ALIAS(E, rct_ind_call, OPTION_DISABLED, STATIC, OP_PCACHE_GLOBAL)
+    OPTION_ALIAS(F, rct_ind_jump, OPTION_DISABLED, STATIC, OP_PCACHE_GLOBAL)
+#   endif /* RCT_IND_BRANCH */
+#  endif /* RETURN_AFTER_CALL */
+
+     /* FIXME: there must be a way to make sure that new security
+      * options are added here; and less importantly make sure that
+      * when options are turned off by default they are taken out
+      */
+    OPTION_COMMAND(bool, X, false, "X", {
+IF_RETURN_AFTER_CALL(options->ret_after_call = false;)
+IF_WINDOWS(options->executable_if_flush = false;)
+        options->executable_if_alloc = false; 
+        options->executable_if_trampoline = false; 
+        options->executable_if_hook = false; 
+        options->executable_if_x = true;
+IF_RCT_IND_BRANCH(options->rct_ind_call = OPTION_DISABLED;)
+IF_RCT_IND_BRANCH(options->rct_ind_jump = OPTION_DISABLED;)
+    }, "duplicate Microsoft's nx: allow x memory only and don't enforce RCT", DYNAMIC, 
+    OP_PCACHE_GLOBAL/*since is not only a relaxation*/)
+
+#endif /* PROGRAM_SHEPHERDING */
+
+    OPTION_DEFAULT(bool, enable_block_mod_load, true,
+         "switch for enabling the block module from being loaded feature, if enabled the modules to block from loading are specified by the block_mod_load_list[_default] options") 
+    /* dynamorio.dll : on this list to prevent non early_inject follow
+     * children from double injecting if the process is already under dr */
+     /* entapi.dll;hidapi.dll : case 2871 for Entercept/VirusScan */
+     /* Caution: add to this list only DLLs whose callers don't crash
+      * if LdrLoadDll calls fail, e.g. do not use the result for
+      * GetProcAddress.
+      */
+    OPTION_DEFAULT(liststring_t, block_mod_load_list_default,
+        OPTION_STRING("dynamorio.dll;entapi.dll;hidapi.dll"),
+        "if -enable_block_mod_load block the loading (at LdrLoadDll) of the following ;-separated modules, note that since this is blocking at LdrLoadDll the module match will be based on the filename of module being loaded NOT the PE name (which is used by most other options)")
+    OPTION_DEFAULT(liststring_t, block_mod_load_list, EMPTY_STRING,
+        "if -enable_block_mod_load, block the loading (at LdrLoadDll) of the following ;-separated modules, note that since this is blocking at LdrLoadDll the module match will be based on the filename of module being loaded NOT the PE name (which is used by most other options)")
+
+    OPTION_DEFAULT(uint, handle_DR_modify, 1 /*DR_MODIFY_NOP*/, "specify how to handle app attempts to modify DR memory protection: either halt with an error, turn into a nop (default), or return failure to the app")
+
+    OPTION_DEFAULT(uint, handle_ntdll_modify, 1 /*DR_MODIFY_NOP*/, "specify how to handle app attempts to modify ntdll code: either halt with an error, turn into a nop (default), or return failure to the app")
+
+    /* generalized DR_MODIFY_NOP for customizable list of modules */
+    OPTION_DEFAULT(liststring_t, patch_proof_default_list,
+        EMPTY_STRING, "ignore protection changes and writes to text of ;-separated module list, or * for all")
+        /* note '*' has to be at first position to mean all modules */
+    OPTION_DEFAULT(liststring_t, patch_proof_list,
+        EMPTY_STRING, "ignore protection changes and writes to text of ;-separated module list, append, or * for all")
+        /* note '*' has to be at first position to mean all modules */
+
+    PC_OPTION_DEFAULT(bool, use_moduledb, false, "activate module database")
+    OPTION_ALIAS(staged, use_moduledb, false, STATIC, OP_PCACHE_GLOBAL) /* xref case 8924 */
+    /* FIXME - can't handle a company name with a ; in it */
+    PC_OPTION_DEFAULT(liststring_t, whitelist_company_names_default, OPTION_STRING(COMPANY_LONG_NAME), "don't relax protections as part of moduledb matching for these ; separated company names")
+    PC_OPTION_DEFAULT(liststring_t, whitelist_company_names, OPTION_STRING("Microsoft Corporation"), "don't relax protections as part of moduledb matching for these ; separated company names")
+    PC_OPTION_DEFAULT(uint, unknown_module_policy, 0xf /*MODULEDB_RCT_EXEMPT_TO|MODULEDB_ALL_SECTIONS_BITS:SECTION_ALLOW|MODULEDB_REPORT_ON_LOAD*/, "module policy control field for non-whitelisted modules")
+    OPTION_DEFAULT(uint, unknown_module_load_report_max, 10, "max number of non whitelist modules to log/report at load time")
+    OPTION_DEFAULT(uint, moduledb_exemptions_report_max, 3, "max number of moduledb security exemptions to report")
+    /* case 9330 detect race in our security policies during DLL
+     * unload, and also case 9364 for .C only after unload
+     */
+    OPTION_DEFAULT(bool, unloaded_target_exception, true,
+        "detect and silently handle as exceptions app races during DLL unload")
+
+#ifdef WINDOWS
+    OPTION_DEFAULT(bool, hide, true, "remove DR dll from module lists")
+    OPTION_DEFAULT(uint, hide_from_query, 
+        3 /* HIDE_FROM_QUERY_BASE_SIZE|HIDE_FROM_QUERY_TYPE_PROTECT */, 
+        "mask to control what option to take to hide dr when the app does a query virtual memory call on our dll base") 
+
+    OPTION_DEFAULT(bool, track_module_filenames, true,
+        "track module file names by watching section creation")
+
+    /* FIXME: since we have dynamic options this option can be false for most of the time,
+     * and the gui should set true only when going to detach to prevent a security risk.
+     * The setting should be removed when detach is complete.
+     * In vault mode: -no_allow_detach -no_dynamic_options
+     */
+    DYNAMIC_OPTION_DEFAULT(bool, allow_detach, true, "allow detaching from process")
+#endif
+
+    /* turn off critical features, right now for experimentation only */
+#ifdef WINDOWS
+    PC_OPTION_INTERNAL(bool, noasynch, "disable asynchronous event interceptions")
+#endif
+    PC_OPTION_DEFAULT_INTERNAL(bool, cache_consistency, true, "keep code cache consistent?")
+    OPTION_DEFAULT_INTERNAL(bool, sandbox_writes, true, "check each sandboxed write for selfmod?")
+    /* FIXME: off by default until dll load perf issues are solved: case 3559 */
+    OPTION_DEFAULT_INTERNAL(bool, safe_translate_flushed, false,
+        "store info at flush time for safe post-flush translation")
+    PC_OPTION_INTERNAL(bool, store_translations,
+        "store info at emit time for fragment translation")
+
+     /* case 8812 - owner validation possible only on Win32 */
+     /* Note that we expect correct ACLs to prevent anyone other than
+      * owner to have overwritten the files.
+      */
+    OPTION_DEFAULT(bool, validate_owner_dir, true,
+                   "validate owner of persisted cache or ASLR directory")
+    OPTION_DEFAULT(bool, validate_owner_file, false,
+                   "validate owner of persisted cache or ASLR, on each file")
+
+    /* PR 326815: off until we fix gcc+gap perf */
+    OPTION_COMMAND(bool, coarse_units, false, "coarse_units", {
+        if (options->coarse_units) {
+            /* PR 326610: we turned off -indirect_stubs by default, but
+             * -coarse_units doesn't support that yet (that's PR 213262).
+             */
+            options->indirect_stubs = true;
+        }
+    }, "use coarse-grain code cache management when possible", STATIC, OP_PCACHE_GLOBAL)
+
+    /* Currently a nop, but left in for the future */
+    OPTION_ALIAS(enable_full_api, coarse_units, false, STATIC, OP_PCACHE_GLOBAL)
+
+    OPTION_DEFAULT(bool, coarse_enable_freeze, false,
+                   "enable freezing of coarse units")
+    DYNAMIC_OPTION_DEFAULT(bool, coarse_freeze_at_exit, false,
+                           "freeze coarse units at process exit")
+    DYNAMIC_OPTION_DEFAULT(bool, coarse_freeze_at_unload, false,
+                           "freeze coarse units at module unload or other flush")
+    /* Remember that this is a threshold on per-module per-run new generated code.
+     * Though bb building time would recommend a fragment count threshold, for
+     * sharing benefits we care about code size.
+     * Remember also that we get wss and time improvements from RCT table
+     * being persisted; yet a large module w/ many RCT targets is likely to have
+     * a lot of code, though we may not execute much of it, so we have
+     * the coarse_freeze_rct_min option.
+     * Xref case 10362 on using pcache files for RCT independently of code caches.
+     */
+    DYNAMIC_OPTION_DEFAULT(uint_size, coarse_freeze_min_size, 512,
+                           "only freeze new coarse code > this cache size (bytes)")
+    DYNAMIC_OPTION_DEFAULT(uint_size, coarse_freeze_append_size, 256,
+                           "only append new coarse code > this cache size (bytes)")
+    DYNAMIC_OPTION_DEFAULT(uint_size, coarse_freeze_rct_min, 2*1024,
+                           "freeze a coarse module w/ > this RCT entries")
+    DYNAMIC_OPTION_DEFAULT(bool, coarse_freeze_clobber, false,
+                           "overwrite existing persisted temp files")
+    DYNAMIC_OPTION_DEFAULT(bool, coarse_freeze_rename, true,
+                           "rename existing persisted files when writing new ones")
+    DYNAMIC_OPTION_DEFAULT(bool, coarse_freeze_clean, true,
+                           "delete renamed persisted files")
+    DYNAMIC_OPTION_DEFAULT(bool, coarse_freeze_merge, true,
+                           "merge unfrozen coarse code with frozen code when persisting")
+    DYNAMIC_OPTION_DEFAULT(bool, coarse_lone_merge, true,
+                           "merge un-persisted unit w/ disk file when persisting")
+    DYNAMIC_OPTION_DEFAULT(bool, coarse_disk_merge, true,
+                           "merge persisted unit w/ disk file when persisting")
+    /* We have explicit support for mixing elision at gen and use so not PC_ */
+    OPTION_DEFAULT(bool, coarse_freeze_elide_ubr, true,
+                   "elide fall-through ubr when freezing coarse units")
+    /* case 9677: unsafe to elide entire-bb-ubr since creates backmap ambiguity */
+    PC_OPTION(bool, unsafe_freeze_elide_sole_ubr,
+                   "elide sole-ubr-bb fall-through ubr when freezing coarse units")
+    OPTION_DEFAULT(bool, coarse_pclookup_table, true,
+                   "use a reverse cache lookup table for faster entry-pc lookup,"
+                   "critical for performance of frozen units + trace building")
+    OPTION_DEFAULT(bool, persist_per_app, false,
+                   "use separate persisted cache per executable")
+    OPTION_DEFAULT(bool, persist_per_user, true,
+                   "use separate persisted cache per user")
+    OPTION_DEFAULT(bool, use_persisted, false,
+                   "use persisted cache if it exists")
+    /* exemptions are based on canonical DR names (case 3858) */
+    OPTION_DEFAULT(liststring_t, persist_exclude_list, EMPTY_STRING,
+        "exclude these ;-separated modules from persisted use and generation")
+#ifdef RCT_IND_BRANCH
+    OPTION_DEFAULT(bool, persist_rct, true,
+                   "persist RCT (call* target) tables; if this option is off, "
+                   "we will still persist for Borland modules, but no others")
+    OPTION_DEFAULT(bool, persist_rct_entire, true,
+                   "if -persist_rct, persist RCT tables for entire module")
+    OPTION_DEFAULT(bool, use_persisted_rct, true,
+                   "use persisted RCT info, if available, instead of analyzing module")
+#endif
+#ifdef HOT_PATCHING_INTERFACE
+    OPTION_DEFAULT(bool, use_persisted_hotp, true,
+                   "use persisted hotp info to avoid flushing perscaches")
+#endif
+    OPTION_DEFAULT(uint, coarse_fill_ibl,
+                   1, /* mask from 1<<IBL_type (1=ret|2=call*|4=jmp*) indicating
+                       * which per-type table(s) to fill */
+                   "fill 1st thread's ibl tables from persisted RAC/RCT tables")
+    /* FIXME case 9599: w/ MEM_MAPPED this option removes COW from the cache */
+    /* FIXME: could make PC_ to coexist for separate values */
+    OPTION_DEFAULT(bool, persist_map_rw_separate, true,
+                   "map persisted read-only sections separately to support sharing"
+                   "(option must be on both when generated and when using)")
+    OPTION_DEFAULT(bool, persist_lock_file, true,
+                   "keep persisted file handle open to prevent writes/deletes")
+    /* FIXME: could make PC_ to coexist for separate values */
+    OPTION_DEFAULT(uint, persist_gen_validation, 0x1d,
+                   /* PERSCACHE_MODULE_MD5_SHORT | PERSCACHE_MODULE_MD5_AT_LOAD |
+                      PERSCACHE_GENFILE_MD5_{SHORT,COMPLETE} */
+        "controls md5 values that we store when we persist")
+    OPTION_DEFAULT(uint, persist_load_validation, 0x5,
+                   /* PERSCACHE_MODULE_MD5_SHORT | PERSCACHE_GENFILE_MD5_SHORT */
+        "controls which md5 values we check when we load a persisted file; must "
+        "be a subset of -persist_gen_validation, else we won't load anything")
+    /* Size of short checksum of file header and footer for PERSCACHE_MODULE_MD5_SHORT.
+     * Size must match that in effect at persist time. */
+    PC_OPTION_DEFAULT(uint_size, persist_short_digest, (4*1024),
+        "size of file header and footer to check, in KB or MB")
+    OPTION_DEFAULT(bool, persist_check_options, true,
+        "consider pcache-affecting options when using pcaches")
+    PC_OPTION_DEFAULT(bool, persist_check_local_options, false,
+        "include all local options in -persist_check_options")
+    PC_OPTION_DEFAULT(bool, persist_check_exempted_options, true,
+        "only check local options for modules affected by exemptions")
+    /* FIXME: make this part of -protect_mask? */
+    OPTION_DEFAULT(bool, persist_protect_stubs, true,
+        "keep persisted cache stubs read-only")
+    OPTION_DEFAULT(uint, persist_protect_stubs_limit, 0,
+        "stop write-protecting stubs after this many writes (0 protects forever)")
+    /* case 10074: we can trade working set to reduce pagefile, strangely */
+    OPTION_DEFAULT(bool, persist_touch_stubs, true,
+        "touch stubs prior to protecting to avoid pagefile cost")
+    /* case 8640: relies on -executable_{if_rx_text,after_load} */
+    PC_OPTION_DEFAULT(bool, coarse_merge_iat, true,
+        "merge iat page into coarse unit at +rx transition")
+
+    /* case 10339: tuned for boot and memory performance, not steady-state */
+    OPTION_COMMAND(bool, desktop, false, "desktop", {
+           if (options->desktop) {
+               options->coarse_enable_freeze = true;
+               options->use_persisted = true;
+               options->coarse_freeze_at_unload = true;
+               DISABLE_TRACES(options);
+               options->shared_bb_ibt_tables = true;
+               /* case 10525/8711: reduce # links via single fine-grained vsyscall bb
+                * N.B.: if we re-enable traces we'll want to turn this back on
+                */
+               options->indcall2direct = false;
+               /* case 9686: for now reset is a nop until we decide what it
+                * should do wrt pcaches */
+               DISABLE_RESET(options);
+           } else {
+               /* -no_desktop: like -no_client, only use in simple
+                * sequences of -desktop -no_desktop
+                */
+               options->coarse_enable_freeze = false;
+               options->use_persisted = false;
+               options->coarse_freeze_at_unload = false;
+               REENABLE_TRACES(options);
+               options->shared_bb_ibt_tables = false;
+               options->indcall2direct = true;
+               REENABLE_RESET(options);
+           }
+        }, "desktop process protection",
+        STATIC, OP_PCACHE_GLOBAL)
+
+     /* should probably always turn on -executable_if_text if turning this on, for
+      * modules loaded by natively-executed modules
+      * these don't affect pcaches since the trampoline bbs won't be coarse-grain.
+      */
+    OPTION_DEFAULT(bool, native_exec, true, "attempt to execute certain libraries natively (WARNING: lots of issues with this, use at own risk)")
+     /* initially populated w/ all dlls we've needed to get .NET, MS JVM, Sun JVM, 
+      * Symantec JVM, and Panda AV working, but with very limited workload testing so far
+      */
+    OPTION_DEFAULT(liststring_t, native_exec_default_list, 
+                   /* case 3453, case 1962 .NET 1.0, 1.1 : mscorsvr.dll;mscorwks.dll;aspnet_isapi.dll */
+                   /* case 6189 .NET 2.0: mscorwks_ntdef.dll(PE name of mscorwks.dll);aspnet_isapi.dll */
+                   /* case 3453 MS JVM: msjava.dll;msawt.dll, Sun JVM: jvm.dll */
+                   /* case 3749 Symantec Java! JIT: symcjit.dll */
+                   /* case 3762 Panda AV: pavdll.dll */
+                   OPTION_STRING("mscorsvr.dll;mscorwks.dll;aspnet_isapi.dll;mscorwks_ntdef.dll;msjava.dll;msawt.dll;jvm.dll;symcjit.dll;pavdll.dll"),
+                   "execute these ;-separated modules natively")
+    /* easy way to add dlls w/o having to re-specify default list, while keeping 
+     * default list visible and settable at global level 
+     */
+    OPTION_DEFAULT(liststring_t, native_exec_list, EMPTY_STRING,
+                   "execute these ;-separated modules natively")
+    OPTION_DEFAULT(bool, native_exec_syscalls, true, "intercept system calls while application is native")
+    OPTION_DEFAULT(bool, native_exec_dircalls, true, "check direct calls as gateways to native_exec modules")
+    OPTION_DEFAULT(bool, native_exec_callcall, true, "put gateway on 1st call of a pair where 2nd targets a native_exec module")
+    OPTION_DEFAULT(bool, native_exec_guess_calls, true, "if TOS looks like a ret addr, assume transition to a new module was via call*")
+    /* case 7266: add exes and dlls with managed code to native_exec_areas */
+    OPTION_DEFAULT(bool, native_exec_managed_code, true,
+        "if module has managed code, execute it natively")
+    /* case 10998: add modules with .pexe sections to native_exec_areas,
+     * FIXME - for case 6765 turn this into a liststring_t so we can consider native exec
+     * for other potentially problematic sections like .aspack, .pcle, and .sforce */
+    OPTION_DEFAULT(bool, native_exec_dot_pexe, true,
+        "if module has .pexe section (proxy for strange int 3 behavior), execute it natively")    
+
+    /* vestiges from our previous life as a dynamic optimizer */
+    OPTION_DEFAULT_INTERNAL(bool, inline_calls, true, "inline calls in traces")
+
+    /* control-flow optimization to convert indirect calls to direct calls.
+     * This is separated from the optimization flags that follow below as it
+     * applies to all fragments, not just traces.
+     * FIXME Delete the setting after sufficient testing & qualification?
+     */
+    PC_OPTION_DEFAULT(bool, indcall2direct, true,
+        "optimization: convert indirect calls to direct calls")
+
+    /* case 85 - for optimization, and case 1948 for its basis for a stronger security check */
+    /* similar to both indcall2direct and emulate_IAT_writes */
+    PC_OPTION_DEFAULT(bool, IAT_convert, false,
+        "convert indirect call or jmp through IAT to direct call or jmp")
+    PC_OPTION_DEFAULT(bool, IAT_elide, false,
+        "elide indirect call or jmp converted by IAT_convert"
+        "unless reached max_elide_{jmp,call}; requires IAT_convert")
+    OPTION_DEFAULT_INTERNAL(bool, unsafe_IAT_ignore_hooker, false, "ignore IAT writes")
+
+    /* compatibility options */
+    OPTION_DEFAULT(uint, thread_policy, 
+        OPTION_DISABLED|OPTION_NO_BLOCK|OPTION_NO_REPORT|OPTION_NO_CUSTOM,
+         "thread delivered to a writable region allowed or squashed (optionally silently)")
+    /* custom bit off restricts thread_policy to VSE shellcode, on makes it general */
+#ifdef WINDOWS
+    OPTION_DEFAULT(uint, apc_policy, 
+        OPTION_DISABLED|OPTION_NO_BLOCK|OPTION_NO_REPORT|OPTION_NO_CUSTOM,
+         "APC delivered to a writable region allowed or squashed (optionally silently)")
+    /* custom bit off restricts apc_policy to VSE shellcode, on makes it general */
+
+    OPTION_DEFAULT_INTERNAL(bool, hook_image_entry, true, "Allow hooking of the image "
+        "entry point when we lose control at a pre-image-entry-point callback return. "
+        "Typically it's not needed to regain control if -native_exec_syscalls is on.")
+    OPTION_DEFAULT(bool, clean_testalert, true, /* case 9288, 10414 SpywareDoctor etc. */
+        "restore NtTestAlert to a pristine state at load by clearing away any hooks")
+    OPTION_DEFAULT(uint, hook_conflict, 1 /* HOOKED_TRAMPOLINE_SQUASH */, /* case 2525 */
+        "action on conflict with existing non Nt* hooks: die, squash or chain")
+    OPTION_DEFAULT(uint, native_exec_hook_conflict, 
+        1 /* HOOKED_TRAMPOLINE_SQUASH */,
+        "action on conflict with existing Nt* hooks: die, squash, or deeper")
+    /* NOTE - be careful about using the default value till the options are
+     * read */
+    OPTION_DEFAULT(bool, dr_sygate_int, false,
+        "Perform dr int system calls in a sygate compatible fashion (indirected via ntdll)")
+    OPTION_DEFAULT(bool, dr_sygate_sysenter, false,
+        "Perform dr int system calls in a sygate compatible fashion (indirected via ntdll)")
+     /* Turn off sygate compatibility int syscall indirection for app system
+      * calls.  Dr system calls will still indirect as controlled by 
+      * above options. */
+    OPTION_DEFAULT(bool, sygate_int, false,
+        "Perform app int system calls in Sygate compatible fashion (indirected via ntdll)")
+    OPTION_DEFAULT(bool, sygate_sysenter, false,
+        "Perform app sysenter system calls in Sygate compatible fashion (indirected via ntdll)")
+    OPTION_DEFAULT(bool, native_exec_hook_create_thread, true, 
+        "if using native_exec hooks, decides whether or not to hook CreateThread (disable for Sygate compatibility)")
+    OPTION_COMMAND(bool, sygate, false, "sygate", {
+            options->dr_sygate_int = true;
+            options->dr_sygate_sysenter = true;
+            options->sygate_int = true;
+            options->sygate_sysenter = true;
+            options->native_exec_hook_conflict = 3; /* HOOK_CONFLICT_HOOK_DEEPER */
+            options->native_exec_hook_create_thread = false;
+        }, "Sets neccesary options for running in Sygate compatible mode",
+        STATIC, OP_PCACHE_NOP)
+
+    /* FIXME - disabling for 64bit due to layout considerations xref PR 215395,
+     * should re-enable once we have a better solution. xref PR 253621 */
+    OPTION_DEFAULT(bool, aslr_dr, IF_X64_ELSE(false, true),
+        /* case 5366 randomize location of dynamorio.dll, uses
+         * aslr_parent_offset to control randomization padding which
+         * currently gives us 8 bits of randomness for wasting 16MB
+         * virtual space; breaks sharing for the relocated
+         * portions of our DLL
+         */
+        "randomization needs to be set in parent process")
+
+     /* Address Space Layout Randomization */
+     /* FIXME: case 2491 for stacks/heaps/PEBs/TEBs, sharing */
+    OPTION_DEFAULT(uint, aslr, 0 /* ASLR_DISABLED */, 
+        "address space layout randomization, from aslr_option_t")
+    OPTION_ALIAS(R, aslr, 0 /* ASLR_DISABLED */, STATIC, OP_PCACHE_NOP)
+
+    OPTION_DEFAULT(uint, aslr_action, 0x111,
+        /* case 7017 0x111 = ASLR_REPORT | ASLR_DETECT_EXECUTE | ASLR_TRACK_AREAS */
+        "address space layout handling and reporting, from aslr_action_t")
+
+    OPTION_DEFAULT(uint, aslr_retry, 2,
+        /* case 6739 - allow private ASLR to search for a good fit,
+         * 1  would allow linear range choice to leapfrog other DLLs
+         * 2+ would also have a chance to deal with unlikely races
+         */
+        "private ASLR attempts for a good fit after failure, 0 fallback to native")
+         
+    OPTION_DEFAULT(uint, aslr_cache, 0, /* ASLR_DISABLED */
+        "address space layout process shared and persistent cache, from aslr_cache_t")
+#endif
+    /* min_free_disk applies to both ASLR and persisted caches */
+    OPTION_DEFAULT(uint_size, min_free_disk, 50*(1024*1024), /* 50MB */
+        "minimum free disk space (or quota) on DLL cache volume")
+        /* case 8494 cache capacity management for ASLR and pcaches
+         * Note default size is in Kilobytes, so it's best to always add M qualifier.
+         * Examples: 1m, 10M
+         */
+#ifdef WINDOWS
+    OPTION_DEFAULT_INTERNAL(uint, aslr_internal, 0 /* ASLR_INTERNAL_DEFAULT */, 
+        "address space layout randomization, internal flags from aslr_internal_option_t")
+
+     /* FIXME: we need to find a good location to allow growth for other allocations */
+    OPTION_DEFAULT(uint_addr, aslr_dll_base, 0x40000000, "starting DLL base addresses")
+
+     /* limit for ASLR_RANGE_BOTTOM_UP, or starting point for ASLR_RANGE_TOP_DOWN */
+     /* FIXME: case 6739 - what to do when reaching top, 
+      *
+      * FIXME: how is STATUS_ILLEGAL_DLL_RELOCATION determined, we
+      * have to know stay out of the way since loader fails app if it
+      * finds that user32.dll is not where it wanted?
+      */
+    OPTION_DEFAULT(uint_addr, aslr_dll_top,  0x77000000, "top of DLL range")
+
+     /* FIXME: the lower 16 bits are ignored (Windows/x86
+      * AllocationGranularity is 64KB).  This here gives us 12bits of
+      * randomness.  Could make it larger if necessary. */
+    OPTION_DEFAULT(uint_addr, aslr_dll_offset,  0x10000000, "maximum random jitter for first DLL")
+
+     /* FIXME: too little (4 bits) randomness between DLLs, vs too much fragmentation */
+    OPTION_DEFAULT(uint_addr, aslr_dll_pad, 0x00100000, "maximum random jitter between DLLs")
+
+     /* case 6287 - first thread's stack can be controlled only by parent */
+     /* ASLR_STACK activates, though affect real heap reservations as well */
+     /* FIXME: the lower 16 bits are ignored (Windows/x86
+      * AllocationGranularity is 64KB).  FIXME: This here gives us 8 bits of
+      * randomness.  Could make it larger if necessary. */
+    OPTION_DEFAULT(uint_addr, aslr_parent_offset,              0x01000000, 
+        "maximum random jitter for parent reservation")
+     /* ASLR_HEAP activates, though affect real stack reservations as
+      * well This here gives us 12 bits of randomness.  Could make it
+      * larger if necessary.
+      */
+    OPTION_DEFAULT(uint_addr, aslr_heap_reserve_offset,        0x10000000,
+        "random jitter for first child reservation (large)")
+    OPTION_DEFAULT(uint_addr, aslr_heap_exe_reserve_offset,    0x01000000,
+        /* if executable ImageBaseAddress is in the middle of virtual address space */
+        "random jitter for reservation after executable (smaller)")
+
+     /* ASLR_HEAP_FILL activates */
+     /* FIXME: too little (4 bits) randomness between heap reservations, 
+      * vs too much fragmentation */
+    OPTION_DEFAULT(uint_addr, aslr_reserve_pad,                0x00100000, 
+        "random jitter between reservations (tiny)")
+
+    /* FIXME: plan for 4.3 only after aslr_safe_save is checked in
+     * ASLR_PERSISTENT_SOURCE_DIGEST | ASLR_PERSISTENT_SHORT_DIGESTS */
+    OPTION_DEFAULT(uint, aslr_validation, 0x1, /* ASLR_PERSISTENT_PARANOID */
+        "consistency and security validation level of stringency")
+
+    OPTION_DEFAULT(uint_size, aslr_short_digest, (16*1024),
+        "size of file header and footer to check, in KB or MB")
+        /* used for checksum comparison of file header and footer,
+         * note that if this value is changed previously persisted
+         * files will not be accepted.  
+         * 0 turns into a full file digest
+         */
+        /* default size is in Kilobytes, Examples: 1024, 1024k, 1m */
+
+    OPTION_DEFAULT(uint_size, aslr_section_prefix, (16*1024),
+        "size of section prefix to match, in KB or MB")
+        /* used for byte comparison of a prefix of each file section,
+         * if enabled by ASLR_PERSISTENT_PARANOID_PREFIX.
+         * 0 disables section validation
+         */
+        /* default size is in Kilobytes, Examples: 1024, 1024k, 1m */
+
+    OPTION_DEFAULT(liststring_t, exempt_aslr_default_list, 
+        /* user32.dll - case 6620 on STATUS_ILLEGAL_DLL_RELOCATION. Not
+         *              clear whether loader is just whining, or there
+         *              is a good reason it shouldn't be rebased.  
+         *       FIXME: without early injection affects only RU=5 processes that load it late,
+         *              SHOWSTOPPER on XP.
+         *       FIXME: Hopefully shouldn't be necessary for all KnownDlls, investigate full list.
+         * ole32.dll - case 7746 on Win2000 and case 7743 
+         *      on NT complaining about STATUS_ILLEGAL_DLL_RELOCATION
+         * sfc.dll - case 8705 
+         *      update.exe targeting directly 
+         *      sfc.dll!MySfcTerminateWatcherThread in winlogon.exe
+         * kbdus.dll,kbdbg.dll - case 6671 FIXME: list not complete, 
+         *              FIXME: case 6740: we don't actually support kbd*.dll
+         * kernel32.dll - with early injection this one also complains of being
+         *                rebased.
+         */
+        OPTION_STRING("kernel32.dll;user32.dll;ole32.dll;sfc.dll;kbdus.dll;kbdbu.dll;kbd*.dll"), 
+        "exempt from randomization these ;-separated modules")
+    OPTION_DEFAULT(liststring_t, exempt_aslr_list,
+        /* Note that allows '*' as a stress option */
+        EMPTY_STRING, "exempt from randomization these ;-separated modules, append")
+
+
+    /* case 7794 - when using private ASLR these large DLLs have a higher
+     * impact on visible memory, and in stress testing on performance.
+     * not really recommended but available until we enable sharing
+     */
+    OPTION_DEFAULT(bool, aslr_extra, false, "ASLR DLL exempt longer list")
+    OPTION_DEFAULT(liststring_t, exempt_aslr_extra_list,
+        /* see case 7794 exempt from both private and shared */
+        /* note that these names are matched to PE name for private,
+         * but to file names for ASLR cache */
+        OPTION_STRING("mshtml.dll;msi.dll;mso.dll;shell32.dll"), 
+        "exempt from randomization these ;-separated modules")
+
+    /* case 9495 - include or exclude list for DLLs to share and
+     * persist. Note that one cannot use simultaneously an include and
+     * exclude list, so to remove an entry from either need to provide the 
+     * whole list as '#advapi32.dll'
+     */
+    OPTION_DEFAULT(uint, aslr_cache_list, 1, /* ASLR_CACHE_LIST_INCLUDE */
+        "controls DLLs to be shared via aslr_cache {all,include,exclude}")
+        /* (note aslr_cache_list_t values match meaning in process_control) */
+    /* exemptions are based on file names, not PE names */
+    OPTION_DEFAULT(liststring_t, aslr_cache_include_list,
+        /* enabled via -aslr_cache_list 1 (ASLR_CACHE_LIST_INCLUDE) (default) */
+        /* All other DLLs are left for private ASLR if not exempt by
+         * exempt_aslr_extra_list. */
+        /* Need a slightly longer list to be worth enabling */
+        /* Note we're bound by MAX_LIST_OPTION_LENGTH so check total
+         * length before adding, and leave some room for DLLs that we
+         * may want to add in the field.
+         */
+        /* Note DLLs that have no or very few relocation pages shouldn't be added */
+        OPTION_STRING(
+        "advapi32.dll;"
+        /* FIXME: browseui.dll; also has high wset in IE after empty.exe */
+        "comctl32.dll;"         /* note two different SxS versions */
+        "gdi32.dll;"
+        "jscript.dll;"          /* IE */
+        "msctf.dll;"
+        "mshtml.dll;"           /* shared by IE and Explorer */
+        /* FIXME: mso.dll is the top DLL in Office apps, yet has startup effects */
+        "msvcrt.dll;"
+        "riched20.dll;"         /* different versions of in Office11\ and system32\ */
+        "rpcrt4.dll;"
+        "sapi.dll;"             /* only Speech\ for Office apps */
+        "setupapi.dll;"
+        "shell32.dll;"
+        /* FIXME: shdocvw.dll; is a top DLL in Office and IE */
+        /* shdoclc.dll companion DLL has resources for shdocvw.dll,
+         * and has no relocations so it's fine to randomize privately
+         * the resources, while not having to validate them if in
+         * shared ASLR cache
+         */
+        "sptip.dll;"            /* only Speech\ related */
+        "sxs.dll;"              /* XP, in all processes, provides Side x Side  */
+        "uxtheme.dll;"           /* FIXME: XP, not very high WSet after startup */
+        "ws2_32.dll"            /* case 9627 make sure any ASLR at all */
+        ),
+        "use shared cache only for these ;-separated modules")
+    /* exemptions are based on file names, not PE names */
+    OPTION_DEFAULT(liststring_t, aslr_cache_exclude_list,
+        /* enabled via -aslr_cache_list 2 (ASLR_CACHE_LIST_EXCLUDE) */
+        /* exempt from shared ASLR but still apply private ASLR */
+        OPTION_STRING("mso.dll;xpsp2res.dll"),
+        "exclude from shared cache these ;-separated modules")
+    OPTION_DEFAULT(bool, aslr_safe_save, true, 
+        /* see case 9696 */
+        "ASLR DLL safe file creation in temporary file before rename")
+
+    /* Syntactic sugar for memory savings at the cost of security controlled from core */
+    OPTION_COMMAND(bool, medium, false, "medium", {
+           if (options->medium) {
+               options->aslr_extra = true;
+               options->thin_client = false;   /* Case 9037. */
+           }
+        }, "medium security/memory mapping",
+        STATIC, OP_PCACHE_NOP)
+
+    OPTION_COMMAND(bool, low, false, "low", {
+           if (options->low) {
+                IF_HOTP(options->hot_patching = true);
+                IF_HOTP(options->hotp_only = true;)
+                IF_HOTP(options->liveshields = true;)
+                IF_HOTP(options->hotp_diagnostics = true;)
+                IF_HOTP(if (options->hotp_only == true) {
+                   /* coordinate with hotp_only any additional option changes */
+                   IF_RETURN_AFTER_CALL(options->ret_after_call = false;)
+                   IF_RCT_IND_BRANCH(options->rct_ind_call = OPTION_DISABLED;)
+                   IF_RCT_IND_BRANCH(options->rct_ind_jump = OPTION_DISABLED;)
+                });
+                /* Matching old behavior */
+                IF_WINDOWS(options->apc_policy =
+                           OPTION_ENABLED|OPTION_BLOCK|OPTION_REPORT|OPTION_CUSTOM;)
+
+                options->vm_size   = 32*1024*1024; /* 32MB */
+                IF_GBOP(options->gbop = 0x6037;) /* GBOP_CLIENT_DEFAULT */
+                options->aslr      = 0x0; /* ASLR_DISABLED */
+                /* Reset has no meaning for hotp_only; see case 8389. */
+                DISABLE_RESET(options);
+                IF_KSTATS(options->kstats = false); /* Cases 6837 & 8869. */
+                options->thin_client = false;   /* Case 9037. */
+           }
+        }, "low security/memory mapping",
+        STATIC, OP_PCACHE_NOP)
+
+    OPTION_COMMAND(bool, client, false, "client", {
+           if (options->client) {
+                IF_HOTP(options->hot_patching = true);
+                IF_HOTP(options->hotp_only = true;)
+                IF_HOTP(options->liveshields = true;)
+                IF_HOTP(options->hotp_diagnostics = true;)
+                IF_HOTP(if (options->hotp_only == true) {
+                   /* coordinate with hotp_only any additional option changes */
+                   IF_RETURN_AFTER_CALL(options->ret_after_call = false;)
+                   IF_RCT_IND_BRANCH(options->rct_ind_call = OPTION_DISABLED;)
+                   IF_RCT_IND_BRANCH(options->rct_ind_jump = OPTION_DISABLED;)
+                });
+                /* Matching old behavior */
+                IF_WINDOWS(options->apc_policy =
+                           OPTION_ENABLED|OPTION_BLOCK|OPTION_REPORT|OPTION_CUSTOM;)
+
+                options->vm_size   = 32*1024*1024; /* 32MB */
+                IF_GBOP(options->gbop = 0x6037;) /* GBOP_CLIENT_DEFAULT */
+                /* making sure -client -low  == -low -client*/
+                if (options->low) {
+                    options->aslr      = 0x0; /* ASLR_DISABLED */
+                    options->aslr_cache= 0x0; /* ASLR_DISABLED */
+                } else {
+                    options->aslr      = 0x7; /* ASLR_CLIENT_DEFAULT */
+                    options->aslr_cache= 0x192; /* ASLR_CACHE_DEFAULT */
+                }
+
+                /* case 2491 ASLR_SHARED_CONTENTS | 
+                 * ASLR_SHARED_ANONYMOUS_CONSUMER | ASLR_SHARED_FILE_PRODUCER,
+                 */
+
+                /* Reset has no meaning for hotp_only; see case 8389. */
+                DISABLE_RESET(options);
+                IF_KSTATS(options->kstats = false); /* Cases 6837 & 8869. */
+                options->thin_client = false;   /* Case 9037. */
+           } else {
+               /* 
+                * case 8283 -no_client.  Note that this will work well
+                * only for simple sequences of -client -no_client, as
+                * we do not attempt to actually recover and is very
+                * very likely we'll forget to keep this in synch
+                */
+                IF_HOTP(options->hotp_only = false;)
+                /* coordinate with hotp_only any additional option changes */
+                IF_RETURN_AFTER_CALL(options->ret_after_call = 
+                                     DEFAULT_OPTION_VALUE(ret_after_call);)
+                IF_RCT_IND_BRANCH(options->rct_ind_call = 
+                                  DEFAULT_OPTION_VALUE(rct_ind_call);)
+                IF_RCT_IND_BRANCH(options->rct_ind_jump = 
+                                  DEFAULT_OPTION_VALUE(rct_ind_jump);)
+                options->vm_size = DEFAULT_OPTION_VALUE(vm_size);
+                IF_GBOP(options->gbop = DEFAULT_OPTION_VALUE(gbop);)
+                options->aslr = DEFAULT_OPTION_VALUE(aslr);
+                options->aslr_cache = DEFAULT_OPTION_VALUE(aslr_cache);
+
+                REENABLE_RESET(options);
+                IF_KSTATS(options->kstats = true);
+           }
+        }, "client process protection",
+        STATIC, OP_PCACHE_NOP)
+
+# ifdef GBOP
+    /* Generically Bypassable Overflow Protection in user mode */
+    OPTION_DEFAULT(uint, gbop, 0 /* GBOP_DISABLED */,
+        "GBOP control, from GBOP_OPTION")
+    OPTION_ALIAS(O, gbop, 0 /* GBOP_DISABLED */, STATIC, OP_PCACHE_NOP)
+    DYNAMIC_OPTION_DEFAULT(uint, gbop_frames, 0, 
+        "GBOP stack backtrace frame depth") /* >0 NYI */
+
+    OPTION_DEFAULT(uint, gbop_include_set, 
+         0x1 /* GBOP_SET_NTDLL_BASE */
+        |0x2 /* KERNEL32 BASE */
+        |0x4 /* MSVCRT BASE */
+        |0x8 /* WS2_32 BASE */
+        |0x10/* WININET BASE */
+        |0x20/* USER32 BASE */
+        |0x40/* SHELL32 BASE */
+        /* FIXME: case 8006 should enable MORE NTDLL KERNEL32 MSVCRT WS2_32 in a later round */
+        , "GBOP policy group control, 0 means all")
+     /* bit positions are as defined in GBOP_ALL_HOOKS */
+
+    OPTION_DEFAULT(uint, gbop_last_hook, 0 /* automatically determine number of hooks */,
+        /* note this should be internal, but currently provides good
+         * control over the performance impact of having extra GBOP hooks
+         */
+        "GBOP number of hooks length, crude override")
+
+    /* FIXME: may want to duplicate the above options for the extra
+     * hooks, that may have different flags and different frame depth
+     */
+    OPTION_DEFAULT(liststring_t, gbop_include_list,
+        /* NYI: case 7127 list of additional hook points module!func */
+        /* FIXME: should allow module!* for stress testing */
+        EMPTY_STRING, "include for GBOP these ;-separated module!func descriptors, append")
+    OPTION_DEFAULT(liststring_t, gbop_exclude_list,
+        /* case 7127 list of GBOP hook points to turn the mode off (disable hook) for */
+        /* e.g. 'KERNEL32.dll!FreeLibrary;WININET.dll!*',
+         * Note the only wildcards supported are module.dll!*,
+         * or '*' as a stress test option to exclude all hooks.
+         */
+        EMPTY_STRING, "disable GBOP hook for these ;-separated module!func descriptors, append")
+    OPTION_DEFAULT(liststring_t, exempt_gbop_from_default_list, 
+        EMPTY_STRING, "allow GBOP violations from these ;-separated modules")
+    OPTION_DEFAULT(liststring_t, exempt_gbop_from_list, 
+        EMPTY_STRING, "allow GBOP violations from these ;-separated modules, append")
+
+   /* FIXME: case 7127 - can make all gbop options dynamic, though
+    * extra hook injection will need a more explicit nudge-like
+    * mechanism
+    */
+# endif /* GBOP */
+    /* FIXME: temporary fix for case 9467 - option to disable if not needed */
+    OPTION_DEFAULT(bool, mute_nudge, true, "mute nudges for thin_clients")
+#endif /* WINDOWS */
+     
+    /* Pseudo Random Number Generator seed affects all random number users:
+     * (currently vm_max_offset, aslr_dll_offset, aslr_dll_pad) */
+    OPTION_DEFAULT(uint, prng_seed, 0 /* get a good seed from the OS */,
+        "if non-0 allows reproducible pseudo random number generator sequences")
+
+    /* FIXME PR 215179 on enabling pad_jmps in all builds */
+#if defined(PROFILE_LINKCOUNT) || defined(TRACE_HEAD_CACHE_INCR) || defined(CUSTOM_EXIT_STUBS)
+    OPTION_DEFAULT(bool, pad_jmps, false, "nop pads jmps in the cache that we might need to patch so that the offset doesn't cross a L1 cache line boundary (necessary for atomic linking/unlinking on an mp machine)")
+#else 
+    OPTION_DEFAULT(bool, pad_jmps, true, "nop pads jmps in the cache that we might need to patch so that the offset doesn't cross a L1 cache line boundary (necessary for atomic linking/unlinking on an mp machine)")
+#endif
+    /* FIXME PR 215179 on getting rid of this tracing restriction. */
+#if defined(LINUX)
+    OPTION_DEFAULT(bool, pad_jmps_mark_no_trace, true, "mark bbs that require added nops "
+                   "for multiple hot patchable exits with CANNOT_BE_TRACE, since tracing "
+                   "through fragments with inserted nops isn't well supported (see PR "
+                   "215179 on fixing this)") 
+#else
+    OPTION_DEFAULT(bool, pad_jmps_mark_no_trace, false, "mark bbs that require added nops "
+                   "for multiple hot patchable exits with CANNOT_BE_TRACE, since tracing "
+                   "through fragments with inserted nops isn't well supported (see PR "
+                   "215179 on fixing this)") 
+#endif
+    OPTION_DEFAULT_INTERNAL(bool, pad_jmps_return_excess_padding, true, "if -pad_jmps returns any excess requested memory to fcache")
+    OPTION_DEFAULT_INTERNAL(bool, pad_jmps_shift_bb, true, 
+        "if -pad_jmps shifts the start_pc for padding the first jmp of a bb instead of inserting a nop")
+    OPTION_DEFAULT_INTERNAL(bool, pad_jmps_shift_trace, true,
+        "if -pad_jmps shifts the start_pc for padding the first jmp of a trace instead of inserting a nop")
+    OPTION_DEFAULT_INTERNAL(uint, pad_jmps_set_alignment, 0, "if non-zero sets the pad_jmps alignment (useful for stress testing -pad_jmps code)")
+
+    OPTION_DEFAULT_INTERNAL(bool, ibl_sentinel_check, true, /* case 2174: FIXME: remove when working fine */
+        "check for sentinel overwraps in IBL routine instead of exit")
+
+    OPTION_DEFAULT(bool, ibl_addr_prefix, false, /* case 5231: FIXME: remove when working fine */
+        "uses shorter but slower encode with addr16 prefix in IBL routine and elsewhere")
+
+     /* Artificial Slowdown Options */
+    OPTION_DEFAULT_INTERNAL(uint, slowdown_ibl_found, 0,
+        "add a loop to slow down the IBL hit path")
+
+     /* Stress Testing Options */
+    OPTION_DEFAULT_INTERNAL(bool, stress_recreate_pc, false, 
+        "stress test recreate pc after each trace or bb")
+    OPTION_COMMAND_INTERNAL(bool, stress_recreate_state, false, "stress_recreate_state",
+    { if (options->stress_recreate_state) 
+        options->stress_recreate_pc = true;
+    }, "stress test recreate state after each trace or bb", STATIC, OP_PCACHE_NOP)
+    OPTION_DEFAULT_INTERNAL(bool, detect_dangling_fcache, false, 
+        "detect any execution of a freed fragment")
+    OPTION_DEFAULT_INTERNAL(bool, stress_detach_with_stacked_callbacks, false,
+        "detach once a thread has 2 levels of nested callbacks (for internal testing)")
+    OPTION_DEFAULT_INTERNAL(bool, detach_fix_sysenter_on_stack, true /* default true */,
+        "if false then detach does not fix sysenter callbacks on the stack and instead "
+        "uses the emitted dispatch code used for other system calls (a fairly minor "
+        "transparency violation).  Used for internal testing.")
+
+     /* for stress testing can use 1 */
+    OPTION_DEFAULT_INTERNAL(uint, vmarea_initial_size, 100, 
+        "initial vmarea vector size")
+    /* FIXME: case 4471 should start smaller and double instead */
+    OPTION_DEFAULT_INTERNAL(uint, vmarea_increment_size, 100, 
+        "incremental vmarea vector size")
+    OPTION_INTERNAL(uint_addr, stress_fake_userva, 
+        "pretend system address space starts at this address (case 9022)")
+
+    /* degenerate options: only used for run-once testing (case 3990) */
+    OPTION_INTERNAL(bool, unsafe_crash_process, "unsafe: generates a DR exception")
+    OPTION_INTERNAL(bool, unsafe_hang_process, "unsafe: hang the process")
+
+    /* unsafe experimental options */
+    OPTION_DEFAULT_INTERNAL(bool, unsafe_ignore_overflow, false,
+        "do not preserve OF flag, unsafe")
+    OPTION_COMMAND_INTERNAL(bool, unsafe_ignore_eflags, false, "unsafe_ignore_eflags", {
+        if (options->unsafe_ignore_eflags) { /* else leave alone */
+            options->unsafe_ignore_eflags_trace = options->unsafe_ignore_eflags;
+            options->unsafe_ignore_eflags_prefix = options->unsafe_ignore_eflags;
+            options->unsafe_ignore_eflags_ibl = options->unsafe_ignore_eflags;
+            /* this also covers NATIVE_RETURN which doesn't have a separate option */
+        }
+     }, "do not preserve EFLAGS on any part of ind br handling, unsafe", STATIC,
+    OP_PCACHE_NOP)
+
+    OPTION_DEFAULT_INTERNAL(bool, unsafe_ignore_eflags_trace, false,
+        "do not preserve EFLAGS on in-trace cmp, unsafe")
+    OPTION_DEFAULT_INTERNAL(bool, unsafe_ignore_eflags_prefix, false,
+        "do not preserve EFLAGS on prefixes, unsafe")
+    OPTION_DEFAULT_INTERNAL(bool, unsafe_ignore_eflags_ibl, false,
+        "do not preserve EFLAGS in ibl proper, unsafe")
+
+    OPTION_DEFAULT(liststring_t, ignore_assert_list, 
+        EMPTY_STRING, "convert into warnings these ;-separated assert identifiers")
+     /* Should be an exact match of message after Internal Error.
+      * Most common ones look like 'x86/arch.c:142', but could also
+      * look like 'Not implemented @x86/arch.c:142' or 
+      * 'Bug #4809 @x86/arch.c:145;Ignore message @x86/arch.c:146'
+      */
+
+#ifdef EXPOSE_INTERNAL_OPTIONS
+    OPTION_NAME(bool, optimize, " synthethic", "set if ANY opts are on")
+
+#define OPTIMIZE_OPTION(type, name) OPTION_COMMAND(type, name, 0, #name, {      \
+    options->optimize = true;                                             \
+}, "optimization", STATIC, OP_PCACHE_NOP)
+
+# ifdef SIDELINE
+    OPTION(bool, sideline, "use sideline thread for optimization")
+# endif
+    /* optimizations */
+
+# if 0 /* this flag does nothing yet...disable so people don't try to use it */
+    OPTION(uint, aggressiveness, "level of aggressiveness in optimizations")
+# endif
+
+# ifdef IA32_ON_IA64
+    OPTIMIZE_OPTION(bool, test_ia64)
+# endif
+    OPTIMIZE_OPTION(bool, prefetch)
+    OPTIMIZE_OPTION(bool, rlr)
+    OPTIMIZE_OPTION(bool, vectorize)
+    OPTIMIZE_OPTION(bool, unroll_loops)
+
+    OPTIMIZE_OPTION(bool, instr_counts)
+    OPTIMIZE_OPTION(bool, stack_adjust)
+# ifdef LOAD_TO_CONST
+    OPTIMIZE_OPTION(bool, loads_to_const)
+    OPTIMIZE_OPTION(bool, safe_loads_to_const)
+# endif
+
+    OPTIMIZE_OPTION(uint, remove_dead_code)             /* aggressiveness level */
+    OPTIMIZE_OPTION(uint, constant_prop)                /* aggressiveness level */
+        /* 2 digits, first controls local aggressiveness second global */
+        /* aggressiveness, i.e. 14 is local aggressiveness 1 and global */
+        /* aggressiveness 4, possible values are 0, 1, 2 for global and */
+        /* 0, 1, 2, 3 for local, larger numbers = more aggressive at the */
+        /* possible (but hopefully unlikely) expense of correctness    */
+
+    OPTIMIZE_OPTION(bool, call_return_matching)
+    OPTIMIZE_OPTION(bool, remove_unnecessary_zeroing) // FIXME: unnecessarily long option
+    OPTIMIZE_OPTION(bool, peephole)
+# undef OPTIMIZE_OPTION
+#endif /* EXPOSE_INTERNAL_OPTIONS */
+#ifdef HOT_PATCHING_INTERFACE
+    OPTION_DEFAULT(bool, hot_patching, false, "enable hot patching")
+
+    /* This is used to create forensics files when a hot patch event is
+     * logged.  There can be numerous hotpatch events and dumping a
+     * forensics file for each one by default would result in too many,
+     * hence the need for this option.  Note: hot patch exceptions and
+     * internal errors are not hot patch requested events, so this option
+     * won't be used to control forensics files for those.
+     */
+    DYNAMIC_OPTION_DEFAULT(bool, hotp_diagnostics, false,
+                           "produces forensics for hot patch events")
+    /* There are many technical challenges to switching dynamically between
+     * full core control mode and hotp_only mode.  So hotp_only is not DYNAMIC.
+     * TODO: Revisit when starting on "attach" mode.
+     * NOTE: hotp_only specifies the non-code cache mode, not liveshields; for
+     *       liveshields must use the -liveshields option.
+     */
+    OPTION_COMMAND(bool, hotp_only, false, "hotp_only", {
+        if (options->hotp_only) {
+            IF_RETURN_AFTER_CALL(options->ret_after_call = false;)
+            IF_RCT_IND_BRANCH(options->rct_ind_call = OPTION_DISABLED;)
+            IF_RCT_IND_BRANCH(options->rct_ind_jump = OPTION_DISABLED;)
+
+            /* no kstats for -hotp_only; case 6837 */
+            IF_KSTATS(options->kstats = false);
+
+            /* reset has no meaning for hotp_only; see case 8389 */
+            DISABLE_RESET(options);
+
+            /* -low and -client set their sizes afterward so no conflict */
+            options->vm_size   = 32*1024*1024; /* 32MB */
+
+            options->thin_client = false;   /* Case 9037. */
+            options->native_exec = false;
+
+            /* FIXME: add other options we should turn off */
+            /* FIXME: coordinate with -client any other option changes _required_ 
+             * for -hotp_only */
+        }
+    }, "enable hot patching only mode, i.e., no code cache", STATIC, OP_PCACHE_NOP)
+    /* NOTE: as of today probe_api and liveshields are mutually exclusive, i.e,
+     * ls-defs and probes can't be specified by one or more clients.  The
+     * ultimate goal is to abstract out the probe api and convert LiveShields
+     * into a client using probe api, at which point -probe_api will be the
+     * same as -hot_patching.  The same applies to gbop too, i.e., excluded
+     * when probe_api is turned on; gbop & livshields can co-exist as before.
+     */
+    OPTION_COMMAND(bool, liveshields, false, "liveshields", { 
+        if (options->liveshields) {
+            options->hot_patching = true;
+            options->hotp_diagnostics = true;
+            IF_CLIENT_INTERFACE(options->probe_api = false;)
+        }
+    }, "enables LiveShields", STATIC, OP_PCACHE_NOP)
+#endif /* HOT_PATCHING_INTERFACE */
+#ifdef PROCESS_CONTROL      /* case 8594 */
+    /* Dynamic because it can be turned on or off using a nudge. */
+    DYNAMIC_OPTION_DEFAULT(uint, process_control, 0,
+        "sets process control mode {off,whitelist,blacklist} thereby "
+        "deciding if a process is allowed to execute or not")
+    /* FIXME: remove this after md5s are obtained from a mapped file; case 9252.*/
+    DYNAMIC_OPTION_DEFAULT(uint, pc_num_hashes, 100,
+        "sets the number of hashes a process control hashlist can contain")
+
+    /* detect_mode for process_control; see case 10610.  Only reason to have it
+     * separate is that it is distinctly different than other core features,
+     * and is exposed in UI as orthogonal to other VPS features.
+     *
+     * FIXME: having a separate detect_mode option for each
+     * security mechanism won't scale even if each used the OPTION_* flags
+     * internally; should make the actual security mechanism parameterized to
+     * take in OPTION_* flags (which can't be done today because OPTION_* flags
+     * assume only on or off state and things like process control, gbop, aslr
+     * have more than one state, in some cases, they have their own set of
+     * operational flags) or parameterize individual actions like
+     * detect_mode, dumpcore, remediation action, forensics file, etc.  */
+    DYNAMIC_OPTION_DEFAULT(bool, pc_detect_mode, false,
+        "provides detect_mode for process control independent of -detect_mode")
+
+    /* Case 11023: don't produce forensices by default; needless load for EV. */
+    DYNAMIC_OPTION_DEFAULT(bool, pc_diagnostics, false,
+        "provides forensics for process control independent of -diagnostics")
+#endif
+
+    /* thin_client mode is just a light weight mode in which the core
+     * executes where there is no code cache, hotp_only, gbop or aslr.
+     * It hooks one or two system calls, enough to follow into child 
+     * processes.  Though it is intended to be used for process_control
+     * today, it has value independent of process_control.  One example
+     * is to inflate/switch the core to hotp_only, gbop or aslr on the 
+     * fly for any unprotected process, which I think would be very handy.
+     * Case 8576.
+     */
+    OPTION_COMMAND(bool, thin_client, false, "thin_client", {
+        if (options->thin_client) {
+            /* Will be running native mostly, so need native_exec_syscalls 
+             * to hook syscalls to follow children.
+             */
+            options->native_exec_syscalls = true;
+
+            /* thin_client is just that, thin; so no hot patching, gbopping 
+             * or aslring here.
+             */
+            IF_HOTP(options->hot_patching = false;)
+            IF_HOTP(options->hotp_only = false;)
+            IF_GBOP(options->gbop = 0;)
+            IF_WINDOWS(options->aslr = 0;)
+
+            /* similarly, client/low/medium modes are incompatible with 
+             * thin_client; though check_options_compatibility() enforces this,
+             * this is a special case as the user may legitimately set both
+             * -client and -thin_client.  Mostly for debugging; case 9037. */
+            IF_WINDOWS(options->client = false;)
+            IF_WINDOWS(options->low = false;)
+            IF_WINDOWS(options->medium = false;)
+
+            /* thin_client mode is intended to have a low foot print;
+             * reserving the default 128 mb takes 256 kb of page table
+             * space (case 8491), so reserve just 4 mb, in case we inflate
+             * to hotp_only mode.
+             */
+            options->vm_size = 4*1024*1024;
+
+            /* Don't randomize the core heap; cygwin app's stack & heaps 
+             * will move, causing them to crash.  Also memory 
+             * contiguity expected by .NET apps would be broken.  The
+             * problem is when we inflate we still won't be randomized!
+             */
+            options->vm_base = 0;
+            options->vm_max_offset = 0;
+
+            /* Reset has no meaning for thin_client; see case 8389. */
+            DISABLE_RESET(options);
+
+            /* No kstats for -thin_client - same issue as case 6837. */
+            IF_KSTATS(options->kstats = false);     /* See case 8869 also. */
+        }
+    }, "run dr in a light weight mode with nothing but a few hooks", STATIC,
+    OP_PCACHE_NOP)
+
+#ifdef VMX86_SERVER
+    OPTION_DEFAULT(uint, vmkuw_version, 0,
+                   "override vmkernel userworld version this library was compiled for")
+    /* FIXME PR 363075: we can distinguish visor from classic but not 32-bit
+     * (3.5) from 64-bit (4.0); we allow running on 3.5 via this option
+     */
+    OPTION_DEFAULT(bool, esx_32bit, false, "running on 32-bit vmkernel")
+#endif
+
+#undef OPTION
+#undef OPTION_NAME
+#undef OPTION_DEFAULT
+
+#undef DYNAMIC_OPTION
+#undef DYNAMIC_OPTION_DEFAULT
diff --git a/core/os_shared.h b/core/os_shared.h
new file mode 100644
index 000000000..dbe2566e9
--- /dev/null
+++ b/core/os_shared.h
@@ -0,0 +1,829 @@
+/* **********************************************************
+ * Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+
+/*
+ * os_shared.h - shared declarations for os facilities from linux/win32 
+ */
+
+#ifndef OS_SHARED_H
+#define OS_SHARED_H
+
+enum {VM_ALLOCATION_BOUNDARY = 64*1024}; /* 64KB allocation size for Windows */
+
+struct _local_state_t; /* in arch_exports.h */
+
+/* in os.c */
+void os_init(void);
+/* called on detach and on process exit in debug builds */
+void os_slow_exit(void);
+/* called on detach and on process exit in both debug and release builds */
+void os_fast_exit(void);
+
+void os_tls_init(void);
+/* Can be called from other threads, so uses the parameter to locate tls */
+void os_tls_exit(struct _local_state_t *local_state);
+void os_thread_init(dcontext_t *dcontext);
+void os_thread_exit(dcontext_t *dcontext);
+void os_thread_under_dynamo(dcontext_t *dcontext);
+void os_thread_not_under_dynamo(dcontext_t *dcontext);
+
+void os_heap_init(void);
+void os_heap_exit(void);
+
+/* os provided heap routines */
+/* caller is required to handle thread synchronization and to update dynamo vm areas.
+ * size must be PAGE_SIZE-aligned.
+ * returns NULL if fails to allocate memory!
+ * N.B.: only heap.c should call these routines, as it contains the logic to
+ * handle memory allocation failure!
+ *
+ * error_code is set to 0 on success
+ * on failure it is set to one of the other enum values below or an OS specific status
+ * code used only for reporting
+ */
+enum {
+    HEAP_ERROR_SUCCESS = 0,
+    /* os_heap_reserve_in_region() only, couldn't find a place to reserve within region */
+    HEAP_ERROR_CANT_RESERVE_IN_REGION = 1,
+    /* os_heap_reserve() only, Linux only, mmap failed to reserve at preferred address */ 
+    HEAP_ERROR_NOT_AT_PREFERRED = 2,
+};
+typedef uint heap_error_code_t;
+
+/* Reserve size bytes of virtual address space in one piece without committing swap
+ * space for it.  If preferred is non-NULL then memory will be reserved at that address
+ * only (if size bytes are unavailable at preferred then the allocation will fail). */
+void *os_heap_reserve(void *preferred, size_t size, heap_error_code_t *error_code);
+/* Reserve size bytes of virtual address space in one piece entirely within the
+ * address range specified without committing swap space for it. */
+void *os_heap_reserve_in_region(void *start, void *end, size_t size,
+                                heap_error_code_t *error_code);
+/* commit previously reserved pages, returns false when out of memory */
+bool os_heap_commit(void *p, size_t size, uint prot, heap_error_code_t *error_code);
+/* decommit previously committed page, so it is reserved for future reuse */
+void os_heap_decommit(void *p, size_t size, heap_error_code_t *error_code);
+/* frees size bytes starting at address p (note - on windows the entire allocation
+ * containing p is freed and size is ignored) */
+void os_heap_free(void *p, size_t size, heap_error_code_t *error_code);
+
+/* prognosticate whether systemwide memory pressure based on
+ * last_error_code and systemwide omens 
+ *
+ * Note it doesn't answer if the hog is current process, nor whether
+ * it is our own fault in current process.
+ */
+bool os_heap_systemwide_overcommit(heap_error_code_t last_error_code);
+
+bool os_heap_get_commit_limit(size_t *commit_used, size_t *commit_limit);
+
+thread_id_t get_thread_id(void);
+process_id_t get_process_id(void);
+void thread_yield(void);
+void thread_sleep(uint64 milliseconds);
+bool thread_suspend(thread_record_t *tr);
+bool thread_resume(thread_record_t *tr);
+bool thread_terminate(thread_record_t *tr);
+
+bool is_thread_currently_native(thread_record_t *tr);
+
+/* If state beyond that in our dr_mcontext_t is needed, os-specific routines
+ * must be used.  These only deal with dr_mcontext_t state.
+ */
+bool thread_get_mcontext(thread_record_t *tr, dr_mcontext_t *mc);
+bool thread_set_mcontext(thread_record_t *tr, dr_mcontext_t *mc);
+
+/* Takes an os-specific context. Does not return. */
+void thread_set_self_context(void *cxt);
+/* Only sets the dr_mcontext_t state.  Does not return. */
+void thread_set_self_mcontext(dr_mcontext_t *mc);
+
+dcontext_t *get_thread_private_dcontext(void);
+void set_thread_private_dcontext(dcontext_t *dcontext);
+
+/* converts a local_state_t offset to a segment offset */
+ushort os_tls_offset(ushort tls_offs);
+
+struct _local_state_t; /* in arch_exports.h */
+struct _local_state_extended_t; /* in arch_exports.h */
+struct _local_state_t *get_local_state(void);
+struct _local_state_extended_t *get_local_state_extended(void);
+ 
+#ifdef X86
+/* Returns POINTER_MAX on failure.
+ * Assumes that cs, ss, ds, and es are flat.
+ */
+byte *
+get_segment_base(uint seg);
+#endif
+
+bool pre_system_call(dcontext_t *dcontext);
+void post_system_call(dcontext_t *dcontext);
+
+char *get_application_pid(void);
+char *get_application_name(void);
+const char *get_application_short_name(void);
+char *get_computer_name(void); /* implemented on win32 only, in eventlog.c */
+
+int get_num_processors(void);
+
+/* Terminate types - the best choice here is TERMINATE_PROCESS, no cleanup*/
+typedef enum {
+    TERMINATE_PROCESS = 0x1,
+    TERMINATE_THREAD  = 0x2, /* terminate thread (and process if last) */
+    /* Use of TERMINATE_THREAD mode is dangerous, 
+       and can result in the following problems (see MSDN):
+       * If the target thread owns a critical section, 
+       the critical section will not be released.
+       * If the target thread is allocating memory from the heap, 
+       the heap lock will not be released.
+       * If the target thread is executing certain kernel32 calls when it is terminated, 
+       the kernel32 state for the thread's process could be inconsistent.
+       * If the target thread is manipulating the global state of a shared DLL, 
+       the state of the DLL could be destroyed, affecting other users of the DLL.
+    */
+    TERMINATE_CLEANUP = 0x4 /* cleanup our state before issuing terminal syscall */
+} terminate_flags_t;
+
+void os_terminate(dcontext_t *dcontext, terminate_flags_t flags);
+
+typedef enum {
+    ILLEGAL_INSTRUCTION_EXCEPTION,
+    UNREADABLE_MEMORY_EXECUTION_EXCEPTION,
+    IN_PAGE_ERROR_EXCEPTION,
+} exception_type_t;
+
+void os_forge_exception(app_pc exception_address, exception_type_t type);
+
+/* events for dumpcore_mask */
+/* NOTE with DUMPCORE_DEADLOCK and DUMPCORE_ASSERT you will get 2 dumps for
+ * rank order violations in debug builds */
+enum {
+    DUMPCORE_INTERNAL_EXCEPTION = 0x0001,
+    DUMPCORE_SECURITY_VIOLATION = 0x0002,
+    DUMPCORE_DEADLOCK           = 0x0004,
+    DUMPCORE_ASSERTION          = 0x0008,
+    DUMPCORE_FATAL_USAGE_ERROR  = 0x0010,
+    DUMPCORE_UNSUPPORTED_APP    = 0x0020,
+    DUMPCORE_TIMEOUT            = 0x0040,
+    DUMPCORE_CURIOSITY          = 0x0080,
+#ifdef HOT_PATCHING_INTERFACE       /* Part of fix for 5357 & 5988. */
+    /* Errors and exceptions in hot patches are treated the same.  Case 5696. */
+    DUMPCORE_HOTP_FAILURE       = 0x0100,
+#endif
+    DUMPCORE_OUT_OF_MEM         = 0x0200,
+    DUMPCORE_OUT_OF_MEM_SILENT  = 0x0400, /* not on by default in DEBUG */
+#ifdef LINUX
+    DUMPCORE_WAIT_FOR_DEBUGGER  = 0x1000, /* not on by default in DEBUG */
+#endif
+#ifdef HOT_PATCHING_INTERFACE       /* Part of fix for 5988. */
+    DUMPCORE_HOTP_DETECTION     = 0x2000, /* not on by default in DEBUG */
+    DUMPCORE_HOTP_PROTECTION    = 0x4000, /* not on by default in DEBUG */
+#endif
+    /* all exception cases here are off by default since we expect them to
+     * usually be the app's fault (or normal behavior for the app) */
+    DUMPCORE_FORGE_ILLEGAL_INST = 0x10000,
+    DUMPCORE_FORGE_UNREAD_EXEC  = 0x20000, /* not including -throw_exception */
+    /* Note this is ALL app exceptions (including those the app may expect and
+     * handle without issue) except those created via RaiseException (note
+     * dr forged exceptions use the eqv. of RaiseException).  Would be nice to
+     * have a flag for just unhandled app exceptions but that's harder to
+     * implement. */
+    DUMPCORE_APP_EXCEPTION      = 0x40000,
+    DUMPCORE_TRY_EXCEPT         = 0x80000, /* even when we do have a handler */
+
+#ifdef LINUX
+    DUMPCORE_OPTION_PAUSE       = DUMPCORE_WAIT_FOR_DEBUGGER  |
+                                  DUMPCORE_INTERNAL_EXCEPTION |
+                                  DUMPCORE_SECURITY_VIOLATION |
+                                  DUMPCORE_DEADLOCK           |
+                                  DUMPCORE_ASSERTION          |
+                                  DUMPCORE_FATAL_USAGE_ERROR  |
+                                  DUMPCORE_UNSUPPORTED_APP    |
+                                  DUMPCORE_TIMEOUT            |
+                                  DUMPCORE_CURIOSITY          |
+                                  DUMPCORE_OUT_OF_MEM         |
+                                  DUMPCORE_OUT_OF_MEM_SILENT,
+#endif
+};
+void os_dump_core(const char *msg);
+
+int os_timeout(int time_in_milliseconds);
+
+void os_syslog(syslog_event_type_t priority, uint message_id, 
+               uint substitutions_num, va_list args);
+
+#if defined(CLIENT_INTERFACE) || defined(HOT_PATCHING_INTERFACE)
+typedef void * shlib_handle_t;
+typedef void (*shlib_routine_ptr_t)();
+
+shlib_handle_t load_shared_library(char *name);
+#endif
+
+#if defined(CLIENT_INTERFACE)
+shlib_routine_ptr_t lookup_library_routine(shlib_handle_t lib, char *name);
+void unload_shared_library(shlib_handle_t lib);
+void shared_library_error(char *buf, int maxlen);
+/* addr is any pointer known to lie within the library */
+bool shared_library_bounds(IN shlib_handle_t lib, IN byte *addr,
+                           OUT byte **start, OUT byte **end);
+#endif
+
+/* NOTE - keep these in sync with the client API counterparts in instrument.h */
+#define MEMPROT_NONE  0x00
+#define MEMPROT_READ  0x01
+#define MEMPROT_WRITE 0x02
+#define MEMPROT_EXEC  0x04
+bool get_memory_info(const byte *pc, byte **base_pc, size_t *size, uint *prot);
+#ifdef LINUX
+bool get_memory_info_from_os(const byte *pc, byte **base_pc, size_t *size, uint *prot);
+#endif
+
+bool get_stack_bounds(dcontext_t *dcontext, byte **base, byte **top);
+
+bool is_readable_without_exception(const byte *pc, size_t size);
+bool is_readable_without_exception_query_os(byte *pc, size_t size);
+bool safe_read(const void *base, size_t size, void *out_buf);
+bool safe_read_ex(const void *base, size_t size, void *out_buf, size_t *bytes_read);
+bool safe_write_ex(void *base, size_t size, const void *in_buf, size_t *bytes_written);
+bool is_user_address(byte *pc);
+/* returns osprot flags preserving all native protection flags except
+ * for RWX, which are replaced according to memprot */
+uint
+osprot_replace_memprot(uint old_osprot, uint memprot);
+
+/* returns false if out of memory */
+bool set_protection(byte *pc, size_t size, uint prot);
+/* Change protections on memory region starting at pc of length size 
+ * (padded to page boundaries).  This method is meant to be used on DR memory 
+ * as part of protect from app and is safe with respect to stats and changing
+ * the protection of the data segment. */
+/* returns false if out of memory */
+bool change_protection(byte *pc, size_t size, bool writable);
+#ifdef WINDOWS
+/* makes pc:pc+size (page_padded) writable preserving other flags 
+ * returns false if out of memory
+ */
+bool make_hookable(byte *pc, size_t size, bool *changed_prot);
+/* if changed_prot makes pc:pc+size (page padded) unwritable preserving 
+ * other flags */
+void make_unhookable(byte *pc, size_t size, bool changed_prot);
+#endif
+/* requires that pc is page aligned and size is multiple of the page size
+ * and marks that memory writable, preserves other flags,
+ * returns false if out of memory
+ */
+bool make_writable(byte *pc, size_t size);
+/* requires that pc is page aligned and size is multiple of the page size
+ * and marks that memory NOT writable, preserves other flags */
+void make_unwritable(byte *pc, size_t size);
+/* like make_writable but adds COW (note: only usable if allocated COW) */
+bool make_copy_on_writable(byte *pc, size_t size);
+
+/***************************************************************************
+ * SELF_PROTECTION
+ */
+
+/* Values for protect_mask to specify what is write-protected from malicious or
+ * inadvertent modification by the application.
+ * DATA_CXTSW and GLOBAL are done on each context switch
+ * the rest are on-demand:
+ * DATASEGMENT, DATA_FREQ, and GENCODE only on the rare occasions when we write to them
+ * CACHE only when emitting or {,un}linking
+ * LOCAL only on path in DR that needs to write to local
+ */
+enum {
+    /* Options specifying protection of our DR dll data sections: */
+    /* .data == variables written only at init or exit time or rarely in between */
+    SELFPROT_DATA_RARE   = 0x001,
+    /* .fspdata == frequently written enough that we separate from .data.
+     * FIXME case 8073: currently these are unprotected on every cxt switch
+     */
+    SELFPROT_DATA_FREQ   = 0x002,
+    /* .cspdata == so frequently written that to protect them requires unprotecting
+     * every context switch.
+     */
+    SELFPROT_DATA_CXTSW  = 0x004,
+
+    /* if GLOBAL && !DCONTEXT, entire dcontext is unprotected, rest of
+     *   global allocs are protected;
+     * if GLOBAL && DCONTEXT, cache-written fields of dcontext are unprotected,
+     *   rest are protected;
+     * if !GLOBAL, DCONTEXT should not be used
+     */
+    SELFPROT_GLOBAL      = 0x008,
+    SELFPROT_DCONTEXT    = 0x010, /* means we split out unprotected_context_t -- 
+                                   * no actual protection unless SELFPROT_GLOBAL */
+    SELFPROT_LOCAL       = 0x020,
+    SELFPROT_CACHE       = 0x040, /* FIXME: thread-safe NYI when doing all units */
+    SELFPROT_STACK       = 0x080, /* essentially always on with clean-dstack dispatch()
+                                   * design, leaving as a bit in case we do more later */
+    /* protect our generated thread-shared and thread-private code */
+    SELFPROT_GENCODE     = 0x100,
+    /* FIXME: TEB page on Win32 
+     * Other global structs, like thread-local callbacks on Win32?
+     * PEB page?
+     */
+    /* options that require action on every context switch
+     * FIXME: global heap used to be much rarer before shared
+     * fragments, only containing "important" data, which is why we
+     * un-protected on every context switch.  We should re-think that
+     * now that most things are shared.
+     */
+    SELFPROT_ON_CXT_SWITCH = (SELFPROT_DATA_CXTSW | SELFPROT_GLOBAL
+                              /* FIXME case 8073: this is only temporary until
+                               * we finish implementing .fspdata unprots */
+                              | SELFPROT_DATA_FREQ),
+    SELFPROT_ANY_DATA_SECTION = (SELFPROT_DATA_RARE | SELFPROT_DATA_FREQ |
+                                 SELFPROT_DATA_CXTSW),
+};
+
+/* Values to refer to individual data sections.  The order is not
+ * important here.
+ */
+enum {
+    DATASEC_NEVER_PROT = 0,
+    DATASEC_RARELY_PROT,
+    DATASEC_FREQ_PROT,
+    DATASEC_CXTSW_PROT,
+    DATASEC_NUM,
+};
+
+/* in dynamo.c */
+extern const uint DATASEC_SELFPROT[];
+extern const char * const DATASEC_NAMES[];
+/* see dynamo.c for why this is not an array */
+extern const uint datasec_writable_neverprot;
+extern uint datasec_writable_rareprot;
+extern uint datasec_writable_freqprot;
+extern uint datasec_writable_cxtswprot;
+
+/* this is a uint not a bool */
+#define DATASEC_WRITABLE(which) \
+    ((which) == DATASEC_RARELY_PROT ? datasec_writable_rareprot : \
+     ((which) == DATASEC_CXTSW_PROT ? datasec_writable_cxtswprot : \
+      ((which) == DATASEC_FREQ_PROT ? datasec_writable_freqprot : \
+       datasec_writable_neverprot)))
+
+/* these must be plain literals since we need these in pragmas/attributes */
+#define NEVER_PROTECTED_SECTION  ".nspdata"
+#define RARELY_PROTECTED_SECTION ".data"
+#define FREQ_PROTECTED_SECTION   ".fspdata"
+#define CXTSW_PROTECTED_SECTION  ".cspdata"
+
+/* note that asserting !protected is safe, but asserting protection
+ * is racy as another thread could be in an unprot window.
+ * see also check_should_be_protected().
+ */
+#define DATASEC_PROTECTED(which) (DATASEC_WRITABLE(which) == 0)
+
+/* User MUST supply an init value, or else cl will leave in .bss (will show up
+ * at end of .data), which is why we hardcode the = here!
+ * We use varargs to allow commas in the init if a struct.
+ */
+#define DECLARE_FREQPROT_VAR(var, ...)                        \
+    START_DATA_SECTION(FREQ_PROTECTED_SECTION, "w")           \
+    var VAR_IN_SECTION(FREQ_PROTECTED_SECTION) = __VA_ARGS__; \
+    END_DATA_SECTION() 
+
+#define DECLARE_CXTSWPROT_VAR(var, ...)                        \
+    START_DATA_SECTION(CXTSW_PROTECTED_SECTION, "w")           \
+    var VAR_IN_SECTION(CXTSW_PROTECTED_SECTION) = __VA_ARGS__; \
+    END_DATA_SECTION() 
+
+#define DECLARE_NEVERPROT_VAR(var, ...)                        \
+    START_DATA_SECTION(NEVER_PROTECTED_SECTION, "w")           \
+    var VAR_IN_SECTION(NEVER_PROTECTED_SECTION) = __VA_ARGS__; \
+    END_DATA_SECTION() 
+
+#define SELF_PROTECT_ON_CXT_SWITCH \
+    (TESTANY(SELFPROT_ON_CXT_SWITCH, DYNAMO_OPTION(protect_mask)) \
+     || INTERNAL_OPTION(single_privileged_thread))
+
+/* macros to put mask check outside of function, for efficiency */
+#define SELF_PROTECT_LOCAL(dc, w) do { \
+    if (TEST(SELFPROT_LOCAL, dynamo_options.protect_mask)) \
+        protect_local_heap(dc, w);                         \
+} while (0);
+
+#define SELF_PROTECT_GLOBAL(w) do { \
+    if (TEST(SELFPROT_GLOBAL, dynamo_options.protect_mask)) \
+        protect_global_heap(w);                             \
+} while (0);
+
+#define ASSERT_LOCAL_HEAP_PROTECTED(dcontext) \
+    ASSERT(!TEST(SELFPROT_LOCAL, dynamo_options.protect_mask) || \
+           local_heap_protected(dcontext))
+#define ASSERT_LOCAL_HEAP_UNPROTECTED(dcontext) \
+    ASSERT(!TEST(SELFPROT_LOCAL, dynamo_options.protect_mask) || \
+           !local_heap_protected(dcontext))
+
+#define SELF_PROTECT_DATASEC(which) do { \
+    if (TEST(DATASEC_SELFPROT[which], DYNAMO_OPTION(protect_mask))) \
+        protect_data_section(which, READONLY);    \
+} while (0);
+#define SELF_UNPROTECT_DATASEC(which) do { \
+    if (TEST(DATASEC_SELFPROT[which], DYNAMO_OPTION(protect_mask))) \
+        protect_data_section(which, WRITABLE);      \
+} while (0);
+
+/***************************************************************************/
+
+#ifdef DEBUG
+void mem_stats_snapshot(void);
+#endif
+
+app_pc get_dynamorio_dll_start(void);
+app_pc get_dynamorio_dll_preferred_base(void);
+
+bool is_in_dynamo_dll(app_pc pc);
+int find_dynamo_library_vm_areas(void);
+int find_executable_vm_areas(void);
+
+/* all_memory_areas is linux only, dummy on win32 */
+void all_memory_areas_lock(void);
+void all_memory_areas_unlock(void);
+void update_all_memory_areas(app_pc start, app_pc end, uint prot);
+bool remove_from_all_memory_areas(app_pc start, app_pc end);
+
+/* file operations */
+/* defaults to read only access, if write is not set ignores others */
+#define OS_OPEN_READ        0x01
+#define OS_OPEN_WRITE       0x02
+#define OS_OPEN_APPEND      0x04
+#define OS_OPEN_REQUIRE_NEW 0x08
+#define OS_EXECUTE          0x10 /* only used on win32, currently */
+#define OS_SHARE_DELETE     0x20 /* only used on win32, currently */
+#define OS_OPEN_FORCE_OWNER 0x40 /* only used on win32, currently */
+#define OS_OPEN_ALLOW_LARGE 0x80 /* only used on linux32, currently */
+/* always use OS_OPEN_REQUIRE_NEW when asking for OS_OPEN_WRITE, in
+ * order to avoid hard link or symbolic link attacks if the file is in
+ * a world writable locations and the process may have high
+ * privileges.
+ */
+file_t os_open(const char *fname, int os_open_flags);
+file_t os_open_directory(const char *fname, int os_open_flags);
+/* FIXME : os_file_exists always returns true on linux */
+bool os_file_exists(const char *fname, bool is_dir);
+bool os_get_file_size(const char *file, uint64 *size); /* NYI on Linux */
+bool os_get_file_size_by_handle(file_t fd, uint64 *size);
+
+typedef enum {
+    CREATE_DIR_ALLOW_EXISTING = 0x0,
+    /* should always use CREATE_DIR_REQUIRE_NEW, see reference in
+     * OS_OPEN_REQUIRE_NEW - to avoid symlink attacks (although only
+     * an issue when files we create in these directories have
+     * predictible names - case 9138)
+     */
+    CREATE_DIR_REQUIRE_NEW    = 0x1,
+    CREATE_DIR_FORCE_OWNER    = 0x2,
+} create_directory_flags_t;
+
+bool os_create_dir(const char *fname, create_directory_flags_t create_dir_flags);
+void os_close(file_t f);
+/* returns number of bytes written, negative if failure */
+ssize_t os_write(file_t f, const void *buf, size_t count);
+/* returns number of bytes read, negative if failure */
+ssize_t os_read(file_t f, void *buf, size_t count);
+void os_flush(file_t f);
+
+/* For use with os_file_seek(), specifies the origin at which to apply the offset
+ * NOTE - keep in synch with DR_SEEK_* in insturment.h and SEEK_* from Linux headers */
+#define OS_SEEK_SET 0  /* start of file */
+#define OS_SEEK_CUR 1  /* current file position */
+#define OS_SEEK_END 2  /* end of file */
+/* seek the current file position to offset bytes from origin, return true if succesful */
+bool os_seek(file_t f, int64 offset, int origin);
+/* return the current file position, -1 on failure */
+int64 os_tell(file_t f);
+
+bool os_delete_file(const wchar_t *file_name,
+                    file_t directory_handle);
+bool os_delete_mapped_file(const char *filename);
+bool os_rename_file(const char *orig_name, const char *new_name, bool replace);
+/* These routines do not update dynamo_areas; use the non-os_-prefixed versions */
+/* File mapping on Linux uses a 32-bit offset, with mmap2 considering
+ * it a multiple of the page size in order to have a larger reach; on
+ * Windows we have a 64-bit offset.  We go with the widest here, a
+ * 64-bit offset, to avoid limiting Windows.
+ */
+byte *os_map_file(file_t f, size_t size, uint64 offs, app_pc addr,
+                  uint prot, bool copy_on_write);
+bool os_unmap_file(byte *map, size_t size);
+
+bool
+os_current_user_directory(char *directory_prefix /* INOUT */,
+                          uint directory_size,
+                          bool create);
+bool
+os_validate_user_owned(file_t file_or_directory_handle);
+
+
+bool
+os_get_disk_free_space(/*IN*/ file_t file_handle,
+                       /*OUT*/ uint64 *AvailableQuotaBytes /*OPTIONAL*/,
+                       /*OUT*/ uint64 *TotalQuotaBytes /*OPTIONAL*/,
+                       /*OUT*/ uint64 *TotalVolumeBytes /*OPTIONAL*/);
+
+#ifdef PROFILE_RDTSC
+extern uint kilo_hertz;
+#endif
+
+/* Despite the name, this enum is used for all types of critical events:
+es
+ * asserts and crashes for all builds, and security violations for PROGRAM_SHEPHERDING builds
+ * When a security_violation is being reported, enum value must be negative! 
+ */
+typedef enum {
+#ifdef PROGRAM_SHEPHERDING
+    STACK_EXECUTION_VIOLATION       = -1,
+    HEAP_EXECUTION_VIOLATION        = -2,
+    RETURN_TARGET_VIOLATION         = -3, 
+    RETURN_DIRECT_RCT_VIOLATION     = -4, /* NYI DIRECT_CALL_CHECK */
+    INDIRECT_CALL_RCT_VIOLATION     = -5,
+    INDIRECT_JUMP_RCT_VIOLATION     = -6,
+# ifdef HOT_PATCHING_INTERFACE
+    HOT_PATCH_DETECTOR_VIOLATION    = -7,
+    HOT_PATCH_PROTECTOR_VIOLATION   = -8,
+    /* Errors and exceptions in hot patches are treated the same.  Case 5696. */
+    HOT_PATCH_FAILURE               = -9,
+# endif
+    /* internal */
+    ATTACK_SIMULATION_VIOLATION     = -10,
+    ATTACK_SIM_NUDGE_VIOLATION      = -11,
+#endif
+    /* not really PROGRAM_SHEPHERDING */
+    ASLR_TARGET_VIOLATION           = -12,
+#ifdef GBOP
+    GBOP_SOURCE_VIOLATION           = -13,
+#endif /* GBOP */
+#ifdef PROCESS_CONTROL
+    PROCESS_CONTROL_VIOLATION       = -14,      /* case 8594 */
+#endif
+    APC_THREAD_SHELLCODE_VIOLATION  = -15, /* note still presented externally as .B */
+    /* Not a valid violation; used for initializing security_violation_t vars. */
+    INVALID_VIOLATION               = 0,
+#ifdef PROGRAM_SHEPHERDING
+    /* Add new violation types above this line as consecutive negative
+     * numbers, and update get_security_violation_name() for the
+     * appropriate letter obfuscation */
+    ALLOWING_OK                     =  1,
+    ALLOWING_BAD                    =  2,    
+#endif
+    NO_VIOLATION_BAD_INTERNAL_STATE =  3,
+    NO_VIOLATION_OK_INTERNAL_STATE  =  4
+} security_violation_t;
+
+/*  in diagnost.c */
+void report_diagnostics(const char *message, const char *name,
+                        security_violation_t violation_type);
+void append_diagnostics(file_t diagnostics_file, const char *message, const char *name,
+                        security_violation_t violation_type);
+void diagnost_exit(void);
+bool check_for_unsupported_modules(void);
+
+#ifdef RETURN_AFTER_CALL
+typedef enum {
+    INITIAL_STACK_EMPTY              = 0,
+    INITIAL_STACK_BOTTOM_REACHED     = 1,
+    INITIAL_STACK_BOTTOM_NOT_REACHED = 2
+} initial_call_stack_status_t;
+
+initial_call_stack_status_t at_initial_stack_bottom(dcontext_t *dcontext, app_pc target_pc);
+bool at_known_exception(dcontext_t *dcontext, app_pc target_pc, app_pc source_fragment);
+#endif
+
+/* contended path of mutex operations */
+void mutex_wait_contended_lock(mutex_t *lock);
+void mutex_notify_released_lock(mutex_t *lock);
+/* contended path of rwlock operations */
+void rwlock_wait_contended_writer(read_write_lock_t *rwlock);
+void rwlock_notify_writer(read_write_lock_t *rwlock);
+void rwlock_wait_contended_reader(read_write_lock_t *rwlock);
+void rwlock_notify_readers(read_write_lock_t *rwlock);
+
+/* The event interface */
+
+/* An event object. */
+#ifdef WINDOWS
+typedef HANDLE event_t;
+#else
+typedef struct linux_event_t *event_t;
+#endif
+
+event_t create_event(void);
+void destroy_event(event_t e);
+void signal_event(event_t e);
+void reset_event(event_t e);
+void wait_for_event(event_t e);
+
+/* get current timer frequency in KHz */
+/* NOTE: Keep in mind that with voltage scaling in power saving mode
+ * this may not be a reliable way to obtain time information */
+timestamp_t
+get_timer_frequency(void);
+
+/* seconds since 1970 */
+uint
+query_time_seconds(void);
+
+/* gives a good but not necessarily crypto-strength random seed */
+uint
+os_random_seed(void);
+
+/* module map/unmap processing relevant to the RCT policies */
+/* cf. rct_analyze_module_at_violation() which can be called lazily */
+void
+rct_process_module_mmap(app_pc module_base, size_t module_size, 
+                        bool add, bool already_relocated);
+
+/* module boundary and code section analysis for RCT policies */
+/* FIXME: should be better abstracted by calling a routine that
+ * enumerates code sections, while keeping the general driver in rct.c
+ */
+bool
+rct_analyze_module_at_violation(dcontext_t *dcontext, app_pc target_pc);
+bool
+aslr_is_possible_attack(app_pc target);
+app_pc
+aslr_possible_preferred_address(app_pc target_addr);
+
+#ifdef WINDOWS
+/* dispatch places next pc in asynch_target and clears it after syscall handling
+ * completes, so a zero value means shared syscall was used and the next pc
+ * is in the esi slot. If asynch_target == BACK_TO_NATIVE_AFTER_SYSCALL then the
+ * thread is native at an intercepted syscall and the real post syscall target is in
+ * native_exec_postsyscall. */
+# define POST_SYSCALL_PC(dc)                                                      \
+     ((dc)->asynch_target == NULL ?                                               \
+      ASSERT(DYNAMO_OPTION(shared_syscalls)), (app_pc) get_mcontext((dc))->xsi :  \
+      ((dc)->asynch_target == BACK_TO_NATIVE_AFTER_SYSCALL ?                      \
+       (dc)->native_exec_postsyscall : (dc)->asynch_target))
+#else
+/* on linux asynch target always holds the post syscall pc */
+# define POST_SYSCALL_PC(dc) ((dc)->asynch_target)
+#endif
+
+/* This has been exposed from win32 module so that hotp_only_gateway can 
+ * return the right return code.
+ */
+typedef enum {
+    AFTER_INTERCEPT_LET_GO,
+    AFTER_INTERCEPT_LET_GO_ALT_DYN, /* alternate direct execution target,
+                                     * usable only with
+                                     * AFTER_INTERCEPT_DYNAMIC_DECISION,
+                                     * not by itself */
+    AFTER_INTERCEPT_TAKE_OVER,
+    AFTER_INTERCEPT_DYNAMIC_DECISION, /* handler returns one of prev values */
+
+    /* handler is expected to restore original instructions */
+    AFTER_INTERCEPT_TAKE_OVER_SINGLE_SHOT, /* static only with alternative target */
+} after_intercept_action_t;
+
+/* Argument structure for intercept function; contains app state at the point
+ * our intercept routine will take over.  Fix for case 7597.
+ *
+ * -- CAUTION -- the number, order and size of fields in this structure are
+ * assumed in emit_intercept_code().  Changing anything here is likely to cause
+ * DR hooks & hotp_only to fail.
+ */
+typedef struct {
+    void *callee_arg;     /* Argument passed to the intercept routine. */
+    app_pc start_pc; /* optimization: could use mc.pc instead */
+    int app_errno;
+#ifdef X64
+    /* 4 bytes of padding since dr_mcontext_t is 8-byte aligned, but it's handled
+     * naturally in emit_intercept_code() since pushes are all 8 bytes */ 
+#endif
+    dr_mcontext_t mc;
+} app_state_at_intercept_t;
+
+/* note that only points intercepted with DYNAMIC_DECISION (currently only
+ * [un]load_dll) care about the return value */ 
+typedef after_intercept_action_t intercept_function_t(app_state_at_intercept_t *args);
+
+/* opcodes and encoding constants that we sometimes directly use */
+#ifdef X86
+/* FIXME: if we add more opcodes maybe should be exported by x86/ */
+enum {
+    JMP_REL32_OPCODE  = 0xe9,
+    JMP_REL32_SIZE    = 5,      /* size in bytes of 32-bit rel jmp */
+    CALL_REL32_OPCODE = 0xe8,
+# ifdef X64
+    JMP_ABS_IND64_OPCODE    = 0xff,
+    JMP_ABS_IND64_SIZE      = 6,    /* size in bytes of a 64-bit abs ind jmp */
+    JMP_ABS_MEM_IND64_MODRM = 0x25,
+# endif
+};
+#else /* !X86 */
+#  error only X86 supported 
+#endif /* X86 */
+
+#ifdef WINDOWS
+/* used for option early_inject_location */
+/* use 0 to supply an arbitrary address via -early_inject_address */
+/* NOTE - these enum values are passed between processes, don't change/reuse
+ * any existing values. */ 
+/* NtMapViewOfSection doesn't work as a place to LoadDll (ldr can't handle the
+ * re-entrancy), but might be a good place to takeover if we remote mapped in
+ * and needed ntdll initialized (first MapView is for kernel32.dll fom what
+ * I've seen, though beware the LdrLists aren't consistent at this point). */
+/* As currently implemented KiUserException location doesn't work, but we might
+ * be able to find a better exception location (maybe point the image's import
+ * section to invalid memory instead? TODO try). */
+/* see early_inject_init() in os.c for more notes on best location for each
+ * os version */
+/* NOTE - be carefull changing this enum as it's shared across proccesses. 
+ * Also values <= LdrDefault are assumed to need address computation while
+ * those > are assumed to not need it. */
+#define INJECT_LOCATION_IS_LDR(loc) (loc <= INJECT_LOCATION_LdrDefault)
+#define INJECT_LOCATION_IS_LDR_NON_DEFAULT(loc) (loc < INJECT_LOCATION_LdrDefault)
+enum {
+    INJECT_LOCATION_Invalid          = -100, /* invalid sentinel */
+    INJECT_LOCATION_LdrpLoadDll      = 0, /* Good for XP, 2k3 */
+    INJECT_LOCATION_LdrpLoadImportModule = 1, /* Good for 2k */
+    INJECT_LOCATION_LdrCustom        = 2, /* Specify custom address */
+    INJECT_LOCATION_LdrLoadDll       = 3, /* Good for 2k3 */
+    INJECT_LOCATION_LdrDefault       = 4, /* Pick best location based on OS */
+    /* For NT the best location is LdrpLoadImportModule, but we can't find that
+     * automatically on NT, so LdrDefault for NT uses -early_inject_address if
+     * specified or else disables early injection (xref 7806). */
+    /* Beyond this point not expected to need address determination */
+    INJECT_LOCATION_KiUserApc        = 5, /* NYI as must use map wich is NYI */
+    INJECT_LOCATION_KiUserException  = 6, /* No good, Ldr init issues */
+    INJECT_LOCATION_MAX = INJECT_LOCATION_KiUserException,
+};
+#endif
+
+void take_over_primary_thread(void);
+
+#ifdef HOT_PATCHING_INTERFACE
+void *
+get_drmarker_hotp_policy_status_table(void);
+void
+set_drmarker_hotp_policy_status_table(void *new_table);
+
+/* Used for -hotp_only; can be exposed if needed elsewhere later on. */
+byte *
+hook_text(byte *hook_code_buf, const app_pc image_addr, 
+          intercept_function_t hook_func, const void *callee_arg, 
+          const after_intercept_action_t action_after,
+          const bool abort_if_hooked, const bool ignore_cti,
+          byte **app_code_copy_p, byte **alt_exit_cti_p);
+void
+unhook_text(byte *hook_code_buf, app_pc image_addr);
+void
+insert_jmp_at_tramp_entry(byte *trampoline, byte *target);
+#endif /* HOT_PATCHING_INTERFACE */
+
+/* checks for compatibility OS specific options, returns true if
+ * modified the value of any options to make them compatible
+ */
+bool
+os_check_option_compatibility(void);
+
+/* Introduced as part of PR 250294 - 64-bit hook reachability. */
+#define LANDING_PAD_AREA_SIZE   64*1024
+#ifdef X64
+/* 8 bytes for the 64-bit abs addr, 6 for abs ind jmp to the trampoline and 5
+ * for return jmp back to the instruction after the hook. */
+# define LANDING_PAD_SIZE    19
+#else
+/* 5 bytes each for the two relative jumps (one to the trampoline and the 
+ * other back to instruction after hook. */
+# define LANDING_PAD_SIZE    10
+#endif
+byte *alloc_landing_pad(app_pc addr_to_hook);
+void landing_pads_to_executable_areas(bool add);
+
+#endif /* OS_SHARED_H */
diff --git a/core/perfctr.c b/core/perfctr.c
new file mode 100644
index 000000000..2bd145317
--- /dev/null
+++ b/core/perfctr.c
@@ -0,0 +1,162 @@
+/* **********************************************************
+ * Copyright (c) 2001-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+
+#include "globals.h"
+
+#ifdef PAPI /* around entire file */
+
+#include "perfctr.h"
+#include "x86-events.h"
+#include "papi.h"
+#include <stdlib.h>
+#include <stdio.h>
+#include <time.h>
+
+extern dr_statistics_t *stats;
+int perfctr_eventset;
+struct {int num;char *name;} papi_events[]={
+    {P6_CPU_CLK_UNHALTED<<8,"Total cycles"},
+    {P6_INST_DECODER<<8,"Instructions decoded"},
+    {P6_INST_RETIRED<<8,"Instructions Retired"},
+    {P6_UOPS_RETIRED<<8,"Micro-ops retired"},
+    {P6_DATA_MEM_REFS<<8,"Total data memory refs (loads + stores"},
+    {P6_DCU_MISS_OUTSTANDING<<8,"DCU miss cycles outstanding"},
+    {P6_IFU_FETCH<<8,"Instruction Fetches"},
+    {P6_IFU_FETCH_MISS<<8,"IFU (instruction L1 ?) misses"},
+    {P6_ITLB_MISS<<8,"ITLB misses"},
+    {P6_IFU_MEM_STALL<<8,"Cycles instruction fetch is stalled"},
+    {P6_ILD_STALL<<8,"Cycles instruction length decoder stalled"},
+    {((0x0f<<8)|P6_L2_IFETCH)<<8,"L2 instruction fetches"},
+    {((0x0f<<8)|P6_L2_LD)<<8,"L2 loads"},
+    {((0x0f<<8)|P6_L2_ST)<<8,"L2 stores"},
+    {P6_BR_INST_DECODED<<8,"Branch instructions decoded"},
+    {P6_BR_INST_RETIRED<<8,"Branch instructions retired"},
+    {P6_BR_MISS_PRED_RETIRED<<8,"Branch insts mispredicted retired"},
+    {P6_BR_TAKEN_RETIRED<<8,"Branch insts taken retired"},
+    {P6_BR_MISS_PRED_TAKEN_RET<<8,"Branch insts mispredicted, taken retired"},
+    {P6_BTB_MISSES<<8,"BTB misses"},
+    {P6_BR_BOGUS<<8,"Bogus Branches"},
+    {P6_RESOURCE_STALLS<<8,"Misc. resource stalls"},
+    {P6_BACLEARS<<8,"BACLEAR asserted"},
+    {P6_DCU_LINES_IN<<8,"DCU lines allocated"},
+    {P6_L2_LINES_IN<<8,"L2 lines allocated"},
+    {PIII_EMON_KNI_PREF_DISPATCHED<<8,"Prefetch NTA dispatched"},
+    {PIII_EMON_KNI_PREF_MISS<<8,"Prefetch NTA miss all caches"}
+};
+    
+
+
+
+void hardware_perfctr_init()
+{
+    int a;
+    perfctr_eventset=PAPI_NULL;
+
+    ASSERT(NUM_EVENTS==(sizeof(papi_events)/8));
+
+#ifdef LINUX
+    ASSERT(!INTERNAL_OPTION(profile_pcs)); 
+#endif
+
+    LOG(GLOBAL, LOG_TOP, 1,"Initializing PAPI\n");
+
+    
+    if (PAPI_library_init(PAPI_VER_CURRENT) !=PAPI_VER_CURRENT) {
+            LOG(GLOBAL, LOG_TOP, 1,"Error initializing PAPI.\n");
+    }
+    
+    if (NUM_EVENTS > 2) {
+        LOG(GLOBAL, LOG_TOP, 3,"Initializing PAPI multiplexing\n");
+        if (PAPI_multiplex_init() != PAPI_OK) {
+            if (stats->loglevel > 0 && (stats->logmask & LOG_TOP) != 0) 
+                LOG(GLOBAL, LOG_TOP, 1,"Error initializing PAPI multiplexing\n");
+        }
+    }
+    if (PAPI_create_eventset(&perfctr_eventset) != PAPI_OK) {
+        LOG(GLOBAL, LOG_TOP, 1,"Error creating PAPI eventset\n");
+    }
+
+
+    
+    if (NUM_EVENTS>2) {
+        if (PAPI_set_multiplex(&perfctr_eventset)!=PAPI_OK) {
+            LOG(GLOBAL, LOG_TOP, 1,"Error setting multiplexed eventset in PAPI\n");
+        }
+    }
+    
+    for (a=0;a<NUM_EVENTS;a++) {
+        if (PAPI_add_event(&perfctr_eventset,papi_events[a].num)!=PAPI_OK) {
+                LOG(GLOBAL, LOG_TOP, 1,"Error adding events in PAPI\n");
+        }
+    }
+
+    if (PAPI_start(perfctr_eventset)!=PAPI_OK) {
+        LOG(GLOBAL, LOG_TOP, 1,"Error starting hardware performance counters\n");
+    }
+    
+}
+
+        
+void hardware_perfctr_exit()
+{
+    int a;
+    uint64 vals[NUM_EVENTS]; /* only used if nullcalls */
+    uint64 *val_array;
+
+    if (dynamo_options.nullcalls) {
+        val_array = vals;
+    } else {
+        val_array = stats->perfctr_vals;
+    }
+    if (PAPI_stop(perfctr_eventset, val_array) != PAPI_OK) {
+        LOG(GLOBAL, LOG_TOP, 1,
+            "Error stopping and reading hardware performance counters\n");
+    }
+    for (a=0; a<NUM_EVENTS; a++) {
+        LOG(GLOBAL, LOG_TOP, 1, "Counter %d = %llu (%s)\n",
+            a+1, val_array[a], papi_events[a].name);
+    }
+    
+}
+
+void perfctr_update_gui()
+{
+    if (PAPI_read(perfctr_eventset,stats->perfctr_vals) != PAPI_OK) {
+        LOG(GLOBAL, LOG_TOP, 1,
+            "Error stopping and reading hardware performance counters\n");
+    }
+}
+
+#endif /* PAPI */
diff --git a/core/perfctr.h b/core/perfctr.h
new file mode 100644
index 000000000..caafb343d
--- /dev/null
+++ b/core/perfctr.h
@@ -0,0 +1,50 @@
+/* **********************************************************
+ * Copyright (c) 2001-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+
+#ifdef PAPI
+
+#include "x86-events.h"
+#define GUI_ALARM_UPDATE_RATIO 1
+extern int perfctr_eventset;
+
+
+
+
+void hardware_perfctr_init(void);
+void hardware_perfctr_exit(void);
+void perfctr_update_gui(void);
+
+
+#endif
diff --git a/core/perscache.c b/core/perscache.c
new file mode 100644
index 000000000..faecf8635
--- /dev/null
+++ b/core/perscache.c
@@ -0,0 +1,4031 @@
+/* **********************************************************
+ * Copyright (c) 2006-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2006-2007 Determina Corp. */
+
+/*
+ * perscache.c - coarse-grain units and persistent cache management
+ */
+
+#include "globals.h"
+#include "link.h"
+#include "fragment.h"
+#include "fcache.h"
+#include "monitor.h"
+#include "perscache.h"
+#include "instr.h"
+#include "decode_fast.h"
+#include "hotpatch.h"
+#include "synch.h"
+#include "module_shared.h"
+#include <string.h>  /* for memset */
+#include <stddef.h> /* for offsetof */
+
+#ifdef DEBUG
+# include "disassemble.h"
+#endif
+
+#define MAX_PCACHE_OPTIONS_STRING (MAX_OPTIONS_STRING/2)
+/* case 10823: align option string to keep hashtable data aligned.
+ * we're not using a cache-line-aligned lookuptable. */
+#define OPTION_STRING_ALIGNMENT (sizeof(app_pc))
+
+/* Forward decls */
+static void
+persist_calculate_module_digest(module_digest_t *digest, app_pc modbase, size_t modsize,
+                                app_pc code_start, app_pc code_end,
+                                uint validation_option);
+static bool
+get_persist_dir(char *directory /* OUT */,
+                uint directory_len,
+                bool create);
+
+#if defined(DEBUG) && defined(INTERNAL)
+static void
+print_module_digest(file_t f, module_digest_t *digest, const char *prefix);
+#endif
+
+static void
+coarse_unit_shift_jmps(dcontext_t *dcontext, coarse_info_t *info,
+                       ssize_t cache_shift, ssize_t stubs_shift, size_t old_mapsz);
+
+static void
+coarse_unit_merge_persist_info(dcontext_t *dcontext, coarse_info_t *dst,
+                               coarse_info_t *info1, coarse_info_t *info2);
+
+#ifdef DEBUG
+/* used below for pcache_dir_check_permissions() */
+DECLARE_CXTSWPROT_VAR(static mutex_t pcache_dir_check_lock,
+                      INIT_LOCK_FREE(pcache_dir_check_lock));
+#endif
+
+/***************************************************************************
+ * COARSE-GRAIN UNITS
+ */
+
+/* case 9653/10380: only one coarse unit in a module's +x region(s) is persisted */
+static void
+coarse_unit_mark_primary(coarse_info_t *info)
+{
+    if (!info->in_use)
+        return;
+#ifdef WINDOWS
+    /* FIXME PR 295529: put in for Linux once we have per-module flags */
+    /* Go ahead and get write lock up front; else have to check again; not
+     * frequently called so don't need perf opt here.
+     */
+    os_get_module_info_write_lock();
+    if (!os_module_get_flag(info->base_pc, MODULE_HAS_PRIMARY_COARSE)) {
+        os_module_set_flag(info->base_pc, MODULE_HAS_PRIMARY_COARSE);
+        ASSERT(os_module_get_flag(info->base_pc, MODULE_HAS_PRIMARY_COARSE));
+        info->primary_for_module = true;
+        LOG(GLOBAL, LOG_CACHE, 1, "marking "PFX"-"PFX" as primary coarse for %s\n",
+            info->base_pc, info->end_pc, info->module);
+    }
+    os_get_module_info_write_unlock();
+#else
+    info->primary_for_module = true;
+#endif
+}
+
+static void
+coarse_unit_unmark_primary(coarse_info_t *info)
+{
+#ifdef WINDOWS
+    /* FIXME PR 295529: put in for Linux once we have per-module flags */
+    if (info->primary_for_module && info->in_use) {
+        ASSERT(os_module_get_flag(info->base_pc, MODULE_HAS_PRIMARY_COARSE));
+        os_module_clear_flag(info->base_pc, MODULE_HAS_PRIMARY_COARSE);
+        info->primary_for_module = false;
+    }
+#else
+    info->primary_for_module = false;
+#endif
+}
+
+void
+coarse_unit_mark_in_use(coarse_info_t *info)
+{
+    info->in_use = true;
+    coarse_unit_mark_primary(info);
+}
+
+coarse_info_t *
+coarse_unit_create(app_pc base_pc, app_pc end_pc, module_digest_t *digest,
+                   bool for_execution)
+{
+    coarse_info_t *info = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, coarse_info_t,
+                                          /* FIXME: have separate heap acct? */
+                                          ACCT_VMAREAS, PROTECTED);
+    memset(info, 0, sizeof(*info));
+    ASSIGN_INIT_LOCK_FREE(info->lock, coarse_info_lock);
+    ASSIGN_INIT_LOCK_FREE(info->incoming_lock, coarse_info_incoming_lock);
+    info->base_pc = base_pc;
+    info->end_pc = end_pc;
+    /* FIXME: set PERSCACHE_X86_{32,64} here since for x64 the live
+     * unit's flags are used for 32-bit code in 64-bit processes.
+     * app_memory_allocation() may need a "bool x86_mode" param that is at
+     * least passed in to here: not clear if it should be stored in the
+     * vmarea.
+     */
+    DODEBUG({
+        info->is_local = false;
+        info->module = os_get_module_name_strdup(info->base_pc HEAPACCT(ACCT_VMAREAS));
+        if (info->module == NULL) {
+            /* else our LOG statements will crash */
+            info->module = dr_strdup("" HEAPACCT(ACCT_VMAREAS));
+        }
+    });
+    if (for_execution)
+        coarse_unit_mark_in_use(info);
+    if (digest != NULL) {
+        memcpy(&info->module_md5, digest, sizeof(info->module_md5));
+    } else if (TEST(PERSCACHE_MODULE_MD5_AT_LOAD,
+                    DYNAMO_OPTION(persist_gen_validation))) {
+        /* case 9735: calculate the module md5 at load time so we have a consistent
+         * point at which to compare it when loading in a persisted cache file.
+         * If we inject at different points we may see different views of
+         * post-loader vs pre-loader module changes but we'll live with that.
+         * Should have consistent injection points in steady state usage.
+         * FIXME case 10601: for 4.4 we'll want to not record the at-mmap md5, but
+         * rather the 1st-execution-time post-rebase md5.
+         */
+        app_pc modbase = get_module_base(info->base_pc);
+        size_t modsize;
+        os_get_module_info_lock();
+        if (os_get_module_info(modbase, NULL, NULL, &modsize,
+                               NULL, NULL, NULL)) {
+            os_get_module_info_unlock();
+            persist_calculate_module_digest(&info->module_md5, modbase, modsize,
+                                            info->base_pc, info->end_pc,
+                                            DYNAMO_OPTION(persist_gen_validation));
+            DOLOG(1, LOG_CACHE, {
+                print_module_digest(GLOBAL, &info->module_md5, "md5 at load time: ");
+            });
+        } else
+            os_get_module_info_unlock();
+    }
+    /* the rest is initialized lazily in coarse_unit_init() */
+    RSTATS_ADD_PEAK(num_coarse_units, 1);
+    return info;
+}
+
+void
+coarse_unit_free(dcontext_t *dcontext, coarse_info_t *info)
+{
+    ASSERT(info != NULL);
+    /* Elements should have been freed in coarse_unit_reset_free() */
+    ASSERT(info->htable == NULL);
+    ASSERT(info->th_htable == NULL);
+    ASSERT(info->pclookup_htable == NULL);
+    ASSERT(info->cache == NULL);
+    ASSERT(info->incoming == NULL);
+    ASSERT(info->stubs == NULL);
+    ASSERT(info->cache_start_pc == NULL);
+    ASSERT(info->stubs_start_pc == NULL);
+    DODEBUG({
+        if (info->module != NULL)
+            dr_strfree(info->module HEAPACCT(ACCT_VMAREAS));
+    });
+    DELETE_LOCK(info->lock);
+    DELETE_LOCK(info->incoming_lock);
+    HEAP_TYPE_FREE(GLOBAL_DCONTEXT, info, coarse_info_t,
+                   ACCT_VMAREAS, PROTECTED);
+    RSTATS_DEC(num_coarse_units);
+}
+
+void
+coarse_unit_init(coarse_info_t *info, void *cache)
+{
+    ASSERT(info != NULL);
+    ASSERT(cache != NULL);
+    ASSERT_OWN_MUTEX(true, &info->lock);
+    fragment_coarse_htable_create(info, 0, 0);
+    coarse_stubs_create(info, NULL, 0);
+    /* cache is passed in since it can't be created while holding info->lock */
+    info->cache = cache;
+}
+
+/* If caller holds change_linking_lock and info->lock, have_locks should be true.
+ * If !need_info_lock, info must be a thread-local, unlinked, private pointer!
+ */
+static void
+coarse_unit_reset_free_internal(dcontext_t *dcontext, coarse_info_t *info,
+                                bool have_locks, bool unlink, bool abdicate_primary,
+                                bool need_info_lock)
+{
+    DEBUG_DECLARE(bool ok;)
+    ASSERT(info != NULL);
+    LOG(THREAD, LOG_CACHE, 2, "coarse_unit_reset_free %s\n", info->module);
+    if (!have_locks) {
+        /* Though only called during all-threads-synch, we still grab our lock here */
+        /* higher rank than info, needed for unlink */
+        if (unlink)
+            acquire_recursive_lock(&change_linking_lock);
+        if (need_info_lock)
+            mutex_lock(&info->lock);
+    }
+    ASSERT(!unlink || self_owns_recursive_lock(&change_linking_lock));
+    ASSERT_OWN_MUTEX(need_info_lock, &info->lock);
+    ASSERT(need_info_lock || !unlink); /* else will get deadlock */
+    /* case 11064: avoid rank order */
+    DODEBUG({
+        if (!need_info_lock)
+            info->is_local = true;
+    });
+    if (unlink)
+        coarse_unit_unlink(dcontext, info);
+    fragment_coarse_htable_free(info);
+    coarse_stubs_delete(info);
+    fcache_coarse_cache_delete(dcontext, info);
+    if (info->in_use && abdicate_primary)
+        coarse_unit_unmark_primary(info);
+    if (info->frozen) {
+        ASSERT(info->mmap_size > 0);
+        if (info->persisted) {
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+            if (info->in_use) {
+                rct_module_table_persisted_invalidate(dcontext, info->base_pc);
+            }
+#endif
+            /* We use GLOBAL_DCONTEXT always for these in case in use */
+#ifdef RCT_IND_BRANCH
+            if (info->rct_table != NULL)
+                rct_table_free(GLOBAL_DCONTEXT, info->rct_table, false/*data mmapped*/);
+#endif
+#ifdef RETURN_AFTER_CALL
+            if (info->rac_table != NULL)
+                rct_table_free(GLOBAL_DCONTEXT, info->rac_table, false/*data mmapped*/);
+#endif
+            ASSERT(info->mmap_pc != NULL);
+            if (info->mmap_ro_size > 0) {
+                /* two views */
+                DEBUG_DECLARE(ok =) unmap_file(info->mmap_pc, info->mmap_ro_size);
+                ASSERT(ok);
+                DEBUG_DECLARE(ok =) unmap_file(info->mmap_pc+info->mmap_ro_size,
+                                               info->mmap_size-info->mmap_ro_size);
+                ASSERT(ok);
+                info->mmap_ro_size = 0;
+            } else {
+                DEBUG_DECLARE(ok =) unmap_file(info->mmap_pc, info->mmap_size);
+                ASSERT(ok);
+            }
+            if (DYNAMO_OPTION(persist_lock_file)) {
+                ASSERT(info->fd != INVALID_FILE);
+                os_close(info->fd);
+                info->fd = INVALID_FILE;
+            }
+        } else {
+            ASSERT(info->cache_start_pc != NULL);
+            ASSERT(info->stubs_start_pc != NULL);
+            ASSERT(info->mmap_ro_size == 0);
+            heap_munmap(info->cache_start_pc, info->mmap_size);
+            if (info->has_persist_info) {
+                /* Persisted units point at their mmaps for these structures;
+                 * non-persisted dynamically allocate them from DR heap.
+                 */
+#ifdef RCT_IND_BRANCH
+                if (info->rct_table != NULL)
+                    rct_table_free(dcontext, info->rct_table, true);
+#endif
+#ifdef RETURN_AFTER_CALL
+                if (info->rac_table != NULL)
+                    rct_table_free(dcontext, info->rac_table, true);
+#endif
+#ifdef HOT_PATCHING_INTERFACE
+                if (info->hotp_ppoint_vec != NULL) {
+                    HEAP_ARRAY_FREE(dcontext, info->hotp_ppoint_vec, app_rva_t,
+                                    info->hotp_ppoint_vec_num, ACCT_HOT_PATCHING, PROTECTED);
+                }
+#endif
+            }
+        }
+    } else {
+        ASSERT(info->mmap_size == 0);
+        ASSERT(info->cache_start_pc == NULL);
+        ASSERT(info->stubs_start_pc == NULL);
+        ASSERT(!info->has_persist_info);
+    }
+    /* This struct may be re-used for a non-frozen/persisted unit if it was reset due
+     * to a non-cache-consistency reason.  Thus we want to preserve the locks, vm
+     * region, and md5, but clear everything else (case 10119).
+     */
+    memset(info, 0, offsetof(coarse_info_t, lock));
+    if (!have_locks) {
+        if (need_info_lock)
+            mutex_unlock(&info->lock);
+        if (unlink)
+            release_recursive_lock(&change_linking_lock);
+    }
+}
+
+/* If caller holds change_linking_lock and info->lock, have_locks should be true */
+void
+coarse_unit_reset_free(dcontext_t *dcontext, coarse_info_t *info,
+                       bool have_locks, bool unlink, bool abdicate_primary)
+{
+    coarse_unit_reset_free_internal(dcontext, info,
+                                    have_locks, unlink, abdicate_primary,
+                                    true/*need_info_lock*/);
+}
+
+/* currently only one such directory expected matching
+ * primary user token, see case 8812
+ */
+static file_t perscache_user_directory = INVALID_FILE;
+
+void
+perscache_init(void)
+{
+    if (DYNAMO_OPTION(use_persisted) && 
+        DYNAMO_OPTION(persist_per_user) &&
+        DYNAMO_OPTION(validate_owner_dir)) {
+        char dir[MAXIMUM_PATH];
+
+        /* case 8812 we need to hold a handle to the user directory
+         * from startup (we could delay until we open our first pcache file)
+         */
+        if (get_persist_dir(dir, BUFFER_SIZE_ELEMENTS(dir), 
+                            true /* note we MUST always create directory
+                                  * even if never persisting */
+                             )) {
+            /* we just need READ_CONTROL (on Windows) to check
+             * ownership, and we are NOT OK with the directory being
+             * renamed (or deleted and recreated by a malactor) while
+             * we still have a handle to it.
+             */
+            perscache_user_directory = os_open_directory(dir, 0);
+            ASSERT(perscache_user_directory != INVALID_FILE);
+
+            /* note that now that we have the actual handle open, we can validate */
+            /* see os_current_user_directory() for details */
+            if (perscache_user_directory != INVALID_FILE && 
+                !os_validate_user_owned(perscache_user_directory)) {
+                SYSLOG_INTERNAL_ERROR("%s is OWNED by an impostor!"
+                                      " Persistent cache use is disabled.",
+                                      dir);
+                os_close(perscache_user_directory);
+                perscache_user_directory = INVALID_FILE;
+                /* we could also turn off use_persisted */
+            } else {
+                /* either FAT32 or we are the proper owner */
+            
+                /* FIXME: we have to verify that the final permissions and
+                 * sharing attributes for cache/ and for the current
+                 * directory, do NOT allow anyone to rename our directory
+                 * while in use, and replace it.  Otherwise we'd still
+                 * have to verify owner for each file as well with
+                 * -validate_owner_file.  See duplicate comment in
+                 * open_relocated_dlls_filecache_directory()
+                 */
+            }
+        }
+    }
+}
+
+void
+perscache_fast_exit(void)
+{
+    if (DYNAMO_OPTION(coarse_freeze_at_exit)) {
+        coarse_units_freeze_all(false/*!in place*/);
+    }
+
+    if (perscache_user_directory != INVALID_FILE) {
+        ASSERT_CURIOSITY(DYNAMO_OPTION(validate_owner_dir));
+        os_close(perscache_user_directory);
+        perscache_user_directory = INVALID_FILE;
+    }
+    ASSERT(perscache_user_directory == INVALID_FILE);
+}
+
+void
+perscache_slow_exit(void)
+{
+    DODEBUG(DELETE_LOCK(pcache_dir_check_lock););
+}
+
+/***************************************************************************
+ * FROZEN UNITS
+ */
+
+/* Separated out to keep dr_mcontext_t out of critical stack path */
+static void
+coarse_units_freeze_translate(thread_record_t *tr,
+                              const thread_synch_state_t desired_state)
+{
+    dr_mcontext_t mc;
+    bool res;
+    res = thread_get_mcontext(tr, &mc);
+    ASSERT(res);
+    /* We're freeing coarse fragments so we must translate all
+     * threads who are currently in a coarse unit, or about
+     * to enter one (case 10030).  We don't translate threads
+     * in fine-grained caches as an optimization.
+     * If we did one unit at a time, could compare to just that unit.
+     */
+    if (!res || !in_fcache((cache_pc)mc.pc) ||
+        get_fcache_coarse_info((cache_pc)mc.pc) != NULL) {
+        /* FIXME optimization: pass cxt for translation */
+        translate_from_synchall_to_dispatch(tr, desired_state);
+    } else {
+        LOG(GLOBAL, LOG_FRAGMENT, 2,
+            "\tin fine-grained cache so no translation needed\n");
+    }
+}
+
+/* If !in_place this routine freezes (if not already) and persists.
+ * FIXME case 9975: provide support for freezing in place and
+ * persisting in one call?  Should we support loading in a newly
+ * persisted version to replace the in-memory unit?
+ */
+void
+coarse_units_freeze_all(bool in_place)
+{
+    thread_record_t **threads = NULL;
+    int i, num_threads = 0;
+    bool own_synch;
+    dcontext_t *my_dcontext = get_thread_private_dcontext();
+    const thread_synch_state_t desired_state =
+        THREAD_SYNCH_SUSPENDED_VALID_MCONTEXT_OR_NO_XFER;
+    if (!DYNAMO_OPTION(coarse_units) || !DYNAMO_OPTION(coarse_enable_freeze) ||
+        RUNNING_WITHOUT_CODE_CACHE())
+        return;
+    KSTART(coarse_freeze_all);
+    /* on a detach we don't need to synch or xlate the threads */
+    own_synch = !dynamo_all_threads_synched;
+    ASSERT(own_synch IF_WINDOWS(|| doing_detach));
+    STATS_INC(coarse_freezes);
+    if (own_synch) {
+        /* Called from nudge threads from the code cache, so 
+         * if the calling fragment could be coarse, we have to
+         * terminate this thread.  Case 8711 does not allow calls from
+         * coarse fragments so we're fine for now.
+         */
+        if (!synch_with_all_threads(desired_state, &threads, &num_threads,
+                                    /* FIXME: can we set mcontext->pc to next_tag and
+                                     * use THREAD_SYNCH_VALID_MCONTEXT?  not if nudge
+                                     * comes here */
+                                    THREAD_SYNCH_NO_LOCKS_NO_XFER, /* Case 6821 */
+                                    /* if we fail to suspend a thread (e.g., for
+                                     * privilege reasons) just abort */
+                                    THREAD_SYNCH_SUSPEND_FAILURE_ABORT
+                                    /* if we get in a race with detach, or are having
+                                     * synch issues for whatever reason, bail out sooner
+                                     * rather than later */
+                                    | THREAD_SYNCH_SMALL_LOOP_MAX)) {
+            /* just give up */
+            ASSERT(!OWN_MUTEX(&all_threads_synch_lock) &&
+                   !OWN_MUTEX(&thread_initexit_lock));
+            ASSERT(threads == NULL);
+            ASSERT(!dynamo_all_threads_synched);
+            STATS_INC(coarse_freeze_abort);
+            LOG(GLOBAL, LOG_CACHE, 2, 
+                "coarse_unit_freeze: aborting due to thread synch failure\n");
+            SYSLOG_INTERNAL_WARNING("coarse freeze aborted due to thread synch failure");
+            KSTOP(coarse_freeze_all);
+            return;
+        }
+    }
+    ASSERT(dynamo_all_threads_synched);
+    ASSERT(OWN_MUTEX(&all_threads_synch_lock) && OWN_MUTEX(&thread_initexit_lock));
+    
+    DOSTATS({
+        SYSLOG_INTERNAL_INFO("freezing all coarse units @ "SSZFMT" fragments",
+                             GLOBAL_STAT(num_fragments));
+    });
+
+    /* This routine does the actual freeze and persist calls
+     * FIXME case 9641: should we end the synch after freezing
+     * so other threads can make progress while we persist?
+     */
+    vm_area_coarse_units_freeze(in_place);
+
+    if (in_place && own_synch) {
+        DEBUG_DECLARE(uint removed;)
+        for (i = 0; i < num_threads; i++) {
+            dcontext_t *dcontext = threads[i]->dcontext;
+            if (dcontext != NULL && dcontext != my_dcontext) {
+                /* FIXME: share these checks w/ other synchall-and-abort users
+                 * (reset) and synchall-and-don't-abort (flush).
+                 */
+                /* Should have aborted if we had any synch failures */
+                ASSERT(thread_synch_successful(threads[i]));
+                if (is_thread_currently_native(threads[i])) {
+                    /* Whether in native_exec or we lost control, since we're not
+                     * freeing the interception buffer, no state to worry about.
+                     */
+                    LOG(GLOBAL, LOG_FRAGMENT, 2,
+                        "\tcurrently native so no translation needed\n");
+                } else if (thread_synch_state_no_xfer(dcontext)) {
+                    /* Case 6821: do not translate other synch-all-thread users. */
+                    LOG(GLOBAL, LOG_FRAGMENT, 2,
+                        "\tat THREAD_SYNCH_NO_LOCKS_NO_XFER so no translation needed\n");
+                } else {
+                    /* subroutine to avoid dr_mcontext_t on our stack when we
+                     * freeze + merge&load */
+                    coarse_units_freeze_translate(threads[i], desired_state);
+                }
+                last_exit_deleted(dcontext);
+                if (is_building_trace(dcontext)) {
+                    LOG(THREAD, LOG_FRAGMENT, 2,
+                        "\tsquashing trace of thread %d\n", i);
+                    trace_abort(dcontext);
+                }
+                if (DYNAMO_OPTION(bb_ibl_targets)) {
+                    /* FIXME: we could just remove the coarse ibl entries */
+                    DEBUG_DECLARE(removed =)
+                        fragment_remove_all_ibl_in_region(dcontext,
+                                                          UNIVERSAL_REGION_BASE,
+                                                          UNIVERSAL_REGION_END);
+                    LOG(THREAD, LOG_FRAGMENT, 2, "\tremoved %d ibl entries\n", removed);
+                }
+            }
+        }
+        if (DYNAMO_OPTION(bb_ibl_targets)) {
+            /* FIXME: we could just remove the coarse ibl entries */
+            DEBUG_DECLARE(removed =)
+                fragment_remove_all_ibl_in_region(GLOBAL_DCONTEXT,
+                                                  UNIVERSAL_REGION_BASE,
+                                                  UNIVERSAL_REGION_END);
+            LOG(GLOBAL, LOG_FRAGMENT, 2, "\tremoved %d ibl entries\n", removed);
+        }
+    }
+
+    if (own_synch)
+        end_synch_with_all_threads(threads, num_threads, true/*resume*/);
+    KSTOP(coarse_freeze_all);
+}
+
+/* Removes dst's data and replaces it with src's data.  Frees src.
+ * Assumes that src is thread-local and not reachable by any other thread,
+ * and that dst's lock is held.
+ */
+static void
+coarse_replace_unit(dcontext_t *dcontext, coarse_info_t *dst, coarse_info_t *src)
+{
+    /* Perhaps we should separately allocate the locks to avoid this copying
+     * for preservation?  Or memcpy all but the lock fields?  Or delete and
+     * re-init them?  If we move to a model where the world isn't suspended
+     * we have to ensure no other thread is trying to lock.
+     */
+    coarse_info_t *non_frozen;
+    mutex_t temp_lock, temp_incoming_lock;
+    DEBUG_DECLARE(const char *modname;)
+    ASSERT_OWN_MUTEX(true, &dst->lock);
+    mutex_lock(&dst->incoming_lock);
+    ASSERT(src->incoming == NULL); /* else we leak */
+    src->incoming = dst->incoming;
+    dst->incoming = NULL; /* do not free incoming */
+    mutex_unlock(&dst->incoming_lock);
+    non_frozen = dst->non_frozen;
+    coarse_unit_reset_free(dcontext, dst, true/*have locks*/,
+                           false/*do not unlink*/, false/*keep primary*/);
+    temp_lock = dst->lock;
+    temp_incoming_lock = dst->incoming_lock;
+    DODEBUG({ modname = dst->module; });
+    memcpy(dst, src, sizeof(*dst));
+    dst->lock = temp_lock;
+    dst->incoming_lock = temp_incoming_lock;
+    dst->non_frozen = non_frozen;
+    DODEBUG({ dst->module = modname; });
+    ASSERT(dst->incoming == src->incoming);
+    /* update pointers from src to dst */
+    fcache_coarse_set_info(dcontext, dst);
+    patch_coarse_exit_prefix(dcontext, dst);
+    coarse_stubs_set_info(dst);
+    DODEBUG({
+        /* avoid asserts */
+        src->htable = NULL;
+        src->th_htable = NULL;
+        src->pclookup_htable = NULL;
+        src->cache = NULL;
+        src->incoming = NULL;
+        src->stubs = NULL;
+        src->cache_start_pc = NULL;
+        src->stubs_start_pc = NULL;
+    });
+    coarse_unit_free(dcontext, src);
+}
+
+/* In-place freezing replaces info with a frozen copy.
+ * Otherwise, a new copy is created for persisting, while the original
+ * copy is undisturbed and unfrozen.
+ * Only purpose of !in_place is to write out to disk a
+ * snapshot while letting coarse unit creation continue.  Can
+ * write in_place out to disk as well, so we leave that up to
+ * caller.
+ * Caller must hold change_linking_lock.
+ * If in_place, caller is responsible for flushing the ibl tables (case 11057).
+ */
+coarse_info_t *
+coarse_unit_freeze(dcontext_t *dcontext, coarse_info_t *info, bool in_place)
+{
+    coarse_info_t *frozen;
+    coarse_info_t *res = NULL;
+    size_t frozen_stub_size, frozen_cache_size;
+    uint num_fragments, num_stubs;
+    coarse_freeze_info_t *freeze_info =
+        HEAP_TYPE_ALLOC(dcontext, coarse_freeze_info_t,
+                        ACCT_MEM_MGT/*appropriate?*/, PROTECTED);
+
+    LOG(THREAD, LOG_CACHE, 2, "coarse_unit_freeze %s\n", info->module);
+    STATS_INC(coarse_units_frozen);
+    /* FIXME: Suspend world needed if not in-place?  Even though unit lock
+     * is not held when changing unit links (e.g., unit flush holds only
+     * target unit lock when unlinking incoming), the change_linking_lock
+     * should give us guarantees.
+     * But if we don't suspend-all we'll have other issues:
+     * - fcache_coarse_init_frozen() will need to
+     *   grab the shared cache lock, which is higher rank than coarse unit lock!
+     * - same issue w/ fcache_coarse_cache_delete (via coarse_unit_reset_free)
+     */
+    /* FIXME: support single-unit freeze by having this routine itself
+     * do a synch-all?
+     */
+    ASSERT(dynamo_all_threads_synched);
+
+    ASSERT(info != NULL);
+    ASSERT_OWN_MUTEX(true, &change_linking_lock);
+    mutex_lock(&info->lock);
+    ASSERT(info->cache != NULL); /* don't freeze empty units */
+    ASSERT(!info->frozen); /* don't freeze already frozen units */
+    if (info->cache == NULL || info->frozen)
+        goto coarse_unit_freeze_exit;
+    /* invalid unit shouldn't get this far */
+    ASSERT(!TEST(PERSCACHE_CODE_INVALID, info->flags));
+    if (TEST(PERSCACHE_CODE_INVALID, info->flags)) /* paranoid */
+        goto coarse_unit_freeze_exit;
+
+    memset(freeze_info, 0, sizeof(*freeze_info));
+    freeze_info->src_info = info;
+
+    /* Tasks:
+     * 1) Calculate final size of cache and stub space:
+     *    Walk entrance stubs and count how many are intra-unit links
+     *    that can be changed to direct jmps
+     * 2) Create single contiguous region to hold both cache and stubs,
+     *    rounding up to a page boundary in the middle for +r->+rw
+     * 3) Copy each fragment and stub over
+     *
+     * FIXME case 9428: shrink the cache to take advantage of elided jmps!
+     * Requires a separate pass to touch up jmps to stubs/prefixes, or
+     * re-ordering w/ stubs on top and cache on bottom.  That would also
+     * put a read-only page at the end, so no guard page needed -- unless we
+     * hook our own cache (case 9673) and are worried about brief periods of +w.
+     */
+
+    frozen_stub_size = coarse_frozen_stub_size(dcontext, info, &num_fragments,
+                                               &num_stubs);
+    frozen_cache_size = coarse_frozen_cache_size(dcontext, info);
+    /* we need the stubs to start on a new page since will be +rw vs cache +r */
+    frozen_cache_size = ALIGN_FORWARD(frozen_cache_size, PAGE_SIZE);
+    freeze_info->cache_start_pc =
+        (cache_pc) heap_mmap(frozen_stub_size + frozen_cache_size);
+    /* FIXME: should show full non-frozen size as well */
+    LOG(THREAD, LOG_CACHE, 2,
+        "%d frozen stubs @ "SZFMT" bytes + %d fragments @ "SZFMT" bytes => "PFX"\n",
+        num_stubs, frozen_stub_size, num_fragments,
+        frozen_cache_size, freeze_info->cache_start_pc);
+    STATS_ADD(coarse_fragments_frozen, num_fragments);
+
+    /* We use raw pcs to build up our cache and stubs, and later we impose
+     * our regular data structures on them
+     */
+
+    /* Whether freezing in-place or not, we create a new coarse_info_t.
+     * If in-place we delete the old one afterward.
+     */
+    frozen = coarse_unit_create(info->base_pc, info->end_pc, &info->module_md5,
+                                in_place && info->in_use);
+    freeze_info->dst_info = frozen;
+    frozen->frozen = true;
+    frozen->cache_start_pc = freeze_info->cache_start_pc;
+    frozen->mmap_size = frozen_stub_size + frozen_cache_size;
+    /* Our relative jmps require that we do not exceed 32-bit reachability */
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(frozen->mmap_size)));
+    /* Same bounds, so same persistence privileges */
+    frozen->primary_for_module = info->primary_for_module;
+
+    freeze_info->stubs_start_pc =
+        coarse_stubs_create(frozen, freeze_info->cache_start_pc + frozen_cache_size,
+                            frozen_stub_size);
+    ASSERT(freeze_info->stubs_start_pc != NULL);
+    ASSERT(ALIGNED(freeze_info->stubs_start_pc, coarse_stub_alignment(info)));
+    frozen->stubs_start_pc = freeze_info->stubs_start_pc;
+    ASSERT(frozen->fcache_return_prefix ==
+           freeze_info->cache_start_pc + frozen_cache_size);
+    ASSERT(frozen->trace_head_return_prefix == frozen->fcache_return_prefix +
+           (info->trace_head_return_prefix - info->fcache_return_prefix));
+    ASSERT(frozen->ibl_ret_prefix == frozen->fcache_return_prefix +
+           (info->ibl_ret_prefix - info->fcache_return_prefix));
+    ASSERT(frozen->ibl_call_prefix == frozen->fcache_return_prefix +
+           (info->ibl_call_prefix - info->fcache_return_prefix));
+    ASSERT(frozen->ibl_jmp_prefix == frozen->fcache_return_prefix +
+           (info->ibl_jmp_prefix - info->fcache_return_prefix));
+
+    fragment_coarse_htable_create(frozen, num_fragments, num_stubs);
+
+    fcache_coarse_init_frozen(dcontext, frozen, freeze_info->cache_start_pc,
+                              frozen_cache_size);
+
+    /* assumption: leave inter-unit links intact for in_place, but not (for
+     * persisting) otherwise
+     */
+    freeze_info->unlink = !in_place;
+    
+    freeze_info->cache_cur_pc = freeze_info->cache_start_pc;
+    freeze_info->stubs_cur_pc = freeze_info->stubs_start_pc;
+
+    fragment_coarse_unit_freeze(dcontext, freeze_info);
+    ASSERT(freeze_info->pending == NULL);
+    ASSERT(freeze_info->cache_cur_pc <=
+           freeze_info->cache_start_pc + frozen_cache_size);
+    ASSERT(freeze_info->stubs_cur_pc <=
+           freeze_info->stubs_start_pc + frozen_stub_size);
+    if (frozen->fcache_return_prefix + frozen_stub_size ==
+        freeze_info->stubs_cur_pc)
+        frozen->stubs_end_pc = freeze_info->stubs_cur_pc;
+    else {
+        /* FIXME case 9428: strange history here: I don't see a problem now,
+         * but leaving some release-build code just in case.
+        */
+        ASSERT_NOT_REACHED();
+        coarse_stubs_set_end_pc(frozen, freeze_info->stubs_cur_pc); 
+    }
+    frozen->cache_end_pc = freeze_info->cache_cur_pc;
+
+    LOG(THREAD, LOG_CACHE, 2, "frozen code stats for %s:\n  %6d app code\n",
+        info->module, freeze_info->app_code_size);
+    LOG(THREAD, LOG_CACHE, 2, "  %6d fallthrough\n", freeze_info->added_fallthrough);
+    LOG(THREAD, LOG_CACHE, 2, "  %6d ind br mangle\n", freeze_info->added_indbr_mangle);
+    LOG(THREAD, LOG_CACHE, 2, "  %6d indr br stubs\n", freeze_info->added_indbr_stub);
+    LOG(THREAD, LOG_CACHE, 2, "  %6d jecxz mangle\n", freeze_info->added_jecxz_mangle);
+    LOG(THREAD, LOG_CACHE, 2, " -%6d = 5 x %d elisions\n",
+        freeze_info->num_elisions*5, freeze_info->num_elisions);
+    LOG(THREAD, LOG_CACHE, 2, "ctis: %5d cbr, %5d jmp, %5d call, %5d ind\n",
+        freeze_info->num_cbr, freeze_info->num_jmp, freeze_info->num_call,
+        freeze_info->num_indbr);
+    LOG(THREAD, LOG_CACHE, 2,
+        "frozen final size: stubs "SZFMT" bytes + cache "SZFMT" bytes\n",
+        freeze_info->stubs_cur_pc - freeze_info->stubs_start_pc,
+        freeze_info->cache_cur_pc - freeze_info->cache_start_pc);
+
+    /* FIXME case 9687: mark cache as read-only */
+
+    if (in_place) {
+        coarse_replace_unit(dcontext, info, frozen);
+        frozen = NULL;
+        mark_executable_area_coarse_frozen(info);
+        coarse_unit_shift_links(dcontext, info);
+        res = info;
+    } else {
+        /* we made separate copy that has no outgoing or incoming links */
+        res = frozen;
+    }
+
+ coarse_unit_freeze_exit:
+    HEAP_TYPE_FREE(dcontext, freeze_info, coarse_freeze_info_t,
+                   ACCT_MEM_MGT/*appropriate?*/, PROTECTED);
+
+    mutex_unlock(&info->lock);
+
+    return res;
+}
+
+/* These decode-and-instr-using routines could go in x86/ as they assume that direct
+ * jump operands are 4 bytes and are at the end of the instruction.
+ */
+
+/* Transfers a coarse stub to a new location.
+ * If freeze_info->dst_info is non-NULL,
+ *   shifts any unlinked stubs to point at the prefixes in freeze_info->dst_info.
+ * If freeze_info->unlink is true, 
+ *   points any linked stubs at freeze_info->dst_info->fcache_return_prefix if
+ *   freeze_info->dst_info is non-NULL, else
+ *   freeze_info->src_info->fcache_return_prefix.  If trace_head, points at
+ *   trace_head_return_prefix instead of fcache_return_prefix.
+ * replace_outgoing really only applies if in_place: should we replace
+ *   outgoing link incoming entries (true), or add new ones (false)
+ */
+void
+transfer_coarse_stub(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info,
+                     cache_pc stub, bool trace_head, bool replace_outgoing)
+{
+    cache_pc tgt = entrance_stub_jmp_target(stub);
+    cache_pc pc = freeze_info->stubs_cur_pc; /* target pc */
+    uint sz;
+    bool update_out = false;
+    /* Should not be targeting the cache, else our later shift will be wrong */
+    ASSERT(tgt < freeze_info->src_info->cache_start_pc ||
+           tgt >= freeze_info->src_info->cache_end_pc);
+    if (tgt == freeze_info->src_info->fcache_return_prefix) {
+        ASSERT(!trace_head);
+        if (freeze_info->dst_info != NULL)
+            tgt = freeze_info->dst_info->fcache_return_prefix;
+        LOG(THREAD, LOG_FRAGMENT, 4,
+            "    transfer_coarse_stub "PFX": tgt is fcache_return_prefix\n", stub);
+    } else if (tgt == freeze_info->src_info->trace_head_return_prefix) {
+        ASSERT(trace_head);
+        if (freeze_info->dst_info != NULL)
+            tgt = freeze_info->dst_info->trace_head_return_prefix;
+        LOG(THREAD, LOG_FRAGMENT, 4,
+            "    transfer_coarse_stub "PFX": tgt is trace_head_return_prefix\n", stub);
+    } else if (freeze_info->unlink) {
+        coarse_info_t *info = (freeze_info->dst_info != NULL) ?
+            freeze_info->dst_info : freeze_info->src_info;
+        if (trace_head) {
+            tgt = info->trace_head_return_prefix;
+            LOG(THREAD, LOG_FRAGMENT, 4,
+                "    transfer_coarse_stub "PFX": unlinking as trace head\n", stub);
+        } else {
+            tgt = info->fcache_return_prefix;
+            LOG(THREAD, LOG_FRAGMENT, 4,
+                "    transfer_coarse_stub "PFX": unlinking as non-trace head\n", stub);
+        }
+    } else
+        update_out = true;
+    sz = exit_stub_size(dcontext, tgt, FRAG_COARSE_GRAIN)
+        - (JMP_LONG_LENGTH - 1/*get opcode*/);
+    memcpy(pc, stub, sz);
+    pc += sz;
+    ASSERT(pc == entrance_stub_jmp(freeze_info->stubs_cur_pc) + 1/*skip opcode*/);
+    ASSERT(*(pc-1) == JMP_OPCODE);
+    /* if tgt unchanged we still need to re-relativize it */
+    ASSERT(dynamo_all_threads_synched); /* thus NOT_HOT_PATCHABLE */
+    pc = insert_relative_target(pc, tgt, NOT_HOT_PATCHABLE);
+    if (update_out) {
+        coarse_update_outgoing(dcontext, stub, freeze_info->stubs_cur_pc,
+                               freeze_info->src_info, replace_outgoing);
+    }
+    pc = (cache_pc) ALIGN_FORWARD(pc, coarse_stub_alignment(freeze_info->src_info));
+    freeze_info->stubs_cur_pc = pc;
+}
+
+void
+transfer_coarse_stub_fix_trace_head(dcontext_t *dcontext,
+                                    coarse_freeze_info_t *freeze_info,
+                                    cache_pc stub)
+{
+    /* We don't know body pc at fragment exit processing time and so can
+     * add a stub and unlink it as a non-trace head if it was linked to
+     * a trace, so we fix it up later
+     */
+    coarse_info_t *info = (freeze_info->dst_info != NULL) ?
+        freeze_info->dst_info : freeze_info->src_info;
+    ASSERT(freeze_info->unlink);
+    if (entrance_stub_jmp_target(stub) == info->fcache_return_prefix) {
+        cache_pc tgt = info->trace_head_return_prefix;
+        ASSERT(dynamo_all_threads_synched); /* thus NOT_HOT_PATCHABLE */
+        insert_relative_target(entrance_stub_jmp(stub) + 1/*skip opcode*/,
+                               tgt, NOT_HOT_PATCHABLE);
+        LOG(THREAD, LOG_FRAGMENT, 4,
+            "    fixing up stub "PFX" to be unlinked as a trace head\n", stub);
+    } else
+        ASSERT(entrance_stub_jmp_target(stub) == info->trace_head_return_prefix);
+}
+
+static void
+push_pending_freeze(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info,
+                    cache_pc exit_tgt, uint cti_len, cache_pc cti_pc,
+                    cache_pc body_start_pc)
+{
+    pending_freeze_t *pending;
+    cache_pc stub_target;
+    uint sz;
+
+    pending = HEAP_TYPE_ALLOC(dcontext, pending_freeze_t,
+                              ACCT_MEM_MGT/*appropriate?*/, UNPROTECTED);
+    ASSERT(coarse_is_entrance_stub(exit_tgt));
+    pending->tag = entrance_stub_target_tag(exit_tgt);
+    stub_target = entrance_stub_jmp_target(exit_tgt);
+    if (entrance_stub_linked(exit_tgt, freeze_info->src_info) &&
+        get_fcache_coarse_info(stub_target) == freeze_info->src_info) {
+        /* Intra-unit non-trace-head target: eliminate stub */
+        pending->entrance_stub = false;
+        pending->cur_pc = stub_target;
+        /* trace heads are discovered in the htable walk, never pushed here */
+        pending->trace_head = false;
+    } else {
+        /* Leave stub */
+        pending->entrance_stub = true;
+        pending->cur_pc = exit_tgt;
+        DODEBUG({
+            cache_pc body;
+            /* A rank order violation (grabbing src htable read lock while
+             * holding frozen htable write lock) prevents us from calling the
+             * exported interface fragment_coarse_lookup_in_unit, so instead we
+             * have a hack where we call the lower-level routine that is
+             * exported only for us.
+             */
+            coarse_body_from_htable_entry(dcontext, freeze_info->src_info,
+                                          pending->tag, exit_tgt, NULL, &body);
+            ASSERT(body == NULL ||
+                   coarse_is_trace_head(exit_tgt) ||
+                   fragment_lookup_trace(dcontext, pending->tag) != NULL);
+        });
+        /* We do not look up body pc to see if a trace head stub linked to
+         * a trace; instead we fix up the unlinked target (for freeze_info->unlink)
+         * when we proactively add the stub when processing the head body
+         */
+        pending->trace_head = coarse_is_trace_head(exit_tgt);
+        /* If target is trace head in same unit, we could add to pending,
+         * but don't have body pc handy, so we let htable walk find it
+         */
+    }
+    ASSERT(cti_len > 4);
+    sz = cti_len - 4;
+    pending->link_cti_opnd = freeze_info->cache_cur_pc + sz;
+    memcpy(freeze_info->cache_cur_pc, cti_pc, sz);
+    if (body_start_pc == cti_pc && !DYNAMO_OPTION(unsafe_freeze_elide_sole_ubr)) {
+        /* case 9677: unsafe to elide entire-bb-ubr */
+        pending->elide_ubr = false;
+    } else /* elide if possible */
+        pending->elide_ubr = true;
+    freeze_info->cache_cur_pc += cti_len;
+    pending->next = freeze_info->pending;
+    freeze_info->pending = pending;
+}
+
+static cache_pc
+redirect_to_tgt_ibl_prefix(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info,
+                           cache_pc tgt)
+{
+    ASSERT(freeze_info != NULL && freeze_info->src_info != NULL &&
+           freeze_info->dst_info != NULL);
+    if (tgt == freeze_info->src_info->ibl_ret_prefix)
+        return freeze_info->dst_info->ibl_ret_prefix;
+    else if (tgt == freeze_info->src_info->ibl_call_prefix)
+        return freeze_info->dst_info->ibl_call_prefix;
+    else if (tgt == freeze_info->src_info->ibl_jmp_prefix)
+        return freeze_info->dst_info->ibl_jmp_prefix;
+    else
+        ASSERT_NOT_REACHED();
+    return tgt; /* best chance of continuing on */
+}
+
+/* Transfers a coarse fragment to a new location.  Queues up all of its
+ * exit targets for transfer as well, scheduling ubr last to enable ubr elision.
+ */
+void
+transfer_coarse_fragment(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info,
+                         cache_pc body)
+{
+    /* FIXME: for maximum code re-use, use decode_fragment instead of trying
+     * to be efficient?
+     * FIXME case 9428: 8-bit conversion
+     */
+    cache_pc pc = body, next_pc = pc; /* source pcs */
+    app_pc tgt;
+    size_t sz;
+    instr_t *instr;
+    instr = instr_create(dcontext);
+    do {
+        instr_reset(dcontext, instr);
+        pc = next_pc;
+        ASSERT(pc - body <= MAX_FRAGMENT_SIZE);
+        next_pc = decode_cti(dcontext, pc, instr);
+        /* Case 8711: we can't distinguish exit ctis from others,
+         * so we must assume that any cti is an exit cti.
+         * Assumption: coarse-grain bbs have 1 ind exit or 2 direct,
+         * and no code beyond the last exit!
+         */
+    } while (!instr_opcode_valid(instr) || !instr_is_cti(instr));
+
+    /* copy body of fragment, up to start of cti */
+    sz = pc - body;
+    memcpy(freeze_info->cache_cur_pc, body, sz);
+    freeze_info->cache_cur_pc += sz;
+    DODEBUG({ freeze_info->app_code_size += sz; });
+
+    /* Ensure we get proper target for short cti sequence */
+    if (instr_is_cti_short_rewrite(instr, pc)) {
+        next_pc = remangle_short_rewrite(dcontext, instr, pc, 0/*same target*/);
+        DODEBUG({
+            /* We mangled 2-byte jecxz/loop* into 9-byte sequence */
+            freeze_info->app_code_size -= 7; 
+            freeze_info->added_jecxz_mangle += 7; 
+        });
+    }
+    tgt = opnd_get_pc(instr_get_target(instr));
+    if (tgt == next_pc) {
+        ASSERT(instr_is_ubr(instr));
+        /* indirect exit stub */
+        ASSERT(coarse_is_indirect_stub(tgt));
+        /* elide the jmp to the stub */
+        pc += JMP_LONG_LENGTH /*ubr to stub*/;
+        sz = coarse_indirect_stub_size(freeze_info->src_info) - 4;
+        memcpy(freeze_info->cache_cur_pc, pc, sz);
+        freeze_info->cache_cur_pc += sz;
+        pc += sz;
+        tgt = PC_RELATIVE_TARGET(pc);
+        DODEBUG({
+            freeze_info->num_indbr++;
+            freeze_info->app_code_size -= 6; /* save ecx */
+            freeze_info->added_indbr_mangle += 6 /* save ecx */;
+            if (tgt == freeze_info->src_info->ibl_ret_prefix) {
+                /* ret imm goes from 3 bytes to 1+4=5 bytes
+                   L3  c2 18 00             ret    $0x0018 %esp (%esp) -> %esp 
+                   =>
+                   L4  67 64 89 0e e8 0e    addr16 mov    %ecx -> %fs:0xee8 
+                   L4  59                   pop    %esp (%esp) -> %ecx %esp 
+                   L4  8d 64 24 18          lea    0x18(%esp) -> %esp
+                */
+                /* guaranteed to be able to read 5 bytes back */
+                if (*(pc - 4) == 0x8d && *(pc - 3) == 0x6d && *(pc - 2) == 0x24) {
+                    freeze_info->app_code_size -= 2;
+                    freeze_info->added_indbr_mangle += 2;
+                }
+            } else if (tgt == freeze_info->src_info->ibl_call_prefix) {
+                /* change from call* to mov is no size diff */
+                freeze_info->added_indbr_mangle += 5 /* push immed */;
+            } else {
+                /* jmp*: change to mov is no size difference */
+            }
+            freeze_info->added_indbr_stub +=
+                coarse_indirect_stub_size(freeze_info->src_info) ; 
+        });
+        tgt = redirect_to_tgt_ibl_prefix(dcontext, freeze_info, tgt);
+        ASSERT(dynamo_all_threads_synched); /* thus NOT_HOT_PATCHABLE */
+        freeze_info->cache_cur_pc =
+            insert_relative_target(freeze_info->cache_cur_pc, tgt, NOT_HOT_PATCHABLE);
+    } else {
+        /* FIXME: if we had profile info we could reverse the branch and
+         * make our cache trace-like
+         */
+        DEBUG_DECLARE(bool is_cbr = false;)
+        if (instr_is_cbr(instr)) {
+            uint cbr_len;
+            /* push cbr target on todo-stack */
+            if (instr_is_cti_short_rewrite(instr, pc))
+                cbr_len = CBR_SHORT_REWRITE_LENGTH;
+            else
+                cbr_len = CBR_LONG_LENGTH;
+            push_pending_freeze(dcontext, freeze_info, tgt, cbr_len, pc, body);
+            ASSERT(pc + cbr_len == next_pc);
+
+            /* process ubr next */
+            instr_reset(dcontext, instr);
+            pc = next_pc;
+            next_pc = decode_cti(dcontext, pc, instr);
+            ASSERT(instr_opcode_valid(instr) && instr_is_ubr(instr));
+            tgt = opnd_get_pc(instr_get_target(instr));
+            DODEBUG({
+                freeze_info->num_cbr++;
+                /* FIXME: assumes 32-bit cbr! */
+                freeze_info->app_code_size += cbr_len;
+                freeze_info->added_fallthrough += 5;
+                is_cbr = true;
+            });
+        }
+
+        ASSERT(instr_is_ubr(instr));
+        /* push ubr last, so we can elide the jmp if we process it next */
+        push_pending_freeze(dcontext, freeze_info, tgt, JMP_LONG_LENGTH, pc, body);
+        ASSERT(pc + JMP_LONG_LENGTH == next_pc);
+        DODEBUG({
+            if (!is_cbr) {
+                if (pc >= body + 5 && *(pc - 5) == 0x68) {
+                    /* FIXME: could be an app push immed followed by app jmp */
+                    /* call => push immed: same size, but adding jmp */
+                    freeze_info->num_call++;
+                    freeze_info->added_fallthrough += 5; /* jmp */
+                } else {
+                    /* FIXME: assumes 32-bit jmp! */
+                    freeze_info->app_code_size += 5;
+                    freeze_info->num_jmp++;
+                }
+            }
+        });
+    }
+    instr_destroy(dcontext, instr);
+}
+
+/* This routine walks info's cache and updates extra-cache jmp targets by cache_shift
+ * and jmps to stubs by stubs_shift.
+ * If !is_cache, assumes these are stubs and decodes and acts appropriately.
+ */
+static void
+coarse_unit_shift_jmps_internal(dcontext_t *dcontext, coarse_info_t *info,
+                                ssize_t cache_shift, ssize_t stubs_shift,
+                                size_t old_mapsz,
+                                cache_pc start, cache_pc end,
+                                cache_pc bounds_start, cache_pc bounds_end,
+                                bool is_cache)
+{
+    /* We must patch up indirect and direct stub jmps to prefixes */
+    cache_pc pc = start;
+    cache_pc next_pc = pc;
+    app_pc tgt;
+    instr_t *instr;
+    ASSERT(dynamo_all_threads_synched); /* thus NOT_HOT_PATCHABLE */
+    ASSERT(info->frozen);
+    instr = instr_create(dcontext);
+    while (next_pc < end) {
+        instr_reset(dcontext, instr);
+        pc = next_pc;
+        next_pc = decode_cti(dcontext, pc, instr);
+        /* Case 8711: we can't distinguish exit ctis from others */
+        if (instr_opcode_valid(instr) && instr_is_cti(instr)) {
+            if (instr_is_cti_short_rewrite(instr, pc))
+                next_pc = remangle_short_rewrite(dcontext, instr, pc, 0/*same target*/);
+            tgt = opnd_get_pc(instr_get_target(instr));
+            if (tgt < bounds_start || tgt >= bounds_end) {
+                ssize_t shift;
+                if (is_cache) {
+                    /* break down into whether targeting stubs or not 
+                     * ok to use new prefix start, which is where old padding was
+                     */
+                    if (tgt >= info->fcache_return_prefix &&
+                        tgt < info->cache_start_pc + old_mapsz)
+                        shift = stubs_shift;
+                    else
+                        shift = cache_shift;
+                } else {
+                    /* Shifting jmps from stubs
+                     * We started with [cache | padding | stubs | padding]
+                     * We then allocate new memory and copy there [cache | stubs]
+                     * Thus, the stubs have a double shift: once for padding bet 
+                     * cache and stubs, and once for shift of whole alloc.
+                     * This doesn't work if stubs target cache, but we assert on
+                     * that in transfer_coarse_stub().
+                     */
+                     shift = cache_shift - stubs_shift;
+                }
+                LOG(THREAD, LOG_FRAGMENT, 4,
+                    "\tshifting jmp @"PFX" "PFX" from "PFX" to "PFX"\n",
+                    pc, next_pc - 4, tgt, tgt + shift);
+                insert_relative_target(next_pc - 4, tgt + shift, NOT_HOT_PATCHABLE);
+                if (!is_cache) {
+                    /* we must update incoming after fixing target, since old_stub is
+                     * inconsistent and we need a complete stub to dereference
+                     */
+                    cache_pc old_stub, new_stub;
+                    /* double-check post-shift: no prefix targets */
+                    ASSERT(tgt + shift < bounds_start || tgt + shift >= bounds_end);
+                    new_stub = (cache_pc) ALIGN_BACKWARD(pc, coarse_stub_alignment(info));
+                    old_stub = new_stub + shift;
+                    /* we can't assert that old_stub or new_stub are entrance_stubs
+                     * since targets are currently inconsistent wrt info
+                     */
+                    /* must update incoming stub for target */
+                    coarse_update_outgoing(dcontext, old_stub, new_stub,
+                                           info, true/*replace*/);
+                }
+            }
+            if (!is_cache) {
+                /* for stubs, skip the padding (which we'll decode as garbage */
+                ASSERT(next_pc + 1 ==
+                       (cache_pc) ALIGN_FORWARD(next_pc, coarse_stub_alignment(info)));
+                next_pc = (cache_pc) ALIGN_FORWARD(next_pc, coarse_stub_alignment(info));
+            }
+        }
+    }
+    instr_destroy(dcontext, instr);
+}
+
+/* This routine walks info's cache and updates extra-cache jmp targets by cache_shift
+ * and jmps to stubs by stubs_shift.
+ * It also walks info's stubs and updates targets that are not prefixes:
+ * in other coarse units or in fine-grained fragment caches.
+ */
+static void
+coarse_unit_shift_jmps(dcontext_t *dcontext, coarse_info_t *info,
+                       ssize_t cache_shift, ssize_t stubs_shift, size_t old_mapsz)
+{
+    LOG(THREAD, LOG_FRAGMENT, 4,
+        "shifting jmps for cache "PFX"-"PFX"\n",
+        info->cache_start_pc, info->cache_end_pc);
+    coarse_unit_shift_jmps_internal(dcontext, info,
+                                    cache_shift, stubs_shift, old_mapsz,
+                                    info->cache_start_pc, info->cache_end_pc,
+                                    info->cache_start_pc, info->cache_end_pc,
+                                    true/*cache*/);
+    LOG(THREAD, LOG_FRAGMENT, 4,
+        "shifting jmps for stubs "PFX"-"PFX"\n",
+        info->stubs_start_pc, info->stubs_end_pc);
+    coarse_unit_shift_jmps_internal(dcontext, info,
+                                    cache_shift, stubs_shift, old_mapsz,
+                                    info->stubs_start_pc, info->stubs_end_pc,
+                                    /* do not re-relativize prefix targets */
+                                    info->fcache_return_prefix, info->stubs_end_pc,
+                                    false/*stubs*/);
+}
+
+/***************************************************************************
+ * MERGING FROZEN UNITS
+ */
+
+/* Processes a stub in the original old source unit at stub whose
+ * targeting cti has been copied into the new being-built merged unit
+ * at dst_cache_pc and has length cti_len.  Passing dst_cache_pc==NULL
+ * causes no cti patching to occur.
+ */
+static void
+coarse_merge_process_stub(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info,
+                          cache_pc old_stub, uint cti_len, cache_pc dst_cache_pc,
+                          bool replace_outgoing)
+{
+    app_pc old_stub_tgt;
+    cache_pc dst_body, dst_stub, patch_pc, src_body;
+    bool trace_head;
+    ASSERT(coarse_is_entrance_stub(old_stub));
+    ASSERT(dynamo_all_threads_synched); /* thus NOT_HOT_PATCHABLE */
+    ASSERT((dst_cache_pc == NULL && cti_len == 0) || cti_len > 4);
+    patch_pc = dst_cache_pc + cti_len - 4;
+    old_stub_tgt = entrance_stub_target_tag(old_stub);
+    fragment_coarse_lookup_in_unit(dcontext, freeze_info->dst_info,
+                                   old_stub_tgt, &dst_stub, &dst_body);
+    /* We need to know for sure whether a trace head as we're not doing
+     * a pass through the htable like we do for regular freezing
+     */
+    fragment_coarse_lookup_in_unit(dcontext, freeze_info->src_info, old_stub_tgt,
+                                   NULL, &src_body);
+    /* Consider both sources for headness */
+    trace_head =
+        coarse_is_trace_head_in_own_unit(dcontext, old_stub_tgt, old_stub,
+                                         src_body, true, freeze_info->src_info) ||
+        (dst_stub != NULL && 
+         coarse_is_trace_head_in_own_unit(dcontext, old_stub_tgt, dst_stub,
+                                          dst_body, true, freeze_info->dst_info));
+    /* Should only be adding w/ no source cti if a trace head or for the stub
+     * walk for the larger unit where we have a dup stub and aren't replacing */
+    ASSERT(dst_cache_pc != NULL || trace_head ||
+           (dst_body == NULL && dst_stub != NULL && !replace_outgoing));
+    if (dst_body != NULL && !trace_head) {
+        /* Directly link and do not copy the stub */
+        LOG(THREAD, LOG_FRAGMENT, 4,
+            "\ttarget "PFX" is in other cache @"PFX": directly linking\n",
+            old_stub_tgt, dst_body);
+        ASSERT(dst_stub == NULL);
+        ASSERT(dst_body >= freeze_info->dst_info->cache_start_pc &&
+               dst_body < freeze_info->dst_info->cache_end_pc);
+        if (dst_cache_pc != NULL)
+            insert_relative_target(patch_pc, dst_body, NOT_HOT_PATCHABLE);
+        if (!freeze_info->unlink && entrance_stub_linked(old_stub,
+                                                         freeze_info->src_info)) {
+            /* ASSUMPTION: unlink == !in_place
+             * If in-place, we must update target incoming info, whether source is
+             * primary (being replaced) or secondary (probably being deleted since
+             * now in merge result, but we don't want to crash while unlinking it
+             * (case 10382)) source.
+             */
+            coarse_remove_outgoing(dcontext, old_stub, freeze_info->src_info);
+        }
+    } else if (dst_stub != NULL) {
+        LOG(THREAD, LOG_FRAGMENT, 4,
+            "\ttarget "PFX" is already in stubs @"PFX"\n", old_stub_tgt, dst_stub);
+        ASSERT(dst_body == NULL || trace_head);
+        /* Stub already exists: point to it */
+        if (dst_cache_pc != NULL)
+            insert_relative_target(patch_pc, dst_stub, NOT_HOT_PATCHABLE);
+        /* Must remove incoming if one mergee had a cross link to the other */
+        if ((dst_body != NULL ||
+            /* If secondary merger was smaller and had a stub for the same target,
+             * we need to remove our outgoing since secondary added a new one.
+             * We want to do this only the 1st time we get here, and not if the
+             * primary merger added the stub, so we have the primary unlink
+             * the old stub (in else code below).
+             */
+             replace_outgoing) &&
+            entrance_stub_linked(old_stub, freeze_info->src_info)) {
+            coarse_remove_outgoing(dcontext, old_stub, freeze_info->src_info);
+        }
+    } else {
+        /* Copy stub */
+        cache_pc stub_pc = freeze_info->stubs_cur_pc;
+        ASSERT(dst_body == NULL || trace_head);
+        LOG(THREAD, LOG_FRAGMENT, 4,
+            "\ttarget "PFX" is %s, adding stub @"PFX"\n",
+            old_stub_tgt, trace_head ? "trace head" : "not present", stub_pc);
+        transfer_coarse_stub(dcontext, freeze_info, old_stub, trace_head,
+                             replace_outgoing);
+        if (replace_outgoing) {
+            /* Signal to later stubs that they don't need to remove the outgoing
+             * entry (as opposed to new stubs added by the secondary merger,
+             * for which we do need to remove).
+             * Assumption: if replace_outgoing then it's ok to unlink the old stub
+             * since it's going away anyway.
+             */
+            unlink_entrance_stub(dcontext, old_stub,
+                                 trace_head ? FRAG_IS_TRACE_HEAD : 0,
+                                 freeze_info->src_info);
+        }
+        ASSERT(freeze_info->stubs_cur_pc == stub_pc +
+               coarse_stub_alignment(freeze_info->src_info));
+        fragment_coarse_th_add(dcontext, freeze_info->dst_info, old_stub_tgt,
+                               stub_pc - (ptr_uint_t)
+                               freeze_info->dst_info->stubs_start_pc);
+        if (dst_cache_pc != NULL)
+            insert_relative_target(patch_pc, stub_pc, NOT_HOT_PATCHABLE);
+    }
+}
+
+/* Assumption: cache has already been copied from src to dst.
+ * This routine walks the copied cache to find inter-unit links; it
+ * directly links them, eliminating their entrance stubs.
+ * replace_outgoing really only applies if in_place: should we replace
+ *   outgoing link incoming entries (true), or add new ones (false)
+ */
+static void
+coarse_merge_update_jmps(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info,
+                         bool replace_outgoing)
+{
+    /* Plan: cache has already been copied from src to dst, but we need to do
+     * inter-unit links.  So we decode from the original cache to find the
+     * target stubs: if a target is present in dst, we do not copy the stub and
+     * we directly link; if not present, we copy the stub and re-relativize the
+     * jmp to the stub.  We must also patch up indirect and direct stub jmps
+     * to prefixes.
+     */
+    cache_pc pc = freeze_info->src_info->cache_start_pc;
+    cache_pc next_pc = pc;
+    cache_pc stop_pc = freeze_info->src_info->cache_end_pc;
+    app_pc tgt;
+    uint sz;
+    /* FIXME: share code w/ decode_fragment() and transfer_coarse_fragment() */
+    instr_t *instr;
+    /* Since mucking with caches, though if thread-private not necessary */
+    ASSERT(dynamo_all_threads_synched);
+    ASSERT(freeze_info->src_info->frozen);
+    LOG(THREAD, LOG_FRAGMENT, 4,
+        "coarse_merge_update_jmps %s "PFX" => "PFX"\n",
+        freeze_info->src_info->module, pc, freeze_info->cache_start_pc);
+    instr = instr_create(dcontext);
+    while (next_pc < stop_pc) {
+        instr_reset(dcontext, instr);
+        pc = next_pc;
+        next_pc = decode_cti(dcontext, pc, instr);
+        /* Case 8711: we can't distinguish exit ctis from others,
+         * so we must assume that any cti is an exit cti.
+         */
+        /* We don't care about fragment boundaries so we can ignore elision.
+         * We only care about jmps to stubs.
+         */
+        if (instr_opcode_valid(instr) && instr_is_cti(instr)) {
+            /* Ensure we get proper target for short cti sequence */
+            if (instr_is_cti_short_rewrite(instr, pc))
+                next_pc = remangle_short_rewrite(dcontext, instr, pc, 0/*same target*/);
+            tgt = opnd_get_pc(instr_get_target(instr));
+            if (in_coarse_stub_prefixes(tgt)) {
+                /* We should not encounter prefix targets other than indirect while
+                 * in the body of the cache (rest are from the stubs) */
+                ASSERT(coarse_is_indirect_stub
+                       (next_pc - coarse_indirect_stub_size(freeze_info->src_info)));
+                /* indirect exit stub: need to update jmp to prefix */
+                ASSERT(instr_is_ubr(instr));
+                sz = JMP_LONG_LENGTH /*ubr to stub*/ - 4;
+                pc += sz;
+                tgt = PC_RELATIVE_TARGET(pc);
+                tgt = redirect_to_tgt_ibl_prefix(dcontext, freeze_info, tgt);
+                ASSERT(dynamo_all_threads_synched); /* thus NOT_HOT_PATCHABLE */
+                insert_relative_target(freeze_info->cache_start_pc +
+                                       (pc - freeze_info->src_info->cache_start_pc),
+                                       tgt, NOT_HOT_PATCHABLE);
+                next_pc = pc + 4;
+            } else if (tgt < freeze_info->src_info->cache_start_pc || tgt >= stop_pc) {
+                /* Must go through a stub */
+                cache_pc dst_cache_pc = freeze_info->cache_start_pc +
+                    (pc - freeze_info->src_info->cache_start_pc);
+                ASSERT(tgt >= freeze_info->src_info->stubs_start_pc &&
+                       tgt < freeze_info->src_info->stubs_end_pc);
+                if (instr_is_cbr(instr)) {
+                    uint cbr_len;
+                    if (instr_is_cti_short_rewrite(instr, pc))
+                        cbr_len = CBR_SHORT_REWRITE_LENGTH;
+                    else
+                        cbr_len = CBR_LONG_LENGTH;
+                    ASSERT(pc + cbr_len == next_pc);
+                    coarse_merge_process_stub(dcontext, freeze_info, tgt, cbr_len,
+                                              dst_cache_pc, replace_outgoing);
+                    /* If there is a ubr next (could be elided) we just hit
+                     * it next time around loop */
+                } else {
+                    ASSERT(instr_is_ubr(instr));
+                    ASSERT(pc + JMP_LONG_LENGTH == next_pc);
+                    coarse_merge_process_stub(dcontext, freeze_info, tgt,
+                                              JMP_LONG_LENGTH, dst_cache_pc,
+                                              replace_outgoing);
+                }
+            } else {
+                /* intra-cache target */
+                /* I would assert that a pclookup finds an entry but that hits
+                 * a recursive lock on non-recursive freeze_info->src_info->lock */
+            }
+        }
+    }
+    instr_destroy(dcontext, instr);
+
+    /* Do the loop even w/o traces in debug for the assert */
+    if (!DYNAMO_OPTION(disable_traces) IF_DEBUG(|| true)) {
+        /* We can have trace heads with no intra-unit targeters (secondary trace
+         * heads!) so we must also walk the stubs.  Rather than require an
+         * iterator or helper routine in fragment or link we directly
+         * walk here. */
+        for (pc = freeze_info->src_info->stubs_start_pc;
+             pc < freeze_info->src_info->stubs_end_pc;
+             pc += coarse_stub_alignment(freeze_info->src_info)) {
+            if (in_coarse_stub_prefixes(pc))
+                continue;
+            ASSERT(coarse_is_entrance_stub(pc));
+            if (entrance_stub_linked(pc, freeze_info->src_info)) {
+                cache_pc src_body;
+                /* for non-in-place merging we don't unlink stubs targeting
+                 * the other mergee, so we must rule that out here.
+                 * the only internally-untargeted stubs we need to add are
+                 * those for our own bodies. */
+                fragment_coarse_lookup_in_unit(dcontext, freeze_info->src_info,
+                                               entrance_stub_target_tag(pc),
+                                               NULL, &src_body);
+                if (src_body != NULL) {
+                    ASSERT(!DYNAMO_OPTION(disable_traces));
+                    coarse_merge_process_stub(dcontext, freeze_info, pc,
+                                              0, NULL, replace_outgoing);
+                }
+            }
+        }
+    }
+}
+
+/* Assumption: cache to be merged with has already been copied to dst.
+ * This routine walks the other src and copies over non-dup fragments,
+ * directly linking inter-unit links along the way.
+ * replace_outgoing really only applies if in_place: should we replace
+ *   outgoing link incoming entries (true), or add new ones (false)
+ */
+static void
+coarse_merge_without_dups(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info,
+                          ssize_t cache_offs, bool replace_outgoing)
+{
+    /* Plan: we need to append the non-dup portions of src to the already-copied
+     * other source, as well as fixing up inter-unit links: if a target is present in
+     * dst, we do not copy the stub and we directly link; if not present, we copy the
+     * stub and re-relativize the jmp to the stub.  We must also patch up indirect
+     * and direct stub jmps to prefixes.
+     * Though we need to know fragment boundaries, note that walking the htable
+     * instead of the cache doesn't buy us much: due to elision we still have to do
+     * pclookups, so we go ahead and walk the cache as it is already laid out.
+     */
+    cache_pc pc = freeze_info->src_info->cache_start_pc;
+    cache_pc src_body, next_pc = pc, fallthrough_body = NULL;
+    cache_pc dst_body = NULL, last_dst_body;
+    cache_pc stop_pc = freeze_info->src_info->cache_end_pc;
+    app_pc tag, fallthrough_tag = NULL, tgt;
+    /* FIXME: share code w/ decode_fragment() and transfer_coarse_fragment() */
+    instr_t *instr;
+    /* Since mucking with caches, though if thread-private not necessary */
+    ASSERT(dynamo_all_threads_synched);
+    ASSERT(freeze_info->src_info->frozen);
+    LOG(THREAD, LOG_FRAGMENT, 4,
+        "coarse_merge_without_dups %s "PFX" => "PFX"\n",
+        freeze_info->src_info->module, pc, freeze_info->cache_cur_pc);
+    instr = instr_create(dcontext);
+    while (next_pc < stop_pc) {
+        last_dst_body = dst_body;
+        if (fallthrough_tag != NULL) {
+            /* still at dup fallthrough pc */
+            ASSERT(fragment_coarse_entry_pclookup(dcontext, freeze_info->src_info,
+                                                  next_pc) == fallthrough_tag);
+            tag = fallthrough_tag;
+            ASSERT(fallthrough_body != NULL);
+            dst_body = fallthrough_body;
+            /* do not go again through the fallthrough code below */
+            instr_reset(dcontext, instr);
+        } else {
+            tag = fragment_coarse_entry_pclookup(dcontext, freeze_info->src_info,
+                                                 next_pc);
+        }
+        /* We come back through the loop for fallthrough jmp of cbr */
+        ASSERT(tag != NULL || (instr_opcode_valid(instr) && instr_is_cbr(instr)));
+        if (tag != NULL && tag != fallthrough_tag) {
+            LOG(THREAD, LOG_FRAGMENT, 4,
+                "\tfragment entry point "PFX" = tag "PFX, next_pc, tag);
+            fragment_coarse_lookup_in_unit(dcontext, freeze_info->dst_info,
+                                           tag, NULL, &dst_body);
+            if (dst_body == NULL) {
+                cache_pc src_stub = NULL;
+                fragment_coarse_add(dcontext, freeze_info->dst_info,
+                                    tag, freeze_info->cache_cur_pc -
+                                    (ptr_uint_t)freeze_info->cache_start_pc +
+                                    cache_offs);
+                LOG(THREAD, LOG_FRAGMENT, 4, " (new => "PFX")\n",
+                    freeze_info->cache_cur_pc);
+                /* this may be a trace head, in which case we need to add its stub
+                 * now in case there are no intra-unit targeters of it (which
+                 * means it is probably a secondary trace head) */
+                fragment_coarse_lookup_in_unit(dcontext, freeze_info->src_info,
+                                               tag, &src_stub, NULL);
+                if (src_stub != NULL) {
+                    ASSERT(!DYNAMO_OPTION(disable_traces));
+                    coarse_merge_process_stub(dcontext, freeze_info, src_stub,
+                                              0, NULL, replace_outgoing);
+                }
+            } else { /* dup */
+                LOG(THREAD, LOG_FRAGMENT, 4, " (duplicate)\n");
+                /* if prev is cbr, this is a fall-through, which is handled below */
+            }
+        } /* else carry through dst_body from last iter */
+        src_body = next_pc;
+        fallthrough_tag = NULL;
+        fallthrough_body = NULL;
+        do {
+            ASSERT(next_pc < stop_pc);
+            if (next_pc >= stop_pc)
+                return; /* paranoid: avoid infinite loop */
+            pc = next_pc;
+            if (next_pc != src_body ||
+                /* fall-through of cbr will be looked up pre-1st iter above */
+                (instr_opcode_valid(instr) && instr_is_cbr(instr))) {
+                /* We assume at least one instr in each fragment, to avoid ambiguity */
+                ASSERT_NOT_IMPLEMENTED(!DYNAMO_OPTION(unsafe_freeze_elide_sole_ubr));
+                if (next_pc == src_body) {
+                    fallthrough_tag = tag;
+                    fallthrough_body = dst_body;
+                } else {
+                    fallthrough_tag =
+                        fragment_coarse_entry_pclookup(dcontext, freeze_info->src_info,
+                                                       next_pc);
+                    if (fallthrough_tag != NULL) {
+                        fragment_coarse_lookup_in_unit(dcontext, freeze_info->dst_info,
+                                                       fallthrough_tag, NULL,
+                                                       &fallthrough_body);
+                    }
+                }
+                if (fallthrough_tag != NULL) {
+                    /* We'd rather keep fall-through elision if we can */
+                    LOG(THREAD, LOG_FRAGMENT, 4,
+                        "\tfall-through tag "PFX" @"PFX, fallthrough_tag, next_pc);
+                    if (fallthrough_body == NULL) {
+                        /* Just keep going and process the fall-through's cti */
+                        LOG(THREAD, LOG_FRAGMENT, 4, " (new => "PFX")\n",
+                            freeze_info->cache_cur_pc + (next_pc - src_body));
+                        if (dst_body != NULL) { /* prev is a dup */
+                            ASSERT_NOT_TESTED();
+                            src_body = next_pc;
+                            tag = fallthrough_tag;
+                        }
+                        if (fallthrough_tag != tag) {
+                            fragment_coarse_add(dcontext, freeze_info->dst_info,
+                                                fallthrough_tag,
+                                                freeze_info->cache_cur_pc +
+                                                (next_pc - src_body) -
+                                                (ptr_uint_t)freeze_info->cache_start_pc +
+                                                cache_offs);
+                            DODEBUG({
+                                /* We should NOT need to add a stub like we might
+                                 * for the entry point add above: fall-through
+                                 * cannot be trace head! */
+                                cache_pc src_stub = NULL;
+                                fragment_coarse_lookup_in_unit(dcontext,
+                                                               freeze_info->src_info,
+                                                               fallthrough_tag,
+                                                               &src_stub, NULL);
+                                ASSERT(src_stub == NULL);
+                            });
+                        }
+                        fallthrough_tag = NULL;
+                    } else {
+                        LOG(THREAD, LOG_FRAGMENT, 4, " (duplicate)\n");
+                        break;
+                    }
+                }
+            }
+            instr_reset(dcontext, instr);
+            next_pc = decode_cti(dcontext, pc, instr);
+            ASSERT(next_pc - src_body <= MAX_FRAGMENT_SIZE);
+            /* Case 8711: we can't distinguish exit ctis from others,
+             * so we must assume that any cti is an exit cti.
+             * Assumption: coarse-grain bbs have 1 ind exit or 2 direct,
+             * and no code beyond the last exit!
+             */
+        } while (!instr_opcode_valid(instr) || !instr_is_cti(instr));
+
+        if (dst_body == NULL) { /* not a dup */
+            /* copy body of fragment, including cti (if not ending @ fall-through) */
+            size_t sz = next_pc - src_body;
+            memcpy(freeze_info->cache_cur_pc, src_body, sz);
+            freeze_info->cache_cur_pc += sz;
+        }
+
+        if (fallthrough_tag != NULL) {
+            ASSERT(next_pc == pc); /* should have short-circuited */
+            /* add intra-cache jmp if elided but fall-through a dup */
+            ASSERT(fallthrough_body != NULL);
+            /* If start bb not a dup, or post-cbr, must un-elide */
+            if (dst_body == NULL || (next_pc == src_body && last_dst_body == NULL)) {
+                LOG(THREAD, LOG_FRAGMENT, 4,
+                    "\tadding jmp @"PFX" to "PFX" for fall-through tag "PFX"\n",
+                    freeze_info->cache_cur_pc, fallthrough_body, fallthrough_tag);
+                freeze_info->cache_cur_pc =
+                    insert_relative_jump(freeze_info->cache_cur_pc, fallthrough_body,
+                                         NOT_HOT_PATCHABLE);
+            }
+        } else {
+            ASSERT(instr_opcode_valid(instr) && instr_is_cti(instr));
+            /* Ensure we get proper target for short cti sequence, and copy full cti */
+            if (instr_is_cti_short_rewrite(instr, pc)) {
+                app_pc old_next_pc = next_pc;
+                next_pc = remangle_short_rewrite(dcontext, instr, pc, 0/*same target*/);
+                if (dst_body == NULL) { /* not a dup */
+                    memcpy(freeze_info->cache_cur_pc, old_next_pc,
+                           next_pc - old_next_pc);
+                    freeze_info->cache_cur_pc += (next_pc - old_next_pc);
+                }
+            }
+            tgt = opnd_get_pc(instr_get_target(instr));
+            if (in_coarse_stub_prefixes(tgt)) {
+                /* We should not encounter prefix targets other than indirect while
+                 * in the body of the cache (rest are from the stubs) */
+                ASSERT(coarse_is_indirect_stub
+                       (next_pc - coarse_indirect_stub_size(freeze_info->src_info)));
+                /* indirect exit stub: need to update jmp to prefix */
+                ASSERT(instr_is_ubr(instr));
+                if (dst_body == NULL) { /* not a dup */
+                    tgt = PC_RELATIVE_TARGET(next_pc - 4);
+                    tgt = redirect_to_tgt_ibl_prefix(dcontext, freeze_info, tgt);
+                    ASSERT(dynamo_all_threads_synched); /* thus NOT_HOT_PATCHABLE */
+                    /* we've already copied the stub as part of the body */
+                    ASSERT(coarse_is_indirect_stub
+                           (freeze_info->cache_cur_pc -
+                            coarse_indirect_stub_size(freeze_info->src_info)));
+                    freeze_info->cache_cur_pc -= 4;
+                    freeze_info->cache_cur_pc =
+                        insert_relative_target(freeze_info->cache_cur_pc,
+                                               tgt, NOT_HOT_PATCHABLE);
+                }
+            } else if (tgt < freeze_info->src_info->cache_start_pc || tgt >= stop_pc) {
+                if (dst_body == NULL) { /* not a dup */
+                    /* currently goes through a stub */
+                    ASSERT(tgt >= freeze_info->src_info->stubs_start_pc &&
+                           tgt < freeze_info->src_info->stubs_end_pc);
+                    if (instr_is_cbr(instr)) {
+                        uint cbr_len;
+                        if (instr_is_cti_short_rewrite(instr, pc))
+                            cbr_len = CBR_SHORT_REWRITE_LENGTH;
+                        else
+                            cbr_len = CBR_LONG_LENGTH;
+                        ASSERT(pc + cbr_len == next_pc);
+                        coarse_merge_process_stub(dcontext, freeze_info, tgt, cbr_len,
+                                                  freeze_info->cache_cur_pc - cbr_len,
+                                                  replace_outgoing);
+                        /* If there is a ubr next (could be elided) we just hit
+                         * it next time around loop */
+                    } else {
+                        ASSERT(instr_is_ubr(instr));
+                        ASSERT(pc + JMP_LONG_LENGTH == next_pc);
+                        coarse_merge_process_stub(dcontext, freeze_info, tgt,
+                                                  JMP_LONG_LENGTH,
+                                                  freeze_info->cache_cur_pc -
+                                                  JMP_LONG_LENGTH,
+                                                  replace_outgoing);
+                    }
+                }
+            } else if (dst_body == NULL) { /* not a dup */
+                /* Intra-cache target, but we're moving things around and have to do
+                 * a separate pass since don't know future locations.  Since the
+                 * layout is changing and later we'd need multiple lookups to find
+                 * the corrsepondence between src and dst, we store the target tag in
+                 * the jmp and replace it w/ the body in the later pass.
+                 */
+                app_pc tgt_tag =
+                    fragment_coarse_entry_pclookup(dcontext, freeze_info->src_info, tgt);
+                ASSERT(tgt_tag != NULL);
+                LOG(THREAD, LOG_FRAGMENT, 4,
+                    "\tintra-cache src "PFX"->"PFX" tag "PFX" dst pre-"PFX"\n",
+                    pc, tgt, tgt_tag, freeze_info->cache_cur_pc);
+                /* This works for mangled short cti sequences as well as regular */
+                insert_relative_target(freeze_info->cache_cur_pc - 4,
+                                       tgt_tag, NOT_HOT_PATCHABLE);
+                /* Just make sure we can recognize it as an app tag */
+                ASSERT(tgt_tag < freeze_info->dst_info->cache_start_pc ||
+                       tgt_tag >= freeze_info->dst_info->stubs_end_pc);
+            }
+        }
+    }
+
+    /* Second pass to update intra-cache targets.
+     * FIXME: combine w/ later coarse_unit_shift_jmps()
+     */
+    next_pc = freeze_info->cache_start_pc;
+    while (next_pc < freeze_info->cache_cur_pc) {
+        instr_reset(dcontext, instr);
+        pc = next_pc;
+        next_pc = decode_cti(dcontext, pc, instr);
+        if (instr_opcode_valid(instr) && instr_is_cti(instr)) {
+            ASSERT(next_pc != NULL);
+            if (instr_is_cti_short_rewrite(instr, pc))
+                next_pc = remangle_short_rewrite(dcontext, instr, pc, 0/*same target*/);
+            tgt = opnd_get_pc(instr_get_target(instr));
+            /* We stored the app tag as the target */
+            if (tgt < freeze_info->dst_info->cache_start_pc ||
+                tgt >= freeze_info->dst_info->stubs_end_pc) {
+                fragment_coarse_lookup_in_unit(dcontext, freeze_info->dst_info,
+                                               tgt, NULL, &dst_body);
+                ASSERT(dst_body != NULL);
+                LOG(THREAD, LOG_FRAGMENT, 4,
+                    "\tintra-cache dst "PFX"->"PFX" tag "PFX"\n", pc, dst_body, tgt);
+                /* FIXME: make 4 a named constant; used elsewhere as well */
+                insert_relative_target(next_pc - 4, dst_body, NOT_HOT_PATCHABLE);
+            }
+        }
+    }
+
+    instr_destroy(dcontext, instr);
+}
+
+/* Returns a new coarse_info_t (or if in_place returns info1) that combines
+ * info1 and info2.  In in_place, info1 is replaced with the result and returned;
+ * else, a separate coarse_info_t is created and returned.
+ * If either of the units is live, then info1 must be live.
+ * If one of the two units covers a different code range, it must be info2,
+ * and it must be a subset of info1's range.
+ * If returns NULL, the merge failed; if in_place, info1 is unchanged on failure.
+ * If in_place, caller is responsible for flushing the ibl tables (case 11057).
+ */
+coarse_info_t *
+coarse_unit_merge(dcontext_t *dcontext, coarse_info_t *info1, coarse_info_t *info2,
+                  bool in_place)
+{
+    coarse_info_t *merged;
+    coarse_info_t *res = NULL;
+    coarse_info_t *src_sm, *src_lg;
+    size_t cache1_size, cachelg_size, cache2_size, merged_cache_size;
+    size_t stubs1_size, stubs2_size;
+    coarse_freeze_info_t freeze_info;
+
+    LOG(THREAD, LOG_CACHE, 2, "coarse_unit_merge %s %s with %s\n",
+        info1->module, info1->persisted ? "persisted" : "non-persisted",
+        info2->persisted ? "persisted" : "non-persisted");
+    STATS_INC(coarse_units_merged);
+
+    ASSERT(info1 != NULL && info2 != NULL);
+    ASSERT(info1->base_pc <= info2->base_pc && info1->end_pc >= info2->end_pc);
+    if (info1->base_pc > info2->base_pc || info1->end_pc < info2->end_pc)
+        return NULL;
+    /* Currently we only do online merging where one unit is live */
+    ASSERT(!info1->persisted || !info2->persisted);
+
+    /* Much more efficient to merge smaller cache into larger */
+    if (fragment_coarse_num_entries(info1) > fragment_coarse_num_entries(info2)) {
+        src_lg = info1;
+        src_sm = info2;
+    } else {
+        src_lg = info2;
+        src_sm = info1;
+    }
+
+    /* Ensure the pclookup table is set up for src_sm, to avoid recursive
+     * lock issues
+     */
+    if (src_sm->pclookup_htable == NULL) { /* read needs no lock */
+        fragment_coarse_entry_pclookup(dcontext, src_sm, NULL);
+        ASSERT(src_sm->pclookup_htable != NULL);
+    }
+
+    acquire_recursive_lock(&change_linking_lock);
+#ifdef HOT_PATCHING_INTERFACE
+    /* we may call coarse_unit_calculate_persist_info() */
+    if (DYNAMO_OPTION(hot_patching))
+        read_lock(hotp_get_lock());
+#endif
+    /* We can't grab both locks due to deadlock potential.  Currently we are
+     * always fully synched, so we rely on that to synch with info2.
+     */
+    ASSERT(dynamo_all_threads_synched);
+    mutex_lock(&info1->lock);
+    ASSERT(info1->cache != NULL && info2->cache != NULL); /* don't merge empty units */
+    ASSERT(info1->frozen && info2->frozen);
+
+    /* Tasks:
+     * 1) Merge the caches, eliminating duplicates.  Various optimizations to
+     *    preserve fall-through pairs if 2nd part is in other cache but 1st is
+     *    not could be employed.
+     * 2) Turn cross-links into direct links and eliminate those entrance stubs.
+     * 3) Copy the rest of the entrance stubs.
+     * 4) Merge the RCT, RAC, and hotp fields, which may require calculating
+     *    persist info if this unit has not been persisted; if neither unit
+     *    has been persisted we do not need to do anything here.
+     *
+     * We will remove the extra post-stubs space since when persisting
+     * we only write up through stubs_end_pc and not the allocation end.
+     */
+    /* Whether merging in-place or not, we create a new coarse_info_t.
+     * If in-place we delete the old one afterward.
+     */
+    merged = coarse_unit_create(info1->base_pc, info1->end_pc, &info1->module_md5,
+                                in_place && info1->in_use);
+    merged->frozen = true;
+    cache1_size = info1->cache_end_pc - info1->cache_start_pc;
+    cache2_size = info2->cache_end_pc - info2->cache_start_pc;
+    /* We shrink the cache to size below, after merging & removing dups */
+    merged_cache_size = cache1_size + cache2_size;
+    /* We need the stubs to start on a new page since will be +rw vs cache +r */
+    merged_cache_size = ALIGN_FORWARD(merged_cache_size, PAGE_SIZE);
+    /* We only need one set of prefixes */
+    stubs1_size = info1->stubs_end_pc - info1->fcache_return_prefix;
+    stubs2_size = info2->stubs_end_pc - info2->stubs_start_pc;
+    merged->mmap_size = merged_cache_size + stubs1_size + stubs2_size;
+    /* Our relative jmps require that we do not exceed 32-bit reachability */
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(merged->mmap_size)));
+    merged->cache_start_pc = (cache_pc) heap_mmap(merged->mmap_size);
+    merged->cache_end_pc = merged->cache_start_pc + cache1_size + cache2_size;
+    merged->stubs_start_pc =
+        coarse_stubs_create(merged, merged->cache_start_pc + merged_cache_size,
+                            stubs1_size + stubs2_size);
+    /* will be tightened up later */
+    merged->stubs_end_pc = merged->cache_start_pc + merged->mmap_size;
+    ASSERT(merged->stubs_start_pc != NULL);
+    ASSERT(ALIGNED(merged->stubs_start_pc, coarse_stub_alignment(info1)));
+    ASSERT(merged->fcache_return_prefix ==
+           merged->cache_start_pc + merged_cache_size);
+    ASSERT(merged->trace_head_return_prefix == merged->fcache_return_prefix +
+           (info1->trace_head_return_prefix - info1->fcache_return_prefix));
+    ASSERT(merged->ibl_ret_prefix == merged->fcache_return_prefix +
+           (info1->ibl_ret_prefix - info1->fcache_return_prefix));
+    ASSERT(merged->ibl_call_prefix == merged->fcache_return_prefix +
+           (info1->ibl_call_prefix - info1->fcache_return_prefix));
+    ASSERT(merged->ibl_jmp_prefix == merged->fcache_return_prefix +
+           (info1->ibl_jmp_prefix - info1->fcache_return_prefix));
+
+    /* Much more efficient to put the larger cache 1st, but we have to be sure
+     * to use the same order for both the htable and cache.
+     */
+
+    /* Try to size the dst tables to avoid collision asserts.
+     * Put the larger unit's entries into the dst table up front.
+     * FIXME: do this earlier and if one is a subset of other then do
+     * a simpler merge?
+     */
+    fragment_coarse_htable_merge(dcontext, merged, src_lg, src_sm,
+                                 false/*do not add src_sm yet*/,
+                                 false/*leave th htable empty*/);
+
+    /* Copy the 1st cache intact, and bring in the non-dup portions of the second
+     * (since know the offsets of the 1st); then, walk the 1st and patch up its
+     * inter-unit links while decoding from the original, now that the 2nd is in
+     * place.
+     */
+    cachelg_size = (src_lg == info2) ? cache2_size : cache1_size;
+    memcpy(merged->cache_start_pc, src_lg->cache_start_pc, cachelg_size);
+
+    memset(&freeze_info, 0, sizeof(freeze_info));
+    freeze_info.dst_info = merged;
+    freeze_info.stubs_start_pc = merged->stubs_start_pc;
+    freeze_info.stubs_cur_pc = merged->stubs_start_pc;
+    /* Just like for freezing: leave inter-unit links intact for in_place.
+     * coarse_merge_process_stub() assumes that unlink == !in_place
+     */
+    freeze_info.unlink = !in_place;
+
+    freeze_info.src_info = src_sm;
+    freeze_info.cache_start_pc = merged->cache_start_pc + cachelg_size;
+    freeze_info.cache_cur_pc = freeze_info.cache_start_pc;
+    coarse_merge_without_dups(dcontext, &freeze_info, cachelg_size,
+                              /* replace for primary unit; add for secondary */
+                              freeze_info.src_info == info1);
+    merged->cache_end_pc = freeze_info.cache_cur_pc;
+
+    freeze_info.src_info = src_lg;
+    freeze_info.cache_start_pc = merged->cache_start_pc;
+    freeze_info.cache_cur_pc = freeze_info.cache_start_pc;
+    coarse_merge_update_jmps(dcontext, &freeze_info,
+                              /* replace for primary unit; add for secondary */
+                             freeze_info.src_info == info1);
+
+    ASSERT((ptr_uint_t)(freeze_info.stubs_cur_pc - merged->fcache_return_prefix) <=
+           stubs1_size + stubs2_size);
+
+    /* We have extra space from extra in each stub region (from case 9428),
+     * from duplicate prefix space, and from eliminated inter-unit stubs, so
+     * we must set end pc.
+     */
+    coarse_stubs_set_end_pc(merged, freeze_info.stubs_cur_pc); 
+
+    LOG(THREAD, LOG_CACHE, 2, "merged size: stubs "SZFMT" => "SZFMT" bytes, "
+        "cache "SZFMT" ("SZFMT" align) => "SZFMT" ("SZFMT" align) bytes\n",
+        stubs1_size + stubs2_size, freeze_info.stubs_cur_pc - merged->stubs_start_pc,
+        cache1_size + cache2_size,
+        (info1->fcache_return_prefix - info1->cache_start_pc) +
+        (info2->fcache_return_prefix - info2->cache_start_pc),
+        merged->cache_end_pc - merged->cache_start_pc,
+        merged->fcache_return_prefix - merged->cache_start_pc);
+
+    if (merged_cache_size - (merged->cache_end_pc - merged->cache_start_pc) > 0) {
+        /* With duplicate elimination we often have a lot of empty space, so we
+         * re-allocate into a proper-fitting space
+         */
+        size_t cachesz = merged->cache_end_pc - merged->cache_start_pc;
+        size_t cachesz_aligned = ALIGN_FORWARD(cachesz, PAGE_SIZE);
+        size_t stubsz = merged->stubs_end_pc - merged->fcache_return_prefix;
+        size_t newsz = cachesz_aligned + stubsz;
+        size_t old_mapsz = merged->mmap_size;
+        cache_pc newmap = (cache_pc) heap_mmap(newsz);
+        ssize_t cache_shift = merged->cache_start_pc - newmap;
+        /* stubs have moved too, so a relative shift not absolute */
+        ssize_t stubs_shift = cachesz_aligned -
+            (merged->fcache_return_prefix - merged->cache_start_pc);
+        LOG(THREAD, LOG_CACHE, 2,
+            "re-allocating merged unit: "SZFMT" @"PFX" "PFX" => "
+            SZFMT" @"PFX" "PFX" "SZFMT" "SZFMT"\n",
+            merged->mmap_size, merged->cache_start_pc, merged->fcache_return_prefix,
+            newsz, newmap, newmap + cachesz_aligned, cache_shift, stubs_shift);
+        memcpy(newmap, merged->cache_start_pc, cachesz);
+        memcpy(newmap + cachesz_aligned, merged->fcache_return_prefix, stubsz);
+        heap_munmap(merged->cache_start_pc, merged->mmap_size);
+        coarse_stubs_delete(merged);
+        merged->mmap_size = newsz;
+        /* Our relative jmps require that we do not exceed 32-bit reachability */
+        IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(merged->mmap_size)));
+        merged->cache_start_pc = newmap;
+        merged->cache_end_pc = merged->cache_start_pc + cachesz;
+        merged->stubs_start_pc =
+            coarse_stubs_create(merged, merged->cache_start_pc + cachesz_aligned,
+                                stubsz);
+        ASSERT(merged->stubs_start_pc != NULL);
+        ASSERT(ALIGNED(merged->stubs_start_pc, coarse_stub_alignment(info1)));
+        ASSERT(merged->fcache_return_prefix == newmap + cachesz_aligned);
+        coarse_stubs_set_end_pc(merged, merged->cache_start_pc + newsz);
+        coarse_unit_shift_jmps(dcontext, merged, cache_shift, stubs_shift, old_mapsz);
+    }
+
+    /* Set cache bounds after we've potentially moved the initial cache */
+    fcache_coarse_init_frozen(dcontext, merged, merged->cache_start_pc,
+                              merged->fcache_return_prefix - merged->cache_start_pc);
+
+    /* Currently we only do online merging where at least one unit is live,
+     * and we expect that to be info1
+     */
+    ASSERT(!info1->persisted);
+    /* Store the source persisted size so we know whether we need to merge
+     * with that on-disk file
+     */
+    if (info2->persisted)
+        merged->persisted_source_mmap_size = info2->mmap_size;
+
+    /* Merge the other fields */
+    coarse_unit_merge_persist_info(dcontext, merged, info1, info2);
+
+    DOLOG(5, LOG_CACHE, {
+        byte *pc = merged->cache_start_pc;
+        LOG(THREAD, LOG_CACHE, 1, "merged cache:\n");
+        do {
+            pc = disassemble_with_bytes(dcontext, pc, THREAD);
+        } while (pc < merged->cache_end_pc);
+        pc = merged->stubs_start_pc;
+        LOG(THREAD, LOG_CACHE, 1, "merged stubs:\n");
+        do {
+            if (((ptr_uint_t)pc) % coarse_stub_alignment(info1) ==
+                coarse_stub_alignment(info1)-1)
+                pc++;
+            pc = disassemble_with_bytes(dcontext, pc, THREAD);
+        } while (pc < merged->stubs_end_pc);
+    });
+    
+    /* FIXME case 9687: mark cache as read-only */
+
+    if (in_place) {
+        coarse_incoming_t *e;
+        coarse_replace_unit(dcontext, info1, merged);
+        merged = NULL;
+        /* up to caller to call mark_executable_area_coarse_frozen() if necessary */
+
+        /* case 10877: must combine the incoming lists
+         * targets should be unique, so can just append
+         */
+        mutex_lock(&info1->incoming_lock);
+        /* can't grab info2 lock, so just like for main lock we rely on synchall */
+        DODEBUG({
+            /* Make sure no inter-incoming left */
+            uint in1 = 0;
+            uint in2 = 0;
+            for (e = info1->incoming; e != NULL; e = e->next, in1++)
+                ASSERT(!e->coarse || get_stub_coarse_info(e->in.stub_pc) != info2);
+            for (e = info2->incoming; e != NULL; e = e->next, in2++)
+                ASSERT(!e->coarse || get_stub_coarse_info(e->in.stub_pc) != info1);
+            LOG(THREAD, LOG_CACHE, 1, "merging %d incoming into %d incoming\n",
+                in2, in1);
+        });
+        e = info1->incoming;
+        if (e == NULL) {
+            info1->incoming = info2->incoming;
+        } else {
+            while (e->next != NULL)
+                e = e->next;
+            e->next = info2->incoming;
+        }
+        mutex_unlock(&info1->incoming_lock);
+        info2->incoming = NULL; /* ensure not freed when info2 is freed */
+        coarse_unit_shift_links(dcontext, info1);
+
+        res = info1;
+    } else {
+        /* we made separate copy that has no outgoing or incoming links */
+        res = merged;
+    }
+    mutex_unlock(&info1->lock);
+#ifdef HOT_PATCHING_INTERFACE
+    /* we may call coarse_unit_calculate_persist_info() */
+    if (DYNAMO_OPTION(hot_patching))
+        read_unlock(hotp_get_lock());
+#endif
+    release_recursive_lock(&change_linking_lock);
+    return res;
+}
+
+/***************************************************************************
+ * PERSISTENT CODE CACHE
+ */
+
+#if defined(RETURN_AFTER_CALL) && defined(WINDOWS)
+extern bool seen_Borland_SEH;
+#endif
+
+/* get global or per-user directory name */
+static bool
+get_persist_dir(char *directory /* OUT */,
+                uint directory_len,
+                bool create)
+{
+    int retval;
+
+    retval = get_parameter((DYNAMO_OPTION(persist_per_user) ? 
+                            L_IF_WIN(DYNAMORIO_VAR_PERSCACHE_ROOT) : 
+                            L_IF_WIN(DYNAMORIO_VAR_PERSCACHE_SHARED)),
+                           directory, directory_len);
+    if (IS_GET_PARAMETER_FAILURE(retval) || strchr(directory, DIRSEP) == NULL) {
+        SYSLOG_INTERNAL_ERROR_ONCE("%s not set!"
+                                   " Persistent cache will not operate.",
+                                   (DYNAMO_OPTION(persist_per_user) ? 
+                                    DYNAMORIO_VAR_PERSCACHE_ROOT : 
+                                    DYNAMORIO_VAR_PERSCACHE_SHARED));
+        return false;
+    }
+    directory[directory_len - 1] = '\0';
+
+    if (DYNAMO_OPTION(persist_per_user)) {
+        bool res = os_current_user_directory(directory, directory_len,
+                                             create);
+        /* null terminated */
+        if (!res) {
+            /* directory name may be set even on failure */
+            LOG(GLOBAL, LOG_CACHE, 2, "\terror opening per-user dir %s\n", directory);
+            return false;
+        }
+    }
+
+    return true;
+}
+
+/* Checks for enough space on the volume where persisted caches are stored */
+bool
+coarse_unit_check_persist_space(file_t fd_in/*OPTIONAL*/, size_t size_needed)
+{
+    bool room = false;
+    file_t fd = fd_in;
+    if (fd == INVALID_FILE) {
+        /* Use directory to get handle on proper volume */
+        char dir[MAXIMUM_PATH];
+        if (get_persist_dir(dir, BUFFER_SIZE_ELEMENTS(dir), 
+                            true /* note we MUST always create directory
+                                  * even if never persisting */
+                            )) {
+            fd = os_open_directory(dir, 0);
+        }
+    }
+    if (fd != INVALID_FILE) {
+        room = check_low_disk_threshold(fd, (uint64)size_needed);
+        if (fd_in == INVALID_FILE) {
+            /* FIXME: cache the handle, combine with -validate_owner_dir */
+            os_close(fd);
+        }
+    }
+    return room;
+}
+
+/* If force_local, pretends module at pc has been exempted (so no effect
+ * unless -persist_check_exempted_options)
+ */
+static inline op_pcache_t
+persist_get_options_level(app_pc pc, coarse_info_t *info, bool force_local)
+{
+    if (!DYNAMO_OPTION(persist_check_options))
+        return OP_PCACHE_NOP;
+    else if (DYNAMO_OPTION(persist_check_local_options) ||
+             (DYNAMO_OPTION(persist_check_exempted_options) &&
+              (force_local ||
+               /* once loaded as local, must remain local, even if this
+                * process never hit the exemption */
+               (info != NULL && TEST(PERSCACHE_EXEMPTION_OPTIONS, info->flags)) ||
+               os_module_get_flag(pc, MODULE_WAS_EXEMPTED)) &&
+              /* don't use local if no such options: else when load will think
+               * local when really global */
+              has_pcache_dynamo_options(&dynamo_options, OP_PCACHE_LOCAL)))
+        return OP_PCACHE_LOCAL;
+    else
+        return OP_PCACHE_GLOBAL;
+}
+
+static const char *
+persist_get_relevant_options(dcontext_t *dcontext, char *option_buf,
+                             uint buf_len, op_pcache_t level)
+{
+    if (level == OP_PCACHE_NOP)
+        return NULL;
+    get_pcache_dynamo_options_string(&dynamo_options, option_buf, buf_len, level);
+    option_buf[buf_len - 1] = '\0';
+    LOG(THREAD, LOG_CACHE, 2, "Pcache-affecting options = %s\n", option_buf);
+    return option_buf;
+}
+
+/* We identify persisted caches by mapping module info into a canonical name.
+ * There can be collisions (including for sub-module coarse units, such as
+ * separate +x module sections (xref case 9834 and case 9653); in addition, we
+ * can have different modules map to the same name), so caller must further
+ * verify matches.
+ * (Note that we use a different scheme than aslr's calculate_publish_name()
+ * as we are not dealing with file handles here but in-memory module images).
+ */
+static bool
+get_persist_filename(char *filename /*OUT*/, uint filename_max /* max #chars */,
+                     app_pc modbase, bool write, persisted_module_info_t *modinfo,
+                     const char *option_string)
+{
+    uint checksum, timestamp;
+    size_t size, code_size;
+    uint64 file_version;
+    const char *name;
+    uint hash;
+    char dir[MAXIMUM_PATH];
+    
+    os_get_module_info_lock();
+    if (!os_get_module_info(modbase, &checksum, &timestamp, &size,
+                            &name, &code_size, &file_version)) {
+        os_get_module_info_unlock();
+        return false;
+    }
+    if (name == NULL) {
+        /* theoretically possible but pathological, unless we came in late */
+        ASSERT_CURIOSITY(IF_WINDOWS_ELSE_0(!dr_early_injected));
+        LOG(GLOBAL, LOG_CACHE, 1, "\tmodule "PFX" has no name\n", modbase);
+        os_get_module_info_unlock();
+        return false;
+    }
+    /* Should not have path chars in the name */
+    ASSERT(get_short_name(name) == name && name[0] != DIRSEP);
+    name = get_short_name(name); /* paranoid */
+
+    /* Exclude list applies to both read and write */
+    if (!IS_STRING_OPTION_EMPTY(persist_exclude_list)) {
+        bool exclude;
+        string_option_read_lock();
+        exclude = check_filter(DYNAMO_OPTION(persist_exclude_list), name);
+        string_option_read_unlock();
+        if (exclude) {
+            LOG(GLOBAL, LOG_CACHE, 1, "\t%s is on exclude list\n", name);
+            DOSTATS({
+                if (write)
+                    STATS_INC(coarse_units_persist_excluded);
+                else
+                    STATS_INC(perscache_load_excluded);
+            });
+            os_get_module_info_unlock();
+            return false;
+        }
+    }
+
+    /* Prepend the perscache dir.  We assume it has already been created.
+     * FIXME: cache this, or better, cache the dir handle and use an
+     * os_open that can take it in.  Note that the directory handle
+     * doesn't help us in Linux - we can neither open files relative to it,
+     * nor there is any strong chown guarantee that we depend on.
+     */
+    if (!get_persist_dir(dir, BUFFER_SIZE_ELEMENTS(dir), write)) {
+        os_get_module_info_unlock();
+        return false;
+    }
+
+    /* FIXME case 8494: version-independent names so we clobber files
+     * from old versions of modules and have less need of stale file
+     * cleanup?  If so should add in (hash of) full path (w/ volume)
+     * to avoid name conflicts?  But if try to share across machines
+     * we do not want to include path since can vary.
+     */
+    /* should we go to a 64-bit hash? */
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(size)));
+    hash = checksum ^ timestamp ^ (uint)size;
+    /* case 9799: make options part of namespace */
+    if (option_string != NULL) {
+        uint i;
+        ASSERT(DYNAMO_OPTION(persist_check_options));
+        for (i = 0; i < strlen(option_string); i++)
+            hash ^= option_string[i] << ((i % 4)*8);
+    }
+    LOG(GLOBAL, LOG_CACHE, 2, "\thash = 0x%08x^0x%08x^"PFX" ^ %s = "PFX"\n",
+        checksum, timestamp, size, option_string == NULL ? "" : option_string, hash);
+    ASSERT_CURIOSITY(hash != 0);
+
+    if (DYNAMO_OPTION(persist_per_app)) {
+        char *dirend = dir + strlen(dir);
+        /* FIXME case 9692: put tls offs instead of "dbg" here, and then
+         * sqlservr can have its own set if it ends up w/ separate tls offs
+         * (once we have non-per-app persisted files, that is).
+         */
+        snprintf(dirend, BUFFER_SIZE_ELEMENTS(dir) - (dirend - dir), "%c%s%s",
+                 DIRSEP, get_application_short_name(), IF_DEBUG_ELSE("-dbg", ""));
+        NULL_TERMINATE_BUFFER(dir);
+        LOG(GLOBAL, LOG_CACHE, 2, "\tper-app dir is %s\n", dir);
+
+        /* check for existence first so we can require new during creation */
+        if (!os_file_exists(dir, true/*is dir*/) && write) {
+            if (!os_create_dir(dir, CREATE_DIR_REQUIRE_NEW)) {
+                LOG(GLOBAL, LOG_CACHE, 2, "\terror creating per-app dir %s\n", dir);
+                os_get_module_info_unlock();
+                return false;
+            } else
+                LOG(GLOBAL, LOG_CACHE, 2, "\tcreated per-app dir %s\n", dir);
+        }
+    }
+    /* FIXME case 9653: should we put the section ordinal or vmarea range into
+     * the name to support simultaneous sub-module files?  If sections are
+     * adjacent they'll be one vmarea, so this affects very few dlls.  For now
+     * we only support one file per module.  We could also support multiple
+     * ranges per file.
+     */
+    snprintf(filename, filename_max, "%s%c%s%s-0x%08x.%s", dir, DIRSEP, name,
+             IF_DEBUG_ELSE("-dbg", ""), hash, PERSCACHE_FILE_SUFFIX);
+    filename[filename_max-1] = '\0';
+    os_get_module_info_unlock();
+    if (modinfo != NULL) {
+        modinfo->base = modbase;
+        modinfo->checksum = checksum;
+        modinfo->timestamp = timestamp;
+        modinfo->image_size = size;
+        modinfo->code_size = code_size;
+        modinfo->file_version = file_version;
+    }
+    return true;
+}
+
+#if defined(DEBUG) && defined(INTERNAL)
+/* FIXME: share w/ aslr.c */
+static void
+print_module_digest(file_t f, module_digest_t *digest, const char *prefix)
+{
+    LOG(f, LOG_CACHE, 1, "%s\n  md5 short: ", prefix);
+    dump_buffer_as_bytes(f, digest->short_MD5, MD5_RAW_BYTES, DUMP_RAW);
+    LOG(f, LOG_CACHE, 1, "\n  md5 long:  ");
+    dump_buffer_as_bytes(f, digest->full_MD5, MD5_RAW_BYTES, DUMP_RAW);
+    LOG(f, LOG_CACHE, 1, "\n");
+}
+#endif
+
+static void
+persist_calculate_self_digest(module_digest_t *digest, coarse_persisted_info_t *pers,
+                              app_pc map, uint validation_option)
+{
+    struct MD5Context self_md5_cxt;
+    if (TEST(PERSCACHE_GENFILE_MD5_COMPLETE, validation_option)) {
+        MD5Init(&self_md5_cxt);
+        /* Even if generated w/ -persist_map_rw_separate but loaded w/o that
+         * option, the md5 should match since the memory layout is the same.
+         */
+        MD5Update(&self_md5_cxt, map, pers->header_len + pers->data_len
+                  - sizeof(persisted_footer_t));
+        MD5Final(digest->full_MD5, &self_md5_cxt);
+    }
+    if (TEST(PERSCACHE_GENFILE_MD5_SHORT, validation_option)) {
+        MD5Init(&self_md5_cxt);
+        MD5Update(&self_md5_cxt, (byte *) pers, pers->header_len);
+        MD5Final(digest->short_MD5, &self_md5_cxt);
+    }
+}
+
+static void
+persist_calculate_module_digest(module_digest_t *digest, app_pc modbase, size_t modsize,
+                                app_pc code_start, app_pc code_end,
+                                uint validation_option)
+{
+    size_t view_size = modsize;
+    if (TESTANY(PERSCACHE_MODULE_MD5_COMPLETE|PERSCACHE_MODULE_MD5_SHORT,
+                validation_option)) {
+        /* case 9717: need view size, not image size */
+        view_size = os_module_get_view_size(modbase);
+    }
+    if (TEST(PERSCACHE_MODULE_MD5_COMPLETE, validation_option)) {
+        /* We can't use a full md5 from module_calculate_digest() since .data
+         * and other sections change between persist and load times (this is
+         * in-memory image, not file).  So we do md5 of code region.  If we have
+         * hooks at persist time but not at load time we will cry foul;
+         * PERSCACHE_MODULE_MD5_AT_LOAD tries to get around this by using
+         * the load-time md5 when persisting.
+         */
+        struct MD5Context code_md5_cxt;
+        MD5Init(&code_md5_cxt);
+        /* Code range should be within a single memory allocation so it should
+         * all be readable.  Xref case 9653.
+         */
+        code_end = MIN(code_end, modbase + view_size);
+        MD5Update(&code_md5_cxt, code_start, code_end - code_start);
+        MD5Final(digest->full_MD5, &code_md5_cxt);
+    }
+    if (TEST(PERSCACHE_MODULE_MD5_SHORT, validation_option)) {
+        /* Examine only the image header and the footer (if non-writable) 
+         * FIXME: if view_size < modsize, better to skip the footer than have it
+         * cover a data section?  Should be ok w/ PERSCACHE_MODULE_MD5_AT_LOAD.
+         */
+        module_calculate_digest(digest, modbase, view_size,
+                                false /* not full */, true /* yes short */,
+                                DYNAMO_OPTION(persist_short_digest),
+                                /* do not consider writable sections */
+                                ~((uint)OS_IMAGE_WRITE));
+    }
+}
+
+/* Compares all but the module base */
+static bool
+persist_modinfo_cmp(persisted_module_info_t *mi1, persisted_module_info_t *mi2)
+{
+    bool match = true;
+    /* We'd like to know if we have an md5 mismatch */
+    ASSERT_CURIOSITY(module_digests_equal(&mi1->module_md5, &mi2->module_md5,
+                                          TEST(PERSCACHE_MODULE_MD5_SHORT,
+                                               DYNAMO_OPTION(persist_load_validation)),
+                                          TEST(PERSCACHE_MODULE_MD5_COMPLETE,
+                                               DYNAMO_OPTION(persist_load_validation)))
+                     || check_filter("partial_map.exe",
+                                     get_short_name(get_application_name())));
+    if (TESTALL(PERSCACHE_MODULE_MD5_SHORT|PERSCACHE_MODULE_MD5_COMPLETE,
+                DYNAMO_OPTION(persist_load_validation))) {
+        return (memcmp(&mi1->checksum, &mi2->checksum,
+                       sizeof(*mi1)-offsetof(persisted_module_info_t, checksum)) == 0);
+    }
+    match = match && (memcmp(&mi1->checksum, &mi2->checksum,
+                             offsetof(persisted_module_info_t, module_md5) -
+                             offsetof(persisted_module_info_t, checksum)) == 0);
+    match = match && module_digests_equal(&mi1->module_md5, &mi2->module_md5,
+                                          TEST(PERSCACHE_MODULE_MD5_SHORT,
+                                               DYNAMO_OPTION(persist_load_validation)),
+                                          TEST(PERSCACHE_MODULE_MD5_COMPLETE,
+                                               DYNAMO_OPTION(persist_load_validation)));
+    return match;
+}
+
+/* key is meant to be a short string to help identify the purpose of this name.
+ * FIXME: right now up to caller to figure out if the name collided w/ an
+ * existing file; maybe this routine should do that and return a file handle?
+ * FIXME: combine w/ get_unique_logfile, which handles file creation race?
+ * As it is this is not mkstemp, and caller must use OS_OPEN_REQUIRE_NEW.
+ */
+static void
+get_unique_name(const char *origname, const char *key,
+                char *filename /*OUT*/, uint filename_max /* max #chars */)
+{
+    /* We need unique names for:
+     * 1) case 9696: a temp file to build our pcache in
+     * before renaming to the real thing
+     * 2) case 9701: to rename the existing file before we replace it, as for
+     * images or mmaps with file handles open we must rename before deleting.
+     */
+    /* FIXME: should we use full 64-bit TSC instead of pseudo-random 32-bit?
+     * FIXME: if we make name w/ full path too long we'll truncate:
+     * could cache dir handle and use relative name only.
+     */
+    /* update aslr_get_unique_wide_name() with any improvements here */
+    size_t timestamp = get_random_offset(UINT_MAX);
+    LOG_DECLARE(int trunc =) /* for DEBUG and INTERNAL */
+    snprintf(filename, filename_max,
+             "%s-"IDFMT"-%010"SZFC"-%s", origname,
+             get_process_id(), timestamp, key);
+    ASSERT_CURIOSITY(trunc > 0 && trunc < (int)filename_max &&
+                     "perscache new name truncated");
+    /* FIXME: case 10677 file name truncation */
+
+    filename[filename_max-1] = '\0';
+}
+
+
+/* Merges a given frozen unit with any new persisted cache file on disk.
+ * Caller must hold read lock hotp_get_lock(), if -hot_patching.
+ * If merge is successful, returns a new coarse_info_t, which caller is
+ * responsible for freeing; else returns NULL.
+ */
+static coarse_info_t *
+coarse_unit_merge_with_disk(dcontext_t *dcontext, coarse_info_t *info,
+                            const char *filename)
+{
+    coarse_info_t *merge_with, *postmerge = NULL;
+    uint64 file_size;
+    size_t existing_size;
+    /* We may have already merged new code with an inuse persisted unit, so we
+     * check the stored size of that one if info is not itself persisted.
+     * FIXME: we could store the file handle: can we tell if two file handles
+     * refer to the same file?
+     */
+    size_t inuse_size = (info->persisted) ? info->mmap_size :
+        info->persisted_source_mmap_size;
+
+    LOG(THREAD, LOG_CACHE, 2, "coarse_unit_merge_with_disk %s\n", info->module);
+    ASSERT(dynamo_all_threads_synched);
+    ASSERT(info != NULL && info->cache != NULL); /* don't merge empty units */
+    ASSERT(info->frozen);
+#ifdef HOT_PATCHING_INTERFACE
+    ASSERT_OWN_READ_LOCK(DYNAMO_OPTION(hot_patching), hotp_get_lock());
+#endif
+
+    /* Strategy: check current pcache file size (not perfect but good enough):
+     *   if different from source size, or source was not persisted, then
+     *   load in and merge.
+     *   FIXME case 10356: need a better check since can have false positive
+     *   and false negatives by only looking at size.
+     * Could repeat, and could also check again after writing to tmp file but
+     * before renaming. FIXME: should we do those things to reduce the race
+     * window where we lose another process's appended code?
+     */
+    if (!os_get_file_size(filename, &file_size)) {
+        LOG(THREAD, LOG_CACHE, 2, "  no existing file %s to merge with\n", filename);
+        return postmerge;
+    }
+    ASSERT_TRUNCATE(existing_size, size_t, file_size);
+    existing_size = (size_t) file_size;
+    LOG(THREAD, LOG_CACHE, 2, "  size of existing %s is "SZFMT" vs our "SZFMT"\n",
+        filename, existing_size, inuse_size);
+    if (existing_size == 0)
+        return postmerge;
+    /* Merge a non-persisted (and not merged with persisted) file w/ any on-disk file
+     * that has appeared since startup; or, our own, if we abandoned it but stayed
+     * coarse due to a reset or hotp flush.
+     */
+    if ((!info->persisted && info->persisted_source_mmap_size == 0 &&
+         DYNAMO_OPTION(coarse_lone_merge)) ||
+        /* FIXME case 10356: need a better check since can have false positive
+         * and false negatives by only looking at size.
+         */
+        (existing_size != inuse_size && DYNAMO_OPTION(coarse_disk_merge))) {
+        merge_with = coarse_unit_load(dcontext, info->base_pc, info->end_pc,
+                                      false/*not for execution*/);
+        /* We rely on coarse_unit_load to reject incompatible pcaches, whether for
+         * tls, trace support, or other reasons.  We do need to check the region
+         * here.  FIXME: once we support relocs we need to handle appropriately.
+         */
+        if (merge_with != NULL) {
+            LOG(THREAD, LOG_CACHE, 2, "  merging to-be-persisted %s with on-disk %s\n",
+                info->module, filename);
+            /* Case 8640: allow merging with a smaller on-disk file, to avoid
+             * being forever stuck at that size by prior cores with no IAT merging
+             * or other features */
+            if (merge_with->base_pc >= info->base_pc &&
+                merge_with->end_pc <= info->end_pc) {
+                /* If want in-place merging, need to arrange for ibl invalidation:
+                 * case 11057 */
+                postmerge = coarse_unit_merge(dcontext, info, merge_with,
+                                              false/*!in-place*/);
+                ASSERT(postmerge != NULL);
+                /* if NULL info is still guaranteed to be unchanged so carry on */
+                DOSTATS({
+                    if (postmerge == NULL)
+                        STATS_INC(coarse_merge_disk_fail);
+                    else
+                        STATS_INC(coarse_merge_disk);
+                });
+            } else {
+                /* FIXME case 10357: w/o -unsafe_ignore_IAT_writes
+                 * we're going to see a lot
+                 * of this w/ process-shared pcaches.  Should we abort
+                 * the persist at this point to keep the small-region
+                 * file on disk?  Otherwise we're going to keep
+                 * trading w/ the small-region process.
+                 */
+                LOG(THREAD, LOG_CACHE, 2,
+                    "  region mismatch: "PFX"-"PFX" on-disk vs "PFX"-"PFX" live\n",
+                    merge_with->base_pc, merge_with->end_pc,
+                    info->base_pc, info->end_pc);
+                STATS_INC(coarse_merge_disk_mismatch);
+            }
+            coarse_unit_reset_free(dcontext, merge_with, false/*no locks*/,
+                                   true/*need to unlink*/, false/*not in use anyway*/);
+            coarse_unit_free(dcontext, merge_with);
+            merge_with = NULL;
+        } else
+            STATS_INC(coarse_merge_disk_fail);
+    }
+    return postmerge;
+}
+
+/* Calculates information for persisting that we don't need for online-generated
+ * units, such as hotp points and RAC/RCT info.
+ * If just_live is true, ignores any currently in-use persisted info, assuming
+ * it will be merged in in a merge step by an in-use persisted unit.
+ * Caller must hold change_linking_lock, read lock hotp_get_lock(), and info's lock.
+ */
+static void
+coarse_unit_calculate_persist_info(dcontext_t *dcontext, coarse_info_t *info)
+{
+#ifdef HOT_PATCHING_INTERFACE
+    int len;
+#endif
+    /* we need real dcontext for rac_entries_in_region() */
+    ASSERT(dcontext != NULL && dcontext != GLOBAL_DCONTEXT);
+
+    ASSERT_OWN_MUTEX(true, &change_linking_lock);
+#ifdef HOT_PATCHING_INTERFACE
+    ASSERT_OWN_READ_LOCK(DYNAMO_OPTION(hot_patching), hotp_get_lock());
+#endif
+    ASSERT_OWN_MUTEX(true, &info->lock);
+    LOG(THREAD, LOG_CACHE, 1, "coarse_unit_calculate_persist_info %s "PFX"-"PFX"\n",
+        info->module, info->base_pc, info->end_pc);
+    ASSERT(info->frozen && !info->persisted && !info->has_persist_info);
+
+#ifdef RCT_IND_BRANCH
+    ASSERT(info->rct_table == NULL);
+    if ((TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) ||
+         TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump))) &&
+        (DYNAMO_OPTION(persist_rct)
+# if defined(RETURN_AFTER_CALL) && defined(WINDOWS)
+         /* case 8648: we can have RCT entries that come from code patterns
+          * not derivable from relocation entries that we must persist.
+          */
+         || os_module_get_flag(info->base_pc, MODULE_HAS_BORLAND_SEH)
+# endif
+         )) {
+        app_pc limit_start = info->base_pc;
+        app_pc limit_end = info->end_pc;
+        if (DYNAMO_OPTION(persist_rct) && DYNAMO_OPTION(persist_rct_entire)) {
+            limit_start = 0;
+            limit_end = (app_pc) POINTER_MAX;
+        }
+        info->flags |= PERSCACHE_SUPPORT_RCT;
+        /* We don't support pulling out just entries from this sub-module-region */
+        ASSERT(DYNAMO_OPTION(persist_rct_entire));
+        info->flags |= PERSCACHE_ENTIRE_MODULE_RCT;
+        /* Get a copy of the live + persisted tables merged together */
+        info->rct_table = rct_module_table_copy(dcontext, info->base_pc, RCT_RCT,
+                                                limit_start, limit_end);
+    }
+#endif
+#ifdef RETURN_AFTER_CALL
+    if (DYNAMO_OPTION(ret_after_call)) {
+        ASSERT(info->rac_table == NULL);
+        info->flags |= PERSCACHE_SUPPORT_RAC;
+        /* If we don't persist RAC we'll get violations when using! */
+        /* Get a copy of the live + persisted tables merged together,
+         * but only the entries in this region.
+         * To make sure we have entries for call sites in the region, not targets,
+         * we ignore target just over start and include just over end: thus the
+         * +1 on both sides.
+         */
+        info->rac_table = rct_module_table_copy(dcontext, info->base_pc, RCT_RAC,
+                                                info->base_pc+1, info->end_pc+1);
+    }
+#endif
+
+#ifdef HOT_PATCHING_INTERFACE
+    if (DYNAMO_OPTION(hot_patching)) {
+        /* We expect 2 patch points per hotpatch on average; most have 1,
+         * some have up to 5.  So we could hardcode a max length; instead
+         * we have a separate pass to get the size.
+         */
+        /* FIXME: we could include the image entry point to avoid flushing the
+         * whole exe, but that only happens when we inject into secondary thread.
+         */
+        /* FIXME: when merging live with in-use-persisted we don't need to
+         * re-calculate this and can just use the persisted array, since we would
+         * have flushed the unit on any new hotp
+         */
+        ASSERT_OWN_READ_LOCK(DYNAMO_OPTION(hot_patching), hotp_get_lock());
+        info->hotp_ppoint_vec_num = 
+            hotp_num_matched_patch_points(info->base_pc, info->end_pc);
+        if (info->hotp_ppoint_vec_num > 0) {
+            info->hotp_ppoint_vec =
+                HEAP_ARRAY_ALLOC(dcontext, app_rva_t, info->hotp_ppoint_vec_num,
+                                 ACCT_HOT_PATCHING, PROTECTED);
+            len = hotp_get_matched_patch_points(info->base_pc, info->end_pc,
+                                                info->hotp_ppoint_vec,
+                                                info->hotp_ppoint_vec_num);
+            /* Should never have mismatch, as we're holding the hotp lock.
+             * Even if len < 0 the routine still fills up the array. */
+            ASSERT(len == (int)info->hotp_ppoint_vec_num);
+            if (len != (int)info->hotp_ppoint_vec_num) {
+                /* abort writing out hotp points */
+                info->hotp_ppoint_vec_num = 0;
+            }
+            LOG(THREAD, LOG_CACHE, 2, "hotp points for %s "PFX"-"PFX":\n",
+                info->module, info->base_pc, info->end_pc);
+            DODEBUG({
+                    uint i;
+                    for (i = 0; i < info->hotp_ppoint_vec_num; i++)
+                        LOG(THREAD, LOG_CACHE, 2, "\t"PFX"\n", info->hotp_ppoint_vec[i]);
+                });
+        } else
+            ASSERT(info->hotp_ppoint_vec == NULL);
+    } else
+        ASSERT(info->hotp_ppoint_vec == NULL);
+#endif
+
+    info->has_persist_info = true;
+}
+
+/* Merges the persist-calculated fields of info1 and info2 into dst,
+ * as well as non-persist-calculated fields like primary_for_module.
+ * Assumption: we've already checked for compatibility and here we just
+ * need to take the union (minus dups).
+ * Caller must hold info1's lock, and we must be dynamo_all_threads_synched.
+ */
+static void
+coarse_unit_merge_persist_info(dcontext_t *dcontext, coarse_info_t *dst,
+                               coarse_info_t *info1, coarse_info_t *info2)
+{
+    ASSERT(dynamo_all_threads_synched);
+    LOG(THREAD, LOG_CACHE, 1, "coarse_unit_merge_persist_info %s "PFX"-"PFX"\n",
+        info1->module, info1->base_pc, info1->end_pc);
+    /* We can't grab both locks due to deadlock potential.  Currently we are
+     * always fully synched, so we rely on that.
+     */
+    ASSERT(dynamo_all_threads_synched);
+    ASSERT_OWN_MUTEX(true, &info1->lock);
+
+    /* We need to incorporate flags from persisted units (Borland, e.g.) */
+    dst->flags |= info1->flags;
+    dst->flags |= info2->flags;
+    /* Some flags are intersection rather than union (but those like
+     * PERSCACHE_SUPPORT_TRACES are weeded out at load time and should
+     * not differ here)
+     */
+    if (!TEST(PERSCACHE_MAP_RW_SEPARATE, info1->flags) ||
+        !TEST(PERSCACHE_MAP_RW_SEPARATE, info2->flags))
+        dst->flags &= ~PERSCACHE_MAP_RW_SEPARATE;
+    /* Same bounds, so same persistence privileges */
+    dst->primary_for_module = info1->primary_for_module || info2->primary_for_module;
+
+    ASSERT(!info2->persisted || !info2->in_use ||
+           info2->has_persist_info); /* must have to use */
+
+    /* Everything else is only necessary if we've already calculated persist
+     * info; if we haven't, then if we do persist later we'll calculate it
+     * fresh, so no need to calculate and merge here
+     */
+    if (!info1->has_persist_info && !info2->has_persist_info)
+        return;
+
+    if (!info1->has_persist_info)
+        coarse_unit_calculate_persist_info(dcontext, info1);
+    if (!info2->has_persist_info)
+        coarse_unit_calculate_persist_info(dcontext, info2);
+    ASSERT(info1->has_persist_info && info2->has_persist_info);
+    ASSERT(!dst->has_persist_info);
+
+    ASSERT((info1->flags &
+            (PERSCACHE_SUPPORT_TRACES|PERSCACHE_SUPPORT_RAC|PERSCACHE_SUPPORT_RCT)) ==
+           (info2->flags &
+            (PERSCACHE_SUPPORT_TRACES|PERSCACHE_SUPPORT_RAC|PERSCACHE_SUPPORT_RCT)));
+
+#ifdef RCT_IND_BRANCH
+    ASSERT(dst->rct_table == NULL);
+    if (TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) ||
+        TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump))) {
+        if (info2->persisted && info2->in_use && !info1->persisted) {
+            /* coarse_unit_calculate_persist_info already merged the new entries
+             * with the in-use persisted entries
+             */
+            dst->rct_table = rct_table_copy(dcontext, info1->rct_table);
+        } else {
+            /* Note that we can't simply take one source if it has
+             * PERSCACHE_ENTIRE_MODULE_RCT as Borland can add entries: we must
+             * merge every time
+             */
+            dst->rct_table = rct_table_merge(dcontext, info1->rct_table,
+                                             info2->rct_table);
+        }
+    }
+#endif
+#ifdef RETURN_AFTER_CALL
+    ASSERT(dst->rac_table == NULL);
+    if (DYNAMO_OPTION(ret_after_call)) {
+        if (info2->persisted && info2->in_use && !info1->persisted) {
+            /* coarse_unit_calculate_persist_info already merged the new entries
+             * with the in-use persisted entries
+             */
+            dst->rac_table = rct_table_copy(dcontext, info1->rac_table);
+        } else {
+            dst->rac_table = rct_table_merge(dcontext, info1->rac_table,
+                                             info2->rac_table);
+        }
+    }
+#endif
+
+#ifdef HOT_PATCHING_INTERFACE
+    ASSERT(dst->hotp_ppoint_vec == NULL);
+    if (info2->persisted && info2->in_use && !info1->persisted) {
+        /* We would have flushed info2 if a new hotp overlapped it so just
+         * take its hotp list.  info1 may have a subset.
+         */
+        dst->hotp_ppoint_vec_num = info2->hotp_ppoint_vec_num;
+        if (dst->hotp_ppoint_vec_num > 0) {
+            dst->hotp_ppoint_vec =
+                HEAP_ARRAY_ALLOC(dcontext, app_rva_t, dst->hotp_ppoint_vec_num,
+                                 ACCT_HOT_PATCHING, PROTECTED);
+            memcpy(dst->hotp_ppoint_vec, info2->hotp_ppoint_vec,
+                   dst->hotp_ppoint_vec_num*sizeof(app_rva_t));
+        }
+    } else {        
+        /* We expect <= 5 entries in each array so quadratic and two passes
+         * shouldn't be a perf issue.
+         */
+        ASSERT(dst->hotp_ppoint_vec_num == 0);
+        ASSERT(sizeof(app_rva_t) == sizeof(void *));
+        array_merge(dcontext, true /* intersect */,
+                    (void **) info1->hotp_ppoint_vec, info1->hotp_ppoint_vec_num,
+                    (void **) info2->hotp_ppoint_vec, info2->hotp_ppoint_vec_num,
+                    (void ***) &dst->hotp_ppoint_vec, &dst->hotp_ppoint_vec_num
+                    HEAPACCT(ACCT_HOT_PATCHING));
+    }
+#endif
+    dst->has_persist_info = true;
+}
+
+static bool
+write_persist_file(dcontext_t *dcontext, file_t fd, const void *buf, size_t count)
+{
+    ASSERT(fd != INVALID_FILE && buf != NULL && count > 0);
+    if (os_write(fd, buf, count) != (ssize_t)count) {
+        LOG(THREAD, LOG_CACHE, 1, "  unable to write %d bytes to file\n", count);
+        SYSLOG_INTERNAL_WARNING_ONCE("unable to write persistent cache file");
+        STATS_INC(coarse_units_persist_error);
+        return false;
+    }
+    return true;
+}
+
+static bool
+pad_persist_file(dcontext_t *dcontext, file_t fd, size_t bytes, coarse_info_t *info)
+{
+    size_t towrite = bytes;
+    size_t thiswrite;
+    /* We don't have a handy page of zeros so we just write the cache start.
+     * FIXME: better to use file positions (just setting the size doesn't
+     * advance the pointer): write_file() has win32 support.
+     */
+    ASSERT(fd != INVALID_FILE);
+    while (towrite > 0) {
+        thiswrite = MIN(towrite, (ptr_uint_t)info->stubs_end_pc -
+                        (ptr_uint_t)info->cache_start_pc);
+        if (!write_persist_file(dcontext, fd, info->cache_start_pc, thiswrite))
+            return false;
+        towrite -= thiswrite;
+    }
+    return true;
+}
+
+/* Fills in pers with data from info */
+static void
+coarse_unit_set_persist_data(dcontext_t *dcontext, coarse_info_t *info,
+                             coarse_persisted_info_t *pers, app_pc modbase,
+                             op_pcache_t option_level, const char *option_string)
+{
+    size_t x_offs = 0; /* offs of start of +x section */
+
+    /* We do not need to freeze the world: merely the state of outgoing links. */
+    ASSERT_OWN_MUTEX(true, &change_linking_lock);
+#ifdef HOT_PATCHING_INTERFACE
+    ASSERT_OWN_READ_LOCK(DYNAMO_OPTION(hot_patching), hotp_get_lock());
+#endif
+    ASSERT_OWN_MUTEX(true, &info->lock);
+
+    /* Put the unit in the proper state for persisting */
+    coarse_unit_unlink_outgoing(dcontext, info);
+
+    /* Fill in the metadata */
+    pers->magic = PERSISTENT_CACHE_MAGIC;
+    pers->version = PERSISTENT_CACHE_VERSION;
+    pers->header_len = sizeof(*pers);
+    pers->data_len = 0;
+
+    /* Take flags from info */
+    pers->flags = info->flags;
+    pers->flags |= PERSCACHE_X86_32; /* FIXME: see above: should be set earlier */
+    if (DYNAMO_OPTION(coarse_freeze_elide_ubr))
+        pers->flags |= PERSCACHE_ELIDED_UBR;
+#if defined(RETURN_AFTER_CALL) && defined(WINDOWS)
+    if (seen_Borland_SEH)
+        pers->flags |= PERSCACHE_SEEN_BORLAND_SEH;
+#endif
+    if (!DYNAMO_OPTION(disable_traces))
+        pers->flags |= PERSCACHE_SUPPORT_TRACES;
+    if (option_level == OP_PCACHE_LOCAL) {
+        ASSERT(option_string != NULL);
+        pers->flags |= PERSCACHE_EXEMPTION_OPTIONS;
+    }
+   
+#ifdef BUILD_NUMBER
+    pers->build_number = BUILD_NUMBER;
+#else
+    pers->build_number = 0;
+#endif
+
+    if (TEST(PERSCACHE_MODULE_MD5_AT_LOAD, DYNAMO_OPTION(persist_gen_validation))) {
+        ASSERT(!is_region_memset_to_char((byte *)&info->module_md5,
+                                         sizeof(info->module_md5), 0));
+        DOLOG(1, LOG_CACHE, {
+            print_module_digest(THREAD, &info->module_md5, "using md5 from load time: ");
+        });
+        memcpy(&pers->modinfo.module_md5, &info->module_md5,
+               sizeof(pers->modinfo.module_md5));
+    } else {
+        persist_calculate_module_digest(&pers->modinfo.module_md5, modbase,
+                                        (size_t) pers->modinfo.image_size,
+                                        info->base_pc, info->end_pc,
+                                        DYNAMO_OPTION(persist_gen_validation));
+    }
+    /* rest of modinfo filled in by get_persist_filename() */
+    ASSERT(pers->modinfo.base == modbase);
+
+    ASSERT(info->base_pc > modbase);
+    ASSERT(info->end_pc > info->base_pc);
+    pers->start_offs = info->base_pc - modbase;
+    pers->end_offs = info->end_pc - modbase;
+    pers->tls_offs_base = os_tls_offset(0);
+
+    /* We always write the footer regardless of whether we calculate the
+     * MD5.  Otherwise we'd need a new flag; not really worth it.
+     * Plus, we have our magic field as an always-on extra check.
+     * Note that we do not set the total file size up front (which could make
+     * our writes faster), so we know if the footer is there then it was
+     * actually written.
+     */
+    pers->data_len += sizeof(persisted_footer_t);
+
+    pers->stubs_len = info->stubs_end_pc - info->stubs_start_pc;
+    pers->data_len += pers->stubs_len;
+    pers->ibl_jmp_prefix_len = info->stubs_start_pc - info->ibl_jmp_prefix;
+    pers->data_len += pers->ibl_jmp_prefix_len;
+    pers->ibl_call_prefix_len = info->ibl_jmp_prefix - info->ibl_call_prefix;
+    pers->data_len += pers->ibl_call_prefix_len;
+    pers->ibl_ret_prefix_len = info->ibl_call_prefix - info->ibl_ret_prefix;
+    pers->data_len += pers->ibl_ret_prefix_len;
+    pers->trace_head_return_prefix_len =
+        info->ibl_ret_prefix - info->trace_head_return_prefix;
+    pers->data_len += pers->trace_head_return_prefix_len;
+    pers->fcache_return_prefix_len =
+        info->trace_head_return_prefix - info->fcache_return_prefix;
+    pers->data_len += pers->fcache_return_prefix_len;
+    pers->cache_len = info->fcache_return_prefix - info->cache_start_pc;
+    pers->post_cache_pad_len = info->fcache_return_prefix - info->cache_end_pc;
+    pers->data_len += pers->cache_len; /* includes post_cache_pad_len */
+    STATS_ADD(coarse_code_persisted, info->cache_end_pc - info->cache_start_pc);
+
+    pers->stub_htable_len =
+        fragment_coarse_htable_persist_size(dcontext, info, false/*stub table*/);
+    x_offs += pers->stub_htable_len;
+    pers->cache_htable_len =
+        fragment_coarse_htable_persist_size(dcontext, info, true/*cache table*/);
+    x_offs += pers->cache_htable_len;
+
+#ifdef RCT_IND_BRANCH
+    pers->rct_htable_len = rct_table_persist_size(dcontext, info->rct_table);
+#else
+    pers->rct_htable_len = 0;
+#endif
+    x_offs += pers->rct_htable_len;
+#ifdef RETURN_AFTER_CALL
+    pers->rac_htable_len = rct_table_persist_size(dcontext, info->rac_table);
+#else
+    pers->rac_htable_len = 0;
+#endif
+    x_offs += pers->rac_htable_len;
+
+    /* FIXME case 9581 NYI: need to add to coarse_unit_calculate_persist_info()
+     * and coarse_unit_merge_persist_info() as well.
+     */
+    pers->reloc_len = 0;
+    x_offs += pers->reloc_len;
+
+#ifdef HOT_PATCHING_INTERFACE
+    pers->hotp_patch_list_len = sizeof(app_rva_t) * info->hotp_ppoint_vec_num;
+    x_offs += pers->hotp_patch_list_len;
+#endif
+
+    /* Add new data section here */
+
+    pers->option_string_len = option_string == NULL ? 0 :
+        (ALIGN_FORWARD((strlen(option_string)+1/*include NULL*/)*sizeof(char),
+                       OPTION_STRING_ALIGNMENT));
+    x_offs += pers->option_string_len;
+
+    /* We need to pad out the data before the +x boundary to get a new page */
+    pers->data_len += x_offs;
+    x_offs += pers->header_len;
+    pers->pad_len = ALIGN_FORWARD(x_offs, PAGE_SIZE) - x_offs;
+    if (DYNAMO_OPTION(persist_map_rw_separate)) {
+        pers->flags |= PERSCACHE_MAP_RW_SEPARATE;
+        /* We need the cache/stub split to be on a file view map boundary,
+         * yet have cache+stubs adjacent
+         */
+        pers->pad_len += ALIGN_FORWARD(x_offs + pers->pad_len + pers->cache_len,
+                                      MAP_FILE_VIEW_ALIGNMENT)
+            - (x_offs + pers->pad_len + pers->cache_len);
+    }
+    pers->data_len += pers->pad_len;
+    /* Our relative jmps require that we do not exceed 32-bit reachability */
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(pers->data_len)));
+    LOG(THREAD, LOG_CACHE, 2, "  header=%d, data=%d, pad=%d\n",
+        pers->header_len, pers->data_len, pers->pad_len);
+}
+
+/* Separated from coarse_unit_persist() to keep the stack space on the
+ *   persist -> merge_with_disk -> load
+ * path to a minimum (case 10712).
+ */
+static bool
+coarse_unit_persist_rename(dcontext_t *dcontext, const char *filename,
+                           const char *tmpname)
+{
+    bool success = false;
+    char rename[MAXIMUM_PATH];
+    if (os_rename_file(tmpname, filename, false/*do not replace*/)) {
+        success = true;
+    } else {
+        ASSERT_CURIOSITY(os_file_exists(filename, false/*!dir*/));
+        if (DYNAMO_OPTION(coarse_freeze_rename)) {
+            get_unique_name(filename, "todel", rename, BUFFER_SIZE_ELEMENTS(rename));
+            LOG(THREAD, LOG_CACHE, 1, "  attempting to rename %s to %s\n",
+                filename, rename);
+            if (os_rename_file(filename, rename, false/*do not replace*/)) {
+                LOG(THREAD, LOG_CACHE, 1, "  succeeded renaming %s to %s\n",
+                    filename, rename);
+                STATS_INC(persist_rename_success);
+                if (DYNAMO_OPTION(coarse_freeze_clean)) {
+                    /* Try to mark the renamed file for deletion.  We have no handle
+                     * open and are re-targeting based on name only but should have a
+                     * unique name so we aren't worried about deleting the wrong file.
+                     */
+                    if (os_delete_mapped_file(rename)) {
+                        LOG(THREAD, LOG_CACHE, 1,
+                            "  succeeded marking for deletion %s\n", rename);
+                        STATS_INC(persist_delete_success);
+                    }
+                }
+                /* We could support os_delete_mapped_file() w/o renaming first but it
+                 * only works for SEC_COMMIT with -no_persist_lock_file */
+                if (os_rename_file(tmpname, filename, false/*do not replace*/))
+                    success = true;
+                else {
+                    /* Race: someone beat us to the rename (or else no permissions).
+                     * FIXME: should we try again? */
+                    ASSERT_CURIOSITY(os_file_exists(filename, false/*!dir*/));
+                    STATS_INC(persist_rename_race);
+                    ASSERT(!success);
+                }
+            } else {
+                STATS_INC(persist_rename_fail);
+                ASSERT(!success);
+            }
+        }
+    }
+    return success;
+}
+
+/* Unlinks all inter-unit stubs.  Can still use info afterward, as
+ * lazy linking should soon re-link them.
+ * Caller must hold change_linking_lock and read lock hotp_get_lock().
+ */
+bool
+coarse_unit_persist(dcontext_t *dcontext, coarse_info_t *info)
+{
+    coarse_persisted_info_t pers;
+    persisted_footer_t footer;
+    /* FIXME: this is a lot of stack space: make static and serialize? */
+    char filename[MAXIMUM_PATH];
+    char tmpname[MAXIMUM_PATH];
+    char option_buf[MAX_PCACHE_OPTIONS_STRING];
+    const char *option_string;
+    op_pcache_t option_level;
+    app_pc modbase;
+    bool success = false;
+    bool created_temp = false;
+    bool free_info = false;
+    file_t fd = INVALID_FILE;
+    /* we need real dcontext for rac_entries_in_region() */
+    ASSERT(dcontext != NULL && dcontext != GLOBAL_DCONTEXT);
+
+    KSTART(persisted_generation);
+    LOG(THREAD, LOG_CACHE, 1, "coarse_unit_persist %s "PFX"-"PFX"\n",
+        info->module, info->base_pc, info->end_pc);
+    STATS_INC(coarse_units_persist_try);
+    ASSERT(info->frozen && !info->persisted);
+
+    /* invalid unit shouldn't get this far */
+    ASSERT(!TEST(PERSCACHE_CODE_INVALID, info->flags));
+    if (TEST(PERSCACHE_CODE_INVALID, info->flags)) /* paranoid */
+        goto coarse_unit_persist_exit;
+
+    modbase = get_module_base(info->base_pc);
+    if (modbase == NULL) {
+        LOG(THREAD, LOG_CACHE, 1, "  no module base for "PFX"\n", info->base_pc);
+        goto coarse_unit_persist_exit;
+    }
+    /* case 9653/10380: only one coarse unit in a module's +x region(s) is persisted */
+    if (!info->primary_for_module) {
+        /* the primary may have been flushed, so see if we can become primary */
+        coarse_unit_mark_primary(info);
+        if (!info->primary_for_module) {
+            LOG(THREAD, LOG_CACHE, 1,
+                "  not primary unit for module %s: not persisting\n", info->module);
+            STATS_INC(coarse_units_persist_dup);
+            goto coarse_unit_persist_exit;
+        }
+    }
+    /* case 9799: store pcache-affecting options */
+    option_level = persist_get_options_level(modbase, info, false/*use exemptions*/);
+    LOG(THREAD, LOG_CACHE, 2, "  persisting option string at %d level\n", option_level);
+    option_string = persist_get_relevant_options(dcontext, option_buf,
+                                                 BUFFER_SIZE_ELEMENTS(option_buf),
+                                                 option_level);
+    /* get_persist_filename() fills in pers.modinfo */
+    memset(&pers, 0, sizeof(pers));
+    if (!get_persist_filename(filename, BUFFER_SIZE_ELEMENTS(filename), modbase,
+                              true/*write*/, &pers.modinfo, option_string)) {
+        LOG(THREAD, LOG_CACHE, 1, "  error calculating filename (or excluded)\n");
+        STATS_INC(coarse_units_persist_error);
+        goto coarse_unit_persist_exit;
+    }
+    LOG(THREAD, LOG_CACHE, 1, "  persisted filename = %s\n", filename);
+    /* Open the file before unlinking since may be common point of failure.
+     * FIXME: better to cache base dir handle and pass just name here
+     * and to os_{rename,delete_mapped}_file?  Would be win32-specific code.
+     */
+    if (!DYNAMO_OPTION(coarse_freeze_rename) &&
+        !DYNAMO_OPTION(coarse_freeze_clobber) &&
+        os_file_exists(filename, false/*FIXME: would like to check all*/)) {
+        /* Check up front to avoid work */
+        LOG(THREAD, LOG_CACHE, 1, "  will be unable to replace existing file %s\n",
+            filename);
+        STATS_INC(coarse_units_persist_error);
+        goto coarse_unit_persist_exit;
+    }
+
+    get_unique_name(filename, "tmp", tmpname, BUFFER_SIZE_ELEMENTS(tmpname));
+    fd = os_open(tmpname, OS_OPEN_WRITE |
+                 /* We ask for read access to map it later for md5 */
+                 OS_OPEN_READ |
+                 /* Case 9964: we want to allow renaming while holding
+                  * the file handle */
+                 OS_SHARE_DELETE |
+                 /* Renaming is the better option; and for SEC_COMMIT
+                  * we can delete immediately, w/o the symlink risk case 9057;
+                  * so we leave the CURIOSITY we get w/ the clobber option.
+                  */
+                 (DYNAMO_OPTION(coarse_freeze_clobber) ? 0 : OS_OPEN_REQUIRE_NEW) |
+                 /* case 10884: needed only for validate_owner_file */
+                 OS_OPEN_FORCE_OWNER);
+    if (fd == INVALID_FILE) {
+        /* FIXME: it's possible but unlikely that our name collided: should we
+         * try again?
+         */
+        LOG(THREAD, LOG_CACHE, 1, "  unable to open temp file %s\n", tmpname);
+        STATS_INC(coarse_units_persist_error);
+        goto coarse_unit_persist_exit;
+    }
+    ASSERT_CURIOSITY(os_validate_user_owned(fd) && "persisted while impersonating?");
+
+    created_temp = true;
+    /* FIXME case 9758: we could mmap the file: would that be more efficient?
+     * We could then more easily write each body section at the same
+     * time as we calculate its size for the header, instead of linear
+     * write order.  We could also pass file offsets to out-of-order writes.
+     * Could we control write-through if we mmap (case 9696)?
+     * Note the prescribed order of unmap and then flush in case 9696.
+     */
+
+    mutex_lock(&info->lock);
+    /* Get info that is not needed for online coarse units but is needed at persist
+     * time (RCT, RAC, and hotp info) now, so we can merge it more easily.
+     */
+    if (!info->has_persist_info)
+        coarse_unit_calculate_persist_info(dcontext, info);
+    ASSERT(info->has_persist_info);
+    /* FIXME: the lock situation in all these routines is ugly: here we must
+     * release since coarse_unit_merge() will acquire.  Should we not grab any
+     * info lock?  We're already relying on dynamo_all_threads_synched.
+     */
+    mutex_unlock(&info->lock);
+
+    /* Attempt to merge with any new on-disk file */
+    if (DYNAMO_OPTION(coarse_lone_merge) || DYNAMO_OPTION(coarse_disk_merge)) {
+        coarse_info_t *postmerge = coarse_unit_merge_with_disk(dcontext, info, filename);
+        if (postmerge != NULL) {
+            /* Just abandon the original info */
+            info = postmerge;
+            free_info = true;
+        }
+    }
+
+    mutex_lock(&info->lock);
+
+    /* Fill in the rest of pers */
+    coarse_unit_set_persist_data(dcontext, info, &pers, modbase,
+                                 option_level, option_string);
+
+    if (!coarse_unit_check_persist_space(fd, pers.header_len+pers.data_len)) {
+        /* We also check this prior to freezing/merging to avoid work on a
+         * perpetually-full volume */
+        LOG(THREAD, LOG_CACHE, 1,
+            "  not enough disk space available for module %s: not persisting\n",
+            info->module);
+        STATS_INC(coarse_units_persist_nospace);
+        goto coarse_unit_persist_exit;
+    }
+
+    /* Write the headers */
+    if (!write_persist_file(dcontext, fd, &pers, pers.header_len))
+        goto coarse_unit_persist_exit; /* logs, stats are in write_persist_file */
+
+    if (pers.option_string_len > 0) {
+        size_t len = strlen(option_string) + 1/*NULL*/;
+        ASSERT(len <= pers.option_string_len);
+        if (!write_persist_file(dcontext, fd, option_string, len))
+            goto coarse_unit_persist_exit; /* logs, stats are in write_persist_file */
+        if (pers.option_string_len - len > 0) {
+            if (!pad_persist_file(dcontext, fd, pers.option_string_len - len, info))
+                goto coarse_unit_persist_exit; /* logs, stats in write_persist_file */
+        }
+    }
+
+    /* New data section goes here */
+
+#ifdef HOT_PATCHING_INTERFACE
+    if (pers.hotp_patch_list_len > 0) {
+        if (!write_persist_file(dcontext, fd, info->hotp_ppoint_vec,
+                                pers.hotp_patch_list_len))
+            goto coarse_unit_persist_exit; /* logs, stats are in write_persist_file */
+    }
+#endif
+
+    /* FIXME case 9581 NYI: write reloc section */
+
+#ifdef RETURN_AFTER_CALL
+    if (pers.rac_htable_len > 0) {
+        if (!rct_table_persist(dcontext, info->rac_table, fd)) {
+            LOG(THREAD, LOG_CACHE, 1, "  unable to write RAC htable to file\n");
+            SYSLOG_INTERNAL_WARNING_ONCE("unable to write RAC htable to pcache file");
+            STATS_INC(coarse_units_persist_error);
+            goto coarse_unit_persist_exit;
+        }
+    }
+#else
+    ASSERT(pers.rac_htable_len == 0);
+#endif
+#ifdef RCT_IND_BRANCH
+    if (pers.rct_htable_len > 0) {
+        if (!rct_table_persist(dcontext, info->rct_table, fd)) {
+            LOG(THREAD, LOG_CACHE, 1, "  unable to write RCT htable to file\n");
+            SYSLOG_INTERNAL_WARNING_ONCE("unable to write RCT htable to pcache file");
+            STATS_INC(coarse_units_persist_error);
+            goto coarse_unit_persist_exit;
+        }
+    }
+#else
+    ASSERT(pers.rct_htable_len == 0);
+#endif
+
+    /* For modularity we split out htable persistence.  We could combine the
+     * tables with our cache+stubs mmap region at freeze time but there's no
+     * need as there are no cross-links so we keep them separate.
+     */
+    if (!fragment_coarse_htable_persist(dcontext, info, true/*cache table*/, fd) ||
+        !fragment_coarse_htable_persist(dcontext, info, false/*stub table*/, fd)) {
+        LOG(THREAD, LOG_CACHE, 1, "  unable to write htable(s) to file\n");
+        SYSLOG_INTERNAL_WARNING_ONCE("unable to write htable(s) to pcache file");
+        STATS_INC(coarse_units_persist_error);
+        goto coarse_unit_persist_exit;
+    }
+
+    /* If we used an image format, or did separate mappings, we could avoid
+     * storing padding on disk
+     */
+    if (pers.pad_len > 0) {
+        if (!pad_persist_file(dcontext, fd, pers.pad_len, info))
+            goto coarse_unit_persist_exit; /* logs, stats in write_persist_file */
+    }
+
+    /* Write the cache + stubs.  We omit the guard pages of course. 
+     * FIXME: when freezing and persisting at once, more efficient to
+     * make our cache+stubs mmap backed by the file from the start.
+     */
+    if (!write_persist_file(dcontext, fd, info->cache_start_pc,
+                            info->stubs_end_pc - info->cache_start_pc))
+        goto coarse_unit_persist_exit; /* logs, stats are in write_persist_file */
+
+    if (TESTANY(PERSCACHE_GENFILE_MD5_SHORT|PERSCACHE_GENFILE_MD5_COMPLETE,
+                DYNAMO_OPTION(persist_gen_validation))) {
+        byte *map = (byte *) &pers;
+        uint which = DYNAMO_OPTION(persist_gen_validation);
+        if (TEST(PERSCACHE_GENFILE_MD5_COMPLETE,
+                 DYNAMO_OPTION(persist_gen_validation))) {
+            /* FIXME if mmap up front (case 9758) don't need this mmap */
+            map = map_file(fd, pers.header_len+pers.data_len-sizeof(persisted_footer_t),
+                           0, NULL, MEMPROT_READ,
+                           false/*won't change so save pagefile by not asking for COW*/);
+            ASSERT(map != NULL);
+            if (map == NULL) {
+                /* give up */
+                which &= ~PERSCACHE_GENFILE_MD5_COMPLETE;
+            }
+        }
+        persist_calculate_self_digest(&footer.self_md5, &pers, map, which);
+        DOLOG(1, LOG_CACHE, {
+            print_module_digest(THREAD, &footer.self_md5, "self md5: ");
+        });
+        if (TEST(PERSCACHE_GENFILE_MD5_COMPLETE,
+                 DYNAMO_OPTION(persist_gen_validation)) &&
+            map != (byte *) &pers) {
+            ASSERT(map != NULL); /* we cleared _COMPLETE so shouldn't get here */
+            unmap_file(map,  pers.header_len+pers.data_len);
+        }
+    } else {
+        memset(&footer, 0, sizeof(footer));
+    }
+    footer.magic = PERSISTENT_CACHE_MAGIC;
+    if (!write_persist_file(dcontext, fd, &footer, sizeof(footer)))
+        goto coarse_unit_persist_exit; /* logs, stats are in write_persist_file */
+
+    os_flush(fd);
+    /* Temp file is all flushed and ready to go.  Only now do we try to replace
+     * the original, to ensure we have a complete file (case 9696).
+     * We can't rename it if we have a handle with write privileges open,
+     * regardless of FILE_SHARE_* flags, so we close it and live with races.
+     * Nobody else should be creating a file of the same name (tmpname includes process id).
+     * Therefore no other process may have racily created an
+     * incomplete file overwriting the file we just produced and are
+     * about to rename.
+     */
+    os_close(fd);
+    fd = INVALID_FILE;
+
+    success = coarse_unit_persist_rename(dcontext, filename, tmpname);
+    if (!success) {
+        LOG(THREAD, LOG_CACHE, 1, "  unable to rename to %s\n", filename);
+        STATS_INC(persist_rename_tmp_fail);
+        goto coarse_unit_persist_exit;
+    }
+    /* success */
+    STATS_INC(coarse_units_persist);
+    ASSERT(success);
+
+ coarse_unit_persist_exit:
+    if (fd != INVALID_FILE)
+        os_close(fd);
+    if (!success && created_temp) {
+        if (!os_delete_mapped_file(tmpname)) {
+            LOG(THREAD, LOG_CACHE, 1,
+                "  failed to delete on failure temp %s\n", tmpname);
+            STATS_INC(persist_delete_tmp_fail);
+       }
+    }
+    if (created_temp)
+        mutex_unlock(&info->lock);
+    if (free_info) {
+        /* if we dup-ed when merging with disk, free now */
+        coarse_unit_reset_free(dcontext, info, false/*no locks*/,
+                               false/*no unlink*/, false/*not in use anyway*/);
+        coarse_unit_free(dcontext, info);
+    }
+    KSTOP(persisted_generation);
+    return success;
+}
+
+static bool
+persist_check_option_compat(dcontext_t *dcontext, coarse_persisted_info_t *pers,
+                            const char *option_string)
+{
+    const char *pers_options;
+
+    if (os_tls_offset(0) != pers->tls_offs_base) {
+        /* Bail out 
+         * WARNING: if we ever change our -tls_align default from 1 we should
+         * consider implications on platform-independence of persisted caches.
+         * FIXME: should we have release-build notification so we can know
+         * when we have problems here?
+         */
+        LOG(THREAD, LOG_CACHE, 1, "  tls offset mismatch %d vs persisted %d\n",
+            os_tls_offset(0), pers->tls_offs_base);
+        STATS_INC(perscache_tls_mismatch);
+        SYSLOG_INTERNAL_WARNING_ONCE("persistent cache tls offset mismatch");
+        return false;
+    }
+    
+    if (!TEST(PERSCACHE_SUPPORT_TRACES, pers->flags) && !DYNAMO_OPTION(disable_traces)) {
+        /* We bail out; the consequences of continuing are huge performance
+         * problems akin to -no_link_ibl.
+         */
+        LOG(THREAD, LOG_CACHE, 1, "  error: persisted cache has no trace support\n");
+        STATS_INC(perscache_trace_mismatch);
+        SYSLOG_INTERNAL_WARNING_ONCE("persistent cache trace support mismatch");
+        return false;
+    }
+
+    /* we don't bother to split the ifdefs */
+#if defined(RCT_IND_BRANCH) || defined (RETURN_AFTER_CALL)
+    if ((!TEST(PERSCACHE_SUPPORT_RAC, pers->flags) &&
+         DYNAMO_OPTION(ret_after_call)) ||
+        (!TEST(PERSCACHE_SUPPORT_RCT, pers->flags) &&
+         (TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) ||
+          TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump))))) {
+        LOG(THREAD, LOG_CACHE, 1, "  error: persisted cache has no RAC/RCT support\n");
+        STATS_INC(perscache_rct_mismatch);
+        SYSLOG_INTERNAL_WARNING_ONCE("persistent cache RAC/RCT support mismatch");
+        return false;
+    }
+#endif
+
+    if (DYNAMO_OPTION(persist_check_options)) {
+        /* Option string is 1st data section; we persisted a NULL so we can
+         * point directly at it. */
+        pers_options = (const char *) (((byte *)pers) + pers->header_len);
+        LOG(THREAD, LOG_CACHE, 2, "  checking pcache options \"%s\" vs current \"%s\"\n",
+            pers_options, option_string);
+        if (strcmp(option_string, pers_options) != 0) {
+            /* Incompatible options!  In many cases the pcache may be safe to
+             * execute but we play it conservatively and don't delve deeper (case 9799).
+             * FIXME: put local options in name?!?
+             */
+            LOG(THREAD, LOG_CACHE, 1, "  error: options mismatch \"%s\" vs \"%s\"\n",
+                pers_options, option_string);
+            STATS_INC(perscache_options_mismatch);
+            SYSLOG_INTERNAL_WARNING_ONCE("persistent cache options mismatch");
+            return false;
+        }
+    }
+
+    return true;
+}
+
+#ifdef DEBUG
+/* case 10712: this would be a lot of stack space so we use static buffers
+ * to avoid stack overflow
+ */
+DECLARE_NEVERPROT_VAR(static char pcache_dir_check_root[MAXIMUM_PATH], {0});
+DECLARE_NEVERPROT_VAR(static char pcache_dir_check_temp[MAXIMUM_PATH], {0});
+
+static void
+pcache_dir_check_permissions(dcontext_t *dcontext, const char *filename)
+{
+    /* Test that we cannot rename per-user directory before
+     * opening the file.  Prevented by sharing/in use
+     * restrictions, not ACLs.  xref case 10504
+     */
+    /* `dirname filename` */
+    const char *file_parent = double_strrchr(filename, DIRSEP, ALT_DIRSEP);
+    size_t per_user_len;
+    mutex_lock(&pcache_dir_check_lock);
+    if (DYNAMO_OPTION(persist_per_app)) {
+        /* back up one level higher */
+        /* note that we only need to check whether the per-user directory
+         * cannot be renamed by an attacker, while the per-user\per-app
+         * directory should be created with correct ACLs
+         */
+        snprintf(pcache_dir_check_temp, BUFFER_SIZE_ELEMENTS(pcache_dir_check_temp),
+                 "%.*s", file_parent - filename, filename);
+        /* `dirname pcache_dir_check_temp` */
+        per_user_len = 
+            double_strrchr(pcache_dir_check_temp, DIRSEP, ALT_DIRSEP) - 
+            pcache_dir_check_temp;
+    } else {
+        per_user_len = file_parent - filename;
+    }
+    snprintf(pcache_dir_check_root, BUFFER_SIZE_ELEMENTS(pcache_dir_check_root), "%.*s", 
+             per_user_len, filename);
+    snprintf(pcache_dir_check_temp, BUFFER_SIZE_ELEMENTS(pcache_dir_check_temp),
+             "%s-bumped", pcache_dir_check_root);
+    LOG(THREAD, LOG_CACHE, 3, "  attempting rename %s -> %s\n", 
+        pcache_dir_check_root, pcache_dir_check_temp);
+    ASSERT(!os_rename_file(pcache_dir_check_root, pcache_dir_check_temp,
+                           false) && 
+           "directory can be bumped!");
+    mutex_unlock(&pcache_dir_check_lock);
+}
+#endif
+
+/* It's up to the caller to do the work of mark_executable_area_coarse_frozen().
+ * Caller must hold read lock hotp_get_lock().
+ */
+coarse_info_t *
+coarse_unit_load(dcontext_t *dcontext, app_pc start, app_pc end,
+                 bool for_execution)
+{
+    coarse_persisted_info_t *pers;
+    persisted_footer_t *footer;
+    coarse_info_t *info = NULL;
+    char filename[MAXIMUM_PATH];
+    char option_buf[MAX_PCACHE_OPTIONS_STRING];
+    const char *option_string;
+    op_pcache_t option_level;
+    file_t fd = INVALID_FILE;
+    byte *map = NULL, *map2 = NULL;
+    size_t map_size = 0, map2_size = 0;
+    uint64 file_size;
+    size_t stubs_and_prefixes_len;
+    byte *pc, *rx_pc, *rwx_pc;
+    persisted_module_info_t modinfo;
+    app_pc modbase = get_module_base(start);
+    DEBUG_DECLARE(bool ok;)
+
+    KSTART(persisted_load);
+    DOLOG(1, LOG_CACHE, {
+        char modname[MAX_MODNAME_INTERNAL];
+        os_get_module_name_buf(modbase, modname, BUFFER_SIZE_ELEMENTS(modname));
+        LOG(THREAD, LOG_CACHE, 1, "coarse_unit_load %s\n", modname);
+    });
+    DOSTATS({
+        if (for_execution)
+            STATS_INC(perscache_load_attempt);
+        else
+            STATS_INC(perscache_load_nox_attempt);
+    });
+#ifdef HOT_PATCHING_INTERFACE
+    ASSERT_OWN_READWRITE_LOCK(DYNAMO_OPTION(hot_patching), hotp_get_lock());
+#endif
+
+    /* case 9799: pcache-affecting options are part of name.
+     * if -persist_check_exempted_options we must first try w/ the local
+     * options and then the global, hence the do-while loop.
+     */
+    option_level = persist_get_options_level(modbase, NULL, true/*force local*/);
+    do {
+        option_string = persist_get_relevant_options(dcontext, option_buf,
+                                                     BUFFER_SIZE_ELEMENTS(option_buf),
+                                                     option_level);
+        memset(&modinfo, 0, sizeof(modinfo));
+        if (!get_persist_filename(filename, BUFFER_SIZE_ELEMENTS(filename), modbase,
+                                  false/*read*/, &modinfo, option_string)) {
+            LOG(THREAD, LOG_CACHE, 1,
+                "  error computing name/excluded for "PFX"-"PFX"\n", start, end);
+            STATS_INC(perscache_load_noname);
+            goto coarse_unit_load_exit;
+        }
+        LOG(THREAD, LOG_CACHE, 1, "  persisted filename = %s\n", filename);
+        
+        if (DYNAMO_OPTION(validate_owner_dir)) {
+            ASSERT(DYNAMO_OPTION(persist_per_user));
+            if (perscache_user_directory == INVALID_FILE) {
+                /* there is no alternative location to test - currently
+                 * only single per-user directories supported 
+                 */
+                LOG(THREAD, LOG_CACHE, 1, 
+                    "  directory is unsafe, cannot use persistent cache\n");
+                goto coarse_unit_load_exit;
+            }
+            
+            DODEBUG({
+                pcache_dir_check_permissions(dcontext, filename);
+            });
+        }
+
+        /* On win32 OS_EXECUTE is required to create a section w/ rwx
+         * permissions, which is in turn required to map a view w/ rwx
+         */
+        fd = os_open(filename, OS_OPEN_READ | OS_EXECUTE |
+                     /* Case 9964: we want to allow renaming while holding
+                      * the file handle */
+                     OS_SHARE_DELETE);
+        if (fd == INVALID_FILE && option_level == OP_PCACHE_LOCAL) {
+            ASSERT(DYNAMO_OPTION(persist_check_exempted_options));
+            LOG(THREAD, LOG_CACHE, 1, "  local-options file not found %s\n", filename);
+            option_level = OP_PCACHE_GLOBAL;
+            /* try again */
+        } else
+            break;
+    } while (true);
+
+    if (fd == INVALID_FILE) {
+        LOG(THREAD, LOG_CACHE, 1, "  error opening file %s\n", filename);
+        STATS_INC(perscache_load_nofile);
+        goto coarse_unit_load_exit;
+    }
+    if (DYNAMO_OPTION(validate_owner_file)) {
+        if (!os_validate_user_owned(fd)) {
+            SYSLOG_INTERNAL_ERROR_ONCE("%s not owned by current process!"
+                                       " Persistent cache may be compromised, not using.",
+                                       filename);
+            /* note we don't attempt to overwrite here */
+            goto coarse_unit_load_exit;
+        }
+    } else {
+        ASSERT(!DYNAMO_OPTION(validate_owner_dir) || 
+               perscache_user_directory != INVALID_FILE);
+        /* note that if have a directory handle open, there is no need to
+         * validate file with DYNAMO_OPTION(validate_owner_file) here.
+         * of course it doesn't hurt to check both.
+         */
+        DODEBUG({
+            ASSERT_CURIOSITY(os_validate_user_owned(fd) && "impostor not detected!");
+        });
+    }
+
+    if (!os_get_file_size_by_handle(fd, &file_size)) {
+        LOG(THREAD, LOG_CACHE, 1, "  error obtaining file size for %s\n", filename);
+        goto coarse_unit_load_exit;
+    }
+    ASSERT_TRUNCATE(map_size, uint, file_size);
+    map_size = (uint) file_size;
+    LOG(THREAD, LOG_CACHE, 1, "  size of %s is %d\n", filename, map_size);
+    /* FIXME case 9642: control where in address space we map the file:
+     * right after vmheap?  Randomized?
+     */
+    map = map_file(fd, map_size, 0, NULL,
+                   /* Ask for max, then restrict pieces */
+                   MEMPROT_READ|MEMPROT_WRITE|MEMPROT_EXEC,
+                   /* case 9599: asking for COW commits pagefile space
+                    * up front, so we map two separate views later: see below
+                    */
+                   true/*writes should not change file*/);
+    /* case 9925: if we keep the file handle open we can prevent writes
+     * to the file while it's mapped in, but it prevents our rename replacement
+     * scheme (case 9701/9720) so we have it under option control.
+     */
+    if (!DYNAMO_OPTION(persist_lock_file)) {
+        os_close(fd);
+        fd = INVALID_FILE;
+    }
+
+    if (map == NULL) {
+        LOG(THREAD, LOG_CACHE, 1, "  error mapping file %s\n", filename);
+        goto coarse_unit_load_exit;
+    }
+    pers = (coarse_persisted_info_t *) map;
+    ASSERT(pers->header_len + pers->data_len == map_size);
+    footer = (persisted_footer_t *)
+        (map + pers->header_len + pers->data_len - sizeof(persisted_footer_t));
+
+    if (pers->magic != PERSISTENT_CACHE_MAGIC ||
+        /* case 10160: don't crash trying to read footer */
+        pers->header_len + pers->data_len != map_size ||
+        footer->magic != PERSISTENT_CACHE_MAGIC ||
+        TEST(PERSCACHE_CODE_INVALID, pers->flags)) {
+        LOG(THREAD, LOG_CACHE, 1, "  invalid persisted file %s\n", filename);
+        /* This flag shouldn't be set for on-disk units */
+        ASSERT(!TEST(PERSCACHE_CODE_INVALID, pers->flags));
+        STATS_INC(perscache_bad_file);
+        /* We didn't have a footer prior to version 4 */
+        ASSERT_CURIOSITY_ONCE(pers->version < 4 && "persistent cache file corrupt");
+        goto coarse_unit_load_exit;
+    }
+
+    if (pers->version != PERSISTENT_CACHE_VERSION) {
+        /* FIXME case 9655: should we clobber such a file when we persist? */
+        LOG(THREAD, LOG_CACHE, 1, "  invalid persisted file version %d for %s\n",
+            pers->version, filename);
+        STATS_INC(perscache_version_mismatch);
+        goto coarse_unit_load_exit;
+    }
+
+    if (!TEST(PERSCACHE_X86_32, pers->flags)) {
+        LOG(THREAD, LOG_CACHE, 1, "  invalid architecture: not IA-32 %s\n", filename);
+        STATS_INC(perscache_version_mismatch);
+        SYSLOG_INTERNAL_WARNING_ONCE("persistent cache architecture mismatch");
+        goto coarse_unit_load_exit;
+    }
+
+    /* Check file's md5 */
+    if (TESTANY(PERSCACHE_GENFILE_MD5_SHORT|PERSCACHE_GENFILE_MD5_COMPLETE,
+                DYNAMO_OPTION(persist_load_validation))) {
+        module_digest_t self_md5;
+        persist_calculate_self_digest(&self_md5, pers, map,
+                                      DYNAMO_OPTION(persist_load_validation));
+        DOLOG(1, LOG_CACHE, {
+            print_module_digest(THREAD, &footer->self_md5, "md5 stored in file: ");
+            print_module_digest(THREAD, &footer->self_md5, "md5 calculated:     ");
+        });
+        if ((TEST(PERSCACHE_GENFILE_MD5_SHORT,
+                  DYNAMO_OPTION(persist_load_validation)) &&
+             !md5_digests_equal(self_md5.short_MD5, footer->self_md5.short_MD5)) ||
+            (TEST(PERSCACHE_GENFILE_MD5_COMPLETE,
+                  DYNAMO_OPTION(persist_load_validation)) &&
+             !md5_digests_equal(self_md5.full_MD5, footer->self_md5.full_MD5))) {
+            LOG(THREAD, LOG_CACHE, 1, "  file header md5 mismatch\n");
+            STATS_INC(perscache_md5_mismatch);
+            ASSERT_CURIOSITY_ONCE(false && "persistent cache md5 mismatch");
+            goto coarse_unit_load_exit;
+        }
+    }
+
+    /* Consistency with original module */
+    persist_calculate_module_digest(&modinfo.module_md5, modbase,
+                                    (size_t) modinfo.image_size,
+                                    modbase + pers->start_offs,
+                                    modbase + pers->end_offs,
+                                    DYNAMO_OPTION(persist_load_validation));
+    /* Compare the module digest and module fields (except base) all at once */
+    if (!persist_modinfo_cmp(&modinfo, &pers->modinfo)) {
+        LOG(THREAD, LOG_CACHE, 1, "  module info mismatch\n");
+        DOLOG(1, LOG_CACHE, {
+            LOG(THREAD, LOG_CACHE, 1, "modinfo stored in file: ");
+            dump_buffer_as_bytes(THREAD, &pers->modinfo, sizeof(pers->modinfo),
+                                 DUMP_RAW|DUMP_DWORD);
+            LOG(THREAD, LOG_CACHE, 1, "\nmodinfo in memory:      ");
+            dump_buffer_as_bytes(THREAD, &modinfo, sizeof(modinfo),
+                                 DUMP_RAW|DUMP_DWORD);
+            LOG(THREAD, LOG_CACHE, 1, "\n");
+        });
+        SYSLOG_INTERNAL_WARNING_ONCE("persistent cache module mismatch");
+        STATS_INC(perscache_modinfo_mismatch);
+        goto coarse_unit_load_exit;
+    }
+
+    /* modinfo cmp checks all but base, which is separate for reloc support */
+    if (modbase != pers->modinfo.base) {
+        /* FIXME case 9581/9649: Bail out for now since relocs NYI
+         * Once we do support them, make sure to do the right thing when merging:
+         * current code will always apply relocs before merging.
+         */
+        LOG(THREAD, LOG_CACHE, 1, "  module base mismatch "PFX" vs persisted "PFX"\n",
+            modbase, pers->modinfo.base);
+        STATS_INC(perscache_base_mismatch);
+        goto coarse_unit_load_exit;
+    }
+
+    if (modbase + pers->start_offs < start ||
+        modbase + pers->end_offs > end) {
+        /* case 9653: only load a file that covers a subset of the vmarea region */
+        LOG(THREAD, LOG_CACHE, 1,
+            "  region mismatch "PFX"-"PFX" vs persisted "PFX"-"PFX"\n",
+            start, end, modbase + pers->start_offs, modbase + pers->end_offs);
+        STATS_INC(perscache_region_mismatch);
+        goto coarse_unit_load_exit;
+    }
+
+    if (!persist_check_option_compat(dcontext, pers, option_string)) {
+        goto coarse_unit_load_exit;
+    }
+
+    stubs_and_prefixes_len =
+        (pers->stubs_len + pers->ibl_jmp_prefix_len + pers->ibl_call_prefix_len
+         + pers->ibl_ret_prefix_len + pers->trace_head_return_prefix_len
+         + pers->fcache_return_prefix_len);
+    
+    if (TEST(PERSCACHE_MAP_RW_SEPARATE, pers->flags) &&
+        DYNAMO_OPTION(persist_map_rw_separate)) {
+        size_t ro_size;
+        map2_size = stubs_and_prefixes_len + sizeof(persisted_footer_t);
+        ro_size = map_size - map2_size;
+        ASSERT(ro_size == ALIGN_FORWARD(pers->header_len + pers->data_len
+                                        - stubs_and_prefixes_len
+                                        - pers->pad_len
+                                        - sizeof(persisted_footer_t),
+                                        MAP_FILE_VIEW_ALIGNMENT));
+        LOG(THREAD, LOG_CACHE, 2,
+            "  attempting double mapping: size "PIFX" and "PIFX"\n", ro_size, map2_size);
+        if (!DYNAMO_OPTION(persist_lock_file)) {
+            fd = os_open(filename, OS_OPEN_READ|OS_EXECUTE|OS_SHARE_DELETE);
+            /* FIXME: case 10547 what do we know about this second
+             * file that we just opened?  Nothing guarantees it is
+             * still the same one we used earlier.
+             */
+            if (fd != INVALID_FILE && DYNAMO_OPTION(validate_owner_file)) {
+                if (!os_validate_user_owned(fd)) {
+                    os_close(fd);
+                    fd = INVALID_FILE;
+                    SYSLOG_INTERNAL_ERROR_ONCE("%s not owned by current process!"
+                                               " Persistent cache may be compromised, "
+                                               "not using.",
+                                               filename);
+                    
+                }
+            }
+        }
+        ASSERT(fd != INVALID_FILE);
+        if (fd != INVALID_FILE) {
+            unmap_file(map, map_size);
+            pers = NULL;
+            map_size = ro_size;
+            /* we know the whole thing fit @map, so try there */
+            map = map_file(fd, map_size, 0, map,
+                           /* Ask for max, then restrict pieces */
+                           MEMPROT_READ|MEMPROT_EXEC,
+                           false/*no COW to avoid pagefile cost*/);
+            map2 = map_file(fd, map2_size, map_size, map + map_size,
+                            /* Ask for max, then restrict pieces */
+                            MEMPROT_READ|MEMPROT_WRITE|MEMPROT_EXEC,
+                            true/*writes should not change file*/);
+            /* FIXME: try again if racy alloc and they both don't fit */
+            if (!DYNAMO_OPTION(persist_lock_file)) {
+                os_close(fd);
+                fd = INVALID_FILE;
+            }
+            if (map == NULL || map2 != map + ro_size) {
+                SYSLOG_INTERNAL_ERROR_ONCE("double perscache mapping failed");
+                LOG(THREAD, LOG_CACHE, 1,
+                    "  error: 2nd map "PFX" not adjacent to 1st "PFX"\n", map, map2);
+                STATS_INC(perscache_maps_not_adjacent);
+                goto coarse_unit_load_exit;
+            }
+            /* the two views are adjacent so most code can treat it as one view */
+            LOG(THREAD, LOG_CACHE, 1,
+                "  mapped view1 @"PFX" and view2 @"PFX"\n", map, map2);
+            pers = (coarse_persisted_info_t *) map;
+        }
+    }
+
+    /* We assume that once info!=NULL we have been successful, though we do
+     * abort for hotp conflicts below */
+    info = coarse_unit_create(modbase + pers->start_offs, modbase + pers->end_offs,
+                              &modinfo.module_md5, for_execution);
+    info->frozen = true;
+    info->persisted = true;
+    info->has_persist_info = true;
+    info->mmap_pc = map;
+    if (map2 != NULL) {
+        info->mmap_ro_size = map_size;
+        info->mmap_size = map_size + map2_size;
+        ASSERT(map2 == info->mmap_pc + info->mmap_ro_size);
+    } else
+        info->mmap_size = map_size;
+    ASSERT(info->mmap_size == pers->header_len + pers->data_len);
+    if (DYNAMO_OPTION(persist_lock_file))
+        info->fd = fd;
+
+    info->flags = pers->flags;
+#if defined(RETURN_AFTER_CALL) && defined(WINDOWS)
+    /* FIXME: provide win32/ interface for setting the flag? */
+    if (TEST(PERSCACHE_SEEN_BORLAND_SEH, pers->flags) && !seen_Borland_SEH) {
+        SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+        seen_Borland_SEH = true;
+        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+    }
+#endif
+    ASSERT(option_level != OP_PCACHE_LOCAL ||
+           TEST(PERSCACHE_EXEMPTION_OPTIONS, info->flags));
+
+    /* Process data sections (other than option string) in reverse order */
+    pc = map + pers->header_len + pers->data_len; /* end of file */
+    pc -= sizeof(persisted_footer_t);
+    info->stubs_end_pc = pc;
+
+    pc -= stubs_and_prefixes_len;
+    info->fcache_return_prefix = pc;        
+    rwx_pc = info->fcache_return_prefix;
+    ASSERT(ALIGNED(rwx_pc, PAGE_SIZE));
+    /* To patch up the prefixes, it's just as easy to re-emit them all as
+     * most are just plain jmps anyway
+     */
+    info->stubs_start_pc = coarse_stubs_create(info, info->fcache_return_prefix,
+                                               stubs_and_prefixes_len);
+    /* We could omit these from the file and assume their sizes */
+    DODEBUG({
+        byte *check = info->fcache_return_prefix;
+        check += pers->fcache_return_prefix_len;
+        ASSERT(check == info->trace_head_return_prefix);
+        check += pers->trace_head_return_prefix_len;
+        ASSERT(check == info->ibl_ret_prefix);
+        check += pers->ibl_ret_prefix_len;
+        ASSERT(check == info->ibl_call_prefix);
+        check += pers->ibl_call_prefix_len;
+        ASSERT(check == info->ibl_jmp_prefix);
+        check += pers->ibl_jmp_prefix_len;
+        ASSERT(check == info->stubs_start_pc);
+        check += pers->stubs_len;
+        ASSERT(check == info->stubs_end_pc);
+    });
+    if (DYNAMO_OPTION(persist_protect_stubs)) {
+        /* Mark prefixes + stubs read-only.  Note that we can't map in as +rx, since
+         * we must have +w to get COW (unless we switch to MEM_IMAGE).  So we have
+         * no way to avoid the pagefile cost, except the mysterious case 10074,
+         * which we take advantage of here by touching every page (yes it works
+         * even after the prefix write above), paying a one-time working set cost.
+         */
+        if (DYNAMO_OPTION(persist_touch_stubs)) {
+            app_pc touch_pc = rwx_pc;
+            volatile byte touch_value;
+            for (; touch_pc < (app_pc) (map + pers->header_len + pers->data_len);
+                 touch_pc += PAGE_SIZE) {
+                touch_value = *touch_pc;
+                STATS_INC(pcache_stub_touched);
+            }
+        }
+        DEBUG_DECLARE(ok =)
+            set_protection(rwx_pc, map + pers->header_len + pers->data_len - rwx_pc,
+                           MEMPROT_READ|MEMPROT_EXEC);
+        ASSERT(ok);
+        info->stubs_readonly = true;
+    } else {
+        /* FIXME case 9650: we could mark the prefixes as read-only now, if we put them
+         * on their own page
+         */
+    }
+
+    info->cache_end_pc = pc - pers->post_cache_pad_len;
+    pc -= pers->cache_len;
+    info->cache_start_pc = pc;
+    rx_pc = info->cache_start_pc;
+    ASSERT(ALIGNED(rx_pc, PAGE_SIZE));
+    fcache_coarse_init_frozen(dcontext, info, info->cache_start_pc,
+                              info->fcache_return_prefix - info->cache_start_pc);
+
+    pc -= pers->pad_len; /* just skip it */
+
+    pc -= pers->stub_htable_len;
+    fragment_coarse_htable_resurrect(GLOBAL_DCONTEXT, info, false/*stub table*/, pc);
+    ASSERT(fragment_coarse_htable_persist_size(dcontext, info, false/*stub*/) ==
+           pers->stub_htable_len);
+    pc -= pers->cache_htable_len;
+    fragment_coarse_htable_resurrect(GLOBAL_DCONTEXT, info, true/*cache table*/, pc);
+    ASSERT(fragment_coarse_htable_persist_size(dcontext, info, true/*cache*/) ==
+           pers->cache_htable_len);
+    ASSERT(offsetof(coarse_persisted_info_t, cache_htable_len) < pers->header_len);
+
+    /* From here on out, check offsets so will work w/ earlier-versioned file
+     */
+
+    if (offsetof(coarse_persisted_info_t, rct_htable_len) < pers->header_len) {
+        pc -= pers->rct_htable_len;
+#ifdef RCT_IND_BRANCH
+        if (pers->rct_htable_len > 0) {
+            /* We may be merging with the on-disk file, so load it in
+             * regardless of whether we care in this process instance
+             * Use GLOBAL_DCONTEXT in case we do use.
+             */
+            info->rct_table = rct_table_resurrect(GLOBAL_DCONTEXT, pc, RCT_RCT);
+            ASSERT(info->rct_table != NULL);
+            if (for_execution && 
+                (TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) ||
+                 TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump))) &&
+                (DYNAMO_OPTION(use_persisted_rct) ||
+                 TEST(PERSCACHE_SEEN_BORLAND_SEH, pers->flags)) &&
+                /* Case 9834: avoid double-add from earlier entire-module resurrect */
+                !os_module_get_flag(info->base_pc, MODULE_RCT_LOADED)) {
+                /* load into the official module table */
+                DEBUG_DECLARE(bool used =)
+                    rct_module_table_set(GLOBAL_DCONTEXT, modbase,
+                                         info->rct_table, RCT_RCT);
+                ASSERT(used);
+# ifdef WINDOWS
+                if (TEST(PERSCACHE_ENTIRE_MODULE_RCT, pers->flags)) {
+                    /* mark as not needing to be analyzed */
+                    os_module_set_flag(info->base_pc, MODULE_RCT_LOADED);
+                }
+# endif
+            }
+        }
+#endif
+    }
+    if (offsetof(coarse_persisted_info_t, rac_htable_len) < pers->header_len) {
+        pc -= pers->rac_htable_len;
+#ifdef RETURN_AFTER_CALL
+        if (pers->rac_htable_len > 0) {
+            /* We may be merging with the on-disk file, so load it in
+             * regardless of whether we care in this process instance.
+             * Use GLOBAL_DCONTEXT in case we do use.
+             */
+            info->rac_table = rct_table_resurrect(GLOBAL_DCONTEXT, pc, RCT_RAC);
+            ASSERT(info->rac_table != NULL);
+            if (for_execution && DYNAMO_OPTION(ret_after_call)) {
+                /* load into the official module table */
+                DEBUG_DECLARE(bool used =)
+                    rct_module_table_set(GLOBAL_DCONTEXT, modbase,
+                                         info->rac_table, RCT_RAC);
+                ASSERT(used);
+            }
+        }
+#endif
+    }
+
+    /* FIXME case 9581 NYI: reloc section */
+    if (offsetof(coarse_persisted_info_t, reloc_len) < pers->header_len) {
+        pc -= pers->reloc_len;
+    }
+
+#ifdef HOT_PATCHING_INTERFACE
+    if (offsetof(coarse_persisted_info_t, hotp_patch_list_len) < pers->header_len) {
+        /* Case 9995: store list of active patch points in perscache file and don't
+         * abort if those are the only matching points at this time.
+         * FIXME if we had an efficient way to see if an app pc is present
+         * in the pcache (case 9969) we could be more precise.
+         */
+        pc -= pers->hotp_patch_list_len;
+        IF_X64(ASSERT_TRUNCATE(info->hotp_ppoint_vec_num, uint,
+                               pers->hotp_patch_list_len / sizeof(app_rva_t)));
+        info->hotp_ppoint_vec_num = (uint) pers->hotp_patch_list_len / sizeof(app_rva_t);
+        if (info->hotp_ppoint_vec_num > 0)
+            info->hotp_ppoint_vec = (app_rva_t *) pc;
+    } else
+        info->hotp_ppoint_vec_num = 0;
+    ASSERT(info->hotp_ppoint_vec == NULL || info->hotp_ppoint_vec_num > 0);
+    if (DYNAMO_OPTION(hot_patching) &&
+        hotp_point_not_on_list(info->base_pc, info->end_pc,
+                               true/*own hotp lock*/,
+                               info->hotp_ppoint_vec, info->hotp_ppoint_vec_num)) {
+        LOG(THREAD, LOG_CACHE, 1,
+            "  error: hotp match prevents using persistence\n");
+        STATS_INC(perscache_hotp_conflict);
+        /* FIXME: we could duplicate the calculation of info->hotp_ppoint_vec
+         * to detect this error before allocating info.  Instead we have to
+         * free what we've created.  A bonus of waiting to detect the error
+         * is that we've already loaded the RCT entries, which the error does
+         * not invalidate.
+         */
+        coarse_unit_reset_free_internal(dcontext, info, false/*no locks*/,
+                                        false/*not linked yet*/,
+                                        true/*give up primary*/,
+                                        /* case 11064: avoid rank order */
+                                        false/*local, no info lock needed*/);
+        coarse_unit_free(dcontext, info);
+        info = NULL;
+        map = NULL; /* reset_free unmapped for us */
+        map2 = NULL; /* reset_free unmapped for us */
+        goto coarse_unit_load_exit;
+    }
+#endif
+
+    ASSERT(pc - map >= (int)pers->header_len);
+
+    DEBUG_DECLARE(ok =)
+        set_protection(map, rx_pc - map, MEMPROT_READ);
+    ASSERT(ok);
+    DEBUG_DECLARE(ok =)
+        set_protection(rx_pc, rwx_pc - rx_pc, MEMPROT_READ|MEMPROT_EXEC);
+    ASSERT(ok);
+
+    /* FIXME case 9648: don't forget to append a guard page -- but we
+     * should only need it if we fill up the full allocation region,
+     * so on win32 we will get a free guard page whenever we're not
+     * end-aligned to 64KB.
+     */
+    RSTATS_INC(perscache_loaded);
+
+ coarse_unit_load_exit:
+    if (info == NULL /* error! */) {
+        if (map != NULL)
+            unmap_file(map, map_size);
+        if (map2 != NULL)
+            unmap_file(map2, map2_size);
+        if (fd != INVALID_FILE)
+            os_close(fd);
+    }
+    KSTOP(persisted_load);
+    return info;
+}
+
+bool
+exists_coarse_ibl_pending_table(dcontext_t *dcontext, /* in case per-thread someday */
+                                coarse_info_t *info, ibl_branch_type_t branch_type)
+{
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+    bool exists = false;
+    if (info != NULL) {
+        if (branch_type == IBL_RETURN)
+            exists = rct_module_persisted_table_exists(dcontext, info->base_pc, RCT_RAC);
+        else if (branch_type == IBL_INDCALL)
+            exists = rct_module_persisted_table_exists(dcontext, info->base_pc, RCT_RCT);
+        else {
+            ASSERT(branch_type == IBL_INDJMP);
+            exists = rct_module_persisted_table_exists(dcontext, info->base_pc, RCT_RCT)
+                || rct_module_persisted_table_exists(dcontext, info->base_pc, RCT_RAC);
+        }
+        return (exists && !TEST(COARSE_FILL_IBL_MASK(branch_type),
+                                info->ibl_pending_used));
+    }
+#endif
+    return false;
+}
+
+/* If pc is in a module, marks that module as exempted (case 9799) */
+void
+mark_module_exempted(app_pc pc)
+{
+    if (DYNAMO_OPTION(persist_check_options) &&
+        /* if persist_check_local_options is on no reason to track exemptions */
+        !DYNAMO_OPTION(persist_check_local_options) &&
+        DYNAMO_OPTION(persist_check_exempted_options) &&
+        module_info_exists(pc) &&
+        !os_module_get_flag(pc, MODULE_WAS_EXEMPTED)) {
+        LOG(GLOBAL, LOG_VMAREAS, 1, "marking module @"PFX" as exempted\n");
+        os_module_set_flag(pc, MODULE_WAS_EXEMPTED);
+    }
+}
+
diff --git a/core/perscache.h b/core/perscache.h
new file mode 100644
index 000000000..e1740cfbd
--- /dev/null
+++ b/core/perscache.h
@@ -0,0 +1,585 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2006-2007 Determina Corp. */
+
+/*
+ * perscache.h - persistent code cache exports
+ */
+
+#ifndef _PERSCACHE_H_
+#define _PERSCACHE_H_ 1
+
+#include "module_shared.h" /* for module_digest_t */
+
+/***************************************************************************
+ * COARSE-GRAIN UNITS
+ */
+
+/* Information kept per coarse-grain region.
+ * FIXME: for sharing we want to keep htable, stubs, and incoming
+ * per unit, not per cache.  That will require changing fcache_unit_t and
+ * having a way to map from app pc to unit.  For this first step toward
+ * sharing we stick with a per-cache scheme with the multiple units
+ * internal to the cache simply being ignored at this level.
+ * Cache consistency needs to treat this as a unit in any case.
+ *
+ * Synchronization model: the struct lock controls writes to the
+ * direct fields.  The cache, htable, th_htable, and stubs fields are
+ * all assumed to only be written at init time, and thus internal
+ * changes to the objects do not require the struct lock.
+ * The struct lock is used at init time and for later writes to incoming
+ * and frozen.
+ * Destruction is assumed to involve all-thread-synch and so reads of
+ * fields do not require the struct lock.  Just like reset, we rely on
+ * all-thread-synch plus redirection to dispatch (rather than resuming
+ * at suspended location, which we only do for native threads) to
+ * allow us to free these shared structures that are read w/o locks.
+ * FIXME: what about where we lack setcontext permission
+ */
+struct _coarse_info_t {
+    bool frozen:1;
+    bool persisted:1;
+    bool in_use:1; /* are we using this unit officially? */
+    /* Flag to indicate whether we've calculated the rac/rct/hotp info that
+     * we only need when persisting.
+     */
+    bool has_persist_info:1;
+    /* Case 9653: only the 1st coarse unit in a module's +x region(s) is persisted
+     * Non-in-use units inherit this from their sources, but do not
+     * change the status on deletion.
+     */
+    bool primary_for_module:1;
+    /* case 10525 where we keep the stubs read-only */
+    bool stubs_readonly:1;
+#ifdef DEBUG
+    /* A local info pointer has not escaped to any other thread.
+     * We only use this flag to get around lock ordering issues (case 11064).
+     */
+    bool is_local:1;           /* no lock needed since only known to this thread */
+#endif
+
+    void *cache; /* opaque type internal to fcache.c */
+    
+    /* For frozen units, htable holds body pcs, while th_htable holds stub pcs.
+     * FIXME case 8628: split these into a body table and a stub table for
+     * non-frozen units.
+     */
+    void *htable; /* opaque htable mapping app pc -> stub/cache entry point */
+    void *th_htable; /* opaque htable mapping trace head app pc -> cache entry point */
+    
+    void *stubs; /* opaque special heap */
+
+    cache_pc fcache_return_prefix;
+    cache_pc trace_head_return_prefix;
+    cache_pc ibl_ret_prefix;
+    cache_pc ibl_call_prefix;
+    cache_pc ibl_jmp_prefix;
+
+    coarse_incoming_t *incoming;
+
+    /* These fields are non-NULL only for frozen units.
+     * Since htable entries are offsets we need to expose the cache and stub pcs
+     */
+    cache_pc cache_start_pc;
+    cache_pc cache_end_pc; /* last instr, not end of allocation */
+    cache_pc stubs_start_pc; /* this is post-prefixes */
+    cache_pc stubs_end_pc; /* may not fill out full mmap_size if overestimate */
+    /* if not persisted, this is the bounds of the region shared by
+     * the frozen cache and stubs, assumed to start at cache_start_pc;
+     * if persisted, this is the bounds of the entire mmapped file.
+     */
+    size_t mmap_size;
+    /* Performance optimization for frozen units (critical for trace building) */
+    void *pclookup_htable; /* opaque htable mapping cache entry point -> app pc */
+    /* end frozen-only fields */
+
+    /* Fields for persisted units */
+    uint flags; /* corresponds to PERSCACHE_ flags for persisted files */
+    cache_pc mmap_pc; /* start of persisted mmapped file; size is mmap_size */
+    /* if this is >0, we mapped the file in two different views:
+     * 1) [mmap_pc,mmap_pc+mmap_ro_size) 
+     * 2) [mmap_pc+mmap_ro_size,mmap_pc+mmap_size)
+     */
+    size_t mmap_ro_size;
+    /* case 9925: we may want to keep the file handle open for the duration */
+    file_t fd;
+
+    /* If we merged with a persisted file, we store the original size so
+     * we can avoid re-merging with that on-disk file.
+     */
+    size_t persisted_source_mmap_size;
+
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+    app_pc_table_t *rct_table;
+    app_pc_table_t *rac_table;
+    /* Case 9639: fill ibl tables from persisted rac/rct table
+     * entries.  We only do once, storing here to enforce that.
+     */
+    uint ibl_pending_used; /* bitmask from (1<<(IBL_<type>-1)) so we use each once */
+#endif
+
+#ifdef HOT_PATCHING_INTERFACE
+    /* Case 9995: array of hotp patch points matched at persist time, which
+     * means they should have been made fine-grained and NOT be present
+     * in the pcache.
+     * Used for persisted units only; for this-run units it would only make
+     * sense to use this if hotp did not ignore idempotent ppoint changes
+     * at nudge time, which may be the case today but better to fix hotp.
+     */
+    app_rva_t *hotp_ppoint_vec;
+    uint hotp_ppoint_vec_num; /* number of elements in array */
+#endif
+
+    /* case 10525: leave stubs as writable if written too many times */
+    uint stubs_write_count;
+
+    /* case 9521: we can have a second unit in the same region for new,
+     * non-frozen coarse code if the primary unit is frozen.
+     * Presumably frozen unit is larger so we put it first.
+     */
+    struct _coarse_info_t *non_frozen;
+
+    /* FIXME: add debug-build stats on #fragments, #stubs,
+     * #trace heads, etc.
+     */
+
+    /***************************************************
+     * Fields below this point are preserved across a coarse_unit_reset_free(),
+     * while those above are cleared.
+     */
+
+    /* Controls access to directly changing the fields of the struct, except
+     * the incoming list */
+    mutex_t lock;
+    /* Controls the incoming list; separated to allow holding the src main
+     * lock while changing target linked unit locks (case 9809) */
+    mutex_t incoming_lock;
+
+    /* FIXME: once we persist/share we will need official labels to identify
+     * source modules
+     */
+    app_pc base_pc; /* base of vm area this unit covers */
+    app_pc end_pc;  /* end of vm area this unit covers */
+#ifdef DEBUG
+    const char *module;
+#endif
+    /* MD5 of the module, used only for persisting but we calculate at load time
+     * so we're comparing the in-memory image at a consistent point.
+     */
+    module_digest_t module_md5;
+    
+    /* Only add a field here if it should be preserved across coarse_unit_reset_free */
+
+}; /* typedef as "coarse_info_t" is in globals.h */
+
+#ifdef X64
+# define COARSE_32_FLAG(info) (TEST(PERSCACHE_X86_32, (info)->flags) ? FRAG_32_BIT : 0)
+#else
+# define COARSE_32_FLAG(info) 0
+#endif
+
+coarse_info_t *
+coarse_unit_create(app_pc base_pc, app_pc end_pc, module_digest_t *digest,
+                   bool for_execution);
+
+void
+coarse_unit_init(coarse_info_t *info, void *cache);
+
+/* If caller holds change_linking_lock and info->lock, have_locks should be true */
+void
+coarse_unit_reset_free(dcontext_t *dcontext, coarse_info_t *info,
+                       bool have_locks, bool unlink, bool abdicate_primary);
+
+void
+coarse_unit_free(dcontext_t *dcontext, coarse_info_t *info);
+
+void
+coarse_unit_mark_in_use(coarse_info_t *info);
+
+/***************************************************************************
+ * FROZEN UNITS
+ */
+
+/* For storing information needed during the freezing process */
+
+typedef struct _pending_freeze_t {
+    bool entrance_stub;
+    bool trace_head;
+    app_pc tag;
+    cache_pc cur_pc;
+    cache_pc link_cti_opnd; /* 4-byte pc-relative opnd to re-target */
+    bool elide_ubr; /* whether to elide the link, if that's an option */
+    struct _pending_freeze_t *next;
+} pending_freeze_t;
+
+struct _coarse_freeze_info_t {
+    coarse_info_t *src_info;
+    coarse_info_t *dst_info;
+    cache_pc cache_start_pc;
+    cache_pc cache_cur_pc;
+    cache_pc stubs_start_pc;
+    cache_pc stubs_cur_pc;
+    bool unlink;
+    pending_freeze_t *pending;
+#ifdef DEBUG
+    /* statistics on frozen code expansion from original app code */
+    size_t app_code_size;
+    uint num_cbr;
+    uint num_jmp;
+    uint num_call;
+    uint num_indbr;
+    /* removed code: */
+    uint num_elisions;
+    /* trying to enumerate all the added code: */
+    uint added_fallthrough;
+    uint added_indbr_mangle; /* all but the stub and the (removable) jmp to stub */
+    uint added_indbr_stub;   /* stub plus the jmp to it */
+    uint added_jecxz_mangle;
+#endif
+}; /* typedef as "coarse_freeze_info_t" is in globals.h */
+
+void
+perscache_init(void);
+
+void
+perscache_fast_exit(void);
+
+void
+perscache_slow_exit(void);
+
+void
+coarse_units_freeze_all(bool in_place);
+
+coarse_info_t *
+coarse_unit_freeze(dcontext_t *dcontext, coarse_info_t *info, bool in_place);
+
+void
+transfer_coarse_stub(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info,
+                     cache_pc stub, bool trace_head, bool replace_outgoing);
+void
+transfer_coarse_stub_fix_trace_head(dcontext_t *dcontext,
+                                    coarse_freeze_info_t *freeze_info,
+                                    cache_pc stub);
+void
+transfer_coarse_fragment(dcontext_t *dcontext, coarse_freeze_info_t *freeze_info,
+                         cache_pc body);
+
+coarse_info_t *
+coarse_unit_merge(dcontext_t *dcontext, coarse_info_t *info1, coarse_info_t *info2,
+                  bool in_place);
+
+/***************************************************************************
+ * PERSISTENT CODE CACHE
+ */
+
+/* For storing frozen caches on disk */
+
+enum {
+    PERSISTENT_CACHE_MAGIC = 0x244f4952, /* RIO$ */
+    PERSISTENT_CACHE_VERSION = 8,
+};
+
+/* Global flags we need to process if present in a persisted cache */
+enum {
+    /* Identify underlying architecture */
+    PERSCACHE_X86_32             = 0x00000001,
+    PERSCACHE_X86_64             = 0x00000002,
+    /* FIXME: should we add cache line info?  Currently we have no
+     * -pad_jmps for coarse bbs, coarse bbs are aligned to 1, our only
+     * hotpatched jmps are in stubs which are 16-byte-aligned and 15 bytes
+     * long, and we have -tls_align 1, so we have no cache line
+     * dependences.
+     */
+
+    PERSCACHE_SEEN_BORLAND_SEH   = 0x00000004,
+
+    /* Does cache contain elided ubrs? */
+    PERSCACHE_ELIDED_UBR         = 0x00000008,
+
+    /* Does cache contain return-after-call or RCT entries? */
+    PERSCACHE_SUPPORT_RAC        = 0x00000010,
+    PERSCACHE_SUPPORT_RCT        = 0x00000020,
+
+    /* Does cache contain persisted RCT for entire module? */
+    PERSCACHE_ENTIRE_MODULE_RCT  = 0x00000040,
+
+    /* Does cache support trace building? */
+    PERSCACHE_SUPPORT_TRACES     = 0x00000080,
+
+    /* Does cache support separately mapping the writable portion? */
+    PERSCACHE_MAP_RW_SEPARATE    = 0x00000100,
+
+    /* Case 9799: local exemption options are part of option string */
+    PERSCACHE_EXEMPTION_OPTIONS  = 0x00000200,
+
+    /* Used only in coarse_info_t, not in coarse_persisted_info_t.
+     * We load and use persisted RCT tables prior to full code consistency checks;
+     * plus, we may continue using RCT tables after code consistency checks fail.
+     * Thus, the code being valid is separate from the pcache file being loaded.
+     * Xref case 10601.
+     */
+    PERSCACHE_CODE_INVALID       = 0x00000400,
+};
+
+/* Consistency and security checking options */
+enum {
+    /* Checks on the app module */
+    PERSCACHE_MODULE_MD5_SHORT       = 0x00000001, /* header + footer */
+    PERSCACHE_MODULE_MD5_COMPLETE    = 0x00000002, /* entire code region */
+    /* Checks on our own generated file */
+    PERSCACHE_GENFILE_MD5_SHORT      = 0x00000004, /* header */
+    PERSCACHE_GENFILE_MD5_COMPLETE   = 0x00000008, /* entire file */
+    /* When to calculate gen-side module md5's 
+     * In 4.4 this will be stored at 1st execution, not load time (case 10601)
+     */
+    PERSCACHE_MODULE_MD5_AT_LOAD     = 0x00000010, /* else, at persist time */
+};
+
+/* FIXME: share with hotp_module_sig_t in hotpatch.c
+ * Xref case 9543 on combining all these module structs.
+ * N.B.: the precise layout of the fields here is relied upon
+ * in persist_modinfo_cmp()
+ */
+typedef struct _persisted_module_info_t {
+    app_pc base; /* base of module at persist time */
+    uint checksum;
+    uint timestamp;
+    /* Ordinarily we'd use size_t for image_size and code_size, but we
+     * want the same header size for both 32 and 64 bit
+     */
+    uint64 image_size;
+    uint64 code_size; /* sum of sizes of executable sections in module */
+    uint64 file_version;
+
+    /* FIXME case 10087: move to module list and share w/ module-level
+     * process control, aslr?
+     */
+    /* FIXME: granularity: may want to support loading small pieces of the
+     * cache+stubs at a time, to reduce cache capacity and consistency checks
+     * and RAC security loosening.  Should then have separate md5's.
+     */
+    module_digest_t module_md5;
+} persisted_module_info_t;
+
+/* We put an MD5 of our generated file at the end, to make it easier
+ * to exclude itself from the calc and to provide assurance the end
+ * of the file was written.
+ */
+typedef struct _persisted_footer_t {
+    /* Self-consistency: MD5 of the header or of the whole file.
+     * We store them separately, even though file is superset, to allow
+     * checking just the header even if we generated the whole.
+     */
+    module_digest_t self_md5; /* should we take module out of type name? */
+    uint magic;
+} persisted_footer_t;
+
+/* The layout of a frozen coarse unit for persisting to disk.
+ * For the htables, cache, and stubs, we store the raw data here and when
+ * reading back in we generate separate header structs for each.
+ *
+ * The layout is split into two pieces: the header and the data.  For
+ * each section of data (stubs, cache, htable) we store a length in
+ * the header.  The lengths are stored in reverse order, allowing for
+ * adding new sections (to the read-only-not-executable side at least)
+ * that are ignored by older versions of the code.
+ *
+ * FIXME: we have our own format here which we keep page-aligned on disk and set
+ * memory privileges properly ourselves upon loading in.  We could instead use
+ * an image format (PE/ELF): with PE the kernel will set up the privileges for
+ * us, though with ELF it's the user-mode loader that does that.  Our own format
+ * lets us map sub-views of the file, though currently only the small header
+ * data is not needed as a map.  An image format will make it easier to use
+ * tools like debuggers with our caches.  It also has an impact on sharing and
+ * copy-on-write.  But, the kernel may make assumptions about images that don't
+ * apply to our files.
+ */
+typedef struct _coarse_persisted_info_t {
+    /* Read-only **************************************/
+
+    /* For verification */
+    uint magic;
+    uint version;
+
+    /* The lengths of the two divisions in the file.
+     * We limit entire file to 2GB on x64, but use size_t as
+     * the natural size for all our length computations.
+     */
+    size_t header_len;
+    size_t data_len;
+
+    /* Unit-wide flags for state such as whether we've seen Borland SEH.
+     * Uses the PERSCACHE_ flags.
+     */
+    uint flags;
+
+    /* For diagnostics */
+    uint build_number;
+
+    /* Consistency with original file */
+    persisted_module_info_t modinfo;
+
+    /* the address range that this cache covers, offset from module_base */
+    size_t start_offs;
+    size_t end_offs;
+
+    /* We require a match here; alternative is to put all uses in relocs */
+    uint tls_offs_base; /* could be ushort */
+
+    /* Now we store the lengths of each data section, in reverse
+     * order, to allow for expansion */
+
+    /* +rwx data sections */
+    size_t stubs_len;
+    size_t ibl_jmp_prefix_len;
+    size_t ibl_call_prefix_len;
+    size_t ibl_ret_prefix_len;
+    size_t trace_head_return_prefix_len;
+    size_t fcache_return_prefix_len;
+
+    /* +rx data sections */
+    size_t cache_len;
+    size_t post_cache_pad_len; /* included in cache_len, this lets us know the
+                              * end of actual instrs in the cache */
+    size_t pad_len;
+
+    /* +r data sections */
+    size_t stub_htable_len;
+    size_t cache_htable_len;
+    size_t rct_htable_len;
+    size_t rac_htable_len;
+    size_t reloc_len;
+#ifdef HOT_PATCHING_INTERFACE
+    size_t hotp_patch_list_len; /* in bytes, like all the other *_len fields */
+#endif
+
+    /* Case 9799: pcache-affecting options that differ from default values */
+    size_t option_string_len;
+
+    /* Add length of new +r data section here (header grows downward)
+     * header_len indicates the start of the data section
+     */
+
+    /* Data sections ****************************************/
+    /* option_string is the 1st data section; it may be padded to a 4-byte alignment */
+
+    /* Add new data section here (data grows upward across versions) */
+
+#ifdef HOT_PATCHING_INTERFACE
+    /* Hotp patch points matched at persist time to avoid flushing if
+     * the same vulns are active in the current run (case 9969).
+     */
+#endif
+
+    /* Relocations
+     * FIXME case 9581 NYI: other than app code relocs, all
+     * we add w/ coarse bbs are "call->push immed" manglings.  Once have traces,
+     * also stay-on-trace cmp.  No off-fragment jmps are currently allowed
+     * except for fcache/trace-head return and ibl, which are indirected.
+     * For app relocs we can either store in our own format at freeze time
+     * or do all processing at load time.
+     * FIXME: our non-entry pclookup is slow -- how slow will applying relocs be?
+     * FIXME case 9649: We could make our own call->push manglings
+     * PIC using pc-relative addressing on x86-64.
+     */
+
+    /* RAC and RCT tables -- RCT only for Borland SEH
+     * FIXME case 8648: instead keep as flag hidden in msb of main htable?
+     *   won't work for persisting the entire RCT tables, but will for Borland
+     *   and RAC where all targets are present in cache.
+     * FIXME case 9777: we're loosening security by allowing ret to target any
+     * after-call executed in any prior run of this app or whatever
+     * app is producing, instead of just this run.
+     */
+
+    /* Hashtables
+     * Up to the table what to persist: current plan has the struct and the table
+     * here, and on load we make a copy of the struct so that the lock can be
+     * writable.
+     */
+
+    /* Padding to get +x section on new page */
+
+    /****** Begin offset-sensitive sequence */
+    /* We have one single contiguous piece of memory containing our cache and
+     * stubs which must be kept together, as they contain relative jmps to each
+     * other, including to the prefixes at the top of the stubs.
+     *
+     * --------------------------------------
+     * executable below here
+     *
+     *   code cache
+     *
+     * read-only above here
+     * --------------------------------------
+     * writable below here
+     *   FIXME case 9650: we can make the prefixes read-only after
+     *   loading if we put them on their own page
+     *
+     *   fcache_return_prefix
+     *   trace_head_return_prefix
+     *   ibl_ret_prefix
+     *   ibl_call_prefix
+     *   ibl_jmp_prefix
+     *   stubs
+     *
+     ****** End offset-sensitive sequence */
+
+    /* persisted_footer_t here */
+
+    /* We place a guard page here at load time if we fill out the allocation
+     * region.  We don't need one in front since we have read-only pages there.
+     */
+
+} coarse_persisted_info_t;
+
+bool
+coarse_unit_persist(dcontext_t *dcontext, coarse_info_t *info);
+
+coarse_info_t *
+coarse_unit_load(dcontext_t *dcontext, app_pc start, app_pc end,
+                 bool for_execution);
+
+bool
+exists_coarse_ibl_pending_table(dcontext_t *dcontext,
+                                coarse_info_t *info, ibl_branch_type_t branch_type);
+
+/* Checks for enough space on the volume where persisted caches are stored */
+bool
+coarse_unit_check_persist_space(file_t fd_in/*OPTIONAL*/, size_t size_needed);
+
+/* If pc is in a module, marks that module as exempted (case 9799) */
+void
+mark_module_exempted(app_pc pc);
+
+#endif /* _PERSCACHE_H_ */
diff --git a/core/rct.c b/core/rct.c
new file mode 100644
index 000000000..1bc4a41c3
--- /dev/null
+++ b/core/rct.c
@@ -0,0 +1,577 @@
+/* **********************************************************
+ * Copyright (c) 2004-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2004-2007 Determina Corp. */
+
+/*
+ * rct.c
+ * Routines for the security features related to indirect calls and indirect jumps
+ * in a platform independent manner.
+ *
+ */
+
+#include "globals.h"
+#ifdef RCT_IND_BRANCH
+
+#include "fragment.h"
+#include "rct.h"
+#include "module_shared.h"
+
+#ifdef WINDOWS
+# include "nudge.h" /* for generic_nudge_target() */
+#endif
+
+#ifdef X64
+#  include "instr.h" /* for instr_raw_is_rip_rel_lea */
+#endif
+
+/* General assumption all indirect branch targets on X86 will have an
+ * absolute address encoded in the code or data sections of the binary
+ * (e.g. address taken functions)
+ *
+ * Should go through each module's (non-zero) image sections meaning
+ * [module_base,+modulesize) and look for any address pointing to the
+ * code section(s) [baseof_code_section,+sizeof_code_section) for each
+ * code section.  
+ *
+ * FIXME: (optimization) Since there can be multiple code sections we
+ * have to do this multiple times for each one in a module.  While
+ * this may result in multiple passes with the current interface,
+ * don't optimize before shown to be a hit.
+ *
+ * FIXME: (optimization) In case there are heavy-weight resource
+ * sections (dialogues, etc.) we may want to skip them.
+ *
+ */
+
+
+/* Only a single thread should be traversing new modules */
+/* Currently this overlaps with the table_rwlock of global_rct_ind_targets */
+DECLARE_CXTSWPROT_VAR(mutex_t rct_module_lock, INIT_LOCK_FREE(rct_module_lock));
+
+/* look in text[start,end) memory range for any reference to values
+ *  in referto[start,end) address range
+ * Caller is responsible for ensuring that [start,end) is readable.
+ * 
+ * TODO: add any such addresses to an OUT parameter hashtable
+ * returns number of added references (for diagnostic purposes)
+ */
+uint
+find_address_references(dcontext_t *dcontext,
+                        app_pc text_start, app_pc text_end, 
+                        app_pc referto_start, app_pc referto_end
+                        )
+{
+    uint references_found = 0;  /* only for debugging  */
+    DEBUG_DECLARE(uint references_already_known = 0;)
+    
+    app_pc cur_addr;
+    app_pc last_addr = text_end - sizeof(app_pc); /* inclusive */
+
+    LOG(GLOBAL, LOG_RCT, 2, "find_address_references: text["PFX", "PFX"), referto["PFX", "PFX")\n",
+        text_start, text_end, referto_start, referto_end);
+
+    ASSERT(text_start <= text_end); /* empty ok */
+    ASSERT(referto_start <= referto_end); /* empty ok */
+
+    ASSERT(sizeof(app_pc) == IF_X64_ELSE(8,4));
+    ASSERT((ptr_uint_t)(last_addr+1) == (((ptr_uint_t)last_addr)+1)); /* byte increments */
+
+    ASSERT(is_readable_without_exception(text_start, text_end - text_start));
+
+    /* FIXME: could try to read dword[pc] dword[pc+4] and then merging them with shifts and |  */
+    /* to get dword[pc+1] dword[pc+2] dword[pc+3]  instead of reading memory  */
+    /* but of course only if KSTAT says the latter is indeed faster! */
+
+    KSTART(rct_no_reloc);
+    for (cur_addr = text_start; cur_addr <= last_addr; cur_addr++) {
+        DEBUG_DECLARE(bool known_ref = false;)
+
+        app_pc ref = *(app_pc*)cur_addr; /* note dereference here */
+        if (rct_check_ref_and_add(dcontext, ref, referto_start, referto_end
+                                  _IF_DEBUG(cur_addr)
+                                  _IF_DEBUG(&known_ref)))
+            references_found++;
+        else {
+            DODEBUG({
+                if (known_ref)
+                    references_already_known++;
+            });
+        }
+#ifdef X64
+        /* PR 215408: look for "lea reg, [rip+disp]" */
+        ref = instr_raw_is_rip_rel_lea(cur_addr, text_end);
+        if (ref != NULL) {
+            LOG(GLOBAL, LOG_RCT, 4,
+                "find_address_references: rip-rel @"PFX" => "PFX"\n", cur_addr, ref);
+            if (rct_check_ref_and_add(dcontext, ref, referto_start, referto_end
+                                      _IF_DEBUG(cur_addr) _IF_DEBUG(&known_ref))) {
+                STATS_INC(rct_ind_rip_rel_scan_new);
+                references_found++;
+            } else {
+                DODEBUG({
+                    if (known_ref) {
+                        references_already_known++;
+                        STATS_INC(rct_ind_rip_rel_scan_old);
+                    } else
+                        STATS_INC(rct_ind_rip_rel_scan_data);
+                });
+            }
+        }
+#endif
+    }
+    KSTOP(rct_no_reloc);
+
+    LOG(GLOBAL, LOG_RCT, 2,
+        "find_address_references: scanned %u addresses, touched %u pages, "
+        "added %u new, %u duplicate ind targets\n", 
+        (last_addr - text_start), (last_addr - text_start) / PAGE_SIZE,
+        references_found, references_already_known);
+    
+    return references_found;
+}
+
+/* Check if the given address refers to [referto_start, refereto_end) and
+ * add it to a hashtable of valid indirect branches.  Returns true if ref
+ * is added to hashtable, false otherwise.
+ * Optional OUT:  known_ref -- true if ref was already known
+ *
+ * FIXME: called from find_relocation_addresses, if perf critical make
+ * this static inline.
+ * FIXME: make find_address_references call this once static inlined
+ */
+
+bool
+rct_check_ref_and_add(dcontext_t *dcontext, app_pc ref,
+                      app_pc referto_start, app_pc referto_end
+                      _IF_DEBUG(app_pc addr)
+                      _IF_DEBUG(bool *known_ref /* OPTIONAL OUT */))
+{
+    /* can be null when called from find_address_references */
+    if (ref == NULL)
+        return false;
+
+    DODEBUG({
+        if (known_ref != NULL)
+            *known_ref = false;
+    });
+
+    /* reference outside range */
+    if (ref < referto_start || ref >= referto_end) {
+        return false;
+    }
+
+    /* indeed points to a code section */
+    DOLOG(3, LOG_RCT, { 
+        char symbuf[MAXIMUM_SYMBOL_LENGTH];
+        LOG(GLOBAL, LOG_RCT, 3, 
+            "rct_check_ref_and_add:  "PFX" addr taken reference at "PFX"\n",
+            ref, addr);
+        print_symbolic_address(ref, symbuf, sizeof(symbuf), true);
+        LOG(GLOBAL, LOG_SYMBOLS, 3, "\t%s\n", symbuf);
+    });
+
+    if (rct_add_valid_ind_branch_target(dcontext, ref)) {
+        STATS_INC(rct_ind_branch_valid_targets);
+        LOG(GLOBAL, LOG_RCT, 3, "\t "PFX" added\n", ref);
+
+        return true;  /* ref added, known_ref == false */
+    } else {
+        STATS_INC(rct_ind_branch_existing_targets);
+        LOG(GLOBAL, LOG_RCT, 3, "\t known\n");
+
+        DODEBUG({
+            if (known_ref != NULL)
+                *known_ref = true;
+        });
+
+        return false;  /* ref not added, known_ref == true */
+    }
+    ASSERT_NOT_REACHED();
+}
+
+/* There are several alternative solutions to keeping the information
+ * about known valid indirect branch targets: a single global
+ * hashtable, per-module hashtables.  We can use the latter for fast flushes
+ * yet we'd need a binary search to find the right module hashtable,
+ * or we can merge all module hashtables into the global one for lookups.
+ *
+ * Considering that we will want to have hashtables that provide the
+ * intersection of the above allowed targets with existing traces (or
+ * in the near future bbs), having yet another middle level table may
+ * not be worth it.  If we do have bb2bb then we'd have to do this
+ * much less often and the cost of the module search will not be
+ * exposed, and in case it is we can at least cache the last two
+ * looked up entries (like vmareas caches last_area).
+ *
+ * FIXME: for now using a single global hashtable and flushing of the whole table
+ * is the only provided implementation
+ */
+
+/* TODO: pass the necessary bookkeeping hashtable if done per-module */
+static inline bool
+is_address_taken(dcontext_t *dcontext, app_pc target)
+{
+    return rct_ind_branch_target_lookup(dcontext, target) != NULL;
+}
+
+static inline bool
+is_address_after_call(dcontext_t *dcontext, app_pc target)
+{
+    /* FIXME: practically equivalent to find_call_site() */
+    return fragment_after_call_lookup(dcontext, target) != NULL;
+}
+
+/* Restricted control transfer check on indirect branches called by
+ * dispatch after inlined indirect branch lookup routine has failed
+ *
+ * function does not return if a security violation is blocked
+ * FIXME: return value is ignored
+ */
+int
+rct_ind_branch_check(dcontext_t *dcontext, app_pc target_addr, app_pc src_addr)
+{
+    bool is_ind_call = TEST(LINK_CALL, dcontext->last_exit->flags);
+    security_violation_t indirect_branch_violation = is_ind_call ? 
+        INDIRECT_CALL_RCT_VIOLATION : INDIRECT_JUMP_RCT_VIOLATION;
+    int res = 0;
+    bool cache = true;
+    DEBUG_DECLARE(const char *ibranch_type = 
+                  is_ind_call ? "call" : "jmp";)
+    ASSERT(is_ind_call || TEST(LINK_JMP, dcontext->last_exit->flags));
+
+    ASSERT((is_ind_call && TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call))) ||
+           (!is_ind_call && TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump))));
+
+    LOG(THREAD, LOG_RCT, 2, "RCT: ind %s target = "PFX"\n", ibranch_type,
+        target_addr);
+
+    STATS_INC(rct_ind_branch_validations);
+
+    DOSTATS({
+        /* since the exports are now added to the global hashtable, 
+         * we have to check this first to collect these stats
+         */
+        /* FIXME: we need symbols at loglevel 0 for this to work */
+        if (rct_is_exported_function(target_addr)) {
+            if (is_ind_call)
+                STATS_INC(rct_ind_call_exports);
+            else
+                STATS_INC(rct_ind_jmp_exports);
+
+            if (DYNAMO_OPTION(IAT_convert)) {
+                LOG(THREAD, LOG_RCT, 2, 
+                    "RCT: address taken export or IAT conversion missed for "PFX, target_addr);
+                /* the module entry point is in fact hit here */
+                /* FIXME: investigate if an export is really not used via IAT or a variation of register */
+            }
+        }
+    });
+
+    /* FIXME: if we use per-module hashtables, we'd first have to
+     * looking up the module in question */
+    if (!is_address_taken(dcontext, target_addr)) {
+        bool is_code_section;
+        /* FIXME: use loglevel 2 when the define is on by default */
+        LOG(THREAD, LOG_RCT, 1, "RCT: bad ind %s target: "PFX", source "PFX"\n",
+            ibranch_type, target_addr, src_addr);
+
+        DOLOG(2, LOG_RCT, { 
+            char symbuf[MAXIMUM_SYMBOL_LENGTH];
+            print_symbolic_address(target_addr, symbuf, sizeof(symbuf), true);
+            LOG(THREAD, LOG_SYMBOLS, 2, "\t%s\n", symbuf);
+        });
+
+#ifdef DR_APP_EXPORTS
+        /* case 9195: Allow certain API calls so we don't get security violations
+         * when using DR with the start / stop interface.
+         * NOTE: this is a security hole so should never be in product build
+         */
+        if (target_addr == (app_pc) dr_app_start ||
+            target_addr == (app_pc) dr_app_take_over ||
+            target_addr == (app_pc) dr_app_stop ||
+            target_addr == (app_pc) dr_app_cleanup)
+            goto good;
+#endif
+        if (!is_ind_call) {
+            /* for DYNAMO_OPTION(rct_ind_jump) we need to allow an
+             * after call location (normally a target of return) to be
+             * targeted.
+             * Instances in ole32.dll abound.
+             * 
+             * 77a7f057 e8ac2ffdff       call    ole32!IIDFromString+0xf6 (77a52008)
+             * ole32!IIDFromString+0x107:
+             * 77a52022 8b08             mov     ecx,[eax]
+             * 77a52024 8b4004           mov     eax,[eax+0x4]
+             * 77a52027 ffe0             jmp     eax
+             * 
+             * We would need to ensure that ret_after_call is turned
+             * on, FIXME: should turn into a security_option_t that needs
+             * to be at least OPTION_ENABLED
+             */
+            
+            if (DYNAMO_OPTION(ret_after_call)) {
+                if (is_address_after_call(dcontext, target_addr)) {
+                    LOG(THREAD, LOG_RCT, 1, 
+                        "RCT: bad ind jump targeting an after call site: "PFX"\n", 
+                        target_addr);
+                    STATS_INC(rct_ind_jmp_allowed_to_ac);
+                    /* the current thread's indirect jmp IBL table will cache this */
+                    goto good;
+                }
+            } else {
+                /* case 4982: we can't use RAC data - we'll get a violation
+                 * FIXME: add better option enforcement after making ret_after_call a security_option_t
+                 */
+                ASSERT_NOT_IMPLEMENTED(false);
+            }
+        }
+
+        /* PR 275723: RVA-table-based switch statements.  We check this prior to
+         * rct_analyze_module_at_violation to avoid excessive scanning of x64
+         * modules from PR 277044/277064.
+         */
+        if (DYNAMO_OPTION(rct_exempt_intra_jmp)) {
+            app_pc code_start, code_end;
+            app_pc modbase = get_module_base(target_addr);
+            if (modbase != NULL &&
+                is_in_code_section(modbase, target_addr, &code_start, &code_end) &&
+                src_addr >= code_start && src_addr < code_end) {
+                STATS_INC(rct_ind_jmp_x64switch);
+                STATS_INC(rct_ind_jmp_exemptions);
+                LOG(THREAD, LOG_RCT, 2,
+                    "RCT: target "PFX" in same code sec as src "PFX" --ok\n",
+                    target_addr, src_addr);
+                /* Though we have per-module tables (except on Linux) (PR 214107)
+                 * we don't check by source (and xref PR 204770) and don't have
+                 * per-module ibl tables, so we can't cache this.
+                 */
+                cache = false;
+                res = 2;
+                goto exempted;
+            }
+        }
+
+        /* Grab the rct_module_lock to ensure no duplicates if two threads
+         * attempt to add the same module
+         */
+        mutex_lock(&rct_module_lock);
+        /* NOTE - under current default options (analyze_at_load) this routine will have
+         * no effect and is only being used for its is_in_code_section (&IMAGE) return
+         * value.  For x64 it is used to scan on violation (PR 277044/277064). */
+        is_code_section = rct_analyze_module_at_violation(dcontext, target_addr);
+        mutex_unlock(&rct_module_lock);
+
+        /* In fact, we have to let all regions that are not modules - so 
+         * .A and .B attacks will still be marked as such instead of failing here.
+         */
+        if (!is_code_section) {
+            /* FIXME: assert DGC */
+            /* we could be targeting a .data section that is within a
+             * module we still don't cache it but only a performance
+             * hit - it will not be reported anyways
+             */
+            STATS_INC(rct_ind_branch_not_code_section);
+
+            /* ASLR: check if is in wouldbe region, if so report as failure */
+            if (aslr_is_possible_attack(target_addr)) {
+                LOG(THREAD, LOG_RCT, 1, 
+                    "RCT: ASLR: wouldbe a preferred DLL, "PFX" --BAD\n", target_addr);
+                STATS_INC(aslr_rct_ind_wouldbe);
+                /* fall through and report */
+            } else {
+                LOG(THREAD, LOG_RCT, 1, "RCT: not a code section, ignoring "PFX" --check\n", target_addr);
+                /* not caching violation target if not in code section */
+                return 2;       /* positive, ignore violation */
+            }
+        }
+
+        /* we could be running in an unload race, and is_code_section
+         * would still be true if target is not yet unmapped, if it
+         * just became unreadable we also ignore like above
+         */
+        if (is_unreadable_or_currently_unloaded_region(target_addr)) {
+            STATS_INC(rct_ind_branch_unload_race);
+            LOG(THREAD, LOG_RCT, 1, "RCT: unload race, ignoring "PFX" --check\n", target_addr);
+            /* not caching violation target since it will disappear */
+            /* note that we should throw exception while processing code origin checks */
+            return 2;       /* positive, ignore violation */
+        }
+
+        if (is_address_taken(dcontext, target_addr)) {
+            LOG(THREAD, LOG_RCT, 1, "RCT: new module added for "PFX" --ok\n", target_addr);
+            STATS_INC(rct_ok_at_vio);
+            goto good;
+        }
+
+        /* FIXME: this code will not be needed if the exports are
+         * automatically added to the list yet I'd like to keep it
+         * around for the STATS that may be relevant to case 1948
+         */
+        /* FIXME: case 3946 we need symbols at loglevel 0 to collect these stats */
+        DOLOG(1, LOG_RCT|LOG_SYMBOLS, {
+            /* this is an expensive bsearch, so we may not want to
+             * collect it as other stats */
+            if (rct_is_exported_function(target_addr)) {
+                DOSTATS({
+                    if (is_ind_call)
+                        STATS_INC(rct_ind_call_exports);
+                    else
+                        STATS_INC(rct_ind_jmp_exports);
+                });
+                SYSLOG_INTERNAL_WARNING_ONCE("missed an export "PFX" caught by "
+                                             "rct_is_exported_function()", target_addr);
+                DOLOG(1, LOG_RCT, {
+                    char name[MAXIMUM_SYMBOL_LENGTH];
+                    print_symbolic_address(target_addr, name, sizeof(name), false);
+                    LOG(THREAD, LOG_RCT, 1, "RCT: exported function "PFX" %s missed!\n", target_addr, name);
+                });
+            }
+        });
+
+        LOG(THREAD, LOG_RCT, 1,
+            "RCT: BAD[%d]  problem target="PFX" src fragment="PFX" type=%s\n",
+            GLOBAL_STAT(rct_ind_call_violations) + GLOBAL_STAT(rct_ind_jmp_violations), 
+            target_addr, src_addr, ibranch_type);
+
+        /* FIXME: case 4331, as a minimal change for 2.1, this
+         * currently reuses several .C exceptions, while in fact only
+         * the -exempt_rct list is needed (but not fibers)
+         */
+        if (at_known_exception(dcontext, target_addr, src_addr)) {
+            LOG(THREAD, LOG_RCT, 1, "RCT: target "PFX" exempted --ok\n");
+            DOSTATS({
+                if (is_ind_call)
+                    STATS_INC(rct_ind_call_exemptions);
+                else
+                    STATS_INC(rct_ind_jmp_exemptions);
+            });
+
+            res = 2;
+            /* we'll cache violation target */
+            goto exempted;
+        }
+
+        DOSTATS({
+            if (is_ind_call)
+                STATS_INC(rct_ind_call_violations);
+            else
+                STATS_INC(rct_ind_jmp_violations);
+        });
+        SYSLOG_INTERNAL_WARNING_ONCE("indirect %s targeting unknown "PFX, 
+                                     TEST(LINK_CALL, dcontext->last_exit->flags)
+                                     ? "call" : "jmp", target_addr);
+        /* does not return when OPTION_BLOCK is enforced */
+        if (security_violation(dcontext, target_addr, indirect_branch_violation, 
+                               is_ind_call ? DYNAMO_OPTION(rct_ind_call) : DYNAMO_OPTION(rct_ind_jump)) ==
+            indirect_branch_violation) {
+            /* running in detect mode */
+            /* we'll cache violation target */
+            res = -2;
+        } else {
+            res = -3;
+            /* exempted Threat ID */
+            /* we'll cache violation target */
+        }
+
+    exempted:                  
+        /* Exempted or bad in detect mode, either way should add the
+         * violating address so future references to it in other
+         * threads do not fail.
+         */
+        if (DYNAMO_OPTION(rct_cache_exempt) != RCT_CACHE_EXEMPT_NONE) {
+            mutex_lock(&rct_module_lock);
+            rct_add_valid_ind_branch_target(dcontext, target_addr);
+            mutex_unlock(&rct_module_lock);
+        }
+        return res;
+    }
+ good:
+    LOG(THREAD, LOG_RCT, 3, "RCT: good ind %s to "PFX"\n", ibranch_type, target_addr);
+    DOSTATS({
+        if (is_ind_call)
+            STATS_INC(rct_ind_call_good);
+        else
+            STATS_INC(rct_ind_jmp_good);
+    });
+    return 1;
+}
+
+#ifdef WINDOWS
+/* Add allowed targets in dynamorio.dll.
+ * Note: currently only needed for WINDOWS
+ * Note: needs dynamo_dll_start to be initialized (currently done by
+ *       vmareas_init->find_dynamo_library_vm_areas)
+ */
+void
+rct_known_targets_init(void)
+{
+    /* Allow targeting dynamorio!generic_nudge_target and
+     * dynamorio!safe_apc_or_thread_target (case 7266)
+     */
+    mutex_lock(&rct_module_lock);
+
+    ASSERT(is_in_dynamo_dll((app_pc)generic_nudge_target));
+    rct_add_valid_ind_branch_target(GLOBAL_DCONTEXT, (app_pc)generic_nudge_target);
+
+    ASSERT(is_in_dynamo_dll((app_pc)safe_apc_or_thread_target));
+    rct_add_valid_ind_branch_target(GLOBAL_DCONTEXT,
+                                   (app_pc)safe_apc_or_thread_target);
+
+    mutex_unlock(&rct_module_lock);
+}
+#endif
+
+void
+rct_init(void)
+{
+    if (!(TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) ||
+          TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump)))) {
+        ASSERT(!(TESTANY(OPTION_REPORT|OPTION_BLOCK, DYNAMO_OPTION(rct_ind_call)) ||
+                 TESTANY(OPTION_REPORT|OPTION_BLOCK, DYNAMO_OPTION(rct_ind_jump))));
+        return;
+    }
+    /* FIXME: hashtable_init currently in fragment.c should be moved here */
+    IF_WINDOWS(rct_known_targets_init());
+}
+
+void
+rct_exit(void)
+{
+    if (!(TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) ||
+          TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump))))
+        return;
+    
+    DELETE_LOCK(rct_module_lock);
+}
+
+#endif /* RCT_IND_BRANCH */
diff --git a/core/rct.h b/core/rct.h
new file mode 100644
index 000000000..8550bad64
--- /dev/null
+++ b/core/rct.h
@@ -0,0 +1,102 @@
+/* **********************************************************
+ * Copyright (c) 2004-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2004-2007 Determina Corp. */
+
+/*
+ * rct.h - RCT policy exports
+ */
+
+#ifndef RCT_H
+#define RCT_H
+
+enum {RCT_CACHE_EXEMPT_NONE = 0,
+      RCT_CACHE_EXEMPT_MODULES = 1,
+      RCT_CACHE_EXEMPT_ALL = 2, /* including DGC */
+};
+
+#ifdef RCT_IND_BRANCH
+/* public */
+int
+rct_ind_branch_check(dcontext_t *dcontext, app_pc target_addr, app_pc src_addr);
+
+bool
+rct_check_ref_and_add(dcontext_t *dcontext, app_pc ref, 
+                      app_pc referto_start, app_pc referto_end
+                      _IF_DEBUG(app_pc addr)
+                      _IF_DEBUG(bool *known_ref /* OPTIONAL OUT */));
+
+uint
+find_address_references(dcontext_t *dcontext, 
+                        app_pc text_start, app_pc text_end, 
+                        app_pc referto_start, app_pc referto_end);
+uint
+invalidate_ind_branch_target_range(dcontext_t *dcontext, 
+                                   app_pc text_start, app_pc text_end);
+
+void rct_known_targets_init(void);
+void rct_init(void);
+void rct_exit(void);
+
+/* private routines for rct.c and fragment.c */
+
+/* FIXME: once we move the needed pieces out into fragment.h then
+ * these routines can be moved from fragment.c and be static in rct.c
+ */
+
+#include "fragment.h"
+
+/* returns NULL if not found */
+app_pc
+rct_ind_branch_target_lookup(dcontext_t *dcontext, app_pc tag);
+
+bool
+rct_add_valid_ind_branch_target(dcontext_t *dcontext, app_pc tag);
+
+void
+rct_flush_ind_branch_target_entry(dcontext_t *dcontext, app_pc tag);
+
+bool
+rct_is_exported_function(app_pc tag);  /* in module.c */
+
+#ifdef X64
+/* in module.c */
+bool
+rct_add_rip_rel_addr(dcontext_t *dcontext, app_pc tgt _IF_DEBUG(app_pc src));
+#endif
+
+/* exported for ASSERT's to verify it's held */
+extern mutex_t rct_module_lock;
+
+#endif /* RCT_IND_BRANCH */
+
+#endif /* RCT_H */
diff --git a/core/stats.c b/core/stats.c
new file mode 100644
index 000000000..6d10a9006
--- /dev/null
+++ b/core/stats.c
@@ -0,0 +1,457 @@
+/* **********************************************************
+ * Copyright (c) 2004-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2004-2007 Determina Corp. */
+
+/*
+ * stats.c - statistics related functionality
+ */
+
+/* 
+ * Note that timer numbers from TSC cannot fully adjust for thread
+ * context switches.  We truly want a virtual time stamp counter that
+ * is thread specific, yet without OS support that's not possible.  If
+ * we could read the ETHREAD data that counts number of context
+ * switches, we should also regularly print the ThreadTimes data from
+ * ZwQueryInformationThread for sanity checking.  See
+ * KSTAT_OUTLIER_THRESHOLD_MS for the current solution of not adding
+ * up at least the largest outliers, although it can't help with all.
+ *
+ * We could also add sampling collected statistics on platforms on which
+ * we can get custom action on profiling interrupts (Linux only).
+ *
+ * Numbers don't seem to be very reliable in VMware which isn't that
+ * surprising - considering RDTSC is not easy to virtualize.  Even the
+ * QueryPerformanceFrequency call is not producing meaningful data,
+ * and minimums of 1 are often seen.
+ */
+
+#include "globals.h"
+#include "dr_stats.h"
+#include "stats.h"
+
+#include <string.h>  /* for memset */
+
+#ifdef KSTATS
+
+#ifdef KSTAT_UNIT_TEST 
+/* FIXME: add a Makefile rule for this */
+kstat_variables_t tkv;
+kstat_stack_t ks;
+#endif
+
+/* STATIC forward declarations */
+static void
+kstat_merge_var(kstat_variable_t *destination, kstat_variable_t *source);
+
+
+
+static void
+kstat_init_variable(kstat_variable_t *kv)
+{
+    memset(kv, 0x0, sizeof(kstat_variable_t));
+    kv->min_cum = (timestamp_t)-1;
+}
+
+/* process all sum equations, any more complicated expressions should
+ * better be done out of the core
+ */
+static void
+kstats_evaluate_expressions(kstat_variables_t *kvars)
+{
+    /* sum can be recomputed at any time and target is reinitialized,
+     * all chained KSTAT_SUM equations should appear in evaluation order
+     */
+#define KSTAT_SUM(desc, name, var1, var2)               \
+        kstat_init_variable(&kvars->name);              \
+        kstat_merge_var(&kvars->name, &kvars->var1);    \
+        kstat_merge_var(&kvars->name, &kvars->var2);
+#define KSTAT_DEF(desc, name)   /* nothing to do */
+#include "kstatsx.h"    
+#undef KSTAT_SUM
+#undef KSTAT_DEF
+}
+
+/* equivalent to KSTAT_DEF for the rest of the file */
+#define KSTAT_SUM(desc, name, var1, var2) KSTAT_DEF(desc, name)
+
+static void
+kstat_init_variables(kstat_variables_t *ks)
+{
+#define KSTAT_DEF(desc, name) kstat_init_variable(&ks->name);
+#include "kstatsx.h"
+#undef KSTAT_DEF
+}
+
+/* There is no good minimum value here since a thread can get switched
+ * back in a shorter time slice, in case another thread is waiting or
+ * has yielded its share.  If valid measurements are indeed taking
+ * longer than 1ms then checkpoints in between will be needed to count
+ * these properly.  However, a millisecond is quite a lot of time and
+ * we shouldn't be doing anything like that.
+ */
+enum {KSTAT_OUTLIER_THRESHOLD_MS = 1}; /* 1 ms for now */
+timestamp_t kstat_ignore_context_switch; 
+
+static timestamp_t
+kstat_frequency_per_msec;
+
+/* PR 312534: reduce stack usage (gcc does not combine locals) */
+static void
+kstat_print_individual(file_t outf, kstat_variable_t *kv,
+                       const char *name, const char *desc)
+{
+    print_file(outf, "%20s:"FIXED_TIMESTAMP_FORMAT" totc,"
+               "%8u num,"
+               FIXED_TIMESTAMP_FORMAT" minc,"
+               FIXED_TIMESTAMP_FORMAT" avg,"
+               FIXED_TIMESTAMP_FORMAT" maxc,"
+               FIXED_TIMESTAMP_FORMAT" self,"
+               FIXED_TIMESTAMP_FORMAT" sub,"
+               "\n"
+               "                   "
+               FIXED_TIMESTAMP_FORMAT" ms,"
+               FIXED_TIMESTAMP_FORMAT" ms out,"
+               "%s\n",
+               name,
+               kv->total_self + kv->total_sub,
+               kv->num_self,
+               (kv->min_cum == (timestamp_t)-1) ? 0 : kv->min_cum,
+               (kv->total_self + kv->total_sub) / kv->num_self,
+               kv->max_cum,
+               kv->total_self,
+               kv->total_sub,
+               (kv->total_self + kv->total_sub) / kstat_frequency_per_msec,
+               kv->total_outliers/ kstat_frequency_per_msec,
+               desc);
+}
+
+static void
+kstat_report(file_t outf, kstat_variables_t *ks)
+{
+    kstats_evaluate_expressions(ks);
+    /* FIXME: Outliers may make the minc number appear smaller than
+     * real, should at least mark with a '*' */
+#define KSTAT_DEF(desc, name)                   \
+    if (ks->name.num_self)                      \
+        kstat_print_individual(outf, &ks->name, #name, desc);
+#include "kstatsx.h"                                          
+#undef KSTAT_DEF
+}
+
+DECLARE_CXTSWPROT_VAR(mutex_t process_kstats_lock, INIT_LOCK_FREE(process_kstats_lock));
+DECLARE_NEVERPROT_VAR(kstat_variables_t process_kstats, {{0,}});
+DECLARE_NEVERPROT_VAR(file_t process_kstats_outfile, INVALID_FILE);
+
+/* Log files are only needed for non-debug builds. */
+#ifndef DEBUG
+static const char *
+kstats_main_logfile_name(void)
+{
+    return "process-kstats";
+}
+
+static char *
+kstats_thread_logfile_name()
+{
+    return "kstats";
+}
+#endif
+
+void
+kstat_init()
+{
+    kstat_frequency_per_msec = get_timer_frequency();
+    kstat_ignore_context_switch = KSTAT_OUTLIER_THRESHOLD_MS * kstat_frequency_per_msec;
+
+    LOG(GLOBAL, LOG_STATS, 1, "Processor speed: "UINT64_FORMAT_STRING"MHz\n", 
+        kstat_frequency_per_msec/1000);
+
+    /* FIXME: There is no check for TSC feature and whether CR4.TSD is set
+     * so we can read it at CPL 3
+     */
+
+    if (!DYNAMO_OPTION(kstats))
+        return;
+
+    kstat_init_variables(&process_kstats);
+#ifdef DEBUG
+    process_kstats_outfile = GLOBAL;
+#else
+    /* Open a process-wide kstats file. open_thread_private_file() does the
+     * job when passed the appropriate basename (2nd arg).
+     */
+    process_kstats_outfile =
+        open_log_file(kstats_main_logfile_name(), NULL, 0);
+#endif
+}
+
+void
+kstat_exit()
+{
+    if (!DYNAMO_OPTION(kstats))
+        return;
+
+    /* report merged process statistics */
+    mutex_lock(&process_kstats_lock);
+    print_file(process_kstats_outfile, "Process KSTATS:\n");
+    kstat_report(process_kstats_outfile, &process_kstats);
+    mutex_unlock(&process_kstats_lock);
+
+    DELETE_LOCK(process_kstats_lock);
+}    
+
+static void
+kstat_calibrate()
+{
+    uint i;
+    static bool kstats_calibrated = false;
+    if (kstats_calibrated)
+        return;
+    kstats_calibrated = true; /* slight innocent race */
+
+    /* FIXME: once we calculate the overhead of calibrate_empty we can
+     * subtract that from every self_time measurement.  
+     * FIXME: The cost of
+     * overhead_nested-overhead_empty should be subtracted from each
+     * subpath_time.
+     */
+    for (i=0; i<10000; i++) {
+        KSTART(overhead_nested);
+          KSTART(overhead_empty);
+          KSTOP_NOT_PROPAGATED(overhead_empty);
+        KSTOP(overhead_nested);
+    }
+}
+
+void
+kstat_thread_init(dcontext_t *dcontext)
+{
+    thread_kstats_t *new_thread_kstats;
+    if (!DYNAMO_OPTION(kstats))
+        return;                 /* dcontext->thread_kstats stays NULL */
+
+    /* allocated on thread heap - use global if timing initialization matters */
+    new_thread_kstats = HEAP_TYPE_ALLOC(dcontext, thread_kstats_t, ACCT_STATS, UNPROTECTED);
+    LOG(THREAD, LOG_STATS, 2, "thread_kstats="PFX" size=%d\n", new_thread_kstats, 
+        sizeof(thread_kstats_t));
+    /* initialize any thread stats bookkeeping fields before assigning to dcontext */
+    kstat_init_variables(&new_thread_kstats->vars_kstats);
+    /* add a dummy node to save one branch in UPDATE_CURRENT_COUNTER */
+    new_thread_kstats->stack_kstats.depth = 1;
+
+    new_thread_kstats->thread_id = get_thread_id();
+#ifdef DEBUG
+    new_thread_kstats->outfile_kstats = THREAD;
+#else
+    new_thread_kstats->outfile_kstats =
+        open_log_file(kstats_thread_logfile_name(), NULL, 0);
+#endif
+    dcontext->thread_kstats = new_thread_kstats;
+
+    /* need to do this in a thread after it's initialized */
+    kstat_calibrate();
+
+    KSTART_DC(dcontext, thread_measured);
+
+    LOG(THREAD, LOG_STATS, 2, "threads_started\n");
+}
+
+static void
+kstat_merge_var(kstat_variable_t *destination, kstat_variable_t *source)
+{
+    destination->num_self += source->num_self;
+    destination->total_self += source->total_self;
+    destination->total_sub += source->total_sub;
+    destination->total_outliers += source->total_outliers;
+    if (destination->min_cum > source->min_cum) 
+        destination->min_cum = source->min_cum;
+    if (destination->max_cum < source->max_cum) 
+        destination->max_cum = source->max_cum;
+}
+
+/* make sure sourcevars are merged in only once */
+static void
+kstat_merge(kstat_variables_t *destinationvars, kstat_variables_t *sourcevars)
+{
+#define KSTAT_DEF(desc, name) kstat_merge_var(&destinationvars->name, &sourcevars->name);
+#include "kstatsx.h"    
+#undef KSTAT_DEF
+}
+
+void
+dump_thread_kstats(dcontext_t *dcontext)
+{
+    if (dcontext->thread_kstats == NULL)
+        return;
+
+    /* add thread id's in case outfile is rerouted to process_kstats_outfile */
+    print_file(dcontext->thread_kstats->outfile_kstats, "Thread %d KSTATS {\n",
+               dcontext->thread_kstats->thread_id);
+    kstat_report(dcontext->thread_kstats->outfile_kstats, 
+                 &dcontext->thread_kstats->vars_kstats);
+    print_file(dcontext->thread_kstats->outfile_kstats, "} KSTATS\n");
+}
+
+#ifdef DEBUG
+/* We don't keep the variable name, but instead we lookup by addr when necessary */
+/* Too convoluted solution but since not common case, we don't bother
+ * to initialize a name for each var in kstat_init_variables() */
+static char *
+kstat_var_name(dcontext_t *dcontext, kstat_variable_t *kvar)
+{
+    kstat_variables_t *kvs = &dcontext->thread_kstats->vars_kstats;
+#define KSTAT_DEF(desc, name) \
+    if (kvar == &kvs->name)   \
+        return #name;
+#include "kstatsx.h"
+#undef KSTAT_DEF
+    ASSERT_NOT_REACHED();
+    return "#ERROR#";
+}
+
+void
+kstats_dump_stack(dcontext_t *dcontext)
+{
+    uint i;
+    LOG(THREAD, 1, LOG_STATS, "Thread KSTAT stack:\n");
+    for (i = dcontext->thread_kstats->stack_kstats.depth - 1;
+         i > 0; i--) {
+        LOG(THREAD, 1, LOG_STATS, "[%d] "PIFX" %s\n",
+            i, &dcontext->thread_kstats->stack_kstats.node[i],
+            kstat_var_name(dcontext, dcontext->thread_kstats->stack_kstats.node[i].var));
+    }
+}
+#endif
+
+void
+kstat_thread_exit(dcontext_t *dcontext)
+{
+    thread_kstats_t *old_thread_kstats = dcontext->thread_kstats;
+    if (old_thread_kstats == NULL)
+        return;
+    LOG(THREAD, LOG_ALL, 2, "kstat_thread_exit: kstats stack is:\n");
+    DOLOG(2, LOG_STATS, {kstats_dump_stack(dcontext);});
+    KSTOP_DC(dcontext, thread_measured);
+    ASSERT(old_thread_kstats->stack_kstats.depth == 1);
+    dump_thread_kstats(dcontext);
+
+    /* a good time to combine all of these with the global statistics */
+    mutex_lock(&process_kstats_lock);
+    kstat_merge(&process_kstats, &old_thread_kstats->vars_kstats);
+    mutex_unlock(&process_kstats_lock);
+
+#ifdef DEBUG
+    /* for non-debug we do fast exit path and don't free local heap */
+    /* no clean up needed */
+    dcontext->thread_kstats = NULL; /* disable thread kstats before freeing memory */
+    HEAP_TYPE_FREE(dcontext, old_thread_kstats, thread_kstats_t, ACCT_STATS, UNPROTECTED);
+#endif /* DEBUG */
+}
+
+#endif /* KSTATS */
+
+#ifdef KSTAT_UNIT_TEST
+uint
+kstat_test()
+{
+    KSTART(measured);
+    printf("test %d\n", __LINE__);
+    KSTART(empty);
+    KSTOP(empty);
+    KSTART(empty);
+    KSTOP(empty);
+
+    printf("test %d\n", __LINE__);
+    KSTART(dr_default);
+    KSWITCH(dr_existing_bb);
+    KSTOP_NOT_MATCHING(dr_default);
+
+    KSTART(dr_default);
+    KSTART(empty);
+    KSTOP(empty);
+    KSTOP_NOT_MATCHING(dr_default);
+
+    KSTART(dr_default);
+    KSTOP_NOT_MATCHING(dr_default);
+
+    printf("test %d\n", __LINE__);
+    KSTART(wait_event);
+    KSTOP_NOT_PROPAGATED(wait_event);
+    printf("test %d\n", __LINE__);
+
+    { 
+        uint i;
+        for (i=0; i<100000; i++) {
+            KSTART(bb);
+            KSTOP(bb);
+        }
+    }
+
+    { 
+        uint i,j,k;
+        for (i=0; i<100; i++) {
+            KSTART(iloop);
+            for (j=0; j<100; j++) {
+                KSTART(jloop);
+                for (k=0; k<100000; k++)
+                    /* nothing */;
+                KSTOP(jloop);
+            }
+
+            KSTOP(iloop);
+        }
+    }
+
+
+    KSTART(syscalls);
+    KSTART(wait_event);
+    KSTOP_NOT_PROPAGATED(wait_event);
+
+    KSTART(wait_event);
+    KSTOP(wait_event);
+
+    KSTOP(syscalls);
+    printf("test %d\n", __LINE__);
+    KSTOP(measured);
+}
+int main()
+{
+    kstat_init();
+    kstat_thread_init();
+
+    kstat_test();
+    
+    kstat_thread_exit();
+    kstat_exit();
+}
+#endif /* KSTAT_UNIT_TEST */
diff --git a/core/stats.h b/core/stats.h
new file mode 100644
index 000000000..2d0994059
--- /dev/null
+++ b/core/stats.h
@@ -0,0 +1,301 @@
+/* **********************************************************
+ * Copyright (c) 2004-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2004-2007 Determina Corp. */
+
+/*
+ * stats.h - statistics related exports and internal macros
+ */
+
+#ifndef STATS_H
+#define STATS_H
+
+#ifdef KSTATS
+void
+kstat_init(void);
+void
+kstat_exit(void);
+void
+kstat_thread_init(dcontext_t *dcontext);
+void
+kstat_thread_exit(dcontext_t *dcontext);
+void
+dump_thread_kstats(dcontext_t *dcontext);
+
+/* for debugging only */
+#ifdef DEBUG
+void
+kstats_dump_stack(dcontext_t *dcontext);
+#endif
+
+/* Kstat variable */
+typedef struct {
+    /* number of executions */
+    uint num_self;
+    /* Currently only time data - more performance counters could be
+     * added in the future if necessary
+     */
+
+    /* We could always measure total elapsed time in subroutines, by
+     * keeping their start times on the stack.  However, it is better
+     * to use essentially a single running timer that gets reset every
+     * time we enter a new level, and then we add up all outstanding
+     * times.  This allows us to be more selective not to time spent
+     * waiting or when context switched.  
+     *
+     * It in effect lets use as a drop-in replacement a sampling based
+     * counter for time in self.  If we could add an action, every
+     * time we get a sample we can increment the TOS entry.
+     * Currently, we are measuring time at start/stop to calculate the
+     * exact time with some overhead.
+     */
+
+    timestamp_t total_self;       /* measured time in this block */
+    timestamp_t total_sub;        /* nested subpaths */
+    /* cumulative = self time + subpaths */
+    /* total time for all calls */
+    timestamp_t min_cum;          /* most stable number for the common case */
+    timestamp_t max_cum;          /* a high number shows need for subpaths */
+    timestamp_t total_outliers;   /* attempt to account for context switches */
+    /* VTune also keeps self_wait time, but we do not propagate that time up */
+} kstat_variable_t;
+
+/* All kstat variables - expanded as a structure instead of an array 
+ * referenced by index allows easy to read debugger pretty prints
+ */
+typedef struct {
+#define KSTAT_DEF(desc, name) kstat_variable_t name;
+#define KSTAT_SUM(description, name, var1, var2) kstat_variable_t name;
+#include "kstatsx.h"
+#undef KSTAT_SUM
+#undef KSTAT_DEF
+} kstat_variables_t;
+
+/* Stack entry for an active execution node */
+typedef struct {
+    kstat_variable_t *var;
+    timestamp_t      self_time;
+    timestamp_t      subpath_time;
+    timestamp_t      outlier_time;
+} kstat_node_t;
+
+enum {KSTAT_MAX_DEPTH = 16};
+
+/* Information about a current execution path */
+typedef struct {
+    uint      depth;
+    timestamp_t last_start_time;
+    timestamp_t last_end_time;
+    kstat_node_t node[KSTAT_MAX_DEPTH];
+} kstat_stack_t;
+
+/* Thread local context and collected data */
+typedef struct {
+    thread_id_t       thread_id;
+    kstat_variables_t vars_kstats;
+    kstat_stack_t     stack_kstats;
+    file_t           outfile_kstats;
+} thread_kstats_t;
+
+extern timestamp_t
+kstat_ignore_context_switch; 
+
+#define KSTAT_THREAD_NO_PV_START(dc) do {                                       \
+    dcontext_t *cur_dcontext = (dc);                                         \
+    if (cur_dcontext != NULL && cur_dcontext->thread_kstats != NULL) {          \
+        kstat_stack_t *ks = &cur_dcontext->thread_kstats->stack_kstats;    
+
+#define KSTAT_THREAD_NO_PV_END()                \
+    }                                           \
+} while (0)
+
+/* ensures statement can safely use ks=stack_kstats and pv=&vars_kstats.name,
+ */
+#define KSTAT_THREAD(name, statement)                           \
+        KSTAT_THREAD_NO_PV_START(get_thread_private_dcontext()) \
+        kstat_variable_t *pv = &cur_dcontext->thread_kstats->      \
+                            vars_kstats.name;                   \
+        UNUSED_VARIABLE(pv);                                    \
+        statement;                                              \
+        KSTAT_THREAD_NO_PV_END()
+
+#define KSTAT_OTHER_THREAD(dc, name, statement)                 \
+        KSTAT_THREAD_NO_PV_START(dc)                            \
+        kstat_variable_t *pv = &cur_dcontext->thread_kstats->      \
+                            vars_kstats.name;                   \
+        UNUSED_VARIABLE(pv);                                    \
+        statement;                                              \
+        KSTAT_THREAD_NO_PV_END()
+
+/* makes sure we're matching start/stop */
+# define KSTAT_TOS_MATCHING(name) KSTAT_THREAD(name, kstat_tos_matching_var(ks, pv))
+
+/* optional - serialize instruction stream before measurement on my
+ * laptop the overhead of an empty inner block is 95 cycles without
+ * serialization vs 222 cycles with serialization, 
+ * too much overhead for little extra stability */
+#define KSTAT_SERIALIZE_INSTRUCTIONS() /* no SERIALIZE_INSTRUCTIONS() */
+
+/* optional - accumulate outliers separately for comparable analysis,
+ * for 88 vs 98 cycles in inner block - definitely worth it, e.g. two
+ * simultaneously run loops get same user CPU as single run, although
+ * wall clock time is twice longer.
+ */
+#ifdef KSTAT_NO_OUTLIERS
+#  define CORRECT_OUTLIER(stmt) /* ignore extra logic */
+#else
+   /* We do not update self_time on outlier so that higher level stats
+    * are also able to discount the context switches from self.
+    */
+#  define CORRECT_OUTLIER(stmt) stmt
+#endif /* KSTAT_NO_OUTLIERS */
+
+
+/* update current timer before starting a new one */
+#define UPDATE_CURRENT_COUNTER(kstack) do {                     \
+    KSTAT_SERIALIZE_INSTRUCTIONS();                             \
+    RDTSC_LL((kstack)->last_end_time);                          \
+    /* a dummy entry at 0 is always present, callers ASSERT  */ \
+    CORRECT_OUTLIER(                                            \
+    if ((kstack)->last_end_time - (kstack)->last_start_time >   \
+         kstat_ignore_context_switch)                           \
+       (kstack)->node[(kstack)->depth - 1].outlier_time +=      \
+           (kstack)->last_end_time - (kstack)->last_start_time; \
+    else )                                                      \
+       (kstack)->node[(kstack)->depth - 1].self_time +=         \
+           (kstack)->last_end_time - (kstack)->last_start_time; \
+    (kstack)->last_start_time = (kstack)->last_end_time;        \
+} while (0)
+
+/* since we want to add kstats to logging functions we can't use LOG in it */
+#define KSTAT_INTERNAL_DEBUG(kstack)                                            \
+    LOG(THREAD, LOG_STATS, 3, "ks start="FIXED_TIMESTAMP_FORMAT                 \
+                                " end="FIXED_TIMESTAMP_FORMAT                   \
+                                " cur diff="FIXED_TIMESTAMP_FORMAT              \
+                                " depth=%d\n",                                  \
+        (kstack)->last_start_time, (kstack)->last_end_time,                     \
+        (kstack)->node[(kstack)->depth - 1].self_time, (kstack)->depth);
+
+#ifdef DEBUG
+/* avoid recursion in logging KSTAT - do not checkin uses */
+#define KSTAT_DUMP_STACK(kstack)                                        \
+    if ((kstack)->node[(kstack)->depth - 1].var !=                      \
+        &cur_dcontext->thread_kstats->vars_kstats.logging) {            \
+        LOG(THREAD_GET, LOG_STATS, 1, "ks %s:%d\n"__FILE__, __LINE__);  \
+        kstats_dump_stack(cur_dcontext);                                \
+    } 
+#endif
+
+/* Most of these could be static inline's but DEBUG builds aren't
+ * inlining these and we want to minimize the measurement overhead
+ * FIXME: consider replacing them if we have too many copies, it will
+ * also clean up the argument evaluation in KSTAT_THREAD_NO_PV.
+ *
+ * KSTAT_THREAD encloses these statements in a do {} while (0) block,
+ * FIXME: if DEBUG stats are at all interesting may want to remove the
+ * one's from here if a compiler in fact adds real loops - e.g. cl does
+ */
+
+/* starts a timer */
+#define kstat_start_var(kstack, pvar) do {              \
+   DODEBUG({if ((kstack)->depth >= KSTAT_MAX_DEPTH)     \
+        kstats_dump_stack(cur_dcontext);});             \
+    ASSERT((kstack)->depth < KSTAT_MAX_DEPTH            \
+           && "probably missing a STOP on return");     \
+    (kstack)->node[(kstack)->depth].var = pvar;         \
+    UPDATE_CURRENT_COUNTER((kstack));                   \
+    /* reset new counter */                             \
+    (kstack)->node[(kstack)->depth].subpath_time = 0;   \
+    (kstack)->node[(kstack)->depth].self_time = 0;      \
+    (kstack)->node[(kstack)->depth].outlier_time = 0;   \
+    (kstack)->depth++;                                  \
+} while(0)
+
+/* updates which variable will be counted */
+#define kstat_switch_var(kstack, pvar) (kstack)->node[(kstack)->depth - 1].var = (pvar)
+
+/* allow mismatched start/stop - for use with KSWITCH */
+#define kstat_stop_not_matching_var(kstack, pvar) do {                  \
+   timestamp_t last_cum;                                                  \
+   kstat_stop_not_propagated_var(kstack, pvar, &last_cum);              \
+   if ((kstack)->depth > 0)       /* update subpath in parent */        \
+      (kstack)->node[(kstack)->depth - 1].subpath_time += last_cum;     \
+} while (0)
+
+#define kstat_stop_matching_var(kstack, pvar) do {              \
+   DODEBUG({if (ks->node[ks->depth - 1].var != pvar)            \
+        kstats_dump_stack(cur_dcontext);});    \
+   ASSERT(ks->node[ks->depth - 1].var == pvar                   \
+          && "stop not matching TOS");                          \
+   kstat_stop_not_matching_var(ks, pvar);                       \
+} while (0)
+
+#define kstat_tos_matching_var(kstack, pvar) (ks->node[ks->depth - 1].var == pv)
+
+#define kstat_stop_not_propagated_var(kstack, pvar, pcum) do {                  \
+    ASSERT((kstack)->depth > 1);                                                \
+    UPDATE_CURRENT_COUNTER((kstack));                                           \
+    (kstack)->depth--;                                                          \
+    (kstack)->node[(kstack)->depth].var->num_self++;                            \
+    (kstack)->node[(kstack)->depth].var->total_self +=                          \
+         (kstack)->node[(kstack)->depth].self_time;                             \
+    (kstack)->node[(kstack)->depth].var->total_sub +=                           \
+         (kstack)->node[(kstack)->depth].subpath_time;                          \
+    (kstack)->node[(kstack)->depth].var->total_outliers +=                      \
+         (kstack)->node[(kstack)->depth].outlier_time;                          \
+    *pcum = (kstack)->node[(kstack)->depth].self_time +                         \
+            (kstack)->node[(kstack)->depth].subpath_time;                       \
+    /* FIXME: an outlier should be counted as a NaN for outliers on subpaths */ \
+    if (*pcum > 0 && (kstack)->node[(kstack)->depth].var->min_cum > *pcum)      \
+        (kstack)->node[(kstack)->depth].var->min_cum = *pcum;                   \
+    if ((kstack)->node[(kstack)->depth].var->max_cum < *pcum)                   \
+        (kstack)->node[(kstack)->depth].var->max_cum = *pcum;                   \
+} while (0)
+
+/* FIXME: we may have to add a type argument to KSTAT_DEF saying whether 
+ * a variable should be propagated or not - here we assume all are propagated
+ */
+#define kstat_stop_rewind_var(kstack, pvar) do {                \
+        kstat_stop_not_matching_var(kstack, ignored);           \
+} while (kstack->node[kstack->depth /* the removed */].var != pvar)
+
+/* This is essentially rewind-until, stopping BEFORE deleting pvar
+ * FIXME: we may have to add a type argument to KSTAT_DEF saying whether 
+ * a variable should be propagated or not - here we assume all are propagated
+ */
+#define kstat_stop_longjmp_var(kstack, pvar)                                  \
+    while (kstack->node[kstack->depth - 1 /* to be removed */].var != pvar) { \
+        kstat_stop_not_matching_var(kstack, ignored);                         \
+    }
+#endif /* KSTATS */
+
+#endif /* STATS_H */
diff --git a/core/synch.c b/core/synch.c
new file mode 100644
index 000000000..6185c886f
--- /dev/null
+++ b/core/synch.c
@@ -0,0 +1,1523 @@
+/* **********************************************************
+ * Copyright (c) 2008-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * thread.c - thread synchronization
+ */
+
+#include "globals.h"
+#include "synch.h"
+#include "instrument.h" /* is_in_client_lib() */
+#include "hotpatch.h" /* hotp_only_in_tramp() */
+#include "fragment.h" /* get_at_syscall() */
+#include "fcache.h" /* in_fcache() */
+#include <string.h> /* for memcpy */
+
+extern vm_area_vector_t *fcache_unit_areas; /* from fcache.c */
+
+static void
+synch_thread_yield(void);
+
+/* Thread-local data
+ */
+typedef struct _thread_synch_data_t {
+    /* the following three fields are used to synchronize for detach, suspend
+     * thread, terminate thread, terminate process */
+    /* synch_lock and pending_synch_count act as a semaphore */
+    /* for check_wait_at_safe_spot() must use a spin_mutex_t */
+    spin_mutex_t *synch_lock;
+    /* we allow pending_synch_count to be read without holding the synch_lock
+     * so all updates should be ATOMIC as well as holding the lock */
+    int pending_synch_count;
+    /* to guarantee that the thread really has this permission you need to hold
+     * the synch_lock when you read this value  */
+    thread_synch_permission_t synch_perm;
+    /* Only valid while holding all_threads_synch_lock and thread_initexit_lock.  Set
+     * to whether synch_with_all_threads was successful in synching this thread.
+     */
+    int synch_with_success;
+    /* Case 10101: allows threads waiting_at_safe_spot() to set their own
+     * contexts.  This use sometimes requires a full os-specific context, which
+     * we hide behind a generic pointer and a size.
+     */
+    dr_mcontext_t *set_mcontext;
+    void *set_context;
+    size_t set_context_size;
+#ifdef X64
+    /* PR 263338: we have to pad for alignment */
+    byte *set_context_alloc;
+#endif
+} thread_synch_data_t;
+
+
+/* This lock prevents more than one thread from being in the synch_with_all_
+ * threads method body at the same time (which would lead to deadlock as they
+ * tried to synchronize with each other)
+ */
+DECLARE_CXTSWPROT_VAR(mutex_t all_threads_synch_lock,
+                      INIT_LOCK_FREE(all_threads_synch_lock));
+
+/* pass either mc or both cxt and cxt_size */
+static void
+free_setcontext(dr_mcontext_t *mc, void *cxt, size_t cxt_size _IF_X64(byte *cxt_alloc))
+{
+    if (mc != NULL) {
+        ASSERT(cxt == NULL);
+        global_heap_free(mc, sizeof(*mc) HEAPACCT(ACCT_OTHER));
+    } else if (cxt != NULL) {
+        ASSERT(cxt_size > 0);
+        global_heap_free(IF_X64_ELSE(cxt_alloc, cxt), cxt_size HEAPACCT(ACCT_OTHER));
+    }
+}
+
+static void
+synch_thread_free_setcontext(thread_synch_data_t *tsd)
+{
+    free_setcontext(tsd->set_mcontext, tsd->set_context, tsd->set_context_size
+                    _IF_X64(tsd->set_context_alloc));
+    tsd->set_mcontext = NULL;
+    tsd->set_context = NULL;
+}
+
+void
+synch_init(void)
+{
+}
+
+void
+synch_exit(void)
+{
+    DELETE_LOCK(all_threads_synch_lock);
+}
+
+void
+synch_thread_init(dcontext_t *dcontext)
+{
+    thread_synch_data_t *tsd = (thread_synch_data_t *)
+        heap_alloc(dcontext, sizeof(thread_synch_data_t) HEAPACCT(ACCT_OTHER));
+    dcontext->synch_field = (void *) tsd;
+    tsd->pending_synch_count = 0;
+    tsd->synch_perm = THREAD_SYNCH_NONE;
+    tsd->synch_with_success = false;
+    tsd->set_mcontext = NULL;
+    tsd->set_context = NULL;
+    /* the synch_lock is in unprotected memory so that check_wait_at_safe_spot
+     * can call the EXITING_DR hook before releasing it */
+    tsd->synch_lock = HEAP_TYPE_ALLOC(dcontext, spin_mutex_t, ACCT_OTHER,
+                                       UNPROTECTED);
+    ASSIGN_INIT_SPINMUTEX_FREE(*tsd->synch_lock, synch_lock);
+}
+
+void
+synch_thread_exit(dcontext_t *dcontext)
+{
+    thread_synch_data_t *tsd = (thread_synch_data_t *) dcontext->synch_field;
+    /* Could be waiting at safe spot when we detach or exit */
+    synch_thread_free_setcontext(tsd);
+    DELETE_SPINMUTEX(*tsd->synch_lock);
+    /* Note that we do need to free this in non-debug builds since, despite
+     * appearances, UNPROTECTED_LOCAL is acutally allocated on a global
+     * heap. */
+    HEAP_TYPE_FREE(dcontext, tsd->synch_lock, spin_mutex_t, ACCT_OTHER,
+                   UNPROTECTED);
+#ifdef DEBUG
+    /* for non-debug we do fast exit path and don't free local heap */
+    /* clean up tsd fields here */
+    heap_free(dcontext, tsd, sizeof(thread_synch_data_t) HEAPACCT(ACCT_OTHER));
+#endif
+}
+
+/* Check for a no-xfer permission.  Currently used only for case 6821,
+ * where we need to distinguish three groups: unsafe (wait for safe
+ * point), safe and translatable, and safe but not translatable.
+ */
+bool
+thread_synch_state_no_xfer(dcontext_t *dcontext)
+{
+    thread_synch_data_t *tsd = (thread_synch_data_t *) dcontext->synch_field;
+    return (tsd->synch_perm == THREAD_SYNCH_NO_LOCKS_NO_XFER ||
+            tsd->synch_perm == THREAD_SYNCH_VALID_MCONTEXT_NO_XFER);
+}
+
+/* Only valid while holding all_threads_synch_lock and thread_initexit_lock.  Set to
+ * whether synch_with_all_threads was successful in synching this thread.
+ * Cannot be called when THREAD_SYNCH_*_AND_CLEANED was requested as the
+ * thread-local memory will be freed on success!
+ */
+bool
+thread_synch_successful(thread_record_t *tr)
+{
+    thread_synch_data_t *tsd;
+    ASSERT(tr != NULL && tr->dcontext != NULL);
+    ASSERT_OWN_MUTEX(true, &all_threads_synch_lock);
+    ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
+    tsd = (thread_synch_data_t *) tr->dcontext->synch_field;
+    return tsd->synch_with_success;
+}
+
+bool
+is_at_do_syscall(dcontext_t *dcontext, app_pc pc, byte *esp)
+{
+    app_pc buf[2];
+    bool res = safe_read(esp, sizeof(buf), buf);
+    if (!res) {
+        ASSERT(res); /* we expect the stack to always be readable */
+        return false;
+    }
+
+    if (does_syscall_ret_to_callsite()) {
+#ifdef WINDOWS
+        if (get_syscall_method() == SYSCALL_METHOD_INT && DYNAMO_OPTION(sygate_int)) {
+            return (pc == after_do_syscall_addr(dcontext) &&
+                    buf[0] == after_do_syscall_code(dcontext));
+        } else {
+            return pc == after_do_syscall_code(dcontext);
+        }
+#else
+        return is_after_do_syscall_addr(dcontext, pc);
+#endif
+    } else if (get_syscall_method() == SYSCALL_METHOD_SYSENTER) {
+#ifdef WINDOWS
+        if (pc == vsyscall_after_syscall) {
+            if (DYNAMO_OPTION(sygate_sysenter))
+                return buf[1] == after_do_syscall_code(dcontext);
+            else 
+                return buf[0] == after_do_syscall_code(dcontext);
+        } else {
+            /* not at a system call, could still have tos match after_do_syscall
+             * either by chance or because we leak that value on the apps stack
+             * (a non transparency) */
+            ASSERT_CURIOSITY(buf[0] != after_do_syscall_code(dcontext));
+            return false;
+        }
+#else
+        /* Even when the main syscall method is sysenter, we also have a
+         * do_int_syscall and do_clone_syscall that use int, so check both.
+         * Note that we don't modify the stack, so once we do sysenter syscalls
+         * inlined in the cache (PR 288101) we'll need some mechanism to
+         * distinguish those: but for now if a sysenter instruction is used it
+         * has to be do_syscall since DR's own syscalls are ints.
+         */
+        return (pc == vsyscall_sysenter_return_pc ||
+                is_after_do_syscall_addr(dcontext, pc));
+#endif
+    }
+    /* we can reach here w/ a fault prior to 1st syscall on Linux */
+    IF_WINDOWS(ASSERT_NOT_REACHED());
+    return false;
+}
+
+/* Helper function for at_safe_spot(). Note state for client-owned threads isn't
+ * considered valid since it may be holding client locks and doesn't correspond to
+ * an actual app state. Caller should handle client-owned threads appropriately. */
+static bool
+is_native_thread_state_valid(dcontext_t *dcontext, app_pc pc, byte *esp)
+{
+    /* ref case 3675, the assumption is that if we aren't executing
+     * out of dr memory and our stack isn't in dr memory (to disambiguate
+     * pc in kernel32, ntdll etc.) then the app has a valid native context.
+     * However, we can't call is_dynamo_address() as it (and its children)
+     * grab too many different locks, all of which we would have to check
+     * here in the same manner as fcache_unit_areas.lock in at_safe_spot().  So 
+     * instead we just check the pc for the dr dll, interception code, and 
+     * do_syscall regions and check the stack against the thread's dr stack 
+     * and the initstack, all of which we can do without grabbing any locks.
+     * That should be sufficient at this point, FIXME try to use something
+     * like is_dynamo_address() to make this more maintainable */
+    /* For sysenter system calls we also have to check the top of the stack
+     * for the after_do_syscall_address to catch the do_syscall @ syscall
+     * itself case. */
+    ASSERT(esp != NULL);
+    ASSERT(is_thread_currently_native(dcontext->thread_record));
+    return (!is_in_dynamo_dll(pc) &&
+            IF_WINDOWS(!is_in_interception_buffer(pc) &&)
+            !vmvector_overlap(landing_pad_areas, pc, pc + 1) &&
+            (!in_generated_routine(dcontext, pc) ||
+             /* we allow native thread to be at do_syscall - for int syscalls the pc
+              * (syscall return point) will be in do_syscall (so in generated routine)
+              * xref case 9333 */
+             is_at_do_syscall(dcontext, pc, esp)) &&
+            !is_on_initstack(esp) &&
+            !is_on_dstack(dcontext, esp) &&
+            IF_CLIENT_INTERFACE(!is_in_client_lib(pc) &&)
+            /* xref PR 200067 & 222812 on client-owned native threads */
+            IF_CLIENT_INTERFACE(!IS_CLIENT_THREAD(dcontext) &&)
+#ifdef HOT_PATCHING_INTERFACE
+            /* Shouldn't be in the middle of executing a hotp_only patch.  The
+             * check for being in hotp_dll is WHERE_HOTPATCH because the patch can 
+             * change esp.
+             */
+            (dcontext->whereami != WHERE_HOTPATCH &&
+             /* dynamo dll check has been done */ 
+             !hotp_only_in_tramp(pc)) &&
+#endif
+            true /* no effect, simplifies ifdef handling with && above */
+            );
+}
+
+/* Translates the context mcontext for the given thread trec.  If
+ * restore_memory is true, also restores any memory values that were
+ * shifted (primarily due to clients).  If restore_memory is true, the
+ * caller should always relocate the translated thread, as it may not
+ * execute properly if left at its current location (it could be in the
+ * middle of client code in the cache).
+ * 
+ * Like any instance where a thread_record_t is used by a thread other than its
+ * owner, the caller must hold the thread_initexit_lock to ensure that it
+ * remains valid.
+ * Requires thread trec is at_safe_spot().
+ */
+bool
+translate_mcontext(thread_record_t *trec, dr_mcontext_t *mcontext,
+                   bool restore_memory)
+{
+    thread_synch_data_t *tsd = (thread_synch_data_t *) trec->dcontext->synch_field;
+    bool res;
+    recreate_success_t success;
+    bool native_translate = false;
+    ASSERT(tsd->pending_synch_count >= 0);
+    /* check if native thread */
+    if (is_thread_currently_native(trec)) {
+        /* running natively, no need to translate unless at do_syscall for an
+         * intercepted-via-trampoline syscall which we allow now for case 9333 */
+#ifdef CLIENT_INTERFACE
+        if (IS_CLIENT_THREAD(trec->dcontext)) {
+            /* don't need to translate anything */
+            LOG(THREAD_GET, LOG_SYNCH, 1, "translate context, thread %d is client "
+                "thread, no translation needed\n", trec->id);
+            return true;
+        }
+#endif
+        if (is_native_thread_state_valid(trec->dcontext, (app_pc)mcontext->xip, 
+                                         (byte *)mcontext->xsp)) {
+            if (is_at_do_syscall(trec->dcontext, (app_pc)mcontext->xip,
+                                 (byte *)mcontext->xsp)) {
+                LOG(THREAD_GET, LOG_SYNCH, 1, "translate context, thread %d running "
+                    "natively, at do_syscall so translation needed\n", trec->id);
+                native_translate = true;
+            } else {
+                LOG(THREAD_GET, LOG_SYNCH, 1, "translate context, thread %d running "
+                    "natively, no translation needed\n", trec->id);
+                return true;
+            }
+        } else {
+            /* now that do_syscall is a safe spot for native threads we shouldn't get
+             * here for get context on self, FIXME - is however possible to get here
+             * via get_context on unsuspended thread (result of which is technically
+             * undefined according to MS), see get_context post sys comments
+             * (should prob. synch there in which case can assert here) */
+            ASSERT(trec->id != get_thread_id());
+            ASSERT_CURIOSITY(false && "translate failure, likely get context on "
+                             "unsuspended native thread");
+            /* we'll just try to translate and hope for the best */
+            native_translate = true;
+        }
+    }
+    if (!native_translate) {
+        /* check if waiting at a good spot */
+        spinmutex_lock(tsd->synch_lock);
+        res = THREAD_SYNCH_SAFE(tsd->synch_perm, THREAD_SYNCH_VALID_MCONTEXT);
+        spinmutex_unlock(tsd->synch_lock);
+        if (res) {
+            LOG(THREAD_GET, LOG_SYNCH, 1, "translate context, thread %d waiting at "
+                "valid mcontext point, copying over\n", trec->id);
+            DOLOG(2, LOG_SYNCH, {
+                LOG(THREAD_GET, LOG_SYNCH, 2, "Thread State\n");
+                dump_mcontext(get_mcontext(trec->dcontext), THREAD_GET, DUMP_NOT_XML);
+            });
+            *mcontext = *get_mcontext(trec->dcontext);
+            return true;
+        }
+    }
+
+    /* In case 4148 we see a thread calling NtGetContextThread on itself, which
+     * is undefined according to MS but it does get the syscall address, so it's
+     * fine with us.  For other threads the app shouldn't be asking about them
+     * unless they're suspended, and the same goes for us.
+     */
+    ASSERT_CURIOSITY(trec->dcontext->whereami == WHERE_FCACHE ||
+                     native_translate ||
+                     trec->id == get_thread_id());
+    LOG(THREAD_GET, LOG_SYNCH, 1, 
+        "translate context, thread %d at pc_recreatable spot translating\n", 
+        trec->id);
+    success = recreate_app_state(trec->dcontext, mcontext, restore_memory);
+    if (success != RECREATE_SUCCESS_STATE) {
+        /* should never happen right? */
+        LOG(THREAD_GET, LOG_SYNCH, 1, 
+            "translate context, thread %d unable to translate context at pc"
+            " = "PFX"\n", trec->id, mcontext->pc);
+        ASSERT_NOT_REACHED();
+        /* at least get pc right */
+        ASSERT(success == RECREATE_SUCCESS_PC);
+    }
+    return true;
+}
+
+static bool
+waiting_at_safe_spot(thread_record_t *trec, thread_synch_state_t desired_state)
+{
+    thread_synch_data_t *tsd = (thread_synch_data_t *) trec->dcontext->synch_field;
+    ASSERT(tsd->pending_synch_count >= 0);
+    /* check if waiting at a good spot, note that we can't spin in
+     * case the suspended thread is holding this lock, note only need
+     * lock to check the synch_perm */
+    if (spinmutex_trylock(tsd->synch_lock)) {
+        bool res = THREAD_SYNCH_SAFE(tsd->synch_perm, desired_state);
+        spinmutex_unlock(tsd->synch_lock);
+        if (res) {
+            LOG(THREAD_GET, LOG_SYNCH, 2, 
+                "thread %d waiting at safe spot\n", trec->id);
+            return true;
+        }
+    } else {
+        LOG(THREAD_GET, LOG_SYNCH, 2, 
+            "at_safe_spot unable to get locks to test if thread %d is waiting "
+            "at safe spot\n", trec->id);
+    } 
+    return false;
+}
+
+/* checks whether the thread trec is at a spot suitable for requested define
+ * desired_state
+ * Requires that trec thread is suspended */
+/* Note that since trec is potentially suspended at an arbitrary point,
+ * this function (and any function it calls) cannot call mutex_lock as 
+ * trec thread may hold a lock.  It is ok for at_safe_spot to return false if 
+ * it can't obtain a lock on the first try. FIXME : in the long term we may 
+ * want to go to a locking model that stores the thread id of the owner in 
+ * which case we can check for this situation directly
+ */
+bool
+at_safe_spot(thread_record_t *trec, dr_mcontext_t *mc,
+             thread_synch_state_t desired_state)
+{
+    bool safe = false;
+
+    if (waiting_at_safe_spot(trec, desired_state))
+        return true;
+
+    /* check if suspended at good spot */
+    /* FIXME: right now don't distinguish between suspend and term privileges
+     * even though suspend is stronger requirement, are the checks below 
+     * sufficient */
+    /* FIXME : check with respect to flush, should be ok */
+    /* test fcache_unit_areas.lock (from fcache.c) before calling recreate_app_state
+     * since it calls in_fcache() which uses the lock (if we are in_fcache()
+     * assume other locks are not a problem (so is_dynamo_address is fine)) */
+    /* Right now the only dr code that ends up in the cache is our DLL main 
+     * (which we'll reduce/get rid of with libc independence), our takeover 
+     * from preinject return stack, and the callback.c interception code.  
+     * FIXME : test for just these and ASSERT(!is_dynamo_address) otherwise */
+    if (is_thread_currently_native(trec)) {
+        /* thread is running native, verify is not in dr code */
+#ifdef CLIENT_INTERFACE
+        /* We treat client-owned threads (such as a client nudge thread) as native and
+         * consider them safe if they are in the client_lib.  Since they might own client
+         * locks that could block application threads from progressing, we synchronize
+         * with them last.  FIXME - xref PR 231301 - since we can't disambiguate
+         * client->ntdll/gencode which is safe from client->dr->ntdll/gencode which isn't
+         * we disallow both.  This could hurt synchronization efficiency if the client
+         * owned thread spent most of its execution time calling out of its lib to ntdll
+         * routines or generated code. */
+        if (IS_CLIENT_THREAD(trec->dcontext)) {
+            safe = trec->dcontext->client_data->client_thread_safe_for_synch ||
+                is_in_client_lib(mc->pc);
+        }
+#endif
+        if (is_native_thread_state_valid(trec->dcontext, mc->pc, 
+                                         (byte *)mc->xsp)) {
+            safe = true;
+            /* we should always be able to translate a valid native state */
+            ASSERT(translate_mcontext(trec, mc, false/*just querying*/));
+        }   
+    } else if ((!WRITE_LOCK_HELD(&fcache_unit_areas->lock) &&
+                /* even though we only need the read lock, if our target holds it
+                 * and a 3rd thread requests the write lock, we'll hang if we
+                 * ask for the read lock (case 7493)
+                 */
+                !READ_LOCK_HELD(&fcache_unit_areas->lock)) &&
+               recreate_app_state(trec->dcontext, mc, false/*just query*/) ==
+               RECREATE_SUCCESS_STATE &&
+               /* is ok to call is_dynamo_address even though it grabs many 
+                * locks because recreate_app_state succeeded */
+               !is_dynamo_address(mc->pc)) {
+        safe = true;
+    }
+    if (safe) {
+        ASSERT(trec->dcontext->whereami == WHERE_FCACHE || 
+               is_thread_currently_native(trec));
+        LOG(THREAD_GET, LOG_SYNCH, 2, 
+            "thread %d suspended at safe spot pc="PFX"\n", trec->id, mc->pc);
+        
+        return true;
+    }
+    LOG(THREAD_GET, LOG_SYNCH, 2, 
+        "thread %d not at safe spot (pc="PFX") for %d\n", 
+        trec->id, mc->pc, desired_state);
+    return false;
+}
+
+/* a fast way to tell a thread if it should call check_wait_at_safe_spot
+ * if translating context would be expensive */
+bool
+should_wait_at_safe_spot(dcontext_t *dcontext)
+{
+    thread_synch_data_t *tsd = (thread_synch_data_t *) dcontext->synch_field;
+    return (tsd->pending_synch_count != 0);
+}
+
+/* checks to see if any threads are waiting to synch with this one and waits 
+ * if they are
+ * cur_state - a given permission define from above that describes the current
+ *             state of the caller
+ * NOTE - Requires the caller is !could_be_linking (i.e. not in an 
+ * enter_couldbelinking state)
+ */
+void
+check_wait_at_safe_spot(dcontext_t *dcontext, thread_synch_permission_t cur_state)
+{
+    thread_synch_data_t *tsd = (thread_synch_data_t *) dcontext->synch_field;
+    app_pc pc;
+    byte cxt[CONTEXT_HEAP_SIZE_OPAQUE];
+    bool set_context = false;
+    bool set_mcontext = false;
+    if (tsd->pending_synch_count == 0 || cur_state == THREAD_SYNCH_NONE) 
+        return;
+    ASSERT(tsd->pending_synch_count >= 0);
+    pc = get_mcontext(dcontext)->pc;
+    LOG(THREAD, LOG_SYNCH, 2, 
+        "waiting for synch with state %d (pc "PFX")\n", 
+        cur_state, pc);
+    if (cur_state == THREAD_SYNCH_VALID_MCONTEXT) {
+        ASSERT(!is_dynamo_address(pc));
+        /* for detach must set this here and now */
+        IF_WINDOWS(set_last_error(dcontext->app_errno));
+    }
+    spinmutex_lock(tsd->synch_lock);
+    tsd->synch_perm = cur_state;
+    /* Since can be killed, suspended, etc. must call the exit dr hook. But, to
+     * avoid races, we must do so before giving up the synch_lock. This is why
+     * that lock has to be in unprotected memory. FIXME - for single thread in
+     * dr this will lead to rank order violation between dr exclusivity lock
+     * and the synch_lock with no easy workaround (real deadlocks possible).
+     * Luckily we'll prob. never use that option. */
+    if (INTERNAL_OPTION(single_thread_in_DR)) {
+        ASSERT_NOT_IMPLEMENTED(false);
+    }
+    EXITING_DR();
+    /* Ref case 5074, for us/app to succesfully SetThreadContext at
+     * this synch point, this thread can NOT be at a system call. So, for
+     * case 10101, we instead have threads that are waiting_at_safe_spot()
+     * set their own contexts, allowing us to make system calls here.
+     * We don't yet handle the detach case, so it still requires no system
+     * calls, including the act of releasing the synch_lock
+     * which is why that lock has to be a user mode spin yield lock.
+     * FIXME: we could change tsd->synch_lock back to a regular lock
+     * once we have detach handling system calls here.
+     */
+    spinmutex_unlock(tsd->synch_lock);
+    while (tsd->pending_synch_count > 0 && 
+           tsd->synch_perm != THREAD_SYNCH_NONE) {
+        STATS_INC_DC(dcontext, synch_loops_wait_safe);
+#ifdef WINDOWS
+        if (doing_detach) {
+            /* We spin for any non-detach synchs encountered during detach
+             * since we have no flag telling us this synch is for detach. */
+            /* Ref case 5074, can NOT use thread_yield here. This must be a user
+             * mode spin loop. */
+            SPINLOCK_PAUSE();
+        } else {
+#endif
+            /* FIXME case 10100: replace this sleep/yield with a wait_for_event() */
+            synch_thread_yield();
+#ifdef WINDOWS
+        }
+#endif
+    }
+    /* Regain the synch_lock before ENTERING_DR to avoid races with getting
+     * suspended/killed in the middle of ENTERING_DR (before synch_perm is
+     * reset to NONE). */
+    /* Ref case 5074, for detach we still can NOT use thread_yield here (no system
+     * calls) so don't allow the spinmutex_lock to yield while grabbing the lock. */
+    spinmutex_lock_no_yield(tsd->synch_lock);
+    ENTERING_DR();
+    tsd->synch_perm = THREAD_SYNCH_NONE;
+    if (tsd->set_mcontext != NULL || tsd->set_context != NULL) {
+        IF_WINDOWS(ASSERT(!doing_detach));
+        /* Make a local copy */
+        ASSERT(CONTEXT_HEAP_SIZE_OPAQUE >= sizeof(dr_mcontext_t));
+        if (tsd->set_mcontext != NULL) {
+            set_mcontext = true;
+            memcpy(cxt, tsd->set_mcontext, sizeof(*tsd->set_mcontext));
+        } else {
+            set_context = true;
+            memcpy(cxt, tsd->set_context, tsd->set_context_size);
+        }
+        synch_thread_free_setcontext(tsd); /* sets to NULL for us */
+    }
+    spinmutex_unlock(tsd->synch_lock);
+    LOG(THREAD, LOG_SYNCH, 2, 
+        "done waiting for synch with state %d (pc "PFX")\n", 
+        cur_state, pc);
+    if (set_mcontext || set_context) {
+        /* FIXME: see comment in dispatch.c check_wait_at_safe_spot() call
+         * about problems with KSTART(fcache_* differences bet the target
+         * being at the synch point vs in the cache.
+         */
+        if (set_mcontext)
+            thread_set_self_mcontext((dr_mcontext_t *)cxt);
+        else
+            thread_set_self_context((void *)cxt);
+        ASSERT_NOT_REACHED();
+    }
+}
+
+/* adjusts the pending synch count */
+void
+adjust_wait_at_safe_spot(dcontext_t *dcontext, int amt)
+{
+    thread_synch_data_t *tsd = (thread_synch_data_t *) dcontext->synch_field;
+    ASSERT(tsd->pending_synch_count >= 0);
+    spinmutex_lock(tsd->synch_lock);
+    ATOMIC_ADD(int, tsd->pending_synch_count, amt);
+    spinmutex_unlock(tsd->synch_lock);
+}
+
+/* Case 10101: Safely sets the context for a target thread that may be waiting at a
+ * safe spot, in which case we do not want to directly do a setcontext as the return
+ * from the yield or wait system call will mess up the state (case 5074).
+ * Assumes that cxt was allocated on the global heap, and frees it, rather than
+ * making its own copy (as an optimization).
+ * Does not work on the executing thread.
+ * Caller must hold thread_initexit_lock.
+ * If used on behalf of the app, it's up to the caller to check for privileges.
+ */
+bool
+set_synched_thread_context(thread_record_t *trec, 
+                           /* pass either mc or both cxt and cxt_size */
+                           dr_mcontext_t *mc, void *cxt, size_t cxt_size,
+                           thread_synch_state_t desired_state
+                           _IF_X64(byte *cxt_alloc)
+                           _IF_WINDOWS(NTSTATUS *status/*OUT*/))
+{
+    bool res = true;
+    ASSERT(trec != NULL && trec->dcontext != NULL);
+    ASSERT(trec->dcontext != get_thread_private_dcontext());
+    ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
+#ifdef WINDOWS
+    if (status != NULL)
+        *status = STATUS_SUCCESS;
+#endif
+    if (waiting_at_safe_spot(trec, desired_state)) {
+        /* case 10101: to allow system calls in check_wait_at_safe_spot() for
+         * performance reasons we have the waiting thread perform its own setcontext.
+         */
+        thread_synch_data_t *tsd = (thread_synch_data_t *) trec->dcontext->synch_field;
+        spinmutex_lock(tsd->synch_lock);
+        if (tsd->set_mcontext != NULL || tsd->set_context != NULL) {
+            /* Two synchs in a row while still waiting; 2nd takes precedence */
+            STATS_INC(wait_multiple_setcxt);
+            synch_thread_free_setcontext(tsd);
+        }
+        LOG(THREAD_GET, LOG_SYNCH, 2, 
+            "set_synched_thread_context %d to pc "PFX"\n", trec->id,
+            (mc != NULL) ? mc->pc : (app_pc)
+            IF_WINDOWS_ELSE(((CONTEXT*)cxt)->CXT_XIP,
+                            ((struct sigcontext *)cxt)->SC_XIP));
+        if (mc != NULL)
+            tsd->set_mcontext = mc;
+        else {
+            ASSERT(cxt != NULL && cxt_size > 0);
+            tsd->set_context = cxt;
+            tsd->set_context_size = cxt_size;
+        }
+        IF_X64(tsd->set_context_alloc = cxt_alloc);
+        ASSERT(THREAD_SYNCH_SAFE(tsd->synch_perm, desired_state));
+        ASSERT(tsd->pending_synch_count >= 0);
+        /* Don't need to change pending_synch_count or anything; when thread is
+         * resumed it will properly reset everything itself */
+        spinmutex_unlock(tsd->synch_lock);
+    } else {
+        if (mc != NULL) {
+            res = thread_set_mcontext(trec, mc);
+        } else {
+#ifdef WINDOWS
+            /* sort of ugly: but NtSetContextThread handling needs the status */
+            if (status != NULL) {
+                *status = nt_set_context(trec->handle, (CONTEXT *) cxt);
+                res = NT_SUCCESS(*status);
+            } else
+                res = thread_set_context(trec->handle, (CONTEXT *) cxt);
+#else
+            /* currently there are no callers who don't pass mc: presumably
+             * PR 212090 will change that */
+            ASSERT_NOT_IMPLEMENTED(false);
+#endif
+        }
+        free_setcontext(mc, cxt, cxt_size _IF_X64(cxt_alloc));
+    }
+    return res;
+}
+
+/* This is used to limit the maximum number of times synch_with_thread or
+ * synch_with_all_threads spin yield loops while waiting on an exiting thread.
+ * We assert if we ever break out of the loop because of this limit.  FIXME make
+ * sure this limit is large enough that if it does ever trigger it's because
+ * of some kind of deadlock situation.  Breaking out of the synchronization loop
+ * early is a correctness issue.  Right now the limits are large but arbitrary.
+ * FIXME : once we are confident about thread synch get rid of these max loop checks.
+ * N.B.: the THREAD_SYNCH_SMALL_LOOP_MAX flag causes us to divide these by 10.
+ */
+#define SYNCH_ALL_THREADS_MAXIMUM_LOOPS (DYNAMO_OPTION(synch_all_threads_max_loops))
+#define SYNCH_MAXIMUM_LOOPS (DYNAMO_OPTION(synch_thread_max_loops))
+
+/* Amt of time in ms to wait for threads to get to a safe spot per a loop,
+ * see comments in synch_with_yield() on value.  Our default value is 5ms which,
+ * depending on the tick resolution could end up being as long as 10 ms. */
+#define SYNCH_WITH_WAIT_MS ((int)DYNAMO_OPTION(synch_with_sleep_time))
+
+/* for use by synch_with_* routines to wait for thread(s) */
+static void
+synch_thread_yield()
+{
+    /* xref 9400, 9488 - thread_yield() works ok on an UP machine, but on an MP machine
+     * yield might not actually do anything (in which case we burn through to the max
+     * loop counts pretty quick).  We actually do want to wait a reasonable amt of time
+     * since the target thread might be doing some long latency dr operation (like
+     * dumping 500kb of registry into a forensics file) so we have the option to sleep
+     * instead. */
+    uint num_procs = get_num_processors();
+    ASSERT(num_procs != 0);
+    if ((num_procs == 1 && DYNAMO_OPTION(synch_thread_sleep_UP)) ||
+        (num_procs > 1 && DYNAMO_OPTION(synch_thread_sleep_MP))) {
+        thread_sleep(SYNCH_WITH_WAIT_MS);
+    } else {
+        thread_yield();
+    }
+}
+
+/* returns a thread_synch_result_t value
+ * id - the thread you want to synch with
+ * block - whether or not should spin until synch is succesful
+ * hold_initexit_lock - whether or not the caller holds the thread_initexit_lock
+ * caller_state - a given permission define from above that describes the 
+ *                current state of the caller (note that holding the initexit
+ *                lock is ok with respect to NO_LOCK
+ * desired_state - a requested state define from above that describes the 
+ *                 desired synchronization
+ * flags - options from THREAD_SYNCH_ bitmask values
+ * NOTE - if you hold the initexit_lock and block with greater than NONE for
+ * caller state, then initexit_lock may be released and re-acquired
+ * NOTE - if any of the nt_ routines fails, it is assumed the thread no longer
+ * exists and returns true
+ * NOTE - if called directly (i.e. not through synch_with_all_threads) 
+ * requires THREAD_SYNCH_IS_SAFE(caller_state, desired_state) to avoid deadlock
+ * NOTE - Requires the caller is !could_be_linking (i.e. not in an 
+ * enter_couldbelinking state)
+ * NOTE - you can't call this with a thread that you've already suspended
+ */
+thread_synch_result_t
+synch_with_thread(thread_id_t id, bool block, bool hold_initexit_lock,
+                  thread_synch_permission_t caller_state,
+                  thread_synch_state_t desired_state,
+                  uint flags)
+{
+    thread_id_t my_id = get_thread_id();
+    uint loop_count = 0;
+    int expect_exiting = 0;
+    thread_record_t *my_tr = thread_lookup(my_id), *trec = NULL;
+    dcontext_t *dcontext = NULL;
+    dr_mcontext_t mc;
+    thread_synch_result_t res = THREAD_SYNCH_RESULT_NOT_SAFE;
+    bool first_loop = true;
+    const uint max_loops = TEST(THREAD_SYNCH_SMALL_LOOP_MAX, flags) ?
+        (SYNCH_MAXIMUM_LOOPS/10) : SYNCH_MAXIMUM_LOOPS;
+
+    ASSERT(id != my_id);
+    /* Must set ABORT or IGNORE.  Only caller can RETRY as need a new
+     * set of threads for that, hoping problematic one is short-lived.
+     */
+    ASSERT(TESTANY(THREAD_SYNCH_SUSPEND_FAILURE_ABORT |
+                   THREAD_SYNCH_SUSPEND_FAILURE_IGNORE, flags) &&
+           !TESTALL(THREAD_SYNCH_SUSPEND_FAILURE_ABORT |
+                    THREAD_SYNCH_SUSPEND_FAILURE_IGNORE, flags));
+
+    if (my_tr != NULL) {
+        dcontext = my_tr->dcontext;
+        expect_exiting = dcontext->is_exiting ? 1 : 0;
+        ASSERT(exiting_thread_count >= expect_exiting);
+    } else {
+        /* calling thread should always be a known thread */
+        ASSERT_NOT_REACHED();
+    }
+
+    LOG(THREAD, LOG_SYNCH, 2, 
+        "Synching with thread "IDFMT", giving %d, requesting %d, blocking=%d\n", 
+        id, caller_state, desired_state, block);
+
+    if (!hold_initexit_lock) 
+        mutex_lock(&thread_initexit_lock);
+
+    while (true) {
+        /* get thread record */
+        /* FIXME : thread id recycling is possible that this could be a 
+         * different thread, perhaps we should take handle instead of id 
+         * FIXME: use the new num field of thread_record_t?
+         */
+        LOG(THREAD, LOG_SYNCH, 3, 
+            "Looping on synch with thread "IDFMT"\n", id);
+        trec = thread_lookup(id);
+        /* We test the exiting thread count to avoid races between terminate/
+         * suspend thread (current thread, though we could be here for other 
+         * reasons) and an exiting thread (who might no longer be on the all 
+         * threads list) who is still using shared resources (ref case 3121) */
+        if ((trec == NULL && exiting_thread_count == expect_exiting)
+            || loop_count++ > max_loops) {
+            /* make sure we didn't exit the loop without synchronizing, FIXME :
+             * in release builds we assume the synchronization is failing and 
+             * continue without it, but that is dangerous.
+             * It is now up to the caller to handle this, and some use
+             * small loop counts and abort on failure, so only a curiosity. */
+            ASSERT_CURIOSITY(loop_count < max_loops);
+            goto exit_synch_with_thread;
+        }
+        DOSTATS({
+            if (trec == NULL && exiting_thread_count > expect_exiting) {
+                LOG(THREAD, LOG_SYNCH, 2, "Waiting for an exiting thread\n");
+                STATS_INC(synch_yields_for_exiting_thread);
+            }
+        });
+        if (trec != NULL) {
+            if (first_loop) {
+                adjust_wait_at_safe_spot(trec->dcontext, 1);
+                first_loop = false;
+            }
+            if (!thread_suspend(trec)) {
+                /* FIXME : eventually should be a real assert once we figure out
+                 * how to handle threads with low privilege handles */
+                ASSERT_CURIOSITY_ONCE(false && "Thead synch unable to suspend target"
+                                      " thread, case 2096?");
+                res = (TEST(THREAD_SYNCH_SUSPEND_FAILURE_IGNORE, flags) ?
+                       THREAD_SYNCH_RESULT_SUCCESS : THREAD_SYNCH_RESULT_SUSPEND_FAILURE);
+                break;
+            }
+            if (!thread_get_mcontext(trec, &mc)) {
+                /* FIXME : eventually should be a real assert once we figure out
+                 * how to handle threads with low privilege handles */
+                ASSERT_CURIOSITY_ONCE(false && "Thead synch unable to get_context target"
+                                      " thread, case 2096?");
+                res = (TEST(THREAD_SYNCH_SUSPEND_FAILURE_IGNORE, flags) ?
+                       THREAD_SYNCH_RESULT_SUCCESS : THREAD_SYNCH_RESULT_SUSPEND_FAILURE);
+                break;
+            }
+            if (at_safe_spot(trec, &mc, desired_state)) {
+                /* FIXME: case 5325 for detach handling and testing */
+                IF_WINDOWS(ASSERT_NOT_IMPLEMENTED
+                           (!dcontext->aslr_context.sys_aslr_clobbered));
+                LOG(THREAD, LOG_SYNCH, 2, 
+                    "Thread "IDFMT" suspended in good spot\n", id);
+                LOG(trec->dcontext->logfile, LOG_SYNCH, 2, 
+                    "@@@@@@@@@@@@@@@@@@ SUSPENDED BY THREAD "IDFMT" synch_with_thread "
+                    "@@@@@@@@@@@@@@@@@@\n", my_id);
+                res = THREAD_SYNCH_RESULT_SUCCESS;
+                break;
+            }
+            if (!thread_resume(trec)) {
+                ASSERT_NOT_REACHED();
+                res = (TEST(THREAD_SYNCH_SUSPEND_FAILURE_IGNORE, flags) ?
+                       THREAD_SYNCH_RESULT_SUCCESS : THREAD_SYNCH_RESULT_SUSPEND_FAILURE);
+                break;
+            }
+        }
+        /* don't loop if !block, before we ever release initexit_lock in case
+         * caller is holding it and not blocking, (i.e. wants to keep it) */
+        if (!block) 
+            break; 
+        /* see if someone is waiting for us */
+        if (dcontext != NULL && caller_state != THREAD_SYNCH_NONE &&
+            should_wait_at_safe_spot(dcontext)) {
+            if (trec != NULL)
+                adjust_wait_at_safe_spot(trec->dcontext, -1);
+            mutex_unlock(&thread_initexit_lock);
+            /* ref case 5552, if we've inc'ed the exiting thread count need to 
+             * adjust it back before calling check_wait_at_safe_spot since we
+             * may end up being killed there */
+            if (dcontext->is_exiting) {
+                ASSERT(exiting_thread_count >= 1);
+                ATOMIC_DEC(int, exiting_thread_count);
+            }
+            check_wait_at_safe_spot(dcontext, caller_state);
+            if (dcontext->is_exiting) {
+                ATOMIC_INC(int, exiting_thread_count);
+            }
+            mutex_lock(&thread_initexit_lock);
+            trec = thread_lookup(id);
+            /* Like above, we test the exiting thread count to avoid races 
+             * between terminate/suspend thread (current thread, though we 
+             * could be here for other reasons) and an exiting thread (who 
+             * might no longer be on the all threads list) who is still using 
+             * shared resources (ref case 3121) */
+            if (trec == NULL && exiting_thread_count == expect_exiting) {
+                if (!hold_initexit_lock) 
+                    mutex_unlock(&thread_initexit_lock);
+                return THREAD_SYNCH_RESULT_SUCCESS;
+            }
+            DOSTATS({
+                if (trec == NULL && exiting_thread_count > expect_exiting) {
+                    LOG(THREAD, LOG_SYNCH, 2, "Waiting for an exiting thread\n");
+                    STATS_INC(synch_yields_for_exiting_thread);
+                }
+            });
+            if (trec != NULL)
+                adjust_wait_at_safe_spot(trec->dcontext, 1);
+        }
+        STATS_INC(synch_yields);
+        mutex_unlock(&thread_initexit_lock);
+        /* Note - we only need call the ENTER/EXIT_DR hooks if single thread
+         * in dr since we are not really exiting DR here (we just need to give
+         * up the exclusion lock for a while to let thread we are trying to
+         * synch with make progress towards a safe synch point). */
+        if (INTERNAL_OPTION(single_thread_in_DR))
+            EXITING_DR(); /* give up DR exclusion lock */
+        synch_thread_yield();
+        if (INTERNAL_OPTION(single_thread_in_DR))
+            ENTERING_DR(); /* re-gain DR exclusion lock */
+        mutex_lock(&thread_initexit_lock);
+    }
+    /* reset this back to before */
+    adjust_wait_at_safe_spot(trec->dcontext, -1);
+    /* success!, is suspended (or already exited) put in desired state */
+    if (res == THREAD_SYNCH_RESULT_SUCCESS) {
+        LOG(THREAD, LOG_SYNCH, 2, 
+            "Success synching with thread "IDFMT" performing cleanup\n", id);
+        if (THREAD_SYNCH_IS_TERMINATED(desired_state)) {
+            thread_terminate(trec);
+        }
+        if (THREAD_SYNCH_IS_CLEANED(desired_state)) {
+            dynamo_other_thread_exit(trec _IF_WINDOWS(false));
+        }
+    }
+ exit_synch_with_thread:
+    if (!hold_initexit_lock) 
+        mutex_unlock(&thread_initexit_lock);
+    return res;
+}
+
+/* desired_synch_state - a requested state define from above that describes
+ *                        the synchronization required
+ * threads, num_threads - must not be NULL, if !THREAD_SYNCH_IS_CLEANED(desired
+ *                        synch_state) then will hold a list and num of threads
+ * cur_state - a given permission from above that describes the state of the
+ *             caller
+ * flags - options from THREAD_SYNCH_ bitmask values
+ * NOTE - Requires that the caller doesn't hold the thread_initexit_lock, on
+ * return caller will hold the thread_initexit_lock
+ * NOTE - Requires the caller is !could_be_linking (i.e. not in an 
+ * enter_couldbelinking state)
+ * NOTE - To avoid deadlock this routine should really only be called with 
+ * cur_state giving maximum permissions, (currently app_exit and detach could
+ * conflict, except our routes to app_exit go through different synch point 
+ * (TermThread or TermProcess) first
+ * NOTE - when !all_synched, if desired_synch_state is not cleaned or synch result is
+ * ignored, the caller is reponsible for resuming threads that are suspended,
+ * freeing allocation for threads array and releasing locks
+ * Caller should call end_synch_with_all_threads when finished to accomplish that.
+ */
+bool
+synch_with_all_threads(thread_synch_state_t desired_synch_state,
+                       /*OUT*/ thread_record_t ***threads_out,
+                       /*OUT*/ int *num_threads_out,
+                       thread_synch_permission_t cur_state,
+                       /* FIXME: turn the ThreadSynch* enums into bitmasks and merge
+                        * into flags param */
+                       uint flags)
+{
+    /* Case 8815: we cannot use the OUT params themselves internally as they
+     * may be volatile, so we need our own values until we're ready to return
+     */
+    thread_record_t **threads = NULL;
+    int num_threads = 0;
+    /* we record ids from before we gave up thread_initexit_lock */
+    thread_id_t *thread_ids_temp = NULL;
+    int num_threads_temp = 0, i, j, expect_exiting = 0;
+    /* synch array contains a SYNCH_WITH_ALL_ value for each thread */
+    uint *synch_array = NULL, *synch_array_temp = NULL;
+    enum {
+        SYNCH_WITH_ALL_NEW      = 0,
+        SYNCH_WITH_ALL_NOTIFIED = 1,
+        SYNCH_WITH_ALL_SYNCHED  = 2,
+    };
+    bool all_synched = false;
+    thread_id_t my_id = get_thread_id();
+    uint loop_count = 0;
+    thread_record_t *tr = thread_lookup(my_id);
+    dcontext_t *dcontext = NULL;
+    uint flags_one; /* flags for synch_with_thread() call */
+    thread_synch_result_t synch_res;
+    const uint max_loops = TEST(THREAD_SYNCH_SMALL_LOOP_MAX, flags) ?
+        (SYNCH_ALL_THREADS_MAXIMUM_LOOPS/10) : SYNCH_ALL_THREADS_MAXIMUM_LOOPS;
+#ifdef CLIENT_INTERFACE
+    /* We treat client-owned threads as native but they don't have a clean native state
+     * for us to suspend them in (they are always in client or dr code).  We need to be
+     * able to suspend such threads so that they're !couldbelinking and holding no dr
+     * locks. We make the assumption that client-owned threads that are in the client
+     * library (or are in a dr routine that has set dcontext->client_thread_safe_to_sync)
+     * meet this requirement (see at_safe_spot()).  As such, all we need to worry about
+     * here are client locks the client-owned thread might hold that could block other
+     * threads from reaching safe spots.  If we only suspend client-owned threads once
+     * all other threads are taken care of then this is not a problem. FIXME - xref
+     * PR 231301 on issues that arise if the client thread spends most of its time
+     * calling out of its lib to dr API, ntdll, or generated code functions. */
+    bool finished_non_client_threads;
+#endif
+
+    ASSERT(!dynamo_all_threads_synched);
+    /* flag any caller who does not give up enough permissions to avoid livelock
+     * with other synch_with_all_threads callers
+     */
+    ASSERT_CURIOSITY(cur_state >= THREAD_SYNCH_NO_LOCKS_NO_XFER);
+    /* also flag anyone asking for full mcontext w/o possibility of no_xfer,
+     * which can also livelock
+     */
+    ASSERT_CURIOSITY(desired_synch_state < THREAD_SYNCH_SUSPENDED_VALID_MCONTEXT
+                     /* detach currently violates this: bug 8942 */
+                     IF_WINDOWS(|| doing_detach));
+
+    /* must set exactly one of these -- FIXME: better way to check? */
+    ASSERT(TESTANY(THREAD_SYNCH_SUSPEND_FAILURE_ABORT |
+                   THREAD_SYNCH_SUSPEND_FAILURE_IGNORE |
+                   THREAD_SYNCH_SUSPEND_FAILURE_RETRY, flags) &&
+           !TESTALL(THREAD_SYNCH_SUSPEND_FAILURE_ABORT |
+                    THREAD_SYNCH_SUSPEND_FAILURE_IGNORE, flags) &&
+           !TESTALL(THREAD_SYNCH_SUSPEND_FAILURE_ABORT |
+                    THREAD_SYNCH_SUSPEND_FAILURE_RETRY, flags) &&
+           !TESTALL(THREAD_SYNCH_SUSPEND_FAILURE_IGNORE |
+                    THREAD_SYNCH_SUSPEND_FAILURE_RETRY, flags));
+    flags_one = flags;
+    /* we'll do the retry */
+    if (TEST(THREAD_SYNCH_SUSPEND_FAILURE_RETRY, flags)) {
+        flags_one &= ~THREAD_SYNCH_SUSPEND_FAILURE_RETRY;
+        flags_one |= THREAD_SYNCH_SUSPEND_FAILURE_ABORT;
+    }
+
+    if (tr != NULL) {
+        dcontext = tr->dcontext;
+        expect_exiting = dcontext->is_exiting ? 1 : 0;
+        ASSERT(exiting_thread_count >= expect_exiting);
+    } else {
+        /* calling thread should always be a known thread */
+        ASSERT_NOT_REACHED();
+    }
+    
+    LOG(THREAD, LOG_SYNCH, 1, 
+        "synch with all threads my id = "SZFMT
+        " Giving %d permission and seeking %d state\n", 
+        my_id, cur_state, desired_synch_state);
+
+    /* grab all_threads_synch_lock */
+    /* since all_threads synch doesn't give any permissions this is neccesary
+     * to prevent deadlock in the case of two threads trying to synch with all
+     * threads at the same time  */
+    /* FIXME: for DEADLOCK_AVOIDANCE, to preserve LIFO, should we
+     * exit DR, trylock, then immediately enter DR?  introducing any
+     * race conditions in doing so?
+     * Ditto on all other thread_yields in this file!
+     */
+    while (!mutex_trylock(&all_threads_synch_lock)) {
+        LOG(THREAD, LOG_SYNCH, 2, "Spinning on all threads synch lock\n");
+        STATS_INC(synch_yields);
+        if (dcontext != NULL && cur_state != THREAD_SYNCH_NONE &&
+            should_wait_at_safe_spot(dcontext)) {
+            /* ref case 5552, if we've inc'ed the exiting thread count need to 
+             * adjust it back before calling check_wait_at_safe_spot since we
+             * may end up being killed there */
+            if (dcontext->is_exiting) {
+                ASSERT(exiting_thread_count >= 1);
+                ATOMIC_DEC(int, exiting_thread_count);
+            }
+            check_wait_at_safe_spot(dcontext, cur_state);
+            if (dcontext->is_exiting) {
+                ATOMIC_INC(int, exiting_thread_count);
+            }
+        }
+        LOG(THREAD, LOG_SYNCH, 2, "Yielding on all threads synch lock\n");
+        /* Note - we only need call the ENTER/EXIT_DR hooks if single thread
+         * in dr since we are not really exiting DR here (we just need to give
+         * up the exclusion lock for a while to let thread we are trying to
+         * synch with make progress towards a safe synch point). */
+        if (INTERNAL_OPTION(single_thread_in_DR))
+            EXITING_DR(); /* give up DR exclusion lock */
+        thread_yield();
+        if (INTERNAL_OPTION(single_thread_in_DR))
+            ENTERING_DR(); /* re-gain DR exclusion lock */
+    }
+
+    mutex_lock(&thread_initexit_lock);
+    /* synch with all threads */
+    /* FIXME: this should be a do/while loop - then we wouldn't have
+     * to initialize all the variables above 
+     */
+    while ((!all_synched || exiting_thread_count > expect_exiting)
+           && loop_count++ < max_loops) {
+        if (threads != NULL){
+            /* Case 8941: must free here rather than when yield (below) since
+             * termination condition can change between there and here
+             */
+            ASSERT(num_threads > 0);
+            global_heap_free(threads, num_threads * sizeof(thread_record_t *)
+                             HEAPACCT(ACCT_THREAD_MGT));
+            /* be paranoid */
+            threads = NULL;
+            num_threads = 0;
+        }
+        get_list_of_threads(&threads, &num_threads);
+        synch_array = (uint *)global_heap_alloc(num_threads * sizeof(uint)
+                                                HEAPACCT(ACCT_THREAD_MGT));
+        for (i = 0; i < num_threads; i++) {
+            synch_array[i] = SYNCH_WITH_ALL_NEW;
+        }
+        /* Fixme : an inefficient algorithm, but is not as bad as it seems
+         * since it is very unlikely that many threads have started or ended
+         * and the list threads routine always puts them in the same order  
+         */
+        /* on first loop num_threads_temp == 0 */
+        for (i = 0; i < num_threads_temp; i++) {
+            /* care only if we have already notified or synched thread */
+            if (synch_array_temp[i] != SYNCH_WITH_ALL_NEW) {
+                for (j = 0; j < num_threads; j++) {
+                    /* FIXME : os recycles thread ids, should have stronger
+                     * check here, could check dcontext equivalence, (but we
+                     * recycle those to), probably should check threads_temp
+                     * handle and be sure thread is still alive since the id
+                     * won't be recycled then */
+                    if (threads[j]->id == thread_ids_temp[i]) {
+                        synch_array[j] = synch_array_temp[i];
+                        break;
+                    }
+                }
+            }
+        }
+        /* free old synch list, old thread id list */
+        if (num_threads_temp > 0) {
+            global_heap_free(thread_ids_temp, num_threads_temp * sizeof(thread_id_t)
+                             HEAPACCT(ACCT_THREAD_MGT));
+            global_heap_free(synch_array_temp, num_threads_temp * sizeof(uint)
+                             HEAPACCT(ACCT_THREAD_MGT));
+            num_threads_temp = 0;
+        }
+        
+        all_synched = true;
+        LOG(THREAD, LOG_SYNCH, 3, 
+            "Looping over all threads (%d threads)\n", num_threads);
+#ifdef CLIENT_INTERFACE
+        finished_non_client_threads = true;
+        for (i = 0; i < num_threads; i++) {
+            if (threads[i]->id != my_id && synch_array[i] != SYNCH_WITH_ALL_SYNCHED &&
+                !IS_CLIENT_THREAD(threads[i]->dcontext)) {
+                finished_non_client_threads = false;
+                break;
+            }
+        }
+#endif
+        for (i = 0; i < num_threads; i++) {
+            if (threads[i]->id != my_id && synch_array[i] != SYNCH_WITH_ALL_SYNCHED) {
+#ifdef CLIENT_INTERFACE
+                if (!finished_non_client_threads &&
+                    IS_CLIENT_THREAD(threads[i]->dcontext)) {
+                    all_synched = false;
+                    continue; /* skip this thread for now till non-client are finished */
+                }
+#endif
+                /* speed things up a tad */
+                if (synch_array[i] != SYNCH_WITH_ALL_NOTIFIED) {
+                    ASSERT(synch_array[i] == SYNCH_WITH_ALL_NEW);
+                    adjust_wait_at_safe_spot(threads[i]->dcontext, 1);
+                    synch_array[i] = SYNCH_WITH_ALL_NOTIFIED;
+                }
+                LOG(THREAD, LOG_SYNCH, 2, 
+                    "About to try synch with thread "IDFMT"\n", threads[i]->id);
+                synch_res = synch_with_thread(threads[i]->id, false, true,
+                                              THREAD_SYNCH_NONE,
+                                              desired_synch_state, flags_one);
+                if (synch_res == THREAD_SYNCH_RESULT_SUCCESS) {
+                    LOG(THREAD, LOG_SYNCH, 2, "Synch succeded!\n");
+                    /* successful synch */
+                    synch_array[i] = SYNCH_WITH_ALL_SYNCHED;
+                    if (!THREAD_SYNCH_IS_CLEANED(desired_synch_state)) 
+                        adjust_wait_at_safe_spot(threads[i]->dcontext, -1);
+                } else {
+                    LOG(THREAD, LOG_SYNCH, 2, "Synch failed!\n");
+                    all_synched = false;
+                    if (synch_res == THREAD_SYNCH_RESULT_SUSPEND_FAILURE) {
+                        if (TEST(THREAD_SYNCH_SUSPEND_FAILURE_ABORT, flags))
+                            goto synch_with_all_abort;
+                    } else
+                        ASSERT(synch_res == THREAD_SYNCH_RESULT_NOT_SAFE);
+                }
+            } else {
+                LOG(THREAD, LOG_SYNCH, 2, 
+                    "Skipping synch with thread "IDFMT"\n", threads[i]->id);
+            }
+        }
+        /* We test the exiting thread count to avoid races between exit
+         * process (current thread, though we could be here for detach or other
+         * reasons) and an exiting thread (who might no longer be on the all
+         * threads list) who is still using shared resources (ref case 3121) */
+        if (!all_synched || exiting_thread_count > expect_exiting) {
+            DOSTATS({
+                if (all_synched && exiting_thread_count > expect_exiting) {
+                    LOG(THREAD, LOG_SYNCH, 2, "Waiting for an exiting thread %d %d %d\n",
+                        all_synched, exiting_thread_count, expect_exiting);
+                    STATS_INC(synch_yields_for_exiting_thread);
+                }
+            });
+            STATS_INC(synch_yields);
+
+            /* make a copy of the thread ids (can't just keep the thread list
+             * since it consists of pointers to live ThreadRecords
+             */
+            thread_ids_temp = (thread_id_t *)
+                global_heap_alloc(num_threads * sizeof(thread_id_t)
+                                  HEAPACCT(ACCT_THREAD_MGT));
+            for (i = 0; i < num_threads; i++)
+                thread_ids_temp[i] = threads[i]->id;
+            num_threads_temp = num_threads;
+            synch_array_temp = synch_array;
+
+            /* release lock in case some other thread waiting on it */
+            mutex_unlock(&thread_initexit_lock);
+            LOG(THREAD, LOG_SYNCH, 2, "Not all threads synched looping again\n");
+            /* Note - we only need call the ENTER/EXIT_DR hooks if single
+             * thread in dr since we are not really exiting DR here (we just
+             * need to give up the exclusion lock for a while to let thread we
+             * are trying to synch with make progress towards a safe synch
+             * point). */
+            if (INTERNAL_OPTION(single_thread_in_DR))
+                EXITING_DR(); /* give up DR exclusion lock */
+            synch_thread_yield();
+            if (INTERNAL_OPTION(single_thread_in_DR))
+                ENTERING_DR(); /* re-gain DR exclusion lock */
+            mutex_lock(&thread_initexit_lock);
+        }
+    }
+    /* case 9392: callers passing in ABORT expect a return value of failure
+     * to correspond w/ no suspended threads, a freed threads array, and no
+     * locks being held, so we go through the abort path
+     */
+    if (!all_synched && TEST(THREAD_SYNCH_SUSPEND_FAILURE_ABORT, flags))
+        goto synch_with_all_abort;
+ synch_with_all_exit:
+    /* make sure we didn't exit the loop without synchronizing, FIXME : in
+     * release builds we assume the synchronization is failing and continue
+     * without it, but that is dangerous.
+     * It is now up to the caller to handle this, and some use
+     * small loop counts and abort on failure, so only a curiosity. */
+    ASSERT_CURIOSITY(loop_count < max_loops);
+    ASSERT(threads != NULL);
+    /* Since the set of threads can change we don't set the success field
+     * until we're passing back the thread list.
+     * We would use an tsd field directly instead of synch_array except
+     * for THREAD_SYNCH_*_CLEAN where tsd is freed.
+     */
+    ASSERT(synch_array != NULL);
+    if (!THREAD_SYNCH_IS_CLEANED(desired_synch_state)) { /* else unsafe to access tsd */
+        for (i = 0; i < num_threads; i++) {
+            if (threads[i]->id != my_id) {
+                thread_synch_data_t *tsd;
+                ASSERT(threads[i]->dcontext != NULL);
+                tsd = (thread_synch_data_t *) threads[i]->dcontext->synch_field;
+                tsd->synch_with_success = (synch_array[i] == SYNCH_WITH_ALL_SYNCHED);
+            }
+        }
+    }
+    global_heap_free(synch_array, num_threads * sizeof(uint) HEAPACCT(ACCT_THREAD_MGT));
+    if (num_threads_temp > 0) {
+        global_heap_free(thread_ids_temp, num_threads_temp * sizeof(thread_id_t)
+                         HEAPACCT(ACCT_THREAD_MGT));
+    }
+    /* FIXME case 9333: on all_synch failure we do not free threads array if
+     * synch_result is ignored.  Callers are responsible for resuming threads that are
+     * suspended and freeing allocation for threads array
+     */
+    if ((!all_synched && TEST(THREAD_SYNCH_SUSPEND_FAILURE_ABORT, flags))
+        || THREAD_SYNCH_IS_CLEANED(desired_synch_state)) {
+        global_heap_free(threads, num_threads * sizeof(thread_record_t *)
+                         HEAPACCT(ACCT_THREAD_MGT));
+        threads = NULL;
+        num_threads = 0;
+    }
+    LOG(THREAD, LOG_SYNCH, 1,
+        "Finished synch with all threads: result=%d\n", all_synched);
+    DOLOG(1, LOG_SYNCH, {
+        if (all_synched) {
+            LOG(THREAD, LOG_SYNCH, 1,
+                "\treturning holding initexit_lock and all_threads_synch_lock\n");
+        }
+    });
+    *threads_out = threads;
+    *num_threads_out = num_threads;
+    dynamo_all_threads_synched = all_synched;
+    /* FIXME case 9392: where on all_synch failure we do not release the locks in the
+     * non-abort exit path */
+    return all_synched;
+
+ synch_with_all_abort:
+    /* undo everything! */
+    for (i = 0; i < num_threads; i++) {
+        DEBUG_DECLARE(bool ok;)
+        if (threads[i]->id != my_id) {
+            if (synch_array[i] == SYNCH_WITH_ALL_SYNCHED) {
+                DEBUG_DECLARE(ok =)
+                    thread_resume(threads[i]);
+                ASSERT(ok);
+                /* ensure synch_with_success is set to false on exit path,
+                 * even though locks are released and not fully valid
+                 */
+                synch_array[i] = SYNCH_WITH_ALL_NEW;
+            } else if (synch_array[i] == SYNCH_WITH_ALL_NOTIFIED) {
+                adjust_wait_at_safe_spot(threads[i]->dcontext, -1);
+            }
+        }
+    }
+    mutex_unlock(&thread_initexit_lock);
+    mutex_unlock(&all_threads_synch_lock);
+    ASSERT(!all_synched); /* ensure our OUT values will be NULL,0
+                             for THREAD_SYNCH_SUSPEND_FAILURE_ABORT */
+    goto synch_with_all_exit;
+}
+
+/* Assumes that the threads were suspended with synch_with_all_threads()
+ * and thus even is_thread_currently_native() threads were suspended.
+ * Assumes that the caller will free up threads if it is dynamically allocated.
+ */
+void
+resume_all_threads(thread_record_t **threads, const uint num_threads)
+{
+    uint i;
+    thread_id_t my_tid;
+    bool res;
+
+    ASSERT_OWN_MUTEX(true, &all_threads_synch_lock);
+    ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
+
+    if (threads == NULL || num_threads == 0)
+        return;
+
+    my_tid = get_thread_id();
+    for (i = 0; i < num_threads; i++) {
+        if (my_tid == threads[i]->id)
+            continue;
+
+        /* This routine assumes that each thread in the array was suspended, so
+         * each one has to successfully resume.
+         */
+        res = thread_resume(threads[i]);
+        ASSERT(res);
+    }
+}
+
+/* Should be called to clean up after synch_with_all_threads as otherwise
+ * dynamo_all_threads_synched will be left as true.
+ * If resume is true, resumes the threads in the threads array.
+ * Unlocks thread_initexit_lock and all_threads_synch_lock.
+ * If threads != NULL, frees the threads array.
+ */
+void
+end_synch_with_all_threads(thread_record_t **threads, uint num_threads, bool resume)
+{
+    /* dynamo_all_threads_synched will be false if synch failed */
+    ASSERT_CURIOSITY(dynamo_all_threads_synched);
+    ASSERT(OWN_MUTEX(&all_threads_synch_lock) && OWN_MUTEX(&thread_initexit_lock));
+    dynamo_all_threads_synched = false;
+    if (resume) {
+        ASSERT(threads != NULL);
+        resume_all_threads(threads, num_threads);
+    }
+    /* if we knew whether THREAD_SYNCH_*_CLEANED was specified we could set
+     * synch_with_success to false, but it's unsafe otherwise
+     */
+    mutex_unlock(&thread_initexit_lock);
+    mutex_unlock(&all_threads_synch_lock);
+    if (threads != NULL) {
+        global_heap_free(threads, num_threads*sizeof(thread_record_t*)
+                         HEAPACCT(ACCT_THREAD_MGT));
+    }
+}
+
+/* Resets a thread's context to start interpreting anew.
+ * ASSUMPTION: the thread is currently suspended.
+ * This was moved here from fcache_reset_all_caches_proactively simply to
+ * get access to win32-private CONTEXT-related routines
+ */
+void
+translate_from_synchall_to_dispatch(thread_record_t *tr, thread_synch_state_t synch_state)
+{
+    bool res;
+    /* we do not have to align dr_mcontext_t */
+    dr_mcontext_t *mc = global_heap_alloc(sizeof(*mc) HEAPACCT(ACCT_OTHER));
+    bool free_cxt = true;
+    dcontext_t *dcontext = tr->dcontext;
+    app_pc pre_translation;
+    ASSERT(OWN_MUTEX(&all_threads_synch_lock) && OWN_MUTEX(&thread_initexit_lock));
+    /* FIXME: would like to assert that suspendcount is > 0 but how? */
+    ASSERT(thread_synch_successful(tr));
+
+    res = thread_get_mcontext(tr, mc);
+    ASSERT(res);
+    pre_translation = (app_pc) mc->xip;
+    LOG(GLOBAL, LOG_CACHE, 2, 
+        "\trecreating address for "PFX"\n", mc->xip);
+    LOG(THREAD, LOG_CACHE, 2, 
+        "translate_from_synchall_to_dispatch: being translated from "PFX"\n",
+        mc->xip);
+    if (get_at_syscall(dcontext)) {
+        /* Don't need to do anything as shared_syscall and do_syscall will not
+         * change due to a reset and will have any inlined ibl updated.  If we
+         * did try to send these guys back to dispatch, have to set asynch_tag
+         * (as well as next_tag since translation looks only at that), restore
+         * TOS to asynch_target/esi (unless still at reset state), and have to
+         * figure out how to avoid post-syscall processing for those who never
+         * did pre-syscall processing (i.e., if at shared_syscall) (else will
+         * get wrong dcontext->sysnum, etc.)
+         * Not to mention that after resuming the kernel will finish the
+         * syscall and clobber several registers, making it hard to set a
+         * clean state (xref case 6113, case 5074, and notes below)!
+         * It's just too hard to redirect while at a syscall.
+         */
+        LOG(GLOBAL, LOG_CACHE, 2, 
+            "\tat syscall so not translating\n");
+        /* sanity check */
+        ASSERT(is_after_syscall_address(dcontext, pre_translation) ||
+               pre_translation == IF_WINDOWS_ELSE(vsyscall_after_syscall,
+                                                  vsyscall_sysenter_return_pc));
+    } else {
+        res = translate_mcontext(tr, mc, true/*restore memory*/);
+        ASSERT(res);
+        if (!thread_synch_successful(tr) || mc->xip == 0) {
+            /* Better to risk failure on accessing a freed cache than
+             * to have a guaranteed crash by sending to NULL.
+             * FIXME: it's possible the real translation is NULL,
+             * but if so should be fine to leave it there since the
+             * current eip should also be NULL.
+             */
+            ASSERT_NOT_REACHED();
+            goto translate_from_synchall_to_dispatch_exit;
+        }
+        LOG(GLOBAL, LOG_CACHE, 2, 
+            "\ttranslation pc = "PFX"\n", mc->xip);
+        ASSERT(!is_dynamo_address((app_pc)mc->xip) &&
+               !in_fcache((app_pc)mc->xip));
+        /* We send all threads, regardless of whether was in DR or not, to
+         * re-interp from translated cxt, to avoid having to handle stale
+         * local state problems if we simply resumed.
+         * We assume no KSTATS or other state issues to deal with.
+         * FIXME: enter hook w/o an exit?
+         */
+        dcontext->next_tag = (app_pc) mc->xip;
+        /* FIXME PR 212266: for linux if we're at an inlined syscall
+         * we may have problems: however, we might be able to rely on the kernel
+         * not clobbering any registers besides eax (which is ok: reset stub
+         * handles it), though presumably it's allowed to write to any
+         * caller-saved registers.  We may need to change inlined syscalls
+         * to set at_syscall (see comments below as well).
+         */
+        if (pre_translation == IF_WINDOWS_ELSE(vsyscall_after_syscall,
+                                               vsyscall_sysenter_return_pc) &&
+            !waiting_at_safe_spot(dcontext->thread_record, synch_state)) {
+            /* FIXME case 7827/PR 212266: shouldn't translate for this case, right?
+             * should have -ignore_syscalls set at_syscall and eliminate
+             * this whole block of code
+             */
+            /* put the proper retaddr back on the stack, as we won't
+             * be doing the ret natively to regain control, but rather
+             * will interpret it
+             */
+            /* FIXME: ensure readable and writable? */
+            app_pc cur_retaddr = *((app_pc *)mc->xsp);
+            app_pc native_retaddr;
+            ASSERT(cur_retaddr != NULL);
+            /* must be ignore_syscalls (else, at_syscall will be set) */
+            IF_WINDOWS(ASSERT(DYNAMO_OPTION(ignore_syscalls)));
+            ASSERT(get_syscall_method() == SYSCALL_METHOD_SYSENTER);
+            /* For DYNAMO_OPTION(sygate_sysenter) we need to restore both stack
+             * values and fix up esp, but we can't do it here since the kernel
+             * will change esp... incompatible w/ -ignore_syscalls anyway
+             */
+            IF_WINDOWS(ASSERT_NOT_IMPLEMENTED(!DYNAMO_OPTION(sygate_sysenter)));
+            /* may still be at syscall from a prior reset -- don't want to grab
+             * locks for in_fcache so we determine via the translation
+             */
+            ASSERT_NOT_TESTED();
+            native_retaddr = recreate_app_pc(dcontext, cur_retaddr, NULL);
+            if (native_retaddr != cur_retaddr) {
+                LOG(GLOBAL, LOG_CACHE, 2, 
+                    "\trestoring TOS to "PFX" from "PFX"\n", native_retaddr,
+                    cur_retaddr);
+                *((app_pc *)mc->xsp) = native_retaddr;
+            } else {
+                LOG(GLOBAL, LOG_CACHE, 2, 
+                    "\tnot restoring TOS since still at previous reset state "PFX"\n",
+                    cur_retaddr);
+            }
+        }
+        /* Send back to dispatch.  Rather than setting up last_exit in eax here,
+         * we point to a special routine to save the correct eax -- in fact it's
+         * simply a direct exit stub.  Originally this was b/c we tried to
+         * translate threads at system calls, and the kernel clobbers eax (and
+         * ecx/edx for sysenter, though preserves eip setcontext change: case
+         * 6113, case 5074) in finishing the system call, but now that we don't
+         * translate them we've kept the stub approach.  It's actually faster
+         * for the stub itself to save eax and set the linkstub than for us to
+         * emulate it here, anyway.
+         * Note that a thread in check_wait_at_safe_spot() spins and will NOT be
+         * at a syscall, avoiding problems there (case 5074).
+         */
+        mc->xip = (app_pc) get_reset_exit_stub(dcontext);
+        LOG(GLOBAL, LOG_CACHE, 2, 
+            "\tsent to reset exit stub "PFX"\n", mc->xip);
+        /* make dispatch happy */
+        dcontext->whereami = WHERE_FCACHE;
+        /* exit stub and subsequent fcache_return will save rest of state */
+        res = set_synched_thread_context(dcontext->thread_record, mc, NULL, 0,
+                                         synch_state _IF_X64((void *)mc)
+                                         _IF_WINDOWS(NULL));
+        ASSERT(res);
+        /* cxt is freed by set_synched_thread_context() or target thread */
+        free_cxt = false;
+    }
+ translate_from_synchall_to_dispatch_exit:
+    if (free_cxt) {
+        global_heap_free(mc, sizeof(*mc) HEAPACCT(ACCT_OTHER));
+    }
+}
+
diff --git a/core/synch.h b/core/synch.h
new file mode 100644
index 000000000..e8b5f320a
--- /dev/null
+++ b/core/synch.h
@@ -0,0 +1,255 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * thread.h - thread synchronization
+ */
+
+#ifndef SYNCH_H
+#define SYNCH_H
+
+/*for synch */
+/* Given permissions */
+/* The order is in increasing permissiveness and the values are chosen to
+ * match up with equivalent requested states below */
+typedef enum {
+    THREAD_SYNCH_NONE=0, 
+
+    /* At consistent state, holding no locks, suitable for terminate,
+     * suitable for suspending */
+    THREAD_SYNCH_NO_LOCKS=3,
+
+    /* At consistent state, holding no locks, suitable for terminate,
+     * suitable for suspending, but cannot be transfered elsewhere:
+     * must be resumed at suspend point in order to finish an
+     * in-progress task (such as flushing or hot patch updating).
+     * Xref case 6821.
+     */
+    THREAD_SYNCH_NO_LOCKS_NO_XFER=4,
+
+    /* At consistent state, holding no locks, with valid mcontext
+     * (including app_errno), suitable for suspending.  But, not
+     * suitable for transferring elsewhere in DR -- ok to transfer
+     * if going native, though.
+     */
+    THREAD_SYNCH_VALID_MCONTEXT_NO_XFER=5,
+
+    /* At consistent state, holding no locks, with valid mcontext, suitable
+     *  for suspending, note that valid mcontext includes app_errno */
+    THREAD_SYNCH_VALID_MCONTEXT=6
+} thread_synch_permission_t;
+
+/* Requested states */
+/* clean means that the supporting dynamorio data structures are cleaned up */
+/* The order is in increasingly strong requests and the values are chosen to
+ * match up with equivalent given permissions above */
+typedef enum {
+    THREAD_SYNCH_SUSPENDED=1,
+    THREAD_SYNCH_SUSPENDED_AND_CLEANED=2,
+    /* xref case 8747, may be best to avoid TERMINATED_AND_CLEANED where possible */
+    THREAD_SYNCH_TERMINATED_AND_CLEANED=3,
+
+    /* A target thread that has THREAD_SYNCH_NO_LOCKS_NO_XFER is acceptable.
+     * Xref case 6821.
+     */
+    THREAD_SYNCH_SUSPENDED_VALID_MCONTEXT_OR_NO_XFER=4,
+
+    THREAD_SYNCH_SUSPENDED_VALID_MCONTEXT=5
+} thread_synch_state_t;
+
+typedef enum {
+    THREAD_SYNCH_RESULT_SUCCESS         = 1,
+    THREAD_SYNCH_RESULT_SUSPEND_FAILURE = 2,
+    THREAD_SYNCH_RESULT_NOT_SAFE        = 3,
+} thread_synch_result_t;
+
+/* option bits that affect both synch_with_all_threads() and synch_with_thread(),
+ * though the two routines interpret some options in different ways
+ */
+enum {
+    /* these 3 are mutually exclusive and apply to failures to suspend or
+     * to get the context of a target thread.  the first two apply to
+     * both synch_with_thread() and all_threads() while the last only
+     * applies to all_threads().
+     */
+    THREAD_SYNCH_SUSPEND_FAILURE_ABORT  = 0x00000001,
+    THREAD_SYNCH_SUSPEND_FAILURE_IGNORE = 0x00000002,
+    /* retry will cause synch_with_all_threads to keep trying until hit
+     * the loop max, so may want to combine w/ SMALL_LOOP_MAX.
+     * retry does not apply to synch_with_thread(), only to all_threads().
+     */
+    THREAD_SYNCH_SUSPEND_FAILURE_RETRY  = 0x00000004,
+
+    /* specifies a much smaller loop max */
+    THREAD_SYNCH_SMALL_LOOP_MAX         = 0x00000008,
+};
+
+/* convenience macros */
+#define THREAD_SYNCH_IS_CLEANED(desired_synch_state)  (                \
+     desired_synch_state == THREAD_SYNCH_SUSPENDED_AND_CLEANED ||      \
+     desired_synch_state == THREAD_SYNCH_TERMINATED_AND_CLEANED        \
+     )
+#define THREAD_SYNCH_IS_TERMINATED(desired_synch_state) (              \
+     desired_synch_state == THREAD_SYNCH_TERMINATED_AND_CLEANED        \
+     )
+#define THREAD_SYNCH_SAFE(synch_perm, desired_perm) (                  \
+     synch_perm >= desired_perm                                        \
+     )
+
+void
+synch_init(void);
+
+void
+synch_exit(void);
+
+void
+synch_thread_init(dcontext_t *dcontext);
+
+void
+synch_thread_exit(dcontext_t *dcontext);
+
+bool
+thread_synch_state_no_xfer(dcontext_t *dcontext);
+
+/* Only valid while holding all_threads_synch_lock and thread_initexit_lock.  Set to
+ * whether synch_with_thread was successful.
+ * Cannot be called when THREAD_SYNCH_*_AND_CLEANED was requested as the
+ * thread-local memory will be freed on success!
+ */
+bool
+thread_synch_successful(thread_record_t *tr);
+
+/* returns a thread_synch_result_t value
+ * id - the thread you want to synch with
+ * block - whether or not should spin until synch is succesful
+ * hold_initexit_lock - whether or not the caller holds the thread_initexit_lock
+ * caller_state - a given permission define from above that describes the 
+ *                current state of the caller (note that holding the initexit
+ *                lock is ok with respect to NO_LOCK
+ * desired_state - a requested state define from above that describes the 
+ *                 desired synchronization
+ * flags - options from THREAD_SYNCH_ bitmask values
+ * NOTE - if you hold the initexit_lock and block with greater than NONE for
+ * caller state, then initexit_lock may be released and reaquired
+ * NOTE - if any of the nt_ routines fails, it is assumed the thread no longer
+ * exists and returns true
+ * NOTE - if called directly (i.e. not through synch_with_all_threads) 
+ * requires THREAD_SYNCH_IS_SAFE(caller_state, desired_state) to avoid deadlock
+ * NOTE - Requires the caller is !could_be_linking (i.e. not in an 
+ * enter_couldbelinking state)
+ * NOTE - you can't call this with a thread that you've already suspended
+ */
+thread_synch_result_t
+synch_with_thread(thread_id_t id, bool block, bool hold_initexit_lock,
+                  thread_synch_permission_t cur_state,
+                  thread_synch_state_t desired_state,
+                  uint flags);
+
+/* held on return from synch_with_all_threads together with thread_initexit_lock*/
+extern mutex_t all_threads_synch_lock;
+
+/* desired_synch_state - a requested state define from above that describes
+ *                        the synchronization required
+ * threads, num_threads - must not be NULL, if !THREAD_SYNCH_IS_CLEANED(desired
+ *                        synch_state) then will hold a list and num of threads
+ * cur_state - a given permission from above that describes the state of the
+ *             caller
+ * flags - options from THREAD_SYNCH_ bitmask values
+ * NOTE - Requires that the caller doesn't hold the thread_initexit_lock, on
+ * return caller will hold the thread_initexit_lock
+ * NOTE - Requires the caller is !could_be_linking (i.e. not in an 
+ * enter_couldbelinking state)
+ * NOTE - To avoid deadlock this routine should really only be called with 
+ * cur_state giving maximum permissions, (currently app_exit and detach could
+ * conflict, except our routes to app_exit go through different synch point 
+ * (TermThread or TermProcess) first
+ * Caller should call end_synch_with_all_threads when finished.
+ * Note - if this is a CLIENT_INTERFACE build and the caller doesn't intend to resume all
+ * threads (say detach or process exit) then it should first call
+ * instrument_client_thread_termination() to inform the client that client-owned threads
+ * will be killed/permanently stopped.
+ */
+bool
+synch_with_all_threads(thread_synch_state_t desired_synch_state,
+                       thread_record_t ***threads,
+                       int *num_threads, thread_synch_permission_t cur_state,
+                       uint flags);
+void
+end_synch_with_all_threads(thread_record_t **threads, uint num_threads, bool resume);
+void
+resume_all_threads(thread_record_t **threads, const uint num_threads);
+
+/* a fast way to tell a thread if it should call check_wait_at_safe_spot
+ * if translating context would be expensive */
+bool
+should_wait_at_safe_spot(dcontext_t *dcontext);
+
+/* checks to see if any threads are waiting to synch with this one and waits 
+ * if they are
+ * cur_state - a given permission define from above that describes the current
+ *             state of the caller
+ * NOTE - Requires the caller is !could_be_linking (i.e. not in an 
+ * enter_couldbelinking state)
+ */
+void
+check_wait_at_safe_spot(dcontext_t *dcontext, thread_synch_permission_t cur_state);
+
+bool
+at_safe_spot(thread_record_t *trec, dr_mcontext_t *mc,
+             thread_synch_state_t desired_state);
+
+bool
+set_synched_thread_context(thread_record_t *trec,
+                           /* pass either mc or both cxt and cxt_size */
+                           dr_mcontext_t *mc, void *cxt, size_t cxt_size,
+                           thread_synch_state_t desired_state
+                           _IF_X64(byte *cxt_alloc)
+                           _IF_WINDOWS(NTSTATUS *status/*OUT*/));
+
+bool
+translate_mcontext(thread_record_t *trec, dr_mcontext_t *mc, bool restore_memory);
+
+/* resets a thread to start interpreting anew */
+void
+translate_from_synchall_to_dispatch(thread_record_t *tr,
+                                    thread_synch_state_t synch_state);
+
+/*** exported for detach only ***/
+
+bool
+is_at_do_syscall(dcontext_t *dcontext, app_pc pc, byte *esp);
+
+/* adjusts the pending synch count */
+void
+adjust_wait_at_safe_spot(dcontext_t *dcontext, int amt);
+
+#endif /* SYNCH_H */
diff --git a/core/unit-rct.c b/core/unit-rct.c
new file mode 100644
index 000000000..05f08fc7e
--- /dev/null
+++ b/core/unit-rct.c
@@ -0,0 +1,509 @@
+/* **********************************************************
+ * Copyright (c) 2004-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2004-2007 Determina Corp. */
+
+/*
+ * unit-rct.c unit tests for rct.c 
+ */
+
+#include "rct.c"
+
+UNIT_TEST_MAIN
+
+/* This part is more of a regression test - but the rest of the unit
+ * test can use its own executable image as a good test case.
+ * FIXME: convert the first part into a regression test to run us on top of it
+ * or even better - run DR on top of the unit tests
+ */
+
+#include <ctype.h>
+
+typedef int (*fconvert_t)(int c);
+typedef int (*fmult_t)(int);
+
+int foo(int a, bool lower)
+{
+    fconvert_t f = toupper;
+    int res;
+    if (lower)
+        f = tolower;
+    res = f(a);
+    LOG(GLOBAL, LOG_ALL, 1, "foo('%c',%d): '%c'\n", a, lower, res);
+    return res;
+}
+
+int f2(int a)
+{
+    return 2*a;
+}
+
+int f3(int a)
+{
+    return 3*a;
+}
+
+int f7(int a)
+{
+    return 7*a;
+}
+
+int 
+bar(int a, fmult_t f)
+{
+    int x = f2(a);
+    int y = f3(a);
+    int z = f(a);
+    
+    LOG(GLOBAL, LOG_ALL, 1, "bar(%d): %d %d %d\n", x, y, z);
+    return z;
+}
+
+/*  Writable yet initialized data indeed needs to be processed.*/
+fmult_t farr[2] = {f2, f7};
+
+static void 
+test_indcalls()
+{
+    EXPECT(foo('a', true), 'a');
+    EXPECT(foo('a', false), 'A');
+    EXPECT(foo('Z', true), 'z');
+    EXPECT(foo('Z', false), 'Z');
+    EXPECT(bar(5, f2), 10);
+    EXPECT(bar(7, f3), 21);
+    EXPECT(bar(7, f3), 21);
+}
+
+
+static 
+char
+test_switch_helper(char c)
+{
+    switch (c) {
+    case 'a': return 'j';
+    case 'b': return 'k';
+    case 'c': return 'o';
+    default: return c;
+    }
+}
+
+static 
+void
+test_switch()
+{
+    EXPECT(test_switch_helper('a'),'j');
+    EXPECT(test_switch_helper('z'),'z');
+}
+
+/* start of real unit test */
+
+/* work on a small arrays of carefully planted values 
+ *
+ * TODO: verify end
+ * of region conditions - and add this at the end of a page to verify
+ * not reaching out to bad memory out of the array
+ */
+static int 
+test_small_array(dcontext_t *dcontext)
+{
+    /*
+     * [0 1 2 3 4 5 6 7] 8)
+     * [4 3 2 1 5 3 2 1]
+     */
+
+    char arr[100];
+
+    arr[0]=4;
+    arr[1]=3;
+    arr[2]=2;
+    arr[3]=1;
+
+    arr[4+0]=5;
+    arr[4+1]=3;
+    arr[4+2]=2;
+    arr[4+3]=1;
+
+    mutex_lock(&rct_module_lock); /* around whole sequence */
+
+    EXPECT(find_address_references(dcontext, (app_pc)arr, (app_pc)(arr+8), 
+                                   (app_pc)0x01020304, (app_pc)0x01020304),
+           0);
+    /* clean up to start over */
+    EXPECT(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1), 
+           0);
+
+    EXPECT(find_address_references(dcontext, arr, arr+8, 
+                            (app_pc)0x01020304, (app_pc)0x01020305),
+           1);
+    EXPECT(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1), 
+           1);
+
+    /* repetition */
+    EXPECT(find_address_references(dcontext, arr, arr+8, 
+                            (app_pc)0x01020304, (app_pc)0x01020305),
+           1);
+    EXPECT(find_address_references(dcontext, arr, arr+8, 
+                            (app_pc)0x01020304, (app_pc)0x01020305),
+           0);
+    EXPECT(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1), 
+           1);
+    EXPECT(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1), 
+           0);
+
+
+    EXPECT(find_address_references(dcontext,arr, arr+8, 
+                                   (app_pc)0x01020304, (app_pc)0x01020309),
+           2);
+    EXPECT(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1), 
+           2);
+
+
+    EXPECT(find_address_references(dcontext,arr, arr+8, 
+                                   (app_pc)0x01020304, (app_pc)0x01020309),
+           2);
+    EXPECT(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)0x01020304), 
+           0);
+    EXPECT(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)0x01020305), 
+           1);
+    EXPECT(invalidate_ind_branch_target_range(dcontext, (app_pc)0x01020306, (app_pc)0x01020309), 
+           0);
+    EXPECT(invalidate_ind_branch_target_range(dcontext, (app_pc)0x01020305, (app_pc)0x01020306), 
+           1);
+    EXPECT(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1), 
+           0);
+
+
+    EXPECT(find_address_references(dcontext,arr+1, arr+8, 
+                                   (app_pc)0x01020304, (app_pc)0x01020309),
+           1);
+    EXPECT(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1), 
+           1);
+
+
+    EXPECT(find_address_references(dcontext,arr+1, arr+8, 
+                                   (app_pc)0x01020305, (app_pc)0x01020309),
+           1);
+    EXPECT(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1), 
+           1);
+
+
+    EXPECT(find_address_references(dcontext,arr+1, arr+8, 
+                            (app_pc)0x01020306, (app_pc)0x01020309),
+           0);
+    EXPECT(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1), 
+           0);
+
+    EXPECT(find_address_references(dcontext,arr+4, arr+8, 
+                            (app_pc)0x01020300, (app_pc)0x01020309),
+           1);
+    EXPECT(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1), 
+           1);
+
+    EXPECT(find_address_references(dcontext,arr+5, arr+8, 
+                            (app_pc)0x01020300, (app_pc)0x01020309),
+           0);
+
+    /* unreadable */
+    EXPECT(find_address_references(dcontext, (app_pc)5, (app_pc)8, 
+                            (app_pc)0x01020300, (app_pc)0x01020309),
+           0);
+
+    /* all address space for code */
+    EXPECT(find_address_references(dcontext,arr, arr+8, 
+                            (app_pc)0, (app_pc)-1),
+           5); /* all match */
+
+    EXPECT(find_address_references(dcontext,arr, arr+8, 
+                            (app_pc)0, (app_pc)-1),
+           0); /* all duplicates of last search */
+
+    EXPECT(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1), 
+           5);
+
+
+    arr[0]=4;
+    arr[1]=3;
+    arr[2]=2;
+    arr[3]=1;
+
+    arr[4+0]=4;                 /* duplicate entry */
+    arr[4+1]=3;
+    arr[4+2]=2;
+    arr[4+3]=1;
+
+    /* all address space for code */
+    EXPECT(find_address_references(dcontext,arr, arr+8, 
+                            (app_pc)0, (app_pc)-1),
+           4); /* all match but we have a duplicate */
+    EXPECT(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1), 
+           4);
+
+    EXPECT(find_address_references(dcontext,arr, arr+8, 
+                            (app_pc)0x01020300, (app_pc)0x01020305),
+           1); /* two matches but with a duplicate */
+    EXPECT(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1), 
+           1);
+
+    EXPECT(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1), 
+           0);
+
+    mutex_unlock(&rct_module_lock);
+
+    return 1;
+}                        
+
+static void
+test_lookup_delete(dcontext_t *dcontext)
+{
+
+    app_pc tag = (app_pc)0x1234567;
+    fragment_t * f = rct_ind_branch_target_lookup(dcontext, tag);
+    EXPECT_RELATION((ptr_uint_t)f, ==, 0);
+
+    mutex_lock(&rct_module_lock);
+    EXPECT(rct_add_valid_ind_branch_target(dcontext, tag), true);
+    EXPECT(rct_add_valid_ind_branch_target(dcontext, tag), false);
+    mutex_unlock(&rct_module_lock);
+
+    f = rct_ind_branch_target_lookup(dcontext, tag);
+    EXPECT_RELATION((ptr_uint_t)f, !=, 0);
+    rct_flush_ind_branch_target_entry(dcontext, (FutureAfterCallFragment*)f);
+    f = rct_ind_branch_target_lookup(dcontext, tag);
+    EXPECT_RELATION((ptr_uint_t)f, ==, 0);
+}
+
+static void
+test_self_direct(dcontext_t *dcontext)
+{
+    app_pc base_pc;
+    size_t size;
+    uint found;
+    uint newfound;
+
+#ifdef WINDOWS
+    /* this will get both code and data FIXME: data2data reference
+     * will be the majority
+     */
+    size = get_allocation_size((app_pc)test_self_direct, &base_pc);
+#else
+    /* platform agnostic but only looks for current CODE section, and
+     * on windows is not quite what we want - since base_pc will just
+     * be just page aligned
+     */
+    get_memory_info((app_pc)test_self_direct, &base_pc, &size, NULL);
+#endif
+
+    mutex_lock(&rct_module_lock);
+    found = find_address_references(dcontext, 
+                                    base_pc, base_pc+size, 
+                                    base_pc, base_pc+size);
+    mutex_unlock(&rct_module_lock);
+
+    /* guesstimate */
+    EXPECT_RELATION(found, >, 140);
+#ifdef WINDOWS
+    /* FIXME: note data2data have a huge part here */
+    EXPECT_RELATION(found, <, 20000);
+#else
+    EXPECT_RELATION(found, <, 1000);
+#endif
+    EXPECT(is_address_taken(dcontext, (app_pc)f3), true);
+    EXPECT(is_address_taken(dcontext, (app_pc)f2), true);
+    EXPECT(is_address_taken(dcontext, (app_pc)f7), true); /* array reference only */
+
+
+    /* it is pretty hard to produce the address of a static
+     * (e.g. test_self) without making it address taken ;) so we just
+     * add a number to known to be good one's */
+    EXPECT(is_address_taken(dcontext, (app_pc)f3 + 1), false);
+    EXPECT(is_address_taken(dcontext, (app_pc)f3 + 2), false);
+    EXPECT(is_address_taken(dcontext, (app_pc)f2 + 1), false);
+    EXPECT(is_address_taken(dcontext, (app_pc)f7 + 1), false);
+
+    mutex_lock(&rct_module_lock);
+    EXPECT(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1), 
+           found);
+    EXPECT_RELATION(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1)
+                    , == , 0);  /* nothing missed */
+    mutex_unlock(&rct_module_lock);
+
+    /* now try manually rct_analyze_module_at_violation */
+
+    mutex_lock(&rct_module_lock);
+    EXPECT(rct_analyze_module_at_violation(dcontext, (app_pc)test_self_direct), true);
+
+    /* should be all found */
+    /* FIXME: with the data2data in fact a few noisy entries show up
+     * since second lookup in data may differ from original
+     */
+    newfound = find_address_references(dcontext, 
+                                       base_pc, base_pc+size, 
+                                       base_pc, base_pc+size);
+    EXPECT_RELATION(newfound, <, 4);
+    EXPECT_RELATION(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1)
+                    , > , found + newfound - 5); /* FIXME: data references uncomparable */
+
+    EXPECT_RELATION(invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1)
+                    , == , 0);  /* nothing missed */
+    mutex_unlock(&rct_module_lock);
+}
+
+static void
+test_rct_ind_branch_check(void)
+{
+    uint found;
+
+    linkstub_t l;
+    fragment_t f;
+
+    /* to pass args security_violation assumes present */
+    dcontext_t *dcontext = create_new_dynamo_context(true/*initial*/);
+
+    f.tag = (app_pc)0xbeef;
+    l.fragment = &f;
+    l.flags = LINK_INDIRECT | LINK_CALL;
+    set_last_exit(dcontext, &l);
+    dcontext->logfile = GLOBAL;
+
+    /* this should auto call rct_analyze_module_at_violation((app_pc)test_self) */
+    EXPECT(rct_ind_branch_check(dcontext, (app_pc)f3), 
+           1);
+    EXPECT(rct_ind_branch_check(dcontext, (app_pc)f3), 
+           1);
+
+    /* running in -detect_mode we should get -1 */
+    EXPECT(rct_ind_branch_check(dcontext, (app_pc)f3+1), 
+           -1);
+
+    /* not code */
+    EXPECT(rct_ind_branch_check(dcontext, (app_pc)0xbad), 
+           2);
+
+    /* starting over */
+    mutex_lock(&rct_module_lock);
+    invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1);
+    mutex_unlock(&rct_module_lock);
+
+    EXPECT(rct_ind_branch_check(dcontext, (app_pc)f3), 1);
+    EXPECT(rct_ind_branch_check(dcontext, (app_pc)f2), 1);
+    EXPECT(rct_ind_branch_check(dcontext, (app_pc)f7), 1); /* array reference only */
+
+    /* it is pretty hard to produce the address of a static
+      * (e.g. test_self) without making it address taken ;) so we just
+      * add a number to known to be good one's */
+    EXPECT(rct_ind_branch_check(dcontext, (app_pc)f3 + 1), -1);
+    EXPECT(rct_ind_branch_check(dcontext, (app_pc)f3 + 2), -1);
+    EXPECT(rct_ind_branch_check(dcontext, (app_pc)f2 + 1), -1);
+    EXPECT(rct_ind_branch_check(dcontext, (app_pc)f7 + 1), -1);
+
+    mutex_lock(&rct_module_lock);
+    found = invalidate_ind_branch_target_range(dcontext, 0, (app_pc)-1);
+    mutex_unlock(&rct_module_lock);
+    
+    EXPECT_RELATION(found, >, 140);
+}
+
+/*
+ * TODO: LoadLibrary(kernel32) and work on that
+ */
+static void
+test_loaddll()
+{
+    /* TODO: LoadLibrary/GetProcAddress */
+    /* and dlopen/dlsym */
+    LOG(GLOBAL, LOG_ALL, 1, "test_loaddll: NYI\n");
+}
+
+/*
+ * TODO: add a unit test that in fact creates multiple sections by using
+ * #pragma(code_section) that is used by device drivers to mark
+ * PAGEABLE code sections
+ *
+*/
+
+#if TEST_MULTI_SECTIONS         /* NYI */
+#pragma code_seg(".my_code1")
+void func2() { 
+}
+
+#pragma code_seg(push, r1, ".my_code2")
+void func3() { 
+}
+
+#pragma code_seg(pop, r1)      /* back to my_code1 */
+void func4() {
+}
+#pragma code_seg                /* back in .text */
+#endif
+
+
+static int
+unit_main(void) {
+    dcontext_t *dcontext = GLOBAL_DCONTEXT;
+
+    standalone_init();
+    /* keep in mind that not all units are properly initialized above */
+    fragment_init();
+
+    /* options have to be set on the command line since after
+     * synchronization any overrides will be gone 
+     */
+    EXPECT(dynamo_options.detect_mode, true);
+    EXPECT(dynamo_options.rct_ind_call, 11);
+    EXPECT(dynamo_options.rct_ind_jump, 11);
+
+    /* FIXME: report_current_process calls is_couldbelinking()
+     * maybe we should just set the TEB entry with a good context
+     */
+    EXPECT(dynamo_options.diagnostics, false);
+
+    TESTRUN(test_indcalls());
+    TESTRUN(test_switch());
+
+    TESTRUN(test_lookup_delete(dcontext));
+    TESTRUN(test_small_array(dcontext));
+
+    TESTRUN(test_self_direct(dcontext));
+
+    TESTRUN(test_rct_ind_branch_check());
+
+    TESTRUN(test_loaddll());
+
+    LOG(GLOBAL, LOG_ALL, 1, "DONE unit-rct.c:unit_main()\n");
+
+    dynamo_exited = true;
+    fragment_exit();
+    standalone_exit();
+
+    return 0;
+}
+
diff --git a/core/utils.c b/core/utils.c
new file mode 100644
index 000000000..c73c9c640
--- /dev/null
+++ b/core/utils.c
@@ -0,0 +1,4106 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * utils.c - miscellaneous utilities
+ */
+
+#include "globals.h"
+#include "utils.h"
+#include "module_shared.h"
+#include <string.h>  /* for memset */
+#include <math.h>
+
+#ifdef PROCESS_CONTROL
+# include "moduledb.h"   /* for process control macros */
+#endif 
+
+#ifdef LINUX
+# include <sys/types.h>
+# include <sys/stat.h>
+# include <fcntl.h>
+# include <stdio.h>
+# include <stdlib.h>
+# include <unistd.h>
+# include <errno.h>
+# include <sys/time.h>          /* gettimeofday */
+# include "syscall.h"           /* SYS_gettimeofday */
+#else
+# include <errno.h>
+/* FIXME : remove when syslog macros fixed */
+# include "events.h"
+#endif
+
+#ifdef SHARING_STUDY
+# include "fragment.h" /* print_shared_stats */
+#endif
+#ifdef DEBUG
+# include "fcache.h"
+# include "synch.h" /* all_threads_synch_lock */
+#endif
+
+#include "defines.h" /* for dynamorio_buildmark, etc. */
+
+#include <stdarg.h> /* for varargs */
+
+#include <ctype.h> /* for tolower */
+
+#ifdef SIDELINE
+extern void sideline_exit(void);
+#endif
+
+#if defined(INTERNAL) || defined(DEBUG) || defined(CLIENT_INTERFACE) \
+    || defined(DR_APP_EXPORTS)
+/* use for soft errors that can handle some cleanup: assertions and apichecks
+ * performs some cleanup and then calls os_terminate 
+ */
+static void
+assertion_terminate()
+{
+#ifdef SIDELINE
+    /* kill child threads */
+    if (dynamo_options.sideline) {
+        sideline_stop();
+        sideline_exit();
+    }
+#endif
+    /* set exited status for shared memory watchers, other threads */
+    DOSTATS({
+        if (stats != NULL)
+            GLOBAL_STAT(exited) = true;
+    });
+
+    /* do not try to clean up */
+    os_terminate(NULL, TERMINATE_PROCESS);
+}
+#endif /* defined(INTERNAL) || defined(DEBUG) || defined(CLIENT_INTERFACE) 
+        * || defined(DR_APP_EXPORTS) */
+
+#if defined(INTERNAL) || defined(DEBUG)
+/* checks whether an assert statement should be ignored, 
+ * produces a warning if so and returns true,
+ * otherwise returns false
+ */
+bool
+ignore_assert(const char *assert_stmt, const char *expr)
+{
+    bool ignore = false;
+    if (!IS_STRING_OPTION_EMPTY(ignore_assert_list)) {
+        string_option_read_lock();
+        ignore = check_filter(DYNAMO_OPTION(ignore_assert_list), assert_stmt);
+        string_option_read_unlock();
+    }
+    if (IS_LISTSTRING_OPTION_FORALL(ignore_assert_list)) {
+        ignore = true;
+    }
+    if (ignore) {
+        /* FIXME: could have passed message around */
+        SYSLOG_INTERNAL_WARNING("Ignoring assert %s %s", assert_stmt, expr);
+    }
+    return ignore;
+}
+
+/* Hand-made DO_ONCE used in internal_error b/c ifdefs in
+ * report_dynamorio_problem prevent DO_ONCE itself
+ */
+DECLARE_FREQPROT_VAR(static bool do_once_internal_error, false);
+
+/* abort on internal dynamo error */
+void
+internal_error(char *file, int line, char *expr)
+{
+    /* note that we no longer obfuscate filenames in non-internal builds
+     * xref PR 303817 */
+
+    /* need to produce a SYSLOG Ignore Error here and return right away */
+    /* to avoid adding another ?: in the ASSERT messsage
+     * we'll reconstruct file and line # here
+     */
+    if (!IS_STRING_OPTION_EMPTY(ignore_assert_list)) {
+        char assert_stmt[MAXIMUM_PATH]; /* max unique identifier */
+        /* note the ignore checks are done with a possible recursive
+         * infinte loop if any asserts fail.  Not very safe to set and
+         * unset a static bool either since we'll be noisy.
+         */
+
+        /* Assert identifiers should be an exact match of message
+         * after Internal Error.  Most common look like 'x86/arch.c:142',
+         * but could also look like 'Not implemented @x86/arch.c:142'
+         * or 'Bug #4809 @x86/arch.c:145;Ignore message
+         * @x86/arch.c:146'
+         */
+        snprintf(assert_stmt, BUFFER_SIZE_ELEMENTS(assert_stmt), 
+                 "%s:%d", file, line);
+        NULL_TERMINATE_BUFFER(assert_stmt);
+        ASSERT_CURIOSITY((strlen(assert_stmt) + 1) !=
+                         BUFFER_SIZE_ELEMENTS(assert_stmt));
+        if (ignore_assert(assert_stmt, expr))
+            return;
+        /* we can ignore multiple asserts without triggering the do_once */
+    }
+
+    if (do_once_internal_error) /* recursing, bail out */
+        return;
+    else
+        do_once_internal_error = true;
+
+    report_dynamorio_problem(NULL, DUMPCORE_ASSERTION, NULL, NULL,
+                         "Internal "PRODUCT_NAME" Error: %s:%d %s"
+#if defined(DEBUG) && defined(INTERNAL)
+                         "\n(Error occurred @%d frags)"
+#endif
+                         , file, line, expr
+#if defined(DEBUG) && defined(INTERNAL)
+                             , stats == NULL ? -1 : GLOBAL_STAT(num_fragments)
+#endif
+                         );
+
+    assertion_terminate();
+}
+#endif /* defined(INTERNAL) || defined(DEBUG) */
+
+#if defined(CLIENT_INTERFACE) || defined(DR_APP_EXPORTS)
+/* abort on external application created error, i.e. apicheck */
+void
+external_error(char *file, int line, char *msg)
+{
+    DO_ONCE({
+        /* this syslog is before any core dump, unlike our other reports, but
+         * not worth fixing
+         */
+        SYSLOG(SYSLOG_ERROR, EXTERNAL_ERROR, 4, get_application_name(), 
+               get_application_pid(), PRODUCT_NAME, msg);
+        report_dynamorio_problem(NULL, DUMPCORE_FATAL_USAGE_ERROR, NULL, NULL,
+                                 "Usage error: %s (%s, line %d)", msg, file, line);
+    });
+    assertion_terminate();
+}
+#endif
+
+#ifndef WINDOWS
+/* Our parameters (option string, logdir, etc.) can be configured
+ * through environment variables or, on win32, the registry.
+ * This routine is used to check both of those places.
+ * value is a buffer allocated by the caller to hold the
+ * resulting value.
+ */
+int
+get_parameter(const char *name, char *value, int maxlen)
+{
+    char *envval = getenv(name);
+    /* env var has top priority, then registry */
+    if (envval != NULL) {
+        strncpy(value, envval, maxlen-1);
+        value[maxlen-1]  = '\0'; /* if max no null */
+        return GET_PARAMETER_SUCCESS;
+    } 
+    return GET_PARAMETER_FAILURE;
+}
+
+int
+get_unqualified_parameter(const char *name, char *value, int maxlen)
+{
+    /* we don't use qualified names on linux yet */
+    return get_parameter(name, value, maxlen);
+}
+/* FIXME: the prototype should go to os_shared.h and the above moved to linux */
+#else  /* WINDOWS */
+/* get_parameter is now shared with preinject and is in inject_shared.c. */
+#endif
+
+/****************************************************************************/
+/* SYNCHRONIZATION */
+
+#ifdef DEADLOCK_AVOIDANCE
+/* Keeps the head of a linked list of all mutexes currently 
+   held by a thread.  We also require a LIFO lock unlock order 
+   to keep things simpler (and stricter).
+*/
+struct _thread_locks_t {
+    mutex_t *last_lock;
+};
+
+/* These two locks are never deleted, although innermost_lock is grabbed */
+DECLARE_CXTSWPROT_VAR(mutex_t outermost_lock, INIT_LOCK_FREE(outermost_lock));
+DECLARE_CXTSWPROT_VAR(mutex_t innermost_lock, INIT_LOCK_FREE(innermost_lock));
+
+/* Case 8075: For selfprot we have no way to put a local-scope mutex into
+ * .cspdata, and {add,remove}_process_lock need to write to the
+ * do_threshold_mutex when managing adjacent entries in the lock list, so
+ * we use a global lock instead.  Could be a contention issue but it's only
+ * DEADLOCK_AVOIDANCE builds and there are few uses of DO_THRESHOLD_SAFE.
+ */
+DECLARE_CXTSWPROT_VAR(mutex_t do_threshold_mutex, INIT_LOCK_FREE(do_threshold_mutex));
+
+/* structure field dumper for both name and value, format with %*s%d */
+#define DUMP_NONZERO(v,field) strlen(#field)+1, (v->field ? #field"=" : ""), v->field
+
+/* common format string used for different log files and loglevels */
+#define DUMP_LOCK_INFO_ARGS(depth, cur_lock, prev)                      \
+     "%d lock "PFX": name=%s\nrank=%d owner=%d owning_dc="PFX" "  \
+     "%*s"PIFX" prev="PFX"\n"                                    \
+     "lock %*s%8d %*s%8d %*s%8d %*s%8d %*s%8d+2 %s\n",                  \
+    depth, cur_lock, cur_lock->name, cur_lock->rank,                    \
+    cur_lock->owner, cur_lock->owning_dcontext,                         \
+    DUMP_NONZERO(cur_lock, contended_event), prev,                      \
+    DUMP_NONZERO(cur_lock, count_times_acquired),                       \
+    DUMP_NONZERO(cur_lock, count_times_contended),                      \
+    DUMP_NONZERO(cur_lock, count_times_spin_pause),                     \
+    DUMP_NONZERO(cur_lock, count_times_spin_only),                      \
+    DUMP_NONZERO(cur_lock, max_contended_requests),                     \
+    cur_lock->name
+
+#ifdef INTERNAL
+static void
+dump_mutex_callstack(mutex_t *lock)
+{
+    /* from windbg proper command is
+     *  0:001> dds @@(&lock->callstack) L4
+     */
+#ifdef MUTEX_CALLSTACK
+    uint i;
+    if (INTERNAL_OPTION(mutex_callstack) == 0)
+        return;
+    LOG(GLOBAL, LOG_THREADS, 1, "dump_mutex_callstack %s\n", lock->name);
+    for (i=0; i<INTERNAL_OPTION(mutex_callstack); i++) {
+        /* some macro's call this function, so it is easier to ifdef
+         * only references to callstack */
+        LOG(GLOBAL, LOG_THREADS, 1, "  "PFX"\n", lock->callstack[i]);
+    }
+#endif /* MUTEX_CALLSTACK */
+}
+#endif
+
+void
+dump_owned_locks(dcontext_t *dcontext)
+{            /* LIFO order even though order in releasing doesn't matter */
+    mutex_t *cur_lock;
+    uint depth = 0;
+    cur_lock = dcontext->thread_owned_locks->last_lock;
+    LOG(THREAD, LOG_THREADS, 1, "Owned locks for thread %d dcontext="PFX"\n", 
+        dcontext->owning_thread, dcontext);
+    while (cur_lock != &outermost_lock) {
+        depth++;
+        LOG(THREAD, LOG_THREADS, 1, DUMP_LOCK_INFO_ARGS(depth, cur_lock, 
+                                                        cur_lock->prev_owned_lock));
+        ASSERT(cur_lock->owner == dcontext->owning_thread);
+        cur_lock = cur_lock->prev_owned_lock;
+    }
+}
+
+bool
+thread_owns_no_locks(dcontext_t *dcontext)
+{
+    ASSERT(dcontext != NULL);
+    if (!INTERNAL_OPTION(deadlock_avoidance))
+        return true; /* can't verify since we aren't keeping track of owned locks */
+    return (dcontext->thread_owned_locks->last_lock == &outermost_lock);
+}
+
+bool
+thread_owns_one_lock(dcontext_t *dcontext, mutex_t *lock)
+{
+    mutex_t *cur_lock;
+    ASSERT(dcontext != NULL);
+    if (!INTERNAL_OPTION(deadlock_avoidance))
+        return true; /* can't verify since we aren't keeping track of owned locks */
+    cur_lock = dcontext->thread_owned_locks->last_lock;
+    return (cur_lock == lock && cur_lock->prev_owned_lock == &outermost_lock);
+}
+
+/* Returns true if dcontext thread owns lock1 and lock2 and no other locks */
+bool
+thread_owns_two_locks(dcontext_t *dcontext, mutex_t *lock1, mutex_t *lock2)
+{
+    mutex_t *cur_lock;
+    ASSERT(dcontext != NULL);
+    if (!INTERNAL_OPTION(deadlock_avoidance))
+        return true; /* can't verify since we aren't keeping track of owned locks */
+    cur_lock = dcontext->thread_owned_locks->last_lock;
+    return (cur_lock == lock1 && 
+            cur_lock->prev_owned_lock == lock2 &&
+            lock2->prev_owned_lock == &outermost_lock);
+}
+
+/* Returns true if dcontext thread owns lock1 and optionally lock2
+ * (aquired before lock1) and no other locks. */
+bool
+thread_owns_first_or_both_locks_only(dcontext_t *dcontext, mutex_t *lock1, mutex_t *lock2)
+{
+    mutex_t *cur_lock;
+    ASSERT(dcontext != NULL);
+    if (!INTERNAL_OPTION(deadlock_avoidance))
+        return true; /* can't verify since we aren't keeping track of owned locks */
+    cur_lock = dcontext->thread_owned_locks->last_lock;
+    return (cur_lock == lock1 && 
+            (cur_lock->prev_owned_lock == &outermost_lock ||
+             (cur_lock->prev_owned_lock == lock2 &&
+              lock2->prev_owned_lock == &outermost_lock)));
+}
+
+/* dump process locks that have been acquired at least once */
+/* FIXME: since most mutexes are global we don't have thread private lock lists */
+void
+dump_process_locks()
+{
+    mutex_t *cur_lock;
+    uint depth = 0;
+    uint total_acquired = 0;
+    uint total_contended = 0;
+
+    LOG(GLOBAL, LOG_STATS, 2, "Currently live process locks:\n");
+    /* global list access needs to be synchronized */
+    mutex_lock(&innermost_lock);
+    cur_lock = &innermost_lock;
+    do {
+        depth++;
+        LOG(GLOBAL, LOG_STATS, (cur_lock->count_times_contended ? 1U: 2U), /* elevate contended ones */
+            DUMP_LOCK_INFO_ARGS(depth, cur_lock, cur_lock->next_process_lock));
+        DOLOG((cur_lock->count_times_contended ? 2U: 3U), /* elevate contended ones */
+              LOG_THREADS, {
+            /* last recorded callstack, not necessarily the most contended path */
+            dump_mutex_callstack(cur_lock);
+        });
+        cur_lock = cur_lock->next_process_lock;
+        total_acquired += cur_lock->count_times_acquired;
+        total_contended += cur_lock->count_times_contended;
+        ASSERT(cur_lock);
+        ASSERT(cur_lock->next_process_lock->prev_process_lock == cur_lock);
+        ASSERT(cur_lock->prev_process_lock->next_process_lock == cur_lock);
+        ASSERT(cur_lock->prev_process_lock != cur_lock || cur_lock == &innermost_lock);
+        ASSERT(cur_lock->next_process_lock != cur_lock || cur_lock == &innermost_lock);
+    } while (cur_lock != &innermost_lock);
+    mutex_unlock(&innermost_lock);
+    LOG(GLOBAL, LOG_STATS, 1, "Currently live process locks: %d, acquired %d, contended %d (current only)\n", 
+        depth, total_acquired, total_contended);
+}
+
+uint
+locks_not_closed()
+{
+    mutex_t *cur_lock;
+    uint forgotten = 0;
+    uint ignored = 0;
+    /* Case 8075: we use a global do_threshold_mutex for DEADLOCK_AVOIDANCE.
+     * Leaving the code for a local var here via this bool in case we run
+     * this routine in release build while somehow avoiding the global lock.
+     */
+    static const bool allow_do_threshold_leaks = false;
+
+    /* we assume that we would have removed them from the process list in mutex_close */
+    
+    /* locks assigned with do_threshold_mutex are 'leaked' because it
+     * is too much hassle to find a good place to DELETE them -- though we
+     * now use a global mutex for DEADLOCK_AVOIDANCE so that's not the case.
+     */
+    mutex_lock(&innermost_lock);
+    /* innermost will stay */
+    cur_lock = innermost_lock.next_process_lock;
+    while (cur_lock != &innermost_lock) {
+        if (allow_do_threshold_leaks && cur_lock->rank == LOCK_RANK(do_threshold_mutex)) {
+            ignored++;
+        } else {
+            LOG(GLOBAL, LOG_STATS, 1, "missing DELETE_LOCK on lock "PFX" %s\n",
+                cur_lock, cur_lock->name);
+            forgotten++;
+        }
+        cur_lock = cur_lock->next_process_lock;
+    }
+    mutex_unlock(&innermost_lock);
+    LOG(GLOBAL, LOG_STATS, 3, "locks_not_closed= %d remaining, %d DO_THRESHOLD_SAFE ignored\n", 
+        forgotten, ignored);
+    return forgotten;
+}
+
+void
+locks_thread_init(dcontext_t *dcontext)
+{
+    thread_locks_t *new_thread_locks;
+    new_thread_locks = (thread_locks_t *)
+        UNPROTECTED_GLOBAL_ALLOC(sizeof(thread_locks_t) HEAPACCT(ACCT_OTHER));
+    LOG(THREAD, LOG_STATS, 2, "thread_locks="PFX" size=%d\n", new_thread_locks, 
+        sizeof(thread_locks_t));
+    /* initialize any thread bookkeeping fields before assigning to dcontext */
+    new_thread_locks->last_lock = &outermost_lock;
+    dcontext->thread_owned_locks = new_thread_locks;
+}
+
+void
+locks_thread_exit(dcontext_t *dcontext)
+{
+    /* using global heap and always have to clean up */
+    if (dcontext->thread_owned_locks) {
+        thread_locks_t *old_thread_locks = dcontext->thread_owned_locks;
+        /* when exiting, another thread may be holding the lock instead of the current,
+           CHECK: is this true for detaching */
+        ASSERT(dcontext->thread_owned_locks->last_lock == &thread_initexit_lock ||
+               dcontext->thread_owned_locks->last_lock == &outermost_lock);
+        dcontext->thread_owned_locks = NULL; /* disable thread lock checks before freeing memory */
+        UNPROTECTED_GLOBAL_FREE(old_thread_locks, sizeof(thread_locks_t) HEAPACCT(ACCT_OTHER));
+    }
+}
+
+static void
+add_process_lock(mutex_t *lock)
+{
+    /* add to global locks circular double linked list */
+    LOG(THREAD_GET, LOG_THREADS, 5, "add_process_lock" 
+        DUMP_LOCK_INFO_ARGS(0, lock, lock->prev_process_lock));
+    mutex_lock(&innermost_lock);
+    if (lock->prev_process_lock != NULL) {
+        /* race: someone already added (can only happen for read locks) */
+        LOG(THREAD_GET, LOG_THREADS, 2, "\talready added\n");
+        ASSERT(lock->next_process_lock != NULL);
+        mutex_unlock(&innermost_lock);
+        return;
+    }
+    ASSERT(lock->next_process_lock == NULL || lock == &innermost_lock);
+    ASSERT(lock->prev_process_lock == NULL || lock == &innermost_lock);
+    if (innermost_lock.prev_process_lock == NULL) {
+        innermost_lock.next_process_lock = &innermost_lock;
+        innermost_lock.prev_process_lock = &innermost_lock;
+    }
+    lock->next_process_lock = &innermost_lock;
+    innermost_lock.prev_process_lock->next_process_lock = lock;
+    lock->prev_process_lock = innermost_lock.prev_process_lock;
+    innermost_lock.prev_process_lock = lock;
+    ASSERT(lock->next_process_lock->prev_process_lock == lock);
+    ASSERT(lock->prev_process_lock->next_process_lock == lock);
+    ASSERT(lock->prev_process_lock != lock || lock == &innermost_lock);
+    ASSERT(lock->next_process_lock != lock || lock == &innermost_lock);
+    mutex_unlock(&innermost_lock);
+
+}
+
+static void
+remove_process_lock(mutex_t *lock)
+{
+    LOG(THREAD_GET, LOG_THREADS, 3, "remove_process_lock" 
+        DUMP_LOCK_INFO_ARGS(0, lock, lock->prev_process_lock));
+    STATS_ADD(total_acquired, lock->count_times_acquired);
+    STATS_ADD(total_contended, lock->count_times_contended);
+    if (lock->count_times_acquired == 0) {
+        ASSERT(lock->prev_process_lock == NULL);
+        LOG(THREAD_GET, LOG_THREADS, 3, "\tnever acquired\n");
+        return;
+    }
+    ASSERT(lock->prev_process_lock && "if ever acquired should be on the list");
+    ASSERT(lock != &innermost_lock && "innermost will be 'leaked'");
+    /* remove from global locks list */
+    mutex_lock(&innermost_lock);
+    /* innermost should always have both fields set here */
+    lock->next_process_lock->prev_process_lock = lock->prev_process_lock;
+    lock->prev_process_lock->next_process_lock = lock->next_process_lock;
+    lock->next_process_lock = NULL;
+    lock->prev_process_lock = NULL; /* so we catch uses after closing */
+    mutex_unlock(&innermost_lock);
+}
+
+#ifdef MUTEX_CALLSTACK
+/* FIXME: generalize and merge w/ CALL_PROFILE? */
+static void
+mutex_collect_callstack(mutex_t *lock)
+{
+    uint max_depth = INTERNAL_OPTION(mutex_callstack);
+    uint depth = 0;
+    uint skip = 2; /* ignore calls from deadlock_avoidance() and mutex_lock() */
+    /* FIXME: write_lock could ignore one level further */
+    byte *fp;
+    dcontext_t *dcontext = get_thread_private_dcontext();
+
+    GET_FRAME_PTR(fp);
+
+    /* only interested in DR addresses which should all be readable */
+    while (depth < max_depth &&
+           (is_on_initstack(fp) || is_on_dstack(dcontext, fp))
+#ifdef STACK_GUARD_PAGE
+           /* is_on_initstack() and is_on_dstack() do include the guard pages
+            * yet we cannot afford to call is_readable_without_exception()
+            */
+           && !is_stack_overflow(dcontext, fp)
+#endif
+           ) {
+        app_pc our_ret = *((app_pc*)fp+1);
+        fp = *(byte **)fp;
+        if (skip) {
+            skip--;
+            continue;
+        }            
+        lock->callstack[depth] = our_ret;
+        depth++;
+    }
+}
+#endif /* MUTEX_CALLSTACK */
+
+enum {LOCK_NOT_OWNABLE, LOCK_OWNABLE};
+
+/* if not acquired only update statistics,
+   if not ownable (i.e. read lock) only check against previous locks, 
+   but don't add to thread owned list
+ */
+static void
+deadlock_avoidance_lock(mutex_t *lock, bool acquired, bool ownable)
+{
+    if (acquired) {
+        lock->count_times_acquired++;
+        /* CHECK: everything here works without mutex_trylock's */
+        LOG(GLOBAL, LOG_THREADS, 6, "acquired lock "PFX" %s rank=%d, %s dcontext, tid:%d, %d time\n", 
+            lock, lock->name, lock->rank,
+            get_thread_private_dcontext() ? "valid" : "not valid",
+            get_thread_id(),
+            lock->count_times_acquired
+            );
+        LOG(THREAD_GET, LOG_THREADS, 6, "acquired lock "PFX" %s rank=%d\n",
+            lock, lock->name, lock->rank);
+        ASSERT(lock->rank > 0 && "initialize with INIT_LOCK_FREE");
+        if (ownable) {
+            ASSERT(!lock->owner);
+            lock->owner = get_thread_id();
+            lock->owning_dcontext = get_thread_private_dcontext();
+        }
+        /* add to global list */
+        if (lock->prev_process_lock == NULL && lock != &innermost_lock) {
+            add_process_lock(lock);
+        }
+
+        /* cannot hold thread_initexit_lock while couldbelinking, else will
+         * deadlock with flushers
+         */
+        ASSERT(lock != &thread_initexit_lock || !is_self_couldbelinking());
+
+        if (INTERNAL_OPTION(deadlock_avoidance) && get_thread_private_dcontext() != NULL) {
+            dcontext_t *dcontext = get_thread_private_dcontext();
+            if (dcontext->thread_owned_locks != NULL) {
+#ifdef CLIENT_INTERFACE
+                /* PR 198871: same label used for all client locks so allow same rank.
+                 * For now we ignore rank order when client lock is 1st, as well,
+                 * to support decode_trace() for 0.9.6 release PR 198871 covers safer
+                 * long-term fix.
+                 */
+                bool first_client = (dcontext->thread_owned_locks->last_lock->rank ==
+                                     dr_client_mutex_rank);
+                bool both_client = (first_client && lock->rank == dr_client_mutex_rank);
+#endif
+                if (dcontext->thread_owned_locks->last_lock->rank >= lock->rank
+                    IF_CLIENT_INTERFACE(&& !first_client/*FIXME PR 198871: remove */
+                                        && !both_client)) {
+                    /* like syslog don't synchronize options for dumpcore_mask */
+                    if (TEST(DUMPCORE_DEADLOCK, DYNAMO_OPTION(dumpcore_mask)))
+                        os_dump_core("rank order violation");
+                    /* report rank order violation */
+                    SYSLOG_INTERNAL_NO_OPTION_SYNCH(SYSLOG_CRITICAL, 
+                                                    "rank order violation %s acquired after %s in tid:%x",
+                                                    lock->name, 
+                                                    dcontext->thread_owned_locks->last_lock->name, get_thread_id());
+                    dump_owned_locks(dcontext);
+                }
+                ASSERT((dcontext->thread_owned_locks->last_lock->rank < lock->rank
+                        IF_CLIENT_INTERFACE(|| first_client/*FIXME PR 198871: remove */
+                                            || both_client)) && "rank order violation");
+                if (ownable) {
+                    lock->prev_owned_lock = dcontext->thread_owned_locks->last_lock;
+                    dcontext->thread_owned_locks->last_lock = lock;
+                }
+                DOLOG(6, LOG_THREADS, {
+                    dump_owned_locks(dcontext);
+                });
+            }
+        }
+        if (INTERNAL_OPTION(mutex_callstack) != 0 && 
+            ownable &&
+            get_thread_private_dcontext() != NULL) {
+#ifdef MUTEX_CALLSTACK
+            mutex_collect_callstack(lock);
+#endif
+        }
+    } else {
+        /* NOTE check_wait_at_safe_spot makes the assumption that no system
+         * calls are made on the non acquired path here */
+        ASSERT(lock->rank > 0 && "initialize with INIT_LOCK_FREE");
+        if (INTERNAL_OPTION(deadlock_avoidance) && ownable) {
+            ASSERT(lock->owner != get_thread_id() && "deadlock on recursive mutex_lock");
+        }
+        lock->count_times_contended++;
+    }
+}
+
+/* FIXME: exported only for the linux hack -- make static once that's fixed */
+void
+deadlock_avoidance_unlock(mutex_t *lock, bool ownable)
+{
+    if (INTERNAL_OPTION(simulate_contention)) {
+        /* with higher chances another thread will have to wait */
+        thread_yield();
+    }
+
+    LOG(GLOBAL, LOG_THREADS, 6, "released lock "PFX" %s rank=%d, %s dcontext, tid:%d \n", 
+        lock, lock->name, lock->rank,
+        get_thread_private_dcontext() ? "valid" : "not valid",
+        get_thread_id());
+    LOG(THREAD_GET, LOG_THREADS, 6, "released lock "PFX" %s rank=%d\n", 
+        lock, lock->name, lock->rank);
+    if (!ownable)
+        return;
+
+    ASSERT(lock->owner == get_thread_id());
+    if (INTERNAL_OPTION(deadlock_avoidance) && lock->owning_dcontext != NULL)
+        {
+            dcontext_t *dcontext = get_thread_private_dcontext();
+            if (dcontext == NULL) {
+#ifdef DEBUG                
+                /* thread_initexit_lock and all_threads_synch_lock
+                 * are unlocked after tearing down thread structures 
+                 */
+# if defined(LINUX) && !defined(HAVE_TLS)
+                extern mutex_t tls_lock;
+# endif
+                bool null_ok = (lock == &thread_initexit_lock ||
+                                lock == &all_threads_synch_lock
+# if defined(LINUX) && !defined(HAVE_TLS)
+                                || lock == &tls_lock
+# endif
+                                );
+                ASSERT(null_ok);
+#endif
+            }
+            else {
+                ASSERT(lock->owning_dcontext == dcontext);
+                if (dcontext->thread_owned_locks != NULL) {
+                    DOLOG(6, LOG_THREADS, {
+                        dump_owned_locks(dcontext);
+                    });
+                    /* LIFO order even though order in releasing doesn't matter */
+                    ASSERT(dcontext->thread_owned_locks->last_lock == lock);
+                    dcontext->thread_owned_locks->last_lock = lock->prev_owned_lock;
+                    lock->prev_owned_lock = NULL;
+                }
+            }
+        }
+    lock->owner = INVALID_THREAD_ID;
+    lock->owning_dcontext = NULL;
+}
+#define DEADLOCK_AVOIDANCE_LOCK(lock, acquired, ownable) deadlock_avoidance_lock(lock, acquired, ownable) 
+#define DEADLOCK_AVOIDANCE_UNLOCK(lock, ownable) deadlock_avoidance_unlock(lock, ownable)
+#else
+#  define DEADLOCK_AVOIDANCE_LOCK(lock, acquired, ownable) /* do nothing */
+#  define DEADLOCK_AVOIDANCE_UNLOCK(lock, ownable) /* do nothing */
+#endif /* DEADLOCK_AVOIDANCE */
+
+static uint spinlock_count = 0;     /* initialized in utils_init, but 0 is always safe */
+DECLARE_FREQPROT_VAR(static uint random_seed, 1234); /* initialized in utils_init */
+DEBUG_DECLARE(static uint initial_random_seed;)
+
+void
+utils_init()
+{
+    /* FIXME: We need to find a formula (or a better constant) based on real experiments
+       also see comment on spinlock_count_on_SMP in optionsx.h */
+    /* we want to make sure it is 0 on UP, the rest is speculation */
+    spinlock_count = (get_num_processors() - 1) * DYNAMO_OPTION(spinlock_count_on_SMP);
+
+    /* allow reproducing PRNG sequence
+     * (of course, thread scheduling may still affect requests) */
+    random_seed = (DYNAMO_OPTION(prng_seed) == 0) ? 
+        os_random_seed() : DYNAMO_OPTION(prng_seed);
+    /* logged only at end, preserved so can be looked up in a dump */
+    DODEBUG(initial_random_seed = random_seed;);
+
+    /* sanity check since we cast back and forth */
+    ASSERT(sizeof(spin_mutex_t) == sizeof(mutex_t));
+
+    ASSERT(sizeof(uint64) == 8);
+    ASSERT(sizeof(uint32) == 4);
+    ASSERT(sizeof(uint) == 4);
+    ASSERT(sizeof(reg_t) == sizeof(void *));
+
+#ifdef LINUX /* after options_init(), before we open logfile or call instrument_init() */
+        /* Problem: tcsh uses descriptors 3-5 for piping stdin, stdout, and
+         * stderr.  Those numbers are hardcoded, so if we open descriptor 3 for
+         * our logfile, it gets tied up with stdout!
+         * Hack to fix: open up 3 descriptors pointing to /dev/null at beginning so
+         * our log files start with 6, and let tcsh close the 3 bogus files.
+         * FIXME: close these on exit
+         */
+        if (DYNAMO_OPTION(open_tcsh_fds)) {
+            file_t foo = 0;
+            while (foo <= 5) {
+                foo = os_open("/dev/null", OS_OPEN_WRITE|OS_OPEN_APPEND);
+                ASSERT(foo != INVALID_FILE);
+            }
+        }
+#endif
+}
+
+/* NOTE since used by spinmutex_lock_no_yield, can make no system calls before
+ * the lock is grabbed (required by check_wait_at_safe_spot). */
+bool
+spinmutex_trylock(spin_mutex_t *spin_lock)
+{
+    mutex_t *lock = &spin_lock->lock;
+    int mutexval;
+    mutexval = atomic_swap(&lock->lock_requests, LOCK_SET_STATE);
+    ASSERT(mutexval == LOCK_FREE_STATE || mutexval == LOCK_SET_STATE);
+    DEADLOCK_AVOIDANCE_LOCK(lock, mutexval == LOCK_FREE_STATE, LOCK_OWNABLE);
+    return (mutexval == LOCK_FREE_STATE);
+}
+
+void
+spinmutex_lock(spin_mutex_t *spin_lock)
+{
+    /* busy-wait until mutex is locked */
+    while (!spinmutex_trylock(spin_lock)) {
+        thread_yield();
+    }
+    return;
+}
+
+/* special version of spinmutex_lock that makes no system calls (i.e. no yield)
+ * as required by check_wait_at_safe_spot */
+void
+spinmutex_lock_no_yield(spin_mutex_t *spin_lock)
+{
+    /* busy-wait until mutex is locked */
+    while (!spinmutex_trylock(spin_lock)) {
+#ifdef DEADLOCK_AVOIDANCE
+        mutex_t *lock = &spin_lock->lock;
+        /* Trylock inc'ed count_times_contended, but since we are prob. going
+         * to spin a lot, we'd rather attribute that to a separate counter
+         * count_times_spin_pause to keep the counts meaningful. */
+        lock->count_times_contended--;
+        lock->count_times_spin_pause++;
+#endif
+        SPINLOCK_PAUSE();
+    }
+    return;
+}
+
+void
+spinmutex_unlock(spin_mutex_t *spin_lock)
+{
+    mutex_t *lock = &spin_lock->lock;
+    /* if this fails, it means you don't already own the lock. */
+    ASSERT(lock->lock_requests > LOCK_FREE_STATE && "lock not owned");
+    ASSERT(lock->lock_requests == LOCK_SET_STATE);
+    DEADLOCK_AVOIDANCE_UNLOCK(lock, LOCK_OWNABLE);
+    lock->lock_requests = LOCK_FREE_STATE;
+    /* NOTE - check_wait_at_safe_spot requires that no system calls be made
+     * after we release the lock */
+    return;
+}
+
+void
+spinmutex_delete(spin_mutex_t *spin_lock)
+{
+    ASSERT(spin_lock->lock.contended_event == CONTENTION_EVENT_NOT_CREATED);
+    mutex_delete(&spin_lock->lock);
+}
+
+void
+mutex_lock(mutex_t *lock)
+{
+    bool acquired;
+
+    if (INTERNAL_OPTION(spin_yield_mutex)) {
+        spinmutex_lock((spin_mutex_t *)lock);
+        return;
+    } 
+    
+    /* we may want to first spin the lock for a while if we are on a multiprocessor machine */
+    /* option is external only so that we can set it to 0 on a uniprocessor */
+    if (spinlock_count) {
+        uint i;
+        /* in the common case we'll just get it */
+        if (mutex_trylock(lock))
+            return;
+
+        /* otherwise contended, we should spin for some time */
+        i = spinlock_count;
+        /* while spinning we are PAUSEing and reading without LOCKing the bus in the spin loop */
+        do {
+            /* hint we are spinning */
+            SPINLOCK_PAUSE();
+
+            /* We spin only while lock_requests == 0 which means that exactly one thread
+               holds the lock, while the current one (and possibly a few others) are
+               contending on who will grab it next.  It doesn't make much sense to spin
+               when the lock->lock_requests > 0 (which means that at least one thread is
+               already blocked).  And of course, we also break if it is LOCK_FREE_STATE.
+            */
+            if (lock->lock_requests != LOCK_SET_STATE) {
+#               ifdef DEADLOCK_AVOIDANCE
+                lock->count_times_spin_only++;
+#               endif
+                break;
+            }
+            i--;
+        } while (i>0);
+    }
+
+    /* we have strong intentions to grab this lock, increment requests */
+    acquired = atomic_inc_and_test(&lock->lock_requests);
+    DEADLOCK_AVOIDANCE_LOCK(lock, acquired, LOCK_OWNABLE);
+
+    if (!acquired) {
+        mutex_wait_contended_lock(lock);
+#       ifdef DEADLOCK_AVOIDANCE
+        DEADLOCK_AVOIDANCE_LOCK(lock, true, LOCK_OWNABLE); /* now we got it  */
+        /* this and previous owner are not included in lock_requests */
+        if (lock->max_contended_requests < (uint)lock->lock_requests)
+            lock->max_contended_requests = (uint)lock->lock_requests;
+#       endif
+    }
+}
+
+/* try once to grab the lock, return whether or not successful */
+bool
+mutex_trylock(mutex_t *lock)
+{
+    bool acquired;
+
+    if (INTERNAL_OPTION(spin_yield_mutex)) {
+        return spinmutex_trylock((spin_mutex_t *)lock);
+    }
+    
+    /* preserve old value in case not LOCK_FREE_STATE */
+    acquired = atomic_compare_exchange(&lock->lock_requests, 
+                                       LOCK_FREE_STATE, LOCK_SET_STATE);
+    /* if old value was free, that means we just obtained lock
+       old value may be >=0 when several threads are trying to acquire lock,
+       so we should return false
+     */
+    DEADLOCK_AVOIDANCE_LOCK(lock, acquired, LOCK_OWNABLE);
+    return acquired;
+}
+
+/* free the lock */
+void
+mutex_unlock(mutex_t *lock)
+{
+    if (INTERNAL_OPTION(spin_yield_mutex)) {
+        spinmutex_unlock((spin_mutex_t *)lock);
+        return;
+    }
+
+    ASSERT(lock->lock_requests > LOCK_FREE_STATE && "lock not owned");
+    DEADLOCK_AVOIDANCE_UNLOCK(lock, LOCK_OWNABLE);
+    
+    if (atomic_dec_and_test(&lock->lock_requests)) 
+        return;
+    /* if we were not the last one to hold the lock, 
+       (i.e. final value is not LOCK_FREE_STATE) 
+       we need to notify another waiting thread */
+    mutex_notify_released_lock(lock);
+}
+
+/* releases any associated kernel objects */
+void
+mutex_delete(mutex_t *lock)
+{
+    LOG(GLOBAL, LOG_THREADS, 3, "mutex_delete lock "PFX"\n", lock);
+#ifdef DEADLOCK_AVOIDANCE
+    LOG(THREAD_GET, LOG_THREADS, 2, "mutex_delete" DUMP_LOCK_INFO_ARGS(0, lock, lock->prev_process_lock));
+    remove_process_lock(lock);
+#endif    
+    ASSERT(lock->lock_requests == LOCK_FREE_STATE);
+
+    if (lock->contended_event != CONTENTION_EVENT_NOT_CREATED) {
+#ifdef WINDOWS
+        /* FIXME: better abstraction.  should be mutex_free_contended_event */
+        os_close(lock->contended_event);
+#else
+        ASSERT_NOT_IMPLEMENTED(false);
+#endif
+    }
+}
+
+static inline 
+void
+own_recursive_lock(recursive_lock_t *lock)
+{
+    ASSERT(lock->owner == INVALID_THREAD_ID);
+    ASSERT(lock->count == 0);
+    lock->owner = get_thread_id();
+    ASSERT(lock->owner != INVALID_THREAD_ID);
+    lock->count = 1;
+}
+
+/* FIXME: rename recursive routines to parallel mutex_ routines */
+void
+acquire_recursive_lock(recursive_lock_t *lock)
+{
+    /* we no longer use the pattern of implementing acquire_lock as a
+       busy try_lock 
+    */
+
+    /* ASSUMPTION: reading owner field is atomic */
+    if (lock->owner == get_thread_id()) {
+        lock->count++;
+    } else {
+        mutex_lock(&lock->lock);
+        own_recursive_lock(lock);
+    }
+}
+
+bool
+try_recursive_lock(recursive_lock_t *lock)
+{
+    /* ASSUMPTION: reading owner field is atomic */
+    if (lock->owner == get_thread_id()) {
+        lock->count++;
+    } else {
+        if (!mutex_trylock(&lock->lock))
+            return false;
+        own_recursive_lock(lock);
+    }
+    return true;
+}
+
+void
+release_recursive_lock(recursive_lock_t *lock)
+{
+    ASSERT(lock->owner == get_thread_id());
+    ASSERT(lock->count > 0);
+    lock->count--;
+    if (lock->count == 0) {
+        lock->owner = INVALID_THREAD_ID;
+        mutex_unlock(&lock->lock);
+    }
+}
+
+bool
+self_owns_recursive_lock(recursive_lock_t *lock)
+{
+    /* ASSUMPTION: reading owner field is atomic */
+    return (lock->owner == get_thread_id());
+}
+
+/* Read write locks */
+/* A read write lock allows multiple readers or alternatively a single writer */
+
+/* We're keeping here an older implementation under
+   INTERNAL_OPTION(spin_yield_rwlock) that spins on the contention
+   path.  In the Attic we also have the initial naive implementation
+   wrapping mutex_t'es !INTERNAL_OPTION(fast_rwlock).
+*/
+
+/* 
+   FIXME: Since we are using multiple words to contain the state,
+   we still have to keep looping on contention events.
+
+   We need to switch to using a single variable for this but for now
+   let's first put all pieces of the kernel objects support together.
+
+   PLAN:  All state should be in one 32bit word.
+   Then we need one atomic operation that decrements readers and tells us:
+   1) whether there was a writer (e.g. MSB set)
+   2) whether this was the last reader (e.g. 0 in all other bits)
+   Only when 1) & 2) are true (e.g. 0x80000000) we need to notify the writer.
+   Think about using XADD: atomic_add_exchange(state, -1)
+*/
+/* FIXME: See /usr/src/linux-2.4/include/asm-i386/rwlock.h,
+   spinlock.h and /usr/src/linux-2.4/arch/i386/kernel/semaphore.c
+   for the Linux kernel implementation on x86.
+ */
+
+/* Currently we are using kernel objects to block on contention paths.
+   Writers are blocked from each other at the mutex_t, and are notified
+   by previous readers by an auto event.  Readers, of course, can have
+   the lock simultaneously, but block on a previous writer - note also
+   on an auto event.  Broadcasting to all readers is done by
+   explicitly waking up each by the previous one, while the writer
+   continues execution.  There is no fairness to the readers that are
+   blocked vs any new readers that will grab the lock immediately, and
+   for that matter vs any new writers.
+    
+   FIXME: Keep in mind that a succesful wait on the kernel events in read
+   locks should not be used as a guarantee that the current thread can
+   proceed with a granted request.  We should rather keep looping to
+   verify that we are back on the fast path.
+
+   Due to the two reasons above we still have unbound loops in the
+   rwlock primitives. It also lets the Linux implementation just yield.  
+*/
+
+void read_lock(read_write_lock_t *rw)
+{
+    /* wait for writer here if lock is held 
+     * FIXME: generalize DEADLOCK_AVOIDANCE to both detect
+     * order violations and gather contention stats for
+     * this mutex-less synch
+     */
+    if (INTERNAL_OPTION(spin_yield_rwlock)) {
+        do {
+            while (mutex_testlock(&rw->lock)) {
+                /* contended read */
+                /* am I the writer?
+                 * ASSUMPTION: reading field is atomic 
+                 * FIXME: for linux get_thread_id() is expensive -- we
+                 * should either address that through special handling
+                 * of native and new thread cases, or switch this
+                 * routine to pass in dcontext and use that
+                 * FIXME: we could also reorganize this check so that it is done only once
+                 * instead of in the loop body but it doesn't seem wortwhile
+                 */
+                if (rw->writer == get_thread_id()) {
+                    /* we would share the code below but we do not want
+                     * the deadlock avoidance to consider this an acquire
+                     */
+                    ATOMIC_INC(int, rw->num_readers);
+                    return;
+                }
+                DEADLOCK_AVOIDANCE_LOCK(&rw->lock, false, LOCK_NOT_OWNABLE);
+                /* FIXME: last places where we yield instead of wait */
+                thread_yield();
+            }
+            ATOMIC_INC(int, rw->num_readers);
+            if (!mutex_testlock(&rw->lock))
+                break;
+            /* else, race with writer, must try again */
+            ATOMIC_DEC(int, rw->num_readers);
+        } while (true);
+        DEADLOCK_AVOIDANCE_LOCK(&rw->lock, true, LOCK_NOT_OWNABLE);
+        return;
+    }
+
+
+    /* event based notification, yet still need to loop */
+    do {
+        while (mutex_testlock(&rw->lock)) {
+            /* contended read */
+            /* am I the writer?
+             * ASSUMPTION: reading field is atomic 
+             * FIXME: for linux get_thread_id() is expensive -- we
+             * should either address that through special handling
+             * of native and new thread cases, or switch this
+             * routine to pass in dcontext and use that
+             */
+            if (rw->writer == get_thread_id()) {
+                /* we would share the code below but we do not want
+                 * the deadlock avoidance to consider this an acquire
+                 */
+                /* we also have to do this check on the read_unlock path  */
+                ATOMIC_INC(int, rw->num_readers);
+                return;
+            }
+            DEADLOCK_AVOIDANCE_LOCK(&rw->lock, false, LOCK_NOT_OWNABLE);
+
+            ATOMIC_INC(int, rw->num_pending_readers);
+            /* if we get interrupted before we have incremented this counter? 
+               Then no signal will be send our way, so we shouldn't be waiting then
+            */
+            if (mutex_testlock(&rw->lock)) {
+                /* still holding up */
+                rwlock_wait_contended_reader(rw);
+            } else {
+                /* otherwise race with writer */
+                /* after the write lock is released pending readers
+                   should no longer wait since noone will wake them up */
+                /* no need to pause */
+            }
+            /* Even if we didn't wait another reader may be waiting for notification */
+            if (!atomic_dec_becomes_zero(&rw->num_pending_readers)) {
+                /* If we were not the last pending reader,
+                   we need to notify another waiting one so that 
+                   it can get out of the contention path.
+                */
+                rwlock_notify_readers(rw);
+                /* Keep in mind that here we don't guarantee that after blocking
+                   we have an automatic right to claim the lock.
+                */ 
+            }
+        }
+        /* fast path */
+        ATOMIC_INC(int, rw->num_readers);
+        if (!mutex_testlock(&rw->lock))
+            break;
+        /* else, race with writer, must try again */
+        /* FIXME: need to get num_readers and the mutex in one place,
+           or otherwise add a mutex grabbed by readers for the above
+           test.
+        */
+        ATOMIC_DEC(int, rw->num_readers);
+        /* What if a writer thought that this reader has
+         already taken turn - and will then wait thinking this guy has
+         grabbed the read lock first?  For now we'll have to wake up
+         the writer to retry even if it spuriously wakes up the next writer.
+        */
+        // FIXME: we need to do only when num_readers has become zero,
+        // but it is OK for now as this won't usually happen
+        rwlock_notify_writer(rw); /* --ok since writers still have to loop */
+        /* hint we are spinning */
+        SPINLOCK_PAUSE();
+    } while (true);
+
+    DEADLOCK_AVOIDANCE_LOCK(&rw->lock, true, LOCK_NOT_OWNABLE);
+}
+
+void write_lock(read_write_lock_t *rw)
+{
+    /* we do not follow the pattern of having lock call trylock in
+     * a loop because that would be unfair to writers -- first guy
+     * in this implementation gets to write
+     */
+    if (INTERNAL_OPTION(spin_yield_rwlock)) {
+        mutex_lock(&rw->lock);
+        while (rw->num_readers > 0) {
+            /* contended write */
+            DEADLOCK_AVOIDANCE_LOCK(&rw->lock, false, LOCK_NOT_OWNABLE);
+            /* FIXME: last places where we yield instead of wait */
+            thread_yield();
+        }
+        rw->writer = get_thread_id();
+        return;
+    }
+
+    mutex_lock(&rw->lock);
+    /* We still do this in a loop, since the event signal doesn't guarantee 
+       that num_readers is 0 when unblocked. 
+     */
+    while (rw->num_readers > 0) {
+        /* contended write */
+        DEADLOCK_AVOIDANCE_LOCK(&rw->lock, false, LOCK_NOT_OWNABLE);
+        rwlock_wait_contended_writer(rw);
+    }
+    rw->writer = get_thread_id();
+}
+
+bool write_trylock(read_write_lock_t *rw)
+{
+    if (mutex_trylock(&rw->lock)) {
+        ASSERT_NOT_TESTED();
+        if (rw->num_readers == 0) {
+            rw->writer = get_thread_id();
+            return true;
+        } else {
+            /* We need to duplicate the bottom of write_unlock() */
+            /* since if a new reader has appeared after we have acquired the lock
+               that one may already be waiting on the broadcast event */
+            mutex_unlock(&rw->lock);
+            /* check whether any reader is currently waiting */
+            if (rw->num_pending_readers > 0) {
+                /* after we've released the write lock, pending readers will no longer wait */
+                rwlock_notify_readers(rw);
+            }
+        }
+    }
+    return false;
+}
+
+void read_unlock(read_write_lock_t *rw)
+{
+    if (INTERNAL_OPTION(spin_yield_rwlock)) {
+        ATOMIC_DEC(int, rw->num_readers);
+        DEADLOCK_AVOIDANCE_UNLOCK(&rw->lock, LOCK_NOT_OWNABLE);
+        return;
+    }
+
+    /* if we were the last reader to hold the lock, (i.e. final value is 0) 
+       we may need to notify a waiting writer */
+
+    /* unfortunately even on the hot path (of a single reader) we have
+       to check if the writer is in fact waiting.  Even though this is
+       not atomic we don't need to loop here - write_lock() will loop.
+    */
+    if (atomic_dec_becomes_zero(&rw->num_readers)) {
+        /* if the writer is waiting it definitely needs to hold the mutex */
+        if (mutex_testlock(&rw->lock)) {
+            /* test that it was not this thread owning both write and read lock */
+            if (rw->writer != get_thread_id()) {
+                /* we're assuming the writer has been forced to wait,
+                   but since we can't tell whether it did indeed wait this 
+                   notify may leave signaled the event for the next turn
+
+                   If the writer has grabbed the mutex and checked
+                   when num_readers==0 and has gone assuming to be the
+                   rwlock owner.  In that case the above
+                   rwlock_notify_writer will give the wrong signal to
+                   the next writer.  
+                   --ok since writers still have to loop 
+                */
+                rwlock_notify_writer(rw);
+            }
+        }
+    }
+
+    DEADLOCK_AVOIDANCE_UNLOCK(&rw->lock, LOCK_NOT_OWNABLE);
+}
+
+void write_unlock(read_write_lock_t *rw)
+{
+#ifdef DEADLOCK_AVOIDANCE
+    ASSERT(rw->writer == rw->lock.owner);
+#endif
+    rw->writer = INVALID_THREAD_ID;
+    if (INTERNAL_OPTION(spin_yield_rwlock)) {
+        mutex_unlock(&rw->lock);
+        return;
+    }
+    /* we need to signal all waiting readers (if any) that they can now go
+       ahead.  No writer should be allowed to lock until all currently
+       waiting readers are unblocked.
+     */
+    /* We first unlock so that any blocked readers can start making
+       progress as soon as they are notified.  Further field
+       accesses however have to be assumed unprotected.
+    */
+    mutex_unlock(&rw->lock);
+    /* check whether any reader is currently waiting */
+    if (rw->num_pending_readers > 0) {
+        /* after we've released the write lock, pending readers will no longer wait */
+        rwlock_notify_readers(rw);
+    }
+}
+
+bool
+self_owns_write_lock(read_write_lock_t *rw)
+{
+    /* ASSUMPTION: reading owner field is atomic */
+    return (rw->writer == get_thread_id());
+}
+
+/***************************************************/
+/* broadcast events */
+
+/* FIXME: once we're happy with these, change rwlock reader
+ *  notification to use these same routines
+ */
+
+struct _broadcast_event_t {
+    event_t event;
+    volatile int num_waiting;
+};
+
+broadcast_event_t *
+create_broadcast_event()
+{
+    broadcast_event_t *be = (broadcast_event_t *)
+        global_heap_alloc(sizeof(broadcast_event_t) HEAPACCT(ACCT_OTHER));
+    be->event = create_event();
+    be->num_waiting = 0;
+    return be;
+}
+
+void
+destroy_broadcast_event(broadcast_event_t *be)
+{
+    destroy_event(be->event);
+    global_heap_free(be, sizeof(broadcast_event_t) HEAPACCT(ACCT_OTHER));
+}
+
+/* NOTE : to avoid races a signaler should always do the required action to 
+ * make any do_wait_condition(s) for the WAIT_FOR_BROADCAST_EVENT(s) on the 
+ * event false BEFORE signaling the event
+ */
+void
+signal_broadcast_event(broadcast_event_t *be)
+{
+    /* we rely on each woken-up thread to wake another */
+    if (be->num_waiting > 0)
+        signal_event(be->event);
+}
+
+/* Note : waiting on a broadcast event is done through the 
+ * WAIT_FOR_BROADCAST_EVENT macro, don't use the helper functions below
+ */
+void 
+intend_wait_broadcast_event_helper(broadcast_event_t *be) 
+{
+    ATOMIC_INC(int, be->num_waiting);
+}
+void 
+unintend_wait_broadcast_event_helper(broadcast_event_t *be)
+{
+    ATOMIC_DEC(int, be->num_waiting);
+}
+void 
+wait_broadcast_event_helper(broadcast_event_t *be)
+{
+    wait_for_event(be->event);
+    /* once woken, we must wake next thread in the chain,
+     * unless we are the last
+     */
+    if (!atomic_dec_becomes_zero(&be->num_waiting)) {
+        signal_event(be->event);
+    }
+}
+
+
+/****************************************************************************/
+/* HASHING */
+
+ptr_uint_t
+hash_value(ptr_uint_t val, hash_function_t func, ptr_uint_t mask, uint bits)
+{
+    if (func == HASH_FUNCTION_NONE)
+        return val;
+    switch(func)
+        {
+        case HASH_FUNCTION_MULTIPLY_PHI:
+            {
+                /* case 8457: keep in sync w/ HASH_VALUE_FOR_TABLE() */
+                return ((val * HASH_PHI) >> (HASH_TAG_BITS - bits));
+            }
+#ifdef INTERNAL
+        case HASH_FUNCTION_LOWER_BSWAP:
+            {
+                IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+                return (((val & 0xFFFF0000)) | 
+                        ((val & 0x000000FF) << 8) | 
+                        ((val & 0x0000FF00) >> 8));
+            }
+        case HASH_FUNCTION_BSWAP_XOR:
+            {
+                IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+                return (val ^ (((val & 0x000000FF) << 24)  | 
+                               ((val & 0x0000FF00) << 8)   |
+                               ((val & 0x00FF0000) >> 8)   |
+                               ((val & 0xFF000000) >> 24)));
+            }
+        case HASH_FUNCTION_SWAP_12TO15:
+            {
+                IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+                return (((val & 0xFFFF0FF0)) |
+                        ((val & 0x0000F000) >> 12) |
+                        ((val & 0x0000000F) << 12));
+            }
+        case HASH_FUNCTION_SWAP_12TO15_AND_NONE:
+            {
+                IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+                return (mask <= 0xFFF ? val : (((val & 0xFFFF0FF0)) |
+                                               ((val & 0x0000F000) >> 12) |
+                                               ((val & 0x0000000F) << 12)));
+            }
+        case HASH_FUNCTION_SHIFT_XOR:
+            {
+                IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+                return val ^ (val >> 12) ^ (val << 12);
+            }
+#endif            
+        default:
+            {
+                ASSERT_NOT_REACHED();
+                return 0;
+            }
+        }
+}
+
+uint
+hashtable_num_bits(uint size)
+{
+    uint bits = 0;
+    uint sz = size;
+    while (sz > 0) {
+        sz = sz >> 1;
+        bits++;
+    }
+    ASSERT(HASHTABLE_SIZE(bits) > size &&
+           HASHTABLE_SIZE(bits) <= size*2);
+    return bits;
+}
+
+/****************************************************************************/
+/* BITMAP */
+
+/* Since there is no ffs() on windows we use the one from
+ *  /usr/src/linux-2.4/include/linux/bitops.h.  TODO: An easier
+ *  x86-specific way using BSF is
+ *  /usr/src/linux-2.4/include/asm/bitops.h
+ */
+
+/* Returns the position of the first set bit - betwen 0 and 31 */
+static inline uint
+bitmap_find_first_set_bit(bitmap_element_t x)
+{
+    int r = 0;
+
+    ASSERT(x);
+    if (!(x & 0xffff)) {
+        x >>= 16;
+        r += 16;
+    }
+    if (!(x & 0xff)) {
+        x >>= 8;
+        r += 8;
+    }
+    if (!(x & 0xf)) {
+        x >>= 4;
+        r += 4;
+    }
+    if (!(x & 3)) {
+        x >>= 2;
+        r += 2;
+    }
+    if (!(x & 1)) {
+        x >>= 1;
+        r += 1;
+    }
+    return r;
+}
+
+/* A block is marked free with a set bit.
+   Returns -1 if no block is found!
+*/
+static inline uint
+bitmap_find_set_block(bitmap_t b, uint bitmap_size)
+{
+    uint i = 0;
+    uint last_index = BITMAP_INDEX(bitmap_size);
+
+    while (b[i] == 0 && i < last_index) 
+        i++;
+    if (i == last_index)
+        return BITMAP_NOT_FOUND;
+    return i*BITMAP_DENSITY + bitmap_find_first_set_bit(b[i]);
+}
+
+/* Looks for a sequence of free blocks 
+ * Returns -1 if no such sequence is found!
+
+ * Considering the fact that the majority of our allocations will be
+ * for a single block this operation is not terribly efficient.
+ */
+static uint
+bitmap_find_set_block_sequence(bitmap_t b, uint bitmap_size, uint requested)
+{
+    uint last_bit = bitmap_size - requested + 1;
+    /* find quickly at least a single block */
+    uint first = bitmap_find_set_block(b, bitmap_size);
+    if (first == BITMAP_NOT_FOUND)
+        return BITMAP_NOT_FOUND;
+    
+    do {
+        /* now check if there is room for the requested number of bits */
+        uint hole_size = 1;
+        while (hole_size < requested && 
+               bitmap_test(b, first + hole_size)) {
+            hole_size++;
+        }
+        if (hole_size == requested)
+            return first;
+        /* otherwise first + hole_size is not set, so we should skip that */
+        first += hole_size + 1;
+        while (first < last_bit && !bitmap_test(b, first))
+            first++;
+    } while (first < last_bit);
+
+    return BITMAP_NOT_FOUND;
+}
+
+void 
+bitmap_initialize_free(bitmap_t b, uint bitmap_size)
+{
+    memset(b, 0xff, BITMAP_INDEX(bitmap_size) * sizeof(bitmap_element_t));
+}
+
+uint
+bitmap_allocate_blocks(bitmap_t b, uint bitmap_size, uint request_blocks)
+{
+    uint i, res;
+    if (request_blocks == 1) {
+        i = bitmap_find_set_block(b, bitmap_size);
+    } else {
+        i = bitmap_find_set_block_sequence(b, bitmap_size, request_blocks);
+    }
+    res = i;
+    if (res == BITMAP_NOT_FOUND)
+        return BITMAP_NOT_FOUND;
+
+    do {
+        bitmap_clear(b, i++);
+    } while (--request_blocks);
+    return res;
+}
+
+void
+bitmap_free_blocks(bitmap_t b, uint bitmap_size, uint first_block, uint num_free)
+{
+    ASSERT(first_block + num_free <= bitmap_size);
+    do {
+        ASSERT(!bitmap_test(b, first_block));
+        bitmap_set(b, first_block++);
+    } while (--num_free);
+}
+
+#ifdef DEBUG
+/* used only for ASSERTs */
+bool
+bitmap_are_reserved_blocks(bitmap_t b, uint bitmap_size, uint first_block, uint num_blocks)
+{
+    ASSERT(first_block + num_blocks <= bitmap_size);
+    do {
+        if (bitmap_test(b, first_block))
+            return false;
+        first_block++;
+    } while (--num_blocks);
+    return true;
+}
+
+static inline
+uint
+bitmap_count_set_bits(bitmap_element_t x)
+{
+    int r = 0;
+
+    /* count set bits in each element */
+    while (x) {
+        r++;
+        x &= x - 1;
+    }
+
+    return r;
+}
+
+bool
+bitmap_check_consistency(bitmap_t b, uint bitmap_size, uint expect_free)
+{
+    uint last_index = BITMAP_INDEX(bitmap_size);
+    uint i;
+    uint current = 0;
+    for (i=0; i < last_index; i++) {
+        current += bitmap_count_set_bits(b[i]);
+    }
+    
+    LOG(GLOBAL, LOG_HEAP, 3, "bitmap_check_consistency(b="PFX", bitmap_size=%d)"
+        " expected=%d current=%d\n", 
+        b, bitmap_size, expect_free, current);
+    return expect_free == current;
+}
+#endif /* DEBUG */
+
+/****************************************************************************/
+/* LOGGING */
+
+file_t
+get_thread_private_logfile() 
+{
+#ifdef DEBUG
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    if (dcontext == NULL)
+        dcontext = GLOBAL_DCONTEXT;
+    return THREAD;
+#else
+    return INVALID_FILE;
+#endif
+}
+
+#ifdef DEBUG
+DECLARE_FREQPROT_VAR(static bool do_once_do_file_write, false);
+#endif
+
+/*  FIXME: add buffering? */
+void 
+do_file_write(file_t f, const char *fmt, va_list ap)
+{
+    size_t size;
+    char logbuf[MAX_LOG_LENGTH];
+
+#ifndef NOLIBC
+    /* W/ libc, we cannot print while .data is protected.  We assume
+     * that DATASEC_RARELY_PROT is .data.
+     */
+    if (DATASEC_PROTECTED(DATASEC_RARELY_PROT)) {
+        ASSERT(TEST(SELFPROT_DATA_RARE, dynamo_options.protect_mask));
+        ASSERT(strcmp(DATASEC_NAMES[DATASEC_RARELY_PROT], ".data") == 0);
+        return;
+    }
+#endif
+    if (f == INVALID_FILE)
+        return;
+    size = vsnprintf(logbuf, BUFFER_SIZE_ELEMENTS(logbuf), fmt, ap);
+    NULL_TERMINATE_BUFFER(logbuf); /* always NULL terminate */
+    /* note that we can't print %f on windows with NOLIBC (returns error 
+     * size == -1), use double_print() or divide_uint64_print() as needed */
+    DODEBUG({
+        /* we have our own do-once to avoid infinite recursion w/ protect_data_section */
+        if (size < 0 || size >= sizeof(logbuf)) {
+            if (!do_once_do_file_write) {
+                do_once_do_file_write = true;
+                ASSERT_CURIOSITY(size >= 0 && size < sizeof(logbuf));
+            }
+        }
+    });
+    /* handle failure values */
+    if (size >= sizeof(logbuf) || size < 0)
+        size = strlen(logbuf);
+    os_write(f, logbuf, size);
+}
+
+/* a little utiliy for printing a float that is formed by dividing 2 uints,
+ * gives back high and low parts for printing, also supports percentages
+ * FIXME : we might need to handle signed numbers at some point (but not yet),
+ * also could be smarter about overflow conditions (i.e. for calculating
+ * bottom or top) but we never call with numbers that big, also truncates
+ * instead of rounding
+ * Usage : given a, b (uint[64]); d_int()==divide_uint64_print(); 
+ *         uint c, d tmp; parameterized on precision p and width w 
+ * note that %f is eqv. to %.6f, 3rd ex. is a percentage ex.
+ * "%.pf", a/(float)b => d_int(a, b, false, p, &c, &d);  "%u.%.pu", c,d
+ * "%w.pf", a/(float)b => d_int(a, b, false, p, &c, &d); "%(w-p-1)u.%.pu", c,d
+ * "%.pf%%", 100*(a/(float)b) => d_int(a, b, true, p, &c, &d); "%u.%.pu%%", c,d
+ */
+void
+divide_uint64_print(uint64 numerator, uint64 denominator, bool percentage, 
+                    uint precision, uint *top, uint *bottom)
+{
+    uint i, precision_multiple, multiple = percentage ? 100 : 1;
+#ifdef HOT_PATCHING_INTERFACE
+    /* case 6657: hotp_only does not have many of the DR stats, so 
+     * numerator and/or denominator may be 0 */
+    ASSERT(denominator != 0 || DYNAMO_OPTION(hotp_only));
+#else
+    ASSERT(denominator != 0);
+#endif
+    ASSERT(top != NULL && bottom != NULL);
+    if (denominator == 0)
+        return;
+    ASSERT_TRUNCATE(*top, uint, ((multiple * numerator) / denominator));
+    *top = (uint) ((multiple * numerator) / denominator);
+    for (i = 0, precision_multiple = 1; i < precision; i++)
+        precision_multiple *= 10;
+    ASSERT_TRUNCATE(*bottom, uint, (((precision_multiple * multiple * numerator) / denominator)
+                                    - (precision_multiple * *top)));
+    /* FUNNY: if I forget the above ) I crash the preprocessor: cc1 internal compiler error 
+     * couldn't reproduce in a smaller set to file a bug against gcc version 3.3.3 (cygwin special)
+     */
+    *bottom = (uint) (((precision_multiple * multiple * numerator) / denominator)
+                      - (precision_multiple * *top));
+}
+
+#if defined(DEBUG) || defined(INTERNAL) || defined(CLIENT_INTERFACE)
+/* for printing a float (can't use %f on windows with NOLIBC), NOTE: you must
+ * preserve floating point state to call this function!! 
+ * FIXME : truncates instead of rounding, also negative with width looks funny,
+ *         finally width can be one off if negative
+ * Usage : given double/float a; uint c, d and char *s tmp; dp==double_print
+ *         parameterized on precision p width w
+ * note that %f is eqv. to %.6f
+ * "%.pf", a => dp(a, p, &c, &d, &s) "%s%u.%.pu", s, c, d
+ * "%w.pf", a => dp(a, p, &c, &d, &s) "%s%(w-p-1)u.%.pu", s, c, d
+ */
+void
+double_print(double val, uint precision, uint *top, uint *bottom, char **sign) 
+{
+    uint i, precision_multiple;
+    ASSERT(top != NULL && bottom != NULL && sign != NULL);
+    if (val < 0.0) {
+        val = -val;
+        *sign = "-";
+    } else {
+        *sign = "";
+    }
+    for (i = 0, precision_multiple = 1; i < precision; i++)
+        precision_multiple *= 10;
+    *top = (uint)val;
+    *bottom = (uint)((val - *top) * precision_multiple);
+}
+#endif /* DEBUG || INTERNAL */
+
+#ifdef WINDOWS
+/* for pre_inject, injector, and core shared files, is just wrapper for syslog
+ * internal */
+void
+display_error(char *msg)
+{
+    SYSLOG_INTERNAL_ERROR("%s", msg);
+}
+#endif
+
+#ifdef DEBUG
+# ifdef WINDOWS
+/* print_symbolic_address is in module.c */
+# else
+/* prints a symbolic name, or best guess of it into a caller provided buffer */
+void
+print_symbolic_address(app_pc tag, char *buf, int max_chars, bool exact_only) {
+    buf[0]='\0';
+}
+# endif
+#endif /* DEBUG */
+
+void 
+print_file(file_t f, char *fmt, ...)
+{
+    va_list ap;
+    va_start(ap, fmt);
+    do_file_write(f, fmt, ap);
+    va_end(ap);
+}
+
+/* N.B.: this routine is duplicated in instrument.c's dr_log!
+ * Maybe export this routine itself, or make another layer of
+ * calls that passes the va_list (-> less efficient)?
+ * For now I'm assuming this routine changes little.
+ */
+void 
+print_log(file_t logfile, uint mask, uint level, char *fmt, ...)
+{
+    va_list ap;
+
+#ifdef DEBUG
+    /* FIXME: now the LOG macro checks these, remove here? */
+    if (logfile == INVALID_FILE || 
+        (stats != NULL &&
+         ((stats->logmask & mask) == 0 ||
+          stats->loglevel < level)))
+        return;
+#else
+    return;
+#endif
+    
+    KSTART(logging);
+    va_start(ap, fmt);
+    do_file_write(logfile, fmt, ap);
+    va_end(ap);
+    KSTOP_NOT_PROPAGATED(logging);
+}
+
+#ifdef WINDOWS
+static void 
+do_syslog(syslog_event_type_t priority, uint message_id, uint substitutions_num, ...)
+{
+    va_list ap;
+    va_start(ap, substitutions_num);
+    os_syslog(priority, message_id, substitutions_num, ap);
+    va_end(ap);
+}
+#endif
+
+/* notify present a notification message to one or more destinations,
+ * depending on the runtime parameters and the priority:
+ *   -syslog_mask controls sending to the system log
+ *   -stderr_mask controls sending to stderr
+ *   -msgbox_mask controls sending to an interactive pop-up window
+ */
+void 
+notify(syslog_event_type_t priority, bool internal, bool synch, 
+       IF_WINDOWS_(uint message_id) uint substitution_num, char *prefix, 
+       char *fmt, ...)
+{
+    char msgbuf[MAX_LOG_LENGTH];
+    int size;
+    va_list ap;
+    va_start(ap, fmt);
+    /* FIXME : the vsnprintf call is not needed in the most common case where
+     * we are going to just os_syslog, but it gets pretty ugly to do that */
+    size = vsnprintf(msgbuf, sizeof(msgbuf), fmt, ap);
+    NULL_TERMINATE_BUFFER(msgbuf);         /* always NULL terminate */
+    ASSERT(size < BUFFER_SIZE_ELEMENTS(msgbuf));
+    va_end(ap);
+
+    LOG(GLOBAL, LOG_ALL, 1, "%s: %s\n", prefix, msgbuf);
+    /* so can skip synchronizing when failure is in option parsing to avoid
+     * infinite recursion, still could be issue with exception, but separate
+     * recursive bailout will at least kill us then, FIXME, note will use 
+     * default masks below if original parse */
+    if (synch)
+        synchronize_dynamic_options(); /* TODO: dynamic THREAD mask */
+    LOG(THREAD_GET, LOG_ALL, 1, "%s: %s\n", prefix, msgbuf);
+
+#ifdef WINDOWS
+    if (TEST(priority, dynamo_options.syslog_mask)) {
+        if (internal) {
+            if (TEST(priority, INTERNAL_OPTION(syslog_internal_mask)))
+                do_syslog(priority, message_id, 3, get_application_name(), 
+                          get_application_pid(), msgbuf);
+        } else {
+            va_start(ap, fmt);
+            os_syslog(priority, message_id, substitution_num, ap);
+            va_end(ap);
+        }
+    }
+#else 
+    /* syslog not yet implemented on linux, FIXME */
+#endif
+
+    if (TEST(priority, dynamo_options.stderr_mask))
+        print_file(STDERR, "<%s>\n", msgbuf);
+
+#ifdef WINDOWS
+    if (TEST(priority, dynamo_options.msgbox_mask)) {
+        /* FIXME : could use os_countdown_msgbox (if ever implemented) here to
+         * do a timed out messagebox, could then also replace the os_timeout in
+         * vmareas.c
+         */
+        debugbox(msgbuf);
+    }
+#endif
+}
+
+/****************************************************************************
+ * REPORTING DYNAMORIO PROBLEMS
+ * Including assertions, curiosity asserts, API usage errors,
+ * deadlock timeouts, internal exceptions, and the app modifying our memory.
+ * 
+ * The following constants are for the pieces of a buffer we will send
+ * to the event log, to diagnostics, and to stderr/msgbox/logfile.
+ * It's static to avoid adding 500+ bytes to the stack on a critical
+ * path, and so needs synchronization, but the risk of a problem with
+ * the lock is worth getting a clear message on the first exception.
+ *
+ * Here's a sample of a report.  First four lines here are the
+ * passed-in custom string fmt, subsequent are the options and call
+ * stack, which are always appended:
+ *   Unrecoverable error at PC 0x15003075
+ *   0xc0000005 0x00000000 0x15003075 0x15003075 0x00000001 0x00000037 
+ *   Registers: eax 0x00000000 ebx 0x00000000 ecx 0x177c9040 edx 0x177c9040
+ *           esi 0x00000b56 edi 0x0000015f esp 0x177e3eb0 eflags 0x00010246
+ *   Delta: 0x0000000
+ *   internal version, custom build
+ *   -loglevel 2 -msgbox_mask 12 -stderr_mask 12 
+ *   0x00342ee8 0x150100f2
+ *   0x00342f64 0x15010576
+ *   0x00342f84 0x1503d77b
+ *   0x00342fb0 0x1503f12c
+ *   0x00342ff4 0x150470e9
+ *   0x77f82b95 0x565308ec
+ */
+/* The magic number 271 comes from MAXIMUM_PATH (on WINDOWS = 260) + 11 for PR 204171
+ * (in other words historical reasons). Xref PR 226547 we use a constant value here
+ * instead of MAXIMUM_PATH since it has different length on Linux and makes this buffer
+ * too long. */
+#ifdef X64
+# define REPORT_MSG_MAX        (271+17*8+8*23+2) /* wider, + more regs */
+#else
+# define REPORT_MSG_MAX        (271)
+#endif
+#define REPORT_LEN_VERSION    37
+  /* example: "\ninternal version, build 94201\n"
+   * For custom builds, the build # is generated as follows
+   * (cut-and-paste from Makefile):
+   * # custom builds: 9XYYZ
+   * # X = developer, YY = tree, Z = diffnum
+   * # YY defaults to 1st 2 letters of CUR_TREE, unless CASENUM is defined,
+   * # in which case it is the last 2 letters of CASENUM (all % 10 of course)
+   */
+#define REPORT_LEN_OPTIONS   192
+  /* still not long enough for ALL non-default options but I'll wager money we'll never
+   * see this option string truncated, at least for non-internal builds
+   * (famous last words?)
+   * List options from staging mode could be problematic though.
+   */
+#define REPORT_NUM_STACK      10
+#ifdef X64
+# define REPORT_LEN_STACK_EACH (22+2*8)
+#else
+# define REPORT_LEN_STACK_EACH 22
+#endif
+  /* just frame ptr, ret addr: "0x0342fc7c 0x77f8c6dd\n" == 22 chars per line */
+#define REPORT_LEN_STACK      (REPORT_LEN_STACK_EACH)*(REPORT_NUM_STACK)
+/* Not persistent across code cache execution, so not protected */
+DECLARE_NEVERPROT_VAR(static char reportbuf[REPORT_MSG_MAX + REPORT_LEN_VERSION +
+                                            REPORT_LEN_OPTIONS + REPORT_LEN_STACK + 1],
+                      {0,});
+DECLARE_CXTSWPROT_VAR(static mutex_t report_buf_lock, INIT_LOCK_FREE(report_buf_lock));
+
+#define ASSERT_ROOM(reportbuf, curbuf, maxlen) \
+    ASSERT(curbuf + maxlen < reportbuf + sizeof(reportbuf))
+
+/* random number generator */
+DECLARE_CXTSWPROT_VAR(static mutex_t prng_lock, INIT_LOCK_FREE(prng_lock));
+
+#ifdef DEBUG
+/* callers should play it safe - no memory allocations, no grabbing locks */
+bool
+under_internal_exception()
+{
+# ifdef DEADLOCK_AVOIDANCE
+    /* ASSUMPTION: reading owner field is atomic */
+    return (report_buf_lock.owner == get_thread_id());
+# else
+    /* mutexes normally don't have an owner, stay safe no matter who owns */
+    return mutex_testlock(&report_buf_lock);
+# endif /* DEADLOCK_AVOIDANCE */
+}
+#endif /* DEBUG */
+
+/* HACK: to avoid duplicating the prefix of the event log message, we
+ * skip it for the SYSLOG, but not the other notifications
+ */
+#ifdef X64
+# define EXCEPTION_PREFIX "Unrecoverable error at PC 0x0000000000000000\n"
+#else
+# define EXCEPTION_PREFIX "Unrecoverable error at PC 0x00000000\n"
+#endif
+#define REPORT_EXCEPTION_SKIP_PREFIX (sizeof(EXCEPTION_PREFIX) - 1/*NULL*/ -1/*include newline!*/)
+
+/* Fine to pass NULL for dcontext, will obtain it for you.
+ * If dumpcore_flag == DUMPCORE_INTERNAL_EXCEPTION, does a full SYSLOG;
+ * else, does a SYSLOG_INTERNAL_ERROR.
+ * Fine to pass NULL for report_ebp: will use current ebp for you.
+ */
+void
+report_dynamorio_problem(dcontext_t *dcontext, uint dumpcore_flag,
+                         app_pc exception_addr, app_pc report_ebp, 
+                         const char *fmt, ...)
+{
+    /* WARNING: this routine is called for fatal errors, and
+     * a fault in DR means that potentially anything could be
+     * inconsistent or corrupted!  Do not grab locks or traverse
+     * data structures or read memory if you can avoid it!
+     */
+    char *curbuf;
+    ptr_uint_t *pc;
+    uint num;
+    int len;
+    va_list ap;
+
+    /* synchronize dynamic options */
+    synchronize_dynamic_options();
+
+    ASSERT(sizeof(reportbuf) < MAX_LOG_LENGTH);
+    if (dcontext == NULL)
+        dcontext = get_thread_private_dcontext();
+    if (dcontext == NULL)
+        dcontext = GLOBAL_DCONTEXT;
+
+    mutex_lock(&report_buf_lock);
+    /* we assume the caller does a DO_ONCE to prevent hanging on a
+     * fault in this routine, if called to report a fatal error.
+     */
+
+    /* now build up the report */
+    curbuf = reportbuf;
+
+    ASSERT_ROOM(reportbuf, curbuf, REPORT_MSG_MAX);
+    va_start(ap, fmt);
+    len = vsnprintf(curbuf, REPORT_MSG_MAX, fmt, ap);
+    curbuf += (len < 0 ? 0 : len); /* any error, even if at max, and we squash */
+    va_end(ap);
+
+    /* don't use dynamorio_version_string, we don't need copyright notice */
+    ASSERT_ROOM(reportbuf, curbuf, REPORT_LEN_VERSION);
+    len = snprintf(curbuf, REPORT_LEN_VERSION, "\n%s, %s\n",
+                       VERSION_NUMBER_STRING, BUILD_NUMBER_STRING);
+    curbuf += (len < 0 ? 0 : len); /* any error, even if at max, and we squash */
+                
+    ASSERT_ROOM(reportbuf, curbuf, REPORT_LEN_OPTIONS);
+    /* leave room for newline */
+    get_dynamo_options_string(&dynamo_options, curbuf, REPORT_LEN_OPTIONS-1, true);
+    /* get_dynamo_options_string will null-terminate even if truncates */
+    curbuf += strlen(curbuf);
+    *(curbuf++) = '\n';
+
+    /* print just frame ptr and ret addr for top of call stack */
+    ASSERT_ROOM(reportbuf, curbuf, REPORT_LEN_STACK);
+    if (report_ebp == NULL) {
+        GET_FRAME_PTR(report_ebp);
+    }
+    for (num = 0, pc = (ptr_uint_t *) report_ebp;
+         num <= REPORT_NUM_STACK && pc != NULL &&
+             is_readable_without_exception_query_os((app_pc) pc, 2*sizeof(reg_t));
+         num++, pc = (ptr_uint_t *) *pc) {
+        len = snprintf(curbuf, REPORT_LEN_STACK_EACH, PFX" "PFX"\n",
+                       pc, *(pc+1));
+        curbuf += (len < 0 ? 0 : len); /* any error, even if at max, and we squash */
+    }
+
+    /* SYSLOG_INTERNAL and diagnostics expect no trailing newline */
+    ASSERT(*(curbuf-1) == '\n');
+    curbuf--;
+    /* now we for sure have room for \0 */
+    *curbuf = '\0';
+    /* now done with reportbuf */
+
+    if (TEST(dumpcore_flag, DYNAMO_OPTION(dumpcore_mask))
+        && DYNAMO_OPTION(live_dump)) {
+        /* non-fatal coredumps attempted before printing further diagnostics */
+        os_dump_core(reportbuf);
+    }
+
+    /* we already synchronized the options at the top of this function and we
+     * might be stack critical so use _NO_OPTION_SYNCH */
+    if (dumpcore_flag == DUMPCORE_INTERNAL_EXCEPTION) {
+        char saddr[IF_X64_ELSE(19,11)];
+        snprintf(saddr, BUFFER_SIZE_ELEMENTS(saddr), PFX, exception_addr);
+        NULL_TERMINATE_BUFFER(saddr);
+        SYSLOG_NO_OPTION_SYNCH(SYSLOG_CRITICAL, EXCEPTION, 4,
+                               get_application_name(), get_application_pid(), 
+                               saddr,
+                               /* skip the prefix since the event log string 
+                                * already has it */
+                               reportbuf + REPORT_EXCEPTION_SKIP_PREFIX);
+    } else if (dumpcore_flag == DUMPCORE_ASSERTION) {
+        /* We need to report ASSERTS in DEBUG=1 INTENERAL=0 builds since we're still
+         * going to kill the process. Xref PR 232783. internal_error() already 
+         * obfuscated the which file info. */
+        SYSLOG_NO_OPTION_SYNCH(SYSLOG_ERROR, INTERNAL_SYSLOG_ERROR, 3,
+                               get_application_name(), get_application_pid(),
+                               reportbuf);
+    } else {
+        SYSLOG_INTERNAL_NO_OPTION_SYNCH(SYSLOG_ERROR, "%s", reportbuf);
+    }
+
+    /* no forensics files for usage error */
+    if (dumpcore_flag != DUMPCORE_FATAL_USAGE_ERROR) {
+        /* NULL for the threat id
+         * We always assume BAD state, even for curiosity asserts, etc., since
+         * diagnostics grabs memory when ok and we can't have that at arbitrary points!
+         */
+        report_diagnostics(reportbuf, NULL, NO_VIOLATION_BAD_INTERNAL_STATE);
+    }
+
+    /* print out pretty call stack */
+    DOLOG(1, LOG_ALL, {
+        if (dumpcore_flag == DUMPCORE_INTERNAL_EXCEPTION)
+            dump_callstack(exception_addr, report_ebp, THREAD, DUMP_NOT_XML);
+        else
+            dump_dr_callstack(THREAD);
+    });
+
+    mutex_unlock(&report_buf_lock);
+
+    if (dumpcore_flag != DUMPCORE_CURIOSITY) {
+        /* print out stats, can't be done inside the report_buf_lock 
+         * because of non-trivial lock rank order violation on the 
+         * snapshot_lock */
+        DOLOG(1, LOG_ALL, {
+            dump_global_stats(false);
+            if (dcontext != GLOBAL_DCONTEXT)
+                dump_thread_stats(dcontext, false);
+        });
+    }
+
+    if (TEST(dumpcore_flag, DYNAMO_OPTION(dumpcore_mask))
+        && !DYNAMO_OPTION(live_dump)) {
+        /* fatal coredump goes last */
+        os_dump_core(reportbuf);
+    }
+}
+
+bool
+is_readable_without_exception_try(byte *pc, size_t size)
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+
+    /* note we need a dcontext for a TRY block */
+    if (dcontext == NULL) {
+        /* FIXME: should rename the current
+         * is_readable_without_exception() to
+         * is_readable_without_exception_os_read(). On each platform
+         * we should pick the fastest implementation for the
+         * non-faulting common case as the default version of
+         * is_readable_without_exception().  Some callers may still call a
+         * specific version if the fast path is not as common.
+         */
+        return is_readable_without_exception(pc, size);
+    }
+
+    TRY(dcontext, {
+        byte *check_pc = (byte *) ALIGN_BACKWARD(pc, PAGE_SIZE);
+        if (size > (size_t)((byte *)POINTER_MAX - pc)) {
+            ASSERT_NOT_TESTED();
+            size = (byte *)POINTER_MAX - pc;
+        }
+        do {
+            PROBE_READ_PC(check_pc);
+            /* note the minor perf benefit - we check the whole loop
+             * in a single TRY/EXCEPT, and no system calls xref
+             * is_readable_without_exception() [based on safe_read]
+             * and is_readable_without_exception_query_os() [based on
+             * query_virtual_memory].
+             */
+
+            check_pc += PAGE_SIZE;
+        } while (check_pc != 0/*overflow*/ && check_pc < pc+size);
+        /* TRY usage note: can't return here */
+    }, EXCEPT(dcontext, {
+        /* no state to preserve */
+        return false;
+    }));
+
+    return true;
+}
+
+bool
+is_string_readable_without_exception(char *str, size_t *str_length /* OPTIONAL OUT */)
+{
+    size_t length = 0;
+    dcontext_t *dcontext = get_thread_private_dcontext();
+
+    if (str == NULL)
+        return false;
+
+    if (dcontext != NULL) {
+        TRY_EXCEPT(dcontext, /* try */ {
+            length = strlen(str);
+            if (str_length != NULL)
+                *str_length = length;
+            /* NOTE - can't return here (try usage restriction) */
+        }, /* except */ {
+            return false;
+        });
+        return true;
+    } else {
+        /* ok have to do this the hard way... */
+        char *cur_page = (char *)ALIGN_BACKWARD(str, PAGE_SIZE);
+        char *cur_str = str;
+        do {
+            if (!is_readable_without_exception((byte *)cur_str,
+                                               (cur_page+PAGE_SIZE)-cur_str)) {
+                return false;
+            }
+            while (cur_str < cur_page + PAGE_SIZE) {
+                if (*cur_str == '\0') {
+                    if (str_length != NULL)
+                        *str_length = length;
+                    return true;
+                }
+                cur_str++;
+                length++;
+            }
+            cur_page += PAGE_SIZE;
+            ASSERT(cur_page == cur_str && ALIGNED(cur_page, PAGE_SIZE));
+        } while (true);
+        ASSERT_NOT_REACHED();
+        return false;
+    }
+}
+
+
+char *
+memprot_string(uint prot)
+{
+    switch (prot) {
+    case (MEMPROT_READ|MEMPROT_WRITE|MEMPROT_EXEC): return "rwx";
+    case (MEMPROT_READ|MEMPROT_WRITE             ): return "rw-";
+    case (MEMPROT_READ|              MEMPROT_EXEC): return "r-x";
+    case (MEMPROT_READ                           ): return "r--";
+    case (             MEMPROT_WRITE|MEMPROT_EXEC): return "-wx";
+    case (             MEMPROT_WRITE             ): return "-w-";
+    case (                           MEMPROT_EXEC): return "--x";
+    case (0                                      ): return "---";
+    }
+    return "<error>";
+}
+
+/* returns true if every byte in the region addr to addr+size is set to val */
+bool
+is_region_memset_to_char(byte *addr, size_t size, byte val)
+{
+    /* FIXME : we could make this much faster with arch specific implementation
+     * (for x86 repe scasd w/proper alignment handling) */
+    size_t i;
+    for (i = 0; i < size; i++) {
+        if (*addr++ != val)
+            return false;
+    }
+    return true;
+}
+
+/* returns pointer to first char of string that matches either c1 or c2 
+ * or NULL if can't find */
+char *
+double_strchr(char *string, char c1, char c2)
+{
+    while (*string != '\0') {
+        if (*string == c1 || *string == c2) {
+            return string;
+        }
+        string++;
+    }
+    return NULL;
+}
+
+#ifndef WINDOWS
+/* returns pointer to last char of string that matches either c1 or c2 
+ * or NULL if can't find */
+const char *
+double_strrchr(const char *string, char c1, char c2)
+{
+    const char *ret = NULL;
+    while (*string != '\0') {
+        if (*string == c1 || *string == c2) {
+            ret = string;
+        }
+        string++;
+    }
+    return ret;
+}
+#else
+/* in inject_shared.c, FIXME : move both copies to a common location */
+#endif
+
+#ifdef WINDOWS
+/* Just like wcslen, but if the string is >= MAX characters long returns MAX
+ * whithout interrogating past str+MAX.  NOTE - this matches most library 
+ * implementations, but does NOT work the same way as the strnlen etc.
+ * functions in the hotpatch2 module (they return MAX+1 for strings > MAX).
+ * The hotpatch2 module implementation is scheduled to be changed. FIXME -
+ * eventually would be nice to share the various string routines used both by
+ * the core and the hotpatch2 module. */
+size_t
+wcsnlen(const wchar_t *str, size_t max)
+{
+    const wchar_t *s = str;
+    size_t i = 0;
+
+    while (i < max && *s != L'\0') {
+        i++;
+        s++;
+    }
+
+    return i;
+}
+#endif
+
+static int
+strcasecmp_with_wildcards(const char *regexp, const char *consider)
+{
+    char cr, cc;
+    while (true) {
+        if (*regexp == '\0') {
+            if (*consider == '\0')
+                return 0;
+            return -1;
+        } else if (*consider == '\0')
+            return 1;
+        ASSERT(*regexp != EOF && *consider != EOF);
+        cr = (char)tolower(*regexp);
+        cc = (char)tolower(*consider);
+        if (cr != '?' && cr != cc) {
+            if (cr < cc)
+                return -1;
+            else
+                return 1;
+        }
+        regexp++;
+        consider++;
+    }
+}
+
+static bool
+check_filter_common(const char *filter, const char *short_name, bool wildcards) 
+{
+    const char *next, *prev;
+    /* FIXME: can we shrink this?  not using full paths here */
+    char consider[MAXIMUM_PATH];
+    bool done = false;
+
+    ASSERT(short_name != NULL && filter != NULL);
+    /* FIXME: consider replacing most of this with
+       strtok_r(copy_filter, ";", &pos) */
+    prev = filter;
+    do {
+        next = strchr(prev, ';');
+        if (next == NULL) {
+            next = prev + strlen(prev);
+            if (next == prev)
+                break;
+            done = true;
+        }
+        strncpy(consider, prev, MIN(BUFFER_SIZE_ELEMENTS(consider), (next-prev)));
+        consider[next-prev] = '\0'; /* if max no null */
+        LOG(THREAD_GET, LOG_ALL, 3, "considering \"%s\" == \"%s\"\n",
+            consider, short_name);
+        if (wildcards && strcasecmp_with_wildcards(consider, short_name) == 0)
+            return true;
+        else if (strcasecmp(consider, short_name) == 0)
+            return true;
+        prev = next + 1;
+    } while (!done);
+    return false;
+}
+
+bool
+check_filter(const char *filter, const char *short_name) 
+{
+    return check_filter_common(filter, short_name, false/*no wildcards*/);
+}
+
+bool
+check_filter_with_wildcards(const char *filter, const char *short_name) 
+{
+    return check_filter_common(filter, short_name, true/*allow wildcards*/);
+}
+
+
+static char logdir[MAXIMUM_PATH];
+static bool logdir_initialized = false;
+static char basedir[MAXIMUM_PATH];
+static bool basedir_initialized = false;
+/* below used in the create_log_dir function to avoid having it on the stack
+ * on what is a critical path for stack depth (diagnostics->create_log_dir->
+ * get_parameter */
+static char old_basedir[MAXIMUM_PATH];
+/* this lock is recursive because current implementation recurses to create the
+ * basedir when called to create the logdir before the basedir is created, is
+ * also useful in case we receive an exception in the create_log_dir function
+ * since it is called in the diagnostics path, we should relook this though
+ * as is probably not the best way to avoid the diagnostics problem FIXME */
+DECLARE_CXTSWPROT_VAR(static recursive_lock_t logdir_mutex,
+                      INIT_RECURSIVE_LOCK(logdir_mutex));
+
+/* enable creating a new base logdir (for a fork, e.g.) */
+void
+enable_new_log_dir()
+{
+    logdir_initialized = false;
+}
+
+void
+create_log_dir(int dir_type)
+{
+#ifdef LINUX
+    char *pre_execve = getenv(DYNAMORIO_VAR_EXECVE_LOGDIR);
+#endif
+    /* synchronize */
+    acquire_recursive_lock(&logdir_mutex);
+    SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+#ifdef LINUX
+    if (dir_type == PROCESS_DIR && pre_execve != NULL) {
+        /* use same dir as pre-execve! */
+        strncpy(logdir, pre_execve, BUFFER_SIZE_ELEMENTS(logdir));
+        NULL_TERMINATE_BUFFER(logdir); /* if max no null */
+        logdir_initialized = true;
+        /* important to remove it, don't want to propagate to forked children */
+        unsetenv(DYNAMORIO_VAR_EXECVE_LOGDIR);
+        /* check that it's gone: we've had problems with unsetenv */
+        ASSERT(getenv(DYNAMORIO_VAR_EXECVE_LOGDIR) == NULL);
+    } else {
+#endif
+        if (dir_type == BASE_DIR) {
+            int retval;
+            ASSERT(sizeof(basedir) == sizeof(old_basedir));
+            strncpy(old_basedir, basedir, sizeof(basedir));
+            retval = get_parameter(L_IF_WIN(DYNAMORIO_VAR_LOGDIR), basedir, 
+                                   sizeof(basedir));
+            if (IS_GET_PARAMETER_FAILURE(retval))
+                basedir[0] = '\0';
+            basedir[sizeof(basedir)-1] =  '\0';
+            if (!basedir_initialized ||
+                strncmp(old_basedir, basedir, sizeof(basedir))) {
+                /* need to create basedir, is changed or not yet created */
+                basedir_initialized = true;
+                /* skip creating dir basedir if is empty */
+                if (basedir[0] == '\0') {
+                    SYSLOG(SYSLOG_WARNING, 
+                           WARNING_EMPTY_OR_NONEXISTENT_LOGDIR_KEY, 2,
+                           get_application_name(), get_application_pid());
+                } else {
+                    if (!os_create_dir(basedir, CREATE_DIR_ALLOW_EXISTING)) {
+                        /* try to create full path */
+                        char swap;
+                        char *end = double_strchr(basedir, DIRSEP, ALT_DIRSEP);
+                        bool res;
+                        while (end) {
+                            swap = *end;
+                            *end = '\0';
+                            res = os_create_dir(basedir, CREATE_DIR_ALLOW_EXISTING);
+                            *end = swap;
+                            end = double_strchr(++end, DIRSEP, ALT_DIRSEP);
+                        }
+                        res = os_create_dir(basedir, CREATE_DIR_ALLOW_EXISTING);
+                        /* check for success */
+                        if (!res) {
+                            SYSLOG(SYSLOG_ERROR, 
+                                   ERROR_UNABLE_TO_CREATE_BASEDIR, 3,
+                                   get_application_name(), 
+                                   get_application_pid(), 
+                                   basedir);
+                            /* everything should work out fine, individual log 
+                             * dirs will also fail to open and just won't be 
+                             * logged to */
+                        }
+                    }
+                }
+            }
+        }
+        /* only create one logging directory (i.e. not dynamic) */
+        else if (dir_type == PROCESS_DIR && !logdir_initialized) {
+            char *base = basedir;
+            if (!basedir_initialized) {
+                create_log_dir(BASE_DIR);
+            }
+            ASSERT(basedir_initialized);
+            logdir_initialized = true;
+            /* skip creating if basedir is empty */
+            if (*base != '\0') {
+                if (!get_unique_logfile("", logdir, sizeof(logdir), true, NULL)) {
+                    SYSLOG_INTERNAL_WARNING("Unable to create log directory %s", 
+                                            logdir);
+                }
+            }
+        }
+#ifdef LINUX
+    }
+#endif
+    SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+    release_recursive_lock(&logdir_mutex);
+    
+#ifdef DEBUG
+    if (stats != NULL) {
+        /* if null, we're trying to report an error (probably via a core dump),
+         * so who cares if we lose logdir name */
+        strncpy(stats->logdir, logdir, sizeof(stats->logdir));
+        stats->logdir[sizeof(stats->logdir)-1]  = '\0'; /* if max no null */
+    }
+    if (dir_type == PROCESS_DIR
+# ifdef LINUX
+        && !post_execve
+# endif
+        )
+        SYSLOG_INTERNAL_INFO("log dir=%s", logdir);
+#endif /* DEBUG */
+}
+
+/* Copies the name of the specified directory into buffer, returns true if 
+ * the specifed buffer has been initialized (if it hasn't then no copying is
+ * done).
+ * Will not copy more than *buffer_length bytes and does not ensure null 
+ * termination.
+ * buffer can be NULL
+ * buffer_length can be NULL, but only if buffer is NULL
+ * on return (if it is not NULL) *buffer_length will hold the length of the 
+ * specified directory's name (including the terminating NULL, i.e. the number 
+ * of chars written to the buffer assuming the buffer was not NULL and large
+ * enough) 
+ */
+bool
+get_log_dir(log_dir_t dir_type, char *buffer, uint *buffer_length)
+{
+    bool target_initialized = false;
+    char *target_dir = NULL;
+    ASSERT(buffer == NULL || buffer_length != NULL);
+    acquire_recursive_lock(&logdir_mutex);
+    if (dir_type == BASE_DIR) {
+        target_dir = basedir;
+        target_initialized = basedir_initialized;
+    } else if (dir_type == PROCESS_DIR) {
+        target_dir = logdir;
+        target_initialized = logdir_initialized;
+    } else {
+        /* should never get here */
+        ASSERT(false);
+    }
+    if (buffer != NULL && target_initialized) {
+        strncpy(buffer, target_dir, *buffer_length);
+    }
+    if (buffer_length != NULL && target_initialized) {
+        ASSERT_TRUNCATE(*buffer_length, uint, strlen(target_dir) + 1);
+        *buffer_length = (uint) strlen(target_dir) + 1;
+    }
+    release_recursive_lock(&logdir_mutex);
+    return target_initialized;
+}
+
+/*#ifdef LINUX
+ *  N.B.: if you create a log file, you'll probably want to create a new one
+ *  upon a fork.  Should we require a callback passed in to this routine?
+ *  For clients we have a dynamorio_fork_init routine.  For internal modules,
+ *  for now make your own fork_init routine.
+ *#endif
+ */
+file_t
+open_log_file(const char *basename, char *finalname_with_path, uint maxlen)
+{
+    file_t file;
+    char name[MAXIMUM_PATH];
+    uint name_size = BUFFER_SIZE_ELEMENTS(name);
+    name[0] = '\0';
+
+    if (!get_log_dir(PROCESS_DIR, name, &name_size)) {
+        create_log_dir(PROCESS_DIR);
+        if (!get_log_dir(PROCESS_DIR, name, &name_size)) {
+            ASSERT_NOT_REACHED();
+        }
+    }
+    NULL_TERMINATE_BUFFER(name);
+    /* skip if logdir empty */
+    if (name[0] == '\0') 
+        return INVALID_FILE;
+    snprintf(&name[strlen(name)], BUFFER_SIZE_ELEMENTS(name) - strlen(name), 
+             "%c%s.%d."IDFMT".html", DIRSEP, basename, 
+             get_thread_num(get_thread_id()), get_thread_id());
+    NULL_TERMINATE_BUFFER(name);
+#ifdef LINUX
+    if (post_execve) /* reuse same log file */
+        file = os_open(name, OS_OPEN_WRITE|OS_OPEN_APPEND|OS_OPEN_ALLOW_LARGE);
+    else
+#endif
+        file = os_open(name, OS_OPEN_WRITE|OS_OPEN_REQUIRE_NEW|OS_OPEN_ALLOW_LARGE);
+    if (file == INVALID_FILE) {
+        SYSLOG_INTERNAL_WARNING_ONCE("Cannot create log file %s", name);
+        /* everything should work out fine, log statements will just fail to 
+         * write since invalid handle */
+    }
+    /* full path is often too long, so just print final dir and file name */
+#ifdef LINUX
+    if (!post_execve) 
+#endif
+        {
+            /* Note that we won't receive a message for the global logfile
+             * since the caller won't have set it yet.  However, we will get
+             * all thread log files logged here. */
+            LOG(GLOBAL, LOG_THREADS, 1, 
+                "created log file %s\n", 
+                double_strrchr(name, DIRSEP, ALT_DIRSEP) + 1);
+        }
+    if (finalname_with_path != NULL) {
+        strncpy(finalname_with_path, name, maxlen);
+        finalname_with_path[maxlen-1]  = '\0'; /* if max no null */
+    }
+    return file;
+}
+
+/* Generalize further as needed
+ * Creates a unique file or directory of the form 
+ * BASEDIR/[app_name].[pid].<unique num of up to 8 digits>[file_type]
+ * If the filename_buffer is not NULL, the filename of the obtained file
+ * is copied there. For creating a directory the file argument is expected
+ * to be null.  Return true if the requested file or directory was created 
+ * and, in the case of a file, returns a handle to file in the file argument.
+ */
+bool
+get_unique_logfile(const char *file_type, char *filename_buffer, uint maxlen,
+                   bool open_directory, file_t *file)
+{
+    char buf[MAXIMUM_PATH];
+    uint size = BUFFER_SIZE_ELEMENTS(buf), counter = 0, base_offset;
+    bool success = false;
+    ASSERT((open_directory && file == NULL) || 
+           (!open_directory && file != NULL));
+    if (!open_directory) 
+        *file = INVALID_FILE;
+    create_log_dir(BASE_DIR);
+    if (get_log_dir(BASE_DIR, buf, &size)) {
+        NULL_TERMINATE_BUFFER(buf);
+        ASSERT_TRUNCATE(base_offset, uint, strlen(buf));
+        base_offset = (uint) strlen(buf);
+        buf[base_offset++] = DIRSEP;
+        size = BUFFER_SIZE_ELEMENTS(buf) - base_offset;
+        do {
+            snprintf(&(buf[base_offset]), size, "%s.%s.%.8d%s", 
+                     get_app_name_for_path(), get_application_pid(),
+                     counter, file_type);
+            NULL_TERMINATE_BUFFER(buf);
+            if (open_directory) {
+                success = os_create_dir(buf, CREATE_DIR_REQUIRE_NEW);
+            } else {
+                *file = os_open(buf, OS_OPEN_REQUIRE_NEW|OS_OPEN_WRITE);
+                success = (*file != INVALID_FILE);
+            }
+        } while (!success && counter++ < 99999999 && 
+                 os_file_exists(buf, open_directory));
+        DOLOG(1, LOG_ALL, {
+            if (!success)
+                LOG(GLOBAL, LOG_ALL, 1, "Failed to create unique logfile %s\n", buf);
+            else 
+                LOG(GLOBAL, LOG_ALL, 1, "Created unique logfile %s\n", buf);
+        });
+    }
+
+    /* copy the filename over if we have a valid buffer */
+    if (NULL != filename_buffer) {
+        strncpy(filename_buffer, buf, maxlen);
+        filename_buffer[maxlen - 1] = '\0'; /* NULL terminate */
+    }
+
+    return success;
+}
+
+const char*
+get_app_name_for_path()
+{
+    return get_short_name(get_application_name());
+}
+
+const char*
+get_short_name(const char *exename)
+{
+    const char *exe;
+    exe = double_strrchr(exename, DIRSEP, ALT_DIRSEP);
+    if (exe == NULL)
+        exe = exename;
+    else
+        exe++; /* skip (back)slash */
+    return exe;
+}
+
+/****************************************************************************/
+
+#ifdef DEBUG
+/* given an array of size size of integers, computes and prints the
+ * min, max, mean, and stddev
+ */
+void
+print_statistics(int *data, int size)
+{
+    int i;
+    int min, max;
+    /* our context switch does not save & restore floating point state,
+     * so we have to do it here!
+     */
+    PRESERVE_FLOATING_POINT_STATE_START();
+    double mean, stddev, sum;
+    uint top, bottom;
+    char *sign;
+
+    sum = 0.;
+    min = max = data[0];
+    for (i=0; i<size; i++) {
+        if (data[i] < min)
+            min = data[i];
+        if (data[i] > max)
+            max = data[i];
+        sum += data[i];
+    }
+    mean = sum / (double)size;
+
+    stddev = 0.;
+    for (i=0; i<size; i++) {
+        double diff = ((double)data[i]) - mean;
+        stddev += diff*diff;
+    }
+    stddev /= (double)size;
+    stddev = sqrt(stddev);
+
+    LOG(GLOBAL, LOG_ALL, 0, "\t#      = %9d\n", size);
+    LOG(GLOBAL, LOG_ALL, 0, "\tmin    = %9d\n", min);
+    LOG(GLOBAL, LOG_ALL, 0, "\tmax    = %9d\n", max);
+    double_print(mean, 1, &top, &bottom, &sign);
+    LOG(GLOBAL, LOG_ALL, 0, "\tmean   =   %s%7u.%.1u\n", sign, top, bottom);
+    double_print(stddev, 1, &top, &bottom, &sign);
+    LOG(GLOBAL, LOG_ALL, 0, "\tstddev =   %s%7u.%.1u\n", sign, top, bottom);
+
+    PRESERVE_FLOATING_POINT_STATE_END();
+}
+
+/* FIXME: these should be under ifdef STATS, not necessarily ifdef DEBUG */
+void
+stats_thread_init(dcontext_t *dcontext)
+{
+    thread_local_statistics_t *new_thread_stats;
+    if (!INTERNAL_OPTION(thread_stats))
+        return;                 /* dcontext->thread_stats stays NULL */
+
+    new_thread_stats =
+        HEAP_TYPE_ALLOC(dcontext, thread_local_statistics_t, ACCT_STATS, UNPROTECTED);
+    LOG(THREAD, LOG_STATS, 2, "thread_stats="PFX" size=%d\n", new_thread_stats, 
+        sizeof(thread_local_statistics_t));
+    /* initialize any thread stats bookkeeping fields before assigning to dcontext */
+    memset(new_thread_stats, 0x0, sizeof(thread_local_statistics_t));
+    new_thread_stats->thread_id = get_thread_id();
+    ASSIGN_INIT_LOCK_FREE(new_thread_stats->thread_stats_lock, thread_stats_lock);
+    dcontext->thread_stats = new_thread_stats;
+}
+
+void
+stats_thread_exit(dcontext_t *dcontext)
+{
+#ifdef DEBUG
+    /* for non-debug we do fast exit path and don't free local heap */
+    /* no clean up needed */
+    if (dcontext->thread_stats) {
+        thread_local_statistics_t *old_thread_stats = dcontext->thread_stats;
+        DELETE_LOCK(old_thread_stats->thread_stats_lock);
+        dcontext->thread_stats = NULL; /* disable thread stats before freeing memory */
+        HEAP_TYPE_FREE(dcontext, old_thread_stats, thread_local_statistics_t,
+                       ACCT_STATS, UNPROTECTED);
+    }
+#endif
+}
+
+void
+dump_thread_stats(dcontext_t *dcontext, bool raw)
+{
+    /* Note that this routine may be called by another thread, but
+       we want the LOGs and the STATs to be for the thread getting dumped
+       Make sure we use the passed dcontext everywhere here.
+       Avoid implicit use of get_thread_private_dcontext (e.g. THREAD_GET).
+    */
+    /* Each use of THREAD causes cl to make two implicit local variables
+     * (without optimizations on) (culprit is the ? : syntax).  Since the
+     * number of locals sums over scopes, this leads to stack usage
+     * of ~3kb for this function due to the many LOGs.  
+     * Instead we use THREAD once here and use a local below, cutting stack 
+     * usage to 12 bytes, ref bug 2203 */
+    file_t logfile = THREAD;
+    if (!THREAD_STATS_ON(dcontext))
+        return;
+
+    /* FIXME: for now we'll have code duplication with dump_global_stats()
+     * with the only difference being THREAD vs GLOBAL, e.g. LOG(GLOBAL and GLOBAL_STAT
+     * Keep in sync or make a template statistics dump macro for both cases. 
+     */
+    LOG(logfile, LOG_STATS, 1, "(Begin) Thread statistics @%d global, %d thread fragments ",
+        GLOBAL_STAT(num_fragments), THREAD_STAT(dcontext, num_fragments));
+    DOLOG(1, LOG_STATS, { print_timestamp(logfile); });
+    /* give up right away if thread stats lock already held, will dump next time
+       most likely thread interrupted while dumping state
+     */
+    if (!mutex_trylock(&dcontext->thread_stats->thread_stats_lock)) {
+        LOG(logfile, LOG_STATS, 1, " WARNING: skipped! Another dump in progress.\n");
+        return;
+    }
+    LOG(logfile, LOG_STATS, 1, ":\n");
+
+#define STATS_DEF(desc, stat) if (THREAD_STAT(dcontext, stat)) {           \
+        if (raw) {                                                         \
+            LOG(logfile, LOG_STATS, 1, "\t%s\t= "SSZFMT"\n",               \
+                #stat, THREAD_STAT(dcontext, stat));                       \
+        } else {                                                           \
+            LOG(logfile, LOG_STATS, 1, "%50s %s:"IF_X64_ELSE("%18","%9")   \
+                SSZFC"\n", desc, "(thread)", THREAD_STAT(dcontext, stat)); \
+        }                                                                  \
+    }
+# include "statsx.h"
+#undef STATS_DEF
+
+    LOG(logfile, LOG_STATS, 1, "(End) Thread statistics\n");
+    mutex_unlock(&dcontext->thread_stats->thread_stats_lock);
+    /* TODO: update all thread statistics, using the thread stats delta when implemented */
+
+#ifdef KSTATS
+    dump_thread_kstats(dcontext);
+#endif
+}
+
+void
+dump_global_stats(bool raw)
+{
+    DOLOG(1, LOG_MEMSTATS, {
+        if (!dynamo_exited_and_cleaned)
+            mem_stats_snapshot();
+    });
+    if (!dynamo_exited_and_cleaned)
+        print_vmm_heap_data(GLOBAL);
+    if (GLOBAL_STATS_ON()) {
+        LOG(GLOBAL, LOG_STATS, 1, "(Begin) All statistics @%d ", GLOBAL_STAT(num_fragments));
+        DOLOG(1, LOG_STATS, { print_timestamp(GLOBAL); });
+        LOG(GLOBAL, LOG_STATS, 1, ":\n");
+#define STATS_DEF(desc, stat) if (GLOBAL_STAT(stat)) {                                  \
+        if (raw) {                                                                      \
+            LOG(GLOBAL, LOG_STATS, 1, "\t%s\t= "SSZFMT"\n", #stat, GLOBAL_STAT(stat));  \
+        } else {                                                                        \
+            LOG(GLOBAL, LOG_STATS, 1, "%50s :"IF_X64_ELSE("%18","%9")SSZFC              \
+                "\n", desc, GLOBAL_STAT(stat));                                         \
+        }                                                                               \
+      }
+# include "statsx.h"
+#undef STATS_DEF
+        LOG(GLOBAL, LOG_STATS, 1, "(End) All statistics\n");
+    }
+#ifdef HEAP_ACCOUNTING
+    DOLOG(1, LOG_HEAP|LOG_STATS, {
+        print_heap_statistics();
+    });
+#endif
+    DOLOG(1, LOG_CACHE, {
+        /* shared cache stats */
+        fcache_stats_exit();
+    });
+#ifdef SHARING_STUDY
+    DOLOG(1, LOG_ALL, {
+        if (INTERNAL_OPTION(fragment_sharing_study) && !dynamo_exited)
+            print_shared_stats();
+    });
+#endif
+# ifdef DEADLOCK_AVOIDANCE
+    dump_process_locks();
+# endif
+}
+
+/* prints elapsed time since program startup to the given logfile */
+/* TODO: should also print absolute timestamp 
+   TODO: and relative time from thread start */
+void
+print_timestamp(file_t logfile)
+{
+    uint min, sec, msec;
+#ifdef LINUX
+    static struct timeval initial_time;
+    struct timeval current_time;
+
+    if (!initial_time.tv_sec) {
+        /* first time around we obtain initial time */
+        dynamorio_syscall(SYS_gettimeofday, 2, &initial_time, NULL);
+    }
+    dynamorio_syscall(SYS_gettimeofday, 2, &current_time, NULL);
+
+    current_time.tv_usec -= initial_time.tv_usec;
+    if (current_time.tv_usec < 0) {
+        current_time.tv_sec--;  /* borrow a second */
+        current_time.tv_usec += 1000*1000;
+    }
+    current_time.tv_sec -= initial_time.tv_sec;
+
+    sec = current_time.tv_sec;
+    msec = current_time.tv_usec / 1000; /* milliseconds are good enough */
+#else  /* !LINUX */
+    static uint initial_time = 0;   /* in milliseconds */
+    uint current_time;
+
+    if (!initial_time)
+        initial_time = query_time_millis();
+    current_time = query_time_millis() - initial_time; /* elapsed */
+    sec = current_time / 1000;
+    msec = current_time % 1000;
+#endif /* LINUX */
+    min = sec / 60;
+    sec = sec % 60;
+    print_file(logfile, "(%ld:%02ld.%03ld)", min, sec, msec);
+}
+
+#endif /* DEBUG */
+
+static void 
+dump_buffer_as_ascii(file_t logfile, char *buffer, size_t len)
+{
+    size_t i;
+    for (i = 0; i < len; i++) {
+        print_file(logfile, "%c", 
+                   isprint_fast(buffer[i]) ? buffer[i] : '.');
+    }
+}
+
+void
+dump_buffer_as_bytes (file_t logfile, void *buffer, size_t len, int flags)
+{
+    bool octal = flags & DUMP_OCTAL;
+    bool raw = flags & DUMP_RAW;
+    bool usechars = !raw && !(flags & DUMP_NO_CHARS);
+    bool replayable = usechars && !(flags & DUMP_NO_QUOTING);
+    bool dword = flags & DUMP_DWORD;
+    bool prepend_address = flags & DUMP_ADDRESS;
+    bool append_ascii = flags & DUMP_APPEND_ASCII;
+
+    unsigned char *buf = (unsigned char*) buffer;
+
+    int per_line = (flags & DUMP_PER_LINE) ? (flags & DUMP_PER_LINE) : DUMP_PER_LINE_DEFAULT;
+    size_t i;
+    int nonprint = 0;
+    size_t line_start = 0;
+
+    if (!raw) 
+        print_file(logfile, "%s", "\"");
+    
+    for (i=0; i + (dword ? 4 : 1) <= len; i += (dword ? 4 : 1)) {
+        if (i > 0 && 0 == i % per_line) {
+            if (append_ascii) {
+                print_file(logfile, "%s", " ");
+                /* append current line as ASCII */
+                ASSERT(line_start == (i - per_line));
+                dump_buffer_as_ascii(logfile, (char *)buf + line_start, per_line);
+                line_start = i;
+            }
+            /* new line */
+            print_file(logfile, "%s", raw ? "\n" : "\"\n\"");
+        }
+        if (prepend_address && 0 == i % per_line) // prepend address on new line
+            print_file(logfile, PFX" ", buf+i);
+
+        if (replayable) {
+            if (isdigit_fast(buf[i]) && nonprint) {
+                print_file(logfile, "%s", "\"\""); // to make \01 into \0""1
+            }         
+            if (buf[i] == '"') {
+                print_file(logfile, "%s", "\\\"");
+                continue;
+            }
+            if (buf[i] == '\\') 
+                print_file(logfile, "%s", "\\");
+        }
+
+        if (usechars && isprint_fast(buf[i])) {
+            print_file(logfile, "%c", buf[i]);
+            nonprint = 0;
+        } else {
+            if (!raw) {
+                print_file(logfile, "%s", octal ? "\\" : "\\x");
+            }
+            if (dword)
+                print_file(logfile, "%08x", *((uint*)(buf+i)));
+            else
+                print_file(logfile, octal ? "%03o" : "%02x", buf[i]);
+            nonprint = 1;
+            if (raw) {
+                print_file(logfile, "%s", " ");
+            }
+        }
+    }
+
+    if (append_ascii) {
+        /* append last line as ASCII */
+        /* pad to align columns */
+        size_t empty = ALIGN_FORWARD(buf + len, per_line);
+        uint size = (dword ? 4 : 1);
+        /* In general we expect dword requests to be dword aligned but
+         * we don't enforce it.  Note that we don't print DWORDs that
+         * may extend beyond valid len, but we'll print as ASCII any
+         * bytes included in valid len even if not printed in hex.
+         */
+        for (i = ALIGN_BACKWARD(buf + len, size); 
+             i < empty; i += size) {
+            if (dword) {
+                print_file(logfile, "%8c ", ' ');
+            } else {
+                print_file(logfile, octal ? "%3c " : "%2c ", ' ');
+            }
+        }
+
+        print_file(logfile, "%s", " ");
+        dump_buffer_as_ascii(logfile, (char *)buf + line_start, len - line_start);
+    }
+
+    if (!raw)
+        print_file(logfile, "%s", "\";\n");
+}
+
+/******************************************************************************/
+/* xml escaping routines, presumes iso-8859-1 encoding */
+
+bool
+is_valid_xml_char(char c)
+{
+    /* FIXME - wld.exe xml parsing complains about any C0 control character other
+     * then \t \r and \n.   However, in this encoding (to my understanding) all values
+     * should be valid and IE doesn't complain opening an xml file in this encoding
+     * with these characters.  Not sure where the wld.exe problem lies, but since it is
+     * our primary consumer we work around here. */
+    if ((uchar)c < 0x20 && c != '\t' && c != '\n' && c != '\r') {
+        return false;
+    }
+    return true;
+}
+
+static bool
+is_valid_xml_string(const char *str)
+{
+    while (*str != '\0') {
+        if (!is_valid_xml_char(*str))
+            return false;
+        str++;
+    }
+    return true;
+}
+
+/* NOTE - string should not include the <![CDATA[   ]]> markup as one thing
+ * this routine checks for is an inadvertant ending sequence ]]>. Caller is
+ * responsible for correct markup. */
+static bool
+is_valid_xml_cdata_string(const char *str)
+{
+    /* check for end CDATA tag */
+    /* FIXME - optimization,combine the two walks of the string into a 
+     * single walk.*/
+    return (strstr(str, "]]>") == NULL && is_valid_xml_string(str)) ;
+}
+
+#if 0 /* Not yet used */
+static bool
+is_valid_xml_body_string(const char *str)
+{
+    /* check for & < > */
+    /* FIXME - optimization, combine into a single walk of the string. */
+    return (strchr(str, '>') == NULL && strchr(str, '<') == NULL &&
+            strchr(str, '&') == NULL && is_valid_xml_string(str));
+}
+
+static bool
+is_valid_xml_attribute_string(const char *str)
+{
+    /* check for & < > ' " */
+    /* FIXME - optimization, combine into a single walk of the string. */
+    return (strchr(str, '\'') == NULL && strchr(str, '\"') == NULL &&
+            is_valid_xml_body_string(str));
+}
+#endif
+
+/* NOTE - string should not include the <![CDATA[   ]]> markup as one thing
+ * this routine checks for is an inadvertant ending sequence ]]> (in which
+ * case the first ] will be escaped). We escape using \%03d, note that since
+ * we don't escape \ , '\003' and "\003" will be indistinguishable (FIXME), 
+ * but given that these should really be normal ascii strings we'll live with
+ * that. */
+void
+print_xml_cdata(file_t f, const char *str)
+{
+    if (is_valid_xml_cdata_string(str)) {
+        print_file(f, "%s", str);
+    } else {
+        while (*str != '\0') {
+            if (!is_valid_xml_char(*str) ||
+                (*str == ']' && *(str+1) == ']' && *(str+2) == '>')) {
+                print_file(f, "\\%03d", (int)*(uchar *)str);
+            } else {
+                /* FIXME : could batch up printing normal chars for perf.
+                 * but we usually expect to have valid strings anyways. */
+                print_file(f, "%c", *str);
+            }
+            str++;
+        }
+    }
+}
+
+/* TODO - NYI print_xml_body_string, print_xml_attribute_string */
+
+#ifdef LINUX
+/* FIXME: this should be reworked, and the copy in preload should be removed */
+extern void 
+getnamefrompid(int pid, char *name, uint maxlen)
+{
+    int fd,n;
+    char tempstring[200+1], *lastpart;
+    /*this is a shitty way of getting the process name,
+    but i can't think of anything better... */
+
+    snprintf(tempstring, 200+1, "/proc/%d/cmdline", pid);
+    fd = open_syscall(tempstring, O_RDONLY, 0);
+    /* buffer overflow even if only off by 1 can be devastating */
+    n = read_syscall(fd, tempstring, 200);   
+    tempstring[n] = '\0';
+    lastpart = rindex(tempstring, '/');
+
+    if (lastpart == NULL)
+      lastpart = tempstring;
+    else
+      lastpart++; /*don't include last '/' in name*/ 
+
+    strncpy(name, lastpart, maxlen-1);
+    name[maxlen-1]  = '\0'; /* if max no null */
+
+    close_syscall(fd);
+}
+#endif
+
+void
+print_version_and_app_info(file_t file)
+{
+    print_file(file, "%s\n", dynamorio_version_string);
+    /* print qualified name (not stats->process_name) to get cmdline */
+    print_file(file, "Running: %s\n", get_application_name());
+#ifdef WINDOWS
+    /* FIXME: also get linux cmdline -- separate since wide on win32 */
+    print_file(file, "App cmdline: %S\n", get_application_cmdline());
+#endif
+    print_file(file, PRODUCT_NAME" built with: %s\n", DYNAMORIO_DEFINES);
+    print_file(file, PRODUCT_NAME" built on: %s\n", dynamorio_buildmark);
+#ifndef _WIN32_WCE
+    print_file(file, DYNAMORIO_VAR_OPTIONS": %s\n", option_string);
+#endif
+}
+
+#ifdef RETURN_STACK
+void
+print_return_stack(dcontext_t *dcontext)
+{
+    byte *rsp = dcontext->top_of_rstack;
+    if (rsp < dcontext->rstack) {
+        int pairs = ((byte *)dcontext->rstack - rsp) / 8;
+        LOG(THREAD, LOG_ALL, 0,
+            "Return stack contents (top="PFX", base="PFX"): %d pairs\n",
+            rsp, dcontext->rstack, pairs);
+        if (stats->loglevel > 2) {
+            while (rsp < dcontext->rstack) {
+                LOG(THREAD, LOG_ALL, 0, "\tapp: "PFX", us: "PFX"\n",
+                    *(((int *)rsp)+1), *((int *)rsp));
+                rsp += 8;
+            }
+        }
+    }
+}
+#endif
+
+void
+utils_exit()
+{
+    LOG(GLOBAL, LOG_STATS, 1, "-prng_seed "PFX" for reproducing random sequence\n",
+        initial_random_seed);
+
+    DELETE_LOCK(report_buf_lock);
+    DELETE_RECURSIVE_LOCK(logdir_mutex);
+    DELETE_LOCK(prng_lock);
+#ifdef DEADLOCK_AVOIDANCE
+    DELETE_LOCK(do_threshold_mutex);
+#endif
+}
+
+/* returns a pseudo random number in [0, max_offset) */
+
+/* FIXME: [minor security] while the first user may get more
+ *   randomness from the lower bits of the seed, I am not sure the
+ *   following users should strongly prefer higher or lower bits.
+ */
+size_t
+get_random_offset(size_t max_offset)
+{
+    /* These linear congruential constants taken from
+     * http://remus.rutgers.edu/~rhoads/Code/random.c 
+     * FIXME: Look up Knuth's recommendations in vol. 2
+     */
+    enum {
+        LCM_A = 279470273,
+        LCM_Q = 15,
+        LCM_R = 102913196
+    };
+    /* I prefer not risking any of the randomness in our seed to go
+     * through the fishy LCM and value generation - it doesn't buy us
+     * anything for the first instance which is all we currently use.
+     * Calculating value based on the previous seed also removes
+     * dependencies from the critical path and will be faster if we
+     * use this on a critical path..
+     */
+    size_t value;
+
+    /* Avoids div-by-zero if offset is 0; see case 8602 for implications. */
+    if (max_offset == 0)
+        return 0;
+
+    /* process-shared random sequence */
+    mutex_lock(&prng_lock);
+    /* FIXME: this is not getting the best randomness
+     * see srand() comments why taking higher order bits is usually better 
+     * j=1+(int) (10.0*rand()/(RAND_MAX+1.0));
+     * but I want to do it without floating point
+     */
+    value = random_seed % max_offset;
+
+    random_seed = LCM_A*(random_seed % LCM_Q) - LCM_R*(random_seed / LCM_Q);
+    mutex_unlock(&prng_lock);
+    LOG(GLOBAL, LOG_ALL, 2, "get_random_offset: value=%d (mod %d), new rs=%d\n", 
+        value, max_offset, random_seed);
+    return value;
+}
+
+const uint crctab[] = {
+    0x00000000, 0x77073096, 0xee0e612c, 0x990951ba,
+    0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3,
+    0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
+    0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91,
+    0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de,
+    0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
+    0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec,
+    0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5,
+    0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,
+    0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b,
+    0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940,
+    0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
+    0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116,
+    0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f,
+    0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
+    0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d,
+    0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a,
+    0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
+    0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818,
+    0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01,
+    0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
+    0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457,
+    0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c,
+    0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
+    0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2,
+    0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb,
+    0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,
+    0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9,
+    0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086,
+    0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
+    0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4,
+    0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad,
+    0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,
+    0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683,
+    0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8,
+    0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
+    0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe,
+    0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7,
+    0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,
+    0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5,
+    0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252,
+    0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
+    0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60,
+    0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79,
+    0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
+    0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f,
+    0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04,
+    0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
+    0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a,
+    0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713,
+    0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,
+    0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21,
+    0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e,
+    0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
+    0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c,
+    0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45,
+    0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,
+    0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db,
+    0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0,
+    0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
+    0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6,
+    0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf,
+    0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
+    0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d
+};
+
+/* This function implements the Ethernet AUTODIN II CRC32 algorithm.  */
+uint
+crc32(const char *buf, const uint len)
+{
+    uint i;
+    uint crc = 0xFFFFFFFF;
+
+    for (i = 0; i < len; i++)
+        crc = (crc >> 8) ^ crctab[(crc ^ buf[i]) & 0xFF];
+
+    return crc;
+}
+
+/* MD5 is used for persistent and process-shared caches, process_control, and
+ * ASLR persistent sharing.  The definition of MD5 below has a public
+ * license; source: http://stuff.mit.edu/afs/sipb/user/kenta/lj/clive/clive-0.4.5/
+ */
+/*----------------------------------------------------------------------------*/
+/* This code implements the MD5 message-digest algorithm.
+ * The algorithm is due to Ron Rivest. This code was
+ * written by Colin Plumb in 1993, no copyright is claimed.
+ * This code is in the public domain; do with it what you wish.
+ *
+ * Equivalent code is available from RSA Data Security, Inc.
+ * This code has been tested against that, and is equivalent,
+ * except that you don't need to include two pages of legalese
+ * with every copy.
+ *
+ * To compute the message digest of a chunk of bytes, declare an
+ * MD5Context structure, pass it to MD5Init, call MD5Update as
+ * needed on buffers full of bytes, and then call MD5Final, which
+ * will fill a supplied 16-byte array with the digest.
+ */
+static void
+MD5Transform(uint32 state[4], const unsigned char block[MD5_BLOCK_LENGTH]);
+
+#define PUT_64BIT_LE(cp, value) do {                                \
+    (cp)[7] = (unsigned char)((value) >> 56);                       \
+    (cp)[6] = (unsigned char)((value) >> 48);                       \
+    (cp)[5] = (unsigned char)((value) >> 40);                       \
+    (cp)[4] = (unsigned char)((value) >> 32);                       \
+    (cp)[3] = (unsigned char)((value) >> 24);                       \
+    (cp)[2] = (unsigned char)((value) >> 16);                       \
+    (cp)[1] = (unsigned char)((value) >> 8);                        \
+    (cp)[0] = (unsigned char)(value); } while (0)
+
+#define PUT_32BIT_LE(cp, value) do {                                \
+    (cp)[3] = (unsigned char)((value) >> 24);                       \
+    (cp)[2] = (unsigned char)((value) >> 16);                       \
+    (cp)[1] = (unsigned char)((value) >> 8);                        \
+    (cp)[0] = (unsigned char)(value); } while (0)
+
+static unsigned char PADDING[MD5_BLOCK_LENGTH] = {
+    0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+};
+
+/*
+ * Start MD5 accumulation.  Set bit count to 0 and buffer to mysterious
+ * initialization constants.
+ */
+void
+MD5Init(struct MD5Context *ctx)
+{
+    ctx->count = 0;
+    ctx->state[0] = 0x67452301;
+    ctx->state[1] = 0xefcdab89;
+    ctx->state[2] = 0x98badcfe;
+    ctx->state[3] = 0x10325476;
+}
+
+/*
+ * Update context to reflect the concatenation of another buffer full
+ * of bytes.
+ */
+void
+MD5Update(struct MD5Context *ctx, const unsigned char *input, size_t len)
+{
+    size_t have, need;
+
+    /* Check how many bytes we already have and how many more we need. */
+    have = (size_t)((ctx->count >> 3) & (MD5_BLOCK_LENGTH - 1));
+    need = MD5_BLOCK_LENGTH - have;
+
+    /* Update bitcount */
+    ctx->count += (uint64)len << 3;
+
+    if (len >= need) {
+        if (have != 0) {
+            memcpy(ctx->buffer + have, input, need);
+            MD5Transform(ctx->state, ctx->buffer);
+            input += need;
+            len -= need;
+            have = 0;
+        }
+
+        /* Process data in MD5_BLOCK_LENGTH-byte chunks. */
+        while (len >= MD5_BLOCK_LENGTH) {
+            MD5Transform(ctx->state, input);
+            input += MD5_BLOCK_LENGTH;
+            len -= MD5_BLOCK_LENGTH;
+        }
+    }
+
+    /* Handle any remaining bytes of data. */
+    if (len != 0)
+        memcpy(ctx->buffer + have, input, len);
+}
+
+/*
+ * Pad pad to 64-byte boundary with the bit pattern
+ * 1 0* (64-bit count of bits processed, MSB-first)
+ */
+static void
+MD5Pad(struct MD5Context *ctx)
+{
+    unsigned char count[8];
+    size_t padlen;
+
+    /* Convert count to 8 bytes in little endian order. */
+    PUT_64BIT_LE(count, ctx->count);
+
+    /* Pad out to 56 mod 64. */
+    padlen = (size_t)
+        (MD5_BLOCK_LENGTH - ((ctx->count >> 3) & (MD5_BLOCK_LENGTH - 1)));
+    if (padlen < 1 + 8)
+        padlen += MD5_BLOCK_LENGTH;
+    MD5Update(ctx, PADDING, padlen - 8);            /* padlen - 8 <= 64 */
+    MD5Update(ctx, count, 8);
+}
+
+/*
+ * Final wrapup--call MD5Pad, fill in digest and zero out ctx.
+ */
+void
+MD5Final(unsigned char digest[MD5_RAW_BYTES], struct MD5Context *ctx)
+{
+    int i;
+
+    MD5Pad(ctx);
+    if (digest != NULL) {
+        for (i = 0; i < 4; i++)
+            PUT_32BIT_LE(digest + i * 4, ctx->state[i]);
+    }
+    memset(ctx, 0, sizeof(*ctx));   /* in case it's sensitive */
+}
+
+
+/* The four core functions - F1 is optimized somewhat */
+
+/* #define F1(x, y, z) (x & y | ~x & z) */
+#define F1(x, y, z) (z ^ (x & (y ^ z)))
+#define F2(x, y, z) F1(z, x, y)
+#define F3(x, y, z) (x ^ y ^ z)
+#define F4(x, y, z) (y ^ (x | ~z))
+
+/* This is the central step in the MD5 algorithm. */
+#define MD5STEP(f, w, x, y, z, data, s)                         \
+    ( w += f(x, y, z) + data,  w = w<<s | w>>(32-s),  w += x )
+
+/*
+ * The core of the MD5 algorithm, this alters an existing MD5 hash to
+ * reflect the addition of 16 longwords of new data.  MD5Update blocks
+ * the data and converts bytes into longwords for this routine.
+ */
+static void
+MD5Transform(uint32 state[4], const unsigned char block[MD5_BLOCK_LENGTH])
+{
+    uint32 a, b, c, d, in[MD5_BLOCK_LENGTH / 4];
+
+#if BYTE_ORDER == LITTLE_ENDIAN
+    memcpy(in, block, sizeof(in));
+#else
+    for (a = 0; a < MD5_BLOCK_LENGTH / 4; a++) {
+        in[a] = (uint32)
+            ((uint32)(block[a * 4 + 0]) |
+             (uint32)(block[a * 4 + 1]) <<  8 |
+             (uint32)(block[a * 4 + 2]) << 16 |
+             (uint32)(block[a * 4 + 3]) << 24);
+    }
+#endif
+
+    a = state[0];
+    b = state[1];
+    c = state[2];
+    d = state[3];
+
+    MD5STEP(F1, a, b, c, d, in[ 0] + 0xd76aa478,  7);
+    MD5STEP(F1, d, a, b, c, in[ 1] + 0xe8c7b756, 12);
+    MD5STEP(F1, c, d, a, b, in[ 2] + 0x242070db, 17);
+    MD5STEP(F1, b, c, d, a, in[ 3] + 0xc1bdceee, 22);
+    MD5STEP(F1, a, b, c, d, in[ 4] + 0xf57c0faf,  7);
+    MD5STEP(F1, d, a, b, c, in[ 5] + 0x4787c62a, 12);
+    MD5STEP(F1, c, d, a, b, in[ 6] + 0xa8304613, 17);
+    MD5STEP(F1, b, c, d, a, in[ 7] + 0xfd469501, 22);
+    MD5STEP(F1, a, b, c, d, in[ 8] + 0x698098d8,  7);
+    MD5STEP(F1, d, a, b, c, in[ 9] + 0x8b44f7af, 12);
+    MD5STEP(F1, c, d, a, b, in[10] + 0xffff5bb1, 17);
+    MD5STEP(F1, b, c, d, a, in[11] + 0x895cd7be, 22);
+    MD5STEP(F1, a, b, c, d, in[12] + 0x6b901122,  7);
+    MD5STEP(F1, d, a, b, c, in[13] + 0xfd987193, 12);
+    MD5STEP(F1, c, d, a, b, in[14] + 0xa679438e, 17);
+    MD5STEP(F1, b, c, d, a, in[15] + 0x49b40821, 22);
+
+    MD5STEP(F2, a, b, c, d, in[ 1] + 0xf61e2562,  5);
+    MD5STEP(F2, d, a, b, c, in[ 6] + 0xc040b340,  9);
+    MD5STEP(F2, c, d, a, b, in[11] + 0x265e5a51, 14);
+    MD5STEP(F2, b, c, d, a, in[ 0] + 0xe9b6c7aa, 20);
+    MD5STEP(F2, a, b, c, d, in[ 5] + 0xd62f105d,  5);
+    MD5STEP(F2, d, a, b, c, in[10] + 0x02441453,  9);
+    MD5STEP(F2, c, d, a, b, in[15] + 0xd8a1e681, 14);
+    MD5STEP(F2, b, c, d, a, in[ 4] + 0xe7d3fbc8, 20);
+    MD5STEP(F2, a, b, c, d, in[ 9] + 0x21e1cde6,  5);
+    MD5STEP(F2, d, a, b, c, in[14] + 0xc33707d6,  9);
+    MD5STEP(F2, c, d, a, b, in[ 3] + 0xf4d50d87, 14);
+    MD5STEP(F2, b, c, d, a, in[ 8] + 0x455a14ed, 20);
+    MD5STEP(F2, a, b, c, d, in[13] + 0xa9e3e905,  5);
+    MD5STEP(F2, d, a, b, c, in[ 2] + 0xfcefa3f8,  9);
+    MD5STEP(F2, c, d, a, b, in[ 7] + 0x676f02d9, 14);
+    MD5STEP(F2, b, c, d, a, in[12] + 0x8d2a4c8a, 20);
+
+    MD5STEP(F3, a, b, c, d, in[ 5] + 0xfffa3942,  4);
+    MD5STEP(F3, d, a, b, c, in[ 8] + 0x8771f681, 11);
+    MD5STEP(F3, c, d, a, b, in[11] + 0x6d9d6122, 16);
+    MD5STEP(F3, b, c, d, a, in[14] + 0xfde5380c, 23);
+    MD5STEP(F3, a, b, c, d, in[ 1] + 0xa4beea44,  4);
+    MD5STEP(F3, d, a, b, c, in[ 4] + 0x4bdecfa9, 11);
+    MD5STEP(F3, c, d, a, b, in[ 7] + 0xf6bb4b60, 16);
+    MD5STEP(F3, b, c, d, a, in[10] + 0xbebfbc70, 23);
+    MD5STEP(F3, a, b, c, d, in[13] + 0x289b7ec6,  4);
+    MD5STEP(F3, d, a, b, c, in[ 0] + 0xeaa127fa, 11);
+    MD5STEP(F3, c, d, a, b, in[ 3] + 0xd4ef3085, 16);
+    MD5STEP(F3, b, c, d, a, in[ 6] + 0x04881d05, 23);
+    MD5STEP(F3, a, b, c, d, in[ 9] + 0xd9d4d039,  4);
+    MD5STEP(F3, d, a, b, c, in[12] + 0xe6db99e5, 11);
+    MD5STEP(F3, c, d, a, b, in[15] + 0x1fa27cf8, 16);
+    MD5STEP(F3, b, c, d, a, in[2 ] + 0xc4ac5665, 23);
+
+    MD5STEP(F4, a, b, c, d, in[ 0] + 0xf4292244,  6);
+    MD5STEP(F4, d, a, b, c, in[7 ] + 0x432aff97, 10);
+    MD5STEP(F4, c, d, a, b, in[14] + 0xab9423a7, 15);
+    MD5STEP(F4, b, c, d, a, in[5 ] + 0xfc93a039, 21);
+    MD5STEP(F4, a, b, c, d, in[12] + 0x655b59c3,  6);
+    MD5STEP(F4, d, a, b, c, in[3 ] + 0x8f0ccc92, 10);
+    MD5STEP(F4, c, d, a, b, in[10] + 0xffeff47d, 15);
+    MD5STEP(F4, b, c, d, a, in[1 ] + 0x85845dd1, 21);
+    MD5STEP(F4, a, b, c, d, in[8 ] + 0x6fa87e4f,  6);
+    MD5STEP(F4, d, a, b, c, in[15] + 0xfe2ce6e0, 10);
+    MD5STEP(F4, c, d, a, b, in[6 ] + 0xa3014314, 15);
+    MD5STEP(F4, b, c, d, a, in[13] + 0x4e0811a1, 21);
+    MD5STEP(F4, a, b, c, d, in[4 ] + 0xf7537e82,  6);
+    MD5STEP(F4, d, a, b, c, in[11] + 0xbd3af235, 10);
+    MD5STEP(F4, c, d, a, b, in[2 ] + 0x2ad7d2bb, 15);
+    MD5STEP(F4, b, c, d, a, in[9 ] + 0xeb86d391, 21);
+
+    state[0] += a;
+    state[1] += b;
+    state[2] += c;
+    state[3] += d;
+}
+#undef F1
+#undef F2
+#undef F3
+#undef F4
+#undef MD5STEP
+/*----------------------------------------------------------------------------*/
+
+bool
+module_digests_equal(const module_digest_t *calculated_digest, 
+                     const module_digest_t *matching_digest,
+                     bool check_short, bool check_full)
+{
+    bool match = true;
+    if (check_short) {
+        match = match && md5_digests_equal(calculated_digest->short_MD5,
+                                           matching_digest->short_MD5);
+    }
+    if (check_full) {
+        match = match && md5_digests_equal(calculated_digest->full_MD5,
+                                           matching_digest->full_MD5);
+    }
+    return match;
+}
+
+/* Reads the full file, returns it in a buffer and sets buf_len to the size
+ * of the buffer allocated on the specified heap, if successful.  Returns NULL
+ * and sets buf_len to 0 on failure.  Defined when fixing case 8187.
+ */
+char *
+read_entire_file(const char *file, size_t *buf_len /* OUT */ HEAPACCT(which_heap_t heap))
+{
+    ssize_t bytes_read;
+    file_t fd = INVALID_FILE;
+    char *buf = NULL;
+    uint64 buf_len64 = 0;
+
+    /* No point in reading the file if the length can't be returned - the caller
+     * won't be able to free the buffer without the size.
+     */
+    if (file == NULL || buf_len == NULL)
+        return NULL;
+
+    *buf_len = 0;
+
+    fd = os_open((char *)file, OS_OPEN_READ);
+    if (fd == INVALID_FILE )
+        return NULL;
+   
+    if (!os_get_file_size(file, &buf_len64)) {
+        os_close(fd);
+        return NULL;
+    }
+    ASSERT_TRUNCATE(*buf_len, uint, buf_len64);
+
+    /* Though only 1 byte is needed for the \0 at the end of the buffer,
+     * 4 may be allocated to work around case 8048.  FIXME: remove 
+     * alignment after case is resolved.
+     */
+    *buf_len = (uint) ALIGN_FORWARD((buf_len64 + 1), 4);
+    buf = (char *) heap_alloc(GLOBAL_DCONTEXT, *buf_len HEAPACCT(heap));
+    bytes_read = os_read(fd, buf, *buf_len);
+    if (bytes_read <= 0) {
+        heap_free(GLOBAL_DCONTEXT, buf, *buf_len HEAPACCT(heap));
+        os_close(fd);
+        return NULL;
+    }
+    ASSERT(CHECK_TRUNCATE_TYPE_uint(bytes_read));
+    ASSERT((uint)bytes_read != *buf_len && "buffer too small");
+    ASSERT((uint)bytes_read < *buf_len); /* use MIN below just to be safe */
+    buf[MIN((uint)bytes_read, *buf_len - 1)] = 0;
+
+    os_close(fd);
+    return buf;
+}
+
+/* returns false if we are too low on disk to create a file of desired size */
+bool
+check_low_disk_threshold(file_t f, uint64 new_file_size)
+{
+    /* FIXME: we only use UserAvailable to find the minimum expressed
+     * as absolute bytes to leave available.  In addition we could
+     * also have percentage limits, where minimum available should be
+     * based on TotalQuotaBytes, and maximum cache size on
+     * TotalVolumeBytes.
+     */
+    uint64 user_available_bytes;
+    /* FIXME: does this work for compressed volumes? */
+    bool ok = os_get_disk_free_space(f, &user_available_bytes, NULL, NULL);
+    if (ok) {
+        LOG(THREAD_GET, LOG_SYSCALLS|LOG_THREADS, 2,
+            "available disk space quota %dMB\n",
+            user_available_bytes/1024/1024);
+
+        /* note that the actual needed bytes are in fact aligned to a
+         * cluster, so this is somewhat imprecise */
+        ok = (user_available_bytes > new_file_size) &&
+            (user_available_bytes - new_file_size) > DYNAMO_OPTION(min_free_disk);
+        if (!ok) {
+            /* FIXME: notify the customer that they are low on disk space? */
+            SYSLOG_INTERNAL_WARNING_ONCE("reached minimal free disk space limit,"
+                                         " available "UINT64_FORMAT_STRING"MB, limit %dMB, "
+                                         "asking for "UINT64_FORMAT_STRING"KB",
+                                         user_available_bytes/1024/1024,
+                                         DYNAMO_OPTION(min_free_disk)/1024/1024,
+                                         new_file_size/1024);
+            /* ONCE, even though later we may succeed and start failing again */
+        }
+    } else {
+        /* impersonated thread may not have rights to even query */
+        /* or we have an invalid path */
+        /* do nothing */
+    }
+    return ok;
+}
+
+#ifdef PROCESS_CONTROL  /* currently used for only for this; need not be so */
+/* Note: Reading the full file in one shot will cause committed memory and
+ * wss to shoot up.  Even if this memory is freed, only wss will come down.
+ * While this may be ok for fcache mode, it is probably not for hotp_only mode,
+ * and definitely not for thin_client mode.  The main use of thin_client today
+ * is process_control, which means that MD5 will be computed in thin_client
+ * mode. 
+ *
+ * The options for this memory issue are to mmap the memory and unmap it after
+ * use rather than use the heap or to use a moderate sized buffer and read the
+ * file in chunks, which is wat I have chosen.  If startup performance becomes
+ * a problem because of this play with bigger pages or just use mmap and munmap.
+ *
+ * Measurements on my laptop on a total of 5076 executables showed,
+ *  Average size        : 233 kb
+ *  Standard deviation  : 684 kb
+ *  Median size         : 68 kb!
+ *  80% percentile      : 205 kb
+ *  90% percentile      : 469 kb
+ * So a buffer of 16 kb seems reasonable, even though the data is not based
+ * on frequency of usage.
+ *
+ * FIXME: use nt_map_view_of_section with SEC_MAPPED && !SEC_IMAGE.
+ */
+#define MD5_FILE_READ_BUF_SIZE  (4 * PAGE_SIZE) 
+
+/* Reads 'file', computes MD5 hash for it, which is returned in hash_buf
+ * when successful and true is returned.  On failure false is returned and 
+ * hash_buf contents invalid.
+ * Note: Reads in file in 16k chunks to avoid private/committed memory
+ * increase.
+ */
+bool
+get_md5_for_file(const char *file, char *hash_buf /* OUT */)
+{
+    ssize_t bytes_read;
+    int i;
+    file_t fd;
+    char *file_buf;
+    struct MD5Context md5_cxt;
+    unsigned char md5_buf[MD5_STRING_LENGTH/2];
+
+    if (file == NULL || hash_buf == NULL)
+        return false;
+
+    fd = os_open((char *)file, OS_OPEN_READ);
+    if (fd == INVALID_FILE)
+        return false;
+    
+    MD5Init(&md5_cxt);
+    file_buf = (char *) heap_alloc(GLOBAL_DCONTEXT, MD5_FILE_READ_BUF_SIZE
+                                   HEAPACCT(ACCT_OTHER));
+    while ((bytes_read = os_read(fd, file_buf, MD5_FILE_READ_BUF_SIZE)) > 0) {
+        ASSERT(CHECK_TRUNCATE_TYPE_uint(bytes_read));
+        MD5Update(&md5_cxt, (byte *) file_buf, (uint)bytes_read);
+    }
+    MD5Final(md5_buf, &md5_cxt);
+
+    /* Convert 16-byte signature into 32-byte string, which is how MD5 is
+     * usually printed/used. n is 3, 2 for chars & 1 for the '\0';
+     */
+    for (i = 0; i < BUFFER_SIZE_ELEMENTS(md5_buf); i++)
+        snprintf(hash_buf + (i * 2), 3, "%02X", md5_buf[i]);
+
+    /* Just be safe & terminate the buffer; assuming it has 33 chars! */
+    hash_buf[MD5_STRING_LENGTH] = '\0';
+
+    heap_free(GLOBAL_DCONTEXT, file_buf, MD5_FILE_READ_BUF_SIZE HEAPACCT(ACCT_OTHER));
+    os_close(fd);
+    return true; 
+}
+#endif /* PROCESS_CONTROL */
+
+/* Computes and caches MD5 for the application if it isn't there; returns it. */
+/* Note: this function isn't under PROCESS_CONTROL even though that is what
+ *       uses this because md5 for the executable may be needed in future.
+ *       Also, this avoids having to define 2 types of process start events.
+ *       Further, if we decide to remove process control in future leaving the 
+ *       md5 in the start event will prevent needless backward compatibility 
+ *       problems caused by removing md5 from it.
+ */
+const char *
+get_application_md5(void)
+{
+    /* Though exe_md5 is used in the process start event, it is currently set
+     * only by process_control.  BTW, 1 for the terminating '\0'.
+     */
+    static char exe_md5[MD5_STRING_LENGTH + 1] = {0};
+
+#ifdef PROCESS_CONTROL
+    /* MD5 is computed only if process control is turned on, otherwise the cost
+     * isn't paid (roughly 10ms for a 350kb exe), i.e. "" is returned.
+     */
+    if (exe_md5[0] == '\0') {
+        if (IS_PROCESS_CONTROL_ON()) {
+            DEBUG_DECLARE(bool res;)
+# ifdef WINDOWS
+            /* FIXME - inefficient, we use stack buffer here to convert wchar to char,
+             * and later we'll use another stack buffer to convert char to wchar to
+             * open the file.  That said this is only done once (either at startup or
+             * for process_control nudge [app_stack]) so as long as the stack doesn't
+             * overflow it doesn't really matter. */ 
+            char exe_name[MAXIMUM_PATH];
+            snprintf(exe_name, BUFFER_SIZE_ELEMENTS(exe_name), "%ls",
+                     get_own_unqualified_name());
+            NULL_TERMINATE_BUFFER(exe_name);
+# else
+            /* FIXME - will need to strip out qualifications if we add those to Linux */
+            const char *exe_name = get_application_name();
+# endif
+
+            /* Change protection to make .data writable; this is a nop at init
+             * time - which is the most common case.  For the nudge case we
+             * will pay full price of data protection if MD5 hasn't been
+             * computed yet; this is rare, so it is ok.
+             */
+            SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+            DEBUG_DECLARE(res = ) get_md5_for_file(exe_name, exe_md5);
+            ASSERT(res && strlen(exe_md5) == MD5_STRING_LENGTH);
+            NULL_TERMINATE_BUFFER(exe_md5);             /* just be safe */
+            SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);  /* restore protection */
+        }
+    } else {
+        /* If it isn't null then it must be a full MD5 and process control
+         * should be turned on.
+         */
+        ASSERT(strlen(exe_md5) == MD5_STRING_LENGTH);
+        ASSERT(IS_PROCESS_CONTROL_ON());
+    }
+#else
+    /* If there is no process control, for now, app MD5 isn't computed. */
+    ASSERT(exe_md5[0] == '\0');
+#endif
+    return exe_md5;
+}
+
+/* Producing a single MD5 digest for a given readable memory region.
+ *
+ * An empty region is not expected, though legal, since produces a
+ * constant value.
+ */
+void
+get_md5_for_region(const byte *region_start, uint len, 
+                   unsigned char digest[MD5_RAW_BYTES] /* OUT */)
+{
+    struct MD5Context md5_cxt;
+    MD5Init(&md5_cxt);
+    ASSERT(region_start != NULL);
+    ASSERT_CURIOSITY(len != 0);
+
+    if (region_start != NULL && len != 0) 
+        MD5Update(&md5_cxt, region_start, len);
+    MD5Final(digest, &md5_cxt);
+    ASSERT_NOT_TESTED();
+}
+
+bool
+md5_digests_equal(const byte digest1[MD5_RAW_BYTES], const byte digest2[MD5_RAW_BYTES])
+{
+    return (memcmp(digest1, digest2, MD5_RAW_BYTES) == 0);
+}
+
+/* calculates intersection of two regions defined as open ended intervals
+ * [region1_start, region1_start + region1_len) \intersect
+ * [region2_start, region2_start + region2_len)
+ * 
+ * intersection_len is set to 0 if the regions do not overlap
+ * otherwise returns the intersecting region 
+ * [intersection_start, intersection_start + intersection_len)
+ */
+void
+region_intersection(app_pc *intersection_start /* OUT */,
+                    size_t *intersection_len /* OUT */,
+                    const app_pc region1_start, size_t region1_len,
+                    const app_pc region2_start, size_t region2_len)
+{
+    /* intersection */
+    app_pc intersection_end = MIN(region1_start + region1_len, 
+                                  region2_start + region2_len);
+    ASSERT(intersection_start != NULL);
+    ASSERT(intersection_len != NULL);
+    *intersection_start = MAX(region1_start, region2_start);
+
+    /* set length as long as result is a proper intersecting region,
+     * max(0, intersection_end - intersection_start) if signed
+     */
+    *intersection_len = 
+        (intersection_end > *intersection_start) ?
+        (intersection_end - *intersection_start) : 0;
+}
+/***************************************************************************/
+#ifdef CALL_PROFILE
+
+typedef struct _profile_callers_t {
+    app_pc caller[MAX_CALL_PROFILE_DEPTH];
+    uint count;
+    struct _profile_callers_t *next;
+} profile_callers_t;
+
+/* debug-only so neverprot */
+DECLARE_NEVERPROT_VAR(static profile_callers_t *profcalls, NULL);
+DECLARE_CXTSWPROT_VAR(static mutex_t profile_callers_lock,
+                      INIT_LOCK_FREE(profile_callers_lock));
+
+/* Usage:
+ * Simply place a profile_callers() call in the routine you wish to profile.
+ * You MUST build without optimizations to enable call stack walking.
+ * Results go to a separate log file and are dumped only at exit.
+ * FIXME: combine w/ a generalized mutex_collect_callstack()?
+ */
+void
+profile_callers()
+{
+    profile_callers_t *entry;
+    uint *pc;
+    uint num = 0;
+    app_pc our_ebp = 0;
+    app_pc caller[MAX_CALL_PROFILE_DEPTH];
+    app_pc saferead[2];
+    if (DYNAMO_OPTION(prof_caller) == 0 || dynamo_exited_and_cleaned/*no heap*/)
+        return;
+    ASSERT(DYNAMO_OPTION(prof_caller) <= MAX_CALL_PROFILE_DEPTH);
+    GET_FRAME_PTR(our_ebp);
+    memset(caller, 0, sizeof(caller));
+    pc = (uint *) our_ebp;
+    /* FIXME: mutex_collect_callstack() assumes caller addresses are in
+     * DR and thus are safe to read, but checks for dstack, etc.
+     * Should combine the two into a general routine.
+     */
+    while (pc != NULL && safe_read((byte *)pc, sizeof(saferead), saferead)) {
+        caller[num] = saferead[1];
+        num++;
+        /* yes I've seen weird recursive cases before */
+        if (pc == (uint *) saferead[0] || num >= DYNAMO_OPTION(prof_caller))
+            break;
+        pc = (uint *) saferead[0];
+    }
+    /* Assumption: there aren't many unique callstacks being profiled, so a
+     * linear search is sufficient!
+     * FIXME: make this more performant if necessary
+     */
+    for (entry = profcalls; entry != NULL; entry = entry->next) {
+        bool match = true;
+        for (num = 0; num < DYNAMO_OPTION(prof_caller); num++) {
+            if (entry->caller[num] != caller[num]) {
+                match = false;
+                break;
+            }
+        }
+        if (match) {
+            entry->count++;
+            break;
+        }
+    }
+    if (entry == NULL) {
+        entry = global_heap_alloc(sizeof(profile_callers_t) HEAPACCT(ACCT_OTHER));
+        memcpy(entry->caller, caller, sizeof(caller));
+        entry->count = 1;
+        mutex_lock(&profile_callers_lock);
+        entry->next = profcalls;
+        profcalls = entry;
+        mutex_unlock(&profile_callers_lock);
+    }
+}
+
+void
+profile_callers_exit()
+{
+    profile_callers_t *entry, *next;
+    file_t file;
+    if (DYNAMO_OPTION(prof_caller) > 0) {
+        mutex_lock(&profile_callers_lock);
+        file = open_log_file("callprof", NULL, 0);
+        for (entry = profcalls; entry != NULL; entry = next) {
+            uint num;
+            next = entry->next;
+            for (num = 0; num < DYNAMO_OPTION(prof_caller); num++) {
+                print_file(file, PFX" ", entry->caller[num]);
+            }
+            print_file(file, "%d\n", entry->count);
+            global_heap_free(entry, sizeof(profile_callers_t) HEAPACCT(ACCT_OTHER));
+        }
+        os_close(file);
+        profcalls = NULL;
+        mutex_unlock(&profile_callers_lock);
+    }
+    DELETE_LOCK(profile_callers_lock);
+}
+
+#endif /* CALL_PROFILE */
+
+
+#ifdef UTILS_UNIT_TEST
+
+# ifdef printf
+#  undef printf
+# endif
+# define printf(...) print_file(STDERR, __VA_ARGS__)
+
+/* some tests for double_print() and divide_uint64_print() */
+int main()
+{
+    char buf[128];
+    uint c, d;
+    char *s;
+
+# define DO_TEST(a, b, p, percent, fmt, result)                           \
+    divide_uint64_print(a, b, percent, p, &c, &d);                        \
+    snprintf(buf, BUFFER_SIZE_ELEMENTS(buf), fmt, c, d);                  \
+    NULL_TERMINATE_BUFFER(buf);                                           \
+    if (strcmp(buf, result) == 0)                                         \
+        printf("PASS\n");                                                 \
+    else                                                                  \
+        printf("FAIL : \"%s\" doesn't match \"%s\"\n", buf, result);      \
+
+    DO_TEST(1, 20, 3, false, "%u.%.3u", "0.050");
+    DO_TEST(2, 5, 2, false, "%3u.%.2u", "  0.40");
+    DO_TEST(100, 7, 4, false, "%u.%.4u", "14.2857");
+    DO_TEST(475, 1000, 2, true, "%u.%.2u%%", "47.50%");
+
+# undef DO_TEST
+# define DO_TEST(a, p, fmt, result)                                       \
+    double_print(a, p, &c, &d, &s);                                       \
+    snprintf(buf, BUFFER_SIZE_ELEMENTS(buf), fmt, s, c, d);               \
+    NULL_TERMINATE_BUFFER(buf);                                           \
+    if (strcmp(buf, result) == 0)                                         \
+        printf("PASS\n");                                                 \
+    else                                                                  \
+        printf("FAIL : \"%s\" doesn't match \"%s\"\n", buf, result);      \
+
+    DO_TEST(-2.06, 3, "%s%u.%.3u", "-2.060");
+    DO_TEST(2.06, 4, "%s%u.%.4u", "2.0600");
+    DO_TEST(.0563, 2, "%s%u.%.2u", "0.05");
+    DO_TEST(-.0563, 2, "%s%u.%.2u", "-0.05");
+    DO_TEST(23.0456, 5, "%s%4u.%.5u", "  23.04560");
+    DO_TEST(-23.0456, 5, "%s%4u.%.5u", "-  23.04560");
+
+# undef DO_TEST
+}
+
+# undef printf
+
+#endif /* UTILS_UNIT_TEST */
+
+
+char *
+dr_strdup(const char *str HEAPACCT(which_heap_t which))
+{
+    char *dup;
+    size_t str_len;
+
+    if (str == NULL)
+        return NULL;
+
+    str_len = strlen(str) + 1;      /* Extra 1 char for the '\0' at the end. */
+    dup = (char*) heap_alloc(GLOBAL_DCONTEXT, str_len HEAPACCT(which));
+    strncpy (dup, str, str_len);
+    dup[str_len - 1] = '\0';        /* Being on the safe side. */
+    return dup;
+}
+
+#ifdef WINDOWS
+/* Allocates a new char *(NOT a new wchar_t*) from a wchar_t* */
+char *
+dr_wstrdup(const wchar_t *str HEAPACCT(which_heap_t which))
+{
+    char *dup;
+    size_t str_len;
+    if (str == NULL)
+        return NULL;
+    /* FIXME: should have a max length and truncate? 
+     * I'm assuming we're using not directly on external inputs.
+     * If we do put in a max length, should do the same for dr_strdup.
+     */
+    str_len = wcslen(str) + 1;      /* Extra 1 char for the '\0' at the end. */
+    dup = (char*) heap_alloc(GLOBAL_DCONTEXT, str_len HEAPACCT(which));
+    snprintf(dup, str_len, "%S", str);
+    dup[str_len - 1] = '\0';        /* Being on the safe side. */
+    return dup;
+}
+#endif
+
+/* Frees a char *string (NOT a wchar_t*) allocated via dr_strdup or
+ * dr_wstrdup that has not been modified since being copied!
+ */
+void
+dr_strfree(const char *str HEAPACCT(which_heap_t which))
+{
+    size_t str_len;
+    ASSERT_CURIOSITY(str != NULL);
+    if (str == NULL)
+        return;
+    str_len = strlen(str) + 1;      /* Extra 1 char for the '\0' at the end. */
+    heap_free(GLOBAL_DCONTEXT, (void *)str, str_len HEAPACCT(which));
+}
+
+/* Merges two unsorted arrays, treating their elements as type void*
+ * (but will work on other types of same size as void*)
+ * as an intersection if intersect is true or as a union (removing
+ * duplicates) otherwise.  Allocates a new array on dcontext's heap
+ * for the result and returns it and its size; if the new size is 0,
+ * does NOT allocate anything and points the new array at NULL.
+ */
+void
+array_merge(dcontext_t *dcontext, bool intersect /* else union */,
+            void **src1, uint src1_num, void **src2, uint src2_num,
+            /*OUT*/ void ***dst, /*OUT*/ uint *dst_num
+            HEAPACCT(which_heap_t which))
+{
+    /* Two passes: one to find number of unique entries, and second to
+     * fill them in.
+     * FIXME: if this routine is ever on a performance-critical path then
+     * we should switch to a temp hashtable and avoid this quadratic cost.
+     */
+    uint num;
+    void **vec = NULL;
+    uint i, j;
+    DEBUG_DECLARE(uint res;)
+    ASSERT(dst != NULL && dst_num != NULL);
+    ASSERT(src1 != NULL || src1_num == 0);
+    ASSERT(src2 != NULL || src2_num == 0);
+    if (src1 == NULL || src2 == NULL || dst == NULL) /* paranoid */
+        return; /* FIXME: return a bool? */
+    if (src1_num == 0 && src2_num == 0) {
+        *dst = NULL;
+        *dst_num = 0;
+        return;
+    }
+    num = intersect ? 0 : src1_num;
+    for (i = 0; i < src2_num; i++) {
+        for (j = 0; j < src1_num; j++) {
+            if (src2[i] == src1[j]) {
+                if (intersect)
+                    num++;
+                break;
+            }
+        }
+        if (!intersect && j == src1_num)
+            num++;
+    }
+    if (num > 0) {
+        vec = HEAP_ARRAY_ALLOC(dcontext, void *, num, which, PROTECTED);
+        if (!intersect)
+            memcpy(vec, src1, sizeof(void *) * src1_num);
+        DODEBUG(res = num;);
+        num = intersect ? 0 : src1_num;
+        for (i = 0; i < src2_num; i++) {
+            for (j = 0; j < src1_num; j++) {
+                if (src2[i] == src1[j]) {
+                    if (intersect)
+                        vec[num++] = src2[i];
+                    break;
+                }
+            }
+            if (!intersect && j == src1_num)
+                vec[num++] = src2[i];
+        }
+        ASSERT(num == res);
+    } else {
+        ASSERT(intersect);
+        ASSERT(vec == NULL);
+    }
+    *dst = vec;
+    *dst_num = num;
+}
+
diff --git a/core/utils.h b/core/utils.h
new file mode 100644
index 000000000..4960d6f9c
--- /dev/null
+++ b/core/utils.h
@@ -0,0 +1,2030 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * utils.h - miscellenaous utilities
+ */
+
+#ifndef _UTILS_H_
+#define _UTILS_H_ 1
+
+#ifdef assert
+# undef assert
+#endif
+/* avoid mistake of lower-case assert */
+#define assert assert_no_good_use_ASSERT_instead
+#ifdef DEBUG
+# ifdef INTERNAL
+/* cast to void to avoid gcc warning "statement with no effect" when used as
+ * a statement and x is a compile-time false
+ * FIXME: cl debug build allocates a local for each ?:, so if this gets
+ * unrolled in some optionsx or other expansion we could have stack problems!
+ */
+#   define ASSERT(x) \
+        ((void)((!(x)) ? (internal_error(__FILE__, __LINE__, #x), 0) : 0))
+/* make type void to avoid gcc 4.1 warnings about "value computed is not used"
+ * (case 7851).  can also use statement expr ({;}) but only w/ gcc, not w/ cl.
+ */
+#   define ASSERT_MESSAGE(msg, x) ((!(x)) ? \
+        (internal_error(msg " @" __FILE__, __LINE__, #x), (void)0) : (void)0)
+#   define REPORT_CURIOSITY(x) do {                                                   \
+            if (!ignore_assert(__FILE__":"STRINGIFY(__LINE__), "curiosity : "#x)) {   \
+                report_dynamorio_problem(NULL, DUMPCORE_CURIOSITY, NULL, NULL,        \
+                                         "CURIOSITY : %s in file %s line %d", #x,     \
+                                          __FILE__, __LINE__);                        \
+            }                                                                         \
+        } while (0)
+#   define ASSERT_CURIOSITY(x) do {                                                   \
+           if (!(x)) {                                                                \
+               REPORT_CURIOSITY(x);                                                   \
+           }                                                                          \
+       } while (0)  
+#   define ASSERT_CURIOSITY_ONCE(x) do {                                              \
+           if (!(x)) {                                                                \
+               DO_ONCE({REPORT_CURIOSITY(x);});                                       \
+           }                                                                          \
+       } while (0)  
+# else 
+/* cast to void to avoid gcc warning "statement with no effect" */
+#   define ASSERT(x) \
+        ((void)((!(x)) ? (internal_error(__FILE__, __LINE__, ""), 0) : 0))
+#   define ASSERT_MESSAGE(msg, x) ((void)((!(x)) ? (internal_error(__FILE__, __LINE__, ""), 0) : 0))
+#   define ASSERT_CURIOSITY(x) ((void) 0)
+#   define ASSERT_CURIOSITY_ONCE(x) ((void) 0)
+# endif /* INTERNAL */
+# define ASSERT_NOT_TESTED() SYSLOG_INTERNAL_WARNING_ONCE("Not tested @%s:%d", \
+                                                          __FILE__, __LINE__)
+#else
+# define ASSERT(x)         ((void) 0)
+# define ASSERT_MESSAGE(msg, x) ASSERT(x)
+# define ASSERT_NOT_TESTED() ASSERT(true)
+# define ASSERT_CURIOSITY(x) ASSERT(true)
+# define ASSERT_CURIOSITY_ONCE(x) ASSERT(true)
+#endif /* DEBUG */
+
+#define ASSERT_NOT_REACHED() ASSERT(false)
+#define ASSERT_BUG_NUM(num, x) ASSERT_MESSAGE("Bug #"#num, x)
+#define ASSERT_NOT_IMPLEMENTED(x) ASSERT_MESSAGE("Not implemented", x)
+#define EXEMPT_TEST(tests) check_filter(tests, get_short_name(get_application_name()))
+
+#if defined(INTERNAL) || defined(DEBUG)
+void internal_error(char *file, int line, char *expr);
+bool ignore_assert(const char *assert_file_line, const char *expr);
+#endif
+
+/* only support apicheck for exported api's */
+#if defined(CLIENT_INTERFACE) || defined(DR_APP_EXPORTS)
+# define apicheck(x, msg) do {                   \
+    if (!(x))                                    \
+        external_error(__FILE__, __LINE__, msg); \
+} while (0)
+void external_error(char *file, int line, char *msg);
+#endif
+
+#ifdef CLIENT_INTERFACE
+# ifdef DEBUG
+#  define CLIENT_ASSERT(x, msg) apicheck(x, msg)
+# else
+#  define CLIENT_ASSERT(x, msg) /* PR 215261: nothing in release builds */
+# endif
+#else
+# define CLIENT_ASSERT(x, msg) ASSERT_MESSAGE(msg, x)
+#endif
+
+#ifdef DR_APP_EXPORTS
+# define APP_EXPORT_ASSERT(x, msg) apicheck(x, msg)
+#else
+# define APP_EXPORT_ASSERT(x, msg) ASSERT_MESSAGE(msg, x)
+#endif
+
+/* truncation assert - should be used wherever addressing cl warning 4244 */
+/* an explicit type cast to lower precision should match the type used here
+ * Later compiler versions cl.13 have an option /RCTc that will make make all
+ * explicit casts have that extra check, so these ASSERTs can then be removed.
+ */
+/* Assumption: int64 is our largest signed type (so casting to it never loses precision).
+ * Assumption C99 promotion rules (if same signess promote to size of larger, if
+ * different signess promote to size and signess of larger, if different signess and
+ * same size promote to unsigned). */
+#define CHECK_TRUNCATE_TYPE_byte(val) ((val) >= 0 && (val) <= UCHAR_MAX)
+#define CHECK_TRUNCATE_TYPE_sbyte(val) ((val) <= SCHAR_MAX && ((int64)(val)) >= SCHAR_MIN)
+#define CHECK_TRUNCATE_TYPE_ushort(val) ((val) >= 0 && (val) <= USHRT_MAX)
+#define CHECK_TRUNCATE_TYPE_short(val) ((val) <= SHRT_MAX && ((int64)(val)) >= SHRT_MIN)
+#define CHECK_TRUNCATE_TYPE_uint(val) ((val) >= 0 && (val) <= UINT_MAX)
+#ifdef LINUX
+/* We can't do the proper int check on Linux because it generates a warning if val has
+ * type uint that I can't seem to cast around and is impossible to ignore -
+ * "comparison is always true due to limited range of data type".
+ * See http://gcc.gnu.org/ml/gcc/2006-01/msg00784.html and note that the suggested
+ * workaround option doesn't exist as far as I can tell. We are potentially in trouble
+ * if val has type int64, is negative, and too big to fit in an int. */
+# define CHECK_TRUNCATE_TYPE_int(val) ((val) <= INT_MAX)
+#else
+# define CHECK_TRUNCATE_TYPE_int(val) ((val) <= INT_MAX && ((int64)(val)) >= INT_MIN)
+#endif
+#ifdef X64
+# define CHECK_TRUNCATE_TYPE_size_t(val) ((val) >= 0)
+/* avoid gcc warning: always true anyway since stats_int_t == int64 */
+# define CHECK_TRUNCATE_TYPE_stats_int_t(val) (true)
+#else
+# define CHECK_TRUNCATE_TYPE_size_t CHECK_TRUNCATE_TYPE_uint
+# define CHECK_TRUNCATE_TYPE_stats_int_t CHECK_TRUNCATE_TYPE_int
+#endif
+
+/* FIXME: too bad typeof is a GCC only extension - so need to pass both var and type */
+
+/* var = (type) val; should always be preceded by a call to ASSERT_TRUNCATE  */
+/* note that it is also OK to use ASSERT_TRUNCATE(type, type, val) for return values */
+#define ASSERT_TRUNCATE(var, type, val) ASSERT(sizeof(type) == sizeof(var) && \
+                                               CHECK_TRUNCATE_TYPE_##type(val) && \
+                                               "truncating "#var" to "#type)
+#define CLIENT_ASSERT_TRUNCATE(var, type, val, msg) \
+    CLIENT_ASSERT(sizeof(type) == sizeof(var) && CHECK_TRUNCATE_TYPE_##type(val), msg)
+
+/* assumes val is unsigned and width<32 */
+#define ASSERT_BITFIELD_TRUNCATE(width, val) \
+    ASSERT((val) < (1 << ((width)+1)) && "truncating to "#width" bits");
+#define CLIENT_ASSERT_BITFIELD_TRUNCATE(width, val, msg) \
+    CLIENT_ASSERT((val) < (1 << ((width)+1)), msg);
+                                               
+/* thread synchronization */
+#define LOCK_FREE_STATE -1     /* allows a quick >=0 test for contention */
+#define LOCK_SET_STATE (LOCK_FREE_STATE + 1)  /* set when requested by a single thread */
+/* any value greater than LOCK_SET_STATE means multiple threads requested the lock */
+
+#define SPINLOCK_FREE_STATE 0 /* for initstack_mutex is a spin lock with values 0 or 1 */
+
+#define CONTENTION_EVENT_NOT_CREATED ((contention_event_t)0)
+
+#ifdef WINDOWS
+typedef HANDLE contention_event_t;
+#else
+typedef void * contention_event_t;
+#endif
+
+typedef struct _mutex_t {
+    volatile int lock_requests; /* number of threads requesting this lock minus 1 */
+    /* a value greater than LOCK_FREE_STATE means the lock has been requested */
+    contention_event_t contended_event; /* handle to event object to wait on when contended */
+#ifdef DEADLOCK_AVOIDANCE
+    /* These fields are initialized with the INIT_LOCK_NO_TYPE macro */
+    char *    name;             /* set to variable lock name and location */
+    /* We flag as a violation if a lock with rank numerically smaller
+     * or equal to the rank of a lock already held by the owning thread is acquired
+     */
+    uint      rank;             /* sets rank order in which this lock can be set */
+    thread_id_t owner;            /* TID of owner (reusable, not available before initialization) */
+    dcontext_t *owning_dcontext; /* dcontext responsible (reusable, multiple per thread) */
+    struct _mutex_t *prev_owned_lock; /* linked list of thread owned locks */
+    uint count_times_acquired;  /* count total times this lock was acquired */
+    uint count_times_contended; /* count total times this lock was contended upon */
+    uint count_times_spin_pause; /* count total times this lock was contended in a spin pause loop */
+    uint max_contended_requests;/* maximum number of simultaneous requests when contended */
+    uint count_times_spin_only; /* count times contended but grabbed after spinning without yielding */
+    /* we need to register all locks in the process to be able to dump regular statistics */
+    /* linked list of all live locks (for all threads), another ad hoc double linked circular list */
+    struct _mutex_t *prev_process_lock;
+    struct _mutex_t *next_process_lock;
+    /* TODO: we should also add cycles spent while holding the lock, KSTATS-like */
+#ifdef MUTEX_CALLSTACK
+    /* FIXME: case 5378: to avoid allocating memory in mutexes use a
+     * static array that will get too fat if larger than 4
+     */
+#  define MAX_MUTEX_CALLSTACK 4
+    byte *callstack[MAX_MUTEX_CALLSTACK];
+    /* keep as last item so not initialized in INIT_LOCK_NO_TYPE */
+#else
+#  define MAX_MUTEX_CALLSTACK 0 /* cannot use */
+#endif /* MUTEX_CALLSTACK */
+#endif /* DEADLOCK_AVOIDANCE */
+    /* Any new field needs to be initialized with INIT_LOCK_NO_TYPE */
+} mutex_t;
+
+/* A spin_mutex_t is the same thing as a mutex_t (and all utils.c users use it as
+ * such).  It exists only to enforce type separation outside of utils.c which
+ * is why we can't simply do a typedef mutex_t spin_mutex_t. */
+typedef struct _spin_mutex_t {
+    mutex_t lock;
+} spin_mutex_t;
+
+/* perhaps for DEBUG all locks should record owner? */
+typedef struct _recursive_lock_t {
+    mutex_t lock;
+    /* ASSUMPTION: reading owner field is atomic!
+     * Thus must allocate this statically (compiler should 4-byte-align
+     * this field, which is good enough) or align it manually!
+     * FIXME: provide creation routine that does that for non-static locks
+     */
+    thread_id_t owner;
+    uint count;
+} recursive_lock_t;
+
+typedef struct _read_write_lock_t {
+    mutex_t lock;
+    /* FIXME: could be merged w/ lock->state if want to get more sophisticated...
+     * we could use the lock->state as a 32-bit counter, incremented 
+     * by readers, and with the MSB bit (sign) set by writers 
+     */
+    volatile int num_readers;
+    /* we store the writer so that writers can be readers */
+    thread_id_t writer;
+    volatile int num_pending_readers;       /* readers that have contended with a writer */
+    contention_event_t writer_waiting_readers; /* event object for writer to wait on */
+    contention_event_t readers_waiting_writer; /* event object for readers to wait on */
+    /* make sure to update the two INIT_READWRITE_LOCK cases if you add new fields  */
+} read_write_lock_t;
+
+#ifdef DEADLOCK_AVOIDANCE
+#  define LOCK_RANK(lock) lock##_rank
+/* This should be the single place where all ranks are declared */
+/* Your lock should preferrably take the last possible rank in this
+ * list, usually at the location marked as ADD HERE  */
+
+enum {
+    LOCK_RANK(outermost_lock), /* pseudo lock, sentinel of thread owned locks list */
+    LOCK_RANK(thread_in_DR_exclusion),  /* outermost */
+    LOCK_RANK(all_threads_synch_lock),  /* < thread_initexit_lock */
+
+    LOCK_RANK(trace_building_lock), /* < bb_building_lock, < table_rwlock */
+
+    LOCK_RANK(bb_building_lock), /* < change_linking_lock + all vm and heap locks */
+    /* decode exception -> check if should_intercept requires all_threads 
+     * FIXME: any other locks that could be interrupted by exception that
+     * could be app's fault?
+     */
+    LOCK_RANK(thread_initexit_lock), /* < all_threads_lock, < snapshot_lock */
+
+#if defined(WINDOWS) && defined(STACK_GUARD_PAGE)
+    LOCK_RANK(exception_stack_lock), /* < all_threads_lock */
+#endif
+    /* FIXME: grabbed on an exception, which could happen anywhere! 
+     * possible deadlock if already held */
+    LOCK_RANK(all_threads_lock),  /* < global_alloc_lock */
+
+    LOCK_RANK(linking_lock),  /* < dynamo_areas < global_alloc_lock */
+
+#ifdef SHARING_STUDY
+    LOCK_RANK(shared_blocks_lock), /* < global_alloc_lock */
+    LOCK_RANK(shared_traces_lock), /* < global_alloc_lock */
+#endif
+
+    LOCK_RANK(synch_lock), /* per thread, < protect_info */
+
+    LOCK_RANK(protect_info), /* < cache and heap traversal locks */
+
+#if defined(CLIENT_SIDELINE) && defined(CLIENT_INTERFACE)
+    LOCK_RANK(sideline_mutex), 
+    LOCK_RANK(sideline_heap_lock),
+    LOCK_RANK(fragment_delete_mutex),
+#endif
+
+    LOCK_RANK(shared_cache_flush_lock), /* < shared_cache_count_lock,
+                                           < shared_delete_lock,
+                                           < change_linking_lock */
+    LOCK_RANK(shared_delete_lock), /* < change_linking_lock < shared_vm_areas */
+    LOCK_RANK(lazy_delete_lock), /* > shared_delete_lock, < shared_cache_lock */
+
+    LOCK_RANK(shared_cache_lock), /* < dynamo_areas, < allunits_lock,
+                                   * < table_rwlock for shared cache regen/replace,
+                                   * < shared_vm_areas for cache unit flush,
+                                   * < change_linking_lock for add_to_free_list */
+
+    LOCK_RANK(change_linking_lock), /* < shared_vm_areas, < all heap locks */
+    LOCK_RANK(shared_vm_areas), /* > change_linking_lock, < executable_areas  */
+    LOCK_RANK(shared_cache_count_lock),
+    LOCK_RANK(tracedump_mutex),  /* < table_rwlock, > change_linking_lock */
+
+    LOCK_RANK(emulate_write_lock), /* in future may be < emulate_write_areas */
+
+    LOCK_RANK(unit_flush_lock), /* > shared_delete_lock */
+
+#ifdef LINUX
+    LOCK_RANK(maps_iter_buf_lock), /* < executable_areas, < module_data_lock,
+                                    * < hotp_vul_table_lock */
+#endif
+
+#ifdef HOT_PATCHING_INTERFACE
+    /* This lock's rank needs to be after bb_building_lock because 
+     * build_bb_ilist() is where injection takes place, which means the 
+     * bb lock has been acquired before any hot patching related work is done 
+     * on a bb.
+     */
+    LOCK_RANK(hotp_vul_table_lock), /* > bb_building_lock, 
+                                     * < dynamo_areas, < heap_unit_lock. */
+#endif
+    LOCK_RANK(coarse_info_lock), /* < special_heap_lock, < global_alloc_lock,
+                                  * > change_linking_lock */
+    LOCK_RANK(special_units_list_lock), /* < special_heap_lock */
+    LOCK_RANK(special_heap_lock), /* > bb_building_lock, > hotp_vul_table_lock
+                                   * < dynamo_areas, < heap_unit_lock */
+    LOCK_RANK(coarse_info_incoming_lock), /* > special_heap_lock, > coarse_info_lock,
+                                           * > change_linking_lock */
+
+    /* (We don't technically need a coarse_table_rwlock separate from table_rwlock
+     * anymore but having it gives us flexibility so I'm leaving it)
+     */
+    LOCK_RANK(coarse_table_rwlock), /* < global_alloc_lock, < coarse_th_table_rwlock */
+    /* We make the th table separate (we look in it while holding master table lock) */
+    LOCK_RANK(coarse_th_table_rwlock), /* < global_alloc_lock */
+    /* We make the pc table separate (we write it while holding master table lock) */
+    LOCK_RANK(coarse_pclookup_table_rwlock), /* < global_alloc_lock */
+
+    LOCK_RANK(executable_areas), /* < dynamo_areas < global_alloc_lock
+                                  * < process_module_vector_lock (diagnostics)
+                                  */
+#ifdef RCT_IND_BRANCH
+    LOCK_RANK(rct_module_lock), /* > coarse_info_lock, > executable_areas,
+                                 * < heap allocation */
+#endif
+#ifdef RETURN_AFTER_CALL
+    LOCK_RANK(after_call_lock), /* < table_rwlock, > bb_building_lock,
+                                 * > coarse_info_lock, > executable_areas */
+#endif
+    LOCK_RANK(process_module_vector_lock), /* < snapshot_lock > all_threads_synch_lock */
+    /* For Loglevel 1 and higher, with LOG_MEMSTATS, the snapshot lock is
+     * grabbed on an exception, possible deadlock if already held FIXME */
+    LOCK_RANK(snapshot_lock),   /* < dynamo_areas */
+    LOCK_RANK(written_areas), /* > executable_areas
+                               * < dynamo_areas < global_alloc_lock */
+#ifdef PROGRAM_SHEPHERDING
+    LOCK_RANK(futureexec_areas), /* > executable_areas
+                                  * < dynamo_areas < global_alloc_lock */
+# ifdef WINDOWS
+    LOCK_RANK(app_flushed_areas), /* < dynamo_areas < global_alloc_lock */
+# endif
+#endif
+    LOCK_RANK(pretend_writable_areas), /* < dynamo_areas < global_alloc_lock */
+    LOCK_RANK(patch_proof_areas), /* < dynamo_areas < global_alloc_lock */
+    LOCK_RANK(emulate_write_areas), /* < dynamo_areas < global_alloc_lock */
+    LOCK_RANK(IAT_areas), /* < dynamo_areas < global_alloc_lock */
+    LOCK_RANK(module_data_lock),  /* < loaded_module_areas */
+#ifdef CLIENT_INTERFACE
+    /* PR 198871: this same label is used for all client locks */
+    LOCK_RANK(dr_client_mutex), /* > module_data_lock */
+    LOCK_RANK(client_nudge_count_lock), /* > dr_client_mutex */
+    LOCK_RANK(client_flush_request_lock), /* > dr_client_mutex */
+    LOCK_RANK(callback_registration_lock), /* > dr_client_mutex */
+#endif
+    LOCK_RANK(table_rwlock), /* > dr_client_mutex */
+    LOCK_RANK(loaded_module_areas),  /* < dynamo_areas < global_alloc_lock */
+    LOCK_RANK(aslr_areas), /* < dynamo_areas < global_alloc_lock */
+    LOCK_RANK(aslr_pad_areas), /* < dynamo_areas < global_alloc_lock */
+    LOCK_RANK(native_exec_areas), /* < dynamo_areas < global_alloc_lock */
+    LOCK_RANK(thread_vm_areas), /* currently never used */
+
+    LOCK_RANK(app_pc_table_rwlock), /* > after_call_lock, > rct_module_lock,
+                                     * > module_data_lock */
+
+    LOCK_RANK(dead_tables_lock), /* < heap_unit_lock */
+    LOCK_RANK(aslr_lock),
+
+#ifdef HOT_PATCHING_INTERFACE
+    LOCK_RANK(hotp_only_tramp_areas_lock),  /* > hotp_vul_table_lock,
+                                             * < global_alloc_lock */
+    LOCK_RANK(hotp_patch_point_areas_lock), /* > hotp_vul_table_lock,
+                                             * < global_alloc_lock */
+#endif
+#ifdef CALL_PROFILE
+    LOCK_RANK(profile_callers_lock), /* < global_alloc_lock */
+#endif
+    LOCK_RANK(coarse_stub_areas), /* < global_alloc_lock */
+    LOCK_RANK(moduledb_lock), /* < global heap allocation */
+    LOCK_RANK(pcache_dir_check_lock),
+#ifdef LINUX
+    LOCK_RANK(suspend_lock),
+#endif
+    /* ADD HERE a lock around section that may allocate memory */
+
+    /* N.B.: the order of allunits < global_alloc < heap_unit is relied on
+     * in the {fcache,heap}_low_on_memory routines.  IMPORTANT - any locks
+     * added between the allunits_lock and heap_unit_lock must have special
+     * handling in the fcache_low_on_memory() routine. 
+     */
+    LOCK_RANK(allunits_lock),  /* < global_alloc_lock */
+    LOCK_RANK(fcache_unit_areas), /* > allunits_lock, 
+                                     < dynamo_areas, < global_alloc_lock */
+    IF_LINUX_(IF_DEBUG(LOCK_RANK(elf_areas))) /* < all_memory_areas */
+    IF_LINUX_(LOCK_RANK(all_memory_areas))    /* < dynamo_areas */
+    LOCK_RANK(landing_pad_areas_lock),  /* < global_alloc_lock, < dynamo_areas */
+    LOCK_RANK(dynamo_areas),    /* < global_alloc_lock */
+    LOCK_RANK(map_intercept_pc_lock), /* < global_alloc_lock */
+    LOCK_RANK(global_alloc_lock),/* < heap_unit_lock */
+    LOCK_RANK(heap_unit_lock),   /* recursive */
+    LOCK_RANK(vmh_lock),        /* lowest level */
+    LOCK_RANK(last_deallocated_lock),
+    /*---- noone below here can be held at a memory allocation site ----*/
+
+#ifdef LINUX
+    LOCK_RANK(tls_lock), /* if used for get_thread_private_dcontext() may
+                          * need to be even lower: as it is, only used for set */
+#endif
+    LOCK_RANK(reset_pending_lock), /* > heap_unit_lock */
+
+    LOCK_RANK(initstack_mutex),  /* FIXME: NOT TESTED */
+
+    LOCK_RANK(event_lock),  /* FIXME: NOT TESTED */
+    LOCK_RANK(do_threshold_mutex),  /* FIXME: NOT TESTED */
+    LOCK_RANK(threads_killed_lock),  /* FIXME: NOT TESTED */
+    LOCK_RANK(child_lock),  /* FIXME: NOT TESTED */
+    LOCK_RANK(shared_lock),  /* FIXME: NOT TESTED */
+    
+#ifdef SIDELINE
+    LOCK_RANK(sideline_lock), /* FIXME: NOT TESTED */
+    LOCK_RANK(do_not_delete_lock),/* FIXME: NOT TESTED */
+    LOCK_RANK(remember_lock),/* FIXME: NOT TESTED */
+    LOCK_RANK(sideline_table_lock),/* FIXME: NOT TESTED */
+#endif
+#ifdef SIMULATE_ATTACK
+    LOCK_RANK(simulate_lock),
+#endif
+#ifdef KSTATS
+    LOCK_RANK(process_kstats_lock),
+#endif
+#ifdef X64
+    LOCK_RANK(request_region_be_heap_reachable_lock), /* > heap_unit_lock, vmh_lock
+                                                       * < report_buf_lock (for assert) */
+#endif
+    LOCK_RANK(report_buf_lock),
+    /* FIXME: if we crash while holding the all_threads_lock, snapshot_lock 
+     * (for loglevel 1+, logmask LOG_MEMSTATS), or any lock below this
+     * line (except the profile_dump_lock, and possibly others depending on
+     * options) we will deadlock
+     */
+#ifdef LINUX
+    LOCK_RANK(memory_info_buf_lock),
+#endif
+#ifdef WINDOWS
+    LOCK_RANK(dump_core_lock),
+#endif
+
+    LOCK_RANK(logdir_mutex),     /* recursive */
+    LOCK_RANK(diagnost_reg_mutex),
+#ifdef WINDOWS_PC_SAMPLE
+    LOCK_RANK(profile_dump_lock),
+#endif
+
+    LOCK_RANK(prng_lock),
+    /* ---------------------------------------------------------- */
+    /* No new locks below this line, reserved for innermost ASSERT,
+     * SYSLOG and STATS facilities */
+    LOCK_RANK(options_lock),
+#ifdef WINDOWS
+    LOCK_RANK(debugbox_lock),
+#endif
+    LOCK_RANK(eventlog_mutex), /* < datasec_selfprot_lock only for hello_message */
+    LOCK_RANK(datasec_selfprot_lock),
+    LOCK_RANK(thread_stats_lock),
+    LOCK_RANK(innermost_lock), /* innermost internal lock, head of all locks list */
+};
+
+struct _thread_locks_t;
+typedef struct _thread_locks_t thread_locks_t;
+
+extern mutex_t outermost_lock;
+
+void locks_thread_init(dcontext_t *dcontext);
+void locks_thread_exit(dcontext_t *dcontext);
+uint locks_not_closed(void);
+bool thread_owns_no_locks(dcontext_t *dcontext);
+bool thread_owns_one_lock(dcontext_t *dcontext, mutex_t *lock);
+bool thread_owns_two_locks(dcontext_t *dcontext, mutex_t *lock1, mutex_t *lock2);
+bool thread_owns_first_or_both_locks_only(dcontext_t *dcontext, mutex_t *lock1, mutex_t *lock2);
+
+/* We need the (mutex_t) type specifier for direct initialization, 
+   but not when defining compound structures, hence NO_TYPE */
+#  define INIT_LOCK_NO_TYPE(name, rank) {LOCK_FREE_STATE,               \
+                                         CONTENTION_EVENT_NOT_CREATED,  \
+                                         name, rank,                    \
+                                         INVALID_THREAD_ID,             \
+                                         NULL, NULL,                    \
+                                         0, 0, 0, 0, 0,                 \
+                                         NULL, NULL}
+#else
+/* Ignore the arguments */
+#  define INIT_LOCK_NO_TYPE(name, rank) {LOCK_FREE_STATE, CONTENTION_EVENT_NOT_CREATED}
+#endif /* DEADLOCK_AVOIDANCE */
+
+/* Structure assignments and initialization don't work the same in gcc and cl
+   in gcc it is ok to have assignment {gcc; var = (mutex_t) {0}; } 
+      and so it helped to have the type specifier.  
+      Windows however doesn't like that at all even for initialization, 
+      so I have to go and add explicit initialized temporaries to be assigned to a variable
+      i.e. {cl; {mutex_t temp = {0}; var = temp; }; }
+      }
+*/
+#ifdef WINDOWS
+#  define STRUCTURE_TYPE(x) 
+#else
+/* FIXME: gcc 4.1.1 complains "initializer element is not constant"
+ * if we have our old (x) here; presumably some older gcc needed it;
+ * once sure nobody does let's remove this define altogether.
+ */
+#  define STRUCTURE_TYPE(x)
+#endif
+
+#define INIT_LOCK_FREE(lock) STRUCTURE_TYPE(mutex_t) INIT_LOCK_NO_TYPE( \
+       #lock "(mutex)" "@" __FILE__ ":" STRINGIFY(__LINE__),          \
+       LOCK_RANK(lock))
+
+#define ASSIGN_INIT_LOCK_FREE(var, lock) do {                           \
+     mutex_t initializer_##lock = STRUCTURE_TYPE(mutex_t) INIT_LOCK_NO_TYPE(\
+            #lock "(mutex)" "@" __FILE__ ":" STRINGIFY(__LINE__),       \
+                                                 LOCK_RANK(lock));      \
+     var = initializer_##lock;                                          \
+   } while (0)
+
+#define ASSIGN_INIT_SPINMUTEX_FREE(var, spinmutex) \
+    ASSIGN_INIT_LOCK_FREE((var).lock, spinmutex)
+
+#define INIT_RECURSIVE_LOCK(lock) STRUCTURE_TYPE(recursive_lock_t) {       \
+     INIT_LOCK_NO_TYPE(                                                 \
+       #lock "(recursive)" "@" __FILE__ ":" STRINGIFY(__LINE__) ,       \
+       LOCK_RANK(lock)),                                                \
+     INVALID_THREAD_ID, 0                                               \
+   }
+
+#define INIT_READWRITE_LOCK(lock) STRUCTURE_TYPE(read_write_lock_t) {       \
+     INIT_LOCK_NO_TYPE(                                                 \
+       #lock "(readwrite)" "@" __FILE__ ":" STRINGIFY(__LINE__) ,       \
+       LOCK_RANK(lock)),                                                \
+       0, INVALID_THREAD_ID,                                            \
+       0,                                                               \
+       CONTENTION_EVENT_NOT_CREATED, CONTENTION_EVENT_NOT_CREATED       \
+   }
+
+#define ASSIGN_INIT_READWRITE_LOCK_FREE(var, lock) do {                 \
+     read_write_lock_t initializer_##lock = STRUCTURE_TYPE(read_write_lock_t)   \
+      {INIT_LOCK_NO_TYPE(                                               \
+            #lock "(readwrite)" "@" __FILE__ ":" STRINGIFY(__LINE__),   \
+                                                 LOCK_RANK(lock)),      \
+       0, INVALID_THREAD_ID,                                            \
+       0,                                                               \
+       CONTENTION_EVENT_NOT_CREATED, CONTENTION_EVENT_NOT_CREATED       \
+      };                                                                \
+     var = initializer_##lock;                                          \
+   } while (0)
+
+#define ASSIGN_INIT_RECURSIVE_LOCK_FREE(var, lock) do {                 \
+     recursive_lock_t initializer_##lock = STRUCTURE_TYPE(recursive_lock_t)   \
+      {INIT_LOCK_NO_TYPE(                                               \
+            #lock "(recursive)" "@" __FILE__ ":" STRINGIFY(__LINE__),   \
+                                                 LOCK_RANK(lock)),      \
+       INVALID_THREAD_ID, 0};                                           \
+     var = initializer_##lock;                                          \
+   } while (0)
+
+#define INIT_SPINLOCK_FREE(lock) STRUCTURE_TYPE(mutex_t) {SPINLOCK_FREE_STATE, }
+
+/* in order to use parallel names to the above INIT_*LOCK routines */
+#define DELETE_LOCK(lock) mutex_delete(&lock)
+#define DELETE_SPINMUTEX(spinmutex) spinmutex_delete(&spinmutex)
+#define DELETE_RECURSIVE_LOCK(rec_lock) mutex_delete(&rec_lock.lock)
+#define DELETE_READWRITE_LOCK(rwlock) mutex_delete(&rwlock.lock)
+/* mutexes need to release any kernel objects that were created */
+void mutex_delete(mutex_t *lock); 
+
+/* basic synchronization functions */
+void mutex_lock(mutex_t *mutex);
+bool mutex_trylock(mutex_t *mutex);
+void mutex_unlock(mutex_t *mutex);
+
+/* spinmutex synchronization */
+bool spinmutex_trylock(spin_mutex_t *spin_lock);
+void spinmutex_lock(spin_mutex_t *spin_lock);
+void spinmutex_lock_no_yield(spin_mutex_t *spin_lock);
+void spinmutex_unlock(spin_mutex_t *spin_lock);
+void spinmutex_delete(spin_mutex_t *spin_lock);
+
+/* tests if a lock is held, but doesn't grab it */
+/* note that this is not a synchronizing function, its intended uses are :
+ * 1. for synch code to guarantee that a thread it has suspened isn't holding
+ * a lock (note that a return of true doesn't mean that the suspended thread 
+ * is holding the lock, could be some other thread)
+ * 2. for when you want to assert that you hold a lock, while you can't 
+ * actually do that, you can assert with this function that the lock is held
+ * by someone
+ * 3. read_{un,}lock use this function to check the state of write lock mutex
+ */
+static inline bool
+mutex_testlock(mutex_t *lock)
+{
+    return lock->lock_requests > LOCK_FREE_STATE;
+}
+
+
+/* A recursive lock can be taken more than once by the owning thread */
+void acquire_recursive_lock(recursive_lock_t *lock);
+bool try_recursive_lock(recursive_lock_t *lock);
+void release_recursive_lock(recursive_lock_t *lock);
+bool self_owns_recursive_lock(recursive_lock_t *lock);
+
+/* A read write lock allows multiple readers or alternatively a single writer */
+void read_lock(read_write_lock_t *rw);
+void write_lock(read_write_lock_t *rw);
+bool write_trylock(read_write_lock_t *rw);
+void read_unlock(read_write_lock_t *rw);
+void write_unlock(read_write_lock_t *rw);
+bool self_owns_write_lock(read_write_lock_t *rw);
+
+/* a broadcast event wakes all waiting threads when signalled */
+struct _broadcast_event_t;
+typedef struct _broadcast_event_t broadcast_event_t;
+broadcast_event_t * create_broadcast_event(void);
+void destroy_broadcast_event(broadcast_event_t *be);
+/* NOTE : to avoid races a signaler should always do the required action to 
+ * make any wait_condition(s) for the WAIT_FOR_BROADCAST_EVENT(s) on the 
+ * event false BEFORE signaling the event
+ */
+void signal_broadcast_event(broadcast_event_t *be);
+/* the wait macro, we force people to use a while loop since our current 
+ * implementation has a race (ATOMIC_INC, else clause ATOMIC_DEC, in the 
+ * middle someone could signal thinking we are waiting)
+ *
+ * event is the broadcast event
+ * wait_condition is the conditional expression we want to become true
+ * pre is code we should execute before we actually do a wait
+ * post is code we should execute after we return from a wait
+ *
+ * in typical usage pre and post might be empty, or might free and regrab a
+ * lock we shouldn't hold during the wait
+ * NOTE : because we loop to handle certain race conditions, wait_condition
+ * pre and post might all be evaluated several times
+ */
+#define WAIT_FOR_BROADCAST_EVENT(wait_condition, pre, post, event) do {     \
+        while (wait_condition) {                                            \
+            intend_wait_broadcast_event_helper(event);                      \
+            if (wait_condition) {                                           \
+                pre;                                                        \
+                wait_broadcast_event_helper(event);                         \
+                post;                                                       \
+            } else {                                                        \
+                unintend_wait_broadcast_event_helper(event);                \
+            }                                                               \
+        }                                                                   \
+    } while (0)
+/* helpers for the broadcast event wait macro, don't call these directly */
+void intend_wait_broadcast_event_helper(broadcast_event_t *be);
+void unintend_wait_broadcast_event_helper(broadcast_event_t *be);
+void wait_broadcast_event_helper(broadcast_event_t *be);
+
+/* test whether locks are held at all */
+#define WRITE_LOCK_HELD(rw) (mutex_testlock(&(rw)->lock) && ((rw)->num_readers == 0))
+#define READ_LOCK_HELD(rw) ((rw)->num_readers > 0)
+
+/* test whether current thread owns locks
+ * for non-DEADLOCK_AVOIDANCE, cannot tell who owns it, so we bundle
+ * into asserts to make sure not used in a way that counts on it
+ * knowing for sure.
+ */
+#ifdef DEADLOCK_AVOIDANCE
+# define OWN_MUTEX(m) ((m)->owner == get_thread_id())
+# define ASSERT_OWN_MUTEX(pred, m) \
+    ASSERT(!(pred) || OWN_MUTEX(m))
+# define ASSERT_DO_NOT_OWN_MUTEX(pred, m) \
+    ASSERT(!(pred) || !OWN_MUTEX(m))
+# define ASSERT_OWN_NO_LOCKS() \
+    ASSERT(get_thread_private_dcontext() == NULL /* no way to tell */ || \
+           thread_owns_no_locks(get_thread_private_dcontext()))
+#else
+/* don't know for sure: imprecise in a conservative direction */
+# define OWN_MUTEX(m) (mutex_testlock(m))
+# define ASSERT_OWN_MUTEX(pred, m) \
+    ASSERT(!(pred) || OWN_MUTEX(m))
+# define ASSERT_DO_NOT_OWN_MUTEX(pred, m) \
+    ASSERT(!(pred) || true/*no way to tell*/)
+# define ASSERT_OWN_NO_LOCKS() /* no way to tell */
+#endif
+#define ASSERT_OWN_WRITE_LOCK(pred, rw) \
+    ASSERT(!(pred) || self_owns_write_lock(rw))
+#define ASSERT_DO_NOT_OWN_WRITE_LOCK(pred, rw) \
+    ASSERT(!(pred) || !self_owns_write_lock(rw))
+/* FIXME: no way to tell if current thread is one of the readers */
+#define ASSERT_OWN_READ_LOCK(pred, rw) \
+    ASSERT(!(pred) || READ_LOCK_HELD(rw))
+#define READWRITE_LOCK_HELD(rw) (READ_LOCK_HELD(rw) || self_owns_write_lock(rw))
+#define ASSERT_OWN_READWRITE_LOCK(pred, rw) \
+    ASSERT(!(pred) || READWRITE_LOCK_HELD(rw))
+#define ASSERT_OWN_RECURSIVE_LOCK(pred, l) \
+    ASSERT(!(pred) || self_owns_recursive_lock(l))
+
+/* in machine-specific assembly file: */
+int atomic_swap(volatile int *addr, int value);
+
+#define SHARED_MUTEX(operation, lock) do {                                    \
+    if (SHARED_FRAGMENTS_ENABLED() && !INTERNAL_OPTION(single_thread_in_DR))  \
+        mutex_##operation(&(lock));                                           \
+} while (0)
+#define SHARED_RECURSIVE_LOCK(operation, lock) do {                           \
+    if (SHARED_FRAGMENTS_ENABLED() && !INTERNAL_OPTION(single_thread_in_DR))  \
+        operation##_recursive_lock(&(lock));                                  \
+} while (0)
+#define USE_BB_BUILDING_LOCK()  \
+    (DYNAMO_OPTION(shared_bbs) && !INTERNAL_OPTION(single_thread_in_DR))
+#define SHARED_BB_MUTEX(operation) do {                                      \
+    if (USE_BB_BUILDING_LOCK())                                              \
+        mutex_##operation(&(bb_building_lock));                              \
+} while (0)
+/* we assume dynamo_resetting is only done w/ all threads suspended */
+#define NEED_SHARED_LOCK(flags)                                          \
+    (TEST(FRAG_SHARED, (flags)) && !INTERNAL_OPTION(single_thread_in_DR) \
+     && !dynamo_exited && !dynamo_resetting)
+#define SHARED_FLAGS_MUTEX(flags, operation, lock) do {  \
+    if (NEED_SHARED_LOCK((flags)))                       \
+        mutex_##operation(&(lock));                      \
+} while (0)
+#define SHARED_FLAGS_RECURSIVE_LOCK(flags, operation, lock) do {  \
+    if (NEED_SHARED_LOCK((flags)))                                \
+        operation##_recursive_lock(&(lock));                      \
+} while (0)
+
+/* Our parameters (option string, logdir, etc.) can be configured
+ * through environment variables or, on win32, the registry.
+ * The routines below are used to check both of those places.
+ * value is a buffer allocated by the caller to hold the
+ * resulting value.
+ */
+
+#ifndef WINDOWS
+int get_parameter(const char *name, char *value, int maxlen);
+int get_unqualified_parameter(const char *name, char *value, int maxlen);
+#else
+/* redeclared in inject_shared.h */
+int get_parameter(const wchar_t *name, char *value, int maxlen);
+#endif
+
+/****************************************************************************
+ * hashing functions
+ */
+/* bits=entries: 8=256, 12=4096, 13=8192, 14=16384, 16=65536 */
+
+/* Our default hash function is a bitwise & with a mask (HASH_FUNC_BITS) 
+ * to select index bits, and maybe offset to ignore least significant bits.
+ * (as we're dealing with pcs here that is generally enough, but for larger
+ *  table sizes a more complicated function is sometimes needed so the 
+ *  HASH_FUNC macro supports other hashing functions)
+ * These macros would be typed as follows:
+ *   extern uint HASH_FUNC(uint val, table* table);
+ *      where table is any struct that contains 
+ *      hash_func, hash_mask and hash_offset elements
+ *   extern uint HASH_FUNC_BITS(uint val, int num_bits);
+ *   extern uint HASH_MASK(int num_bits);
+ *   extern uint HASHTABLE_SIZE(int num_bits);
+ */
+
+/* FIXME - xref 8139 which is best 2654435769U 2654435761U or 0x9e379e37 
+ * FIXME PR 212574 (==8139): would we want the 32-bit one for smaller index values?
+ */
+#define PHI_2_32  2654435769U /* (sqrt(5)-1)/2 * (2^32) */
+#define PHI_2_64  11400714819323198485U /* (sqrt(5)-1)/2 * (2^64) */
+#ifdef X64
+# define HASH_PHI PHI_2_64
+# define HASH_TAG_BITS 64
+#else
+# define HASH_PHI PHI_2_32
+# define HASH_TAG_BITS 32
+#endif
+
+/* bitmask selecting num_bits least significant bits */
+#define HASH_MASK(num_bits) ((~PTR_UINT_0)>>(HASH_TAG_BITS-(num_bits)))
+
+/* evaluate hash function and select index bits.  Although bit
+ * selection and shifting could be done in reverse, better assembly
+ * code can be emitted when hash_mask selects the index bits */
+#define HASH_FUNC(val, table)                                           \
+      ((uint)((HASH_VALUE_FOR_TABLE(val, table) & ((table)->hash_mask))        \
+       >> (table)->hash_mask_offset))
+
+#ifdef X86
+  /* no instruction alignment -> use the lsb! 
+   * for 64-bit we assume the mask is taking out everything beyond uint range
+   */
+# define HASH_FUNC_BITS(val, num_bits) ((uint)((val) & (HASH_MASK(num_bits))))
+#else
+  /* do not use the lsb (alignment!) */
+  /* FIXME: this function in product builds is in fact not used on
+   * addresses so ignoring the LSB is not helping.
+   * Better use the more generic HASH_FUNC that allows for hash_offset other than 1
+   */
+# define HASH_FUNC_BITS(val, num_bits) (((val) & (HASH_MASK(num_bits))) >> 1)
+#endif
+
+/* FIXME - xref 8139, what's the best shift for multiply phi? In theory for
+ * m bit table should take the middle m bits of the qword result. We currently
+ * take the top m bits of the lower dword which is prob. almost as good, but
+ * could experiment (however, I suspect probing strategy might make more of a
+ * difference at that point). You're not allowed to touch this code without
+ * first reading Knuth vol 3 sec 6.4 */
+#define HASH_VALUE_FOR_TABLE(val, table)                                \
+    ((table)->hash_func == HASH_FUNCTION_NONE ?                         \
+         (val)                                :                         \
+         ((table)->hash_func == HASH_FUNCTION_MULTIPLY_PHI ?            \
+             /* All ibl tables use NONE so don't need to worry about    \
+              * the later hash_mask_offset shift. FIXME - structure all \
+              * these macros a little clearly/efficiently. */           \
+             /* case 8457: keep in sync w/ hash_value()'s calc */       \
+             (((val) * HASH_PHI) >> (HASH_TAG_BITS - (table)->hash_bits)) : \
+             hash_value((val), (table)->hash_func, (table)->hash_mask,  \
+                        (table)->hash_bits)))
+
+#define HASHTABLE_SIZE(num_bits) (1U << (num_bits))
+
+/* Note that 1 will be the default */
+typedef enum {
+    HASH_FUNCTION_NONE = 0,
+    HASH_FUNCTION_MULTIPLY_PHI = 1,
+#ifdef INTERNAL
+    HASH_FUNCTION_LOWER_BSWAP = 2,
+    HASH_FUNCTION_BSWAP_XOR = 3,
+    HASH_FUNCTION_SWAP_12TO15 = 4,
+    HASH_FUNCTION_SWAP_12TO15_AND_NONE = 5,
+    HASH_FUNCTION_SHIFT_XOR = 6,
+#endif
+    HASH_FUNCTION_ENUM_MAX,
+} hash_function_t;
+
+ptr_uint_t hash_value(ptr_uint_t val, hash_function_t func, ptr_uint_t mask, uint bits);
+uint hashtable_num_bits(uint size);
+/****************************************************************************/
+
+/* Reachability helpers */
+/* Given a region, returns the start of the enclosing region that can reached by a
+ * 32bit displacement from everywhere in the supplied region. Checks for underflow. If the
+ * supplied region is too large then returned value may be > reachable_region_start
+ * (caller should check) as the constraint may not be satisfiable. */
+# define REACHABLE_32BIT_START(reachable_region_start, reachable_region_end) \
+    (((reachable_region_end) + INT_MIN < (reachable_region_end)) ?           \
+     (reachable_region_end) + INT_MIN : (byte *)PTR_UINT_0)
+/* Given a region, returns the end of the enclosing region that can reached by a
+ * 32bit displacement from everywhere in the supplied region. Checks for overflow. If the
+ * supplied region is too large then returned value may be < reachable_region_end
+ * (caller should check) as the constraint may not be satisfiable. */
+# define REACHABLE_32BIT_END(reachable_region_start, reachable_region_end)   \
+    (((reachable_region_start) + INT_MAX > (reachable_region_start)) ?  \
+     (reachable_region_start) + INT_MAX : (byte *)POINTER_MAX)
+
+/* alignment helpers, alignment must be power of 2 */
+#define ALIGNED(x, alignment) ((((ptr_uint_t)x) & ((alignment)-1)) == 0)
+#define ALIGN_FORWARD(x, alignment) \
+    ((((ptr_uint_t)x) + ((alignment)-1)) & (~((alignment)-1)))
+#define ALIGN_FORWARD_UINT(x, alignment) \
+    ((((uint)x) + ((alignment)-1)) & (~((alignment)-1)))
+#define ALIGN_BACKWARD(x, alignment) (((ptr_uint_t)x) & (~((alignment)-1)))
+#define PAD(length, alignment) (ALIGN_FORWARD((length), (alignment)) - (length))
+#define ALIGN_MOD(addr, size, alignment) \
+    ((((ptr_uint_t)addr)+(size)-1) & ((alignment)-1))
+#define CROSSES_ALIGNMENT(addr, size, alignment) \
+    (ALIGN_MOD(addr, size, alignment) < (size)-1)
+/* number of bytes you need to shift addr forward so that it's !CROSSES_ALIGNMENT */
+#define ALIGN_SHIFT_SIZE(addr, size, alignment) \
+    (CROSSES_ALIGNMENT(addr, size, alignment) ?   \
+        ((size) - 1 - ALIGN_MOD(addr, size, alignment)) : 0)
+
+#define IS_POWER_OF_2(x) ((x) == 0 || ((x) & ((x)-1)) == 0)
+
+/* bitmap_t operations */
+
+/* Current implementation uses integers representing 32 bits */
+typedef uint bitmap_element_t;
+typedef bitmap_element_t bitmap_t[];
+
+/* Note that we have some bitmap operations in linux/signal.c for
+ *  kernel version of sigset_t as well as in
+ *  win32/ntdll.c:tls_{alloc,free} which could use some of these
+ *  facilities, but for now we leave those as more OS specific
+ */
+
+#define BITMAP_DENSITY   32
+#define BITMAP_MASK(i)   (1 << ((i) % BITMAP_DENSITY))
+#define BITMAP_INDEX(i)  ((i) / BITMAP_DENSITY)
+#define BITMAP_NOT_FOUND ((uint)-1)
+
+/* bitmap_t primitives */
+
+/* TODO: could use BT for bit test, BTS/BTR for set/clear, and BSF for
+ *  bit scan forward.  See /usr/src/linux-2.4/include/asm/bitops.h for
+ *  an all assembly implementation.  Here we stick to plain C.
+ */
+
+/* returns non-zero value if bit is set */
+static inline bool
+bitmap_test(bitmap_t b, uint i)
+{
+    return b[BITMAP_INDEX(i)] & BITMAP_MASK(i);
+}
+
+static inline void
+bitmap_set(bitmap_t b, uint i)
+{
+    b[BITMAP_INDEX(i)] |= BITMAP_MASK(i);
+}
+
+static inline void
+bitmap_clear(bitmap_t b, uint i)
+{
+    b[BITMAP_INDEX(i)] &= ~BITMAP_MASK(i);
+}
+
+/* bitmap_size is number of bits in the bitmap_t */
+void bitmap_initialize_free(bitmap_t b, uint bitmap_size);
+uint bitmap_allocate_blocks(bitmap_t b, uint bitmap_size, uint request_blocks);
+void bitmap_free_blocks(bitmap_t b, uint bitmap_size, uint first_block, uint num_free);
+
+#ifdef DEBUG
+/* used only for ASSERTs */
+bool bitmap_are_reserved_blocks(bitmap_t b, uint bitmap_size, uint first_block, uint num_blocks);
+bool bitmap_check_consistency(bitmap_t b, uint bitmap_size, uint expect_free);
+#endif /* DEBUG */
+
+/* logging functions */
+/* use the following three defines to control the logging directory format */
+#define LOGDIR_MAX_NUM          1000
+#define LOGDIR_FORMAT_STRING    "%s.%03d"
+#define LOGDIR_FORMAT_ARGS(num) "dynamorio", num
+ /* longest message we would put in a log or messagebox 
+  * 512 is too short for internal exception w/ app + options + callstack
+  */
+#define MAX_LOG_LENGTH IF_X64_ELSE(1280, 768)
+#define MAX_LOG_LENGTH_MINUS_ONE (MAX_LOG_LENGTH-1) /* for splitting long buffers */
+
+#ifdef DEBUG
+# define LOG(file, mask, level, ...) do {        \
+  if (stats != NULL &&                           \
+      stats->loglevel >= (level) &&              \
+      (stats->logmask & (mask)) != 0)            \
+      print_log(file, mask, level, __VA_ARGS__); \
+  } while (0)
+  /* use DOELOG for customer visible logging. statement can be a {} block */
+# define DOELOG(level, mask, statement) do {    \
+  if (stats != NULL &&                          \
+      stats->loglevel >= (level) &&             \
+      (stats->logmask & (mask)) != 0)           \
+    statement                                   \
+  } while (0)
+# define DOCHECK(level, statement) do {     \
+  if (DYNAMO_OPTION(checklevel) >= (level)) \
+    statement                               \
+  } while (0)
+# ifdef INTERNAL
+#  define DOLOG DOELOG
+#  define LOG_DECLARE(declaration) declaration
+# else
+#  define DOLOG(level, mask, statement)
+#  define LOG_DECLARE(declaration)
+# endif /* INTERNAL */
+# define THREAD ((dcontext == NULL) ? INVALID_FILE : \
+                 ((dcontext == GLOBAL_DCONTEXT) ? main_logfile : dcontext->logfile))
+# define THREAD_GET get_thread_private_logfile()
+# define GLOBAL main_logfile
+#else  /* !DEBUG */
+/* make use of gcc macro varargs, LOG's args may be ifdef DEBUG */
+/* the macro is actually ,fmt... but C99 requires one+ argument which we just strip */
+# define LOG(file, mask, level, ...)
+# define DOLOG(level, mask, statement)
+# define DOELOG DOLOG
+# define LOG_DECLARE(declaration)
+# define DOCHECK(level, statement) /* nothing */
+#endif
+void print_log(file_t logfile, uint mask, uint level, char *fmt, ...);
+void print_file(file_t f, char *fmt, ...);
+
+char *memprot_string(uint prot);
+
+char * double_strchr(char *string, char c1, char c2);
+#ifndef WINDOWS
+const char * double_strrchr(const char *string, char c1, char c2);
+#else
+/* double_strrchr() defined in win32/inject_shared.h */
+size_t wcsnlen(const wchar_t *str, size_t max);
+#endif
+
+bool is_region_memset_to_char(byte *addr, size_t size, byte val);
+
+/* calculates intersection of two regions defined as open ended intervals
+ * [region1_start, region1_start + region1_len) \intersect
+ * [region2_start, region2_start + region2_len)
+ * 
+ * intersection_len is set to 0 if the regions do not overlap
+ * otherwise returns the intersecting region as
+ * [intersection_start, intersection_start + intersection_len)
+ */
+void
+region_intersection(app_pc *intersection_start /* OUT */,
+                    size_t *intersection_len /* OUT */,
+                    const app_pc region1_start, size_t region1_len,
+                    const app_pc region2_start, size_t region2_len);
+
+int check_filter(const char *filter, const char *short_name);
+int check_filter_with_wildcards(const char *filter, const char *short_name);
+
+typedef enum {
+    BASE_DIR,   /* Only creates directory specified in env */
+    PROCESS_DIR /* Creates a process subdir off of base (e.g. dynamorio.000) */
+} log_dir_t;
+void enable_new_log_dir(void); /* enable creating a new base logdir (for a fork, e.g.) */
+void create_log_dir(int dir_type);
+bool get_log_dir(log_dir_t dir_type, char *buffer, uint *buffer_length);
+file_t open_log_file(const char *basename, char *finalname_with_path, uint maxlen);
+file_t get_thread_private_logfile(void);
+bool get_unique_logfile(const char *file_type, char *filename_buffer, uint maxlen, bool open_directory, file_t *file);
+const char *get_app_name_for_path(void);
+const char *get_short_name(const char *exename);
+
+/* Self-protection: we can't use pragmas in local scopes so no convenient way to
+ * place the do_once var elsewhere than .data.  Since it's only written once we
+ * go ahead and unprotect here.  Even if we have dozens of these (there aren't
+ * that many in release builds currently) it shouldn't hurt us.
+ * FIXME: this means that if the protection routines call a routine that has
+ * a do-once, we have a deadlock!  Could switch to a recursive lock.
+ */
+#define DO_ONCE(statement) {                                    \
+        /* no mutual exclusion, should be used only with logging */      \
+        static int do_once = 0;                                 \
+        if (!do_once) {                                         \
+                SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);\
+                do_once = 1;                                    \
+                SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);  \
+                statement;                                      \
+        }                                                       \
+}
+
+/* This is more heavy-weight and includes its own static mutex 
+ * The counter is only incremented if it is less than the threshold
+ */
+/* Self-protection case 8075: can't use pragma at local scope.  We put
+ * the burden on the caller to make the do_threshold_cur in .data
+ * writable.  For do_threshold_mutex, even if it's writable at the
+ * macro site, {add,remove}_process_lock will crash on adjacent
+ * entries in the lock list (and an attempt there to unprot .data will
+ * deadlock as the datasec lock is acquired and hits the same
+ * unprot!).  So we use a single global mutex in debug builds.  There
+ * aren't currently any uses of this macro that will be hurt by this
+ * serialization so we could also do it in release builds.
+ */
+#ifdef DEADLOCK_AVOIDANCE
+extern mutex_t do_threshold_mutex;
+# define DECLARE_THRESHOLD_LOCK(section) /* nothing */
+#else
+# define DECLARE_THRESHOLD_LOCK(section) \
+    static mutex_t do_threshold_mutex VAR_IN_SECTION(section) \
+        = INIT_LOCK_FREE(do_threshold_mutex);
+#endif
+/* The section argument is our support for the user wrapping entire
+ * function in a separate section, which for gcc also requires
+ * annotating each var declaration.
+ */
+#define DO_THRESHOLD_SAFE(threshold, section, statement_below, statement_after) {\
+        DECLARE_THRESHOLD_LOCK(section)                                 \
+        static uint do_threshold_cur VAR_IN_SECTION(section) = 0;       \
+        mutex_lock(&do_threshold_mutex);                                \
+        if (do_threshold_cur < threshold) {                             \
+            do_threshold_cur++;                                         \
+            mutex_unlock(&do_threshold_mutex);                          \
+            statement_below;                                            \
+        } else {                                                        \
+            mutex_unlock(&do_threshold_mutex);                          \
+            statement_after; /* or at */                                \
+        }                                                               \
+}
+
+/* TRY/EXCEPT and TRY/FINALLY 
+ * usage notes:
+ * any automatic variables that you want to use in the except block
+ * should be declared as volatile - see case 5891
+ * 
+ * Note we do not have language support - do not use return within a
+ * TRY block!  otherwise we can't rollback or execute FINALLY.
+ *
+ * Note we do not support filters in EXCEPT statements so the
+ * innermost handler will be called.  Also finally blocks are not
+ * implemented (FIXME: we would have to unwind all nested finally blocks
+ * before the EXCEPT block).
+ *
+ * Note no locks should be grabbed within a TRY/EXCEPT block (FIXME:
+ * until we have FINALLY support to release them).
+ *
+ * (tip: compile your TRY blocks first outside of this macro for
+ * easier line matching and debugging)
+ */
+
+#define TRY_EXCEPT_ALLOW_NO_DCONTEXT(dcontext, try_statement, except_statement) do { \
+    if ((dcontext) != NULL && (dcontext) != GLOBAL_DCONTEXT) {                       \
+        TRY(dcontext, try_statement, EXCEPT(dcontext, except_statement));            \
+    } else {                                                                         \
+        try_statement;                                                               \
+    }                                                                                \
+} while (0)
+
+#define TRY_EXCEPT(dcontext, try_statement, except_statement)           \
+    TRY(dcontext, try_statement, EXCEPT(dcontext, except_statement))
+
+#define TRY_FINALLY(dcontext, try_statement, finally_statement)         \
+    TRY(dcontext, try_statement, FINALLY(dcontext, except_statement))
+
+/* internal versions */
+#define TRY(cur_dcontext, try_statement, except_or_finally) do {        \
+    try_except_context_t try__state;                                    \
+    /* must be current thread -> where we'll fault */                   \
+    ASSERT(cur_dcontext == get_thread_private_dcontext());              \
+    try__state.prev_context = cur_dcontext->try_except_state;           \
+    cur_dcontext->try_except_state = &try__state;                       \
+    if (DR_SETJMP(&try__state.context) == 0) {  /* TRY block */         \
+        try_statement                                                   \
+        /* make sure there is no return in try_statement */             \
+        POP_TRY_BLOCK(cur_dcontext, try__state);                        \
+    }                                                                   \
+    except_or_finally                                                   \
+    /* EXCEPT or FINALLY will POP_TRY_BLOCK on exception */             \
+} while (0)
+
+/* implementation notes: */
+/* FIXME: it is more secure yet not as flexible to use a scheme
+ * like the Exception Tables in the Linux kernel where a static
+ * mapping from a faulting PC to a fixup code (in
+ * exception_table_entry) can be kept in read-only memory.  That
+ * scheme works really well for tight assembly, and the fast path
+ * is somewhat faster than the 10 instructions in dr_setjmp().
+ * Note however, that a return address on our thread stack is just
+ * as vulnerable, so the security advantage is minor.
+ * The tighter scheme also makes it hard to cover up faults that are
+ * at unexpected instructions in a block.
+ */
+
+
+/* no filters
+ * FIXME: we may want filters in debug builds to make sure we can
+ * detect the proper EXCEPT condition (need to register in the TRY).
+ * Alternatively, should match against a list of instructions that are
+ * the only ones known to possibly fail.
+ *
+ * Note we also don't provide any access to the exception context,
+ * since we don't plan on recovering at the fault point (which a
+ * filter may recommend).
+ */
+
+/* Only called within a TRY block that contains the proper try__state */
+#define EXCEPT(cur_dcontext, statement) else { /* EXCEPT */             \
+        /* a failure in the EXCEPT should be thrown higher up */        \
+        /* rollback first */                                            \
+        POP_TRY_BLOCK(cur_dcontext, try__state);                        \
+        statement;                                                      \
+        /* FIXME: stop unwinding */                                     \
+    }
+
+/* FIXME: should be called only nested within another TRY/EXCEPT
+ * block.  (We don't support __leave so there is no other use).  If it
+ * was called not nested in an EXCEPT handler, we can't just hide
+ * there was an exception at all, otherwise this will change behavior
+ * if it is ever nested.
+ */
+/* Only called within a TRY block */
+/* NYI */
+#define FINALLY(cur_dcontext, statement) /* ALWAYS */ {                 \
+        ASSERT_NOT_IMPLEMENTED(false);                                  \
+        if (cur_dcontext->unwinding_exception) {                        \
+            /* only on exception we have to POP here */                 \
+            /* normal execution would have already POPped */            \
+                                                                        \
+            /* pop before executing finally statement */                \
+            /* so an exception in it is delivered to the */             \
+            /* previous handler */                                      \
+            /* only parent TRY block has proper try__state */           \
+            POP_TRY_BLOCK(cur_dcontext, try__state);                    \
+        }                                                               \
+        ASSERT(cur_dcontext->try_except_state != NULL                   \
+               && "try/finally should be nested in try/except");        \
+        /* executed for both normal execution, or exception */          \
+        statement;                                                      \
+        if (cur_dcontext->unwinding_exception) {                        \
+           /* FIXME: on nested exception must keep UNWINDing */         \
+           /* and give control to the previous nested handler */        \
+           /* until an EXCEPT handler resumes to normal execution */    \
+           /* we don't keep any exception context */                    \
+           ASSERT_NOT_IMPLEMENTED(false);                               \
+        }                                                               \
+    }
+
+/* internal helper */
+#define POP_TRY_BLOCK(cur_dcontext, state)                          \
+        ASSERT(cur_dcontext->try_except_state == &state);           \
+        cur_dcontext->try_except_state =                            \
+                  cur_dcontext->try_except_state->prev_context;
+
+enum {LONGJMP_EXCEPTION = 1};
+/* the return value of setjmp() returned on exception (or unwinding) */
+
+/* volatile to ensure the compiler doesn't completely skip a READ */
+#define PROBE_READ_PC(pc) ((*(volatile char *)(pc)))
+#define PROBE_WRITE_PC(pc) ATOMIC_ADD_PTR(volatile char*, pc, 0)
+/* FIXME: while handling a read exception thread stack expansion in
+ * other threads may lose its guard page.  Since current thread won't
+ * know if it is ok to expand, therefore the stacks won't grow any
+ * further.  See MSDN: IsBadReadPtr(), We may want to mark back any
+ * PAGE_GUARD faults before we handle them in our EXCEPT block. For
+ * most purposes it is unlikely to be an issue that is not already an
+ * app bug causing us to touch these.  Our LOCKed ADD is somewhat
+ * better than IsBadWritePtr() but is best not to have to use it.
+ */
+
+#include "stats.h"
+
+/* Use to shut up the compiler about an unused variable when the
+ * alternative is a painful modification of more src code. Our
+ * standard is to use this macro just after the variable is declared
+ * and to use it judiciously.
+ */
+#define UNUSED_VARIABLE(pv) {void *unused_pv = (void*) pv; unused_pv = NULL; }
+
+/* Both release and debug builds share these common stats macros */
+/* If -no_global_rstats, all values will be 0, so user does not have to
+ * use DO_GLOBAL_STATS or check runtime option.
+ */
+#define GLOBAL_STAT(stat) stats->stat##_pair.value
+/* explicit macro for addr so no assumptions on GLOBAL_STAT being lvalue */
+#define GLOBAL_STAT_ADDR(stat) &(stats->stat##_pair.value)
+#define DO_GLOBAL_STATS(statement) do {   \
+        if (GLOBAL_STATS_ON()) {             \
+           statement;                        \
+        }                                    \
+    } while (0)
+
+#ifdef X64
+# define XSTATS_ATOMIC_INC(var) ATOMIC_INC(int64, var)
+# define XSTATS_ATOMIC_DEC(var) ATOMIC_DEC(int64, var)
+# define XSTATS_ATOMIC_ADD(var, val) ATOMIC_ADD(int64, var, val)
+# define XSTATS_ATOMIC_MAX(max, cur) ATOMIC_MAX(int64, max, cur)
+# define XSTATS_ATOMIC_ADD_EXCHANGE(var, val) atomic_add_exchange_int64(var, val)
+#else
+# define XSTATS_ATOMIC_INC(var) ATOMIC_INC(int, var)
+# define XSTATS_ATOMIC_DEC(var) ATOMIC_DEC(int, var)
+# define XSTATS_ATOMIC_ADD(var, val) ATOMIC_ADD(int, var, val)
+# define XSTATS_ATOMIC_MAX(max, cur) ATOMIC_MAX(int, max, cur)
+# define XSTATS_ATOMIC_ADD_EXCHANGE(var, val) atomic_add_exchange_int(var, val)
+#endif
+
+/* XSTAT_* macros are pointed at by either STATS_* or RSTATS_*
+ * XSTAT_* should not be called directly outside this file
+ */
+#define XSTATS_INC_DC(dcontext, stat) do {                              \
+        DO_THREAD_STATS(dcontext, THREAD_STAT(dcontext, stat) += 1);    \
+        DO_GLOBAL_STATS(XSTATS_ATOMIC_INC(GLOBAL_STAT(stat)));          \
+    } while (0)
+#define XSTATS_INC(stat)                                                 \
+        XSTATS_WITH_DC(stats_inc__dcontext,                              \
+                      XSTATS_INC_DC(stats_inc__dcontext, stat)) 
+
+#define XSTATS_DEC_DC(dcontext, stat) do {                              \
+        DO_THREAD_STATS(dcontext, THREAD_STAT(dcontext, stat) -= 1);    \
+        DO_GLOBAL_STATS(XSTATS_ATOMIC_DEC(GLOBAL_STAT(stat)));          \
+    } while (0)
+#define XSTATS_DEC(stat)                                                 \
+        XSTATS_WITH_DC(stats_dec__dcontext,                              \
+                      XSTATS_DEC_DC(stats_dec__dcontext, stat)) 
+
+#define XSTATS_ADD_DC(dcontext, stat, value) do {                                       \
+        stats_int_t stats_add_dc__value = (stats_int_t) (value);                        \
+        ASSERT_TRUNCATE(stats_add_dc__value, stats_int_t, value);                       \
+        DO_THREAD_STATS(dcontext, THREAD_STAT(dcontext, stat) += stats_add_dc__value);  \
+        DO_GLOBAL_STATS(XSTATS_ATOMIC_ADD(GLOBAL_STAT(stat), stats_add_dc__value));     \
+    } while (0)
+#define XSTATS_ADD(stat, value)                                             \
+        XSTATS_WITH_DC(stats_add__dcontext,                                 \
+                       XSTATS_ADD_DC(stats_add__dcontext, stat, value)) 
+#define XSTATS_SUB(stat, value) XSTATS_ADD(stat, -(stats_int_t)(value))
+#define XSTATS_ADD_ASSIGN_DC(dcontext, stat, var, value) do {               \
+        stats_int_t stats_add_assign_dc__value = (stats_int_t) (value);     \
+        ASSERT_TRUNCATE(stats_add_assign_dc__value, stats_int_t, value);    \
+        DO_THREAD_STATS(dcontext,                                           \
+            THREAD_STAT(dcontext, stat) += stats_add_assign_dc__value);     \
+        /* would normally DO_GLOBAL_STATS(), but need to assign var */      \
+        var = XSTATS_ATOMIC_ADD_EXCHANGE(&GLOBAL_STAT(stat),                \
+                                         stats_add_assign_dc__value);       \
+    } while (0)
+#define XSTATS_INC_ASSIGN_DC(dcontext, stat, var)                         \
+        XSTATS_ADD_ASSIGN_DC(dcontext, stats, var, 1)
+#define XSTATS_ADD_ASSIGN(stat, var, value)                                  \
+        XSTATS_WITH_DC(stats_add_assign__dcontext,                           \
+            XSTATS_ADD_ASSIGN_DC(stats_add_assign__dcontext, stat, var, value))
+#define XSTATS_INC_ASSIGN(stat, var) XSTATS_ADD_ASSIGN(stat, var, 1)
+
+
+#define XSTATS_MAX_HELPER(dcontext, stat, global_val, thread_val) do {   \
+        DO_THREAD_STATS(dcontext, {                                      \
+            stats_int_t stats_max_helper__value;                         \
+            stats_max_helper__value = (thread_val);                      \
+            if (THREAD_STAT(dcontext, stat) < stats_max_helper__value)   \
+                THREAD_STAT(dcontext, stat) = stats_max_helper__value;   \
+        });                                                              \
+        DO_GLOBAL_STATS({                                                \
+            XSTATS_ATOMIC_MAX(GLOBAL_STAT(stat), global_val);            \
+        });                                                              \
+    } while (0)
+#define XSTATS_MAX_DC(dcontext, stat_max, stat_cur)                       \
+        XSTATS_MAX_HELPER(dcontext, stat_max, GLOBAL_STAT(stat_cur),      \
+                         THREAD_STAT(dcontext, stat_cur))
+#define XSTATS_PEAK_DC(dcontext, stat)                                 \
+        XSTATS_MAX_DC(dcontext, peak_##stat, stat)
+#define XSTATS_MAX(stat_max, stat_cur)                                    \
+        XSTATS_WITH_DC(stats_max__dcontext,                               \
+                      XSTATS_MAX_DC(stats_max__dcontext, stat_max, stat_cur))
+#define XSTATS_TRACK_MAX(stats_track_max, val) do {                      \
+        stats_int_t stats_track_max__value = (stats_int_t) (val);        \
+        ASSERT_TRUNCATE(stats_track_max__value, stats_int_t, val);       \
+        XSTATS_WITH_DC(stats_track_max__dcontext,                        \
+                      XSTATS_MAX_HELPER(stats_track_max__dcontext,       \
+                                       stats_track_max,                  \
+                                       stats_track_max__value,           \
+                                       stats_track_max__value));         \
+    } while (0)
+#define XSTATS_PEAK(stat)                                                 \
+        XSTATS_WITH_DC(stats_peak__dcontext,                              \
+                      XSTATS_PEAK_DC(stats_peak__dcontext, stat))
+
+
+#define XSTATS_ADD_MAX_DC(dcontext, stat_max, stat_cur, value) do {           \
+        stats_int_t stats_add_max__temp;                                      \
+        XSTATS_ADD_ASSIGN_DC(dcontext, stat_cur, stats_add_max__temp, value); \
+        XSTATS_MAX_HELPER(dcontext, stat_max, stats_add_max__temp,            \
+                         THREAD_STAT(dcontext, stat_cur));                    \
+    } while (0) 
+#define XSTATS_ADD_MAX(stat_max, stat_cur, value)                             \
+        XSTATS_WITH_DC(stats_add_max__dcontext,                               \
+                      XSTATS_ADD_MAX_DC(stats_add_max__dcontext, stat_max,    \
+                                       stat_cur, value))
+#define XSTATS_ADD_PEAK_DC(dcontext, stat, value)                          \
+        XSTATS_ADD_MAX_DC(dcontext, peak_##stat, stat, value)
+#define XSTATS_ADD_PEAK(stat, value)                                          \
+        XSTATS_WITH_DC(stats_add_peak__dcontext,                              \
+                      XSTATS_ADD_PEAK_DC(stats_add_peak__dcontext, stat, value))
+
+
+#define XSTATS_RESET_DC(dcontext, stat) do {                            \
+        DO_THREAD_STATS(dcontext, THREAD_STAT(dcontext, stat) = 0);     \
+        DO_GLOBAL_STATS(GLOBAL_STAT(stat) = 0);                         \
+    } while (0)
+#define XSTATS_RESET(stat)                                               \
+        XSTATS_WITH_DC(stats_reset__dcontext,                            \
+                      XSTATS_RESET_DC(stats_reset__dcontext, stat)) 
+
+/* common to both release and debug build */
+#define RSTATS_INC XSTATS_INC
+#define RSTATS_DEC XSTATS_DEC
+#define RSTATS_ADD XSTATS_ADD
+#define RSTATS_SUB XSTATS_SUB
+#define RSTATS_ADD_PEAK XSTATS_ADD_PEAK
+
+#ifdef DEBUG
+#   define DODEBUG(statement) do { statement } while (0)
+#   define DEBUG_DECLARE(declaration) declaration
+#   define DOSTATS(statement) do { statement } while (0)
+    /* FIXME: move to stats.h */
+    /* Note : stats macros are called in places where it is not safe to hold any lock
+     * (such as special_heap_create_unit, others?), if ever go back to using a mutex 
+     * to protect the stats need to update such places
+     */  
+    /* global and thread local stats, can be used as lvalues, 
+     * not used if not DEBUG */
+    /* We assume below that all stats are aligned and thus reading and writing
+     * stats are atomic operations on Intel x86 */
+    /* In general should prob. be using stats_add_[peak, max] instead of
+     * stats_[peak/max] since they tie the adjustment of the stat to the 
+     * setting of the max, otherwise you're open to race conditions involving
+     * multiple threads adjusting the same stats and setting peak/max FIXME
+     */
+#   define GLOBAL_STATS_ON() (stats != NULL && INTERNAL_OPTION(global_stats))
+#   define THREAD_STAT(dcontext, stat)       \
+        (dcontext->thread_stats)->stat##_thread
+#   define THREAD_STATS_ON(dcontext)                                    \
+        (dcontext != NULL && INTERNAL_OPTION(thread_stats) &&           \
+         dcontext->thread_stats != NULL)
+#   define DO_THREAD_STATS(dcontext, statement) do {                    \
+        if (THREAD_STATS_ON(dcontext)) {                                \
+            statement;                                                  \
+        }                                                               \
+    } while (0)
+#   define XSTATS_WITH_DC(var, statement) do {                          \
+        dcontext_t *var = NULL;                                      \
+        if (INTERNAL_OPTION(thread_stats))                              \
+           var = get_thread_private_dcontext();                         \
+        statement;                                                      \
+    } while (0)
+
+#   define STATS_INC XSTATS_INC
+/* we'll expose more *_DC as we need them */
+#   define STATS_INC_DC XSTATS_INC_DC
+#   define STATS_DEC XSTATS_DEC
+#   define STATS_ADD XSTATS_ADD
+#   define STATS_SUB XSTATS_SUB
+#   define STATS_INC_ASSIGN XSTATS_INC_ASSIGN
+#   define STATS_ADD_ASSIGN XSTATS_ADD_ASSIGN
+#   define STATS_MAX XSTATS_MAX
+#   define STATS_TRACK_MAX XSTATS_TRACK_MAX
+#   define STATS_PEAK XSTATS_PEAK
+#   define STATS_ADD_MAX XSTATS_ADD_MAX
+#   define STATS_ADD_PEAK XSTATS_ADD_PEAK
+#   define STATS_RESET XSTATS_RESET
+
+#else
+#   define DODEBUG(statement)
+#   define DEBUG_DECLARE(declaration)
+#   define DOSTATS(statement)
+#   define THREAD_STATS_ON(dcontext) false
+#   define XSTATS_WITH_DC(var, statement) statement
+#   define DO_THREAD_STATS(dcontext, statement) /* nothing */
+#   define GLOBAL_STATS_ON() (stats != NULL && DYNAMO_OPTION(global_rstats))
+
+/* Would be nice to catch incorrect usage of STATS_INC on a release-build
+ * stat: if rename release vars, have to use separate GLOBAL_RSTAT though.
+ */
+#   define STATS_INC(stat) /* nothing */
+#   define STATS_INC_DC(dcontext, stat) /* nothing */
+#   define STATS_DEC(stat) /* nothing */
+#   define STATS_ADD(stat, value) /* nothing */
+#   define STATS_SUB(stat, value) /* nothing */
+#   define STATS_INC_ASSIGN(stat, var) /* nothing */
+#   define STATS_ADD_ASSIGN(stat, var, value) /* nothing */
+#   define STATS_MAX(stat_max, stat_cur) /* nothing */
+#   define STATS_TRACK_MAX(stats_track_max, val) /* nothing */
+#   define STATS_PEAK(stat) /* nothing */
+#   define STATS_ADD_MAX(stat_max, stat_cur, value) /* nothing */
+#   define STATS_ADD_PEAK(stat, value) /* nothing */
+#   define STATS_RESET(stat) /* nothing */
+#endif /* DEBUG */
+
+#ifdef KSTATS
+# define DOKSTATS(statement) do { statement } while (0)
+
+/* The proper use is most commonly KSTART(name)/KSTOP(name), or
+ * occasionally KSTART(name)/KSWITCH(better_name)/KSTOP(name), and in
+ * ignorable cases KSTART(name)/KSTOP_NOT_PROPAGATED(name)
+ */
+/* starts a timer */
+# define KSTART(name) KSTAT_THREAD(name, kstat_start_var(ks, pv))
+
+/* makes sure we're matching start/stop */
+# define KSTOP(name) KSTAT_THREAD(name, kstat_stop_matching_var(ks, pv))
+
+/* modifies the variable against which this path should be counted */
+# define KSWITCH(name) KSTAT_THREAD(name, kstat_switch_var(ks, pv))
+
+/* allow mismatched start/stop - for use with KSWITCH */
+# define KSTOP_NOT_MATCHING(name)                               \
+    KSTAT_THREAD_NO_PV_START(get_thread_private_dcontext())     \
+        ASSERT(ks->depth > 2 && "stop_not_matching not allowed to clear kstack"); \
+        kstat_stop_not_matching_var(ks, ignored);               \
+    KSTAT_THREAD_NO_PV_END()
+
+/* rewind the callstack exiting multiple entries - for exception cases */
+# define KSTOP_REWIND(name) KSTAT_THREAD(name, kstat_stop_rewind_var(ks, pv))
+# define KSTOP_REWIND_UNTIL(name) KSTAT_THREAD(name, kstat_stop_longjmp_var(ks, pv))
+
+/* simultanously switch to a path and stop timer */
+# define KSWITCH_STOP(name) KSTAT_THREAD(name, {        \
+    kstat_switch_var(ks, pv);                           \
+    kstat_stop_not_matching_var(ks, ignored);           \
+})
+
+/* simultanously switch to a path and stop timer w/o propagating to parent */
+# define KSWITCH_STOP_NOT_PROPAGATED(name) KSTAT_THREAD(name, { \
+    timestamp_t ignore_cum;                                       \
+    kstat_switch_var(ks, pv);                                   \
+    kstat_stop_not_propagated_var(ks, ignored, &ignore_cum);    \
+})
+
+/* do not propagate subpath time to parent */
+# define KSTOP_NOT_MATCHING_NOT_PROPAGATED(name) KSTAT_THREAD(name,  { \
+    timestamp_t ignore_cum;                                              \
+    ASSERT(ks->depth > 2 && "stop_not_matching_np not allowed to clear kstack"); \
+    kstat_stop_not_propagated_var(ks, pv, &ignore_cum);                \
+})
+
+/* do not propagate subpath time to parent */
+# define KSTOP_NOT_PROPAGATED(name) KSTAT_THREAD(name,  {       \
+    timestamp_t ignore_cum;                                       \
+    DODEBUG({if (ks->node[ks->depth - 1].var != pv)             \
+        kstats_dump_stack(cur_dcontext);});                     \
+    ASSERT(ks->node[ks->depth - 1].var == pv                    \
+          && "stop not matching TOS");                          \
+    kstat_stop_not_propagated_var(ks, pv, &ignore_cum);         \
+})
+
+
+/* in some cases we do need to pass a dcontext for another thread */
+/* since get_thread_private_dcontext() may be expensive we can pass a dcontext
+ * to this version of the macro, however we should then use this everywhere
+ * to have comparable overheads
+ */
+# define KSTART_DC(dc, name)                                    \
+        KSTAT_OTHER_THREAD(dc, name, kstat_start_var(ks, pv))
+# define KSTOP_DC(dc, name)                                     \
+        KSTAT_OTHER_THREAD(dc, name, kstat_stop_matching_var(ks, pv))
+# define KSTOP_NOT_MATCHING_DC(dc, name)                        \
+    KSTAT_THREAD_NO_PV_START(dc)                                \
+        kstat_stop_not_matching_var(ks, ignored);               \
+    KSTAT_THREAD_NO_PV_END()
+# define KSTOP_REWIND_DC(dc, name) KSTAT_OTHER_THREAD(dc, name, kstat_stop_rewind_var(ks, pv))
+#else  /* !KSTATS */
+# define DOKSTATS(statement)    /* nothing */
+# define KSTART(name)           /* nothing */
+# define KSWITCH(name)          /* nothing */
+# define KSWITCH_STOP(name)     /* nothing */
+# define KSWITCH_STOP_NOT_PROPAGATED(name) /* nothing */
+# define KSTOP_NOT_MATCHING_NOT_PROPAGATED(name) /* nothing */
+# define KSTOP_NOT_PROPAGATED(name) /* nothing */
+# define KSTOP_NOT_MATCHING(name)   /* nothing */
+# define KSTOP(name)            /* nothing */
+# define KSTOP_REWIND(name)     /* nothing */
+# define KSTOP_REWIND_UNTIL(name) /* nothing */
+
+# define KSTART_DC(dc, name)    /* nothing */
+# define KSTOP_DC(dc, name)     /* nothing */
+# define KSTOP_NOT_MATCHING_DC(dc, name) /* nothing */
+# define KSTOP_REWIND_DC(dc, name) /* nothing */
+#endif /* KSTATS */
+
+#ifdef INTERNAL
+#  define DODEBUG_ONCE(statement) DODEBUG(DO_ONCE(statement))
+#  define DOLOG_ONCE(level, mask, statement) DOLOG(level, mask, DO_ONCE(statement))
+#else
+#  define DODEBUG_ONCE(statement) /* nothing */
+#  define DOLOG_ONCE(level, mask, statement) /* nothing */
+#endif /* INTERNAL */
+
+#define MAX_FP_STATE_SIZE (512+16) /* maximum buffer size plus alignment */
+/* for convenice when want to save floating point state around a statement
+ * that contains ifdefs, need to be used at same nesting depth */
+/* fpstate_junk is used so that this macro can be used before, or in the
+ * middle of a list of declerations without bothering the compiler or creating
+ * a new nesting block */
+
+/* keep in mind that each use of PRESERVE_FLOATING_POINT_STATE takes
+ * 512 bytes - avoid nesting uses
+ */
+
+#define PRESERVE_FLOATING_POINT_STATE_START()                   \
+    {                                                           \
+    byte  fpstate_buf[MAX_FP_STATE_SIZE];                       \
+    byte *fpstate = (byte*)ALIGN_FORWARD(fpstate_buf, 16);      \
+    size_t fpstate_junk = proc_save_fpstate(fpstate)
+
+#define PRESERVE_FLOATING_POINT_STATE_END()                     \
+    proc_restore_fpstate(fpstate);                              \
+    fpstate_junk = 0xdead;                                      \
+    }
+
+#define PRESERVE_FLOATING_POINT_STATE(statement) do {           \
+    PRESERVE_FLOATING_POINT_STATE_START();                      \
+    statement;                                                  \
+    PRESERVE_FLOATING_POINT_STATE_END();                        \
+ } while (0)
+
+/* argument counting, http://gcc.gnu.org/ml/gcc/2000-09/msg00604.html */
+#define ARGUMENT_COUNTER(...) _ARGUMENT_COUNT1( , ## __VA_ARGS__)
+#define _ARGUMENT_COUNT1(...) _ARGUMENT_COUNT2(__VA_ARGS__, 5,4,3,2,1,0)
+#define _ARGUMENT_COUNT2(_,x0,x1,x2,x3,x4, argc,...) argc
+
+/* write to the system event log facilities - using syslogd or EventLog */
+/* For true portability we have to get the message strings as well as 
+   the argument count out of the .mc files.  It will be nice to get them 
+   compile time type checked by way of having the format strings as 
+   static arguments for LOGging.  (As for the explicit number of substitions, 
+   we can easily count args with macros, but for now we'll at least have to 
+   lookup the number of arguments in events.mc.)
+*/
+#include "event_strings.h"
+#ifdef WINDOWS
+# include "events.h"
+#endif
+
+void
+report_dynamorio_problem(dcontext_t *dcontext, uint dumpcore_flag,
+                         app_pc exception_addr, app_pc report_ebp,
+                         const char *fmt, ...);
+
+void
+notify(syslog_event_type_t priority, bool internal, bool synch,
+            IF_WINDOWS_(uint message_id) uint substitution_nam, char *prefix, 
+            char *fmt, ...);
+
+#define SYSLOG_COMMON(synch, type, id, sub, ...) \
+    notify(type, false, synch, IF_WINDOWS_(MSG_##id) sub, #type, MSG_##id##_STRING, __VA_ARGS__)
+
+#define SYSLOG_INTERNAL_COMMON(synch, type, ...) \
+    notify(type, true, synch, IF_WINDOWS_(MSG_INTERNAL_##type) 0, #type, __VA_ARGS__)
+
+/* For security messages use passed in fmt string instead of eventlog fmt 
+ * string for LOG/stderr/msgbox to avoid breaking our regression suite,
+ * NOTE assumes actual id passed, not name of id less MSG_ (so can use array
+ * of id's in vmareas.c, another reason need seperate fmt string)
+ * FIXME : use events.mc string instead (SYSLOG_COMMON), breaks regression
+ * This is now used for out-of-memory as well, for the same reason --
+ * we should have a mechanism to strip the application name & pid prefix,
+ * then we could use the eventlog string.
+ */
+#define SYSLOG_CUSTOM_NOTIFY(type, id, sub, ...) \
+    notify(type, false, true, IF_WINDOWS_(id) sub, #type, __VA_ARGS__)
+
+#define SYSLOG(type, id, sub, ...) \
+    SYSLOG_COMMON(true, type, id, sub, __VA_ARGS__)
+#define SYSLOG_NO_OPTION_SYNCH(type, id, sub, ...) \
+    SYSLOG_COMMON(false, type, id, sub, __VA_ARGS__) 
+
+#ifdef INTERNAL
+# define SYSLOG_INTERNAL(type, ...) \
+      SYSLOG_INTERNAL_COMMON(true, type, __VA_ARGS__)
+# define SYSLOG_INTERNAL_NO_OPTION_SYNCH(type, ...) \
+      SYSLOG_INTERNAL_COMMON(false, type, __VA_ARGS__)
+#else
+# define SYSLOG_INTERNAL(...)
+# define SYSLOG_INTERNAL_NO_OPTION_SYNCH(...)
+#endif /* INTERNAL */
+
+/* for convenience */
+#define SYSLOG_INTERNAL_INFO(...) \
+    SYSLOG_INTERNAL(SYSLOG_INFORMATION, __VA_ARGS__)
+#define SYSLOG_INTERNAL_WARNING(...) \
+    SYSLOG_INTERNAL(SYSLOG_WARNING, __VA_ARGS__)
+#define SYSLOG_INTERNAL_ERROR(...) \
+    SYSLOG_INTERNAL(SYSLOG_ERROR, __VA_ARGS__)
+#define SYSLOG_INTERNAL_CRITICAL(...) \
+    SYSLOG_INTERNAL(SYSLOG_CRITICAL, __VA_ARGS__)
+
+#define SYSLOG_INTERNAL_INFO_ONCE(...) \
+    DODEBUG_ONCE(SYSLOG_INTERNAL_INFO(__VA_ARGS__))
+#define SYSLOG_INTERNAL_WARNING_ONCE(...) \
+    DODEBUG_ONCE(SYSLOG_INTERNAL_WARNING(__VA_ARGS__))
+#define SYSLOG_INTERNAL_ERROR_ONCE(...) \
+    DODEBUG_ONCE(SYSLOG_INTERNAL_ERROR(__VA_ARGS__))
+#define SYSLOG_INTERNAL_CRITICAL_ONCE(...) \
+    DODEBUG_ONCE(SYSLOG_INTERNAL_CRITICAL(__VA_ARGS__))
+
+/* FIXME, eventually want usage_error to also be external (may also eventually
+ * need non dynamic option synch form as well for usage errors while updating
+ * dynamic options), but lot of work to get all in eventlog and currently only 
+ * really triggered by internal options */
+/* FIXME : could leave out the asserts, this is a recoverable error */
+#define USAGE_ERROR(...)                                                     \
+    do {                                                                     \
+        SYSLOG_INTERNAL_ERROR(__VA_ARGS__);                                  \
+        ASSERT_NOT_REACHED();                                                \
+    } while (0)
+#define FATAL_USAGE_ERROR(id, sub, ...)                                      \
+    do {                                                                     \
+        /* synchronize dynamic options for dumpcore_mask */                  \
+        synchronize_dynamic_options();                                       \
+        if (TEST(DUMPCORE_FATAL_USAGE_ERROR, DYNAMO_OPTION_NOT_STRING(dumpcore_mask)))  \
+            os_dump_core("fatal usage error");                               \
+        SYSLOG(SYSLOG_CRITICAL, id, sub,  __VA_ARGS__);                      \
+        os_terminate(NULL, TERMINATE_PROCESS);                               \
+    } while (0)
+#define OPTION_PARSE_ERROR(id, sub, ...)                                     \
+    do {                                                                     \
+        SYSLOG_NO_OPTION_SYNCH(SYSLOG_ERROR, id, sub, __VA_ARGS__);          \
+        DODEBUG(os_terminate(NULL, TERMINATE_PROCESS););                     \
+    } while (0)
+
+#ifdef DEBUG
+/* only for temporary tracing - do not leave in source, use make ugly to remind you */
+# define TRACELOG(level) LOG(GLOBAL, LOG_TOP, level, "%s:%d ", __FILE__, __LINE__)
+#else
+# define TRACELOG(level)
+#endif
+
+/* what-to-log bitmask values */
+/* N.B.: if these constants are changed, win32gui must also be changed!
+ * They are also duplicated in instrument.h -- too hard to get them to
+ * automatically show up in right place in header files for release.
+ */
+#define LOG_NONE           0x00000000
+#define LOG_STATS          0x00000001
+#define LOG_TOP            0x00000002
+#define LOG_THREADS        0x00000004
+#define LOG_SYSCALLS       0x00000008
+#define LOG_ASYNCH         0x00000010
+#define LOG_INTERP         0x00000020
+#define LOG_EMIT           0x00000040
+#define LOG_LINKS          0x00000080
+#define LOG_CACHE          0x00000100
+#define LOG_FRAGMENT       0x00000200
+#define LOG_DISPATCH       0x00000400
+#define LOG_MONITOR        0x00000800
+#define LOG_HEAP           0x00001000
+#define LOG_VMAREAS        0x00002000
+#define LOG_SYNCH          0x00004000
+#define LOG_MEMSTATS       0x00008000
+#define LOG_OPTS           0x00010000
+#define LOG_SIDELINE       0x00020000
+#define LOG_SYMBOLS        0x00040000
+#define LOG_RCT            0x00080000
+#define LOG_NT             0x00100000
+#define LOG_HOT_PATCHING   0x00200000
+#define LOG_HTABLE         0x00400000
+#define LOG_MODULEDB       0x00800000
+
+#define LOG_ALL_RELEASE    0x00e0ffff
+#define LOG_ALL            0x00ffffff
+
+#ifdef WINDOWS_PC_SAMPLE
+# define LOG_PROFILE       LOG_ALL
+#endif
+
+/* buffer size supposed to handle undecorated names like
+ * "kernel32!CreateFile" or "kernel32.dll!CreateProcess" or ranges as
+ * needed by print_symbolic_address() */
+#define MAXIMUM_SYMBOL_LENGTH 80
+
+#ifdef DEBUG
+/* given an array of size size of integers, computes and prints the
+ * min, max, mean, and stddev
+ */
+void print_statistics(int *data, int size);
+/* global and thread local STATS */
+void dump_global_stats(bool raw);
+void dump_thread_stats(dcontext_t *dcontext, bool raw);
+void stats_thread_init(dcontext_t *dcontext);
+void stats_thread_exit(dcontext_t *dcontext);
+void print_timestamp(file_t logfile);
+
+/* prints a symbolic name, or best guess of it into a caller provided buffer */
+void
+print_symbolic_address(app_pc tag, char *buf, int max_chars, bool exact_only);
+
+#endif /* DEBUG */
+
+bool
+under_internal_exception(void);
+
+enum {
+    DUMP_NO_QUOTING   = 0x01000, // no quoting for C string replay
+    DUMP_OCTAL        = 0x02000, // hex otherwise
+    DUMP_NO_CHARS     = 0x04000, // no printable characters 
+    DUMP_RAW          = 0x08000, // do not keep as a string
+    DUMP_DWORD        = 0x10000, // dump as 4-byte chunks
+    DUMP_ADDRESS      = 0x20000, // prepend address before each line of output
+    DUMP_APPEND_ASCII = 0x40000, // append printable ASCII after each line
+    DUMP_PER_LINE     = 0x000ff  // mask for bytes per line flag
+};
+#define DUMP_PER_LINE_DEFAULT 16
+
+/* dump buffer in a desired hex/octal/char combination */
+void dump_buffer_as_bytes (file_t logfile, void *buf, size_t len, int flags);
+
+bool
+is_readable_without_exception_try(byte *pc, size_t size);
+/* Note: this is OS independent version
+ * cf. is_readable_without_exception() and is_readable_without_exception_query_os()
+ */
+
+/* NOTE - uses try/except when possible, is_readable_without_exception otherwise. NOTE -
+ * like our other is_readable_without_exception routines this routine offers no guarantee
+ * that the string will remain valid after the call returns.  FIXME - may be worthwhile
+ * to extend this routine to copy the string into a buffer while in the try/using
+ * safe_read instead of making the caller do it. */
+bool
+is_string_readable_without_exception(char *str, size_t *str_length /* OPTIONAL OUT */);
+
+#ifdef DEBUG
+bool
+is_valid_xml_char(char c);
+#endif
+
+/* prints str, escaping out char or char seq. for a xml CDATA block */
+void
+print_xml_cdata(file_t f, const char *str);
+
+/* Reads the full file, returns it in a buffer and sets buf_len to the size
+ * of the buffer allocated on the specified heap, if successful.  Returns NULL
+ * and sets buf_len to 0 on failure.
+ */
+char *read_entire_file(const char *file, size_t *buf_len /* OUT */
+                       HEAPACCT(which_heap_t heap));
+
+/* returns false if we are too low on disk to create a file of desired size */
+bool
+check_low_disk_threshold(file_t f, uint64 new_file_size);
+
+/* MD5 */
+/* Note: MD5 is only 16 bytes in length, but it is usually used as a
+ * string, so each byte will result in 2 chars being used.
+ */
+#define MD5_BLOCK_LENGTH 64
+#define MD5_RAW_BYTES 16
+#define MD5_STRING_LENGTH  (2*MD5_RAW_BYTES)
+
+/* To compute the message digest of several chunks of bytes, declare
+ * an MD5Context structure, pass it to MD5Init, call MD5Update as
+ * needed on buffers full of bytes, and then call MD5Final, which will
+ * fill a supplied 16-byte array with the digest.
+ */
+struct MD5Context {
+    uint32 state[4];                         /* state */
+    uint64 count;                            /* number of bits, mod 2^64 */
+    unsigned char buffer[MD5_BLOCK_LENGTH];  /* input buffer */
+};
+
+void MD5Init(struct MD5Context *ctx);
+void MD5Update(struct MD5Context *ctx, const unsigned char *buf, size_t len);
+void MD5Final(unsigned char digest[16], struct MD5Context *ctx);
+
+#ifdef PROCESS_CONTROL
+bool get_md5_for_file(const char *file, char *hash_buf /* OUT */);
+#endif
+const char *get_application_md5(void);
+void get_md5_for_region(const byte *region_start, uint len, 
+                        unsigned char digest[MD5_RAW_BYTES] /* OUT */);
+bool md5_digests_equal(const byte digest1[MD5_RAW_BYTES],
+                       const byte digest2[MD5_RAW_BYTES]);
+
+void
+print_version_and_app_info(file_t file);
+
+#ifdef RETURN_STACK
+void print_return_stack(dcontext_t *dcontext);
+#endif
+
+/* returns a pseudo random number in [0, max_offset)
+ * Not crypto strong, and not thread safe!
+ */
+size_t
+get_random_offset(size_t max_offset);
+
+uint crc32(const char *buf, const uint len);
+void utils_init(void);
+void utils_exit(void);
+
+#ifdef NOLIBC
+# ifdef WINDOWS
+#  ifdef isprint
+#   undef isprint
+bool isprint(int c);
+#  endif
+
+#  ifdef isdigit
+#   undef isdigit
+bool isdigit(int c);
+#  endif
+# endif
+#endif
+
+#define isprint_fast(c) (((c) >= 0x20) && ((c) < 0x7f))
+#define isdigit_fast(c) (((c) >= '0') && ((c) <= '9'))
+
+#ifdef LINUX
+/* PR 251709 / PR 257565: avoid __ctype_b linking issues for standalone
+ * and start/stop clients.  We simply avoid linking with the locale code
+ * altogether.
+ */
+# ifdef isprint
+#  undef isprint
+# endif
+# ifdef isdigit
+#  undef isdigit
+# endif
+# define isprint isprint_HAS_LINK_ERRORS_USE_isprint_fast_INSTEAD
+# define isdigit isdigit_HAS_LINK_ERRORS_USE_isdigit_fast_INSTEAD
+/* to avoid calling isprint()/isdigit() and localization tables
+ * xref PR 251709 / PR 257565
+ */
+#endif
+
+/* a little utiliy for printing a float that is formed by dividing 2 ints,
+ * gives back high and low parts for printing, also supports percentages
+ * Usage : given a, b (uint[64]); d_int()==divide_uint64_print(); 
+ *         uint c, d tmp; parameterized on precision p and width w 
+ * note that %f is eqv. to %.6f, 3rd ex. is a percentage ex.
+ * "%.pf", a/(float)b => d_int(a, b, false, p, &c, &d);  "%u.%.pu", c,d
+ * "%w.pf", a/(float)b => d_int(a, b, false, p, &c, &d); "%(w-p)u.%.pu", c,d
+ * "%.pf%%", 100*(a/(float)b) => d_int(a, b, true, p, &c, &d); "%u.%.pu%%", c,d
+ */
+void
+divide_uint64_print(uint64 numerator, uint64 denominator, bool percentage, 
+                    uint precision, uint *top, uint *bottom);
+
+#if defined(DEBUG) || defined(INTERNAL) || defined(CLIENT_INTERFACE)
+/* for printing a float (can't use %f on windows with NOLIBC), NOTE: you must
+ * preserve floating point state to call this function!! 
+ * Usage : given double/float a; uint c, d and char *s tmp; dp==double_print
+ *         parameterized on precision p width w
+ * note that %f is eqv. to %.6f
+ * "%.pf", a => dp(a, p, &c, &d, &s) "%s%u.%.pu", s, c, d
+ * "%w.pf", a => dp(a, p, &c, &d, &s) "%s%(w-p)u.%.pu", s, c, d
+ */
+void
+double_print(double val, uint precision, uint *top, uint *bottom, char **sign);
+#endif /* DEBUG || INTERNAL */
+
+#ifdef CALL_PROFILE
+/* Max depth of call stack to maintain.
+ * We actually maintain DYNAMO_OPTION(prof_caller) depth.
+ */
+#define MAX_CALL_PROFILE_DEPTH 8
+void profile_callers(void);
+void profile_callers_exit(void);
+#endif
+
+#ifdef STANDALONE_UNIT_TEST
+
+/* an ASSERT replacement for use in unit tests */
+#define EXPECT(expr, expected) do {                             \
+    ptr_uint_t value_once = expr;                               \
+    EXPECT_RELATION_INTERNAL(#expr, value_once, ==, expected);  \
+} while (0)
+
+#define EXPECT_RELATION(expr, REL, expected) do {               \
+    ptr_uint_t value_once = expr;                               \
+    EXPECT_RELATION_INTERNAL(#expr, value_once, REL, expected); \
+} while (0)
+
+#define EXPECT_RELATION_INTERNAL(exprstr, value, REL, expected) do {    \
+    LOG(GLOBAL, LOG_ALL, 1, "%s = %d [expected "#REL" %d] %s\n",        \
+        exprstr, value_once, expected,                                  \
+        value_once REL expected ? "good" : "BAD");                      \
+    ASSERT_MESSAGE(exprstr, value_once REL expected);                   \
+} while (0)
+
+#define TESTRUN(test) do {                              \
+    test_number++;                                      \
+    print_file(STDERR,                             \
+               "Test %d: "#test":\n", test_number);     \
+    test;                                               \
+    print_file(STDERR,                             \
+               "\t"#test" [OK]\n");                     \
+} while (0)
+
+/* define this on top of each unit test, and of course unit_main()
+ * has to be declared.
+ * Note that unit_main() is
+ * responsible for calling standalone_init() and any other
+ * initialization routines as needed
+ */
+#define UNIT_TEST_MAIN                                  \
+uint test_number = 0;                                   \
+static int unit_main(void);                             \
+int main() {                                            \
+    print_file(STDERR, "%s:\n", __FILE__);         \
+    unit_main();                                        \
+    print_file(STDERR, "%d tests\n", test_number); \
+    print_file(STDERR, "%s: SUCCESS\n", __FILE__); \
+    return 0;                                           \
+}
+
+#endif /* STANDALONE_UNIT_TEST */
+
+/* internal strdup */
+char *dr_strdup(const char *str HEAPACCT(which_heap_t which));
+
+#ifdef WINDOWS
+/* Allocates a new char* (NOT a new wchar_t*) from a wchar_t* */
+char *
+dr_wstrdup(const wchar_t *str HEAPACCT(which_heap_t which));
+#endif
+
+/* Frees a char* string (NOT a wchar_t*) allocated via dr_strdup or
+ * dr_wstrdup that has not been modified since being copied!
+ */
+void
+dr_strfree(const char *str HEAPACCT(which_heap_t which));
+
+/* Merges two arrays, treating their elements as type void*.
+ * Allocates a new array on dcontext's heap for the result
+ * and returns it and its size.
+ */
+void
+array_merge(dcontext_t *dcontext, bool intersect /* else union */,
+            void **src1, uint src1_num, void **src2, uint src2_num,
+            /*OUT*/ void ***dst, /*OUT*/ uint *dst_num
+            HEAPACCT(which_heap_t which));
+
+#endif /* _UTILS_H_ */
diff --git a/core/vmareas.c b/core/vmareas.c
new file mode 100644
index 000000000..421d104b6
--- /dev/null
+++ b/core/vmareas.c
@@ -0,0 +1,10938 @@
+/* **********************************************************
+ * Copyright (c) 2002-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+
+/*
+ * vmareas.c - virtual memory executable areas
+ */
+
+#include "globals.h"
+
+/* all of this for selfmod handling */
+#include "fragment.h"
+#include "instr.h"
+#include "decode.h"
+#include "decode_fast.h"
+
+#include "link.h"
+#include "disassemble.h"
+#include "fcache.h"
+#include "hotpatch.h"
+#include "moduledb.h"
+#include "module_shared.h"
+#include "perscache.h"
+
+#ifdef WINDOWS
+# include "events.h"             /* event log messages - not supported yet on Linux  */
+#endif
+
+#ifdef CLIENT_INTERFACE
+# include "instrument.h"
+#endif
+
+#ifdef DEBUG
+# include "synch.h" /* all_threads_synch_lock */
+#endif
+
+#include <string.h>
+
+enum {
+    /* VM_ flags to distinguish region types 
+     * We also use some FRAG_ flags (but in a separate field so no value space overlap)
+     * Adjacent regions w/ different flags are never merged.
+     */
+    VM_WRITABLE     = 0x0001,    /* app memory writable? */
+    /* UNMOD_IMAGE means the region was mmapped in and has been read-only since then
+     * this excludes even loader modifications (IAT update, relocate, etc.) on win32!
+     */
+    VM_UNMOD_IMAGE  = 0x0002,
+    VM_DELETE_ME    = 0x0004,    /* on delete queue -- for thread-local only */
+     /* NOTE : if a new area is added that overlaps an existing area with a 
+      * different VM_WAS_FUTURE flag, the areas will be merged with the flag 
+      * taken from the new area, see FIXME in add_vm_area */
+    VM_WAS_FUTURE   = 0x0008,    /* moved from future list to exec list */
+    VM_DR_HEAP      = 0x0010,    /* DR heap area */
+    VM_ONCE_ONLY    = 0x0020,    /* on future list but should be removed on 
+                                  * first exec */
+    /* FIXME case 7877, 3744: need to properly merge pageprot regions with
+     * existing selfmod regions before we can truly separate this.  For now we
+     * continue to treat selfmod as pageprot.
+     */
+    VM_MADE_READONLY = VM_WRITABLE/* FIXME: should be 0x0040 -- see above */,
+                                 /* DR has marked this region read 
+                                  * only for consistency, should only be used
+                                  * in conjunction with VM_WRITABLE */
+    VM_DELAY_READONLY = 0x0080,  /* dr has not yet marked this region read 
+                                  * only for consistency, should only be used
+                                  * in conjunction with VM_WRITABLE */
+#ifdef PROGRAM_SHEPHERDING
+    /* re-verify this region for code origins policies every time it is
+     * encountered.  only used with selfmod regions that are only allowed if
+     * they match patterns, to prevent other threads from writing non-pattern
+     * code though and executing after the region has been approved.
+     * xref case 4020.  can remove once we split code origins list from
+     * cache consistency list (case 3744).
+     */
+    VM_PATTERN_REVERIFY = 0x0100,  
+#endif
+
+    VM_DRIVER_ADDRESS   = 0x0200,
+    /* a driver hooker area, needed for case 9022.  Note we can
+     * normally read properties only of user mode addresses, so we
+     * have to probe addresses in this area.  Also note that we're
+     * still executing all of this code in user mode e.g. there is no
+     * mode switch, no conforming segments, etc.
+     */
+
+    /* Does this region contain a persisted cache?
+     * Must also be FRAG_COARSE_GRAIN of course.
+     * This is a shortcut to reading custom.client->persisted.
+     * This is not guaranteed to be set on shared_data: only on executable_areas.
+     */
+    VM_PERSISTED_CACHE     = 0x0400,
+
+    /* Case 10584: avoid flush synch when no code has been executed */
+    VM_EXECUTED_FROM       = 0x0800,
+
+    /* A workaround for lock rank issues: we delay adding loaded persisted
+     * units to shared_data until first asked about.
+     * This flags is NOT propagated on vmarea splits.
+     */
+    VM_ADD_TO_SHARED_DATA  = 0x1000,
+};
+
+/* simple way to disable sandboxing */
+#define SANDBOX_FLAG() (INTERNAL_OPTION(cache_consistency) ? FRAG_SELFMOD_SANDBOXED : 0)
+
+/* Fields only used for written_areas */
+typedef struct _ro_vs_sandbox_data_t {
+    /* written_count only used for written_areas vector.
+     * if > 0, areas will NOT be merged, so we can keep separate
+     * counts by page (hopefully not making the list too long).
+     */
+    uint written_count;
+    /* Used only for -sandbox2ro_threshold.  It's only in the
+     * written_areas vector b/c executable_areas has its regions removed
+     * on a flush while threads could still be accessing counters in
+     * selfmod fragments in the cache.  We lose some granularity here but
+     * it's not a big deal.
+     * We could make these both ushorts, but it'd be more of a pain
+     * to increment this counter from the cache then, worrying about overflows.
+     */
+    uint selfmod_execs;
+#ifdef DEBUG
+    uint ro2s_xfers;
+    uint s2ro_xfers;
+#endif
+} ro_vs_sandbox_data_t;
+
+/* Our executable area list has three types of areas.  Each type can be merged
+ * with adjacent areas of the same type but not with any of the other types!
+ * 1) originally RO code   == we leave alone
+ * 2) originally RW code   == we mark RO
+ * 3) originally RW code, written to from within itself  == we leave RW and sandbox
+ * We keep all three types in the same list b/c any particular address interval
+ * can only be of one type at any one time, and all three are executable, meaning
+ * code cache code was copied from there.
+ */
+typedef struct vm_area_t {
+    app_pc start;
+    app_pc end;         /* open end interval */
+    /* We have two different flags fields to allow easy use of the FRAG_ flags.
+     * The two combined are used to distinguish different regions.
+     * Adjacent regions w/ different flags are never merged.
+     */
+    /* Flags that start with VM_ */
+    uint vm_flags;
+    /* Flags that start with FRAG_
+     * In use now are FRAG_SELFMOD_SANDBOXED and FRAG_DYNGEN.
+     */
+    uint frag_flags;
+#ifdef DEBUG
+    char *comment;
+#endif
+    /********************
+     * custom fields not used in all vectors
+     * FIXME: separate into separately-allocated piece?  or have a struct
+     * extension (poor man's subclass, like trace_t, etc.) and make our vector
+     * iterators handle it?
+     * once we have a generic interval data structure (case 6208) this
+     * hardcoding of individual uses will go away.
+     */
+    union {
+        /* Used in per-thread and shared vectors, not in master area lists.
+         * We identify vectors using this via VECTOR_FRAGMENT_LIST, needed
+         * b/c {add,remove}_vm_area have special behavior for frags.
+         */
+        fragment_t *frags;
+        /* for clients' custom use via vmvector interfaces */
+        void *client;
+    } custom;
+} vm_area_t;
+
+/* for each thread we record all executable areas, to make it faster
+ * to decide whether we need to flush any fragments on an munmap
+ */
+typedef struct thread_data_t {
+    vm_area_vector_t areas;
+    /* cached pointer to last area encountered by thread */
+    vm_area_t *last_area;
+    /* FIXME: for locality would be nice to have per-thread last_shared_area
+     * (cannot put shared in private last_area, that would void its usefulness
+     *  since couldn't tell if area really in shared list or not)
+     * but then have to update all other threads whenever change shared
+     * vmarea vector, so for now we use a global last_area 
+     */
+    /* cached pointer of a PC in the last page decoded by thread -- set only
+     * in thread-private structures, not in shared structures like shared_data */
+    app_pc last_decode_area_page_pc;
+    bool   last_decode_area_valid; /* since no sentinel exists */
+#ifdef PROGRAM_SHEPHERDING
+    uint thrown_exceptions; /* number of responses to execution violations */
+#endif
+} thread_data_t;
+
+#define SHOULD_LOCK_VECTOR(v)                               \
+       (TEST(VECTOR_SHARED, (v)->flags) &&                  \
+        !TEST(VECTOR_NO_LOCK, (v)->flags) &&                \
+        !self_owns_write_lock(&(v)->lock))
+
+#define LOCK_VECTOR(v, release_lock, RW) do {               \
+    if (SHOULD_LOCK_VECTOR(v)) {                            \
+        (release_lock) = true;                              \
+        RW##_lock(&(v)->lock);                              \
+    }                                                       \
+    else                                                    \
+        (release_lock) = false;                             \
+} while (0);
+
+#define UNLOCK_VECTOR(v, release_lock, RW) do {             \
+    if ((release_lock)) {                                   \
+        ASSERT(TEST(VECTOR_SHARED, (v)->flags));            \
+        ASSERT(!TEST(VECTOR_NO_LOCK, (v)->flags));          \
+        ASSERT_OWN_READWRITE_LOCK(true, &(v)->lock);        \
+        RW##_unlock(&v->lock);                              \
+    }                                                       \
+} while (0);
+
+/* these two global vectors store all executable areas and all dynamo
+ * areas (executable or otherwise).
+ * executable_areas' custom field is used to store coarse unit info.
+ * for a FRAG_COARSE_GRAIN region, an info struct is always present, even
+ * if not yet executed from (initially, or after a flush).
+ */
+static vm_area_vector_t *executable_areas;
+static vm_area_vector_t *dynamo_areas;
+
+/* Protected by executable_areas lock; used only to delete coarse_info_t
+ * while holding executable_areas lock during execute-less flushes
+ * (case 10995).  Extra layer of indirection to get on heap and avoid .data
+ * unprotection.
+ */
+static coarse_info_t **coarse_to_delete;
+                        
+/* used for DYNAMO_OPTION(handle_DR_modify),
+ * DYNAMO_OPTION(handle_ntdll_modify) == DR_MODIFY_NOP or
+ * DYNAMO_OPTION(patch_proof_list)
+ */
+static vm_area_vector_t *pretend_writable_areas;
+
+/* used for DYNAMO_OPTION(patch_proof_list) areas to watch */
+vm_area_vector_t *patch_proof_areas;
+
+/* used for DYNAMO_OPTION(emulate_IAT_writes), though in future may be
+ * expanded, so not just ifdef WINDOWS or ifdef PROGRAM_SHEPHERDING
+ */
+vm_area_vector_t *emulate_write_areas;
+
+/* used for DYNAMO_OPTION(IAT_convert)
+ * IAT or GOT areas of all mapped DLLs - note the exact regions are added here.
+ * While the IATs for modules in native_exec_areas are not added here - 
+ * note that any module's IAT may still be importing native modules.
+ */
+vm_area_vector_t *IAT_areas;
+
+/* Keeps persistent written-to and execution counts for switching back and
+ * forth from page prot to sandboxing.
+ */
+static vm_area_vector_t *written_areas;
+
+#ifdef PROGRAM_SHEPHERDING
+/* for executable_if_flush and executable_if_alloc, we need a future list, so their regions
+ * are considered executable until de-allocated -- even if written to!
+ */
+static vm_area_vector_t *futureexec_areas;
+# ifdef WINDOWS
+/* FIXME: for -xdata_rct we only need start pc called on, so htable would do,
+ * once we have reusable htable for storing single pc
+ */
+static vm_area_vector_t *app_flushed_areas;
+
+# endif
+#endif
+
+/* tamper resistant region see tamper_resistant_region_add() for current use.
+ * If needed this should be turned into a vm_area_vector_t as well.
+ */
+static app_pc tamper_resistant_region_start, tamper_resistant_region_end;
+
+
+/* shared_data is synchronized via either single_thread_in_DR or
+ * the vector lock (cannot use bb_building_lock b/c both trace building
+ * and pc translation need read access and neither can/should grab
+ * the bb building lock, plus it's cleaner to not depend on it, and now
+ * with -shared_traces it's not sufficient).
+ * N.B.: the vector lock is used to protect not just the vector, but also
+ * the whole thread_data_t struct (including last_area) and sequences
+ * of vector operations.
+ * Kept on the heap for selfprot (case 7957).
+ */
+static thread_data_t *shared_data; /* set in vm_areas_reset_init() */
+
+typedef struct _pending_delete_t {
+#ifdef DEBUG
+    /* record bounds of original deleted region, for debugging only */
+    app_pc start;
+    app_pc end;
+#endif
+    /* list of unlinked fragments that are waiting to be deleted */
+    fragment_t *frags;
+    /* ref count and timestamp to determine when it's safe to delete them */
+    uint ref_count;
+    uint flushtime_deleted;
+    /* we use a simple linked list of entries */
+    struct _pending_delete_t *next;
+} pending_delete_t;
+
+/* We keep these list pointers on the heap for selfprot (case 8074). */
+typedef struct _deletion_lists_t {
+    /* Unlike private vm lists, we cannot simply mark shared_data vm areas as
+     * deleted since new fragments come in concurrently, so we have to have a
+     * separate list of flushed-but-not-yet-deleted areas.  We can't use a
+     * vm_area_vector_t b/c newly flushed fragments spoil our ref count by resetting
+     * it, so we keep a linked list of fragment lists.
+     */
+    pending_delete_t *shared_delete;
+    /* We maintain the tail solely for fcache_free_pending_units() */
+    pending_delete_t *shared_delete_tail;
+    /* count used for reset threshold */
+    uint shared_delete_count;
+
+    /* shared lazy deletion: a list of fragment_t chained via next_vmarea that
+     * are pending deletion, but are only freed when a shared deletion event
+     * shows that it is safe to do so.
+     */
+    fragment_t *lazy_delete_list;
+    /* stores the end of the list, for appending */
+    fragment_t *lazy_delete_tail;
+    /* stores the length of the lazy list */
+    uint lazy_delete_count;
+    /* ensure only one thread tries to move to pending deletion list */
+    bool move_pending;
+} deletion_lists_t;
+
+static deletion_lists_t *todelete;
+
+typedef struct _last_deallocated_t {
+    /* case 9330 - we want to detect races during DLL unloads, and to
+     * silence a reported violation during unload.  At least DLLs are
+     * expected to be already serialized by the loader so keeping only
+     * one is sufficient (note Win2K3 doesn't hold lock only during
+     * process initialization).  We'll also keep references to the
+     * last DLL that was unloaded for diagnostics.  Although, that is
+     * not reliable enough when multiple DLLs are involved - case 6061
+     * should be used for better tracking after unload.
+     */
+    /* Yet loss of integrity is tolerable, as long as detected.  Since
+     * we currently mark all mappings they are not necessarily
+     * serialized (and potentially other apps can directly map, so
+     * can't really count on the loader lock for integrity).  We
+     * should make sure that we do not set unload_in_progress unless
+     * [last_unload_base, last_unload_size) is really still the
+     * current module.
+     */
+    bool unload_in_progress;
+    app_pc last_unload_base;
+    size_t last_unload_size;
+    /* FIXME: we may want to overload the above or add different
+     * fields for non image (MEM_MAPPED) unmaps, and DGC (MEM_PRIVATE)
+     * frees.  Note that we avoid keeping lists of active unloads, or
+     * even to deal with case 9371 we would need intersection of
+     * overlapping app syscalls.  If we serialize app syscalls as
+     * proposed case 545 a single one will be sufficient.
+     */
+} last_deallocated_t;
+static last_deallocated_t *last_deallocated;
+/* synchronization currently used only for the contents of
+ * last_deallocated: last_unload_base and last_unload_size
+ */
+DECLARE_CXTSWPROT_VAR(static mutex_t last_deallocated_lock, 
+                      INIT_LOCK_FREE(last_deallocated_lock));
+
+/* synchronization for shared_delete, not a rw lock since readers usually write */
+DECLARE_CXTSWPROT_VAR(mutex_t shared_delete_lock, INIT_LOCK_FREE(shared_delete_lock));
+/* synchronization for the lazy deletion list */
+DECLARE_CXTSWPROT_VAR(static mutex_t lazy_delete_lock, INIT_LOCK_FREE(lazy_delete_lock));
+
+/* multi_entry_t allocation is either global or local heap */
+#define MULTI_ALLOC_DC(dc, flags) FRAGMENT_ALLOC_DC(dc, flags)
+#define GET_DATA(dc, flags) \
+    (((dc) == GLOBAL_DCONTEXT || TEST(FRAG_SHARED, (flags))) ? shared_data : \
+     (thread_data_t *) (dc)->vm_areas_field)
+#define GET_VECTOR(dc, flags)                                                  \
+    (((dc) == GLOBAL_DCONTEXT || TEST(FRAG_SHARED, (flags))) ?                 \
+     (TEST(FRAG_WAS_DELETED, (flags)) ? NULL : &shared_data->areas) :           \
+     (&((thread_data_t *)(dc)->vm_areas_field)->areas))
+#define SHARED_VECTOR_RWLOCK(v, rw, op) do    { \
+    if (TEST(VECTOR_SHARED, (v)->flags)) {      \
+        ASSERT(SHARED_FRAGMENTS_ENABLED());     \
+        rw##_##op(&(v)->lock);                  \
+    }                                           \
+} while (0)
+#define ASSERT_VMAREA_DATA_PROTECTED(data, RW)                      \
+    ASSERT_OWN_##RW##_LOCK((data == shared_data &&                 \
+                            !INTERNAL_OPTION(single_thread_in_DR)), \
+                           &shared_data->areas.lock)
+
+/* list of native_exec module regions 
+ * FIXME: since using general routines, could allocate this elsewhere if add
+ * vmvector_* interface heap alloc support
+ */
+vm_area_vector_t *native_exec_areas;
+
+/* FIXME: find a way to assert that an area by itself is synchronized if
+ * it points into a vector for the routines that take in only areas
+ */
+#ifdef DEBUG
+# define ASSERT_VMAREA_VECTOR_PROTECTED(v, RW) do {                    \
+    ASSERT_OWN_##RW##_LOCK(SHOULD_LOCK_VECTOR(v) &&                    \
+                           !dynamo_exited, &(v)->lock);                \
+    if ((v) == dynamo_areas) {                                         \
+        ASSERT(dynamo_areas_uptodate || dynamo_areas_synching);        \
+    }                                                                  \
+} while (0);
+#else
+# define ASSERT_VMAREA_VECTOR_PROTECTED(v, RW) /* nothing */
+#endif
+
+/* size of security violation string - must be at least 16 */
+#define MAXIMUM_VIOLATION_NAME_LENGTH 16
+
+#define VMVECTOR_INITIALIZE_VECTOR(v, flags, lockname) do {    \
+        vmvector_init_vector((v), (flags));            \
+        ASSIGN_INIT_READWRITE_LOCK_FREE((v)->lock, lockname); \
+    } while (0);
+
+/* forward declarations */
+static void
+vmvector_free_vector(dcontext_t *dcontext, vm_area_vector_t *v);
+
+static void
+vm_area_clean_fraglist(dcontext_t *dcontext, vm_area_t *area);
+static bool
+lookup_addr(vm_area_vector_t *v, app_pc addr, vm_area_t **area);
+#if defined(DEBUG) && defined(INTERNAL)
+static void
+print_fraglist(dcontext_t *dcontext, vm_area_t *area, char *prefix);
+static void
+print_written_areas(file_t outf);
+#endif
+
+static void
+update_dynamo_vm_areas(bool have_writelock);
+static void
+dynamo_vm_areas_start_reading(void);
+static void
+dynamo_vm_areas_done_reading(void);
+
+#ifdef PROGRAM_SHEPHERDING
+static bool
+remove_futureexec_vm_area(app_pc start, app_pc end);
+
+DECLARE_CXTSWPROT_VAR(static mutex_t threads_killed_lock,
+                      INIT_LOCK_FREE(threads_killed_lock));
+void
+mark_unload_future_added(app_pc module_base, size_t size);
+#endif
+
+static void
+vm_area_coarse_region_freeze(dcontext_t *dcontext, coarse_info_t *info,
+                             vm_area_t *area, bool in_place);
+
+#ifdef SIMULATE_ATTACK
+/* synch simulate_at string parsing */
+DECLARE_CXTSWPROT_VAR(static mutex_t simulate_lock, INIT_LOCK_FREE(simulate_lock));
+#endif
+
+/* used to determine when we need to do another heap walk to keep
+ * dynamo vm areas up to date (can't do it incrementally b/c of
+ * circular dependencies).
+ * protected for both read and write by dynamo_areas->lock
+ */
+/* Case 3045: areas inside the vmheap reservation are not added to the list,
+ * so the vector is considered uptodate until we run out of reservation
+ */
+DECLARE_FREQPROT_VAR(static bool dynamo_areas_uptodate, true);
+
+#ifdef DEBUG
+/* used for debugging to tell when uptodate can be false.
+ * protected for both read and write by dynamo_areas->lock
+ */
+DECLARE_FREQPROT_VAR(static bool dynamo_areas_synching, false);
+#endif
+
+/* HACK to make dynamo_areas->lock recursive 
+ * protected for both read and write by dynamo_areas->lock
+ * FIXME: provide general rwlock w/ write portion recursive
+ */
+DECLARE_CXTSWPROT_VAR(uint dynamo_areas_recursion, 0);
+
+/* used for DR area debugging */
+bool vm_areas_exited = false;
+
+/***************************************************
+ * flushing by walking entire hashtable is too slow, so we keep a list of
+ * all fragments in each region.
+ * to save memory, we use the fragment_t struct as the linked list entry
+ * for these lists.  However, some fragments are on multiple lists due to
+ * crossing boundaries (usually traces).  For those, the other entries are
+ * pointed to by an "also" field, and the entries themselves use this struct,
+ * which plays games (similar to fcache's empty_slot_t) to be able to be used
+ * like a fragment_t struct in the lists.
+ *
+ * this is better than the old fragment_t->app_{min,max}_pc performance wise,
+ * and granularity-wise for blocks that bounce over regions, but worse
+ * granularity-wise since if want to flush singe page in text
+ * section, will end up flushing entire region.  especially scary in face of
+ * merges of adjacent regions, but merges are rare for images since
+ * they usually have more than just text, so texts aren't adjacent.
+ *
+ * FIXME: better way, now that fcache supports multiple units, is to have
+ * a separate unit for each source vmarea.  common case will be a flush to
+ * an un-merged or clipped area, so just toss whole unit.
+ */
+typedef struct _multi_entry_t {
+    fragment_t *f; /* backpointer */
+    /* flags MUST be at same location as fragment_t->flags
+     * we set flags==FRAG_IS_EXTRA_VMAREA to indicate a multi_entry_t
+     * we also use FRAG_SHARED to indicate that a multi_entry_t is on global heap
+     */
+    uint      flags;
+    /* officially all list entries are fragment_t *, really some are multi_entry_t */
+    fragment_t *next_vmarea;
+    fragment_t *prev_vmarea;
+    fragment_t *also_vmarea; /* if in multiple areas */
+    /* need to be able to look up vmarea: area not stored since vmareas
+     * shift and merge, so we store original pc */
+    app_pc pc;
+} multi_entry_t;
+
+/* macros to make dealing with both fragment_t and multi_entry_t easier */
+#define FRAG_MULTI(f) (TEST(FRAG_IS_EXTRA_VMAREA, (f)->flags))
+
+#define FRAG_MULTI_INIT(f) (TESTALL((FRAG_IS_EXTRA_VMAREA|FRAG_IS_EXTRA_VMAREA_INIT), (f)->flags))
+
+#define FRAG_NEXT(f) ((TEST(FRAG_IS_EXTRA_VMAREA, (f)->flags)) ? \
+    ((multi_entry_t *)(f))->next_vmarea : (f)->next_vmarea)
+
+#define FRAG_NEXT_ASSIGN(f, val) do { \
+    if (TEST(FRAG_IS_EXTRA_VMAREA, (f)->flags)) \
+        ((multi_entry_t *)(f))->next_vmarea = (val); \
+    else \
+        (f)->next_vmarea = (val); \
+} while (0)
+
+#define FRAG_PREV(f) ((TEST(FRAG_IS_EXTRA_VMAREA, (f)->flags)) ? \
+    ((multi_entry_t *)(f))->prev_vmarea : (f)->prev_vmarea)
+
+#define FRAG_PREV_ASSIGN(f, val) do { \
+    if (TEST(FRAG_IS_EXTRA_VMAREA, (f)->flags)) \
+        ((multi_entry_t *)(f))->prev_vmarea = (val); \
+    else \
+        (f)->prev_vmarea = (val); \
+} while (0)
+
+/* Case 8419: also_vmarea is invalid once we 1st-stage-delete a fragment */
+#define FRAG_ALSO(f) ((TEST(FRAG_IS_EXTRA_VMAREA, (f)->flags)) ? \
+    ((multi_entry_t *)(f))->also_vmarea : \
+    (ASSERT(!TEST(FRAG_WAS_DELETED, (f)->flags)), (f)->also.also_vmarea))
+/* Only call this one to avoid the assert when you know it's safe */
+#define FRAG_ALSO_DEL_OK(f) ((TEST(FRAG_IS_EXTRA_VMAREA, (f)->flags)) ? \
+    ((multi_entry_t *)(f))->also_vmarea : (f)->also.also_vmarea)
+
+#define FRAG_ALSO_ASSIGN(f, val) do { \
+    if (TEST(FRAG_IS_EXTRA_VMAREA, (f)->flags)) \
+        ((multi_entry_t *)(f))->also_vmarea = (val); \
+    else { \
+        ASSERT(!TEST(FRAG_WAS_DELETED, (f)->flags)); \
+        (f)->also.also_vmarea = (val); \
+    } \
+} while (0)
+
+/* assumption: if multiple units, fragment_t is on list of region owning tag */
+#define FRAG_PC(f) ((TEST(FRAG_IS_EXTRA_VMAREA, (f)->flags)) ? \
+    ((multi_entry_t *)(f))->pc : (f)->tag)
+
+#define FRAG_PC_ASSIGN(f, val) do { \
+    if (TEST(FRAG_IS_EXTRA_VMAREA, (f)->flags)) \
+        ((multi_entry_t *)(f))->pc = (val); \
+    else \
+        ASSERT_NOT_REACHED(); \
+} while (0)
+
+#define FRAG_FRAG(fr) ((TEST(FRAG_IS_EXTRA_VMAREA, (fr)->flags)) ? \
+    ((multi_entry_t *)(fr))->f : (fr))
+
+#define FRAG_FRAG_ASSIGN(fr, val) do { \
+    if (TEST(FRAG_IS_EXTRA_VMAREA, (fr)->flags)) \
+        ((multi_entry_t *)(fr))->f = (val); \
+    else \
+        ASSERT_NOT_REACHED(); \
+} while (0)
+
+#define FRAG_ID(fr) ((TEST(FRAG_IS_EXTRA_VMAREA, (fr)->flags)) ? \
+    ((multi_entry_t *)(fr))->f->id : (fr)->id)
+
+/***************************************************/
+
+/* FIXME : is problematic to page align subpage regions */
+static void
+vm_make_writable(byte *pc, size_t size)
+{
+    byte *start_pc = (byte *)ALIGN_BACKWARD(pc, PAGE_SIZE);
+    size_t final_size = ALIGN_FORWARD(size + (pc - start_pc), PAGE_SIZE);
+    DEBUG_DECLARE(bool ok = )
+        make_writable(start_pc, final_size);
+    ASSERT(ok);
+    ASSERT(INTERNAL_OPTION(cache_consistency));
+}
+
+static void
+vm_make_unwritable(byte *pc, size_t size)
+{
+    byte *start_pc = (byte *)ALIGN_BACKWARD(pc, PAGE_SIZE);
+    size_t final_size = ALIGN_FORWARD(size + (pc - start_pc), PAGE_SIZE);
+    ASSERT(INTERNAL_OPTION(cache_consistency));
+    make_unwritable(start_pc, final_size);
+
+    /* case 8308: We should never call vm_make_unwritable if
+     * -sandbox_writable is on, or if -sandbox_non_text is on and this
+     * is a non-text region.
+     */
+    ASSERT(!DYNAMO_OPTION(sandbox_writable));
+    DODEBUG({
+        if (DYNAMO_OPTION(sandbox_non_text)) {
+            app_pc modbase = get_module_base(pc);
+            ASSERT(modbase != NULL && is_range_in_code_section(modbase, pc,
+                                                               pc + size,
+                                                               NULL, NULL));
+        }
+    });
+}
+
+/* since dynamorio changes some readwrite memory regions to read only, 
+ * this changes all regions memory permissions back to what they should be, 
+ * since dynamorio uses this mechanism to ensure code cache coherency, 
+ * once this method is called stale code could be executed out of the 
+ * code cache */
+void 
+revert_memory_regions()
+{
+    int i;
+
+    /* executable_areas doesn't exist in thin_client mode. */
+    ASSERT(!DYNAMO_OPTION(thin_client));
+
+    read_lock(&executable_areas->lock);
+    for (i = 0; i < executable_areas->length; i++) {
+        if (TEST(VM_MADE_READONLY, executable_areas->buf[i].vm_flags)) {
+            /* this is a region that dynamorio has marked read only, fix */
+            LOG(GLOBAL, LOG_VMAREAS, 1, 
+                " fixing permissions for RW executable area "PFX"-"PFX" %s\n", 
+                executable_areas->buf[i].start, executable_areas->buf[i].end, 
+                executable_areas->buf[i].comment);
+            vm_make_writable(executable_areas->buf[i].start, 
+                             executable_areas->buf[i].end -
+                             executable_areas->buf[i].start);
+        }
+    }
+    read_unlock(&executable_areas->lock);
+}
+
+static void
+print_vm_flags(uint vm_flags, uint frag_flags, file_t outf)
+{
+    print_file(outf, " %s%s%s%s",
+               (vm_flags & VM_WRITABLE) != 0 ? "W" : "-",
+               (vm_flags & VM_WAS_FUTURE) != 0 ? "F" : "-",
+               (frag_flags & FRAG_SELFMOD_SANDBOXED) != 0 ? "S" : "-",
+               TEST(FRAG_COARSE_GRAIN, frag_flags) ? "C" : "-");
+#ifdef PROGRAM_SHEPHERDING
+    print_file(outf, "%s%s",
+               TEST(VM_PATTERN_REVERIFY, vm_flags) ? "P" : "-",
+               (frag_flags & FRAG_DYNGEN) != 0 ? "D" : "-");
+#endif
+}
+
+/* ok to pass NULL for v, only used to identify use of custom field */
+static void
+print_vm_area(vm_area_vector_t *v, vm_area_t *area, file_t outf, const char *prefix)
+{
+    print_file(outf, "%s"PFX"-"PFX, prefix, area->start, area->end);
+    print_vm_flags(area->vm_flags, area->frag_flags, outf);
+    if (v == executable_areas && TEST(FRAG_COARSE_GRAIN, area->frag_flags)) {
+        coarse_info_t *info = (coarse_info_t *) area->custom.client;
+        if (info != NULL) {
+            if (info->persisted)
+                print_file(outf, "R");
+            else if (info->frozen)
+                print_file(outf, "Z");
+            else
+                print_file(outf, "-");
+        }
+    }
+#ifdef DEBUG
+    print_file(outf, " %s", area->comment);
+    DOLOG(1, LOG_VMAREAS, {
+        app_pc modbase = get_module_base(area->start);
+        if (modbase != NULL &&
+            /* avoid rank order violations */
+            v != dynamo_areas &&
+            /* we free module list before vmareas */
+            !dynamo_exited_and_cleaned &&
+            is_mapped_as_image(modbase)/*avoid asserts in getting name */) {
+            const char *name;
+            os_get_module_info_lock();
+            os_get_module_name(modbase, &name);
+            print_file(outf, " %s", name == NULL ? "" : name);
+            os_get_module_info_unlock();
+        }
+    });
+#endif
+    if (v == written_areas) {
+        ro_vs_sandbox_data_t *ro2s = (ro_vs_sandbox_data_t *) area->custom.client;
+#ifdef DEBUG
+        if (ro2s != NULL) { /* can be null if in middle of adding */
+            uint tot_w = ro2s->ro2s_xfers * DYNAMO_OPTION(ro2sandbox_threshold);
+            uint tot_s = ro2s->s2ro_xfers * DYNAMO_OPTION(sandbox2ro_threshold);
+            print_file(outf, " w %3d, %3d tot; x %3d, %5d tot; ro2s %d, s2ro %d",
+                       ro2s->written_count, tot_w, ro2s->selfmod_execs, tot_s,
+                       ro2s->ro2s_xfers, ro2s->s2ro_xfers);
+        }
+#else
+        print_file(outf, " written %3d, exec %5d",
+                   ro2s->written_count, ro2s->selfmod_execs);
+#endif
+    }
+    print_file(outf, "\n");
+}
+
+/* Assumes caller holds v->lock for coherency */
+static void
+print_vm_areas(vm_area_vector_t *v, file_t outf)
+{
+    int i;
+    ASSERT_VMAREA_VECTOR_PROTECTED(v, READWRITE);
+    for(i = 0; i < v->length; i++) {
+        print_vm_area(v, &v->buf[i], outf, "  ");
+    }
+}
+
+#if defined(DEBUG) && defined(INTERNAL)
+static void
+print_contig_vm_areas(vm_area_vector_t *v, app_pc start, app_pc end, file_t outf, char *prefix)
+{
+    vm_area_t *new_area;
+    app_pc pc = start;
+    do {
+        lookup_addr(v, pc, &new_area);
+        if (new_area == NULL)
+            break;
+        print_vm_area(v, new_area, outf, prefix);
+        pc = new_area->end + 1;
+    } while (new_area->end < end);
+}
+#endif
+
+#if defined(DEBUG) && defined(INTERNAL)
+static void
+print_pending_list(file_t outf)
+{
+    pending_delete_t *pend;
+    int i;
+    ASSERT_OWN_MUTEX(true, &shared_delete_lock);
+    for (i = 0, pend = todelete->shared_delete; pend != NULL; i++, pend = pend->next) {
+        print_file(outf, "%d: "PFX"-"PFX" ref=%d, stamp=%d\n",
+                   i, pend->start, pend->end, pend->ref_count, pend->flushtime_deleted);
+    }
+}
+#endif
+
+/* If v requires a lock and the calling thread does not hold that lock,
+ * this routine acquires the lock and returns true; else it returns false.
+ */
+static bool
+writelock_if_not_already(vm_area_vector_t *v)
+{
+    if (TEST(VECTOR_SHARED, v->flags) && !self_owns_write_lock(&v->lock)) {
+        SHARED_VECTOR_RWLOCK(v, write, lock);
+        return true;
+    }
+    return false;
+}
+
+static void
+vm_area_vector_check_size(vm_area_vector_t *v)
+{
+    /* only called by add_vm_area which does the assert that the vector is
+     * protected */
+    /* check if at capacity */
+    if (v->size == v->length){
+        if (v->length == 0) {
+            v->size = INTERNAL_OPTION(vmarea_initial_size);
+            v->buf = (vm_area_t*) global_heap_alloc(v->size*sizeof(struct vm_area_t)
+                                                  HEAPACCT(ACCT_VMAREAS));
+        }
+        else {
+            /* FIXME: case 4471 we should be doubling size here */
+            int new_size = (INTERNAL_OPTION(vmarea_increment_size) + v->length);
+            STATS_INC(num_vmareas_resized);
+            v->buf = global_heap_realloc(v->buf, v->size, new_size,
+                                         sizeof(struct vm_area_t)
+                                         HEAPACCT(ACCT_VMAREAS));
+            v->size = new_size;
+        }
+        ASSERT(v->buf != NULL);
+    }
+}
+
+static void
+vm_area_merge_fraglists(vm_area_t *dst, vm_area_t *src)
+{
+    /* caller must hold write lock for vector of course: FIXME: assert that here */
+    LOG(THREAD_GET, LOG_VMAREAS, 3,
+        "\tmerging frag lists for "PFX"-"PFX" and "PFX"-"PFX"\n",
+        src->start, src->end, dst->start, dst->end);
+    if (dst->custom.frags == NULL)
+        dst->custom.frags = src->custom.frags;
+    else if (src->custom.frags == NULL)
+        return;
+    else {
+        /* put src's frags at end of dst's frags */
+        fragment_t *top1 = dst->custom.frags;
+        fragment_t *top2 = src->custom.frags;
+        fragment_t *tmp = FRAG_PREV(top1);
+        FRAG_NEXT_ASSIGN(tmp, top2);
+        FRAG_PREV_ASSIGN(top1, FRAG_PREV(top2));
+        FRAG_PREV_ASSIGN(top2, tmp);
+        DOLOG(4, LOG_VMAREAS, {
+            print_fraglist(get_thread_private_dcontext(),
+                           dst, "after merging fraglists:");
+        });
+    }
+}
+
+/* Assumes caller holds v->lock, if necessary.
+ * Does not return the area added since it may be merged or split depending
+ * on existing areas->
+ * If a last_area points into this vector, the caller must make sure to
+ *   clear or update the last_area pointer.
+ *   FIXME: make it easier to keep them in synch -- too easy to add_vm_area
+ *   somewhere to a thread vector and forget to clear last_area.
+ * Adds a new area to v, merging it with adjacent areas of the same type.
+ * A new area is only allowed to overlap an old area of a different type if it
+ *   meets certain criteria (see asserts below).  For VM_WAS_FUTURE and 
+ *   VM_ONCE_ONLY we may clear the flag from an existing region if the new 
+ *   region doesn't have the flag and overlaps the existing region.  Otherwise 
+ *   the new area is split such that the overlapping portion remains part of
+ *   the old area.  This tries to keep entire new area from becoming selfmod
+ *   for instance. FIXME : for VM_WAS_FUTURE and VM_ONCE_ONLY may want to split
+ *   region if only paritally overlapping
+ * 
+ * FIXME: change add_vm_area to return NULL when merged, and otherwise
+ * return the new complete area, so callers don't have to do a separate lookup
+ * to access the added area.
+ */
+static void
+add_vm_area(vm_area_vector_t *v, app_pc start, app_pc end,
+            uint vm_flags, uint frag_flags _IF_DEBUG(char *comment))
+{
+    int i, j, diff;
+    /* if we have overlap, we extend an existing area -- else we add a new area */
+    int overlap_start = -1, overlap_end = -1;
+    DEBUG_DECLARE(uint flagignore;)
+
+    ASSERT(start < end);
+
+    ASSERT_VMAREA_VECTOR_PROTECTED(v, WRITE);
+    LOG(GLOBAL, LOG_VMAREAS, 4, "in add_vm_area "PFX" "PFX" %s\n", start, end, comment);
+    /* N.B.: new area could span multiple existing areas! */
+    for (i = 0; i < v->length; i++) {
+        /* look for overlap, or adjacency of same type (including all flags, and never
+         * merge adjacent if keeping write counts) 
+         */
+        if ((start < v->buf[i].end && end > v->buf[i].start) ||
+            (start <= v->buf[i].end && end >= v->buf[i].start &&
+             vm_flags == v->buf[i].vm_flags &&
+             frag_flags == v->buf[i].frag_flags &&
+             /* never merge coarse-grain */
+             !TEST(FRAG_COARSE_GRAIN, v->buf[i].frag_flags) &&
+             !TEST(VECTOR_NEVER_MERGE, v->flags))) {
+            if (overlap_start == -1) {
+                /* assume we'll simply expand an existing area rather than
+                 * add a new one -- we'll reset this if we hit merge conflicts */
+                overlap_start = i;
+            }
+            /* overlapping regions of different properties are often
+             * problematic so we add a lot of debugging output
+             */
+            DOLOG(4, LOG_VMAREAS, {
+                LOG(GLOBAL, LOG_VMAREAS, 1,
+                    "==================================================\n"
+                    "add_vm_area "PFX"-"PFX" %s %x-%x overlaps "PFX"-"PFX" %s %x-%x\n",
+                    start, end, comment, vm_flags, frag_flags,
+                    v->buf[i].start, v->buf[i].end,
+                    v->buf[i].comment, v->buf[i].vm_flags, v->buf[i].frag_flags);
+                print_vm_areas(v, GLOBAL);
+                /* rank order problem if holding heap_unit_lock, so only print
+                 * if not holding a lock for v right now, though ok to print
+                 * for shared vm areas since its lock is higher than the lock
+                 * for executable/written areas
+                 */
+                if (v != dynamo_areas &&
+                    (!TEST(VECTOR_SHARED, v->flags) || v == &shared_data->areas)) {
+                    LOG(GLOBAL, LOG_VMAREAS, 1, "\nexecutable areas:\n");
+                    print_executable_areas(GLOBAL);
+                    LOG(GLOBAL, LOG_VMAREAS, 1, "\nwritten areas:\n");
+                    print_written_areas(GLOBAL);
+                }
+                LOG(GLOBAL, LOG_VMAREAS, 1,
+                    "==================================================\n\n");
+            });
+
+            /* we have some restrictions on overlapping regions with 
+             * different flags */
+
+            /* no restrictions on WAS_FUTURE flag, but if new region is
+             * not was future and old region is then should drop from old 
+             * region FIXME : partial overlap? we don't really care about
+             * this flag anyways */
+            if (TEST(VM_WAS_FUTURE, v->buf[i].vm_flags) &&
+                !TEST(VM_WAS_FUTURE, vm_flags)) {
+                v->buf[i].vm_flags &= ~VM_WAS_FUTURE;
+                LOG(GLOBAL, LOG_VMAREAS, 1,
+                    "Warning : removing was_future flag from area "PFX
+                    "-"PFX" %s that overlaps new area "PFX"-"PFX" %s\n",
+                     v->buf[i].start, v->buf[i].end, v->buf[i].comment,
+                    start, end, comment);
+            }
+            /* no restrictions on ONCE_ONLY flag, but if new region is not
+             * should drop fom existing region FIXME : partial overlap? is
+             * not much of an additional security risk */
+            if (TEST(VM_ONCE_ONLY, v->buf[i].vm_flags) &&
+                !TEST(VM_ONCE_ONLY, vm_flags)) {
+                v->buf[i].vm_flags &= ~VM_ONCE_ONLY;
+                LOG(GLOBAL, LOG_VMAREAS, 1,
+                    "Warning : removing once_only flag from area "PFX
+                    "-"PFX" %s that overlaps new area "PFX"-"PFX" %s\n",
+                     v->buf[i].start, v->buf[i].end, v->buf[i].comment,
+                    start, end, comment);
+            }
+            /* shouldn't be adding unmod image over existing not unmod image,
+             * reverse could happen with os region merging though */
+            ASSERT(TEST(VM_UNMOD_IMAGE, v->buf[i].vm_flags) ||
+                   !TEST(VM_UNMOD_IMAGE, vm_flags));
+            /* for VM_WRITABLE only allow new region to not be writable and
+             * existing region to be writable to handle cases of os region
+             * merging due to our consistency protection changes */
+            ASSERT(TEST(VM_WRITABLE, v->buf[i].vm_flags) ||
+                   !TEST(VM_WRITABLE, vm_flags));
+            /* FIXME: case 7877: if new is VM_MADE_READONLY and old is not, we
+             * must mark old overlapping portion as VM_MADE_READONLY.  Things only
+             * worked now b/c VM_MADE_READONLY==VM_WRITABLE, so we can add
+             * pageprot regions that overlap w/ selfmod.
+             */
+#ifdef PROGRAM_SHEPHERDING
+            /* !VM_PATTERN_REVERIFY trumps having the flag on, so for new having
+             * the flag and old not, we're fine, but when old has it we'd like
+             * to remove it from the overlap portion: FIXME: need better merging
+             * control, also see all the partial overlap fixmes above.
+             * for this flag not a big deal, just a possible perf hit as we
+             * re-check every time.
+             */
+#endif
+            /* disallow any other vm_flag differences */
+            DODEBUG({ flagignore = VM_UNMOD_IMAGE | VM_WAS_FUTURE |
+                          VM_ONCE_ONLY | VM_WRITABLE; });
+#ifdef PROGRAM_SHEPHERDING
+            DODEBUG({ flagignore = flagignore | VM_PATTERN_REVERIFY; });
+#endif
+            ASSERT((v->buf[i].vm_flags & ~flagignore) == (vm_flags & ~flagignore));
+            
+            /* new region must be more innocent with respect to selfmod */
+            ASSERT(TEST(FRAG_SELFMOD_SANDBOXED, v->buf[i].frag_flags) || 
+                   !TEST(FRAG_SELFMOD_SANDBOXED, frag_flags));
+            /* disallow other frag_flag differences */
+#ifndef PROGRAM_SHEPHERDING
+            ASSERT((v->buf[i].frag_flags & ~FRAG_SELFMOD_SANDBOXED) == 
+                   (frag_flags & ~FRAG_SELFMOD_SANDBOXED));
+#else
+# ifdef DGC_DIAGNOSTICS
+            /* FIXME : no restrictions on differing FRAG_DYNGEN_RESTRICTED 
+             * flags? */
+            ASSERT((v->buf[i].frag_flags & 
+                    ~(FRAG_SELFMOD_SANDBOXED|FRAG_DYNGEN|FRAG_DYNGEN_RESTRICTED)) == 
+                   (frag_flags & 
+                    ~(FRAG_SELFMOD_SANDBOXED|FRAG_DYNGEN|FRAG_DYNGEN_RESTRICTED)));
+# else
+            ASSERT((v->buf[i].frag_flags & 
+                    ~(FRAG_SELFMOD_SANDBOXED|FRAG_DYNGEN)) == 
+                   (frag_flags & 
+                    ~(FRAG_SELFMOD_SANDBOXED|FRAG_DYNGEN)));
+# endif
+            /* shouldn't add non-dyngen overlapping existing dyngen, FIXME
+             * is the reverse possible? right now we allow it */
+            ASSERT(TEST(FRAG_DYNGEN, frag_flags) ||
+                   !TEST(FRAG_DYNGEN, v->buf[i].frag_flags));
+#endif
+            /* Never split FRAG_COARSE_GRAIN */
+            ASSERT(TEST(FRAG_COARSE_GRAIN, frag_flags) ||
+                   !TEST(FRAG_COARSE_GRAIN, v->buf[i].frag_flags));
+
+            /* must overlap same type -- else split */
+            if (vm_flags != v->buf[i].vm_flags || frag_flags != v->buf[i].frag_flags) {
+                LOG(GLOBAL, LOG_VMAREAS, 1,
+                    "add_vm_area "PFX"-"PFX" %s vm_flags=0x%08x "
+                    "frag_flags=0x%08x\n  overlaps diff type "PFX"-"PFX" %s"
+                    "vm_flags=0x%08x frag_flags=0x%08x\n  in vect at "PFX"\n",
+                    start, end, comment, vm_flags, frag_flags,
+                    v->buf[i].start, v->buf[i].end, v->buf[i].comment,
+                    v->buf[i].vm_flags, v->buf[i].frag_flags, v);
+                LOG(GLOBAL, LOG_VMAREAS, 3,
+                    "before splitting b/c adding "PFX"-"PFX":\n",
+                    start, end);
+                DOLOG(3, LOG_VMAREAS, { print_vm_areas(v, GLOBAL); });
+
+                /* split off the overlapping part from the new region 
+                 * reasoning: old regions get marked selfmod, then see new code,
+                 * its region overlaps old selfmod -- don't make new all selfmod,
+                 * split off the part that hasn't been proved selfmod yet.
+                 * since we never split the old region, we don't need to worry
+                 * about splitting its frags list.
+                 */
+                if (start < v->buf[i].start) {
+                    if (end > v->buf[i].end) {
+                        /* need two areas, one for either side */
+                        LOG(GLOBAL, LOG_VMAREAS, 3,
+                            "=> will add "PFX"-"PFX" after i\n", v->buf[i].end, end);
+                        /* safe to recurse here, new area will be after the area
+                         * we are currently looking at in the vector */
+                        add_vm_area(v, v->buf[i].end, end, vm_flags, frag_flags
+                                    _IF_DEBUG(comment));
+                    }
+                    /* if had been merging, let this routine finish that off -- else,
+                     * need to add a new area
+                     */
+                    end = v->buf[i].start;
+                    if (overlap_start == i) {
+                        /* no merging */
+                        overlap_start = -1;
+                    }
+                    LOG(GLOBAL, LOG_VMAREAS, 3,
+                        "=> will add/merge "PFX"-"PFX" before i\n", start, end);
+                    overlap_end = i;
+                    break;
+                } else if (end > v->buf[i].end) {
+                    /* shift area of consideration to end of i, and keep going,
+                     * can't act now since don't know areas overlapping beyond i
+                     */
+                    LOG(GLOBAL, LOG_VMAREAS, 3,
+                        "=> ignoring "PFX"-"PFX", only adding "PFX"-"PFX"\n",
+                        start, v->buf[i].end, v->buf[i].end, end);
+                    start = v->buf[i].end;
+                    /* reset overlap vars */
+                    ASSERT(overlap_start <= i);
+                    overlap_start = -1;
+                } else {
+                    /* completely inside -- ok, we'll leave it that way and won't split */
+                    LOG(GLOBAL, LOG_VMAREAS, 3,
+                        "=> ignoring "PFX"-"PFX", forcing to be part of "PFX"-"PFX"\n",
+                        start, end, v->buf[i].start, v->buf[i].end);
+                }
+                ASSERT(end > start);
+            }
+        } else if (overlap_start > -1) {
+            overlap_end = i; /* not inclusive */
+            break;
+        } else if (end <= v->buf[i].start)
+            break;
+    }
+
+    if (overlap_start == -1) {
+        /* brand-new area, goes before v->buf[i] */
+        struct vm_area_t new_area = {start, end, vm_flags, frag_flags, /* rest 0 */};
+#ifdef DEBUG
+        /* get comment */
+        size_t len = strlen(comment);
+        ASSERT(len < 1024);
+        new_area.comment = (char *) global_heap_alloc(len+1 HEAPACCT(ACCT_VMAREAS));
+        strncpy(new_area.comment, comment, len);
+        new_area.comment[len]  = '\0'; /* if max no null */
+#endif
+        LOG(GLOBAL, LOG_VMAREAS, 3, "=> adding "PFX"-"PFX"\n", start, end);
+        vm_area_vector_check_size(v);
+        /* shift subsequent entries */
+        for (j = v->length; j > i; j--)
+            v->buf[j] = v->buf[j-1];
+        v->buf[i] = new_area;
+        /* assumption: no overlaps between areas in list! */
+#ifdef DEBUG
+        if (!((i == 0 || v->buf[i-1].end <= v->buf[i].start) &&
+              (i == v->length || v->buf[i].end <= v->buf[i+1].start))) {
+            LOG(GLOBAL, LOG_VMAREAS, 1,
+                "ERROR: add_vm_area illegal overlap "PFX" "PFX" %s\n", start, end, comment);
+            print_vm_areas(v, GLOBAL);
+        }
+#endif
+        ASSERT((i == 0 || v->buf[i-1].end <= v->buf[i].start) &&
+               (i == v->length || v->buf[i].end <= v->buf[i+1].start));
+        v->length++;
+        STATS_TRACK_MAX(max_vmareas_length, v->length);
+        DOSTATS({
+            if (v == dynamo_areas)
+                STATS_TRACK_MAX(max_DRareas_length, v->length);
+            else if (v == executable_areas)
+                STATS_TRACK_MAX(max_execareas_length, v->length);
+        });
+#ifdef WINDOWS
+        DOSTATS({
+            extern vm_area_vector_t *loaded_module_areas;
+            if (v == loaded_module_areas)
+                STATS_TRACK_MAX(max_modareas_length, v->length);
+        });
+#endif
+    } else {
+        /* overlaps one or more areas, modify first to equal entire range,
+         * delete rest
+         */
+        if (overlap_end == -1)
+            overlap_end = v->length;
+        LOG(GLOBAL, LOG_VMAREAS, 3, "=> changing "PFX"-"PFX,
+            v->buf[overlap_start].start, v->buf[overlap_start].end);
+        if (start < v->buf[overlap_start].start)
+            v->buf[overlap_start].start = start;
+        if (end > v->buf[overlap_end-1].end)
+            v->buf[overlap_start].end = end;
+        else
+            v->buf[overlap_start].end = v->buf[overlap_end-1].end;
+        LOG(GLOBAL, LOG_VMAREAS, 3, " to "PFX"-"PFX"\n",
+            v->buf[overlap_start].start, v->buf[overlap_start].end);
+        /* when merge, use which comment?  could combine them all
+         * FIXME
+         */
+        /* now delete */
+        for (i = overlap_start+1; i < overlap_end; i++) {
+            LOG(GLOBAL, LOG_VMAREAS, 3, "=> completely removing "PFX"-"PFX" %s\n",
+                v->buf[i].start, v->buf[i].end, v->buf[i].comment);
+#ifdef DEBUG
+            global_heap_free(v->buf[i].comment, strlen(v->buf[i].comment)+1
+                             HEAPACCT(ACCT_VMAREAS));
+#endif
+            if (v == written_areas) {
+                /* FIXME: generalize vmvector further to allow custom merging
+                 * control so we don't have to hardcode this vector pointer check
+                 */
+                HEAP_TYPE_FREE(GLOBAL_DCONTEXT, (ro_vs_sandbox_data_t *)
+                               written_areas->buf[i].custom.client,
+                               ro_vs_sandbox_data_t, ACCT_VMAREAS, UNPROTECTED);
+                
+            }
+            /* merge frags lists */
+            if (TEST(VECTOR_FRAGMENT_LIST, v->flags) && v->buf[i].custom.frags != NULL)
+                vm_area_merge_fraglists(&v->buf[overlap_start], &v->buf[i]);
+        }
+        diff = overlap_end - (overlap_start+1);
+        for (i = overlap_start+1; i < v->length-diff; i++)
+            v->buf[i] = v->buf[i+diff];
+        v->length -= diff;
+        i = overlap_start; /* for return value */
+        if (TEST(VECTOR_FRAGMENT_LIST, v->flags) && v->buf[i].custom.frags != NULL) {
+            dcontext_t *dcontext = get_thread_private_dcontext();
+            ASSERT(dcontext != NULL);
+            /* have to remove all alsos that are now in same area as frag */
+            vm_area_clean_fraglist(dcontext, &v->buf[i]);
+        }
+    }
+    DOLOG(5, LOG_VMAREAS, { print_vm_areas(v, GLOBAL); });
+}
+
+static void
+adjust_coarse_unit_bounds(vm_area_t *area, bool if_invalid)
+{
+    coarse_info_t *info = (coarse_info_t *) area->custom.client;
+    ASSERT(TEST(FRAG_COARSE_GRAIN, area->frag_flags));
+    ASSERT(!RUNNING_WITHOUT_CODE_CACHE());
+    ASSERT(info != NULL);
+    if (info == NULL) /* be paranoid */
+        return;
+    /* FIXME: we'd like to grab info->lock but we have a rank order w/
+     * exec_areas lock -- so instead we rely on all-thread-synch flushing
+     * being the only reason to get here; an empty flush won't have synchall,
+     * but we won't be able to get_executable_area_coarse_info w/o the
+     * exec areas write lock so we're ok there.
+     */
+    ASSERT(dynamo_all_threads_synched ||
+           (!TEST(VM_EXECUTED_FROM, area->vm_flags) &&
+            READWRITE_LOCK_HELD(&executable_areas->lock)));
+    if (!if_invalid && TEST(PERSCACHE_CODE_INVALID, info->flags)) {
+        /* Don't change bounds of primary or secondary; we expect vm_area_t to
+         * be merged back to this size post-rebind; if not, we'll throw out this
+         * pcache at validation time due to not matching the vm_area_t.
+         */
+        return;
+    }
+    while (info != NULL) { /* loop over primary and secondary unit */
+        /* We should have reset this coarse info when flushing */
+        ASSERT(info->cache == NULL);
+        ASSERT(!info->frozen && !info->persisted);
+        /* No longer covers the removed region */
+        if (info->base_pc < area->start)
+            info->base_pc = area->start;
+        if (info->end_pc > area->end)
+            info->end_pc = area->end;
+        ASSERT(info->frozen || info->non_frozen == NULL);
+        info = info->non_frozen;
+        ASSERT(info == NULL || !info->frozen);
+    }
+}
+
+/* Assumes caller holds v->lock, if necessary
+ * Returns false if no area contains start..end
+ * Ignores type of area -- removes all within start..end
+ * Caller should probably clear last_area as well
+ */
+static bool
+remove_vm_area(vm_area_vector_t *v, app_pc start, app_pc end, bool restore_prot)
+{
+    int i, diff;
+    int overlap_start = -1, overlap_end = -1;
+    bool add_new_area = false;
+    vm_area_t new_area = {0};     /* used only when add_new_area, wimpy compiler */
+    /* FIXME: cleaner test? shared_data copies flags, but uses
+     * custom.frags and not custom.client
+     */
+    bool official_coarse_vector = (v == executable_areas);
+
+    ASSERT_VMAREA_VECTOR_PROTECTED(v, WRITE);
+    LOG(GLOBAL, LOG_VMAREAS, 4, "in remove_vm_area "PFX" "PFX"\n", start, end);
+    /* N.B.: removed area could span multiple areas! */
+    for (i = 0; i < v->length; i++) {
+        /* look for overlap */
+        if (start < v->buf[i].end && end > v->buf[i].start) {
+            if (overlap_start == -1)
+                overlap_start = i;
+        } else if (overlap_start > -1) {
+            overlap_end = i; /* not inclusive */
+            break;
+        } else if (end <= v->buf[i].start)
+            break;
+    }
+    if (overlap_start == -1)
+        return false;
+    if (overlap_end == -1)
+        overlap_end = v->length;
+    /* since it's sorted and there are no overlaps, we do not have to re-sort.
+     * we just delete entire intervals affected, and shorten non-entire
+     */
+    if (start > v->buf[overlap_start].start) {
+        /* need to split? */
+        if (overlap_start == overlap_end-1 && end < v->buf[overlap_start].end) {
+            /* don't call add_vm_area now, that will mess up our vector */
+            new_area = v->buf[overlap_start]; /* make a copy */
+            new_area.start = end;
+            /* rest of fields are correct */
+            add_new_area = true;
+        }
+        /* move ending bound backward */
+        LOG(GLOBAL, LOG_VMAREAS, 3, "\tchanging "PFX"-"PFX" to "PFX"-"PFX"\n",
+            v->buf[overlap_start].start, v->buf[overlap_start].end,
+            v->buf[overlap_start].start, start);
+        if (restore_prot && TEST(VM_MADE_READONLY, v->buf[overlap_start].vm_flags)) {
+            vm_make_writable(start, end - start);
+        }
+        v->buf[overlap_start].end = start;
+        if (TEST(FRAG_COARSE_GRAIN, v->buf[overlap_start].frag_flags) &&
+            official_coarse_vector) {
+            adjust_coarse_unit_bounds(&v->buf[overlap_start], false/*leave invalid*/);
+        }
+        overlap_start++; /* don't delete me */
+    }
+    if (end < v->buf[overlap_end-1].end) {
+        /* move starting bound forward */
+        LOG(GLOBAL, LOG_VMAREAS, 3, "\tchanging "PFX"-"PFX" to "PFX"-"PFX"\n",
+            v->buf[overlap_end-1].start, v->buf[overlap_end-1].end,
+            end, v->buf[overlap_end-1].end);
+        if (restore_prot && TEST(VM_MADE_READONLY, v->buf[overlap_end-1].vm_flags)) {
+            vm_make_writable(v->buf[overlap_end-1].start, end - v->buf[overlap_end-1].start);
+        }
+        v->buf[overlap_end-1].start = end;
+        if (TEST(FRAG_COARSE_GRAIN, v->buf[overlap_end-1].frag_flags) &&
+            official_coarse_vector) {
+            adjust_coarse_unit_bounds(&v->buf[overlap_end-1], false/*leave invalid*/);
+        }
+        overlap_end--; /* don't delete me */
+    }
+    /* now delete */
+    if (overlap_start < overlap_end) {
+        for (i = overlap_start; i < overlap_end; i++) {
+            LOG(GLOBAL, LOG_VMAREAS, 3, "\tcompletely removing "PFX"-"PFX" %s\n",
+                v->buf[i].start, v->buf[i].end, v->buf[i].comment);
+            if (restore_prot && TEST(VM_MADE_READONLY, v->buf[i].vm_flags)) {
+                vm_make_writable(v->buf[i].start, v->buf[i].end - v->buf[i].start);
+            }
+            if (TEST(FRAG_COARSE_GRAIN, v->buf[i].frag_flags) &&
+                official_coarse_vector) {
+                coarse_info_t *info = (coarse_info_t *) v->buf[i].custom.client;
+                coarse_info_t *next_info;
+                ASSERT(info != NULL);
+                ASSERT(!RUNNING_WITHOUT_CODE_CACHE());
+                while (info != NULL) { /* loop over primary and secondary unit */
+                    ASSERT(info->base_pc >= v->buf[i].start &&
+                           info->end_pc <= v->buf[i].end);
+                    ASSERT(info->frozen || info->non_frozen == NULL);
+                    /* Should have already freed fields, unless we flushed a region
+                     * that has not been executed from (case 10995): in which case
+                     * we must delay as we cannot grab change_linking_lock or
+                     * special_heap_lock or info->lock while holding exec_areas lock.
+                     */
+                    if (info->cache != NULL) {
+                        ASSERT(info->persisted);
+                        ASSERT(!TEST(VM_EXECUTED_FROM, v->buf[i].vm_flags));
+                        ASSERT(info->non_frozen != NULL);
+                        ASSERT(coarse_to_delete != NULL);
+                        /* Both primary and secondary must be un-executed */
+                        info->non_frozen->non_frozen = *coarse_to_delete;
+                        *coarse_to_delete = info;
+                        info = NULL;
+                    } else {
+                        ASSERT(info->cache == NULL && info->stubs == NULL);
+                        next_info = info->non_frozen;
+                        coarse_unit_free(GLOBAL_DCONTEXT, info);
+                        info = next_info;
+                        ASSERT(info == NULL || !info->frozen);
+                    }
+                }
+                v->buf[i].custom.client = NULL;
+            }
+#ifdef DEBUG
+            global_heap_free(v->buf[i].comment, strlen(v->buf[i].comment)+1
+                             HEAPACCT(ACCT_VMAREAS));
+#endif
+            /* frags list should always be null here (flush should have happened,
+             * etc.) */
+            ASSERT(!TEST(VECTOR_FRAGMENT_LIST, v->flags) || v->buf[i].custom.frags == NULL);
+        }
+        diff = overlap_end - overlap_start;
+        for (i = overlap_start; i < v->length-diff; i++)
+            v->buf[i] = v->buf[i+diff];
+#ifdef DEBUG
+        memset(v->buf + v->length - diff, 0, diff * sizeof(vm_area_t));
+#endif
+        v->length -= diff;
+    }
+    if (add_new_area) {
+        /* Case 8640: Do not propagate coarse-grain-ness to split-off region,
+         * for now only for simplicity.  FIXME: come up with better policy.  We
+         * do keep it on original part of split region.  FIXME: assert that
+         * there the unit is fully flushed.  Better to remove in
+         * vm_area_allsynch_flush_fragments() and then re-add if warranted?
+         */
+        new_area.frag_flags &= ~FRAG_COARSE_GRAIN;
+        /* With flush of partial module region w/o remove (e.g., from
+         * -unsafe_ignore_IAT_writes) we can have VM_ADD_TO_SHARED_DATA set
+         */
+        new_area.vm_flags &= ~VM_ADD_TO_SHARED_DATA;
+        LOG(GLOBAL, LOG_VMAREAS, 3, "\tadding "PFX"-"PFX"\n", new_area.start, new_area.end);
+        add_vm_area(v, new_area.start, new_area.end, new_area.vm_flags,
+                    new_area.frag_flags _IF_DEBUG(new_area.comment));
+    }
+    DOLOG(5, LOG_VMAREAS, { print_vm_areas(v, GLOBAL); });
+    return true;
+}
+
+/* Returns true if start..end overlaps any area in v.
+ * If end==NULL, assumes that end is very top of address space (wraparound).
+ * If area!=NULL, sets *area to an overlapping area in v
+ *   If index!=NULL, sets *index to the vector index of area; if no match
+ *   is found, sets *index to the index before [start,end) (may be -1).
+ *   If first, makes sure *area is the 1st overlapping area
+ * Assumes caller holds v->lock, if necessary
+ * N.B.: the pointer returned by this routine is volatile!  Only use it while
+ * you have exclusive control over the vector v, either by holding its lock
+ * or by being its owning thread if it has no lock.
+ */
+static bool
+binary_search(vm_area_vector_t *v, app_pc start, app_pc end, vm_area_t **area/*OUT*/,
+              int *index/*OUT*/, bool first)
+{
+    /* BINARY SEARCH -- assumes the vector is kept sorted by add & remove! */
+    int min = 0;
+    int max = v->length - 1;
+
+    ASSERT(start < end || end == NULL /* wraparound */);
+
+    ASSERT_VMAREA_VECTOR_PROTECTED(v, READWRITE);
+    LOG(GLOBAL, LOG_VMAREAS, 7, "Binary search for "PFX"-"PFX" on this vector:\n",
+        start, end);
+    DOLOG(7, LOG_VMAREAS, { print_vm_areas(v, GLOBAL); });
+    /* binary search */
+    while (max >= min) {
+        int i = (min + max) / 2;
+        if (end != NULL && end <= v->buf[i].start)
+            max = i - 1;
+        else if (start >= v->buf[i].end)
+            min = i + 1;
+        else {
+            if (area != NULL || index != NULL) {
+                if (first) {
+                    /* caller wants 1st matching area */
+                    for (; i >= 1 && v->buf[i-1].end > start; i--)
+                        ;
+                }
+                /* returning pointer to volatile array dangerous -- see comment above */
+                if (area != NULL)
+                    *area = &(v->buf[i]);
+                if (index != NULL)
+                    *index = i;
+            }
+            LOG(GLOBAL, LOG_VMAREAS, 7, "\tfound "PFX"-"PFX" in area "PFX"-"PFX"\n",
+                start, end, v->buf[i].start, v->buf[i].end);
+            return true;
+        }
+    }
+    /* now max < min */
+    LOG(GLOBAL, LOG_VMAREAS, 7, "\tdid not find "PFX"-"PFX"!\n", start, end);
+    if (index != NULL) {
+        ASSERT((max < 0 || v->buf[max].end <= start) &&
+               (min > v->length - 1 || v->buf[min].start >= end));
+        *index = max;
+    }
+    return false;
+}
+
+/* lookup an addr in the current area
+ * RETURN true if address area is found, false otherwise
+ * if area is non NULL it is set to the area found
+ * Assumes caller holds v->lock, if necessary
+ * N.B.: the pointer returned by this routine is volatile!  Only use it while
+ * you have exclusive control over the vector v, either by holding its lock
+ * or by being its owning thread if it has no lock.
+ */
+/* FIXME: change lookup_addr to two routines, one for readers which
+ * returns a copy, and the other for writers who must hold a lock
+ * across all uses of the pointer
+ */
+static bool
+lookup_addr(vm_area_vector_t *v, app_pc addr, vm_area_t **area)
+{
+    /* binary search asserts v is protected */
+    return binary_search(v, addr, addr+1/*open end*/, area, NULL, false);
+}
+
+/* returns true if the passed in area overlaps any known executable areas
+ * Assumes caller holds v->lock, if necessary
+ */
+static bool
+vm_area_overlap(vm_area_vector_t *v, app_pc start, app_pc end)
+{
+    /* binary search asserts v is protected */
+    return binary_search(v, start, end, NULL, NULL, false);
+}
+
+/*********************** EXPORTED ROUTINES **********************/
+
+/* thread-shared initialization that should be repeated after a reset */
+void
+vm_areas_reset_init(void)
+{
+    memset(shared_data, 0, sizeof(*shared_data));
+    VMVECTOR_INITIALIZE_VECTOR(&shared_data->areas,
+                               VECTOR_SHARED | VECTOR_FRAGMENT_LIST, shared_vm_areas);
+}
+
+void
+dynamo_vm_areas_init()
+{
+    VMVECTOR_ALLOC_VECTOR(dynamo_areas, GLOBAL_DCONTEXT, VECTOR_SHARED,
+                          dynamo_areas);
+}
+
+/* calls find_executable_vm_areas to get per-process map 
+ * N.B.: add_dynamo_vm_area can be called before this init routine!
+ * N.B.: this is called after vm_areas_thread_init()
+ */
+int
+vm_areas_init()
+{
+    int areas;
+
+    /* Case 7957: we allocate all vm vectors on the heap for self-prot reasons.
+     * We're already paying the indirection cost by passing their addresses
+     * to generic routines, after all.
+     */
+    VMVECTOR_ALLOC_VECTOR(executable_areas, GLOBAL_DCONTEXT, VECTOR_SHARED,
+                          executable_areas);
+    VMVECTOR_ALLOC_VECTOR(pretend_writable_areas, GLOBAL_DCONTEXT, VECTOR_SHARED,
+                          pretend_writable_areas);
+    VMVECTOR_ALLOC_VECTOR(patch_proof_areas, GLOBAL_DCONTEXT, VECTOR_SHARED,
+                          patch_proof_areas);
+    VMVECTOR_ALLOC_VECTOR(emulate_write_areas, GLOBAL_DCONTEXT, VECTOR_SHARED,
+                          emulate_write_areas);
+    VMVECTOR_ALLOC_VECTOR(IAT_areas, GLOBAL_DCONTEXT, VECTOR_SHARED,
+                          IAT_areas);
+    VMVECTOR_ALLOC_VECTOR(written_areas, GLOBAL_DCONTEXT,
+                          VECTOR_SHARED | VECTOR_NEVER_MERGE,
+                          written_areas);
+#ifdef PROGRAM_SHEPHERDING
+    VMVECTOR_ALLOC_VECTOR(futureexec_areas, GLOBAL_DCONTEXT, VECTOR_SHARED,
+                          futureexec_areas);
+# ifdef WINDOWS
+    VMVECTOR_ALLOC_VECTOR(app_flushed_areas, GLOBAL_DCONTEXT, VECTOR_SHARED,
+                          app_flushed_areas);
+# endif
+#endif
+    VMVECTOR_ALLOC_VECTOR(native_exec_areas, GLOBAL_DCONTEXT, VECTOR_SHARED,
+                          native_exec_areas);
+
+    shared_data = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, thread_data_t, ACCT_VMAREAS, PROTECTED);
+
+    todelete = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, deletion_lists_t, ACCT_VMAREAS, PROTECTED);
+    memset(todelete, 0, sizeof(*todelete));
+
+    coarse_to_delete = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, coarse_info_t *,
+                                       ACCT_VMAREAS, PROTECTED);
+    *coarse_to_delete = NULL;
+
+    if (DYNAMO_OPTION(unloaded_target_exception)) {
+        last_deallocated = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, last_deallocated_t, 
+                                           ACCT_VMAREAS, PROTECTED);
+        memset(last_deallocated, 0, sizeof(*last_deallocated));
+    } else
+        ASSERT(last_deallocated == NULL);
+
+    vm_areas_reset_init();
+
+    /* initialize dynamo list first */
+    LOG(GLOBAL, LOG_VMAREAS, 2,
+        "\n--------------------------------------------------------------------------\n");
+    dynamo_vm_areas_lock();
+    areas = find_dynamo_library_vm_areas();
+    dynamo_vm_areas_unlock();
+
+    /* initialize executable list 
+     * this routine calls app_memory_allocation() w/ dcontext==NULL and so we
+     * won't go adding rwx regions, like the linux stack, to our list, even w/
+     * -executable_if_alloc
+     */
+    areas = find_executable_vm_areas();
+    DOLOG(1, LOG_VMAREAS, {
+        if (areas > 0) {
+            LOG(GLOBAL, LOG_VMAREAS, 1, "\nExecution is allowed in %d areas\n", areas);
+            print_executable_areas(GLOBAL);
+        }
+        LOG(GLOBAL, LOG_VMAREAS, 2,
+            "--------------------------------------------------------------------------\n");
+    });
+
+    return areas;
+}
+
+static void
+vm_areas_statistics()
+{
+#ifdef PROGRAM_SHEPHERDING
+    DOLOG(1, LOG_VMAREAS|LOG_STATS, {
+        uint top; uint bottom;
+        divide_uint64_print(GLOBAL_STAT(looked_up_in_last_area), 
+                            GLOBAL_STAT(checked_addresses), true, 2, &top, &bottom);
+        LOG(GLOBAL, LOG_VMAREAS|LOG_STATS, 1,
+            "Code Origin: %d address lookups, %d in last area, hit ratio %u.%.2u\n",
+            GLOBAL_STAT(checked_addresses), GLOBAL_STAT(looked_up_in_last_area),
+            top, bottom);
+    });
+#endif /* PROGRAM_SHEPHERDING */
+    DOLOG(1, LOG_VMAREAS, {
+        LOG(GLOBAL, LOG_VMAREAS, 1, "\nexecutable_areas at exit:\n");
+        print_executable_areas(GLOBAL);
+    });
+}
+
+/* Free all thread-shared state not critical to forward progress;
+ * vm_areas_reset_init() will be called before continuing.
+ */
+void
+vm_areas_reset_free(void)
+{
+    if (SHARED_FRAGMENTS_ENABLED()) {
+        /* all deletion entries should be removed in fragment_exit(),
+         * else we'd have to free the frags lists and entries here
+         */
+        ASSERT(todelete->shared_delete == NULL);
+        ASSERT(todelete->shared_delete_tail == NULL); 
+        /* FIXME: don't free lock so init has less work */
+        vmvector_free_vector(GLOBAL_DCONTEXT, &shared_data->areas);
+    }
+    /* vm_area_coarse_units_reset_free() is called in fragment_reset_free() */
+}
+
+int
+vm_areas_exit()
+{
+    int i;
+    vm_areas_exited = true;
+    vm_areas_statistics();
+
+    if (DYNAMO_OPTION(thin_client)) {
+        vmvector_delete_vector(GLOBAL_DCONTEXT, dynamo_areas);
+        dynamo_areas = NULL;
+        /* For thin_client none of the following areas should have been
+         * initialized because they aren't used.
+         * FIXME: wonder if I can do something like this for -client and see
+         * what I am using unnecessarily.
+         */
+        ASSERT(shared_data == NULL);
+        ASSERT(todelete == NULL);
+        ASSERT(executable_areas == NULL);
+        ASSERT(pretend_writable_areas == NULL);
+        ASSERT(patch_proof_areas == NULL);
+        ASSERT(emulate_write_areas == NULL);
+        ASSERT(written_areas == NULL);
+#ifdef PROGRAM_SHEPHERDING
+        ASSERT(futureexec_areas == NULL);
+        IF_WINDOWS(ASSERT(app_flushed_areas == NULL);)
+#endif
+        ASSERT(native_exec_areas == NULL);
+        ASSERT(IAT_areas == NULL);
+        return 0;
+    }
+
+    vm_areas_reset_free();
+    DELETE_LOCK(shared_delete_lock);
+    DELETE_LOCK(lazy_delete_lock);
+    ASSERT(todelete->lazy_delete_count == 0);
+    ASSERT(!todelete->move_pending);
+
+    HEAP_TYPE_FREE(GLOBAL_DCONTEXT, shared_data, thread_data_t, ACCT_VMAREAS, PROTECTED);
+    shared_data = NULL;
+
+    HEAP_TYPE_FREE(GLOBAL_DCONTEXT, todelete, deletion_lists_t, ACCT_VMAREAS, PROTECTED);
+    todelete = NULL;
+
+    ASSERT(coarse_to_delete != NULL);
+    /* should be freed immediately after each use, during a no-exec flush */
+    ASSERT(*coarse_to_delete == NULL);
+    HEAP_TYPE_FREE(GLOBAL_DCONTEXT, coarse_to_delete, coarse_info_t *,
+                   ACCT_VMAREAS, PROTECTED);
+
+    if (DYNAMO_OPTION(unloaded_target_exception)) {
+        HEAP_TYPE_FREE(GLOBAL_DCONTEXT, last_deallocated, 
+                       last_deallocated_t, ACCT_VMAREAS, PROTECTED);
+        last_deallocated = NULL;
+    } else
+        ASSERT(last_deallocated == NULL);
+    DELETE_LOCK(last_deallocated_lock);
+
+    vmvector_delete_vector(GLOBAL_DCONTEXT, executable_areas);
+    executable_areas = NULL;
+    DOLOG(1, LOG_VMAREAS, {
+        if (dynamo_areas->buf != NULL) {
+            LOG(GLOBAL, LOG_VMAREAS, 1, "DR regions at exit are:\n");
+            print_dynamo_areas(GLOBAL);
+            LOG(GLOBAL, LOG_VMAREAS, 1, "\n");
+        }
+    });
+    vmvector_delete_vector(GLOBAL_DCONTEXT, dynamo_areas);
+    dynamo_areas = NULL;
+    DOLOG(1, LOG_VMAREAS, {
+        if (written_areas->buf != NULL) {
+            LOG(GLOBAL, LOG_VMAREAS, 1, "Code write and selfmod exec counts:\n");
+            print_written_areas(GLOBAL); 
+            LOG(GLOBAL, LOG_VMAREAS, 1, "\n");
+        }
+    });
+    vmvector_delete_vector(GLOBAL_DCONTEXT, pretend_writable_areas);
+    pretend_writable_areas = NULL;
+    vmvector_delete_vector(GLOBAL_DCONTEXT, patch_proof_areas);
+    patch_proof_areas = NULL;
+    vmvector_delete_vector(GLOBAL_DCONTEXT, emulate_write_areas);
+    emulate_write_areas = NULL;
+
+    for (i = 0; i < written_areas->length; i++) {
+        HEAP_TYPE_FREE(GLOBAL_DCONTEXT,
+                       (ro_vs_sandbox_data_t *) written_areas->buf[i].custom.client,
+                       ro_vs_sandbox_data_t, ACCT_VMAREAS, UNPROTECTED);
+    }
+    vmvector_delete_vector(GLOBAL_DCONTEXT, written_areas);
+    written_areas = NULL;
+
+#ifdef PROGRAM_SHEPHERDING
+    DOLOG(1, LOG_VMAREAS, {
+        if (futureexec_areas->buf != NULL)
+            LOG(GLOBAL, LOG_VMAREAS, 1, "futureexec %d regions at exit are:\n",
+                futureexec_areas->length);
+        print_futureexec_areas(GLOBAL);
+    });
+    vmvector_delete_vector(GLOBAL_DCONTEXT, futureexec_areas);
+    futureexec_areas = NULL;
+    DELETE_LOCK(threads_killed_lock);
+# ifdef WINDOWS
+    ASSERT(DYNAMO_OPTION(xdata_rct) || vmvector_empty(app_flushed_areas));
+    vmvector_delete_vector(GLOBAL_DCONTEXT, app_flushed_areas);
+    app_flushed_areas = NULL;
+# endif
+#endif
+#ifdef SIMULATE_ATTACK
+    DELETE_LOCK(simulate_lock);
+#endif
+    vmvector_delete_vector(GLOBAL_DCONTEXT, native_exec_areas);
+    native_exec_areas = NULL;
+    vmvector_delete_vector(GLOBAL_DCONTEXT, IAT_areas);
+    IAT_areas = NULL;
+    return 0;
+}
+
+void
+vm_areas_thread_reset_init(dcontext_t *dcontext)
+{
+    thread_data_t *data = (thread_data_t *) dcontext->vm_areas_field;
+    memset(dcontext->vm_areas_field, 0, sizeof(thread_data_t));
+    VMVECTOR_INITIALIZE_VECTOR(&data->areas, VECTOR_FRAGMENT_LIST, thread_vm_areas);
+    /* data->areas.lock is never used, but we may want to grab it one day, 
+       e.g. to print other thread areas */
+}
+
+/* N.B.: this is called before vm_areas_init() */
+void
+vm_areas_thread_init(dcontext_t *dcontext)
+{
+    thread_data_t *data = HEAP_TYPE_ALLOC(dcontext, thread_data_t, ACCT_OTHER, PROTECTED);
+    dcontext->vm_areas_field = data;
+    vm_areas_thread_reset_init(dcontext);
+}
+
+void
+vm_areas_thread_reset_free(dcontext_t *dcontext)
+{
+    /* we free the local areas vector so it will match fragments post-reset 
+     * FIXME: put it in nonpersistent heap
+     */
+    thread_data_t *data = (thread_data_t *) dcontext->vm_areas_field;
+    /* yes, we end up using global heap for the thread-local area
+     * vector...not a big deal, but FIXME sometime
+     */
+    vmvector_free_vector(GLOBAL_DCONTEXT, &data->areas);
+}
+
+void
+vm_areas_thread_exit(dcontext_t *dcontext)
+{
+    vm_areas_thread_reset_free(dcontext);
+#ifdef DEBUG
+    /* for non-debug we do fast exit path and don't free local heap */
+    HEAP_TYPE_FREE(dcontext, dcontext->vm_areas_field, thread_data_t, ACCT_OTHER, PROTECTED);
+#endif
+}
+
+/****************************************************************************
+ * external interface to vm_area_vector_t 
+ *
+ * FIXME: add user data field to vector and to add routine
+ * FIXME: have init and destroy routines so don't have to expose
+ *        vm_area_vector_t struct or declare vector in this file
+ */
+
+void
+vmvector_print(vm_area_vector_t *v, file_t outf)
+{
+    bool release_lock; /* 'true' means this routine needs to unlock */
+    LOCK_VECTOR(v, release_lock, read);
+    print_vm_areas(v, outf);
+    UNLOCK_VECTOR(v, release_lock, read);
+}
+
+/* ASSUMPTION: no merges are allowed! */
+void
+vmvector_add(vm_area_vector_t *v, app_pc start, app_pc end, void *data)
+{
+    vm_area_t *a = NULL;
+    bool release_lock; /* 'true' means this routine needs to unlock */
+    DEBUG_DECLARE(bool ok;)
+
+    LOCK_VECTOR(v, release_lock, write);
+    ASSERT_OWN_WRITE_LOCK(SHOULD_LOCK_VECTOR(v), &v->lock);
+    add_vm_area(v, start, end, 0, 0 _IF_DEBUG(""));
+    DEBUG_DECLARE(ok = ) lookup_addr(v, start, &a);
+    ASSERT(ok);
+    /* no merging combined w/ client data! */
+    ASSERT(data == NULL || TEST(VECTOR_NEVER_MERGE, v->flags));
+    if (TEST(VECTOR_NEVER_MERGE, v->flags)) {
+        /* flag region overlap, unless client data is not critical */
+        ASSERT((a->start == start && a->end == end) ||
+               TEST(VECTOR_OVERLAP_NON_CRITICAL, v->flags));
+        a->custom.client = data;
+    }
+    UNLOCK_VECTOR(v, release_lock, write);
+}
+
+bool
+vmvector_remove(vm_area_vector_t *v, app_pc start, app_pc end)
+{
+    bool ok;
+    bool release_lock; /* 'true' means this routine needs to unlock */
+    LOCK_VECTOR(v, release_lock, write);
+    ASSERT_OWN_WRITE_LOCK(SHOULD_LOCK_VECTOR(v), &v->lock);
+    ok = remove_vm_area(v, start, end, false);
+    UNLOCK_VECTOR(v, release_lock, write);
+    return ok;
+}
+
+/* Looks up area encapsulating target pc and removes.
+ * returns true if found and removed, and optional area boundaries are set
+ * returns false if not found
+ */
+bool
+vmvector_remove_containing_area(vm_area_vector_t *v, app_pc pc, 
+                                app_pc *area_start /* OUT optional */, 
+                                app_pc *area_end /* OUT optional */)
+{
+    vm_area_t *a;
+    bool ok;
+    bool release_lock; /* 'true' means this routine needs to unlock */
+
+    /* common path should be to find one, and would need write lock to
+     * remove */
+    LOCK_VECTOR(v, release_lock, write);
+    ASSERT_OWN_WRITE_LOCK(SHOULD_LOCK_VECTOR(v), &v->lock);
+    ok = lookup_addr(v, pc, &a);
+    if (ok) {
+        if (area_start != NULL)
+            *area_start = a->start;
+        if (area_end != NULL)
+            *area_end = a->end;
+        remove_vm_area(v, a->start, a->end, false);
+    }
+    UNLOCK_VECTOR(v, release_lock, write);
+    return ok;
+}
+
+
+bool
+vmvector_overlap(vm_area_vector_t *v, app_pc start, app_pc end)
+{
+    bool overlap;
+    bool release_lock; /* 'true' means this routine needs to unlock */
+    if (vmvector_empty(v))
+        return false;
+    LOCK_VECTOR(v, release_lock, read);
+    ASSERT_OWN_READWRITE_LOCK(SHOULD_LOCK_VECTOR(v), &v->lock);
+    overlap = vm_area_overlap(v, start, end);
+    UNLOCK_VECTOR(v, release_lock, read);
+    return overlap;
+}
+
+/* returns custom data field, or NULL if not found.  NOTE: Access to
+ * custom data needs explicit synchronization in addition to
+ * vm_area_vector_t's locks!
+ */
+void *
+vmvector_lookup(vm_area_vector_t *v, app_pc pc)
+{
+    void *data = NULL;
+    vmvector_lookup_data(v, pc, NULL, NULL, &data);
+    return data;
+}
+  
+/* Looks up if pc is in a vmarea and optionally returns the areas's bounds
+ * and any custom data. NOTE: Access to custom data needs explicit
+ * synchronization in addition to vm_area_vector_t's locks!
+ */
+bool
+vmvector_lookup_data(vm_area_vector_t *v, app_pc pc,
+                     app_pc *start /* OUT */, app_pc *end /* OUT */,
+                     void **data /* OUT */)
+{
+    bool overlap;
+    vm_area_t *area = NULL;
+    bool release_lock; /* 'true' means this routine needs to unlock */
+    
+    LOCK_VECTOR(v, release_lock, read);
+    ASSERT_OWN_READWRITE_LOCK(SHOULD_LOCK_VECTOR(v), &v->lock);
+    overlap = lookup_addr(v, pc, &area);
+    if (overlap) {
+        if (start != NULL)
+            *start = area->start;
+        if (end != NULL)
+            *end = area->end;
+        if (data != NULL)
+            *data = area->custom.client;
+    }
+    UNLOCK_VECTOR(v, release_lock, read);
+    return overlap;
+}
+
+/* Sets custom data field if a vmarea is present. Returns true if found,
+ * false if not found.  NOTE: Access to custom data needs explicit
+ * synchronization in addition to vm_area_vector_t's locks!
+ */
+bool
+vmvector_modify_data(vm_area_vector_t *v, app_pc start, app_pc end, void *data)
+{
+    bool overlap;
+    vm_area_t *area = NULL;
+    bool release_lock; /* 'true' means this routine needs to unlock */
+
+    LOCK_VECTOR(v, release_lock, write);
+    ASSERT_OWN_WRITE_LOCK(SHOULD_LOCK_VECTOR(v), &v->lock);
+    overlap = lookup_addr(v, start, &area);
+    if (overlap && start == area->start && end == area->end)
+        area->custom.client = data;
+    UNLOCK_VECTOR(v, release_lock, write);
+    return overlap;
+}
+
+/* this routine does NOT initialize the rw lock!  use VMVECTOR_INITIALIZE_VECTOR */
+void
+vmvector_init_vector(vm_area_vector_t *v, uint flags)
+{
+    memset(v, 0, sizeof(*v));
+    v->flags = flags;
+}
+
+/* this routine does NOT initialize the rw lock!  use VMVECTOR_ALLOC_VECTOR instead */
+vm_area_vector_t *
+vmvector_create_vector(dcontext_t *dcontext, uint flags)
+{
+    vm_area_vector_t *v =
+        HEAP_TYPE_ALLOC(dcontext, vm_area_vector_t, ACCT_VMAREAS, PROTECTED);
+    vmvector_init_vector(v, flags);
+    return v;
+}
+
+/* frees the fields of vm_area_vector_t v (not v itself) */
+void
+vmvector_reset_vector(dcontext_t *dcontext, vm_area_vector_t *v)
+{
+    DODEBUG({
+        int i;
+        /* walk areas and delete coarse info and comments */
+        for (i = 0; i < v->length; i++) {
+            /* FIXME: this code is duplicated in remove_vm_area() */
+            if (TEST(FRAG_COARSE_GRAIN, v->buf[i].frag_flags) &&
+                /* FIXME: cleaner test? shared_data copies flags, but uses
+                 * custom.frags and not custom.client
+                 */
+                v == executable_areas) {
+                coarse_info_t *info = (coarse_info_t *) v->buf[i].custom.client;
+                coarse_info_t *next_info;
+                ASSERT(!RUNNING_WITHOUT_CODE_CACHE());
+                ASSERT(info != NULL);
+                while (info != NULL) { /* loop over primary and secondary unit */
+                    next_info = info->non_frozen;
+                    ASSERT(info->frozen || info->non_frozen == NULL);
+                    coarse_unit_free(GLOBAL_DCONTEXT, info);
+                    info = next_info;
+                    ASSERT(info == NULL || !info->frozen);
+                }
+                v->buf[i].custom.client = NULL;
+            }
+            global_heap_free(v->buf[i].comment, strlen(v->buf[i].comment)+1
+                             HEAPACCT(ACCT_VMAREAS));
+        }
+    });
+    /* with thread shared cache it is in fact possible to have no thread local vmareas */
+    if (v->buf != NULL) {
+        /* FIXME: walk through and make sure frags lists are all freed */
+        global_heap_free(v->buf, v->size*sizeof(struct vm_area_t) HEAPACCT(ACCT_VMAREAS));
+        v->size = 0;
+        v->length = 0;
+        v->buf = NULL;
+    } else
+        ASSERT(v->size == 0 && v->length == 0);
+}
+
+static void
+vmvector_free_vector(dcontext_t *dcontext, vm_area_vector_t *v)
+{
+    vmvector_reset_vector(dcontext, v);
+    DELETE_READWRITE_LOCK(v->lock);
+}
+
+/* frees the vm_area_vector_t v and its associated memory */
+void
+vmvector_delete_vector(dcontext_t *dcontext, vm_area_vector_t *v)
+{
+    vmvector_free_vector(dcontext, v);
+    HEAP_TYPE_FREE(dcontext, v, vm_area_vector_t, ACCT_VMAREAS, PROTECTED);
+}
+
+/* vmvector iterator */
+/* initialize an iterator, has to be released with
+ * vmvector_iterator_stop.  The iterator doesn't support mutations.
+ * In fact shared vectors should detect a deadlock
+ * if vmvector_add() and vmvector_remove() is erroneously called.
+ */
+void
+vmvector_iterator_start(vm_area_vector_t *v, vmvector_iterator_t *vmvi)
+{
+    ASSERT(v != NULL);
+    ASSERT(vmvi != NULL);
+    if (SHOULD_LOCK_VECTOR(v))
+        read_lock(&v->lock);
+    vmvi->vector = v;
+    vmvi->index = -1;
+}
+
+bool
+vmvector_iterator_hasnext(vmvector_iterator_t *vmvi)
+{
+    ASSERT_VMAREA_VECTOR_PROTECTED(vmvi->vector, READWRITE);
+    return (vmvi->index + 1) < vmvi->vector->length;
+}
+
+/* iterator accessor
+ * has to be initialized with vmvector_iterator_start, and should be
+ * called only when vmvector_iterator_hasnext() is true
+ *
+ * returns custom data and 
+ * sets the area boundaries in area_start and area_end 
+ */
+void*
+vmvector_iterator_next(vmvector_iterator_t *vmvi, /* IN/OUT */
+                       app_pc *area_start /* OUT */, app_pc *area_end /* OUT */)
+{
+    ASSERT(vmvector_iterator_hasnext(vmvi));
+    ASSERT_VMAREA_VECTOR_PROTECTED(vmvi->vector, READWRITE);
+    vmvi->index++;
+    ASSERT(vmvi->index < vmvi->vector->length);
+    *area_start = vmvi->vector->buf[vmvi->index].start;
+    *area_end = vmvi->vector->buf[vmvi->index].end;
+    return vmvi->vector->buf[vmvi->index].custom.client;
+}
+
+void
+vmvector_iterator_stop(vmvector_iterator_t *vmvi)
+{
+    ASSERT_VMAREA_VECTOR_PROTECTED(vmvi->vector, READWRITE);
+    if (SHOULD_LOCK_VECTOR(vmvi->vector))
+        read_unlock(&vmvi->vector->lock);
+    DODEBUG({
+        vmvi->vector = NULL;    /* crash incorrect reuse */
+        vmvi->index = -1;
+    });
+}
+
+/****************************************************************************
+ * routines specific to our own vectors
+ */
+
+void
+print_executable_areas(file_t outf)
+{
+    vmvector_print(executable_areas, outf);
+}
+
+void
+print_dynamo_areas(file_t outf)
+{
+    dynamo_vm_areas_start_reading();
+    print_vm_areas(dynamo_areas, outf);
+    dynamo_vm_areas_done_reading();
+}
+
+#ifdef PROGRAM_SHEPHERDING
+void
+print_futureexec_areas(file_t outf)
+{
+    vmvector_print(futureexec_areas, outf);
+}
+#endif
+
+#if defined(DEBUG) && defined(INTERNAL)
+static void
+print_written_areas(file_t outf)
+{
+    vmvector_print(written_areas, outf);
+}
+#endif
+
+/* Functions as a lookup routine if an entry is already present.
+ * Returns true if an entry was already present, false if not, in which
+ * case an entry for [start, end) is added.
+ */
+static bool
+add_written_area(vm_area_vector_t *v, app_pc tag, app_pc start,
+                 app_pc end, vm_area_t **area)
+{
+    vm_area_t *a = NULL;
+    bool already;
+    DEBUG_DECLARE(bool ok;)
+    /* currently only one vector */
+    ASSERT(v == written_areas);
+    ASSERT_OWN_WRITE_LOCK(true, &v->lock);
+    ASSERT(tag >= start && tag < end);
+    /* re-adding fails for written_areas since no merging, so lookup first */
+    already = lookup_addr(v, tag, &a);
+    if (!already) {
+        LOG(GLOBAL, LOG_VMAREAS, 2,
+            "new written executable vm area: "PFX"-"PFX"\n",
+            start, end);
+        /* case 9179: With no flags, any overlap (in non-tag portion of [start,
+         * end)) will result in a merge: so we'll inherit and share counts from
+         * any adjacent region(s): maybe better to split?  Rare in any case and
+         * not critical.  In case of simultaneous overlap, we take counter from
+         * first region, since that's how add_vm_area does the merge.
+         */
+        add_vm_area(v, start, end, /* no flags */ 0, 0 _IF_DEBUG(""));
+        DEBUG_DECLARE(ok = ) lookup_addr(v, tag, &a);
+        ASSERT(ok && a != NULL);
+        /* If we merged, we already have an ro2s struct */
+        if (a->custom.client == NULL) {
+            /* Since selfmod_execs is written from the cache this must be
+             * unprotected.  Attacker changing selfmod_execs or written_count
+             * shouldn't be able to cause problems.
+             */
+            ro_vs_sandbox_data_t *ro2s =
+                HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, ro_vs_sandbox_data_t,
+                                ACCT_VMAREAS, UNPROTECTED);
+            /* selfmod_execs is inc-ed from the cache, and if it crosses a cache
+             * line we could have a problem with large thresholds.  We assert on
+             * 32-bit alignment here, which our heap alloc currently provides, to
+             * ensure no cache line is crossed.
+             */
+            ASSERT(ALIGNED(ro2s, sizeof(uint)));
+            memset(ro2s, 0, sizeof(*ro2s));
+            a->custom.client = (void *) ro2s;
+        }
+    } else {
+        LOG(GLOBAL, LOG_VMAREAS, 3,
+            "request for written area "PFX"-"PFX" vs existing "PFX"-"PFX"\n",
+            start, end, a->start, a->end);
+    }
+    ASSERT(a != NULL);
+    if (area != NULL)
+        *area = a;
+    return already;
+}
+
+#ifdef WINDOWS
+/* Adjusts a new executable area with respect to the IAT.
+ * Returns whether it should remain coarse or not.
+ */
+static bool
+add_executable_vm_area_check_IAT(app_pc *start /*IN/OUT*/, app_pc *end /*IN/OUT*/,
+                                 uint vm_flags,
+                                 vm_area_t **existing_area /*OUT*/,
+                                 coarse_info_t **info_out /*OUT*/,
+                                 coarse_info_t **tofree /*OUT*/,
+                                 app_pc *delay_start /*OUT*/,
+                                 app_pc *delay_end /*OUT*/)
+{
+    bool keep_coarse = false;
+    app_pc IAT_start = NULL, IAT_end = NULL;
+    app_pc orig_start = *start, orig_end = *end;
+    ASSERT(existing_area != NULL && info_out != NULL && tofree != NULL);
+    ASSERT(delay_start != NULL && delay_end != NULL);
+    if (DYNAMO_OPTION(coarse_merge_iat) &&
+        get_module_base(*start) != NULL &&
+        get_IAT_section_bounds(get_module_base(*start), &IAT_start, &IAT_end) &&
+        /* case 1094{5,7}: to match the assumptions of case 10600 we adjust
+         * to post-IAT even if the IAT is in the middle, if it's toward the front
+         */
+        (*start >= IAT_start || (IAT_start - *start < *end - IAT_end)) &&
+        *start < IAT_end &&
+        /* be paranoid: multi-page IAT where hooker fooled our loader matching
+         * could add just 1st page of IAT? */
+        *end > IAT_end /* for == avoid an empty region */) {
+        /* If a pre-IAT region exists, split if off separately (case 10945).
+         * We want to keep as coarse, but we need the post-IAT region to be the
+         * primary coarse and the one we try to load a pcache for: so we delay
+         * the add.
+         * FIXME: should we do a general split around the IAT and make both sides
+         * coarse with larger the primary instead of assuming pre-IAT is smaller?
+         */
+        if (orig_start < IAT_start) {
+            LOG(GLOBAL, LOG_VMAREAS, 2,
+                "splitting pre-IAT "PFX"-"PFX" off from exec area "PFX"-"PFX"\n",
+                orig_start, IAT_start, orig_start, orig_end);
+            *delay_start = orig_start;
+            *delay_end = IAT_start;
+            DODEBUG({
+                /* When IAT is in the middle of +rx region we expect .orpc */
+                app_pc orpc_start = NULL;
+                app_pc orpc_end = NULL;
+                get_named_section_bounds(get_module_base(orig_start), ".orpc",
+                                         &orpc_start, &orpc_end);
+                ASSERT_CURIOSITY(orpc_start == orig_start && orpc_end == IAT_start);
+            });
+        }
+        /* Just abandon [*start, IAT_end) */
+        *start = IAT_end;
+        ASSERT(*end > *start);
+        LOG(GLOBAL, LOG_VMAREAS, 2,
+            "adjusting exec area "PFX"-"PFX" to post-IAT "PFX"-"PFX"\n",
+            orig_start, *end, *start, *end);
+    } else {
+        LOG(GLOBAL, LOG_VMAREAS, 2,
+            "NOT adjusting exec area "PFX"-"PFX" vs IAT "PFX"-"PFX"\n",
+            orig_start, *end, IAT_start, IAT_end);
+    }
+    if (TEST(VM_UNMOD_IMAGE, vm_flags))
+        keep_coarse = true;
+    else {
+        /* Keep the coarse-grain flag for modified pages only if IAT pages.
+         * We want to avoid repeated coarse flushes, so we are
+         * very conservative about marking if_rx_text regions coarse: we count on
+         * our IAT loader check to make this a do-once.
+         * FIXME: Should extend this to also merge on left with .orpc:
+         * .orpc at page 1, IAT on page 2, and .text continuing on
+         */
+        ASSERT(ALIGNED(*end, PAGE_SIZE));
+        if (DYNAMO_OPTION(coarse_merge_iat) &&
+            vm_flags == 0 /* no other flags */ &&
+            /* FIXME: used our stored bounds */
+            is_IAT(orig_start, orig_end, true/*page-align*/, NULL, NULL) &&
+            is_module_patch_region(GLOBAL_DCONTEXT, orig_start, orig_end,
+                                   true/*be conservative*/) &&
+            /* We stored the IAT code at +rw time */
+            os_module_cmp_IAT_code(orig_start)) {
+            vm_area_t *area = NULL;
+            bool all_new = !executable_vm_area_overlap(orig_start, orig_end-1,
+                                                       true/*wlock*/);
+            ASSERT(IAT_start != NULL); /* should have found bounds above */
+            if (all_new && /* elseif assumes next call happened */
+                lookup_addr(executable_areas, *end, &area) &&
+                TEST(FRAG_COARSE_GRAIN, area->frag_flags) &&
+                /* Only merge if no execution has yet occurred: else this
+                 * must not be normal rebinding */
+                !TEST(VM_EXECUTED_FROM, area->vm_flags) &&
+                /* Should be marked invalid; else no loader +rw => not rebinding */
+                area->custom.client != NULL &&
+                TEST(PERSCACHE_CODE_INVALID,
+                     ((coarse_info_t *)area->custom.client)->flags)) {
+                /* Case 8640: merge IAT page back in to coarse area.
+                 * Easier to merge here than in add_vm_area.
+                 */
+                coarse_info_t *info = (coarse_info_t *) area->custom.client;
+                keep_coarse = true;
+                LOG(GLOBAL, LOG_VMAREAS, 2,
+                    "merging post-IAT ("PFX"-"PFX") with "PFX"-"PFX"\n",
+                    IAT_end, orig_end, area->start, area->end);
+                ASSERT(area != NULL);
+                ASSERT(area->start == *end);
+                ASSERT(IAT_end > orig_start && IAT_end < area->start);
+                ASSERT(*start == IAT_end); /* set up above */
+                *end = area->end;
+                area->start = *start;
+                *existing_area = area;
+                STATS_INC(coarse_merge_IAT);
+                /* If info was loaded prior to rebinding just use it.
+                 * Else, we need a fresh coarse_info_t if persisted, so rather than
+                 * adjust_coarse_unit_bounds on info we must free it.
+                 * Due to lock constraints we can't do that while holding
+                 * exec areas lock.
+                 */
+                /* Bounds should match exactly, since we did not adjust them
+                 * on the flush; if they don't, don't use the pcache. */
+                if (info->base_pc == area->start && info->end_pc == area->end) {
+                    info->flags &= ~PERSCACHE_CODE_INVALID;
+                    *info_out = info;
+                    STATS_INC(coarse_marked_valid);
+                    LOG(GLOBAL, LOG_VMAREAS, 2,
+                        "\tkeeping now-valid info %s "PFX"-"PFX"\n",
+                        info->module, info->base_pc, info->end_pc);
+                } else {
+                    /* Go ahead and merge, but don't use this pcache */
+                    ASSERT_CURIOSITY(false && "post-rebind pcache bounds mismatch");
+                    *tofree = info;
+                    area->custom.client = NULL;
+                    /* FIXME: we'll try to load again: prevent that?  We
+                     * know the image hasn't been modified so no real danger. */
+                    STATS_INC(perscache_rebind_load);
+                }
+            } else if (all_new && area == NULL /*nothing following*/) {
+                /* Code section is a single page, so was completely flushed
+                 * We'll try to re-load the pcache.
+                 * FIXME: we already merged the persisted rct tables into
+                 * the live tables when we flushed the pcache: so now
+                 * we'll have redundancy, and if we flush again we'll waste
+                 * time tryingn to re-add (we do check for dups).
+                 */
+                ASSERT(!lookup_addr(executable_areas, *start, NULL));
+                LOG(GLOBAL, LOG_VMAREAS, 2,
+                    "marking IAT/code region ("PFX"-"PFX" vs "PFX"-"PFX") as coarse\n",
+                    IAT_start, IAT_end, orig_start, orig_end);
+                keep_coarse = true;
+                STATS_INC(coarse_merge_IAT); /* we use same stat */
+            } else {
+                LOG(GLOBAL, LOG_VMAREAS, 2,
+                    "NOT merging IAT-containing "PFX"-"PFX": abuts non-inv-coarse\n",
+                    orig_start, orig_end);
+                DODEBUG({
+                    if (all_new && area != NULL &&
+                        TEST(FRAG_COARSE_GRAIN, area->frag_flags) &&
+                        TEST(VM_EXECUTED_FROM, area->vm_flags)) {
+                        coarse_info_t *info = (coarse_info_t *) area->custom.client;
+                        ASSERT(!info->persisted);
+                        ASSERT(!TEST(PERSCACHE_CODE_INVALID, info->flags));
+                    }
+                });
+            }
+        } else {
+            LOG(GLOBAL, LOG_VMAREAS, 2,
+                "NOT merging .text "PFX"-"PFX" vs IAT "PFX"-"PFX" %d %d %d %d %d\n",
+                orig_start, orig_end, IAT_start, IAT_end,
+                DYNAMO_OPTION(coarse_merge_iat), vm_flags == 0,
+                is_IAT(orig_start, *end, true/*page-align*/, NULL, NULL),
+                is_module_patch_region(GLOBAL_DCONTEXT, orig_start, orig_end,
+                                       true/*be conservative*/),
+                os_module_cmp_IAT_code(orig_start));
+        }
+    }
+    return keep_coarse;
+}
+#endif
+
+static void
+add_executable_vm_area_helper(app_pc start, app_pc end, uint vm_flags, uint frag_flags,
+                              coarse_info_t *info _IF_DEBUG(char *comment))
+{
+    ASSERT_OWN_WRITE_LOCK(true, &executable_areas->lock);
+
+    add_vm_area(executable_areas, start, end,
+                vm_flags, frag_flags _IF_DEBUG(comment));
+
+    if (TEST(VM_WRITABLE, vm_flags)) {
+        /* N.B.: the writable flag indicates the natural state of the memory,
+         * not what we have made it be -- we make it read-only before adding
+         * to the executable list!
+         * FIXME: win32 callback's intercept_call code appears in fragments
+         * and is writable...would like to fix that, and coalesce that memory
+         * with the generated routines or something
+         */
+        LOG(GLOBAL, LOG_VMAREAS, 2,
+            "WARNING: new executable vm area is writable: "PFX"-"PFX" %s\n",
+            start, end, comment);
+#if 0
+        /* this syslog causes services.exe to hang (ref case 666) once case 666
+         * is fixed re-enable if desired FIXME */
+        SYSLOG_INTERNAL_WARNING_ONCE("new executable vm area is writable.");
+#endif
+    }
+#ifdef PROGRAM_SHEPHERDING
+    if (!DYNAMO_OPTION(selfmod_futureexec) &&
+        TEST(FRAG_SELFMOD_SANDBOXED, frag_flags)) {
+        /* We do not need future entries for selfmod regions.  We mark
+         * the futures as once-only when they are selfmod at future add time, and
+         * here we catch those who weren't selfmod then but are now.
+         */
+        remove_futureexec_vm_area(start, end);
+    }
+#endif
+    if (TEST(FRAG_COARSE_GRAIN, frag_flags)) {
+        vm_area_t *area = NULL;
+        DEBUG_DECLARE(bool found = )
+            lookup_addr(executable_areas, start, &area);
+        ASSERT(found && area != NULL);
+        /* case 9521: always have one non-frozen coarse unit per coarse region */
+        if (info == NULL || info->frozen) {
+            coarse_info_t *new_info = coarse_unit_create(start, end,
+                                                         (info == NULL) ? NULL :
+                                                         &info->module_md5,
+                                                         true/*for execution*/);
+            LOG(GLOBAL, LOG_VMAREAS, 1, "new %scoarse unit %s "PFX"-"PFX"\n",
+                info == NULL ? "" : "secondary ", new_info->module, start, end);
+            if (info == NULL)
+                info = new_info;
+            else
+                info->non_frozen = new_info;
+        }
+        area->custom.client = (void *) info;
+    }
+    DOLOG(2, LOG_VMAREAS, {
+        /* new area could have been split into multiple */
+        print_contig_vm_areas(executable_areas, start, end, GLOBAL, "new executable vm area: ");
+    });
+}
+
+/* NOTE : caller is responsible for ensuring that consistency conditions are
+ * met, thus if the region is writable the caller must either mark it read
+ * only or pass in the VM_DELAY_READONLY flag in which case 
+ * check_thread_vm_area will mark it read only when a thread goes to build a 
+ * block from the region */
+static bool
+add_executable_vm_area(app_pc start, app_pc end, uint vm_flags, uint frag_flags,
+                       bool have_writelock _IF_DEBUG(char *comment))
+{
+    vm_area_t *existing_area = NULL;
+    coarse_info_t *info = NULL;
+    coarse_info_t *tofree = NULL;
+    app_pc delay_start = NULL, delay_end = NULL;
+    /* only expect to see the *_READONLY flags on WRITABLE regions */
+    ASSERT(!TEST(VM_DELAY_READONLY, vm_flags) ||
+           TEST(VM_WRITABLE, vm_flags));
+    ASSERT(!TEST(VM_MADE_READONLY, vm_flags) ||
+           TEST(VM_WRITABLE, vm_flags));
+#ifdef DEBUG /* can't use DODEBUG b/c of ifdef inside */
+    {
+        /* we only expect certain flags */
+        uint expect = VM_WRITABLE|VM_UNMOD_IMAGE|VM_MADE_READONLY|
+            VM_DELAY_READONLY|VM_WAS_FUTURE|VM_EXECUTED_FROM|VM_DRIVER_ADDRESS;
+# ifdef PROGRAM_SHEPHERDING
+        expect |= VM_PATTERN_REVERIFY;
+# endif
+        ASSERT(!TESTANY(~expect, vm_flags));
+    }
+#endif /* DEBUG */
+    if (!have_writelock) {
+#ifdef HOT_PATCHING_INTERFACE
+        /* case 9970: need to check hotp vs perscache; rank order hotp < exec_areas */
+        if (DYNAMO_OPTION(hot_patching))
+            read_lock(hotp_get_lock());
+#endif
+        write_lock(&executable_areas->lock);
+    }
+    ASSERT_OWN_WRITE_LOCK(true, &executable_areas->lock);
+    /* FIXME: rather than change all callers who already hold exec_areas lock
+     * to first grab hotp lock, we don't support perscache in those cases.
+     * We expect to only be adding a coarse-grain area for module loads.
+     */
+    ASSERT(!TEST(FRAG_COARSE_GRAIN, frag_flags) || !have_writelock);
+    if (TEST(FRAG_COARSE_GRAIN, frag_flags) && !have_writelock) {
+#ifdef WINDOWS
+        if (!add_executable_vm_area_check_IAT(&start, &end, vm_flags,
+                                              &existing_area, &info, &tofree,
+                                              &delay_start, &delay_end))
+            frag_flags &= ~FRAG_COARSE_GRAIN;
+#else
+        ASSERT(TEST(VM_UNMOD_IMAGE, vm_flags));
+#endif
+        ASSERT(!RUNNING_WITHOUT_CODE_CACHE());
+        if (TEST(FRAG_COARSE_GRAIN, frag_flags) && DYNAMO_OPTION(use_persisted) &&
+            info == NULL) {
+            /* We load persisted cache files at mmap time primarily for RCT
+             * tables; but to avoid duplicated code, and for simplicity, we do
+             * so if -use_persisted even if not -use_persisted_rct.
+             */
+            dcontext_t *dcontext = get_thread_private_dcontext();
+            /* FIXME: we're called before 1st thread is set up.  Only a problem
+             * right now for rac_entries_resurrect() w/ private after-call
+             * which won't happen w/ -coarse_units that requires shared bbs.
+             */
+            info = coarse_unit_load(dcontext == NULL ? GLOBAL_DCONTEXT : dcontext,
+                                    start, end, true/*for execution*/);
+            if (info != NULL) {
+                ASSERT(info->base_pc >= start && info->end_pc <= end);
+                LOG(GLOBAL, LOG_VMAREAS, 1,
+                    "using persisted coarse unit %s "PFX"-"PFX" for "PFX"-"PFX"\n",
+                    info->module, info->base_pc, info->end_pc, start, end);
+                /* Case 8640/9653/8639: adjust region bounds so that a
+                 * cache consistency event outside the persisted region
+                 * does not invalidate it (mainly targeting loader rebinding).
+                 * We count on FRAG_COARSE_GRAIN preventing any merging of regions.
+                 * We could delay this until code validation, as RCT tables don't care,
+                 * and then we could avoid splitting the region in case validation
+                 * fails: but our plan for lazy per-page validation (case 10601)
+                 * means we can fail post-split even that way.  So we go ahead and split
+                 * up front here.  For 4.4 we should move this to 1st exec.
+                 */
+                if (info->base_pc > start) {
+                    add_executable_vm_area_helper(start, info->base_pc,
+                                                  vm_flags, frag_flags, NULL
+                                                  _IF_DEBUG(comment));
+                    start = info->base_pc;
+                }
+                if (info->end_pc < end) {
+                    add_executable_vm_area_helper(info->end_pc, end,
+                                                  vm_flags, frag_flags, NULL
+                                                  _IF_DEBUG(comment));
+                    end = info->end_pc;
+                }
+                ASSERT(info->frozen && info->persisted);
+                vm_flags |= VM_PERSISTED_CACHE;
+                /* For 4.4 we would mark as PERSCACHE_CODE_INVALID here and
+                 * mark valid only at 1st execution when we do md5 checks;
+                 * for 4.3 we're valid until a rebind action.
+                 */
+                ASSERT(!TEST(PERSCACHE_CODE_INVALID, info->flags));
+                /* We must add to shared_data, but we cannot here due to lock
+                 * rank issues (shared_vm_areas lock is higher rank than
+                 * executable_areas, and we have callers doing flushes and
+                 * already holding executable_areas), so we delay.
+                 */
+                vm_flags |= VM_ADD_TO_SHARED_DATA;
+            }
+        }
+    }
+
+    if (existing_area == NULL) {
+        add_executable_vm_area_helper(start, end, vm_flags, frag_flags, info
+                                      _IF_DEBUG(comment));
+    } else {
+        /* we shouldn't need the other parts of _helper() */
+        ASSERT(!TEST(VM_WRITABLE, vm_flags));
+#ifdef PROGRAM_SHEPHERDING
+        ASSERT(DYNAMO_OPTION(selfmod_futureexec) ||
+               !TEST(FRAG_SELFMOD_SANDBOXED, frag_flags));
+#endif
+    }
+
+    if (delay_start != NULL) {
+        ASSERT(delay_end > delay_start);
+        add_executable_vm_area_helper(delay_start, delay_end, vm_flags, frag_flags, NULL
+                                      _IF_DEBUG(comment));
+    }
+
+    DOLOG(2, LOG_VMAREAS, {
+        /* new area could have been split into multiple */
+        print_contig_vm_areas(executable_areas, start, end, GLOBAL, "new executable vm area: ");
+    });
+
+    if (!have_writelock) {
+        write_unlock(&executable_areas->lock);
+#ifdef HOT_PATCHING_INTERFACE
+        if (DYNAMO_OPTION(hot_patching))
+            read_unlock(hotp_get_lock());
+#endif
+    }
+    if (tofree != NULL) {
+        /* Since change_linking_lock and info->lock are higher rank than exec areas we
+         * must free down here.  FIXME: this should move to 1st exec for 4.4.
+         */
+        ASSERT(tofree->non_frozen == NULL);
+        coarse_unit_reset_free(GLOBAL_DCONTEXT, tofree, false/*no locks*/,
+                               true/*unlink*/, true/*give up primary*/);
+        coarse_unit_free(GLOBAL_DCONTEXT, tofree);
+    }
+    return true;
+}
+
+/* Used to add dr allocated memory regions that may execute out of the cache */
+/* NOTE : region is assumed to not be writable, caller is responsible for
+ * ensuring this (see fixme in signal.c adding sigreturn code)
+ */
+bool
+add_executable_region(app_pc start, size_t size _IF_DEBUG(char *comment))
+{
+    return add_executable_vm_area(start, start+size, 0, 0, false/*no lock*/
+                                  _IF_DEBUG(comment));
+}
+
+/* remove an executable area from the area list 
+ * the caller is responsible for ensuring that all threads' local vm lists
+ * are updated by calling flush_fragments_and_remove_region (can't just
+ * remove local vm areas and leave existing fragments hanging...)
+ */
+static bool
+remove_executable_vm_area(app_pc start, app_pc end, bool have_writelock)
+{
+    bool ok;
+    LOG(GLOBAL, LOG_VMAREAS, 2, "removing executable vm area: "PFX"-"PFX"\n",
+        start, end);
+    if (!have_writelock)
+        write_lock(&executable_areas->lock);
+    ok = remove_vm_area(executable_areas, start, end, true/*restore writability!*/);
+    if (!have_writelock)
+        write_unlock(&executable_areas->lock);
+    return ok;
+}
+
+/* removes a region from the executable list */
+/* NOTE :the caller is responsible for ensuring that all threads' local 
+ * vm lists are updated by calling flush_fragments_and_remove_region
+ */
+bool
+remove_executable_region(app_pc start, size_t size, bool have_writelock)
+{
+    return remove_executable_vm_area(start, start+size, have_writelock);
+}
+
+/* case 10995: we have to delay freeing un-executed coarse units until
+ * we can release the exec areas lock when we flush an un-executed region.
+ * This routine frees the queued-up coarse units, and releases the
+ * executable areas lock, which the caller must hold.
+ */
+bool
+free_nonexec_coarse_and_unlock()
+{
+    bool freed_any = false;
+    coarse_info_t *info = NULL;
+    coarse_info_t *next_info;
+    /* We must hold the exec areas lock while traversing the to-delete list,
+     * yet we cannot delete while holding it, so we use a temp var
+     */
+    ASSERT_OWN_WRITE_LOCK(true, &executable_areas->lock);
+    ASSERT(coarse_to_delete != NULL);
+    if (coarse_to_delete != NULL/*paranoid*/ && *coarse_to_delete != NULL) {
+        freed_any = true;
+        info = *coarse_to_delete;
+        *coarse_to_delete = NULL;
+    }
+    /* Now we can unlock, and then it's safe to delete */
+    executable_areas_unlock();
+    if (freed_any) {
+        /* units are chained by non_frozen field */
+        while (info != NULL) {
+            next_info = info->non_frozen;
+            if (info->cache != NULL) {
+                ASSERT(info->persisted);
+                /* We shouldn't need to unlink since no execution has occurred
+                 * (lazy linking)
+                 */
+                ASSERT(info->incoming == NULL);
+                ASSERT(!coarse_unit_outgoing_linked(GLOBAL_DCONTEXT, info));
+            }
+            coarse_unit_reset_free(GLOBAL_DCONTEXT, info,
+                                   false/*no locks*/, false/*!unlink*/,
+                                   true/*give up primary*/);
+            coarse_unit_free(GLOBAL_DCONTEXT, info);
+            info = next_info;
+        }
+    }
+    return freed_any;
+}
+
+#ifdef PROGRAM_SHEPHERDING
+/* add a "future executable area" (e.g., mapped EW) to the future list 
+ *
+ * FIXME: now that this is vmareas.c-internal we should change it to
+ * take in direct VM_ flags, and make separate flags for each future-adding
+ * code origins policy.  Then we can have policy-specific removal from future list.
+ */
+static bool
+add_futureexec_vm_area(app_pc start, app_pc end, bool once_only
+                       _IF_DEBUG(char *comment))
+{
+    /* FIXME: don't add portions that overlap w/ exec areas */
+    LOG(GLOBAL, LOG_VMAREAS, 2, "new FUTURE executable vm area: "PFX"-"PFX" %s%s\n",
+        start, end, (once_only?"ONCE ":""), comment);
+
+    if (DYNAMO_OPTION(unloaded_target_exception)) {
+        /* case 9371 - to avoid possible misclassification in a tight race
+         * between NtUnmapViewOfSection and a consecutive future area
+         * allocated in the same place, we clear our the unload in progress flag
+         */
+        mark_unload_future_added(start, end - start);
+    }
+
+    write_lock(&futureexec_areas->lock);
+    add_vm_area(futureexec_areas, start, end,
+                (once_only ? VM_ONCE_ONLY : 0),
+                0 /* frag_flags */ _IF_DEBUG(comment));
+    write_unlock(&futureexec_areas->lock);
+    return true;
+}
+
+/* remove a "future executable area" from the future list */
+static bool
+remove_futureexec_vm_area(app_pc start, app_pc end)
+{
+    bool ok;
+    LOG(GLOBAL, LOG_VMAREAS, 2, "removing FUTURE executable vm area: "PFX"-"PFX"\n",
+        start, end);
+    write_lock(&futureexec_areas->lock);
+    ok = remove_vm_area(futureexec_areas, start, end, false);
+    write_unlock(&futureexec_areas->lock);
+    return ok;
+}
+
+/* returns true if the passed in area overlaps any known future executable areas */
+static bool
+futureexec_vm_area_overlap(app_pc start, app_pc end)
+{
+    bool overlap;
+    read_lock(&futureexec_areas->lock);
+    overlap = vm_area_overlap(futureexec_areas, start, end);
+    read_unlock(&futureexec_areas->lock);
+    return overlap;
+}
+#endif /* PROGRAM_SHEPHERDING */
+
+/* lookup against the per-process executable addresses map */
+bool
+is_executable_address(app_pc addr)
+{
+    bool found;
+    read_lock(&executable_areas->lock);
+    found = lookup_addr(executable_areas, addr, NULL);
+    read_unlock(&executable_areas->lock);
+    return found;
+}
+
+/* returns any VM_ flags associated with addr's vm area 
+ * returns 0 if no area is found
+ * cf. get_executable_area_flags() for FRAG_ flags
+ */
+bool
+get_executable_area_vm_flags(app_pc addr, uint *vm_flags)
+{
+    bool found = false;
+    vm_area_t *area;
+    read_lock(&executable_areas->lock);
+    if (lookup_addr(executable_areas, addr, &area)) {
+        *vm_flags = area->vm_flags;
+        found = true;
+    }
+    read_unlock(&executable_areas->lock);
+    return found;
+}
+
+/* if addr is an executable area, returns true and returns in *flags
+ *   any FRAG_ flags associated with addr's vm area 
+ * returns false if area not found 
+ *
+ * cf. get_executable_area_vm_flags() for VM_ flags
+ */
+bool
+get_executable_area_flags(app_pc addr, uint *frag_flags)
+{
+    bool found = false;
+    vm_area_t *area;
+    read_lock(&executable_areas->lock);
+    if (lookup_addr(executable_areas, addr, &area)) {
+        *frag_flags = area->frag_flags;
+        found = true;
+    }
+    read_unlock(&executable_areas->lock);
+    return found;
+}
+
+/* For coarse-grain operation, we use a separate cache and htable per region.
+ * See coarse_info_t notes on synchronization model.
+ * Returns NULL when region is not coarse.
+ * Assumption: this routine is called prior to the first execution from a
+ * coarse vm area region.
+ */
+static coarse_info_t *
+get_coarse_info_internal(app_pc addr, bool init, bool have_shvm_lock)
+{
+    coarse_info_t *coarse = NULL;
+    vm_area_t *area = NULL;
+    vm_area_t area_copy = {0,};
+    bool is_coarse = false;
+    bool add_to_shared = false;
+    bool reset_unit = false;
+    /* FIXME perf opt: have a last_area */
+    /* FIXME: could use vmvector_lookup_data() but I need area->{vm,frag}_flags */
+    read_lock(&executable_areas->lock);
+    if (lookup_addr(executable_areas, addr, &area)) {
+        ASSERT(area != NULL);
+        /* The custom field is initialized to 0 in add_vm_area */
+        coarse = (coarse_info_t *) area->custom.client;
+        is_coarse = TEST(FRAG_COARSE_GRAIN, area->frag_flags);
+        /* We always create coarse_info_t up front in add_executable_vm_area */
+        ASSERT((is_coarse && coarse != NULL) || (!is_coarse && coarse == NULL));
+        if (init && coarse != NULL && TEST(PERSCACHE_CODE_INVALID, coarse->flags)) {
+            /* Reset the unit as the validating event did not occur
+             * (can't do it here due to lock rank order vs exec areas lock) */
+            reset_unit = true;
+            /* We do need to adjust coarse unit bounds for 4.3 when we don't see
+             * the rebind +rx event */
+            adjust_coarse_unit_bounds(area, true/*even if invalid*/);
+            STATS_INC(coarse_executed_invalid);
+            /* FIXME for 4.4: validation won't happen post-rebind like 4.3, so we
+             * will always get here marked as invalid.  Here we'll do full md5
+             * modulo rebasing check (split into per-page via read-only as opt).
+             */
+        }
+        /* We cannot add to shared_data when we load in a persisted unit
+         * due to lock rank issues, so we delay until first asked about.
+         */
+        if (init && TEST(VM_ADD_TO_SHARED_DATA, area->vm_flags)) {
+            add_to_shared = true;
+            area->vm_flags &= ~VM_ADD_TO_SHARED_DATA;
+            area->vm_flags |= VM_EXECUTED_FROM;
+            area_copy = *area;
+        } else {
+            DODEBUG({ area_copy = *area; }); /* for ASSERT below */
+        }
+    }
+    read_unlock(&executable_areas->lock);
+
+    if (coarse != NULL && init) {
+        /* For 4.3, bounds check is done at post-rebind validation;
+         * FIXME: in 4.4, we need to do it here and adjust bounds or invalidate
+         * pcache if not a superset (we'll allow any if_rx_text to merge into coarse).
+         */
+        ASSERT(coarse->base_pc == area_copy.start && coarse->end_pc == area_copy.end);
+        if (reset_unit) {
+            coarse_unit_reset_free(get_thread_private_dcontext(),
+                                   coarse, false/*no locks*/, true/*unlink*/,
+                                   true/*give up primary*/);
+        }
+        if (add_to_shared) {
+            if (!have_shvm_lock)
+                SHARED_VECTOR_RWLOCK(&shared_data->areas, write, lock);
+            ASSERT_VMAREA_VECTOR_PROTECTED(&shared_data->areas, WRITE);
+            /* avoid double-add from a race */
+            if (!lookup_addr(&shared_data->areas, coarse->base_pc, NULL)) {
+                LOG(GLOBAL, LOG_VMAREAS, 2,
+                    "adding coarse region "PFX"-"PFX" to shared vm areas\n",
+                    area_copy.start, area_copy.end);
+                add_vm_area(&shared_data->areas, area_copy.start, area_copy.end,
+                            area_copy.vm_flags, area_copy.frag_flags
+                            _IF_DEBUG(area_copy.comment));
+            }
+            if (!have_shvm_lock)
+                SHARED_VECTOR_RWLOCK(&shared_data->areas, write, unlock);
+        }
+    } else
+        ASSERT(!add_to_shared && !reset_unit);
+
+    return coarse;
+}
+
+coarse_info_t *
+get_executable_area_coarse_info(app_pc addr)
+{
+    return get_coarse_info_internal(addr, true/*init*/, false/*no lock*/);
+}
+
+/* Ensures there is a non-frozen coarse unit for the executable_areas region
+ * corresponding to "frozen", which is now frozen.
+ */
+void
+mark_executable_area_coarse_frozen(coarse_info_t *frozen)
+{
+    vm_area_t *area = NULL;
+    coarse_info_t *info;
+    ASSERT(frozen->frozen); /* caller should mark */
+    write_lock(&executable_areas->lock); /* since writing flags */
+    if (lookup_addr(executable_areas, frozen->base_pc, &area)) {
+        ASSERT(area != NULL);
+        /* The custom field is initialized to 0 in add_vm_area */
+        if (area->custom.client != NULL) {
+            ASSERT(TEST(FRAG_COARSE_GRAIN, area->frag_flags));
+            info = (coarse_info_t *) area->custom.client;
+            ASSERT(info == frozen && frozen->non_frozen == NULL);
+            info = coarse_unit_create(frozen->base_pc, frozen->end_pc,
+                                      &frozen->module_md5, true/*for execution*/);
+            LOG(GLOBAL, LOG_VMAREAS, 1,
+                "new secondary coarse unit %s "PFX"-"PFX"\n",
+                info->module, frozen->base_pc, frozen->end_pc);
+            frozen->non_frozen = info;
+        } else
+            ASSERT(!TEST(FRAG_COARSE_GRAIN, area->frag_flags));
+    }
+    write_unlock(&executable_areas->lock);
+}
+
+/* iterates through all executable areas overlapping the pages touched
+ * by the region addr_[start,end) 
+ * if are_all_matching is false 
+ *    returns true if any overlapping region has matching vm_flags and frag_flags;
+ *    false otherwise
+ * if are_all_matching is true
+ *    returns true only if all overlapping regions have matching vm_flags
+ *    and matching frag_flags, or if there are no overlapping regions;
+ *    false otherwise
+ * a match of 0 matches all
+ */
+static bool
+executable_areas_match_flags(app_pc addr_start, app_pc addr_end, bool *found_area, 
+                             bool are_all_matching /* ALL when true,
+                                                      EXISTS when false */,
+                             uint match_vm_flags, uint match_frag_flags)
+{
+    /* binary search below will assure that we hold an executable_areas lock */
+    app_pc page_start = (app_pc)ALIGN_BACKWARD(addr_start, PAGE_SIZE);
+    app_pc page_end = (app_pc)ALIGN_FORWARD(addr_end, PAGE_SIZE);
+    vm_area_t *area;
+    if (found_area != NULL)
+        *found_area = false;
+    ASSERT(page_start < page_end || page_end == NULL); /* wraparound */
+    /* We have subpage regions from some of our rules, we should return true
+     * if any area on the list that overlaps the pages enclosing the addr_[start,end) 
+     * region is writable */
+    while (binary_search(executable_areas, page_start, 
+                         page_end, &area, NULL, true)) {
+        if (found_area != NULL)
+            *found_area = true;
+        /* TESTALL will return true for a match of 0 */
+        if (are_all_matching) {
+            if (!TESTALL(match_vm_flags, area->vm_flags) ||
+                !TESTALL(match_frag_flags, area->frag_flags))
+                return false;
+        } else {
+            if (TESTALL(match_vm_flags, area->vm_flags) &&
+                TESTALL(match_frag_flags, area->frag_flags))
+                return true;
+        }
+        if (area->end < page_end || page_end == NULL)
+            page_start = area->end;
+        else
+            break;
+    }
+    return are_all_matching;    /* false for EXISTS, true for ALL */
+}
+
+/* returns true if addr is on a page that was marked writable by the 
+ * application but that we marked RO b/c it contains executable code
+ * does NOT check if addr is executable, only that something on its page is!
+ */
+bool
+is_executable_area_writable(app_pc addr)
+{
+    bool writable;
+    read_lock(&executable_areas->lock);
+    writable = executable_areas_match_flags(addr, addr+1 /* open ended */, 
+                                            NULL, false /* EXISTS */,
+                                            VM_MADE_READONLY, 0);
+    read_unlock(&executable_areas->lock);
+    return writable;
+}
+
+#if defined(DEBUG) /* since only used for a stat right now */
+/* returns true if region [start, end) overlaps pages that match match_vm_flags
+ * e.g. VM_WRITABLE is set when all pages marked writable by the 
+ *      application but that we marked RO b/c they contain executable code.
+ *
+ * Does NOT check if region is executable, only that something
+ * overlapping its pages is!  are_all_matching determines
+ * whether all regions need to match flags, or whether a matching
+ * region exists.
+ */
+static bool
+is_executable_area_writable_overlap(app_pc start, app_pc end,
+                                    bool are_all_matching, uint match_vm_flags)
+{
+    bool writable;
+    read_lock(&executable_areas->lock);
+    writable = executable_areas_match_flags(start, end, NULL, 
+                                            are_all_matching, match_vm_flags, 0);
+    read_unlock(&executable_areas->lock);
+    return writable;
+}
+#endif
+
+/* Returns true if region [start, end) overlaps any regions that are
+ * marked as FRAG_COARSE_GRAIN.
+ */
+bool
+executable_vm_area_coarse_overlap(app_pc start, app_pc end)
+{
+    bool match;
+    read_lock(&executable_areas->lock);
+    match = executable_areas_match_flags(start, end, NULL, false/*exists, not all*/,
+                                         0, FRAG_COARSE_GRAIN);
+    read_unlock(&executable_areas->lock);
+    return match;
+}
+
+/* Returns true if region [start, end) overlaps any regions that are
+ * marked as VM_PERSISTED_CACHE.
+ */
+bool
+executable_vm_area_persisted_overlap(app_pc start, app_pc end)
+{
+    bool match;
+    read_lock(&executable_areas->lock);
+    match = executable_areas_match_flags(start, end, NULL, false/*exists, not all*/,
+                                         VM_PERSISTED_CACHE, 0);
+    read_unlock(&executable_areas->lock);
+    return match;
+}
+
+/* Returns true if any part of region [start, end) has ever been executed from */
+bool
+executable_vm_area_executed_from(app_pc start, app_pc end)
+{
+    bool match;
+    read_lock(&executable_areas->lock);
+    match = executable_areas_match_flags(start, end, NULL, false/*exists, not all*/,
+                                         VM_EXECUTED_FROM, 0);
+    read_unlock(&executable_areas->lock);
+    return match;
+}
+
+/* If there is no overlap between executable_areas and [start,end), returns false.
+ * Else, returns true and sets [overlap_start,overlap_end) as the bounds of the first
+ * and last executable_area regions that overlap [start,end); i.e.,
+ *   overlap_start starts the first area that overlaps [start,end);
+ *   overlap_end ends the last area that overlaps [start,end).
+ * Note that overlap_start may be > start and overlap_end may be < end.
+ *
+ * If frag_flags != 0, the region described above is expanded such that the regions
+ * before and after [overlap_start,overlap_end) do NOT match
+ * [overlap_start,overlap_end) in TESTALL of frag_flags, but only considering
+ * non-contiguous regions if !contig.
+ * For example, we pass in FRAG_COARSE_GRAIN and contig=true; then, if
+ * the overlap_start region is FRAG_COARSE_GRAIN and it has a contiguous region
+ * to its left that is also FRAG_COARSE_GRAIN, but beyond that there is no
+ * contiguous region, we will return the start of the region to the left rather than
+ * the regular overlap_start.
+ */
+bool
+executable_area_overlap_bounds(app_pc start, app_pc end,
+                               app_pc *overlap_start/*OUT*/, app_pc *overlap_end/*OUT*/,
+                               uint frag_flags, bool contig)
+{
+    int start_index, end_index; /* must be signed */
+    int i; /* must be signed */
+    ASSERT(overlap_start != NULL && overlap_end != NULL);
+    read_lock(&executable_areas->lock);
+
+    /* Find first overlapping region */
+    if (!binary_search(executable_areas, start, end, NULL, &start_index, true/*first*/))
+        return false;
+    ASSERT(start_index >= 0);
+    if (frag_flags != 0) {
+        for (i = start_index - 1; i >= 0; i--) {
+            if ((contig &&
+                 executable_areas->buf[i].end != executable_areas->buf[i+1].start) ||
+                (TESTALL(frag_flags, executable_areas->buf[i].frag_flags) !=
+                 TESTALL(frag_flags, executable_areas->buf[start_index].frag_flags)))
+                break;
+        }
+        ASSERT(i + 1 >= 0);
+        *overlap_start = executable_areas->buf[i + 1].start;
+    } else
+        *overlap_start = executable_areas->buf[start_index].start;
+
+    /* Now find region just at or before end */
+    binary_search(executable_areas, end-1, end, NULL, &end_index, true/*first*/);
+    ASSERT(end_index >= 0); /* else 1st binary search would have failed */
+    ASSERT(end_index >= start_index);
+    if (end_index < executable_areas->length - 1 && frag_flags != 0) {
+        for (i = end_index + 1; i < executable_areas->length; i++) {
+            if ((contig &&
+                 executable_areas->buf[i].start != executable_areas->buf[i-1].end) ||
+                (TESTALL(frag_flags, executable_areas->buf[i].frag_flags) !=
+                 TESTALL(frag_flags, executable_areas->buf[end_index].frag_flags)))
+                break;
+        }
+        ASSERT(i - 1 < executable_areas->length);
+        *overlap_end = executable_areas->buf[i - 1].end;
+    } else /* no extension asked for, or nowhere to extend to */
+        *overlap_end = executable_areas->buf[end_index].end;
+
+    read_unlock(&executable_areas->lock);
+    return true;
+}
+
+/***************************************************
+ * Iterator over coarse units in executable_areas that overlap [start,end) */
+void
+vm_area_coarse_iter_start(vmvector_iterator_t *vmvi, app_pc start)
+{
+    int start_index; /* must be signed */
+    ASSERT(vmvi != NULL);
+    vmvector_iterator_start(executable_areas, vmvi);
+    ASSERT_OWN_READ_LOCK(true, &executable_areas->lock);
+    /* Find first overlapping region */
+    if (start != NULL &&
+        binary_search(executable_areas, start, start+1, NULL,
+                      &start_index, true/*first*/)) {
+        ASSERT(start_index >= 0);
+        vmvi->index = start_index - 1 /*since next is +1*/;
+    }
+}
+static bool
+vm_area_coarse_iter_find_next(vmvector_iterator_t *vmvi, app_pc end, bool mutate,
+                              coarse_info_t **info_out/*OUT*/)
+{
+    int forw;
+    ASSERT_VMAREA_VECTOR_PROTECTED(vmvi->vector, READWRITE);
+    ASSERT(vmvi->vector == executable_areas);
+    for (forw = 1; vmvi->index + forw < vmvi->vector->length; forw++) {
+        if (end != NULL && executable_areas->buf[vmvi->index+forw].start >= end)
+            break;
+        if (TEST(FRAG_COARSE_GRAIN,
+                 executable_areas->buf[vmvi->index+forw].frag_flags)) {
+            coarse_info_t *info = executable_areas->buf[vmvi->index+forw].custom.client;
+            if (mutate)
+                vmvi->index = vmvi->index+forw;
+            ASSERT(info != NULL); /* we always allocate up front */
+            if (info_out != NULL)
+                *info_out = info;
+            return true;
+        }
+    }
+    return false;
+}
+bool
+vm_area_coarse_iter_hasnext(vmvector_iterator_t *vmvi, app_pc end)
+{
+    return vm_area_coarse_iter_find_next(vmvi, end, false/*no mutate*/, NULL);
+}
+/* May want to return region bounds if have callers who care about that. */
+coarse_info_t *
+vm_area_coarse_iter_next(vmvector_iterator_t *vmvi, app_pc end)
+{
+    coarse_info_t *info = NULL;
+    vm_area_coarse_iter_find_next(vmvi, end, true/*mutate*/, &info);
+    return info;
+}
+void
+vm_area_coarse_iter_stop(vmvector_iterator_t *vmvi)
+{
+    ASSERT(vmvi->vector == executable_areas);
+    vmvector_iterator_stop(vmvi);
+}
+/***************************************************/
+
+/* returns true if addr is on a page that contains at least one selfmod
+ * region and no non-selfmod regions.
+ */
+bool
+is_executable_area_on_all_selfmod_pages(app_pc start, app_pc end)
+{
+    bool all_selfmod;
+    bool found;
+    read_lock(&executable_areas->lock);
+    all_selfmod = executable_areas_match_flags(start, end,
+                                               &found, true /* ALL */,
+                                               0, FRAG_SELFMOD_SANDBOXED);
+    read_unlock(&executable_areas->lock);
+    /* we require at least one area to be present */
+    return all_selfmod && found;
+}
+
+/* Meant to be called from a seg fault handler.
+ * Returns true if addr is on a page that was marked writable by the 
+ * application but that we marked RO b/c it contains executable code, OR if 
+ * addr is on a writable page (since another thread could have removed addr
+ * from exec list before seg fault handler was scheduled).
+ * does NOT check if addr is executable, only that something on its page is!
+ */
+bool
+was_executable_area_writable(app_pc addr)
+{
+    bool found_area = false, was_writable = false;
+    read_lock(&executable_areas->lock);
+    was_writable = executable_areas_match_flags(addr, addr+1, &found_area, 
+                                                false /* EXISTS */,
+                                                VM_MADE_READONLY, 0);
+    /* seg fault could have happened, then area was made writable before
+     * thread w/ exception was scheduled.
+     * we assume that area was writable at time of seg fault if it's
+     * exec writable now (above) OR no area was found and it's writable now
+     * and not on DR area list (below).
+     * Need to check DR area list since a write to protected DR area from code 
+     * cache can end up here, as DR area may be made writable once in fault handler
+     * due to self-protection un-protection for entering DR!
+     * FIXME: checking for threads_ever_created==1 could further rule out other
+     * causes for some apps.
+     * Keep readlock to avoid races.
+     */
+    if (!found_area) {
+        uint prot;
+        if (get_memory_info(addr, NULL, NULL, &prot))
+            was_writable = TEST(MEMPROT_WRITE, prot) && !is_dynamo_address(addr);
+    }
+    read_unlock(&executable_areas->lock);
+    return was_writable;
+}
+
+/* returns true if addr is in an executable area that contains
+ * self-modifying code, and so should be sandboxed
+ */
+bool
+is_executable_area_selfmod(app_pc addr)
+{
+    uint flags;
+    if (get_executable_area_flags(addr, &flags))
+        return TEST(FRAG_SELFMOD_SANDBOXED, flags);
+    else
+        return false;
+}
+
+#ifdef DGC_DIAGNOSTICS
+/* returns false if addr is not in an executable area marked as dyngen */
+bool
+is_executable_area_dyngen(app_pc addr)
+{
+    uint flags;
+    if (get_executable_area_flags(addr, &flags))
+        return TEST(FRAG_DYNGEN, flags);
+    else
+        return false;
+}
+#endif
+
+/* lookup against the per-process addresses map */
+bool
+is_valid_address(app_pc addr)
+{
+    ASSERT_NOT_IMPLEMENTED(false && "is_valid_address not implemented");
+    return false;
+}
+
+/* Due to circular dependencies bet vmareas and global heap, we cannot
+ * incrementally keep dynamo_areas up to date.
+ * Instead, we wait until people ask about it, when we do a complete
+ * walk through the heap units and add them all (yes, re-adding
+ * ones we've seen).
+ */
+static void
+update_dynamo_vm_areas(bool have_writelock)
+{
+    if (dynamo_areas_uptodate)
+        return;
+    if (!have_writelock)
+        dynamo_vm_areas_lock();
+    ASSERT(dynamo_areas != NULL);
+    ASSERT_OWN_WRITE_LOCK(true, &dynamo_areas->lock);
+    /* avoid uptodate asserts from heap needed inside add_vm_area */
+    DODEBUG({ dynamo_areas_synching = true; });
+    /* check again with lock, and repeat until done since
+     * could require more memory in the middle for vm area vector
+     */ 
+    while (!dynamo_areas_uptodate) {
+        dynamo_areas_uptodate = true;
+        heap_vmareas_synch_units();
+        LOG(GLOBAL, LOG_VMAREAS, 3, "after updating dynamo vm areas:\n");
+        DOLOG(3, LOG_VMAREAS, { print_vm_areas(dynamo_areas, GLOBAL); });
+    }
+    DODEBUG({ dynamo_areas_synching = false; });
+    if (!have_writelock)
+        dynamo_vm_areas_unlock();
+}
+
+/* Used for DR heap area changes as circular dependences prevent
+ * directly adding or removing DR vm areas->
+ * Must hold the DR areas lock across the combination of calling this and
+ * modifying the heap lists.
+ */
+void
+mark_dynamo_vm_areas_stale()
+{
+    /* ok to ask for locks or mark stale before dynamo_areas is allocated */
+    ASSERT((dynamo_areas == NULL && get_num_threads() <= 1 /*must be only DR thread*/)
+           || self_owns_write_lock(&dynamo_areas->lock));
+    dynamo_areas_uptodate = false;
+}
+
+/* HACK to get recursive write lock for internal and external use */
+void
+dynamo_vm_areas_lock()
+{
+    all_memory_areas_lock();
+    /* ok to ask for locks or mark stale before dynamo_areas is allocated,
+     * during heap init and before we can allocate it.  no lock needed then.
+     */
+    ASSERT(dynamo_areas != NULL || get_num_threads() <= 1 /*must be only DR thread*/);
+    if (dynamo_areas == NULL)
+        return;
+    if (self_owns_write_lock(&dynamo_areas->lock)) {
+        dynamo_areas_recursion++;
+        /* we have a 5-deep path:
+         *   global_heap_alloc | heap_create_unit | get_guarded_real_memory | 
+         *   heap_low_on_memory | release_guarded_real_memory
+         */
+        ASSERT_CURIOSITY(dynamo_areas_recursion <= 4);
+    } else
+        write_lock(&dynamo_areas->lock);
+}
+
+void
+dynamo_vm_areas_unlock()
+{
+    /* ok to ask for locks or mark stale before dynamo_areas is allocated,
+     * during heap init and before we can allocate it.  no lock needed then.
+     */
+    ASSERT(dynamo_areas != NULL || get_num_threads() <= 1 /*must be only DR thread*/);
+    if (dynamo_areas == NULL)
+        return;
+    if (dynamo_areas_recursion > 0) {
+        ASSERT_OWN_WRITE_LOCK(true, &dynamo_areas->lock);
+        dynamo_areas_recursion--;
+    } else
+        write_unlock(&dynamo_areas->lock);
+    all_memory_areas_unlock();
+}
+
+bool
+self_owns_dynamo_vm_area_lock()
+{
+    /* heap inits before dynamo_areas (which now needs heap to init) so
+     * we ignore the lock prior to dynamo_areas init, assuming single-DR-thread.
+     */
+    ASSERT(dynamo_areas != NULL || get_num_threads() <= 1 /*must be only DR thread*/);
+    return dynamo_areas == NULL || self_owns_write_lock(&dynamo_areas->lock);
+}
+
+/* grabs read lock and checks for update -- when it returns it guarantees
+ * to hold read lock with no updates pending
+ */
+static void
+dynamo_vm_areas_start_reading()
+{
+    read_lock(&dynamo_areas->lock);
+    while (!dynamo_areas_uptodate) {
+        /* switch to write lock 
+         * cannot rely on uptodate value prior to a lock so must
+         * grab read and then check it, and back out if necessary
+         * as we have no reader->writer transition
+         */
+        read_unlock(&dynamo_areas->lock);
+        dynamo_vm_areas_lock();
+        update_dynamo_vm_areas(true);
+        /* FIXME: more efficient if we could safely drop from write to read
+         * lock -- could simply reverse order here and then while becomes if,
+         * but a little fragile in that properly nested rwlocks may be assumed
+         * elsewhere
+         */
+        dynamo_vm_areas_unlock();
+        read_lock(&dynamo_areas->lock);
+    }
+}
+
+static void
+dynamo_vm_areas_done_reading()
+{
+    read_unlock(&dynamo_areas->lock);
+}
+
+/* add dynamo-internal area to the dynamo-internal area list
+ * this should be atomic wrt the memory being allocated to avoid races
+ * w/ the app executing from it -- thus caller must hold DR areas write lock!
+ */
+bool
+add_dynamo_vm_area(app_pc start, app_pc end, uint prot, bool unmod_image
+                   _IF_DEBUG(char *comment))
+{
+    uint vm_flags = (TEST(MEMPROT_WRITE, prot) ? VM_WRITABLE : 0) |
+                    (unmod_image ? VM_UNMOD_IMAGE : 0);
+    /* case 3045: areas inside the vmheap reservation are not added to the list */
+    ASSERT(!is_vmm_reserved_address(start, end - start));
+    LOG(GLOBAL, LOG_VMAREAS, 2, "new dynamo vm area: "PFX"-"PFX" %s\n",
+        start, end, comment);
+    ASSERT(dynamo_areas != NULL);
+    ASSERT_OWN_WRITE_LOCK(true, &dynamo_areas->lock);
+    if (!dynamo_areas_uptodate)
+        update_dynamo_vm_areas(true);
+    ASSERT(!vm_area_overlap(dynamo_areas, start, end));
+    add_vm_area(dynamo_areas, start, end, vm_flags, 0 /* frag_flags */
+                _IF_DEBUG(comment));
+    update_all_memory_areas(start, end, prot);
+    return true;
+}
+
+/* remove dynamo-internal area from the dynamo-internal area list 
+ * this should be atomic wrt the memory being freed to avoid races
+ * w/ it being re-used and problems w/ the app executing from it --
+ * thus caller must hold DR areas write lock!
+ */
+bool
+remove_dynamo_vm_area(app_pc start, app_pc end)
+{
+    bool ok;
+    DEBUG_DECLARE(bool removed);
+    LOG(GLOBAL, LOG_VMAREAS, 2, "removing dynamo vm area: "PFX"-"PFX"\n",
+        start, end);
+    ASSERT(dynamo_areas != NULL);
+    ASSERT_OWN_WRITE_LOCK(true, &dynamo_areas->lock);
+    if (!dynamo_areas_uptodate)
+        update_dynamo_vm_areas(true);
+    ok = remove_vm_area(dynamo_areas, start, end, false);
+    DEBUG_DECLARE(removed = )
+        remove_from_all_memory_areas(start, end);
+    ASSERT(removed);
+    return ok;
+}
+
+/* adds dynamo-internal area to the dynamo-internal area list, but
+ * doesn't grab the dynamo areas lock.  intended to be only used for
+ * heap walk updates, where the lock is grabbed prior to the walk and held
+ * throughout the entire walk.
+ */
+bool
+add_dynamo_heap_vm_area(app_pc start, app_pc end, bool writable, bool unmod_image
+                        _IF_DEBUG(char *comment))
+{
+    LOG(GLOBAL, LOG_VMAREAS, 2, "new dynamo vm area: "PFX"-"PFX" %s\n",
+        start, end, comment);
+    ASSERT(!vm_area_overlap(dynamo_areas, start, end));
+    /* case 3045: areas inside the vmheap reservation are not added to the list */
+    ASSERT(!is_vmm_reserved_address(start, end - start));
+    /* add_vm_area will assert that write lock is held */
+    add_vm_area(dynamo_areas, start, end, 
+                VM_DR_HEAP |
+                (writable ? VM_WRITABLE : 0) |
+                (unmod_image ? VM_UNMOD_IMAGE : 0),
+                0 /* frag_flags */ _IF_DEBUG(comment));
+    return true;
+}
+
+/* breaking most abstractions here we return whether current vmarea
+ * vector starts at given heap_pc.  The price of circular dependency
+ * is that abstractions can no longer be safely used.  case 4196
+ */
+bool
+is_dynamo_area_buffer(byte *heap_unit_start_pc)
+{
+    return (void*)heap_unit_start_pc == dynamo_areas->buf;
+}
+
+/* assumes caller holds dynamo_areas->lock */
+void 
+remove_dynamo_heap_areas()
+{
+    int i;
+    /* remove_vm_area will assert that write lock is held, but let's make
+     * sure we're holding it as we walk the vector, even if make no removals
+     */
+    ASSERT_VMAREA_VECTOR_PROTECTED(dynamo_areas, WRITE);
+    LOG(GLOBAL, LOG_VMAREAS, 4, "remove_dynamo_heap_areas:\n");
+    /* walk backwards to avoid O(n^2) */
+    for (i = dynamo_areas->length - 1; i >= 0; i--) {
+        if (TEST(VM_DR_HEAP, dynamo_areas->buf[i].vm_flags)) {
+            app_pc start = dynamo_areas->buf[i].start;
+            app_pc end = dynamo_areas->buf[i].end;
+            /* ASSUMPTION: remove_vm_area, given exact bounds, simply shifts later
+             * areas down in vector!
+             */
+            LOG(GLOBAL, LOG_VMAREAS, 4, "Before removing vm area:\n");
+            DOLOG(3, LOG_VMAREAS, { print_vm_areas(dynamo_areas, GLOBAL); });
+            remove_vm_area(dynamo_areas, start, end, false);
+            LOG(GLOBAL, LOG_VMAREAS, 4, "After removing vm area:\n");
+            DOLOG(3, LOG_VMAREAS, { print_vm_areas(dynamo_areas, GLOBAL); });
+            remove_from_all_memory_areas(start, end);
+        }
+    }
+}
+
+bool
+is_dynamo_address(app_pc addr)
+{
+    bool found;
+    /* case 3045: areas inside the vmheap reservation are not added to the list */
+    if (is_vmm_reserved_address(addr, 1))
+        return true;
+    dynamo_vm_areas_start_reading();
+    found = lookup_addr(dynamo_areas, addr, NULL);
+    dynamo_vm_areas_done_reading();
+    return found;
+}
+
+/* returns true iff address is an address that the app thinks is writable
+ * but really is not, as it overlaps DR memory (or did at the prot time);
+ * or we're preventing function patching in specified application modules.
+ */
+bool
+is_pretend_writable_address(app_pc addr)
+{
+    bool found;
+    ASSERT(DYNAMO_OPTION(handle_DR_modify) == DR_MODIFY_NOP ||
+           DYNAMO_OPTION(handle_ntdll_modify) == DR_MODIFY_NOP
+           || !IS_STRING_OPTION_EMPTY(patch_proof_list) 
+           || !IS_STRING_OPTION_EMPTY(patch_proof_default_list));
+    read_lock(&pretend_writable_areas->lock);
+    found = lookup_addr(pretend_writable_areas, addr, NULL);
+    read_unlock(&pretend_writable_areas->lock);
+    return found;
+}
+
+/* returns true if the passed in area overlaps any known pretend writable areas */
+static bool
+pretend_writable_vm_area_overlap(app_pc start, app_pc end)
+{
+    bool overlap;
+    read_lock(&pretend_writable_areas->lock);
+    overlap = vm_area_overlap(pretend_writable_areas, start, end);
+    read_unlock(&pretend_writable_areas->lock);
+    return overlap;
+}
+
+#ifdef DEBUG
+/* returns comment for addr, if there is one, else NULL
+ */
+char *
+get_address_comment(app_pc addr)
+{
+    char *res = NULL;
+    vm_area_t *area;
+    bool ok;
+    read_lock(&executable_areas->lock);
+    ok = lookup_addr(executable_areas, addr, &area);
+    if (ok)
+        res = area->comment;
+    read_unlock(&executable_areas->lock);
+    if (!ok) {
+        read_lock(&dynamo_areas->lock);
+        ok = lookup_addr(dynamo_areas, addr, &area);
+        if (ok)
+            res = area->comment;
+        read_unlock(&dynamo_areas->lock);
+    }
+    return res;
+}
+#endif
+
+/* returns true if the passed in area overlaps any known executable areas
+ * if !have_writelock, acquires the executable_areas read lock
+ */
+bool
+executable_vm_area_overlap(app_pc start, app_pc end, bool have_writelock)
+{
+    bool overlap;
+    if (!have_writelock)
+        read_lock(&executable_areas->lock);
+    overlap = vm_area_overlap(executable_areas, start, end);
+    if (!have_writelock)
+        read_unlock(&executable_areas->lock);
+    return overlap;
+}
+
+void
+executable_areas_lock()
+{
+    write_lock(&executable_areas->lock);
+}
+
+void
+executable_areas_unlock()
+{
+    ASSERT_OWN_WRITE_LOCK(true, &executable_areas->lock);
+    write_unlock(&executable_areas->lock);
+}
+
+/* returns true if the passed in area overlaps any dynamo areas */
+bool
+dynamo_vm_area_overlap(app_pc start, app_pc end)
+{
+    bool overlap;
+    dynamo_vm_areas_start_reading();
+    overlap = vm_area_overlap(dynamo_areas, start, end);
+    dynamo_vm_areas_done_reading();
+    return overlap;
+}
+
+/* Checks to see if pc is on the stack
+ * If pc has already been resolved into an area, pass that in.
+ */
+static bool
+is_on_stack(dcontext_t *dcontext, app_pc pc, vm_area_t *area)
+{
+    byte *stack_base, *stack_top; /* "official" stack */
+    byte *esp = (byte *) get_mcontext(dcontext)->xsp;
+    byte *esp_base;
+    size_t size;
+    bool ok, query_esp = true;
+    /* First check the area if we're supplied one. */
+    if (area != NULL) {
+        LOG(THREAD, LOG_VMAREAS, 3,
+            "stack vs "PFX": area "PFX".."PFX", esp "PFX"\n",
+            pc, area->start, area->end, esp);
+        ASSERT(pc >= area->start && pc < area->end);
+        if (esp >= area->start && esp < area->end)
+            return true;
+    }
+    /* Now check the "offical" stack bounds.  These are cached so cheap to
+     * look up.  Xref case 8180, these might not always be available, 
+     * get_stack_bounds() takes care of any asserts on availability. */
+    ok = get_stack_bounds(dcontext, &stack_base, &stack_top);
+    if (ok) {
+        LOG(THREAD, LOG_VMAREAS, 3,
+            "stack vs "PFX": official "PFX".."PFX", esp "PFX"\n",
+            pc, stack_base, stack_top, esp);
+        ASSERT(stack_base < stack_top);
+        if (pc >= stack_base && pc < stack_top)
+            return true;
+        /* We optimize away the expensive query of esp region bounds if esp
+         * is in within the "offical" stack cached allocation bounds. */
+        if (esp >= stack_base && esp < stack_top)
+            query_esp = false;
+    }
+    if (query_esp) {
+        ok = get_memory_info(esp, &esp_base, &size, NULL);
+        ASSERT(ok);
+        LOG(THREAD, LOG_VMAREAS, 3,
+            "stack vs "PFX": region "PFX".."PFX", esp "PFX"\n",
+            pc, esp_base, esp_base+size, esp);
+        /* FIXME - stack could be split into multiple os regions by prot
+         * differences, could check alloc base equivalence. */
+        if (pc >= esp_base && pc < esp_base + size)
+            return true;
+    }
+    return false;
+}
+
+bool
+is_address_on_stack(dcontext_t *dcontext, app_pc address)
+{
+    return is_on_stack(dcontext, address, NULL);
+}
+
+/* returns true if an executable area exists with VM_DRIVER_ADDRESS,
+ * not a strict opposite of is_user_address() */
+bool
+is_driver_address(app_pc addr)
+{
+    uint vm_flags;
+    if (get_executable_area_vm_flags(addr, &vm_flags)) {
+        return TEST(VM_DRIVER_ADDRESS, vm_flags);
+    }
+    return false;
+}
+
+#ifdef PROGRAM_SHEPHERDING /********************************************/
+
+/* forward declaration */
+static int
+check_origins_bb_pattern(dcontext_t *dcontext, app_pc addr, app_pc *base, size_t *size,
+                         uint *vm_flags, uint *frag_flags);
+
+/* The following two arrays need to be in synch with enum action_type_t defined in
+ * vmareas.h.
+ */
+#define MESSAGE_EXEC_VIOLATION "Execution security violation was intercepted!\n"
+#define MESSAGE_CONTACT_VENDOR "Contact your vendor for a security vulnerability fix.\n"
+const char * const action_message[] = {
+    /* no trailing newlines for SYSLOG_INTERNAL */
+    MESSAGE_EXEC_VIOLATION MESSAGE_CONTACT_VENDOR "Program terminated.",
+    MESSAGE_EXEC_VIOLATION MESSAGE_CONTACT_VENDOR "Program continuing!",
+    MESSAGE_EXEC_VIOLATION MESSAGE_CONTACT_VENDOR "Program continuing after terminating thread.",
+    MESSAGE_EXEC_VIOLATION MESSAGE_CONTACT_VENDOR "Program continuing after throwing an exception."
+};
+
+/* event log message IDs */
+#ifdef WINDOWS
+const uint action_event_id[] = {
+    MSG_SEC_VIOLATION_TERMINATED,
+    MSG_SEC_VIOLATION_CONTINUE,
+    MSG_SEC_VIOLATION_THREAD,
+    MSG_SEC_VIOLATION_EXCEPTION,
+# ifdef HOT_PATCHING_INTERFACE
+    MSG_HOT_PATCH_VIOLATION,
+# endif
+};
+#endif
+
+/* fills the target component of a threat ID */
+static void
+fill_security_violation_target(char name[MAXIMUM_VIOLATION_NAME_LENGTH],
+                               const byte target_contents[4])
+{
+    int i;
+    for (i = 0; i < 4; i++)
+        name[i + 5] = (char) ((target_contents[i] % 10) + '0');
+}
+
+static void
+get_security_violation_name(dcontext_t *dcontext, 
+                            app_pc addr, char *name, int name_length, 
+                            security_violation_t violation_type, 
+                            const char *threat_id)
+{
+    ptr_uint_t addr_as_int;
+    app_pc name_addr = NULL;
+    int i;
+
+    ASSERT(name_length >= MAXIMUM_VIOLATION_NAME_LENGTH);
+
+    /* Hot patches & process_control use their own threat IDs. */
+    if (IF_HOTP(violation_type == HOT_PATCH_DETECTOR_VIOLATION || 
+                violation_type == HOT_PATCH_PROTECTOR_VIOLATION ||)
+        IF_PROC_CTL(violation_type == PROCESS_CONTROL_VIOLATION ||) false) {
+        ASSERT(threat_id != NULL);
+        strncpy(name, threat_id, MAXIMUM_VIOLATION_NAME_LENGTH);
+    } else {
+        bool unreadable_addr = false;
+        byte target_contents[4];    /* 4 instruction bytes read from target */
+        ASSERT(threat_id == NULL);  /* Supplied only for hot patch violations.*/
+
+        /* First four characters are alphabetics calculated from the address 
+           of the beginning of the basic block from which the violating
+           contol transfer instruction originated.  Ideally we would use the
+           exact CTI address rather than the beginning of its block, but 
+           we don't want to translate it back to an app address to reduce 
+           possible failure points on this critical path. */
+        name_addr = dcontext->last_fragment->tag;
+#ifdef WINDOWS
+        /* Move PC relative to preferred base for consistent naming */
+        name_addr += get_module_preferred_base_delta(name_addr);
+#endif
+        addr_as_int = (ptr_uint_t) name_addr;
+        for (i = 0; i < 4; i++) {
+            name[i] = (char) ((addr_as_int % 26) + 'A');
+            addr_as_int /= 256;
+        }
+
+        /* Fifth character is a '.' */
+        name[4] = '.';
+
+        unreadable_addr = !safe_read(addr, 
+                                     sizeof(target_contents), &target_contents);
+
+        /* if at unreadable memory see if an ASLR preferred address can be used */
+        if (unreadable_addr) {
+            app_pc likely_target_pc = 
+                aslr_possible_preferred_address(addr);
+            if (likely_target_pc != NULL) {
+                unreadable_addr = 
+                    !safe_read(likely_target_pc, 
+                               sizeof(target_contents), &target_contents);
+            } else {
+                unreadable_addr = true;
+            }
+        }
+
+        /* Next four characters are decimal numerics from the target code */
+        if (unreadable_addr) {
+            for (i = 0; i < 4; i++)
+                name[i + 5] = 'X';
+        } else {
+            fill_security_violation_target(name, target_contents);
+        }
+    }
+
+    /* Tenth character is a '.' */
+    name[9] = '.';    
+
+    /* Next character indicates the security violation type;
+     * sequential letter choices used rather than semantic ones to
+     * obfuscate meaning.   */
+    switch (violation_type) {
+    case STACK_EXECUTION_VIOLATION:    name[10] = 'A'; break;
+    case HEAP_EXECUTION_VIOLATION:     name[10] = 'B'; break;
+    case RETURN_TARGET_VIOLATION:      name[10] = 'C'; break;
+    case RETURN_DIRECT_RCT_VIOLATION:  name[10] = 'D'; ASSERT_NOT_IMPLEMENTED(false); break;
+    case INDIRECT_CALL_RCT_VIOLATION:  name[10] = 'E'; break;
+    case INDIRECT_JUMP_RCT_VIOLATION:  name[10] = 'F'; break;
+#ifdef HOT_PATCHING_INTERFACE
+    case HOT_PATCH_DETECTOR_VIOLATION: name[10] = 'H'; break;
+    case HOT_PATCH_PROTECTOR_VIOLATION:name[10] = 'P'; break;
+#endif
+#ifdef PROCESS_CONTROL
+    case PROCESS_CONTROL_VIOLATION:    name[10] = 'K'; break;
+#endif
+#ifdef GBOP
+    case GBOP_SOURCE_VIOLATION:        name[10] = 'O'; break;
+#endif
+    case ASLR_TARGET_VIOLATION:        name[10] = 'R'; break;
+    case ATTACK_SIM_NUDGE_VIOLATION: /* share w/ normal attack sim */
+    case ATTACK_SIMULATION_VIOLATION:  name[10] = 'S'; break;
+    case APC_THREAD_SHELLCODE_VIOLATION: 
+        /* injected shellcode threat names are custom generated */
+        ASSERT_NOT_REACHED(); 
+        name[10] = 'B'; 
+        break;
+    default:
+        name[10] = 'X';
+        ASSERT_NOT_REACHED();
+    }
+
+    /* Null-terminate */
+    name[11] = '\0';    
+
+    LOG(GLOBAL, LOG_ALL, 1, "Security violation name: %s\n", name);
+}
+
+
+bool
+is_exempt_threat_name(const char *name)
+{
+    if (DYNAMO_OPTION(exempt_threat) && 
+        !IS_STRING_OPTION_EMPTY(exempt_threat_list)) {
+        bool onlist;
+        string_option_read_lock();
+        onlist = check_filter_with_wildcards(DYNAMO_OPTION(exempt_threat_list), name);
+        string_option_read_unlock();
+        if (onlist) {
+            LOG(THREAD_GET, LOG_INTERP|LOG_VMAREAS, 1,
+                "WARNING: threat %s is on exempt list, suppressing violation\n", name);
+            SYSLOG_INTERNAL_WARNING_ONCE("threat %s exempt", name);
+            STATS_INC(num_exempt_threat);
+            return true;
+        }
+    }
+    return false;
+}
+
+/***************************************************************************
+ * Case 8075: we don't want to unprotect .data during violation reporting, so we
+ * place all the local-scope static vars (from DO_THRESHOLD) into .fspdata.
+ */
+START_DATA_SECTION(FREQ_PROTECTED_SECTION, "w");
+
+/* Report security violation to all outputs - syslog, diagnostics, and interactive 
+ * returns false if violation was not reported
+ */
+static bool
+security_violation_report(app_pc addr, security_violation_t violation_type, 
+                          const char *name, action_type_t action)
+{
+    bool dump_forensics = true;
+    /* shouldn't report anything if on silent_block_threat_list */
+    if (!IS_STRING_OPTION_EMPTY(silent_block_threat_list)) {
+        bool onlist;
+        string_option_read_lock();
+        onlist = check_filter_with_wildcards(DYNAMO_OPTION(silent_block_threat_list), name);
+        string_option_read_unlock();
+        if (onlist) {
+            LOG(THREAD_GET, LOG_INTERP|LOG_VMAREAS, 1,
+                "WARNING: threat %s is on silent block list, suppressing reporting\n", name);
+            SYSLOG_INTERNAL_WARNING_ONCE("threat %s silently blocked", name);
+            STATS_INC(num_silently_blocked_threat);
+            return false;
+        }
+    }
+
+    if (dynamo_options.report_max) {
+        /* need bool since ctr only inc-ed when < threshold, so no way
+         * to tell 1st instance beyond threshold from subsequent
+         */
+        static bool reached_max = false;
+        /* do not report in any way if report threshold is reached */
+        DO_THRESHOLD_SAFE(dynamo_options.report_max, FREQ_PROTECTED_SECTION,
+                          {/* < report_max */}, {
+            /* >= report_max */
+            if (!reached_max) {
+                reached_max = true;
+                SYSLOG(SYSLOG_WARNING, WARNING_REPORT_THRESHOLD, 2, 
+                       get_application_name(), get_application_pid());
+            }
+            return false;
+        });
+    }
+
+    /* options already synchronized by security_violation() */
+    if ((TEST(DUMPCORE_SECURITY_VIOLATION, DYNAMO_OPTION(dumpcore_mask))
+#ifdef HOT_PATCHING_INTERFACE   /* Part of fix for 5367. */
+         && violation_type != HOT_PATCH_DETECTOR_VIOLATION
+         && violation_type != HOT_PATCH_PROTECTOR_VIOLATION
+#endif
+        )
+#ifdef HOT_PATCHING_INTERFACE   /* Part of fix for 5367. */
+        /* Dump core if violation was for hot patch detector/protector and
+         * the corresponding dumpcore_mask flag was set.
+         */
+        || (TEST(DUMPCORE_HOTP_DETECTION, DYNAMO_OPTION(dumpcore_mask)) &&
+         violation_type == HOT_PATCH_DETECTOR_VIOLATION) ||
+        (TEST(DUMPCORE_HOTP_PROTECTION, DYNAMO_OPTION(dumpcore_mask)) &&
+         violation_type == HOT_PATCH_PROTECTOR_VIOLATION)
+#endif
+       ) {
+        DO_THRESHOLD_SAFE(DYNAMO_OPTION(dumpcore_violation_threshold),
+                          FREQ_PROTECTED_SECTION, os_dump_core(name) /* < threshold */,);
+    }
+
+#ifdef HOT_PATCHING_INTERFACE
+    if (violation_type == HOT_PATCH_DETECTOR_VIOLATION ||
+        violation_type == HOT_PATCH_PROTECTOR_VIOLATION) {
+        SYSLOG_CUSTOM_NOTIFY(SYSLOG_ERROR,
+                             IF_WINDOWS_ELSE_0(MSG_HOT_PATCH_VIOLATION), 3, 
+                             (char *) action_message[action], get_application_name(), 
+                             get_application_pid(), name);
+    } else
+#endif
+        SYSLOG_CUSTOM_NOTIFY(SYSLOG_ERROR,
+                             IF_WINDOWS_ELSE_0(action_event_id[action]), 3, 
+                             (char *) action_message[action], get_application_name(), 
+                             get_application_pid(), name);
+
+#ifdef HOT_PATCHING_INTERFACE
+    /* Part of fix for 5367.  For hot patches core dumps and forensics should
+     * be generated only if needed, which is not the case for other violations.
+     */
+    if (!(DYNAMO_OPTION(hotp_diagnostics)) &&
+        (violation_type == HOT_PATCH_DETECTOR_VIOLATION ||
+         violation_type == HOT_PATCH_PROTECTOR_VIOLATION))
+        dump_forensics = false;
+#endif
+#ifdef PROCESS_CONTROL
+    if (!DYNAMO_OPTION(pc_diagnostics) &&           /* Case 11023. */
+        violation_type == PROCESS_CONTROL_VIOLATION)
+        dump_forensics = false;
+#endif
+    /* report_max (above) will limit the number of files created */
+    if (dump_forensics)
+        report_diagnostics(action_message[action], name, violation_type);
+
+    return true;
+}
+
+/* attack handling - reports violation and decides on action - possibly terminates the process
+ * N.B.: we make assumptions about whether the callers of this routine hold
+ * various locks, so be careful when adding new callers.
+ * 
+ * type_handling prescribes per-type handling and is combined with
+ * global options.  It can be used to specify whether to take an
+ * action (and may request specific alternative handling with
+ * OPTION_HANDLING), and whether to report.
+ *
+ * The optional out value result_type can differ from the passed-in violation_type
+ * for exemptions.
+ * Returns an action, with the caller responsible for calling
+ * security_violation_action() if action != ACTION_CONTINUE
+ */
+static action_type_t
+security_violation_internal_main(dcontext_t *dcontext, app_pc addr,
+                                 security_violation_t violation_type, 
+                                 security_option_t type_handling, const char *threat_id,
+                                 const action_type_t desired_action,
+                                 read_write_lock_t *lock,
+                                 security_violation_t *result_type/*OUT*/)
+{
+    /* All violations except hot patch ones will request the safest solution, i.e.,
+     * to terminate the process.  Based on the options used, different ones may be
+     * selected in this function.  However, hot patches can request specific actions
+     * as specified by the hot patch writer.
+     */
+    action_type_t action = desired_action;
+    /* probably best to simply use the default TERMINATE_PROCESS */
+    char name[MAXIMUM_VIOLATION_NAME_LENGTH];
+    bool action_selected = false;
+    bool found_unsupported = false;
+#ifdef HOT_PATCHING_INTERFACE
+    /* Passing the hotp lock as an argument is ugly, but it is the cleanest way
+     * to release the hotp lock for case 7988, otherwise, will have to release 
+     * it in hotp_event_notify and re-acquire it after reporting - really ugly.
+     * Anyway, cleaning up the interface to security_violation is in plan 
+     * for Marlin, a FIXME, case 8079.
+     */
+    ASSERT((DYNAMO_OPTION(hot_patching) && lock == hotp_get_lock()) || lock == NULL);
+#else
+    ASSERT(lock == NULL);
+#endif
+    /* though ASLR handling is currently not using this routine */
+    ASSERT(violation_type != ASLR_TARGET_VIOLATION);
+
+    DOLOG(2, LOG_ALL, {
+        SYSLOG_INTERNAL_INFO("security_violation("PFX", %d)", addr, violation_type);
+        LOG(THREAD, LOG_VMAREAS, 2, "executable areas are:\n");
+        print_executable_areas(THREAD);
+        LOG(THREAD, LOG_VMAREAS, 2, "future executable areas are:\n");
+        read_lock(&futureexec_areas->lock);
+        print_vm_areas(futureexec_areas, THREAD);
+        read_unlock(&futureexec_areas->lock);
+    });
+
+    /* case 8075: we no longer unprot .data on the violation path */
+    ASSERT(check_should_be_protected(DATASEC_RARELY_PROT));
+
+    /* CHECK: all options for attack handling and reporting are dynamic, synchronized only once */
+    synchronize_dynamic_options(); 
+
+#ifdef HOT_PATCHING_INTERFACE
+    if (violation_type == HOT_PATCH_DETECTOR_VIOLATION || 
+        violation_type == HOT_PATCH_PROTECTOR_VIOLATION) {
+        /* For hot patches, the action is provided by the hot patch writer; 
+         * nothing should be selected here.
+         */
+        action_selected = true;
+    }
+#endif
+#ifdef PROCESS_CONTROL
+    /* A process control violation (which can only happen if process control is
+     * turned on) results in the process being killed unless it is running in
+     * detect mode.
+     */
+    if (violation_type == PROCESS_CONTROL_VIOLATION) {
+        ASSERT(IS_PROCESS_CONTROL_ON());
+        ASSERT((action == ACTION_TERMINATE_PROCESS &&
+                !DYNAMO_OPTION(pc_detect_mode)) ||
+               (action == ACTION_CONTINUE && DYNAMO_OPTION(pc_detect_mode)));
+        action_selected = true;
+    }
+#endif
+    /* one last chance to avoid a violation */
+    get_security_violation_name(dcontext, addr, name,
+                                MAXIMUM_VIOLATION_NAME_LENGTH, violation_type,
+                                threat_id);
+    if (!IS_STRING_OPTION_EMPTY(exempt_threat_list)) {
+        if (is_exempt_threat_name(name)) {
+            if (result_type != NULL)
+                *result_type = ALLOWING_BAD;
+            mark_module_exempted(addr);
+            return ACTION_CONTINUE;
+        }
+    }
+
+    /* FIXME: if we reinstate case 6141 where we acquire the thread_initexit_lock
+     * we'll need to release our locks!
+     * See ifdef FORENSICS_ACQUIRES_INITEXIT_LOCK in the Attic.
+     * FIXME: even worse, we'll crash w/ case 9381 if we get a flush
+     * while we're nolinking due to init-extra-vmareas on the frags list!
+     */
+
+    /* diagnose_violation_mode says to check if would have allowed if were allowing patterns */
+    if (dynamo_options.diagnose_violation_mode && !dynamo_options.executable_if_trampoline) {
+        size_t junk;
+        if (check_origins_bb_pattern(dcontext, addr, (app_pc *) &junk, &junk,
+                                     (uint *) &junk, (uint *) &junk)
+            == ALLOWING_OK) {
+            /* FIXME: change later user-visible message to indicate this may be
+             * a false positive
+             */
+            SYSLOG_INTERNAL_WARNING_ONCE("would have allowed pattern DGC.");
+        }
+    }
+#ifdef DGC_DIAGNOSTICS
+    LOG(GLOBAL, LOG_VMAREAS, 1, "violating basic block target:\n");
+    DOLOG(1, LOG_VMAREAS, { disassemble_app_bb(dcontext, addr, GLOBAL); });
+#endif
+    /* for non-debug build, give some info on violating block */
+    DODEBUG({
+        if (is_readable_without_exception(addr, 12)) {
+            SYSLOG_INTERNAL_WARNING("violating basic block target @"PFX": "
+                                    "%x %x %x %x %x %x %x %x %x %x %x %x", addr, 
+                                    *addr, *(addr+1), *(addr+2), *(addr+3), *(addr+4), 
+                                    *(addr+5), *(addr+6), *(addr+7), *(addr+8), 
+                                    *(addr+9), *(addr+10), *(addr+11));
+        } else
+            SYSLOG_INTERNAL_WARNING("violating basic block target @"PFX": not readable!",
+                                    addr);
+    });
+
+    if (DYNAMO_OPTION(detect_mode) && !TEST(OPTION_BLOCK_IGNORE_DETECT, type_handling)
+        /* As of today, detect mode for hot patches is set using modes files. */
+        IF_HOTP(&& violation_type != HOT_PATCH_DETECTOR_VIOLATION 
+                && violation_type != HOT_PATCH_PROTECTOR_VIOLATION)) {
+        bool allow = true;
+        /* would be nice to keep the count going when no max, so if dynamically impose
+         * one later all the previous ones count toward it, but then have to worry about
+         * overflow of counter, etc. -- so we ignore count while there's no max
+         */
+        if (DYNAMO_OPTION(detect_mode_max) > 0) {
+            /* global counter for violations in all threads */
+            DO_THRESHOLD_SAFE(DYNAMO_OPTION(detect_mode_max), FREQ_PROTECTED_SECTION,
+                              {/* < max */
+                                  LOG(GLOBAL, LOG_ALL, 1, 
+                                      "security_violation: allowing violation #%d [max %d], tid=%d\n", 
+                                      do_threshold_cur, DYNAMO_OPTION(detect_mode_max),
+                                      get_thread_id());
+                              },
+                              {/* >= max */
+                                  allow = false;
+                                  LOG(GLOBAL, LOG_ALL, 1, 
+                                      "security_violation: reached maximum allowed %d, tid=%d\n", 
+                                      DYNAMO_OPTION(detect_mode_max), get_thread_id());
+                              });
+        } else {
+            LOG(GLOBAL, LOG_ALL, 1, 
+                "security_violation: allowing violation, no max, tid=%d\n", get_thread_id());
+        }
+        if (allow) {
+            /* we have priority over other handling options */
+            action = ACTION_CONTINUE;
+            action_selected = true;
+            mark_module_exempted(addr);
+       }
+    }
+
+    /* FIXME: case 2144 we need to TEST(OPTION_BLOCK early on so that
+     * we do not impact the counters, in addition we need to
+     * TEST(OPTION_HANDLING to specify an alternative attack handling
+     * (e.g. -throw_exception if default is -kill_thread) 
+     * FIXME: We may also want a different message to allow 'staging' events to be
+     * considered differently, maybe with a DO_ONCE semantics...
+     */
+
+    /* decide on specific attack handling action if not continuing */
+    if (!action_selected && DYNAMO_OPTION(throw_exception)) {
+        thread_data_t *thread_local = (thread_data_t *) dcontext->vm_areas_field;
+        /* maintain a thread local counter to bail out and avoid infinite exceptions */
+        if (thread_local->thrown_exceptions < DYNAMO_OPTION(throw_exception_max_per_thread)) {
+#  ifdef WINDOWS
+            /* If can't verify consistent SEH chain should fall through to kill path */
+            /* UnhandledExceptionFilter is always installed. */
+            /* There is no point in throwing an exception if no other
+               handlers are installed to unwind.
+               We may still get there when our exception is not handled, 
+               but at least cleanup code will be given a chance. 
+            */
+            enum {
+                MIN_SEH_DEPTH = 1
+                /* doesn't seem to deserve a separate option */
+            };
+            int seh_chain_depth = exception_frame_chain_depth(dcontext);
+            if (seh_chain_depth > MIN_SEH_DEPTH) {
+                /* note the check is best effort, 
+                   e.g. attacked handler can still point to valid RET */
+                bool global_max_reached = true;
+                /* check global counter as well */
+                DO_THRESHOLD_SAFE(DYNAMO_OPTION(throw_exception_max),
+                                  FREQ_PROTECTED_SECTION,
+                                  {global_max_reached = false;}, 
+                                  {global_max_reached = true;});
+                if (!global_max_reached) {
+                    thread_local->thrown_exceptions++;
+                    LOG(GLOBAL, LOG_ALL, 1, 
+                        "security_violation: throwing exception %d for this thread [max pt %d] [global max %d]\n", 
+                        thread_local->thrown_exceptions,
+                        dynamo_options.throw_exception_max_per_thread, 
+                        dynamo_options.throw_exception_max);
+                    action = ACTION_THROW_EXCEPTION;
+                    action_selected = true;
+                }
+            } else {
+                LOG(GLOBAL, LOG_ALL, 1, 
+                    "security_violation: SEH chain invalid [%d], better kill\n", seh_chain_depth);
+            }
+#  else
+            ASSERT_NOT_IMPLEMENTED(false);
+#  endif /* WINDOWS */
+        } else {
+            LOG(GLOBAL, LOG_ALL, 1, 
+                "security_violation: reached maximum exception count, kill now\n");
+        }
+    }
+    
+    /* kill process or maybe thread */
+    if (!action_selected) {
+        ASSERT(action == ACTION_TERMINATE_PROCESS);
+        if (DYNAMO_OPTION(kill_thread)) {
+            /* check global counter as well */
+            DO_THRESHOLD_SAFE(DYNAMO_OPTION(kill_thread_max), FREQ_PROTECTED_SECTION,
+                {/* < max */
+                    LOG(GLOBAL, LOG_ALL, 1, 
+                        "security_violation: \t killing thread #%d [max %d], tid=%d\n", 
+                        do_threshold_cur, DYNAMO_OPTION(kill_thread_max),
+                        get_thread_id());
+                    /* FIXME: can't check if get_num_threads()==1 then say we're killing process
+                     * because it is possible that another thread has not been scheduled yet 
+                     * and we wouldn't have seen it.
+                     * Still, only our message will be wrong if we end up killing the process, 
+                     * when we terminate the last thread 
+                     */
+                    action = ACTION_TERMINATE_THREAD;
+                    action_selected = true;
+                },
+                {/* >= max */
+                    LOG(GLOBAL, LOG_ALL, 1, 
+                        "security_violation: reached maximum thread kill, kill process now\n");
+                    action = ACTION_TERMINATE_PROCESS;
+                    action_selected = true;
+                });
+        } else {
+            action = ACTION_TERMINATE_PROCESS;
+            action_selected = true;
+        }
+    }
+    ASSERT(action_selected);
+
+#ifdef CLIENT_INTERFACE
+    /* Case 9712: Inform the client of the security violation and 
+     * give it a chance to modify the action.
+     */
+    if (!IS_INTERNAL_STRING_OPTION_EMPTY(client_lib)) {
+        instrument_security_violation(dcontext, addr, violation_type, &action);
+    }
+#endif
+
+    /* now we know what is the chosen action and we can report */
+    if (TEST(OPTION_REPORT, type_handling)) 
+        security_violation_report(addr, violation_type, name, action);
+
+    /* FIXME: walking the loader data structures at arbitrary 
+     * points is dangerous due to data races with other threads 
+     * -- see is_module_being_initialized and get_module_name */
+    if (check_for_unsupported_modules()) {
+        /* found an unsupported module */
+        action = ACTION_TERMINATE_PROCESS;
+        found_unsupported = true;
+        /* NOTE that because of the violation_threshold this
+         * check isn't actually sufficient to ensure we get a dump file
+         * (if for instance already got several violations) but it's good
+         * enough */
+        if (TEST(DUMPCORE_UNSUPPORTED_APP, 
+                 DYNAMO_OPTION(dumpcore_mask)) &&
+            !TEST(DUMPCORE_SECURITY_VIOLATION, 
+                  DYNAMO_OPTION(dumpcore_mask))) {
+            os_dump_core("unsupported module");
+        }
+    }
+
+#ifdef WINDOWS
+    if (ACTION_TERMINATE_PROCESS == action &&
+        (TEST(DETACH_UNHANDLED_VIOLATION, 
+              DYNAMO_OPTION(internal_detach_mask)) || 
+         (found_unsupported && TEST(DETACH_UNSUPPORTED_MODULE, 
+                                    DYNAMO_OPTION(internal_detach_mask))))) {
+        /* set pc to right value and detach */
+        get_mcontext(dcontext)->pc = addr;
+        /* FIXME - currently detach_internal creates a new thread to do the
+         * detach (case 3312) and if we hold an app lock used by the init apc
+         * such as the loader lock (case 4486) we could livelock the process
+         * if we used a synchronous detach.  Instead, we set detach in motion
+         * disable all future violations and continue.
+         */
+        detach_internal();
+        options_make_writable();
+        /* make sure synchronizes won't clobber the changes here */
+        dynamo_options.dynamic_options = false;
+        dynamo_options.detect_mode = true;
+        dynamo_options.detect_mode_max = 0; /* no limit on detections */
+        dynamo_options.report_max = 1; /* don't report any more */
+        options_restore_readonly();
+        action = ACTION_CONTINUE;
+    }
+#endif
+
+    /* FIXME: move this into hotp code like we've done for bb building so we
+     * don't need to pass the lock in anymore
+     */
+#ifdef HOT_PATCHING_INTERFACE
+    /* Fix for case 7988.  Release the hotp lock when the remediation action 
+     * is to terminate the {thread,process} or to throw an exception, otherwise 
+     * we will deadlock trying to access the hotp_vul_table in another thread.
+     */
+    if (lock != NULL && (action == ACTION_TERMINATE_THREAD ||
+                         action == ACTION_TERMINATE_PROCESS ||
+                         action == ACTION_THROW_EXCEPTION)) {
+#ifdef GBOP
+        ASSERT(violation_type == HOT_PATCH_DETECTOR_VIOLATION ||
+               violation_type == HOT_PATCH_PROTECTOR_VIOLATION ||
+               violation_type == GBOP_SOURCE_VIOLATION);
+#else
+        ASSERT(violation_type == HOT_PATCH_DETECTOR_VIOLATION ||
+               violation_type == HOT_PATCH_PROTECTOR_VIOLATION);
+#endif
+        ASSERT_OWN_READ_LOCK(true, lock);
+        read_unlock(lock);
+    }
+#endif
+
+    if (result_type != NULL)
+        *result_type = violation_type;
+    return action;
+}
+
+/* Meant to be called after security_violation_internal_main().
+ * Caller should only call for action!=ACTION_CONTINUE.
+ */
+void
+security_violation_action(dcontext_t *dcontext, action_type_t action, app_pc addr)
+{
+    ASSERT(action != ACTION_CONTINUE);
+    if (action == ACTION_CONTINUE)
+        return;
+
+    /* timeout before we take an action */
+    if (dynamo_options.timeout) {
+        /* For now assuming only current thread sleeps.
+           FIXME: If we are about the kill the process anyways,
+           it may be safer to stop_the_world,
+           so attacks in this time window do not get through.
+           
+           TODO: On the other hand sleeping in one thread,
+           while the rest are preparing for controlled shutdown sounds better,
+           yet we have no way of telling them that process death is pending.
+        */
+        /* FIXME: shouldn't we suspend all other threads for the messagebox too? */
+        
+        /* For services you can get a similar effect to -timeout on kill process
+           by settings in Services\service properties\Recovery.
+           Restart service after x minutes.
+           0 is very useful - then you get your app back immediately.
+           1 minute however may be too much in some circumstances,
+           Our option is then useful for finer control, e.g. -timeout 10s
+        */
+        os_timeout(dynamo_options.timeout);
+    }
+
+    if (ACTION_THROW_EXCEPTION == action) {
+        os_forge_exception(addr, UNREADABLE_MEMORY_EXECUTION_EXCEPTION);
+        ASSERT_NOT_REACHED();
+    }
+    if (ACTION_CONTINUE != action) {
+        uint terminate_flags_t = TERMINATE_PROCESS;
+        if (is_self_couldbelinking()) {
+            /* must be nolinking for terminate cleanup to avoid deadlock w/ flush */
+            enter_nolinking(dcontext, NULL, false/*not a real cache transition*/);
+        }
+        if (action == ACTION_TERMINATE_THREAD) {
+            terminate_flags_t = TERMINATE_THREAD;
+            /* clean up when terminating a thread */
+            terminate_flags_t |= TERMINATE_CLEANUP;
+        } else {
+            ASSERT(action == ACTION_TERMINATE_PROCESS &&
+                   terminate_flags_t == TERMINATE_PROCESS);
+        }
+#ifdef HOT_PATCHING_INTERFACE
+        ASSERT(!DYNAMO_OPTION(hot_patching) ||
+               !READ_LOCK_HELD(hotp_get_lock()));   /* See case 7998. */
+#endif
+        os_terminate(dcontext, terminate_flags_t);
+        ASSERT_NOT_REACHED();
+    }
+    ASSERT_NOT_REACHED();
+}
+
+/* Caller must call security_violation_action() if return != ACTION_CONTINUE */
+static action_type_t
+security_violation_main(dcontext_t *dcontext, app_pc addr,
+                        security_violation_t violation_type,
+                        security_option_t type_handling)
+{
+    return security_violation_internal_main(dcontext, addr, violation_type, 
+                                            type_handling, NULL, 
+                                            ACTION_TERMINATE_PROCESS, NULL, NULL);
+}
+
+/* See security_violation_internal_main() for further comments.
+ *
+ * Returns ALLOWING_BAD if on exempt_threat_list, or if in detect mode
+ * returns the passed violation_type (a negative value)
+ * Does not return if protection action is taken.  
+ */
+security_violation_t
+security_violation_internal(dcontext_t *dcontext, app_pc addr,
+                            security_violation_t violation_type, 
+                            security_option_t type_handling, const char *threat_id,
+                            const action_type_t desired_action,
+                            read_write_lock_t *lock)
+{
+    security_violation_t result_type;
+    action_type_t action =
+        security_violation_internal_main(dcontext, addr, violation_type, 
+                                         type_handling, threat_id, desired_action,
+                                         lock, &result_type);
+    DOKSTATS(if (ACTION_CONTINUE != action) {
+        KSTOP_REWIND_UNTIL(dispatch_num_exits);
+    });
+    if (action != ACTION_CONTINUE)
+        security_violation_action(dcontext, action, addr);
+    return result_type;
+}
+
+/* security_violation_internal() is the real function.  This a wrapper exists
+ * for two reasons; one, hot patching needs to send extra arguments for event
+ * notification and two, existing calls to security_violation() in the code
+ * shouldn't have to change the interface.
+ */
+security_violation_t
+security_violation(dcontext_t *dcontext, app_pc addr,
+                   security_violation_t violation_type, 
+                   security_option_t type_handling)
+{
+    return security_violation_internal(dcontext, addr, violation_type, 
+                                       type_handling, NULL, 
+                                       ACTION_TERMINATE_PROCESS, NULL);
+}
+
+/* back to normal section */
+END_DATA_SECTION()
+/****************************************************************************/
+
+bool
+is_dyngen_vsyscall(app_pc addr)
+{
+    /* FIXME: on win32, should we only allow portion of page? */
+    /* CHECK: likely to be true on all Linux versions by the time we ship */
+    /* if vsyscall_page_start == 0, then this exception doesn't apply */
+    /* Note vsyscall_page_start is a global defined in the corresponding os.c files */
+    if (vsyscall_page_start == 0) 
+        return false;
+    return (addr >= (app_pc) vsyscall_page_start && 
+            addr < (app_pc) (vsyscall_page_start+PAGE_SIZE));
+}
+
+bool
+is_in_futureexec_area(app_pc addr)
+{
+    bool future;
+    read_lock(&futureexec_areas->lock);
+    future = lookup_addr(futureexec_areas, addr, NULL);
+    read_unlock(&futureexec_areas->lock);
+    return future;
+}
+
+bool
+is_dyngen_code(app_pc addr)
+{
+    uint flags;
+    if (get_executable_area_flags(addr, &flags)) {
+        /* assuming only true DGC is marked DYNGEN  */
+        return TEST(FRAG_DYNGEN, flags);
+    }
+
+    return is_in_futureexec_area(addr);
+}
+
+/* Returns true if in is a direct jmp targeting a known piece of non-DGC code
+ */
+static bool
+is_direct_jmp_to_image(dcontext_t *dcontext, instr_t *in)
+{
+    bool ok = false;
+    if (instr_get_opcode(in) == OP_jmp && /* no short jmps */
+        opnd_is_near_pc(instr_get_target(in))) {
+        app_pc target = opnd_get_pc(instr_get_target(in));
+        uint flags;
+        if (get_executable_area_flags(target, &flags)) {
+            /* we could test for UNMOD_IMAGE but that would ruin windows
+             * loader touch-ups, which can happen for any dll!
+             * so we test FRAG_DYNGEN instead
+             */
+            ok = !TEST(FRAG_DYNGEN, flags);
+        }
+    }
+    return ok;
+}
+
+/* allow original code displaced by a hook, seen for Citrix 4.0 (case 6615):
+ *   <zero or more non-cti and non-syscall instrs whose length < 5>
+ *   <one more such instr, making length sum X>
+ *   jmp <dll:Y>, where <dll:Y-X> contains a jmp to this page
+ */
+static bool
+check_trampoline_displaced_code(dcontext_t *dcontext, app_pc addr, bool on_stack,
+                                instrlist_t *ilist, size_t *len)
+{
+    uint size = 0;
+    bool match = false;
+    instr_t *in, *last = instrlist_last(ilist);
+    ASSERT(DYNAMO_OPTION(trampoline_displaced_code));
+    if (on_stack || !is_direct_jmp_to_image(dcontext, last))
+        return false;
+    ASSERT(instr_length(dcontext, last) == JMP_LONG_LENGTH);
+    for (in = instrlist_first(ilist);
+         in != NULL/*sanity*/ && in != last;
+         in = instr_get_next(in)) {
+        /* build_app_bb_ilist should fully decode everything */
+        ASSERT(instr_opcode_valid(in));
+        if (instr_is_cti(in) || instr_is_syscall(in) || instr_is_interrupt(in))
+            break;
+        size += instr_length(dcontext, in);
+        if (instr_get_next(in) == last) {
+            if (size < JMP_LONG_LENGTH)
+                break;
+        } else {
+            if (size >= JMP_LONG_LENGTH)
+                break;
+        }
+    }
+    ASSERT(in != NULL);
+    if (in == last) {
+        app_pc target;
+        LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 3,
+            "check_trampoline_displaced_code @"PFX": size=%d\n",
+            addr, size);
+        DOLOG(3, LOG_INTERP|LOG_VMAREAS, {
+            instrlist_disassemble(dcontext, addr, ilist, THREAD);
+        });
+        /* is_direct_jmp_to_image should have checked for us */
+        ASSERT(opnd_is_near_pc(instr_get_target(last)));
+        target = opnd_get_pc(instr_get_target(last));
+        if (is_readable_without_exception(target - size, JMP_LONG_LENGTH)) {
+            instr_t *tramp = instr_create(dcontext);
+            /* Ensure a racy unmap causing a decode crash is passed to the app */
+            set_thread_decode_page_start(dcontext, (app_pc) PAGE_START(target - size));
+            target = decode_cti(dcontext, target - size, tramp);
+            if (target != NULL && instr_opcode_valid(tramp) &&
+                instr_is_ubr(tramp) && opnd_is_near_pc(instr_get_target(tramp))) {
+                app_pc hook = opnd_get_pc(instr_get_target(tramp));
+                /* FIXME: could be tighter by ensuring that hook targets a jmp
+                 * or call right before addr but that may be too specific.
+                 * FIXME: if the pattern crosses a page we could fail to match.
+                 * we could check for being inside region instead.
+                 */
+                if (PAGE_START(hook) == PAGE_START(addr)) {
+                    *len = size + JMP_LONG_LENGTH;
+                    LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+                        "WARNING: allowing hook-displaced code "PFX" -> "PFX" -> "PFX"\n",
+                        addr, target, hook);
+                    SYSLOG_INTERNAL_WARNING_ONCE("hook-displaced code allowed.");
+                    STATS_INC(trampolines_displaced_code);
+                    match = true;
+                }
+            }
+            instr_destroy(dcontext, tramp);
+        }
+    }
+    return match;
+}
+
+/* other than JITed code, we allow a small set of specific patterns of DGC such
+ * as function closure trampolines, which this routine checks for.
+ * returns ALLOWING_OK if bb matches, else returns ALLOWING_BAD
+ */
+static int
+check_origins_bb_pattern(dcontext_t *dcontext, app_pc addr, app_pc *base, size_t *size,
+                         uint *vm_flags, uint *frag_flags)
+{
+    /* we assume this is not a cti target (flag diffs will prevent direct cti here)
+     * we only check for the bb beginning at addr
+     */
+    instrlist_t *ilist;
+    instr_t *in, *first;
+    opnd_t op;
+    size_t len = 0;
+    int res = ALLOWING_BAD; /* signal to caller not a match */
+    bool on_stack = is_on_stack(dcontext, addr, NULL);
+
+    /* FIXME: verify bb memory is readable prior to decoding it
+     * we shouldn't get here if addr is unreadable, but rest of bb could be
+     * note that may end up looking at win32 GUARD page -- don't need to do
+     * anything special since that will look unreadable
+     */
+    /* FIXME bug 9376: if unreadable check_thread_vm_area() will
+     * assert vmlist!=NULL and throw an exception, which is ok
+     */
+    ilist = build_app_bb_ilist(dcontext, addr, INVALID_FILE);
+    first = instrlist_first(ilist);
+    if (first == NULL) /* empty bb: perhaps invalid instr */
+        goto check_origins_bb_pattern_exit;
+
+    LOG(GLOBAL, LOG_VMAREAS, 3, "check_origins_bb_pattern:\n");
+    DOLOG(3, LOG_VMAREAS, { instrlist_disassemble(dcontext, addr, ilist, GLOBAL); });
+
+#ifndef X86
+    /* FIXME: move the x86-specific analysis to an x86/ file! */
+    ASSERT_NOT_IMPLEMENTED();
+#endif
+
+#ifdef LINUX
+    /* is this a sigreturn pattern placed by kernel on the stack or vsyscall page? */
+    if (is_signal_restorer_code(addr, &len)) {
+        LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 2,
+            "found signal restorer code @"PFX", allowing it\n", addr);
+        SYSLOG_INTERNAL_WARNING_ONCE("signal restorer code allowed.");
+        res = ALLOWING_OK;
+        goto check_origins_bb_pattern_exit;
+    }
+#endif
+
+    /* is this a closure trampoline that looks like this:
+     *   mov immed -> 0x4(esp)             (put frame ptr directly in slot)
+     *   jmp known-non-DGC-address
+     * or like this (gcc-style, also seen in dfrgui):
+     *   mov immed -> %ecx                 (put frame ptr in ecx, callee puts in slot)
+     *   jmp known-non-DGC-address
+     * OR, is this some sort of C++ exception chaining (seen in soffice):
+     *   mov immed -> %eax                 (put try index in eax)
+     *   jmp known-non-DGC-address
+     * these can be on the stack or on the heap, except the soffice one, which must
+     * be on the heap (simply b/c we've never seen it on the stack)
+     * all of these must be targeted by a call
+     */
+    if (instr_get_opcode(first) == OP_mov_imm ||
+        /* funny case where store of immed is mov_st -- see x86/decode_table.c */
+        (instr_get_opcode(first) == OP_mov_st &&
+         opnd_is_immed(instr_get_src(first, 0)))) {
+        bool ok = false;
+        LOG(GLOBAL, LOG_VMAREAS, 3, "testing for mov immed pattern\n");
+        /* mov_imm always has immed src, just check dst */
+        op = instr_get_dst(first, 0);
+        ok = (opnd_is_near_base_disp(op) && opnd_get_base(op) == REG_XSP &&
+              opnd_get_disp(op) == 4 && opnd_get_scale(op) == REG_NULL);
+
+        if (!ok && opnd_is_reg(op) && opnd_get_size(instr_get_src(first, 0)) == OPSZ_4) {
+            uint immed = (uint) opnd_get_immed_int(instr_get_src(first, 0));
+            /* require immed be addr for ecx, non-addr plus on heap for eax */
+            /* FIXME: PAGE_SIZE is arbitrary restriction, assuming eax values
+             * are small indices, and it's a nice way to distinguish pointers
+             */
+            IF_X64(ASSERT_NOT_TESTED()); /* on x64 will these become rcx & rax? */
+            ok = (opnd_get_reg(op) == REG_ECX && immed > PAGE_SIZE) ||
+                (opnd_get_reg(op) == REG_EAX && immed < PAGE_SIZE && !on_stack);
+        }
+
+        if (ok) {
+            /* check 2nd instr */
+            ok = false;
+            len += instr_length(dcontext, first);
+            in = instr_get_next(first);
+            if (instr_get_next(in) == NULL && /* only 2 instrs in this bb */
+                is_direct_jmp_to_image(dcontext, in)) {
+                len += instr_length(dcontext, in);
+                ok = true;
+            } else
+                LOG(GLOBAL, LOG_VMAREAS, 3, "2nd instr not jmp to good code!\n");
+        } else
+            LOG(GLOBAL, LOG_VMAREAS, 3, "immed bad!\n");
+
+        if (ok) {
+            /* require source to be known and to be a call
+             * cases where source is unknown are fairly pathological
+             * (another thread flushing and deleting the fragment, etc.)
+             */
+            ok = TEST(LINK_CALL, dcontext->last_exit->flags);
+        }
+        if (ok) {
+            LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 2,
+                "WARNING: found trampoline block @"PFX", allowing it\n", addr);
+            SYSLOG_INTERNAL_WARNING_ONCE("trampoline DGC allowed.");
+            res = ALLOWING_OK;
+            goto check_origins_bb_pattern_exit;
+        }
+    }
+
+    /* is this a PLT-type push/jmp, where the push uses its own address
+     * (this is seen in soffice):
+     *   push own-address
+     *   jmp known-non-DGC-address
+     */
+    if (instr_get_opcode(first) == OP_push_imm &&
+        opnd_get_size(instr_get_src(first, 0)) == OPSZ_4) {
+        ptr_uint_t immed = opnd_get_immed_int(instr_get_src(first, 0));
+        LOG(GLOBAL, LOG_VMAREAS, 3, "testing for push immed pattern\n");
+        if ((app_pc)immed == addr) {
+            len += instr_length(dcontext, first);
+            in = instr_get_next(first);
+            if (instr_get_next(in) == NULL && /* only 2 instrs in this bb */
+                is_direct_jmp_to_image(dcontext, in)) {
+                len += instr_length(dcontext, in);
+                LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 2,
+                    "WARNING: found push/jmp block @"PFX", allowing it\n", addr);
+                SYSLOG_INTERNAL_WARNING_ONCE("push/jmp DGC allowed.");
+                res = ALLOWING_OK;
+                goto check_origins_bb_pattern_exit;
+            }
+        }
+    }
+
+    /* look for the DGC ret on the stack that office xp uses, beyond TOS! 
+     * it varies between having no arg or having an immed arg -- my guess
+     * is they use it to handle varargs with stdcall: callee must
+     * clean up args but has to deal w/ dynamically varying #args, so
+     * they use DGC ret, only alternative is jmp* and no ret
+     */
+    if (instr_is_return(first) && on_stack &&
+        addr < (app_pc) get_mcontext(dcontext)->xsp) { /* beyond TOS */
+        ASSERT(instr_get_next(first) == NULL); /* bb should have only ret in it */
+        len = instr_length(dcontext, first);
+        LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 2,
+            "WARNING: found ret-beyond-TOS @"PFX", allowing it\n", addr);
+        SYSLOG_INTERNAL_WARNING_ONCE("ret-beyond-TOS DGC allowed.");
+        res = ALLOWING_OK;
+        goto check_origins_bb_pattern_exit;
+    }
+
+    if (DYNAMO_OPTION(trampoline_dirjmp) &&
+        !on_stack && is_direct_jmp_to_image(dcontext, first)) {
+        /* should be a lone jmp */
+        ASSERT(instr_get_next(first) == NULL);
+        len = instr_length(dcontext, first);
+        LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+            "WARNING: allowing targeted direct jmp @"PFX"\n", addr);
+        SYSLOG_INTERNAL_WARNING_ONCE("trampoline direct jmp allowed.");
+        STATS_INC(trampolines_direct_jmps);
+        res = ALLOWING_OK;
+        goto check_origins_bb_pattern_exit;
+    }
+
+    /* allow a .NET COM method table: a lone direct call on the heap, and a
+     * ret immediately preceding it (see case 3558 and case 3564)
+     */
+    if (DYNAMO_OPTION(trampoline_dircall) &&
+        !on_stack && instr_is_call_direct(first)) {
+        len = instr_length(dcontext, first);
+        /* ignore rest of ilist -- may or may not follow call for real bb, as
+         * will have separate calls to check_thread_vm_area() and thus
+         * separate code origins checks being applied to the target, making this
+         * not really a security hole at all as attack could have sent control
+         * directly to target
+         */
+        LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 2,
+            "WARNING: allowing targeted direct call @"PFX"\n", addr);
+        SYSLOG_INTERNAL_WARNING_ONCE("trampoline direct call allowed.");
+        STATS_INC(trampolines_direct_calls);
+        res = ALLOWING_OK;
+        goto check_origins_bb_pattern_exit;
+    }
+    if (DYNAMO_OPTION(trampoline_com_ret) && !on_stack && instr_is_return(first)) {
+        app_pc nxt_pc = addr + instr_length(dcontext, first);
+        if (is_readable_without_exception(nxt_pc, MAX_INSTR_LENGTH)) {
+            instr_t *nxt = instr_create(dcontext);
+            /* WARNING: until our decoding is more robust, as this is AFTER a
+             * ret this could fire a decode assert if not actually code there,
+             * so we avoid any more decoding than we have to do w/ decode_cti.
+             */
+            /* A racy unmap could cause a fault here so we track the page
+             * that's being decoded. */
+            set_thread_decode_page_start(dcontext, (app_pc) PAGE_START(nxt_pc));
+            nxt_pc = decode_cti(dcontext, nxt_pc, nxt);
+            if (nxt_pc != NULL && instr_opcode_valid(nxt) && instr_is_call_direct(nxt)) {
+                /* actually we don't get here w/ current native_exec early-gateway
+                 * design since we go native at the PREVIOUS call to this ret's call
+                 */
+                ASSERT_NOT_TESTED();
+                instr_destroy(dcontext, nxt);
+                len = instr_length(dcontext, first);
+                LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 2,
+                    "WARNING: allowing .NET COM ret in method table @"PFX"\n", addr);
+                SYSLOG_INTERNAL_WARNING_ONCE(".NET COM method table ret allowed.");
+                STATS_INC(trampolines_com_rets);
+                res = ALLOWING_OK;
+                goto check_origins_bb_pattern_exit;
+            }
+            instr_destroy(dcontext, nxt);
+        }
+    }
+
+    if (DYNAMO_OPTION(trampoline_displaced_code) &&
+        check_trampoline_displaced_code(dcontext, addr, on_stack, ilist, &len)) {
+        res = ALLOWING_OK;
+        goto check_origins_bb_pattern_exit;
+    }
+
+ check_origins_bb_pattern_exit:
+    if (res == ALLOWING_OK) {
+        /* bb matches pattern, let's allow it, but only this block, not entire region! */
+        LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 2,
+            "Trimming exec area "PFX"-"PFX" to match pattern bb "PFX"-"PFX"\n",
+            *base, *base+*size, addr, addr+len);
+        *base = addr;
+        ASSERT(len > 0);
+        *size = len;
+        /* Since this is a sub-page region that shouldn't be frequently
+         * executed, it's best to use sandboxing.
+         */
+        *frag_flags |= SANDBOX_FLAG();
+        /* ensure another thread is not able to use this memory region for
+         * a non-pattern-matching code sequence
+         */
+        *vm_flags |= VM_PATTERN_REVERIFY;
+        STATS_INC(num_selfmod_vm_areas);
+    }
+    instrlist_clear_and_destroy(dcontext, ilist);
+    return res;
+}
+
+/* trims [base, base+size) to its intersection with [start, end)
+ * NOTE - regions are required to intersect */
+static void
+check_origins_trim_region_helper(app_pc *base /*INOUT*/, size_t *size /*INOUT*/,
+                                 app_pc start, app_pc end) {
+    app_pc original_base = *base;
+    ASSERT(*base + *size > *base); /* shouldn't overflow */
+    ASSERT(start < end); /* [start, end) should be an actual region */
+    ASSERT(*base + *size > start &&  *base < end); /* region must intersect */
+    LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 2,
+        "Trimming exec area "PFX"-"PFX" to intersect area "PFX"-"PFX"\n",
+        *base, *base+*size, start, end);
+    *base = MAX(*base, start);
+    /* don't use new base here! (case 8152) */
+    *size = MIN(original_base+*size, end) - *base;
+}
+
+/* Checks if the given PC is trusted and to what level
+ * if execution for the referenced area is not allowed program execution
+ * should be aborted
+ */
+static INLINE_ONCE security_violation_t
+check_origins_helper(dcontext_t *dcontext, app_pc addr, app_pc *base, size_t *size,
+                     uint prot, uint *vm_flags, uint *frag_flags, const char *modname)
+{
+    vm_area_t *fut_area;
+
+    if (is_dyngen_vsyscall(addr) && *size == PAGE_SIZE && (prot & MEMPROT_WRITE) == 0) {
+        /* FIXME: don't allow anyone to make this region writable? */
+        LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 2,
+            PFX" is the vsyscall page, ok to execute\n", addr);
+        return ALLOWING_OK;
+    }
+#if 0
+    /* this syslog causes services.exe to hang (ref case 666) once case 666
+     * is fixed re-enable if desired FIXME */
+    SYSLOG_INTERNAL_WARNING_ONCE("executing region at "PFX" not on executable list.",
+                                 addr);
+#else
+    LOG(GLOBAL, LOG_VMAREAS, 1, 
+        "executing region at "PFX" not on executable list. Thread %d\n", 
+        addr, dcontext->owning_thread);
+#endif
+
+    if (USING_FUTURE_EXEC_LIST) {
+        bool ok;
+        bool once_only;
+        read_lock(&futureexec_areas->lock);
+        ok = lookup_addr(futureexec_areas, addr, &fut_area);
+        if (!ok)
+            read_unlock(&futureexec_areas->lock);
+        else {
+            LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+                "WARNING: pc = "PFX" is future executable, allowing\n", addr);
+            LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 2,
+                "WARNING: pc = "PFX" is future executable, allowing\n", addr);
+#if 0
+            /* this syslog causes services.exe to hang (ref case 666) 
+             * once case 666 is fixed re-enable if desired FIXME */
+            SYSLOG_INTERNAL_WARNING_ONCE("future executable region allowed.");
+#else
+            DODEBUG_ONCE(LOG(GLOBAL, LOG_ALL, 1, "future executable region allowed."));
+#endif
+            if (*base < fut_area->start || *base+*size > fut_area->end) {
+                check_origins_trim_region_helper(base, size,
+                                                 fut_area->start, fut_area->end);
+            }
+            once_only = TEST(VM_ONCE_ONLY, fut_area->vm_flags);
+            /* now done w/ fut_area */
+            read_unlock(&futureexec_areas->lock);
+            fut_area = NULL;
+            if (is_on_stack(dcontext, addr, NULL)) {
+                /* normally futureexec regions are persistent, to allow app to
+                 * repeatedly write and then execute (yes this happens a lot).
+                 * we don't want to do that for the stack, b/c it amounts to
+                 * permanently allowing a certain piece of stack to be executed!
+                 * besides, we don't see the write-exec iter scheme for the stack.
+                 */
+                STATS_INC(num_exec_future_stack);
+                LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+                    "future exec "PFX"-"PFX" is on stack, removing from future list\n",
+                    *base, *base+*size);
+                ok = remove_futureexec_vm_area(*base, *base+*size);
+                ASSERT(ok);
+            } else {
+                STATS_INC(num_exec_future_heap);
+                if (!DYNAMO_OPTION(selfmod_futureexec)) {
+                    /* if on all-selfmod pages, then we shouldn't need to keep it on
+                     * the futureexec list
+                     */
+                    if (is_executable_area_on_all_selfmod_pages(*base, *base+*size))
+                        once_only = true;
+                }
+                if (once_only) {
+                    LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+                        "future exec "PFX"-"PFX" is once-only, removing from future list\n",
+                        *base, *base+*size);
+                    ok = remove_futureexec_vm_area(*base, *base+*size);
+                    ASSERT(ok);
+                    STATS_INC(num_exec_future_once);
+                }
+            }
+            *vm_flags |= VM_WAS_FUTURE;
+            return ALLOWING_OK;
+        }
+    }
+
+    if (DYNAMO_OPTION(executable_if_text) ||
+        DYNAMO_OPTION(executable_if_rx_text) ||
+        (DYNAMO_OPTION(exempt_text) ||
+        !IS_STRING_OPTION_EMPTY(exempt_text_list))) {
+        app_pc modbase = get_module_base(addr);
+        if (modbase != NULL) { /* PE, and is readable */
+            /* note that it could still be a PRIVATE mapping */
+            /* don't expand region to match actual text section bounds -- if we split
+             * let's keep this region smaller.
+             */
+            app_pc sec_start = NULL, sec_end = NULL;
+            if (is_in_code_section(modbase, addr, &sec_start, &sec_end)) {
+                bool allow = false;
+                if (DYNAMO_OPTION(executable_if_text)) {
+                    LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+                        "exec region is in code section of module @"PFX" (%s), allowing\n",
+                        modbase, modname == NULL? "<invalid name>" : modname);
+                    STATS_INC(num_text);
+                    mark_module_exempted(addr);
+                    allow = true;
+                } else {
+                    uint prot = 0;
+                    list_default_or_append_t deflist = LIST_NO_MATCH;
+                    /* Xref case 10526, in the common case app_mem_prot_change() adds 
+                     * this region, however it can miss -> rx transitions if they
+                     * overlapped more then one section (fixing it to do so would
+                     * require signifigant restructuring of that routine, see comments
+                     * there) so we also check here. */
+                    if (DYNAMO_OPTION(executable_if_rx_text) &&
+                        get_memory_info(addr, NULL, NULL, &prot) &&
+                        (TEST(MEMPROT_EXEC, prot) && !TEST(MEMPROT_WRITE, prot))) {
+                        /* matches -executable_if_rx_text */
+                        /* case 9799: we don't mark exempted for default-on options */
+                        allow = true;
+                        SYSLOG_INTERNAL_WARNING_ONCE("allowable rx text section not "
+                                                     "found till check_origins");
+                    }
+                    if (!allow && modname != NULL) {
+                        bool onlist;
+                        string_option_read_lock();
+                        LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 3,
+                            "exec region is in code section of module %s, vs list %s\n",
+                            modname, DYNAMO_OPTION(exempt_text_list));
+                        onlist = check_filter(DYNAMO_OPTION(exempt_text_list), modname);
+                        string_option_read_unlock();
+                        if (onlist) {
+                            LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+                                "module %s is on text list, allowing execution\n", modname);
+                            STATS_INC(num_text_list);
+                            SYSLOG_INTERNAL_WARNING_ONCE("code origins: module %s text "
+                                                         "section exempt", modname);
+                            mark_module_exempted(addr);
+                            allow = true;
+                        }
+                    }
+
+                    if (!allow && modname != NULL) {
+                        deflist =
+                            check_list_default_and_append(dynamo_options.
+                                                          exempt_mapped_image_text_default_list,
+                                                          dynamo_options.
+                                                          exempt_mapped_image_text_list,
+                                                          modname);
+                    }
+                    if (deflist != LIST_NO_MATCH) {
+                        bool image_mapping = is_mapped_as_image(modbase);
+                        if (image_mapping) {
+                            LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+                                "module %s is on text list, of a mapped IMAGE"
+                                " allowing execution\n", modname);
+                            STATS_INC(num_image_text_list);
+                            SYSLOG_INTERNAL_WARNING_ONCE("code origins: module %s IMAGE text "
+                                                         "section exempt", modname);
+                            if (deflist == LIST_ON_APPEND) /* case 9799: not default */
+                                mark_module_exempted(addr);
+                            allow = true;
+                        } else {
+                            ASSERT_NOT_TESTED();
+                            SYSLOG_INTERNAL_WARNING_ONCE("code origins: module %s text "
+                                                         "not IMAGE, attack!", modname);
+                        }
+                    }
+                }
+                if (allow) {
+                    /* trim exec area to allowed bounds */
+                    check_origins_trim_region_helper(base, size, sec_start, sec_end);
+                    return ALLOWING_OK;
+                }
+            }
+        }
+    }
+
+    if (DYNAMO_OPTION(executable_if_dot_data) ||
+        DYNAMO_OPTION(executable_if_dot_data_x) ||
+        (DYNAMO_OPTION(exempt_dot_data) &&
+         !IS_STRING_OPTION_EMPTY(exempt_dot_data_list)) ||
+        (DYNAMO_OPTION(exempt_dot_data_x) &&
+         !IS_STRING_OPTION_EMPTY(exempt_dot_data_x_list))) {
+        /* FIXME: get_module_base() is called all over in this function. 
+         *        This function could do with some refactoring. */
+        app_pc modbase = get_module_base(addr);
+        if (modbase != NULL) {
+            /* A loaded module exists for addr; now see if addr is in .data. */
+            app_pc sec_start = NULL, sec_end = NULL;
+            if (is_in_dot_data_section(modbase, addr, &sec_start, &sec_end)) {
+                bool allow = false;
+                bool onlist = false;
+                uint prot = 0;
+                if (!DYNAMO_OPTION(executable_if_dot_data) &&
+                    DYNAMO_OPTION(exempt_dot_data) &&
+                    !IS_STRING_OPTION_EMPTY(exempt_dot_data_list)) {
+                    if (modname != NULL) {
+                        string_option_read_lock();
+                        LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 3,
+                            "exec region is in data of module %s, vs list %s\n",
+                            modname, DYNAMO_OPTION(exempt_dot_data_list));
+                        onlist = check_filter(DYNAMO_OPTION(exempt_dot_data_list), modname);
+                        string_option_read_unlock();
+                        DOSTATS({
+                            if (onlist) 
+                                STATS_INC(num_dot_data_list);
+                        });
+                    }
+                }
+                DOSTATS({
+                    if (DYNAMO_OPTION(executable_if_dot_data)) 
+                        STATS_INC(num_dot_data);
+                });
+                if (onlist || DYNAMO_OPTION(executable_if_dot_data)) {
+                    LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+                        "exec region is in .data section of module %s\n",
+                        modname == NULL? "<invalid name>" : modname);
+                    SYSLOG_INTERNAL_WARNING_ONCE(
+                            "code origins: .data section of module %s exempt",
+                            modname == NULL? "<invalid name>" : modname);
+                    /* case 9799: FIXME: we don't want to mark as exempted for the
+                     * default modules on the list: should split into a separate
+                     * default list so we can tell!  Those modules will have private
+                     * pcaches if in a process w/ ANY exemption options */
+                    mark_module_exempted(addr);
+                    allow = true;;
+                }
+                if (!allow && get_memory_info(addr, NULL, NULL, &prot) &&
+                    TEST(MEMPROT_EXEC, prot)) {
+                    /* check the _x versions */
+                    if (!DYNAMO_OPTION(executable_if_dot_data_x) &&
+                        DYNAMO_OPTION(exempt_dot_data_x) &&
+                        !IS_STRING_OPTION_EMPTY(exempt_dot_data_x_list)) {
+                        if (modname != NULL) {
+                            string_option_read_lock();
+                            LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 3,
+                                "exec region is in x data of module %s, vs list %s\n",
+                                modname, DYNAMO_OPTION(exempt_dot_data_x_list));
+                            onlist = check_filter_with_wildcards(DYNAMO_OPTION(exempt_dot_data_x_list), modname);
+                            string_option_read_unlock();
+                            DOSTATS({
+                                if (onlist) 
+                                    STATS_INC(num_dot_data_x_list);
+                            });
+                        }
+                        DOSTATS({
+                            if (DYNAMO_OPTION(executable_if_dot_data_x)) 
+                                STATS_INC(num_dot_data_x);
+                        });
+                    }
+                    if (DYNAMO_OPTION(executable_if_dot_data_x) || onlist) {
+                        LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+                            "exec region is in x .data section of module %s\n",
+                            modname == NULL? "<invalid name>" : modname);
+                        SYSLOG_INTERNAL_WARNING_ONCE(
+                                "code origins: .data section of module %s exempt",
+                                modname == NULL? "<invalid name>" : modname);
+                        /* case 9799: FIXME: we don't want to mark as exempted for
+                         * the default modules on the list: should split into a
+                         * separate default list so we can tell!  Those modules will
+                         * have private pcaches if in a process w/ ANY exemption
+                         * options */
+                        mark_module_exempted(addr);
+                        allow = true;
+                    }
+                }
+                if (allow) {
+                    /* trim exec area to allowed bounds */
+                    check_origins_trim_region_helper(base, size, sec_start, sec_end);
+                    return ALLOWING_OK;
+                }
+            }
+        }
+    }
+
+    if (DYNAMO_OPTION(executable_if_image) ||
+        (DYNAMO_OPTION(exempt_image) &&
+         !IS_STRING_OPTION_EMPTY(exempt_image_list)) ||
+        !moduledb_exempt_list_empty(MODULEDB_EXEMPT_IMAGE)) {
+        app_pc modbase = get_module_base(addr);
+        if (modbase != NULL) {
+            /* A loaded module exists for addr; we allow the module (xref 10526 we
+             * used to limit to just certain sections).  FIXME - we could use the
+             * relaxed is_in_any_section here, but other relaxations (such as dll2heap)
+             * exclude the entire module so need to match that to prevent there being
+             * non exemptable areas. */                
+            bool onlist = false;
+            bool mark_exempted = true;
+            if (!DYNAMO_OPTION(executable_if_image)) {
+                if (modname != NULL) {
+                    string_option_read_lock();
+                    LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 3,
+                        "exec region is in image of module %s, vs list %s\n",
+                        modname, DYNAMO_OPTION(exempt_image_list));
+                    onlist = check_filter(DYNAMO_OPTION(exempt_image_list),
+                                          modname);
+                    string_option_read_unlock();
+                    DOSTATS({
+                        if (onlist) 
+                            STATS_INC(num_exempt_image_list);
+                    });
+                    if (!onlist &&
+                        !moduledb_exempt_list_empty(MODULEDB_EXEMPT_IMAGE)) {
+                        onlist =
+                            moduledb_check_exempt_list(MODULEDB_EXEMPT_IMAGE,
+                                                       modname);
+                        DOSTATS({
+                            if (onlist) 
+                                STATS_INC(num_moduledb_exempt_image);
+                        });
+                        /* FIXME - could be that a later policy would
+                         * allow this in which case we shouldn't report,
+                         * however from layout this is should be the last
+                         * place that could allow this target. */
+                        if (onlist) {
+                            /* Case 9799: We don't want to set this for
+                             * default-on options like moduledb to avoid
+                             * non-shared pcaches when other exemption options
+                             * are turned on in the process.
+                             */
+                            mark_exempted = false;
+                            moduledb_report_exemption("Moduledb image exemption"
+                                                      " "PFX" to "PFX" from "
+                                                      "module %s", *base,
+                                                      *base + *size, modname);
+                        }
+                    }
+                }
+            } else {
+                STATS_INC(num_exempt_image);
+            }
+            if (onlist || DYNAMO_OPTION(executable_if_image)) {
+                LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+                        "exec region is in the loaded image of module %s\n",
+                    modname == NULL ? "<invalid name>" : modname);
+                SYSLOG_INTERNAL_WARNING_ONCE("code origins: loaded image of module %s"
+                                             "exempt", modname == NULL ?
+                                             "<invalid name>" : modname);
+                if (mark_exempted)
+                    mark_module_exempted(addr);
+                return ALLOWING_OK;
+            }
+        }
+    }
+    
+    if (((DYNAMO_OPTION(exempt_dll2heap) &&
+          !IS_STRING_OPTION_EMPTY(exempt_dll2heap_list)) ||
+         !moduledb_exempt_list_empty(MODULEDB_EXEMPT_DLL2HEAP) ||
+         (DYNAMO_OPTION(exempt_dll2stack) &&
+          !IS_STRING_OPTION_EMPTY(exempt_dll2stack_list)) ||
+         !moduledb_exempt_list_empty(MODULEDB_EXEMPT_DLL2STACK)) &&
+        /* FIXME: any way to find module info for deleted source? */
+        !LINKSTUB_FAKE(dcontext->last_exit)) {
+        /* no cutting corners here -- find exact module that exit cti is from */
+        app_pc modbase;
+        app_pc translated_pc = 
+            recreate_app_pc(dcontext, EXIT_CTI_PC(dcontext->last_fragment,
+                                                  dcontext->last_exit),
+                            dcontext->last_fragment);
+        ASSERT(translated_pc != NULL);
+        modbase = get_module_base(translated_pc);
+        LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 3,
+            "check_origins: dll2heap and dll2stack for "PFX": cache "PFX" => app "PFX" == mod "PFX"\n",
+            addr, EXIT_CTI_PC(dcontext->last_fragment, dcontext->last_exit),
+            translated_pc, modbase);
+        if (modbase != NULL) { /* PE, and is readable */
+            if (modname != NULL) {
+                bool onheaplist = false, onstacklist = false;
+                bool on_moddb_heaplist = false, on_moddb_stacklist = false;
+                string_option_read_lock();
+                LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 3,
+                    "source region is in module %s\n", modname);
+                if (DYNAMO_OPTION(exempt_dll2heap)) {
+                    onheaplist = check_filter(DYNAMO_OPTION(exempt_dll2heap_list),
+                                              modname);
+                    LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 3, "exempt heap list: %s\n",
+                        DYNAMO_OPTION(exempt_dll2heap_list));
+                }
+                if (DYNAMO_OPTION(exempt_dll2stack)) {
+                    onstacklist = check_filter(DYNAMO_OPTION(exempt_dll2stack_list),
+                                               modname);
+                    LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 3, "exempt stack list: %s\n",
+                        DYNAMO_OPTION(exempt_dll2stack_list));
+                }
+                string_option_read_unlock();
+                if (!onheaplist) {
+                    on_moddb_heaplist =
+                        moduledb_check_exempt_list(MODULEDB_EXEMPT_DLL2HEAP, modname);
+                }
+                if (!onstacklist) {
+                    on_moddb_stacklist =
+                        moduledb_check_exempt_list(MODULEDB_EXEMPT_DLL2STACK, modname);
+                }
+
+                /* make sure targeting non-stack, non-module memory */
+                if ((onheaplist || on_moddb_heaplist) &&
+                    !is_on_stack(dcontext, addr, NULL) &&
+                    get_module_base(addr) == NULL) {
+                    LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+                        "source module %s is on exempt list, target is heap => allowing "
+                        "execution\n", modname);
+                    if (on_moddb_heaplist) {
+                        STATS_INC(num_moduledb_exempt_dll2heap);
+                        moduledb_report_exemption("Moduledb dll2heap exemption "PFX" to"
+                                                  " "PFX" from module %s",
+                                                  translated_pc, addr, modname);
+                    } else {
+                        STATS_INC(num_exempt_dll2heap);
+                        SYSLOG_INTERNAL_WARNING_ONCE("code origins: dll2heap from %s "
+                                                     "exempt", modname);
+                    }
+                    return ALLOWING_OK;
+                }
+                if ((onstacklist || on_moddb_stacklist) &&
+                    is_on_stack(dcontext, addr, NULL)) {
+                    LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+                        "source module %s is on exempt list, target is stack => allowing"
+                        "execution\n", modname);
+                    if (on_moddb_stacklist) {
+                        STATS_INC(num_moduledb_exempt_dll2stack);
+                        moduledb_report_exemption("Moduledb dll2stack exemption "PFX" "
+                                                  "to "PFX" from module %s",
+                                                  translated_pc, addr, modname);
+                    } else {
+                        SYSLOG_INTERNAL_WARNING_ONCE("code origins: dll2stack from %s is"
+                                                     " exempt", modname);
+                        STATS_INC(num_exempt_dll2stack);
+                    }
+                    return ALLOWING_OK;
+                }
+            }
+        }
+    }
+
+    if (dynamo_options.executable_if_trampoline) {
+        /* check for specific bb patterns we allow */
+        if (check_origins_bb_pattern(dcontext, addr, base, size, vm_flags, frag_flags)
+            == ALLOWING_OK) {
+            DOSTATS({
+                if (is_on_stack(dcontext, addr, NULL)) {
+                    STATS_INC(num_trampolines_stack);
+                } else {
+                    STATS_INC(num_trampolines_heap);
+                }
+            });
+            return ALLOWING_OK;
+        }
+    }
+
+    if (DYNAMO_OPTION(executable_if_driver)) {
+        if (TEST(VM_DRIVER_ADDRESS, *vm_flags)) {
+            ASSERT(*size == PAGE_SIZE);
+            LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+                "check origins: pc = "PFX" is in a new driver area\n", addr);
+            STATS_INC(num_driver_areas);
+            return ALLOWING_OK;
+        }
+    }
+
+    if (is_on_stack(dcontext, addr, NULL)) {
+        /* WARNING: stack check not bulletproof since attackers control esp */
+        LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+            "check origins: pc = "PFX" is on the stack\n", addr);
+        STATS_INC(num_stack_violations);
+        if (!dynamo_options.executable_stack) {
+            LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 1,
+                "ERROR: Address "PFX" on the stack is not executable!\n",
+                addr);
+            return STACK_EXECUTION_VIOLATION;
+        } else {
+            LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 1,
+                "WARNING: Execution violation @ stack address "PFX" detected. "
+                "Continuing...\n", addr);
+            return ALLOWING_BAD;
+        }
+    } else {
+        STATS_INC(num_heap_violations);
+        if (!dynamo_options.executable_heap) {
+            LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 1,
+                "ERROR: Address "PFX" on the heap is not executable!\n", addr);
+            SYSLOG_INTERNAL_WARNING_ONCE("Address "PFX" on the heap is not executable",
+                                         addr);
+            return HEAP_EXECUTION_VIOLATION;
+        } else {
+            LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 1,
+                "WARNING: Execution violation @ heap address "PFX" detected. "
+                "Continuing...\n", addr);
+            return ALLOWING_BAD;
+        }
+    }
+
+    /* CHECK: why did we get here? */
+    ASSERT_NOT_REACHED();
+}
+
+/* It is up to the caller to raise a violation if return value is < 0 */
+static INLINE_ONCE int
+check_origins(dcontext_t *dcontext, app_pc addr, app_pc *base, size_t *size,
+              uint prot, uint *vm_flags, uint *frag_flags, bool xfer)
+{
+    security_violation_t res;
+    /* Many exemptions need to know the module name, so we obtain here */
+    char modname_buf[MAX_MODNAME_INTERNAL];
+    const char *modname =
+        os_get_module_name_buf_strdup(addr, modname_buf,
+                                      BUFFER_SIZE_ELEMENTS(modname_buf)
+                                      HEAPACCT(ACCT_VMAREAS));
+
+    ASSERT(DYNAMO_OPTION(code_origins));
+    LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 3,
+        "check origins: pc = "PFX"\n", addr);
+    res = check_origins_helper(dcontext, addr, base, size, prot, vm_flags, frag_flags,
+                               modname);
+# ifdef DGC_DIAGNOSTICS
+    if (res != ALLOWING_OK) {
+        /* set flag so we can call this area BAD in the future */
+        *frag_flags |= FRAG_DYNGEN_RESTRICTED;
+    }
+# endif
+    if (res < 0) {
+        /* if_x shouldn't have to check here, should catch all regions marked x
+         * at DR init time or app allocation time
+         */
+        /* FIXME: turn these into a SYSLOG_INTERNAL_WARNING_ONCE(in case an
+         * external agent has added that code) 
+         * and then we'd need to add them now.
+         * FIXME: xref case 3742
+         */
+        ASSERT_BUG_NUM(3742, !DYNAMO_OPTION(executable_if_x) || !TEST(MEMPROT_EXEC, prot));
+        ASSERT(!DYNAMO_OPTION(executable_if_rx) || !TEST(MEMPROT_EXEC, prot) || 
+                              TEST(MEMPROT_WRITE, prot));
+    }
+    if (modname != NULL && modname != modname_buf)
+        dr_strfree(modname HEAPACCT(ACCT_VMAREAS));
+    return res;
+}
+
+/* returns whether it ended up deleting the self-writing fragment
+ * by flushing the region
+ */
+bool
+vm_area_fragment_self_write(dcontext_t *dcontext, app_pc tag)
+{
+    if (!dynamo_options.executable_stack && is_on_stack(dcontext, tag, NULL)) {
+        /* stack code is NOT persistently executable, nor is it allowed to be
+         * written, period!  however, in keeping with our philosophy of only
+         * interfering with the program when it executes, we don't stop it
+         * at the write here, we simply remove the code from the executable
+         * list and remove its sandboxing.  after all, the code on the stack
+         * may be finished with, and now the stack is just being used as data!
+         *
+         * FIXME: there is a hole here due to selfmod fragments
+         * being private: a second thread can write to a stack region and then
+         * execute from the changed region w/o kicking it off the executable
+         * list.  case 4020 fixed this for pattern-matched regions.
+         */
+        bool ok;
+        vm_area_t *area = NULL;
+        app_pc start, end;
+        read_lock(&executable_areas->lock);
+        ok = lookup_addr(executable_areas, tag, &area);
+        ASSERT(ok);
+        /* grab fields since can't hold lock entire time */
+        start = area->start;
+        end = area->end;
+        read_unlock(&executable_areas->lock);
+        LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 1,
+            "WARNING: code on stack "PFX"-"PFX" @tag "PFX" written to\n",
+            start, end, tag);
+        SYSLOG_INTERNAL_WARNING_ONCE("executable code on stack written to.");
+        /* FIXME: fragment could extend into multiple areas, we should flush
+         * them all to cover the written-to region (which we don't know)
+         */
+        flush_fragments_and_remove_region(dcontext, start, end - start,
+                                          false /* don't own initexit_lock */,
+                                          false /* keep futures */);
+        return true;
+    }
+    return false;
+}
+#endif /* PROGRAM_SHEPHERDING ******************************************/
+
+#ifdef SIMULATE_ATTACK
+
+enum {
+    SIMULATE_INIT       = 0,
+    SIMULATE_GENERIC    = 1,
+    SIMULATE_AT_ADDR    = 2,
+    SIMULATE_AT_FRAGNUM = 4,
+    SIMULATE_WIPE_STACK = 8,
+    SIMULATE_OVER       = 0x1000,
+};
+
+/* attack simulation list */
+/* comma separated list of simulate points.
+     @fragnum
+        fragment number available only in DEBUG builds
+     0xfragpc 
+        will test addr only whenever check_thread_vm_area is called 
+        start of bb, pc at end of direct cti instr, target of direct cti, 
+        pc at end of final instr in bb
+     s: prefix wipes the stack 
+   Ex: -simulate_at @100,s:@150,0x77e9e8d6,s:0x77e9e8f0,@777,@2000,s:@19999,@29999
+*/
+/* simulate_at is modified in place, hence caller needs to synchronize
+   and should be 0 after the first call just like strtok */
+int
+next_simulate_at_fragment(char **tokpos /* OUT */, int *action /* OUT */)
+{
+    char *fragnum;
+
+    // assumes sscanf won't get confused with the ,s
+    for(fragnum = *tokpos; fragnum; fragnum = *tokpos) {
+        int num;
+        *tokpos = strchr(fragnum, ','); /* next ptr */
+        if (*tokpos) 
+            (*tokpos)++;
+
+        if (sscanf(fragnum, PIFX, &num) == 1) {
+            LOG(GLOBAL, LOG_VMAREAS, 1, 
+                "next_simulate_at_fragment: %s="PIFX" addr\n", fragnum, num);
+            *action = SIMULATE_AT_ADDR;
+            return num;
+        } else if (sscanf(fragnum, "s:"PIFX, &num) == 1) {
+            LOG(GLOBAL, LOG_VMAREAS, 1, 
+                "next_simulate_at_fragment: wipe stack %s="PIFX"\n", fragnum, num);
+            *action = SIMULATE_WIPE_STACK | SIMULATE_AT_ADDR;
+            return num;
+        } 
+#ifdef DEBUG                    /* for fragment count */
+        else if (sscanf(fragnum, "s:@%d", &num) == 1) {
+            LOG(GLOBAL, LOG_VMAREAS, 1, 
+                "next_simulate_at_fragment: wipe stack %s=%d\n", fragnum, num);
+            *action = SIMULATE_WIPE_STACK | SIMULATE_AT_FRAGNUM;
+            return num;
+        } else if (sscanf(fragnum, "@%d", &num) == 1) {
+            LOG(GLOBAL, LOG_VMAREAS, 1, 
+                "next_simulate_at_fragment: %s=%d num\n", fragnum, num);
+            *action = SIMULATE_AT_FRAGNUM;
+            return num;
+        }
+#endif
+        else {
+            LOG(GLOBAL, LOG_VMAREAS, 1, 
+                "next_simulate_at_fragment: frg=%s ignored\n", fragnum);
+        }
+    }
+
+    *action = SIMULATE_OVER;
+    LOG(GLOBAL, LOG_VMAREAS, 1, 
+        "next_simulate_at_fragment: simulate attack over\n");
+
+    return 0;
+}
+
+void
+simulate_attack(dcontext_t *dcontext, app_pc pc)
+{
+    static char *tokpos;
+    static int next_frag = 0;   /* number or address */
+    static int action = SIMULATE_INIT;
+
+    bool attack = false;
+
+    if (TEST(SIMULATE_AT_FRAGNUM, action)) {
+        attack = GLOBAL_STAT(num_fragments) > next_frag;
+    }
+    if (TEST(SIMULATE_AT_ADDR, action)) {
+        if (pc == (app_pc)next_frag)
+            attack = true;
+    }
+
+    if (attack) {
+        LOG(GLOBAL, LOG_VMAREAS, 1,
+            "SIMULATE ATTACK for "PFX" @%d frags\n", pc, GLOBAL_STAT(num_fragments));
+        
+        if (TEST(SIMULATE_WIPE_STACK, action)) {
+            reg_t esp = get_mcontext(dcontext)->xsp;
+            uint overflow_size = 1024;
+            LOG(THREAD_GET, LOG_VMAREAS, 1, "simulate_attack: wipe stack "PFX"-"PFX"\n", 
+                esp, esp + overflow_size-1);
+
+            /* wipe out a good portion of the app stack */
+            memset((void*)esp, 0xbf, overflow_size); /* LOOK for 0xbf in the log */
+            LOG(THREAD_GET, LOG_VMAREAS, 1, "simulate_attack: wiped stack "PFX"-"PFX"\n", 
+                esp, esp + overflow_size-1);
+
+            /* FIXME: we may want to just wipe the stack and return to app */
+        }
+    }
+
+    /* prepare for what to do next */
+    if (attack || action == SIMULATE_INIT) {
+        mutex_lock(&simulate_lock);
+        string_option_read_lock();
+        tokpos = dynamo_options.simulate_at;
+        if (action == SIMULATE_INIT) {
+            if ('\0' == *tokpos)
+                tokpos = NULL;
+        }
+        next_frag = next_simulate_at_fragment(&tokpos, &action);
+        /* dynamic changes to the string may have truncated it in front of original */
+        ASSERT(tokpos < strchr(dynamo_options.simulate_at, '\0'));
+        string_option_read_unlock();
+        /* FIXME: tokpos ptr is kept beyond release of lock! */
+        mutex_unlock(&simulate_lock);
+    }
+
+    if (attack) {
+        security_violation(dcontext, pc, ATTACK_SIMULATION_VIOLATION,
+                           OPTION_BLOCK|OPTION_REPORT);
+    }
+}
+#endif /* SIMULATE_ATTACK */
+
+#if defined(DEBUG) && defined(INTERNAL)
+static void
+print_entry(dcontext_t *dcontext, fragment_t *entry, char *prefix)
+{
+    if (entry == NULL)
+        LOG(THREAD, LOG_VMAREAS, 1, "%s<NULL>\n", prefix);
+    else if (FRAG_MULTI(entry)) {
+        if (FRAG_MULTI_INIT(entry)) {
+            LOG(THREAD, LOG_VMAREAS, 1, "%s"PFX" <init: tag="PFX"> pc="PFX"\n",
+                prefix, entry, FRAG_FRAG(entry), FRAG_PC(entry));
+        } else {
+            LOG(THREAD, LOG_VMAREAS, 1, "%s"PFX" F="PFX" pc="PFX"\n",
+                prefix, entry, FRAG_FRAG(entry), FRAG_PC(entry));
+        }
+    } else {
+        fragment_t *f = (fragment_t *) entry;
+        LOG(THREAD, LOG_VMAREAS, 1, "%s"PFX" F%d tag="PFX"\n",
+            prefix, f, f->id, f->tag);
+    }
+}
+
+static void
+print_fraglist(dcontext_t *dcontext, vm_area_t *area, char *prefix)
+{
+    fragment_t *entry, *last;
+    LOG(THREAD, LOG_VMAREAS, 1, "%sFragments for area ("PFX") "PFX".."PFX"\n",
+        prefix, area, area->start, area->end);
+    for (entry = area->custom.frags, last = NULL; entry != NULL;
+         last = entry, entry = FRAG_NEXT(entry)) {
+        print_entry(dcontext, entry, "\t");
+        DOLOG(7, LOG_VMAREAS, {
+            print_entry(dcontext, FRAG_PREV(entry), "\t    <=");
+            print_entry(dcontext, FRAG_NEXT(entry), "\t    =>");
+        });
+        if (FRAG_ALSO(entry) != NULL) {
+            fragment_t *also = FRAG_ALSO(entry);
+            print_entry(dcontext, FRAG_ALSO(entry), "\t    also =>");
+
+            /* check for also in same area == inconsistency in data structs */
+            if (FRAG_PC(also) >= area->start && FRAG_PC(also) < area->end) {
+                if (FRAG_MULTI_INIT(also)) {
+                    LOG(THREAD, LOG_VMAREAS, 1, "WARNING: self-also frag tag "PFX"\n",
+                        FRAG_FRAG(also));
+                } else {
+                    fragment_t *f = FRAG_FRAG(also);
+                    LOG(THREAD, LOG_VMAREAS, 1, "WARNING: self-also frag F%d("PFX")%s\n",
+                        f->id, f->tag, TEST(FRAG_IS_TRACE, f->flags) ? " trace" : "");
+                }
+                /* not an assertion b/c we sometimes print prior to cleaning */
+            }
+        }
+
+        ASSERT(last == NULL || last == FRAG_PREV(entry));
+    }
+    ASSERT(area->custom.frags == NULL || FRAG_PREV(area->custom.frags) == last);
+}
+
+static void
+print_fraglists(dcontext_t *dcontext)
+{
+    thread_data_t *data = GET_DATA(dcontext, 0);
+    int i;
+    ASSERT_VMAREA_DATA_PROTECTED(data, READWRITE);
+    LOG(THREAD, LOG_VMAREAS, 1, "\nFragment lists for ALL AREAS:\n");
+    for (i = 0; i < data->areas.length; i++) {
+        print_fraglist(dcontext, &(data->areas.buf[i]), "");
+    }
+    LOG(THREAD, LOG_VMAREAS, 1, "\n");
+}
+
+static void
+print_frag_arealist(dcontext_t *dcontext, fragment_t *f)
+{
+    fragment_t *entry;
+    if (FRAG_MULTI(f)) {
+        LOG(THREAD, LOG_VMAREAS, 1, "Areas for F="PFX" ("PFX")\n",
+            FRAG_FRAG(f), FRAG_PC(f));
+    } else
+        LOG(THREAD, LOG_VMAREAS, 1, "Areas for F%d ("PFX")\n", f->id, f->tag);
+    for (entry = f; entry != NULL; entry = FRAG_ALSO(entry)) {
+        print_entry(dcontext, entry, "\t");
+    }
+}
+#endif /* DEBUG && INTERNAL */
+
+
+/* adds entry to front of area's frags list 
+ * caller must synchronize modification of area
+ * FIXME: how assert that caller has done that w/o asking for whole vector
+ * to be passed in, or having backpointer from area?
+ * See general FIXME of same flavor at top of file.
+ */
+static void
+prepend_entry_to_fraglist(vm_area_t *area, fragment_t *entry)
+{
+    FRAG_NEXT_ASSIGN(entry, area->custom.frags);
+    /* prev wraps around, but not next */
+    if (area->custom.frags != NULL) {
+        FRAG_PREV_ASSIGN(entry, FRAG_PREV(area->custom.frags));
+        FRAG_PREV_ASSIGN(area->custom.frags, entry);
+    } else
+        FRAG_PREV_ASSIGN(entry, entry);
+    area->custom.frags = entry;
+}
+
+/* adds a multi_entry_t to the list of fragments for area.
+ * cross-links with prev if prev != NULL.
+ * sticks tag in for f (will be fixed in vm_area_add_fragment, once f is created)
+ */
+static fragment_t *
+prepend_fraglist(dcontext_t *dcontext, vm_area_t *area, app_pc entry_pc,
+                 app_pc tag, fragment_t *prev)
+{
+    multi_entry_t *e = (multi_entry_t *)
+        nonpersistent_heap_alloc(dcontext, sizeof(multi_entry_t) HEAPACCT(ACCT_VMAREA_MULTI));
+    fragment_t * entry = (fragment_t *) e;
+    e->flags = FRAG_FAKE | FRAG_IS_EXTRA_VMAREA | /* distinguish from fragment_t */
+        FRAG_IS_EXTRA_VMAREA_INIT; /* indicate f field is a tag, not a fragment_t yet */
+    if (dcontext == GLOBAL_DCONTEXT) /* shared */
+        e->flags |= FRAG_SHARED;
+    e->f = (fragment_t *) tag; /* placeholder */
+    e->pc = entry_pc;
+    if (prev != NULL)
+        FRAG_ALSO_ASSIGN(prev, entry);
+    FRAG_ALSO_ASSIGN(entry, NULL);
+    prepend_entry_to_fraglist(area, entry);
+    DOLOG(7, LOG_VMAREAS, {
+        print_fraglist(dcontext, area, "after prepend_fraglist, ");
+    });
+    return entry;
+}
+
+#ifdef DGC_DIAGNOSTICS
+void
+dyngen_diagnostics(dcontext_t *dcontext, app_pc pc, app_pc base_pc, size_t size,
+                   uint prot)
+{
+    bool future, stack;
+    char buf[MAXIMUM_SYMBOL_LENGTH];
+    app_pc translated_pc;
+
+    read_lock(&futureexec_areas->lock);
+    future = lookup_addr(futureexec_areas, pc, NULL);
+    read_unlock(&futureexec_areas->lock);
+    stack = is_on_stack(dcontext, pc, NULL);
+
+    if (!future)
+        future = is_dyngen_vsyscall(pc);
+
+    print_symbolic_address(pc, buf, sizeof(buf), false);
+    LOG(GLOBAL, LOG_VMAREAS, 1,
+        "DYNGEN in %d: target="PFX" => "PFX"-"PFX" %s%s%s%s%s %s\n",
+        dcontext->owning_thread, pc, base_pc, base_pc+size,
+        ((prot & MEMPROT_READ) != 0) ? "R":"",
+        ((prot & MEMPROT_WRITE) != 0)? "W":"",
+        ((prot & MEMPROT_EXEC) != 0) ? "E":"",
+        future ? " future":" BAD", stack ? " stack":"", buf);
+
+    if (LINKSTUB_FAKE(dcontext->last_exit)) {
+        LOG(GLOBAL, LOG_VMAREAS, 1,
+            "source=!!! fake last_exit, must have been flushed?\n");
+        return;
+    }
+
+    /* FIXME: risky if last fragment is deleted -- should check for that
+     * here and instead just print type from last_exit, since recreate
+     * may fail
+     */
+    translated_pc = 
+        recreate_app_pc(dcontext, EXIT_CTI_PC(dcontext->last_fragment,
+                                              dcontext->last_exit),
+                        dcontext->last_fragment);
+    if (translated_pc != NULL) {
+        print_symbolic_address(translated_pc, buf, sizeof(buf), false);
+        LOG(GLOBAL, LOG_VMAREAS, 1,
+            "source=F%d("PFX") @"PFX" \"%s\"\n",
+            dcontext->last_fragment->id,
+            dcontext->last_fragment->tag,
+            EXIT_CTI_PC(dcontext->last_fragment, dcontext->last_exit), buf);
+        disassemble_with_bytes(dcontext, translated_pc, main_logfile);
+    }
+    DOLOG(4, LOG_VMAREAS, {
+        disassemble_fragment(dcontext, dcontext->last_fragment, false);
+    });
+}
+#endif
+
+/***************************************************************************
+ * APPLICATION MEMORY STATE TRACKING
+ */
+
+/* newly allocated or mapped in memory region, returns true if added to exec list 
+ * ok to pass in NULL for dcontext -- in fact, assumes dcontext is NULL at initialization
+ */
+bool
+app_memory_allocation(dcontext_t *dcontext, app_pc base, size_t size, uint prot,
+                      bool image _IF_DEBUG(char *comment))
+{
+    /* FIXME (case 68): to guard against external agents freeing memory, we
+     * could remove this region from the executable list here -- is it worth the
+     * performance hit?  DR itself could allocate memory that was freed
+     * externally -- but our DR overlap checks would catch that.
+     */
+    ASSERT_CURIOSITY(!executable_vm_area_overlap(base, base + size, false/*have no lock*/));
+#ifdef PROGRAM_SHEPHERDING
+    DODEBUG({
+        /* case 4175 - reallocations will overlap with no easy way to
+         * enforce this 
+         */
+        if (futureexec_vm_area_overlap(base, base + size)) {
+            SYSLOG_INTERNAL_WARNING_ONCE("existing future area overlapping ["PFX", "
+                                         PFX")", base, base + size);
+        }
+    });
+#endif
+
+    /* no current policies allow non-x code at allocation time onto exec list */
+    if (!TEST(MEMPROT_EXEC, prot))
+        return false;
+
+    LOG(GLOBAL, LOG_VMAREAS, 1, "New +x app memory region: "PFX"-"PFX" %s\n",
+        base, base+size, memprot_string(prot));
+
+    /* Do not add our own code cache and other data structures
+     * to executable list -- but do add our code segment
+     * FIXME: checking base only is good enough?
+     */
+    if (dynamo_vm_area_overlap(base, base + size)) {
+        LOG(GLOBAL, LOG_VMAREAS, 2, "\t<dynamorio region>\n");
+        /* assumption: preload/preinject library is not on DR area list since unloaded */
+        if (!is_in_dynamo_dll(base)) /* our own text section is ok */
+            return false;
+    }
+
+    if (!TEST(MEMPROT_WRITE, prot)) {
+        uint frag_flags = 0;
+        if (DYNAMO_OPTION(coarse_units) && image && !RUNNING_WITHOUT_CODE_CACHE()) {
+            /* all images start out with coarse-grain management */
+            frag_flags |= FRAG_COARSE_GRAIN;
+        }
+        add_executable_vm_area(base, base + size, image ? VM_UNMOD_IMAGE : 0, frag_flags,
+                               false/*no lock*/ _IF_DEBUG(comment));
+        return true;
+    } else if (dcontext==NULL || !is_on_stack(dcontext, base, NULL)) {
+        LOG(GLOBAL, LOG_VMAREAS, 1,
+            "WARNING: "PFX"-"PFX" is writable, NOT adding to executable list\n",
+            base, base+size);
+
+#ifdef PROGRAM_SHEPHERDING
+        if (DYNAMO_OPTION(executable_if_x)) {
+            LOG(GLOBAL, LOG_VMAREAS, 1,
+                "app_memory_allocation: New future exec region b/c x: "PFX"-"PFX" %s\n",
+                base, base+size, memprot_string(prot));
+            STATS_INC(num_mark_if_x);
+            add_futureexec_vm_area(base, base+size, false/*permanent*/
+                                   _IF_DEBUG("alloc executable_if_x"));
+            mark_module_exempted(base);
+        } else if (DYNAMO_OPTION(executable_if_alloc)) {
+            bool future = false;
+            /* rwx regions are not added at init time unless in images */
+# ifdef WINDOWS
+            if (image) {
+                /* anything marked rwx in an image is added to future list
+                 * otherwise it is not added -- must be separately allocated,
+                 * not just be present at init or in a mapped non-image file
+                 */
+                future = true;
+                LOG(GLOBAL, LOG_VMAREAS, 1,
+                    "New future exec region b/c x from image: "PFX"-"PFX" %s\n",
+                    base, base+size, memprot_string(prot));
+            } else if (dcontext != NULL && dcontext->alloc_no_reserve) {
+                /* we only add a region marked rwx at allocation time to the
+                 * future list if it is allocated and reserved at the same time
+                 * (to distinguish from the rwx heap on 2003)
+                 */
+                future = true;
+                LOG(GLOBAL, LOG_VMAREAS, 1,
+                    "New future exec region b/c x @alloc & no reserve: "PFX"-"PFX" %s\n",
+                    base, base+size, memprot_string(prot));
+            }
+# else
+            if (dcontext != NULL || image) {
+                /* can't distinguish stack -- saved at init time since we don't add rwx then,
+                 * but what about stacks whose creation we see?  FIXME
+                 */
+                future = true;
+                LOG(GLOBAL, LOG_VMAREAS, 1,
+                    "New future exec region b/c x @alloc: "PFX"-"PFX" %s\n",
+                    base, base+size, memprot_string(prot));
+            }
+# endif
+            if (future) {
+                STATS_INC(num_alloc_exec);
+                add_futureexec_vm_area(base, base+size, false/*permanent*/
+                                       _IF_DEBUG("alloc x"));
+            }
+        }
+#endif /* PROGRAM_SHEPHERDING */
+    }
+    return false;
+}
+
+/* de-allocated or un-mapped memory region */
+void
+app_memory_deallocation(dcontext_t *dcontext, app_pc base, size_t size,
+                        bool own_initexit_lock, bool image)
+{
+    ASSERT(dcontext != NULL);
+    ASSERT(!dynamo_vm_area_overlap(base, base + size));
+    /* we check for overlap regardless of memory protections, to allow flexible
+     * policies that are independent of rwx bits -- if any overlap we remove,
+     * no shortcuts
+     */
+    if (executable_vm_area_overlap(base, base + size, false/*have no lock*/)) {
+        /* ok for overlap to have changed in between, flush checks again */
+        flush_fragments_and_remove_region(dcontext, base, size, own_initexit_lock,
+                                          true/*free futures*/);
+
+#ifdef RETURN_AFTER_CALL
+        if (DYNAMO_OPTION(ret_after_call) && !image 
+            && !DYNAMO_OPTION(rac_dgc_sticky)) {
+            /* we can have after call targets in DGC in addition to DLLs */
+            /* Note IMAGE mappings are handled in process_image() on
+             * Windows, so that they can be handled more efficiently
+             * as a single region. FIXME: case 4983 on Linux
+             */
+            /* only freeing if we have ever interp/executed from this area */
+
+            /* FIXME: note that on app_memory_protection_change() we
+             * do NOT want to free these entries, therefore we'd have
+             * a leak if a portion gets marked writable and is thus no
+             * longer on our list.  Note we can't flush the areas on
+             * memory protection because the likelihood of introducing
+             * a false positives in doing so is vastly greater than
+             * the security risk of not flushing.  (Many valid after
+             * call locations may still be active, and our vmarea
+             * boundaries can not precisely capture the application
+             * intent.)  Note that we would not leak on DLLs even if
+             * they are made writable, because we treat separately.
+             */
+            /* FIXME: see proposal in case 2236 about using a
+             * heuristic that removes only when too numerous, if that
+             * works well as a heuristic that DGC is being reused, and
+             * unlikely that it will be so densely filled.
+             */
+
+            /* FIXME: [perf] case 9331 this is not so good on all
+             * deallocations, if we can't tell whether we have
+             * executed from it.  On every module LOAD, before mapping
+             * it as MEM_IMAGE the loader first maps a DLL as
+             * MEM_MAPPED, and on each of the corresponding unmaps
+             * during LoadLibrary(), we'd be walking the cumulative
+             * hashtable.  Although there shouldn't be that many valid
+             * AC entries at process startup, maybe best to leave the
+             * DGC leak for now if this will potentially hurt startup
+             * time in say svchost.exe.  Currently rac_dgc_sticky is
+             * on by default so we don't reach this code.
+             */
+            /* case 9331: should find out if there was any true
+             * execution in any thread here before we go through a
+             * linear walk of the hashtable.  More directly we need a
+             * vmvector matching all vmareas that had a .C added for
+             * them, considering the common case should be that this
+             * is an app memory deallocation that has nothing to do
+             * with us.
+             * 
+             * FIXME: for now just checking if base is declared DGC,
+             * and ignoring any others possible vm_areas for the same
+             * OS region, so we may still have a leak.
+             */
+            if (is_dyngen_code(base)) {
+                ASSERT_NOT_TESTED();
+                invalidate_after_call_target_range(dcontext, base, base+size);
+            }
+        }
+#endif /* RETURN_AFTER_CALL */
+    }
+
+#ifdef PROGRAM_SHEPHERDING
+    if (USING_FUTURE_EXEC_LIST && futureexec_vm_area_overlap(base, base + size)) {
+        remove_futureexec_vm_area(base, base + size);
+        LOG(GLOBAL, LOG_VMAREAS, 2,
+            "removing future exec "PFX"-"PFX" since now freed\n", base, base+size);
+    }
+#endif
+}
+
+/* A convenience routine that starts the two-phase flushing protocol */
+/* Note this is not flush_fragments_and_remove_region */
+static bool
+flush_and_remove_executable_vm_area(dcontext_t *dcontext, 
+                                    app_pc base, size_t size)
+{
+    DEBUG_DECLARE(bool res;)
+    flush_fragments_in_region_start(dcontext, base, size, 
+                                    false /* don't own initexit_lock */,
+                                    false /* case 2236: keep futures */,
+                                    true /* exec invalid */,
+                                    false /* don't force synchall */
+                                    _IF_DGCDIAG(NULL));
+    DEBUG_DECLARE(res = )
+        remove_executable_vm_area(base, base + size, true/*have lock*/);
+    DODEBUG(if (!res) {
+        /* area doesn't have to be executable in fact when called 
+         * on executable_if_hook path
+         */
+        LOG(THREAD, LOG_VMAREAS, 2,
+            "\tregion was in fact not on executable_areas, so nothing to remove\n");
+    });
+    /* making sure there is no overlap now */
+    ASSERT(!executable_vm_area_overlap(base, base+size, true /* holding lock */));
+            
+    return true;
+}
+
+void
+tamper_resistant_region_add(app_pc start, app_pc end)
+{
+    /* For now assuming a single area for specially protected areas
+     * that is looked up in addition to dynamo_vm_areas.  Assuming
+     * modifications to any location is ntdll.dll is always
+     * interesting to us, instead of only those pieces we trampoline
+     * this should be sufficient.
+     *
+     * FIXME: we could add a new vm_area_vector_t for protected possibly
+     * subpage regions that we later turn into pretend_writable_areas
+     *
+     * Note that ntdll doesn't have an IAT section so we only worry
+     * about function patching
+     */
+    ASSERT(tamper_resistant_region_start == NULL);
+    tamper_resistant_region_start = start;
+    tamper_resistant_region_end = end;
+}
+
+/* returns true if [start, end) overlaps with a tamper_resistant region 
+ * as needed for DYNAMO_OPTION(handle_ntdll_modify)
+ */
+bool
+tamper_resistant_region_overlap(app_pc start, app_pc end)
+{
+    return (end > tamper_resistant_region_start && 
+            start < tamper_resistant_region_end);
+}
+
+/* memory region base:base+size now has privileges prot 
+ * returns a value from the enum in vmareas->h about whether to perform the
+ * system call or not and if not what the return code to the app should be
+ */
+/* FIXME : This is called before the system call that will change 
+ * the memory permission which could be race condition prone! If another
+ * thread executes from a region added by this function before the system call 
+ * goes through we could get a disconnect on what the memory premissions of the
+ * region really are vs what vmareas expects for consistency, see bug 2833 
+*/
+/* N.B.: be careful about leaving code read-only and returning
+ * PRETEND_APP_MEM_PROT_CHANGE or SUBSET_APP_MEM_PROT_CHANGE, or other
+ * cases where mixed with native execution we may have incorrect page settings -
+ * e.g. make sure all pages that need to be executable are executable!
+ *
+ * Note new_memprot is set only for SUBSET_APP_MEM_PROT_CHANGE,
+ * and old_memprot is set for PRETEND_APP_MEM_PROT_CHANGE or SUBSET_APP_MEM_PROT_CHANGE.
+ */
+/* Note: hotp_only_mem_prot_change() relies on executable_areas to find out
+ * previous state, so eliminating it should be carefully; see case 6669.
+ */
+uint
+app_memory_protection_change(dcontext_t *dcontext, app_pc base, size_t size, 
+                             uint prot, /* platform independent MEMPROT_ */
+                             uint *new_memprot, /* OUT */
+                             uint *old_memprot /* OPTIONAL OUT*/)
+{
+    /* FIXME: look up whether image, etc. here?  
+     * but could overlap multiple regions! 
+     */
+    bool is_executable;
+
+    bool should_finish_flushing = false;
+
+    bool dr_overlap = DYNAMO_OPTION(handle_DR_modify) != DR_MODIFY_OFF /* we don't care */
+        && dynamo_vm_area_overlap(base, base + size);
+
+    bool system_overlap = DYNAMO_OPTION(handle_ntdll_modify) != DR_MODIFY_OFF /* we don't care */
+        && tamper_resistant_region_overlap(base, base + size);
+    
+    bool patch_proof_overlap = false;
+#ifdef WINDOWS
+    uint frag_flags;
+#endif
+    ASSERT(new_memprot != NULL);
+    /* old_memprot is optional */
+
+#if defined(PROGRAM_SHEPHERDING) && defined(WINDOWS)
+    patch_proof_overlap = (!IS_STRING_OPTION_EMPTY(patch_proof_default_list) ||
+                           !IS_STRING_OPTION_EMPTY(patch_proof_list)) &&
+        vmvector_overlap(patch_proof_areas, base, base + size);
+    /* FIXME: [minor perf] all the above tests can be combined into a
+     * single vmarea lookup when this feature default on, case 6632 */
+    ASSERT(base != NULL);
+    if (patch_proof_overlap) {
+        app_pc modbase = get_module_base(base);
+        bool loader = is_module_patch_region(dcontext, base, base+size,
+                                             false/*be liberal: don't miss loader*/);
+        bool patching_code = is_range_in_code_section(modbase, base, base+size,
+                                                      NULL, NULL);
+        bool patching_IAT = is_IAT(base, base+size, true/*page-align*/, NULL, NULL);
+        /* FIXME: [perf] could have added CODE sections instead of modules to patch_proof_areas */
+        /* FIXME: [minor perf] is_module_patch_region already collected these */
+        /* FIXME: [minor perf] same check is done later for all IATs for emulate_IAT_writes */
+
+        bool patch_proof_IAT = false; /* NYI - case 6622 */
+        /* FIXME: case 6622 IAT hooker protection for some modules is
+         * expected to conflict with emulate_IAT_writes, need to make
+         * sure emulate_write_areas will not overlap with this
+         */
+        ASSERT_NOT_IMPLEMENTED(!patch_proof_IAT);
+
+        patch_proof_overlap = !loader && patching_code &&
+            /* even if it is not the loader we protect IAT sections only */
+            (!patching_IAT || patch_proof_IAT);
+
+        LOG(THREAD, LOG_VMAREAS, 1, "patch proof module "PFX"-"PFX" modified %s, by %s,%s=>%s\n",
+            base, base+size,
+            patching_code ? "code!" : "data --ok",
+            loader ? "loader --ok" : patching_code ? "hooker!" : "loader or hooker",
+            patching_IAT ? "IAT hooker" : "patching!",
+            patch_proof_overlap ? "SQUASH" : "allow");
+        /* curiosly the loader modifies the .reloc section of Dell\QuickSet\dadkeyb.dll */
+    }
+#endif /* defined(PROGRAM_SHEPHERDING) && defined(WINDOWS) */
+
+    /* FIXME: case 6622 IAT hooking should be controlled separately,
+     * note that when it is not protecting all IAT areas - exemptions
+     * tracked by module name there may have to handle two different
+     * cases.  If making sure a particular DLL is always using the
+     * real exports current implementation above will work.  Yet in
+     * the use case of avoiding a particular IAT hooker replacing
+     * imports from kernel32, _all_ modules will have to be pretend
+     * writable.  xref case 1948 for tracking read/written values
+     */
+
+    if (dr_overlap || system_overlap || patch_proof_overlap) {
+        uint how_handle;
+        const char *target_area_name;
+        /* FIXME: separate this in a function */
+        if (dr_overlap) {
+            how_handle = DYNAMO_OPTION(handle_DR_modify);
+            STATS_INC(app_modify_DR_prot);
+            target_area_name = PRODUCT_NAME;
+        } else if (system_overlap) {
+            ASSERT(system_overlap);
+            how_handle = DYNAMO_OPTION(handle_ntdll_modify);
+            STATS_INC(app_modify_ntdll_prot);
+            target_area_name = "system";
+        } else {
+            ASSERT(patch_proof_overlap);
+            target_area_name = "module";
+            how_handle = DR_MODIFY_NOP; /* use pretend writable */
+            STATS_INC(app_modify_module_prot);
+        }
+
+        /* we can't be both pretend writable and emulate write */
+        ASSERT(!vmvector_overlap(emulate_write_areas, base, base+size));
+
+        if (how_handle == DR_MODIFY_HALT) {
+            /* Until we've fixed our DR area list problems and gotten shim.dll to work,
+             * we will issue an unrecoverable error
+             */
+            report_dynamorio_problem(dcontext, DUMPCORE_SECURITY_VIOLATION, NULL, NULL,
+                                     "Application changing protections of "
+                                     "%s memory @"PFX"-"PFX,
+                                     target_area_name, base, base+size);
+            /* FIXME: walking the loader data structures at arbitrary 
+             * points is dangerous due to data races with other threads 
+             * -- see is_module_being_initialized and get_module_name
+             */
+            check_for_unsupported_modules();
+            os_terminate(dcontext, TERMINATE_PROCESS);
+            ASSERT_NOT_REACHED();
+        } else {
+            SYSLOG_INTERNAL_WARNING("Application changing protections of "
+                                    "%s memory @"PFX"-"PFX, 
+                                    target_area_name, base, base+size);
+            if (how_handle == DR_MODIFY_NOP) {
+                /* we use a separate list, rather than a flag on DR areas, as the
+                 * affected region could include non-DR memory
+                 */
+                /* FIXME: note that we do not intersect with a concrete
+                 * region that we want to protect - considering Win32
+                 * protection changes allowed only separately
+                 * allocated regions this may be ok.  If we want to
+                 * have subpage regions then it becomes an issue:
+                 * we'd have to be able to emulate a write on a
+                 * page that has pretend writable regions
+                 */
+                ASSERT(ALIGNED(base, PAGE_SIZE));
+                ASSERT(ALIGNED(size, PAGE_SIZE));
+                write_lock(&pretend_writable_areas->lock);
+                if (TEST(MEMPROT_WRITE, prot)) {
+                    LOG(THREAD, LOG_VMAREAS, 2, "adding pretend-writable region "PFX"-"PFX"\n",
+                        base, base+size);
+                    add_vm_area(pretend_writable_areas, base, base+size,
+                                true, 0 _IF_DEBUG("DR_MODIFY_NOP"));
+                } else {
+                    LOG(THREAD, LOG_VMAREAS, 2, "removing pretend-writable region "PFX"-"PFX"\n",
+                        base, base+size);
+                    remove_vm_area(pretend_writable_areas, base, base+size, false);
+                }
+                write_unlock(&pretend_writable_areas->lock);
+                LOG(THREAD, LOG_VMAREAS, 2, "turning system call into a nop\n");
+
+                if (old_memprot != NULL) {
+                    /* FIXME: case 10437 we should keep track of any previous values */
+                    if (!get_memory_info(base, NULL, NULL, old_memprot)) {
+                        /* FIXME: should we fail instead of feigning success? */
+                        ASSERT_CURIOSITY(false && "prot change nop should fail");
+                        *old_memprot = MEMPROT_NONE;
+                    }
+                }
+                return PRETEND_APP_MEM_PROT_CHANGE; /* have syscall be a nop! */
+            } else if (how_handle == DR_MODIFY_FAIL) {
+                /* not the default b/c hooks that target our DLL often ignore the return
+                 * code of the syscall and blindly write, failing on the write fault.
+                 */
+                LOG(THREAD, LOG_VMAREAS, 2, "turning system call into a failure\n");
+                return FAIL_APP_MEM_PROT_CHANGE; /* have syscall fail! */
+            } else if (how_handle == DR_MODIFY_ALLOW) {
+                LOG(THREAD, LOG_VMAREAS, 2, "ALLOWING system call!\n");
+                return DO_APP_MEM_PROT_CHANGE;
+            }
+        }
+        ASSERT_NOT_REACHED();
+    }
+
+    /* DR areas may have changed, but we still have to remove from pretend list */
+    if (USING_PRETEND_WRITABLE() && !TEST(MEMPROT_WRITE, prot) && 
+        pretend_writable_vm_area_overlap(base, base+size)) {
+        ASSERT_NOT_TESTED();
+        /* FIXME: again we have the race -- if we could go from read to write
+         * it would be a simple fix, else have to grab write up front, or check again
+         */
+        write_lock(&pretend_writable_areas->lock);
+        LOG(THREAD, LOG_VMAREAS, 2, "removing pretend-writable region "PFX"-"PFX"\n",
+            base, base+size);
+        remove_vm_area(pretend_writable_areas, base, base+size, false);
+        write_unlock(&pretend_writable_areas->lock);
+    }
+
+#ifdef PROGRAM_SHEPHERDING
+    if (USING_FUTURE_EXEC_LIST && futureexec_vm_area_overlap(base, base + size)) {
+        /* something changed */
+        if (!TEST(MEMPROT_EXEC, prot)) {
+            /* we DO remove future regions just b/c they're now marked non-x
+             * but we may want to re-consider this -- some hooks briefly go to rw, e.g.
+             * although we MUST do this for executable_if_exec
+             * we should add flags to future areas indicating which policy put it here
+             * (have to not merge different policies, I guess -- problematic for
+             * sub-page flush combined w/ other policies?)
+             */
+            DEBUG_DECLARE(bool ok =) remove_futureexec_vm_area(base, base + size);
+            ASSERT(ok);
+            LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                "future region "PFX"-"PFX" is being made non-x, removing\n",
+                base, base + size);
+        } else {
+            /* Maybe nothing is changed in fact. */
+            /* In fact this happens when a protection size larger than
+             * necessary for a hook leaves some pages on the
+             * futureexec_vm_area_overlap region (case 2871 for a two
+             * page hooker).  There is nothing to do here,
+             * executable_if_hook should re-add the pages.
+             */
+            /* case 3279 - probably similar behaviour -- when a second
+             * NOP memory protection change happens to a region
+             * already on the future list - we'd need to power it up
+             * again
+             */
+            /* xref case 3102 - where we don't care about VM_WRITABLE */
+#if 0 /* this syslog may causes services.exe to hang (ref case 666)  */
+            SYSLOG_INTERNAL_WARNING("future executable area overlapping with "PFX"-"PFX" made %s",
+                                    base, base + size, memprot_string(prot));
+#endif
+        }
+    }
+#endif
+
+#if defined(PROGRAM_SHEPHERDING) && defined(WINDOWS)
+    /* Just remove up front if changing anything about an emulation region.
+     * Should certainly remove if becoming -w, but should also remove if
+     * being added to exec list -- current usage expects to be removed on
+     * next protection change (hooker restoring IAT privileges).
+     * FIXME: should make the ->rx restoration syscall a NOP for performance
+     */
+    if (DYNAMO_OPTION(emulate_IAT_writes) &&
+        !vmvector_empty(emulate_write_areas) &&
+        vmvector_overlap(emulate_write_areas, base, base+size)) {
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+            "removing emulation region "PFX"-"PFX"\n", base, base+size);
+        vmvector_remove(emulate_write_areas, base, base+size);
+    }
+#endif
+
+    /* look for calls making code writable! 
+     * cache is_executable here w/o holding lock -- if decide to perform state
+     * change via flushing, we'll re-check overlap there and all will be atomic
+     * at that point, no reason to try and make atomic from here, will hit
+     * deadlock issues w/ thread_initexit_lock
+     */
+    is_executable = executable_vm_area_overlap(base, base + size, false/*have no lock*/);
+    if (is_executable && TEST(MEMPROT_WRITE, prot) && !TEST(MEMPROT_EXEC, prot)) {
+#ifdef WINDOWS
+        app_pc IAT_start, IAT_end;
+        /* Could not page-align and ask for original params but some hookers
+         * page-align even when targeting only IAT */
+        bool is_iat = is_IAT(base, base+size, true/*page-align*/, &IAT_start, &IAT_end);
+        bool is_patch = is_module_patch_region(dcontext, base, base+size,
+                                               true/*be conservative*/);
+        DOSTATS({
+            if (is_iat && is_patch)
+                STATS_INC(num_app_rebinds);
+        });
+#ifdef PROGRAM_SHEPHERDING
+        /* This potentially unsafe option is superseded by -coarse_merge_iat 
+         * FIXME: this should be available for !PROGRAM_SHEPHERDING
+         */
+        if (DYNAMO_OPTION(unsafe_ignore_IAT_writes) && is_iat && is_patch) {
+            /* do nothing: let go writable and then come back */
+            LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                "WARNING: letting IAT be written w/o flushing: potentially unsafe\n");
+            return DO_APP_MEM_PROT_CHANGE; /* let syscall go through */
+        }
+#endif
+        /* Case 11072: must match these conditions w/ the assert on freeing */
+        if (DYNAMO_OPTION(coarse_units) && DYNAMO_OPTION(coarse_merge_iat) &&
+# ifdef PROGRAM_SHEPHERDING
+            /* Ensure we'll re-mark as valid */
+            (DYNAMO_OPTION(executable_if_rx_text) ||
+             DYNAMO_OPTION(executable_after_load)) &&
+# endif
+            is_iat && is_patch &&
+            !executable_vm_area_executed_from(IAT_start, IAT_end) &&
+            /* case 10830/11072: ensure currently marked coarse-grain to avoid
+             * blessing the IAT region as coarse when it was in fact made non-coarse
+             * due to a rebase (or anything else) prior to a rebind.  check the end,
+             * since we may have adjusted the exec area bounds to be post-IAT.
+             */
+            get_executable_area_flags(base+size-1, &frag_flags) &&
+            TEST(FRAG_COARSE_GRAIN, frag_flags)) {
+            coarse_info_t *info = 
+                get_coarse_info_internal(IAT_end, false/*no init*/, false/*no lock*/);
+            /* loader rebinding
+             * We cmp and free the stored code at +rx time; if that doesn't happen,
+             * we free at module unload time.
+             */
+            DEBUG_DECLARE(bool success =)
+                os_module_store_IAT_code(base);
+            ASSERT(success);
+            ASSERT(!RUNNING_WITHOUT_CODE_CACHE()); /* FRAG_COARSE_GRAIN excludes */
+            LOG(GLOBAL, LOG_VMAREAS, 2,
+                "storing IAT code for "PFX"-"PFX"\n", IAT_start, IAT_end);
+            if (info != NULL) {
+                /* Only expect to do this for empty or persisted units */
+                ASSERT(info->cache == NULL ||
+                       (info->persisted && info->non_frozen != NULL &&
+                        info->non_frozen->cache == NULL));
+                /* Do not reset/free during flush as we hope to see a validating
+                 * event soon. 
+                 */
+                ASSERT(!TEST(PERSCACHE_CODE_INVALID, info->flags));
+                info->flags |= PERSCACHE_CODE_INVALID;
+                STATS_INC(coarse_marked_invalid);
+            }
+        }
+# ifdef PROGRAM_SHEPHERDING
+        if (DYNAMO_OPTION(emulate_IAT_writes) && is_iat &&
+            /* We do NOT want to emulate hundreds of writes by the loader -- we
+             * assume no other thread will execute in the module until it's
+             * initialized.  We only need our emulation for hookers who come in
+             * after initialization when another thread may be in there.
+             */
+            !is_patch) {
+            /* To avoid having the IAT page (which often includes the start of the
+             * text section) off the exec areas list, we only remove the IAT itself,
+             * and emulate writes to it.
+             * FIXME: perhaps this should become an IAT-only vector, and be used
+             * for when we have the IAT read-only to protect it security-wise.
+             */
+            /* unfortunately we have to flush to be conservative */
+            should_finish_flushing =
+                flush_and_remove_executable_vm_area(dcontext, IAT_start,
+                                                    IAT_end - IAT_start);
+            /* a write to IAT gets emulated, but to elsewhere on page is a code mod */
+            vmvector_add(emulate_write_areas, IAT_start, IAT_end, NULL);
+            /* must release the exec areas lock, even if expect no flush */
+            if (should_finish_flushing) {
+                flush_fragments_in_region_finish(dcontext,
+                                                 false /*don't keep initexit_lock*/);
+            }
+            LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                "executable region == IAT so not marking %s, emulating writes\n",
+                memprot_string(prot));
+            /* now leave as read-only.
+             * we do not record what other flags they're using here -- we assume
+             * they're going to restore IAT back to what it was
+             */
+            /* FIXME: case 10437 we should keep track of any previous values */
+            if (old_memprot != NULL) {
+                if (!get_memory_info(base, NULL, NULL, old_memprot)) {
+                    /* FIXME: should we fail instead of feigning success? */
+                    ASSERT_CURIOSITY(false && "prot change nop should fail");
+                    *old_memprot = MEMPROT_NONE;
+                }
+            }
+            return PRETEND_APP_MEM_PROT_CHANGE;
+        }
+# endif /* PROGRAM_SHEPHERDING */
+#endif /* WINDOWS */
+        /* being made writable but non-executable! 
+         * kill all current fragments in the region (since a
+         * non-executable region is ignored by flush routine)
+         */
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+            "WARNING: executable region being made writable and non-executable\n");
+        flush_fragments_and_remove_region(dcontext, base, size,
+                                          false /* don't own initexit_lock */,
+                                          false /* case 2236: keep futures */);
+#ifdef HOT_PATCHING_INTERFACE
+        if (DYNAMO_OPTION(hotp_only))
+            hotp_only_mem_prot_change(base, size, true, false);
+#endif 
+    }
+    else if (is_executable && TESTALL(MEMPROT_WRITE | MEMPROT_EXEC, prot)) {
+        /* Need to flush all fragments in [base, base+size), unless
+         * they are ALL already writable 
+         */
+        DOSTATS({
+            /* If all the overlapping executable areas are VM_WRITABLE|
+             * VM_DELAY_READONLY then we could optimize away the flush since
+             * we haven't made any portion of this region read only for 
+             * consistency purposes.  We haven't implemented this optimization
+             * as it's quite rare (though does happen xref case 8104) and
+             * previous implementations of this optimization proved buggy. */
+            if (is_executable_area_writable_overlap(base, base + size, 
+                                                     true /* ALL regions are: */,
+                                                     VM_WRITABLE|VM_DELAY_READONLY)) {
+                STATS_INC(num_possible_app_to_rwx_skip_flush);
+            }
+        });
+        /* executable region being made writable
+         * flush all current fragments, and mark as non-executable
+         */
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+            "WARNING: executable region "PFX"-"PFX" is being made writable!\n"
+            "\tRemoving from executable list\n",
+            base, base + size);
+        /* use two-part flush to make futureexec & exec changes atomic w/ flush */
+        should_finish_flushing = flush_and_remove_executable_vm_area(dcontext, base, size);
+        /* we flush_fragments_finish after security checks to keep them atomic */
+    } 
+    else if (is_executable && is_executable_area_writable(base) &&
+             !TEST(MEMPROT_WRITE, prot) && TEST(MEMPROT_EXEC, prot)) {
+        /* executable & writable region being made read-only
+         * make sure any future write faults are given to app, not us
+         */
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+            "executable writable region "PFX"-"PFX" => read-only!\n",
+            base, base + size);
+        /* remove writable exec area, then add read-only exec area */
+        /* use two-part flush to make futureexec & exec changes atomic w/ flush */
+        should_finish_flushing = flush_and_remove_executable_vm_area(dcontext, base, size);
+        /* FIXME: this is wrong -- this will make all pieces in the middle executable,
+         * which is not what we want -- we want all pieces ON THE EXEC LIST to
+         * change from rw to r.  thus this should be like the change-to-selfmod case
+         * in handle_modified_code => add new vector routine?  (case 3570)
+         */
+        add_executable_vm_area(base, base + size,
+                               0 /* not image? FIXME */, 0,
+                               should_finish_flushing/* own lock if flushed */
+                               _IF_DEBUG("protection change"));
+    }
+    /* also look for calls making data executable 
+     * FIXME: perhaps should do a write_keep for this is_executable, to bind
+     * to the subsequent exec areas changes -- though case 2833 would still be there
+     */
+    else if (!is_executable && TEST(MEMPROT_EXEC, prot)) {
+        if (TEST(MEMPROT_WRITE, prot)) {
+            /* do NOT add to executable list if writable */
+            LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                "WARNING: data region "PFX"-"PFX" made executable and "
+                "writable, not adding to exec list\n", base, base + size);
+        } else {
+            bool add_to_exec_list = false;
+#ifdef WINDOWS
+            bool check_iat = false;
+            bool free_iat = false;
+#endif
+            uint frag_flags = 0;
+            DEBUG_DECLARE(char *comment = "";)
+            LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                "WARNING: data region "PFX" is being made executable\n", base);
+#ifdef PROGRAM_SHEPHERDING
+            /* if on future, no reason to add to exec list now
+             * if once-only, no reason to add to exec list and remove from future
+             * wait until actually executed!
+             */
+            /* none of our policies allow this on the stack */
+            if (is_address_on_stack(dcontext, base)) {
+                LOG(THREAD, LOG_VMAREAS, 2,
+                    "not allowing data->x for stack region\n");
+# ifdef WINDOWS
+            } else if (DYNAMO_OPTION(executable_after_load) &&
+                       is_module_patch_region(dcontext, base, base+size,
+                                              false/*be liberal: can't miss loader*/)) {
+                STATS_INC(num_mark_after_load);
+                add_to_exec_list = true;
+                check_iat = true;
+                DODEBUG({ comment = "if_after_load"; });
+                LOG(THREAD, LOG_VMAREAS, 2,
+                    "module is being initialized, adding region to executable list\n");
+
+# endif
+            } else if (DYNAMO_OPTION(executable_if_rx_text)) {
+                /* FIXME: this should be moved out of the if (!executable) branch? 
+                 * to where executable_if_x is handled
+                 */
+                /* NOTE - xref case 10526, the check here is insufficient to implement
+                 * this policy because [*base, *base+*size) could overlap multiple
+                 * sections (some of which might not be code) which would cause this
+                 * check to fail.  Fixing this here would require us to find the
+                 * intersection of this region and any code section(s) and add the
+                 * resulting region(s) (there could be more then one). Instead we leave
+                 * this check here to catch the common case but extend
+                 * check_origins_helper to catch anything unusual. */
+                app_pc modbase = get_module_base(base);
+                if (modbase != NULL && 
+                    is_range_in_code_section(modbase, base, base+size, NULL, NULL)) {
+                    STATS_INC(num_2rx_text);
+                    add_to_exec_list = true;
+                    IF_WINDOWS(check_iat = true;)
+                    DODEBUG({ comment = "if_rx_text"; });
+                    LOG(THREAD, LOG_VMAREAS, 2,
+                        "adding code region being marked rx to executable list\n");
+                }
+            } /* Don't use an  else if here, the else if for 
+               * -executable_if_rx_text if doesn't check all its 
+               * conditionals in the first if */
+
+            if (DYNAMO_OPTION(executable_if_rx)) {
+                STATS_INC(num_mark_if_rx);
+                add_to_exec_list = true;
+                mark_module_exempted(base);
+                DODEBUG({ comment = "if_rx"; });
+                LOG(THREAD, LOG_VMAREAS, 2, "adding region marked only rx "
+                                            "to executable list\n");
+            }
+#else
+            add_to_exec_list = true;
+            IF_WINDOWS(check_iat = true;)
+#endif
+#ifdef WINDOWS
+            if (check_iat) {
+                if (DYNAMO_OPTION(coarse_units) && DYNAMO_OPTION(coarse_merge_iat) &&
+                    is_IAT(base, base+size, true/*page-align*/, NULL, NULL))
+                    free_iat = true;
+                LOG(THREAD, LOG_VMAREAS, 2,
+                    "IAT is being made rx again "PFX"-"PFX"\n", base, base+size);
+                if (!RUNNING_WITHOUT_CODE_CACHE()) {
+                    /* case 8640: let add_executable_vm_area() decide whether to
+                     * keep the coarse-grain flag
+                     */
+                    frag_flags |= FRAG_COARSE_GRAIN;
+                } else {
+                    free_iat = false;
+                    ASSERT(!os_module_free_IAT_code(base));
+                }
+            }
+#endif
+            if (add_to_exec_list) {
+                /* FIXME : see note at top of function about bug 2833 */
+                ASSERT(!TEST(MEMPROT_WRITE, prot)); /* sanity check */
+                add_executable_vm_area(base, base + size, 
+                                       0 /* not an unmodified image */, frag_flags,
+                                       false/*no lock*/ _IF_DEBUG(comment));
+            }
+#ifdef WINDOWS
+            if (free_iat) {
+                DEBUG_DECLARE(bool had_iat =)
+                    os_module_free_IAT_code(base);
+                DEBUG_DECLARE(app_pc text_start;)
+                DEBUG_DECLARE(app_pc text_end;)
+                DEBUG_DECLARE(app_pc iat_start;)
+                DEBUG_DECLARE(app_pc iat_end;)
+                /* calculate IAT bounds */
+                ASSERT(is_IAT(base, base+size, true/*page-align*/,
+                              &iat_start, &iat_end));
+                ASSERT(had_iat ||
+                       /* duplicate the reasons we wouldn't have stored the IAT: */
+                       !is_module_patch_region(dcontext, base, base+size,
+                                               true/*be conservative*/) ||
+                       executable_vm_area_executed_from(iat_start, iat_end) ||
+                       /* case 11072: rebase prior to rebind prevents IAT storage */
+                       (get_module_preferred_base_delta(base) != 0 &&
+                        is_in_code_section(get_module_base(base), base,
+                                           &text_start, &text_end) &&
+                        iat_start >= text_start && iat_end <= text_end));
+            }
+#endif
+#ifdef HOT_PATCHING_INTERFACE
+            if (DYNAMO_OPTION(hotp_only))
+                hotp_only_mem_prot_change(base, size, false, true);
+#endif 
+        }
+    }
+
+#ifdef PROGRAM_SHEPHERDING
+    /* These policies do not depend on a transition taking place. */
+    /* Make sure weaker policies are considered first, so that
+     * the region is kept on the futureexec list with the least restrictions
+     */
+    if (DYNAMO_OPTION(executable_if_x) && TEST(MEMPROT_EXEC, prot)) {
+        /* The executable_if_x policy considers all code marked ..x to be executable */
+        
+        /* Note that executable_if_rx may have added a region directly
+         * to the executable_areas, while here we only add to the futureexec_areas
+         * FIXME: move executable_if_rx checks as an 'else if' following this if.
+         */
+        LOG(GLOBAL, LOG_VMAREAS, 1,
+            "New future region b/c x, "PFX"-"PFX" %s, was %sexecutable\n",
+            base, base+size, memprot_string(prot), is_executable ? "" : "not ");
+        STATS_INC(num_mark_if_x);
+        add_futureexec_vm_area(base, base+size, false/*permanent*/
+                               _IF_DEBUG(TEST(MEMPROT_WRITE, prot) ? 
+                                         "executable_if_x protect exec .wx" :
+                                         "executable_if_x protect exec .-x"
+                                         ));
+        mark_module_exempted(base);
+    } else if (DYNAMO_OPTION(executable_if_hook) && TESTALL(MEMPROT_WRITE | MEMPROT_EXEC, prot)) {
+        /* Note here we're strict in requesting a .WX setting by the
+         * hooker, won't be surprising if some don't do even this 
+         */
+        /* FIXME: could restrict to sub-page piece of text section,
+         * since should only be targeting 4 or 5 byte area
+         */
+        app_pc modbase = get_module_base(base);
+        if (modbase != NULL) { /* PE, and is readable */
+            /* FIXME - xref case 10526, if the base - base+size overlaps more than
+             * one section then this policy won't apply, though not clear if we'd want
+             * it to for such an unusual hooker. */
+            if (is_range_in_code_section(modbase, base, base+size, NULL, NULL)) {
+                uint vm_flags;
+                DOLOG(2, LOG_INTERP|LOG_VMAREAS, {
+                    char modname[MAX_MODNAME_INTERNAL];
+                    os_get_module_name_buf(modbase, modname,
+                                           BUFFER_SIZE_ELEMENTS(modname));
+                    LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+                        "adding hook to future list: "PFX"-"PFX" in code of "
+                        "module @"PFX" == %s made rwx\n",
+                        base, base+size, modbase,
+                        modname == NULL? "<invalid name>" : modname);
+                });
+                STATS_INC(num_hook);
+
+                /* add as a once-only future area */
+                add_futureexec_vm_area(base, base + size, 
+                                       true/*once-only*/
+                                       _IF_DEBUG(memprot_string(prot)));
+                /* This is text section, leave area on executable list
+                 * so app can execute here, write, and then execute
+                 * again (via future list) to handle cases of hooking
+                 * kernel32 functions ref case 2803 and case 3097 */
+
+                if (!should_finish_flushing) {
+                    /* FIXME: as a quick fix we flush the existing area
+                     * just in case anyways, so that we don't think about
+                     * merging properly the FRAG_DYNGEN
+                     */
+                    should_finish_flushing =
+                        flush_and_remove_executable_vm_area(dcontext, base, size);
+                }
+
+                /* FIXME: we could optimize away the VM_DELAY_READONLY
+                 * path if we actually knew that the current real
+                 * protection flag is not writable.  Yet we've removed
+                 * any internal data about it, so we need
+                 * restructuring or an extra system call here vs the
+                 * safe one at make_unwritable().
+                 *
+                 * case 8308: Don't mark as DELAY_READONLY if -sandbox_writable
+                 * is on.  We don't need to check for -sandbox_non_text here
+                 * since we know we're in a text region here.
+                 */
+                vm_flags = VM_WRITABLE;
+                if (!DYNAMO_OPTION(sandbox_writable))
+                    vm_flags |= VM_DELAY_READONLY;
+
+                add_executable_vm_area(base, base + size, vm_flags,
+                                       0, should_finish_flushing/* own the lock if
+                                                                   we have flushed */
+                                       _IF_DEBUG("prot chg text rx->rwx not yet written"));
+                /* leave read only since we are leaving on exec list */
+                if (should_finish_flushing) {
+                    flush_fragments_in_region_finish(dcontext, 
+                                                     false /*don't keep initexit_lock*/);
+                }
+
+                if (old_memprot != NULL) {
+                    /* FIXME: case 10437 we should keep track of any previous values */
+                    if (!get_memory_info(base, NULL, NULL, old_memprot)) {
+                        /* FIXME: should we fail instead of feigning success? */
+                        ASSERT_CURIOSITY(false && "prot change nop should fail");
+                        *old_memprot = MEMPROT_NONE;
+                    }
+                }
+                /* case 10387 initial fix - on a DEP machine to
+                 * support properly native execution we must set the X
+                 * bit: most needed for -hotp_only when we provide our
+                 * code origins policies for GBOP enforcement, but
+                 * similar need in native_exec or other possible mixed
+                 * modes.
+                 */
+
+                /* We really should be setting everything according to
+                 * app request except for writability.  Hopefully we
+                 * don't have sophisticated hookers using PAGE_GUARD
+                 * so ok to use only the memprot supported flags.
+                 */
+                prot &= ~MEMPROT_WRITE;
+                ASSERT_CURIOSITY(TESTALL(MEMPROT_READ|MEMPROT_EXEC, prot));
+
+                *new_memprot = prot;
+                return SUBSET_APP_MEM_PROT_CHANGE;
+            }
+        }
+    }
+#endif /* PROGRAM_SHEPHERDING */
+    if (should_finish_flushing)
+        flush_fragments_in_region_finish(dcontext, false /*don't keep initexit_lock*/);
+
+    return DO_APP_MEM_PROT_CHANGE; /* let syscall go through */
+}
+
+#ifdef WINDOWS
+/* memory region base:base+size was flushed from hardware icache by app */
+void
+app_memory_flush(dcontext_t *dcontext, app_pc base, size_t size, uint prot)
+{
+# ifdef PROGRAM_SHEPHERDING
+    if (DYNAMO_OPTION(executable_if_flush)) {
+        /* We want to ignore the loader calling flush, since our current
+         * impl makes a flush region permanently executable.
+         * The loader always follows the order "rw, rx, flush", but we have
+         * seen real DGC marking rx before flushing as well, so we use
+         * our module-being-loaded test:
+         */
+        if (!is_module_patch_region(dcontext, base, base+size,
+                                    false/*be liberal: don't miss loader*/)) {
+            /* FIXME case 280: we'd like to always be once-only, but writes
+             * to data on the same page make it hard to do that.
+             */
+            bool onceonly = false;
+            /* we do NOT go to page boundaries, instead we put sub-page
+             * regions on our future list
+             */
+            LOG(GLOBAL, LOG_VMAREAS, 1,
+                "New future exec region b/c flushed: "PFX"-"PFX" %s\n",
+                base, base+size, memprot_string(prot));
+            if (!DYNAMO_OPTION(selfmod_futureexec) &&
+                is_executable_area_on_all_selfmod_pages(base, base+size)) {
+                /* for selfmod we can be onceonly, as writes to data on the
+                 * same page won't kick us off the executable list
+                 */
+                onceonly = true;
+            }
+            add_futureexec_vm_area(base, base + size, onceonly
+                                   _IF_DEBUG("NtFlushInstructionCache"));
+            if (DYNAMO_OPTION(xdata_rct)) {
+                /* FIXME: for now we only care about start pc */
+                vmvector_add(app_flushed_areas, base, base+1, NULL);
+                /* FIXME: remove when region de-allocated? */
+            }
+            DOSTATS({
+                if (is_executable_area_writable(base))
+                    STATS_INC(num_NT_flush_w2r); /* pretend writable (we made RO) */
+                if (TEST(MEMPROT_WRITE, prot))
+                    STATS_INC(num_NT_flush_w);
+                else
+                    STATS_INC(num_NT_flush_r);
+                if (is_address_on_stack(dcontext, base)) {
+                    STATS_INC(num_NT_flush_stack);
+                } else {
+                    STATS_INC(num_NT_flush_heap);
+                }
+            });
+        } else {
+            LOG(THREAD, LOG_VMAREAS, 1, "module is being loaded, ignoring flush\n");
+            STATS_INC(num_NT_flush_loader);
+        }
+    }
+# else
+    /* NOP */
+# endif /* PROGRAM_SHEPHERDING */
+}
+
+# ifdef PROGRAM_SHEPHERDING
+bool
+was_address_flush_start(dcontext_t *dcontext, app_pc pc)
+{
+    ASSERT(DYNAMO_OPTION(xdata_rct));
+    /* FIXME: once we have flags marking where each futureexec region
+     * came from we can distinguish NtFlush, but for now we need our own list,
+     * which as FIXME above says could be simply htable since we only care about
+     * start_pc (for now).
+     * We assume we only add start pcs to the vector.
+     */
+    return vmvector_overlap(app_flushed_areas, pc, pc + 1);
+}
+# endif
+#endif
+
+/****************************************************************************/
+
+/* a helper function for check_thread_vm_area
+ * assumes caller owns executable_areas write lock */
+static void
+handle_delay_readonly(dcontext_t *dcontext, app_pc pc, vm_area_t *area)
+{
+    ASSERT_OWN_WRITE_LOCK(true, &executable_areas->lock);
+    ASSERT(TESTALL(VM_DELAY_READONLY|VM_WRITABLE, area->vm_flags));
+    /* should never get a selfmod region here, to be marked selfmod
+     * would already have had to execute (to get faulting write)
+     * so region would already have had to go through here */
+    ASSERT(!TEST(FRAG_SELFMOD_SANDBOXED, area->frag_flags));
+    if (!is_on_stack(dcontext, pc, NULL) && INTERNAL_OPTION(cache_consistency)) {
+        vm_make_unwritable(area->start, area->end - area->start);
+        area->vm_flags |= VM_MADE_READONLY;
+    } else {
+        /* this could happen if app changed mem protection on its 
+         * stack that triggered us adding a delay_readonly writable
+         * region to the executable list in 
+         * app_memory_protection_change() */
+        ASSERT_CURIOSITY(false);
+        area->frag_flags |= FRAG_SELFMOD_SANDBOXED;
+    }
+    area->vm_flags &= ~VM_DELAY_READONLY;
+    LOG(GLOBAL, LOG_VMAREAS, 2,
+        "\tMarking existing wx vm_area_t ro for consistency, "
+        "area "PFX" - "PFX", target pc "PFX"\n",
+        area->start, area->end, pc);
+    STATS_INC(num_delayed_rw2r);
+}
+
+/* Frees resources acquired in check_thread_vm_area().
+ * data and vmlist need to match those used in check_thread_vm_area().
+ * abort indicates that we are forging and exception or killing a thread
+ * or some other drastic action that will not return to the caller
+ * of check_thread_vm_area.
+ * own_execareas_writelock indicates whether the executable_areas
+ * write lock is currently held, while caller_execareas_writelock
+ * indicates whether the caller held that lock and thus we should not
+ * free it unless we're aborting.
+ * If both clean_bb and abort are true, calls bb_build_abort.
+ */
+static void
+check_thread_vm_area_cleanup(dcontext_t *dcontext, bool abort, bool clean_bb,
+                             thread_data_t *data, void **vmlist,
+                             bool own_execareas_writelock,
+                             bool caller_execareas_writelock)
+{
+    if (own_execareas_writelock && (!caller_execareas_writelock || abort)) {
+        ASSERT(self_owns_write_lock(&executable_areas->lock));
+        write_unlock(&executable_areas->lock);
+#ifdef HOT_PATCHING_INTERFACE
+        if (DYNAMO_OPTION(hot_patching)) {
+            ASSERT(self_owns_write_lock(hotp_get_lock()));
+            write_unlock(hotp_get_lock());
+        }
+#endif
+    }
+    ASSERT(!caller_execareas_writelock || self_owns_write_lock(&executable_areas->lock));
+    /* FIXME: could we have multiply-nested vmlist==NULL where we'd need to
+     * release read lock more than once? */
+    if (vmlist == NULL)
+        SHARED_VECTOR_RWLOCK(&data->areas, read, unlock);
+    if (self_owns_write_lock(&data->areas.lock) && (vmlist != NULL || abort)) {
+        /* Case 9376: we can forge an exception for vmlist==NULL, in which case
+         * we must release the write lock from the prior layer;
+         * we can also have a decode fault with vmlist!=NULL but w/o holding
+         * the vm areas lock.
+         */
+        SHARED_VECTOR_RWLOCK(&data->areas, write, unlock);
+    } /* we need to not unlock vmareas for nested check_thread_vm_area() call */
+    if (abort) {
+        if (vmlist != NULL && *vmlist != NULL) {
+            vm_area_destroy_list(dcontext, *vmlist);
+        }
+        if (clean_bb) {
+            /* clean up bb_building_lock and IR */
+            bb_build_abort(dcontext, false/*don't call back*/);
+        }
+    }
+}
+
+/* Releases any held locks.  Up to caller to free vmlist.
+ * Flags are reverse logic, just like for check_thread_vm_area()
+ */
+void
+check_thread_vm_area_abort(dcontext_t *dcontext, void **vmlist, uint flags)
+{
+    thread_data_t *data;
+    if (DYNAMO_OPTION(shared_bbs) &&
+        !TEST(FRAG_SHARED, flags)) { /* yes, reverse logic, see comment above */
+        data = shared_data;
+    } else {
+        data = (thread_data_t *) dcontext->vm_areas_field;
+    }
+    check_thread_vm_area_cleanup(dcontext, true, false/*caller takes care of bb*/,
+                                 data, vmlist,
+                                 self_owns_write_lock(&executable_areas->lock),
+                                 self_owns_write_lock(&data->areas.lock));
+}
+
+/* check origins of code for several purposes:
+ * 1) we need list of areas where this thread's fragments come
+ *    from, for faster flushing on munmaps
+ * 2) also for faster flushing, each vmarea has a list of fragments
+ * 3) we need to mark as read-only any writable region that
+ *    has a fragment come from it, to handle self-modifying code
+ * 4) for PROGRAM_SHEPHERDING for security
+ *
+ * We keep a list of vm areas per thread, to make flushing fragments
+ * due to memory unmaps faster
+ * This routine adds the page containing start to the thread's list.
+ * Adds any FRAG_ flags relevant for a fragment overlapping start's page.
+ * If xfer and encounters change in vmareas flags, returns false and does NOT
+ * add the new page to the list for this fragment -- assumes caller will NOT add
+ * it to the current bb.  This allows for selectively not following direct ctis.
+ * Assumes only building a real app bb if vmlist!=NULL -- assumes that otherwise
+ * caller is reconstructing an app bb or some other secondary bb walk.
+ * If returns true, returns in the optional stop OUT parameter the final pc of
+ * this region (open-ended).
+ */
+bool
+check_thread_vm_area(dcontext_t *dcontext, app_pc pc, app_pc tag, void **vmlist,
+                     uint *flags, app_pc *stop, bool xfer)
+{
+    bool result;
+    thread_data_t *data;
+    bool in_last = false;
+    uint frag_flags = 0;
+    uint vm_flags = 0;
+    bool ok;
+    bool shared_to_private = false;
+    /* used for new area */
+    app_pc base_pc = 0;
+    size_t size = 0;                  /* set only for unknown areas */
+    uint prot = 0;              /* set only for unknown areas */
+    /* both area and local_area either point to thread-local vector, for which
+     * we do not need a lock, or to a shared area, for which we hold
+     * a read or a write lock (either is sufficient) the entire time
+     */
+    vm_area_t *area = NULL;
+    vm_area_t *local_area = NULL; /* entry for this thread */
+    vm_area_t area_copy; /* local copy, so can let go of lock */
+    /* the flags we don't allow a direct cti to bridge if different */
+    const uint frag_flags_cmp = FRAG_SELFMOD_SANDBOXED | FRAG_COARSE_GRAIN
+#ifdef PROGRAM_SHEPHERDING
+        | FRAG_DYNGEN
+#endif
+        ;
+    /* we can be recursively called (check_origins() calling build_app_bb_ilist())
+     * so make sure we don't re-try to get a lock we already hold
+     */
+    bool caller_execareas_writelock = self_owns_write_lock(&executable_areas->lock);
+    bool own_execareas_writelock = caller_execareas_writelock;
+    DEBUG_DECLARE(char *new_area_prefix;)
+
+    /* deadlock issues if write lock is held already for vmlist!=NULL case */
+    ASSERT(vmlist == NULL || !caller_execareas_writelock);
+#ifdef HOT_PATCHING_INTERFACE
+    /* hotp_vul_table_lock goes hand in hand w/ executable_areas lock here */
+    ASSERT(!DYNAMO_OPTION(hot_patching) ||
+           (own_execareas_writelock && self_owns_write_lock(hotp_get_lock())) ||
+           (!own_execareas_writelock && !self_owns_write_lock(hotp_get_lock())));
+#endif
+
+    ASSERT(flags != NULL);
+
+    /* don't know yet whether this bb will be shared, but a good chance,
+     * so we guess shared and will rectify later.
+     * later, to add to local instead, we call again, and to tell the difference
+     * we perversely pass FRAG_SHARED
+     */
+    if (DYNAMO_OPTION(shared_bbs) &&
+        /* for TEMP_PRIVATE we make private up front */
+        !TEST(FRAG_TEMP_PRIVATE, *flags) &&
+        !TEST(FRAG_SHARED, *flags)) { /* yes, reverse logic, see comment above */
+        data = shared_data;
+        DODEBUG({new_area_prefix = "new shared vm area: ";});
+        if (vmlist == NULL) { /* not making any state changes to vm lists */
+            /* need read access only, for lookup and holding ptr into vector */
+            SHARED_VECTOR_RWLOCK(&data->areas, read, lock);
+        } else { /* building a bb */
+            /* need write access later, and want our lookup to be bundled
+             * with our writes so we don't rely on the bb building lock,
+             * so we grab the write lock for the whole routine
+             */
+            SHARED_VECTOR_RWLOCK(&data->areas, write, lock);
+        }
+    } else {
+        DODEBUG({new_area_prefix = "new vm area for thread: ";});
+        data = (thread_data_t *) dcontext->vm_areas_field;
+        if (DYNAMO_OPTION(shared_bbs) && TEST(FRAG_SHARED, *flags))
+            shared_to_private = true;
+    }
+
+    LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 4,
+        "check_thread_vm_area: pc = "PFX"\n", pc);
+
+    /* no lock on data->areas needed if thread-local,
+     * if shared we grabbed either read or write lock above
+     */
+    /* check cached last area first to avoid lookup cost */
+    if (data->last_area != NULL)
+        in_last = (pc < data->last_area->end && data->last_area->start <= pc);
+
+    DOSTATS({
+        STATS_INC(checked_addresses);
+        if (in_last)
+            STATS_INC(looked_up_in_last_area);
+    });
+
+    if (in_last) {
+        local_area = data->last_area;
+        area = local_area;
+    } else if (lookup_addr(&data->areas, pc, &local_area)) {
+        /* ok to hold onto pointer since it's this thread's */
+        area = local_area;
+    } else {
+        /* not in this thread's current executable list
+         * try the global executable area list
+         */
+        if (!own_execareas_writelock)
+            read_lock(&executable_areas->lock);
+        ok = lookup_addr(executable_areas, pc, &area);
+        if (ok && TEST(VM_DELAY_READONLY, area->vm_flags)) {
+            /* need to mark region read only for consistency
+             * need to upgrade to write lock, have to release lock first
+             * then recheck conditions after grabbing hotp + write lock */
+            if (!own_execareas_writelock) {
+                read_unlock(&executable_areas->lock);
+#ifdef HOT_PATCHING_INTERFACE
+                /* Case 8780: due to lock rank issues we must grab the hotp lock
+                 * prior to the exec areas lock, as the hotp lock may be needed
+                 * for pc recreation in check_origins().  We assume this will
+                 * not cause noticeable lock contention.
+                 */
+                if (DYNAMO_OPTION(hot_patching))
+                    write_lock(hotp_get_lock());
+#endif
+                write_lock(&executable_areas->lock);
+                own_execareas_writelock = true;
+                ok = lookup_addr(executable_areas, pc, &area);
+            }
+            if (ok && TEST(VM_DELAY_READONLY, area->vm_flags)) 
+                handle_delay_readonly(dcontext, pc, area);
+        }
+        if ((!ok || (ok && vmlist != NULL && !TEST(VM_EXECUTED_FROM, area->vm_flags))) &&
+            !own_execareas_writelock) {
+            /* we must hold the write lock until we add the new region, as we
+             * may want to give it selfmod or other properties that will not mix
+             * well if we have a race and another thread adds an overlapping
+             * region with different properties!
+             * or if never executed from, we need to mark the area as such
+             * (if we didn't support thread-private, we would just grab write
+             * lock up front and not bother with read lock).
+             */
+            read_unlock(&executable_areas->lock);
+#ifdef HOT_PATCHING_INTERFACE
+            if (DYNAMO_OPTION(hot_patching))
+                write_lock(hotp_get_lock()); /* case 8780 -- see comments above */
+#endif
+            write_lock(&executable_areas->lock);
+            own_execareas_writelock = true;
+            ok = lookup_addr(executable_areas, pc, &area);
+        }
+        if (ok) {
+            if (vmlist != NULL && !TEST(VM_EXECUTED_FROM, area->vm_flags)) {
+                ASSERT(self_owns_write_lock(&executable_areas->lock));
+                area->vm_flags |= VM_EXECUTED_FROM;
+            }
+            area_copy = *area;
+            area = &area_copy;
+            /* if we already have an area, we do not need to hold an execareas
+             * lock, as there is no race within this routine.  any removal of
+             * the area must go through the flush synch and so cannot be
+             * concurrent to this routine.
+             */
+            if (own_execareas_writelock) {
+                if (!caller_execareas_writelock) {
+                    write_unlock(&executable_areas->lock);
+#ifdef HOT_PATCHING_INTERFACE
+                    if (DYNAMO_OPTION(hot_patching))
+                        write_unlock(hotp_get_lock()); /* case 8780 -- see above */
+#endif
+                    own_execareas_writelock = false;
+                }
+            } else
+                read_unlock(&executable_areas->lock);
+        }
+        /* if ok we should not own the readlock but we can't assert on that */
+        ASSERT(ok || (self_owns_write_lock(&executable_areas->lock) &&
+                      own_execareas_writelock
+                      IF_HOTP(&& (!DYNAMO_OPTION(hot_patching) ||
+                                  self_owns_write_lock(hotp_get_lock())))));
+        if (!ok) {
+            /* we no longer allow execution from arbitrary dr mem, our dll is
+             * on the executable list and we specifically add the callback
+             * interception code */
+            bool is_in_dr = is_dynamo_address(pc);
+            /* this is an unknown or dr area
+             * we may need to return false, if flags change or if pc is
+             * unreadable (and so we don't want to follow a direct cti there
+             * until the app actually does)
+             */
+            bool is_allocated_mem = get_memory_info(pc, &base_pc, &size, &prot);
+            bool is_being_unloaded = false;
+
+            if (!is_allocated_mem) {
+                /* case 9022 - Kaspersky sports JMPs to a driver in
+                 * kernel address space e.g. jmp f7ab7d67
+                 * and system call queries refuse to provide any information.
+                 * We need to just try reading from that address.
+                 */
+
+                /* we first compare to
+                 * SYSTEM_BASIC_INFORMATION.HighestUserAddress (2GB or
+                 * 3GB) to know for sure we're testing a kernel
+                 * address, and not dealing with a race instead.
+                 */
+
+                if (!is_user_address(pc) && 
+                    is_readable_without_exception_try(pc, 1)) {
+                    SYSLOG_INTERNAL_WARNING_ONCE(
+                       "Readable kernel address space memory at "PFX".\n"
+                       "case 9022 seen with Kaspersky AV", 
+                       pc);
+                    /* FIXME: we're constructing these flags with the
+                     * intent to allow this region, any other
+                     * characteristics are hard to validate
+                     */
+                    is_allocated_mem = true;
+                    base_pc = (app_pc)ALIGN_BACKWARD(pc, PAGE_SIZE);
+                    size = PAGE_SIZE;
+                    prot = MEMPROT_READ | MEMPROT_EXEC;
+
+                    /* FIXME: note we could also test for
+                     * MEMPROT_WRITE, note that explicitly turn on
+                     * SANDBOX_FLAG() anyways.  Luckily, the one known
+                     * case where this is needed doesn't leave its
+                     * driver space writable.
+                     */
+
+                    vm_flags |= VM_DRIVER_ADDRESS;
+                    /* we mark so that we can add to executable_areas
+                     * list later, and as in the only current example
+                     * so that we can allow execution.  FIXME: Note
+                     * we'll never remove this area.  We could check
+                     * on a future access whether such an address is
+                     * still readable, and then we can remove it if
+                     * the address stops being readable.  Note that we
+                     * can never tell if this area has disappeared -
+                     * since we won't get notified on memory changes.
+                     * So we may be more likely to get a decode fault
+                     * if these ever happen.
+                     */
+                    /* FIXME: we don't support this on Linux where
+                     * we'd have to also add to all_memory_areas */
+
+                    /* Note it is better to ALWAYS turn on
+                     * SANDBOX_FLAG for these fragments since it is
+                     * not clear that we can control any writes to
+                     * them from kernel space FIXME: may be
+                     * unnecessary in the case of Kaspersky.  
+                     * insert_selfmod_sandbox() will suppress
+                     * sandbox2ro_threshold for VM_DRIVER_ADDRESS areas
+                     */
+                    frag_flags |= SANDBOX_FLAG();
+                    /* FIXME: could do this under an option */
+                } else {
+                    /* just a bad address in kernel space - like 0xdeadbeef */
+                }
+            } else {
+                /* check for race where DLL is still present, but no
+                 * longer on our list.
+                 */
+                is_being_unloaded = is_unreadable_or_currently_unloaded_region(pc);
+                /* note here we'll forge an exception to the app, 
+                 * even if the address is practically still readable
+                 */
+                if (is_being_unloaded) {
+                    STATS_INC(num_unloaded_race_code_origins);
+                    SYSLOG_INTERNAL_WARNING_ONCE("Application executing from unloaded "
+                                                 "address "PFX"\n", pc);
+                }
+            }
+
+            /* if target unreadable, app will die, so make sure we don't die 
+             * instead, NOTE we treat dr memory as unreadable because of app
+             * races (see bug 2574) and the fact that we don't yet expect
+             * targeted attacks against dr */
+            /* case 9330 tracks a violation while we are unloading,
+             * but address shouldn't be on a new futureexec_area (case 9371)
+             */
+            if (is_in_dr || !is_allocated_mem || prot == 0/*no access flags*/ ||
+                is_being_unloaded) {
+                if (xfer) {
+                    /* don't follow cti, wait for app to get there and then 
+                     * handle this (might be pathological case where cti is 
+                     * never really followed)
+                     */
+
+                    /* Note for case 9330 that for direct xfer we want
+                     * to be able to recreate the scenario after we
+                     * stop.  Even though is_being_unloaded is a
+                     * transient property, since we treat unreadable
+                     * the same way, next time we get here we'll be
+                     * ok.  We already have to make sure we don't
+                     * missclassify futureexec_areas so can't really
+                     * get here.  Normal module unloads would have
+                     * flushed all other bb's.
+                     */
+
+                    LOG(THREAD, LOG_VMAREAS, 3,
+                        "cti targets %s "PFX", stopping bb here\n", 
+                        is_in_dr ? "dr" : "unreadable", pc);
+                    result = false;
+                    goto check_thread_return;
+                } else {
+                    /* generate sigsegv as though target application
+                     * instruction being decoded generated it
+                     */
+                    /* FIXME : might be pathalogical selfmod case where
+                     * app in fact jumps out of block before reaching the 
+                     * unreadable memory */
+                    if (vmlist == NULL) {
+                        /* Case 9376: check_origins_bb_pattern() can get here
+                         * w/ vmlist==NULL.  We have to be careful to free
+                         * resources of the prior vmlist and the vmarea write lock.
+                         */
+                        SYSLOG_INTERNAL_INFO("non-bb-build app decode found "
+                                             "unreadable memory");
+                    }
+                    LOG(GLOBAL, LOG_VMAREAS, 1,
+                        "application tried to execute from %s "PFX" is_allocated_mem=%d prot=0x%x\n",
+                        is_in_dr ? "dr" : "unreadable", pc, is_allocated_mem, prot);
+                    LOG(THREAD, LOG_VMAREAS, 1,
+                        "application tried to execute from %s "PFX" is_allocated_mem=%d prot=0x%x\n",
+                        is_in_dr ? "dr" : "unreadable", pc, is_allocated_mem, prot);
+                    DOLOG(1, LOG_VMAREAS, {
+                        dump_callstack((app_pc)get_mcontext(dcontext)->xsi,
+                                       (app_pc)get_mcontext(dcontext)->xbp, 
+                                       THREAD, DUMP_NOT_XML);
+                    });
+                    
+                    /* FIXME: what if the app masks it with an exception 
+                     * handler? */
+                    SYSLOG_INTERNAL_WARNING_ONCE(
+                        "Application tried to execute from %s memory "PFX".\n"
+                        "This may be a result of an unsuccessful attack or a potential "
+                        "application vulnerability.", is_in_dr ? "dr" : "unreadable",
+                        pc);
+                    /* Not logged as a security violation, but still an 
+                     * external warning, We don't want to take blame for all 
+                     * program bugs that overwrite EIP with invalid addresses, 
+                     * yet it may help discovering new security holes. 
+                     * [Although, watching for crashes of 0x41414141 can't read
+                     *  0x41414141 helps.]
+                     *It may also be a failing attack..
+                     */
+
+                    check_thread_vm_area_cleanup(dcontext, true/*abort*/,
+                                                 true/*clean bb*/, data, vmlist,
+                                                 own_execareas_writelock,
+                                                 caller_execareas_writelock);
+
+                    /* Create an exception record for this failure */
+                    if (TEST(DUMPCORE_FORGE_UNREAD_EXEC,
+                             DYNAMO_OPTION(dumpcore_mask)))
+                        os_dump_core("Warning: App trying to execute from unreadable memory");
+                    os_forge_exception(pc, UNREADABLE_MEMORY_EXECUTION_EXCEPTION);
+                    ASSERT_NOT_REACHED();
+                }
+            }
+
+            /* set all flags that don't intermix now */
+#ifdef PROGRAM_SHEPHERDING
+# ifdef WINDOWS
+            /* Don't classify the vsyscall code page as DGC for our purposes,
+             * since we permit execution from that region. This is needed
+             * for Windows XP/2003 pre-SP2 on which the code page is not
+             * part of ntdll.
+             * FIXME What about SP1?
+             * FIXME A better soln is to add the region to the exec list
+             * during os init and remove this specialized check.
+             */
+            if (!is_dyngen_vsyscall(pc))
+# endif
+                frag_flags |= FRAG_DYNGEN;
+#endif
+#ifdef WINDOWS
+            if ((prot & MEMPROT_WRITE) != 0 && is_on_stack(dcontext, pc, NULL)) {
+                /* On win32, kernel kills process if esp is bad,
+                 * doesn't even call KiUserExceptionDispatcher entry point!
+                 * Thus we cannot make this region read-only.
+                 * We must treat it as self-modifying code, and sandbox
+                 * the whole thing, to guarantee cache consistency.
+                 * FIXME: esp can point anywhere, so other regions we make
+                 * read-only may end up becoming "stack", and then we'll
+                 * just silently fail on a write there!!!
+                 */
+                frag_flags |= SANDBOX_FLAG();
+                STATS_INC(num_selfmod_vm_areas);
+            }
+#endif
+        }
+    }
+    if (area != NULL) {
+        ASSERT_CURIOSITY(vmlist == NULL || !TEST(VM_DELETE_ME, area->vm_flags));
+        if (vmlist != NULL && TEST(FRAG_COARSE_GRAIN, area->frag_flags)) {
+            /* We assume get_executable_area_coarse_info() is called prior to
+             * execution in a coarse region.  We go ahead and initialize here
+             * though we could wait if a xfer since the bb will not cross.
+             */
+            DEBUG_DECLARE(coarse_info_t *info =)
+                get_coarse_info_internal(pc, true/*init*/, true/*have shvm lock*/);
+            ASSERT(info != NULL);
+        }
+        ASSERT(!TEST(FRAG_COARSE_GRAIN, area->frag_flags) ||
+               get_coarse_info_internal(pc, false/*no init*/, false/*no lock*/) != NULL);
+        frag_flags |= area->frag_flags;
+
+#ifdef PROGRAM_SHEPHERDING
+        if (vmlist != NULL && /* only for bb building */
+            TEST(VM_PATTERN_REVERIFY, area->vm_flags) &&
+            !shared_to_private /* ignore shared-to-private conversion */) {
+            /* case 8168: sandbox2ro_threshold can turn into a non-sandboxed region,
+             * and our re-verify won't change that as the region is already on the
+             * executable list.  It will all work fine though.
+             */
+            ASSERT(DYNAMO_OPTION(sandbox2ro_threshold) > 0 ||
+                   TEST(FRAG_SELFMOD_SANDBOXED, area->frag_flags));
+            /* Re-verify the code origins policies, unless we are ensuring that
+             * the end of the pattern is ok.  This fixes case 4020 where another
+             * thread can use a pattern region for non-pattern code.
+             */
+            area = NULL; /* clear to force a re-verify */
+            /* satisfy lock asumptions when area == NULL */
+            if (!own_execareas_writelock) {
+# ifdef HOT_PATCHING_INTERFACE
+                if (DYNAMO_OPTION(hot_patching))
+                    write_lock(hotp_get_lock()); /* case 8780 -- see comments above */
+# endif
+                write_lock(&executable_areas->lock);
+                own_execareas_writelock = true;
+            }
+       }
+#endif
+    }
+
+    /* FIXME: fits nicely down below as alternative to marking read-only,
+     * but must be here for vm==NULL so will stop bb at cti -- although
+     * here it gets executed multiple times until actually switch to sandboxing
+     */
+    if (area == NULL && DYNAMO_OPTION(ro2sandbox_threshold) > 0 &&
+        TEST(MEMPROT_WRITE, prot) && !TEST(FRAG_SELFMOD_SANDBOXED, frag_flags)) {
+        vm_area_t *w_area; /* can't clobber area here */
+        ro_vs_sandbox_data_t *ro2s = NULL;
+        /* even though area==NULL this can still be an exec-writable area
+         * if area is sub-page!  we can't change to sandboxing w/ sub-page
+         * regions on the same page, so we wait until come here the 1st time
+         * after a flush (which will flush the whole os region).  thus, the
+         * threshold is really just a lower bound.  FIXME: add stats on this case!
+         */
+        ASSERT(own_execareas_writelock);
+#ifdef HOT_PATCHING_INTERFACE
+        ASSERT(!DYNAMO_OPTION(hot_patching) || self_owns_write_lock(hotp_get_lock()));
+#endif
+        ASSERT(self_owns_write_lock(&executable_areas->lock));
+        if (!is_executable_area_writable(pc)) { /* ok to read as a writer */
+            /* see whether this region has been cycling on and off the list due
+             * to being written to -- if so, switch to sandboxing
+             */
+            read_lock(&written_areas->lock);
+            ok = lookup_addr(written_areas, pc, &w_area);
+            if (ok)
+                ro2s = (ro_vs_sandbox_data_t *) w_area->custom.client;
+            if (ok && ro2s->written_count >=
+                DYNAMO_OPTION(ro2sandbox_threshold)) {
+                LOG(GLOBAL, LOG_VMAREAS, 1,
+                    "new executable area "PFX"-"PFX" written >= %dX => "
+                    "switch to sandboxing\n",
+                    base_pc, base_pc+size, DYNAMO_OPTION(ro2sandbox_threshold));
+                DOSTATS({
+                    if (vmlist != NULL) /* don't count non-build calls */
+                        STATS_INC(num_ro2sandbox);
+                });
+                /* TODO FOR PERFORMANCE:
+                 * -- if app appending to area of jitted code, make threshold big enough
+                 * so will get off page
+                 * -- modern jit shouldn't really have data on same page: all jitted
+                 * code should be combined
+                 * -- we're using OS regions b/c we merge ours, but if writer and writee
+                 * are on sep pages but in same OS region, we'll keep in cycle when we
+                 * could simply split region!  even if peel off written-to pages here,
+                 * (can't at flush time as must flush whole vm region)
+                 * if exec even once from target page, will add entire since we
+                 * merge, and will flush entire since flush bounds suggested by
+                 * OS regions (and must flush entire merged vmarea since that's
+                 * granularity of frags list).  still, worth splitting, even if
+                 * will merge back, to not lose perf if writee is on
+                 * never-executed page!  to impl, want another vm vector in
+                 * which, at flush time, we store bounds for next exec.
+                 */
+                frag_flags |= SANDBOX_FLAG();
+                /* for sandboxing best to stay at single-page regions */
+                base_pc = (app_pc) PAGE_START(pc);
+                size = PAGE_SIZE;
+                /* We do not clear the written count as we're only doing one page
+                 * here.  We want the next exec in the same region to also be
+                 * over the threshold.
+                 */
+                DODEBUG({ ro2s->ro2s_xfers++; });
+                LOG(GLOBAL, LOG_VMAREAS, 2,
+                    "\tsandboxing just the page "PFX"-"PFX"\n", base_pc, base_pc+size);
+            }
+            read_unlock(&written_areas->lock);
+        } else
+            STATS_INC(num_ro2sandbox_other_sub);
+    }
+
+    /* now that we know about new area, decide whether it's compatible to be
+     * in the same bb as previous areas, as dictated by old flags
+     * N.B.: we only care about FRAG_ flags here, not VM_ flags
+     */
+    if (xfer) {
+        uint src_cmp = *flags & frag_flags_cmp;
+        uint tgt_cmp = frag_flags & frag_flags_cmp;
+        bool allow = (src_cmp == tgt_cmp) ||
+            /* Case 8917: hack to allow elision of call* to vsyscall-in-ntdll,
+             * while still ruling out fine fragments coming in to coarse regions
+             * (where we'd rather stop the fine and build a (cheaper) coarse bb).
+             * Use == instead of TEST to rule out any other funny flags.
+             */
+            (src_cmp == 0 /* we removed FRAG_COARSE_GRAIN to make this fine */
+             && tgt_cmp == FRAG_COARSE_GRAIN /* still in coarse region though */
+             && TEST(FRAG_HAS_SYSCALL, *flags));
+        if (TEST(FRAG_COARSE_GRAIN, *flags)) {
+            /* FIXME case 8606: we can allow intra-module xfers but we have no
+             * way of checking here -- would have to check in
+             * interp.c:check_new_page_jmp().  So for now we disallow all xfers.
+             * If our regions match modules exactly we shouldn't see any
+             * intra-module direct xfers anyway.
+             */
+            /* N.B.: ibl entry removal (case 9636) assumes coarse fragments
+             * stay bounded within contiguous FRAG_COARSE_GRAIN regions
+             */
+            allow = false;
+        }
+        if (!allow) {
+            LOG(THREAD, LOG_VMAREAS, 3,
+                "change in vm area flags (0x%08x vs. 0x%08x %d): "
+                "stopping at "PFX"\n", *flags, frag_flags,
+                TEST(FRAG_COARSE_GRAIN, *flags), pc);
+            DOSTATS({
+                    if (TEST(FRAG_COARSE_GRAIN, frag_flags))
+                        STATS_INC(elisions_prevented_for_coarse);
+                });
+            result = false;
+            goto check_thread_return;
+        }
+    }
+
+    /* Normally we return the union of flags from all vmarea regions touched.
+     * But if one region is coarse and another fine, we do NOT want the union,
+     * but rather we want the whole thing to be fine.  FIXME: We could also try
+     * to put in functionality to truncate at the region boundary.
+     * Case 9932: in fact we cannot allow touching two adjacent coarse regions.
+     */
+    /* N.B.: ibl entry removal (case 9636) assumes coarse fragments
+     * stay bounded within a single FRAG_COARSE_GRAIN region
+     */
+    if (TEST(FRAG_COARSE_GRAIN, frag_flags) && pc != tag/*don't cmp to nothing*/ &&
+        ((*flags & FRAG_COARSE_GRAIN) != (frag_flags & FRAG_COARSE_GRAIN) ||
+         area == NULL || area->start > tag)) {
+        *flags &= ~FRAG_COARSE_GRAIN;
+        frag_flags &= ~FRAG_COARSE_GRAIN; /* else we'll re-add below */
+        DOSTATS({
+            if (vmlist != NULL)
+                STATS_INC(coarse_overlap_with_fine);
+        });
+    }
+
+    if (vmlist == NULL) {
+        /* caller only cared about whether to follow direct cti, so exit now, don't
+         * make any persistent state changes
+         */
+        *flags |= frag_flags;
+        if (stop != NULL) {
+            if (area == NULL)
+                *stop = base_pc + size;
+            else
+                *stop = area->end;
+        }
+        ASSERT(*stop != NULL);
+        result = true;
+        goto check_thread_return;
+    }
+    /* once reach this point we're building a real bb */
+
+#ifdef SIMULATE_ATTACK
+    simulate_attack(dcontext, pc);
+#endif /* SIMULATE_ATTACK */
+
+    if (area == NULL /* unknown area */) {
+        LOG(GLOBAL, LOG_VMAREAS, 2,
+            "WARNING: "PFX" -> "PFX"-"PFX" %s%s is not on executable list (thread %d)\n",
+            pc, base_pc, base_pc+size,
+            ((prot & MEMPROT_WRITE) != 0)?"W":"", ((prot & MEMPROT_EXEC) != 0)?"E":"",
+            dcontext->owning_thread);
+        DOLOG(3, LOG_VMAREAS, { print_executable_areas(GLOBAL); });
+        DODEBUG({
+            if (is_on_stack(dcontext, pc, NULL))
+                SYSLOG_INTERNAL_WARNING_ONCE("executing region with pc "PFX" on "
+                                             "the stack.", pc);
+        });
+#ifdef DGC_DIAGNOSTICS
+        dyngen_diagnostics(dcontext, pc, base_pc, size, prot);
+#endif
+
+#ifdef PROGRAM_SHEPHERDING
+        /* give origins checker a chance to change region 
+         * N.B.: security violation reports in detect_mode assume that at
+         * this point we aren't holding pointers into vectors, since the
+         * shared vm write lock is released briefly for the diagnostic report.
+         */
+        if (DYNAMO_OPTION(code_origins) &&
+            !shared_to_private) { /* don't check for shared-to-private conversion */
+            int res = check_origins(dcontext, pc, &base_pc, &size, prot, &vm_flags,
+                                    &frag_flags, xfer);
+            if (res < 0) {
+                if (!xfer) {
+                    action_type_t action =
+                        security_violation_main(dcontext, pc, res,
+                                                OPTION_BLOCK|OPTION_REPORT);
+                    if (action != ACTION_CONTINUE) {
+                        check_thread_vm_area_cleanup(dcontext, true/*abort*/,
+                                                     true/*clean bb*/, data, vmlist,
+                                                     own_execareas_writelock,
+                                                     caller_execareas_writelock);
+                        security_violation_action(dcontext, action, pc);
+                        ASSERT_NOT_REACHED();
+                    }
+                } else {
+                    /* if xfer, we simply don't follow the xfer */
+                    LOG(THREAD, LOG_VMAREAS, 3,
+                        "xfer to "PFX" => violation, so stopping at "PFX"\n",
+                        base_pc, pc);
+                    result = false;
+                    goto check_thread_return;
+                }
+            }
+        }
+#endif
+
+        /* make sure code is either read-only or selfmod sandboxed */
+        /* making unwritable and adding to exec areas must be atomic
+         * (another thread could get what would look like app seg fault in between!)
+         * and selfmod flag additions, etc. have restrictions, so we must have
+         * held the write lock the whole time
+         */
+        ASSERT(own_execareas_writelock);
+        ok = lookup_addr(executable_areas, pc, &area);
+        if (ok) {
+            LOG(GLOBAL, LOG_VMAREAS, 1,
+                "\tNew executable region is on page already added!\n");
+#ifdef FORENSICS_ACQUIRES_INITEXIT_LOCK
+            /* disabled until case 6141 is resolved: no lock release needed for now */
+            /* if we release the exec areas lock to emit forensic info, then
+             * someone else could have added the region since we checked above.
+             * see if we need to handle the DELAY_READONLY flag.
+             */
+            if (TEST(VM_DELAY_READONLY, area->vm_flags))
+                handle_delay_readonly(dcontext, pc, area);
+            else {
+#endif
+#ifdef PROGRAM_SHEPHERDING
+                /* else, this can only happen for pattern reverification: no races! */ 
+                ASSERT(TEST(VM_PATTERN_REVERIFY, area->vm_flags) &&
+                       TEST(FRAG_SELFMOD_SANDBOXED, area->frag_flags));
+#else
+                ASSERT_NOT_REACHED();
+#endif
+#ifdef FORENSICS_ACQUIRES_INITEXIT_LOCK
+            }
+#endif
+        } else {
+            /* need to add the region */
+            if (TEST(MEMPROT_WRITE, prot)) {
+                vm_flags |= VM_WRITABLE;
+                STATS_INC(num_writable_code_regions);
+                /* Now that new area bounds are finalized, see if it should be
+                 * selfmod.  Mainly this is a problem with a subpage region on the
+                 * same page as an existing subpage selfmod region.  We want the new
+                 * region to be selfmod to avoid forcing the old to switch to page
+                 * protection.  We won't have to do this once we separate the
+                 * consistency region list from the code origins list (case 3744):
+                 * then we'd have the whole page as selfmod on the consistency list,
+                 * with only the valid subpage on the origins list.  We don't mark
+                 * pieces of a large region, for simplicity.
+                 */
+                if (is_executable_area_on_all_selfmod_pages(base_pc, base_pc+size)) {
+                    frag_flags |= SANDBOX_FLAG();
+                }
+                /* case 8308: We've added options to force certain regions to
+                 * use selfmod instead of RO.  -sandbox_writable causes all writable
+                 * regions to be selfmod.  -sandbox_non_text causes all non-text
+                 * writable regions to be selfmod.
+                 */
+                else if (DYNAMO_OPTION(sandbox_writable)) {
+                    frag_flags |= SANDBOX_FLAG();
+                }
+                else if (DYNAMO_OPTION(sandbox_non_text)) {
+                    app_pc modbase = get_module_base(base_pc);
+                    if (modbase == NULL || !is_range_in_code_section
+                        (modbase, base_pc, base_pc + size, NULL, NULL)) {
+                        frag_flags |= SANDBOX_FLAG();
+                    }
+                }
+
+                if (TEST(FRAG_SELFMOD_SANDBOXED, frag_flags)) {
+                    LOG(GLOBAL, LOG_VMAREAS, 2,
+                        "\tNew executable region "PFX"-"PFX" is writable, but selfmod, "
+                        "so leaving as writable\n", base_pc, base_pc+size);
+                } else if (INTERNAL_OPTION(cache_consistency)) {
+                    /* Make entire region read-only
+                     * If that's too big, i.e., it contains some data, the
+                     * region size will be corrected when we get a write
+                     * fault in the region
+                     */
+                    LOG(GLOBAL, LOG_VMAREAS, 2,
+                        "\tNew executable region "PFX"-"PFX" is writable, "
+                        "making it read-only\n", base_pc, base_pc+size);
+#if 0
+                    /* this syslog causes services.exe to hang 
+                     * (ref case 666) once case 666 is fixed re-enable if 
+                     * desired FIXME */
+                    SYSLOG_INTERNAL_WARNING_ONCE("new executable vm area is writable.");
+#endif
+                    vm_make_unwritable(base_pc, size);
+                    vm_flags |= VM_MADE_READONLY;
+                    STATS_INC(num_rw2r_code_regions);
+                }
+            }
+            /* now add the new region to the global list */
+            ASSERT(!TEST(FRAG_COARSE_GRAIN, frag_flags)); /* else no pre-exec query */
+            add_executable_vm_area(base_pc, base_pc+size, vm_flags | VM_EXECUTED_FROM,
+                                   frag_flags, true/*own lock*/
+                                   _IF_DEBUG("unexpected vm area"));
+            ok = lookup_addr(executable_areas, pc, &area);
+            ASSERT(ok);
+            DOLOG(2, LOG_VMAREAS, {
+                /* new area could have been split into multiple */
+                print_contig_vm_areas(executable_areas, base_pc, base_pc+size,
+                                      GLOBAL, "new executable vm area: ");
+            });
+        }
+        ASSERT(area != NULL);
+        area_copy = *area;
+        area = &area_copy;
+    }
+    if (local_area == NULL) {
+        /* new area for this thread */
+        ASSERT(TEST(VM_EXECUTED_FROM, area->vm_flags)); /* marked above */
+#ifdef DGC_DIAGNOSTICS
+        if (!TESTANY(VM_UNMOD_IMAGE|VM_WAS_FUTURE, area->vm_flags)) {
+            LOG(GLOBAL, LOG_VMAREAS, 1,
+                "DYNGEN in %d: non-unmod-image exec area "PFX"-"PFX" %s\n",
+                get_thread_id(), area->start, area->end, area->comment);
+        }
+#endif
+#ifdef PROGRAM_SHEPHERDING
+        DOSTATS({
+            if (!TEST(VM_UNMOD_IMAGE, area->vm_flags) && TEST(VM_WAS_FUTURE, area->vm_flags)) {
+                /* increment for other threads (1st thread will be inc-ed in check_origins_helper) */
+                if (is_on_stack(dcontext, area->start, area)) {
+                    STATS_INC(num_exec_future_stack);
+                } else {
+                    STATS_INC(num_exec_future_heap);
+                }
+            }
+        });
+# ifdef WINDOWS
+        DOSTATS({
+            if (!TEST(VM_UNMOD_IMAGE, area->vm_flags) && !TEST(VM_WAS_FUTURE, area->vm_flags))
+                STATS_INC(num_exec_after_load);
+        });
+# endif
+#endif
+        
+        add_vm_area(&data->areas, area->start, area->end, area->vm_flags,
+                    area->frag_flags _IF_DEBUG(area->comment));
+        /* get area for actual pc (new area could have been split up) */
+        ok = lookup_addr(&data->areas, pc, &local_area);
+        ASSERT(ok);
+        DOLOG(2, LOG_VMAREAS, { print_vm_area(&data->areas, local_area,
+                                              THREAD, new_area_prefix); });
+        DOLOG(5, LOG_VMAREAS, { print_vm_areas(&data->areas, THREAD); });
+    }
+
+    ASSERT(local_area != NULL);
+    data->last_area = local_area;
+
+    /* for adding new bbs to frag lists */
+    if (tag != NULL) {
+        bool already = false;
+        fragment_t *entry, *prev;
+        /* see if this frag is already on this area's list.
+         * prev entry may not be first on list due to area merging or due to
+         * trace building that requires bb creation in middle.
+         */
+        /* vmlist has to point to front, so must walk every time
+         * along the way check to see if existing entry points to this area
+         */
+        for (entry = (fragment_t *) *vmlist, prev = NULL; entry != NULL;
+             prev = entry, entry = FRAG_ALSO(entry)) {
+            if (FRAG_PC(entry) >= local_area->start && FRAG_PC(entry) < local_area->end) {
+                already = true;
+                break;
+            }
+        }
+        if (!already) {
+            /* always allocate global, will re-allocate later if not shared */
+            prev = prepend_fraglist(MULTI_ALLOC_DC(dcontext,
+                       (data == shared_data) ? FRAG_SHARED : 0),
+                       local_area, pc, tag, prev);
+            ASSERT(FRAG_PREV(prev) != NULL);
+            if (*vmlist == NULL) {
+                /* write back first */
+                *vmlist = (void *) prev;
+            }
+        }
+        DOLOG(6, LOG_VMAREAS, {
+            print_fraglist(dcontext, local_area, "after check_thread_vm_area, ");
+        });
+        DOLOG(7, LOG_VMAREAS, { print_fraglists(dcontext); });
+    }
+
+    *flags |= frag_flags;
+    if (stop != NULL) {
+        *stop = area->end;
+        ASSERT(*stop != NULL);
+    }
+    result = true;
+    
+    /* we are building a real bb, assert consistency checks */
+    DODEBUG({
+        uint prot2;
+        ok = get_memory_info(pc, NULL, NULL, &prot2);
+        ASSERT(!ok || !TEST(MEMPROT_WRITE, prot2) || 
+               TEST(FRAG_SELFMOD_SANDBOXED, *flags));
+        ASSERT(is_readable_without_exception_try(pc, 1));
+    });
+
+ check_thread_return:
+    check_thread_vm_area_cleanup(dcontext, false/*not aborting*/,
+                                 false/*leave bb*/, data, vmlist,
+                                 own_execareas_writelock,
+                                 caller_execareas_writelock);
+    return result;
+}
+
+static void
+remove_fraglist_entry(dcontext_t *dcontext, fragment_t *entry, vm_area_t *area);
+
+/* page_pc must be aligned to the start of a page */
+void
+set_thread_decode_page_start(dcontext_t *dcontext, app_pc page_pc)
+{
+    thread_data_t *data;
+    /* Regardless of the dcontext that's passed in, we want to track the
+     * page_pc for the thread so get a real dcontext. */
+#ifdef LINUX
+    /* FIXME On Linux, fetching a context requires a syscall, which is a
+     * relatively costly operation, so we don't even try. Note that this can
+     * be misleading when the dcontext that's passed in isn't the one for
+     * the executing thread (such as in case 5388 on Windows).
+     */
+    if (dcontext == GLOBAL_DCONTEXT) {
+        ASSERT_CURIOSITY(dynamo_exited);
+        return;
+    }
+#else
+    dcontext = get_thread_private_dcontext();
+    if (dcontext == NULL) {
+        ASSERT_CURIOSITY(dynamo_exited);
+        return;
+    }
+#endif
+    data = (thread_data_t *) dcontext->vm_areas_field;
+    ASSERT(page_pc == (app_pc) PAGE_START(page_pc));
+    data->last_decode_area_page_pc = page_pc;
+    data->last_decode_area_valid = true;
+}
+
+/* Check if address is in the last area that passed the check_thread_vm_area tests.
+ * Used for testing for an application race condition (case 845),
+ * where code executed by one thread is unmapped by another.
+ * The last decoded application pc should always be in the thread's last area. 
+ */
+bool
+check_in_last_thread_vm_area(dcontext_t *dcontext, app_pc pc)
+{
+    thread_data_t *data = NULL;
+    bool in_last = false;
+    app_pc last_decode_area_page_pc;
+    /* extra paranoia since called by intercept_exception */
+    if (is_readable_without_exception((app_pc)&dcontext->vm_areas_field, 4)) 
+        data = (thread_data_t *) dcontext->vm_areas_field;
+    /* note that if data is NULL &data->last_area will not be readable either */
+    if (is_readable_without_exception((app_pc)&data->last_area, 4) &&
+        is_readable_without_exception((app_pc)&data->last_area->end, 4) &&
+        is_readable_without_exception((app_pc)&data->last_area->start, 4))
+        in_last = (pc < data->last_area->end && data->last_area->start <= pc);
+    /* last decoded app pc may be in last shared area instead */
+    if (!in_last && DYNAMO_OPTION(shared_bbs)) {
+        /* FIXME: bad to grab on failure path...
+         * can we assume only grabbed then, and not synch?
+         */
+        SHARED_VECTOR_RWLOCK(&shared_data->areas, read, lock);
+        if (is_readable_without_exception((app_pc)&shared_data->last_area->end, 4) &&
+            is_readable_without_exception((app_pc)&shared_data->last_area->start, 4))
+            in_last = (pc < shared_data->last_area->end && shared_data->last_area->start <= pc);
+        SHARED_VECTOR_RWLOCK(&shared_data->areas, read, unlock);
+    }
+    /* the last decoded app pc may be in the last decoded page or the page after
+     * if the instr crosses a page boundary. This can help us more gracefully
+     * handle a race during the origins pattern check between a thread unmapping
+     * a region and another thread decoding in that region (xref case 7103).
+     */
+    if (!in_last && data != NULL &&
+        safe_read(&data->last_decode_area_page_pc, sizeof(last_decode_area_page_pc),
+                  &last_decode_area_page_pc) &&
+        /* I think the above "safety" checks are ridiculous so not doing them here */
+        data->last_decode_area_valid) {
+        /* Check the last decoded pc's current page and the page after. */
+        app_pc last_decode_page_end = last_decode_area_page_pc + 2*PAGE_SIZE;
+        in_last = ((last_decode_page_end < last_decode_area_page_pc /* overflow */ ||
+                    pc < last_decode_page_end) && last_decode_area_page_pc <= pc);
+    }
+    return in_last;
+}
+
+/* Removes vmlist entries added to the global vmarea list for f.
+ * If new_vmlist != NULL, adds locally in addition to removing globally, and
+ * removes the global area itself if empty.
+ */
+static void
+remove_shared_vmlist(dcontext_t *dcontext, void *vmlist, fragment_t *f,
+                     void **local_vmlist)
+{
+    vm_area_t *area = NULL;
+    fragment_t *entry = (fragment_t *) vmlist;
+    fragment_t *next;
+    bool remove;
+    bool ok;
+    uint check_flags;
+    app_pc pc;
+    LOG(THREAD, LOG_VMAREAS, 4, "\tremoving shared vm data for F%d("PFX")\n",
+        f->id, f->tag);
+    SHARED_VECTOR_RWLOCK(&shared_data->areas, write, lock);
+    while (entry != NULL) {
+        ASSERT(FRAG_MULTI_INIT(entry));
+        ASSERT(FRAG_FRAG(entry) == (fragment_t *) f->tag); /* for this frag */
+        /* If area will become empty, remove it, since it was only added for
+         * this bb that is not actually shared.
+         * Case 8906: do NOT remove the area for coarse fragments, as they are
+         * still shared!  We need the area, just not the fragment on the frags
+         * list(s).
+         */
+        remove = (local_vmlist != NULL && FRAG_PREV(entry) == entry &&
+                  !TEST(FRAG_COARSE_GRAIN, f->flags));
+        if (remove) {
+            ok = lookup_addr(&shared_data->areas, FRAG_PC(entry), &area);
+            ASSERT(ok && area != NULL);
+            if (TEST(FRAG_COARSE_GRAIN, area->frag_flags)) {
+                /* Case 9806: do NOT remove the coarse area even if this
+                 * particular fragment is fine-grained.  We also test f->flags
+                 * up front to avoid the lookup cost as an optimization.
+                 */
+                remove = false;
+            } else {
+                LOG(THREAD, LOG_VMAREAS, 4,
+                    "sole fragment in added shared area, removing\n");
+            }
+        } else
+            area = NULL;
+        next = FRAG_ALSO(entry);
+        pc = FRAG_PC(entry);
+        remove_fraglist_entry(GLOBAL_DCONTEXT, entry, area /* ok to be NULL */);
+        if (remove) {
+            /* FIXME case 8629: lots of churn if frequent removals (e.g., coarse grain) */
+            remove_vm_area(&shared_data->areas, area->start, area->end, false);
+            shared_data->last_area = NULL;
+        }
+        if (local_vmlist != NULL) {
+            /* add area to local and add local heap also entry */
+            if (DYNAMO_OPTION(shared_bbs))
+                check_flags = f->flags | FRAG_SHARED; /*indicator to NOT use global*/
+            ok = check_thread_vm_area(dcontext, pc, f->tag, local_vmlist, &check_flags,
+                                      NULL, false /*xfer should not matter now*/);
+            ASSERT(ok);
+        }
+        entry = next;
+    }
+    SHARED_VECTOR_RWLOCK(&shared_data->areas, write, unlock);
+}
+
+void
+vm_area_add_fragment(dcontext_t *dcontext, fragment_t *f, void *vmlist)
+{
+    thread_data_t *data;
+    vm_area_t *area = NULL;
+    fragment_t *entry = (fragment_t *) vmlist;
+    fragment_t *prev = NULL;
+
+    LOG(THREAD, LOG_VMAREAS, 4, "vm_area_add_fragment for F%d("PFX")\n", f->id, f->tag);
+
+    if (TEST(FRAG_COARSE_GRAIN, f->flags)) {
+        /* We went ahead and built up vmlist since we might decide later to not
+         * make a fragment coarse-grain.  If it is emitted as coarse-grain,
+         * we need to clean up the vmlist as it is not needed.
+         */
+        remove_shared_vmlist(dcontext, vmlist, f, NULL/*do not add local*/);
+        return;        
+    }
+
+    if (TEST(FRAG_SHARED, f->flags)) {
+        data = shared_data;
+        /* need write lock since writing area->frags */
+        SHARED_VECTOR_RWLOCK(&shared_data->areas, write, lock);
+    } else if (!DYNAMO_OPTION(shared_bbs) ||
+               /* should already be in private vmareas */
+               TESTANY(FRAG_IS_TRACE | FRAG_TEMP_PRIVATE, f->flags))
+        data = (thread_data_t *) dcontext->vm_areas_field;
+    else {
+        void *local_vmlist = NULL;
+        /* turns out bb isn't shared, so we have to transfer also entries
+         * to local heap and vector.  we do that by removing from global
+         * and then calling check_thread_vm_area, telling it to add local.
+         */
+        ASSERT(dcontext != GLOBAL_DCONTEXT);
+        /* only bbs do we build shared and then switch to private */
+        ASSERT(!TEST(FRAG_IS_TRACE, f->flags));
+        data = (thread_data_t *) dcontext->vm_areas_field;
+        LOG(THREAD, LOG_VMAREAS, 4, "\tbb not shared, shifting vm data to thread-local\n");
+        remove_shared_vmlist(dcontext, vmlist, f, &local_vmlist);
+        /* now proceed as though everything were local to begin with */
+        vmlist = local_vmlist;
+        entry = (fragment_t *) vmlist;
+    }
+
+    /* swap f for the first multi_entry_t (the one in region of f->tag) */
+    ASSERT(entry != NULL);
+    FRAG_NEXT_ASSIGN(f, FRAG_NEXT(entry));
+    FRAG_PREV_ASSIGN(f, FRAG_PREV(entry));
+    FRAG_ALSO_ASSIGN(f, FRAG_ALSO(entry));
+    prev = FRAG_PREV(f);
+    ASSERT(prev != NULL); /* prev is never null */
+    if (FRAG_NEXT(prev) == NULL) {
+        DEBUG_DECLARE(bool ok =)
+            /* need to know area */
+            lookup_addr(&data->areas, FRAG_PC(entry), &area);
+        ASSERT(ok);
+        /* remember: prev wraps around, next does not */
+        ASSERT(area->custom.frags == entry);
+        area->custom.frags = f;
+        /* if single entry will be circular */
+        if (prev == entry)
+            FRAG_PREV_ASSIGN(f, f);
+    } else
+        FRAG_NEXT_ASSIGN(prev, f);
+    if (FRAG_NEXT(f) == NULL) {
+        if (area == NULL) {
+            DEBUG_DECLARE(bool ok =)
+                /* need to know area for area->frags */
+                lookup_addr(&data->areas, FRAG_PC(entry), &area);
+            ASSERT(ok);
+        }
+        if (area->custom.frags == f) {
+            ASSERT(FRAG_PREV(area->custom.frags) == f);
+        } else {
+            ASSERT(FRAG_PREV(area->custom.frags) == entry);
+            FRAG_PREV_ASSIGN(area->custom.frags, f);
+        }
+    } else {
+        prev = FRAG_NEXT(f);
+        FRAG_PREV_ASSIGN(prev, f);
+    }
+
+    prev = FRAG_ALSO(entry);
+    nonpersistent_heap_free(MULTI_ALLOC_DC(dcontext, entry->flags), entry,
+                            sizeof(multi_entry_t) HEAPACCT(ACCT_VMAREA_MULTI));
+    entry = prev;
+
+    DOSTATS({
+        if (entry != NULL)
+            STATS_INC(num_bb_also_vmarea);
+    });
+
+    /* now put backpointers in */
+    while (entry != NULL) {
+        ASSERT(FRAG_MULTI_INIT(entry));
+        ASSERT(FRAG_FRAG(entry) == (fragment_t *) f->tag); /* for this frag */
+        DOLOG(4, LOG_VMAREAS, { print_entry(dcontext, entry, "\talso "); });
+        FRAG_FRAG_ASSIGN(entry, f);
+        /* remove the init flag now that the real fragment_t is in the f field
+         * The vector lock protects this non-atomic flag change.
+         */
+        entry->flags &= ~FRAG_IS_EXTRA_VMAREA_INIT;
+        entry = FRAG_ALSO(entry);
+    }
+
+    DOLOG(6, LOG_VMAREAS, { print_frag_arealist(dcontext, f); });
+    DOLOG(7, LOG_VMAREAS, { print_fraglists(dcontext); });
+
+    /* can't release lock once done w/ prev/next values since alsos can 
+     * be changed as well by vm_area_clean_fraglist()!
+     */
+    SHARED_VECTOR_RWLOCK(&data->areas, write, unlock);
+}
+
+void
+acquire_vm_areas_lock(dcontext_t *dcontext, uint flags)
+{
+    thread_data_t *data = GET_DATA(dcontext, flags);
+    SHARED_VECTOR_RWLOCK(&data->areas, write, lock);
+}
+
+void
+release_vm_areas_lock(dcontext_t *dcontext, uint flags)
+{
+    thread_data_t *data = GET_DATA(dcontext, flags);
+    SHARED_VECTOR_RWLOCK(&data->areas, write, unlock);
+}
+
+/* Case 8419: this routine will fail and return false if f is marked as
+ * FRAG_WAS_DELETED, since that means f's also entries have been deleted!
+ * Caller can make an atomic no-fail region by holding f's vm area lock
+ * and the change_linking_lock and passing true for have_locks.
+ */
+bool
+vm_area_add_to_list(dcontext_t *dcontext, app_pc tag, void **vmlist,
+                    uint list_flags, fragment_t *f, bool have_locks)
+{
+    thread_data_t *src_data = GET_DATA(dcontext, f->flags);
+    thread_data_t *tgt_data = GET_DATA(dcontext, list_flags);
+    vm_area_t *area = NULL;
+    bool ok;
+    fragment_t *prev = (fragment_t *) *vmlist;
+    fragment_t *already;
+    fragment_t *entry = f;
+    bool success = true;
+    bool lock;
+    if (!have_locks)
+        SHARED_FLAGS_RECURSIVE_LOCK(f->flags, acquire, change_linking_lock);
+    else {
+        ASSERT((!TEST(VECTOR_SHARED, tgt_data->areas.flags) &&
+                !TEST(VECTOR_SHARED, src_data->areas.flags)) ||
+               self_owns_recursive_lock(&change_linking_lock));
+    }
+    /* support caller already owning write lock */
+    lock = writelock_if_not_already(&src_data->areas);
+    if (src_data != tgt_data) {
+        /* we assume only one of the two is shared, or that they are both the same,
+         * and we thus grab only one lock in this routine:
+         * otherwise we need to do more work to avoid deadlocks here!
+         */
+        ASSERT(!TEST(VECTOR_SHARED, tgt_data->areas.flags) ||
+               !TEST(VECTOR_SHARED, src_data->areas.flags));
+        if (TEST(VECTOR_SHARED, tgt_data->areas.flags)) {
+            ASSERT(!lock);
+            lock = writelock_if_not_already(&tgt_data->areas);
+        }
+    }
+    ASSERT((lock && !have_locks) || (!lock && have_locks) ||
+           (!TEST(VECTOR_SHARED, tgt_data->areas.flags) &&
+            !TEST(VECTOR_SHARED, src_data->areas.flags)));
+    /* If deleted, the also field is invalid and we cannot handle that! */
+    if (TEST(FRAG_WAS_DELETED, f->flags)) {
+        success = false;
+        goto vm_area_add_to_list_done;
+    }
+    /* vmlist has to point to front, so must walk every time to find end */
+    while (prev != NULL && FRAG_ALSO(prev) != NULL)
+        prev = FRAG_ALSO(prev);
+    /* walk f's areas */
+    while (entry != NULL) {
+        /* see if each of f's areas is already on trace's list */
+        ok = lookup_addr(&src_data->areas, FRAG_PC(entry), &area);
+        ASSERT(ok);
+        ok = false; /* whether found existing entry in area or not */
+        for (already = (fragment_t *) *vmlist; already != NULL;
+             already = FRAG_ALSO(already)) {
+            ASSERT(FRAG_MULTI(already));
+            if (FRAG_PC(already) >= area->start && FRAG_PC(already) < area->end) {
+                ok = true;
+                break;
+            }
+        }
+        if (!ok) {
+            /* found new area that trace is on */
+            /* src may be shared bb, its area may not be on tgt list (e.g., private trace) */
+            if (src_data != tgt_data) { /* else, have area already */
+                vm_area_t *tgt_area = NULL;
+                ok = lookup_addr(&tgt_data->areas, FRAG_PC(entry), &tgt_area);
+                if (!ok) {
+                    add_vm_area(&tgt_data->areas, area->start, area->end, area->vm_flags,
+                                area->frag_flags _IF_DEBUG(area->comment));
+                    ok = lookup_addr(&tgt_data->areas, FRAG_PC(entry), &tgt_area);
+                    ASSERT(ok);
+                    /* modified vector, must clear last_area */
+                    tgt_data->last_area = NULL;
+                    DOLOG(2, LOG_VMAREAS, {
+                        print_vm_area(&tgt_data->areas, tgt_area, THREAD,
+                                      "new vm area for thread: ");
+                    });
+                    DOLOG(5, LOG_VMAREAS, { print_vm_areas(&tgt_data->areas, THREAD); });
+                }
+                area = tgt_area;
+            }
+            ASSERT(area != NULL);
+            prev = prepend_fraglist(MULTI_ALLOC_DC(dcontext, list_flags),
+                                    area, FRAG_PC(entry), tag, prev);
+            if (*vmlist == NULL) {
+                /* write back first */
+                *vmlist = (void *) prev;
+            }
+        }
+        entry = FRAG_ALSO(entry);
+    }
+    DOLOG(6, LOG_VMAREAS, { print_frag_arealist(dcontext, (fragment_t *) *vmlist); });
+    DOLOG(7, LOG_VMAREAS, { print_fraglists(dcontext); });
+ vm_area_add_to_list_done:
+    if (lock) {
+        if (src_data != tgt_data)
+            SHARED_VECTOR_RWLOCK(&tgt_data->areas, write, unlock);
+        SHARED_VECTOR_RWLOCK(&src_data->areas, write, unlock);
+    }
+    if (!have_locks)
+        SHARED_FLAGS_RECURSIVE_LOCK(f->flags, release, change_linking_lock);
+    return success;
+}
+
+/* Frees storage for any multi-entries in the list (NOT for any fragment_t).
+ * FIXME: this is now used on bb abort, where we may want to remove a vmarea
+ * that was added only for a unreadable region (if decode fault will have been
+ * added already)!  Yet we don't know whether any coarse fragments in area,
+ * etc., so we go ahead and leave there: cached in last_area will lead to decode
+ * fault rather than explicit detection in check_thread_vm_area but that's ok.
+ * If we do want to remove should share code between this routine and
+ * remove_shared_vmlist().
+ */
+void
+vm_area_destroy_list(dcontext_t *dcontext, void *vmlist)
+{
+    vm_area_remove_fragment(dcontext, (fragment_t *)vmlist);
+}
+
+bool
+vm_list_overlaps(dcontext_t *dcontext, void *vmlist, app_pc start, app_pc end)
+{
+    vm_area_vector_t *v = GET_VECTOR(dcontext, ((fragment_t *)vmlist)->flags);
+    fragment_t *entry;
+    bool ok;
+    vm_area_t *area;
+    bool result = false;
+    LOG(THREAD, LOG_VMAREAS, 4, "vm_list_overlaps "PFX" vs "PFX"-"PFX"\n",
+        vmlist, start, end);
+    /* don't assert if can't find anything -- see usage in handle_modified_code() */
+    if (v == NULL)
+        return false;
+    SHARED_VECTOR_RWLOCK(v, read, lock);
+    for (entry = vmlist; entry != NULL; entry = FRAG_ALSO(entry)) {
+        ok = lookup_addr(v, FRAG_PC(entry), &area);
+        if (!ok)
+            break;
+        if (start < area->end && end > area->start) {
+            result = true;
+            break;
+        }
+    }
+    SHARED_VECTOR_RWLOCK(v, read, unlock);
+    return result;
+}
+
+/* Removes an entry from the fraglist of area.
+ * If area is NULL, looks it up based on dcontext->vm_areas_field->areas,
+ * or the shared areas, depending on entry.
+ * That lookup may need to be synchronized: this routine checks if the
+ * caller holds the write lock before grabbing it.
+ * If entry is a multi_entry_t, frees its heap
+ * DOES NOT update the also chain!
+ */
+static void
+remove_fraglist_entry(dcontext_t *dcontext, fragment_t *entry, vm_area_t *area)
+{
+    thread_data_t *data = GET_DATA(dcontext, entry->flags);
+    fragment_t *prev;
+    vm_area_vector_t *vector = &data->areas;
+    /* need write lock since may modify area->frags */
+    bool lock = writelock_if_not_already(vector);
+    /* entry is only in shared vector if still live -- if not
+     * we shouldn't get here
+     */
+    ASSERT(!TEST(VECTOR_SHARED, vector->flags) || !TEST(FRAG_WAS_DELETED, entry->flags));
+
+    prev = FRAG_PREV(entry);
+    if (FRAG_NEXT(prev) == NULL || FRAG_NEXT(entry) == NULL) {
+        /* need to know area */
+        DEBUG_DECLARE(bool ok =)
+            lookup_addr(vector, FRAG_PC(entry), &area);
+        ASSERT(ok);
+        ASSERT(area != NULL);
+    }
+
+    /* remember: prev wraps around, next does not */
+    if (FRAG_NEXT(prev) == NULL) {
+        ASSERT(area->custom.frags == entry);
+        area->custom.frags = FRAG_NEXT(entry);
+    } else {
+        FRAG_NEXT_ASSIGN(prev, FRAG_NEXT(entry));
+    }
+    if (FRAG_NEXT(entry) == NULL) {
+        if (area->custom.frags != NULL) {
+            ASSERT(FRAG_PREV(area->custom.frags) == entry);
+            FRAG_PREV_ASSIGN(area->custom.frags, FRAG_PREV(entry));
+        }
+    } else {
+        fragment_t *next = FRAG_NEXT(entry);
+        FRAG_PREV_ASSIGN(next, FRAG_PREV(entry));
+    }
+    /* next MUST be NULL-ed for fragment_remove_shared_no_flush() */
+    FRAG_NEXT_ASSIGN(entry, NULL);
+    DODEBUG({
+        FRAG_PREV_ASSIGN(entry, NULL);
+        FRAG_ALSO_ASSIGN(entry, NULL);
+    });
+    if (FRAG_MULTI(entry)) {
+        nonpersistent_heap_free(MULTI_ALLOC_DC(dcontext, entry->flags), entry,
+                                sizeof(multi_entry_t) HEAPACCT(ACCT_VMAREA_MULTI));
+    }
+    if (lock)
+        SHARED_VECTOR_RWLOCK(vector, write, unlock);
+}
+
+/* Removes redundant also entries in area's frags list
+ * (viz., those also entries that are now in same area as frag)
+ * Meant to be called after merging areas
+ */
+static void
+vm_area_clean_fraglist(dcontext_t *dcontext, vm_area_t *area)
+{
+    fragment_t *entry, *next, *f;
+    fragment_t *also, *also_prev, *also_next;
+    LOG(THREAD, LOG_VMAREAS, 4,
+        "vm_area_clean_fraglist for "PFX"-"PFX"\n", area->start, area->end);
+    DOLOG(6, LOG_VMAREAS, { print_fraglist(dcontext, area, "before cleaning "); });
+    /* FIXME: would like to assert we hold write lock but only have area ptr */
+    for (entry = area->custom.frags; entry != NULL; entry = next) {
+        next = FRAG_NEXT(entry); /* might delete entry */
+        /* Strategy: look at each multi, see if its fragment_t is here or if the next
+         * multi in also chain is here.
+         * This cleaning doesn't happen very often so this shouldn't be perf critical.
+         */
+        if (FRAG_MULTI(entry)) {
+            f = FRAG_FRAG(entry);
+            /* Remove later also entries first */
+            also = FRAG_ALSO(entry);
+            also_prev = entry;
+            while (also != NULL) {
+                app_pc pc = FRAG_PC(also);
+                also_next = FRAG_ALSO(also);
+                if (pc >= area->start && pc < area->end) {
+                    ASSERT(FRAG_FRAG(also) == f);
+                    DOLOG(5, LOG_VMAREAS, { print_entry(dcontext, also, "\tremoving "); });
+                    /* we have to remove from also chain ourselves */
+                    FRAG_ALSO_ASSIGN(also_prev, also_next);
+                    /* now remove from area frags list */
+                    remove_fraglist_entry(dcontext, also, area);
+                } else
+                    also_prev = also;
+                also = also_next;
+            }
+            /* fragment_t itself is always in area of its tag */
+            if (!FRAG_MULTI_INIT(entry) && f->tag >= area->start && f->tag < area->end) {
+                /* Remove this multi entry */
+                DOLOG(5, LOG_VMAREAS, { print_entry(dcontext, entry, "\tremoving "); });
+                /* we have to remove from also chain ourselves */
+                for (also_prev = f; FRAG_ALSO(also_prev) != entry; also_prev = FRAG_ALSO(also_prev))
+                    ;
+                FRAG_ALSO_ASSIGN(also_prev, FRAG_ALSO(entry));
+                /* now remove from area frags list */
+                remove_fraglist_entry(dcontext, entry, area);
+            }
+        }
+    }
+    DOLOG(6, LOG_VMAREAS, { print_fraglist(dcontext, area, "after cleaning "); });
+}
+
+void
+vm_area_remove_fragment(dcontext_t *dcontext, fragment_t *f)
+{
+    fragment_t *entry, *next, *match;
+    /* must grab lock across whole thing since alsos can be changed
+     * by vm_area_clean_fraglist()
+     */
+    vm_area_vector_t *vector = &(GET_DATA(dcontext, f->flags))->areas;
+    bool multi = FRAG_MULTI(f);
+    bool lock = writelock_if_not_already(vector);
+
+    if (!multi) {
+        LOG(THREAD, LOG_VMAREAS, 4,
+            "vm_area_remove_fragment: F%d tag="PFX"\n", f->id, f->tag);
+        match = f;
+    } else {
+        /* we do get called for multi-entries from vm_area_destroy_list */
+        LOG(THREAD, LOG_VMAREAS, 4,
+            "vm_area_remove_fragment: entry "PFX"\n", f);
+        match = FRAG_FRAG(f);
+    }
+    ASSERT(FRAG_PREV(f) != NULL); /* prev wraps around, should never be null */
+
+    entry = f;
+    while (entry != NULL) {
+        DOLOG(5, LOG_VMAREAS, { print_entry(dcontext, entry, "\tremoving "); });
+        /* from vm_area_destroy_list we can end up deleting a multi-init */
+        ASSERT(FRAG_FRAG(entry) == match);
+        next = FRAG_ALSO(entry);
+        remove_fraglist_entry(dcontext, entry, NULL);
+        entry = next;
+    }
+    if (!multi) /* else f may have been freed */
+        FRAG_ALSO_ASSIGN(f, NULL);
+    
+    DOLOG(7, LOG_VMAREAS, { print_fraglists(dcontext); });
+
+    /* f may no longer exist if it is FRAG_MULTI */
+    if (lock)
+        SHARED_VECTOR_RWLOCK(vector, write, unlock);
+}
+
+/* adds the fragment list chained by next_vmarea starting at f to a new
+ * pending deletion entry
+ */
+static void
+add_to_pending_list(dcontext_t *dcontext, fragment_t *f,
+                    uint refcount, uint flushtime
+                    _IF_DEBUG(app_pc start) _IF_DEBUG(app_pc end))
+{
+    pending_delete_t *pend;
+    ASSERT_OWN_MUTEX(true, &shared_delete_lock);
+    pend = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, pending_delete_t,
+                           ACCT_VMAREAS, PROTECTED);
+    DODEBUG({
+        pend->start = start;
+        pend->end = end;
+    });
+    pend->frags = f;
+    if (DYNAMO_OPTION(shared_deletion)) {
+        /* Set up ref count and timestamp for delayed deletion */
+        pend->ref_count = refcount;
+        pend->flushtime_deleted = flushtime;
+        LOG(GLOBAL, LOG_VMAREAS, 2,
+            "deleted area ref count=%d timestamp=%u start="PFX" end="PFX"\n",
+            pend->ref_count, pend->flushtime_deleted, start, end);
+    }
+    /* add to front of list */
+    pend->next = todelete->shared_delete;
+    todelete->shared_delete = pend;
+    todelete->shared_delete_count++;
+    if (pend->next == NULL) {
+        ASSERT(todelete->shared_delete_tail == NULL);
+        todelete->shared_delete_tail = pend;
+    }
+
+    if (DYNAMO_OPTION(reset_every_nth_pending) > 0 &&
+        DYNAMO_OPTION(reset_every_nth_pending) == todelete->shared_delete_count) {
+        /* if too many pending entries are piling up, suspend all threads
+         * in order to free them immediately.
+         * we can get here multiple times before we actually do the reset
+         * (can dec and then re-inc shared_delete_count),
+         * but that's not a problem, except we have to move our stats inc
+         * into the reset routine itself.
+         */
+        schedule_reset(RESET_PENDING_DELETION/*NYI: currently this is ignored and we
+                                              * do a full reset*/);
+    }
+
+    STATS_INC(num_shared_flush_regions);
+    LOG(GLOBAL, LOG_VMAREAS, 3,
+        "Pending list after adding deleted vm area:\n");
+    DOLOG(3, LOG_VMAREAS, { print_pending_list(GLOBAL); });
+}
+
+#if defined(DEBUG) && defined(INTERNAL)
+static void
+print_lazy_deletion_list(dcontext_t *dcontext, char *msg)
+{
+    uint i = 0;
+    fragment_t *f;
+    ASSERT_OWN_MUTEX(true, &lazy_delete_lock);
+    LOG(THREAD, LOG_VMAREAS, 1, "%s", msg);
+    for (f = todelete->lazy_delete_list; f != NULL; f = f->next_vmarea) {
+        LOG(THREAD, LOG_VMAREAS, 1, "\t%d: F%d ("PFX")\n", i, f->id, f->tag);
+        i++;
+    }
+}
+#endif
+
+#ifdef DEBUG
+static void
+check_lazy_deletion_list_consistency()
+{
+    uint i =0;
+    fragment_t *f;
+    ASSERT_OWN_MUTEX(true, &lazy_delete_lock);
+    for (f = todelete->lazy_delete_list; f != NULL; f = f->next_vmarea) {
+        i++;
+    }
+    ASSERT(i == todelete->lazy_delete_count);
+}
+#endif
+
+bool
+remove_from_lazy_deletion_list(dcontext_t *dcontext, fragment_t *remove)
+{
+    fragment_t *f, *prev_f = NULL;
+    mutex_lock(&lazy_delete_lock);
+    /* FIXME: start using prev_vmarea?!? (case 7165) */
+    for (f = todelete->lazy_delete_list; f != NULL; prev_f = f, f = f->next_vmarea) {
+        if (f == remove) {
+            if (prev_f == NULL)
+                todelete->lazy_delete_list = f->next_vmarea;
+            else
+                prev_f->next_vmarea = f->next_vmarea;
+            if (f == todelete->lazy_delete_tail)
+                todelete->lazy_delete_tail = prev_f;
+            todelete->lazy_delete_count--;
+            mutex_unlock(&lazy_delete_lock);
+            return true;
+        }
+    }
+    mutex_unlock(&lazy_delete_lock);
+    return false;
+}
+
+/* Moves all lazy list entries into a real pending deletion entry.
+ * Can only be called when !couldbelinking.
+ */
+static void
+move_lazy_list_to_pending_delete(dcontext_t *dcontext)
+{ 
+    ASSERT_OWN_NO_LOCKS();
+    ASSERT(is_self_couldbelinking());
+    /* to properly set up ref count we MUST get a flushtime synched with a
+     * thread count (otherwise we may have too many threads decrementing
+     * the ref count, or vice versa, causing either premature or
+     * never-occurring freeing), so we must grab thread_initexit_lock,
+     * meaning we must be nolinking, meaning the caller must accept loss
+     * of locals.
+     * FIXME: should switch to a flag-triggered addition in dispatch
+     * to avoid this nolinking trouble.
+     */
+    enter_nolinking(dcontext, NULL, false/*not a cache transition*/);
+    mutex_lock(&thread_initexit_lock);
+    /* to ensure no deletion queue checks happen in the middle of our update */
+    mutex_lock(&shared_cache_flush_lock);
+    mutex_lock(&shared_delete_lock);
+    mutex_lock(&lazy_delete_lock);
+    if (todelete->move_pending) {
+        /* it's possible for remove_from_lazy_deletion_list to drop the count */
+        DODEBUG({
+            if (todelete->lazy_delete_count <=
+                DYNAMO_OPTION(lazy_deletion_max_pending)) {
+                SYSLOG_INTERNAL_WARNING_ONCE("lazy_delete_count dropped below "
+                                             "threshold before move to pending");
+            }
+        });
+        LOG(THREAD, LOG_VMAREAS, 3, "moving lazy list to a pending deletion entry\n");
+        STATS_INC(num_lazy_del_to_pending);
+        STATS_ADD(num_lazy_del_frags_to_pending, todelete->lazy_delete_count);
+        /* ensure all threads in ref count will actually check the queue */
+        increment_global_flushtime();
+        add_to_pending_list(dcontext, todelete->lazy_delete_list,
+                            /* we do count this thread, as we aren't checking the
+                             * pending list here or inc-ing our flushtime
+                             */
+                            get_num_threads(), flushtime_global
+                            _IF_DEBUG(NULL) _IF_DEBUG(NULL));
+        todelete->lazy_delete_list = NULL;
+        todelete->lazy_delete_tail = NULL;
+        todelete->lazy_delete_count = 0;
+        todelete->move_pending = false;
+    } else /* should not happen */
+        ASSERT(false && "race in move_lazy_list_to_pending_delete");
+    DODEBUG({ check_lazy_deletion_list_consistency(); });
+    mutex_unlock(&lazy_delete_lock);
+    mutex_unlock(&shared_delete_lock);
+    mutex_unlock(&shared_cache_flush_lock);
+    mutex_unlock(&thread_initexit_lock);
+    enter_couldbelinking(dcontext, NULL, false/*not a cache transition*/);
+}
+
+/* adds the list of fragments beginning with f and chained by {next,prev}_vmarea
+ * to a new pending-lazy-deletion entry.
+ * This routine may become nolinking, meaning that fragments may be freed
+ * before this routine returns, so the caller should invalidate all pointers.
+ * It also means that no locks may be held by the caller!
+ */
+void
+add_to_lazy_deletion_list(dcontext_t *dcontext, fragment_t *f)
+{
+    /* rather than allocate memory for a pending operation to save memory,
+     * we re-use f->incoming_stubs's slot (via a union), which is no longer needed
+     * (caller should have already called incoming_remove_fragment()), to store our
+     * timestamp, and next_vmarea to chain.
+     */
+    fragment_t *tail, *prev = NULL;
+    uint flushtime;
+    bool perform_move = false;
+    ASSERT_OWN_NO_LOCKS();
+    ASSERT(is_self_couldbelinking());
+    mutex_lock(&shared_cache_flush_lock); /* for consistent flushtime */
+    mutex_lock(&lazy_delete_lock);
+    /* We need a flushtime as we are compared to shared deletion pending
+     * entries, but we don't need to inc flushtime_global.  We need a value
+     * larger than any thread has already signed off on, and thus larger than
+     * the current flushtime_global.  We hold shared_cache_flush_lock to ensure
+     * our flushtime retains that property until the lazy list is updated.
+     *
+     * (Optimization to allow lazy adds to proceed concurrently with deletion
+     * list checks: don't grab the shared_cache_flush_lock.  Since we're
+     * couldbelinking, the flusher won't inc flushtime until we're done here,
+     * and the lazy lock prevents other lazy adders from incing flushtime global
+     * for a shift to pending deletion list (in code below).  Then non-flusher
+     * must hold lazy lock in general to inc flushtime.).
+     */
+    ASSERT(flushtime_global < UINT_MAX);
+    /* currently we reset if flushtime hits a threshold -- in which case we
+     * may never reach this flushtime, but the reset if we hit threshold again,
+     * moving lazy entries to pending delete (below), and -reset_every_nth_pending
+     * combined should ensure we delete these fragments
+     */
+    flushtime = flushtime_global + 1;
+    /* we support adding a string of fragments at once 
+     * FIXME: if a string is common, move to a data structure w/ a
+     * single timestamp for a group of fragments -- though lazy_deletion_max_pending
+     * sort of does that for us.
+     */
+    /* must append to keep the list reverse-sorted by flushtime */
+    if (todelete->lazy_delete_list == NULL) {
+        ASSERT(todelete->lazy_delete_tail == NULL);
+        todelete->lazy_delete_list = f;
+    } else {
+        ASSERT(todelete->lazy_delete_tail->next_vmarea == NULL);
+        todelete->lazy_delete_tail->next_vmarea = f;
+    }
+    for (tail = f; tail != NULL; prev = tail, tail = tail->next_vmarea) {
+        ASSERT(tail->also.also_vmarea == NULL);
+        ASSERT(TEST(FRAG_SHARED, tail->flags));
+        tail->also.flushtime = flushtime;
+        todelete->lazy_delete_count++;
+    }
+    todelete->lazy_delete_tail = prev;
+    ASSERT(todelete->lazy_delete_tail != NULL);
+    LOG(THREAD, LOG_VMAREAS, 3, "adding F%d to lazy deletion list @ timestamp %u\n",
+        f->id, flushtime);
+    STATS_INC(num_lazy_deletion_appends);
+    DOLOG(5, LOG_VMAREAS, {
+        print_lazy_deletion_list(dcontext,
+                                 "Lazy deletion list after adding deleted fragment:\n");
+    });
+    DODEBUG({ check_lazy_deletion_list_consistency(); });
+    /* case 9115: ensure only one thread calls move_lazy_list_to_pending_delete,
+     * to reduce thread_initexit_lock contention and subsequent synch_with_all_threads
+     * performance issues
+     */
+    if (!todelete->move_pending &&
+        todelete->lazy_delete_count > DYNAMO_OPTION(lazy_deletion_max_pending)) {
+        perform_move = true;
+        todelete->move_pending = true;
+    }
+    mutex_unlock(&lazy_delete_lock);
+    mutex_unlock(&shared_cache_flush_lock);
+    if (perform_move) {
+        /* hit threshold -- move to real pending deletion entry */
+        /* had to release lazy_delete_lock and re-grab for proper rank order */
+        move_lazy_list_to_pending_delete(dcontext);
+    }
+}
+
+/* frees all fragments on the lazy list with flushtimes less than flushtime
+ */
+static void
+check_lazy_deletion_list(dcontext_t *dcontext, uint flushtime)
+{
+    fragment_t *f, *next_f;
+    mutex_lock(&lazy_delete_lock);
+    LOG(THREAD, LOG_VMAREAS, 3,
+        "checking lazy list @ timestamp %u\n", flushtime);
+    for (f = todelete->lazy_delete_list; f != NULL; f = next_f) {
+        next_f = f->next_vmarea; /* may be freed so cache now */
+        LOG(THREAD, LOG_VMAREAS, 4,
+            "\tf->id %u vs %u\n", f->id, f->also.flushtime, flushtime);
+        if (f->also.flushtime <= flushtime) {
+            /* it is safe to free! */
+            LOG(THREAD, LOG_VMAREAS, 3,
+                "freeing F%d on lazy deletion list @ timestamp %u\n",
+                f->id, flushtime);
+            DOSTATS({
+                if (dcontext == GLOBAL_DCONTEXT) /* at exit */
+                    STATS_INC(num_lazy_deletion_frees_atexit);
+                else
+                    STATS_INC(num_lazy_deletion_frees);
+            });
+            /* FIXME: separate stats for frees at exit time */
+            ASSERT(TEST(FRAG_SHARED, f->flags));
+            /* we assume we're freeing the entire head of the list */
+            todelete->lazy_delete_count--;
+            todelete->lazy_delete_list = next_f;
+            if (f == todelete->lazy_delete_tail) {
+                ASSERT(todelete->lazy_delete_list == NULL);
+                todelete->lazy_delete_tail = NULL;
+            }
+            fragment_delete(GLOBAL_DCONTEXT, f,
+                            FRAGDEL_NO_OUTPUT | FRAGDEL_NO_UNLINK |
+                            FRAGDEL_NO_HTABLE | FRAGDEL_NO_VMAREA);
+        } else {
+            /* the lazy list is appended to and thus reverse-sorted, so
+             * we can stop now as the oldest items are at the front
+             */
+            break;
+        }
+    }
+    DOLOG(5, LOG_VMAREAS, {
+        print_lazy_deletion_list(dcontext,
+                                 "Lazy deletion list after freeing fragments:\n");
+    });
+    DODEBUG({ check_lazy_deletion_list_consistency(); });
+    mutex_unlock(&lazy_delete_lock);
+}
+
+/* Prepares a list of shared fragments for deletion..
+ * Caller should have already called vm_area_remove_fragment() on
+ * each and chained them together via next_vmarea.
+ * Caller must hold the shared_cache_flush_lock.
+ * Returns the number of fragments unlinked 
+ */
+int
+unlink_fragments_for_deletion(dcontext_t *dcontext, fragment_t *list,
+                              int pending_delete_threads)
+{
+    fragment_t *f, *next;
+    uint num = 0;
+    /* only applies to lists of shared fragments -- we check the head now */
+    ASSERT(TEST(FRAG_SHARED, list->flags));
+    /* for shared_deletion we have to protect this whole walk w/ a lock so
+     * that the flushtime_global value remains higher than any thread's
+     * flushtime.
+     */
+    ASSERT_OWN_MUTEX(DYNAMO_OPTION(shared_deletion), &shared_cache_flush_lock);
+
+    acquire_recursive_lock(&change_linking_lock);
+    for (f = list; f != NULL; f = next) {
+        ASSERT(!FRAG_MULTI(f));
+        next = f->next_vmarea;
+        if (SHARED_IB_TARGETS()) {
+            /* Invalidate shared targets from all threads' ibl tables
+             * (if private) or from shared ibl tables.  Right now this
+             * routine is only called mid-flush so it's safe to do
+             * this here.
+             */
+            flush_invalidate_ibl_shared_target(dcontext, f);
+        }
+        fragment_unlink_for_deletion(dcontext, f);
+        num++;
+    }
+    release_recursive_lock(&change_linking_lock);
+
+    mutex_lock(&shared_delete_lock);
+    /* add area's fragments as a new entry in the pending deletion list */
+    add_to_pending_list(dcontext, list,
+                        pending_delete_threads, flushtime_global
+                        _IF_DEBUG(NULL) _IF_DEBUG(NULL));
+    mutex_unlock(&shared_delete_lock);
+    STATS_ADD(list_entries_unlinked_for_deletion, num);
+    return num;
+}
+
+/* returns the number of fragments unlinked */
+int
+vm_area_unlink_fragments(dcontext_t *dcontext, app_pc start, app_pc end,
+                         int pending_delete_threads
+                         _IF_DGCDIAG(app_pc written_pc))
+{
+    thread_data_t *data = GET_DATA(dcontext, 0);
+    fragment_t *entry, *next;
+    int num = 0, i;
+    if (data == shared_data) {
+        /* we also need to add to the deletion list */
+        mutex_lock(&shared_delete_lock);
+
+        acquire_recursive_lock(&change_linking_lock);
+
+        /* we do not need the bb building lock, only the vm lock and
+         * the fragment hashtable write lock, which is grabbed by fragment_remove
+         */
+        SHARED_VECTOR_RWLOCK(&data->areas, write, lock);
+
+        /* clear shared last_area now, don't want a new bb in flushed area
+         * thought to be ok b/c of a last_area hit
+         */
+        shared_data->last_area = NULL;
+
+        /* for shared_deletion we have to protect this whole walk w/ a lock so
+         * that the flushtime_global value remains higher than any thread's
+         * flushtime.
+         */
+        ASSERT_OWN_MUTEX(DYNAMO_OPTION(shared_deletion), &shared_cache_flush_lock);
+    }
+    
+    LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2, "vm_area_unlink_fragments "PFX".."PFX"\n",
+        start, end);
+
+    /* walk backwards to avoid O(n^2)
+     * FIXME case 9819: could use executable_area_overlap_bounds() to avoid linear walk
+     */
+    for (i = data->areas.length - 1; i >= 0; i--) {
+        /* look for overlap */
+        if (start < data->areas.buf[i].end && end > data->areas.buf[i].start) {
+            LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2,
+                "\tmarking region "PFX".."PFX" for deletion & unlinking all its frags\n",
+                data->areas.buf[i].start, data->areas.buf[i].end);
+            data->areas.buf[i].vm_flags |= VM_DELETE_ME;
+            if (data->areas.buf[i].start < start ||
+                data->areas.buf[i].end > end) {
+                /* FIXME: best to only delete within asked-for flush area
+                 * however, checking every fragment's bounds is way too expensive
+                 * (surprisingly).  we've gone through several different schemes,
+                 * including keeping a min_page and max_page in fragment_t, or
+                 * various multi-page flags, to make checking every fragment faster,
+                 * but keeping vm area lists is the most efficient.
+                 * HOWEVER, deleting outside the flush bounds can cause problems
+                 * if the caller holds fragment_t pointers and expects them not
+                 * to be flushed (e.g., a faulting write on a read-only code region).
+                 */
+                LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2,
+                    "\tWARNING: region "PFX".."PFX" is larger than "
+                    "flush area "PFX".."PFX"\n",
+                    data->areas.buf[i].start, data->areas.buf[i].end,
+                    start, end);                    
+            }
+            ASSERT(!TEST(FRAG_COARSE_GRAIN, data->areas.buf[i].frag_flags));
+            for (entry = data->areas.buf[i].custom.frags; entry != NULL; entry = next) {
+                fragment_t *f = FRAG_FRAG(entry);
+                next = FRAG_NEXT(entry);
+                /* case 9381: this shouldn't happen but we handle it to avoid crash */
+                if (FRAG_MULTI_INIT(entry)) {
+                    ASSERT(false && "stale multi-init entry on frags list");
+                    /* stale init entry, just remove it */
+                    vm_area_remove_fragment(dcontext, entry);
+                    continue;
+                }
+                /* case 9118: call fragment_unlink_for_deletion() even if fragment
+                 * is already unlinked 
+                 */
+                if (!TEST(FRAG_WAS_DELETED, f->flags) || data == shared_data) {
+                    LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 5,
+                        "\tunlinking "PFX"%s F%d("PFX")\n",
+                        entry, FRAG_MULTI(entry) ? " multi": "", FRAG_ID(entry),
+                        FRAG_PC(entry));
+                    /* need to remove also entries from other vm lists 
+                     * thread-private doesn't have to do this b/c only unlinking,
+                     * so ok if encounter an also in same flush, except we
+                     * now do incoming_remove_fragment() for thread-private for
+                     * use of fragment_t.incoming_stubs as a union.  so we
+                     * do this for all fragments.
+                     */
+                    if (FRAG_ALSO(entry) != NULL || FRAG_MULTI(entry)) {
+                        if (FRAG_MULTI(entry)) {
+                            vm_area_remove_fragment(dcontext, f);
+                            /* move to this area's frags list so will get
+                             * transfered to deletion list if shared, or
+                             * freed from this marked-vmarea if private
+                             */
+                            prepend_entry_to_fraglist(&data->areas.buf[i], f);
+                        } else {
+                            /* entry is the fragment, remove all its alsos */
+                            vm_area_remove_fragment(dcontext, FRAG_ALSO(entry));
+                        }
+                        FRAG_ALSO_ASSIGN(f, NULL);
+                    }
+                    if (data == shared_data && SHARED_IB_TARGETS()) {
+                        /* Invalidate shared targets from all threads' ibl
+                         * tables (if private) or from shared ibl tables
+                         */
+                        flush_invalidate_ibl_shared_target(dcontext, f);
+                    }
+                    fragment_unlink_for_deletion(dcontext, f);
+#ifdef DGC_DIAGNOSTICS
+                    /* try to find out exactly which fragment contained written_pc */
+                    if (written_pc != NULL) {
+                        app_pc bb;
+                        DOLOG(2, LOG_VMAREAS, {
+                            LOG(THREAD, LOG_VMAREAS, 1, "Flushing F%d "PFX":\n",
+                                FRAG_ID(entry), FRAG_PC(entry));
+                            disassemble_fragment(dcontext, entry, false);
+                            LOG(THREAD, LOG_VMAREAS, 1, "First app bb for frag:\n");
+                            disassemble_app_bb(dcontext, FRAG_PC(entry), THREAD);
+                        });
+                        if (fragment_overlaps(dcontext, entry, written_pc,
+                                              written_pc+1, false, NULL, &bb)) {
+                            LOG(THREAD, LOG_VMAREAS, 1,
+                                "Write target is actually inside app bb @"PFX":\n",
+                                written_pc);
+                            disassemble_app_bb(dcontext, bb, THREAD);
+                        }
+                    }
+#endif
+                    num++;
+                } else {
+                    LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 5,
+                        "\tnot unlinking "PFX"%s F%d("PFX") (already unlinked)\n",
+                        entry, FRAG_MULTI(entry) ? " multi": "", FRAG_ID(entry), FRAG_PC(entry));
+                }
+                /* let recreate_fragment_ilist() know that this fragment
+                 * is pending deletion and might no longer match the app's
+                 * state. note that if we called fragment_unlink_for_deletion()
+                 * then we already set this flag above. */
+                f->flags |= FRAG_WAS_DELETED;
+            }
+            DOLOG(6, LOG_VMAREAS, {
+                print_fraglist(dcontext, &data->areas.buf[i], "Fragments after unlinking\n");
+            });
+            if (data == shared_data) {
+                if (data->areas.buf[i].custom.frags != NULL) {
+                    /* add area's fragments as a new entry in the pending deletion list */
+                    add_to_pending_list(dcontext, data->areas.buf[i].custom.frags,
+                                        pending_delete_threads, flushtime_global
+                                        _IF_DEBUG(data->areas.buf[i].start)
+                                        _IF_DEBUG(data->areas.buf[i].end));
+                    /* frags are moved over completely */
+                    data->areas.buf[i].custom.frags = NULL;
+                    STATS_INC(num_shared_flush_regions);
+                }
+
+                /* ASSUMPTION: remove_vm_area, given exact bounds, simply shifts later
+                 * areas down in vector!
+                 */
+                LOG(THREAD, LOG_VMAREAS, 3, "Before removing vm area:\n");
+                DOLOG(3, LOG_VMAREAS, { print_vm_areas(&data->areas, THREAD); });
+                LOG(THREAD, LOG_VMAREAS, 2, "Removing shared vm area "PFX"-"PFX"\n",
+                    data->areas.buf[i].start, data->areas.buf[i].end);
+                remove_vm_area(&data->areas, data->areas.buf[i].start,
+                               data->areas.buf[i].end, false);
+                LOG(THREAD, LOG_VMAREAS, 3, "After removing vm area:\n");
+                DOLOG(3, LOG_VMAREAS, { print_vm_areas(&data->areas, THREAD); });
+            }
+        }
+    }
+    
+    if (data == shared_data) {
+        SHARED_VECTOR_RWLOCK(&data->areas, write, unlock);
+        release_recursive_lock(&change_linking_lock);
+        mutex_unlock(&shared_delete_lock);
+    }
+    
+    LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2, "  Unlinked %d frags\n", num);
+    return num;
+}
+
+/* removes incoming links for all private fragments in the dcontext
+ * thread that contain 'pc' 
+ */
+void
+vm_area_unlink_incoming(dcontext_t *dcontext, app_pc pc)
+{
+    int i;
+    thread_data_t *data;
+
+    ASSERT(dcontext != GLOBAL_DCONTEXT);
+    data = GET_DATA(dcontext, 0);
+
+    for (i = data->areas.length - 1; i >= 0; i--) {
+        if (pc >= data->areas.buf[i].start &&
+            pc <  data->areas.buf[i].end) {
+
+            fragment_t *entry;
+            for (entry = data->areas.buf[i].custom.frags; 
+                 entry != NULL; entry = FRAG_NEXT(entry)) {
+                fragment_t *f = FRAG_FRAG(entry);
+                ASSERT(!TEST(FRAG_SHARED, f->flags));
+
+                /* Note that we aren't unlinking or ibl-invalidating
+                 * (i.e., making unreachable) any fragments in other
+                 * threads containing pc.
+                 */
+                if ((f->flags & FRAG_LINKED_INCOMING) != 0)
+                    unlink_fragment_incoming(dcontext, f);
+                fragment_remove_from_ibt_tables(dcontext, f, false);
+            }
+        }
+    }
+}
+
+/* Decrements ref counts for thread-shared pending-deletion fragments,
+ * and deletes those whose count has reached 0.
+ * If dcontext==GLOBAL_DCONTEXT, does NOT check the ref counts and assumes it's
+ * safe to free EVERYTHING.
+ * Returns false iff was_I_flushed has been flushed (not necessarily
+ * fully freed yet though, but may be at any time after this call
+ * returns, so caller should drop its ref to it).
+ */
+bool
+vm_area_check_shared_pending(dcontext_t *dcontext, fragment_t *was_I_flushed)
+{
+    pending_delete_t *pend;
+    pending_delete_t *pend_prev = NULL;
+    pending_delete_t *pend_nxt;
+    /* a local list used to arrange in reverse order of flushtime */
+    pending_delete_t *tofree = NULL;
+    fragment_t *entry, *next;
+    int num = 0;
+    DEBUG_DECLARE(int i = 0;)
+    bool not_flushed = true;
+    ASSERT(DYNAMO_OPTION(shared_deletion) || dynamo_exited);
+    /* must pass in real dcontext, unless exiting or resetting */
+    ASSERT(dcontext != GLOBAL_DCONTEXT || dynamo_exited || dynamo_resetting);
+
+    LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2, 
+        "thread %d (flushtime %d) walking pending deletion list (was_I_flushed==F%d)\n",
+        get_thread_id(), dcontext == GLOBAL_DCONTEXT ? flushtime_global :
+        get_flushtime_last_update(dcontext),
+        (was_I_flushed==NULL) ? -1 : was_I_flushed->id);
+    STATS_INC(num_shared_flush_walks);
+
+    /* synch w/ anyone incrementing flushtime_global and using its
+     * value when adding to the shared deletion list (currently flushers
+     * and lazy list transfers).
+     */
+    mutex_lock(&shared_cache_flush_lock);
+        
+    /* check if was_I_flushed has been flushed, prior to dec ref count and
+     * allowing anyone to be fully freed
+     */
+    if (was_I_flushed != NULL &&
+        TESTALL(FRAG_SHARED|FRAG_WAS_DELETED, was_I_flushed->flags)) {
+        not_flushed = false;
+        if (was_I_flushed == dcontext->last_fragment)
+            last_exit_deleted(dcontext);
+    }
+    /* we can hit check points before we re-enter the cache, so we cannot
+     * rely on the enter_couldbelinking of exiting the cache for invalidating
+     * last_fragment -- we must check here as well (case 7453) (and case 7666,
+     * where a non-null was_I_flushed prevented this check from executing).
+     */
+    if (dcontext != GLOBAL_DCONTEXT && dcontext->last_fragment != NULL &&
+        TESTALL(FRAG_SHARED|FRAG_WAS_DELETED, dcontext->last_fragment->flags)) {
+        last_exit_deleted(dcontext);
+    }
+
+    mutex_lock(&shared_delete_lock);
+    for (pend = todelete->shared_delete; pend != NULL; pend = pend_nxt) {
+        bool delete_area = false;
+        pend_nxt = pend->next;
+        LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2,
+            "  Considering #%d: "PFX".."PFX" flushtime %d\n",
+            i, pend->start, pend->end, pend->flushtime_deleted);
+        if (dcontext == GLOBAL_DCONTEXT) {
+            /* indication that it's safe to free everything */
+            delete_area = true;
+            if (dynamo_exited)
+                STATS_INC(num_shared_flush_atexit);
+            else
+                STATS_INC(num_shared_flush_atreset);
+        } else if (get_flushtime_last_update(dcontext) < pend->flushtime_deleted) {
+            ASSERT(pend->ref_count > 0);
+            pend->ref_count--;
+            STATS_INC(num_shared_flush_refdec);
+            LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2,
+                "\tdec => ref_count is now %d, flushtime diff is %d\n",
+                pend->ref_count, flushtime_global - pend->flushtime_deleted);
+            delete_area = (pend->ref_count == 0);
+            DODEBUG({
+                if (INTERNAL_OPTION(detect_dangling_fcache) && delete_area) {
+                    /* don't actually free fragments until exit so we can catch any
+                     * lingering links or ibt entries
+                     */
+                    delete_area = false;
+                    for (entry = pend->frags; entry != NULL; entry = FRAG_NEXT(entry)) {
+                        /* we do have to notify caller of flushing held ptrs */
+                        if (FRAG_FRAG(entry) == was_I_flushed)
+                            ASSERT(!not_flushed); /* should have been caught up top */
+                        /* catch any links or ibt entries allowing access to deleted
+                         * fragments by filling w/ int3 instead of reusing the cache
+                         * space.  this will show up as a pc translation assert,
+                         * typically.
+                         */
+                        /* should only get fragment_t here */
+                        ASSERT(!FRAG_MULTI(entry));
+                        LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 4,
+                            "\tfilling F%d "PFX"-"PFX" with 0x%x\n",
+                            entry->id, entry->start_pc, entry->start_pc+entry->size,
+                            DEBUGGER_INTERRUPT_BYTE);
+                        memset(entry->start_pc, DEBUGGER_INTERRUPT_BYTE, entry->size);
+                    }
+                }
+            });
+            DOSTATS({ 
+                if (delete_area)
+                    STATS_INC(num_shared_flush_refzero);
+            });
+        } else {
+            /* optimization: since we always pre-pend, can skip all the rest, as
+             * they are guaranteed to have been ok-ed by us already
+             */
+            LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2,
+                "\t(aborting now since rest have already been ok-ed)\n");
+            break;
+        }
+
+        if (delete_area) {
+            /* we want to delete in increasing order of flushtime so that
+             * fcache unit flushing will not occur before all lazily deleted
+             * fragments in a unit are freed
+             */
+            if (pend_prev == NULL)
+                todelete->shared_delete = pend->next;
+            else
+                pend_prev->next = pend->next;
+            if (pend == todelete->shared_delete_tail) {
+                ASSERT(pend->next == NULL);
+                todelete->shared_delete_tail = pend_prev;
+            }
+            pend->next = tofree;
+            tofree = pend;
+        } else
+            pend_prev = pend;
+        DODEBUG({ i++; });
+    }
+
+    for (pend = tofree; pend != NULL; pend = pend_nxt) {
+        pend_nxt = pend->next;
+
+        /* we now know that any objects unlinked at or before this entry's
+         * timestamp are safe to be freed (although not all earlier objects have
+         * yet been freed, so containers cannot necessarily be freed: case 8242).
+         * free these before this entry's fragments as they are older
+         * (fcache unit flushing relies on this order).
+         */
+        check_lazy_deletion_list(dcontext, pend->flushtime_deleted);
+        
+        STATS_TRACK_MAX(num_shared_flush_maxdiff,
+                        flushtime_global - pend->flushtime_deleted);
+        DOSTATS({
+            /* metric: # times flushtime diff is > #threads */
+            if (flushtime_global - pend->flushtime_deleted > (uint) get_num_threads())
+                STATS_INC(num_shared_flush_diffthreads);
+        });
+        LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2,
+            "\tdeleting all fragments in region "PFX".."PFX" flushtime %u\n",
+            pend->start, pend->end, pend->flushtime_deleted);
+        ASSERT(pend->frags != NULL);
+        for (entry = pend->frags; entry != NULL; entry = next) {
+            next = FRAG_NEXT(entry);
+            LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 5,
+                "\tremoving "PFX"%s F%d("PFX")\n",
+                entry, FRAG_MULTI(entry) ? " multi": "", FRAG_ID(entry), FRAG_PC(entry));
+            if (FRAG_FRAG(entry) == was_I_flushed)
+                ASSERT(!not_flushed); /* should have been caught up top */
+#ifdef NATIVE_RETURN
+            fragment_add_deleted(GLOBAL_DCONTEXT, FRAG_FRAG(entry));
+#endif
+            /* vm_area_unlink_fragments should have removed all multis/alsos */
+            ASSERT(!FRAG_MULTI(entry));
+            /* FRAG_ALSO is used by lazy list so it may not be NULL */
+            ASSERT(TEST(FRAG_WAS_DELETED, FRAG_FRAG(entry)->flags));
+            /* do NOT call vm_area_remove_fragment, as it will freak out trying
+             * to look up the area this fragment is in
+             */
+            fragment_delete(GLOBAL_DCONTEXT, FRAG_FRAG(entry),
+                            FRAGDEL_NO_OUTPUT | FRAGDEL_NO_UNLINK |
+                            FRAGDEL_NO_HTABLE | FRAGDEL_NO_VMAREA);
+            STATS_INC(num_fragments_deleted_consistency);
+            num++;
+        }
+
+        ASSERT(todelete->shared_delete_count > 0);
+        todelete->shared_delete_count--;
+        HEAP_TYPE_FREE(GLOBAL_DCONTEXT, pend, pending_delete_t,
+                       ACCT_VMAREAS, PROTECTED);
+    }
+
+    if (tofree != NULL) { /* if we freed something (careful: tofree is dangling) */
+        /* case 8242: due to -syscalls_synch_flush, a later entry can
+         * reach refcount 0 before an earlier entry, so we cannot free
+         * units until all earlier entries have been freed.
+         */
+        if (todelete->shared_delete_tail == NULL)
+            fcache_free_pending_units(dcontext, flushtime_global);
+        else {
+            fcache_free_pending_units(dcontext,
+                                      todelete->shared_delete_tail->flushtime_deleted
+                                      - 1);
+        }
+    }
+    
+    if (dcontext == GLOBAL_DCONTEXT) { /* need to free everything */
+        check_lazy_deletion_list(dcontext, flushtime_global+1);
+        fcache_free_pending_units(dcontext, flushtime_global+1);
+        /* reset_every_nth_pending relies on this */
+        ASSERT(todelete->shared_delete_count == 0);
+    }
+    mutex_unlock(&shared_delete_lock);
+    STATS_TRACK_MAX(num_shared_flush_maxpending, i);
+
+    /* last_area cleared in vm_area_unlink_fragments */
+    LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2, 
+        "thread %d done walking pending list @flushtime %d\n",
+        get_thread_id(), flushtime_global);
+    if (dcontext != GLOBAL_DCONTEXT) {
+        /* update thread timestamp */
+        set_flushtime_last_update(dcontext, flushtime_global);
+    }
+    mutex_unlock(&shared_cache_flush_lock);
+
+    LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2, "  Flushed %d frags\n", num);
+    return not_flushed;
+}
+
+/* Deletes all pending-delete thread-private vm areas belonging to dcontext.
+ * Returns false iff was_I_flushed ends up being deleted.
+ */
+bool
+vm_area_flush_fragments(dcontext_t *dcontext, fragment_t *was_I_flushed)
+{
+    thread_data_t *data = GET_DATA(dcontext, 0);
+    vm_area_vector_t *v = &data->areas;
+    fragment_t *entry, *next;
+    int i, num = 0;
+    bool not_flushed = true;
+    /* should call vm_area_check_shared_pending for shared flushing */
+    ASSERT(data != shared_data);
+
+    LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2, "vm_area_flush_fragments\n");
+    /* walk backwards to avoid O(n^2) */
+    for (i = v->length - 1; i >= 0; i--) {
+        LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2,
+            "  Considering %d == "PFX".."PFX"\n", i,
+            v->buf[i].start, v->buf[i].end);
+        if (TEST(VM_DELETE_ME, v->buf[i].vm_flags)) {
+            LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2,
+                "\tdeleting all fragments in region "PFX".."PFX"\n",
+                v->buf[i].start, v->buf[i].end);
+            for (entry = v->buf[i].custom.frags; entry != NULL; entry = next) {
+                next = FRAG_NEXT(entry);
+                LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 5,
+                    "\tremoving "PFX"%s F%d("PFX")\n",
+                    entry, FRAG_MULTI(entry) ? " multi": "", FRAG_ID(entry), FRAG_PC(entry));
+                if (FRAG_FRAG(entry) == was_I_flushed) {
+                    not_flushed = false;
+                    if (was_I_flushed == dcontext->last_fragment)
+                        last_exit_deleted(dcontext);
+                }
+#ifdef NATIVE_RETURN
+                fragment_add_deleted(dcontext, FRAG_FRAG(entry));
+#endif
+                ASSERT(TEST(FRAG_WAS_DELETED, FRAG_FRAG(entry)->flags));
+                ASSERT(FRAG_ALSO_DEL_OK(entry) == NULL);
+                fragment_delete(dcontext, FRAG_FRAG(entry),
+                                /* We used to leave link, vmarea, and htable removal
+                                 * until here for private fragments, but for case
+                                 * 3559 we wanted link removal at unlink time, and
+                                 * the 3 of them must go together, so we now do all 3
+                                 * at unlink time just like for shared fragments.
+                                 */
+                                FRAGDEL_NO_OUTPUT | FRAGDEL_NO_UNLINK |
+                                FRAGDEL_NO_HTABLE | FRAGDEL_NO_VMAREA);
+                STATS_INC(num_fragments_deleted_consistency);
+                num++;
+            }
+            v->buf[i].custom.frags = NULL;
+            /* could just remove flush region...but we flushed entire vm region 
+             * ASSUMPTION: remove_vm_area, given exact bounds, simply shifts later
+             * areas down in vector!
+             */
+            LOG(THREAD, LOG_VMAREAS, 3, "Before removing vm area:\n");
+            DOLOG(3, LOG_VMAREAS, { print_vm_areas(v, THREAD); });
+            remove_vm_area(v, v->buf[i].start, v->buf[i].end, false);
+            LOG(THREAD, LOG_VMAREAS, 3, "After removing vm area:\n");
+            DOLOG(3, LOG_VMAREAS, { print_vm_areas(v, THREAD); });
+        }
+    }
+
+#ifdef WINDOWS
+    /* The relink needs a real thread dcontext, so don't pass a GLOBAL_DCONTEXT
+     * in. This can occur when flushing shared fragments. Functionally, this is
+     * fine since only private fragments are routed thru shared syscall, and
+     * flush requests for such fragments are provided with a real thread
+     * context.
+     */
+    if (DYNAMO_OPTION(shared_syscalls) && dcontext != GLOBAL_DCONTEXT &&
+        !IS_SHARED_SYSCALL_THREAD_SHARED) {
+        /* re-link shared syscall */
+        link_shared_syscall(dcontext);
+    }
+#endif
+
+    data->last_area = NULL;
+
+    DOSTATS({
+        if (num == 0)
+            STATS_INC(num_flushq_actually_empty);
+    });
+    LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2, "  Flushed %d frags\n", num);
+    DOLOG(7, LOG_VMAREAS, { 
+        SHARED_VECTOR_RWLOCK(&data->areas, read, lock);
+        print_fraglists(dcontext);
+        SHARED_VECTOR_RWLOCK(&data->areas, read, unlock);
+    });
+
+    return not_flushed;
+}
+
+/* Flushes all units grouped with info.
+ * Caller must hold change_linking_lock, read lock hotp_get_lock(), and
+ * executable_areas lock.
+ */
+static void
+vm_area_flush_coarse_unit(dcontext_t *dcontext, coarse_info_t *info_in,
+                          vm_area_t *area, bool all_synched, bool entire)
+{
+    coarse_info_t *info = info_in, *next_info;
+    ASSERT(info != NULL);
+    ASSERT_OWN_MUTEX(true, &change_linking_lock);
+#ifdef HOT_PATCHING_INTERFACE
+    ASSERT_OWN_READWRITE_LOCK(DYNAMO_OPTION(hot_patching), hotp_get_lock());
+#endif
+    ASSERT(READ_LOCK_HELD(&executable_areas->lock));
+    /* Need a real dcontext for persisting rac */
+    if (dcontext == GLOBAL_DCONTEXT)
+        dcontext = get_thread_private_dcontext();
+    if (DYNAMO_OPTION(coarse_freeze_at_unload)) {
+        /* we do not try to freeze if we've failed to suspend the world */
+        if (all_synched) {
+            /* in-place builds a separate unit anyway so no savings that way */
+            vm_area_coarse_region_freeze(dcontext, info, area, false/*!in place*/);
+            STATS_INC(persist_unload_try);
+        } else {
+            SYSLOG_INTERNAL_WARNING_ONCE("not freezing due to synch failure");
+            STATS_INC(persist_unload_suspend_failure);
+        }
+    }
+    while (info != NULL) { /* loop over primary and secondary unit */
+        next_info = info->non_frozen;
+        ASSERT(info->frozen || info->non_frozen == NULL);
+        if (!entire && TEST(PERSCACHE_CODE_INVALID, info->flags)) {
+            /* Do not reset yet as it may become valid again.
+             * Assumption: if !entire, we will leave this info there.
+             */
+            /* Should only mark invalid if no or empty secondary unit */
+            ASSERT(next_info == NULL || next_info->cache == NULL);
+            break;
+        }
+        DOSTATS({
+            if (info->persisted) {
+                STATS_INC(flush_persisted_units);
+                if (os_module_get_flag(info->base_pc, MODULE_BEING_UNLOADED))
+                    STATS_INC(flush_persisted_unload);
+            }
+            STATS_INC(flush_coarse_units);
+        });
+        coarse_unit_reset_free(dcontext, info, false/*no locks*/, true/*unlink*/,
+                               true/*give up primary*/);
+        /* We only want one non-frozen unit per region; we keep the 1st unit */
+        if (info != info_in) {
+            coarse_unit_free(GLOBAL_DCONTEXT, info);
+            info = NULL;
+        } else
+            coarse_unit_mark_in_use(info); /* still in-use if re-used */
+        /* The remaining info itself is freed from exec list in remove_vm_area,
+         * though may remain if only part of this region is removed
+         * and will be lazily re-initialized if we execute from there again.
+         * FIXME: case 8640: better to remove it all here?
+         */
+        info = next_info;
+        ASSERT(info == NULL || !info->frozen);
+    }
+}
+
+/* Assumes that all threads are suspended at safe synch points.
+ * Flushes fragments in the region [start, end) in the vmarea
+ * list for del_dcontext.
+ * If dcontext == del_dcontext == GLOBAL_DCONTEXT, 
+ *   removes shared fine fragments and coarse units in the region.
+ * If dcontext == thread and del_dcontext == GLOBAL_DCONTEXT,
+ *   removes any ibl table entries for shared fragments in the region.
+ *   WARNING: this routine will not remove coarse ibl entries!
+ * Else (both dcontexts are the local thread's), deletes private fragments
+ *   in the region.
+ * FIXME: share code w/ vm_area_unlink_fragments() and vm_area_flush_fragments()!
+ * all_synched is ignored unless dcontext == GLOBAL_DCONTEXT
+ */
+void
+vm_area_allsynch_flush_fragments(dcontext_t *dcontext, dcontext_t *del_dcontext,
+                                 app_pc start, app_pc end, bool exec_invalid,
+                                 bool all_synched)
+{
+    thread_data_t *data = GET_DATA(del_dcontext, 0);
+    vm_area_vector_t *v = &data->areas;
+    fragment_t *entry, *next;
+    int i;
+    bool remove_shared_vm_area = true;
+    DEBUG_DECLARE(int num_fine = 0;)
+    DEBUG_DECLARE(int num_coarse = 0;)
+
+    LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2,
+        "vm_area_allsynch_flush_fragments "PFX" "PFX"\n",
+        dcontext, del_dcontext);
+    ASSERT(OWN_MUTEX(&all_threads_synch_lock) && OWN_MUTEX(&thread_initexit_lock));
+    ASSERT(is_self_allsynch_flushing());
+
+    /* change_linking_lock is higher ranked than shared_vm_areas lock and is
+     * acquired for fragment_delete()'s unlinking as well as fcache removal to
+     * add to free list, so we must grab it up front.
+     * coarse_unit_persist and coarse_unit_freeze also require it to be held.
+     */
+    acquire_recursive_lock(&change_linking_lock);
+
+    if (dcontext == GLOBAL_DCONTEXT && del_dcontext == GLOBAL_DCONTEXT) {
+        /* We can't add persisted units to shared vector at load time due to
+         * lock rank orders, so we normally add on first access -- but we can
+         * flush before any access, so we must walk exec areas here.
+         * While we're at it we do our coarse unit freeing here, so don't have
+         * to do lookups in exec areas while walking shared vmarea vector below.
+         */
+#ifdef HOT_PATCHING_INTERFACE
+        if (DYNAMO_OPTION(hot_patching))
+            read_lock(hotp_get_lock()); /* case 9970: rank hotp < exec_areas */
+#endif
+        read_lock(&executable_areas->lock); /* no need to write */
+        for (i = 0; i < executable_areas->length; i++) {
+            if (TEST(FRAG_COARSE_GRAIN, executable_areas->buf[i].frag_flags) &&
+                start < executable_areas->buf[i].end &&
+                end > executable_areas->buf[i].start) {
+                coarse_info_t *coarse =
+                    (coarse_info_t *) executable_areas->buf[i].custom.client;
+                bool do_flush = (coarse != NULL);
+#ifdef HOT_PATCHING_INTERFACE
+                /* Case 9995: do not flush for 1-byte (mostly hotp) regions that are
+                 * still valid execution regions and that are recorded as not being
+                 * present in persistent caches.
+                 */
+                if (do_flush && !exec_invalid &&
+                    start + 1 == end && coarse->hotp_ppoint_vec != NULL) {
+                    app_pc modbase = get_module_base(coarse->base_pc);
+                    ASSERT(modbase <= start);
+                    /* Only persisted units store vec, though we could store for
+                     * frozen but not persisted if we had frequent nudges throwing
+                     * them out. */
+                    ASSERT(coarse->persisted);
+                    if (hotp_ppoint_on_list((app_rva_t)(start - modbase),
+                                            coarse->hotp_ppoint_vec,
+                                            coarse->hotp_ppoint_vec_num)) {
+                        do_flush = false;
+                        STATS_INC(perscache_hotp_flush_avoided);
+                        remove_shared_vm_area = false;
+                    }
+                }
+#endif
+                if (do_flush) {
+                    vm_area_flush_coarse_unit(dcontext, coarse,
+                                              &executable_areas->buf[i], all_synched,
+                                              start <= executable_areas->buf[i].start &&
+                                              end >= executable_areas->buf[i].end);
+                    DODEBUG({ num_coarse++; });
+                    if (TEST(VM_ADD_TO_SHARED_DATA, executable_areas->buf[i].vm_flags)) {
+                        LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2,
+                            "\tdeleting coarse unit not yet in shared vector "
+                            PFX".."PFX"\n",
+                            executable_areas->buf[i].start,
+                            executable_areas->buf[i].end);
+                        /* This flag is only relevant for persisted units, so we clear it
+                         * here since this same coarse_info_t may be re-used
+                         */
+                        executable_areas->buf[i].vm_flags &= ~VM_ADD_TO_SHARED_DATA;
+                    }
+                }
+            }
+        }
+        read_unlock(&executable_areas->lock);
+#ifdef HOT_PATCHING_INTERFACE
+        if (DYNAMO_OPTION(hot_patching))
+            read_unlock(hotp_get_lock());
+#endif
+    }
+
+    SHARED_VECTOR_RWLOCK(v, write, lock);
+    /* walk backwards to avoid O(n^2)
+     * FIXME case 9819: could use executable_area_overlap_bounds() to avoid linear walk
+     */
+    for (i = v->length - 1; i >= 0; i--) {
+        if (start < v->buf[i].end && end > v->buf[i].start) {
+            if (v->buf[i].start < start ||
+                v->buf[i].end > end) {
+                /* see comments in vm_area_unlink_fragments() */
+                LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2,
+                    "\tWARNING: region "PFX".."PFX" is larger than flush area"
+                    " "PFX".."PFX"\n",
+                    v->buf[i].start, v->buf[i].end, start, end);                    
+            }
+            LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2,
+                "\tdeleting all fragments in region "PFX".."PFX"\n",
+                v->buf[i].start, v->buf[i].end);
+            /* We flush coarse units in executable_areas walk down below */
+            /* We can have fine fragments here as well */
+            if (v->buf[i].custom.frags != NULL) {
+                for (entry = v->buf[i].custom.frags; entry != NULL; entry = next) {
+                    next = FRAG_NEXT(entry);
+                    if (dcontext == del_dcontext) {
+                        LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 5,
+                            "\tremoving "PFX"%s F%d("PFX")\n",
+                            entry, FRAG_MULTI(entry) ? " multi": "", FRAG_ID(entry),
+                            FRAG_PC(entry));
+#ifdef NATIVE_RETURN
+                        fragment_add_deleted(dcontext, FRAG_FRAG(entry));
+#endif
+                        if (SHARED_IBT_TABLES_ENABLED()) {
+                            /* fragment_remove() won't remove from shared ibt tables,
+                             * b/c assuming we didn't do the synch for it, so we
+                             * have to explicitly remove
+                             */
+                            fragment_remove_from_ibt_tables(dcontext, FRAG_FRAG(entry),
+                                                            true/*rm from shared*/);
+                        }
+                        fragment_delete(dcontext, FRAG_FRAG(entry), FRAGDEL_ALL);
+                        STATS_INC(num_fragments_deleted_consistency);
+                        DODEBUG({ num_fine++; });
+                    } else {
+                        ASSERT(dcontext != GLOBAL_DCONTEXT &&
+                               del_dcontext == GLOBAL_DCONTEXT);
+                        fragment_remove_from_ibt_tables(dcontext, FRAG_FRAG(entry),
+                                                        false/*shouldn't be in shared*/);
+                    }
+                }
+                if (dcontext == del_dcontext)
+                    v->buf[i].custom.frags = NULL;
+            }
+            if (dcontext == del_dcontext && remove_shared_vm_area) {
+                /* could just remove flush region...but we flushed entire vm region 
+                 * ASSUMPTION: remove_vm_area, given exact bounds, simply shifts later
+                 * areas down in vector!
+                 */
+                LOG(THREAD, LOG_VMAREAS, 3, "Before removing vm area:\n");
+                DOLOG(3, LOG_VMAREAS, { print_vm_areas(v, THREAD); });
+                remove_vm_area(v, v->buf[i].start, v->buf[i].end, false);
+                LOG(THREAD, LOG_VMAREAS, 3, "After removing vm area:\n");
+                DOLOG(3, LOG_VMAREAS, { print_vm_areas(v, THREAD); });
+            } else {
+                ASSERT(dcontext != del_dcontext ||
+                       /* should only not flush for special hotp case 9995 */
+                       start + 1 == end);
+            }
+        }
+    }
+
+    if (dcontext == del_dcontext)
+        data->last_area = NULL;
+    SHARED_VECTOR_RWLOCK(v, write, unlock);
+    release_recursive_lock(&change_linking_lock);
+
+    LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2,
+        "  Flushed %d fine frags & %d coarse units\n", num_fine, num_coarse);
+    DOLOG(7, LOG_VMAREAS, { 
+        SHARED_VECTOR_RWLOCK(v, read, lock);
+        print_fraglists(dcontext);
+        SHARED_VECTOR_RWLOCK(v, read, unlock);
+    });
+}
+
+/* Deletes all coarse units */
+void
+vm_area_coarse_units_reset_free()
+{
+    vm_area_vector_t *v = executable_areas;
+    int i;
+    ASSERT(DYNAMO_OPTION(coarse_units));
+    LOG(GLOBAL, LOG_FRAGMENT|LOG_VMAREAS, 2, "vm_area_coarse_units_reset_free\n");
+    ASSERT(dynamo_exited || dynamo_resetting);
+    DOLOG(1, LOG_VMAREAS, {
+        LOG(GLOBAL, LOG_VMAREAS, 1, "\nexecutable_areas before reset:\n");
+        print_executable_areas(GLOBAL);
+    });
+    /* We would grab executable_areas_lock but coarse_unit_reset_free() grabs
+     * change_linking_lock and coarse_info_lock, both of higher rank.  We could
+     * grab change_linking_lock first here and raise executable_areas_lock above
+     * coarse_info_lock's rank, but executable_areas_lock can be acquired during
+     * coarse_unit_unlink after special_heap_lock -- so the best solution is to
+     * not grab executable_areas_lock here and rely on reset synch.
+     */
+    for (i = 0; i < v->length; i++) {
+        if (TEST(FRAG_COARSE_GRAIN, v->buf[i].frag_flags)) {
+            coarse_info_t *info_start = (coarse_info_t *) v->buf[i].custom.client;
+            coarse_info_t *info = info_start, *next_info;
+            ASSERT(info != NULL);
+            while (info != NULL) { /* loop over primary and secondary unit */
+                next_info = info->non_frozen;
+                ASSERT(info->frozen || info->non_frozen == NULL);
+                LOG(GLOBAL, LOG_FRAGMENT|LOG_VMAREAS, 2,
+                    "\tdeleting all fragments in region "PFX".."PFX"\n",
+                    v->buf[i].start, v->buf[i].end);
+                coarse_unit_reset_free(GLOBAL_DCONTEXT, info, false/*no locks*/,
+                                       true/*unlink*/, true/*give up primary*/);
+                /* We only want one non-frozen unit per region; we keep the 1st one */
+                if (info != info_start) {
+                    coarse_unit_free(GLOBAL_DCONTEXT, info);
+                    info = NULL;
+                }
+                /* The start info itself is freed in remove_vm_area, if exiting */
+                /* FIXME case 9686: should re-load persisted caches after reset */
+                info = next_info;
+                ASSERT(info == NULL || !info->frozen);
+            }
+        }
+    }
+}
+
+/* Returns true if info && info->non_frozen meet the size requirements 
+ * for persisting.
+ */
+static bool
+coarse_region_should_persist(dcontext_t *dcontext, coarse_info_t *info)
+{
+    bool cache_large_enough = false;
+    size_t cache_size = 0;
+    /* Must hold lock to get size but ok for size to change afterward;
+     * normal usage has all threads synched */
+    if (!info->persisted) {
+        mutex_lock(&info->lock);
+        cache_size += coarse_frozen_cache_size(dcontext, info);
+        mutex_unlock(&info->lock);
+    }
+    if (info->non_frozen != NULL) {
+        mutex_lock(&info->non_frozen->lock);
+        cache_size += coarse_frozen_cache_size(dcontext, info->non_frozen);
+        mutex_unlock(&info->non_frozen->lock);
+    }
+    LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2,
+        "\tconsidering persisting coarse unit %s with cache size %d\n",
+        info->module, cache_size);
+    /* case 10107: check for disk space before freezing, if persisting.
+     * A crude estimate is all we need up front (we'll do a precise check at file
+     * write time): estimate that hashtables, stubs, etc. double cache size.
+     */
+    if (!coarse_unit_check_persist_space(INVALID_FILE, cache_size * 2)) {
+        LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2,
+            "\tnot enough disk space for %s\n", info->module);
+        STATS_INC(coarse_units_persist_nospace);
+        return false;
+    }
+    cache_large_enough =
+        (cache_size > DYNAMO_OPTION(coarse_freeze_min_size) ||
+         (info->persisted &&
+          /* FIXME: should use append size if merging only w/ disk as well */
+          cache_size > DYNAMO_OPTION(coarse_freeze_append_size)));
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+    /* Real cost is in pages touched while walking reloc, which is
+     * typically 80% of module.
+     */
+    if (rct_module_live_entries(dcontext, info->base_pc, RCT_RCT)
+        > DYNAMO_OPTION(coarse_freeze_rct_min)) {
+        DOSTATS({
+            if (!cache_large_enough)
+                STATS_INC(persist_code_small);
+        });
+        LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2,
+            "\tRCT entries are over threshold so persisting %s\n", info->module);
+        return true;
+    }
+#endif /* defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH) */
+    DOSTATS({
+        if (!cache_large_enough) {
+            LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2,
+                "\tnot persisting %s since too small\n", info->module);
+            STATS_INC(persist_too_small);
+        }
+    });
+    return cache_large_enough;
+}
+
+/* FIXME case 9975: we should provide separate control over persistence
+ * (today we assume !in_place==persist) so we can persist and use in_place
+ * rather than having to wait until next run to get the benefit.
+ */
+/* FIXME: if we map in a newly persisted unit we need to set
+ * VM_PERSISTED_CACHE, but we only care about it in executable_areas.
+ */
+/* Caller must hold change_linking_lock, read lock hotp_get_lock(), and
+ * either executable_areas lock or dynamo_all_threads_synched.
+ */
+static void
+vm_area_coarse_region_freeze(dcontext_t *dcontext, coarse_info_t *info,
+                             vm_area_t *area, bool in_place)
+{
+    coarse_info_t *frozen_info = NULL; /* the already-frozen info */
+    coarse_info_t *unfrozen_info = NULL; /* the un-frozen info */
+    if (!DYNAMO_OPTION(coarse_enable_freeze) || RUNNING_WITHOUT_CODE_CACHE())
+        return;
+    ASSERT(!RUNNING_WITHOUT_CODE_CACHE());
+    ASSERT(info != NULL);
+    ASSERT_OWN_MUTEX(true, &change_linking_lock);
+#ifdef HOT_PATCHING_INTERFACE
+    ASSERT_OWN_READWRITE_LOCK(DYNAMO_OPTION(hot_patching), hotp_get_lock());
+#endif
+    ASSERT(READ_LOCK_HELD(&executable_areas->lock) || dynamo_all_threads_synched);
+    /* Note that freezing in place will call mark_executable_area_coarse_frozen and
+     * add a new unit, so next_info should not be traversed after freezing.
+     */
+    if (info->frozen) {
+        frozen_info = info;
+        unfrozen_info = info->non_frozen;
+    } else {
+        unfrozen_info = info;
+        ASSERT(info->non_frozen == NULL);
+    }
+    if (unfrozen_info != NULL &&
+        unfrozen_info->cache != NULL /*skip empty units*/ &&
+        !TEST(PERSCACHE_CODE_INVALID, unfrozen_info->flags) &&
+         /* we only freeze a unit in presence of a frozen unit if we're merging
+          * (we don't support side-by-side frozen units) */
+        (DYNAMO_OPTION(coarse_freeze_merge) || frozen_info == NULL)) {
+        if (in_place || coarse_region_should_persist(dcontext, info)) {
+            coarse_info_t *frozen;
+            coarse_info_t *premerge;
+            LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2,
+                "\tfreezing coarse unit for region "PFX".."PFX" %s\n",
+                info->base_pc, info->end_pc, info->module);
+            if (frozen_info != NULL && in_place) {
+                /* We're freezing unfrozen_info, merging frozen_info into it, and
+                 * then deleting frozen_info, so we need to replace it with just
+                 * unfrozen_info (soon to be frozen); we do it this way since
+                 * mark_executable_area_coarse_frozen assumes being-frozen info is
+                 * the 1st info.
+                 */
+                area->custom.client = (void *) unfrozen_info;
+            }                
+            frozen = coarse_unit_freeze(dcontext, unfrozen_info, in_place);
+            ASSERT(frozen != NULL && frozen->frozen);
+            /* mark_executable_area_coarse_frozen creates new non_frozen for in_place */
+            ASSERT(!in_place || frozen->non_frozen != NULL);
+            premerge = frozen;
+            if (frozen_info != NULL) {
+                ASSERT(DYNAMO_OPTION(coarse_freeze_merge));
+                /* case 9701: more efficient to merge while freezing, but
+                 * this way we share code w/ offline merger
+                 */
+                /* I would put most-likely-larger unit as first source since more
+                 * efficient to merge into, but we need frozen first in case
+                 * we are in_place.
+                 */
+                frozen = coarse_unit_merge(dcontext, frozen, frozen_info, in_place);
+                ASSERT(frozen != NULL);
+                ASSERT(!in_place || frozen->non_frozen != NULL);
+                if (frozen == NULL) {
+                    /* Shouldn't happen w/ online units; if it does we end up
+                     * tossing frozen_info w/o merging it
+                     */
+                    frozen = premerge;
+                }
+                if (!in_place && premerge != NULL) {
+                    coarse_unit_reset_free(dcontext, premerge, false/*no locks*/,
+                                           false/*already unlinked*/,
+                                           false/*not in use anyway*/);
+                    coarse_unit_free(dcontext, premerge);
+                    if (frozen == premerge)
+                        frozen = NULL;
+                    premerge = NULL;
+                }
+                if (in_place) {
+                    coarse_unit_reset_free(dcontext, frozen_info, false/*no locks*/,
+                                           true/*need to unlink*/,
+                                           false/*keep primary*/);
+                    coarse_unit_free(dcontext, frozen_info);
+                    frozen_info = NULL;
+                }
+            }
+            if (!in_place && frozen != NULL) {
+                coarse_unit_persist(dcontext, frozen);
+                coarse_unit_reset_free(dcontext, frozen, false/*no locks*/,
+                                       false/*already unlinked*/,
+                                       false/*not in use anyway*/);
+                coarse_unit_free(dcontext, frozen);
+                frozen = NULL;
+            } else
+                ASSERT(frozen == unfrozen_info);
+        }
+    } else if (frozen_info != NULL && frozen_info->cache != NULL &&
+               !in_place && !frozen_info->persisted) {
+        ASSERT(!TEST(PERSCACHE_CODE_INVALID, frozen_info->flags));
+        if (coarse_region_should_persist(dcontext, frozen_info))
+            coarse_unit_persist(dcontext, frozen_info);
+    }
+}
+
+/* FIXME: could create iterator and move this and vm_area_coarse_units_reset_free()
+ * into callers
+ * If !in_place this routine freezes (if not already) and persists.
+ */
+void
+vm_area_coarse_units_freeze(bool in_place)
+{
+    vm_area_vector_t *v = executable_areas;
+    int i;
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    if (!DYNAMO_OPTION(coarse_units) || !DYNAMO_OPTION(coarse_enable_freeze) ||
+        RUNNING_WITHOUT_CODE_CACHE())
+        return;
+    ASSERT(!RUNNING_WITHOUT_CODE_CACHE());
+    ASSERT(dcontext != NULL);
+    LOG(THREAD, LOG_FRAGMENT|LOG_VMAREAS, 2, "vm_area_coarse_units_freeze\n");
+    ASSERT(dynamo_all_threads_synched);
+    acquire_recursive_lock(&change_linking_lock);
+#ifdef HOT_PATCHING_INTERFACE
+    if (DYNAMO_OPTION(hot_patching))
+        read_lock(hotp_get_lock());
+#endif
+    /* We would grab executable_areas_lock but coarse_unit_freeze() grabs
+     * change_linking_lock and coarse_info_lock, both of higher rank.  We could
+     * grab change_linking_lock first here and raise executable_areas_lock above
+     * coarse_info_lock's rank, but executable_areas_lock can be acquired during
+     * coarse_unit_unlink after special_heap_lock -- so the best solution is to
+     * not grab executable_areas_lock here and rely on all_threads_synched.
+     * Could make executable_areas_lock recursive and grab all locks here?
+     */
+    for (i = 0; i < v->length; i++) {
+        if (TEST(FRAG_COARSE_GRAIN, v->buf[i].frag_flags)) {
+            coarse_info_t *info = (coarse_info_t *) v->buf[i].custom.client;
+            ASSERT(info != NULL);
+            if (info != NULL)
+                vm_area_coarse_region_freeze(dcontext, info, &v->buf[i], in_place);
+        }
+    }
+#ifdef HOT_PATCHING_INTERFACE
+    if (DYNAMO_OPTION(hot_patching))
+        read_unlock(hotp_get_lock());
+#endif
+    release_recursive_lock(&change_linking_lock);
+}
+
+#if 0 /* not used */
+/* remove a thread's vm area */
+static bool
+remove_thread_vm_area(dcontext_t *dcontext, app_pc start, app_pc end)
+{
+    thread_data_t *data = GET_DATA(dcontext, 0);
+    bool ok;
+    LOG(THREAD, LOG_VMAREAS, 2, "removing thread %d vm area: "PFX"-"PFX"\n",
+        dcontext->owning_thread, start, end);
+    /* no lock needed, this is thread-private */
+    ok = remove_vm_area(&data->areas, start, end, false);
+    /* due to re-sorting, areas move around...not worth trying to shift,
+     * just clear the cache area */
+    data->last_area = NULL;
+    return ok;
+}
+#endif
+
+/* returns true if the passed in area overlaps any thread executable areas */
+bool
+thread_vm_area_overlap(dcontext_t *dcontext, app_pc start, app_pc end)
+{
+    thread_data_t *data = GET_DATA(dcontext, 0);
+    bool res;
+    if (data == shared_data) {
+        ASSERT(!self_owns_write_lock(&shared_data->areas.lock));
+        SHARED_VECTOR_RWLOCK(&data->areas, write, lock);
+    }
+    res = vm_area_overlap(&data->areas, start, end);
+    if (data == shared_data) {
+        SHARED_VECTOR_RWLOCK(&data->areas, write, unlock);
+    }
+    return res;
+}
+
+/* Returns NULL if should re-execute the faulting write
+ * Else returns the target pc for a new basic block -- caller should
+ * return to dispatch rather than the code cache
+ * If instr_cache_pc==NULL, assumes the cache is unavailable (due to reset).
+ */
+app_pc
+handle_modified_code(dcontext_t *dcontext, cache_pc instr_cache_pc,
+                     app_pc instr_app_pc, app_pc target)
+{
+    /* FIXME: for Linux, this is all happening inside signal handler...
+     * flushing could take a while, and signals are blocked the entire time!
+     */
+    app_pc base_pc, flush_start = NULL, next_pc;
+    app_pc instr_size_pc;
+    size_t size, flush_size = 0, instr_size;
+    uint opnd_size = 0;
+    uint prot;
+    overlap_info_t info = {0,/* init to 0 so info.overlap is false */};
+    app_pc bb_start = NULL;
+    app_pc bb_end = NULL;
+    app_pc bb_pstart = NULL, bb_pend = NULL; /* pages occupied by instr's bb */
+    vm_area_t *a = NULL;
+    fragment_t wrapper;
+    fragment_t *f = (instr_cache_pc == NULL) ? NULL :
+        fragment_pclookup(dcontext, instr_cache_pc, &wrapper);
+    /* get the "region" size (don't use exec list, it merges regions),
+     * the os merges regions too, and we might have changed the protections
+     * on the region and caused it do so, so below we take the intersection
+     * with the enclosing executable_areas region if it exists */
+    bool ok = get_memory_info(target, &base_pc, &size, &prot);
+    /* FIXME: what if seg fault is b/c target is unreadable?  then should have
+     * app die, not us trigger assertion!
+     */
+    /* In the absence of reset, f MUST still be in the cache since we're still
+     * nolinking, and pclookup will find it even if it's no longer in htables.
+     * But, a reset can result in not having the fragment available at all.  In
+     * that case we just flush the whole region and hope that in the future
+     * we'll eventually identify the writer, but there's a possibility of no
+     * forward progress if another thread keeps flushing writing fragment
+     * (ro2sandbox_threshold would alleviate that).
+     */
+    DOLOG(1, LOG_VMAREAS, {
+        if (instr_cache_pc == NULL) {
+            LOG(THREAD, LOG_VMAREAS, 1,
+                "WARNING: cache unavailable for processing code mod @ app pc "PFX"\n",
+                instr_app_pc);
+        } else if (f == NULL) {
+            LOG(THREAD, LOG_VMAREAS, 1,
+                "WARNING: cannot find fragment @ writer pc "PFX" -- was deleted, "
+                "or native\n", instr_cache_pc);
+        }
+    });
+    ASSERT(ok);
+    SYSLOG_INTERNAL_WARNING_ONCE("writing to executable region.");
+    STATS_INC(num_write_faults);
+    read_lock(&executable_areas->lock);
+    lookup_addr(executable_areas, (app_pc)target, &a);
+    if (a == NULL) {
+        LOG(THREAD, LOG_VMAREAS, 1,
+            "\tRegion for "PFX" not exec, probably data on same page\n", target);
+        DOLOG(2, LOG_VMAREAS, { print_vm_areas(executable_areas, THREAD); });
+    } else {
+        /* The os may have merged regions  because we made a region read 
+         * only! (ref case 2803), thus we should take the intersection of 
+         * the region on our list and the os region */
+        /* make sure to handle sub-page regions, pad to page boundary */
+        app_pc a_pstart = (app_pc)ALIGN_BACKWARD(a->start, PAGE_SIZE);
+        app_pc a_pend = (app_pc)ALIGN_FORWARD(a->end, PAGE_SIZE);
+        if (a_pstart > base_pc) {
+            size -= a_pstart - base_pc;
+            base_pc = a_pstart;
+        }
+        if (a_pend < base_pc + size) {
+            size = a_pend - base_pc;
+        }
+        LOG(THREAD, LOG_VMAREAS, 1,
+            "WARNING: Exec "PFX"-"PFX" %s%s written @"PFX" by "PFX" == app "PFX"\n",
+            base_pc, base_pc+size,
+            ((a->vm_flags & VM_WRITABLE) != 0) ? "W" : "",
+            ((prot & MEMPROT_EXEC) != 0) ? "E" : "",
+            target, instr_cache_pc, instr_app_pc);
+    }
+    read_unlock(&executable_areas->lock);
+#ifdef DGC_DIAGNOSTICS
+    DOLOG(1, LOG_VMAREAS, {
+        /* it's hard to locate frag owning an app pc in the cache, so we wait until
+         * we flush and only check the flushed frags
+         */
+        char buf[MAXIMUM_SYMBOL_LENGTH];
+        print_symbolic_address(instr_app_pc, buf, sizeof(buf), false);
+        LOG(THREAD, LOG_VMAREAS, 1, "code written by app pc "PFX" from bb %s:\n",
+            instr_app_pc, buf);
+        disassemble_app_bb(dcontext, instr_app_pc, THREAD);
+    });
+#endif
+    if (TEST(MEMPROT_WRITE, prot)) {
+        LOG(THREAD, LOG_VMAREAS, 1,
+            "\tWARNING: region now writable: assuming another thread already flushed it\n"
+            "\tgoing to flush again just to make sure\n");
+        /* we could just bail here, but could have no forward progress if repeated
+         * races between selfmod writer and out-of-region writer
+         */
+        STATS_INC(num_write_fault_races);
+    }
+
+    /* see if writer is inside our region
+     * need instr size and opnd size to check for page boundary overlaps!
+     * For reset when the cache is not present, we decode from the app code,
+     * though that's racy!  solution is to have reset store a copy of the app instr
+     * (FIXME case 7393).
+     */
+    instr_size_pc = (instr_cache_pc == NULL) ? instr_app_pc : instr_cache_pc;
+    next_pc = decode_memory_reference_size(dcontext, instr_size_pc, &opnd_size);
+    ASSERT(next_pc != NULL);
+    ASSERT(opnd_size != 0);
+    instr_size = next_pc - instr_size_pc;
+    /* FIXME case 7492: if write crosses page boundary, the reported faulting
+     * target for win32 will be in the middle of the instr's target (win32
+     * reports the first unwritable byte).  (On Linux we're fine as we calculate
+     * the target ourselves.)
+     */
+    if (target + opnd_size > base_pc + size) {
+        /* must expand to cover entire target, even if crosses OS regions */
+        app_pc t_pend = (app_pc)ALIGN_FORWARD(target + opnd_size, PAGE_SIZE);
+        size = t_pend - base_pc;
+    }
+    /* see if instr's bb is in region
+     * not good enough to only check instr!
+     * will end up in infinite loop if any part of bb overlaps the executable
+     * region removed!
+     * if f was deleted, we threw away its also info, so we have to do a full
+     * overlaps lookup.  f cannot have been removed completely since we 
+     * count as being in the shared cache and could be inside f.
+     */
+    if (f != NULL &&
+         /* faster check up front if frag not deleted -- BUT, we are in
+          * a race w/ any flusher marking as deleted!
+          * so, we make vm_list_overlaps not assert on a not-there fragment,
+          * and only if it finds it and it's STILL not marked do we trust the
+          * return value.
+          */
+         (vm_list_overlaps(dcontext, (void *)f, base_pc, base_pc+size) ||
+          TEST(FRAG_WAS_DELETED, f->flags))) {
+        fragment_overlaps(dcontext, f, instr_app_pc, instr_app_pc+1,
+                          false /* fine-grain! */, &info, &bb_start);
+        /* if did fast check and it said overlap, slow check should too */
+        ASSERT(TEST(FRAG_WAS_DELETED, f->flags) || info.overlap);
+    } 
+    if (info.overlap) {
+        /* instr_t may be in region, but could also be from a different region
+         * included in a trace.  Determine if instr bb overlaps with target
+         * region.
+         * Move to page boundaries, with inclusive end pages.
+         * We must look at entire bb containing instr, not just instr
+         * itself (can't isolate write from its bb -- will always
+         * enter from top of bb, even across direct cti)
+         */
+        ASSERT(info.overlap && bb_start != NULL);
+        if (info.contiguous)
+            bb_end = info.bb_end;
+        else {
+            /* FIXME: could be smart and have info include list of all pages,
+             * handle situations like start outside of region and jmp/call in,
+             * but this is going to be rare -- let's just take min and max of
+             * entire bb, even if that includes huge area (in which case we'll
+             * consider it self-modifying code, even if jumped over middle)
+             */
+            bb_start = info.min_pc;
+            bb_end = info.max_pc;
+            ASSERT(bb_start != NULL && bb_end != NULL);
+        }
+        bb_pstart = (app_pc) PAGE_START(bb_start);
+        bb_pend = (app_pc) PAGE_START(bb_end);
+        ASSERT(instr_app_pc >= bb_pstart &&
+               instr_app_pc+instr_size <= bb_pend+PAGE_SIZE);
+        ASSERT(f != NULL); /* else info.overlap should not be set */
+    }
+    /* Now we can check if source bb overlaps target region. */
+    if (info.overlap &&
+        base_pc < (bb_pend + PAGE_SIZE) &&
+        (base_pc + size) > bb_pstart) {
+        /* bb pages overlap target region -
+         * We want to split up region to keep instr exec but target writable.
+         * All pages touched by target will become writable.
+         * All pages in instr's bb must remain executable (can't isolate
+         * write from its bb -- will always enter from top of bb)
+         */
+        /* pages occupied by target */
+        app_pc tgt_pstart = (app_pc) PAGE_START(target);
+        app_pc tgt_pend = (app_pc) PAGE_START(target+opnd_size);
+        
+        DOSTATS({
+            /* race condition case of another thread flushing 1st */
+            if (TEST(MEMPROT_WRITE, prot))
+                STATS_INC(num_write_fault_races_selfmod);
+        });
+        
+        LOG(THREAD, LOG_VMAREAS, 2,
+            "Write instr is inside F%d "PFX"\n", f->id, f->tag);
+        
+        LOG(THREAD, LOG_VMAREAS, 1,
+            "\tinstr's bb src "PFX"-"PFX" overlaps target "PFX"-"PFX"\n",
+            bb_start, bb_end, target, target+opnd_size);
+        
+        /* look for selfmod overlap */
+        if (bb_pstart <= tgt_pend && bb_pend >= tgt_pstart) {
+            vm_area_t *execarea;
+            app_pc nxt_on_page;
+            LOG(THREAD, LOG_VMAREAS, 1,
+                "WARNING: self-modifying code: instr @"PFX" (in bb "PFX"-"PFX")\n"
+                "\twrote to "PFX"-"PFX"\n",
+                instr_app_pc, bb_start, bb_end, target, target+opnd_size);
+            SYSLOG_INTERNAL_WARNING_ONCE("self-modifying code.");
+            /* can leave non-intersection part of instr pages as executable,
+             * no need to flush them
+             */
+            /* DGC_DIAGNOSTICS: have flusher pass target to
+             * vm_area_unlink_fragments to check if code was actually overwritten
+             */
+            flush_fragments_in_region_start(dcontext, (app_pc)tgt_pstart,
+                                            (tgt_pend+PAGE_SIZE-tgt_pstart),
+                                            false /* don't own initexit_lock */,
+                                            false /* keep futures */,
+                                            true /* exec invalid */,
+                                            false /* don't force synchall */
+                                            _IF_DGCDIAG(target));
+            /* flush_* grabbed exec areas lock for us, to make following sequence atomic */
+            /* need to change all exec areas on these pages to be selfmod */
+            for (ok = true, nxt_on_page = (app_pc) tgt_pstart;
+                 ok && nxt_on_page < (app_pc)tgt_pend + PAGE_SIZE; ) {
+                ok = binary_search(executable_areas, nxt_on_page, (app_pc)tgt_pend+PAGE_SIZE,
+                                   &execarea, NULL, true /* want 1st match! */);
+                if (ok) {
+                    nxt_on_page = execarea->end;
+                    if (TESTANY(FRAG_SELFMOD_SANDBOXED, execarea->frag_flags)) {
+                        /* not calling remove_vm_area so we have to vm_make_writable
+                         * FIXME: why do we have to do anything if already selfmod?
+                         */
+                        if (TEST(VM_MADE_READONLY, execarea->vm_flags))
+                            vm_make_writable(execarea->start, execarea->end - execarea->start);
+                        continue;
+                    }
+                    if (execarea->start < (app_pc)tgt_pstart ||
+                        execarea->end > (app_pc)tgt_pend + PAGE_SIZE) {
+                        /* this area sticks out from our target area, so we split it
+                         * by removing and then re-adding (as selfmod) the overlap portion
+                         */
+                        uint old_vmf = execarea->vm_flags;
+                        uint old_ff = execarea->frag_flags;
+                        app_pc old_start = (execarea->start < tgt_pstart) ?
+                            tgt_pstart : execarea->start;
+                        app_pc old_end = (execarea->end > tgt_pend + PAGE_SIZE) ?
+                            tgt_pend + PAGE_SIZE : execarea->end;
+                        LOG(GLOBAL, LOG_VMAREAS, 2,
+                            "removing executable vm area to mark selfmod: "PFX"-"PFX"\n",
+                            old_start, old_end);
+                        remove_vm_area(executable_areas, old_start, old_end, true);
+                        /* now re-add */
+                        add_executable_vm_area(old_start, old_end,
+                                               old_vmf, old_ff | FRAG_SELFMOD_SANDBOXED,
+                                               true /*own lock */
+                                               _IF_DEBUG("selfmod replacement"));
+                        STATS_INC(num_selfmod_vm_areas);
+                        /* this won't hurt our iteration since it's stateless except for
+                         * nxt_on_page
+                         */
+                    } else {
+                        LOG(THREAD, LOG_VMAREAS, 2, "\tmarking "PFX"-"PFX" as selfmod\n",
+                            execarea->start, execarea->end);
+                        execarea->frag_flags |= SANDBOX_FLAG();
+                        STATS_INC(num_selfmod_vm_areas);
+                        /* not calling remove_vm_area so we have to vm_make_writable */
+                        if (TEST(VM_MADE_READONLY, execarea->vm_flags))
+                            vm_make_writable(execarea->start, execarea->end - execarea->start);
+                    }
+                }
+            }
+            LOG(GLOBAL, LOG_VMAREAS, 3,
+                "After marking all areas in "PFX"-"PFX" as selfmod:\n",
+                tgt_pstart, tgt_pend+PAGE_SIZE);
+            DOLOG(3, LOG_VMAREAS, { print_vm_areas(executable_areas, GLOBAL); });
+            flush_fragments_in_region_finish(dcontext,
+                                             false /*don't keep initexit_lock*/);
+            /* must execute instr_app_pc next, even though that new bb will be
+             * useless afterward (will most likely re-enter from bb_start)
+             */
+            return instr_app_pc;
+        } else {
+            /* Not selfmod, but target and bb region may still overlap -
+             * heuristic: split the region up -- assume will keep writing
+             * to higher addresses and keep executing at higher addresses.
+             */
+            if (tgt_pend < bb_pstart) {
+                /* make all pages from tgt_pstart up to bb_pstart or
+                 * region end (which ever is first) non-exec */
+                /* FIXME - CHECK - should we really be starting at
+                 * base_pc instead? Not clear why we shouldn't start at
+                 * region start (like we would if we didn't have an
+                 * overlap). */
+                flush_start = tgt_pstart;
+                ASSERT(bb_pstart < (base_pc + size) &&
+                       bb_pstart > tgt_pstart);
+                flush_size = bb_pstart - tgt_pstart;
+            } else if (tgt_pstart > bb_pend) {
+                /* make all pages from tgt_pstart to end of region non-exec */
+                flush_start = tgt_pstart;
+                flush_size = (base_pc + size) - tgt_pstart;
+            } else {
+                /* should never get here -- all cases covered above */
+                ASSERT_NOT_REACHED();
+            }
+            LOG(THREAD, LOG_VMAREAS, 2,
+                "splitting region up, flushing just "PFX"-"PFX"\n",
+                flush_start, flush_start+flush_size);
+        }
+    } else {
+        ASSERT(!info.overlap || (f != NULL && TEST(FRAG_IS_TRACE, f->flags)));
+        /* instr not in region, so move entire region off the executable list */
+        flush_start = base_pc;
+        flush_size = size;
+        LOG(THREAD, LOG_VMAREAS, 2, "instr not in region, flushing entire "PFX"-"PFX"\n",
+            flush_start, flush_start+flush_size);
+    }
+
+    /* DGC_DIAGNOSTICS: have flusher pass target to
+     * vm_area_unlink_fragments to check if code was actually overwritten
+     */
+    flush_fragments_in_region_start(dcontext, flush_start, flush_size,
+                                    false /* don't own initexit_lock */,
+                                    false /* keep futures */,
+                                    true /* exec invalid */,
+                                    false /* don't force synchall */
+                                    _IF_DGCDIAG(target));
+    f = NULL; /* after the flush we don't know if it's safe to deref f */
+
+    if (DYNAMO_OPTION(ro2sandbox_threshold) > 0) {
+        /* add removed region to written list to track # of times this has happened
+         * actually, we only track by the written-to page
+         * FIXME case 8161: should we add more than just the page?
+         * we'll keep adding the whole region until it hits the ro2sandbox threshold,
+         * at which point we'll just add the page
+         */
+        ro_vs_sandbox_data_t *ro2s;
+        write_lock(&written_areas->lock);
+        /* use the add routine to lookup if present, add if not */
+        add_written_area(written_areas, target, (app_pc) PAGE_START(target),
+                         (app_pc) PAGE_START(target+opnd_size) + PAGE_SIZE, &a);
+        ASSERT(a != NULL);
+        ro2s = (ro_vs_sandbox_data_t *) a->custom.client;
+        ro2s->written_count++;
+        LOG(GLOBAL, LOG_VMAREAS, 2,
+            "written area "PFX"-"PFX" now written %d X\n",
+            a->start, a->end, ro2s->written_count);
+        DOLOG(3, LOG_VMAREAS, {
+            LOG(GLOBAL, LOG_VMAREAS, 2, "\nwritten areas:\n");
+            print_vm_areas(written_areas, GLOBAL);
+        });
+        write_unlock(&written_areas->lock);
+    }
+
+    if (
+#ifdef PROGRAM_SHEPHERDING
+        !DYNAMO_OPTION(selfmod_futureexec) &&
+#endif
+        is_executable_area_on_all_selfmod_pages(target, target+opnd_size)) {
+        /* We can be in various races with another thread in handling write
+         * faults to this same region.  We check at the start of this routine,
+         * but in practice (case 7911) I've seen the race more often show up
+         * here, after the flush synch.  If another thread has already switched
+         * the target region to selfmod, then we shouldn't remove it from
+         * executable_areas here.  In fact if we were to remove it we would foil
+         * the selfmod->remove future optimizations (case 280) (once-only at
+         * NtFlush, selfmod when used to validate exec area, and remove
+         * overlapping futures w/ new selfmod exec area).
+         */
+        /* FIXME: is it worth checking this selfmod overlap in earlier places,
+         * like the start of this routine, or at the start of the flush synch,
+         * which could save some synch work and perhaps avoid the flush
+         * altogether?
+         */
+        STATS_INC(flush_selfmod_race_no_remove);
+        LOG(THREAD, LOG_VMAREAS, 2,
+            "Target "PFX" is already selfmod, race, no reason to remove\n",
+            target);
+    } else {
+        /* flush_* grabbed exec areas lock for us, to make vm_make_writable, 
+         * remove global vm area, and lookup an atomic sequence 
+         */
+        LOG(GLOBAL, LOG_VMAREAS, 2,
+            "removing executable vm area since written: "PFX"-"PFX"\n",
+            flush_start, flush_start+flush_size);
+        /* FIXME : are we removing regions that might not get re-added here?
+         * what about things that came from once only future or mem prot changes,
+         * the region removed here can be much larger then just the page written
+         */
+        /* FIXME (part of case 3744): should remove only non-selfmod regions here!
+         * Then can eliminate the if above.   Could pass filter flag to remove_vm_area,
+         * but better to just split code origins from consistency and not have
+         * sub-page regions on the consistency list (case 3744).
+         */
+        remove_vm_area(executable_areas, flush_start, flush_start+flush_size,
+                       true/*restore writability!*/);
+        LOG(THREAD, LOG_VMAREAS, 2,
+            "Removed "PFX"-"PFX" from exec list, continuing @ write\n",
+            flush_start, flush_start+flush_size);
+    }
+    DOLOG(3, LOG_VMAREAS, {
+        thread_data_t *data = GET_DATA(dcontext, 0);
+        LOG(THREAD, LOG_VMAREAS, 2, "\nexecutable areas:\n");
+        print_vm_areas(executable_areas, THREAD);
+        LOG(THREAD, LOG_VMAREAS, 2, "\nthread areas:\n");
+        print_vm_areas(&data->areas, THREAD);
+    });
+
+    /* There is no good way to tell if we flushed f or not, so need to start
+     * interpreting at instr_app_pc. If f was a trace could overlap flushed
+     * region even if the src bb didn't and anyways flushing can end up
+     * flushing outside the requested region (entire vm_area_t). If we could tell
+     * we could return NULL instead (which is a special flag that says redo the
+     * write instead of going to dispatch) if f wasn't flushed. 
+     * FIXME - Redoing the write would be more efficient then going back to
+     * dispatch and should be the common case. */
+    flush_fragments_in_region_finish(dcontext, false /*don't keep initexit_lock*/);
+    return instr_app_pc;
+}
+
+/* Returns the counter a selfmod fragment should execute for -sandbox2ro_threshold */
+uint *
+get_selfmod_exec_counter(app_pc tag)
+{
+    vm_area_t *area = NULL;
+    ro_vs_sandbox_data_t *ro2s;
+    uint *counter;
+    bool ok;
+    read_lock(&written_areas->lock);
+    ok = lookup_addr(written_areas, tag, &area);
+    if (!ok) {
+        read_unlock(&written_areas->lock);
+        read_lock(&executable_areas->lock);
+        write_lock(&written_areas->lock);
+        ok = lookup_addr(executable_areas, tag, &area);
+        ASSERT(ok && area != NULL);
+        /* FIXME: do this addition whenever add new exec area marked as
+         * selfmod?
+         * FIXME case 8161: add only add one page?  since never split written_areas?
+         * For now we add the whole region, reasoning that as a selfmod
+         * region it's probably not very big anyway.
+         * In Sun's JVM 1.4.2 we actually never get here b/c we always
+         * have an executable region already present before we make it selfmod,
+         * so we're only adding to written_areas when we get a write fault,
+         * at which point we only use the surrounding page.
+         */
+        STATS_INC(num_sandbox_before_ro);
+        add_written_area(written_areas, tag, area->start, area->end, &area);
+        ASSERT(area != NULL);
+        ro2s = (ro_vs_sandbox_data_t *) area->custom.client;
+        counter = &ro2s->selfmod_execs;
+        /* Inc of selfmod_execs from cache can have problems if it crosses a
+         * cache line, so we assert on the 32-bit alignment we should get from
+         * the heap.  add_written_area already asserts but we double-check here.
+         */
+        ASSERT(ALIGNED(counter, sizeof(uint)));
+        write_unlock(&written_areas->lock);
+        read_unlock(&executable_areas->lock);
+    } else {
+        ASSERT(ok && area != NULL);
+        ro2s = (ro_vs_sandbox_data_t *) area->custom.client;
+        counter = &ro2s->selfmod_execs;
+        read_unlock(&written_areas->lock);
+    }
+    /* ref to counter will be accessed in-cache w/o read lock but
+     * written_areas is never merged and counter won't be freed until
+     * exit time.
+     */
+    return counter;
+}
+
+/* Returns true if f has been flushed */
+bool
+vm_area_selfmod_check_clear_exec_count(dcontext_t *dcontext, fragment_t *f)
+{
+    ro_vs_sandbox_data_t *ro2s = NULL;
+    vm_area_t *exec_area = NULL, *written_area;
+    app_pc start, end;
+    bool ok;
+    bool convert_s2ro = true;
+    if (DYNAMO_OPTION(sandbox2ro_threshold) == 0)
+        return false;
+
+    /* NOTE - we could only grab the readlock here.  Even though we're going to
+     * write to selfmod_execs count, it's not really protected by the written_areas
+     * lock since we read and write to it from the cache.  Should change to read lock
+     * if contention ever becomes an issue.  Note that we would then have to later
+     * grab the write lock if we need to write to ro2s->written_count below. */
+    write_lock(&written_areas->lock);
+
+    ok = lookup_addr(written_areas, f->tag, &written_area);
+    if (ok) {
+        ro2s = (ro_vs_sandbox_data_t *) written_area->custom.client;
+    } else {
+        /* never had instrumentation */
+        write_unlock(&written_areas->lock);
+        return false;
+    }
+    if (ro2s->selfmod_execs < DYNAMO_OPTION(sandbox2ro_threshold)) {
+        /* must be a real fragment modification, reset the selfmod_execs count
+         * xref case 9908 */
+        LOG(THREAD, LOG_VMAREAS, 3,
+            "Fragment "PFX" self-write -> "PFX"-"PFX" selfmod exec counter reset, old"
+            " count=%d\n",
+            f->tag, written_area->start, written_area->end, ro2s->selfmod_execs);
+        /* Write must be atomic since we access this field from the cache, an aligned
+         * 4 byte write is atomic on the architectures we support. */
+        ASSERT(sizeof(ro2s->selfmod_execs) == 4 && ALIGNED(&(ro2s->selfmod_execs), 4));
+        ro2s->selfmod_execs = 0;
+        write_unlock(&written_areas->lock);
+        return false;
+    }
+
+    LOG(THREAD, LOG_VMAREAS, 1,
+        "Fragment "PFX" caused "PFX"-"PFX" to cross sandbox2ro threshold %d vs %d\n",
+        f->tag, written_area->start, written_area->end,
+        ro2s->selfmod_execs, DYNAMO_OPTION(sandbox2ro_threshold));
+    start = written_area->start;
+    end = written_area->end;
+    /* reset to avoid immediate re-trigger */
+    ro2s->selfmod_execs = 0;
+
+    if (is_on_stack(dcontext, f->tag, NULL)) {
+        /* Naturally we cannot make the stack ro.  We checked when we built f,
+         * but esp must now point elsewhere.  We go ahead and flush and assume
+         * that when we rebuild f we won't put the instrumentation in. */
+        convert_s2ro = false;
+        STATS_INC(num_sandbox2ro_onstack);
+        LOG(THREAD, LOG_VMAREAS, 1, "Fragment "PFX" is on stack now!\n", f->tag);
+        ASSERT_CURIOSITY(false && "on-stack selfmod bb w/ counter inc");
+    }
+
+    if (convert_s2ro && DYNAMO_OPTION(ro2sandbox_threshold) > 0) {
+        /* We'll listen to -sandbox2ro_threshold even if a selfmod region
+         * didn't become that way via -ro2sandbox_threshold, to avoid perf
+         * problems w/ other code in the same region, and to take advantage of
+         * patterns of write at init time and then never selfmod again.
+         * FIXME: have a different threshold for regions made selfmod for actual
+         * self-writes versus -ro2sandbox_threshold regions?
+         * If there is a written_count, we reset it so it can trigger again.
+         * We reset here rather than when ro2sandbox_threshold is triggered as
+         * ro2sandbox only does a page at a time and if keeping a count for
+         * multiple pages doesn't want to clear that count too early.
+         */
+        LOG(THREAD, LOG_VMAREAS, 2,
+            "re-setting written executable vm area: "PFX"-"PFX" written %d X\n",
+            written_area->start, written_area->end,
+            ro2s->written_count);
+        ro2s->written_count = 0;
+    }
+    DOLOG(3, LOG_VMAREAS, {
+        LOG(THREAD, LOG_VMAREAS, 2, "\nwritten areas:\n");
+        print_vm_areas(written_areas, THREAD);
+    });
+
+    write_unlock(&written_areas->lock);
+
+    /* Convert the selfmod region to a ro region. 
+     * FIXME case 8161: should we flush and make ro the executable area,
+     * or the written area?  Written area may only be a page if made
+     * selfmod due to a code write, but then it should match the executable area
+     * in the common case, though written area may be larger if executable area
+     * is from a tiny NtFlush.  If we make a sub-piece of the executable area ro,
+     * the rest will remain selfmod and will eventually come here anyway.
+     */
+    flush_fragments_in_region_start(dcontext, start, end - start,
+                                    false /* don't own initexit_lock */,
+                                    false /* keep futures */,
+                                    true /* exec invalid */,
+                                    false /* don't force synchall */
+                                    _IF_DGCDIAG(NULL));
+    if (convert_s2ro) {
+        DODEBUG(ro2s->s2ro_xfers++;);
+        /* flush_* grabbed executable_areas lock for us */
+        ok = lookup_addr(executable_areas, f->tag, &exec_area);
+        if (ok) {
+            if (TEST(FRAG_SELFMOD_SANDBOXED, exec_area->frag_flags)) {
+                /* FIXME: if exec area is larger than flush area, it's
+                 * ok since marking fragments in a ro region as selfmod
+                 * is not a correctness problem.  Current flush impl, though,
+                 * will flush whole region.
+                 */
+                LOG(THREAD, LOG_VMAREAS, 1,
+                    "\tconverting "PFX"-"PFX" from sandbox to ro\n",
+                    exec_area->start, exec_area->end);
+                exec_area->frag_flags &= ~FRAG_SELFMOD_SANDBOXED;
+                /* can't ASSERT(!TEST(VM_MADE_READONLY, area->vm_flags)) (case 7877) */
+                vm_make_unwritable(exec_area->start, exec_area->end - exec_area->start);
+                exec_area->vm_flags |= VM_MADE_READONLY;
+                LOG(THREAD, LOG_VMAREAS, 3,
+                    "After marking "PFX"-"PFX" as NOT selfmod:\n",
+                    exec_area->start, exec_area->end);
+                DOLOG(3, LOG_VMAREAS, { print_vm_areas(executable_areas, THREAD); });
+                STATS_INC(num_sandbox2ro);
+            } else {
+                /* must be a race! */
+                LOG(THREAD, LOG_VMAREAS, 3,
+                    "Area "PFX"-"PFX" is ALREADY not selfmod!\n",
+                    exec_area->start, exec_area->end);
+                STATS_INC(num_sandbox2ro_race);
+            }
+        } else {
+            /* must be a flushing race */
+            LOG(THREAD, LOG_VMAREAS, 3, "Area "PFX"-"PFX" is no longer there!\n",
+                start, end);
+            STATS_INC(num_sandbox2ro_flush_race);
+        }
+    }
+
+    ASSERT(exec_area == NULL || /* never looked up */
+           (start < exec_area->end && end > exec_area->start));
+
+    flush_fragments_in_region_finish(dcontext,
+                                     false /*don't keep initexit_lock*/);
+    return true;
+}
+
+void
+mark_unload_start(app_pc module_base, size_t module_size)
+{
+    /* in thin client mode we don't allocate this, 
+     * but we do track unloads in -client mode 
+     */
+    if (last_deallocated == NULL)
+        return;
+    ASSERT(DYNAMO_OPTION(unloaded_target_exception));
+    ASSERT_CURIOSITY(!last_deallocated->unload_in_progress);
+    /* we may have a race, or a thread killed during unload syscall,
+     * either way we just mark our last region on top of the old one
+     */
+    mutex_lock(&last_deallocated_lock);
+    last_deallocated->last_unload_base = module_base;
+    last_deallocated->last_unload_size = module_size;
+    last_deallocated->unload_in_progress = true;
+    mutex_unlock(&last_deallocated_lock);
+}
+
+void
+mark_unload_future_added(app_pc module_base, size_t size)
+{
+    /* case 9371: if a thread gets preempted before returning from
+     * unmapviewofsection and in the mean time another has a _future_ exec
+     * area allocated at the same place and executes from it, we should not
+     * throw exception mistakenly if the area would have been allowed
+     */
+    if (last_deallocated == NULL)
+        return;
+    ASSERT(DYNAMO_OPTION(unloaded_target_exception));
+
+    ASSERT_CURIOSITY(!last_deallocated->unload_in_progress && "future while unload");
+
+    /* FIXME: more preciselly we should only remove our intersection
+     * with the last module, otherwise don't need to, but it is never
+     * expected to happen, so not optimizing at all
+     */
+    last_deallocated->unload_in_progress = false;
+}
+
+void
+mark_unload_end(app_pc module_base)
+{
+    if (last_deallocated == NULL)
+        return;
+    ASSERT(DYNAMO_OPTION(unloaded_target_exception));
+
+    /* We're trying to avoid a spurious security violation while we
+     * are flushing our security policies, but before the address is
+     * actually fully unloaded.  So if we don't have an entry in our
+     * executable_areas or RAC or RCT policies then we should either
+     * find the address unreadable with query_virtual_memory(), or we
+     * should make sure that we find it as is_currently_unloaded_region().
+     */
+
+    /* The fact that we have reached this routine already guarantees
+     * that the memory was made unreadable (whether the memory is
+     * still unreadable is not guaranteed, see below).  Yet if we do
+     * checks in proper order is_currently_unloaded_region() _before_
+     * is_readable_without_exception(), as we do in the convenience
+     * routine is_unreadable_or_currently_unloaded_region(), we can
+     * get away without a barrier here.
+     */
+
+    /* FIXME: Otherwise we'd need a barrier, such that until a security
+     * policy reader is done, we cannot mark the module as unloaded,
+     * and if they start doing their check after this - then they
+     * should get a policy consistent with the memory already being
+     * unreadable.  (For example, we can synchronize with
+     * check_thread_vm_area() via
+     * {executable_areas_lock();executable_areas_unlock()} but since
+     * all other policies have sufficient information from unreadable
+     * memory, we're OK with a DLL being completely unloaded.
+     */
+
+    /* FIXME: note we may want to grab the appropriate policy locks so
+     * that we can thus delay our declaring we're no longer unloading
+     * a module until the policy processing is done, e.g. if one has
+     * started querying a security policy while we are unloading, we
+     * should preserve the marker until they are done.
+     * for .B we hold a writable executable_areas_lock(), 
+     * watch out here if for case 9371 we want to also mark_unload_end()
+     * on any new allocations
+     * FIXME: the RCT policies however we don't hold a lock.
+     */
+
+    /* FIXME: case 9372 Note that we may still have a problem primarily if a DLL
+     * gets subsequently reloaded at the same location, (so we have
+     * lost our flag) so after a time in which we make our checks
+     * whether the target is unreadable, the new version will show up
+     * and may not yet be fully processed in postsys_MapViewOfSection
+     * (and even if it is, we may have already checked our
+     * policies). I assume this should be less frequent than the
+     * unload side (although it still shows up in our
+     * win32/reload-race.c).  At least not a problem if the DLL gets
+     * reloaded at a different address, like case 9121 or with -aslr 1
+     */
+
+    /* note grabbing this lock is only useful for the ASSERTs, setting
+     * the flag is atomic even without it.
+     * is_unreadable_or_currently_unloaded_region() when used in
+     * proper order doesn't need to synchronize with this lock either */
+    mutex_lock(&last_deallocated_lock);
+
+    /* note, we mark_unload_start on MEM_IMAGE but mark_unload_end on
+     * MEM_MAPPED as well.  Note base doesn't have to match as long as
+     * it is within the module */
+    ASSERT_CURIOSITY(!last_deallocated->unload_in_progress || 
+                     ((last_deallocated->last_unload_base <= module_base && 
+                       module_base < 
+                       (last_deallocated->last_unload_base + 
+                        last_deallocated->last_unload_size)) && "race - multiple unmaps"));
+    DOLOG(1, LOG_VMAREAS, {
+        /* there are a few cases where DLLs aren't unloaded by real
+         * base uxtheme.dll, but I haven't seen them */
+        ASSERT_CURIOSITY(!last_deallocated->unload_in_progress || 
+                         (last_deallocated->last_unload_base == module_base && 
+                         "not base"));
+    });
+
+    /* multiple racy unmaps can't be handled simultaneously anyways */
+    last_deallocated->unload_in_progress = false;
+    mutex_unlock(&last_deallocated_lock);
+}
+
+bool
+is_in_last_unloaded_region(app_pc pc)
+{
+    bool in_last = true;
+    if (last_deallocated == NULL)
+        return false;
+    ASSERT(DYNAMO_OPTION(unloaded_target_exception));
+
+    mutex_lock(&last_deallocated_lock);
+    /* if we are in such a tight race that we're no longer
+     * last_deallocated->unload_in_progress we can still use the
+     * already unloaded module 
+     */
+    if ((pc < last_deallocated->last_unload_base) ||
+        (pc >= (last_deallocated->last_unload_base 
+                + last_deallocated->last_unload_size)))
+        in_last = false;
+    mutex_unlock(&last_deallocated_lock);
+    return in_last;
+}
+
+static bool
+is_currently_unloaded_region(app_pc pc)
+{
+    if (last_deallocated == NULL)
+        return false;
+    ASSERT(DYNAMO_OPTION(unloaded_target_exception));
+
+    if (!last_deallocated->unload_in_progress)
+        return false;
+
+    return is_in_last_unloaded_region(pc);
+}
+
+bool
+is_unreadable_or_currently_unloaded_region(app_pc pc)
+{
+    /* we want one atomic query - so if we are before the completion
+     * of the UnMap system call we should be
+     * is_currently_unloaded_region(), but afterwards the address
+     * should be !is_readable_without_exception
+     */
+    /* order of execution is important - so that we don't have to grab
+     * a lock to synchronize with mark_unload_end().  
+     */
+
+    if (is_currently_unloaded_region(pc)) {
+        STATS_INC(num_unloaded_race);
+        return true;
+    }
+    /* If we are not in a currently unloaded module then target is
+     * either not being unloaded or we are beyond system call.
+     */
+    if (!is_readable_without_exception(pc, 1)) {
+        return true;
+    }
+    return false;
+}
+
+void
+print_last_deallocated(file_t outf)
+{
+    if (last_deallocated == NULL)
+        return;
+
+    ASSERT(DYNAMO_OPTION(unloaded_target_exception));
+    if (last_deallocated->last_unload_base == NULL) {
+        print_file(outf, "never unloaded\n");
+        return;
+    }
+
+    print_file(outf, "last unload: "PFX"-"PFX"%s\n", 
+               last_deallocated->last_unload_base,
+               last_deallocated->last_unload_base +
+               last_deallocated->last_unload_size,
+               last_deallocated->unload_in_progress ? " being unloaded": "");
+}
+
+
+#ifdef PROGRAM_SHEPHERDING
+/* Note that rerouting an APC to this target should safely popup the arguments
+ * and continue.
+ *
+ * Since ThreadProc and APCProc have the same signature, we handle a
+ * remote thread in a similar way, instead of letting attack handling
+ * decide its fate - which may be an exception instead of killing the
+ * thread.
+ *
+ * FIXME: we're interpreting dynamorio.dll code here
+ */
+/* FIXME clean up: safe_apc_or_thread_target, apc_thread_policy_helper and
+ * aslr_report_violation should all be ifdef WINDOWS, and may be in a
+ * different file
+ */
+/* could do naked to get a single RET 4 emitted with no prologue */
+void
+APC_API
+safe_apc_or_thread_target(reg_t arg)
+{
+    /* NOTHING */
+}
+/* FIXME: case 9023: this is WRONG for NATIVE APCs!
+ * kernel32!BaseDispatchAPC+0x33:
+ * 7c82c13a c20c00           ret     0xc
+ * FIXME: add safe_native_apc(PVOID context, PAPCFUNC func, reg_t arg)
+ */
+
+/* a helper procedure for DYNAMO_OPTION(apc_policy) or DYNAMO_OPTION(thread_policy) 
+ *
+ * FIXME: currently relevant only on WINDOWS
+ */
+void
+apc_thread_policy_helper(app_pc *apc_target_location, /* IN/OUT */
+                         security_option_t target_policy,
+                         apc_thread_type_t target_type)
+{
+    bool is_apc = 
+        (target_type == APC_TARGET_NATIVE) || 
+        (target_type == APC_TARGET_WINDOWS);
+    /* if is_win32api we're evaluating the Win32 API targets of
+     * QueueUserAPC/CreateThreadEx, otherwise it is the native
+     * NtQueueApcThread/NtCreateThreadEx targets
+     */
+    bool is_win32api = 
+        (target_type == THREAD_TARGET_WINDOWS) || 
+        (target_type == APC_TARGET_WINDOWS);
+
+    bool match = false;
+    /* FIXME: note taking the risk here of reading from either the
+     * word on the stack, or from a Cxt.  While the app would fail in
+     * either case this should be safer.  I don't want the extra
+     * is_readable_without_exception() here though.
+     */
+    app_pc injected_target = *apc_target_location;
+    uint injected_code = 0; /* first bytes of shellcode */
+
+    /* match PIC shellcode header, for example
+     * 0013004c 53               push    ebx
+     * 0013004d e800000000       call    00130052
+     */
+    enum {PIC_SHELLCODE_MATCH = 0x0000e853};
+
+    /* Now we quickly check a stipped down code origins policy instead
+     * of letting the bb builder do this.  ALTERNATIVE design: We could save
+     * the target and have this extra work done only after a code
+     * origins violations.  Then we would not modify application state
+     * unnecessarily.  The problem however is that we need to make
+     * sure we do that only _immediately_ after an APC.
+     */
+
+    /* using only executable area - assuming areas added by
+     * -executable_if_x are only added to futureexec_areas, so that
+     * this test can be done and acted upon independently of us
+     * running in NX compatibility
+     */
+    if (is_executable_address(injected_target)) {
+        return;         /* not a match */
+    }
+
+    if (safe_read(injected_target, 
+                  sizeof(injected_code), &injected_code)) {
+        LOG(GLOBAL, LOG_ASYNCH, 2,
+            "ASYNCH intercepted APC: APC pc="PFX", APC code="PFX" %s\n", 
+            injected_target, injected_code,
+            injected_code == PIC_SHELLCODE_MATCH ? "MATCH" : "");
+    } else {
+        ASSERT_NOT_TESTED();
+    }
+
+    /* target is a non-executable area, but we may want to be more specific */
+    if (TEST(OPTION_CUSTOM, target_policy)) {
+        match = true;   /* no matter what is in the shellcode */
+    } else {
+        if (injected_code == PIC_SHELLCODE_MATCH)
+            match = true;
+    }
+
+    if (match) {
+        bool squashed = false;
+        char injected_threat_buf[MAXIMUM_VIOLATION_NAME_LENGTH]
+            = "APCS.XXXX.B";
+        char *name = injected_threat_buf;
+
+        bool block = TEST(OPTION_BLOCK, target_policy);
+
+        /* we need the constructed name before deciding to really
+         * block, in case we exempt by ID */
+        if (TEST(OPTION_REPORT, target_policy)) {
+            /* mangle injected_code into a name */
+            if (injected_code == PIC_SHELLCODE_MATCH) {
+                /* keeping the well known hardcoded ones for VSE */
+                name = is_apc ? "VVPP.3200.B" : "YCRP.3200.B";
+            } else {
+                /* FIXME: nativs vs non-native could get a different prefix as well */
+                if (!is_apc) {
+                    const char *INJT = "INJT";
+                    /* (injected) shellcode thread */
+                    ASSERT_NOT_TESTED();
+                    /* gcc warns if we use the string "INJT" directly */
+                    strncpy(name, INJT, 4);
+                }
+                fill_security_violation_target(name, (const byte*)&injected_code);
+            }
+
+            /* we allow -exempt_threat_list to override our action */
+            if (!IS_STRING_OPTION_EMPTY(exempt_threat_list)) {
+                if (is_exempt_threat_name(name)) {
+                    /* we want to ALLOW unconditionally so we don't
+                     * immediately get a regular .B violation after we
+                     * let it through the APC check
+                     */
+                    block = false;
+                }
+                /* FIXME: we don't have a good way to express allow
+                 * everyone except for the ones on this list while we
+                 * could say block = !block that doesn't match the
+                 * general meaning of exempt_threat_list
+                 */
+            }
+        }
+
+        if (block) {
+            /* always using custom attack handling */
+            /* We cannot let default attack handling take care
+             * of this because a main thread may get affected very early.
+             *
+             * It is also hard to reuse security_violation() call here
+             * (since we are not under dispatch()).  If we want to see a
+             * code origins failure, we can just disable this policy.
+             */
+            ASSERT(!TEST(OPTION_HANDLING, target_policy) && "handling cannot be modified");
+
+            SYSLOG_INTERNAL_WARNING("squashed %s %s at bad target pc="PFX" %s",
+                                    is_apc ? "APC" : "thread", 
+                                    is_win32api ? "win32" : "native",
+                                    injected_target, name);
+
+            /* FIXME: case 9023 : should squash appropriately native
+             * vs non-native since the number of arguments may be
+             * different, hence stdcall RET size 
+             */
+            *apc_target_location = 
+                is_win32api ? 
+                (app_pc)safe_apc_or_thread_target :
+                (app_pc)safe_apc_or_thread_target;
+
+            squashed = true;
+        } else {
+            /* allow */
+            app_pc base = (app_pc)PAGE_START(injected_target);
+            SYSLOG_INTERNAL_WARNING("allowing %s %s at bad target pc="PFX" %s",
+                                    is_apc ? "APC" : "thread", 
+                                    is_win32api ? "win32" : "native",
+                                    injected_target, name);
+
+            /* FIXME: for HIGH mode, unfortunately the target code
+             * may be selfmod, so adding a hook-style policy is hard.
+             */
+            /* FIXME: It looks like in VirusScan (case 2871) they
+             * eventually free this memory, so not that bad hole.
+             * Although I haven't found how would they properly
+             * synchronize that entapi.dll is loaded. */
+
+            /* we can't safely determine a subpage region so adding whole page */
+            add_futureexec_vm_area(base, base + PAGE_SIZE,
+                                   false/*permanent*/                                    
+                                   _IF_DEBUG(is_apc ? "apc_helper" : "thread_policy"));
+        }
+
+        if (TEST(OPTION_REPORT, target_policy)) {
+            /* report a violation adjusted for appropriate action */
+            /* FIXME: should come up with a new name for this
+             * violation otherwise it is pretty inconsistent to say
+             * running in detect mode and -B policies
+             */
+            /* note that we may not actually report if silent_block_threat_list */
+            security_violation_report(injected_target, APC_THREAD_SHELLCODE_VIOLATION, 
+                                      name,
+                                      squashed ? ACTION_TERMINATE_THREAD :
+                                                 ACTION_CONTINUE);
+        }
+
+        DOSTATS({
+            if (is_apc)
+                STATS_INC(num_used_apc_policy);
+            else
+                STATS_INC(num_used_thread_policy);
+        });
+    }
+}
+
+/* a helper procedure for reporting ASLR violations */
+void
+aslr_report_violation(app_pc execution_fault_pc,
+                      security_option_t handling_policy)
+{
+    STATS_INC(aslr_wouldbe_exec);
+
+    /* note OPTION_BLOCK has to be set since there is nothing we can
+     * do to not block the attack, there is no detect mode here, yet
+     * we let the original exception be passed.  For default
+     * applications where ASLR can be hit natively, the attack
+     * handling policy is to throw an exception.
+     */
+    ASSERT(TEST(OPTION_BLOCK, handling_policy));
+
+    /* FIXME: yet we should have a choice whether to override the
+     * exception that would normally be delivered to the application,
+     * with a -kill_thread or -kill_process in case the SEH chain is
+     * corrupt, and to allow the attack handling thresholds to take
+     * effect.
+     */
+    ASSERT(!TEST(OPTION_HANDLING, handling_policy));
+    /* FIXME: if using report security_violation() to provide attack
+     * handling decisions should make sure it prefers exceptions,
+     * FIXME: make sure not trying to release locks, FIXME: also clean
+     * kstats (currently hotp_only is already broken)
+     */
+
+    ASSERT(!TEST(OPTION_CUSTOM, handling_policy));
+
+    if (TEST(OPTION_REPORT, handling_policy)) {
+        /* report a violation, adjusted for appropriate action */
+        char aslr_threat_id[MAXIMUM_VIOLATION_NAME_LENGTH];
+
+        /* in -hotp_only mode cannot have the regular distinction
+         * between stack and heap targets (usually marked as .A and
+         * .B), instead marking all as the same .R violation.
+         */
+        security_violation_t aslr_violation_type = ASLR_TARGET_VIOLATION;
+
+        /* source cannot be obtained */
+        /* FIXME: case 8160 on possibly setting the source to something useful */
+
+        /* FIXME: target is currently unreadable, forensic and Threat
+         * ID generation will adjust to a likely current mapping to
+         * print its contents
+         */
+        dcontext_t *dcontext = get_thread_private_dcontext();
+
+        /* should be in hotp_only */
+        ASSERT(dcontext != NULL && dcontext->last_fragment != NULL && 
+               dcontext->last_fragment->tag == NULL);
+
+        /* note we clobber next_tag here, not bothering to preserve */
+        /* report_dcontext_info() uses next_tag for target (and
+         * preferred target) diagnostics */
+        dcontext->next_tag = execution_fault_pc;
+
+        /* if likely_target_pc is unreadable (and it should be)
+         * get_security_violation_name will use as target the contents
+         * of a likely would be target */
+        get_security_violation_name(dcontext, 
+                                    execution_fault_pc, 
+                                    aslr_threat_id, MAXIMUM_VIOLATION_NAME_LENGTH, 
+                                    aslr_violation_type, NULL);
+        security_violation_report(execution_fault_pc, 
+                                  aslr_violation_type,
+                                  aslr_threat_id,
+                                  ACTION_THROW_EXCEPTION);
+   }
+}
+#endif /* PROGRAM_SHEPHERDING */
+
+#ifdef VMAREAS_UNIT_TEST
+static void
+print_vector_msg(vm_area_vector_t *v, file_t f, char *msg)
+{
+    print_file(f, "%s:\n", msg);
+    print_vm_areas(v, f);
+}
+
+static void
+check_vec(vm_area_vector_t *v, int i, app_pc start, app_pc end,
+          uint vm_flags, uint frag_flags)
+{
+    ASSERT(i < v->length);
+    ASSERT(v->buf[i].start == start);
+    ASSERT(v->buf[i].end == end);
+    ASSERT(v->buf[i].vm_flags == vm_flags);
+    ASSERT(v->buf[i].frag_flags == frag_flags);
+}
+
+void
+vmvector_tests()
+{
+    vm_area_vector_t v = {0, 0, 0, VECTOR_SHARED | VECTOR_NEVER_MERGE, 
+                        INIT_READWRITE_LOCK(thread_vm_areas)};
+    bool res;
+    app_pc start, end;
+    /* FIXME: not tested */
+    vmvector_add(&v, (app_pc)0x100, (app_pc)0x103, NULL);
+    vmvector_add(&v, (app_pc)0x200, (app_pc)0x203, NULL);
+    vmvector_print(&v, STDERR);
+    vmvector_add(&v, (app_pc)0x202, (app_pc)0x210, NULL); /* should complain */
+    vmvector_add(&v, (app_pc)0x203, (app_pc)0x221, NULL);
+    vmvector_print(&v, STDERR);
+    check_vec(&v, 2, (app_pc)0x203, (app_pc)0x221, 0, 0);
+
+    res = vmvector_remove_containing_area(&v, (app_pc)0x103, NULL, NULL); /* not in */
+    EXPECT(res, false);
+    check_vec(&v, 0, (app_pc)0x100, (app_pc)0x103, 0, 0);
+    res = vmvector_remove_containing_area(&v, (app_pc)0x100, NULL, &end);
+    EXPECT(end, 0x103);
+    EXPECT(res, true);
+    vmvector_print(&v, STDERR);
+    check_vec(&v, 0, (app_pc)0x200, (app_pc)0x203, 0, 0);
+    res = vmvector_remove_containing_area(&v, (app_pc)0x100, NULL, NULL); /* not in */
+    EXPECT(res, false);
+    vmvector_print(&v, STDERR);
+    res = vmvector_remove_containing_area(&v, (app_pc)0x202, &start, NULL); /* not in */
+    EXPECT(res, false);
+    EXPECT(start, 0x200);
+    vmvector_print(&v, STDERR);
+    res = vmvector_remove(&v, (app_pc)0x20, (app_pc)0x210); /* truncation allowed? */
+    EXPECT(res, true);
+    vmvector_print(&v, STDERR);
+}
+
+/* initial vector tests
+ * FIXME: should add a lot more, esp. wrt other flags -- these only
+ * test no flags or interactions w/ selfmod flag
+ */
+int
+main() {
+    vm_area_vector_t v = {0,0,0,false};
+    /* not needed yet: dcontext_t *dcontext = */
+    standalone_init();
+    ASSIGN_INIT_READWRITE_LOCK_FREE(v.lock, thread_vm_areas);
+
+    /* TEST 1: merge a bunch of areas
+     */
+    add_vm_area(&v, (app_pc)1, (app_pc)3, 0, 0 _IF_DEBUG("A"));
+    add_vm_area(&v, (app_pc)5, (app_pc)7, 0, 0 _IF_DEBUG("B"));
+    add_vm_area(&v, (app_pc)9, (app_pc)11, 0, 0 _IF_DEBUG("C"));
+    print_vector_msg(&v, STDERR, "after adding areas");
+    check_vec(&v, 0, (app_pc)1, (app_pc)3, 0, 0);
+    check_vec(&v, 1, (app_pc)5, (app_pc)7, 0, 0);
+    check_vec(&v, 2, (app_pc)9, (app_pc)11, 0, 0);
+
+    add_vm_area(&v, (app_pc)0, (app_pc)12, 0, 0 _IF_DEBUG("D"));
+    print_vector_msg(&v, STDERR, "after merging with D");
+    check_vec(&v, 0, (app_pc)0, (app_pc)12, 0, 0);
+
+    /* clear for next test */
+    remove_vm_area(&v, (app_pc)0, UNIVERSAL_REGION_END, false);
+    print_file(STDERR, "\n");
+
+    /* TEST 2: add an area that covers several smaller ones, including one
+     * that cannot be merged
+     */
+    add_vm_area(&v, (app_pc)1, (app_pc)3, 0, 0 _IF_DEBUG("A"));
+    add_vm_area(&v, (app_pc)5, (app_pc)7, 0, FRAG_SELFMOD_SANDBOXED _IF_DEBUG("B"));
+    add_vm_area(&v, (app_pc)9, (app_pc)11, 0, 0 _IF_DEBUG("C"));
+    print_vector_msg(&v, STDERR, "after adding areas");
+    check_vec(&v, 0, (app_pc)1, (app_pc)3, 0, 0);
+    check_vec(&v, 1, (app_pc)5, (app_pc)7, 0, FRAG_SELFMOD_SANDBOXED);
+    check_vec(&v, 2, (app_pc)9, (app_pc)11, 0, 0);
+
+    add_vm_area(&v, (app_pc)2, (app_pc)10, 0, 0 _IF_DEBUG("D"));
+    print_vector_msg(&v, STDERR, "after merging with D");
+    check_vec(&v, 0, (app_pc)1, (app_pc)5, 0, 0);
+    check_vec(&v, 1, (app_pc)5, (app_pc)7, 0, FRAG_SELFMOD_SANDBOXED);
+    check_vec(&v, 2, (app_pc)7, (app_pc)11, 0, 0);
+
+    remove_vm_area(&v, (app_pc)6, (app_pc)8, false);
+    print_vector_msg(&v, STDERR, "after removing 6-8");
+    check_vec(&v, 0, (app_pc)1, (app_pc)5, 0, 0);
+    check_vec(&v, 1, (app_pc)5, (app_pc)6, 0, FRAG_SELFMOD_SANDBOXED);
+    check_vec(&v, 2, (app_pc)8, (app_pc)11, 0, 0);
+
+    /* clear for next test */
+    remove_vm_area(&v, (app_pc)0, UNIVERSAL_REGION_END, false);
+    print_file(STDERR, "\n");
+
+    /* TEST 3: add an area that covers several smaller ones, including two
+     * that cannot be merged
+     */
+    add_vm_area(&v, (app_pc)1, (app_pc)3, 0, FRAG_SELFMOD_SANDBOXED _IF_DEBUG("A"));
+    add_vm_area(&v, (app_pc)5, (app_pc)7, 0, FRAG_SELFMOD_SANDBOXED _IF_DEBUG("B"));
+    add_vm_area(&v, (app_pc)9, (app_pc)11, 0, 0 _IF_DEBUG("C"));
+    print_vector_msg(&v, STDERR, "after adding areas");
+    check_vec(&v, 0, (app_pc)1, (app_pc)3, 0, FRAG_SELFMOD_SANDBOXED);
+    check_vec(&v, 1, (app_pc)5, (app_pc)7, 0, FRAG_SELFMOD_SANDBOXED);
+    check_vec(&v, 2, (app_pc)9, (app_pc)11, 0, 0);
+
+    add_vm_area(&v, (app_pc)2, (app_pc)12, 0, 0 _IF_DEBUG("D"));
+    print_vector_msg(&v, STDERR, "after merging with D");
+    check_vec(&v, 0, (app_pc)1, (app_pc)3, 0, FRAG_SELFMOD_SANDBOXED);
+    check_vec(&v, 1, (app_pc)3, (app_pc)5, 0, 0);
+    check_vec(&v, 2, (app_pc)5, (app_pc)7, 0, FRAG_SELFMOD_SANDBOXED);
+    check_vec(&v, 3, (app_pc)7, (app_pc)12, 0, 0);
+
+    remove_vm_area(&v, (app_pc)2, (app_pc)11, false);
+    print_vector_msg(&v, STDERR, "after removing 2-11");
+    check_vec(&v, 0, (app_pc)1, (app_pc)2, 0, FRAG_SELFMOD_SANDBOXED);
+    check_vec(&v, 1, (app_pc)11, (app_pc)12, 0, 0);
+
+    /* FIXME: would be nice to be able to test that an assert is generated... 
+     * say, for this:
+     * add_vm_area(&v, (app_pc)7, (app_pc)12, 0, FRAG_SELFMOD_SANDBOXED _IF_DEBUG("E"));
+     */
+
+    /* clear for next test */
+    remove_vm_area(&v, (app_pc)0, UNIVERSAL_REGION_END, false);
+    print_file(STDERR, "\n");
+
+    /* TEST 4: add an area completely inside one that cannot be merged
+     */
+    add_vm_area(&v, (app_pc)1, (app_pc)5, 0, FRAG_SELFMOD_SANDBOXED _IF_DEBUG("A"));
+    print_vector_msg(&v, STDERR, "after adding areas");
+    check_vec(&v, 0, (app_pc)1, (app_pc)5, 0, FRAG_SELFMOD_SANDBOXED);
+
+    add_vm_area(&v, (app_pc)3, (app_pc)4, 0, 0 _IF_DEBUG("B"));
+    print_vector_msg(&v, STDERR, "after merging with B");
+    check_vec(&v, 0, (app_pc)1, (app_pc)5, 0, FRAG_SELFMOD_SANDBOXED);
+
+    vmvector_tests();
+
+    return 0;
+}
+#endif
diff --git a/core/vmareas.h b/core/vmareas.h
new file mode 100644
index 000000000..311885ee3
--- /dev/null
+++ b/core/vmareas.h
@@ -0,0 +1,755 @@
+/* **********************************************************
+ * Copyright (c) 2002-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+
+/*
+ * vmareas.h - virtual memory areas
+ */
+
+#ifndef _VMAREAS_H_
+#define _VMAREAS_H_ 1
+
+/* case 9750: use these constants to specify empty and all-encompassing address
+ * space regions for flushing, etc.
+ */
+#define EMPTY_REGION_BASE  ((app_pc)PTR_UINT_1) /* avoid [0,0) == wrapped universal */
+#define EMPTY_REGION_SIZE  ((size_t)0U)
+#define UNIVERSAL_REGION_BASE  ((app_pc)NULL)
+/* really open-ended should have this 1 larger */
+#define UNIVERSAL_REGION_SIZE  POINTER_MAX
+/* really open-ended would make this wrap around to 0 */
+#define UNIVERSAL_REGION_END   ((app_pc)POINTER_MAX)
+
+/* opaque struct */
+struct vm_area_t;
+
+enum {
+    /* these are bitmask flags */
+    VECTOR_SHARED        = 0x0001,
+    VECTOR_FRAGMENT_LIST = 0x0002, /* for vmareas.c-only use */
+    VECTOR_NEVER_MERGE   = 0x0004,
+    /* Used as a qualifier for VECTOR_NEVER_MERGE: while a merge of
+     * adjacent areas is not desired, data does not contain critical
+     * information.  Overlapping regions will preserve the data from
+     * an arbitrary region's version, and data is not guaranteed to be
+     * preserved if any of these regions is removed.  
+     */
+    VECTOR_OVERLAP_NON_CRITICAL = 0x0008,
+    /* case 10335: if a higher-level lock is being used, set this
+     * flag to avoid the redundant vector-level lock
+     */
+    VECTOR_NO_LOCK       = 0x0010,
+};
+
+/* This vector data structure is only exposed here for quick length checks.
+ * For external (non-vmareas.c) users, the vmvector_* interface is the
+ * preferred way of manipulating vectors.
+ *
+ * Each vector is kept sorted by area.  Since there are no overlaps allowed
+ * among areas in the same vector (they're merged to preserve that), sorting
+ * by start_pc or by end_pc produce identical results.
+ */
+struct vm_area_vector_t {
+    struct vm_area_t *buf;
+    int size; /* capacity */
+    int length;
+    uint flags; /* VECTOR_* flags */
+    /* often thread-shared, so needs a lock
+     * read-write lock for performance, and to allow a high-level writer
+     * to perform a read (don't need full recursive lock)
+     */
+    read_write_lock_t lock;
+}; /* typedef-ed in globals.h */
+
+/* vm_area_vectors should NOT be declared statically if their locks need to be
+ * accessed on a regular basis.  Instead, allocate them on the heap with this macro:
+ */
+#define VMVECTOR_ALLOC_VECTOR(v, dc, flags, lockname) do {    \
+        v = vmvector_create_vector((dc), (flags));            \
+        ASSIGN_INIT_READWRITE_LOCK_FREE((v)->lock, lockname); \
+    } while (0);
+
+/* iterator over a vm_area_vector_t */
+typedef struct vmvector_iterator_t {
+    vm_area_vector_t *vector;
+    int index;
+} vmvector_iterator_t;
+
+/* rather than exporting specialized routines for just these vectors we
+ * export the vectors and general routines
+ */
+extern vm_area_vector_t *native_exec_areas;
+extern vm_area_vector_t *emulate_write_areas;
+
+extern vm_area_vector_t *patch_proof_areas;
+
+/* IAT or GOT areas of all mapped DLLs except for those in
+ * native_exec_areas - note the exact regions are added here */
+extern vm_area_vector_t *IAT_areas; 
+
+extern mutex_t shared_delete_lock;
+
+/* operations on the opaque vector struct */
+void
+vmvector_print(vm_area_vector_t *v, file_t outf);
+
+void
+vmvector_add(vm_area_vector_t *v, app_pc start, app_pc end, void *data);
+
+bool
+vmvector_remove(vm_area_vector_t *v, app_pc start, app_pc end);
+
+bool
+vmvector_remove_containing_area(vm_area_vector_t *v, app_pc pc, 
+                                app_pc *area_start /* OUT optional */, 
+                                app_pc *area_end /* OUT optional */);
+
+static inline bool
+vmvector_empty(vm_area_vector_t *v)
+{
+    ASSERT(v != NULL);
+    if (v == NULL) /* defensive programming */
+        return true;
+    else
+        return (v->length == 0);
+}
+
+bool
+vmvector_overlap(vm_area_vector_t *v, app_pc start, app_pc end);
+
+void *
+vmvector_lookup(vm_area_vector_t *v, app_pc pc);
+
+/* this routine does NOT initialize the rw lock!  use VMVECTOR_INITIALIZE_VECTOR */
+void
+vmvector_init_vector(vm_area_vector_t *v, uint flags);
+
+/* this routine does NOT initialize the rw lock!  use VMVECTOR_ALLOC_VECTOR instead */
+vm_area_vector_t *
+vmvector_create_vector(dcontext_t *dcontext, uint flags);
+
+bool
+vmvector_lookup_data(vm_area_vector_t *v, app_pc pc, app_pc *start, app_pc *end,
+                     void **data);
+
+bool
+vmvector_modify_data(vm_area_vector_t *v, app_pc start, app_pc end, void *data);
+
+void
+vmvector_delete_vector(dcontext_t *dcontext, vm_area_vector_t *v);
+
+void
+vmvector_reset_vector(dcontext_t *dcontext, vm_area_vector_t *v);
+
+/* vmvector iterator.  Initialized with vmvector_iterator_start(), and
+ * destroyed with vmvector_iterator_stop() to release vector lock.
+ * Accessor vmvector_iterator_next() should be called only when
+ * predicate vmvector_iterator_hasnext() is true.  No add/remove
+ * mutations are allowed, although only shared vectors will detect that.
+ */
+void
+vmvector_iterator_start(vm_area_vector_t *v, vmvector_iterator_t *vmvi);
+bool
+vmvector_iterator_hasnext(vmvector_iterator_t *vmvi);
+void*
+vmvector_iterator_next(vmvector_iterator_t *vmvi, /* IN/OUT */
+                       app_pc *area_start /* OUT */, app_pc *area_end /* OUT */);
+void
+vmvector_iterator_stop(vmvector_iterator_t *vmvi);
+
+/* called earlier than vm_areas_init() to allow add_dynamo_vm_area to be called */
+void
+dynamo_vm_areas_init(void);
+
+/* initialize per process */
+int
+vm_areas_init(void);
+
+/* cleanup and print statistics per process */
+int
+vm_areas_exit(void);
+
+/* thread-shared initialization that should be repeated after a reset */
+void
+vm_areas_reset_init(void);
+
+/* Free all thread-shared state not critical to forward progress;
+ * vm_areas_reset_init() will be called before continuing.
+ */
+void
+vm_areas_reset_free(void);
+
+void
+vm_area_coarse_units_reset_free(void);
+
+void
+vm_area_coarse_units_freeze(bool in_place);
+
+/* print list of all currently allowed executable areas */
+void
+print_executable_areas(file_t outf);
+
+#ifdef PROGRAM_SHEPHERDING
+/* print list of all future-exec areas */
+void
+print_futureexec_areas(file_t outf);
+#endif
+
+/* print list of all current dynamo areas */
+void
+print_dynamo_areas(file_t outf);
+
+/* initialize per thread */
+void
+vm_areas_thread_init(dcontext_t *dcontext);
+
+/* cleanup per thread */
+void
+vm_areas_thread_exit(dcontext_t *dcontext);
+
+/* re-initializes non-persistent memory */
+void
+vm_areas_thread_reset_init(dcontext_t *dcontext);
+
+/* frees all non-persistent memory */
+void
+vm_areas_thread_reset_free(dcontext_t *dcontext);
+
+/* lookup an address against the per-process executable map */
+bool
+is_executable_address(app_pc addr);
+
+/* if addr is an executable area, returns true and returns in *flags
+ *   any FRAG_ flags associated with addr's vm area 
+ * returns false if area not found 
+ */
+bool
+get_executable_area_flags(app_pc addr, uint *frag_flags);
+
+/* returns any VM_ flags associated with addr's vm area 
+ * returns 0 if no area is found
+*/
+bool
+get_executable_area_vm_flags(app_pc addr, uint *vm_flags);
+
+/* For coarse-grain operation, we use a separate cache and htable per region.
+ * See coarse_info_t notes on synchronization model.
+ */
+coarse_info_t *
+get_executable_area_coarse_info(app_pc addr);
+
+/* Marks the executable_areas region corresponding to info as frozen. */
+void
+mark_executable_area_coarse_frozen(coarse_info_t *info);
+
+/* returns true if addr is on a page that was marked writable by the application
+ * but that we marked RO b/c it contains executable code
+ * does NOT check if addr is executable, only that something on its page is!
+ */
+bool
+is_executable_area_writable(app_pc addr);
+
+bool
+was_executable_area_writable(app_pc addr);
+
+/* returns false if addr is not in an executable area that
+ * contains self-modifying code */
+bool
+is_executable_area_selfmod(app_pc addr);
+
+#ifdef DGC_DIAGNOSTICS
+/* returns false if addr is not in an executable area marked as dyngen */
+bool
+is_executable_area_dyngen(app_pc addr);
+#endif
+
+#ifdef PROGRAM_SHEPHERDING
+/* returns true if addr is in the dyngen syscall area of XP+/RH9+ */
+bool
+is_dyngen_vsyscall(app_pc addr);
+
+/* returns true if addr is a DGC,
+   Note either executed or would be allowed in the future */
+bool
+is_dyngen_code(app_pc addr);
+
+bool
+is_in_futureexec_area(app_pc addr);
+#endif
+
+/* lookup an address against the per-process map
+ * used to distinguish security violations from other segmentation faults
+ */
+bool
+is_valid_address(app_pc addr);
+
+/* Used for DR heap area changes as circular dependences prevent
+ * directly adding or removing DR vm areas.
+ * Must hold the DR areas lock across the combination of calling this and
+ * modifying the heap lists.
+ * Used to determine when we need to do another heap walk to keep
+ * dynamo vm areas up to date (can't do it incrementally b/c of
+ * circular dependencies)
+ */
+void
+mark_dynamo_vm_areas_stale(void);
+
+void
+dynamo_vm_areas_lock(void);
+
+void
+dynamo_vm_areas_unlock(void);
+
+bool
+self_owns_dynamo_vm_area_lock(void);
+
+/* adds dynamo area but doesn't grab lock -- only use this when called to
+ * update heap areas!
+ */
+bool
+add_dynamo_heap_vm_area(app_pc start, app_pc end, bool writable, bool from_unmod_image
+                        _IF_DEBUG(char *comment));
+
+/* assumes caller holds a private vmareas lock --
+ * only for use by heap_vmareas_synch_units()
+ */
+void 
+remove_dynamo_heap_areas(void);
+
+/* used by detach to fixup the permissions of any app writable regions that
+ * dynamorio has marked read only for its cache consistency effort */
+void
+revert_memory_regions(void);
+
+/* returns true if address is inside some dynamo-internal vm area
+ * note that this includes not only dynamo library executable regions but
+ * also all memory allocated by us, including the code cache!
+ */
+bool
+is_dynamo_address(app_pc addr);
+
+/* check if any features that need pretend writable areas are enabled,
+ * in default product configuration should always be true 
+ */
+#define USING_PRETEND_WRITABLE()                                \
+        (DYNAMO_OPTION(handle_DR_modify) == DR_MODIFY_NOP ||    \
+         DYNAMO_OPTION(handle_ntdll_modify) == DR_MODIFY_NOP || \
+         !IS_STRING_OPTION_EMPTY(patch_proof_list) ||           \
+         !IS_STRING_OPTION_EMPTY(patch_proof_default_list))
+
+/* returns true iff address is an address that the app thinks is writable
+ * but really is not, as it overlaps DR memory
+ */
+bool
+is_pretend_writable_address(app_pc addr);
+
+#ifdef DEBUG
+/* returns comment for addr, if there is one, else NULL
+ */
+char *
+get_address_comment(app_pc addr);
+#endif
+
+/* returns true if the passed in area overlaps any known executable areas */
+bool
+executable_vm_area_overlap(app_pc start, app_pc end, bool have_writelock);
+
+/* Returns true if region [start, end) overlaps any regions that are
+ * marked as FRAG_COARSE_GRAIN.
+ */
+bool
+executable_vm_area_coarse_overlap(app_pc start, app_pc end);
+
+/* Returns true if region [start, end) overlaps any regions that are
+ * marked as VM_PERSISTED_CACHE.
+ */
+bool
+executable_vm_area_persisted_overlap(app_pc start, app_pc end);
+
+/* Returns true if any part of region [start, end) has ever been executed from
+ * (including having ibl targets loaded from a persistent cache)
+ */
+bool
+executable_vm_area_executed_from(app_pc start, app_pc end);
+
+/* If there is no overlap between executable_areas and [start,end), returns false.
+ * Else, returns true and sets [overlap_start,overlap_end) as the bounds of the first
+ * and last executable_area regions that overlap [start,end); i.e.,
+ *   overlap_start starts the first area that overlaps [start,end);
+ *   overlap_end ends the last area that overlaps [start,end).
+ * Thus, overlap_start may be > start and overlap_end may be < end.
+ *
+ * If frag_flags != 0, the region described above is expanded such that the regions
+ * before and after [overlap_start,overlap_end) do NOT match
+ * [overlap_start,overlap_end) in TESTALL of frag_flags, but only considering
+ * non-contiguous regions if !contig.
+ */
+bool
+executable_area_overlap_bounds(app_pc start, app_pc end,
+                               app_pc *overlap_start/*OUT*/, app_pc *overlap_end/*OUT*/,
+                               uint frag_flags, bool contig);
+
+/* Coarse unit iterator with support for vm area start and end bounds */
+void
+vm_area_coarse_iter_start(vmvector_iterator_t *vmvi, app_pc start);
+bool
+vm_area_coarse_iter_hasnext(vmvector_iterator_t *vmvi, app_pc end);
+/* Note that NULL is a valid return value for intermediate iterations
+ * if that coarse region has not yet been executed from, due to our
+ * new lazy init scheme (case 8640).
+ * FIXME: may need to return region bounds or something if have other callers.
+ */
+coarse_info_t *
+vm_area_coarse_iter_next(vmvector_iterator_t *vmvi, app_pc end);
+void
+vm_area_coarse_iter_stop(vmvector_iterator_t *vmvi);
+
+void
+executable_areas_lock(void);
+
+void
+executable_areas_unlock(void);
+
+/* returns true if the passed in area overlaps any dynamo areas */
+bool
+dynamo_vm_area_overlap(app_pc start, app_pc end);
+
+/* Used to add dr allocated memory regions that may execute out of the cache */
+/* NOTE : region is assumed to not be writable, caller is responsible for
+ * ensuring this (see fixme in signal.c adding sigreturn code)
+ */
+bool
+add_executable_region(app_pc start, size_t size _IF_DEBUG(char *comment));
+
+/* removes a region from the executable list */
+/* NOTE :the caller is responsible for ensuring that all threads' local 
+ * vm lists are updated by calling flush_fragments_and_remove_region
+ */
+bool
+remove_executable_region(app_pc start, size_t size, bool have_writelock);
+
+bool
+free_nonexec_coarse_and_unlock(void);
+
+/* add dynamo-internal area to the dynamo-internal area list */
+bool
+add_dynamo_vm_area(app_pc start, app_pc end, uint prot, bool from_unmod_image
+                   _IF_DEBUG(char *comment));
+bool
+is_dynamo_area_buffer(byte *heap_pc);
+
+/* remove dynamo-internal area from the dynamo-internal area list */
+bool
+remove_dynamo_vm_area(app_pc start, app_pc end);
+
+#ifdef PROGRAM_SHEPHERDING  /* Fix for case 5061.  See case 5075. */
+extern const char * const action_message[];
+
+/* be sure to keep this enum and the two arrays, action_message[] & 
+ * action_event_id[] located in vmareas.c, in synch
+ */
+typedef enum {
+    ACTION_TERMINATE_PROCESS,
+    ACTION_CONTINUE,            /* detect mode */
+    ACTION_TERMINATE_THREAD,
+    ACTION_THROW_EXCEPTION,
+} action_type_t;
+
+/* Wrapper for security_violation_internal. */
+security_violation_t
+security_violation(dcontext_t *dcontext, app_pc addr, 
+                   security_violation_t violation_type, 
+                   security_option_t type_handling);
+
+/* attack handling - reports violation and possibly terminates the process */
+security_violation_t
+security_violation_internal(dcontext_t *dcontext, app_pc addr, 
+                            security_violation_t violation_type, 
+                            security_option_t type_handling, const char *threat_id,
+                            const action_type_t desired_action,
+                            read_write_lock_t *lock);
+
+/* returns whether it ended up deleting the self-writing fragment
+ * by flushing the region
+ */
+bool
+vm_area_fragment_self_write(dcontext_t *dcontext, app_pc tag);
+
+# ifdef WINDOWS
+#  define USING_FUTURE_EXEC_LIST (dynamo_options.executable_if_alloc ||  \
+                                  dynamo_options.executable_if_x     ||  \
+                                  dynamo_options.executable_if_flush ||  \
+                                  dynamo_options.executable_if_hook)
+# else
+#  define USING_FUTURE_EXEC_LIST (dynamo_options.executable_if_alloc ||  \
+                                  dynamo_options.executable_if_x)
+# endif
+
+#endif
+
+/* newly allocated or mapped in memory region */
+bool
+app_memory_allocation(dcontext_t *dcontext, app_pc base, size_t size,
+                      uint prot, bool image _IF_DEBUG(char *comment));
+
+/* de-allocated or un-mapped memory region */
+void
+app_memory_deallocation(dcontext_t *dcontext, app_pc base, size_t size, 
+                        bool own_initexit_lock, bool image);
+
+/* memory region base:base+size now has privileges prot
+ * returns one of the following codes
+ */
+enum {
+    DO_APP_MEM_PROT_CHANGE,   /* Let the system call execute normally */
+    FAIL_APP_MEM_PROT_CHANGE, /* skip the system call and return a 
+                               * failure code to the app */
+    PRETEND_APP_MEM_PROT_CHANGE, /* skip the system call but return success */
+    SUBSET_APP_MEM_PROT_CHANGE,  /* make a system call with modified protection,
+                                  * expect to return success, 
+                                  */
+};
+
+/* values taken by the option handle_DR_modify and handle_ntdll_modify
+ * specifies how to handle app attempts to modify DR memory protection:
+ * either halt with an error, return failure to the app, or turn into a nop
+ */
+enum {
+    /* these are mutually exclusive */
+    DR_MODIFY_HALT  = 0, /* throw an internal error at the mem prot attempt */
+    DR_MODIFY_NOP   = 1, /* turn the mem prot and later write faults into nops */
+    DR_MODIFY_FAIL  = 2, /* have the mem prot fail */
+    DR_MODIFY_ALLOW = 3, /* let the app muck with us -- WARNING: use at own risk */
+    DR_MODIFY_OFF   = 4, /* we don't even check for attempts -- WARNING: use at own risk */
+};
+
+uint
+app_memory_protection_change(dcontext_t *dcontext, app_pc base, size_t size, 
+                             uint prot, /* platform independent MEMPROT_ */
+                             uint *new_memprot, /* OUT */
+                             uint *old_memprot /* OPTIONAL OUT*/);
+
+#ifdef WINDOWS
+/* memory region base:base+size was flushed from hardware icache by app */
+void app_memory_flush(dcontext_t *dcontext, app_pc base, size_t size, uint prot);
+# ifdef PROGRAM_SHEPHERDING
+/* was pc ever the start pc arg to a flush request? */
+bool was_address_flush_start(dcontext_t *dcontext, app_pc pc);
+# endif
+#endif
+
+/* we keep a list of vm areas per thread, to make flushing fragments
+ * due to memory unmaps faster
+ * This routine adds the page containing start to the thread's list.
+ * Adds any FRAG_ flags relevant for a fragment overlapping start's page.
+ * If xfer and encounters change in vmareas flags, returns false and does NOT
+ * add the new page to the list for this fragment -- assumes caller will NOT add
+ * it to the current bb.  This allows for selectively not following direct ctis.
+ */
+bool
+check_thread_vm_area(dcontext_t *dcontext, app_pc start, app_pc tag, void **vmlist,
+                     uint *flags, app_pc *stop, bool xfer);
+
+void
+check_thread_vm_area_abort(dcontext_t *dcontext, void **vmlist, uint flags);
+
+/* Indicate the PC of the page in which instructions are about to be decoded. */
+void
+set_thread_decode_page_start(dcontext_t *dcontext, app_pc page_start);
+
+bool
+check_in_last_thread_vm_area(dcontext_t *dcontext, app_pc pc);
+
+void
+vm_area_add_fragment(dcontext_t *dcontext, fragment_t *f, void *vmlist);
+
+void
+acquire_vm_areas_lock(dcontext_t *dcontext, uint fragment_flags);
+
+void
+release_vm_areas_lock(dcontext_t *dcontext, uint fragment_flags);
+
+bool
+vm_area_add_to_list(dcontext_t *dcontext, app_pc tag, void **vmlist,
+                    uint list_flags, fragment_t *f, bool have_locks);
+
+void
+vm_area_destroy_list(dcontext_t *dcontext, void *vmlist);
+
+bool
+vm_list_overlaps(dcontext_t *dcontext, void *vmlist, app_pc start, app_pc end);
+
+void
+vm_area_remove_fragment(dcontext_t *dcontext, fragment_t *f);
+
+/* Prepares a list of shared fragments for deletion..
+ * Caller should have already called vm_area_remove_fragment() on
+ * each and chained them together via next_vmarea.
+ * Caller must hold the shared_cache_flush_lock.
+ * Returns the number of fragments unlinked 
+ */
+int
+unlink_fragments_for_deletion(dcontext_t *dcontext, fragment_t *list,
+                              int pending_delete_threads);
+
+/* returns the number of fragments unlinked */
+int
+vm_area_unlink_fragments(dcontext_t *dcontext, app_pc start, app_pc end,
+                         int pending_delete_threads
+                         _IF_DGCDIAG(app_pc written_pc));
+
+/* removes incoming links for all private fragments in the dcontext
+ * thread that contain 'pc' 
+ */
+void
+vm_area_unlink_incoming(dcontext_t *dcontext, app_pc pc);
+
+/* Flushes thread-private pending-deletion fragments.
+ * Returns false iff was_I_flushed ends up being deleted.
+ */
+bool
+vm_area_flush_fragments(dcontext_t *dcontext, fragment_t *was_I_flushed);
+
+/* Decrements ref counts for thread-shared pending-deletion fragments,
+ * and deletes those whose count has reached 0.
+ * Returns false iff was_I_flushed has been flushed (not necessarily
+ * fully freed yet though, but may be at any time after this call
+ * returns, so caller should drop its ref to it).
+ */
+bool
+vm_area_check_shared_pending(dcontext_t *dcontext, fragment_t *was_I_flushed);
+
+/* Assumes that all threads are suspended at safe synch points.
+ * Deletes all fine fragments and coarse units in [start, end).
+ */
+void
+vm_area_allsynch_flush_fragments(dcontext_t *dcontext, dcontext_t *del_dcontext,
+                                 app_pc start, app_pc end, bool exec_invalid,
+                                 bool all_synched);
+
+/* adds the list of fragments beginning with f and chained by {next,prev}_vmarea
+ * to a new pending-lazy-deletion entry
+ */
+void
+add_to_lazy_deletion_list(dcontext_t *dcontext, fragment_t *f);
+
+bool
+remove_from_lazy_deletion_list(dcontext_t *dcontext, fragment_t *remove);
+
+/* returns true if the passed in area overlaps any vm area of given thread */
+bool
+thread_vm_area_overlap(dcontext_t *dcontext, app_pc start, app_pc end);
+
+/* Returns NULL if should re-execute the faulting write
+ * Else returns the target pc for a new basic block -- caller should
+ * return to dispatch rather than the code cache
+ */
+app_pc
+handle_modified_code(dcontext_t *dcontext, cache_pc instr_cache_pc,
+                     app_pc instr_app_pc, app_pc target);
+
+/* Returns the counter a selfmod fragment should execute for -sandbox2ro_threshold */
+uint *
+get_selfmod_exec_counter(app_pc tag);
+
+/* Returns true if f has been flushed */
+bool
+vm_area_selfmod_check_clear_exec_count(dcontext_t *dcontext, fragment_t *f);
+
+/* return true if the address is determined to be on the application's stack
+ * otherwise return false
+ */
+bool
+is_address_on_stack(dcontext_t *dcontext, app_pc address);
+
+/* returns true if an executable area exists with VM_DRIVER_ADDRESS,
+ * not a strict opposite of is_user_address() */
+bool
+is_driver_address(app_pc addr);
+
+
+/* FIXME clean up: safe_apc_or_thread_target, apc_thread_policy_helper and
+ * aslr_report_violation should all be ifdef WINDOWS, and may be in a
+ * different file
+ */
+#ifdef PROGRAM_SHEPHERDING
+#  ifdef WINDOWS
+#    define APC_API __stdcall
+#  else  /* !WINDOWS */
+#    define APC_API
+#  endif
+void
+APC_API
+safe_apc_or_thread_target(reg_t arg);
+#endif /* PROGRAM_SHEPHERDING */
+
+/* a helper procedure for DYNAMO_OPTION(apc_policy) or
+ * DYNAMO_OPTION(thread_policy) 
+ */
+typedef enum {
+    APC_TARGET_NATIVE, 
+    APC_TARGET_WINDOWS, 
+    THREAD_TARGET_NATIVE, 
+    THREAD_TARGET_WINDOWS
+} apc_thread_type_t; 
+
+void
+apc_thread_policy_helper(app_pc *apc_target_location, /* IN/OUT */
+                         security_option_t target_policy,
+                         apc_thread_type_t target_type);
+
+/* a helper procedure for reporting ASLR violations */
+void
+aslr_report_violation(app_pc execution_fault_pc,
+                      security_option_t handling_policy);
+
+/* tamper resistant areas used for handle_ntdll_modify */
+void
+tamper_resistant_region_add(app_pc start, app_pc end);
+
+bool
+tamper_resistant_region_overlap(app_pc start, app_pc end);
+
+void
+mark_unload_start(app_pc module_base, size_t module_size);
+void
+mark_unload_end(app_pc module_base);
+void
+print_last_deallocated(file_t outf);
+bool
+is_unreadable_or_currently_unloaded_region(app_pc pc);
+bool
+is_in_last_unloaded_region(app_pc pc);
+
+#endif /* _VMAREAS_H_ */
diff --git a/core/win32/Makefile b/core/win32/Makefile
new file mode 100644
index 000000000..9c650bfb6
--- /dev/null
+++ b/core/win32/Makefile
@@ -0,0 +1,45 @@
+# **********************************************************
+# Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# Copyright (c) 2003-2007 Determina Corp.
+
+# local submodule Makefile wrapper
+#   keep in sync the copies in win32 and linux
+# execute any rule using the top level makefile
+
+# You can actually override any rule here, 
+# but currently there is no reason to do so
+
+all:
+
+# for any action we go to the master Makefile
+%: force
+	@$(MAKE) -C .. $@
+force: ;
diff --git a/core/win32/aslr.c b/core/win32/aslr.c
new file mode 100644
index 000000000..19d5fe7f6
--- /dev/null
+++ b/core/win32/aslr.c
@@ -0,0 +1,6963 @@
+/* **********************************************************
+ * Copyright (c) 2005-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2005-2007 Determina Corp. */
+
+/*
+ * aslr.c - ASLR: address space layout randomization from user mode
+ */
+
+#include "globals.h"
+#include "ntdll.h"
+#include "os_private.h"
+#include "aslr.h"
+#include "instr.h"      /* for instr_t and OP_* opcodes */
+#include "decode.h"     /* for decode_opcode */
+#ifdef GBOP
+# include "gbop.h"
+# include "fragment.h"
+# include "hotpatch.h"
+#endif
+#include "module_shared.h"
+
+enum {ASLR_MAP_GRANULARITY = 64*1024}; /* 64KB - OS allocation granularity */
+
+/* A signature appended to relocated files in our DLL cache providing
+ * stronger consistency check between source and target.
+ * 
+ * Note that this uses another page or sector on disk but at least we
+ * don't waste another file and directory entry and any additional
+ * security descriptors.  Raw reads of data after the end of a
+ * MEM_IMAGE may result in a new file cache mapping, yet unlikely to
+ * be worse in performance or memory than having a separate
+ * file. FIXME: should measure
+ */
+typedef struct {
+    module_digest_t original_source;
+    module_digest_t relocated_target;
+
+    /* minimal file corruption check.  A mismatched signature is most
+     * likely due to version mismatch, or power failure.  Note that we
+     * do not require guaranteed order of flushing, so a matching
+     * signature doesn't guarantee that the previous blocks are
+     * consistently written.  We must maintain internal file
+     * consistency by making sure that any failed file write
+     * immediately terminates further work, such incomplete file
+     * prefixes should never be published under well-known name.
+     * 
+     */
+    uint magic;
+    /* although old files should be invalidated anyways, in case we'd
+     * want to report suspiciously corrupt files we better be sure
+     * we're not matching against the wrong version.
+     */
+    uint version;
+    /* do not add any fields after version - it has to be last word in file */
+} aslr_persistent_digest_t;
+
+/* version number for file signature */
+enum {ASLR_PERSISTENT_CACHE_VERSION = 1};
+/* magic footer: ADPE */
+enum {ASLR_PERSISTENT_CACHE_MAGIC = 0x45504441};
+
+/* all ASLR state protected by this lock */
+DECLARE_CXTSWPROT_VAR(static mutex_t aslr_lock, INIT_LOCK_FREE(aslr_lock));
+
+/* We keep these vars on the heap for selfprot (case 8074). */
+typedef struct _aslr_last_dll_bounds_t {
+    app_pc end;
+    /* used by ASLR_RANGE_BOTTOM_UP to capture failures 
+     * FIXME: should allow UnmapViewOfSection to rewind last DLL */
+    app_pc start;
+} aslr_last_dll_bounds_t;
+
+static aslr_last_dll_bounds_t *aslr_last_dll_bounds;
+
+/* FIXME: case 6739 to properly keep track on UnmapViewOfSection we
+ * should either QueryMemory for the jitter block or keep preceding
+ * padding plus the modules we've bumped into a vmarea.
+ */
+
+/* FIXME: ASLR_RANGE_TOP_DOWN needs aslr_last_dll_bounds->start and not the end */
+
+/* used for ASLR_TRACK_AREAS and ASLR_AVOID_AREAS.  Tracks preferred
+ * address ranges where a DLL would usually be located without ASLR.
+ * data is base of current mapping of rebased DLL that would be in that area
+ * Kept on the heap for selfprot (case 7957).
+ */
+vm_area_vector_t *aslr_wouldbe_areas;
+
+/* used for ASLR_HEAP and ASLR_HEAP_FILL - tracks added pad areas that should be freed.
+ * Data is base of associated real heap allocation that precedes allocation,
+ *
+ * FIXME: (TOTEST) We currently expect to be able to lookup the pad
+ * region whenever the application heap region is freed, if any
+ * version of Windows allows a subregion of original to be freed or
+ * free crossing boundaries, we'll just add a real backmap as well.
+ * Kept on the heap for selfprot (case 7957).
+ */
+vm_area_vector_t *aslr_heap_pad_areas;
+
+/* shared Object directory for publishing Sections */
+HANDLE shared_object_directory = INVALID_HANDLE_VALUE;
+
+/* file_t directory of relocated DLL cache - shared 
+ * FIXME: should have one according to starting user SID
+ */
+HANDLE relocated_dlls_filecache_initial = INVALID_HANDLE_VALUE;
+
+#define KNOWN_DLLS_OBJECT_DIRECTORY L"\\KnownDlls"
+HANDLE known_dlls_object_directory = INVALID_HANDLE_VALUE;
+
+#define KNOWN_DLL_PATH_SYMLINK L"KnownDllPath"
+wchar_t known_dll_path[MAX_PATH];
+/* needed even by consumers to be handle NtOpenSection */
+
+/* forwards */
+#ifdef DEBUG
+static bool
+aslr_doublecheck_wouldbe_areas(void);
+#endif
+
+static void aslr_free_heap_pads(void);
+static app_pc aslr_reserve_initial_heap_pad(app_pc preferred_base, size_t reserve_offset);
+
+static bool
+aslr_publish_file(const wchar_t *module_name);
+
+static void
+aslr_process_worklist(void);
+
+static HANDLE 
+open_relocated_dlls_filecache_directory(void);
+static void
+aslr_get_known_dll_path(wchar_t *w_known_dll_path, /* OUT */
+                        uint max_length_characters);
+static bool
+aslr_generate_relocated_section(IN HANDLE unmodified_section,
+                                IN OUT app_pc *new_base, /* presumably random */
+                                bool search_fitting_base,
+                                OUT app_pc *mapped_base,
+                                OUT size_t *mapped_size,
+                                OUT module_digest_t *file_digest);
+
+
+void aslr_free_dynamorio_loadblock(void);
+/* end of forwards */
+
+void
+aslr_init(void)
+{
+    /* big delta should be harder to guess or brute force */
+    size_t big_delta;
+    ASSERT(ALIGNED(DYNAMO_OPTION(aslr_dll_base), ASLR_MAP_GRANULARITY));
+    ASSERT_NOT_IMPLEMENTED(!TESTANY(~(ASLR_DLL|ASLR_STACK|ASLR_HEAP|ASLR_HEAP_FILL), 
+                                    DYNAMO_OPTION(aslr)));
+    ASSERT_NOT_IMPLEMENTED(!TESTANY(~(ASLR_SHARED_INITIALIZE|ASLR_SHARED_INITIALIZE_NONPERMANENT|
+                                      ASLR_SHARED_CONTENTS|
+                                      ASLR_SHARED_PUBLISHER|ASLR_SHARED_SUBSCRIBER|
+                                      ASLR_SHARED_ANONYMOUS_CONSUMER|
+                                      ASLR_SHARED_WORKLIST|ASLR_SHARED_FILE_PRODUCER|
+                                      ASLR_ALLOW_ORIGINAL_CLOBBER|ASLR_RANDOMIZE_EXECUTABLE|
+                                      ASLR_AVOID_NET20_NATIVE_IMAGES|
+                                      ASLR_SHARED_PER_USER
+                                      ), DYNAMO_OPTION(aslr_cache)));
+    ASSERT_NOT_IMPLEMENTED(!TESTANY(~(ASLR_PERSISTENT_PARANOID 
+                                      | ASLR_PERSISTENT_SOURCE_DIGEST 
+                                      | ASLR_PERSISTENT_TARGET_DIGEST
+                                      | ASLR_PERSISTENT_SHORT_DIGESTS
+                                      | ASLR_PERSISTENT_PARANOID_TRANSFORM_EXPLICITLY
+                                      | ASLR_PERSISTENT_PARANOID_PREFIX
+                                      ), DYNAMO_OPTION(aslr_validation)));
+
+    ASSERT_NOT_IMPLEMENTED(!TESTANY(~(ASLR_INTERNAL_SAME_STRESS|ASLR_INTERNAL_RANGE_NONE|
+                                      ASLR_INTERNAL_SHARED_NONUNIQUE),
+                                    INTERNAL_OPTION(aslr_internal)));
+    ASSERT_NOT_IMPLEMENTED(!TESTANY(~
+         (ASLR_TRACK_AREAS | ASLR_DETECT_EXECUTE | ASLR_REPORT),
+                                    DYNAMO_OPTION(aslr_action)));
+    /* FIXME: NYI ASLR_AVOID_AREAS|ASLR_RESERVE_AREAS|
+     * ASLR_DETECT_READ|ASLR_DETECT_WRITE|
+     * ASLR_HANDLING|ASLR_NORMALIZE_ID 
+     */
+
+    ASSERT_CURIOSITY(!TEST(ASLR_RANDOMIZE_EXECUTABLE, 
+                          DYNAMO_OPTION(aslr_cache))
+                     || TEST(ASLR_ALLOW_ORIGINAL_CLOBBER, 
+                             DYNAMO_OPTION(aslr_cache))
+                     && "case 8902 - need to duplicate handle in child");
+    /* case 8902 tracks the extra work if we want to support this
+     * non-recommended configuration */
+
+    ASSERT(ASLR_CLIENT_DEFAULT == 0x7);
+    ASSERT(ASLR_CACHE_DEFAULT == 0x192); /* match any numeric use in optionsx.h */
+#ifdef GBOP
+    ASSERT(GBOP_CLIENT_DEFAULT == 0x6037);
+#endif
+
+    aslr_last_dll_bounds = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, aslr_last_dll_bounds_t,
+                                           ACCT_OTHER, PROTECTED);
+    aslr_last_dll_bounds->start = NULL;
+    big_delta = get_random_offset(DYNAMO_OPTION(aslr_dll_offset));
+    aslr_last_dll_bounds->end = (app_pc) ALIGN_FORWARD(DYNAMO_OPTION(aslr_dll_base)
+                                                       + big_delta,
+                                                       ASLR_MAP_GRANULARITY);
+    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+        "ASLR: -aslr "PFX", dll end="PFX", "
+        "base="PFX", offset="PFX" -> delta="PFX", pad="PFX"\n", 
+        DYNAMO_OPTION(aslr),
+        aslr_last_dll_bounds->end, 
+        DYNAMO_OPTION(aslr_dll_base), DYNAMO_OPTION(aslr_dll_offset), big_delta,
+        DYNAMO_OPTION(aslr_dll_pad));
+
+    VMVECTOR_ALLOC_VECTOR(aslr_wouldbe_areas, GLOBAL_DCONTEXT,
+                          VECTOR_SHARED | VECTOR_NEVER_MERGE | 
+                          /* allow overlap due to conflicting DLLs */
+                          VECTOR_OVERLAP_NON_CRITICAL, 
+                          aslr_areas);
+    VMVECTOR_ALLOC_VECTOR(aslr_heap_pad_areas, GLOBAL_DCONTEXT,
+                          VECTOR_SHARED | VECTOR_NEVER_MERGE,
+                          aslr_pad_areas);
+
+    if (DYNAMO_OPTION(aslr_dr)) {
+        /* free loadblocks if injected by parent */
+        aslr_free_dynamorio_loadblock();
+    } else {
+        /* note if parent had the flag enabled while the child doesn't
+         * we won't risk freeing */
+    }
+
+    if (TEST(ASLR_HEAP, DYNAMO_OPTION(aslr))) {
+        /* we only reserve a random padding from the beginning of memory
+         * and let the OS handle all other allocations normally
+         */
+        app_pc big_front_pad_base = 
+            aslr_reserve_initial_heap_pad(NULL /* earliest possible */,
+                                          DYNAMO_OPTION(aslr_heap_reserve_offset));
+
+        /* FIXME: If we want to consider ASLR_HEAP (but not
+         * ASLR_HEAP_FILL) as a default option we may want to use this
+         * padding as the randomization for our own memory.  If we
+         * want to avoid address space fragmentation for important
+         * services, we may want to add the initial padding before
+         * vmm_heap_init() disable -vm_max_offset 0x0 and use -vm_base 0x0. */
+
+        /* FIXME: Our large reservation maybe too large to fit in
+         * front of the executable, when we're not in early there may
+         * be heap regions already allocated.  While apps commonly
+         * start at 0x00400000, many windows services start at
+         * 0x01000000 (16MB) and the initial hole may be too small to
+         * randomize anyways.
+         *
+         * Office apps start at 0x30000000 so they may end up having
+         * two heap regions if an attacker is able to control memory
+         * allocations.  We only use the smaller
+         * aslr_heap_exe_reserve_offset for after the executable in
+         * case the original mapping was before the imagebase.
+         *
+         */
+
+        /* FIXME: though just grabbing big and small usually works
+         * should just fill in any space in front of the
+         * executable, 
+         *
+         * FIXME: add a random pad after the executable to make sure
+         * no heap allocation will eventually be at a predictable
+         * location.
+         */
+
+        app_pc small_pad_after_executable_base = 
+            aslr_reserve_initial_heap_pad(NULL /* FIXME: should be after executable */,
+                                          DYNAMO_OPTION(aslr_heap_exe_reserve_offset));
+    }
+
+    /* initialize shared object directory - note that this should be
+     * done in a high privilege process (e.g. winlogon.exe) that may
+     * otherwise have no other role to serve in ASLR_SHARED_CONTENTS
+     */
+    if (TEST(ASLR_SHARED_INITIALIZE, DYNAMO_OPTION(aslr_cache))) {
+        HANDLE initialize_directory;
+        NTSTATUS res = nt_initialize_shared_directory(&initialize_directory, 
+                                                      true /* permanent */);
+        /* we currently don't need to do anything else with this
+         * handle, unless we can't make the object permanent then may
+         * want to 'leak' the handle to persist the object directory
+         * until this process dies.
+         */
+        /* FIXME: would be nice to provide a drcontrol -shared
+         * -destroy (using NtMakeTemporaryObject()) to clear the permanent directory
+         * -init to recreate it for easier testing and saving a reboot.
+         */
+        /* FIXME: Note that in a model in which per-session or
+         * per-user sharing is allowed we may have extra levels to
+         * create.  Otherwise, this nt_close_object_directory() can be
+         * done inside nt_initialize_shared_directory() for permanent
+         * directories.
+         */
+        if (NT_SUCCESS(res)) {
+            nt_close_object_directory(initialize_directory);
+        } else {
+            /* STATUS_PRIVILEGE_NOT_HELD (0xc0000061) is an expected
+             * failure code for low privileged processes.  Note for
+             * testing may need a dummy process with high enough
+             * privileges
+             */
+
+            /* FIXME: may want to make this non-internal flag to allow
+             * simple experiments with unprivileged processes in
+             * release builds as well
+             */
+            ASSERT_CURIOSITY(res == STATUS_PRIVILEGE_NOT_HELD);
+            if (TEST(ASLR_SHARED_INITIALIZE_NONPERMANENT, 
+                     DYNAMO_OPTION(aslr_cache))) {
+                NTSTATUS res = nt_initialize_shared_directory(&initialize_directory, 
+                                                              false /* temporary */);
+                ASSERT(NT_SUCCESS(res) && "unable to initialize");
+                /* must 'leak' initialize_directory to persist
+                 * directory until process terminates,
+                 * so there is no corresponding nt_close_object_directory()
+                 */
+            }
+        }
+    }
+
+    if (TESTANY(ASLR_SHARED_SUBSCRIBER|ASLR_SHARED_PUBLISHER, 
+                DYNAMO_OPTION(aslr_cache))) {
+        /* Open shared DLL object directory '\Determina\SharedCache' */
+        /* publisher will ask for permission to create objects in that
+         * directory, consumer needs read only access */
+        /* FIXME: this should change to become SID related */
+        NTSTATUS res = nt_open_object_directory(&shared_object_directory,
+                                                DYNAMORIO_SHARED_OBJECT_DIRECTORY,
+                                                TEST(ASLR_SHARED_PUBLISHER, DYNAMO_OPTION(aslr_cache))
+                                                );
+        /* Only trusted publishers should be allowed to publish in the
+         * SharedCache */
+        /* Note  */
+
+        /* if any of these fail in release build (most likely if the
+         * root is not created, or it is created with restrictive
+         * permissions) we won't be able to publish named sections.
+         * Not a critical failure though.
+         */
+
+        /* FIXME: should test shared_object_directory before any
+         * consumer requests, so that we don't waste any time trying
+         * to request sharing */
+        ASSERT_CURIOSITY(NT_SUCCESS(res) && "can't open \\Determina\\SharedCache");
+    }
+
+    if (DYNAMO_OPTION(track_module_filenames) ||
+        TESTANY(ASLR_SHARED_SUBSCRIBER | ASLR_SHARED_ANONYMOUS_CONSUMER |
+                ASLR_SHARED_PUBLISHER /* just in case */, 
+                DYNAMO_OPTION(aslr_cache))) {
+        /* we'll need to match sections from \KnownDlls, note that all
+         * direct or indirect consumers have to handle NtOpenSection()
+         * here to deal with KnownDlls
+         */
+        NTSTATUS res = nt_open_object_directory(&known_dlls_object_directory,
+                                       KNOWN_DLLS_OBJECT_DIRECTORY,
+                                       false);
+        ASSERT(NT_SUCCESS(res));
+
+        /* open the \KnowdnDlls\KnownDllPath directory */
+        aslr_get_known_dll_path(known_dll_path, 
+                                BUFFER_SIZE_ELEMENTS(known_dll_path));
+    }
+
+    if (TESTANY(ASLR_SHARED_PUBLISHER | ASLR_SHARED_ANONYMOUS_CONSUMER,
+                DYNAMO_OPTION(aslr_cache))) {
+        /* Open shared cache file directory */
+        relocated_dlls_filecache_initial = 
+            open_relocated_dlls_filecache_directory();
+
+        /* FIXME: may need to open one shared and in addition one
+         * per-user
+         */
+
+        /* FIXME: a ASLR_SHARED_FILE_PRODUCER | ASLR_SHARED_WORKLIST
+         * producer may want to be able to write to the filecache
+         * directory
+         */
+    }
+
+    if (TEST(ASLR_SHARED_WORKLIST, DYNAMO_OPTION(aslr_cache))) {
+        aslr_process_worklist();
+    }
+
+    /* FIXME: case 6725 ASLR functionality is not fully dynamic.  The
+     * only state that needs to be set up is the above random number,
+     * which we can just always initialize here.  Yet not enough for
+     * the product:
+     *  o we can't really undo changes, so not very useful to begin with, but 
+     *    at least DLLs after a change can be controlled
+     *  o not planning on synchronizing options, yet may allow nudge to do so
+     *  o post_syscall mappings does attempt to handle dynamic changes, not tested
+     */
+    if (DYNAMO_OPTION(aslr) == ASLR_DISABLED)
+        return;
+}
+
+void
+aslr_exit(void)
+{
+    if (TEST(ASLR_TRACK_AREAS, DYNAMO_OPTION(aslr_action))) {
+        /* doublecheck and print entries to make sure they match */
+        DOLOG(1, LOG_VMAREAS, { print_modules_safe(GLOBAL, DUMP_NOT_XML); });
+        ASSERT(aslr_doublecheck_wouldbe_areas());
+    }
+    /* dynamic option => free no matter the option value now */
+
+    /* at startup: ASLR_TRACK_AREAS */
+    vmvector_delete_vector(GLOBAL_DCONTEXT, aslr_wouldbe_areas);
+
+    /* at startup: ASLR_HEAP_FILL|ASLR_HEAP */
+    aslr_free_heap_pads();
+    vmvector_delete_vector(GLOBAL_DCONTEXT, aslr_heap_pad_areas);
+
+    if (shared_object_directory != INVALID_HANDLE_VALUE) {
+        ASSERT_CURIOSITY(TEST(ASLR_SHARED_CONTENTS, DYNAMO_OPTION(aslr_cache)));
+        nt_close_object_directory(shared_object_directory);
+    }
+
+    if (known_dlls_object_directory != INVALID_HANDLE_VALUE) {
+        ASSERT_CURIOSITY(DYNAMO_OPTION(track_module_filenames) ||
+                         TEST(ASLR_SHARED_CONTENTS, DYNAMO_OPTION(aslr_cache)));
+        nt_close_object_directory(known_dlls_object_directory);
+    }
+
+    if (relocated_dlls_filecache_initial != INVALID_HANDLE_VALUE) {
+        ASSERT_CURIOSITY(TEST(ASLR_SHARED_CONTENTS, DYNAMO_OPTION(aslr_cache)));
+        close_handle(relocated_dlls_filecache_initial);
+    }
+
+    HEAP_TYPE_FREE(GLOBAL_DCONTEXT, aslr_last_dll_bounds, aslr_last_dll_bounds_t,
+                   ACCT_OTHER, PROTECTED);
+    aslr_last_dll_bounds = NULL;
+
+    /* always release lock in case -aslr was dynamically changed
+     * although currently it is not dynamic 
+     */
+    DELETE_LOCK(aslr_lock);
+}
+
+void
+aslr_thread_init(dcontext_t *dcontext)
+{
+}
+
+void
+aslr_set_last_section_file_name(dcontext_t *dcontext, const wchar_t *short_name)
+{
+    aslr_free_last_section_file_name(dcontext);
+    dcontext->aslr_context.last_section_file_name =
+        (const char *) dr_wstrdup(short_name HEAPACCT(ACCT_OTHER));
+}
+
+void
+aslr_free_last_section_file_name(dcontext_t *dcontext)
+{
+    if (dcontext->aslr_context.last_section_file_name != NULL) {
+        dr_strfree(dcontext->aslr_context.last_section_file_name
+                   HEAPACCT(ACCT_OTHER));
+    }
+    dcontext->aslr_context.last_section_file_name = NULL;
+}
+
+void
+aslr_thread_exit(dcontext_t *dcontext)
+{
+    aslr_free_last_section_file_name(dcontext);
+}
+
+/* ASLR random range choice */
+/* use aslr_get_next_base() to start using a range, and in combination
+ * with aslr_update_failed() on failure to use it, and
+ * aslr_update_view_size() to flag success and proceed to the next base.
+ */
+static app_pc
+aslr_get_next_base(void)
+{
+    /* although the loader holds a lock for the DLL mappings, other
+     * MapViewOfFile calls may be racy.  If really serialzed by app,
+     * there will never be contention on the locks grabbed here.
+     */
+
+    size_t jitter = get_random_offset(DYNAMO_OPTION(aslr_dll_pad));
+    app_pc returned_base;
+    /* 
+     * FIXME: [minor security] Although DLLs are definitely not loaded
+     * racily, if we are using this for other potentially racy
+     * allocations from the same region we may have races.  The
+     * aslr_last_dll_bounds->end won't be updated so multiple callers may get
+     * based not far from the same last end.  If aslr_dll_pad is
+     * comparable to the size of an average mapping, the jitter here
+     * will make it possible for multiple racy callers to receive
+     * bases that may succeed.  Nevertheless, that is not really
+     * necessary nor sufficient to avoid collisions.  Still even
+     * though on collision we'll currently give up attackers can't
+     * rely much on this.
+     */
+
+    mutex_lock(&aslr_lock);
+    /* note that we always lose the low 16 bits of randomness of the
+     * padding, so adding to last dll page-aligned doesn't matter */
+    aslr_last_dll_bounds->start = aslr_last_dll_bounds->end + jitter;
+    aslr_last_dll_bounds->start = (app_pc)
+        ALIGN_FORWARD(aslr_last_dll_bounds->start, ASLR_MAP_GRANULARITY);
+    returned_base = aslr_last_dll_bounds->start; /* for racy callers */
+    mutex_unlock(&aslr_lock);
+
+    LOG(THREAD_GET, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+        "ASLR: next dll recommended="PFX"\n",  returned_base);
+    return returned_base; 
+}
+
+/* preverify the range is available, leaving possibility of failure
+ * only to race.  Allows us to skip ranges that get in our way,
+ * especially common when used for ASLR sharing, where we quickly
+ * fragment our address space when DLLs are generated by multiple
+ * processs.
+ */
+/* returns NULL if no valid range exists */
+static app_pc
+aslr_get_fitting_base(app_pc requested_base, size_t view_size)
+{
+    bool available = true;
+    app_pc current_base = requested_base;
+    
+    ASSERT(ALIGNED(current_base, ASLR_MAP_GRANULARITY));
+    /* currently march forward through OS allocated regions */
+
+    do {
+        app_pc allocated_base;
+        size_t size;
+        if ((ptr_uint_t)(current_base + view_size) > DYNAMO_OPTION(aslr_dll_top)) {
+            /* FIXME: case 6739 could try to wrap around (ONCE!) */
+            ASSERT_CURIOSITY((ptr_uint_t)current_base <= DYNAMO_OPTION(aslr_dll_top) ||
+                             /* case 9844: suppress for short regression for now */
+                             check_filter("reload-race.exe",
+                                          get_short_name(get_application_name())));
+            ASSERT_CURIOSITY(false && "exhausted DLL range" ||
+                             /* case 9378: suppress for short regression for now */
+                             check_filter("reload-race.exe",
+                                          get_short_name(get_application_name())));
+            return NULL;
+        }
+
+        size = get_allocation_size(current_base, &allocated_base);
+        if (size == 0) {
+            /* very unusual, can't have passed into kernel ranges */
+            ASSERT_NOT_REACHED();
+            return NULL;
+        }
+
+        /* note that get_allocation_size() returns allocation size of
+         * non FREE regions, while for FREE regions is the available
+         * region size (exactly what we need)
+         */
+        if (allocated_base != NULL /* taken, skip */) {
+            ASSERT(current_base < allocated_base + size);
+            current_base = allocated_base + size;
+            /* skip unusable unaligned MEM_FREE region */
+            current_base = (app_pc)ALIGN_FORWARD(current_base, ASLR_MAP_GRANULARITY);
+            available = false;
+        } else { /* free */
+            if (size < view_size) { /* we don't fit in free size, skip */
+                available = false;
+                ASSERT(size > 0);
+                current_base = current_base + size;
+                /* free blocks should end aligned at allocation granularity  */
+                ASSERT_CURIOSITY(ALIGNED(current_base, ASLR_MAP_GRANULARITY));
+                /* can't be too sure - could be in the middle of freed TEB entries */
+                current_base = (app_pc)ALIGN_FORWARD(current_base, ASLR_MAP_GRANULARITY);
+            } else {
+                /* we can take this, unless someone beats us */
+                available = true;
+            } 
+        }
+    } while (!available);
+    
+    if (requested_base != current_base) {
+        /* update our expectations, so that aslr_update_view_size()
+         * doesn't get surprised */
+        mutex_lock(&aslr_lock);
+        if (aslr_last_dll_bounds->start == requested_base) {
+            aslr_last_dll_bounds->start = current_base;
+        } else {
+            /* racy requests? */
+            ASSERT_CURIOSITY(false && "aslr_get_fitting_base: racy ASLR mapping");
+            ASSERT_NOT_TESTED();
+        }
+        mutex_unlock(&aslr_lock);
+    }
+    ASSERT(ALIGNED(current_base, ASLR_MAP_GRANULARITY));
+    return current_base;
+}
+
+/* update on failure, if request_new is true, we should look for a
+ * better fit given the module needed_size.  Note requested_base is
+ * just a hint for what we have tried
+ */
+static app_pc
+aslr_update_failed(bool request_new, 
+                   app_pc requested_base,
+                   size_t needed_size)
+{
+    app_pc new_base = NULL;     /* default to native preferred base */
+    LOG(THREAD_GET, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+        "ASLR: aslr_update_failed for "PFX"\n",  aslr_last_dll_bounds->start);
+
+    if (request_new) {
+        ASSERT(requested_base != NULL);
+        ASSERT(needed_size != 0);
+        if (requested_base != NULL && needed_size != 0) {
+            new_base = aslr_get_fitting_base(requested_base, needed_size);
+            ASSERT_CURIOSITY(new_base != NULL ||
+                             /* case 9894: suppress for short regression for now */
+                             check_filter("reload-race.exe",
+                                          get_short_name(get_application_name())));
+        } else {
+            /* give up, something is not right, just reset */
+            ASSERT(new_base == NULL);
+        }
+    }
+
+    if (new_base == NULL) {
+        /* update old base, currently just so we can ASSERT elsewhere */
+        mutex_lock(&aslr_lock);
+        aslr_last_dll_bounds->start = NULL;
+        mutex_unlock(&aslr_lock);
+        /* just giving up, no need for new base */
+    }
+    return new_base;
+}
+
+static void
+aslr_update_view_size(app_pc view_base, size_t view_size)
+{
+    ASSERT(view_base != NULL);
+    ASSERT(view_size != 0);
+    ASSERT_CURIOSITY_ONCE((ptr_uint_t)(view_base + view_size) <=
+                          DYNAMO_OPTION(aslr_dll_top) ||
+                          /* case 7059: suppress for short regr for now */
+                          EXEMPT_TEST("reload-race.exe"));
+
+    /* FIXME: if aslr_dll_top is not reachable should wrap around, or
+     * know not to try anymore.  Currently we'll just keep trying to
+     * rebase and giving up all the time.
+     */
+
+    if (TEST(ASLR_INTERNAL_SAME_STRESS, INTERNAL_OPTION(aslr_internal))) {
+        return;
+    }
+    
+    /* NOTE we don't have a lock for the actual system call so we can
+     * get out of order here
+     */
+    mutex_lock(&aslr_lock);
+    if (aslr_last_dll_bounds->start == view_base) {
+        aslr_last_dll_bounds->end = view_base + view_size;
+    } else {
+        /* racy requests? */
+        ASSERT_CURIOSITY(false && "racy ASLR mapping");
+        /* when the last known request is not the same we just bump to
+         * largest value to resynch, although it is more likely that a
+         * collision would have prevented one from reaching here 
+         */
+        aslr_last_dll_bounds->end = MAX(aslr_last_dll_bounds->end, view_base + view_size);
+        ASSERT_NOT_TESTED();
+    }
+    mutex_unlock(&aslr_lock);
+}
+
+/* used for tracking potential violations in ASLR_TRACK_AREAS */
+static void
+aslr_track_randomized_dlls(dcontext_t *dcontext, app_pc base, size_t size, bool map,
+                           bool our_shared_file)
+{
+    if (map) {
+        /* note can't use get_module_preferred_base_safe() here, since
+         * not yet added to loaded_module_areas */
+        app_pc preferred_base;
+        if (our_shared_file) {
+            DEBUG_DECLARE(app_pc our_relocated_preferred_base = 
+                          get_module_preferred_base(base););
+            ASSERT(TEST(ASLR_SHARED_CONTENTS, DYNAMO_OPTION(aslr_cache)));
+            ASSERT(dcontext->aslr_context.original_section_base != ASLR_INVALID_SECTION_BASE);
+
+            ASSERT_CURIOSITY(our_relocated_preferred_base == base 
+                             && "useless conflicting shared");
+
+            LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 2,
+                "ASLR: SHARED: mapped base "PFX", preferred random "PFX
+                ", original "PFX"\n", base, 
+                our_relocated_preferred_base, 
+                dcontext->aslr_context.original_section_base);
+
+            preferred_base = dcontext->aslr_context.original_section_base;
+        } else {
+            preferred_base = get_module_preferred_base(base);
+        }
+
+        /* FIXME: should be rare, otherwise could retry if this happens */
+        ASSERT_CURIOSITY(preferred_base != base && "randomly preferred base");
+
+        /* FIXME: with ASLR_SHARED_CONTENTS we now have three bases to
+         * consider original preferred base, shared preferred base,
+         * real base (our shared DLL can be rebased due to conflict).
+         */
+        if (preferred_base != NULL
+            && preferred_base != base /* for the rare case of staying at base */) {
+            /* FIXME: if overlap in aslr_wouldbe_areas then we cannot
+             * tell which DLL is the one really being targeted.  Yet
+             * unlikely that attackers would bother targeting one of
+             * these, can still use the first loaded as most likely.
+             * Note we can't properly remove overlapping DLLs either.
+             * FIXME: Maybe we shouldn't flag compatibility issues and
+             * accidental read/write in such contested areas.
+             */
+
+            DOLOG(0, LOG_SYSCALLS, {
+                if (vmvector_overlap(aslr_wouldbe_areas, 
+                                     preferred_base, preferred_base + size)) {
+                    LOG(THREAD_GET, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                        "aslr: conflicting preferred range "PFX"-"PFX" currently "PFX, 
+                        preferred_base, preferred_base + size, base);
+                }
+            });
+
+            vmvector_add(aslr_wouldbe_areas, preferred_base, preferred_base + size, 
+                         base /* current mapping of DLL */);
+        } else {
+            /* FIXME: shouldn't happen for ASLR_DLL */
+            ASSERT_CURIOSITY(false && "not a PE or no image base");
+        }
+    } else {
+        /* not all unmappings are to modules, and double mappings of a
+         * PE file both as a module and as a linear memory mapping
+         * exist - e.g. USER32!ExtractIconFromEXE.  Would need an
+         * explicit MEM_IMAGE check on the area.
+         */
+        /* It should be faster to check in loaded_module_areas.
+         * Ignore if unmapped view was not loaded as DLL.  Called
+         * before process_mmap(unmap), still ok to use loaded module
+         * list. */
+        app_pc preferred_base = get_module_preferred_base_safe(base);
+        if (preferred_base != NULL /* tracked module */
+            && preferred_base != base /* randomized by us, or simply rebased? */
+            ) {
+            /* FIXME: we don't know which DLLs we have randomized
+             * ourselves and which have had a conflict, but not a
+             * significant loss if we remove the range from tracking.
+             * Note that simple technique for silencing the ASSERT
+             * doesn't work for rebased DLLs that have been loaded
+             * before we're loaded.
+             */
+            DOLOG(0, LOG_SYSCALLS, {
+                /* case 7797 any conflicting natively DLLs may hit this */
+                if (!vmvector_overlap(aslr_wouldbe_areas, 
+                                      preferred_base, preferred_base + size)) {
+                    LOG(THREAD_GET, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                        "ASLR: unmap missing preferred range "PFX"-"PFX", "
+                        "probably conflict?", preferred_base, preferred_base + size);
+                }
+            });
+            
+            /* doublecheck unsafe base, since PE still mapped in,
+             * however preferred base from PE is not what we want in ASLR shared
+             * see case 8507
+             */
+            ASSERT(preferred_base == get_module_preferred_base(base)
+                   || TEST(ASLR_SHARED_CONTENTS,
+                           DYNAMO_OPTION(aslr_cache)));
+
+            /* FIXME: if multiple DLLs preferred regions overlap we
+             * wouldn't know not to remove a hole, need refcounting, but
+             * since whole notification is best effort, not doing that */
+            vmvector_remove(aslr_wouldbe_areas, 
+                            preferred_base, preferred_base + size);
+        }
+    }
+}
+
+/* PRE hook for NtMapViewOfSection */
+void 
+aslr_pre_process_mapview(dcontext_t *dcontext)
+{
+    reg_t *param_base = dcontext->sys_param_base;
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    
+    HANDLE section_handle = (HANDLE) sys_param(dcontext, param_base, 0);
+    HANDLE process_handle = (HANDLE) sys_param(dcontext, param_base, 1);
+    void **pbase_unsafe = (void *) sys_param(dcontext, param_base, 2);
+    uint zerobits = (uint) sys_param(dcontext, param_base, 3);
+    size_t commit_size = (size_t) sys_param(dcontext, param_base, 4);
+    LARGE_INTEGER *psection_offs_unsafe = 
+        (LARGE_INTEGER *) sys_param(dcontext, param_base, 5); /* OPTIONAL */
+    size_t *pview_size_unsafe = (size_t *) sys_param(dcontext, param_base, 6);
+    uint inherit_disposition = (uint) sys_param(dcontext, param_base, 7);
+    uint allocation_type = (uint) sys_param(dcontext, param_base, 8);
+    uint prot = (uint) sys_param(dcontext, param_base, 9);
+
+    app_pc requested_base;
+    size_t requested_size;
+    app_pc modified_base = 0;
+        
+    /* flag currently used only for MapViewOfSection */
+    dcontext->aslr_context.sys_aslr_clobbered = false;
+
+    if (!safe_read(pbase_unsafe, sizeof(requested_base), &requested_base) || 
+        !safe_read(pview_size_unsafe, sizeof(requested_size), &requested_size)) {
+        /* we expect the system call to fail */
+        DODEBUG(dcontext->expect_last_syscall_to_fail = true;);
+        return;
+    }
+
+    DOLOG(1, LOG_SYSCALLS, {
+        uint queried_section_attributes = 0;
+        bool attrib_ok = 
+            get_section_attributes(section_handle, &queried_section_attributes, NULL);
+
+        /* Unfortunately, the loader creates sections that do not have
+         * Query access (SECTION_QUERY 0x1), and we can't rely on
+         * being able to use this
+         *
+         * windbg> !handle 0 f section
+         * GrantedAccess 0xe: 
+         *    None, MapWrite,MapRead,MapExecute 
+         * vs
+         * GrantedAccess 0xf001f:
+         *      Delete,ReadControl,WriteDac,WriteOwner
+         *      Query,MapWrite,MapRead,MapExecute,Extend
+         * Object Specific Information
+         *   Section base address 0
+         *   Section attributes 0x4000000
+         */
+
+        /* FIXME: unknown flag 0x20000000
+         * when running notepad I get
+         * Section attributes 0x21800000 only on two DLLs
+         * I:\Program Files\WIDCOMM\Bluetooth Software\btkeyind.dll (my bluetooth extension)
+         * I:\Program Files\Dell\QuickSet\dadkeyb.dll are using 0x20000000, why are they special?
+         */
+        ASSERT_CURIOSITY(!TESTANY(~(SEC_BASED_UNSUPPORTED | SEC_NO_CHANGE_UNSUPPORTED
+                                    | SEC_FILE | SEC_IMAGE | SEC_VLM | SEC_RESERVE
+                                    | SEC_COMMIT | SEC_NOCACHE
+                                    /* FIXME: value is 0x20000000
+                                     * could also be IMAGE_SCN_MEM_EXECUTE , or MEM_LARGE_PAGES
+                                     */
+                                    | GENERIC_EXECUTE
+                                    ), 
+                                    queried_section_attributes));
+
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+            "syscall: pre NtMapViewOfSection *base="PFX" *size="PIFX" prot=%s\n"
+            "         sh="PIFX" zero=%d commit=%d &secoffs="PIFX" inherit=%d type=0x%x;"
+            "%s%x\n",
+            requested_base, requested_size, prot_string(prot), 
+            section_handle, zerobits, commit_size, psection_offs_unsafe, inherit_disposition,
+            allocation_type, 
+            attrib_ok ? "attrib=0x" : "unknown ", queried_section_attributes);
+    });
+
+    /* Reversing notes: on XP SP2
+     *
+     * o Protect - all modules are first attempted with --x- but
+     *   DLLs that need rebasing are remapped as rw-- 
+     * 
+     * intercept_load_dll: I:\Program Files\Dell\QuickSet\dadkeyb.dll (always conflicts)
+     * syscall: NtMapViewOfSection *base=0x00000000 *size=0x0 prot=--x-
+     *          sh=1832 zero=0 commit=0 &secoffs=0 inherit=1 type=0
+     * syscall: NtMapViewOfSection 0x00980000 size=0x12000 prot=--x- => 0
+     * syscall: NtUnmapViewOfSection 0x00980000 size=0x12000
+     * 
+     * syscall: NtMapViewOfSection *base=0x00000000 *size=0x0 prot=rw--
+     *          sh=1836 zero=0 commit=0 &secoffs=0 inherit=1 type=0
+     * syscall: NtMapViewOfSection 0x00980000 size=0x13000 prot=rw-- => 0x40000003
+     *   Note the size is now larger, in fact mapping is MEM_IMAGE, and
+     *   so get STATUS_IMAGE_NOT_AT_BASE, yet we can't always even query our section,
+     *   so we would have to track NtCreateSection to determine that.
+     *
+     * syscall: NtProtectVirtualMemory process=0xffffffff base=0x00981000 size=0x8000 prot=rw-- 0x4
+     *
+     * And most weird is a call that always fails while processing the above DLL
+     * syscall: NtMapViewOfSection *base=0x00980000 *size=0x13000 prot=rw--
+     *          sh=1832 zero=0 commit=0 &secoffs=0 inherit=1 type=0
+     * syscall: failed NtMapViewOfSection prot=rw-- => 0xc0000018 STATUS_CONFLICTING_ADDRESSES
+     */
+
+    if (is_phandle_me(process_handle)) {
+        ASSERT_CURIOSITY(psection_offs_unsafe == NULL || prot != PAGE_EXECUTE);
+        /* haven't seen a DLL mapping that specifies an offset  */
+
+        /* * SectionOffset is NULL for the loader,
+         * kernel32!MapViewOfFileEx (on Windows XP and Win2k)
+         * always passes the psection_offs_unsafe as a stack
+         * variable, since offset is user exposed.  
+         * DLL loading on the other hand doesn't need this argument.
+         */
+        ASSERT_NOT_IMPLEMENTED(!TEST(ASLR_MAPPED, DYNAMO_OPTION(aslr)));
+
+        if (psection_offs_unsafe == NULL && prot != PAGE_READONLY) {
+            /* FIXME: should distinguish SEC_IMAGE for the
+             * purpose of ASLR_MAPPED in pre-processing, and
+             * should be able to tell MEM_IMAGE from
+             * MEM_MAPPED.  Can do only if tracking
+             * NtCreateSection(), or better yet should just
+             * NtQuerySection() which would work for 
+             * NtCreateSection(), but the loader uses NtOpenSection() 
+             * without SECTION_QUERY.
+             *
+             * FIXME: see if using queried_section_attributes would
+             * help.  There is nothing interesting in
+             * SectionImageInformation (other than that
+             * NtQuerySection() would return STATUS_SECTION_NOT_IMAGE
+             * when asking for it, if not ).  We should touch only
+             * SEC_IMAGE and definitely not mess with SEC_BASED
+             *
+             * Extra syscall here won't be too critical - we're
+             * already calling at least NtQueryVirtualMemory() in
+             * process_mmap(), and currently safe_read/safe_write are
+             * also system calls.
+             */
+            /* FIXME: still unclear whether the loader always
+             * first maps as PAGE_EXECUTE and only afterwards
+             * it tries a rw- mapping
+             */
+
+            /* on xp sp2 seen this use of NtMapViewOfSection PAGE_READONLY
+             * kernel32!BasepCreateActCtx+0x3d8:
+             * 7c8159b1 push    0x2
+             * 7c8159cf call dword ptr [kernel32!_imp__NtMapViewOfSection]
+             */
+
+            ASSERT_CURIOSITY(zerobits == 0);
+            ASSERT_CURIOSITY(commit_size == 0);
+
+            /* only nodemgr and services have been observed to use
+             * ViewUnmap, in nodemgr it is on Module32Next from the
+             * ToolHelp.  
+
+             * FIXME: unclear whether we'll want to do something
+             * different for ViewShare handle inheritance if we go
+             * after ASLR_SHARED_PER_USER.  Unlikely that a high
+             * privilege service will share handles with a low
+             * privilege one though.
+             */
+            ASSERT_CURIOSITY(inherit_disposition == 0x1 /* ViewShare */ ||
+                             inherit_disposition == 0x2 /* ViewUnmap */);
+            /* cygwin uses AT_ROUND_TO_PAGE but specify file offset,
+             * not seen in DLL mappings */
+            ASSERT_CURIOSITY(allocation_type == 0);
+            ASSERT_CURIOSITY(prot == PAGE_EXECUTE || prot == PAGE_READWRITE);
+
+            DOSTATS({
+                if (prot == PAGE_EXECUTE)
+                    STATS_INC(app_mmap_section_x);
+                else {
+                    STATS_INC(app_mmap_section_rw);
+                }
+            });
+
+            /* seen only either both 0 or both set */
+            ASSERT_CURIOSITY(requested_size == 0 || requested_base != 0);
+
+            /* assumption: loader never suggests base in 1st map */
+            if (requested_base == 0) {
+                DODEBUG({
+                    if (TEST(ASLR_SHARED_CONTENTS, DYNAMO_OPTION(aslr_cache)) && 
+                        dcontext->aslr_context.randomized_section_handle !=
+                        section_handle) {
+                        STATS_INC(aslr_dlls_not_shared);
+                        
+                        ASSERT_CURIOSITY(dcontext->aslr_context.
+                                         last_app_section_handle == section_handle);
+                        /* note that unusual uses of sections other than the loader can trigger this */
+
+                        if (dcontext->aslr_context.
+                            last_app_section_handle == section_handle)
+                            /* FIXME: with MapViewOfSection private
+                             * ASLR processing we don't quite know
+                             * whether we're dealing with an image or
+                             * mapped file.  This is always hit by
+                             * LdrpCheckForLoadedDll, it suggests that
+                             * only SEC_IMAGE should be bumped,
+                             * instead of SEC_COMMIT as well.  Maybe
+                             * there is nothing with doing this and
+                             * should take out this warning.
+                             */
+                            SYSLOG_INTERNAL_WARNING_ONCE("non-image DLL pre-processed for private ASLR");
+                        else {
+                            /* could have been exempted */
+                            SYSLOG_INTERNAL_WARNING_ONCE("image DLL ASLRed without sharing");
+                        }
+                    }
+                });
+                
+                if (TEST(ASLR_SHARED_CONTENTS, DYNAMO_OPTION(aslr_cache)) &&
+                    dcontext->aslr_context.randomized_section_handle == 
+                    section_handle) {
+                    /* shared DLL mapping at presumably randomized location,
+                     * leave base unset for preferred mapping
+                     */
+
+                    /* we may want to check whether preferred
+                     * base+size is available, but since racy we
+                     * anyways have to check the success afterwards
+                     */
+                    STATS_INC(aslr_dlls_shared_mapped);
+
+                    /* mark so that we can handle failures */
+                    dcontext->aslr_context.sys_aslr_clobbered = true;
+                } else {        /* private ASLR */
+                    /* FIXME: we may want to take a hint from prot and expected size */
+                    modified_base = aslr_get_next_base();
+
+                    if (!TEST(ASLR_INTERNAL_RANGE_NONE, 
+                              INTERNAL_OPTION(aslr_internal))) {
+                        /* really modify base now */
+                        /* note that pbase_unsafe is an IN/OUT argument,
+                         * so it is not likely that the application would
+                         * have used the passed value.  If we instead
+                         * passed a pointer to our own (dcontext) variable
+                         * we'd have to safe_write it back in aslr_post_process_mapview.
+                         */
+                        DEBUG_DECLARE(bool ok = )
+                            safe_write(pbase_unsafe, sizeof(modified_base), &modified_base);
+                        ASSERT(ok);
+                        STATS_INC(aslr_dlls_bumped);
+                        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                            "ASLR: NtMapViewOfSection prot=%s BUMPED to "PFX"\n",
+                            prot_string(prot), modified_base);
+                        /* mark so that we can handle failures, not allow
+                         * detach when system call arguments are modified
+                         * from what the application can handle if we do
+                         * not deal with possible failures */
+                        dcontext->aslr_context.sys_aslr_clobbered = true;
+                    } else {
+                        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                            "ASLR: NtMapViewOfSection prot=%s RANGE_NONE: would be at "
+                            PFX"\n", prot_string(prot), modified_base);
+                    }
+                }
+            } else {
+                /* Apparently the loader maps again with the known base and size
+                 * since we have modified the base already, we'll just leave it alone
+                 * in same example as noted in the above dadkeyb.dll
+                 * syscall: NtMapViewOfSection *base=0x00980000 *size=0x13000 prot=rw--
+                 *          sh=1832 zero=0 commit=0 &secoffs=0 inherit=1 type=0
+                 * syscall: failed NtMapViewOfSection prot=rw-- => 0xc0000018
+                 * Since it fails and goes to already randomized DLL nothing to do here.
+                 *
+                 * All other yet to be seen users that set base are
+                 * also assumed to not need to be randomized.  We may
+                 * have to revisit for MEM_MAPPED.
+                 */
+                ASSERT_CURIOSITY(aslr_last_dll_bounds->start == 0 || /* given up */
+                                 aslr_last_dll_bounds->start ==
+                                 requested_base  /* may be race? */
+                                 || TEST(ASLR_SHARED_CONTENTS, 
+                                         DYNAMO_OPTION(aslr_cache))
+                                 /* not keeping keep track for shared */);
+                /* FIXME: for ASLR_SHARED_CONTENTS would be at the
+                 * requested shared preferred mapping address which is
+                 * not the same as the private address!  or if it is
+                 * hitting a conflict it is in fact the base of the
+                 * last mapping that was left to the kernel */
+                LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                    "ASLR: not touching NtMapViewOfSection prot=%s requested "PFX"\n",
+                    prot_string(prot), requested_base);
+                STATS_INC(app_mmap_requested_base);
+            }
+        } else {
+            DOSTATS({
+                if (psection_offs_unsafe == NULL) {
+                    if (prot == PAGE_READONLY)
+                        STATS_INC(app_mmap_section_r);
+                    else {
+                        /* not seen other prot requests */
+                        ASSERT_CURIOSITY(false && "unseen protection");
+                    }
+                }
+            });
+        }
+    } else {
+        IPC_ALERT("WARNING: MapViewOfSection on another process\n");
+    }
+}
+
+NTSTATUS
+aslr_retry_map_syscall(dcontext_t *dcontext, reg_t *param_base)
+{
+    /* FIXME: we could issue a system call from app and just pass the
+     * sysnum and param_base, yet we don't have the facility to handle
+     * post_system_call for that case.  Instead we issue our own copy
+     * of the arguments, note that all OUT arguments will be modified
+     * directly in the app space anyways.  Only any IN argument races
+     * and overwrites won't be transparent.
+     */
+
+    NTSTATUS res;
+    /* Minor hit of unnecessary argument copying, allows us to work
+     * with any special handling needed by NT_SYSCALL
+     */
+    HANDLE section_handle = (HANDLE) postsys_param(dcontext, param_base, 0);
+    HANDLE process_handle = (HANDLE) postsys_param(dcontext, param_base, 1);
+    void **pbase_unsafe = (void *) postsys_param(dcontext, param_base, 2);
+    uint zerobits = (uint) postsys_param(dcontext, param_base, 3);
+    size_t commit_size = (size_t) postsys_param(dcontext, param_base, 4);
+    LARGE_INTEGER *section_offs = (LARGE_INTEGER *)
+        postsys_param(dcontext, param_base, 5);
+    SIZE_T *view_size = (SIZE_T *) postsys_param(dcontext, param_base, 6);
+    uint inherit_disposition = (uint) postsys_param(dcontext, param_base, 7);
+    uint type = (uint) postsys_param(dcontext, param_base, 8);
+    uint prot = (uint) postsys_param(dcontext, param_base, 9);
+
+    /* Atypical use of NT types in nt_map_view_of_section to reaffirm
+     * that we are using this on behalf of the application. */
+    res = nt_map_view_of_section(section_handle, /* 0 */
+                                 process_handle, /* 1 */
+                                 pbase_unsafe,   /* 2 */
+                                 zerobits,       /* 3 */
+                                 commit_size,    /* 4 */
+                                 section_offs,   /* 5 */
+                                 view_size,      /* 6 */
+                                 inherit_disposition, /* 7 */
+                                 type,           /* 8 */
+                                 prot);          /* 9 */
+
+    LOG(THREAD_GET, LOG_SYSCALLS|LOG_VMAREAS, 1,
+        "syscall: aslr_retry_map_syscall NtMapViewOfSection *pbase="PFX
+        ", prot=%s, res "PFX"\n", *pbase_unsafe, prot_string(prot), res);
+    ASSERT_CURIOSITY(NT_SUCCESS(res));
+    return res;
+}
+
+/* get mapping size needed for an application section */
+bool
+aslr_get_module_mapping_size(HANDLE section_handle,
+                             size_t *module_size,
+                             uint prot)
+{
+    NTSTATUS res;
+    app_pc base = (app_pc)0x0;  /* default mapping */
+    size_t commit_size = 0;       
+    SIZE_T view_size = 0;       /* we need to know full size */
+    uint type = 0;              /* commit is default */
+
+    /* note the section characteristics determine whether MEM_MAPPED
+     * or MEM_IMAGE is needed */
+
+    /* we need protection flags given by the caller, so we can avert a
+     * STATUS_SECTION_PROTECTION error - A view to a section specifies
+     * a protection which is incompatible with the initial view's
+     * protection.
+     */
+
+    /* FIXME: case 9669 - if we have SECTION_QUERY privilege we can
+     * try to get the size from SectionBasicInformation.Size, and map
+     * only on failure
+     */
+    res = nt_map_view_of_section(section_handle, /* 0 */
+                                 NT_CURRENT_PROCESS, /* 1 */
+                                 &base, /* 2 */
+                                 0, /* 3 */
+                                 commit_size, /* 4 */
+                                 NULL, /* 5 */
+                                 &view_size, /* 6 */
+                                 ViewShare, /* 7 */
+                                 type, /* 8 */
+                                 prot); /* 9 */
+    ASSERT(NT_SUCCESS(res));
+    if (!NT_SUCCESS(res))
+        return false;
+    /* side note: windbg receives a ModLoad: for our temporary mapping
+     * at the NtMapViewOfSection(), no harm */
+    *module_size = view_size;
+
+    res = nt_unmap_view_of_section(NT_CURRENT_PROCESS, 
+                                   base);
+    ASSERT(NT_SUCCESS(res));
+    return true;
+}
+
+/* since always coming from dispatch now, only need to set mcontext, but we
+ * continue to set reg_eax in case it's read later in the routine
+ * FIXME: assumes local variable reg_eax
+ */
+#define SET_RETURN_VAL(dc, val) \
+  reg_eax = (reg_t) (val); \
+  get_mcontext(dc)->xax = (reg_t) (val);
+
+/* POST processing of NtMapViewOfSection.  Should be called only when
+ * base is clobbered by us.  Potentially modifies app registers and system
+ * call parameters.
+ */
+void
+aslr_post_process_mapview(dcontext_t *dcontext)
+{
+    reg_t *param_base = dcontext->sys_param_base;
+    reg_t reg_eax = get_mcontext(dcontext)->xax;
+    NTSTATUS status = (NTSTATUS) reg_eax; /* get signed result */
+
+    HANDLE section_handle = (HANDLE) postsys_param(dcontext, param_base, 0);
+    HANDLE process_handle = (HANDLE) postsys_param(dcontext, param_base, 1);
+    void **pbase_unsafe = (void *) postsys_param(dcontext, param_base, 2);
+    uint zerobits = (uint) postsys_param(dcontext, param_base, 3);
+    size_t commit_size = (size_t) postsys_param(dcontext, param_base, 4);
+    uint *section_offs = (uint *) postsys_param(dcontext, param_base, 5);
+    size_t *view_size = (size_t *) postsys_param(dcontext, param_base, 6);
+    uint inherit_disposition = (uint) postsys_param(dcontext, param_base, 7);
+    uint type = (uint) postsys_param(dcontext, param_base, 8);
+    uint prot = (uint) postsys_param(dcontext, param_base, 9);
+    size_t size;
+    app_pc base;
+
+    /* retries to recover private ASLR from range conflict */
+    uint retries_left = DYNAMO_OPTION(aslr_retry) + 1 /* must fallback to native */;
+
+    DEBUG_DECLARE(uint section_attributes = 0;)
+
+    ASSERT(dcontext->aslr_context.sys_aslr_clobbered);
+
+    /* unlikely that a dynamic option change happened in-between */
+    ASSERT_CURIOSITY(TESTANY(ASLR_DLL|ASLR_MAPPED, DYNAMO_OPTION(aslr)));
+
+    ASSERT(is_phandle_me(process_handle));
+
+    /* FIXME: should distinguish SEC_IMAGE for the purpose of
+     * ASLR_MAPPED in pre-processing.  Should be able to tell
+     * MEM_IMAGE from MEM_MAPPED, here at least ASSERT.
+     */
+
+    /* expected attributes only when we have decided to clobber,
+     * under ASLR_DLL it is only loader objects.
+     */
+    DODEBUG(get_section_attributes(section_handle, &section_attributes, NULL););
+    ASSERT_CURIOSITY(section_attributes == 0 || 
+                     TESTALL(SEC_IMAGE | SEC_FILE, section_attributes));
+    ASSERT_CURIOSITY(section_attributes == 0 || /* no Query access */
+                     !TESTANY(~(SEC_IMAGE | SEC_FILE | GENERIC_EXECUTE), section_attributes));
+
+    ASSERT_CURIOSITY(status == STATUS_SUCCESS || 
+                     status == STATUS_IMAGE_NOT_AT_BASE || 
+                     status == STATUS_CONFLICTING_ADDRESSES);
+
+    /* handle shared DLL ASLR mapping */
+    if (TEST(ASLR_SHARED_CONTENTS, DYNAMO_OPTION(aslr_cache)) && 
+        dcontext->aslr_context.randomized_section_handle == section_handle) {
+        if (NT_SUCCESS(status)) {
+            if (status == STATUS_SUCCESS) {
+                STATS_INC(aslr_dlls_shared_mapped_good);
+            } else if (status == STATUS_IMAGE_NOT_AT_BASE) {
+                /* we can live with not being at our choice as well,
+                 * though it breaks all the work we did to share this
+                 * mapping.
+                 */
+                /* If we fail to map a shared DLL at its preferred
+                 * base we're not gaining any sharing.  Should revert
+                 * this DLL back to private randomization for better
+                 * controlled randomization worse the kernel will pick
+                 * the lowest possible address that may be easier to
+                 * predict.  TOFILE currently useful to leave
+                 * as is for testing full sharing.
+                 */
+                SYSLOG_INTERNAL_WARNING("conflicting shared mapping "
+                                        "should use private instead\n");
+                /* FIXME: should get some systemwide stats on how
+                 * often do we get the correct base so we can measure
+                 * the effectiveness of the randomization mapping  */
+                STATS_INC(aslr_dlls_shared_map_rebased);
+            } else 
+                ASSERT_NOT_REACHED();
+
+            /* if succesful, we'll use the original base from our records,
+             * not from mapped PE, so we can detect attacks.
+             *
+             * case 8507 similarly we have to register to fool hotpatching's
+             * timestamp/checksum.  Saved on section create or open 
+             * aslr_context.original_section_{base,checksum,timestamp}.
+             */
+
+            /* add to preferred module range */
+            if (TEST(ASLR_TRACK_AREAS, DYNAMO_OPTION(aslr_action))) {
+                ASSERT(NT_SUCCESS(status));
+
+                /* we assume that since syscall succeeded these dereferences are safe 
+                 * FIXME : could always be multi-thread races though */
+                size = *view_size; /* ignore commit_size? */
+                base = *((app_pc *)pbase_unsafe);
+
+                LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                    "ASLR: SHARED NtMapViewOfSection "PFX" size="PIFX" prot=%s => "
+                    PFX"\n", base, size, prot_string(prot), reg_eax);
+
+                /* We need to provide the original preferred address
+                 * which was preserved at the section creation in
+                 * aslr_context.  We also keep the original base in
+                 * the module list so that on UnMapViewOfSection we
+                 * can remove the preferred region
+                 */
+                aslr_track_randomized_dlls(dcontext, base, size, true /* Map */,
+                                           true /* Our Shared File */);
+            }
+        } else {
+            /* FIXME: we've went too far here - we can still switch the
+             * file handle to the original handle for creating a new
+             * section, and then map that instead and recover the
+             * application's intent.  Or should have kept the
+             * original_section_handle open until here?
+             */
+            STATS_INC(aslr_dlls_shared_map_failed);
+
+            LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                "ASLR: unexpected failure on shared NtMapViewOfSection"
+                " prot=%s => "PFX"\n",
+                prot_string(prot), reg_eax);
+
+            /* we can't simply restore application request below, and retry */
+            ASSERT_CURIOSITY(false && "unexpected error status");
+
+            /* FIXME: return error to app hoping it would have been a
+             * native error as well.  Would we be out of virtual
+             * address space?
+             */
+            ASSERT_NOT_IMPLEMENTED(false);
+        }
+
+        dcontext->aslr_context.randomized_section_handle = 
+            INVALID_HANDLE_VALUE;
+        dcontext->aslr_context.sys_aslr_clobbered = false;
+        return;
+    } else if (TEST(ASLR_SHARED_CONTENTS, DYNAMO_OPTION(aslr_cache)) && 
+        dcontext->aslr_context.randomized_section_handle != section_handle) {
+        /* flag that private mapping should be processed in
+         * update_module_list()
+         */
+        dcontext->aslr_context.original_section_base = ASLR_INVALID_SECTION_BASE;
+    }
+
+    /* handle private rebasing on ASLR mapping */
+
+    /* FIXME: STATUS_ILLEGAL_DLL_RELOCATION: what defines the
+     * "The system DLL %hs was relocated in memory. The
+     * application will not run properly.  The relocation
+     * occurred because the DLL %hs occupied an address range
+     * reserved for Windows system DLLs. "
+     */
+
+    /* Three potential problems that prevent us from randomizing all
+     * mappings: DLL exemptions by name, DLL FIXED, and races due to
+     * non-atomic interposition on system calls.
+     *
+     * Three approaches to solving them:
+     * 1) keep track of handles from file to section - 
+     *     may be able to do exemption on file name
+     * 2) presyscall 
+     *     o may grab a lock to deal with in-process races
+     *     o extra map/unmap - can figure out FIXED or PE name
+     * 3) postsyscall undo and redo
+     *    o can handle racy allocation failure by retrying on failure
+     *    o can figure out PE name and FIXED exemption, unmap & retry
+     *
+     * Currently using 3) to avoid adding the Map/UnMap on the normal
+     * path, assuming exemptions are exceptions not the norm, and also
+     * allows dealing with IPC allocations.
+     */
+
+    if (!NT_SUCCESS(status)) {
+        app_pc retry_base = 0;
+        int retry_result = 0;
+
+        /* Need to handle failures and retry.  For sure we can cause
+         * STATUS_CONFLICTING_ADDRESSES and since the loader doesn't
+         * retry we have to retry for it.  Conservatively we should
+         * retry on any other unknown failure.
+         */
+
+        /* FIXME: should we look for the end/beginning of the current
+         * mapping at the conflicting location and try one more time?
+         * mostly needed for ASLR_RANGE_BOTTOM_UP/ASLR_RANGE_TOP_DOWN.
+         * ASLR_RANGE_RANDOM should have a full address space map to
+         * allow it to choose any location.
+         */
+
+        /* Note that SQL server is grabbing a lot of virtual
+         * address space in the example I've seen it has taken
+         * everything from after sqlsort.dll 42b70000 and
+         * reserves all the memory until rpcrt4.dll 77d30000
+         * So a scheme that simply gives up randomizing after
+         * hitting these will not do us much good here.
+         * Should wrap around and continue looking for good
+         * ranges.
+         *
+         * Side note that due to the above reservation some
+         * dynamically loaded DLLs are not at predictable
+         * locations, since loaded by multiple threads. SQL
+         * Slammer used a stable location in the statically
+         * linked sqlsort.dll as a trampoline.
+         */
+
+        /* FIXME: alternative solution is to retry with no
+         * base address - and use the returned mapping as a
+         * hint where the OS would rather have us, then unmap,
+         * add jitter and try again.  The problem is that most
+         * DLLs in the usual case will prefer to be at their
+         * preferred base.
+         */
+
+        ASSERT_CURIOSITY(status == STATUS_CONFLICTING_ADDRESSES);
+
+        ASSERT(*pbase_unsafe != 0); /* ASSERT can take a risk */
+        ASSERT(*pbase_unsafe == aslr_last_dll_bounds->start);
+
+        ASSERT(retries_left >= 0);
+        /* possibly several ASLR attempts, and a final native base retry */
+        /* retry syscall */
+        do {
+            if (status == STATUS_CONFLICTING_ADDRESSES) {
+                /* we can modify the arguments and give it another shot */
+                if (retries_left > 1) {
+                    /* note aslr_last_dll_bounds->start is global so
+                     * subject to race, while the *pbase_unsafe is app
+                     * memory similarly beyond our control, so neither
+                     * one can really be trusted to be what the
+                     * syscall really used.  We choose to use the app
+                     * for the base_requested hint.
+                     */
+                    app_pc base_requested = 0;
+                    size_t size_needed;
+
+                    TRY_EXCEPT(dcontext, { 
+                        base_requested = *pbase_unsafe;
+                    }, { /* nothing */ });
+
+                    /* although we could skip the first MEM_FREE block
+                     * and assume we were too big, we're not
+                     * guaranteed we'd find enough room in the next
+                     * hole either in a small number of retries, so
+                     * we're doing a full NtMapViewOfSection() to
+                     * obtain the actual size needed
+                     */
+                    if (aslr_get_module_mapping_size(section_handle, &size_needed, prot)) {
+                        retry_base = aslr_update_failed(true /* request a better fit */,
+                                                        base_requested,
+                                                        size_needed);
+                        ASSERT_CURIOSITY(retry_base != 0 ||
+                                         /* case 9893: suppress for short regr for now */
+                                         check_filter("reload-race.exe",
+                                                get_short_name(get_application_name())));
+                    } else {
+                        retry_base = NULL;
+                    }
+                    if (retry_base == NULL) {
+                        SYSLOG_INTERNAL_WARNING_ONCE("ASLR conflict at "PFX", "
+                                                     "no good fit, giving up",
+                                                     *pbase_unsafe);
+                        /* couldn't find a better match */
+                        STATS_INC(aslr_dll_conflict_giveup);
+
+                        /* if giving up we just process as if application request */
+                        retries_left = 0;
+                        /* same as handling any other error */
+                    } else {
+                        SYSLOG_INTERNAL_WARNING_ONCE("ASLR conflict at "PFX
+                                                     ", retrying at "PFX,
+                                                     *pbase_unsafe, retry_base);
+                
+                        /* we'll give it another shot at the new address 
+                         * although it may still fail there due to races,
+                         * so we have to be ready to retry the original app
+                         */
+                        ASSERT(dcontext->aslr_context.sys_aslr_clobbered);
+                        retries_left--;
+                        ASSERT(retries_left > 0);
+                        STATS_INC(aslr_dll_conflict_fit_retry);
+                    }
+                } else {
+                    /* first solution: give up our randomization and move on */
+                    SYSLOG_INTERNAL_WARNING_ONCE("ASLR conflict at "PFX", giving up",
+                                                 *pbase_unsafe);
+                    /* if giving up we just process as if application request */
+                    retries_left = 0;
+                    retry_base = aslr_update_failed(false /* no new request */, NULL, 0);
+                    STATS_INC(aslr_dll_conflict_giveup);
+                }
+                /* side note: WinDbg seems to get notified even when the system call fails
+                 * so when executing this under a debugger a sequence like this is seen:
+                 * when run with ASLR_RANGE_SAME_STRESS
+                 *
+                 * WARNING: WS2HELP overlaps Msi
+                 * ModLoad: 43b40000 43b40000   I:\WINDOWS\system32\WS2HELP.dll
+                 * ModLoad: 71aa0000 71aa8000   I:\WINDOWS\system32\WS2HELP.dll
+                 * 
+                 * WARNING: WSOCK32 overlaps IMAGEHLP
+                 * WARNING: WSOCK32 overlaps urlmon
+                 * WARNING: WSOCK32 overlaps appHelp
+                 * WARNING: WSOCK32 overlaps btkeyind
+                 * ModLoad: 43aa0000 43aeb000   I:\WINDOWS\system32\WSOCK32.dll
+                 * ModLoad: 71ad0000 71ad9000   I:\WINDOWS\system32\WSOCK32.dll
+                 */
+            } else {
+                ASSERT_NOT_TESTED();
+                LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                    "ASLR: unexpected failure on NtMapViewOfSection prot=%s => "PFX"\n",
+                    prot_string(prot), reg_eax);
+
+                /* FIXME: note that we may be able to retry on out of
+                 * page file memory if there was a transient memory
+                 * use, though success is unlikely to be worthwhile
+                 */
+
+                /* just restore application request below, and retry */
+                ASSERT_CURIOSITY(false && "unexpected error status");
+                /* directly pass retried result to the application */
+                retries_left = 0;
+                retry_base = aslr_update_failed(false /* no retry */, NULL, 0);
+            }
+
+            if (retries_left == 0) {
+                dcontext->aslr_context.sys_aslr_clobbered = false;
+                ASSERT(retry_base == NULL);
+                /* we get here only when aslr_pre_process_mapview()
+                 * has verified the app request was for base 0
+                 */
+            }
+
+            safe_write(pbase_unsafe, sizeof(retry_base), &retry_base);
+
+            /* here we reset all IN/OUT arguments */
+
+            /* make sure that even on syscall failure OUT arguments aren't set */
+            ASSERT(*view_size == 0); /* we handle only when not set */
+            ASSERT(section_offs == NULL); /* optional, we handle only when not set */
+
+            /* we have to be able to handle failure of new base */
+            ASSERT(retry_base == 0 || dcontext->aslr_context.sys_aslr_clobbered);
+            ASSERT(*pbase_unsafe == retry_base); /* retry at base, 
+                                                  * unsafe ASSERT can take a risk */
+            /* retry with new mapping base - passing arguments */
+            retry_result = aslr_retry_map_syscall(dcontext, param_base);
+            SET_RETURN_VAL(dcontext, retry_result); /* sets reg_eax */
+
+            /* reread all OUT arguments since we have to handle
+             * the retried system call as if that's what really happened
+             */
+            ASSERT(section_handle == (HANDLE) postsys_param(dcontext, param_base, 0));
+            ASSERT(process_handle == (HANDLE) postsys_param(dcontext, param_base, 1));
+            pbase_unsafe = (void *) postsys_param(dcontext, param_base, 2);/* OUT */
+            ASSERT(zerobits == (uint) postsys_param(dcontext, param_base, 3));
+            ASSERT(commit_size == (size_t) postsys_param(dcontext, param_base, 4));
+            section_offs = (uint *) postsys_param(dcontext, param_base, 5);/* OUT */
+            view_size = (size_t *) postsys_param(dcontext, param_base, 6); /* OUT */
+            ASSERT(inherit_disposition == (uint) postsys_param(dcontext, param_base, 7));
+            ASSERT(type == (uint) postsys_param(dcontext, param_base, 8));
+            ASSERT(prot == (uint) postsys_param(dcontext, param_base, 9));
+
+            STATS_INC(aslr_error_retry);
+            DOSTATS({
+                if (!NT_SUCCESS(status)) {
+                    STATS_INC(aslr_error_on_retry);
+                } else {
+                    if (status == STATUS_SUCCESS) 
+                        STATS_INC(aslr_retry_at_base);
+                    else if (status == STATUS_IMAGE_NOT_AT_BASE)
+                        STATS_INC(aslr_retry_not_at_base);
+                    else
+                        ASSERT_NOT_REACHED();
+                }
+            });
+
+            /* we retry further only if we tried a different base, and
+             * otherwise leave to the application as it was
+             */
+        } while (!NT_SUCCESS(status) && 
+                 (retries_left > 0));
+
+        /* last retry is native, implication */
+        ASSERT(!(retries_left == 0) ||
+               !dcontext->aslr_context.sys_aslr_clobbered);
+        ASSERT(!dcontext->aslr_context.sys_aslr_clobbered || 
+               NT_SUCCESS(status));
+    }
+
+    DODEBUG({
+        if (dcontext->aslr_context.sys_aslr_clobbered
+            && NT_SUCCESS(status)) {
+            /* really handle success later, after safe read of base and size */
+
+            /* verify that we always get a (success) code */
+            /* STATUS_IMAGE_NOT_AT_BASE ((NTSTATUS)0x40000003L) 
+             *
+             * FIXME: I presume the loader maps MEM_MAPPED as
+             * MapViewOfSection(--x) and it maybe just reads the PE
+             * headers?  Only the MapViewOfSection(rw-) in fact returns
+             * STATUS_IMAGE_NOT_AT_BASE
+             */
+
+            /* Note the confusing mapping of MEM_MAPPED as --x, and of MEM_IMAGE as rw-! */
+            ASSERT_CURIOSITY(prot == PAGE_EXECUTE && status == STATUS_SUCCESS || 
+                             prot == PAGE_READWRITE && status == STATUS_IMAGE_NOT_AT_BASE);
+            /* FIXME: case 6736 is hitting this as well - assumed
+             * SEC_RESERVE 0x4000000, prot = RW, inherit_disposition = ViewUnmap
+             * and should simply allow that to get STATUS_SUCCESS
+             */
+
+            /* FIXME: case 2298 needs to check for /FIXED DLLs - are they
+             * going to fail above, or loader will fail when presented
+             * with them.
+             *
+             * FIXME: -exempt_aslr_list needs to be handled here 
+             * FIXME: need to reset all IN/OUT arguments
+             */
+        }
+    });
+
+    /* note this is failure after retrying at default base */
+    /* so if it fails it is not our fault
+     */
+    if (!NT_SUCCESS(status)) { 
+        ASSERT_NOT_TESTED();
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+            "ASLR: retry failed NtMapViewOfSection prot=%s => "PFX"\n",
+            prot_string(prot), reg_eax);
+        ASSERT_CURIOSITY(false);
+        
+        /* directly pass retried result to the application */
+        return;
+    }
+    
+    ASSERT(NT_SUCCESS(status));
+
+    /* we assume that since syscall succeeded these dereferences are safe 
+     * FIXME : could always be multi-thread races though */
+    size = *view_size; /* ignore commit_size? */
+    base = *((app_pc *)pbase_unsafe);
+
+    LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+        "ASLR: NtMapViewOfSection "PFX" size="PIFX" prot=%s => "PFX"\n",
+        base, size, prot_string(prot), reg_eax);
+
+    /* verify if need to exempt, only if we are still processing our randomization */
+    /* we are exempting only after the fact here */
+    /* keep in synch with is_aslr_exempted_file_name() */
+    if (dcontext->aslr_context.sys_aslr_clobbered && 
+        (!IS_STRING_OPTION_EMPTY(exempt_aslr_default_list) ||
+         !IS_STRING_OPTION_EMPTY(exempt_aslr_list) ||
+         !IS_STRING_OPTION_EMPTY(exempt_aslr_extra_list))) {
+        MEMORY_BASIC_INFORMATION mbi;
+
+        bool exempt = false; /* NOTE: we do not give up if can't find
+                              * name, case 3858 if no name */
+
+        /* -exempt_aslr_list '*' is really only interesting as a
+         * stress test option, otherwise should just turn off ASLR_DLL
+         */
+        if (IS_LISTSTRING_OPTION_FORALL(exempt_aslr_list))
+            exempt = true;
+
+        if (query_virtual_memory(base, &mbi, sizeof(mbi)) == sizeof(mbi)) {
+            ASSERT(mbi.Type == MEM_IMAGE || mbi.Type == MEM_MAPPED);
+
+            LOG(THREAD, LOG_SYSCALLS, 2, "ASLR: !vprot "PFX"\n", base);
+            DOLOG(2, LOG_SYSCALLS, {
+                dump_mbi(THREAD, &mbi, false);
+            });
+        } else
+            ASSERT_NOT_REACHED();
+
+        if (is_readable_pe_base(base)) {
+            /* Note that the loader first maps an image as MEM_MAPPED */
+            /* FIXME: in those allocations RVAs have to be converted
+             * for our reads of export table and thus PE name to work
+             * properly!
+             */
+            /*
+             * 0:000> !vprot 0x43ab0000 
+             *   BaseAddress:       43ab0000
+             *   AllocationBase:    43ab0000
+             *   AllocationProtect: 00000010  PAGE_EXECUTE
+             *   RegionSize:        00048000
+             *   State:             00001000  MEM_COMMIT
+             *   Protect:           00000010  PAGE_EXECUTE
+             *   Type:              00040000  MEM_MAPPED
+             */
+            if (mbi.Type == MEM_IMAGE) {
+                /* For MEM_IMAGE can properly get PE name.  We haven't yet added
+                 * to the loaded_module_areas so we can't use
+                 * get_module_short_name().  We could use
+                 * get_module_short_name_uncached(), but
+                 * is_aslr_exempted_file_name() uses file name only, so we use
+                 * that as well. (For example, in IE we have browselc.dll
+                 * filename vs BROWSEUI.DLL rsrc name, and we don't want the
+                 * user having to specify a different name for private vs shared
+                 * exemptions).
+                 */
+                const char *module_name = NULL;
+                bool alloc = false;
+                uint module_characteristics;
+                if (DYNAMO_OPTION(track_module_filenames))
+                    module_name = dcontext->aslr_context.last_section_file_name;
+                if (module_name == NULL) {
+                    alloc = true;
+                    module_name =
+                        get_module_short_name_uncached(dcontext, base, true/*at map*/
+                                                       HEAPACCT(ACCT_OTHER));
+                }
+
+                LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                    "ASLR: NtMapViewOfSection prot=%s mapped %s\n",
+                    prot_string(prot), module_name ? module_name : "<noname>");
+
+                /* note that although we are undoing randomization
+                 * of the MEM_IMAGE mapping (usually done in
+                 * ntdll!LdrpMapDll), we don't handle it when
+                 * loaded as MEM_MAPPED earlier */
+
+                ASSERT(module_name != NULL);
+                if (module_name != NULL &&
+                    check_list_default_and_append(dynamo_options.exempt_aslr_default_list,
+                                                  dynamo_options.exempt_aslr_list,
+                                                  module_name)) {
+                    SYSLOG_INTERNAL_WARNING("ASLR exempted DLL %s",
+                                            module_name);
+                    exempt = true;
+                }
+
+                if (module_name != NULL &&
+                    DYNAMO_OPTION(aslr_extra) &&
+                    check_list_default_and_append("", /* no default list */
+                                                  dynamo_options.exempt_aslr_extra_list,
+                                                  module_name)) {
+                    SYSLOG_INTERNAL_WARNING("ASLR exempted extra DLL %s",
+                                            module_name);
+                    exempt = true;
+                }
+
+                module_characteristics = get_module_characteristics(base);
+                if (TEST(IMAGE_FILE_DLL, module_characteristics) && 
+                    TEST(IMAGE_FILE_RELOCS_STRIPPED, module_characteristics)) {
+                    /* Note that we still privately ASLR EXEs that are
+                     * presumed to not be executable but only loaded
+                     * for ther resources */
+                    /* FIXME: case 2298 this test doesn't really work
+                     * for one version of /FIXED in our test suite as
+                     * security-win32/secalign-fixed.dll.c, yet works
+                     * for sec-fixed.dll.c*/
+                    SYSLOG_INTERNAL_WARNING("ASLR exempted /FIXED DLL %s",
+                                            module_name ? module_name : "noname");
+                    exempt = true;
+                }
+                DODEBUG({
+                    if (!exempt && !TEST(IMAGE_FILE_DLL, module_characteristics)) {
+                        /* EXE usually have no PE name, and note that we
+                         * see for example in notepad.exe help on (xp sp2)
+                         * we get helpctr.exe loaded as
+                         * C:\WINDOWS\PCHealth\HelpCtr\Binaries\HelpCtr.exe
+                         *  LDRP_ENTRY_PROCESSED
+                         *  LDRP_IMAGE_NOT_AT_BASE
+                         */
+                        SYSLOG_INTERNAL_INFO("ASLR note randomizing mapped EXE %s",
+                                             module_name != NULL ? module_name : 
+                                             "noname");
+                    }
+                });
+
+                /* add to preferred module range only if MEM_IMAGE */
+                if (TEST(ASLR_TRACK_AREAS, DYNAMO_OPTION(aslr_action))
+                    && !exempt) {
+                    /* FIXME: only DLLs that are randomized by us get added,
+                     * not any DLL rebased due to other conflicts (even if
+                     * due to overlap our own allocations we don't take blame)
+                     */
+                    /* FIXME: case 8490 on moving out */
+                    aslr_track_randomized_dlls(dcontext, base, size, true /* Map */,
+                                               false /* Original File */); 
+                }
+
+                if (alloc && module_name != NULL)
+                    dr_strfree(module_name HEAPACCT(ACCT_OTHER));
+            } else {
+                ASSERT(mbi.Type == MEM_MAPPED);
+                /* FIXME: case 5325 still have to call get_dll_short_name()
+                 * alternative that knows to use our ImageRvaToVa() FIXME: case 6766 
+                 * to get the PE name and properly exempt these mappings
+                 */
+                /* Note: Although ntdll!LdrpCheckForLoadedDll maps DLL
+                 * as MEM_MAPPED and we'll currently randomize that,
+                 * it in fact doesn't depend on this mapping to be at
+                 * the normal DLL location.  We will not exempt here.
+                 */
+                LOG(THREAD, LOG_SYSCALLS, 1, 
+                    "ASLR: NtMapViewOfSection "PFX" module not mapped as image!\n", base);
+                STATS_INC(app_mmap_PE_as_MAPPED);
+                /* FIXME: we do not check nor set exempt here! */
+            }
+        } else {
+            /* FIXME: case 6737 ASLR_MAPPED should we rebase other
+             * mappings that are not PEs?  Reversing note: seen in
+             * notepad help, and currenty rebased even for ASLR_DLL
+             *
+             * <?xml version="1.0" ...> 
+             * <assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0"> 
+             * <assemblyIdentity processorArchitecture="*" version="5.1.0.0" 
+             * type="win32" name="Microsoft.Windows.Shell.shell32"/>     
+             * <description>Windows Shell</description>  
+             * 
+             * 00b664e4 7c91659e ntdll!LdrGetDllHandleEx+0x258
+             * 00b66500 7c801d1f ntdll!LdrGetDllHandle+0x18
+             * 00b66568 7c816f55 kernel32!LoadLibraryExW+0x161 "I:\WINDOWS\WindowsShell.manifest"
+             * 00b66594 7c816ed5 kernel32!BasepSxsFindSuitableManifestResourceFor+0x51
+             * 00b66894 7d58f157 kernel32!CreateActCtxW+0x69e
+             * 00b66acc 7d58f0a8 mshtml!DllGetClassObject+0x1291
+            */
+            LOG(THREAD, LOG_SYSCALLS, 1, 
+                "ASLR: NtMapViewOfSection "PFX" not a module!\n", base);
+            STATS_INC(app_mmap_not_PE_rebased);
+        }
+
+        if (exempt) {
+            /* have to undo and redo app mapping */
+            app_pc redo_base = 0;
+            app_pc redo_size = 0;
+            int redo_result;
+            /* undo: issue unmap on what we have bumped */
+            NTSTATUS res = nt_unmap_view_of_section(process_handle, base);
+            LOG(THREAD_GET, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                "syscall: aslr exempt: NtUnmapViewOfSection base="PFX", res "PFX"\n",
+                base, res);
+            ASSERT(NT_SUCCESS(res));
+            /* if we cannot unmap our own mapping we're in trouble, but app should be ok
+             * it will just have some wasted memory, we can continue
+             */
+
+            /* here we reset IN/OUT arguments in our current param_base 
+             * (currently only pbase_unsafe and view_size),
+             * then retry just as above to remap at good base. */
+            safe_write(pbase_unsafe, sizeof(redo_base), &redo_base);
+            /* redo OUT argument view_size, whose value would have changed */
+            ASSERT_CURIOSITY(*view_size != 0);
+            safe_write(view_size, sizeof(redo_size), &redo_size);
+            ASSERT(*view_size == 0); /* we handle only when not set originally */
+
+            ASSERT(section_offs == NULL); /* optional, we handle only when not set */
+
+            /* no plans on trying a different base */
+            ASSERT(*pbase_unsafe == 0); /* retry at base, unsafe ASSERT can take a risk */
+
+            /* retry with new mapping base - passing arguments */
+            redo_result = aslr_retry_map_syscall(dcontext, param_base);
+            SET_RETURN_VAL(dcontext, redo_result); /* sets reg_eax */
+
+            LOG(THREAD_GET, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                "syscall: aslr exempt: NtMapViewOfSection got base="PFX", res "PFX"\n",
+                *pbase_unsafe, res);
+
+            /* no further processing of arguments here */
+
+            /* this worked for us, it should succeed for app, though there
+             * may be a conflict at the original base, while ours was good
+             */
+            ASSERT_CURIOSITY(NT_SUCCESS(status));
+
+            ASSERT(dcontext->aslr_context.sys_aslr_clobbered);
+            aslr_update_failed(false /* no retry */, NULL, 0);
+            dcontext->aslr_context.sys_aslr_clobbered = false;
+            STATS_INC(aslr_dlls_exempted);
+        }
+    }
+
+    /* update if randomized, but not if had to retry on conflict, or if exempted */
+    if (dcontext->aslr_context.sys_aslr_clobbered) {
+        aslr_update_view_size(base, size);
+        dcontext->aslr_context.sys_aslr_clobbered = false;
+    }
+}
+
+/* PRE hook for NtUnmapViewOfSection */
+void aslr_pre_process_unmapview(dcontext_t *dcontext, 
+                                app_pc base, size_t size)
+{
+    reg_t *param_base = dcontext->sys_param_base;
+
+    /* remove from preferred module range */
+    if (TEST(ASLR_TRACK_AREAS, DYNAMO_OPTION(aslr_action))) {
+        /* FIXME: should move to post processing in
+         * aslr_post_process_mapview, for the unlikely case
+         * NtUnmapViewOfSection fails, and so that we remove only when
+         * really removed.  We need to preserve all our data across
+         * system call.
+         */
+        aslr_track_randomized_dlls(dcontext, base, size, false /* Unmap */, 
+                                   false);
+    }
+
+    /* FIXME: need to mark in our range or vmmap that memory
+     * is available Note that the loader always does a
+     * MapViewOfSection(--x);UnmapViewOfSection();MapViewOfSection(rw-); 
+     * so we'll leave a growing hole in case of DLL churn - 
+     * case 6739 about virtual memory reclamation
+     * case 6729 on committed memory leaks and optimizations this also affects
+     */
+    ASSERT_NOT_IMPLEMENTED(true);
+}
+
+/* POST processing of NtUnmapViewOfSection with possibly clobbered by us base */
+reg_t
+aslr_post_process_unmapview(dcontext_t *dcontext)
+{
+    reg_t *param_base = dcontext->sys_param_base;
+    reg_t reg_eax = get_mcontext(dcontext)->xax;
+    NTSTATUS status = (NTSTATUS) reg_eax; /* get signed result */
+
+    ASSERT_NOT_IMPLEMENTED(false);
+    return reg_eax;
+}
+
+#ifdef DEBUG
+/* doublecheck wouldbe areas subset of loaded module preferred ranges 
+ * by removing all known loaded modules preferred ranges
+ * returns true if vmvector_empty(aslr_wouldbe_areas)
+ * Call once only!
+ */
+static bool
+aslr_doublecheck_wouldbe_areas(void)
+{
+    module_iterator_t *iter = module_iterator_start();
+    while (module_iterator_hasnext(iter)) {
+        size_t size;
+        module_area_t *ma = module_iterator_next(iter);
+        ASSERT (ma != NULL);
+        size = (ma->end - ma->start);
+
+        /* not all modules are randomized, ok not to find an overlapping one */
+        vmvector_remove(aslr_wouldbe_areas, 
+                        ma->os_data.preferred_base, 
+                        ma->os_data.preferred_base + size);
+    }
+    module_iterator_stop(iter);
+
+    return vmvector_empty(aslr_wouldbe_areas);
+}
+#endif /* DEBUG */
+
+bool
+aslr_is_possible_attack(app_pc target_addr)
+{
+    /* FIXME: split by ASLR_DETECT_EXECUTE, ASLR_DETECT_READ,
+     * ASLR_DETECT_WRITE */
+
+    /* FIXME: case 7017 case 6287 check aslr_heap_pad_areas - less
+     * clear that this is an attack rather than stray execution, so
+     * we'd want that check under a different flag.
+     * 
+     * FIXME: case TOFILE: should have a flag to detect any read/write
+     * exceptions in the aslr_wouldbe_areas or aslr_heap_pad_areas
+     * areas and make sure they are incompatibilities or real
+     * application bugs, although maybe present only with
+     * randomization so considered incompatibilities.
+     */
+    return TESTALL(ASLR_TRACK_AREAS | ASLR_DETECT_EXECUTE, DYNAMO_OPTION(aslr_action)) && 
+        vmvector_overlap(aslr_wouldbe_areas, target_addr, target_addr + 1);
+}
+
+/* returns NULL if not relevant or not found */
+app_pc
+aslr_possible_preferred_address(app_pc target_addr)
+{
+    if (TESTALL(ASLR_TRACK_AREAS | ASLR_DETECT_EXECUTE, DYNAMO_OPTION(aslr_action))) {
+        app_pc wouldbe_module_current_base = 
+            vmvector_lookup(aslr_wouldbe_areas, target_addr);
+        app_pc wouldbe_preferred_base;
+        if (wouldbe_module_current_base == NULL) {
+            /* note we check according to aslr_action (e.g. always
+             * since default on) even in case ASLR was never enabled,
+             * to be able to handle having -aslr dynamically disabled.
+             * We add areas only when ASLR is enabled.
+             */
+            return NULL;
+        }
+
+        /* note that we don't have a vmvector interface to get the
+         * base of the wouldbe area, from which we got this */
+        /* but we anyways doublecheck with the loaded_module_areas as well */
+        /* FIXME: such an interface is being added on the Marlin branch, use when ready */
+        wouldbe_preferred_base = 
+            get_module_preferred_base_safe(wouldbe_module_current_base);
+        ASSERT(vmvector_lookup(aslr_wouldbe_areas, wouldbe_preferred_base)
+               == wouldbe_module_current_base ||
+               /* FIXME case 10727: if serious then let's fix this */
+               check_filter("reload-race.exe",
+                            get_short_name(get_application_name())));
+        return target_addr - wouldbe_preferred_base 
+            + wouldbe_module_current_base;
+    } else {
+        ASSERT_NOT_TESTED();
+        return NULL;
+    }
+}
+
+static bool
+aslr_reserve_remote_random_pad(HANDLE process_handle, size_t pad_size)
+{
+    NTSTATUS res;
+    HANDLE child_handle = process_handle;
+    void *early_reservation_base = NULL; /* allocate earliest possible address */
+
+    size_t early_reservation_delta = get_random_offset(pad_size);
+    size_t early_reservation_size = ALIGN_FORWARD(early_reservation_delta,
+                                                ASLR_MAP_GRANULARITY);
+    ASSERT(!is_phandle_me(process_handle));
+    
+    res = nt_remote_allocate_virtual_memory(child_handle,
+                                            &early_reservation_base,
+                                            early_reservation_size, PAGE_NOACCESS,
+                                            MEMORY_RESERVE_ONLY);
+    ASSERT(NT_SUCCESS(res));
+    /* not a critical failure if reservation has failed */
+
+    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+        "ASLR: initial padding in child "PIFX", pad base="PFX", size="PIFX", res=0x%x\n",
+        child_handle, 
+        early_reservation_base, early_reservation_size, res);
+
+    /* FIXME: case 7017 should pass the early
+     * reservation region to child for detecting exploits
+     * targeting a predictable stack (for ASLR_STACK), need to identify a good
+     * stable across core versions mechanism.  Once that's there
+     * child should ASSERT its stack immediately follows this.
+     *
+     * Alternatively, for case 5366 we may choose to free this
+     * padding, and if freeing we can use a lot larger initial one,
+     * risking only fragmentation
+     */
+    return NT_SUCCESS(res);
+}
+
+/* FIXME: this routine bases its decisions on the parent options
+ * instead of the target process, currently controlled by option
+ * string options, too much effort to check remotely.
+ */
+/* may decide that the target is not a stack */
+void
+aslr_maybe_pad_stack(dcontext_t *dcontext, HANDLE process_handle)
+{
+    /* note that should be careful to properly detect only this is
+     * done before very first thread injection in a newly created
+     * process, otherwise we'd risk a virtual memory leak
+     *
+     * FIXME: case 7682 tracks correctly identifying remote thread
+     * injectors other than parent process
+     */
+    ASSERT(!is_phandle_me(process_handle));
+
+    /* we should only handle remote reservation from parent to
+     * child */
+
+    /* we check if child is at all configured, note that by doing this
+     * check only for a presumed thread stack, we can rely on
+     * ProcessParameters being created.  FIXME: Since the
+     * ProcessParameters will get normalized from offsets to pointers
+     * only when the child starts running, if this is not a first
+     * child we may get a random or incorrect value - e.g. the global
+     * settings if the read name is not good enough.
+     */
+    /* Remotely injected threads should not need this since will get
+     * their padding from the general ASLR_HEAP in the child.
+     */
+    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+        "ASLR: check if thread is in new child "PIFX"\n", process_handle);
+
+    if (TEST(ASLR_STACK, DYNAMO_OPTION(aslr)) &&
+        DYNAMO_OPTION(aslr_parent_offset) > 0 &&
+        should_inject_into_process(get_thread_private_dcontext(), 
+                                   process_handle,
+                                   NULL, NULL)) {
+        /* Case 9173: ensure we only do this once, as 3rd party
+         * hookers allocating memory can cause this routine to be
+         * invoked many times for the same child
+         */
+        process_id_t pid = process_id_from_handle(process_handle);
+        if (pid == dcontext->aslr_context.last_child_padded) {
+            SYSLOG_INTERNAL_WARNING_ONCE("extra memory allocations for child "PIFX
+                                         " %d: hooker?", process_handle, pid);
+        } else {
+            bool ok = aslr_reserve_remote_random_pad(process_handle, 
+                                                     DYNAMO_OPTION(aslr_parent_offset));
+            ASSERT(ok);
+            if (ok)
+                dcontext->aslr_context.last_child_padded = pid;
+        }
+    } else {
+        DODEBUG({
+            if (TEST(ASLR_STACK, DYNAMO_OPTION(aslr)) &&
+                DYNAMO_OPTION(aslr_parent_offset) > 0) {
+                LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+                    "ASLR: child not configured for protection, not padding\n");
+            }
+        });
+    }
+}
+
+#define LOADBLOCK_PAGE_PROTECT (PAGE_READWRITE | PAGE_GUARD)
+
+/* Prevents the loader from mapping our DLL at its preferred base, by
+ * allocating a random initial padding before we're loaded, padding is
+ * independent of ASLR_STACK or ASLR_PROCESS_PARAM if those aren't
+ * enabled.
+ *
+ * Child process should call aslr_free_dynamorio_loadblock() to free
+ * FIXME: may want to make this routine available for pre_inject.c
+ *
+ * Note option active in parent and applies to its children.
+ * FIXME: Eventually should share views similar to ASLR_SHARED_CONTENT.
+ */
+void
+aslr_force_dynamorio_rebase(HANDLE process_handle)
+{
+    /* We'll assume that the preferred base is the same in parent and
+     * child */
+    app_pc preferred_base = get_dynamorio_dll_preferred_base();
+    NTSTATUS res;
+    DEBUG_DECLARE(bool ok;)
+    LOG(THREAD_GET, LOG_SYSCALLS|LOG_THREADS, 1, "\ttaking over expected DLL base\n");
+
+    ASSERT(DYNAMO_OPTION(aslr_dr));
+
+    ASSERT(!is_phandle_me(process_handle));
+
+    res = nt_remote_allocate_virtual_memory(process_handle, 
+                                            &preferred_base,
+                                            1*PAGE_SIZE, LOADBLOCK_PAGE_PROTECT,
+                                            MEM_RESERVE);
+    ASSERT(NT_SUCCESS(res));
+    /* not critical if we fail, though failure expected only if
+     * the target executable is also at our preferred base */
+
+    /* child process should free the page at preferred base if it
+     * looks like what we have created to not fragment the address
+     * space.
+     */
+
+    /* no need to do both */
+    if (!(TEST(ASLR_STACK, DYNAMO_OPTION(aslr)))) {
+        /* random padding to have the loader load us in a not so
+         * determinstic location */
+        DEBUG_DECLARE(ok = )
+            aslr_reserve_remote_random_pad(process_handle, 
+                                           DYNAMO_OPTION(aslr_parent_offset));
+        ASSERT(ok);
+    } else {
+        /* do nothing, ASLR_STACK will add a pad */
+    }
+    /* FIXME: note that we should pass this region just as ASLR_STACK
+     * is supposed to so that the child can free that region, yet only
+     * at beginning of address space, and can double as extra heap
+     * randomization
+     */
+}
+
+void
+aslr_free_dynamorio_loadblock(void)
+{
+    /* we don't want the l-roadblock to be a tombstone and get in the
+     * way of other allocations, so we'll try to clean it up.  
+     */
+
+    /* we also need to make sure that we have the preferred_base base
+     * collected earlier */
+    app_pc preferred_base = get_dynamorio_dll_preferred_base();
+    NTSTATUS res;
+    MEMORY_BASIC_INFORMATION mbi;    
+
+    /* note that parent may have had different settings */
+    ASSERT(DYNAMO_OPTION(aslr_dr));
+
+    if (get_dynamorio_dll_start() == preferred_base) {
+        /* not rebased, no loadblock to free */
+        return;
+    }
+
+    /* first check whether we have allocated this */
+    if (query_virtual_memory(preferred_base, &mbi, sizeof(mbi)) == sizeof(mbi)) {
+        /* FIXME: the only way to get a strong guarantee that no other
+         * block is allocated at our preferred base by passing the
+         * loadblock information to the child.  This check using
+         * trying unusual combination of State and AllocationProtect
+         * will make very unlikely we'd accidentally free something
+         * else. */
+        if (mbi.RegionSize == PAGE_SIZE &&
+            mbi.State == MEM_RESERVE && 
+            mbi.Type == MEM_PRIVATE &&
+            mbi.AllocationProtect == LOADBLOCK_PAGE_PROTECT) {
+            LOG(GLOBAL, LOG_SYSCALLS|LOG_THREADS, 1, "\t freeing loadblock at preferred base\n");
+            res = nt_free_virtual_memory(preferred_base);
+            ASSERT(NT_SUCCESS(res));
+        } else {
+            /* We'd expect mbi.State==MEM_FREE, or the large reserved block
+             * that cygwin apps use if we come in late, or an executable
+             * at our preferred base (for which this will fire).
+             */
+            ASSERT_CURIOSITY(mbi.State == MEM_FREE || !dr_early_injected);
+            LOG(GLOBAL, LOG_SYSCALLS|LOG_THREADS, 1,
+                "something other than loadblock, leaving as is\n");
+        }
+    }
+}
+
+
+/* post processing of succesful application reservations */
+void
+aslr_post_process_allocate_virtual_memory(dcontext_t *dcontext, 
+                                          app_pc last_allocation_base, 
+                                          size_t last_allocation_size)
+{
+    ASSERT(ALIGNED(last_allocation_base, PAGE_SIZE));
+    ASSERT(ALIGNED(last_allocation_size, PAGE_SIZE));
+
+    ASSERT(TEST(ASLR_HEAP_FILL, DYNAMO_OPTION(aslr)));
+    if (DYNAMO_OPTION(aslr_reserve_pad) > 0) {
+        /* We need to randomly pad memory around each memory
+         * allocation as well.  Conservatively, we reserve a new
+         * region after each succesful native reservation and would
+         * have to free it whenever the target region itself is freed.
+         * Assumption: one can't free separately allocated regions
+         * with a single NtFreeVirtualMemory.
+         *
+         * Alternatively we can increase the size of the allocation,
+         * at the risk of breaking some application.  Further even
+         * more risky, within a larger reservation, we could return a
+         * base that is not at the allocation granularity (but I
+         * wouldn't consider not returning at page granularity).
+         * Instead of actually keeping the reservation we could just
+         * forcefully reserve at a slighly padded address without
+         * really keeping the reservation ourselves.
+         */
+        heap_error_code_t error_code;
+        size_t heap_pad_delta = 
+            get_random_offset(DYNAMO_OPTION(aslr_reserve_pad));
+        size_t heap_pad_size = ALIGN_FORWARD(heap_pad_delta,
+                                             ASLR_MAP_GRANULARITY);
+        app_pc heap_pad_base;
+        app_pc append_heap_pad_base = (app_pc) 
+            ALIGN_FORWARD(last_allocation_base + last_allocation_size,
+                          ASLR_MAP_GRANULARITY);
+        bool immediate_taken = 
+            get_memory_info(append_heap_pad_base, NULL, NULL, NULL);
+        /* there may be an allocation immediately tracking us, or a
+         * hole too small for our request.  
+         *
+         * FIXME: get_memory_info() should provide size of hole, but
+         * can't change the interface on Linux easily, so not using
+         * that for now, so we just try
+         */
+
+        if (immediate_taken) {
+            STATS_INC(aslr_heap_giveup_filling);
+            /* FIXME: TOFILE we shouldn't give up here if we also want
+             * to fill uniformly */
+
+            /* currently not adding a pad if the immediate next region
+             * is already allocated e.g. MEM_MAPPED, or due to best
+             * fit allocation/fragmentation virtual memory allocation
+             * is in non-linear order)
+             */
+
+            LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                "ASLR: ASLR_HEAP: giving up since next region "PFX" is taken\n",
+                append_heap_pad_base);
+            
+            return;
+        }
+
+        /* racy memory reservation with respect to other threads, but
+         * if we didn't get one where we wanted it to be, unlikely to
+         * be useful to attackers if not deterministic.
+         */
+        heap_pad_base = os_heap_reserve(append_heap_pad_base,
+                                        heap_pad_size,
+                                        &error_code);
+        if (heap_pad_base == NULL) {
+            /* unable to get preferred, let the os pick a spot */
+            /* FIXME - remove this - no real reason to reserve if we can't get our
+             * preferred, but the old os_heap_reserve implementation automatically
+             * tried again for us and the code below assumes so. */
+            heap_pad_base = os_heap_reserve(NULL, heap_pad_size, &error_code);
+        }
+
+        LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 2,
+            "ASLR: ASLR_HEAP: reserved pad base="PFX", size="PIFX", err=%x, after "PFX"\n",
+            heap_pad_base, heap_pad_size, error_code, append_heap_pad_base);
+
+        ASSERT_CURIOSITY(NT_SUCCESS(error_code) || 
+                         check_filter("oomtest.exe",
+                                      get_short_name(get_application_name())));
+        /* not critical functionality loss if we have failed to
+         * reserve this memory, but shouldn't happen */
+        if (NT_SUCCESS(error_code)) {
+            /* FIXME: currently nt_remote_allocate_virtual_memory()
+             * automatically retries for the next available region and for
+             * dual meaning of padding to mean waste some memory to detect
+             * brute force fill attacks, we can keep the allocation.
+             *
+             * However, we'd need a way to quickly lookup a region getting
+             * freed to find its corresponding pad.
+             * FIXME: For now on race I'd immediately give up the padding.
+             */
+            /* FIXME: we checked earlier only if the immediate next
+             * region is already allocated, but when the size of the
+             * allocation is too large we also miss here
+             */
+            if (heap_pad_base != append_heap_pad_base) {
+                size_t existing_size = 0;
+                bool now_immediate_taken = 
+                    get_memory_info(append_heap_pad_base, NULL, &existing_size, NULL);
+                /* FIXME: possible to simply not have enough room in current hole */
+                /* or somebody else already got the immediate next region */
+                ASSERT_CURIOSITY(!now_immediate_taken && "racy allocate");
+
+                /* FIXME: get_memory_info() DOESN'T fill in size when MEM_FREE, 
+                 * this DOESN'T actually check existing_size it's just 0 */
+                if (!now_immediate_taken && existing_size < heap_pad_size) {
+                    /* FIXME: should we at least fill the hole? */
+                    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 2,
+                        "ASLR: ASLR_HEAP: giving up, hole after region "PFX
+                        " is too small, req "PIFX" hole\n",
+                        append_heap_pad_base, heap_pad_size);
+                    /* FIXME: need to keep track of these - is there too much fragmentation? */
+                }
+
+                STATS_INC(aslr_heap_giveup_filling);
+                os_heap_free(heap_pad_base, heap_pad_size, &error_code);
+                LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 2,
+                    "ASLR: ASLR_HEAP: giving up, freed pad base="PFX", size="
+                    PIFX", err=0x%x\n",
+                    heap_pad_base, heap_pad_size, error_code);
+                ASSERT(NT_SUCCESS(error_code));
+            } else {
+                /* register this allocation 
+                 * 1) so that we can lookup free from original base and return the memory
+                 * 
+                 * 2) so that we can detect attempted execution from it and flag
+                 */
+                ASSERT(!vmvector_overlap(aslr_heap_pad_areas, 
+                                         heap_pad_base, 
+                                         heap_pad_base + heap_pad_size));
+                /* FIXME: case 7017 should check the reservation region
+                 * for detecting attacks targeting predictable heaps or
+                 * brute force heap fill style attacks */
+                vmvector_add(aslr_heap_pad_areas, 
+                             heap_pad_base, 
+                             heap_pad_base + heap_pad_size,
+                             last_allocation_base /* tag to match reservations */);
+                ASSERT(vmvector_overlap(aslr_heap_pad_areas, 
+                                        heap_pad_base, 
+                                        heap_pad_base + heap_pad_size));
+                ASSERT(vmvector_lookup(aslr_heap_pad_areas, 
+                                       heap_pad_base) == last_allocation_base);
+                STATS_ADD_PEAK(aslr_heap_total_reservation, heap_pad_size/1024);
+                STATS_ADD_PEAK(aslr_heap_pads, 1);
+                STATS_INC(ever_aslr_heap_pads);
+            }
+        } else {
+            SYSLOG_INTERNAL_WARNING("ASLR_HEAP_FILL: error "PIFX" on ("PFX","PFX")\n", 
+                                    error_code, append_heap_pad_base, 
+                                    append_heap_pad_base + heap_pad_size);
+
+            /* FIXME: should try to flag if out of memory - could be
+             * an application incompatible with too aggressive ASLR_HEAP_FILL
+             *
+             * (NTSTATUS) 0xc00000f2 - An invalid parameter was passed to a
+             * service or function as the fourth argument.
+             *
+             * This was the result of 0x7ff90000+80000 = 0x80010000 which of course is an invalid region.
+             */
+
+            /* or
+             * Error code: (NTSTATUS) 0xc0000017 (3221225495) - {Not Enough Quota}  Not enough virtual memory or paging file quota is available to complete the specified operation.
+             */
+            ASSERT_CURIOSITY(error_code == STATUS_INVALID_PARAMETER_4 ||
+                             error_code == STATUS_NO_MEMORY);
+
+        }
+    }
+}
+
+/* should be called before application memory reservation is released.
+ * Note that currently in addition to explicit memory free it is also
+ * called for implicit stack release on XP+.
+ * If application system call fails not a critical failure that we have freed a pad.
+ */
+void
+aslr_pre_process_free_virtual_memory(dcontext_t *dcontext, 
+                                     app_pc freed_base, size_t freed_size)
+{
+    /* properly adjusted base and size for next allocation unit */
+    app_pc expected_pad_base = (app_pc)
+        ALIGN_FORWARD(freed_base + freed_size,
+                      ASLR_MAP_GRANULARITY);
+    app_pc heap_pad_base, heap_pad_end;
+    size_t heap_pad_size;
+    heap_error_code_t error_code;
+    ASSERT(ALIGNED(freed_base, PAGE_SIZE));
+    ASSERT(ALIGNED(freed_size, PAGE_SIZE));
+
+    /* should have had a pad */
+
+    if (vmvector_lookup(aslr_heap_pad_areas,
+                        expected_pad_base) != NULL) {
+        /* case 6287: due to handling MEM_COMMIT on stack allocations
+         * now it is possible that the original MEM_RESERVE allocation
+         * fails to pad (e.g. due to a MEM_MAPPED) allocation, yet the
+         * later MEM_RESERVE|MEM_COMMIT has a second chance.  Rare, so
+         * leaving for now. */
+        ASSERT_CURIOSITY(vmvector_lookup(aslr_heap_pad_areas, 
+                                         expected_pad_base) == freed_base);
+        
+        /* need to remove atomically to make sure that nobody else is
+         * freeing the same region at this point, otherwise on an
+         * application double free race, we may attempt to double free
+         * a region that may have been given back to the application.
+         */
+        vmvector_remove_containing_area(aslr_heap_pad_areas, expected_pad_base,
+                                        &heap_pad_base, &heap_pad_end);
+        ASSERT(heap_pad_base == expected_pad_base);
+        ASSERT_CURIOSITY(!vmvector_overlap(aslr_heap_pad_areas, 
+                                           expected_pad_base, 
+                                           expected_pad_base + 1));
+
+        /* have to free it up, even if we picked the wrong pad we
+         * already removed it from vmvector */
+        heap_pad_size = heap_pad_end - heap_pad_base;
+        os_heap_free(heap_pad_base, heap_pad_size, &error_code);
+        LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 2,
+            "ASLR: ASLR_HEAP: freed pad base="PFX", size="PIFX", err=0x%x\n",
+            heap_pad_base, heap_pad_size, error_code);
+        ASSERT(NT_SUCCESS(error_code));
+
+        STATS_SUB(aslr_heap_total_reservation, (heap_pad_size/1024));
+        STATS_DEC(aslr_heap_pads);
+    } else {
+        /* no overlap */
+        ASSERT_CURIOSITY(!vmvector_overlap(aslr_heap_pad_areas, 
+                                           expected_pad_base, 
+                                           expected_pad_base + 1));
+    }
+}
+
+/* called at startup to randomize immediately after known fixed
+ * addresses, note that if a hole at preferred_base is not available
+ * we let the OS choose an allocation
+ */
+static
+app_pc
+aslr_reserve_initial_heap_pad(app_pc preferred_base, size_t reserve_offset)
+{
+    size_t heap_initial_delta = 
+        get_random_offset(reserve_offset);
+    heap_error_code_t error_code;
+    size_t heap_reservation_size = ALIGN_FORWARD(heap_initial_delta,
+                                               ASLR_MAP_GRANULARITY);
+    app_pc heap_reservation_base = os_heap_reserve(preferred_base, 
+                                                   heap_reservation_size, 
+                                                   &error_code);
+    if (heap_reservation_base == NULL) {
+        /* unable to get a preferred, let the os pick a spot */
+        /* FIXME - remove this - no real reason to reserve if we can't get our
+         * preferred, but the old os_heap_reserve implementation automatically
+         * tried again for us and the code below assumes so. */
+        heap_reservation_base = os_heap_reserve(NULL, heap_reservation_size, &error_code);
+    }
+    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+        "ASLR: ASLR_HEAP: requested random offset="PFX",\n"
+        "ASLR: reservation base="PFX", real size="PFX", err=0x%x\n",
+        reserve_offset, 
+        heap_reservation_base, heap_reservation_size, error_code);
+
+    ASSERT_CURIOSITY(NT_SUCCESS(error_code));
+    /* not critical functionality loss if we have failed to
+     * reserve this memory, but shouldn't happen */
+    if (NT_SUCCESS(error_code)) {
+        /* register this allocation */
+        STATS_ADD(aslr_heap_initial_reservation, heap_reservation_size/1024);
+        STATS_ADD_PEAK(aslr_heap_total_reservation, heap_reservation_size/1024);
+        STATS_ADD_PEAK(aslr_heap_pads, 1);
+        STATS_INC(ever_aslr_heap_pads);
+
+        ASSERT(!vmvector_overlap(aslr_heap_pad_areas, 
+                                 heap_reservation_base, 
+                                 heap_reservation_base + heap_reservation_size));
+        /* FIXME: case 7017 should check the reservation region
+         * for detecting attacks targeting predictable heaps or
+         * brute force heap fill style attacks */
+        vmvector_add(aslr_heap_pad_areas, 
+                     heap_reservation_base, 
+                     heap_reservation_base + heap_reservation_size,
+                     preferred_base);
+        /* Note breaking invariant for custom field - this is not
+         * base of previous allocation but initial padding or
+         * executable are not supposed to be freed, and in case
+         * there was a smaller region in front of our pad that
+         * gets freed we still get to keep it 
+         */
+    }
+    return heap_reservation_base;
+}
+
+/* release all heap reservation pads go through the
+ * aslr_heap_pad_areas, used on exit or detach.  There will still
+ * be lasting effects due to fragmentation.
+ *
+ * FIXME: case 6287 on application! or on DR out of reservation memory
+ * should release all heap pads as well - the big initial reservations
+ * should help free up some.  Should do if case 6498 can be reproduced
+ * with inflated reservation sizes.  Yet attackers may control the
+ * reservation sizes and would force a failing large request, or may
+ * be able to fill all available heap in smaller requests.
+ */
+static
+void
+aslr_free_heap_pads(void)
+{
+    
+    DEBUG_DECLARE(uint count_freed = 0;)
+
+    vmvector_iterator_t vmvi;
+    vmvector_iterator_start(aslr_heap_pad_areas, &vmvi);
+    while (vmvector_iterator_hasnext(&vmvi)) {
+        app_pc start, end;
+        app_pc previous_base = vmvector_iterator_next(&vmvi, &start, &end);
+        app_pc heap_pad_base = start; /* assuming not overlapping */
+        size_t heap_pad_size = (end - start);
+        heap_error_code_t error_code;
+        
+        os_heap_free(heap_pad_base, heap_pad_size, &error_code);
+        LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 2,
+            "ASLR: ASLR_HEAP: final cleanup pad base="PFX", size="PIFX
+            ", app_base="PFX", err=0x%x\n",
+            heap_pad_base, heap_pad_size, previous_base, error_code);
+        ASSERT(NT_SUCCESS(error_code));
+
+        STATS_SUB(aslr_heap_total_reservation, (heap_pad_size/1024));
+        STATS_DEC(aslr_heap_pads);
+        DODEBUG({count_freed++;});
+    }
+    vmvector_iterator_stop(&vmvi);
+    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, "aslr_free_heap_pads: %d freed\n", count_freed);
+}
+
+/* ASLR_SHARED_CONTENTS related functions */
+
+/* File backing seems to be unavoidable to allow using our own private
+ * section of a file, that is sharing the relocated version.  See
+ * comments in aslr_experiment_with_section_handle() about the
+ * different layers of file, section, and view content backing and
+ * sharing.
+ */
+
+/* currently doesn't require callers to close_handle(),
+ * Note: subject to change if we support impersonation
+ */
+HANDLE
+get_relocated_dlls_filecache_directory(bool write_access)
+{
+    /* FIXME: today both publishers and producers are getting full access */
+
+    /* the file cache may be per user (though impersonation may make
+     * this harder) we'll assume that only cache of initial caller
+     * should be used, while all other later impersonations will
+     * simply fail to share anything.  (The FTP problem still exists
+     * if the FTP user is allowed to create files.)
+     */
+    /* FIXME: currently only a single shared one, with full permissions */
+    return relocated_dlls_filecache_initial;
+}
+
+/* opens the DLL cache directory for this user,
+ * for now assuming both read and write privileges and opening a shared directory.
+ */
+/* FIXME: if per user may keep in e.g. \??\Program
+ * Files\Determina\Determina Agent\cache\USER or some other better
+ * per-USER location not under \Program Files.  Note nodemgr may not
+ * be able to create these directories in advance, e.g. in a domain
+ * where a new user may log in at any time.  For such a scenario,
+ * maybe we really wouldn't want the per-user directory at all...
+ */
+static HANDLE
+open_relocated_dlls_filecache_directory(void)
+{
+    char base_directory[MAXIMUM_PATH];
+    wchar_t wbuf[MAXIMUM_PATH];
+    HANDLE directory_handle;
+    int retval;
+    bool per_user = TEST(ASLR_SHARED_PER_USER, DYNAMO_OPTION(aslr_cache));
+    
+    /* FIXME: note a lot of overlap with code in os_create_dir() yet
+     * that assumes we'll deal with file names, while I want to avoid
+     * further string concatenation.  It also goes through a lot more
+     * hoops for unique and not yet created paths, while here we
+     * assume proper installer.
+     */
+
+
+    /* If not per user we use the SHARED directory which requires
+     * content validation.   FIXME: note that ASLR_SHARED_INHERIT may
+     * ask for opening two directories as trusted sources
+     * DYNAMORIO_VAR_CACHE_ROOT (\cache) in addition to the a per USER
+     * subdirectory \cache\SID
+     */
+    retval = get_parameter((per_user ? 
+                            L_IF_WIN(DYNAMORIO_VAR_CACHE_ROOT) :
+                            L_IF_WIN(DYNAMORIO_VAR_CACHE_SHARED)),
+                           base_directory, sizeof(base_directory));
+    if (IS_GET_PARAMETER_FAILURE(retval) || 
+        strchr(base_directory, DIRSEP) == NULL) {
+        SYSLOG_INTERNAL_ERROR(" %s not set!"
+                              " ASLR sharing ineffective.\n", 
+                              (per_user ? 
+                               DYNAMORIO_VAR_CACHE_ROOT :
+                               DYNAMORIO_VAR_CACHE_SHARED));
+        return INVALID_HANDLE_VALUE;
+    }
+    NULL_TERMINATE_BUFFER(base_directory);
+
+    LOG(GLOBAL, LOG_ALL, 1, "ASLR_SHARED: Opening file cache directory %s\n", 
+        base_directory);
+
+    if (per_user) {
+        /* for now we'll always create directory, since without
+         * ASLR_SHARED_INHERIT almost every process will need to
+         * create some non-exempt from sharing DLLs
+         */
+        bool res = os_current_user_directory(base_directory, 
+                                             BUFFER_SIZE_ELEMENTS(base_directory),
+                                             true /* create if missing */);
+        if (!res) {
+            /* directory may be set even on failure */
+            LOG(GLOBAL, LOG_CACHE, 2, "\terror creating per-user dir %s\n", base_directory);
+            return INVALID_HANDLE_VALUE;
+        }
+    }
+
+    /* now using potentially modified base_directory per-user */
+    snwprintf(wbuf, BUFFER_SIZE_ELEMENTS(wbuf), GLOBAL_NT_PREFIX L"%hs", 
+              base_directory);
+    NULL_TERMINATE_BUFFER(wbuf);
+
+    /* the shared directory is supposed to be created by nodemgr as
+     * world writable.  We should not create it if it doesn't exist
+     * with FILE_OPEN_IF (if we did it would inherit the permissions
+     * of the parent which are too restrictive).
+     */
+    directory_handle = create_file(wbuf, true /* is_dir */, 
+                                   READ_CONTROL /* generic rights */,
+                                   FILE_SHARE_READ
+                                   /* case 10255: allow persisted cache files
+                                    * in same directory */
+                                   | FILE_SHARE_WRITE,
+                                   FILE_OPEN, true);
+    if (directory_handle == INVALID_HANDLE_VALUE) {
+        SYSLOG_INTERNAL_ERROR("%s=%s is invalid!"
+                              " ASLR sharing is ineffective.\n",
+                              (per_user ? 
+                               DYNAMORIO_VAR_CACHE_ROOT :
+                               DYNAMORIO_VAR_CACHE_SHARED), base_directory);
+    } else {
+        /* note that now that we have the actual handle open, we can validate */
+        if (per_user && 
+            (DYNAMO_OPTION(validate_owner_dir) || DYNAMO_OPTION(validate_owner_file))) {
+            /* see os_current_user_directory() for details */
+            if (!os_validate_user_owned(directory_handle)) {
+                /* we could report in release, but it's unlikely that
+                 * it will get reported */
+                SYSLOG_INTERNAL_ERROR("%s -> %s is OWNED by an impostor!"
+                                      " ASLR sharing is disabled.",
+                                      (per_user ? 
+                                       DYNAMORIO_VAR_CACHE_ROOT :
+                                       DYNAMORIO_VAR_CACHE_SHARED),
+                                      base_directory);
+                close_handle(directory_handle);
+                directory_handle = INVALID_HANDLE_VALUE;
+            } else {
+                /* either FAT32 or we are the proper owner */
+
+                /* FIXME: case 10504 we have to verify that the final permissions and
+                 * sharing attributes for cache/ and for the current
+                 * directory, do NOT allow anyone to rename our directory
+                 * while in use, and replace it.  Otherwise we'd still
+                 * have to verify owner for each file as well with -validate_owner_file.
+                 */
+            }
+        }
+    }
+    return directory_handle;
+}
+
+/* note that this is currently done mostly as a hack, to allow fast
+ * first level checksum comparison just based on a file handle.
+ * Returns true if the files were the same size, or we have
+ * succesfully made them so.
+ */
+static bool
+aslr_module_force_size(IN HANDLE app_file_handle,
+                       IN HANDLE randomized_file_handle,
+                       const wchar_t *file_name,
+                       OUT uint64 *final_file_size)
+{
+    uint64 app_file_size;
+    uint64 randomized_file_size;
+    bool ok;
+    ok = os_get_file_size_by_handle(app_file_handle,
+                                    &app_file_size);
+    if (!ok) {
+        ASSERT_NOT_TESTED();
+        return false;
+    }
+
+    ok = os_get_file_size_by_handle(randomized_file_handle,
+                                    &randomized_file_size);
+    if (!ok) {
+        ASSERT_NOT_TESTED();
+        return false;
+    }
+
+    if (randomized_file_size != app_file_size) {
+        ASSERT(randomized_file_size < app_file_size);
+        SYSLOG_INTERNAL_WARNING("aslr_module_force_size: "
+                                "forcing %ls, padding %d bytes\n", file_name,
+                                (app_file_size - randomized_file_size));
+
+        /* note that Certificates Directory or debugging information
+         * are the usual sources of such not-loaded by
+         * NtMapViewOfSection memory.  Since we pass such file handle
+         * only to SEC_IMAGE NtCreateSection() calls, we don't need to
+         * call os_copy_file() to fill the missing data.  The
+         * SEC_COMMIT use by the loader in ntdll!LdrpCheckForLoadedDll
+         * will be given the original file.
+         */
+        ok = os_set_file_size(randomized_file_handle, app_file_size);
+        if (!ok) {
+            ASSERT_NOT_TESTED();
+            return false;
+        }
+
+        ok = os_get_file_size_by_handle(randomized_file_handle,
+                                        final_file_size);
+        if (!ok) {
+            ASSERT_NOT_TESTED();
+            return false;
+        }
+        ASSERT(*final_file_size == app_file_size);
+        if (*final_file_size != app_file_size) {
+            ASSERT_NOT_TESTED();
+            return false;
+        }
+        /* note we don't care whether we have had to force */
+    } else {
+        *final_file_size = randomized_file_size;
+    }
+
+    return true;
+}
+
+/* we expect produced_file_pointer to be a location where the file's
+ * signature can be written
+ */
+static bool
+aslr_module_append_signature(HANDLE produced_file,
+                             uint64* produced_file_pointer,
+                             aslr_persistent_digest_t *persistent_digest)
+{
+    bool ok;
+    size_t num_written;
+
+    persistent_digest->version = ASLR_PERSISTENT_CACHE_VERSION;
+    persistent_digest->magic = ASLR_PERSISTENT_CACHE_MAGIC;
+
+    /* Note we do not preclude having aslr_module_force_size()
+     * always force the size to be the final size |app size|+|aslr_persistent_digest_t|
+     * but unlikely we'd care to do this
+     */
+    DOLOG(1, LOG_SYSCALLS|LOG_VMAREAS, {
+        LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+            "ASLR: aslr_module_append_signature:");
+        LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+            "\n\t source.full :");
+        /* FIXME: should abstract out the md5sum style printing from
+         * get_md5_for_file() */
+        dump_buffer_as_bytes(GLOBAL, 
+                             persistent_digest->original_source.full_MD5, 
+                             MD5_RAW_BYTES, DUMP_RAW);
+        LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+            "\n\t source.short:");
+        dump_buffer_as_bytes(GLOBAL, 
+                             persistent_digest->original_source.short_MD5, 
+                             MD5_RAW_BYTES, DUMP_RAW);
+
+        LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+            "\n\t target.full :");
+        dump_buffer_as_bytes(GLOBAL, 
+                             persistent_digest->relocated_target.full_MD5, 
+                             MD5_RAW_BYTES, DUMP_RAW);
+        LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+            "\n\t target.short:");
+        dump_buffer_as_bytes(GLOBAL, 
+                             persistent_digest->relocated_target.short_MD5, 
+                             MD5_RAW_BYTES, DUMP_RAW);
+        LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, "\n");
+    });
+
+    ok = write_file(produced_file, 
+                    persistent_digest, sizeof(aslr_persistent_digest_t),
+                    produced_file_pointer,
+                    &num_written);
+    ASSERT(ok);
+    ASSERT(num_written == sizeof(aslr_persistent_digest_t));
+    ok = ok && (num_written == sizeof(aslr_persistent_digest_t));
+    return ok;
+}
+
+static bool
+aslr_module_read_signature(HANDLE randomized_file,
+                           uint64* randomized_file_pointer,
+                           OUT aslr_persistent_digest_t *persistent_digest)
+{
+    size_t num_read;
+    bool ok;
+    ok = read_file(randomized_file, 
+                   persistent_digest, sizeof(*persistent_digest),
+                   randomized_file_pointer,
+                   &num_read);
+    ASSERT(ok);
+
+    ok = ok && (num_read == sizeof(aslr_persistent_digest_t));
+    ASSERT(ok);
+
+    if (ok) {
+        ok = persistent_digest->version == ASLR_PERSISTENT_CACHE_VERSION;
+        ASSERT_CURIOSITY(ok && "invalid version");
+    }
+
+    if (ok) {
+        ok = persistent_digest->magic == ASLR_PERSISTENT_CACHE_MAGIC;
+        ASSERT_CURIOSITY(ok && "bad magic");
+    }
+
+    /* Where can we store any additional checksums and metadata:
+     * - [CURRENTLY] after the end of the file - just like certificates and
+     *   debugging information is in PEs, we could replace the
+     *   existing certificates, but we simply pad the file with 0 for those,
+     *   and add our signature after the end of the file.
+
+     * Alternatives:
+     * - in a PE field - good enough if using only a 32bit Checksum
+     * - NTFS streams - no, since we need to support FAT32
+
+     * - in a separate file or .chksum or for many files in a .cat
+     * most flexible though adds overhead.  Could throw in the
+     * registry, but we already have to secure the files so easier to
+     * use the same permissions.
+     *
+     *  <metadata>  <!- not really going be in xml -->
+     *    name=""
+     *    <original> 
+     *     <checksum md5|crc32|sha1= /> <-- staleness -->
+     *    <rebased> 
+     *     <checksum md5|crc32|sha1= /> <-- corruption -->
+     *  </metadata>
+     *  <hash>md5(metadata)</hash>
+     *
+     * - append to file name - content-based addressing
+     *   possible only for data based on original application file
+     */
+
+    /* FIXME: for unique name we can add the PE section Image.Checksum
+     * to generate different ID's.  Note we do not keep different
+     * possible mappings for the same name.  So we hope no two
+     * simultaneously needed files will clobber each other due to name
+     * collision.
+     *
+     * FIXME: yet we still need to verify any calculated checksum between our
+     * generated file and the file that it purportedly backs
+     * or better yet fully compare it
+     */
+
+    /* see reactos/0.2.9/lib/ntdll/ldr/utils.c for the original
+     * LdrpCheckImageChecksum, though we could produce our own crc32()
+     * checksum on original file as well and store it as checksum of
+     * our generated file in some PE orifice.
+     */
+
+    /* see pecoff v6 Appendix B, or pecoff v8 Appendix A: Calculating
+     * Authenticode PE Image Hash for reference
+     * where Checksum and Certificate Tables are excluded
+     */
+
+    return ok;
+}
+
+/* For our relocated version we should be validating a private section
+ * before publishing one.  Note that when calculating digest on
+ * original application section we have a section handle already that
+ * is assumed to be private.
+ */
+static
+bool
+aslr_get_section_digest(OUT module_digest_t *digest,
+                        HANDLE section_handle,
+                        bool short_digest_only)
+{
+    NTSTATUS res;
+    app_pc base = (app_pc)0x0; 
+    size_t commit_size = 0;       
+
+    SIZE_T view_size = 0;         
+    /* full file view, since even our short digest includes both
+     * header and footer */
+
+    uint type = 0;              /* commit not needed for original DLL */
+    uint prot = PAGE_READONLY; 
+
+    res = nt_map_view_of_section(section_handle, /* 0 */
+                                 NT_CURRENT_PROCESS, /* 1 */
+                                 &base, /* 2 */
+                                 0, /* 3 */
+                                 commit_size, /* 4 */
+                                 NULL, /* 5 */
+                                 &view_size, /* 6 */
+                                 ViewShare, /* 7 */
+                                 type, /* 8 */
+                                 prot); /* 9 */
+    ASSERT(NT_SUCCESS(res));
+    if (!NT_SUCCESS(res))
+        return false;
+    /* side note: windbg receives a ModLoad: for our temporary mapping
+     * at the NtMapViewOfSection(), no harm */
+
+    module_calculate_digest(digest,
+                            base,
+                            view_size,
+                            !short_digest_only, /* full */
+                            short_digest_only, /* short */
+                            DYNAMO_OPTION(aslr_short_digest),
+                            UINT_MAX/*all secs*/);
+    res = nt_unmap_view_of_section(NT_CURRENT_PROCESS, 
+                                   base);
+    ASSERT(NT_SUCCESS(res));
+    return true;
+}
+
+/* returns a private image section.
+ * a simple wrapper around nt_create_section() with common attributes
+ * on success callers need to close_handle() after use 
+ */
+static inline
+bool
+aslr_create_private_module_section(OUT HANDLE private_section,
+                                   HANDLE file_handle)
+{
+    NTSTATUS res;
+    res = nt_create_section(private_section,
+                            SECTION_ALL_ACCESS, /* FIXME: maybe less privileges needed */
+                            NULL, /* full file size */
+                            PAGE_EXECUTE, 
+                            /* PAGE_EXECUTE gives us COW in readers
+                             * but can't share any changes.
+                             * Unmodified pages are always shared.
+                             */
+                            /* PAGE_EXECUTE_READWRITE - gives us true
+                             * overwrite ability only in SEC_COMMIT */
+                            /* PAGE_EXECUTE_WRITECOPY is still COW,
+                             * though it also needs FILE_READ_DATA
+                             * privileges to at all create the section
+                             * which the loader doesn't use */
+                            SEC_IMAGE,
+                            file_handle,
+
+                            /* process private - no security needed */
+                            /* object name attributes */
+                            NULL /* unnamed */, 0, NULL, NULL);
+    ASSERT_CURIOSITY(NT_SUCCESS(res) && "create failed - maybe invalid PE");
+    /* seen STATUS_INVALID_IMAGE_FORMAT when testing non-aligned PE base */
+    if (!NT_SUCCESS(res)) 
+        return false;
+    return true;
+}
+
+static 
+bool
+aslr_get_file_digest(OUT module_digest_t *digest,
+                     HANDLE relocated_file_handle,
+                     bool short_only)
+{
+    /* Keep in mind that we have to create a private section mapping
+     * before we publish it for other consumers to use 
+     * in aslr_publish_section_handle
+     */
+
+    /* note we produce all of these on MEM_IMAGE versions so that the
+     * kernel doesn't have to page in both MEM_IMAGE and MEM_MAPPED
+     * copies, and the only cost of these is the extra virtual address
+     * remappings
+     */
+
+    /* see comments in aslr_get_original_metadata() about sharing some
+     * of the extraneous mappings 
+     */
+    HANDLE private_section;
+    bool ok;
+    
+    ok = aslr_create_private_module_section(&private_section,
+                                            relocated_file_handle);
+    if (!ok)
+        return false;
+
+    ok = aslr_get_section_digest(digest, private_section, 
+                                 short_only);
+                            
+    close_handle(private_section);
+    /* Note: we may need to keep this handle OPEN if that is to
+     * guarantee that the file cannot be overwritten.  Assuming that
+     * effect is already achieved by the flags we use to open the file
+     * and we will not close the file handle until finished.
+     */
+
+    return ok;
+}
+
+/* Caller must unmap mapping if original_mapped_base != NULL
+ * regardless of return value.
+ *
+ * Also see notes in aslr_generate_relocated_section() which this
+ * routine mostly borrows from.  Comparing in place avoids the
+ * CopyOnWrite faults and associated page copies.
+ */
+static bool
+aslr_compare_in_place(IN HANDLE original_section,
+                      OUT app_pc *original_mapped_base,
+                      OUT size_t *original_mapped_size,
+
+                      app_pc suspect_mapped_base,
+                      size_t suspect_mapped_size,
+                      app_pc suspect_preferred_base,
+                      size_t validation_prefix
+                      )
+{
+    bool ok;
+    NTSTATUS res;
+
+    HANDLE section_handle = original_section;
+    app_pc base = (app_pc)0x0; 
+    size_t commit_size = 0;       
+    SIZE_T view_size = 0;         /* full file view */
+    uint type = 0;              /* commit not needed for original DLL */
+    uint prot = PAGE_READWRITE; 
+    /* PAGE_READWRITE would allow us to update the backing section */
+    /* PAGE_WRITECOPY - will only provide the current mapping */
+
+    app_pc original_preferred_base;
+
+    ASSERT(*original_mapped_base == NULL);
+
+    res = nt_map_view_of_section(section_handle, /* 0 */
+                                 NT_CURRENT_PROCESS, /* 1 */
+                                 &base, /* 2 */
+                                 0, /* 3 */
+                                 commit_size, /* 4 */
+                                 NULL, /* 5 */
+                                 &view_size, /* 6 */
+                                 ViewShare, /* 7 */
+                                 type, /* 8 */
+                                 prot); /* 9 */
+    ASSERT_CURIOSITY(NT_SUCCESS(res));
+    if (!NT_SUCCESS(res)) {
+        *original_mapped_base = NULL;
+        return false;
+    }
+
+    *original_mapped_base = base;
+    *original_mapped_size = view_size;
+
+    /* Be aware of LdrVerifyImageMatchesChecksum() for our relocations
+     * - but that maps in as SEC_COMMIT based on the original file, so
+     * even if it is called for anything other than what is exported
+     * in KnownDlls we'd be ok.  If we want to match that checksum we
+     * can follow suit and process the file image, or we can emulate
+     * that on a mapped image Section.
+     *
+     * FIXME: check what is the meaning of
+     * "IMAGE_DLL_CHARACTERISTICS_FORCE_INTEGRITY 0x0080 Code
+     * Integrity checks are enforced", documented in PECOFF v8.0
+     */
+    original_preferred_base = get_module_preferred_base(base);
+    if (original_preferred_base == NULL) {
+        ASSERT_CURIOSITY(false && "base at 0, bad PE?");
+        /* maybe not a PE */
+        ASSERT_NOT_TESTED();
+        return false;
+    }
+
+
+    if (suspect_preferred_base == original_preferred_base) {
+        /* note we don't really care */
+        ASSERT_CURIOSITY(false && "old and new base the same!");
+        ASSERT_NOT_TESTED();
+        /* FIXME: we may want to force the new base in
+         * aslr_generate_relocated_section() to never be the same as
+         * original, but that may or may not be generally good,
+         * remember Enigma */
+    }
+
+    ok = (*original_mapped_size == suspect_mapped_size) &&
+        module_contents_compare(*original_mapped_base, 
+                                suspect_mapped_base, *original_mapped_size,
+                                false /* not relocated */,
+                                suspect_preferred_base - original_preferred_base,
+                                validation_prefix);
+    return ok;
+}
+
+/* paranoid mode check that a provided memory area is what it claims
+ * to be.  FIXME: note the relocated file should have such permissions
+ * that its contents cannot be overwritten after this point.
+ *
+ */
+static bool
+aslr_module_verify_relocated_contents(HANDLE original_file_handle,
+                                      HANDLE suspect_file_handle)
+{
+    /* In paranoid mode: should verify that the image is exactly the
+     * same as the original except for the relocations which should be
+     * exactly what we expect.
+     */
+
+    HANDLE original_file_section;
+    app_pc relocated_original_mapped_base = NULL;
+    size_t relocated_original_size = 0;
+
+    HANDLE suspect_file_section;
+    app_pc suspect_base = NULL; /* any base */
+    SIZE_T suspect_size = 0;  /* request full file view */
+    app_pc suspect_preferred_base;
+    bool ok;
+    NTSTATUS res;
+
+    size_t validation_prefix = (TEST(ASLR_PERSISTENT_PARANOID_PREFIX, 
+                                     DYNAMO_OPTION(aslr_validation)) ? 
+                                DYNAMO_OPTION(aslr_section_prefix) : POINTER_MAX);
+
+    /* create a private section for suspect  */
+    ok = aslr_create_private_module_section(&suspect_file_section,
+                                            suspect_file_handle);
+    if (!ok) {
+        return false;
+    }
+
+    /* map relocated suspect copy */
+    res = nt_map_view_of_section(suspect_file_section, /* 0 */
+                                 NT_CURRENT_PROCESS, /* 1 */
+                                 &suspect_base, /* 2 */
+                                 0, /* 3 */
+                                 0, /* 4 commit_size*/
+                                 NULL, /* 5 */
+                                 &suspect_size, /* 6 */
+                                 ViewShare, /* 7 */
+                                 0, /* 8 type */
+                                 PAGE_READWRITE); /* 9 prot */
+    /* FIXME: we are asking for PAGE_READWRITE on the whole file -
+     * affecting commit memory case 10251 */
+
+    /* we can close the handle as soon as we have a mapping */
+    close_handle(suspect_file_section);
+
+    ASSERT_CURIOSITY(NT_SUCCESS(res) && "map failed - maybe invalid PE");
+    if (!NT_SUCCESS(res)) {
+        ASSERT_NOT_TESTED();
+        return false;
+    }
+
+    /* FIXME: [minor perf] we should pass a handle to original section
+     * which is available to all publishers
+     */
+    ok = aslr_create_private_module_section(&original_file_section,
+                                            original_file_handle);
+    if (!ok) {
+        nt_unmap_view_of_section(NT_CURRENT_PROCESS, suspect_base);
+        return false;
+    }
+
+    /* Compare relocated files byte by byte
+     * 1.1) memcmp() relocated copy at the same base
+     *
+     * 1.1.1) [CURRENTLY] apply relocations to original DLL as if
+     * going to the relocated DLL location
+     *    memcmp(original privately relocated, relocated DLL)
+     *    Note that our changes to a mapping of the original are only
+     *    temporary (even if we did reuse the application section).
+     * 1.1.2) alternatively we could transform the relocated section
+     *    back into the original.  We would also have to be extra
+     *    careful when processing a potentially rogue PE.
+     *
+     * Note that the MD5 sum of the relocated DLL may be more expensive
+     * than comparing the DLLs adjusting for relocations.
+     * Note we can't trust MD5s saved in the file.
+     * 1.2) (MD5(relocated DLL) == MD5(original privately relocated)
+     *
+     * FIXME: [perf] Need to do some perf testing to see 1.1 is good enough -
+     * note we will have to check this only once for publisher, not consumer
+     *
+     * 2.1) relocation at a time we'd save the extra private copy of
+     * the pages we need to touch if we do this smarter. We need to
+     * provide a compare function that for each section traverses
+     * relocations to do the proper match and compares verbatim all
+     * bytes between relocations.
+     */
+
+    suspect_preferred_base = get_module_preferred_base(suspect_base);
+    ASSERT_CURIOSITY(suspect_preferred_base != NULL && "bad PE file");
+    DODEBUG({
+        if (suspect_preferred_base != suspect_base) {
+            /* this is the earliest we know this DLL won't fit in this process */
+            SYSLOG_INTERNAL_WARNING("DLL mapping is not shareable");
+            /* of course we may have a conflict, and so DLL won't really be
+             * shared if not loaded at preferred base
+             */
+        }
+    });
+    
+   if (TEST(ASLR_PERSISTENT_PARANOID_TRANSFORM_EXPLICITLY, 
+             DYNAMO_OPTION(aslr_validation))) {
+        KSTART(aslr_validate_relocate);
+        /* note we're transforming our good section into the relocated one
+         * including any header modifications
+         */
+        ok = (suspect_preferred_base != NULL) &&
+            aslr_generate_relocated_section(original_file_section,
+                                            &suspect_preferred_base,
+                                            false,
+                                            &relocated_original_mapped_base,
+                                            &relocated_original_size,
+                                            NULL /* no digest */);
+        KSTOP(aslr_validate_relocate);
+        if (!ok) {
+            ASSERT(relocated_original_mapped_base == NULL);
+        } else {
+            ASSERT(relocated_original_mapped_base != NULL);
+        }
+
+        ASSERT_CURIOSITY(ok && "invalid source file!");
+
+        if (ok) {
+            KSTART(aslr_compare);
+            ok = (relocated_original_size == suspect_size) &&
+                module_contents_compare(relocated_original_mapped_base, 
+                                        suspect_base, relocated_original_size,
+                                        true /* already relocated */,
+                                        0,
+                                        validation_prefix);
+            KSTOP(aslr_compare);
+        }
+    } else {
+        /* we must do the comparison in place */
+        KSTART(aslr_compare);
+        ok = aslr_compare_in_place(original_file_section,
+                                   &relocated_original_mapped_base,
+                                   &relocated_original_size,
+                                   suspect_base,
+                                   suspect_size,
+                                   suspect_preferred_base,
+                                   validation_prefix);
+        KSTOP(aslr_compare);
+        /* note we don't keep track whether failed due to bad original
+         * file or due to mismatch with suspect file
+         */
+    }
+
+    ASSERT_CURIOSITY((relocated_original_size == suspect_size) && "mismatched PE size");
+    ASSERT_CURIOSITY(ok && "mismatched relocated file!");
+    /* on failure here inspect with
+     * 0:000> c poi(relocated_original_mapped_base) L13000 poi(suspect_base) */
+    if (!ok) {
+        SYSLOG_INTERNAL_ERROR("ASLR_SHARED: stale, corrupt or rogue file!");
+    }
+
+    if (relocated_original_mapped_base != NULL) {
+        res = nt_unmap_view_of_section(NT_CURRENT_PROCESS, relocated_original_mapped_base);
+        ASSERT(NT_SUCCESS(res));
+    }
+
+    /* FIXME: [perf] we unmap everything that was paged in (presumably
+     * the whole file), yet we expect that the system cache will keep
+     * the file views until the app maps it again.  Alternatively, we
+     * could preserve this mapping and present it to the application
+     * whenever it calls NtMapViewOfSection().  Note that then our
+     * working set will visibly include these pages, so letting the
+     * system cache keep these for us may be better (in addition to easier).
+     */
+    if (suspect_base != NULL) {
+        res = nt_unmap_view_of_section(NT_CURRENT_PROCESS, suspect_base);
+        ASSERT(NT_SUCCESS(res));
+    }
+
+    close_handle(original_file_section);
+
+    return ok;
+}
+
+/* verify whether a randomized_file_handle is a valid backer of the
+ * application file that's given to us.  Needed only for persistence.
+ * FIXME: Otherwise, for sharing only have to figure out how to
+ * safely create a file exclusively and allow only the producer to
+ * become a publisher.
+ */
+static bool
+aslr_verify_file_checksum(IN HANDLE app_file_handle,
+                          IN HANDLE randomized_file_handle)
+{
+    /* we do some basic sanity checking - is the
+     * FileStandardInformation.EndOfFile the same for both the
+     * original and the rebased file.  Otherwise disk file may be
+     * corrupt.  Should preferably verify its new checksum as well if
+     * requested.  For best consistency and security check a byte by
+     * byte comparison is performed.
+     *
+     * case 8492 covers discussion on alternative checks to allow for
+     * patching/tampering detection
+     */
+    uint64 app_file_size;
+    uint64 randomized_file_size;
+    uint64 adjusted_file_size;
+    aslr_persistent_digest_t persistent_digest;
+
+    module_digest_t calculated_digest;
+    bool short_only = 
+        TEST(ASLR_PERSISTENT_SHORT_DIGESTS, DYNAMO_OPTION(aslr_validation));
+
+    bool ok;
+    ok = os_get_file_size_by_handle(app_file_handle,
+                                    &app_file_size);
+    if (!ok) {
+        ASSERT_NOT_TESTED();
+        return false;
+    }
+
+    ok = os_get_file_size_by_handle(randomized_file_handle,
+                                    &randomized_file_size);
+    if (!ok) {
+        ASSERT_NOT_TESTED();
+        return false;
+    }
+
+    /* minimum check is whether original_file_size matches, so that we
+     * know we're not dealing with a corrupt randomized file Note
+     * aslr_create_relocated_dll_file() shouldn't provide an
+     * allocation hint in case that would immediately set the file
+     * size.
+     */
+
+    adjusted_file_size = 
+        randomized_file_size - sizeof(aslr_persistent_digest_t);
+
+    /*
+     * Note that this scheme will not work if some other software (AV,
+     * backup) adds any data after the end of our own files in the
+     * DATA stream.
+     *
+     * FIXME: could make these size checks optional but there will be
+     * very few things we can really do without our own signature
+     * record.
+     *
+     * Note, somewhat not orthogonal, we in fact ignore any changes
+     * that are not a part of the file, so we will not complain about
+     * modifications of the original application, but we will complain
+     * about appends.
+     */
+
+    if (adjusted_file_size != app_file_size) {
+        SYSLOG_INTERNAL_WARNING("aslr_verify_file_checksum: "
+                                "wrong size - stale or possibly corrupt file\n");
+        /* note that as a publisher we should not proactively attempt
+         * deleting the file, but we'll ask producer to attempt to
+         * produce again.  Hopefully this time it will be produced
+         * properly.  If we have a bug in the field with some DLL the
+         * exemptions by name should allow us to skip trying over and
+         * over.
+         */
+        return false;
+    }
+
+    /* at least we know that our signature version and magic match */
+    /* always reading signature even if we won't need the fields */
+    ok = aslr_module_read_signature(randomized_file_handle,
+                                    /* expected pointer to signature */
+                                    &adjusted_file_size,
+                                    &persistent_digest);
+    if (!ok) {
+        return false;
+    }
+
+    /* FIXME: in order to not break the abstraction here we'll need to
+     * use another private nt_create_section(),
+     * nt_map_view_of_section() before officially publishing
+     * Measure for performance problems and may streamline.
+     */
+
+    if (TEST(ASLR_PERSISTENT_MODIFIED_TIME,
+             DYNAMO_OPTION(aslr_validation))) {
+        /* FIXME: currently impossible to check application times */
+        ASSERT_NOT_IMPLEMENTED(false);
+        if (!ok) {
+            SYSLOG_INTERNAL_WARNING("aslr_verify_file_checksum: "
+                                    "modified time differs - stale file!\n");
+            return false;
+        }
+    }
+    
+    if (TEST(ASLR_PERSISTENT_PARANOID,
+             DYNAMO_OPTION(aslr_validation))) {
+        ok = aslr_module_verify_relocated_contents(app_file_handle,
+                                                   randomized_file_handle);
+        if (!ok) {
+            SYSLOG_INTERNAL_WARNING("aslr_verify_file_checksum: "
+                                    "paranoid check failed - stale, corrupt, or rogue file!\n");
+            /* FIXME: do we want to report to the authorities?  Maybe
+             * only for rogues, then caller needs to verify in other
+             * ways.  To make sure file wasn't truncated due to power
+             * failure - corrupt; and doublecheck with MD5 of original
+             * file to see if it is not just a stale file.
+             */
+            return false;
+        }
+    }
+
+    if (TEST(ASLR_PERSISTENT_SOURCE_DIGEST,
+             DYNAMO_OPTION(aslr_validation))) {
+        /* FIXME: note that we should pass the original section to
+         * aslr_publish_section_handle() and use
+         * aslr_get_section_digest() instead of a private mapping
+         */
+
+        /* FIXME: the original file may have been opened in
+         * non-exclusive mode but that is very unlikely, so we'll
+         * assume our section can be used without a race
+         */
+        ok = aslr_get_file_digest(&calculated_digest, 
+                                  app_file_handle, short_only);
+        if (!ok) {
+            ASSERT_NOT_TESTED();
+            return false;
+        }
+            
+        if (!module_digests_equal(&persistent_digest.original_source,
+                                  &calculated_digest, short_only, !short_only)) {
+            SYSLOG_INTERNAL_WARNING("aslr_verify_file_checksum: "
+                                    "invalid source checksum - stale!\n");
+            return false;
+        }
+    }
+
+    if (TEST(ASLR_PERSISTENT_TARGET_DIGEST,
+             DYNAMO_OPTION(aslr_validation))) {
+        /* FIXME: note that this routine should not be completely
+         * trusted, if we're trying to prevent a high privileged
+         * process from crashing on a bad DLL for extra safety we
+         * may need to wrap this call in a try/except block.
+         */
+        ok = aslr_get_file_digest(&calculated_digest, 
+                                  randomized_file_handle, short_only);
+        if (!ok) {
+            ASSERT_NOT_TESTED();
+            return false;
+        }
+            
+        if (!module_digests_equal(&persistent_digest.relocated_target,
+                                  &calculated_digest, short_only, !short_only)) {
+            SYSLOG_INTERNAL_ERROR("aslr_verify_file_checksum: "
+                                  "invalid target checksum - corrupt!\n");
+            return false;
+        }
+    }
+
+    if (!TESTANY(ASLR_PERSISTENT_PARANOID|
+                 ASLR_PERSISTENT_SOURCE_DIGEST|
+                 ASLR_PERSISTENT_TARGET_DIGEST,
+                 DYNAMO_OPTION(aslr_validation))) {
+        SYSLOG_INTERNAL_WARNING_ONCE("aslr_verify_file_checksum: no checksum\n");
+    }
+
+    return true;
+}
+
+/* used by section publishers for providing our alternative file
+ * backing in current user DLL file cache, 
+ * 
+ * returns true if module_name exists and is not stale.  Caller should
+ * close file on success.  
+ */
+static bool
+aslr_open_relocated_dll_file(OUT HANDLE *relocated_file, 
+                             IN HANDLE original_file,
+                             const wchar_t *module_name)
+{
+    HANDLE relocated_dlls_directory = 
+        get_relocated_dlls_filecache_directory(false);
+    NTSTATUS res;
+    HANDLE new_file = INVALID_HANDLE_VALUE;
+
+    if (relocated_dlls_directory == INVALID_HANDLE_VALUE) {
+        return false;
+    }
+
+    /* FIXME: case 8494: staleness trigger
+     * we may want to check for stale files - e.g. if not
+     * asking for ASLR_PERSISTENT we want only freshly produced and
+     * still open by publisher file.  In that case may want to try
+     * exclusive access first - and if we can get it then the file is
+     * not freshly produced.  
+     * 
+     * Alternatively, even if we allow persistence we may use the file
+     * creation time to decide that a file has been created too long
+     * ago (aslr_module_get_times()), or that is has been used too
+     * many times (e.g. brute forcing a process).  Although such
+     * measures improve resistance to brute force attack only as much
+     * as one bit of randomness.
+     * 
+     * If we do want to refuse loading a file on brute forcing, best
+     * recourse is to switch to private ASLR.  If the file is too old,
+     * but borderline old, such that some processes are still using
+     * the old version, and for some reason we do not allow a new copy
+     * to to supersede the existing produced file or published section
+     * we should just wait until next reboot and use private ASLR in
+     * the mean time as well.  Could also use alternative bases.
+     * 
+     * Probably, best is to not bother with the above directly, but
+     * instead use a reaper process (nodemgr.exe) which will regularly
+     * schedule for removal files in FIFO, both for capacity
+     * management and for staleness.
+     */
+    /* case 8494 FILE_SHARE_DELETE doesn't allow deletion of a memory
+     * mapped file, so aside from the tight interval before we
+     * nt_create_section() the file cannot be deleted.  Setting that
+     * flag now would also allow succesfully opening a file, if it has
+     * been already marked for deletion, but we don't expect any such
+     * in common use.
+     */
+    res = nt_create_module_file(&new_file, module_name,
+                                relocated_dlls_directory,
+                                (DYNAMO_OPTION(validate_owner_file) ? READ_CONTROL : 0) |
+                                FILE_EXECUTE | FILE_READ_DATA,
+                                FILE_ATTRIBUTE_NORMAL,
+                                FILE_SHARE_READ,
+                                FILE_OPEN, 0);
+    if (NT_SUCCESS(res)) {
+        if (DYNAMO_OPTION(validate_owner_file)) {
+            if (!os_validate_user_owned(new_file)) {
+                SYSLOG_INTERNAL_ERROR_ONCE("%S not owned by current process!"
+                                           " ASLR cache may be compromised, not using.",
+                                           module_name);
+                close_handle(new_file);
+                return false;
+            }
+        }
+
+        if (!aslr_verify_file_checksum(original_file, new_file)) {
+            close_handle(new_file);
+            return false;
+        }
+
+        *relocated_file = new_file;
+        return true;
+    } else {
+        LOG(GLOBAL, LOG_ALL, 2, "aslr_open_relocated_dll_file: res %x\n",res);
+        if (res == STATUS_OBJECT_NAME_NOT_FOUND) {
+            LOG(GLOBAL, LOG_ALL, 1, "aslr_open_relocated_dll_file: DLL not found\n");
+        } else {
+            if (res == STATUS_ACCESS_DENIED) {
+                ASSERT_CURIOSITY(false && 
+                                 "insufficient permissions, or non-executable file");
+            } else if (res == STATUS_SHARING_VIOLATION) {
+                ASSERT_NOT_TESTED();
+                /* currently this may happen in a race with producer */
+                ASSERT_CURIOSITY(false && "a producer has exclusive read lock");
+
+            } else if (res == STATUS_DELETE_PENDING) {
+                ASSERT_CURIOSITY(false && "some process has a handle to a deleted file");
+                /* Maybe you're debugging and need to close windbg.
+                 * Otherwise use procexp to find who owns the handle.
+                 */
+                /* very mysteriously windbg was holding a handle to a DLL that was consecutively
+                 * rm'ed from cygwin which wasn't truly enough to allow
+                 * $ ls -l
+                 *   ls: dadkeyb.dll-12628e13: No such file or directory
+                 *   total 104147
+                 *   -rwxr-xr-x  1 vlk None   163903 May  3 21:09 dll.dll.dll-885d0011
+                 * SIC! although listing the whole directory ls was complaining about the file.
+                 *
+                 */
+
+                /* see SDK comments on what should DeleteFile() which
+                 * uses NtSetInformationFile If we similarly delete the
+                 * files, until all consumers are done with the file,
+                 * new consumers may be allowed to map the exported
+                 * section, new publishers should fail, while producers
+                 * are supposed to request to supersede the file.
+                 * There is only a tiny window in which we can delete file
+                 * - before we create a section based on it.
+                 */
+            } else if (res == STATUS_FILE_IS_A_DIRECTORY) {
+                /* 0xc00000ba - The file that was specified as a target is a
+                 * directory and the caller specified that it could be anything
+                 * but a directory.
+                 */
+                /* test example: mkdir unique_name */
+                ASSERT_CURIOSITY(false && "a directory is in the way");
+                /* FIXME: we should expect nodemgr to clean all files and directories */
+            } else {
+                ASSERT(false);
+            }
+
+            /* maybe insufficient privileges */
+        }
+        return false;
+    }
+}
+
+/* returns false if we are too low on disk to create a file of desired size */
+bool
+aslr_check_low_disk_threshold(uint64 new_file_size)
+{
+    /* see case 8494 on other capacity triggers: low disk threshold,
+     * should share heuristic with nodemgr though that won't depend on
+     * request size, but on current total use, as well as available
+     * disk space.  Note that we do not track current use but instead
+     * leave nodemgr to calculate that itself (if not done too
+     * frequently).  We may want to have nodemgr run any cache
+     * trimming only in case the total available disk space is below
+     * another threshold.
+     */
+    bool ok;
+    HANDLE producer_directory = 
+        get_relocated_dlls_filecache_directory(true);
+
+    ok = check_low_disk_threshold(producer_directory, new_file_size);
+
+    /* FIXME: should we memoize the value on failure so that we don't
+     * bother even with a syscall in the future?  We'll then ignore
+     * the potential for someone freeing up disk space. */
+    /* Note that this should be present as a quick check.  If we are
+     * the only writer to the volume then we'll keep checking for each
+     * file until we get really close to the cache size.  If there are
+     * other producers this check is racy so we may easily end up
+     * beyond the desired minimum.  The same of course will happen if
+     * any other application is writing to the disk.
+     */
+
+    return ok;
+}
+
+/* used by file producers for providing our alternative file backing */
+/* callers should close the handle */
+static bool
+aslr_create_relocated_dll_file(OUT HANDLE *new_file, 
+                               const wchar_t *unique_name, 
+                               uint64 original_file_size,
+                               bool persistent 
+                               /* hint whether file is persistent */)
+{
+    HANDLE our_relocated_dlls_directory = 
+        get_relocated_dlls_filecache_directory(true);
+    uint attributes;
+    size_t allocation_size = 0x0;
+    NTSTATUS res;
+    bool retry;
+    uint attempts = 0;
+
+    /* note that we could pass the original_file_size as allocation
+     * size, but it's not obvious what does that really mean on
+     * compressed volumes so it's better to not optimze.  While we
+     * could get an immediate feedback if we're out of disk space, we
+     * have to handle it anyways when we write to the file as
+     * well. Note that aslr_verify_file_checksum() can now assume a
+     * file is not truncated.
+     */
+
+    /* FIXME: should append a suffix to the file image */
+
+    if (!persistent) {
+        /* FIXME: if we're creating the file only for sharing but not for
+         * persistence we should compare performance compared to FILE_ATTRIBUTE_NORMAL */
+        /* FIXME: maybe should add FILE_FLAG_DELETE_ON_CLOSE? */
+
+        /* FIXME: we may want to always use FILE_ATTRIBUTE_TEMPORARY
+         * even if persistent to push disk write to better times, but
+         * if never written before power off may be bad for persistence. */
+        attributes = FILE_ATTRIBUTE_TEMPORARY;
+    } else {
+        attributes = FILE_ATTRIBUTE_NORMAL;
+        ASSERT_NOT_TESTED();
+    }
+
+    /* FIXME: in a race between producers the exclusive access should
+     * prevent them from overwriting the same file.  using
+     * FILE_SUPERSEDE for the case of a file has been created already
+     * we wouldn't get this far unless it was deemed corrupt!  Note
+     * that there is a tiny window in which serialized producers will
+     * supersede each other's copies, which wouldn't happen with
+     * FILE_CREATE but we cannot return success in that case either.
+     * also cf. FILE_OVERWRITE_IF */
+
+    /* to prevent malicious hard link (or symbolic link introduced in
+     * Longhorn) we need to make sure we overwrite the link itself not
+     * its target.  FILE_SUPERSEDE unfortunately overwrites the target
+     * FIXME: there is a race so we should return failure and not
+     * attempt to produce a file if after we have deleted a file it is
+     * still present.
+     */
+    do {
+        retry = false;
+        attempts++;
+
+        res = nt_create_module_file(new_file, unique_name, 
+                                    our_relocated_dlls_directory,
+                                    READ_CONTROL |
+                                    FILE_READ_DATA | FILE_WRITE_DATA,
+                                    attributes,
+                                    0,
+                                    /* exclusive read/write access */
+                                    FILE_CREATE /* create only if non-existing */
+                                    /* case 10884: really needed only for validate_owner_file */
+                                    | FILE_DISPOSITION_SET_OWNER,
+                                    allocation_size);
+        /* FIXME: adding FILE_SHARE_DELETE would allow us to supersede
+         * a file that has been marked for deletion while in use.
+         * However that normally isn't useful since we map sections
+         * from these files which precludes deletion
+         * (STATUS_CANNOT_DELETE) .  Rogue users getting in our way
+         * can always just open exclusively.
+         */
+        
+        if (NT_SUCCESS(res)) {
+            ASSERT_CURIOSITY(os_validate_user_owned(*new_file) && 
+                             "DLL loaded while impersonating?");
+            return true;
+        }
+
+        /* note that name collision error should be returned before
+         * any other reason for failure is found, e.g. non-executable
+         * etc. so we can attempt to delete the file
+         */
+        if (res == STATUS_OBJECT_NAME_COLLISION) {
+            bool deleted;
+
+            ASSERT_CURIOSITY(attempts == 1 && 
+                             "ln attack, in use, or race with another producer");
+            /* the file could legally be in use only if valid for a
+             * different core version */
+
+            deleted = os_delete_file(unique_name,
+                                     our_relocated_dlls_directory);
+            if (deleted) {
+                SYSLOG_INTERNAL_WARNING("deleted (invalid) file %ls in the way",
+                                        unique_name);
+                retry = true;
+
+                /* note that even if we have marked for deletion the file
+                 * will really disappear only when the last user is done
+                 * with it
+                 */
+
+                /* Note deleted file creation time may be preserved : see
+                 * MSDN on CreateFile "If you rename or delete a file and
+                 * then restore it shortly afterward, the system searches
+                 * the cache for file information to restore. Cached
+                 * information includes its short/long name pair and
+                 * creation time."
+                 */
+            } else {
+                ASSERT_CURIOSITY(deleted && "can't delete: maybe directory");
+            }
+        }
+        /* normally attempt 1 should succeed,
+         * if corrupt and succesfully deleted should succeed on attempt 2
+         * only in some odd race one would try for a third time
+         */
+    } while (retry && attempts <= 3);
+
+    return false;
+}
+
+/* returns true if file name is on exempt from ASLR list */
+/* FIXME: merge or keep in synch with the checks based on a PE mapping
+ * in aslr_post_process_mapview()
+ */
+static bool
+is_aslr_exempted_file_name(const wchar_t *short_file_name) 
+{
+    if (!IS_STRING_OPTION_EMPTY(exempt_aslr_default_list) ||
+        !IS_STRING_OPTION_EMPTY(exempt_aslr_list) ||
+        !IS_STRING_OPTION_EMPTY(exempt_aslr_extra_list) ||
+        DYNAMO_OPTION(aslr_cache_list) != ASLR_CACHE_LIST_DEFAULT
+        ) {
+        char file_name[MAXIMUM_PATH];
+        /* need to convert since exemption lists work on char strings */
+        /* name may also come directly from section name which for the
+         * KnownDlls will have to match
+         */
+
+        /* -exempt_aslr_list '*' is really only interesting as a
+         * stress test option, otherwise should just turn off ASLR_DLL
+         */
+        if (IS_LISTSTRING_OPTION_FORALL(exempt_aslr_list))
+            return true;
+
+        wchar_to_char(file_name, BUFFER_SIZE_ELEMENTS(file_name), 
+                      short_file_name, 
+                      wcslen(short_file_name) * sizeof(wchar_t) /* size in bytes */);
+        NULL_TERMINATE_BUFFER(file_name);
+
+        /* note that almost all exempted DLLs are KnownDlls, and
+         * kbdus.dll is ok in -hotp_only, yet to be prepared we still
+         * get the name.  Note that we use a FILE name, not a PE name here.
+         */
+
+        /* we're using the same exemption list as private ASLR, though we
+         * may separate these 
+         */
+        if (check_list_default_and_append(dynamo_options.exempt_aslr_default_list,
+                                          dynamo_options.exempt_aslr_list,
+                                          file_name)) {
+            SYSLOG_INTERNAL_WARNING("ASLR exempted from sharing DLL %s",
+                                    file_name);
+            return true;
+        }
+
+        /* FIXME: in fact we may want to share only these 'extra'
+         * exempted from private ASLR DLLs only due to memory concerns */
+        if (DYNAMO_OPTION(aslr_extra) &&
+            check_list_default_and_append("", /* no default list */
+                                          dynamo_options.exempt_aslr_extra_list,
+                                          file_name)) {
+            ASSERT_NOT_TESTED();
+            SYSLOG_INTERNAL_WARNING("ASLR exempted extra DLL %s",
+                                    file_name);
+            return true;
+        }
+
+        if (DYNAMO_OPTION(aslr_cache_list) == ASLR_CACHE_LIST_INCLUDE &&
+            /* using include list, exempt if NOT on list */
+            !check_list_default_and_append("", /* no default list */
+                                           dynamo_options.aslr_cache_include_list,
+                                           file_name)) {
+            SYSLOG_INTERNAL_WARNING("ASLR exempted DLL %s not on include list",
+                                    file_name);
+            return true;
+        }
+        if (DYNAMO_OPTION(aslr_cache_list) == ASLR_CACHE_LIST_EXCLUDE &&
+            /* using exclude list, exempt if on list */
+            check_list_default_and_append("", /* no default list */
+                                          dynamo_options.aslr_cache_exclude_list,
+                                          file_name)) {
+            SYSLOG_INTERNAL_WARNING("ASLR exempted DLL %s on exclude list",
+                                    file_name);
+            return true;
+        }
+    }
+    return false;
+}
+
+/* Returns a pointer to the short name within the long name that is copied
+ * into name_info
+ */
+const wchar_t *
+get_file_short_name(IN HANDLE file_handle, IN OUT FILE_NAME_INFORMATION *name_info)
+{
+    NTSTATUS res;
+    /* note FileName is not NULL-terminated */
+    res = nt_query_file_info(file_handle, name_info,
+                             sizeof(*name_info), FileNameInformation);
+    if (!NT_SUCCESS(res))
+        return NULL;
+
+    /* now have to properly NULL terminate the wide string we got */
+    /* ok to overwrite the last character  */
+    NULL_TERMINATE_BUFFER(name_info->FileName);
+    if ((name_info->FileNameLength - sizeof(wchar_t)) <= sizeof(name_info->FileName)) {
+        /* Length is supposed to be in bytes */
+        name_info->FileName[name_info->FileNameLength/sizeof(wchar_t)] = 0;
+    }
+
+    /* very unlikely that we'd get a relative name, then we'll get full name */
+    return w_get_short_name(name_info->FileName);
+}
+
+/* returns true if a likely unique generated_name name was succesfully
+ * produced.  Note name collisions are possible, so callers need to
+ * ensure sections correspond to the same file by other means.
+ *
+ * produced name is guaranteed to have no backslashes,
+ */
+static bool
+calculate_publish_name(wchar_t *generated_name /* OUT */, 
+                       uint max_name_length /* in elements */,
+                       HANDLE file_handle, HANDLE section_handle)
+{
+    /* FIXME: if we are post-processing a succesful app
+     * NtCreateSection/NtOpenSection can also map it and calculate any
+     * other interesting properties.  Note we won't have the FILE for
+     * KnownDlls!  We need to somehow inherit from the original
+     * section, or we'd need to guess the file name.  Or, otherwise
+     * recreate it in the way we need it based on KnownDllPath.  If we
+     * do this based on the section not file can handle both the same
+     * way, otherwise need to assume that KnownDlls are the only ones
+     * opened via OpenSection, and we'd open them appropriately and
+     * maybe use are_mapped_files_the_same() to doublecheck, or use
+     * NtQueryObject ObjectNameInformation to verify the directory
+     * used is the expected KnownDlls.
+     */
+
+    /* We're using only file attributes here:
+     * - FileNameInformation - path name.  See DDK for the odd case in
+     * which a relative path will be returned, if a user has
+     * SeChangeNotifyPrivilege ZwQueryInformationFile returns the full
+     * path in all cases. Note usually all process do have
+     * SeChangeNotifyPrivilege so we can always expect a full path.
+     * Since we will only use it for a hash, even that is OK as long
+     * as all users get it the same way.
+     * 
+     * - FileStandardInformation - EndOfFile as a byte offset 
+     *
+     * Other more restrained sources we could have used -
+     * file: FileBasicInformation - can use create/access/times, however
+     * requires FILE_READ_ATTRIBUTES; FileInternalInformation - gives
+     * us a unique file ID but is valid only on NTFS
+     * 
+     * Section: SectionBasicInformation.Size or
+     * SectionImageInformation.EntryPoint can also be used for an
+     * image if we had Query access but we don't have that on
+     * KnownDlls until we map them in.
+     * 
+     * Mapped PE: PE name, timestamp, checksum - none is reliable
+     * enough by itself. We could use add a hash of the PE header
+     * (like FX!32) or the short MD5 module_digest_t.  Yet for most
+     * practical cases files with same name but different contents
+     * will not have simulatenous lifetimes.
+     *
+     * One could use consider only content-based naming and use a
+     * hash of the mapping (maybe short) and ignore names - then any
+     * files with the same content will use the same copy.  Main
+     * benefit that it allows multiple paths to the same files to
+     * match.  Not so useful if we use a publisher cache that
+     * consumers can quickly lookup with a faster name hash and file
+     * handle comparison.
+     */
+    NTSTATUS res;
+    const wchar_t *short_name;
+    uint name_hash;
+    uint final_hash;
+
+    FILE_STANDARD_INFORMATION standard_info;
+    FILE_NAME_INFORMATION name_info; /* note: large struct */
+    /* note FileName is not NULL-terminated */
+
+    res = nt_query_file_info(file_handle,
+                             &standard_info,
+                             sizeof(standard_info),
+                             FileStandardInformation);
+    if (!NT_SUCCESS(res)) {
+        /* should always be able to get this */
+        ASSERT(false && "bad handle?");
+        return false;           /* can't generate name */
+    }
+    
+    short_name = get_file_short_name(file_handle, &name_info);
+    if (short_name == NULL) {
+        ASSERT(false);
+        return false;           /* can't generate name */
+    }
+
+    /* name hash over the wide char as bytes (many will be 0's but OK) */
+    name_hash = crc32((char*)name_info.FileName, 
+                      name_info.FileNameLength);
+
+    /* xor over the file size as bytes */
+    final_hash = name_hash ^ 
+        (standard_info.EndOfFile.LowPart ^ standard_info.EndOfFile.HighPart);
+
+    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 2,
+        "ASLR: calculate_publish_name: short name %ls\n"
+        "  full file name '%ls', file size %d\n"
+        "  name_hash "PFX", final_hash "PFX"\n",
+        short_name,
+        name_info.FileName,
+        standard_info.EndOfFile.LowPart,
+        name_hash, final_hash);
+
+    if (is_aslr_exempted_file_name(short_name)) {
+        return false; /* exempted, shouldn't publish */
+    }
+
+    _snwprintf(generated_name, max_name_length, 
+               L"%s-"L_PFMT, short_name, final_hash);
+
+    if (TEST(ASLR_INTERNAL_SHARED_NONUNIQUE, 
+             INTERNAL_OPTION(aslr_internal))) {
+        /* stress testing: temporarily testing multiple file sections by unique
+         * within process name */
+        static int unique = 0;
+        _snwprintf(generated_name, max_name_length, 
+                   L"unique-7ababcd-%d", unique++);
+    }
+
+    generated_name[max_name_length-1] = 0;
+    ASSERT(w_get_short_name(generated_name) == generated_name
+           && generated_name[0] != L_EXPAND_LEVEL(DIRSEP));
+    return true; /* name should be usable */
+}
+
+/* assumes mapped_module_base's header page is writable */
+static 
+bool
+aslr_write_header(app_pc mapped_module_base, size_t module_size, 
+                  app_pc new_preferred_base,
+                  uint new_checksum, uint new_timestamp)
+{
+    IMAGE_DOS_HEADER *dos;
+    IMAGE_NT_HEADERS *nt_hdr;
+
+    ASSERT(is_readable_pe_base(mapped_module_base));
+    ASSERT_CURIOSITY(new_preferred_base != mapped_module_base && 
+           "usually relocated at original address");
+    /* note that mapped_module_base is not necessarily the original
+     * preferred image base for DLLs with poorly chosen base */
+
+    dos = (IMAGE_DOS_HEADER *) mapped_module_base;
+    nt_hdr = (IMAGE_NT_HEADERS *) (((byte *)dos) + dos->e_lfanew);
+
+    /* from pecoff_v8.doc CheckSum The image file checksum. The
+     * algorithm for computing the checksum is incorporated into
+     * IMAGHELP.DLL. The following are checked for validation at load
+     * time: all drivers, any DLL loaded at boot time, and any DLL
+     * that is loaded into a critical Windows process.
+     */
+
+    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+        "ASLR: aslr_write_header checksum old "PFX", new "PFX"\n",
+        nt_hdr->OptionalHeader.CheckSum, new_checksum);
+    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+        "ASLR: aslr_write_header ImageBase old "PFX", new "PFX"\n",
+        OPT_HDR(nt_hdr, ImageBase), mapped_module_base);
+    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+        "ASLR: aslr_write_header TimeDateStamp old "PFX", new "PFX"\n",
+        nt_hdr->FileHeader.TimeDateStamp, new_timestamp);
+
+    /* note that the FileHeader.TimeDateStamp is different than
+     * IMAGE_EXPORT_DIRECTORY.TimeDateStamp - yet IAT addresses should
+     * better match the one used.  Since we have a new base and yet
+     * the loader doesn't know that the file is relocated we have to
+     * change the timestamp.
+     */
+
+    /* FIXME: may need to set Imagebase if the loader decides based on
+     * a comparison with that whether to relocate, instead of using
+     * the result code of NtMapViewOfSection */
+
+    /* case 8507 discusses ramifications of modifying or
+     * preserving the original value for each of these fields.
+     */
+    nt_hdr->OptionalHeader.CheckSum = new_checksum;
+    nt_hdr->FileHeader.TimeDateStamp = new_timestamp;
+    if (module_is_32bit(mapped_module_base)) {
+        /* wow64 process - new base can't be > 32 bits even in 64-bit process
+         * address space for 32-bit dlls.  */
+        IF_X64(ASSERT(is_wow64_process(NT_CURRENT_PROCESS));)
+        IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint((ptr_uint_t)new_preferred_base));)
+        /* ImageBase for a 32-bit dll is 32-bits for 32-bit and 64-bit DR. */
+        *(uint *)(OPT_HDR_P(nt_hdr, ImageBase)) = (uint)(ptr_uint_t)new_preferred_base;
+    }
+    else {
+#ifdef X64
+        /* ImageBase for a 64-bit dll is 64-bits. */
+        *(uint64 *)(OPT_HDR_P(nt_hdr, ImageBase)) = (uint64)new_preferred_base;
+#else
+        ASSERT_NOT_REACHED();
+#endif
+    }
+
+    return true;
+}
+
+/* returns true if succesful, caller is responsible for unmapping the
+ * mapped view if mapped_base is set. 
+ * If search_fitting_base then new_base is set to the new random base.
+ *
+ * Returned view is writable, but is not intended to be used for
+ * execution.  Note that the section handle is irrelevant for COW (
+ * i.e. always for SEC_IMAGE), we need to keep the section mapped if
+ * the produced private contents is to be used.
+ *
+ * Note this function can be called by producers as well as consumers
+ * to verify a mapping.
+ */
+static bool
+aslr_generate_relocated_section(IN HANDLE unmodified_section,
+                                IN OUT app_pc *new_base, /* presumably random */
+                                bool search_fitting_base,
+                                OUT app_pc *mapped_base,
+                                OUT size_t *mapped_size,
+                                OUT module_digest_t *file_digest)
+{
+    bool success;
+    NTSTATUS res;
+    HANDLE section_handle = unmodified_section;
+    app_pc base = (app_pc)0x0; 
+    /* we won't necessarily use this mapping's base, no need to
+     * require it to be at new_base.  If producer is going to be a
+     * consumer it better choose a good new_base that will work. */
+    /* Note that we could force the new base, but we may prefer to not
+     * ask the kernel to do more than necessary (though all PE fields
+     * should be RVAs that do not to be modified).  We may have wanted
+     * to pass a mapping here earlier if we had read the image size to
+     * support ASLR_RANGE_TOP_DOWN */
+
+    size_t commit_size = 0;       
+    /* commit_size for an explicit anonymous mapping
+     * will need to match section size */
+    SIZE_T view_size = 0;         /* full file view */
+    uint type = 0;              /* commit not needed for original DLL */
+    uint prot = PAGE_READWRITE; 
+    /* PAGE_READWRITE would allow us to update the backing section */
+    /* PAGE_WRITECOPY - will only provide the current mapping */
+
+    app_pc original_preferred_base;
+    uint module_characteristics;
+
+    ASSERT(*mapped_base == NULL);
+
+    res = nt_map_view_of_section(section_handle, /* 0 */
+                                 NT_CURRENT_PROCESS, /* 1 */
+                                 &base, /* 2 */
+                                 0, /* 3 */
+                                 commit_size, /* 4 */
+                                 NULL, /* 5 */
+                                 &view_size, /* 6 */
+                                 ViewShare, /* 7 */
+                                 type, /* 8 */
+                                 prot); /* 9 */
+    ASSERT(NT_SUCCESS(res));
+    if (!NT_SUCCESS(res)) {
+        *mapped_base = NULL;
+        return false;
+    }
+    /* side note: windbg receives a ModLoad: for our temporary mapping
+     * at the NtMapViewOfSection(), no harm.  Note that the path name
+     * the debugger uses is the value the loader sets in
+     * PEB->SubSystemData = PathFileName.Buffer.  In case it is
+     * setting that before NtMapViewOfSection(), but not in front of
+     * NtCreateSection() then the DLL path the debugger may be
+     * confused.  Though on XPSP2 it showed the correct current DLL
+     * name.
+     */
+
+    *mapped_base = base;
+    *mapped_size = view_size;
+
+    /* Be aware of LdrVerifyImageMatchesChecksum() for our relocations
+     * - but that maps in as SEC_COMMIT based on the original file, so
+     * even if it is called for anything other than what is exported
+     * in KnownDlls we'd be ok.  If we want to match that checksum we
+     * can follow suit and process the file image, or we can emulate
+     * that on a mapped image Section.
+     *
+     * FIXME: check what is the meaning of
+     * "IMAGE_DLL_CHARACTERISTICS_FORCE_INTEGRITY 0x0080 Code
+     * Integrity checks are enforced", documented in PECOFF v8.0
+     */
+    original_preferred_base = get_module_preferred_base(base);
+    if (original_preferred_base == NULL) {
+        ASSERT_CURIOSITY(false && "base at 0, bad PE?");
+        /* maybe not a PE */
+        ASSERT_NOT_TESTED();
+        goto unmap_and_exit;
+    }
+
+    /* This is the earliest we can tell that an EXECUTABLE is being
+     * mapped, e.g. for CreateProcess.  FIXME: case 8459 - unless we
+     * need to apply any other binary transformations, here we return
+     * failure so not to create a copy of the original executable.
+     */
+    /* check for PE already done by get_module_preferred_base() */
+    module_characteristics = get_module_characteristics(base);
+    if (TEST(IMAGE_FILE_RELOCS_STRIPPED, module_characteristics)) {
+        LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+            "ASLR: aslr_generate_relocated_section skipping non-relocatable module\n");
+        goto unmap_and_exit;
+    }
+    if (!TEST(IMAGE_FILE_DLL, module_characteristics)) {
+        if (TEST(ASLR_RANDOMIZE_EXECUTABLE, DYNAMO_OPTION(aslr_cache))) {
+            /* note that we have no problem randomizing an executable with
+             * relocations, when we're in the parent 
+             */
+            SYSLOG_INTERNAL_INFO("randomizing executable with .reloc");
+        } else {
+            /* FIXME: minor perf: every time we're starting an executable 
+             * we'd be wasting a lot of work until we get here, 
+             * see if any other mapping already exists
+             */
+            SYSLOG_INTERNAL_INFO("skipping executable, though it has .reloc");
+            goto unmap_and_exit;
+        }
+    }
+
+    /* check if we can deal with all sections in the module - .shared, non-readable */
+    if (!module_file_relocatable(base)) {
+        /* note that attackers don't have a chance to fool our checks
+         * with a fake .shared section since we do match the PE
+         * headers on verification.  But still minimal cost on
+         * verification to keep this routine the same for both
+         * producers and verifiers.
+         */
+        SYSLOG_INTERNAL_INFO("non relocatable DLL .shared section - can't replicate");
+        goto unmap_and_exit;
+    }
+
+    /* .NET DLLs */
+    if (TEST(ASLR_AVOID_NET20_NATIVE_IMAGES, DYNAMO_OPTION(aslr_cache))
+        && module_has_cor20_header(base)) {
+        /* FIXME:case 9164 once we have better capacity management
+         * currently only fear of new temporary DLLs generated by ASP.NET */
+        SYSLOG_INTERNAL_INFO_ONCE("not producing .NET 2.0 DLL - case 9164");
+        goto unmap_and_exit;
+    }
+
+    if (search_fitting_base) {
+        /* expected to be called by producer only */
+        *new_base = aslr_get_fitting_base(*new_base, view_size);
+        if (*new_base == NULL) {
+            SYSLOG_INTERNAL_INFO_ONCE("no good fit, don't bother producing");
+            goto unmap_and_exit;
+        }
+    }
+
+    /* optional: could check here for valid image checksum, or any
+     * future restrictions on original file contents which may need to
+     * be preserved */
+    if (file_digest != NULL) {
+        module_calculate_digest(file_digest,
+                                base,
+                                view_size,
+                                true, true, /* both short and full */
+                                DYNAMO_OPTION(aslr_short_digest),
+                                UINT_MAX/*all secs*/);
+    }
+
+    success = module_rebase(base, view_size, 
+                            *new_base - original_preferred_base);
+
+    /* need to perform all actions usually taken by rebase.exe note
+     * rebase modifies in the header the timestamp, imagebase and
+     * checksum
+     */
+
+    /* FIXME: case 8507: test bound DLLs - a too clean relocation may
+     * fool other DLLs bound to this one that they have the correct
+     * prebound IAT entries.
+     *
+     * The loader checks only timestamp and whether a DLL was
+     * relocated.  We need to tell the loader ImageBase matches
+     * mapping so it does not break our sharing, but then bounding
+     * will be fooled too.  FIXME: find if we can control separately,
+     * otherwise we'll have to hack somehow one idea is to increment
+     * the timestamp and have our hotpatching match [timestamp,
+     * timestamp+1]
+     */
+
+    /* windbg will not be happy with our mapped images having
+     * incorrect timestamp and checksum
+     * 
+     * May be better solution is to set IMAGE_NOT_AT_BASE in the
+     * MODULEITEM which hopefully will be used by the loader for any
+     * binding requests.  We cannot return STATUS_IMAGE_NOT_AT_BASE
+     * because then the loader will relocate and lose our sharing, and
+     * for that reason we may need to set the MODULEITEM at some later
+     * point.  Yet it is not obvious what would be a good time to
+     * modify these loader structures.
+     *
+     * Alternative, is to disable BINDing in the
+     * importers - they are anyways supposed to fail, so we can walk
+     * through the list of IMAGE_IMPORT_DESCRIPTOR (old BIND?),
+     * IMAGE_DIRECTORY_ENTRY_BOUND_IMPORT (new BIND),
+     * yet worst kind is the DelayLoad timestamp that may now be found to be bound.
+     * pecoff.doc: 5.8.1. The Delay-Load Directory Table, Delay Import Descriptor
+     * offset 28, size 4, Time Stamp, Time stamp of DLL to which this image has been bound.
+     * e.g. ImgDelayDescr.dwTimeStamp in Microsoft Visual Studio/VC98/Include/DELAYIMP.H
+     * So it may require too many possibly custom delay import implementations.
+     *
+     * presumably we cannot use IMAGE_DLLCHARACTERISTICS_ NO_BIND
+     * 0x0800 Do not bind the image (in pecoff v8.0), however should
+     * check whether the binder or the loader checks this.
+     *
+     * FIXME: case 8508: As an optimization we could bind all of our cached DLLs
+     * to our randomized version to recoup any losses.
+     * see "Optimizing DLL Load Time Performance"by Matt Pietrek 
+     *   http://msdn.microsoft.com/msdnmag/issues/0500/hood/
+     * to measure if that is at all worth it on current machines.
+     *
+     * "Even under the slowest scenario, [his P3 550MHz] machine still
+     * loaded the program under Windows 2000 in less than 1/50th of a
+     * second.  On Windows 2000, properly basing the DLLs improved the
+     * load time by roughly 12 percent. Basing and binding the EXE and
+     * the DLLs improved the load time by around 18 percent. "
+     */
+    /* Note we leave the
+     * PE.Checksum invalid since it is the value of the
+     * original file.  In case anyone matches the original
+     * file they will be happy, or in case they validates
+     * hopefully they will do against the disk image as
+     * SEC_COMMIT/MEM_MAPPED which will be OK since we'll pass the original image.
+     */
+
+    if (success) {
+        uint old_checksum;
+        uint old_timestamp;
+        uint new_timestamp;
+
+        /* we could use the current time as a new timestamp, but using
+         * old_timestamp + 1 will give us at least a way of finding
+         * the module in case of limited diagnostic information */
+        bool ok = get_module_info_pe(*mapped_base, &old_checksum,
+                                     &old_timestamp, NULL, NULL, NULL);
+        ASSERT(ok);
+
+        /* imagine any other product like our one-off hotpatches would
+         * get fooled by this non-transparency */
+        new_timestamp = aslr_timestamp_transformation(old_timestamp);
+        /* coordinate any changes here with aslr_compare_header() */
+
+        aslr_write_header(*mapped_base, *mapped_size, 
+                          *new_base,
+                          old_checksum, new_timestamp);
+        /* FIXME: we need to somehow preserve original_preferred_base
+         * for detection, see aslr_get_original_metadata() for other considerations */
+    }
+
+    return success;
+ unmap_and_exit:
+    /* we do not need the section mapping */
+    res = nt_unmap_view_of_section(NT_CURRENT_PROCESS, *mapped_base);
+    ASSERT(NT_SUCCESS(res));
+    *mapped_base = NULL;
+    return false;
+}
+
+#ifdef DEBUG 
+/* UNUSED */
+/* keeping this scaffolding code to still compile to allow
+ * experimentation with any new mapping and sharing features that may
+ * have different attributes
+ *
+ * given a handle to original section with original file contents,
+ * publishes in the appropriate section directory a randomized copy
+ */
+/* returns true if caller should expect to find a mapped relocated copy */
+bool
+aslr_experiment_with_section_handle(IN HANDLE file_handle,
+                                    const wchar_t *mostly_unique_name)
+{
+    HANDLE object_directory = shared_object_directory; /* publish in shared or private view */
+    HANDLE new_published_handle;
+    NTSTATUS res;
+    PSECURITY_DESCRIPTOR dacl = NULL;
+    bool permanent = false;
+
+    /* section<PAGE_EXECUTE, SEC_IMAGE, app_file> gives us CoW in each
+     * process, and we can't share the relocation information
+     */
+    /* section<PAGE_EXECUTE_READWRITE, SEC_IMAGE, original app_file>
+     * gives access denied since file is open only for execution.
+     * Though even proper privileges do not overwrite the original
+     * file - SEC_IMAGE is always copy on write.
+     */
+
+    /* only using SEC_COMMIT either with page file, or with a
+     * {file<FILE_EXECUTE | FILE_READ_DATA | FILE_WRITE_DATA>,
+     * section<PAGE_EXECUTE_READWRITE, SEC_COMMIT, file>,
+     * map<PAGE_READWRITE>} allows writers to write to a true shared
+     * memory with readers.  If a particular reader needs private
+     * writes they can use map<PAGE_WRITECOPY> (can even track the
+     * pages that have transitioned from PAGE_WRITECOPY into
+     * PAGE_READWRITE to find which ones have been touched.  
+     * 
+     * Note if we could use SEC_COMMIT for mapping DLLs we'd always
+     * need PAGE_WRITECOPY to allow hotp or other hookers to modify
+     * privately.  We may also depend on CoW for a shared DR cache if
+     * there similarly may be some rare private invalidations.
+     */
+
+    /* most likely places to experiment with flags are marked with CHANGEME */
+
+    /* FIXME: doublecheck flags and privileges with what smss does */
+    res = nt_create_section(&new_published_handle,
+                            SECTION_ALL_ACCESS, /* FIXME: maybe less privileges needed */
+                            NULL, /* full file size */
+                            PAGE_EXECUTE_READWRITE, 
+                            /* PAGE_EXECUTE_READWRITE - gives us true overwrite ability */
+                            /* PAGE_EXECUTE gives us COW but not sharing */
+                            /* PAGE_EXECUTE_WRITECOPY is still COW,
+                             * though it needs FILE_READ_DATA
+                             * privileges to at all create a section,
+                             * CHANGEME */
+                            SEC_COMMIT, /* CHANGEME SEC_IMAGE or SEC_COMMIT (default) */
+                            file_handle, /* CHANGEME */
+                            /* NULL for page file backed */
+                            /* file_handle for file backed */
+
+                            /* object name attributes */
+                            mostly_unique_name, 
+                            (permanent ? OBJ_PERMANENT : 0),
+                            object_directory, dacl);
+
+    /* FIXME: is SEC_BASED supported - and what good does that do to
+     * us?  For sure a convenient place to keep our current
+     * BaseAddress, since SECTION_BASIC_INFORMATION.BaseAddress is
+     * supposed to be valid only for SEC_BASED_UNSUPPORTED
+     */
+    if (NT_SUCCESS(res)) {
+        /* FIXME: this is done for real in aslr_file_relocate_cow(),
+         * FIXME: duplication here is left just for future experimentation
+         * now comes the interesting part of rebasing the executable to a random new address
+         * relocating and possibly updating all other fields that need to change
+         */
+        app_pc mapped_base;
+        size_t mapped_size;
+        app_pc new_base = (app_pc)(ptr_uint_t)0x12340000;
+        bool relocated = aslr_generate_relocated_section(new_published_handle,
+                                                         &new_base,
+                                                         false,
+                                                         &mapped_base, &mapped_size,
+                                                         NULL);
+        if (relocated) {
+            /* for testing purposes we're just touching the checksum
+             * to verify sharing and private pages */
+            /* FIXME: note that we want unique value for testing only! */
+            relocated = aslr_write_header(mapped_base, mapped_size, 
+                                          mapped_base,
+                                          (uint) win32_pid, 1);
+        }
+
+
+        if (0 && relocated) {  /* CHANGEME:  */
+            /* finally verifying that the data doesn't stick around if it's not unmapped */
+            res = nt_unmap_view_of_section(NT_CURRENT_PROCESS, mapped_base);
+            ASSERT(NT_SUCCESS(res));
+        }
+
+        /* CHANGEME now for experiment we do it again to see whether
+         * first write was a private COW or it is a still globally
+         * visible shared write */
+        /* unfortunately it is not - visible */
+        new_base = (app_pc)(ptr_uint_t)0x23450000;
+        aslr_generate_relocated_section(new_published_handle,
+                                        &new_base,
+                                        false,
+                                        &mapped_base, &mapped_size,
+                                        NULL);
+        if (relocated) {
+            /* for testing purposes just touching the checksum to verify
+             * sharing and private pages */
+            /* FIXME: note that we want unique value for testing only! */
+            relocated = aslr_write_header(mapped_base, mapped_size, 
+                                          mapped_base,
+                                          (uint) win32_pid, 2); 
+        }
+
+        if (!relocated) {
+            ASSERT_NOT_TESTED();
+            close_handle(new_published_handle);
+            return false;
+        }
+
+        if (permanent) {
+            ASSERT_NOT_TESTED();
+            close_handle(new_published_handle);
+        }
+        ASSERT_NOT_TESTED();
+        return true;
+    } else {
+        /* FIXME: need to check for name collisions in a race, we
+         * should still return true so that the caller tries to open
+         * the created object
+         */
+        if (res == STATUS_OBJECT_NAME_COLLISION) {
+            /* we don't need to create a section if it already exists,
+             * STATUS_OBJECT_NAME_EXISTS presumably on OBJ_OPENIF? */
+            ASSERT_CURIOSITY(false);
+            /* we assume caller should now try to use this - ok for
+             * SEC_IMAGE since published only in consistent views */
+            return true;
+        } else {
+            /* any other error presumed to mean sharing is not possible */
+            ASSERT_CURIOSITY(false);
+            return false;
+        }
+    }
+    ASSERT_NOT_REACHED();
+    return false;
+}
+#endif /* DEBUG */
+
+/* returns true if a section has been mapped locally as Copy on Write,
+ * and has been relocated at a randomly chosen base.  Caller has to
+ * unmap the returned private view on success.
+ */
+static bool
+aslr_file_relocate_cow(IN HANDLE original_file_handle, 
+                       OUT app_pc *relocated_module_mapped_base,
+                       OUT size_t *relocated_module_size,
+                       OUT app_pc *random_preferred_module_base,
+                       OUT module_digest_t *original_digest)
+{
+    NTSTATUS res;
+    HANDLE relocated_section;
+    res = nt_create_section(&relocated_section,
+                            SECTION_ALL_ACCESS, /* FIXME: maybe less privileges needed */
+                            NULL, /* full file size */
+                            PAGE_EXECUTE, 
+                            /* PAGE_EXECUTE gives us COW in readers
+                             * but can't share any changes.
+                             * Unmodified pages are always shared.
+                             */
+                            /* PAGE_EXECUTE_READWRITE - gives us true
+                               overwrite ability only in SEC_COMMIT */
+                            /* PAGE_EXECUTE_WRITECOPY is still COW,
+                             * though it also needs FILE_READ_DATA
+                             * privileges to at all create the section
+                             * which the loader doesn't use */
+                            SEC_IMAGE,
+                            /* note we can't map a SEC_IMAGE as PAGE_READWRITE, also
+                             * original_file_handle can't be pagefile - since we can't open
+                             * such section as a SEC_IMAGE later.
+                             */
+                            original_file_handle, 
+
+                            /* process private - no security needed */
+                            /* object name attributes */
+                            NULL /* unnamed */, 0, NULL, NULL);
+    ASSERT(NT_SUCCESS(res));
+    if (NT_SUCCESS(res)) {
+        /* now comes the interesting part of rebasing the executable
+         * to a random new address relocating and possibly updating
+         * all other fields that need to change
+         */
+ 
+        /* FIXME: have to pick a random address, yet such that we can
+         * share across processes
+         */
+        /* FIXME: if we'll do top down we'll need to map first to
+         * obtain the section size before we choose an address,
+         * similarly if we read the preferred address
+         */
+        /* FIXME: currently using the same address generation as the
+         * private mappings (a linearly growing random range for each
+         * process.  TOFILE better strategy may be needed
+         * e.g. an affine transformation that converts the usually
+         * non-overlapping region of system DLLs into some other
+         * region with a different base and possibly increased holes
+         * between DLLs.  Could also use different copies in different
+         * applications, due to conflicts, or if we simply don't want
+         * to share between different users.
+         */
+        bool relocated;
+        *random_preferred_module_base = aslr_get_next_base();
+        *relocated_module_mapped_base = NULL; 
+        /* note that if the producer is not using this mapping of the
+         * DLL, we don't care about it really being mapped where we
+         * want it in other processes, so relocated_module_mapped_base
+         * will not be the same as random_preferred_module_base.  Yet
+         * we do want the random_preferred_module_base to fit at least
+         * in the current producer's layout, so once we map the module
+         * and know its size we may choose a different base. */
+        relocated = 
+            aslr_generate_relocated_section(relocated_section,
+                                            random_preferred_module_base,
+                                            true, /* search to avoid conflict */
+                                            relocated_module_mapped_base,
+                                            relocated_module_size, 
+                                            original_digest);
+        if (!relocated) {
+            ASSERT(*relocated_module_mapped_base == NULL);
+        }
+
+        /* caller doesn't care about the section handle, but only
+         * about the mapping base and size
+         */
+        close_handle(relocated_section);
+        return relocated;       /* caller will unmap the view */
+    }
+    return false;
+}
+
+/* Note: this routine cannot be used on original handles since we
+ * don't have proper permissions for the application handle.  Leaving
+ * the routine in case we find it useful for files that we have opened
+ * ourselves and we want to detect too old files.
+ *
+ * Collect module source times to keep in our signature field.  Note
+ * that we can't just copy to our own fields for exact match since
+ * NTFS and FAT have different time granularites.
+ *
+ * Note our target file creation time is a good measure for a
+ * cleanup tool to remove files that are too old 'expired'.  FIXME: If
+ * we update our produced files in place then the modification time
+ * will be the correct time to use.
+ */
+bool
+aslr_module_get_times(HANDLE file_handle, uint64 *last_write_time)
+{
+    NTSTATUS res;
+    
+    FILE_BASIC_INFORMATION basic_info;
+
+    /* FileBasicInformation A FILE_BASIC_INFORMATION structure. The
+     * caller must have opened the file with the FILE_READ_ATTRIBUTES flag
+     * specified in the DesiredAccess parameter.
+     */
+
+    /* Note we're missing FILE_READ_ATTRIBUTES when using original
+     * application handles * and we don't ask for them in
+     * aslr_recreate_known_dll_file() either,
+     */
+
+    /* It looks like files are 
+     * Type             File
+     * Attributes       0
+     * GrantedAccess    0x100020:
+     *   Synch
+     *   Execute/Traverse
+     */
+
+    HANDLE read_attrib_handle = file_handle;
+
+    /* FIXME: we can't even use DuplicateHandle() SDK: For example, a
+     * file handle created with the GENERIC_READ access right cannot
+     * be duplicated so that it has both the GENERIC_READ and
+     * GENERIC_WRITE access right.
+     */
+    ASSERT_CURIOSITY(TESTALL(FILE_READ_ATTRIBUTES|SYNCHRONIZE, 
+                             nt_get_handle_access_rights(read_attrib_handle)));
+
+    /* FIXME: the only possibility left is to try to reopen the file 
+     * starting with full path, but we don't really have that.
+     * For now giving up on this route for original files.
+     */
+    res = nt_query_file_info(read_attrib_handle,
+                             &basic_info,
+                             sizeof(basic_info),
+                             FileBasicInformation);
+
+    if (!NT_SUCCESS(res)) {
+        ASSERT(false && "insufficient privilege or bad handle?");
+        return false;           /* can't read times */
+    }
+    
+    /* the LastAccessTime and FileAttributes aren't useful to us The
+     * other three times are interesting - most likely LastWriteTime
+     * by itself is sufficient for normal use.  Still interesting
+     * whether we should invalidate if any of the other fields are
+     * notably ChangeTime which is not exposed through Win32 makes a
+     * good candidate.
+     *
+     * DDK 
+     *   CreationTime 
+     *        Specifies the time that the file was created. 
+     *   LastWriteTime 
+     *        Specifies the time that the file was last written to. 
+     *   ChangeTime 
+     *        Specifies the last time the file was changed. 
+     *   
+     *  rumors and speculations that: this is the time the MFT entry is changed
+     * FIXME: should test
+     * 
+     * http://www.cygwin.com/ml/cygwin/2005-04/msg00492.html
+     * " Windows NT supports a fourth timestamp which is inaccessible from the
+     *   Win32 API.  The NTFS filesystem actually implements it.  It behaves
+     *   as a ctime in a POSIX-like fashion.  Cygwin's st_ctime stat member now
+     *   contains this ChangeTime, if it's available.
+     * "
+     * 
+     * "ctime attribute keeps track of when the content or meta information
+     * about the file has changed - the owner, group, file permission,
+     * etc. Ctime may also be used as an approximation of when a file was
+     * deleted."
+     */
+    ASSERT_NOT_TESTED();
+    *last_write_time = basic_info.LastWriteTime.QuadPart;
+    return true;
+}
+
+/* given original application file and hashed name, returns a handle
+ * to a relocated version of the file, caller should close_handle().
+ * 
+ * otherwise returns INVALID_HANDLE_VALUE if not found
+ */
+/*
+ * FIXME: alternative 1) add an item to a work queue - have a trusted
+ * process (nodemgr) produce=copy+rebase the file next time and
+ * (winlogon) can publish the section mappings and can even get rid of
+ * the image itself.
+ */
+
+static bool
+aslr_produce_randomized_file(IN HANDLE original_file_handle,
+                             const wchar_t *mostly_unique_name,
+                             OUT HANDLE *produced_file)
+{
+    aslr_persistent_digest_t aslr_digest;
+    /* FIXME: TOFILE: need to create a file from a properly secured
+     * location to avoid privilege elevation!  FIXME: there is still a
+     * hole in that in the current implementation we let go of our
+     * exclusive write (0) access handle to convert from producer into
+     * publisher which allows someone else to replace the file!
+     */
+
+    /* FIXME: alternatively: can create a new one on the fly with
+     * FILE_ATTRIBUTE_TEMPORARY FILE_SHARE_READ and use these
+     * attributes.  Yet (FILE_SHARE_READ)
+     * may let publishers see an incomplete file, so we'll need to get
+     * a safe checksum for synchronization.
+     */
+
+    /* FIXME: case 8458 about consulting with sharing and persistence
+     * threshold heuristics to decide whether to produce a sharable
+     * file.  FIXME: may also want to add a random jitter to the
+     * persistence thresholds to smoothen the transition over multiple
+     * runs - though it will make it hard to benchmark what we're
+     * doing.
+     */
+
+    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+        "ASLR: aslr_produce_randomized_file for %ls\n",
+        mostly_unique_name);
+
+    if (TEST(ASLR_SHARED_FILE_PRODUCER, DYNAMO_OPTION(aslr_cache))) {
+        /* Note that SEC_IMAGE is always mapped as
+         * PAGE_EXECUTE_WRITECOPY therefore one can't write back
+         * relocations to a file mapped as image, rather only
+         * SEC_COMMIT mappings allow writing back - and ImageRVAToVa
+         * conversions are neccessary when applying any relocations.
+         */
+
+        /* We have two options for producing a rebased copy:
+         *
+         * rebase+save: Rebase into created section, we relocate in
+         * memory the original file (as SEC_IMAGE), we'll end up with
+         * a COW copy, and then we'd have to write that to a file (as
+         * SEC_COMMIT) (converting from raw memory to PE aligned
+         * sections).  Not very efficient since results in two memory
+         * copies.  Reusable module relocate and a PE dump, current
+         * plan.  Any data that is not in a PE section is not going
+         * the be mapped in memory by later SEC_IMAGE, so we can
+         * ignore that.
+         * 
+         * copy+rebase: Alternatively, we can copy the whole file to disk (as
+         * SEC_COMMIT) and then relocate that one again as SEC_COMMIT
+         * in the private copy (with no COW) - easy file copy, harder
+         * file relocate.
+         */
+        bool ok;
+        app_pc relocated_module_mapped_base; /* mapping in current process */
+        size_t module_size; 
+        app_pc new_preferred_module_base; /* new random preferred base */
+
+        uint64 randomized_file_size;
+        uint64 original_file_size;
+        uint64 requested_size;
+
+        ok = os_get_file_size_by_handle(original_file_handle,
+                                        &original_file_size);
+        if (!ok)
+            return false;
+
+        requested_size = original_file_size + sizeof(aslr_persistent_digest_t);
+        if (!aslr_check_low_disk_threshold(requested_size)) {
+            /* Note there is no point in checking earlier whether
+             * available disk space is already beyond the minimum. If
+             * we preserve the original file size, we could save that
+             * one syscall here.  [perf minor] We may want to memoize
+             * the value and decide to never try again.
+             */
+            return false; 
+        }
+
+        ok = aslr_file_relocate_cow(original_file_handle,
+                                    &relocated_module_mapped_base, 
+                                    &module_size,
+                                    &new_preferred_module_base,
+                                    &aslr_digest.original_source);
+
+        if (ok) {
+            NTSTATUS res;
+            bool persistent = 
+                TEST(ASLR_PERSISTENT, DYNAMO_OPTION(aslr_cache));
+            
+            /* note that SEC_IMAGE is larger than the real file size,
+             * but could use module_size to be slightly more conservative */
+
+            /* note we test whether we can create a file after we've
+             * done a lot of work, but in fact as close as possible to
+             * actually producing the file is good */
+            ok = aslr_create_relocated_dll_file(produced_file, 
+                                                mostly_unique_name, 
+                                                original_file_size,
+                                                persistent);
+            
+            if (ok) {
+                /* FIXME: case 8459 now that we have a private copy of
+                 * the file we can also apply any other binary
+                 * rewriting here: hooking all exported functions, or
+                 * rewriting the export table; applying hotpatches as
+                 * coldpatches - e.g. if we have many functions to
+                 * patch, or for GBOP, etc.
+                 */
+
+                ok = module_dump_pe_file(*produced_file,
+                                         relocated_module_mapped_base, 
+                                         module_size);
+            } else {
+                *produced_file = INVALID_HANDLE_VALUE;
+            }
+
+            if (ok) {
+                module_calculate_digest(&aslr_digest.relocated_target,
+                                        relocated_module_mapped_base,
+                                        module_size,
+                                        true, true, /* both short and full */
+                                        DYNAMO_OPTION(aslr_short_digest),
+                                        UINT_MAX/*all secs*/);
+                /* other than crashing digest can't fail  */
+            }
+
+            /* we do not use the private section mapping any more */
+            res = nt_unmap_view_of_section(NT_CURRENT_PROCESS, 
+                                           relocated_module_mapped_base);
+            ASSERT(NT_SUCCESS(res));
+
+            if (ok) {
+                /* not all file contents are mapped in memory - see if
+                 * there is more to preserve to appease aslr_verify_file_checksum() */
+                /* FIXME: case 8496 tracks possibly removing that part */
+                ok = aslr_module_force_size(original_file_handle, *produced_file, 
+                                            mostly_unique_name,
+                                            &randomized_file_size);
+                ASSERT(ok);
+            }
+
+            if (ok) {
+                /* always produce and append signature in case clients
+                 * need to use it, not necessary in lax or fully
+                 * strict mode.  */
+                /* Target MD5 also allows a cleanup tool to detect
+                 * files that are corrupt 'rotten'.  Such a tool can't
+                 * determine if a file is 'stale' (not matching its
+                 * original) since we don't keep the original path
+                 * only its hash is in the name.
+                 */
+
+                ok = aslr_module_append_signature(*produced_file,
+                                                  &randomized_file_size,
+                                                  &aslr_digest);
+                ASSERT(ok);
+            }
+
+            if (!ok) {
+                if (*produced_file != INVALID_HANDLE_VALUE)
+                    close_handle(*produced_file);
+
+                return false;
+            }
+
+            /* at this point the file is still exlusive access to the
+             * producer and our current handle disallows execute
+             * access to make sure we don't map that as an exclusive
+             * copy.  In case of a power failure the file will be
+             * accessible to others so unless we are using
+             * aslr_safe_save, we need to have publishers do complete
+             * validation checks.
+             */
+            aslr_update_view_size(new_preferred_module_base, module_size);
+
+            SYSLOG_INTERNAL_INFO("ASLR: produced DLL cache copy %ls",
+                                 mostly_unique_name);
+
+            return true;
+        } else {
+            /* release attempted region to randomize */
+            aslr_update_failed(false /* no retry */, NULL, 0);
+        }
+    } else {
+        /* Need to ask a producer to produce a relocated version of a
+         * file at a given path, message should probably be keyed with
+         * the mostly_unique_name to allow faster processing of
+         * duplicate requests.
+         */
+
+        /* FIXME: note that we don't have a good way to obtain fully qualified name
+         * the names from FileNameInformation don't contain the volume path!
+         */
+
+        /* FIXME: if we want to use the workqueue model we should find the
+         * volume name from the file  */
+        /* Note that we currently only have the unqualified with volume path */
+        SYSLOG_INTERNAL_WARNING("ASLR: ask somebody to produce file %ls",
+                                mostly_unique_name);
+
+        /* FIXME: if we want synchronous LPC to nodemgr or winlogon we
+         * need to write a secure server side. */
+        /* It looks harder to ask somebody else to do the work than to do
+         * it ourselves */
+
+        /* see also aslr_process_worklist() about using the registry
+         * as an asynchronous mailbox */
+    }
+
+    return false;
+}
+
+
+static void
+aslr_get_unique_wide_name(const wchar_t *origname, const wchar_t *key,
+                          wchar_t *newname /*OUT*/, uint newname_max /* max #wchars */)
+{
+    /* note this routine is a copy of get_unique_name but for
+     * wchar_t should keep in synch any improvements 
+     */
+    uint timestamp = (uint) get_random_offset(UINT_MAX);
+    DEBUG_DECLARE(int trunc =) /* for DEBUG and INTERNAL */
+    _snwprintf(newname, newname_max,
+               L"%s-%d-%010u-%s", origname, 
+               get_process_id(), timestamp, key);
+
+    ASSERT(trunc > 0 && trunc < (int)newname_max &&
+           "aslr_get_unique_wide_name name truncated");
+    /* FIXME: case 10677 */
+    /* Truncation may result in incorrect use of the wrong file - we
+     * should not truncate any strings at end, but should rather
+     * truncate the app controlled name, and have a fixed format (zero
+     * padded) for any known suffix that we are adding.
+     */
+    newname[newname_max - 1] = L'\0';
+}
+
+/* produced_temporary_file is closed regardless of success */
+static bool
+aslr_rename_temporary_file(const wchar_t *mostly_unique_name_target,
+                           IN HANDLE produced_temporary_file,
+                           const wchar_t *temporary_unique_name)
+{
+    HANDLE our_relocated_dlls_directory = 
+        get_relocated_dlls_filecache_directory(true);
+    ASSERT(our_relocated_dlls_directory != INVALID_HANDLE_VALUE);
+
+    ASSERT(DYNAMO_OPTION(aslr_safe_save));
+    /* Note that we are providing an implicit guarantee that the file
+     * that we are renaming is safe to load without complete file
+     * validation:
+     * 1) it corresponds to the correct application file
+     * 2) we accept the risk of not allowing byte patching within files
+     * 3) FIXME: case 10378 files are internally consistent - e.g. out of disk errors 
+     * 4) files are completely flushed on disk
+     * 5) the file that we are renaming has been freshly produced
+     */
+    /* FIXME: if we do pass the original file handle while still open,
+     * we know we are dealing with the same file.
+     */
+    close_handle(produced_temporary_file);
+
+    /* FIXME: we may want to be able to rename the file while holding
+     * this handle, otherwise we have to count on names being unique
+     * enough, so we don't get our file overwritten before rename */
+
+    /* to use os_rename_file() we'd have had to convert the full path
+     * names and the per-user directory paths.
+     */
+    if (!os_rename_file_in_directory(our_relocated_dlls_directory,
+                                     temporary_unique_name, 
+                                     mostly_unique_name_target, 
+                                     false/*do not replace*/)) {
+        SYSLOG_INTERNAL_WARNING_ONCE("aslr_rename_temporary_file failed");
+        return false;
+    }
+
+    return true;
+}
+
+/* given a handle to original file, publishes in the appropriate
+ * section directory a section to a randomized copy, on success calls
+ * to aslr_subscribe_section_handle() should be able to find one
+ *
+ * if anonymous then the created section is not published, just a
+ * private section is created instead
+ *
+ * returns true if caller should expect to find a mapped relocated copy
+ * on subscribe, (may return true if a published section already exists)
+ *
+ * if new_section_handle is not INVALID_HANDLE_VALUE caller may use
+ * that section handle instead of subscribing
+ */
+static bool
+aslr_publish_section_handle(IN HANDLE original_file_handle,
+                            const wchar_t *mostly_unique_name,
+                            bool anonymous,
+                            OUT HANDLE *new_section_handle)
+{
+    HANDLE object_directory = 
+        shared_object_directory; /* publish in shared or private view */
+    HANDLE new_published_handle;
+    NTSTATUS res;
+    PSECURITY_DESCRIPTOR dacl = NULL;
+    HANDLE randomized_file_handle = NULL;
+
+    bool permanent = false; 
+    /* Whether published handle should be left after all consumers
+     * unmap their views. 
+     * Note that in asynchronous
+     * consumer/publisher it should always be persisted until reboot.
+     *
+     * Note this notion of 'permanence' is different from
+     * ASLR_SHARED_INITIALIZE_NONPERMANENT which only controls the
+     * lifetime of the directory itself.
+     *
+     * TOFILE: publisher may or may not create permanent
+     * sections probably shouldn't so that we keep a matching original
+     * application handle with the same lifetime as normal DLL
+     * mappings, as well as a non-leaking publisher handle.  If we
+     * leak within the process DLL churn will be handled a lot faster.
+     * Persistence (across application restarts) will require full
+     * validation of produced files on process churn.  Even more
+     * importantly we _cannot_ create permanent section objects from
+     * all applications, so our only option is to keep a handle open.
+     */
+
+    if (permanent && !anonymous) {
+        permanent = false;
+    }
+
+    ASSERT(new_section_handle != NULL);
+    *new_section_handle = INVALID_HANDLE_VALUE;
+
+    if (object_directory == INVALID_HANDLE_VALUE && !anonymous) {
+        /* FIXME: currently this could be evaluated in caller, yet 
+         * in the future may have multiple possible locations to try
+         */
+        return false;
+    }
+
+    /* Note that the two alternative implementations here are to share
+     * a relocated Section produced from the original file, or a
+     * Section produced from a randomized file produced by relocating
+     * the original file.  Since SEC_IMAGE is always COW and can't
+     * share writes, currently pursuing the second option - opening a
+     * randomized file on disk.
+     */
+
+    /* No section was published, we are the first to publish */
+    /* Note distinction between section publisher and file producer.
+     * It is possible to consider a separate process creating the
+     * files, in that case the publisher has to decide whether the
+     * file is corrupt or stale - especially when we don't need
+     * persistence.
+     */
+
+    /* FIXME: for perisistence: look for an already created randomized
+     * file, for pure randomization without persistence should attempt
+     * overwriting the previously generated file.  If in a race with
+     * another publisher who has created the file but not exported the
+     * section, then should make sure the overwrite will fail due to
+     * exclusive write access.
+     */
+    if (!aslr_open_relocated_dll_file(&randomized_file_handle,
+                                      original_file_handle,
+                                      mostly_unique_name)) {
+        HANDLE produced_file_handle;
+        const wchar_t *randomized_file_name;
+        wchar_t temporary_more_unique_name[MAX_PUBLISHED_SECTION_NAME];
+
+        /* should attempt to produce a new one if allowed to produce,
+         * or request for one to be produced by a trusted producer
+         */
+        randomized_file_handle = NULL;
+        if (DYNAMO_OPTION(aslr_safe_save)) {
+            /* we first create the randomized file version in a
+             * temporary file */
+            aslr_get_unique_wide_name(mostly_unique_name, 
+                                      L"tmp",
+                                      temporary_more_unique_name,
+                                      BUFFER_SIZE_ELEMENTS(temporary_more_unique_name));
+            randomized_file_name = temporary_more_unique_name;
+        } else {
+            randomized_file_name = mostly_unique_name;
+        }
+
+        if (aslr_produce_randomized_file(original_file_handle, 
+                                         randomized_file_name,
+                                         &produced_file_handle)) {
+            /* TOFILE: note that currently we cannot cleanly hand-off
+             * from producer handle to allow for a non-persistent file
+             * handle, see discussion in
+             * aslr_produce_randomized_file() about possible
+             * alternatives.  FIXME: We should be closing the
+             * exclusive write producer handle only after we open it
+             * as shared read/no write as a publisher.  Currently
+             * produced_file_handle has exclusive read hence we cannot
+             * publish.
+             */
+            /* case 9696 for DYNAMO_OPTION(aslr_safe_save) should
+             * guarantee file is written correctly before we rename */
+            if (DYNAMO_OPTION(aslr_safe_save)) {
+                os_flush(produced_file_handle);
+                /* temporary file version is self-consistent */
+                /* consumers don't need full validation */
+                if (!aslr_rename_temporary_file(mostly_unique_name,
+                                                produced_file_handle,
+                                                temporary_more_unique_name)) {
+                    ASSERT_CURIOSITY(false && "couldn't rename just produced temp file!");
+                    randomized_file_handle = NULL;
+                }
+                /* produced_file_handle is closed regardless of success */
+                LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+                    "ASLR: aslr_publish_section_handle: renamed %ls to %ls\n",
+                    temporary_more_unique_name, mostly_unique_name);
+
+            } else {
+                /* file is created with well-known name,
+                 * consumers must validate carefully
+                 */
+                close_handle(produced_file_handle);
+            }
+            
+            if (aslr_open_relocated_dll_file(&randomized_file_handle,
+                                             original_file_handle,
+                                             mostly_unique_name)) {
+            } else {
+                ASSERT_CURIOSITY(false && "couldn't open just produced file!");
+                randomized_file_handle = NULL;
+            }
+        }
+    } else {
+        LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+            "ASLR: aslr_publish_section_handle existing copy of %ls\n",
+            mostly_unique_name);
+        /* more visibility only when logging */
+        DOLOG(1, LOG_VMAREAS, {
+            SYSLOG_INTERNAL_INFO("ASLR: using existing DLL cache copy %ls",
+                                 mostly_unique_name);
+        });
+    }
+
+    if (TEST(ASLR_INTERNAL_SHARED_APPFILE, 
+             INTERNAL_OPTION(aslr_internal))) {
+        ASSERT_CURIOSITY(randomized_file_handle == NULL);
+        /* stress testing: temporarily testing application file
+         * sections instead of our own files, provides original file,
+         * so nothing is really randomized, just testing the other
+         * basic components
+         */
+        ASSERT_NOT_TESTED();
+        duplicate_handle(NT_CURRENT_PROCESS, original_file_handle, NT_CURRENT_PROCESS,
+                         &randomized_file_handle, 0, 0, 
+                         DUPLICATE_SAME_ACCESS|DUPLICATE_SAME_ATTRIBUTES);
+    }
+
+    if (randomized_file_handle == NULL) {
+        return false;
+    }
+
+    /* FIXME: doublecheck flags and privileges with what smss does for
+     * exporting KnownDlls */
+    res = nt_create_section(&new_published_handle,
+                            /* even as publisher we don't need any of
+                             * SECTION_ALL_ACCESS rights after
+                             * creating the object */
+                            SECTION_QUERY | /* optional */
+                            SECTION_MAP_WRITE | 
+                            SECTION_MAP_READ |  
+                            SECTION_MAP_EXECUTE,
+                            NULL, /* full file size */
+                            PAGE_EXECUTE, 
+                            /* PAGE_EXECUTE 
+                             * gives us COW in readers but not sharing */
+                            /* PAGE_EXECUTE_READWRITE 
+                             * - gives us true overwrite ability only in SEC_COMMIT*/
+                            /* PAGE_EXECUTE_WRITECOPY is still COW,
+                             * though it also needs FILE_READ_DATA
+                             * privileges to at all create the section
+                             * which loader doesn't use */
+                            SEC_IMAGE, /* PE file mapping */
+                            randomized_file_handle, 
+                            /* NULL for page file backed */
+                            /* object name attributes */
+                            /* if anonymous sections is not named after all */
+
+                            anonymous ? NULL : mostly_unique_name,
+                            (permanent ? OBJ_PERMANENT : 0),
+                            anonymous ? NULL : object_directory, 
+                            anonymous ? NULL : dacl);
+    /* we can close the file handle whether check section was created or not */
+    close_handle(randomized_file_handle);
+
+    /* FIXME: is SEC_BASED supported - and what good does that do to
+     * us?  Only potential value is
+     * SECTION_BASIC_INFORMATION.BaseAddress as a convenient place to
+     * keep our current BaseAddress, since is supposed to be valid
+     * only for SEC_BASED_UNSUPPORTED
+     */
+    if (NT_SUCCESS(res)) {
+        /* FIXME: TOFILE if not permanent may want to keep this section
+         * handle around - memory and handle leaked until process
+         * dies.  Permanent is even worse - leaked until reboot.
+         */
+
+        /* Note that a single process reloading a DLL should now find
+         * it all the time, so we'd leak a handle only for each unique
+         * DLL.  However, a lot more problematic is the commit memory
+         * leak if a process is producing temporary DLLs (like
+         * ASP.NET) that are later not needed.
+         * 
+         * Maybe the right thing to do is to close the handle right
+         * after the publisher opens it as a subscriber as well.  That
+         * way a section will exist - though I am not sure whether it
+         * will be visible - if there is at least one user at any
+         * time, and will disappear if not needed.  That's all that's
+         * needed for true sharing.  Otherwise need STICKY bit to mark
+         * DLLs that end up being reused.
+         */
+        if (permanent) {
+            ASSERT_NOT_TESTED();
+            close_handle(new_published_handle);
+            *new_section_handle = INVALID_HANDLE_VALUE;
+        } else {
+            *new_section_handle = new_published_handle;
+            /* leaked, or up to caller */
+        }
+        return true;
+    } else {
+        if (res == STATUS_OBJECT_NAME_COLLISION) {
+            /* FIXME: need to check for name collisions in a race, we
+             * should still return true so that the caller tries to open
+             * the created object
+             */
+            /* we don't need to create a section if it already exists,
+             * STATUS_OBJECT_NAME_EXISTS is a Warning presumably on
+             * OBJ_OPENIF? */
+
+            /* shouldn't happen if callers always first attempt to
+             * subscribe, multiple workqueue publishers may hit this
+             * as well */
+            ASSERT_CURIOSITY(false && "already published");
+            /* we assume caller should now try to use this - ok for
+             * SEC_IMAGE since published only in consistent views */
+            *new_section_handle = INVALID_HANDLE_VALUE;
+            return true;        /* we don't give out any new handles */
+        } else {
+            /* any other error presumed to mean sharing is not possible */
+            /* e.g. insufficient permissions STATUS_ACCESS_DENIED,*/
+            if (res == STATUS_INVALID_FILE_FOR_SECTION) {
+                /* an invalid PE file is used - e.g. created by us, or
+                 * maybe truncated due to power loss */
+                /* FIXME: if persistent we should request producer to redo,
+                 * otherwise someone should have caught this as stale.
+                 * Producer while writing should be exlusive. 
+                 */
+                ASSERT_CURIOSITY(false && "bad PE file");
+            } else if (res == STATUS_ACCESS_DENIED) {
+                ASSERT_CURIOSITY(false && "insufficient privileges");
+            } else
+                ASSERT_CURIOSITY(false && "unexpected failure on nt_create_section");
+            return false;
+        }
+    }
+    ASSERT_NOT_REACHED();
+    return false;
+}
+
+/* FIXME: for persistent file generation, we may want to observe a
+ * code generation rule that says that if current user has rights to
+ * modify the original DLL that is getting loaded, then it's not a
+ * problem to let them create a new copy (that may get overwritten).
+ * For transient files however publisher(==producer) only creates a
+ * section of the produced file and never looks at the file again, so
+ * there is no attack vector.
+ */
+ 
+/* preserve state about not having to aslr privately this section */
+static void
+aslr_set_randomized_handle(dcontext_t *dcontext,
+                           HANDLE relocated_section_handle,
+                           app_pc original_preferred_base,
+                           uint original_checksum,
+                           uint original_timestamp)
+{
+    /* FIXME: at this point we should keep track of this handle
+     * and if it is indeed truly randomized then we can skip
+     * randomizing it again in our aslr_pre_process_mapview()
+     * handling, should also keep original base and maybe
+     * timestamp/checksum.
+     */
+    /* it appears that windbg is capable of finding the correct
+     * symbols for a rebased DLL just fine.
+     * FIXME: case 8439 tracks verifying how exactly it works.
+     *
+     * The only user known to be unhappy about a different timestamp
+     * is hotpatching.
+     *
+     * FIXME: case 8507 should make sure that all possible bindings
+     * really care about timestamp+1 (negative test - easier to see
+     * which ones crash on leaving timestamp).  Regular statically
+     * loaded and bound DLLs do crash.  Delay loaded and bound DLLs
+     * are also expected to crash.
+     */
+
+    /* FIXME: case 1272: now can also add to module list 
+     * (short filename: already added in syscall routines)
+     * full path (except for volume name)
+     * original base
+     * original timestamp
+     * original checksum
+     *
+     * Slightly complicated to track until the section is mapped,
+     * allocating memory here for the structure may sometimes leak -
+     * so maybe not worth dealing with the full path (and more than
+     * some limited short filename).  Can keep a file handle per
+     * thread assuming single threaded, yet should defer allocating
+     * any memory for these until a NtMapViewOfSection() - on success
+     * add to module list, on failure can free.  App's file handle may get
+     * closed after NtCreateSection() so we can't really use it to
+     * reread the names.  In general other than for matching, we
+     * should use duplicate_handle(), but then we open a handle leak
+     * problem if we never see a corresponding event to
+     * close_handle().  We will have to
+     * close_handle(previous_file_handle) if it is not closed by the
+     * next time we get here, or similarly could free any memory that
+     * we may have allocated, so will always have at most one
+     * outstanding.
+     *
+     * FIXME: should we keep the original application handle just in
+     * case it is needed to handle failure of NtMapViewOfSection(), we
+     * can't duplicate?
+     */
+    dcontext->aslr_context.
+        randomized_section_handle = relocated_section_handle;
+    dcontext->aslr_context.
+        original_section_base = original_preferred_base;
+    dcontext->aslr_context.
+        original_section_checksum = original_checksum;
+    dcontext->aslr_context.
+        original_section_timestamp = original_timestamp;
+
+    if (TEST(ASLR_INTERNAL_SHARED_AND_PRIVATE, INTERNAL_OPTION(aslr_internal))) {
+        dcontext->aslr_context.
+            randomized_section_handle = INVALID_HANDLE_VALUE;
+    }
+}
+
+static
+bool
+aslr_get_original_metadata(HANDLE original_app_section_handle,
+                           OUT app_pc *original_preferred_base,
+                           OUT uint *original_checksum,
+                           OUT uint *original_timestamp)
+{
+    /* currently mapping the original section and read all the values
+     * that we need */
+    /* FIXME: If consumers don't need to map this file for any other
+     * reason it may be faster to have publishers publish this data
+     * with the relocated section.  Could possibly be kept as metadata
+     * by the producers, though it is easy enough to regenerate by
+     * publishers.
+     */
+
+    /* side note: currently producers need to map in the original
+     * file, publishers need to do so to calculate any hashes, and
+     * consumers may need to do this in some faster way to verify that
+     * exported sections are still valid, so anyone can produce these
+     * if we allow for a mapping to exist.
+     */
+    bool ok;
+    NTSTATUS res;
+    app_pc base = (app_pc)0x0; 
+    size_t commit_size = 0;       
+
+    SIZE_T view_size = 0;         /* full file view */
+    /* FIXME: we really only need the header, if that makes things
+     * faster otherwise system cache will get up to a 256KB view
+     */
+
+    uint type = 0;              /* commit not needed for original DLL */
+    uint prot = PAGE_READONLY; 
+
+    res = nt_map_view_of_section(original_app_section_handle, /* 0 */
+                                 NT_CURRENT_PROCESS, /* 1 */
+                                 &base, /* 2 */
+                                 0, /* 3 */
+                                 commit_size, /* 4 */
+                                 NULL, /* 5 */
+                                 &view_size, /* 6 */
+                                 ViewShare, /* 7 */
+                                 type, /* 8 */
+                                 prot); /* 9 */
+    ASSERT(NT_SUCCESS(res));
+    if (!NT_SUCCESS(res))
+        return false;
+    /* side note: windbg receives a ModLoad: for our temporary mapping
+     * at the NtMapViewOfSection(), no harm */
+
+    *original_preferred_base = get_module_preferred_base(base);
+    ASSERT(*original_preferred_base != NULL);
+
+    ok = get_module_info_pe(base, original_checksum,
+                            original_timestamp, NULL, NULL, NULL);
+    ASSERT(ok);
+
+    res = nt_unmap_view_of_section(NT_CURRENT_PROCESS, 
+                                   base);
+    ASSERT(NT_SUCCESS(res));
+    
+    return ok;
+}
+
+/* FIXME: TOFILE we may need to export the original file handle to preserve
+ * no-clobber transparency
+ * FIXME: verification is not yet implemented, but
+ * good enoguh for a demo and casual use
+ */
+static bool
+aslr_verify_section_backing(IN HANDLE original_app_section_handle,
+                            IN HANDLE new_relocated_handle)
+{
+    /* FIXME: on Win2k+ we can count on
+     * are_mapped_files_the_same(addr1, addr2) if we have made an
+     * incorrect guess of the DLL name. Since our published sections
+     * can't be based on the original file, we can use
+     * are_mapped_files_the_same() only if we keep a section mapped to
+     * the original file.  We'd just need to Map that section to
+     * verify the file is the same (e.g. it hasn't been renamed).  A
+     * modification to a file on the other hand should be prevented by
+     * keeping a handle to the existing one.
+     */
+
+    SYSLOG_INTERNAL_WARNING_ONCE("ASLR consumer relying on name hash only");
+
+    ASSERT_NOT_IMPLEMENTED(true);
+    return true;
+}
+
+static bool
+aslr_replace_section_handle(IN HANDLE original_app_section_handle,
+                            IN HANDLE new_relocated_handle)
+
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+
+    bool ok;
+    app_pc original_preferred_base;
+    uint original_checksum;
+    uint original_timestamp;
+    /* get metadata about original section that is lost in the rebased section */
+    ok = aslr_get_original_metadata(original_app_section_handle,
+                                    &original_preferred_base,
+                                    &original_checksum,
+                                    &original_timestamp);
+    if (!ok) {
+        ASSERT_NOT_TESTED();
+        ASSERT(false && "can't read metadata");
+        close_handle(new_relocated_handle);
+        return false;
+    }
+                                   
+    aslr_set_randomized_handle(dcontext, 
+                               new_relocated_handle,
+                               original_preferred_base,
+                               original_checksum,
+                               original_timestamp);
+
+    /* We need to preserve original handle to maintain
+     * transparent behavior with regards to attempts to
+     * modify a DLL while in use.
+     * Note that each consumer will keep a handle and only when all
+     * are done with it the file would be replacable.  
+     */
+    if (!TEST(ASLR_ALLOW_ORIGINAL_CLOBBER, DYNAMO_OPTION(aslr_cache))) {
+        /* Since we'll keep the app handle after mangling
+         * NtCreateSection/NtOpenSection we'll have to close the old
+         * handle the next time we're at NtCreateSection/NtOpenSection.
+         * At most one per-thread may be missing - and there
+         * is a mess tracking these across threads!  e.g. if a
+         * MapViewOfSection is done in another thread we'll not
+         * maintain our handle.  That's not the way the loader
+         * currently does things, so this fragile solution should hold
+         * up.
+         */
+        if (dcontext->aslr_context.
+            original_image_section_handle != INVALID_HANDLE_VALUE) {
+            /* FIXME: we don't follow NtCreateProcess, for a
+             * known such incorrect leak in parent instead of child
+             * if we randomize EXEs case 8902
+             */
+            ASSERT_CURIOSITY("unexpected unused handle");
+            ASSERT(dcontext->aslr_context.
+                   original_image_section_handle != original_app_section_handle);
+            close_handle(dcontext->aslr_context.
+                         original_image_section_handle);
+        }
+
+        dcontext->aslr_context.
+            original_image_section_handle = original_app_section_handle;
+        /* note that the app has never seen this handle,
+         * ignoring the miniscule race for another thread
+         * watching the OUT argument that this system call
+         * returns.  Very unlikely a supported app would use
+         * undefined values, and we already make enough
+         * assumptions about single threaded sequences.
+         */
+    } else {
+        /* we don't need to preserve anything */
+        ASSERT(dcontext->aslr_context.
+               original_image_section_handle == INVALID_HANDLE_VALUE);
+        dcontext->aslr_context.
+            original_image_section_handle = INVALID_HANDLE_VALUE;
+    }
+
+    return true;
+}
+
+/* given a handle to original file and hashed name, subscribes to a
+ * copy of the appropriate section directory a section to a presumably
+ * randomized copy.  After verifying it is really for the same file.
+ * Returns true if a shared relocated section is found.
+ */
+static bool
+aslr_subscribe_section_handle(IN HANDLE original_app_section_handle,
+                              IN HANDLE file_handle,
+                              const wchar_t *mostly_unique_name,
+                              OUT HANDLE *new_relocated_handle)
+{
+    HANDLE object_directory = 
+        shared_object_directory; /* publish in shared or private view */
+    NTSTATUS res;
+
+    if (object_directory == INVALID_HANDLE_VALUE) {
+        /* FIXME: currently this could be evaluated in caller, yet 
+         * in the future may have multiple possible locations to try
+         */
+        return false;
+    }
+
+    /* open our candidate section based on expected name */
+
+    /* note on XP SP2 that when the loader creates a section it typically has 
+     * the following access flags and attributes
+     * 0:000> !handle 750 f
+     * Handle 750
+     *   Type           Section
+     *   Attributes     0
+     *   GrantedAccess  0xf:
+     *          None
+     *          Query,MapWrite,MapRead,MapExecute
+     *   Name           <none>
+     *   Object Specific Information
+     *     Section base address 0
+     *     Section attributes 0x1800000  SEC_IMAGE | SEC_FILE
+     *     Section max size 0x4b000
+     */
+    /* note that loader doesn't use SECTION_QUERY on open section
+     * (KnownDlls) but we may want to be able to query, instead of
+     * adding only SECTION_MAP_READ | SECTION_MAP_WRITE |
+     * SECTION_MAP_EXECUTE,
+     * also we don't need STANDARD_RIGHTS_REQUIRED either
+     */
+    /* FIXME: on XP SP2+ there is a new flag
+     * SECTION_MAP_EXECUTE_EXPLICIT that the loader doesn't seem to
+     * use.  Not clear what it does, we should experiment with it, but
+     * easier not to depend on it.
+     */
+   res = nt_open_section(new_relocated_handle,
+                         SECTION_QUERY | /* optional */
+                         SECTION_MAP_WRITE | 
+                         SECTION_MAP_READ |  
+                         SECTION_MAP_EXECUTE,
+                         mostly_unique_name, 0, 
+                         object_directory);
+    if (NT_SUCCESS(res)) {
+        /* FIXME: now should for sure check whether this new mapping is
+         * related to the original file.
+         */
+        
+        if (!aslr_verify_section_backing(original_app_section_handle,
+                                        *new_relocated_handle)) {
+            ASSERT(false && "stale published section");
+            ASSERT_NOT_TESTED();
+            close_handle(*new_relocated_handle);
+            return false;
+        }
+
+        /* FIXME: should make sure that sizes are for sure the same,
+         * can't do that in case of query access, for now debug only check
+         */
+        DODEBUG({
+            uint new_section_attributes = 0;
+            uint original_section_attributes = 0;
+            LARGE_INTEGER new_section_size;
+            LARGE_INTEGER original_section_size;
+            bool new_attrib_ok = 
+                get_section_attributes(*new_relocated_handle, 
+                                       &new_section_attributes,
+                                       &new_section_size);
+            bool original_attrib_ok = 
+                get_section_attributes(original_app_section_handle, 
+                                       &original_section_attributes,
+                                       &original_section_size);
+            /* if we don't have Query access (e.g. for KnownDlls) we
+             * can't even tell what else we have or don't have */
+            if (!original_attrib_ok) {
+                SYSLOG_INTERNAL_WARNING_ONCE("ASLR sharing on KnownDll %ls", mostly_unique_name);
+            }
+
+            ASSERT(new_section_attributes == 
+                   original_section_attributes
+                   || !original_attrib_ok);
+            ASSERT(new_section_size.QuadPart == original_section_size.QuadPart
+                   || !original_attrib_ok);
+
+            SYSLOG_INTERNAL_INFO("ASLR: consumer: using section cache %ls",
+                                 mostly_unique_name);
+        });
+
+        return aslr_replace_section_handle(original_app_section_handle, 
+                                           *new_relocated_handle);
+    } else {
+        if (res == STATUS_OBJECT_NAME_NOT_FOUND) {
+            return false;
+        } else {
+            ASSERT_CURIOSITY(false && "nt_open_section failure");
+            return false;
+        }
+    }
+    ASSERT_NOT_REACHED();
+    return false;
+}
+
+/* returns true and sets new_relocated_handle if a randomized section
+ * should be used instead of the original handle returned by the OS
+ */
+static bool
+aslr_post_process_create_section_internal(IN HANDLE old_app_section_handle,
+                                          IN HANDLE file_handle,
+                                          OUT HANDLE *new_relocated_handle)
+{
+    wchar_t mostly_unique_name[MAX_PUBLISHED_SECTION_NAME];
+    bool ok;
+    HANDLE new_published_handle;
+
+    ASSERT(TEST(ASLR_SHARED_CONTENTS, DYNAMO_OPTION(aslr_cache)));
+    ASSERT(TESTANY(ASLR_SHARED_PUBLISHER | ASLR_SHARED_SUBSCRIBER | 
+                   ASLR_SHARED_ANONYMOUS_CONSUMER, 
+                   DYNAMO_OPTION(aslr_cache)));
+
+    /* Obtain our unique name - 
+     * based on file name and path hash
+     */
+    ok = calculate_publish_name(mostly_unique_name, 
+                                BUFFER_SIZE_ELEMENTS(mostly_unique_name),
+                                file_handle, old_app_section_handle);
+
+    /* FIXME: may need to append suffixes L".new" or L".orig" if
+     * necessary to publish both relocated and original sections in the namespace
+     * in aslr_verify_section_backing()
+     */
+    if (!ok) {
+        LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+            "ASLR: shared: exempted DLL\n");
+        return false;
+    }
+
+    /* FIXME: we may want to create another section based on the
+     * original file so that we maintain file consistency checks -
+     * e.g. one can't replace the original file with a new copy and
+     * when we close this handle we may want the exported sections to
+     * disappear as well.  Both publishers and subscribers may need to
+     * do that to keep reference counters matching.
+     */
+
+    /* if we are a subscriber, try opening a published section, and
+     * verify whether it is from correct DLL
+     */
+    if (TEST(ASLR_SHARED_SUBSCRIBER, DYNAMO_OPTION(aslr_cache)) &&
+        aslr_subscribe_section_handle(old_app_section_handle,
+                                      file_handle,
+                                      mostly_unique_name,
+                                      new_relocated_handle)) {
+        return true;
+    }
+
+    /* if we are a publisher, publish relocated section */
+    if (TESTANY(ASLR_SHARED_PUBLISHER | ASLR_SHARED_ANONYMOUS_CONSUMER, 
+                DYNAMO_OPTION(aslr_cache)) &&
+        aslr_publish_section_handle(file_handle, 
+                                    mostly_unique_name,
+                                    TEST(ASLR_SHARED_ANONYMOUS_CONSUMER, 
+                                         DYNAMO_OPTION(aslr_cache)),
+                                    &new_published_handle)) {
+        /* anonymous publisher==subscriber */
+        if (TEST(ASLR_SHARED_ANONYMOUS_CONSUMER, DYNAMO_OPTION(aslr_cache))) {
+            /* reuses the private section handle, just needs to
+             * register the metadata */
+            ASSERT(new_published_handle != INVALID_HANDLE_VALUE);
+            *new_relocated_handle = new_published_handle;
+            /* note handle may be closed on error,
+             * otherwise we'll just return the private handle
+             */
+            return aslr_replace_section_handle(old_app_section_handle, 
+                                               *new_relocated_handle);
+        }
+
+        /* see discussion in aslr_publish_section_handle() */
+        /* note we will leak the new_published_handle which will make
+         * hanlding of DLL churn within single process faster,
+         * FIXME: we still have to do more work as a subscriber to
+         * make sure it is consistent */
+
+        /* We don't need to always consume what we have
+         * produced - producer may want to produce only for
+         * others' consumption but refrain from using these...
+         */
+        if (TEST(ASLR_SHARED_SUBSCRIBER, DYNAMO_OPTION(aslr_cache))) {
+            /* we now reopen the object as a regular subscriber, so
+             * that we don't pass to the application a handle that may
+             * have higher privileges than necessary.  FIXME: this
+             * shouldn't be the case for file based handles, so could
+             * just return a handle from aslr_publish_section_handle()
+             */
+            if (aslr_subscribe_section_handle(old_app_section_handle,
+                                              file_handle, 
+                                              mostly_unique_name,
+                                              new_relocated_handle)) {
+                return true;
+            } else {
+                ASSERT_CURIOSITY(false && "publisher can't subscribe?");
+            }
+        } else {
+            /* just publishing, then we don't get sharing in this process */
+            /* FIXME: xref case 8458 may want to count number of
+             * simultaneous process mappings for deciding what to
+             * share
+             */
+            /* note we will leak the new_published_handle */
+            ASSERT_CURIOSITY(new_published_handle != INVALID_HANDLE_VALUE);
+            return false;
+        }
+    }
+
+    return false;
+}
+
+/* aslr_publish_file() may be used for proactive loading of a list of
+ *  files to relocate (similar to KnownDlls) the main problem with
+ *  this scheme is that we'd need the file path.  We could provide
+ *  full paths, or expect all important files to be only in %system32%
+ *  (which precludes IE or Office DLLs from being preprocessed).
+ *  Alternatively a work queue (ASLR_SHARED_WORKLIST) to publish can
+ *  be generated by previous runs.  
+ */
+bool
+aslr_publish_file(const wchar_t *module_name)
+{
+    HANDLE file;
+    HANDLE published_section;
+    bool ok;
+    wchar_t mostly_unique_name[MAX_PUBLISHED_SECTION_NAME];
+    NTSTATUS res;
+
+    HANDLE preloaded_dlls_directory = NULL;
+    /* FIXME: module_name should provide a full path */
+    res = nt_create_module_file(&file, module_name, preloaded_dlls_directory,
+                                FILE_EXECUTE | FILE_READ_DATA,
+                                FILE_ATTRIBUTE_NORMAL,
+                                FILE_SHARE_READ,
+                                FILE_OPEN, 0);
+    if (!NT_SUCCESS(res)) {
+        return false;
+    }
+    ok = calculate_publish_name(mostly_unique_name, 
+                                BUFFER_SIZE_ELEMENTS(mostly_unique_name),
+                                file, NULL);
+    if (!ok) {
+        ASSERT_NOT_TESTED();
+        return false;
+    }
+    aslr_publish_section_handle(file, mostly_unique_name, false, &published_section);
+    /* may leak published_section */
+    close_handle(file);
+    return true;
+}
+
+/* get target of symbolic link as file path and keep a directory handle open */
+static void
+aslr_get_known_dll_path(wchar_t *known_dll_path_buffer, /* OUT */
+                        uint max_length_characters)
+{
+    UNICODE_STRING link_target_name;
+    /* initialize with \??\ */
+    wchar_t link_target_name_prefixed[MAX_PATH] = GLOBAL_NT_PREFIX;
+    NTSTATUS res;
+    uint bytes_length;
+
+    /* FIXME: for now we don't keep the link_target_name, but just
+     * open a file directory.   
+     * FIXME: We may not need the hash for full path, if we later
+     * query the file handles.
+     */
+    /* FIXME: we may not need to use the full path name string.
+     * According to Windows Internals the symlinks should be usable
+     * directly as the object manager should apply the substitions -
+     * so \KnownDlls\KnownDllPath\kernel32.dll should open the right
+     * file.
+     */
+
+    /* initialize using stack buffer using room after \??\ prefix */
+    link_target_name.Length = 0;
+    link_target_name.MaximumLength = (ushort)
+        (sizeof(link_target_name_prefixed) -
+         wcslen(link_target_name_prefixed) * sizeof(wchar_t));
+    link_target_name.Buffer = link_target_name_prefixed + 
+        wcslen(link_target_name_prefixed);
+    bytes_length = link_target_name.MaximumLength;
+
+    ASSERT(known_dlls_object_directory != NULL);
+
+    res = nt_get_symlink_target(known_dlls_object_directory,
+                                KNOWN_DLL_PATH_SYMLINK, 
+                                &link_target_name,
+                                &bytes_length);
+    ASSERT(NT_SUCCESS(res));
+    ASSERT(bytes_length == (uint)(link_target_name.Length + sizeof(wchar_t) /* final NULL */ ));
+
+    if (!NT_SUCCESS(res)) {
+        ASSERT_NOT_TESTED();
+        known_dll_path_buffer[0] = 0;
+        return;
+    }
+    
+    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+        "ASLR: aslr_get_known_dll_path KnownDllPath = %ls\n",
+        link_target_name.Buffer);
+
+
+
+    wcsncpy(known_dll_path_buffer, link_target_name_prefixed,
+            max_length_characters);
+    known_dll_path_buffer[max_length_characters - 1] = 0;
+
+    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+        "ASLR: known_dll_path = %ls\n", 
+        known_dll_path_buffer);
+}
+
+/* given a handle, returns whether its name exactly matches \KnownDlls  */
+bool
+aslr_is_handle_KnownDlls(HANDLE directory_handle)
+{
+    /* We are always doing a slowish name comparison. */
+
+    /* Expected that loader uses the same handle for the whole process
+     * so we can cache the handle value it is using.  We can
+     * still support multiple handles, but we'll fail in the unlikely
+     * case that this one is closed and another reopened.
+     * Only once we have found the loader handle we can quickly check
+     * and discard the \BaseNamedObjects.  We could also traverse all
+     * handles to find the value instead of waiting until a first use
+     * of a KnownDll and set last_known_dlls_handle_is_valid.
+     */
+
+    /* ObjectBasicInformation NameInformationLength can tell us whether it is our length,
+     * but we anyways want an exact match
+     */
+
+    /* ObjectNameInformation will give us complete name match
+     * verify it is \KnownDlls
+     */
+    NTSTATUS res;
+    OBJECT_NAME_INFORMATION name_info;
+    uint bytes_length;
+    uint returned_byte_length;
+
+    name_info.ObjectName.Length = 0;
+    name_info.ObjectName.MaximumLength = 
+        sizeof(name_info.object_name_buffer);
+    name_info.object_name_buffer[0] = L'\0';
+    name_info.ObjectName.Buffer = name_info.object_name_buffer;
+    bytes_length = sizeof(name_info);
+
+    STATS_INC(aslr_dlls_known_dlls_query);
+    res = nt_get_object_name(directory_handle, &name_info,
+                             bytes_length, &returned_byte_length);
+    ASSERT(NT_SUCCESS(res));
+    /* UNICODE_STRING doesn't guarantee NULL termination */
+    NULL_TERMINATE_BUFFER(name_info.object_name_buffer);
+    if (!NT_SUCCESS(res) || /* xref 9984 */ name_info.ObjectName.Buffer == NULL) {
+        return false;
+    }
+    ASSERT_CURIOSITY(name_info.ObjectName.Buffer == name_info.object_name_buffer); 
+
+    if (wcscmp(name_info.ObjectName.Buffer, 
+               KNOWN_DLLS_OBJECT_DIRECTORY) == 0) {
+        return true;
+    } else {
+        return false;
+    }        
+}
+
+bool
+aslr_recreate_known_dll_file(OBJECT_ATTRIBUTES *obj_attr, 
+                             OUT HANDLE *recreated_file)
+{
+    /* NOTE: we are making the assumption that all KnownDlls and their
+     * dependents are all physically located in the KnownDllPath.
+     * Presumably the loader makes the same assumption so hopefully
+     * that is enforced somehow - e.g. smss.exe's DllPath may not
+     * allow other paths to resolve dependencies etc.
+     */
+
+    /* we need the section name to allow us to open the
+     * appropriate original file
+     */
+
+    /* To avoid risking reliance on having SeChangeNotifyPrivilege
+     * we'll construct the absolute path name instead of using a saved
+     * directory name, so that calculate_publish_name() can look it
+     * up.  Note currently that the full path name is all that we use
+     * as a hash, but we may need to use more file properties for hash
+     * and for validation, so we are still creating a file.
+     */
+
+    wchar_t dll_full_file_name[MAX_PATH];
+    NTSTATUS res;
+
+    /* Name of handle is \KnownDlls\appHelp.dll but when NtOpenSection
+     * is called with \KnownDlls directory root, the object name is
+     * simply appHelp.dll */
+
+    /* FIXME: we should use the obj_attr->ObjectName with careful
+     * checks or case 1800 try/except.  Alternatively we can use a
+     * single syscall to load it in our own buffer by calling
+     * nt_get_object_name() just like the code in
+     * aslr_is_handle_KnownDlls() and have some extra string
+     * manipulation.  For now trusting the loader not to change the
+     * object name in a race.
+     */
+    
+    _snwprintf(dll_full_file_name, BUFFER_SIZE_ELEMENTS(dll_full_file_name), 
+               L"%s\\%s", known_dll_path, obj_attr->ObjectName->Buffer);
+    NULL_TERMINATE_BUFFER(dll_full_file_name);
+
+    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+        "ASLR: aslr_recreate_known_dll_file = %ls\n", 
+        dll_full_file_name);
+
+    res = nt_create_module_file(recreated_file, dll_full_file_name, 
+                                NULL /* absolute path name */,
+                                FILE_EXECUTE | FILE_READ_DATA,
+                                FILE_ATTRIBUTE_NORMAL,
+                                FILE_SHARE_READ,
+                                FILE_OPEN, 0);
+    ASSERT(NT_SUCCESS(res));
+    if (!NT_SUCCESS(res)) {
+        return false;
+    }
+
+    /* Note that KnownDlls sections are cached and if the file has
+     * been modified the proper semantics is to still use the original
+     * file!  While the file may not be modifiable it may have been
+     * superseded. */
+    
+    /* FIXME: case 8503 need to verify that the file that we have
+     * opened is the same as the one in KnownDlls - e.g. it may have
+     * been modified, in that case we cannot use the current file and
+     * should give up on doing anything that would use the new file
+     * contents.
+     */
+    /* FIXME: should use are_mapped_files_the_same() to validate that
+     * a section created from the new file is the same as one from the
+     * original section */
+    /* FIXME: Alternatively could use some of the checks in
+     * aslr_verify_file_checksum() file to verify that a section from
+     * that matches the opened application section.  Especially for
+     * producers this may be a good additional sanity check.
+     */
+
+    SYSLOG_INTERNAL_WARNING_ONCE("ASLR sharing assuming KnownDll file %ls hasn't changed", 
+                                 dll_full_file_name);
+    /* FIXME: since we currently don't really care about anything
+     * other than the name itself, it is safe to ignore potential
+     * change */
+
+    return true;
+}
+
+/* common handler for NtCreateSection and NtOpenSection, note that
+ *
+ * NtOpenSection doesn't have a file_handle to the original file the
+ * section was created from, and instead it passes a file handle to a
+ * reopened expected matching original
+ * 
+ * returns true if application section is replaced with our own
+ */
+bool
+aslr_post_process_create_or_open_section(dcontext_t *dcontext,
+                                         bool is_create,
+                                         IN HANDLE file_handle /* OPTIONAL */,
+                                         OUT HANDLE *sysarg_section_handle)
+{
+    /* reading handle is unsafe to race only, since syscall succeeded */
+    HANDLE safe_section_handle;
+    bool result = false;
+
+    ASSERT(NT_SUCCESS(get_mcontext(dcontext)->xax));
+
+    safe_read(sysarg_section_handle,
+              sizeof(safe_section_handle), &safe_section_handle);
+
+    ASSERT(TEST(ASLR_DLL, DYNAMO_OPTION(aslr)));
+    ASSERT(file_handle != NULL && file_handle != INVALID_HANDLE_VALUE);
+
+    if (TEST(ASLR_SHARED_CONTENTS, DYNAMO_OPTION(aslr_cache)) &&
+        TESTANY(ASLR_SHARED_PUBLISHER | ASLR_SHARED_SUBSCRIBER | 
+                ASLR_SHARED_ANONYMOUS_CONSUMER, 
+                DYNAMO_OPTION(aslr_cache))) {
+        HANDLE new_section = NULL;
+
+        if (aslr_post_process_create_section_internal(safe_section_handle,
+                                                      file_handle,
+                                                      &new_section)) {
+            /* we'll replace original application handle with our
+             * new mapping, so closing the original handle */
+            /* FIXME: see note in aslr_post_process_mapview about handling failures
+             * where we may need to preserve the original handle as well
+             */
+            if (TEST(ASLR_ALLOW_ORIGINAL_CLOBBER, DYNAMO_OPTION(aslr_cache))) {
+                /* we can just close the handle to the original file */
+                close_handle(safe_section_handle);
+            } else {
+                ASSERT(safe_section_handle == 
+                       dcontext->aslr_context.original_image_section_handle);
+            }
+
+            /* safe_write() our new handle into argument location
+             * address in case old application pointer is no
+             * longer writable.
+             */
+            safe_write(sysarg_section_handle, sizeof(new_section), &new_section);
+            result = true;
+        } else {
+            /* we leave original handle as is */
+        }
+    }
+    return result;
+}
+
+static void
+aslr_process_worklist(void)
+{
+    ASSERT_NOT_IMPLEMENTED(false);
+    /* FIXME: case 8505 worklist - synchronous or asynchronous communication 
+     *
+     * This should use either an option string that lists all files
+     * that we'd want to publish, or better yet a global registry key
+     * that others may be allowed to write to.  Since non-critical
+     * loss due to races or malicious intent the latter may be good enough.
+     *
+     * FIXME: a more complicated worklist based on IPC with message
+     * passing/writable shared memory will require a lot more work.
+     * 
+     * For a portable worklist scheme we may use files.  Though on
+     * Windows the registry may be lighter weight.  In fact, both the
+     * registry and file system allow notification on additions with
+     * either ZwNotifyChangeKey or ZwNotifyChangeDirectoryFile so
+     * timely (though not synchronous) response is possible.  (Loop
+     * every 60s and work if CPU is idle is probably also acceptable.)
+     */
+}
+
+#ifdef GBOP
+/* Generic Buffer Overflow Protection
+ *
+ *   Detection is not based on controlling the PC under DR, but on
+ * running natively and expecting select hook locations to be targeted
+ * by injected shellcode or manipulated activation records.
+ * 
+ * o BOP can be bypassed by setting up the stack frame to look as
+ *   whatever would pass any policies whether from user or kernel mode.
+ *
+ * o usermode BOP can simply be bypassed by directly going to the kernel 
+ *
+ * o usermode BOP can also easily be bypassed if hooks go just after our hooks
+ * 
+ * o Simplest bypass is to use a non-hooked routine, and for shallow
+ *   hookers simply a level higher
+ *
+ * The last two make it very easy to bypass our usermode hooking BOP,
+ * but add the extra inconvenience for attackers to lookup the windows
+ * version (KUSER_SHARED_DATA or TEB->PEB) to find the correct syscall
+ * numbers.  They could also traverse our data structures and execute
+ * from our copy, yet to bypass all other BOPs as well, the first
+ * attack is most likely.
+ *
+ * FIXME: we're mostly ignoring new attacks from having our
+ * data structures writable.  Yet unlike MF here the threat is a lot
+ * higher since attackers are running active shellcode and can look up
+ * interesting data.
+ */
+
+/* FIXME: scramble this table so that an attacker can't search for it in 
+ *          memory and overwrite it with what they want.
+ */
+
+#define GBOP_DEFINE_HOOK(module, symbol) {module, #symbol},
+#define GBOP_DEFINE_HOOK_MODULE(module_name, set_name)                  \
+    GBOP_DEFINE_##module_name ## _ ## set_name##_HOOKS(#module_name".dll")
+
+/* For each point to be hooked for generic buffer overflow protection, add
+ * an entry in gbop_hooks as a module name, function name pair.
+ * Note: this array is only for those gbop hooks that are to be injected with
+ * the hotp_only interface, not for piggy-backing gbop
+ * Note: the names are case sensitive, so enter the correct name.
+ */
+static const gbop_hook_desc_t gbop_hooks[] = {
+    GBOP_ALL_HOOKS
+};
+
+#undef GBOP_DEFINE_HOOK
+#undef GBOP_DEFINE_HOOK_MODULE
+
+#define GBOP_HOOK_LIST_END_SENTINEL ((uint)-1)
+
+/* way too hacky linearization of a two dimensional array */
+/* via templates: expands each set to a list of +1+1 = 2 which would
+ * be the number of entries in that set */
+#define GBOP_DEFINE_HOOK(module, symbol) +1
+#define GBOP_DEFINE_HOOK_MODULE(module_name, set_name)                  \
+    GBOP_DEFINE_##module_name ## _ ## set_name##_HOOKS(unused), 
+
+/* size of array is determined by number of modules in GBOP_ALL_HOOKS,
+ * end is demarcated by GBOP_HOOK_LIST_END_SENTINEL
+ * Note that GBOP_SET_NTDLL_BASE is also included
+ */
+static const uint gbop_hooks_set_sizes[] = {
+    0,                          /* GBOP_SET_NTDLL_BASE */
+    GBOP_ALL_HOOKS
+    GBOP_HOOK_LIST_END_SENTINEL
+};
+
+#undef GBOP_DEFINE_HOOK
+#undef GBOP_DEFINE_HOOK_MODULE
+
+/* size of array is determined by number of modules in GBOP_ALL_HOOKS,
+ * end is demarkated by GBOP_HOOK_LIST_END_SENTINEL,
+ * Note that GBOP_SET_NTDLL_BASE is also included
+ */
+/* each set expands into a ON entry, so all sets are enabled by default */
+#define GBOP_DEFINE_HOOK_MODULE(module_name, set_name) 1,
+
+static int gbop_hooks_set_enabled[] = {
+    1,                          /* GBOP_SET_NTDLL_BASE */
+    GBOP_ALL_HOOKS
+    GBOP_HOOK_LIST_END_SENTINEL
+};
+#undef GBOP_DEFINE_HOOK_MODULE
+
+static const uint gbop_num_hooks = BUFFER_SIZE_ELEMENTS(gbop_hooks);
+
+/* FIXME: need better gbop for gdc; today a simple vm load would render our
+ * gbop useless.  case 8087.
+ */
+/* This flag tracks if a VM has been loaded; used by gbop to identify the
+ * presence of gdc.  Note: it doesn't track unloads or the actual execution of
+ * dgc that lead to a gbop hook.
+ */
+bool gbop_vm_loaded = false;
+
+/* converting from a condensed index (order after eliminating disabled
+ * sets) into a real index into gbop_hooks[] */
+const gbop_hook_desc_t *
+gbop_get_hook(uint condensed_index)
+{
+    uint set_index = 0;
+    uint real_index = 0;
+
+    ASSERT(condensed_index < gbop_num_hooks);
+    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, "gbop_get_hook: %d hook\n", condensed_index);
+   
+     /* skip earlier sets and expand from condensed index not including disabled sets */
+    while (condensed_index >= gbop_hooks_set_sizes[set_index] || 
+           !gbop_hooks_set_enabled[set_index]) {
+        if (gbop_hooks_set_enabled[set_index]) {
+            condensed_index -= gbop_hooks_set_sizes[set_index];
+        } else 
+            ASSERT_NOT_TESTED();
+
+        real_index += gbop_hooks_set_sizes[set_index];
+        ASSERT(real_index < gbop_num_hooks);
+        set_index++;
+        ASSERT(set_index < gbop_num_hooks); /* at least less than all possible hooks */
+    }
+    real_index += condensed_index;
+    ASSERT(real_index < gbop_num_hooks);
+    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, 
+        "gbop_get_hook: => %d real, %s!%s\n", real_index,
+        gbop_hooks[real_index].mod_name, gbop_hooks[real_index].func_name);
+
+    return &gbop_hooks[real_index];
+}
+
+uint
+gbop_get_num_hooks(void)
+{
+    static uint num_hooks = BUFFER_SIZE_ELEMENTS(gbop_hooks);
+    static bool gbop_hooks_initialized = false;
+
+    if (gbop_hooks_initialized)
+        return num_hooks;
+
+    /* we evaluate only before .data is protected, and we do not
+     * support dynamically changing the hooks
+     */
+    ASSERT(!dynamo_initialized);
+
+    if (DYNAMO_OPTION(gbop_include_set) != 0) {
+        /* case 8246: the set of hooks is currently not dynamic should
+         * unprotect .data around this if gbop_include_set is made dynamic
+         */
+        uint set_index = 0;
+        size_t total_size = 0;
+        while (gbop_hooks_set_sizes[set_index] != GBOP_HOOK_LIST_END_SENTINEL) {
+            if (TEST((1 << set_index), DYNAMO_OPTION(gbop_include_set))) {
+                gbop_hooks_set_enabled[set_index] = 1;
+                LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, "gbop_get_num_hooks: 0x%x => %d enabled \n", 
+                    (1 << set_index),
+                    gbop_hooks_set_sizes[set_index]);
+                total_size += gbop_hooks_set_sizes[set_index];
+            } else {
+                gbop_hooks_set_enabled[set_index] = 0;
+                LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1, "gbop_get_num_hooks: 0x%x => %d disabled \n",
+                    (1 << set_index),
+                    gbop_hooks_set_sizes[set_index]);
+            }
+            set_index++;
+        }
+        ASSERT(total_size <= num_hooks);
+        IF_X64(ASSERT_TRUNCATE(num_hooks, uint, total_size));
+        num_hooks = (uint) total_size;
+    }
+
+    if (DYNAMO_OPTION(gbop_last_hook) != 0 && !dynamo_initialized) {
+        num_hooks = MIN(num_hooks, DYNAMO_OPTION(gbop_last_hook));
+    }
+
+    gbop_hooks_initialized = true;
+    return num_hooks;
+}
+
+bool
+gbop_exclude_filter(const gbop_hook_desc_t *gbop_hook)
+{
+    char qualified_name[MAXIMUM_SYMBOL_LENGTH];
+    char *os_exclude_list = "";
+
+    if (IS_LISTSTRING_OPTION_FORALL(gbop_exclude_list)) {
+        return true;
+    }
+
+    if (get_os_version() == WINDOWS_VERSION_VISTA) {
+        /* xref 9772, on Vista shell32!RealShellExecuteA == shell32!RealShellExecuteW
+         * and shell32!RealShellExecuteExA == shell32!RealShellExecuteExW so we have to
+         * avoid hooking the same spot twice. Arbitrarily exclude W versions. */
+        os_exclude_list =
+            "shell32.dll!RealShellExecuteW;shell32.dll!RealShellExecuteExW";
+        DODEBUG_ONCE({
+            HANDLE shell_mod = get_module_handle(L"shell32.dll");
+            ASSERT(get_proc_address(shell_mod, "RealShellExecuteA") ==
+                   get_proc_address(shell_mod, "RealShellExecuteW"));
+            ASSERT(get_proc_address(shell_mod, "RealShellExecuteExA") ==
+                   get_proc_address(shell_mod, "RealShellExecuteExW"));
+        });
+    }
+
+    /* optimization - skip the string concatenation below if nothing to check */
+    if (IS_STRING_OPTION_EMPTY(gbop_exclude_list) && *os_exclude_list == '\0') {
+        return false;
+    }
+
+    /* concatenating names */
+    snprintf(qualified_name, BUFFER_SIZE_ELEMENTS(qualified_name),
+             "%s!%s", gbop_hook->mod_name, gbop_hook->func_name);
+    NULL_TERMINATE_BUFFER(qualified_name);
+    if (check_list_default_and_append(os_exclude_list,
+                                      dynamo_options.gbop_exclude_list,
+                                      qualified_name)) {
+        return true;
+    }
+
+    /* optimization - skip the string concatenation below if nothing to check */
+    if (IS_STRING_OPTION_EMPTY(gbop_exclude_list)) {
+        return false;
+    }
+
+    /* check for all */
+    snprintf(qualified_name, BUFFER_SIZE_ELEMENTS(qualified_name),
+             "%s!%s", gbop_hook->mod_name, "*");
+    NULL_TERMINATE_BUFFER(qualified_name);
+    if (check_list_default_and_append("", /* no default list, we checked os above */
+                                      dynamo_options.gbop_exclude_list,
+                                      qualified_name)) {
+        return true;
+    }
+    return false;
+}
+
+/* NOTE: Assumes x86 
+ * NOTE: CTI sizes do not include prefixes, and assumes prefixes
+ *       do not change the opcode (cf. ff)
+ */
+enum {
+    CTI_MIN_LENGTH = CTI_IND1_LENGTH,
+    CTI_MAX_LENGTH = CTI_FAR_ABS_LENGTH,
+};
+
+/* Check if instruction preceding return address on TOS is a call */
+/* FIXME: add stats */
+static bool
+gbop_is_after_cti(const app_pc ret_addr)
+{
+    /* Instructions are checked for cti in this order, put the most common
+     * cti first
+     */
+    const uint cti_sizes[] = { CTI_DIRECT_LENGTH,
+                               CTI_IAT_LENGTH,
+                               CTI_IND1_LENGTH,
+                               CTI_IND2_LENGTH,
+                               CTI_IND3_LENGTH,
+                               CTI_FAR_ABS_LENGTH,
+                             };
+    const int num_cti_types = sizeof(cti_sizes)/sizeof(cti_sizes[0]);
+
+    /* While decoding we could be looking for a CTI instruction of size 2
+     * e.g., and we could end up decoding  beyond CTI_MAX_LENGTH if raw bits
+     * there look like other long instructions.  This could even result in
+     * stack underflows.  So raw_bytes has some extra padding.
+     */
+    byte raw_bytes[CTI_MAX_LENGTH + MAX_INSTR_LENGTH /*padding*/];
+    int i = 0;
+    uint bytes_read = 0;
+    bool done = false;
+    dcontext_t *dcontext;
+
+    ASSERT(TEST(GBOP_CHECK_INSTR_TYPE, DYNAMO_OPTION(gbop)));
+    if (!TEST(GBOP_IS_CALL, DYNAMO_OPTION(gbop))) {
+        DODEBUG_ONCE(LOG(THREAD_GET, LOG_ALL, 1,
+                         "GBOP: gbop_is_after_cti: GBOP_CHECK_INSTR_TYPE is "
+                         "enabled, but GBOP_IS_CALL is not\n"));
+        return false;
+    }
+
+    memset((void *)raw_bytes, 0, sizeof(raw_bytes));
+
+    /* safe_read instructions before ret_addr. Try and read the max, if
+     * unsuccessful try reading one byte less at a time until min.  Common
+     * case we'd do one read of CTI_MAX_LENGTH bytes.
+     *
+     * FIXME: OPTIMIZATION: If the first read fails, we could instead
+     * align_backward on page_size, find the delta and just try that size.
+     * Not worth doing unless this routine proves to be expensive.
+     */
+    for (bytes_read = CTI_MAX_LENGTH; bytes_read >= CTI_MIN_LENGTH; bytes_read--) {
+        done = safe_read(ret_addr - bytes_read, bytes_read, (void *)raw_bytes);
+        if (done)
+            break;
+        ASSERT_NOT_TESTED();
+    }
+
+    if (!done) {
+        LOG(THREAD_GET, LOG_INTERP, 1,
+            "GBOP: gbop_is_after_cti: could not read %d to %d bytes above "
+            "return addr=0x%0x\n", CTI_MIN_LENGTH, CTI_MAX_LENGTH, ret_addr);
+
+        ASSERT_NOT_TESTED();
+        return false; /* cannot read instructions above return addr */
+    }
+
+    ASSERT(bytes_read >= CTI_MIN_LENGTH);
+
+    /* FIXME: CLEANUP: dcontext is not used at all in decode_opcode, but
+     * don't want to pass GLOBAL_DCONTEXT or NULL
+     */
+    dcontext = get_thread_private_dcontext();
+
+    /* Now that we have read raw instructions, check to see if ret_addr was
+     * preceded by a call.  Check if we find a call opcode at offsets listed
+     * in cti_sizes[].
+     */
+    for(i = 0; i < num_cti_types; i++) {
+        app_pc pc = NULL;
+        app_pc next_pc = NULL;
+        instr_t instr;
+
+        /* skip call instruction types that are larger than bytes read */
+        if (bytes_read < cti_sizes[i]) {
+            ASSERT_NOT_TESTED();
+            continue;
+        }
+
+        /* We come here only if bytes_read <= cti_sizes[i], and use
+         * bytes_read - cti_sizes[i] as an index into raw_bytes read.
+         * e.g., if bytes_read is 7 and if cti_sizes[i] = 2, an indirect
+         * call, we check for this type of call at index 5.  Also, if we
+         * find a call we do not expect its size to be > 2.
+         */
+        pc = &raw_bytes[bytes_read - cti_sizes[i]];
+
+        /* set up instr for decode_opcode */
+        instr_init(dcontext, &instr);
+        DODEBUG({
+            /* case 9151: only report invalid instrs for normal code decoding */
+            instr.flags |= INSTR_IGNORE_INVALID;
+        });
+
+        /* NOTE: Make sure we do not do any allocations here (see note of
+         * CAUTION below about hotpach limitations).  decode, if asked for
+         * operands e.g., can allocate memory trying to up-decode the
+         * instruction.  Even if the first src operand is statically
+         * allocated and is asked for, decode may fill up other implicit
+         * operands and could lead to memory allocation.
+         * GBOP_EMULATE_SOURCE needs to be careful while looking at
+         * operands.
+         */
+        next_pc = decode_opcode(dcontext, pc, &instr);
+        ASSERT(!instr_has_allocated_bits(&instr));
+
+        if (next_pc != NULL) {
+            ASSERT(!instr_operands_valid(&instr));
+            ASSERT(!instr_needs_encoding(&instr));
+
+            /* If we found a valid call instruction and it is of the size we
+             * expect to find, then return valid, otherwise continue looking.
+             * FIXME: GBOP_IS_JMP NYI
+             */
+            if (instr_is_call(&instr)) {
+                /* FIXME: make sure next_pc == ret_addr (NOTE: next_pc is inside
+                 *        raw_bytes buffer)
+                 *        something like: next_pc - pc == cti_sizes[i], since
+                 *        call instructions do not have prefix
+                 *
+                 * FIXME: in debug build, after finding a match continue looking
+                 *        and incr. stat on conflicts
+                 * Note: GBOP_EMULATE_SOURCE may have to do PC relativization
+                 */
+                LOG(THREAD_GET, LOG_ALL, 3,
+                    "GBOP: gbop_is_after_cti: found valid call preceding return "
+                    "addr=0x%0x\n", ret_addr);
+                return true;
+            }
+        }
+        /* loop back and check for other possible ctis */
+    }
+
+    LOG(THREAD_GET, LOG_ALL, 1,
+        "GBOP: gbop_is_after_cti: no valid call preceding return addr=0x%0x\n",
+        ret_addr);
+    return false; /* didn't find a valid call instruction preceding ret_addr */
+}
+
+/* note we currently don't care which rule was broken, an exemption
+ * will overrule any */
+static inline
+bool
+check_exempt_gbop_addr(app_pc violating_source_addr)
+{
+    /* Currently exempting only if source is a named DLL */
+    /* Most violations will be failing GBOP source memory page rules
+     * and not really in a proper DLL.  Yet, it is possible that
+     * source is a PE module that is not properly loaded as MEM_IMAGE
+     * and breaks all other allow rules based on source page
+     * properties, this will look for a PE name at the allocation
+     * base.  FIXME: considered a feature not a bug to allow
+     * MEM_MAPPED
+     *
+     * case 8245 about an example where a DLL breaks the source
+     * instruction type properties.
+     */
+    if (!IS_STRING_OPTION_EMPTY(exempt_gbop_from_default_list) ||
+        !IS_STRING_OPTION_EMPTY(exempt_gbop_from_list)) {
+        const char *source_module_name;
+        os_get_module_info_lock();
+        os_get_module_name(violating_source_addr, &source_module_name);
+        LOG(THREAD_GET, LOG_INTERP, 2, 
+            "check_exempt_gbop_addr: source_fragment="PFX" module_name=%s\n", 
+            violating_source_addr, source_module_name != NULL ? 
+            source_module_name : "<none>");
+        /* note check_list_default_and_append will grab string_option_read_lock */
+        if (source_module_name != NULL
+            && check_list_default_and_append(dynamo_options.exempt_gbop_from_default_list,
+                                             dynamo_options.exempt_gbop_from_list,
+                                             source_module_name)) {
+            LOG(THREAD_GET, LOG_INTERP, 1, 
+                "GBOP: exception from exempt source module --ok\n");
+            os_get_module_info_unlock();
+            return true;
+        }
+        os_get_module_info_unlock();
+    }
+    return false;
+}
+
+/* CAUTION: this routine is called by hotp_only_gbop_detector, which means 
+ *          that it has to adhere to all limitations prescribed for hot patch 
+ *          code, i.e., no system calls, no calls to dr code, allocating 
+ *          memory, holding locks or changing whereami.  If any of those need 
+ *          to be done, the code should be carefully examined.
+ * FIXME: as of now this function hasn't been examined as stated above.
+ */
+/* returns true if caller to hooked routine looks valid
+ */
+bool
+gbop_check_valid_caller(app_pc reg_ebp, app_pc reg_esp, app_pc cur_pc,
+                        app_pc *violating_source_addr /* OUT */)
+{
+    /* FIXME: optional: check PC -- should check PC if this is done in
+     * kernel mode, otherwise we can detect only locations we have
+     * hooked */
+    
+    /* optional: adjust ESP to TOS (needs FPO information).  Here we
+     * assume that we have hooked at function entry points, or at
+     * least early enough that [ESP] still points to the return
+     * address.  Can use [EBP] in earlier frames, and only if we know
+     * it is really used.
+     */
+    app_pc ret_on_stack = reg_esp;
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    uint depth = 0;             /* NYI stack walk */
+    ASSERT(violating_source_addr != NULL);
+    ASSERT(dcontext != NULL && dcontext != GLOBAL_DCONTEXT);
+    if (dcontext == NULL /* case 9385: unknown thread */ ||
+        dcontext == GLOBAL_DCONTEXT) {
+        /* if we don't know what's going on, we shouldn't block nor crash */
+        return true;
+    }
+
+    /* for each frame */
+    do {
+        /* assume app stack is valid if attackers are trying to execute code */
+        /* FIXME: make this a safe read, or a TRY/EXCEPT */
+        /* otherwise an attack with ESP just starting off a good page
+         * will be reported as DR crash.  Not very useful for attacks
+         * other than that instead of return an exception is produced
+         * instead, (and still generated on the new good stack) */
+        app_pc purported_ret_addr = *(app_pc*)ret_on_stack; /* unsafe_read */
+
+        /* We want to make sure that we check the properties of a page
+         * on which the expected CALL instruction for sure is.  case
+         * 3821 about our own interop problems with ones incorrectly
+         * dealing with the very beginning of a page, and using the
+         * ret addr as is may already be on the next page (though
+         * interesting only if counting on exception on return).
+         */
+        app_pc suspect_shellcode_addr = purported_ret_addr - 1;
+    
+        bool source_page_ok = false;
+        /* collect all properties for single shot diagnostics */
+        bool is_exec = false;
+        bool is_x = false;      /* updated only if !is_exec */
+        bool is_image = false;  /* updated only if !is_exec */
+        bool on_stack = false;
+        bool is_future_exec = false; /* updated only if !is_exec */
+
+        reg_t spill_mc_esp;
+
+        bool source_instr_ok = !(TEST(GBOP_CHECK_INSTR_TYPE, DYNAMO_OPTION(gbop)));
+        bool is_call = false;   /* NYI */
+        bool is_hp_jmp = false; /* NYI */
+        bool is_jmp = false;    /* NYI */
+
+        /* In case of failure, we point to the next instruction in
+         * shellcode (not the arbitrary one in the middle of a CALL
+         * that we check as suspect_shellcode_addr).  We obviously
+         * can't know the beginning of the shellcode, so it's really
+         * last not first bad instruction
+         */
+        *violating_source_addr = purported_ret_addr;
+
+        ASSERT(DYNAMO_OPTION(gbop) != GBOP_DISABLED);
+        ASSERT(TESTANY(GBOP_IS_EXECUTABLE | GBOP_IS_X | GBOP_IS_IMAGE
+                       | GBOP_IS_FUTURE_EXEC, 
+                       DYNAMO_OPTION(gbop)));
+
+        /* You cannot have GBOP_CHECK_INSTR_TYPE w/o one of
+         * GBOP_IS_{CALL,JMP,HOTPATCH_JMP}
+         */
+        ASSERT(!TESTANY(GBOP_CHECK_INSTR_TYPE, DYNAMO_OPTION(gbop)) ||
+               (TESTANY(GBOP_CHECK_INSTR_TYPE, DYNAMO_OPTION(gbop)) &&
+                TESTANY(GBOP_IS_CALL | GBOP_IS_JMP | GBOP_IS_HOTPATCH_JMP,
+                        DYNAMO_OPTION(gbop))));
+
+        ASSERT_NOT_IMPLEMENTED(!TESTANY(~(GBOP_IS_EXECUTABLE |
+                                          GBOP_IS_X |
+                                          GBOP_IS_IMAGE |
+                                          GBOP_CHECK_INSTR_TYPE |
+                                          GBOP_IS_CALL |
+                                          GBOP_IS_DGC |
+                                          GBOP_IS_FUTURE_EXEC |
+                                          GBOP_IS_NOT_STACK),
+                                        DYNAMO_OPTION(gbop)));
+        /* FIXME: NYI: GBOP_IS_JMP | GBOP_IS_HOTPATCH_JMP
+         * GBOP_EMULATE_SOURCE | GBOP_IS_RET_TO_ENTRY
+         * GBOP_WHEN_NATIVE_EXEC 
+         * GBOP_DIAGNOSE_SOURCE
+         */
+
+        LOG(THREAD_GET, LOG_SYSCALLS, 2,
+            "GBOP: checking pc="PFX", reg_esp="PFX", reg_ebp="PFX"\n",
+            cur_pc, reg_esp, reg_ebp);
+
+        /* As long as we make the check fast enough it will be OK to hook
+         * even all functions in kernel32, or e.g. ntdll!Nt* to hook all
+         * system calls.  Could use a last_area (will have to be writable
+         * so not very secure) - per function or a global last_area though
+         * could make check that it points within the array so can't be
+         * overwritten trivially (and assuming DR memory is not writable).
+         */
+        if (TEST(GBOP_IS_EXECUTABLE, DYNAMO_OPTION(gbop))) {
+            is_exec = is_executable_address(suspect_shellcode_addr);
+        }
+
+#ifdef PROGRAM_SHEPHERDING
+        if (!is_exec && 
+            TEST(GBOP_IS_FUTURE_EXEC, DYNAMO_OPTION(gbop))) {
+            /* is_future_exec is cheaper to evaluate policy
+             * than the the policies that use query_virtual_memory()
+             * so doing before the rest
+             */
+            is_future_exec = is_in_futureexec_area(suspect_shellcode_addr);
+            LOG(THREAD_GET, LOG_VMAREAS, 1,
+                "GBOP: using GBOP_IS_FUTURE_EXEC "PFX" %s\n",
+                suspect_shellcode_addr,
+                is_future_exec ? "allowing future" : "not future");
+            /*
+             * FIXME: not supporting GBOP_DIAGNOSE_SOURCE, so
+             * evaluating only if needed
+             */
+        }
+#endif /* PROGRAM_SHEPHERDING */
+
+        /* may still allow with the alternative policies, note that
+         * GBOP_IS_EXECUTABLE is not always a superset, of even
+         * GBOP_IS_X, but usually should match what we'd need.
+         */
+        if (!is_exec && !is_future_exec) {
+            /* note that we always check whether the weaker policies would
+             * have worked even if they are not currently enabled.
+             * FIXME: should be done only under GBOP_DIAGNOSE_SOURCE
+             *
+             * FIXME: Maybe should just have these turned on all the time,
+             * (GBOP_IS_X is a problem only on Win2003 RTM), but could
+             * have staging use the information to propose adding
+             * post-factum since known to work.
+             */
+            /* FIXME: should have a vm_area_t that we track our quick list,
+             * and in addition have to doublecheck for any memory that may
+             * have shown up in our process without our knowledge.  Since
+             * we anyways have to do check for the latter, adding to list
+             * on violation instead of tracking protection changes.
+             */
+
+            /* FIXME: for the time being will do the system call all
+             * the time.  Shoul add the entry to a vmarea, for the
+             * latter we should be getting a real lock and make DR
+             * datastructures writable.  We could otherwise process
+             * these hooks without protection changes (other than locks).
+             */
+            app_pc check_pc = (app_pc)ALIGN_BACKWARD(suspect_shellcode_addr, PAGE_SIZE);
+            MEMORY_BASIC_INFORMATION mbi;
+            if (query_virtual_memory(check_pc, &mbi, sizeof(mbi)) == sizeof(mbi)) {
+                is_image = (mbi.Type == MEM_IMAGE);
+                is_x = prot_is_executable(mbi.Protect);
+            }
+            /* we only use current Protect mapping for these stateless
+             * policies.  Here MEM_FREE or MEM_RESERVE are still
+             * worthy of reporting a violation - since after all they
+             * somehow managed to make this call - so maybe somebody
+             * is fooling with us. */
+            DOLOG(2, LOG_VMAREAS, {
+                if (is_x && TEST(GBOP_IS_X, DYNAMO_OPTION(gbop))) {
+                    LOG(THREAD_GET, LOG_VMAREAS, 1, "GBOP: using is GBOP_IS_X\n");
+                }
+                if (is_image && TEST(GBOP_IS_IMAGE, DYNAMO_OPTION(gbop))) {
+                    /* FIXME: all we check for GBOP_IS_IMAGE is whether MEM_IMAGE
+                     * is set, note executable_if_image in fact only counts on
+                     * getting module base, but should also be checking this.
+                     */
+                    LOG(THREAD_GET, LOG_VMAREAS, 1, "GBOP: using GBOP_IS_IMAGE\n");
+                }
+            });
+        }
+
+        /* Is the bad address on the stack?  Case 8085.  Have to save, use and 
+         * restore mcontext esp because is_address_on_stack() directly gets 
+         * the esp from the mcontext to find the stack base & size.
+         * Note: The app stack can change when we walk the frames (app. may
+         *       switch stacks), so compute on_stack for each frame walked.
+         */
+        spill_mc_esp = get_mcontext(dcontext)->xsp;
+        get_mcontext(dcontext)->xsp = (reg_t) reg_esp;
+        on_stack = is_address_on_stack(dcontext, purported_ret_addr);
+        get_mcontext(dcontext)->xsp = spill_mc_esp;
+
+        ASSERT(!is_exec || TEST(GBOP_IS_EXECUTABLE, DYNAMO_OPTION(gbop)));
+        ASSERT(!is_future_exec || TEST(GBOP_IS_FUTURE_EXEC, DYNAMO_OPTION(gbop)));
+
+        /* CAUTION: the order of the source page checks shouldn't be changed! */
+        source_page_ok = is_exec || is_future_exec
+            || (TEST(GBOP_IS_IMAGE, DYNAMO_OPTION(gbop)) && is_image)
+            || (TEST(GBOP_IS_X, DYNAMO_OPTION(gbop)) && is_x);
+
+        /* Allow any target but the current stack; case 8085. */
+        if (!source_page_ok &&
+            (TEST(GBOP_IS_NOT_STACK, DYNAMO_OPTION(gbop)) && !on_stack)) {
+            LOG(THREAD_GET, LOG_VMAREAS, 1, "GBOP: using GBOP_IS_NOT_STACK\n");
+            source_page_ok = true;
+        }
+
+        /* Allow any target but the current stack, if a vm is loaded; case 8087. */
+        if (!source_page_ok &&
+            (TEST(GBOP_IS_DGC, DYNAMO_OPTION(gbop)) && gbop_vm_loaded &&
+             !on_stack)) {
+            LOG(THREAD_GET, LOG_VMAREAS, 1, "GBOP: using GBOP_IS_DGC\n");
+            source_page_ok = true;
+        }
+
+        DOLOG(2, LOG_VMAREAS, {
+            /* note regular callstack dumps assume EBP chain, yet comes
+             * with expensive checks for is_readable_without_exception()
+             */
+            dump_callstack(cur_pc, reg_ebp, THREAD_GET, DUMP_NOT_XML);
+        });
+
+        if (!source_page_ok) {
+            if (!check_exempt_gbop_addr(*violating_source_addr)) {
+                return false; /* bad source memory type */
+            }
+            LOG(THREAD_GET, LOG_VMAREAS, 1, 
+                "GBOP: exempted bad source memory properties\n");
+            /* continuing in case we're walking stack frames */
+        }
+
+        if (TEST(GBOP_CHECK_INSTR_TYPE, DYNAMO_OPTION(gbop)))
+            source_instr_ok = gbop_is_after_cti(purported_ret_addr);
+
+        if (!source_instr_ok) {
+            if (!check_exempt_gbop_addr(*violating_source_addr)) {
+                return false; /* bad source instruction type */
+            }
+            LOG(THREAD_GET, LOG_VMAREAS, 1, 
+                "GBOP: exempted bad source instruction type\n");
+            SYSLOG_INTERNAL_WARNING("GBOP exempted instr type @"PFX"\n", 
+                                    *violating_source_addr);
+            /* continuing in case we're walking stack frames */
+        }
+
+        /* FIXME: stack walking will just have to iterate in the above
+         * loop for the next frame, if needed
+         */
+        if (DYNAMO_OPTION(gbop_frames) > 0) {
+            /* FIXME: how to reliably walk the stack:
+             * - could check if EBP is within current thread stack
+             *   as long as that is not changed by the app or for fibers
+             * - it's ok to skip frames if we follow EBP,
+             * - can also use SEH frames as a guideline to where function
+             *   frames should be, but those may have been overwritten as well
+             * - see windbg "Manually Walking a Stack" for manual ones, though 
+             *   kb isn't doing much
+             * - look at gdb source code to see if they have a good heuristic
+             */
+            ASSERT_NOT_IMPLEMENTED(false);
+            /* get next frame:   ra = poi(reg_ebp+4), reg_ebp = poi(reg_ebp) if safe */
+            /* FIXME: check if readable without exception safely */
+        }
+    } while (0); /* FIXME: currently not walking frames */
+
+    return true; /* valid, or haven't found reason */
+}
+
+/* used to validate the hooks in ntdll.dll on system calls and loader
+ * routines, but not for any additional hotpatch hooks */
+void
+gbop_validate_and_act(app_state_at_intercept_t *state,
+                      byte fpo_adjustment,                      
+                      app_pc hooked_target)
+{
+    /* FIXME: while the 'extra' hook locations (hotpatched) from
+     * gbop_include_list will simply not be injected the always hooked
+     * (native_exec hooked) locations may need a bitmask say
+     * gbop_active[] for syscalls.  Callers should check so not to
+     * burden this routine with indexing or lookup.
+     *
+     * FIXME: plan should add LdrLoadDll as a fake syscall and could
+     * even add to syscall_trampoline_hook_pc[i], so we can deal with
+     * all of these uniformly - we need index for quick check, and
+     * original PC for producing the generic obfuscated threat ID.
+     */
+    app_pc bad_addr;
+
+    ASSERT(DYNAMO_OPTION(gbop) != GBOP_DISABLED);
+    if (!TEST(GBOP_SET_NTDLL_BASE, DYNAMO_OPTION(gbop_include_set))) {
+        return;
+    }
+
+    STATS_INC(gbop_validations);
+    if (!gbop_check_valid_caller((app_pc)state->mc.xbp, 
+                                 (app_pc)state->mc.xsp + fpo_adjustment, 
+                                 hooked_target,
+                                 &bad_addr)) {
+        dcontext_t *dcontext = get_thread_private_dcontext();
+        security_option_t type_handling = OPTION_BLOCK|OPTION_REPORT;
+#ifdef PROGRAM_SHEPHERDING
+        app_pc old_next_tag;
+        fragment_t src_frag = {0}, *old_last_frag;
+        dr_mcontext_t old_mc = {0};
+#endif
+
+        STATS_INC(gbop_violations);
+        /* FIXME: should provide the failure depth or simpy first bad
+         * target to report 
+         */
+        LOG(THREAD_GET, LOG_ASYNCH, 1, "GBOP invalid source to "PFX"!\n", 
+            hooked_target);
+        SYSLOG_INTERNAL_ERROR("GBOP: execution attempt to "PFX" from bad "PFX"\n",
+                              hooked_target, bad_addr);
+        /* FIXME: reporting: have to reverse the usual meaning of good source,
+         * BAD target, here Threat ID should be of hooked target as
+         * constant part, and contents of source
+         */
+
+        /* FIXME: case 7946 action - standard attack handling,
+         * or alternative handling return error - easy for the ntdll!Nt*, 
+         * not so easy for the other entry points - will need to cleanup arguments,
+         * and probably not going to lead to more than crashes anyways.
+         * xref case 846 about competitors silently eating these
+         */
+        /* FIXME: may want to return after_intercept_action_t in that
+         * case to be able to modify the target to go to the function
+         * exit.  However, that also requires knowing a correct
+         * address viable only as a liveshield update.  A better solution 
+         * is to pass the number of arguments and just clean those up.
+         */
+        /* FIXME: need to test separately the handling of nt syscalls,
+         * LdrLoadDll and the 'extra' hooks
+         */
+
+        /* FIXME: reporting for gbop should be via one path; today it is 
+         * different for core gbop hooks and hotp_only gbop hooks; case 8096.
+         * changes here must be kept in synch with hotp_event_notify till then.
+         */
+#ifdef PROGRAM_SHEPHERDING
+        /* Save the last fragment, next tag & registers state, use the correct
+         * ones, report & then restore.
+         */
+        hotp_spill_before_notify(dcontext, &old_last_frag, &src_frag,
+                                 hooked_target, &old_next_tag, bad_addr,
+                                 &old_mc, state, CXT_TYPE_CORE_HOOK);
+
+        /* does not return when OPTION_BLOCK is enforced */
+        if (security_violation(dcontext, bad_addr,
+                               GBOP_SOURCE_VIOLATION,
+                               type_handling) == GBOP_SOURCE_VIOLATION) {
+            /* running in detect mode, or action didn't kill control flow */
+            ASSERT_NOT_TESTED();
+        } else {
+            /* exempted Threat ID */
+            ASSERT_NOT_TESTED();
+        }
+        hotp_restore_after_notify(dcontext, old_last_frag, old_next_tag, &old_mc);
+#endif /* PROGRAM_SHEPHERDING */
+        /* FIXME: we may want to cache violation source location, if
+         * survived either due to either detect mode or exemption */
+    }
+}
+
+void
+gbop_init(void)
+{
+    /* all hooks already used by DR for native_exec and hotp_only do
+     * not need special initialization
+     */
+    /* FIXME: here should hook all routines other than the ones we
+     * have already hooked in ntdll!Ldr* and ntdll!Nt*
+     */
+
+    /* FIXME: note that DLLs that are not yet loaded will have to be
+     * hooked at normal hotp_only times */
+
+    /* FIXME: on hook conflicts we should give up our optional hooks -
+     *   just like hotp_only should already do.
+     *
+     * FIXME: conflict with hotp_only hashing - we'll have to
+     * recalculate hashes.  Best to treat the extra hooks as
+     * hotpatches, and if we don't chain our hotpatches, should kick
+     * out the existing extra hooks.
+     */
+
+    /* FIXME: Tim brought up that we can't use LdrGetProcedureAddress
+     * at the time of load, but rather have to read the exports
+     * ourselves - a problem mostly for already loaded DLLs that we'd
+     * process at startup */
+
+    /* FIXME: currently not called - decide its relationship with
+     * hotp_init: whether before so that it uses the loader list walk,
+     * or after so it can explicitly use hotp routines
+     */
+    ASSERT_NOT_IMPLEMENTED(false);
+}
+
+void
+gbop_exit(void)
+{
+    /* FIXME: have to unhook all additional routines on detach */
+    ASSERT_NOT_IMPLEMENTED(false);
+}
+#endif /* GBOP */
diff --git a/core/win32/aslr.h b/core/win32/aslr.h
new file mode 100644
index 000000000..5cd93a413
--- /dev/null
+++ b/core/win32/aslr.h
@@ -0,0 +1,693 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2005-2007 Determina Corp. */
+
+/*
+ * aslr.h - Types and definitions for ASLR address space layout randomization
+ *
+ * Plan to implement only for win32.
+ */
+
+#ifndef ASLR_H
+#define ASLR_H
+
+#include "ntdll.h"
+
+/* ASLR control flags for -aslr */
+enum {
+    ASLR_DISABLED          = 0x00, 
+
+    /* application memory areas to randomize */
+    ASLR_DLL               = 0x01, /* randomize DLLs mapped as image */
+    ASLR_STACK             = 0x02, /* case 6287 - padding initial stack from parent */
+    /* Note option active in parent applies to its children */
+
+    ASLR_HEAP              = 0x04, /* random initial virtual memory padding */
+    ASLR_MAPPED            = 0x08, /* FIXME: NYI case 2491, case 6737 */
+    ASLR_EXECUTABLE        = 0x10, /* FIXME: NYI case 2491 + case 1948  */
+
+    ASLR_PROCESS_PARAM     = 0x20, /* FIXME: NYI case 6840 */ 
+    /* Note option active in parent applies to its children */
+    /* controls earliest possible padding from parent, yet interop
+     * problem with device drivers assuming ProcessParameters is at a
+     * fixed location, separately controlled from ASLR_STACK which
+     * gets allocated later
+     */
+    
+    ASLR_HEAP_FILL         = 0x40, /* random padding between virtual allocations */
+    ASLR_TEB               = 0x80, /* FIXME: NYI case 2491 */
+
+    /* FIXME: for TEB can reserve the pages after the PEB allocation
+     * unit, yet the first threads are still in the PEB allocation
+     * unit.  We could fill the original threads space from the parent
+     * before any other threads are created, but not that valuable
+     * since still providing known writable location */
+
+    /* FIXME: PEB probably CAN'T be moved from user mode at all, 
+     * definitely can't relocate in running process since can't change protections
+     */
+
+    /* range allocation - case 6739 */
+    /* A virtual memory range should be allocated to each group,
+     * (DLLs, mappings, heap).  Default range allocation within a
+     * group is to add new areas bottom up (from a defined starting
+     * value + random); after each mapping the size is known and some
+     * smaller random padding can be added.  
+     * FIXME: Reclaiming ranges is not currently controlled,
+     *  committed memory issues due to that tracked in case 6729
+     */
+    ASLR_RANGE_BOTTOM_UP   = 0x00100, /* default behaviour, reserved */
+    ASLR_RANGE_TOP_DOWN    = 0x00200, /* NYI: FIXME: need size for
+                                       * going top down requires an
+                                       * extra system call */
+    ASLR_RANGE_RANDOM      = 0x00400, /* NYI: FIXME: may cause too
+                                       * much fragmentation, best done
+                                       * with full vmmap, then we can
+                                       * choose a random location
+                                       * anywhere in our range */
+
+    ASLR_SHARE_DR_DLL      = 0x10000000, /* FIXME: NYI case 8129 */
+    /* Note option active in parent applies to its children */
+    /* In addition to the current -aslr_dr feature, should eventually
+     * share views similar to ASLR_SHARED_CONTENTS */
+    
+    /* default in client mode currently is */
+    ASLR_CLIENT_DEFAULT = ASLR_DLL | ASLR_STACK | ASLR_HEAP, /* 0x7 */
+} aslr_option_t;
+
+/* ASLR control flags for -aslr_cache */
+enum {
+    /* sharing flags - case 2491 scheme 2.2
+     * NYI we may publish section handles in a global
+     * namespace to allow sharing of private mappings,
+     * and share addresses for compatibility, or both addresses and contents 
+     * for memory reduction
+     */
+    ASLR_PROCESS_PRIVATE           = 0x1, /* default behaviour, flag RESERVED */
+
+    ASLR_SHARED_PER_USER           = 0x2,
+    /* NYI: allows per-user sharing so that all instances of a
+     * module by 'user' use the same mapping, but other users use
+     * private copies.  
+     *   0) sharing - good - for the general desktop only SYSTEM and Administrator
+     *   1) integrity- permission-wise created by 'user'
+     *      so other processes may write to it without impact to others 
+     *   2) disclosure - other users base addresses will be different
+     */
+
+    ASLR_SHARED_INHERIT            = 0x4,
+     /* NYI: allow trusted users to create mappings that other users
+      * share (kernel deals with ref counting, and expected to be
+      * sticky! so low priv process doesn't need to do anything about
+      * removing ).  
+      *    0) sharing better than ASLR_SHARED_PER_USER 
+      *    1) integrity same 
+      *
+      *    2) disclosure worse, since now plain users know
+      *       the randomized mappings for SYSTEM processes
+      *       and allows local attacks
+      *
+      *    3) there is also a more theoretical information disclosure
+      *    in case a published executable has 'secrets' that low
+      *    privilege users aren't supposed to be able to read or
+      *    execute from
+      */
+
+    ASLR_SHARED_IN_SESSION         = 0x8,
+    /* NYI: \Local\ limits sharing for terminal service users to only
+     * current session, can be combined with INHERIT to prevent
+     * disclosure of mappings of services from remotely logged users.
+     * (Although in Vista services will be the only processes in
+     * Session 0, on earlier Windows versions still allows desktop
+     * user to completely share and inherit mappings).  May apply only
+     * to inherit, or could be used to disallow user in multiple sessions...
+     */
+    /* Note that different security models based on the user
+     * privileges may have some processes independently take some but
+     * not necessarily all of file 'producer', section 'publisher' and
+     * mapping 'subscriber' roles. */
+
+    ASLR_SHARED_CONTENTS           = 0x10,
+    /* NYI: If not set sharing will be of address only to resolve
+     * incompatible applications requiring a DLL to be mapped at the
+     * same address, if set sharing also means saving memory by not
+     * using private copies.
+     */
+
+    ASLR_SHARED_PUBLISHER          = 0x20,
+    /* Process is allowed and has permissions to publish sections for
+     * other users to map contents.  Note may be different from
+     * file producer marked as ASLR_SHARED_FILE_PRODUCER.
+     */
+
+    ASLR_SHARED_SUBSCRIBER         = 0x40,
+    /* Process is to take the risk to use published sections, with
+     * risk of privilege escalation if sections or backing files are
+     * improperly secured, and according to the sharing inheritance
+     * flags.
+     */
+
+    ASLR_SHARED_ANONYMOUS_CONSUMER = 0x80,
+    /* Anonymous section mappings directly from an already produced
+     * file if current user doesn't have permission to publish,
+     * e.g. direct file consumer.  Useful if consistency and security
+     * needs can be satisfied without need for exclusive file access
+     * and other shared objects.  Performance hit depends on
+     * consistency checks performed.
+     *
+     * FIXME: we may want to fallback to this option even if we use as
+     * default option a publisher and use any of the further options
+     * tracked in case 8812
+     */
+
+    ASLR_SHARED_FILE_PRODUCER      = 0x100,
+    /* Publisher may only be opening files created by a different
+     * process if not set, or may produce the relocated files from
+     * within DR.
+     *
+     * FIXME: can also queue up request to generate a particular file
+     * even if not materializing immediately in that case it will be
+     * combined with ASLR_SHARED_WORKLIST modifier.
+     */
+
+    ASLR_SHARED_WORKLIST           = 0x200,
+    /* Process a worklist of modules to optionally produce and/or
+     * publish.  FIXME: Note that separate queues for publishing and
+     * producing may be needed.
+     */
+
+    ASLR_SHARED_INITIALIZE         = 0x1000,
+    /* Creates object directories if not yet created - needs to be
+     * done by first highly privileged process, although multiple ones
+     * attempting to do so is OK.  FIXME: TOFILE Security
+     * risk for privileged processes if directories and subdirectories
+     * can be created by non-privileged process, needs to make sure
+     * there is no race in which a low-privileged process creates
+     * entries before an older one.  See comments in
+     * nt_initialize_shared_directory() about required privileges if a
+     * permanent (until reboot) directory is to be created.
+     */
+
+    ASLR_SHARED_INITIALIZE_NONPERMANENT = 0x2000, 
+    /* make ASLR_SHARED_INITIALIZE initialize as a temporary object
+     * instead of the default permanent directory, useful for running
+     * a user process without proper permissions.  FIXME: may want to
+     * force it to use a per-user directory in this case.
+     */
+
+    ASLR_PERSISTENT                = 0x100000, /* NYI: namespace placeholder */
+    /* allow persistent copies to remain reusable possibly until
+     * reboot, or even longer.  Currently this is the default behavior
+     * for produced files.
+     */
+
+    /* non transparent option, not recommended */
+    ASLR_ALLOW_ORIGINAL_CLOBBER  = 0x1000000,
+    /* FIXME: as a feature case 9033 - would allow overwriting
+     * original files without anyone noticiing.  This would allow one
+     * to apply M$ patches without a reboot, and they would take
+     * effect for any other service using them.  However, it may
+     * prevent the use of M$ or third party tools to appropriately
+     * determine that application restarts are necessary.  xref case
+     * 8623 about the opposite problem of publishers keeping such
+     * handles even after all subscribers have been closed.
+     */
+    
+    /* non transparent option, not recommended */
+    ASLR_RANDOMIZE_EXECUTABLE    = 0x2000000,
+    /*
+     *   executables with relocations can now be randomized
+     *   from the parent process (especially on Vista). 
+     * case 8902 - however shows in taskmgr the name of our mangled file
+     *   we need to change our naming scheme to make this appear the same
+     * FIXME: to support !ASLR_ALLOW_ORIGINAL_CLOBBER 
+     * we also need to prevent overwrites of the executablewe by duplicating 
+     * our handle in the target process, so it gets closed when the child dies.
+     * 
+     */
+
+    /* case 9164: default off, may want if many temporary ASP.NET DLLs get created */
+    ASLR_AVOID_NET20_NATIVE_IMAGES = 0x4000000,
+
+    /* default in client mode currently should be 0x192 */
+    ASLR_CACHE_DEFAULT = ASLR_SHARED_PER_USER | /*  0x2 */
+                         ASLR_SHARED_CONTENTS | /* 0x10 */
+               ASLR_SHARED_ANONYMOUS_CONSUMER | /* 0x80 */
+                     ASLR_SHARED_FILE_PRODUCER /* 0x100 */
+} aslr_cache_t;
+
+/* ASLR cache coverage options for -aslr_cache_list */
+enum {
+    /* almost all controlled by other general ASLR exemptions  */
+    ASLR_CACHE_LIST_DEFAULT,
+    ASLR_CACHE_LIST_INCLUDE,    /* -aslr_cache_list_include */
+    ASLR_CACHE_LIST_EXCLUDE     /* -aslr_cache_list_exclude */
+    /* (note aslr_cache_list_t values match meaning of whitelist blacklist
+     * as in process_control) 
+     */
+} aslr_cache_list_t;
+
+enum {
+    /* consistency check flags for use with aslr_validation.  These
+     * are enforced by publishers when validating already produced
+     * files.  Note that with better security guarantees different
+     * levels of caching can be employed.  We currently demand that
+     * producers produce all supported metadata even if used only by
+     * some publishers, that may have different tradeoffs between
+     * security and/or performance objectives.
+     */
+
+    ASLR_PERSISTENT_LAX           = 0x0,
+    /* rely only on file size and matching magic
+     * insufficient for small patches,
+     * least consistency and no security 
+     */
+
+    ASLR_PERSISTENT_PARANOID      = 0x1,
+    /* provides strongest consistency and security: full byte
+     * comparison of original DLL and a provided relocated DLL 
+     * most expensive to evaluate
+     */
+
+    ASLR_PERSISTENT_SOURCE_DIGEST = 0x2,
+    /* provides source consistency and staleness check: MD5 digest of
+     * the source file, assuming reliable and trustworthy producer 
+     */
+
+    ASLR_PERSISTENT_TARGET_DIGEST = 0x4,
+    /* target corruption only: MD5 digest of the produced file,
+     * nonmalicious corruption only, otherwise no guarantee of the
+     * relationship between source and target */
+    /* FIXME: This one may not be very useful, but allows for a better
+     * check for corrupt files in case we can't guarantee atomicity,
+     * e.g. in case files are to be shared over the network. 
+     *
+     * Note that the combination of target digest and paranoid
+     * verification gives us somewhat more information than either
+     * one: an attacker trying to bypass an MD5 can always calculate
+     * the correct value, while a corrupted file will not have it.
+     * Interesting information only if we run processes with different
+     * privileged with differing level of validation.
+     */
+
+    ASLR_PERSISTENT_MODIFIED_TIME = 0x8, /* NYI */
+    /* FIXME: currently not possible to read the source file time.
+     * Otherwise would provide a weak staleness check only, no
+     * security: has potential for missing an update that preserves
+     * file times.  Also has some possibility of false positives in
+     * case say some antivirus program modifies a file or stores
+     * information past the end or in a separate stream of the
+     * original DLL.
+     *
+     * Note that FAT32 vs NTFS times differ so we store this in our
+     * own signature instead of trying to touch our file to match.
+     */
+
+    ASLR_PERSISTENT_NOTOWNER_PARANOIA = 0x10,
+    /* FIXME: NYI force paranoia in case file is not securely owned by
+     * the current user, e.g. could have never been overwritten by
+     * another user.  This allows us to share between users even in
+     * case files are produced by others, and skipping the check if we
+     * have safe private copies.
+     */
+
+    ASLR_PERSISTENT_SHORT_DIGESTS     = 0x20,
+    /* a qualifer on ASLR_PERSISTENT_SOURCE_DIGEST and
+     * ASLR_PERSISTENT_TARGET_DIGEST to use short as defined by
+     * aslr_short_digest.  Allows us to set a limit on verification time
+     * and working set impact, with acceptable consistency, but with
+     * an obvious security risk.
+     */
+
+    ASLR_PERSISTENT_PARANOID_TRANSFORM_EXPLICITLY = 0x10000, /* case 8858 */
+    /* When set we'll do the comparison by explicitly making a private
+     * copy just as a publisher, otherwise we compare in place
+     * relocation by relocation.  FIXME: this flag for
+     * ASLR_PERSISTENT_PARANOID should be internal, but for now
+     * leaving the old implementation for perf comparison, and added
+     * late in the game anyways.
+     */
+
+    /* FIXME: not recommended in production, should really make INTERNAL */
+    ASLR_PERSISTENT_PARANOID_PREFIX = 0x20000, 
+    /* This flag for ASLR_PERSISTENT_PARANOID similarly to
+     * ASLR_PERSISTENT_SHORT_DIGESTS makes it not really so paranoid.
+     * Allows to trade security and consistency risk for performance.
+     * 
+     * Cannot be used for security if target files are world writable -
+     * planting bad code for Administrator running explorer.exe is a
+     * bad enough elevation of privileges.
+     *
+     * FIXME: if in the future we do on-demand comparison, we may
+     * still want to first verify a prefix before we decide that the
+     * DLL is usable.
+     */
+} aslr_consistency_check_t;
+
+
+/* ASLR control flags for -aslr_action */
+enum {
+    ASLR_NO_ACTION      = 0x0,  /* do not track */
+
+    /* Track likely attempts to use preferred addresses */
+    ASLR_TRACK_AREAS    = 0x1,  /* keep track of would_be regions */
+    ASLR_AVOID_AREAS    = 0x2,  /* NYI disallow other DLL mappings */
+    ASLR_RESERVE_AREAS  = 0x4,  /* NYI virtually reserve to avoid any allocation */
+
+    /* Reporting exceptions */
+    /* intercept execute faults when run native, or RCT violations
+     * when targeting unreadable memory.
+     *
+     * FIXME: if areas are not reserved an RCT violation in a
+     * would_be area may also be attributed here.
+     */
+    ASLR_DETECT_EXECUTE = 0x10, 
+    /* FIXME: cannot reliably distinguish read from Execute when not
+     * enforcing DR security policies on a machine without NX */
+    ASLR_DETECT_READ    = 0x20, /* NYI */
+    ASLR_DETECT_WRITE   = 0x40, /* NYI */
+
+    /* report likely violations */
+    ASLR_REPORT         = 0x100, 
+    /* if not set, stays silent, yet detection worthwhile in
+     * combination with alternative handling where we'd kill an
+     * injected thread */
+
+    /* Alternative attack handling */
+    ASLR_HANDLING =     0x1000,/* NYI the default of throw_exception or kill_thread */
+    /* if not set an exception is simply passed to the application */
+
+    /* ThreatIDs can be normalized */
+    ASLR_NORMALIZE_ID = 0x10000, /* NYI use bytes at current mapping
+                                  * instead of would_be address */
+    /* as long as there are no relocations should maintain ThreatID */
+} aslr_action_t;
+
+/* testing and stress testing range flags for -aslr_internal */
+enum {
+    ASLR_INTERNAL_DEFAULT       = 0x0000, /* normal handling */
+
+    /* stress test option to verify proper dealing with address conflicts */
+    /* doesn't increment base, so most requests will overlap */
+    ASLR_INTERNAL_SAME_STRESS   = 0x1000, 
+
+    /* testing option - actually not choosing base, FIXME: remove soon */
+    ASLR_INTERNAL_RANGE_NONE    = 0x2000, 
+
+    ASLR_INTERNAL_SHARED_NONUNIQUE = 0x800000, 
+    /* stress test naming conflicts */
+
+    ASLR_INTERNAL_SHARED_APPFILE = 0x1000000, 
+    /* stress test - use application file to test sections */
+
+    ASLR_INTERNAL_SHARED_AND_PRIVATE = 0x2000000, 
+    /* stress test - use our files but still randomize privately as well */
+
+    /* Note that -exempt_aslr_list '*' can also be used as a stress
+     * option, instead of another flag here.
+     */
+} aslr_internal_option_t;
+
+typedef struct {
+    bool           sys_aslr_clobbered; 
+    /* mark syscalls modified by ASLR, and need additional handling */
+
+    /* ASLR_SHARED_CONTENTS needs to preserve some context across
+     * NtCreateSection and NtMapViewOfSection system calls
+     * xref case 9028 about using a more robust scheme that doesn't depend
+     * on these being consecutive
+     */
+    HANDLE randomized_section_handle; /* for shared randomization */
+    app_pc original_section_base; /* for detecting attacks */
+    uint   original_section_timestamp; /* for hotpatching */
+    uint   original_section_checksum; /* for hotpatching */
+    HANDLE original_image_section_handle;
+    /* used for !ASLR_ALLOW_ORIGINAL_CLOBBER to pass information from
+     * NtCreateSection to NtMapViewOfSection or NtCreateProcess* */
+
+    /* Case 1272: store short file name from section creation for use when
+     * module is mapped in.  These two fields are thus used for non-aslr
+     * purposes.  We will fail to keep the file name if there are
+     * non-properly-nested calls to create a section and map it (case 9028).
+     */
+    const char *last_section_file_name;
+    HANDLE last_app_section_handle; /* flagging unusual section handle uses */
+    /* e.g. a NtMapViewOfSection that doesn't immediately follow
+     * NtCreateSection or NtOpenSection  */
+
+    /* Case 9173: only pad each child once.  Rather than record every child seen
+     * (which has problems w/ pid reuse, as well as unbounded growth, as we
+     * won't see child death), we only record the previous one, leaving a corner
+     * case with alternate memory allocations to multiple pre-thread children
+     * causing us to pad multiply; likewise with separate threads each
+     * allocating in the same child.  We'll live with both risks.
+     */
+    process_id_t last_child_padded; /* pid */
+} aslr_syscall_context_t;
+
+#define ASLR_INVALID_SECTION_BASE ((app_pc)PTR_UINT_MINUS_1)
+
+/* names should look like kernel32.dll-12349783 */
+#define MAX_PUBLISHED_SECTION_NAME 64
+
+void aslr_init(void);
+void aslr_exit(void);
+void aslr_thread_init(dcontext_t *dcontext);
+void aslr_thread_exit(dcontext_t *dcontext);
+
+void aslr_free_last_section_file_name(dcontext_t *dcontext);
+
+void aslr_pre_process_mapview(dcontext_t *dcontext);
+void aslr_post_process_mapview(dcontext_t *dcontext);
+
+void aslr_pre_process_unmapview(dcontext_t *dcontext, app_pc base, size_t size);
+reg_t aslr_post_process_unmapview(dcontext_t *dcontext);
+
+void aslr_post_process_allocate_virtual_memory(dcontext_t *dcontext, 
+                                               app_pc base, size_t size);
+void aslr_pre_process_free_virtual_memory(dcontext_t *dcontext, 
+                                          app_pc base, size_t size);
+
+/* FIXME: wrap in aslr_post_process_create_or_open_section */
+bool
+aslr_post_process_create_section(IN HANDLE old_app_section_handle,
+                                 IN HANDLE file_handle,
+                                 OUT HANDLE *new_relocated_handle);
+
+bool
+aslr_post_process_create_or_open_section(dcontext_t *dcontext,
+                                         bool is_create,
+                                         IN HANDLE file_handle /* OPTIONAL */,
+                                         OUT HANDLE *sysarg_section_handle);
+bool
+aslr_is_handle_KnownDlls(HANDLE directory_handle);
+
+bool
+aslr_recreate_known_dll_file(OBJECT_ATTRIBUTES *obj_attr, 
+                             OUT HANDLE *recreated_file);
+
+void
+aslr_maybe_pad_stack(dcontext_t *dcontext, HANDLE process_handle);
+
+void
+aslr_force_dynamorio_rebase(HANDLE process_handle);
+
+/* deterministic (and reversible) timestamp transformation */
+static inline uint
+aslr_timestamp_transformation(uint old_timestamp)
+{
+    return old_timestamp + 1;
+}
+
+#ifdef GBOP
+/* Generically Bypassable Overflow Protection in user mode
+ * 
+ * For reference see 
+ * P. Szor, "Virus Research and Defense", Chapter 13, 13.3.1.1 and 13.3.4.2,
+ *   (skipping 13.2.6 on program shepherding and 13.3.4.1 ASLR)
+ * or "Bypassing 3rd Party Windows Buffer Overflow Protection" 
+ *     http://www.phrack.org/show.php?p=62&a=5 on how to break one
+ *
+ * FIXME: For interoperability purposes need to identify a
+ * compatibility mode that doesn't do more than competitor's desktop
+ * suite offerings, so their BOP functionality can be turned off.  Yet
+ * to avoid reversing them (although legally allowed for
+ * interoperability purposes), for now we'll do the best they may be
+ * doing.
+ */
+
+/* GBOP control flags for the -gbop option */
+enum {
+    GBOP_DISABLED          = 0x0,
+    /* source is identified as [TOS] for the first level,
+     * and if non-zero gbop_frames a stack backtrace is attempted.
+     *
+     * Note that providing FPO information (requires product updates
+     * for new versions) would allow deeper hooking and reliable
+     * backtraces.
+     */
+
+    /* source memory properties, note these are ORed together,
+     * at least one of the following set needs to be enabled */
+    GBOP_IS_EXECUTABLE     = 0x1, /* using our own tracking definitions */
+    GBOP_IS_X              = 0x2, /* allowing all ..X pages, cf -executable_if_x */
+    GBOP_IS_IMAGE          = 0x4, /* allowing all MEM_IMAGE pages, 
+                                   * c.f. -executable_if_image */
+    /* FIXME: another realistic policy would be to allow RWX as
+     * long as it is in an image, but not otherwise
+     */
+    /* FIXME: add GBOP_IS_NOT_W to allow as long as not writable */
+
+    GBOP_IS_NOT_STACK      = 0x8,
+    /* If set, allows returns to anything but the current stack.  Case 8085. */
+
+    /* see also GBOP_IS_FUTURE_EXEC = 0x04000 */
+
+
+    /* source instruction type.  Checks if instruction according to
+     * [TOS] is of valid type to have targeted the hook location.
+     * Note the allowed type checks are ORed. Evaluated only if source
+     * memory protection is satisfied.  See also stronger validation
+     * in GBOP_EMULATE_SOURCE.
+     */
+    GBOP_CHECK_INSTR_TYPE  = 0x10, /* no source instruction type restriction if not set */
+    GBOP_IS_CALL           = 0x20, /* verify source is at all a CALL instruction */
+    GBOP_IS_JMP            = 0x40, /* FIXME: not needed - app JMP won't be seen on TOS */
+    GBOP_IS_HOTPATCH_JMP   = 0x80, /* our JMP in case we have hotpatched purported source */
+    /* FIXME: we can't find the source if it was a JMP or JMP*, unless
+     * the caller explicitly wants to fool us with one.
+     *
+     * Note that in case of tail recursion elimination [TOS] is really
+     * of the caller of the previous function, should check that there
+     * are no internal XREFs to a hooked location, but we don't really
+     * care.
+     */
+
+    /* source instruction validation, if correct instruction type,
+     * check further whether it could have targeted our hook or it is
+     * used as chaff to bypass the previous simple validations.
+     */
+    GBOP_EMULATE_SOURCE   = 0x100, /* NYI */
+    /* We do know that this was the previous instruction and there is
+     * no reason it should have lost its state in normal operations.
+     * All registers can be restored to original expected state when
+     * hooking at entry point, only ESP on say call [esp+8] is the
+     * hardest where should revert ESP a little bit to get back to
+     * original state.  Of course call [esp-4] is impossible to
+     * recover since it will be overwritten, but not expected in real
+     * code.
+     *
+     * Needs to support CALL intrasegment to hook, CALL *eax or CALL
+     * *[IAT] intrasegment to hook, and CALL PLT -> JMP *[IAT] to hook.
+     * Other custom jump tables may get more messy.
+     */
+
+    /* target instruction check for simple ret2libc */
+    GBOP_IS_RET_TO_ENTRY  = 0x00200, /* NYI */
+
+    /* FIXME: See Szor's idea of checking whether PC=[ESP-4] as a
+     * check for RET_LIBC attack.  Is at all that safe to do - would
+     * there be a valid program doing PUSH target, RET targeting the
+     * exported routines, or simply remnants from a stack frame where
+     * the entry point is pushed (e.g. register spill).  Attacker
+     * could have used a RET 4 instr - then the API entry point is at
+     * [ESP-8] and can no longer be validated
+     */
+
+    /* FIXME: GBOP_WHEN_NATIVE_EXEC = 0x01000, NYI */
+    /* should this be applied when running in native exec mode - need
+     * to tell apart native_exec from hotp_only .NET, Java, maybe VB.
+     * Today -gbop is not by default, and recommended to use only for
+     * hotp_only.  This flag would be useful only when running in a
+     * future mix of -hotp_only and DR in the same process the two
+     * modes may need to differentiated.
+     */
+
+    GBOP_IS_DGC           = 0x02000,
+    /* If set, allows returns to heap (not stack) if a known vm has been loaded.
+     * native_exec today runs all vms/dgc natively; gbop for dgc involves 
+     * doing all the same bookkeepping to identify vms and uses -hotp_only 
+     * (which uses native_exec), so they are the same.  The bookkeepping may 
+     * have to be split up if native_exec's definition changes.  See case 8087.
+     * The main difference between just native_exec & gbop is that the former 
+     * is used only to run a dll/dgc natively, not when the control is in dr, 
+     * whereas in gbop it is used to avoid false positives for dgc.
+     */
+
+    GBOP_IS_FUTURE_EXEC   = 0x04000,
+    /* using our tracking definition for allowing a region as a
+     * futureexec
+     */
+
+    GBOP_DIAGNOSE_SOURCE  = 0x10000, /* NYI */
+    /* all enabled checks are evaluated to allow diagnosing all
+     * failures, and evaluate any disabled OR checks to provide
+     * recommendations to workaround a false positive based on a
+     * single run without staging.
+     */
+
+    /* FIXME: we may want to further control whether the hooked
+     * locations for other purposes do a GBOP check, or leave it only
+     * to the 'extra' hooks from gbop_include_list */
+
+    /* default in client mode currently should be 0x6037 */
+    GBOP_CLIENT_DEFAULT = GBOP_IS_DGC | GBOP_IS_FUTURE_EXEC |         /* 0x06000 */ 
+                          GBOP_CHECK_INSTR_TYPE | GBOP_IS_CALL |         /* 0x30 */
+                          GBOP_IS_EXECUTABLE | GBOP_IS_X | GBOP_IS_IMAGE /*  0x7 */
+};
+
+extern bool gbop_vm_loaded;
+
+typedef struct {
+    const char *mod_name;
+    const char *func_name;
+} gbop_hook_desc_t;
+
+const gbop_hook_desc_t *
+gbop_get_hook(uint hook_index);
+uint
+gbop_get_num_hooks(void);
+
+bool
+gbop_exclude_filter(const gbop_hook_desc_t *gbop_hook);
+
+bool
+gbop_check_valid_caller(app_pc reg_ebp, app_pc reg_esp, app_pc cur_pc,
+                        app_pc *violating_source_addr /* OUT */);
+void
+gbop_validate_and_act(app_state_at_intercept_t *state, 
+                      byte fpo_adjustment,
+                      app_pc hooked_target);
+#endif /* GBOP */
+
+#endif /* ASLR_H */
diff --git a/core/win32/callback.c b/core/win32/callback.c
new file mode 100644
index 000000000..c4a0b5af9
--- /dev/null
+++ b/core/win32/callback.c
@@ -0,0 +1,7465 @@
+/* **********************************************************
+ * Copyright (c) 2002-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+/*
+ * callback.c - windows-specific callback, APC, and exception handling routines
+ */
+
+/* This whole file assumes x86 */
+#ifndef X86
+#error X86 must be defined
+#endif
+
+#include "globals.h"
+#include "arch.h"
+#include "instr.h"
+#include "decode.h"
+#include "monitor.h"
+#include "fcache.h"
+#include "fragment.h"
+#include "decode_fast.h"
+#include "disassemble.h"
+#include "instr_create.h"
+#include "ntdll.h"
+#include "events.h"
+#include "os_private.h"
+#include "moduledb.h"
+#include "aslr.h"
+#include "nudge.h" /* for generic_nudge_target() address */
+#ifdef RETURN_AFTER_CALL
+# include "rct.h" /* for rct_ind_branch_target_lookup */
+#endif
+#include "instrument.h"
+#include "perscache.h"
+
+#include <string.h> /* for memcpy */
+#include <windows.h>
+
+/* forward declarations */
+static dcontext_t * callback_setup(app_pc next_pc);
+static byte * insert_image_entry_trampoline(dcontext_t *dcontext);
+static void swap_dcontexts(dcontext_t *done, dcontext_t *dtwo);
+static void asynch_take_over(app_state_at_intercept_t *state);
+
+/* currently we do not intercept top level exceptions */
+#ifdef INTERCEPT_TOP_LEVEL_EXCEPTIONS
+/* the app's top-level exception handler */
+static LPTOP_LEVEL_EXCEPTION_FILTER app_top_handler;
+#endif
+
+/* interception information 
+ * if it weren't for syscall trampolines this could be a single page
+ * Note: if you add more intercept points, make sure to adjust
+ * NUM_INTERCEPT_POINTS below.
+ */
+static byte * interception_code = NULL;
+static byte * interception_cur_pc = NULL;
+static byte * ldr_init_pc = NULL;
+static byte * callback_pc = NULL;
+static byte * apc_pc = NULL;
+static byte * exception_pc = NULL;
+static byte * raise_exception_pc = NULL;
+static byte * after_callback_orig_pc = NULL;
+static byte * after_apc_orig_pc = NULL;
+static byte * load_dll_pc = NULL;
+static byte * unload_dll_pc = NULL;
+static byte * image_entry_pc = NULL;
+static byte * image_entry_trampoline = NULL;
+static byte * syscall_trampolines_start = NULL;
+static byte * syscall_trampolines_end = NULL;
+
+/* For detach */
+bool init_apc_go_native = false;
+bool init_apc_go_native_pause = false;
+
+/* overridden by dr_preinjected, or retakeover_after_native() */
+static retakeover_point_t interception_point = INTERCEPT_PREINJECT;
+
+/* While emiting the trampoline, the alt. target is unknown for hotp_only. */
+#define CURRENTLY_UNKNOWN ((byte *)(ptr_uint_t) 0xdeadc0de)
+
+#ifdef DEBUG
+#define INTERCEPT_POINT(point) STRINGIFY(point),
+static const char * const retakeover_names[] = {
+    INTERCEPT_ALL_POINTS
+};
+#undef INTERCEPT_POINT
+#endif
+
+/* We keep a list of mappings from intercept points to original app PCs */
+typedef struct _intercept_map_elem_t {
+    byte *interception_pc;
+    app_pc original_app_pc;
+    size_t displace_length;
+    struct _intercept_map_elem_t *next;
+} intercept_map_elem_t;
+
+typedef struct _intercept_map_t {
+    intercept_map_elem_t *head;
+    intercept_map_elem_t *tail;
+} intercept_map_t;
+
+static intercept_map_t *intercept_map;
+
+DECLARE_CXTSWPROT_VAR(static mutex_t map_intercept_pc_lock,
+                      INIT_LOCK_FREE(map_intercept_pc_lock));
+
+DECLARE_CXTSWPROT_VAR(static mutex_t emulate_write_lock,
+                      INIT_LOCK_FREE(emulate_write_lock));
+
+#ifdef STACK_GUARD_PAGE
+DECLARE_CXTSWPROT_VAR(static mutex_t exception_stack_lock,
+                      INIT_LOCK_FREE(exception_stack_lock));
+#endif
+
+/* Only used for Vista, new threads start directly here instead of going 
+ * through KiUserApcDispatcher first. Isn't in our lib (though is exported
+ * on 2k, xp and vista at least) so we get it dynamically. */
+static byte *LdrInitializeThunk = NULL;
+/* On vista this is the address the kernel sets (via NtCreateThreadEx, used by all the
+ * api routines) as Xip in the context the LdrInitializeThunk NtContinue's to (is eqv.
+ * to the unexported kernel32!Base[Process,Thread]StartThunk in pre-Vista).  Fortunately
+ * ntdll!RtlUserThreadStart is exported and we cache it here for use in
+ * intercept_new_thread().  Note that threads created by the legacy native
+ * NtCreateThread don't have to target this address. */
+static byte *RtlUserThreadStart = NULL;
+
+static inline app_pc
+get_setcontext_interceptor()
+{
+    return (app_pc) nt_continue_dynamo_start;
+}
+
+/* if tid != self, must hold thread_initexit_lock */
+void
+set_asynch_interception(thread_id_t tid, bool intercept)
+{
+    /* Needed to turn on and off asynchronous event interception
+     * for non-entire-application-under-dynamo-control situations
+     */
+    thread_record_t *tr = thread_lookup(tid);
+    ASSERT(tr != NULL);
+    tr->under_dynamo_control = intercept;
+}
+
+static inline bool
+intercept_asynch_global()
+{
+    return (intercept_asynch && !INTERNAL_OPTION(nullcalls));
+}
+
+/* if tr is not for calling thread, must hold thread_initexit_lock */
+static bool
+intercept_asynch_common(thread_record_t *tr, bool intercept_unknown)
+{
+    if (!intercept_asynch_global())
+        return false;
+    if (tr == NULL) {
+        if (intercept_unknown)
+            return true;
+        /* caller should have made all attempts to get tr */
+        if (control_all_threads) {
+            /* we should know about all threads! */
+            SYSLOG_INTERNAL_WARNING("Received asynch event for unknown thread %d", get_thread_id());
+            /* try to make everything run rather than assert -- just do
+             * this asynch natively, we probably received it for a thread that's
+             * been created but not scheduled?  
+             */
+        }
+        return false;
+    }
+    /* FIXME: under_dynamo_control should be an enum w/ separate
+     * values for 1) truly native, 2) under DR but currently native_exec,
+     * 3) temporarily native b/c DR lost control (== UNDER_DYN_HACK), and
+     * 4) fully under DR
+     */
+    DOSTATS({
+        if (tr->under_dynamo_control == UNDER_DYN_HACK)
+            STATS_INC(num_asynch_while_lost);
+    });
+    return (tr->under_dynamo_control);
+}
+
+/* if tid != self, must hold thread_initexit_lock */
+bool
+intercept_asynch_for_thread(thread_id_t tid, bool intercept_unknown)
+{
+    /* Needed to turn on and off asynchronous event interception
+     * for non-entire-application-under-dynamo-control situations
+     */
+    thread_record_t *tr = thread_lookup(tid);
+    return intercept_asynch_common(tr, intercept_unknown);
+}
+
+bool
+intercept_asynch_for_self(bool intercept_unknown)
+{
+    /* To avoid problems with the all_threads_lock required to look 
+     * up a thread in the thread table, we first see if it has a
+     * dcontext, and if so we get the thread_record_t from there.
+     * If not, it probably is a native thread and grabbing the lock
+     * should cause no problems as it should not currently be holding
+     * any locks.
+     */
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    if (dcontext != NULL)
+        return intercept_asynch_common(dcontext->thread_record, intercept_unknown);
+    else
+        return intercept_asynch_for_thread(get_thread_id(), intercept_unknown);
+}
+
+
+/***************************************************************************
+ * INTERCEPTION CODE FOR TRAMPOLINES INSERTED INTO APPLICATION CODE
+
+interception code either assumes that the app's xsp is valid, or uses
+dstack if available, or as a last resort uses initstack.  when using
+initstack, must make sure all paths exiting handler routine clear the
+initstack mutex once not using the initstack itself!
+
+We clobber TIB->PID, which is believed to be safe since no user-mode
+code will execute there b/c thread is not alertable, and the kernel
+shouldn't be reading (and trusting) user mode TIB structures.
+FIXME: allocate and use a TIB scratch slot instead
+
+N.B.: this interception code, if encountered by DR, is let run natively,
+so make sure DR takes control at the end!
+
+For trying to use the dstack, we have to be careful and check if we're
+already on the dstack, which can happen for internal exceptions --
+hopefully not for callbacks or apcs, we should assert on that =>
+FIXME: add such checks to the cb and apc handlers, and split dstack
+check as a separate parameter, once we make cbs and apcs not
+assume_xsp (they still do for now since we haven't tested enough to
+convince ourselves we never get them while on the dstack)
+
+Unfortunately there's no easy way to check w/o modifying flags, so for
+now we assume eflags whenever we do not assume xsp.
+Assumption should be ok for Ki*, also for Ldr*.
+
+Alternatives: check later when we're in exception handler, only paths
+there are terminate or forge exception.  Thus we can get away w/o
+reading anything on stack placed by kernel, but we won't have clean
+call stack or anything else for diagnostics, and we'll have clobbered
+the real xsp in the mcontext slot, which we use for forging the
+exception.
+
+Could perhaps use whereami==WHERE_FCACHE, but could get exception during
+clean call or cxt switch when on dstack but prior to whereami change.
+
+Note: the app registers passed to the handler are restored when going back to
+      the app, which means any changes made by the handler will be reflected
+      in the app state; 
+      FIXME: change handler prototype to make all registers volatile so that the
+      compiler doesn't clobber them; for now it is the user's responsibility.
+
+if (!assume_xsp)
+      mov xcx, fs:$PID_TIB_OFFSET  # save xcx
+      mov fs:$TLS_DCONTEXT_OFFSET, xcx
+      jecxz no_local_stack
+ if (!assume_not_on_dstack)
+      # need to check if already on dstack
+      # assumes eflags!
+      mov $DSTACK(xcx), xcx
+      cmp xsp, xcx
+      jge not_on_dstack
+      lea -DYNAMORIO_STACK_SIZE(xcx), xcx
+      cmp xsp, xcx
+      jl  not_on_dstack
+      # record stack method: using dstack/initstack unmodified
+      push xsp
+      push $2
+      jmp have_stack_now
+  not_on_dstack:
+      mov fs:$TLS_DCONTEXT_OFFSET, xcx
+ endif
+      # store app xsp in dcontext & switch to dstack; this will be used to save
+      # app xsp on the switched stack, i.e., dstack; not used after that.
+    if TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)
+      mov $MCONTEXT_OFFSET(xcx), xcx
+    endif
+      mov xsp, $XSP_OFFSET(xcx)
+    if TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)
+      mov fs:$TLS_DCONTEXT_OFFSET, xcx
+    endif
+      mov $DSTACK(xcx), xsp
+
+      # now get the app xsp from the dcontext and put it on the dstack; this 
+      # will serve as the app xsp cache and will be used to send the correct 
+      # app xsp to the handler and to restore app xsp at exit
+    if TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)
+      mov $MCONTEXT_OFFSET(xcx), xcx
+    endif
+      mov $XSP_OFFSET(xcx), xcx
+      push xcx
+
+      # need to record stack method, since dcontext could change in handler
+      push $1
+      jmp have_stack_now
+  no_local_stack:
+      # use initstack -- it's a global synch, but should only have no
+      # dcontext for initializing thread (where we actually use the app's
+      # stack) or exiting thread
+      # If we are already on the initstack, should just continue to use it.
+      # need to check if already on initstack
+      # assumes eflags, but we already did on this path for checking dstack
+      mov $INITSTACK, xcx
+      cmp xsp, xcx
+      jge grab_initstack
+      lea -DYNAMORIO_STACK_SIZE(xcx), xcx
+      cmp xsp, xcx
+      jl  grab_initstack
+      push xsp
+      # record stack method: using dstack/initstack unmodified
+      push $2
+      jmp have_stack_now
+    grab_initstack:
+      mov $1, ecx # upper 32 bits zeroed on x64
+     if x64 # can't directly address initstack_mutex or initstack_app_xsp
+            # (though we could use rip-relative, nice to not have reachability issues
+            # if located far from dynamorio.dll, for general hooks (PR 250294)!)
+      # if a new thread we can't easily (w/o syscall) replace tid, so we use peb
+      mov xax, fs:$PEB_TIB_OFFSET  # save xax
+     endif
+    get_lock:
+     if x64 # can't directly address initstack_mutex or initstack_app_xsp
+      mov $INITSTACK_MUTEX, xax
+     endif
+      # initstack_mutex.lock_requests is 32-bits, but on x64 we're not xchg-ing w/ 
+      #   memory, so ok
+      xchg xcx, IF_X64_ELSE(xax, initstack_mutex)
+      jecxz have_lock
+      pause  # improve spin-loop perf on P4
+      jmp get_lock # no way to sleep or anything, must spin
+    have_lock:
+      # app xsp is saved in initstack_app_xsp only so that it can be accessed after
+      # switching to initstack; used only to set up the app xsp on the initstack
+     if x64 # we don't need to set initstack_app_xsp, just push the app xsp value
+      mov xsp, xcx
+      mov initstack, xax
+      xchg xax, xsp
+      push xcx
+     else
+      mov xsp, initstack_app_xsp
+      mov initstack, xsp
+      push initstack_app_xsp
+     endif
+      # need to record stack method, since dcontext could change in handler
+      push $0
+     if x64
+      mov $peb_ptr, xax
+      xchg fs:$PEB_TIB_OFFSET, xax  # restore xax and peb ptr
+     endif
+  have_stack_now:
+     if x64
+      mov $global_pid, xcx
+      xchg fs:$PID_TIB_OFFSET, xcx  # restore xcx and pid
+     else
+      mov fs:$PID_TIB_OFFSET, xcx  # restore xcx
+      mov $global_pid, fs:$PID_TIB_OFFSET  # restore TIB PID
+     endif
+else
+      push xsp # cache app xsp so that it can be used to send the right value
+               # to the handler and to restore app xsp safely at exit
+      push $3  # recording stack type when using app stack
+endif
+      # we assume here that we've done two pushes on the stack,
+      # which combined w/ the push0 and pushf give us 16-byte alignment
+      # for 32-bit and 64-bit prior to push-all-regs
+  clean_call_setup:
+      # lay out pc, eflags, regs, etc. in app_state_at_intercept_t order
+      push $0 # pc slot; unused; could use instead of state->start_pc
+      pushf
+      pusha (or push all regs for x64)
+      push $0 # ASSUMPTION: clearing, not preserving, is good enough
+              # FIXME: this won't work at CPL0 if we ever run there!
+      popf
+
+      # get the cached app xsp and write it to pusha location,
+      # so that the handler gets the correct app xsp
+      mov sizeof(dr_mcontext_t)+XSP_SZ(xsp), xax
+      mov xax, offsetof(dr_mcontext_t, xsp)(xsp)
+
+      # the 4 bytes of x64 app_state_at_intercept_t padding are
+      # handled by the 8-byte push of eax
+      mov fs:$ERRNO_TIB_OFFSET, eax  # instead of call GetLastError
+      push xax
+if (ENTER_DR_HOOK != NULL)
+      call ENTER_DR_HOOK
+endif
+    if x64
+      mov no_cleanup, xax
+      push xax
+      mov handler_arg, xax
+      push xax
+    else
+      push no_cleanup
+      push handler_arg
+    endif
+      # now we've laid out app_state_at_intercept_t on the stack
+      push/mov xsp      # a pointer to the pushed values; this is the argument;
+                        # see case 7597.  may be passed in a register.
+      call handler
+      <clean up args>
+      lea 2*XSP_SZ(xsp), lea # pop handler_arg + no_cleanup
+if (AFTER_INTERCEPT_DYNAMIC_DECISION)
+      cmp xax, AFTER_INTERCEPT_LET_GO
+      je let_go
+ if (alternate target provided)
+      cmp xax, AFTER_INTERCEPT_LET_GO_ALT_DYN
+      je let_go_alt
+ endif
+endif
+if (AFTER_INTERCEPT_TAKE_OVER || AFTER_INTERCEPT_TAKE_OVER_SINGLE_SHOT
+      || AFTER_INTERCEPT_DYNAMIC_DECISION)
+    if x64
+      mov no_cleanup, xax
+      push xax
+    else
+      push no_cleanup # state->start_pc
+    endif
+      push $0 # we assume always want !save_dcontext as arg to asynch_take_over
+      push/mov xsp # app_state_at_intercept_t *
+      call asynch_take_over
+      # should never reach here
+      push $0
+      push $-3  # internal_error will report -3 as line number
+      push $0
+      call internal_error
+endif
+if (AFTER_INTERCEPT_DYNAMIC_DECISION && alternate target provided)
+  let_go_alt:
+      <complete duplicate of let_go, but ending in a jmp to alternate target>
+      <(cannot really share much of let_go cleanup w/o more scratch slots)>
+      <(has to be first since original app instrs are placed by caller, not us)>
+endif
+if (!AFTER_INTERCEPT_TAKE_OVER)
+  let_go:
+ if (EXIT_DR_HOOK != NULL)
+      call EXIT_DR_HOOK
+ endif
+      pop xax # retrieve last error
+      mov eax, fs:$ERRNO_TIB_OFFSET # instead of call SetLastError
+      # the 4 bytes of x64 app_state_at_intercept_t padding are
+      # handled by the 8-byte pop of eax
+
+      # get the xsp passed to the handler, which may have been
+      # changed; store it in the xsp cache to restore at exit
+      mov offsetof(dr_mcontext_t, xsp)(xsp), xax
+      mov xax, sizeof(dr_mcontext_t)+XSP_SZ(xsp)
+      popa # or pop all regs on x64
+      popf
+      lea XSP_SZ(xsp), xsp # clear pc slot
+ if (!assume_xsp)
+      mov xcx, fs:$PID_TIB_OFFSET  # save xcx
+      pop xcx # get back const telling stack used
+      pop xsp
+      jecxz restore_initstack
+      jmp done_restoring
+  restore_initstack:
+    if x64
+      mov &initstack_mutex, xcx
+      mov $0, (xcx)
+    else
+      mov $0, initstack_mutex
+    endif
+  done_restoring:
+    if x64
+      mov $global_pid, xcx
+      xchg fs:$PID_TIB_OFFSET, xcx  # restore xcx and pid
+    else
+      mov fs:$PID_TIB_OFFSET, xcx  # restore xcx
+      mov $global_pid, fs:$PID_TIB_OFFSET  # restore TIB PID
+    endif
+ else
+      lea XSP_SZ(xsp), xsp       # clear out the stack type
+      pop xsp  # handler may have changed xsp; so get it from the xsp cache
+ endif
+endif (!AFTER_INTERCEPT_TAKE_OVER)
+  no_cleanup:
+      <original app instructions>
+
+=> handler signature, exported as typedef intercept_function_t:
+void handler(void *callee_arg, app_pc start_pc, int app_errno, reg_t eflags, regs...)
+
+if AFTER_INTERCEPT_TAKE_OVER, then asynch_take_over is called, with "false" for
+its save_dcontext parameter
+
+handler must make sure all paths exiting handler routine clear the
+initstack mutex once not using the initstack itself!
+
+*/
+
+#define APP  instrlist_append
+
+/* common routine separate since used for let go and alternate let go */
+static void
+insert_let_go_cleanup(dcontext_t *dcontext, instrlist_t *ilist, instr_t *decision,
+                      bool assume_xsp, bool assume_not_on_dstack,
+                      after_intercept_action_t action_after)
+{
+    instr_t *first = NULL;
+    if (action_after == AFTER_INTERCEPT_DYNAMIC_DECISION) {
+        /* placeholder so can find 1st of this path */
+        first = instrlist_last(ilist);
+    }
+    
+    if (EXIT_DR_HOOK != NULL) {
+        /* make sure to use dr_insert_call() rather than a raw OP_call instr,
+         * since x64 windows requires 32 bytes of stack space even w/ no args.
+         */
+        dr_insert_call((void *)dcontext, ilist, NULL/*append*/, (void *)EXIT_DR_HOOK, 0);
+    }
+
+    /* must preserve the LastErrorCode (if call a Win32 API routine could
+     * overwrite the app's error code) */
+    /* saved error code is currently on the top of the stack */
+    APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_XAX)));
+    /* for x64, errno is still 32 bits */
+    preinsert_set_last_error(dcontext, ilist, NULL, REG_EAX);
+    /* the 4 bytes of x64 app_state_at_intercept_t padding are
+     * handled by the 8-byte pop of eax */
+
+    /* Get the app xsp passed to the handler from the popa location and store
+     * it in the app xsp cache; this is because the handler could have changed
+     * the app xsp that was passed to it.  CAUTION: do this before the popa.
+     */
+    APP(ilist, INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XAX),
+                                   OPND_CREATE_MEMPTR(REG_XSP,
+                                                      offsetof(dr_mcontext_t, xsp))));
+    APP(ilist, INSTR_CREATE_mov_st(dcontext,
+                                   OPND_CREATE_MEMPTR(REG_XSP,
+                                                      sizeof(dr_mcontext_t)+XSP_SZ),
+                                   opnd_create_reg(REG_XAX)));
+    /* now restore everything */
+    insert_pop_all_registers(dcontext, ilist, NULL, false/*see push_all use*/);
+    APP(ilist, INSTR_CREATE_RAW_popf(dcontext));
+    APP(ilist, INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XSP), 
+                                opnd_create_base_disp(REG_XSP, REG_NULL, 0,
+                                                      XSP_SZ, OPSZ_lea)));
+        
+    if (action_after == AFTER_INTERCEPT_DYNAMIC_DECISION) {
+        /* now that instrs are there, take 1st */
+        ASSERT(first != NULL);
+        instr_set_target(decision, opnd_create_instr(instr_get_next(first)));
+    }
+
+    if (!assume_xsp) {
+        instr_t *restore_initstack = INSTR_CREATE_label(dcontext);
+        instr_t *done_restoring = INSTR_CREATE_label(dcontext);
+        APP(ilist,
+            INSTR_CREATE_mov_st(dcontext,
+                                opnd_create_far_base_disp(SEG_TLS, REG_NULL, REG_NULL,
+                                                          0, PID_TIB_OFFSET, OPSZ_PTR),
+                                opnd_create_reg(REG_XCX)));
+        APP(ilist,
+            INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_XCX)));
+
+        /* popa doesn't restore xsp; the handler might have changed it, so 
+         * restore it from the app xsp cache, which is now the top of stack.
+         */
+        APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_XSP)));
+        APP(ilist, INSTR_CREATE_jecxz(dcontext, opnd_create_instr(restore_initstack)));
+        APP(ilist,
+            INSTR_CREATE_jmp(dcontext, opnd_create_instr(done_restoring)));
+        /* use initstack to avoid any assumptions about app xsp */
+        APP(ilist, restore_initstack);
+#ifdef X64
+        APP(ilist,
+            INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XCX),
+                                 OPND_CREATE_INTPTR((ptr_uint_t)&initstack_mutex)));
+#endif
+        APP(ilist,
+            INSTR_CREATE_mov_st(dcontext,
+                                IF_X64_ELSE(OPND_CREATE_MEM32(REG_XCX, 0),
+                                            OPND_CREATE_ABSMEM((void *)&initstack_mutex,
+                                                               OPSZ_4)),
+                                OPND_CREATE_INT32(0)));
+        APP(ilist, done_restoring);
+#ifdef X64
+        /* we could perhaps assume the top 32 bits of win32_pid are zero, but
+         * xchg works just as well */
+        APP(ilist,
+            INSTR_CREATE_mov_imm(dcontext,
+                                 opnd_create_reg(REG_XCX),
+                                 OPND_CREATE_INTPTR((ptr_uint_t)win32_pid)));
+        APP(ilist,
+            INSTR_CREATE_xchg(dcontext,
+                              opnd_create_far_base_disp(SEG_TLS, REG_NULL, REG_NULL,
+                                                        0, PID_TIB_OFFSET, OPSZ_PTR),
+                              opnd_create_reg(REG_XCX)));
+#else
+        APP(ilist,
+            INSTR_CREATE_mov_ld(dcontext,
+                                opnd_create_reg(REG_XCX),
+                                opnd_create_far_base_disp(SEG_TLS, REG_NULL, REG_NULL,
+                                                          0, PID_TIB_OFFSET, OPSZ_PTR)));
+        APP(ilist,
+            INSTR_CREATE_mov_st(dcontext,
+                                opnd_create_far_base_disp(SEG_TLS, REG_NULL, REG_NULL,
+                                                          0, PID_TIB_OFFSET, OPSZ_PTR),
+                                OPND_CREATE_INTPTR(win32_pid)));
+#endif
+    } else {
+        /* popa doesn't restore xsp; the handler might have changed it, so 
+         * restore it from the app xsp cache, which is now the top of stack.
+         */
+        APP(ilist, INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XSP), 
+                                    opnd_create_base_disp(REG_XSP, REG_NULL, 0,
+                                                          XSP_SZ, OPSZ_0)));
+        APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_XSP)));
+    }
+}
+
+/* Emits a landing pad (shown below) and returns the address to the first
+ * instruction in it.
+ *
+ *  CAUTION: These landing pad layouts are assumed in intercept_call() and in
+ *           read_and_verify_dr_marker().
+ *ifndef X64
+ *  32-bit landing pad:
+ *      jmp tgt_pc          ; 5 bytes, 32-bit relative jump
+ *      jmp after_hook_pc   ; 5 bytes, 32-bit relative jump
+ *else
+ *  64-bit landing pad:
+ *      tgt_pc              ; 8 bytes of absolute address, i.e., tgt_pc
+ *      jmp [tgt_pc]        ; 6 bytes, 64-bit absolute indirect jmp
+ *      jmp after_hook_pc   ; 5 bytes, 32-bit relative jump
+ *endif
+ *
+ * Note: For 64-bit landing pad, tgt_pc can be stored at the bottom of the
+ * trampoline too.  I chose the top because it helps avoid a minor reachability
+ * problem: iff the landing pad is allocated at the topmost part of the
+ * reachability region for a given addr_to_hook, then there is a possiblity
+ * that the return jmp from the landing pad may not reach the instruction after
+ * the hook address.  This is because the number of bytes of the hook (5 bytes)
+ * and the number of bytes of the instruction(s) clobbered at the hook point
+ * might be different.  If the clobbered bytes are more than 5 bytes, then the
+ * return jmp from the landing pad won't be able to reach it.  By placing 8
+ * bytes above the landing pad, we give it the extra reachability needed.   
+ * Also, having the tgt_pc at the top of the landing pad makes it easy to see
+ * the disassembly of the whole landing pad while debugging, else there will be
+ * jmp and garbage after it.
+ *
+ * This isn't a problem for 32-bit landing pad because in 32-bit everything is
+ * reachable.
+ *
+ * See heap.c for details about what landing pads are.
+ */
+#define JMP_SIZE (IF_X64_ELSE(JMP_ABS_IND64_SIZE, JMP_REL32_SIZE))
+static byte *
+emit_landing_pad_code(byte *lpad_buf, const byte *tgt_pc,
+                      const byte *after_hook_pc)
+{
+    byte *lpad_entry = lpad_buf;
+    bool changed_prot, res;
+    DEBUG_DECLARE(byte *lpad_start = lpad_buf;)
+    ASSERT(lpad_buf != NULL);
+
+    res = make_hookable(lpad_buf, LANDING_PAD_SIZE, &changed_prot);
+    ASSERT(res);
+    
+#ifndef X64
+    *lpad_buf = JMP_REL32_OPCODE;
+    lpad_buf++;
+    *((int *)lpad_buf) = (int)(tgt_pc - lpad_buf - 4);
+    lpad_buf += 4;
+#else
+    *((byte **)lpad_buf) = (byte *)tgt_pc; /* save tgt_pc for the rip-rel jmp */
+    lpad_buf += sizeof(tgt_pc);
+    lpad_entry = lpad_buf;              /* entry is after the first 8 bytes */
+    *lpad_buf = JMP_ABS_IND64_OPCODE;
+    lpad_buf++;
+    *lpad_buf = JMP_ABS_MEM_IND64_MODRM;
+    lpad_buf++;
+    /* rip relative address to 8-bytes, i.e., start of lpad_buf */
+    *((int *)lpad_buf) = -(int)(JMP_ABS_IND64_SIZE + sizeof(tgt_pc));
+    lpad_buf += 4;
+#endif
+
+    /* The return 32-bit relative jump is common to both 32-bit and 64-bit
+     * landing pads.  Make sure that the second jmp goes into the right address.
+     */
+    ASSERT(lpad_buf - lpad_start == JMP_SIZE IF_X64(+ sizeof(tgt_pc)));
+    *lpad_buf = JMP_REL32_OPCODE;
+    lpad_buf++;
+    *((int *)lpad_buf) = (int)(after_hook_pc - lpad_buf - 4);
+    lpad_buf += 4;
+
+    /* Even though we have the 8 byte space up front for 64-bit, just make sure
+     * that the return jmp can reach the instruction after the hook.
+     */
+    ASSERT(REL32_REACHABLE(lpad_buf, after_hook_pc));
+
+    /* Make sure that the landing pad size match with definitions. */
+    ASSERT(lpad_buf - lpad_start == LANDING_PAD_SIZE);
+
+    make_unhookable(lpad_buf, LANDING_PAD_SIZE, changed_prot);
+    return lpad_entry;
+}
+
+/* Assumes that ilist contains decoded instrs for [start_pc, start_pc+size).
+ * Copies size bytes of the app code at start_pc into buf by encoding
+ * the ilist, re-relativizing rip-relative and ctis as it goes along.
+ * Also converts short ctis into 32-bit-offset ctis.
+ *
+ * hotp_only does not support ctis in the middle of the ilist, only at
+ * the end, nor size changes in the middle of the ilist: to support
+ * that we'd need a relocation table mapping old instruction offsets
+ * to the newly emitted ones.
+ *
+ *      As of today only one cti is allowed in a patch region and that too at 
+ * the end of it, so the starting location of that cti won't change even if we 
+ * convert and re-relativize it.  This means hot patch control flow changes into
+ * the middle of a patch region won't have to worry about using an offset table.
+ * 
+ *      The current patch region definition doesn't allow ctis to be in the 
+ * middle of patch regions.  This means we don't have to worry about 
+ * re-relativizing ctis in the middle of a patch region.  However Alex has an 
+ * argument about allowing cbrs to be completely inside a patch region as 
+ * control flow can never reach the following instruction other than fall 
+ * through, i.e., not from outside.  This is a matter for debate, but one 
+ * which will need the ilist & creating the relocation table per patch point.
+ */
+static byte *
+copy_app_code(dcontext_t *dcontext, const byte *start_pc, 
+              byte *buf, size_t size, instrlist_t *ilist)
+{
+    instr_t *instr;
+    byte *buf_nxt;
+    DEBUG_DECLARE(byte *buf_start = buf;)
+    DEBUG_DECLARE(bool size_change = false;)
+    ASSERT(dcontext != NULL && start_pc != NULL && buf != NULL);
+    /* Patch region should be at least 5 bytes in length, but no more than 5 
+     * plus the length of the last instruction in the region.
+     */
+    ASSERT(size >= 5 && size <
+           (size_t)(5 + instr_length(dcontext, instrlist_last(ilist))));
+
+    /* We have to walk the instr list to lengthen short (8-bit) ctis */
+    for (instr = instrlist_first(ilist); instr != NULL; instr = instr_get_next(instr)) {
+        /* For short ctis in the loop to jecxz range, the cti conversion 
+         * will set the target in the raw bits, so the raw bits will be valid.  
+         * For other short ctis, the conversion will invalidate the raw bits, 
+         * so a full encoding is enforced.  For other ctis, the raw bits aren't 
+         * valid for encoding because we are relocating them; so invalidate
+         * them explicitly.
+         */
+        if (instr_opcode_valid(instr) && instr_is_cti(instr)) {
+            if (instr_is_cti_short(instr)) {
+                DODEBUG({ size_change = true; });
+                convert_to_near_rel(dcontext, instr);
+            } else
+                instr_set_raw_bits_valid(instr, false);
+            /* see notes above: hotp_only doesn't support non-final cti */
+            ASSERT(!instr_is_cti(instr) || instr == instrlist_last(ilist));
+        }
+#ifdef X64
+        /* If we have reachability issues, instrlist_encode() below
+         * will fail.  We try to do an assert here for that case
+         * (estimating where the relative offset will be encoded at).
+         * PR 250294's heap pad proposal will solve this.
+         */
+        DODEBUG({
+            app_pc target;
+            instr_get_rel_addr_target(instr, &target);
+            ASSERT_NOT_IMPLEMENTED
+                ((!instr_has_rel_addr_reference(instr) ||
+                  REL32_REACHABLE(buf, target)) &&
+                 "PR 250294: displaced code too far from rip-rel target");
+        });
+#endif
+    }
+
+    /* now encode and re-relativize x64 rip-relative instructions */
+    buf_nxt = instrlist_encode(dcontext, ilist, buf, false/*no instr_t targets*/);
+    ASSERT(buf_nxt != NULL);
+    ASSERT((buf_nxt - buf) == (ssize_t)size ||
+           size_change && (buf_nxt - buf) > (ssize_t)size);
+    return buf_nxt;
+}
+
+/* N.B.: !assume_xsp implies eflags assumptions!
+ * Could add another choice of assuming not on dstack, in which case the
+ * eflags assumption is not required.
+ * Could optimize by having a bool indicating whether to have a callee arg or not,
+ * but then the intercept_function_t typedef must be void, or must have two, so we
+ * just make every callee take an arg.
+ * Currently only hotp_only uses alt_after_cti_p.
+ */
+static byte *
+emit_intercept_code(dcontext_t *dcontext, byte *pc, intercept_function_t callee,
+                    void *callee_arg, bool assume_xsp,
+                    after_intercept_action_t action_after, byte *alternate_after,
+                    byte **alt_after_cti_p)
+{
+    instrlist_t ilist;
+    instr_t *inst, *push_start, *push_start2 = NULL;
+    instr_t *decision = NULL, *alt_decision = NULL, *alt_after = NULL;
+    uint len;
+    byte *push_pc, *push_pc2 = NULL;
+    bool assume_not_on_dstack = assume_xsp;
+    app_pc no_cleanup;
+    
+    /* AFTER_INTERCEPT_LET_GO_ALT_DYN is used only dynamically to select alternate */
+    ASSERT(action_after != AFTER_INTERCEPT_LET_GO_ALT_DYN);
+
+    /* alternate_after provided only when possibly using alternate target */
+    ASSERT(alternate_after == NULL || 
+           action_after == AFTER_INTERCEPT_DYNAMIC_DECISION ||
+           action_after == AFTER_INTERCEPT_TAKE_OVER_SINGLE_SHOT);
+
+    /* initialize the ilist */
+    instrlist_init(&ilist);
+
+    if (!assume_xsp) {
+        instr_t *no_local_stack = INSTR_CREATE_label(dcontext);
+        instr_t *grab_initstack = INSTR_CREATE_label(dcontext);
+        instr_t *get_lock = INSTR_CREATE_label(dcontext);
+        instr_t *have_lock = INSTR_CREATE_label(dcontext);
+        instr_t *have_stack_now = INSTR_CREATE_label(dcontext);
+        APP(&ilist,
+            INSTR_CREATE_mov_st(dcontext,
+                                opnd_create_far_base_disp(SEG_TLS, REG_NULL, REG_NULL,
+                                                          0, PID_TIB_OFFSET, OPSZ_PTR),
+                                opnd_create_reg(REG_XCX)));
+        APP(&ilist,
+            INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XCX),
+                                opnd_create_tls_slot(os_tls_offset(TLS_DCONTEXT_SLOT))));
+        APP(&ilist, INSTR_CREATE_jecxz(dcontext, opnd_create_instr(no_local_stack)));
+
+        if (!assume_not_on_dstack) {
+            instr_t *not_on_dstack = INSTR_CREATE_label(dcontext);
+            APP(&ilist,
+                instr_create_restore_from_dc_via_reg(dcontext, REG_XCX, REG_XCX,
+                                                     DSTACK_OFFSET));
+            APP(&ilist,
+                INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_XSP),
+                                 opnd_create_reg(REG_XCX)));
+            APP(&ilist,
+                INSTR_CREATE_jcc(dcontext, OP_jge, opnd_create_instr(not_on_dstack)));
+            APP(&ilist,
+                INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XCX),
+                                 opnd_create_base_disp(REG_XCX, REG_NULL, 0,
+                                                       -(int)DYNAMORIO_STACK_SIZE, OPSZ_0)));
+            APP(&ilist,
+                INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_XSP),
+                                 opnd_create_reg(REG_XCX)));
+            APP(&ilist,
+                INSTR_CREATE_jcc(dcontext, OP_jl, opnd_create_instr(not_on_dstack)));
+            APP(&ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XSP)));
+            APP(&ilist,
+                INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32(2)));
+            APP(&ilist,
+                INSTR_CREATE_jmp(dcontext, opnd_create_instr(have_stack_now)));
+            APP(&ilist, not_on_dstack);
+            APP(&ilist, INSTR_CREATE_mov_ld
+                (dcontext, opnd_create_reg(REG_XCX),
+                 opnd_create_tls_slot(os_tls_offset(TLS_DCONTEXT_SLOT))));
+        }
+
+        /* Store the app xsp in dcontext and switch to dstack. */
+        if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)) {
+            APP(&ilist,
+                instr_create_restore_from_dc_via_reg(dcontext, REG_XCX, REG_XCX,
+                                                     PROT_OFFS));
+        }
+        APP(&ilist,
+            instr_create_save_to_dc_via_reg(dcontext, REG_XCX, REG_XSP, XSP_OFFSET));
+        if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)) {
+            APP(&ilist, INSTR_CREATE_mov_ld
+                (dcontext, opnd_create_reg(REG_XCX),
+                 opnd_create_tls_slot(os_tls_offset(TLS_DCONTEXT_SLOT))));
+        }
+        APP(&ilist,
+            instr_create_restore_from_dc_via_reg(dcontext, REG_XCX, REG_XSP,
+                                                 DSTACK_OFFSET));
+
+        /* Get the app xsp from the dcontext and put it on the dstack to serve
+         * as the app xsp cache.
+         */
+        if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)) {
+            APP(&ilist,
+                instr_create_restore_from_dc_via_reg(dcontext,REG_XCX, REG_XCX,
+                                                     PROT_OFFS));
+        }
+        APP(&ilist,
+            instr_create_restore_from_dc_via_reg(dcontext, REG_XCX, REG_XCX, XSP_OFFSET));
+        APP(&ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XCX)));
+        APP(&ilist,
+            INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32(1)));
+        APP(&ilist,
+            INSTR_CREATE_jmp(dcontext, opnd_create_instr(have_stack_now)));
+
+        /* use initstack to avoid any assumptions about app xsp */
+        /* first check if we are already on it */
+        APP(&ilist, no_local_stack);
+        APP(&ilist, 
+            INSTR_CREATE_mov_imm(dcontext,
+                                 opnd_create_reg(REG_XCX),
+                                 OPND_CREATE_INTPTR((ptr_int_t)initstack)));
+       APP(&ilist,
+            INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_XSP),
+                             opnd_create_reg(REG_XCX)));
+        APP(&ilist,
+            INSTR_CREATE_jcc(dcontext, OP_jge, opnd_create_instr(grab_initstack)));
+        APP(&ilist,
+            INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XCX),
+                             opnd_create_base_disp(REG_XCX, REG_NULL, 0,
+                                                   -(int)DYNAMORIO_STACK_SIZE, OPSZ_0)));
+        APP(&ilist,
+            INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_XSP),
+                             opnd_create_reg(REG_XCX)));
+        APP(&ilist,
+            INSTR_CREATE_jcc(dcontext, OP_jl, opnd_create_instr(grab_initstack)));
+        APP(&ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XSP)));
+        APP(&ilist,
+            INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32(2)));
+        APP(&ilist,
+            INSTR_CREATE_jmp(dcontext, opnd_create_instr(have_stack_now)));
+        APP(&ilist, grab_initstack);
+        APP(&ilist,
+            INSTR_CREATE_mov_imm(dcontext,
+                                 /* on x64 the upper 32 bits will be zeroed for us */
+                                 opnd_create_reg(REG_ECX), OPND_CREATE_INT32(1)));
+#ifdef X64
+        APP(&ilist,
+            INSTR_CREATE_mov_st(dcontext,
+                                opnd_create_far_base_disp(SEG_TLS, REG_NULL, REG_NULL,
+                                                          0, PEB_TIB_OFFSET, OPSZ_PTR),
+                                opnd_create_reg(REG_XAX)));
+#endif
+        APP(&ilist, get_lock);
+#ifdef X64
+        APP(&ilist,
+            INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XAX),
+                                 OPND_CREATE_INTPTR((ptr_uint_t)&initstack_mutex)));
+#endif
+        APP(&ilist,
+           INSTR_CREATE_xchg(dcontext,
+                             /* initstack_mutex is 32 bits always */
+                             IF_X64_ELSE(opnd_create_reg(REG_XAX),
+                                         OPND_CREATE_ABSMEM((void *)&initstack_mutex,
+                                                            OPSZ_4)),
+                             opnd_create_reg(REG_XCX)));
+        APP(&ilist,
+            INSTR_CREATE_jecxz(dcontext, opnd_create_instr(have_lock)));
+        APP(&ilist,
+            INSTR_CREATE_pause(dcontext));
+        APP(&ilist,
+            INSTR_CREATE_jmp(dcontext, opnd_create_instr(get_lock)));
+        APP(&ilist, have_lock);
+        APP(&ilist,
+            INSTR_CREATE_mov_st(dcontext,
+                                IF_X64_ELSE(opnd_create_reg(REG_XCX),
+                                            OPND_CREATE_ABSMEM((void *)&initstack_app_xsp,
+                                                               OPSZ_PTR)),
+                                opnd_create_reg(REG_XSP)));
+#ifdef X64
+        /* we can do a 64-bit absolute address into xax only */
+        APP(&ilist,
+            INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XAX),
+                                OPND_CREATE_ABSMEM((void *)&initstack, OPSZ_PTR)));
+        APP(&ilist,
+            INSTR_CREATE_xchg(dcontext, opnd_create_reg(REG_XSP),
+                              opnd_create_reg(REG_XAX)));
+#else
+        APP(&ilist,
+            INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XSP),
+                                OPND_CREATE_ABSMEM((void *)&initstack, OPSZ_PTR)));
+#endif
+        APP(&ilist,
+            INSTR_CREATE_push(dcontext,
+                              IF_X64_ELSE(opnd_create_reg(REG_XCX),
+                                          OPND_CREATE_ABSMEM((void *)&initstack_app_xsp,
+                                                             OPSZ_PTR))));
+        APP(&ilist,
+            INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32(0)));
+#ifdef X64
+        APP(&ilist,
+            INSTR_CREATE_mov_imm(dcontext,
+                                 opnd_create_reg(REG_XAX),
+                                 OPND_CREATE_INTPTR((ptr_uint_t)peb_ptr)));
+        APP(&ilist,
+            INSTR_CREATE_xchg(dcontext,
+                              opnd_create_far_base_disp(SEG_TLS, REG_NULL, REG_NULL,
+                                                        0, PEB_TIB_OFFSET, OPSZ_PTR),
+                              opnd_create_reg(REG_XAX)));
+#endif
+        APP(&ilist, have_stack_now);
+#ifdef X64
+        /* we could perhaps assume the top 32 bits of win32_pid are zero, but
+         * xchg works just as well */
+        APP(&ilist,
+            INSTR_CREATE_mov_imm(dcontext,
+                                 opnd_create_reg(REG_XCX),
+                                 OPND_CREATE_INTPTR((ptr_uint_t)win32_pid)));
+        APP(&ilist,
+            INSTR_CREATE_xchg(dcontext,
+                              opnd_create_far_base_disp(SEG_TLS, REG_NULL, REG_NULL,
+                                                        0, PID_TIB_OFFSET, OPSZ_PTR),
+                              opnd_create_reg(REG_XCX)));
+#else
+        APP(&ilist,
+            INSTR_CREATE_mov_ld(dcontext,
+                                opnd_create_reg(REG_XCX),
+                                opnd_create_far_base_disp(SEG_TLS, REG_NULL, REG_NULL,
+                                                          0, PID_TIB_OFFSET, OPSZ_PTR)));
+        APP(&ilist,
+            INSTR_CREATE_mov_st(dcontext,
+                                opnd_create_far_base_disp(SEG_TLS, REG_NULL, REG_NULL,
+                                                          0, PID_TIB_OFFSET, OPSZ_PTR),
+                                OPND_CREATE_INTPTR(win32_pid)));
+#endif /* X64 */
+    } else { /* assume_xsp */
+        /* Cache app xsp so that the right value can be passed to the handler
+         * and to restore at exit.  Push stack type too: 3 for app stack.
+         */
+        APP(&ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XSP)));
+        APP(&ilist, INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32(3)));
+    }
+
+    /* pc slot: not used: could use instead of state->start_pc */
+    APP(&ilist, INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32(0)));/*sign-extended*/
+    APP(&ilist, INSTR_CREATE_RAW_pushf(dcontext));
+    /* We assume that if !assume_xsp we've done two pushes on the stack,
+     * which combined w/ the push0 and pushf give us 16-byte alignment
+     * for 32-bit and 64-bit at this point.
+     * PR 270115: However if already on dstack we may not be aligned: we play it
+     * safe and use unaligned instrs, rather than add code to test and align.
+     */
+    insert_push_all_registers(dcontext, &ilist, NULL, false);
+    /* clear eflags for callee's usage */
+    APP(&ilist,
+        INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32(0)));
+    APP(&ilist, INSTR_CREATE_RAW_popf(dcontext));
+
+    /* Get the cached app xsp and update the pusha's xsp with it; this is the
+     * right app xsp.
+     */
+    APP(&ilist,
+        INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XAX),
+                            OPND_CREATE_MEMPTR(REG_XSP, /* mcxt + stack type */
+                                               sizeof(dr_mcontext_t)+XSP_SZ)));
+    APP(&ilist,
+        INSTR_CREATE_mov_st(dcontext,
+                            OPND_CREATE_MEMPTR(REG_XSP,
+                                               offsetof(dr_mcontext_t, xsp)),
+                            opnd_create_reg(REG_XAX)));
+
+    /* the 4 bytes of x64 app_state_at_intercept_t padding are
+     * handled by the 8-byte push of eax */
+    /* must preserve the LastErrorCode (if call a Win32 API routine could
+     * overwrite the app's error code) */
+    preinsert_get_last_error(dcontext, &ilist, NULL, REG_EAX); /* 32 bits for x64 too */
+    /* save errno on stack (top 32 bits zeroed for x64; can't push just eax) */
+    APP(&ilist,
+        INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XAX)));
+
+    /* FIXME: don't want hooks for trampolines that run natively like
+     * LdrLoadDll or image entry, right?
+     */
+    if (ENTER_DR_HOOK != NULL) {
+        /* make sure to use dr_insert_call() rather than a raw OP_call instr,
+         * since x64 windows requires 32 bytes of stack space even w/ no args.
+         */
+        dr_insert_call((void *)dcontext, &ilist, NULL/*append*/,
+                       (void *)ENTER_DR_HOOK, 0);
+    }
+
+    /* these are part of app_state_at_intercept_t struct so we have to
+     * push them on the stack, rather than pass in registers
+     */
+    /* will fill in immed with no_cleanup pointer later */
+#ifdef X64
+    push_start = INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XAX),
+                                      OPND_CREATE_INTPTR(0));
+    APP(&ilist, push_start);
+    APP(&ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XAX)));
+    APP(&ilist, INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XAX),
+                                     OPND_CREATE_INTPTR(callee_arg)));
+    APP(&ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XAX)));
+#else
+    push_start = INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INTPTR(0));
+    APP(&ilist, push_start);
+    APP(&ilist, INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INTPTR(callee_arg)));
+#endif
+
+    /* We pass xsp as a pointer to all the values on the stack; this is the actual
+     * argument to the intercept routine.  Fix for case 7597.
+     * -- CAUTION -- if app_state_at_intercept_t changes in anyway, this can
+     * blow up!  That structure's field's types, order & layout are assumed
+     * here.  These two should change only in synch. 
+     */
+    if (parameters_stack_padded()) {
+        /* xsp won't have proper value due to stack padding */
+        APP(&ilist, INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XAX),
+                                        opnd_create_reg(REG_XSP)));
+    }
+    dr_insert_call(dcontext, &ilist, NULL, (byte *)callee, 1,
+                   parameters_stack_padded() ? opnd_create_reg(REG_XAX) :
+                   opnd_create_reg(REG_XSP));
+    /* clean up 2 pushes */
+    APP(&ilist, INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XSP),
+                                 opnd_create_base_disp(REG_XSP, REG_NULL, 0,
+                                                       2*XSP_SZ, OPSZ_0)));
+    if (action_after == AFTER_INTERCEPT_DYNAMIC_DECISION) {
+        /* our 32-bit immed will be sign-extended.
+         * perhaps we could assume upper bits not set and use eax to save a rex.w.
+         */
+        APP(&ilist, INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_XAX),
+                                     OPND_CREATE_INT32(AFTER_INTERCEPT_LET_GO)));
+        /* will fill in later */
+        decision = INSTR_CREATE_jcc(dcontext, OP_je, opnd_create_instr(NULL));
+        APP(&ilist, decision);
+        if (alternate_after != NULL) {
+            APP(&ilist, INSTR_CREATE_cmp
+                (dcontext, opnd_create_reg(REG_XAX),
+                 OPND_CREATE_INT32(AFTER_INTERCEPT_LET_GO_ALT_DYN))); /*sign-extended*/
+            /* will fill in later */
+            alt_decision = INSTR_CREATE_jcc(dcontext, OP_je, opnd_create_instr(NULL));
+            APP(&ilist, alt_decision);
+        }
+    }
+
+    if (action_after == AFTER_INTERCEPT_TAKE_OVER ||
+        action_after == AFTER_INTERCEPT_TAKE_OVER_SINGLE_SHOT ||
+        action_after == AFTER_INTERCEPT_DYNAMIC_DECISION) {
+        /* will fill in immed with no_cleanup pointer later */
+#ifdef X64
+        push_start2 = INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XAX),
+                                           OPND_CREATE_INTPTR(0));
+        APP(&ilist, push_start2);
+        APP(&ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XAX)));
+#else
+        push_start2 = INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INTPTR(0));
+        APP(&ilist, push_start2);
+#endif
+        APP(&ilist, INSTR_CREATE_push_imm(dcontext,
+                                          OPND_CREATE_INT32(0/*don't save dcontext*/)));
+        if (parameters_stack_padded()) {
+            /* xsp won't have proper value due to stack padding */
+            APP(&ilist, INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XAX),
+                                            opnd_create_reg(REG_XSP)));
+        }
+        dr_insert_call(dcontext, &ilist, NULL, (app_pc)asynch_take_over, 1,
+                       parameters_stack_padded() ? opnd_create_reg(REG_XAX) :
+                       opnd_create_reg(REG_XSP));
+#ifdef INTERNAL
+        dr_insert_call(dcontext, &ilist, NULL, (app_pc)internal_error, 3,
+                       OPND_CREATE_INTPTR(0),
+                       OPND_CREATE_INT32(-3),
+                       OPND_CREATE_INTPTR(0));
+#endif
+    }
+
+    if (action_after == AFTER_INTERCEPT_LET_GO ||
+        action_after == AFTER_INTERCEPT_DYNAMIC_DECISION) {
+        if (alternate_after != NULL) {
+            insert_let_go_cleanup(dcontext, &ilist, alt_decision,
+                                  assume_xsp, assume_not_on_dstack, action_after);
+            /* alternate after cleanup target */
+            alt_after = INSTR_CREATE_jmp(dcontext, opnd_create_pc((app_pc)alternate_after));
+            APP(&ilist, alt_after);
+        }
+        /* the normal let_go target */
+        insert_let_go_cleanup(dcontext, &ilist, decision,
+                              assume_xsp, assume_not_on_dstack, action_after);
+    }
+
+    /* now encode the instructions */
+    /* must set note fields first with offset */
+    len = 0;
+    push_pc = NULL;
+    for (inst = instrlist_first(&ilist); inst; inst = instr_get_next(inst)) {
+        instr_set_note(inst, (void *)(ptr_int_t)len);
+        len += instr_length(dcontext, inst);
+    }
+    for (inst = instrlist_first(&ilist); inst; inst = instr_get_next(inst)) {
+        pc = instr_encode(dcontext, inst, pc);
+        ASSERT(pc != NULL);
+        if (inst == push_start)
+            push_pc = (pc - sizeof(ptr_uint_t));
+        if (inst == push_start2)
+            push_pc2 = (pc - sizeof(ptr_uint_t));
+        if (inst == alt_after && alt_after_cti_p != NULL)
+            *alt_after_cti_p = pc - JMP_LONG_LENGTH;
+    }
+
+    /* now can point start_pc arg of callee at beyond-cleanup pc */
+    if (action_after == AFTER_INTERCEPT_TAKE_OVER_SINGLE_SHOT) {
+        /* Note the interface here allows any target. Yet as the name
+         * suggests it should mainly be used to directly transfer to
+         * the now restored trampoline target.
+         */
+        ASSERT(alternate_after != NULL);
+        no_cleanup = alternate_after;
+    } else {
+        /* callers are supposed to append the original target prefix */
+        no_cleanup = pc;
+    }
+
+    ASSERT(push_pc != NULL);
+    *((ptr_uint_t*)push_pc) = (ptr_uint_t)no_cleanup;
+    if (push_pc2 != NULL)
+        *((ptr_uint_t*)push_pc2) = (ptr_uint_t)no_cleanup;
+
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+
+    return pc;
+}
+#undef APP
+
+static void
+map_intercept_pc_to_app_pc(byte *interception_pc, app_pc original_app_pc,
+                           size_t displace_length)
+{
+    intercept_map_elem_t *elem = HEAP_TYPE_ALLOC
+        (GLOBAL_DCONTEXT, intercept_map_elem_t, ACCT_OTHER, UNPROTECTED);
+
+    elem->interception_pc = interception_pc;
+    elem->original_app_pc = original_app_pc;
+    elem->displace_length = displace_length;
+    elem->next = NULL;
+
+    mutex_lock(&map_intercept_pc_lock);
+
+    if (intercept_map->head == NULL) {
+        intercept_map->head = elem;
+        intercept_map->tail = elem;
+    }
+    else {
+        intercept_map->tail->next = elem;
+        intercept_map->tail = elem;
+    }
+
+    mutex_unlock(&map_intercept_pc_lock);
+}
+
+static void
+unmap_intercept_pc(app_pc original_app_pc)
+{
+    intercept_map_elem_t *curr, *prev;
+
+    mutex_lock(&map_intercept_pc_lock);
+
+    prev = NULL;
+    curr = intercept_map->head;
+    while (curr != NULL) {
+        if (curr->original_app_pc == original_app_pc) {
+            if (prev != NULL) {
+                prev->next = curr->next;
+            }
+            if (curr == intercept_map->head) {
+                intercept_map->head = curr->next;
+            }
+            if (curr == intercept_map->tail) {
+                intercept_map->tail = prev;
+            }
+
+            HEAP_TYPE_FREE(GLOBAL_DCONTEXT, curr, intercept_map_elem_t, 
+                           ACCT_OTHER, UNPROTECTED);
+            break;
+        }
+
+        prev = curr;
+        curr = curr->next;
+    }
+     
+    mutex_unlock(&map_intercept_pc_lock);
+}
+
+/* We assume no mangling of code placed in the interception buffer,
+ * other than re-relativizing ctis.  As such, we can uniquely correlate
+ * interception buffer PCs to their original app PCs.
+ * Caller must check that pc is actually in the intercept buffer.
+ */
+app_pc
+get_app_pc_from_intercept_pc(byte *pc)
+{
+    intercept_map_elem_t *iter = intercept_map->head;
+    while (iter != NULL) {
+        byte *start = iter->interception_pc;
+        byte *end = start + iter->displace_length;
+        if (pc >= start && pc < end)
+            return iter->original_app_pc + (pc - start);
+
+        iter = iter->next;
+    }
+
+    ASSERT_NOT_REACHED();
+    return NULL;
+}
+
+bool
+is_intercepted_app_pc(app_pc pc, byte **interception_pc)
+{
+    intercept_map_elem_t *iter = intercept_map->head;
+    while (iter != NULL) {
+        if (iter->original_app_pc == pc) {
+            /* PR 219351: For syscall trampolines, while building bbs we replace
+             * the jmp and never execute from the displaced app code in the
+             * buffer, so the bb looks normal.  FIXME: should we just not add to
+             * the map?  For now, better safe than sorry so
+             * get_app_pc_from_intercept_pc will work in case we ever ask about
+             * that displaced app code.
+             */
+            if (is_syscall_trampoline(iter->interception_pc))
+                return false;
+            if (interception_pc != NULL)
+                *interception_pc = iter->interception_pc;
+
+            return true;
+        }
+
+        iter = iter->next;
+    }
+
+    return false;
+}
+
+/* Redirects code at tgt_pc to jmp to our_pc, which is filled with generated
+ * code to call prof_func and then return to the original code.
+ * Assumes that the original tgt_pc should be unwritable.
+ * The caller is responsible for adding the generated
+ * code at our_pc to the dynamo/executable list(s).
+ *
+ * We assume we're being called either before any threads are created
+ * or while all threads are suspended, as our code-overwriting is not atomic!
+ * The only fix is to switch from code-overwriting to import-table modifying,
+ * which is more complicated, see Richter chap22 for example: and import-table
+ * modifying will not allow arbitrary hook placement of course, which we
+ * support for probes and hot patches.
+ *
+ * We guarantee to use a 5-byte jump instruction, even on x64 (PR 250294: we
+ * sometimes have to allocate nearby landing pads there.  See PR 245169 for all
+ * of the possibilities for x64 hooking, all of which are either very large or
+ * have caveats; we decided that allocating several 64K chunks and sticking w/
+ * 5-byte jumps was the cleanest).  It is up to the caller to ensure that we
+ * aren't crossing a cti target point and that displacing these 5 bytes is safe
+ * (though we will take care of re-relativizing the displaced code)).
+ *
+ * When cti_safe_to_ignore true, we expect to restore the code
+ * immediately after hitting our trampoline then we can treat the
+ * first 5 bytes as raw.  Otherwise, we may need to PC-relativize or
+ * deal with conflicting hookers (case 2525).  Assuming a CTI in the
+ * target is a good sign for hookers, we may decide to treat that
+ * specially based on DYNAMO_OPTION(hook_conflict) or we can give up
+ * and not intercept this call when abort_on_incompatible_hooker is
+ * true.
+ * FIXME: if we add one more flag we should switch to a single flag enum
+ * 
+ * Currently only hotp_only uses app_code_copy_p and alt_exit_cti_p.
+ * Returns pc after last instruction of emitted interception code,
+ * or NULL when abort_on_incompatible_hooker is true and tgt_pc starts with a CTI.
+ */
+static byte *
+intercept_call(byte *our_pc, byte *tgt_pc, intercept_function_t prof_func,
+               void *callee_arg, bool assume_xsp, after_intercept_action_t action_after,
+               bool abort_on_incompatible_hooker,
+               bool cti_safe_to_ignore,
+               byte **app_code_copy_p,
+               byte **alt_exit_cti_p)
+{
+    byte *pc, *our_pc_end, *lpad_pc;
+    size_t size;
+    instrlist_t ilist;
+    instr_t *instr;
+    bool changed_prot;
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    bool is_hooked = false;
+    bool ok;
+
+#ifdef HOT_PATCHING_INTERFACE
+    /* During core init time when hotp_init() is executed, for hotp_only 
+     * injection takes place.  At this time thread initializations haven't 
+     * been done, so dcontext is NULL, therefore the call to intercept_call() 
+     * fails because it calls decode() which needs a valid dcontext to 
+     * allocate memory. 
+     */
+    if (DYNAMO_OPTION(hotp_only) && dcontext == NULL)
+        dcontext = GLOBAL_DCONTEXT;
+#endif
+
+    ASSERT(tgt_pc != NULL);
+    /* can't detect hookers if ignoring CTIs */
+    ASSERT(!abort_on_incompatible_hooker || !cti_safe_to_ignore);
+
+    /* we need 5 bytes for a jump
+     * find instr boundary >= 5 bytes after pc
+     */
+    LOG(GLOBAL, LOG_ASYNCH, 3, "before intercepting:\n");
+    instrlist_init(&ilist);
+    pc = tgt_pc;
+    do {
+        app_pc next_pc;
+        DOLOG(3, LOG_ASYNCH, {
+            disassemble_with_bytes(dcontext, pc, main_logfile);
+        });
+        instr = instr_create(dcontext);
+        next_pc = decode_cti(dcontext, pc, instr);
+        ASSERT(instr_valid(instr));
+        instrlist_append(&ilist, instr);
+
+        /* we do not handle control transfer instructions very well here! (case 2525) */
+        if (instr_opcode_valid(instr) && instr_is_cti(instr)) {
+            /* allow for only a single cti at first instruction,
+             *
+             * unless CTIs are safe to ignore since never actually
+             * re-relativized (case 4086 == once-only so don't execute copy)
+             */
+            ASSERT(!is_hooked);
+            ASSERT(tgt_pc == pc || cti_safe_to_ignore);
+            if (!cti_safe_to_ignore) {
+                /* we treat this as a sign of a third party hooking before us */
+                is_hooked =  true;
+            }
+        }
+
+        pc = next_pc;
+
+        /* some of our trampolines are best effort anyways: LdrLoadDll
+         * shouldn't matter much, yet we like to keep it when we can
+         */
+        if (is_hooked && abort_on_incompatible_hooker) {
+            SYSLOG_INTERNAL_WARNING_ONCE("giving up interception: "PFX" already hooked\n",
+                                         tgt_pc);
+            LOG(GLOBAL, LOG_ASYNCH, 1, "intercept_call: giving up "PFX" already hooked\n", tgt_pc);
+            instrlist_clear(dcontext, &ilist);
+            return NULL;
+        }
+        
+        if (pc == NULL || is_hooked && DYNAMO_OPTION(hook_conflict) == HOOKED_TRAMPOLINE_DIE) {
+            FATAL_USAGE_ERROR(TAMPERED_NTDLL, 2, get_application_name(), get_application_pid());
+        }
+
+        size = (pc - tgt_pc);
+    } while (size < 5);
+
+    pc = our_pc;
+
+    if (is_hooked && DYNAMO_OPTION(hook_conflict) == HOOKED_TRAMPOLINE_SQUASH) {
+        /* squash over original with expected code, so that both
+         * copies we make later (one for actual execution and one for
+         * uninterception) have the supposedly original values
+         * see use in intercept_syscall_wrapper()
+         */
+        /* FIXME: it is not easy to get the correct original bytes
+         * probably best solution is to read from the original
+         * ntdll.dll on disk.  To avoid having to deal with RVA disk
+         * to virtual address transformations, it may be even easier
+         * to call LdrLoadDll with a different path to a load a
+         * pristine copy e.g. \\?C:\WINNT\system32\ntdll.dll
+         */
+        /* FIXME: even if we detach we don't restore the original
+         * values, since what we have here should be good enough
+         */
+        ASSERT_NOT_IMPLEMENTED(false);
+    }
+
+    /* Store 1st 5 bytes of original code at start of our code
+     * (won't be executed, original code will jump to after it)
+     * We do this for convenience of un-intercepting, so we don't have to
+     * record offset of the copy in the middle of the interception code
+     * CAUTION: storing the exact copy of the 5 bytes from the app image at 
+     *          the start of the trampoline is assumed in hotp_only for 
+     *          case 7279 - change only in synch.
+     */
+    memcpy(pc, tgt_pc, 5);
+    pc += 5;
+
+    /* Allocate the landing pad, store its address (4 bytes in 32-bit builds
+     * and 8 in 64-bit ones) in the trampoline, just after the original app
+     * code, and emit it.
+     */
+    lpad_pc = alloc_landing_pad(tgt_pc);
+    memcpy(pc, &lpad_pc, sizeof(lpad_pc));
+    pc += sizeof(lpad_pc);
+    lpad_pc = emit_landing_pad_code(lpad_pc, pc, tgt_pc + size);
+
+    pc = emit_intercept_code(dcontext, pc, prof_func, callee_arg,
+                             assume_xsp, action_after, 
+                             (action_after ==
+                              AFTER_INTERCEPT_TAKE_OVER_SINGLE_SHOT) ?
+                                 tgt_pc : 
+                                 ((alt_exit_cti_p != NULL) ?
+                                     CURRENTLY_UNKNOWN :
+                                     NULL),
+                             alt_exit_cti_p);
+
+    /* If we are TAKE_OVER_SINGLE_SHOT then the handler routine has promised to 
+     * restore the original code and supply the appropriate continuation address.
+     * As such there is no need for us to copy the code here as we will never use it.
+     * (Note not copying the code also gives us a quick fix for the Vista image entry
+     * problem in PR 293452 from not yet handling non-reaching cits in hook displaced
+     * code PR 268988). FIXME - not having a displaced copy to decode breaks the 
+     * redirection deoode_as_bb() (but not other deocde routines) uses to hide the 
+     * hook from the client (see PR 293465 for other reasons we need a better solution
+     * to that problem). */
+    if (action_after != AFTER_INTERCEPT_TAKE_OVER_SINGLE_SHOT) {
+        /* Map interception buffer PCs to original app PCs */
+        if (is_in_interception_buffer(pc)) {
+            map_intercept_pc_to_app_pc
+                (pc, tgt_pc, size + IF_X64_ELSE(6, 5) /* include jmp back */);
+        }
+        
+        /* Copy original instructions to our version, re-relativizing where necessary */
+        if (app_code_copy_p != NULL)
+            *app_code_copy_p = pc;
+        pc = copy_app_code(dcontext, tgt_pc, pc, size, &ilist);
+    } else {
+        /* single shot hooks shouldn't need a copy of the app code */
+        ASSERT(app_code_copy_p == NULL);
+    }
+
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+
+    if (is_hooked) {
+        if (DYNAMO_OPTION(hook_conflict) == HOOKED_TRAMPOLINE_CHAIN) {
+            /* we only have to rerelativize rel32, yet indirect
+             * branches can also be used by hookers, in which case we
+             * don't need to do anything special when copying as bytes
+             */
+
+            /* FIXME: now re-relativize at target location */
+            ASSERT_NOT_IMPLEMENTED(false);
+            ASSERT_NOT_TESTED();
+        }
+    }
+
+    /* Must return to the second jmp in the landing pad; the first jmp is 6
+     * bytes for 64-bit and 5 for 32-bit (see emit_landing_pad_code() for
+     * details).  Also, for 64-bit 8 bytes of the trampoline address are stored
+     * at the top of the landing pad.  See emit_landing_pad_code().
+     */
+#ifndef X64
+    *pc = JMP_REL32_OPCODE; pc++;
+    *((int *)pc) = (int)(lpad_pc + JMP_REL32_SIZE - pc - 4);
+    pc += 4;    /* 4 is the size of the relative offset */
+#else
+    *pc = JMP_ABS_IND64_OPCODE; pc++;
+    *pc = JMP_ABS_MEM_IND64_MODRM; pc++;
+    /* 64-bit abs address is placed after the jmp instr., i.e., rip rel is 0.
+     * We can't place it before the jmp as in the case of the landing pad
+     * because there is code in the trampoline immediately preceding this jmp.
+     */
+    *((int *)pc) = 0; pc += 4;  /* 4 here is the rel offset to the lpad entry */
+    *((byte **)pc) = lpad_pc + JMP_ABS_IND64_SIZE; pc += sizeof(lpad_pc);
+#endif
+    our_pc_end = pc;
+
+    /* Replace original code with jmp to our version (after 5-byte backup) */
+    /* copy-on-write will give us a copy of this page */
+    ok = make_hookable(tgt_pc, JMP_REL32_SIZE, &changed_prot);
+    if (!ok) {
+        /* FIXME: we fail to insert our hook but for now it is easier
+         * to pretend that we succeeded. */
+        /* should really return NULL and have callers handle this better */
+        return our_pc_end;
+    }
+    pc = tgt_pc;
+    *pc = JMP_REL32_OPCODE; pc++;
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(lpad_pc - pc - 4)));
+    *((int *)pc) = (int)(ptr_int_t)(lpad_pc - pc - 4);
+    /* make our page unwritable now */
+    make_unhookable(tgt_pc, JMP_REL32_SIZE, changed_prot);
+
+    ASSERT(our_pc_end != NULL);
+    return our_pc_end;
+}
+
+/* Assumes that tgt_pc should be unwritable */
+static void
+un_intercept_call(byte *our_pc, byte *tgt_pc)
+{
+    bool changed_prot;
+    bool ok;
+    byte *lpad_entry;
+    /* if intercept_call() has failed we shouldn't be un-intercepting */
+    if (our_pc == NULL)
+        return;
+
+    lpad_entry = (tgt_pc + JMP_REL32_SIZE) + *((int *)(tgt_pc + 1));
+
+    /* restore 1st 5 bytes of original code */
+    ok = make_hookable(tgt_pc, JMP_REL32_SIZE, &changed_prot);
+    /* if we were able to hook we can't fail on unhook */
+    ASSERT(ok || memcmp(tgt_pc, our_pc, JMP_REL32_SIZE) == 0 /* hook wasn't applied */);
+    if (!ok) {
+        return;
+    }
+    ASSERT(memcmp(tgt_pc, our_pc, JMP_REL32_SIZE) != 0 /* hook was applied */);
+    memcpy(tgt_pc, our_pc, JMP_REL32_SIZE);
+    make_unhookable(tgt_pc, JMP_REL32_SIZE, changed_prot);
+
+    /* Redirect the first jump in the landing pad to the hooked address (which we just
+     * restored above) - in case someone has chained with our hook.  
+     */
+    ok = make_hookable(lpad_entry, JMP_SIZE, &changed_prot);
+    ASSERT(ok);
+    if (ok) {
+        /* patch jmp to go back to target */
+        /* Note - not a hot_patch, caller must have synchronized already  to make the
+         * memcpy restore above safe. */
+        insert_relative_target(lpad_entry+1, tgt_pc, false /* not a hotpatch */);
+        make_unhookable(lpad_entry, JMP_SIZE, changed_prot);
+    }
+
+    DOLOG(3, LOG_ASYNCH, {
+        byte *pc = tgt_pc;
+        LOG(GLOBAL, LOG_ASYNCH, 3, "after un-intercepting:\n");
+        do {
+            /* Use GLOBAL_DCONTEXT here since we may have already 
+             * called dynamo_thread_exit()
+             */
+            pc = disassemble_with_bytes(GLOBAL_DCONTEXT, pc, main_logfile);
+        } while (pc < tgt_pc + JMP_REL32_SIZE);
+    });
+
+    unmap_intercept_pc((app_pc)tgt_pc);
+}
+
+/* Returns the syscall wrapper at nt_wrapper to a pristine (unhooked) state. Currently
+ * used for -clean_testalert to block the problematic injection of SpywareDoctor (9288)
+ * and similar apps. Returns true if syscall wrapper required cleaning */
+/* FIXME - use this for our hook conflict squash policy in intercept_syscall_wrapper as
+ * this can handle more complicated hooks. */
+static bool
+clean_syscall_wrapper(byte *nt_wrapper, int sys_enum)
+{
+    dcontext_t *dcontext = GLOBAL_DCONTEXT;
+    instr_t *instr_new, *instr_old = instr_create(dcontext);
+    instrlist_t *ilist = instrlist_create(dcontext);
+    app_pc pc = nt_wrapper;
+    bool hooked = false;
+    int sysnum = syscalls[sys_enum];
+    uint arg_bytes = syscall_argsz[sys_enum];
+
+    if (nt_wrapper == NULL || sysnum == SYSCALL_NOT_PRESENT)
+        goto exit_clean_syscall_wrapper;
+
+    /* syscall wrapper should look like
+     * For NT/2000
+     * mov eax, sysnum         {5 bytes}
+     * lea edx, [esp+4]        {4 bytes}
+     * int 2e                  {2 bytes}
+     * ret arg_bytes           {1 byte (0 args) or 3 bytes}
+     *
+     * For XPsp0/XPsp1/2003sp0
+     * mov eax, sysnum         {5 bytes}
+     * mov edx, VSYSCALL_ADDR  {5 bytes}
+     * call edx                {2 bytes}
+     * ret arg_bytes           {1 byte (0 args) or 3 bytes}
+     *
+     * For XPsp2/2003sp1/Vista
+     * mov eax, sysnum         {5 bytes}
+     * mov edx, VSYSCALL_ADDR  {5 bytes}
+     * call [edx]              {2 bytes}
+     * ret arg_bytes           {1 byte (0 args) or 3 bytes}
+     *
+     * For WOW64 (case 3922), there are two types: if setting ecx to 0, xor is used.
+     * mov eax, sysnum         {5 bytes}
+     * mov ecx, wow_index      {5 bytes}  --OR--   xor ecx,ecx  {2 bytes}
+     * lea edx, [esp+4]        {4 bytes}
+     * call fs:0xc0            {7 bytes}
+     * ret arg_bytes           {1 byte (0 args) or 3 bytes}
+     *
+     * x64 syscall (PR 215398):
+     *   mov r10, rcx          {3 bytes}
+     *   mov eax, sysnum       {5 bytes}
+     *   syscall               {2 bytes}
+     *   ret                   {1 byte}
+     */
+
+    /* build correct instr list */
+#define APP(list, inst) instrlist_append((list), (inst))
+#ifdef X64
+    APP(ilist, INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_R10),
+                                   opnd_create_reg(REG_RCX)));
+    APP(ilist, INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_EAX),
+                                    OPND_CREATE_INT32(sysnum)));
+    APP(ilist, INSTR_CREATE_syscall(dcontext));
+    APP(ilist, INSTR_CREATE_ret(dcontext));
+#else
+    APP(ilist, INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_EAX),
+                                    opnd_create_immed_int(sysnum, OPSZ_4)));
+    /* NOTE - the structure of the wrapper depends only on the OS version, not on the
+     * syscall method (for ex. using int on XPsp2 just changes the target on the
+     * vsyscall page, not the wrapper layout). */
+    if (get_os_version() <= WINDOWS_VERSION_2000) {
+        APP(ilist,
+            INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XDX),
+                             opnd_create_base_disp(REG_XSP, REG_NULL, 0, 4, OPSZ_0)));
+        APP(ilist, INSTR_CREATE_int(dcontext, opnd_create_immed_int(0x2e, OPSZ_1)));
+    } else if (is_wow64_process(NT_CURRENT_PROCESS)) {
+        ASSERT(get_syscall_method() == SYSCALL_METHOD_WOW64);
+        ASSERT(wow64_index != NULL);
+        ASSERT(wow64_index[sys_enum] != SYSCALL_NOT_PRESENT);
+        if (wow64_index[sys_enum] == 0) {
+            APP(ilist, INSTR_CREATE_xor(dcontext, opnd_create_reg(REG_XCX),
+                                        opnd_create_reg(REG_XCX)));
+        } else {
+            APP(ilist, INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XCX),
+                                            OPND_CREATE_INT32(wow64_index[sys_enum])));
+        }
+        APP(ilist,
+            INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XDX),
+                             opnd_create_base_disp(REG_XSP, REG_NULL, 0, 4, OPSZ_0)));
+        APP(ilist, create_syscall_instr(dcontext));
+    } else { /* XP or greater */
+        APP(ilist,
+            INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XDX),
+                                 OPND_CREATE_INTPTR((ptr_int_t)VSYSCALL_BOOTSTRAP_ADDR)));
+        if (use_ki_syscall_routines()) {
+            /* call through vsyscall addr to Ki*SystemCall routine */
+            APP(ilist,
+                INSTR_CREATE_call_ind(dcontext, opnd_create_base_disp(REG_XDX, REG_NULL,
+                                                                      0, 0, OPSZ_4_short2)));
+        } else {
+            /* call to vsyscall addr */
+            APP(ilist, INSTR_CREATE_call_ind(dcontext, opnd_create_reg(REG_XDX)));
+        }
+    }
+    if (arg_bytes == 0) {
+        APP(ilist, INSTR_CREATE_ret(dcontext));
+    } else {
+        APP(ilist,
+            INSTR_CREATE_ret_imm(dcontext, opnd_create_immed_int(arg_bytes, OPSZ_1)));
+    }
+#endif /* X64 */
+#undef APP
+    
+    /* we've seen 3 different ways of hooking syscall wrappers :
+     * 1) jmp overwriting first 5 bytes (mov eax, sysnum), most common.
+     * 2) jmp overwriting second 5 bytes (certain versions of Sygate)
+     * 3) overwriting first 8 bytes with push eax (x3) then jmp (Spyware Doctor 9288, A^2
+     *      anti-spyware 10414). */
+    
+    /* NOTE - we could finish the walk whether hooked or not, but not much point and
+     * I don't fully trust are decode routine w/ junk input (if for ex. hook doesn't end
+     * on an instr boundary). */
+    for (instr_new = instrlist_first(ilist); instr_new != NULL;
+         instr_new = instr_get_next(instr_new)) {
+        instr_reset(dcontext, instr_old);
+        pc = decode(dcontext, pc, instr_old);
+        if (!instr_same(instr_new, instr_old)) {
+            /* We haven't seen hookers where the opcode would match, so in that case
+             * seems likely could be our fault (got an immed wrong or something). */
+            ASSERT_CURIOSITY(instr_get_opcode(instr_new) != instr_get_opcode(instr_old));
+            /* we haven't seen any hook start deeper then the 2nd instruction */
+            ASSERT_CURIOSITY(instr_new == instrlist_first(ilist) ||
+                             instr_new == instr_get_next(instrlist_first(ilist)));
+            hooked = true;
+            break;
+        }
+    }
+    
+    LOG(GLOBAL, LOG_SYSCALLS, hooked ? 1U : 2U,
+        "Syscall wrapper @ "PFX" syscall_num=0x%03x%s hooked.\n",
+        nt_wrapper, sysnum, hooked ? "" : " not");
+
+    if (hooked) {
+        bool changed_prot;
+        int length = 0, encode_length;
+        byte *nxt_pc;
+        instr_t *in;
+
+        SYSLOG_INTERNAL_WARNING_ONCE("Cleaning hooked Nt wrapper @"PFX" sysnum=0x%03x",
+                                     nt_wrapper, sysnum);
+        for (in = instrlist_first(ilist); in != NULL; in = instr_get_next(in))
+            length += instr_length(dcontext, in);
+        DOLOG(1, LOG_SYSCALLS, {
+            LOG(GLOBAL, LOG_SYSCALLS, 1, "Replacing hooked wrapper :\n");
+            pc = nt_wrapper;
+            /* Note - we may disassemble junk here (if hook doesn't end on instr
+             * boundary) but our decode routines should handle it; is debug anyways. */
+            while (pc - nt_wrapper < length)
+                pc = disassemble_with_bytes(dcontext, pc, GLOBAL);
+            LOG(GLOBAL, LOG_SYSCALLS, 1, "With :\n");
+            instrlist_disassemble(dcontext, nt_wrapper, ilist, GLOBAL);
+        });
+#ifdef X64
+        ASSERT(length == 11);
+#else
+        DODEBUG({
+            if (get_os_version() <= WINDOWS_VERSION_2000) {
+                ASSERT((length == 14 && arg_bytes > 0) ||
+                       (length == 12 && arg_bytes == 0));
+            } else {
+                ASSERT((length == 15 && arg_bytes > 0) ||
+                       (length == 13 && arg_bytes == 0));
+            }
+        });
+#endif /* X64 */
+
+        make_hookable(nt_wrapper, length, &changed_prot);
+        nxt_pc = instrlist_encode(dcontext, ilist, nt_wrapper,
+                                  false /* no jmp targets */);
+        ASSERT(nxt_pc != NULL);
+        encode_length = (int) (nxt_pc - nt_wrapper);
+        ASSERT(encode_length == length && "clean syscall encoded length mismatch");
+        make_unhookable(nt_wrapper, length, changed_prot);
+
+        DOLOG(1, LOG_SYSCALLS, {
+            LOG(GLOBAL, LOG_SYSCALLS, 1, "Cleaned wrapper is now :\n");
+            pc = nt_wrapper;
+            while (pc - nt_wrapper < length)
+                pc = disassemble_with_bytes(dcontext, pc, GLOBAL);
+        });
+    }
+
+exit_clean_syscall_wrapper:
+    instr_destroy(dcontext, instr_old);
+    instrlist_clear_and_destroy(dcontext, ilist);
+    return hooked;
+}
+
+/* Inserts a trampoline in a system call wrapper.
+ * All uses should end up using dstack -- else watch out for initstack
+ * infinite loop (see comment above).
+ * Returns in skip_syscall_pc the native pc for skipping the system call altogether.
+ *
+ * Since the only safe point is the first instr, and not right at the syscall
+ * instr itself (no 5-byte spot there), we have to copy the whole series of app
+ * instrs up until the syscall instr into our buffer to be executed prior to the
+ * callee.  This means any intercepted syscall from the cache will have that
+ * sequence run NATIVELY!  A solution is to set a flag to go back to native
+ * after the next syscall, and take over right away, but a little more worrisome
+ * than only executing the syscall under DR in terms of potential to miss the
+ * re-native trigger.
+ *
+ * For x64, we still use a 5-byte jump, assuming our main heap is within 2GB of
+ * ntdll.dll (xref PR 215395); if not we'll need an auxiliary landing pad
+ * trampoline within 2GB (xref PR 250294 where we need to support such
+ * trampolines for general hooks).  Also xref PR 245169 on x64 hooking
+ * possibilities, none of which is ideal.
+ *
+ * FIXME: other interception ideas: could do at instr after mov-immed,
+ * and arrange own int 2e for win2k, and emulate rest of sequence when
+ * handling syscall from handler -- this would eliminate some issues
+ * with the pre-syscall sequence copy, but not clear if better overall.
+ * Would be nice to have a single shared syscall handler, but since
+ * wrappers are stdcall that would be difficult.
+ *
+ * We allow the callee to execute the syscall itself, and by returning
+ * AFTER_INTERCEPT_LET_GO_ALT_DYN, it signals to skip the actual syscall,
+ * so we have control returned to the instr after the syscall instr.
+ * For AFTER_INTERCEPT_LET_GO or AFTER_INTERCEPT_TAKE_OVER, the syscall
+ * instr itself is the next instr to be executed.
+ *
+ * N.B.: this routine makes assumptions about the exact sequence of instrs in
+ * syscall wrappers, in particular that the indirect call to the vsyscall page
+ * can be turned into a direct call, which is only safe for XP SP2 if the
+ * vsyscall page is not writable, and cannot be made writable, which is what we
+ * have observed to be true.
+ */
+byte *
+intercept_syscall_wrapper(byte **ptgt_pc /* IN/OUT */, 
+                          intercept_function_t prof_func,
+                          void *callee_arg, after_intercept_action_t action_after,
+                          app_pc *skip_syscall_pc /* OUT */,
+                          byte **orig_bytes_pc /* OUT */,
+                          byte *fpo_stack_adjustment /* OUT OPTIONAL */)
+{
+    byte *pc, *emit_pc, *ret_pc, *after_hook_target, *entrance_pc, *tgt_pc;
+    instr_t *instr, *hook_return_instr = NULL;
+    instrlist_t ilist;
+    bool changed_prot;
+    int opcode;
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    int native_sys_num = syscalls[(ptr_uint_t)callee_arg];
+    bool ok;
+    DEBUG_DECLARE(bool is_hooked = false;)
+
+    instrlist_init(&ilist);
+
+    ASSERT(ptgt_pc != NULL && *ptgt_pc != NULL);
+    LOG(GLOBAL, LOG_ASYNCH, 3, "before intercepting:\n");
+    tgt_pc = *ptgt_pc;
+    pc = tgt_pc;
+
+    /* we need 5 bytes for a jump, and we assume that the first instr
+     * (2nd instr for x64, where we skip the 1st) is a 5-byte mov immed! 
+     */
+    DOLOG(3, LOG_ASYNCH, { disassemble_with_bytes(dcontext, pc, main_logfile); });
+    instr = instr_create(dcontext);
+    pc = decode(dcontext, pc, instr);
+    /* FIXME: handle other hookers gracefully by chaining!
+     * Note that moving trampoline point 5 bytes in could help here (see above).
+     */
+#ifndef X64
+    ASSERT(instr_length(dcontext, instr) >= 5);
+#endif
+    if (fpo_stack_adjustment != NULL)
+        *fpo_stack_adjustment = 0; /* for GBOP case 7127 */
+
+    if (instr_is_cti(instr)) {
+        DEBUG_DECLARE(is_hooked = true;)
+        /* we only have to rerelativize rel32, yet indirect
+         * branches can also be used by hookers, in which case we
+         * don't need to do anything special when copying as bytes
+         * FIXME: should we still die?
+         */
+
+        /* see case 2525 for background discussion */
+        if (DYNAMO_OPTION(native_exec_hook_conflict) == HOOKED_TRAMPOLINE_DIE) {
+            /* FIXME: we could still print the message but we don't have to kill the app here */
+            FATAL_USAGE_ERROR(TAMPERED_NTDLL, 2, get_application_name(),
+                              get_application_pid());
+        } else if (DYNAMO_OPTION(native_exec_hook_conflict) == HOOKED_TRAMPOLINE_CHAIN) {
+            /* we assume 5-byte hookers as well - so only need to relativize in our own copy */
+            /* and we need to introduce a PUSH in case of a CALL here */
+
+            ASSERT(instr_get_opcode(instr) != OP_call_ind);
+            if (instr_is_mbr(instr)) {
+                /* one can imagine mbr being used on x64 */
+                FATAL_USAGE_ERROR(TAMPERED_NTDLL, 2, get_application_name(),
+                                  get_application_pid());
+            }
+            if (instr_get_opcode(instr) == OP_call) {
+                LOG(GLOBAL, LOG_ASYNCH, 2, 
+                    "intercept_syscall_wrapper: mangling hooked call at "PFX"\n", pc);
+                /* replace the call w/ a push/jmp hoping this will
+                 * eventually return to us unless the hooker decides
+                 * to squash the system call or execute without going
+                 * back here.
+                 * FIXME: keep in mind the code on the instrlist is executed natively
+                 */
+#ifdef X64
+                /* do push-64-bit-immed in two pieces */
+                insert_push_immed_ptrsz(dcontext, &ilist, NULL, (ptr_int_t)pc);
+                /* check reachability from new location */
+                /* allow interception code to be up to a page: don't bother
+                 * to calculate exactly where our jmp will be encoded */
+                if (!REL32_REACHABLE(interception_cur_pc,
+                                     opnd_get_pc(instr_get_target(instr))) ||
+                    !REL32_REACHABLE(interception_cur_pc + PAGE_SIZE,
+                                     opnd_get_pc(instr_get_target(instr)))) {
+                    FATAL_USAGE_ERROR(TAMPERED_NTDLL, 2, get_application_name(),
+                                      get_application_pid());
+                }
+#else
+                instrlist_append(&ilist, INSTR_CREATE_push_imm
+                                 (dcontext, OPND_CREATE_INTPTR((ptr_int_t)pc)));
+#endif
+                instrlist_append(&ilist,
+                                 INSTR_CREATE_jmp(dcontext, 
+                                                  opnd_create_pc(opnd_get_pc(instr_get_target(instr)))));
+                /* skip original instruction */
+                instr_destroy(dcontext, instr);
+                /* interp still needs to be updated */
+                ASSERT_NOT_IMPLEMENTED(false);
+            } else if (instr_get_opcode(instr) == OP_jmp) {
+                /* FIXME - no good way to regain control after the hook */
+                ASSERT_NOT_IMPLEMENTED(false);
+                LOG(GLOBAL, LOG_ASYNCH, 2, "intercept_syscall_wrapper: hooked with jmp "PFX"\n", pc);
+                /* just append instruction as is */
+                instrlist_append(&ilist, instr);
+            } else {
+                ASSERT_NOT_IMPLEMENTED(false && "unchainable CTI");
+                /* FIXME PR 215397: need to re-relativize pc-relative memory reference */
+                IF_X64(ASSERT_NOT_IMPLEMENTED(!instr_has_rel_addr_reference(instr)));
+                /* just append instruction as is, emit re-relativises if neccesary */
+                instrlist_append(&ilist, instr);
+                /* FIXME: if instr's length doesn't match normal 1st instr we'll
+                 * get off down below: really shouldn't continue here */
+            }
+        } else if (DYNAMO_OPTION(native_exec_hook_conflict) == HOOKED_TRAMPOLINE_SQUASH) {
+            SYSLOG_INTERNAL_WARNING_ONCE("intercept_syscall_wrapper: "
+                                         "squashing hooked syscall at "PFX, pc);
+            LOG(GLOBAL, LOG_ASYNCH, 2, 
+                "intercept_syscall_wrapper: squashing hooked syscall %02x at "PFX"\n", 
+                native_sys_num, pc);
+#ifdef X64
+            /* in this case we put our hook at the 1st instr */
+            instrlist_append(&ilist,
+                             INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_R10),
+                                                 opnd_create_reg(REG_RCX)));
+#endif            
+            /* we normally ASSERT that 1st instr is always mov imm -> eax */
+            instrlist_append(&ilist,
+                             INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_EAX), 
+                                                  OPND_CREATE_INT32(native_sys_num)));
+            /* FIXME: even if we detach we don't restore the original
+             * values, since what we have here should be good enough
+             */
+            /* skip original instruction */
+            instr_destroy(dcontext, instr);
+        } else if (DYNAMO_OPTION(native_exec_hook_conflict) == HOOKED_TRAMPOLINE_HOOK_DEEPER) {
+            /* move our hook one instruction deeper assuming hooker will
+             * return to right after the hook, verify that's an 
+             * instruction boundary */
+#ifdef X64
+            /* not much room for two hooks before the syscall; we don't support
+             * for now */
+            ASSERT_NOT_REACHED();
+            FATAL_USAGE_ERROR(TAMPERED_NTDLL, 2, get_application_name(),
+                              get_application_pid());
+#else
+            ASSERT(instr_length(dcontext, instr) == 5 /* length of normal mov_imm */);
+            *ptgt_pc = pc;
+            tgt_pc = pc;
+            /* skip original instruction */
+            instr_destroy(dcontext, instr);
+#endif
+        } else {
+            ASSERT_NOT_REACHED();
+            FATAL_USAGE_ERROR(TAMPERED_NTDLL, 2, get_application_name(),
+                              get_application_pid());
+        }
+    } else {
+#ifdef X64
+        /* first instr is mov rcx -> r10, which we skip to reach the 5-byte mov immed */
+        ASSERT(instr_get_opcode(instr) == OP_mov_ld &&
+               opnd_is_reg(instr_get_src(instr, 0)) &&
+               opnd_get_reg(instr_get_src(instr, 0)) == REG_RCX &&
+               opnd_is_reg(instr_get_dst(instr, 0)) &&
+               opnd_get_reg(instr_get_dst(instr, 0)) == REG_R10);
+        /* we hook after the 1st instr.  will this confuse other hookers who
+         * will think there currently is no hook b/c not on 1st instr? */
+        *ptgt_pc = pc;
+        tgt_pc = pc;
+        instr_destroy(dcontext, instr);
+        /* now decode the 2nd instr which should be a mov immed */
+        DOLOG(3, LOG_ASYNCH, { disassemble_with_bytes(dcontext, pc, main_logfile); });
+        instr = instr_create(dcontext);
+        pc = decode(dcontext, pc, instr);
+        ASSERT(instr_length(dcontext, instr) == 5 /* length of normal mov_imm */);
+        opcode = instr_get_opcode(instr);
+        /* now fall through */
+#endif
+        /* normally a mov eax, native_sys_num */
+        ASSERT(instr_get_opcode(instr) == OP_mov_imm);
+        ASSERT(opnd_get_immed_int(instr_get_src(instr, 0)) == native_sys_num);
+        LOG(GLOBAL, LOG_ASYNCH, 3, "intercept_syscall_wrapper: hooked syscall %02x at "PFX"\n", 
+            native_sys_num, pc);
+        /* append instruction (non-CTI) */
+        instrlist_append(&ilist, instr);
+    }
+
+#ifdef X64
+    /* 3rd instr: syscall */
+    instr = instr_create(dcontext);
+    after_hook_target = pc;
+    pc = decode(dcontext, pc, instr);
+    ret_pc = pc;
+    ASSERT(instr_get_opcode(instr) == OP_syscall);
+    instr_destroy(dcontext, instr);
+#else
+    /* second instr is either a lea, a mov immed, or an xor */
+    DOLOG(3, LOG_ASYNCH, { disassemble_with_bytes(dcontext, pc, main_logfile); });
+    instr = instr_create(dcontext);
+    pc = decode(dcontext, pc, instr);
+    instrlist_append(&ilist, instr);
+    opcode = instr_get_opcode(instr);
+    if (get_syscall_method() == SYSCALL_METHOD_WOW64) {
+        ASSERT(opcode == OP_xor || opcode == OP_mov_imm);
+        /* third instr is a lea */
+        instr = instr_create(dcontext);
+        pc = decode(dcontext, pc, instr);
+        ASSERT(instr_get_opcode(instr) == OP_lea);
+        instrlist_append(&ilist, instr);
+
+        /* fourth instr is a call*, what we consider the system call instr */
+        after_hook_target = pc;
+        instr = instr_create(dcontext);
+        ret_pc = decode(dcontext, pc, instr); /* skip call* to skip syscall */
+        ASSERT(instr_get_opcode(instr) == OP_call_ind);
+        instr_destroy(dcontext, instr);
+    } else if (opcode == OP_mov_imm) {
+        ptr_int_t immed = opnd_get_immed_int(instr_get_src(instr, 0));
+        ASSERT(PAGE_START(immed) == (ptr_uint_t) VSYSCALL_PAGE_START_BOOTSTRAP_VALUE);
+        ASSERT(get_syscall_method() == SYSCALL_METHOD_SYSENTER);
+        ASSERT(get_os_version() >= WINDOWS_VERSION_XP);
+
+        /* third instr is an indirect call */
+        instr = instr_create(dcontext);
+        pc = decode(dcontext, pc, instr);
+        ret_pc = pc;
+        ASSERT(instr_get_opcode(instr) == OP_call_ind);
+        if (fpo_stack_adjustment != NULL) {
+            /* for GBOP case 7127 */
+            *fpo_stack_adjustment = 4;
+        }
+        /* replace the call w/ a push */
+        instrlist_append(&ilist, INSTR_CREATE_push_imm
+                         (dcontext, OPND_CREATE_INTPTR((ptr_int_t)pc)));
+
+        /* the callee, either on vsyscall page or at KiFastSystemCall */
+        if (opnd_is_reg(instr_get_src(instr, 0)))
+            pc = (byte *) immed;
+        else /* KiFastSystemCall */
+            pc = *((byte **)immed);
+
+        /* fourth instr: mov %xsp -> %xdx */
+        instr_reset(dcontext, instr); /* re-use ind call container */
+        pc = decode(dcontext, pc, instr);
+        instrlist_append(&ilist, instr);
+        ASSERT(instr_get_opcode(instr) == OP_mov_ld);
+
+        /* fifth instr: sysenter */
+        instr = instr_create(dcontext);
+        after_hook_target = pc;
+        pc = decode(dcontext, pc, instr);
+        ASSERT(instr_get_opcode(instr) == OP_sysenter);
+        instr_destroy(dcontext, instr);
+
+        /* ignore ret after sysenter, we'll return to ret after call */
+    } else {
+        ASSERT(opcode == OP_lea);
+        /* third instr: int 2e */
+        instr = instr_create(dcontext);
+        ret_pc = decode(dcontext, pc, instr);
+        ASSERT(instr_get_opcode(instr) == OP_int);
+        /* if we hooked deeper, will need to hook over the int too */
+        if (pc - tgt_pc < 5 /* length of our hook */) {
+            /* Need to add an int 2e to the return path since hook clobbered
+             * the original one. We use create_syscall_instr(dcontext) for
+             * the sygate int fix. FIXME - the pc will now show up as
+             * after_do/share_syscall() but should be ok since anyone
+             * checking for those on this thread should have already checked
+             * for it being native. */
+            hook_return_instr = create_syscall_instr(dcontext);
+            after_hook_target = ret_pc;
+            ASSERT(DYNAMO_OPTION(native_exec_hook_conflict) == 
+                   HOOKED_TRAMPOLINE_HOOK_DEEPER);
+        } else {
+            /* point after_hook_target to int 2e */
+            after_hook_target = pc;
+        }
+        instr_destroy(dcontext, instr);
+    }
+#endif
+
+    pc = interception_cur_pc; /* current spot in interception buffer */
+
+    /* copy original 5 bytes to ease unhooking, we won't execute this */
+    *orig_bytes_pc = pc;
+    memcpy(pc, tgt_pc, 5);
+    pc += 5;
+
+    emit_pc = pc;
+    /* we assume that interception buffer is still writable */
+
+    /* we need to enter at copy of pre-syscall sequence, since we need
+     * callee to be at app state exactly prior to syscall instr itself.
+     * this means this sequence is executed natively even for syscalls
+     * in the cache (since interception code is run natively) -- only
+     * worry would be stack faults, whose context we might xlate incorrectly
+     *
+     * N.B.: bb_process_ubr() assumes that the target of the trampoline
+     * is the original mov immed!
+     */
+    entrance_pc = pc;
+
+    /* insert our copy of app instrs leading up to syscall
+     * first instr doubles as the clobbered original code for un-intercepting.
+     */
+    for (instr = instrlist_first(&ilist); instr != NULL; instr = instr_get_next(instr)) {
+        pc = instr_encode(dcontext, instr, pc);
+        ASSERT(pc != NULL);
+    }
+    instrlist_clear(dcontext, &ilist);
+
+    pc = emit_intercept_code(dcontext, pc, prof_func, callee_arg,
+                             false /*do not assume xsp*/, action_after,
+                             ret_pc /* alternate target to skip syscall */, NULL);
+
+    /* Map interception buffer PCs to original app PCs */
+    if (is_in_interception_buffer(pc))
+        map_intercept_pc_to_app_pc(pc, tgt_pc, 10 /* 5 bytes + jmp back */);
+
+    /* The normal target, for really doing the system call native, used
+     * for letting go normally and for take over.
+     * We already did pre-syscall sequence, so we go straight to syscall itself.
+     */
+    /* have to include syscall instr here if we ended up hooking over it */
+    if (hook_return_instr != NULL) {
+        pc = instr_encode(dcontext, hook_return_instr, pc);
+        ASSERT(pc != NULL);
+        instr_destroy(dcontext, hook_return_instr);
+    }
+    *pc = JMP_REL32_OPCODE; pc++;
+#ifdef X64
+    if (!REL32_REACHABLE(pc + 4, after_hook_target)) {
+        ASSERT_NOT_IMPLEMENTED(false && "PR 245169: hook target too far: NYI");
+        /* FIXME PR 245169: we need use landing_pad_areas to alloc landing
+         * pads to trampolines, as done for PR 250294.
+         */
+    }
+#endif
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(after_hook_target - pc - 4)));
+    *((int *)pc) = (int)(ptr_int_t) (after_hook_target - pc - 4);
+    pc += 4;
+
+    /* update interception buffer pc */
+    interception_cur_pc = pc;
+
+    /* Replace original code with jmp to our version's entrance */
+    /* copy-on-write will give us a copy of this page */
+    ok = make_hookable(tgt_pc, 5, &changed_prot);
+    if (ok) {
+        ptr_int_t offset = (entrance_pc - (tgt_pc + 5));
+#ifdef X64
+        if (!REL32_REACHABLE_OFFS(offset)) {
+            ASSERT_NOT_IMPLEMENTED(false && "PR 245169: hook target too far: NYI");
+            /* FIXME PR 245169: we need use landing_pad_areas to alloc landing
+             * pads to trampolines, as done for PR 250294.
+             */
+        }
+#endif
+        pc = tgt_pc;
+        *pc = JMP_REL32_OPCODE;
+        pc++;
+        IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(offset)));
+        *((int *)pc) = (int) offset;
+    }
+    /* make our page unwritable now */
+    make_unhookable(tgt_pc, 5, changed_prot);
+
+    if (skip_syscall_pc != NULL)
+        *skip_syscall_pc = ret_pc;
+
+    return emit_pc;
+}
+
+/* two convenience routines for intercepting using the code[] buffer
+ * after the initialization routine has completed
+ *
+ * WARNING: only call this when there is only one thread going!
+ * This is not thread-safe!
+ */
+byte *
+insert_trampoline(byte *tgt_pc, intercept_function_t prof_func,
+                  void *callee_arg, bool assume_xsp, after_intercept_action_t action_after,
+                  bool cti_safe_to_ignore)
+{
+    byte *pc = interception_cur_pc;
+    /* make interception code writable, NOTE the interception code may
+     * be in vmareas executable list, we make the interception code temporarily
+     * writable here without removing or flushing the region, this is ok since
+     * we should be single threaded when this function is called and we never
+     * overwrite existing interception code */
+    DEBUG_DECLARE(bool ok =)
+        make_writable(interception_code, INTERCEPTION_CODE_SIZE);
+    ASSERT(ok);
+
+    /* FIXME: worry about inserting trampoline across bb boundaries? */
+    interception_cur_pc = intercept_call(interception_cur_pc, tgt_pc, prof_func, callee_arg,
+                                         assume_xsp, action_after, 
+                                         false, /* need the trampoline at all costs */
+                                         cti_safe_to_ignore, NULL, NULL);
+    /* FIXME: we assume early intercept_call failures are ok to
+     * ignore.  Note we may want to crash instead if trying to sandbox
+     * malicious programs that may be able to prevent us from
+     * committing memory.
+     */
+
+    ASSERT(interception_cur_pc - interception_code < INTERCEPTION_CODE_SIZE);
+
+    /* return interception code to read only state */
+    make_unwritable(interception_code, INTERCEPTION_CODE_SIZE);
+
+    return pc;
+}
+
+void
+remove_trampoline(byte *our_pc, byte *tgt_pc)
+{
+    un_intercept_call(our_pc, tgt_pc);
+}
+
+bool
+is_in_interception_buffer(byte *pc)
+{
+    return (pc >= interception_code &&
+            pc < interception_code + INTERCEPTION_CODE_SIZE);
+}
+
+bool
+is_syscall_trampoline(byte *pc)
+{
+    return (syscall_trampolines_start != NULL &&
+            pc >= syscall_trampolines_start && pc < syscall_trampolines_end);
+}
+
+/****************************************************************************
+ */
+/* TRACK_NTDLL: try to find where kernel re-emerges into user mode when it
+ * dives into kernel mode
+ */
+#if TRACK_NTDLL
+static byte *
+make_writable_incr(byte *pc)
+{
+    PBYTE pb = (PBYTE) pc;
+    MEMORY_BASIC_INFORMATION mbi;
+    DWORD old_prot;
+    int res;
+
+    res = query_virtual_memory(pb, &mbi, sizeof(mbi));
+    ASSERT(res == sizeof(mbi));
+
+    res = protect_virtual_memory(mbi.BaseAddress, mbi.RegionSize,
+                                 PAGE_EXECUTE_WRITECOPY, &old_prot);
+    ASSERT(res);
+    return (byte *)((int)mbi.BaseAddress + (int)mbi.RegionSize);
+}
+
+static byte *
+make_inaccessible(byte *pc)
+{
+    PBYTE pb = (PBYTE) pc;
+    MEMORY_BASIC_INFORMATION mbi;
+    DWORD old_prot;
+    int res;
+
+    res = query_virtual_memory(pb, &mbi, sizeof(mbi));
+    ASSERT(res == sizeof(mbi));
+
+    res = protect_virtual_memory(mbi.BaseAddress, mbi.RegionSize,
+                                 PAGE_NOACCESS, &old_prot);
+    ASSERT(res);
+    return (byte *)((int)mbi.BaseAddress + (int)mbi.RegionSize);
+}
+
+void
+wipe_out_ntdll()
+{
+    byte * start = (byte *) 0x77F81000;
+    byte * stop  = (byte *) 0x77FCD95B;
+    byte *pc;
+
+    /* first suspend all other threads */
+    thread_record_t **threads;
+    int i, num_threads;
+    mutex_lock(&thread_initexit_lock);
+    get_list_of_threads(&threads, &num_threads);
+    for (i=0; i<num_threads; i++) {
+        if (threads[i]->id != get_thread_id()) {
+            LOG(GLOBAL, LOG_ASYNCH, 1, "Suspending thread %d == "PFX"\n", 
+                tr->id, tr->handle);
+            SuspendThread(threads[i]->handle);
+        }
+    }
+    mutex_unlock(&thread_initexit_lock);
+    global_heap_free(threads, num_threads*sizeof(thread_record_t*)
+                     HEAPACCT(ACCT_THREAD_MGT));
+
+    LOG(GLOBAL, LOG_ASYNCH, 1, "INVALIDATING ENTIRE NTDLL.DLL!!!\n");
+    pc = start;
+    while (pc < stop) {
+        LOG(GLOBAL, LOG_ASYNCH, 1, "\t"PFX"\n", pc);
+#if 0
+        pc = make_inaccessible(pc);
+#else
+        pc = make_writable_incr(pc);
+#endif
+    }
+#if 1
+    for (pc=start; pc<stop; pc++) {
+        *pc = 0xcc;
+    }
+#endif
+}
+#endif /* TRACK_NTDLL */
+/*
+ ****************************************************************************/
+
+/* If we receive an asynch event while we've lost control but before we
+ * reach the image entry point or our other retakeover points we should
+ * retakeover, to minimize the amount of code run natively -- these should
+ * be rare during init and perf hit of repeated flushing and re-walking
+ * memory list shouldn't be an issue.
+ * Separated from asynch_take_over to not force its callers to do this.
+ */
+static inline void
+asynch_retakeover_if_native()
+{
+    thread_record_t *tr = thread_lookup(get_thread_id());
+    ASSERT(tr != NULL);
+    if (tr->under_dynamo_control == UNDER_DYN_HACK) {
+        ASSERT(!reached_image_entry_yet());
+        /* must do a complete takeover-after-native */
+        retakeover_after_native(tr, INTERCEPT_EARLY_ASYNCH);
+    }
+}
+
+/* This routine is called by a DynamoRIO routine that was invoked natively,
+ * i.e., not under DynamoRIO control.
+ * This routine takes control using the application state in its arguments,
+ * and starts execution under DynamoRIO at start_pc.
+ * state->callee_arg is a boolean "save_dcontext":
+ * If save_dcontext is true, it saves the cur dcontext on the callback stack
+ * of dcontexts and proceeds to execute with a new dcontext.
+ * Otherwise, it uses the current dcontext, which has its trace squashed.
+ */
+static void
+asynch_take_over(app_state_at_intercept_t *state)
+{
+    dcontext_t *dcontext;
+    bool save_dcontext = (bool)(ptr_uint_t) state->callee_arg;
+    if (save_dcontext) {
+        /* save cur dcontext and get a new one */
+        dcontext = callback_setup(state->start_pc);
+    } else {
+        dcontext = get_thread_private_dcontext();
+        ASSERT(dcontext->initialized);
+        /* case 9347 we want to let go after image entry point */
+        if (RUNNING_WITHOUT_CODE_CACHE() 
+            && dcontext->next_tag == BACK_TO_NATIVE_AFTER_SYSCALL
+            && state->start_pc == image_entry_pc) {
+            ASSERT(dcontext->native_exec_postsyscall == image_entry_pc);
+        } else {
+            ASSERT(!RUNNING_WITHOUT_CODE_CACHE());
+            dcontext->next_tag = state->start_pc;
+        }
+        /* if we were building a trace, kill it */
+        if (is_building_trace(dcontext)) {
+            LOG(THREAD, LOG_ASYNCH, 2, "asynch_take_over: squashing old trace\n");
+            trace_abort(dcontext);
+        }
+    }
+    LOG(THREAD, LOG_ASYNCH, 2, "asynch_take_over 0x%08x\n", state->start_pc);
+    /* may have been inside syscall...now we're in app! */
+    set_at_syscall(dcontext, false);
+    /* tell dispatch() why we're coming there */
+    if (dcontext->whereami != WHERE_APP) /* new thread, typically: leave it that way */
+        dcontext->whereami = WHERE_TRAMPOLINE;
+    set_last_exit(dcontext, (linkstub_t *) get_asynch_linkstub());
+
+    transfer_to_dispatch(dcontext, state->app_errno, &state->mc);
+    ASSERT_NOT_REACHED();
+}
+
+static void
+possible_new_thread_wait_for_dr_init()
+{
+    /* Because of problems with injected threads while we are initializing
+     * (case 5167, 5020, 5103 bunch of others) we block here while the main
+     * thread finishes initializing. Once dynamo_exited is set is safe to
+     * let the thread continue since dynamo_thread_init will imediately
+     * return. */
+    while (!dynamo_initialized && !dynamo_exited) {
+        STATS_INC(apc_yields_while_initializing);
+        thread_yield();
+    }
+}
+
+/* returns true if intercept function should return immediately and let go,
+ * false if intercept function should continue processing and maybe takeover */
+static bool
+intercept_new_thread(CONTEXT *cxt)
+{
+    /* init apc, check init_apc_go_native to sync w/detach */
+    if (init_apc_go_native) {
+        /* need to wait after checking _go_native to avoid a thread 
+         * going native to early because of races between setting
+         * _go_native and _pause */
+        if (init_apc_go_native_pause) {
+            /* FIXME : this along with any other logging in this 
+             * method could potentially be race condition with detach 
+             * cleanup, though is unlikely */
+            LOG(GLOBAL, LOG_ALL, 2, 
+                "Thread waiting at init_apc for detach to finish\n");
+        }
+        while (init_apc_go_native_pause) {
+            thread_yield();
+        }
+        /* just return, FIXME : see concerns in detach_helper about
+         * getting to native code before the interception_code is 
+         * freed and getting out of here before the dll is unloaded
+         */
+# if 0 /* this is not a dynamo controlled thread! */
+        SELF_PROTECT_LOCAL(get_thread_private_dcontext(), READONLY);
+# endif
+        return true /* exit intercept function and let go */;
+    }
+
+    /* should keep in sync with changes in intercept_image_entry() for
+     * thread initialization
+     */
+
+    /* initialize thread now */
+    if (dynamo_thread_init() != -1) {
+        app_pc thunk_xip = (app_pc)cxt->CXT_XIP;
+        dcontext_t *dcontext = get_thread_private_dcontext();
+        LOG_DECLARE(char sym_buf[MAXIMUM_SYMBOL_LENGTH];)
+        bool is_nudge_thread = false;
+
+        /* Xref case 552, to ameliorate the risk of an attacker
+         * leveraging our detach routines etc. against us, we detect
+         * an incoming nudge thread here during thread init and set
+         * a dcontext flag that the nudge routines can later verify.
+         * Attacker could still bypass if can control the start addr
+         * of a new thread (FIXME).  We check both Xax and Xip since
+         * nodemgr has the ability to target directly or send through
+         * kernel32 start thunk (though only start thunk, i.e. xax,
+         * is currently used).  If we move to just directly targeted,
+         * i.e. xip, would be a lot harder for the attacker since
+         * the documented API routines all hardcode that value.
+         *
+         * The nudge related checks below were moved above thread_policy checks
+         * because there is no dependency and because process control nudge for
+         * thin_client needs it; part of cases 8884, 8594 & 8888. */
+        ASSERT(dcontext != NULL && dcontext->nudge_target == NULL);
+        if ((void *)cxt->CXT_XIP == (void *)generic_nudge_target ||
+            (void *)cxt->CXT_XAX == (void *)generic_nudge_target) {
+            LOG(THREAD, LOG_ALL, 1, "Thread targeting nudge.\n");
+            if (dcontext != NULL) {
+                dcontext->nudge_target = (void *)generic_nudge_target;
+            }
+            is_nudge_thread = true;
+        }
+        /* FIXME: temporary fix for case 9467 - mute nudges for cygwin apps.
+         * Long term fix is to make nudge threads go directly to their targets.
+         */
+        if (is_nudge_thread && DYNAMO_OPTION(thin_client) && DYNAMO_OPTION(mute_nudge)) {
+            TRY_EXCEPT(dcontext, { /* to prevent crashes when walking the ldr list */
+                PEB *peb = get_own_peb();
+                PEB_LDR_DATA *ldr = peb->LoaderData;
+                LIST_ENTRY *e;
+                LIST_ENTRY *start = &ldr->InLoadOrderModuleList;
+                LDR_MODULE *mod;
+                uint traversed = 0;
+
+                /* Note: this loader module list walk is racy with the loader;
+                 * can't really grab the loader lock here.  Shouldn't be a big
+                 * problem as this is a temp fix anyway. */
+                for (e = start->Flink; e != start; e = e->Flink) {
+                    mod = (LDR_MODULE *) e;
+                    if (wcsstr(mod->BaseDllName.Buffer, L"cygwin1.dll") != NULL) {
+                        os_terminate(dcontext, TERMINATE_THREAD|TERMINATE_CLEANUP);
+                        ASSERT_NOT_REACHED();
+                    }
+                    if (traversed++ > MAX_MODULE_LIST_INFINITE_LOOP_THRESHOLD) {
+                        SYSLOG_INTERNAL_WARNING("nudge muting: too many modules");
+                        break;
+                    }
+                }
+            }, {  /* do nothing */  });
+        }
+
+        /* For thin_client, let go right after we init the thread, i.e., create
+         * the dcontext; don't do the thread policy stuff, that requires locks
+         * that aren't initialized in this mode!  */
+        if (DYNAMO_OPTION(thin_client))
+            return true /* exit intercept function and let go */;
+
+        /* In fact the apc_target is ntdll!LdrInitializeThunk
+         * (for all threads not only the first one).
+         * Note for vista that threads do not start with an apc, but rather
+         * directly show up at ntdll!LdrInitializeThunk (which we hook on
+         * vista to call this routine).  Note that the thunk will return via
+         * an NtContinue to a context on the stack so really we see the same
+         * behavior as before except we don't go through the apc dispatcher.
+         *
+         * For threads created by kernel32!CreateRemoteThread pre vista
+         * the cxt->Xip then is kernel32!Base{Process,Thread}StartThunk (not exported),
+         * while the cxt->Xax is the user thread procedure and cxt->Xbx is the arg.
+         * On vista it's the same except cxt->Xip is set to ntdll!RtlUserThreadStart
+         * (which is exported in ntdll.dll) by the kernel.
+         *
+         * kernel32!BaseProcessStartThunk, or kernel32!BaseThreadStartThunk 
+         * on all versions I've tested start with  
+         * 0xed33  xor ebp,ebp
+         * 
+         * Note, of course, that direct NtCreateThread calls
+         * can go anywhere they want (including on Vista).  For example toolhelp
+         * uses NTDLL!RtlpQueryProcessDebugInformationRemote
+         * as the xip so shouldn't count much on this. NtCreateThreadEx threads
+         * (vista only) will, however, always have xip=ntdll!RtlUserThreadStart
+         * since the kernel sets that.
+         */
+        /* keep in mind this is a 16-bit match */
+#define BASE_THREAD_START_THUNK_USHORT  0xed33
+#define THREAD_START_ADDR IF_X64_ELSE(CXT_XCX, CXT_XAX)
+#define THREAD_START_ARG  IF_X64_ELSE(CXT_XDX, CXT_XBX)
+        
+        /* see comments in os.c pre_system_call CreateThread, Xax holds
+         * the win32 start address (Nebbett), Xbx holds the argument
+         * (observation). Same appears to hold for CreateThreadEx. */
+        /* Note that the initial thread won't log here */
+        LOG(THREAD_GET, LOG_THREADS, 1,
+            "New Thread : Win32 start address "PFX" arg "PFX", thunk xip="PFX"\n",
+            cxt->THREAD_START_ADDR, cxt->THREAD_START_ARG, cxt->CXT_XIP);
+        DOLOG(1, LOG_THREADS, {
+            print_symbolic_address((app_pc)cxt->THREAD_START_ADDR, sym_buf, sizeof(sym_buf),
+                                   false);
+            LOG(THREAD_GET, LOG_THREADS, 1,
+                "Symbol information for start address %s\n", sym_buf);
+        });
+        DOLOG(2, LOG_THREADS, {
+            print_symbolic_address((app_pc)cxt->CXT_XIP, sym_buf, sizeof(sym_buf),
+                                   false);
+            LOG(THREAD_GET, LOG_THREADS, 2,
+                "Symbol information for thunk address %s\n", sym_buf);
+        });
+        
+        /* start address should be set at thread initialization */
+        if (dcontext->win32_start_addr == (app_pc)cxt->THREAD_START_ARG) {
+            /* case 10965/PR 215400: WOW64 & x64 query returns arg for some reason */
+#ifndef X64
+            ASSERT(is_wow64_process(NT_CURRENT_PROCESS));
+#endif
+            dcontext->win32_start_addr = (app_pc)cxt->THREAD_START_ADDR;
+        }
+        ASSERT(dcontext->win32_start_addr == (app_pc)cxt->THREAD_START_ADDR);
+
+#ifdef PROGRAM_SHEPHERDING
+        /* We expect target address (xip) to be on our executable list 
+         * (is usually one of the start thunks). */ 
+        ASSERT_CURIOSITY(executable_vm_area_overlap(thunk_xip, thunk_xip+2, false));
+        /* On vista it appears all new threads target RtlUserThreadStart
+         * (the kernel sets in in NtCreateThreadEx). Thread created via the legacy
+         * NtCreateThread however can target anywhere (such as our internal nudges). */
+        ASSERT_CURIOSITY(get_os_version() != WINDOWS_VERSION_VISTA ||
+                         is_nudge_thread || thunk_xip == RtlUserThreadStart ||
+                         /* The security_win32/execept-execution.exe regr test does a
+                          * raw create thread using the old NtCreateThread syscall. */
+                         check_filter("except-execution.exe",
+                                      get_short_name(get_application_name())));
+        /* check for hooker's shellcode delivered via a remote thread */
+        if (TEST(OPTION_ENABLED, DYNAMO_OPTION(thread_policy))) {
+            /* Most new threads (and all of the ones that target injected code
+             * so far) have xip targeting one of the start thunks.  For these
+             * threads the start address we want to apply the policy to is in
+             * eax.  However we don't want to apply the policy to the random
+             * value in eax if the thread isn't targeting a start thunk (such
+             * as injected toolhelp [RtlpQueryProcessDebugInformationRemote]
+             * or debugger threads).  For Vista we can check that xip = 
+             * RtlUserThreadStart, but the kernel32 thunks used pre Vista
+             * aren't exported so as a sanity check for those we check if
+             * the first few bytes of xip match the kernel32 start thunks.
+             * FIXME - this is only a 2 byte comparison (xor ebp,ebp) so a
+             * false match is certainly not impossible.  We should try to find
+             * a better way to check. Could also check that it's in kernel32
+             * etc.
+             * FIXME - the deref of cxt->CXT_XIP is potentially unsafe, we
+             * assume it's ok since is on the executable list and the thread is
+             * about to execute from there.  Should prob. use safe_read()
+             * FIXME - for this to work we're also assuming that the thunks
+             * will always be on the executable list.
+             */
+            if (executable_vm_area_overlap(thunk_xip, thunk_xip+2, false) &&
+                (get_os_version() == WINDOWS_VERSION_VISTA ?
+                 thunk_xip == RtlUserThreadStart :
+                 BASE_THREAD_START_THUNK_USHORT == *(ushort*)thunk_xip)) {
+                apc_thread_policy_helper((app_pc *)&cxt->THREAD_START_ADDR,
+                                         /* target code is in CONTEXT structure */
+                                         DYNAMO_OPTION(thread_policy),
+                                         THREAD_TARGET_WINDOWS
+                                         /* CreateThreadEx target */);
+            }
+            /* FIXME - threads can directly target new code without going
+             * through one of the start thunks (though for our purposes here
+             * that's the uncommon case) so we should consider also applying
+             * the thread policy to the cxt->CXT_XIP address. Doesn't apply
+             * to threads created by NtCreateThreadEx though since they will
+             * always go through the RtlUserThreadStart thunk. */
+        }
+#endif /* PROGRAM_SHEPHERDING */
+#ifdef HOT_PATCHING_INTERFACE
+        /* For hotp_only, this is where newly created threads should 
+         * be let go native, i.e., do the thread_policy enforcement. 
+         */
+        if (DYNAMO_OPTION(hotp_only))
+            return true /* exit intercept function and let go */;
+#endif
+    } else {
+        ASSERT_NOT_REACHED();
+    }
+    return false /* continue intercept function and maybe takeover */;
+}
+
+/****************************************************************************
+ * New Threads
+ * On os_versions prior to Vista new threads start KiUserApcDispatcher with an
+ * APC to LdrInitializeThunk.  We catch those with our KiUserApcDispatcher
+ * hook. On Vista new threads skip the dispatcher and go directly to 
+ * LdrInitializeThunk (stack is similar to APC, i.e. does an NtContinue to
+ * go on) so we need to hook there to catch new threads.
+ */
+
+/*
+<Vista, note other platforms differ>
+ntdll!LdrInitializeThunk:
+  77F40229: 8B FF              mov         edi,edi
+  77F4022B: 55                 push        ebp
+  77F4022C: 8B EC              mov         ebp,esp
+  77F4022E: FF 75 0C           push        dword ptr [ebp+0Ch]
+  77F40231: FF 75 08           push        dword ptr [ebp+8]
+  77F40234: E8 6B 10 00 00     call        ntdll!LdrpInitialize (77F412A4)
+  77F40239: 6A 01              push        1
+  77F4023B: FF 75 08           push        dword ptr [ebp+8]
+  77F4023E: E8 78 2E FF FF     call        ntdll!NtContinue (77F330BB)
+  77F40243: 50                 push        eax
+  77F40244: E8 EF 49 FF FF     call        ntdll!RtlRaiseStatus (77F34C38)
+  77F40249: CC                 int         3
+  77F4024A: 90                 nop
+
+ * At interception point esp+4 holds the new threads context (first arg, rcx on 64-bit).
+ * FIXME - need code to try and verify this offset during hooking.
+ */
+#define LDR_INIT_CXT_XSP_OFFSET 0x4
+
+static after_intercept_action_t  /* note return value will be ignored */
+intercept_ldr_init(app_state_at_intercept_t *state)
+{
+    /* we only hook this routine on vista */
+    ASSERT(get_os_version() == WINDOWS_VERSION_VISTA);
+
+    /* this might be a new thread */
+    possible_new_thread_wait_for_dr_init();
+
+    if (intercept_asynch_for_self(true/*we want unknown threads*/)) {
+        CONTEXT *cxt;
+#ifdef X64
+        cxt = (CONTEXT *)(state->mc.xcx);
+#else
+        cxt = (*(CONTEXT **)(state->mc.xsp + LDR_INIT_CXT_XSP_OFFSET));
+#endif
+
+        if (!is_thread_initialized()) {
+            if (intercept_new_thread(cxt))
+                return AFTER_INTERCEPT_LET_GO;
+        } else {
+            /* ntdll!LdrInitializeThunk is only used for initializing new
+             * threads so we should never get here */
+            ASSERT_NOT_REACHED();
+        }
+        asynch_retakeover_if_native(); /* FIXME - this is unneccesary */
+        state->callee_arg = (void *) false /* use cur dcontext */;
+        asynch_take_over(state);
+    } else {
+        /* ntdll!LdrInitializeThunk is only used for initializing new
+         * threads so we should never get here */
+        ASSERT_NOT_REACHED();
+    }
+
+    return AFTER_INTERCEPT_LET_GO;
+}
+
+/****************************************************************************
+ * APCs
+ *
+ * Interception routine for an Asynchronous Procedure Call
+ * We intercept this point in ntdll:
+
+KiUserApcDispatcher:
+  77F9F028: 8D 7C 24 10        lea         edi,[esp+10h]
+  77F9F02C: 58                 pop         eax
+  77F9F02D: FF D0              call        eax
+  77F9F02F: 6A 01              push        1
+  77F9F031: 57                 push        edi
+  77F9F032: E8 BC 30 FE FF     call        77F820F3 <NtContinue>
+  77F9F037: 90                 nop
+
+2003 SP1 looks a little different, w/ SEH code
+Vista is similar
+KiUserApcDispatcher:
+  7c8362c8 8d8424dc020000   lea     eax,[esp+0x2dc]
+  7c8362cf 648b0d00000000   mov     ecx,fs:[00000000]
+  7c8362d6 baab62837c       mov     edx,0x7c8362ab (KiUserApcExceptionHandler)
+  7c8362db 8908             mov     [eax],ecx
+  7c8362dd 895004           mov     [eax+0x4],edx
+  7c8362e0 64a300000000     mov     fs:[00000000],eax
+  7c8362e6 58               pop     eax
+  7c8362e7 8d7c240c         lea     edi,[esp+0xc]
+  7c8362eb ffd0             call    eax
+  7c8362ed 8b8fcc020000     mov     ecx,[edi+0x2cc]
+  7c8362f3 64890d00000000   mov     fs:[00000000],ecx
+  7c8362fa 6a01             push    0x1
+  7c8362fc 57               push    edi
+  7c8362fd e88328ffff       call    ntdll!NtContinue (7c828b85)
+  7c836302 8bf0             mov     esi,eax
+  7c836304 56               push    esi
+  7c836305 e88e010000       call    ntdll!RtlRaiseStatus (7c836498)
+  7c83630a ebf8             jmp     ntdll!KiUserApcDispatcher+0x3c (7c836304)
+  7c83630c c21000           ret     0x10
+
+x64 XP, where they use the PxHome CONTEXT fields:
+ntdll!KiUserApcDispatcher:
+  00000000`78ef3910 488b0c24        mov     rcx,qword ptr [rsp]
+  00000000`78ef3914 488b542408      mov     rdx,qword ptr [rsp+8]
+  00000000`78ef3919 4c8b442410      mov     r8,qword ptr [rsp+10h]
+  00000000`78ef391e 4c8bcc          mov     r9,rsp
+  00000000`78ef3921 ff542418        call    qword ptr [rsp+18h]
+  00000000`78ef3925 488bcc          mov     rcx,rsp
+  00000000`78ef3928 b201            mov     dl,1
+  00000000`78ef392a e861ddffff      call    ntdll!NtContinue (00000000`78ef1690)
+  00000000`78ef392f 85c0            test    eax,eax
+  00000000`78ef3931 74dd            je      ntdll!KiUserApcDispatcher (00000000`78ef3910)
+  00000000`78ef3933 8bf0            mov     esi,eax
+  00000000`78ef3935 8bce            mov     ecx,esi
+  00000000`78ef3937 e834f90500      call    ntdll!RtlRaiseException+0x10d (00000000`78f53270)
+  00000000`78ef393c cc              int     3
+  00000000`78ef393d 90              nop
+  00000000`78ef393e ebf7            jmp     ntdll!KiUserApcDispatch+0x27 (00000000`78ef3937)
+  00000000`78ef3940 cc              int     3
+
+
+FIXME case 6395/case 6050: what are KiUserCallbackExceptionHandler and
+KiUserApcExceptionHandler, added on 2003 sp1?  We're assuming not
+entered from kernel mode despite Ki prefix.  They are not exported
+entry points.
+Case 10579: KiUserCallbackExceptionHandler is used to pop the kernel
+cb stack when an exception will abandon that cb frame.
+
+ * The target APC function pointer is the would-be return address for
+ * this routine.  The first argument is in fact the argument of the
+ * APC
+ *
+ * ASSUMPTIONS:
+ *    1) *(esp+0x0) == PKNORMAL_ROUTINE ApcRoutine 
+ *       (IN PVOID NormalContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2)
+ *      call* native target 
+ *      The next three arguments on the stack are just passed through
+ *      to this function, and are the arguments passed by 
+ *      NtQueueApcThread(thread, ApcRoutine, ApcContext, Argument1, Argument2)
+ *
+ *      On XP SP2 user mode APCs target kernel32!BaseDispatchAPC,
+ *      and the following arguments have been observed to be:
+ *    2') *(esp+0x4) == ApcContext 
+ *        call* Win32 target PAPCFUNC for user mode APCs
+ *    FIXME:  need to check the other platforms - it is completely
+ *      up to the caller
+ *    3') *(esp+0x8) == Argument1 
+ *        win32_APC_argument
+ *    4') *(esp+0xc) == Argument2
+ *      on XP SP2 for BaseDispatchAPC, seems to be SXS activation context related
+ *    5) *(esp+0x10) == CONTEXT
+ * For x64, it looks like the CONTEXT is at the top of the stack, and the
+ * PxHome fields hold the APC parameters.
+ */
+#define APC_CONTEXT_XSP_OFFS IF_X64_ELSE(0, 0x10)
+#define APC_TARGET_XSP_OFFS  IF_X64_ELSE(0x18, 0)
+/* Remember that every path out of here must invoke the DR exit hook.
+ * The normal return path will do so as the interception code has an
+ * enter and exit hook around the call to this routine.
+ */
+static after_intercept_action_t  /* note return value will be ignored */
+intercept_apc(app_state_at_intercept_t *state)
+{
+    /* this might be a new thread */
+    possible_new_thread_wait_for_dr_init();
+
+    /* FIXME: should we only intercept apc for non-initialized thread
+     * with start/stop interface?
+     * (fine to have start/stop interface also call dynamo_thread_init,
+     *  2nd call turns into nop)
+     */
+    if (intercept_asynch_for_self(true/*we want unknown threads*/)) {
+        CONTEXT *cxt;
+        dcontext_t *dcontext;
+        DEBUG_DECLARE(app_pc apc_target;)
+        if (get_thread_private_dcontext() != NULL)
+            SELF_PROTECT_LOCAL(get_thread_private_dcontext(), WRITABLE);
+        /* won't be re-protected until dispatch->fcache */
+
+        /* the CONTEXT is laid out on the stack itself
+         * from examining KiUserApcDispatcher, we know it's 16 bytes up
+         * we try to verify that at interception time via check_apc_context_offset
+         */
+        cxt = ((CONTEXT *)(state->mc.xsp + APC_CONTEXT_XSP_OFFS));
+
+        RSTATS_INC(num_APCs);
+
+        
+#ifdef DEBUG
+        /* retrieve info on this APC call */
+        apc_target = *((app_pc *)(state->mc.xsp + APC_TARGET_XSP_OFFS));
+        /* FIXME: invalid app parameters would have been caught already, right? */
+        ASSERT(apc_target != 0 && cxt != NULL);
+        LOG(GLOBAL, LOG_ASYNCH, 2,
+            "ASYNCH intercepted apc: thread=%d, apc pc="PFX", cont pc="PFX"\n", 
+            get_thread_id(), apc_target, cxt->CXT_XIP);
+#endif
+
+        /* this is the same check as in dynamorio_init */
+        if (!is_thread_initialized()) {
+            ASSERT(get_os_version() != WINDOWS_VERSION_VISTA);
+            LOG(GLOBAL, LOG_ASYNCH|LOG_THREADS, 2,
+                "APC thread was not initialized!\n");
+            LOG(GLOBAL, LOG_ASYNCH, 1,
+                "ASYNCH intercepted thread init apc: apc pc="PFX", cont pc="PFX"\n", 
+                apc_target, cxt->CXT_XIP);
+            if (intercept_new_thread(cxt))
+                return AFTER_INTERCEPT_LET_GO;
+        } else {
+            /* should not receive APC while in DR code! */
+            ASSERT(get_thread_private_dcontext()->whereami == WHERE_FCACHE);
+            LOG(GLOBAL, LOG_ASYNCH|LOG_THREADS, 2, "APC thread was already initialized!\n");
+            LOG(THREAD_GET, LOG_ASYNCH, 2,
+                "ASYNCH intercepted non-init apc: apc pc="PFX", cont pc="PFX"\n", 
+                apc_target, cxt->CXT_XIP);
+
+#ifdef PROGRAM_SHEPHERDING
+            /* check for hooker's shellcode delivered via APC */
+            if (TEST(OPTION_ENABLED, DYNAMO_OPTION(apc_policy))) {
+                apc_thread_policy_helper((app_pc *)(state->mc.xsp + APC_TARGET_XSP_OFFS),
+                                         DYNAMO_OPTION(apc_policy),
+                                         APC_TARGET_NATIVE 
+                                         /* NtQueueApcThread, likely from kernel mode */);
+                /* case: 9024 test WINDOWS APC as well
+                 * FIXME: we may want to attempt to give an exemption
+                 * for user mode APCs as long as we can determine
+                 * safely that the routine is
+                 * kernel32!BaseDispatchAPC.  Then we'd know which
+                 * argument is indeed going to be a target for an
+                 * indirect call so we can test whether that is
+                 * some shellcode that we need to block or allow.
+                 */
+            }
+#endif /* PROGRAM_SHEPHERDING */
+
+        }
+
+        /* Strategy: we want to use the same dcontext for the APC.
+         * Since we're not stealing a register or anything, and we're squashing
+         * traces, we can rely on the CONTEXT to store the only state we need.
+         * We simply change the CONTEXT right now to point to the next app
+         * pc to execute, and we're all set.
+         */
+        dcontext = get_thread_private_dcontext();
+        if ((cache_pc)cxt->CXT_XIP == after_do_syscall_addr(dcontext) ||
+            (cache_pc)cxt->CXT_XIP == after_shared_syscall_addr(dcontext)) {
+            /* to avoid needing to save this dcontext, just have cxt point to
+             * app pc for after syscall, stored in asynch_target/esi slot
+             * since next_tag holds do/share_syscall address
+             */
+            LOG(THREAD, LOG_ASYNCH, 2,
+                "\tchanging cont pc "PFX" from after do/share syscall to "PFX" or "PFX"\n",
+                cxt->CXT_XIP, dcontext->asynch_target, get_mcontext(dcontext)->xsi);
+            ASSERT(does_syscall_ret_to_callsite());
+            if (DYNAMO_OPTION(sygate_int) &&
+                get_syscall_method() == SYSCALL_METHOD_INT) {
+                /* This should be an int system call and since for sygate
+                 * compatility we redirect those with a call to an ntdll.dll
+                 * int 2e ret 0 we need to pop the stack once to match app. */
+                ASSERT(*(app_pc *)cxt->CXT_XSP == after_do_syscall_code(dcontext) ||
+                       *(app_pc *)cxt->CXT_XSP == after_shared_syscall_code(dcontext));
+                cxt->CXT_XSP += XSP_SZ; /* pop the stack */
+            }
+            if (dcontext->asynch_target != 0)
+                cxt->CXT_XIP = (ptr_uint_t) dcontext->asynch_target;
+            else
+                cxt->CXT_XIP = get_mcontext(dcontext)->xsi;
+        }
+        else if (get_syscall_method() == SYSCALL_METHOD_SYSENTER &&
+                 cxt->CXT_XIP == (ptr_uint_t) vsyscall_after_syscall) {
+            /* Windows XP/2003: kernel ignores the return address
+             * of the caller of the sysenter wrapper code and instead sends control
+             * straight to the ret at 0x7ffe0304.  Since a trampoline there is not
+             * very transparent, we instead clobber the retaddr to point at the
+             * caller of the wrapper.
+             * For an APC interrupting a syscall (i.e., a non-init apc), we
+             * need to change the retaddr back to its native value for transparency,
+             * and also since the storage for it (esi or asynch_target)
+             * might get clobbered before the NtContinue restores us back to
+             * the syscall.
+             * We'll re-fix-up the retaddr to retain control at NtContinue.
+             */
+            ASSERT(get_os_version() >= WINDOWS_VERSION_XP);
+            /* change after syscall ret addr to be app after syscall addr,
+             * since asynch/esi slot is going to get clobbered
+             */
+            /* for the case 5441 Sygate hack, esp will point to
+             * sysenter_ret_address while esp+4/8 will point to after_*_syscall
+             * we'll need to restore both stack values */
+            if (*((cache_pc *)(cxt->CXT_XSP+
+                               (DYNAMO_OPTION(sygate_sysenter) ? XSP_SZ : 0)))
+                == after_do_syscall_code(dcontext)) {
+                LOG(THREAD, LOG_ASYNCH, 2,
+                    "\tcont pc is vsyscall ret, changing ret addr @"PFX" "
+                    "from "PFX" to "PFX"\n",
+                    cxt->CXT_XSP, *((app_pc *)cxt->CXT_XSP), dcontext->asynch_target);
+                if (DYNAMO_OPTION(sygate_sysenter)) {
+                    ASSERT(*((app_pc *)cxt->CXT_XSP) == sysenter_ret_address);
+                    *((app_pc *)(cxt->CXT_XSP+XSP_SZ)) =
+                        dcontext->sysenter_storage;
+                }
+                *((app_pc *)cxt->CXT_XSP) = dcontext->asynch_target;
+            } else if (*((cache_pc *)(cxt->CXT_XSP+
+                                      (DYNAMO_OPTION(sygate_sysenter) ? XSP_SZ : 0)))
+                       == after_shared_syscall_code(dcontext)) {
+                ASSERT(DYNAMO_OPTION(shared_syscalls));
+                /* change after syscall ret addr to be app after syscall addr,
+                 * since esi slot is going to get clobbered
+                 */
+                LOG(THREAD, LOG_ASYNCH, 2,
+                    "\tcont pc is vsyscall ret, changing ret addr @"PFX" "
+                    "from "PFX" to "PFX"\n", cxt->CXT_XSP,
+                    *((app_pc *)cxt->CXT_XSP), get_mcontext(dcontext)->xsi);
+                if (DYNAMO_OPTION(sygate_sysenter)) {
+                    ASSERT(*((app_pc *)cxt->CXT_XSP) == sysenter_ret_address);
+                    *((app_pc *)(cxt->CXT_XSP+XSP_SZ)) =
+                        dcontext->sysenter_storage;
+                }
+                /* change after syscall ret addr to be app after syscall addr,
+                 * since esi slot is going to get clobbered
+                 */
+                *((app_pc *)cxt->CXT_XSP) = (app_pc)get_mcontext(dcontext)->xsi;
+            } else {
+                /* should only get here w/ non-DR-mangled syscall if was native! */
+                ASSERT(dcontext->thread_record->under_dynamo_control == UNDER_DYN_HACK);
+            }
+        } else if (cxt->CXT_XIP == (ptr_uint_t) nt_continue_dynamo_start) {
+            /* NtContinue entered kernel and was interrupted for another APC
+             * we have to restore as though NtContinue never happened, this APC will
+             * execute its own NtContinue (remember, we're stateless)
+             */
+            /* asynch_target is zeroed out when handle_sysem_call is done, so
+             * a zero value indicates that the syscall was handled in-cache.
+             */
+            if (dcontext->asynch_target != NULL)
+                cxt->CXT_XIP = (ptr_uint_t) dcontext->asynch_target;
+            else {
+                ASSERT(DYNAMO_OPTION(shared_syscalls));
+                cxt->CXT_XIP = (ptr_uint_t) dcontext->next_tag;
+            }
+            LOG(THREAD, LOG_ASYNCH, 2,
+                "\tnew APC interrupted nt_continue_dynamo_start, restoring "
+                PFX" as cxt->Xip\n", cxt->CXT_XIP);
+        } else {
+            /* possibilities: for thread init APC I usually see what I think is the
+             * thread entry point in kernel32 (very close to image entry point), but
+             * I also sometimes see a routine in ntdll @0x77f9e9b9:
+             * <ntdll.dll~RtlConvertUiListToApiList+0x2fc,~RtlCreateQueryDebugBuffer-0x315>
+             */
+            LOG(THREAD, LOG_ASYNCH, 2,
+                "\tAPC return point "PFX" needs no translation\n", cxt->CXT_XIP);
+            /* our internal nudge creates a thread that directly targets
+             * generic_nudge_target() */
+            ASSERT(!is_dynamo_address((app_pc)cxt->CXT_XIP) || 
+                   cxt->CXT_XIP == (ptr_uint_t)generic_nudge_target);
+        }
+
+        asynch_retakeover_if_native();
+        state->callee_arg = (void *) false /* use cur dcontext */;
+        asynch_take_over(state);
+    } else
+        STATS_INC(num_APCs_noasynch);
+    return AFTER_INTERCEPT_LET_GO;
+}
+
+/* Verify the 16 byte offset of the CONTEXT structure */
+void
+check_apc_context_offset(byte *apc_entry)
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    instr_t instr;
+    instr_init(dcontext, &instr);
+
+    LOG(GLOBAL, LOG_ASYNCH, 3, "check_apc_context_offset\n");
+    DOLOG(3, LOG_ASYNCH, { disassemble_with_bytes(dcontext, apc_entry, GLOBAL); });
+
+    decode(dcontext, apc_entry, &instr);
+
+#ifdef X64
+    ASSERT(instr_get_opcode(&instr) == OP_mov_ld &&
+           opnd_is_reg(instr_get_dst(&instr, 0)) &&
+           opnd_get_reg(instr_get_dst(&instr, 0)) == REG_RCX &&
+           opnd_is_base_disp(instr_get_src(&instr, 0)) &&
+           opnd_get_disp(instr_get_src(&instr, 0)) == 0 &&
+           opnd_get_base(instr_get_src(&instr, 0)) == REG_XSP &&
+           opnd_get_index(instr_get_src(&instr, 0)) == REG_NULL);
+#else
+    /* In Win 2003 SP1, the context offset used is 0xc, and DR works with it;  
+     * the first lea used there has an offset of 0x2dc, not 0xc.  See case 3522.
+     */
+    ASSERT(instr_get_opcode(&instr) == OP_lea &&
+           (opnd_get_disp(instr_get_src(&instr, 0)) == 0x10 ||
+            opnd_get_disp(instr_get_src(&instr, 0)) == 0x2dc));
+#endif
+    instr_free(dcontext, &instr);
+}
+
+
+/****************************************************************************
+ * NtContinue
+ *
+ * NtContinue is used both by exceptions and APCs (both thread-creation APCs
+ * and non-creation APCs, the latter distinguished by always interrupting
+ * the app inside a system call, just like a callback).
+ * We can avoid needing to save the prev dcontext and restore it here b/c
+ * user mode passes a CONTEXT so we know where we're going, and we can simply
+ * start interpreting anew at that point.  We aren't stealing a register and
+ * we're squashing traces, so we have no baggage that needs to be restored
+ * on the other end.  We just have to be careful about shared syscall return
+ * addresses and the exception fragment ==> FIXME!  what if get APC while handling
+ * exception, then another exception?  will our exception fragment get messed up?
+ *
+ * We used to need to restore a register, so we worried about non-init APCs and
+ * exceptions that re-executed the faulting instruction.  We couldn't tell them apart
+ * from init APCs and other exceptions, so we saved the dcontext every time and restored
+ * it here.  No more.
+ *
+NtContinue:
+  77F820F3: B8 1C 00 00 00     mov         eax,1Ch
+  77F820F8: 8D 54 24 04        lea         edx,[esp+4]
+  77F820FC: CD 2E              int         2Eh
+ *
+ * NtContinue takes CONTEXT *cxt and flag (0=exception, 1=APC?!?)
+ * This routine is called by pre_system_call, NOT intercepted from
+ * ntdll kernel entry point, as it's user-driven.
+ */
+void
+intercept_nt_continue(CONTEXT *cxt, int flag)
+{
+    if (intercept_asynch_for_self(false/*no unknown threads*/)) {
+        dcontext_t *dcontext = get_thread_private_dcontext();
+        
+        LOG(THREAD, LOG_ASYNCH, 2,
+            "ASYNCH intercept_nt_continue in thread %d, xip="PFX"\n",
+            get_thread_id(), cxt->CXT_XIP);
+
+        LOG(THREAD, LOG_ASYNCH, 3, "target context:\n");
+        DOLOG(3, LOG_ASYNCH, { dump_context_info(cxt, THREAD, true); });
+        
+        if (is_building_trace(dcontext)) {
+            LOG(THREAD, LOG_ASYNCH, 2, "intercept_nt_continue: squashing old trace\n");
+            trace_abort(dcontext);
+        }
+       
+        if (get_syscall_method() == SYSCALL_METHOD_SYSENTER &&
+            cxt->CXT_XIP == (ptr_uint_t) vsyscall_after_syscall) {
+            /* We need to go back to after shared/do syscall, to do post syscall and other
+             * activities, so we restore the special esi pointer from the ret addr
+             */
+            /* This works w/optimize syscalls w/no changes. Since
+             * intercept_nt_continue is called from pre_system_call only, the
+             * #ifdefs ensure that the correct location is used for the
+             * after-syscall address.
+             */
+            /* Note that our stateless handling re-uses the same dcontext, and
+             * that we assume we can match the dstack for the NtContinue
+             * fcache entrance with the fcache return from the return-to syscall
+             */
+            /* NOTE - the stack mangling must match that of handle_system_call()
+             * and shared_syscall as not all routines looking at the stack
+             * differentiate. */
+            ASSERT(get_os_version() >= WINDOWS_VERSION_XP);
+            LOG(THREAD, LOG_ASYNCH, 2,
+                "\txip=vsyscall "PFX", changing ret addr @"PFX" from "PFX" to "PFX"\n",
+                cxt->CXT_XIP, cxt->CXT_XSP,
+                *((app_pc *)(cxt->CXT_XSP+
+                             (DYNAMO_OPTION(sygate_sysenter) ? XSP_SZ : 0))),
+                after_do_syscall_code(dcontext));
+            dcontext->asynch_target = *((app_pc *)cxt->CXT_XSP);
+            if (DYNAMO_OPTION(sygate_sysenter)) {
+                /* case 5441 Sygate hack, tos in esi/asynch, next stack slot saved
+                 * in sysenter_storage. Stack then looks like
+                 * esp +0   sysenter_ret_address (ret in ntdll.dll)
+                 *     +4/8 after_do/shared_syscall
+                 */
+                /* get the app ret addr into the proper asynch target slot */
+                dcontext->sysenter_storage = *((app_pc *)(cxt->CXT_XSP+XSP_SZ));
+                /* now replace the ret addr w/ do syscall */
+                *((app_pc *)cxt->CXT_XSP) = sysenter_ret_address;
+                *((app_pc *)(cxt->CXT_XSP+XSP_SZ)) =
+                    (app_pc) after_do_syscall_code(dcontext);
+            } else {
+                /* now replace the ret addr w/ do syscall */
+                *((app_pc *)cxt->CXT_XSP) = (app_pc) after_do_syscall_code(dcontext);
+            }
+        } else if (!in_fcache((cache_pc)cxt->CXT_XIP) &&
+                   /* FIXME : currently internal nudges (detach on violation 
+                    * for ex.) create a thread that directly targets the
+                    * generic_nudge_target() function. Therefore, we have to check for
+                    * it here. */
+                   (!is_dynamo_address((cache_pc)cxt->CXT_XIP) ||
+                    cxt->CXT_XIP == (ptr_uint_t)generic_nudge_target) &&
+                   !in_generated_routine(dcontext, (cache_pc)cxt->CXT_XIP)) {
+            /* Going to non-code-cache address, need to make sure get control back 
+             * Use next_tag slot to hold original Xip 
+             */
+            LOG(THREAD, LOG_ASYNCH, 2,
+                "\txip="PFX" not in fcache, intercepting at "PFX"\n",
+                cxt->CXT_XIP, nt_continue_dynamo_start);
+            /* we have to use a different slot since next_tag ends up holding the do_syscall
+             * entry when entered from dispatch (we're called from pre_syscall, prior to entering cache)
+             */
+            dcontext->asynch_target = (app_pc) cxt->CXT_XIP;
+            /* Point Xip to allow dynamo to retain control
+             * FIXME: w/ stateless handling here, can point at fcache_return
+             * like signals do for better performance?
+             */
+            cxt->CXT_XIP = (ptr_uint_t) nt_continue_dynamo_start;
+        } else {
+            /* No explanation for this one! */
+            SYSLOG_INTERNAL_ERROR("ERROR: intercept_nt_continue: xip="PFX
+                                  " not an app pc!", cxt->CXT_XIP);
+            ASSERT_NOT_REACHED();
+        }
+    }
+}
+
+/* This routine is called by pre_system_call
+ * Assumes caller holds thread_initexit_lock
+ * dcontext is the context of the target thread, not this thread
+ */
+void
+intercept_nt_setcontext(dcontext_t *dcontext, CONTEXT *cxt)
+{
+    /* b/c it needs the registers passed in, pre_system_call does the
+     * synch, and b/c post_system_call needs to test the same
+     * condition, pre also does the interception check.
+     */
+    ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
+    ASSERT(intercept_asynch_for_thread(dcontext->owning_thread, false/*no unknown threads*/));
+    ASSERT(dcontext != NULL && dcontext->initialized);
+    LOG(THREAD, LOG_ASYNCH, 1,
+        "ASYNCH intercept_nt_setcontext: thread %d targeting thread %d\n",
+        get_thread_id(), dcontext->owning_thread);
+    LOG(THREAD, LOG_ASYNCH, 3, "target context:\n");
+    DOLOG(3, LOG_ASYNCH, { dump_context_info(cxt, THREAD, true); });
+
+    if (is_building_trace(dcontext)) {
+        LOG(THREAD, LOG_ASYNCH, 2, "intercept_nt_setcontext: squashing old trace\n");
+        trace_abort(dcontext);
+    }
+        
+    /* Yes, we use the same x86.asm and x86_code.c procedures as
+     * NtContinue: nt_continue_dynamo_start and nt_continue_start_setup
+     */
+    if (!in_fcache((cache_pc)cxt->CXT_XIP) &&
+        !in_generated_routine(dcontext, (cache_pc)cxt->CXT_XIP)) {
+        /* Going to non-code-cache address, need to make sure get control back 
+         * Use next_tag slot to hold original Xip 
+         */
+        LOG(THREAD, LOG_ASYNCH, 1,
+            "intercept_nt_setcontext: xip="PFX" not in fcache, intercepting\n",
+            cxt->CXT_XIP);
+        /* This works w/optimize syscalls w/no changes. Since
+         * intercept_nt_setcontext is called from pre_system_call only, the
+         * #ifdefs ensure that the correct location is used for the xip.
+         */
+        /* we have to use a different slot since next_tag ends up holding the do_syscall
+         * entry when entered from dispatch (we're called from pre_syscall, prior to entering cache)
+         */
+        dcontext->asynch_target = (app_pc) cxt->CXT_XIP;
+        /* Point Xip to allow dynamo to retain control 
+         * FIXME: w/ stateless handling here, can point at fcache_return
+         * like signals do for better performance?
+         */
+        cxt->CXT_XIP = (ptr_uint_t) get_setcontext_interceptor();
+    } else {
+        LOG(THREAD, LOG_ASYNCH, 1,
+            "ERROR: intercept_nt_setcontext: xip="PFX" in fcache!\n", cxt->CXT_XIP);
+        /* This should not happen!  Does this indicate
+         * malicious/erroneous application code?
+         */
+        SYSLOG_INTERNAL_ERROR("intercept_nt_setcontext: targeting fcache!");
+        ASSERT_NOT_REACHED();
+    }
+}
+
+
+/****************************************************************************
+ * EXCEPTIONS
+ * 
+ */
+
+#ifdef INTERCEPT_TOP_LEVEL_EXCEPTIONS
+/* top-level exception handler 
+ * currently we don't need this, so it's not operational
+ * to make operational, add this to callback_init:
+ *   app_top_handler = 
+ *     SetUnhandledExceptionFilter((LPTOP_LEVEL_EXCEPTION_FILTER) our_top_handler);
+ * also need to intercept the app calling SetUnhandledExceptionFilter, so need
+ * to investigate whether it turns into syscall RtlUnhandledExceptionFilter
+ * or what -- FIXME
+ */
+static LONG
+our_top_handler(struct _EXCEPTION_POINTERS * pExceptionInfo)
+{
+    SYSLOG_INTERNAL_INFO("in top level exception handler!");
+    if (app_top_handler != NULL)
+        return (*app_top_handler)(pExceptionInfo);
+    else
+        return EXCEPTION_CONTINUE_SEARCH; /* let default action happen */
+}
+#endif
+
+static void
+transfer_to_fcache_return(dcontext_t *dcontext, CONTEXT *cxt, app_pc next_pc,
+                          linkstub_t *last_exit)
+{
+    /* Do not resume execution in cache, go back to dispatch.
+     * Do a direct nt_continue to fcache_return!
+     * Note that even if we were in the shared cache, we
+     * still go to the private fcache_return for simplicity.
+     */
+    cxt->CXT_XIP = (ptr_uint_t) fcache_return_routine(dcontext);
+    get_mcontext(dcontext)->xax = cxt->CXT_XAX;
+    cxt->CXT_XAX = (ptr_uint_t) last_exit;
+    /* fcache_return will save rest of state */
+    dcontext->next_tag = next_pc;
+    LOG(THREAD, LOG_ASYNCH, 2,
+        "\tset next_tag to "PFX", resuming in fcache_return\n", next_pc);
+    EXITING_DR();
+    nt_continue(cxt);
+}
+
+/* Due to lack of parameter space when calling found_modified_code()
+ * we use flags.  We also use them for check_for_modified_code() for
+ * consistency.
+ */
+enum {
+    MOD_CODE_TAKEOVER      = 0x01,
+    MOD_CODE_EMULATE_WRITE = 0x02,
+    MOD_CODE_APP_CXT       = 0x04,
+};
+
+/* To allow execution from a writable memory region, we mark it read-only.
+ * When we get a write seg fault from that region, we call this routine.
+ * It removes the region from the executable list, flushes fragments
+ * originating there, marks it writable again, and then calls NtContinue
+ * to resume execution of the faulting write.
+ * This function does not return!
+ */
+/* exported since we can't do inline asm anymore and must call from x86.asm */
+void
+found_modified_code(dcontext_t *dcontext, EXCEPTION_RECORD *pExcptRec,
+                    CONTEXT *cxt, app_pc target, uint flags)
+{
+    app_pc next_pc = NULL;
+    cache_pc instr_cache_pc = (app_pc) pExcptRec->ExceptionAddress;
+    app_pc translated_pc;
+    if (!TEST(flags, MOD_CODE_TAKEOVER) || TEST(flags, MOD_CODE_APP_CXT)) {
+        LOG(THREAD, LOG_ASYNCH, 2,
+            "found_modified_code: native/app "PFX"\n", instr_cache_pc);
+        /* for !takeover: assumption: native pc -- FIXME: vs
+         * thread-noasynch general usage? 
+         */
+        ASSERT(!in_generated_routine(dcontext, instr_cache_pc) &&
+               !in_fcache(instr_cache_pc));
+        translated_pc = instr_cache_pc;
+        instr_cache_pc = NULL;
+    } else {
+        LOG(THREAD, LOG_ASYNCH, 2,
+            "found_modified_code: translating "PFX"\n", instr_cache_pc);
+        /* For safe recreation we need to either be couldbelinking or hold the
+         * initexit lock (to keep someone from flushing current fragment), the
+         * initexit lock is easier
+         */
+        mutex_lock(&thread_initexit_lock);
+        /* FIXME: currently this will fail for a pc inside a pending-deletion
+         * fragment!  == case 3567
+         */
+        translated_pc = recreate_app_pc(dcontext, instr_cache_pc, NULL);
+        mutex_unlock(&thread_initexit_lock);
+        LOG(THREAD, LOG_ASYNCH, 2,
+            "\tinto "PFX"\n", translated_pc);
+    }
+    ASSERT(translated_pc != NULL);
+    if (USING_PRETEND_WRITABLE() &&
+        is_pretend_writable_address(target)) {
+        /* now figure out why is this pretend_writable, here only for debugging */
+
+        /* case 6632: we may want to report even in release build if
+         * we've prevented a function patch.  or at least should add a
+         * release build statistic to show that.
+         */
+        DEBUG_DECLARE(bool system_overlap = tamper_resistant_region_overlap(target, target+1);)
+        DEBUG_DECLARE(bool patch_module_overlap = vmvector_overlap(patch_proof_areas,
+                                                                   target, target+1);)
+        SYSLOG_INTERNAL_WARNING_ONCE("app tried to write to pretend-writable "
+                                     "code "PFX, target);
+        LOG(THREAD, LOG_ASYNCH, 2, "app tried to write to pretend-writable %s code "
+            PFX"\n",
+            system_overlap ? "system" : (patch_module_overlap ? "patch module" : "DR"), 
+            target);
+            
+        DOSTATS({if (system_overlap)
+            STATS_INC(app_modify_ntdll_writes);
+        else if (patch_module_overlap)
+            STATS_INC(app_modify_patch_module_writes);
+        else 
+            STATS_INC(app_modify_DR_writes);
+        });
+        /* if there are more than a handful of writes we're dealing not with a
+         * hooking dll but w/ something else.
+         * we see 48 ntdll hook writes in case 9149.
+         */
+        ASSERT_CURIOSITY_ONCE(GLOBAL_STAT(app_modify_DR_writes) < 10);
+        ASSERT_CURIOSITY_ONCE(GLOBAL_STAT(app_modify_ntdll_writes) < 50);
+        ASSERT_CURIOSITY_ONCE(GLOBAL_STAT(app_modify_patch_module_writes) < 50);
+
+        /* skip the write */
+        next_pc = decode_next_pc(dcontext, translated_pc);
+        LOG(THREAD, LOG_ASYNCH, 2, "skipping to after write pc "PFX"\n", next_pc);
+    } else if (TEST(flags, MOD_CODE_EMULATE_WRITE)) {
+        app_pc prot_start = (app_pc) PAGE_START(target);
+        uint write_size;
+        size_t prot_size;
+        dr_mcontext_t mcontext;
+        bool ok;
+        DEBUG_DECLARE(app_pc result =)
+            decode_memory_reference_size(dcontext, translated_pc, &write_size);
+        ASSERT(result != NULL);
+        /* In current usage, we only use emulation for cases where a sub-page
+         * region NOT containing code is being written to -- otherwise there's
+         * no real advantage over normal page protection consistency or sandboxing,
+         * especially with multiple writes in a row w/ no executions in between.
+         * If we do decide to use it for executable regions, must call flush here.
+         */
+        ASSERT(!executable_vm_area_overlap(target, target+write_size, false/*no lock*/));
+        SYSLOG_INTERNAL_WARNING_ONCE("app tried to write emulate-write region @"PFX,
+                                     target);
+        LOG(THREAD, LOG_ASYNCH, 2, "emulating writer @"PFX" writing "PFX"-"PFX"\n",
+            translated_pc, target, target+write_size);
+        prot_size = (app_pc) PAGE_START(target+write_size) + PAGE_SIZE - prot_start;
+        context_to_mcontext(&mcontext, cxt);
+        /* can't have two threads in here at once mixing up writability w/ the write */
+        mutex_lock(&emulate_write_lock);
+        /* FIXME: this opens up a window where an executable region on the same
+         * page will not have code modifications detected
+         */
+        ok = make_writable(prot_start, prot_size);
+        ASSERT_CURIOSITY(ok);
+        if (ok) {
+            next_pc = emulate(dcontext, translated_pc, &mcontext);
+        } else {
+            /* FIXME: case 10550 note that it is possible that this
+             * would have failed natively, so we should have executed
+             * the call to make a page writable when the app requested
+             * it.  Then we wouldn't have to worry about this write
+             * failing.  There is a small chance the app wouldn't have
+             * even tried to write.
+             *
+             * It is too late for us to return the proper error from
+             * the system call, so we could either skip this
+             * instruction, or crash the app.  We do the latter: we'll
+             * reexecute app write on a still read only page!
+             */
+            next_pc = NULL;
+        }
+        if (next_pc == NULL) {
+            /* using some instr our emulate can't handle yet
+             * abort and remove page-expanded region from exec list
+             */
+            mutex_unlock(&emulate_write_lock);
+            LOG(THREAD, LOG_ASYNCH, 1, "emulation of instr @"PFX" failed, bailing\n",
+                translated_pc);
+            flush_fragments_and_remove_region(dcontext, prot_start, prot_size,
+                                              false /* don't own initexit_lock */,
+                                              false /* keep futures */);
+            /* could re-execute the write in-cache, but could be inside region
+             * being flushed, so safest to exit */
+            next_pc = translated_pc;
+            STATS_INC(num_emulated_write_failures);
+        } else {
+            LOG(THREAD, LOG_ASYNCH, 1,
+                "successfully emulated writer @"PFX" writing "PFX" to "PFX"\n",
+                translated_pc, *((int *)target), target);
+            make_unwritable(prot_start, prot_size);
+            mutex_unlock(&emulate_write_lock);
+            /* will go back to dispatch for next_pc below */
+            STATS_INC(num_emulated_writes);
+        }
+        ASSERT(next_pc != NULL);
+        if (DYNAMO_OPTION(IAT_convert)) {
+            /* FIXME: case 85: very crude solution just flush ALL
+             * fragments if an IAT hooker shows up to make sure we're
+             * executing consistently
+             */
+            /* we depend on emulate_IAT_writes to get these faults,
+             * loader patching up IAT will not trigger these since
+             * it is exempted as would also like
+             */
+            if (vmvector_overlap(IAT_areas, target, target+1)) {
+                LOG(THREAD, LOG_ASYNCH, 1,
+                    "IAT hooker at @"PFX" invalidating all caches\n",
+                    translated_pc);
+                if (!INTERNAL_OPTION(unsafe_IAT_ignore_hooker)) {
+                    SYSLOG_INTERNAL_WARNING_ONCE("IAT hooker resulted in whole "
+                                                 "cache flush");
+                    invalidate_code_cache();
+                } else {
+                    SYSLOG_INTERNAL_WARNING_ONCE("IAT hooker - ignoring write");
+                }
+                STATS_INC(num_invalidate_IAT_hooker);
+            } else {
+                ASSERT_NOT_TESTED();
+            }
+        }
+    } else {
+        next_pc = handle_modified_code(dcontext, instr_cache_pc, 
+                                       translated_pc, target);
+    }
+    /* if !takeover, re-execute the write no matter what -- the assumption
+     * is that the write is native */
+    if (!TEST(flags, MOD_CODE_TAKEOVER) || next_pc == NULL) {
+        /* now re-execute the write
+         * don't try to go through entire exception route by setting up
+         * our own exception handler directly in TIB -- not transparent,
+         * requires user stack!  just call NtContinue here
+         */
+        if (next_pc != NULL) {
+            cxt->CXT_XIP = (ptr_uint_t) next_pc;
+            LOG(THREAD, LOG_ASYNCH, 2,
+                "\tresuming after write instr @ "PFX"\n", cxt->CXT_XIP);
+        } else
+            LOG(THREAD, LOG_ASYNCH, 2, "\tresuming write instr @ "PFX"\n", cxt->CXT_XIP);
+        EXITING_DR();
+        nt_continue(cxt);
+    } else {
+        /* Cannot resume execution in cache (was flushed), go back to dispatch
+         * via fcache_return
+         */
+        transfer_to_fcache_return(dcontext, cxt, next_pc,
+                                  (linkstub_t *) get_selfmod_linkstub());
+    }
+    ASSERT_NOT_REACHED(); /* should never get here */
+}
+
+#ifdef STACK_GUARD_PAGE
+static bool
+is_dstack_overflow(dcontext_t *dcontext, EXCEPTION_RECORD *pExcptRec,
+                   CONTEXT *cxt)
+{
+    if (pExcptRec->ExceptionCode == EXCEPTION_GUARD_PAGE) {
+        /* Richter book says that only access violation fills in info array,
+         * but on win2k guard page seems to fill it in!
+         */
+        if (pExcptRec->NumberParameters >= 2) {
+            app_pc target = (app_pc) pExcptRec->ExceptionInformation[1];
+            LOG(THREAD, LOG_ASYNCH, 2,
+                "is_dstack_overflow: target is "PFX"\n", target);
+            return is_stack_overflow(dcontext, target);
+        }
+    }
+    return false;
+}
+#endif /* STACK_GUARD_PAGE */
+
+/* To allow execution from a writable memory region, we mark it read-only.
+ * When we get a seg fault, we call this routine, which determines if it's
+ * a write to a region we've marked read-only.  If so, it does not return.
+ * In that case, if takeover, we re-execute the faulting instr under our
+ * control, while if !takeover, we re-execute the faulting instr natively.
+ * If app_cxt, the exception record and context contain app state, not
+ * code cache state (happens in some circumstances such as case 7393).
+ */
+static void
+check_for_modified_code(dcontext_t *dcontext, EXCEPTION_RECORD *pExcptRec,
+                        CONTEXT *cxt, uint flags)
+{
+    /* special case: we expect a seg fault for executable regions
+     * that were writable and marked read-only by us
+     */
+    if (pExcptRec->ExceptionCode == EXCEPTION_ACCESS_VIOLATION &&
+        pExcptRec->ExceptionInformation[0]==1 /* write */) {
+        app_pc target = (app_pc) pExcptRec->ExceptionInformation[1];
+        bool emulate_write = false;
+        uint mod_flags;
+        LOG(THREAD, LOG_ASYNCH, 2,
+            "check_for_modified_code: exception was write to "PFX"\n", target);
+        if (!vmvector_empty(emulate_write_areas)) {
+            uint write_size = 0;
+            /* FIXME: we duplicate this write size lookup w/ found_modified_code */
+            DEBUG_DECLARE(app_pc result =)
+                decode_memory_reference_size(dcontext, (app_pc) pExcptRec->ExceptionAddress,
+                                             &write_size);
+            ASSERT(result != NULL);
+            /* only emulate if completely inside -- no quick check for that, good
+             * enough to say if overlap w/ emulate areas and no overlap w/ exec areas.
+             * o/w we have to flush the written part outside of emulate areas.
+             */
+            /* FIXME: case 7492: reported target may be in the middle of the write! */
+            emulate_write = 
+                vmvector_overlap(emulate_write_areas, target, target+write_size) &&
+                !executable_vm_area_overlap(target, target+write_size, false/*no lock*/);
+        }
+        if (was_executable_area_writable(target) || emulate_write ||
+            ((DYNAMO_OPTION(handle_DR_modify) == DR_MODIFY_NOP || 
+              DYNAMO_OPTION(handle_ntdll_modify) == DR_MODIFY_NOP) &&
+             /* FIXME: should pass written-to range and not just single target */
+             is_pretend_writable_address(target))) {
+            app_pc cur_esp;
+
+            /* not an app exception */
+            RSTATS_DEC(num_exceptions);
+            DOSTATS({
+                if (!TEST(MOD_CODE_TAKEOVER, flags))
+                    STATS_INC(num_native_cachecons_faults);
+            });
+            LOG(THREAD, LOG_ASYNCH, 2,
+                "check_for_modified_code: seg fault in exec-writable region @"PFX"\n",
+                target);
+            /* we're not going to return through either of the usual
+             * methods, so we have to free the initstack mutex, but
+             * we need a stack -- so, we use a separate method to avoid
+             * stack conflicts, and switch to dstack now.
+             * FIXME: the TOS dstack offset constant (12 or 84) is hardcoded below.
+             * use an asm routine to make use of existing x86.asm constants?
+             */
+            GET_STACK_PTR(cur_esp);
+            /* prepare flags param for found_modified_code */
+            mod_flags = flags;
+            if (emulate_write)
+                mod_flags |= MOD_CODE_EMULATE_WRITE;
+            call_modcode_alt_stack(dcontext, pExcptRec, cxt, target,
+                                   mod_flags, is_on_initstack(cur_esp));
+            ASSERT_NOT_REACHED();
+        }
+#ifdef DGC_DIAGNOSTICS
+        else {
+            /* make all heap RO in attempt to view generation of DGC */
+            DOLOG(3, LOG_VMAREAS, {
+                /* WARNING: assuming here that app never seg faults on its own */
+                char buf[MAXIMUM_SYMBOL_LENGTH];
+                app_pc base;
+                size_t size;
+                bool ok = get_memory_info(target, &base, &size, NULL);
+                cache_pc instr_cache_pc = (app_pc) pExcptRec->ExceptionAddress;
+                app_pc translated_pc;
+                ASSERT(ok);
+                LOG(THREAD, LOG_ASYNCH, 1,
+                    "got seg fault @"PFX" in non-E region we made RO "PFX"-"PFX"\n",
+                    target, base, base + size);
+                LOG(THREAD, LOG_ASYNCH, 2,
+                    "found_modified_code: traslating "PFX"\n", instr_cache_pc);
+                /* For safe recreation we need to either be couldbelinking or hold the
+                 * initexit lock (to keep someone from flushing current fragment), the
+                 * initexit lock is easier
+                 */
+                mutex_lock(&thread_initexit_lock);
+                translated_pc = recreate_app_pc(dcontext, instr_cache_pc, NULL);
+                ASSERT(translated_pc != NULL);
+                mutex_unlock(&thread_initexit_lock);
+                LOG(THREAD, LOG_ASYNCH, 2, "\tinto "PFX"\n", translated_pc);
+                print_symbolic_address(translated_pc, buf, sizeof(buf), false);
+                LOG(THREAD, LOG_VMAREAS, 1,
+                    "non-code written by app pc "PFX" from bb %s:\n",
+                    translated_pc, buf);
+                DOLOG(1, LOG_VMAREAS, { 
+                    disassemble_app_bb(dcontext, translated_pc, THREAD); });
+                LOG(THREAD, LOG_ASYNCH, 1,
+                    "Making "PFX"-"PFX" writable\n", base, base + size);
+                ok = make_writable(base, size);
+                ASSERT(ok);
+                /* now re-execute the write
+                 * don't try to go through entire exception route by setting up
+                 * our own exception handler directly in TIB -- not transparent,
+                 * requires user stack!  just call NtContinue here
+                 */
+                LOG(THREAD, LOG_ASYNCH, 1, "\tresuming write instr @ "PFX", esp="PFX"\n",
+                    cxt->CXT_XIP, cxt->CXT_XSP);
+                EXITING_DR();
+                nt_continue(cxt);
+                ASSERT_NOT_REACHED();
+            });
+        }
+#endif
+    }
+}
+
+
+/* SEH Definitions */
+/* returns current head of exception list
+   assumes we haven't installed our own exception handler
+   hence we can just read it off TIB
+*/
+EXCEPTION_REGISTRATION*
+get_exception_list()
+{
+    /* typedef struct _NT_TIB { */
+    /*     struct _EXCEPTION_REGISTRATION_RECORD *ExceptionList; */
+    return (EXCEPTION_REGISTRATION *) get_tls(EXCEPTION_LIST_TIB_OFFSET);
+}
+
+/* verify exception handler list is consistent */
+/* Used as a first level check for handler integrity before throwing an exception. */
+/* These checks are best effort and following through a handler may
+   still result in subsequent violations of our policies,
+   e.g. attacked handler can point to a valid RET that will fail our checks later. 
+*/
+/* returns depth so that can caller can decide whether
+    it is worth throwing an exception if anyone would be there to catch it
+    1 only the default handler is there
+    0 empty shouldn't happen
+   -1 when invalid
+*/
+int
+exception_frame_chain_depth(dcontext_t *dcontext)
+{
+    int depth = 0;
+    EXCEPTION_REGISTRATION* pexcrec = get_exception_list();
+    app_pc stack_base, stack_top;
+    get_stack_bounds(dcontext, &stack_base, &stack_top);
+
+    LOG(THREAD_GET, LOG_ASYNCH, 2, "ASYNCH exception_frame_chain_depth head: "PFX"\n", 
+        pexcrec);
+
+    for (; (EXCEPTION_REGISTRATION*)PTR_UINT_MINUS_1 != pexcrec;
+         pexcrec = pexcrec->prev) {
+        if (!ALIGNED(pexcrec, 4)) {
+            LOG(THREAD_GET, LOG_ASYNCH, 1, "WARNING: ASYNCH invalid chain - not DWORD aligned\n");
+            return -1;
+        }
+        /* each memory location should be readable (we don't want to die while checking) */
+        if (!is_readable_without_exception((app_pc)pexcrec, sizeof(EXCEPTION_REGISTRATION))) /* heavy weight check */ {
+            LOG(THREAD_GET, LOG_ASYNCH, 1, 
+                "ASYNCH exception_frame_chain_depth "PFX" invalid! "
+                "possibly under attack\n", pexcrec);
+            return -1;
+        }
+        LOG(THREAD_GET, LOG_ASYNCH, 2, 
+            "ASYNCH exception_frame_chain_depth[%d] "PFX", handler: "PFX
+            ", prev: "PFX"\n", depth, pexcrec, pexcrec->handler, pexcrec->prev);
+        /* prev address should be higher in memory than current */
+        if (pexcrec->prev <= pexcrec) {
+            LOG(THREAD_GET, LOG_ASYNCH, 1, 
+                "WARNING: ASYNCH invalid chain - not strictly up on the stack\n");
+            return -1;
+        }
+        /* check against stack limits */
+        if ((stack_base > (app_pc)pexcrec)
+            || (stack_top < (app_pc)pexcrec + sizeof(EXCEPTION_REGISTRATION))) {
+            LOG(THREAD_GET, LOG_ASYNCH, 1, 
+                "WARNING: ASYNCH invalid chain - "PFX" not on 'official' stack "
+                PFX"-"PFX"\n", pexcrec, stack_base, stack_top);
+            return -1;
+        }
+
+        /* FIXME: the handler pc should pass the code origins check -- 
+           it is possible that we have failed on it to begin with
+           - check_origins_helper() unfortunately has side effects that we may not want..)
+           and furthermore we may be coming from there - need to restructure that code 
+           we want a quick check with no action taken there - something like
+           check_thread_vm_area(dcontext, pexcrec->handler, NULL, NULL)
+           or maybe a variant of check_origins_helper(dcontext, pexcrec->handler, ...)
+        */
+        ASSERT_NOT_IMPLEMENTED(true); /* keep going for now  */
+
+        /* make sure we don't get in an infinite loop
+           (shouldn't be possible after the prev <= exrec check) */
+        if (depth++ > 100) {
+            LOG(THREAD_GET, LOG_ASYNCH, 1,
+                "ASYNCH frame[%d]: too deep chain, possibly corrupted\n", depth);
+            return -1;
+        }
+    }
+    LOG(THREAD_GET, LOG_ASYNCH, 1,
+        "ASYNCH exception_frame_chain_depth depth=%d\n", depth);
+
+    return depth;                /* FIXME: return length */
+}
+
+#ifdef DEBUG
+void
+dump_context_info(CONTEXT *context, file_t file, bool all)
+{
+#define DUMP(r) LOG(file, LOG_ASYNCH, 2, #r"="PFX" ", context->r);
+#define DUMPNM(r,nm) LOG(file, LOG_ASYNCH, 2, #nm"="PFX" ", context->r);
+#define NEWLINE LOG(file, LOG_ASYNCH, 2, "\n  ");
+    DUMP(ContextFlags);
+    NEWLINE;
+
+    if (all || context->ContextFlags & CONTEXT_INTEGER) {
+        DUMPNM(CXT_XDI, Xdi);
+        DUMPNM(CXT_XSI, Xsi);
+        DUMPNM(CXT_XBX, Xbx);
+        NEWLINE;
+        DUMPNM(CXT_XDX, Xdx);
+        DUMPNM(CXT_XCX, Xcx);
+        DUMPNM(CXT_XAX, Xax);
+        NEWLINE;
+#ifdef X64
+        DUMPNM(CXT_XBP, Xbp);
+        DUMP(R8);
+        DUMP(R9);
+        NEWLINE;
+        DUMP(R10);
+        DUMP(R11);
+        DUMP(R12);
+        NEWLINE;
+        DUMP(R13);
+        DUMP(R14);
+        DUMP(R15);
+        NEWLINE;
+#endif
+    }
+
+    if (all || context->ContextFlags & CONTEXT_CONTROL) {
+#ifndef X64
+        DUMPNM(CXT_XBP, Xbp);
+#endif
+        DUMPNM(CXT_XIP, Xip);
+        DUMP(SegCs);                // MUST BE SANITIZED
+        NEWLINE;
+        DUMPNM(CXT_XFLAGS, XFlags); // MUST BE SANITIZED
+        DUMPNM(CXT_XSP, Xsp);
+        DUMP(SegSs);
+        NEWLINE;
+    }
+
+    if (all || context->ContextFlags & CONTEXT_DEBUG_REGISTERS) {
+        DUMP(Dr0);
+        DUMP(Dr1);
+        DUMP(Dr2);
+        DUMP(Dr3);
+        NEWLINE;
+        DUMP(Dr6);
+        DUMP(Dr7);
+        NEWLINE;
+    }
+
+    /* For PR 264138 */
+    if (all || TESTALL(CONTEXT_XMM_FLAG, context->ContextFlags)) {
+        int i, j;
+        for (i=0; i<NUM_XMM_SAVED; i++) {
+            LOG(file, LOG_ASYNCH, 2, "xmm%d=0x", i);
+            /* This would be simpler if we had uint64 fields in dr_xmm_t but
+             * that complicates our struct layouts */
+            for (j = 0; j < 4; j++) {
+                LOG(file, LOG_ASYNCH, 2, "%08x", CXT_XMM(context, i)->u32[j]);
+            }
+            NEWLINE;
+        }
+    }
+
+    if (all || context->ContextFlags & CONTEXT_FLOATING_POINT) {
+        LOG(THREAD_GET, LOG_ASYNCH, 2, "<floating point area>\n  ");
+    }
+
+    if (all || context->ContextFlags & CONTEXT_SEGMENTS) {
+        DUMP(SegGs);
+        DUMP(SegFs);
+        DUMP(SegEs);
+        DUMP(SegDs);
+    }
+    LOG(file, LOG_ASYNCH, 2, "\n");
+#undef DUMP
+#undef DUMPNM
+#undef NEWLINE
+}
+
+static const char *
+exception_access_violation_type(ptr_uint_t code)
+{
+    if (code == EXCEPTION_INFORMATION_READ_EXECUTE_FAULT)
+        return "read";
+    else if (code == EXCEPTION_INFORMATION_WRITE_FAULT)
+        return "write";
+    else if (code == EXCEPTION_INFORMATION_EXECUTE_FAULT)
+        return "execute";
+    else
+        return "UNKNOWN";
+}
+
+static void
+dump_exception_info(EXCEPTION_RECORD* exception, CONTEXT *context)
+{
+    LOG(THREAD_GET, LOG_ASYNCH, 2, 
+        "\texception code = "PFX", ExceptionFlags="PFX"\n\trecord="PFX", params=%d\n",
+        exception->ExceptionCode,
+        exception->ExceptionFlags,
+        exception->ExceptionRecord, /* follow if non NULL */
+        exception->NumberParameters);
+    if (exception->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
+        LOG(THREAD_GET, LOG_ASYNCH, 2, 
+            "\tPC "PFX" tried to %s address "PFX"\n",
+            exception->ExceptionAddress, 
+            exception_access_violation_type(exception->ExceptionInformation[0]),
+            exception->ExceptionInformation[1]);
+    }
+    dump_context_info(context, THREAD_GET, false);
+}
+
+static void
+dump_exception_frames()
+{
+    int depth = 0;
+    EXCEPTION_REGISTRATION* pexcrec = get_exception_list();
+
+    LOG(THREAD_GET, LOG_ASYNCH, 2,
+        "ASYNCH dump_exception_frames SEH frames head: "PFX"\n", pexcrec);
+
+    // 0xFFFFFFFF indicates the end of list
+    while ((EXCEPTION_REGISTRATION*)PTR_UINT_MINUS_1 != pexcrec) {
+        if (!is_readable_without_exception((app_pc)pexcrec, sizeof(EXCEPTION_REGISTRATION))) /* heavy weight check */ {
+            LOG(THREAD_GET, LOG_ASYNCH, 1, 
+                "ASYNCH dump_exception_frames "PFX" invalid! possibly corrupt\n",
+                pexcrec);
+            return;
+        }
+        DOLOG(2, LOG_ASYNCH, {
+            char symbolbuf[MAXIMUM_SYMBOL_LENGTH];
+            print_symbolic_address(pexcrec->handler,
+                                   symbolbuf, sizeof(symbolbuf), false);
+            LOG(THREAD_GET, LOG_ASYNCH, 2,
+                "ASYNCH frame[%d]: "PFX"  handler: "PFX" %s, prev: "PFX"\n",
+                depth,
+                pexcrec, pexcrec->handler, symbolbuf,
+                pexcrec->prev);
+        });
+        
+        pexcrec = pexcrec->prev;
+        if (depth++ > 100) {
+            LOG(THREAD_GET, LOG_ASYNCH, 2,
+                "ASYNCH frame[%d]: too deep chain, possibly corrupted\n", depth);
+            break;
+        }
+    }
+}
+
+#endif /* DEBUG */
+
+/* Data structure(s) pointed to by Visual C++ extended exception frame
+ * WARNING: these are compiler-dependent and we cannot count on any
+ * particular exception frame looking like this
+ */
+typedef struct scopetable_entry_t {
+    DWORD previousTryLevel;
+    PVOID lpfnFilter;
+    PVOID lpfnHandler;
+} scopetable_entry_t;
+ 
+/* The extended exception frame used by Visual C++ */
+typedef struct _vc_exception_registration_t
+{
+    EXCEPTION_REGISTRATION   exception_base;
+    struct scopetable_entry_t *scopetable;
+    int                      trylevel;
+    int                      _ebp;
+} vc_exception_registration_t;
+
+#ifdef DEBUG
+/* display the extended exception frame used by Visual C++
+ * There is at most one EXCEPTION_REGISTRATION record per function, 
+ * the rest is compiler dependent and we don't want to depend on that...
+ */
+void dump_vc_exception_frame(EXCEPTION_REGISTRATION * pexcreg)
+{
+    vc_exception_registration_t *pVCExcRec = (vc_exception_registration_t*) pexcreg;
+    struct scopetable_entry_t *pScopeTableEntry = pVCExcRec->scopetable;
+    int i;
+    for (i = 0; i <= pVCExcRec->trylevel; i++) {
+        LOG(THREAD_GET, LOG_ASYNCH, 2, "\t scope[%u] PrevTry: "PFX"  "
+            "filter: "PFX"  __except: "PFX"\n", i,
+            pScopeTableEntry->previousTryLevel,
+            pScopeTableEntry->lpfnFilter,
+            pScopeTableEntry->lpfnHandler );
+        pScopeTableEntry++;
+    }
+}
+#endif /* DEBUG */
+
+void
+internal_exception_info(dcontext_t *dcontext, EXCEPTION_RECORD *pExcptRec,
+                        CONTEXT *cxt, bool dstack_overflow)
+{
+    /* WARNING: a fault in DR means that potentially anything could be
+     * inconsistent or corrupted!  Do not grab locks or traverse
+     * data structures or read memory if you can avoid it!
+     */
+    /* Note this format string is at its limit.  Do not add anything
+     * else without compressing.  xref PR 204171.
+     * report_dynamorio_problem has an allocated buffer size that
+     * assumes the size here is MAX_PATH+11
+     */
+    const char *fmt = dstack_overflow ?
+        "Unrecoverable error at PC "PFX"\nStack Overflow\n"
+        PFX" "PFX" "PFX" "PFX" "PFX" "PFX"\nDelta: "PFX"\n" 
+        :
+        "Unrecoverable error at PC "PFX"\n"
+        PFX" "PFX" "PFX" "PFX" "PFX" "PFX"\n"
+        "Delta: "PFX"\n"
+        "Registers: eax="PFX" ebx="PFX" ecx="PFX" edx="PFX"\n"
+        "\tesi="PFX" edi="PFX" esp="PFX" ebp="PFX"\n"
+#ifdef X64
+        "\tr8 ="PFX" r9 ="PFX" r10="PFX" r11="PFX"\n"
+        "\tr12="PFX" r13="PFX" r14="PFX" r15="PFX"\n"
+#endif
+        "\teflags="PFX ;
+
+    /* case 5366 we may have randomized our location, for easier recognition,
+     * adjusting printed PC to preferred address.
+     */
+    app_pc adjusted_exception_addr = (app_pc) pExcptRec->ExceptionAddress;
+    ssize_t preferred_base_delta = 0; 
+    if (DYNAMO_OPTION(aslr_dr)) {
+        /* adjust if exception in possibly rebased DLL */
+        preferred_base_delta = get_dynamorio_dll_preferred_base() - 
+            get_dynamorio_dll_start();
+        /* presumably both are memoized */
+        if (is_in_dynamo_dll(adjusted_exception_addr))
+            adjusted_exception_addr += preferred_base_delta;
+    }
+
+    DODEBUG({
+        /* also check for a self-protection bug: write fault accessing data section */
+        if (pExcptRec->ExceptionCode == EXCEPTION_ACCESS_VIOLATION &&
+            pExcptRec->ExceptionInformation[0]==1 /* write */) {
+            app_pc target = (app_pc) pExcptRec->ExceptionInformation[1];
+            if (is_in_dynamo_dll(target)) {
+                const char *sec = get_data_section_name(target);
+                SYSLOG_INTERNAL_CRITICAL("Self-protection bug: %s written to @"PFX"/"PFX,
+                                         sec == NULL ? "" : sec, target, 
+                                         target + preferred_base_delta);
+            }
+        }
+    });
+
+    /* FIXME: we need to test whether the exception is due to 
+       Guard page violation - code 80000001 (not our own stack guard though)
+       Stack overflow - code c00000fd (last guard page touched)
+
+       We may get there because intercept_exception uses the application stack
+       before calling asynch_take_over and in case of an invalid exception handler
+       this will trickle down to it.
+               
+       We can currently get here if trying to read application pages marked as GUARD pages,
+       or marked as RESERVEd but not committed - in which case we'll receive a general
+       Access violation - code c0000005 and we'd have to verify the page flags.
+       Again we may want to treat differently our own guard pages than the application
+       uncommitted pages.
+    */
+
+    /* note that the first adjusted_exception_addr is used for
+     * eventlog, and the second for forensics, and so both need to be adjusted 
+     */
+    report_dynamorio_problem(dcontext, DUMPCORE_INTERNAL_EXCEPTION,
+                             adjusted_exception_addr, (app_pc) cxt->CXT_XBP,
+                             fmt,
+                             adjusted_exception_addr,
+                             pExcptRec->ExceptionCode, pExcptRec->ExceptionFlags,
+                             cxt->CXT_XIP, pExcptRec->ExceptionAddress,
+                             (pExcptRec->NumberParameters >= 1) ?
+                             pExcptRec->ExceptionInformation[0] : 0,
+                             (pExcptRec->NumberParameters >= 2) ?
+                             pExcptRec->ExceptionInformation[1] : 0,
+                             preferred_base_delta,
+                             cxt->CXT_XAX, cxt->CXT_XBX, cxt->CXT_XCX, cxt->CXT_XDX,
+                             cxt->CXT_XSI, cxt->CXT_XDI, cxt->CXT_XSP, cxt->CXT_XBP,
+#ifdef X64
+                             cxt->R8, cxt->R9, cxt->R10, cxt->R11,
+                             cxt->R12, cxt->R13, cxt->R14, cxt->R15,
+#endif
+                             cxt->CXT_XFLAGS);
+}
+
+static void
+internal_dynamo_exception(dcontext_t *dcontext, EXCEPTION_RECORD *pExcptRec, CONTEXT *cxt)
+{
+    /* recursive bailout: avoid infinite loop due to fault in fault handling
+     * by using DO_ONCE
+     * FIXME: any worries about lack of mutex w/ DO_ONCE?
+     */
+#ifdef STACK_GUARD_PAGE
+    /* PR 203701: If we've exhausted the dstack, then we'll switch
+     * to a separate exception handling stack to make sure we have
+     * enough space to report the problem.  One guard page is not
+     * always sufficient.
+     */
+    DO_ONCE({
+        if (is_dstack_overflow(dcontext, pExcptRec, cxt) && exception_stack != NULL) {
+            mutex_lock(&exception_stack_lock);
+            call_intr_excpt_alt_stack(dcontext, pExcptRec, cxt, exception_stack);
+            mutex_unlock(&exception_stack_lock);
+        }
+        else {
+            internal_exception_info(dcontext, pExcptRec, cxt, false);
+        }
+    });
+#else
+    DO_ONCE({ internal_exception_info(dcontext, pExcptRec, cxt, false); });
+#endif
+    os_terminate(dcontext, TERMINATE_PROCESS);
+    ASSERT_NOT_REACHED();
+}
+
+/* heuristic check whether an exception is due to execution or due to
+ * a read from unreadable memory */
+static bool
+is_execution_exception(EXCEPTION_RECORD *pExcptRec)
+{
+    app_pc fault_pc = pExcptRec->ExceptionAddress;
+    app_pc target = (app_pc) pExcptRec->ExceptionInformation[1];
+    bool execution = false;
+
+    ASSERT(pExcptRec->ExceptionCode == EXCEPTION_ACCESS_VIOLATION);
+ 
+    if (pExcptRec->ExceptionInformation[0] == 
+        EXCEPTION_INFORMATION_EXECUTE_FAULT) {
+        /* certainly execution */
+        execution = true;
+    }
+
+    /* FIXME: case 5879 should know if running on an NX capable
+     * machine and whether this information code depends on whether
+     * the current application is NX compatible. Verify that if this
+     * is not set, it is expected to be just a read fault.  For the
+     * time being using the read/execute heuristic all the time 
+     */
+
+    if (pExcptRec->ExceptionInformation[0] == 
+        EXCEPTION_INFORMATION_READ_EXECUTE_FAULT) {
+        if (fault_pc == target) {
+            /* certainly execution */
+            execution = true;
+        } else if (fault_pc < target && 
+                   target < fault_pc + MAX_INSTR_LENGTH) {
+            /* near a page boundary crossing a read exception can
+             * happen either while reading the instruction (execution)
+             * or while the instruction is reading from a nearby
+             * location.  Without decoding cannot say which one is it,
+             * but most likely it is execution.
+             */
+
+            /* unclear whether execution or read */
+
+            /* if we are not crossing a page boundary we can't
+             * possibly fail, so should ASSERT on that, and also on
+             * is_readable_without_exception() to make sure we don't
+             * have a race
+             */
+            
+            execution = true;   /* execution more likely */
+            /* FIXME: case 1948 actually has to deal with this more
+             * precisely when instructions may cross pages with
+             * different permissions */
+            ASSERT_NOT_IMPLEMENTED(false);
+        } else {
+            /* read otherwise */
+            execution = false;
+        }
+    }
+    return execution;
+}
+
+/*
+ * Exceptions:
+ * Kernel gives control to KiUserExceptionDispatcher.
+ * It examines linked list of EXCEPTION_REGISTRATION_RECORDs, which
+ * each contain a callback function and a next pointer.
+ * Calls each callback function, passing 0 for the exception flags,
+ * asking what they want to do.  Callback function corresponds to the filter
+ * for the __except block.  Typically a Visual C++ wrapper routine
+ * __except_handler3 is used, which expects filter return values of:
+ *   EXCEPTION_EXECUTE_HANDLER = execute this handler
+ *   NT_CONTINUE_SEARCH = keep walking chain of handlers
+ *   NT_CONTINUE_EXECUTION = re-execute faulting instruction
+ * Once an accepting filter is found, walks chain of records again
+ * by calling __global_unwind2(), which calls the callback functions again,
+ * passing 2 (==EH_UNWINDING) as the flag, giving each a chance
+ * to clean up.  Accepting handler is responsible for properly setting
+ * app state (including stack to be same as frame that contains handler
+ * code) and then jumping to the right pc -- __except_handler3 does all this,
+ * then calls code corresponding to __except block {} itself.
+ *
+ * Error during exception handler search -> raises an unhandleable exception
+ * Global unwind = handler itself calls RtlUnwind
+ * RtlUnwind calls NtContinue to continue execution at faulty instruction or
+ *   after handler
+ *
+KiUserExceptionDispatcher:
+  77F9F054: 8B 4C 24 04        mov         ecx,dword ptr [esp+4]
+  77F9F058: 8B 1C 24           mov         ebx,dword ptr [esp]
+  77F9F05B: 51                 push        ecx
+  ...
+
+ * This is entered directly from the kernel, so there is no return address on
+ * the stack like there is with NtContinue
+ *
+ * ASSUMPTIONS:
+ *    1) *esp = EXCEPTION_RECORD*
+ *    2) *(esp+4) == CONTEXT*
+ * For x64:
+ *    1) *rsp = CONTEXT
+ *    2) *(rsp+sizeof(CONTEXT)) = EXCEPTION_RECORD
+ */
+/* Remember that every path out of here must invoke the DR exit hook.
+ * The normal return path will do so as the interception code has an
+ * enter and exit hook around the call to this routine.
+ */
+static after_intercept_action_t  /* note return value will be ignored */
+intercept_exception(app_state_at_intercept_t *state)
+{
+    /* FIXME : if dr is calling through Nt* wrappers that are hooked
+     * (say by sygate's sysfer.dll) they may generate and handle exceptions
+     * (throw) for which we should backout here.  (We should neither take
+     * responsibility nor start interpreting).  Keep in mind we make some
+     * syscalls on the app stack (intercept_apc -> init etc.) and that we may
+     * also make some before a dcontext is created for the thread.  Right now
+     * we avoid calling through sygate's hooks (do our own system calls). */
+    /* FIXME - no real scheme to handle us calling SYSFER hooks that trip over
+     * a region we made read-only. */
+
+    /* we intercept known threads even if !intercept_asynch_for_self, to
+     * handle write faults on regions we made RO
+     */
+
+    /* if we have a valid dcontext then we're really valid, but we
+     * could also have been just created so we also allow
+     * is_thread_known().  
+     * FIXME: is_thread_known() may be unnecessary */
+    dcontext_t *dcontext = get_thread_private_dcontext();
+
+    if (intercept_asynch_global() && 
+        (dcontext != NULL || is_thread_known(get_thread_id()))) {
+        dr_mcontext_t mcontext;
+        app_pc forged_exception_addr;
+        EXCEPTION_RECORD *pExcptRec;
+        CONTEXT *cxt;
+        cache_pc faulting_pc;
+        byte *fault_xsp;
+        /* if !takeover, we handle our-fault write faults, but then let go */
+        bool takeover = intercept_asynch_for_self(false/*no unknown threads*/);
+        bool thread_is_lost = false; /* temporarily native (UNDER_DYN_HACK) */
+        DEBUG_DECLARE(bool known_source = false;)
+        
+        /* grab parameters to native method */
+#ifdef X64
+        pExcptRec = (EXCEPTION_RECORD *) (state->mc.xsp + sizeof(CONTEXT));
+        cxt = (CONTEXT *) state->mc.xsp;
+#else
+        pExcptRec = *((EXCEPTION_RECORD **)(state->mc.xsp));
+        cxt = *((CONTEXT **)(state->mc.xsp + XSP_SZ));
+#endif
+        fault_xsp = (byte *) cxt->CXT_XSP;
+
+        if (dcontext == NULL) {
+            ASSERT_NOT_TESTED();
+            SYSLOG_INTERNAL_CRITICAL("Early thread failure, no dcontext\n");
+            /* there is no good reason for this, other than DR error */
+            ASSERT(is_dynamo_address((app_pc)pExcptRec->ExceptionAddress));
+            pExcptRec->ExceptionFlags = 0xbadDC;
+            internal_dynamo_exception(dcontext, pExcptRec, cxt);
+            ASSERT_NOT_REACHED();
+        }
+
+        forged_exception_addr = dcontext->forged_exception_addr;
+
+        /* FIXME : we'd like to retakeover lost-control threads, but we need
+         * to correct writable->read_only faults etc. as if for a native thread
+         * and the helper routines (below code, check/found/handle modified
+         * code) don't support a native thread that we want to retakeover (ref
+         * case 6069). So, we just treat the thread as a native_exec thread
+         * and wait for a later retakeover point to regain control. */
+        if (takeover == UNDER_DYN_HACK) {
+            STATS_INC(num_except_while_lost);
+            thread_is_lost = true;
+            takeover = false;
+        }
+
+        SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+        /* won't be re-protected until dispatch->fcache */
+
+        RSTATS_INC(num_exceptions);
+
+        dcontext->forged_exception_addr = NULL;
+
+        LOG(THREAD, LOG_ASYNCH, 1,
+            "ASYNCH intercepted exception in %sthread %d at pc "PFX"\n", 
+            takeover ? "" : "non-asynch ", get_thread_id(),
+            pExcptRec->ExceptionAddress);
+        DOLOG(2, LOG_ASYNCH, {
+            if (cxt->CXT_XIP != (ptr_uint_t) pExcptRec->ExceptionAddress)
+                LOG(THREAD, LOG_ASYNCH, 2, "\tcxt pc is different: "PFX"\n",
+                    cxt->CXT_XIP);
+        });
+        DOLOG(2, LOG_ASYNCH, {
+            dump_exception_info(pExcptRec, cxt);
+            dump_exception_frames(); /* check what handlers are installed */
+        });
+        DOLOG(2, LOG_ASYNCH, {
+            /* verify attack handling assumptions on valid frames */
+            if (IF_X64_ELSE(is_wow64_process(NT_CURRENT_PROCESS), true))
+                exception_frame_chain_depth(dcontext);
+        });
+
+#ifdef HOT_PATCHING_INTERFACE
+        /* Recover from a hot patch exception. */
+        if (dcontext->whereami == WHERE_HOTPATCH) {
+            /* Note: If we use a separate stack for executing hot patches, this
+             * assert should be changed.
+             */
+            ASSERT(is_on_dstack(dcontext, (byte *)cxt->CXT_XSP));
+            if (is_on_dstack(dcontext, (byte *)cxt->CXT_XSP)) {
+                char excpt_addr[16];
+                snprintf(excpt_addr, BUFFER_SIZE_ELEMENTS(excpt_addr), PFX,
+                         (byte *)cxt->CXT_XIP);
+                NULL_TERMINATE_BUFFER(excpt_addr);
+
+                /* Forensics for this event are dumped in hotp_execute_patch() 
+                 * because only that has vulnerability specific information. 
+                 */
+                SYSLOG_CUSTOM_NOTIFY(SYSLOG_ERROR, MSG_HOT_PATCH_FAILURE, 3,
+                                     "Hot patch exception, continuing.", 
+                                     get_application_name(), 
+                                     get_application_pid(), excpt_addr);
+                if (TEST(DUMPCORE_HOTP_FAILURE, DYNAMO_OPTION(dumpcore_mask)))
+                    os_dump_core("hotp exception");
+
+                /* we don't support filters, so a single pass through
+                 * all FINALLY and EXCEPT handlers is sufficient
+                 */
+
+                /* The exception interception code did an ENTER so we must EXIT here */
+                EXITING_DR();
+                DR_LONGJMP(&dcontext->hotp_excpt_state, LONGJMP_EXCEPTION);
+            } 
+            /* Else, if it is on init stack, the control flow below would
+             * report an internal error.
+             */
+        }
+#endif
+
+        if (dcontext->try_except_state != NULL) {
+            /* handle our own TRY/EXCEPT */
+            /* similar to hotpatch exceptions above */
+
+            /* FIXME: noisy syslog, but actually not even enough.  How
+             * should we make sure we don't have a TRY/EXCEPT that is
+             * encompassing too much?  While in release build
+             * recovering is a good thing, any unexpected faults
+             * should be visible in debug builds.
+             */
+            SYSLOG_INTERNAL_WARNING("Handling our fault in a TRY at "PFX, cxt->CXT_XIP);
+
+            ASSERT(is_on_dstack(dcontext, (byte *)cxt->CXT_XSP));
+            ASSERT(pExcptRec->ExceptionCode == EXCEPTION_ACCESS_VIOLATION);
+            ASSERT_CURIOSITY((pExcptRec->NumberParameters >= 2) && 
+                             (pExcptRec->ExceptionInformation[0] == 
+                              EXCEPTION_INFORMATION_READ_EXECUTE_FAULT
+                              && "currently only racy reads"));
+
+            if (TEST(DUMPCORE_TRY_EXCEPT, DYNAMO_OPTION(dumpcore_mask)))
+                os_dump_core("try/except fault");
+
+            /* The exception interception code did an ENTER so we must EXIT here */
+            EXITING_DR();
+            DR_LONGJMP(&dcontext->try_except_state->context, LONGJMP_EXCEPTION);
+            ASSERT_NOT_REACHED();
+        }
+
+#ifdef CLIENT_INTERFACE
+        if (!IS_INTERNAL_STRING_OPTION_EMPTY(client_lib) &&
+            is_in_client_lib(pExcptRec->ExceptionAddress)) {
+            char excpt_addr[IF_X64_ELSE(20,12)];
+            snprintf(excpt_addr, BUFFER_SIZE_ELEMENTS(excpt_addr),
+                     PFX, pExcptRec->ExceptionAddress);
+            NULL_TERMINATE_BUFFER(excpt_addr);
+            SYSLOG_CUSTOM_NOTIFY(SYSLOG_ERROR, MSG_CLIENT_EXCEPTION, 3,
+                                 "Exception in client library.", 
+                                 get_application_name(), 
+                                 get_application_pid(),
+                                 excpt_addr);
+            /* kill process on a crash in client code */
+            os_terminate(NULL, TERMINATE_PROCESS);
+        }
+#endif
+
+        /* If we set a thread's context after it received a fault but
+         * before the kernel copied the faulting context to the user
+         * mode structures for the handler, we can come here and think
+         * it faulted at the pc we set its context to (case 7393).
+         * ASSUMPTION: we will not actually fault at any of these addresses.
+         * We could set a flag in the dcontext before we setcontext and
+         * clear it afterward to reduce the mis-diagnosis chance.
+         */
+        if ((app_pc) pExcptRec->ExceptionAddress == get_reset_exit_stub(dcontext)) {
+            ASSERT(!RUNNING_WITHOUT_CODE_CACHE());
+            /* Restore to faulting address, which is an app address.
+             * Thus we skip the translation steps below.
+             * We must do our own modified code check as well, and since the original
+             * cache is gone (so saving the address wouldn't help) we need custom
+             * handling along that path.
+             */
+            ASSERT(!in_fcache(dcontext->next_tag));
+            pExcptRec->ExceptionAddress = (PVOID) dcontext->next_tag;
+            cxt->CXT_XIP = (ptr_uint_t) dcontext->next_tag;
+            STATS_INC(num_reset_setcontext_at_fault);
+            SYSLOG_INTERNAL_WARNING("reset SetContext at faulting instruction");
+            check_for_modified_code(dcontext, pExcptRec, cxt,
+                                    MOD_CODE_TAKEOVER | MOD_CODE_APP_CXT);
+            /* now handle the fault just like RaiseException */
+            DODEBUG({ known_source = true; });
+        } else if ((app_pc) pExcptRec->ExceptionAddress == 
+                   get_setcontext_interceptor()) {
+            ASSERT(!RUNNING_WITHOUT_CODE_CACHE());
+            /* Restore to faulting address, no need to go to our interception routine
+             * as natively we'd have the fault and go from there.
+             */
+            /* FIXME case 7456: we don't have original fault address so we
+             * can't properly process a codemod fault!  Need to have SetContext
+             * pre-handler store that somewhere.  Here we need new code paths to
+             * do the codemod handling but then go to the SetContext target
+             * rather than re-execute the write.  NOT IMPLEMENTED!
+             */
+            ASSERT_NOT_IMPLEMENTED(false && "app SetContext on faulting instr");
+            STATS_INC(num_app_setcontext_at_fault);
+            pExcptRec->ExceptionAddress = (PVOID) dcontext->asynch_target;
+            cxt->CXT_XIP = (ptr_uint_t) dcontext->asynch_target;
+            /* now handle the fault just like RaiseException */
+            DODEBUG({ known_source = true; });
+        }
+
+        /* even though in_fcache is the much more common path (we hope! :)),
+         * it grabs a lock, so we check for DR exceptions first, hoping to
+         * avoid livelock due to us crashing while holding the fcache lock
+         */
+        /* FIXME : we still might pass exceptions that are our fault back to
+         * the app (in a client library, global do syscall, client library dgc
+         * maybe others?). Also is the on_dstack/on_initstack check too 
+         * general?  We might take responsibility for app crashes if their esp 
+         * gets set to random address that happens to match one of our stacks.
+         * We could additionally require that the pc is also in ntdll.dll or 
+         * kernel32.dll (that would cover cases (like bug 3516) where we call 
+         * out to other dlls) though as it is now it may cover some of the 
+         * remaining holes (client library for instance).  Is also possible 
+         * that we could take responsibility for an app exception that occurs
+         * in the first few instructions of a location we hooked (since if
+         * we didn't takover at the hook, it would execute out of the 
+         * interception buffer (guard page on stack for instance)).
+         */
+        /* Note the is_on_[init/d]stack routines count any guard pages as part
+         * of the stack */
+        if ((is_on_dstack(dcontext, (byte *)cxt->CXT_XSP)
+             /* PR 302951: clean call arg processing => pass to app/client.
+              * Rather than call the risky in_fcache we check whereami. */
+             IF_CLIENT_INTERFACE(&& (dcontext->whereami != WHERE_FCACHE))) ||
+            is_on_initstack((byte *)cxt->CXT_XSP) ||
+            /* Is this an exception forged by DR that should be passed on
+             * to the app? */
+            (forged_exception_addr != (app_pc) pExcptRec->ExceptionAddress &&
+             (is_in_dynamo_dll((app_pc)pExcptRec->ExceptionAddress) ||
+             /* we go ahead and grab locks here to do a negative test for
+              * !in_fcache rather than trying to enumerate all non-cache
+              * categories, as we'll have to grab a lock anyway to find
+              * whether in a separate stub region.  we do this last to
+              * reduce the scenarios in which we won't report a crash.
+              */
+              (is_dynamo_address((app_pc)pExcptRec->ExceptionAddress) && 
+               !in_fcache(pExcptRec->ExceptionAddress))))) {
+            /* Check if we ended up decoding from unreadable memory due to an 
+             * app race condition (case 845) or hit an IN_PAGE_ERROR (case 10567) */
+            if ((pExcptRec->ExceptionCode == EXCEPTION_ACCESS_VIOLATION ||
+                 pExcptRec->ExceptionCode == EXCEPTION_IN_PAGE_ERROR) &&
+                (pExcptRec->NumberParameters >= 2) && 
+                (pExcptRec->ExceptionInformation[0] == 
+                 EXCEPTION_INFORMATION_READ_EXECUTE_FAULT
+                 )) {
+                app_pc target_addr = (app_pc)pExcptRec->ExceptionInformation[1];
+                ASSERT((pExcptRec->ExceptionCode == EXCEPTION_IN_PAGE_ERROR)
+                       || !is_readable_without_exception(target_addr, 4));
+                /* for shared fragments, the bb building lock is what prevents
+                 * another thread from changing the shared last_area before
+                 * we check it
+                 * note: for hotp_only or thin_client, this shouldn't trigger,
+                 *       especially for thin_client because it will crash as
+                 *       uninitialized vmarea_vectors will be accessed.
+                 */
+                if (!RUNNING_WITHOUT_CODE_CACHE() &&
+                    check_in_last_thread_vm_area(dcontext, target_addr)) {
+
+                    exception_type_t exception_type =
+                        (pExcptRec->ExceptionCode == EXCEPTION_IN_PAGE_ERROR) ?
+                        IN_PAGE_ERROR_EXCEPTION : UNREADABLE_MEMORY_EXECUTION_EXCEPTION;
+
+                    /* The last decoded application pc should always be in the 
+                     * thread's last area, yet if code executed by one thread 
+                     * is unmapped by another we may have let it through 
+                     * check_thread_vm_area and into decode*()
+                     */
+                    SYSLOG_INTERNAL_ERROR("(decode) exception in last area, "
+                                          "%s: dr pc="PFX", app pc="PFX,
+                                          (exception_type == IN_PAGE_ERROR_EXCEPTION) ?
+                                          "in_page_error" : "probably app race condition",
+                                          (app_pc)pExcptRec->ExceptionAddress,
+                                          target_addr);
+                    STATS_INC(num_exceptions_decode);
+                    if (is_building_trace(dcontext)) {
+                        LOG(THREAD, LOG_ASYNCH, 2, "intercept_exception: "
+                                                   "squashing old trace\n");
+                        trace_abort(dcontext);
+                    }
+                    /* we do get faults when not building a bb: e.g.,
+                     * ret_after_call_check does decoding (case 9396) */
+                    if (dcontext->bb_build_info != NULL) {
+                        /* must have been building a bb at the time */
+                        bb_build_abort(dcontext, true/*clean vm area*/);
+                    }
+                    /* FIXME: if necessary, have a separate dump core mask for
+                     * in_page_error */
+                    /* Let's pass it back to the application - memory is unreadable */
+                    if (TEST(DUMPCORE_FORGE_UNREAD_EXEC,
+                             DYNAMO_OPTION(dumpcore_mask)))
+                        os_dump_core("Warning: Racy app execution "
+                                     "(decode unreadable)");
+                    os_forge_exception(target_addr, exception_type);
+
+                    /* CHECK: I hope we're not covering up the symptom instead 
+                     * of fixing the real cause
+                     */
+                    ASSERT_NOT_REACHED();
+                }
+            }
+
+            internal_dynamo_exception(dcontext, pExcptRec, cxt);
+            ASSERT_NOT_REACHED();
+        }
+
+        /* don't do this for DR exception in case it's exception in trace_abort! */
+        if (is_building_trace(dcontext)) {
+            LOG(THREAD, LOG_ASYNCH, 2, "intercept_exception: squashing old trace\n");
+            trace_abort(dcontext);
+        }
+
+        /* FIXME: we may want to distinguish the exception that we
+         * have generated from interp from potential races that will
+         * result in this really being generated in the fcache. */
+        /* FIXME: there is no reason a native_exec thread can't natively
+         * generate (and even handle) one of these exceptions. Of course,
+         * judging by the need for the filter, doesn't even look like it needs
+         * to be native. */
+        ASSERT_CURIOSITY(pExcptRec->ExceptionCode != STATUS_ILLEGAL_INSTRUCTION ||
+                         check_filter("decode-prefixes.exe;decode-bad.exe;"
+                                      "decode-bad-stack.exe;decode.exe;"
+                                      "decode-stress.exe",
+                                      get_short_name(get_application_name())));
+        ASSERT_CURIOSITY(pExcptRec->ExceptionCode != STATUS_PRIVILEGED_INSTRUCTION
+                         || check_filter("decode-priv.exe;decode-bad.exe", 
+                                         get_short_name(get_application_name())));
+
+        /* if !takeover, the thread could be native and not in fcache */
+        if (!takeover || in_fcache((void *)(pExcptRec->ExceptionAddress))) {
+            recreate_success_t res;
+            /* special case: we expect a seg fault for executable regions
+             * that were writable and marked read-only by us.
+             * if it is modified code, routine won't return to us
+             * (it takes care of initstack though).
+             * checking for modified code shouldn't be done for thin_client.
+             * note: hotp_only plays around with memory protection to smash
+             *       hooks, so can't be ignore; vlad: in -client mode we
+             *       smash hooks using emulate_write
+             */
+            if (!DYNAMO_OPTION(thin_client)) {
+                check_for_modified_code(dcontext, pExcptRec, cxt,
+                                        takeover ? MOD_CODE_TAKEOVER : 0);
+            }
+            if (!takeover) {
+#ifdef PROGRAM_SHEPHERDING
+                /* check for an ASLR execution violation - currently
+                 * should only be hit in hotp_only, but could also be
+                 * in native_exec threads
+                 * FIXME: can we tell hotp from native if we wanted to?
+                 */
+                /* FIXME: Currently we only track execution faults in
+                 * unreadable memory.  If we allow mapping of data,
+                 * e.g. !ASLR_RESERVE_AREAS then should track any
+                 * other exception, ILLEGAL_INSTRUCTION_EXCEPTION or
+                 * PRIVILEGED_INSTRUCTION or anything else with an
+                 * ExceptionAddress in that region.  However, can't do
+                 * the same if other DLLs can map there,
+                 * e.g. !(ASLR_AVOID_AREAS|ASLR_RESERVE_AREAS).
+                 */
+                if (DYNAMO_OPTION(aslr) != ASLR_DISABLED && 
+                    (pExcptRec->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) &&
+                    is_execution_exception(pExcptRec)) {
+                    app_pc execution_addr = pExcptRec->ExceptionAddress;
+                    if (aslr_is_possible_attack(execution_addr) &&
+                        /* ignore if we have just forged_exception_addr */
+                        execution_addr != forged_exception_addr) {
+                        security_option_t handling_policy = OPTION_BLOCK;
+
+                        LOG(THREAD, LOG_ASYNCH, 1, "Exception at "PFX" is due "
+                            "to randomization, under attack!\n", execution_addr);
+                        SYSLOG_INTERNAL_ERROR("ASLR: execution attempt "PFX" "
+                                              "in preferred DLL range\n",
+                                              execution_addr);
+
+                        if (TEST(ASLR_HANDLING, DYNAMO_OPTION(aslr_action))) 
+                            handling_policy |= OPTION_HANDLING;
+
+                        if (TEST(ASLR_REPORT, DYNAMO_OPTION(aslr_action)))
+                            handling_policy |= OPTION_REPORT;
+
+                        /* for reporting purposes copy application
+                         * context in our context, we don't need our
+                         * dr_mcontext_t at all otherwise
+                         */
+                        context_to_mcontext(get_mcontext(dcontext), cxt);
+                        aslr_report_violation(execution_addr,
+                                              handling_policy);
+                        /* should normally return so we pass the
+                         * exception to the application, unless we
+                         * want to forcefully handle it
+                         */
+                        ASSERT(!TEST(OPTION_HANDLING, handling_policy) &&
+                               "doesn't return");
+                    }
+                }
+#endif /* PROGRAM_SHEPHERDING */
+
+                /* Note - temporarily lost control threads (UNDER_DYN_HACK) are
+                 * whereami == WHERE_FCACHE (FIXME would be more logical to be
+                 * WHERE_APP) and !takeover, but unlike the forge case we don't
+                 * need to fix them up here. */ 
+                if (!thread_is_lost) {
+                    /* xref 8267, can't just check that exception addr matches
+                     * forged addr because that can falsely match at 0, so
+                     * we base on whereami instead. */
+                    if (dcontext->whereami == WHERE_FCACHE) {
+                        /* Xref case 8219 - forge exception sets whereami to
+                         * WHERE_FCACHE while we perform the RaiseException.  Need
+                         * to set the whereami back to WHERE_APP now since not
+                         * taking over. */
+                        ASSERT_CURIOSITY(pExcptRec->ExceptionAddress ==
+                                         forged_exception_addr);
+#ifdef PROGRAM_SHEPHERDING
+                        /* only known to happen on throw exception */
+                        ASSERT_CURIOSITY(DYNAMO_OPTION(throw_exception));
+#else
+                        ASSERT_CURIOSITY(false && "should not be reached");
+#endif
+                        dcontext->whereami = WHERE_APP;
+                    } else {
+                        /* should already be WHERE_APP then */
+                        ASSERT_CURIOSITY(dcontext->whereami == WHERE_APP);
+                        /* this should not be a forged exception */
+                        ASSERT_CURIOSITY(pExcptRec->ExceptionAddress !=
+                                         forged_exception_addr || /* 8267 */
+                                         forged_exception_addr == NULL);
+                    }
+                }
+
+                /* wasn't our fault, let it go back to app */
+                if (TEST(DUMPCORE_APP_EXCEPTION, DYNAMO_OPTION(dumpcore_mask)))
+                    os_dump_core("Info: App exception, not taking over");
+#ifdef PROTECT_FROM_APP
+                SELF_PROTECT_LOCAL(dcontext, READONLY);
+#endif
+                return AFTER_INTERCEPT_LET_GO;
+            }
+
+            LOG(THREAD, LOG_ASYNCH, 1, "Exception is in code cache\n");
+            ASSERT(!RUNNING_WITHOUT_CODE_CACHE());
+            DOLOG(2, LOG_ASYNCH, {
+                fragment_t wrapper;
+                fragment_t *f;
+                f = fragment_pclookup(dcontext,
+                                      (void *)(pExcptRec->ExceptionAddress), &wrapper);
+                LOG(THREAD, LOG_ASYNCH, 2, "Exception is in this fragment:\n");
+                /* We might not find the fragment since if it is shared and
+                 * pending deletion for flush, may have already been removed
+                 * from the lookup tables */
+                if (f != NULL)
+                    disassemble_fragment(dcontext, f, false);
+                else
+                    LOG(THREAD, LOG_ASYNCH, 2, "Fragment not found");
+            });
+            /* Need to fix EXCEPTION_RECORD's pc and CONTEXT's registers
+             * to pretend it was original code, not cache code 
+             */
+            /* remember faulting pc */
+            faulting_pc = (cache_pc) pExcptRec->ExceptionAddress;
+            /* For safe recreation we need to either be couldbelinking or hold the
+             * initexit lock (to keep someone from flushing current fragment), the
+             * initexit lock is easier
+             */
+            mutex_lock(&thread_initexit_lock);
+            if (cxt->CXT_XIP != (ptr_uint_t)pExcptRec->ExceptionAddress) {
+                /* sometimes this happens, certainly cxt can be changed by exception
+                 * handlers, but not clear why kernel changes it before getting here.
+                 * I saw it pointing to next instr while ExceptionAddress was faulting
+                 * instr...who knows, in any case we translate them both, but do
+                 * exceptionaddress first since cxt is the one we want for real, we
+                 * just want pc for exceptionaddress.
+                 */
+                app_pc translated_pc = 
+                    recreate_app_pc(dcontext, pExcptRec->ExceptionAddress, NULL);
+                ASSERT(translated_pc != NULL);
+                LOG(THREAD, LOG_ASYNCH, 2, "Translated ExceptionAddress "
+                    PFX" to "PFX"\n",
+                    pExcptRec->ExceptionAddress, translated_pc);
+                pExcptRec->ExceptionAddress = (PVOID) translated_pc;
+            }
+            context_to_mcontext(&mcontext, cxt);
+            res = recreate_app_state(dcontext, &mcontext, true/*memory too*/);
+            if (res != RECREATE_SUCCESS_STATE) {
+                /* We don't expect to get here: means an exception from an
+                 * instruction we added.  FIXME: today we do have some
+                 * translation pieces that are NYI: selfmod(PR 267764),
+                 * native_exec or windows sysenter (PR 303413), a flushed
+                 * fragment (PR 208037), or a hot patch fragment (PR
+                 * 213251: we can get here if a hot patched bb excepts
+                 * because of app code and the hot patch was applied at the
+                 * first instr; this is because the translation_target for
+                 * that will be wrong and recreating app state will get
+                 * messed up.  However, it will end up with the right
+                 * state.  See case 5981.)
+                 */
+                SYSLOG_INTERNAL_WARNING("Unable to fully translate context for "
+                                        "exception in the cache");
+                /* we should always at least get pc right */
+                ASSERT(res == RECREATE_SUCCESS_PC);
+            }
+            mutex_unlock(&thread_initexit_lock);
+            if (cxt->CXT_XIP == (ptr_uint_t)pExcptRec->ExceptionAddress)
+                pExcptRec->ExceptionAddress = (PVOID) mcontext.pc;
+            LOG(THREAD, LOG_ASYNCH, 2, "Translated cxt->Xip "PFX" to "PFX"\n",
+                cxt->CXT_XIP, mcontext.pc);
+            mcontext_to_context(cxt, &mcontext);
+
+            /* PR 306410: if exception while on dstack but going to app,
+             * copy SEH frame over to app stack and update handler xsp.
+             */
+            if (is_on_dstack(dcontext, fault_xsp)) {
+                size_t frame_sz = sizeof(CONTEXT) + sizeof(EXCEPTION_RECORD);
+                bool frame_copied;
+                IF_NOT_X64(frame_sz += XSP_SZ*2 /* 2 args */);
+                ASSERT(!is_on_dstack(dcontext, (byte *)cxt->CXT_XSP));
+                frame_copied = safe_write((byte *)cxt->CXT_XSP - frame_sz, frame_sz,
+                                          (byte *)state->mc.xsp);
+                LOG(THREAD, LOG_ASYNCH, 2,
+                    "exception on dstack; copied %d-byte SEH frame from "PFX
+                    " to app stack "PFX"\n",
+                    frame_sz, state->mc.xsp, cxt->CXT_XSP - frame_sz);
+                state->mc.xsp = cxt->CXT_XSP - frame_sz;
+#ifndef X64
+                /* update pointers */
+                *((byte **)state->mc.xsp) = (byte *)state->mc.xsp + 2*XSP_SZ;
+                *((byte **)(state->mc.xsp+XSP_SZ)) = (byte *)
+                    state->mc.xsp + 2*XSP_SZ + sizeof(EXCEPTION_RECORD);
+#endif
+                /* x64 KiUserExceptionDispatcher does not take any args */
+                if (!frame_copied) {
+                    SYSLOG_INTERNAL_WARNING("Unable to copy on-dstack app SEH frame "
+                                            "to app stack");
+                    /* FIXME: terminate app?  forge exception (though that does
+                     * a getcontext right here)?  can this be just a guard page? */
+                    ASSERT_NOT_REACHED();
+                }
+            }
+
+            /* we interpret init and other pieces of our own dll */
+            if (is_dynamo_address(mcontext.pc)) {
+                SYSLOG_INTERNAL_CRITICAL("Exception in cache "PFX" "
+                                         "interpreting DR code "PFX,
+                                         faulting_pc, mcontext.pc);
+                /* to avoid confusion over whether the original DR pc was
+                 * native or in the cache we hack the exception flags
+                 */
+                pExcptRec->ExceptionFlags = 0xbadcad;
+                internal_dynamo_exception(dcontext, pExcptRec, cxt);
+                ASSERT_NOT_REACHED();
+            }
+
+#ifdef CLIENT_INTERFACE
+            /* Inform client of exceptions */
+            if (!IS_INTERNAL_STRING_OPTION_EMPTY(client_lib)) {
+                dr_exception_t exception;
+                exception.record = pExcptRec;
+                context_to_mcontext(&exception.mcontext, cxt);
+                instrument_exception(dcontext, &exception);
+            }
+#endif
+            /* Fixme : we could do this higher up in the function (before
+             * translation) but then wouldn't be able to separate out the case
+             * of faulting interpreted dynamo address above. Prob. is nicer
+             * to have the final translation available in the dump anyways.*/
+            if (TEST(DUMPCORE_APP_EXCEPTION, DYNAMO_OPTION(dumpcore_mask)))
+                os_dump_core("Info: App exception, taking over");
+        } else {
+            /* If the exception pc is not in the fcache, then the exception
+             * was generated by calling RaiseException, or it's one of the
+             * SetContext cases up above.
+             * RaiseException calls RtlRaiseException, and the return address
+             * of that call site becomes the exception pc
+             * RtlRaiseException stores all registers, eflags, and segment
+             * registers somewhere (presumably a CONTEXT struct), and then
+             * calls NtRaiseException, which does int 0x2e
+             * User mode is re-entered in the exception handler with
+             * a pc equal to the return address from way up the call stack
+             *
+             * Now, the question is, how do we make sure the CONTEXT is ok?
+             * It's only not ok if we've optimized stuff around the call to
+             * RaiseException, right?
+             * FIXME: should we simpy not optimize there?  Else we need
+             * a special translator that knows to look back there, rather
+             * than at the exception pc (== after call to RtlRaiseException)
+             *
+             * Also, note that exceptions we generate via os_forge_exception 
+             * end up here as well, they need no translation.
+             */
+            DOLOG(1, LOG_ASYNCH, {
+                  if (!known_source) {
+                      LOG(THREAD, LOG_ASYNCH, 1,
+                          "Exception was generated by call to RaiseException\n");
+                  }
+            });
+        }
+
+        /* We won't get here for UNDER_DYN_HACK since at the top of the routine
+         * we set takeover to false for that case.  FIXME - if we clean up the
+         * above if and the check/found/handle modified code paths to handle
+         * UNDER_DYN_HACK correctly then we can use this as a retakeover point
+         */
+        asynch_retakeover_if_native();
+        /* We want to squash the current trace (don't want traces
+         * following exceptions), asynch_take_over does that for us.
+         * We don't save the cur dcontext.
+         */
+        state->callee_arg = (void *) false /* use cur dcontext */;
+        asynch_take_over(state);
+    } else
+        STATS_INC(num_exceptions_noasynch);
+    return AFTER_INTERCEPT_LET_GO;
+}
+
+/* We have yet to ever see this!
+ * Disassembly reveals that all it does is call RtlRaiseException,
+ * which will dive right back into the kernel and re-appear in
+ * KiUserExceptionDispatcher.
+ * We intercept this simply for completeness, to run ALL the user mode
+ * code.  We do nothing special, just take over.
+ *
+KiRaiseUserExceptionDispatcher:
+  77FA0384: 50                 push        eax
+  77FA0385: 55                 push        ebp
+  77FA0386: 8B EC              mov         ebp,esp
+  77FA0388: 83 EC 50           sub         esp,50h
+  77FA038B: 89 44 24 0C        mov         dword ptr [esp+0Ch],eax
+  77FA038F: 64 A1 18 00 00 00  mov         eax,fs:[00000018]
+  77FA0395: 8B 80 A4 01 00 00  mov         eax,dword ptr [eax+000001A4h]
+  77FA039B: 89 04 24           mov         dword ptr [esp],eax
+  77FA039E: C7 44 24 04 00 00  mov         dword ptr [esp+4],0
+            00 00
+  77FA03A6: C7 44 24 08 00 00  mov         dword ptr [esp+8],0
+            00 00
+  77FA03AE: C7 44 24 10 00 00  mov         dword ptr [esp+10h],0
+            00 00
+  77FA03B6: 54                 push        esp
+  77FA03B7: E8 5C 0B 01 00     call        77FB0F18  <RtlRaiseException>
+  77FA03BC: 8B 04 24           mov         eax,dword ptr [esp]
+  77FA03BF: 8B E5              mov         esp,ebp
+  77FA03C1: 5D                 pop         ebp
+  77FA03C2: C3                 ret
+ *
+ * ASSUMPTIONS: none
+ */
+/* Remember that every path out of here must invoke the DR exit hook.
+ * The normal return path will do so as the interception code has an
+ * enter and exit hook around the call to this routine.
+ */
+static after_intercept_action_t  /* note return value will be ignored */
+intercept_raise_exception(app_state_at_intercept_t *state)
+{
+    ASSERT_NOT_TESTED();
+    if (intercept_asynch_for_self(false/*no unknown threads*/)) {
+        SELF_PROTECT_LOCAL(get_thread_private_dcontext(), WRITABLE);
+        /* won't be re-protected until dispatch->fcache */
+
+        LOG(THREAD_GET, LOG_ASYNCH, 1, "ASYNCH intercept_raise_exception()\n");
+        STATS_INC(num_raise_exceptions);
+        
+        asynch_retakeover_if_native();
+        /* We want to squash the current trace (don't want traces
+         * following exceptions), asynch_take_over does that for us.
+         * We don't save the cur dcontext.
+         */
+        state->callee_arg = (void *) false /* use cur dcontext */;
+        asynch_take_over(state);
+    } else
+        STATS_INC(num_raise_exceptions_noasynch);
+    return AFTER_INTERCEPT_LET_GO;
+}
+
+/* creates an exception record for a forged exception */
+/* Access violations on read:
+   ff 12 call dword ptr [edx]
+        PC 0x00401124 tried to read address 0x00baddaa
+   Access violations on execution:
+   ff d2 call edx
+        PC 0xdeadbeef tried to read address 0xdeadbeef
+*/
+static void
+initialize_exception_record(EXCEPTION_RECORD* rec, app_pc exception_address,
+                             uint exception_code)
+{
+    rec->ExceptionFlags = 0;
+    rec->ExceptionRecord = NULL;
+    rec->ExceptionAddress = exception_address;
+    rec->NumberParameters = 0;
+    switch (exception_code) {
+    case UNREADABLE_MEMORY_EXECUTION_EXCEPTION:
+        rec->ExceptionCode = EXCEPTION_ACCESS_VIOLATION;
+        rec->NumberParameters = 2;
+        rec->ExceptionInformation[0]=0 /* read/execute */;
+        rec->ExceptionInformation[1]=(ptr_uint_t)exception_address;
+        break;
+    case IN_PAGE_ERROR_EXCEPTION:
+        rec->ExceptionCode = EXCEPTION_IN_PAGE_ERROR;
+        rec->NumberParameters = 2;
+        rec->ExceptionInformation[0]=0 /* read/execute */;
+        rec->ExceptionInformation[1]=(ptr_uint_t)exception_address;
+        break;
+    case ILLEGAL_INSTRUCTION_EXCEPTION:
+        rec->ExceptionCode = EXCEPTION_ILLEGAL_INSTRUCTION;
+        break;
+    default:
+        ASSERT_NOT_REACHED();
+    }
+}
+
+/* forge an exception (much like calling RaiseException)
+   1) we fill in the CONTEXT and EXCEPTION_RECORD structure
+      with proper context
+   2) then pass along to the kernel for delivery
+      - switch to application stack
+      - call ZwRaiseException(excrec, context, true)
+        -- kernel handler (pseudocode in Nebbett p.439)
+           (not clear how that pseudocode propagates the context of ZwRaiseException
+           see  CONTEXT context = {CONTEXT_FULL | CONTEXT_DEBUG};
+           if (valid user stack with enough space)
+             copy EXCEPTION_RECORD and CONTEXT to user mode stack
+             push &EXCEPTION_RECORD
+             push &CONTEXT
+        -- will return to user mode in intercept_exception (KiUserExceptionDispatcher)
+
+  An ALTERNATE less transparent step 2) would be:
+  push on the user stack the EXCEPTION_POINTERS (pretending to be the kernel)
+  and then get to our own intercept_exception to pass this to the application
+*/
+void
+os_forge_exception(app_pc exception_address, exception_type_t exception_type)
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    EXCEPTION_RECORD excrec;
+    int res;
+    /* In order to match the native exception we need a really full context  */
+    CONTEXT context;
+    context.ContextFlags = CONTEXT_FULL | CONTEXT_FLOATING_POINT |
+        IF_X64_ELSE(0/*doesn't exist*/, CONTEXT_EXTENDED_REGISTERS) |
+        CONTEXT_DEBUG_REGISTERS;
+    /* keep in mind the above structure is 716 bytes */
+
+    LOG(THREAD, LOG_ASYNCH, 1, 
+        "ASYNCH os_forge_exception(type "PFX" addr "PFX")\n", 
+        exception_type, exception_address);
+
+    initialize_exception_record(&excrec, exception_address, exception_type);
+    dcontext->forged_exception_addr = exception_address;
+
+    /* we first get full context, and then convert it using saved app context */
+    /* FIXME Rather than having a thread call get_context() on itself, should we
+     * assemble the context data ourself? This would get us around any handle
+     * permission problem.
+     * FIXME: MSDN says that GetThreadContext for the current thread returns
+     * an invalid context: so should use GET_OWN_CONTEXT, if it's sufficient.
+     */
+    res = nt_get_context(NT_CURRENT_THREAD, &context);
+    ASSERT(NT_SUCCESS(res));
+    DOLOG(2, LOG_ASYNCH, {
+        LOG(THREAD, LOG_ASYNCH, 2, "ASYNCH context before remapping\n");
+        dump_exception_info(&excrec, &context);
+    });
+
+    /* get application context */
+    mcontext_to_context(&context, get_mcontext(dcontext));
+    context.CXT_XIP = (ptr_uint_t)exception_address;
+ 
+    DOLOG(2, LOG_ASYNCH, {
+        LOG(THREAD, LOG_ASYNCH, 2, "\nASYNCH context after remapping\n");
+        dump_exception_info(&excrec, &context);
+    });
+
+    /* FIXME: App Context issues. */
+    /* For some uses of forge_exception we expect registers to differ from 
+     * native since we forge the exception at the start of the basic block we
+     * think will cause the exception (FIXME).  But even when that is not the
+     * case the eflags register still sometimes differs from native for unknown
+     * reasons, in my tests with the decode_prefixes unit test the resume 
+     * flags would be set natively, but not under us and the parity flags 
+     * would be set under us but not natively. FIXME
+     */
+    
+    /* FIXME : We might want to use nt_raise_exception here instead of 
+     * os_raise_exception, then we could get rid of that function & issue_last
+     * _system_call_from_app.  Also if we call this too early we might
+     * not know the syscall method yet, in which case we could screw up the
+     * args in os_raise_exception.  Which leads to the next problem that if the
+     * syscall fails (can happen if args are bad) then nt_raise_exception will
+     * return to us but os_raise_exception will return in to random app memory
+     * for xp/2003 and into our global syscall buffer on 2000 (which will
+     * prob. fault).  nt_raise_exception also allows the possibility for 
+     * recovering  (though we currently assert).  As far as os_transparency
+     * goes, nt_raise_exception will have the right return addresses,
+     * (thought shouldn't matter to the OS).  os_raise_exception has the 
+     * advantage of going through our cache entering routines before generating
+     * the exception so will reach our exception handler with internal dynamo
+     * state more similar to the app exception we are trying to forge.  This
+     * is especially relevant for self-protection and using nt_raise_exception
+     * would entail getting dynamo state into an appropriate configuration for
+     * receiving and app exception.
+     */
+    os_raise_exception(dcontext, &excrec, &context);
+    ASSERT_NOT_REACHED();
+}
+
+/****************************************************************************
+ * CALLBACKS
+ *
+ * Callbacks: start with kernel entering user mode in KiUserCallbackDispatcher,
+ * which this routine intercepts.  They end with an "int 2b" instruction
+ * (the ones I see go through 0x77e163c1 in ntdll.dll on win2000), if they
+ * come back here they will also hit the int 2b seen below.
+ * N.B.: This "int 2b" ending is my theory, I can find no documentation on it!
+ * UPDATE: Inside Win2K book confirms that int 2b maps to NtCallbackReturn
+ * 
+KiUserCallbackDispatcher:
+  77F9F038: 83 C4 04           add         esp,4
+  77F9F03B: 5A                 pop         edx
+  77F9F03C: 64 A1 18 00 00 00  mov         eax,fs:[00000018]
+  77F9F042: 8B 40 30           mov         eax,dword ptr [eax+30h]
+  77F9F045: 8B 40 2C           mov         eax,dword ptr [eax+2Ch]
+  77F9F048: FF 14 90           call        dword ptr [eax+edx*4]
+  77F9F04B: 33 C9              xor         ecx,ecx
+  77F9F04D: 33 D2              xor         edx,edx
+  77F9F04F: CD 2B              int         2Bh
+  77F9F051: CC                 int         3
+  77F9F052: 8B FF              mov         edi,edi
+ *
+ * 2003 SP1 has an explicit call to ZwCallbackReturn instead of the int2b:
+KiUserCallbackDispatcher:
+  7c836330 648b0d00000000   mov     ecx,fs:[00000000]
+  7c836337 ba1063837c       mov     edx,0x7c836310 (KiUserCallbackExceptionHandler)
+  7c83633c 8d442410         lea     eax,[esp+0x10]
+  7c836340 894c2410         mov     [esp+0x10],ecx
+  7c836344 89542414         mov     [esp+0x14],edx
+  7c836348 64a300000000     mov     fs:[00000000],eax
+  7c83634e 83c404           add     esp,0x4
+  7c836351 5a               pop     edx
+  7c836352 64a130000000     mov     eax,fs:[00000030]
+  7c836358 8b402c           mov     eax,[eax+0x2c]
+  7c83635b ff1490           call    dword ptr [eax+edx*4]
+  7c83635e 50               push    eax
+  7c83635f 6a00             push    0x0
+  7c836361 6a00             push    0x0
+  7c836363 e85d27ffff       call    ntdll!ZwCallbackReturn (7c828ac5)
+  7c836368 8bf0             mov     esi,eax
+  7c83636a 56               push    esi
+  7c83636b e828010000       call    ntdll!RtlRaiseStatus (7c836498)
+  7c836370 ebf8             jmp  ntdll!KiUserCallbackDispatcher+0x3a (7c83636a)
+  7c836372 c20c00           ret     0xc
+ *
+  x64 xp:
+  ntdll!KiUserCallbackDispatch:
+  00000000`77ef3160 488b4c2420      mov     rcx,qword ptr [rsp+20h]
+  00000000`77ef3165 8b542428        mov     edx,dword ptr [rsp+28h]
+  00000000`77ef3169 448b44242c      mov     r8d,dword ptr [rsp+2Ch]
+  00000000`77ef316e 65488b042560000000 mov   rax,qword ptr gs:[60h]
+  00000000`77ef3177 4c8b4858        mov     r9,qword ptr [rax+58h]
+  00000000`77ef317b 43ff14c1        call    qword ptr [r9+r8*8]
+  ntdll!KiUserCallbackDispatcherContinue:
+  00000000`77ef317f 33c9            xor     ecx,ecx
+  00000000`77ef3181 33d2            xor     edx,edx
+  00000000`77ef3183 448bc0          mov     r8d,eax
+  00000000`77ef3186 e8a5d8ffff      call    ntdll!ZwCallbackReturn (00000000`77ef0a30)
+  00000000`77ef318b 8bf0            mov     esi,eax
+  00000000`77ef318d 8bce            mov     ecx,esi
+  00000000`77ef318f e85cf50500      call    ntdll!RtlRaiseException+0x118 (00000000`77f526f0)
+  00000000`77ef3194 cc              int     3
+  00000000`77ef3195 eb01            jmp     ntdll!KiUserCallbackDispatcherContinue+0x15 (00000000`77ef3198)
+  00000000`77ef3197 90              nop
+  00000000`77ef3198 ebf7            jmp     ntdll!KiUserCallbackDispatcherContinue+0x12 (00000000`77ef3191)
+  00000000`77ef319a cc              int     3
+ *
+ * ASSUMPTIONS:
+ *    1) peb->KernelCallbackTable[*(esp+IF_X64_ELSE(0x2c,4))] == 
+ *       call* target (used for debug only)
+ */
+/* Remember that every path out of here must invoke the DR exit hook.
+ * The normal return path will do so as the interception code has an
+ * enter and exit hook around the call to this routine.
+ */
+static after_intercept_action_t  /* note return value will be ignored */
+intercept_callback_start(app_state_at_intercept_t *state)
+{
+    /* We only hook this in thin_client mode to be able to read the DRmarker. */
+    if (DYNAMO_OPTION(thin_client))
+        return AFTER_INTERCEPT_LET_GO;
+
+    if (intercept_callbacks && intercept_asynch_for_self(false/*no unknown threads*/)) {
+        SELF_PROTECT_LOCAL(get_thread_private_dcontext(), WRITABLE);
+        /* won't be re-protected until dispatch->fcache */
+        ASSERT(is_thread_initialized());
+        /* should not receive APC while in DR code! */
+        ASSERT(get_thread_private_dcontext()->whereami == WHERE_FCACHE);
+        DODEBUG({
+            /* get callback target address
+             * we want ((fs:0x18):0x30):0x2c => TEB->PEB->KernelCallbackTable
+             * on x64 it's the same table ((gs:60):0x58), but the index comes
+             * from rsp+0x2c.
+             */
+            app_pc target = NULL;
+            uint *reg_esp = (uint *) state->mc.xsp;
+            app_pc *cbtable = (app_pc *) get_own_peb()->KernelCallbackTable;
+            target = cbtable[*(uint*)(state->mc.xsp+IF_X64_ELSE(0x2c,4))];
+            LOG(THREAD_GET, LOG_ASYNCH, 2,
+                "ASYNCH intercepted callback #%d: target="PFX", thread=%d\n",
+                GLOBAL_STAT(num_callbacks)+1, target, get_thread_id());
+            DOLOG(3, LOG_ASYNCH, {
+                dump_mcontext(&state->mc, THREAD_GET, DUMP_NOT_XML);
+            });
+        });
+        /* FIXME: we should be able to strongly assert that only
+         * non-ignorable syscalls should be allowed to get here - all
+         * ignorable one's shouldn't be interrupted for callbacks.  We
+         * also start syscall_fcache at callback return yet another
+         * callback can be delivered.
+         */
+
+#if 0 /* disabled b/c not fully flushed out yet, may be useful in future */
+        /* attempt to find where thread was suspended for callback by walking
+         * stack backward, finding last ret addr before KiUserCallbackDispatcher,
+         * and walking forward looking for the syscall that triggered callback.
+         * This method is fragile, since ind branches could be in the middle...
+         */
+        uint *pc = (uint *) state->mc.xbp;
+        DOLOG(2, LOG_ASYNCH, {
+            dump_callstack(NULL, (app_pc) state->mc.xbp, THREAD_GET, DUMP_NOT_XML);
+        });
+        while (pc != NULL && is_readable_without_exception((byte *)pc, 8)) {
+            if (*(pc+1) == (ptr_uint_t)after_callback_orig_pc ||
+                *(pc+1) == (ptr_uint_t)after_apc_orig_pc) {
+                uint *stop; /* one past ret for call to syscall wrapper */
+                if (*(pc+1) == (ptr_uint_t)after_callback_orig_pc) {
+                    LOG(cur_dcontext->logfile, LOG_ASYNCH, 2,
+                        "\tafter_callback_orig_pc == "PFX", pointed to @"PFX"\n",
+                        after_callback_orig_pc, pc);
+                    stop = pc + 29; 
+                } else {
+                    LOG(cur_dcontext->logfile, LOG_ASYNCH, 2,
+                        "\tafter_apc_orig_pc == "PFX", pointed to @"PFX"\n",
+                        after_apc_orig_pc, pc);
+                    stop = pc + 1000;
+                }
+                while (pc < stop) {
+                    LOG(cur_dcontext->logfile, LOG_ASYNCH, 2,
+                        "\t*"PFX" = "PFX"\n", pc, *pc);
+                    if (*pc != NULL && is_readable_without_exception(((byte*)(*pc))-5, 5)) {
+                        byte *nxt = ((byte*)(*pc))-5;
+                        while (nxt < (byte *)(*pc)) {
+                            nxt = disassemble_with_bytes(cur_dcontext, nxt,
+                                                         cur_dcontext->logfile);
+                        }
+                    }
+                    pc++;
+                }
+                if (*stop != NULL && is_readable_without_exception(*stop, 4)) {
+                    byte *cpc = (byte *) *(pc-1);
+                    instr_t instr;
+                    instr_init(dcontext, &instr);
+                    /* Have to decode previous call...tough to do, how
+                     * tell call* from call?
+                    e8 f3 cd ff ff       call   $0x77f4386b 
+                    ff 15 20 11 f4 77    call   0x77f41120 
+                      can you have call 0x?(?,?,?) such that last 5 bytes look like direct call?
+                    ff 14 90             call   (%eax,%edx,4) 
+                    ff 55 08             call   0x8(%ebp) 
+                    ff 56 5c             call   0x5c(%esi) 
+                    ff d3                call   %ebx 
+                    */
+                    cpc -= 5;
+                    LOG(cur_dcontext->logfile, LOG_ASYNCH, 2,
+                        "\tdecoding "PFX"\n", cpc);
+                    cpc = decode_cti(cur_dcontext, cpc, &instr);
+                    if (instr_opcode_valid(&instr) &&
+                        instr_is_call(&instr) &&
+                        cpc == (byte *) *(pc-1)) {
+                        LOG(cur_dcontext->logfile, LOG_ASYNCH, 2,
+                            "\tafter instr is "PFX"\n", cpc);
+                        cpc = opnd_get_pc(instr_get_target(&instr));
+                        LOG(cur_dcontext->logfile, LOG_ASYNCH, 2,
+                            "\ttarget is "PFX"\n", cpc);
+                        /* now if win2k, target will be int 2e, then 3-byte ret
+                         * how insert trampoline into 3-byte ret??!?!
+                         *   suspend all other threads?
+                         * know where it's going -- back to after call, could try for
+                         * trampoline there, but what if buffer overflow?
+                         * don't want native ret to be executed!
+                         */
+                    } else {
+                        ASSERT(false && "Bad guess of direct near call");
+                    }
+                }
+            }
+            pc = (uint *) *pc;
+        }
+#endif /* if 0: find callback interruption point */
+
+        RSTATS_INC(num_callbacks);
+
+        asynch_retakeover_if_native();
+        state->callee_arg = (void *)(ptr_uint_t) true /* save cur dcontext */;
+        asynch_take_over(state);
+    } else
+        STATS_INC(num_callbacks_noasynch);
+    return AFTER_INTERCEPT_LET_GO;
+}
+
+
+/* NtCallbackReturn: the other way (other than int 2b) to return from
+ * a callback
+ *
+ * Like NtContinue, NtCallbackReturn is a system call that is entered
+ * from user mode, but when control emerges from the kernel it does
+ * not resume after the syscall point.
+ * Note that sometimes an indirect call (for example, at 77E25A5A in user32.dll)
+ * targets NtCallbackReturn
+ *
+ * Is it possible to get at target pc?
+ * I tried looking at various points on the stack:
+ *   intercept_callback_return: thread=520, target=0x03f10705
+ *   intercept_callback_return: thread=520, target=0x77e25a60
+ *   intercept_callback_return: thread=520, target=0x00000000
+ *   intercept_callback_return: thread=520, target=0x00000000
+ *   intercept_callback_return: thread=520, target=0x00000000
+ *   intercept_callback_return: thread=520, target=0x77f9f04b
+ *   intercept_callback_return: thread=520, target=0x040bff98
+ * First 2 are proc call returns, that 77f9f04b is inside CallbackDispatcher!
+ * Sounds very promising but our after callback interception would have caught
+ * anything that went there...why is that on stack?  Could be proc call return
+ * address for callback call...
+ * Final stack value I thought might be pointer to CONTEXT on stack, but
+ * no.
+ * Resolution: impossible, just like int 2b
+ */
+/*
+NtCallbackReturn:
+  77F83103: B8 13 00 00 00     mov         eax,13h
+  77F83108: 8D 54 24 04        lea         edx,[esp+4]
+  77F8310C: CD 2E              int         2Eh
+  77F8310E: E9 FA 57 01 00     jmp         77F9890D
+  77F83113: B8 93 00 00 00     mov         eax,93h
+  77F83118: 8D 54 24 04        lea         edx,[esp+4]
+  77F8311C: CD 2E              int         2Eh
+  77F8311E: C2 14 00           ret         14h
+*/
+
+/**************************************************
+ * Dealing with dcontext stack for callbacks
+ * Since the kernel maintains a stack of contexts for callbacks that we
+ * cannot read, we must keep our state for when the kernel unwinds the stack
+ * and sends control back.  To do that, we have a stack of dcontexts.
+ */
+
+/* Called by asynch_take_over to initialize the dcontext structure for
+ * the current thread and return a ptr to it.  The parameter "next_pc"
+ * indicates where the Dynamo interpreter should begin.
+ */
+static dcontext_t *
+callback_setup(app_pc next_pc)
+{
+    dcontext_t *old_dcontext;
+    dcontext_t *new_dcontext;
+#ifndef DCONTEXT_IN_EDI
+    dcontext_t *dc;
+#endif
+
+    old_dcontext = get_thread_private_dcontext();
+    ASSERT(old_dcontext);
+
+    if (!old_dcontext->initialized) {
+        /* new threads are created via APC, so they come in here
+         * uninitialized -- we could not create new dcontext and use old
+         * one, then have to check for that in callback_start_return,
+         * instead we simply initialize old one:
+         */
+        initialize_dynamo_context(old_dcontext);
+    }
+
+    /* if we were building a trace, kill it */
+    if (is_building_trace(old_dcontext)) {
+        LOG(old_dcontext->logfile, LOG_ASYNCH, 2,
+            "callback_setup: squashing old trace\n");
+        trace_abort(old_dcontext);
+    }
+
+    /* kill any outstanding pointers -- once we switch to the new dcontext,
+     * they will not be cleared if we delete their target, and we'll have
+     * problems when we return to old dcontext and deref them!
+     */
+    set_last_exit(old_dcontext, (linkstub_t *) get_asynch_linkstub());
+#ifdef PROFILE_RDTSC
+    old_dcontext->prev_fragment = NULL;
+#endif
+
+    /* need to save old dcontext and get new dcontext for callback execution */
+
+#ifndef DCONTEXT_IN_EDI
+    /* must always use same dcontext because fragment code is hardwired
+     * for it, so we use stack of dcontexts to save old ones.
+     * what we do here: find or create new dcontext for use with callback.
+     * initialize new dcontext.  then swap the new and old dcontexts!
+     */
+    dc = old_dcontext;
+    /* go to end of valid (==saved) contexts */
+    while (dc->prev_unused != NULL && dc->prev_unused->valid)
+        dc = dc->prev_unused;
+    if (INTERNAL_OPTION(stress_detach_with_stacked_callbacks) && 
+        dc != old_dcontext && dc != old_dcontext->prev_unused &&
+        dc != old_dcontext->prev_unused->prev_unused) {
+        /* internal stress testing of detach (once app has multiple stacked callbacks) */
+        DO_ONCE(detach_internal(););
+    }
+    if (dc->prev_unused != NULL) {
+        new_dcontext = dc->prev_unused;
+        ASSERT(!new_dcontext->valid);
+    } else {
+        /* need to make a new dcontext */
+        /* FIXME: how do we know we're not getting a new callback 
+           while in a system call for allocating more memory?
+           This routine should be organized so that we spend minimal time 
+           setting up a new context and then we should be able to handle a new one.
+           We need a per-thread flag/counter saying are we handling a callback stack.
+           Not that we can do anything about it when we come here again and the flag is set,
+           but at least we can STAT such occurrences.
+
+           Actually that should only happen if we call an alertable system call
+           (see end of bug 326 )...we should see if we're calling any.and deal with it,
+           otherwise if we're safe with respect to callbacks then remove the above comment.
+        */
+        new_dcontext = create_callback_dcontext(old_dcontext);
+        /* stick at end of list */
+        dc->prev_unused = new_dcontext;
+        new_dcontext->prev_unused = NULL;
+    }
+    LOG(old_dcontext->logfile, LOG_ASYNCH, 2,
+        "\tsaving prev dcontext @"PFX"\n", new_dcontext);
+
+    DOLOG(4, LOG_ASYNCH, {
+        LOG(old_dcontext->logfile, LOG_ASYNCH, 4,
+            "old dcontext "PFX" w/ next_tag "PFX":\n",
+            old_dcontext, old_dcontext->next_tag);
+        dump_mcontext(get_mcontext(old_dcontext), old_dcontext->logfile,
+                      DUMP_NOT_XML);
+        LOG(old_dcontext->logfile, LOG_ASYNCH, 4,
+            "new dcontext "PFX" w/ next_tag "PFX":\n",
+            new_dcontext, new_dcontext->next_tag);
+        dump_mcontext(get_mcontext(new_dcontext), old_dcontext->logfile,
+                      DUMP_NOT_XML);
+    });
+
+    /* now swap new and old */
+    swap_dcontexts(new_dcontext, old_dcontext);
+    /* saved and current dcontext should both be valid */
+    new_dcontext->valid = true;
+    old_dcontext->valid = true;
+
+    /* now prepare to use new dcontext, pointed to by old_dcontext ptr */
+    initialize_dynamo_context(old_dcontext);
+    old_dcontext->whereami = WHERE_TRAMPOLINE;
+    old_dcontext->next_tag = next_pc;
+    ASSERT(old_dcontext->next_tag != NULL);
+    return old_dcontext;
+
+#else /* DCONTEXT_IN_EDI */
+    /* since we use edi to point to the dcontext, we can simply use a
+     * different dcontext
+     */
+    if (old_dcontext->prev_unused) {
+        new_dcontext = old_dcontext->prev_unused;
+        LOG(old_dcontext->logfile, LOG_ASYNCH, 2,
+            "callback_setup(): re-using unused dcontext\n");
+    } 
+    else {
+        new_dcontext = create_callback_dcontext(old_dcontext);
+        old_dcontext->prev_unused = new_dcontext;
+        new_dcontext->next_saved = old_dcontext;
+    }
+    new_execution_environment(new_dcontext);
+    new_dcontext->next_tag = next_pc;
+    ASSERT(new_dcontext->next_tag != NULL);
+    set_thread_private_dcontext(new_dcontext);
+    return new_dcontext;
+#endif /* DCONTEXT_IN_EDI */
+}
+
+/* Called when a callback has completed execution and is about to return
+ * (either at an int 2b, or an NtCallbackReturn system call).
+ * Note that we restore the old dcontext now and then execute the rest
+ * of the current fragment that's been using the new dcontext, so we
+ * rely on the interrupt instruction being the NEXT INSTRUCTION after this call
+ * (& call cleanup)!
+ * Arguments: pusha right before call -> pretend getting all regs as args
+ * Do not need them as args!  You can pass me anything!
+ */
+void
+callback_start_return(int app_errno, dr_mcontext_t *mc)
+{
+    dcontext_t *cur_dcontext;
+    dcontext_t *prev_dcontext;
+
+    if (!intercept_callbacks || !intercept_asynch_for_self(false/*no unknown threads*/))
+        return;
+
+    /* both paths here, int 2b and syscall, go back through dispatch,
+     * so self-protection has already been taken care of -- we don't come
+     * here straight from cache!
+     */
+
+    cur_dcontext = get_thread_private_dcontext();
+    ASSERT(cur_dcontext && cur_dcontext->initialized);
+
+    /* if we were building a trace, kill it */
+    if (is_building_trace(cur_dcontext)) {
+        LOG(cur_dcontext->logfile, LOG_ASYNCH, 2,
+            "callback_start_return: squashing old trace\n");
+        trace_abort(cur_dcontext);
+    }
+
+    LOG(cur_dcontext->logfile, LOG_ASYNCH, 2,
+        "ASYNCH callback_start_return(): restoring previous dcontext\n");
+    DOLOG(3, LOG_ASYNCH, {
+        dump_mcontext(mc, cur_dcontext->logfile, DUMP_NOT_XML);
+        if (mc->xbp != 0) {
+            dump_callstack(NULL, (app_pc) mc->xbp, cur_dcontext->logfile,
+                           DUMP_NOT_XML);
+        }
+    });
+
+#ifndef DCONTEXT_IN_EDI
+    /* must always use same dcontext because fragment code is hardwired
+     * for it, so we use stack of dcontexts to save old ones.
+     * what we do here: find the last dcontext that holds saved values.
+     * then swap the new and old dcontexts!
+     */
+    prev_dcontext = cur_dcontext;
+    /* go to end of valid (==saved) contexts */
+    while (prev_dcontext->prev_unused != NULL && prev_dcontext->prev_unused->valid)
+        prev_dcontext = prev_dcontext->prev_unused;
+
+    if (prev_dcontext == cur_dcontext) {
+        /* There's no prior dcontext! */
+        /* If this is a callback return, the kernel will send control to a native context!
+         * We see this when we take control in the middle of a callback that's
+         * in the middle of the init APC, which is used to initialize the main thread.
+         */
+        thread_record_t *tr = thread_lookup(get_thread_id());
+        /* we may end up losing control, so use this to signal as a hack,
+         * if we intercept any asynchs we'll report an error if we see UNDER_DYN_HACK
+         */
+        tr->under_dynamo_control = UNDER_DYN_HACK;
+
+        /* We assume this is during the init APC.
+         * We try to regain control by adding a trampoline at the
+         * image entry point -- we ignore race conditions here b/c there's only
+         * one thread (not always true [injected], but the races are minimal).
+         * FIXME: kernel32's base process start point is actually where
+         * control appears, but it's not exported.  It only executes a dozen or
+         * so instructions before calling the image entry point, but it would
+         * be nice if we had a way to find it -- could get it from the
+         * initialization apc context. */
+        /* We rely on -native_exec_syscalls to retake control before any app
+         * state changes dr needs to see occur. This lets us handle injected
+         * threads (or the strange case of a statically linked dll creating a
+         * thread in its dllmain). If not using native_exec_syscalls then we
+         * can't maintain the executable list, retakeover will have to flush
+         * and re-walk the list, and we won't be safe with injected threads.
+         */
+        if (!DYNAMO_OPTION(native_exec_syscalls)) {
+            /* we need to restore changed memory protections because we won't
+             * be intercepting system calls to fix things up */
+            /* not multi-thread safe */
+            ASSERT(check_sole_thread() && get_num_threads() == 1);
+            revert_memory_regions();
+        }
+
+        if (INTERNAL_OPTION(hook_image_entry)) {
+            /* potentially racy hook (injected threads) */
+            insert_image_entry_trampoline(cur_dcontext);
+        }
+
+        DODEBUG({
+            /* we should never see this after we have reached the image entry point */
+            if (reached_image_entry_yet()) {
+                /* Nothing we can do -- except try to walk stack frame back and hope to decode
+                 * exactly where thread was suspended, but there's no guarantee we can do that
+                 * if there were indirect jmps.
+                 */
+                SYSLOG_INTERNAL_ERROR("non-process-init callback return with native callback context for %s thread %d",
+                                      (tr == NULL) ? "unknown" : "known", 
+                                      get_thread_id());
+                /* might be injected late, refer to bug 426 for discussion
+                 * of instance here where that was the case */
+                ASSERT_NOT_REACHED();
+            }
+        });
+        return;
+    }
+
+    LOG(cur_dcontext->logfile, LOG_ASYNCH, 2,
+        "\trestoring previous dcontext saved @"PFX"\n", prev_dcontext);
+
+    DOLOG(4, LOG_ASYNCH, {
+        LOG(cur_dcontext->logfile, LOG_ASYNCH, 4,
+            "current dcontext "PFX" w/ next_tag "PFX":\n",
+            cur_dcontext, cur_dcontext->next_tag);
+        dump_mcontext(get_mcontext(cur_dcontext), cur_dcontext->logfile,
+                      DUMP_NOT_XML);
+        LOG(cur_dcontext->logfile, LOG_ASYNCH, 4,
+            "prev dcontext "PFX" w/ next_tag "PFX":\n",
+            prev_dcontext, prev_dcontext->next_tag);
+        dump_mcontext(get_mcontext(prev_dcontext), cur_dcontext->logfile,
+                      DUMP_NOT_XML);
+    });
+
+    /* now swap cur and prev 
+     * N.B.: callback return brings up a tricky dual-dcontext problem, where
+     * we need the cur dcontext to restore to native state right before interrupt
+     * (after this routine returns), but we need to restore to prev dcontext now
+     * since we won't get another chance -- we solve the problem by assuming we
+     * never deallocate dstack of prev dcontext and letting the clean call restore
+     * from the stack, plus using a special unswapped_scratch slot in dcontext
+     * to restore app esp properly to native value.
+     * for self-protection, we solve the problem by using a custom fcache_enter
+     * routine that restores native state from the prev dcontext.
+     */
+    swap_dcontexts(prev_dcontext, cur_dcontext);
+    /* invalidate prev */
+    prev_dcontext->valid = false;
+
+    DOLOG(5, LOG_ASYNCH, {
+        LOG(cur_dcontext->logfile, LOG_ASYNCH, 4,
+            "after swap, current dcontext "PFX" w/ next_tag "PFX":\n",
+            cur_dcontext, cur_dcontext->next_tag);
+        dump_mcontext(get_mcontext(cur_dcontext), cur_dcontext->logfile,
+                      DUMP_NOT_XML);
+    });
+#else /* DCONTEXT_IN_EDI */
+    /* restore previous dcontext */
+    prev_dcontext = cur_dcontext->next_saved;
+    ASSERT(prev_dcontext != NULL);
+    set_thread_private_dcontext(prev_dcontext);
+
+    /* note that we do not need to adjust numthreads in cache, since we
+     * are currently in the cache and will be returning to the cache
+     */
+
+    /* don't delete cur_dcontext, leave in list for use next time */
+    ASSERT(prev_dcontext->prev_unused == cur_dcontext);
+#endif
+}
+
+/* Returns the prev dcontext that was just swapped by callback_start_return */
+dcontext_t *
+get_prev_swapped_dcontext(dcontext_t *dcontext)
+{
+    dcontext_t *prev = dcontext;
+    /* find first invalid dcontext */
+    while (prev->prev_unused != NULL && prev->valid)
+        prev = prev->prev_unused;
+    return prev;
+}
+
+/****************************************************************************
+ * MISC
+ */
+
+/* finds the pc after the call to the callback routine in
+ * KiUserCallbackDispatcher or KiUserApcDispatcher, useful for examining
+ * call stacks (the pc after the call shows up on the call stack)
+ * For callbacks:
+ *   on Win2k Workstation should get: 0x77F9F04F
+ *   on Win2K Server: 0x77F9FB83
+ * Will also look beyond the call for the 1st cti/int, and if an int2b
+ * or a call to NtCallbackReturn, sets cbret to that pc.
+ */
+byte *
+get_pc_after_call(byte *entry, byte **cbret)
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    byte *pc = entry;
+    byte *after_call = NULL;
+    instr_t instr;
+    int num_instrs = 0;
+
+    /* find call to callback */
+    instr_init(dcontext, &instr);
+    do {
+        instr_reset(dcontext, &instr);
+        pc = decode_cti(dcontext, pc, &instr);
+        ASSERT(pc != NULL);
+        num_instrs++;
+        ASSERT_CURIOSITY(num_instrs <= 11);
+    } while (!instr_opcode_valid(&instr) || !instr_is_call(&instr));
+    after_call = pc;
+
+    /* find next cti, see if it's an int 2b or a call to ZwCallbackReturn */
+    if (cbret != NULL) {
+        *cbret = NULL;
+        do {
+            instr_reset(dcontext, &instr);
+            pc = decode_cti(dcontext, pc, &instr);
+            ASSERT_CURIOSITY(pc != NULL);
+            num_instrs++;
+            ASSERT_CURIOSITY(num_instrs <= 20);     /* case 3522. */
+            if (instr_opcode_valid(&instr)) {
+                if (instr_is_interrupt(&instr)) {
+                    int num = instr_get_interrupt_number(&instr);
+                    if (num == 0x2b) {
+                        LOG(THREAD_GET, LOG_ASYNCH, 2,
+                            "after dispatcher found int 2b @"PFX"\n", pc);
+                        *cbret = pc;
+                    }
+                } else if (instr_is_call(&instr)) {
+                    GET_NTDLL(NtCallbackReturn, (IN PVOID Result OPTIONAL,
+                                                 IN ULONG ResultLength,
+                                                 IN NTSTATUS Status));
+                    if (opnd_get_pc(instr_get_target(&instr)) ==
+                        (app_pc)NtCallbackReturn) {
+                        LOG(THREAD_GET, LOG_ASYNCH, 2,
+                            "after dispatcher found call to NtCallbackReturn @"PFX"\n",
+                            pc);
+                        *cbret = pc;
+                    }
+                }
+            }
+        } while (!instr_opcode_valid(&instr));
+    }
+    instr_free(dcontext, &instr);
+    LOG(THREAD_GET, LOG_ASYNCH, 2, "after dispatcher pc is: "PFX"\n", after_call);
+    return after_call;
+}
+
+/****************************************************************************/
+/* it's possible to load w/o going through the LdrLoad routines (by
+ * using MapViewOfSectino), we only use these trampolines for debugging, and
+ * also for the AppInit injection in order to regain control at an earlier point
+ * than waiting for the main image entry point (and, another thread is sometimes
+ * scheduled before the main thread gets to its entry point!)
+ * Thus we go ahead and intercept load_dll for the non-debug build.
+ * We also use this hook for our do not load list, though we could prob. get
+ * away with doing that at MapViewOfSection.
+ */
+GET_NTDLL(LdrLoadDll, (IN PWSTR DllPath OPTIONAL,
+                       IN PULONG DllCharacteristics OPTIONAL,
+                       IN PUNICODE_STRING DllName,
+                       OUT PVOID *DllHandle
+                       ));
+GET_NTDLL(LdrUnloadDll, (IN PVOID DllHandle));
+
+after_intercept_action_t
+intercept_load_dll(app_state_at_intercept_t *state)
+{
+    thread_record_t *tr = thread_lookup(get_thread_id());
+    /* grab args to original routine */
+    wchar_t *path = (wchar_t *) APP_PARAM(&state->mc, 0);
+    uint *characteristics = (uint *) APP_PARAM(&state->mc, 1);
+    UNICODE_STRING *name = (UNICODE_STRING *) APP_PARAM(&state->mc, 2);
+    HMODULE *out_handle = (HMODULE *) APP_PARAM(&state->mc, 3);
+
+    LOG(GLOBAL, LOG_VMAREAS, 1, "intercept_load_dll: %S\n", name->Buffer);
+    LOG(GLOBAL, LOG_VMAREAS, 2, "\tpath=%S\n", path);
+    LOG(GLOBAL, LOG_VMAREAS, 2, "\tcharacteristics=%d\n",
+        characteristics ? *characteristics : 0);
+
+#ifdef GBOP
+    if (DYNAMO_OPTION(gbop) != GBOP_DISABLED) {
+        /* FIXME: case 7127: currently doesn't obey
+         * -exclude_gbop_list, which should set a flag.
+         */
+        gbop_validate_and_act(state, 
+                              0 /* no ESP offset - at entry point */,
+                              load_dll_pc);
+        /* if GBOP validation at all returns it accepted the source */
+        /* FIXME: case 7127: may want alternative handling */
+    }
+#endif /* GBOP */
+
+    /* we should be able to block loads even in unknown threads */
+    if (DYNAMO_OPTION(enable_block_mod_load) &&
+        (!IS_STRING_OPTION_EMPTY(block_mod_load_list) ||
+         !IS_STRING_OPTION_EMPTY(block_mod_load_list_default))) {
+        static const UNICODE_STRING dummy_ustring = {0, 0, L""};
+        static const HMODULE dummy_handle = NULL;
+        /* UNICODE_STRINGs don't have to be NULL terminated (though usually
+         * are) we copy to a buf here just to be safe. */
+        wchar_t buf[MAXIMUM_PATH] = {0};
+        const wchar_t *short_name;
+        char name_buf[MAXIMUM_PATH];
+        UNICODE_STRING name_cpy = {0};
+        /* arg pointers are user supplied so be extra carefull */
+        if (safe_read(name, sizeof(UNICODE_STRING), &name_cpy) &&
+            safe_read(name_cpy.Buffer, 
+                      MIN(BUFFER_SIZE_BYTES(buf), name_cpy.Length), buf)) {
+            ASSERT_CURIOSITY(name_cpy.Length <= BUFFER_SIZE_BYTES(buf));
+            /* null terminate */
+            buf[MIN(BUFFER_SIZE_ELEMENTS(buf)-1,
+                    name_cpy.Length/sizeof(buf[0]))] = L'\0';
+            short_name = w_get_short_name(buf);
+            snprintf(name_buf, BUFFER_SIZE_ELEMENTS(name_buf), "%ls",
+                     short_name);
+            NULL_TERMINATE_BUFFER(name_buf);
+            string_option_read_lock();
+            if ((!IS_STRING_OPTION_EMPTY(block_mod_load_list) &&
+                 check_filter(DYNAMO_OPTION(block_mod_load_list), name_buf)) ||
+                (!IS_STRING_OPTION_EMPTY(block_mod_load_list_default) &&
+                 check_filter(DYNAMO_OPTION(block_mod_load_list_default),
+                              name_buf))) {
+                string_option_read_unlock();
+                /* Modify args so call fails, stdcall so caller shouldn't care
+                 * about the args being modified. FIXME - alt. we could just
+                 * do the stdcall ret here (for non-takeover need to supply
+                 * a dr location with a ret 4 instruction at hook time and
+                 * return alt_dyn here, for takeover need to modify the
+                 * interception code or pass a flag to asynch_takeover/dispath
+                 * to modify the app state) */
+                LOG(GLOBAL, LOG_ALL, 1,
+                    "Blocking load of module %s\n", name_buf);
+                SYSLOG_INTERNAL_WARNING_ONCE("Blocking load of module %s", name_buf);
+                *((wchar_t **)(state->mc.xsp + XSP_SZ)) = NULL; /* NULL out path */
+                *((const UNICODE_STRING **)(state->mc.xsp + 12)) = &dummy_ustring;
+                /* LdrLoadDll should do this, but let's be sure */
+                safe_write(out_handle, sizeof(HANDLE), &dummy_handle);
+            } else {
+                string_option_read_unlock();
+            }
+        }
+    }
+
+    if (tr == NULL) {
+        LOG(GLOBAL, LOG_VMAREAS, 1, "WARNING: native thread in intercept_load_dll\n");
+        if (control_all_threads) {
+            SYSLOG_INTERNAL_ERROR("LdrLoadDll reached by unexpected %s thread %d", 
+                                  (tr == NULL) ? "unknown" : "known",
+                                  get_thread_id());
+            /* case 9385 tracks an instance */
+            ASSERT_CURIOSITY(false);
+        }
+        return AFTER_INTERCEPT_LET_GO;
+    } else if (control_all_threads && tr->under_dynamo_control == UNDER_DYN_HACK) {
+        dcontext_t *dcontext = get_thread_private_dcontext();
+        /* trying to open debugbox causes IIS to fail, so don't SYSLOG_INTERNAL */
+        LOG(THREAD, LOG_ASYNCH, 1, "ERROR: load_dll: we lost control of thread %d\n",
+            tr->id);
+        DOLOG(2, LOG_ASYNCH, {
+            dump_callstack(NULL, (app_pc) state->mc.xbp, THREAD, DUMP_NOT_XML);
+        });
+        retakeover_after_native(tr, INTERCEPT_LOAD_DLL);
+        /* we want to take over, but we'll do that anyway on return from this routine
+         * since our interception code was told to take over afterward, so we wait.
+         * this routine executes natively so even if we were already in control
+         * we want to take over so no conditional is needed there.
+         */
+    } else if (!intercept_asynch_for_self(false/*no unknown threads*/)) {
+        LOG(GLOBAL, LOG_VMAREAS, 1, "WARNING: no-asynch thread loading a dll\n");
+        return AFTER_INTERCEPT_LET_GO;
+    } else {
+        /* unnecessary trampoline exit when we were in full control */
+        /* LdrLoadDll will be executed as a normal fragment */
+    }
+
+    /* we're taking over afterward so we need local writability */
+    SELF_PROTECT_LOCAL(get_thread_private_dcontext(), WRITABLE);
+    /* won't be re-protected until dispatch->fcache */
+
+#ifdef DEBUG
+    if (get_thread_private_dcontext() != NULL)
+        LOG(THREAD_GET, LOG_VMAREAS, 1, "intercept_load_dll: %S\n", name->Buffer);
+    DOLOG(3, LOG_VMAREAS, { print_modules(GLOBAL, DUMP_NOT_XML); });
+#endif
+    return AFTER_INTERCEPT_TAKE_OVER;
+}
+
+/* used for log messages in normal operation
+   and also needed for the svchost_timeout hack */
+after_intercept_action_t
+intercept_unload_dll(app_state_at_intercept_t *state)
+{
+    /* grab arg to original routine */
+    HMODULE h = (HMODULE) APP_PARAM(&state->mc, 0);
+    static int in_svchost = -1; /* unknown yet */
+    thread_record_t *tr = thread_lookup(get_thread_id());
+
+    if (tr == NULL) {
+        LOG(GLOBAL, LOG_VMAREAS, 1, "WARNING: native thread in "
+                                    "intercept_unload_dll\n");
+        if (control_all_threads) {
+            SYSLOG_INTERNAL_ERROR("LdrUnloadDll reached by unexpected %s thread %d", 
+                                  (tr == NULL) ? "unknown" : "known",
+                                  get_thread_id());
+            /* case 9385 tracks an instance */
+            ASSERT_CURIOSITY(false);
+        }
+        return AFTER_INTERCEPT_LET_GO;
+    } else if (tr->under_dynamo_control != UNDER_DYN_HACK &&
+        !intercept_asynch_for_self(false/*no unknown threads*/)) {
+        LOG(GLOBAL, LOG_VMAREAS, 1, "WARNING: no-asynch thread unloading a dll\n");
+        return AFTER_INTERCEPT_LET_GO;
+    }
+
+    if (in_svchost && dynamo_options.svchost_timeout &&
+        /* case 10509: avoid the timeout on platforms where we haven't seen problems */
+        get_os_version() <= WINDOWS_VERSION_2000) {
+        /* ENTERING GROSS HACK AREA, case 374 */
+#       define HACK_EXE_NAME SVCHOST_EXE_NAME "-netsvcs"
+#       define L_PIN_DLL_NAME  L"wzcsvc.dll"
+        /* The unload order is wzcsvc.dll, WINSTA.dll, CRYPT32.dll, MSASN1.DLL, 
+           so waiting on the first one should be sufficient */
+        /* We don't want to let this dll be unloaded immediately before it
+         * handles some callbacks, so we need to add a time out to the 
+         * unloading thread.  It looks like we're too late to pin it in memory 
+         * so our substitute for 
+         * WinXP::GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_PIN) didn't work.
+         */
+
+        if (in_svchost < 0) {   /* unknown yet */
+            /* NOTE: the hack is needed only for the bug in svchost-netsvcs, 
+               seems to be an issue only on Win2k SP4
+               if that grouping changes this may become a problem again.
+            */
+            SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+            in_svchost = strcasecmp(HACK_EXE_NAME,
+                                    get_short_name(get_application_name())) == 0;
+            SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+            LOG(GLOBAL, LOG_VMAREAS, 3, "intercept_unload_dll: svchost_timeout "
+                "hack_name=%s app_name=%s in_svchost=%d\n", 
+                HACK_EXE_NAME, get_short_name(get_application_name()), in_svchost);
+        }
+
+        if (in_svchost) {
+            extern LDR_MODULE *get_ldr_module_by_pc(app_pc pc); /* in module.c */
+
+            /* FIXME: we're not holding the loader lock here 
+               we may want a deeper interception point, 
+               or maybe we should just as well grab the lock */
+            LDR_MODULE* mod = get_ldr_module_by_pc((app_pc)h);
+
+            if (mod && (wcscasecmp(L_PIN_DLL_NAME, mod->BaseDllName.Buffer) == 0)) {
+                LOG(GLOBAL, LOG_VMAREAS, 1, "intercept_unload_dll: "
+                    "svchost_timeout found target app_name=%s dll_name=%ls\n", 
+                    HACK_EXE_NAME, mod->BaseDllName.Buffer);
+
+                SYSLOG_INTERNAL_WARNING("WARNING: svchost timeout in progress");
+                /* let the events get delivered */
+                os_timeout(dynamo_options.svchost_timeout);
+
+                /* This event can happen any time explorer.exe is restarted so 
+                 * we stay alert
+                 */
+            }
+        }
+    }                                           /* EXITING GROSS HACK */
+
+    /* we're taking over afterward so we need local writability */
+    SELF_PROTECT_LOCAL(get_thread_private_dcontext(), WRITABLE);
+    /* won't be re-protected until dispatch->fcache */
+
+    DOLOG(1,  LOG_VMAREAS, {
+        char buf[MAXIMUM_PATH];
+        /* assumption: h is base address! */
+        size_t size = get_allocation_size((byte *) h, NULL);
+        get_module_name((app_pc) h, buf, sizeof(buf));
+
+        if (buf[0] != '\0') {
+            LOG(GLOBAL, LOG_VMAREAS, 1,
+                "intercept_unload_dll: %s @"PFX" size "PIFX"\n",
+                buf, h, size);
+        } else {
+            LOG(GLOBAL, LOG_VMAREAS, 1,
+                "intercept_unload_dll: <unknown> @"PFX" size "PIFX"\n", h, size);
+        }
+        if (get_thread_private_dcontext() != NULL)
+            LOG(THREAD_GET, LOG_VMAREAS, 1, "intercept_unload_dll: %s @"PFX" "
+                                            "size "PIFX"\n", buf, h, size);
+        DOLOG(3, LOG_VMAREAS, { print_modules(GLOBAL, DUMP_NOT_XML); });
+    });
+    /* we do not flush fragments here b/c this call only decrements
+     * the reference count, we wait until the library is actually
+     * unloaded in the syscall NtUnmapViewOfSection
+     * (there are many unload calls that do not end up unmapping)
+     */
+    if (control_all_threads && tr->under_dynamo_control == UNDER_DYN_HACK)
+        retakeover_after_native(tr, INTERCEPT_UNLOAD_DLL);
+    return AFTER_INTERCEPT_TAKE_OVER;
+}
+
+/****************************************************************************/
+
+void
+retakeover_after_native(thread_record_t *tr, retakeover_point_t where)
+{
+    ASSERT(tr->under_dynamo_control == UNDER_DYN_HACK || 
+           dr_injected_secondary_thread);
+    tr->under_dynamo_control = true;
+    SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+    interception_point = where;
+
+    if (INTERNAL_OPTION(hook_image_entry)) {
+        /* remove the image entry trampoline */
+        ASSERT(image_entry_trampoline != NULL);
+        /* ensure we didn't take over and forget to call this routine -- we shouldn't
+         * get here via interpreting, only natively, so no fragment should exist
+         */
+        ASSERT(image_entry_pc != NULL &&
+               fragment_lookup(tr->dcontext, image_entry_pc) == NULL);
+        /* potentially slightly racy with injected threads */
+        remove_image_entry_trampoline();
+    }
+
+    STATS_INC(num_retakeover_after_native);
+    ASSERT_CURIOSITY(GLOBAL_STAT(num_retakeover_after_native) == 1);
+    SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+
+    LOG(GLOBAL, LOG_VMAREAS, 1, "\n*** re-taking-over @%s after losing control ***\n",
+        retakeover_names[where]);
+
+    /* we don't need to rescan even if we had injected in secondary
+     * thread, since we don't expect anything interesting to have been
+     * done by the primary thread: all initialization should have been
+     * done by the secondary thread, so it should just go to the entry
+     * point.
+     */
+    if (!DYNAMO_OPTION(native_exec_syscalls)) {
+        dcontext_t *dcontext = get_thread_private_dcontext();
+        /* ensure we're (still) the only thread! */
+        /* xref case 3053 where that is not always the case */
+        /* FIXME: we may need to suspend all other threads or grab a lock
+         * to make sure other threads aren't blocked before modifying
+         * protection that everything we're doing here is still safe
+         */
+        ASSERT_CURIOSITY(check_sole_thread() && get_num_threads() == 1);
+        DOSTATS({ ASSERT_CURIOSITY(GLOBAL_STAT(num_threads_created) == 1); });
+        /* If we weren't watching memory alloc/dealloc while the thread was
+         * native we have to redo our exec list completely here.
+         * FIXME: To avoid races w/ other threads (which until we have early
+         * injection we will continue to see: CTRL_SHUTDOWN and other injected
+         * threads (xref case 4181)) it would be best to hold the executable_areas
+         * lock across the removal and re-add, but that would require retooling
+         * find_executable_vm_areas().  So we don't bother to do it now since
+         * native_exec_syscalls is on by default.
+         */
+        LOG(GLOBAL, LOG_VMAREAS, 1,
+            "re-walking executable regions after native execution period\n");
+        /* need to re-walk exec areas since may have changed
+         * while app was native
+         */
+        /* first clear the executable list, FIXME : do we need to clear the 
+         * futurexec list too? */
+        flush_fragments_and_remove_region(dcontext, UNIVERSAL_REGION_BASE,
+                                          UNIVERSAL_REGION_SIZE,
+                                          false /* don't own initexit_lock */,
+                                          true /* remove futures */);
+        
+        /* need to clean any existing regions */
+        DOLOG(SYMBOLS_LOGLEVEL, LOG_SYMBOLS, { 
+            module_cleanup();
+        });
+        modules_reset_list();
+
+#if defined(RCT_IND_BRANCH) || defined(RETURN_AFTER_CALL)
+        /* case 9926: we invalidate to avoid stale targets: but
+         * (case 10518) modules_reset_list() removed all the rct and rac tables for us;
+         * besides, invalidate_{ind_branch,after_call}_target_range doesn't
+         * support cross-module ranges
+         */
+#endif
+
+        find_executable_vm_areas();
+        /* N.B.: this is duplicated from callback_interception_init() because
+         * after we takeover from native executable_areas is built again to
+         * handle the case of the app loading or unloadeing any modules while
+         * we didn't have control.  We shouldn't need this with
+         * native_exec_syscalls turned on, but need it if not.
+         * FIXME: for every special exec region marked during init, must duplicate
+         * marking here!  how ensure that?  today we have two such regions.
+         */
+        add_executable_region(interception_code, INTERCEPTION_CODE_SIZE
+                              _IF_DEBUG("heap mmap callback interception code"));
+        landing_pads_to_executable_areas(true /* add */);
+        LOG(GLOBAL, LOG_VMAREAS, 1, "after re-walking, executable regions are:\n");
+        DOLOG(1, LOG_VMAREAS, { print_executable_areas(GLOBAL); });
+    }
+}
+
+void
+remove_image_entry_trampoline()
+{
+    ASSERT(image_entry_trampoline != NULL);
+    if (image_entry_trampoline != NULL)
+        remove_trampoline(image_entry_trampoline, image_entry_pc);
+    /* FIXME: should set image_entry_trampoline unless we have multiple calls */
+}
+
+/* early inject and user32 should reach this, drinject /
+ * late inject should never get here, and make sure we have
+ * dr_late_injected_primary_thread = true set for that. Assumption: we
+ * assume that with our method of injection, the primary thread will
+ * not be allowed to execute when the secondary thread is in control,
+ * e.g. holds the loader lock.  Any other (tertiary) threads that have
+ * reached their APC before us will be left running as 'unknown'.
+ * We are also assuming that only the primary thread will
+ * ever go through the EXE's image entry.
+ */
+void
+take_over_primary_thread(void)
+{
+    /* note that if we have initialized this thread we already have the value,
+     * but making this more generic in case we move it, 
+     */
+    app_pc win32_start_addr = 0;
+    NTSTATUS res = query_win32_start_addr(NT_CURRENT_THREAD, &win32_start_addr);
+    ASSERT_CURIOSITY(NT_SUCCESS(res) && "failed to obtain win32 start address");
+    if (!NT_SUCCESS(res)) {
+        /* assume it was primary if we can't tell */
+        win32_start_addr = NULL;
+    }
+
+    if ((ptr_uint_t)win32_start_addr < 0x10000 && /* can't be on NULL page */
+        win32_start_addr != NULL) {
+        /* the value is not reliable if the thread has run and there
+         * was a ReplyWaitRecievePort - this would no longer be the
+         * correct start address.  We're making an assumption that if
+         * that is the case then possibly via user32.dll injection
+         * some other DLL that got to run before us may have done this.
+         */
+        ASSERT_NOT_TESTED();
+        win32_start_addr = NULL;
+    }
+    /* unfortunately the value is also not reliable if Native threads
+     * are used as well, and winlogon.exe's parent smss.exe is
+     * creating processes directly.  FIXME: pstat however is able to
+     * show the value
+     */
+
+    /* FIXME: could exempt winlogon.exe by name instead */
+    ASSERT_CURIOSITY(win32_start_addr != NULL);
+
+    /* FIXME: while we should be able to always intercept the image
+     * entry point, regardless of which threads we really control, we
+     * don't need to risk these extra moving parts unless we are
+     * certain we are in a secondary thread
+     */
+    if (win32_start_addr != NULL &&
+        win32_start_addr != get_image_entry()) {
+        dcontext_t *secondary_dcontext = get_thread_private_dcontext();
+        SYSLOG_INTERNAL_WARNING("took over non-primary thread!\n");
+        dr_injected_primary_thread = false;
+        dr_late_injected_primary_thread = false;
+        /* these will be set to to true only when we reach the image entry */
+
+        /* flags the reason for taking over late for use by
+         * insert_image_entry_trampoline() 
+         */
+        dr_injected_secondary_thread = true;
+
+        /* Although we don't really control_all_threads we'll leave it
+         * set, and we're hoping that we'll soon take over the primary
+         * thread since it really has nothing interesting to do before
+         * reaching the image entry point, so it is ok to let it execute
+         * as unknown thread for a little bit.  (We will still have to
+         * furnish it with a dcontext and stack when we take over.)
+         */
+
+        /* potentially racy hook (injected threads) */
+        insert_image_entry_trampoline(secondary_dcontext);
+    } else {
+        /* we are in the primary thread */
+        dr_injected_primary_thread = true;
+    }
+}
+
+/* When we are forced to lose control of a thread during an initialization
+ * APC, we try to regain control w/ a trampoline at the image entry point
+ */
+static after_intercept_action_t  /* note return value will be ignored */
+intercept_image_entry(app_state_at_intercept_t *state)
+{
+    if (dr_injected_secondary_thread) {
+        /* we finally took over the primary thread */
+        SYSLOG_INTERNAL_WARNING("image entry point - should be in primary thread\n");
+
+        DODEBUG({
+            /* check other threads don't reach image entry point for some reason */
+            app_pc win32_start_addr = 0;
+            NTSTATUS res = query_win32_start_addr(NT_CURRENT_THREAD, &win32_start_addr);
+            ASSERT(NT_SUCCESS(res) && "failed to obtain win32 start address");
+            if (win32_start_addr != get_image_entry()) {
+                ASSERT(false && "reached by non-primary thread");
+                /* FIXME: if this can happen we may want to wait for
+                 * the primary and do this out of DODEBUG as well 
+                 */
+            }
+        });
+
+        /* if entry point is never reached debug build will complain */
+
+        /* we could have taken over the primary thread already if it
+         * has executed its KiUserApcDispatcher routine after the
+         * secondary thread has intercepted it, in which case we'd be
+         * on a dstack.  Alternatively, we would be on initstack and
+         * have no dcontext and dstack */
+
+        /* we must create a new dcontext to be a 'known' thread */
+        /* initialize thread now */
+        if (dynamo_thread_init() != -1) {
+            LOG(THREAD_GET, LOG_ASYNCH, 1, "just initialized primary thread \n");
+            /* keep in synch if we do anything else in intercept_new_thread() */
+        } else {
+            /* Note that it is possible that we were in full control
+             * of the primary thread, if it hasn't reached its APC
+             * at the time the secondary thread was started
+             */
+            LOG(THREAD_GET, LOG_ASYNCH, 1, "primary thread was already known\n");
+            /* we'll still treat as if it needed to be injected late */
+
+            if (!RUNNING_WITHOUT_CODE_CACHE()) {
+                dcontext_t *existing_dcontext = get_thread_private_dcontext();
+
+                /* we MUST flush our image entry point fragment that
+                 * would currently use the trampoline.  
+                 */ 
+                ASSERT(fragment_lookup(existing_dcontext, image_entry_pc) != NULL);
+                /* We can lookup at that tag almost for sure,
+                 * currently since the indirect call will always have
+                 * us start a new bb.  Note we were just there, and
+                 * the new bb we'll build won't be executed ever
+                 * again.
+                 */
+                /* note we only flush, but not remove region, since we will not rewalk */
+                flush_fragments_in_region_start(existing_dcontext,
+                                                image_entry_pc, 1,
+                                                false /* don't own initexit_lock */,
+                                                false /* keep futures */,
+                                                false /* exec still valid */,
+                                                false /* don't force sychall */
+                                                _IF_DGCDIAG(NULL));
+                flush_fragments_in_region_finish(existing_dcontext, false);
+
+                ASSERT_NOT_TESTED();
+            }
+        }
+
+        /* for presys_TerminateThread() need to set after we have become 'known' */
+        SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+        dr_late_injected_primary_thread = true;
+        dr_injected_primary_thread = true;
+        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+        /* three protect/unprotect pairs on this path, still rare enough */
+
+        /* in both -client and -thin_client mode */
+        if (RUNNING_WITHOUT_CODE_CACHE()) {
+            /* nothing more to do here, we just wanted to have a
+             * proper dcontext created and set in TLS
+             */
+            dcontext_t *dcontext = get_thread_private_dcontext();
+
+            /* potentially slightly racy with injected threads */
+            remove_image_entry_trampoline();
+            /* note that we are not freeing the emitted code, we are
+             * just fixing up the original application code
+             */
+
+            /* case 9347 - we will incorrectly reach asynch_take_over() */
+            /* FIXME: we chould have created the trampoline to 'let
+             * go' directly to the now restored application code.
+             * AFTER_INTERCEPT_LET_GO_ALT_DYN is closest to what we
+             * need - direct native JMP to the image entry, instead of
+             * potentially non-pc-relativized execution from our copy,
+             * and we only have to make sure we use the initstack properly.
+             *
+             * Instead of changing the assembly, we're using a gross
+             * hack here - we'll have to rely on asynch_take_over() ->
+             * transfer_to_dispatch() -> is_stopping_point(),
+             * dispatch_enter_native() to cleanly jump back to the
+             * original application code (releasing initstack etc)
+             */
+            dcontext->next_tag = BACK_TO_NATIVE_AFTER_SYSCALL;
+            /* start_pc is the take-over pc that will jmp to the syscall instr, while
+             * we need the post-syscall pc, which we stored when generating the trampoline
+             */
+            ASSERT(image_entry_pc != NULL);
+            dcontext->native_exec_postsyscall = image_entry_pc;
+
+            /* ignored, we are created as AFTER_INTERCEPT_TAKE_OVER_SINGLE_SHOT */
+            return AFTER_INTERCEPT_LET_GO;
+        }
+
+        ASSERT(thread_lookup(get_thread_id())->under_dynamo_control != UNDER_DYN_HACK); 
+    }
+
+    if (dynamo_initialized) {
+        thread_record_t *tr = thread_lookup(get_thread_id());
+        /* FIXME: we must unprot .data here and again for retakeover: worth optimizing? */
+        set_reached_image_entry();
+        if ((tr != NULL && tr->under_dynamo_control == UNDER_DYN_HACK)
+            || dr_injected_secondary_thread) {
+            LOG(THREAD_GET, LOG_ASYNCH, 1, "inside intercept_image_entry\n");
+            /* we were native, retakeover */
+            retakeover_after_native(tr, INTERCEPT_IMAGE_ENTRY);
+#ifdef RETURN_AFTER_CALL
+            /* ref case 3565, we need to add the return address to the allowed
+             * return after call transitions table */
+            ASSERT(tr->dcontext == get_thread_private_dcontext());
+
+            /* Addition to the fix for case 3565; ref case 3896, can't add the
+             * return address for -no_ret_after_call; hash table not set up.
+             */
+            if (DYNAMO_OPTION(ret_after_call)) {
+                if (is_readable_without_exception((byte *)state->mc.xsp,
+                                                  sizeof(app_pc))) {
+                    fragment_add_after_call(tr->dcontext,
+                                            *(app_pc *) state->mc.xsp);
+                } else {
+                    ASSERT_NOT_REACHED();
+                }
+            }
+#endif
+            /* we want to take over, but we'll do that anyway on return from this routine
+             * since our interception code was told to take over afterward, so we wait.
+             * this routine executes natively so even if we were already in control
+             * we want to take over so no conditional is needed there.
+             */
+        } else {
+            SYSLOG_INTERNAL_ERROR("Image entry interception point reached by unexpected %s thread %d", 
+                                  (tr == NULL) ? "unknown" : "known",
+                                  get_thread_id());
+            ASSERT_NOT_REACHED();
+        }
+    }
+    return AFTER_INTERCEPT_TAKE_OVER;
+}
+
+/* WARNING: only call this when there is only one thread going!
+ * This is not thread-safe!
+ */
+static byte *
+insert_image_entry_trampoline(dcontext_t *dcontext)
+{
+    /* Adds a trampoline at the image entry point.
+     * We ignore race conditions here b/c we assume there's only one thread.
+     * FIXME: kernel32's base process start point is actually where
+     * control appears, but it's not exported.  you can find it by seeing
+     * what the starting point is using our targeted injection.
+     * It only executes a dozen or so instructions before calling the
+     * image entry point, but it would be nice if we had a way to find
+     * it -- how does loader/whoever know what it is?  how do you get there,
+     * if the thread's pc points at image entry point to begin with?
+     *
+     * (kernel32 call to image entry point for Win2K Server is 0x77e87900
+     * (so retaddr on stack is 0x77e87903).  the actual entry point, from
+     * examining the ntcontinue finishing up the init apc, is 0x77e878c1.
+     * FIXME: how find that programatically?  is it used for every process?
+     * Inside Windows 2000 indicates that it is.
+     * Promising approach: use QueryThreadInformation with ThreadQuerySetWin32StartAddress
+     * -- although that can get clobbered, hopefully will still be set to start 
+     * address when we read it.
+     * Note that will most likely be the image entry point (win32), not the 
+     * actual entry point (it is the address in eax of the CreateThread system
+     * call) as that's how it behaves for a normal CreateThread
+     */
+    static bool image_entry_hooked = false;
+    /* need to set a flag to prevent double injection - e.g. if we
+     * have a callback return in a secondary thread
+     */
+    if (image_entry_hooked) {
+        ASSERT_NOT_TESTED();
+        LOG(THREAD, LOG_ASYNCH, 1, "WARNING: already hooked!\n");
+        ASSERT(dr_injected_secondary_thread);
+        return NULL;
+    }
+    SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+
+    /* note we are thread safe, but we are not worried about a race,
+     * but only about getting a callback in a secondary thread after
+     * it has already hooked */
+    image_entry_hooked = true;
+    image_entry_pc = get_image_entry();
+    if (dr_injected_secondary_thread) {
+        LOG(THREAD, LOG_ASYNCH, 1,
+            "WARNING: image entry hook to catch primary thread!\n");
+    } else {
+        LOG(THREAD, LOG_ASYNCH, 1,
+            "WARNING: callback return with native cb context!\n");
+    }
+
+    LOG(THREAD, LOG_ASYNCH, 1,
+        "\tInserting trampoline at image entry point "PFX"\n",
+        image_entry_pc);
+
+    image_entry_trampoline = insert_trampoline(image_entry_pc, intercept_image_entry,
+                                               0, /* no arg */
+                                               false /* do not assume esp */,
+                                               /* handler should restore target */
+                                               AFTER_INTERCEPT_TAKE_OVER_SINGLE_SHOT,
+                                               true /* single shot - safe to ignore CTI */);
+    SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+    return image_entry_pc;
+}
+
+/****************************************************************************/
+
+/* Note that we don't want to be overwriting ntdll code when another
+ * thread is in there, so this init routine should be called prior to
+ * creating any other threads...if not using run-entire-program-under-dynamo
+ * approach, I'm not sure how to guarantee no race conditions...FIXME
+ */
+
+/* We rely on the compiler doing the right thing
+   so when we dereference an imported function we get its real address
+   instead of a stub in our module. The loader does the rest of the magic.
+*/
+GET_NTDLL(KiUserApcDispatcher, (IN PVOID Unknown1, 
+                                IN PVOID Unknown2, 
+                                IN PVOID Unknown3, 
+                                IN PVOID ContextStart, 
+                                IN PVOID ContextBody));
+GET_NTDLL(KiUserCallbackDispatcher, (IN PVOID Unknown1, 
+                                     IN PVOID Unknown2, 
+                                     IN PVOID Unknown3));
+GET_NTDLL(KiUserExceptionDispatcher, (IN PVOID Unknown1, 
+                                      IN PVOID Unknown2));
+GET_NTDLL(KiRaiseUserExceptionDispatcher, (void));
+
+void
+callback_interception_init()
+{
+    byte *pc;
+    byte *int2b_after_cb_dispatcher;
+    DEBUG_DECLARE(dr_marker_t test_marker);
+
+    intercept_asynch = true;
+    intercept_callbacks = true;
+
+    interception_code = heap_mmap(INTERCEPTION_CODE_SIZE);
+
+#ifdef INTERCEPT_TOP_LEVEL_EXCEPTIONS
+    app_top_handler = 
+        SetUnhandledExceptionFilter((LPTOP_LEVEL_EXCEPTION_FILTER) our_top_handler);
+#endif
+    
+    /* Note that we go ahead and assume that the app's esp is valid for
+     * most of these interceptions.  This is for efficiency -- the alternative
+     * is to use the global initstack, which imposes a synchronization point,
+     * which we don't want for multi-thread frequent events like callbacks.
+     * Re: transparency, if the app's esp were not valid, the native
+     * routine would crash in a similar, though not necessarily
+     * exactly the same, manner as the program will under DynamoRIO.
+     */
+
+    /* We place a small struct at the base of the interception code to pass
+     * information to outside processes (such as build type, number, etc.).
+     * The outside process finds the struct by page aligning the target of our
+     * KiUserCallbackDispatcher hook and verifies it by checking certain magic
+     * number fields in the struct.  This is also used to determine if dr
+     * is running in the process */
+    ASSERT(ALIGNED(interception_code, PAGE_SIZE));
+    init_dr_marker((dr_marker_t *)interception_code);
+
+    pc = interception_code + sizeof(dr_marker_t);
+
+    /* Order of hooking matters to some degree.  LdrInitializeThunk, then APC
+     * dispatcher and then callback dispatcher.
+     */
+    if (!DYNAMO_OPTION(thin_client)) {
+        if (DYNAMO_OPTION(handle_ntdll_modify) != DR_MODIFY_OFF) {
+            app_pc ntdll_base = get_ntdll_base();
+            size_t ntdll_module_size = get_allocation_size(ntdll_base, NULL);
+
+            /* FIXME: should only add code section(s!), but for now adding
+             * whole module
+             */
+            app_pc ntdll_code_start = ntdll_base;
+            app_pc ntdll_code_end = ntdll_base + ntdll_module_size;
+
+            tamper_resistant_region_add(ntdll_code_start, ntdll_code_end);
+        }
+    }
+
+    intercept_map = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, intercept_map_t,
+                                    ACCT_OTHER, PROTECTED);
+    memset(intercept_map, 0, sizeof(*intercept_map));
+
+    /* Note that if we switch to using a non-5-byte-reljmp for our Ki hooks
+     * we need to change our drmarker reader. */
+
+    /* LdrInitializeThunk is hooked for thin_client too, so that
+     * each thread can have a dcontext (case 8884). */
+    if (get_os_version() == WINDOWS_VERSION_VISTA) {
+        module_handle_t ntdllh = get_ntdll_base();
+        LdrInitializeThunk = (byte *)get_proc_address(ntdllh,
+                                                      "LdrInitializeThunk");
+        ASSERT(LdrInitializeThunk != NULL);
+        /* initialize this now for use later in intercept_new_thread() */
+        RtlUserThreadStart = (byte *)get_proc_address(ntdllh,
+                                                      "RtlUserThreadStart");
+        ASSERT(RtlUserThreadStart != NULL);
+        ldr_init_pc = pc;
+        pc = intercept_call(pc, (byte*)LdrInitializeThunk, intercept_ldr_init,
+                            0, /* no arg */
+                            true /* FIXME: assume esp only until dstack check
+                                  * separated! */,
+                            AFTER_INTERCEPT_LET_GO,
+                            false /* cannot ignore on CTI */,
+                            false /* handle CTI */,
+                            NULL, NULL);
+    }
+
+    /* hook APC dispatcher, also check context offset */
+    /* APC dispatcher is hooked for thin_client too, so that
+     * each thread can have a dcontext (case 8884).  Needed for handling 
+     * process control/detach nudges for thin_client and for muting other ones 
+     * (case 8888).  Also, in future, if we implement inflating to hotp_only
+     * mode, then each thread needs to have a dcontext. 
+     */
+    check_apc_context_offset((byte*)KiUserApcDispatcher);
+    after_apc_orig_pc = get_pc_after_call((byte*)KiUserApcDispatcher, NULL);
+    apc_pc = pc;
+    pc = intercept_call(pc, (byte*)KiUserApcDispatcher, intercept_apc,
+                        0, /* no arg */
+                        true /* FIXME: assume esp only until dstack check
+                              * separated! */,
+                        AFTER_INTERCEPT_LET_GO,
+                        false /* cannot ignore on CTI */,
+                        false /* handle CTI */,
+                        NULL, NULL);
+
+    /* The apc hook is how we catch new threads, make sure none sneaked in
+     * while we were still initializing. Also the nodemgr etc. use the
+     * callback hook to find the drmarker, once it's inserted they might
+     * start injecting threads, so be sure to do the apc hook first so we
+     * see them. */
+    DODEBUG({
+        /* case 9423 - just SYSLOG, we can somewhat handle these */
+        if (!check_sole_thread()) {
+            SYSLOG_INTERNAL_WARNING("!check_sole_thread() case 9423");
+        }
+    });
+
+    callback_pc = pc;
+    /* make sure nobody ever comes back to instr after callback call: */
+    after_callback_orig_pc =
+        get_pc_after_call((byte*)KiUserCallbackDispatcher,
+                          &int2b_after_cb_dispatcher);
+    /* make sure dispatcher concludes with an int 2b */
+    /* In Win 2003 SP1, the dispatcher concludes with a ret.  See case 3522. */
+    /* For thin_client we need to hook this to be able to read the DRmarker. */
+    ASSERT_CURIOSITY(int2b_after_cb_dispatcher != NULL);
+    pc = intercept_call(pc, (byte*)KiUserCallbackDispatcher,
+                        intercept_callback_start,
+                        0, /* no arg */
+                        true /* FIXME: assume esp only until dstack check separated! */,
+                        AFTER_INTERCEPT_LET_GO,
+                        false /* cannot ignore on CTI */, false /* handle CTI */,
+                        NULL, NULL);
+
+    /* We would like to not assume esp for exceptions, since we mark
+     * executable regions read-only and if the stack contains code
+     * we'd like to be able to catch write faults on the stack,
+     * but the kernel just silently kills the process if the user stack
+     * is not valid!  So may as well be more efficient and assume esp ourselves.
+     * Note: thin_client needs this hook to catch and report core exceptions.
+     */
+    exception_pc = pc;
+    pc = intercept_call(pc, (byte*)KiUserExceptionDispatcher,
+                        intercept_exception,
+                        0, /* no arg */
+                        false /* do not assume esp */,
+                        AFTER_INTERCEPT_LET_GO,
+                        false /* cannot ignore on CTI */,
+                        false /* handle CTI */,
+                        NULL, NULL);
+
+    if (!DYNAMO_OPTION(thin_client)) {
+        raise_exception_pc = pc;
+        pc = intercept_call(pc, (byte*)KiRaiseUserExceptionDispatcher, 
+                            intercept_raise_exception,
+                            0, /* no arg */
+                            false /* do not assume esp */,
+                            AFTER_INTERCEPT_LET_GO,
+                            false /* cannot ignore on CTI */, 
+                            false /* handle CTI */,
+                            NULL, NULL);
+
+        /* WARNING: these two routines are entered from user mode, so the
+         * interception code for them ends up being executed under DynamoRIO,
+         * and we rely on the list of do-not-inline to allow the actual
+         * intercept_{un,}load_dll routine to execute natively.
+         * We also count on the interception code to not do anything that won't
+         * cause issues when passed through dispatch().
+         * FIXME: a better way to do this?  assume entry point will always
+         * be start of fragment, add clean call in mangle?
+         */
+        load_dll_pc = pc;
+        pc = intercept_call(pc, (byte*)LdrLoadDll, intercept_load_dll,
+                            0, /* no arg */
+                            false /* do not assume esp */, 
+                            AFTER_INTERCEPT_DYNAMIC_DECISION,
+                            true /* not critical trampoline, can ignore if 
+                                  * hooked with CTI */, 
+                            false /* handle CTI */,
+                            NULL, NULL);
+        if (pc == NULL) {
+            /* failed to hook, reset pointer for next routine */
+            pc = load_dll_pc;
+            load_dll_pc = NULL;
+        }
+
+        unload_dll_pc = pc;
+        /* FIXME: used only for DEBUG and svchost_hack (case 374), shouldn't 
+         * do it all the time
+         */
+        pc = intercept_call(pc, (byte*)LdrUnloadDll, intercept_unload_dll,
+                            0, /* no arg */
+                            false /* do not assume esp */, 
+                            AFTER_INTERCEPT_DYNAMIC_DECISION,
+                            true /* not critical trampoline, can ignore if 
+                                  * hooked with CTI */,
+                            false /* handle CTI */,
+                            NULL, NULL);
+        if (pc == NULL) {
+            /* failed to hook, reset pointer for next routine */
+            pc = unload_dll_pc;
+            unload_dll_pc = NULL;
+        }
+    }
+
+    ASSERT(pc - interception_code < INTERCEPTION_CODE_SIZE);
+    interception_cur_pc = pc; /* set global pc for future trampoline insertions */
+
+    if (DYNAMO_OPTION(native_exec_syscalls)) {
+        syscall_trampolines_start = interception_cur_pc;
+        init_syscall_trampolines();
+        syscall_trampolines_end = interception_cur_pc;
+    }
+
+    if (DYNAMO_OPTION(clean_testalert)) {
+        GET_NTDLL(NtTestAlert, (void));
+        clean_syscall_wrapper((byte*)NtTestAlert, SYS_TestAlert);
+    }
+
+#ifdef PROGRAM_SHEPHERDING
+    /* a fragment will be built from this code, but it's not for
+     * general execution, so add as dynamo area but not executable area
+     * should already be added since it's part of data section
+     * FIXME: use generated-code region rather than data section?
+     */
+    if (!is_dynamo_address(interception_code) ||
+        !is_dynamo_address(interception_code + INTERCEPTION_CODE_SIZE-1)) { /* check endpoints */
+        /* probably was already added but just to make sure */
+        add_dynamo_vm_area(interception_code,
+                           interception_code + INTERCEPTION_CODE_SIZE-1,
+                           MEMPROT_READ | MEMPROT_WRITE,
+                           true /* from image since static */
+                           _IF_DEBUG("intercept_call"));
+    }
+#endif
+
+    DOLOG(3, LOG_EMIT, {
+        dcontext_t *dcontext = get_thread_private_dcontext();
+        bool skip8 = false;
+        byte *end_asynch_pc = pc;
+        pc = interception_code + sizeof(dr_marker_t);
+        LOG(GLOBAL, LOG_EMIT, 3, "\nCreated these interception points:\n");
+        do {
+            if (pc == callback_pc) {
+                LOG(GLOBAL, LOG_EMIT, 3, "KiUserCallbackDispatcher:\n");
+                LOG(GLOBAL, LOG_EMIT, 3, "  <backup of 1st 5 bytes>\n");
+                LOG(GLOBAL, LOG_EMIT, 3, "  <landing pad address>\n");
+                pc += 5 + sizeof(byte*);
+            } else if (pc == apc_pc) {
+                LOG(GLOBAL, LOG_EMIT, 3, "KiUserApcDispatcher:\n");
+                LOG(GLOBAL, LOG_EMIT, 3, "  <backup of 1st 5 bytes>\n");
+                LOG(GLOBAL, LOG_EMIT, 3, "  <landing pad address>\n");
+                pc += 5 + sizeof(byte*);
+            } else if (pc == exception_pc) {
+                LOG(GLOBAL, LOG_EMIT, 3, "KiUserExceptionDispatcher:\n");
+                LOG(GLOBAL, LOG_EMIT, 3, "  <backup of 1st 5 bytes>\n");
+                LOG(GLOBAL, LOG_EMIT, 3, "  <landing pad address>\n");
+                pc += 5 + sizeof(byte*);
+            } else if (pc == raise_exception_pc) {
+                LOG(GLOBAL, LOG_EMIT, 3, "KiRaiseUserExceptionDispatcher:\n");
+                LOG(GLOBAL, LOG_EMIT, 3, "  <backup of 1st 5 bytes>\n");
+                LOG(GLOBAL, LOG_EMIT, 3, "  <landing pad address>\n");
+                pc += 5 + sizeof(byte*);
+            } else if (pc == load_dll_pc) {
+                LOG(GLOBAL, LOG_EMIT, 3, "LdrLoadDll:\n");
+                LOG(GLOBAL, LOG_EMIT, 3, "  <backup of 1st 5 bytes>\n");
+                LOG(GLOBAL, LOG_EMIT, 3, "  <landing pad address>\n");
+                pc += 5 + sizeof(byte*);
+            } else if (pc == unload_dll_pc) {
+                LOG(GLOBAL, LOG_EMIT, 3, "LdrUnloadDll:\n");
+                LOG(GLOBAL, LOG_EMIT, 3, "  <backup of 1st 5 bytes>\n");
+                LOG(GLOBAL, LOG_EMIT, 3, "  <landing pad address>\n");
+                pc += 5 + sizeof(byte*);
+            } else if (pc != NULL && pc == ldr_init_pc) {
+                LOG(GLOBAL, LOG_EMIT, 3, "LdrInitializeThunk:\n");
+                LOG(GLOBAL, LOG_EMIT, 3, "  <backup of 1st 5 bytes>\n");
+                LOG(GLOBAL, LOG_EMIT, 3, "  <landing pad address>\n");
+                pc += 5 + sizeof(byte*);
+            } else if (pc == end_asynch_pc) {
+                LOG(GLOBAL, LOG_EMIT, 3, "\nSyscall trampolines:\n\n");
+            }
+            IF_X64({
+                /* handle 8 bytes of address at end */
+                if (pc + JMP_ABS_IND64_SIZE + sizeof(byte*) <= interception_cur_pc &&
+                    *pc==JMP_ABS_IND64_OPCODE && *(pc+1)==JMP_ABS_MEM_IND64_MODRM &&
+                    *(int*)(pc+2)==0/*next pc*/)
+                    skip8 = true;
+            });
+            pc = disassemble_with_bytes(dcontext, pc, main_logfile);
+            IF_X64({
+                if (skip8) {
+                    LOG(GLOBAL, LOG_EMIT, 3, "  <return target address: "PFX">\n",
+                        *(byte **)pc);
+                    pc += sizeof(byte*);
+                    skip8 = false;
+                }
+            });
+        } while (pc < interception_cur_pc);
+        LOG(GLOBAL, LOG_EMIT, 3, "\n");
+    });
+
+    make_unwritable(interception_code, INTERCEPTION_CODE_SIZE);
+
+    /* No vm areas except dynamo_areas exists in thin_client mode. */
+    if (!DYNAMO_OPTION(thin_client)) {
+        /* add interception code to the executable list */
+        /* N.B.: we duplicate this call after losing control and re-doing exec 
+         * regions
+         */
+        add_executable_region(interception_code, INTERCEPTION_CODE_SIZE
+                              _IF_DEBUG("heap mmap callback interception code"));
+        landing_pads_to_executable_areas(true /* add */);
+    }
+
+    ASSERT(read_and_verify_dr_marker(NT_CURRENT_PROCESS, &test_marker) ==
+           DR_MARKER_FOUND);
+}
+
+DEBUG_DECLARE(static bool callback_interception_unintercepted = false;);
+
+/* N.B.: not thread-safe! */
+void
+callback_interception_unintercept()
+{
+    /* remove syscall trampolines BEFORE turning off asynch, to avoid
+     * thinking we're a native thread! */
+    if (DYNAMO_OPTION(native_exec_syscalls)) {
+        exit_syscall_trampolines();
+        syscall_trampolines_start = NULL;
+        syscall_trampolines_end = NULL;
+    }
+
+    intercept_asynch = false;
+    intercept_callbacks = false;
+
+    LOG(GLOBAL, LOG_ASYNCH|LOG_STATS, 1,
+        "Total # of asynchronous events for process:\n");
+    LOG(GLOBAL, LOG_ASYNCH|LOG_STATS, 1, "   Callbacks:  %d\n", GLOBAL_STAT(num_callbacks));
+    LOG(GLOBAL, LOG_ASYNCH|LOG_STATS, 1, "   APCs:       %d\n", GLOBAL_STAT(num_APCs));
+    LOG(GLOBAL, LOG_ASYNCH|LOG_STATS, 1, "   Exceptions: %d\n", GLOBAL_STAT(num_exceptions));
+
+    un_intercept_call(load_dll_pc, (byte*)LdrLoadDll);
+    un_intercept_call(unload_dll_pc, (byte*)LdrUnloadDll);
+    un_intercept_call(raise_exception_pc,
+                      (byte*)KiRaiseUserExceptionDispatcher);
+    un_intercept_call(callback_pc, (byte*)KiUserCallbackDispatcher);
+    un_intercept_call(apc_pc, (byte*)KiUserApcDispatcher);
+    if (get_os_version() == WINDOWS_VERSION_VISTA) {
+        ASSERT(ldr_init_pc != NULL && LdrInitializeThunk != NULL);
+        un_intercept_call(ldr_init_pc, (byte *)LdrInitializeThunk);
+    }
+    /* remove exception dispatcher last to catch errors in the meantime */
+    un_intercept_call(exception_pc, (byte*)KiUserExceptionDispatcher);
+
+    DODEBUG(callback_interception_unintercepted = true;);
+}
+
+void
+callback_interception_exit()
+{
+    ASSERT(callback_interception_unintercepted);
+    /* FIXME : we are exiting so no need to flush here right? */
+    if (!DYNAMO_OPTION(thin_client))
+        remove_executable_region(interception_code, INTERCEPTION_CODE_SIZE,
+                                 false/*no lock*/);
+    heap_munmap(interception_code, INTERCEPTION_CODE_SIZE);
+
+    HEAP_TYPE_FREE(GLOBAL_DCONTEXT, intercept_map, intercept_map_t,
+                   ACCT_OTHER, PROTECTED);
+
+    landing_pads_to_executable_areas(false /* remove */);
+}
+
+static void 
+swap_dcontexts(dcontext_t *d1, dcontext_t *d2)
+{
+    dcontext_t temp;
+    /* be careful some fields can't be blindly swapped */
+    if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)) {
+        /* deep swap of upcontext */
+        unprotected_context_t uptemp;
+        memcpy((void *)&uptemp, (void *)d1->upcontext.separate_upcontext,
+               sizeof(unprotected_context_t));
+        memcpy((void *)d1->upcontext.separate_upcontext,
+               (void *)d2->upcontext.separate_upcontext,
+               sizeof(unprotected_context_t));
+        memcpy((void *)d2->upcontext.separate_upcontext, (void *)&uptemp,
+               sizeof(unprotected_context_t));
+    }
+    memcpy((void *)&temp, (void *)d1, sizeof(dcontext_t));
+    memcpy((void *)d1, (void *)d2, sizeof(dcontext_t));
+    memcpy((void *)d2, (void *)&temp, sizeof(dcontext_t));
+    if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)) {
+        /* must swap upcontext pointers back since code is hardcoded for main one */
+        temp.upcontext.separate_upcontext = d2->upcontext.separate_upcontext;
+        d2->upcontext.separate_upcontext = d1->upcontext.separate_upcontext;
+        d1->upcontext.separate_upcontext = temp.upcontext.separate_upcontext;
+    }
+    /* must swap self pointers back so that asm routines work */
+    temp.upcontext_ptr = d2->upcontext_ptr;
+    d2->upcontext_ptr = d1->upcontext_ptr;
+    d1->upcontext_ptr = temp.upcontext_ptr;
+    /* swap nonswapped field back */
+    temp.nonswapped_scratch = d2->nonswapped_scratch;
+    d2->nonswapped_scratch = d1->nonswapped_scratch;
+    d1->nonswapped_scratch = temp.nonswapped_scratch;
+    /* swap allocated starts back */
+    temp.allocated_start = d2->allocated_start;
+    d2->allocated_start = d1->allocated_start;
+    d1->allocated_start = temp.allocated_start;
+    /* swap list pointers back */
+    temp.prev_unused = d1->prev_unused;
+    d1->prev_unused = d2->prev_unused;
+    d2->prev_unused = temp.prev_unused;
+}
+
+#ifdef RETURN_AFTER_CALL
+/* return EMPTY if we are past the stack bottom - target_pc should NOT be let through
+          BOTTOM_REACHED just reached stack bottom, let this through, but start enforcing
+          BOTTOM_NOT_REACHED haven't yet reached the stack bottom, let this through do not enforce
+ */
+initial_call_stack_status_t
+at_initial_stack_bottom(dcontext_t *dcontext, app_pc target_pc)
+{
+    /* FIXME: this is quite messy - should figure a better way to keep track
+     * of the initial call stack:
+     *  - either explicitly save it after all, or maybe only use the depth
+     *    instead of matching the address
+     */
+    LOG(THREAD, LOG_ASYNCH|LOG_STATS, 1, 
+        "get_initial_stack_bottom: preinjected=%d interception_point=%d "
+        "after_callback="PFX"\n", 
+        dr_preinjected, interception_point, after_callback_orig_pc);
+    /* CHECK: drinject AND follow children go through dynamo_auto_start 
+       instead of dynamorio_app_take_over which sets dr_preinjected */
+    
+    /* we start with an empty stack when explicitly injected  */
+    if (!dr_preinjected)
+        return INITIAL_STACK_EMPTY;
+
+    if (interception_point == INTERCEPT_IMAGE_ENTRY) {
+        /* intercept_image_entry does not execute any DR returns so this is a violation.
+         * At image entry trampoline we explicitly added the kernel32 thunk as a RAC target,
+         * so even though we didn't see the real stack bottom we've added the bottom frame
+         * and so cannot bottom out.
+         */
+        return INITIAL_STACK_EMPTY;
+    }
+
+    if (interception_point == INTERCEPT_LOAD_DLL
+        || interception_point == INTERCEPT_UNLOAD_DLL
+        || interception_point == INTERCEPT_EARLY_ASYNCH
+        || interception_point == INTERCEPT_SYSCALL
+        || interception_point == INTERCEPT_PREINJECT) {
+        /* initial APC still has control */
+        /* Checking for after_call instruction in KiUserApcDispatcher is problematic --
+         * could see it too early if there's a nested APC before the init APC finishes,
+         * and if we take control after the end of the init APC but before the image entry
+         * trampoline we may never interpret that pc.  Instead we wait until the image
+         * entry, fixing those problems, and only opening up a hole by delaying during the
+         * kernel32 thunk, which doesn't execute much code.
+         */
+        if (reached_image_entry_yet()) {
+            /* we're past the image entry, so this is a violation, not a bottoming
+             * out -- and any issues w/ bottoming out between the image entry
+             * and the kernel32 thunk on the way back down should have been
+             * handled in the image entry trampoline
+             */
+            return INITIAL_STACK_EMPTY;
+        } else {
+            /* FIXME: if we never get to that address we'll never trigger a violation, very unsafe */
+            return INITIAL_STACK_BOTTOM_NOT_REACHED;
+        }
+    }
+
+    ASSERT_NOT_REACHED();
+    /* safe default */
+    return INITIAL_STACK_EMPTY;
+}
+
+/* Allow a ret to target an address inside an .xdata section that was
+ * the argument to an NtFlushInstructionCache syscall.
+ * (xref case 7319)
+ */
+static bool
+at_xdata_rct_exception(dcontext_t *dcontext, app_pc target_pc)
+{
+    app_pc modbase = get_module_base(target_pc);
+    ASSERT(DYNAMO_OPTION(xdata_rct));
+    if (modbase != NULL && is_in_xdata_section(modbase, target_pc, NULL, NULL) &&
+        was_address_flush_start(dcontext, target_pc)) {
+        SYSLOG_INTERNAL_INFO("RCT: .xdata NtFlush-target matched @"PFX, target_pc);
+        STATS_INC(ret_after_call_xdata);
+        return true;
+    }
+    return false;
+}
+
+/* allow any RCT - though seen a .E when a CALL* from a Kaspersky
+ * kernel driver goes to the entry points of the API routines whose
+ * exports were hijacked
+ */
+static bool
+at_driver_rct_exception(dcontext_t *dcontext, app_pc source_pc)
+{
+    ASSERT(DYNAMO_OPTION(driver_rct));
+    if (!is_user_address(source_pc) &&
+        is_driver_address(source_pc)) {
+        SYSLOG_INTERNAL_INFO_ONCE("RCT: kernel driver source @"PFX, source_pc);
+        STATS_INC(num_rct_driver_address);
+        return true;
+    }
+    return false;
+}
+
+/* Fibers on Win2003 RAC false positive - see case 1543, see also 9726 on Vista */
+/* On Win2003 - the initial trampoline needed to initialize the
+   function call to the FiberFunc is not registered as an after call
+   site when the SwitchToFiber is called for the first time.  This
+   piece of code seems to be the same in kernel32.dll on Windows 2000
+   and Windows 2003.  So I'll take the risk they won't change this
+   instruction to use a different register, and they won't optimize it
+   like in our case 1307.  The match will be simply of the first 4
+   bytes SEG_FS MOV_EAX 10 00 and only one target location will be
+   exempted.
+
+   FIXME: It would be nicer if we could get a pattern on the
+   source_fragment (e.g. like at_vbjmp_exception), then we wouldn't
+   have to worry about readability of the target address and it is in
+   a way safer.
+  
+   In terms of multithread safety - it is OK to have multiple threads
+   test for the condition, and as long as any of them sets the
+   exempted address I don't expect attackers to have any chance here.
+
+   FIXME: We also explicitly check if target_pc is readable, although
+   a higher level check should be added to return
+   UNREADABLE_MEMORY_EXECUTION_EXCEPTION in that case.
+
+   > u 0x77e65927  
+   kernel32!ConvertFiberToThread+0x44:  [ with symbols kernel32!BaseFiberStart: ]
+   77e65927 64a110000000     mov     eax,fs:[00000010]  ; FIBER_DATA_TIB_OFFSET
+
+   From XP (which doesn't have a problem since we jmp * instead of ret here)
+   kernel32!BaseFiberStart:
+   7c82ff92 64a110000000     mov     eax,fs:[00000010]
+   7c82ff98 ffb0b8000000     push    dword ptr [eax+0xb8]
+   7c82ff9e ffb0c4000000     push    dword ptr [eax+0xc4]
+   7c82ffa4 e8a3b6fdff       call    kernel32!BaseThreadStart (7c80b64c)
+   7c82ffa9 c3               ret
+
+   on Vista routine has been changed with the addition of an SEH frame first and other
+   modification (likely since kernel32!BaseThreadStart no longer exists)
+   kernel32!BaseFiberStart:
+   76cde8ca 6a0c             push    0xc
+   76cde8cc 68f8e8cd76       push    0x76cde8f8
+   76cde8d1 e85a8f0300       call    kernel32!_SEH_prolog4 (76d17830)
+   76cde8d6 64a110000000     mov     eax,fs:[00000010]
+   76cde8dc 8b88c4000000     mov     ecx,[eax+0xc4]
+   76cde8e2 8b80b8000000     mov     eax,[eax+0xb8]
+   76cde8e8 8365fc00         and     dword ptr [ebp-0x4],0x0
+   76cde8ec 50               push    eax
+   76cde8ed ffd1             call    ecx
+   76cde8ef 6a00             push    0x0
+   76cde8f1 ff150810cd76 call dword ptr [kernel32!_imp__RtlExitUserThread (76cd1008)]
+   76cde8f7 90               nop
+   76cde8f8 feff             ???     bh
+   76cde8fa ffff             ???
+
+   On 64-bit XP:
+   kernel32!BaseFiberStart:
+   00000000`77d687c0 65488b0c2520000000 mov   rcx,qword ptr gs:[20h]
+   00000000`77d687c9 488b91b8000000  mov     rdx,qword ptr [rcx+0B8h]
+   00000000`77d687d0 488b89b0000000  mov     rcx,qword ptr [rcx+0B0h]
+   00000000`77d687d7 e9c42e0000      jmp     kernel32!BaseThreadStart (00000000`77d6b6a0)
+
+   On 64-bit Vista:
+   Note - There is no kernel32!BaseFiberStart symbol, but this routine takes its
+   place and is reached in the same way.
+   00000000`772aa1d0 4883ec28        sub     rsp, 28h
+   00000000`772aa1d4 65488b042520000000 mov   rax,qword ptr gs:[20h]
+   00000000`772aa1dd 488b90b0000000  mov     rdx,qword ptr [rax+0B0h]
+   00000000`772aa1e4 488b88b8000000  mov     rcx,qword ptr [rax+0B8h]
+   00000000`772aa1eb ffd2            call    rdx
+   00000000'772aa1ed 33c9            xor     ecx,ecx
+   00000000'772aa1ef ff151b1e0b00    call    qword ptr [kernel32!_imp__RtlExitUserThread]
+
+   Returns true if target_pc is readable and is the known fiber initialization routine.
+*/
+
+static bool
+at_fiber_init_known_exception(dcontext_t *dcontext, app_pc target_pc)
+{
+    static app_pc fiber_init_known_pc = 0;
+    int os_ver = get_os_version();
+
+    if (os_ver <= WINDOWS_VERSION_XP || target_pc == 0) {
+        /* only 2003 and Vista are known to have this problem */
+        return false;
+    }
+
+    /* check if this is the first time we got to create a fiber,
+     * and save the value as the only exception that is allowed to start with 
+     * this pattern */
+    if (fiber_init_known_pc == 0) { /* never seen before */
+        /* match first 7 bytes of mov eax/rax/rcx, fs/gs:[FIBER_DATA_TIB_OFFSET] */
+        static const byte FIBER_CODE_32[] =
+            {0x64,0xa1,0x10,0x00,0x00,0x00,0x00};
+        static const byte FIBER_CODE_rcx_64[] =
+            {0x65,0x48,0x8b,0x0c,0x25,0x20,0x00};
+        static const byte FIBER_CODE_rax_64[] =
+            {0x65,0x48,0x8b,0x04,0x25,0x20,0x00};
+        enum {
+            SUB_RSP_LENGTH = 4,
+            FIBER_SEH_LENGTH = 12
+        };
+        byte buf[sizeof(FIBER_CODE_32)+FIBER_SEH_LENGTH]; /* Vista needs extra */
+        byte *cur = buf;
+        const byte *pattern;
+        size_t pattern_size;
+        ASSERT(sizeof(FIBER_CODE_32) == sizeof(FIBER_CODE_rcx_64) &&
+               sizeof(FIBER_CODE_32) == sizeof(FIBER_CODE_rax_64));
+
+        if (!safe_read(target_pc, sizeof(buf), &buf))
+            return false; /* target not sufficiently readable */
+
+        /* For wow64 we expect to only see 32-bit kernel32 */
+        if (IF_X64_ELSE(is_wow64_process(NT_CURRENT_PROCESS), true)) {
+            pattern = FIBER_CODE_32;
+            pattern_size = sizeof(FIBER_CODE_32);
+            if (os_ver >= WINDOWS_VERSION_VISTA) {
+                /* we expect some SEH code before the instruction to match
+                 * 76cde8ca 6a0c             push    0xc
+                 * 76cde8cc 68f8e8cd76       push    0x76cde8f8
+                 * 76cde8d1 e85a8f0300       call    kernel32!_SEH_prolog4 (76d17830) */
+                if (*cur == 0x6a && *(cur+2) == 0x68 && *(cur+7) == 0xe8)
+                    cur += FIBER_SEH_LENGTH;
+                else
+                    return false; /* not a match */
+            }
+        } else {
+            if (os_ver >= WINDOWS_VERSION_VISTA) {
+                /* we expect a sub rsp first and to use rax instead of rcx
+                 * 00000000`772aa1d0 4883ec28        sub     rsp, 28h
+                 */
+                if (*cur == 0x48 && *(cur+1) == 0x83 && *(cur+2) == 0xec) {
+                    cur += SUB_RSP_LENGTH;
+                    pattern = FIBER_CODE_rax_64;
+                    pattern_size = sizeof(FIBER_CODE_rax_64);
+                } else
+                    return false; /* not a match */
+            } else {
+                pattern = FIBER_CODE_rcx_64;
+                pattern_size = sizeof(FIBER_CODE_rcx_64);
+            }
+        }
+
+        if (memcmp(cur, pattern, pattern_size) == 0) {
+            /* We have a match! Now ensure target is in kernel32.dll */
+            const char *target_module_name =
+                os_get_module_name_strdup(target_pc HEAPACCT(ACCT_OTHER));
+            if (target_module_name != NULL &&
+                check_filter("kernel32.dll", target_module_name)) {
+                /* We have a full match! */
+                SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+                fiber_init_known_pc = target_pc;
+                SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+                SYSLOG_INTERNAL_INFO("RCT: fiber matched @"PFX, fiber_init_known_pc);
+            } else {
+                ASSERT_CURIOSITY(false && "RCT: false fiber match");
+            }
+            dr_strfree(target_module_name HEAPACCT(ACCT_OTHER));
+        }
+    }
+
+    if (fiber_init_known_pc == target_pc && fiber_init_known_pc != 0) {
+        return true;
+    }
+    return false;
+}
+
+enum { 
+    MAX_SEH_TRYLEVEL = 8,
+    INSTR_PUSH_IMMED32_LENGTH = 5,
+    INSTR_PUSH_IMMED32_OPCODE = 0x68,
+};
+
+/* We allow non-standard uses of ret with SEH that we have seen on NT4
+ * in kernel32 and ntdll.  An example is a bad ret from 77f65b33 to
+ * 77f1d9eb, via a push immed that is used to skip a ret that is part
+ * of a handler:
+ *
+ *   kernel32!TlsFree:
+ *   ...
+ *   77f1d9df 68ebd9f177       push    0x77f1d9eb
+ *   77f1d9e4 ff2548c0f377 jmp dword ptr [KERNEL32!_imp__RtlReleasePebLock (77f3c048)]
+ *   77f1d9ea c3               ret
+ *   77f1d9eb 0fb645e7         movzx   eax,byte ptr [ebp-0x19]
+ *   
+ *   at top of routine we have:
+ *   KERNEL32!TlsFree:
+ *   77f1d92d 64a100000000     mov     eax,fs:[00000000]
+ *   77f1d933 55               push    ebp
+ *   77f1d934 8bec             mov     ebp,esp
+ *   77f1d936 6aff             push    0xff
+ *   77f1d938 6808d3f377       push    0x77f3d308
+ *   77f1d93d 6844b7f377       push    0x77f3b744
+ *   
+ *   and indeed those are on top of the SEH stack:
+ *   0:000> !teb
+ *   TEB at 7ffde000
+ *       ExceptionList:    12fdb4
+ *   0:000> dds 12fdb4
+ *   0012fdb4  0012fe2c
+ *   0012fdb8  77f3b744 KERNEL32!_except_handler3
+ *   0012fdbc  77f3d308 KERNEL32!ntdll_NULL_THUNK_DATA+0xebc
+ *   
+ *   and the handler is the instr after the push immed:
+ *   0:000> dds 77f3d308
+ *   77f3d308  ffffffff
+ *   77f3d30c  00000000
+ *   77f3d310  77f1d9e4 KERNEL32!TlsFree+0xb7
+ *
+ * We allow a ret to target an address that is pushed as an immediate
+ * immediately prior to the handler at the current trylevel in the scopetable
+ */
+static bool
+at_SEH_rct_exception(dcontext_t *dcontext, app_pc target_pc)
+{
+    TEB *teb = get_own_teb();
+    vc_exception_registration_t *vcex;
+    scopetable_entry_t *ste;
+    int trylevel;
+    app_pc pc, modbase;
+    bool result = false;
+    /* first, we only allow this in a text section */
+    modbase = get_module_base(target_pc);
+    if (modbase == NULL ||
+        !is_in_code_section(modbase, target_pc, NULL, NULL)) 
+        return false;
+    /* now read SEH data structs, being careful not to fault */
+    if (!is_readable_without_exception((app_pc)teb->ExceptionList,
+                                       /* make sure can read extra fields */
+                                       sizeof(vc_exception_registration_t)))
+        return false;
+    vcex = (vc_exception_registration_t *) teb->ExceptionList;
+    trylevel = vcex->trylevel;
+    /* sanity check: array offset by -1, don't go too far */
+    if (trylevel < -1 || trylevel > MAX_SEH_TRYLEVEL)
+        return false;
+    /* be even more careful: may not be compiled by VC! */
+    if (!is_readable_without_exception((app_pc)vcex->scopetable,
+                                       (1+trylevel)*sizeof(scopetable_entry_t)))
+        return false;
+    ste = (scopetable_entry_t *) vcex->scopetable;
+    /* -1 becomes 0 */
+    ste += (trylevel + 1);
+    pc = (app_pc) ste->lpfnHandler;
+    if (!is_readable_without_exception(pc - INSTR_PUSH_IMMED32_LENGTH,
+                                       INSTR_PUSH_IMMED32_LENGTH))
+        return false;
+    LOG(GLOBAL, LOG_INTERP, 3,
+        "RCT: at_SEH_rct_exception: testing "PFX" for push $"PFX"\n", 
+        pc - INSTR_PUSH_IMMED32_LENGTH, target_pc);
+    /* not worth risk of decoding -- we check raw bytes */
+    if (*(pc - INSTR_PUSH_IMMED32_LENGTH) == INSTR_PUSH_IMMED32_OPCODE &&
+        *((app_pc *)(pc - INSTR_PUSH_IMMED32_LENGTH + 1)) == target_pc) {
+        STATS_INC(ret_after_call_SEH);
+        SYSLOG_INTERNAL_INFO_ONCE("RCT: SEH matched @"PFX, target_pc);
+        ASSERT_CURIOSITY(ste->previousTryLevel == (DWORD)trylevel);
+        return true;
+    }
+    return false;
+}
+
+/* Whether we've seen any Borland SEH constructs or not, set by
+ * -process_SEH_push in interp.c and used to enable
+ * at_Borland_SEH_rct_exemption() which covers the rct exemptions not covered
+ * already in interp.c.  We could make this tighter and keep track in which
+ * modules we've seen Borland SEH constructs and only allow the additional
+ * exemption in those, FIXME prob. overkill. */
+bool seen_Borland_SEH = false;
+
+/* The interp.c -process_SEH_push -borland_SEH_rct processing covers 99.9% of
+ * the Borland SEH rct problems I've seen.  See notes there for more general
+ * explanation of the Borland SEH constructs.  However, I've seen (at least
+ * in one spot in one dll) an optimization that isn't covered by watching the
+ * frame pushes (you'd think with all the clever, and problematic for us,
+ * optimization done, they'd do something about screwing up the rsb predictor
+ * on the hot [fall through] path of try finally blocks, oh well).  In what
+ * looks like part of the exception handler itself, there appears to be an
+ * optimization that rather than pop then push an SEH frame, it modifies the
+ * SEH frame directly on the stack (I've only seen it do the top frame, but
+ * perhaps it sometimes needs to do a deeper frame in which case this would
+ * make more sense).  The x: y: pattern (see interp.c) still holds so we can
+ * match on that (even though isn't too much to match on) and use
+ * seen_Borland_SEH to limit when we open this hole. (Note that I saw this
+ * exemption in spybot, but only with the case 8123 bug, hopefully not missing
+ * any other rarely hit corner cases)
+ */
+static bool
+at_Borland_SEH_rct_exemption(dcontext_t *dcontext, app_pc target_pc)
+{
+    /* See pattern in interp.c -borland_SEH_rct (try except ind branch to y).
+     * If we've seen_Borland_SEH already and target_pc is in a module code
+     * section and target_pc - JMP_LONG_LENGTH is on the .E/.F table already
+     * and is a jmp rel32 whose target is in a code section of the same module
+     * then allow.  FIXME - if we can reliably recreate the src cti could also
+     * check that it is in a code section of the same module.  FIXME - if we
+     * give up the seen_Borland_SEH flag restriction this could cover all try
+     * except Borland SEH rct violations (if we feel the above checks are
+     * strong enough standalone), however we can't cover the try finally
+     * exemptions here unless we can accurately recreate the src cti to have a
+     * starting point for pattern matching.  There are some impediments in our
+     * system to being able to accurately get the src cti at this point, though
+     * if we do that may be preferable to the process_SEH code in interp.c
+     * (lazy  rather then upfront processing and less impact to non-Borland
+     * apps).  Is also not clear how we could reactively cover the call to a:
+     * case in the try finally construct as there is very little to match there
+     * (target follows a push imm of an in module addr).  Still I've never
+     * actually triggered a call to a (though I've seen it compute the address.
+     * FIXME revist handling the Borland SEH execmptions reactively
+     * instead of in interp.
+     */
+    app_pc base, jmp_target, jmp_loc = target_pc - JMP_LONG_LENGTH;
+    byte buf[JMP_LONG_LENGTH];
+
+    if (!seen_Borland_SEH || 
+        /* see above this routine is only for a certain .E/.F violation */
+        (DYNAMO_OPTION(rct_ind_jump) == OPTION_DISABLED &&
+         DYNAMO_OPTION(rct_ind_call) == OPTION_DISABLED)) {
+        return false;
+    }
+
+    base = get_module_base(target_pc);
+    if (base != NULL && 
+        /* even without rct_analyze_at_load should have processed this
+         * module by now (before we hit the exemption code) */
+        rct_ind_branch_target_lookup(dcontext, jmp_loc) != NULL &&
+        is_in_code_section(base, target_pc, NULL, NULL) &&
+        safe_read(jmp_loc, sizeof(buf), &buf) &&
+        is_jmp_rel32(buf, jmp_loc, &jmp_target) &&
+        /* perf opt, we use get_allocation_base instead of get_module_base
+         * since already checking if matches a known module base (base) */
+        get_allocation_base(jmp_target) == base &&
+        is_in_code_section(base, jmp_target, NULL, NULL)) {
+        /* we have a match */
+        return true;
+    }
+    return false;
+}
+
+static bool
+at_rct_exempt_module(dcontext_t *dcontext, app_pc target_pc, app_pc source_fragment)
+{
+    const char *target_module_name;
+    const char *source_module_name;
+    list_default_or_append_t onlist;
+    os_get_module_info_lock();
+    os_get_module_name(target_pc, &target_module_name);
+    os_get_module_name(source_fragment, &source_module_name);
+
+    LOG(THREAD, LOG_INTERP, 2, "at_rct_exempt_module: target_pc="PFX" module_name=%s\n",
+        target_pc, target_module_name != NULL ? target_module_name : "<none>");
+
+    if (source_module_name != NULL &&
+        (!IS_STRING_OPTION_EMPTY(exempt_rct_list) ||
+         !IS_STRING_OPTION_EMPTY(exempt_rct_default_list))) {
+        /* note check_list_default_and_append will grab string_option_read_lock */
+        onlist = check_list_default_and_append(dynamo_options.exempt_rct_default_list,
+                                               dynamo_options.exempt_rct_list,
+                                               source_module_name);
+        if (onlist != LIST_NO_MATCH) {
+            LOG(THREAD, LOG_INTERP, 1,
+                "at_rct_exempt_module: source_fragment="PFX" same=%d is_dyngen=%d\n", 
+                source_fragment, in_same_module(target_pc, source_fragment),
+                is_dyngen_code(target_pc));
+            if (in_same_module(target_pc, source_fragment) || is_dyngen_code(target_pc)) {
+                LOG(THREAD, LOG_INTERP, 1,
+                    "RCT: exception in exempt module %s --ok\n", source_module_name);
+                STATS_INC(ret_after_call_exempt_exceptions); /* also counted as known */
+                os_get_module_info_unlock();
+                if (onlist == LIST_ON_APPEND) /* case 9799: not if on default */
+                    mark_module_exempted(target_pc);
+                return true;
+            }
+        }
+    }
+
+    if (target_module_name != NULL &&
+        (!IS_STRING_OPTION_EMPTY(exempt_rct_to_default_list) ||
+         !IS_STRING_OPTION_EMPTY(exempt_rct_to_list) ||
+         !moduledb_exempt_list_empty(MODULEDB_EXEMPT_RCT))) {
+        /* note check_list_default_and_append will grab string_option_read_lock */
+        onlist = check_list_default_and_append(dynamo_options.exempt_rct_to_default_list,
+                                               dynamo_options.exempt_rct_to_list,
+                                               target_module_name);
+        if (onlist != LIST_NO_MATCH) {
+            LOG(THREAD, LOG_INTERP, 1,
+                "RCT: exception to exempt target module %s --ok\n", target_module_name);
+            STATS_INC(ret_after_call_exempt_exceptions); /* also counted as known */
+            os_get_module_info_unlock();
+            if (onlist == LIST_ON_APPEND) /* case 9799: not if on default */
+                mark_module_exempted(target_pc);
+            return true;
+        } else if (!moduledb_exempt_list_empty(MODULEDB_EXEMPT_RCT) &&
+                   moduledb_check_exempt_list(MODULEDB_EXEMPT_RCT, target_module_name)) {
+            LOG(THREAD, LOG_MODULEDB|LOG_INTERP, 1,
+                "RCT: exemption for moduledb exempted target module %s --ok\n",
+                target_module_name);
+            STATS_INC(num_rct_moduledb_exempt);
+            moduledb_report_exemption("Moduledb rct exemption from "PFX" to "PFX" in %s",
+                                      target_pc, source_fragment, target_module_name);
+            os_get_module_info_unlock();
+            /* case 9799: do not mark as exempted for default-on option */
+            return true;
+        }
+    }
+
+    if (source_module_name != NULL &&
+        (!IS_STRING_OPTION_EMPTY(exempt_rct_from_default_list) ||
+         !IS_STRING_OPTION_EMPTY(exempt_rct_from_list))) {
+        LOG(THREAD, LOG_INTERP, 2,
+            "at_rct_exempt_module: source_fragment="PFX" module_name=%s\n", 
+            source_fragment, source_module_name != NULL ? source_module_name : "<none>");
+        /* note check_list_default_and_append will grab string_option_read_lock */
+        if (source_module_name != NULL &&
+            (onlist =
+             check_list_default_and_append(dynamo_options.exempt_rct_from_default_list,
+                                           dynamo_options.exempt_rct_from_list,
+                                           source_module_name)) != LIST_NO_MATCH) {
+            LOG(THREAD, LOG_INTERP, 1,
+                "RCT: exception from exempt source module %s --ok\n",
+                source_module_name);
+            STATS_INC(ret_after_call_exempt_exceptions); /* also counted as known */
+            os_get_module_info_unlock();
+            if (onlist == LIST_ON_APPEND) /* case 9799: not if on default */
+                mark_module_exempted(target_pc);
+            return true;
+        }
+    }
+
+    os_get_module_info_unlock();
+    return false;
+}
+
+/* FIXME - this currently used for both .C and .E/.F violations, but almost all
+ * the specific exemptions are particular to one or the other.  We should
+ * really split this routine. Xref case 8170.
+ */
+bool
+at_known_exception(dcontext_t *dcontext, app_pc target_pc, app_pc source_fragment)
+{
+    /* check for known exception with fibers on Windows2003 */
+    if (DYNAMO_OPTION(fiber_rct) &&
+        at_fiber_init_known_exception(dcontext, target_pc)) {
+        LOG(THREAD, LOG_INTERP, 1, "RCT: known exception on fiber init --ok\n");
+        return true;
+    }
+
+    /* check for known exception with SEH on Windows NT4 */
+    if (DYNAMO_OPTION(seh_rct) &&
+        at_SEH_rct_exception(dcontext, target_pc)) {
+        LOG(THREAD, LOG_INTERP, 1, "RCT: known exception on SEH target --ok\n");
+        return true;
+    }
+
+    /* check for additional Borland SEH exemptions */
+    if (DYNAMO_OPTION(borland_SEH_rct) &&
+        at_Borland_SEH_rct_exemption(dcontext, target_pc)) {
+        LOG(THREAD, LOG_INTERP, 1, "RCT: at known Borland exception --ok\n");
+        STATS_INC(num_borland_SEH_modified);
+        return true;
+    }
+
+    if (DYNAMO_OPTION(xdata_rct) &&
+        at_xdata_rct_exception(dcontext, target_pc)) {
+        LOG(THREAD, LOG_INTERP, 1, "RCT: known exception on .xdata target --ok\n");
+        return true;
+    }
+
+    /* check whether withing an exempt module or targeting DGC from a known module */
+    if (DYNAMO_OPTION(exempt_rct) && 
+        at_rct_exempt_module(dcontext, target_pc, source_fragment)) {
+        DODEBUG({
+            const char *name;
+            os_get_module_info_lock();
+            os_get_module_name(target_pc, &name);
+            SYSLOG_INTERNAL_WARNING_ONCE("RCT: target_pc "PFX" exempt in module %s", 
+                                         target_pc, name == NULL ? "<null>" : name);
+            os_get_module_info_unlock();
+        });
+        return true;
+    }
+
+    if (DYNAMO_OPTION(driver_rct) &&
+        at_driver_rct_exception(dcontext, source_fragment)) {
+        /* FIXME: we need to ensure that we do not form traces, so that the
+         * fragile source tag is still from an exit from the driver */
+        LOG(THREAD, LOG_INTERP, 1, "RCT: known exception from driver area --ok\n");
+
+        return true;
+    }
+
+    return false;
+}
+
+#endif /* RETURN_AFTER_CALL */
+
+void
+callback_init()
+{
+}
+
+void
+callback_exit()
+{
+    DELETE_LOCK(emulate_write_lock);
+    DELETE_LOCK(map_intercept_pc_lock);
+#ifdef STACK_GUARD_PAGE
+    DELETE_LOCK(exception_stack_lock);
+#endif
+}
+
+dr_marker_t*
+get_drmarker(void)
+{
+    return (dr_marker_t*)interception_code;
+}
+
+#ifdef HOT_PATCHING_INTERFACE
+/* This function provides an interface to hook any instruction in a loaded
+ * module.  For now, the consumer is hotp_only.
+ */
+byte *
+hook_text(byte *hook_code_buf, const app_pc image_addr, 
+          intercept_function_t hook_func, const void *callee_arg, 
+          const after_intercept_action_t action_after,
+          const bool abort_if_hooked, const bool ignore_cti,
+          byte **app_code_copy_p, byte **alt_exit_cti_p)
+{
+    byte *res;
+    ASSERT(DYNAMO_OPTION(hotp_only));
+    ASSERT(hook_code_buf != NULL && image_addr != NULL && hook_func != NULL);
+
+    /* Currently hotp_only is the only user for this.  However if any other 
+     * module wants to use this, we better find out if they are trying to hook 
+     * something other than the .text section; note, it will still hook if it 
+     * isn't the text section. 
+     */ 
+    ASSERT_CURIOSITY(is_in_code_section(get_module_base(image_addr), image_addr,
+                                        NULL, NULL));
+
+    res = intercept_call(hook_code_buf, image_addr, hook_func, (void *) callee_arg,
+                          /* use dr stack now, later on hotp stack - FIXME */
+                          false, action_after,
+                          abort_if_hooked, ignore_cti,
+                          app_code_copy_p, alt_exit_cti_p);
+
+    /* Hooking can only fail if there was a cti at the patch region.  There 
+     * better not be any there!
+     */
+    ASSERT(res != NULL);
+
+    /* If app_code_copy_p isn't null, then *app_code_copy_p can't be null;
+     * same for alt_exit_cti_p if after_action is dynamic decision.
+     */
+    ASSERT(app_code_copy_p == NULL || *app_code_copy_p != NULL);
+    ASSERT(action_after != AFTER_INTERCEPT_DYNAMIC_DECISION ||
+           alt_exit_cti_p == NULL || *app_code_copy_p != NULL);
+    return res;
+}
+
+/* Just a wrapper to export unhook_text; may evolve in future. */
+void
+unhook_text(byte *hook_code_buf, app_pc image_addr)
+{
+    un_intercept_call(hook_code_buf, image_addr);
+}
+
+/* Introduced as part of fix for case 9593, which required leaking trampolines. */
+void
+insert_jmp_at_tramp_entry(byte *trampoline, byte *target)
+{
+    ASSERT(trampoline != NULL && target != NULL);
+
+    /* Note: first 5 bytes of the trampoline contain the copy of app code which
+     * was overwritten with the hook;  so, entry point is 5 bytes after that.
+     */
+    *(trampoline + 5) = JMP_REL32_OPCODE;
+    patch_branch(trampoline + 5, target, false /* Don't have to hot_patch. */);
+}
+#endif
+
+#ifdef X86
+/* Returns POINTER_MAX on failure.
+ * Assumes that cs, ss, ds, and es are flat.
+ */
+byte *
+get_segment_base(uint seg)
+{
+    if (seg == SEG_TLS)
+        return (byte *) get_own_teb();
+    else if (seg == SEG_CS || seg == SEG_SS || seg == SEG_DS || seg == SEG_ES)
+        return NULL;
+    else
+        return (byte *) POINTER_MAX;
+}
+#endif
+
diff --git a/core/win32/diagnost.c b/core/win32/diagnost.c
new file mode 100644
index 000000000..6c649a41a
--- /dev/null
+++ b/core/win32/diagnost.c
@@ -0,0 +1,1521 @@
+/* **********************************************************
+ * Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+
+/* diagnost.c - maintains information about modules (dll or executable images) */
+
+#include "globals.h"
+
+#include "ntdll.h"
+#include "fragment.h"
+#include "link.h"
+#include "utils.h"
+#include "diagnost.h"
+#include "os_private.h"
+#include "hotpatch.h"
+#include "moduledb.h"
+#include "module_shared.h"
+
+/* Declared globally to reduce recursion overhead */
+/* Not persistent across code cache execution, so not protected */
+DECLARE_NEVERPROT_VAR(static DIAGNOSTICS_KEY_NAME_INFORMATION diagnostic_key_info, {0});
+DECLARE_NEVERPROT_VAR(static DIAGNOSTICS_KEY_VALUE_FULL_INFORMATION diagnostic_value_info,
+                      {0});
+DECLARE_NEVERPROT_VAR(static char keyinfo_name[DIAGNOSTICS_MAX_KEY_NAME_SIZE], {0});
+DECLARE_NEVERPROT_VAR(static char keyinfo_data[DIAGNOSTICS_MAX_NAME_AND_DATA_SIZE], {0});
+DECLARE_NEVERPROT_VAR(static wchar_t diagnostic_keyname[DIAGNOSTICS_MAX_KEY_NAME_SIZE], {0});
+
+/* Enforces unique access to shared reg data structures */
+DECLARE_CXTSWPROT_VAR(static mutex_t reg_mutex, INIT_LOCK_FREE(diagnost_reg_mutex));
+
+static const char * const separator =
+"-----------------------------------------------------------------------\n";
+
+/* FIXME: The following is a list of relevant registry key entries as
+ * reported by autoruns-8.53.  Since autorunsc -a only shows non-empty
+ * keys, I've compiled this list by aggregating the output on several
+ * different machines. Some keys may be missing...
+ */
+static const wchar_t * const HKLM_entries[] = {
+    L"\\Registry\\Machine\\SOFTWARE\\Classes\\Folder\\Shellex\\ColumnHandlers",
+    L"\\Registry\\Machine\\SOFTWARE\\Classes\\Protocols\\Filter",
+    L"\\Registry\\Machine\\SOFTWARE\\Classes\\Protocols\\Handler",
+    L"\\Registry\\Machine\\SOFTWARE\\Microsoft\\Active Setup\\Installed Components",
+    L"\\Registry\\Machine\\SOFTWARE\\Microsoft\\Internet Explorer\\Extensions",
+    L"\\Registry\\Machine\\SOFTWARE\\Microsoft\\Internet Explorer\\Toolbar",
+    L"\\Registry\\Machine\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows\\Appinit_Dlls",
+    L"\\Registry\\Machine\\SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Winlogon",
+    L"\\Registry\\Machine\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\Browser Helper Objects",
+    L"\\Registry\\Machine\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\SharedTaskScheduler",
+    L"\\Registry\\Machine\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Explorer\\ShellExecuteHooks",
+    L"\\Registry\\Machine\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer\\Run",
+    L"\\Registry\\Machine\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System",
+    L"\\Registry\\Machine\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Run",
+    L"\\Registry\\Machine\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\RunOnce",
+    L"\\Registry\\Machine\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\RunOnceEx",
+    L"\\Registry\\Machine\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Shell Extensions\\Approved",
+    L"\\Registry\\Machine\\SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\ShellServiceObjectDelayLoad",
+    L"\\Registry\\Machine\\SOFTWARE\\Policies\\Microsoft\\Windows\\System\\Scripts\\Logon",
+    L"\\Registry\\Machine\\SYSTEM\\CurrentControlSet\\Control\\Lsa",
+    L"\\Registry\\Machine\\SYSTEM\\CurrentControlSet\\Control\\Print\\Monitors",
+    L"\\Registry\\Machine\\SYSTEM\\CurrentControlSet\\Control\\Session Manager",
+    L"\\Registry\\Machine\\SYSTEM\\CurrentControlSet\\Control\\Terminal Server\\Wds\\rdpwd",
+    L"\\Registry\\Machine\\SYSTEM\\CurrentControlSet\\Services",
+};
+
+/* These are prefixed with \\Registry\\User\\<SSID>\\ in 
+ * report_autostart_programs()
+ */
+static const wchar_t * const HKCU_entries[] = {
+    L"Control Panel\\Desktop",
+    L"SOFTWARE\\Microsoft\\Internet Explorer\\Desktop\\Components",
+    L"SOFTWARE\\Microsoft\\Internet Explorer\\UrlSearchHooks",
+    L"SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Winlogon",
+    L"SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows\\Load",
+    L"SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows\\Run",
+    L"SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\Explorer\\Run",
+    L"SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Policies\\System\\Shell",
+    L"SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\Run",
+    L"SOFTWARE\\Microsoft\\Windows\\CurrentVersion\\RunOnce",
+    L"SOFTWARE\\Policies\\Microsoft\\Windows\\System\\Scripts\\Logon",
+};
+
+/* Extracts and logs the data field of DIAGNOSTIC_KEY_VALUE_FULL_INFORMATION.
+ * Only REG_SZ, REG_EXPAND_SZ (w/o expanding), REG_MULTI_SZ (just the first 
+ * string reported), REG_DWORD and REG_BINARY are currently supported.
+ */
+static void 
+diagnostics_log_data(IN file_t diagnostics_file, IN uint log_mask)
+{
+    uint i;
+
+    ASSERT_OWN_MUTEX(true, &reg_mutex);
+    if (log_mask & DIAGNOSTICS_REG_NAME) {
+        /* there is no trailing null after name so don't print to far */
+        snprintf(keyinfo_name, BUFFER_SIZE_ELEMENTS(keyinfo_name), "%.*S\n",
+                 diagnostic_value_info.NameLength / sizeof(wchar_t),
+                 (wchar_t *)&diagnostic_value_info.NameAndData);
+        NULL_TERMINATE_BUFFER(keyinfo_name);
+        print_xml_cdata(diagnostics_file, keyinfo_name);
+    }
+
+    if (log_mask & DIAGNOSTICS_REG_DATA) {
+        if ((diagnostic_value_info.Type == REG_SZ) ||
+            (diagnostic_value_info.Type == REG_EXPAND_SZ) ||
+            (diagnostic_value_info.Type == REG_MULTI_SZ)) {
+            snprintf(keyinfo_data, BUFFER_SIZE_ELEMENTS(keyinfo_data), "%S\n",
+                     diagnostic_value_info.NameAndData  + 
+                     diagnostic_value_info.DataOffset - 
+                     DECREMENT_FOR_DATA_OFFSET);
+            NULL_TERMINATE_BUFFER(keyinfo_data);
+            print_xml_cdata(diagnostics_file, keyinfo_data);
+        }
+        else if (diagnostic_value_info.Type == REG_DWORD) {
+            print_file(diagnostics_file, "0x%.8x\n", 
+                       *(uint *) (diagnostic_value_info.NameAndData  + 
+                       diagnostic_value_info.DataOffset - 
+                       DECREMENT_FOR_DATA_OFFSET));
+        }
+        else if (diagnostic_value_info.Type == REG_BINARY) {
+            for (i = diagnostic_value_info.DataOffset - DECREMENT_FOR_DATA_OFFSET;
+                 (i < diagnostic_value_info.DataOffset - DECREMENT_FOR_DATA_OFFSET +
+                      diagnostic_value_info.DataLength) && 
+                 (i < DIAGNOSTICS_MAX_NAME_AND_DATA_SIZE );
+                 i++) {
+                    print_file(diagnostics_file, "%.2x ", 
+                        diagnostic_value_info.NameAndData[i]);
+                    if ((i % DIAGNOSTICS_BYTES_PER_LINE)== 0)
+                        print_file(diagnostics_file, "\n");
+
+             }
+             print_file(diagnostics_file, "\n");
+        }
+
+    }
+}
+
+/* Determines the location of the logging directory and create a new log file
+ * that is sequentially higher than the previous log file.  The logging 
+ * directory is obtained from the produt settings in the registry, and the
+ * file name is obtained by opening existing files until one is not found.
+ */
+static void
+open_diagnostics_file(IN file_t *file, OUT char *buf, IN uint maxlen)
+{
+    const char *file_extension = DIAGNOSTICS_FILE_XML_EXTENSION;
+    get_unique_logfile(file_extension, buf, maxlen, false, file);
+}
+
+/* flags */
+#define PRINT_MEM_BUF_BYTE     0x1 /* print as byte values, default dword */
+
+#define PRINT_MEM_BUF_START    0x2 /* print region starting at address */
+    /* default is to print region centered at address */
+
+#define PRINT_MEM_BUF_NO_ALIGN 0x4 /* print exact region */
+    /* default extends to nice alignments */
+
+#define PRINT_MEM_BUF_ASCII    0x8 /* print ASCII characters for each line */
+    /* applicable to either DWORD or BYTE mode */
+
+/* Prints [-length/2, +length/2] from memory address (if readable) or, for
+ * PRINT_MEM_BUF_START, prints [0, length] from memory address (if readable) */
+static void
+print_memory_buffer(file_t diagnostics_file, byte *address, uint length,
+                    const char *label, uint flags)
+{
+    byte *start, *end;
+   
+    if (TEST(PRINT_MEM_BUF_START, flags)) {
+        start = address;
+        end = address + length;
+    } else {
+        start = address - length/2;
+        end = address + length/2;
+    }
+
+    if (!TEST(PRINT_MEM_BUF_NO_ALIGN, flags)) {
+        /* Align macros require power of 2 */
+        ASSERT((DUMP_PER_LINE_DEFAULT & (DUMP_PER_LINE_DEFAULT - 1)) == 0);
+        start = (byte *)ALIGN_BACKWARD(start, DUMP_PER_LINE_DEFAULT);
+        end = (byte *)ALIGN_FORWARD(end, DUMP_PER_LINE_DEFAULT);
+    }
+
+    print_file(diagnostics_file, "%s: 0x%.8x\n", label, address);
+    print_file(diagnostics_file, "<![CDATA[\n");
+    while (start < end) {
+        byte *cur_end = (byte *)MIN(ALIGN_FORWARD(start+1, PAGE_SIZE), (ptr_uint_t)end);
+        if (is_readable_without_exception(start, cur_end - start)) {
+            dump_buffer_as_bytes(diagnostics_file, start, cur_end - start,
+                                 DUMP_RAW|DUMP_ADDRESS
+                                 |(TEST(PRINT_MEM_BUF_BYTE, flags) ? 0 : DUMP_DWORD)
+                                 |(TEST(PRINT_MEM_BUF_ASCII, flags) ? DUMP_APPEND_ASCII : 0)
+                                 );
+            print_file(diagnostics_file, "\n");
+        } else {
+            print_file(diagnostics_file, 
+                       "Can\'t print 0x%.8x-0x%.8x (unreadable)\n", start, cur_end);
+        }
+        start = cur_end;
+    }
+    print_file(diagnostics_file, "]]>\n");
+}
+
+static void
+report_addr_info(file_t diagnostics_file, app_pc addr, const char *tag)
+{
+    /* FIXME: Add closest exported function from Vlad's code, when ready */
+    /* This is only used for violations so safe to allocate memory, and
+     * we need the full name so we can't just stick w/ the buffer
+     */
+    char modname_buf[MAX_MODNAME_INTERNAL];
+    const char *mod_name =
+        os_get_module_name_buf_strdup(addr, modname_buf,
+                                      BUFFER_SIZE_ELEMENTS(modname_buf)
+                                      HEAPACCT(ACCT_OTHER));
+    print_file(diagnostics_file,
+               "\t\taddress=             \"0x%.8x\"\n"
+               "\t\tmodule=              \"%s\"\n"
+               "\t\tin_IAT=              \"%s\"\n"
+               "\t\tpreferred_base=      \""PFX"\"\n",
+               addr,
+               mod_name == NULL ? "(none)" : mod_name, 
+               is_in_IAT(addr) ? "yes" : "no",
+               get_module_preferred_base(addr));
+    if (mod_name != NULL && mod_name != modname_buf)
+        dr_strfree(mod_name HEAPACCT(ACCT_OTHER));
+    mod_name = NULL;
+
+    print_module_section_info(diagnostics_file, addr);
+
+    /* Dump memory permission and region information */
+    dump_mbi_addr(diagnostics_file, addr, DUMP_XML);
+    
+    /* Also dump 1 page around address */
+    print_file(diagnostics_file, "\t\t><content>\n");
+    print_memory_buffer(diagnostics_file, addr, PAGE_SIZE, tag,
+                        PRINT_MEM_BUF_BYTE | PRINT_MEM_BUF_ASCII);
+    print_file(diagnostics_file, "\t\t</content>\n");
+}
+
+static void
+report_src_info(file_t diagnostics_file, dcontext_t *dcontext)
+{
+    const char *name = "source";
+    fragment_t *f = dcontext->last_fragment;
+
+    /* Note: last_fragment may span across different dlls because 
+     * DR's basic blocks span across unconditional direct branches.  If
+     * recreate_app_pc() isn't used, current module may be wrong. Case 2152.
+     * Make sure to check for LINKSTUB_FAKE() before recreating.
+     * Note that last_exit can be NULL if DR is still initializing -- but if
+     * we crash then we should not come here (will print conservative info only).
+     */
+    ASSERT(dcontext->last_exit != NULL);
+
+    print_file(diagnostics_file, "\t<%s-properties \n", name);
+     
+    report_addr_info(diagnostics_file, f->tag, name); 
+
+    /* Dump src fragment information: flags, bb_tags, cache dump etc. */
+    print_file(diagnostics_file, "\t\t<cache-content\n"
+               "\t\t\tflags= \"0x%0x\"",
+               f->flags);
+    if (TEST(FRAG_IS_TRACE, f->flags)) {
+        uint i = 0;
+        trace_only_t *t = TRACE_FIELDS(f);
+        if (t != NULL) {
+            print_file(diagnostics_file, "\n\t\t\ttags=  \"");
+            /* FIXME: all app tags are printed in one line, if it proves to
+             * create unreadably long lines, change this
+             */
+            for(i = 0; i < t->num_bbs; i++)
+                print_file(diagnostics_file, PFX" ", t->bbs[i].tag);
+            print_file(diagnostics_file, "\"");
+        } else
+            ASSERT_CURIOSITY(false && "frag is trace, but no trace specific data?");
+    }
+    print_file(diagnostics_file, ">\n");
+
+    /* For fake link stubs start_pc is null, for all other cases dump the
+     * bb/trace
+     */
+    if (!LINKSTUB_FAKE(dcontext->last_exit)) {
+        /* Print as raw bytes, just to be obscure and to not allocate any
+         * memory (decoding does), from last_fragment start_pc to size
+         */
+        print_memory_buffer(diagnostics_file,
+                            dcontext->last_fragment->start_pc,
+                            dcontext->last_fragment->size,
+                            "pc",
+                            PRINT_MEM_BUF_BYTE  |
+                            PRINT_MEM_BUF_START |
+                            PRINT_MEM_BUF_NO_ALIGN);
+    }
+
+    print_file(diagnostics_file, "\t\t</cache-content>\n\t</%s-properties>\n",
+               name);
+}
+
+static void
+report_target_info(file_t diagnostics_file, dcontext_t *dcontext)
+{
+    const char *name = "target";
+    print_file(diagnostics_file, "\t<%s-properties \n", name);
+    report_addr_info(diagnostics_file, dcontext->next_tag, name);
+    print_file(diagnostics_file, "\t</%s-properties>\n", name);
+}
+
+/* applies to any violation though in most cases expected to provide
+ * extra information for covering RCT (.C .E .F) or .R failures.
+ * (Note we don't report on other preferred targets in DLLs
+ * rebased due to other conflicts)
+ */
+static void
+report_preferred_target_info(file_t diagnostics_file, dcontext_t *dcontext)
+{
+    const char *name = "preferred-target";
+    app_pc aslr_preferred_address =     
+        aslr_possible_preferred_address(dcontext->next_tag);
+
+    /* no report if we don't have a preferred address */
+    if (aslr_preferred_address == NULL) {
+        return;
+    }
+
+    print_file(diagnostics_file, "\t<%s-properties \n", name);
+    report_addr_info(diagnostics_file, aslr_preferred_address, name);
+    print_file(diagnostics_file, "\t</%s-properties>\n", name);
+}
+
+static void
+report_vm_counters(file_t diagnostics_file, VM_COUNTERS *vmc)
+{
+    print_file(diagnostics_file,
+               "<vm-counters>\n"
+               "%.10d %.10d %.10d %.10d %.10d %.10d %.10d\n"
+               "%.10d %.10d %.10d %.10d\n"
+               "</vm-counters>\n",
+               vmc->PeakVirtualSize,
+               vmc->VirtualSize,
+               vmc->PageFaultCount,
+               vmc->PeakWorkingSetSize,
+               vmc->WorkingSetSize,
+               vmc->QuotaPeakPagedPoolUsage,
+               vmc->QuotaPagedPoolUsage,
+               vmc->QuotaPeakNonPagedPoolUsage,
+               vmc->QuotaNonPagedPoolUsage,
+               vmc->PagefileUsage,
+               vmc->PeakPagefileUsage);
+}
+
+/* Prints out dcontext information.  If the dcontext is from the current thread,
+ * additional information is reported.  
+ * The conservative flag indicates we may have come here from a crash.  Print
+ * information that do not need any allocations etc.
+ */
+static void
+report_dcontext_info(IN file_t diagnostics_file, IN dcontext_t *dcontext,
+                     IN bool conservative)
+{
+
+    byte *start_ptr = NULL;
+
+    if (dcontext == NULL) {     /* see case 8830 - dcontext can be NULL! */
+        print_file(diagnostics_file, "\tNo thread specific data available\n");
+        return;
+    }
+
+    print_file(diagnostics_file, 
+               "\t<whereami> %d </whereami>\n", dcontext->whereami);
+    dump_mcontext(get_mcontext(dcontext), diagnostics_file, DUMP_XML);
+    dump_callstack(NULL, (app_pc)get_mcontext(dcontext)->xbp, diagnostics_file,
+                   DUMP_XML);
+
+    if (dcontext == get_thread_private_dcontext()) {
+        if (!conservative) {
+            /* Print out additional current thread information */
+            report_src_info(diagnostics_file, dcontext);
+            report_target_info(diagnostics_file, dcontext);
+            /* We print both really targeted address and the contents of the
+             * module potentially targeted by an attack thwarted by ASLR.
+             */
+            report_preferred_target_info(diagnostics_file, dcontext);
+        }
+
+        /* Dump 1 page before/after ESP, making no assumptions about EBP. */
+        /* Verify beginning and end of region (which span at most 2 pages) */
+        print_file(diagnostics_file, "\t<stack>\n\t\t<content>\n",
+                   (byte *) get_mcontext(dcontext)->xsp);
+        print_memory_buffer(diagnostics_file, 
+                            (byte *) get_mcontext(dcontext)->xsp, 
+                            PAGE_SIZE, 
+                            "Current Stack",
+                             PRINT_MEM_BUF_ASCII);
+        print_file(diagnostics_file, "\t\t</content>\n\t</stack>\n");
+    }
+    else {
+        /* Dump a mini-stack */
+        print_file(diagnostics_file,
+                   "\t<stack>\n\t\t<content>\n",
+                   (byte *) get_mcontext(dcontext)->xsp);
+        print_memory_buffer(diagnostics_file,
+                            (byte *) get_mcontext(dcontext)->xsp, 
+                            DIAGNOSTICS_MINI_DUMP_SIZE, 
+                            "Stack",
+                            PRINT_MEM_BUF_ASCII);
+        print_file(diagnostics_file, "\t\t</content>\n\t</stack>\n");
+    }
+}
+
+/* Collects and displays all DynamoRIO data structures that provide useful
+ * diagnostic information.
+ * violation_type of NO_VIOLATION_* is diagnostics; other is forensics.
+ */
+static void
+report_internal_data_structures(IN file_t diagnostics_file,
+                                IN security_violation_t violation_type)
+{
+    dcontext_t *dcontext;
+    char optstring_buf[MAX_OPTIONS_STRING];
+    /* make sure this is changed to a protected static buf if gets too long
+     * report_registry_settings has already used more than this (552 bytes
+     * per invocation & recurses up to 5 times) so no risk of overflow */
+    ASSERT(MAX_OPTIONS_STRING <= 512);
+
+    print_file(diagnostics_file, "<internal-data-structures>\n"
+               "automatic_startup  : %d\ncontrol_all_threads: %d\n" 
+               "dynamo_initialized : %d\ndynamo_exited      : %d\n"
+               "num_threads        : %d\ndynamorio.dll      = "PFX"\n",
+               automatic_startup, control_all_threads,
+               dynamo_initialized, dynamo_exited,
+               get_num_threads(), get_dynamorio_dll_start());
+
+    /* skip for non-attack calls to avoid risk of any global locks */
+    if (violation_type != NO_VIOLATION_BAD_INTERNAL_STATE) {
+        print_vmm_heap_data(diagnostics_file);
+        if (dynamo_initialized && !DYNAMO_OPTION(thin_client)) {    /* case 8830 */
+            print_file(diagnostics_file, "Exec areas:\n");
+            print_executable_areas(diagnostics_file);
+#ifdef PROGRAM_SHEPHERDING
+            print_file(diagnostics_file, "Future exec areas:\n");
+            print_futureexec_areas(diagnostics_file);
+#endif
+            print_moduledb_exempt_lists(diagnostics_file);
+        }
+    }
+    
+    print_last_deallocated(diagnostics_file);
+
+    /* case 5442: always dump dcontext */
+    dcontext = get_thread_private_dcontext();
+    if (dcontext != NULL) {
+        print_memory_buffer(diagnostics_file, (byte *)dcontext, 
+                            sizeof(dcontext_t), "current dcontext", 
+                            PRINT_MEM_BUF_START|PRINT_MEM_BUF_NO_ALIGN); 
+    }
+
+    /* Include mini-call stacks for non-attack calls */
+    if (violation_type == NO_VIOLATION_BAD_INTERNAL_STATE ||
+        violation_type == NO_VIOLATION_OK_INTERNAL_STATE) {
+        app_pc our_esp = NULL, our_ebp = NULL;
+        print_file(diagnostics_file, "\nCall stack for DR:\n");
+        dump_dr_callstack(diagnostics_file);
+        GET_STACK_PTR(our_esp);
+        GET_FRAME_PTR(our_ebp);
+        /* dump whole stack */
+        print_file(diagnostics_file, "ebp="PFX" esp="PFX"\n", 
+                   our_ebp, our_esp);
+        print_memory_buffer(diagnostics_file, (byte *)
+                            ALIGN_BACKWARD(our_esp, PAGE_SIZE), 3*PAGE_SIZE, 
+                            "DR Stack", PRINT_MEM_BUF_START);
+        /* application call stack is printed by report_dcontext_info */
+    }
+#ifdef HOT_PATCHING_INTERFACE
+    /* As long as hotp_diagnostics is turned on dump diagnostics about
+     * hotpatches.
+     */
+    if (DYNAMO_OPTION(hotp_diagnostics)) {
+        hotp_print_diagnostics(diagnostics_file);
+    }
+#endif
+
+    get_dynamo_options_string(&dynamo_options, optstring_buf,
+                              BUFFER_SIZE_ELEMENTS(optstring_buf), true);
+    NULL_TERMINATE_BUFFER(optstring_buf);
+    print_file(diagnostics_file, "option string = \"%s\"\n", optstring_buf);
+
+    DOLOG(1, LOG_ALL, {
+        uchar test_buf[UCHAR_MAX+2];
+        uint i;
+        print_file(diagnostics_file, "<debug_xml_encoding_test>\n<![CDATA[\n");
+        /* test CDATA escaping routines */
+        print_xml_cdata(diagnostics_file, "testing premature ending ]]> for cdata\n");
+        /* test encoding */
+        for (i = 0; i <= UCHAR_MAX; i++) {
+            test_buf[i] = (uchar)i;
+            if (!is_valid_xml_char((char)i)) {
+                ASSERT(i < 0x20 && i != '\n' && i != '\r' && i != '\t');
+                test_buf[i] = 'a';
+            }
+        }
+        BUFFER_LAST_ELEMENT(test_buf) = '\n';
+        os_write(diagnostics_file, test_buf, sizeof(test_buf));
+        for (test_buf[0] = 'a', i = 1; i <= UCHAR_MAX; i++)
+            test_buf[i] = (uchar)i;
+        NULL_TERMINATE_BUFFER(test_buf);
+        print_xml_cdata(diagnostics_file, (const char *)test_buf);
+        print_file(diagnostics_file, "\n]]>\n</debug_xml_encoding_test>\n");
+    });
+
+    print_file(diagnostics_file, "</internal-data-structures>\n");
+}
+
+/* Collects and displays information about NTDLL.DLL.  Finds the SystemRoot
+ * registry key & adds "System32" to it to find NTDLL.DLL.
+ */
+static void
+report_ntdll_info(IN file_t diagnostics_file)
+{
+
+    reg_query_value_result_t value_result;
+    FILE_NETWORK_OPEN_INFORMATION file_info;
+    wchar_t filename[MAXIMUM_PATH + 1];
+
+    print_file(diagnostics_file, "<ntdll-file-information><![CDATA[\n");
+
+    memset (&file_info, 0, sizeof(file_info));
+    value_result = reg_query_value(DIAGNOSTICS_OS_REG_KEY, 
+                                   DIAGNOSTICS_SYSTEMROOT_REG_KEY,
+                                   KeyValueFullInformation,
+                                   &diagnostic_value_info, 
+                                   sizeof(diagnostic_value_info), 0);    
+    if (value_result == REG_QUERY_SUCCESS) {
+        _snwprintf(filename, BUFFER_SIZE_ELEMENTS(filename), L"\\??\\%s\\%s", 
+                   (wchar_t*)(diagnostic_value_info.NameAndData  + 
+                              diagnostic_value_info.DataOffset - 
+                              DECREMENT_FOR_DATA_OFFSET),
+                    DIAGNOSTICS_NTDLL_DLL_LOCATION);
+        NULL_TERMINATE_BUFFER(filename);
+        print_file(diagnostics_file, "%S\n", filename);
+
+        if (query_full_attributes_file(filename, &file_info)) {
+            print_file(diagnostics_file, 
+                "0x%.11x%.8x 0x%.11x%.8x 0x%.11x%.8x\n"
+                "0x%.11x%.8x 0x%.11x%.8x 0x%.11x%.8x\n"
+                "0x%.8x\n",
+                file_info.CreationTime.HighPart,
+                file_info.CreationTime.LowPart,
+                file_info.LastAccessTime.HighPart,
+                file_info.LastAccessTime.LowPart,
+                file_info.LastWriteTime.HighPart,
+                file_info.LastWriteTime.LowPart,
+                file_info.ChangeTime.HighPart,
+                file_info.ChangeTime.LowPart,
+                file_info.AllocationSize.HighPart,
+                file_info.AllocationSize.LowPart,
+                file_info.EndOfFile.HighPart,
+                file_info.EndOfFile.LowPart,
+                file_info.FileAttributes);
+        }
+    }
+
+    print_file(diagnostics_file, "]]></ntdll-file-information>\n\n");
+}
+
+/* conservative flag indicates we may have come here from a crash.  Print
+ * information that do not need any allocations etc. */
+static void
+report_thread(file_t diagnostics_file, int num, thread_id_t id, dcontext_t *dcontext,
+              bool conservative)
+{
+    print_file(diagnostics_file, 
+               "\n<thread id=\"%d\" current-thread=\"%s\" num=\"%d\">\n",
+               id,
+               ((dcontext == get_thread_private_dcontext()) ? "yes" : "no"),
+               num + 1);
+    report_dcontext_info(diagnostics_file, dcontext, conservative);
+    print_file(diagnostics_file, "</thread>\n");
+}
+
+/* Displays process-specific information for the current process.
+ * The conservative flag indicates we may have come here from a crash.  Print
+ * information that does not need any allocations, etc.
+ */
+static void
+report_current_process(IN file_t diagnostics_file, IN PSYSTEM_PROCESSES sp,
+                       IN security_violation_t violation_type, IN bool conservative)
+{
+    PEB *peb = get_own_peb();
+    thread_record_t **threads;
+    int i, num_threads;
+    size_t s, buffer_length;
+    wchar_t *buffer;
+    bool couldbelinking = false;
+    bool report_thread_list = true;
+
+    print_file(diagnostics_file, "<current-process\n");
+
+    ASSERT(conservative || sp != NULL);
+
+    /* FIXME: There are several in-memory depedencies on strings that could
+       be used in an attack.  Risk should be assessed. */
+    if (conservative) {
+        print_file(diagnostics_file, "name=                    \"%s\"\n",
+                   get_application_name());        
+    } else {
+        print_file(diagnostics_file, "name=                    \"%S\"\n",
+                   sp->ProcessName.Buffer);
+    }
+    print_file(diagnostics_file, "image-path=              \"%S\"\n", 
+               peb->ProcessParameters->ImagePathName.Buffer);
+    print_file(diagnostics_file, "full-qualified-name=     \"%s\"\n",
+               get_application_name());
+    print_file(diagnostics_file, "short-qualified-name=    \"%S\"\n",
+               get_own_short_qualified_name());
+    print_file(diagnostics_file, "current-directory-path=  \"%S\"\n", 
+               peb->ProcessParameters->CurrentDirectoryPath.Buffer);        
+    if (conservative) {
+        print_file(diagnostics_file, "process-id=              \"%s\"\n",
+                   get_application_pid());
+    } else {
+        print_file(diagnostics_file, "process-id=              \"%d\"\n",
+                   sp->ProcessId);
+    }
+    print_file(diagnostics_file,
+               "being-debugged=          \"%s\"\n"
+               "image-base-address=      \"0x%.8x\"\n", 
+               peb->BeingDebugged ? "yes" : "no",
+               peb->ImageBaseAddress);
+    print_file(diagnostics_file, "shell-info=              \"%S\"\n",
+               peb->ProcessParameters->ShellInfo.Buffer);
+    /* This can be NULL sometimes */
+    print_file(diagnostics_file, "runtime-info=            \"%S\"\n",
+               peb->ProcessParameters->RuntimeData.Buffer == NULL ?
+               L"(null)" : peb->ProcessParameters->RuntimeData.Buffer);
+    print_file(diagnostics_file, "console-flags=           \"0x%.8x\"\n",
+               peb->ProcessParameters->ConsoleFlags);
+    if (!conservative) {
+        print_file(diagnostics_file,
+                   "thread-count=            \"%d\"\n"
+                   "handle-count=            \"%d\"\n"
+                   "base-priority=           \"%d\"\n"
+                   "creation-time=           \"0x%.8x%.8x\"\n"
+                   "user-time=               \"0x%.8x%.8x\"\n"
+                   "kernel-time=             \"0x%.8x%.8x\"\n",
+                   sp->ThreadCount,
+                   sp->HandleCount,
+                   sp->BasePriority,
+                   sp->CreateTime.HighPart, sp->CreateTime.LowPart,
+                   sp->UserTime.HighPart, sp->UserTime.LowPart,
+                   sp->KernelTime.HighPart, sp->KernelTime.LowPart);
+    }
+
+    /* note cmdline is sometimes already quoted and sometimes not, to avoid
+     * problems for xml we dump it as a separate CDATA tag instead of an
+     * attribute */
+    print_file(diagnostics_file,
+               "><command-line><![CDATA[ %S\n]]></command-line>\n", 
+               peb->ProcessParameters->CommandLine.Buffer);
+
+    /* DllPath can get pretty large -- splitting it up here.
+     * FIXME: Splitting buffers can be generalized fairly simply (1 for wide, 
+     * 1 for ascii) if this kind of thing happens a lot */
+    /* For xml can't be done as an additional in tag field since I've seen
+     * quotes in the dll-path string. */
+    print_file(diagnostics_file, "<dll-path><![CDATA[      ");
+    buffer = peb->ProcessParameters->DllPath.Buffer;
+    buffer_length = wcslen(peb->ProcessParameters->DllPath.Buffer);
+    /* MAX_LOG_LENGTH_MINUS_ONE allows a \0 to be appended without overflow */
+    ASSERT(MAX_LOG_LENGTH_MINUS_ONE == MAX_LOG_LENGTH - 1);
+    for (s = 0; s < buffer_length; s += MAX_LOG_LENGTH_MINUS_ONE) {
+        print_file(diagnostics_file,"%."STRINGIFY(MAX_LOG_LENGTH_MINUS_ONE)"S",
+                   buffer + s);
+    }
+    print_file(diagnostics_file, "\n]]></dll-path>\n");
+
+    if (conservative) {
+        /* Can be called while unstable, don't allocate memory for mem dynamically */
+        VM_COUNTERS mem;
+        if (get_process_mem_stats(NT_CURRENT_PROCESS, &mem))
+            report_vm_counters(diagnostics_file, &mem);
+    } else {
+        report_vm_counters(diagnostics_file, &sp->VmCounters);
+        print_file(diagnostics_file,
+                   "<io-counters>\n"
+                   "0x%.11x%.8x 0x%.11x%.8x 0x%.11x%.8x\n"
+                   "0x%.11x%.8x 0x%.11x%.8x 0x%.11x%.8x\n"
+                   "</io-counters>\n\n",
+                   sp->IoCounters.ReadOperationCount.HighPart,
+                   sp->IoCounters.ReadOperationCount.LowPart,
+                   sp->IoCounters.WriteOperationCount.HighPart,
+                   sp->IoCounters.WriteOperationCount.LowPart,
+                   sp->IoCounters.OtherOperationCount.HighPart,
+                   sp->IoCounters.OtherOperationCount.LowPart,
+                   sp->IoCounters.ReadTransferCount.HighPart,
+                   sp->IoCounters.ReadTransferCount.LowPart,
+                   sp->IoCounters.WriteTransferCount.HighPart,
+                   sp->IoCounters.WriteTransferCount.LowPart,
+                   sp->IoCounters.OtherTransferCount.HighPart,
+                   sp->IoCounters.OtherTransferCount.LowPart);
+    }
+
+    /* Print out DLL information */
+    /* FIXME: walking the loader data structures at arbitrary points is dangerous
+     * due to data races with other threads -- see is_module_being_initialized
+     * and get_module_name
+     */
+    print_modules_ldrlist_and_ourlist(diagnostics_file, DUMP_XML, conservative);
+
+    /* Print out all thread information */
+    print_file(diagnostics_file, "\n<thread-list>");
+
+    if (is_self_couldbelinking()) {
+        /* case 6093: we can 3-way deadlock w/ a flusher and a thread wanting
+         * the bb building lock if we come here holding it (.B/.A violation).
+         * FIXME: as a short-term fix we do not print the list of all threads.
+         * case 6141 covers re-enabling.
+         */
+        report_thread_list = false;
+
+        if (report_thread_list) {
+            /* cannot grab thread_initexit_lock if couldbelinking since could deadlock
+             * with a flushing thread, so we go nolinking for the thread snapshot
+             */
+            enter_nolinking(get_thread_private_dcontext(), NULL,
+                            false/*not a cache transition*/);
+        }
+        couldbelinking = true;
+    }
+#if defined(PROGRAM_SHEPHERDING) && defined(HOT_PATCHING_INTERFACE)
+    /* case 7528: hotp violations are nolinking yet hold the read lock
+     * when reporting.  our solution for now is to not list the threads,
+     * which we're already not doing for other violations (case 6093).
+     * case 6141 covers re-enabling.
+     */
+    if (violation_type == HOT_PATCH_DETECTOR_VIOLATION ||
+        violation_type == HOT_PATCH_PROTECTOR_VIOLATION ||
+        violation_type == HOT_PATCH_FAILURE)
+        report_thread_list = false;
+#endif
+#if defined(PROGRAM_SHEPHERDING) && defined(GBOP)   /* xref case 7960. */
+    if (violation_type == GBOP_SOURCE_VIOLATION)
+        report_thread_list = false;
+#endif
+    if (violation_type == ASLR_TARGET_VIOLATION) {
+        /* we should in fact be able to report the thread list, 
+         * if it wasn't the ASSERT, and to keep things mostly same 
+         */
+        report_thread_list = false;
+    }
+    if (violation_type == APC_THREAD_SHELLCODE_VIOLATION) {
+        report_thread_list = false;
+    }
+#if defined(PROGRAM_SHEPHERDING) && defined(PROCESS_CONTROL)
+    if (violation_type == PROCESS_CONTROL_VIOLATION)
+        report_thread_list = false;
+#endif
+
+    if (conservative) {
+        /* cannot call malloc, don't list all threads */
+        report_thread_list = false;
+    }
+
+    /* we do not support acquiring the thread_initexit_lock for any
+     * violations.  case 6141 covers re-enabling.  see also
+     * FORENSICS_ACQUIRES_INITEXIT_LOCK in vmareas.c.
+     */
+    ASSERT(!report_thread_list || violation_type >= 0/*non-violation*/);
+    if (report_thread_list) {
+        mutex_lock(&thread_initexit_lock);
+        get_list_of_threads(&threads, &num_threads);
+        for (i = 0; i < num_threads; i++) {
+            if (threads[i]->dcontext != NULL) {
+                report_thread(diagnostics_file, i,
+                              threads[i]->id, threads[i]->dcontext,
+                              conservative);
+            }
+        }
+        mutex_unlock(&thread_initexit_lock);
+        if (couldbelinking) {
+            enter_couldbelinking(get_thread_private_dcontext(), NULL,
+                                 false/*not a cache transition*/);
+        }
+        global_heap_free(threads, num_threads*sizeof(thread_record_t*)
+                         HEAPACCT(ACCT_THREAD_MGT));
+    } else {
+        report_thread(diagnostics_file, 0, get_thread_id(),
+                      get_thread_private_dcontext(), conservative);
+    }
+
+    print_file(diagnostics_file, "</thread-list>\n</current-process>\n\n");
+}
+
+/* Using the NtQuerySystemInformation() system call, the
+ * SystemProcessesAndThreadsInformation structure is filled in.
+ * Since the structure is of variable size, repeated calls to
+ * NtQuerySystemInformation() are made until the buffer is big
+ * enough to hold all the information.  This also explains why the
+ * returned pointer is a byte * rather than PSYSTEM_PROCESSES.  The
+ * calling function can cast the buffer to PSYSTEM_PROCESSES for
+ * each process chained by the NextEntryDelta field.
+ */
+static byte *
+get_system_processes(IN file_t diagnostics_file, OUT uint *info_bytes_needed)
+{
+    NTSTATUS result;
+    byte *process_info;
+
+    *info_bytes_needed = sizeof(SYSTEM_PROCESSES);
+    /* FIXME: Not ideal to dynamically allocate memory in unstable situation. */
+    process_info = (byte *) global_heap_alloc(*info_bytes_needed HEAPACCT(ACCT_OTHER));
+    memset(process_info, 0, *info_bytes_needed);
+    do {
+        result = query_system_info(SystemProcessesAndThreadsInformation, 
+                                   *info_bytes_needed, 
+                                   process_info);
+        if (result == STATUS_INFO_LENGTH_MISMATCH) {
+            global_heap_free(process_info, *info_bytes_needed HEAPACCT(ACCT_OTHER));
+            *info_bytes_needed = (*info_bytes_needed) * 2;
+            process_info = (byte *) global_heap_alloc(*info_bytes_needed
+                                                      HEAPACCT(ACCT_OTHER));
+            memset(process_info, 0, *info_bytes_needed);
+        }
+    } while (result == STATUS_INFO_LENGTH_MISMATCH);
+
+    if (!NT_SUCCESS(result))
+        return NULL;
+
+    return process_info;
+}
+
+/* Collects and displays all process information.  First displays all process
+ * names.  Then displays additional information for the current process 
+ */
+static void
+report_processes(IN file_t diagnostics_file, IN security_violation_t violation_type)
+{
+    byte *process_info = NULL;
+    byte *next_process = NULL;
+    PSYSTEM_PROCESSES sp = NULL;
+    uint info_bytes_needed;
+    bool found_current_process;
+    bool found_last_process;
+
+    /* We use byte * for process_info because
+     * SystemProcessesAndThreadsInformation is variable,
+     * each entry is cast to a SYSTEM_PROCESSES prior to access */
+    process_info = get_system_processes(diagnostics_file, &info_bytes_needed);
+
+    if (process_info != NULL) {
+        /* Initialize to first process */
+        next_process = process_info;
+        sp = (PSYSTEM_PROCESSES) next_process;
+        found_last_process = false;
+
+        print_file(diagnostics_file, "<process-list> <![CDATA[ \n");
+
+        /* Print out all process names here */
+        do {        
+            /* A NextEntryDelta of 0 indicates the last process in the structure */
+            if (sp->NextEntryDelta == 0)
+                found_last_process = true;
+            if (sp->ProcessName.Buffer !=  NULL) {
+                print_file(diagnostics_file, "%S\n", sp->ProcessName.Buffer);
+            }
+            next_process = (byte *) (next_process + sp->NextEntryDelta);
+            sp = (PSYSTEM_PROCESSES) next_process;
+        } while  (found_last_process == false);
+
+        print_file(diagnostics_file, "]]> </process-list>\n\n");
+
+        /* Initialize to first process */
+        next_process = process_info;
+        sp = (PSYSTEM_PROCESSES) next_process;
+        found_last_process = false;
+        found_current_process = false;
+
+        /* Print out current process info */
+        do {        
+            /* A NextEntryDelta of 0 indicates the last process in the structure */
+            if (sp->NextEntryDelta == 0)
+                found_last_process = true;
+            if (is_pid_me(sp->ProcessId)) {
+                found_current_process = true;
+                report_current_process(diagnostics_file, sp, violation_type,
+                                       false /*not conservative*/);
+            }
+            next_process = (byte *) (next_process + * (uint *) next_process);
+            sp = (PSYSTEM_PROCESSES) next_process;
+        } while ((found_last_process == false) && (found_current_process == false));
+
+        global_heap_free(process_info, info_bytes_needed HEAPACCT(ACCT_OTHER));
+    }
+}
+
+/* forward decl */
+static void
+report_registry_settings_helper(file_t diagnostics_file, uint log_mask,
+                                uint *total_keys, uint *recursion_level);
+
+/* Collects and displays all diagnostic information collected from the
+ * registry key 'keyname'. Recursively walk subkeys and values up to
+ * DIAGNOSTICS_MAX_RECURSION_LEVEL depth. No more then
+ * DIAGNOSTICS_MAX_REG_KEYS will be investigated in this way.
+ */
+static void
+report_registry_settings(IN file_t diagnostics_file,
+                         IN wchar_t *keyname,
+                         IN uint  log_mask)
+{
+    uint recursion_level = 0;
+    uint total_keys = 0;
+    ASSERT_OWN_MUTEX(true, &reg_mutex);
+    wcsncpy(diagnostic_keyname, keyname,
+            BUFFER_SIZE_ELEMENTS(diagnostic_keyname));
+    report_registry_settings_helper(diagnostics_file, log_mask,
+                                    &total_keys, &recursion_level);
+}
+
+/* static buffer diagnostic_keyname holds the registry key to output
+ * information for, report_registry_settings modifies it in place and
+ * recursively calls itself to walk the subkeys. total_keys and recursion_level
+ * are used to bound the max number of keys walked and the max recursion depth
+ * respectively. */
+static void
+report_registry_settings_helper(file_t diagnostics_file, uint log_mask,
+                                uint *total_keys, uint *recursion_level)
+{
+    int current_enum_key = 0;
+    int current_enum_value = 0;
+    int key_result;
+
+    if (*recursion_level == 0) {
+        *total_keys = 0;
+        print_file(diagnostics_file, "%sRegistry Settings\n%S\n%s",
+                   separator, diagnostic_keyname, separator);
+    }
+    *total_keys = *total_keys + 1;
+
+    memset(&diagnostic_value_info, 0, sizeof(diagnostic_value_info));
+
+    if (log_mask & DIAGNOSTICS_REG_ALLKEYS) {
+        int value_result;
+        print_file(diagnostics_file, "%S\n\n", diagnostic_keyname);            
+        current_enum_value = 0;
+        do {
+            value_result = reg_enum_value(diagnostic_keyname, 
+                                          current_enum_value,
+                                          KeyValueFullInformation,
+                                          &diagnostic_value_info, 
+                                          sizeof(diagnostic_value_info));    
+            if (value_result) {
+                diagnostics_log_data(diagnostics_file, log_mask);
+                print_file(diagnostics_file, "\n"); 
+            }
+            current_enum_value++;
+        } while ((value_result) &&
+                 (current_enum_value < DIAGNOSTICS_MAX_REG_VALUES));
+    }
+    else if (log_mask & DIAGNOSTICS_REG_HARDWARE) {
+        reg_query_value_result_t value_result;
+        value_result = reg_query_value(diagnostic_keyname, 
+                                       DIAGNOSTICS_DESCRIPTION_KEY,
+                                       KeyValueFullInformation,
+                                       &diagnostic_value_info, 
+                                       sizeof(diagnostic_value_info), 0);    
+        if (value_result == REG_QUERY_SUCCESS) {
+            diagnostics_log_data(diagnostics_file, log_mask);
+
+            /* Try to get the manufacturer */
+            value_result = reg_query_value(diagnostic_keyname, 
+                                           DIAGNOSTICS_MANUFACTURER_KEY,
+                                           KeyValueFullInformation,
+                                           &diagnostic_value_info, 
+                                           sizeof(diagnostic_value_info), 0);    
+            if (value_result == REG_QUERY_SUCCESS) 
+                diagnostics_log_data(diagnostics_file, log_mask);
+            /* Try to get the friendly name */
+            value_result = reg_query_value(diagnostic_keyname, 
+                                           DIAGNOSTICS_FRIENDLYNAME_KEY,
+                                           KeyValueFullInformation,
+                                           &diagnostic_value_info, 
+                                           sizeof(diagnostic_value_info), 0);
+            if (value_result == REG_QUERY_SUCCESS)
+                diagnostics_log_data(diagnostics_file, log_mask);
+    
+            print_file(diagnostics_file, "\n");
+        } 
+    }
+
+    /* See if there are more subkeys to recursively call */
+    /* Re-using the same diagnostic_key_info structure to reduce stack overhead */
+    if (log_mask & DIAGNOSTICS_REG_ALLSUBKEYS) {
+        do {
+            memset(&diagnostic_key_info, 0, sizeof(diagnostic_key_info));
+            key_result = reg_enum_key(diagnostic_keyname, 
+                                      current_enum_key, 
+                                      KeyBasicInformation, 
+                                      &diagnostic_key_info, 
+                                      sizeof(diagnostic_key_info));
+            current_enum_key++;
+
+            if ((*recursion_level < DIAGNOSTICS_MAX_RECURSION_LEVEL) && 
+                (*total_keys < DIAGNOSTICS_MAX_REG_KEYS) && 
+                (key_result)) {
+                size_t index = wcslen(diagnostic_keyname);
+
+                /* append subkey name */
+                _snwprintf(&diagnostic_keyname[index],
+                           BUFFER_SIZE_ELEMENTS(diagnostic_keyname) - index, 
+                           L"\\%s", diagnostic_key_info.Name);
+                NULL_TERMINATE_BUFFER(diagnostic_keyname);
+
+                *recursion_level = *recursion_level + 1;
+                report_registry_settings_helper(diagnostics_file, log_mask,
+                                                total_keys, recursion_level);
+                *recursion_level = *recursion_level - 1;
+
+                /* remove subkey name */
+                diagnostic_keyname[index] = L'\0';
+            }
+        } while ((*recursion_level < DIAGNOSTICS_MAX_RECURSION_LEVEL) && 
+                (*total_keys < DIAGNOSTICS_MAX_REG_KEYS) && 
+                (key_result));
+    }
+}
+
+static void
+report_autostart_programs(file_t diagnostics_file)
+{
+    int i;
+    wchar_t sid[DIAGNOSTICS_MAX_KEY_NAME_SIZE];
+    NTSTATUS result;
+
+    ASSERT_OWN_MUTEX(true, &reg_mutex);
+    print_file(diagnostics_file, "<autostart-programs>\n<![CDATA[\n");
+
+    /* HKEY_LOCAL_MACHINE entries */
+    for (i=0; i<BUFFER_SIZE_ELEMENTS(HKLM_entries); i++) {
+        report_registry_settings(diagnostics_file,
+                                 (wchar_t *)HKLM_entries[i],
+                                 (DIAGNOSTICS_REG_ALLKEYS |
+                                  DIAGNOSTICS_REG_ALLSUBKEYS |
+                                  DIAGNOSTICS_REG_NAME | 
+                                  DIAGNOSTICS_REG_DATA));
+    }     
+
+    /* HKEY_CURRENT_USER entries */
+    result = get_current_user_SID(sid, sizeof(sid));
+    if (NT_SUCCESS(result)) {
+        wchar_t entry[DIAGNOSTICS_MAX_KEY_NAME_SIZE];
+
+        for (i=0; i<BUFFER_SIZE_ELEMENTS(HKCU_entries); i++) {
+            _snwprintf(entry, BUFFER_SIZE_ELEMENTS(entry),
+                       L"\\Registry\\User\\%s\\%s", sid, HKCU_entries[i]);
+            NULL_TERMINATE_BUFFER(entry);
+
+            report_registry_settings(diagnostics_file,
+                                     entry,
+                                     (DIAGNOSTICS_REG_ALLKEYS |
+                                      DIAGNOSTICS_REG_ALLSUBKEYS |
+                                      DIAGNOSTICS_REG_NAME | 
+                                      DIAGNOSTICS_REG_DATA));
+        }
+    }
+    else {
+        ASSERT(false && "query of current user's SID failed");
+    }
+
+    print_file(diagnostics_file, "]]>\n</autostart-programs>\n\n");
+}
+
+/* Displays diagnostic intro. */
+static void
+report_intro(IN file_t diagnostics_file,
+             IN const char *message,
+             IN const char *name /* NULL if not a violation */)
+{
+    static const char *months[] = {"???", "Jan", "Feb", "Mar", "Apr", "May", "Jun",
+                                   "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"};
+    SYSTEMTIME st;
+    query_system_time(&st);
+
+    /* make sure index to months array is okay */
+    if (st.wMonth < 1 || st.wMonth > 12) {
+        ASSERT(false && "query_system_time() returning bad month");
+        st.wMonth = 0;
+    }
+
+    print_file(diagnostics_file, "\n<diagnostic-report>\n"
+               "<date> %s %d, %d </date>\n"
+               "<time> %.2d:%.2d:%.2d.%.3d GMT </time>\n",
+               months[st.wMonth], st.wDay, st.wYear,
+               st.wHour, st.wMinute, st.wSecond, st.wMilliseconds);
+
+    /* FIXME - can message be long enough that this runs into our buffer length
+     * limits? Could write message as a direct file write. */
+    print_file(diagnostics_file,
+               "<description> <![CDATA[[ \n"
+               "Generated by "PRODUCT_NAME" %s, %s\n%s\n"
+               "]]> </description>\n",
+               VERSION_NUMBER_STRING, BUILD_NUMBER_STRING, message);
+
+    if (name != NULL) {
+        print_file(diagnostics_file,
+                   "<threat-id> Threat ID: %s </threat-id>\n", name);  
+    }
+
+    print_file(diagnostics_file, "</diagnostic-report>\n\n");
+}
+
+static void
+report_processor_info(file_t diagnostics_file)
+{
+    features_t *features;
+
+    print_file(diagnostics_file, "<processor-information\n");
+
+    print_file(diagnostics_file, "Brand=        \"%s\"\n",
+               proc_get_brand_string());
+
+    print_file(diagnostics_file, "Type=         \"0x%x\"\n",
+               proc_get_type());
+
+    print_file(diagnostics_file, "Family=       \"0x%x\"\n",
+               proc_get_family());
+
+    print_file(diagnostics_file, "Model=        \"0x%x\"\n",
+               proc_get_model());
+
+    print_file(diagnostics_file, "Stepping=     \"0x%x\"\n",
+               proc_get_stepping());
+
+    print_file(diagnostics_file, "L1_icache=    \"%s\"\n",
+               proc_get_cache_size_str(proc_get_L1_icache_size()));
+
+    print_file(diagnostics_file, "L1_dcache=    \"%s\"\n",
+               proc_get_cache_size_str(proc_get_L1_dcache_size()));
+
+    print_file(diagnostics_file, "L2_cache=     \"%s\"\n",
+               proc_get_cache_size_str(proc_get_L2_cache_size()));
+
+    features = proc_get_all_feature_bits();
+    print_file(diagnostics_file, "Feature_bits= \"%.8x %.8x %.8x %.8x\"\n",
+               features->flags_edx, features->flags_ecx,
+               features->ext_flags_edx, features->ext_flags_ecx);
+
+    print_file(diagnostics_file, "/>\n");
+}
+
+/* Collects and displays all system diagnostic information. */
+static void
+report_system_diagnostics(IN file_t diagnostics_file)
+{
+
+    DIAGNOSTICS_INFORMATION diag_info;
+    NTSTATUS result;
+    /* Declare large structure here, excluding it from recursion storage */
+
+    print_file(diagnostics_file, "<system-settings>\n"
+               "<computer name=\"%s\" />\n",
+               get_computer_name());
+
+    report_processor_info(diagnostics_file);
+
+    memset(&diag_info, 0, sizeof(diag_info));
+    result = query_system_info(SystemBasicInformation, 
+                               sizeof(SYSTEM_BASIC_INFORMATION), 
+                               &(diag_info.sbasic_info));
+    if (NT_SUCCESS(result)) {
+        print_file(diagnostics_file,
+                   "<basic-information>\n"
+                   "\t%.10d %.10d %.10d %.10d %.10d %.10d %.10d\n"
+                   "\t0x%.8x 0x%.8x %.10d %.10d \n"
+                   "</basic-information>\n",
+                   diag_info.sbasic_info.Unknown,
+                   diag_info.sbasic_info.MaximumIncrement,
+                   diag_info.sbasic_info.PhysicalPageSize,
+                   diag_info.sbasic_info.NumberOfPhysicalPages,
+                   diag_info.sbasic_info.LowestPhysicalPage,
+                   diag_info.sbasic_info.HighestPhysicalPage,
+                   diag_info.sbasic_info.AllocationGranularity,
+                   diag_info.sbasic_info.LowestUserAddress,
+                   diag_info.sbasic_info.HighestUserAddress,
+                   diag_info.sbasic_info.ActiveProcessors,
+                   diag_info.sbasic_info.NumberProcessors);
+    }
+
+    result = query_system_info(SystemPerformanceInformation, 
+                               sizeof(SYSTEM_PERFORMANCE_INFORMATION), 
+                               &(diag_info.sperf_info));
+    if (NT_SUCCESS(result)) {
+        /* FIXME: good we started with all, but we should cut most of the
+         * useless ones */
+        print_file(diagnostics_file,
+                   "<performance-information>\n"
+                   "\t0x%.11x%.8x 0x%.11x%.8x 0x%.11x%.8x 0x%.11x%.8x\n",
+                   diag_info.sperf_info.IdleTime.HighPart,
+                   diag_info.sperf_info.IdleTime.LowPart,
+                   diag_info.sperf_info.ReadTransferCount.HighPart,
+                   diag_info.sperf_info.ReadTransferCount.LowPart,
+                   diag_info.sperf_info.WriteTransferCount.HighPart,
+                   diag_info.sperf_info.WriteTransferCount.LowPart,
+                   diag_info.sperf_info.OtherTransferCount.HighPart,
+                   diag_info.sperf_info.OtherTransferCount.LowPart);
+        print_file(diagnostics_file, 
+                   "\t%.10d %.10d %.10d %.10d %.10d %.10d %.10d\n"
+                   "\t%.10d %.10d %.10d %.10d %.10d %.10d %.10d\n"
+                   "\t%.10d %.10d %.10d %.10d %.10d %.10d %.10d\n"
+                   "\t%.10d %.10d %.10d %.10d %.10d %.10d %.10d\n"
+                   "\t%.10d %.10d %.10d %.10d %.10d %.10d %.10d\n",
+                   diag_info.sperf_info.ReadOperationCount,
+                   diag_info.sperf_info.WriteOperationCount,
+                   diag_info.sperf_info.OtherOperationCount,
+                   diag_info.sperf_info.AvailablePages,
+                   diag_info.sperf_info.TotalCommittedPages,
+                   diag_info.sperf_info.TotalCommitLimit,
+                   diag_info.sperf_info.PeakCommitment,
+                   diag_info.sperf_info.PageFaults,
+                   diag_info.sperf_info.WriteCopyFaults,
+                   diag_info.sperf_info.TranstitionFaults,
+                   diag_info.sperf_info.Reserved1,
+                   diag_info.sperf_info.DemandZeroFaults,
+                   diag_info.sperf_info.PagesRead,
+                   diag_info.sperf_info.PageReadIos,
+                   diag_info.sperf_info.Reserved2[0],
+                   diag_info.sperf_info.Reserved2[1],
+                   diag_info.sperf_info.PageFilePagesWritten,
+                   diag_info.sperf_info.PageFilePagesWriteIos,
+                   diag_info.sperf_info.MappedFilePagesWritten,
+                   diag_info.sperf_info.PagedPoolUsage,
+                   diag_info.sperf_info.NonPagedPoolUsage,
+                   diag_info.sperf_info.PagedPoolAllocs,
+                   diag_info.sperf_info.PagedPoolFrees,
+                   diag_info.sperf_info.NonPagedPoolAllocs,
+                   diag_info.sperf_info.NonPagedPoolFrees,
+                   diag_info.sperf_info.TotalFreeSystemPtes,
+                   diag_info.sperf_info.SystemCodePage,
+                   diag_info.sperf_info.TotalSystemDriverPages,
+                   diag_info.sperf_info.TotalSystemCodePages,
+                   diag_info.sperf_info.SmallNonPagedLookasideListAllocateHits,
+                   diag_info.sperf_info.SmallPagedLookasieListAllocateHits,
+                   diag_info.sperf_info.Reserved3,
+                   diag_info.sperf_info.MmSystemCachePage,
+                   diag_info.sperf_info.PagedPoolPage,
+                   diag_info.sperf_info.SystemDriverPage);
+        print_file(diagnostics_file, 
+                   "\t%.10d %.10d %.10d %.10d %.10d %.10d %.10d\n"
+                   "\t%.10d %.10d %.10d %.10d %.10d %.10d %.10d\n"
+                   "\t%.10d %.10d %.10d %.10d %.10d %.10d %.10d\n"
+                   "\t%.10d %.10d %.10d %.10d %.10d %.10d %.10d\n"
+                   "\t%.10d %.10d %.10d %.10d %.10d %.10d\n"
+                   "</performance-information>\n",
+                   diag_info.sperf_info.FastReadNoWait,
+                   diag_info.sperf_info.FastReadWait,
+                   diag_info.sperf_info.FastReadResourceMiss,
+                   diag_info.sperf_info.FastReadNotPossible,
+                   diag_info.sperf_info.FastMdlReadNoWait,
+                   diag_info.sperf_info.FastMdlReadWait,
+                   diag_info.sperf_info.FastMdlReadResourceMiss,
+                   diag_info.sperf_info.FastMdlReadNotPossible,
+                   diag_info.sperf_info.MapDataNoWait,
+                   diag_info.sperf_info.MapDataWait,
+                   diag_info.sperf_info.MapDataNoWaitMiss,
+                   diag_info.sperf_info.MapDataWaitMiss,
+                   diag_info.sperf_info.PinMappedDataCount,
+                   diag_info.sperf_info.PinReadNoWait,
+                   diag_info.sperf_info.PinReadWait,
+                   diag_info.sperf_info.PinReadNoWaitMiss,
+                   diag_info.sperf_info.PinReadWaitMiss,
+                   diag_info.sperf_info.CopyReadNoWait,
+                   diag_info.sperf_info.CopyReadWait,
+                   diag_info.sperf_info.CopyReadNoWaitMiss,
+                   diag_info.sperf_info.CopyReadWaitMiss,
+                   diag_info.sperf_info.MdlReadNoWait,
+                   diag_info.sperf_info.MdlReadWait,
+                   diag_info.sperf_info.MdlReadNoWaitMiss,
+                   diag_info.sperf_info.MdlReadWaitMiss,
+                   diag_info.sperf_info.ReadAheadIos,
+                   diag_info.sperf_info.LazyWriteIos,
+                   diag_info.sperf_info.LazyWritePages,
+                   diag_info.sperf_info.DataFlushes,
+                   diag_info.sperf_info.DataPages,
+                   diag_info.sperf_info.ContextSwitches,
+                   diag_info.sperf_info.FirstLevelTbFills,
+                   diag_info.sperf_info.SecondLevelTbFills,
+                   diag_info.sperf_info.SystemCalls);
+    }
+
+    result = query_system_info(SystemTimeOfDayInformation, 
+                               sizeof(SYSTEM_TIME_OF_DAY_INFORMATION), 
+                               &(diag_info.stime_info));
+    if (NT_SUCCESS(result)) {
+        print_file(diagnostics_file, 
+                   "<time-of-day-information>\n"
+                   "\t0x%.11x%.8x 0x%.11x%.8x 0x%.11x%.8x %.10d\n"
+                   "</time-of-day-information>\n",
+                   diag_info.stime_info.BootTime.HighPart,
+                   diag_info.stime_info.BootTime.LowPart,
+                   diag_info.stime_info.CurrentTime.HighPart,
+                   diag_info.stime_info.CurrentTime.LowPart,
+                   diag_info.stime_info.TimeZoneBias.HighPart,
+                   diag_info.stime_info.TimeZoneBias.LowPart,
+                   diag_info.stime_info.CurrentTimeZoneId);
+    }
+
+    result = query_system_info(SystemProcessorTimes, 
+                               sizeof(SYSTEM_PROCESSOR_TIMES), 
+                               &(diag_info.sptime_info));
+    if (NT_SUCCESS(result)) {
+        print_file(diagnostics_file,
+                   "<processor-times>\n"
+                   "\t0x%.11x%.8x 0x%.11x%.8x 0x%.11x%.8x 0x%.11x%.8x\n"
+                   "\t0x%.11x%.8x %.10d\n"
+                   "</processor-times>\n",
+                   diag_info.sptime_info.IdleTime.HighPart,
+                   diag_info.sptime_info.IdleTime.LowPart,
+                   diag_info.sptime_info.KernelTime.HighPart,
+                   diag_info.sptime_info.KernelTime.LowPart,
+                   diag_info.sptime_info.UserTime.HighPart,
+                   diag_info.sptime_info.UserTime.LowPart,
+                   diag_info.sptime_info.DpcTime.HighPart,
+                   diag_info.sptime_info.DpcTime.LowPart,
+                   diag_info.sptime_info.InterruptTime.HighPart,
+                   diag_info.sptime_info.InterruptTime.LowPart,
+                   diag_info.sptime_info.InterruptCount);
+    }
+
+    result = query_system_info(SystemGlobalFlag, 
+                               sizeof(SYSTEM_GLOBAL_FLAG), 
+                               &(diag_info.global_flag));
+    if (NT_SUCCESS(result)) {
+        print_file(diagnostics_file, "<global-flag> 0x%.8x </global-flag>\n",
+                   diag_info.global_flag.GlobalFlag);
+    }
+
+    print_file(diagnostics_file, "</system-settings>\n\n");
+}
+
+static void
+add_diagnostics_xml_header(file_t diagnostics_file)
+{
+    /* FIXME - xref case 9425, iso-8859-1 encoding is chosen because all 8 bit values
+     * are valid and wld.exe's library knows how to handle it.  Other choices may be
+     * more appropriate in the future. */
+    print_file(diagnostics_file,
+               "<?xml version=\""DIAGNOSTICS_XML_FILE_VERSION"\" encoding=\"iso-8859-1\" ?>\n"
+               "<!--\n"
+               "  =====================================================================\n"
+               "  Copyright @ "COMPANY_LONG_NAME" (2007). All rights reserved\n"
+               "  =====================================================================\n"
+               " -->\n"
+               "<forensic-report title=\""PRODUCT_NAME" Forensic File\">\n");
+}
+
+static void
+report_diagnostics_common(file_t diagnostics_file, const char *message, const char *name,
+                          security_violation_t violation_type)
+{
+    uint total_keys = 0;
+    uint recursion_level = 0;
+
+    report_intro(diagnostics_file, message, name);
+
+    /* Process snapshot requires memory allocation -- only use if genuine attack */
+    if (violation_type == NO_VIOLATION_BAD_INTERNAL_STATE)
+        report_current_process(diagnostics_file, NULL /*no snaphot*/,
+                               violation_type, true /*be conservative*/);
+    else
+        report_processes(diagnostics_file, violation_type);
+    
+    report_system_diagnostics(diagnostics_file);
+
+    mutex_lock(&reg_mutex);
+    print_file(diagnostics_file, "<registry-settings>\n<![CDATA[\n");
+    report_registry_settings(diagnostics_file,
+                             DYNAMORIO_REGISTRY_BASE, 
+                             (DIAGNOSTICS_REG_ALLKEYS |
+                              DIAGNOSTICS_REG_ALLSUBKEYS |
+                              DIAGNOSTICS_REG_NAME | 
+                              DIAGNOSTICS_REG_DATA));
+    report_registry_settings(diagnostics_file,
+                             DIAGNOSTICS_OS_REG_KEY, 
+                             (DIAGNOSTICS_REG_ALLKEYS | 
+                              DIAGNOSTICS_REG_NAME | 
+                              DIAGNOSTICS_REG_DATA));
+    /* delve deeper into OS reg key for our two injection method keys */
+    report_registry_settings(diagnostics_file,
+                             INJECT_ALL_HIVE_L INJECT_ALL_KEY_L,
+                             (DIAGNOSTICS_REG_ALLKEYS |
+                              DIAGNOSTICS_REG_NAME |
+                              DIAGNOSTICS_REG_DATA));
+    report_registry_settings(diagnostics_file,
+                             DEBUGGER_INJECTION_HIVE_L DEBUGGER_INJECTION_KEY_L,
+                             (DIAGNOSTICS_REG_ALLKEYS |
+                              DIAGNOSTICS_REG_ALLSUBKEYS |
+                              DIAGNOSTICS_REG_NAME |
+                              DIAGNOSTICS_REG_DATA));
+    report_registry_settings(diagnostics_file,
+                             DIAGNOSTICS_BIOS_REG_KEY, 
+                             (DIAGNOSTICS_REG_ALLKEYS | 
+                              DIAGNOSTICS_REG_NAME | 
+                              DIAGNOSTICS_REG_DATA));
+    report_registry_settings(diagnostics_file,
+                             DIAGNOSTICS_HARDWARE_REG_KEY, 
+                             (DIAGNOSTICS_REG_HARDWARE | 
+                              DIAGNOSTICS_REG_ALLSUBKEYS |
+                              DIAGNOSTICS_REG_DATA));
+    report_registry_settings(diagnostics_file,
+                             DIAGNOSTICS_CONTROL_REG_KEY, 
+                             (DIAGNOSTICS_REG_ALLKEYS |
+                              DIAGNOSTICS_REG_NAME | 
+                              DIAGNOSTICS_REG_DATA));
+    report_registry_settings(diagnostics_file,
+                             DIAGNOSTICS_OS_HOTFIX_REG_KEY, 
+                             (DIAGNOSTICS_REG_ALLKEYS |
+                              DIAGNOSTICS_REG_ALLSUBKEYS |
+                              DIAGNOSTICS_REG_NAME | 
+                              DIAGNOSTICS_REG_DATA));
+    print_file(diagnostics_file, "]]>\n</registry-settings>\n\n");
+    report_ntdll_info(diagnostics_file);
+    report_autostart_programs(diagnostics_file);
+    mutex_unlock(&reg_mutex);  
+
+    report_internal_data_structures(diagnostics_file, violation_type);
+
+    return;
+}
+
+/* Collects and displays all diagnostic information.
+ * violation_type of NO_VIOLATION_* is diagnostics; other is forensics.
+ */
+void 
+report_diagnostics(IN const char *message,
+                   IN const char *name, /* NULL if not a violation */
+                   IN security_violation_t violation_type)
+{
+
+    char diagnostics_filename[MAXIMUM_PATH];
+    file_t diagnostics_file = INVALID_FILE;
+
+    /* caller is assumed to have synchronized options */
+    if (!DYNAMO_OPTION(diagnostics))
+        return;
+
+    open_diagnostics_file(&diagnostics_file, diagnostics_filename, 
+                          BUFFER_SIZE_ELEMENTS(diagnostics_filename));
+
+    if (diagnostics_file == INVALID_FILE)
+        return;
+
+    /* Begin file with appropriate header */
+    add_diagnostics_xml_header(diagnostics_file);
+
+    report_diagnostics_common(diagnostics_file, message, name, violation_type);
+
+    /* End-of-file */
+    print_file(diagnostics_file, "</forensic-report>\n");
+
+    if (diagnostics_file != INVALID_FILE)
+        os_close(diagnostics_file);
+
+    /* write an event indicating the file was created */
+    SYSLOG(SYSLOG_INFORMATION, SEC_FORENSICS, 
+           3, get_application_name(), get_application_pid(), 
+           diagnostics_filename);
+}
+
+/* Functions similarly to report_diagnostics only appends to a supplied file instead of
+ * creating a file. It also skips adding a header in case this is part of a larger xml
+ * structure. */
+void
+append_diagnostics(file_t diagnostics_file, const char *message, const char *name,
+                   security_violation_t violation_type)
+{
+    /* Begin Report */
+    print_file(diagnostics_file, "<forensic-report title=\""PRODUCT_NAME" Forensic File\" "
+               "version=\""DIAGNOSTICS_XML_FILE_VERSION"\" encoding=\"iso-8859-1\">\n");
+    
+    report_diagnostics_common(diagnostics_file, message, name, violation_type);
+    
+    /* End-of-file */
+    print_file(diagnostics_file, "</forensic-report>\n");
+}
+
+void
+diagnost_exit()
+{
+    DELETE_LOCK(reg_mutex);
+}
+
diff --git a/core/win32/diagnost.h b/core/win32/diagnost.h
new file mode 100644
index 000000000..33935419f
--- /dev/null
+++ b/core/win32/diagnost.h
@@ -0,0 +1,117 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+
+/*
+ * diagnost.h - Diagnostics declarations
+ */
+
+#ifndef _DIAGNOST_H_
+#define _DIAGNOST_H_ 1
+
+/* FIXME: The key for the log directory should be in a shared file! */
+#define DIAGNOSTICS_LOGDIR_KEY L"DYNAMORIO_LOGDIR"
+#define DIAGNOSTICS_FILE_XML_EXTENSION ".xml"
+#define DIAGNOSTICS_XML_FILE_VERSION "1.0"
+#define DIAGNOSTICS_NTDLL_DLL_LOCATION L"System32\\NTDLL.DLL"
+#define DIAGNOSTICS_HARDWARE_REG_KEY L"\\Registry\\Machine\\System\\CurrentControlSet\\Enum"
+#define DIAGNOSTICS_CONTROL_REG_KEY L"\\Registry\\Machine\\System\\CurrentControlSet\\Control"
+#define DIAGNOSTICS_TEST_REG_KEY L"\\Registry\\Machine\\Software"
+#define DIAGNOSTICS_OS_REG_KEY L"\\Registry\\Machine\\Software\\Microsoft\\Windows NT\\CurrentVersion"
+#define DIAGNOSTICS_OS_HOTFIX_REG_KEY L"\\Registry\\Machine\\Software\\Microsoft\\Windows NT\\CurrentVersion\\Hotfix"  
+#define DIAGNOSTICS_BIOS_REG_KEY L"\\Registry\\Machine\\Hardware\\Description\\System"
+#define DIAGNOSTICS_SYSTEMROOT_REG_KEY L"SystemRoot"
+#define DIAGNOSTICS_DESCRIPTION_KEY L"DeviceDesc"
+#define DIAGNOSTICS_MANUFACTURER_KEY L"Mfg"
+#define DIAGNOSTICS_FRIENDLYNAME_KEY L"FriendlyName"
+
+#define DIAGNOSTICS_MAX_REG_KEYS 1000                  /* Arbitrary - seems sufficient */
+#define DIAGNOSTICS_MAX_REG_VALUES 1000                /* Arbitrary - seems sufficient */
+#define DIAGNOSTICS_MAX_RECURSION_LEVEL 5              /* Arbitrary, bit should keep small */
+#define DIAGNOSTICS_MAX_NAME_AND_DATA_SIZE 500         /* Arbitrary - seems sufficient */
+#define DIAGNOSTICS_MAX_KEY_NAME_SIZE 257              /* From SDK, + 1 for a null */
+#define DIAGNOSTICS_MAX_LOG_BUFFER_SIZE 1000
+#define DIAGNOSTICS_MAX_LOG_FILES 99999999             /* Supports 8.3 */
+#define DIAGNOSTICS_MINI_DUMP_SIZE 104                 /* multiple of 8, so is dumped aligned */
+
+#define DIAGNOSTICS_REG_NAME       0x00000001L         /* Log key name */
+#define DIAGNOSTICS_REG_DATA       0x00000002L         /* Log key data */
+#define DIAGNOSTICS_REG_HARDWARE   0x00000004L         /* Look for device keys */
+#define DIAGNOSTICS_REG_ALLKEYS    0x00000008L         /* Search all keys */
+#define DIAGNOSTICS_REG_ALLSUBKEYS 0x00000010L         /* Search all subkeys (recursive)*/
+#define DIAGNOSTICS_INITIAL_PROCESS_TOTAL 10
+
+#define DIAGNOSTICS_BYTES_PER_LINE 32
+
+ /* The DataOffset field in KEY_VALUE_FULL_INFORMATION uses the size of the structure as 
+   part of the offset.  When offsetting into the NameAndData member (see below), these
+   bytes are not present, and must be decremented.  The 2 WCHARs readjusts back for the 
+   null-terminated Name[1] (which IS included in NameAndData).  Confusing, no? */
+#define DECREMENT_FOR_DATA_OFFSET (sizeof(KEY_VALUE_FULL_INFORMATION) - (sizeof(WCHAR) * 2))
+
+typedef struct _DIAGNOSTICS_INFORMATION {
+    SYSTEM_BASIC_INFORMATION sbasic_info;
+    SYSTEM_PROCESSOR_INFORMATION sproc_info;
+    SYSTEM_PERFORMANCE_INFORMATION sperf_info;
+    SYSTEM_TIME_OF_DAY_INFORMATION stime_info;
+    SYSTEM_PROCESSOR_TIMES sptime_info;
+    SYSTEM_GLOBAL_FLAG global_flag;
+} DIAGNOSTICS_INFORMATION, *PDIAGNOSTICS_INFORMATION; 
+
+/* FIXME: combine this definition and KEY_VALUE_INFORMATION_CLASS from ntdll.h */
+/* The properly-sized name & data field for KEY_VALUE_FULL_INFORMATION 
+   are not included in the KEY_VALUE_FULL_INFORMATION structure, but 
+   needed to receive the data after calling NtEnumerateValueKey().  
+   This structure includes a buffer (NameAndData) for the largest reasonable 
+   size for both variable items; anything longer will simply be discarded).
+   The variable name field in NameAndData will always be null-terminated,
+   and the variable data field in NameAndData will always begin at 
+   (DataOffset - DECREMENT_FOR_DATAOFFSET). */
+typedef struct _DIAGNOSTICS_KEY_VALUE_FULL_INFORMATION {
+    ULONG   TitleIndex;
+    ULONG   Type;
+    ULONG   DataOffset;
+    ULONG   DataLength;
+    ULONG   NameLength; /* in BYTES (including NULL term) */
+    BYTE    NameAndData[DIAGNOSTICS_MAX_NAME_AND_DATA_SIZE];
+} DIAGNOSTICS_KEY_VALUE_FULL_INFORMATION, *PDIAGNOSTICS_KEY_VALUE_FULL_INFORMATION;
+
+/* Same story as above with the key basic info here */
+typedef struct _DIAGNOSTICS_KEY_NAME_INFORMATION {
+    LARGE_INTEGER   LastWriteTime;
+    ULONG   TitleIndex;
+    ULONG   NameLength; /* in BYTES (including NULL term) */
+    WCHAR   Name[DIAGNOSTICS_MAX_KEY_NAME_SIZE];       
+} DIAGNOSTICS_KEY_NAME_INFORMATION, *PDIAGNOSTICS_KEY_NAME_INFORMATION;
+
+#endif /* #ifndef _DIAGNOST_H_ */
diff --git a/core/win32/drmarker.c b/core/win32/drmarker.c
new file mode 100644
index 000000000..e2cf9902d
--- /dev/null
+++ b/core/win32/drmarker.c
@@ -0,0 +1,296 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2005-2007 Determina Corp. */
+
+/*
+ * drmarker.c
+ *
+ * shared between core/gui nodemgr, functions to tell if a process is running
+ * under dr and pass information out of the running process
+ *
+ */
+
+#if !defined(NOT_DYNAMORIO_CORE) && !defined(NOT_DYNAMORIO_CORE_PROPER)
+# include "globals.h"
+# include "os_private.h"
+# include "nudge.h"
+# include "module_shared.h"
+#else  /* NOT_DYNAMORIO_CORE */
+# define WIN32_LEAN_AND_MEAN
+# include <windows.h> /* no longer included in globals_shared.h */
+# include "globals_shared.h"
+#endif
+
+#include "ntdll.h"
+#include "drmarker.h"
+
+#if !defined(NOT_DYNAMORIO_CORE) && !defined(NOT_DYNAMORIO_CORE_PROPER)
+# define READ_FUNC nt_read_virtual_memory
+# define DR_MARKER_VERSION_CURRENT DR_MARKER_VERSION_1
+#else /* NOT_DYNAMORIO_CORE */
+# pragma warning( disable : 4054) /* from function pointer 'void (__cdecl *)(void )' to data pointer 'unsigned char *' */
+/* complains about size_t* vs SIZE_T* */
+# pragma warning( disable : 4057) /* 'function' : 'unsigned long *' differs in indirection to slightly different base types from 'int *' */
+# pragma warning( disable : 4100) /* 'envp' : unreferenced formal parameter */
+# ifndef byte
+typedef unsigned char byte;
+# endif
+# define PAGE_START(x) (((ptr_uint_t)(x)) & ~((PAGE_SIZE)-1))
+# define PAGE_SIZE (4*1024)
+# define READ_FUNC ReadProcessMemory
+#endif
+
+#define OP_jmp_byte 0xe9
+
+#ifdef NOT_DYNAMORIO_CORE
+/* avoid having to link in ntdll.c */
+bool
+is_wow64_process(HANDLE hProcess)
+{
+    /* IsWow64Pocess is only available on XP+ */
+    typedef DWORD (WINAPI *IsWow64Process_Type)(HANDLE hProcess,
+                                                PBOOL isWow64Process);
+    static HANDLE kernel32_handle;
+    static IsWow64Process_Type IsWow64Process;
+    if (kernel32_handle == NULL)
+        kernel32_handle = GetModuleHandle(L"kernel32.dll");
+    if (IsWow64Process == NULL && kernel32_handle != NULL) {
+        IsWow64Process = (IsWow64Process_Type)
+            GetProcAddress(kernel32_handle, "IsWow64Process");
+    }
+    if (IsWow64Process == NULL) {
+        /* should be NT or 2K */
+        return false;
+    } else {
+        bool res;
+        if (!IsWow64Process(hProcess, &res))
+            return false;
+        return res;
+    }
+}
+#endif
+
+/* For 32-bit build, supports looking for x64 marker (in WOW64 process).
+ * For 64-bit build, only supports looking for x64 marker.
+ */
+static int
+read_and_verify_dr_marker_common(HANDLE process, dr_marker_t *marker, bool x64)
+{
+    byte buf[8]; /* only needs to be 5, but dword pad just in case */
+    size_t res;
+    void *target, *landing_pad;
+#if !defined(NOT_DYNAMORIO_CORE) && !defined(NOT_DYNAMORIO_CORE_PROPER)
+    GET_NTDLL(DR_MARKER_HOOKED_FUNCTION, DR_MARKER_HOOKED_FUNCTION_ARGS);
+    void *hook_func = (void *)DR_MARKER_HOOKED_FUNCTION;
+#else
+    void *hook_func;
+# ifndef X64
+    if (x64) {
+        hook_func = (void *) get_proc_address_64
+            (get_module_handle_64(L_DR_MARKER_HOOKED_DLL),
+             DR_MARKER_HOOKED_FUNCTION_STRING);
+    } else
+# endif
+        hook_func = (void *)GetProcAddress(GetModuleHandle(DR_MARKER_HOOKED_DLL),
+                                           DR_MARKER_HOOKED_FUNCTION_STRING);
+#endif
+    if (IF_X64_ELSE(!x64, x64 && !is_wow64_process(NT_CURRENT_PROCESS)))
+        return DR_MARKER_ERROR;
+    
+    if (hook_func == NULL)
+        return DR_MARKER_ERROR;
+    
+    if (!READ_FUNC(process, hook_func, buf, 5, &res) || res != 5)
+        return DR_MARKER_ERROR;
+
+    if (buf[0] != OP_jmp_byte)
+        return DR_MARKER_NOT_FOUND;
+
+    /* jmp offset + EIP (after jmp = hook_func + size of jmp (5 bytes)) */
+    landing_pad = (void *)(*(int *)&buf[1] + (ptr_int_t)hook_func + 5);
+    /* for 64-bit, the target is stored in front of the trampoline */
+    if (x64)
+        landing_pad = (byte *)landing_pad - 8;
+    /* see emit_landing_pad_code() for layout of landing pad */
+    if (!READ_FUNC(process, landing_pad, buf, (x64 ? 8 : 5), &res) ||
+        res != (x64 ? 8U : 5U))
+        return DR_MARKER_ERROR;
+
+    if (x64) {
+        /* trampoline address is stored at the top of the landing pad for 64-bit */
+        target = (void *)PAGE_START(*(ptr_int_t *)buf);
+    } else {
+        /* jmp offset + EIP (after jmp = landing_pad + size of jmp (5 bytes)) */
+        target = (void *)PAGE_START(*(int *)&buf[1] + (ptr_int_t)landing_pad + 5);
+    }
+    
+    if (!READ_FUNC(process, target, marker, sizeof(dr_marker_t), &res) ||
+        res != sizeof(dr_marker_t)) {
+        return DR_MARKER_NOT_FOUND;
+    }
+
+    if (dr_marker_verify(process, marker)) {
+        return DR_MARKER_FOUND;
+    }
+    
+    return DR_MARKER_NOT_FOUND; /* probably some other hooker */
+}
+
+#ifndef X64
+int
+read_and_verify_dr_marker_64(HANDLE process, dr_marker_t *marker)
+{
+    return read_and_verify_dr_marker_common(process, marker, true);
+}
+#endif
+
+int
+read_and_verify_dr_marker(HANDLE process, dr_marker_t *marker)
+{
+    return read_and_verify_dr_marker_common(process, marker, IF_X64_ELSE(true, false));
+}
+
+/* FIXME : in the future we may want to obfuscate so is not a constant? */
+/* generated by dropping hand on number keypad and converting to hex */
+/* we pass in the process handle to dr_marker_verify and dr_marker_magic_init
+ * in case we decide to make the magic numbers process specific (xor w/pid or
+ * something similar) */
+#define DR_MARKER_MAGIC1 0xB1D2AE58
+#define DR_MARKER_MAGIC2 0xCA50C356
+#define DR_MARKER_MAGIC3 0x63000089
+#define DR_MARKER_MAGIC4 0x3FA898F0
+
+bool
+dr_marker_verify(HANDLE process, dr_marker_t *marker)
+{
+    return (marker->magic1 == DR_MARKER_MAGIC1 &&
+            marker->magic2 == DR_MARKER_MAGIC2 &&
+            marker->magic3 == DR_MARKER_MAGIC3 &&
+            marker->magic4 == DR_MARKER_MAGIC4);
+}
+
+#if !defined(NOT_DYNAMORIO_CORE) && !defined(NOT_DYNAMORIO_CORE_PROPER)
+/* takes a dr marker with all non magic fields filled in and fills in the
+ * magic fields */
+static void
+dr_marker_magic_init(HANDLE process, dr_marker_t *marker)
+{
+    marker->magic1 = DR_MARKER_MAGIC1;
+    marker->magic2 = DR_MARKER_MAGIC2;
+    marker->magic3 = DR_MARKER_MAGIC3;
+    marker->magic4 = DR_MARKER_MAGIC4;
+}
+
+# ifdef HOT_PATCHING_INTERFACE
+/* FIXME: Including hotpatch.h here breaks the share module because it doesn't
+ *        define a whole bunch of types like app_pc.  Once that is fixed, this
+ *        extern declaration should be removed and hotpatch.h included; see
+ *        case 4999.
+ */
+extern void *hotp_policy_status_table;  /* See FIXME above. */
+extern read_write_lock_t *hotp_get_lock(void);  /* See FIXME above. */
+# endif
+
+void
+init_dr_marker(dr_marker_t *marker)
+{
+    memset(marker, 0, sizeof(dr_marker_t));
+# ifdef DEBUG
+    marker->flags |= DR_MARKER_DEBUG_BUILD;
+# endif
+# ifdef PROFILE
+    marker->flags |= DR_MARKER_PROFILE_BUILD;
+# endif
+# ifdef RELEASE
+    marker->flags |= DR_MARKER_RELEASE_BUILD;
+# endif
+    /* make sure we set one of the above flags and not more then one */
+    ASSERT(TESTANY(DR_MARKER_BUILD_TYPES, marker->flags) &&
+           ((DR_MARKER_BUILD_TYPES & marker->flags) & 
+            ((DR_MARKER_BUILD_TYPES & marker->flags) - 1)) == 0);
+    /* TODO : add any additional flags? */
+    marker->build_num = BUILD_NUMBER;
+    marker->dr_base_addr = get_module_base((app_pc)init_dr_marker);
+    marker->dr_generic_nudge_target = (void *)generic_nudge_target;
+# ifdef HOT_PATCHING_INTERFACE
+    marker->dr_hotp_policy_status_table = (void *)hotp_policy_status_table;
+# endif
+    marker->dr_marker_version = DR_MARKER_VERSION_CURRENT;
+    marker->stats = get_dr_stats();
+    dr_marker_magic_init(NT_CURRENT_PROCESS, marker);
+}
+
+# ifdef HOT_PATCHING_INTERFACE
+void*
+get_drmarker_hotp_policy_status_table()
+{
+    dr_marker_t *dr_marker = get_drmarker();
+
+    ASSERT_OWN_READWRITE_LOCK(true, hotp_get_lock());
+
+    if (dr_marker == NULL)      /* dr_marker_t has been initialized. */
+        return NULL;
+
+    return dr_marker->dr_hotp_policy_status_table;
+}
+
+void
+set_drmarker_hotp_policy_status_table(void *new_table)
+{
+    dr_marker_t *dr_marker = get_drmarker();
+
+    ASSERT_OWN_WRITE_LOCK(true, hotp_get_lock());
+
+    /* We don't want to write to the dr_marker_t before it is initialized;  we
+     * could get an exception.
+     */
+    if (dr_marker == NULL)      /* Part of fix for case 5367. */
+        return;
+    /* Ok, dr_marker_t has been initialized. */
+
+    /* It is ok to do this memory protection change here even though this can
+     * be any arbitrary time due to the nature of nudge.  This is because
+     * once initialized, the dr_marker_t isn't touched by any one except the hot
+     * patch nudge.
+     *
+     * TODO: In future other parts of the core may need to change the dr_marker_t.
+     *       It might be a good idea to introduce a lock for the dr_marker_t and
+     *       generic accessor functions.
+     */
+    make_writable((byte*)dr_marker, INTERCEPTION_CODE_SIZE);
+    dr_marker->dr_hotp_policy_status_table = new_table;
+    make_unwritable((byte*)dr_marker, INTERCEPTION_CODE_SIZE);
+}
+# endif  /* HOT_PATCHING_INTERFACE */
+
+#endif /* !NOT_DYNAMORIO_CORE && !NOT_DYNAMORIO_CORE_PROPER */
diff --git a/core/win32/drmarker.h b/core/win32/drmarker.h
new file mode 100644
index 000000000..aa97571c5
--- /dev/null
+++ b/core/win32/drmarker.h
@@ -0,0 +1,130 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2005-2007 Determina Corp. */
+
+/*
+ * drmarker.h
+ * 
+ * shared between core/gui nodemgr, functions to tell if a process is running
+ * under dr and pass information out of the running process
+ */
+
+#ifndef _DRMARKER_H_
+#define _DRMARKER_H_
+
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#define L_DR_MARKER_HOOKED_DLL L"ntdll.dll"
+#ifdef UNICODE
+# define DR_MARKER_HOOKED_DLL L_DR_MARKER_HOOKED_DLL
+#else
+# define DR_MARKER_HOOKED_DLL "ntdll.dll"
+#endif
+
+#define DR_MARKER_HOOKED_FUNCTION KiUserCallbackDispatcher
+#define DR_MARKER_HOOKED_FUNCTION_ARGS (IN PVOID Unknown1, IN PVOID Unknown2, \
+                                        IN PVOID Unknown3)
+#define DR_MARKER_HOOKED_FUNCTION_STRING STRINGIFY(DR_MARKER_HOOKED_FUNCTION)
+
+enum {
+    DR_MARKER_RELEASE_BUILD  = 0x1,
+    DR_MARKER_DEBUG_BUILD    = 0x2,
+    DR_MARKER_PROFILE_BUILD  = 0x4,
+    DR_MARKER_BUILD_TYPES    = 
+        DR_MARKER_RELEASE_BUILD|DR_MARKER_DEBUG_BUILD|DR_MARKER_PROFILE_BUILD
+};
+
+#define DR_MARKER_VERSION_1 1
+
+/* CAUTION: This structure is shared across processes, so any changes to it
+ *          should only be field addtions.  NO DELETIONS ALLOWED; to obsolete
+ *          fields fill it with an invalid value.  Also, each change should
+ *          result in the DR_MARKER_VERSION_CURRENT being increased.
+ * FIXME: use size to denote a newer structures in future; same issue needs to
+ *        be handled for hotp_policy_status_table.
+ */
+struct _dr_statistics_t;
+typedef struct _dr_marker_t {
+    uint magic1;
+    uint magic2;
+    uint magic3;
+    uint magic4;
+    uint flags;
+    uint build_num;
+    uint dr_marker_version; /* this offset cannot be changed */
+    void *dr_base_addr;
+    void *dr_generic_nudge_target;
+    void *dr_hotp_policy_status_table;
+    struct _dr_statistics_t *stats;
+    /* future fields */
+    /* NOTE: rct_known_targets_init needs to be updated if new targets
+     * into DR are added */
+} dr_marker_t;
+
+enum {
+    DR_MARKER_FOUND                   = 0,
+    DR_MARKER_NOT_FOUND               = 1,
+    DR_MARKER_ERROR                   = 2
+};
+
+/* process must grant PROCESS_VM_READ */
+int
+read_and_verify_dr_marker(HANDLE process, dr_marker_t *marker);
+
+#ifndef X64
+/* 32-bit code to check for 64-bit marker. process must grant PROCESS_VM_READ */
+int
+read_and_verify_dr_marker_64(HANDLE process, dr_marker_t *marker);
+#endif
+
+#ifndef bool
+typedef int bool;
+#endif
+
+bool
+dr_marker_verify(HANDLE process, dr_marker_t *marker);
+
+#if !defined(NOT_DYNAMORIO_CORE) && !defined(NOT_DYNAMORIO_CORE_PROPER)
+void
+init_dr_marker(dr_marker_t *marker);
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _DRMARKER_H_ */
diff --git a/core/win32/early_inject_helper1.c b/core/win32/early_inject_helper1.c
new file mode 100644
index 000000000..4209c9701
--- /dev/null
+++ b/core/win32/early_inject_helper1.c
@@ -0,0 +1,63 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2006-2007 Determina Corp. */
+
+#pragma warning( disable : 4100) //'Reserved' : unreferenced formal parameter
+/* shows up in buildtools/VC/8.0/dist/VC/include/vadefs.h
+ * supposed to include identifier in the pop pragma and they don't
+ */
+#pragma warning( disable : 4159) // #pragma pack has popped previously pushed identifier
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+
+__declspec(dllimport) void dr_early_inject_helper2_dummy_func();
+
+__declspec(dllexport) void
+dr_early_inject_helper1_dummy_func()
+{
+    /* nothing: just here so we have export section for easy name finding */
+}
+
+BOOL WINAPI /* get link warning 4216 if export via APIENTRY */
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    /* Want to do nothing but still have the dependency, hModule is the base
+     * address of this DLL so should never equal -1 */
+    if ((LONG_PTR)hModule == -1) {
+        /* call does nothing anyways, just inserted so the linker doesn't
+         * optimize out the dependency, but we ensure isn't reached anyways */
+        dr_early_inject_helper2_dummy_func();
+    }
+    return TRUE;
+}
diff --git a/core/win32/early_inject_helper2.c b/core/win32/early_inject_helper2.c
new file mode 100644
index 000000000..8d8199980
--- /dev/null
+++ b/core/win32/early_inject_helper2.c
@@ -0,0 +1,55 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2006-2007 Determina Corp. */
+
+#pragma warning( disable : 4100) //'hModule' : unreferenced formal parameter
+/* shows up in buildtools/VC/8.0/dist/VC/include/vadefs.h
+ * supposed to include identifier in the pop pragma and they don't
+ */
+#pragma warning( disable : 4159) // #pragma pack has popped previously pushed identifier
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+
+__declspec(dllexport) void
+dr_early_inject_helper2_dummy_func()
+{
+    /* nothing */
+}
+
+BOOL WINAPI /* get link warning 4216 if export via APIENTRY */
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    /* nothing */
+    return TRUE;
+}
diff --git a/core/win32/eventlog.c b/core/win32/eventlog.c
new file mode 100644
index 000000000..c28610b40
--- /dev/null
+++ b/core/win32/eventlog.c
@@ -0,0 +1,939 @@
+/* **********************************************************
+ * Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+
+/* eventlog.c - Windows specific event logging issues */
+
+#include "globals.h"
+
+#include <windows.h>
+#include <tchar.h>
+#include "utils.h"
+
+#include "events.h"             /* generated from message file  */
+#include "ntdll.h"
+
+/* Types for Named pipe communication to the Event Log  */
+#define NONCE_LENGTH 20
+
+#define MAX_MESSAGE_SIZE 1024   /* total message size communicated to the eventlog */
+
+/* Connection state */
+typedef struct eventlog_state_t {
+    HANDLE eventlog_pipe;
+    HANDLE eventlog_completion; /* used for synchronization */
+    uint message_seq; /* message sequence number */
+    char nonce[NONCE_LENGTH];   /* nonce received from server on handshake */
+    mutex_t eventlog_mutex; /* sync persistent thread shared logging connection */
+    /* place buffers here to save stack space, used by [de]register and report
+     * all of whom protect them with the above lock, this structure is single
+     * instance static anyways so not wasting much memory doing it this way */
+    char outbuf[MAX_MESSAGE_SIZE];
+    size_t outlen;
+    char buf[MAX_MESSAGE_SIZE];
+    int request_length;
+} eventlog_state_t;
+
+/* writes a message to the Windows Event Log */
+static void os_eventlog(syslog_event_type_t priority, uint message_id, 
+                        uint substitutions_num, char **arguments,
+                        size_t size_data, char *raw_data);
+
+/* Custom logfile key and properties */
+/* This enables administrators to control the size of the log file  */
+/* and we can attach SACLs for security purposes, without affecting other applications.  */
+
+// Make sure the registry key is all set up, maybe better done in the installer?
+// The minimum we need:
+
+/* addsource.reg:
+Windows Registry Editor Version 5.00
+                                                                                
+[HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\Eventlog\Araksha]
+                                                                                 
+[HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Services\Eventlog\Araksha\DynamoRIO]
+"TypesSupported"=dword:00000007
+"EventMessageFile"="C:\\cygwin\\home\\vlk\\exports\\x86_win32_dbg\\dynamorio.dll"
+*/
+
+/* sets the values for the already existing event source key */
+static int
+set_event_source_registry_values()
+{
+    int res;
+    HANDLE heventsource = reg_open_key(L_EVENT_SOURCE_KEY, KEY_SET_VALUE);
+    /* the message file is in our main dll */
+    char *message_file = get_dynamorio_library_path();
+    wchar_t wide_message_file[MAXIMUM_PATH];
+
+    if (!heventsource) {
+        return 0;
+    }
+
+    ASSERT(message_file && strlen(message_file) < MAXIMUM_PATH);
+    _snwprintf(wide_message_file, MAXIMUM_PATH, L"%S", message_file);
+    NULL_TERMINATE_BUFFER(wide_message_file);
+
+    // FIXME: BUFOV? is the conversion locale dependent
+
+    res = reg_set_dword_key_value(heventsource,
+                                  L"TypesSupported", // which messages can go in
+                                  EVENTLOG_ERROR_TYPE | EVENTLOG_WARNING_TYPE | 
+                                  EVENTLOG_INFORMATION_TYPE);
+
+    res |= reg_set_dword_key_value(heventsource,
+                                   L"CategoryCount", // # of event categories supported
+                                   0);
+
+    /* REG_EXPAND_SZ can contain a level of "indirection" in the form of a
+       system variable that can be resolved at the time of use of the
+       entry. For example: EventMessageFile="%WINDIR%\\dynamorio.dll"
+    */
+ 
+    //FIXME: I'd rather set the full REG_EXPAND_SZ to be prepared
+    res |= reg_set_key_value(heventsource, 
+                             L"EventMessageFile",  
+                             /* should be the name of our DLL (or RLL if we put in
+                              * a separate file) */
+                             wide_message_file);
+
+    res |= reg_set_key_value(heventsource,
+                             L"CategoryMessageFile",
+                             wide_message_file);
+
+    /* we don't use these
+       DisplayNameFile 
+       DisplayNameID 
+       ParameterMessageFile 
+    */
+        
+    reg_close_key(heventsource);
+
+    return 1;
+}
+
+/* returns 1 on successfully setting up the registry variables for an event source */
+static int
+init_registry_source(void)
+{
+    static int initialized = 0; 
+    /* FIXME: we assume no one should have access to modify after we check */
+    /* FIXME: may want to register for notifications .. */
+    /* FIXME: if we fail we'll do this over and over for each event .. */
+
+    /* use our registry routines to avoid Win32 reentrancy issues */
+
+    /* FIXME: let's do it with access rights only as needed- I got fed up at one point*/
+    if (!initialized) {
+        /* first make sure all keys are created */
+        HANDLE heventsource = reg_open_key(L_EVENT_SOURCE_KEY, KEY_READ | KEY_WRITE);
+        HANDLE heventlogroot = 0;
+        HANDLE heventlog = 0;
+
+        if (!heventsource) {    /* we're not in */
+            /* test and create eventlog key  */
+            heventlog = reg_open_key(L_EVENT_LOG_KEY, KEY_READ | KEY_WRITE);
+            // KEY_READ == KEY_QUERY_VALUE | KEY_NOTIFY | 
+            //             KEY_ENUMERATE_SUB_KEYS seems too strong
+
+            if (!heventlog) {
+                heventlogroot = reg_open_key(L_EVENTLOG_REGISTRY_KEY, 
+                                             KEY_READ | KEY_WRITE);
+                if (!heventlogroot) {
+                    LOG(GLOBAL, LOG_TOP, 1, "WARNING: Registration failure.  Could not open root %ls.", 
+                        L_EVENTLOG_REGISTRY_KEY);
+                    return 0;   
+                }
+                heventlog = reg_create_key(heventlogroot, L_EVENT_LOG_NAME,
+                                           KEY_ALL_ACCESS);
+            }
+            if (!heventlog) {
+                LOG(GLOBAL, LOG_TOP, 1, "WARNING: Could not create event log key %s.", EVENTLOG_NAME);
+                return 0;       
+            }
+
+            /* obviously we'll need SET_VALUE later but to keep the logic simple we take minimal here */
+            heventsource = reg_create_key(heventlog, L_EVENT_SOURCE_NAME, KEY_QUERY_VALUE); 
+            if (heventlog) {
+                reg_close_key(heventlog);
+            }
+            if (heventlogroot) {
+                reg_close_key(heventlogroot);
+            }
+
+        }
+        if (!heventsource) {
+            LOG(GLOBAL, LOG_TOP, 1, "WARNING: Could not create event source key %s.", EVENTSOURCE_NAME);
+            return 0;   
+        }
+        
+        reg_close_key(heventsource);
+        initialized = set_event_source_registry_values();
+    }
+
+    return initialized;
+}
+
+
+#define MAX_SYSLOG_ARGS 6       /* increase if necessary */
+/* collect arguments in an array and pass along */
+void os_syslog(syslog_event_type_t priority, uint message_id, uint substitutions_num,
+               va_list vargs)
+{
+    uint arg;
+    char *arg_arr[MAX_SYSLOG_ARGS];
+
+    // pointer to raw data TODO: SYSLOG_DATA entry point that also adds data arguments
+    char *other_data = "";
+    size_t size_data = strlen(other_data); /* 0 - for no data */
+
+    ASSERT(substitutions_num < MAX_SYSLOG_ARGS);
+
+    for(arg = 0; arg < substitutions_num; arg++) {
+        arg_arr[arg] = va_arg(vargs, char*);
+    }
+    
+    /* don't need to check syslog, mask, caller is responsible for 
+     * checking the mask and synchronizing the options
+     */
+    os_eventlog(priority, message_id, substitutions_num, arg_arr, size_data, other_data);
+}
+
+/* Here starts the gross hack for direct message passing to the EventLog service */
+
+/* Macros for adding variable length fields to a message buffer 
+   p should point to the current position in the string appended to
+   pend points to the first location after the end of the buffer  */
+/* Modifies p */
+#define FIELD(p, pend, type, val) do {                  \
+    if (!(p) || (p) + sizeof(type) > (pend))            \
+        return 0;                                       \
+    *(type*)(p) = (type)(val);                          \
+    (p) += sizeof(type);                                \
+} while (0)
+
+/* pass &p */                                           \
+#define VARFIELD(pp, pend, val, len) do {               \
+    if (!(*(pp)) || (*(pp)) + (len) > (pend))           \
+        return 0;                                       \
+    memcpy((*(pp)), (val), (len));                      \
+    *(pp) += (len);                                     \
+} while (0)
+
+/* The advapi functions don't actually zero out the padding */
+/* Modifies p (i.e. *pp) */
+#define PADDING(pp, pend, len, boundary) do {               \
+  int skip = (int)PAD((len), (boundary));                   \
+  if (!(*(pp)) || (*(pp)) + (skip) > (pend))                \
+      return 0;                                             \
+  while (skip--)                                            \
+      *(*(pp))++ = '\0';                                    \
+} while(0)
+
+/* Encodes an ASCIIZ string into some weird format */
+/* Example "\011\0\012\0""03B\0""\12\0\0\0DynamoRio\0\0\0"
+   CHECK: Is this a documented M$ string representation?
+   It seems like there is a lot of redundancy in the encoding,
+   but it maybe useful in later reincarnations of the message. 
+   This is in fact a plain UNICODE_STRING.
+*/
+static inline 
+char*
+append_string(char **pp /* INOUT */, char *pend, char *str)
+{
+    size_t length = strlen(str) + 1;
+    WORD len;
+
+    ASSERT_TRUNCATE(len, ushort, length); /* ushort == WORD */
+    len = (WORD) length;
+
+    FIELD(*pp, pend, WORD, len-1);
+    FIELD(*pp, pend, WORD, len);
+    FIELD(*pp, pend, void*, str);
+    FIELD(*pp, pend, DWORD, len);
+    VARFIELD(pp, pend, str, len);
+    PADDING(pp, pend, len, sizeof(DWORD));
+    return *pp;
+}
+
+#define HEADER_SIZE 24
+#define HEADER_OFFSET 28
+
+static inline 
+char*
+prepend_header(char *p, char *pend, 
+               char *header, int length, int sequence,
+               DWORD unknown)
+{
+    if (!p) 
+        return NULL;
+    VARFIELD(&p, pend, header, 8);
+    FIELD(p, pend, DWORD, length);
+    FIELD(p, pend, DWORD, sequence);
+    FIELD(p, pend, DWORD, length - HEADER_SIZE);
+    FIELD(p, pend, DWORD, unknown);
+    FIELD(p, pend, DWORD, 0);
+    return p;
+}
+
+/* FIXME: this value needs to be decoded using Ethereal too. See case 5655. */
+#define EVENTLOG "\20\0\0\0"    /* always 16 */
+
+/* The first byte of the hello_message string should be \x05, but this
+ * triggers a false positive in McAfee. That's why that byte is set to
+ * RPC_VERSION_BOGUS and replaced with RPC_VERSION_5 before it is used.
+ * See case 5002 for more details.*/
+
+#define RPC_VERSION_BOGUS   "\xFF"      /* must be a string */
+#define RPC_VERSION_5       '\x05'      /* must be a character */
+
+/* advapi sends this message for several days with different computer names,
+   if I break the protocol then its hello request starts with H\0\0\0\5... */
+static char hello_message[] = /* DCE RPC request, decoded by Ethereal */
+RPC_VERSION_BOGUS   /* Version: Should be 5, but we set it to a bogus value
+                     * because of a false positive in McAfee. See case 5002 */
+"\x00"              /* Version (minor): 0 */
+"\x0B"              /* Packet type: Bind (11) */
+"\x03"              /* Packet Flags: 0x03 */
+
+"\x10\x00\x00\x00"  /* Data Representation: 10000000 */
+"\x48\x00"          /* Frag Length: 72 */
+"\x00\x00"          /* Auth Length: 0 */
+"\x01\x00\x00\x00"  /* Call ID: 1 */
+"\xB8\x10"          /* Max Xmit Frag: 4280 */
+"\xB8\x10"          /* Max Recv Frag: 4280 */
+
+"\x00\x00\x00\x00"  /* Assoc Group: 0x00000000 */
+"\x01\x00\x00\x00"  /* Num Ctx Items: 1 */
+"\x00\x00"          /* Context ID: 0 */
+"\x01\x00"          /* Num Trans Items: 1 */
+"\xDC\x3F\x27\x82\x2A\xE3\xC3\x18" /* Interface UUID: */
+"\x3F\x78\x82\x79\x29\xDC\x23\xEA" /*   82273fdc-e32a-18c3-3f78-827929dc23ea */
+"\x00\x00"          /* Interface Ver: 0 */
+"\x00\x00"          /* Interface Ver Minor: 0 */
+
+"\x04\x5D\x88\x8A\xEB\x1C\xC9\x11" /* Transfer Syntax: */
+"\x9F\xE8\x08\x00\x2B\x10\x48\x60" /*   8a885d04-1ceb-11c9-9fe8-08002b104860 */
+"\x02\x00\x00\x00";
+
+/* We ignore this response to the hello message:
+ *
+ *                      // DCE RPC response, decoded by Ethereal
+ *  "\x05"              // Version: 5
+ *  "\x00"              // Version (minor): 0
+ *  "\x0C"              // Packet type: Bind_ack (12)
+ *  "\x03"              // Packet Flags: 0x03
+ *  "\x10\x00\x00\x00"  // Data Representation: 10000000
+ *  "\x44\x00"          // Frag Length: 68
+ *  "\x00\x00"          // Auth Length: 0
+ *  "\x01\x00\x00\x00"  // Call ID: 1
+ *  "\xB8\x10"          // Max Xmit Frag: 4280
+ *  "\xB8\x10"          // Max Recv Frag: 4280
+ *  "\x54\x13\x01\x00"  // Assoc Group: 0x00011354 (may vary)
+ *  "\x0D\x00"          // Scndry Addr len: 13
+ *  "\\PIPE\\ntsvcs\x00"    // Scndry Addr: \PIPE\ntsvcs
+ *  "\x00"              // for alignment? (gets to 4 byte alignment)
+ *  "\x01\x00\x00\x00"  // Num results: 1
+ *  "\x00\x00\x00\x00"  // Ack result: Acceptance (0)
+ *  "\x04\x5D\x88\x8A\xEB\x1C\xC9\x11" // Transfer Syntax:
+ *  "\x9F\xE8\x08\x00\x2B\x10\x48\x60" //   8a885d04-1ceb-11c9-9fe8-08002b104860
+ *  "\x02\x00\x00\x00"
+ *
+ * Note that the hello response has changed slightly in Vista (from hand
+ * comparison) :
+ * ...
+ * ! "\x0F\x00"          // Scndry Addr len: 15
+ * ! "\\PIPE\\eventlog\x00"    // Scndry Addr: \PIPE\eventlog
+ * ! "\x00\x73\x00"      // for alignment? (gets to 4 byte alignment)
+ * ...
+ */
+
+#define REPORT "\5\0\0\3"
+
+#define REGISTER_UNKNOWN_HEADER *(int*)"\0\0\17\0"
+#define REPORT_UNKNOWN_HEADER *(int*)"\0\0\22\0"
+#define DEREGISTER_UNKNOWN_HEADER *(int*)"\0\0\3\0"
+
+/* TODO: The client can talk to a named pipe server on a remote machine,
+   then we will be able to get messages out even before the local services are started! */
+
+#define EVENTLOG_NAMED_PIPE L"\\??\\PIPE\\EVENTLOG"
+
+// debugging facility 
+
+#ifdef DEBUG
+#define PRINT(form, arg) LOG(GLOBAL, LOG_TOP, 3, form, arg)
+static void
+print_buffer_as_bytes (unsigned char *buf, size_t len)
+{
+    size_t i;
+    int nonprint = 0;
+    PRINT ("%s", "\"");
+    for (i=0; i<len; i++) {
+        if (isdigit (buf[i]) && nonprint) {
+            PRINT ("%s", "\"\""); // to make \01 into \0""1
+        }         
+        if (buf[i] == '\\')
+            PRINT ("%s", "\\");
+
+        if (isprint (buf[i])) {
+            PRINT ("%c", buf[i]);
+            nonprint = 0;
+        } else {
+            PRINT ("\\%o", buf[i]);
+            nonprint = 1;
+        }
+    }
+    PRINT ("%s", "\"");
+    PRINT ("%s", ";\n");
+}
+#undef PRINT
+#endif /* DEBUG */
+
+/* see comments above, the response message length changed in Vista */
+#define HELLO_RESPONSE_LENGTH \
+    (get_os_version() < WINDOWS_VERSION_VISTA ? 68U : 72U)
+#define REGISTER_RESPONSE_LENGTH 48
+#define REPORT_RESPONSE_LENGTH 36
+
+/* returns 0 on failure  */
+/* must hold the eventlog mutex */
+int
+eventlog_register(eventlog_state_t *evconnection)
+{
+    char *p;
+
+    ASSERT(evconnection != NULL);
+    ASSERT_OWN_MUTEX(true, &evconnection->eventlog_mutex);
+
+    evconnection->eventlog_completion = create_iocompletion();
+    evconnection->eventlog_pipe = open_pipe(EVENTLOG_NAMED_PIPE, 
+                                            evconnection->eventlog_completion);
+
+    if (!evconnection->eventlog_pipe) {
+        LOG(GLOBAL, LOG_TOP, 1, "Couldn't open EVENTLOG\n");
+        goto failed_to_register;
+    }
+
+    /* we can't do this in eventlog_init() b/c sometimes eventlog_register()
+     * is called before eventlog_init()
+     */
+    SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+    /* Restore the first byte of hello_message before it is used. It was set
+     * to a bogus value to avoid a McAfee false positive. See case 5002. */
+    hello_message[0] = RPC_VERSION_5;
+    SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+
+    evconnection->request_length = sizeof(hello_message) - 1;
+    evconnection->outlen = nt_pipe_transceive(evconnection->eventlog_pipe, 
+                                              hello_message, 
+                                              evconnection->request_length, 
+                                              evconnection->outbuf, 
+                                              sizeof(evconnection->outbuf),
+                                              DYNAMO_OPTION(eventlog_timeout));
+    DOLOG(2, LOG_TOP, {
+        LOG(GLOBAL, LOG_TOP, 3, "inlen= %d; outlen = "SZFMT"\"\n", 
+            evconnection->request_length, evconnection->outlen);
+        LOG(GLOBAL, LOG_TOP, 3, "char hello[] = ");
+        print_buffer_as_bytes((byte*)hello_message, evconnection->request_length);
+        LOG(GLOBAL, LOG_TOP, 3, "char hello_resp[] = ");
+        print_buffer_as_bytes((byte*)evconnection->outbuf, evconnection->outlen);
+    });
+
+    if (evconnection->outlen != HELLO_RESPONSE_LENGTH) {
+        LOG(GLOBAL, LOG_TOP, 1,
+            "eventlog_register: Mismatch on HELLO_RESPONSE outlen="SZFMT"\n", 
+            evconnection->outlen);
+        goto failed_to_register;
+    }
+
+    /* the only expected message length, we're lenient on contents */
+
+    evconnection->message_seq = 1; /* we start counting from source registration */
+    p = evconnection->buf + HEADER_OFFSET;
+    append_string(&p, evconnection->buf + sizeof(evconnection->buf),  
+                  EVENTSOURCE_NAME);
+
+#define REPORT_IN_LOG "Application"
+    /* CHECK: I don't quite get how the log name here matters for Event Viewer, 
+       since the source is registered only under EVENTLOG_NAME subtree,
+       TODO: yet we may want to have our own event file, and it may matter then.*/
+    append_string(&p, evconnection->buf + sizeof(evconnection->buf),
+                  REPORT_IN_LOG);
+    VARFIELD(&p, evconnection->buf + sizeof(evconnection->buf),
+             "\1\0\0\0\1\0\0\0", 8); /*UNKNOWN*/
+    ASSERT(p); // our buffer should be large enough
+
+    IF_X64(ASSERT_TRUNCATE(evconnection->request_length, int, p - evconnection->buf));
+    evconnection->request_length = (int) (p - evconnection->buf);
+    p = prepend_header(evconnection->buf, 
+                       evconnection->buf + sizeof(evconnection->buf), 
+                       REPORT EVENTLOG, evconnection->request_length, 
+                       evconnection->message_seq, REGISTER_UNKNOWN_HEADER);
+    ASSERT(p);    
+    evconnection->message_seq++;
+
+    evconnection->outlen = nt_pipe_transceive(evconnection->eventlog_pipe, 
+                                              evconnection->buf, 
+                                              evconnection->request_length, 
+                                              evconnection->outbuf, 
+                                              sizeof(evconnection->outbuf),
+                                              DYNAMO_OPTION(eventlog_timeout));
+    DOLOG(2, LOG_TOP, {
+        LOG(GLOBAL, LOG_TOP, 3, "inlen= %d; outlen = "SZFMT"\n", 
+            evconnection->request_length, evconnection->outlen);
+        LOG(GLOBAL, LOG_TOP, 3, "char reg[] = ");
+        print_buffer_as_bytes((byte*)evconnection->buf, evconnection->request_length);
+        LOG(GLOBAL, LOG_TOP, 3, "char reg_resp[] = ");
+        print_buffer_as_bytes((byte*)evconnection->outbuf, evconnection->outlen);
+    });
+
+    /* the only expected message length, we're lenient on contents */
+    if (evconnection->outlen != REGISTER_RESPONSE_LENGTH) {
+        LOG(GLOBAL, LOG_TOP, 1,
+            "eventlog_register: Mismatch on REGISTER_RESPONSE outlen="SZFMT"\n",
+            evconnection->outlen);
+        goto failed_to_register;
+    }
+
+    // we can parse the output to verify its contents, yet we care only about the nonce
+    memcpy(evconnection->nonce, &evconnection->outbuf[HEADER_OFFSET], 
+           NONCE_LENGTH);
+
+    return 1;
+    /* on error */
+failed_to_register:
+    if (evconnection->eventlog_completion) {
+        close_handle(evconnection->eventlog_completion);
+        evconnection->eventlog_completion = 0;
+    }
+    if (evconnection->eventlog_pipe) {
+        close_file(evconnection->eventlog_pipe);
+        evconnection->eventlog_pipe = 0;
+    }
+    return 0;
+
+}
+
+/* get computer name, (cached) */
+char*
+get_computer_name() 
+{
+    static char computer_name[MAX_COMPUTERNAME_LENGTH + 5]; /* 15 + 5 */
+
+    /* returns the Windows name of the current computer just like GetComputerName() */
+    if (!computer_name[0]) {
+        char buf[sizeof(KEY_VALUE_PARTIAL_INFORMATION)+ 
+                 sizeof(wchar_t)*(MAX_COMPUTERNAME_LENGTH + 1)]; // wide 
+        KEY_VALUE_PARTIAL_INFORMATION* kvpi = (KEY_VALUE_PARTIAL_INFORMATION*)buf;
+        
+        if (reg_query_value(L"\\Registry\\Machine\\System\\CurrentControlSet"
+                            L"\\Control\\ComputerName\\ActiveComputerName",
+                            L"ComputerName",
+                            KeyValuePartialInformation,
+                            kvpi,
+                            sizeof(buf), 0) ==
+            REG_QUERY_SUCCESS) {
+            /* Case 8185: this reg key may not be set until after winlogon
+             * starts up, and our first event may be post-init as well once
+             * the eventlog service is up.  So we may need to unprotect
+             * .data here.
+             */
+            if (dynamo_initialized) {
+                ASSERT(check_should_be_protected(DATASEC_RARELY_PROT));
+                SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+            } else
+                ASSERT(!DATASEC_PROTECTED(DATASEC_RARELY_PROT));
+            snprintf(computer_name, sizeof(computer_name) - 1, "%*ls", 
+                     kvpi->DataLength / sizeof(wchar_t) - 1,
+                     (wchar_t*)kvpi->Data);
+            /* computer_name is static so last element is zeroed */
+            if (dynamo_initialized)
+                SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+        }
+    }
+    return computer_name;
+}
+
+/* must hold the eventlog mutex */
+static int
+eventlog_report(eventlog_state_t *evconnection,
+                WORD severity, WORD category, DWORD message_id, 
+                void *pSID,
+                uint substitutions_num, size_t raw_data_size, 
+                char **substitutions, char *raw_data)
+{
+    char *p;
+    uint i;
+
+    ULONG sec = query_time_seconds();
+
+    ASSERT_OWN_MUTEX(true, &evconnection->eventlog_mutex);
+
+    p = evconnection->buf + HEADER_OFFSET;
+    VARFIELD(&p, evconnection->buf + sizeof(evconnection->buf), 
+             evconnection->nonce, NONCE_LENGTH);
+    p -= 4;                     /* overwrite timestamp? */
+    FIELD(p, evconnection->buf + sizeof(evconnection->buf), DWORD, sec);
+    FIELD(p, evconnection->buf + sizeof(evconnection->buf),  WORD, severity);
+    FIELD(p, evconnection->buf + sizeof(evconnection->buf),  WORD, category);
+    FIELD(p, evconnection->buf + sizeof(evconnection->buf), DWORD, message_id);
+    FIELD(p, evconnection->buf + sizeof(evconnection->buf),  WORD, 
+          substitutions_num);
+
+    /* FIXME: should write this constant in hex instead - \333 is not meaningful anyways
+     * with the following broken code we've been writing
+     * $SG23701 DB     0dbH, 'w', 00H
+     *   which is 0x77db.
+     * We should either keep using the magic value that has worked, 
+     * or figure out what should have really been written there.
+     */
+    FIELD(p, evconnection->buf + sizeof(evconnection->buf),  WORD, 
+          *(WORD*)"\333w"); /* FIXME: ReservedFlags? */
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(raw_data_size)));
+    FIELD(p, evconnection->buf + sizeof(evconnection->buf), DWORD, 
+          (DWORD)raw_data_size);
+    append_string(&p, evconnection->buf + sizeof(evconnection->buf),
+                  get_computer_name());
+
+    /* FIXME: This used to be type DWORD: I'm guessing that it should be widened */
+    IF_X64(ASSERT_NOT_TESTED());
+    FIELD(p, evconnection->buf + sizeof(evconnection->buf), void*, pSID);
+    if (pSID) {
+        // FIXME: dump a SID in binary format
+        // FIXME: the actual structure order seems to be 
+        // WORD(sub_authorities_num), 
+        // 48 bit authority value,
+        // sub_authorities_num * ( 48 bit sub-authority values)
+    }
+
+    // FIXME: these don't seem to be either offsets nor pointers
+    // but are some function of the number of substitutions
+
+    FIELD(p, evconnection->buf + sizeof(evconnection->buf), DWORD, 
+          *(DWORD*)"\230y\23\0"); /* FIXME pointer placeholder */
+    FIELD(p, evconnection->buf + sizeof(evconnection->buf), DWORD, 
+          substitutions_num);
+    for(i=0; i<substitutions_num; i++) {
+        /* FIXME unknown pointer placeholder */
+        FIELD(p, evconnection->buf + sizeof(evconnection->buf), DWORD, 
+              *(DWORD*)"\210y\23\0"); 
+    }
+
+    for(i=0; i<substitutions_num; i++) {
+        append_string(&p, evconnection->buf + sizeof(evconnection->buf), 
+                      substitutions[i]);
+    }
+
+    /* just the pointer */
+    /* FIXME: This used to be type DWORD: I'm guessing that it should be widened */
+    IF_X64(ASSERT_NOT_TESTED());
+    FIELD(p, evconnection->buf + sizeof(evconnection->buf), char*, raw_data);
+    FIELD(p, evconnection->buf + sizeof(evconnection->buf), DWORD, 
+          raw_data_size);
+    LOG(GLOBAL, LOG_TOP, 3, "datalen=%d data= %*s\n", raw_data_size,
+        raw_data_size, raw_data);
+    if (raw_data_size) {
+        VARFIELD(&p, evconnection->buf + sizeof(evconnection->buf), raw_data, 
+                 raw_data_size); /* now the data */
+        PADDING(&p, evconnection->buf + sizeof(evconnection->buf), 
+                raw_data_size, sizeof(DWORD));
+    }
+
+    /* FIXME: extra padding
+       It seems like the server can handle more but not less padding */
+    VARFIELD(&p, evconnection->buf + sizeof(evconnection->buf), "\0\0\0\0", 4);
+    VARFIELD(&p, evconnection->buf + sizeof(evconnection->buf), "\0\0\0\0", 4);
+    VARFIELD(&p, evconnection->buf + sizeof(evconnection->buf), "\0\0\0\0", 4);
+    
+    IF_X64(ASSERT_TRUNCATE(evconnection->request_length, int, p - evconnection->buf));
+    evconnection->request_length = (int) (p - evconnection->buf);
+    p = prepend_header(evconnection->buf, 
+                       evconnection->buf + sizeof(evconnection->buf), 
+                       REPORT EVENTLOG, evconnection->request_length, 
+                       evconnection->message_seq, REPORT_UNKNOWN_HEADER);
+    ASSERT(p);    
+
+    evconnection->message_seq++;
+    evconnection->outlen = nt_pipe_transceive(evconnection->eventlog_pipe,
+                                              evconnection->buf, 
+                                              evconnection->request_length, 
+                                              evconnection->outbuf, 
+                                              sizeof(evconnection->outbuf),
+                                              DYNAMO_OPTION(eventlog_timeout));
+
+    DOLOG(2, LOG_TOP, {
+        LOG(GLOBAL, LOG_TOP, 3, "inlen= %d; outlen = "SZFMT"\n", 
+            evconnection->request_length, evconnection->outlen);
+        LOG(GLOBAL, LOG_TOP, 3, "char report[] = ");
+        print_buffer_as_bytes((byte*)evconnection->buf, evconnection->request_length);
+        LOG(GLOBAL, LOG_TOP, 3, "char report_resp[] = ");
+        print_buffer_as_bytes((byte*)evconnection->outbuf, evconnection->outlen);
+        if (evconnection->outbuf[2]=='\3') {
+            LOG(GLOBAL, LOG_TOP, 2, "//5 0 3 3 is bad news\n");
+        }
+    });
+
+    /* the only expected message length, we're lenient on contents */
+    if (evconnection->outlen != REPORT_RESPONSE_LENGTH) {
+        LOG(GLOBAL, LOG_TOP, 1,
+            "WARNING: Mismatch on REPORT_RESPONSE outlen=:"SZFMT"\n", 
+            evconnection->outlen);
+        return 0;
+    }
+
+    return 1;
+}
+
+/* must hold the eventlog mutex */
+uint
+eventlog_deregister(eventlog_state_t *evconnection)
+{
+    char *p;
+    int res;
+
+    ASSERT_OWN_MUTEX(true, &evconnection->eventlog_mutex);
+
+    p = evconnection->buf + HEADER_OFFSET;
+    VARFIELD(&p, evconnection->buf + sizeof(evconnection->buf),
+             evconnection->nonce, NONCE_LENGTH);
+    IF_X64(ASSERT_TRUNCATE(evconnection->request_length, int, p - evconnection->buf));
+    evconnection->request_length = (int) (p - evconnection->buf);
+    p = prepend_header(evconnection->buf, 
+                       evconnection->buf + sizeof(evconnection->buf), 
+                       REPORT EVENTLOG, evconnection->request_length, 
+                       evconnection->message_seq, DEREGISTER_UNKNOWN_HEADER);
+    ASSERT(p);
+
+
+    evconnection->outlen = nt_pipe_transceive(evconnection->eventlog_pipe, 
+                                              evconnection->buf, 
+                                              evconnection->request_length, 
+                                              evconnection->outbuf, 
+                                              sizeof(evconnection->outbuf),
+                                              DYNAMO_OPTION(eventlog_timeout));
+
+    if (evconnection->outlen != /*DE*/REGISTER_RESPONSE_LENGTH) {
+        LOG(GLOBAL, LOG_TOP, 1,
+            "WARNING: Mismatch on DEREGISTER_RESPONSE outlen="SZFMT"\n", 
+            evconnection->outlen);
+    }
+
+    if (evconnection->eventlog_completion) {
+        close_handle(evconnection->eventlog_completion);
+        evconnection->eventlog_completion = 0;
+    }
+
+    ASSERT(evconnection->eventlog_pipe);
+    res = close_file(evconnection->eventlog_pipe);
+    evconnection->eventlog_pipe = 0;
+    return res;
+}
+
+/* Getting a new handle may be not very performant, and also may fail 
+ * at unexpected times we cache session state across messages and across 
+ * threads */
+static eventlog_state_t *shared_eventlog_connection;
+/* we use this if we have to syslog prior to heap being initialized */
+static eventlog_state_t temp_shared_eventlog_connection;
+
+/* separate so it can be called for pre-eventlog_init() syslogs */
+static void
+eventlog_alloc()
+{
+    /* Shouldn't come in here later when would need multi-thread synch.
+     * Sometimes eventlog registration fails until post-init (for lsass, e.g.)
+     * but the alloc should happen during init regardless.
+     */
+    ASSERT(!dynamo_initialized);
+    if (shared_eventlog_connection != NULL &&
+        shared_eventlog_connection != &temp_shared_eventlog_connection) {
+        /* an early syslog was post-heap-init and we are fully initialized */
+        return;
+    }
+    if (!dynamo_heap_initialized) {
+        /* No heap available, so we use our temp static struct.
+         * eventlog_init() will call this routine again and we'll copy
+         * to the heap (else condition below).
+         */
+        ASSERT(shared_eventlog_connection == NULL);
+        shared_eventlog_connection = &temp_shared_eventlog_connection;
+        ASSIGN_INIT_LOCK_FREE(shared_eventlog_connection->eventlog_mutex,
+                              eventlog_mutex);
+    } else {
+        eventlog_state_t *alloc = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, eventlog_state_t,
+                                                ACCT_OTHER, PROTECTED);
+        if (shared_eventlog_connection == &temp_shared_eventlog_connection) {
+            /* transfer from the temp static structure to the heap */
+            memcpy(alloc, &temp_shared_eventlog_connection,
+                   sizeof(temp_shared_eventlog_connection));
+        } else {
+            memset(alloc, 0, sizeof(*shared_eventlog_connection));
+            ASSIGN_INIT_LOCK_FREE(alloc->eventlog_mutex, eventlog_mutex);
+        }
+        shared_eventlog_connection = alloc;
+    }
+}
+
+void 
+eventlog_init() 
+{
+    uint res;
+
+    /* TODO: Check a persistent (registry) counter for the current application 
+     whether to report to the system log on this run, decrement it if present */
+
+    /* syslog_mask is dynamic, so even if 0 now we init in case it changes later */
+
+    /* we call get computer name to make sure it's static buffer is initialized
+     * while we are still single threaded */
+    get_computer_name();
+
+    /* FIXME: We don't actually get our own log as intended  */
+    /* on error we just go in the Application EventLog */
+    if (DYNAMO_OPTION(syslog_init) &&
+        !init_registry_source()) { /* update registry keys */
+        DOLOG_ONCE(1, LOG_TOP, {
+            LOG(GLOBAL, LOG_TOP, 1, "WARNING: Could not add the event source registry keys."
+                "Events are reported with no message files.\n");
+        });
+    }
+
+    /* may have already been allocated for early syslogs */
+    eventlog_alloc();
+
+    mutex_lock(&shared_eventlog_connection->eventlog_mutex);
+    if (!shared_eventlog_connection->eventlog_pipe) {
+        /* initialize thread shared connection */
+        res = eventlog_register(shared_eventlog_connection);
+        if (!res) {
+            LOG(GLOBAL, LOG_TOP, 1, "WARNING: Could not register event source.\n");
+        }
+    }
+    mutex_unlock(&shared_eventlog_connection->eventlog_mutex);
+}
+
+void 
+eventlog_fast_exit(void)
+{
+    uint res = 1;               /* maybe nothing to do */
+    mutex_lock(&shared_eventlog_connection->eventlog_mutex);
+    if (shared_eventlog_connection->eventlog_pipe) 
+        res = eventlog_deregister(shared_eventlog_connection);
+    shared_eventlog_connection->eventlog_pipe = 0;
+    mutex_unlock(&shared_eventlog_connection->eventlog_mutex);
+    DOLOG(1, LOG_TOP, if (!res) {
+        LOG(GLOBAL, LOG_TOP, 1, "WARNING: DeregisterEventSource failed.\n"); 
+    });
+}
+
+void 
+eventlog_slow_exit()
+{
+    ASSERT_CURIOSITY(shared_eventlog_connection->eventlog_pipe == 0 && 
+                     "call after eventlog_fast_exit");
+    /* syslog_mask is dynamic, so even if 0 now we init in case it changes later */
+    DELETE_LOCK(shared_eventlog_connection->eventlog_mutex);
+    ASSERT(shared_eventlog_connection != NULL &&
+           shared_eventlog_connection != &temp_shared_eventlog_connection);
+    HEAP_TYPE_FREE(GLOBAL_DCONTEXT, shared_eventlog_connection, eventlog_state_t,
+                   ACCT_OTHER, PROTECTED);
+    /* try to let syslogs during later cleanup go through
+     * FIXME: won't re-deregister in that case
+     */
+    shared_eventlog_connection = &temp_shared_eventlog_connection;
+    ASSIGN_INIT_LOCK_FREE(shared_eventlog_connection->eventlog_mutex,
+                          eventlog_mutex);
+}
+
+/* writes a message to the Windows Event Log */
+static
+void 
+os_eventlog(syslog_event_type_t priority, uint message_id, 
+            uint substitutions_num, char **arguments,
+            size_t size_data, char *raw_data)
+{
+    WORD native_priority = 0;
+    WORD category = 0;          /* we don't use any */
+    uint res = 0;
+
+
+    /* check mask on event_type whether to log this type of message */
+    if (!TEST(priority, dynamo_options.syslog_mask))
+        return;
+
+    switch (priority) {
+    case SYSLOG_INFORMATION: native_priority = EVENTLOG_INFORMATION_TYPE; break;
+    case SYSLOG_WARNING:     native_priority = EVENTLOG_WARNING_TYPE; break;
+    case SYSLOG_CRITICAL:    /* report as error */
+    case SYSLOG_ERROR:       native_priority = EVENTLOG_ERROR_TYPE; break;
+    default:
+        ASSERT_NOT_REACHED();
+    }
+
+    if (shared_eventlog_connection == NULL)
+        eventlog_alloc();
+    mutex_lock(&shared_eventlog_connection->eventlog_mutex);
+    if (!shared_eventlog_connection->eventlog_pipe) {
+        /* Retry to open connection, since may have been unable 
+           to do that early on for system services started before EventLog */
+        res = eventlog_register(shared_eventlog_connection);
+        if (!res) {
+            DOLOG_ONCE(1, LOG_TOP, {
+                LOG(GLOBAL, LOG_TOP, 1, "WARNING: Could not register event source on second attempt.\n");
+            });
+        } else {
+            LOG(GLOBAL, LOG_TOP, 1, "Registered event source after program started. "
+                "Events may be missing. --ok\n");
+        }
+    }
+
+    if (shared_eventlog_connection->eventlog_pipe) {
+        // TODO: add current user SID (thread maybe impersonated)
+        res = eventlog_report(shared_eventlog_connection,
+                              (WORD)native_priority,
+                              category,             
+                              message_id,
+                              NULL, /* pSID */
+                              (WORD)substitutions_num,
+                              size_data,
+                              arguments,
+                              raw_data);
+    }
+    mutex_unlock(&shared_eventlog_connection->eventlog_mutex);
+    
+    DOLOG(1, LOG_TOP, if (!res) {
+        LOG(GLOBAL, LOG_TOP, 1, "WARNING: Could not report event 0x%x. \n", message_id); 
+    });
+}
+
diff --git a/core/win32/events.mc b/core/win32/events.mc
new file mode 100644
index 000000000..c0588ec94
--- /dev/null
+++ b/core/win32/events.mc
@@ -0,0 +1,510 @@
+;// **********************************************************
+;// Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+;// **********************************************************
+
+;// Redistribution and use in source and binary forms, with or without
+;// modification, are permitted provided that the following conditions are met:
+;// 
+;// * Redistributions of source code must retain the above copyright notice,
+;//   this list of conditions and the following disclaimer.
+;// 
+;// * Redistributions in binary form must reproduce the above copyright notice,
+;//   this list of conditions and the following disclaimer in the documentation
+;//   and/or other materials provided with the distribution.
+;// 
+;// * Neither the name of VMware, Inc. nor the names of its contributors may be
+;//   used to endorse or promote products derived from this software without
+;//   specific prior written permission.
+;// 
+;// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+;// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+;// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+;// ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+;// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+;// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+;// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+;// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+;// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+;// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+;// DAMAGE.
+
+;// Copyright (c) 2003-2007 Determina Corp.
+
+;// events.mc messages for Event logging
+
+;// FIXME: I don't seem to get the !d! format specifiers on Win2k - will need to rework this
+
+;// CHECK: is there a buffer overflow possibility in these facilities?
+;// CHECK: should we always have the * specifier for the width component of a %!s!
+
+;// TODO: We have to figure how to get all of this export something nice for Linux syslog
+
+;// DO not use TABS or COMMAS (,) in the messages 
+;// ADD NEW MESSAGES AT THE END
+
+MessageIdTypedef=uint
+
+;// defaults
+SeverityNames=(
+    Success=0x0
+    Informational=0x1
+    Warning=0x2
+    Error=0x3
+    )
+
+;// FIXME: unclear on can we override these..
+FacilityNames=(
+    DRCore     =0x0FF   ; core messages
+    Security   =0x7FF   ; security violations
+)
+
+;// default
+;//FIXME: can't override - I still want to change the name of the .BIN files 
+;//LanguageNames=(English=1:EVMSG001)
+
+;//;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;// Categories
+;// unused: we may be happy with message types for now
+;// FIXME: if the FacilityName thing works, otherwise we'll use these
+MessageId=0x1
+Severity=Success
+SymbolicName=MSG_CATEGORY_SECURITY
+Language=English
+Security
+.
+
+;//;;;;;;;;;;;;;;;;;;; Core
+;// Info messages
+MessageId = +1000
+Severity = Informational
+Facility = DRCore
+SymbolicName = MSG_INFO_PRODUCT_VERSION
+Language=English
+%1!s! (R) %2!s! (R) %3!s! %4!s! installed.
+.
+
+;// Info messages
+;// NOTE: All message regarding applications should have app name and pid as the
+;// first 2 arguments - otherwise controller will break.
+MessageId = 
+Severity = Informational
+Facility = DRCore
+SymbolicName = MSG_INFO_PROCESS_START
+Language=English
+Starting application %1!s! (%2!s!)
+MD5: %3!s!
+.
+
+;// Avoid the MD5 output for CLIENT_INTERFACE
+MessageId = 
+Severity = Informational
+Facility = DRCore
+SymbolicName = MSG_INFO_PROCESS_START_CLIENT
+Language=English
+Starting application %1!s! (%2!s!)
+.
+
+;// Info messages
+;// empty MessageId value means previous +1
+MessageId = 
+Severity = Informational
+Facility = DRCore
+SymbolicName = MSG_INFO_PROCESS_STOP
+Language=English
+Stopping application %1!s! (%2!s!)
+.
+
+;// Info messages
+MessageId = 
+Severity = Informational
+Facility = DRCore
+SymbolicName = MSG_INFO_DETACHING
+Language=English
+Detaching from application %1!s! (%2!s!)
+.
+
+;//;;;;;;;;;;;;;;;;;;; Security
+
+MessageId = 
+Severity = Informational
+Facility = Security
+SymbolicName = MSG_SEC_VIOLATION_TERMINATED
+Language=English
+A security violation was intercepted in application %1!s! (%2!s!).  
+Threat ID: %3!s!.  
+Program terminated.
+.
+
+MessageId = 
+Severity = Informational
+Facility = Security
+SymbolicName = MSG_SEC_VIOLATION_CONTINUE
+Language=English
+A security violation was intercepted in application %1!s! (%2!s!).  
+Threat ID: %3!s!.  
+Running in test mode - program continuing.
+Your system may have been compromised.
+.
+
+MessageId = 
+Severity = Informational
+Facility = Security
+SymbolicName = MSG_SEC_VIOLATION_THREAD
+Language=English
+A security violation was intercepted in application %1!s! (%2!s!).
+Threat ID: %3!s!.  
+Program continuing after terminating faulty thread. 
+Some application functionality may be lost.
+.
+
+MessageId = 
+Severity = Informational
+Facility = Security
+SymbolicName = MSG_SEC_VIOLATION_EXCEPTION
+Language=English
+A security violation was intercepted in application %1!s! (%2!s!).
+Threat ID: %3!s!.  
+Program continuing after throwing an exception.
+Application may terminate.
+.
+
+MessageId = 
+Severity = Error
+Facility = DRCore
+SymbolicName = MSG_INTERNAL_SYSLOG_CRITICAL
+Language=English
+Application %1!s! (%2!s!).  Internal Critical Error %3!s!
+.
+
+MessageId = 
+Severity = Error
+Facility = DRCore
+SymbolicName = MSG_INTERNAL_SYSLOG_ERROR
+Language=English
+Application %1!s! (%2!s!).  Internal Error %3!s!
+.
+
+MessageId = 
+Severity = Warning
+Facility = DRCore
+SymbolicName = MSG_INTERNAL_SYSLOG_WARNING
+Language=English
+Application %1!s! (%2!s!).  Internal Warning %3!s!
+.
+
+MessageId = 
+Severity = Informational
+Facility = DRCore
+SymbolicName = MSG_INTERNAL_SYSLOG_INFORMATION
+Language=English
+Application %1!s! (%2!s!).  Internal Information %3!s!
+.
+
+MessageId = 
+Severity = Error
+Facility = DRCore
+SymbolicName = MSG_EXCEPTION
+Language=English
+Application %1!s! (%2!s!).  Unrecoverable Error at PC %3!s!.  Program aborted. %4!s!
+.
+
+MessageId = 
+Severity = Error
+Facility = DRCore
+SymbolicName = MSG_CLIENT_EXCEPTION
+Language=English
+Application %1!s! (%2!s!).  Client exception at PC %3!s!.
+.
+
+MessageId = 
+Severity = Error
+Facility = DRCore
+SymbolicName = MSG_OUT_OF_MEMORY
+Language=English
+Application %1!s! (%2!s!).  Out of memory.  Program aborted. Status %3!s! 0x%4!s!
+.
+
+MessageId = 
+Severity = Error
+Facility = DRCore
+SymbolicName = MSG_UNSUPPORTED_APPLICATION
+Language=English
+Application %1!s! (%2!s!) is not supported due to dll %3!s!.  Program aborted.
+.
+
+MessageId = 
+Severity = Warning
+Facility = DRCore
+SymbolicName = MSG_WARNING_REPORT_THRESHOLD
+Language=English
+Application %1!s! (%2!s!) has reached its report threshold.  No more violations will be logged.
+.
+
+MessageId = 
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_ERROR_UNABLE_TO_CREATE_BASEDIR
+Language=English
+Application %1!s! (%2!s!) unable to create directory %3!s! for forensics file
+.
+
+MessageId = 
+Severity = Warning
+Facility = DRCore
+SymbolicName = MSG_WARNING_EMPTY_OR_NONEXISTENT_LOGDIR_KEY
+Language=English
+Application %1!s! (%2!s!) has no directory specified for forensics files
+.
+
+MessageId = 
+Severity = Warning
+Facility = DRCore
+SymbolicName = MSG_UNSUPPORTED_OS_VERSION
+Language=English
+Application %1!s! (%2!s!) is running on an unsupported operating system (%3!s!)
+.
+
+MessageId = 
+Severity = Informational
+Facility = Security
+SymbolicName = MSG_SEC_FORENSICS
+Language=English
+Forensics file for security violation in application %1!s! (%2!s!) created at %3!s!
+.
+
+MessageId = 
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_ERROR_OPTION_TOO_LONG_TO_PARSE
+Language=English
+Application %1!s! (%2!s!) option %3!s! is too long to parse. %4!s!
+.
+
+MessageId = 
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_ERROR_OPTION_BAD_NUMBER_FORMAT
+Language=English
+Application %1!s! (%2!s!). Option parsing error : unrecognized number format %3!s!. %4!s!
+.
+
+MessageId = 
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_ERROR_OPTION_UNKNOWN_SIZE_SPECIFIER
+Language=English
+Application %1!s! (%2!s!). Option parsing error : unrecognized size factor %3!s!. %4!s!
+.
+
+MessageId = 
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_ERROR_OPTION_UNKNOWN_TIME_SPECIFIER
+Language=English
+Application %1!s! (%2!s!). Option parsing error : unrecognized time factor %3!s!. %4!s!
+.
+
+MessageId = 
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_ERROR_OPTION_UNKNOWN
+Language=English
+Application %1!s! (%2!s!). Option parsing error : unknown option %3!s!. %4!s!
+.
+
+MessageId = 
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_BAD_OS_VERSION
+Language=English
+Application %1!s! (%2!s!) %3!%s! does not run on %4!s!
+.
+
+MessageId = 
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_OPTION_VERIFICATION_RECURSION
+Language=English
+Application %1!s! (%2!s!) bad option string, unable to continue.
+.
+
+MessageId = 
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_TAMPERED_NTDLL
+Language=English
+Application %1!s! (%2!s!). System library ntdll.dll has been tampered with, unable to continue.
+.
+
+;#ifdef CHECK_RETURNS_SSE2
+MessageId = 
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_CHECK_RETURNS_SSE2_XMM_USED
+Language=English
+Application %1!s! (%2!s!). Check returns using SSE2 assumption violated, app is using the xmm registers
+.
+
+MessageId = 
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_CHECK_RETURNS_SSE2_REQUIRES_SSE2
+Language=English
+Application %1!s! (%2!s!). Check returns using SSE2 requires that the processor support SSE2
+.
+
+;#endif
+
+;#if defined(CLIENT_INTERFACE) || defined(STRACE_CLIENT_INTERFACE) || defined(DR_APP_EXPORTS)
+MessageId = 
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_EXTERNAL_ERROR
+Language=English
+Application %1!s! (%2!s!) %3!s! usage error : %4!s!
+.
+
+;#endif
+
+;#ifdef LINUX
+;// is ok to use x! d!, this won't be used on windows
+MessageId = 
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_SIGSEGV_IN_SECURE_CORE
+Language=English
+Application %1!s! (%2!s!) 
+** Received SIG%3!s! at%4!s! %5!s! pc 0x%6!x! in thread %7!d!
+.
+
+;#endif
+
+MessageId = 
+Severity = Informational
+Facility = Security
+SymbolicName = MSG_LDMP
+Language=English
+Core dump file for application %1!s! (%2!s!) created at %3!s!
+.
+
+;#ifdef HOT_PATCHING_INTERFACE
+MessageId = 
+Severity = Informational
+Facility = Security
+SymbolicName = MSG_HOT_PATCH_VIOLATION
+Language=English
+A security violation was intercepted in application %1!s! (%2!s!).  
+Threat ID: %3!s!.
+.
+
+MessageID =
+Severity = Error
+Facility = DRCore
+SymbolicName = MSG_HOT_PATCH_FAILURE
+Language=English
+A LiveShield Sentry failure was intercepted in application %1!s! (%2!s!) at address %3!s!.
+.
+
+;#endif
+
+MessageId = 
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_ERROR_REGISTRY_PARAMETER_TOO_LONG
+Language=English
+Application %1!s! (%2!s!). Error reading registry : registry parameter %3!s! exceeds maximum length.
+.
+
+;// FIXME - do we want a more cryptic error messages (such as the out of memory one) that
+;// requires going back to Determina for resolution, or do we want the message to be 
+;// potentially actionable by the customer? FIXME - warning or error?
+MessageId = 
+Severity = Warning
+Facility = DRCore
+SymbolicName = MSG_LOW_ON_VMM_MEMORY
+Language=English
+Application %1!s! (%2!s!).  Potentially thrashing on low virtual memory; suggest increasing the -vm_size option for this application.
+.
+MessageId = 
+Severity = Warning
+Facility = DRCore
+SymbolicName = MSG_LOW_ON_COMMITTABLE_MEMORY
+Language=English
+Application %1!s! (%2!s!).  Potentially thrashing on low available system memory; suggest increasing initial pagefile size or adding additional RAM to this machine.
+.
+
+;#ifdef PROCESS_CONTROL
+MessageId =
+Severity = Error
+Facility = DRCore
+SymbolicName = MSG_PROC_CTL_HASH_LIST_TOO_LONG
+Language = English
+Application %1!s! (%2!s!). Process control hash list %3!s! has more than %4!s! MD5 hashes, so process control has been disable.  Try doubling -pc_num_hashes to start process control again.
+.
+;#endif
+
+;#ifdef X64
+MessageId =
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_HEAP_CONTRAINTS_UNSATISFIABLE
+Language=English
+Application %1!s! (%2!s!). Unable to place the heap in a manner that satisfies all 32bit displacement requirements.  Check -vm_base, -vm_offset, -heap_in_lower_4GB, and dll prefferred base addresses for issues.
+.
+
+MessageId =
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_UNSUPPORTED_PROCESSOR_LAHF
+Language=English
+Application %1!s! (%2!s!). Unsupported processor: LAHF/SAHF instructions required in 64-bit mode.
+.
+;#endif
+
+;#ifdef CLIENT_INTERFACE
+MessageId =
+Severity = WARNING
+Facility = DRCore
+SymbolicName = MSG_CLIENT_VERSION_INCOMPATIBLE
+Language=English
+Application %1!s! (%2!s!). Client library targets an incompatible API version and should be re-compiled.
+.
+;#endif
+
+;#ifdef CLIENT_INTERFACE
+MessageId =
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_INSTRUMENTATION_TOO_LARGE
+Language=English
+Application %1!s! (%2!s!). Basic block or trace instrumentation exceeded maximum size.  Try lowering -max_bb_instrs and/or -max_trace_bbs.
+.
+;#endif
+
+MessageId = 
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_SYSENTER_NOT_SUPPORTED
+Language=English
+Application %1!s! (%2!s!). System calls using sysenter are not supported on this operating system.
+.
+
+MessageId = 
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_ESX_VERSION_MISMATCH
+Language=English
+Application %1!s! (%2!s!). Please verify that the DynamoRIO build corresponds to the ESX/Visor build you are using.
+.
+
+;#ifdef CLIENT_INTERFACE
+MessageId =
+Severity = ERROR
+Facility = DRCore
+SymbolicName = MSG_WAITING_FOR_DEBUGGER
+Language=English
+Application %1!s! (%2!s!). Waiting for debugger to attach.
+.
+;#endif
+
+;// ADD NEW MESSAGES HERE
diff --git a/core/win32/gbop.h b/core/win32/gbop.h
new file mode 100644
index 000000000..f24299740
--- /dev/null
+++ b/core/win32/gbop.h
@@ -0,0 +1,327 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2006-2007 Determina Corp. */
+
+/*
+ * gbop.h - GBOP hook definitions
+ *
+ * Plan to implement only for win32.
+ *
+ */
+
+#ifndef GBOP_H
+#define GBOP_H
+
+/* Attempting to keep GBOP hook policies more flexbile, though the
+ * right place will be in a hotpatch policy file.
+ */
+
+/* Support for gbop_include_set */
+/* GBOP_DEFINE_HOOK and GBOP_DEFINE_HOOK_MODULE should be defined as
+ * appropriate by users of hook definitions, note that module name for
+ * each list is specified by the user since some may need different
+ * variants of the name.
+ */
+
+/* note that we currently do only shallow GBOP checking therefore we
+ * need all entry points even if they all eventually call a common
+ * exported implementation that we also hook */
+
+/* Creating processes, modifying files, or creating new network
+ * connections are the primary behaviors that need to be watched.
+ * More generic shellcodes will be stopped while trying to setup their
+ * plumbing when the more generic ones use LoadLibrary/GetProcAddress.
+ * To minimize overhead due to our checks they shouldn't be added to
+ * hot routines, and should be best positioned to include the
+ * initializing call of some facility, but not the more common methods:
+ * e.g. hook socket() but not necessarily recv().
+ */
+
+
+/* KERNEL32.dll base */
+/* likely targets for generic shellcode */
+/* mostly in alphabetic order */
+#define GBOP_DEFINE_KERNEL32_BASE_HOOKS(module) /* "KERNEL32.dll" */    \
+    GBOP_DEFINE_HOOK(module, CreateFileA)                               \
+    GBOP_DEFINE_HOOK(module, CreateFileW)                               \
+    GBOP_DEFINE_HOOK(module, CreateProcessA)                            \
+    GBOP_DEFINE_HOOK(module, CreateProcessInternalA)                    \
+    GBOP_DEFINE_HOOK(module, CreateProcessInternalW)                    \
+    GBOP_DEFINE_HOOK(module, CreateProcessW)                            \
+    /* kernel32!CreateProcessInternalWSecure is a RET on XP SP2 */      \
+    GBOP_DEFINE_HOOK(module, CreateRemoteThread)                        \
+    GBOP_DEFINE_HOOK(module, CreateThread)                              \
+    GBOP_DEFINE_HOOK(module, FreeLibrary)                               \
+                                                                        \
+    GBOP_DEFINE_HOOK(module, GetModuleHandleA)                          \
+    GBOP_DEFINE_HOOK(module, GetModuleHandleW)                          \
+    GBOP_DEFINE_HOOK(module, GetModuleHandleExW)                        \
+    GBOP_DEFINE_HOOK(module, GetModuleHandleExA)                        \
+                                                                        \
+    GBOP_DEFINE_HOOK(module, GetProcAddress)                            \
+    GBOP_DEFINE_HOOK(module, LoadLibraryA)                              \
+    GBOP_DEFINE_HOOK(module, LoadLibraryExA)                            \
+    GBOP_DEFINE_HOOK(module, LoadLibraryExW)                            \
+    GBOP_DEFINE_HOOK(module, LoadLibraryW)                              \
+    GBOP_DEFINE_HOOK(module, LoadModule)                                \
+    /* wrapper around CreateProcess */                                  \
+                                                                        \
+    GBOP_DEFINE_HOOK(module, OpenProcess)                               \
+    GBOP_DEFINE_HOOK(module, VirtualAlloc)                              \
+    GBOP_DEFINE_HOOK(module, VirtualAllocEx)                            \
+    GBOP_DEFINE_HOOK(module, VirtualProtect)                            \
+    GBOP_DEFINE_HOOK(module, VirtualProtectEx)                          \
+    GBOP_DEFINE_HOOK(module, WinExec)                                   \
+    /* wrapper around CreateProcess */                                  \
+                                                                        \
+    GBOP_DEFINE_HOOK(module, WriteProcessMemory)
+
+/* KERNEL32.dll complete set */
+/* adding the less likely to be used by a generic exploit for
+ * completeness, yet short of KERNEL32!*
+ */
+#define GBOP_DEFINE_KERNEL32_MORE_HOOKS(module) /* "KERNEL32.dll" */    \
+    GBOP_DEFINE_HOOK(module, CopyFileA)                                 \
+    GBOP_DEFINE_HOOK(module, CopyFileW)                                 \
+    GBOP_DEFINE_HOOK(module, CopyFileExA)                               \
+    GBOP_DEFINE_HOOK(module, CopyFileExW)                               \
+    GBOP_DEFINE_HOOK(module, CreatePipe)                                \
+    GBOP_DEFINE_HOOK(module, CreateNamedPipeA)                          \
+    GBOP_DEFINE_HOOK(module, CreateNamedPipeW)                          \
+                                                                        \
+    GBOP_DEFINE_HOOK(module, CreateDirectoryA) /* low risk */           \
+    GBOP_DEFINE_HOOK(module, CreateDirectoryExA)                        \
+    GBOP_DEFINE_HOOK(module, CreateDirectoryExW)                        \
+    GBOP_DEFINE_HOOK(module, CreateDirectoryW)                          \
+                                                                        \
+    GBOP_DEFINE_HOOK(module, DeleteFileA)                               \
+    GBOP_DEFINE_HOOK(module, DeleteFileW)                               \
+                                                                        \
+    GBOP_DEFINE_HOOK(module, ExitProcess)                               \
+                                                                        \
+    GBOP_DEFINE_HOOK(module, GetStartupInfoA)                           \
+    GBOP_DEFINE_HOOK(module, GetStartupInfoW)                           \
+                                                                        \
+    GBOP_DEFINE_HOOK(module, LZCreateFileW)                             \
+     /* may be used to overwrite a file */                              \
+     /* note there is no LZCreateFileA version */                       \
+    /* skipping LZOpenFile[AW] which open only compressed files */      \
+                                                                        \
+    GBOP_DEFINE_HOOK(module, MoveFileA)                                 \
+    GBOP_DEFINE_HOOK(module, MoveFileExA)                               \
+    GBOP_DEFINE_HOOK(module, MoveFileExW)                               \
+    GBOP_DEFINE_HOOK(module, MoveFileW)                                 \
+    GBOP_DEFINE_HOOK(module, MoveFileWithProgressA)                     \
+    GBOP_DEFINE_HOOK(module, MoveFileWithProgressW)                     \
+                                                                        \
+    GBOP_DEFINE_HOOK(module, OpenFile)                                  \
+    GBOP_DEFINE_HOOK(module, OpenDataFile)                              \
+                                                                        \
+    GBOP_DEFINE_HOOK(module, PeekNamedPipe)                             \
+    GBOP_DEFINE_HOOK(module, PrivCopyFileExW)                           \
+                                                                        \
+    /* skipping ReadFile, though hooked by others */                    \
+    GBOP_DEFINE_HOOK(module, ReplaceFileA)                              \
+    GBOP_DEFINE_HOOK(module, ReplaceFileW)                              \
+     /* skipping kernel32!RemoveDirectoryW on an empty directory */     \
+                                                                        \
+    GBOP_DEFINE_HOOK(module, SetEndOfFile)                              \
+        /* interesting if a handle is open */                           \
+    GBOP_DEFINE_HOOK(module, WriteFile)                                 \
+     /* FIXME: performance, interesting if a handle is open */          \
+    GBOP_DEFINE_HOOK(module, WriteFileEx)                               \
+     /* FIXME: performance, interesting if a handle is open */          \
+    /* KERNEL32 presumed to be complete */
+
+#define GBOP_DEFINE_WININET_BASE_HOOKS(module) /* "WININET.dll" */      \
+    GBOP_DEFINE_HOOK(module, FtpGetFileA)                               \
+        /* risk of creating a local file */                             \
+    GBOP_DEFINE_HOOK(module, InternetConnectA)                          \
+    GBOP_DEFINE_HOOK(module, InternetConnectW)                          \
+    GBOP_DEFINE_HOOK(module, InternetOpenA)                             \
+    GBOP_DEFINE_HOOK(module, InternetOpenUrlA)                          \
+    GBOP_DEFINE_HOOK(module, InternetOpenUrlW)                          \
+    GBOP_DEFINE_HOOK(module, InternetOpenW)
+    /* InternetReadFile needs a handle created by the above */
+    /* WININET presumed to be complete */
+
+#define GBOP_DEFINE_MSVCRT_BASE_HOOKS(module) /* "MSVCRT.dll" */        \
+    GBOP_DEFINE_HOOK(module, system)                                    \
+
+/* FIXME: case 8006 currently unused */
+#define GBOP_DEFINE_MSVCRT_MORE_HOOKS(module) /* "MSVCRT.dll" */        \
+    GBOP_DEFINE_HOOK(module, _execl)                                    \
+    GBOP_DEFINE_HOOK(module, _execle)                                   \
+    GBOP_DEFINE_HOOK(module, _execlp)                                   \
+    GBOP_DEFINE_HOOK(module, _execlpe)                                  \
+    GBOP_DEFINE_HOOK(module, _execv)                                    \
+    GBOP_DEFINE_HOOK(module, _execve)                                   \
+    GBOP_DEFINE_HOOK(module, _execvp)                                   \
+    GBOP_DEFINE_HOOK(module, _execvpe)                                  \
+                                                                        \
+    GBOP_DEFINE_HOOK(module, _getdllprocaddr)                           \
+    GBOP_DEFINE_HOOK(module, _loaddll)                                  \
+    GBOP_DEFINE_HOOK(module, _popen  /* process+pipe open */)           \
+                                                                        \
+    GBOP_DEFINE_HOOK(module, _spawnl)                                   \
+    GBOP_DEFINE_HOOK(module, _spawnle)                                  \
+    GBOP_DEFINE_HOOK(module, _spawnlp)                                  \
+    GBOP_DEFINE_HOOK(module, _spawnlpe)                                 \
+    GBOP_DEFINE_HOOK(module, _spawnv)                                   \
+    GBOP_DEFINE_HOOK(module, _spawnve)                                  \
+    GBOP_DEFINE_HOOK(module, _spawnvp)                                  \
+    GBOP_DEFINE_HOOK(module, _spawnvpe)                                 \
+                                                                        \
+    GBOP_DEFINE_HOOK(module, _wexecl)                                   \
+    GBOP_DEFINE_HOOK(module, _wexecle)                                  \
+    GBOP_DEFINE_HOOK(module, _wexeclp)                                  \
+    GBOP_DEFINE_HOOK(module, _wexeclpe)                                 \
+    GBOP_DEFINE_HOOK(module, _wexecv)                                   \
+    GBOP_DEFINE_HOOK(module, _wexecve)                                  \
+    GBOP_DEFINE_HOOK(module, _wexecvp)                                  \
+    GBOP_DEFINE_HOOK(module, _wexecvpe)                                 \
+    GBOP_DEFINE_HOOK(module, _wspawnl)                                  \
+    GBOP_DEFINE_HOOK(module, _wspawnle)                                 \
+    GBOP_DEFINE_HOOK(module, _wspawnlp)                                 \
+    GBOP_DEFINE_HOOK(module, _wspawnlpe)                                \
+    GBOP_DEFINE_HOOK(module, _wspawnv)                                  \
+    GBOP_DEFINE_HOOK(module, _wspawnve)                                 \
+    GBOP_DEFINE_HOOK(module, _wspawnvp)                                 \
+    GBOP_DEFINE_HOOK(module, _wspawnvpe)                                \
+    GBOP_DEFINE_HOOK(module, _wsystem)                                  \
+    /* FIXME: more file creation related to add */
+
+#define GBOP_DEFINE_WS2_32_BASE_HOOKS(module) /* "WS2_32.dll" */        \
+    GBOP_DEFINE_HOOK(module, WSASocketA)                                \
+    GBOP_DEFINE_HOOK(module, WSASocketW)                                \
+    GBOP_DEFINE_HOOK(module, bind)                                      \
+    GBOP_DEFINE_HOOK(module, getpeername)                               \
+    GBOP_DEFINE_HOOK(module, socket)
+    /* most operations create a new socket(), or reuse one */
+
+/* FIXME: case 8006 currently unused in default gbop_include_set */
+#define GBOP_DEFINE_WS2_32_MORE_HOOKS(module) /* "WS2_32.dll" */        \
+    GBOP_DEFINE_HOOK(module, connect)                                   \
+    GBOP_DEFINE_HOOK(module, listen)                                    \
+    GBOP_DEFINE_HOOK(module, WSAConnect)                                \
+    /* not adding the send family: WSASend, WSASendTo, send, sendto */
+
+/* note that WSOCK32.dll is WinSock 1.1 and WS2_32.DLL is WinSock 2,
+ * yet all interesting routines that we hook of identical names are
+ * simply forwarded from WSOCK32.dll to WS2_32.DLL so our hooks there
+ * are sufficient.
+ */
+
+#define GBOP_DEFINE_USER32_BASE_HOOKS(module) /* "USER32.dll" */        \
+    GBOP_DEFINE_HOOK(module, MessageBoxIndirectA)                       \
+    GBOP_DEFINE_HOOK(module, MessageBoxA)                               \
+    GBOP_DEFINE_HOOK(module, MessageBoxExW)                             \
+    GBOP_DEFINE_HOOK(module, MessageBoxExA)                             \
+    GBOP_DEFINE_HOOK(module, MessageBoxTimeoutW)                        \
+    GBOP_DEFINE_HOOK(module, MessageBoxTimeoutA)                        \
+    GBOP_DEFINE_HOOK(module, MessageBoxIndirectW)                       \
+    GBOP_DEFINE_HOOK(module, MessageBoxW)                               \
+    GBOP_DEFINE_HOOK(module, ExitWindowsEx)
+    /* USER32 presumed to be complete */
+
+/* FIXME: case 8006 currently unused in default gbop_include_set */
+#define GBOP_DEFINE_SHELL32_BASE_HOOKS(module) /* "SHELL32.dll" */      \
+    GBOP_DEFINE_HOOK(module, RealShellExecuteA)                         \
+    GBOP_DEFINE_HOOK(module, RealShellExecuteExA)                       \
+    GBOP_DEFINE_HOOK(module, RealShellExecuteExW)                       \
+    GBOP_DEFINE_HOOK(module, RealShellExecuteW)                         \
+    GBOP_DEFINE_HOOK(module, SHCreateDirectory)                         \
+    GBOP_DEFINE_HOOK(module, SHCreateDirectoryExA)                      \
+    GBOP_DEFINE_HOOK(module, SHCreateProcessAsUserW)                    \
+    /* SHFileOperation is an alias for SHFileOperationA on XP SP2 */    \
+    GBOP_DEFINE_HOOK(module, SHFileOperationA)                          \
+    GBOP_DEFINE_HOOK(module, SHFileOperationW)                          \
+    GBOP_DEFINE_HOOK(module, ShellExec_RunDLLA)                         \
+    GBOP_DEFINE_HOOK(module, ShellExec_RunDLLW)                         \
+    GBOP_DEFINE_HOOK(module, ShellExecuteA)                             \
+    GBOP_DEFINE_HOOK(module, ShellExecuteExA)                           \
+    GBOP_DEFINE_HOOK(module, ShellExecuteExW)                           \
+    GBOP_DEFINE_HOOK(module, ShellExecuteW)                             \
+    GBOP_DEFINE_HOOK(module, WOWShellExecute)
+
+/* caution these shouldn't overlap with hooks that DR normally has. */
+/* Fortunately hotp_only doesn't allow duplicate hooks so it will
+ * detect any such overlap. */
+#define GBOP_DEFINE_NTDLL_MORE_HOOKS(module) /* "ntdll.dll" */  \
+    GBOP_DEFINE_HOOK(module, LdrGetDllHandle)                   \
+    GBOP_DEFINE_HOOK(module, LdrGetDllHandleEx)                 \
+    GBOP_DEFINE_HOOK(module, LdrGetProcedureAddress)            \
+    GBOP_DEFINE_HOOK(module, NtCreateFile)                      \
+    GBOP_DEFINE_HOOK(module, NtCreateKey)                       \
+    GBOP_DEFINE_HOOK(module, NtCreateToken)                     \
+    GBOP_DEFINE_HOOK(module, NtDeleteKey)                       \
+    GBOP_DEFINE_HOOK(module, NtDeleteValueKey)                  \
+    GBOP_DEFINE_HOOK(module, NtSetValueKey)                     \
+    GBOP_DEFINE_HOOK(module, NtShutdownSystem)                  \
+    GBOP_DEFINE_HOOK(module, NtWriteVirtualMemory)
+    /* FIXME: case 8006 have to add a complete set for NTDLL.DLL */
+
+/* FIXME: case 8006, need to add ADVAPI32 wrappers as
+ * ADVAPI32!CreateProcessAsUser* 
+ */
+
+/* FIXME: I am not sure if this will scale to the number of modules -
+ * since there is a limit on the macro size (at least in cl) */
+/* all users are expected to define appropriately
+ * GBOP_DEFINE_HOOK_MODULE and GBOP_DEFINE_HOOK 
+ */
+#define GBOP_ALL_HOOKS                                  \
+   /* counting hooks that we have without hotp */       \
+                /* GBOP_SET_NTDLL_BASE         0x1 */   \
+    GBOP_DEFINE_HOOK_MODULE(KERNEL32, BASE) /* 0x2 */   \
+    GBOP_DEFINE_HOOK_MODULE(MSVCRT, BASE)   /* 0x4 */   \
+    GBOP_DEFINE_HOOK_MODULE(WS2_32, BASE)   /* 0x8 */   \
+    GBOP_DEFINE_HOOK_MODULE(WININET, BASE)  /*0x10 */   \
+    /* case 8006: gbop_include_set defaults stop here */\
+    GBOP_DEFINE_HOOK_MODULE(USER32, BASE)  /* 0x20 */   \
+    GBOP_DEFINE_HOOK_MODULE(SHELL32, BASE) /* 0x40 */   \
+                                                        \
+    GBOP_DEFINE_HOOK_MODULE(NTDLL, MORE)  /* 0x100 */   \
+    GBOP_DEFINE_HOOK_MODULE(KERNEL32, MORE)/*0x200 */   \
+    GBOP_DEFINE_HOOK_MODULE(MSVCRT, MORE) /* 0x400 */   \
+    GBOP_DEFINE_HOOK_MODULE(WS2_32, MORE)  /*0x800 */   \
+    /* FIXME: need GBOP_DEFINE_NTDLL_MORE_HOOKS */
+
+enum {
+    GBOP_SET_NTDLL_BASE         = 0x1,
+    /* note that all other flags used in -gbop_set have the same bit
+     * position equal to their order in GBOP_ALL_HOOKS above
+     */
+};
+
+#endif /* GBOP_H */
diff --git a/core/win32/inject.c b/core/win32/inject.c
new file mode 100644
index 000000000..51e238dd4
--- /dev/null
+++ b/core/win32/inject.c
@@ -0,0 +1,772 @@
+/* **********************************************************
+ * Copyright (c) 2000-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * inject.c - injects dynamo into a new thread
+ */
+
+/* FIXME: Unicode support?!?! case 61 */
+#include "globals.h"            /* for pragma warning's and assert defines */
+#include "module_shared.h"      /* for get_proc_address() */
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "ntdll.h"  /* for get/set context etc. */
+#include "os_private.h"  /* for load_dynamo */
+
+#ifndef NOT_DYNAMORIO_CORE_PROPER
+# include "os_private.h" /* for get_proc_address() */
+# define GET_PROC_ADDR get_proc_address
+#else
+# define GET_PROC_ADDR GetProcAddress
+#endif
+
+/* this entry point is hardcoded, FIXME : abstract */
+#define DYNAMORIO_ENTRY "dynamo_auto_start"
+
+#ifdef DEBUG
+/* for asserts, we import globals.h now (for pragmas) so don't need to 
+ * duplicate assert defines, declarations */
+extern void display_error(char *msg);
+#else
+# define display_error(msg) ((void) 0)
+#endif
+
+/* get_module_handle is unsafe to call at arbitrary point from the core so move
+ * all uses in inject.c to separate init function which can be called at a safe
+ * point */
+static ptr_uint_t addr_getprocaddr;
+static ptr_uint_t addr_loadlibrarya;
+#ifdef LOAD_DYNAMO_DEBUGBREAK
+static ptr_uint_t addr_debugbreak;
+#endif
+static bool inject_initialized = false;
+void
+inject_init()
+{
+    HANDLE kern32 = get_module_handle(L"KERNEL32.DLL");
+    ASSERT(kern32 != NULL);
+    addr_getprocaddr = (ptr_uint_t) GET_PROC_ADDR(kern32, "GetProcAddress");
+    ASSERT(addr_getprocaddr != 0);
+    addr_loadlibrarya = (ptr_uint_t) GET_PROC_ADDR(kern32, "LoadLibraryA");
+    ASSERT(addr_loadlibrarya != 0);
+# ifdef LOAD_DYNAMO_DEBUGBREAK
+    addr_debugbreak = (ptr_uint_t) GET_PROC_ADDR(kern32, "DebugBreak");
+    ASSERT(addr_debugbreak != NULL);
+# endif
+    inject_initialized = true;
+}
+
+/* change this if load_dynamo changes 
+ * 128 is more than enough room even with all debugging code in there
+ */
+#define SIZE_OF_LOAD_DYNAMO 128
+
+/* pass non-NULL for thandle if you want this routine to use
+ *   Get/SetThreadContext to get the context -- you must still pass
+ *   in a pointer to a cxt
+ */
+BOOL
+inject_into_thread(HANDLE phandle, CONTEXT *cxt, HANDLE thandle,
+                   char *dynamo_path)
+{
+    size_t              nbytes;
+    BOOL                success = FALSE;
+    ptr_uint_t          dynamo_entry_esp;
+    ptr_uint_t          dynamo_path_esp;
+    LPVOID              load_dynamo_code = NULL; /* = base of code allocation */
+    ptr_uint_t          addr;
+    reg_t               *bufptr;
+    char                buf[MAX_PATH];
+    uint                old_prot;
+
+    ASSERT(cxt != NULL);
+
+#ifndef NOT_DYNAMORIO_CORE_PROPER
+    /* FIXME - if we were early injected we couldn't call inject_init during
+     * startup because kernel32 wasn't loaded yet, so we call it here which
+     * isn't safe because it uses app locks. If we want to support a mix
+     * of early and late follow children injection we should change load_dynamo
+     * to use Nt functions (which we can link) rather then kernel32 functions
+     * (which we have to look up).  We could also use module.c code to safely
+     * walk the exports of kernel32.dll (we can cache its mod handle when it
+     * is loaded). */ 
+    if (!inject_initialized) {
+        SYSLOG_INTERNAL_WARNING("Using late inject follow children from early injected process, unsafe LdrLock usage");
+        SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+        inject_init();
+        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+    }
+#else
+    ASSERT(inject_initialized);
+#endif
+
+    /* soon we'll start using alternative injection with case 102 - leaving block */
+    {
+        reg_t app_xsp;
+        if (thandle != NULL) {
+            /* grab the context of the app's main thread */                 
+            cxt->ContextFlags = CONTEXT_DR_STATE;
+            if (!NT_SUCCESS(nt_get_context(thandle, cxt))) {
+                display_error("GetThreadContext failed");
+                goto error;
+            }
+        }
+        app_xsp = cxt->CXT_XSP;
+
+        /* copy load_dynamo() into the address space of the new process */
+        ASSERT(BUFFER_SIZE_BYTES(buf) > SIZE_OF_LOAD_DYNAMO);
+        memcpy(buf, (char*)load_dynamo, SIZE_OF_LOAD_DYNAMO);
+        /* R-X protection is adequate for our non-self modifying code,
+         * and we'll update that after we're done with
+         * nt_write_virtual_memory() calls */
+
+        /* get allocation, this will be freed by os_heap_free, so make sure
+         * is compatible allocation method */
+        if (!NT_SUCCESS(nt_remote_allocate_virtual_memory(phandle, &load_dynamo_code, 
+                                                          SIZE_OF_LOAD_DYNAMO,
+                                                          PAGE_EXECUTE_READWRITE,
+                                                          MEMORY_COMMIT))) {
+            display_error("Failed to allocate memory for injection code");
+            goto error;
+        }
+        if (!nt_write_virtual_memory(phandle, load_dynamo_code, buf,
+                                     SIZE_OF_LOAD_DYNAMO, &nbytes)) {
+            display_error("WriteMemory failed");
+            goto error;
+        }
+
+        /* Xref PR 252745 & PR 252008 - we can use the app's stack to hold our data
+         * even on WOW64 and 64-bit since we're using set context to set xsp. */
+   
+        /* copy the DYNAMORIO_ENTRY string to the app's stack */
+        _snprintf(buf, BUFFER_SIZE_ELEMENTS(buf), "%s", DYNAMORIO_ENTRY);
+        NULL_TERMINATE_BUFFER(buf);
+        nbytes = strlen(buf) + 1; // include the trailing '\0'
+        /* keep esp at pointer-sized alignment */
+        cxt->CXT_XSP -= ALIGN_FORWARD(nbytes, XSP_SZ);
+        dynamo_entry_esp = cxt->CXT_XSP;
+        if (!nt_write_virtual_memory(phandle, (LPVOID)cxt->CXT_XSP, 
+                                     buf, nbytes, &nbytes)) {
+            display_error("WriteMemory failed");
+            goto error;
+        }
+
+        /* copy the dynamorio_path string to the app's stack */
+        _snprintf(buf, BUFFER_SIZE_ELEMENTS(buf), "%s", dynamo_path);
+        NULL_TERMINATE_BUFFER(buf);
+        nbytes = strlen(buf) + 1; // include the trailing '\0'
+        /* keep esp at pointer-sized byte alignment */
+        cxt->CXT_XSP -= ALIGN_FORWARD(nbytes, XSP_SZ);
+        dynamo_path_esp = cxt->CXT_XSP;
+        if (!nt_write_virtual_memory(phandle, (LPVOID)cxt->CXT_XSP, 
+                                     buf, nbytes, &nbytes)) {
+            display_error("WriteMemory failed");
+            goto error;
+        }
+
+        /* copy the current context to the app's stack. Only need the
+         * control registers, so we use a dr_mcontext_t layout.
+         */
+        bufptr = (reg_t*) buf;
+        *bufptr++ = cxt->CXT_XDI;
+        *bufptr++ = cxt->CXT_XSI;
+        *bufptr++ = cxt->CXT_XBP;
+        *bufptr++ = app_xsp;
+        *bufptr++ = cxt->CXT_XBX;
+        *bufptr++ = cxt->CXT_XDX;
+        *bufptr++ = cxt->CXT_XCX;
+        *bufptr++ = cxt->CXT_XAX;
+#ifdef X64
+        *bufptr++ = cxt->R8;
+        *bufptr++ = cxt->R9;
+        *bufptr++ = cxt->R10;
+        *bufptr++ = cxt->R11;
+        *bufptr++ = cxt->R12;
+        *bufptr++ = cxt->R13;
+        *bufptr++ = cxt->R14;
+        *bufptr++ = cxt->R15;
+#endif
+        /* It would be nice to use preserve_xmm_caller_saved(), but we'd need to
+         * link proc.c and deal w/ messy dependencies to get it into arch_exports.h,
+         * so we do our own check.  We go ahead and put in the xmm slots even
+         * if the underlying processor has no xmm support: no harm done.
+         */
+        if (IF_X64_ELSE(true, is_wow64_process(NT_CURRENT_PROCESS))) {
+            /* PR 264138: preserve xmm0-5.  We fill in all slots even though
+             * for 32-bit we don't use them (PR 306394).
+             */
+            int i, j;
+            for (i = 0; i < NUM_XMM_SLOTS; i++) {
+                for (j = 0; j < IF_X64_ELSE(2,4); j++) {
+                    *bufptr++ = CXT_XMM(cxt, i)->reg[j];
+                }
+            }
+        } else {
+            /* skip xmm slots */
+            bufptr += XMM_SLOTS_SIZE/sizeof(*bufptr);
+        }
+        *bufptr++ = cxt->CXT_XFLAGS;
+        *bufptr++ = cxt->CXT_XIP;
+        ASSERT((char *)bufptr - (char *)buf == sizeof(dr_mcontext_t));
+        *bufptr++ = (ptr_uint_t)load_dynamo_code;
+        *bufptr++ = SIZE_OF_LOAD_DYNAMO;
+        nbytes = sizeof(dr_mcontext_t) + 2*sizeof(reg_t);
+        cxt->CXT_XSP -= nbytes;
+        if (!nt_write_virtual_memory(phandle, (LPVOID)cxt->CXT_XSP,
+                                     buf, nbytes, &nbytes)) {
+            display_error("WriteMemory failed");
+            goto error;
+        }
+
+        /* push the address of the DYNAMORIO_ENTRY string on the app's stack */
+        cxt->CXT_XSP -= XSP_SZ;
+        if (!nt_write_virtual_memory(phandle, (LPVOID)cxt->CXT_XSP, 
+                                     &dynamo_entry_esp, sizeof(dynamo_entry_esp),
+                                     &nbytes)) {
+            display_error("WriteMemory failed");
+            goto error;
+        }
+
+        /* push the address of GetProcAddress on the app's stack */
+        ASSERT(addr_getprocaddr);
+        addr = addr_getprocaddr;
+        cxt->CXT_XSP -= XSP_SZ;
+        if (!nt_write_virtual_memory(phandle, (LPVOID)cxt->CXT_XSP, 
+                                     &addr, sizeof(addr), &nbytes)) {
+            display_error("WriteMemory failed");
+            goto error;
+        }
+
+        /* push the address of the dynamorio_path string on the app's stack */
+        cxt->CXT_XSP -= XSP_SZ;
+        if (!nt_write_virtual_memory(phandle, (LPVOID)cxt->CXT_XSP, 
+                                     &dynamo_path_esp, sizeof(dynamo_path_esp),
+                                     &nbytes)) {
+            display_error("WriteMemory failed");
+            goto error;
+        }
+
+        /* push the address of LoadLibraryA on the app's stack */
+        ASSERT(addr_loadlibrarya);
+        addr = addr_loadlibrarya;
+        cxt->CXT_XSP -= XSP_SZ;
+        if (!nt_write_virtual_memory(phandle, (LPVOID)cxt->CXT_XSP, 
+                                     &addr, sizeof(addr), &nbytes)) {
+            display_error("WriteMemory failed");
+            goto error;
+        }
+
+#ifdef LOAD_DYNAMO_DEBUGBREAK
+        /* push the address of DebugBreak on the app's stack */
+        ASSERT(addr_debugbreak);
+        addr = addr_debugbreak;
+        cxt->CXT_XSP -= XSP_SZ;
+        if (!nt_write_virtual_memory(phandle, (LPVOID)cxt->CXT_XSP, 
+                                     &addr, sizeof(addr), &nbytes)) {
+            display_error("WriteMemory failed");
+            goto error;
+        }
+#endif
+
+        /* make the code R-X now */
+        if (!nt_remote_protect_virtual_memory(phandle, load_dynamo_code, 
+                                              SIZE_OF_LOAD_DYNAMO,
+                                              PAGE_EXECUTE_READ, &old_prot)) {
+            display_error("Failed to make injection code R-X");
+            goto error;
+        }
+        ASSERT(old_prot == PAGE_EXECUTE_READWRITE);
+
+        /* now change Eip to point to the entry point of load_dynamo(), so that
+           when we resume, load_dynamo is invoked automatically */
+        cxt->CXT_XIP = (ptr_uint_t)load_dynamo_code;
+        cxt->CXT_XFLAGS = 0;
+        if (thandle != NULL) {
+            if (!NT_SUCCESS(nt_set_context(thandle, cxt))) {
+                display_error("SetThreadContext failed");
+                goto error;
+            }
+        }
+
+        success = TRUE;
+    }
+    error:
+        /* we do not recover any changes in the child's address space */
+
+    return success;
+}
+
+/* FIXME - would be nicer to use instrlist etc. to generate and emit the code
+ * (with patch list for the calls), but we'll also likely want to use this for
+ * drinject which would mean getting most of the core compiled into that. Prob.
+ * should still do it, but writing like this isn't that hard. Another
+ * possibility is to export this from a special/standalone build of dr that
+ * injector can load, that would also make it easier for injector to find
+ * Ldr* addresses. At the very least we should combine all these enums (instr.h
+ * os_shared.h, emit_utils.c etc.) in one place. */
+enum {
+    PUSHF                 = 0x9c,
+    POPF                  = 0x9d,
+    PUSHA                 = 0x60,
+    POPA                  = 0x61,
+    PUSH_EAX              = 0x50,
+    POP_EAX               = 0x58,
+    PUSH_ECX              = 0x51,
+    POP_ECX               = 0x59,
+    PUSH_IMM32            = 0x68,
+    PUSH_IMM8             = 0x6a,
+
+    JMP_REL8              = 0xeb,
+    JMP_REL32             = 0xe9,
+    CALL_REL32            = 0xe8,
+    CALL_RM32             = 0xff,
+    CALL_EAX_RM           = 0xd0,
+
+    MOV_RM32_2_REG32      = 0x8b,
+    MOV_ESP_2_EAX_RM      = 0xc4,
+    MOV_derefEAX_2_EAX_RM = 0x00,
+    MOV_IMM8_2_RM8        = 0xc6,
+    MOV_IMM32_2_RM32      = 0xc7,
+    MOV_IMM_RM_ABS        = 0x05,
+
+    ADD_EAX_IMM32         = 0x05,
+
+    CMP_EAX_IMM32         = 0x3d,
+    JZ_REL8               = 0x74,
+    JNZ_REL8              = 0x75,
+};
+
+#define DEBUG_LOOP 0
+
+#define ASSERT_ROOM(cur, buf, maxlen) \
+    ASSERT(cur + maxlen < buf + sizeof(buf))
+
+/* Early injection. */
+/* FIXME - like inject_into_thread we assume esp, but we could allocate our
+ * own stack in the child and swap to that for transparency. */
+bool
+inject_into_new_process(HANDLE phandle, char *dynamo_path, bool map,
+                        uint inject_location, void *inject_address)
+{
+    void *hook_target = NULL, *hook_location = NULL;
+    uint old_prot; 
+    size_t num_bytes_out;
+    byte hook_buf[5];
+
+    /* Possible child hook points */
+    GET_NTDLL(KiUserApcDispatcher, (IN PVOID Unknown1, 
+                                    IN PVOID Unknown2, 
+                                    IN PVOID Unknown3, 
+                                    IN PVOID ContextStart, 
+                                    IN PVOID ContextBody));
+    GET_NTDLL(KiUserExceptionDispatcher, (IN PVOID Unknown1, 
+                                          IN PVOID Unknown2));
+
+    /* Only ones that work, though I have hopes for KiUserException if can
+     * find a better spot to trigger the exception, or we should implement
+     * KiUserApc map requirement. */
+    ASSERT_NOT_IMPLEMENTED(INJECT_LOCATION_IS_LDR(inject_location));
+    switch(inject_location) {
+    case INJECT_LOCATION_LdrLoadDll:
+    case INJECT_LOCATION_LdrpLoadDll:
+    case INJECT_LOCATION_LdrCustom:
+    case INJECT_LOCATION_LdrpLoadImportModule:
+    case INJECT_LOCATION_LdrDefault:
+        /* caller provides the ldr address to use */
+        ASSERT(inject_address != NULL);
+        hook_location = inject_address;
+        if (hook_location == NULL) {
+            goto error;
+        }
+        break;
+    case INJECT_LOCATION_KiUserApc:
+        hook_location = (void *)KiUserApcDispatcher;
+        ASSERT(map);
+        break;
+    case INJECT_LOCATION_KiUserException:
+        hook_location = (void *)KiUserExceptionDispatcher;
+        break;
+    default:
+        ASSERT_NOT_REACHED();
+        goto error;
+    }
+
+    /* read in code at hook */
+    if (!nt_read_virtual_memory(phandle, hook_location, hook_buf,
+                                sizeof(hook_buf), &num_bytes_out) ||
+        num_bytes_out != sizeof(hook_buf)) {
+        goto error;
+    }
+
+    if (map) {
+        hook_target = NULL; /* for compiler */
+        /* NYI see case 102, plan is to remote map in our dll, link and rebase if
+         * neccesary and hook to a routine in our dll */
+        ASSERT_NOT_IMPLEMENTED(false);
+    } else {
+        byte *remote_code_buffer = NULL, *remote_data_buffer;
+        /* max usage for local_buf is for writing the dr library name
+         * 2*MAX_PATH (unicode) + sizoef(UNICODE_STRING) + 2, round up to
+         * 3*MAX_PATH to be safe */
+        byte local_buf[3*MAX_PATH];
+        byte *cur_local_pos, *cur_remote_pos, *jmp_fixup1, *jmp_fixup2;
+        char *takeover_func = "dynamorio_app_init_and_early_takeover";
+        PUNICODE_STRING mod, mod_remote;
+        PANSI_STRING func, func_remote;
+        int res;
+        size_t num_bytes_in;
+
+        GET_NTDLL(LdrLoadDll, (IN PCWSTR PathToFile OPTIONAL,
+                               IN PULONG Flags OPTIONAL,
+                               IN PUNICODE_STRING ModuleFileName,
+                               OUT PHANDLE ModuleHandle));
+        GET_NTDLL(LdrGetProcedureAddress, (IN HANDLE ModuleHandle,
+                                           IN PANSI_STRING ProcedureName OPTIONAL,
+                                           IN ULONG Ordinal OPTIONAL,
+                                           OUT FARPROC *ProcedureAddress));
+#define GET_PROC_ADDR_BAD_ADDR 0xffbadd11
+        GET_NTDLL(NtProtectVirtualMemory, (IN HANDLE ProcessHandle,
+                                           IN OUT PVOID *BaseAddress,
+                                           IN OUT PULONG ProtectSize,
+                                           IN ULONG NewProtect,
+                                           OUT PULONG OldProtect));
+        GET_NTDLL(NtContinue, (IN PCONTEXT Context,
+                               IN BOOLEAN TestAlert));
+
+        /* get buffer for emitted code and data */
+        if (!NT_SUCCESS(nt_remote_allocate_virtual_memory(phandle, &remote_code_buffer,
+                                                          2*PAGE_SIZE, PAGE_READWRITE,
+                                                          MEM_COMMIT))) {
+            goto error;
+        }
+        remote_data_buffer = remote_code_buffer + PAGE_SIZE;
+        
+        /* write data */
+        /* FIXME the two writes are similar (unicode vs ascii), could combine */
+        /* First UNICODE_STRING to library */
+        cur_remote_pos = remote_data_buffer;
+        cur_local_pos = local_buf;
+        ASSERT_ROOM(cur_local_pos, local_buf, sizeof(UNICODE_STRING));
+        mod = (PUNICODE_STRING)cur_local_pos;
+        memset(mod, 0, sizeof(UNICODE_STRING));
+        cur_local_pos += sizeof(UNICODE_STRING);
+        mod->Buffer = (wchar_t *)(cur_remote_pos + (cur_local_pos - local_buf));
+        ASSERT_ROOM(cur_local_pos, local_buf, 2*MAX_PATH+2 /* plus null */);
+        res = snwprintf((wchar_t *)cur_local_pos, 2*MAX_PATH, L"%hs", dynamo_path);
+        ASSERT(res > 0);
+        if (res > 0) {
+            cur_local_pos += (2*res);
+            ASSERT_TRUNCATE(mod->Length, ushort, 2*res);
+            mod->Length = (ushort)(2*res);
+            mod->MaximumLength = (ushort)(2*res);
+        }
+        /* ensure NULL termination, just in case */
+        *(wchar_t *)cur_local_pos = L'\0';
+        cur_local_pos += sizeof(wchar_t);
+        /* write to remote process */
+        num_bytes_in = cur_local_pos - local_buf;
+        if (!nt_write_virtual_memory(phandle, cur_remote_pos, local_buf,
+                                     num_bytes_in, &num_bytes_out) ||
+            num_bytes_out != num_bytes_in) {
+            goto error;
+        }
+        mod_remote = (PUNICODE_STRING)cur_remote_pos;
+        cur_remote_pos += num_bytes_out;
+
+        /* now write init/takeover func */
+        cur_local_pos = local_buf;
+        ASSERT_ROOM(cur_local_pos, local_buf, sizeof(ANSI_STRING));
+        func = (PANSI_STRING)cur_local_pos;
+        memset(func, 0, sizeof(ANSI_STRING));
+        cur_local_pos += sizeof(ANSI_STRING);
+        func->Buffer = (PCHAR) cur_remote_pos + (cur_local_pos - local_buf);
+        ASSERT_ROOM(cur_local_pos, local_buf, strlen(takeover_func)+1);
+        strncpy((char *)cur_local_pos, takeover_func, strlen(takeover_func));
+        cur_local_pos += strlen(takeover_func);
+        ASSERT_TRUNCATE(func->Length, ushort, strlen(takeover_func));
+        func->Length = (ushort)strlen(takeover_func);
+        func->MaximumLength = (ushort)strlen(takeover_func);
+        *cur_local_pos++ = '\0'; /* ensure NULL termination, just in case */
+        /* write to remote_process */
+        num_bytes_in = cur_local_pos - local_buf;
+        if (!nt_write_virtual_memory(phandle, cur_remote_pos, local_buf,
+                                     num_bytes_in, &num_bytes_out) ||
+            num_bytes_out != num_bytes_in) {
+            goto error;
+        }
+        func_remote = (PANSI_STRING)cur_remote_pos;
+        cur_remote_pos += num_bytes_out;
+        
+        /* now make data page read only */
+        res = nt_remote_protect_virtual_memory(phandle, remote_data_buffer, 
+                                               PAGE_SIZE, PAGE_READONLY,
+                                               &old_prot);
+        ASSERT(res);
+        
+#define INSERT_INT(value)         \
+  *(int *)cur_local_pos = value;  \
+  cur_local_pos += sizeof(int)
+
+#define PUSH_IMMEDIATE(value)     \
+  *cur_local_pos++ = PUSH_IMM32;  \
+  INSERT_INT(value)
+
+#define PUSH_SHORT_IMMEDIATE(value)     \
+  *cur_local_pos++ = PUSH_IMM8;         \
+  *cur_local_pos++ = value
+
+#define MOV_ESP_TO_EAX()                \
+  *cur_local_pos++ = MOV_RM32_2_REG32;  \
+  *cur_local_pos++ = MOV_ESP_2_EAX_RM
+
+/* FIXME - all values are small use imm8 version */
+#define ADD_TO_EAX(value)               \
+  *cur_local_pos++ = ADD_EAX_IMM32;     \
+  INSERT_INT(value)
+
+#define INSERT_REL32_ADDRESS(target)    \
+  IF_X64(ASSERT_NOT_IMPLEMENTED(false)); \
+  INSERT_INT((int)(ptr_int_t)((byte *)target - \
+                              (((cur_local_pos - local_buf)+4)+cur_remote_pos)))
+
+#define CALL(target_func)               \
+  *cur_local_pos++ = CALL_REL32;        \
+  INSERT_REL32_ADDRESS(target_func)
+
+/* ecx will hold OldProtection afterwards */
+#define PROT_IN_ECX 0xbad15bad /* doesn't match a PAGE_* define */
+#define CHANGE_PROTECTION(start, size, new_protection)                \
+  *cur_local_pos++ = PUSH_EAX; /* OldProtect slot */                  \
+  MOV_ESP_TO_EAX(); /* get &OldProtect */                             \
+  IF_X64(ASSERT_NOT_IMPLEMENTED(false));                              \
+  PUSH_IMMEDIATE((int)(ALIGN_FORWARD(start+size, PAGE_SIZE) -         \
+                 ALIGN_BACKWARD(start, PAGE_SIZE))); /* ProtectSize */ \
+  PUSH_IMMEDIATE((int)ALIGN_BACKWARD(start, PAGE_SIZE)); /* BaseAddress */ \
+  *cur_local_pos++ = PUSH_EAX; /* arg 5 &OldProtect */                \
+  if (new_protection == PROT_IN_ECX) {                                \
+      *cur_local_pos++ = PUSH_ECX; /* arg 4 NewProtect */             \
+  } else {                                                            \
+      PUSH_IMMEDIATE(new_protection); /* arg 4 NewProtect */          \
+  }                                                                   \
+  ADD_TO_EAX(-4); /* get &ProtectSize */                              \
+  *cur_local_pos++ = PUSH_EAX; /* arg 3 &ProtectSize */               \
+  ADD_TO_EAX(-4); /* get &BaseAddress */                              \
+  *cur_local_pos++ = PUSH_EAX; /* arg 2 &BaseAddress */               \
+  PUSH_IMMEDIATE((int)(ptr_int_t)NT_CURRENT_PROCESS); /* arg ProcessHandle */ \
+  CALL(NtProtectVirtualMemory);                                       \
+  /* no error checking, can't really do anything about it, FIXME */   \
+  /* stdcall so just the three slots we made for the ptr arguments    \
+   * left on the stack */                                             \
+  *cur_local_pos++ = POP_ECX; /* pop BaseAddress */                   \
+  *cur_local_pos++ = POP_ECX; /* pop ProtectSize */                   \
+  *cur_local_pos++ = POP_ECX /* pop OldProtect into ecx */
+
+
+        /* write code */
+        /* xref case 3821, first call to a possibly hooked routine should be
+         * more then 5 bytes into the page, which is satisfied (though is not
+         * clear if any hookers would manage to get in first). */
+        cur_remote_pos = remote_code_buffer;
+        cur_local_pos = local_buf;
+        hook_target = cur_remote_pos;
+        /* for inject_location INJECT_LOCATION_Ldr* we stick the address used
+         * at the start of the code for the child's use */
+        if (INJECT_LOCATION_IS_LDR(inject_location)) {
+            IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+            INSERT_INT((int)(ptr_int_t)inject_address);
+            hook_target = cur_remote_pos + 4;  /* skip the address */
+        }
+
+#if DEBUG_LOOP
+        *cur_local_pos++ = JMP_REL8;
+        *cur_local_pos++ = 0xfe;
+#endif
+
+        /* save current state */
+        *cur_local_pos++ = PUSHA;
+        *cur_local_pos++ = PUSHF;
+
+        /* restore trampoline, first make writable */
+        CHANGE_PROTECTION(hook_location, 5, PAGE_EXECUTE_READWRITE);
+        *cur_local_pos++ = MOV_IMM32_2_RM32; /* restore first 4 bytes of hook */
+        *cur_local_pos++ = MOV_IMM_RM_ABS;
+        IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+        INSERT_INT((int)(ptr_int_t)hook_location);
+        INSERT_INT(*(int *)hook_buf);
+        *cur_local_pos++ = MOV_IMM8_2_RM8; /* restore 5th byte of the hook */
+        *cur_local_pos++ = MOV_IMM_RM_ABS;
+        IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+        INSERT_INT((int)(ptr_int_t)hook_location+4);
+        *cur_local_pos++ = hook_buf[4];
+        /* hook restored, restore protection */
+        CHANGE_PROTECTION(hook_location, 5, PROT_IN_ECX);
+
+        if (inject_location == INJECT_LOCATION_KiUserException) {
+            /* Making the first page of the image unreadable triggers an exception
+             * to early to use the loader, might try pointing the import table ptr
+             * to bad memory instead TOTRY, whatever we do should fixup here */
+            ASSERT_NOT_IMPLEMENTED(false);
+        }
+        
+        /* call LdrLoadDll to load dr library */
+        *cur_local_pos++ = PUSH_EAX; /* need slot for OUT hmodule*/
+        MOV_ESP_TO_EAX();
+        *cur_local_pos++ = PUSH_EAX; /* arg 4 OUT *hmodule */
+        IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+        PUSH_IMMEDIATE((int)(ptr_int_t)mod_remote); /* our library name */
+        PUSH_SHORT_IMMEDIATE(0x0); /* Flags OPTIONAL */
+        PUSH_SHORT_IMMEDIATE(0x0); /* PathToFile OPTIONAL */
+        CALL(LdrLoadDll); /* see signature at decleration above */
+
+        /* stdcall so removed args so top of stack is now the slot containing the
+         * returned handle.  Use LdrGetProcedureAddress to get the address of the
+         * dr init and takeover function. Is ok to call even if LdrLoadDll failed,
+         * so we check for errors afterwards. */
+        *cur_local_pos++ = POP_ECX; /* dr module handle */
+        *cur_local_pos++ = PUSH_ECX; /* need slot for out ProcedureAddress */
+        MOV_ESP_TO_EAX();
+        *cur_local_pos++ = PUSH_EAX; /* arg 4 OUT *ProcedureAddress */
+        PUSH_SHORT_IMMEDIATE(0x0); /* Ordinal OPTIONAL */
+        IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+        PUSH_IMMEDIATE((int)(ptr_int_t)func_remote); /* func name */
+        *cur_local_pos++ = PUSH_ECX; /* module handle */
+        CALL(LdrGetProcedureAddress); /* see signature at decleration above */
+
+        /* Top of stack is now the dr init and takeover function (stdcall removed
+         * args). Check for errors and bail (FIXME debug build report somehow?) */
+        *cur_local_pos++ = CMP_EAX_IMM32;
+        INSERT_INT(STATUS_SUCCESS);
+        *cur_local_pos++ = POP_EAX; /* dr init_and_takeover function */
+        *cur_local_pos++ = JNZ_REL8; /* FIXME - should check >= 0 instead? */
+        jmp_fixup1 = cur_local_pos++; /* jmp to after call below */
+        /* Xref case 8373, LdrGetProcedureAdderss sometimes returns an
+         * address of 0xffbadd11 even though it returned STATUS_SUCCESS */
+        *cur_local_pos++ = CMP_EAX_IMM32;
+        INSERT_INT(GET_PROC_ADDR_BAD_ADDR);
+        *cur_local_pos++ = JZ_REL8; /* JZ == JE */
+        jmp_fixup2 = cur_local_pos++; /* jmp to after call below */
+        IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+        PUSH_IMMEDIATE((int)(ptr_int_t)remote_code_buffer); /* arg to takeover func */
+        PUSH_IMMEDIATE(inject_location); /* arg to takeover func */
+        *cur_local_pos++ = CALL_RM32; /* call EAX */
+        *cur_local_pos++ = CALL_EAX_RM;
+        *cur_local_pos++ = POP_ECX; /* cdecl so pop arg */
+        *cur_local_pos++ = POP_ECX; /* cdecl so pop arg */
+        /* Now patch the jnz above (if error) to go to here */
+        ASSERT_TRUNCATE(*jmp_fixup1, byte, cur_local_pos - (jmp_fixup1+1));
+        *jmp_fixup1 = (byte)(cur_local_pos - (jmp_fixup1+1)); /* target of jnz */
+        ASSERT_TRUNCATE(*jmp_fixup2, byte, cur_local_pos - (jmp_fixup2+1));
+        *jmp_fixup2 = (byte)(cur_local_pos - (jmp_fixup2+1)); /* target of jz */
+        *cur_local_pos++ = POPF;
+        *cur_local_pos++ = POPA;
+        if (inject_location != INJECT_LOCATION_KiUserException) {
+            /* jmp back to the hook location to resume execution */
+            *cur_local_pos++ = JMP_REL32;
+            INSERT_REL32_ADDRESS(hook_location);
+        } else {
+            /* we triggered the exception, so do an NtContinue back */
+            /* see callback.c, esp+4 holds CONTEXT ** */
+            *cur_local_pos++ = POP_EAX;  /* EXCEPTION_RECORD ** */
+            *cur_local_pos++ = POP_EAX;  /* CONTEXT ** */
+            PUSH_SHORT_IMMEDIATE(FALSE); /* arg 2 TestAlert */
+            *cur_local_pos++ = MOV_RM32_2_REG32;
+            *cur_local_pos++ = MOV_derefEAX_2_EAX_RM; /* CONTEXT * -> EAX */
+            *cur_local_pos++ = PUSH_EAX; /* push CONTEXT * (arg 1) */
+            CALL(NtContinue);
+            /* should never get here, will be zeroed memory so will crash if
+             * we do happen to get here, good enough reporting */
+        }
+
+        /* Our emitted code above is much less then the sizeof local_buf,
+         * but we'll add a check here (after the fact so not robust if really
+         * overflowed) that we didn't even come close (someon adding large amounts
+         * of code should hit this. FIXME - do better? */
+        ASSERT_ROOM(cur_local_pos, local_buf, MAX_PATH);
+        num_bytes_in = cur_local_pos - local_buf;
+        if (!nt_write_virtual_memory(phandle, cur_remote_pos, local_buf,
+                                     num_bytes_in, &num_bytes_out) ||
+            num_bytes_out != num_bytes_in) {
+            goto error;
+        }
+        cur_remote_pos += num_bytes_out;
+        /* now make code page rx */
+        res = nt_remote_protect_virtual_memory(phandle, remote_code_buffer, 
+                                               PAGE_SIZE, PAGE_EXECUTE_READ,
+                                               &old_prot);
+        ASSERT(res);
+
+#undef INSERT_INT
+#undef PUSH_IMMEDIATE
+#undef PUSH_SHORT_IMMEDIATE
+#undef MOV_ESP_TO_EAX
+#undef ADD_TO_EAX
+#undef INSERT_REL32_ADDRESS
+#undef CALL
+#undef PROT_IN_ECX
+#undef CHANGE_PROTECTION
+    }
+
+    /* place hook */
+    ASSERT(sizeof(hook_buf) == 5); /* standard 5 byte jmp rel32 hook */
+    hook_buf[0] = JMP_REL32;
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    *(int *)(&hook_buf[1]) = (int)((byte *)hook_target - ((byte *)hook_location + 5));
+    if (!nt_remote_protect_virtual_memory(phandle, hook_location,
+                                          sizeof(hook_buf),
+                                          PAGE_EXECUTE_READWRITE, &old_prot)) {
+        goto error;
+    }
+    if (!nt_write_virtual_memory(phandle, hook_location, hook_buf,
+                                 sizeof(hook_buf), &num_bytes_out) ||
+        num_bytes_out != sizeof(hook_buf)) {
+        goto error;
+    }
+    if (!nt_remote_protect_virtual_memory(phandle, hook_location,
+                                          sizeof(hook_buf),
+                                          old_prot, &old_prot)) {
+        goto error;
+    }
+
+    return true;
+
+    error:
+    /* we do not recover any changes in the child's address space */
+    return false;
+}
diff --git a/core/win32/inject_shared.c b/core/win32/inject_shared.c
new file mode 100644
index 000000000..9d7655159
--- /dev/null
+++ b/core/win32/inject_shared.c
@@ -0,0 +1,1422 @@
+/* **********************************************************
+ * Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+
+/*
+ * inject_shared.c
+ * 
+ * Facilities shared between the core library, the preinject library 
+ * and the drinject executable.
+ *
+ * getting parameters from registry keys
+ * rununderdr/runall logic
+ * and some debugging support
+ *
+ */
+
+/* for policy.dll linkage of qualified name utils */
+#ifndef NOT_DYNAMORIO_CORE
+
+#include "globals.h"
+
+#include <windows.h>
+#include <tchar.h>
+#include <stdio.h>
+#include "ntdll.h"  /* should link with ntdll.o */
+#ifdef DEBUG
+# include "moduledb.h"   /* for macros for 9252 fix. */
+#endif
+
+/* for asserts, copied from utils.h */
+#ifdef assert
+# undef assert
+#endif
+/* avoid mistake of lower-case assert */
+#define assert assert_no_good_use_ASSERT_instead
+extern void internal_error(char *file, int line, char *msg);
+#ifdef DEBUG
+extern void display_error(char *msg);
+# ifdef NOT_DYNAMORIO_CORE_PROPER   /* Part of case 9252 fix. */
+#  ifdef ASSERT
+#   undef ASSERT
+#  endif
+#  ifdef INTERNAL
+#   define ASSERT(x)         if (!(x)) internal_error(__FILE__, __LINE__, #x)
+#  else 
+#   define ASSERT(x)         if (!(x)) internal_error(__FILE__, __LINE__, "")
+#  endif /* INTERNAL */
+# endif  /* !NOT_DYNAMORIO_CORE_PROPER */
+#else
+# define ASSERT(x)         ((void) 0)
+# define display_error(msg)
+#endif
+
+#define MAX_RUNVALUE_LENGTH 12  /* 1 for sign, 10 digits and a \0 */
+#ifdef NOT_DYNAMORIO_CORE_PROPER
+# define VERBOSE 0
+#else
+# define VERBOSE 0
+#endif
+
+typedef enum {
+    REGISTRY_DEFAULT,
+    /* These apply only to 64-bit Windows and only matter for WOW64 */
+    REGISTRY_32, /* Look in 32-bit WOW64 registry settings */
+    REGISTRY_64, /* Look in 64-bit registry settings */
+} reg_platform_t;
+
+#if defined(NOT_DYNAMORIO_CORE_PROPER) && defined(DEBUG)
+/* for ASSERT_CURIOSITY as defined in utils.h */
+bool
+ignore_assert(const char *assert_stmt, const char *expr)
+{
+    return false;
+}
+
+void
+report_dynamorio_problem(dcontext_t *unused_dcontext, uint unused_dumpcore_flag,
+                         app_pc unused_exception_addr, app_pc unused_report_ebp, 
+                         const char *unused_fmt, ...)
+{
+    /* FIXME: not supporting here - cannot print the message but
+     * rather its format string */
+    display_error("ASSERT_CURIOSITY hit - attach a debugger\n");
+}
+#endif
+
+#if VERBOSE
+/* display_verbose_message also used by pre_inject */
+
+/* Have to reinvent the wheel, unfortunately, since pre_inject does not link
+ * utils.c.  Only need this to be able to get the app name for the
+ * display_verbose_message title, but that's nice to have.
+ */
+static void
+notcore_mutex_lock(int *thelock)
+{
+    while (_InterlockedExchange((LONG *)thelock, 1) != 0) {
+        _mm_pause();
+    }
+}
+
+static void
+notcore_mutex_unlock(int *thelock)
+{
+    *thelock = 0;
+}
+
+void
+display_verbose_message(char *format, ...) 
+{
+    char char_msg_buf[512];
+    wchar_t msg_buf[512];
+    static bool title_set = false;
+    static bool title_set_in_progress = false;
+    static wchar_t title_buf[MAX_PATH + 64];
+    static int title_lock;
+    static uint title_sz;
+    wchar_t *title = title_buf;
+    uint msg_sz;
+    size_t written;
+    va_list ap;
+    va_start(ap, format);
+    if (!title_set) {
+        /* avoid recursive infinite loop of get_application_name()'s registry
+         * reads calling this routine
+         */
+        if (title_set_in_progress) {
+            /* we ourselves hold the lock, just use const for this msg */
+            title = L"<title set in progress>";
+        } else {
+            notcore_mutex_lock(&title_lock);
+            if (!title_set) { /* handle race */
+                title_set_in_progress = true;
+                title_sz = _snwprintf(title_buf, BUFFER_SIZE_ELEMENTS(title_buf),
+                                      L_PRODUCT_NAME L" Notice: %hs(%hs)", 
+                                      get_application_name(), get_application_pid());
+                NULL_TERMINATE_BUFFER(title_buf);
+                if (title_sz < 0) /* at max count: print all but NULL to stderr */
+                    title_sz = BUFFER_SIZE_ELEMENTS(title_buf) - 1;
+                title_set_in_progress = false;
+                title_set = true;
+            }
+            notcore_mutex_unlock(&title_lock);
+        }
+    }
+    vsnprintf(char_msg_buf, BUFFER_SIZE_ELEMENTS(char_msg_buf), format, ap);
+    NULL_TERMINATE_BUFFER(char_msg_buf);
+    msg_sz = _snwprintf(msg_buf, BUFFER_SIZE_ELEMENTS(msg_buf), L"%hs", char_msg_buf);
+    NULL_TERMINATE_BUFFER(msg_buf);
+    if (msg_sz < 0) /* at max count: print all but NULL to stderr */
+        msg_sz = BUFFER_SIZE_ELEMENTS(msg_buf) - 1;
+    /* stderr and messagebox */
+    write_file(STDERR, title_buf, title_sz*sizeof(title_buf[0]), NULL, &written);
+    write_file(STDERR, msg_buf, msg_sz*sizeof(msg_buf[0]), NULL, &written);
+    write_file(STDERR, "\n", sizeof("\n"), NULL, &written);
+    nt_messagebox(msg_buf, title);
+}
+#endif
+
+/* returns pointer to last char of string that matches either c1 or c2 
+ * or NULL if can't find */
+const char *
+double_strrchr(const char *string, char c1, char c2)
+{
+    const char *ret = NULL;
+    while (*string != '\0') {
+        if (*string == c1 || *string == c2) {
+            ret = string;
+        }
+        string++;
+    }
+    return ret;
+}
+
+int
+wchar_to_char(char *cdst, size_t buflen, 
+              PCWSTR wide_src, size_t bytelen)
+{
+    int res;
+    size_t wlen = (bytelen / sizeof(wchar_t));
+    if (wlen >= buflen) {         /* wide string length + NULL */
+        cdst[0] = '\0';
+        return 0;
+    }
+
+    res = snprintf(cdst, buflen, "%*ls", wlen, wide_src);
+    cdst[wlen] = '\0';          /* always NULL terminate */
+    ASSERT(strlen(cdst) < buflen); /* off by one, lets us see if we're pushing it */
+    return res;
+}
+
+/* Description: Sets the value data for a value name that belongs to a given 
+ *              registry key.
+ * Input:       keyname - fully qualified name of the registry key to 
+ *                        which valuename belongs to.
+ *              valuename - name of the value for which the data is to be set.
+ *              value - data that is to be set for valuename.
+ * Output:      None.
+ * Return value:SET_PARAMETER_SUCCESS if valuename is changed/created.
+ *              SET_PARAMETER_FAILURE if keyname is invalid or if valuename 
+ *              can't be changed or created.
+ * Notes:       If valuename doesn't exist, it will be created with value as 
+ *              data.  Implemented as part of case 3702.
+ */
+static int
+set_registry_parameter(const wchar_t *keyname, const wchar_t *valuename,
+                       const char *value)
+{
+    HANDLE hkey;
+    int size, res;
+
+    wchar_t wvalue[MAX_REGISTRY_PARAMETER];
+
+    size = _snwprintf(wvalue, BUFFER_SIZE_ELEMENTS(wvalue), L"%S", value);
+    NULL_TERMINATE_BUFFER(wvalue);
+    ASSERT(size >= 0 && size < BUFFER_SIZE_ELEMENTS(wvalue));
+
+    hkey = reg_open_key(keyname, KEY_SET_VALUE);
+    if (NULL != hkey) {
+        res = reg_set_key_value(hkey, valuename, wvalue);
+        if (res) {
+            /* Need to flush registry writes to disk, otherwise a
+             * power cycle will throw out the changes.  See case 4138.
+             */
+            reg_flush_key(hkey);
+            reg_close_key(hkey);
+            return SET_PARAMETER_SUCCESS;
+        }
+        reg_close_key(hkey);
+    }
+    return SET_PARAMETER_FAILURE;
+}
+
+static int
+get_registry_parameter(PCWSTR keyname, PCWSTR valuename,
+                       char *value, /* OUT */
+                       int maxlen /* up to MAX_REGISTRY_PARAMETER */,
+                       reg_platform_t whichreg) 
+{
+    int retval = GET_PARAMETER_FAILURE;
+    KEY_VALUE_PARTIAL_INFORMATION *kvpi;
+    reg_query_value_result_t result;
+    /* we could probably get rid of this buffer by using the callers buffer
+     * though would be kind of ugly for the caller */
+    char buf_array[sizeof(KEY_VALUE_PARTIAL_INFORMATION) +
+                   sizeof(wchar_t)*(MAX_REGISTRY_PARAMETER + 1)]; // wide
+
+    /* For injectors and for all core registry reads (except process control
+     * hash lists) use the local array; for process control hash lists use a
+     * dynamic buffer.  This messy code was needed to handle case 9252.
+     */
+    char *buf = (char *)&buf_array;
+    int alloc_size = sizeof(buf_array);
+#if !defined(NOT_DYNAMORIO_CORE_PROPER) && !defined(NOT_DYNAMORIO_CORE)
+    /* Only for core; injectors shouldn't use this and don't have heap mgt.
+     * Even if core is using, can't use this till heap is initialized. */
+    if (maxlen > MAX_REGISTRY_PARAMETER && dynamo_heap_initialized) {
+        /* Only process control hashlist reads may read more chars than 
+         * MAX_REGISTRY_PARAMETER.  Case 9252.
+         */
+#ifdef PROCESS_CONTROL
+        ASSERT(IS_PROCESS_CONTROL_ON() &&
+               maxlen == (int)(DYNAMO_OPTION(pc_num_hashes) *
+                               (MD5_STRING_LENGTH + 1)));
+#endif
+        /* Registry takes wchar buf so can't use maxlen directly to allocate. */
+        alloc_size = sizeof(KEY_VALUE_PARTIAL_INFORMATION) +
+                     sizeof(wchar_t)*(maxlen + 1);
+        buf = heap_alloc(GLOBAL_DCONTEXT, alloc_size HEAPACCT(ACCT_OTHER));
+    }
+#else   /* NOT_DYNAMORIO_CORE_PROPER || NOT_DYNAMORIO_CORE */
+    ASSERT(maxlen <= MAX_REGISTRY_PARAMETER);
+#endif  /* !NOT_DYNAMORIO_CORE_PROPER && !NOT_DYNAMORIO_CORE */
+
+    kvpi = (KEY_VALUE_PARTIAL_INFORMATION*)buf;
+
+    result = reg_query_value(keyname, 
+                             valuename,
+                             KeyValuePartialInformation,
+                             kvpi,
+                             alloc_size,
+                             whichreg == REGISTRY_64 ? KEY_WOW64_64KEY :
+                             (whichreg == REGISTRY_32 ? KEY_WOW64_32KEY :0));
+    if (result == REG_QUERY_SUCCESS) {
+        snprintf(value, maxlen - 1, "%*ls", 
+                 kvpi->DataLength / sizeof(wchar_t) - 1,
+                 (wchar_t*)kvpi->Data);
+        value[maxlen - 1] = '\0'; /* make sure it is terminated */
+#if VERBOSE
+        display_verbose_message("got registry value of %hs for value %ls in key %ls", 
+                                value, valuename, keyname);
+#endif VERBOSE
+        retval = GET_PARAMETER_SUCCESS;
+    }
+#if !defined(NOT_DYNAMORIO_CORE_PROPER) && !defined(NOT_DYNAMORIO_CORE)
+    else if (result == REG_QUERY_BUFFER_TOO_SMALL) {
+        /* reuse buf */
+        snprintf(buf, alloc_size, "%ls - %ls", keyname, valuename);
+        buf[(alloc_size/sizeof(wchar_t)) - 1] = '\0';
+        retval = GET_PARAMETER_BUF_TOO_SMALL;
+        /* we might be reading the option string right now so don't synch */
+        SYSLOG_NO_OPTION_SYNCH(SYSLOG_ERROR, ERROR_REGISTRY_PARAMETER_TOO_LONG,
+                               3, get_application_name(),
+                               get_application_pid(), buf);
+    }
+#endif
+
+#if VERBOSE
+    if (IS_GET_PARAMETER_FAILURE(retval)) {
+        display_verbose_message("didn't get registry value %ls in key %ls", 
+                                valuename, keyname);
+    }
+#endif /* VERBOSE */
+
+#if !defined(NOT_DYNAMORIO_CORE_PROPER) && !defined(NOT_DYNAMORIO_CORE)
+    if (maxlen > MAX_REGISTRY_PARAMETER && dynamo_heap_initialized) /* case 9252 */
+        heap_free(GLOBAL_DCONTEXT, buf, alloc_size HEAPACCT(ACCT_OTHER));
+#endif  /* !NOT_DYNAMORIO_CORE_PROPER && !NOT_DYNAMORIO_CORE */
+    return retval;
+}
+
+/* reads remote process PEB.Ldr field 
+ * returns 
+ *   -1 if can't read remote process, 
+ *    0 if PEB.Ldr is NULL
+ *    1 if PEB.Ldr has been initialized 
+ */
+int
+get_remote_process_ldr_status(HANDLE process_handle)
+{
+    size_t nbytes;
+    size_t res;
+    PEB peb;
+
+    LPVOID peb_base = get_peb(process_handle);
+
+    /* Read process PEB */
+    res = nt_read_virtual_memory(process_handle, (LPVOID)peb_base, 
+                                 &peb, sizeof(peb), &nbytes);
+    if (!res) {
+        /* xref case 9800 - the app handle may not always have sufficient rights
+         * FIXME - could dup the handle and retry */
+        return -1;
+    }
+
+    if (peb.LoaderData != NULL) {
+        return 1;               /* already created process */
+    } else {
+        return 0;               /* new process */
+    }
+}
+
+static bool is_windows_version_vista(void); /* forward decl */
+
+/* 
+ * this assumes it will be called on process initialization, when
+ *  the PEB apparently uses offsets in the ProcessParameters block
+ *  instead of direct pointers.
+ * image name and cmdline combined into one call to reduce 
+ *  read process memory calls (whether this is actually true depends on 
+ *  usage)
+ */
+void 
+get_process_imgname_cmdline(HANDLE process_handle, 
+                            wchar_t *image_name, int max_image_wchars,
+                            wchar_t *command_line, int max_cmdl_wchars)
+{
+    size_t nbytes;
+    int res;
+    int len;
+    PEB peb;
+    LPVOID peb_base = get_peb(process_handle);
+    RTL_USER_PROCESS_PARAMETERS process_parameters;
+    void *param_location; 
+    /* It is supposed to be at process_parameters.ImagePathName.Buffer */
+
+    /* Read process PEB */
+    res = nt_read_virtual_memory(process_handle, (LPVOID)peb_base, 
+                                 &peb, sizeof(peb), &nbytes);
+    /* FIXME: is this always possible?
+       although we assume we can even do WriteProcessMemory for an explicit inject */
+    if (!res) {
+        display_error("Warning: could not read process memory!");
+        if (image_name) 
+            image_name[0] = L'\0';
+        if (command_line) 
+            command_line[0] = L'\0';
+        return;
+    }
+
+    /* Follow on to process parameters */
+    res = nt_read_virtual_memory(process_handle, (LPVOID)peb.ProcessParameters,
+                                 &process_parameters, 
+                                 sizeof(process_parameters), &nbytes);
+    
+
+    /* apparently {ImagePathName,CommandLine}.Buffer contains the offset 
+     * from the beginning of the ProcessParameters structure during
+     * process init on os versions prior to Vista */
+    
+    if (image_name) {
+        if (is_windows_version_vista()) {
+            param_location = process_parameters.ImagePathName.Buffer;
+        } else {
+            param_location = (void *)
+                ((ptr_uint_t)process_parameters.ImagePathName.Buffer +
+                 (ptr_uint_t)peb.ProcessParameters);
+        }
+
+        len = process_parameters.ImagePathName.Length;
+        if (len >  2*(max_image_wchars - 1))
+            len = 2*(max_image_wchars - 1);
+
+        /* Read the image file name in our memory too */
+        res = nt_read_virtual_memory(process_handle, (LPVOID)param_location,
+                                     image_name, len, &nbytes);
+        if (!res) {
+            len = 0;
+            display_error("Warning: could not read image name from PEB");
+        }
+        image_name[len/2] = 0;
+    }
+
+    if (command_line) {
+        if (is_windows_version_vista()) {
+            param_location = process_parameters.CommandLine.Buffer;
+        } else {
+            param_location = (void*)((ptr_uint_t)process_parameters.CommandLine.Buffer 
+                                     + (ptr_uint_t)peb.ProcessParameters);
+        }
+        
+        len = process_parameters.CommandLine.Length;
+        if (len >  2*(max_cmdl_wchars - 1))
+            len = 2*(max_cmdl_wchars - 1);
+        
+        /* Read the image file name in our memory too */
+        res = nt_read_virtual_memory(process_handle, (LPVOID)param_location,
+                                     command_line, len, &nbytes);
+        if (!res) {
+            len = 0;
+            display_error("Warning: could not read cmdline from PEB");
+        }
+        command_line[len/2] = 0;
+    }
+
+    return;
+}
+
+static inline
+int
+get_rununder_value(const char *runvalue)
+{
+    /* For now we allow only decimal, but with more flags it will be easier to work on hex.
+       FIXME: share the logic in parse_uint() from after options.c -r 1.4 
+       to allow both hex and decimal values
+    */
+    return atoi(runvalue);
+}
+
+#else /* !defined(NOT_DYNAMORIO_CORE) */
+
+/* need some definitions for get_commandline_qualifier below */
+#  include <windows.h>
+#  include <globals_shared.h> 
+
+#define DIRSEP '\\'
+#define ALT_DIRSEP '/'
+#endif
+
+
+/* shared utilities */
+/* unicode version of double_strrchr */
+const wchar_t *
+double_wcsrchr(const wchar_t *string, wchar_t c1, wchar_t c2)
+{
+    const wchar_t *ret = NULL;
+    while (*string != L'\0') {
+        if (*string == c1 || *string == c2) {
+            ret = string;
+        }
+        string++;
+    }
+    return ret;
+}
+
+const wchar_t*
+w_get_short_name(const wchar_t *exename)
+{
+    const wchar_t *exe;
+    exe = double_wcsrchr(exename, L_EXPAND_LEVEL(DIRSEP), 
+                         L_EXPAND_LEVEL(ALT_DIRSEP));
+    if (exe == NULL)
+        exe = exename;
+    else
+        exe++; /* skip (back)slash */
+    return exe;
+}
+
+/* 
+   We derive an application specific name to differentiate instances
+   based on the canonicalized commandline of the process.
+   Originally we did that only for svchost, but now it is for anything 
+     marked with RUNUNDER_COMMANDLINE_DISPATCH.
+   The current scheme asks for adding all alphanumeric characters from
+   the original commandline after skipping the executable name itself.
+
+   If no_strip (e.g. RUNUNDER_COMMANDLINE_NO_STRIP is set) then the
+   first argument on the commandline is not stripped.  This is for
+   backwards compatibility where we stripped the -k argument in the
+   svchost groups, e.g. 'svchost -k rpcss'
+
+   max_derived_length is in number of elements, so most callers should
+   simply use BUFFER_SIZE_ELEMENTS on the buffer passed as derived_name
+
+    Returns 1 if group command line qualifier present 
+      (normally should be 1 if called for matching executables, 
+      but on an empty commandline we do return 0)
+*/
+bool
+get_commandline_qualifier(const wchar_t *command_line,
+                          wchar_t *derived_name, uint max_derived_length /* in elements */,
+                          bool no_strip)
+{
+    wchar_t *derived_ptr = derived_name;
+    wchar_t *derived_end = derived_name + max_derived_length - 1; /* last usable char */
+
+    /* Find last piece of the executable name  */
+    const wchar_t *cmdptr;
+
+    /* Long paths (that may have spaces) are assumed to be in quotes on the command line */
+    if (command_line[0] == L'"') {
+        cmdptr = wcschr(command_line+1, L'"');
+        if (cmdptr)
+            cmdptr++;
+    } else {
+        cmdptr = wcschr(command_line, L' ');
+    }
+
+    if (!cmdptr) {
+        *derived_name = L'\0';
+        return 0;
+    }
+    
+    do {
+        /* Skip any leading delimiters before each argument, e.g. "svchost.exe   -k          netsvcs" */
+        while (*cmdptr && !iswalnum(*cmdptr))
+            cmdptr++;
+        if (!*cmdptr) 
+            break;
+
+        /* Skip sequence of alphanums unless no_strip */
+        if (!no_strip) {
+            while (*cmdptr && iswalnum(*cmdptr))
+                cmdptr++;
+            no_strip = true;
+            if (!*cmdptr) 
+                break;
+        }
+
+        /* copy out all valid characters */
+        while (*cmdptr && iswalnum(*cmdptr)) {
+            if (derived_ptr == derived_end) 
+                goto out;
+            *derived_ptr++ = *cmdptr++;
+        }        
+
+        /* We do not add any normalized delimiters, e.g. "/t /e /st" is the same as "/test",
+           since currently there is no need to be that punctual. */
+    } while (*cmdptr);
+ out:
+    *derived_ptr = L'\0'; /* NULL terminate */
+    if (derived_ptr == derived_name)
+        return 0; /* no commandline given */
+    else
+        return 1;
+}
+
+/* for policy.dll linkage of qualified name utils */
+#ifndef NOT_DYNAMORIO_CORE
+
+/* types for get_process_qualified_name results */
+typedef enum {
+    QUALIFIED_FULL_NAME, 
+    QUALIFIED_SHORT_NAME,
+    UNQUALIFIED_FULL_NAME,
+    UNQUALIFIED_SHORT_NAME,
+} qualified_name_type_t;
+
+#define NAME_TYPE_IS_UNQUALIFIED(name_type) \
+    (name_type == UNQUALIFIED_FULL_NAME || name_type == UNQUALIFIED_SHORT_NAME)
+#define NAME_TYPE_IS_SHORT(name_type) \
+    (name_type == QUALIFIED_SHORT_NAME || name_type == UNQUALIFIED_SHORT_NAME)
+
+/*
+  We test the rununder_mask of a process and if it specifies that a
+  fully qualified name is needed we lookup at the fully qualified
+  location first.
+
+  See comments in get_commandline_qualifier() and case 1324 for a list
+  of executables which we further distinguish based on commandline.
+
+  If we hardcode only a few more entries we could then avoid this
+  buffer and registry access altogether, yet for generality sake
+  we'll do this.
+
+  FIXME[inefficiency]: The systemwide_should_inject() thus ends up
+     checking twice for RUNUNDERDR, but the OS is good at caching this.
+*/
+static uint
+commandline_qualifier_needed(const wchar_t *process_short_name,
+                             reg_platform_t whichreg)
+{
+    char runvalue[MAX_RUNVALUE_LENGTH];
+    uint ret_val = 0;
+    int res;
+
+    wchar_t app_specific_base[MAXIMUM_PATH] = DYNAMORIO_REGISTRY_BASE L"\\";
+    /* FIXME: this extra buffer has exactly the same contents as that
+       passed by get_subkey_parameter(,,QUALIFIED_SHORT_NAME) and in
+       fact have the same contents as we'll now prepare in a different buffer.
+       Other callers of get_process_qualified_name() do not have an
+       extra buffer information prepared. For now we'll allocate another buffer.
+     */
+
+    /* We now need to use direct registry access to get RUNUNDER flags */
+    wcsncat(app_specific_base, process_short_name, BUFFER_ROOM_LEFT_W(app_specific_base));
+    NULL_TERMINATE_BUFFER(app_specific_base);
+    res = get_registry_parameter(app_specific_base, L_DYNAMORIO_VAR_RUNUNDER,
+                                 runvalue, sizeof(runvalue), whichreg);
+    if (IS_GET_PARAMETER_SUCCESS(res)) {
+        ret_val = get_rununder_value(runvalue) &
+            (RUNUNDER_COMMANDLINE_DISPATCH | RUNUNDER_COMMANDLINE_NO_STRIP);
+    }
+
+    return ret_val;
+}
+
+/* Returns the executable image path appended with the command line qualifier (if
+ * requested in name_type) into the user provided buffer.  If short is requested in 
+ * name_type only the executable name portion is added, otherwise the full path is used.
+ *
+ * max_exename_length is in number of elements, so most callers should
+ * simply use BUFFER_SIZE_ELEMENTS on the buffer passed as w_exename
+ *
+ * If process_handle is NULL we read from the local PEB entries. */
+static void
+get_process_qualified_name(HANDLE process_handle,
+                           wchar_t *w_exename,
+                           size_t max_exename_length,
+                           qualified_name_type_t name_type,
+                           reg_platform_t whichreg)
+{
+    const wchar_t *full_name;
+    const wchar_t *short_name;
+    uint commandline_dispatch = 0;
+
+    PEB *own_peb;
+    wchar_t other_process_img_or_cmd[MAXIMUM_PATH];
+
+    /* FIXME: This buffer is only needed for reading other process
+       data, we need to check stack depths for the follow children
+       case.  Although not needed when reading current process this
+       function should be called only at startup with known stack layout.
+    */
+
+    if (process_handle == NULL) {
+        /* get our own subkey */
+        own_peb = get_own_peb();
+        ASSERT(own_peb && own_peb->ProcessParameters);
+        ASSERT(own_peb->ProcessParameters->ImagePathName.Buffer);
+
+        full_name = own_peb->ProcessParameters->ImagePathName.Buffer;
+    } else {
+        own_peb = NULL;
+        /* get foreign process subkey */
+        /* to avoid another buffer and save stack space, we do this in stages */
+        /* just get image name first */
+        get_process_imgname_cmdline(process_handle, other_process_img_or_cmd /* image name */, 
+                                    BUFFER_SIZE_ELEMENTS(other_process_img_or_cmd),
+                                    NULL, 0);
+        full_name = other_process_img_or_cmd;
+    }
+
+    /* CHECK: can we safely assume that all UNICODE_STRINGs we read do have a final 0? */
+    short_name = w_get_short_name(full_name);
+    wcsncpy(w_exename, (NAME_TYPE_IS_SHORT(name_type)) ? short_name : full_name, 
+            max_exename_length);
+    w_exename[max_exename_length - 1] = 0;          /* always NULL terminate */
+
+    if (NAME_TYPE_IS_UNQUALIFIED(name_type)) {
+        /* off by one, lets us see if we're pushing it */
+        ASSERT_CURIOSITY(wcslen(w_exename) < max_exename_length - 1);
+        return;
+    }
+
+    commandline_dispatch = commandline_qualifier_needed(short_name, whichreg);
+    if (TEST(RUNUNDER_COMMANDLINE_DISPATCH, commandline_dispatch)) {
+        wchar_t cmdline_qualifier[MAXIMUM_PATH] = L"-";
+        /* FIXME: we could do all this processing in w_exename so that
+           no other buffer is needed at all, but for the sake of
+           readability keeping this extra */
+        const wchar_t *process_commandline;
+
+        if (process_handle == NULL) {
+            /* get our own commandline */
+            ASSERT(own_peb->ProcessParameters->CommandLine.Buffer);
+            process_commandline = own_peb->ProcessParameters->CommandLine.Buffer;
+        } else {
+            /* get only command line from other process */
+            get_process_imgname_cmdline(process_handle, NULL, 0, 
+                                        other_process_img_or_cmd, 
+                                        BUFFER_SIZE_ELEMENTS(other_process_img_or_cmd));
+            process_commandline = other_process_img_or_cmd;
+        }
+
+        get_commandline_qualifier(process_commandline,
+                                  cmdline_qualifier + 1,   /* skip the '-' */
+                                  BUFFER_SIZE_ELEMENTS(cmdline_qualifier) - 1,
+                                  TEST(RUNUNDER_COMMANDLINE_NO_STRIP,
+                                       commandline_dispatch));
+
+        /* append "qualifier" which already has a '-' and may in fact be only '-' if
+         * no qualifier was found (we still want the '-' to separate out the registry
+         * entries xref 9119) */
+        wcsncat(w_exename, cmdline_qualifier, 
+                max_exename_length - wcslen(w_exename) - 1);
+    }
+    w_exename[max_exename_length - 1] = 0;          /* always NULL terminate */
+    /* off by one, lets us see if we're pushing it */
+    ASSERT_CURIOSITY(wcslen(w_exename) < max_exename_length - 1);
+}
+
+/* NOTE - get_own_*_name routines cache their values and are primed by os_init() since
+ * it might not be safe to read the process parameters later. */
+
+/* Returns the cached full path of the image, including the command line qualifier when
+ * necessary */
+const wchar_t*
+get_own_qualified_name()
+{
+    static wchar_t full_qualified_name[MAXIMUM_PATH];
+
+    if (full_qualified_name[0] == L'\0') {
+        get_process_qualified_name(NULL, full_qualified_name,
+                                   BUFFER_SIZE_ELEMENTS(full_qualified_name),
+                                   QUALIFIED_FULL_NAME, REGISTRY_DEFAULT);
+        ASSERT(full_qualified_name[0] != L'\0');
+    }
+    return full_qualified_name;
+}
+
+/* Returns the cached full path of the image with no qualifiers */
+const wchar_t*
+get_own_unqualified_name()
+{
+    static wchar_t full_unqualified_name[MAXIMUM_PATH];
+
+    if (full_unqualified_name[0] == L'\0') {
+        get_process_qualified_name(NULL, full_unqualified_name,
+                                   BUFFER_SIZE_ELEMENTS(full_unqualified_name),
+                                   UNQUALIFIED_FULL_NAME, REGISTRY_DEFAULT);
+        ASSERT(full_unqualified_name[0] != L'\0');
+    }
+    return full_unqualified_name;
+}
+
+/* Returns the cached short image name, including the command line qualifier when
+ * necessary */
+const wchar_t*
+get_own_short_qualified_name()
+{
+    static wchar_t short_qualified_name[MAXIMUM_PATH];
+
+    if (!short_qualified_name[0]) {
+        get_process_qualified_name(NULL, short_qualified_name, 
+                                   BUFFER_SIZE_ELEMENTS(short_qualified_name),
+                                   QUALIFIED_SHORT_NAME, REGISTRY_DEFAULT);
+        ASSERT(short_qualified_name[0]);
+    }
+    return short_qualified_name;
+}
+
+/* Returns the cached short image name with no qualifiers */
+const wchar_t*
+get_own_short_unqualified_name()
+{
+    static wchar_t short_unqualified_name[MAXIMUM_PATH];
+
+    if (!short_unqualified_name[0]) {
+        get_process_qualified_name(NULL, short_unqualified_name, 
+                                   BUFFER_SIZE_ELEMENTS(short_unqualified_name),
+                                   UNQUALIFIED_SHORT_NAME, REGISTRY_DEFAULT);
+        ASSERT(short_unqualified_name[0]);
+    }
+    return short_unqualified_name;
+}
+
+static int
+get_subkey_parameter(HANDLE process_handle, const wchar_t *uname, 
+                     char *value, int maxlen, bool use_qualified,
+                     reg_platform_t whichreg)
+{
+    int retval;
+    wchar_t app_specific_base[MAXIMUM_PATH] = DYNAMORIO_REGISTRY_BASE L"\\";
+    /* DYNAMORIO_REGISTRY_BASE is not user controlled, ASSERT only */
+    ASSERT(wcslen(app_specific_base) < BUFFER_SIZE_ELEMENTS(app_specific_base));
+
+    if (process_handle == NULL) {
+        wcsncat(app_specific_base, 
+                use_qualified ?
+                get_own_short_qualified_name() : get_own_short_unqualified_name(),
+                BUFFER_ROOM_LEFT_W(app_specific_base));
+    } else {
+        /* instead of using another buffer for the temporary, 
+           we just append to the current one */
+        get_process_qualified_name(process_handle, 
+                                   app_specific_base + wcslen(app_specific_base),
+                                   BUFFER_ROOM_LEFT_W(app_specific_base), 
+                                   use_qualified ?
+                                   QUALIFIED_SHORT_NAME : UNQUALIFIED_SHORT_NAME,
+                                   whichreg);
+    }
+    NULL_TERMINATE_BUFFER(app_specific_base);
+
+    retval = get_registry_parameter(app_specific_base, uname, value, maxlen, whichreg);
+#if VERBOSE
+    display_verbose_message("gskp: %ls -- %ls\n\"%hs\"", app_specific_base, uname,
+                            IS_GET_PARAMETER_SUCCESS(retval) ?  value : "");
+#endif
+
+     if (IS_GET_PARAMETER_FAILURE(retval)) {
+        HANDLE hkey = reg_open_key(app_specific_base, KEY_READ);
+        if (hkey == NULL) {
+            retval = GET_PARAMETER_NOAPPSPECIFIC;
+        }
+        reg_close_key(hkey);
+    }
+    return retval;
+}
+
+
+/* value is a buffer allocated by the caller to hold the
+ * resulting value. If not succesful leaves original buffer contents intact.
+ *
+ * The same parameter is looked up first in the application specific registry 
+ * subtree and then in the global registry tree.  We no longer look for 
+ * environment variables.
+ */
+static int
+get_process_parameter_internal(HANDLE phandle, const wchar_t *name, 
+                               char *value, int maxlen, bool use_qualified,
+                               reg_platform_t whichreg)
+{
+    int err, err2;
+    
+#if VERBOSE
+    display_verbose_message("get_parameter:%ls", name);
+#endif VERBOSE
+    
+    /* first check app specific options */
+    err = get_subkey_parameter(phandle, name, value, maxlen, use_qualified, whichreg);
+    
+    if (err != GET_PARAMETER_SUCCESS) {
+        err2 = get_registry_parameter(DYNAMORIO_REGISTRY_BASE, name,
+                                      value, maxlen, whichreg);
+        if (IS_GET_PARAMETER_SUCCESS(err2)) {
+            /* if there's no app-specific but there is a global, return
+             * GET_PARAMETER_NOAPPSPECIFIC; otherwise, if there's a
+             * global, return success. */
+            if (err != GET_PARAMETER_NOAPPSPECIFIC)
+                err = GET_PARAMETER_SUCCESS;
+        }
+        else if (err == GET_PARAMETER_BUF_TOO_SMALL || 
+                 err2 == GET_PARAMETER_BUF_TOO_SMALL) {
+            /* On error, buffer too small takes precedence. */
+            err = GET_PARAMETER_BUF_TOO_SMALL;
+        } else
+            err = GET_PARAMETER_FAILURE;
+    }
+    return err;
+}
+
+/* get parameter for a different process */
+int
+get_process_parameter(HANDLE phandle, const wchar_t *name, char *value, int maxlen)
+{
+    return get_process_parameter_internal(phandle, name, value, maxlen, true/*qual*/,
+                                          REGISTRY_DEFAULT);
+}
+
+/* get parameter for current process */
+int
+get_parameter(const wchar_t *name, char *value, int maxlen)
+{
+    return get_process_parameter_internal(NULL, name, value, maxlen, true/*qual*/,
+                                          REGISTRY_DEFAULT);
+}
+
+#ifdef X64
+/* get parameter for current process name using 32-bit registry key */
+int
+get_parameter_32(const wchar_t *name, char *value, int maxlen)
+{
+    return get_process_parameter_internal(NULL, name, value, maxlen, true/*qual*/,
+                                          REGISTRY_32);
+}
+#else
+/* get parameter for current process name using 64-bit registry key */
+int
+get_parameter_64(const wchar_t *name, char *value, int maxlen)
+{
+    return get_process_parameter_internal(NULL, name, value, maxlen, true/*qual*/,
+                                          REGISTRY_64);
+}
+#endif
+
+/* get parameter for current processes root app key (not qualified app key)
+ * for ex. would get parameter from svchost.exe instead of svchost.exe-netsvc */
+int
+get_unqualified_parameter(const wchar_t *name, char *value, int maxlen)
+{
+    return get_process_parameter_internal(NULL, name, value, maxlen, false/*!qual*/,
+                                          REGISTRY_DEFAULT);
+}
+
+/* Description: Modifies the value name corresponding to a DR parameter.  This 
+ *              value name should belong to the registry key associated with the
+ *              executable for the given process under DYNAMORIO_REGISTRY_BASE.
+ * Input:       phandle - handle for the process whose registry value is to be
+ *                        changed.
+ *              name - value name for the corresponding DR parameter.
+ *              value - value to which the registry parameter should be set to.
+ * Output:      None.
+ * Return value:SET_PARAMETER_SUCCESS if name is changed.
+ *              SET_PARAMETER_FAILURE if not.
+ * Notes:       If called with an incorrect parameter name, a value with that 
+ *              parameter name will be created in the registry for the current 
+ *              executable name under DYNAMORIO_REGISTRY_BASE.  Implemented as
+ *              part of case 3702.
+ */
+int
+set_process_parameter(HANDLE phandle, const wchar_t *name, const char *value)
+{
+    wchar_t app_specific_base[MAXIMUM_PATH] = DYNAMORIO_REGISTRY_BASE L"\\";
+
+    /* Even though DYNAMORIO_REGISTRY_BASE is a constant, we need to null 
+     * terminate because windows compiler doesn't complain or null terminate 
+     * the array if the constant is longer than the array size!
+     */
+    NULL_TERMINATE_BUFFER(app_specific_base);
+
+    /* DYNAMORIO_REGISTRY_BASE is not user controlled, ASSERT only */
+    ASSERT(wcslen(app_specific_base) < BUFFER_SIZE_ELEMENTS(app_specific_base) - 1);
+
+    if (phandle == NULL) {
+        wcsncat(app_specific_base, get_own_short_qualified_name(),
+                BUFFER_ROOM_LEFT_W(app_specific_base));
+    } else {
+        /* instead of using another buffer for the temporary, 
+           we just append to the current one */
+        get_process_qualified_name(phandle, 
+                                   app_specific_base + wcslen(app_specific_base),
+                                   BUFFER_ROOM_LEFT_W(app_specific_base), 
+                                   QUALIFIED_SHORT_NAME,
+                                   false/*no cross-arch set needed yet*/);
+    }
+    NULL_TERMINATE_BUFFER(app_specific_base);
+    ASSERT(wcslen(app_specific_base) < BUFFER_SIZE_ELEMENTS(app_specific_base) - 1);
+
+    return set_registry_parameter(app_specific_base, name, value);
+}
+
+/* on NT */
+static bool
+is_nt_or_custom_safe_mode()
+{
+    char start_options[MAX_REGISTRY_PARAMETER];
+    int retval;
+
+    retval = get_registry_parameter(L"\\Registry\\Machine\\System\\CurrentControlSet"
+                                    L"\\Control",
+                                    L"SystemStartOptions",
+                                    start_options, sizeof(start_options),
+                                    REGISTRY_DEFAULT);
+    if (IS_GET_PARAMETER_SUCCESS(retval)) {
+        /* FIXME: should do only when non empty start options given */
+        /* let's see if we have an override */
+        char safemarker_override_buf[MAX_PARAMNAME_LENGTH];
+        char *safemarker = "SOS";
+        /* Currently doing only on NT, otherwise to preserve the
+         * distinction in is_safe_mode() between MINIMAL and NETWORK
+         * we'd need to check for SAFEBOOT:MINIMAL, since SOS will be
+         * set for SAFEBOOT:NETWORK as well.
+         * Note: There is no app specific override for safe boot, just global.
+         */
+        retval = get_registry_parameter(DYNAMORIO_REGISTRY_BASE,
+                                        L_DYNAMORIO_VAR_SAFEMARKER,
+                                        safemarker_override_buf,
+                                        sizeof(safemarker_override_buf),
+                                        /* currently only called on NT where there
+                                         * is no wow64 */
+                                        REGISTRY_DEFAULT);
+        if (IS_GET_PARAMETER_SUCCESS(retval))
+            safemarker = safemarker_override_buf;
+        
+        /* note that match is case sensitive, yet ntldr always
+         * converts the boot.ini options in all CAPS, so the value in
+         * DYNAMORIO_SAFEMARKER should always be all CAPS as well
+         */
+        if (strstr(start_options, safemarker)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+/* note that windows_version_init does a lot more checks and messages
+ * which we cannot use in drpreinject.dll
+ * otherwise this should be equivalent to get_os_version() == WINDOWS_VERSION_NT
+ */
+static inline bool
+is_windows_version_nt()
+{
+    PEB *peb = get_own_peb();
+    /* we won't work on any other anyways */
+    ASSERT(peb->OSPlatformId == VER_PLATFORM_WIN32_NT);
+    return (peb->OSMajorVersion == 4);
+}
+
+/* see comments at is_windows_version_nt() */
+static bool
+is_windows_version_vista()
+{
+    PEB *peb = get_own_peb();
+    /* we won't work on any other anyways */
+    ASSERT(peb->OSPlatformId == VER_PLATFORM_WIN32_NT);
+    return (peb->OSMajorVersion == 6);
+}
+
+/* verify safe mode registry key on Win2000+ */
+bool
+is_safe_mode() 
+{
+    char buf[sizeof(KEY_VALUE_PARTIAL_INFORMATION)+ sizeof(uint)];
+    KEY_VALUE_PARTIAL_INFORMATION* kvpi = (KEY_VALUE_PARTIAL_INFORMATION*)buf;
+    
+    enum {
+        MINIMAL = 1,
+        NETWORK = 2,
+    };
+    /* On safe mode boot we should override all current settings and Run Nothing. */
+    /* If the value is MINIMAL we do not inject.  Note we still inject
+       when running with == NETWORK, we don't want to expose network
+       listening services to risks, and to disable us administrators
+       only need the local console */
+
+    if (reg_query_value(L"\\Registry\\Machine\\System\\CurrentControlSet"
+                        L"\\Control\\SafeBoot\\Option",
+                        L"OptionValue",
+                        KeyValuePartialInformation,
+                        kvpi,
+                        sizeof(buf), 0) ==
+        REG_QUERY_SUCCESS) {
+        if (*(uint*)(kvpi->Data) == MINIMAL) {
+            return true;
+        }
+    }
+
+    /* FIXME: case 5307: based on some other our registry key value we should
+     * allow this even on Win2000 so that /DISABLESC can be passed on
+     * the command line.
+     */
+    if (is_windows_version_nt()) {
+        return is_nt_or_custom_safe_mode();
+    }
+
+    return false;
+}
+
+/* check AppInit key of current architecture (so if currently wow64,
+ * checks wow64; if x64, checks x64.  we don't support cross-arch follow-children
+ * (PR 254193).
+ */
+bool
+systemwide_inject_enabled()
+{
+    /* FIXME: is it better to memoize the result for multiple uses in os.c? */
+    /* There is always going to be a TOCTOU race condition anyways. */
+    char appinit[MAXIMUM_PATH]; 
+    int retval;
+
+    retval = get_registry_parameter(INJECT_ALL_HIVE_L INJECT_ALL_KEY_L,
+                                    INJECT_ALL_SUBKEY_L, appinit, sizeof(appinit),
+                                    REGISTRY_DEFAULT/*no cross-arch support: PR 254193*/);
+    if (IS_GET_PARAMETER_SUCCESS(retval)) {
+        // assumption: nobody else would use this name!
+        if ((strstr(appinit, INJECT_DLL_NAME) != NULL ||
+             strstr(appinit, INJECT_DLL_8_3_NAME) != NULL))
+            return 1;
+    }
+    return 0;
+}
+
+/* returns true if the process commandline matches the string in
+ * the DYNAMORIO_VAR_CMDLINE parameter.
+ * if callers need REGISTRY_{32,64} they should add that parameter: not needed currently.
+ */
+int
+check_commandline_match(HANDLE process)
+{
+    char process_cmdline[MAX_PATH];
+    wchar_t w_process_cmdline[MAX_PATH];
+
+    if (process == NULL) {
+        /* get our own cmdline */
+        PEB *peb = get_own_peb();
+        ASSERT(peb && peb->ProcessParameters);
+        ASSERT(peb->ProcessParameters->CommandLine.Buffer);
+        wcsncpy(w_process_cmdline,
+                peb->ProcessParameters->CommandLine.Buffer,
+                MAX_PATH);
+        w_process_cmdline[MAX_PATH-1] = L'\0';
+    } else {
+        get_process_imgname_cmdline(process, NULL, 0,
+                                    w_process_cmdline, MAX_PATH);
+        w_process_cmdline[MAX_PATH-1] = L'\0';
+    }
+    wchar_to_char(process_cmdline, sizeof(process_cmdline), w_process_cmdline,
+                  (wcslen(w_process_cmdline)+1)*sizeof(wchar_t));
+
+    {
+        /* share buffer between cmd_line_to_match and w_process_cmdline to save
+         * stack space */
+        char *cmdline_to_match = (char *)w_process_cmdline;
+        /* we expect an app-specific parameter only */
+        if ((GET_PARAMETER_SUCCESS == 
+             get_subkey_parameter(process, L_DYNAMORIO_VAR_CMDLINE,
+                                  cmdline_to_match, MAX_PATH, true, REGISTRY_DEFAULT)) &&
+            NULL != strstr(process_cmdline, cmdline_to_match))
+            return 1;
+        return 0;
+    }
+}
+
+/* look up RUNUNDER param.
+ * if it's defined in app-specific key, check against RUNUNDER_ON
+ *    if RUNUNDER_ON is set check against RUNUNDER_EXPLICIT
+ *        if set return INJECT_TRUE|INJECT_EXPLICIT else return INJECT_TRUE
+ *    if RUNUNDER_ON is not set return INJECT_EXCLUDED
+ *
+ * if no app-specific key, check global key against RUNUNDER_ALL
+ *    if set return INJECT_TRUE else return INJECT_FALSE
+ *
+ * if no app-specific key and no global key return INJECT_FALSE
+ *
+ * if mask isn't NULL, the DYNAMORIO_RUNUNDER mask is returned in it.
+ *
+ * NOTE - if return INJECT_TRUE and !INJECT_EXPLICIT then preinjector should inject if
+ *   systemwide_inject_enabled()
+ */
+static inject_setting_mask_t
+systemwide_should_inject_common(HANDLE process, int *mask, reg_platform_t whichreg)
+{
+    char runvalue[MAX_RUNVALUE_LENGTH];
+    int rununder_mask, retval, err;
+
+#if VERBOSE
+    display_verbose_message("systemwide_should_inject");
+#endif
+
+    /* get_process_parameter properly terminates short buffer */
+    err = get_process_parameter_internal(process, L_DYNAMORIO_VAR_RUNUNDER,
+                                         runvalue, sizeof(runvalue),
+                                         true/*qual*/, whichreg);
+
+    if (IS_GET_PARAMETER_FAILURE(err))
+        return INJECT_FALSE;
+    
+    rununder_mask = get_rununder_value(runvalue);
+    if (NULL != mask)
+        if (IS_GET_PARAMETER_SUCCESS(err))
+            *mask = rununder_mask;
+        else
+            *mask = 0;
+    
+    /* if there is no app-specific subkey, then we should compare
+       against runall */
+    if (err == GET_PARAMETER_NOAPPSPECIFIC) {
+        if (rununder_mask & RUNUNDER_ALL)
+            return INJECT_TRUE;
+        else
+            return INJECT_FALSE;
+    }
+    else { /* err == GET_PARAMETER_SUCCESS */
+        if (!(rununder_mask & RUNUNDER_ON))
+            return INJECT_EXCLUDED;
+        else {
+            /* now there is the possibility of needing to use an alternate
+             *  injection technique. */
+
+            int inject_mask = INJECT_FALSE;
+
+            if (rununder_mask & RUNUNDER_EXPLICIT)
+                inject_mask |= INJECT_EXPLICIT;
+
+            if (rununder_mask & RUNUNDER_COMMANDLINE_MATCH) {
+                
+                /* if the commandline matches, return INJECT_TRUE 
+                 * if the commandline doesn't match and runall is on,
+                 *  return INJECT_TRUE
+                 * else return INJECT_FALSE */
+                
+                if (check_commandline_match(process)) {
+                    inject_mask |= INJECT_TRUE;
+                } else {
+                    /* no match; check global runall */
+                    retval = get_registry_parameter(DYNAMORIO_REGISTRY_BASE,
+                                                    L_DYNAMORIO_VAR_RUNUNDER,
+                                                    runvalue, sizeof(runvalue),
+                                                    whichreg);
+                    if (IS_GET_PARAMETER_SUCCESS(retval)) {
+                        if (RUNUNDER_ALL == get_rununder_value(runvalue))
+                            inject_mask |= INJECT_TRUE;
+                    }
+                }
+                
+                return inject_mask;
+                
+            } else /* just normal injection */
+                return (inject_mask | INJECT_TRUE);
+        }
+    }
+}
+
+#ifndef X64
+inject_setting_mask_t
+systemwide_should_inject_64(HANDLE process, int *mask)
+{
+    return systemwide_should_inject_common(process, mask, REGISTRY_64);
+}
+#endif
+
+inject_setting_mask_t
+systemwide_should_inject(HANDLE process, int *mask)
+{
+    return systemwide_should_inject_common(process, mask, REGISTRY_DEFAULT);
+}
+
+/* Description: If RUNUNDER_ONCE flag exists in the given mask, the RUNUNDER_ON 
+ *              flag for the registry value DYNAMORIO_RUNUNDER is cleared for 
+ *              the given process, so that the application won't start under
+ *              DR during the next invocation.
+ * Input:       process - handle of the process whose registry value should be 
+ *                        modified.
+ *              rununder_mask - contains the process' DYNAMORIO_RUNUNDER mask.
+ * Output:      None.
+ * Return value:None.
+ * Notes:       See case 3702.
+ */
+void 
+check_for_run_once(HANDLE process, int rununder_mask)
+{
+    int size;
+    char mask_string[MAX_RUNVALUE_LENGTH];
+
+    if (TEST(rununder_mask, RUNUNDER_ONCE)) {
+        rununder_mask &= ~RUNUNDER_ON;
+        size = snprintf(mask_string, BUFFER_SIZE_ELEMENTS(mask_string),
+                        "%d", rununder_mask);
+        NULL_TERMINATE_BUFFER(mask_string);
+        ASSERT(size >= 0 && size < BUFFER_SIZE_ELEMENTS(mask_string) - 1);
+
+        /* All registry keys set up by our product are writable only by 
+         * SYSTEM and Admin users.  If another user runs an executable with
+         * RUNUNDER_ONCE, the core won't turn of RUNUNDER_ON because registry 
+         * write will fail.  This is an EV limitation: RUNUNDER_ONCE won't work
+         * for non privileged processes.  Will be fixed in 2.5.  See case 4249.
+         */
+        if (set_process_parameter(process, L_DYNAMORIO_VAR_RUNUNDER,
+                                  mask_string) !=
+            SET_PARAMETER_SUCCESS) {
+            /* FIXME: Till 2.5 ASSERT_NOT_REACHED/display_error should actually 
+             * be ASSERT_CURIOSITY.  Defining ASSERT_CURIOSITY for core, 
+             * drinject.exe and drpreinject.dll is an ugly redefinition hack; 
+             * better not do it just for case 4249.
+             */
+            display_error("Can't enforce RUNUNDER_ONCE.");
+            ASSERT_NOT_REACHED();
+        }
+    }
+}
+
+#endif /* !defined(NOT_DYNAMORIO_CORE) */
+
+#ifdef UNIT_TEST
+
+int
+test(char *name, char *filter, int expected)
+{
+    int res = check_filter(name, filter);
+    printf("check_filter(\"%s\", \"%s\") = %s\n", 
+           name, filter, res ? "true" : "false");
+    if (res != expected)
+        printf("FAILURE!\n");
+}
+
+int
+nametest(const char *commandline,
+         const char *expect,
+         )
+{
+    wchar_t wcommandline[2048];
+    wchar_t wexpect[2048];
+    wchar_t derived[MAX_PATH];
+    int ret;
+    _snwprintf(wcommandline, sizeof(wcommandline), L"%S", commandline);
+    _snwprintf(wexpect, sizeof(wexpect), L"%S", expect);
+
+    ret = get_commandline_qualifier(wcommandline,
+                                    derived, BUFFER_SIZE_ELEMENTS(derived), true);
+    printf("get_commandline_qualifier(\"%s\") => \"%ls\" [%d]\n", 
+           commandline, derived, ret);
+
+    if (wcscmp(derived, expect))
+        printf("FAILED!\n");
+
+    ret = get_commandline_qualifier(wcommandline,
+                                    derived, BUFFER_SIZE_ELEMENTS(derived),
+                                    REGISTRY_DEFAULT);
+    printf("get_commandline_qualifier(\"%s\") => \"%ls\" [%d]\n", 
+           commandline, derived, ret);
+}
+
+int
+main()
+{
+    char *filter = "calc.exe;notepad.exe";
+
+    test("notepad.exe", filter, 1);
+    test("calc.exe", filter, 1);
+    test("write.exe", filter, 0);
+    test("test.exe", filter, 0);
+
+    filter = NULL;
+    test("notepad.exe", filter, 0);
+    test("calc.exe", filter, 0);
+    test("write.exe", filter, 0);
+    test("test.exe", filter, 0);
+
+    nametest("C:\WINNT\System32\dllhost.exe /Processid:{3D14228D-FBE1-11D0-995D-00C04FD919C1}",
+             "");
+
+    /* short name lots of spaces */
+    nametest("svchost.exe   -k    netsvc    ", "");
+    
+    /* a real example: in fact the service executable name for RpcSs doesn't have a .exe in it */
+    nametest("system32\svchost -k rpcss    ", "");
+
+    /* spaces in long name actually require " */
+    nametest("\"c:\program files\test\my test\sqlserver.exe\"   -s uddi    ", "");
+    /* spaces in long name and no .exe */
+    nametest("\"c:\program files\test\my test\sqlserver\" -s uddi    ", "");
+
+    /* capital .EXE */
+    nametest("c:\program files\test\my test\sqlserver.EXE   -s uddi    ", "");
+    /* capital .EXE and a backslashes in case trying using short_name  */
+    nametest("c:\program files\test\my test\sqlserver.EXE   -s uddi -u test\test    ", "");
+    nametest("C:\WINNT\System32\dllhost.exe /Processid:{3D14228D-FBE1-11D0-995D-00C04FD919C1} /Processid:{3D14228D-FBE1-11D0-995D-00C04FD919C1} /Processid:{3D14228D-FBE1-11D0-995D-00C04FD919C1} /Processid:{3D14228D-FBE1-11D0-995D-00C04FD919C1} /Processid:{3D14228D-FBE1-11D0-995D-00C04FD919C1} /Processid:{3D14228D-FBE1-11D0-995D-00C04FD919C1} /Processid:{3D14228D-FBE1-11D0-995D-00C04FD919C1} /Processid:{3D14228D-FBE1-11D0-995D-00C04FD919C1}", "");
+
+    nametest("dllhost.exe ////// /P#%@#$%-k    netsvc    ", "");
+
+    /* test that snprintf/snwprintf are respecting bounds */
+    /* FIXME : move to some place shared with linux and expand to more
+     * cases (and add vs* functions) */
+    {
+        char buf[20];
+        memset(buf, 0x1, sizeof(buf));
+        snprintf(buf, 10, "0123456789%dfoo", 555);
+        if (buf[10] != 0x1 || buf[9] != '5')
+            printf("FAILED snprintf safety check\n");
+    }
+    {
+        wchar_t buf[20];
+        memset(buf, 0x1, sizeof(buf));
+        snwprintf(buf, 10, "0123456789%dfoo", 555);
+        if (buf[10] != 0x11 || buf[9] != L'5')
+            printf("FAILED snprintf safety check\n");
+    }
+    return 0;
+}
+
+#endif /* UNIT_TEST */
diff --git a/core/win32/inject_shared.h b/core/win32/inject_shared.h
new file mode 100644
index 000000000..d467dcac9
--- /dev/null
+++ b/core/win32/inject_shared.h
@@ -0,0 +1,158 @@
+/* **********************************************************
+ * Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+
+/*
+ * inject_shared.h
+ * 
+ * Prototypes of facilities shared between the core library, the preinject library 
+ * and the drinject executable.
+ */
+
+#ifndef INJECT_SHARED_H
+#define INJECT_SHARED_H
+
+/* return codes for systemwide_should_inject, see notes there for more info */
+typedef enum {
+    INJECT_FALSE    = 0,
+    INJECT_TRUE     = 1,
+    INJECT_EXCLUDED = 2,
+    INJECT_EXPLICIT = 4
+} inject_setting_mask_t;
+
+/* value is a buffer allocated by the caller to hold the resulting value.
+ *
+ * The same parameter is looked up first in the application specific registry 
+ * subtree and then in the global registry tree.  
+ */
+int
+get_process_parameter(HANDLE phandle, 
+                      const wchar_t *name, char *value, int maxlen);
+
+/* Only the process specific registry value is set.  */
+int 
+set_process_parameter(HANDLE phandle, const wchar_t *name, const char *value);
+
+/* Get parameter for current process, using registry view matching the
+ * build of DR (so 64-bit DR looks at 64-bit registry, even for WO64 process).
+ * If return value is not GET_PARAMETER_SUCCESS leaves original buffer contents 
+ * intact (may not be true if GET_PARAMETER_BUF_TOO_SMALL is returned).
+ */
+int
+get_parameter(const wchar_t *name, char *value, int maxlen);
+
+#ifdef X64
+/* Get parameter for current process name from 32-bit registry view.
+ * If return value is not GET_PARAMETER_SUCCESS leaves original buffer contents 
+ * intact (may not be true if GET_PARAMETER_BUF_TOO_SMALL is returned).
+ */
+int
+get_parameter_32(const wchar_t *name, char *value, int maxlen);
+#else
+/* Get parameter for current process name from 64-bit registry view.
+ * If return value is not GET_PARAMETER_SUCCESS leaves original buffer contents 
+ * intact (may not be true if GET_PARAMETER_BUF_TOO_SMALL is returned).
+ */
+int
+get_parameter_64(const wchar_t *name, char *value, int maxlen);
+#endif
+
+/* Get parameter for current processes root app key (not qualified app key).
+ * For ex. would get parameter from svchost.exe instead of svchost.exe-netsvc. 
+ * If return value is not GET_PARAMETER_SUCCESS leaves original buffer contents
+ * intact (may not be true if GET_PARAMETER_BUF_TOO_SMALL is returned).
+ */
+int
+get_unqualified_parameter(const wchar_t *name, char *value, int maxlen);
+
+/* get_own_*_name routines cache their values and are primed by os_init() */
+const wchar_t*
+get_own_qualified_name(void);
+
+const wchar_t*
+get_own_unqualified_name(void);
+
+const wchar_t*
+get_own_short_qualified_name(void);
+
+const wchar_t*
+get_own_short_unqualified_name(void);
+
+bool
+is_safe_mode(void);
+
+bool
+systemwide_inject_enabled(void);
+
+void 
+check_for_run_once(HANDLE process, int rununder_mask);
+
+#ifndef X64
+inject_setting_mask_t
+systemwide_should_inject_64(HANDLE process, int *mask);
+#endif
+
+inject_setting_mask_t
+systemwide_should_inject(HANDLE process, int *mask);
+
+int
+get_remote_process_ldr_status(HANDLE process_handle);
+
+
+/* utility functions */
+const char *
+double_strrchr(const char *string, char c1, char c2);
+
+const wchar_t *
+double_wcsrchr(const wchar_t *string, wchar_t c1, wchar_t c2);
+
+const wchar_t*
+w_get_short_name(const wchar_t *exename);
+
+/* in case we ever want to build as unicode app, normally set by the compiler,
+ * if we want to build unicode we will have to set, however, since we 
+ * preprocess with gcc, this is the define used in the windows headers */
+#ifdef _UNICODE
+# define double_tcsrchr double_wcsrchr
+#else
+# define double_tcsrchr double_strrchr
+#endif
+
+int
+wchar_to_char(char *cdst, size_t buflen, 
+              PCWSTR wide_src, size_t bytelen);
+
+void
+display_message(char *msg, int wait_for_user);
+
+#endif
diff --git a/core/win32/injector.c b/core/win32/injector.c
new file mode 100644
index 000000000..4c4b69584
--- /dev/null
+++ b/core/win32/injector.c
@@ -0,0 +1,964 @@
+/* **********************************************************
+ * Copyright (c) 2002-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+/*
+ * injector.c - standalone injector for win32
+ */
+
+/* FIXME: Unicode support?!?! case 61 */
+
+/* Disable warnings from vc8 include files */
+/* buildtools\VC\8.0\dist\VC\PlatformSDK\Include\prsht.h(201)
+ * "nonstandard extension used : nameless struct/union"
+ */
+#pragma warning( disable : 4201 )
+/* buildtools\VC\8.0\dist\VC\PlatformSDK\Include\wintrust.h(459)
+ * "named type definition in parentheses"
+ */
+#pragma warning( disable : 4115 )
+
+#include "globals.h"
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include <commdlg.h>
+#include <imagehlp.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>             /* memset */
+#include <time.h>
+#include <tchar.h>
+
+#include "globals_shared.h"
+#include "ntdll.h"
+#include "inject_shared.h"
+#include "os_private.h"
+
+#define VERBOSE 0
+#if VERBOSE
+# define DO_VERBOSE(x) x
+# undef printf
+# define VERBOSE_PRINT(x) printf x
+#else
+# define DO_VERBOSE(x) 
+# define VERBOSE_PRINT(x)
+#endif
+#define FP stderr
+
+/* FIXME: case 64 would like ^C to kill child process, it doesn't.
+ * also, child process seems able to read stdin but not to write
+ * to stdout or stderr (in fact it dies if it tries)
+ * I think Ctrl-C is delivered only if you have a debugger(windbg) attached.
+ */
+#define HANDLE_CONTROL_C 0
+
+/* global vars */
+static int limit; /* in seconds */
+static BOOL showmem;
+static BOOL showstats;
+static BOOL inject;
+static BOOL force_injection;
+static BOOL use_environment = TRUE; /* FIXME : for now default to using
+                                     * the environment, below we check and
+                                     * never use the environment if using
+                                     * debug injection.  Revisit.
+                                     */
+static double wallclock; /* in seconds */
+
+/* FIXME : assert stuff, internal error, display_message duplicated from 
+ * pre_inject, share? */
+
+/* for asserts, copied from utils.h */
+#ifdef assert
+# undef assert
+#endif
+/* avoid mistake of lower-case assert */
+#define assert assert_no_good_use_ASSERT_instead
+void internal_error(char *file, int line, char *msg);
+#undef ASSERT
+#ifdef DEBUG
+void display_error(char *msg);
+# ifdef INTERNAL
+#   define ASSERT(x)         if (!(x)) internal_error(__FILE__, __LINE__, #x)
+# else 
+#   define ASSERT(x)         if (!(x)) internal_error(__FILE__, __LINE__, "")
+# endif /* INTERNAL */
+#else
+# define display_error(msg) ((void) 0)
+# define ASSERT(x)         ((void) 0)
+#endif /* DEBUG */
+
+/* in ntdll.c */
+extern char *get_application_name(void);
+extern char *get_application_pid(void);
+
+static void
+display_error_helper(wchar_t *msg) 
+{
+    wchar_t title_buf[MAX_PATH + 64];
+    _snwprintf(title_buf, BUFFER_SIZE_ELEMENTS(title_buf), 
+              L_PRODUCT_NAME L" Notice: %hs(%hs)", 
+              get_application_name(), get_application_pid());
+    NULL_TERMINATE_BUFFER(title_buf);
+
+    /* for unit tests: assume that if a limit is set, we are in a
+     *  script so it's ok to just display to stderr. avoids hangs when
+     *  an error is encountered. */
+    if (limit <= 0)
+        nt_messagebox(msg, title_buf);
+    else {
+        fprintf(FP, "\n\n%ls\n%ls\n\n", title_buf, msg);
+        fflush(FP);
+    }
+}
+
+void
+internal_error(char *file, int line, char *expr)
+{
+#ifdef INTERNAL
+# define FILENAME_LENGTH L""
+#else 
+/* truncate file name to first character */
+# define FILENAME_LENGTH L".1"   
+#endif
+    wchar_t buf[512];
+    _snwprintf(buf, BUFFER_SIZE_ELEMENTS(buf), 
+              L"Injector Error %" FILENAME_LENGTH L"hs:%d %hs\n", 
+              file, line, expr);
+    NULL_TERMINATE_BUFFER(buf);
+    display_error_helper(buf);
+    TerminateProcess(GetCurrentProcess(), (uint)-1);
+}
+
+#ifdef DEBUG
+void 
+display_error(char *msg)
+{
+# ifdef EXTERNAL_INJECTOR
+    fprintf(FP, msg);
+# else
+    wchar_t buf[512];
+    _snwprintf(buf, BUFFER_SIZE_ELEMENTS(buf), L"%hs", msg);
+    NULL_TERMINATE_BUFFER(buf);
+    display_error_helper(buf);
+# endif
+}
+#endif /* DEBUG */
+
+#if HANDLE_CONTROL_C
+BOOL WINAPI HandlerRoutine(DWORD dwCtrlType   //  control signal type
+                           )
+{
+    printf("Inside HandlerRoutine!\n");
+    fflush(stdout);
+    /*    GenerateConsoleCtrlEvent(dwCtrlType, phandle);*/
+    return TRUE;
+}
+#endif 
+
+static void
+print_mem_stats(HANDLE h)
+{
+    VM_COUNTERS mem;
+    /* not in DR library - floating point use is OK */
+    int secs = (int) wallclock;
+
+    if (!get_process_mem_stats(h, &mem)) {
+        /* failed */
+        memset(&mem, 0, sizeof(VM_COUNTERS));
+    }
+
+    if (showstats) {
+        int cpu = get_process_load(h);
+        /* Elapsed real (wall clock) time.  */
+        if (secs >= 3600) {     /* One hour -> h:m:s.  */
+            fprintf(FP, "%ld:%02ld:%02ldelapsed ",
+                    secs / 3600,
+                    (secs % 3600) / 60,
+                    secs % 60);
+        } else {
+            fprintf(FP, "%ld:%02ld.%02ldelapsed ",      /* -> m:s.  */
+                    secs / 60,
+                    secs % 60,
+                    0 /* for now */);
+        }
+        fprintf(FP, "%d%%CPU \n", cpu);
+        fprintf(FP, "(%lu tot, %lu RSS, %lu paged, %lu non, %lu swap)k\n",
+                mem.PeakVirtualSize/1024,
+                mem.PeakWorkingSetSize/1024,
+                mem.QuotaPeakPagedPoolUsage/1024,
+                mem.QuotaPeakNonPagedPoolUsage/1024,
+                mem.PeakPagefileUsage/1024);
+    }
+    if (showmem) {
+        fprintf(FP, "Process Memory Statistics:\n");
+        fprintf(FP, "\tPeak virtual size:         %6d KB\n",
+               mem.PeakVirtualSize/1024);
+        fprintf(FP, "\tPeak working set size:     %6d KB\n",
+               mem.PeakWorkingSetSize/1024);
+        fprintf(FP, "\tPeak paged pool usage:     %6d KB\n",
+               mem.QuotaPeakPagedPoolUsage/1024);
+        fprintf(FP, "\tPeak non-paged pool usage: %6d KB\n",
+               mem.QuotaPeakNonPagedPoolUsage/1024);
+        fprintf(FP, "\tPeak pagefile usage:       %6d KB\n",
+               mem.PeakPagefileUsage/1024);
+    }
+}
+
+/*************************************************************************/
+/* Following code handles the copying of environment variables to the 
+ * registry (the -env option, default on) and unsetting them later
+ *
+ * FIXME : race conditions with someone else modifing this registry key,
+ *         doesn't restore registry if -no_wait
+ * NOTE : doesn't propagate if using debug injection method (by design)
+ *         
+ */
+
+static BOOL created_product_reg_key;
+static BOOL created_image_reg_key;
+static HKEY image_name_key;
+static HKEY product_name_key;
+
+/* use macros to avoid duplication */
+/* This check is neccesary since we don't have 
+ * preprocessor warnings enabled on windows */
+#if defined(TEMP_CMD) || defined(DO_CMD)
+#error TEMP_CMD or DO_ENV_VARS macro already declared!
+#endif
+/* if need to add or remove environment variables looked for, do it here */
+#define DO_ENV_VARS()                             \
+ TEMP_CMD(options, OPTIONS);                      \
+ TEMP_CMD(logdir, LOGDIR);                        \
+ TEMP_CMD(unsupported, UNSUPPORTED);              \
+ TEMP_CMD(rununder, RUNUNDER);                    \
+ TEMP_CMD(autoinject, AUTOINJECT);                \
+ TEMP_CMD(cache_root, CACHE_ROOT);                \
+ TEMP_CMD(cache_shared, CACHE_SHARED)
+
+#define TEMP_CMD(name, NAME)                               \
+ static BOOL overwrote_##name##_value;                     \
+ static BOOL created_##name##_value;                       \
+ static TCHAR old_##name##_value[MAX_REGISTRY_PARAMETER]
+
+/* Note - leading spacs is to avoid make ugly rule on zero argument function
+ * declerations not using void - xref PR 215100 */
+ DO_ENV_VARS();
+
+static void
+set_registry_from_env(const TCHAR *image_name, const TCHAR *dll_path) 
+{
+#undef TEMP_CMD
+#define TEMP_CMD(name, NAME)                     \
+ BOOL do_##name;                                 \
+ TCHAR name##_value[MAX_REGISTRY_PARAMETER]
+
+    DO_ENV_VARS();
+
+    DWORD disp, size, type;
+    int res, len;
+    int rununder_int_value;
+
+    /* get environment variable values if they are set */
+#undef TEMP_CMD    
+#define TEMP_CMD(name, NAME)                                                  \
+ name##_value[0] = '\0';  /* to be pedantic */                                \
+ len = GetEnvironmentVariable(_TEXT(DYNAMORIO_VAR_##NAME), name##_value,      \
+                              BUFFER_SIZE_ELEMENTS(name##_value));            \
+ do_##name = (use_environment &&                                              \
+              (len > 0 ||                                                     \
+              (len == 0 && GetLastError() != ERROR_ENVVAR_NOT_FOUND)));       \
+ ASSERT(len < BUFFER_SIZE_ELEMENTS(name##_value));                            \
+ VERBOSE_PRINT(("Environment var %s for %s, value = %s\n",                    \
+        do_##name ? "set" : "not set", #name, name##_value));
+
+    DO_ENV_VARS();
+
+    /* we always want to set the rununder to make sure RUNUNDER_ON is on
+     * to support following children; we set RUNUNDER_EXPLICIT to allow
+     * injecting even when preinject is configured. */
+    /* FIXME: we read only decimal */
+    rununder_int_value = _ttoi(rununder_value);
+    rununder_int_value |= RUNUNDER_ON | RUNUNDER_EXPLICIT;
+
+    do_rununder = true;
+    _itot(rununder_int_value, rununder_value, 
+          10 /* FIXME : is the radix abstracted somewhere */);
+
+    /* for follow_children, we set DYNAMORIO_AUTOINJECT (unless
+     * overridden by env var: then child will use env value, while
+     * parent uses cmdline path) */
+    if (!do_autoinject && dll_path != NULL) {
+        _tcsncpy(autoinject_value, dll_path, BUFFER_SIZE_ELEMENTS(autoinject_value));
+        do_autoinject = true;
+    }
+
+    /* FIXME : doesn't support svchost-* yet */
+    ASSERT(_tcsicmp(_TEXT(SVCHOST_EXE_NAME), image_name));
+    res = RegCreateKeyEx(DYNAMORIO_REGISTRY_HIVE, 
+                         _TEXT(DYNAMORIO_REGISTRY_KEY), 0,  NULL, 
+                         REG_OPTION_NON_VOLATILE, KEY_CREATE_SUB_KEY, NULL,
+                         &product_name_key, &disp);
+    ASSERT(res == ERROR_SUCCESS);
+    if (disp == REG_CREATED_NEW_KEY) {
+        created_product_reg_key = TRUE;
+    }
+    res = RegCreateKeyEx(product_name_key, image_name, 0, NULL, 
+                         REG_OPTION_NON_VOLATILE, KEY_QUERY_VALUE|KEY_SET_VALUE, 
+                         NULL, &image_name_key, &disp);
+    ASSERT(res == ERROR_SUCCESS);
+    if (disp == REG_CREATED_NEW_KEY) {
+        created_image_reg_key = TRUE;
+    }
+    
+    DO_VERBOSE({
+        printf("created product key? %s\ncreated image key? %s\n",
+               created_product_reg_key ? "yes" : "no", 
+               created_image_reg_key ? "yes" : "no");
+        fflush(stdout);
+    });
+ 
+    /* Now set values */
+#undef TEMP_CMD
+#define TEMP_CMD(name, NAME)                                                  \
+ if (do_##name) {                                                             \
+    size = BUFFER_SIZE_BYTES(old_##name##_value);                             \
+    res = RegQueryValueEx(image_name_key, _TEXT(DYNAMORIO_VAR_##NAME),        \
+                          NULL, &type, (BYTE *) old_##name##_value, &size);   \
+    ASSERT(size <= BUFFER_SIZE_BYTES(old_##name##_value));                    \
+    if (res == ERROR_SUCCESS) {                                               \
+        overwrote_##name##_value = TRUE;                                      \
+        ASSERT(type == REG_SZ);                                               \
+    } else {                                                                  \
+        created_##name##_value = TRUE;                                        \
+    }                                                                         \
+    res = RegSetValueEx(image_name_key, _TEXT(DYNAMORIO_VAR_##NAME),          \
+                        0, REG_SZ, (BYTE *) name##_value,                     \
+                        (DWORD) (_tcslen(name##_value)+1)*sizeof(TCHAR));     \
+    ASSERT(res == ERROR_SUCCESS);                                             \
+    VERBOSE_PRINT(("%s %s registry value with \"%s\" replacing \"%s\"\n",     \
+           overwrote_##name##_value  ? "overwrote" : "created", #name,        \
+           name##_value, overwrote_##name##_value ? old_##name##_value : ""));\
+ }
+    
+    DO_ENV_VARS();
+
+    DO_VERBOSE({fflush(stdout);});
+}
+
+static void
+unset_registry_from_env(const TCHAR *image_name)
+{
+    int res;
+
+    VERBOSE_PRINT(("Restoring registry configuration\n"));
+
+    /* restore registry values */
+#undef TEMP_CMD
+#define TEMP_CMD(name, NAME)                                                 \
+ if (overwrote_##name##_value) {                                             \
+     res = RegSetValueEx(image_name_key, _TEXT(DYNAMORIO_VAR_##NAME), 0,     \
+                         REG_SZ /* FIXME : abstracted somewhere? */,         \
+                         (BYTE *) old_##name##_value,                        \
+                         (DWORD)(_tcslen(old_##name##_value)+1) * sizeof(TCHAR));\
+     ASSERT(res == ERROR_SUCCESS);                                           \
+     VERBOSE_PRINT(("Restored %s value to %s\n", #name, old_##name##_value));\
+ } else if (created_##name##_value) {                                        \
+     res = RegDeleteValue(image_name_key, _TEXT(DYNAMORIO_VAR_##NAME));      \
+     ASSERT(res == ERROR_SUCCESS);                                           \
+     VERBOSE_PRINT(("Deleted %s value\n", #name));                           \
+ }
+
+    DO_ENV_VARS();
+
+    /* delete keys if created them */
+    if (created_image_reg_key) {
+        res = RegDeleteKey(product_name_key, image_name);
+        ASSERT(res == ERROR_SUCCESS);
+        VERBOSE_PRINT(("deleted image reg key\n"));
+    }
+    if (created_product_reg_key) {
+        res = RegDeleteKey(DYNAMORIO_REGISTRY_HIVE, 
+                           _TEXT(DYNAMORIO_REGISTRY_KEY));
+        ASSERT(res == ERROR_SUCCESS);
+        VERBOSE_PRINT(("deleted product reg key\n"));
+    }
+    DO_VERBOSE({fflush(stdout);});
+}
+
+#undef DO_ENV_VARS
+#undef TEMP_CMD
+
+/************************** end environment->registry **********************/
+
+/***************************************************************************/
+/* The following code handles checking for, setting and unsetting of the
+ * debug key injection method
+ *
+ * This whole section can go away once we have our own version of create 
+ * process that doesn't check the debugger key FIXME
+ */
+
+static HKEY debugger_key;
+static TCHAR debugger_key_full_name[MAX_PATH];
+static TCHAR debugger_key_value[3*MAX_PATH];
+static DWORD debugger_key_value_size = BUFFER_SIZE_BYTES(debugger_key_value);
+static BOOL (*debug_stop_function)(int); 
+
+static BOOL
+using_debugger_key_injection(const TCHAR *image_name)
+{
+    int res;
+
+    debug_stop_function = (BOOL (*)(int))
+        (GetProcAddress(GetModuleHandle(TEXT("Kernel32")), 
+                        "DebugActiveProcessStop"));
+    
+    _sntprintf(debugger_key_full_name, 
+               BUFFER_SIZE_ELEMENTS(debugger_key_full_name), _TEXT("%hs\\%s"),
+               DEBUGGER_INJECTION_KEY, image_name);
+    NULL_TERMINATE_BUFFER(debugger_key_full_name);
+
+    VERBOSE_PRINT(("debugger key %s\n", debugger_key_full_name));
+
+    res = RegOpenKeyEx(DEBUGGER_INJECTION_HIVE, debugger_key_full_name, 
+                       0, KEY_QUERY_VALUE + KEY_SET_VALUE, &debugger_key);
+    if (ERROR_SUCCESS != res) 
+        return FALSE;
+    res = RegQueryValueEx(debugger_key, _TEXT(DEBUGGER_INJECTION_VALUE_NAME),
+                          NULL, NULL, (BYTE *) debugger_key_value, 
+                          &debugger_key_value_size);
+    if (ERROR_SUCCESS != res || 
+        /* FIXME : it would be better to check if our commandline matched 
+         * what was in the registry value, instead of looking for drinject */
+        _tcsstr(debugger_key_value, _TEXT(DRINJECT_NAME)) == 0) {
+        RegCloseKey(debugger_key);
+        return FALSE;
+    }
+
+    /* since returning TRUE, we don't close the debugger_key (it will be 
+     * needed by the unset and restore functions). The restore function will
+     * close it */
+
+    return TRUE;
+}
+
+static
+void unset_debugger_key_injection()
+{
+    if (debug_stop_function == NULL) {
+        int res = RegDeleteValue(debugger_key, 
+                                 _TEXT(DEBUGGER_INJECTION_VALUE_NAME));
+        VERBOSE_PRINT(("Succesfully deleted debugger registry value? %s\n", 
+                       (ERROR_SUCCESS == res) ? "yes" : "no"));
+        if (ERROR_SUCCESS != res) {
+            ASSERT(FALSE);
+            /* prevent infinite recursion, die now */
+            abort();
+        }
+    }
+}
+
+static
+void restore_debugger_key_injection(int id, BOOL started)
+{
+    int res;
+    if (debug_stop_function == NULL) {
+        res = RegSetValueEx(debugger_key, 
+                            _TEXT(DEBUGGER_INJECTION_VALUE_NAME), 0, REG_SZ, 
+                            (BYTE *) debugger_key_value, debugger_key_value_size);
+        VERBOSE_PRINT(("Succesfully restored debugger registry value? %s\n", 
+                       (ERROR_SUCCESS == res) ? "yes" : "no"));
+    } else {
+        if (started) {
+            res = (*debug_stop_function)(id);
+            VERBOSE_PRINT(("Succesfully detached from debugging process? %s\n",
+                           res ? "yes" : "no"));
+        }
+    }
+    /* close the global debugger key */
+    RegCloseKey(debugger_key);
+}
+
+/***************************** end debug key injection ********************/
+
+static const TCHAR*
+get_image_name(TCHAR *app_name)
+{
+    const TCHAR *name_start = double_tcsrchr(app_name, _TEXT('\\'), _TEXT('/'));
+    if (name_start == NULL) 
+        name_start = app_name;
+    else
+        name_start++;
+    return name_start;
+}
+
+static char *
+strip_quotes(char *str)
+{
+    if (str[0] == '\"' || str[0] == '\'' || str[0] == '`') {
+        /* remove quotes */
+        str++;
+        str[strlen(str)-1] = '\0';
+    }
+    return str;
+}
+
+int usage(char *us)
+{
+#ifdef EXTERNAL_INJECTOR
+    fprintf(FP,
+            "Usage : %s [-s limit_sec | -m limit_min | -h limit_hr | -no_wait] \n"
+            "        [-noinject] <application> [args...]\n"
+            "    -no_wait   don't wait for application to exit before returning\n"
+            "    -s -m -h   kill the application if runs longer than specified limit\n"
+            "    -noinject  just launch the process without injecting\n",
+            us);
+#else
+    fprintf(FP, "Usage: %s [-s limit_sec | -m limit_min | -h limit_hr | -no_wait] [-stats]\n"
+            "      [-mem] [-no_env | -env]  [-force] -noinject|<dll_to_inject> <program> <args...>\n", us);
+#endif
+    return 0;
+}
+
+#define MAX_CMDLINE 2048
+
+/* we must be a console application in order to have the process
+ * we launch NOT get a brand new console window!
+ */
+int __cdecl
+main(int argc, char *argv[], char *envp[])
+{
+    LPTSTR              app_name = NULL;
+    TCHAR               full_app_name[2*MAX_PATH];
+    TCHAR               app_cmdline[MAX_CMDLINE];
+    LPTSTR              ch;
+    const TCHAR*        image_name;
+    int                 i;
+    LPTSTR              pszCmdLine = GetCommandLine();
+    LPTSTR              DYNAMORIO_PATH = NULL;
+    PLOADED_IMAGE       li;
+    STARTUPINFO         si;
+    PROCESS_INFORMATION pi;
+    CONTEXT             cxt;
+    HANDLE              phandle;
+    BOOL                success;
+    BOOL                res;
+    BYTE *              app_entry;
+    DWORD               wait_result;
+    int                 arg_offs = 1;
+    time_t              start_time, end_time;
+    BOOL                debugger_key_injection;
+    BOOL                limit_default = TRUE;
+#ifdef EXTERNAL_INJECTOR
+    char                library_path_buf[MAXIMUM_PATH];
+#endif
+
+    STARTUPINFO mysi;
+    GetStartupInfo(&mysi);
+
+    /* parse args */
+    showmem = FALSE;
+    inject = TRUE;
+
+#ifdef EXTERNAL_INJECTOR
+    if (argc < 2) {
+        return usage(argv[0]);
+    }
+#else
+    if (argc < 3) {
+        return usage(argv[0]);
+    }
+#endif
+    while (argv[arg_offs][0] == '-') {
+        if (strcmp(argv[arg_offs], "-s") == 0) {
+            if (argc <= arg_offs+1)
+                return usage(argv[0]);
+            limit = atoi(argv[arg_offs+1]);
+            limit_default = FALSE;
+            arg_offs += 2;
+        } else if (strcmp(argv[arg_offs], "-m") == 0) {
+            if (argc <= arg_offs+1)
+                return usage(argv[0]);
+            limit = atoi(argv[arg_offs+1])*60;
+            limit_default = FALSE;
+            arg_offs += 2;
+        } else if (strcmp(argv[arg_offs], "-h") == 0) {
+            if (argc <= arg_offs+1)
+                return usage(argv[0]);
+            limit = atoi(argv[arg_offs+1])*3600;
+            limit_default = FALSE;
+            arg_offs += 2;
+        } else if (strcmp(argv[arg_offs], "-no_wait") == 0) {
+            limit = -1;
+            limit_default = FALSE;
+            arg_offs += 1;
+        } else if (strcmp(argv[arg_offs], "-noinject") == 0) {
+            inject = FALSE;
+            arg_offs += 1;
+        }
+#ifndef EXTERNAL_INJECTOR
+        else if (strcmp(argv[arg_offs], "-v") == 0) {
+            /* just ignore -v, only here for compatibility with texec */
+            arg_offs += 1;
+        } else if (strcmp(argv[arg_offs], "-mem") == 0) {
+            showmem = TRUE;
+            arg_offs += 1;
+        } else if (strcmp(argv[arg_offs], "-stats") == 0) {
+            showstats = TRUE;
+            arg_offs += 1;
+        } else if (strcmp(argv[arg_offs], "-force") == 0) {
+            VERBOSE_PRINT(("Forcing injection\n"));
+            /* note this overrides INJECT_EXCLUDED */
+            force_injection = TRUE;
+            arg_offs += 1;
+        } else if (strcmp(argv[arg_offs], "-no_env") == 0) {
+            use_environment = FALSE;
+            arg_offs += 1;
+        } else if (strcmp(argv[arg_offs], "-env") == 0) {
+            use_environment = TRUE;
+            arg_offs += 1;
+        }
+#endif /* ! EXTERNAL_INJECTOR */
+        else {
+            return usage(argv[0]);
+        }
+        if (limit < -1 && !limit_default) {
+            return usage(argv[0]);
+        }
+        if (argc - arg_offs < 1) {
+            return usage(argv[0]);
+        }
+    }
+
+#ifndef EXTERNAL_INJECTOR
+    if (inject) {
+        if (argc - arg_offs < 2) {
+            return usage(argv[0]);
+        }
+        /* we don't want quotes included in our dll path string or our
+         * application path string (app_name), but we do want to put
+         * quotes around every member of the command line
+         */
+        DYNAMORIO_PATH = strip_quotes(argv[arg_offs]);
+        arg_offs += 1;
+    }
+#endif
+
+    /* we don't want quotes included in our
+     * application path string (app_name), but we do want to put
+     * quotes around every member of the command line
+     */
+    app_name = strip_quotes(argv[arg_offs]);
+    arg_offs += 1;
+
+    /* note that we want target app name as part of cmd line
+     * (FYI: if we were using WinMain, the pzsCmdLine passed in
+     *  does not have our own app name in it)
+     * since cygwin shell ends up putting extra quote characters, etc.
+     * in pszCmdLine ('"foo"' => "\"foo\""), we can't easily walk past
+     * the 1st 2 args (our path and the dll path), so we reconstruct
+     * the cmd line from scratch:
+     */
+    ch = app_cmdline;
+    ch += _snprintf(app_cmdline, BUFFER_SIZE_ELEMENTS(app_cmdline), "\"%s\"", 
+                    app_name);
+    for (i = arg_offs; i < argc; i++)
+        ch += _snprintf(ch, 
+                        BUFFER_SIZE_ELEMENTS(app_cmdline) - (ch - app_cmdline),
+                        " \"%s\"", argv[i]);
+    NULL_TERMINATE_BUFFER(app_cmdline);
+    ASSERT(ch - app_cmdline < BUFFER_SIZE_ELEMENTS(app_cmdline));
+
+#if HANDLE_CONTROL_C
+    /* Note that one must call SetConsoleCtrlHandler(NULL, FALSE) to enable
+     * ^C event reception: but under bash/rxvt it still won't work
+     */
+    if (!SetConsoleCtrlHandler(NULL, FALSE) ||
+        !SetConsoleCtrlHandler(HandlerRoutine, TRUE)) {
+        display_error("SetConsoleCtrlHandler failed");
+        return 1;
+    }
+    {
+        int flags;
+        /* FIXME: hStdInput apparently is not the right handle...how do I get
+         * the right handle?!?
+         */
+        if (!GetConsoleMode(mysi.hStdInput, &flags)) {
+            display_error("GetConsoleMode failed");
+            return 1;
+        }
+        printf("console mode flags are: "PFX"\n", flags);
+    }
+#endif
+
+    _tsearchenv(app_name, _TEXT("PATH"), full_app_name);
+    if (full_app_name[0] == _TEXT('\0')) {
+        /* may need to append .exe, FIXME : other executable types */
+        TCHAR tmp_buf[BUFFER_SIZE_ELEMENTS(app_name)+5];
+        _sntprintf(tmp_buf, BUFFER_SIZE_ELEMENTS(tmp_buf), 
+                   _TEXT("%s%s"), app_name, _TEXT(".exe"));
+        NULL_TERMINATE_BUFFER(tmp_buf);
+        VERBOSE_PRINT(("app name with exe %s\n", tmp_buf));
+        _tsearchenv(tmp_buf, _TEXT("PATH"), full_app_name);
+    }
+    VERBOSE_PRINT(("app name %s, full app name %s\n", app_name, 
+                   full_app_name));
+    app_name = full_app_name;
+
+    if (NULL != (li = ImageLoad(app_name, NULL))) {
+        app_entry = (BYTE *)
+            (li->MappedAddress +
+             li->FileHeader->OptionalHeader.AddressOfEntryPoint);
+        ImageUnload(li);
+    } else {
+        _snprintf(app_cmdline, BUFFER_SIZE_ELEMENTS(app_cmdline), 
+                  "Failed to load executable image \"%s\"", app_name);
+        NULL_TERMINATE_BUFFER(app_cmdline);
+        display_error(app_cmdline);
+        return 1;
+    }
+
+    DO_VERBOSE({
+        printf("Running %s with cmdline \"%s\"\n", app_name, app_cmdline);
+        fflush(stdout);
+    });
+    
+    /* Launch the application process. */
+    ZeroMemory(&si, sizeof(si));
+    si.cb = sizeof(si);
+    si.dwFlags = STARTF_USESTDHANDLES;
+    si.hStdInput = mysi.hStdInput;
+    si.hStdOutput = mysi.hStdOutput;
+    si.hStdError = mysi.hStdError;
+
+    image_name = get_image_name(app_name);
+
+    /* FIXME, won't need to check this, or unset/restore debugger_key_injection
+     * if we have our own version of CreateProcess that doesn't check the 
+     * debugger key */
+    debugger_key_injection = using_debugger_key_injection(image_name);
+
+    /* set defaults for limit */
+    if (limit_default) {
+        if (debugger_key_injection)
+            limit = -1; /* no wait */
+        else 
+            limit = 0; /* infinite wait */
+    }
+    
+    DO_VERBOSE({
+        printf("Using debug method? %s, debug_stop_function available %s\n", 
+               debugger_key_injection ? "Yes" : "No", 
+               debug_stop_function ? "Yes" : "No");
+        fflush(stdout);
+    });
+
+#ifndef EXTERNAL_INJECTOR
+    /* don't set registry from environment if using debug key */
+    if (!debugger_key_injection) {
+        set_registry_from_env(image_name, DYNAMORIO_PATH);
+    }
+#endif
+
+    if (debugger_key_injection) {
+        unset_debugger_key_injection();
+    }
+
+    DO_VERBOSE({
+        printf("starting process\n");
+        fflush(stdout);
+    });
+
+    /* Must specify TRUE for bInheritHandles so child inherits stdin! */
+    res = CreateProcess(app_name, app_cmdline, NULL, NULL, TRUE,
+                        CREATE_SUSPENDED + 
+                        ((debug_stop_function && 
+                          debugger_key_injection) ? DEBUG_PROCESS | 
+                                                    DEBUG_ONLY_THIS_PROCESS 
+                                                  : 0),
+                        NULL, NULL, &si, &pi);
+
+    DO_VERBOSE({
+        printf("done starting process\n");
+        fflush(stdout);
+    });
+
+    if (debugger_key_injection) {
+        restore_debugger_key_injection(pi.dwProcessId, res);
+    }
+    
+    if (!res) {
+        display_error("Failed to launch application");
+        goto error;
+    }
+
+    if ((phandle = OpenProcess(PROCESS_ALL_ACCESS,
+                               FALSE, pi.dwProcessId)) == NULL) {
+        display_error("Cannot open application process");
+        TerminateProcess(pi.hProcess, 0);
+        goto error;
+    }
+
+    /* force_injection prevents overriding of inject based on registry */
+    if (inject && !force_injection) {
+        int inject_flags = systemwide_should_inject(phandle, NULL);
+        bool syswide_will_inject = systemwide_inject_enabled() && 
+            TEST(INJECT_TRUE, inject_flags) && 
+            !TEST(INJECT_EXPLICIT, inject_flags);
+        bool should_not_takeover = TEST(INJECT_EXCLUDED, inject_flags) && 
+            debugger_key_injection;
+        /* case 10794: to support follow_children we inject even if
+         * syswide_will_inject.  we use RUNUNDER_EXPLICIT to avoid
+         * user32 injection from happening, to get consistent injection.
+         * (if we didn't things would work but we'd have 
+         * a warning "<Blocking load of module dynamorio.dll>" on the 2nd
+         * injection)
+         */
+        inject = !should_not_takeover;
+        if (!inject) {
+            /* we should always be injecting (we set the registry above) 
+             * unless we are using debugger_key_injection, in which 
+             * case we use what is in the registry (whoever wrote the registry 
+             * should take care of late or nonexistent user32 loading in the 
+             * rununder value) */
+            ASSERT(debugger_key_injection);
+            display_error("application is excluded from injection\n");
+        } else {
+            /* Don't inject if in safe mode */
+            if (is_safe_mode()) {
+                inject = false;
+                display_error("System is in safe mode, not injecting\n");
+            }
+        }
+    }
+
+    VERBOSE_PRINT(("%s in %s (%s)\n", inject ? "Injecting" : "Native", 
+                   app_name, argv[0]));
+
+#ifdef EXTERNAL_INJECTOR
+    if (inject) {
+        int err = get_process_parameter(phandle, L_DYNAMORIO_VAR_AUTOINJECT,
+                                        library_path_buf, sizeof(library_path_buf));
+        if (err != GET_PARAMETER_SUCCESS && err != GET_PARAMETER_NOAPPSPECIFIC) {
+            inject = false;
+            display_error("application not properly configured\n"
+                          "use drdeploy.exe to configure the application first");
+        }
+        NULL_TERMINATE_BUFFER(library_path_buf);
+        DYNAMORIO_PATH = library_path_buf;
+    }
+#endif
+
+    if (inject) {
+        inject_init();
+        if (!inject_into_thread(phandle, &cxt, pi.hThread, DYNAMORIO_PATH)) {
+            display_error("injection failed");
+            close_handle(phandle);
+            TerminateProcess(pi.hProcess, 0);
+            goto error;
+        }
+        DO_VERBOSE({
+            printf("Successful at inserting dynamo\n");
+            fflush(stdout);
+        });
+    }
+
+    success = FALSE;
+    /* FIXME: indentation left from removing a try block */
+        start_time = time(NULL);
+
+        /* resume the suspended app process so its main thread can run */
+        ResumeThread(pi.hThread);
+    
+        /* detach from the app process */
+        close_handle(pi.hThread);
+        close_handle(pi.hProcess);
+
+        if (limit >= 0) {
+            VERBOSE_PRINT(("Waiting for app to finish\n"));
+            /* now wait for app process to finish */
+            wait_result = WaitForSingleObject(phandle, (limit==0) ? INFINITE : (limit*1000));
+            end_time = time(NULL);
+            wallclock = difftime(end_time, start_time);
+#ifndef EXTERNAL_INJECTOR
+            if (!debugger_key_injection) {
+                unset_registry_from_env(image_name);
+            }
+#endif
+            if (wait_result == WAIT_OBJECT_0)
+                success = TRUE;
+            else
+                fprintf(FP, "Timeout after %d seconds\n", limit);
+        } else {
+            /* if we are using env -> registry our changes won't get undone!
+             * we can't unset now, the app may still reference them */
+            success = TRUE;
+        }
+
+        /* FIXME: this is my attempt to get ^C, but it doesn't work... */
+#if HANDLE_CONTROL_C
+        /* should we register a control handler for this to work? */
+        if (!success) {
+            printf("Terminating child process!\n");
+            fflush(stdout);
+            TerminateProcess(phandle, 0);
+        } else
+            printf("Injector exiting peacefully\n");
+#endif
+#ifndef EXTERNAL_INJECTOR
+        if (showmem || showstats)
+            print_mem_stats(phandle);
+#endif
+        if (!success) {
+            /* kill the child */
+            TerminateProcess(phandle, 0);
+        }
+        close_handle(phandle);
+        DO_VERBOSE({
+            printf("All done\n");
+            fflush(stdout);
+        });
+
+    return 0;
+
+ error:
+#ifndef EXTERNAL_INJECTOR
+    if (!debugger_key_injection) {
+        unset_registry_from_env(image_name);
+    }
+#endif
+    return 1;
+}
diff --git a/core/win32/module.c b/core/win32/module.c
new file mode 100644
index 000000000..7da29af3d
--- /dev/null
+++ b/core/win32/module.c
@@ -0,0 +1,6174 @@
+/* **********************************************************
+ * Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+
+/* module.c - maintains information about modules (dll or executable images) */
+
+
+#include "globals.h"
+#include "ntdll.h"
+#include <stddef.h> /* for offsetof */
+
+#ifdef RCT_IND_BRANCH
+#  include "rct.h"
+#endif
+
+#include "utils.h"
+#include "os_private.h"
+#include "aslr.h"
+#include "instrument.h"
+#include "perscache.h" /* for coarse_info_t.rct_loaded */
+
+/* used to hold the version information we get from the .rsrc section */
+typedef struct _version_info_t {
+    version_number_t file_version;
+    version_number_t product_version;
+    wchar_t *company_name;
+    wchar_t *product_name;
+    wchar_t *original_filename;
+} version_info_t;
+
+static const char *
+get_module_original_filename(app_pc mod_base HEAPACCT(which_heap_t which));
+
+static bool
+module_area_free_IAT(module_area_t *ma);
+
+/* NOTE the strings returned in info_out are pointing to the .rsrc version
+ * directory and as such they're only valid while the module is loaded. */
+static bool
+get_module_resource_version_info(app_pc mod_base, version_info_t *info_out);
+
+/****************************************************************************/
+#ifdef DEBUG                    /* around symbols related part of the file */
+
+#include <windows.h>
+
+/* Currently only uses dll export functions for DEBUG symbol information */
+
+/* Private data types */
+typedef struct export_entry_t {
+    app_pc  entry_point;          /* exported function entry point */
+    char *export_name;
+} export_entry_t;
+
+/* FIXME: a module can have multiple code sections and each should
+ * have a separate searchable entry, yet all relevant per-module
+ * structures should be thrown away when a module is unloaded.
+ * Caution with data sections (or other invalidated vmareas) that may
+ * be within the module region.
+ */
+
+/* use module_area_t for any release build needs */
+typedef struct module_info_t {
+    app_pc start;
+    app_pc end;                 /* open end interval */
+    char * module_name;
+    size_t exports_size;        /* initial export table length */
+    uint   exports_num;         /* number of unique exports */
+
+    export_entry_t *exports_table; /* sorted array to allow range searches */
+    /* FIXME: this is necessary only for debug symbol lookup, if
+     * implementing export restrictions (case 286), we only need
+     * membership tests so we only have to populate the entry points
+     * into a hashtable
+     */
+
+    /* FIXME: case 3927
+     * It will be quite useful for some debugging sessions to
+     * use real symbols from PDB's.  It maybe easier if they are
+     * already preprocessed with windbg -c 'x module!*' or some other
+     * PDB parser.  Then we'll have an easier time debugging and
+     * reverse engineering some external components.  We could even
+     * think about security policies (or constraints) that would
+     * require symbols, since at least M$ has at least public PDB's
+     * (e.g. not private with variable names) but a good starting
+     * point.
+     */
+} module_info_t;
+
+/* This structure is parallel to the one kept by the loader - 
+ * yet may be safer to use.  See comment in vm_area_vector_t.
+ *
+ * FIXME: We may want to abstract this and vm_area_vector_t into a common data structure
+ * with polymorphic elements with a start,end
+ */
+
+/* The module vector is kept sorted by area.  Since there are no overlaps allowed
+ * among areas in the same vector, sorting by start_pc or by end_pc produce
+ * identical results.
+ */
+
+typedef struct module_info_vector_t {
+    struct module_info_t *buf;
+    int   capacity;
+    int   length;
+    /* thread-shared, so needs a lock */
+    /* FIXME: make this a read/write lock if readers contend too often */
+    mutex_t lock;
+} module_info_vector_t;
+
+/* debug-only so we don't need to efficiently protect it */
+DECLARE_CXTSWPROT_VAR(static module_info_vector_t process_module_vector,
+                      {NULL, 0, 0, INIT_LOCK_FREE(process_module_vector_lock)});
+
+static void
+print_module_list(module_info_vector_t *v)
+{
+    int i;
+    LOG(GLOBAL, LOG_SYMBOLS, 4,
+        "print_module_list("PFX") capacity=%d, length=%d, lock=%d, buf="PFX, 
+        v, v->capacity, v->length, v->lock, v->buf);
+
+    mutex_lock(&v->lock);
+    for(i = 0; i < v->length; i++) {
+        LOG(GLOBAL, LOG_SYMBOLS, 3, "  "PFX"-"PFX" %s, %d exports ["SZFMT" size]\n", 
+            v->buf[i].start,
+            v->buf[i].end,
+            v->buf[i].module_name,
+            v->buf[i].exports_num,
+            v->buf[i].exports_size);
+    }
+    mutex_unlock(&v->lock);
+}
+
+
+/* For binary search */
+int
+module_info_compare(const void *vkey, const void *vel)
+{
+    module_info_t *key = ((module_info_t *)vkey);
+    module_info_t *el = ((module_info_t *)vel);
+    if (key->end <= el->start)
+        return (-1); /* key less than element */
+
+    if (key->start >= el->end)
+        return (1); /* key greater than element */
+
+    return (0); /* key equals (overlaps) element */
+}
+
+/* lookup a module by address, 
+   assumes the v->lock is held by caller! 
+   returned module_info_t *should not be used after releasing lock
+   returns NULL if no module found
+*/
+static module_info_t*
+lookup_module_info(module_info_vector_t *v, app_pc addr)
+{
+    /* BINARY SEARCH -- assumes the vector is kept sorted by add & remove! */
+    module_info_t key = {addr, addr+1}; /* end is open */
+#ifdef NOLIBC
+    /* FIXME : copied from find_predecessor(), would be nice to share with
+     * that routine and with binary range search (w/linear backsearch) in 
+     * vmareas.c */
+    int min = 0;
+    int max = v->length - 1;
+    /* binary search */
+    while (max >= min) {
+        int i = (min + max) / 2;
+        int cmp = module_info_compare(&key, &(v->buf[i]));
+        if (cmp < 0) 
+            max = i - 1;
+        else if (cmp > 0) 
+            min = i + 1;
+        else {
+            return &(v->buf[i]);
+        }
+    }
+    return NULL;
+#else
+    return bsearch(&key, v->buf, v->length,
+                   sizeof(module_info_t), module_info_compare);
+#endif
+}
+
+#define INITIAL_MODULE_NUMBER 4
+
+/* Creates a new module info, allocates its exports table, and adds to module vector, 
+ * module_name is caller allocated (module_name is from the exports section for PE dlls)
+ * 
+ * Returns a pointer to the module's export table
+ */
+static
+export_entry_t *
+module_info_create(module_info_vector_t *v, app_pc start, app_pc end, 
+                   char *module_name, uint exports_num)
+{
+    struct module_info_t new_module = {start, end, module_name, exports_num, exports_num, 0 /* table */}; 
+    int i,j;
+
+    if (exports_num > 0) {
+        new_module.exports_table = global_heap_alloc(exports_num * sizeof(export_entry_t)
+                                                     HEAPACCT(ACCT_SYMBOLS));
+    } else {
+        new_module.exports_table = NULL;
+    }
+
+    mutex_lock(&v->lock);
+    /* FIXME: the question is what to do when an overlap occurs.
+     * If we assume that we should have removed the references from an old DLL.
+     * A possibly new DLL overlapping the same range should not show up,
+     * this indeed would be an error worth investigating.
+     */
+    /* FIXME: need a real overlap check */
+    if (lookup_module_info(v, start) != NULL) {
+        ASSERT_NOT_REACHED();
+        return NULL;
+    }
+
+    for (i = 0; i < v->length; i++) {
+        if (end <= v->buf[i].start)
+            break;
+    }
+    /* check if at full capacity */
+    if (v->capacity == v->length){
+        int new_size = v->capacity ? v->capacity * 2 : INITIAL_MODULE_NUMBER;
+        v->buf = global_heap_realloc(v->buf, v->capacity, new_size,
+                                     sizeof(struct module_info_t) HEAPACCT(ACCT_SYMBOLS));
+        v->capacity = new_size;
+        ASSERT(v->buf);
+    }
+    /* shift subsequent to i entries */
+    for (j = v->length; j > i; j--)
+        v->buf[j] = v->buf[j-1];
+
+    v->buf[i] = new_module;
+    v->length++;
+    mutex_unlock(&v->lock);
+    DOLOG(3, LOG_SYMBOLS, { print_module_list(v); });
+
+    /* we can not return &v->buf[i] since buf may get realloc-ed, or buf[i] may be shifted */
+    return new_module.exports_table;
+}
+
+
+/* remove from module vector and free up memory */
+static
+void
+remove_module_info_vector(module_info_vector_t *v, app_pc start, app_pc end)
+{
+    int i,j;
+
+    export_entry_t *exports_table = NULL;
+    size_t exports_size = 0;
+
+    mutex_lock(&v->lock);
+    /* linear search, we don't have a find_predecessor on module_info_t's to get i */
+    for (i = 0; i < v->length; i++) {
+        if (start == v->buf[i].start && end == v->buf[i].end) {
+            exports_table = v->buf[i].exports_table;
+            exports_size = v->buf[i].exports_size;
+            break;
+        }
+    }
+
+    LOG(GLOBAL, LOG_SYMBOLS, 2, "remove_module_info_vector("PFX","PFX") dll=%s\n",
+        start, end, v->buf[i].module_name);
+    ASSERT_CURIOSITY(exports_table);      /* curiosity */
+    if (!exports_table) {
+        /* it could have disappeared since we last checked */
+        mutex_unlock(&v->lock);
+        return;                
+    }
+
+    /* shift subsequent to i entries over */
+    for (j = i+1; j < v->length; j++)
+        v->buf[j-1] = v->buf[j];
+    v->length--;
+    mutex_unlock(&v->lock);
+
+    if (exports_size > 0) {
+        global_heap_free(exports_table, exports_size * sizeof(export_entry_t)
+                         HEAPACCT(ACCT_SYMBOLS));
+    }
+        
+    DOLOG(3, LOG_SYMBOLS, { print_module_list(v); });
+}
+
+/* remove internal bookkeeping for unloaded module
+   returns 1 if the range is a known module, and that is removed,
+           0 otherwise
+*/
+int
+remove_module_info(app_pc start, size_t size)
+{
+    module_info_t *pmod;
+    mutex_lock(&process_module_vector.lock);
+    pmod = lookup_module_info(&process_module_vector, start);
+    mutex_unlock(&process_module_vector.lock);
+
+    if (!pmod) { /* FIXME: need a real overlap check */
+        LOG(GLOBAL, LOG_SYMBOLS, 2,
+            "WARNING:remove_module_info called on unknown module "PFX", size "PIFX"\n",
+            start, size);
+        /* my assert_curiosity was triggered, yet unexplained */
+        return 0; 
+    }
+
+    remove_module_info_vector(&process_module_vector, (app_pc)start, (app_pc)(start + size));
+
+    return 1;
+}
+
+void
+module_cleanup(void)
+{
+    module_info_vector_t *v = &process_module_vector;
+    int i;
+    export_entry_t *exports_table = NULL;
+    size_t exports_size = 0;
+    mutex_lock(&v->lock);
+    /* linear search, we don't have a find_predecessor on module_info_t's to get i */
+    for (i = 0; i < v->length; i++) {
+        exports_table = v->buf[i].exports_table;
+        exports_size = v->buf[i].exports_size;
+        if (exports_size > 0) {
+            global_heap_free(exports_table, exports_size * sizeof(export_entry_t)
+                             HEAPACCT(ACCT_SYMBOLS));
+        }
+    }
+    if (v->buf != NULL)
+        global_heap_free(v->buf, v->capacity * sizeof(struct module_info_t)
+                         HEAPACCT(ACCT_SYMBOLS));
+    v->buf = NULL;
+    v->capacity = 0;
+    v->length = 0;
+    mutex_unlock(&v->lock);
+}
+
+void
+module_info_exit()
+{
+    module_cleanup();
+    DELETE_LOCK(process_module_vector.lock);
+}
+
+int
+export_entry_compare(const void *vkey, const void *vel)
+{
+    /* used for qsort so only care about sign; truncation is ok */
+    return (int)(((export_entry_t*)vkey)->entry_point
+                 - ((export_entry_t*)vel)->entry_point);
+}
+
+
+/* Returns the offset within table[] of the last element equal or smaller than key, 
+   table[] must be sorted in ascending order. 
+   Returns -1 when smaller than the first element, or array empty
+*/
+int 
+find_predecessor(export_entry_t table[], int n, app_pc tag) 
+{
+  int min = 0;
+  int max = n - 1;
+  /* binary search */
+  while (max >= min) {
+      int i = (min + max) / 2;
+
+      if (tag < table[i].entry_point) 
+          max = i - 1;
+      else if (tag > table[i].entry_point) 
+          min = i + 1;
+      else {
+          return i;
+      }
+  }
+  
+  /* now max < min */
+  return max;                   /* may be -1 */
+}
+
+/* remove duplicate export entries, 
+   returns number of unique entry points
+   (assumes table is ordered by address) 
+*/
+int
+remove_export_duplicates(export_entry_t table[], int n) 
+{
+    int i=0, j=1;
+
+    if (n < 2)
+        return n;
+
+    while (j < n) {
+        if (table[i].entry_point == table[j].entry_point) {
+            LOG(GLOBAL, LOG_SYMBOLS, 3,
+                "Export alias %s == %s\n", table[i].export_name, table[j].export_name);
+        } else {
+            i++;
+            table[i] = table[j];
+        }
+
+        j++;
+    }
+    i++;
+
+    return i;
+}
+
+/* prints a symbolic name, or best guess of it into a caller provided buffer */
+void
+print_symbolic_address(app_pc tag, char *buf, int max_chars, bool exact_only)
+{
+    module_info_t *pmod;          /* volatile pointer */
+    module_info_t mod = {0};            /* copy of module info */
+
+    /* FIXME: cannot grab this lock under internal_exception_lock */
+    if (under_internal_exception()) {
+        pmod = NULL;
+    } else {
+        mutex_lock(&process_module_vector.lock); /* FIXME: this can be a shared read lock */
+        {
+            pmod = lookup_module_info(&process_module_vector, tag);
+            if (pmod) {
+                mod = *pmod; /* keep a copy in case of reallocations */
+                /* the data will be invalid only in a race condition,
+                   where some other thread frees the library */
+            }
+        }
+        mutex_unlock(&process_module_vector.lock); 
+    }
+
+    buf[0]='\0';
+    if (pmod != NULL) {
+        int i = find_predecessor(mod.exports_table, mod.exports_num, tag);
+        if (i<0) {              /* tag smaller than first exported function */
+            /* convert to offset from base */
+            if (!exact_only) {
+                snprintf(buf, max_chars, "[%s~%s+"PIFX"]",
+                         mod.module_name, ".begin", 
+                         tag - mod.start);
+            }
+        } else {
+            if (mod.exports_table[i].entry_point == tag) {
+                /* tag exactly matches an export, i.e. <ntdll!CsrIdentifyAlertableThread> */
+                snprintf(buf, max_chars, "[%s!%s]", mod.module_name, mod.exports_table[i].export_name);
+            } else if (!exact_only) {
+                uint prev = (uint)i;
+                uint next = (uint)i+1;
+                
+                ASSERT((uint)i < mod.exports_num);
+                /* <KERNEL32.dll~CreateProcessW+0x1564,~RegisterWaitForInputIdle-0x9e> */
+                snprintf(buf, max_chars, "[%s~%s+"PIFX",~%s-"PIFX"]",
+                         mod.module_name, mod.exports_table[prev].export_name, 
+                         tag - (ptr_uint_t)mod.exports_table[prev].entry_point,
+                         next < mod.exports_num ?
+                         mod.exports_table[next].export_name : ".end",
+                         (next < mod.exports_num ?
+                          mod.exports_table[next].entry_point :
+                          mod.end) - (ptr_uint_t)tag);
+            }
+        }
+    } else {
+        char modname_buf[MAX_MODNAME_INTERNAL];
+        const char *short_name = NULL;
+        if (under_internal_exception()) {
+            /* We're called in fragile situations so we explicitly check here.
+             * Will get lock rank order in accessing module_data_lock so just
+             * use PE name.  This is for debugging only anyway.
+             */
+            app_pc base = get_allocation_base(tag);
+            if (base != NULL && is_readable_pe_base(base))
+                short_name = get_dll_short_name(base);
+            if (short_name == NULL)
+                short_name = "";
+        } else {
+            os_get_module_name_buf(tag, modname_buf, BUFFER_SIZE_ELEMENTS(modname_buf));
+            short_name = modname_buf;
+        }
+        /* since currently we aren't working well w/ dynamically loaded dlls, and
+         * certain things are disabled at lower loglevels, fall back to the short name
+         */
+        DODEBUG({
+            get_module_name(tag, buf, max_chars);
+            /* check if we get the same name */
+            if (strcasecmp(get_short_name(buf), short_name) != 0 && buf[0]!='\0') {
+                /* after a module is off the module list some code from it still
+                 * gets executed */
+                /* In addition there are modules with different file names, 
+                   e.g. wdmaud.drv != wdmaud.dll (export section name) */
+                LOG(GLOBAL, LOG_SYMBOLS, 3,
+                    "WARNING: print_symbolic_address("PFX"): ldr name='%s' "
+                    "pe name='%s'\n", tag, get_short_name(buf), short_name);
+            }
+        });
+        snprintf(buf, max_chars, "[%s]", short_name);
+    }
+    buf[max_chars-1]='\0';      /* to make sure */
+    LOG(GLOBAL, LOG_SYMBOLS, 5, "print_symbolic_address("PFX")='%s'\n", tag, buf);
+}
+
+/* adds a module to the module_info_t list, and parses its exports table,
+   this can be done as soon as the module is mapped in the address space */
+/* returns 1 if successfully added
+           0 if address range is not a PE file */
+int
+add_module_info(app_pc base_addr, size_t image_size)
+{
+    size_t size;
+    IMAGE_EXPORT_DIRECTORY *exports =
+        get_module_exports_directory_check(base_addr, &size, true);
+            
+    if (exports != NULL) {
+        PULONG functions =  (PULONG) (base_addr + exports->AddressOfFunctions);
+        PUSHORT ordinals = (PUSHORT) (base_addr + exports->AddressOfNameOrdinals);
+        PULONG fnames    =  (PULONG) (base_addr + exports->AddressOfNames);
+        char *dll_name =  (char*) (base_addr + exports->Name);
+        uint exports_num = 0;
+        uint i;
+
+        export_entry_t *exports_table;
+
+        LOG(GLOBAL, LOG_SYMBOLS, 4, "\tnumnames=%d numfunc=%d", 
+            exports->NumberOfNames, exports->NumberOfFunctions);
+
+        if (exports->NumberOfFunctions != exports->NumberOfNames) {
+            /* TODO: we should also use the knowledge about the noname [ordinal] entry points  */
+            /* shlwapi.dll or winspool.drv  are good examples
+             * find more with dumpbin /exports shlwapi.dll | grep "number of"
+             */
+            /* These are in fact much more important for
+             * rct_add_exports() where we traverse functions otherwise
+             * we'd have a .E in shlwapi on a noname export
+             * SHLWAPI!Ordinal80
+             */
+            LOG(GLOBAL, LOG_SYMBOLS, 2,
+                "add_module_info: %s functions %d != %d names\n", dll_name, 
+                exports->NumberOfFunctions, exports->NumberOfNames);
+        }
+        /* FIXME: Once we do use noname entry points this if should change to 
+         * check NumberOfFunctions, but for now we only look at names
+         */
+        if (exports->NumberOfNames == 0) {
+            /* riched32.dll from mmc.exe actually has NumberOfFunctions==0 */
+            LOG(GLOBAL, LOG_SYMBOLS, 1,
+                "dll_name=%s has no exported symbols\n", dll_name);
+            return 1;
+        }
+
+        /* FIXME: old comment, should work now.
+         * Dynamically loading modules is still not working very
+         * well, e.g. the itss.dll when loaded in notepad.exe on
+         * F1 seems to point to a zero page for its exports
+         * directory..  We are reading the right exports address
+         * but there is nothing there.  There may be some VA/RVA
+         * issue, and note we try to do this on a MapViewOfSection
+         * before the loader has done anything to the file */
+
+        LOG(GLOBAL, LOG_SYMBOLS, 3, 
+            "dll_name=%s exports="PFX" functions="PFX" ordinals="PFX" fnames="PFX
+            " numnames=%d numfunc=%d %s"
+            "baseord=%d\n", /* the linker adds this to the ords we see, safe to ignore */
+            dll_name,
+            exports, functions, ordinals, fnames, 
+            exports->NumberOfNames, 
+            exports->NumberOfFunctions,
+            (exports->NumberOfFunctions == exports->NumberOfNames) ? "" : "NONAMES ",
+            exports->Base);
+
+        DOLOG(6, LOG_SYMBOLS, { 
+            dump_buffer_as_bytes(GLOBAL, exports, size, 16); 
+        });
+
+
+        exports_table = module_info_create(&process_module_vector, 
+                                           (app_pc)base_addr, (app_pc)(base_addr + image_size), 
+                                           dll_name, exports->NumberOfNames);
+        /* FIXME: for a security policy to restrict transfers to exports only,
+           we actually need all functions and they simply need to be put in a hash table */
+        /* FIXME: for RCT_IND_BRANCH we don't need to travel
+         * through the string names or forwarders - we should only
+         * scan through all functions[] instead of
+         * functions[ordinals[i]]
+         */
+        ASSERT(exports_table != NULL);
+        for (i = 0; i < exports->NumberOfNames; i++) {
+            PSTR name = (PSTR) (base_addr + fnames[i]);
+            ULONG ord = ordinals[i];
+            app_pc func = (app_pc) (base_addr + functions[ord]);  /* note ord here, not i */
+
+            /* check if it points within the exports section in real address space, not RVA */
+            if (func < (app_pc)exports || func >= (app_pc)exports + size) {
+                LOG(GLOBAL, LOG_SYMBOLS, 3, "\t%s -> "PFX"\n", name, func);
+                /* insert in exports table, coming sorted by name order */
+                exports_table[exports_num].export_name = name;
+                exports_table[exports_num].entry_point = func;
+                exports_num++;
+            } else {
+                char *forwardto = (char*)(functions[ord] + base_addr);
+                // skip forwarded function if it forwards to a named import
+                // i.e. NTDLL.RtlAllocateHeap will be reported instead of HeapAlloc
+
+                LOG(GLOBAL, LOG_SYMBOLS, 3,
+                    "Forward found for %s -> "PFX" %s.  Skipping...\n", name, 
+                    functions[ord], forwardto);
+
+                // FIXME: Report the name under which it should show up if it is an ordinal import
+                // if it is referenced as ordinal DLLNAME.#232, then we'll get more from the current name.
+                // The problem though is that now the address range of the forwarded function
+                // is not going to give us the module name... haven't seen any so far
+            }
+        }
+
+        /* FIXME: take this post processing step out of this function */
+        /* the exports_table now needs to be sorted by function address instead of name */
+        qsort(exports_table, exports_num, /* non skipped entries only */
+              sizeof(export_entry_t), export_entry_compare);
+
+        /* need to remove duplicates and update entry in process_module_vector */
+        mutex_lock(&process_module_vector.lock);
+        {
+            module_info_t *pmod;
+            int unique_num = remove_export_duplicates(exports_table, exports_num);
+                
+            pmod = lookup_module_info(&process_module_vector, (app_pc)base_addr); /* FIXME: need a real overlap check */
+            ASSERT(pmod);
+            pmod->exports_num = unique_num;
+        }
+        mutex_unlock(&process_module_vector.lock);
+        return 1;
+    } else {
+        DOLOG(SYMBOLS_LOGLEVEL, LOG_SYMBOLS, { 
+            char short_name[MAX_MODNAME_INTERNAL];
+            os_get_module_name_buf(base_addr, short_name,
+                                   BUFFER_SIZE_ELEMENTS(short_name));
+
+            /* the executable itself is OK */
+            if (base_addr != get_own_peb()->ImageBaseAddress) {
+                if (short_name) 
+                    LOG(GLOBAL, LOG_SYMBOLS, 2, "No exports %s\n", short_name);
+                else
+                    LOG(GLOBAL, LOG_SYMBOLS, 2, "Not a PE at "PFX"\n", base_addr);
+            }
+        });
+        return 0;
+    }
+}
+
+/* The following functions depend on traversing loader data */
+
+/* This routine is here so we know how to walk all 3 loader lists */
+static void
+print_ldr_data()
+{
+    PEB *peb = get_own_peb();
+    PEB_LDR_DATA *ldr = peb->LoaderData;
+    LIST_ENTRY *e, *mark;
+    LDR_MODULE *mod;
+    int i;
+    LOG(GLOBAL, LOG_ALL, 1, "PEB LoaderData:\n");
+    LOG(GLOBAL, LOG_ALL, 1, "\tLength = %d\n", ldr->Length);
+    LOG(GLOBAL, LOG_ALL, 1, "\tInitialized = %d\n", ldr->Initialized);
+    LOG(GLOBAL, LOG_ALL, 1, "\tSsHandle = "PFX"\n", ldr->SsHandle);
+    LOG(GLOBAL, LOG_ALL, 1, "InLoadOrder:\n");
+    mark = &ldr->InLoadOrderModuleList;
+    for (i = 0, e = mark->Flink; e != mark; i++, e = e->Flink) {
+        LOG(GLOBAL, LOG_ALL, 5,
+            "  %d  e="PFX" => "PFX" "PFX" "PFX" "PFX" "PFX" "PFX"\n", i,
+            e, *((ptr_uint_t *)e), *((ptr_uint_t *)e+1), *((ptr_uint_t *)e+2),
+            *((ptr_uint_t *)e+3), *((ptr_uint_t *)e+4), *((ptr_uint_t *)e+5));
+        mod = (LDR_MODULE *) e;
+        LOG(GLOBAL, LOG_ALL, 1, "\t%d  "PFX" "PFX" 0x%x %S %S\n", i,
+            mod->BaseAddress, mod->EntryPoint, mod->SizeOfImage,
+            mod->FullDllName.Buffer, mod->BaseDllName.Buffer);
+        if (i > MAX_MODULE_LIST_INFINITE_LOOP_THRESHOLD) {
+            SYSLOG_INTERNAL_WARNING_ONCE("print_ldr_data: too many modules, maybe "
+                                         "in a race");
+            break;
+        }
+    }
+    LOG(GLOBAL, LOG_ALL, 1, "InMemoryOrder:\n");
+    /* FIXME: why doesn't this turn out to be in memory order? */
+    mark = &ldr->InMemoryOrderModuleList;
+    for (i = 0, e = mark->Flink; e != mark; i++, e = e->Flink) {
+        LOG(GLOBAL, LOG_ALL, 5,
+            "  %d  e="PFX" => "PFX" "PFX" "PFX" "PFX" "PFX" "PFX"\n", i,
+            e, *((ptr_uint_t *)e), *((ptr_uint_t *)e+1), *((ptr_uint_t *)e+2),
+            *((ptr_uint_t *)e+3), *((ptr_uint_t *)e+4), *((ptr_uint_t *)e+5));
+        mod = (LDR_MODULE *) ((char *)e - offsetof(LDR_MODULE, InMemoryOrderModuleList));
+        LOG(GLOBAL, LOG_ALL, 1, "\t%d  "PFX" "PFX" 0x%x %S %S\n", i,
+            mod->BaseAddress, mod->EntryPoint, mod->SizeOfImage,
+            mod->FullDllName.Buffer, mod->BaseDllName.Buffer);
+        if (i > MAX_MODULE_LIST_INFINITE_LOOP_THRESHOLD) {
+            SYSLOG_INTERNAL_WARNING_ONCE("print_ldr_data: too many modules, maybe "
+                                         "in a race");
+            break;
+        }
+    }
+    LOG(GLOBAL, LOG_ALL, 1, "InInitOrder:\n");
+    mark = &ldr->InInitializationOrderModuleList;
+    for (i = 0, e = mark->Flink; e != mark; i++, e = e->Flink) {
+        LOG(GLOBAL, LOG_ALL, 5,
+            "  %d  e="PFX" => "PFX" "PFX" "PFX" "PFX" "PFX" "PFX"\n", i,
+            e, *((ptr_uint_t *)e), *((ptr_uint_t *)e+1), *((ptr_uint_t *)e+2),
+            *((ptr_uint_t *)e+3), *((ptr_uint_t *)e+4), *((ptr_uint_t *)e+5));
+        mod = (LDR_MODULE *)
+            ((char *)e - offsetof(LDR_MODULE, InInitializationOrderModuleList));
+        LOG(GLOBAL, LOG_ALL, 1, "\t%d  "PFX" "PFX" 0x%x %S %S\n", i,
+            mod->BaseAddress, mod->EntryPoint, mod->SizeOfImage,
+            mod->FullDllName.Buffer, mod->BaseDllName.Buffer);
+        if (i > MAX_MODULE_LIST_INFINITE_LOOP_THRESHOLD) {
+            SYSLOG_INTERNAL_WARNING_ONCE("print_ldr_data: too many modules, maybe "
+                                         "in a race");
+            break;
+        }
+    }
+}
+
+#endif /* DEBUG */
+/****************************************************************************/
+/* release build routines */
+
+/* remember our struct in case we want to put it back */
+static LDR_MODULE *DR_module;
+
+/* FIXME : to cleanly detach we need to add ourselves back on to the module
+ * list so we can free library NYI, right now is memory leak but not a big
+ * deal since vmmheap is already leaking a lot more then that */
+
+/* NOTE We are walking the loader lists without holding the lock which is 
+ * potentially dangerous, however we are doing this at init time where we
+ * expect to be single threaded and to be in a clean app state, is more of 
+ * a problem for unhide when it is implemented */
+
+/* NOTE the loader lists appear to be doubly linked circular lists with
+ * each element being a LDR_MODULE (cast to LIST_ENTRYs at various offsets
+ * for actual inclusion on the lists) except the initial list entry in the
+ * PEB_LDR_DATA. NOTE, we assume here (and everywhere else) that the forward 
+ * links are circularly linked for our iteration loops, we ASSERT that the 
+ * backwards pointer is valid before updating below, FIXME should we
+ * be checking the forward pointers here and elsewhere for the loop? */
+
+/* FIXME : also where is the unloaded module list kept? Might be nice to 
+ * remove our pre-inject dll from that. */
+
+static void
+hide_from_module_lists()
+{
+    /* remove us from the module lists! */
+    PEB *peb = get_own_peb();
+    PEB_LDR_DATA *ldr = peb->LoaderData;
+    LIST_ENTRY *e, *mark;
+    LDR_MODULE *mod;
+    int i;
+    app_pc DRbase;
+    MEMORY_BASIC_INFORMATION mbi;
+    size_t len;
+
+    /* FIXME: have os find DR bounds earlier so we don't duplicate work */
+    len = query_virtual_memory((app_pc) hide_from_module_lists, &mbi, sizeof(mbi));
+    ASSERT(len == sizeof(mbi));
+    ASSERT(mbi.State != MEM_FREE);
+    DRbase = (app_pc) mbi.AllocationBase;
+    LOG(GLOBAL, LOG_TOP, 1, "DR dll base = "PFX"\n", DRbase);
+
+    /* FIXME: build iterator so all loopers aren't duplicating all this code */
+    mark = &ldr->InLoadOrderModuleList;
+    ASSERT(mark->Flink != NULL && mark->Blink != NULL); /* sanity check */
+    ASSERT(offsetof(LDR_MODULE, InLoadOrderModuleList) == 0);
+    for (i = 0, e = mark->Flink; e != mark; i++, e = e->Flink) {
+        mod = (LDR_MODULE *) e;
+        /* sanity check */
+        ASSERT(e->Flink != NULL && e->Blink != NULL && 
+               e->Flink->Blink != NULL && e->Blink->Flink != NULL);
+        if ((app_pc) mod->BaseAddress == DRbase) {
+            /* we store the LDR_MODULE struct and do not attempt to de-allocate it,
+             * in case we want to put it back
+             */
+            DR_module = mod;
+            LOG(GLOBAL, LOG_ALL, 1, "Removing "PFX" %S from load order module list\n",
+                mod->BaseAddress, mod->FullDllName.Buffer);
+            /* doubly linked circular list */
+            e->Flink->Blink = e->Blink;
+            e->Blink->Flink = e->Flink;
+            break;
+        }
+        if (i > MAX_MODULE_LIST_INFINITE_LOOP_THRESHOLD) {
+            SYSLOG_INTERNAL_WARNING_ONCE("modules_init: too many modules, maybe "
+                                         "in a race");
+            break;
+        }
+    }
+    mark = &ldr->InMemoryOrderModuleList;
+    ASSERT(mark->Flink != NULL && mark->Blink != NULL); /* sanity check */
+    for (i = 0, e = mark->Flink; e != mark; i++, e = e->Flink) {
+        mod = (LDR_MODULE *) ((char *)e - offsetof(LDR_MODULE, InMemoryOrderModuleList));
+        /* sanity check */
+        ASSERT(e->Flink != NULL && e->Blink != NULL && 
+               e->Flink->Blink != NULL && e->Blink->Flink != NULL);
+        if ((app_pc) mod->BaseAddress == DRbase) {
+            ASSERT(mod == DR_module);
+            LOG(GLOBAL, LOG_ALL, 1, "Removing "PFX" %S from memory order module list\n",
+                mod->BaseAddress, mod->FullDllName.Buffer);
+            /* doubly linked circular list */
+            e->Flink->Blink = e->Blink;
+            e->Blink->Flink = e->Flink;
+            break;
+        }
+        if (i > MAX_MODULE_LIST_INFINITE_LOOP_THRESHOLD) {
+            SYSLOG_INTERNAL_WARNING_ONCE("modules_init: too many modules, maybe "
+                                         "in a race");
+            break;
+        }
+    }
+    mark = &ldr->InInitializationOrderModuleList;
+    ASSERT(mark->Flink != NULL && mark->Blink != NULL); /* sanity check */
+    for (i = 0, e = mark->Flink; e != mark; i++, e = e->Flink) {
+        mod = (LDR_MODULE *) ((char *)e - offsetof(LDR_MODULE, InInitializationOrderModuleList));
+        /* sanity check */
+        ASSERT(e->Flink != NULL && e->Blink != NULL && 
+               e->Flink->Blink != NULL && e->Blink->Flink != NULL);
+        if ((app_pc) mod->BaseAddress == DRbase) {
+            ASSERT(mod == DR_module);
+            LOG(GLOBAL, LOG_ALL, 1, "Removing "PFX" %S from init order module list\n",
+                mod->BaseAddress, mod->FullDllName.Buffer);
+            /* doubly linked circular list */
+            e->Flink->Blink = e->Blink;
+            e->Blink->Flink = e->Flink;
+            break;
+        }
+        if (i > MAX_MODULE_LIST_INFINITE_LOOP_THRESHOLD) {
+            SYSLOG_INTERNAL_WARNING_ONCE("modules_init: too many modules, maybe "
+                                         "in a race");
+            break;
+        }
+    }
+    LOG(GLOBAL, LOG_ALL, 2, "After removing, module lists are:\n");
+    DOLOG(2, LOG_ALL, { print_ldr_data(); });
+}
+
+/* N.B.: walking loader data structures at random times is dangerous!
+ * Do not call this for non-debug reasons if you can help it!
+ * See is_module_being_initialized for a safer approach to walking loader structs.
+ * FIXME: other routines use the same iteration, should we abstract it out?
+ * Other routines include check_for_unsupported_modules, loaded_modules_exports,
+ * get_ldr_module_by_pc, and (in ntdll.c since used by pre-inject)
+ * get_ldr_module_by_name (though the last two use the memory-order list)
+ */
+void
+print_modules(file_t f, bool dump_xml) {
+    print_modules_ldrlist_and_ourlist(f, dump_xml, false /*not conservative*/);
+}
+
+/* conservative flag indicates we may have come here from a crash.  Print
+ * information that do not need any allocations or lock acquisitions */
+void
+print_modules_ldrlist_and_ourlist(file_t f, bool dump_xml, bool conservative)
+{
+    /* We used to walk through every block in memory and call GetModuleFileName
+     * That's not re-entrant, so instead we walk the loader's data structures in the PEB
+     */
+    PEB *peb = get_own_peb();
+    PEB_LDR_DATA *ldr = peb->LoaderData;
+    LIST_ENTRY *e, *mark;
+    LDR_MODULE *mod;
+    uint traversed = 0;
+#ifdef DEBUG
+    RTL_CRITICAL_SECTION *lock = (RTL_CRITICAL_SECTION *) peb->LoaderLock;
+    thread_id_t owner = (thread_id_t) lock->OwningThread;
+    LOG(GLOBAL, LOG_ALL, 2, "LoaderLock owned by %d\n", owner);
+    if (owner != 0 && owner != get_thread_id()) {
+        LOG(GLOBAL, LOG_ALL, 1, "WARNING: print_modules called w/o holding LoaderLock\n");
+        DOLOG_ONCE(2, LOG_ALL, {
+            SYSLOG_INTERNAL_WARNING("print_modules w/o holding LoaderLock");
+        });
+    }
+#endif
+
+    print_file(f, dump_xml ? "<loaded-modules>\n" : "\nLoaded modules:\n");
+
+    /* We use the memory order list instead of the init order list, as
+     * it includes the .exe, and is updated first upon loading a new dll
+     */
+    mark = &ldr->InMemoryOrderModuleList;
+    for (e = mark->Flink; e != mark; e = e->Flink) {
+        uint checksum = 0;
+        char *pe_name = NULL;
+        app_pc preferred_base;
+        version_info_t info;
+        mod = (LDR_MODULE *) ((char *)e - offsetof(LDR_MODULE, InMemoryOrderModuleList));
+        get_module_info_pe(mod->BaseAddress, &checksum, NULL, NULL, &pe_name, NULL);
+        preferred_base = get_module_preferred_base(mod->BaseAddress);
+        print_file(f, dump_xml ? 
+                   "\t<dll range=\""PFX"-"PFX"\" name=\"%ls\" "
+                   "entry=\""PFX"\" count=\"%-3d\"\n"
+                   "\t     flags=\"0x%08x\" "
+                   "timestamp=\"0x%08x\" checksum=\"0x%08x\" pe_name=\"%s\"\n"
+                   "\t     path=\"%ls\" preferred_base=\""PFX"\"\n"
+                   "\t     dll_relocated=\"%s\" "
+                   :
+                   "  "PFX"-"PFX" %-13ls entry="PFX" count=%-3d\n"
+                   "\tflags=0x%08x timestamp=0x%08x checksum=0x%08x\n"
+                   "\tpe_name=%s  %ls\n\tpreferred_base="PFX"\n"
+                   "\tdll_relocated=%s\n",
+                   mod->BaseAddress, 
+                   (char *)mod->BaseAddress + mod->SizeOfImage - 1,
+                   mod->BaseDllName.Buffer, mod->EntryPoint, mod->LoadCount,
+                   mod->Flags, mod->TimeDateStamp, checksum,
+                   pe_name == NULL ? "(null)" : pe_name,
+                   mod->FullDllName.Buffer,
+                   preferred_base,
+                   preferred_base == (app_pc) mod->BaseAddress ? "no" : "yes"
+                   );
+        if (get_module_resource_version_info(mod->BaseAddress, &info)) {
+            print_file(f, dump_xml ?
+                       "file_version=\"%d.%d.%d.%d\" product_version=\"%d.%d.%d.%d\"\n"
+                       "\t     original_filename=\"%S\" company_name=\"%S\"\n"
+                       "\t     product_name=\"%S\" "
+                       :
+                       "\tfile_version=%d.%d.%d.%d product_version=%d.%d.%d.%d"
+                       "\toriginal_filename=%S\n\tcompany_name=%S"
+                       "product_name=%S\n",
+                       info.file_version.version_parts.p1,
+                       info.file_version.version_parts.p2,
+                       info.file_version.version_parts.p3,
+                       info.file_version.version_parts.p4,
+                       info.product_version.version_parts.p1,
+                       info.product_version.version_parts.p2,
+                       info.product_version.version_parts.p3,
+                       info.product_version.version_parts.p4,
+                       info.original_filename == NULL ? L"none" : info.original_filename,
+                       info.company_name == NULL ? L"none" : info.company_name,
+                       info.product_name == NULL ? L"none" : info.product_name);
+        } else {
+            print_file(f, dump_xml ?
+                       "no_version_information=\"true\" " :
+                       "\tmodule_has_no_version_information\n");
+        }
+        if (dump_xml) {
+            print_file(f, "/> \n");
+        }
+        if (traversed++ > MAX_MODULE_LIST_INFINITE_LOOP_THRESHOLD) {
+            SYSLOG_INTERNAL_WARNING_ONCE("print_modules: too many modules");
+            break;
+        }
+    }
+    if (dump_xml)
+        print_file(f, "</loaded-modules>\n");
+    else
+        print_file(f, "\n");
+
+    /* FIXME: currently updated only under aslr_action */
+    if (TEST(ASLR_DLL, DYNAMO_OPTION(aslr)) &&
+        TEST(ASLR_TRACK_AREAS, DYNAMO_OPTION(aslr_action)) &&
+        /* FIXME: xref case 10750: could print w/o lock inside a TRY  */
+        !conservative) {
+        print_file(f, "<print_modules_safe/>\n");
+        if (is_module_list_initialized()) {
+            print_modules_safe(f, dump_xml);
+        }
+    }
+}
+
+void
+print_modules_safe(file_t f, bool dump_xml)
+{
+    module_iterator_t *mi;
+
+    /* we walk our own module list that is populated on an initial walk through memory,
+     * and further kept consistent on memory mappings of likely DLLs */
+    print_file(f, dump_xml ? "<loaded-modules>\n" : "\nLoaded modules:\n");
+
+    mi = module_iterator_start();
+    while (module_iterator_hasnext(mi)) {
+        module_area_t *ma = module_iterator_next(mi);
+        print_file(f, dump_xml ? 
+                   "\t<dll range=\""PFX"-"PFX"\" name=\"%ls\" "
+                   "entry=\""PFX"\" count=\"%-3d\"\n"
+                   "\t     flags=\"0x%08x\" "
+                   "timestamp=\"0x%08x\" checksum=\"0x%08x\" pe_name=\"%s\"\n"
+                   "\t     path=\"%ls\" preferred_base=\""PFX"\" />\n"
+                   :
+                   "  "PFX"-"PFX" %-13ls entry="PFX" count=%-3d\n"
+                   "\tflags=0x%08x timestamp=0x%08x checksum=0x%08x\n"
+                   "\tpe_name=%s  %ls\n\tpreferred_base="PFX"\n",
+                   
+                   ma->start, ma->end - 1, /* inclusive */
+                   L"name",    /* FIXME: dll name is often quite useful */
+                   ma->entry_point, 
+                   0 /* no LoadCount */,
+                   0 /* no Flags */, 
+                   ma->os_data.timestamp, ma->os_data.checksum,
+                   GET_MODULE_NAME(&ma->names) == NULL ?
+                       "(null)" : GET_MODULE_NAME(&ma->names),
+                   L"path",    /* FIXME: path is often quite useful */
+                   ma->os_data.preferred_base);
+    }
+    module_iterator_stop(mi);
+
+    if (dump_xml)
+        print_file(f, "</loaded-modules>\n");
+    else
+        print_file(f, "\n");
+}
+
+
+/* N.B.: see comments on print_modules about why this is a
+ * dangerous routine, especially on a critical path like diagnostics...
+ * FIXME!  Returns true if found an unsupported module, false otherwise
+ */
+bool
+check_for_unsupported_modules()
+{
+
+    PEB *peb = get_own_peb();
+    PEB_LDR_DATA *ldr = peb->LoaderData;
+    LIST_ENTRY *e, *mark;
+    LDR_MODULE *mod;
+    char filter[MAXIMUM_PATH];
+    char dllname[MAXIMUM_PATH];
+    const char *short_name;
+    uint traversed = 0;
+    int retval = get_parameter(L_DYNAMORIO_VAR_UNSUPPORTED, filter, sizeof(filter));
+    if (IS_GET_PARAMETER_FAILURE(retval) ||
+        filter[0] == 0 /* empty UNSUPPORTED list */) {
+        /* no unsupported list, so nothing to look for */
+        return false;
+    }
+
+    LOG(GLOBAL, LOG_ALL, 4, "check_for_unsupported_modules: %s\n", filter);
+    /* FIXME: check peb->LoaderLock? */
+    /* FIXME: share iteration w/ the other routines that do this? */
+    mark = &ldr->InInitializationOrderModuleList;
+    for (e = mark->Flink; e != mark; e = e->Flink) {
+        mod = (LDR_MODULE *) ((char *)e - offsetof(LDR_MODULE, InInitializationOrderModuleList));
+        wchar_to_char(dllname, MAXIMUM_PATH, mod->FullDllName.Buffer,
+                      /* Length is size in bytes not counting final 0 */
+                      mod->FullDllName.Length);
+        short_name = get_short_name(dllname);
+        LOG(GLOBAL, LOG_ALL, 4, "\tchecking %s => %s\n", dllname, short_name);
+        if (check_filter(filter, short_name)) {
+            /* critical since it's unrecoverable and to distinguish from attacks */
+            /* dumpcore if warranted and not already dumped at the security 
+             * violation, options are already synchronized at the security 
+             * violation */
+            SYSLOG(SYSLOG_CRITICAL, UNSUPPORTED_APPLICATION, 3,
+                   get_application_name(), get_application_pid(), dllname);
+            return true;
+        }
+        if (traversed++ > MAX_MODULE_LIST_INFINITE_LOOP_THRESHOLD) {
+            SYSLOG_INTERNAL_WARNING_ONCE("check_for_unsupported_modules: "
+                                         "too many modules");
+            break;
+        }
+    }
+    return false;
+}
+
+/* non-DEBUG routines for parsing PE files */
+
+#define RVA_TO_VA(base, rva)    ((ptr_uint_t)(base) + (ptr_uint_t)(rva))
+
+/* FIXME: make this a static inline function get_nt_header
+ * that verifies and returns nt header */
+#define DOS_HEADER(base)  ((IMAGE_DOS_HEADER *)(base))
+#define NT_HEADER(base)   ((IMAGE_NT_HEADERS *)((ptr_uint_t)(base) + \
+                                                DOS_HEADER(base)->e_lfanew))
+
+#define VERIFY_DOS_HEADER(base)  {                                  \
+        DEBUG_DECLARE(IMAGE_DOS_HEADER *dos = DOS_HEADER(base));    \
+        ASSERT(dos->e_magic == IMAGE_DOS_SIGNATURE);                \
+    }
+
+#define VERIFY_NT_HEADER(base)   {                                 \
+    DEBUG_DECLARE(IMAGE_NT_HEADERS *nt = NT_HEADER(base));     \
+    VERIFY_DOS_HEADER(base);                                   \
+    ASSERT(nt != NULL && nt->Signature == IMAGE_NT_SIGNATURE); \
+    ASSERT_CURIOSITY(nt->OptionalHeader.Magic == IMAGE_NT_OPTIONAL_HDR32_MAGIC || \
+                     nt->OptionalHeader.Magic == IMAGE_NT_OPTIONAL_HDR64_MAGIC);  \
+}
+
+/* returns true iff [start2, start2+size2] covers the same or a subset of the pages
+ * covered by [start1, start1+size1]
+ */
+static inline bool
+on_subset_of_pages(app_pc start1, size_t size1, app_pc start2, size_t size2)
+{
+    return (PAGE_START(start1) <= PAGE_START(start2) &&
+            PAGE_START(start1 + size1) >= PAGE_START(start2 + size2));
+}
+
+bool
+is_readable_pe_base(app_pc base)
+{
+    IMAGE_DOS_HEADER *dos = (IMAGE_DOS_HEADER *) base;
+    IMAGE_NT_HEADERS *nt;
+    size_t size;
+    /* would be nice to batch the is_readables into one, but we need
+     * to dereference in turn...
+     */
+    if (!is_readable_without_exception((app_pc)dos, sizeof(*dos)) ||
+        dos->e_magic != IMAGE_DOS_SIGNATURE)
+        return false;
+    nt = (IMAGE_NT_HEADERS *) (((ptr_uint_t)dos) + dos->e_lfanew);
+    if (nt == NULL ||
+        /* optimization: reduce number of system calls for safe reads */
+        (!on_subset_of_pages((app_pc)dos, sizeof(*dos), (app_pc)nt, sizeof(*nt)) &&
+         !is_readable_without_exception((app_pc)nt, sizeof(*nt))) ||
+        nt->Signature != IMAGE_NT_SIGNATURE)
+        return false;
+    /* make sure section headers are readable */
+    size = nt->FileHeader.NumberOfSections * sizeof(IMAGE_SECTION_HEADER);
+    if (/* optimization: reduce number of system calls for safe reads */
+        !on_subset_of_pages((app_pc)dos, sizeof(*dos),
+                            (app_pc) IMAGE_FIRST_SECTION(nt), size) &&
+        !is_readable_without_exception((app_pc) IMAGE_FIRST_SECTION(nt), size))
+        return false;
+    return true;
+}
+
+/* Returns the size of the image section when loaded not counting alignment bytes
+ * added by the image loader. */
+static inline size_t
+get_image_section_unpadded_size(IMAGE_SECTION_HEADER *sec
+                                _IF_DEBUG(IMAGE_NT_HEADERS *nt))
+{
+    ASSERT(sec != NULL && nt != NULL);
+    /* Curiosity if VirtualSize/SizeOfRawData relationship doesn't match one of the
+     * cases we've seen.  Note that this will fire for the (experimentally legal, but
+     * never seen in practice) case of raw data much larger than virtual size (as in
+     * past the next file alignment), though the various size routines here will handle
+     * that correctly (see 5355, 9053). */
+    ASSERT_CURIOSITY(sec->Misc.VirtualSize > sec->SizeOfRawData /* case 5355 */ ||
+                     sec->Misc.VirtualSize == 0 /* case 10501 */ ||
+                     ALIGN_FORWARD(sec->Misc.VirtualSize,
+                                   nt->OptionalHeader.FileAlignment) ==
+                     ALIGN_FORWARD(sec->SizeOfRawData, /* case 8868 not always aligned */
+                                   nt->OptionalHeader.FileAlignment));
+    ASSERT_CURIOSITY(sec->Misc.VirtualSize != 0 || sec->SizeOfRawData != 0);
+    if (sec->Misc.VirtualSize == 0) /* case 10501 */
+        return sec->SizeOfRawData;
+    return sec->Misc.VirtualSize;  /* case 5355 */
+}
+
+/* Returns the size in bytes of the image section when loaded, including image loader
+ * allocated alignment/padding bytes.  To include non-allocated (MEM_RESERVE) padding
+ * bytes align this value forward to nt->OptionalHeader.SectionAlignment (only comes
+ * into play when SectionAlignment is > PAGE_SIZE). */
+static inline size_t
+get_image_section_size(IMAGE_SECTION_HEADER *sec, IMAGE_NT_HEADERS *nt)
+{
+    /* Xref case 9797, drivers (which we've seen mapped in on Vista) don't
+     * usually use page size section alignment (use 0x80 alignment instead). */
+    size_t unpadded_size = get_image_section_unpadded_size(sec _IF_DEBUG(nt));
+    uint alignment = MIN(PAGE_SIZE, nt->OptionalHeader.SectionAlignment);
+    return ALIGN_FORWARD(unpadded_size, alignment);
+}
+
+/* Returns the size of the portion of the image file that's mapped into the image section
+ * when it's loaded. */
+static inline size_t
+get_image_section_map_size(IMAGE_SECTION_HEADER *sec, IMAGE_NT_HEADERS *nt)
+{
+    /* Xref case 5355 - this is mapped in irregardless of sec->Characteristics flags
+     * (including the UNINITIALIZED_DATA flag) so can ignore them. */
+    size_t virtual_size = get_image_section_size(sec, nt);
+    /* FileAlignment - the alignment factor (in bytes) that is used to align the raw data
+     * of sections in the image file. The value should be a power of 2 between 512
+     * (though note the lower bound is not enforced, xref 9798) and 64 K, inclusive. The
+     * default is 512. If the SectionAlignment is less than the architecture's page size,
+     * then FileAlignment must match SectionAlignment. */
+    size_t raw_data_size =
+        ALIGN_FORWARD(sec->SizeOfRawData, nt->OptionalHeader.FileAlignment);
+    /* Xref 5355, the size of the mapping is the lesser of the virtual size (as
+     * determined above) and the FileAlignment aligned size of SizeOfRawData. Any
+     * extra space up to virtual size is 0 filled. */
+    return MIN(virtual_size, raw_data_size);
+}
+
+/* returns the offset into the PE file at which the mapping for section sec starts */
+static inline size_t
+get_image_section_file_offs(IMAGE_SECTION_HEADER *sec, IMAGE_NT_HEADERS *nt)
+{
+    ASSERT(sec != NULL && nt != NULL);
+    /* Xref 5355, despite PE specifications it appears that PointerToRawData is not
+     * required to be aligned (the image loader apparently back aligns it before use). */
+    return ALIGN_BACKWARD(sec->PointerToRawData, nt->OptionalHeader.FileAlignment);
+}
+
+void
+print_module_section_info(file_t file, app_pc addr)
+{
+    IMAGE_DOS_HEADER *dos;
+    IMAGE_NT_HEADERS *nt;
+    IMAGE_SECTION_HEADER *sec;
+    uint i;
+    app_pc module_base = get_module_base(addr);
+    
+    if (module_base == NULL)
+        return;
+    
+    dos = (IMAGE_DOS_HEADER *) module_base;
+    nt = (IMAGE_NT_HEADERS *) (((ptr_uint_t)dos) + dos->e_lfanew);
+    sec = IMAGE_FIRST_SECTION(nt);
+    /* FIXME : can we share this loop with is_in_executable_file_section? */
+    for (i = 0; i < nt->FileHeader.NumberOfSections; i++, sec++) {
+        app_pc sec_start = module_base + sec->VirtualAddress;
+        app_pc sec_end =  module_base + sec->VirtualAddress +
+            get_image_section_size(sec, nt);
+
+        /* xref case 6799, section is [start, end) */
+        if (sec_start <= addr && addr < sec_end) {
+            print_file(file,
+                       "\t\tmod_base=            \""PFX"\"\n"
+                       "\t\tsec_name=            \"%.*s\"\n"
+                       "\t\tsec_start=           \""PFX"\"\n"
+                       "\t\tsec_end=             \""PFX"\"\n"
+                       "\t\tVirtualSize=         \"0x%08x\"\n"
+                       "\t\tSizeOfRawData=       \"0x%08x\"\n"
+                       "\t\tsec_characteristics= \"0x%08x\"\n",
+                       module_base, IMAGE_SIZEOF_SHORT_NAME, sec->Name,
+                       sec_start, sec_end, sec->Misc.VirtualSize, sec->SizeOfRawData,
+                       sec->Characteristics);   
+        }
+    }
+}
+
+/* Looks for a section or (if merge) group of sections that satisfies the following
+ * criteria:
+ * - if start_pc != NULL, that contains [start_pc, end_pc);
+ * - if sec_characteristics_match != 0, that matches ANY of sec_characteristics_match;
+ * - if name != NULL, that matches name.
+ *
+ * If a section or (if merge) group of sections are found that satisfy the above,
+ * then returns the bounds of the section(s) in sec_start_out and sec_end_out (both are
+ * optional) and returns true.  If no matching section(s) are found returns false.
+ * If !merge the actual characteristics are returned in sec_characteristics_out,
+ * which is optional and must be NULL if merge.
+ *
+ * FIXME - with case 10526 fix letting the exemption polices trim to section boundaries
+ * is there any reason we still need merging support?
+ */
+static bool
+is_in_executable_file_section(app_pc module_base, app_pc start_pc, app_pc end_pc,
+                              app_pc *sec_start_out /* OPTIONAL OUT */,
+                              app_pc *sec_end_out /* OPTIONAL OUT */, 
+                              uint *sec_characteristics_out /* OPTIONAL OUT */,
+                              IMAGE_SECTION_HEADER *sec_header_out /* OPTIONAL OUT */,
+                              uint sec_characteristics_match /* TESTANY, 0 to ignore */,
+                              const char *name /* OPTIONAL */, bool merge)
+{
+    IMAGE_DOS_HEADER *dos;
+    IMAGE_NT_HEADERS *nt;
+    IMAGE_SECTION_HEADER *sec;
+    uint i;
+    bool prev_sec_same_chars = false, result = false, stop_at_next_non_matching = false;
+    app_pc sec_start = 0, sec_end = 0;
+
+    /* See case 7998 where a NULL base was passed. */
+    ASSERT_CURIOSITY(module_base != NULL);
+    if (module_base == NULL)
+        return false;
+
+    dos = (IMAGE_DOS_HEADER *) module_base;
+    nt = (IMAGE_NT_HEADERS *) (((ptr_uint_t)dos) + dos->e_lfanew);
+    if (dos->e_magic != IMAGE_DOS_SIGNATURE ||
+        nt == NULL || nt->Signature != IMAGE_NT_SIGNATURE)
+        return false;
+    /* must specify some criteria */
+    ASSERT(start_pc != NULL || sec_characteristics_match != 0 || name != NULL);
+    ASSERT(start_pc == NULL || start_pc < end_pc);
+    /* sec_characteristics_out & section_header_out only makes sense if !merge */
+    ASSERT(sec_characteristics_out == NULL || !merge);
+    ASSERT(sec_header_out == NULL || !merge);
+    /* We cannot use the OptionalHeader fields BaseOfCode or SizeOfCode or SizeOfData
+     * since for multiple sections the SizeOfCode is the sum of the
+     * non-page-align-expanded sizes, and sections need not be contiguous!
+     * Instead we walk all sections for ones that match our criteria. */
+    LOG(GLOBAL, LOG_VMAREAS, 4, "module @ "PFX":\n", module_base);
+    sec = IMAGE_FIRST_SECTION(nt);
+    for (i = 0; i < nt->FileHeader.NumberOfSections; i++, sec++) {
+        LOG(GLOBAL, LOG_VMAREAS, 4, "\tName = %.*s\n", IMAGE_SIZEOF_SHORT_NAME,
+            sec->Name);
+        LOG(GLOBAL, LOG_VMAREAS, 4, "\tVirtualSize    = "PFX"\n", sec->Misc.VirtualSize);
+        LOG(GLOBAL, LOG_VMAREAS, 4, "\tVirtualAddress = "PFX"\n", sec->VirtualAddress);
+        LOG(GLOBAL, LOG_VMAREAS, 4, "\tSizeOfRawData  = 0x%08x\n", sec->SizeOfRawData);
+        LOG(GLOBAL, LOG_VMAREAS, 4, "\tCharacteristics= 0x%08x\n", sec->Characteristics);
+
+        if ((sec_characteristics_match == 0 ||
+             TESTANY(sec_characteristics_match, sec->Characteristics)) &&
+            (name == NULL ||
+             (sec->Name != NULL && strncmp((const char *)sec->Name,
+                                           name, strlen(name)) == 0))) {
+            app_pc new_start = module_base + sec->VirtualAddress;
+            if (prev_sec_same_chars && sec_end == new_start) {
+                /* os will merge adjacent regions w/ same privileges, so consider
+                 * these one region by leaving sec_start at its old value if merge */
+                ASSERT(merge);
+                LOG(GLOBAL, LOG_VMAREAS, 2,
+                    "is_in_executable_file_section: adjacent sections @"PFX" and "PFX"\n",
+                    sec_start, new_start);
+            } else {
+                if (stop_at_next_non_matching)
+                    break;
+                sec_start = new_start;
+            }
+            if (merge)
+                prev_sec_same_chars = true;
+            sec_end = module_base + sec->VirtualAddress +
+                get_image_section_size(sec, nt);
+            LOG(GLOBAL, LOG_VMAREAS, 2,
+                "is_in_executable_file_section (module "PFX", region "PFX"-"PFX"): "
+                "%.*s == "PFX"-"PFX"\n",
+                module_base, start_pc, end_pc, IMAGE_SIZEOF_SHORT_NAME,
+                sec->Name, sec_start, sec_end);
+            if (start_pc == NULL ||
+                (start_pc >= sec_start && start_pc <= sec_end)) {
+                if (sec_start_out != NULL)
+                    *sec_start_out = sec_start; /* merged section start */
+                if (sec_end_out != NULL)
+                    *sec_end_out = sec_end; /* merged section end */
+                if (sec_characteristics_out != NULL)
+                    *sec_characteristics_out = sec->Characteristics;
+                if (sec_header_out != NULL)
+                    *sec_header_out = *sec;
+                if (start_pc == NULL || end_pc <= sec_end) {
+                    /* we found what we were looking for, stop looping as soon as we
+                     * finish merging into the current region. */
+                    result = true;
+                    if (merge)
+                        stop_at_next_non_matching = true;
+                    else 
+                        break;
+                }
+            }
+        } else
+            prev_sec_same_chars = false;
+    }
+    return result;
+}
+
+bool
+module_pc_section_lookup(app_pc module_base, app_pc pc, IMAGE_SECTION_HEADER *section_out)
+{
+    ASSERT(is_readable_pe_base(module_base));
+    if (section_out != NULL)
+        memset(section_out, 0, sizeof(*section_out));
+    return is_in_executable_file_section(module_base, pc, pc+1,
+                                         NULL, NULL, NULL, section_out,
+                                         0 /* any section */, NULL, false);
+}
+
+/* Returns true if [start_pc, end_pc) is within a single code section.
+ * Returns the bounds of the enclosing section in sec_start and sec_end.
+ * Note that unlike is_in_*_section routines, does not merge sections. */
+bool
+is_range_in_code_section(app_pc module_base, app_pc start_pc, app_pc end_pc,
+                         app_pc *sec_start /* OPTIONAL OUT */,
+                         app_pc *sec_end /* OPTIONAL OUT */)
+{
+    return is_in_executable_file_section(module_base, start_pc, end_pc,
+                                         sec_start, sec_end, NULL, NULL,
+                                         IMAGE_SCN_CNT_CODE, NULL,
+                                         false /* don't merge */);
+}
+
+/* Returns true if addr is in a code section and if so returns in sec_start and sec_end
+ * the bounds of the section containing addr (merged with adjacent code sections). */
+bool
+is_in_code_section(app_pc module_base, app_pc addr,
+                   app_pc *sec_start /* OPTIONAL OUT */,
+                   app_pc *sec_end /* OPTIONAL OUT */)
+{
+    return is_in_executable_file_section(module_base, addr, addr+1,
+                                         sec_start, sec_end, NULL, NULL,
+                                         IMAGE_SCN_CNT_CODE, NULL,
+                                         true /* merge */);
+}
+
+/* Same as above only for initalized data sections instead of code. */
+bool
+is_in_dot_data_section(app_pc module_base, app_pc addr,
+                       app_pc *sec_start /* OPTIONAL OUT */,
+                       app_pc *sec_end /* OPTIONAL OUT */)
+{
+    return is_in_executable_file_section(module_base, addr, addr+1,
+                                         sec_start, sec_end, NULL, NULL,
+                                         IMAGE_SCN_CNT_INITIALIZED_DATA |
+                                         IMAGE_SCN_CNT_UNINITIALIZED_DATA,
+                                         NULL, true /* merge */);
+}
+
+/* Same as above only for xdata sections (see below) instead of code. */
+bool
+is_in_xdata_section(app_pc module_base, app_pc addr,
+                    app_pc *sec_start /* OPTIONAL OUT */,
+                    app_pc *sec_end /* OPTIONAL OUT */)
+{
+    /* .xdata is present in .NET2.0 .ni.dll files
+     * it is marked as +rwx initialized data
+     */
+    uint sec_flags = 0;
+    if (is_in_executable_file_section(module_base, addr, addr+1, 
+                                      sec_start, sec_end, &sec_flags, NULL,
+                                      IMAGE_SCN_CNT_INITIALIZED_DATA, ".xdata",
+                                      false /* don't merge */)) {
+        bool xdata_prot_match = TESTALL(IMAGE_SCN_MEM_READ | IMAGE_SCN_MEM_WRITE |
+                                        IMAGE_SCN_MEM_EXECUTE, sec_flags);
+        ASSERT_CURIOSITY(xdata_prot_match && "unexpected xdata section characteristics");
+        return xdata_prot_match;
+    }
+    return false;
+}
+
+/* This is a more restrictive test than (get_module_base() != NULL) because it
+ * checks for the start of the PE and examines at least one section in it before
+ * concluding that addr belongs to that module. */
+bool
+is_in_any_section(app_pc module_base, app_pc addr,
+                  app_pc *sec_start /* OPTIONAL OUT */,
+                  app_pc *sec_end /* OPTIONAL OUT */)
+{
+    return is_in_executable_file_section(module_base, addr, addr+1,
+                                         sec_start, sec_end, NULL, NULL,
+                                         0 /* any section */, NULL,
+                                         true /* merge */);
+}
+
+/* allow only true MEM_IMAGE mappings */
+bool
+is_mapped_as_image(app_pc module_base)
+{
+    MEMORY_BASIC_INFORMATION mbi;
+
+    if (query_virtual_memory(module_base, &mbi, sizeof(mbi)) == sizeof(mbi)
+        && mbi.State == MEM_COMMIT /* header should always be committed */
+        && mbi.Type == MEM_IMAGE) {
+        return true;
+    }
+
+    /* although mbi.Type may be undefined, most callers should get
+     * this far only if not MEM_FREE, so ok to ASSERT.  Note all
+     * Type's are MEM_FREE, MEM_PRIVATE, MEM_MAPPED, and MEM_IMAGE */
+    ASSERT_CURIOSITY(mbi.Type == MEM_PRIVATE ||
+                     mbi.Type == MEM_MAPPED);
+
+    return false;
+}
+
+/* returns true if a matching section is found, false otherwise */
+bool
+get_named_section_bounds(app_pc module_base, const char *name,
+                         app_pc *start/*OPTIONAL OUT*/, app_pc *end/*OPTIONAL OUT*/)
+{
+    if (!is_in_executable_file_section(module_base, NULL, NULL, start, end, NULL, NULL,
+                                       0 /* any section */, name, true /* merge */)) {
+        if (start != NULL)
+            *start = NULL;
+        if (end != NULL)
+            *end = NULL;
+        return false;
+    }
+    return true;
+}
+
+bool
+get_IAT_section_bounds(app_pc module_base, app_pc *iat_start, app_pc *iat_end)
+{
+    IMAGE_DOS_HEADER *dos = (IMAGE_DOS_HEADER *) module_base;
+    IMAGE_NT_HEADERS *nt = (IMAGE_NT_HEADERS *) (((ptr_uint_t)dos) + dos->e_lfanew);
+    IMAGE_DATA_DIRECTORY *dir;
+    if (dos->e_magic != IMAGE_DOS_SIGNATURE ||
+        nt == NULL || nt->Signature != IMAGE_NT_SIGNATURE)
+        return false;
+    dir = &OPT_HDR(nt, DataDirectory)[IMAGE_DIRECTORY_ENTRY_IAT];
+    *iat_start = module_base + dir->VirtualAddress;
+    *iat_end = module_base + dir->VirtualAddress + dir->Size;
+    return true;
+}
+
+bool
+is_IAT(app_pc start, app_pc end, bool page_align, app_pc *iat_start, app_pc *iat_end)
+{
+    app_pc IAT_start, IAT_end;
+    app_pc base = get_module_base(start);
+    if (base == NULL)
+        return false;
+    if (!get_IAT_section_bounds(base, &IAT_start, &IAT_end))
+        return false;
+    if (iat_start != NULL)
+        *iat_start = IAT_start;
+    if (iat_end != NULL)
+        *iat_end = IAT_end;
+    if (page_align) {
+        IAT_start = (app_pc) ALIGN_BACKWARD(IAT_start, PAGE_SIZE);
+        IAT_end = (app_pc) ALIGN_FORWARD(IAT_end, PAGE_SIZE);
+    }
+    LOG(THREAD_GET, LOG_VMAREAS, 3,
+        "is_IAT("PFX","PFX") vs ("PFX","PFX") == %d\n", start, end,
+        IAT_start, IAT_end, IAT_start == start && IAT_end == end);
+    return (IAT_start == start && IAT_end == end);
+}
+
+bool
+is_in_IAT(app_pc addr)
+{
+    app_pc IAT_start, IAT_end;
+    app_pc base = get_module_base(addr);
+    if (base == NULL)
+        return false;
+    if (!get_IAT_section_bounds(base, &IAT_start, &IAT_end))
+        return false;
+    return (IAT_start <= addr && addr < IAT_end);
+}
+
+app_pc
+get_module_entry(app_pc module_base)
+{
+    /* N.B.: do not use imagehlp routines like ImageNtHeader here, since the
+     * dependency on that dll causes sqlsrvr to crash.
+     * It's not that hard to directly read the headers.
+     */
+    IMAGE_DOS_HEADER *dos = (IMAGE_DOS_HEADER *) module_base;
+    IMAGE_NT_HEADERS *nt = (IMAGE_NT_HEADERS *) (((ptr_uint_t)dos) + dos->e_lfanew);
+    ASSERT(is_readable_pe_base(module_base));
+    ASSERT(dos->e_magic == IMAGE_DOS_SIGNATURE);
+    ASSERT(nt != NULL && nt->Signature == IMAGE_NT_SIGNATURE);
+    /* note that the entry point for .NET executables is clobbered by
+     * mscoree.dll to point directly at either mscoree!_CorDllMain or
+     * mscoree!_CorExeMain (though the LDR_MODULE struct entry is still the
+     * original one), so don't assume that it's inside the PE module itself
+     * (see case 3714)
+     */
+    return ((app_pc)dos) + nt->OptionalHeader.AddressOfEntryPoint;
+}
+
+app_pc
+get_module_base(app_pc pc)
+{
+    /* We get the base from the allocation region.  We cannot simply back-align
+     * to 64K, the Windows allocation granularity on all platforms, since some
+     * modules have code sections beyond 64K from the start of the module.
+     */
+    app_pc base = get_allocation_base(pc);
+    if (!is_readable_pe_base(base)) {
+        /* not readable, or not PE */
+        return NULL;
+    }
+    return base;
+}
+
+/* gets the preferred base of the module containing pc from PE header */
+app_pc
+get_module_preferred_base(app_pc pc)
+{
+    IMAGE_DOS_HEADER *dos;
+    IMAGE_NT_HEADERS *nt;
+    app_pc module_base = get_allocation_base(pc);
+    
+    if (!is_readable_pe_base(module_base))
+        return NULL;
+    dos = (IMAGE_DOS_HEADER *) module_base;
+    nt = (IMAGE_NT_HEADERS *) (((ptr_uint_t)dos) + dos->e_lfanew);
+    /* we return NULL on error above, make sure no one actually sets their
+     * preferred base address to NULL */
+    ASSERT_CURIOSITY(OPT_HDR(nt, ImageBase) != 0);
+    return (app_pc) OPT_HDR(nt, ImageBase);
+}
+
+/* we simply test if allocation bases of a region are the same */
+bool
+in_same_module(app_pc target, app_pc source)
+{
+    app_pc target_base = get_allocation_base(target);
+    app_pc source_base = get_allocation_base(source);
+    LOG(THREAD_GET, LOG_VMAREAS, 2,
+        "in_same_module("PFX","PFX") => ("PFX","PFX") == %d\n", target, source,
+        target_base, source_base, (target_base == source_base));
+    /* all unallocated memory regions will get a base of 0 */
+    return (target_base != NULL) && (target_base == source_base);
+}
+
+/* Use get_module_short_name for arbitrary pcs -- only call this if
+ * you KNOW this is the base addr of a non-executable module, as it
+ * bypasses some safety checks in get_module_short_name to avoid 4
+ * system calls.
+ * Returns the short module name from the PE exports section, or NULL if invalid 
+ */
+char*
+get_dll_short_name(app_pc base_addr)
+{
+    /* FIXME: We'll have a name pointer in a DLL that may get unloaded
+       by another thread, so it would be nice to synchronize this call
+       with UnmapViewOfSection so that we can get a safe copy of the name.
+
+       FIXME: How can make sure we can't fail on strncpy(buf, name, max_chars);
+       we can't test is_readable_without_exception(dll_name, max_chars) 
+       because our max_chars may be too long and of course we can't use strlen(),
+       a TRY block would work.
+
+       For now we avoid copying altogether and callers are expected to
+       synchronize with DLL unloads, or otherwise to be ready to take the risk.
+
+       Nearly all callers should be looking up in the loaded_module_areas vector
+       and using the copy there, which is copied under TRY/EXCEPT, so the
+       racy window while update_module_list() copies from here is now safe.
+    */
+    IMAGE_EXPORT_DIRECTORY *exports;
+    ASSERT(base_addr == get_allocation_base(base_addr) &&
+           is_readable_pe_base(base_addr));
+    exports = get_module_exports_directory(base_addr, NULL);
+    if (exports != NULL) {
+        char *dll_name = (char*) (base_addr + exports->Name /* RVA */);
+        /* sanity check whether really MEM_IMAGE, but too late */
+        if (!is_string_readable_without_exception(dll_name, NULL)) {
+            ASSERT_CURIOSITY(false && "Exports name not readable, partial map?" ||
+                             EXEMPT_TEST("partial_map.exe"));
+            dll_name = NULL;
+        }
+        LOG(THREAD_GET, LOG_SYMBOLS, 3, 
+            "get_dll_short_name(base_addr="PFX") exports=%d dll_name=%s\n", 
+            base_addr, exports, dll_name == NULL ? "<invalid>" : dll_name);
+        return dll_name;
+    }
+
+    return NULL;
+}
+
+/* Get all possible names for the module corresponding to pc.  Part of fix for 
+ * case 9842.  We have to maintain all different module names, can't just use
+ * a precedence rule for deciding at all points.
+ */
+static void
+get_all_module_short_names_uncached(dcontext_t *dcontext, app_pc pc, bool at_map,
+                                    module_names_t *names
+                                    HEAPACCT(which_heap_t which))
+{
+    const char *name;
+    app_pc base;
+
+    ASSERT(names != NULL);
+    if (names == NULL)
+        return;
+    memset(names, 0, sizeof(*names));
+
+    base = get_allocation_base(pc);
+    LOG(THREAD_GET, LOG_VMAREAS, 5,
+        "get_all_module_short_names_uncached: start "PFX" -> base "PFX"\n", pc, base);
+    if (!is_readable_pe_base(base)) {
+        LOG(THREAD_GET, LOG_VMAREAS, 5,
+            "get_all_module_short_names_uncached: not a module\n");
+        return;
+    }
+#ifndef X64
+    if (module_is_64bit(base)) {
+        /* For 32-bit dr we ignore 64-bit dlls in a wow64 process. */
+        ASSERT_CURIOSITY(is_wow64_process(NT_CURRENT_PROCESS));
+        LOG(THREAD_GET, LOG_VMAREAS, 5,
+            "get_all_module_short_names_uncached: ignoring 64-bit module in "
+            "wow64 process\n");
+        return;
+    }
+#endif
+    /* FIXME: we do have a race here where the module can be unloaded
+     * before we finish making a copy of its name
+     */
+    if (dynamo_exited)
+        return; /* no heap for strdup */
+
+    /* Ensure we don't crash if a dll is unloaded racily underneath us */
+    TRY_EXCEPT_ALLOW_NO_DCONTEXT(dcontext, { 
+        app_pc process_image;
+
+        /* Choice #1: PE exports name */
+        name = get_dll_short_name(base);
+        if (name != NULL)
+            names->module_name = dr_strdup(name HEAPACCT(which));
+        else
+            names->module_name = NULL;
+
+        /* Choice #2: executable qualified name
+         * This would be the last choice except historically it's been #2 so
+         * we'll stick with that.
+         * check if target is in process image - 
+         * in which case we use our unqualified name for the executable
+         */
+        process_image = get_own_peb()->ImageBaseAddress;
+
+        /* check if pc region base matches the image base  */
+        /* FIXME: they should be aligned anyways, can remove this */
+        ASSERT(ALIGNED(process_image, PAGE_SIZE) && ALIGNED(base, PAGE_SIZE));
+        if (process_image == base) {
+            name = get_short_name(get_application_name());
+            if (name != NULL)
+                names->exe_name = dr_strdup(name HEAPACCT(which));
+            else
+                names->exe_name = NULL;
+        }
+
+        /* Choice #3: .rsrc original filename, already strduped */
+        names->rsrc_name = (char *) get_module_original_filename(base HEAPACCT(which));
+
+        /* Choice #4: file name */
+        if (at_map && DYNAMO_OPTION(track_module_filenames) && dcontext != NULL) {
+            name = dcontext->aslr_context.last_section_file_name;
+            if (name != NULL)
+                names->file_name = dr_strdup(name HEAPACCT(which));
+            else 
+                names->file_name = NULL;
+        }
+
+        DOLOG(3, LOG_VMAREAS, {
+            LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 1,
+                "get_all_module_short_names_uncached "PFX":\n", base);
+            LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 1, "\tPE name=%s\n",
+                names->module_name == NULL ? "<unavailable>" : names->module_name);
+            LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 1, "\texe name=%s\n",
+                names->exe_name == NULL ? "<unavailable>" : names->exe_name);
+            LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 1, "\t.rsrc original filename=%s\n",
+                names->rsrc_name == NULL ? "<unavailable>" : names->rsrc_name);
+            if (at_map && DYNAMO_OPTION(track_module_filenames) && dcontext != NULL)
+                LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 1, "\tfilename=%s\n",
+                    names->file_name == NULL ? "<unavailable>" : names->file_name);
+        });
+    }, {
+        /* Free all allocations in the event of an exception and return NULL 
+         * for all names. */
+        free_module_names(names HEAPACCT(which));
+        memset(names, 0, sizeof(*names));
+    });
+
+    /* theoretically possible to fail, since section matching can be thwarted,
+     * or if we came in late
+     */
+    ASSERT_CURIOSITY(names->module_name != NULL || names->exe_name != NULL ||
+                     names->rsrc_name != NULL || names->file_name != NULL ||
+                     !at_map ||
+                     /* PR 229284: a partial map can cause this */
+                     check_filter("partial_map.exe",
+                                  get_short_name(get_application_name())));
+}
+
+/* Caller should use get_module_short_name() unless calling before or after
+ * we set up the loaded_module_areas vector.
+ *
+ * If map is true, this routine finds our official internal name for a
+ * module, which is done in this priority order:
+ * 1) PE exports name
+ * 2) If pc is in the main executable image we use our fully qualified name
+ * 3) .rsrc original file name
+ * 4) if at_map, file name; else unavailable
+ *    FIXME: is PEB->SubSystemData = PathFileName.Buffer (see
+ *    notes in aslr_generate_relocated_section()) available w/o a debugger,
+ *    and should we check whether it equals our stored name?
+ *
+ * 1 and 2 need not be present, and 3 can be invalid if the app creates multiple
+ * sections before mapping any, so we can have a NULL name for a module.
+ * Also returns NULL if pc is not in a valid module.
+ * This name precedence is established by GET_MODULE_NAME().
+ *
+ * The name string is dr_strduped with HEAPACCT(which) and must be freed
+ * by the caller calling dr_strfree.
+ */
+const char *
+get_module_short_name_uncached(dcontext_t *dcontext, app_pc pc, bool at_map
+                               HEAPACCT(which_heap_t which))
+{
+    module_names_t names = {0};
+    const char *res;
+
+    get_all_module_short_names_uncached(dcontext, pc, at_map, &names
+                                        HEAPACCT(which));
+    res = dr_strdup(GET_MODULE_NAME(&names) HEAPACCT(which));
+    free_module_names(&names HEAPACCT(which));
+    return res;
+}
+
+/* All internal uses of module names should be calling this routine,
+ * which not only looks up the cached name but uses the priority-order
+ * naming scheme that avoids modules without names, rather than
+ * explicitly get_dll_short_name() (PE name only) or the other
+ * individual name gathering routines.
+ * For safety this routine makes a copy of the name.
+ * For more-performant uses, use os_get_module_name(), which allows the caller
+ * to hold a lock and use the original string.
+ */
+const char *
+get_module_short_name(app_pc pc HEAPACCT(which_heap_t which))
+{
+    /* Our module list name is the short name */
+    return os_get_module_name_strdup(pc HEAPACCT(which));
+}
+
+/* If the PC resides in a module that has been relocated from 
+ * its preferred base, get_module_preferred_base_delta() returns 
+ * the delta of the preferred base and its actual base (used to 
+ * normalize the Threat ID).  If the PC does not reside in a 
+ * module or it is invalid, the function returns 0 (since there 
+ * is no need to normalize the Threat ID in those cases).
+ */
+ssize_t
+get_module_preferred_base_delta(app_pc pc)
+{
+    app_pc preferred_base_addr = get_module_preferred_base(pc);
+    app_pc current_base_addr = get_allocation_base(pc);
+    /* FIXME : optimization add out argument to get_module_preferred_base to
+     * return the allocation base */
+
+    if (preferred_base_addr == NULL || current_base_addr == NULL)
+        return 0;
+
+    return (preferred_base_addr - current_base_addr);
+}
+
+
+/* returns 0 if no loader module is found 
+ * N.B.: walking loader data structures at random times is dangerous!
+ */
+LDR_MODULE *
+get_ldr_module_by_pc(app_pc pc)
+{
+    PEB *peb = get_own_peb();
+    PEB_LDR_DATA *ldr = peb->LoaderData;
+    LIST_ENTRY *e, *mark;
+    LDR_MODULE *mod;
+    uint traversed = 0;     /* a simple infinite loop break out */
+
+#ifdef DEBUG
+    RTL_CRITICAL_SECTION *lock = (RTL_CRITICAL_SECTION *) peb->LoaderLock;
+    thread_id_t owner = (thread_id_t) lock->OwningThread;
+    if (owner != 0 && owner != get_thread_id()) {
+        /* This will be a risky operation but we'll live with it.
+           In case we walk in a list in an inconsistent state
+           1) we may get trapped in an infinite loop when following a partially updated list
+              so we'll bail out in case of a deep loop 
+           2) list entries and pointed data may be removed and even deallocated
+              we can't just check for is_readable_without_exception 
+              since it won't help if we're in a race
+           FIXME: we should mark we started this routine,
+              and if we get an exception retry or give up gracefully
+        */
+        LOG(GLOBAL, LOG_ALL, 3, "WARNING: get_ldr_module_by_pc w/o holding LoaderLock\n");
+        DOLOG_ONCE(2, LOG_ALL, {
+            SYSLOG_INTERNAL_WARNING("get_ldr_module_by_pc w/o holding LoaderLock");
+        });
+    }
+#endif
+
+    /* Now, you'd think these would actually be in memory order, but they
+     * don't seem to be for me!
+     */
+    mark = &ldr->InMemoryOrderModuleList;
+
+    for (e = mark->Flink; e != mark; e = e->Flink) {
+        app_pc start, end;
+        mod = (LDR_MODULE *) ((char *)e - offsetof(LDR_MODULE, InMemoryOrderModuleList));
+        start = (app_pc) mod->BaseAddress;
+        end = start + mod->SizeOfImage;
+        if (pc >= start && pc < end) {
+            return mod;
+        }
+
+        if (traversed++ > MAX_MODULE_LIST_INFINITE_LOOP_THRESHOLD) {
+            LOG(GLOBAL, LOG_ALL, 1, 
+                "WARNING: get_ldr_module_by_pc too many modules, or an infinte loop due to a race\n");
+            SYSLOG_INTERNAL_WARNING_ONCE("get_ldr_module_by_pc too many modules");
+            /* TODO: In case we ever hit this we may want to retry the traversal once more */
+            return NULL;
+        }
+    }
+    return NULL;
+}
+
+/* N.B.: walking loader data structures at random times is dangerous!
+ * Do not call this for non-debug reasons if you can help it!
+ * See get_module_status for a safer approach to walking loader structs.
+ */
+void
+get_module_name(app_pc pc, char *buf, int max_chars)
+{
+    LDR_MODULE *mod = get_ldr_module_by_pc(pc);
+    if (mod != NULL) {
+        wchar_to_char(buf, max_chars, mod->FullDllName.Buffer, mod->FullDllName.Length);
+        return;
+    }
+    buf[0] = '\0';
+}
+
+static
+IMAGE_BASE_RELOCATION *
+get_module_base_reloc(app_pc module_base, size_t *base_reloc_size /* OPTIONAL OUT */)
+{
+    IMAGE_NT_HEADERS *nt;
+    IMAGE_DATA_DIRECTORY *base_reloc_dir = NULL;
+    app_rva_t base_reloc_vaddr = 0;
+    size_t size = 0;
+    IMAGE_BASE_RELOCATION *base_reloc = NULL;
+
+    VERIFY_NT_HEADER(module_base);
+    /* callers should have done this in release builds */
+    ASSERT(is_readable_pe_base(module_base));
+
+    nt = NT_HEADER(module_base);
+    base_reloc_dir = OPT_HDR(nt, DataDirectory) +
+                                        IMAGE_DIRECTORY_ENTRY_BASERELOC;
+
+    if (base_reloc_size != NULL)
+        *base_reloc_size = 0;
+
+    /* Dont expect base_reloc_dir to be NULL, but to be safe */
+    if (base_reloc_dir == NULL) {
+        ASSERT_CURIOSITY(false &&
+                         "DataDirectory[IMAGE_DIRECTORY_ENTRY_BASERELOC] NULL");
+        /* base_reloc_size set to 0 */
+        return NULL;
+    }
+
+    /* sanity check */
+    ASSERT(is_readable_without_exception((app_pc)base_reloc_dir, 8));
+
+    base_reloc_vaddr = base_reloc_dir->VirtualAddress;
+    size = base_reloc_dir->Size;
+
+    /* /FIXED dlls have the vaddr as 0, but size may be garbage */
+    if (base_reloc_vaddr == 0) {
+        /* $ dumpbin /headers xpsp2res.dll 
+         *             210E characteristics
+         *      0 [       0] RVA [size] of Base Relocation Directory
+         * has only one section .rsrc
+         */
+        return NULL; /* base_reloc_size set to 0 */
+    }
+
+    if (base_reloc_vaddr != 0 && size == 0) {
+        ASSERT_CURIOSITY(false && "expect non-zero base_reloc");
+        return NULL; /* base_reloc_size set to 0 */
+    }
+
+    LOG(GLOBAL, LOG_RCT, 2,
+        "reloc: get_module_base_reloc: module_base="PFX", "
+        "base_reloc_dir="PFX", base_reloc_vaddr="PFX", size="PFX")\n",
+        module_base, base_reloc_dir, base_reloc_vaddr, size);
+
+    base_reloc = (IMAGE_BASE_RELOCATION *) RVA_TO_VA(module_base, base_reloc_vaddr);
+
+    if (is_readable_without_exception((app_pc)base_reloc, size)) {
+        if (base_reloc_size != NULL)
+            *base_reloc_size = size;
+        return base_reloc;
+    } else {
+        ASSERT_CURIOSITY(false && "bad base relocation" ||
+                         EXEMPT_TEST("partial_map.exe")); /* expected for partial map */
+    }
+
+    return NULL;
+}
+
+/* returns FileHeader.Characteristics */
+/* should be used only under after is_readable_pe_base */
+uint
+get_module_characteristics(app_pc module_base)
+{
+    IMAGE_NT_HEADERS *nt = NULL;
+    IMAGE_DATA_DIRECTORY *com_desc_dir = NULL;
+
+    VERIFY_NT_HEADER(module_base);
+    /* callers should have done this in release builds */
+    ASSERT(is_readable_pe_base(module_base));
+
+    nt = NT_HEADER(module_base);
+    /* note this is not the same as OptionalHeader.DllCharacteristics */
+    return nt->FileHeader.Characteristics;
+}
+
+/* Parse PE and return IMAGE_COR20_HEADER * if it has a valid COM header.
+ * Optional OUT: cor20_header_size.
+ * NOTE: returning a pointer into a dll that could be unloaded could be racy
+ * (though we dont expect a race, see case 1272 for _safe/_unsafe routines)
+ */
+IMAGE_COR20_HEADER *
+get_module_cor20_header(app_pc module_base, size_t *cor20_header_size)
+{
+    IMAGE_NT_HEADERS *nt = NULL;
+    IMAGE_DATA_DIRECTORY *com_desc_dir = NULL;
+
+    VERIFY_NT_HEADER(module_base);
+    /* callers should have done this in release builds */
+    ASSERT(is_readable_pe_base(module_base));
+
+    nt = NT_HEADER(module_base);
+
+    /* IMAGE_DIRECTORY_ENTRY_COM_DESCRIPTOR < IMAGE_NUMBEROF_DIRECTORY_ENTRIES */
+    com_desc_dir = OPT_HDR(nt, DataDirectory) +
+                        IMAGE_DIRECTORY_ENTRY_COM_DESCRIPTOR;
+    /* sanity check */
+    ASSERT(is_readable_without_exception((app_pc)com_desc_dir, 8));
+
+    LOG(GLOBAL, LOG_RCT, 3,
+        "get_module_cor20_header: module_base="PFX", com_desc_dir="PFX")\n",
+        module_base, com_desc_dir);
+
+    if (cor20_header_size != NULL)
+        *cor20_header_size = 0;
+
+    if (com_desc_dir != NULL) {
+        app_rva_t com_desc_vaddr = com_desc_dir->VirtualAddress;
+        size_t size = com_desc_dir->Size; /* only a dword but we plan for future */
+
+        LOG(GLOBAL, LOG_RCT, 3,
+            "get_module_cor20_header: module_base="PFX", "
+            "com_desc_dir="PFX", com_desc_vaddr="PFX", size="PFX")\n",
+            module_base, com_desc_dir, com_desc_vaddr, size);
+        
+        if (com_desc_vaddr != 0 && size == 0 ||
+            com_desc_vaddr == 0 && size > 0) {
+            ASSERT_CURIOSITY(false && "bad cor20 header");
+            /* cor20_header_size set to 0 */
+            return NULL;
+        }
+
+        if (size > 0) {
+            IMAGE_COR20_HEADER *cor20_header = (IMAGE_COR20_HEADER *)
+                RVA_TO_VA(module_base, com_desc_dir->VirtualAddress);
+
+            if (is_readable_without_exception((app_pc)cor20_header,
+                                              sizeof(IMAGE_COR20_HEADER))) {
+                if (cor20_header_size != NULL) {
+                    *cor20_header_size = size;
+                }
+                return cor20_header;
+            } else 
+                ASSERT_CURIOSITY(false && "bad cor20 header");
+        }
+    }
+    else
+        ASSERT_CURIOSITY(false && "no cor20_header directory entry");
+
+    return NULL;
+}
+
+/* PE files, for exes and dlls, with managed code have IMAGE_COR20_HEADER
+ * defined in their PE.  Return if PE has cor20 header or not.
+ */
+bool
+module_has_cor20_header(app_pc module_base)
+{
+    size_t cor20_header_size = 0;
+    IMAGE_COR20_HEADER *cor20_header = 
+        get_module_cor20_header(module_base, &cor20_header_size);
+
+    return (cor20_header != NULL && cor20_header_size > 0);
+}
+
+static WORD
+get_module_magic(app_pc module_base)
+{
+    IMAGE_NT_HEADERS *nt = NULL;
+    if (!is_readable_pe_base(module_base))
+        return false;
+    VERIFY_NT_HEADER(module_base);
+    nt = NT_HEADER(module_base);
+    return nt->OptionalHeader.Magic;
+}
+
+bool
+module_is_32bit(app_pc module_base)
+{
+    return (get_module_magic(module_base) == IMAGE_NT_OPTIONAL_HDR32_MAGIC);
+}
+
+bool
+module_is_64bit(app_pc module_base)
+{
+    return (get_module_magic(module_base) == IMAGE_NT_OPTIONAL_HDR64_MAGIC);
+}
+
+/* WARNING: this routine relies on observed behavior and data structures that
+ * may change in future versions of Windows
+ *
+ * Returns true if start:end matches a code/IAT section of a module that the loader
+ * would legitimately update, AND the module is currently being initialized
+ * by this thread (or a guess as to that effect for 2003).
+ * If conservative is true, makes fewer guesses and uses stricter guidelines,
+ * so may have false negatives but should have no false positives after
+ * the image entry point.
+ * Caller must distinguish IAT in .rdata from IAT in .text.
+ */
+bool
+is_module_patch_region(dcontext_t *dcontext, app_pc start, app_pc end,
+                       bool conservative)
+{
+    PEB *peb = get_own_peb();
+    LDR_MODULE *mod;
+    RTL_CRITICAL_SECTION *lock = (RTL_CRITICAL_SECTION *) peb->LoaderLock;
+    app_pc IAT_start, IAT_end;
+    bool match_IAT = false;
+    app_pc base = get_module_base(start);
+    LOG(THREAD, LOG_VMAREAS, 2,
+        "is_module_patch_region: start "PFX" -> base "PFX"\n", start, base);
+    if (base == NULL) {
+        LOG(THREAD, LOG_VMAREAS, 2,
+            "is_module_patch_region: not readable or not PE => NO\n");
+        return false;
+    }
+    /* The only module changes we recognize are rebasing, where the entire
+     * code section should be written to, and rebinding, where only
+     * the IAT should be written to.  We ignore relocation of other data.
+     * We allow for page rounding at end.
+     */
+    if (is_IAT(start, (app_pc) ALIGN_FORWARD(end, PAGE_SIZE),
+               true /*page align*/, &IAT_start, &IAT_end)) {
+        LOG(THREAD, LOG_VMAREAS, 2,
+            "is_module_patch_region: matches IAT "PFX"-"PFX"\n", IAT_start, IAT_end);
+        match_IAT = true;
+    } else {
+        /* ASSUMPTION: if multiple code sections, they are always protected separately.
+         * We walk the code sections and see if our region is inside one of them.
+         */
+        app_pc sec_start = NULL, sec_end = NULL;
+        if (!is_range_in_code_section(base, start, end, &sec_start, &sec_end)) {
+            LOG(THREAD, LOG_VMAREAS, 2,
+                "is_module_patch_region: not IAT or inside code section => NO\n");
+            return false;
+        }
+        LOG(THREAD, LOG_VMAREAS, 2,
+            "is_module_patch_region: target "PFX"-"PFX" => section "PFX"-"PFX"\n",
+            start, end, sec_start, sec_end);
+        /* FIXME - check what alignment the loader uses when section alignment is
+         * < than page size (check on all platforms) to tighten this up. According to
+         * Derek the IAT requests are very percise, but the loader may do exact
+         * (rebase restore) or page-aligned (rebind restore) on xpsp2 at least. */
+        if (ALIGN_BACKWARD(start, PAGE_SIZE) != ALIGN_BACKWARD(sec_start, PAGE_SIZE) ||
+            ALIGN_FORWARD(end, PAGE_SIZE) != ALIGN_FORWARD(sec_end, PAGE_SIZE)) {
+            LOG(THREAD, LOG_VMAREAS, 2,
+                "is_module_patch_region: not targeting whole code or IAT section => NO\n");
+            return false;
+        }
+    }
+
+    /* On 2K and XP, the LoaderLock is always held when loading a module,
+     * but on 2003 it is not held for loads prior to the image entry point!
+     * Even worse, we've seen apps that create a 2nd thread prior to the entry
+     * point, meaning we cannot safely walk the list.
+     */
+    if ((thread_id_t) lock->OwningThread == get_thread_id()) {
+        /* Walk the list
+         * FIXME: just look at the last entry, since it's appended to the memory-order
+         * list?
+         */
+        mod = get_ldr_module_by_pc(start);
+        if (mod != NULL) {
+            /* How do we know if module is initialized?  LoadCount is 0 for a while, but
+             * on win2003 it becomes 1 and the loader is still mucking around.  But when it
+             * does become 1, the flags have 0x1000 set.  So we have this hack.
+             * ASSUMPTION: module is uninitialized if either LoadCount is 0 or
+             * flags have 0x1000 set.
+             * Note that LoadCount is -1 for statically linked dlls and the exe itself.
+             * We also see cases where a module's IAT is patched, and later is re-patched
+             * once the module's count and flags indicate it's initialized.
+             * We go ahead and allow that, since it's only data and not much of a security
+             * risk.
+             * FIXME: figure out what the loader is doing there -- I saw it on sqlservr
+             * on 2003 loading msdtcprx.dll and then a series of dependent dlls was loaded
+             * and patched and re-patched, perhaps due to forwarding?
+             */
+            LOG(THREAD, LOG_VMAREAS, 2,
+                "is_module_patch_region: count=%d, flags=0x%x, %s\n",
+                mod->LoadCount, mod->Flags, match_IAT ? "IAT" : "not IAT");
+            if (mod->LoadCount == 0 || TEST(LDR_LOAD_IN_PROGRESS, mod->Flags) ||
+                /* case 10180: executable itself has unknown flag 0x00004000 set; we
+                 * relax to consider it the loader if the lock is held and we are
+                 * before the image entry, but only when we track the image entry */
+                (!reached_image_entry_yet() && !RUNNING_WITHOUT_CODE_CACHE()) || 
+                (!conservative && match_IAT))
+                return true;
+            else
+                return false;
+        }
+    } else {
+        if (get_os_version() >= WINDOWS_VERSION_2003 && !reached_image_entry_yet()) {
+#ifdef HOT_PATCHING_INTERFACE
+            /* This one of the uses of reached_image_entry that may conflict 
+             * with -hotp_only not setting it because interp is not done. Others
+             * are safe.  If this is hit, then the image_entry hook will have 
+             * to placed in callback_interception_init for -hotp_only.  A TODO.
+             */
+            if(DYNAMO_OPTION(hotp_only)) {
+                LOG(GLOBAL, LOG_HOT_PATCHING, 1, "Warning: On w2k3, for "
+                    "hotp_only, image entry won't be detected because no "
+                    "interp is done and hook is placed late");
+            }
+#endif
+            /* On 2003, we cannot safely walk the module list (grabbing
+             * the LoaderLock is fraught with deadlock problems...)
+             * We haven't put in the effort in analyzing the
+             * LdrLockLoaderLock routine that decides not to grab it
+             * to find the flag the loader is using to decide when to
+             * start using the lock, but it coincides with the image
+             * entry point (or loader finishing initialization), so we
+             * use that. 
+             * FIXME: this isn't as narrow as we'd like --
+             * we're letting anyone modify a .text section prior to
+             * image entry on 2003!
+             */
+            return true;
+        }
+    }
+    return false;
+}
+
+#define IMAGE_REL_BASED_TYPE(x)      ((x) >> 12)
+#define IMAGE_REL_BASED_OFFSET_MASK  0x0FFF
+#define IMAGE_REL_BASED_OFFSET(x)    ((ushort)((x) & IMAGE_REL_BASED_OFFSET_MASK))
+
+/* Processes a single relocation and returns the relocated address.  If
+ * apply_reloc is false, the actual relocation isn't performed on the
+ * image only the relocated address is returned.
+ *
+ * Note: this routine handles 32-bit dlls for both 32-bit dr and for 64-bit dr
+ * (wow64 process), and 64-bit dlls for 64-bit dr.
+ *
+ * X86 relocation types can be: IMAGE_REL_BASED_HIGHLOW |
+ * IMAGE_REL_BASED_ABSOLUTE --  offsets pointing to 32-bit
+ * immediate (see case 6424).  For X64 it is IMAGE_REL_BASED_DIR64 
+ * or IMAGE_REL_BASED_ABSOLUTE.
+ * 
+ * Returns relocated_addr for HIGHLOW & DIR64.
+ */
+static app_pc
+process_one_relocation(const app_pc module_base, app_pc reloc_entry_p, 
+                       uint reloc_array_rva, ssize_t relocation_delta, 
+                       bool apply_reloc, bool *null_ref /* OUT */,
+                       bool *unsup_reloc /* OUT */,
+                       app_pc *relocatee_addr /* OUT */, bool is_module_32bit
+                       _IF_DEBUG(size_t module_size))
+{
+    ushort reloc_entry = *(ushort *)reloc_entry_p;
+    int reloc_type = IMAGE_REL_BASED_TYPE(reloc_entry);
+    ushort offset  = IMAGE_REL_BASED_OFFSET(reloc_entry);
+    app_pc cur_addr, addr_to_reloc, relocated_addr = NULL;
+    DEBUG_DECLARE(char *rel_name = "unsupported";)
+
+    cur_addr = (app_pc) RVA_TO_VA(module_base, (reloc_array_rva + offset));
+    /* relocatee_addr is used to return the address of the value that is to be
+     * relocated. */
+    if (relocatee_addr != NULL)
+        *relocatee_addr = cur_addr;
+    /* curiosity: sometimes cur_addr is not within module */
+    ASSERT_CURIOSITY(module_base <= cur_addr && 
+                     cur_addr < (module_base + module_size));
+
+#ifdef X64
+    if (reloc_type == IMAGE_REL_BASED_DIR64) {
+        /* this relocation is only on 64-bits */
+        addr_to_reloc = (app_pc)(*(uint64 *)cur_addr);
+        if (addr_to_reloc == NULL) {
+            if (null_ref != NULL)
+                *null_ref = true;
+            ASSERT_CURIOSITY(false && "relocation entry for a null ref?");
+        }
+        relocated_addr = relocation_delta + addr_to_reloc;
+        if (apply_reloc) {
+            *(uint64 *)cur_addr = (uint64) relocated_addr;
+        }
+        DEBUG_DECLARE(rel_name = "DIR64");
+    } else
+#endif
+    if (reloc_type == IMAGE_REL_BASED_HIGHLOW) {
+        /* This is a 32-bit relocation type and can found only in a 32-bit dll.
+         * If it is found in a 64-bit process then the process must be wow64.
+         */
+        IF_X64(ASSERT(is_wow64_process(NT_CURRENT_PROCESS));)
+        IF_X64(ASSERT(is_module_32bit);)
+        if (!is_module_32bit) {
+            if (unsup_reloc != NULL)
+                *unsup_reloc = true;
+            return NULL;
+        }
+
+        /* Relocation delta can't be greater than 32-bits!  It is a bug if so
+         * because 32-bit dlls only have 32-bit quantities to be relocated,
+         * i.e., only within a 2 GB space - so can't add a relocation delta
+         * that is greater than 31 bits (which is possible to generate while
+         * rebasing in a 64-bit address space).
+         */
+        ASSERT(CHECK_TRUNCATE_TYPE_int(relocation_delta));
+
+        /* IMAGE_REL_BASED_HIGHLOW relocations are defined to operate
+         * on 32-bits, irrespective of 32-bit or 64-bit DR, so read
+         * exactly 32-bits.
+         */
+        ASSERT(sizeof(uint) == 4);    /* this relocation is only on 32-bits */
+        addr_to_reloc = (app_pc)(ptr_uint_t)(*(uint*)cur_addr);
+        if (addr_to_reloc == NULL) {
+            if (null_ref != NULL)
+                *null_ref = true;
+            ASSERT_CURIOSITY(false && "relocation entry for a null ref?");
+        }
+        relocated_addr = relocation_delta + addr_to_reloc;
+
+        /* Just like relocation delta, relocated addr can't be greater than
+         * 32-bits too.
+         */
+        IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint((ptr_uint_t)relocated_addr));)
+        if (apply_reloc) {
+            *(uint *)cur_addr = (uint)(ptr_uint_t) relocated_addr;
+        }
+        DEBUG_DECLARE(rel_name = "HIGHLOW");
+    } else if (reloc_type == IMAGE_REL_BASED_ABSOLUTE) {
+        /* This is just a padding type so ignore. */
+        DEBUG_DECLARE(rel_name = "ABS");
+    } else {
+        /* unsupported types */
+        /* LOW:  *(ushort*) cur_addr += LOWORD(relocation_delta)
+         * HIGH: *(ushort*) cur_addr += HIWORD(relocation_delta)
+         */
+        /* FIXME: case 8515: it is better to implement these 
+         * in case some stupid compiler is generating them for random reasons
+         */
+        ASSERT_CURIOSITY("Unsupported relocation encountered");
+        if (unsup_reloc != NULL)
+            *unsup_reloc = true;
+    }
+    LOG(GLOBAL, LOG_RCT, 6, "\t%8x %s\n", offset, rel_name);
+    return relocated_addr;
+}
+
+/****************************************************************************/
+#ifdef RCT_IND_BRANCH
+
+# ifdef X64
+static void
+add_SEH_address(dcontext_t *dcontext, app_pc addr, app_pc modbase, size_t modsize)
+{
+    /* If except_handler isn't within the image, don't add to RCT table */
+    if (addr > modbase && addr < modbase + modsize) {
+        if (rct_add_valid_ind_branch_target(dcontext, addr)) {
+            STATS_INC(rct_ind_branch_valid_targets);
+            STATS_INC(rct_ind_seh64_new);
+        } else
+            STATS_INC(rct_ind_seh64_old);
+    } else {
+        ASSERT_CURIOSITY(false && "SEH address out of module");
+    }
+}
+
+/* This routine analyzes the .pdata section in a PE32+ module and adds
+ * exception handler addresses to rct table.  Note, this isn't applicable for
+ * 32-bit dlls as SEH aren't specified like this in those dlls.  PR 250395.
+ */
+static void
+add_SEH_to_rct_table(dcontext_t *dcontext, app_pc module_base)
+{
+    IMAGE_NT_HEADERS *nt = NT_HEADER(module_base);
+    IMAGE_DATA_DIRECTORY *except_dir;
+    PIMAGE_RUNTIME_FUNCTION_ENTRY func_entry, func_entry_end;
+    byte *pdata_start;
+    uint image_size;
+    ASSERT_OWN_MUTEX(true, &rct_module_lock);
+
+    ASSERT(module_base != NULL);
+    if (module_base == NULL)
+        return;
+
+    /* Ignore 32-bit dlls in a wow64 process. */
+    if (!module_is_64bit(module_base))
+        return;
+
+    nt = NT_HEADER(module_base);
+
+    /* Exception directories can be NULL if the compiler didn't put it in, but
+     * unusual for PE32+ dlls to not have an exception directory.
+     */
+    except_dir = OPT_HDR(nt, DataDirectory) + IMAGE_DIRECTORY_ENTRY_EXCEPTION;
+    if (except_dir == NULL) {
+        ASSERT_CURIOSITY(false && "no exception data directory (no .pdata?)");
+        return;
+    }
+    image_size = nt->OptionalHeader.SizeOfImage;
+    /* Exception directory entry must lie within the image */
+    ASSERT((app_pc)except_dir > module_base &&
+           (app_pc)except_dir < module_base + image_size);
+
+    /* Exception directory, i.e., function table entries, must lie within the
+     * image.
+     */
+    func_entry = (PIMAGE_RUNTIME_FUNCTION_ENTRY)(module_base +
+                                                 except_dir->VirtualAddress);
+    pdata_start = (byte *) func_entry;
+    func_entry_end = (PIMAGE_RUNTIME_FUNCTION_ENTRY)(module_base +
+                                                     except_dir->VirtualAddress +
+                                                     except_dir->Size);
+    /* spec says func_entry must be dword-aligned */
+    ASSERT_CURIOSITY(ALIGNED(func_entry, sizeof(DWORD)));
+    ASSERT((app_pc)func_entry_end >= module_base &&
+           (app_pc)func_entry < module_base + image_size);
+
+    LOG(GLOBAL, LOG_RCT, 2, "parsing .pdata of pe32+ module "PFX"\n", module_base);
+    for (; func_entry < func_entry_end; func_entry++) {
+        unwind_info_t *info = (unwind_info_t*)
+            (module_base + func_entry->UnwindInfoAddress);
+        /* Spec says unwind info must be dword-aligned, but we also have
+         * special entries that point at other RUNTIME_FUNCTION slots in
+         * the .pdata array, but with a 1-byte offset.  It seems to be a
+         * way to share unwind info for non-contiguous pieces of the same
+         * function without wasting space on a chained info structure
+         * (see PR 250395).
+         */
+        if (info > (unwind_info_t *) pdata_start &&
+            info < (unwind_info_t *) func_entry_end) {
+            /* All the ones I've seen have been 1 byte in */
+            ASSERT_CURIOSITY(ALIGNED(((byte*)info)-1, sizeof(DWORD)));
+            /* We just skip this entry, as it's subsumed by the one it points at */
+            STATS_INC(rct_ind_seh64_plus1);
+            continue;
+        }
+        ASSERT_CURIOSITY(ALIGNED(info, sizeof(DWORD)));
+
+        /* If it is a chain entry, then walk the chain to get the exception
+         * handler.
+         */
+        while (TEST(UNW_FLAG_CHAININFO, info->Flags)) {
+            /* Note that while one page of the specs, and the
+             * GetChainedFunctionEntry() macro, say that there is a pointer to a
+             * RUNTIME_FUNCTION, another page, and all the instances I've seen, have
+             * the RUNTIME_FUNCTION inlined.  We handle both.
+             */
+            byte *ptr;
+            uint rva;
+            PIMAGE_RUNTIME_FUNCTION_ENTRY chain_func;
+            /* if chained, can't have handler flags set */
+            ASSERT_CURIOSITY(!TESTANY(UNW_FLAG_EHANDLER|UNW_FLAG_UHANDLER, info->Flags));
+            ptr = module_base + UNWIND_INFO_PTR_RVA(info);
+            if (ptr > pdata_start && ptr < (byte *) func_entry_end)
+                chain_func = (PIMAGE_RUNTIME_FUNCTION_ENTRY) ptr;
+            else /* inlined */
+                chain_func = (PIMAGE_RUNTIME_FUNCTION_ENTRY) UNWIND_INFO_PTR_ADDR(info);
+            if (!safe_read(&chain_func->UnwindInfoAddress, sizeof(rva), &rva)) {
+                ASSERT_CURIOSITY(false && "unwind_info_t corrupted/misinterpreted");
+                continue;
+            }
+            info = (unwind_info_t *)(module_base + rva/*chain_func->UnwindInfoAddress*/);
+            /* spec says unwind info must be dword-aligned */
+            ASSERT_CURIOSITY(ALIGNED(info, sizeof(DWORD)));
+        }
+
+        /* If unwind info is either UNW_FLAG_EHANDLER or UNW_FLAG_UHANDLER,
+         * then it has an exception handler address.
+         */
+        if (TESTANY(UNW_FLAG_EHANDLER|UNW_FLAG_UHANDLER, info->Flags)) {
+            app_pc handler = module_base + UNWIND_INFO_PTR_RVA(info);
+            /* PR 276527: also process the scope table addresses */
+            scope_table_t *scope;
+            uint i;
+            bool is_scope = true;
+            add_SEH_address(dcontext, handler, module_base, image_size);
+            LOG(GLOBAL, LOG_RCT, 4, "added RCT SEH64 handler "PFX
+                " (from "PFX")\n", handler, info);
+            /* Note that I'm seeing user32 and kernel32 each having a single
+             * master handler, which calls the ntdll master handler, which uses
+             * the scope table to dive down into details.  Given the single
+             * master, why not provide an efficient way to have a single master
+             * handler, which would allow handling misaligned stack pointers?
+             */
+            /* Like the chained info, the scope table is described as being
+             * out-of-line, but I'm seeing it inlined.
+             */
+            scope =  (scope_table_t *) UNWIND_INFO_DATA_ADDR(info);
+            /* Not all entries have this: e.g., calc.exe's _CxxFrameHandler doesn't
+             * use this setup; only the _C_specific_handler routines do.
+             * We use a heuristic where we assume there won't be over 4K entries;
+             * most other fields in these tables are > 0x1000.  That breaks down
+             * when we hit other lang-specific structs, though, so we have
+             * further checks below.
+             */
+            if (scope->Count == 0 || scope->Count >= 0x1000)
+                is_scope = false;
+            else {
+                /* Do one pass through to make sure it all looks right.  FIXME: we
+                 * need a stronger way to tell when there's a scope table and when
+                 * not.  It would be nice to check the scope entry's range, but it
+                 * seems to not need to be a subset of func_entry's range: many
+                 * func_entries point at the same unwind_info, which then re-expands
+                 * via the scope table.  We could check that [Begin, End) is in a
+                 * code section.
+                 */
+                for (i = 0; i < scope->Count; i++) {
+                    if (scope->ScopeRecord[i].EndAddress <=
+                        scope->ScopeRecord[i].BeginAddress ||
+                        /* Yes, you can have tiny dlls, but we'll adjust when we hit
+                         * that: I'm seeing other lang-specific structs and I need
+                         * heuristics to distinguish from the scope table we know */
+                        scope->ScopeRecord[i].BeginAddress < PAGE_SIZE ||
+                        scope->ScopeRecord[i].EndAddress > image_size ||
+                        (scope->ScopeRecord[i].HandlerAddress >
+                         EXCEPTION_EXECUTE_HANDLER &&
+                         scope->ScopeRecord[i].HandlerAddress < PAGE_SIZE) ||
+                        scope->ScopeRecord[i].HandlerAddress > image_size ||
+                        scope->ScopeRecord[i].JumpTarget > image_size) {
+                        LOG(GLOBAL, LOG_RCT, 4, "NOT a scope table entry %d info "PFX
+                            "\n", i, info);
+                        is_scope = false;
+                        break;
+                    }
+                }
+            }
+            if (is_scope) {
+                for (i = 0; i < scope->Count; i++) {
+                    /* Add the filter address */
+                    if (scope->ScopeRecord[i].HandlerAddress !=
+                        EXCEPTION_EXECUTE_HANDLER &&
+                        /* Often they're all the same */ 
+                        (i == 0 || scope->ScopeRecord[i].HandlerAddress !=
+                         scope->ScopeRecord[i-1].HandlerAddress)) {
+                        add_SEH_address(dcontext, module_base +
+                                        scope->ScopeRecord[i].HandlerAddress,
+                                        module_base, image_size);
+                        LOG(GLOBAL, LOG_RCT, 4, "added RCT SEH64 filter %d "PFX"\n",
+                            i, module_base +
+                            scope->ScopeRecord[i].HandlerAddress);
+                    }
+                    if (scope->ScopeRecord[i].JumpTarget != 0 &&
+                        /* Often they're all the same */ 
+                        (i == 0 || scope->ScopeRecord[i].JumpTarget !=
+                         scope->ScopeRecord[i-1].JumpTarget)) {
+                        /* Add the catch block entry address */
+                        add_SEH_address(dcontext,
+                                        module_base + scope->ScopeRecord[i].JumpTarget,
+                                        module_base, image_size);
+                        LOG(GLOBAL, LOG_RCT, 4, "added RCT SEH64 catch %d "PFX
+                            "\n", i, module_base + scope->ScopeRecord[i].JumpTarget);
+                    }
+                }
+            } else {
+                LOG(GLOBAL, LOG_RCT, 4,
+                    "assuming scope "PFX" w/ count %d is not a scope table\n",
+                    scope, scope->Count);
+            }
+        }
+    }
+}
+
+bool
+rct_add_rip_rel_addr(dcontext_t *dcontext, app_pc tgt _IF_DEBUG(app_pc src))
+{
+    /* PR 215408: Whether we scan or use relocations, for x64 we also need to add
+     * rip-rel references (unless we update the scan to consider those as well).  We
+     * need to look up the section and if it matches the characteristics checked by
+     * add_rct_module() add it like rct_check_ref_and_add() does.  Faster to do a
+     * scan than all these section lookups?  Keep a vmvector of appropriate sections?
+     * It should be faster to check the rct table before doing the section walk,
+     * so we do that.
+     */
+    app_pc modbase = get_module_base(tgt);
+    uint secchar;
+    bool res = false;
+    if (modbase != NULL &&
+        rct_ind_branch_target_lookup(dcontext, tgt) == NULL &&
+        is_in_executable_file_section(modbase, tgt, tgt+1, NULL, NULL,
+                                      &secchar, NULL, 0, NULL, false)) {
+        ASSERT(DYNAMO_OPTION(rct_section_type) != 0 &&
+               !TESTANY(~(IMAGE_SCN_CNT_CODE|IMAGE_SCN_CNT_INITIALIZED_DATA|
+                          IMAGE_SCN_CNT_UNINITIALIZED_DATA),
+                        DYNAMO_OPTION(rct_section_type)));
+        if (TESTANY(DYNAMO_OPTION(rct_section_type), secchar) &&
+            (DYNAMO_OPTION(rct_section_type_exclude) == 0 || 
+             !TESTALL(DYNAMO_OPTION(rct_section_type_exclude), secchar))) {
+            DOLOG(3, LOG_RCT, { 
+                char symbuf[MAXIMUM_SYMBOL_LENGTH];
+                LOG(GLOBAL, LOG_RCT, 3, 
+                    "rct_add_rip_rel_addr: "PFX" rip-rel addr referenced at "PFX"\n",
+                    tgt, src);
+                print_symbolic_address(tgt, symbuf, sizeof(symbuf), true);
+                LOG(GLOBAL, LOG_SYMBOLS, 3, "\t%s\n", symbuf);
+            });
+            mutex_lock(&rct_module_lock);
+            if (rct_add_valid_ind_branch_target(dcontext, tgt)) {
+                STATS_INC(rct_ind_branch_valid_targets);
+                STATS_INC(rct_ind_rip_rel_new);
+                res = true;
+            } else {
+                STATS_INC(rct_ind_rip_rel_old);
+                ASSERT_CURIOSITY(false && "TOCTOU race");
+            }
+            mutex_unlock(&rct_module_lock);
+        }
+    } else {
+        DOSTATS({
+            if (rct_ind_branch_target_lookup(dcontext, tgt) != NULL)
+                STATS_INC(rct_ind_rip_rel_old);
+        });
+    }
+    return res;
+}
+# endif /* X64 */
+
+/* The exported functions of a particular module are in fact not
+ * absolute references - the linker keeps them in RVA format therefore
+ * we do need to walk the exports table for them.  For the same
+ * reason, we can do this walk whether the actual module is relocated or
+ * as soon as the module is mapped in the address space
+ */
+static void
+rct_add_exports(dcontext_t *dcontext, app_pc module_base, size_t module_size)
+{
+    /* a more complicated version that also walks the parallel function
+     * names is done in add_module_info().
+     */
+    size_t size;
+    IMAGE_EXPORT_DIRECTORY *exports =
+        get_module_exports_directory_check(module_base, &size,
+                                           false /* only check functions array */);
+            
+    if (exports != NULL) {
+        /* RVA array of all exported addresses */
+        PULONG functions =  (PULONG) (module_base + exports->AddressOfFunctions);
+        uint i;
+
+        LOG(GLOBAL, LOG_SYMBOLS, 3, "\tnumnames=%d numfunc=%d", 
+            exports->NumberOfNames, exports->NumberOfFunctions);
+
+        if (exports->NumberOfFunctions == 0) {
+            /* no functions to add */
+            /* riched32.dll from mmc.exe actually hits this */
+            return;
+        }
+
+        LOG(GLOBAL, LOG_RCT, 3, 
+            "rct_add_exports: dll_name=%s exports="PFX" numnames=%d numfunc=%d %s",
+            (char*) (module_base + exports->Name),
+            exports, 
+            exports->NumberOfNames, 
+            exports->NumberOfFunctions,
+            (exports->NumberOfFunctions == exports->NumberOfNames) ? "" : "NONAMES ");
+
+        /* for our security policy to restrict transfers to exports
+         * only, we need all functions (no matter whether named or not)
+         * just watch out for forwarders
+         */
+        for (i = 0; i < exports->NumberOfFunctions; i++) {
+            /* plain walk through AddressOfFunctions array */
+            app_pc func = module_base + functions[i];
+
+            /* check if it points within the exports section in real address space */
+            if (func < (app_pc)exports || func >= (app_pc)exports + size) {
+                /* FIXME: use print_symbolic_address() */
+                LOG(GLOBAL, LOG_RCT, 3, "\tadding i=%d "PFX"\n", i, func);
+                /* interestingly there are ordinals in shell32.dll
+                 * that are at module_base, so can't make this point
+                 * to code sections only
+                 */
+                ASSERT(func >= module_base && func < (module_base + module_size) ||
+                       EXEMPT_TEST("partial_map.exe"));
+                /* FIXME: note that we may add not only functions but
+                 * export data as well! 
+                 * FIXME: currently we leave on code origins to cover this up,
+                 * FIXME: while instead we should check for code sections here
+                 * just like we need to do in rct_analyze_module() anyways
+                 */
+                if (rct_add_valid_ind_branch_target(dcontext, func)) {
+                    STATS_INC(rct_ind_branch_valid_targets);
+                    STATS_INC(rct_ind_added_exports);
+                } else {
+                    LOG(GLOBAL, LOG_RCT, 3,
+                        "\t already added export entry i=%d "PFX"\n", i, func);
+                    /* most likely address taken - 
+                     * FIXME: verify that they are all really address taken, 
+                     * and not say forwards, although those can't possibly be added
+                     */
+                    STATS_INC(rct_ind_already_added_exports);
+                }
+            } else {
+                /* skip forwarded function - it forwards to a named
+                 * import e.g. NTDLL.RtlAllocateHeap which will be
+                 * added in its own module's exports */
+                LOG(GLOBAL, LOG_RCT, 3, "Forward to "PFX" %s.  Skipping...\n", func,
+                    (char*)func);
+            }
+        }
+        
+        DOLOG(2, LOG_RCT, { 
+            char short_name[MAX_MODNAME_INTERNAL];
+            os_get_module_name_buf(module_base, short_name,
+                                   BUFFER_SIZE_ELEMENTS(short_name));
+            LOG(GLOBAL, LOG_RCT, 2, "rct_add_exports: %s : %d exports added\n", 
+                short_name, exports->NumberOfFunctions);
+        });
+    } else {
+        DOLOG(SYMBOLS_LOGLEVEL, LOG_SYMBOLS, { 
+            char short_name[MAX_MODNAME_INTERNAL];
+            os_get_module_name_buf(module_base, short_name,
+                                   BUFFER_SIZE_ELEMENTS(short_name));
+
+            /* the executable itself is OK */
+            if (module_base != get_own_peb()->ImageBaseAddress) {
+                if (short_name) 
+                    LOG(GLOBAL, LOG_SYMBOLS, 2, "No exports %s\n", short_name);
+                else
+                    LOG(GLOBAL, LOG_SYMBOLS, 2, "Not a PE at "PFX"\n", module_base);
+            }
+        });
+    }
+}
+
+/* For each relocation entry, check if the address refers to code section
+ * [referto_start, referto_end).  Add all such valid references to the indirect
+ * branch hashtable.
+ * returns: -1 if there were no relocation entries
+ *           0 if there was a valid entry refering to some section
+ *          references_found, otherwise
+ */
+/* FIXME: should switch to using module_reloc_iterator_start() */
+static
+int
+find_relocation_references(dcontext_t *dcontext,
+                           app_pc module_base, size_t module_size,
+                           IMAGE_BASE_RELOCATION *base_reloc,
+                           size_t base_reloc_size,
+                           ssize_t relocation_delta,
+                           app_pc referto_start, app_pc referto_end)
+{
+    app_pc relocs = NULL;          /* reloc entries */
+    app_pc relocs_end = 0;         /* to iterate through reloc entries */
+    app_pc relocs_block_end = 0;   /* to iterate through reloc blocks */
+    bool is_module_32bit;
+
+    int references_found = -1;
+    DEBUG_DECLARE(
+        uint references_already_known = 0;
+        uint addresses_scanned = 0;
+        uint pages_touched = 0;
+        char symbuf[MAXIMUM_SYMBOL_LENGTH] = "";
+    );
+
+    ASSERT(sizeof(uint)   == 4);   /* size of rva and block_size - 32 bits */
+    ASSERT(sizeof(ushort) == 2);   /* size of reloc entry - 16 bits */
+
+    ASSERT(module_base != NULL);   /* caller should have verified base */
+
+    /* callers set up base_reloc and base_reloc_size by calling
+     * get_module_base_reloc */
+    if (base_reloc == NULL || base_reloc_size == 0) {
+        ASSERT(false && "expect relocations");
+        return 0;
+    }
+    is_module_32bit = module_is_32bit(module_base);
+
+    ASSERT(is_readable_without_exception((app_pc)base_reloc, base_reloc_size));
+    ASSERT(referto_start <= referto_end); /* empty ok */
+
+    DOLOG(2, LOG_RCT, {
+        print_symbolic_address(module_base, symbuf, sizeof(symbuf), true);
+        NULL_TERMINATE_BUFFER(symbuf);
+        LOG(GLOBAL, LOG_RCT, 2, "reloc: find_relocation_references: "
+            "module=%s, module_base="PFX", base_reloc="PFX", "
+            "base_reloc_size="PFX", referto["PFX", "PFX")\n",
+            symbuf, module_base, base_reloc, base_reloc_size,
+            referto_start, referto_end);
+    });
+
+    /* Iterate through relocation entries and check if they contain
+     * references to code section
+     */
+    relocs     = (app_pc) base_reloc;
+    relocs_end = relocs + base_reloc_size;
+
+    KSTART(rct_reloc);
+    while (relocs < relocs_end) {
+        /* Image based relocation is stored as:
+         * DWORD RVA,
+         * DWORD SizeOfBlock
+         * followed by a WORD array relocation entries
+         *   number of relocation entries = 
+         *       (SizeOfBlock - IMAGE_SIZEOF_BASE_RELOCATION) / sizeof(ushort)
+         */
+        uint rva = (uint) ((IMAGE_BASE_RELOCATION *)relocs)->VirtualAddress;
+        uint block_size = (uint) ((IMAGE_BASE_RELOCATION *)relocs)->SizeOfBlock;
+
+        LOG(GLOBAL, LOG_RCT, 6, "\t%s %8x RVA, %8x SizeofBlock\n",
+            symbuf, rva, block_size);
+        /* WORD array of relocation entries
+         *   top 4-bits are type
+         *       [IMAGE_REL_BASED_ABSOLUTE | IMAGE_REL_BASED_HIGHLOW] - PE32/x86
+         *       [IMAGE_REL_BASED_ABSOLUTE | IMAGE_REL_BASED_DIR64] - PE32+/x64
+         *   remaining 12-bits are offset relative to RVA
+         *
+         * 0:000> dw 77c10000 + 55000
+         * 77c65000  1000 0000 0158 0000 327c 3288 328c 3290 
+         * 77c65010  3294 3298 32a4 32a8 32ac 32b8 32ec 32f0
+         * ...
+         */
+        relocs_block_end = relocs + block_size;
+        relocs = relocs + IMAGE_SIZEOF_BASE_RELOCATION;
+
+        KSTART(rct_reloc_per_page);
+        while (relocs < relocs_block_end) {
+            DEBUG_DECLARE(bool known_ref = false;)
+            bool null_ref = false;
+            app_pc cur_addr = NULL;
+            app_pc ref = process_one_relocation(module_base, relocs, rva,
+                                                relocation_delta, 
+                                                false /* don't apply reloc */,
+                                                &null_ref, NULL, &cur_addr,
+                                                is_module_32bit
+                                                _IF_DEBUG(module_size));
+            if (!null_ref) {
+                /* There is at least one valid entry, so no longer
+                 * return -1 */
+                if (references_found < 0)
+                    references_found = 0;
+
+                /* sanity check */
+                DODEBUG({
+                    app_pc module_end = module_base + module_size; 
+                    /* There are many cases where ref is outside
+                     * module, but with early_injection and
+                     * analzye_at_load it should be rare. Have seen
+                     * this in USER32.dll, where 77d9d414 is outside
+                     * module:
+                     *   77D9D410: 82B60F09 77D3394F 836656EB C01BE0F9 
+                     USER32!UserIsFELineBreakEnd+0x9:
+                     77d9d36b 0fb7d1           movzx   edx,cx
+                     ...
+                     77d9d40b 6683f99f         cmp     cx,0xff9f
+                     77d9d40f 7709             ja      USER32!UserIsFELineBreakEnd+0xb9 (77d9d41a)
+                     77d9d411 0fb6824f39d377   movzx   eax,byte ptr [edx+0x77d3394f]
+                     */
+                    if (ref < module_base || ref >= module_end) {
+                        LOG(GLOBAL, LOG_RCT, 2,
+                            "find_relocation_references: ref "PFX" taken at "
+                            "addr "PFX" not in module ["PFX","PFX")\n", 
+                            ref, cur_addr, module_base, module_end);
+                        STATS_INC(rct_ind_branch_ref_outside_module);
+                    }
+                });
+            }                
+
+            if (rct_check_ref_and_add(dcontext, ref,
+                                      referto_start, referto_end
+                                      _IF_DEBUG(cur_addr)
+                                      _IF_DEBUG(&known_ref)))
+                references_found++;
+            else {
+                DODEBUG({
+                    if (known_ref)
+                        references_already_known++;
+                });
+            }
+
+            DODEBUG(addresses_scanned++;);
+            /* IMAGE_REL_BASED_ABSOLUTE is just used for padding, ignore */
+
+            relocs = relocs + sizeof(ushort);
+        } /* a block of relocation entries */
+        KSTOP(rct_reloc_per_page);
+
+        /* each block of relocation entries is for a 4k (PAGE_SIZE) page */
+        DODEBUG(pages_touched++;);
+    } /* all relocation entries */
+    KSTOP(rct_reloc);
+
+    LOG(GLOBAL, LOG_RCT, 1, "reloc: find_relocation_references:  "
+        "scanned %u addresses, touched %u pages, "
+        "added %d new, %u duplicate ind targets\n", 
+        addresses_scanned, pages_touched, 
+        references_found, references_already_known);
+    
+    return references_found;
+}
+
+/* Case 7275: an image with alignment > PAGE_SIZE can have reserved-but-not-committed
+ * pages within its allocation region.  Thus we must scan each region individually.
+ * This routine is normally called only on a single MEM_IMAGE allocation region,
+ * but we don't restrict the caller and blindly process all committed pieces in
+ * [text_start, text_end), calling find_address_references() on each.
+ */
+static uint
+find_address_references_by_region(dcontext_t *dcontext,
+                                  app_pc text_start, app_pc text_end, 
+                                  app_pc referto_start, app_pc referto_end)
+{
+    app_pc pc;
+    MEMORY_BASIC_INFORMATION mbi;
+    uint found = 0;
+    LOG(GLOBAL, LOG_RCT, 2, "find_address_references_by_region ["PFX", "PFX")\n",
+        text_start, text_end);
+#if defined(X64) && (defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH))
+    os_module_set_flag(text_start, MODULE_RCT_SCANNED);
+#endif
+    for (pc = text_start; pc < text_end; pc += mbi.RegionSize) {
+        if (query_virtual_memory(pc, &mbi, sizeof(mbi)) != sizeof(mbi)) {
+            ASSERT(false && "error querying memory for rct analysis");
+            break;
+        }
+        if (pc + mbi.RegionSize < pc) /* overflow check */
+            break;
+        if (mbi.State == MEM_COMMIT) {
+            /* safe to read */
+            found += find_address_references(dcontext,
+                                             pc, MIN(pc + mbi.RegionSize, text_end),
+                                             referto_start, referto_end);
+        } else {
+            LOG(GLOBAL, LOG_RCT, 2, "\t["PFX", "PFX") not committed (state 0x%x)\n",
+                pc, pc + mbi.RegionSize, mbi.State);
+        }
+    }
+    return found;
+}
+
+/* Called while analyzing a module at load time or at the point of a
+ * violation. For the latter, we defer until a failing indirect call/jmp
+ * happens and then walk the corresponding code sections in the yet
+ * undefined module.  Since the entry points of DLLs are targeted by
+ * indirect calls we'll have all DLLs tested quickly anyways.
+ *
+ * As an optimization, we go through relocation table whenever present,
+ * walk it and only compute addresses that are relevant.  Note that most
+ * DLLs (other than /FIXED ones) will have a relocation table and even some
+ * executables are relocatable.  Note that some modules may be quite large,
+ * so going through the relocation list will be worthwhile.  If no
+ * relocations are present we then scan code sections for valid ind branch
+ * targets.
+ */
+static void
+add_rct_module(dcontext_t *dcontext,
+               app_pc module_base, size_t module_size, ssize_t relocation_delta,
+               bool at_violation)
+{
+    IMAGE_DOS_HEADER *dos = (IMAGE_DOS_HEADER *) module_base;
+    IMAGE_NT_HEADERS *nt = (IMAGE_NT_HEADERS *) (((ptr_uint_t)dos) + dos->e_lfanew);
+    IMAGE_SECTION_HEADER *sec;
+    uint i;
+
+    /* FIXME: PRECISION/speed: limit searched range only to code and
+     * initialized data section, although unclear whether resources
+     * may or may not have functions or function pointers
+     */
+    app_pc text_start = module_base;
+    app_pc text_end = module_base + module_size;
+    uint found = 0;
+    app_pc entry_point; 
+    DEBUG_DECLARE(uint code_sections = 0;)
+    DEBUG_DECLARE(char modname[MAX_MODNAME_INTERNAL];)
+    DODEBUG({
+        os_get_module_name_buf(module_base, modname, BUFFER_SIZE_ELEMENTS(modname));
+    });
+
+    ASSERT(is_readable_pe_base(module_base));
+    ASSERT_OWN_MUTEX(true, &rct_module_lock);
+
+    STATS_INC(rct_ind_branch_modules_analyzed);
+
+    ASSERT(DYNAMO_OPTION(rct_section_type) != 0 &&
+           !TESTANY(~(IMAGE_SCN_CNT_CODE|IMAGE_SCN_CNT_INITIALIZED_DATA|IMAGE_SCN_CNT_UNINITIALIZED_DATA),
+                    DYNAMO_OPTION(rct_section_type)));
+           
+    /* #define IMAGE_SCN_CNT_CODE                   0x00000020  // Section contains code. */
+    /* #define IMAGE_SCN_CNT_INITIALIZED_DATA       0x00000040  // Section contains initialized data. */
+    /* #define IMAGE_SCN_CNT_UNINITIALIZED_DATA     0x00000080  // Section contains uninitialized data. */
+
+    /* For code, there can be multiple sections (.text, .orpc, perhaps others)
+     * so we walk the section headers and check for the "code" flag.
+     * iterator from is_in_executable_file_section()
+     */
+    LOG(GLOBAL, LOG_VMAREAS, 4, "module @ "PFX":\n", module_base);
+    /* PRECISION/memory: limit code only to a code section, and
+     * process one section at a time.
+     */
+    sec = IMAGE_FIRST_SECTION(nt);
+    for (i = 0; i < nt->FileHeader.NumberOfSections; i++, sec++) {
+        LOG(GLOBAL, LOG_VMAREAS, 4, "\tName = %.*s\n", IMAGE_SIZEOF_SHORT_NAME,
+            sec->Name);
+        LOG(GLOBAL, LOG_VMAREAS, 4, "\tVirtualSize    = "PFX"\n", sec->Misc.VirtualSize);
+        LOG(GLOBAL, LOG_VMAREAS, 4, "\tVirtualAddress = "PFX"\n", sec->VirtualAddress);
+        LOG(GLOBAL, LOG_VMAREAS, 4, "\tSizeOfRawData  = "PFX"\n", sec->SizeOfRawData);
+        LOG(GLOBAL, LOG_VMAREAS, 4, "\tCharacteristics= "PFX"\n", sec->Characteristics);
+
+        /* FIXME: case 5355, case 10526 - note we are not following the convoluted
+         * section size matching from is_in_executable_file_section -
+         * we don't have an OS region to be matching against. */
+
+        if (TESTANY(DYNAMO_OPTION(rct_section_type), sec->Characteristics) &&
+            (DYNAMO_OPTION(rct_section_type_exclude) == 0 || 
+             !TESTALL(DYNAMO_OPTION(rct_section_type_exclude), sec->Characteristics))) {
+            bool scan_all_addresses = true;
+            app_pc code_start = module_base + sec->VirtualAddress;
+            app_pc code_end = module_base + sec->VirtualAddress +
+                get_image_section_size(sec, nt);
+            LOG(GLOBAL, LOG_VMAREAS, 2,
+                "add_rct_module (module "PFX"): %.*s == "PFX"-"PFX"\n",
+                module_base, IMAGE_SIZEOF_SHORT_NAME, sec->Name, code_start, code_end);
+            
+            DODEBUG(code_sections++;);
+
+            /* we don't expect too many code sections, otherwise we
+             * can invert the loop and go through the whole file only
+             * once.  Three sections in suite/tests/win32/multisec.
+             */
+            /* if we're enforcing on all section types we can get a few more sections */
+            ASSERT_CURIOSITY((DYNAMO_OPTION(rct_section_type) == IMAGE_SCN_CNT_CODE
+                              && code_sections < 5) /* default */
+                             || code_sections < 10);
+
+            if (DYNAMO_OPTION(rct_reloc)) {
+                IMAGE_BASE_RELOCATION *base_reloc = NULL;
+                size_t base_reloc_size = 0;
+
+                base_reloc = get_module_base_reloc(module_base, &base_reloc_size);
+                LOG(GLOBAL, LOG_RCT, 2,
+                    "reloc: add_rct_module: module_base="PFX", "
+                    "base_reloc="PFX", base_reloc_size="PFX")\n",
+                    module_base, base_reloc, base_reloc_size);
+
+                /* FIXME: We walk through relocations for each code section
+                 * and hence stats can be counted more than once.
+                 */
+                if (base_reloc != NULL && base_reloc_size > 0) {
+                    int refs_found = 
+                            find_relocation_references(dcontext,
+                                                       module_base, module_size,
+                                                       base_reloc,
+                                                       base_reloc_size,
+                                                       relocation_delta,
+                                                       code_start, code_end);
+                    if (refs_found >= 0) {
+                        found += refs_found;
+                        /* PR 215408: even when we have reloc info, we need to scan
+                         * for rip-rel lea for any module whose code we haven't
+                         * executed.  We assume we don't need to scan the executable
+                         * (not executed from until post-inject).  We assume that all
+                         * modules loaded at inject time need to be scanned (even
+                         * though they may not all have been executed from).  Since
+                         * early inject should be the norm, we should usually only be
+                         * scanning ntdll, which reduces the perf impact.  For
+                         * native_exec (PR 277044) and for hardcoded hooks (PR 277064)
+                         * we also scan on violation for modules we didn't scan
+                         * proactively here.
+                         */
+                        if (IF_X64(at_violation ||)
+                            (!dynamo_initialized && DYNAMO_OPTION(rct_scan_at_init) && 
+                             module_base != get_own_peb()->ImageBaseAddress)) {
+                            LOG(GLOBAL, LOG_RCT, 1,
+                                "add_rct_module: scanning "PFX" even though has relocs\n",
+                                module_base);
+                            DOSTATS({
+                                if (IF_X64_ELSE(at_violation, false))
+                                    STATS_INC(rct_scan_at_vio);
+                                else
+                                    STATS_INC(rct_scan_at_init);
+                            });
+                        } else
+                            scan_all_addresses = false;
+                    }
+                    else {
+                        /* relocation section found, but no relocations?
+                         * fall back to scanning all addresses, as backup
+                         */
+                        LOG(GLOBAL, LOG_RCT, 1,
+                            "add_rct_module: relocation section found, but "
+                            "no relocations.  Falling back to scanning all "
+                            "addresses\n");
+                        STATS_INC(rct_ind_branch_no_valid_targets);
+                    }
+                }
+            }
+
+            /* Don't use or couldn't find relocation info, scan all
+             * addresses to find ind branch targets
+             */
+            if (scan_all_addresses) {
+                found += find_address_references_by_region(dcontext,
+                                                           text_start, text_end, 
+                                                           code_start, code_end);
+            }
+
+            STATS_INC(rct_ind_branch_sections_analyzed);
+        }
+    }
+
+    entry_point = get_module_entry(module_base);
+
+    /* add the module entry point (executable's main and DllMain()) */
+    /* since they don't have to be exported nor address taken */
+    if (entry_point != NULL) {
+        DODEBUG({
+            if (entry_point < module_base || entry_point >= module_base + module_size) {
+                const char *target_module_name = NULL;
+                /* may not be added to our own module list yet as we often get
+                 * here when processing the exe, often the 1st module we see
+                 */
+                if (module_info_exists(entry_point)) {
+                    target_module_name =
+                        os_get_module_name_strdup(entry_point HEAPACCT(ACCT_VMAREAS));
+                    ASSERT(target_module_name != NULL);
+                }
+                if (target_module_name == NULL) {
+                    target_module_name =
+                        get_module_short_name_uncached(dcontext, entry_point,
+                                                       false/*!at map*/
+                                                       HEAPACCT(ACCT_VMAREAS));
+                }
+                ASSERT_CURIOSITY(target_module_name != NULL ||
+                                 /* for partial map, entry point may not be mapped in */
+                                 EXEMPT_TEST("partial_map.exe"));
+
+                /* case 5776 see if it was rerouted outside of the module, e.g. in .NET */
+                /* FIXME: we still can't tell whether it was modified */
+                LOG(GLOBAL, LOG_RCT, 1,"entry point outside of module: %s "
+                    PFX"-"PFX", entry point="PFX", in %s\n",
+                    modname, module_base, module_base + module_size,
+                    entry_point,
+                    target_module_name == NULL ? "<null>" : target_module_name);
+
+                if ((target_module_name == NULL ||
+                     !check_filter("mscoree.dll", target_module_name)) &&
+                    !check_filter("partial_map.exe",
+                                  get_short_name(get_application_name()))) {
+                    SYSLOG_INTERNAL_WARNING("entry point outside of module: %s "
+                                            PFX"-"PFX", entry point="PFX" %s\n",
+                                            modname,
+                                            module_base, module_base + module_size,
+                                            entry_point, target_module_name == NULL ?
+                                            "<null>" : target_module_name);
+                    /* Quiet for partial_map test case (where isn't actually modified,
+                     * just not mapped in). */
+                    ASSERT_CURIOSITY(false && "modified entry point" ||
+                                     EXEMPT_TEST("partial_map.exe"));
+                }
+                if (target_module_name != NULL)
+                    dr_strfree(target_module_name HEAPACCT(ACCT_VMAREAS));
+            }
+        });
+        rct_add_valid_ind_branch_target(dcontext, entry_point);
+    } else {
+        /* curious if any module other than ntdll.dll has no entry point,
+         * see above note why this would be important
+         */
+        LOG(GLOBAL, LOG_RCT, 1, "add_rct_module: %s, NULL entry point="PFX"\n", 
+            modname, entry_point);
+    }
+
+    LOG(GLOBAL, LOG_RCT, 2,
+        "add_rct_module: %s : %d ind targets for %d size, entry="PFX"\n", 
+        modname, found, module_size, entry_point);
+
+    rct_add_exports(dcontext, module_base, module_size);
+    /* FIXME: case 1948 curiosity: dump any exported entries that are also address taken */
+
+    /* PR 250395: add SEH handlers from .pdata section */
+    IF_X64(add_SEH_to_rct_table(dcontext, module_base);)
+}
+
+/* Analyze a range in a possibly new module
+ * and add all valid targets for rct_ind_branch_check 
+ */
+static void
+rct_analyze_module_at_load(dcontext_t *dcontext,
+                           app_pc module_base, size_t module_size,
+                           ssize_t relocation_delta)
+{
+    ASSERT(module_size != 0 && is_readable_pe_base(module_base));
+    DODEBUG({
+        MEMORY_BASIC_INFORMATION mbi;
+        /* xref case 8192, expect to only be analyzing IMAGE memory */
+        ASSERT(query_virtual_memory(module_base, &mbi, sizeof(mbi)) == sizeof(mbi) &&
+               mbi.Type == MEM_IMAGE);
+    });
+
+    /* do not analyze and add targets in dynamorio.dll */
+    if (is_in_dynamo_dll(module_base))
+        return;
+
+    LOG(GLOBAL, LOG_RCT, 1,
+        "rct_analyze_module_at_load: module_base="PFX", module_size=%d, "
+        "relocation_delta=%c"PFX"\n",
+        module_base, module_size,
+        relocation_delta < 0 ? '-':' ', 
+        relocation_delta < 0 ? -relocation_delta : relocation_delta);
+
+    add_rct_module(dcontext, module_base, module_size, relocation_delta, false);
+}
+
+/* Analyze a range, in a possibly new module (if we don't analyze_at_load),
+ * return false if not a code section in a module
+ * otherwise return true and add all valid targets for rct_ind_branch_check
+ * NOTE - under current default options (analyze_at_load) this routine is only
+ * used for its is_in_code_section (&IMAGE) return value. */
+bool
+rct_analyze_module_at_violation(dcontext_t *dcontext, app_pc target_pc)
+{
+    app_pc module_base;
+    MEMORY_BASIC_INFORMATION mbi;
+    size_t module_size = get_allocation_size(target_pc, &module_base);
+    uint sec_flags = 0;
+
+    if (module_base == NULL || !is_readable_pe_base(module_base))
+        return false;
+
+    /* xref case 8192, don't consider target a code section if it's not MEM_IMAGE 
+     * (since if MEM_PRIVATE or MEM_MAPPED we won't have analyzed at_load) */
+    if (query_virtual_memory(module_base, &mbi, sizeof(mbi)) == sizeof(mbi) &&
+        mbi.Type != MEM_IMAGE) {
+        SYSLOG_INTERNAL_WARNING_ONCE("Transfer to non-IMAGE memory "PFX" that looks "
+                                     "like a pe module.", target_pc);
+        return false;
+    }
+
+    LOG(GLOBAL, LOG_RCT, 1,
+        "rct_analyze_module_at_violation: target_pc="PFX"\n", target_pc);
+
+    /* test what area is target_pc in and enforce and analyze only if
+     * in desired section type.  Default rct_section_type is code section only.
+     */
+    ASSERT(DYNAMO_OPTION(rct_section_type) != 0); /* sentinel for is_in_executable_... */
+    if (!is_in_executable_file_section(module_base, target_pc, target_pc+1, NULL, NULL,
+                                       &sec_flags, NULL, DYNAMO_OPTION(rct_section_type),
+                                       NULL, false /* no need to merge */) ||
+        (DYNAMO_OPTION(rct_section_type_exclude) != 0 &&
+         TESTALL(DYNAMO_OPTION(rct_section_type_exclude), sec_flags))) {
+        SYSLOG_INTERNAL_WARNING_ONCE("RCT executing from non-analyzed module "
+                                     "section at "PFX, target_pc);
+        /* FIXME: heavy-weight check if done every time for execution
+         * off .data section until it makes it into a trace
+         */
+        return false;
+    }
+
+    /* analyze the module if we haven't already done so at load and if it is
+     * not dynamorio.dll (case 7266)
+     */
+    if ((!DYNAMO_OPTION(rct_analyze_at_load) && !is_in_dynamo_dll(module_base))
+#if defined(X64) && (defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH))
+        /* We need to scan for rip-rel leas we didn't see execute:
+         * for native_exec (PR 277044) and hardcoded hooks (PR 277064) */
+        || !os_module_get_flag(module_base, MODULE_RCT_SCANNED)
+#endif
+        ) {
+        add_rct_module(dcontext, module_base, module_size, 0/* already relocated */,
+                       true);
+    }
+
+    return true;
+}
+
+/* module map/unmap processing relevant to the RCT policies */
+void
+rct_process_module_mmap(app_pc module_base, size_t module_size, 
+                        bool add, bool already_relocated)
+{
+    DEBUG_DECLARE(char modname[MAX_MODNAME_INTERNAL];)
+    /* although we expect MEM_IMAGE regions to be indeed only PE's,
+     * not taking chances
+     */
+    if (!is_readable_pe_base(module_base))
+        return;
+    DODEBUG({
+        os_get_module_name_buf(module_base, modname, BUFFER_SIZE_ELEMENTS(modname));
+    });
+
+    if (add) {
+        ssize_t delta = 0;
+
+        /* we can't just check -no_use_persisted_rct since Borland overrides that */
+        if (DYNAMO_OPTION(use_persisted) &&
+            os_module_get_flag(module_base, MODULE_RCT_LOADED)) {
+            /* We can only skip analyzing the module if the persisted data
+             * covers the ENTIRE module, and we rely on the persisted cache
+             * using a flag stored at persist time to indicate that, as the RCT
+             * entries are not just drawn from the text section(s) but from
+             * other sections (like .orpc) that do not correspond to the normal
+             * persisted cache bounds.  If we persisted only some of the entries
+             * we'll re-analyze the whole module and waste some work, but we do
+             * check for duplicates before adding.
+             *
+             * FIXME case 8648: invisible IAT hooker (e.g., Kaspersky) could
+             * cause problems, so perhaps we should also always do
+             * rct_add_exports() here?
+             */
+            STATS_INC(rct_ind_branch_modules_persist_loaded);
+            LOG(GLOBAL, LOG_RCT, 2,
+                "rct_process_module_mmap: not processing "PFX" b/c persisted\n",
+                module_base);
+            return;
+        }
+
+        /* we need to know whether the module was relocated already
+         * (e.g. when we're taking over), or it's just mapped and
+         * therefore we should relocate.  If it has been relocated and we
+         * assume it needs further relocation we'll have the wrong
+         * offsets, so need to know that.
+         */
+        if (!already_relocated) {
+            /* delta = OLD - NEW since usually used to add to an
+             * address in a relocated file, while here we have not yet
+             * relocated one.  We want to add to OLD addresses the
+             * negated value NEW - OLD = - delta */
+            delta = - get_module_preferred_base_delta(module_base);
+            LOG(GLOBAL, LOG_RCT, 2,
+                "rct_process_module_mmap: "PFX" relocation_delta=%c"PFX"\n",
+                module_base, 
+                delta < 0 ? '-':' ', 
+                delta < 0 ? -delta : delta);
+        }
+
+        /* Grab the rct_module_lock to ensure no conflicts while
+         * processing entries
+         */
+        mutex_lock(&rct_module_lock);
+        if (DYNAMO_OPTION(rct_analyze_at_load)) {
+            /* should use GLOBAL_DCONTEXT since called early */
+            rct_analyze_module_at_load(GLOBAL_DCONTEXT,
+                                       module_base, module_size, delta);
+        }
+
+        if (DYNAMO_OPTION(rct_modified_entry)) {
+            /* case 5776 - where mscoree.dll modifies the image entry point */
+            /* FIXME: case 5354: rct_analyze_at_load took care of most of this.
+             * Clean up.
+             */
+            app_pc entry_point = get_module_entry(module_base);
+            /* assume unmodified entry_point if within module or NULL.
+             * NOTE that entry_point RVA of 0 is the module_base, 
+             * yet that can't possibly be valid NT header as well as entry point.
+             * We'll assume we DON'T need to read the LDR structure in that case.
+             */
+            bool use_ldr = ((entry_point < module_base)
+                            || (entry_point >= module_base + module_size));
+
+            if (already_relocated) {
+                /* loaded before we were in control */
+                if (use_ldr) {
+                    /* modified PE image entry - we should look up the LDR entry point. */
+                    /* Only seen for executables, but statically linked
+                     * DLLs may have this too.  Though still unclear
+                     * what's the point of modifying the PE entry if the
+                     * loader uses the original entry point from LDR structure.
+                     */
+                    LDR_MODULE *mod = get_ldr_module_by_pc(module_base);
+                    /* walking the loader list is, of course unsafe,
+                     * unless we're the single thread at the time of
+                     * DR initialization
+                     */
+                    ASSERT_CURIOSITY(check_sole_thread());
+
+                    ASSERT(use_ldr ||
+                           (mod != NULL && mod->EntryPoint == entry_point) || 
+                           /* msvcrt40.dll is a good example of no entry point */
+                           (mod != NULL && mod->EntryPoint == 0 && entry_point == module_base));
+                    if (use_ldr && mod != NULL) {
+                        entry_point = mod->EntryPoint;
+                        LOG(GLOBAL, LOG_RCT, 1, "rct_process_module_mmap: %s "
+                            ".NET modified entry point="PFX"\n", 
+                            modname, entry_point);
+                        DODEBUG({
+                            SYSLOG_INTERNAL_WARNING("rct_process_module_mmap: %s "
+                                                ".NET modified entry point="PFX"\n", 
+                                                modname, entry_point);
+                            /* expected to be mscoree!_CorDllMain or mscoree!_CorExeMain */
+                        });
+                    }
+                }
+            } else {
+                /* Quiet assert for partial map test case where we are tricked above
+                 * into thinking the entry point has been modified (it just hasn't been
+                 * mapped in). */
+                ASSERT_CURIOSITY(!use_ldr || EXEMPT_TEST("partial_map.exe"));
+                ASSERT(get_thread_private_dcontext() != NULL);
+                /* newly loaded module - we should add the PE entry point now */
+            }
+
+            DODEBUG({
+                /* FIXME: rct_analyze_module_at_violation counts on most DLLs
+                 * having an entry point, so analysis is not deferred
+                 * for too long - see case 5354
+                 */
+                if (entry_point == module_base) {
+                    /* shdoclc.dll, xpsp2res.dll and msls31.dll are known 
+                     * instances in notepad on XP SP2 */
+                    LOG(GLOBAL, LOG_RCT, 1, "rct_process_module_mmap: %s, "
+                        "entry point=NULL\n",
+                        modname != NULL ? modname : "<null>");
+                }
+            });
+
+            /* add the module entry point DllMain() for mmap'ed region, before it is modified */
+            if (entry_point != NULL) {
+                /* should use GLOBAL_DCONTEXT since called early */
+                rct_add_valid_ind_branch_target(GLOBAL_DCONTEXT, entry_point);
+                LOG(GLOBAL, LOG_RCT, 2,
+                    "rct_process_module_mmap: %s, entry point="PFX"\n", 
+                    modname, entry_point);
+
+            } else {
+                LOG(GLOBAL, LOG_RCT, 1,
+                    "rct_process_module_mmap: %s, entry point=NULL\n", 
+                    modname);
+                ASSERT_NOT_REACHED();
+            }
+        }
+
+        mutex_unlock(&rct_module_lock);
+    } else {
+        /* case 9672: we now use per-module tables, so we don't need to
+         * take any explicit action here; the tables will simply be removed.
+         */
+    }
+}
+
+# ifdef DEBUG
+/* FIXME: this is an inefficient hack using the find_predecessor data
+ * structures. The right solution for this problem is to add all
+ * entries to a hashtable when walking the module exports table.
+ */
+/* we're still using this to collect some statistics in rct_  */
+bool
+rct_is_exported_function(app_pc tag)
+{
+    module_info_t mod = {0}, *pmod;
+    mutex_lock(&process_module_vector.lock); /* FIXME: this can be a shared read lock */
+    {
+        pmod = lookup_module_info(&process_module_vector, tag);
+        if (pmod) {
+            mod = *pmod; /* keep a copy in case of reallocations */
+            /* the data will be invalid only in a race condition,
+             * where some other thread frees the library
+             */
+        }
+    }
+    mutex_unlock(&process_module_vector.lock); 
+
+    if (pmod) {
+        int i = find_predecessor(mod.exports_table, mod.exports_num, tag);
+        if (i>=0 && mod.exports_table[i].entry_point == tag) {
+                return true;
+        }
+    }
+    return false;
+}
+# endif /* DEBUG */
+
+#endif /* RCT_IND_BRANCH */
+/****************************************************************************/
+
+void
+os_modules_init(void)
+{
+    if (DYNAMO_OPTION(hide))
+        hide_from_module_lists();
+}
+
+void
+os_modules_exit(void)
+{
+
+}
+
+void
+free_module_names(module_names_t *mod_names HEAPACCT(which_heap_t which))
+{
+    ASSERT(mod_names != NULL);
+
+    if (mod_names->module_name != NULL)
+        dr_strfree(mod_names->module_name HEAPACCT(which));
+    if (mod_names->file_name != NULL)
+        dr_strfree(mod_names->file_name HEAPACCT(which));
+    if (mod_names->exe_name != NULL)
+        dr_strfree(mod_names->exe_name HEAPACCT(which));
+    if (mod_names->rsrc_name != NULL)
+        dr_strfree(mod_names->rsrc_name HEAPACCT(which));
+}
+
+/* destructor for os-specific module area fields */
+void
+os_module_area_reset(module_area_t *ma HEAPACCT(which_heap_t which))
+{
+    ASSERT(TEST(MODULE_BEING_UNLOADED, ma->flags));
+
+    if (ma->os_data.company_name != NULL)
+        dr_strfree(ma->os_data.company_name HEAPACCT(which));
+    if (ma->os_data.product_name != NULL)
+        dr_strfree(ma->os_data.product_name HEAPACCT(which));
+
+    if (ma->os_data.noclobber_section_handle != INVALID_HANDLE_VALUE) {
+        bool ok = close_handle(ma->os_data.noclobber_section_handle);
+        ASSERT_CURIOSITY(ok);
+    }
+
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+    LOG(GLOBAL, LOG_RCT, 1, "freeing RCT/RAC tables for %s "PFX"-"PFX"\n",
+        GET_MODULE_NAME(&ma->names), ma->start, ma->end);
+    {
+        uint i;
+        for (i = 0; i < RCT_NUM_TYPES; i++) {
+            rct_module_table_free(GLOBAL_DCONTEXT, &ma->os_data.rct_table[i], ma->start);
+        }
+    }
+#endif
+
+    if (ma->os_data.iat_code != NULL) {
+        /* iat code should be deleted earlier unless hooker that
+         * fools our loader match */
+        ASSERT_CURIOSITY(false && "iat code not deleted until unload");
+        module_area_free_IAT(ma);
+    }
+}
+
+/* In general, callers should use os_get_module_info() instead.
+ *
+ * Returns the timestamp from the PE file header, the checksum & image size from the
+ * PE optional header for the module at the given base address. If pe_name is
+ * non-null also returns the PE name as well (NOTE this can be NULL). code_size is
+ * computed by summing up the unpadded sizes for all code or executable sections.
+ * file_version is the 64-bit word from the resource section. All of the out arguments
+ * can be NULL. Returns true on success.
+ *
+ * We use a combined routine for the checksum, timestamp, image size, code_size,
+ * and pe_name since calls to is_readable_pe_base are expensive.
+ *
+ * FIXME : like many routines in module.c this routine is unsafe since the
+ * module in question could be unloaded while we are still looking around its
+ * header or before caller finishes using pe_name.  Need try-except.
+ */
+bool
+get_module_info_pe(const app_pc module_base, uint *checksum, uint *timestamp,
+                   size_t *image_size, char **pe_name, size_t *code_size)
+{
+    int i;
+    IMAGE_DOS_HEADER *dos;
+    IMAGE_NT_HEADERS *nt_hdr;
+    IMAGE_SECTION_HEADER *sec;
+
+    if (!is_readable_pe_base(module_base)) {
+        /* TODO: dump some sort of message here?  for customer or dbg. engr.? */
+        return false;
+    }
+
+    dos = (IMAGE_DOS_HEADER *) module_base;
+    nt_hdr = (IMAGE_NT_HEADERS *) (((ptr_uint_t)dos) + dos->e_lfanew);
+    if (timestamp != NULL)
+        *timestamp = nt_hdr->FileHeader.TimeDateStamp;
+    if (checksum != NULL)
+        *checksum = nt_hdr->OptionalHeader.CheckSum;
+    if (image_size != NULL)
+        *image_size = nt_hdr->OptionalHeader.SizeOfImage;
+
+    /* get_dll_short_name() usually shouldn't be called by itself, but through
+     * get_module_short_name() which does checks.  In this particular case, 
+     * the exact PE name is needed and only get_dll_short_name() provides that; 
+     * get_module_short_name() can return the name from the command line 
+     * string, so we can't use it to find the PE name.  
+     *
+     * CAUTION: If dll is unloaded the name is lost.  Also, ensure that checks 
+     *          performed by get_module_short_name() are replicated here.
+     *
+     * FIXME: if there is no exports section should we use the resource section
+     * to get the file name?  alex thinks not, I agree; tim thinks we should;
+     * I also think it is time to introduce distinct 'ignore' and 'not
+     * available' values.
+     */
+    if (pe_name != NULL)
+        *pe_name = get_dll_short_name(module_base);
+        
+    if (code_size != NULL) {
+        *code_size = 0;
+        sec = IMAGE_FIRST_SECTION(nt_hdr);
+        for (i = 0; i < nt_hdr->FileHeader.NumberOfSections; i++, sec++) {
+            /* Note a section may be executable but not marked as code; this
+             * isn't common, but isn't rare - kbdus.dll.  See case 9053.
+             */
+            if (TESTANY(IMAGE_SCN_CNT_CODE | IMAGE_SCN_MEM_EXECUTE,
+                        sec->Characteristics)) {
+                /* Executable sections should be loadable. */
+                DODEBUG({
+                    /* PR 214227: we do see INIT sections that are discardable */
+                    if (TEST(IMAGE_SCN_MEM_DISCARDABLE, sec->Characteristics)) {
+                        SYSLOG_INTERNAL_WARNING("found code section that is discardable");
+                    }
+                });
+                *code_size += get_image_section_unpadded_size(sec _IF_DEBUG(nt_hdr));
+            }
+        }
+        /* Can't assert on code_size > 0 because dlls can have no code sections. */
+    }
+
+    return true;
+}
+
+/* update our data structures that keep track of PE modules */
+void
+os_module_area_init(module_area_t *ma, app_pc base, size_t view_size,
+                    bool at_map HEAPACCT(which_heap_t which))
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    app_pc preferred_base = NULL;
+    uint timestamp = 0;
+    uint checksum = 0;
+    size_t pe_size = 0;
+    version_info_t info = {0};
+
+    /* currently add is done post-map, and remove is pre-unmap
+     * FIXME: we should remove at post-unmap, though unmap is unlikely to fail
+     */
+    ASSERT(is_readable_pe_base(base));
+
+    /* FIXME: theoretically need to grab a lock to prevent
+     * unmapping of a DLL that one thread is mapping and another
+     * one is unmapping.  We don't know for sure that all
+     * allocations that we see are really by the loader and
+     * can't assume all to be synchronized.
+     */
+    ma->entry_point = get_module_entry(base);
+    get_module_info_pe(base, &checksum, &timestamp, &pe_size, NULL, 
+                       &ma->os_data.code_size);
+    get_all_module_short_names_uncached(dcontext, base, at_map, &ma->names
+                                        HEAPACCT(which));
+    get_module_resource_version_info(base, &info); /* we inited to zero so ok if fails */
+    ma->os_data.file_version = info.file_version;
+    ma->os_data.product_version = info.product_version;
+    /* This converts unicode to ascii which might not always be good, but we do select
+     * the english version of the strings if available in the .rsrc section and
+     * all current users compare with ascii strings anyways.  FIXME */
+    if (info.company_name != NULL)
+        ma->os_data.company_name = dr_wstrdup(info.company_name HEAPACCT(which));
+    if (info.product_name != NULL)
+        ma->os_data.product_name = dr_wstrdup(info.product_name HEAPACCT(which));
+
+    if (TEST(ASLR_SHARED_CONTENTS, DYNAMO_OPTION(aslr_cache)) && 
+        dcontext != NULL && /* if during initialization */
+        dcontext->aslr_context.original_section_base != 
+        ASLR_INVALID_SECTION_BASE) {
+        DEBUG_DECLARE(uint pe_timestamp = timestamp;)
+        preferred_base = dcontext->aslr_context.original_section_base;
+        /* note that modules loaded before we have taken over
+         * are assumed to be using the native DLLs (unless we
+         * have full detach and one day a reattach) */
+        
+        /* keep in parallel with aslr_track_areas() unmap logic,
+         * to preserve _original_ app preferred base for shared mappings
+         * which have PE.ImageBase set to the current mapping address
+         */
+        
+        /* FIXME: case 1272 see also aslr_set_randomized_handle() for possibly
+         * producing more identifying data when we are watching NtCreateSection 
+         */
+        checksum = dcontext->aslr_context.original_section_checksum;
+        timestamp = dcontext->aslr_context.original_section_timestamp;
+        ASSERT(timestamp != 0 &&
+               pe_timestamp == aslr_timestamp_transformation(timestamp));
+        
+        /* register handle that needs to be closed on unmap */
+        ma->os_data.noclobber_section_handle = dcontext->aslr_context.
+            original_image_section_handle;
+        /* we could also duplicate the original we'd like to
+         * emulate ObReferenceObjectByPointer() on the section
+         * when a MapViewOfSection succeeds.
+         */
+        /* invalidate, so that we know we had a succesful map */
+        dcontext->aslr_context.
+            original_image_section_handle = INVALID_HANDLE_VALUE;
+    } else {
+        preferred_base = get_module_preferred_base(base);
+        ma->os_data.noclobber_section_handle = INVALID_HANDLE_VALUE;
+    }
+    
+    /* Xref case 9782: pe_size isn't always page aligned.  Drivers don't require
+     * page alignment and are sometimes mapped (though not loaded) into user
+     * processes. According to Nebbett the kernel rounds up view size to the next
+     * page multiple so since this is post syscall view size should already be
+     * page aligned.
+     * Xref case 9717: on Vista at least we sometimes see a view that isn't
+     * the full image.  Shouldn't be a problem here, but is a problem for other
+     * core components (reloc walking etc.). */
+    ASSERT_CURIOSITY(ALIGN_FORWARD(pe_size, PAGE_SIZE) == view_size ||
+                     EXEMPT_TEST("partial_map.exe"));
+    
+    ASSERT(preferred_base != NULL);
+    ma->os_data.preferred_base = preferred_base;
+    ma->os_data.checksum = checksum;
+    ma->os_data.timestamp = timestamp;
+    ma->os_data.module_internal_size = pe_size;
+    
+    /* FIXME: case 9032 about getting MemorySectionName */
+}
+
+/* gets the preferred base of the module containing pc, cached from our module list,
+ * returns NULL, if not in module
+ */
+app_pc
+get_module_preferred_base_safe(app_pc pc)
+{
+    /* FIXME: currently just a little safer */
+    module_area_t *ma;
+    app_pc preferred_base = NULL;
+
+    /* read lock to protect custom data */
+    os_get_module_info_lock();
+    ma = module_pc_lookup(pc);
+    if (ma != NULL) {
+        preferred_base = ma->os_data.preferred_base;
+    } 
+    os_get_module_info_unlock();
+
+    return preferred_base;
+}
+
+/* Gets module information of module containing pc, cached from our module list.
+ * Returns false if not in module; none of the OUT arguments are set in that case.
+ *
+ * Note: this function returns only one module name using the rule established
+ * by GET_MODULE_NAME(); for getting all possible ones use
+ * os_get_module_info_all_names() directly.  Part of fix for case 9842.
+ *
+ * If name != NULL, caller must acquire the module_data_lock beforehand
+ * and call os_get_module_info_unlock() when finished with the name
+ * (caller can use dr_strdup to make a copy first if necessary; validity of the
+ * name is guaranteed only as long as the caller holds module_data_lock).
+ * If name == NULL, this routine acquires and releases the lock and the
+ * caller has no obligations.
+ */
+bool
+os_get_module_info(const app_pc pc, OUT uint *checksum, OUT uint *timestamp,
+                   OUT size_t *module_size, OUT const char **name,
+                   size_t *code_size, uint64 *file_version)
+{
+    bool ok = false;
+    module_names_t *names = NULL;
+
+    /* read lock to protect custom data */
+    if (name == NULL)
+        os_get_module_info_lock();
+
+    ASSERT(os_get_module_info_locked());;
+
+    ok = os_get_module_info_all_names(pc, checksum, timestamp, module_size, 
+                                      &names, code_size, file_version);
+    if (name != NULL) {
+        if (ok) {
+            /* os_get_module_info_all_names() may pass and return NULL because
+             * of a bug, guard against it! */
+            ASSERT(names != NULL);
+            if (names != NULL)
+                *name = GET_MODULE_NAME(names);
+        } else
+            *name = NULL;
+    }
+
+    if (name == NULL)
+        os_get_module_info_unlock();
+
+    return ok;
+}
+
+/* Gets module information of module containing pc, cached in our module list.
+ * Returns false if not in module; none of the OUT arguments are set in that case.
+ * Note: this function returns all types of module names as fix for case 9842.
+ */
+/* cf. get_module_info_pe() which should be called on the original PE
+ * to gather this information */
+/* If names != NULL, caller must acquire the module_data_lock beforehand
+ * and call os_get_module_info_unlock() when finished with the names
+ * (caller can use dr_strdup to make a copy first if necessary; validity of the
+ * names are guaranteed only as long as the caller holds module_data_lock).
+ * If names == NULL, this routine acquires and releases the lock and the
+ * caller has no obligations.
+ * NOTE: though there is no use of this routine with names == NULL, this was
+ * don't to make sure that both os_get_module_info*() have a consistent interface.
+ */
+bool
+os_get_module_info_all_names(const app_pc pc, OUT uint *checksum, 
+                             OUT uint *timestamp, OUT size_t *module_size, 
+                             OUT module_names_t **names,
+                             size_t *code_size, uint64 *file_version)
+{
+    /* FIXME: currently just a little safer than looking up in PE itself */
+    module_area_t *ma;
+    bool ok = false;
+
+    /* Lock rank order if holding report_buf_lock */
+    ASSERT(!under_internal_exception());
+
+    if (!is_module_list_initialized())
+        return false;
+
+    /* read lock to protect custom data */
+    if (names == NULL)
+        os_get_module_info_lock();
+
+    ASSERT(os_get_module_info_locked());;
+    ma = module_pc_lookup(pc);
+    if (ma != NULL) {
+        if (checksum != NULL)
+            *checksum = ma->os_data.checksum;
+        if (timestamp != NULL)
+            *timestamp = ma->os_data.timestamp;
+        if (module_size != NULL)
+            *module_size = ma->os_data.module_internal_size; /* pe_size */
+        if (names != NULL)
+            *names = &(ma->names);
+        if (code_size != NULL)
+            *code_size = ma->os_data.code_size;
+        if (file_version != NULL)
+            *file_version = ma->os_data.file_version.version;
+    } else {
+        /* hotpatch DLLs show up here */
+        /* FIXME case 5381: assert these are really only hotpatch dlls */
+    }
+
+    if (names == NULL)
+        os_get_module_info_unlock();
+    else {
+        DODEBUG({
+            /* Try to ensure nobody is calling us prior to adding or after removing */
+            const char *tmp =
+                get_module_short_name_uncached(get_thread_private_dcontext(),
+                                               pc, false/*!at map*/
+                                               HEAPACCT(ACCT_VMAREAS));
+            /* Unfortunately we can't tell a coding error (calling too early/late)
+             * from an app race so we syslog instead of asserting.
+             * An unloaded list (case 6061) would help us out here.
+             * Also, we don't want to complain about modules that need the
+             * filename, so we only check one direction.
+             */
+            if (ma == NULL && tmp != NULL) {
+                SYSLOG_INTERNAL_WARNING_ONCE("os_get_module_info: module list data "
+                                             "mismatch w/ image: DR error, or race");
+            }
+            if (tmp != NULL)
+                dr_strfree(tmp HEAPACCT(ACCT_VMAREAS));
+        });
+    }
+
+    return ok;
+}
+
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+/* Caller must hold module_data_lock */
+rct_module_table_t *
+os_module_get_rct_htable(app_pc pc, rct_type_t which)
+{
+    module_area_t *ma;
+    ASSERT(which >= 0 && which < RCT_NUM_TYPES);
+    ma = module_pc_lookup(pc);
+    if (ma != NULL)
+        return &ma->os_data.rct_table[which];
+    return NULL;
+}
+#endif
+
+bool
+os_module_store_IAT_code(app_pc addr)
+{
+    module_area_t *ma;
+    bool found = false;
+    app_pc IAT_start, IAT_end;
+    os_get_module_info_write_lock();
+    ma = module_pc_lookup(addr);
+    ASSERT_CURIOSITY((ma == NULL || ma->os_data.iat_code == NULL) && "double store");
+    if (ma != NULL && ma->os_data.iat_code == NULL /* no double store */ &&
+        get_IAT_section_bounds(ma->start, &IAT_start, &IAT_end)) {
+        ma->os_data.iat_len = (app_pc) ALIGN_FORWARD(IAT_end, PAGE_SIZE) - IAT_end;
+        ma->os_data.iat_code = (byte *) global_heap_alloc(ma->os_data.iat_len HEAPACCT(ACCT_VMAREAS));
+        memcpy(ma->os_data.iat_code, IAT_end, ma->os_data.iat_len);
+        found = true;
+    } 
+    os_get_module_info_write_unlock();
+    return found;
+}
+
+bool os_module_cmp_IAT_code(app_pc addr)
+{
+    module_area_t *ma;
+    bool match = false;
+    app_pc IAT_start, IAT_end;
+    os_get_module_info_lock();
+    ma = module_pc_lookup(addr);
+    if (ma != NULL && ma->os_data.iat_code != NULL &&
+        get_IAT_section_bounds(ma->start, &IAT_start, &IAT_end)) {
+#ifdef INTERNAL
+        DEBUG_DECLARE(app_pc text_start;)
+        DEBUG_DECLARE(app_pc text_end;)
+#endif
+        size_t IAT_len = ((app_pc)ALIGN_FORWARD(IAT_end, PAGE_SIZE)) - IAT_end;
+        match = (ma->os_data.iat_len == IAT_len &&
+                 memcmp(ma->os_data.iat_code, IAT_end, ma->os_data.iat_len) == 0);
+        LOG(GLOBAL, LOG_VMAREAS, 2,
+            "comparing stored "PFX"-"PFX" with IAT "PFX"-"PFX"\n",
+            ma->os_data.iat_code, ma->os_data.iat_code + ma->os_data.iat_len, IAT_end, IAT_end + IAT_len);
+        /* in all uses so far we always expect to match, except when we mistake
+         * a rebase of a single-page .text section for a rebind (case 10830).
+         * note that we can't use the check here to distinguish those.
+         */
+        ASSERT_CURIOSITY(match ||
+                         (ma->os_data.preferred_base != ma->start &&
+                          is_in_code_section(ma->start, addr,
+                                             &text_start, &text_end) &&
+                          /* IAT and .text occupying same pages is what counts */
+                          PAGE_START(text_start) == PAGE_START(IAT_start) &&
+                          PAGE_START(text_end-1) == PAGE_START(IAT_end-1)));
+    } else /* assert if we have stored code but fail to get iat bounds */
+        ASSERT(ma == NULL || ma->os_data.iat_code == NULL);
+    os_get_module_info_unlock();
+    return match;
+}
+
+static bool
+module_area_free_IAT(module_area_t *ma)
+{
+    ASSERT(os_get_module_info_write_locked());
+    if (ma != NULL && ma->os_data.iat_code != NULL /* no double free */) {
+        DODEBUG({
+            app_pc IAT_start;
+            app_pc IAT_end;
+            get_IAT_section_bounds(ma->start, &IAT_start, &IAT_end);
+            ASSERT(ALIGN_FORWARD(IAT_end, PAGE_SIZE) -
+                   (ptr_uint_t)IAT_end == ma->os_data.iat_len);
+        });
+        global_heap_free(ma->os_data.iat_code, ma->os_data.iat_len HEAPACCT(ACCT_VMAREAS));
+        ma->os_data.iat_code = NULL;
+        ma->os_data.iat_len = 0;
+        return true;
+    }
+    return false;
+}
+
+bool os_module_free_IAT_code(app_pc addr)
+{
+    module_area_t *ma;
+    bool found = false;
+    os_get_module_info_write_lock();
+    ma = module_pc_lookup(addr);
+    if (ma != NULL && ma->os_data.iat_code != NULL) {
+        found = module_area_free_IAT(ma);
+        ASSERT(found);
+    }
+    os_get_module_info_write_unlock();
+    return found;
+}
+
+/* applies relocations to PE (SEC_IMAGE) file.  Assumes all sections
+ * have been made writable, similarly caller is responsible for
+ * restoring any section protection if needed.
+ * 
+ * returns false if some unhandled error condition (e.g. unknown
+ * relocation type), should signal failure to relocate.
+ *
+ * FIXME: need to generalize this as a callback interface to fit
+ * find_relocation_references() which it is mostly copied from.
+ * should start using module_reloc_iterator_start()
+ * Currently used only for ASLR_SHARED_CONTENTS
+ */
+static bool
+module_apply_relocations(app_pc module_base, size_t module_size,
+                         IMAGE_BASE_RELOCATION *base_reloc,
+                         size_t base_reloc_size,
+                         ssize_t relocation_delta)
+{
+    app_pc relocs = NULL;          /* reloc entries */
+    app_pc relocs_end = 0;         /* to iterate through reloc entries */
+    app_pc relocs_block_end = 0;   /* to iterate through reloc blocks */
+    bool is_module_32bit;
+
+    int references_found = -1;
+    DEBUG_DECLARE(
+        uint addresses_fixedup = 0;
+        uint pages_touched = 0;
+        app_pc module_end = module_base + module_size;
+        app_pc original_preferred_base = get_module_preferred_base(module_base);
+    );
+
+    ASSERT(sizeof(uint)   == 4);   /* size of rva and block_size - 32 bits */
+    ASSERT(sizeof(ushort) == 2);   /* size of reloc entry - 16 bits */
+
+    ASSERT(module_base != NULL);   /* caller should have verified base */
+
+    /* callers set up base_reloc and base_reloc_size by calling
+     * get_module_base_reloc */
+    if (base_reloc == NULL || base_reloc_size == 0) {
+        ASSERT(false && "expect relocations");
+        return 0;
+    }
+    is_module_32bit = module_is_32bit(module_base);
+
+    ASSERT(is_readable_without_exception((app_pc)base_reloc, base_reloc_size));
+
+    /* Iterate through relocation entries and check if they contain
+     * references to code section
+     */
+    relocs     = (app_pc) base_reloc;
+    relocs_end = relocs + base_reloc_size;
+
+    while (relocs < relocs_end) {
+        /* Image based relocation is stored as:
+         * DWORD RVA,
+         * DWORD SizeOfBlock
+         * followed by a WORD array relocation entries
+         *   number of relocation entries = 
+         *       (SizeOfBlock - IMAGE_SIZEOF_BASE_RELOCATION) / sizeof(ushort)
+         */
+        uint rva = (uint) ((IMAGE_BASE_RELOCATION *)relocs)->VirtualAddress;
+        uint block_size = (uint) ((IMAGE_BASE_RELOCATION *)relocs)->SizeOfBlock;
+
+        LOG(GLOBAL, LOG_RCT, 6, "\t %8x RVA, %8x SizeofBlock\n",
+            rva, block_size);
+        /* WORD array of relocation entries
+         *   top 4-bits are type
+         *       [IMAGE_REL_BASED_ABSOLUTE | IMAGE_REL_BASED_HIGHLOW]
+         *   remaining 12-bits are offset relative to RVA
+         *
+         * 0:000> dw 77c10000 + 55000
+         * 77c65000  1000 0000 0158 0000 327c 3288 328c 3290 
+         * 77c65010  3294 3298 32a4 32a8 32ac 32b8 32ec 32f0
+         * ...
+         */
+        relocs_block_end = relocs + block_size;
+        relocs = relocs + IMAGE_SIZEOF_BASE_RELOCATION;
+
+        while (relocs < relocs_block_end) {
+            bool unsup_reloc = false;
+            app_pc ref = process_one_relocation(module_base, relocs, rva,
+                                                relocation_delta, 
+                                                true /* apply reloc */, NULL,
+                                                &unsup_reloc, NULL,
+                                                is_module_32bit
+                                                _IF_DEBUG(module_size));
+            if (unsup_reloc)
+                return false;   /* unsupported fixup */
+
+            DODEBUG({
+                /* 
+                 * curiosity: sometimes ref is not within module,
+                 * is this in result of some compiler optimization on pointer arithmetic?  
+                 * or something else we don't get
+                 * 75f80000 7607d000   browseui   (export symbols)       I:\WINDOWS\System32\browseui.dll
+                 * 
+                 * browseui!Ordinal103+0x741:
+                 * 75fa5e0a ff34bd44e9f775   push    dword ptr [75f7e944+edi*4]
+                 */
+
+                /* ref has fixup applied so subract it to get original */
+                app_pc original_ref = ref - relocation_delta;
+                if (original_ref < original_preferred_base || 
+                    original_ref >= original_preferred_base + module_size) { 
+                    LOG(GLOBAL, LOG_RCT, 1, "ref 0x%08 outside module "PFX"-"PFX, 
+                        original_ref, original_preferred_base, 
+                        original_preferred_base + module_size);
+                }
+
+            });
+            DODEBUG(addresses_fixedup++;);
+
+            relocs = relocs + sizeof(ushort);
+        } /* a block of relocation entries */
+
+        /* each block of relocation entries is for a 4k (PAGE_SIZE) page */
+        DODEBUG(pages_touched++;);
+    } /* all relocation entries */
+
+    LOG(GLOBAL, LOG_RCT, 2, "reloc: module_apply_relocations:  "
+        "fixed up %u addresses, touched %u pages\n",
+        addresses_fixedup, pages_touched);
+    
+    return true;
+}
+
+/* iterator over a PE .reloc section */
+/* 
+ * Currently used only for ASLR_SHARED_CONTENTS validation.
+ * FIXME: see module_apply_relocations() and
+ * find_relocation_references() which can take advantage of this.
+ * Note that inner loop may be somewhat slower than a custom iterator but
+ * considering on average there are about 100 relocations per 4K page
+ * the overhead of read, or worse yet write, page-in faults may dwarf it.
+ */
+typedef struct reloc_iterator_t {
+    app_pc relocs;              /* current reloc entry pointer */
+    app_pc relocs_end;          /* end of all reloc entries */
+    app_pc relocs_block_end;    /* end of current reloc blocks */
+    uint rva_page;              /* current page RVA */
+    app_pc module_base;
+
+    /* FIXME: note that we currently assume clients need sequential
+     * iteration in sorted order of requests.  We also assume that the
+     * .reloc entries themselves are in sorted order.  If any of these
+     * is not true - e.g. we want to lookup any random page on-demand
+     * or if a DLL exists with relocations not in order, we'd have to
+     * do a search through the .reloc block entries on backwards
+     * accesses, or would keep in a sorted array pointers to each
+     * reloc block for really fast random access.
+     */
+    DEBUG_DECLARE(app_pc last_addr;) /* helping verify requests are sorted */
+
+    /* FIXME: some of the above types should be improved to allow
+     * pointer arithmetic instead of explicit conversions, e.g. relocs
+     * should be ushort*, yet the first implementation is copy&paste
+     * from find_relocation_references() and for now is better left
+     * matching the well tested code.
+     */
+} reloc_iterator_t;
+
+static void
+module_reloc_iterator_next_block_internal(reloc_iterator_t *ri)
+{
+    while (ri->relocs >= ri->relocs_block_end 
+           && ri->relocs < ri->relocs_end) {
+        /* checking if relocs are really sorted in known DLLs */
+        DEBUG_DECLARE(uint last_rva_page = ri->rva_page;)
+        uint block_size = (uint) ((IMAGE_BASE_RELOCATION *)ri->relocs)->SizeOfBlock;
+        ri->rva_page = (uint) ((IMAGE_BASE_RELOCATION *)ri->relocs)->VirtualAddress;
+
+        ASSERT((ri->rva_page > last_rva_page || 
+                last_rva_page == 0 /* odbcint.dll has an empty .reloc */)
+               && ".reloc RVA blocks not sorted");
+        LOG(GLOBAL, LOG_RCT, 6, "\t %8x RVA, %8x SizeofBlock\n",
+            ri->rva_page, block_size);
+
+        ri->relocs_block_end = ri->relocs + block_size;
+        ri->relocs = ri->relocs + IMAGE_SIZEOF_BASE_RELOCATION;
+
+        ASSERT(ri->relocs <= ri->relocs_block_end);
+        ASSERT(ri->relocs_block_end <= ri->relocs_end);
+        /* the while loop handles the probably never to be seen corner
+         * case when there are no reloc entries, yet there is a block
+         * entry */
+    } 
+}
+
+/* returns false if iterator cannot be started, if there are no
+ * relocations, but even then allows module_reloc_iterator_next() to
+ * be safely called 
+ */
+bool
+module_reloc_iterator_start(reloc_iterator_t *ri /* OUT */,
+                            app_pc module_base, size_t module_size)
+{
+    IMAGE_BASE_RELOCATION *base_reloc;
+    size_t base_reloc_size;
+    ASSERT(sizeof(uint)   == 4);   /* size of rva and block_size - 32 bits */
+    ASSERT(sizeof(ushort) == 2);   /* size of reloc entry - 16 bits */
+
+    ASSERT(module_base != NULL);   /* caller should have verified base */
+
+    base_reloc = get_module_base_reloc(module_base, &base_reloc_size);
+
+    if (base_reloc == NULL || base_reloc_size == 0) {
+        /* we may currently process DLLs that don't have relocations
+         * e.g. xpsp2res.dll
+         */
+        SYSLOG_INTERNAL_WARNING_ONCE("module_reloc_iterator_start: no relocations");
+
+        /* allows one to call module_reloc_iterator_next() even if
+         * there are no relocations */
+        ri->relocs = ri->relocs_end = NULL;
+
+        return false;
+    }
+
+    ASSERT(is_readable_without_exception((app_pc)base_reloc, base_reloc_size));
+
+    ri->module_base = module_base;
+    ri->relocs = (app_pc) base_reloc;
+    ri->relocs_end = ri->relocs + base_reloc_size;
+    ri->rva_page = 0;
+    DODEBUG({ri->last_addr = 0;});
+    ri->relocs_block_end = 0;   /* to iterate through reloc blocks */
+    /* need to set up first block */
+    module_reloc_iterator_next_block_internal(ri);
+    if (ri->relocs >= ri->relocs_end) {
+        /* bad .reloc section */
+        /* odbcint.dll is a good example here */
+        /* BASE RELOCATIONS #2        ; (section #2)
+         * 0 RVA,        8 SizeOfBlock
+         */
+        /* note we can still call module_reloc_iterator_next() */
+        ASSERT_NOT_TESTED();
+        return false;
+    }
+
+    return true;
+}
+
+/* FIXME: if need arises should add an iterator destructor
+ * module_reloc_iterator_stop(reloc_iterator_t *ri)
+ * currently all state is local
+ */
+
+/* returns location for next relocation to be applied
+ * Note it is ok for someone to ask multiple times for the same relocation 
+ * (e.g. if it was beyond a section bound in a previous request).
+ * Yet once an address is skipped we don't go back.
+ * currently IMAGE_REL_BASED_HIGHLOW is the only supported type on x86-32 &
+ * IMAGE_REL_BASED_DIR64 is the only supported type on x86-64
+ */
+/* FIXME: [perf] we could also pass the section
+ * boundary so that we move the iterator in the common
+ * case, and not move only when reaching next section.
+ */
+/* Note this interface currently doesn't provide random access.  All
+ * users are expected to only need sequential access, with a single
+ * read-ahead, and we only we support randomly skipping forward.*/
+static app_pc
+module_reloc_iterator_next(reloc_iterator_t *ri,
+                           app_pc successor_of /* greater or equal */)
+{
+    DEBUG_DECLARE(int skipped = 0;)
+    DEBUG_DECLARE(size_t module_size = get_allocation_size(ri->module_base, NULL);)
+    bool is_module_32bit = module_is_32bit(ri->module_base);
+    /* make sure requests are in increasing order, otherwise we'd have
+     * to be able to start searching from the beginning.  At least
+     * PEcoff v.8 Section 4 prescribes that VAs for sections must be
+     * in ascending order and adjacent.
+     */
+    ASSERT_CURIOSITY(ri->last_addr < successor_of);
+    DODEBUG({
+        ri->last_addr = successor_of;
+    });
+
+
+    /* Image based relocation is stored as:
+     * DWORD RVA,
+     * DWORD SizeOfBlock
+     * followed by a WORD array relocation entries
+     *   number of relocation entries = 
+     *       (SizeOfBlock - IMAGE_SIZEOF_BASE_RELOCATION) / sizeof(ushort)
+     */
+    /* WORD array of relocation entries
+     *   top 4-bits are type
+     *       [IMAGE_REL_BASED_ABSOLUTE | IMAGE_REL_BASED_HIGHLOW]
+     *   remaining 12-bits are offset relative to RVA
+     *
+     * 0:000> dw 77c10000 + 55000
+     * 77c65000  1000 0000 0158 0000 327c 3288 328c 3290 
+     * 77c65010  3294 3298 32a4 32a8 32ac 32b8 32ec 32f0
+     * ...
+     */
+
+    /* all relocation entries */
+    while (ri->relocs < ri->relocs_end) {
+        /* FIXME: [minor perf] we could take this check out of the
+         * innermost loop, but will need another routine to check if
+         * requests are after the end. */
+
+        /* a block of relocation entries */
+        do {
+            /* module_reloc_iterator_next_block_internal() makes sure
+             * we're always within a block 
+             */
+            bool unsup_reloc = false;
+            app_pc cur_addr;
+            app_pc ref = process_one_relocation(ri->module_base, ri->relocs,
+                                                ri->rva_page,
+                                                0 /* not relocating here */,
+                                                false /* don't apply reloc */,
+                                                NULL, &unsup_reloc, &cur_addr,
+                                                is_module_32bit
+                                                _IF_DEBUG(module_size));
+            if (unsup_reloc)
+                return NULL;   /* unsupported fixup */
+
+            if (cur_addr >= successor_of) {
+                /* found an address larger than requested */
+                return cur_addr;
+            }
+            
+            DODEBUG({skipped++;});
+            /* otherwise keep churning */
+
+            /* we don't normally expect anyone to skip a relocation 
+             * unless we're using a short ASLR_PERSISTENT_PARANOID_PREFIX
+             */
+            ASSERT_CURIOSITY(skipped == 1 || 
+                             TEST(ASLR_PERSISTENT_PARANOID_PREFIX, 
+                                  DYNAMO_OPTION(aslr_validation)));
+
+            ri->relocs = ri->relocs + sizeof(ushort);
+        } while (ri->relocs < ri->relocs_block_end); /* page block */
+
+        module_reloc_iterator_next_block_internal(ri);
+    }
+    
+    /* once we are beyond the end - (or if there were no relocations
+     * at all) all future requests will always get NULL.  We don't
+     * need a module_reloc_iterator_hasnext() in this scheme.
+     */
+    return NULL;                /* not found */
+}
+
+bool
+module_make_writable(app_pc module_base, size_t module_size)
+{
+    /* We can't always change the protections for the whole module
+     * with a single call, instead we have to do it image section by
+     * section.  Although, when used for a module that we have just
+     * mapped and nobody has written to, there should be no copy on
+     * write (PAGE_WRITECOPY) to complicate matters.  Only chink is that
+     * this doesn't work for sections with alignments larger than page
+     * that leave reserved memory holes.
+     *
+     * Note we get charged page file usage as soon as we make a page
+     * privately writable, even if we do not write to it to bring in
+     * a private copy.  FIXME: case 8683 Which means this call can in
+     * fact fail in out of commit memory ~= pagefile situations.
+     */
+
+    /* we could copy the section walking code from
+     * add_rct_module() so that we make each section writable.  or
+     * even most efficient in terms of peak page file use we could
+     * mark private only one page at a time
+     */
+    return make_writable(module_base, module_size);
+
+    /* note we don't care about restoring section protection bits, so
+     * we don't save them here.  We can restore them based on section
+     * headers attributes if we need to in
+     * module_restore_permissions().
+     */
+}
+
+bool
+module_restore_permissions(app_pc module_base, size_t module_size)
+{
+    /* FIXME: necessary for a real loader */
+    /* FIXME: need to walk the sections and restore their requested
+     * permissions
+     */
+
+    /* optional: if we are to execute from this mapping, we may want
+     * to do a NtFlushInstructionCache(0,0) just as the loader keeps
+     * doing on x86
+     */
+
+
+    ASSERT_NOT_IMPLEMENTED(false);
+    return false;
+}
+
+/* verifies section characteristics before we have started relocating the file */
+/* returns true if none of the sections have properties precluding
+ * correct use of our mirror file.  Currently only .shared sections
+ * are presumed to be such a problematic example. */
+/* module_base should be of valid module */
+bool
+module_file_relocatable(app_pc module_base)
+{
+    /* .shared sections - do not allow us to produce a different copy
+     * unless we can guarantee that the original DLLs aren't used by
+     * any process.  Otherwise original DLL users may see a different
+     * value in the shared section than the fresh mapping we'll get
+     * from the other sections.  Note that such sections are
+     * non-securable therefore should in general be avoided if a high
+     * privilege process is to use such a DLL.
+     */
+
+    /* note we handle the rare non-readable sections so we don't need
+     * to give up on such 
+     */
+    IMAGE_DOS_HEADER *dos;
+    IMAGE_NT_HEADERS *nt_hdr;
+    IMAGE_SECTION_HEADER *sec;
+    bool relocatable = true;
+    int i;
+
+    ASSERT(is_readable_pe_base(module_base));
+    dos = (IMAGE_DOS_HEADER *) module_base;
+    nt_hdr = (IMAGE_NT_HEADERS *) (((ptr_uint_t)dos) + dos->e_lfanew);
+
+    sec = IMAGE_FIRST_SECTION(nt_hdr);
+    for (i = 0; i < nt_hdr->FileHeader.NumberOfSections; i++, sec++) {
+        if (TEST(IMAGE_SCN_MEM_SHARED, sec->Characteristics)) {
+            relocatable = false;
+        } else {
+            /* FIXME: probably best is to list all section flags that we
+             * understand and assume that with all others we are looking
+             * for trouble
+             */
+            ASSERT_CURIOSITY(!TESTANY(~(IMAGE_SCN_CNT_CODE|
+                                        IMAGE_SCN_CNT_INITIALIZED_DATA|
+                                        IMAGE_SCN_CNT_UNINITIALIZED_DATA|
+                                        IMAGE_SCN_LNK_OTHER|
+                                        IMAGE_SCN_LNK_INFO|
+                                        IMAGE_SCN_LNK_REMOVE|
+                                        IMAGE_SCN_ALIGN_MASK|
+                                        IMAGE_SCN_LNK_NRELOC_OVFL|
+                                        IMAGE_SCN_MEM_DISCARDABLE|
+                                        IMAGE_SCN_MEM_NOT_CACHED|
+                                        IMAGE_SCN_MEM_NOT_PAGED|
+                                        /* IMAGE_SCN_MEM_SHARED known bad */
+                                        IMAGE_SCN_MEM_EXECUTE|
+                                        IMAGE_SCN_MEM_READ|
+                                        IMAGE_SCN_MEM_WRITE), sec->Characteristics) 
+                             && "not seen section characteristic");
+        }
+    }
+    return relocatable;
+}
+
+
+/* returns true if succesful.  Note the module mapping is left
+ * writable on success, and it is up to callers to call
+ * module_restore_permissions() to make unwritable.  
+ * FIXME: may change this interface to require users to guarantee writability.
+ * especially if module_restore_permissions() can get away as stateless
+ */
+bool
+module_rebase(app_pc module_base, size_t module_size,
+              ssize_t relocation_delta /* value will be added to each relocation */)
+{
+    IMAGE_BASE_RELOCATION *base_reloc = NULL;
+    size_t base_reloc_size = 0;
+    bool ok;
+
+    ASSERT(module_base != NULL);
+    ASSERT(module_size != 0);
+    ASSERT_CURIOSITY(relocation_delta != 0);
+    ASSERT(ALIGNED(relocation_delta, PAGE_SIZE));
+    if (!is_readable_pe_base(module_base)) {
+        return false;
+    }
+
+    /* unprotect all sections - even if there are no relocations to apply */ 
+    ok = module_make_writable(module_base, module_size);
+    ASSERT_CURIOSITY(ok && "out of commit space?");
+    if (!ok) {
+        ASSERT_NOT_TESTED();
+        return false;
+    }
+
+    base_reloc = get_module_base_reloc(module_base, &base_reloc_size);
+    LOG(GLOBAL, LOG_RCT, 2,
+        "reloc: add_rct_module: module_base="PFX", "
+        "base_reloc="PFX", base_reloc_size="PFX")\n",
+        module_base, base_reloc, base_reloc_size);
+
+    /* unless a module is IMAGE_FILE_RELOCS_STRIPPED, even when there
+     * are no relocations apparently there usually is a relocation
+     * directory, other than DLLs with .rsrc only like xpsp2res.dll.
+     *
+     * For example:
+     * $ dumpbin /headers ms05-002-ani-41006.lss 
+     *             3000 [       8] RVA [size] of Base Relocation Directory
+     * $ dumpbin /relocations ms05-002-ani-41006.lss 
+     *     BASE RELOCATIONS #3
+     *     0 RVA,        8 SizeOfBlock
+     *     Summary...
+     */
+
+    if (base_reloc != NULL && base_reloc_size > 0) {
+        ok = module_apply_relocations(module_base, module_size,
+                                      base_reloc,
+                                      base_reloc_size,
+                                      relocation_delta);
+        ASSERT(ok);
+        /* note we don't care about requested permissions here, but if
+         * part of a real loader we'd need to restore the original
+         * section permissions.  Or alternatively protect and
+         * unprotect page by page.
+         */
+
+        if (!ok) 
+            return false;
+    } else {
+        /* for example xpsp2res.dll */
+        /* no relocations needed - even better for us */
+
+        /* FIXME: we may want skip this DLL from sharing.  Since it
+         * has no relocations no memory is wasted when it is
+         * 'privately' ASLRed, while we have to pay a heavy cost to
+         * validate.  The only reason to use sharing would be for
+         * interoperability benefits for having it at the same address.
+         */
+    }
+
+    return true;
+}
+
+/* dump PE components that are loaded in memory to a file */
+/* Note that only sections that are loaded in memory will be
+ * dumped to the new file.  
+ * e.g. "Certificate Table entry points to a table of attribute
+ * certificates. These certificates are not loaded into memory as
+ * part of the image" pecoff v6.
+ * We expect all sections to be marked readable.
+ */
+
+/* Note that if new_file exists we would
+ * overwrite only the appropriate portions we have in memory, so
+ * in fact we could first copy the whole file and then call this
+ * routine to overwrite our changes.
+ */
+bool
+module_dump_pe_file(HANDLE new_file,
+                    app_pc module_base, size_t module_size)
+{
+    IMAGE_NT_HEADERS *nt;
+
+    uint64 file_position;
+    DEBUG_DECLARE(uint64 last_written_position;)
+    size_t num_written;
+    bool ok;
+
+    IMAGE_SECTION_HEADER *sec;
+    uint i;
+
+    ASSERT(module_base != NULL);
+    ASSERT(module_size != 0);
+    ASSERT(new_file != INVALID_HANDLE_VALUE);
+
+    if (!is_readable_pe_base(module_base)) {
+        ASSERT_NOT_REACHED();
+        return false;
+    }
+    nt = NT_HEADER(module_base);
+
+    /* SizeOfHeaders The combined size of an MS DOS stub, PE header,
+     * and section headers rounded up to a multiple of
+     * FileAlignment. 
+     */
+    file_position = 0;
+    ok = write_file(new_file, 
+               module_base + 0, nt->OptionalHeader.SizeOfHeaders,
+               &file_position,
+               &num_written);
+    if (!ok || num_written != nt->OptionalHeader.SizeOfHeaders) {
+        ASSERT_NOT_TESTED();
+        /* we don't delete the file here assuming we'll retry to produce */
+        return false;
+    }
+    DODEBUG({last_written_position = file_position + num_written;});
+
+    sec = IMAGE_FIRST_SECTION(nt);
+    for (i = 0; i < nt->FileHeader.NumberOfSections; i++, sec++) {
+        LOG(GLOBAL, LOG_VMAREAS, 4, "\tName = %.*s\n", IMAGE_SIZEOF_SHORT_NAME,
+            sec->Name);
+        LOG(GLOBAL, LOG_VMAREAS, 2, "\tVirtualAddress = 0x%08x\n", sec->VirtualAddress);
+        LOG(GLOBAL, LOG_VMAREAS, 2, "\tPointerToRawData  = 0x%08x\n",
+            sec->PointerToRawData);
+        LOG(GLOBAL, LOG_VMAREAS, 2, "\tSizeOfRawData  = 0x%08x\n", sec->SizeOfRawData);
+        
+        /* comres.dll for an example of an empty physical section
+         *   .data name
+         *      0 size of raw data
+         *      0 file pointer to raw data
+         */
+        if (get_image_section_map_size(sec, nt) == 0) {
+            LOG(GLOBAL, LOG_VMAREAS, 1, "skipping empty physical section %.*s\n", 
+                IMAGE_SIZEOF_SHORT_NAME, sec->Name);
+            /* note that such sections will still get 0-filled
+             * according to their VirtualAddress VirtualSize, as
+             * normally happens whenever SizeOfRawData < VirtualSize (case 5355) */
+            continue;
+        }
+
+        /* we simply emulate what was already calculated in the original file */
+        file_position = get_image_section_file_offs(sec, nt);
+        /* ASSERT the PE specification prescribes that sections are linearly
+         * consecutive, but apparently the header doesn't have to be adjacent.
+         * We don't care anyways since we seek to the correct PointerToRawData. Only
+         * a curioisty since, while not strictly legal, non-adjacent or even overlapping
+         * raw data is allowed, though not yet seen once alignment (xref case 8868)
+         * is taken into account (xref case 5355). */
+        ASSERT_CURIOSITY(last_written_position == file_position || 
+                         i == 0 /* allowing header to be zero padded */ );
+        DODEBUG({
+            /* seen in an HP printer driver HPBAFD32.DLL-a19ef539
+             * produced by 2.25 linker version
+             * file header ends at RVA 400,
+             * but first section starts at RVA 600, with 0s inbetween,
+             */
+            if (last_written_position != file_position) {
+                SYSLOG_INTERNAL_WARNING_ONCE("header or section padded\n");
+            }
+        });
+        /* Yes, strangely, NtWriteFile takes ULONG instead of size_t */
+        IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(get_image_section_map_size(sec, nt))));
+        ok = write_file(new_file, 
+                        module_base + sec->VirtualAddress,
+                        (uint) get_image_section_map_size(sec, nt),
+                        &file_position,
+                        &num_written);
+        if (!ok || num_written != get_image_section_map_size(sec, nt)) {
+            /* we don't delete the file here assuming we'll retry to produce */
+            /* note that with aslr_safe_save the temporary file will in fact get orphaned! */
+            /* we have to count on nodemgr cleaning up */
+            return false;
+        }
+        DODEBUG({last_written_position = file_position + num_written;});
+    }
+    return true;
+}
+
+/* add only the intersection of the two regions to the running MD5 sum */
+static void
+region_intersection_MD5update(struct MD5Context *ctx, 
+                              app_pc region1_start, size_t region1_len,
+                              app_pc region2_start, size_t region2_len)
+{
+    app_pc intersection_start;
+    size_t intersection_len;
+    ASSERT(ctx != NULL);
+    region_intersection(&intersection_start, &intersection_len,
+                        region1_start, region1_len, 
+                        region2_start, region2_len);
+    if (intersection_len != 0) {
+        LOG(GLOBAL, LOG_SYSCALLS, 2,
+            "adding to short hash region "PFX"-"PFX"\n", 
+            intersection_start, intersection_start + intersection_len);
+        MD5Update(ctx, intersection_start, intersection_len);
+    }
+}
+
+/* keeps track of both short and full digests on each region */
+static
+void
+module_calculate_digest_helper(struct MD5Context * md5_full_cxt OPTIONAL, 
+                               struct MD5Context * md5_short_cxt OPTIONAL,
+                               app_pc region_start, size_t region_len,
+                               app_pc start_header, size_t len_header,
+                               app_pc start_footer, size_t len_footer)
+{
+    ASSERT(md5_full_cxt != NULL || md5_short_cxt != NULL);
+    if (md5_full_cxt != NULL) 
+        MD5Update(md5_full_cxt, region_start, region_len);
+    if (md5_short_cxt == NULL)
+        return;
+    if (len_header != 0) {
+        region_intersection_MD5update(md5_short_cxt, 
+                                      region_start, region_len,
+                                      start_header, len_header);
+    }
+    if (len_footer != 0) {
+        region_intersection_MD5update(md5_short_cxt, 
+                                      region_start, region_len,
+                                      start_footer, len_footer);
+    }
+}
+
+/* Verifies that according to section Characteristics it's mapping is expected to be
+ * readable (and if not VirtualProtects to makes it so).  NOTE this only operates on
+ * the mapped portion of the section (via get_image_section_map_size()) which may be
+ * smaller then the virtual size (get_image_section_size()) of the section (in which
+ * case it was zero padded).
+ *
+ * Note this is NOT checking the current protection settings with
+ * is_readable_without_exception(), so the actual current state may well vary.
+ *
+ * returns false if an unreadable section has been made readable
+ */
+static bool
+ensure_section_readable(app_pc module_base, IMAGE_SECTION_HEADER *sec, 
+                        IMAGE_NT_HEADERS *nt, OUT uint *old_prot,
+                        app_pc view_start, size_t view_len)
+{
+    int ok;
+    app_pc intersection_start;
+    size_t intersection_len;
+    VERIFY_NT_HEADER(module_base);
+
+    region_intersection(&intersection_start, &intersection_len,
+                        view_start, view_len, 
+                        module_base + sec->VirtualAddress, 
+                        ALIGN_FORWARD(get_image_section_map_size(sec, nt), PAGE_SIZE));
+    if (intersection_len == 0)
+        return true;
+
+    /* on X86-32 as long as any of RWX is set the contents is readable */
+    if (TESTANY(IMAGE_SCN_MEM_EXECUTE|IMAGE_SCN_MEM_READ|IMAGE_SCN_MEM_WRITE,
+                sec->Characteristics)) {
+        ASSERT(is_readable_without_exception(intersection_start, intersection_len));
+        return true;
+    }
+    /* such a mapping could potentially be used for some protection
+     * scheme in which sections are made readable only on demand */
+
+    /* Otherwise we just mark readable the raw bytes of the section,
+     * NOTE: we'll leave readable, so only users of our private
+     * mappings should use this function!
+     */
+    SYSLOG_INTERNAL_WARNING("unreadable section %.*s\n", 
+                            IMAGE_SIZEOF_SHORT_NAME, sec->Name);
+    ok = protect_virtual_memory(intersection_start, intersection_len,
+                                PAGE_READONLY, 
+                                old_prot);
+    ASSERT(ok);
+    ASSERT_CURIOSITY(*old_prot == PAGE_NOACCESS || 
+                     *old_prot == PAGE_WRITECOPY); /* expecting unmodifed even if writable */
+    return false;
+}
+
+static bool
+restore_unreadable_section(app_pc module_base, IMAGE_SECTION_HEADER *sec, 
+                           IMAGE_NT_HEADERS *nt, uint restore_prot,
+                           app_pc view_start, size_t view_len)
+{
+    int ok;
+    app_pc intersection_start;
+    size_t intersection_len;
+    uint old_prot;
+    VERIFY_NT_HEADER(module_base);
+
+    ASSERT(!TESTANY(IMAGE_SCN_MEM_EXECUTE|IMAGE_SCN_MEM_READ|IMAGE_SCN_MEM_WRITE,
+                    sec->Characteristics));
+
+    region_intersection(&intersection_start, &intersection_len,
+                        view_start, view_len, 
+                        module_base + sec->VirtualAddress, 
+                        ALIGN_FORWARD(get_image_section_map_size(sec, nt), PAGE_SIZE));
+    if (intersection_len == 0)
+        return true;
+
+    ok = protect_virtual_memory(intersection_start, intersection_len,
+                                restore_prot, 
+                                &old_prot);
+    ASSERT(ok);
+    ASSERT(old_prot == PAGE_READONLY);
+
+    return ok;
+}
+
+/* note it operates on a PE mapping so it can be passed either a
+ * relocated or the original file.  Either full or short digest or both can be requested.
+ * If short_digest is set the short version of the digest is
+ * calculated and set.  Note that if short_digest_size crosses an
+ * unreadable boundary it is truncated to the smallest consecutive
+ * memory region from each of the header and the footer.  If
+ * short_digest_size is 0 or larger than half of the file size the
+ * short and full digests are supposed to be equal.
+ * If sec_characteristics != 0, only sections TESTANY matching those
+ * characteristics (and the PE headers) are considered.
+ * It is the caller's responsibility to ensure that module_size is not
+ * larger than the mapped view size.
+ */
+void
+module_calculate_digest(OUT module_digest_t *digest,
+                        app_pc module_base, 
+                        size_t module_size,
+                        bool full_digest,
+                        bool short_digest,
+                        uint short_digest_size,
+                        uint sec_characteristics)
+{
+    struct MD5Context md5_short_cxt;
+    struct MD5Context md5_full_cxt;
+
+    IMAGE_NT_HEADERS *nt;
+    IMAGE_SECTION_HEADER *sec;
+    uint i;
+    app_pc module_end = module_base + module_size;
+
+    /* tentative starts */
+    /* need to adjust these buffers in case of crossing unreadable areas,
+     * or if overlapping
+     */
+    /* Note that simpler alternative would have been to only produce a
+     * digest on the PE header (0x400), and maybe the last section.
+     * However for better consistency guarantees, yet with a
+     * predictable performance, used this more involved definition of
+     * short digest.  While a 64KB digest may be acceptable, full
+     * checks on some 8MB DLLs may be noticable.
+     */
+
+    app_pc header_start = module_base;
+    size_t header_len = MIN(module_size, short_digest_size);
+    app_pc footer_start = module_end - short_digest_size;
+    size_t footer_len;
+
+    app_pc region_start;
+    size_t region_len;
+
+    ASSERT(digest != NULL);
+    ASSERT(module_base != NULL);
+    ASSERT(module_size != 0);
+
+    LOG(GLOBAL, LOG_SYSCALLS, 2,
+        "module_calculate_digest: module "PFX"-"PFX"\n", 
+        module_base, module_base + module_size);
+
+    if (short_digest_size == 0) {
+        header_len = module_size;
+    }
+
+    footer_start = MAX(footer_start, header_start + header_len);
+    footer_len = module_end - footer_start;
+    /* footer region will be unused if footer_len is 0 - in case of
+     * larger than file size short size, or if short_digest_size = 0
+     * which also means unbounded */
+
+    /* note that this function has significant overlap with
+     * module_dump_pe_file(), and in fact we could avoid a second
+     * traversal and associated cache pollution on producing a file if
+     * we provide this functionality in module_dump_pe_file().  Of
+     * course for verification we still need this separately 
+     */
+
+    if (!is_readable_pe_base(module_base)) {
+        ASSERT_NOT_REACHED();
+        return;
+    }
+    nt = NT_HEADER(module_base);
+
+    if (short_digest)
+        MD5Init(&md5_short_cxt);
+    if (full_digest) 
+        MD5Init(&md5_full_cxt);
+
+    /* first region to consider is module header */
+    region_start = module_base + 0;
+    region_len = nt->OptionalHeader.SizeOfHeaders;
+
+    /* FIXME: note that if we want to provide/match an Authenticode
+     * hash we'd have to skip the Checksum field in the header - see
+     * pecoff_v8 */
+
+    /* at each step intersect with the possible short regions */
+    module_calculate_digest_helper(full_digest ? &md5_full_cxt : NULL,
+                                   short_digest ? &md5_short_cxt : NULL,
+                                   region_start, region_len,
+                                   header_start, header_len,
+                                   footer_start, footer_len);
+
+    sec = IMAGE_FIRST_SECTION(nt);
+    for (i = 0; i < nt->FileHeader.NumberOfSections; i++, sec++) {
+        uint old_section_prot;
+        bool readable;
+        LOG(GLOBAL, LOG_VMAREAS, 4, "\tName = %.*s\n", IMAGE_SIZEOF_SHORT_NAME,
+            sec->Name);
+        LOG(GLOBAL, LOG_VMAREAS, 2, "\tVirtualAddress = 0x%08x\n", sec->VirtualAddress);
+        LOG(GLOBAL, LOG_VMAREAS, 2, "\tPointerToRawData  = 0x%08x\n",
+            sec->PointerToRawData);
+        LOG(GLOBAL, LOG_VMAREAS, 2, "\tSizeOfRawData  = 0x%08x\n", sec->SizeOfRawData);
+        
+        /* comres.dll for an example of an empty physical section
+         *   .data name
+         *      0 size of raw data
+         *      0 file pointer to raw data
+         */
+        if (get_image_section_map_size(sec, nt) == 0) {
+            LOG(GLOBAL, LOG_VMAREAS, 1, "skipping empty physical section %.*s\n", 
+                IMAGE_SIZEOF_SHORT_NAME, sec->Name);
+            /* note that such sections will still get 0-filled 
+             * but we only look at raw bytes */
+            continue;
+        }
+        if (!TESTANY(sec_characteristics, sec->Characteristics)) {
+            LOG(GLOBAL, LOG_VMAREAS, 2, "skipping non-matching section %.*s\n", 
+                IMAGE_SIZEOF_SHORT_NAME, sec->Name);
+            continue;
+        }
+
+        /* make sure region is readable. Alternatively, we could just
+         * ignore unreadable (according to characteristics) portions
+         */
+        readable = ensure_section_readable(module_base, sec, nt, &old_section_prot,
+                                           module_base, module_size);
+
+        region_start = module_base + sec->VirtualAddress;
+        region_len = get_image_section_map_size(sec, nt);
+
+        module_calculate_digest_helper(full_digest ? &md5_full_cxt : NULL,
+                                       short_digest ? &md5_short_cxt : NULL,
+                                       region_start, region_len,
+                                       header_start, header_len,
+                                       footer_start, footer_len);                                   
+        if (!readable) {
+            bool ok = restore_unreadable_section(module_base, sec, nt, old_section_prot,
+                                                 module_base, module_size);
+            ASSERT(ok);
+        }
+    }
+
+    if (short_digest) 
+        MD5Final(digest->short_MD5, &md5_short_cxt);
+    if (full_digest) 
+        MD5Final(digest->full_MD5, &md5_full_cxt);
+
+    DODEBUG({
+        if (full_digest && short_digest && 
+            (short_digest_size == 0 ||
+             short_digest_size * 2 > module_size)) {
+            ASSERT(md5_digests_equal(digest->short_MD5, digest->full_MD5));
+        }
+    });
+
+    /* FIXME: Note that if we did want to have an md5sum-matching
+     * digest we'd have to append the module bytes with the extra
+     * bytes that are only present on disk in our digest.  Since
+     * usually quite small that could be handled by a read_file()
+     * instead of remapping the whole file as MEM_MAPPED. It would be
+     * applicable only if we have the appropriate file handle of
+     * course. */
+}
+
+/* We don't want the compiler to use a byte comparison (rep cmpsb)
+ * when the library version of memcmp can do a faster word comparison
+ * as much as possible (rep cmpsd).  We'll end up using the
+ * ntdll!memcmp, which is the same as msvcrt!memcmp,
+ * ntdll!RtlCompareMemory.  case 9277 tracks exploring faster variants
+ * Note the opposite of this is #pragma intrinsic(memcmp) which would
+ * always force memcmp intrinsic.
+ */
+/* default release build behaviour of VC98 in the following uses
+ * matches #pragma intrinsic(memcmp) */
+/* FIXME: should experiment whether reps cmpsb or reps cmpsd is
+ * faster if we are comparing shorter intervals between relocations
+ */
+#pragma function(memcmp) /* do not use memcmp intrinsic */
+
+/* verbatim comparison of a region, simple wrapper around memcmp()
+ * but with somewhat more convenient range expression
+ */
+static bool
+module_region_compare(app_pc original_module_start,
+                      app_pc original_module_end, /* not inclusive */
+                      ssize_t suspect_module_mapped_delta
+                      /* N.B. difference in mapped memory addresses
+                       * _not_ preferred addresses */
+                      )
+{
+    ASSERT(original_module_start <= original_module_end);
+    /* empty region always matches */
+    ASSERT(suspect_module_mapped_delta != 0);
+    return memcmp(original_module_start, 
+                  original_module_start + suspect_module_mapped_delta,
+                  original_module_end - original_module_start) == 0;
+}
+
+/* compare consecutive readable bytes mapped as a PE section */
+static inline bool
+module_pe_section_compare(app_pc original_module_section,
+                          app_pc suspect_module_section,
+                          size_t matching_section_size,
+                          bool relocated,
+                          /* preferred base delta of suspect */
+                          ssize_t relocation_delta, /* used only if !relocated */
+                          reloc_iterator_t *ri /* used only if !relocated */)
+{
+    LOG(GLOBAL, LOG_VMAREAS, 2, "module_pe_section_compare = %d bytes\n", 
+        matching_section_size);
+
+    if (relocated) {
+        /* as long as we check each section description from original
+         * matching each section from suspect - base size and
+         * characteristics we don't need to do explicit
+         * is_readable_without_exception() or a real TRY block
+         */
+        return memcmp(original_module_section, suspect_module_section, 
+                      matching_section_size) == 0;
+    } else {
+        /* if matching_section_size == 0 we'd always return true here.
+         * the caller we would in fact be only comparing the PE header.
+         */
+        
+        /* note that sections don't have to be page aligned! */
+        /* Note we'll assume sections ARE sorted */
+        /* otherwise we'll have to be able to search for relocation entry
+         * starting from the beginning
+         */
+        
+        /* find RVA of next relocation entry
+         * compare from last entry to the new one (or the end of the region)
+         * match the relocated RVA contents if within the section
+         */
+        app_pc verbatim_start = original_module_section;
+        ssize_t mapped_delta = suspect_module_section - original_module_section;
+
+        do {
+            /* note that iterator doesn't really consume an item until
+             * a larger one is asked for */
+            app_pc next_reloc_original = module_reloc_iterator_next(ri, verbatim_start);
+            app_pc next_reloc_suspect;
+
+            if (next_reloc_original > original_module_section + matching_section_size
+                || next_reloc_original == NULL /* beyond last relocation */) {
+                /* set limit around whole region */
+                next_reloc_original = original_module_section + matching_section_size;
+                /* there is no real relocation entry to compare */
+                next_reloc_suspect = NULL;
+            } else {
+                next_reloc_suspect = next_reloc_original + mapped_delta;
+            }
+
+            if (!module_region_compare(verbatim_start, 
+                                       next_reloc_original, /* not inclusive */
+                                       mapped_delta)) {
+                SYSLOG_INTERNAL_WARNING("mismatch in verbatim region ["PFX"-"PFX")", 
+                                        verbatim_start, next_reloc_original);
+                return false;
+            }
+
+            if (next_reloc_suspect != NULL &&
+                /* full pointer value at next_reloc_original is relocated */
+                (*(app_pc*)(next_reloc_original) + relocation_delta != 
+                 *(app_pc*)(next_reloc_suspect))) {
+                SYSLOG_INTERNAL_WARNING("mismatch at relocated entry "PFX" = "PFX, 
+                                        next_reloc_original, *(app_pc*)next_reloc_original);
+                return false;
+            }
+            verbatim_start = next_reloc_original + sizeof(uint);
+            /* FIXME: note that a very sneaky .reloc section may have
+             * overlapping relocation entries.  Other than malicious
+             * DLLs trying to thwart us, (we won't be able to match
+             * that), there is no reason to expect anyone else would
+             * have such a DLL.  Even if increment by 1 since we
+             * wouldn't have applied the previous relocation, we'd
+             * still fail.  We'll simply ASSERT if we skip any
+             * entries.
+             */
+
+        } while (verbatim_start < 
+                 original_module_section + matching_section_size);
+
+        return true;
+    }
+}
+
+/* compare PE header, verbatim with the exception of the fields which
+ * may be modified if produced by aslr_generate_relocated_section(),
+ * FileHeader.TimeDateStamp, OptionalHeader.ImageBase, and
+ * OptionalHeader.CheckSum
+ */
+bool
+aslr_compare_header(app_pc original_module_base, size_t original_header_len,
+                    app_pc suspect_module_base)
+{
+    IMAGE_DOS_HEADER *dos;
+    IMAGE_NT_HEADERS *nt_hdr;
+
+    uint old_checksum;
+    uint old_timestamp;
+
+    uint new_checksum;
+    uint new_timestamp;
+
+    bool ok;
+
+    /* FIXME: [perf] note that get_module_info_pe()'s and our similar
+     * calls to is_readable_pe_base() are heavy weight. should use TRY
+     * here */
+
+    ASSERT(is_readable_pe_base(original_module_base));
+    if (!is_readable_pe_base(original_module_base)) {
+        return false;
+    }
+
+    dos = (IMAGE_DOS_HEADER *) original_module_base;
+    nt_hdr = (IMAGE_NT_HEADERS *) (((ptr_uint_t)dos) + dos->e_lfanew);
+
+    old_timestamp = nt_hdr->FileHeader.TimeDateStamp;
+    old_checksum = nt_hdr->OptionalHeader.CheckSum;
+
+    ok = get_module_info_pe(suspect_module_base, &new_checksum,
+                            &new_timestamp, NULL, NULL, NULL);
+    ASSERT(ok);
+    if (!ok) {
+        return false;
+    }
+
+    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 2, 
+        "ASLR: aslr_compare_header checksum old "PFX", new "PFX"\n",
+        old_checksum, new_checksum);
+    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 2, 
+        "ASLR: aslr_compare_header TimeDateStamp old "PFX", new "PFX"\n",
+        old_timestamp, new_timestamp);
+
+    /* aslr_generate_relocated_section() adjusts timestamp */
+    if (new_timestamp != aslr_timestamp_transformation(old_timestamp)) {
+        /* this in most cases will be due to benign update of source, */
+        /* but could also indicate (malicious) modification of target */
+        return false;
+    }
+
+    /* Note CheckSum is currently left identical to original, but in
+     * the future we may have to set it to the correct checksum for
+     * the current DLL
+     */
+    if (new_checksum != old_checksum) {
+        ASSERT_CURIOSITY(false && "checksum tampering!");
+        return false;
+    }
+        
+    /* instead of aslr_write_header() we're supposed to compare here in place */
+    /* we need to verify the order of these three variables */
+    /* and then do memcmp() of the four regions that it splits the address space into */
+    /* FileHeader.TimeDateStamp < OptionalHeader.ImageBase < OptionalHeader.CheckSum */
+    ASSERT((byte *)&nt_hdr->FileHeader.TimeDateStamp <
+           (byte *)OPT_HDR_P(nt_hdr, ImageBase));
+    ASSERT((byte *)OPT_HDR_P(nt_hdr, ImageBase) <
+           (byte *)&nt_hdr->OptionalHeader.CheckSum);
+
+    if (original_header_len < (ptr_uint_t)
+        (((app_pc)&nt_hdr->OptionalHeader.CheckSum) - original_module_base)) {
+        ASSERT_NOT_TESTED();
+        ASSERT_CURIOSITY(false && "bad DOS header?");
+        return false;
+    }
+
+    ok = ok && module_region_compare(original_module_base,
+                                     (app_pc)&nt_hdr->FileHeader.TimeDateStamp,
+                                     suspect_module_base - original_module_base);
+    ASSERT_CURIOSITY(ok && "header tampered with");
+
+    ok = ok && module_region_compare(((app_pc)&nt_hdr->FileHeader.TimeDateStamp) + 
+                                     sizeof(nt_hdr->FileHeader.TimeDateStamp),
+                                     (app_pc)OPT_HDR_P(nt_hdr, ImageBase),
+                                     suspect_module_base - original_module_base);
+    ASSERT_CURIOSITY(ok && "header tampered with");
+
+    ok = ok && module_region_compare(((app_pc)OPT_HDR_P(nt_hdr, ImageBase)) + 
+                                     sizeof(OPT_HDR(nt_hdr, ImageBase)),
+                                     (app_pc)&nt_hdr->OptionalHeader.CheckSum, 
+                                     suspect_module_base - original_module_base);
+    ASSERT_CURIOSITY(ok && "header tampered with");
+
+    ok = ok && module_region_compare(((app_pc)&nt_hdr->OptionalHeader.CheckSum) + 
+                                     sizeof(nt_hdr->OptionalHeader.CheckSum),
+                                     original_module_base + original_header_len,
+                                     suspect_module_base - original_module_base);
+    ASSERT_CURIOSITY(ok && "header tampered with");
+    return ok;
+}
+
+/* Compares the PE .sections of a Mapping of a Section of a file_t
+ * note that original_module_base is presumed to be more trustworthy,
+ * though best to be very careful with both
+ *
+ * if !relocated transparently applies relocations
+ * without mappings breaking COW with private copies
+ * 
+ * validation_section_prefix controls maximum per-section comparison
+ * note that PE header is always compared in full.
+ */
+bool
+module_contents_compare(app_pc original_module_base,
+                        app_pc suspect_module_base,
+                        size_t matching_module_size,
+                        bool relocated, 
+                        ssize_t relocation_delta 
+                        /* preferred base delta of suspect */,
+                        size_t validation_section_prefix)
+{
+    /* FIXME: probably a good time to provide an iterator 
+     * that optionally returns the PE header as well
+     * and of course avoids any holes 
+     * just like module_calculate_digest() and module_dump_pe_file()
+     */
+
+    IMAGE_NT_HEADERS *nt_original;
+    IMAGE_NT_HEADERS *nt_suspect;
+    IMAGE_SECTION_HEADER *sec_original;
+    IMAGE_SECTION_HEADER *sec_suspect;
+    uint i;
+
+    size_t region_offset;
+    size_t region_len;
+    size_t suspect_len;
+
+    uint original_section_prot = 0;
+    uint suspect_section_prot = 0;
+
+    reloc_iterator_t reloc_iter;
+    reloc_iterator_t *ri = &reloc_iter;
+
+    ASSERT(original_module_base != NULL);
+    ASSERT(suspect_module_base != NULL);
+    ASSERT(matching_module_size != 0);
+
+    if (!is_readable_pe_base(original_module_base)) {
+        ASSERT_NOT_REACHED();
+        return false;
+    }
+    /* all section headers should be readable now */
+    nt_original = NT_HEADER(suspect_module_base);
+
+    if (!is_readable_pe_base(suspect_module_base)) {
+        ASSERT_CURIOSITY(false && "bad suspect PE header!");
+        return false;
+    }
+    /* all suspect section headers should be readable */
+    nt_suspect = NT_HEADER(suspect_module_base);
+
+    /* first region to consider is module header */
+    region_offset = 0;
+    region_len = nt_original->OptionalHeader.SizeOfHeaders;
+    suspect_len = nt_suspect->OptionalHeader.SizeOfHeaders;
+
+    if (region_len != suspect_len) {
+        ASSERT_CURIOSITY(false && "different header size!");
+        return false;
+    }
+
+    if (!relocated) {
+        /* header comparison has to match our header modifications in
+         * aslr_generate_relocated_section() */
+        if (!aslr_compare_header(original_module_base, region_len,
+                                 suspect_module_base)) {
+            /* commonly just a new version */
+            ASSERT_CURIOSITY(false && "mismatched PE header, new version?");
+            return false;
+        }
+    } else if (!module_pe_section_compare(original_module_base + region_offset, 
+                                          suspect_module_base + region_offset, 
+                                          region_len, relocated, relocation_delta,
+                                          NULL /* no iterator */)) {
+        ASSERT(relocated);
+        /* commonly just a new version */
+        ASSERT_CURIOSITY(false && "mismatched PE header, new version?");
+        return false;
+    }
+
+    if (nt_original->FileHeader.NumberOfSections !=
+        nt_suspect->FileHeader.NumberOfSections) {
+        ASSERT_CURIOSITY(false && "not matching number of sections!");
+        return false;
+    }
+
+    if (!relocated) {
+        module_reloc_iterator_start(&reloc_iter, original_module_base,
+                                    matching_module_size);
+        ASSERT(ri == &reloc_iter);
+    } else {
+        ri = NULL;
+    }
+    
+    sec_original = IMAGE_FIRST_SECTION(nt_original);
+    sec_suspect = IMAGE_FIRST_SECTION(nt_suspect);
+    for (i = 0; i < nt_original->FileHeader.NumberOfSections; 
+         i++, sec_original++, sec_suspect++) {
+        bool readable;
+        LOG(GLOBAL, LOG_VMAREAS, 4, "\tName = %.*s\n", IMAGE_SIZEOF_SHORT_NAME,
+            sec_original->Name);
+        LOG(GLOBAL, LOG_VMAREAS, 2, "\tVirtualAddress = 0x%08x\n",
+            sec_original->VirtualAddress);
+        LOG(GLOBAL, LOG_VMAREAS, 2, "\tPointerToRawData  = 0x%08x\n",
+            sec_original->PointerToRawData);
+        LOG(GLOBAL, LOG_VMAREAS, 2, "\tSizeOfRawData  = 0x%08x\n",
+            sec_original->SizeOfRawData);
+
+        if (get_image_section_map_size(sec_original, nt_original) == 0) {
+            if (get_image_section_map_size(sec_suspect, nt_suspect) != 0) {
+                ASSERT_CURIOSITY(false && "not matching empty section!");
+                return false;
+            }
+            LOG(GLOBAL, LOG_VMAREAS, 1, "skipping empty physical section %.*s\n", 
+                IMAGE_SIZEOF_SHORT_NAME, sec_original->Name);
+            /* note that such sections will still get 0-filled 
+             * but we only look at raw bytes */
+            continue;
+        }
+
+        ASSERT(sec_original->VirtualAddress ==
+               sec_suspect->VirtualAddress);
+
+        /* should be checked already in the header, but doublechecking */
+        if ((sec_original->Characteristics != 
+             sec_suspect->Characteristics) ||
+            (sec_original->VirtualAddress != 
+             sec_suspect->VirtualAddress)
+            ) {
+            ASSERT(false && "mismatched PE section characteristics");
+            /* DoS deflected, if section not readable or privilege
+             * escalation if made shared to allow attacker to change
+             * .data section!
+             */
+            return false;
+        }
+
+        region_offset = sec_original->VirtualAddress;
+        region_len = get_image_section_map_size(sec_original, nt_original);
+        suspect_len = get_image_section_map_size(sec_suspect, nt_suspect);
+
+        readable = ensure_section_readable(original_module_base, sec_original,
+                                           nt_original, &original_section_prot,
+                                           /* FIXME case 9791: must pass view size! */
+                                           original_module_base + region_offset,
+                                           region_len);
+        if (!readable) {
+            bool also_unreadable = 
+                !ensure_section_readable(suspect_module_base, sec_suspect,
+                                         nt_suspect, &suspect_section_prot,
+                                         /* FIXME case 9791: must pass view size! */
+                                         suspect_module_base + region_offset,
+                                         suspect_len);
+            ASSERT(also_unreadable);
+        }
+
+        DODEBUG({
+            if (region_len > validation_section_prefix) {
+                SYSLOG_INTERNAL_WARNING_ONCE("comparing section prefix %d"
+                                             " instead of full %d\n", 
+                                             validation_section_prefix, region_len);
+            }
+        });
+
+        if (region_len != suspect_len
+            || !module_pe_section_compare(original_module_base + region_offset, 
+                                          suspect_module_base + region_offset, 
+                                          MIN(region_len, validation_section_prefix), 
+                                          relocated, relocation_delta, ri)) {
+            SYSLOG_INTERNAL_ERROR("mismatched PE section %.*s\n", 
+                                  IMAGE_SIZEOF_SHORT_NAME, sec_original->Name);
+            /* we also want ldump */
+            ASSERT_CURIOSITY(false && "mismatched PE section!");
+            return false;
+        }
+
+        if (!readable) {
+            /* both should not be */
+            bool ok;
+            ok = restore_unreadable_section(original_module_base, sec_original,
+                                            nt_original, original_section_prot,
+                                            /* FIXME case 9791: must pass view size! */
+                                            original_module_base + region_offset,
+                                            region_len);
+            ok = restore_unreadable_section(suspect_module_base, sec_suspect,
+                                            nt_suspect, suspect_section_prot,
+                                            /* FIXME case 9791: must pass view size! */
+                                            suspect_module_base + region_offset,
+                                            suspect_len);
+            ASSERT(ok);
+        }
+    }
+
+    return true;
+}
+
+/***** Resource Directory routines ****
+ *
+ * See the various PE format guides for more details, types below are 
+ * defined in winnt.h with constants in winnt.h and winuser.h.  See also the
+ * wine implementations in dlls/version/info.c etc.
+ *
+ * At the high level the resource directory is structured as a tree of 
+ * IMAGE_RESOURCE_DIRECTORY elements.  Each resource directory is followed
+ * by an array of IMAGE_RESOURCE_DIRECTORY_ENTRY structs.  Each of which
+ * either points to an IMAGE_RESOURCE_DIRECTORY subdir or an 
+ * IMAGE_RESOURCE_DATA_ENTRY leaf that points to the actual data.  The tree
+ * is structured in layers with the top layer being the resource type, the
+ * second layer being the resource id/name, the third layer being the
+ * resource language id, and the fourth layer being the data entry leaf.
+ */
+
+/* FIXME - like many other module.c routines these are racy w/ respect to an
+ * unmap. */
+
+/* Checks if [read_start, read_start+read_size) is within
+ * [valid_start, valid_size). NOTE evaluates args multiple times. */
+#define CHECK_SAFE_READ(read_start, read_size, valid_start, valid_size) \
+  ((ptr_uint_t)(read_start)+(ptr_uint_t)(read_size) >=      \
+   (ptr_uint_t)(read_start) && /* overflow */               \
+   (ptr_uint_t)(valid_start)+(ptr_uint_t)(valid_size) >=    \
+   (ptr_uint_t)(valid_start) && /* overflow */              \
+   (ptr_uint_t)(read_start) >= (ptr_uint_t)(valid_start) && \
+   (ptr_uint_t)(read_start)+(ptr_uint_t)(read_size) <=      \
+   (ptr_uint_t)(valid_start)+(ptr_uint_t)(valid_size))
+
+/* Gets the address of the memory following the struct at ptr. Note this is not
+ * a deref, only an address computation. */
+#define GET_FOLLOWING_ADDRESS(ptr) (&(ptr)[1])
+
+static inline IMAGE_RESOURCE_DIRECTORY_ENTRY *
+get_resource_directory_entries(IMAGE_RESOURCE_DIRECTORY *dir)
+{
+    return (IMAGE_RESOURCE_DIRECTORY_ENTRY *) GET_FOLLOWING_ADDRESS(dir);
+}
+
+static IMAGE_RESOURCE_DIRECTORY *
+get_module_resource_directory(app_pc mod_base, size_t *rsrc_size /* OUT */)
+{
+    IMAGE_NT_HEADERS *nt = NULL;
+    IMAGE_DATA_DIRECTORY *resource_dir = NULL;
+
+    /* callers should have already done this */
+    ASSERT(is_readable_pe_base(mod_base));
+    VERIFY_NT_HEADER(mod_base);
+
+    nt = NT_HEADER(mod_base);
+
+    resource_dir = (IMAGE_DATA_DIRECTORY *)
+        (OPT_HDR(nt, DataDirectory) + IMAGE_DIRECTORY_ENTRY_RESOURCE);
+    /* sanity check */
+    if (!is_readable_without_exception((app_pc)resource_dir,
+                                       sizeof(*resource_dir))) {
+        ASSERT_CURIOSITY(false && ".rsrc section directory not readable");
+        return NULL;
+    }
+
+    ASSERT_CURIOSITY((resource_dir->VirtualAddress == 0 &&
+                      resource_dir->Size == 0) ||
+                     (resource_dir->VirtualAddress != 0 &&
+                      resource_dir->Size != 0));
+
+    if (resource_dir->VirtualAddress != 0 && resource_dir->Size != 0) {
+        IMAGE_RESOURCE_DIRECTORY *out_dir = (IMAGE_RESOURCE_DIRECTORY *)
+            RVA_TO_VA(mod_base, resource_dir->VirtualAddress);
+        /* xref case 8740 - we've seen at least once case where the size of the .rsrc
+         * section was reported wrong in the pe header (was smaller 0x800 then the actual
+         * section size 0x834).  Since at this level we only care about the size for
+         * avoiding future ptr safety checks (by checking up front that the whole region
+         * is readable), we'll push the size up to the next memory region boundary. */
+        if (is_readable_without_exception((app_pc)out_dir, resource_dir->Size)) {
+            if (rsrc_size != NULL) {
+                byte *base;
+                size_t size;
+                uint prot;
+                bool res = get_memory_info(((byte *)out_dir) + (resource_dir->Size - 1),
+                                           &base, &size, &prot);
+                ASSERT(res && base + size >= ((byte *)out_dir) + resource_dir->Size);
+                *rsrc_size = (base + size) - (byte *)out_dir;
+            }
+            return out_dir;
+        } else {
+            ASSERT_CURIOSITY(false && "resource directory not readable" ||
+                             /* for partial map, .rsrc dir was't mapped in */
+                             EXEMPT_TEST("partial_map.exe"));
+        }
+    } else {
+        ASSERT_CURIOSITY(resource_dir->VirtualAddress == 0 &&
+                         resource_dir->Size == 0);
+    }
+    return NULL;
+}
+
+static IMAGE_RESOURCE_DIRECTORY_ENTRY *
+get_resource_directory_entry_by_id(IMAGE_RESOURCE_DIRECTORY *dir,
+                                   uint id, byte *valid_start, size_t valid_size)
+{
+    int i, j;
+    /* the resource directory entries immediately follow the directory */
+    IMAGE_RESOURCE_DIRECTORY_ENTRY *entries = get_resource_directory_entries(dir);
+
+    if (dir == NULL)
+        return NULL;
+    if (!CHECK_SAFE_READ(dir, sizeof(*dir), valid_start, valid_size)) {
+        ASSERT_CURIOSITY(false && "unreadable resource directory");
+        return NULL;
+    }
+    
+    /* named entries are first and we're looking for a numbered entry */
+    j = dir->NumberOfNamedEntries;
+
+    /* FIXME entries are in order so we could binary search this, but we
+     * only look for small entries (16 for VS_FILE_INFO & 1 for VS_VERSION_INFO) */
+    for (i = 0; i < dir->NumberOfIdEntries; i++) {
+        if (!CHECK_SAFE_READ(&entries[i+j], sizeof(entries[i+j]),
+                             valid_start, valid_size)) {
+            ASSERT_CURIOSITY(false && "rsrc dir parse error");
+            return NULL;
+        }
+        if (entries[i+j].NameIsString) {
+            /* only the first j entries should be indentified by name, the
+             * rest should be indentified by id (!NameIsString) */
+            ASSERT_CURIOSITY(false && "unexpected named entry");
+            continue;
+        }
+        if (entries[i+j].Name == id) {
+            return &entries[i+j];
+        } else {
+            LOG(GLOBAL, LOG_SYMBOLS, 3, "Skipping rsrc dir entry %d\n",
+                entries[i+j].Name);
+        }
+    }
+    return NULL;
+}
+
+/* returns pointer to the start of the VS_VERSIONINFO structure 
+ * also returns the size of the stucture in version_size */
+static void *
+get_module_resource_version_data(app_pc mod_base, size_t *version_size)
+{
+    size_t rsrc_dir_size;
+    IMAGE_RESOURCE_DIRECTORY *resource_base =
+        get_module_resource_directory(mod_base, &rsrc_dir_size);
+    IMAGE_RESOURCE_DIRECTORY_ENTRY *ver_entry = NULL;
+    IMAGE_RESOURCE_DIRECTORY *subdir;
+    IMAGE_RESOURCE_DATA_ENTRY *data;
+    void *version_info;
+
+    if (resource_base == NULL)
+        return NULL;
+
+    LOG(GLOBAL, LOG_SYMBOLS, 3, "Found rsrc section @"PFX"\n", resource_base);
+    
+    /* top level, look for VS_FILE_INFO type entry */
+    /* actually we should be going down to a ushort but we'll stick w/ the uint
+     * interface to get_resource_directory_entry_by_id()
+     */
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint((ptr_uint_t)VS_FILE_INFO)));
+    ver_entry =
+        get_resource_directory_entry_by_id(resource_base, (uint)(ptr_uint_t)VS_FILE_INFO,
+                                           (byte *)resource_base, rsrc_dir_size);
+    if (ver_entry == NULL)
+        return NULL;
+    /* We found a Version type directory entry, recurse down to version leaf */
+    LOG(GLOBAL, LOG_SYMBOLS, 3, "Found version rsrc entry\n");
+    if (!ver_entry->DataIsDirectory) {
+        ASSERT_CURIOSITY(false && "expected resource name/id subdirectory");
+        return NULL;
+    }
+
+    /* Next level should be resource identifier/name, we are looking for
+     * VS_VERSION_INFO (which is 1) */
+    subdir = (IMAGE_RESOURCE_DIRECTORY *)
+        RVA_TO_VA(resource_base, ver_entry->OffsetToDirectory);
+    ver_entry =
+        get_resource_directory_entry_by_id(subdir, (uint)VS_VERSION_INFO,
+                                           (byte *)resource_base, rsrc_dir_size);
+    if (ver_entry == NULL) {
+        DODEBUG({
+            const char *short_name = get_dll_short_name(mod_base);
+            /* xref case 9099 for detoured.dll exemption */
+            if (short_name == NULL || strcmp(short_name, "detoured.dll") != 0)
+                ASSERT_CURIOSITY(false && "expected VS_VERSION_INFO entry");
+        });
+        return NULL;
+    }
+    if (!ver_entry->DataIsDirectory) {
+        ASSERT_CURIOSITY(false && "expected resource lang subdirectory");
+        return false;
+    }
+
+    /* Next Level should be the language id. */
+    /* There is usually only one entry (having version strings in multiple languages is
+     * normally done via multiple string tables within the VS_VERSION_INFO which is more
+     * space efficient), however xref case 8742 for an ex. dll that splits the languages
+     * here instead. We check for English US first (would expect all MS dlls to include
+     * this) and failing that we just take the first entry instead (the keys are always
+     * in English, the fields we care about are usually also in english, and any entry
+     * will work for version numbers). */
+    subdir = (IMAGE_RESOURCE_DIRECTORY *)
+        RVA_TO_VA(resource_base, ver_entry->OffsetToDirectory);
+    ver_entry = NULL;
+    if (!CHECK_SAFE_READ(subdir, sizeof(*subdir) + sizeof(*ver_entry),
+                         resource_base, rsrc_dir_size) ||
+        subdir->NumberOfNamedEntries != 0 || subdir->NumberOfIdEntries < 1) {
+        ASSERT_CURIOSITY(false && "rsrc dir parse error");
+        return NULL;
+    }
+    if (subdir->NumberOfIdEntries > 1) {
+        /* multiple entries, try for US English */
+        ver_entry = get_resource_directory_entry_by_id(subdir,
+                                                       MAKELANGID(LANG_ENGLISH,
+                                                                  SUBLANG_ENGLISH_US),
+                                                       (byte *)resource_base,
+                                                       rsrc_dir_size);
+        DODEBUG({
+            if (ver_entry == NULL) {
+                char name[MAX_MODNAME_INTERNAL];
+                os_get_module_name_buf(mod_base, name, BUFFER_SIZE_ELEMENTS(name));
+                SYSLOG_INTERNAL_WARNING("Module %s @"PFX" with multiple lang id dirs "
+                                        "has no US english version info, 0x%04x.",
+                                        name != NULL ? name : "<none>", mod_base,
+                                        get_resource_directory_entries(subdir)->Name);
+            }
+        });
+    }
+    /* if only one entry or no us english entries, just take first */
+    if (ver_entry == NULL) {
+        ver_entry = get_resource_directory_entries(subdir);
+    }
+    if (ver_entry->DataIsDirectory) {
+        ASSERT_CURIOSITY(false && "expected resource data entry");
+        return NULL;
+    }
+    
+    /* Now we are finally to the IMAGE_RESOURCE_DATA_ENTRY. */
+    data = (IMAGE_RESOURCE_DATA_ENTRY *)
+        RVA_TO_VA(resource_base, ver_entry->OffsetToDirectory);
+    if (!CHECK_SAFE_READ(data, sizeof(*data), resource_base, rsrc_dir_size)) {
+        ASSERT_CURIOSITY(false && "rsrc dir parse error");
+        return NULL;
+    }
+    ASSERT_CURIOSITY(data->OffsetToData != 0 && data->Size != 0);
+    if (version_size != NULL)
+        *version_size = data->Size;
+    /* Yes this is relative to the module base, not the resource base like all
+     * the above offsets */
+    version_info = (void *)RVA_TO_VA(mod_base, data->OffsetToData);
+
+    /* safety check */
+    if (CHECK_SAFE_READ(version_info, data->Size, resource_base, rsrc_dir_size) ||
+        /* xref case 10542, version info might be outside .rsrc section */
+        is_readable_without_exception(version_info, data->Size)) {
+        return version_info;
+    } else {
+        ASSERT_CURIOSITY(false && "rsrc version data not readable");
+    }
+    return NULL;
+}
+
+/* The resource version information structures aren't proper C (they contain
+ * variable sized and alignment specific fields).  Pseudo descriptions can
+ * be found on 'msdn -> library -> win32 & com development -> user interface ->
+ * windows user interface -> resources -> version information' and are briefly
+ * duplicated here.  See also the wine implementations in dlls/version/info.c
+ * etc. for code that walks these, we add some extra complexity to ours to
+ * make these reads of app memory more safe.
+ *
+ * VS_VERSIONINFO is the top level structure
+ * {
+ *   WORD wLength; // size in bytes of vs_versioninfo stucture
+ *   WORD wValueLength; // length of Value field below
+ *   WORD wType; // type of data in Value (expect 0 -> binary)
+ *   WCHAR szKey[]; // contains "VS_VERSION_INFO"
+ *   WORD Padding1[]; // possible padding word to 32 bit align subsequent fields
+ *   VS_FIXEDFILEINFO Value; // see winver.h
+ *   WORD Padding2[]; // possible padding word to 32 bit align subsequent fields
+ *   StringFileInfo/VarFileInfo info[]; // 0 or 1 StringFileInfo and 0 or 1
+ *                                      // VarFileInfo in either order 
+ * }
+ *
+ * The StringFileInfo holds extra information in key/value string pairs (such
+ * as "CompanyName" "Microsoft Corp" etc.) that is displayed by properties etc.
+ * StringFileInfo {
+ *   WORD wLength; // size in bytes of this struture
+ *   WORD wValueLength; // 0
+ *   WORD wType; // type of data (expect 1 -> string)
+ *   WCHAR szKey; // contains "StringFileInfo"
+ *   WORD Padding[]; // if neccesary padding word to align 32 bit
+ *   StringTable tables[]; // 1 or more
+ * }
+ * StringTable {
+ *   WORD wLength; // size in bytes of this struture
+ *   WORD wValueLength; // 0
+ *   WORD wType; // type of data (expect 1 -> string)
+ *   WCHAR szKey; // 8 digit hex string specifying lang id and code page
+ *   WORD Padding[]; // if neccesary padding word to align 32 bit
+ *   String strings[]; // 1 or more
+ * }
+ * String {
+ *   WORD wLength; // size in bytes of this struture
+ *   WORD wValueLength; // size in bytes of value
+ *   WORD wType; // type of data (expect 1 -> string)
+ *   WCHAR szKey; // key (for ex. "CompanyName")
+ *   WORD Padding[]; // if neccesary padding word to align 32 bit
+ *   WCHAR szValue; // value (for ex. "Microsoft Corporation")
+ * }
+ *
+ * The VarFileInfo is used to specify which language/unicode code page pairs
+ * this dll supports.
+ * VarFileInfo {
+ *   WORD wLength; // size in bytes of this struture
+ *   WORD wValueLength; // 0
+ *   WORD wType; // type of data (expect 1 -> string)
+ *   WCHAR szKey; // contains "VarFileInfo"
+ *   WORD Padding[]; // if neccesary padding word to align 32 bit
+ *   Var vars[]; // 1 or more
+ * }
+ * Var {
+ *   WORD wLength; // size in bytes of this struture
+ *   WORD wValueLength; // size in bytes of value member
+ *   WORD wType; // type of data (expect 1 -> string)
+ *   WCHAR szKey; // contains "Translation"
+ *   WORD Padding[]; // if neccesary padding word to align 32 bit
+ *   DWORD Value[]; // array of 1 or more supported language code page pairs
+ * }
+ */
+/* We define our own versions of the version info types (since they aren't
+ * valid C) using ptrs to the variable sized parts. NOTE all ptrs are
+ * into the module .rsrc directory so don't count on them persisting beyond
+ * the lifetime of the module. */
+typedef struct ver_rsrc_header {
+    size_t length; /* length of structure in bytes */
+    size_t value_length; /* length of value in bytes */
+    uint type;
+    wchar_t *key;
+    size_t key_length; /* in bytes & including NULL terminator */
+} ver_rsrc_header_t;
+typedef struct vs_version_info {
+    VS_FIXEDFILEINFO *info; /* see winver.h */
+    void *string_or_var_info; /* can be NULL, note either var info or string
+                               * info can be first, have to disambiguate based
+                               * on the internal key strings and of course both
+                               * are optional. */
+} vs_version_info_t;
+typedef struct string_file_info {
+    size_t size; /* size in bytes of the string table(s) */
+    void *string_table; /* ptr to first string table */
+} string_file_info_t;
+typedef struct string_table {
+    size_t size; /* size in bytes of the string(s) */
+    wchar_t *lang; /* language identifier */
+    void *string; /* ptr to first rsrc string */
+} string_table_t;
+typedef struct rsrc_string {
+    size_t key_length; /* in bytes & including NULL terminator */
+    size_t value_length; /* in bytes & including NULL terminator */
+    wchar_t *key;
+    wchar_t *value;
+} rsrc_string_t;
+/* currently don't need var fields, or missing entries from above structs */
+
+#define RSRC_TYPE_STRING 1
+#define RSRC_TYPE_BINARY 0
+
+/* all version info structs are aligned 32 bit */
+#define RSRC_ALIGNMENT 4
+
+/* Since String, StringTable, StringFileInfo and VS_VERSIONINFO all start out
+ * with the same fields we use this common routine to read them into the above
+ * struct.  valid_start and valid_size delineate the memory region it is safe to
+ * access.  The header to read is at start and a ptr to the value field is
+ * returned (or NULL if read off the valid region).  If key_ref and bool are not
+ * NULL the key string is compared with key_ref and the result is returned in
+ * match. */
+static byte *
+read_version_struct_header(byte *start, byte *valid_start, size_t valid_size,
+                           ver_rsrc_header_t *head /* OUT */, wchar_t *key_ref,
+                           bool *match /* OUT */)
+{
+    size_t space_needed =
+        sizeof(head->length) + sizeof(head->value_length) + sizeof(head->type);
+    size_t max_wchars_left;
+    size_t key_length = 0;
+    byte *cur = start;
+    ushort *cur_u;
+
+    ASSERT(head != NULL && ((key_ref == NULL && match == NULL) ||
+                            (key_ref != NULL && match != NULL)));
+
+    if (key_ref != NULL) {
+        key_length = sizeof(wchar_t)*(wcslen(key_ref) + 1);
+        space_needed += key_length;
+    }
+    if (!CHECK_SAFE_READ(cur, space_needed, valid_start, valid_size))
+        return NULL;
+    cur_u = (ushort *)cur;
+    head->length = *cur_u++;
+    head->value_length = *cur_u++;
+    head->type = *cur_u++;
+    cur = (byte *)cur_u;
+    head->key = (wchar_t *)cur;
+
+    if (key_ref != NULL) {
+        if (wcscmp(key_ref, (wchar_t *)cur) == 0) {
+            *match = true;
+        } else {
+            key_length = 0;
+            *match = false;
+        }
+    }
+    if (key_length == 0) {
+        max_wchars_left = (valid_size - (cur - start)) / sizeof(wchar_t);
+        if (wcsnlen((wchar_t *)cur, max_wchars_left) >= max_wchars_left) {
+            return NULL;
+        }
+        key_length = sizeof(wchar_t) * (wcslen((wchar_t *)cur) + 1);
+    }
+    head->key_length = key_length;
+
+    /* advance cur past key string and alignment padding */
+    cur = (byte *)ALIGN_FORWARD(cur + key_length, RSRC_ALIGNMENT);
+    if (head->type == RSRC_TYPE_STRING && head->value_length != 0) {
+        /* If type == string then value_length is in wchars instead of bytes
+         * for the Microsoft Compiler, but not the Borland Compiler. However,
+         * xref case 10588, sometimes the value_length is just plain wrong.
+         * We just set the length to be the rest of the struct, we'll let
+         * the null termination tell us where the actual end is. */
+        head->value_length = head->length - (cur - (byte *)start);
+    }
+    LOG(GLOBAL, LOG_SYMBOLS, 3, "Read rsrc version structure header @"PFX":\n\t"
+        "length="PFX" value_length="PFX" type=0x%x\n\tkey=\"%S\" value @"PFX"\n",
+        start, head->length, head->value_length, head->type, head->key, cur);
+    return cur;
+}
+
+/* version_info - ptr to VS_VERSIONINFO to read
+ * version_info_size - size of said VS_VERSIONINFO
+ * info - OUT gets populated with the VS_VERSIONINFO data
+ * returns true if succesfully read version info */ 
+static bool
+read_vs_version_info(void *version_info, size_t version_info_size,
+                     vs_version_info_t *info /* OUT */)
+{
+    ver_rsrc_header_t head;
+    byte *cur = (byte *)version_info;
+    bool match;
+
+    ASSERT(version_info != NULL && info != NULL);
+    LOG(GLOBAL, LOG_SYMBOLS, 3, "Reading VS_VERSIONINFO @"PFX"\n", version_info);
+
+    cur = read_version_struct_header(cur, (byte *)version_info, version_info_size,
+                                     &head, L"VS_VERSION_INFO", &match);
+    if (cur == NULL) {
+        ASSERT_CURIOSITY(false && "read off end of rsrc version info");
+        return false;
+    }
+    if (!match) {
+        ASSERT_CURIOSITY(false && "invalid version info structure");
+        return false;
+    }
+    ASSERT_CURIOSITY(head.type == RSRC_TYPE_BINARY);
+
+    ASSERT_CURIOSITY(head.value_length == sizeof(VS_FIXEDFILEINFO));
+    if (!CHECK_SAFE_READ(cur, sizeof(VS_FIXEDFILEINFO),
+                         version_info, version_info_size)) {
+        ASSERT_CURIOSITY(false && "read off end of rsrc version info");
+        return false;
+    }
+    info->info = (VS_FIXEDFILEINFO *)cur;
+    info->string_or_var_info =
+        (void *)ALIGN_FORWARD(cur + head.value_length, RSRC_ALIGNMENT);
+    if ((byte *)info->string_or_var_info >= (byte *)version_info + head.length) {
+        /* has no string or var info */
+        LOG(GLOBAL, LOG_SYMBOLS, 2,
+            "Rsrc VS_VERSIONINFO @"PFX" has no String/VarFileInfo structs\n",
+            version_info);
+        info->string_or_var_info = NULL;
+    }
+    return true;
+}
+
+/* string_or_var_info - ptr to String/VarFileInfo to read
+ * version_info - ptr to the VS_VERSIONINFO containing this StringFileInfo
+ * version_info_size - size of said VS_VERSIONINFO
+ * info - OUT gets populated with the StringFileInfo data if string_or_var_info
+ *        is a StringFileInfo
+ * returns the address of the following StringFileInfo or VarFileInfo struct */
+static void *
+read_string_or_var_info(void *string_or_var_info, void *version_info,
+                        size_t version_info_size, string_file_info_t *info /* OUT */)
+{
+    ver_rsrc_header_t head;
+    size_t bytes_left;
+    byte *cur = (byte *)string_or_var_info;
+    bool match;
+
+    ASSERT(ALIGNED(string_or_var_info, RSRC_ALIGNMENT));
+    ASSERT(string_or_var_info != NULL && info != NULL);
+    memset(info, 0, sizeof(*info));
+    LOG(GLOBAL, LOG_SYMBOLS, 3, "Reading String/VarFileInfo @"PFX"\n",
+        string_or_var_info);
+
+    /* we check for VarFileInfo below */
+    cur = read_version_struct_header(cur, (byte *)version_info, version_info_size,
+                                     &head, L"StringFileInfo", &match);
+    if (cur == NULL) {
+        ASSERT_CURIOSITY(false && "read off end of rsrc version");
+        return NULL;
+    }
+    if (!match) {
+        wchar_t *ref_var = L"VarFileInfo";
+        if (wcscmp(ref_var, head.key) == 0) {
+            /* is a VarFileInfo field */
+            LOG(GLOBAL, LOG_SYMBOLS, 3, "Ignoring version rsrc VarFileInfo struct\n");
+            ASSERT_CURIOSITY(head.value_length == 0); /* no value field */
+        } else {
+            /* Xref case 9276, resvc.dll strangely has zero padding after its string and
+             * var infos.  It doesn't pose a problem for us so relax the assert here. */
+            /* happens many times in AcroRd32.exe 8.0 */
+            DODEBUG({
+                if (!is_region_memset_to_char(string_or_var_info,
+                                              version_info_size - 
+                                              ((byte *)string_or_var_info - 
+                                               (byte *)version_info), 0))
+                    SYSLOG_INTERNAL_WARNING_ONCE(".rsrc @"PFX": expected var or string "
+                                                 "info, or padding",
+                                                 string_or_var_info);
+            });
+            return NULL;
+        }
+    } else {
+        ASSERT_CURIOSITY(head.value_length == 0); /* no value field */
+    }
+
+    if (head.length < (size_t)(cur - (byte *)string_or_var_info)) {
+        ASSERT_CURIOSITY(false && "FileInfo length too short");
+        return NULL;
+    }
+
+    bytes_left = head.length - (cur - (byte *)string_or_var_info);
+
+    if (match) {
+        if (!CHECK_SAFE_READ(cur, bytes_left, version_info, version_info_size)) {
+            ASSERT_CURIOSITY(false && "string file info too large");
+            return NULL;
+        }
+        info->size = bytes_left;
+        info->string_table = cur;
+    }
+    /* else is VarFileInfo so nothing to fill in */
+
+    /* advance to the next String or Var FileInfo */
+    cur = (byte *)ALIGN_FORWARD(cur + bytes_left, RSRC_ALIGNMENT);
+    if (cur >= (byte *)version_info + version_info_size)
+        return NULL; /* last info */
+    return cur;
+}
+
+/* string_table - ptr to StringTable to read
+ * remaining_string_table_size - IN/OUT bytes left in the StringTable array
+ * table - OUT gets populated with the StringTable data
+ * Returns a ptr to the next string table in the array (or NULL if last one) */
+static void *
+read_string_table(void *string_table, size_t *remaining_table_size,
+                  string_table_t *table /* OUT */)
+{
+    ver_rsrc_header_t head;
+    byte *cur = (byte *)string_table;
+
+    ASSERT(ALIGNED(string_table, RSRC_ALIGNMENT));
+    ASSERT(string_table != NULL && remaining_table_size != NULL && table != NULL);
+    memset(table, 0, sizeof(*table)); /* in case return early */
+    LOG(GLOBAL, LOG_SYMBOLS, 3, "Reading StringTable @"PFX"\n", string_table);
+
+    cur = read_version_struct_header(cur, (byte *)string_table,
+                                     *remaining_table_size,
+                                     &head, NULL, NULL);
+    if (cur == NULL) {
+        ASSERT_CURIOSITY(false && "read off end of string table array");
+        return NULL;
+    }
+    ASSERT_CURIOSITY(head.value_length == 0); /* no value field */
+    /* check expected length of lang string */
+    ASSERT_CURIOSITY(head.key_length == (8 + 1 /* NULL */) * sizeof(wchar_t));
+
+    if (head.length > *remaining_table_size) {
+        ASSERT_CURIOSITY(false && "string table too large");
+        return NULL;
+    }
+    if (head.length < (size_t)(cur - (byte *)string_table)) {
+        ASSERT_CURIOSITY(false && "string table too small");
+        return NULL;
+    }
+
+    /* checked for underflow above */
+    table->size = head.length - (cur - (byte *)string_table);
+    table->lang = head.key;
+    table->string = cur;
+    if (ALIGN_FORWARD(head.length, RSRC_ALIGNMENT) >= *remaining_table_size) {
+        *remaining_table_size = 0;
+        return NULL;
+    } else {
+        *remaining_table_size -= ALIGN_FORWARD(head.length, RSRC_ALIGNMENT);
+        return (byte *)string_table + ALIGN_FORWARD(head.length, RSRC_ALIGNMENT);
+    }
+}
+
+/* rsrc_string - ptr to the String to read
+ * remaining_rsrc_string_size - IN/OUT bytes left in the String array
+ * string - OUT gets populated with the String data
+ * Returns a ptr to the next String in the array (or NULL if last one) */
+static void *
+read_rsrc_string(void *rsrc_string, size_t *remaining_rsrc_string_size,
+                 rsrc_string_t *string /* OUT */)
+{
+    ver_rsrc_header_t head;
+    byte *cur = (byte *)rsrc_string;
+
+    ASSERT(ALIGNED(rsrc_string, RSRC_ALIGNMENT));
+    ASSERT(rsrc_string != NULL && remaining_rsrc_string_size != NULL
+           && string != NULL);
+    memset(string, 0, sizeof(*string)); /* in case return early */
+    LOG(GLOBAL, LOG_SYMBOLS, 3, "Reading Rsrc String @"PFX"\n", rsrc_string);
+
+    cur = read_version_struct_header(cur, (byte *)rsrc_string,
+                                     *remaining_rsrc_string_size,
+                                     &head, NULL, NULL);
+    if (cur == NULL) {
+        ASSERT_CURIOSITY(false && "read off end of rsrc version string array");
+        return NULL;
+    }
+
+    if (head.length < (size_t)(cur - (byte *)rsrc_string)) {
+        ASSERT_CURIOSITY(false && "Rsrc string length too short");
+        return NULL;
+    }
+
+    /* Expect the type to always be string, but xref case 8797 for an instance where it
+     * isn't (in that case value was 0, so could still be read as a string, though
+     * was sized in bytes instead of wchars so presumably ment to be binary). We'll just
+     * ignore non string type rsrc Strings. */ 
+    if (head.type == RSRC_TYPE_STRING && head.value_length > 0) {
+        if (!CHECK_SAFE_READ(cur, head.value_length,
+                             rsrc_string, *remaining_rsrc_string_size)) {
+            ASSERT_CURIOSITY(false && "rsrc string value extends too far");
+            return NULL;
+        }
+        if (wcsnlen((wchar_t *)cur, head.value_length / sizeof(wchar_t)) >=
+            (head.value_length / sizeof(wchar_t))) {
+            ASSERT_CURIOSITY(false && "rsrc value string isn't null terminated");
+            return NULL;
+        }
+        /* Don't normally expect dead value space after then end of the string, but xref
+         * case 8797 for an ex. where the string has a NULL in the middle of it so we
+         * can't have an assert curiosity here. */
+        string->value = (wchar_t *)cur;
+        LOG(GLOBAL, LOG_SYMBOLS, 3, "\tvalue=\"%S\"\n", string->value);
+    } else {
+        /* make sure that if it isn't string type, it's not one of the fields we want */
+        ASSERT_CURIOSITY(head.type == RSRC_TYPE_STRING ||
+                         (wcscmp(L"CompanyName", head.key) != 0 &&
+                          wcscmp(L"ProductName", head.key) != 0 &&
+                          wcscmp(L"OriginalFilename", head.key) != 0));
+        string->value = NULL;
+    }
+
+    string->value_length = head.value_length;
+    string->key_length = (head.key_length+1)*sizeof(wchar_t);
+    string->key = head.key;
+
+    if (ALIGN_FORWARD(head.length, RSRC_ALIGNMENT) >= *remaining_rsrc_string_size) {
+        *remaining_rsrc_string_size = 0;
+        return NULL;
+    } else {
+        *remaining_rsrc_string_size -= ALIGN_FORWARD(head.length, RSRC_ALIGNMENT);
+        return (byte *)rsrc_string + ALIGN_FORWARD(head.length, RSRC_ALIGNMENT);
+    }
+}
+
+/* NOTE the strings returned in info_out are pointing to the .rsrc version
+ * directory and as such they're only valid while the module is loaded. */
+static bool
+get_module_resource_version_info(app_pc mod_base, version_info_t *info_out)
+{
+    size_t size, remaining_table, remaining_string;
+    vs_version_info_t ver_info;
+    string_file_info_t string_info;
+    string_table_t string_table;
+    rsrc_string_t string;
+    void *version_rsrc, *cur_table, *cur_string;
+    DEBUG_DECLARE(uint num_string_file_info = 0;)
+    DEBUG_DECLARE(const char *mod_name = "";) /* only set at loglevel 2+ */
+
+    ASSERT(info_out != NULL);
+    memset(info_out, 0, sizeof(version_info_t));
+    DOLOG(2, LOG_SYMBOLS, { 
+        /* We will have an infinite loop if we call get_module_short_name()
+         * so we go w/ PE name, just for debugging */
+        mod_name = get_dll_short_name(mod_base);
+        if (mod_name == NULL)
+            mod_name = "";
+    });
+    LOG(GLOBAL, LOG_SYMBOLS, 3,
+        "Reading rsrc version information for module %s @"PFX"\n",
+        mod_name, mod_base);
+
+    version_rsrc = get_module_resource_version_data(mod_base, &size);
+    if (version_rsrc == NULL) {
+        LOG(GLOBAL, LOG_SYMBOLS, 2, "Module %s has no rsrc section\n", mod_name);
+        return false;
+    }
+
+    if (!read_vs_version_info(version_rsrc, size, &ver_info)) {
+        LOG(GLOBAL, LOG_SYMBOLS, 2, "Module %s has no version rsrc\n", mod_name);
+        return false;
+    }
+
+    info_out->file_version.version_uint.ms = ver_info.info->dwFileVersionMS;
+    info_out->file_version.version_uint.ls = ver_info.info->dwFileVersionLS;
+    info_out->product_version.version_uint.ms = ver_info.info->dwProductVersionMS;
+    info_out->product_version.version_uint.ls = ver_info.info->dwProductVersionLS;
+    LOG(GLOBAL, LOG_SYMBOLS, 3,
+        "Module %s file_version=%d.%d.%d.%d product_version=%d.%d.%d.%d\n"
+        "\tflags_mask=0x%08x flags=0x%08x\n", mod_name,
+        info_out->file_version.version_parts.p1,
+        info_out->file_version.version_parts.p2,
+        info_out->file_version.version_parts.p3,
+        info_out->file_version.version_parts.p4,
+        info_out->product_version.version_parts.p1,
+        info_out->product_version.version_parts.p2,
+        info_out->product_version.version_parts.p3,
+        info_out->product_version.version_parts.p4,
+        ver_info.info->dwFileFlagsMask, ver_info.info->dwFileFlags);
+
+    while (ver_info.string_or_var_info != NULL) {
+        ver_info.string_or_var_info =
+            read_string_or_var_info(ver_info.string_or_var_info, version_rsrc,
+                                    size, &string_info);
+        if (string_info.string_table == NULL)
+            continue;
+
+        /* should be only 0 or 1 string_file_info structs */
+        ASSERT_CURIOSITY(num_string_file_info++ == 0);
+
+        remaining_table = string_info.size;
+        cur_table = string_info.string_table;
+        while (cur_table != NULL && remaining_table > 0) {
+            cur_table = read_string_table(cur_table, &remaining_table,
+                                          &string_table);
+            /* FIXME - there can be several tables (for different languages),
+             * right now we just scan all of them and for the fields we care
+             * about use the last value we find. Prob. better to give pref.
+             * to english though (note the key strings always appear to be in
+             * English, at least the standard ones, only the values vary). */
+            remaining_string = string_table.size;
+            cur_string = string_table.string;
+            while (cur_string != NULL && remaining_string > 0) {
+                cur_string = read_rsrc_string(cur_string, &remaining_string,
+                                              &string);
+                if (string.key != NULL) {
+                    if (string.value == NULL)
+                        string.value = L""; /* has key, but empty value */
+                    if (wcscmp(L"CompanyName", string.key) == 0) {
+                        info_out->company_name = string.value;
+                    } else if (wcscmp(L"ProductName", string.key) == 0) {
+                        info_out->product_name = string.value;
+                    } else if (wcscmp(L"OriginalFilename", string.key) == 0) {
+                        info_out->original_filename = string.value;
+                    }
+                    LOG(GLOBAL, LOG_SYMBOLS, 4,
+                        "read .rsrc version string key=\"%S\" value=\"%S\"\n",
+                        string.key, string.value);
+                }
+            }
+        }
+    }
+    return true;
+}
+
+bool
+get_module_company_name(app_pc mod_base, char *out_buf, size_t out_buf_size)
+{
+    version_info_t info;
+    if (get_module_resource_version_info(mod_base, &info) &&
+        info.company_name != NULL) {
+        snprintf(out_buf, out_buf_size, "%S", info.company_name);
+        out_buf[out_buf_size-1] = '\0';
+        return true;
+    }
+    return false;
+}
+
+/* Using strdup rather than a passed-in buffer b/c of module_area_t needs:
+ * we've already exported module_data_t that does not have an inlined buffer.
+ * Caller is responsible for freeing the string heap space, of course!
+ */
+static const char *
+get_module_original_filename(app_pc mod_base HEAPACCT(which_heap_t which))
+{
+    version_info_t info;
+    if (get_module_resource_version_info(mod_base, &info) &&
+        info.original_filename != NULL) {
+        return (const char *) dr_wstrdup(info.original_filename HEAPACCT(which));
+    }
+    return NULL;
+}
+
+#ifdef DEBUG /* currently no release build user */
+thread_id_t
+get_loader_lock_owner()
+{
+    PEB *peb = get_own_peb();
+    RTL_CRITICAL_SECTION *lock = (RTL_CRITICAL_SECTION *) peb->LoaderLock;
+    return (thread_id_t) lock->OwningThread;
+}
+#endif
diff --git a/core/win32/module.h b/core/win32/module.h
new file mode 100644
index 000000000..b688f7a79
--- /dev/null
+++ b/core/win32/module.h
@@ -0,0 +1,91 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifndef MODULE_H
+#define MODULE_H
+
+#include "fragment.h" /* for rct_module_table_t, is inlined so need struct definition
+                       * FIXME - remove this header dependency */
+
+#define OS_IMAGE_READ    IMAGE_SCN_MEM_READ
+#define OS_IMAGE_WRITE   IMAGE_SCN_MEM_READ
+#define OS_IMAGE_EXECUTE IMAGE_SCN_MEM_EXECUTE
+
+typedef struct _os_module_data_t {
+    app_pc preferred_base; /* module preferred base from the PE headers */
+    uint checksum; /* module checksum from the PE headers */
+    uint timestamp; /* module timestamp from the PE headers */
+    size_t module_internal_size; /* module internal size (from PE headers SizeOfImage) */
+
+    size_t code_size; /* sum of the size of all code sections */
+
+    /* from the .rsrc section */
+    version_number_t file_version;
+    version_number_t product_version;
+    char *company_name;
+    char *product_name;
+
+    /* ALSR sharing - we'll to keep a reference to the original
+     * application section to maintain no-clobber transparency 
+     * disallowing modifications to the original file.
+     *
+     * Note: on detach we do NOT release any such handles: detached
+     * processes will have to be killed for us to release the file
+     * handles, non-transparent only with respect to DLLs that get 
+     * unloaded after detach while native.
+     */
+    HANDLE noclobber_section_handle;
+
+    /* FIXME: the loader also maintains Name and Path that may be
+     * different than PE name */
+
+    /* FIXME: this structure should replace the now deprecated
+     * module_info_t since all of its once advanced features have been
+     * incorporated in vmarea vectors
+     */
+
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+    /* case 9672: we split our RCT and RAC targets into per-module tables.
+     * FIXME: once we have a module list on linux, move these to module_data.
+     */
+    rct_module_table_t rct_table[RCT_NUM_TYPES];
+#endif
+
+    /* Case 8640: store original code on IAT page.  We store bounds here,
+     * even though computable, to avoid a modified PE header causing
+     * us to read beyond the bounds.
+     */
+    byte *iat_code;
+    size_t iat_len;
+} os_module_data_t;
+
+#endif /* MODULE_H */
diff --git a/core/win32/module_shared.c b/core/win32/module_shared.c
new file mode 100644
index 000000000..5dc5da36d
--- /dev/null
+++ b/core/win32/module_shared.c
@@ -0,0 +1,614 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * module_shared.c
+ * Windows DLL routines that are shared between the core, preinject, and
+ * drmarker-using code like libutil.
+ * It's a pain to link module.c with non-core targets like preinject,
+ * so we split these routines out here.
+ * Note that NOT_DYNAMORIO_CORE_PROPER still links ntdll.c, while
+ * NOT_DYNAMORIO_CORE (i.e., libutil) does not (since it is a pain
+ * to link both ntdll.lib and a libc.lib).
+ */
+
+#if defined(NOT_DYNAMORIO_CORE)
+# define ASSERT(x)
+# define ASSERT_NOT_REACHED()
+# define ASSERT_NOT_IMPLEMENTED(x)
+# define DODEBUG(x)
+# define DECLARE_NEVERPROT_VAR(var, ...) var = __VA_ARGS__;
+# define ALIGN_BACKWARD(x, alignment) (((ULONG_PTR)x) & (~((alignment)-1)))
+# define PAGE_SIZE 4096
+#else
+/* we include globals.h mainly for ASSERT, even though we're
+ * used by preinject.
+ * preinject just defines its own internal_error!
+ */
+# include "globals.h"
+# if !defined(NOT_DYNAMORIO_CORE_PROPER)
+#  include "os_private.h" /* for is_readable_pe_base() */
+#  include "module_shared.h" /* for is_in_code_section() */
+# endif
+#endif
+
+#include "ntdll.h"
+
+/* We have to hack away things we use here that won't work for non-core */
+#if defined(NOT_DYNAMORIO_CORE_PROPER) || defined(NOT_DYNAMORIO_CORE)
+# undef strcasecmp
+# define strcasecmp _stricmp
+# define wcscasecmp _wcsicmp
+# undef TRY_EXCEPT_ALLOW_NO_DCONTEXT
+# define TRY_EXCEPT_ALLOW_NO_DCONTEXT(dc, try, except) try
+# undef ASSERT_OWN_NO_LOCKS
+# define ASSERT_OWN_NO_LOCKS() /* who cares if not the core */
+# undef ASSERT_CURIOSITY
+# define ASSERT_CURIOSITY(x) /* who cares if not the core */
+/* since not including os_shared.h (this is impl in ntdll.c): */
+bool is_readable_without_exception(const byte *pc, size_t size);
+/* allow converting functions to and from data pointers */
+# pragma warning(disable : 4055)
+# pragma warning(disable : 4054)
+# define convert_data_to_function(func) ((generic_func_t) (func))
+#endif
+
+/* This routine was moved here from os.c since we need it for 
+ * get_proc_address_64 (via get_module_exports_directory_*()) for preinject
+ * and drmarker, neither of which link os.c.
+ */
+/* is_readable_without_exception checks to see that all bytes with addresses
+ * from pc to pc+size-1 are readable and that reading from there won't
+ * generate an exception.  this is a stronger check than
+ * !not_readable() below.
+ * FIXME : beware of multi-thread races, just because this returns true, 
+ * doesn't mean another thread can't make the region unreadable between the 
+ * check here and the actual read later.  See safe_read() as an alt.
+ */
+/* throw-away buffer */
+DECLARE_NEVERPROT_VAR(static char is_readable_buf[4/*efficient read*/], {0});
+bool 
+is_readable_without_exception(const byte *pc, size_t size)
+{
+    /* Case 7967: NtReadVirtualMemory is significantly faster than
+     * NtQueryVirtualMemory (probably even for large regions where NtQuery can
+     * walk by mbi.RegionSize but we have to walk by page size).  We don't care
+     * if multiple threads write into the buffer at once.  Nearly all of our
+     * calls ask about areas smaller than a page.
+     */
+    byte *check_pc = (byte *) ALIGN_BACKWARD(pc, PAGE_SIZE);
+    if (size > (size_t)((byte *)POINTER_MAX - pc))
+        size = (byte *)POINTER_MAX - pc;
+    do {
+        size_t bytes_read = 0;
+#if defined(NOT_DYNAMORIO_CORE)
+        if (!ReadProcessMemory(NT_CURRENT_PROCESS, check_pc, is_readable_buf,
+                               sizeof(is_readable_buf), (SIZE_T*) &bytes_read) ||
+            bytes_read != sizeof(is_readable_buf))
+#else
+        if (!nt_read_virtual_memory(NT_CURRENT_PROCESS, check_pc, is_readable_buf,
+                                    sizeof(is_readable_buf), &bytes_read) ||
+            bytes_read != sizeof(is_readable_buf))
+#endif
+            return false;
+        check_pc += PAGE_SIZE;
+    } while (check_pc != 0/*overflow*/ && check_pc < pc+size);
+    return true;
+}
+
+/* returns NULL if exports directory doesn't exist
+ * if exports_size != NULL returns also exports section size
+ * assumes base_addr is a safe is_readable_pe_base() 
+ *
+ * NOTE - only verifies readability of the IMAGE_EXPORT_DIRECTORY, does not verify target
+ * readability of any RVAs it contains (for that use get_module_exports_directory_check
+ * or verify in the caller at usage). Xref case 9717.
+ */
+static
+IMAGE_EXPORT_DIRECTORY*
+get_module_exports_directory_common(app_pc base_addr,
+                                    size_t *exports_size /* OPTIONAL OUT */
+                                    _IF_NOT_X64(bool ldr64))
+{
+    IMAGE_DOS_HEADER *dos = (IMAGE_DOS_HEADER *) base_addr;
+    IMAGE_NT_HEADERS *nt = (IMAGE_NT_HEADERS *) (base_addr + dos->e_lfanew);
+    IMAGE_DATA_DIRECTORY *expdir;
+    ASSERT(dos->e_magic == IMAGE_DOS_SIGNATURE);
+    ASSERT(nt != NULL && nt->Signature == IMAGE_NT_SIGNATURE);
+#ifndef X64
+    if (ldr64) {
+        expdir = ((IMAGE_OPTIONAL_HEADER64 *)&(nt->OptionalHeader))->DataDirectory +
+            IMAGE_DIRECTORY_ENTRY_EXPORT;
+    } else
+#endif
+        expdir = OPT_HDR(nt, DataDirectory) + IMAGE_DIRECTORY_ENTRY_EXPORT;
+
+    /* avoid link issues: we don't have is_readable_pe_base */
+#if !defined(NOT_DYNAMORIO_CORE_PROPER) && !defined(NOT_DYNAMORIO_CORE)
+    /* callers should have done this in release builds */
+    ASSERT(is_readable_pe_base(base_addr));
+#endif
+
+    /* RVA conversions are trivial only for MEM_IMAGE */
+#if !defined(NOT_DYNAMORIO_CORE_PROPER) && !defined(NOT_DYNAMORIO_CORE)
+    DODEBUG({
+        MEMORY_BASIC_INFORMATION mbi;
+        ASSERT(query_virtual_memory(base_addr, &mbi, sizeof(mbi)) == sizeof(mbi));
+        /* We do see MEM_MAPPED PE files: case 7947 */
+        if (mbi.Type != MEM_IMAGE) {
+            LOG(THREAD_GET, LOG_SYMBOLS, 1, 
+                "get_module_exports_directory(base_addr="PFX"): !MEM_IMAGE\n",
+                base_addr);
+            ASSERT_CURIOSITY(expdir == NULL || expdir->Size == 0);
+        }
+    });
+#endif
+
+    /* libutil doesn't support vararg macros so can't undef-out */
+#if !defined(NOT_DYNAMORIO_CORE_PROPER) && !defined(NOT_DYNAMORIO_CORE)
+    LOG(THREAD_GET, LOG_SYMBOLS, 5, 
+        "get_module_exports_directory(base_addr="PFX", expdir="PFX")\n",
+        base_addr, expdir);
+#endif
+
+    if (expdir != NULL) {
+        ULONG size = expdir->Size;
+        ULONG exports_vaddr = expdir->VirtualAddress;
+
+        /* libutil doesn't support vararg macros so can't undef-out */
+#if !defined(NOT_DYNAMORIO_CORE_PROPER) && !defined(NOT_DYNAMORIO_CORE)
+        LOG(THREAD_GET, LOG_SYMBOLS, 5, 
+            "get_module_exports_directory(base_addr="PFX") expdir="PFX
+            " size=%d exports_vaddr=%d\n", 
+            base_addr, expdir, size, exports_vaddr);
+#endif
+
+        /* not all DLLs have exports - e.g. drpreinject.dll, or
+         * shdoclc.dll in notepad help */
+        if (size > 0) {
+            IMAGE_EXPORT_DIRECTORY *exports = (IMAGE_EXPORT_DIRECTORY *)
+                (base_addr + exports_vaddr);
+            ASSERT_CURIOSITY(size >= sizeof(IMAGE_EXPORT_DIRECTORY));
+            if (is_readable_without_exception((app_pc)exports,
+                                              sizeof(IMAGE_EXPORT_DIRECTORY))) {
+                if (exports_size != NULL)
+                    *exports_size = size;
+                ASSERT_CURIOSITY(exports->Characteristics == 0);
+                return exports;
+            } else {
+                ASSERT_CURIOSITY(false && "bad exports directory, partial map?" ||
+                                 EXEMPT_TEST("partial_map.exe"));
+            }
+        }
+    } else 
+        ASSERT_CURIOSITY(false && "no exports directory");
+
+    return NULL;
+}
+
+/* Same as get_module_exports_directory except also verifies that the functions (and,
+ * if check_names, ordinals and fnames) arrays are readable. NOTE - does not verify that
+ * the RVA names pointed to by fnames are themselves readable strings. */
+static
+IMAGE_EXPORT_DIRECTORY*
+get_module_exports_directory_check_common(app_pc base_addr,
+                                          size_t *exports_size /*OPTIONAL OUT*/,
+                                          bool check_names
+                                          _IF_NOT_X64(bool ldr64))
+{
+    IMAGE_EXPORT_DIRECTORY *exports = get_module_exports_directory_common
+        (base_addr, exports_size _IF_NOT_X64(ldr64));
+    if (exports != NULL) {
+        PULONG functions =  (PULONG) (base_addr + exports->AddressOfFunctions);
+        PUSHORT ordinals = (PUSHORT) (base_addr + exports->AddressOfNameOrdinals);
+        PULONG fnames    =  (PULONG) (base_addr + exports->AddressOfNames);
+        if (exports->NumberOfFunctions > 0) {
+            if (!is_readable_without_exception((byte *)functions,
+                                               exports->NumberOfFunctions *
+                                               sizeof(*functions))) {
+                ASSERT_CURIOSITY(false &&  "ill-formed exports directory, unreadable "
+                                 "functions array, partial map?" ||
+                                 EXEMPT_TEST("partial_map.exe"));
+                return NULL;
+            }
+        }
+        if (exports->NumberOfNames > 0 && check_names) {
+            ASSERT_CURIOSITY(exports->NumberOfFunctions > 0 &&
+                             "ill-formed exports directory");
+            if (!is_readable_without_exception((byte *)ordinals, exports->NumberOfNames *
+                                               sizeof(*ordinals)) ||
+                !is_readable_without_exception((byte *)fnames, exports->NumberOfNames *
+                                               sizeof(*fnames))) {
+                ASSERT_CURIOSITY(false &&  "ill-formed exports directory, unreadable "
+                                 "ordinal or names array, partial map?" ||
+                                 EXEMPT_TEST("partial_map.exe"));
+                return NULL;
+            }
+        }
+    }
+    return exports;
+}
+
+/* FIXME - this walk is similar to that used in several other module.c
+ * functions, we should look into sharing. Also, like almost all of the
+ * module.c routines this could be racy with app memory deallocations.
+ * FIXME - we could also have a version that lookups by ordinal. We could
+ * also allow wildcards and, if desired, extend it to return multiple
+ * matching addresses.
+ */
+/* Interface is similar to msdn GetProcAddress, takes in a module_handle_t
+ * (this is just the allocation base of the module) and a name and returns
+ * the address of the export with said name.  Returns NULL on failure.
+ * NOTE - will return NULL for forwarded exports, exports pointing outside of
+ * the module and for exports not in a code section (FIXME - is this the
+ * behavior we want?). Name is case insensitive.
+ */ 
+generic_func_t
+get_proc_address_common(module_handle_t lib, const char *name _IF_NOT_X64(bool ldr64))
+{
+    app_pc module_base;
+    size_t exports_size;
+    uint i;
+    IMAGE_EXPORT_DIRECTORY *exports;
+    PULONG functions; /* array of RVAs */
+    PUSHORT ordinals;
+    PULONG fnames; /* array of RVAs */
+
+    /* avoid non-core issues: we don't have get_allocation_size or dcontexts */
+#if !defined(NOT_DYNAMORIO_CORE_PROPER) && !defined(NOT_DYNAMORIO_CORE)
+    size_t module_size;
+    dcontext_t *dcontext = get_thread_private_dcontext();
+#endif
+
+    ASSERT(name != NULL && *name != '\0'); /* verify valid args */
+    if (lib == NULL || name == NULL || name == '\0')
+        return NULL;
+
+    /* avoid non-core issues: we don't have get_allocation_size or is_readable_pe_base */
+#if !defined(NOT_DYNAMORIO_CORE_PROPER) && !defined(NOT_DYNAMORIO_CORE)
+    /* FIXME - get_allocation_size and is_readable_pe_base are expensive
+     * operations, we could put the onus on the caller to only pass in a
+     * valid module_handle_t/pe_base and just assert instead if performance of
+     * this routine becomes a concern, esp. since the caller has likely
+     * already done it. */
+    module_size = get_allocation_size(lib, &module_base);
+    if (!is_readable_pe_base(module_base))
+        return NULL;
+#else
+    module_base = (app_pc) lib;
+#endif
+
+    exports = get_module_exports_directory_check_common
+        (module_base, &exports_size, true _IF_NOT_X64(ldr64));
+    if (exports == NULL || exports_size == 0 || exports->NumberOfNames == 0 ||
+        /* just extra sanity check */ exports->AddressOfNames == 0)
+        return NULL;
+
+    /* avoid non-core issues: we don't have module_size */
+#if !defined(NOT_DYNAMORIO_CORE_PROPER) && !defined(NOT_DYNAMORIO_CORE)
+    /* sanity check */
+    ASSERT(exports->AddressOfFunctions < module_size &&
+           exports->AddressOfFunctions > 0 &&
+           exports->AddressOfNameOrdinals < module_size &&
+           exports->AddressOfNameOrdinals > 0 &&
+           exports->AddressOfNames < module_size &&
+           exports->AddressOfNames > 0);
+#endif
+
+    functions = (PULONG)(module_base + exports->AddressOfFunctions);
+    ordinals = (PUSHORT)(module_base + exports->AddressOfNameOrdinals);
+    fnames = (PULONG)(module_base + exports->AddressOfNames);
+    
+    /* FIXME - linear walk, if this routine becomes performance critical we
+     * we should use a binary search. */
+    for (i = 0; i < exports->NumberOfNames; i++) {
+        char *export_name = (char *)(module_base + fnames[i]);
+        bool match = false;
+        ASSERT_CURIOSITY((app_pc)export_name > module_base &&  /* sanity check */
+                         (app_pc)export_name < module_base + module_size ||
+                         EXEMPT_TEST("partial_map.exe"));
+        /* FIXME - xref case 9717, we haven't verified that export_name string is
+         * safely readable (might not be the case for improperly formed or partially
+         * mapped module) and the try will only protect us if we have a thread_private
+         * dcontext. Could use is_string_readable_without_exception(), but that may be
+         * too much of a perf hit for the no private dcontext case. */
+        TRY_EXCEPT_ALLOW_NO_DCONTEXT(dcontext, {
+            match = (strcasecmp(name, export_name) == 0);
+        }, {
+            ASSERT_CURIOSITY_ONCE(false && "Exception during get_proc_address" ||
+                                  EXEMPT_TEST("partial_map.exe"));
+        });
+        if (match) {
+            /* we have a match */
+            uint ord = ordinals[i];
+            app_pc func;
+            /* note - function array is indexed by ordinal */
+            if (ord >= exports->NumberOfFunctions) {
+                ASSERT_CURIOSITY(false && "invalid ordinal index");
+                return NULL;
+            }
+            func = (app_pc)(module_base + functions[ord]);
+            if (func == module_base) {
+                /* entries can be 0 when no code/data is exported for that
+                 * ordinal */
+                ASSERT_CURIOSITY(false &&
+                                 "get_proc_addr of name with empty export");
+                return NULL;
+            }
+            /* avoid non-core issues: we don't have module_size */
+#if !defined(NOT_DYNAMORIO_CORE_PROPER) && !defined(NOT_DYNAMORIO_CORE)
+            if (func < module_base || func >= module_base + module_size) {
+                /* FIXME - export isn't in the module, should we still return
+                 * it?  Will shimeng.dll or the like ever do this to replace
+                 * a function?  For now we return NULL. Xref case 9717, can also
+                 * happen for a partial map in which case NULL is the right thing
+                 * to return. */
+                ASSERT_CURIOSITY(false && "get_proc_addr export location "
+                                 "outside of module bounds" ||
+                                  EXEMPT_TEST("partial_map.exe"));
+                return NULL;
+            }
+#endif
+            if (func >= (app_pc)exports &&
+                func < (app_pc)exports + exports_size) {
+                /* FIXME - is forwarded function, should we still return it
+                 * or return the target? Check - what does GetProcAddress do?
+                 * For now we return NULL. Looking up the target would require
+                 * a get_module_handle call which might not be safe here. 
+                 * With current and planned usage we shouldn' be looking these
+                 * up anyways. */
+                ASSERT_NOT_IMPLEMENTED(false &&
+                                       "get_proc_addr export is forwarded");
+                return NULL;
+            }
+            /* avoid non-core issues: we don't have is_in_code_section */
+#if !defined(NOT_DYNAMORIO_CORE_PROPER) && !defined(NOT_DYNAMORIO_CORE)
+# ifndef CLIENT_INTERFACE
+            /* CLIENT_INTERFACE uses a data export for versioning (PR 250952) */
+            /* FIXME - this check is also somewhat costly. */
+            if (!is_in_code_section(module_base, func, NULL, NULL)) {
+                /* FIXME - export isn't in a code section. Prob. a data
+                 * export?  For now we return NULL as all current users are
+                 * going to call or hook the returned value. */
+                ASSERT_CURIOSITY(false &&
+                                 "get_proc_addr export not in code section");
+                return NULL;
+            }
+# endif
+#endif
+            /* get around warnings converting app_pc to generic_func_t */
+            return convert_data_to_function(func);
+        }
+    }
+
+    /* export name wasn't found */
+    return NULL;
+}
+
+IMAGE_EXPORT_DIRECTORY*
+get_module_exports_directory(app_pc base_addr, size_t *exports_size /* OPTIONAL OUT */)
+{
+    return get_module_exports_directory_common
+        (base_addr, exports_size _IF_NOT_X64(false));
+}
+
+IMAGE_EXPORT_DIRECTORY*
+get_module_exports_directory_check(app_pc base_addr,
+                                   size_t *exports_size /*OPTIONAL OUT*/,
+                                   bool check_names)
+{
+    return get_module_exports_directory_check_common
+        (base_addr, exports_size, check_names _IF_NOT_X64(false));
+}
+
+generic_func_t
+get_proc_address(module_handle_t lib, const char *name)
+{
+    return get_proc_address_common(lib, name _IF_NOT_X64(false));
+}
+
+#ifndef NOT_DYNAMORIO_CORE /* else need get_own_peb() alternate */
+/* returns NULL if no loader module is found
+ * N.B.: walking loader data structures at random times is dangerous! See
+ * get_ldr_module_by_pc in module.c for code to grab the ldr lock (which is
+ * also unsafe).  Here we presume that we already own the ldr lock and that
+ * the ldr list is consistent, which should be the case for preinject (the only
+ * user).  FIXME stick this in module.c with get_ldr_module_by_pc, would need
+ * to get module.c compiled into preinjector which is a significant hassle.
+ */
+LDR_MODULE *
+get_ldr_module_by_name(wchar_t *name)
+{
+    PEB *peb = get_own_peb();
+    PEB_LDR_DATA *ldr = peb->LoaderData;
+    LIST_ENTRY *e, *mark;
+    LDR_MODULE *mod;
+    uint traversed = 0;     /* a simple infinite loop break out */
+
+    /* Now, you'd think these would actually be in memory order, but they
+     * don't seem to be for me!
+     */
+    mark = &ldr->InMemoryOrderModuleList;
+
+    for (e = mark->Flink; e != mark; e = e->Flink) {
+        mod = (LDR_MODULE *) ((char *)e -
+                              offsetof(LDR_MODULE, InMemoryOrderModuleList));
+        /* NOTE - for comparison we could use pe_name or mod->BaseDllName.
+         * Our current usage is just to get user32.dll for which BaseDllName
+         * is prob. better (can't rename user32, and a random dll could have
+         * user32.dll as a pe_name).  If wanted to be extra certain could
+         * check FullDllName for %systemroot%/system32/user32.dll as that
+         * should ensure uniqueness. 
+         */
+        ASSERT(mod->BaseDllName.Length <= mod->BaseDllName.MaximumLength &&
+               mod->BaseDllName.Buffer != NULL);
+        if (wcscasecmp(name, mod->BaseDllName.Buffer) == 0) {
+            return mod;
+        }
+
+        if (traversed++ > MAX_MODULE_LIST_INFINITE_LOOP_THRESHOLD) {
+            /* Only caller (preinject) should hold the ldr lock and the ldr
+             * state should be consistent so we don't expect to get stuck. */
+            ASSERT_NOT_REACHED();
+            /* TODO: In case we ever hit this we may want to retry the
+             * traversal once more */
+            return NULL;
+        }
+    }
+    return NULL;
+}
+#endif /* !NOT_DYNAMORIO_CORE */
+
+/****************************************************************************/
+#ifndef X64
+
+/* PR 271719: Access x64 loader data from WOW64.
+ * We duplicate a bunch of data structures and code here, but this is cleaner
+ * than compiling the original code as x64 and hacking the build process to
+ * get it linked: stick as char[] inside code section or something.
+ * We assume that all addresses are in the low 4GB (above 4GB is off-limits
+ * for a WOW64 process), so we can ignore the high dword of each pointer.
+ */
+
+typedef struct ALIGN_VAR(8) _LIST_ENTRY_64 {
+    struct _LIST_ENTRY_64 *Flink;
+    uint Flink_hi;
+    struct _LIST_ENTRY_64 *Blink;
+    uint Blink_hi;
+} LIST_ENTRY_64;
+
+/* UNICODE_STRING_64 is in ntdll.h */
+
+/* module information filled by the loader */
+typedef struct ALIGN_VAR(8) _PEB_LDR_DATA_64 {  
+    ULONG Length;
+    BOOLEAN Initialized;
+    PVOID SsHandle;
+    uint  SsHandle_hi;
+    LIST_ENTRY_64 InLoadOrderModuleList;
+    LIST_ENTRY_64 InMemoryOrderModuleList;
+    LIST_ENTRY_64 InInitializationOrderModuleList;
+} PEB_LDR_DATA_64;
+
+/* Note that these lists are walked through corresponding LIST_ENTRY pointers
+ * i.e., for InInit*Order*, Flink points 16 bytes into the LDR_MODULE structure
+ */
+typedef struct ALIGN_VAR(8) _LDR_MODULE_64 {
+    LIST_ENTRY_64 InLoadOrderModuleList;
+    LIST_ENTRY_64 InMemoryOrderModuleList;
+    LIST_ENTRY_64 InInitializationOrderModuleList;
+    PVOID BaseAddress;
+    uint BaseAddress_hi;
+    PVOID EntryPoint;
+    uint EntryPoint_hi;
+    ULONG SizeOfImage;
+    int padding;
+    UNICODE_STRING_64 FullDllName;
+    UNICODE_STRING_64 BaseDllName;
+    ULONG Flags;
+    SHORT LoadCount;
+    SHORT TlsIndex;
+    LIST_ENTRY_64 HashTableEntry; /* see notes for LDR_MODULE */
+    ULONG TimeDateStamp;
+} LDR_MODULE_64;
+
+static PEB_LDR_DATA_64 *
+get_ldr_data_64(void)
+{
+    /* __readgsqword is not supported for 32-bit */
+    byte *peb64;
+    __asm {
+        mov eax, dword ptr gs:X64_PEB_TIB_OFFSET
+        mov peb64, eax
+    };
+    return *(PEB_LDR_DATA_64 **)(peb64 + X64_LDR_PEB_OFFSET);
+}
+
+/* returns NULL if no loader module is found
+ * N.B.: walking loader data structures at random times is dangerous! See
+ * get_ldr_module_by_pc in module.c for code to grab the ldr lock (which is
+ * also unsafe).  Here we presume that we already own the ldr lock and that
+ * the ldr list is consistent, which should be the case for preinject (the only
+ * user).  FIXME stick this in module.c with get_ldr_module_by_pc, would need
+ * to get module.c compiled into preinjector which is a significant hassle.
+ */
+HANDLE
+get_module_handle_64(wchar_t *name)
+{
+    PEB_LDR_DATA_64 *ldr = get_ldr_data_64();
+    LIST_ENTRY_64 *e, *mark;
+    LDR_MODULE_64 *mod;
+    uint traversed = 0;     /* a simple infinite loop break out */
+
+    /* Now, you'd think these would actually be in memory order, but they
+     * don't seem to be for me!
+     */
+    mark = &ldr->InMemoryOrderModuleList;
+
+    for (e = mark->Flink; e != mark; e = e->Flink) {
+        mod = (LDR_MODULE_64 *) ((char *)e -
+                              offsetof(LDR_MODULE_64, InMemoryOrderModuleList));
+        /* NOTE - for comparison we could use pe_name or mod->BaseDllName.
+         * Our current usage is just to get user32.dll for which BaseDllName
+         * is prob. better (can't rename user32, and a random dll could have
+         * user32.dll as a pe_name).  If wanted to be extra certain could
+         * check FullDllName for %systemroot%/system32/user32.dll as that
+         * should ensure uniqueness. 
+         */
+        ASSERT(mod->BaseDllName.Length <= mod->BaseDllName.MaximumLength &&
+               mod->BaseDllName.Buffer != NULL);
+        if (wcscasecmp(name, mod->BaseDllName.Buffer) == 0) {
+            return (HANDLE) mod->BaseAddress;
+        }
+
+        if (traversed++ > MAX_MODULE_LIST_INFINITE_LOOP_THRESHOLD) {
+            /* Only caller (preinject) should hold the ldr lock and the ldr
+             * state should be consistent so we don't expect to get stuck. */
+            ASSERT_NOT_REACHED();
+            /* TODO: In case we ever hit this we may want to retry the
+             * traversal once more */
+            return NULL;
+        }
+    }
+    return NULL;
+}
+
+/* we return void* since that's easier for preinject and drmarker to deal with */
+void *
+get_proc_address_64(HANDLE lib, const char *name)
+{
+    return (void *) get_proc_address_common(lib, name _IF_NOT_X64(true));
+}
+
+#endif /* !X64 */
+/****************************************************************************/
+
diff --git a/core/win32/ntdll.c b/core/win32/ntdll.c
new file mode 100644
index 000000000..e8259d20a
--- /dev/null
+++ b/core/win32/ntdll.c
@@ -0,0 +1,4727 @@
+/* **********************************************************
+ * Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+
+/*
+ * ntdll.c
+ * Routines for calling Windows system calls via the ntdll.dll wrappers.
+ *
+ * This file is used by the main library, the preinject library, and the
+ * standalone injector.
+ */
+
+#ifdef NOT_DYNAMORIO_CORE
+# define ASSERT(x)
+# define ASSERT_CURIOSITY(x)
+# define ASSERT_NOT_REACHED()
+# define ASSERT_NOT_IMPLEMENTED(x)
+# define DODEBUG(x)
+# define DEBUG_DECLARE(x)
+# pragma warning(disable : 4210) //nonstandard extension used : function given file scope
+# pragma warning( disable : 4204) //nonstandard extension used : non-constant aggregate initializer
+# define INVALID_FILE INVALID_HANDLE_VALUE
+# define snprintf   _snprintf
+# include <stdio.h> /* _snprintf */
+#else
+/* we include globals.h mainly for ASSERT, even though we're
+ * used by preinject.
+ * preinject just defines its own internal_error!
+ */
+# include "globals.h"
+# include "module_shared.h"
+#endif
+
+/* We have to hack away things we use here that won't work for non-core */
+#if defined(NOT_DYNAMORIO_CORE_PROPER) || defined(NOT_DYNAMORIO_CORE)
+# undef ASSERT_OWN_NO_LOCKS
+# define ASSERT_OWN_NO_LOCKS() /* who cares if not the core */
+#endif
+
+#include "ntdll.h"
+#ifndef NOT_DYNAMORIO_CORE
+# include "os_private.h"
+#endif
+
+#include <wchar.h> /* _snwprintf */
+
+/* WARNING: these routines use the Native API, an undocumented API
+ * exported by ntdll.dll.
+ * It could change without warning with a new version of Windows.
+ */
+
+/* FIXME : combine NTPRINT with NTLOG */
+/* must turn on VERBOSE in inject_shared.c as well since we're now
+ * using display_verbose_message() -- FIXME: link them automatically */
+#if defined(NOT_DYNAMORIO_CORE_PROPER) || defined(NOT_DYNAMORIO_CORE)
+# define VERBOSE 0
+#else
+# define VERBOSE 0
+#endif
+
+#if VERBOSE
+/* in inject_shared.c: must turn on VERBOSE=1 there as well */
+void display_verbose_message(char *format, ...);
+# define NTPRINT(...) display_verbose_message(__VA_ARGS__)
+#else
+# define NTPRINT(...)
+#endif
+
+/* needed for injector and preinject, to avoid them requiring asm and syscalls */
+#if defined(NOT_DYNAMORIO_CORE_PROPER) || defined(NOT_DYNAMORIO_CORE)
+/* use ntdll wrappers for simplicity, to keep ntdll.c standalone */
+# define GET_SYSCALL(name, ...)  GET_NTDLL(Nt##name, (__VA_ARGS__))
+# define GET_RAW_SYSCALL GET_SYSCALL
+# define NT_SYSCALL(name, ...)   Nt##name (__VA_ARGS__)
+# define NTLOG(...)
+#else
+# define NTLOG LOG
+/* Our own syscall wrapper to avoid relying on ntdll, for 4 reasons:
+ * 1) Maximum interoperability w/ ntdll hookers
+ * 2) Security by avoiding being disabled via ntdll being messed up
+ * 3) Early injection: although ntdll is already in the address space,
+ *    this way we don't need the loader
+ * 4) Easier trampolines on ntdll syscall wrappers for handling native code
+ *    (don't have to worry about DR syscalls going through the trampolines)
+ *
+ * For now we only use our own wrapper for syscalls in the app-relevant array.
+ * When we add the rest we can:
+ *   1) leave out of array, and dynamically determine
+ *      (ideally using our own version of GetProcAddress)
+ *   2) add to array, then everything's consistent
+ *   3) eliminate array completely and always dynamically determine
+ * But, relying on dynamic determination means we won't work if there's a hook
+ * already placed there (losing a big advantage of our own wrappers), and
+ * dynamically determining doesn't give us that much more independence --
+ * we still need to manually verify each new ntdll for other types of
+ * syscall changes (new syscalls we care about, semantic changes, etc.)
+ */
+/* decides which of dynamorio_syscall_{int2e,sysenter,wow64} to use */
+static enum {
+    DR_SYSCALL_INT2E,
+    DR_SYSCALL_SYSENTER,
+    DR_SYSCALL_SYSCALL,
+    DR_SYSCALL_WOW64,
+} dr_which_syscall_t;
+
+/* For x64 "raw syscalls", i.e., those we call directly w/o invoking the
+ * ntdll wrapper routine, we play some games with types to work more
+ * easily w/ the x64 calling convention:
+ */
+# define GET_RAW_SYSCALL(name, arg1, ...) \
+    GET_NTDLL(Nt##name, (arg1, __VA_ARGS__)); \
+    typedef NTSTATUS name##_type (int sysnum, arg1, __VA_ARGS__); \
+    typedef NTSTATUS name##_dr_type (int sys_enum, __VA_ARGS__, arg1)
+
+# define GET_SYSCALL(name, ...) \
+    GET_NTDLL(Nt##name, (__VA_ARGS__)); \
+    typedef NTSTATUS name##_type (int sysnum, __VA_ARGS__)
+
+/* FIXME - since it doesn't vary we could have a variable to store the dr
+ * syscall routine to use, but would be yet another function pointer in
+ * our data segment... */
+/* We use the wrappers till the native_exec Nt hooks go in (at which point
+ * the options have been read) so that we can have sygate compatibility as a
+ * runtime option. */
+/* For X64 sycall we need the 1st arg last to preserve the rest in their
+ * proper registers.  If we ever support 0-arg syscalls here we'll
+ * need a separate macro for those.
+ * Any syscall called using this macro must be declared with GET_RAW_SYSCALL
+ * rather than GET_SYSCALL to get the types to match up.
+ */
+# define NT_SYSCALL(name, arg1, ...)                 \
+    (nt_wrappers_intercepted ?                       \
+      Nt##name(arg1, __VA_ARGS__) :                  \
+      ((dr_which_syscall_t == DR_SYSCALL_WOW64) ?                                    \
+       (((name##_type *) dynamorio_syscall_wow64) (SYS_##name, arg1, __VA_ARGS__)) : \
+       ((IF_X64_ELSE(dr_which_syscall_t == DR_SYSCALL_SYSCALL, false)) ?             \
+        ((name##_dr_type *) IF_X64_ELSE(dynamorio_syscall_syscall, NULL))            \
+         (SYS_##name, __VA_ARGS__, arg1) :                                           \
+        (((name##_type *) ((dr_which_syscall_t == DR_SYSCALL_SYSENTER) ?             \
+         (DYNAMO_OPTION(dr_sygate_sysenter) ?        \
+          dynamorio_syscall_sygate_sysenter :        \
+          dynamorio_syscall_sysenter) :              \
+         (DYNAMO_OPTION(dr_sygate_int) ?             \
+          dynamorio_syscall_sygate_int2e :           \
+          dynamorio_syscall_int2e)))                 \
+        (syscalls[SYS_##name], arg1,  __VA_ARGS__)))))
+
+/* check syscall numbers without using any heap */
+# ifdef X64
+#  define SYSNUM_OFFS 4
+# else
+#  define SYSNUM_OFFS 1
+# endif
+# define CHECK_SYSNUM_AT(pc, idx) \
+     ASSERT(*((int *)((pc) + SYSNUM_OFFS)) == syscalls[idx] || ALLOW_HOOKER(pc) || \
+            (idx == SYS_TestAlert && *(uint*)(pc) == 0xe9505050)); /* xref 9288 */
+/* assuming relative CTI's are the only one's used by hookers */
+# define ALLOW_HOOKER(pc) (*(unsigned char*)(pc) == JMP_REL32_OPCODE ||      \
+                           *(unsigned char*)(pc) == CALL_REL32_OPCODE)
+/* FIXME: we'll evaluate pc multiple times in the above macro */
+
+#endif /* !NOT_DYNAMORIO_CORE_PROPER */
+
+/****************************************************************************
+ * Defines only needed internally to this file
+ */
+
+/* TlsSlots offset is hardcoded into kernel32!TlsGetValue as 0xe10 on all
+ * 32-bit platforms we've seen, 0x1480 for 64-bit:
+ */
+#ifdef X64
+# define TEB_TLS64_OFFSET 0x1480
+# define PEB_SIZE          0x358
+# define TEB_SIZE         0x17d8
+#else
+# define TEB_TLS64_OFFSET 0xe10
+# define PEB_SIZE         0x230 
+# define TEB_SIZE         0xfbc
+#endif
+
+//
+// Thread Information Classes
+//
+
+typedef enum _THREADINFOCLASS {
+    ThreadBasicInformation,
+    ThreadTimes,
+    ThreadPriority,
+    ThreadBasePriority,
+    ThreadAffinityMask,
+    ThreadImpersonationToken,
+    ThreadDescriptorTableEntry,
+    ThreadEnableAlignmentFaultFixup,
+    ThreadEventPair_Reusable,
+    ThreadQuerySetWin32StartAddress,
+    ThreadZeroTlsCell,
+    ThreadPerformanceCount,
+    ThreadAmILastThread,
+    ThreadIdealProcessor,
+    ThreadPriorityBoost,
+    ThreadSetTlsArrayAddress,
+    ThreadIsIoPending,
+    ThreadHideFromDebugger,
+    MaxThreadInfoClass
+    } THREADINFOCLASS;
+
+typedef enum _SECTION_INFORMATION_CLASS {
+    SectionBasicInformation,
+    SectionImageInformation
+} SECTION_INFORMATION_CLASS;
+
+typedef struct _SECTION_BASIC_INFORMATION { // Information Class 0
+    PVOID BaseAddress;
+    ULONG Attributes;
+    LARGE_INTEGER Size;
+} SECTION_BASIC_INFORMATION, *PSECTION_BASIC_INFORMATION;
+
+typedef struct _SECTION_IMAGE_INFORMATION { // Information Class 1
+    PVOID EntryPoint;
+    ULONG Unknown1;
+    ULONG StackReserve;
+    ULONG StackCommit;
+    ULONG Subsystem;
+    USHORT MinorSubsystemVersion;
+    USHORT MajorSubsystemVersion;
+    ULONG Unknown2;
+    ULONG Characteristics;
+    USHORT ImageNumber;
+    BOOLEAN Executable;
+    UCHAR Unknown3;
+    ULONG Unknown4[3];
+} SECTION_IMAGE_INFORMATION, *PSECTION_IMAGE_INFORMATION;
+
+typedef struct _IO_STATUS_BLOCK {
+    union {
+        NTSTATUS Status;
+        PVOID Pointer;
+    };
+    ULONG_PTR Information;
+} IO_STATUS_BLOCK, *PIO_STATUS_BLOCK;
+
+typedef enum _KPROFILE_SOURCE {
+    ProfileTime
+} KPROFILE_SOURCE;
+
+typedef enum _OBJECT_INFORMATION_CLASS {
+    ObjectBasicInformation,
+    ObjectNameInformation,
+    ObjectTypeInformation,
+    ObjectAllTypesInformation,
+    ObjectHandleInformation /* can be queried and set */
+} OBJECT_INFORMATION_CLASS;
+
+typedef struct _OBJECT_BASIC_INFORMATION {
+    ULONG Attributes;
+    ACCESS_MASK GrantedAccess;
+    ULONG HandleCount;
+    ULONG PointerCount;
+    ULONG PagePoolUsage;
+    ULONG NonPagedPoolUsage;
+    ULONG Reserved[3];
+    ULONG NameInformationLength;
+    ULONG TypeInformationLength;
+    ULONG SecurityDescriptorLength;
+    LARGE_INTEGER CreateTime;
+} OBJECT_BASIC_INFORMATION, *POBJECT_BASIC_INFORMATION;
+
+/***************************************************************************
+ * declarations for ntdll exports shared by several routines in this file
+ */
+
+GET_NTDLL(NtQueryInformationThread, (IN HANDLE ThreadHandle,
+                                     IN THREADINFOCLASS ThreadInformationClass,
+                                     OUT PVOID ThreadInformation,
+                                     IN ULONG ThreadInformationLength,
+                                     OUT PULONG ReturnLength OPTIONAL));
+
+GET_NTDLL(NtQueryInformationProcess, (IN HANDLE ProcessHandle,
+                                      IN PROCESSINFOCLASS ProcessInformationClass,
+                                      OUT PVOID ProcessInformation,
+                                      IN ULONG ProcessInformationLength,
+                                      OUT PULONG ReturnLength OPTIONAL));
+
+GET_NTDLL(NtQueryInformationFile, (IN HANDLE FileHandle,
+                                   OUT PIO_STATUS_BLOCK IoStatusBlock,
+                                   OUT PVOID FileInformation,
+                                   IN ULONG FileInformationLength,
+                                   IN FILE_INFORMATION_CLASS  FileInformationClass));
+
+GET_NTDLL(NtSetInformationFile, (IN HANDLE FileHandle,
+                                 OUT PIO_STATUS_BLOCK IoStatusBlock,
+                                 IN PVOID FileInformation,
+                                 IN ULONG FileInformationLength,
+                                 IN FILE_INFORMATION_CLASS  FileInformationClass));
+
+GET_NTDLL(NtQuerySection, (IN HANDLE SectionHandle,
+                           IN SECTION_INFORMATION_CLASS SectionInformationClass,
+                           OUT PVOID SectionInformation,
+                           IN ULONG SectionInformationLength,
+                           OUT PULONG ResultLength OPTIONAL));
+
+GET_NTDLL(NtOpenFile, (OUT PHANDLE FileHandle,
+                       IN ACCESS_MASK DesiredAccess,
+                       IN POBJECT_ATTRIBUTES ObjectAttributes,
+                       OUT PIO_STATUS_BLOCK IoStatusBlock,
+                       IN ULONG ShareAccess,
+                       IN ULONG OpenOptions));
+
+GET_NTDLL(NtOpenThreadToken, ( IN HANDLE ThreadHandle,
+                               IN ACCESS_MASK DesiredAccess,
+                               IN BOOLEAN OpenAsSelf,
+                               OUT PHANDLE TokenHandle
+                               ));
+GET_NTDLL(NtOpenProcessToken, (IN HANDLE ProcessToken,
+                               IN ACCESS_MASK DesiredAccess,
+                               OUT PHANDLE TokenHandle));
+GET_NTDLL(NtQueryInformationToken, (IN HANDLE TokenHandle,
+                                    IN TOKEN_INFORMATION_CLASS TokenInformationClass,
+                                    OUT PVOID TokenInformation,
+                                    IN ULONG TokenInformationLength,
+                                    OUT PULONG ReturnLength));
+
+
+/* routines that we may hook if specified in
+ * syscall_requires_action[], all new routines can use GET_SYSCALL
+ * instead of GET_NTDLL if we provide the syscall numbers - see
+ * comments in GET_SYSCALL definition.
+ */
+
+GET_RAW_SYSCALL(MapViewOfSection, 
+                IN HANDLE           SectionHandle,
+                IN HANDLE           ProcessHandle,
+                IN OUT PVOID       *BaseAddress,
+                IN ULONG            ZeroBits,
+                IN SIZE_T           CommitSize,
+                IN OUT PLARGE_INTEGER  SectionOffset OPTIONAL,
+                IN OUT PSIZE_T      ViewSize,
+                IN SECTION_INHERIT  InheritDisposition,
+                IN ULONG            AllocationType,
+                IN ULONG            Protect);
+
+GET_RAW_SYSCALL(UnmapViewOfSection,
+                IN HANDLE         ProcessHandle,
+                IN PVOID          BaseAddress);
+
+GET_RAW_SYSCALL(CreateSection, 
+                OUT PHANDLE SectionHandle,
+                IN ACCESS_MASK DesiredAccess,
+                IN POBJECT_ATTRIBUTES ObjectAttributes,
+                IN PLARGE_INTEGER SectionSize OPTIONAL,
+                IN ULONG Protect,
+                IN ULONG Attributes,
+                IN HANDLE FileHandle);
+
+GET_RAW_SYSCALL(OpenSection, 
+                OUT PHANDLE SectionHandle,
+                IN ACCESS_MASK DesiredAccess,
+                IN POBJECT_ATTRIBUTES ObjectAttributes);
+
+GET_RAW_SYSCALL(AllocateVirtualMemory,
+                IN HANDLE ProcessHandle,
+                IN OUT PVOID *BaseAddress,
+                IN ULONG ZeroBits,
+                IN OUT PSIZE_T AllocationSize,
+                IN ULONG AllocationType,
+                IN ULONG Protect);
+
+GET_RAW_SYSCALL(FreeVirtualMemory,
+                IN HANDLE ProcessHandle,
+                IN OUT PVOID *BaseAddress,
+                IN OUT PSIZE_T FreeSize,
+                IN ULONG FreeType);
+
+GET_RAW_SYSCALL(ProtectVirtualMemory,
+                IN HANDLE ProcessHandle,
+                IN OUT PVOID *BaseAddress,
+                IN OUT PSIZE_T ProtectSize,
+                IN ULONG NewProtect,
+                OUT PULONG OldProtect);
+
+GET_RAW_SYSCALL(CreateFile,
+                OUT PHANDLE  FileHandle,
+                IN ACCESS_MASK  DesiredAccess,
+                IN POBJECT_ATTRIBUTES  ObjectAttributes,
+                OUT PIO_STATUS_BLOCK IoStatusBlock,
+                IN PLARGE_INTEGER  AllocationSize  OPTIONAL,
+                IN ULONG  FileAttributes,
+                IN ULONG  ShareAccess,
+                IN ULONG  CreateDisposition,
+                IN ULONG  CreateOptions,
+                IN PVOID  EaBuffer  OPTIONAL,
+                IN ULONG  EaLength);
+
+/***************************************************************************
+ * Implementation
+ */
+
+#if !defined(NOT_DYNAMORIO_CORE_PROPER) && !defined(NOT_DYNAMORIO_CORE)
+/* for Sygate 5441 compatibility hack, we need a tls slot for NT_SYSCALL when
+ * using sysenter system calls */
+uint sysenter_tls_offset = 0xffffffff; /* something that will fault */
+/* will be set to false once the options are read but before the native_exec
+ * Nt* hooks are put in. Till then lets NT_SYSCALL know it's safe to call via
+ * the wrappers for Sygate compatiblity before the option string is read in. */
+static bool nt_wrappers_intercepted = true;
+
+void
+syscalls_init_options_read()
+{
+    if (DYNAMO_OPTION(dr_sygate_sysenter)) {
+        tls_alloc(false /* don't grab lock */, &sysenter_tls_offset);
+    }
+    nt_wrappers_intercepted = false;
+}
+
+/* Called very early, prior to any system call use by us, making error
+ * reporting problematic once we have all syscalls requiring this!
+ * See windows_version_init() comments.
+ * The other problem w/ error reporting is that other code assumes
+ * that things are initialized -- that's all fixed now, with stats, dcontext,
+ * etc. checked for NULL in all the right places.
+ */
+void
+syscalls_init()
+{
+    /* Determine which syscall routine to use 
+     * We don't have heap available yet (no syscalls yet!) so
+     * we can't decode easily.
+     * FIXME: for app syscalls, we wait until we see one so we know
+     * the method being used -- should we move that decision up, since
+     * we're checking here for DR?
+     */
+    /* pick a syscall that is unlikely to be hooked, ref case 5217 Sygate
+     * requires all int system call to occur in ntdll.dll or sysfer.dll
+     * so we borrow the int 2e from NtYieldExecution for system calls!
+     * (both our own and the apps via shared_syscall).  The Nt* wrappers
+     * are stdcall so NtYieldExecution is convenient since it has zero
+     * args and is unlikely to be hooked. Ref case 5441, Sygate also sometimes
+     * verifies the top of the stack for sysenter system calls in a similar
+     * fashion (must be in ntdll/sysfer). For that we again borrow out of
+     * NtYieldExecution (this time just the ret) to fix up our stack. */
+    GET_NTDLL(NtYieldExecution, (VOID));
+    /* N.B.: if we change which syscall, for WOW64 the wrapper can change */
+    app_pc pc = (app_pc) NtYieldExecution;
+    app_pc int_target = pc + 9;
+    ushort check = *((ushort *)(int_target));
+
+    windows_version_init();
+    ASSERT(syscalls != NULL);
+
+    /* FIXME : ref case 5463, we should follow through to actual system call
+     * for sysenter cases to be sure os isn't actually using int because of
+     * old hardware not supporting sysenter */
+    /* check 10th and 11th bytes:
+     *  int 2e: {2k}
+     *    77F97BFA: B8 BA 00 00 00     mov         eax,0BAh
+     *    77F97BFF: 8D 54 24 04        lea         edx,[esp+4]
+     *    77F97C03: CD 2E              int         2Eh
+     *    ret (stdcall)
+     *  sysenter:  {xpsp[0,1] 2k3sp0}
+     *    0x77f7eb23   b8 77 00 00 00       mov    $0x00000077 -> %eax 
+     *    0x77f7eb28   ba 00 03 fe 7f       mov    $0x7ffe0300 -> %edx 
+     *    0x77f7eb2d   ff d2                call   %edx
+     *    ret (stdcall)
+     *  sysenter:  {xpsp2 2k3sp1}
+     *    0x77f7eb23   b8 77 00 00 00       mov    $0x00000077 -> %eax 
+     *    0x77f7eb28   ba 00 03 fe 7f       mov    $0x7ffe0300 -> %edx 
+     *    0x77f7eb2d   ff 12                call   [%edx]
+     *    ret (stdcall)
+     *  wow64 xp64 (case 3922):
+     *    7d61ce3f b843000000       mov     eax,0x43
+     *    7d61ce44 b901000000       mov     ecx,0x1
+     *    7d61ce49 8d542404         lea     edx,[esp+0x4]
+     *    7d61ce4d 64ff15c0000000   call    dword ptr fs:[000000c0]
+     *    7d61ce54 c3               ret
+     * x64 syscall (PR 215398):
+     *    00000000`78ef16c0 4c8bd1          mov     r10,rcx
+     *    00000000`78ef16c3 b843000000      mov     eax,43h
+     *    00000000`78ef16c8 0f05            syscall
+     *    00000000`78ef16ca c3              ret
+     */
+    if (check == 0x2ecd) {
+        dr_which_syscall_t = DR_SYSCALL_INT2E;
+        set_syscall_method(SYSCALL_METHOD_INT);
+        int_syscall_address = int_target;
+        /* ASSERT is simple ret (i.e. 0 args) */
+        ASSERT(*(byte *)(int_target + 2) == 0xc3 /* ret 0 */);
+    } else if (check == 0x8d00) {
+        ASSERT(is_wow64_process(NT_CURRENT_PROCESS));
+        dr_which_syscall_t = DR_SYSCALL_WOW64;
+        set_syscall_method(SYSCALL_METHOD_WOW64);
+        /* FIXME: check for and support other platforms */
+        wow64_index = (int *) windows_XP_wow64_index;
+        ASSERT(*((uint *)(int_target+5)) == 0xc015ff64);
+        ASSERT(*((uint *)(int_target+8)) == WOW64_TIB_OFFSET);
+        DODEBUG({
+            /* We assume syscalls go through teb->WOW32Reserved */
+            TEB *teb = get_own_teb();
+            ASSERT(teb != NULL && teb->WOW32Reserved != NULL);
+        });
+#ifdef X64 /* PR 205898 covers 32-bit syscall support */
+    } else if (check == 0xc305) {
+        dr_which_syscall_t = DR_SYSCALL_SYSCALL;
+        set_syscall_method(SYSCALL_METHOD_SYSCALL);
+        /* ASSERT is syscall */
+        ASSERT(*(byte *)(int_target - 1) == 0x0f);
+#endif
+    } else {
+        ASSERT(check == 0xff7f);
+        /* verifiy is call %edx or call [%edx] followed by ret 0 [0xc3] */
+        ASSERT(*((ushort *)(int_target+2)) == 0xc3d2 ||
+               *((ushort *)(int_target+2)) == 0xc312);
+        /* Double check use_ki_syscall_routines() matches type of ind call used */
+        ASSERT((!use_ki_syscall_routines() && *((ushort *)(int_target+1)) == 0xd2ff) ||
+               (use_ki_syscall_routines() && *((ushort *)(int_target+1)) == 0x12ff));
+        /* verify VSYSCALL_BOOTSTRAP_ADDR */
+        IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+        ASSERT(*((uint *)(int_target-3)) == (uint)(ptr_uint_t)VSYSCALL_BOOTSTRAP_ADDR);
+        sysenter_ret_address = (app_pc)int_target+3; /* save addr of ret */
+        set_syscall_method(SYSCALL_METHOD_SYSENTER);
+        dr_which_syscall_t = DR_SYSCALL_SYSENTER;
+    }
+    
+    /* Prime use_ki_syscall_routines() */
+    use_ki_syscall_routines();
+
+    /* quick sanity check that the syscall numbers we care about are what's
+     * in our static array.  we still do our later full-decode sanity checks.
+     */
+    DODEBUG({
+        int i;
+        HMODULE h = get_ntdll_base();
+        ASSERT(h != NULL);
+        for (i = 0; i < SYS_MAX; i++) {
+            if (syscalls[i] == SYSCALL_NOT_PRESENT)
+                continue;
+            /* note that this check allows a hooker so we'll need a
+             * better way of determining syscall numbers 
+             */
+            CHECK_SYSNUM_AT((byte *) get_proc_address(h, syscall_names[i]), i);
+        }
+    });
+}
+
+/* Returns true if machine is using the Ki*SysCall routines (indirection via vsyscall
+ * page), false otherwise. */
+bool
+use_ki_syscall_routines()
+{
+    /* FIXME - two ways to do this.  We could use the byte matching above in
+     * syscalls_init to match call edx vs call [edx] or we could check for the 
+     * existence of the Ki*SystemCall* routines.  We do the latter and have
+     * syscalls_init assert that the two methods agree. */
+    /* We use KiFastSystemCall, but KiIntSystemCall and KiFastSystemCallRet would
+     * work just as well. */
+    static generic_func_t ki_fastsyscall_addr = (generic_func_t)PTR_UINT_MINUS_1;
+    if (ki_fastsyscall_addr == (generic_func_t)PTR_UINT_MINUS_1) {
+        ki_fastsyscall_addr = get_proc_address(get_ntdll_base(), "KiFastSystemCall");
+        ASSERT(ki_fastsyscall_addr != (generic_func_t)PTR_UINT_MINUS_1);
+    }
+    return (ki_fastsyscall_addr != NULL);
+}
+#endif /* !NOT_DYNAMORIO_CORE_PROPER */
+
+void
+ntdll_init()
+{
+    /* FIXME: decode kernel32!TlsGetValue and get the real offset
+     * from there?
+     */
+    ASSERT(offsetof(TEB, TlsSlots) == TEB_TLS64_OFFSET);
+    /* Ensure we have all our other types right */
+    ASSERT(sizeof(TEB) == TEB_SIZE);
+    ASSERT(sizeof(PEB) == PEB_SIZE);
+}
+
+/* note that this function is called even on the release fast exit path 
+ * (via os_exit) and thus should only do neccesary cleanup without ifdef
+ * DEBUG, but also be carefull about ifdef DEBUG since Detach wants to remove
+ * as much of us as possible
+ */
+void
+ntdll_exit()
+{
+}
+
+
+/* export this if needed elsewhere */
+static NTSTATUS
+query_thread_info(HANDLE h, THREAD_BASIC_INFORMATION *info)
+{
+    NTSTATUS res;
+    ULONG got;
+    memset(info, 0, sizeof(THREAD_BASIC_INFORMATION));
+    res = NtQueryInformationThread(h, ThreadBasicInformation,
+                                   info, sizeof(THREAD_BASIC_INFORMATION), &got);
+    ASSERT(!NT_SUCCESS(res) || got == sizeof(THREAD_BASIC_INFORMATION));
+    return res;
+}
+
+/* Get a segment descriptor. This code assumes the selector is set
+ * appropriately in entry->Selector */
+NTSTATUS
+query_seg_descriptor(HANDLE hthread, DESCRIPTOR_TABLE_ENTRY *entry)
+{
+    NTSTATUS res;
+    ULONG got;
+    res = NtQueryInformationThread(hthread, ThreadDescriptorTableEntry,
+                                   entry, sizeof(DESCRIPTOR_TABLE_ENTRY), &got);
+
+    /* This call only writes the LDT_ENTRY portion of the table entry */
+    ASSERT(!NT_SUCCESS(res) || got == sizeof(LDT_ENTRY));
+    return res;
+}
+
+/* Get a win32 start address.  NOTE: According to Nebbet, the value
+ * retrieved with ThreadQuerySetWin32StartAddress is invalid if the
+ * thread has call ZwReplyWaitReplyPort or ZwReplyWaitReceivePort.
+ */
+NTSTATUS
+query_win32_start_addr(HANDLE hthread, PVOID start_addr)
+{
+    NTSTATUS res;
+    ULONG got;
+    res = NtQueryInformationThread(hthread, ThreadQuerySetWin32StartAddress,
+                                   start_addr, sizeof(app_pc), &got);    
+    ASSERT(!NT_SUCCESS(res) || got == sizeof(PVOID));
+    return res;
+}
+
+/* Collects system information available through the NtQuerySystemInformation
+ * system call.
+ */
+NTSTATUS
+query_system_info(IN SYSTEM_INFORMATION_CLASS info_class,
+                  IN int info_size,
+                  OUT PVOID info)
+{
+    NTSTATUS result;
+    ULONG bytes_received = 0;
+
+    GET_NTDLL(NtQuerySystemInformation, (IN  SYSTEM_INFORMATION_CLASS info_class,
+                                         OUT PVOID  info, 
+                                         IN  ULONG  info_size, 
+                                         OUT PULONG bytes_received));
+   
+    result = NtQuerySystemInformation(info_class, info, info_size, &bytes_received);
+
+    return result;
+}
+
+/* since not exporting get_own_teb() */
+#ifndef NOT_DYNAMORIO_CORE
+thread_id_t 
+get_thread_id()
+{
+    return (thread_id_t) get_own_teb()->ClientId.UniqueThread;
+}
+
+process_id_t
+get_process_id()
+{
+    return (process_id_t) get_own_teb()->ClientId.UniqueProcess;
+}
+
+int
+get_last_error()
+{
+    return get_own_teb()->LastErrorValue;
+}
+
+void
+set_last_error(int error)
+{
+    get_own_teb()->LastErrorValue = error;
+}
+#endif /* !NOT_DYNAMORIO_CORE */
+
+HANDLE
+get_stderr_handle()
+{
+    HANDLE herr = get_own_peb()->ProcessParameters->StdErrorHandle;
+    if (herr == NULL)
+        return INVALID_HANDLE_VALUE;
+    return herr;
+}
+
+HANDLE
+get_stdout_handle()
+{
+    HANDLE hout = get_own_peb()->ProcessParameters->StdOutputHandle;
+    if (hout == NULL)
+        return INVALID_HANDLE_VALUE;
+    return hout;
+}
+
+thread_exited_status_t
+is_thread_exited(HANDLE hthread)
+{
+    LARGE_INTEGER timeout;
+    wait_status_t result;
+    /* Keep the timeout small, just want to check if signaled. Don't want to wait at all
+     * really, but no way to specify that. Note negative => relative time offset (so is
+     * a 1 millisecond timeout). */
+    timeout.QuadPart = -((int)1 * TIMER_UNITS_PER_MILLISECOND);
+
+    if (thread_id_from_handle(hthread) == (thread_id_t)PTR_UINT_MINUS_1) {
+        /* not a thread handle */
+        ASSERT(false && "Not a valid thread handle.");
+        return THREAD_EXIT_ERROR;
+    }
+    if (!TEST(SYNCHRONIZE, nt_get_handle_access_rights(hthread))) {
+        /* Note that our own thread handles will have SYNCHRONIZE since, like
+         * THREAD_TERMINATE, that seems to be a right the thread can always get for
+         * itself (prob. due to how stacks are freed). So only a potential issue with
+         * app handles for which we try to dup with the required rights. xref 9529 */
+        HANDLE ht = INVALID_HANDLE_VALUE;
+        NTSTATUS res = duplicate_handle(NT_CURRENT_PROCESS, hthread, NT_CURRENT_PROCESS,
+                                        &ht, SYNCHRONIZE, 0, 0);
+        if (!NT_SUCCESS(res)) {
+            ASSERT_CURIOSITY(false && "Unable to check if thread has exited.");
+            return THREAD_EXIT_ERROR;
+        }
+        result = nt_wait_event_with_timeout(ht, &timeout);
+        close_handle(ht);
+    } else {
+        result = nt_wait_event_with_timeout(hthread, &timeout);
+    }
+    if (result == WAIT_SIGNALED)
+        return THREAD_EXITED;
+    if (result == WAIT_TIMEDOUT)
+        return THREAD_NOT_EXITED;
+    ASSERT(result == WAIT_ERROR);
+    ASSERT_CURIOSITY(false && "is_thread_exited() unknown error");
+    return THREAD_EXIT_ERROR;
+}
+
+/* The other ways to get thread info, like OpenThread and Toolhelp, don't
+ * let you go from handle to id (remember handles can be duplicated and
+ * there's no way to tell equivalence), plus are only on win2k.
+ * Returns POINTER_MAX on failure
+ */
+thread_id_t
+thread_id_from_handle(HANDLE h)
+{
+    THREAD_BASIC_INFORMATION info;
+    NTSTATUS res = query_thread_info(h, &info);
+    if (!NT_SUCCESS(res))
+        return POINTER_MAX;
+    else
+        return (thread_id_t) info.ClientId.UniqueThread;
+}
+
+/* export this if needed elsewhere */
+static NTSTATUS
+query_process_info(HANDLE h, PROCESS_BASIC_INFORMATION *info)
+{
+    NTSTATUS res;
+    ULONG got;
+
+    memset(info, 0, sizeof(PROCESS_BASIC_INFORMATION));
+    res = NtQueryInformationProcess(h, ProcessBasicInformation,
+                                    info, sizeof(PROCESS_BASIC_INFORMATION), &got);
+    ASSERT(!NT_SUCCESS(res) || got == sizeof(PROCESS_BASIC_INFORMATION));
+    return res;
+}
+
+/* Returns POINTER_MAX on failure */
+process_id_t
+process_id_from_handle(HANDLE h)
+{
+    PROCESS_BASIC_INFORMATION info;
+    NTSTATUS res = query_process_info(h, &info);
+    if (!NT_SUCCESS(res))
+        return POINTER_MAX;
+    else
+        return (process_id_t) info.UniqueProcessId;
+}
+
+/* Returns POINTER_MAX on failure */
+process_id_t
+process_id_from_thread_handle(HANDLE h)
+{
+    THREAD_BASIC_INFORMATION info;
+    NTSTATUS res = query_thread_info(h, &info);
+    if (!NT_SUCCESS(res))
+        return POINTER_MAX;
+    else
+        return (process_id_t) info.ClientId.UniqueProcess;
+}
+
+/* PEB:
+ * for a running thread this is stored at fs:[30h] 
+ * it's always at 0x7FFDF000 according to InsideWin2k p.290
+ * but that's out of date, is randomized within 0x7ffd... on XPsp2
+ * so use query_process_info to get it
+ */
+PEB *
+get_peb(HANDLE h)
+{
+    PROCESS_BASIC_INFORMATION info;
+    NTSTATUS res = query_process_info(h, &info);
+    if (!NT_SUCCESS(res))
+        return NULL;
+    else
+        return info.PebBaseAddress;
+}
+
+PEB *
+get_own_peb()
+{
+    /* alt. we could use get_own_teb->PEBptr, but since we're remembering the
+     * results of the first lookup doesn't really gain us much */
+    static PEB *own_peb;
+    if (own_peb == NULL) {
+        own_peb = get_peb(NT_CURRENT_PROCESS);
+        ASSERT(own_peb != NULL);
+    }
+    return own_peb;
+}
+
+/****************************************************************************/
+#ifndef NOT_DYNAMORIO_CORE
+/* avoid needing CXT_ macros and SELF_TIB_OFFSET from os_exports.h */
+
+TEB *
+get_teb(HANDLE h)
+{
+    THREAD_BASIC_INFORMATION info;
+    NTSTATUS res = query_thread_info(h, &info);
+    if (!NT_SUCCESS(res))
+        return NULL;
+    else
+        return (TEB *)info.TebBaseAddress;
+}
+
+TEB *
+get_own_teb()
+{
+    return (TEB *) get_tls(SELF_TIB_OFFSET);
+}
+
+/* FIXME: this can be done by simply walking the PEB */
+void *
+get_ntdll_base(void)
+{
+    static app_pc ntdll_base;
+    if (ntdll_base == NULL) {
+        ntdll_base = (app_pc)get_module_handle(L"ntdll.dll");
+        ASSERT(ntdll_base != NULL);
+    }
+    return ntdll_base;
+}
+
+/* routines for conversion between CONTEXT and dr_mcontext_t */
+/* assumes our segment registers are the same as the app and that
+   we never touch floating-point state and debug registers 
+*/
+/* all we need is CONTEXT_INTEGER and non-segment CONTEXT_CONTROL,
+ * and for PR 264138 we need the XMM registers
+ */
+void
+context_to_mcontext(dr_mcontext_t *mcontext, CONTEXT* cxt)
+{
+    ASSERT(TESTALL(CONTEXT_DR_STATE, cxt->ContextFlags));
+    /* CONTEXT_INTEGER */
+    mcontext->xax    = cxt->CXT_XAX;
+    mcontext->xbx    = cxt->CXT_XBX;
+    mcontext->xcx    = cxt->CXT_XCX;
+    mcontext->xdx    = cxt->CXT_XDX;
+    mcontext->xsi    = cxt->CXT_XSI;
+    mcontext->xdi    = cxt->CXT_XDI;
+#ifdef X64
+    mcontext->r8     = cxt->R8;
+    mcontext->r9     = cxt->R9;
+    mcontext->r10    = cxt->R10;
+    mcontext->r11    = cxt->R11;
+    mcontext->r12    = cxt->R12;
+    mcontext->r13    = cxt->R13;
+    mcontext->r14    = cxt->R14;
+    mcontext->r15    = cxt->R15;
+#endif
+    if (CONTEXT_PRESERVE_XMM) { /* no harm done if no sse support */
+        /* CONTEXT_FLOATING_POINT or CONTEXT_EXTENDED_REGISTERS */
+        memcpy(&mcontext->xmm[0], CXT_XMM(cxt, 0), XMM_SAVED_SIZE);
+    }
+    /* CONTEXT_CONTROL without the segments */
+    mcontext->xbp    = cxt->CXT_XBP;
+    mcontext->xsp    = cxt->CXT_XSP;
+    mcontext->xflags = cxt->CXT_XFLAGS;
+    mcontext->pc     = (app_pc) cxt->CXT_XIP; /* including XIP */
+}
+
+void
+mcontext_to_context(CONTEXT* cxt, dr_mcontext_t *mcontext)
+{
+    ASSERT(TESTALL(CONTEXT_DR_STATE, cxt->ContextFlags));
+    /* CONTEXT_INTEGER */
+    cxt->CXT_XAX    = mcontext->xax;
+    cxt->CXT_XBX    = mcontext->xbx;
+    cxt->CXT_XCX    = mcontext->xcx;
+    cxt->CXT_XDX    = mcontext->xdx;
+    cxt->CXT_XSI    = mcontext->xsi;
+    cxt->CXT_XDI    = mcontext->xdi;
+#ifdef X64
+    cxt->R8         = mcontext->r8;
+    cxt->R9         = mcontext->r9;
+    cxt->R10        = mcontext->r10;
+    cxt->R11        = mcontext->r11;
+    cxt->R12        = mcontext->r12;
+    cxt->R13        = mcontext->r13;
+    cxt->R14        = mcontext->r14;
+    cxt->R15        = mcontext->r15;
+#endif
+    if (CONTEXT_PRESERVE_XMM) { /* no harm done if no sse support */
+        /* CONTEXT_FLOATING_POINT or CONTEXT_EXTENDED_REGISTERS */
+        memcpy(CXT_XMM(cxt, 0), &mcontext->xmm[0], XMM_SAVED_SIZE);
+    }
+    /* CONTEXT_CONTROL without the segments */
+    cxt->CXT_XBP    = mcontext->xbp;
+    cxt->CXT_XSP    = mcontext->xsp;
+    IF_X64(ASSERT_TRUNCATE(cxt->CXT_XFLAGS, uint, mcontext->xflags));
+    cxt->CXT_XFLAGS = (uint) mcontext->xflags;
+    cxt->CXT_XIP    = (ptr_uint_t)mcontext->pc; /* including XIP */
+}
+#endif /* !NOT_DYNAMORIO_CORE */
+/****************************************************************************/
+
+/****************************************************************************/
+#if !defined(NOT_DYNAMORIO_CORE_PROPER) && !defined(NOT_DYNAMORIO_CORE)
+/* avoid needing x86_code.c from x86.asm from get_own_context_helper(),
+
+/* unstatic for use by GET_OWN_CONTEXT macro */
+void
+get_own_context_integer_control(CONTEXT *cxt, reg_t cs, reg_t ss,
+                                dr_mcontext_t *mc)
+{
+    /* We could change the parameter types to cxt_seg_t, but the args
+     * passed by get_own_context_helper() in x86.asm are best simply
+     * widened in passing
+     */
+    DEBUG_DECLARE(uint origflags = cxt->ContextFlags;)
+    IF_X64(ASSERT_TRUNCATE(cxt->SegCs, short, cs));
+    cxt->SegCs = (WORD) cs; /* FIXME : need to sanitize? */
+    IF_X64(ASSERT_TRUNCATE(cxt->SegSs, short, ss));
+    cxt->SegSs = (WORD) ss;
+    /* avoid assert in mcontext_to_context about not having xmm flags.
+     * get rid of this once we implement PR 266070. */
+    DODEBUG({ cxt->ContextFlags = CONTEXT_DR_STATE; });
+    mcontext_to_context(cxt, mc);
+    DODEBUG({ cxt->ContextFlags = origflags; });
+}
+
+/* don't call this directly, use GET_OWN_CONTEXT macro instead (it fills
+ * in CONTEXT_INTEGER and CONTEXT_CONTROL values) */
+void
+get_own_context(CONTEXT *cxt)
+{
+    if (TEST(CONTEXT_SEGMENTS, cxt->ContextFlags)) {
+        get_segments_defg(&cxt->SegDs, &cxt->SegEs, &cxt->SegFs, &cxt->SegGs);
+    }
+    /* FIXME : do we want CONTEXT_DEBUG_REGISTERS or CONTEXT_FLOATING_POINT 
+     * or CONTEXT_EXTENDED_REGISTERS at some point?
+     * Especially in light of PR 264138.  However, no current uses need
+     * to get our own xmm registers.
+     */
+    ASSERT_NOT_IMPLEMENTED((cxt->ContextFlags & 
+                            ~(CONTEXT_SEGMENTS|CONTEXT_INTEGER|CONTEXT_CONTROL)) 
+                           == 0);
+}
+
+static inline uint
+tls_segment_offs(int slot)
+{
+    return (uint) (offsetof(TEB, TlsSlots) + slot * sizeof(uint *));
+}
+
+/* returns the first block sequence of num_slots found either bottom
+ * up or top_down, that has the selected slot aligned to given alignment.
+ * Returns -1 on failure to find properly aligned sequence.
+ *
+ * Note that if we only want the whole sequence to fit in a cache line, callers 
+ * should try either align_which_slot for either first or last. 
+ */
+static int
+tls_find_free_block_sequence(byte *rtl_bitmap, int bitmap_size,
+                             int num_requested_slots, bool top_down,
+                             int align_which_slot, /* 0 based index */
+                             uint alignment)
+{
+    /* note: bitmap_find_set_block_sequence() works similarly on our
+     * internal bitmap_t which starts initialized to 0
+     */
+    uint *p = (uint*)rtl_bitmap; /* we access in 32-bit words */
+    int start, open_end;
+    int step;                   /* +/- 1 */
+    int i;
+    int contig = 0;
+    int result;
+
+    ASSERT(ALIGNED(rtl_bitmap, sizeof(uint))); /* they promised */
+    ASSERT_CURIOSITY(bitmap_size == 64);
+    ASSERT(num_requested_slots < bitmap_size);
+    ASSERT_CURIOSITY(alignment < 256);
+    ASSERT(align_which_slot >= 0 && /* including after last */
+           align_which_slot <= num_requested_slots); 
+
+    if (top_down) {
+        start = bitmap_size - 1;
+        open_end = -1;          /* 0 included */
+        step = -1;
+    } else {
+        start = 0;
+        open_end = bitmap_size;
+        step = +1;
+    }
+
+    for (i = start; i != open_end; i += step) {
+        uint taken = p[i/32] & (1 << (i % 32));
+        NTPRINT("tls slot %d is %d\n", i, taken);
+        if (!taken) {
+            if (contig == 0) {
+                /* check whether first element will be aligned */
+                /* don't bother starting if not */
+                /* FIXME: could add an argument which slot should be aligned here  */
+                int proposed_align_slot = /* first */
+                    (top_down ? i - (num_requested_slots - 1) : 
+                                i)
+                    + align_which_slot;
+                /* ALIGNED doesn't work for 0 so we have to special-case it */
+                bool aligned = (alignment == 0 || 
+                                ALIGNED(tls_segment_offs(proposed_align_slot), alignment));
+
+                NTPRINT("\t => @ "PFX", pivot "PFX" %saligned to 0x%x\n",
+                        tls_segment_offs(i),
+                        tls_segment_offs(proposed_align_slot), 
+                        aligned ? "" : "not ", alignment);
+
+                if (aligned)
+                    contig++;
+                else
+                    contig = 0; /* try at next */
+            } else
+                contig++;
+            NTPRINT("\t => %d contig @ "PFX"\n", contig, tls_segment_offs(i));
+            ASSERT(contig <= num_requested_slots);
+            if (contig == num_requested_slots)
+                break;
+        } else {
+            contig = 0; /* start over! */
+        }
+    }
+    if (contig < num_requested_slots) {
+        result = -1; /* failure */
+    } else {
+        result = top_down ? i : i - (num_requested_slots - 1);
+        ASSERT(i >= 0 && i < bitmap_size);
+        /* ALIGNED doesn't work for 0 so we have to special-case it */
+        ASSERT(alignment == 0 || 
+               ALIGNED(tls_segment_offs(result + align_which_slot), alignment));
+    }
+    return result;
+}
+
+static void
+tls_mark_taken_block_sequence(byte *rtl_bitmap, int bitmap_size,
+                              int first_slot, int last_slot_open_end)
+{
+    int i;
+    uint *p = (uint*)rtl_bitmap; /* we access in 32-bit words */
+
+    ASSERT(ALIGNED(rtl_bitmap, sizeof(uint))); /* they promised */
+    ASSERT(first_slot > 0 && last_slot_open_end <= bitmap_size);
+    for (i = first_slot; i < last_slot_open_end; i++)
+        p[i/32] |= (1 << (i % 32));
+}
+
+/* Our version of kernel32's TlsAlloc
+ * If synch is false, assumes that the peb lock does not need to be obtained,
+ * which may be safer than acquiring the lock, though when there's only a single
+ * thread it shouldn't make any difference (it's a recursive lock).
+ */
+static int
+tls_alloc_helper(int synch, uint *teb_offs /* OUT */, int num_slots, 
+                 uint alignment, uint tls_flags)
+{
+    PEB *peb = get_own_peb();
+    uint start;
+
+    NTSTATUS res;
+    GET_NTDLL(RtlEnterCriticalSection, (IN OUT RTL_CRITICAL_SECTION *crit));
+    GET_NTDLL(RtlLeaveCriticalSection, (IN OUT RTL_CRITICAL_SECTION *crit));
+    if (synch) {
+        /* FIXME: I read somewhere they are removing more PEB pointers in Vista or earlier..  */
+        /* TlsAlloc calls RtlAcquirePebLock which calls RtlEnterCriticalSection */
+        res = RtlEnterCriticalSection(peb->FastPebLock);
+        if (!NT_SUCCESS(res))
+            return 0;
+    }
+
+    /* we align the fs offset and assume that the fs base is page-aligned */
+    ASSERT(alignment < PAGE_SIZE);
+    
+    /* Transparency notes: we doubt any app relies on a particular slot to be available.
+     * These are dynamic TLS slots, after all, used only for dlls, who don't know
+     * which other dlls may be in the address space.  The app is going to use
+     * static TLS.  Furthermore, NT only has 64 slots available, so it's unlikely
+     * an app uses up all the available TLS slots (though we have to have one that's
+     * in the TEB itself, meaning one of the first 64).
+     * We walk backward in an attempt to not disrupt the dynamic sequence if only a
+     * few are in use.
+     */
+    /* case 6770: SQL Server 2005 broke most of the above assumptions:
+     * - it allocates 38 TLS entries and expects them to all be in
+     * TLS64 furthermore it assumes that 38 consecutive calls to
+     * TlsAlloc() return consecutive TLS slots.  Therefore we should
+     * have to make sure we do not leave any slots in a shorter
+     * earlier sequence available.  Although SQL can't handle going
+     * into the TlsExpansionBitMap 
+     */
+
+    ASSERT(peb->TlsBitmap != NULL);
+    /* TlsBitmap always points to next field, TlsBitmapBits, but we'll only
+     * use the pointer for generality
+     */
+    ASSERT(&peb->TlsBitmapBits == (void*)peb->TlsBitmap->BitMapBuffer);
+
+    DODEBUG({
+        int first_available = tls_find_free_block_sequence(peb->TlsBitmap->BitMapBuffer, 
+                                                           peb->TlsBitmap->SizeOfBitMap,
+                                                           1, /* single */
+                                                           false, /* bottom up */
+                                                           0, 0 /* no alignment */);
+        /* On XP ntdll seems to grab slot 0 of the TlsBitmap before loading
+         * kernel32, see if early injection gets us before that */
+        /* On Win2k usually first_available == 0, but not in some
+         * runall tests, so can't assert on the exact value */
+        ASSERT_CURIOSITY(first_available >= 0);
+    });
+
+    /* only when filling need to find a first_empty in release */
+    /* TLS_FLAG_BITMAP_FILL - should first find a single slot
+     * available, then look for whole sequence, then should go through
+     * and mark ALL entries inbetween.  Of course we know we can't go
+     * beyond index 63 in either request.
+     */
+    if (TEST(TLS_FLAG_BITMAP_FILL, tls_flags)) {
+        uint first_to_fill = tls_find_free_block_sequence(peb->TlsBitmap->BitMapBuffer, 
+                                                          peb->TlsBitmap->SizeOfBitMap,
+                                                          1, /* single */
+                                                          false, /* bottom up */
+                                                          0, 0 /* no alignment */);
+        ASSERT_NOT_TESTED();
+        /* we only fill from the front - and taking all up to the top isn't nice */
+        ASSERT(!TEST(TLS_FLAG_BITMAP_TOP_DOWN, tls_flags));
+        ASSERT_NOT_IMPLEMENTED(false);
+        /* FIXME: need to save first slot, so we can free the
+         * filled slots on exit */
+    }
+
+    /* TLS_FLAG_BITMAP_TOP_DOWN will take a slot at end if possible
+     * for better transparency, also for better reproducibility */
+
+    /* TLS_FLAG_CACHE_LINE_START - will align the first entry,
+     * otherwise align either first or last since we only care to fit
+     * on a line */
+    /* FIXME: align at specific element - not necessary since not
+     * aligning at all works well for our current choice
+     */
+
+    /*  Note the TLS64 is at fs:[0xe10-0xf10)
+     *  0xf00 is a cache line start for either 32 or 64 byte
+     *
+     *  If we want to have commonly used items on the same cache line,
+     *  but also could care about starting at its beginning (not
+     *  expected to matter for data but should measure).
+     *
+     *  If we only needed 4 slots 0xf00 would be at a cache line start
+     *  and satisfy all requirements.
+     *
+     *  If we can get not so important items to cross the line, then
+     *  we can have 0xf00 as the balancing item, and the previous 8
+     *  slots will be in one whole cache line on both 32 and 64 byte.
+     *  If we keep it at that then we don't really need alignment hint
+     *  at all - grabbing last is good enough.
+     *
+     *  Only on P4 we can fit more than 8 entries on the same cache
+     *  line if presumed to all be hot, then we have to use 0xec0 as
+     *  start and leave empty the 0xf00 line.  On P3 however we can
+     *  use 0xee0 - only in DEBUG=+HASHTABLE_STATISTICS we use one
+     *  extra slot that ends up at 0xec0.  The minor point for P4 is
+     *  then whether we use the first 12 or the last 12 slots in the
+     *  cache line.
+     */
+
+    /* FIXME: cache line front, otherwise should retry when either
+     * start or end is fine, and choose closest to desired end of
+     * bitmap */
+    start = tls_find_free_block_sequence(peb->TlsBitmap->BitMapBuffer, 
+                                         peb->TlsBitmap->SizeOfBitMap,
+                                         num_slots,
+                                         TEST(TLS_FLAG_BITMAP_TOP_DOWN, tls_flags),
+                                         0 /* align first element */, 
+                                         alignment);
+
+    if (!TEST(TLS_FLAG_CACHE_LINE_START, tls_flags)) {
+        /* try either way, worthwhile only if we fit into an alignment unit */
+        uint end_aligned = 
+            tls_find_free_block_sequence(peb->TlsBitmap->BitMapBuffer, 
+                                         peb->TlsBitmap->SizeOfBitMap,
+                                         num_slots,
+                                         TEST(TLS_FLAG_BITMAP_TOP_DOWN, tls_flags),
+                                         /* align the end of last
+                                          * element, so open ended */ num_slots, 
+                                         alignment);
+        if (start < 0) {
+            ASSERT_NOT_TESTED();
+            start = end_aligned;
+        } else {
+            if (TEST(TLS_FLAG_BITMAP_TOP_DOWN, tls_flags)) {
+                /* prefer latest start */
+                if (start < end_aligned) {
+                    start = end_aligned;
+                    ASSERT_NOT_TESTED();
+                }
+            } else {
+                /* bottom up, prefer earlier start */
+                if (start > end_aligned) {
+                    start = end_aligned;
+                }
+            }
+        }
+    }
+
+    if (start < 0) {
+        ASSERT_NOT_TESTED();
+        NTPRINT("Failed to find %d slots aligned at %d\n", num_slots, alignment);
+        goto tls_alloc_exit;
+    }
+
+    tls_mark_taken_block_sequence(peb->TlsBitmap->BitMapBuffer,
+                                  peb->TlsBitmap->SizeOfBitMap,
+                                  start,
+                                  /* FIXME: TLS_FLAG_BITMAP_FILL should use first_to_fill */
+                                  start + num_slots);
+
+    if (teb_offs != NULL) {
+        *teb_offs = tls_segment_offs(start);
+        /* mostly safe since using the small TLS map (of 64 entries)
+         * and that is on TEB so reachable with a short */
+        /* to avoid ASSERT_TRUNCATE in os_tls_offset() checking here */
+        ASSERT_TRUNCATE(ushort, ushort, *teb_offs);
+        NTPRINT("Taking %d tls slot(s) %d-%d at offset 0x%x\n", num_slots, start, start + num_slots, *teb_offs);
+    }
+
+    DODEBUG({
+        int first_available = 
+            tls_find_free_block_sequence(peb->TlsBitmap->BitMapBuffer, 
+                                         peb->TlsBitmap->SizeOfBitMap,
+                                         1, /* single */
+                                         false, /* bottom up */
+                                         0, 0 /* no alignment */);
+        ASSERT(first_available >= 0);
+
+        /* SQL2005 assumes that first available slot means start of a
+         * sequence of 38 blanks that fit in TLS64.  Unfortunately
+         * can't assert this for all processes, since even for make
+         * progrun (notepad on XP SP2, late injection) 16 bits are
+         * already taken by others.  Worse, exactly in SQL server on
+         * Win2k, at the time we are started there is room, but later
+         * loaded DLLs use it.  Case 6859 on other attempts to catch
+         * the problem.
+         */
+    });
+    
+ tls_alloc_exit:
+    if (synch) {
+        res = RtlLeaveCriticalSection(peb->FastPebLock);
+        if (!NT_SUCCESS(res))
+            return 0;
+    }
+
+    /* ntdll seems to grab slot 0 of the TlsBitmap before loading
+     * kernel32, see if early injection gets us before that if we go
+     * bottom up, FIXME: if hit change interface, since 0 is returned
+     * on error 
+     */
+    ASSERT(start > 0);
+
+    return start;
+}
+
+int
+tls_alloc(int synch, uint *teb_offs /* OUT */)
+{
+    return tls_alloc_helper(synch, teb_offs, 1, 0/* any alignment */, 
+                            /* same top down or bottom up choice as tls_calloc */
+                            DYNAMO_OPTION(tls_flags));
+}
+
+/* Allocates num tls slots aligned with particular alignment
+ * Alignment must be sub-page
+ */
+int
+tls_calloc(int synch, uint *teb_offs /* OUT */, int num, uint alignment)
+{
+    return tls_alloc_helper(synch, teb_offs, num, alignment, 
+                            DYNAMO_OPTION(tls_flags));
+}
+
+static int
+tls_free_helper(int synch, uint teb_offs, int num)
+{
+    PEB *peb = get_own_peb();
+    int i;
+    int slot;
+    uint *p;
+
+    NTSTATUS res;
+    GET_NTDLL(RtlTryEnterCriticalSection, (IN OUT RTL_CRITICAL_SECTION *crit));
+    GET_NTDLL(RtlLeaveCriticalSection, (IN OUT RTL_CRITICAL_SECTION *crit));
+    GET_NTDLL(NtSetInformationThread, (IN HANDLE ThreadHandle,
+                                       IN THREADINFOCLASS ThreadInformationClass,
+                                       IN PVOID ThreadInformation,
+                                       IN ULONG ThreadInformationLength));
+    if (synch) {
+        /* TlsFree calls RtlAcquirePebLock which calls RtlEnterCriticalSection
+         * I'm worried about synch problems so I'm going to just do a Try
+         * and if it fails I simply will not free the slot, not too bad of a leak.
+         * On a detach a suspended thread might be holding this lock, or a thread
+         * killed due to an attack might have held it.  We could, on failure to
+         * get the lock, xchg and read back what we write and try to fix up the bits,
+         * with the worst case being the app hasn't written but has read and thus
+         * our free won't go through, but in the past we just called TlsFree and
+         * never had a lock problem so I'm going to assume Try will work the vast
+         * majority of the time and the times it doesn't we can eat the leak.
+         */
+        res = RtlTryEnterCriticalSection(peb->FastPebLock);
+        ASSERT_CURIOSITY(NT_SUCCESS(res));
+        if (!NT_SUCCESS(res))
+            return 0;
+    }
+    
+    ASSERT(peb->TlsBitmap != NULL);
+    /* TlsBitmap always points to next field, TlsBitmapBits, but we'll only
+     * use the pointer for generality
+     */
+    p = (uint *) peb->TlsBitmap->BitMapBuffer;
+    i = (teb_offs - offsetof(TEB, TlsSlots)) / sizeof(uint *);
+    for (slot = 0; slot < num; slot++, i++) {
+        NTPRINT("Freeing tls slot %d at offset 0x%x -> index %d\n", slot, teb_offs, i);
+        /* In case we aren't synched, zero the tls field before we release it,
+         * (of course that only takes care of one of many possible races if we
+         * aren't synched). */
+        /* This will zero this tls index for all threads (see disassembly of
+         * FreeTls in kernel32, wine srcs).  Strange interface using a
+         * thread handle, would be more sensical as a process info class (esp.
+         * with respect to permissions).  Note that in the wine srcs at least
+         * this syscall will only accept NT_CURRENT_THREAD as the handle. Xref
+         * case 8143 for why we need to zero the tls slot for all threads. */
+        res = NtSetInformationThread(NT_CURRENT_THREAD, ThreadZeroTlsCell,
+                                     &i, sizeof(i));
+        ASSERT(NT_SUCCESS(res));
+        p[i/32] &= ~(1 << (i % 32));
+    }
+
+    if (synch) {
+        res = RtlLeaveCriticalSection(peb->FastPebLock);
+        ASSERT(NT_SUCCESS(res));
+        if (!NT_SUCCESS(res))
+            return 0;
+    }
+
+    return 1;
+}
+
+int
+tls_free(int synch, uint teb_offs)
+{
+    return tls_free_helper(synch, teb_offs, 1);
+}
+
+int
+tls_cfree(int synch, uint teb_offs, int num)
+{
+    return tls_free_helper(synch, teb_offs, num);
+}
+
+#endif /* !NOT_DYNAMORIO_CORE_PROPER */
+/***************************************************************************/
+
+int
+get_process_mem_stats(HANDLE h, VM_COUNTERS *info)
+{
+    NTSTATUS res;
+    ULONG got;
+    res = NtQueryInformationProcess(h, ProcessVmCounters, info, sizeof(VM_COUNTERS), &got);
+    ASSERT(!NT_SUCCESS(res) || got == sizeof(VM_COUNTERS));
+    return NT_SUCCESS(res);
+}
+
+/* Get process quota limits information */
+/* Note returns raw NTSTATUS */
+NTSTATUS
+get_process_mem_quota(HANDLE h, QUOTA_LIMITS *qlimits)
+{
+    NTSTATUS res;
+    ULONG got;
+    res = NtQueryInformationProcess(h, ProcessQuotaLimits, qlimits, sizeof(QUOTA_LIMITS), &got);
+    ASSERT(!NT_SUCCESS(res) || got == sizeof(QUOTA_LIMITS));
+    return res;
+}
+
+/* Get process quota limits information */
+/* Note returns raw NTSTATUS */
+NTSTATUS
+get_process_handle_count(HANDLE ph, ULONG *handle_count)
+{
+    NTSTATUS res;
+    ULONG got;
+    res = NtQueryInformationProcess(ph, ProcessHandleCount, handle_count, sizeof(ULONG), &got);
+    ASSERT(!NT_SUCCESS(res) || got == sizeof(ULONG));
+    return res;
+}
+
+/* forward -- static since others probably want seconds as unit */
+static LONGLONG query_time_100ns(void);
+
+int
+get_process_load(HANDLE h)
+{
+    KERNEL_USER_TIMES times;
+    LONGLONG scheduled_time;
+    LONGLONG wallclock_time;
+    NTSTATUS res;
+    ULONG len = 0;
+    res = NtQueryInformationProcess((HANDLE) h, ProcessTimes,
+                                    &times, sizeof(times), &len);
+    if (!NT_SUCCESS(res))
+        return -1;
+    /* return length not trustworthy, according to Nebbett, so we don't test it */
+    /* we want %CPU == (scheduled time) / (wall clock time) */
+    scheduled_time = times.UserTime.QuadPart + times.KernelTime.QuadPart;
+    wallclock_time = query_time_100ns() - times.CreateTime.QuadPart;
+    if (wallclock_time <= 0)
+        return -1;
+    return (int) ((100 * scheduled_time) / wallclock_time);
+}
+
+/* Returns 0 for both known false and error 
+ * FIXME: do we still have the restriction of not returning a bool for ntdll.c
+ * routines?!?
+ */
+int
+is_wow64_process(HANDLE h)
+{
+    /* since this is called a lot we remember the result for the current process */
+    static int self_is_wow64 = -1; /* uninitialized */
+    if (self_is_wow64 == -1 || h != NT_CURRENT_PROCESS) {
+        ptr_uint_t is_wow64;
+        NTSTATUS res;
+        ULONG len = 0;
+
+        res = NtQueryInformationProcess((HANDLE) h, ProcessWow64Information,
+                                        &is_wow64, sizeof(is_wow64), &len);
+        if (!NT_SUCCESS(res) || len != sizeof(is_wow64)) {
+#if !defined(NOT_DYNAMORIO_CORE_PROPER) && !defined(NOT_DYNAMORIO_CORE)
+            /* PR 233191: we expect failure on NT but nowhere else */
+            ASSERT(res == STATUS_INVALID_INFO_CLASS &&
+                   get_os_version() == WINDOWS_VERSION_NT);
+#endif
+            is_wow64 = 0;
+        }
+
+        if (h == NT_CURRENT_PROCESS) {
+#if !defined(NOT_DYNAMORIO_CORE_PROPER) && !defined(NOT_DYNAMORIO_CORE)
+            ASSERT(!dynamo_initialized); /* .data should be writable */
+#endif
+            self_is_wow64 = (is_wow64 != 0);
+        }
+        return (is_wow64 != 0);
+    }
+    return self_is_wow64;
+}
+
+/* use base hint if present; will bump size up to PAGE_SIZE multiple
+ * Note returns raw NTSTATUS.
+ */
+NTSTATUS
+nt_remote_allocate_virtual_memory(HANDLE process, void **base, size_t size,
+                                  uint prot, memory_commit_status_t commit)
+{
+    NTSTATUS res;
+    SIZE_T sz = size;
+    ASSERT(ALIGNED(*base, PAGE_SIZE) && "base argument not initialized at PAGE_SIZE");
+    res = NT_SYSCALL(AllocateVirtualMemory, process, base, 0 /* zero bits */,
+                     &sz, commit, prot);
+    if (res == STATUS_CONFLICTING_ADDRESSES) {
+        NTPRINT("NtAllocateVirtualMemory: conflict at base "PFX
+                ", res="PFX"\n", *base, res);
+        /* Let caller decide whether to retry or not. */
+    }
+
+    /* FIXME: alert caller if sz > size? only happens if size not PAGE_SIZE multiple */
+    NTPRINT("NtAllocateVirtualMemory: asked for %d bytes, got %d bytes at "PFX"\n",
+            size, sz, *base);
+    ASSERT(sz >= size);
+    return res;
+}
+
+/* use base hint is present; will bump size up to PAGE_SIZE multiple
+ * Note returns raw NTSTATUS.
+ */
+NTSTATUS
+nt_allocate_virtual_memory(void **base, size_t size, uint prot, memory_commit_status_t commit)
+{
+    return nt_remote_allocate_virtual_memory(NT_CURRENT_PROCESS, base, 
+                                             size, prot, commit);
+}
+
+/* commit memory previously reserved with nt_allocate_virtual_memory()
+ * Note returns raw NTSTATUS.
+ */
+NTSTATUS
+nt_commit_virtual_memory(void *base, size_t size, uint prot)
+{
+    NTSTATUS res;
+    DEBUG_DECLARE(void *original_base = base;)
+    DEBUG_DECLARE(size_t original_size = size;)
+    res = NT_SYSCALL(AllocateVirtualMemory, NT_CURRENT_PROCESS, &base, 0, (SIZE_T*)&size,
+                     MEM_COMMIT, /* should be already reserved */ prot);
+    ASSERT(base == original_base);
+    ASSERT(size == original_size);
+    ASSERT_CURIOSITY(NT_SUCCESS(res));
+    return res;
+}
+
+/* Decommit memory previously committed with nt_commit_virtual_memory() or
+ * nt_allocate_virtual_memory().  Still available for committing again. 
+ * Note returns raw NTSTATUS.
+ */
+NTSTATUS
+nt_decommit_virtual_memory(void *base, size_t size)
+{
+    NTSTATUS res;
+    SIZE_T sz = size;             /* copied to compare with OUT value */
+    res = NT_SYSCALL(FreeVirtualMemory, NT_CURRENT_PROCESS, &base, &sz, MEM_DECOMMIT);
+    ASSERT(sz == size);
+    NTPRINT("NtFreeVirtualMemory: decommitted %d bytes [res=%d]\n", sz, res);
+    ASSERT_CURIOSITY(NT_SUCCESS(res));
+    return res;
+}
+
+/* Decommit memory previously committed with nt_commit_virtual_memory() or
+ * nt_allocate_virtual_memory().  Still available for committing again. 
+ * Note returns raw NTSTATUS.
+ */
+NTSTATUS
+nt_free_virtual_memory(void *base)
+{
+    NTSTATUS res;
+    SIZE_T sz = 0;                /* has to be 0 for MEM_RELEASE */
+    res = NT_SYSCALL(FreeVirtualMemory, NT_CURRENT_PROCESS, &base, &sz, MEM_RELEASE);
+    NTPRINT("NtFreeVirtualMemory: freed "SZFMT" bytes\n", sz);
+    ASSERT_CURIOSITY(NT_SUCCESS(res));
+    return res;
+}
+
+/* FIXME: change name to nt_protect_virtual_memory() and use
+ * nt_remote_protect_virtual_memory(), or maybe just change callers to
+ * pass NT_CURRENT_PROCESS to nt_remote_protect_virtual_memory()
+ * instead to avoid the extra function call, especially with self-protection on
+ */
+int
+protect_virtual_memory(void *base, size_t size, uint prot, uint *old_prot)
+{
+    NTSTATUS res;
+    SIZE_T sz = size;
+    res = NT_SYSCALL(ProtectVirtualMemory, NT_CURRENT_PROCESS, &base, &sz, prot,
+                     (ULONG*)old_prot);
+    NTPRINT("NtProtectVirtualMemory: "PFX"-"PFX" 0x%x => 0x%x\n",
+            base, (byte *)base + size, prot, res);
+    ASSERT(sz == size);
+    return NT_SUCCESS(res);
+}
+
+int
+nt_remote_protect_virtual_memory(HANDLE process, 
+                                 void *base, size_t size, uint prot, uint *old_prot)
+{
+    NTSTATUS res;
+    SIZE_T sz = size;
+    res = NT_SYSCALL(ProtectVirtualMemory, process, &base, &sz, prot, (ULONG*)old_prot);
+    NTPRINT("NtProtectVirtualMemory: process "PFX" "PFX"-"PFX" 0x%x => 0x%x\n",
+            process, base, (byte *)base + size, prot, res);
+
+    ASSERT(ALIGNED(base, PAGE_SIZE) && "base argument not initialized at PAGE_SIZE");
+    NTPRINT("NtProtectVirtualMemory: intended to change %d bytes, "
+            "modified %d bytes at "PFX"\n", size, sz, base);
+    ASSERT(sz >= size);
+    return NT_SUCCESS(res);
+}
+
+/* We use this instead of VirtualQuery b/c there are problems using
+ * win32 API routines inside of the app using them
+ */
+/* We make our signature look like VirtualQuery */
+size_t
+query_virtual_memory(const byte *pc, MEMORY_BASIC_INFORMATION *mbi, size_t mbilen)
+{
+    NTSTATUS res;
+    SIZE_T got;
+    GET_RAW_SYSCALL(QueryVirtualMemory,
+                    IN HANDLE ProcessHandle,
+                    IN const void *BaseAddress,
+                    IN MEMORY_INFORMATION_CLASS MemoryInformationClass,
+                    OUT PVOID MemoryInformation,
+                    IN SIZE_T MemoryInformationLength,
+                    OUT PSIZE_T ReturnLength OPTIONAL);
+    ASSERT(mbilen == sizeof(MEMORY_BASIC_INFORMATION));
+    memset(mbi, 0, sizeof(MEMORY_BASIC_INFORMATION));
+    res = NT_SYSCALL(QueryVirtualMemory, NT_CURRENT_PROCESS, pc, MemoryBasicInformation,
+                        mbi, mbilen, &got);
+    ASSERT(!NT_SUCCESS(res) || got == sizeof(MEMORY_BASIC_INFORMATION));
+    /* only 0 and sizeof(MEMORY_BASIC_INFORMATION) should be expected by callers */
+    if (!NT_SUCCESS(res))
+        got = 0;
+
+#if !defined(NOT_DYNAMORIO_CORE_PROPER) && !defined(NOT_DYNAMORIO_CORE)
+    /* for stress testing a fake driver access */
+    if (INTERNAL_OPTION(stress_fake_userva) != 0) {
+        if (pc > (app_pc)INTERNAL_OPTION(stress_fake_userva))
+            return 0;
+    }
+#endif
+
+    return got;
+}
+
+int
+nt_read_virtual_memory(HANDLE process, const void *base, void *buffer, 
+                       size_t buffer_length, size_t *bytes_read)
+{
+    NTSTATUS res;
+    GET_NTDLL(NtReadVirtualMemory, (IN HANDLE ProcessHandle,
+                                    IN const void *BaseAddress,
+                                    OUT PVOID Buffer,
+                                    IN SIZE_T BufferLength,
+                                    OUT PSIZE_T ReturnLength OPTIONAL));
+    res = NtReadVirtualMemory(process, base, buffer, 
+                              buffer_length, (SIZE_T*)bytes_read);
+    return NT_SUCCESS(res);
+}
+
+int
+nt_write_virtual_memory(HANDLE process, void *base, const void *buffer, 
+                        size_t buffer_length, size_t *bytes_written)
+{
+    NTSTATUS res;
+    GET_RAW_SYSCALL(WriteVirtualMemory, IN HANDLE ProcessHandle,
+                    IN PVOID BaseAddress,
+                    IN const void *Buffer,
+                    IN SIZE_T BufferLength,
+                    OUT PSIZE_T ReturnLength OPTIONAL);
+    res = NT_SYSCALL(WriteVirtualMemory, process, base, buffer,
+                     buffer_length, (SIZE_T*)bytes_written);
+    return NT_SUCCESS(res);
+}
+
+/* There are no Win32 API routines to do this, so we use NtContinue */
+void
+nt_continue(CONTEXT *cxt)
+{
+    GET_RAW_SYSCALL(Continue, IN PCONTEXT Context, IN BOOLEAN TestAlert);
+    NT_SYSCALL(Continue, cxt, 0/* don't change APC status */);
+    /* should not get here */
+    ASSERT_NOT_REACHED();
+}
+
+NTSTATUS
+nt_get_context(HANDLE hthread, CONTEXT *cxt)
+{
+    GET_RAW_SYSCALL(GetContextThread, IN HANDLE ThreadHandle,
+                    OUT PCONTEXT Context);
+    /* PR 263338: we get STATUS_DATATYPE_MISALIGNMENT if not aligned */
+    IF_X64(ASSERT(ALIGNED(cxt, 16)));
+    return NT_SYSCALL(GetContextThread, hthread, cxt);
+    /* Don't assert here -- let the caller do so if it expects a particular value.
+     * If we asserted here when an ldmp is being generated, we could prevent
+     * generation of the ldmp if there is a handle privilege problem between
+     * the calling thread and hthread.
+     */
+}
+
+/* WARNING: any time we set a thread's context we must make sure we can
+ * handle two cases:
+ * 1) the thread was at a syscall and now we won't recognize it as such
+ *    (case 6113) (not to mention that the kernel will finish the
+ *    syscall and clobber eax and ecx+edx after setting to cxt: case 5074)
+ * 2) the thread just hit a fault but the kernel has not yet copied the
+ *    faulting context to the user mode structures for the handler
+ *    (case 7393)
+ */
+NTSTATUS
+nt_set_context(HANDLE hthread, CONTEXT *cxt)
+{
+    GET_RAW_SYSCALL(SetContextThread, IN HANDLE ThreadHandle,
+                    IN PCONTEXT Context);
+    /* PR 263338: we get STATUS_DATATYPE_MISALIGNMENT if not aligned */
+    IF_X64(ASSERT(ALIGNED(cxt, 16)));
+    return NT_SYSCALL(SetContextThread, hthread, cxt);
+}
+
+int
+nt_thread_suspend(HANDLE hthread, int *previous_suspend_count)
+{
+    NTSTATUS res;
+    GET_RAW_SYSCALL(SuspendThread, IN HANDLE ThreadHandle,
+                    OUT PULONG PreviousSuspendCount OPTIONAL);
+    res = NT_SYSCALL(SuspendThread, hthread, (ULONG *)previous_suspend_count);
+    /* Don't assert here -- let the caller do so if it expects a particular value.
+     * If we asserted here when an ldmp is being generated, we could prevent
+     * generation of the ldmp if there is a handle privilege problem between
+     * the calling thread and hthread.
+     */
+    return NT_SUCCESS(res);
+}
+
+int
+nt_thread_resume(HANDLE hthread, int *previous_suspend_count)
+{
+    NTSTATUS res;
+    GET_RAW_SYSCALL(ResumeThread, IN HANDLE ThreadHandle,
+                    OUT PULONG PreviousSuspendCount OPTIONAL);
+    res = NT_SYSCALL(ResumeThread, hthread, (ULONG *)previous_suspend_count);
+    return NT_SUCCESS(res);
+}
+
+int
+nt_terminate_thread(HANDLE hthread, NTSTATUS exit_code)
+{
+    NTSTATUS res;
+    GET_RAW_SYSCALL(TerminateThread, IN HANDLE ThreadHandle OPTIONAL,
+                    IN NTSTATUS ExitStatus);
+    /* hthread == 0 means current thread, match kernel32 TerminateThread which 
+     * disallows null to avoid bugs in our code (we should always be passing
+     * a valid handle or NT_CURRENT_THREAD) */
+    ASSERT(hthread != (HANDLE)0);
+    res = NT_SYSCALL(TerminateThread, hthread, exit_code);
+    ASSERT(hthread != NT_CURRENT_THREAD && "terminate current thread failed");
+    return NT_SUCCESS(res);
+}
+
+int 
+nt_terminate_process(HANDLE hprocess, NTSTATUS exit_code)
+{
+    NTSTATUS res;
+    GET_RAW_SYSCALL(TerminateProcess, IN HANDLE ProcessHandle OPTIONAL,
+                    IN NTSTATUS ExitStatus);
+    /* hprocess == 0 has special meaning (terminate all threads but this one),
+     * kernel32!TerminateProcess disallows it and we currently don't use 
+     * that functionality */
+    ASSERT(hprocess != (HANDLE)0);
+    res = NT_SYSCALL(TerminateProcess, hprocess, exit_code);
+    ASSERT(hprocess != NT_CURRENT_PROCESS && "terminate current process failed");
+    return NT_SUCCESS(res);
+}
+
+NTSTATUS
+nt_terminate_process_for_app(HANDLE hprocess, NTSTATUS exit_code)
+{
+    GET_RAW_SYSCALL(TerminateProcess, IN HANDLE ProcessHandle OPTIONAL,
+                    IN NTSTATUS ExitStatus);
+    /* we allow any argument or result values */
+    return NT_SYSCALL(TerminateProcess, hprocess, exit_code);
+}
+
+int
+am_I_sole_thread(HANDLE hthread, int *amI /*OUT*/)
+{
+    NTSTATUS res;
+    ULONG got;
+    res = NtQueryInformationThread(hthread, ThreadAmILastThread, amI, sizeof(*amI), &got);
+    return NT_SUCCESS(res);
+}
+
+/* checks current thread, and turns errors into false */
+int
+check_sole_thread()
+{
+    int amI;
+    if (!am_I_sole_thread(NT_CURRENT_THREAD, &amI))
+        return 0;
+    else
+        return amI;
+}
+
+HANDLE
+nt_create_and_set_timer(PLARGE_INTEGER due_time, LONG period) 
+{
+    NTSTATUS res;
+    HANDLE htimer;
+    enum {
+        NotificationTimer,
+        SynchronizationTimer
+    };
+
+    GET_NTDLL(NtCreateTimer, (OUT PHANDLE TimerHandle,
+                              IN ACCESS_MASK DesiredAccess,
+                              IN POBJECT_ATTRIBUTES ObjectAttributes,
+                              IN DWORD TimerType /* TIMER_TYPE */
+                              ));
+    res = NtCreateTimer(&htimer, TIMER_ALL_ACCESS, NULL /* no name */, 
+                        SynchronizationTimer); 
+    ASSERT(NT_SUCCESS(res));
+    {
+        GET_NTDLL(NtSetTimer, (IN HANDLE TimerHandle,
+                           IN PLARGE_INTEGER DueTime,
+                           IN PVOID TimerApcRoutine, /* PTIMER_APC_ROUTINE */
+                           IN PVOID TimerContext,
+                           IN BOOLEAN Resume,
+                           IN LONG Period,
+                           OUT PBOOLEAN PreviousState));
+        res = NtSetTimer(htimer, due_time, NULL, NULL, false, period, NULL);
+        ASSERT(NT_SUCCESS(res));
+    }
+    return htimer;
+}
+
+int
+nt_sleep(PLARGE_INTEGER due_time) 
+{
+    NTSTATUS res;
+    GET_NTDLL(NtDelayExecution, (IN BOOLEAN Alertable,
+                               IN PLARGE_INTEGER Interval
+                               ));
+    res = NtDelayExecution(false, /* non alertable sleep */
+                           due_time);
+    return NT_SUCCESS(res);
+}
+
+void
+nt_yield()
+{
+    GET_NTDLL(NtYieldExecution, (VOID));
+    NtYieldExecution();
+}
+
+void *
+get_section_address(HANDLE h)
+{
+    SECTION_BASIC_INFORMATION info;
+    NTSTATUS res;
+    ULONG got;
+    memset(&info, 0, sizeof(SECTION_BASIC_INFORMATION));
+    res = NtQuerySection(h, SectionBasicInformation,
+                         &info, sizeof(SECTION_BASIC_INFORMATION), &got);
+    ASSERT(NT_SUCCESS(res) && got == sizeof(SECTION_BASIC_INFORMATION));
+    return info.BaseAddress;
+}
+
+/* returns true if attributes can be read and sets them,
+ * otherwise the values are not modified
+ */
+int
+get_section_attributes(HANDLE h, uint *section_attributes /* OUT */,
+                       LARGE_INTEGER* section_size /* OPTIONAL OUT */)
+{
+    SECTION_BASIC_INFORMATION info;
+    NTSTATUS res;
+    ULONG got;
+    memset(&info, 0, sizeof(SECTION_BASIC_INFORMATION));
+    ASSERT(section_attributes != NULL);
+    res = NtQuerySection(h, SectionBasicInformation,
+                         &info, sizeof(SECTION_BASIC_INFORMATION), &got);
+    if (NT_SUCCESS(res)) {
+        ASSERT(got == sizeof(SECTION_BASIC_INFORMATION));
+        *section_attributes = info.Attributes;
+        if (section_size != NULL) {
+            *section_size = info.Size;
+        }
+        return true;
+    } else {
+        /* Unfortunately, we are often passed section handles that are
+         * created as GrantedAccess 0xe: None,
+         * MapWrite,MapRead,MapExecute which cannot be queried
+         */
+        return false;
+    }
+}
+
+int
+close_handle(HANDLE h)
+{
+    NTSTATUS res;
+    GET_NTDLL(NtClose, (IN HANDLE Handle));
+    res = NtClose(h);
+    return NT_SUCCESS(res);
+}
+
+/* Note returns raw NTSTATUS */
+NTSTATUS
+duplicate_handle(HANDLE source_process, HANDLE source, HANDLE target_process, 
+                 HANDLE *target, ACCESS_MASK access, uint attributes, 
+                 uint options)
+{
+    NTSTATUS res;
+    GET_NTDLL(NtDuplicateObject, (IN HANDLE SourceProcessHandle,
+                                  IN HANDLE SourceHandle,
+                                  IN HANDLE TargetProcessHandle,
+                                  OUT PHANDLE TargetHandle OPTIONAL,
+                                  IN ACCESS_MASK DesiredAcess,
+                                  IN ULONG Atrributes,
+                                  IN ULONG options_t));
+    res = NtDuplicateObject(source_process, source, target_process, target, 
+                            access, attributes, options);
+    return res;
+}
+
+GET_NTDLL(NtQueryObject, (IN HANDLE ObjectHandle,
+                          IN OBJECT_INFORMATION_CLASS ObjectInformationClass,
+                          OUT PVOID ObjectInformation,
+                          IN ULONG ObjectInformationLength,
+                          OUT PULONG ReturnLength OPTIONAL));
+
+ACCESS_MASK
+nt_get_handle_access_rights(HANDLE handle)
+{
+    NTSTATUS res;
+    OBJECT_BASIC_INFORMATION obj_info;
+    ULONG needed_length;
+    res = NtQueryObject(handle, ObjectBasicInformation, &obj_info, 
+                        sizeof(obj_info), &needed_length);
+    ASSERT(needed_length == sizeof(obj_info));
+    ASSERT(NT_SUCCESS(res));
+    return obj_info.GrantedAccess;
+}
+
+/* byte_length is total size of UNICODE_STRING struct and an embedded buffer */
+NTSTATUS
+nt_get_object_name(HANDLE handle, OBJECT_NAME_INFORMATION* object_name /* OUT */,
+                   uint byte_length, uint *returned_byte_length /* OUT */)
+{
+    NTSTATUS res;
+    res = NtQueryObject(handle, ObjectNameInformation, object_name, 
+                        byte_length, (ULONG *)returned_byte_length);
+    ASSERT(NT_SUCCESS(res));
+    return res;
+}
+
+static NTSTATUS
+wchar_to_unicode(PUNICODE_STRING dst, PCWSTR src)
+{
+    NTSTATUS res;
+    GET_NTDLL(RtlInitUnicodeString, (IN OUT PUNICODE_STRING DestinationString,
+                                     IN PCWSTR SourceString));
+    res = RtlInitUnicodeString(dst, src);
+    return res;
+}
+
+
+/* we don't want to allocate memory, so caller must provide
+ * a buffer that's big enough for char -> wchar conversion
+ */
+static NTSTATUS
+char_to_unicode(PUNICODE_STRING dst, PCSTR src, PWSTR buf, size_t buflen)
+{
+    _snwprintf(buf, buflen, L"%S", src);
+    return wchar_to_unicode(dst, buf);
+}
+
+static void
+char_to_ansi(PANSI_STRING dst, const char *str)
+{
+    GET_NTDLL(RtlInitAnsiString, (IN OUT PANSI_STRING DestinationString,
+                                  IN PCSTR SourceString));
+    RtlInitAnsiString(dst, str);
+}
+
+/* Collects file attributes.  
+ * Returns 1 if successful; 0 otherwise.
+ * (Using bool is problematic for non-core users.)
+ */
+int
+query_full_attributes_file(IN PCWSTR filename,
+                           OUT PFILE_NETWORK_OPEN_INFORMATION info)
+{
+    NTSTATUS result;
+    OBJECT_ATTRIBUTES attributes;
+    UNICODE_STRING objname;
+    GET_NTDLL(NtQueryFullAttributesFile, (IN  POBJECT_ATTRIBUTES attributes,
+                                          OUT PFILE_NETWORK_OPEN_INFORMATION info));
+
+    memset(&attributes, 0, sizeof(attributes));
+    wchar_to_unicode(&objname, filename);
+    InitializeObjectAttributes(&attributes, &objname,
+                               OBJ_CASE_INSENSITIVE,
+                               NULL, NULL);
+
+    result = NtQueryFullAttributesFile(&attributes, info);
+
+    return NT_SUCCESS(result);
+}
+
+/* rights should be KEY_READ or KEY_WRITE or both */
+/* parent handle HAS to be opened with an absolute name */
+HANDLE
+reg_create_key(HANDLE parent, PCWSTR keyname, ACCESS_MASK rights)
+{
+    NTSTATUS res;
+    OBJECT_ATTRIBUTES attr;
+    UNICODE_STRING objname;
+    ULONG disp;
+    HANDLE hkey;
+    GET_RAW_SYSCALL(CreateKey,
+                    OUT PHANDLE KeyHandle,
+                    IN ACCESS_MASK DesiredAccess,
+                    IN POBJECT_ATTRIBUTES ObjectAttributes,
+                    IN ULONG TitleIndex,
+                    IN PUNICODE_STRING Class OPTIONAL,
+                    IN ULONG CreateOptions,
+                    OUT PULONG Disposition OPTIONAL);
+    res = wchar_to_unicode(&objname, keyname);
+    if (!NT_SUCCESS(res))
+        return NULL;
+    InitializeObjectAttributes(&attr, &objname,
+                               OBJ_CASE_INSENSITIVE,
+                               parent, NULL);
+    res = NT_SYSCALL(CreateKey, &hkey, rights, &attr, 0, NULL, 0, &disp);
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("Error 0x%x in create key for \"%S\"\n", res, objname.Buffer);
+        return NULL;
+    } else
+        return hkey;
+}
+
+/* rights should be KEY_READ or KEY_WRITE or both */
+HANDLE
+reg_open_key(PCWSTR keyname, ACCESS_MASK rights)
+{
+    NTSTATUS res;
+    HANDLE hkey;
+    OBJECT_ATTRIBUTES attr;
+    UNICODE_STRING objname;
+    GET_RAW_SYSCALL(OpenKey,
+                    OUT PHANDLE KeyHandle,
+                    IN ACCESS_MASK DesiredAccess,
+                    IN POBJECT_ATTRIBUTES ObjectAttributes);
+    res = wchar_to_unicode(&objname, keyname);
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("Error in wchar to unicode\n");
+        return NULL;
+    }
+
+    InitializeObjectAttributes(&attr, &objname,
+                               OBJ_CASE_INSENSITIVE,
+                               NULL, NULL);
+    res = NT_SYSCALL(OpenKey, &hkey, rights, &attr);
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("Error 0x%x in open key for \"%S\"\n", res, objname.Buffer);
+        return NULL;
+    } else
+        return hkey;
+}
+
+
+int
+reg_close_key(HANDLE hkey)
+{
+    return close_handle(hkey);
+}
+
+
+int
+reg_delete_key(HANDLE hkey)
+{
+    NTSTATUS res;
+    GET_NTDLL(NtDeleteKey, (IN HANDLE KeyHandle));
+    res = NtDeleteKey(hkey);
+    NTPRINT("Got %d for deleting key\n", res);
+    return NT_SUCCESS(res);
+}
+
+/* Enumerates the values of a registry key via the NtEnumerateValueKey
+ * system call.  
+ *
+ * Note that the caller must allocate memory at the end of
+ * KEY_VALUE_FULL_INFORMATION to store the actual data.
+ * WARNING: the Name field often has no null terminating it.  It
+ * either runs right up next to Data or has an un-initialized value
+ * in it -- so make sure you zero out your buffer before calling
+ * this routine, and use the NameLength field (bytes not chars) and
+ * then check for null and skip over it if nec. to find the data start.
+ */
+reg_query_value_result_t
+reg_query_value(IN PCWSTR keyname,
+                IN PCWSTR subkeyname, 
+                IN KEY_VALUE_INFORMATION_CLASS info_class,
+                OUT PVOID info,
+                IN ULONG info_size,
+                IN ACCESS_MASK rights)
+{
+    int res;
+    ULONG outlen = 0;
+    UNICODE_STRING valuename;
+    HANDLE hkey = reg_open_key(keyname, KEY_READ | rights);
+
+    GET_NTDLL(NtQueryValueKey, (IN HANDLE KeyHandle,
+                                IN PUNICODE_STRING ValueName,
+                                IN KEY_VALUE_INFORMATION_CLASS KeyValueInformationClass,
+                                OUT PVOID KeyValueInformation,
+                                IN ULONG Length,
+                                OUT PULONG ResultLength));
+
+    if (hkey == NULL) 
+        return REG_QUERY_FAILURE;
+    
+    res = wchar_to_unicode(&valuename, subkeyname);
+    if (!NT_SUCCESS(res)) 
+        return REG_QUERY_FAILURE;
+
+    res = NtQueryValueKey(hkey, &valuename, info_class, info, info_size, &outlen);
+    reg_close_key(hkey);
+#if VERBOSE
+    if (!NT_SUCCESS(res))
+        NTPRINT("Error 0x%x in query key \"%S\"\n", res, subkeyname);
+#endif
+    /* When buffer is insufficient I see it return BUFFER_OVERFLOW, but nebbet
+     * mentions BUFFER_TOO_SMALL as well. */
+    if (res == STATUS_BUFFER_TOO_SMALL || res == STATUS_BUFFER_OVERFLOW) {
+        return REG_QUERY_BUFFER_TOO_SMALL;
+    }
+    return NT_SUCCESS(res) ? REG_QUERY_SUCCESS : REG_QUERY_FAILURE;
+}
+
+GET_RAW_SYSCALL(SetValueKey,
+                IN HANDLE KeyHandle,
+                IN PUNICODE_STRING ValueName,
+                IN ULONG TitleIndex OPTIONAL,
+                IN ULONG Type,
+                IN PVOID Data,
+                IN ULONG DataSize);
+
+int
+reg_set_key_value(HANDLE hkey, PCWSTR subkey, PCWSTR val)
+{
+    UNICODE_STRING name;
+    UNICODE_STRING value;
+    NTSTATUS res;
+    res = wchar_to_unicode(&name, subkey);
+    if (!NT_SUCCESS(res))
+        return NT_SUCCESS(res);
+    res = wchar_to_unicode(&value, val);
+    if (!NT_SUCCESS(res))
+        return NT_SUCCESS(res);
+    /* Length field is really size in bytes, have to add 1 for final 0 */
+    res = NT_SYSCALL(SetValueKey,
+                     hkey, &name, 0, REG_SZ, (LPBYTE) value.Buffer,
+                     value.Length+sizeof(wchar_t));
+    return NT_SUCCESS(res);
+}
+
+int
+reg_set_dword_key_value(HANDLE hkey, PCWSTR subkey, DWORD value)
+{
+    UNICODE_STRING name;
+    NTSTATUS res;
+    res = wchar_to_unicode(&name, subkey);
+    if (!NT_SUCCESS(res))
+        return NT_SUCCESS(res);
+    res = NT_SYSCALL(SetValueKey,
+                     hkey, &name, 0, REG_DWORD, &value, sizeof(DWORD));
+    return NT_SUCCESS(res);
+}
+
+/* Flushes registry changes for the given key to the disk.
+ * Returns 1 on success, 0 otherwise.
+ * Notes:       See case 4138.  For a valid opened key, failure can happen
+ *              only if registry IO fails, i.e., this function shouldn't fail
+ *              for most cases.
+ */
+int
+reg_flush_key(HANDLE hkey)
+{
+    NTSTATUS res;
+    GET_NTDLL(NtFlushKey, (IN HANDLE KeyHandle));
+    res = NtFlushKey(hkey);
+    return NT_SUCCESS(res);
+}
+
+/* Enumerates the subkeys of a registry key via the NtEnumerateKey
+ * system call.  
+ *
+ * Note that the caller must allocate memory at the end of
+ * KEY_VALUE_FULL_INFORMATION to store the actual data.
+ * WARNING: the Name field often has no null terminating it.  It
+ * either runs right up next to Data or has an un-initialized value
+ * in it -- so make sure you zero out your buffer before calling
+ * this routine, and use the NameLength field (bytes not chars) and
+ * then check for null and skip over it if nec. to find the data start.
+ *
+ * Returns 1 on success, 0 otherwise.
+ */
+int
+reg_enum_key(IN PCWSTR keyname,
+             IN ULONG index,
+             IN KEY_INFORMATION_CLASS info_class,
+             OUT PVOID key_info,
+             IN ULONG key_info_size)
+{
+    NTSTATUS result;
+    ULONG received = 0;
+    HANDLE hkey = reg_open_key(keyname, KEY_READ);
+
+    GET_NTDLL(NtEnumerateKey, (IN  HANDLE hkey, 
+                               IN  ULONG index, 
+                               IN  KEY_INFORMATION_CLASS info_class,
+                               OUT PVOID key_info, 
+                               IN  ULONG key_info_size,
+                               OUT PULONG bytes_received));
+
+    if (hkey == NULL)
+        return 0;
+
+    result = NtEnumerateKey(hkey, index, info_class, key_info, 
+                            key_info_size, &received);
+    reg_close_key(hkey);
+
+    return NT_SUCCESS(result);
+}
+
+/* Enumerates the values of a registry key via the NtEnumerateValueKey
+ * system call.  
+ *
+ * Note that the caller must allocate memory at the end of
+ * KEY_VALUE_FULL_INFORMATION to store the actual data.
+ * WARNING: the Name field often has no null terminating it.  It
+ * either runs right up next to Data or has an un-initialized value
+ * in it -- so make sure you zero out your buffer before calling
+ * this routine, and use the NameLength field (bytes not chars) and
+ * then check for null and skip over it if nec. to find the data start.
+ * Returns 1 on success, 0 otherwise.
+ */
+int
+reg_enum_value(IN PCWSTR keyname,
+               IN ULONG index,
+               IN KEY_VALUE_INFORMATION_CLASS info_class,
+               OUT PVOID key_info,
+               IN ULONG key_info_size)
+{
+    NTSTATUS result;
+    ULONG bytes_received = 0;
+    HANDLE hkey = reg_open_key(keyname, KEY_READ);
+
+    GET_NTDLL(NtEnumerateValueKey, (IN  HANDLE hKey, 
+                                    IN  ULONG index, 
+                                    IN  KEY_VALUE_INFORMATION_CLASS info_class,
+                                    OUT PVOID key_info, 
+                                    IN  ULONG key_info_size,
+                                    OUT PULONG bytes_received));
+
+    if (hkey == NULL)
+        return 0;
+
+    result = NtEnumerateValueKey(hkey, index, info_class, key_info, 
+                                 key_info_size, &bytes_received);
+    reg_close_key(hkey);
+
+    return NT_SUCCESS(result);
+}
+
+/* thread token can be primary token, impersonated, or anonymous */
+NTSTATUS
+get_current_user_token(PTOKEN_USER ptoken, USHORT token_buffer_length)
+{
+    NTSTATUS res;
+    HANDLE htoken;
+    ULONG len = 0;
+
+    res = NtOpenThreadToken(NT_CURRENT_THREAD, TOKEN_QUERY, TRUE, &htoken);
+    if (!NT_SUCCESS(res)) {
+        /* anonymous impersonation token cannot be opened  */
+        res = NtOpenProcessToken(NT_CURRENT_PROCESS, TOKEN_QUERY, &htoken);
+        if (!NT_SUCCESS(res)) {
+            return res;
+        }
+    }
+
+    res = NtQueryInformationToken(htoken, TokenUser, ptoken, token_buffer_length, &len);
+    close_handle(htoken);
+
+    ASSERT(len <= token_buffer_length);
+    if (!NT_SUCCESS(res)) {
+        ASSERT_CURIOSITY(false && "can't query token, impersonated?");
+    }
+    return res;
+}
+
+NTSTATUS
+get_primary_user_token(PTOKEN_USER ptoken, USHORT token_buffer_length)
+{
+    NTSTATUS res;
+    HANDLE htoken;
+    ULONG len = 0;
+
+    res = NtOpenProcessToken(NT_CURRENT_PROCESS, TOKEN_QUERY, &htoken);
+    if (!NT_SUCCESS(res)) {
+        return res;
+    }
+
+    res = NtQueryInformationToken(htoken, TokenUser, ptoken, token_buffer_length, &len);
+    close_handle(htoken);
+
+    ASSERT(len <= token_buffer_length);
+    if (!NT_SUCCESS(res)) {
+        ASSERT_CURIOSITY(false && "can't query token?");
+    }
+    return res;
+}
+
+/* returns the Owner that will be recorded for any objects created by
+ * this process (when not impersonating)
+ */
+NTSTATUS
+get_primary_owner_token(PTOKEN_OWNER powner, USHORT owner_buffer_length)
+{
+    NTSTATUS res;
+    HANDLE htoken;
+    ULONG len = 0;
+
+    res = NtOpenProcessToken(NT_CURRENT_PROCESS, TOKEN_QUERY, &htoken);
+    if (!NT_SUCCESS(res)) {
+        return res;
+    }
+
+    res = NtQueryInformationToken(htoken, TokenOwner, powner, owner_buffer_length, &len);
+    close_handle(htoken);
+
+    ASSERT(len <= owner_buffer_length);
+    if (!NT_SUCCESS(res)) {
+        ASSERT_CURIOSITY(false && "can't query token?");
+    }
+    return res;
+}
+
+/* Note that the caller must allocate buffer_length bytes in sid_string */
+NTSTATUS
+get_current_user_SID(PWSTR sid_string, USHORT buffer_length)
+{
+    GET_NTDLL(RtlConvertSidToUnicodeString, (OUT PUNICODE_STRING UnicodeString,
+                                             IN PSID Sid,
+                                             BOOLEAN AllocateDestinationString));
+    NTSTATUS res;
+    UNICODE_STRING ustr;
+    UCHAR buf[SECURITY_MAX_TOKEN_SIZE];
+    PTOKEN_USER ptoken = (PTOKEN_USER)buf;
+
+    res = get_current_user_token(ptoken, sizeof(buf));
+    if (!NT_SUCCESS(res)) {
+        return res;
+    }
+
+    ustr.Length = 0;
+    ustr.MaximumLength = buffer_length;
+    ustr.Buffer = sid_string;
+
+    /* We assume that by passing FALSE, no memory will be allocated
+     * and the routine is reentrant.
+     */
+    res = RtlConvertSidToUnicodeString(&ustr, ptoken->User.Sid, FALSE);
+    return res;
+}
+
+
+const PSID
+get_process_primary_SID()
+{
+    static PSID primary_SID = NULL;
+    static UCHAR buf[SECURITY_MAX_TOKEN_SIZE];
+
+    if (primary_SID == NULL) {
+        PTOKEN_USER ptoken = (PTOKEN_USER)buf;
+        NTSTATUS res;
+        res = get_primary_user_token(ptoken, sizeof(buf));
+        ASSERT(NT_SUCCESS(res));
+           
+        if (!NT_SUCCESS(res)) {
+            return NULL;
+        }
+        primary_SID = ptoken->User.Sid;
+    }
+    return primary_SID;
+}
+
+
+/* based on RtlpQuerySecurityDescriptorPointers from reactos/0.2.9/lib/rtl/sd.c */
+static
+void
+get_sd_pointers(IN PISECURITY_DESCRIPTOR SecurityDescriptor,
+                OUT PSID *Owner  OPTIONAL,
+                OUT PSID *Group  OPTIONAL,
+                OUT PACL *Sacl  OPTIONAL,
+                OUT PACL *Dacl  OPTIONAL)
+{
+    /* we usually deal with self-relative SIDs as returned by NtQuerySecurityObject */
+    if (TEST(SE_SELF_RELATIVE, SecurityDescriptor->Control)) {
+        PISECURITY_DESCRIPTOR_RELATIVE RelSD = 
+            (PISECURITY_DESCRIPTOR_RELATIVE)SecurityDescriptor;
+        if(Owner != NULL) {
+            *Owner = ((RelSD->Owner != 0) ? 
+                      (PSID)((ULONG_PTR)RelSD + RelSD->Owner) : NULL);
+        }
+        if(Group != NULL) {
+            *Group = ((RelSD->Group != 0) ? 
+                      (PSID)((ULONG_PTR)RelSD + RelSD->Group) : NULL);
+        }
+        if(Sacl != NULL) {
+            *Sacl = (((RelSD->Control & SE_SACL_PRESENT) && (RelSD->Sacl != 0)) ?
+                     (PSID)((ULONG_PTR)RelSD + RelSD->Sacl) : NULL);
+        }
+        if(Dacl != NULL) {
+            *Dacl = (((RelSD->Control & SE_DACL_PRESENT) && (RelSD->Dacl != 0)) ?
+                     (PSID)((ULONG_PTR)RelSD + RelSD->Dacl) : NULL);
+        }
+    } else {
+        if(Owner != NULL) {
+            *Owner = SecurityDescriptor->Owner;
+        }
+        if(Group != NULL) {
+            *Group = SecurityDescriptor->Group;
+        }
+        if(Sacl != NULL) {
+            *Sacl = ((SecurityDescriptor->Control & SE_SACL_PRESENT) ? 
+                     SecurityDescriptor->Sacl : NULL);
+        }
+        if(Dacl != NULL) {
+            *Dacl = ((SecurityDescriptor->Control & SE_DACL_PRESENT) ? 
+                     SecurityDescriptor->Dacl : NULL);
+        }
+    }
+}
+
+bool
+get_owner_sd(PISECURITY_DESCRIPTOR SecurityDescriptor,
+             OUT PSID *Owner)
+{
+    /* RtlGetOwnerSecurityDescriptor is clean enough, so could be used
+     * without reentrancy risks instead of writing ours here
+     */
+
+    if (SecurityDescriptor->Revision != SECURITY_DESCRIPTOR_REVISION1) {
+        return false;
+    }
+
+    get_sd_pointers(SecurityDescriptor,
+                    Owner,
+                    NULL,
+                    NULL,
+                    NULL);
+    return true;
+}
+
+void
+initialize_security_descriptor(PISECURITY_DESCRIPTOR SecurityDescriptor)
+{
+   SecurityDescriptor->Revision = SECURITY_DESCRIPTOR_REVISION1;
+   SecurityDescriptor->Sbz1 = 0;
+   /* note using absolute format, not SE_SELF_RELATIVE */
+   SecurityDescriptor->Control = 0;
+   SecurityDescriptor->Owner = NULL;
+   SecurityDescriptor->Group = NULL;
+   SecurityDescriptor->Sacl = NULL;
+   SecurityDescriptor->Dacl = NULL;
+}
+
+/* use only on security descriptors created with initialize_security_descriptor() */
+bool
+set_owner_sd(PISECURITY_DESCRIPTOR SecurityDescriptor,
+             PSID Owner)
+{
+    /* RtlGetOwnerSecurityDescriptor is clean enough, so could be used
+     * without reentrancy risks instead of writing ours here
+     */
+
+    if (SecurityDescriptor->Revision != SECURITY_DESCRIPTOR_REVISION1) {
+        return false;
+    }
+    if (TEST(SE_SELF_RELATIVE, SecurityDescriptor->Control)) {
+        ASSERT(false && "we only create absolute security descriptors");
+        return false;
+    }
+
+    ASSERT(ALIGNED(SecurityDescriptor->Owner, sizeof(void*)));
+    SecurityDescriptor->Owner = Owner;
+
+    /* In case we are editing an existing SD makes remove possible tag
+     * that Owner field was provided with default or inheritance
+     * mechanisms..  Otherwise practically a nop for us when building
+     * an SD from scratch 
+     */
+    SecurityDescriptor->Control &= ~SE_OWNER_DEFAULTED;
+
+    return true;
+}
+
+static int
+length_sid(IN PSID Sid_)
+{
+  PISID Sid =  Sid_;
+  /* we only know about usable length of SID */
+  return LengthRequiredSID(Sid->SubAuthorityCount);
+}
+
+bool
+equal_sid(IN PSID Sid1_, IN PSID Sid2_)
+{
+    PISID Sid1 =  Sid1_;
+    PISID Sid2 =  Sid2_;
+
+    /* note ntdll!RtlEqualSid returns BOOLEAN and so its result is
+     * just in AL!  I don't want to deal with here after it got me
+     * once when assuming regular bool=int.
+     * 
+     * ntdll!RtlEqualSid+0x2e:
+     * 7c91a493 32c0             xor     al,al
+     * ...
+     * 7c91a498 c20800           ret     0x8
+     */
+    /* preferred to reimplement based on reactos/0.2.x/lib/rtl/sid.c*/
+    SIZE_T SidLen;
+
+    if (Sid1->Revision != Sid2->Revision ||
+        Sid1->SubAuthorityCount != Sid2->SubAuthorityCount) {
+        return(FALSE);
+    }
+   
+    SidLen = length_sid(Sid1);
+    return memcmp(Sid1, Sid2, SidLen) == 0;
+}
+
+#ifndef NOT_DYNAMORIO_CORE
+/* To avoid any possible races, we ensure tbat the static buffers are 
+ * initialized before we become multi-threaded via 
+ * os_init->init_debugbox_title_buf() which calls these routines */
+
+/* get application name, (cached), used for options, event logging and
+ * following children */
+char*
+get_application_name()
+{
+    static char exename[MAXIMUM_PATH];
+    if (!exename[0]) {
+        snprintf(exename, BUFFER_SIZE_ELEMENTS(exename), "%ls",
+                 get_own_qualified_name());
+        NULL_TERMINATE_BUFFER(exename);
+    }
+    return exename;
+}
+
+const char *
+get_application_short_name()
+{
+    static char short_exename[MAXIMUM_PATH];
+    if (!short_exename[0]) {
+        snprintf(short_exename, BUFFER_SIZE_ELEMENTS(short_exename), "%ls",
+                 get_own_short_qualified_name());
+        NULL_TERMINATE_BUFFER(short_exename);
+    }
+    return short_exename;
+}
+
+const char *
+get_application_short_unqualified_name()
+{
+    static char short_unqual_exename[MAXIMUM_PATH];
+    if (!short_unqual_exename[0]) {
+        snprintf(short_unqual_exename, BUFFER_SIZE_ELEMENTS(short_unqual_exename), "%ls",
+                 get_own_short_unqualified_name());
+        NULL_TERMINATE_BUFFER(short_unqual_exename);
+    }
+    return short_unqual_exename;
+}
+
+/* get application pid, (cached), used for event logging */
+char*
+get_application_pid()
+{
+    static char pidstr[16];
+
+    if (!pidstr[0]) {
+        process_id_t pid = get_process_id();
+        snprintf(pidstr, BUFFER_SIZE_ELEMENTS(pidstr), IDFMT, pid);
+        NULL_TERMINATE_BUFFER(pidstr);
+    }
+    return pidstr;
+}
+#endif /* NOT_DYNAMORIO_CORE */
+
+wchar_t *get_application_cmdline()
+{
+    PEB *peb = get_own_peb();
+    return peb->ProcessParameters->CommandLine.Buffer;
+}
+
+static LONGLONG
+query_time_100ns()
+{
+    /* FIXME: we could use KUSER_SHARED_DATA here, but it's too volatile
+     * since we can't programmatically grab its address (all we know is
+     * 0x7ffe0000) and it changed on win2003 (tickcount deprecated, e.g.).
+     * Since these time routines aren't currently on critical path we just
+     * use the more-stable syscalls.
+     */
+    LARGE_INTEGER systime;
+    GET_NTDLL(NtQuerySystemTime, (IN PLARGE_INTEGER SystemTime));
+    NtQuerySystemTime(&systime);
+    return systime.QuadPart;
+}
+
+/* FIXME : check how this is used, is definitely overflowing a uint, is this
+ * used for elapsed time (should use tickcount then) or a seed (ok)? */
+uint
+query_time_millis()
+{
+    LONGLONG time100ns = query_time_100ns();
+    return (uint) (time100ns / TIMER_UNITS_PER_MILLISECOND);
+}
+
+uint
+query_time_seconds()
+{
+    /* see query_time_100ns comments on KUSER_SHARED_DATA */
+    LARGE_INTEGER systime;
+    GET_NTDLL(NtQuerySystemTime, (IN PLARGE_INTEGER SystemTime));
+    
+    NtQuerySystemTime(&systime);
+    {
+        ULONG sec;
+        GET_NTDLL (RtlTimeToSecondsSince1970, (IN PLARGE_INTEGER SystemTime,
+                                               OUT PULONG Seconds));
+        /* we ignore result, it is not NT_SUCCESS */
+        RtlTimeToSecondsSince1970(&systime, &sec);
+        return sec;
+    }
+}
+
+/* NOTE - month is zero indexed */
+static const uint days_per_month_normal[12] = 
+    {31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
+static const uint days_per_month_leap[12] = 
+    {31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31};
+
+/* note that ntdll!RtlTimeToTimeFields has this same functionality */
+void
+query_system_time(SYSTEMTIME *st)
+{
+    LONGLONG time = query_time_100ns() / TIMER_UNITS_PER_MILLISECOND;
+    WORD year, month;
+    bool leap_year;
+
+    st->wMilliseconds = (WORD)(time % 1000);
+    time /= 1000;
+    st->wSecond = (WORD)(time % 60);
+    time /= 60;
+    st->wMinute = (WORD)(time % 60);
+    time /= 60;
+    st->wHour = (WORD)(time % 24);
+    /* FIXME - optimization - at this point we could move to a uint
+     * number_of_days which should be much faster in the later / and %
+     * operations than continuing to use LONGLONG time. */
+    time /= 24;
+
+    /* time is now num. of days since Sun. Jan. 1, 1601 */
+    st->wDayOfWeek = (WORD)(time % 7); /* Sun. is 0 */
+
+    /* Since 1601 is the first year of a 400 year leap year cycle, we can use
+     * the following to figure out the correct year. NOTE the 100 year and 4
+     * year values are only correct if not crossing a 400 year of 100 year
+     * (respectively) alignment. */
+#define BASE_YEAR 1601
+#define DAYS_IN_400_YEARS (400*365 + 97)
+#define DAYS_IN_100_YEARS (100*365 + 24)
+#define DAYS_IN_4_YEARS (4*365 + 1)
+    ASSERT(BASE_YEAR % 400 == 1); /* verify alignment */
+    year = (WORD)(BASE_YEAR + 400*(time / DAYS_IN_400_YEARS));
+    time %= DAYS_IN_400_YEARS;
+    year = (WORD)(year + (100*(time / DAYS_IN_100_YEARS)));
+    time %= DAYS_IN_100_YEARS;
+    year = (WORD)(year + (4*(time / DAYS_IN_4_YEARS)));
+    time %= DAYS_IN_4_YEARS;
+    year = (WORD)(year + (time / 365));
+    time %= 365;
+    leap_year = (year % 4 == 0) && ((year % 100 != 0) || (year % 400 == 0));
+    st->wYear = year;
+    
+    /* time is now num. of days since the first of the year */
+    month = 1;
+    while (month <= 12) {
+        uint days_in_month = leap_year ? 
+            days_per_month_leap[month-1] :
+            days_per_month_normal[month-1];
+        if (time >= days_in_month) {
+            month++;
+            time -= days_in_month;
+        } else
+            break;
+    }
+    ASSERT (month != 13);
+    st->wMonth = month;
+    st->wDay = (WORD)(time+1); /* day, like month, is not zero indexed */
+}
+
+
+/* returns NULL (default security descriptor) if can't setup owner,
+ * otherwise edits in place the passed in security descriptor
+ */
+static
+PSECURITY_DESCRIPTOR
+set_primary_user_owner(PSECURITY_DESCRIPTOR psd)
+{
+    bool ok;
+    initialize_security_descriptor(psd);
+
+    /* for consistency, we override the NoDefaultAdminOwner feature
+     * which creates files with owner Administrators instead of current user
+     */
+    /* note we could also just return a NULL SD, if TokenOwner == TokenUser
+     * and create an explicit one only if we really need it
+     */
+    ok = set_owner_sd(psd, get_process_primary_SID());
+    ASSERT(ok);
+    if (!ok)
+        return NULL;
+    /* FIXME: (not verified) note that even if we set owner, we may not be allowed to use it
+     * as an owner it it is not present in the current token.
+     */
+
+    /* we rely on the correct DACL to be provided through inheritance */
+    /* FIXME: we don't specify primary Group, we may end up with no primary group, 
+     * which should be OK too */
+    return psd; /* use the constructed security descriptor */
+}
+
+/* Exposes full power of NtCreateFile.  Caller should first consider
+ * create_file() or nt_create_module_file() before calling this
+ * routine directly.  See comments above nt_create_module_file() for
+ * more details on some of these arguments.
+ *
+ * Note that instead of asking for raw OBJECT_ATTRIBUTES we have
+ * enriched the NT interface with a directory handle and an additional
+ * disposition FILE_DISPOSITION_SET_OWNER.
+ */
+NTSTATUS
+nt_create_file(OUT HANDLE *file_handle, 
+               const wchar_t *filename, HANDLE dir_handle OPTIONAL,
+               size_t alloc_size, ACCESS_MASK rights, uint attributes, uint sharing,
+               uint create_disposition, uint create_options)
+{
+    NTSTATUS res;
+    OBJECT_ATTRIBUTES file_attributes;
+    IO_STATUS_BLOCK iob = {0,0};
+    UNICODE_STRING file_path_unicode;
+    SECURITY_DESCRIPTOR sd;
+    PSECURITY_DESCRIPTOR pSD = NULL; /* default security descriptor */
+    LARGE_INTEGER create_allocation_size;
+    create_allocation_size.QuadPart = alloc_size;
+
+    res = wchar_to_unicode(&file_path_unicode, filename);
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("nt_create_file: name conversion failed, res: %x\n", res);
+        return res;
+    }
+
+    if (TEST(FILE_DISPOSITION_SET_OWNER, create_disposition)) {
+        pSD = set_primary_user_owner(&sd);
+        create_disposition &= ~FILE_DISPOSITION_SET_OWNER;
+    }
+
+    InitializeObjectAttributes(&file_attributes,
+                               &file_path_unicode,
+                               OBJ_CASE_INSENSITIVE,
+                               dir_handle,
+                               pSD);
+    ASSERT(create_disposition <= FILE_MAXIMUM_DISPOSITION);
+    res = NT_SYSCALL(CreateFile, file_handle,
+                     rights, &file_attributes, &iob,
+                     (alloc_size == 0 ? NULL : &create_allocation_size),
+                     attributes, sharing,
+                     create_disposition, create_options,
+                     NULL, 0);
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("Error 0x%x in nt_create_file %S \"%S\"\n", res, filename,
+                file_path_unicode.Buffer);
+    }
+    return res;
+}
+
+/* For ordinary use of NtCreateFile */
+/* FIXME : can't simultaneously have GENERIC_READ, GENERIC_WRITE and SYNCH_IO 
+ * get invalid parameter error, but any two succeeds, makes sense because 
+ * <speculation> SYNCH_IO tells the io system to keep track of the 
+ * current file postion which should start at 0 for read and end of file for 
+ * write </speculation>, could do non_synch_io but had trouble getting that to 
+ * work with read/write */
+HANDLE
+create_file(PCWSTR filename, bool is_dir, ACCESS_MASK rights, 
+            uint sharing, uint create_disposition, bool synch)
+{
+    NTSTATUS res;
+    HANDLE hfile;
+    DEBUG_DECLARE(static const uint access_allow =
+                  READ_CONTROL|
+                  GENERIC_READ|GENERIC_WRITE|GENERIC_EXECUTE|
+                  FILE_GENERIC_READ|FILE_GENERIC_WRITE|FILE_GENERIC_EXECUTE;)
+    DEBUG_DECLARE(static const uint dir_access_allow =
+                  READ_CONTROL|
+                  0;)
+
+    /* FIXME : only support these possibilities for access mask for now 
+     * should be all we need unless we decide to export more functionality 
+     * from os_open/write/read */
+    ASSERT((synch && ((!is_dir && ((rights & (~access_allow)) == 0)) ||
+                      (is_dir && ((rights & (~dir_access_allow)) == 0)) )) ||
+           (!synch && !is_dir && (GENERIC_READ|GENERIC_WRITE) == rights));
+
+    res = nt_create_file(&hfile, filename, NULL, 0,
+                         rights|SYNCHRONIZE| (is_dir? FILE_LIST_DIRECTORY : 
+                                              FILE_READ_ATTRIBUTES), 
+                         /* CreateDirectory uses F_ATTRIB_NORM too, even though
+                          * there is a F_ATTRIB_DIR as well */
+                         FILE_ATTRIBUTE_NORMAL,
+                         sharing,
+                         create_disposition,
+                         (synch ? FILE_SYNCHRONOUS_IO_NONALERT : 0)|
+                         /* FIXME: MSDN instructs to use FILE_FLAG_BACKUP_SEMANTICS
+                          * for opening a dir handle but we don't seem to need it */
+                         (is_dir ?
+                          FILE_DIRECTORY_FILE|FILE_OPEN_FOR_BACKUP_INTENT :
+                          FILE_NON_DIRECTORY_FILE));
+    if (!NT_SUCCESS(res))
+        return INVALID_FILE;
+    else
+        return hfile;
+}
+
+#if 0
+/* FIXME : enable and test once we have a use for it */
+int
+delete_file(PCWSTR filename)
+{
+    NTSTATUS res;
+    OBJECT_ATTRIBUTES attr;
+    UNICODE_STRING objname;
+
+    GET_NTDLL(NtDeleteFile, (IN POBJECT_ATTRIBUTES  ObjectAttributes));
+    
+    res = wchar_to_unicode(&objname, filename);
+    if (!NT_SUCCESS(res))
+        return 0;
+    
+    InitializeObjectAttributes(&attr, &objname,
+                               OBJ_CASE_INSENSITIVE,
+                               NULL, NULL);
+    
+    res = NtDeleteFile(&attr);
+    
+    return NT_SUCCESS(res);
+}
+#endif
+
+int
+flush_file_buffers(HANDLE file_handle)
+{
+    NTSTATUS res;
+    IO_STATUS_BLOCK ret;
+    
+    GET_NTDLL(NtFlushBuffersFile, (IN HANDLE FileHandle,
+                                   OUT PIO_STATUS_BLOCK IoStatusBlock));
+
+    res = NtFlushBuffersFile(file_handle, &ret);
+
+    return NT_SUCCESS(res);
+}
+
+/* we don't actually use this but for cleanliness sake, is from ntddk.h */
+typedef 
+VOID
+(NTAPI *PIO_APC_ROUTINE) (
+                          IN PVOID ApcContext,
+                          IN PIO_STATUS_BLOCK IoStatusBlock,
+                          IN ULONG Reserved
+                          );
+
+int
+read_file(HANDLE file_handle, void *buffer, uint num_bytes_to_read, 
+          IN uint64* file_byte_offset OPTIONAL,
+          OUT size_t *num_bytes_read)
+{
+    NTSTATUS res;
+    IO_STATUS_BLOCK ret = {0};
+    LARGE_INTEGER ByteOffset;   /* should be the same as uint64 */
+
+    GET_NTDLL(NtReadFile, (IN HANDLE FileHandle,
+                           IN HANDLE Event OPTIONAL,
+                           IN PIO_APC_ROUTINE ApcRoutine OPTIONAL,
+                           IN PVOID ApcContext OPTIONAL,
+                           OUT PIO_STATUS_BLOCK IoStatusBlock,
+                           OUT PVOID Buffer,
+                           IN ULONG Length,
+                           IN PLARGE_INTEGER ByteOffset OPTIONAL,
+                           IN PULONG Key OPTIONAL));
+
+    if (file_byte_offset != NULL) {
+        ByteOffset.QuadPart = *file_byte_offset;
+    }
+
+    /* if file is opened with FILE_SYNCHRONOUS_IO_[NON]ALERT then can pass
+     * NULL for ByteOffset to read from current file positions, otherwise 
+     * need to pass special value to ByteOffset to read from current position
+     * (special value is highpart -1, low part FILE_USE_FILE_POINTER_POSITION),
+     *  but I can't get this to work so assuming opened with SYNCH_IO */ 
+    res = NtReadFile(file_handle, NULL, NULL, NULL, &ret, buffer, 
+                     num_bytes_to_read, 
+                     file_byte_offset != NULL ? &ByteOffset : NULL,
+                     NULL);
+
+    *num_bytes_read = ret.Information;
+    return NT_SUCCESS(res);
+}
+
+int
+write_file(HANDLE file_handle, const void *buffer, uint num_bytes_to_write, 
+           OPTIONAL uint64* file_byte_offset,
+           OUT size_t *num_bytes_written)
+{
+    NTSTATUS res;
+    IO_STATUS_BLOCK ret = {0};
+    LARGE_INTEGER ByteOffset;
+    
+    GET_NTDLL(NtWriteFile, (IN HANDLE FileHandle,
+                            IN HANDLE Event OPTIONAL,
+                            IN PIO_APC_ROUTINE ApcRoutine OPTIONAL,
+                            IN PVOID ApcContext OPTIONAL,
+                            OUT PIO_STATUS_BLOCK IoStatusBlock,
+                            IN const void *Buffer, /* PVOID, but need const */
+                            IN ULONG Length,
+                            IN PLARGE_INTEGER ByteOffset OPTIONAL,
+                            IN PULONG Key OPTIONAL));
+
+    if (file_byte_offset != NULL) {
+        ByteOffset.QuadPart = *file_byte_offset;
+    }
+    /* if file is opened with FILE_SYNCHRONOUS_IO_[NON]ALERT then can pass
+     * NULL for ByteOffset to append to end (well append to after the last 
+     * write or end if file just opened), otherwise need to pass special value
+     * to ByteOffset to append, unless opened with just append permissions in 
+     * which case always appends (special value is highpart -1, low part
+     * FILE_WRITE_TO_END_OF_FILE for middle case), but I can't get this to
+     * work so assuming opened with FILE_SYNCHRONOUS_IO_* */ 
+    res = NtWriteFile(file_handle, NULL, NULL, NULL, &ret, buffer, 
+                      num_bytes_to_write, 
+                      file_byte_offset != NULL ? &ByteOffset : NULL, 
+                      NULL);
+
+    *num_bytes_written = ret.Information;
+    return NT_SUCCESS(res);
+}
+
+int
+close_file(HANDLE hfile)
+{
+    return close_handle(hfile);
+}
+
+HANDLE
+create_iocompletion()
+{
+    NTSTATUS res;
+    HANDLE hiocompletion;
+
+    GET_NTDLL(NtCreateIoCompletion, (OUT PHANDLE IoCompletionHandle,
+                                     IN ACCESS_MASK DesiredAccess,
+                                     IN POBJECT_ATTRIBUTES ObjectAttributes,
+                                     IN ULONG NumberOfConcurrentThreads
+                                     ));
+
+    res = NtCreateIoCompletion(&hiocompletion, 
+                               EVENT_ALL_ACCESS /* 0x1f0003 */,
+                               NULL /* no name */,
+                               0 /* FIXME: 0 observed, shouldn't it be 1? */);
+
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("Error 0x%x in create IoCompletion \n", res);
+        return NULL;
+    } else
+        return hiocompletion;
+}
+
+typedef struct _FILE_PIPE_INFORMATION { // Information Class 23
+    ULONG ReadModeMessage;
+    ULONG WaitModeBlocking;
+} FILE_PIPE_INFORMATION, *PFILE_PIPE_INFORMATION;
+
+typedef struct _FILE_COMPLETION_INFORMATION { // Information Class 30
+    HANDLE IoCompletionHandle;
+    ULONG CompletionKey;
+} FILE_COMPLETION_INFORMATION, *PFILE_COMPLETION_INFORMATION;
+
+/* Takes a pipename, and an optional IoCompletion object */
+HANDLE
+open_pipe(PCWSTR pipename, HANDLE hsync)
+{
+    NTSTATUS res;
+    HANDLE h;
+    IO_STATUS_BLOCK iob;
+    FILE_PIPE_INFORMATION pipeinfo = {1 /* message */, 0 /* no wait*/}; 
+                                     /* setting this to wait doesn't work */
+    GET_RAW_SYSCALL(SetInformationFile,
+                    IN HANDLE FileHandle,
+                    OUT PIO_STATUS_BLOCK IoStatusBlock,
+                    IN PVOID FileInformation,
+                    IN ULONG FileInformationLength,
+                    IN FILE_INFORMATION_CLASS FileInformationClass);
+
+    // CHECK: object attributes we see in RegisterEventSource // 1242580, "name"
+    h = create_file(pipename, false, GENERIC_WRITE|GENERIC_READ, FILE_SHARE_READ,
+                    FILE_OPEN, false);
+    if (h == INVALID_FILE) 
+        return NULL;
+    /* FIXME: call nt_set_file_info */
+    res = NT_SYSCALL(SetInformationFile, h, &iob, &pipeinfo, sizeof(pipeinfo), FilePipeInformation);
+    if (!NT_SUCCESS(res)) {
+        /* FIXME: get __FUNCTION__ working for windows */
+        NTPRINT("Error 0x%x in %s:%d\n", res, __FILE__, __LINE__);
+        return NULL;
+    }
+
+    /* CHECK: How should synchronization work here; all I want is blocking I/O
+       then we'd skip this step, (yet we fail with FILE_SYNCHRONOUS_IO_NONALERT) */
+    
+    /* set FileCompletionInformation just like RegisterSource does
+       FIXME: The problem is that the IoCompletion that is used here is not created
+       by RegisterSource and instead an earlier one is used.
+
+       FIXME: There is a NtCreateEvent call, but that is what should go in 
+       NtFsControlFile calls, and I can't match how that handle gets used either.
+    */
+    if (hsync) {
+        FILE_COMPLETION_INFORMATION completioninfo;
+        completioninfo.IoCompletionHandle = hsync;
+        completioninfo.CompletionKey = 0xffff0000; /* observed key */
+
+        /* FIXME: call nt_set_file_info */
+        res = NT_SYSCALL(SetInformationFile, h, &iob, &completioninfo, 
+                         sizeof(completioninfo), FileCompletionInformation);
+        if (!NT_SUCCESS(res)) {
+            /* FIXME: get __FUNCTION__ working for windows */
+            NTPRINT("Error 0x%x in %s:%d\n", res, __FILE__, __LINE__);
+            return NULL;
+        }
+    }
+
+    return h;
+}
+
+/* see example in Nebbett p. 419 */
+/* status codes ntstatus.h (arg 1), also includes descriptions of expected 
+ * arguments which must match the actual arguments described by args 2-4
+ * (which are just for packaging), (i.e the status code defines the format 
+ * string), note that many status codes won't produce message box 
+ * arg 2 is number of substitutions, 
+ * arg 3 is a mask of what substitutions are pointers (i.e. strings), i.e. if
+ * substitutions 2 and 4 are strings then the 2nd and 4th bits will be set and
+ * will get 1010 or 10
+ * arg 4 is an array of ULONGS comprimising the substitutions (ULONG will be
+ * interpreted as pointer as defined by arg 3)
+ * arg5 is response options, eqv. of MB_OK, MB_YESNO etc, see Nebbet enum 418
+ * (1 is OK)
+ * arg 6 is return value from box, see Nebbet enum 418 */
+
+/* bad news is that the following is somewhat brittle, the format changed 
+ * between win2k and XP substantially (though in a forward, but not backwards
+ * compatible way).  The following uses the format for XP since that works for
+ * win2k too, but the reverse is not true.  On both platforms 
+ * ServiceMessageBox (what we use, probably because of service notification 
+ * flag?) uses the undocumented status code 0x50000018L.  On win2k a three
+ * element array with first being the msg string, the second being the title
+ * and the third element being 0x10 (which seems to be ignored) is used and 
+ * the MsgBoxType arg specifies the msg box and buttons shown.  On XP a four 
+ * element array with the first element being the msg string, 
+ * the second being the title, the third being 0x10 (which controls the msg 
+ * box and buttons shown), and the fourth being 0xffffffffL (seems to be 
+ * ignored) is used and the MsgBoxType arg seems to be ignored.  Also 
+ * having the wrong arguments for RaiseHardError can leave the machine in a
+ * bad state.  The offending thread will hang at the syscall and (if for ex.
+ * you use the 2k form on XP) the machine can be left in a state where it is
+ * unable to display any message box from any process (any thread that tries
+ * just hangs).  At one point had to power cycle the machine since couldn't
+ * shut down or get task manager to appear.  But following seems to work.
+ */
+
+#define STATUS_SHOW_MESSAGEBOX_UNDOCUMENTED (NTSTATUS)0x50000018L
+
+int
+nt_messagebox(wchar_t *msg, wchar_t *title)
+{
+    UNICODE_STRING m, t;
+    NTSTATUS res;
+    GET_NTDLL(NtRaiseHardError, (IN NTSTATUS ErrorStatus,
+                                 IN ULONG NumberOfArguments,
+                                 /* FIXME: ReactOS claims this is a PUNICODE_STRING */
+                                 IN ULONG UnicodeStringArgumentsMask,
+                                 IN PVOID Arguments,
+                                 IN ULONG MessageBoxType, /* HARDERROR_RESPONSE_OPTION */
+                                 OUT PULONG MessageBoxResult
+                                 ));
+
+    /* the 0xfff... is only for XP, win2k has three element args array, its
+     * function is unknown (doesn't seem to matter what is there) */
+    /* the 0x10 argument is ignored ? on win2k, on XP chooses the icon and
+     * response options of the resulting dialog box */
+    ULONG ret;
+    void * args[] = {NULL, NULL, (void *)(ptr_uint_t)0x10L, (void *)PTR_UINT_MINUS_1};
+
+    /* make UNICODE_STRINGs */
+    res = wchar_to_unicode(&m, msg);
+    ASSERT(NT_SUCCESS(res));
+    if (!NT_SUCCESS(res))
+        return NT_SUCCESS(res);
+    res = wchar_to_unicode(&t, title);
+    ASSERT(NT_SUCCESS(res));
+    if (!NT_SUCCESS(res))
+        return NT_SUCCESS(res);
+
+    args[0] = (void *)&m;
+    args[1] = (void *)&t;
+
+    /* see notes above */
+    /* 4 = length of args, set to 3 to match native behavior on win2k */
+    /* 1 is OptionOK dialog for win2k, but is ignored ? on XP */
+    res =  NtRaiseHardError(STATUS_SHOW_MESSAGEBOX_UNDOCUMENTED, 4, 0x1 | 0x2, 
+                            args, 1, &ret);
+
+    return NT_SUCCESS(res);
+}
+
+int
+nt_raise_exception(EXCEPTION_RECORD* pexcrec, CONTEXT* pcontext)
+{
+    NTSTATUS res;
+    GET_NTDLL(NtRaiseException, (IN PEXCEPTION_RECORD ExceptionRecord,
+                                 IN PCONTEXT Context,
+                                 IN BOOLEAN SearchFrames));
+    
+    res = NtRaiseException(pexcrec, pcontext, true);
+
+    /* we just threw an exception, shouldn't get here */
+    ASSERT_NOT_REACHED();
+
+    return NT_SUCCESS(res);
+}
+
+HANDLE
+nt_create_event(EVENT_TYPE event_type)
+{
+    NTSTATUS res;
+    HANDLE hevent;
+
+    GET_NTDLL(NtCreateEvent, (OUT PHANDLE EventHandle,
+                              IN ACCESS_MASK DesiredAccess,
+                              IN POBJECT_ATTRIBUTES ObjectAttributes,
+                              IN EVENT_TYPE EventType,
+                              IN BOOLEAN InitialState
+                              ));
+
+    res = NtCreateEvent(&hevent, 
+                        EVENT_ALL_ACCESS,
+                        NULL /* no name */,
+                        event_type, 
+                        0 /* start non-signaled */);
+
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("Error 0x%x in create event \n", res);
+        return NULL;
+    } else
+        return hevent;
+}
+
+void
+nt_close_event(HANDLE hevent)
+{
+    close_handle(hevent);
+}
+
+wait_status_t
+nt_wait_event_with_timeout(HANDLE hevent, PLARGE_INTEGER timeout)
+{
+    GET_NTDLL(NtWaitForSingleObject, (IN HANDLE ObjectHandle, 
+                                      IN BOOLEAN Alertable, 
+                                      IN PLARGE_INTEGER TimeOut ));
+    NTSTATUS res;
+    res = NtWaitForSingleObject(hevent, 
+                                false /* not alertable */,
+                                timeout);
+    if (!NT_SUCCESS(res))
+        return WAIT_ERROR;
+    if (res == STATUS_TIMEOUT)
+        return WAIT_TIMEDOUT;
+    return WAIT_SIGNALED;
+}
+
+void
+nt_set_event(HANDLE hevent)
+{
+    GET_NTDLL(NtSetEvent, (IN HANDLE EventHandle, 
+                           OUT PLONG PreviousState OPTIONAL));
+    NTSTATUS res;
+
+    res = NtSetEvent(hevent, NULL /* no previous */);
+    /* on WinXP critical sections use ZwSetEventBoostPriority, yet 
+       Inside Win2k p.362 claims we always get a boost on Win2000 */
+
+#if VERBOSE
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("Error 0x%x in set event \n", res);
+    }
+#endif
+}
+
+/* This is currently used only for manual broadcast events
+ * It looks like NtPulseEvent will not be a good idea.
+ * MSDN says that PulseEvent is bad because of kernel APCs 
+ * taking a thread out of the wait queue.  If it was only user APCs
+ * we wouldn't have to worry about it.  However, MSDN should have said that
+ * non-alertable waits will not be affected, instead they say don't use it.
+ * Therefore we are stuck with manual event handling.
+ */
+void
+nt_clear_event(HANDLE hevent)
+{
+    GET_NTDLL(NtClearEvent, (IN HANDLE EventHandle));
+    NTSTATUS res;
+
+    res = NtClearEvent(hevent);
+#if VERBOSE
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("Error 0x%x in clear event \n", res);
+    }
+#endif
+}
+
+void
+nt_signal_and_wait(HANDLE hevent_to_signal, HANDLE hevent_to_wait)
+{
+    GET_NTDLL(NtSignalAndWaitForSingleObject, 
+              (IN HANDLE ObjectToSignal, IN HANDLE WaitableObject, 
+               IN BOOLEAN Alertable, IN PLARGE_INTEGER Time OPTIONAL)); 
+    NTSTATUS res;
+    
+    res = NtSignalAndWaitForSingleObject(hevent_to_signal, 
+                                         hevent_to_wait,
+                                         false /* not alertable */,
+                                         NULL /* no timeout */);
+#if VERBOSE
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("Error 0x%x in set event \n", res);
+    }
+#endif
+}
+
+void
+nt_query_performance_counter(PLARGE_INTEGER counter, PLARGE_INTEGER frequency)
+{
+    GET_NTDLL(NtQueryPerformanceCounter, (OUT PLARGE_INTEGER PerformanceCount,
+                                          OUT PLARGE_INTEGER PerformanceFrequency OPTIONAL
+                                          ));
+    NTSTATUS res;
+    res = NtQueryPerformanceCounter(counter, frequency);
+#if VERBOSE
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("Error 0x%x in query_performance_counter  \n", res);
+    }
+#endif
+    ASSERT(NT_SUCCESS(res));
+}
+
+/* Pipe transceive macros and types */
+
+/* macros from WinIoCtl.h */
+#define CTL_CODE( DeviceType, Function, Method, Access ) (                      \
+    ((DeviceType) << 16) | ((Access) << 14) | ((Function) << 2) | (Method)      \
+)
+
+// Define the interesting device type values
+#define FILE_DEVICE_FILE_SYSTEM         0x00000009
+#define FILE_DEVICE_NAMED_PIPE          0x00000011
+
+// Define the method codes for how buffers are passed for I/O and FS controls
+#define METHOD_BUFFERED                 0
+#define METHOD_IN_DIRECT                1
+#define METHOD_OUT_DIRECT               2
+#define METHOD_NEITHER                  3
+
+// Define the access check value for any access
+#define FILE_READ_ACCESS          ( 0x0001 )    // file & pipe
+#define FILE_WRITE_ACCESS         ( 0x0002 )    // file & pipe
+
+#define FSCTL_PIPE_TRANSCEIVE CTL_CODE(FILE_DEVICE_NAMED_PIPE, 5, \
+              METHOD_NEITHER,  FILE_READ_DATA | FILE_WRITE_DATA) /* 0x11c017 */
+
+#if _MSC_VER <= 1200
+/* from ntstatus.h, NtFsControlFile typically returns this, if not using 
+ * overlapped (i.e. asynch io) then TransactNamedPipe specifically checks for 
+ * this value to determine whether or not it should wait on the pipe, this is
+ * the only return code it specifically checks for */
+# define STATUS_PENDING 0x103
+#endif
+
+#ifdef DEBUG
+# if defined(NOT_DYNAMORIO_CORE_PROPER) || defined(NOT_DYNAMORIO_CORE)
+static bool do_once_nt_pipe_transceive = false;
+# else
+DECLARE_FREQPROT_VAR(static bool do_once_nt_pipe_transceive, false);
+# endif
+#endif
+
+/* read/write from a pipe. Returns length of output if successful
+ * 0 on failure */
+size_t
+nt_pipe_transceive(HANDLE hpipe, void *input, uint input_size,
+                   void *output, uint output_size, uint timeout_ms)
+{
+    LARGE_INTEGER liDueTime;
+    NTSTATUS res;
+    IO_STATUS_BLOCK iob = {0,0};
+
+    /* NOTE use an event => asynch IO, if event caller will be notified 
+     * that routine finishes by signaling the event */
+    GET_NTDLL(NtFsControlFile, (IN HANDLE               FileHandle,
+                                IN HANDLE               Event OPTIONAL,
+                                IN PIO_APC_ROUTINE      ApcRoutine OPTIONAL,
+                                IN PVOID                ApcContext OPTIONAL,
+                                OUT PIO_STATUS_BLOCK    IoStatusBlock,
+                                IN ULONG                FsControlCode,
+                                IN PVOID                InputBuffer OPTIONAL,
+                                IN ULONG                InputBufferLength,
+                                OUT PVOID               OutputBuffer OPTIONAL,
+                                IN ULONG                OutputBufferLength));
+
+    /* FIXME shared utility for this style of computation, 
+     * is used in several places in os.c */
+    liDueTime.QuadPart= -((int)timeout_ms * TIMER_UNITS_PER_MILLISECOND);
+
+    ASSERT(hpipe);
+    res = NtFsControlFile(hpipe, NULL, 
+                          NULL, NULL, 
+                          &iob,                    
+                          FSCTL_PIPE_TRANSCEIVE,
+                          input,
+                          input_size,
+                          output,
+                          output_size);
+
+    /* make sure that I/O is complete before we go back to the client 
+       - otherwise we may corrupt the stack if output is a stack allocated buffer */
+
+    if (NT_SUCCESS(res)) {
+        /* it worked, check if we need to wait for the IO to finish */
+        if (res == STATUS_PENDING) {
+            /* we need to wait on the pipe handle */
+            /* ref case 666, sometimes when we are in services.exe this wait
+             * can hang (presumably this thread is needed on the other side
+             * of the pipe or something like that) so we timeout the wait */
+            /* name is a bit misleading (pipe vs event), but does the 
+             * right thing */
+            res = nt_wait_event_with_timeout(hpipe, &liDueTime);
+            if (res == WAIT_TIMEDOUT) {
+                /* CancelIO is wrapper for NtCancelIO, msdn claims only works
+                 * on asynch IO, but as they point out it shouldn't be possible
+                 * to use it on synch IO (the routine shouldn't return for you
+                 * to cancel).  Still not sure why the Fs... returns when synch
+                 * IO.  Try to cancel, we may have to eventually go to quasi
+                 * asynch IO though this appears to work. */
+                /* pipe == file */
+                GET_NTDLL(NtCancelIoFile, (IN HANDLE FileHandle,
+                                           OUT PIO_STATUS_BLOCK IoStatusBlock));
+                IO_STATUS_BLOCK cancel_iob;
+                NTLOG(GLOBAL, LOG_NT, 1, "pipe transceive timed out\n");
+                NTLOG(THREAD_GET, LOG_NT, 1, "pipe transceive timed out\n");
+                res = NtCancelIoFile(hpipe, &cancel_iob);
+                if (!NT_SUCCESS(res)) {
+                    /* check, maybe finished before we could cancel (if so 
+                     * our timeout was too short) */
+                    NTLOG(GLOBAL, LOG_NT, 1, 
+                          "pipe transceive cancel failed code="PFX"\n", res);
+                    NTLOG(THREAD_GET, LOG_NT, 1, 
+                          "pipe transceive cancel failed code="PFX"\n", res);
+                    res = nt_wait_event_with_timeout(hpipe, &liDueTime);
+                    if (res == WAIT_TIMEDOUT) {
+                        /* Now we are in a world of hurt, just return and hope
+                         * for the best */
+                        NTLOG(GLOBAL, LOG_NT, 1, 
+                              "pipe transceive 2nd try FAILED!\n");
+                        NTLOG(THREAD_GET, LOG_NT, 1, 
+                              "pipe transceive 2nd try FAILED!\n");
+                        /* DO_ONCE to avoid an infinite recursion here */
+                        DODEBUG({
+                            /* custom DO_ONCE to avoid selfprot link issues with NOT_DYNAMORIO_CORE_PROPER */
+                            if (!do_once_nt_pipe_transceive) {
+                                do_once_nt_pipe_transceive = true;
+                                ASSERT_NOT_REACHED();
+                            }
+                        });
+                        return 0;
+                    }
+                } else {
+                    NTLOG(GLOBAL, LOG_NT, 1, 
+                          "pipe transceive cancel succeeded code="PFX"\n", res);
+                    NTLOG(THREAD_GET, LOG_NT, 1, 
+                          "pipe transceive cancel succeeded code="PFX"\n", res);
+                    return 0;
+                }
+            }
+        } else {
+            /* completed synchronously (success) */
+            NTLOG(GLOBAL, LOG_NT, 1, "pipe transceive completed sync\n");
+            NTLOG(THREAD_GET, LOG_NT, 1, "pipe transceive completed sync\n");
+        }
+    } else {
+        NTLOG(GLOBAL, LOG_NT, 1, "pipe transceive fail\n");
+        NTLOG(THREAD_GET, LOG_NT, 1, "pipe transceive fail\n");
+        return 0;
+    }
+
+    /* length of output */
+    return iob.Information;
+};
+
+#ifdef PURE_NTDLL
+
+/* FIXME: The following should be pure ntdll.dll replacements of kernel32.dll */
+
+/* FIXME: Currently kernel32 counterparts are used */
+/* FIXME: Impersonation needs to be handled */
+
+typedef struct _THREAD_IMPERSONATION_INFORMATION {
+    HANDLE ThreadImpersonationToken;
+} THREAD_IMPERSONATION_INFORMATION, *PTHREAD_IMPERSONATION_INFORMATION;
+
+
+/* gets a handle to thread impersonation token, returns NULL on failure  */
+static
+HANDLE
+get_thread_impersonation_token(HANDLE hthread)
+{
+    NTSTATUS res;
+    HANDLE htoken;
+    ACCESS_MASK rights = 0XC;               /* CHECK */
+
+    res = NtOpenThreadToken(hthread, rights, true /* as self */, &htoken);
+
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("Error 0x%x in get thread token\n", res);
+        return NULL;
+    } else
+        return htoken;
+}
+
+/* sets impersonation token, returns 0 on failure  */
+static
+uint
+set_thread_impersonation_token(HANDLE hthread, HANDLE himptoken)
+{
+    NTSTATUS res;
+    THREAD_IMPERSONATION_INFORMATION imp_info = {himptoken};
+
+    GET_NTDLL(NtSetInformationThread, (IN HANDLE ThreadHandle,
+                                       IN THREADINFOCLASS ThreadInformationClass,
+                                       IN PVOID ThreadInformation,
+                                       IN ULONG ThreadInformationLength
+                                       ));
+    res = NtSetInformationThread(hthread, ThreadImpersonationToken, &imp_info, sizeof(imp_info));
+
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("Error 0x%x in set thread token\n", res);
+        return 0;
+    } else
+        return 1;
+}
+
+#endif /* PURE_NTDLL */
+
+#ifdef WINDOWS_PC_SAMPLE
+/* for profiling */
+
+/* buffer size is in bytes, buffer_size >= 4 * (size / 2^shift <rounded up>) */
+HANDLE
+nt_create_profile(HANDLE process_handle, void *start, uint size, uint *buffer, 
+                  uint buffer_size, uint shift)
+{
+    NTSTATUS res;
+    HANDLE prof_handle;
+
+    GET_NTDLL(NtCreateProfile, (OUT PHANDLE ProfileHandle,
+                                IN HANDLE ProcessHandle,
+                                IN PVOID Base,
+                                IN ULONG Size,
+                                IN ULONG BucketShift,
+                                IN PULONG Buffer,
+                                IN ULONG BufferLength,
+                                IN KPROFILE_SOURCE Source,
+                                IN ULONG ProcessorMask
+                                ));
+
+    /* there are restrictions on shift, check FIXME */
+
+    res = NtCreateProfile(&prof_handle, process_handle, start, size, shift, 
+                          (ULONG*)buffer, buffer_size, ProfileTime, 0);
+
+    ASSERT(NT_SUCCESS(res));
+    
+    return prof_handle;
+}
+
+void
+nt_set_profile_interval(uint nanoseconds)
+{
+    NTSTATUS res;
+
+    GET_NTDLL(NtSetIntervalProfile, (IN ULONG Interval,
+                                     IN KPROFILE_SOURCE Source
+                                     ));
+
+    res = NtSetIntervalProfile(nanoseconds, ProfileTime);
+
+    ASSERT(NT_SUCCESS(res));
+}
+
+int
+nt_query_profile_interval()
+{
+    NTSTATUS res;
+    ULONG interval;
+    
+    GET_NTDLL(NtQueryIntervalProfile, (IN KPROFILE_SOURCE Source,
+                                       OUT PULONG Interval
+                                       ));
+
+    res = NtQueryIntervalProfile(ProfileTime, &interval);
+
+    ASSERT(NT_SUCCESS(res));
+
+    return interval;
+}
+
+void
+nt_start_profile(HANDLE profile_handle)
+{
+    NTSTATUS res;
+
+    GET_NTDLL(NtStartProfile, (IN HANDLE ProfileHandle));
+
+    res = NtStartProfile(profile_handle);
+
+    ASSERT(NT_SUCCESS(res));
+}
+
+void
+nt_stop_profile(HANDLE profile_handle)
+{
+    NTSTATUS res;
+
+    GET_NTDLL(NtStopProfile, (IN HANDLE ProfileHandle));
+
+    res = NtStopProfile(profile_handle);
+
+    ASSERT(NT_SUCCESS(res));
+}
+#endif
+
+/****************************************************************************
+ * These process creation routines are based on Nebbett example 6.2 
+ */
+
+typedef struct _PORT_MESSAGE {
+    USHORT DataSize;
+    USHORT MessageSize;
+    USHORT MessageType;
+    USHORT VirtualRangesOffset;
+    CLIENT_ID ClientId;
+    ULONG MessageId;
+    ULONG SectionSize;
+    // UCHAR Data[];
+} PORT_MESSAGE, *PPORT_MESSAGE;
+
+typedef struct _CSRSS_MESSAGE {
+    ULONG Unknown1;
+    ULONG Opcode;
+    ULONG Status;
+    ULONG Unknown2;
+} CSRSS_MESSAGE, *PCSRSS_MESSAGE;
+
+/* N.B.: we now rely on this Csr routine, it works on 2K, XP, and 2003, let's
+ * hope it doesn't change in the future
+ */
+static int
+inform_csrss(HANDLE hProcess, HANDLE hthread, process_id_t pid, thread_id_t tid)
+{
+    GET_NTDLL(CsrClientCallServer, (IN OUT PVOID Message,
+                                    IN PVOID,
+                                    IN ULONG Opcode,
+                                    IN ULONG Size));
+    /* We pass a layered message with two headers to csrss.
+     * However, the two headers, PORT_MESSAGE and CSRSS_MESSAGE, are OUT values,
+     * not IN at all.  CsrClientCallServer fills in the first 4 fields of
+     * PORT_MESSAGE and the first 2 fields of CSRSS_MESSAGE.
+     * It adds 0x10, the size of CSRSS_MESSAGE, to the size passed in when
+     * it passes this buffer to NtRequestWaitReplyPort, as everything after
+     * PORT_MESSAGE is data for the LPC to csrss.
+     * Coming out, everything is now filled in except the final field of CSRSS_MESSAGE.
+     * The CreateProcessInternalW code that calls CsrClientCallServer pushes
+     * the opcode and the size as immediates so this is all known at compile time.
+     */
+    struct {
+        PORT_MESSAGE PortMessage; /* port header */
+        /* port data follows */
+        CSRSS_MESSAGE CsrssMessage; /* csrss header */
+        /* csrss data follows */
+        PROCESS_INFORMATION ProcessInformation;
+        CLIENT_ID Debugger;
+        ULONG CreationFlags;
+        ULONG VdmInfo[2];
+        /* the above csrss data fields (size 0x24) are all that's passed to NT,
+         * but other platforms have more, always observed to be 0, max of 0x98 on XP:
+         */
+        ULONG Unknown[0x98-0x24];
+    } csrmsg = {{0}, {0}, {hProcess, hthread, (DWORD)pid, (DWORD)tid}, {0}, 0, {0}, {0}};
+
+    uint size = 0x24;
+    PEB *peb = get_own_peb();
+    /* Note the discrepancy: CLIENT_ID and PROCESS_BASIC_INFORMATION use
+     * HANDLE or ULONG_PTR for the ids, but here we have DWORD, and
+     * Windows API routines like kernel32!GetProcessId return DWORD.
+     */
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(pid)));
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(tid)));
+    if (peb->OSMajorVersion == 4)
+        size = 0x24; /* NT */
+    else {
+        ASSERT(peb->OSMajorVersion == 5);
+        if (peb->OSMinorVersion == 0)
+            size = 0x28; /* 2000 */
+        else if (peb->OSMinorVersion == 1)
+            size = 0x98; /* XP */
+        else if (peb->OSMinorVersion == 2)
+            size = 0x90; /* 2003 */
+    }
+    return NT_SUCCESS(CsrClientCallServer(&csrmsg, 0, 0x10000, size));
+}
+
+static wchar_t *
+copy_environment(HANDLE hProcess)
+{
+    /* this is precisely what KERNEL32!GetEnvironmentStringsW returns: */
+    wchar_t *env = (wchar_t *) get_own_peb()->ProcessParameters->Environment;
+    SIZE_T n;
+    SIZE_T m;
+    void *p;
+
+    for (n = 0; env[n] != 0; n += wcslen(env + n) + 1)
+        ;
+    n *= sizeof(*env);
+
+    m = n;
+    p = NULL;
+    if (!NT_SUCCESS(NT_SYSCALL(AllocateVirtualMemory, hProcess, &p, 0, &m,
+                               MEM_COMMIT, PAGE_READWRITE)))
+        return NULL;
+    if (!nt_write_virtual_memory(hProcess, p, env, n, 0))
+        return NULL;
+    return (wchar_t *) p;
+}
+
+static NTSTATUS
+create_process_parameters(HANDLE hProcess, PEB *peb, UNICODE_STRING *imagefile,
+                          UNICODE_STRING *cmdline)
+{
+    RTL_USER_PROCESS_PARAMETERS *pp;
+    SIZE_T n;
+    void *p;
+    GET_NTDLL(RtlCreateProcessParameters, (OUT PRTL_USER_PROCESS_PARAMETERS *ProcessParameters,
+                                           IN PUNICODE_STRING ImageFile,
+                                           IN PUNICODE_STRING DllPath OPTIONAL,
+                                           IN PUNICODE_STRING CurrentDirectory OPTIONAL,
+                                           IN PUNICODE_STRING CommandLine OPTIONAL,
+                                           IN ULONG CreationFlags,
+                                           IN PUNICODE_STRING WindowTitle OPTIONAL,
+                                           IN PUNICODE_STRING Desktop OPTIONAL,
+                                           IN PUNICODE_STRING Reserved OPTIONAL,
+                                           IN PUNICODE_STRING Reserved2 OPTIONAL));
+    GET_NTDLL(RtlDestroyProcessParameters, (IN PRTL_USER_PROCESS_PARAMETERS ProcessParameters));
+
+    RtlCreateProcessParameters(&pp, imagefile, 0, 0, cmdline, 0, 0, 0, 0, 0);
+    pp->Environment = copy_environment(hProcess);
+    if (pp->Environment == NULL)
+        return 0;
+    n = pp->Length;
+    p = NULL;
+    if (!NT_SUCCESS(NT_SYSCALL(AllocateVirtualMemory, hProcess, &p, 0, &n,
+                               MEM_COMMIT, PAGE_READWRITE)))
+        return 0;
+    if (!nt_write_virtual_memory(hProcess, p, pp, pp->Length, 0))
+        return 0;
+    /* update the pointer in child's PEB */
+    if (!nt_write_virtual_memory(hProcess, &peb->ProcessParameters, &p, sizeof(void*), 0))
+        return 0;
+    if (!NT_SUCCESS(RtlDestroyProcessParameters(pp)))
+        return 0;
+    return 1;
+}
+
+/* avoid needing x86_code.c from x86.asm from get_own_context_helper() */
+#if !defined(NOT_DYNAMORIO_CORE) && !defined(NOT_DYNAMORIO_CORE_PROPER)
+/* Executable name must be in kernel object name form
+ * (e.g., \SystemRoot\System32\notepad.exe, or \??\c:\foo\bar.exe)
+ * The executable name on the command line can be in any form.
+ * On success returns a handle for the child
+ * On failure returns INVALID_HANDLE_VALUE
+ */
+HANDLE
+create_process(wchar_t *exe, wchar_t *cmdline)
+{
+    UNICODE_STRING uexe;
+    UNICODE_STRING ucmdline;
+    HANDLE hProcess = INVALID_HANDLE_VALUE;
+    HANDLE hthread= INVALID_HANDLE_VALUE;
+    HANDLE hSection= INVALID_HANDLE_VALUE;
+    HANDLE hFile= INVALID_HANDLE_VALUE;
+    OBJECT_ATTRIBUTES oa;
+    IO_STATUS_BLOCK iosb;
+    SECTION_IMAGE_INFORMATION sii;
+    thread_id_t tid;
+    PROCESS_BASIC_INFORMATION pbi;
+
+    GET_NTDLL(NtCreateSection, (OUT PHANDLE SectionHandle,
+                                IN ACCESS_MASK DesiredAccess,
+                                IN POBJECT_ATTRIBUTES ObjectAttributes,
+                                IN PLARGE_INTEGER SectionSize OPTIONAL,
+                                IN ULONG Protect,
+                                IN ULONG Attributes,
+                                IN HANDLE FileHandle));
+    GET_NTDLL(NtCreateProcess, (OUT PHANDLE ProcessHandle,
+                                IN ACCESS_MASK DesiredAccess,
+                                IN POBJECT_ATTRIBUTES ObjectAttributes,
+                                IN HANDLE InheritFromProcessHandle,
+                                IN BOOLEAN InheritHandles,
+                                IN HANDLE SectionHandle OPTIONAL,
+                                IN HANDLE DebugPort OPTIONAL,
+                                IN HANDLE ExceptionPort OPTIONAL));
+    GET_NTDLL(NtTerminateProcess, (IN HANDLE ProcessHandle OPTIONAL,
+                                   IN NTSTATUS ExitStatus));
+
+    NTPRINT("create_process starting\n");
+    if (!NT_SUCCESS(wchar_to_unicode(&uexe, exe)))
+        goto creation_error;
+    if (!NT_SUCCESS(wchar_to_unicode(&ucmdline, cmdline)))
+        goto creation_error;
+
+    /* create a section and a process that maps it in */
+    InitializeObjectAttributes(&oa, &uexe, OBJ_CASE_INSENSITIVE, NULL, NULL);
+    if (!NT_SUCCESS(NtOpenFile(&hFile, FILE_EXECUTE | SYNCHRONIZE, &oa, &iosb,
+                               FILE_SHARE_READ, FILE_SYNCHRONOUS_IO_NONALERT))) {
+        NTPRINT("create_process: failed to open file %S\n", uexe.Buffer);
+        goto creation_error;
+    }
+    oa.ObjectName = 0;
+    if (!NT_SUCCESS(NtCreateSection(&hSection, SECTION_ALL_ACCESS, &oa,
+                                    0, PAGE_EXECUTE, SEC_IMAGE, hFile))) {
+        NTPRINT("create_process: failed to create section\n");
+        goto creation_error;
+    }
+    close_file(hFile);
+    if (!NT_SUCCESS(NtCreateProcess(&hProcess, PROCESS_ALL_ACCESS, &oa,
+                                    NT_CURRENT_PROCESS, TRUE, hSection, 0, 0))) {
+        NTPRINT("create_process: failed to create process\n");
+        goto creation_error;
+    }
+    if (!NT_SUCCESS(NtQuerySection(hSection, SectionImageInformation, &sii, sizeof(sii), 0))) {
+        NTPRINT("create_process: failed to query section\n");
+        goto creation_error;
+    }
+    close_handle(hSection);
+    NTPRINT("create_process: created section and process\n");
+
+    /* FIXME : if thread returns from its EntryPoint function will crash because
+     * create_thread skips the kernel32 ThreadStartThunk */
+    /* FIXME : need to know whether target process is 32bit or 64bit, for now
+     * assume 32bit. */
+    hthread = create_thread(hProcess, false, sii.EntryPoint, NULL, NULL, 0,
+                            sii.StackReserve, sii.StackCommit, TRUE, &tid);
+
+    if (hthread == INVALID_HANDLE_VALUE) {
+        NTPRINT("create_process: failed to create thread\n");
+        goto creation_error;
+    }
+
+    if (!NT_SUCCESS(query_process_info(hProcess, &pbi))) {
+        NTPRINT("create_process: failed to query process info\n");
+        goto creation_error;
+    }
+
+    if (!create_process_parameters(hProcess, pbi.PebBaseAddress, &uexe, &ucmdline)) {
+        NTPRINT("create_process: failed to create process parameters\n");
+        goto creation_error;
+    }
+
+    if (!inform_csrss(hProcess, hthread, pbi.UniqueProcessId, tid)) {
+        NTPRINT("create_process: failed to inform csrss\n");
+        goto creation_error;
+    }
+
+    if (!nt_thread_resume(hthread, NULL)) {
+        NTPRINT("create_process: failed to resume initial thread\n");
+        goto creation_error;
+    }
+
+    close_handle(hthread);
+    NTPRINT("create_process: successfully created process %d!\n", pbi.UniqueProcessId);
+    return hProcess;
+
+ creation_error:
+    if (hFile != INVALID_HANDLE_VALUE)
+        close_file(hFile);
+    if (hSection != INVALID_HANDLE_VALUE)
+        close_handle(hSection);
+    if (hthread != INVALID_HANDLE_VALUE)
+        close_handle(hthread);
+    if (hProcess != INVALID_HANDLE_VALUE) {
+        NtTerminateProcess(hProcess, 0);
+        close_handle(hProcess);
+    }
+    return INVALID_HANDLE_VALUE;
+}
+
+/* NOTE does not inform csrss, if caller wants csrss informed must do it
+ * themselves (see inform_csrss). If csrss isn't informed then the stack will
+ * probably not be freed when the thread exits and certain other apps (some 
+ * cygwin versions, debuggers) will choke on these threads. Threads created with
+ * this routine must also kill themselves as opposed to returning from their
+ * start routines (we skip the kernel32 ThreadStartThunk since we can't
+ * programatically get its address) and no top-level exception handler is set up
+ * (again the kernel32 StartThunk does that). FIXME on Vista the StartThunk equivalent
+ * ntdll!RtlUserThreadStart is exported so we could target it on that platform.
+ */
+/* If arg_buf != NULL then arg_buf_size bytes are copied from arg_buf to the new thread's
+ * stack and a pointer to that is passed as the argument to the thread routine instead of
+ * arg.
+ */
+/* returns INVALID_HANDLE_VALUE on error */
+HANDLE
+create_thread(HANDLE hProcess, bool target_64bit, void *start_addr,
+              void *arg, const void *arg_buf, size_t arg_buf_size,
+              uint stack_reserve, uint stack_commit, bool suspended, thread_id_t *tid)
+{
+    HANDLE hthread;
+    USER_STACK stack = {0};
+    OBJECT_ATTRIBUTES oa;
+    CLIENT_ID cid;
+    /* Context must be 16 byte aligned on 64bit */
+    byte context_buf[sizeof(CONTEXT)+16] = {0};
+    CONTEXT *context = (CONTEXT *)ALIGN_FORWARD(context_buf, 16);
+    uint num_commit_bytes, old_prot;
+    void *p, *thread_arg = arg;
+    size_t written;
+    ptr_uint_t final_stack = 0;
+    NTSTATUS res;
+    GET_RAW_SYSCALL(CreateThread, 
+                    OUT PHANDLE ThreadHandle,
+                    IN ACCESS_MASK DesiredAccess,
+                    IN POBJECT_ATTRIBUTES ObjectAttributes,
+                    IN HANDLE ProcessHandle,
+                    OUT PCLIENT_ID ClientId,
+                    IN PCONTEXT ThreadContext,
+                    IN PUSER_STACK UserStack,
+                    IN BOOLEAN CreateSuspended);
+
+    InitializeObjectAttributes(&oa, NULL, OBJ_CASE_INSENSITIVE, NULL, NULL);
+  
+    ASSERT(stack_commit + PAGE_SIZE <= stack_reserve &&
+           ALIGNED(stack_commit, PAGE_SIZE) && 
+           ALIGNED(stack_reserve, PAGE_SIZE));
+    if (!NT_SUCCESS(nt_remote_allocate_virtual_memory(hProcess, 
+                                                      &stack.ExpandableStackBottom,
+                                                      stack_reserve, PAGE_READWRITE,
+                                                      MEM_RESERVE))) {
+        NTPRINT("create_thread: failed to allocate stack\n");
+        return INVALID_HANDLE_VALUE;
+    }
+    /* FIXME : for failures beyond this point we don't bother deallocating the
+     * stack. */
+    stack.ExpandableStackBase = 
+        ((byte *)stack.ExpandableStackBottom) + stack_reserve;
+    stack.ExpandableStackLimit = 
+        ((byte *)stack.ExpandableStackBase) - stack_commit;
+    num_commit_bytes = stack_commit + PAGE_SIZE;
+    p = ((byte *)stack.ExpandableStackBase) - num_commit_bytes;
+    if (!NT_SUCCESS(nt_remote_allocate_virtual_memory(hProcess, &p, num_commit_bytes,
+                                                      PAGE_READWRITE, MEM_COMMIT))) {
+        NTPRINT("create_thread: failed to commit stack pages\n");
+        return INVALID_HANDLE_VALUE;
+    }
+    if (!nt_remote_protect_virtual_memory(hProcess, p, PAGE_SIZE,
+                                          PAGE_READWRITE|PAGE_GUARD,
+                                          &old_prot)) {
+        NTPRINT("create_thread: failed to protect stack guard page\n");
+        return INVALID_HANDLE_VALUE;
+    }
+
+    /* set the context: initialize with our own
+     * We need CONTEXT_CONTROL and CONTEXT_INTEGER for setting the state here.  Also,
+     * on 2k3 (but not XP) we appear to need CONTEXT_SEGMENTS (xref PR 269230) as well.
+     * FIXME - on 64-bit CONTEXT_FULL includes CONTEXT_FLOATING_POINT (though not
+     * CONTEXT_SEGMENTS) so might be nice to grab that as well once PR 266070 is
+     * implemented. */
+    context->ContextFlags = CONTEXT_CONTROL | CONTEXT_INTEGER | CONTEXT_SEGMENTS;
+    GET_OWN_CONTEXT(context);
+    context->CXT_XSP = (ptr_uint_t) stack.ExpandableStackBase;
+    context->CXT_XIP = (ptr_uint_t) start_addr;
+
+    /* write the argument(s) */
+    if (arg_buf != NULL) {
+        context->CXT_XSP -= arg_buf_size;
+        thread_arg = (void *)context->CXT_XSP;
+        if (!nt_write_virtual_memory(hProcess, thread_arg, arg_buf,
+                                     arg_buf_size, &written) ||
+            written != arg_buf_size) {
+            NTPRINT("create_thread: failed to write arguments\n");
+            return INVALID_HANDLE_VALUE;
+        }
+    }
+
+    /* set up function call */
+    if (target_64bit) {
+        context->CXT_XCX = (ptr_uint_t)thread_arg;
+        /* 64-bit Windows requires 16-byte stack alignment (see calling convention) */
+        context->CXT_XSP = ALIGN_BACKWARD(context->CXT_XSP, 16);
+        /* leave spill space for 4 64-byte registers (see calling convention) */
+        context->CXT_XSP -= 4 * sizeof(uint64);
+        /* push the return address (i.e. 0) */
+        context->CXT_XSP -= 8;
+    } else {
+        void *buf[2];
+        buf[1] = thread_arg;
+        buf[0] = NULL; /* would be return address */
+        context->CXT_XSP -= sizeof(buf);
+        if (!nt_write_virtual_memory(hProcess, (void *)context->CXT_XSP, buf,
+                                     sizeof(buf), &written) || 
+            written != sizeof(buf)) {
+            NTPRINT("create_thread: failed to write argument\n");
+            return INVALID_HANDLE_VALUE;
+        }
+    }
+    final_stack = context->CXT_XSP;
+
+    /* create the thread - NOTE always creating suspended (see below) */
+    res = NT_SYSCALL(CreateThread, &hthread, THREAD_ALL_ACCESS, &oa, 
+                     hProcess, &cid, context, &stack, (byte)TRUE);
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("create_thread: failed to create thread\n");
+        return INVALID_HANDLE_VALUE;
+    }
+    /* Xref PR 252008 & PR 252745 - on 32-bit Windows the kernel will set esp 
+     * for the initialization APC to the value supplied in the context when the thread
+     * was created. However, on WOW64 and 64-bit Windows the kernel sets xsp to 20 bytes
+     * in from the stack base for the initialization APC. Since we have data/arguments
+     * sitting on the stack we need to explicitly set the context before we let the
+     * thread run the initialization APC. */
+    nt_get_context(hthread, context);
+    context->CXT_XSP = final_stack;
+    nt_set_context(hthread, context);
+    if (!suspended) {
+        int prev_count;
+        nt_thread_resume(hthread, &prev_count);
+    }
+    if (tid != NULL)
+        *tid = (thread_id_t)cid.UniqueThread;
+    return hthread;
+}
+#endif /* !defined(NOT_DYNAMORIO_CORE) && !defined(NOT_DYNAMORIO_CORE_PROPER) */
+
+
+/* Except where otherwise noted the following notes are taken from analysis of
+ * LoadLibrary on xpsp2 (\\araksha-tim). */
+/* The kernel32 version of this function does some additional work. It checks
+ * to see if you are loading twain_32.dll and handles that separately. It also
+ * does the necessary string conversions to ntdll formats. When kernel32 calls
+ * LdrLoadDll it passes in a ; separated path string for PathToFile. The second
+ * argument is a little trickier. Some sources say it's a ULONG flags argument
+ * (that corresponds to Ex ver flags) while others say its PDWORD LdrErr. On
+ * the platforms observed, xpsp2 (\\araksha-tim) and win2k (test-east), it's
+ * definitely a pointer to a stack location that appears to hold a flag value.
+ * The flag value does not match the Ex flags however. It doesn't appear to be
+ * IN/OUT argument (or OUT error) since I've never seen it written to, even when
+ * the loader had an error). A Summary of observed 2nd arguments:
+ *
+ * for xpsp2 (araksha-tim) & win2k (test-east) (all values hex)
+ * Ex Flag             2nd arg to LdrLoadDll    2nd arg deref
+ * 1                   stack ptr                2
+ * 0,8                 stack ptr                0
+ * 2 (dll not loaded) calls BasepLoadLibraryAsDataFile instead of LdrLoadDll
+ * 2 (dll loaded)      NULL
+ * 10 (xpsp2 only)     stack ptr                1000
+ *
+ * See msdn for explanation of Ex flag values (0 is what we want and what the
+ * non Ex versions use). The 2nd argument definitely appears to PDWORD/PULONG
+ * and not a pointer to a larger struct, the next value on the stack after the
+ * deref value is uninitialized (both before and after LdrLoadDll is called, even
+ * if LdrLoadDll has an error). The argument does indeed appear to be optional.
+ * Our load_library appears to work fine with either a ptr to 0 or NULL as the
+ * 2nd argument (DllMain is called, load count adjusted correctly etc. either
+ * way). Another mystery is that LoadLibraryExW also goes to the trouble of
+ * building a UNICODE_STRING version of PathToFile, but then doesn't appear to
+ * use it, perhaps it is for an unusual path through the function.
+ * FIXME : understand behavior more.
+ */
+/* returns NULL on failure */
+module_handle_t
+load_library(wchar_t *lib_name)
+{
+    UNICODE_STRING ulib_name;
+    HANDLE hMod;
+    NTSTATUS res;
+    ULONG flags = 0;
+    GET_NTDLL(LdrLoadDll, (IN PCWSTR PathToFile OPTIONAL,
+                           IN PULONG Flags OPTIONAL,
+                           IN PUNICODE_STRING ModuleFileName,
+                           OUT PHANDLE ModuleHandle));
+
+    /* we CANNOT be holding any DR locks here, since we are going to
+     * execute app code (we call LdrLoadDll) that may grab app locks
+     */
+    ASSERT_OWN_NO_LOCKS();
+    wchar_to_unicode(&ulib_name, lib_name);
+    res = LdrLoadDll(NULL, &flags, &ulib_name, &hMod);
+    if (!NT_SUCCESS(res))
+        return NULL;
+    return hMod;
+}
+
+/* Kernel32 FreeLibrary is a simple wrapper around this routine normally.
+ * However, if the lsb of the module handle is set, unmaps and calls 
+ * LdrUnloadAlternateResourceModule. For our usage (which is always real dlls)
+ * I think this should be fine. */
+bool
+free_library(module_handle_t lib)
+{
+    NTSTATUS res;
+    GET_NTDLL(LdrUnloadDll, (IN HANDLE ModuleHandle));
+    /* we CANNOT be holding any DR locks here, since we are going to
+     * execute app code (we call LdrLoadDll) that may grab app locks
+     */
+    ASSERT_OWN_NO_LOCKS();
+    res = LdrUnloadDll(lib);
+    return NT_SUCCESS(res);
+}
+
+/* FIXME : the following function (get_module_handle)
+ * should really be implemented in module.c rather than as wrappers to the
+ * undocumented ntdll ldr routines. */
+
+/* The Kernel32 version appears to be more or less a wrapper around this 
+ * function. The kernel32 version has lots of code for processing the name into
+ * a unicode string and what looks like handling the flags for the ex version. */
+/* returns NULL on failure */
+module_handle_t
+get_module_handle(wchar_t *lib_name)
+{
+    UNICODE_STRING ulib_name;
+    HANDLE hMod;
+    NTSTATUS res;
+    /* NOTE - I've seen the first arugment be 0, 1, or a pointer to a ; seperated
+     * path string.  GetModuleHandle usually seems to use 1 though I have no idea
+     * what that means.  Seems to work fine either way (doesn't seem to adjust 
+     * the load count which was my first guess). */
+#define LDR_GET_DLL_HANDLE_ARG1 ((PCWSTR)PTR_UINT_1)
+    GET_NTDLL(LdrGetDllHandle, (IN PCWSTR PathToFile OPTIONAL,
+                                IN ULONG Unused OPTIONAL,
+                                IN PUNICODE_STRING ModuleFileName,
+                                OUT PHANDLE ModuleHandle));
+    
+    /* we CANNOT be holding any DR locks here, since we are going to
+     * execute app code (we call LdrLoadDll) that may grab app locks
+     */
+    ASSERT_OWN_NO_LOCKS();
+    wchar_to_unicode(&ulib_name, lib_name);
+    res = LdrGetDllHandle(LDR_GET_DLL_HANDLE_ARG1, 0, &ulib_name, &hMod);
+    if (!NT_SUCCESS(res))
+        return NULL;
+    return hMod;
+}
+
+/* complete wrapper around NtMapViewOfSection 
+ * Note returns raw NTSTATUS.
+ *
+ */
+NTSTATUS
+nt_map_view_of_section(IN HANDLE           SectionHandle,
+                       IN HANDLE           ProcessHandle,
+                       IN OUT PVOID       *BaseAddress,
+                       IN ULONG            ZeroBits,
+                       IN SIZE_T           CommitSize,
+                       IN OUT PLARGE_INTEGER  SectionOffset OPTIONAL,
+                       IN OUT PSIZE_T      ViewSize,
+                       IN SECTION_INHERIT  InheritDisposition,
+                       IN ULONG            AllocationType,
+                       IN ULONG            Protect
+                       )
+{
+    NTSTATUS res;
+    res = NT_SYSCALL(MapViewOfSection, 
+                     SectionHandle, ProcessHandle, BaseAddress,
+                     ZeroBits, CommitSize, SectionOffset, 
+                     ViewSize, InheritDisposition, AllocationType, Protect);
+    return res;
+}
+
+/* complete wrapper around NtUnmapViewOfSection
+ * Note returns raw NTSTATUS
+ */
+NTSTATUS
+nt_unmap_view_of_section(IN HANDLE         ProcessHandle,
+                         IN PVOID          BaseAddress
+                         )
+{
+    NTSTATUS res;
+    res = NT_SYSCALL(UnmapViewOfSection, 
+                     ProcessHandle, BaseAddress);
+    return res;
+}
+
+#if !defined(NOT_DYNAMORIO_CORE_PROPER) && !defined(NOT_DYNAMORIO_CORE)
+/* Mostly a wrapper around NtCreateDirectoryObject.
+ *
+ * Note that dacl == NULL allows only owner to use the object - 
+ * sufficient for sharing only between processes of one user.
+ */
+NTSTATUS
+nt_create_object_directory(OUT HANDLE *directory /* OUT */,
+                           PCWSTR object_directory_name,
+                           bool permanent_directory,
+                           PSECURITY_DESCRIPTOR dacl)
+{
+    NTSTATUS res;
+    UNICODE_STRING directory_name;
+    OBJECT_ATTRIBUTES directory_attributes = {0};
+
+    GET_NTDLL(NtCreateDirectoryObject, (OUT PHANDLE DirectoryHandle,
+                                        IN ACCESS_MASK DesiredAccess,
+                                        IN POBJECT_ATTRIBUTES ObjectAttributes
+                                        ));
+
+    res = wchar_to_unicode(&directory_name, object_directory_name);
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("nt_create_object_directory: base name conversion failed, res: %x\n", res);
+        return res;
+    }
+
+    /* see DDK about all other flags */
+    InitializeObjectAttributes(&directory_attributes,
+                               &directory_name,
+                               (permanent_directory ? OBJ_PERMANENT : 0) | OBJ_OPENIF | OBJ_CASE_INSENSITIVE,
+                               NULL, 
+                               /* no root, directory name should be fully qualified */
+                               dacl);
+    res = NtCreateDirectoryObject(directory,
+                                  DIRECTORY_ALL_ACCESS, /* for creation */
+                                  &directory_attributes);
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("nt_create_object_directory: failed to create directory\n");
+        return res;
+    }
+
+    return res;
+}
+
+/* only privileged processes will be allowed to create the directory
+ * and set DACLs.
+ *
+ * creating a permanent (until next reboot) directory requires
+ * SeCreatePermanentPrivilege.  Note that most user mode processes do
+ * not have this privilege.  Most services, including winlogon.exe,
+ * for sure have it though.  
+ *
+ * If we pick any of these with ASLR_SHARED_INITIALIZE we'll be able
+ * to set up a directory for all other processes (in fact all
+ * processes can try but only the services will be able to create it,
+ * and only the first one will succeed so indeed can have all.  Only
+ * unusual case will be if a low user process is started before a
+ * privileged process and therefore prevents a permanent directory
+ * from being created.
+ *
+ */
+NTSTATUS
+nt_initialize_shared_directory(HANDLE *shared_directory /* OUT */, 
+                               bool permanent)
+{
+    NTSTATUS res;
+    HANDLE basedh = INVALID_HANDLE_VALUE;
+    HANDLE dh = INVALID_HANDLE_VALUE;
+
+    /* FIXME: TOFILE: need to create at least some reasonable DACL,
+     * note that NULL allows only creator to use, so it is not as bad
+     * as Everyone, but then prevents lower privileged users from even
+     * using this Directory
+     */
+    PSECURITY_DESCRIPTOR dacl = NULL;
+    /* The ACLs in the default security descriptor come from the
+     * primary or impersonation token of the creator.  So in fact we
+     * won't be able to do open this from others as is too restrictive
+     * in this instance.
+     */
+    
+    /* Create base object directory '\Determina' */
+    res = nt_create_object_directory(&basedh, DYNAMORIO_SHARED_OBJECT_BASE,
+                                     permanent, dacl);
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("nt_create_shared_directory: failed to create shared directory\n");
+        return res;
+    }
+
+
+    /* SECURITY_LOCAL_SID_AUTHORITY should be allowed to read all contents
+     * to allow sharing between all process
+     */
+    dacl = NULL;
+
+    /* Create shared DLL object directory '\Determina\SharedCache' */
+    /* FIXME: we will need directories for specific SIDs, and further
+     * restrict which processes can read what.  See
+     * ASLR_SHARED_INITIALIZE.  Even this shared cache security
+     * settings would need to be strengthened.
+     */
+    res = nt_create_object_directory(&dh, DYNAMORIO_SHARED_OBJECT_DIRECTORY,
+                                     permanent, dacl);
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("nt_create_shared_directory: failed to create shared directory\n");
+        return res;
+    }
+
+    /* FIXME: note the dual use of the permanent flag here - in
+     * addition to controlling the OBJ_PERMANENT object creation
+     * attribute, for INTERNAL uses when we don't have a proper
+     * initializer we simulate permanence by keeping a handle open in
+     * the creating process */
+    if (permanent) {
+        /* close base handle only if permanent, otherwise dh=\...\SharedCache,
+         * but subsequent lookup by name can't find \Determina\SharedCache,
+         * so closing this handle would not be really useful for a non-permanent
+         */
+        close_handle(basedh);
+    }
+    
+    /* caller only needs leaf node */
+    *shared_directory = dh;
+    return res;
+}
+
+/* any process should be able to open the shared mappings directory,
+ * and maybe even add entries to it given high enough permissions
+ */
+NTSTATUS
+nt_open_object_directory(HANDLE *shared_directory /* OUT */, 
+                         PCWSTR object_directory_name,
+                         bool allow_object_creation)
+{
+    NTSTATUS res;
+    UNICODE_STRING directory_name;
+    OBJECT_ATTRIBUTES directory_attributes = {0};
+    HANDLE dh = INVALID_HANDLE_VALUE;
+
+    GET_NTDLL(NtOpenDirectoryObject, (OUT PHANDLE DirectoryHandle,
+                                      IN ACCESS_MASK DesiredAccess,
+                                      IN POBJECT_ATTRIBUTES ObjectAttributes
+                                      ));
+
+    res = wchar_to_unicode(&directory_name, object_directory_name);
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("nt_open_object_directory: name conversion failed, res: %x\n", res);
+        return res;
+    }
+
+    /* see DDK about all other flags */
+    InitializeObjectAttributes(&directory_attributes,
+                               &directory_name,
+                               OBJ_CASE_INSENSITIVE,
+                               NULL, 
+                               /* no root, directory name should be fully qualified */
+                               NULL);
+    res = NtOpenDirectoryObject(&dh,
+                                DIRECTORY_QUERY | DIRECTORY_TRAVERSE | 
+                                /* should it try to obtain permission
+                                 * to create objects (e.g. publisher) */
+                                (allow_object_creation ? DIRECTORY_CREATE_OBJECT : 0),
+                                &directory_attributes);
+    /* note DIRECTORY_CREATE_OBJECT doesn't allow creating subdirs,
+     * for which DIRECTORY_CREATE_SUBDIRECTORY is needed */
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("nt_open_object_directory: failed to open res: %x\n", res);
+        return res;
+    }
+    /* FIXME: we could retry if we can't get higher permissions */
+
+    *shared_directory = dh;
+    return res;
+}
+
+void
+nt_close_object_directory(HANDLE hobjdir)
+{
+    close_handle(hobjdir);
+}
+
+/* Returns the symbolic link target in target_name. */
+/* note target_name should be initialized with a valid Buffer and MaximumLength
+ * Also according to the DDK returned_byte_length may be an IN
+ * argument setting max bytes to copy 
+ */
+NTSTATUS
+nt_get_symlink_target(IN HANDLE              directory_handle,
+                      IN PCWSTR              symlink_name,
+                      IN OUT UNICODE_STRING *target_name,
+                      OUT uint              *returned_byte_length)
+{
+    NTSTATUS res;
+    UNICODE_STRING link_unicode_name;
+    OBJECT_ATTRIBUTES link_attributes = {0};
+    HANDLE link_handle = INVALID_HANDLE_VALUE;
+
+    GET_NTDLL(NtOpenSymbolicLinkObject, (OUT PHANDLE DirectoryHandle,
+                                         IN ACCESS_MASK DesiredAccess,
+                                         IN POBJECT_ATTRIBUTES ObjectAttributes
+                                         ));
+    GET_NTDLL(NtQuerySymbolicLinkObject, (IN HANDLE DirectoryHandle,
+                                          IN OUT PUNICODE_STRING TargetName,
+                                          OUT PULONG ReturnLength OPTIONAL
+                                          ));
+    
+    res = wchar_to_unicode(&link_unicode_name, symlink_name);
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("nt_get_symlink_target: name conversion failed, res: %x\n", res);
+        return res;
+    }
+
+    InitializeObjectAttributes(&link_attributes,
+                               &link_unicode_name,
+                               OBJ_CASE_INSENSITIVE 
+                               /* note it doesn't seem to require |
+                                * OBJ_OPENLINK and in fact returns
+                                * STATUS_INVALID_PARAMETER when that
+                                * is set
+                                */,
+                               directory_handle,
+                               NULL);
+    res = NtOpenSymbolicLinkObject(&link_handle,
+                                   SYMBOLIC_LINK_QUERY,
+                                   &link_attributes);
+    if (!NT_SUCCESS(res)) {
+        return res;
+    }
+
+    res = NtQuerySymbolicLinkObject(link_handle,
+                                    target_name,
+                                    (ULONG *)returned_byte_length);
+    close_handle(link_handle);
+    ASSERT(NT_SUCCESS(res));
+    return res;
+}
+
+/* General notes about sharing memory */
+/* section<PAGE_EXECUTE, SEC_IMAGE, app_file> gives us CoW in each
+ * process, and we can't share the relocation information
+ */
+/* section<PAGE_EXECUTE_READWRITE, SEC_IMAGE, original app_file>
+ * gives access denied since file is open only for execution.
+ * Though even proper privileges do not overwrite the original
+ * file - SEC_IMAGE is always copy on write.
+ */
+
+/* only using SEC_COMMIT either with page file, or with a
+ * {file<FILE_EXECUTE | FILE_READ_DATA | FILE_WRITE_DATA>,
+ * createsection<PAGE_EXECUTE_READWRITE, SEC_COMMIT, file>,
+ * map<PAGE_READWRITE>} allows writers to write to a true shared
+ * memory with readers.  
+
+ * If a particular reader needs private writes they can use a mapping
+ * created as above by writers {file<FILE_EXECUTE | FILE_READ_DATA>,
+ * opensection<SEC_COMMIT>, map<PAGE_WRITECOPY>} (can even track the
+ * pages that have transitioned from PAGE_WRITECOPY into
+ * PAGE_READWRITE to find which ones have been touched.
+ */
+
+/* complete wrapper around NtCreateSection but embeds InitializeObjectAttributes */
+NTSTATUS
+nt_create_section(OUT PHANDLE SectionHandle,
+                  IN ACCESS_MASK DesiredAccess,
+                  IN PLARGE_INTEGER SectionSize OPTIONAL,
+                  IN ULONG Protect,
+                  IN ULONG section_creation_attributes,
+                  IN HANDLE FileHandle,
+                  /* object name attributes */
+                  IN PCWSTR section_name OPTIONAL,
+                  IN ULONG object_name_attributes,
+                  IN HANDLE object_directory,
+                  IN PSECURITY_DESCRIPTOR dacl
+                  )
+{
+    NTSTATUS res;
+    UNICODE_STRING section_name_unicode;
+    OBJECT_ATTRIBUTES section_attributes;
+
+    if (section_name != NULL) {
+        res = wchar_to_unicode(&section_name_unicode, section_name);
+        ASSERT(NT_SUCCESS(res));
+        if (!NT_SUCCESS(res))
+            return res;
+    }
+    InitializeObjectAttributes(&section_attributes,
+                               ((section_name != NULL) ? 
+                                &section_name_unicode : NULL),
+                               OBJ_CASE_INSENSITIVE | object_name_attributes,
+                               object_directory,
+                               dacl);
+
+    res = NT_SYSCALL(CreateSection, 
+                     SectionHandle,
+                     DesiredAccess,
+                     &section_attributes,
+                     SectionSize,
+                     Protect,
+                     section_creation_attributes,
+                     FileHandle);
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("nt_create_section: failed res: %x\n", res);
+    }
+    return res;
+}
+
+/* complete wrapper around NtOpenSection */
+/* note that section_name is required and is case insensitive to
+ * support normal Windows case insensitivity of DLL lookup.  
+ * FIXME: unlikely may need to be changed for POSIX support
+ */
+NTSTATUS
+nt_open_section(OUT PHANDLE SectionHandle,
+                IN ACCESS_MASK DesiredAccess,
+                /* object name attributes */
+                IN PCWSTR section_name, /* required */
+                IN ULONG object_name_attributes,
+                IN HANDLE object_directory)
+{
+    NTSTATUS res;
+    UNICODE_STRING section_name_unicode;
+    OBJECT_ATTRIBUTES section_attributes;
+
+    ASSERT(section_name != NULL);
+    res = wchar_to_unicode(&section_name_unicode, section_name);
+    ASSERT(NT_SUCCESS(res));
+    if (!NT_SUCCESS(res))
+        return res;
+    InitializeObjectAttributes(&section_attributes,
+                               &section_name_unicode,
+                               OBJ_CASE_INSENSITIVE | object_name_attributes,
+                               object_directory,
+                               NULL);
+    res = NT_SYSCALL(OpenSection, SectionHandle,
+                                  DesiredAccess,
+                                  &section_attributes);
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("nt_open_section: failed res: %x\n", res);
+    }
+    return res;
+}
+
+bool
+are_mapped_files_the_same(app_pc addr1, app_pc addr2)
+{
+#if 0                           /* NYI: case 8502 */
+
+    /* FIXME: this doesn't exist on NT4 - make sure we handle
+     * gracefully not finding the target - needs a very explicit
+     * get_proc_address() here.  
+     */
+    GET_NTDLL(ZwAreMappedFilesTheSame, (
+                                        IN PVOID Address1,
+                                        IN PVOID Address2
+                                        ));
+#endif
+
+    ASSERT_NOT_TESTED();
+    ASSERT_NOT_IMPLEMENTED(false);
+    /* Testing: check return values for: addresses in different DLLs;
+     * addresses in same DLL; addresses in same DLL but coming from
+     * different mappings - the key one for us.
+     */
+
+    return false;
+}
+
+/* Mostly a wrapper around NtCreateFile, geared to opening existing
+ * module files.  See the DDK and SDK for complete argument documentation.
+ *
+ * callers have to close_handle() after use.
+ *
+ * file_path can be a path relative to root_directory_handle, or if
+ * root_directory_handle is NULL file_path has to be an NT absolute
+ * path (e.g. produced by the likes of RtlDosPathNameToNtPathName_U)
+ */
+
+/* desired_access_rights - a subset of FILE_EXECUTE | FILE_READ_DATA |
+ * FILE_WRITE_DATA.  Note: FILE_READ_DATA would be necessary for a
+ * later section mapping as PAGE_WRITECOPY, and FILE_WRITE_DATA may be
+ * needed for a PAGE_EXECUTE_WRITECOPY mapping if not SEC_IMAGE
+ */
+
+/* file_special_attributes typically FILE_ATTRIBUTE_NORMAL, possible
+ * other flags to use here FILE_ATTRIBUTE_TEMPORARY and maybe
+ * FILE_FLAG_DELETE_ON_CLOSE? */
+
+/* file_sharing_flags  */
+NTSTATUS
+nt_create_module_file(OUT HANDLE *file_handle,
+                      const wchar_t *file_path, 
+                      HANDLE root_directory_handle OPTIONAL,
+                      ACCESS_MASK desired_access_rights,
+                      uint file_special_attributes,
+                      uint file_sharing_flags,
+                      uint create_disposition, 
+                      size_t allocation_size)
+{
+    NTSTATUS res =
+        nt_create_file(file_handle, file_path, root_directory_handle,
+                       allocation_size,
+                       SYNCHRONIZE | desired_access_rights,
+                       file_special_attributes, file_sharing_flags,
+                       create_disposition,
+                       FILE_SYNCHRONOUS_IO_NONALERT | FILE_NON_DIRECTORY_FILE);
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("nt_open_module_file: can't open file, res: %x\n", res);
+    }
+    return res;
+}
+
+/* thin wrapper around ZwQueryInformationFile - 
+ * see DDK for documented information classes */
+NTSTATUS
+nt_query_file_info(IN HANDLE FileHandle,
+                   OUT PVOID FileInformation,
+                   IN ULONG FileInformationLength,
+                   IN FILE_INFORMATION_CLASS FileInformationClass)
+{
+    NTSTATUS res;
+    IO_STATUS_BLOCK iob = {0,0};
+
+    res = NtQueryInformationFile(FileHandle,
+                                 &iob,
+                                 FileInformation,
+                                 FileInformationLength,
+                                 FileInformationClass);
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("nt_query_file_info: can't open file, res: %x\n", res);
+    }
+
+    return res;
+}
+
+/* thin wrapper around ZwSetInformationFile - 
+ * see DDK for fully documented information classes
+ */
+NTSTATUS
+nt_set_file_info(IN HANDLE FileHandle,
+                 IN PVOID FileInformation,
+                 IN ULONG FileInformationLength,
+                 IN FILE_INFORMATION_CLASS FileInformationClass)
+{
+    NTSTATUS res;
+    IO_STATUS_BLOCK iob = {0,0};
+
+    res = NtSetInformationFile(FileHandle,
+                               &iob,
+                               FileInformation,
+                               FileInformationLength,
+                               FileInformationClass);
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("nt_set_file_info: can't open file, res: %x\n", res);
+    }
+
+    return res;
+}
+
+
+
+/* thin wrapper around ZwQueryVolumeInformationFile - 
+ * see Windows Driver Kit: Installable File System Drivers 
+ * for documented information classes,
+ * Note handle can be file, directory, device or volume.
+ */
+NTSTATUS
+nt_query_volume_info(IN HANDLE FileHandle,
+                     OUT PVOID FsInformation,
+                     IN ULONG FsInformationLength,
+                     IN FS_INFORMATION_CLASS FsInformationClass)
+{
+    NTSTATUS res;
+    IO_STATUS_BLOCK iob = {0,0};
+
+    GET_NTDLL(NtQueryVolumeInformationFile, (
+                                             IN HANDLE FileHandle,
+                                             OUT PIO_STATUS_BLOCK IoStatusBlock,
+                                             OUT PVOID FsInformation,
+                                             IN ULONG Length,
+                                             IN FS_INFORMATION_CLASS FsInformationClass
+                                             ));
+
+    res = NtQueryVolumeInformationFile(FileHandle,
+                                       &iob,
+                                       FsInformation,
+                                       FsInformationLength,
+                                       FsInformationClass);
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("nt_query_volume_info: can't open file, res: %x\n", res);
+    }
+    else {
+        ASSERT(iob.Information == FsInformationLength);
+    }
+    return res;
+}
+
+
+/* thin wrapper around ZwQuerySecurityObject - 
+ * Note handle can be any executive objective: including file, directory
+ */
+NTSTATUS
+nt_query_security_object(IN HANDLE Handle,
+                         IN SECURITY_INFORMATION RequestedInformation,
+                         OUT PSECURITY_DESCRIPTOR SecurityDescriptor,
+                         IN ULONG SecurityDescriptorLength,
+                         OUT PULONG ReturnLength
+                         )
+{
+    NTSTATUS res;
+
+    /* note that SecurityDescriptor returned is always PISECURITY_DESCRIPTOR_RELATIVE */
+    GET_NTDLL(NtQuerySecurityObject, (
+                                      IN HANDLE Handle,
+                                      IN SECURITY_INFORMATION RequestedInformation,
+                                      OUT PSECURITY_DESCRIPTOR SecurityDescriptor,
+                                      IN ULONG SecurityDescriptorLength,
+                                      OUT PULONG ReturnLength
+                                      ));
+    res = NtQuerySecurityObject(Handle,
+                                RequestedInformation,
+                                SecurityDescriptor,
+                                SecurityDescriptorLength,
+                                ReturnLength);
+    /* if SecurityDescriptorLength is too small ReturnLength is set to
+     * the number of bytes required for the available data 
+     */
+ 
+   /* for a file handle possibly insufficient permissions or unsupported file system */
+    if (!NT_SUCCESS(res)) {
+        NTPRINT("nt_query_security_object: can't open file, res: %x\n", res);
+    }
+    return res;
+}
+
+/* expect SID to have already been allocated statically
+ * (note we cannot use RtlAllocateAndInitializeSid to allocate memory!)
+ */
+void
+initialize_known_SID(PSID_IDENTIFIER_AUTHORITY IdentifierAuthority,
+                     ULONG SubAuthority0,
+                     SID *pSid)
+{
+    UCHAR SubAuthorityCount = 1;
+    ASSERT(pSid != NULL);
+
+    pSid->Revision = SID_REVISION;
+    pSid->SubAuthorityCount = SubAuthorityCount;
+    memcpy(&pSid->IdentifierAuthority,
+           IdentifierAuthority,
+           sizeof(SID_IDENTIFIER_AUTHORITY));
+
+    pSid->SubAuthority[0] = SubAuthority0;
+}
+
+
+#endif /* NOT_DYNAMORIO_CORE_PROPER */
diff --git a/core/win32/ntdll.h b/core/win32/ntdll.h
new file mode 100644
index 000000000..89419df3e
--- /dev/null
+++ b/core/win32/ntdll.h
@@ -0,0 +1,2031 @@
+/* **********************************************************
+ * Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+
+/*
+ * ntdll.h
+ * Routines for calling Windows system calls via the ntdll.dll wrappers.
+ * We return a boolean (but type int) instead of NTSTATUS, for most cases.
+ * 1 indicates success, 0 indicates an error.
+ * 
+ * New routines however should return the raw NTSTATUS and leave to
+ * the callers to report or act on some specific failure.  Should use
+ * NT_SUCCESS to verify success, luckily here 0 indicates success, so
+ * misuse as a bool will be caught easily.
+ */
+
+#ifndef _NTDLL_H_
+#define _NTDLL_H_ 1
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+
+#include <stddef.h> /* for offsetof */
+
+#include "globals_shared.h" /* for reg_t */
+
+/* Current method is to statically link with ntdll.lib obtained from the DDK */
+/* We cannot call get_module_handle at arbitrary points.
+ * Some syscalls are at certain points in win32 API internal state
+ * such that doing so causes problems.
+ */
+/* We used to dynamically get the proc address but that led to some untimely
+ *  loader lock acquisitions by kernel32.GetProcAddress  (Bug 411).  
+ *  This should serve as an example why using kernel32 functions is not safe.
+*/
+
+/* A simple wrapper to define ntdll entry points used inside our functions.
+   Since there is no official header file exporting these, 
+   we encapsulate signatures obtained from other sources.
+ */
+#define GET_NTDLL(NtFunction, signature)                             \
+  NTSYSAPI NTSTATUS NTAPI NtFunction signature
+
+/***************************************************************************
+ * structs and defines
+ * mostly from either Windows NT/2000 Native API Reference's ntdll.h
+ * or from the ddk's header files
+ */
+
+typedef unsigned int uint;
+typedef LONG NTSTATUS;
+/* make sure to cast to signed in case passed reg_t */
+#define NT_SUCCESS(Status) ((NTSTATUS)(Status) >= 0)
+#define STATUS_SUCCESS ((NTSTATUS)0x00000000L)
+
+#define NT_CURRENT_PROCESS ( (HANDLE) PTR_UINT_MINUS_1 )
+#define NT_CURRENT_THREAD  ( (HANDLE) (ptr_uint_t)-2 )
+
+typedef struct _UNICODE_STRING {
+    /* Length field is size in bytes not counting final 0 */
+    USHORT Length;
+    USHORT MaximumLength;
+    PWSTR  Buffer;
+} UNICODE_STRING;
+typedef UNICODE_STRING *PUNICODE_STRING;
+
+#ifndef X64
+typedef struct ALIGN_VAR(8) _UNICODE_STRING_64 {
+    /* Length field is size in bytes not counting final 0 */
+    USHORT Length;
+    USHORT MaximumLength;
+    int padding;
+    PWSTR  Buffer;
+    uint   Buffer_hi;
+} UNICODE_STRING_64;
+#endif
+
+typedef struct _STRING {
+  USHORT  Length;
+  USHORT  MaximumLength;
+  PCHAR  Buffer;
+} ANSI_STRING;
+typedef ANSI_STRING *PANSI_STRING;
+
+typedef struct _OBJECT_ATTRIBUTES {
+    ULONG Length;
+    HANDLE RootDirectory;
+    PUNICODE_STRING ObjectName;
+    ULONG Attributes;
+    PSECURITY_DESCRIPTOR SecurityDescriptor;
+    PVOID SecurityQualityOfService;  // Points to type SECURITY_QUALITY_OF_SERVICE
+} OBJECT_ATTRIBUTES;
+typedef OBJECT_ATTRIBUTES *POBJECT_ATTRIBUTES;
+
+#define InitializeObjectAttributes( p, n, a, r, s ) { \
+    (p)->Length = sizeof( OBJECT_ATTRIBUTES );          \
+    (p)->RootDirectory = r;                             \
+    (p)->Attributes = a;                                \
+    (p)->ObjectName = n;                                \
+    (p)->SecurityDescriptor = s;                        \
+    (p)->SecurityQualityOfService = NULL;               \
+    }
+
+/* from DDK2003SP1/3790.1830/inc/wnet/ntdef.h */
+#define OBJ_INHERIT             0x00000002L
+#define OBJ_PERMANENT           0x00000010L
+#define OBJ_EXCLUSIVE           0x00000020L
+#define OBJ_CASE_INSENSITIVE    0x00000040L
+#define OBJ_OPENIF              0x00000080L
+#define OBJ_OPENLINK            0x00000100L
+#define OBJ_KERNEL_HANDLE       0x00000200L /* N.B.: this is an invalid parameter on NT4! */
+#define OBJ_FORCE_ACCESS_CHECK  0x00000400L /* N.B.: introduced with Win2003 */
+
+typedef ULONG ACCESS_MASK;
+
+/* from DDK2003SP1/3790.1830/inc/ddk/wnet/ntddk.h */
+#define DIRECTORY_QUERY                 (0x0001)
+#define DIRECTORY_TRAVERSE              (0x0002)
+#define DIRECTORY_CREATE_OBJECT         (0x0004)
+#define DIRECTORY_CREATE_SUBDIRECTORY   (0x0008)
+#define DIRECTORY_ALL_ACCESS (STANDARD_RIGHTS_REQUIRED | 0xF)
+
+/* From http://undocumented.ntinternals.net/UserMode/Structures/RTL_USER_PROCESS_PARAMETERS.html */
+
+typedef struct _RTL_USER_PROCESS_PARAMETERS {
+    ULONG MaximumLength;
+    ULONG Length;
+    ULONG Flags;
+    ULONG DebugFlags;
+    PVOID ConsoleHandle;
+    ULONG ConsoleFlags;
+    HANDLE StdInputHandle;
+    HANDLE StdOutputHandle;
+    HANDLE StdErrorHandle;
+    UNICODE_STRING CurrentDirectoryPath;
+    HANDLE CurrentDirectoryHandle;
+    UNICODE_STRING DllPath;
+    UNICODE_STRING ImagePathName;
+    UNICODE_STRING CommandLine;
+    PVOID Environment;
+    ULONG StartingPositionLeft;
+    ULONG StartingPositionTop;
+    ULONG Width;
+    ULONG Height;
+    ULONG CharWidth;
+    ULONG CharHeight;
+    ULONG ConsoleTextAttributes;
+    ULONG WindowFlags;
+    ULONG ShowWindowFlags;
+    UNICODE_STRING WindowTitle;
+    UNICODE_STRING DesktopName;
+    UNICODE_STRING ShellInfo;
+    UNICODE_STRING RuntimeData;
+    // RTL_DRIVE_LETTER_CURDIR DLCurrentDirectory[0x20]
+} RTL_USER_PROCESS_PARAMETERS, *PRTL_USER_PROCESS_PARAMETERS;
+
+/* module information filled by the loader */
+typedef struct _PEB_LDR_DATA {  
+    ULONG Length;
+    BOOLEAN Initialized;
+    PVOID SsHandle;
+    LIST_ENTRY InLoadOrderModuleList;
+    LIST_ENTRY InMemoryOrderModuleList;
+    LIST_ENTRY InInitializationOrderModuleList;
+} PEB_LDR_DATA, *PPEB_LDR_DATA;
+
+/* Note that these lists are walked through corresponding LIST_ENTRY pointers
+ * i.e., for InInit*Order*, Flink points 16 bytes into the LDR_MODULE structure
+ */
+typedef struct _LDR_MODULE {
+    LIST_ENTRY InLoadOrderModuleList;
+    LIST_ENTRY InMemoryOrderModuleList;
+    LIST_ENTRY InInitializationOrderModuleList;
+    PVOID BaseAddress;
+    PVOID EntryPoint;
+    ULONG SizeOfImage;
+    UNICODE_STRING FullDllName;
+    UNICODE_STRING BaseDllName;
+    ULONG Flags;
+    SHORT LoadCount;
+    SHORT TlsIndex;
+    LIST_ENTRY HashTableEntry; /* CHECK: according to WINE this is HANDLE SectionHandle, ULONG CheckSum */
+    /* FIXME: on XP dt ntdll!_LDR_DATA_TABLE_ENTRY does indeed agree with WINE */
+    ULONG TimeDateStamp;
+} LDR_MODULE, *PLDR_MODULE;
+
+/* This macro is defined so that 32-bit dlls can be handled in 64-bit DR. 
+ * Not all IMAGE_OPTIONAL_HEADER fields are affected, only ImageBase,
+ * LoaderFlags, NumberOfRvaAndSizes, SizeOf{Stack,Heap}{Commit,Reserve},
+ * and DataDirectory, of which we use only ImageBase and DataDirectory.
+ * All other fields happen to have the same offsets and sizes in both
+ * IMAGE_OPTIONAL_HEADER32 and IMAGE_OPTIONAL_HEADER64.
+ */
+#ifdef X64
+/* Don't need to use module_is_32bit() here as that is heavyweight.  Also, as
+ * it is used directly in process_image() just when the module processing
+ * begins, we don't have to do all the checks here.
+ */
+# define OPT_HDR(nt_hdr_p, field) OPT_HDR_BASE(nt_hdr_p, field, )
+# define OPT_HDR_P(nt_hdr_p, field) OPT_HDR_BASE(nt_hdr_p, field, (app_pc)&)
+# define OPT_HDR_BASE(nt_hdr_p, field, amp) \
+    ((nt_hdr_p)->OptionalHeader.Magic == IMAGE_NT_OPTIONAL_HDR32_MAGIC ?      \
+        amp(((IMAGE_OPTIONAL_HEADER32 *)&((nt_hdr_p)->OptionalHeader))->field) : \
+        amp(((IMAGE_OPTIONAL_HEADER64 *)&((nt_hdr_p)->OptionalHeader))->field))
+#else
+# define OPT_HDR(nt_hdr_p, field) ((nt_hdr_p)->OptionalHeader.field)
+# define OPT_HDR_P(nt_hdr_p, field) (&((nt_hdr_p)->OptionalHeader.field))
+#endif
+
+/* For use by routines that walk the module lists. */
+enum {MAX_MODULE_LIST_INFINITE_LOOP_THRESHOLD = 2048};
+
+/* from winternl.h from wine (thus not official)
+ * these defines are (some of the) regular LDR_MODULE.Flags values
+ */
+#define LDR_IMAGE_IS_DLL                0x00000004
+#define LDR_LOAD_IN_PROGRESS            0x00001000
+#define LDR_UNLOAD_IN_PROGRESS          0x00002000
+#define LDR_NO_DLL_CALLS                0x00040000
+#define LDR_PROCESS_ATTACHED            0x00080000
+#define LDR_MODULE_REBASED              0x00200000
+
+typedef struct _PEBLOCKROUTINE *PPEBLOCKROUTINE;
+typedef struct _PEB_FREE_BLOCK *PPEB_FREE_BLOCK;
+typedef PVOID *PPVOID;
+
+typedef struct _RTL_BITMAP {
+    ULONG  SizeOfBitMap; /* Number of bits in the bitmap */
+    LPBYTE BitMapBuffer; /* Bitmap data, assumed sized to a DWORD boundary */
+} RTL_BITMAP, *PRTL_BITMAP;
+typedef const RTL_BITMAP *PCRTL_BITMAP;
+
+/* The layout here is from ntdll pdb on x64 xpsp2, though we
+ * changed some PVOID types to more specific types.
+ */
+typedef struct _PEB {                                     /* offset: 32bit / 64bit */
+    BOOLEAN                      InheritedAddressSpace;           /* 0x000 / 0x000 */
+    BOOLEAN                      ReadImageFileExecOptions;        /* 0x001 / 0x001 */
+    BOOLEAN                      BeingDebugged;                   /* 0x002 / 0x002 */
+#if 0
+    /* x64 xpsp2 lists this as a bitfield but compiler only accepts int bitfields: */
+    BOOLEAN                      ImageUsesLargePages:1;           /* 0x003 / 0x003 */
+    BOOLEAN                      SpareBits:7;                     /* 0x003 / 0x003 */
+#else
+    BOOLEAN                      ImageUsesLargePages;             /* 0x003 / 0x003 */
+#endif
+    HANDLE                       Mutant;                          /* 0x004 / 0x008 */
+    PVOID                        ImageBaseAddress;                /* 0x008 / 0x010 */
+    PPEB_LDR_DATA                LoaderData;                      /* 0x00c / 0x018 */
+    PRTL_USER_PROCESS_PARAMETERS ProcessParameters;               /* 0x010 / 0x020 */
+    PVOID                        SubSystemData;                   /* 0x014 / 0x028 */
+    PVOID                        ProcessHeap;                     /* 0x018 / 0x030 */
+    PRTL_CRITICAL_SECTION        FastPebLock;                     /* 0x01c / 0x038 */
+#if 0
+    /* x64 xpsp2 lists these fields as: */
+    PVOID                        AtlThunkSListPtr;                /* 0x020 / 0x040 */
+    PVOID                        SparePtr2;                       /* 0x024 / 0x048 */
+#else
+    /* xpsp2 and earlier */
+    PPEBLOCKROUTINE              FastPebLockRoutine;              /* 0x020 / 0x040 */
+    PPEBLOCKROUTINE              FastPebUnlockRoutine;            /* 0x024 / 0x048 */
+#endif
+    DWORD                        EnvironmentUpdateCount;          /* 0x028 / 0x050 */
+    PVOID                        KernelCallbackTable;             /* 0x02c / 0x058 */
+#if 0
+    /* x64 xpsp2 lists these fields as: */
+    DWORD                        SystemReserved[1];               /* 0x030 / 0x060 */
+    DWORD                        SpareUlong;                      /* 0x034 / 0x064 */
+#else
+    /* xpsp2 and earlier */
+    DWORD                        EvengLogSection;                 /* 0x030 / 0x060 */
+    DWORD                        EventLog;                        /* 0x034 / 0x064 */
+#endif
+    PPEB_FREE_BLOCK              FreeList;                        /* 0x038 / 0x068 */
+    DWORD                        TlsExpansionCounter;             /* 0x03c / 0x070 */
+    PRTL_BITMAP                  TlsBitmap;                       /* 0x040 / 0x078 */
+    DWORD                        TlsBitmapBits[2];                /* 0x044 / 0x080 */
+    PVOID                        ReadOnlySharedMemoryBase;        /* 0x04c / 0x088 */
+    PVOID                        ReadOnlySharedMemoryHeap;        /* 0x050 / 0x090 */
+    PPVOID                       ReadOnlyStaticServerData;        /* 0x054 / 0x098 */
+    PVOID                        AnsiCodePageData;                /* 0x058 / 0x0a0 */
+    PVOID                        OemCodePageData;                 /* 0x05c / 0x0a8 */
+    PVOID                        UnicodeCaseTableData;            /* 0x060 / 0x0b0 */
+    DWORD                        NumberOfProcessors;              /* 0x064 / 0x0b8 */
+    DWORD                        NtGlobalFlag;                    /* 0x068 / 0x0bc */
+    LARGE_INTEGER                CriticalSectionTimeout;          /* 0x070 / 0x0c0 */
+    ptr_uint_t                   HeapSegmentReserve;              /* 0x078 / 0x0c8 */
+    ptr_uint_t                   HeapSegmentCommit;               /* 0x07c / 0x0d0 */
+    ptr_uint_t                   HeapDeCommitTotalFreeThreshold;  /* 0x080 / 0x0d8 */
+    ptr_uint_t                   HeapDeCommitFreeBlockThreshold;  /* 0x084 / 0x0e0 */
+    DWORD                        NumberOfHeaps;                   /* 0x088 / 0x0e8 */
+    DWORD                        MaximumNumberOfHeaps;            /* 0x08c / 0x0ec */
+    PPVOID                       ProcessHeaps;                    /* 0x090 / 0x0f0 */
+    PVOID                        GdiSharedHandleTable;            /* 0x094 / 0x0f8 */
+    PVOID                        ProcessStarterHelper;            /* 0x098 / 0x100 */
+    DWORD                        GdiDCAttributeList;              /* 0x09c / 0x108 */
+    PRTL_CRITICAL_SECTION        LoaderLock;                      /* 0x0a0 / 0x110 */
+    DWORD                        OSMajorVersion;                  /* 0x0a4 / 0x118 */
+    DWORD                        OSMinorVersion;                  /* 0x0a8 / 0x11c */
+    WORD                         OSBuildNumber;                   /* 0x0ac / 0x120 */
+    WORD                         OSCSDVersion;                    /* 0x0ae / 0x122 */
+    DWORD                        OSPlatformId;                    /* 0x0b0 / 0x124 */
+    DWORD                        ImageSubsystem;                  /* 0x0b4 / 0x128 */
+    DWORD                        ImageSubsystemMajorVersion;      /* 0x0b8 / 0x12c */
+    DWORD                        ImageSubsystemMinorVersion;      /* 0x0bc / 0x130 */
+    ptr_uint_t                   ImageProcessAffinityMask;        /* 0x0c0 / 0x138 */
+#ifdef X64
+    DWORD                        GdiHandleBuffer[60];             /* 0x0c4 / 0x140 */
+#else
+    DWORD                        GdiHandleBuffer[34];             /* 0x0c4 / 0x140 */
+#endif
+    PVOID                        PostProcessInitRoutine;          /* 0x14c / 0x230 */
+    PVOID                        TlsExpansionBitmap;              /* 0x150 / 0x238 */
+    DWORD                        TlsExpansionBitmapBits[32];      /* 0x154 / 0x240 */
+    DWORD                        SessionId;                       /* 0x1d4 / 0x2c0 */
+    ULARGE_INTEGER               AppCompatFlags;                  /* 0x1d8 / 0x2c8 */
+    ULARGE_INTEGER               AppCompatFlagsUser;              /* 0x1e0 / 0x2d0 */
+    PVOID                        pShimData;                       /* 0x1e8 / 0x2d8 */
+    PVOID                        AppCompatInfo;                   /* 0x1ec / 0x2e0 */
+    UNICODE_STRING               CSDVersion;                      /* 0x1f0 / 0x2e8 */
+    PVOID                        ActivationContextData;           /* 0x1f8 / 0x2f8 */
+    PVOID                        ProcessAssemblyStorageMap;       /* 0x1fc / 0x300 */
+    PVOID                        SystemDefaultActivationContextData;/* 0x200 / 0x308 */
+    PVOID                        SystemAssemblyStorageMap;        /* 0x204 / 0x310 */
+    ptr_uint_t                   MinimumStackCommit;              /* 0x208 / 0x318 */
+    PPVOID                       FlsCallback;                     /* 0x20c / 0x320 */
+    LIST_ENTRY                   FlsListHead;                     /* 0x210 / 0x328 */
+    PVOID                        FlsBitmap;                       /* 0x218 / 0x338 */
+    DWORD                        FlsBitmapBits[4];                /* 0x21c / 0x340 */
+    DWORD                        FlsHighIndex;                    /* 0x22c / 0x350 */
+} PEB, *PPEB;
+
+#ifndef _W64
+# define _W64
+#endif
+#ifndef X64
+typedef _W64 long LONG_PTR, *PLONG_PTR;
+typedef _W64 unsigned long ULONG_PTR, *PULONG_PTR;
+typedef ULONG KAFFINITY;
+#endif
+typedef LONG KPRIORITY;
+
+typedef struct _KERNEL_USER_TIMES {
+    LARGE_INTEGER CreateTime;
+    LARGE_INTEGER ExitTime;
+    LARGE_INTEGER KernelTime;
+    LARGE_INTEGER UserTime;
+} KERNEL_USER_TIMES;
+
+/* from DDK2003SP1/3790.1830/inc/ddk/wnet/ntddk.h */
+typedef enum _PROCESSINFOCLASS {
+    ProcessBasicInformation,
+    ProcessQuotaLimits,
+    ProcessIoCounters,
+    ProcessVmCounters,
+    ProcessTimes,
+    ProcessBasePriority,
+    ProcessRaisePriority,
+    ProcessDebugPort,
+    ProcessExceptionPort,
+    ProcessAccessToken,
+    ProcessLdtInformation,
+    ProcessLdtSize,
+    ProcessDefaultHardErrorMode,
+    ProcessIoPortHandlers,          // Note: this is kernel mode only
+    ProcessPooledUsageAndLimits,
+    ProcessWorkingSetWatch,
+    ProcessUserModeIOPL,
+    ProcessEnableAlignmentFaultFixup,
+    ProcessPriorityClass,
+    ProcessWx86Information,
+    ProcessHandleCount,
+    ProcessAffinityMask,
+    ProcessPriorityBoost,
+    ProcessDeviceMap,
+    ProcessSessionInformation,
+    ProcessForegroundInformation,
+    ProcessWow64Information,
+    /* added after XP+ */
+    ProcessImageFileName,
+    ProcessLUIDDeviceMapsEnabled,
+    ProcessBreakOnTermination,
+    ProcessDebugObjectHandle,
+    ProcessDebugFlags,
+    ProcessHandleTracing,
+    ProcessIoPriority,
+    ProcessExecuteFlags,
+    ProcessResourceManagement,
+    ProcessCookie,
+    ProcessImageInformation,
+    MaxProcessInfoClass             // MaxProcessInfoClass should always be the last enum
+} PROCESSINFOCLASS;
+
+/* Process Information Structures */
+
+typedef struct _PROCESS_BASIC_INFORMATION {
+    NTSTATUS ExitStatus;
+    PPEB PebBaseAddress;
+    ULONG_PTR AffinityMask;
+    KPRIORITY BasePriority;
+    ULONG_PTR UniqueProcessId;
+    ULONG_PTR InheritedFromUniqueProcessId;
+} PROCESS_BASIC_INFORMATION;
+typedef PROCESS_BASIC_INFORMATION *PPROCESS_BASIC_INFORMATION;
+
+typedef struct _DESCRIPTOR_TABLE_ENTRY {
+    ULONG Selector;
+    LDT_ENTRY Descriptor;
+} DESCRIPTOR_TABLE_ENTRY, *PDESCRIPTOR_TABLE_ENTRY;
+
+/* format of data returned by QueryInformationProcess ProcessVmCounters */
+typedef struct _VM_COUNTERS {
+    SIZE_T PeakVirtualSize;
+    SIZE_T VirtualSize;
+    ULONG PageFaultCount;
+    SIZE_T PeakWorkingSetSize;
+    SIZE_T WorkingSetSize;
+    SIZE_T QuotaPeakPagedPoolUsage;
+    SIZE_T QuotaPagedPoolUsage;
+    SIZE_T QuotaPeakNonPagedPoolUsage;
+    SIZE_T QuotaNonPagedPoolUsage;
+    SIZE_T PagefileUsage;
+    SIZE_T PeakPagefileUsage;
+} VM_COUNTERS;
+
+#if defined(NOT_DYNAMORIO_CORE)
+# ifndef bool
+typedef int bool;
+# endif /* bool */
+typedef unsigned __int64 uint64;
+#endif /* NOT_DYNAMORIO_CORE */
+
+/* Case 7395: mmcgui build fails with redefinition of
+ * _JOBOBJECT_EXTENDED_LIMIT_INFORMATION and _IO_COUNTERS
+ */
+#if !defined(NOT_DYNAMORIO_CORE) && !defined(NOT_DYNAMORIO_CORE_PROPER)
+
+/* For NtQueryInformationProcess using ProcessQuotaLimits or
+ * ProcessPooledQuotaLimits or for NtSetInformationProcess using
+ * ProcessQuotaLimits */
+/* QUOTA_LIMITS defined in VC98/Include/WINNT.H - from WinNT+ */
+
+/* note for NtSetInformationProcess when setting can set either
+ * working set or the other values: only when both
+ * MinimumWorkingSetSize and MaximumWorkingSetSize are non-zero
+ * working set is adjusted, and the other values are ignored. 
+ * (Nebbett p.141)
+ * 
+ * Job and working set note from MSDN "Processes can still empty their
+ * working sets using the SetProcessWorkingSetSize function, even when
+ * JOB_OBJECT_LIMIT_WORKINGSET is used.  However, you cannot use
+ * SetProcessWorkingSetSize to change the minimum or maximum working
+ * set size."
+ */
+
+/* PageFaultHistory Information 
+ * NtQueryInformationProcess ProcessWorkingSetWatch 
+ */ 
+typedef struct _PROCESS_WS_WATCH_INFORMATION {
+    PVOID FaultingPc;
+    PVOID FaultingVa;
+} PROCESS_WS_WATCH_INFORMATION, *PPROCESS_WS_WATCH_INFORMATION;
+
+/* NtQueryInformationProcess ProcessPooledUsageAndLimits */
+typedef struct _POOLED_USAGE_AND_LIMITS {
+    SIZE_T PeakPagedPoolUsage;
+    SIZE_T PagedPoolUsage;
+    SIZE_T PagedPoolLimit;
+    SIZE_T PeakNonPagedPoolUsage;
+    SIZE_T NonPagedPoolUsage;
+    SIZE_T NonPagedPoolLimit;
+    SIZE_T PeakPagefileUsage;
+    SIZE_T PagefileUsage;
+    SIZE_T PagefileLimit;
+} POOLED_USAGE_AND_LIMITS;
+typedef POOLED_USAGE_AND_LIMITS *PPOOLED_USAGE_AND_LIMITS;
+
+/* Process Security Context Information
+ *  NtSetInformationProcess ProcessAccessToken
+ *  PROCESS_SET_ACCESS_TOKEN access needed to use
+ */
+typedef struct _PROCESS_ACCESS_TOKEN {
+    //
+    // Handle to Primary token to assign to the process.
+    // TOKEN_ASSIGN_PRIMARY access to this token is needed.
+    //
+    HANDLE Token;
+
+    //
+    // Handle to the initial thread of the process.
+    // A process's access token can only be changed if the process has
+    // no threads or one thread.  If the process has no threads, this
+    // field must be set to NULL.  Otherwise, it must contain a handle
+    // open to the process's only thread.  THREAD_QUERY_INFORMATION access
+    // is needed via this handle.
+    HANDLE Thread;
+} PROCESS_ACCESS_TOKEN, *PPROCESS_ACCESS_TOKEN;
+
+/* End of Process Information Structures */
+
+/* Basic Job Limit flags, specified in JOBOBJECT_BASIC_LIMIT_INFORMATION */
+#define JOB_OBJECT_LIMIT_WORKINGSET                 0x00000001
+#define JOB_OBJECT_LIMIT_PROCESS_TIME               0x00000002
+#define JOB_OBJECT_LIMIT_JOB_TIME                   0x00000004
+#define JOB_OBJECT_LIMIT_ACTIVE_PROCESS             0x00000008
+#define JOB_OBJECT_LIMIT_AFFINITY                   0x00000010
+#define JOB_OBJECT_LIMIT_PRIORITY_CLASS             0x00000020
+#define JOB_OBJECT_LIMIT_PRESERVE_JOB_TIME          0x00000040
+#define JOB_OBJECT_LIMIT_SCHEDULING_CLASS           0x00000080
+
+/* Extended Job Limit flags, specified in JOBOBJECT_EXTENDED_LIMIT_INFORMATION */
+#define JOB_OBJECT_LIMIT_PROCESS_MEMORY             0x00000100
+#define JOB_OBJECT_LIMIT_JOB_MEMORY                 0x00000200
+#define JOB_OBJECT_LIMIT_DIE_ON_UNHANDLED_EXCEPTION 0x00000400
+#define JOB_OBJECT_LIMIT_BREAKAWAY_OK               0x00000800
+#define JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK        0x00001000
+#define JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE          0x00002000
+
+/* End of Job Limits */
+#endif /* !NOT_DYNAMORIO_CORE && !NOT_DYNAMORIO_CORE_PROPER */
+
+typedef struct _CLIENT_ID {
+    /* These are numeric ids */
+    HANDLE UniqueProcess;
+    HANDLE UniqueThread;
+} CLIENT_ID;
+typedef CLIENT_ID *PCLIENT_ID;
+
+/* OS dependent SEH frame supported by ntdll.dll 
+   referred to in WINNT.H as _EXCEPTION_REGISTRATION_RECORD */
+typedef struct _EXCEPTION_REGISTRATION {
+     struct _EXCEPTION_REGISTRATION* prev;
+     PVOID                   handler;
+} EXCEPTION_REGISTRATION, *PEXCEPTION_REGISTRATION_RECORD;
+
+typedef struct _GDI_TEB_BATCH
+{
+    ULONG  Offset;
+    HANDLE HDC;
+    ULONG  Buffer[0x136];
+} GDI_TEB_BATCH;
+
+/* The layout here is from ntdll pdb on x64 xpsp2 */
+typedef struct _TEB {                               /* offset: 32bit / 64bit */
+    /* We lay out NT_TIB, which is declared in winnt.h */    
+    PEXCEPTION_REGISTRATION_RECORD ExceptionList;           /* 0x000 / 0x000 */
+    PVOID                     StackBase;                    /* 0x004 / 0x008 */
+    PVOID                     StackLimit;                   /* 0x008 / 0x010 */
+    PVOID                     SubSystemTib;                 /* 0x00c / 0x018 */
+    union {
+        PVOID                 FiberData;                    /* 0x010 / 0x020 */
+        DWORD                 Version;                      /* 0x010 / 0x020 */
+    };
+    PVOID                     ArbitraryUserPointer;         /* 0x014 / 0x028 */
+    struct _TEB*              Self;                         /* 0x018 / 0x030 */
+    PVOID                     EnvironmentPointer;           /* 0x01c / 0x038 */
+    CLIENT_ID                 ClientId;                     /* 0x020 / 0x040 */
+    PVOID                     ActiveRpcHandle;              /* 0x028 / 0x050 */
+    PVOID                     ThreadLocalStoragePointer;    /* 0x02c / 0x058 */
+    PEB*                      ProcessEnvironmentBlock;      /* 0x030 / 0x060 */
+    DWORD                     LastErrorValue;               /* 0x034 / 0x068 */
+    DWORD                     CountOfOwnedCriticalSections; /* 0x038 / 0x06c */
+    PVOID                     CsrClientThread;              /* 0x03c / 0x070 */
+    PVOID                     Win32ThreadInfo;              /* 0x040 / 0x078 */
+    DWORD                     User32Reserved[26];           /* 0x044 / 0x080 */
+    DWORD                     UserReserved[5];              /* 0x0ac / 0x0e8 */
+    PVOID                     WOW32Reserved;                /* 0x0c0 / 0x100 */
+    DWORD                     CurrentLocale;                /* 0x0c4 / 0x108 */
+    DWORD                     FpSoftwareStatusRegister;     /* 0x0c8 / 0x10c */
+    PVOID /* kernel32 data */ SystemReserved1[54];          /* 0x0cc / 0x110 */
+    LONG                      ExceptionCode;                /* 0x1a4 / 0x2c0 */
+    PVOID                     ActivationContextStackPointer;/* 0x1a8 / 0x2c8 */
+#ifdef X64
+    byte                      SpareBytes1[28];              /* 0x1ac / 0x2d0 */
+#else
+    byte                      SpareBytes1[40];              /* 0x1ac / 0x2d0 */
+#endif
+    GDI_TEB_BATCH             GdiTebBatch;                  /* 0x1d4 / 0x2f0 */
+    CLIENT_ID                 RealClientId;                 /* 0x6b4 / 0x7d8 */
+    PVOID                     GdiCachedProcessHandle;       /* 0x6bc / 0x7e8 */
+    DWORD                     GdiClientPID;                 /* 0x6c0 / 0x7f0 */
+    DWORD                     GdiClientTID;                 /* 0x6c4 / 0x7f4 */
+    PVOID                     GdiThreadLocalInfo;           /* 0x6c8 / 0x7f8 */
+    ptr_uint_t                Win32ClientInfo[62];          /* 0x6cc / 0x800 */
+    PVOID                     glDispatchTable[233];         /* 0x7c4 / 0x9f0 */
+    ptr_uint_t                glReserved1[29];              /* 0xb68 / 0x1138 */
+    PVOID                     glReserved2;                  /* 0xbdc / 0x1220 */
+    PVOID                     glSectionInfo;                /* 0xbe0 / 0x1228 */
+    PVOID                     glSection;                    /* 0xbe4 / 0x1230 */
+    PVOID                     glTable;                      /* 0xbe8 / 0x1238 */
+    PVOID                     glCurrentRC;                  /* 0xbec / 0x1240 */
+    PVOID                     glContext;                    /* 0xbf0 / 0x1248 */
+    DWORD                     LastStatusValue;              /* 0xbf4 / 0x1250 */
+    UNICODE_STRING            StaticUnicodeString;          /* 0xbf8 / 0x1258 */
+    WORD                      StaticUnicodeBuffer[261];     /* 0xc00 / 0x1268 */
+    PVOID                     DeallocationStack;            /* 0xe0c / 0x1478 */
+    PVOID                     TlsSlots[64];                 /* 0xe10 / 0x1480 */
+    LIST_ENTRY                TlsLinks;                     /* 0xf10 / 0x1680 */
+    PVOID                     Vdm;                          /* 0xf18 / 0x1690 */
+    PVOID                     ReservedForNtRpc;             /* 0xf1c / 0x1698 */
+    PVOID                     DbgSsReserved[2];             /* 0xf20 / 0x16a0 */
+    DWORD                     HardErrorMode;                /* 0xf28 / 0x16b0 */
+    PVOID                     Instrumentation[14];          /* 0xf2c / 0x16b8 */
+    PVOID                     SubProcessTag;                /* 0xf64 / 0x1728 */
+    PVOID                     EtwTraceData;                 /* 0xf68 / 0x1730 */
+    PVOID                     WinSockData;                  /* 0xf6c / 0x1738 */
+    DWORD                     GdiBatchCount;                /* 0xf70 / 0x1740 */
+    byte                      InDbgPrint;                   /* 0xf74 / 0x1744 */
+    byte                      FreeStackOnTermination;       /* 0xf75 / 0x1745 */
+    byte                      HasFiberData;                 /* 0xf76 / 0x1746 */
+    byte                      IdealProcessor;               /* 0xf77 / 0x1747 */
+    DWORD                     GuaranteedStackBytes;         /* 0xf78 / 0x1748 */
+    PVOID                     ReservedForPerf;              /* 0xf7c / 0x1750 */
+    PVOID                     ReservedForOle;               /* 0xf80 / 0x1758 */
+    DWORD                     WaitingOnLoaderLock;          /* 0xf84 / 0x1760 */
+    ptr_uint_t                SparePointer1;                /* 0xf88 / 0x1768 */
+    ptr_uint_t                SoftPatchPtr1;                /* 0xf8c / 0x1770 */
+    ptr_uint_t                SoftPatchPtr2;                /* 0xf90 / 0x1778 */
+    PPVOID                    TlsExpansionSlots;            /* 0xf94 / 0x1780 */
+#ifdef X64
+    PVOID                     DeallocationBStore;           /* ----- / 0x1788 */
+    PVOID                     BStoreLimit;                  /* ----- / 0x1790 */
+#endif
+    DWORD                     ImpersonationLocale;          /* 0xf98 / 0x1798 */
+    DWORD                     IsImpersonating;              /* 0xf9c / 0x179c */
+    PVOID                     NlsCache;                     /* 0xfa0 / 0x17a0 */
+    PVOID                     pShimData;                    /* 0xfa4 / 0x17a8 */
+    DWORD                     HeapVirtualAffinity;          /* 0xfa8 / 0x17b0 */
+    PVOID                     CurrentTransactionHandle;     /* 0xfac / 0x17b8 */
+    PVOID                     ActiveFrame;                  /* 0xfb0 / 0x17c0 */
+    PVOID                     FlsData;                      /* 0xfb4 / 0x17c8 */
+    byte                      SafeThunkCall;                /* 0xfb8 / 0x17d0 */
+    byte                      BooleanSpare[3];              /* 0xfb9 / 0x17d1 */
+} TEB;
+
+typedef struct _THREAD_BASIC_INFORMATION { // Information Class 0
+    NTSTATUS ExitStatus;
+    PNT_TIB TebBaseAddress;
+    CLIENT_ID ClientId;
+    KAFFINITY AffinityMask;
+    KPRIORITY Priority;
+    KPRIORITY BasePriority;
+} THREAD_BASIC_INFORMATION, *PTHREAD_BASIC_INFORMATION;
+
+typedef struct _USER_STACK {
+    PVOID FixedStackBase;
+    PVOID FixedStackLimit;
+    PVOID ExpandableStackBase;
+    PVOID ExpandableStackLimit;
+    PVOID ExpandableStackBottom;
+} USER_STACK, *PUSER_STACK;
+
+typedef enum _KEY_VALUE_INFORMATION_CLASS {
+    KeyValueBasicInformation,
+    KeyValueFullInformation,
+    KeyValuePartialInformation,
+    KeyValueFullInformationAlign64,
+    KeyValuePartialInformationAlign64
+} KEY_VALUE_INFORMATION_CLASS;
+
+typedef struct _KEY_VALUE_FULL_INFORMATION {
+    ULONG   TitleIndex;
+    ULONG   Type;
+    ULONG   DataOffset;
+    ULONG   DataLength;
+    ULONG   NameLength;
+    WCHAR   Name[1];            // Variable size
+//          Data[1]             // Variable size data not declared
+} KEY_VALUE_FULL_INFORMATION, *PKEY_VALUE_FULL_INFORMATION;
+
+typedef struct _KEY_VALUE_PARTIAL_INFORMATION {
+    ULONG   TitleIndex;
+    ULONG   Type;
+    ULONG   DataLength;
+    UCHAR   Data[1];            // Variable size
+} KEY_VALUE_PARTIAL_INFORMATION, *PKEY_VALUE_PARTIAL_INFORMATION;
+
+typedef enum _KEY_INFORMATION_CLASS {
+    KeyBasicInformation,
+    KeyNodeInformation,
+    KeyFullInformation,
+    KeyNameInformation
+} KEY_INFORMATION_CLASS;
+
+typedef struct _KEY_NAME_INFORMATION {
+    ULONG   NameLength;
+    WCHAR   Name[1];            // Variable size
+} KEY_NAME_INFORMATION, *PKEY_NAME_INFORMATION;
+
+typedef enum _SYSTEM_INFORMATION_CLASS {
+    SystemBasicInformation = 0,
+    SystemProcessorInformation = 1,
+    SystemPerformanceInformation = 2,
+    SystemTimeOfDayInformation = 3,
+    SystemProcessesAndThreadsInformation = 5,
+    SystemProcessorTimes = 8,
+    SystemGlobalFlag = 9,
+    SystemModuleInformation = 11,
+    SystemLockInformation = 12,
+    SystemHandleInformation = 16,
+    SystemObjectInformation = 17
+    /* many more, see Nebbett */
+} SYSTEM_INFORMATION_CLASS;
+
+typedef struct _SYSTEM_BASIC_INFORMATION {
+    ULONG   Unknown;
+    ULONG   MaximumIncrement;
+    ULONG   PhysicalPageSize;
+    ULONG   NumberOfPhysicalPages;
+    ULONG   LowestPhysicalPage;
+    ULONG   HighestPhysicalPage;
+    ULONG   AllocationGranularity;
+    PVOID   LowestUserAddress;
+    PVOID   HighestUserAddress;
+    ULONG_PTR ActiveProcessors;
+    UCHAR   NumberProcessors;
+#ifdef X64
+    ULONG   Unknown2; /* set to 0: probably just padding to 8-byte max field align */
+#endif
+} SYSTEM_BASIC_INFORMATION, *PSYSTEM_BASIC_INFORMATION;
+
+typedef struct _SYSTEM_PROCESSOR_INFORMATION {
+    USHORT  ProcessorArchitecture;
+    USHORT  ProcessorLevel;
+    USHORT  ProcessorRevision;
+    USHORT  Unknown;
+    ULONG   FeatureBits;
+} SYSTEM_PROCESSOR_INFORMATION, *PSYSTEM_PROCESSOR_INFORMATION;
+
+typedef struct _SYSTEM_PERFORMANCE_INFORMATION {
+    LARGE_INTEGER IdleTime;
+    LARGE_INTEGER ReadTransferCount;
+    LARGE_INTEGER WriteTransferCount;
+    LARGE_INTEGER OtherTransferCount;
+    ULONG   ReadOperationCount;
+    ULONG   WriteOperationCount;
+    ULONG   OtherOperationCount;
+    ULONG   AvailablePages;
+    ULONG   TotalCommittedPages;
+    ULONG   TotalCommitLimit;
+    ULONG   PeakCommitment;
+    ULONG   PageFaults;
+    ULONG   WriteCopyFaults;
+    ULONG   TranstitionFaults;
+    ULONG   Reserved1;
+    ULONG   DemandZeroFaults;
+    ULONG   PagesRead;
+    ULONG   PageReadIos;
+    ULONG   Reserved2[2];
+    ULONG   PageFilePagesWritten;
+    ULONG   PageFilePagesWriteIos;
+    ULONG   MappedFilePagesWritten;
+    ULONG   PagedPoolUsage;
+    ULONG   NonPagedPoolUsage;
+    ULONG   PagedPoolAllocs;
+    ULONG   PagedPoolFrees;
+    ULONG   NonPagedPoolAllocs;
+    ULONG   NonPagedPoolFrees;
+    ULONG   TotalFreeSystemPtes;
+    ULONG   SystemCodePage;
+    ULONG   TotalSystemDriverPages;
+    ULONG   TotalSystemCodePages;
+    ULONG   SmallNonPagedLookasideListAllocateHits;
+    ULONG   SmallPagedLookasieListAllocateHits;
+    ULONG   Reserved3;
+    ULONG   MmSystemCachePage;
+    ULONG   PagedPoolPage;
+    ULONG   SystemDriverPage;
+    ULONG   FastReadNoWait;
+    ULONG   FastReadWait;
+    ULONG   FastReadResourceMiss;
+    ULONG   FastReadNotPossible;
+    ULONG   FastMdlReadNoWait;
+    ULONG   FastMdlReadWait;
+    ULONG   FastMdlReadResourceMiss;
+    ULONG   FastMdlReadNotPossible;
+    ULONG   MapDataNoWait;
+    ULONG   MapDataWait;
+    ULONG   MapDataNoWaitMiss;
+    ULONG   MapDataWaitMiss;
+    ULONG   PinMappedDataCount;
+    ULONG   PinReadNoWait;
+    ULONG   PinReadWait;
+    ULONG   PinReadNoWaitMiss;
+    ULONG   PinReadWaitMiss;
+    ULONG   CopyReadNoWait;
+    ULONG   CopyReadWait;
+    ULONG   CopyReadNoWaitMiss;
+    ULONG   CopyReadWaitMiss;
+    ULONG   MdlReadNoWait;
+    ULONG   MdlReadWait;
+    ULONG   MdlReadNoWaitMiss;
+    ULONG   MdlReadWaitMiss;
+    ULONG   ReadAheadIos;
+    ULONG   LazyWriteIos;
+    ULONG   LazyWritePages;
+    ULONG   DataFlushes;
+    ULONG   DataPages;
+    ULONG   ContextSwitches;
+    ULONG   FirstLevelTbFills;
+    ULONG   SecondLevelTbFills;
+    ULONG   SystemCalls;
+} SYSTEM_PERFORMANCE_INFORMATION, *PSYSTEM_PERFORMANCE_INFORMATION;
+
+typedef struct _SYSTEM_TIME_OF_DAY_INFORMATION {
+    LARGE_INTEGER BootTime;
+    LARGE_INTEGER CurrentTime;
+    LARGE_INTEGER TimeZoneBias;
+    ULONG   CurrentTimeZoneId;
+} SYSTEM_TIME_OF_DAY_INFORMATION, *PSYSTEM_TIME_OF_DAY_INFORMATION;
+
+typedef struct _SYSTEM_PROCESSOR_TIMES {
+    LARGE_INTEGER IdleTime;
+    LARGE_INTEGER KernelTime;
+    LARGE_INTEGER UserTime;
+    LARGE_INTEGER DpcTime;
+    LARGE_INTEGER InterruptTime;
+    ULONG   InterruptCount;
+} SYSTEM_PROCESSOR_TIMES, *PSYSTEM_PROCESSOR_TIMES;
+
+typedef struct _IO_COUNTERSEX {
+    LARGE_INTEGER ReadOperationCount;
+    LARGE_INTEGER WriteOperationCount;
+    LARGE_INTEGER OtherOperationCount;
+    LARGE_INTEGER ReadTransferCount;
+    LARGE_INTEGER WriteTransferCount;
+    LARGE_INTEGER OtherTransferCount;
+} IO_COUNTERSEX, *PIO_COUNTERSEX;
+
+typedef enum _THREAD_STATE {
+    StateInitialized,
+    StateReady,
+    StateRunning,
+    StateStandby,
+    StateTerminated,
+    StateWait,
+    StateTransition,
+    StateUnknown
+} THREAD_STATE;
+
+typedef enum _KWAIT_REASON {
+    Executive,
+    FreePage,
+    PageIn,
+    PoolAllocation,
+    DelayExecution,
+    Suspended,
+    UserRequest,
+    WrExecutive,
+    WrFreePage,
+    WrPageIn,
+    WrPoolAllocation,
+    WrDelayExecution,
+    WrSuspended,
+    WrUserRequest,
+    WrEventPair,
+    WrQueue,
+    WrLpcReceive,
+    WrVirtualMemory,
+    WrPageOut,
+    WrRendevous,
+    WrSpare2,
+    WrSpare3,
+    WrSpare4,
+    WrSpare5,
+    WrSpare6,
+    WrKernel
+} KWAIT_REASON;
+
+typedef struct _SYSTEM_THREADS {
+    LARGE_INTEGER CreateTime;
+    LARGE_INTEGER UserTime;
+    LARGE_INTEGER KernelTime;
+    ULONG   WaitTime;
+    PVOID   StartAddress;
+    CLIENT_ID ClientId;
+    KPRIORITY Priority;
+    KPRIORITY BasePriority;
+    ULONG   ContextSwitchCount;
+    THREAD_STATE ThreadState;
+    KWAIT_REASON WaitReason;
+} SYSTEM_THREADS, *PSYSTEM_THREADS;
+
+typedef struct _SYSTEM_PROCESSES { 
+    ULONG NextEntryDelta;
+    ULONG ThreadCount;
+    ULONG Reserved1[6];
+    LARGE_INTEGER CreateTime;
+    LARGE_INTEGER UserTime;
+    LARGE_INTEGER KernelTime;
+    UNICODE_STRING ProcessName;
+    KPRIORITY BasePriority;
+    ULONG ProcessId;
+    ULONG InheritedFromProcessId;
+    ULONG HandleCount;
+    ULONG Reserved2[2];
+    VM_COUNTERS VmCounters;
+    IO_COUNTERSEX IoCounters;  /* Windows 2000+ only */
+    SYSTEM_THREADS Threads[1]; /* Variable size */
+} SYSTEM_PROCESSES, *PSYSTEM_PROCESSES;
+
+
+typedef struct _SYSTEM_GLOBAL_FLAG {
+    ULONG   GlobalFlag;
+} SYSTEM_GLOBAL_FLAG, *PSYSTEM_GLOBAL_FLAG;
+
+typedef struct _FILE_NETWORK_OPEN_INFORMATION {
+    LARGE_INTEGER CreationTime;
+    LARGE_INTEGER LastAccessTime;
+    LARGE_INTEGER LastWriteTime;
+    LARGE_INTEGER ChangeTime;
+    LARGE_INTEGER AllocationSize;
+    LARGE_INTEGER EndOfFile;
+    ULONG   FileAttributes;
+} FILE_NETWORK_OPEN_INFORMATION, *PFILE_NETWORK_OPEN_INFORMATION;
+
+typedef enum _MEMORY_INFORMATION_CLASS {
+    MemoryBasicInformation,
+    MemoryWorkingSetList,
+    MemorySectionName,
+    MemoryBasicVlmInformation
+} MEMORY_INFORMATION_CLASS;
+
+#define SYMBOLIC_LINK_QUERY (0x1)
+#define SYMBOLIC_LINK_ALL_ACCESS (STANDARD_RIGHTS_REQUIRED | SYMBOLIC_LINK_QUERY)
+
+/* Speculated arg 10 to NtCreateUserProcess.
+ * Note the similarities to CreateThreadEx arg 11 below.  Struct starts with size then
+ * after that looks kind of like an array of 16 byte (32 on 64-bit) elements corresponding
+ * to the IN and OUT informational ptrs.  Each array elment consists of a ?flags? then
+ * the sizeof of the IN/OUT ptr buffer then the ptr itself then 0.
+ */
+
+typedef enum { /* NOTE - these are speculative */
+    THREAD_INFO_ELEMENT_BUFFER_IS_INOUT = 0x00000, /* buffer is ??IN/OUT?? */
+    THREAD_INFO_ELEMENT_BUFFER_IS_OUT   = 0x10000, /* buffer is IN (?) */
+    THREAD_INFO_ELEMENT_BUFFER_IS_IN    = 0x20000, /* buffer is OUT (?) */
+} thread_info_elm_buf_access_t;
+
+typedef enum { /* NOTE - these are speculative */
+    THREAD_INFO_ELEMENT_CLIENT_ID       = 0x3, /* buffer is CLIENT_ID - OUT */
+    THREAD_INFO_ELEMENT_TEB             = 0x4, /* buffer is TEB * - OUT */
+    THREAD_INFO_ELEMENT_NT_PATH_TO_EXE  = 0x5, /* buffer is wchar * path to exe
+                                                * [ i.e. L"\??\c:\foo.exe" ] - IN */
+    THREAD_INFO_ELEMENT_EXE_STUFF       = 0x6, /* buffer is exe_stuff_t (see above) 
+                                                * - INOUT */
+    THREAD_INFO_ELEMENT_UNKNOWN_1       = 0x9, /* Unknown - ptr_uint_t sized
+                                                * [ observed 1 ] - IN */
+} thread_info_elm_buf_type_t;
+
+typedef struct _thread_info_element_t { /* NOTE - this is speculative */
+    ptr_uint_t flags;   /* thread_info_elm_buf_access_t | thread_info_elm_buf_type_t */
+    size_t buffer_size; /* sizeof of buffer, in bytes */
+    void *buffer;       /* flags determine disposition, could be IN or OUT or both */
+    ptr_uint_t unkown;  /* [ observed always 0 ] */
+} thread_info_elm_t;
+
+typedef struct _exe_stuff_t { /* NOTE - this is speculative */
+    OUT void *exe_entrypoint_addr; /* Entry point to the exe being started. */
+    // ratio of uint32 to ptr_uint_t assumes no larger changes between 32 and 64-bit
+    ptr_uint_t unknown1[3]; // possibly intermixed with uint32s below IN? OUT?
+    uint32 unknown2[8];     // possible intermixed with ptr_uint_ts above IN? OUT?
+} exe_stuff_t;
+
+typedef struct _create_proc_thread_info_t { /* NOTE - this is speculative */
+    size_t struct_size; /* observed 0x34 or 0x44 (0x68 on 64-bit) = sizeof(this struct) */
+    /* Observed - first thread_info_elm_t always 
+     * flags = 0x20005
+     * buffer_size = varies (sizeof buffer string in bytes)
+     * buffer = wchar * : nt path to executable i.e. "\??\c:\foo.exe" - IN */
+    thread_info_elm_t nt_path_to_exe;
+    /* Observed - second thread_info_elm_t always
+     * flags = 0x10003
+     * buffer_size = sizeof(CLIENT_ID)
+     * buffer = PCLIENT_ID : OUT */
+    thread_info_elm_t client_id;
+    /* Observed - third thread_info_elm_t always
+     * flags = 0x6
+     * buffer_size = 0x30 (or 0x40 on 64-bit) == sizeof(exe_stuff_t)
+     * buffer = exe_stuff_t * : IN/OUT */
+    thread_info_elm_t exe_stuff;
+    /* While the first three thread_info_elm_t have been present in every call I've seen
+     * (and attempts to remove or re-arrange them caused the system call to fail,
+     * assuming I managed to do it right), there's more variation in the later fields
+     * (sometimes present, sometimes not) - most commonly there'll be nothing or just the
+     * TEB * info field (flags = 0x10003) which I've seen here a lot on 32bit. */
+#if 0 /* 0 sized array is non-standard extension */
+    thread_info_elm_t info[];
+#endif
+}  create_proc_thread_info_t;
+
+/* Speculated arg 11 to NtCreateThreadEx.  See the similar arg 10 of
+ * NtCreateUserProcess above. */
+typedef struct _create_thread_info_t { /* NOTE - this is speculative */
+    size_t struct_size; /* observed 0x24 (0x48 on 64-bit) == sizeof(this struct) */
+    /* Note kernel32!CreateThread hardcodes all the values in this structure and
+     * I've never seen any variation elsewhere. Trying to swap the order caused the
+     * system call to fail when I tried it (assuming I did it right). */
+    /* Observed - always
+     * flags = 0x10003
+     * buffer_size = sizeof(CLIENT_ID)
+     * buffer = PCLIENT_ID : OUT */
+    thread_info_elm_t client_id;
+    /* Observed - always
+     * flags = 0x10004
+     * buffer_size = sizeof(CLIENT_ID)
+     * buffer = TEB ** : OUT */
+    thread_info_elm_t teb;
+} create_thread_info_t;
+
+/***************************************************************************
+ * convenience enums
+ */
+typedef enum {MEMORY_RESERVE_ONLY = MEM_RESERVE, 
+              MEMORY_COMMIT = MEM_RESERVE|MEM_COMMIT
+} memory_commit_status_t;
+
+/***************************************************************************
+ * function declarations
+ */
+
+void
+ntdll_init(void);
+
+void
+ntdll_exit(void);
+
+int
+close_handle(HANDLE h);
+
+/* from winddk.h used by cygwin */
+#define DUPLICATE_SAME_ATTRIBUTES   0x00000004
+NTSTATUS
+duplicate_handle(HANDLE source_process, HANDLE source, HANDLE target_process, 
+                 HANDLE *target, ACCESS_MASK access, uint attributes, 
+                 uint options);
+
+ACCESS_MASK
+nt_get_handle_access_rights(HANDLE handle);
+
+typedef enum {THREAD_EXITED, THREAD_NOT_EXITED, THREAD_EXIT_ERROR} thread_exited_status_t;
+thread_exited_status_t
+is_thread_exited(HANDLE hthread);
+
+thread_id_t
+thread_id_from_handle(HANDLE h);
+
+process_id_t
+process_id_from_handle(HANDLE h);
+
+process_id_t
+process_id_from_thread_handle(HANDLE h);
+
+PEB *
+get_peb(HANDLE h);
+
+PEB *
+get_own_peb(void);
+
+TEB *
+get_teb(HANDLE h);
+
+TEB *
+get_own_teb(void);
+
+void *
+get_ntdll_base(void);
+
+/* replacement for API call VirtualQuery */
+size_t
+query_virtual_memory(const byte *pc, MEMORY_BASIC_INFORMATION *mbi, size_t mbilen);
+
+/* for allocating in another process, 
+ * keep in mind base is now IN/OUT value, a NULL value means no preference
+ * will bump size up to PAGE_SIZE multiple */
+NTSTATUS
+nt_remote_allocate_virtual_memory(HANDLE process, void **base, size_t size,
+                                  uint prot, memory_commit_status_t commit);
+
+/* will bump size up to PAGE_SIZE multiple 
+ * keep in mind base is now IN/OUT value, a NULL value means no preference
+ */
+NTSTATUS
+nt_allocate_virtual_memory(void **base, size_t size, uint prot,
+                           memory_commit_status_t commit);
+
+NTSTATUS
+nt_commit_virtual_memory(void *base, size_t size, uint prot);
+NTSTATUS
+nt_decommit_virtual_memory(void *base, size_t size);
+
+NTSTATUS
+nt_free_virtual_memory(void *base);
+
+int
+protect_virtual_memory(void *base, size_t size, uint prot, uint *old_prot);
+
+int
+nt_remote_protect_virtual_memory(HANDLE process,
+                       void *base, size_t size, uint prot, uint *old_prot);
+
+int
+nt_read_virtual_memory(HANDLE process, const void *base, void *buffer, 
+                       size_t buffer_length, size_t *bytes_read);
+
+int
+nt_write_virtual_memory(HANDLE process, void *base, const void *buffer, 
+                        size_t buffer_length, size_t *bytes_written);
+
+void
+nt_continue(CONTEXT *cxt);
+
+NTSTATUS
+nt_get_context(HANDLE hthread, CONTEXT *cxt);
+
+NTSTATUS
+nt_set_context(HANDLE hthread, CONTEXT *cxt);
+
+int
+nt_thread_suspend(HANDLE hthread, int *previous_suspend_count);
+
+int
+nt_thread_resume(HANDLE hthread, int *previous_suspend_count);
+
+int
+nt_terminate_thread(HANDLE hthread, NTSTATUS exit_code);
+
+int 
+nt_terminate_process(HANDLE hprocess, NTSTATUS exit_code);
+
+NTSTATUS
+nt_terminate_process_for_app(HANDLE hprocess, NTSTATUS exit_code);
+
+int
+am_I_sole_thread(HANDLE hthread, int *amI /*OUT*/);
+
+/* checks current thread, and turns errors into false */
+int
+check_sole_thread(void);
+
+/* creates a waitable timer */
+HANDLE
+nt_create_and_set_timer(PLARGE_INTEGER due_time, LONG period);
+
+/* thread sleeps for given relative or absolute time */
+int
+nt_sleep(PLARGE_INTEGER due_time);
+
+void
+nt_yield(void);
+
+int
+nt_raise_exception(EXCEPTION_RECORD* pexcrec, CONTEXT* pcontext);
+
+int
+nt_messagebox(wchar_t *msg, wchar_t *title);
+
+int
+tls_alloc(int synch, uint *teb_offs /* OUT */);
+
+/* Allocates num tls slots aligned with alignment align */
+int
+tls_calloc(int synch, uint *teb_offs /* OUT */, int num, uint alignment);
+
+int
+tls_free(int synch, uint teb_offs);
+
+int
+tls_cfree(int synch, uint teb_offs, int num);
+
+int
+get_process_mem_stats(HANDLE h, VM_COUNTERS *info);
+
+NTSTATUS
+get_process_mem_quota(HANDLE h, QUOTA_LIMITS *qlimits);
+
+NTSTATUS
+get_process_handle_count(HANDLE ph, ULONG *handle_count);
+
+int
+get_process_load(HANDLE h);
+
+int
+is_wow64_process(HANDLE h);
+
+void *
+get_section_address(HANDLE h);
+
+/* Section attributes as described in Nebbett, p. 105.
+ * Some are documented in the SDK CreateFileMapping */
+/* from wnet/winnt.h, new in Win2003 */
+#define SEC_LARGE_PAGES           0x80000000     
+/* SEC_VLM in VC98/winnt.h but in DDK, apparently not implemented on x86 */
+#ifndef SEC_VLM /* not defined in VC8 headers */
+# define SEC_VLM                  0x02000000
+#endif
+/* Similarly these may have existed in earlier windows versions as well */
+#define SEC_BASED_UNSUPPORTED     0x00200000
+#define SEC_NO_CHANGE_UNSUPPORTED 0x00400000
+
+int
+get_section_attributes(HANDLE h, uint *section_attributes /* OUT */,
+                       LARGE_INTEGER* section_size /* OUT OPTIONAL */);
+
+
+HANDLE
+reg_create_key(HANDLE parent, PCWSTR keyname, ACCESS_MASK rights);
+
+HANDLE
+reg_open_key(PCWSTR keyname, ACCESS_MASK rights);
+
+int
+reg_close_key(HANDLE hkey);
+
+int
+reg_delete_key(HANDLE hkey);
+
+typedef enum _reg_query_value_result {
+    REG_QUERY_FAILURE,
+    REG_QUERY_BUFFER_TOO_SMALL,
+    REG_QUERY_SUCCESS
+} reg_query_value_result_t;
+
+reg_query_value_result_t
+reg_query_value(PCWSTR hkey, PCWSTR subkey, 
+                KEY_VALUE_INFORMATION_CLASS key_class, PVOID info, ULONG info_size,
+                ACCESS_MASK rights);
+
+int
+reg_set_key_value(HANDLE hkey, PCWSTR subkey, PCWSTR val);
+int
+reg_set_dword_key_value(HANDLE hkey, PCWSTR subkey, DWORD val);
+
+int
+reg_flush_key(HANDLE hkey);
+
+int
+reg_enum_key(PCWSTR keyname, ULONG index, KEY_INFORMATION_CLASS info_class,
+             PVOID key_info, ULONG key_info_size);
+
+int
+reg_enum_value(PCWSTR keyname, ULONG index, KEY_VALUE_INFORMATION_CLASS key_class,
+               PVOID key_info, ULONG key_info_length);
+
+
+#define LengthRequiredSID(subauthorities)                                       \
+      (sizeof(SID) - (ANYSIZE_ARRAY * sizeof(DWORD)) /* SID.SubAuthority */     \
+       + ((subauthorities) * sizeof(DWORD)))
+
+#ifndef SECURITY_MAX_SID_SIZE
+/* see DDK2003SP1/3790.1830/inc/wnet/winnt.h */
+# define SECURITY_MAX_SID_SIZE \
+      (LengthRequiredSID(SID_MAX_SUB_AUTHORITIES))
+#endif
+
+/* Note: the above static SID size may change in future versions, so
+ * we need to keep track of winnt.h
+ */
+#define SECURITY_MAX_TOKEN_SIZE (SECURITY_MAX_SID_SIZE + sizeof(SID_AND_ATTRIBUTES))
+
+NTSTATUS
+get_primary_user_token(PTOKEN_USER ptoken, USHORT token_buffer_length);
+NTSTATUS
+get_current_user_token(PTOKEN_USER ptoken, USHORT token_buffer_length);
+NTSTATUS
+get_primary_owner_token(PTOKEN_OWNER powner, USHORT owner_buffer_length);
+
+NTSTATUS
+get_current_user_SID(PWSTR sid_string, USHORT buffer_length);
+
+bool
+get_owner_sd(PISECURITY_DESCRIPTOR SecurityDescriptor,
+             PSID* Owner);
+void
+initialize_security_descriptor(PISECURITY_DESCRIPTOR SecurityDescriptor);
+/* use only on security descriptors created with initialize_security_descriptor() */
+bool
+set_owner_sd(PISECURITY_DESCRIPTOR SecurityDescriptor,
+             PSID Owner);
+
+bool
+equal_sid(IN PSID Sid1, IN PSID Sid2);
+void
+initialize_known_SID(PSID_IDENTIFIER_AUTHORITY IdentifierAuthority,
+                     ULONG SubAuthority0,
+                     SID *pSid);
+
+NTSTATUS
+query_seg_descriptor(HANDLE hthread, DESCRIPTOR_TABLE_ENTRY *entry);
+
+NTSTATUS
+query_win32_start_addr(HANDLE hthread, PVOID start_addr);
+
+NTSTATUS 
+query_system_info(SYSTEM_INFORMATION_CLASS info_class, int info_size, PVOID info);
+
+int
+query_full_attributes_file(PCWSTR filename,                           
+                           PFILE_NETWORK_OPEN_INFORMATION info);
+
+/* access mask flags, from ntddk.h rest are in winnt.h so no need to replicate
+ * them here */
+#define FILE_ANY_ACCESS                 0
+#define FILE_SPECIAL_ACCESS    (FILE_ANY_ACCESS)
+#define FILE_READ_ACCESS          ( 0x0001 )    // file & pipe
+#define FILE_WRITE_ACCESS         ( 0x0002 )    // file & pipe
+
+/* share flags, from ntddk.h, rest are in winnt.h */
+#define FILE_SHARE_VALID_FLAGS          0x00000007
+
+/* attribute flags, from ntddk.h, rest are in winnt.h */
+#define FILE_ATTRIBUTE_VALID_FLAGS          0x00007fb7
+#define FILE_ATTRIBUTE_VALID_SET_FLAGS      0x000031a7
+
+/* flags for NtCreateFile and NtOpenFile */
+/* Create/Open options from ntddk.h */
+#define FILE_DIRECTORY_FILE                     0x00000001
+#define FILE_WRITE_THROUGH                      0x00000002
+#define FILE_SEQUENTIAL_ONLY                    0x00000004
+#define FILE_NO_INTERMEDIATE_BUFFERING          0x00000008
+
+#define FILE_SYNCHRONOUS_IO_ALERT               0x00000010
+#define FILE_SYNCHRONOUS_IO_NONALERT            0x00000020
+#define FILE_NON_DIRECTORY_FILE                 0x00000040
+#define FILE_CREATE_TREE_CONNECTION             0x00000080
+
+#define FILE_COMPLETE_IF_OPLOCKED               0x00000100
+#define FILE_NO_EA_KNOWLEDGE                    0x00000200
+#define FILE_OPEN_FOR_RECOVERY                  0x00000400
+#define FILE_RANDOM_ACCESS                      0x00000800
+
+#define FILE_DELETE_ON_CLOSE                    0x00001000
+#define FILE_OPEN_BY_FILE_ID                    0x00002000
+#define FILE_OPEN_FOR_BACKUP_INTENT             0x00004000
+#define FILE_NO_COMPRESSION                     0x00008000
+
+#define FILE_RESERVE_OPFILTER                   0x00100000
+#define FILE_OPEN_REPARSE_POINT                 0x00200000
+#define FILE_OPEN_NO_RECALL                     0x00400000
+#define FILE_OPEN_FOR_FREE_SPACE_QUERY          0x00800000
+
+#define FILE_COPY_STRUCTURED_STORAGE            0x00000041
+#define FILE_STRUCTURED_STORAGE                 0x00000441
+
+#define FILE_VALID_OPTION_FLAGS                 0x00ffffff
+#define FILE_VALID_PIPE_OPTION_FLAGS            0x00000032
+#define FILE_VALID_MAILSLOT_OPTION_FLAGS        0x00000032
+#define FILE_VALID_SET_FLAGS                    0x00000036
+
+/* iob.Information for NtCreateFile or NtOpenFile */
+#define FILE_SUPERSEDED     0x00000000
+#define FILE_OPENED         0x00000001
+#define FILE_CREATED        0x00000002
+#define FILE_OVERWRITTEN    0x00000003
+#define FILE_EXISTS         0x00000004
+#define FILE_DOES_NOT_EXIST 0x00000005
+
+/* CreateDisposition, from the ntddk.h */
+#define FILE_SUPERSEDE                  0x00000000
+#define FILE_OPEN                       0x00000001
+#define FILE_CREATE                     0x00000002
+#define FILE_OPEN_IF                    0x00000003
+#define FILE_OVERWRITE                  0x00000004
+#define FILE_OVERWRITE_IF               0x00000005
+#define FILE_MAXIMUM_DISPOSITION        0x00000005
+
+/* note this is our own private flag, to explicitly set file owner */
+#define FILE_DISPOSITION_SET_OWNER      0x10000000
+
+/* from ntddk.h, special ByteOffset Parameters */
+#define FILE_WRITE_TO_END_OF_FILE       0xffffffff
+#define FILE_USE_FILE_POINTER_POSITION  0xfffffffe
+
+/* From NTSTATUS.H -- this shouldn't change, but you never know... */
+/* DDK2003SP1/3790.1830/inc/wnet/ntstatus.h */
+#define STATUS_INFO_LENGTH_MISMATCH      ((NTSTATUS)0xC0000004L)
+
+/* {Conflicting Address Range}  The specified address range conflicts with the address space. */
+#define STATUS_CONFLICTING_ADDRESSES     ((NTSTATUS)0xC0000018L)
+
+/* "The address handle given to the transport was invalid." */
+#define STATUS_INVALID_ADDRESS           ((NTSTATUS)0xC0000141L)
+
+/* "The data was too large to fit into the specified buffer." */
+#define STATUS_BUFFER_OVERFLOW           ((NTSTATUS)0x80000005L)
+
+/*  {Bad File} The attributes of the specified mapping file for a
+*  section of memory cannot be read. */
+#define STATUS_INVALID_FILE_FOR_SECTION  ((NTSTATUS)0xC0000020L)
+
+/*  {Access Denied} A process has requested access to an object, but
+ *  has not been granted those access rights.*/
+#define STATUS_ACCESS_DENIED             ((NTSTATUS)0xC0000022L)
+
+/* "The buffer is too small to contain the entry. No information has been
+ * written to the buffer." */
+#define STATUS_BUFFER_TOO_SMALL          ((NTSTATUS)0xC0000023L)
+
+/*  Object Name not found. */
+#define STATUS_OBJECT_NAME_NOT_FOUND     ((NTSTATUS)0xC0000034L)
+
+/* Error: Object Name already exists */
+#define STATUS_OBJECT_NAME_COLLISION     ((NTSTATUS)0xC0000035L)
+
+/*  A file cannot be opened because the share access flags are incompatible. */
+#define STATUS_SHARING_VIOLATION         ((NTSTATUS)0xC0000043L)
+
+/*  A non close operation has been requested of a file object with a delete pending. */
+#define STATUS_DELETE_PENDING            ((NTSTATUS)0xC0000056L)
+
+/*  The file that was specified as a target is a directory and the
+ * caller specified that it could be anything but a directory.
+ */
+#define STATUS_FILE_IS_A_DIRECTORY       ((NTSTATUS)0xC00000BAL)
+
+/* Warning: An attempt was made to create an object and the object name already existed. */
+#define STATUS_OBJECT_NAME_EXISTS        ((NTSTATUS)0x40000000L)
+
+/*  An invalid parameter was passed to a service or function as the fourth argument. */
+#define STATUS_INVALID_PARAMETER_4       ((NTSTATUS)0xC00000F2L)
+
+/* Warning: Image Relocated
+ *  "An image file could not be mapped at the address specified in the image file. "
+ *  "Local fixups must be performed on this image."
+ */
+#define STATUS_IMAGE_NOT_AT_BASE         ((NTSTATUS)0x40000003L)
+
+#if !defined(STATUS_NO_MEMORY)
+/* {Not Enough Quota} Not enough virtual memory or paging file quota
+ *  is available to complete the specified operation. */
+#  define STATUS_NO_MEMORY                 ((NTSTATUS)0xC0000017L)   /* winnt */
+/* only seen on virtual memory reservation, maybe on NT it served multiple purposes */
+#endif
+
+/*  Page file quota was exceeded. */
+#define STATUS_PAGEFILE_QUOTA_EXCEEDED   ((NTSTATUS)0xC000012CL)
+/* not seen */
+
+/* {Out of Virtual Memory} Your system is low on virtual memory. To
+ *  ensure that Windows runs properly, increase the size of your
+ *  virtual memory paging file. For more information, see Help.
+ */
+#define STATUS_COMMITMENT_LIMIT          ((NTSTATUS)0xC000012DL)
+/* Seen both when reaching the Commit limit (memory + page file size), 
+ * as well as when a process in a Job reaches its ProcessMemoryLimit
+ */
+
+/* {Virtual Memory Minimum Too Low} Your system is low on virtual
+ *  memory. Windows is increasing the size of your virtual memory
+ *  paging file.  During this process, memory requests for some
+ *  applications may be denied. For more information, see Help.
+ */
+#define STATUS_COMMITMENT_MINIMUM        ((NTSTATUS)0xC00002C8L)
+
+/* This is the exception code microsoft uses for throw exception */
+#define EXCEPTION_THROWN 0xe06d7363
+
+/* Status to use with NtIsProcessInJob() on Win2003+ */
+/*  The specified process is not part of a job */
+#define STATUS_PROCESS_NOT_IN_JOB        ((NTSTATUS)0x00000123L)
+/*  The specified process is part of a job. */
+#define STATUS_PROCESS_IN_JOB            ((NTSTATUS)0x00000124L)
+
+/* A specified privilege does not exist. */
+#define STATUS_NO_SUCH_PRIVILEGE         ((NTSTATUS)0xC0000060L)
+/*  A required privilege is not held by the client. */
+#define STATUS_PRIVILEGE_NOT_HELD        ((NTSTATUS)0xC0000061L)
+
+/* Case 10579: pop but do not transfer control for NtCallbackReturn */
+#define STATUS_CALLBACK_POP_STACK        ((NTSTATUS)0xC0000423L)
+
+/* needed for PR 233191 */
+#define STATUS_INVALID_INFO_CLASS        ((NTSTATUS)0xC0000003L)
+
+/* This is in VS2005 winnt.h but not in SDK winnt.h */
+#ifndef IMAGE_SIZEOF_BASE_RELOCATION
+# define IMAGE_SIZEOF_BASE_RELOCATION         8
+#endif
+
+NTSTATUS
+nt_create_file(OUT HANDLE *file_handle, 
+               const wchar_t *filename, HANDLE dir_handle OPTIONAL,
+               size_t alloc_size, ACCESS_MASK rights, uint attributes, uint sharing,
+               uint create_disposition, uint create_options);
+
+/* is_dir and synch are really bools */
+/* note synch must be true if you are ever going to use read_file or write_file
+ * on the returned handle */
+HANDLE
+create_file(PCWSTR filename, int is_dir, ACCESS_MASK rights, uint sharing,
+            uint create_disposition, int synch);
+
+int
+delete_file(PCWSTR filename);
+
+int
+flush_file_buffers(HANDLE file_handle);
+
+int
+read_file(HANDLE file_handle, void *buffer, uint num_bytes_to_read, 
+          IN uint64* file_byte_offset OPTIONAL,
+          OUT size_t *num_bytes_read);
+int
+write_file(HANDLE file_handle, const void *buffer, uint num_bytes_to_write, 
+           OPTIONAL uint64* file_byte_offset,
+           OUT size_t *num_bytes_written);
+
+int
+close_file(HANDLE hfile);
+
+
+/* from DDK2003SP1/3790.1830/inc/ddk/wnet/ntddk.h */
+/* Define the file information class values */
+typedef enum _FILE_INFORMATION_CLASS {
+    FileDirectoryInformation         = 1,
+    FileFullDirectoryInformation,   // 2
+    FileBothDirectoryInformation,   // 3
+    FileBasicInformation,           // 4  wdm
+    FileStandardInformation,        // 5  wdm
+    FileInternalInformation,        // 6
+    FileEaInformation,              // 7
+    FileAccessInformation,          // 8
+    FileNameInformation,            // 9
+    FileRenameInformation,          // 10
+    FileLinkInformation,            // 11
+    FileNamesInformation,           // 12
+    FileDispositionInformation,     // 13
+    FilePositionInformation,        // 14 wdm
+    FileFullEaInformation,          // 15
+    FileModeInformation,            // 16
+    FileAlignmentInformation,       // 17
+    FileAllInformation,             // 18
+    FileAllocationInformation,      // 19
+    FileEndOfFileInformation,       // 20 wdm
+    FileAlternateNameInformation,   // 21
+    FileStreamInformation,          // 22
+    FilePipeInformation,            // 23
+    FilePipeLocalInformation,       // 24
+    FilePipeRemoteInformation,      // 25
+    FileMailslotQueryInformation,   // 26
+    FileMailslotSetInformation,     // 27
+    FileCompressionInformation,     // 28
+    FileObjectIdInformation,        // 29
+    FileCompletionInformation,      // 30
+    FileMoveClusterInformation,     // 31
+    FileQuotaInformation,           // 32
+    FileReparsePointInformation,    // 33
+    FileNetworkOpenInformation,     // 34
+    FileAttributeTagInformation,    // 35
+    FileTrackingInformation,        // 36
+    FileIdBothDirectoryInformation, // 37
+    FileIdFullDirectoryInformation, // 38
+    /* the following types introduced in XP and later */
+    FileValidDataLengthInformation, // 39
+    FileShortNameInformation,       // 40
+    FileMaximumInformation
+} FILE_INFORMATION_CLASS, *PFILE_INFORMATION_CLASS;
+
+/* Information class structures returned by NtQueryInformationFile */
+typedef struct _FILE_BASIC_INFORMATION {                    
+    LARGE_INTEGER CreationTime;                             
+    LARGE_INTEGER LastAccessTime;                           
+    LARGE_INTEGER LastWriteTime;                            
+    LARGE_INTEGER ChangeTime;                               
+    ULONG FileAttributes;                                   
+} FILE_BASIC_INFORMATION, *PFILE_BASIC_INFORMATION;         
+                                                            
+typedef struct _FILE_STANDARD_INFORMATION {                 
+    LARGE_INTEGER AllocationSize;                           
+    LARGE_INTEGER EndOfFile;                                
+    ULONG NumberOfLinks;                                    
+    BOOLEAN DeletePending;                                  
+    BOOLEAN Directory;                                      
+} FILE_STANDARD_INFORMATION, *PFILE_STANDARD_INFORMATION;   
+                                                            
+typedef struct _FILE_POSITION_INFORMATION {                 
+    LARGE_INTEGER CurrentByteOffset;                        
+} FILE_POSITION_INFORMATION, *PFILE_POSITION_INFORMATION;   
+                                                            
+typedef struct _FILE_ALIGNMENT_INFORMATION {                
+    ULONG AlignmentRequirement;                             
+} FILE_ALIGNMENT_INFORMATION, *PFILE_ALIGNMENT_INFORMATION; 
+                                                            
+#define MAX_FILE_NAME_LENGTH MAX_PATH
+typedef struct _FILE_NAME_INFORMATION {                     
+    ULONG FileNameLength;       /* length in _bytes_ */
+    WCHAR FileName[MAX_FILE_NAME_LENGTH];
+    /* NOTE that the official structure is FileName[1] so that
+     * callers control the length they are interested in */
+} FILE_NAME_INFORMATION, *PFILE_NAME_INFORMATION;
+                                                            
+typedef struct _FILE_RENAME_INFORMATION {
+    BOOLEAN ReplaceIfExists;
+    HANDLE RootDirectory;
+    ULONG FileNameLength;
+    WCHAR FileName[MAX_FILE_NAME_LENGTH];
+} FILE_RENAME_INFORMATION, *PFILE_RENAME_INFORMATION;
+
+typedef struct _FILE_ATTRIBUTE_TAG_INFORMATION {               
+    ULONG FileAttributes;                                       
+    ULONG ReparseTag;                                           
+} FILE_ATTRIBUTE_TAG_INFORMATION, *PFILE_ATTRIBUTE_TAG_INFORMATION;  
+                                                                
+typedef struct _FILE_DISPOSITION_INFORMATION {                  
+    BOOLEAN DeleteFile;                                         
+} FILE_DISPOSITION_INFORMATION, *PFILE_DISPOSITION_INFORMATION; 
+                                                                
+typedef struct _FILE_END_OF_FILE_INFORMATION {                  
+    LARGE_INTEGER EndOfFile;                                    
+} FILE_END_OF_FILE_INFORMATION, *PFILE_END_OF_FILE_INFORMATION; 
+                                                                
+typedef struct _FILE_VALID_DATA_LENGTH_INFORMATION {                                    
+    LARGE_INTEGER ValidDataLength;                                                      
+} FILE_VALID_DATA_LENGTH_INFORMATION, *PFILE_VALID_DATA_LENGTH_INFORMATION;             
+
+/* FileSystem types for nt_query_volume_info */
+/* should be available in ntifs.h from IFS.
+ * This version is from reactos/0.2.9/include/ndk/iotypes.h 
+ */
+typedef enum _FS_INFORMATION_CLASS {
+    FileFsVolumeInformation=1,
+    FileFsLabelInformation,     /* not documented in IFS */
+    FileFsSizeInformation,
+    FileFsDeviceInformation,
+    FileFsAttributeInformation,
+    FileFsControlInformation,
+    FileFsFullSizeInformation,
+    FileFsObjectIdInformation,
+    FileFsDriverPathInformation, /* in reactos */
+    FileFsMaximumInformation
+} FS_INFORMATION_CLASS, *PFS_INFORMATION_CLASS;
+
+typedef struct _FILE_FS_SIZE_INFORMATION {
+    LARGE_INTEGER TotalAllocationUnits;
+    LARGE_INTEGER AvailableAllocationUnits;
+    ULONG SectorsPerAllocationUnit;
+    ULONG BytesPerSector;
+} FILE_FS_SIZE_INFORMATION, *PFILE_FS_SIZE_INFORMATION;
+
+typedef struct _FILE_FS_FULL_SIZE_INFORMATION {
+    LARGE_INTEGER TotalAllocationUnits;
+    LARGE_INTEGER CallerAvailableAllocationUnits;
+    LARGE_INTEGER ActualAvailableAllocationUnits;
+    ULONG SectorsPerAllocationUnit;
+    ULONG BytesPerSector;
+} FILE_FS_FULL_SIZE_INFORMATION, *PFILE_FS_FULL_SIZE_INFORMATION;
+
+
+/* Event types and functions */
+typedef enum _EVENT_TYPE {
+    NotificationEvent,   /* manual-reset event - used for broadcasting to multiple waiting threads */
+    SynchronizationEvent /* automatically changes state to non-signaled after releasing a waiting thread */
+} EVENT_TYPE, *PEVENT_TYPE;
+
+/* event synchronization, using both automatic and manual events */
+HANDLE
+nt_create_event(EVENT_TYPE event_type);
+void
+nt_close_event(HANDLE hevent);
+
+typedef enum {WAIT_TIMEDOUT, WAIT_SIGNALED, WAIT_ERROR} wait_status_t;
+#define INFINITE_WAIT ((PLARGE_INTEGER)NULL)
+wait_status_t
+nt_wait_event_with_timeout(HANDLE hevent, PLARGE_INTEGER timeout);
+void
+nt_set_event(HANDLE hevent);
+/* should be used only for manual events (NotificationEvent) */
+void
+nt_clear_event(HANDLE hevent);
+void
+nt_signal_and_wait(HANDLE hevent_to_signal, HANDLE hevent_to_wait);
+
+HANDLE
+create_iocompletion(void);
+
+HANDLE
+open_pipe(PCWSTR pipename, HANDLE hsync);
+
+size_t 
+nt_pipe_transceive(HANDLE hpipe, void *input, uint input_size,
+                   void *output, uint output_size, uint timeout_ms);
+
+#define TIMER_UNITS_PER_MILLISECOND (1000 * 10) /* 100ns intervals */
+
+/* uint query_time_seconds() declared in os_shared.h */
+
+uint
+query_time_millis(void);
+
+void
+query_system_time(SYSTEMTIME *st);
+
+void
+nt_query_performance_counter(PLARGE_INTEGER counter, PLARGE_INTEGER frequency);
+
+#ifdef WINDOWS_PC_SAMPLE
+HANDLE
+nt_create_profile(HANDLE process_handle, void  *start, uint size, uint *buffer, 
+               uint buffer_size, uint shift);
+
+void
+nt_set_profile_interval(uint nanoseconds);
+
+int
+nt_query_profile_interval(void);
+
+void
+nt_start_profile(HANDLE profile_handle);
+
+void
+nt_stop_profile(HANDLE profile_handle);
+#endif
+
+/* avoid needing x86_code.c from x86.asm from get_own_context_helper() */
+#if !defined(NOT_DYNAMORIO_CORE) && !defined(NOT_DYNAMORIO_CORE_PROPER)
+/* Executable name must be in kernel object name form
+ * (e.g., \SystemRoot\System32\notepad.exe, or \??\c:\foo\bar.exe)
+ * The executable name on the command line can be in any form.
+ * On success returns a handle for the child
+ * On failure returns INVALID_HANDLE_VALUE
+ */
+HANDLE
+create_process(wchar_t *exe, wchar_t *cmdline);
+
+/* NOTE see important usage information in ntdll.c, threads created with this
+ * function can NOT return from their start routine */
+HANDLE
+create_thread(HANDLE hProcess, bool target_64bit, void *start_addr,
+              void *arg, const void *arg_buf, size_t arg_buf_size,
+              uint stack_reserve, uint stack_commit, bool suspended, thread_id_t *tid);
+
+/* NOTE : this isn't equivalent to nt_get_context(NT_CURRENT_THREAD, cxt)
+ * (where the returned context is undefined) so use this to get the context
+ * of the current thread */
+/* FIXME : no support for float/mmx/sse/debug, 
+ *         also if integer or control does both
+ * Xref PR 264138 where we have to preserve xmm registers: however, no
+ * current uses need to get our own xmm registers, so we don't.  
+ * PR 266070: If any future use uses this context to either set a
+ * dr_mcontext_t or passes it to nt_set_context() we'll have to add
+ * the xmm regs.
+ */
+/* NOTE : we use a macro so that we get the register state of the calling
+ * function (we don't want ebp/esp to point to a leaf routine's stack frame) */
+#define GET_OWN_CONTEXT(cxt) {                                          \
+    if (TESTANY(CONTEXT_CONTROL|CONTEXT_INTEGER, (cxt)->ContextFlags)) {\
+        byte *__get_own_context_cur_esp;                                \
+        get_own_context_helper((cxt));                                  \
+        GET_STACK_PTR(__get_own_context_cur_esp);                       \
+        (cxt)->CXT_XSP = (reg_t) __get_own_context_cur_esp;             \
+    }                                                                   \
+    get_own_context((cxt));                                             \
+}
+
+/* unstatic for use by GET_OWN_CONTEXT macro */
+void
+get_own_context_integer_control(CONTEXT *cxt, reg_t cs, reg_t ss,
+                                dr_mcontext_t *mc);
+
+/* don't call this directly, use GET_OWN_CONTEXT macro instead (it fills
+ * in CONTEXT_INTEGER and CONTEXT_CONTROL values, this fills the rest) */
+void
+get_own_context(CONTEXT *cxt);
+#endif /* !defined(NOT_DYNAMORIO_CORE) && !defined(NOT_DYNAMORIO_CORE_PROPER) */
+
+
+/***************************************************/
+/* in module_shared.c */
+
+enum { /* can't put w/ os_exports.h enum b/c needed for non-core */
+    /* for accessing x64 data from WOW64 */
+    X64_PEB_TIB_OFFSET        = 0x060,
+    X64_LDR_PEB_OFFSET        = 0x018,
+};
+
+LDR_MODULE *
+get_ldr_module_by_name(wchar_t *name);;
+
+#ifndef X64
+HANDLE
+get_module_handle_64(wchar_t *name);
+
+void *
+get_proc_address_64(HANDLE lib, const char *name);
+#endif
+
+IMAGE_EXPORT_DIRECTORY*
+get_module_exports_directory(app_pc base_addr, size_t *exports_size /* OPTIONAL OUT */);
+
+IMAGE_EXPORT_DIRECTORY*
+get_module_exports_directory_check(app_pc base_addr,
+                                   size_t *exports_size /*OPTIONAL OUT*/,
+                                   bool check_names);
+
+/***************************************************/
+
+
+/* Kernel32 uses HMODULE type for Load/FreeLibrary and GetModuleHandle. This
+ * is, at least in the platforms we've seen so far, just a pointer to the base
+ * address of the dll. We use a separate type here to keep things neat. */
+/* FIXME - this duplicates the typedef in os_private.h */
+typedef void * module_handle_t;
+
+module_handle_t
+load_library(wchar_t *lib_name);
+
+int
+free_library(module_handle_t lib);
+
+module_handle_t
+get_module_handle(wchar_t *lib_name);
+
+/* from ntddk.h */
+typedef enum _SECTION_INHERIT {
+    ViewShare = 1,
+    ViewUnmap = 2
+} SECTION_INHERIT;
+
+NTSTATUS
+nt_map_view_of_section(IN HANDLE           SectionHandle,
+                       IN HANDLE           ProcessHandle,
+                       IN OUT PVOID       *BaseAddress,
+                       IN ULONG            ZeroBits,
+                       IN SIZE_T           CommitSize,
+                       IN OUT PLARGE_INTEGER  SectionOffset OPTIONAL,
+                       IN OUT PSIZE_T      ViewSize,
+                       IN SECTION_INHERIT  InheritDisposition,
+                       IN ULONG            AllocationType,
+                       IN ULONG            Protect
+                       );
+NTSTATUS
+nt_unmap_view_of_section(IN HANDLE         ProcessHandle,
+                         IN PVOID          BaseAddress
+                         );
+
+/* From WINNT.H for .NET 2.0 (Visual Studio.NET VC7)
+ * Needed for IMAGE_COR20_HEADER
+ */
+#ifndef __IMAGE_COR20_HEADER_DEFINED__
+#define __IMAGE_COR20_HEADER_DEFINED__
+
+typedef enum replaces_cor_hdr_numeric_defines_t
+{
+// COM+ Header entry point flags.
+    COMIMAGE_FLAGS_ILONLY               =0x00000001,
+    COMIMAGE_FLAGS_32BITREQUIRED        =0x00000002,
+    COMIMAGE_FLAGS_IL_LIBRARY           =0x00000004,
+    COMIMAGE_FLAGS_STRONGNAMESIGNED     =0x00000008,
+    COMIMAGE_FLAGS_TRACKDEBUGDATA       =0x00010000,
+
+// Version flags for image.
+    COR_VERSION_MAJOR_V2                =2,
+    COR_VERSION_MAJOR                   =COR_VERSION_MAJOR_V2,
+    COR_VERSION_MINOR                   =0,
+    COR_DELETED_NAME_LENGTH             =8,
+    COR_VTABLEGAP_NAME_LENGTH           =8,
+
+// Maximum size of a NativeType descriptor.
+    NATIVE_TYPE_MAX_CB                  =1,   
+    COR_ILMETHOD_SECT_SMALL_MAX_DATASIZE=0xFF,
+
+// #defines for the MIH FLAGS
+    IMAGE_COR_MIH_METHODRVA             =0x01,
+    IMAGE_COR_MIH_EHRVA                 =0x02,    
+    IMAGE_COR_MIH_BASICBLOCK            =0x08,
+
+// V-table constants
+    COR_VTABLE_32BIT                    =0x01,          // V-table slots are 32-bits in size.   
+    COR_VTABLE_64BIT                    =0x02,          // V-table slots are 64-bits in size.   
+    COR_VTABLE_FROM_UNMANAGED           =0x04,          // If set, transition from unmanaged.
+    COR_VTABLE_CALL_MOST_DERIVED        =0x10,          // Call most derived method described by
+
+// EATJ constants
+    IMAGE_COR_EATJ_THUNK_SIZE           =32,            // Size of a jump thunk reserved range.
+
+// Max name lengths    
+    //@todo: Change to unlimited name lengths.
+    MAX_CLASS_NAME                      =1024,
+    MAX_PACKAGE_NAME                    =1024,
+} replaces_cor_hdr_numeric_defines_t;
+
+// COM+ 2.0 header structure.
+typedef struct IMAGE_COR20_HEADER
+{
+    // Header versioning
+    DWORD                   cb;              
+    WORD                    MajorRuntimeVersion;
+    WORD                    MinorRuntimeVersion;
+    
+    // Symbol table and startup information
+    IMAGE_DATA_DIRECTORY    MetaData;        
+    DWORD                   Flags;           
+    DWORD                   EntryPointToken;
+    
+    // Binding information
+    IMAGE_DATA_DIRECTORY    Resources;
+    IMAGE_DATA_DIRECTORY    StrongNameSignature;
+
+    // Regular fixup and binding information
+    IMAGE_DATA_DIRECTORY    CodeManagerTable;
+    IMAGE_DATA_DIRECTORY    VTableFixups;
+    IMAGE_DATA_DIRECTORY    ExportAddressTableJumps;
+
+    // Precompiled image info (internal use only - set to zero)
+    IMAGE_DATA_DIRECTORY    ManagedNativeHeader;
+    
+} IMAGE_COR20_HEADER, *PIMAGE_COR20_HEADER;
+
+#endif // __IMAGE_COR20_HEADER_DEFINED__
+
+
+#ifndef IMAGE_SCN_ALIGN_MASK
+#define IMAGE_SCN_ALIGN_MASK 0x00F00000 /* from latest winnt.h */
+#endif /* IMAGE_SCN_ALIGN_MASK */
+
+NTSTATUS
+nt_initialize_shared_directory(HANDLE *shared_directory /* OUT */, bool permanent);
+
+NTSTATUS
+nt_open_object_directory(HANDLE *shared_directory /* OUT */, 
+                         PCWSTR object_directory_name,
+                         bool allow_creation);
+
+void
+nt_close_object_directory(HANDLE hobjdir);
+
+NTSTATUS
+nt_get_symlink_target(IN HANDLE              directory_handle,
+                      IN PCWSTR              symlink_name,
+                      IN OUT UNICODE_STRING* target_name,
+                      OUT uint *             returned_byte_length);
+
+#define MAX_OBJECT_NAME_LENGTH MAX_PATH
+typedef struct _OBJECT_NAME_INFORMATION {
+    UNICODE_STRING ObjectName;
+    /* note that the nt interface allows callers to specify any size,
+     * yet we do not expect needs for longer */
+    wchar_t object_name_buffer[MAX_OBJECT_NAME_LENGTH];
+} OBJECT_NAME_INFORMATION, *POBJECT_NAME_INFORMATION;
+
+NTSTATUS
+nt_get_object_name(HANDLE handle, OBJECT_NAME_INFORMATION* object_name /* OUT */,
+                   uint byte_length, uint *returned_byte_length);
+
+NTSTATUS
+nt_create_section(OUT PHANDLE SectionHandle,
+                  IN ACCESS_MASK DesiredAccess,
+                  IN PLARGE_INTEGER SectionSize OPTIONAL,
+                  IN ULONG Protect,
+                  IN ULONG section_creation_attributes,
+                  IN HANDLE FileHandle,
+                  /* object name attributes */
+                  IN PCWSTR new_section_name,
+                  IN ULONG object_name_attributes,
+                  IN HANDLE object_directory,
+                  IN PSECURITY_DESCRIPTOR dacl
+                  );
+NTSTATUS
+nt_open_section(OUT PHANDLE SectionHandle,
+                IN ACCESS_MASK DesiredAccess,
+                /* object name attributes */
+                IN PCWSTR section_name,
+                IN ULONG object_name_attributes,
+                IN HANDLE object_directory);
+
+NTSTATUS
+nt_create_module_file(OUT HANDLE* file_handle,
+                      const wchar_t *file_path, 
+                      IN HANDLE root_directory_handle OPTIONAL,
+                      ACCESS_MASK desired_access_rights,
+                      uint file_special_attributes,
+                      uint file_sharing_flags,
+                      uint create_disposition, 
+                      size_t allocation_size);
+
+NTSTATUS
+nt_query_file_info(IN HANDLE FileHandle,
+                   OUT PVOID FileInformation,
+                   IN ULONG FileInformationLength,
+                   IN FILE_INFORMATION_CLASS FileInformationClass);
+NTSTATUS
+nt_set_file_info(IN HANDLE FileHandle,
+                 IN PVOID FileInformation,
+                 IN ULONG FileInformationLength,
+                 IN FILE_INFORMATION_CLASS FileInformationClass);
+
+NTSTATUS
+nt_query_volume_info(IN HANDLE FileHandle,
+                     OUT PVOID FsInformation,
+                     IN ULONG FsInformationLength,
+                     IN FS_INFORMATION_CLASS FsInformationClass);
+NTSTATUS
+nt_query_security_object(IN HANDLE Handle,
+                         IN SECURITY_INFORMATION RequestedInformation,
+                         OUT PSECURITY_DESCRIPTOR SecurityDescriptor,
+                         IN ULONG SecurityDescriptorLength,
+                         OUT PULONG ReturnLength
+                         );
+
+
+#endif /* _NTDLL_H_ */
diff --git a/core/win32/os.c b/core/win32/os.c
new file mode 100644
index 000000000..3245bd18b
--- /dev/null
+++ b/core/win32/os.c
@@ -0,0 +1,6975 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * os.c - win32 specific routines
+ */
+
+#include "globals.h"
+#include "fragment.h"
+#include "fcache.h"
+#include "ntdll.h"
+#include "os_private.h"
+#include "nudge.h"
+#include "moduledb.h"
+#include "hotpatch.h"
+#ifdef DEBUG
+# include "vmareas.h"
+#endif
+#include "dispatch.h"
+#include "instrument.h" /* is_in_client_lib() */
+
+#include <windows.h>
+#include <stddef.h> /* for offsetof */
+
+#include "events.h"             /* event log messages */
+#include "aslr.h"
+#include "synch.h"
+
+#ifdef DEBUG
+DECLARE_CXTSWPROT_VAR(static mutex_t snapshot_lock, INIT_LOCK_FREE(snapshot_lock));
+#endif
+
+DECLARE_CXTSWPROT_VAR(static mutex_t dump_core_lock, INIT_LOCK_FREE(dump_core_lock));
+DECLARE_CXTSWPROT_VAR(static mutex_t debugbox_lock, INIT_LOCK_FREE(debugbox_lock));
+
+/* globals */
+bool intercept_asynch = false;
+bool intercept_callbacks = false;
+/* we store here to enable TEB.ClientId.ProcessHandle as a spill slot */
+process_id_t win32_pid = 0;
+/* we store here to enable TEB.ProcessEnvironmentBlock as a spill slot */
+void *peb_ptr;
+static int os_version;
+static const char *os_name;
+app_pc vsyscall_page_start = NULL;
+/* pc kernel will claim app is at while in syscall */
+app_pc vsyscall_after_syscall = NULL;
+/* pc of the end of the syscall instr itself */
+app_pc vsyscall_syscall_end_pc = NULL;
+/* atomic variable to prevent multiple threads from trying to detach at the same time */
+DECLARE_CXTSWPROT_VAR(static volatile int dynamo_detaching_flag, LOCK_FREE_STATE);
+
+static bool reached_image_entry = false;
+
+#ifdef PROFILE_RDTSC
+uint kilo_hertz; /* cpu clock speed */
+#endif
+
+#define HEAP_INITIAL_SIZE 1024*1024
+
+/* pc values delimiting dynamo dll image */
+app_pc dynamo_dll_start = NULL;
+app_pc dynamo_dll_end = NULL; /* open-ended */
+
+/* needed for randomizing DR library location */
+static app_pc dynamo_dll_preferred_base = NULL;
+
+/* thread-local storage slots */
+enum {TLS_UNINITIALIZED = (ushort) 0U};
+static ushort tls_local_state_offs = TLS_UNINITIALIZED;
+/* we keep this cached for easy asm access */
+static ushort tls_dcontext_offs = TLS_UNINITIALIZED;
+
+/* used for early inject */
+app_pc parent_early_inject_address = NULL; /* dynamo.c fills in */
+/* note that this is the early inject location we'll use for child processes
+ * dr_early_injected_location is the location (if any) that the current
+ * process was injected at */
+static int early_inject_location = INJECT_LOCATION_Invalid;
+static app_pc early_inject_address = NULL;
+static app_pc ldrpLoadDll_address_not_NT = NULL;
+static app_pc ldrpLoadDll_address_NT = NULL;
+static app_pc ldrpLoadImportModule_address = NULL;
+dcontext_t *early_inject_load_helper_dcontext = NULL;
+
+/* forwards */
+static void
+get_system_basic_info(void);
+static bool
+is_child_in_thin_client(HANDLE process_handle);
+static const char*
+get_process_SID_string(void);
+static const PSID
+get_Everyone_SID(void);
+static const PSID
+get_process_owner_SID(void);
+
+bool
+os_user_directory_supports_ownership(void);
+
+/* Safely gets the target of the call (assumed to be direct) to the nth stack
+ * frame (i.e. the entry point to that function), returns NULL on failure. 
+ * NOTE - Be aware this routine may be called by DllMain before dr is
+ * initialized (before even syscalls_init, though note that safe_read should
+ * be fine as will just use the nt wrapper). */
+static app_pc
+get_nth_stack_frames_call_target(int num_frames, reg_t *ebp)
+{
+    reg_t *cur_ebp = ebp;
+    reg_t next_frame[2];
+    int i;
+    
+    /* walk up the stack */
+    for (i = 0; i < num_frames; i++) {
+        if (!safe_read(cur_ebp, sizeof(next_frame), next_frame))
+            break;
+        cur_ebp = (reg_t *)next_frame[0];
+    }
+
+    if (i == num_frames) {
+        /* success walking frames, return address should be the after
+         * call address of the call that targeted this frame */
+        /* FIXME - would be nice to get this with decode_cti, but dr might
+         * not even be initialized yet and this is safer */
+        byte buf[5]; /* sizeof call rel32 */
+        if (safe_read((byte *)(next_frame[1] - sizeof(buf)), sizeof(buf), &buf) && 
+            buf[0] == CALL_REL32_OPCODE) {
+            app_pc return_point = (app_pc)next_frame[1];
+            return (return_point + *(int *)&buf[1]);
+        }
+    }
+    return NULL;
+}
+
+/* Should be called from NtMapViewOfSection interception with *base
+ * pointing to the just mapped region. */ 
+void
+check_for_ldrpLoadImportModule(byte *base, uint *ebp)
+{
+    MEMORY_BASIC_INFORMATION mbi;
+    if (query_virtual_memory(base, &mbi, sizeof(mbi)) == sizeof(mbi)
+        && mbi.Type == MEM_IMAGE && is_readable_pe_base(base)) {
+        /* loaded a module, check name */
+        const char *name;
+        bool match = false;
+        
+        name = get_dll_short_name(base); /* we only need pe name */
+        if (name != NULL) {
+            LOG(GLOBAL, LOG_TOP, 1,
+                "early_inject hit mapview of image %s\n", name);
+            string_option_read_lock();
+            /* we control both the pe_name and the option value so use
+             * strcmp (vs. strcasecmp), just to keep things tight */
+            match = 
+                strcmp(DYNAMO_OPTION(early_inject_helper_name),
+                       name) == 0;
+            string_option_read_unlock();
+        }
+        if (match) {
+            /* Found it. We expect the stack to look like this
+             *   (in NtMapViewOfSection)
+             *   ntdll!LdrpMapDll
+             *   ntdll!LdrpLoadImportModule (what we want)
+             * After that don't really care (is one of the
+             * Ldrp*ImportDescriptor* routines. So we walk the
+             * stack back and get the desired address.
+             * FIXME - would be nice if we had some way to double
+             * check this address, could try to decode and check against
+             * the versions we've seen.
+             * Note that NtMapViewOfSection in all its various platform forms
+             * (i.e. int, vsyscall, KiFastSystemCall etc.) doesn't set up a
+             * new frame (nor do its callees) so will always be depth 2
+             */
+#define STACK_DEPTH_LdrpLoadImportModule 2
+            ldrpLoadImportModule_address =
+                get_nth_stack_frames_call_target(STACK_DEPTH_LdrpLoadImportModule,
+                                                 (reg_t *)ebp);
+            LOG(GLOBAL, LOG_TOP, 1,
+                "early_inject found address "PFX" for LdrpLoadImportModule\n",
+                ldrpLoadImportModule_address);
+        }
+    }
+}
+    
+/****************************************************************************
+ ** DllMain Routines
+ **
+ **/
+
+#ifdef INTERNAL
+/* we have interp not inline calls to this routine */
+void
+DllMainThreadAttach()
+{
+    if (INTERNAL_OPTION(noasynch) && dynamo_initialized && !dynamo_exited) {
+        /* we normally intercept thread creation in callback.c, but with
+         * noasynch, we do it here (which is later, but better than nothing)
+         */
+        LOG(GLOBAL, LOG_TOP|LOG_THREADS, 1,
+            "DllMain: initializing new thread %d\n", get_thread_id());
+        dynamo_thread_init();
+    }
+}
+#endif
+
+/* Hand-made DO_ONCE since DllMain is executed prior to DR init */
+DECLARE_FREQPROT_VAR(static bool do_once_DllMain, false);
+
+/* DLL entry point 
+ * N.B.: dynamo interprets this routine!
+ */
+bool WINAPI /* get link warning 4216 if export via APIENTRY */
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    switch (reason_for_call) {
+    case DLL_PROCESS_ATTACH:
+        /* case 8097: with -no_hide, DllMain will be called a second time
+         * after all the statically-bound dlls are loaded (the loader
+         * blindly calls all the init routines regardless of whether a dll
+         * was explicitly loaded and already had its init routine called).
+         * We make that 2nd time a nop via a custom DO_ONCE (since default
+         * DO_ONCE will try to unprotect .data, but we're pre-init).
+         */
+        if (!do_once_DllMain) {
+            byte *cur_ebp;
+            do_once_DllMain = true;
+            ASSERT(!dynamo_initialized);
+            ASSERT(ldrpLoadDll_address_NT == NULL);
+            ASSERT(ldrpLoadDll_address_not_NT == NULL);
+            /* Carefully walk stack to find address of LdrpLoadDll. */
+            /* Remember dr isn't initialized yet, no need to worry about
+             * protect from app etc., but also can't check options. */
+            GET_FRAME_PTR(cur_ebp);
+            /* For non early_inject (late follow children, preinject) expect
+             * stack to look like (for win2k and higher)
+             *   here (our DllMain)
+             *   ntdll!LdrpCallInitRoutine
+             *   ntdll!LdrpRunInitializeRoutines
+             *   ntdll!LdrpLoadDll
+             *   ntdll!LdrLoadDll 
+             * For NT is the same only doesn't have ntdll!LdrpCallInitRoutine.
+             *
+             * That's as far we care, after that is likely to be shimeng.dll
+             * or kernel32 (possibly someone else?) depending on how we were
+             * injected. For -early_inject, ntdll!LdrGetProcedureAddress is
+             * usually the root of the call to our DLLMain (likely something
+             * to do with load vs. init order at process startup? FIXME
+             * understand better, is there a flag we can send to have this
+             * called on load?), but in that case we use the address passed to
+             * us by the parent. */
+#define STACK_DEPTH_LdrpLoadDll_NT 3
+#define STACK_DEPTH_LdrpLoadDll 4
+            /* Since dr isn't initialized yet we can't call get_os_version()
+             * so just grab both possible LdrpLoadDll addresses (NT and non NT)
+             * and we'll sort it out later in early_inject_init(). */
+            ldrpLoadDll_address_NT =
+                get_nth_stack_frames_call_target(STACK_DEPTH_LdrpLoadDll_NT,
+                                                 (reg_t *)cur_ebp);
+            ldrpLoadDll_address_not_NT =
+                get_nth_stack_frames_call_target(STACK_DEPTH_LdrpLoadDll,
+                                                 (reg_t *)cur_ebp);
+            /* FIXME - would be nice to have extra verification here,
+             * but after this frame there are too many possibilites (many
+             * of which are unexported) so is hard to find something we
+             * can check. */
+        } else
+            ASSERT(dynamo_initialized);
+        break;
+#ifdef INTERNAL
+    case DLL_THREAD_ATTACH:
+        DllMainThreadAttach();
+        break;
+#endif
+    /* we don't care about DLL_PROCESS_DETACH or DLL_THREAD_DETACH */
+    }
+    return true;
+}
+
+#ifdef WINDOWS_PC_SAMPLE
+static profile_t *global_profile = NULL;
+static profile_t *dynamo_dll_profile = NULL;
+static profile_t *ntdll_profile = NULL;
+file_t profile_file = INVALID_FILE;
+DECLARE_CXTSWPROT_VAR(mutex_t profile_dump_lock, INIT_LOCK_FREE(profile_dump_lock));
+
+static void
+get_dll_bounds(wchar_t *name, app_pc *start, app_pc *end)
+{
+    HMODULE dllh;
+    size_t len;
+    PBYTE pb;
+    MEMORY_BASIC_INFORMATION mbi;
+
+    dllh = get_module_handle(name);
+    ASSERT(dllh != NULL);
+    pb = (PBYTE) dllh;
+    /* FIXME: we should just call get_allocation_size() */
+    len = query_virtual_memory(pb, &mbi, sizeof(mbi));
+    ASSERT(len == sizeof(mbi));
+    ASSERT(mbi.State != MEM_FREE);
+    *start = (app_pc) mbi.AllocationBase;
+    do {
+        if (mbi.State == MEM_FREE || (app_pc) mbi.AllocationBase != *start)
+            break;
+        if (pb + mbi.RegionSize < pb)
+            break;
+        pb += mbi.RegionSize;
+    } while (query_virtual_memory(pb, &mbi, sizeof(mbi)) == sizeof(mbi));
+    *end = (app_pc) pb;
+}
+
+static void
+init_global_profiles()
+{
+    app_pc start, end;
+
+    /* create the profile file */
+    /* if logging is on create in log directory, else use base directory */
+    DOLOG(1, LOG_ALL, {
+        char buf[MAX_PATH];
+        uint size = BUFFER_SIZE_ELEMENTS(buf);
+        if (get_log_dir(PROCESS_DIR, buf, &size)) {
+            NULL_TERMINATE_BUFFER(buf);
+            strncat(buf, "\\profile", BUFFER_SIZE_ELEMENTS(buf) - strlen(buf));
+            NULL_TERMINATE_BUFFER(buf);
+            profile_file = os_open(buf, OS_OPEN_REQUIRE_NEW|OS_OPEN_WRITE);
+            LOG(GLOBAL, LOG_PROFILE, 1, "Profile file is \"%s\"\n", buf);
+        }
+    });
+    if (profile_file == INVALID_FILE) {
+        get_unique_logfile(".profile", NULL, 0, false, &profile_file); 
+    }
+    DOLOG(1, LOG_PROFILE, {
+        if (profile_file == INVALID_FILE) 
+            LOG(GLOBAL, LOG_PROFILE, 1, "Failed to create profile file\n");
+    });
+    ASSERT(profile_file != INVALID_FILE);
+
+    /* Case 7533: put basic run info in profile file. */
+    print_version_and_app_info(profile_file);
+
+    /* set the interval, don't assert success, on my desktop anything less than
+     * 1221 gets set to 1221 on laptop was different minimum value, at least
+     * appears that os sets it close as possible to the requested (starting
+     * value was 39021 for me) */
+    LOG(GLOBAL, LOG_PROFILE, 1, "Profile interval was %d, setting to %d,", 
+        nt_query_profile_interval(), dynamo_options.prof_pcs_freq);
+    nt_set_profile_interval(dynamo_options.prof_pcs_freq);
+    LOG(GLOBAL, LOG_PROFILE, 1, " is now %d (units of 100 nanoseconds)\n", 
+        nt_query_profile_interval());
+    print_file(profile_file, "Interval %d\n\n", nt_query_profile_interval());
+
+    /* create profiles */
+    /* Default shift of 30 gives 4 buckets for the global profile,
+     * allows us to separate kernel and user space (even in the case
+     * of 3GB user space).  Note if invalid range given we default to
+     * 30, so we always have a global profile to use as denominator
+     * later.
+     */
+    global_profile = create_profile(UNIVERSAL_REGION_BASE, UNIVERSAL_REGION_END, 
+                                    DYNAMO_OPTION(prof_pcs_global), NULL);
+    if (dynamo_options.prof_pcs_DR >= 2 && dynamo_options.prof_pcs_DR <= 32) {
+        get_dll_bounds(L_DYNAMORIO_LIBRARY_NAME, &start, &end);
+        dynamo_dll_profile = create_profile(start, end,
+                                            dynamo_options.prof_pcs_DR, NULL);
+    }
+    if (dynamo_options.prof_pcs_ntdll >= 2 && 
+        dynamo_options.prof_pcs_ntdll <= 32) {
+        get_dll_bounds(L"ntdll.dll", &start, &end);
+        ntdll_profile = create_profile(start, end, 
+                                       dynamo_options.prof_pcs_ntdll, NULL);
+    }
+    
+    /* start profiles */
+    start_profile(global_profile);
+    if (dynamo_dll_profile)
+        start_profile(dynamo_dll_profile);
+    if (ntdll_profile)
+        start_profile(ntdll_profile);
+}
+
+static void
+dump_dll_profile(profile_t *profile, uint global_sum, char *dll_name)
+{
+    uint dll_sum;
+    uint top=0, bottom=0;
+
+    dll_sum = sum_profile(profile);
+    if (global_sum > 0)
+        divide_uint64_print(dll_sum, global_sum, true, 2, &top, &bottom);
+    print_file(profile_file, 
+               "\nDumping %s profile\n%d hits out of %d, %u.%.2u%%\n",
+               dll_name, dll_sum, global_sum, top, bottom);
+    LOG(GLOBAL, LOG_PROFILE, 1, 
+        "%s profile had %d hits out of %d total, %u.%.2u%%\n",
+        dll_name, dll_sum, global_sum, top, bottom);
+    dump_profile(profile_file, profile);
+    free_profile(profile);
+}
+
+static void
+exit_global_profiles()
+{
+    int global_sum;
+
+    if (dynamo_dll_profile)
+        stop_profile(dynamo_dll_profile);
+    if (ntdll_profile)
+        stop_profile(ntdll_profile);
+    stop_profile(global_profile);
+
+    global_sum = sum_profile(global_profile);
+
+    /* we expect to be the last thread at this point. 
+       FIXME: we can remove the mutex_lock/unlock then */
+    mutex_lock(&profile_dump_lock);
+    if (dynamo_dll_profile)
+        dump_dll_profile(dynamo_dll_profile, global_sum, "dynamorio.dll");
+    if (ntdll_profile)
+        dump_dll_profile(ntdll_profile, global_sum, "ntdll.dll");
+
+    print_file(profile_file, "\nDumping global profile\n%d hits\n", global_sum);
+    dump_profile(profile_file, global_profile);
+    mutex_unlock(&profile_dump_lock);
+    LOG(GLOBAL, LOG_PROFILE, 1, 
+        "\nDumping global profile\n%d hits\n", global_sum);
+    DOLOG(1, LOG_PROFILE, dump_profile(GLOBAL, global_profile););
+    free_profile(global_profile);
+
+    DELETE_LOCK(profile_dump_lock);
+}
+#endif
+
+/**
+ **
+ ****************************************************************************/
+
+/* FIXME: Right now error reporting will work here, but once we have our
+ * error reporting syscalls going through wrappers and requiring this
+ * init routine, we'll have to have a fallback here that dynamically
+ * determines the syscalls and finishes init, and then reports the error.
+ * We may never be able to report errors for the non-NT OS family.
+ * N.B.: this is too early for LOGs so don't do any -- any errors reported
+ * will not die, they will simply skip LOG.
+ * N.B.: this is prior to eventlog_init(), but then we've been reporting
+ * usage errors prior to that for a long time now anyway.
+ */
+void
+windows_version_init()
+{
+    PEB *peb = get_own_peb();
+    /* choose appropriate syscall array (the syscall numbers change from
+     * one version of windows to the next!
+     * they may even change at different patch levels)
+     */
+    syscalls = NULL;
+
+    DODEBUG({ check_syscall_array_sizes(); });
+
+    if (peb->OSPlatformId == VER_PLATFORM_WIN32_NT) {
+        /* WinNT or descendents */
+        if (peb->OSMajorVersion == 6 && peb->OSMinorVersion == 0) {
+            module_handle_t ntdllh = get_ntdll_base();
+            /* Vista system call number differ between service packs, we use
+             * the existence of NtReplacePartitionUnit to detect sp1 - see
+             * PR 246402.  They also differ for 32-bit vs 64-bit/wow64. */
+            if (get_proc_address(ntdllh, "NtReplacePartitionUnit") != NULL) {
+                if (module_is_64bit(get_ntdll_base()) ||
+                    is_wow64_process(NT_CURRENT_PROCESS)) {
+                    syscalls = (int *) windows_vista_sp1_x64_syscalls;
+                    os_name = "Microsoft Windows Vista x64 SP1";
+                } else {
+                    syscalls = (int *) windows_vista_sp1_syscalls;
+                    os_name = "Microsoft Windows Vista SP1";
+                }
+            } else {
+                if (module_is_64bit(get_ntdll_base()) ||
+                    is_wow64_process(NT_CURRENT_PROCESS)) {
+                    syscalls = (int *) windows_vista_sp0_x64_syscalls;
+                    os_name = "Microsoft Windows Vista x64 SP0";
+                } else {
+                    syscalls = (int *) windows_vista_sp0_syscalls;
+                    os_name = "Microsoft Windows Vista SP0";
+                }
+            }
+            os_version = WINDOWS_VERSION_VISTA;
+        } else if (peb->OSMajorVersion == 5 && peb->OSMinorVersion == 2) {
+            /* Version 5.2 can mean 32- or 64-bit 2003, or 64-bit XP */
+            /* Assumption: get_ntll_base makes no system calls */
+            if (module_is_64bit(get_ntdll_base()) ||
+                is_wow64_process(NT_CURRENT_PROCESS)) {
+                /* We expect x64 2003 and x64 XP to have the same system call
+                 * numbers but that has not been verified.  System call numbers
+                 * remain the same even under WOW64 (ignoring the extra WOW
+                 * system calls, anyway).  We do not split the version for WOW
+                 * as most users do not care to distinguish; those that do must
+                 * use a separate is_wow64_process() check.
+                 */
+                syscalls = (int *) windows_XP_x64_syscalls;
+                /* We don't yet have need to split the version enum */
+                os_version = WINDOWS_VERSION_2003;
+                os_name = "Microsoft Windows x64 XP/2003";
+            } else {
+                syscalls = (int *) windows_2003_syscalls;
+                os_version = WINDOWS_VERSION_2003;
+                os_name = "Microsoft Windows 2003";
+            }
+        } else if (peb->OSMajorVersion == 5 && peb->OSMinorVersion == 1) {
+            syscalls = (int *) windows_XP_syscalls;
+            os_version = WINDOWS_VERSION_XP;
+            os_name = "Microsoft Windows XP";
+        } else if (peb->OSMajorVersion == 5 && peb->OSMinorVersion == 0) {
+            syscalls = (int *) windows_2000_syscalls;
+            os_version = WINDOWS_VERSION_2000;
+            os_name = "Microsoft Windows 2000";
+        } else if (peb->OSMajorVersion == 4) {
+            module_handle_t ntdllh = get_ntdll_base();
+            os_version = WINDOWS_VERSION_NT;
+            /* NT4 syscalls differ among service packs.
+             * Rather than reading the registry to find the service pack we
+             * directly check which system calls are there.  We don't just
+             * check the number of the last syscall in our list b/c we want to
+             * avoid issues w/ hookers.
+             * We rely on these observations:
+             *   SP3: + Nt{Read,Write}FileScatter
+             *   SP4: - NtW32Call
+             */
+            if (get_proc_address(ntdllh, "NtW32Call") != NULL) {
+                /* < SP4 */
+                /* we don't know whether SP1 and SP2 fall in line w/ SP0 or w/ SP3,
+                 * or possibly are different from both, but we don't support them
+                 */
+                if (get_proc_address(ntdllh, "NtReadFileScatter") != NULL) {
+                    /* > SP0 */
+                    syscalls = (int *) windows_NT_sp3_syscalls;
+                    os_name = "Microsoft Windows NT SP3";
+                } else {
+                    /* < SP3 */
+                    syscalls = (int *) windows_NT_sp0_syscalls;
+                    os_name = "Microsoft Windows NT SP0";
+                }
+            } else {
+                syscalls = (int *) windows_NT_sp4_syscalls;
+                os_name = "Microsoft Windows NT SP4, 5, 6, or 6a";
+            }
+        } else {
+            SYSLOG_INTERNAL_ERROR("Unknown Windows NT-family version: major=%d, minor=%d\n",
+                                  peb->OSMajorVersion, peb->OSMinorVersion);
+            os_name = "Unrecognized Windows NT-family version";
+            FATAL_USAGE_ERROR(BAD_OS_VERSION, 4, get_application_name(),
+                              get_application_pid(), PRODUCT_NAME, os_name);
+        }
+    } else if (peb->OSPlatformId == VER_PLATFORM_WIN32_WINDOWS) {
+        /* Win95 or Win98 */
+        uint ver_high = (peb->OSBuildNumber >> 8) & 0xff;
+        uint ver_low = peb->OSBuildNumber & 0xff;
+        if (ver_low >= 90 || ver_high >= 5)
+            os_name = "Windows ME";
+        else if (ver_low >= 10 && ver_low < 90)
+            os_name = "Windows 98";
+        else if (ver_low < 5)
+            os_name = "Windows 3.1 / WfWg";
+        else if (ver_low < 10)
+            os_name = "Windows 98";
+        else
+            os_name = "this unknown version of Windows";
+        FATAL_USAGE_ERROR(BAD_OS_VERSION, 4, get_application_name(),
+                          get_application_pid(), PRODUCT_NAME, os_name);
+    } else {
+        os_name = "Win32s";
+        /* Win32S on Windows 3.1 */
+        FATAL_USAGE_ERROR(BAD_OS_VERSION, 4, get_application_name(),
+                          get_application_pid(), PRODUCT_NAME, os_name);
+    }
+}
+
+/* Note that assigning a process to a Job is done only after it has
+ * been created - with ZwAssignProcessToJobObject(), and we may start
+ * before or after that has been done.
+ */
+static void
+print_mem_quota()
+{
+    QUOTA_LIMITS qlimits;
+    NTSTATUS res = get_process_mem_quota(NT_CURRENT_PROCESS, &qlimits);
+    if (!NT_SUCCESS(res)) {
+        ASSERT(false && "print_mem_quota");
+        return;
+    }
+    LOG(GLOBAL, LOG_TOP, 1, "Process Memory Limits:\n");
+    LOG(GLOBAL, LOG_TOP, 1, "\t Paged pool limit:         %6d KB\n",
+        qlimits.PagedPoolLimit/1024);
+    LOG(GLOBAL, LOG_TOP, 1, "\t Non Paged pool limit:     %6d KB\n",
+        qlimits.NonPagedPoolLimit/1024);
+    LOG(GLOBAL, LOG_TOP, 1, "\t Minimum working set size: %6d KB\n",
+        qlimits.MinimumWorkingSetSize/1024);
+    LOG(GLOBAL, LOG_TOP, 1, "\t Maximum working set size: %6d KB\n",
+        qlimits.MaximumWorkingSetSize/1024);
+
+    /* 4GB for unlimited */
+    LOG(GLOBAL, LOG_TOP, 1, "\t Pagefile limit:          %6d KB\n",
+        qlimits.PagefileLimit/1024);
+    /* TimeLimit not supported on Win2k, but WSRM (Windows System
+     * Resource Manager) can definitely set, so expected to be
+     * supported on Win2003.  Time in 100ns units.
+     */
+    LOG(GLOBAL, LOG_TOP, 1, "\t TimeLimit:    0x%.8x%.8x\n",
+        qlimits.TimeLimit.HighPart, qlimits.TimeLimit.LowPart);
+}
+
+/* os-specific initializations */
+static void init_debugbox_title_buf(void);
+void
+os_init(void)
+{
+    PEB *peb = get_own_peb();
+    uint alignment = 0;
+    uint offs;
+    int res;
+
+    if (dynamo_options.max_supported_os_version <
+        peb->OSMajorVersion * 10 + peb->OSMinorVersion) {
+        SYSLOG(SYSLOG_WARNING, UNSUPPORTED_OS_VERSION, 3, 
+               get_application_name(), get_application_pid(), os_name);
+    }
+
+    /* make sure we create the message box title string before we are 
+     * multi-threaded and is no longer safe to do so on demand, this also
+     * takes care of initializing the static buffer get_appilication_name
+     * and get_application_pid */
+    init_debugbox_title_buf();
+
+    win32_pid =  get_process_id();
+    LOG(GLOBAL, LOG_TOP, 1, "Process id: %d\n", win32_pid);
+    peb_ptr = (void *) get_own_peb();
+    LOG(GLOBAL, LOG_TOP, 1, "PEB: "PFX"\n", peb_ptr);
+    ASSERT((PEB *)peb_ptr == get_own_teb()->ProcessEnvironmentBlock);
+
+    /* match enums in os_exports.h with TEB definition from ntdll.h */
+    ASSERT(EXCEPTION_LIST_TIB_OFFSET == offsetof(TEB, ExceptionList));
+    ASSERT(TOP_STACK_TIB_OFFSET == offsetof(TEB, StackBase));
+    ASSERT(BASE_STACK_TIB_OFFSET == offsetof(TEB, StackLimit));
+    ASSERT(FIBER_DATA_TIB_OFFSET == offsetof(TEB, FiberData));
+    ASSERT(SELF_TIB_OFFSET == offsetof(TEB, Self));
+    ASSERT(TID_TIB_OFFSET == offsetof(TEB, ClientId) +
+           offsetof(CLIENT_ID, UniqueThread));
+    ASSERT(PID_TIB_OFFSET == offsetof(TEB, ClientId) +
+           offsetof(CLIENT_ID, UniqueProcess));
+    ASSERT(ERRNO_TIB_OFFSET == offsetof(TEB, LastErrorValue));
+    ASSERT(WOW64_TIB_OFFSET == offsetof(TEB, WOW32Reserved));
+    ASSERT(PEB_TIB_OFFSET == offsetof(TEB, ProcessEnvironmentBlock));
+
+    /* windows_version_init should have already been called */
+    ASSERT(syscalls != NULL);
+    LOG(GLOBAL, LOG_TOP, 1, "Running on %s\n", os_name);
+
+    ntdll_init();
+    callback_init();
+
+    eventlog_init(); /* os dependent and currently Windows specific */
+
+    if (os_version >= WINDOWS_VERSION_XP) {
+        /* FIXME: bootstrapping problem where we see 0x7ffe0300 before we see
+         * the 1st sysenter...solution for now is to hardcode initial values so
+         * we pass the 1st PROGRAM_SHEPHERDING code origins test, then re-set these once
+         * we see the 1st syscall.
+         */
+        /* on XP service pack 2 the syscall enter and exit stubs are Ki 
+         * routines in ntdll.dll FIXME : as a hack for now will leave 
+         * page_start as 0 (as it would be for 2000, since region is 
+         * executable no need for the code origins exception) and 
+         * after syscall to the appropriate value, this means will still 
+         * execute the return natively (as in xp/03) for simplicity even 
+         * though we could intercept it much more easily than before since 
+         * the ki routines are aligned (less concern about enough space for the
+         * interception stub, nicely exported for us etc.)
+         */
+        /* initializing so get_module_handle should be safe, FIXME */
+        module_handle_t ntdllh = get_ntdll_base();
+        app_pc return_point = (app_pc) get_proc_address(ntdllh, 
+                                                        "KiFastSystemCallRet");
+        if (return_point) {
+            vsyscall_after_syscall = (app_pc) return_point;
+            vsyscall_syscall_end_pc = NULL; /* wait until 1st one */
+        } else {
+            /* FIXME : if INT syscalls are being used then this opens up a
+             * security hole for the followin page */
+            vsyscall_page_start = VSYSCALL_PAGE_START_BOOTSTRAP_VALUE;
+            vsyscall_after_syscall = VSYSCALL_AFTER_SYSCALL_BOOTSTRAP_VALUE;
+            vsyscall_syscall_end_pc = vsyscall_after_syscall;
+        }
+    }
+
+    /* TLS alignment use either preferred on processor, or hardcoded option value */
+    if (DYNAMO_OPTION(tls_align) == 0) {
+        IF_X64(ASSERT_TRUNCATE(alignment, uint, proc_get_cache_line_size()));
+        alignment = (uint) proc_get_cache_line_size();
+    } else {
+        alignment = DYNAMO_OPTION(tls_align);
+    }
+    /* case 3701 about performance gains, 
+     * and case 6670 about TLS conflict in SQL2005 */
+    
+    /* FIXME: could control which entry should be cache aligned, but
+     * we should be able to restructure the state to ensure first
+     * entry is indeed important.  Should make sure we choose same
+     * position in both release and debug, see local_state_t.stats.
+     */
+
+    /* allocate thread-private storage */
+    res = tls_calloc(false/*no synch required*/, &offs, TLS_NUM_SLOTS,
+                     alignment);
+
+    DODEBUG({
+        /* FIXME: elevate failure here to a release-build syslog? */
+        if (!res)
+            SYSLOG_INTERNAL_ERROR("Cannot allocate %d tls slots at %d alignment", TLS_NUM_SLOTS,
+                                  alignment);
+    });
+
+    /* retry with no alignment on failure */
+    if (!res) {
+        alignment = 0;
+        ASSERT_NOT_TESTED();
+            
+        /* allocate thread-private storage with no alignment */
+        res = tls_calloc(false/*no synch required*/, &offs, TLS_NUM_SLOTS,
+                         alignment);
+
+        /* report even in release build that we really can't grab in TLS64 */
+        if (!res) {
+            ASSERT_NOT_TESTED();
+            SYSLOG_INTERNAL_ERROR("Cannot allocate %d tls slots at %d alignment", TLS_NUM_SLOTS,
+                                  alignment);
+
+            report_dynamorio_problem(NULL, DUMPCORE_INTERNAL_EXCEPTION, NULL, NULL,
+                                     "Unrecoverable error on TLS allocation",
+                                     NULL, NULL, NULL);
+        }
+    }
+
+    ASSERT(res);
+    ASSERT(offs != TLS_UNINITIALIZED);
+    ASSERT_TRUNCATE(tls_local_state_offs, ushort, offs);
+    tls_local_state_offs = (ushort) offs;
+    LOG(GLOBAL, LOG_TOP, 1, "%d TLS slots are @ %s:0x%x\n",
+        TLS_NUM_SLOTS, IF_X64_ELSE("gs", "fs"), tls_local_state_offs);
+    ASSERT_CURIOSITY(proc_is_cache_aligned(get_local_state()) || 
+                     DYNAMO_OPTION(tls_align != 0));
+    tls_dcontext_offs = os_tls_offset(TLS_DCONTEXT_SLOT);
+    ASSERT(tls_dcontext_offs != TLS_UNINITIALIZED);
+
+    DOLOG(1, LOG_VMAREAS, { print_modules(GLOBAL, DUMP_NOT_XML); });
+    DOLOG(2, LOG_TOP, { print_mem_quota(); });
+    
+#ifdef WINDOWS_PC_SAMPLE
+    if (dynamo_options.profile_pcs)
+        init_global_profiles();
+#endif
+    
+#ifdef PROFILE_RDTSC
+    if (dynamo_options.profile_times) {
+        ASSERT_NOT_TESTED();
+        kilo_hertz = get_timer_frequency();
+        LOG(GLOBAL, LOG_TOP|LOG_STATS, 1, "CPU MHz is %d\n", kilo_hertz/1000);
+    }
+#endif
+
+    if (!dr_early_injected)
+        inject_init();
+
+    get_dynamorio_library_path(); 
+    /* just to preserve side effects. If not done yet in eventlog,
+     * path needs to be be preserved before hiding from module list.
+     */
+
+    aslr_init();
+
+    /* ensure static cache buffers are primed, both for .data protection purposes and
+     * because it may not be safe to get this information later */
+    get_own_qualified_name();
+    get_own_unqualified_name();
+    get_own_short_qualified_name();
+    get_own_short_unqualified_name();
+    get_application_short_name();
+    get_process_primary_SID();
+    get_process_SID_string();
+    get_process_owner_SID();
+    get_Everyone_SID();
+
+    /* avoid later .data-unprotection calls */
+    get_dynamorio_dll_preferred_base();
+    get_image_entry();
+    get_system_basic_info();
+    os_user_directory_supports_ownership();
+    is_wow64_process(NT_CURRENT_PROCESS);
+}
+
+static void
+print_mem_stats()
+{
+    VM_COUNTERS mem;
+    bool ok = get_process_mem_stats(NT_CURRENT_PROCESS, &mem);
+    ASSERT(ok);
+    LOG(GLOBAL, LOG_TOP, 1, "Process Memory Statistics:\n");
+    LOG(GLOBAL, LOG_TOP, 1, "\tPeak virtual size:         %6d KB\n",
+        mem.PeakVirtualSize/1024);
+    LOG(GLOBAL, LOG_TOP, 1, "\tPeak working set size:     %6d KB\n",
+        mem.PeakWorkingSetSize/1024);
+    LOG(GLOBAL, LOG_TOP, 1, "\tPeak paged pool usage:     %6d KB\n",
+        mem.QuotaPeakPagedPoolUsage/1024);
+    LOG(GLOBAL, LOG_TOP, 1, "\tPeak non-paged pool usage: %6d KB\n",
+        mem.QuotaPeakNonPagedPoolUsage/1024);
+    LOG(GLOBAL, LOG_TOP, 1, "\tPeak pagefile usage:       %6d KB\n",
+        mem.PeakPagefileUsage/1024);
+}
+
+/* os-specific atexit cleanup 
+ * note that this is called even on the fast exit release path so don't add
+ * uneccesary cleanup without ifdef DEBUG, but be careful with ifdef DEBUG's 
+ * also as Detach wants to leave nothing from us behind
+ * Called by dynamo_shared_exit() and the fast path in dynamo_process_exit(). 
+ */
+void
+os_fast_exit(void)
+{
+    /* make sure we never see an .exe that does all its work in
+     * DllMain()'s -- unlikely, even .NET apps have an image entry
+     * just to jump to mscoree
+     *
+     * The curiosity is relaxed for thin_client and hotp_only; if nothing else
+     * in the core is has run into this, then reached_image_entry doesn't have
+     * to be set for thin_client & hotp_only.  TODO: put in the image entry
+     * hook or not?
+     *
+     * The curiosity is also relaxed if we enter DR using the API
+     */
+    ASSERT_CURIOSITY(reached_image_entry_yet() ||
+                     RUNNING_WITHOUT_CODE_CACHE()
+                     IF_APP_EXPORTS( || dr_api_entry));
+
+    DOLOG(1, LOG_TOP, { print_mem_quota(); });
+    DOLOG(1, LOG_TOP, {
+        print_mem_stats();
+    });
+
+#ifdef WINDOWS_PC_SAMPLE
+    if (dynamo_options.profile_pcs) {
+        exit_global_profiles();
+        /* check to see if we are using the fast exit path
+         * if so dump profiles that were skipped */
+# ifndef DEBUG
+        if (dynamo_detaching_flag == LOCK_FREE_STATE) {
+            /* fast exit path, get remaining ungathered profile data */
+            if (dynamo_options.prof_pcs_gencode >= 2 && 
+                dynamo_options.prof_pcs_gencode <= 32) {
+                thread_record_t **threads;
+                int num, i;
+                /* get surviving threads */
+                arch_profile_exit();
+                mutex_lock(&thread_initexit_lock);
+                get_list_of_threads(&threads, &num);
+                for (i = 0; i < num; i++) {
+                    arch_thread_profile_exit(threads[i]->dcontext);
+                }
+                global_heap_free(threads, num*sizeof(thread_record_t*) 
+                                 HEAPACCT(ACCT_THREAD_MGT));
+                mutex_unlock(&thread_initexit_lock);
+            }
+            if (dynamo_options.prof_pcs_fcache >= 2 && 
+                dynamo_options.prof_pcs_fcache <= 32) {
+                /* note that fcache_exit() is called before os_fast_exit(),
+                 * we are here on fast exit path in which case fcache_exit() 
+                 * is not called
+                 */
+                fcache_profile_exit();
+            }
+            if (dynamo_options.prof_pcs_stubs >= 2 && 
+                dynamo_options.prof_pcs_stubs <= 32) {
+                special_heap_profile_exit();
+            }
+        }
+# endif
+        print_file(profile_file, "\nFinished dumping all profile info\n");
+        close_file(profile_file);
+    }
+#endif
+
+    eventlog_fast_exit();
+
+#ifdef DEBUG
+    module_info_exit();
+    DELETE_LOCK(snapshot_lock);
+#endif
+
+    /* case 10338: we don't free TLS on the fast path, in case there
+     * are other active threads: we don't want to synchall on exit so
+     * we let other threads run and try not to crash them until
+     * the process is terminated.
+     */
+
+    DELETE_LOCK(dump_core_lock);
+    DELETE_LOCK(debugbox_lock);
+
+    callback_exit();
+    ntdll_exit();
+}
+
+/* os-specific atexit cleanup since Detach wants to leave nothing from
+ * us behind.  In addition any debug cleanup should only be DODEBUG.
+ * Called by dynamo_shared_exit().
+ * Note it is expected to be called _after_ os_fast_exit().
+ */
+void
+os_slow_exit(void)
+{
+    /* free and zero thread-private storage (case 10338: slow path only) */
+    DEBUG_DECLARE(int res = )
+        tls_cfree(true/*need to synch*/, (uint) tls_local_state_offs,
+                  TLS_NUM_SLOTS);
+    ASSERT(res);
+
+    aslr_exit();
+    eventlog_slow_exit();
+}
+
+/* FIXME: what are good values here? */
+#define KILL_PROC_EXIT_STATUS -1
+#define KILL_THREAD_EXIT_STATUS -1
+
+byte *
+os_terminate_static_arguments(bool exit_process)
+{
+    byte *arguments;
+        
+    /* arguments for NtTerminate{Process,Thread} */
+    typedef struct _terminate_args_t {
+        union {
+            const byte *debug_code;
+            byte pad_bytes[SYSCALL_PARAM_MAX_OFFSET];
+        } padding;
+        struct {
+            IN HANDLE ProcessOrThreadHandle;
+            IN NTSTATUS ExitStatus;
+        } args;
+    } terminate_args_t;
+    /* It is not safe to use app stack and hope application will work.
+     * We need to stick the arguments for NtTerminate* in a place that
+     * doesn't exacerbate the problem - esp may have been in attacker's
+     * hands - so we place args in const static (read only) dr memory.
+     */
+    /* To facilitate detecting syscall failure for SYSENTER, we set a
+     * retaddr at edx (two extra slots there) as esp will be set to edx 
+     * by the kernel at the return from the sysenter. The kernel then sends
+     * control to a native ret which targets the debug infinite loop.
+     * (DEBUG only).
+     */
+    static const terminate_args_t term_thread_args = {
+        IF_DEBUG_ELSE_0((byte *)debug_infinite_loop), /* 0 -> NULL for release */
+        {NT_CURRENT_THREAD, KILL_THREAD_EXIT_STATUS}
+    };
+    static const terminate_args_t term_proc_args = {
+        IF_DEBUG_ELSE_0((byte *)debug_infinite_loop), /* 0 -> NULL for release */
+        {NT_CURRENT_PROCESS, KILL_PROC_EXIT_STATUS}
+    };
+    /* special sygate froms (non-const) */
+    static terminate_args_t sygate_term_thread_args = {
+        0, /* will be set to sysenter_ret_address */
+        {NT_CURRENT_THREAD, KILL_THREAD_EXIT_STATUS}
+    };
+    static terminate_args_t sygate_term_proc_args = {
+        0, /* will be set to sysenter_ret_address */
+        {NT_CURRENT_PROCESS, KILL_PROC_EXIT_STATUS}
+    };
+    
+    /* for LOG statement just pick proc vs. thread here, will adjust for
+     * offset below */
+    if (exit_process) {
+        if (DYNAMO_OPTION(sygate_sysenter) && 
+            get_syscall_method() == SYSCALL_METHOD_SYSENTER) {
+            byte *tgt = (byte *)&sygate_term_proc_args;
+            /* Note we overwrite every time we use this, but is ATOMIC and
+             * always with the same value so is ok */
+            SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+            ATOMIC_ADDR_WRITE(tgt, sysenter_ret_address, false);
+            DODEBUG({
+                    ATOMIC_ADDR_WRITE(tgt+sizeof(sysenter_ret_address),
+                                      (byte *)debug_infinite_loop, false);});
+            SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+            arguments = (byte *)&sygate_term_proc_args;
+        } else
+            arguments = (byte *)&term_proc_args;
+    } else {
+        if (DYNAMO_OPTION(sygate_sysenter) && 
+            get_syscall_method() == SYSCALL_METHOD_SYSENTER) {
+            byte *tgt = (byte *)&sygate_term_thread_args;
+            SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+            ATOMIC_ADDR_WRITE(tgt, sysenter_ret_address, false);
+            DODEBUG({
+                tgt += sizeof(sysenter_ret_address);
+                ATOMIC_ADDR_WRITE(tgt, (byte *)debug_infinite_loop, false);});
+            SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+            arguments = (byte *)&sygate_term_thread_args;
+        } else
+            arguments = (byte *)&term_thread_args;
+    }
+    
+    LOG(THREAD_GET, LOG_SYSCALLS, 2,
+        "Placing terminate arguments tombstone at "PFX" offset=0x%x\n", 
+        arguments, SYSCALL_PARAM_OFFSET());
+    
+    arguments += offsetof(terminate_args_t, args) - SYSCALL_PARAM_OFFSET();
+    return arguments;
+}
+
+/* dcontext is not needed for TERMINATE_PROCESS, so can pass NULL in
+ */
+void
+os_terminate(dcontext_t *dcontext, terminate_flags_t terminate_type)
+{
+    HANDLE currentThreadOrProcess = NT_CURRENT_PROCESS;
+    bool exit_process = true;
+    
+    ASSERT(TEST(TERMINATE_PROCESS, terminate_type) != /* xor */
+           TEST(TERMINATE_THREAD, terminate_type));
+
+    /* We could be holding the bb_building_lock at this point -- if we cleanup,
+     * we will get a rank order violation with all_threads_synch_lock.  if we
+     * unlock the bb lock, we risk an error about the non-owning thread
+     * releasing the lock.
+     * Our solution is for the caller to release it when possible --
+     * on an attack we know if we hold it or not.  But for other callers
+     * they do not know who holds it...for now we do nothing, none of them
+     * terminate just a thread, so the process is going down anyway, and it's
+     * just a debug build assertion :)
+     */
+
+    /* clean up may be dangerous - just terminate */
+    if (terminate_type == TERMINATE_PROCESS) {
+        /* skip synchronizing dynamic options, is risky and caller has almost 
+         * certainly already done so for a syslog */
+        if (TESTANY(DETACH_ON_TERMINATE|DETACH_ON_TERMINATE_NO_CLEAN, 
+                    DYNAMO_OPTION(internal_detach_mask))) {
+            /* FIXME : if we run into stack problems we could reset the stack
+             * here though caller has likely alredy gone as deep as detach 
+             * will since almost everyone SYSLOG's before calling this */
+            detach_helper(TEST(DETACH_ON_TERMINATE_NO_CLEAN, 
+                               DYNAMO_OPTION(internal_detach_mask)) ? 
+                          DETACH_BAD_STATE_NO_CLEANUP : DETACH_BAD_STATE);
+            /* skip option synch, make this as safe as possible */
+            SYSLOG_INTERNAL_NO_OPTION_SYNCH(SYSLOG_WARNING, 
+                                            "detach on terminate failed or already started by another thread, killing thread %d\n", 
+                                            get_thread_id());
+            /* if we get here, either we recursed or someone is already trying
+             * to detach, just kill this thread so progress is made we don't
+             * have anything better to do with it */
+            /* skip cleanup, our state is likely messed up and we'd just like 
+             * to get out alive, also avoids recursion problems, see caveat at 
+             * remove_thread below */
+            terminate_type = TERMINATE_THREAD;
+        } else {
+            nt_terminate_process(currentThreadOrProcess, KILL_PROC_EXIT_STATUS);
+            ASSERT_NOT_REACHED();
+        }
+    }
+
+    /* CHECK: Can a process disallow PROCESS_TERMINATE or THREAD_TERMINATE 
+       access even to itself? 
+    */
+    if (TEST(TERMINATE_THREAD, terminate_type)) {
+        exit_process = (get_num_threads() == 1 && !dynamo_exited);
+        if (!exit_process) {
+            currentThreadOrProcess = NT_CURRENT_THREAD;
+        } 
+    }
+    
+    STATS_INC(num_threads_killed);
+    if (TEST(TERMINATE_CLEANUP, terminate_type)) {
+        byte *arguments = os_terminate_static_arguments(exit_process);
+
+        /* Make sure debug loop pointer is in expected place since this makes
+         * assumptions about offsets.  We don't use the debug loop pointer for
+         * int2e/syscall/wow64 system calls (since they return to the invocation
+         * and can be handled there).  For SYSENTER the SYSCALL_PARAM_OFFSET should
+         * match up with arguments such that arguments is pointing to debugme */
+        ASSERT(does_syscall_ret_to_callsite() ||
+               *(byte **)arguments == (byte *)&debug_infinite_loop ||
+               (DYNAMO_OPTION(sygate_sysenter) && 
+                *(((byte **)arguments)+1) == (byte *)&debug_infinite_loop));
+      
+        STATS_INC(num_threads_killed_cleanly);
+
+        /* we enter from several different places, so rewind until top-level kstat */
+        KSTOP_REWIND_UNTIL(thread_measured);
+
+        /* now we issue a syscall by number */
+        /* we can't use issue_system_call_for_app because it relies on 
+         * dstack that we should release */
+        /* FIXME: what happens now if we get some callbacks that are still on 
+         * their way? Shouldn't happen since Terminate* are believed to be 
+         * non-alertable. */
+        /* FIXME: we only want the last part of cleanup_and_terminate */
+        ASSERT(dcontext != NULL);
+        cleanup_and_terminate
+            (dcontext, 
+             syscalls[exit_process ? SYS_TerminateProcess : SYS_TerminateThread],
+             (ptr_uint_t)
+             IF_X64_ELSE((exit_process ? NT_CURRENT_PROCESS : NT_CURRENT_THREAD),
+                         arguments),
+             (ptr_uint_t)
+             IF_X64_ELSE((exit_process ? KILL_PROC_EXIT_STATUS:KILL_THREAD_EXIT_STATUS),
+                         arguments /* no 2nd arg, just a filler */),
+             exit_process);
+    } else {
+        /* may have decided to terminate process */
+        if (exit_process) {
+            nt_terminate_process(currentThreadOrProcess, KILL_PROC_EXIT_STATUS);
+            ASSERT_NOT_REACHED();
+        } else {
+            /* FIXME: this is now very dangerous - we even leave our own state */
+            /* we should at least remove this thread from the all threads list 
+             * to avoid synchronizing issues, though we are running the risk of
+             * an infinite loop with a failure in this function and detach on 
+             * failure */
+            if (all_threads != NULL)
+                remove_thread(NULL, get_thread_id());
+            nt_terminate_thread(currentThreadOrProcess, KILL_THREAD_EXIT_STATUS);
+            ASSERT_NOT_REACHED();
+        }
+
+        /* CHECK: who is supposed to clean up the thread's stack?  
+           ZwFreeVirtualMemory can be called by another thread
+           waiting on the thread object, hopefully someone will do it */
+    }
+    ASSERT_NOT_REACHED();
+}
+
+void
+os_tls_init()
+{
+    /* everything was done in os_init, even TEB TLS slots are initialized to 0 for us */
+}
+
+void
+os_tls_exit(local_state_t *local_state)
+{
+    /* not needed for windows, everything is done is os_slow_exit including zeroing
+     * the freed TEB tls slots */
+}
+
+void
+os_thread_init(dcontext_t *dcontext)
+{
+    NTSTATUS res;
+    DEBUG_DECLARE(bool ok;)
+    os_thread_data_t *ostd = (os_thread_data_t *)
+        heap_alloc(dcontext, sizeof(os_thread_data_t) HEAPACCT(ACCT_OTHER));
+    dcontext->os_field = (void *) ostd;
+    /* init ostd fields here */
+    ostd->stack_base = NULL;
+    ostd->stack_top = NULL;
+    ostd->teb_stack_no_longer_valid = false;
+    DEBUG_DECLARE(ok = )get_stack_bounds(dcontext, NULL, NULL);
+    ASSERT(ok);
+    
+    /* case 8721: save the win32 start address and print it in the ldmp */
+    res = query_win32_start_addr(NT_CURRENT_THREAD, &dcontext->win32_start_addr);
+    if (!NT_SUCCESS(res)) {
+        ASSERT(false && "failed to obtain win32 start address");
+        dcontext->win32_start_addr = (app_pc)0;
+    } else {
+        LOG(THREAD, LOG_THREADS, 2, "win32 start addr is "PFX"\n",
+            dcontext->win32_start_addr);
+    }
+    aslr_thread_init(dcontext);
+}
+
+void
+os_thread_exit(dcontext_t *dcontext)
+{
+    os_thread_data_t *ostd = (os_thread_data_t *) dcontext->os_field;
+    aslr_thread_exit(dcontext);
+#ifdef DEBUG
+    /* for non-debug we do fast exit path and don't free local heap */
+    /* clean up ostd fields here */
+    heap_free(dcontext, ostd, sizeof(os_thread_data_t) HEAPACCT(ACCT_OTHER));
+#endif
+}
+
+void
+os_thread_stack_exit(dcontext_t *dcontext)
+{
+    os_thread_data_t *ostd = (os_thread_data_t *) dcontext->os_field;
+    ASSERT_OWN_MUTEX(true, &thread_initexit_lock);
+    /* see case 3768: a thread's stack is not de-allocated by this process,
+     * so we remove its stack from our executable region here
+     * ref also case 5518 where it is sometimes freed in process, we watch for
+     * that and set stack_base to NULL
+     * note: thin_client doesn't have executable or aslr areas, so this is moot.
+     */
+    if (DYNAMO_OPTION(thin_client))
+        return;
+
+    if (ostd->stack_base != NULL) {
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+            "os_thread_stack_exit : removing "PFX" - "PFX"\n",
+            ostd->stack_base, ostd->stack_top);
+
+        ASSERT(ostd->stack_top != NULL);
+        DODEBUG({
+            /* ASSERT that os region matches region stored in ostd */
+            byte *alloc_base;
+            size_t size = get_allocation_size(ostd->stack_base, &alloc_base);
+            /* Xref case 5877, this assert can fire if the exiting thread has already
+             * exited (resulting in freed stack) before we clean it up.  This could be do
+             * to using THREAD_SYNCH_TERMINATED_AND_CLEANED with a synch_with* routine
+             * (no current uses) or a race with detach resuming a translated thread
+             * before cleaning it up.  The detach race is harmless so we allow it. */
+            ASSERT(doing_detach ||
+                   ((size == (size_t)(ostd->stack_top - ostd->stack_base) ||
+                     /* PR 252008: for WOW64 nudges we allocate an extra page. */
+                     (size == PAGE_SIZE + (size_t)(ostd->stack_top - ostd->stack_base) &&
+                      is_wow64_process(NT_CURRENT_PROCESS) &&
+                      dcontext->nudge_target != NULL)) &&
+                    ostd->stack_base == alloc_base));
+        });
+        /* believe <= win2k frees the stack in process, would like to check
+         * that but we run into problems with stacks that are never freed
+         * (TerminateThread, threads killed by TerminateProcess 0, last thread
+         * calling TerminateProcess, etc.) FIXME figure out way to add an
+         * assert_curiosity */
+        /* make sure we use our dcontext (dcontext could belong to another thread
+         * from other_thread_exit) since flushing will end up using this dcontext
+         * for synchronization purposes */
+        app_memory_deallocation(get_thread_private_dcontext(), ostd->stack_base,
+                                ostd->stack_top - ostd->stack_base,
+                                true /* own thread_initexit_lock */,
+                                false /* not image */);
+        if (TEST(ASLR_HEAP_FILL, DYNAMO_OPTION(aslr))) {
+            size_t stack_reserved_size = ostd->stack_top - ostd->stack_base;
+            /* verified above with get_allocation_size() this is not only the committed portion */
+            aslr_pre_process_free_virtual_memory(dcontext, ostd->stack_base, 
+                                                 stack_reserved_size);
+        }
+    } else {
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+            "os_thread_stack_exit : Thread's os stack has alread been freed\n");
+        /* believe >= XP free the stack out of process */
+        ASSERT(ostd->stack_top == NULL);
+        ASSERT_CURIOSITY(get_os_version() <= WINDOWS_VERSION_2000);
+    }
+}
+
+void
+os_thread_under_dynamo(dcontext_t *dcontext)
+{
+    /* add cur thread to callback list */
+    set_asynch_interception(get_thread_id(), true);
+}
+
+void
+os_thread_not_under_dynamo(dcontext_t *dcontext)
+{
+    /* remove cur thread from callback list */
+    set_asynch_interception(get_thread_id(), false);
+}
+
+#ifdef CLIENT_SIDELINE /* PR 222812: tied to sideline usage */
+/* We have two choices for the stack:
+ * 1) Hide it from user, allocate a 1-or-2-page stack to get to the apc
+ *    dispatcher and swap to our dstack, which the client uses; then free
+ *    the dstack using existing mechanisms, and let the small stack leak
+ *    or eventually free it if worth the effort.
+ * 2) Expose the stack size to the user and do not create a dcontext
+ *    or a dstack.  Then we need a custom stack de-allocation.
+ * I'm going with #2.  dr_terminate_client_thread() de-allocates.
+ * 
+ * FIXME PR 210591: transparency issues:
+ * 1) All dlls will be notifed of thread creation by DLL_THREAD_ATTACH
+ * 2) The thread will show up in the list of threads accessed by
+ *    NtQuerySystemInformation's SystemProcessesAndThreadsInformation structure.
+ *
+ * FIXME PR 202669: if the client leaves reservation space we should have
+ * the stack auto-expand.
+ */
+DR_API bool
+dr_create_client_thread(void (*func)(void *param), void *arg,
+                        size_t stack_reserve, size_t stack_commit)
+{
+    HANDLE hthread;
+    bool res;
+    thread_id_t tid;
+    CLIENT_ASSERT(stack_reserve >= stack_commit,
+                  "stack_reserve must be >= stack_commit");
+    /* FIXME PR 225714: does this work on Vista? */
+    hthread = create_thread(NT_CURRENT_PROCESS, (void *)func, arg, NULL, 0,
+                            stack_reserve, stack_commit, false, &tid);
+    CLIENT_ASSERT(hthread != INVALID_HANDLE_VALUE, "error creating thread");
+    if (hthread == INVALID_HANDLE_VALUE)
+        return false;
+    /* mark the thread now as a native thread */
+    add_thread(hthread, tid, false/*!under_dynamo_control*/, NULL/*no dcontext*/);
+    /* FIXME: what about all of our check_sole_thread() checks? */
+    res = close_handle(hthread);
+    CLIENT_ASSERT(res, "error closing thread handle");
+    return res;        
+}
+
+DR_API bool
+dr_terminate_client_thread(void)
+{
+    thread_id_t tid = get_thread_id();
+    thread_record_t *tr = thread_lookup(tid);
+    byte *stack_base;
+    byte *term_args;
+    if (tr == NULL || tr->dcontext != NULL || tr->under_dynamo_control) {
+        CLIENT_ASSERT(false, "dr_terminate_client_thread called on non-client thread");
+        return false;
+    }
+    /* We don't have a dcontext+dstack to clean up so we do not call os_terminate;
+     * we simply do the necessary cleanup here.
+     */
+    /* increment _exiting_thread_count so that we don't get killed after 
+     * we're off the all_threads list */
+    ATOMIC_INC(int, exiting_thread_count);
+    remove_thread(NT_CURRENT_THREAD, tid);
+    /* Once we're using CreateThreadEx, for get_os_version() >= WINDOWS_VERSION_VISTA
+     * the kernel will clean up the stack for us and we can directly call
+     * nt_terminate_thread here.
+     */
+    get_stack_bounds(NULL, &stack_base, NULL);
+    term_args = os_terminate_static_arguments(false/*thread only*/);
+    cleanup_and_terminate_client_thread
+        (syscalls[SYS_TerminateThread], stack_base,
+         IF_X64_ELSE(NT_CURRENT_THREAD, (ptr_uint_t)term_args),
+         IF_X64_ELSE(KILL_THREAD_EXIT_STATUS, (ptr_uint_t)term_args/*filler*/));
+    ASSERT_NOT_REACHED();
+    return true; /* satisfy compiler */
+}
+#endif CLIENT_SIDELINE /* PR 222812: tied to sideline usage */
+
+int
+get_os_version()
+{
+    return os_version;
+}
+
+bool
+is_in_dynamo_dll(app_pc pc)
+{
+    ASSERT(dynamo_dll_start != NULL && dynamo_dll_end != NULL);
+    return (pc >= dynamo_dll_start && pc < dynamo_dll_end);
+}
+
+static char *
+mem_state_string(uint state)
+{
+    switch (state) {
+    case 0:            return "none";
+    case MEM_COMMIT:   return "COMMIT";
+    case MEM_FREE:     return "FREE";
+    case MEM_RESERVE:  return "RESERVE";
+    }
+    return "(error)";
+}
+
+static char *
+mem_type_string(uint type)
+{
+    switch (type) {
+    case 0:            return "none";
+    case MEM_IMAGE:    return "IMAGE";
+    case MEM_MAPPED:   return "MAPPED";
+    case MEM_PRIVATE:  return "PRIVATE";
+    }
+    return "(error)";
+}
+
+char *
+prot_string(uint prot)
+{
+    uint ignore_extras = prot & ~PAGE_PROTECTION_QUALIFIERS;
+    switch (ignore_extras) {
+    case PAGE_NOACCESS:          return "----";
+    case PAGE_READONLY:          return "r---";
+    case PAGE_READWRITE:         return "rw--";
+    case PAGE_WRITECOPY:         return "rw-c";
+    case PAGE_EXECUTE:           return "--x-";
+    case PAGE_EXECUTE_READ:      return "r-x-";
+    case PAGE_EXECUTE_READWRITE: return "rwx-";
+    case PAGE_EXECUTE_WRITECOPY: return "rwxc";
+    }
+    return "(error)";
+}
+
+static bool
+prot_is_readable(uint prot) 
+{
+    prot &= ~PAGE_PROTECTION_QUALIFIERS;
+    /* FIXME: consider just E to be unreadable?
+     * do not do exclusions, sometimes prot == 0 or something
+     */
+    switch (prot) {
+    case PAGE_READONLY:         
+    case PAGE_READWRITE:        
+    case PAGE_WRITECOPY:        
+    case PAGE_EXECUTE:          
+    case PAGE_EXECUTE_READ:     
+    case PAGE_EXECUTE_READWRITE:
+    case PAGE_EXECUTE_WRITECOPY: return true;
+    }
+    return false;
+}
+
+bool
+prot_is_writable(uint prot) 
+{
+    prot &= ~PAGE_PROTECTION_QUALIFIERS;
+    return (prot == PAGE_READWRITE || prot == PAGE_WRITECOPY ||
+            prot == PAGE_EXECUTE_READWRITE || prot == PAGE_EXECUTE_WRITECOPY);
+}
+
+bool
+prot_is_executable(uint prot) 
+{
+    prot &= ~PAGE_PROTECTION_QUALIFIERS;
+    return (prot == PAGE_EXECUTE || prot == PAGE_EXECUTE_READ ||
+            prot == PAGE_EXECUTE_READWRITE || prot == PAGE_EXECUTE_WRITECOPY);
+}
+
+/* true when page hasn't been written to */
+bool
+prot_is_copyonwrite(uint prot) 
+{
+    prot &= ~PAGE_PROTECTION_QUALIFIERS;
+    /* although really providing an enumeration, the known PAGE_
+     * values use separate bit flags.  We use TESTANY in case new
+     * PAGE_PROTECTION_QUALIFIERS show up.
+     */
+    return TESTANY(PAGE_WRITECOPY|PAGE_EXECUTE_WRITECOPY, prot);
+}
+
+/* translate platform independent protection bits to native flags */
+int
+memprot_to_osprot(uint prot)
+{
+    uint os_prot = 0;
+    if (TEST(MEMPROT_EXEC, prot)) {
+        ASSERT(TEST(MEMPROT_READ, prot));
+        if (TEST(MEMPROT_WRITE, prot))
+            os_prot = PAGE_EXECUTE_READWRITE;
+        else
+            os_prot = PAGE_EXECUTE_READ;
+    } else if (TEST(MEMPROT_READ, prot)) {
+        if (TEST(MEMPROT_WRITE, prot))
+            os_prot = PAGE_READWRITE;
+        else
+            os_prot = PAGE_READONLY;
+    } else
+        os_prot = PAGE_NOACCESS;
+    return os_prot;
+}
+
+/* translate native flags to platform independent protection bits */
+int
+osprot_to_memprot(uint prot)
+{
+    uint mem_prot = 0;
+    if (prot_is_readable(prot))
+        mem_prot |= MEMPROT_READ;
+    if (prot_is_writable(prot))
+        mem_prot |= MEMPROT_WRITE;
+    if (prot_is_executable(prot))
+        mem_prot |= MEMPROT_EXEC;
+    return mem_prot;
+}
+
+static int
+osprot_add_writecopy(uint prot)
+{
+    int pr = prot & ~PAGE_PROTECTION_QUALIFIERS;
+    switch (pr) {
+    case PAGE_READWRITE: return (prot & (~pr)) | PAGE_WRITECOPY;
+    case PAGE_EXECUTE_READWRITE: return (prot & (~pr)) | PAGE_EXECUTE_WRITECOPY;
+    default: ASSERT_NOT_REACHED();
+    }
+    return prot;
+}
+
+/* returns osprot flags preserving all native protection flags except
+ * for RWX, which are replaced according to memprot */
+uint
+osprot_replace_memprot(uint old_osprot, uint memprot)
+{
+    uint old_qualifiers = 
+        old_osprot & PAGE_PROTECTION_QUALIFIERS;
+    uint new_osprot = memprot_to_osprot(memprot);
+
+    /* preserve any old WRITECOPY 'flag' if page hasn't been touched */
+    if (prot_is_copyonwrite(old_osprot) && prot_is_writable(new_osprot))
+        new_osprot = osprot_add_writecopy(new_osprot);
+    new_osprot |= old_qualifiers;
+    return new_osprot;
+}
+
+void
+dump_mbi(file_t file, MEMORY_BASIC_INFORMATION *mbi, bool dump_xml)
+{
+    print_file(file, dump_xml ?
+               "\t\tBaseAddress=         \""PFX"\"\n"
+               "\t\tAllocationBase=      \""PFX"\"\n"
+               "\t\tAllocationProtect=   \"0x%08x %s\"\n"
+               "\t\tRegionSize=          \"0x%08x\"\n"
+               "\t\tState=               \"0x%08x %s\"\n"
+               "\t\tProtect=             \"0x%08x %s\"\n"
+               "\t\tType=                \"0x%08x %s\"\n"
+               :
+               "BaseAddress:       "PFX"\n"
+               "AllocationBase:    "PFX"\n"
+               "AllocationProtect: 0x%08x %s\n"
+               "RegionSize:        0x%08x\n"
+               "State:             0x%08x %s\n"
+               "Protect:           0x%08x %s\n"
+               "Type:              0x%08x %s\n",
+               mbi->BaseAddress,
+               mbi->AllocationBase,
+               mbi->AllocationProtect, prot_string(mbi->AllocationProtect),
+               mbi->RegionSize,
+               mbi->State, mem_state_string(mbi->State),
+               mbi->Protect, prot_string(mbi->Protect),
+               mbi->Type, mem_type_string(mbi->Type));
+}
+
+void
+dump_mbi_addr(file_t file, app_pc target, bool dump_xml)
+{
+    MEMORY_BASIC_INFORMATION mbi;
+    size_t len;
+    len = query_virtual_memory(target, &mbi, sizeof(mbi));
+    if (len == sizeof(mbi))
+        dump_mbi(file, &mbi, dump_xml);
+    else {
+        if (dump_xml) {
+            print_file(file, "<-- Unable to dump mbi for addr "PFX"\n -->",
+                       target);
+        } else {
+            print_file(file, "Unable to dump mbi for addr "PFX"\n", target);
+        }
+    }
+}
+
+/* FIXME:
+ * We need to be able to distinguish our own pid from that of a child
+ * process.  We observe that after CreateProcess a child has pid of 0 (as
+ * determined by process_id_from_handle, calling NtQueryInformationProcess).
+ * For our current injection methods pid is always set when we take over,
+ * but for future early-injection methods what if the pid is still 0 when
+ * we start executing in the process' context?
+ */
+bool
+is_pid_me(process_id_t pid)
+{
+    return (pid ==  get_process_id());
+}
+
+bool
+is_phandle_me(HANDLE phandle)
+{
+    /* make the common case of NT_CURRENT_PROCESS faster */
+    if (phandle == NT_CURRENT_PROCESS) {
+        return true;
+    } else {
+        /* we know of no way to detect whether two handles point to the same object,
+         * so we go to pid
+         */
+        process_id_t pid = process_id_from_handle(phandle);
+        return is_pid_me(pid);
+    }
+}
+
+/* used only in get_dynamorio_library_path() but file level namespace
+ * so it is easily available to windbg scripts */
+static char dynamorio_library_path[MAXIMUM_PATH];
+
+/* get full path to our own library, (cached), used for forking and message file name */
+char*
+get_dynamorio_library_path()
+{
+    /* This operation could be dangerous, so it is still better that we do it
+     *  once at startup when there is a single thread only 
+     */
+    if (!dynamorio_library_path[0]) { /* not cached */
+        /* get_module_name can take any pc in the dll,
+         * so we simply take the address of this function
+         * instead of using get_module_handle to find the base
+         */
+        app_pc pb = (app_pc)&get_dynamorio_library_path;
+
+        /* here's where we set the library path */
+        get_module_name(pb, dynamorio_library_path, MAXIMUM_PATH);
+    }
+    return dynamorio_library_path;
+}
+
+/* based on a process handle to a process that is not yet running,
+ * verify whether we should be taking control over it */
+/* if target process should be injected into returns true, and
+ * inject_settings is set if non-NULL */
+bool
+should_inject_into_process(dcontext_t *dcontext, HANDLE process_handle,
+                           int *rununder_mask, /* OPTIONAL OUT */
+                           inject_setting_mask_t *inject_settings /* OPTIONAL OUT */)
+{
+    bool inject = false;
+    synchronize_dynamic_options();
+    if (DYNAMO_OPTION(follow_children) || DYNAMO_OPTION(follow_explicit_children) ||
+        DYNAMO_OPTION(follow_systemwide)) {
+
+        inject_setting_mask_t should_inject = 
+            systemwide_should_inject(process_handle, rununder_mask);
+
+        if (DYNAMO_OPTION(follow_systemwide) && TEST(INJECT_TRUE, should_inject)) {
+            LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, 1,
+                "\tconfigured child should be injected\n");
+            inject = true;
+        }
+
+        if (!inject && DYNAMO_OPTION(follow_explicit_children) &&
+            TESTALL(INJECT_EXPLICIT|INJECT_TRUE, should_inject)) {
+            LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, 1,
+                "\texplicit child should be injected\n");
+            inject = true;
+        }
+
+        if (!inject && DYNAMO_OPTION(follow_children)) {
+            inject = true; /* -follow_children defaults to inject */
+
+            /* check if child should be excluded from running under dr */
+            if (TEST(INJECT_EXCLUDED, should_inject)) {
+                LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, 1,
+                    "\tchild is excluded, not injecting\n");
+                inject = false;
+            }
+
+            /* check if we should leave injection to preinjector */
+            if (TEST(INJECT_TRUE, should_inject) && systemwide_inject_enabled() &&
+                !TEST(INJECT_EXPLICIT, should_inject)) {
+                ASSERT(!DYNAMO_OPTION(follow_systemwide));
+                LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, 1,
+                    "\tletting preinjector inject into child\n");
+                inject = false;
+            }
+
+            DODEBUG({
+                if (inject)
+                    LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, 1,
+                        "\tnon-excluded, non-preinjected child should be injected\n");
+            });
+        }
+        if (inject) {
+            ASSERT(!TEST(INJECT_EXCLUDED, should_inject));
+            if (inject_settings != NULL)
+                *inject_settings = should_inject;
+        }
+    }
+    DODEBUG({
+        if (inject) {
+            LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, 1,
+                "\tinjecting into child process\n");
+            
+        } else {
+            LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, 1, "\tletting child execute natively "
+                "(may still get injected by systemwide injector!)\n");
+        }
+    });
+    return inject;
+}
+
+/* cxt may be NULL if -inject_at_create_process */
+static int
+inject_into_process(dcontext_t *dcontext, HANDLE process_handle, CONTEXT *cxt,
+                    inject_setting_mask_t should_inject)
+{
+    /* Here in fact we don't want to have the default argument override
+       mechanism take place.  If an app specific AUTOINJECT value is
+       provided, then we should of course use it.  However, if no
+       specific one is given we should not use the global default when
+       follow_children.  For follow_explicit_children it is actually OK
+       to use the global default value, it will be the GUI's
+       responsibility to set both the parent and child if it is desired
+       to have them use the same library.
+    */
+    char  library_path_buf[MAXIMUM_PATH];
+    char *library = library_path_buf;
+    bool res;
+
+    int err = get_process_parameter(process_handle, L_DYNAMORIO_VAR_AUTOINJECT,
+                                    library_path_buf, sizeof(library_path_buf));
+
+    /* If there is no app-specific subkey, then we should check in what mode are we injecting */
+    /* If we are in fact in follow_children - meaning all children are followed, 
+       and there is no app specific option then we should use the parent library,
+       unless the child is in fact explicit in which case we just use the global library.
+    */
+
+    switch (err) {
+    case GET_PARAMETER_SUCCESS:
+        break;
+    case GET_PARAMETER_NOAPPSPECIFIC:
+        /* We got the global key's library, use parent's library instead if the only
+         * reason we're injecting is -follow_children (i.e. reading RUNUNDER gave us 
+         * !INJECT_TRUE). */
+        if (!TEST(INJECT_TRUE, should_inject)) {
+            ASSERT(DYNAMO_OPTION(follow_children));
+            library = get_dynamorio_library_path();
+        }
+        break;
+    case GET_PARAMETER_BUF_TOO_SMALL:
+    case GET_PARAMETER_FAILURE:
+        library = get_dynamorio_library_path();
+        break;
+    default:
+        ASSERT_NOT_REACHED();
+    }
+                   
+    LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, 1, "\tinjecting %s into child process\n", library);
+    
+    if (DYNAMO_OPTION(aslr_dr) &&
+        /* case 8749 - can't aslr dr for thin_clients */
+        process_handle != NULL && !is_child_in_thin_client(process_handle)) {
+        aslr_force_dynamorio_rebase(process_handle);
+    }
+
+    /* Can't early inject 32-bit DR into a wow64 process as there is no
+     * ntdll32.dll at early inject point, so thread injection only.  PR 215423.
+     */
+    if (DYNAMO_OPTION(early_inject) && !is_wow64_process(process_handle)) {
+        ASSERT(early_inject_address != NULL);
+        /* FIXME if early_inject_address == NULL then early_inject_init failed
+         * to find the correct address to use.  Don't expect that to happen,
+         * but if it does could fall back to late injection (though we can't
+         * be sure that would work, i.e. early thread process for ex.) or
+         * do a SYSLOG error. */
+        res = inject_into_new_process(process_handle, library,
+                                      DYNAMO_OPTION(early_inject_map),
+                                      early_inject_location,
+                                      early_inject_address);
+    } else {
+        ASSERT(cxt != NULL);
+        res = inject_into_thread(process_handle, cxt, NULL, library);
+    }
+    
+    if (!res) {
+        SYSLOG_INTERNAL_ERROR("ERROR: injection into child process failed");
+        ASSERT_NOT_REACHED();
+        return false;       /* for compilation correctness and release builds */
+    }
+    return true;
+}
+
+bool
+is_first_thread_in_new_process(HANDLE process_handle, CONTEXT *cxt)
+{
+    /* ASSUMPTION: based on what I've seen, on win2k a new process has 
+     * pid 0 until its first thread is created.  This is not true on XP
+     * so we also check if the argument value is the PEB address 
+     * (which it should be if it is the first thread in the process,
+     * according to inside win2k).  This is a slight risk of double
+     * or late injection if someone creates a remote thread that 
+     * happens to have an argument that equals the address of PEB.  
+     * Better would be able to tell from Eip if it is pointing at the 
+     * kernel32 thread start thunk or the kernel32 process start thunk,
+     * or to check if the number of threads in the process equals 0, 
+     * but no easy way to do either here.  FIXME 
+     */
+    process_id_t pid = process_id_from_handle(process_handle);
+    return (pid == 0 || 
+            (!is_pid_me(pid) && cxt->CXT_XBX == (ptr_uint_t)get_peb(process_handle)));
+}
+
+/* Depending on registry and options maybe inject into child process with
+ * handle process_handle.  Called by SYS_CreateThread in pre_system_call (in
+ * which case cxt is non-NULL) and by CreateProcess[Ex] in post_system_call (in
+ * which case cxt is NULL). */
+void
+maybe_inject_into_process(dcontext_t *dcontext, HANDLE process_handle,
+                          CONTEXT *cxt)
+{
+    /* if inject_at_create_process becomes dynamic, need to move this check below
+     * the synchronize dynamic options */
+    /* FIXME - can't read process parameters, at process create time is NULL
+     * value in peb field except in Vista.  Could pass it in. */
+    /* Can't early inject 32-bit DR into a wow64 process as there is no
+     * ntdll32.dll at early inject point, so thread injection only.  PR 215423.
+     */
+    if ((cxt == NULL && (DYNAMO_OPTION(inject_at_create_process) ||
+                         (get_os_version() == WINDOWS_VERSION_VISTA && 
+                          DYNAMO_OPTION(vista_inject_at_create_process)))
+         && !is_wow64_process(process_handle)) ||
+        (cxt != NULL && is_first_thread_in_new_process(process_handle, cxt))) {
+        int rununder_mask;
+        inject_setting_mask_t should_inject;
+        /* Creating a new process & at potential inject point */
+        DEBUG_DECLARE(process_id_t pid = process_id_from_handle(process_handle);)
+        DOLOG(3, LOG_SYSCALLS|LOG_THREADS, {
+            SYSLOG_INTERNAL_INFO("found a fork: pid %d", pid);
+        });
+        LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, 1, "found a fork: pid %d\n", pid);
+
+        if (should_inject_into_process(dcontext, process_handle,
+                                       &rununder_mask, &should_inject)) {
+            ASSERT(cxt != NULL || DYNAMO_OPTION(early_inject));
+            /* FIXME : if not -early_inject, we are going to read and write
+             * to cxt, which may be unsafe */
+            if (inject_into_process(dcontext, process_handle, cxt,
+                                    should_inject)) {
+                check_for_run_once(process_handle, rununder_mask);
+            }
+        }
+    }
+}
+
+/* For case 8749: can't aslr dr for thin_client because cygwin apps will die. */
+static bool
+is_child_in_thin_client(HANDLE process_handle)
+{
+    bool res;
+    const options_t *opts;
+
+    /* Shouldn't be using this for the current process. */
+    ASSERT(process_handle != NT_CURRENT_PROCESS && 
+           process_handle != NT_CURRENT_THREAD && process_handle != NULL);
+
+    opts = get_process_options(process_handle);
+    ASSERT_OWN_READWRITE_LOCK(true, &options_lock);
+    ASSERT(opts != NULL);
+
+    /* In this case the option is used only for preventing aslr_dr, so be safe
+     * if you can't read it and say yes which will prevent aslr dr.  Note: this
+     * isn't the secure option, which is to say no, so that we alsr dr.
+     * Interesting tradeoff; choosing safety as this scenario is rare in which
+     * case first goal is to do no harm.
+     */
+    if (opts == NULL) {
+        res = false;
+    } else {
+        res = opts->thin_client;
+    }
+    write_unlock(&options_lock);
+    return res;
+}
+
+app_pc
+get_dynamorio_dll_start() {
+    if (dynamo_dll_start == NULL)
+        dynamo_dll_start = get_allocation_base((app_pc) get_dynamorio_dll_start);
+    return dynamo_dll_start;
+}
+
+app_pc
+get_dynamorio_dll_preferred_base(void)
+{
+    if (dynamo_dll_preferred_base == NULL) {
+        dynamo_dll_preferred_base = get_module_preferred_base(get_dynamorio_dll_start());
+        ASSERT(dynamo_dll_preferred_base != NULL);
+    }
+    return dynamo_dll_preferred_base;
+}
+
+/* IF_X64(ASSERT_NOT_IMPLEMENTED(false)) -- need to update */
+static app_pc highest_user_address = (app_pc)(ptr_uint_t)0x7ffeffff;
+/* 0x7ffeffff on 2GB:2GB default */
+/* or 0xbffeffff with /3GB in boot.ini, */
+/* /userva switch may also change the actual value seen */
+
+static void
+get_system_basic_info(void)
+{
+    SYSTEM_BASIC_INFORMATION sbasic_info;
+
+    NTSTATUS result = query_system_info(SystemBasicInformation, 
+                                        sizeof(SYSTEM_BASIC_INFORMATION), 
+                                        &sbasic_info);
+    ASSERT(NT_SUCCESS(result));
+    highest_user_address = (app_pc)sbasic_info.HighestUserAddress;
+    /* typically we have 2GB:2GB split between user and kernel virtual memory 
+     * lkd> dd nt!MmUserProbeAddress  l1
+     *  8055ee34  7fff0000
+     * lkd> dd nt!MmHighestUserAddress  l1
+     *  8055ee3c  7ffeffff
+     */
+
+    LOG(GLOBAL, LOG_VMAREAS, 1, "get_system_basic_info: "
+        "HighestUserAddress "PFX"\n", highest_user_address);
+
+    /* for testing purposes we can pretend all other addresses are inaccessible */
+    if (INTERNAL_OPTION(stress_fake_userva) != 0) {
+        if (highest_user_address > (app_pc)INTERNAL_OPTION(stress_fake_userva)) {
+            highest_user_address = (app_pc)INTERNAL_OPTION(stress_fake_userva);
+            SYSLOG_INTERNAL_WARNING("using stress_fake_userva "PFX"\n", 
+                                    highest_user_address);
+        } else {
+            ASSERT_CURIOSITY("useless stress_fake_userva");
+        }
+    }
+
+    ASSERT(OS_ALLOC_GRANULARITY == sbasic_info.AllocationGranularity);
+}
+
+bool
+is_user_address(app_pc pc)
+{
+    /* we don't worry about LowestUserAddress which is the first 64KB
+     * page which should normally be invalid.
+     *
+     * FIXME: case 10899 although users can in fact allocate in the
+     * NULL allocation region (by using base=1), as typically done in
+     * a local NULL pointer attack.  Natively the address is still
+     * visible for execution, and the OS should handle base=NULL on
+     * our queries, but we should check if we will.  Of course, this
+     * is likely an attack so it is OK for us to fail it. 
+     * 
+     * we only check upper bound and treat all smaller addresses as user addresses
+     */
+    return pc <= highest_user_address;
+}
+
+void
+merge_writecopy_pages(app_pc start, app_pc end)
+{
+    MEMORY_BASIC_INFORMATION mbi;
+    PBYTE pb = start;
+    uint prot;
+    size_t len = query_virtual_memory(pb, &mbi, sizeof(mbi));
+    ASSERT(len == sizeof(mbi));
+    LOG(GLOBAL, LOG_VMAREAS, 2, "merge_writecopy_pages "PFX"-"PFX"\n", start, end);
+    do {
+        if ((app_pc)mbi.BaseAddress >= end)
+            break;
+        ASSERT(mbi.State == MEM_COMMIT);
+        ASSERT(prot_is_writable(mbi.Protect));
+        prot = mbi.Protect &
+            ~PAGE_PROTECTION_QUALIFIERS;
+        if (prot == PAGE_WRITECOPY) {
+            /* HACK (xref case 8069): make a process-local copy to try and merge
+             * entire section into single region, for more efficient protection!
+             * Yes all the writable regions are already contiguous, but they
+             * have different flags, and so are different regions, and
+             * NtProtectVirtualMemory refuses to do more than one region at a time.
+             * However, regions seem to be merged when they have the
+             * same flags, so we just remove the C flag.
+             * Calling NtProtectVirtualMemory w/ PAGE_READWRITE to try
+             * and remove the copy-on-write bits does not work, so we
+             * write to every page!
+             * FIXME: test on other versions of windows!
+             * it's not documented so it may not be everywhere!
+             * works on Win2K Professional
+             * N.B.: since make_writable doesn't preserve copy-on-write,
+             * it's a good thing we do this hack.
+             * FIXME: how many of these pages would never have been made private?
+             * (case 8069 covers that inquiry)
+             */
+            volatile app_pc pc = mbi.BaseAddress;
+            app_pc stop = ((app_pc)mbi.BaseAddress) + mbi.RegionSize;
+            ASSERT(stop <= end);
+            LOG(GLOBAL, LOG_VMAREAS, 2, "writing to "SZFMT" pages to get local copy of"
+                " copy-on-write section @"PFX"\n", mbi.RegionSize/PAGE_SIZE,
+                pc);
+            while (pc < stop) {
+                *pc = *pc;
+                pc += PAGE_SIZE;
+            }
+        }
+        if (pb + mbi.RegionSize < pb)
+            break;
+        pb += mbi.RegionSize;
+    } while (query_virtual_memory(pb, &mbi, sizeof(mbi)) == sizeof(mbi));
+
+    LOG(GLOBAL, LOG_VMAREAS, 2, "checking that "PFX"-"PFX" merger worked\n",
+        start, end);
+    len = query_virtual_memory(start, &mbi, sizeof(mbi));
+    ASSERT(len == sizeof(mbi));
+    ASSERT(prot_is_writable(mbi.Protect));
+    /* OS could merge w/ another writable region so may not end at end */
+    ASSERT(end <= start + mbi.RegionSize);
+    /* we only call this on DR data sections right now */
+    ASSERT(dynamo_dll_end == NULL || /* FIXME: init it earlier */
+           (is_in_dynamo_dll(start) && is_in_dynamo_dll(end)));
+    LOG(GLOBAL, LOG_VMAREAS, 2, "DR regions post-merger:\n");
+    DOLOG(1, LOG_VMAREAS, {
+        print_dynamo_regions();
+        LOG(GLOBAL, LOG_VMAREAS, 2, "\n");
+    });
+}
+
+int
+find_dynamo_library_vm_areas()
+{
+    /* walk through memory regions in our own dll */
+    size_t len;
+    PBYTE pb;
+    MEMORY_BASIC_INFORMATION mbi;
+    int num_regions = 0;
+
+    get_dynamorio_library_path(); /* just to preserve side effects */
+    LOG(GLOBAL, LOG_VMAREAS, 1, PRODUCT_NAME" dll path: %s\n", get_dynamorio_library_path());
+
+    get_dynamorio_dll_start(); /* for side effects: probably already called though */
+    ASSERT(dynamo_dll_start != NULL);
+    pb = dynamo_dll_start;
+    len = query_virtual_memory(pb, &mbi, sizeof(mbi));
+    ASSERT(len == sizeof(mbi));
+    ASSERT(mbi.State != MEM_FREE);
+
+    LOG(GLOBAL, LOG_VMAREAS, 1, "\nOur regions:\n");
+    do {
+        if (mbi.State == MEM_FREE || (app_pc) mbi.AllocationBase != dynamo_dll_start)
+            break;
+        if (mbi.State == MEM_COMMIT) {
+            /* only look at committed regions */
+            LOG(GLOBAL, LOG_VMAREAS, 1, PFX"-"PFX" %s\n", mbi.BaseAddress,
+                ((app_pc)mbi.BaseAddress) + mbi.RegionSize,
+                prot_string(mbi.Protect));
+            num_regions++;
+            add_dynamo_vm_area(mbi.BaseAddress,
+                               ((app_pc)mbi.BaseAddress) + mbi.RegionSize,
+                               osprot_to_memprot(mbi.Protect),
+                               true /* from image */
+                               _IF_DEBUG(prot_string(mbi.Protect)));
+            /* we need all writable regions to be inside the
+             * sections that we protect */
+            ASSERT(!prot_is_writable(mbi.Protect) ||
+                   data_sections_enclose_region((app_pc)mbi.BaseAddress,
+                                                ((app_pc)mbi.BaseAddress) +
+                                                mbi.RegionSize));
+        }
+        if (pb + mbi.RegionSize < pb)
+            break;
+        pb += mbi.RegionSize;
+    } while (query_virtual_memory(pb, &mbi, sizeof(mbi)) == sizeof(mbi));
+
+    dynamo_dll_end = (app_pc) pb;
+    LOG(GLOBAL, LOG_VMAREAS, 1, PRODUCT_NAME" dll: from "PFX" to "PFX"\n\n",
+        dynamo_dll_start, dynamo_dll_end);
+    return num_regions;
+}
+
+void
+print_dynamo_regions()
+{
+    /* walk through memory regions in our own dll */
+    size_t len;
+    PBYTE pb;
+    MEMORY_BASIC_INFORMATION mbi;
+    /* dynamo_dll_start is a global defined in find_dynamo_library_vm_areas */
+    ASSERT(dynamo_dll_start != NULL);
+    pb = dynamo_dll_start;
+    len = query_virtual_memory(pb, &mbi, sizeof(mbi));
+    ASSERT(len == sizeof(mbi));
+    ASSERT(mbi.State != MEM_FREE);
+
+    do {
+        if (mbi.State == MEM_FREE || (app_pc) mbi.AllocationBase != dynamo_dll_start)
+            break;
+        if (mbi.State == MEM_COMMIT) {
+            /* only look at committed regions */
+            LOG(GLOBAL, LOG_ALL, 1, PFX"-"PFX" %s\n", mbi.BaseAddress,
+                ((app_pc)mbi.BaseAddress) + mbi.RegionSize,
+                prot_string(mbi.Protect));
+        }
+        if (pb + mbi.RegionSize < pb)
+            break;
+        pb += mbi.RegionSize;
+    } while (query_virtual_memory(pb, &mbi, sizeof(mbi)) == sizeof(mbi));
+}
+
+#ifdef DEBUG /* MEMORY STATS ****************************************/
+/* to make it easy to control log statement */
+# define MEM_STATS_ADD(stat, sz) do {                   \
+    if ((sz) != 0) {                                    \
+        STATS_ADD(stat, sz);                            \
+        LOG(GLOBAL, LOG_MEMSTATS, 4, #stat" sz "SZFMT"\n", sz); \
+    }                                                   \
+} while (0);
+
+/* N.B.: "reserved" here means reserved but not committed, so reserved
+ * and committed are disjoint, returns whether or not it was our memory
+ */
+static bool
+add_mem_stats(app_pc region, size_t r_commit, size_t r_reserve, bool r_is_stack,
+              uint r_type, size_t r_exec, size_t r_rw, size_t r_ro)
+{
+    bool ours = false;
+    /* add region to stats */
+    if (r_type == MEM_IMAGE) {
+        if (is_in_dynamo_dll(region)) {
+            ours = true;
+            MEM_STATS_ADD(dr_library_space, r_commit);
+            ASSERT(r_reserve == 0);
+        } else {
+            /* an image can have reserve-only sections (e.g., mscorlib has 2!) */
+            MEM_STATS_ADD(app_image_capacity, r_commit+r_reserve);
+        }
+    } else {
+        if (is_dynamo_address(region)) {
+            ours = true;
+        } else if (r_type == MEM_MAPPED) {
+            MEM_STATS_ADD(app_mmap_capacity, r_commit);
+        } else {
+            if (r_is_stack) {
+                MEM_STATS_ADD(app_stack_capacity, r_commit);
+            } else {
+                MEM_STATS_ADD(app_heap_capacity, r_commit);
+            }
+        }
+    }
+    LOG(GLOBAL, LOG_MEMSTATS, 4,
+        "Region "PFX"-"PFX" commit="SZFMT" reserve="SZFMT" stack="SZFMT" ours="SZFMT"\n",
+        region, region+r_commit+r_reserve, r_commit, r_reserve, r_is_stack, ours);
+    if (ours) {
+        MEM_STATS_ADD(dr_commited_capacity, r_commit);
+        MEM_STATS_ADD(dr_reserved_capacity, r_reserve);
+        MEM_STATS_ADD(dr_vsize, r_commit + r_reserve);
+    } else {
+        MEM_STATS_ADD(app_reserved_capacity, r_reserve);
+        MEM_STATS_ADD(app_committed_capacity, r_commit);
+        MEM_STATS_ADD(app_vsize, r_commit + r_reserve);
+        MEM_STATS_ADD(app_exec_capacity, r_exec);
+        MEM_STATS_ADD(app_rw_capacity, r_rw);
+        MEM_STATS_ADD(app_ro_capacity, r_ro);
+    }
+    /* yes, on windows vsize includes reserved */
+    MEM_STATS_ADD(total_vsize, r_commit + r_reserve);
+    /* count unaligned allocations (PEB TEB etc. see inside win2k pg 420) */
+    if (!ALIGNED(region, OS_ALLOC_GRANULARITY)) {
+        STATS_INC(unaligned_allocations);
+    }
+    return ours;
+}
+
+/* Since incremental app memory stats are too hard, we use snapshots */
+void
+mem_stats_snapshot()
+{
+    PBYTE pb = NULL;
+    MEMORY_BASIC_INFORMATION mbi;
+    /* stats strategy: have to wait until end of region to know everything,
+     * so locally cache sum-of-block values until then
+     */
+    size_t r_commit = 0, r_reserve = 0, r_exec = 0, r_ro = 0, r_rw = 0;
+    bool r_is_stack = false;
+    uint r_type = 0;
+    app_pc r_start = NULL;
+    if (!dynamo_initialized) {
+        /* Now that vm_areas_init() is after dynamo_thread_init()'s call to
+         * dump_global_stats() we come here prior to dynamo_areas or DR's
+         * library bounds being set up: best to just abort until we can
+         * gather accurate stats.
+         */
+        return;
+    }
+    /* It's too hard to keep track of these incrementally -- would have to
+     * record prior to NtAllocateVirtualMemory all of the reserved regions to
+     * know which went from reserved to committed, and on freeing to know what
+     * was committed and what reserved, etc., so we only do complete snapshots,
+     * resetting the stats to 0 each time.
+     */
+    mutex_lock(&snapshot_lock);
+    STATS_RESET(unaligned_allocations);
+    STATS_RESET(dr_library_space);
+    STATS_RESET(dr_commited_capacity);
+    STATS_RESET(dr_reserved_capacity);
+    STATS_RESET(total_wasted_vsize);
+    STATS_RESET(dr_wasted_vsize);
+    STATS_RESET(app_wasted_vsize);
+    STATS_RESET(total_vsize);
+    STATS_RESET(dr_vsize);
+    STATS_RESET(app_vsize);
+    STATS_RESET(app_reserved_capacity);
+    STATS_RESET(app_committed_capacity);
+    STATS_RESET(app_stack_capacity);
+    STATS_RESET(app_heap_capacity);
+    STATS_RESET(app_image_capacity);
+    STATS_RESET(app_mmap_capacity);
+    STATS_RESET(app_exec_capacity);
+    STATS_RESET(app_ro_capacity);
+    STATS_RESET(app_rw_capacity);
+    /* walk through every block in memory */
+    while (query_virtual_memory(pb, &mbi, sizeof(mbi)) == sizeof(mbi)) {
+        /* Standard block iteration that groups blocks with the same
+         * allocation base into a single region
+         */
+        if (mbi.State == MEM_FREE || mbi.AllocationBase == mbi.BaseAddress) {
+            bool ours = false;
+            if (r_start != NULL)
+                ours = add_mem_stats(r_start, r_commit, r_reserve, r_is_stack, 
+                                     r_type, r_exec, r_ro, r_rw);
+            /* reset for next region */
+            r_commit = r_reserve = r_exec = r_ro = r_rw = 0;
+            r_is_stack = false;
+            r_type = mbi.Type;
+            if (mbi.State == MEM_FREE) {
+                LOG(GLOBAL, LOG_MEMSTATS, 4, "Free "PFX"-"PFX"\n",
+                    mbi.BaseAddress, ((app_pc)mbi.BaseAddress)+mbi.RegionSize);
+                if (r_start != NULL && 
+                    !ALIGNED(mbi.BaseAddress, OS_ALLOC_GRANULARITY)) {
+                    /* wasted virtual address space, at least part of this free
+                     * region is unusable */
+                    size_t wasted = ALIGN_FORWARD(mbi.BaseAddress, 
+                                                  OS_ALLOC_GRANULARITY) - 
+                        (ptr_uint_t)mbi.BaseAddress;
+                    if (ours) {
+                        /* last region is ours, we are wasting */
+                        MEM_STATS_ADD(dr_wasted_vsize, (stats_int_t)wasted);
+                    } else {
+                        /* last region apps, its wasting */
+                        MEM_STATS_ADD(app_wasted_vsize, (stats_int_t)wasted);
+                    }
+                    MEM_STATS_ADD(total_wasted_vsize, (stats_int_t)wasted);
+                }
+                r_start = NULL;
+            } else {
+                r_start = mbi.AllocationBase;
+            }
+        }
+        /* incremental work until have end of region */
+        if (mbi.State == MEM_RESERVE) {
+            r_reserve += mbi.RegionSize;
+        } else if (mbi.State == MEM_COMMIT) {
+            r_commit += mbi.RegionSize;
+            if (TEST(PAGE_GUARD, mbi.Protect)) {
+                /* if any guard blocks inside region, assume entire region
+                 * is a stack
+                 */
+                r_is_stack = true;
+            }
+            /* protection stats could be incremental but that would duplicate checks
+             * for being DR memory.
+             * mbi.Protect is invalid for reserved memory, only useful for committed.
+             */
+            if (prot_is_executable(mbi.Protect))
+                r_exec += mbi.RegionSize;
+            else if (prot_is_writable(mbi.Protect))
+                r_rw += mbi.RegionSize;
+            else if (prot_is_readable(mbi.Protect))
+                r_ro += mbi.RegionSize;
+            /* we don't add up no-access memory! */
+        }
+        if (pb + mbi.RegionSize < pb)
+            break;
+        pb += mbi.RegionSize;
+    }
+    if (r_start != NULL)
+        add_mem_stats(r_start, r_commit, r_reserve, r_is_stack, r_type, r_exec, r_ro, r_rw);
+    STATS_PEAK(unaligned_allocations);
+    STATS_PEAK(dr_commited_capacity);
+    STATS_PEAK(dr_reserved_capacity);
+    STATS_PEAK(total_wasted_vsize);
+    STATS_PEAK(dr_wasted_vsize);
+    STATS_PEAK(app_wasted_vsize);
+    STATS_PEAK(total_vsize);
+    STATS_PEAK(dr_vsize);
+    STATS_PEAK(app_vsize);
+    STATS_PEAK(app_reserved_capacity);
+    STATS_PEAK(app_committed_capacity);
+    STATS_PEAK(app_stack_capacity);
+    STATS_PEAK(app_heap_capacity);
+    STATS_PEAK(app_image_capacity);
+    STATS_PEAK(app_mmap_capacity);
+    STATS_PEAK(app_exec_capacity);
+    STATS_PEAK(app_ro_capacity);
+    STATS_PEAK(app_rw_capacity);
+    mutex_unlock(&snapshot_lock);
+}
+#endif /* DEBUG (MEMORY STATS) ****************************************/
+
+/* update our data structures that record info on PE modules */
+/* rewalking is set when walking existing memory mappings, and is
+ * unset if called when processing a system call for (un)map 
+ */
+static void
+process_image(app_pc base, size_t size, uint prot, bool add, bool rewalking)
+{
+    const char *name = NULL;
+    bool module_is_native_exec = false;
+    /* ensure header is readable */
+    ASSERT(prot_is_readable(prot));
+    ASSERT(!rewalking || add);  /* when rewalking can only add */
+
+    /* FIXME: we only know that we are in a MEM_IMAGE
+     * we still need to be careful to check it is a real PE
+     * We could optimize out these system calls, but for now staying safe
+     */
+    if (!is_readable_pe_base(base)) {
+        ASSERT_CURIOSITY(false);
+        return;
+    }
+    /* Our WOW64 design for 32-bit DR involves ignoring all 64-bit dlls
+     * (several are visible: wow64cpu.dll, wow64win.dll, wow64.dll, and ntdll.dll)
+     * For 64-bit DR both should be handled.
+     */
+#ifdef X64
+    DODEBUG({
+        if (module_is_32bit(base)) {
+            LOG(GLOBAL, LOG_VMAREAS, 1,
+                "image "PFX"-"PFX" is 32-bit dll (wow64 process?)\n",
+                base, base+size);
+            ASSERT(is_wow64_process(NT_CURRENT_PROCESS));
+        }
+    });
+#else
+    if (module_is_64bit(base)) {
+        LOG(GLOBAL, LOG_VMAREAS, 1,
+            "image "PFX"-"PFX" is 64-bit dll (wow64 process?): ignoring it!\n", 
+            base, base+size);
+        ASSERT(is_wow64_process(NT_CURRENT_PROCESS));
+        return;
+    }
+#endif
+
+    /* Track loaded module list.  Needs to be done before
+     * hotp_process_image() and any caller of get_module_short_name()
+     * or other data that we cache in the list.
+     */
+    if (add) /* add first */
+        module_list_add(base, size, !rewalking /* !rewalking <=> at_map */);
+    else
+        os_module_set_flag(base, MODULE_BEING_UNLOADED);
+
+    /* DYNAMO_OPTION(native_exec) and DYNAMO_OPTION(use_moduledb) are the
+     * primary user of module name */
+    name = os_get_module_name_strdup(base HEAPACCT(ACCT_VMAREAS));
+    LOG(GLOBAL, LOG_VMAREAS, 1, "image %-15s %smapped @ "PFX"-"PFX"\n",
+        name == NULL ? "<no name>" : name, add ? "" : "un", base, base+size);
+
+    if (DYNAMO_OPTION(native_exec) && name != NULL &&
+        on_native_exec_list(base, name)) {
+        LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 1,
+            "module %s is on native_exec list\n", name);
+        module_is_native_exec = true;
+
+#ifdef GBOP
+        /* FIXME: if some one just loads a vm, our gbop would become useless;
+         * need better dgc identification for gbop; see case 8087.
+         */
+        if (add && TEST(GBOP_IS_DGC, DYNAMO_OPTION(gbop)) && !gbop_vm_loaded) {
+            /* !gbop_vm_loaded in the check above would prevent this memory 
+             * protection change from happenning for each vm load, not that any
+             * process load a vm multiple times or multiple vms.
+             */
+            SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+            gbop_vm_loaded = true;
+            SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+        }
+#endif
+    }
+
+    moduledb_process_image(name, base, add);
+
+    /* Case 7266: add all exes and dlls with managed code to native_exec_areas,
+     * for now.
+     * FIXME: should try to execute non-managed code under DR, when possible.
+     */
+    if (DYNAMO_OPTION(native_exec) && DYNAMO_OPTION(native_exec_managed_code) &&
+        module_has_cor20_header(base)) {
+        DODEBUG({
+            if (add) {
+                LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 1,
+                    "process_image: module=%s, base="PFX" has cor20 header, "
+                    "adding to native exec areas\n", name ? name : "<noname>", base);
+                SYSLOG_INTERNAL_INFO_ONCE("cor20 module %s added to native exec area",
+                                          name ? name : "<noname>");
+            }
+        });
+        module_is_native_exec = true;
+    }
+    /* xref case 10998 - we native exec modules with .pexe sections to handle all the
+     * int 3 strangeness.  FIXME - restrict further? only observed on Vista, known .pexe
+     * sections from problematic dlls all begin with mostly the same 0x60 first bytes,
+     * .pexe is observed to always be the first section, etc. */
+    if (DYNAMO_OPTION(native_exec) && DYNAMO_OPTION(native_exec_dot_pexe) && 
+        get_named_section_bounds(base, ".pexe", NULL, NULL)) {
+        DODEBUG({
+            if (add) {
+                LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 1,
+                    "process_image: module=%s, base="PFX" has .pexe section, "
+                    "adding to native exec areas\n", name ? name : "<noname>", base);
+                SYSLOG_INTERNAL_INFO(".pexe module %s added to native exec area",
+                                     name ? name : "<noname>");
+                /* check is one of the known .pexe dlls */
+                ASSERT_CURIOSITY(name != NULL && check_filter("AuxiliaryDisplayCpl.dll;AuxiliaryDisplayDriverLib.dll;AuxiliaryDisplayServices.dll;NetProjW.dll;P2PGraph.dll;localspl.dll;lsasrv.dll;mssrch.dll;p2psvc.dll;pmcsnap.dll;shell32.dll;spoolss.dll;uDWM.dll", name));
+            }
+        });
+        module_is_native_exec = true;
+    }
+    if (module_is_native_exec && add) {
+        RSTATS_INC(num_native_module_loads);
+        vmvector_add(native_exec_areas, base, base+size, NULL);
+    } else {
+        /* For safety we'll just always remove the region (even if add==true) to avoid
+         * any possibility of having stale entries in the vector overlap into new 
+         * non-native regions. Also see case 7628. */
+        ASSERT(!module_is_native_exec || /* if not native_exec shouldn't be in vector */
+               !vmvector_overlap(native_exec_areas, base, base+size));
+        vmvector_remove(native_exec_areas, base, base+size);
+    }
+
+    if (!IS_STRING_OPTION_EMPTY(patch_proof_list) ||
+        !IS_STRING_OPTION_EMPTY(patch_proof_default_list)) {
+        /* even if name is not valid we should match ALL */
+        if ((IS_LISTSTRING_OPTION_FORALL(patch_proof_list) ||
+             IS_LISTSTRING_OPTION_FORALL(patch_proof_default_list))
+            || (name != NULL && 
+                check_list_default_and_append(dynamo_options.patch_proof_default_list,
+                                              dynamo_options.patch_proof_list,
+                                              name))) {
+            if (add) {
+                LOG(GLOBAL, LOG_INTERP|LOG_VMAREAS, 1,
+                    "module %s is on patch proof list\n", name ? name : "<noname>");
+                STATS_INC(num_patch_proof_module_loads);
+                /* assuming code sections are added as non-writable we
+                 * will prevent them from becoming writable */
+
+                /* Note adding full module region here,
+                 * app_memory_protection_change() will filter out only
+                 * CODE.  FIXME: [minor perf] alternatively could walk
+                 * module and add only code sections here.
+                 */
+                vmvector_add(patch_proof_areas, base, base+size, NULL);
+            } else {
+                /* remove all areas in range */
+                vmvector_remove(patch_proof_areas, base, base+size);
+            }
+        }
+    }
+
+#ifdef HOT_PATCHING_INTERFACE
+    if (DYNAMO_OPTION(hot_patching)) {
+        if (!DYNAMO_OPTION(hotp_only)) {
+            hotp_process_image(base, add, false, false, NULL, NULL, 0);
+        } else {
+            bool needs_processing = false;
+            int num_threads = 0;
+            thread_record_t **all_threads = NULL;
+
+            /* For hotp_only, image processing is done in two steps.  The 
+             * first one is done without suspending all threads (expensive if 
+             * done for each dll load or unload).  Only if the first step 
+             * identified a module match, all threads (known to the core, of 
+             * course) are suspended and the image is processed, i.e., hot 
+             * patches are either injected or removed both of which in 
+             * hotp_only need all threads to be suspended. 
+             */
+            hotp_process_image(base, add, false/*no locks*/, 
+                               /* Do single-step at init: assume no other threads.
+                                * Risk is low; rest of DR assumes it as well.
+                                * Can't do two-step since have no dcontext yet
+                                * and hit synch_with_all_threads assert. */
+                               dynamo_initialized/*just check?*/,
+                               dynamo_initialized ? &needs_processing : NULL,
+                               NULL, 0);
+            if (needs_processing) {
+                DEBUG_DECLARE(bool ok =)
+                    synch_with_all_threads(THREAD_SYNCH_SUSPENDED, &all_threads, 
+                                           /* Case 6821: other synch-all-thread uses that
+                                            * only care about threads carrying fcache
+                                            * state can ignore us
+                                            */
+                                           &num_threads, THREAD_SYNCH_NO_LOCKS_NO_XFER,
+                                           /* if we fail to suspend a thread (e.g.,
+                                            * privilege problems) ignore it.
+                                            * FIXME: retry instead? */
+                                           THREAD_SYNCH_SUSPEND_FAILURE_IGNORE);
+                ASSERT(ok);
+                hotp_process_image(base, add, false, false, NULL, all_threads, 
+                                   num_threads);
+                end_synch_with_all_threads(all_threads, num_threads, true/*resume*/);
+            }
+        }
+    }
+#endif
+
+    if (DYNAMO_OPTION(IAT_convert)) { /* case 85 */
+        /* add IAT areas to a vmarea for faster lookup */
+        app_pc IAT_start, IAT_end;
+        bool valid = get_IAT_section_bounds(base, &IAT_start, &IAT_end);
+        if (valid && IAT_start != IAT_end) {
+            LOG(GLOBAL, LOG_INTERP, 2,
+                "module %s IAT("PFX","PFX") %s\n", name ? name : "<noname>", 
+                IAT_start, IAT_end, add ? "added" : "removed");
+            ASSERT_CURIOSITY(IAT_start != NULL && IAT_end != NULL);
+            ASSERT(IAT_start < IAT_end);
+            if (add) {
+                ASSERT(!vmvector_overlap(IAT_areas, IAT_start, IAT_end));
+                STATS_INC(num_IAT_areas);
+                if (!module_is_native_exec) {
+                    LOG(GLOBAL, LOG_INTERP, 1,
+                        "module %s IAT("PFX","PFX") added\n", name ? name : "<noname>", 
+                        IAT_start, IAT_end);
+                    vmvector_add(IAT_areas, IAT_start, IAT_end, NULL);
+                } else {
+                    LOG(GLOBAL, LOG_INTERP, 1,
+                        "skipping native module %s IAT("PFX","PFX"), native modules seen\n", 
+                        name ? name : "<noname>", 
+                        IAT_start, IAT_end);
+                }
+            } else {
+                STATS_DEC(num_IAT_areas);
+                vmvector_remove(IAT_areas, IAT_start, IAT_end);
+            }
+        } else {
+            ASSERT(!valid || IAT_start == base);
+            ASSERT_CURIOSITY(valid && "bad module");
+        }
+    }
+
+#ifdef RETURN_AFTER_CALL
+    DODEBUG({
+        if (!add && DYNAMO_OPTION(ret_after_call)) { 
+            /* case 5329 (see comments in process_image_post_vmarea()) --
+             * here we just check for exec areas before we flush them */
+            /* although some have no .text section
+             * e.g. hpzst3zm.dll from case 9121 
+             */
+            if (!executable_vm_area_overlap(base, 
+                                            base + size, false/*have no lock*/)) {
+                SYSLOG_INTERNAL_WARNING_ONCE("DLL with no executable areas "PFX"-"PFX"\n",
+                                             base, base + size);
+            }
+        }
+    });
+#endif /* RET_AFTER_CALL */
+
+    /* add module and its export symbols to our list only if logging */
+    DOLOG(1, LOG_SYMBOLS, { 
+        if (add) {
+            /* we need to touch memory to check for PE and that doesn't always work 
+             * FIXME: but, this is MEM_IMAGE, and above we verify the header
+             * is readable, so we can get rid of all of these system calls here
+             */
+            add_module_info((app_pc)base, size);
+        } else {
+            /* remove module if we have it added to our list */
+            remove_module_info((app_pc)base, size); 
+        }
+    });
+
+    if (name != NULL)
+        dr_strfree(name HEAPACCT(ACCT_VMAREAS));
+}
+
+/* Image processing that must be done after vmarea processing (mainly
+ * persisted cache loading)
+ */
+/* rewalking is set when walking existing memory mappings, and is
+ * unset if called when processing a system call for (un)map 
+ */
+static void
+process_image_post_vmarea(app_pc base, size_t size, uint prot, bool add, bool rewalking)
+{
+    /* ensure header is readable */
+    ASSERT(prot_is_readable(prot));
+    ASSERT(!rewalking || add);  /* when rewalking can only add */
+
+    /* FIXME: we only know that we are in a MEM_IMAGE
+     * we still need to be careful to check it is a real PE
+     * We could optimize out these system calls, but for now staying safe
+     */
+    if (!is_readable_pe_base(base)) {
+        ASSERT_CURIOSITY(false);
+        return;
+    }
+    /* Our WOW64 design for 32-bit DR involves ignoring all 64-bit dlls
+     * (several are visible: wow64cpu.dll, wow64win.dll, wow64.dll, and ntdll.dll)
+     * For 64-bit DR both should be handled.
+     */
+#ifndef X64
+    if (module_is_64bit(base))
+        return;
+#endif
+
+#ifdef RCT_IND_BRANCH
+    if (TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) ||
+        TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump))) {
+        /* we need to know about module addition or removal
+         * whether or not we'll act on it right now
+         */
+        rct_process_module_mmap(base, size, add, rewalking);
+    }
+#endif /* RCT_IND_BRANCH */
+
+    if (!add) /* remove last */
+        module_list_remove(base, size);
+}
+
+/* returns true if it added an executable region 
+ * ok for dcontext to be NULL if init==true and add==true
+ */
+static bool
+process_memory_region(dcontext_t *dcontext, MEMORY_BASIC_INFORMATION *mbi,
+                      bool init, bool add)
+{
+    bool from_image = (mbi->Type == MEM_IMAGE);
+    /* Our WOW64 design involves ignoring all 64-bit dlls
+     * (several are visible: wow64cpu.dll, wow64win.dll, wow64.dll, and ntdll.dll)
+     * We go ahead and track the memory, but we do not treat as an image
+     */
+    if (is_wow64_process(NT_CURRENT_PROCESS) &&
+        from_image && module_is_64bit(mbi->AllocationBase/*NOT BaseAddress*/))
+        from_image = false;
+    ASSERT(dcontext != NULL || (init && add));
+    DOLOG(2, LOG_VMAREAS, {
+        if (mbi->State != MEM_FREE) {
+            LOG(GLOBAL, LOG_VMAREAS, prot_is_executable(mbi->Protect) ? 1U : 2U,
+                PFX"-"PFX" %s %s allocbase="PFX"\n",
+                mbi->BaseAddress, ((app_pc)mbi->BaseAddress) + mbi->RegionSize,
+                prot_string(mbi->Protect),
+                (mbi->State == MEM_RESERVE) ? "reserve" : "commit ", mbi->AllocationBase);
+        }
+    });
+    /* MEM_RESERVE has meaningless mbi->Protect field, so we ignore it here */
+    if (mbi->State != MEM_COMMIT)
+        return false;
+    /* call these even if not marked as x, esp. the de-alloc, since some policy
+     * could have them on future list or something
+     */
+    if (add) {
+        return app_memory_allocation(dcontext, mbi->BaseAddress, mbi->RegionSize,
+                                     osprot_to_memprot(mbi->Protect), from_image
+                                     _IF_DEBUG(from_image ? "module" : "alloc"));
+    } else {
+        app_memory_deallocation(dcontext, mbi->BaseAddress, mbi->RegionSize,
+                                false /* don't own thread_initexit_lock */,
+                                from_image);
+    }
+    return false;
+}
+
+/* returns the number of executable areas added to DR's list */
+int
+find_executable_vm_areas()
+{
+    PBYTE pb = NULL;
+    MEMORY_BASIC_INFORMATION mbi;
+    PBYTE image_base = NULL;
+    size_t view_size = 0;
+    uint image_prot = 0;
+    int num_executable = 0;
+    LOG(GLOBAL, LOG_VMAREAS, 2, "Executable regions:\n");
+    DOLOG(1, LOG_MEMSTATS, {
+        mem_stats_snapshot();
+    });
+    /* Strategy: walk through every block in memory */
+    while (query_virtual_memory(pb, &mbi, sizeof(mbi)) == sizeof(mbi)) {
+        ASSERT(pb == mbi.BaseAddress);
+        if (mbi.State != MEM_FREE && mbi.Type == MEM_IMAGE && pb == mbi.AllocationBase) {
+            /* first region in an image */
+            MEMORY_BASIC_INFORMATION mbi_image;
+            PBYTE pb_image = pb + mbi.RegionSize;
+            image_base = pb;
+            image_prot = mbi.Protect;
+
+            /* We want to add to our module list right away so we can use it to
+             * obtain info when processing each +x region. We need the view size
+             * to call process_image with so we walk the image here. */
+            /* FIXME - if it ever becomes a perf issue we can prob. change process_image
+             * to not require the view size (by moving more things into
+             * process_image_post_vmarea or remembering the queryies). */
+            while (query_virtual_memory(pb_image, &mbi_image, sizeof(mbi_image)) ==
+                   sizeof(mbi_image) &&
+                   mbi_image.State != MEM_FREE && mbi_image.AllocationBase == pb) {
+                ASSERT(mbi_image.Type == MEM_IMAGE);
+                pb_image += mbi_image.RegionSize;
+            }
+            view_size = pb_image - pb;
+            process_image(image_base, view_size, image_prot,
+                          true /* add */, true /* rewalking */);
+        }
+        if (process_memory_region(NULL, &mbi, true/*init*/, true/*add*/))
+            num_executable++;
+        if (pb + mbi.RegionSize < pb)
+            break;
+        pb += mbi.RegionSize;
+        if (image_base != NULL && pb >= image_base + view_size) {
+            ASSERT(pb == image_base + view_size);
+            process_image_post_vmarea(image_base, view_size, image_prot, 
+                                      true /* add */, true /* rewalking */);
+            image_base = NULL;
+        }
+    }
+    ASSERT(image_base == NULL); /* check we don't have outstanding post call */
+    LOG(GLOBAL, LOG_VMAREAS, 2, "\n");
+    STATS_ADD(num_app_code_modules, num_executable);
+    return num_executable;
+}
+
+/* all_memory_areas is linux only, dummy on win32 */
+void all_memory_areas_lock()   { /* do nothing */ }
+void all_memory_areas_unlock() { /* do nothing */ }
+void update_all_memory_areas(app_pc start, app_pc end, uint prot) { /* do nothing */ }
+bool remove_from_all_memory_areas(app_pc start, app_pc end) { return true; }
+
+/* Processes a mapped-in section, which may or may not be an image.
+ * if add is false, assumes caller has already called flush_fragments_and_remove_region
+ * for all executable areas in region (probably just for entire super-region).
+ * returns the number of executable areas added to DR's list
+ */
+int
+process_mmap(dcontext_t *dcontext, app_pc pc, size_t size, bool add)
+{
+    PBYTE pb;
+    MEMORY_BASIC_INFORMATION mbi;
+    app_pc region_base;
+    int num_executable = 0;
+    bool image = false;
+    uint image_prot = 0;
+
+    ASSERT(!DYNAMO_OPTION(thin_client));
+    LOG(GLOBAL, LOG_VMAREAS, 2, "%s exec areas in region "PFX"\n",
+        add ? "adding" : "removing", pc);
+    pb = (PBYTE) pc;
+    if (query_virtual_memory(pb, &mbi, sizeof(mbi)) != sizeof(mbi))
+        ASSERT(false);
+    if (mbi.State == MEM_FREE)
+        return num_executable;
+    region_base = (app_pc) mbi.AllocationBase;
+    if (mbi.Type == MEM_IMAGE) {
+        process_image(region_base, size, mbi.Protect, add, false /* not rewalking */);
+        image = true;
+        image_prot = mbi.Protect;
+    }
+    /* Now update our vm areas executable region lists.
+     * The protection flag doesn't tell us if there are executable areas inside,
+     * must walk all the individual regions.
+     * FIXME: for remove, optimize to do single flush but multiple area removals?
+     */
+    while (query_virtual_memory(pb, &mbi, sizeof(mbi)) == sizeof(mbi)) {
+        if (mbi.State == MEM_FREE || mbi.AllocationBase != region_base)
+            break;
+        if (process_memory_region(dcontext, &mbi, false/*!init*/, add)) {
+            num_executable++;
+            STATS_INC(num_app_code_modules);
+        }
+        if (pb + mbi.RegionSize < pb) /* overflow check */
+            break;
+        pb += mbi.RegionSize;
+    }
+    if (image) {
+        process_image_post_vmarea(region_base, size, image_prot, add,
+                                  false /* not rewalking */);
+    }
+    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 3,
+        "Executable areas are now:\n");
+    DOLOG(3, LOG_SYSCALLS|LOG_VMAREAS, { print_executable_areas(GLOBAL); });
+    return num_executable;
+}
+
+app_pc
+get_image_entry()
+{
+    static app_pc image_entry_point = NULL;
+    if (image_entry_point == NULL) {
+        SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+        /* Note that peb->ImageBaseAddress = GetModuleHandle(NULL) */
+        image_entry_point = get_module_entry(get_own_peb()->ImageBaseAddress);
+        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+    }
+    return image_entry_point;
+}
+
+bool
+check_for_image_entry(app_pc bb_start)
+{
+    if (!reached_image_entry && bb_start == get_image_entry()) {
+        LOG(THREAD_GET, LOG_ALL, 1, "Reached image entry point "PFX"\n", bb_start);
+        SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+        reached_image_entry = true;
+        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+        return true;
+    }
+    return false;
+}
+
+void
+set_reached_image_entry()
+{
+    SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+    reached_image_entry = true;
+    SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+}
+
+bool
+reached_image_entry_yet()
+{
+    return reached_image_entry;
+}
+
+/* converts a local_state_t offset to a segment offset */
+ushort
+os_tls_offset(ushort tls_offs)
+{
+    ASSERT_TRUNCATE(tls_offs, ushort, tls_local_state_offs + tls_offs);    
+    return (ushort) (tls_local_state_offs + tls_offs);
+}
+
+local_state_t *
+get_local_state()
+{
+    byte *teb_addr = (byte *) get_own_teb();
+    return (local_state_t *) (teb_addr + tls_local_state_offs);
+}
+
+local_state_extended_t *
+get_local_state_extended()
+{
+    ASSERT(DYNAMO_OPTION(ibl_table_in_tls));
+    return (local_state_extended_t *) get_local_state();
+}
+
+/* returns the thread-private dcontext pointer for the calling thread */
+dcontext_t*
+get_thread_private_dcontext()
+{
+    /* This routine cannot be used before processwide os_init sets up the TLS index. */
+    if (tls_dcontext_offs == TLS_UNINITIALIZED)
+        return NULL;
+    /*
+     * We don't need to check whether this thread has been initialized under us - 
+     * Windows sets the value to 0 for us, so we'll just return NULL.
+     */
+    return (dcontext_t *) get_tls(tls_dcontext_offs);
+}
+
+/* sets the thread-private dcontext pointer for the calling thread */
+void
+set_thread_private_dcontext(dcontext_t *dcontext)
+{
+    set_tls(tls_dcontext_offs, dcontext);
+}
+
+#ifdef WINDOWS_PC_SAMPLE
+/* routines for pc sampling on windows */
+
+profile_t *
+create_profile(void *start, void *end, uint bucket_shift, dcontext_t *dcontext)
+{
+    profile_t *profile;
+    size_t buffer_size = ((((ptr_uint_t)end - (ptr_uint_t)start) >> bucket_shift) + 1)
+        * sizeof(uint);
+    if (dcontext == NULL) {
+        LOG(GLOBAL, LOG_PROFILE, 1, 
+            "Creating global profile from "PFX" to "PFX" with shift %d "
+            "for buffer size "SZFMT" bytes\n", start, end, bucket_shift, buffer_size); 
+        profile = (profile_t *) global_heap_alloc(sizeof(*profile) 
+                                                HEAPACCT(ACCT_STATS));
+        profile->buffer = (uint *) UNPROTECTED_GLOBAL_ALLOC(buffer_size 
+                                                            HEAPACCT(ACCT_STATS));
+    } else {
+        LOG(THREAD, LOG_PROFILE, 1, 
+            "Creating local profile from "PFX" to "PFX" with shift %d "
+            "(buffer size "SZFMT" bytes)\n", start, end, bucket_shift, buffer_size);
+        profile = (profile_t *) heap_alloc(dcontext, sizeof(*profile) 
+                                         HEAPACCT(ACCT_STATS));
+        profile->buffer = (uint *) UNPROTECTED_LOCAL_ALLOC(dcontext, buffer_size 
+                                                           HEAPACCT(ACCT_STATS));
+    }
+    memset(profile->buffer, 0, buffer_size);
+    profile->start = start;
+    profile->end = end;
+    profile->bucket_shift = bucket_shift;
+    profile->buffer_size = buffer_size;
+    profile->enabled = false;
+    profile->dcontext = dcontext;
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint((byte *)end - (byte *)start)));
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(buffer_size)));
+    profile->handle = nt_create_profile(NT_CURRENT_PROCESS, start, 
+                                        (uint)((byte *)end-(byte *)start),
+                                        profile->buffer,
+                                        (uint)buffer_size, bucket_shift);
+    return profile;
+}
+
+void
+free_profile(profile_t *profile) 
+{
+    ASSERT(!profile->enabled);
+    close_handle(profile->handle);
+    if (profile->dcontext == NULL) {
+        LOG(GLOBAL, LOG_PROFILE, 2, 
+            "Freeing global profile from "PFX" to "PFX" with shift %d "
+            "(buffer size "SZFMT" bytes)\n", 
+            profile->start, profile->end, profile->bucket_shift, 
+            profile->buffer_size); 
+        UNPROTECTED_GLOBAL_FREE(profile->buffer, profile->buffer_size 
+                                HEAPACCT(ACCT_STATS));
+        global_heap_free(profile, sizeof(*profile) 
+                         HEAPACCT(ACCT_STATS));
+    } else {
+        dcontext_t *dcontext = profile->dcontext;
+        LOG(THREAD, LOG_PROFILE, 2, 
+            "Freeing local profile from "PFX" to "PFX" with shift %d "
+            "(buffer size "SZFMT" bytes)\n",
+            profile->start, profile->end, profile->bucket_shift, 
+            profile->buffer_size); 
+        UNPROTECTED_LOCAL_FREE(dcontext, profile->buffer, profile->buffer_size 
+                               HEAPACCT(ACCT_STATS));
+        heap_free(dcontext, profile, sizeof(*profile) HEAPACCT(ACCT_STATS));
+    }
+}
+
+void
+start_profile(profile_t *profile)
+{
+    ASSERT(!profile->enabled);
+    nt_start_profile(profile->handle);
+    profile->enabled = true;
+}
+
+void
+stop_profile(profile_t *profile)
+{
+    ASSERT(profile->enabled);
+    nt_stop_profile(profile->handle);
+    profile->enabled = false;
+}
+
+void
+dump_profile_range(file_t file, profile_t *profile, byte *start, byte *end)
+{
+    uint i, step = (1 << profile->bucket_shift);
+    uint start_i = (uint) (start - (byte *)profile->start) / step;
+    uint end_i = (uint) (end - (byte *)profile->start) / step;
+    IF_X64(ASSERT_TRUNCATE(start_i, uint, (start - (byte *)profile->start) / step));
+    IF_X64(ASSERT_TRUNCATE(start_i, uint, (end - (byte *)profile->start) / step));
+
+    print_file(file, "Profile Dump\nRange "PFX"-"PFX"\nStep "PFX
+               " (%u-%u)\n", start, end, step, start_i, end_i);
+    ASSERT(start_i < profile->buffer_size / sizeof(uint) &&
+           end_i < profile->buffer_size / sizeof(uint));
+    for (i = start_i; i <= end_i; i++) {
+        if (profile->buffer[i] > 0)
+            print_file(file, PFX" %10d\n", (byte *)profile->start + i * step, 
+                       profile->buffer[i]);
+    }
+    print_file(file, "Finished Profile Dump\n");
+}
+
+void
+dump_profile(file_t file, profile_t *profile)
+{
+    dump_profile_range(file, profile, (byte *) profile->start,
+                       (byte *) profile->end);
+}
+
+uint
+sum_profile_range(profile_t *profile, byte *start, byte *end)
+{
+    uint i, ret = 0, step = (1 << profile->bucket_shift);
+    uint start_i = (uint) (start - (byte *)profile->start) / step;
+    uint end_i = (uint) (end - (byte *)profile->start) / step;
+    IF_X64(ASSERT_TRUNCATE(start_i, uint, (start - (byte *)profile->start) / step));
+    IF_X64(ASSERT_TRUNCATE(start_i, uint, (end - (byte *)profile->start) / step));
+
+    ASSERT(start_i < profile->buffer_size / sizeof(uint) &&
+           end_i < profile->buffer_size / sizeof(uint));
+    for (i = start_i; i <= end_i; i++) {
+        if (profile->buffer[i] > 0)
+            ret += profile->buffer[i];
+    }
+    return ret;
+}
+
+uint
+sum_profile(profile_t *profile)
+{
+    return sum_profile_range(profile, (byte *) profile->start,
+                             (byte *) profile->end);
+}
+
+void
+reset_profile(profile_t *profile)
+{
+    memset(profile->buffer, 0, profile->buffer_size);
+}
+#endif
+
+/* caller is required to handle thread synchronization */
+/* see inject.c, this must be able to free an nt_allocate_virtual_memory
+ * pointer */
+void
+os_heap_free(void *p, size_t size, heap_error_code_t *error_code)
+{
+    ASSERT(error_code != NULL);
+    DOSTATS({
+        if (!dynamo_exited_log_and_stats)
+            LOG(GLOBAL, LOG_HEAP, 4, "os_heap_free: "SZFMT" bytes @ "PFX"\n", size, p);
+    });
+    *error_code = nt_free_virtual_memory(p);
+    ASSERT(NT_SUCCESS(*error_code));
+}
+
+/* reserve virtual address space without committing swap space for it, 
+   and of course no physical pages since it will never be touched */
+void *
+os_heap_reserve(void *preferred, size_t size, heap_error_code_t *error_code)
+{
+    void *p = preferred;
+    ASSERT(error_code != NULL);
+    /* should only be used on aligned pieces */
+    ASSERT(size > 0 && ALIGNED(size, PAGE_SIZE));
+
+    *error_code = nt_allocate_virtual_memory(&p, size, 
+                                             PAGE_NOACCESS, MEMORY_RESERVE_ONLY);
+    if (!NT_SUCCESS(*error_code))
+        return NULL;
+    LOG(GLOBAL, LOG_HEAP, 2, "os_heap_reserve: "SZFMT" bytes @ "PFX"\n", size, p);
+    ASSERT(preferred == NULL || p == preferred); /* verify correct location */
+    return p;
+}
+
+void *
+os_heap_reserve_in_region(void *start, void *end, size_t size,
+                          heap_error_code_t *error_code)
+{
+    byte *cur, *p = NULL;
+    MEMORY_BASIC_INFORMATION mbi;
+
+    ASSERT(start < end);
+
+    LOG(GLOBAL, LOG_HEAP, 3,
+        "os_heap_reserve_in_region: "SZFMT" bytes in "PFX"-"PFX"\n", size, start, end);
+
+    /* if no restriction on location use regular os_heap_reserve() */
+    if (start == (void *)PTR_UINT_0 && end == (void *)POINTER_MAX)
+        return os_heap_reserve(NULL, size, error_code);
+
+    /* walk bounds to find a suitable location */
+    cur = (byte *)ALIGN_FORWARD(start, VM_ALLOCATION_BOUNDARY);
+    *error_code = HEAP_ERROR_CANT_RESERVE_IN_REGION;
+    while (p == NULL && cur + size <= (byte *)end &&
+           query_virtual_memory(cur, &mbi, sizeof(mbi)) == sizeof(mbi)) { 
+        if (mbi.State == MEM_FREE &&
+            mbi.RegionSize - (cur - (byte *)mbi.BaseAddress) >= size) {
+            /* we have a slot */
+            p = (byte *)os_heap_reserve(cur, size, error_code);
+            /* note - p could be NULL if someone grabbed some of the memory first */
+            LOG(GLOBAL, LOG_HEAP, (p == NULL) ? 1U : 3U,
+                "os_heap_reserve_in_region: got "PFX" trying to reserve "SZFMT" byte @ "
+                PFX" in free region "PFX"-"PFX"\n",
+                p, size, cur, mbi.BaseAddress, (byte *)mbi.BaseAddress + mbi.RegionSize);
+        }
+        cur = (byte *)ALIGN_FORWARD((byte *)mbi.BaseAddress + mbi.RegionSize,
+                                    VM_ALLOCATION_BOUNDARY);
+        /* check for overflow or 0 region size to prevent infinite loop */
+        if (cur <= (byte *)mbi.BaseAddress)
+            break; /* give up */
+    }
+
+    LOG(GLOBAL, LOG_HEAP, 2,
+        "os_heap_reserve_in_region: reserved "SZFMT" bytes @ "PFX" in "PFX"-"PFX"\n",
+        size, p, start, end);
+    return p;
+}
+
+/* commit previously reserved with os_heap_reserve pages */
+/* returns false when out of memory */
+/* A replacement of os_heap_alloc can be constructed by using os_heap_reserve 
+   and os_heap_commit on a subset of the reserved pages. */
+/* caller is required to handle thread synchronization */
+bool
+os_heap_commit(void *p, size_t size, uint prot, heap_error_code_t *error_code)
+{
+    uint os_prot = memprot_to_osprot(prot);
+    ASSERT(error_code != NULL);
+    /* should only be used on aligned pieces */
+    ASSERT(size > 0 && ALIGNED(size, PAGE_SIZE));
+    ASSERT(p);
+
+    LOG(GLOBAL, LOG_HEAP, 4, "os_heap_commit attempt: "SZFMT" bytes @ "PFX"\n", size, p);
+
+    *error_code = nt_commit_virtual_memory(p, size, os_prot);
+    if (!NT_SUCCESS(*error_code))
+        return false;           /* out of memory */
+
+    LOG(GLOBAL, LOG_HEAP, 3, "os_heap_commit: "SZFMT" bytes @ "PFX"\n", size, p);
+    return true;
+}
+
+/* caller is required to handle thread synchronization and to update dynamo vm areas */
+void
+os_heap_decommit(void *p, size_t size, heap_error_code_t *error_code)
+{
+    ASSERT(error_code != NULL);
+    if (!dynamo_exited)
+        LOG(GLOBAL, LOG_HEAP, 3, "os_heap_decommit: "SZFMT" bytes @ "PFX"\n", size, p);
+
+    *error_code = nt_decommit_virtual_memory(p, size);
+    ASSERT(NT_SUCCESS(*error_code));
+}
+
+bool
+os_heap_systemwide_overcommit(heap_error_code_t last_error_code)
+{
+    /* some error_codes may be worth retrying,
+     * e.g. for win32/ STATUS_COMMITMENT_MINIMUM may be a good one
+     * to retry, and maybe worth trying if systemwide memory
+     * pressure has brought us to the limit 
+     *
+     * FIXME: case 7032 covers detecting this.  In fact a pagefile resize,
+     *   will also cause an allocation failure, and TotalCommitLimit seems to be
+     *   the current pagefile size + physical memory not used by the OS.
+     *
+     *     PeakCommitment should be close to TotalCommitLimit, unless
+     *     the pagefile has been resized, or if the OS has trimmed the
+     *     system cache and has made it available in the
+     *     TotalCommitLimit
+     */
+
+    /* FIXME: conservative answer yes */
+    return true;
+}
+
+bool
+os_heap_get_commit_limit(size_t *commit_used, size_t *commit_limit)
+{
+    NTSTATUS res;
+    SYSTEM_PERFORMANCE_INFORMATION sperf_info;
+
+    STATS_INC(commit_limit_queries);
+    res = query_system_info(SystemPerformanceInformation,
+                            sizeof(sperf_info), &sperf_info);
+    if (NT_SUCCESS(res)) {
+        *commit_used = sperf_info.TotalCommittedPages;
+        *commit_limit = sperf_info.TotalCommitLimit;
+        return true;
+    } else {
+        ASSERT_NOT_REACHED();
+        return false;
+    }
+}
+
+/* yield the current thread */
+void
+thread_yield()
+{
+    /* main use in the busy path in mutex_lock  */
+    nt_yield();
+}
+
+void
+thread_sleep(uint64 milliseconds)
+{
+    LARGE_INTEGER liDueTime;
+    /* negative == relative */
+    liDueTime.QuadPart= - (int64)milliseconds * TIMER_UNITS_PER_MILLISECOND;
+    nt_sleep(&liDueTime);
+}
+
+/* probably should have an option to stop all threads and then nt_sleep()
+*/
+int
+os_timeout(int time_in_milliseconds)
+{
+    int res;
+    LARGE_INTEGER liDueTime;
+    liDueTime.QuadPart= - time_in_milliseconds * TIMER_UNITS_PER_MILLISECOND;
+    LOG(THREAD_GET, LOG_ALL, 2, "os_timeout(%d)\n", time_in_milliseconds);
+
+    res = nt_sleep(&liDueTime);
+    LOG(THREAD_GET, LOG_ALL, 2, "Timeout expired res=%d.\n", res);
+    return res;
+}
+
+bool
+thread_suspend(thread_record_t *tr)
+{
+    return nt_thread_suspend(tr->handle, NULL);
+}
+
+bool
+thread_resume(thread_record_t *tr)
+{
+    return nt_thread_resume(tr->handle, NULL);
+}
+
+bool
+thread_terminate(thread_record_t *tr)
+{
+    return nt_terminate_thread(tr->handle, 0);
+}
+
+bool
+thread_get_mcontext(thread_record_t *tr, dr_mcontext_t *mc)
+{
+    CONTEXT cxt;
+    cxt.ContextFlags = CONTEXT_DR_STATE;
+    if (thread_get_context(tr, &cxt)) {
+        context_to_mcontext(mc, &cxt);
+        return true;
+    }
+    return false;
+}
+
+bool
+thread_set_mcontext(thread_record_t *tr, dr_mcontext_t *mc)
+{
+    CONTEXT cxt;
+    cxt.ContextFlags = CONTEXT_DR_STATE;
+    mcontext_to_context(&cxt, mc);
+    return thread_set_context(tr->handle, &cxt);
+}
+
+bool
+thread_get_context(thread_record_t *tr, CONTEXT *context)
+{
+    return NT_SUCCESS(nt_get_context(tr->handle, context));
+}
+
+bool
+thread_set_context(thread_record_t *tr, CONTEXT *context)
+{
+    return NT_SUCCESS(nt_set_context(tr->handle, context));
+}
+
+/* Takes an os-specific context */
+void
+thread_set_self_context(void *cxt)
+{
+    /* We use NtContinue to avoid privilege issues with NtSetContext */
+    nt_continue((CONTEXT *)cxt);
+    ASSERT_NOT_REACHED();
+}
+
+/* Takes a dr_mcontext_t */
+void
+thread_set_self_mcontext(dr_mcontext_t *mc)
+{
+    CONTEXT cxt;
+    cxt.ContextFlags = CONTEXT_DR_STATE;
+    mcontext_to_context(&cxt, mc);
+    thread_set_self_context(&cxt);
+    ASSERT_NOT_REACHED();
+}
+
+int
+get_num_processors()
+{
+    static uint num_cpu = 0;         /* cached value */
+    if (!num_cpu) {
+        SYSTEM_BASIC_INFORMATION sbasic_info;
+        NTSTATUS result = query_system_info(SystemBasicInformation, 
+                                            sizeof(SYSTEM_BASIC_INFORMATION), 
+                                            &sbasic_info);
+        if (!NT_SUCCESS(result))
+            num_cpu = 1;               /* assume single CPU */
+        else 
+            num_cpu = sbasic_info.NumberProcessors;
+        ASSERT(num_cpu);
+    }
+    return num_cpu;
+}
+
+/* Static to save stack space, is initialized at first call to debugbox or
+ * at os_init (whichever is earlier), we are guaranteed to be single threaded
+ * at os_init so no race conditions even though there shouldn't be any anyways
+ * unless snwprintf does something funny with the buffer. This also ensures
+ * that the static buffers in get_application_name and get_application_pid
+ * get initialized while we are still single threaded. */
+static wchar_t debugbox_title_buf[MAXIMUM_PATH+64];
+static void init_debugbox_title_buf()
+{
+    snwprintf(debugbox_title_buf, BUFFER_SIZE_ELEMENTS(debugbox_title_buf), 
+              L_PRODUCT_NAME L" Notice: %hs(%hs)", 
+              get_application_name(), get_application_pid());
+    NULL_TERMINATE_BUFFER(debugbox_title_buf);
+}
+
+/* Static buffer for debugbox.  If stack-allocated, debugbox is one of
+ * the big space hogs when reporting a crash and we risk exhausting
+ * the stack.
+ */
+DECLARE_NEVERPROT_VAR(static wchar_t debugbox_msg_buf[MAX_LOG_LENGTH], {0});
+
+/* draw a message box on the screen with msg */
+int
+debugbox(char *msg)
+{
+    int res;
+
+    if (debugbox_title_buf[0] == 0)
+        init_debugbox_title_buf();
+
+    /* FIXME: If we hit an assert in nt_messagebox, we'll deadlock when
+     * we come back here.
+     */
+    mutex_lock(&debugbox_lock);
+
+    snwprintf(debugbox_msg_buf, BUFFER_SIZE_ELEMENTS(debugbox_msg_buf), L"%hs", msg);
+    NULL_TERMINATE_BUFFER(debugbox_msg_buf);
+    res = nt_messagebox(debugbox_msg_buf, debugbox_title_buf);
+    
+    mutex_unlock(&debugbox_lock);
+
+    return res;
+}
+
+#ifdef FANCY_COUNTDOWN          /* NOT IMPLEMENTED */
+/* Fancy countdown box for a message with timeout */
+
+// This is STATIC window control ID for a message box
+#define ID_MSGBOX_STATIC_TEXT    0x0000ffff
+
+typedef struct {
+    char *message;
+    char *title;
+    HANDLE timer;
+    int seconds_left;
+    bool done;
+} timeout_context_t;
+
+#define print_timeout_message(buf, context) \
+   snprintf(buf, sizeof(buf), "%s\n""You have %d seconds to respond", \
+            ((timeout_context_t*)context)->message, ((timeout_context_t*)context)->seconds_left);
+
+/* FIXME: Be careful about creating a thread -- make sure we 
+don't intercept its asynch events.  Not clear how to do that -- you can
+turn off interception once it's created, but to not intercept its init
+APC, currently all you can do is globally turn off event interception,
+or else try to identify it when we see the init APC. */
+
+/* based on Richter's 11-TimedMsgBox */
+DWORD WINAPI 
+message_box_timeout_thread(void *context) {
+    timeout_context_t *tcontext = (timeout_context_t*)context;
+    return 0;
+
+    LOG(GLOBAL, LOG_ALL, 2, "message_box_timeout_thread(%d)\n", tcontext->seconds_left);
+    do {
+        WaitForSingleObject(tcontext->timer, tcontext->seconds_left * 1000);
+        {
+            HWND hwnd = FindWindow(NULL, tcontext->title);
+
+            LOG(THREAD_GET, LOG_ALL, 2, "message_box_timeout_thread(%d) hwnd="PIFX"\n", 
+                tcontext->seconds_left, hwnd);
+            if (hwnd != NULL) {
+                char countdown[MAX_LOG_LENGTH];
+                tcontext->seconds_left--;
+                print_timeout_message(countdown, context);
+                SetDlgItemText(hwnd, ID_MSGBOX_STATIC_TEXT, countdown);
+
+                if (tcontext->seconds_left == 0) {
+                    /* timeout */
+                    EndDialog(hwnd, IDOK);
+                    return 1;
+                }
+            }
+        }
+    } while(!tcontext->done);
+    return 0;
+}
+
+int
+os_countdown_messagebox(char *message, int time_in_milliseconds)
+{
+    char title[MAXIMUM_PATH+64];
+    char buf[MAX_LOG_LENGTH];
+
+    LONG update_period = 1000; /* milliseconds = 1s */
+    uint seconds_left = time_in_milliseconds / update_period;
+    LARGE_INTEGER liDueTime;
+    HANDLE htimer;
+    HANDLE hthread;
+    timeout_context_t context = {message, title, 0, seconds_left, false};
+    int res;
+
+    LOG(THREAD_GET, LOG_ALL, 2, 
+        "os_countdown_messagebox(%s, %d)\n", message, time_in_milliseconds); 
+    ASSERT_NOT_IMPLEMENTED(false);
+
+    get_debugbox_title(title, sizeof(title));
+    print_timeout_message(buf, &context);
+
+    liDueTime.QuadPart= - update_period * TIMER_UNITS_PER_MILLISECOND;
+
+    /* create a waitable timer to get signaled periodically */
+    htimer = nt_create_and_set_timer(&liDueTime, update_period);
+    context.timer = htimer;
+    
+    hthread = CreateThread(NULL, 0, &message_box_timeout_thread, NULL, 0, NULL);
+    LOG(THREAD_GET, LOG_ALL, 2, 
+        "os_countdown_messagebox(%s, %d)\n", message, time_in_milliseconds); 
+
+    debugbox(buf);
+    context.done = true;
+
+    WaitForSingleObject(hthread, INFINITE);
+
+    close_handle(htimer);
+    close_handle(hthread);
+
+    return 0;
+}
+#else
+int
+os_countdown_messagebox(char *message, int time_in_milliseconds) {
+    char buf[MAX_LOG_LENGTH];
+    snprintf(buf, sizeof(buf), "%sTimeout ignored", message);
+    NULL_TERMINATE_BUFFER(buf);
+    debugbox(buf);
+    return 0;
+}
+#endif /* FANCY_COUNTDOWN */
+
+#if defined(CLIENT_INTERFACE) || defined(HOT_PATCHING_INTERFACE)
+shlib_handle_t 
+load_shared_library(char *name)
+{
+    wchar_t buf[MAX_PATH];
+    snwprintf(buf, BUFFER_SIZE_ELEMENTS(buf), L"%hs", name);
+    NULL_TERMINATE_BUFFER(buf);
+    return load_library(buf);
+}
+#endif
+
+#if defined(CLIENT_INTERFACE)
+shlib_routine_ptr_t
+lookup_library_routine(shlib_handle_t lib, char *name)
+{
+    return (shlib_routine_ptr_t)get_proc_address(lib, name);
+}
+
+void
+unload_shared_library(shlib_handle_t lib)
+{
+   free_library(lib);
+}
+
+void
+shared_library_error(char *buf, int maxlen)
+{
+    /* FIXME : this routine does nothing. It used to use kernel32 FormatMessage
+     * to report errors, but now that we are kernel32 independant that will no
+     * longer work. Would be nice if we could do something with the nt status
+     * codes, but unclear how to propagate them to here. */
+    buf[0] = '\0';
+}
+
+bool
+shared_library_bounds(IN shlib_handle_t lib, IN byte *addr,
+                      OUT byte **start, OUT byte **end)
+{
+    size_t sz = get_allocation_size(lib, start);
+    ASSERT(start != NULL && end != NULL);
+    *end = *start + sz;
+    ASSERT(addr == NULL || (addr >= *start && addr < *end));
+    return true;
+}
+#endif /* defined(CLIENT_INTERFACE) */
+
+/* Returns base of the "allocation region" containing pc for allocated memory, 
+ * Note the current protection settings may not be uniform in the whole region.
+ * Returns NULL for free memory or invalid user mode addresses.
+ * Use get_allocation_size() when size is also needed.
+ */
+byte *
+get_allocation_base(byte *pc)
+{
+    MEMORY_BASIC_INFORMATION mbi;
+    size_t res = query_virtual_memory(pc, &mbi, sizeof(mbi));
+    if (res != sizeof(mbi)) {
+        /* invalid address given, e.g. POINTER_MAX */
+        return NULL;
+    }
+    if (mbi.State == MEM_FREE) {
+        ASSERT_CURIOSITY(mbi.BaseAddress == (byte*)ALIGN_BACKWARD(pc, PAGE_SIZE));
+        return NULL;
+    }
+
+    return mbi.AllocationBase;
+}
+
+/* See comments below -- this max will go away once we're sure
+ * we won't infinite loop.  Until then we keep it very large
+ * (we've seen 128MB with a ton of single-page regions inside in case 4502)
+ * such that should never hit it (@ 1 block per page will hit 4GB first)
+ */
+enum { MAX_QUERY_VM_BLOCKS = 512*1024 };
+
+/* Returns size of the "allocation region" containing pc
+ * Note that this may include several pieces of memory with different
+ * protection and state attributes.
+ * If base_pc != NULL returns base pc as well 
+
+ * If memory is free we set base_pc to NULL, but return free region
+ * size - note that we can't efficiently go backwards to find the
+ * maximum possible allocation size in a free hole.
+ */
+size_t
+get_allocation_size(byte *pc, byte **base_pc)
+{
+    PBYTE pb = (PBYTE) pc;
+    MEMORY_BASIC_INFORMATION mbi;
+    PVOID region_base;
+    PVOID pb_base;
+    size_t pb_size;
+    size_t res;
+    int num_blocks = 0;
+    size_t size;
+
+    res = query_virtual_memory(pb, &mbi, sizeof(mbi));
+    if (res != sizeof(mbi) /* invalid address given, e.g. POINTER_MAX */) {
+        if (base_pc != NULL)
+            *base_pc = NULL;
+        return 0;
+    }
+
+    if (mbi.State == MEM_FREE /* free memory doesn't have AllocationBase */) {
+        if (base_pc != NULL)
+            *base_pc = NULL;
+        /* note free region from requested ALIGN_BACKWARD(pc base */
+        return mbi.RegionSize;
+    }
+
+    pb_base = mbi.BaseAddress;
+    pb_size = mbi.RegionSize;
+    region_base = mbi.AllocationBase;
+    /* start beyond queried region */
+    pb = (byte *) pb_base + mbi.RegionSize;
+    size = (app_pc)pb - (app_pc)region_base;
+
+    /* must keep querying contiguous blocks until reach next region
+     * to find this region's size
+     */
+    LOG(GLOBAL, LOG_VMAREAS, 3,
+        "get_allocation_size pc="PFX" base="PFX" region="PFX" size="PIFX"\n",
+        pc, pb_base, region_base, mbi.RegionSize);
+    do {
+        res = query_virtual_memory(pb, &mbi, sizeof(mbi));
+        if (res != sizeof(mbi) || mbi.State == MEM_FREE || mbi.AllocationBase != region_base)
+            break;
+        ASSERT(mbi.RegionSize > 0); /* if > 0, we will NOT infinite loop */
+        size += mbi.RegionSize;
+        if (pb + mbi.RegionSize < pb)
+            break;
+        pb += mbi.RegionSize;
+        /* WARNING: if app is changing memory at same time as we're examining
+         * it, we could have problems: but, if region becomes free, we'll break,
+         * and so long as RegionSize > 0, we should make progress and hit
+         * end of address space in worst case -- so we shouldn't need this
+         * num_blocks max, but we'll keep it for now.  FIXME.
+         */
+        num_blocks++;
+    } while (num_blocks < MAX_QUERY_VM_BLOCKS);
+    ASSERT_CURIOSITY(num_blocks < MAX_QUERY_VM_BLOCKS);
+    /* size may push to overflow to 0 if at end of address space */
+    ASSERT((app_pc)region_base + size > pc || (app_pc)region_base + size == NULL);
+    if (base_pc != NULL)
+        *base_pc = (byte *) region_base;
+    return size;
+}
+
+/* Returns size and writability of the memory area (not allocation region)
+ * containing pc.  This is a single memory area all from the same allocation
+ * region and all with the same protection and state attributes.
+ * If base_pc != NULL returns base pc of the area.
+ */
+bool
+get_memory_info(const byte *pc, byte **base_pc, size_t *size, uint *prot)
+{
+    MEMORY_BASIC_INFORMATION mbi;
+    size_t res = query_virtual_memory(pc, &mbi, sizeof(mbi));
+    if (res != sizeof(mbi) || mbi.State == MEM_FREE)
+        return false;
+    if (base_pc != NULL)
+        *base_pc = mbi.BaseAddress;
+    if (size != NULL)
+        *size = mbi.RegionSize;
+    if (prot != NULL) {
+        *prot = osprot_to_memprot(mbi.Protect);
+    }
+    return true;
+}
+
+/* It is ok to pass NULL for dcontext */
+bool
+get_stack_bounds(dcontext_t *dcontext, byte **base, byte **top)
+{
+    os_thread_data_t *ostd = NULL;
+    if (dcontext != NULL) {
+        ostd = (os_thread_data_t *) dcontext->os_field;
+        if (ostd->teb_stack_no_longer_valid) {
+            /* Typically this means we are on NT or 2k and the TEB is being used
+             * as the stack for ExitThread. Xref fixme in check_for_stack_free()
+             * about possibly handling this differently. */
+            return false;
+        }
+    }
+    if (dcontext == NULL || ostd->stack_base == NULL) {
+        byte * stack_base = NULL;
+        byte * stack_top = NULL;
+        /* This only works if the dcontext is for the current thread. */
+        ASSERT(dcontext == NULL || dcontext == get_thread_private_dcontext());
+        /* use the TIB fields:
+         *   PVOID   pvStackUserTop;     // 04h Top of user stack
+         *   PVOID   pvStackUserBase;    // 08h Base of user stack
+         * and assume fs is always a valid TIB pointer when called here
+         */
+        stack_top = (byte *) get_tls(TOP_STACK_TIB_OFFSET);
+        stack_base = (byte *) get_tls(BASE_STACK_TIB_OFFSET);
+        LOG(THREAD, LOG_THREADS, 1, "app stack now is "PFX"-"PFX"\n",
+            stack_base, stack_top); /* NULL dcontext => nop */
+        /* we only have current base, we need to find reserved base */
+        stack_base = get_allocation_base(stack_base);
+        LOG(THREAD, LOG_THREADS, 1, "app stack region is "PFX"-"PFX"\n",
+            stack_base, stack_top); /* NULL dcontext => nop */
+        /* FIXME - make curiosity? prob. could create a thread with no official
+         * stack and we would largely be fine with that. */
+        ASSERT(stack_base != NULL);
+        ASSERT(stack_base < stack_top);
+        ASSERT(get_allocation_base(stack_top - 1) == stack_base &&
+               (get_allocation_base(stack_top) != stack_base ||
+                /* PR 252008: for WOW64 nudges we allocate an extra page.
+                 * We would test dcontext->nudge_thread but that's not set yet. */
+                is_wow64_process(NT_CURRENT_PROCESS)));
+        if (dcontext == NULL) {
+            if (base != NULL)
+                *base = stack_base;
+            if (top != NULL)
+                *top = stack_top;
+            return true;
+        }
+        ostd->stack_base = stack_base;
+        ostd->stack_top = stack_top;
+    }
+    if (base != NULL)
+        *base = ostd->stack_base;
+    if (top != NULL)
+        *top = ostd->stack_top;
+    return true;
+}
+
+/*
+winnt.h:#define PAGE_READONLY 2
+winnt.h:#define PAGE_READWRITE 4
+winnt.h:#define PAGE_WRITECOPY 8
+winnt.h:#define PAGE_EXECUTE 16
+winnt.h:#define PAGE_EXECUTE_READ 32
+winnt.h:#define PAGE_EXECUTE_READWRITE 64
+winnt.h:#define PAGE_GUARD 256
+winnt.h:#define PAGE_NOACCESS 1
+winnt.h:#define PAGE_NOCACHE 512
+winnt.h:#define PAGE_EXECUTE_WRITECOPY 128
+*/
+
+/* is_readable_without_exception checks to see that all bytes with addresses
+ * from pc to pc+size-1 are readable and that reading from there won't
+ * generate an exception.  this is a stronger check than
+ * !not_readable() below.
+ * FIXME : beware of multi-thread races, just because this returns true, 
+ * doesn't mean another thread can't make the region unreadable between the 
+ * check here and the actual read later.  See safe_read() as an alt.
+ */
+static bool 
+query_is_readable_without_exception(byte *pc, size_t size)
+{
+    MEMORY_BASIC_INFORMATION mbi;
+    byte *check_pc = (byte *) ALIGN_BACKWARD(pc, PAGE_SIZE);
+    size_t res;
+    if (size > (size_t)((byte *)POINTER_MAX - pc))
+        size = (byte *)POINTER_MAX - pc;
+    do {
+        res = query_virtual_memory(check_pc, &mbi, sizeof(mbi));
+        if (res != sizeof(mbi)) {
+            return false;
+        } else {
+            if (mbi.State != MEM_COMMIT ||
+                TEST(PAGE_GUARD, mbi.Protect) ||
+                !prot_is_readable(mbi.Protect))
+                return false;
+        }
+        /* FIXME: this routine can walk by mbi.RegionSize instead of pages */
+        check_pc += PAGE_SIZE;
+    } while (check_pc != 0/*overflow*/ && check_pc < pc+size);
+    return true;
+}
+
+/* On Windows, same as is_readable_without_exception */
+bool 
+is_readable_without_exception_query_os(byte *pc, size_t size)
+{
+    return is_readable_without_exception(pc, size);
+}
+
+/* Reads size bytes starting at base and puts them in out_buf, this is safe
+ * to call even if the memory at base is unreadable, returns true if the 
+ * read succeeded */
+/* FIXME : This avoids the races with an is_readable_without_exception followed
+ * by a read. We get the os to do the read for us via ReadVirtualMemory, 
+ * however this is still much slower then a structured exception handling 
+ * solution since we expect this to succeed most of the time.  Ref PR 206278 
+ * and 208562 on using the faster TRY/EXCEPT. */
+bool
+safe_read_ex(const void *base, size_t size, void *out_buf, size_t *bytes_read)
+{
+    if (bytes_read != NULL)
+        *bytes_read = 0;
+    STATS_INC(num_safe_reads);
+    return nt_read_virtual_memory(NT_CURRENT_PROCESS, base, out_buf, size, bytes_read);
+}
+
+/* FIXME - fold this together with safe_read_ex() (is a lot of places to update) */
+bool
+safe_read(const void *base, size_t size, void *out_buf)
+{
+    size_t bytes_read = 0;
+    return (safe_read_ex(base, size, out_buf, &bytes_read) && bytes_read == size);
+}
+
+/* Writes size bytes starting at base from in_buf, this is safe
+ * to call even if the memory at base is unreadable, returns true if the 
+ * write succeeded.  See safe_read_ex() on using more performant TRY/EXCEPT. */
+bool
+safe_write_ex(void *base, size_t size, const void *in_buf, size_t *bytes_written)
+{
+    if (bytes_written != NULL)
+        *bytes_written = 0;
+    STATS_INC(num_safe_writes);
+    return nt_write_virtual_memory(NT_CURRENT_PROCESS, base, in_buf,
+                                   size, bytes_written);
+}
+
+/* FIXME - fold this together with safe_write_ex() (is a lot of places to update) */
+bool
+safe_write(void *base, size_t size, const void *in_buf)
+{
+    size_t written_bytes = 0;
+    return (safe_write_ex(base, size, in_buf, &written_bytes) && written_bytes == size);
+}
+
+/* unlike get_memory_info() we return osprot preserving complete
+ * protection info.  Note errors or bad addresses are ignored and
+ * return PAGE_NOACCESS instead.  If the difference between invalid
+ * address or PAGE_NOACCESS is essential users must use
+ * query_virtual_memory()
+ */
+uint
+get_current_protection(byte *pc)
+{
+    PBYTE pb = (PBYTE) pc;
+    MEMORY_BASIC_INFORMATION mbi;
+    size_t res = query_virtual_memory(pb, &mbi, sizeof(mbi));
+    ASSERT(res == sizeof(mbi));
+    ASSERT(mbi.State != MEM_FREE); /* caller assumes this is a valid page */
+    if (res != sizeof(mbi) || mbi.State == MEM_FREE) {
+        /* note we could also return 0 since PAGE_NOACCESS is 1 */
+        ASSERT_CURIOSITY(false && "in get_memory_osprot");
+        return PAGE_NOACCESS;
+    }
+    return mbi.Protect;
+}
+
+/* see note on is_readable_without_exception for differences between the two
+ * returns true if any byte with address from pc to pc+size-1 is not_readable
+ * FIXME: reverse the logic to make this is_readable
+ * Also CHECK that we actually need this routine
+ */
+bool
+not_readable(byte *pc, size_t size)
+{
+    MEMORY_BASIC_INFORMATION mbi;
+    byte *check_pc = (byte *) ALIGN_BACKWARD(pc, PAGE_SIZE);
+    size_t res;
+    if (size > (size_t)((byte *)POINTER_MAX - pc))
+        size = (byte *)POINTER_MAX - pc;
+    while (check_pc != 0/*overflow*/ && check_pc < pc+size) {
+        res = query_virtual_memory(check_pc, &mbi, sizeof(mbi));
+        if (res != sizeof(mbi) || mbi.State == MEM_FREE)
+            return true;
+        else if (!prot_is_readable(mbi.Protect))
+            return true;
+        check_pc += PAGE_SIZE;
+    }
+    return false;
+}
+
+void
+mark_page_as_guard(byte *pc)
+{
+    uint old_prot;
+    int res;
+    /* NOACCESS combined w/ GUARD is invalid -- apparently you specify what you want
+     * after the guard triggers
+     */
+    uint flags = PAGE_READWRITE | PAGE_GUARD;
+    ASSERT(ALIGNED(pc, PAGE_SIZE));
+    res = protect_virtual_memory((void *) pc, PAGE_SIZE, flags, &old_prot);
+    ASSERT(res);
+}
+
+/* Change page protection for pc:pc+size.
+ * If set is false, makes [un]writable depending on add_writable argument,
+ * preserving other flags; else, sets protection to new_prot.
+ * If cow is true and set is false and writable is true, sets to
+ * be not only writable but copy-on-write.
+ * Requires pc and size are multiples of the
+ * PAGE_SIZE.  
+ *
+ * Returns true if all protection requests succeeded, false if
+ * protection on any subregion fails: all callers that make memory
+ * writable should be able to handle the unrecoverable yet failure on
+ * out of commit memory.  
+ * changed_protection is set to true if changes were necessary, or
+ * false if protection already meets requirements.  Note that any
+ * reserved yet not committed subregion will be skipped (and change
+ * protection is not needed).
+ */
+static bool
+internal_change_protection(byte *start, size_t requested_size, bool set,
+                           bool writable, bool cow, uint new_prot, 
+                           bool *changed_protection /* OUT */)
+{
+    byte *pc = start;
+    size_t remaining_size = requested_size;
+    bool changed_permissions = false;
+    bool subregions_failed = false;
+
+    /* while this routine may allow crossing allocation bases 
+     * it is supposed to be in error, a MEM_FREE block would terminate it */
+    DEBUG_DECLARE(app_pc first_allocation_base = NULL;)
+
+    /* we no longer allow you to pass in 0 */
+    ASSERT(requested_size > 0);
+    ASSERT(ALIGNED(start, PAGE_SIZE) && ALIGNED(requested_size, PAGE_SIZE));
+
+    /* we can call protect_virtual_memory only on regions that have
+     * the same attributes, we have to split the requested region into
+     * multiple proper subregions
+     */
+    do {
+        MEMORY_BASIC_INFORMATION mbi;
+        uint old_prot;
+        size_t res;
+        uint flags, new_flags;
+        size_t allow_size;        /* remaining size with same privileges */
+        size_t subregion_size;    /* should be a subregion <= allow_size */
+
+        ASSERT(remaining_size > 0);
+
+        /* FIXME: note that a faster version of this routine when we
+         * know the desired flags can do without the
+         * query_virtual_memory() calls and only needs to process the
+         * results of protect_virtual_memory() to decide whether needs
+         * more iterations.
+         */
+
+        /* needed for current flags and region size */
+        res = query_virtual_memory((PBYTE)pc, &mbi, sizeof(mbi));
+        if (res != sizeof(mbi)) {
+            /* can get here if executing from kernel address space - case 9022 */
+            goto finish;
+        }
+        ASSERT(res == sizeof(mbi));
+        ASSERT(mbi.State != MEM_FREE);
+        ASSERT(mbi.State == MEM_COMMIT || mbi.State == MEM_RESERVE);
+        ASSERT(ALIGNED(pc, PAGE_SIZE) && ALIGNED(remaining_size, PAGE_SIZE));
+        ASSERT(first_allocation_base == NULL ||
+               first_allocation_base == mbi.AllocationBase);
+        DODEBUG({first_allocation_base = mbi.AllocationBase;});
+        ASSERT(pc == mbi.BaseAddress); /* if pc is page aligned, but just in case */
+        allow_size = mbi.RegionSize - (pc - (byte *)mbi.BaseAddress);
+
+        /* to maintain old prot flags, 
+         * we have to do each os region separately */
+        if (remaining_size > allow_size) {
+            LOG(THREAD_GET, LOG_VMAREAS, 2,
+                "WARNING: make_%swritable "PFX": param size "PIFX" vs. "
+                "mbi size "PIFX" base "PFX"\n",
+                writable ? "" : "un", pc, remaining_size, mbi.RegionSize, mbi.BaseAddress);
+            /* we see this on make_writable when we've merged regions that we made read-only
+             * and we go to restore their permissions.
+             * we can see it for the same region many times in a row (e.g., on javac in SPECJVM98),
+             */
+            /* flag in GLOBAL LOG */
+            LOG(GLOBAL, LOG_VMAREAS, pc == start ? 1U : 2U, 
+                "make_%swritable called with size "PFX
+                "> region size "PFX" at pc "PFX"\n",
+                writable ? "" : "un", remaining_size, allow_size, pc);
+            /* needed most commonly when a PAGE_WRITECOPY breaks up a
+             * region or when MEM_RESERVE subregion is processed,
+             * for the time being adding a curiosity on any other use */
+
+            /* for this invocation, just do region size */
+            subregion_size = allow_size;
+        } else {
+            subregion_size = remaining_size;
+        }
+
+        ASSERT( subregion_size <= allow_size);
+
+        LOG(THREAD_GET, LOG_VMAREAS, 3, "make_%swritable: pc "PFX"-"PFX
+            ", currently %s %s\n",
+            writable ? "" : "un", pc, pc+subregion_size, prot_string(mbi.Protect), 
+            mbi.State == MEM_COMMIT ? "committed" : "reserved");
+        /* mbi.Protect is defined only for mbi.State == MEM_COMMIT 
+         * we use gratuitously in this LOG
+         */
+
+        if (mbi.State == MEM_RESERVE) {
+            LOG(THREAD_GET, LOG_VMAREAS, 2, 
+                "make_%swritable: WARNING skipping reserved region "PFX"-"PFX"\n",
+                pc, pc+subregion_size);
+            /* There is nothing we can do about reserved memory.
+             * Assume nobody will really reference this uncomitted
+             * memory, and in case it is caller error, that we'll find
+             * out on write.
+             */
+            goto skip;
+        }
+        if (mbi.State == MEM_FREE) {
+            /* now this is always supposed to be an error */
+            ASSERT_NOT_REACHED();
+            subregions_failed = true;
+            goto finish;
+        }
+
+        flags = mbi.Protect & ~PAGE_PROTECTION_QUALIFIERS;
+
+        if (set) {
+            new_flags = new_prot;
+        } else if (writable) {
+            switch (flags) {
+            case PAGE_NOACCESS:          new_flags = PAGE_READWRITE; break;
+            case PAGE_READONLY:          new_flags = PAGE_READWRITE; break;
+            case PAGE_READWRITE:         goto skip;
+            case PAGE_WRITECOPY:         goto skip;
+            case PAGE_EXECUTE:           new_flags = PAGE_EXECUTE_READWRITE; break;
+            case PAGE_EXECUTE_READ:      new_flags = PAGE_EXECUTE_READWRITE; break;
+            case PAGE_EXECUTE_READWRITE: goto skip;
+            case PAGE_EXECUTE_WRITECOPY: goto skip;
+            default:
+                ASSERT_NOT_REACHED();
+                /* not possible since we handle MEM_RESERVE earlier */
+                /* do not attempt changing permissions to be robust */
+                goto skip;
+            }
+            if (cow)
+                new_flags = osprot_add_writecopy(new_flags);
+        } else {
+            switch (flags) {
+            case PAGE_NOACCESS:          goto skip;
+            case PAGE_READONLY:          goto skip;
+            case PAGE_READWRITE:         new_flags = PAGE_READONLY; break;
+            case PAGE_WRITECOPY:         new_flags = PAGE_READONLY; break;
+            case PAGE_EXECUTE:           goto skip;
+            case PAGE_EXECUTE_READ:      goto skip;
+            case PAGE_EXECUTE_READWRITE: new_flags = PAGE_EXECUTE_READ; break;
+            case PAGE_EXECUTE_WRITECOPY: new_flags = PAGE_EXECUTE_READ; break;
+            default:
+                ASSERT_NOT_REACHED();
+                goto skip;
+            }
+        }
+        /* preserve other flags */
+        new_flags = (mbi.Protect & ~flags) | new_flags;
+
+        DOSTATS({
+            /* once on each side of prot, to get on right side of writability */
+            if (!writable) {
+                STATS_INC(protection_change_calls);
+                STATS_ADD(protection_change_pages, subregion_size / PAGE_SIZE);
+            }
+        });
+        res = protect_virtual_memory((void *)pc, subregion_size, new_flags, &old_prot);
+        if (!res) {
+            /* FIXME: we may want to really make sure that we are out
+             * of commit memory, if we are marking this up as failure
+             * here
+             */
+            subregions_failed = true;
+            /* FIXME: case 10551 we may want to use the techniques in
+             * vmm_heap_commit to wait a little for someone else to
+             * free up memory, or free any of our own.
+             */
+        }
+        /* we ignore any failures due to TOCTOU races on subregion protection */
+        ASSERT_CURIOSITY(res && "protect_virtual_memory failed");
+        DOSTATS({
+            /* once on each side of prot, to get on right side of writability */
+            if (writable) {
+                STATS_INC(protection_change_calls);
+                STATS_ADD(protection_change_pages, subregion_size / PAGE_SIZE);
+            }
+        });
+        changed_permissions = true;
+    skip:
+        pc += subregion_size;
+        remaining_size -= subregion_size;
+    } while ( remaining_size > 0);
+
+ finish:
+    if (changed_protection != NULL)
+        *changed_protection = changed_permissions;
+    return !subregions_failed;
+}
+
+/* Set protections on memory region starting at pc of length size 
+ * (padded to page boundaries).
+ * returns false on failure, e.g. out of commit memory
+ */
+bool
+set_protection(byte *pc, size_t size, uint prot)
+{
+    byte *start_page = (byte *)ALIGN_BACKWARD(pc, PAGE_SIZE);
+    size_t num_bytes = ALIGN_FORWARD(size + (pc - start_page), PAGE_SIZE);
+    return internal_change_protection(start_page, num_bytes, true/*set*/,
+                                      false/*ignored*/, false/*ignored*/,
+                                      memprot_to_osprot(prot), NULL);
+}
+
+/* Change protections on memory region starting at pc of length size 
+ * (padded to page boundaries).  This method is meant to be used on DR memory 
+ * as part of protect from app and is safe with respect to stats and the data
+ * segment.
+ */
+bool
+change_protection(byte *pc, size_t size, bool writable)
+{
+    byte *start_page = (byte *)ALIGN_BACKWARD(pc, PAGE_SIZE);
+    size_t num_bytes = ALIGN_FORWARD(size + (pc - start_page), PAGE_SIZE);
+    return internal_change_protection(start_page, num_bytes, false/*relative*/, 
+                                      writable, false/*not cow*/, 0, NULL);
+}
+
+/* makes pc-:c+size (page_padded) writable preserving other flags */
+bool
+make_hookable(byte *pc, size_t size, bool *changed_prot)
+{
+    byte *start_pc = (byte *)ALIGN_BACKWARD(pc, PAGE_SIZE);
+    size_t num_bytes = ALIGN_FORWARD(size + (pc - start_pc), PAGE_SIZE);
+    ASSERT(changed_prot != NULL);
+    return internal_change_protection(start_pc, num_bytes,
+                                      false/*relative*/, true, false/*not cow*/, 0, 
+                                      changed_prot);
+}
+
+/* if changed_prot makes pc:pc+size (page padded) unwritable preserving 
+ * other flags */
+void
+make_unhookable(byte *pc, size_t size, bool changed_prot)
+{
+    if (changed_prot) {
+        byte *start_pc = (byte *)ALIGN_BACKWARD(pc, PAGE_SIZE);
+        size_t num_bytes = ALIGN_FORWARD(size + (pc - start_pc), PAGE_SIZE);
+        internal_change_protection(start_pc, num_bytes, false/*relative*/, 
+                                   false, false/*ignored*/, 0, NULL);
+    }
+}
+
+/* requires that pc is page aligned and size is multiple of the page size
+ * and marks that memory writable, preserves other flags */
+/* returns false if out of commit memory! */
+bool
+make_writable(byte *pc, size_t size)
+{
+    return internal_change_protection(pc, size, false/*relative*/, true,
+                                      false/*not cow*/, 0, NULL);
+}
+
+/* requires that pc is page aligned and size is multiple of the page size
+ * and marks that memory writable and copy-on-write, preserving other flags.
+ * note: only usable if allocated COW
+ */
+bool
+make_copy_on_writable(byte *pc, size_t size)
+{
+    return internal_change_protection(pc, size, false/*relative*/, true,
+                                      true/*cow*/, 0, NULL);
+}
+
+/* requires that pc is page aligned and size is multiple of the page size
+ * and marks that memory NOT writable, preserves other flags */
+void
+make_unwritable(byte *pc, size_t size)
+{
+    internal_change_protection(pc, size, false/*relative*/, false, false/*ignored*/,
+                               0, NULL);
+}
+
+static bool
+convert_to_NT_file_path(OUT wchar_t *buf, IN const char *fname,
+                        IN size_t buf_len/*# elements*/)
+{
+    bool is_UNC = false;
+    const char *name = fname;
+    uint i;
+    /* need nt file path, prepend \??\ so is \??\c:\.... make sure everyone
+     * gives us a fullly qualified absolute path, no . .. relative etc.
+     * For UNC names(//server/name), the path should be \??\UNC\server\name. */
+    /* NOTE - for process control we use an app path (image location) with this routine
+     * so we should handle all possible file name prefixes, we've seen - 
+     * c:\ \??\c:\ \\?\c:\ \\server \??\UNC\server \\?\UNC\server */
+    /* FIXME - could we ever get any other path formats here (xref case 9146 and the
+     * reactos src.  See DEVICE_PATH \\.\foo, UNC_DOT_PATH \\., etc. */ 
+    /* CHECK - at the api level, paths longer then MAX_PATH require \\?\ prefix, unclear
+     * if we would need to use that at this level instead of \??\ for long paths (not
+     * that it matters since our buffer in this routine limits us to MAX_PATH anyways).
+    /* FIXME - handle . and .. */
+    /* FIMXE : there is also ntdll!RtlDosPathNameToNtPathName_U that does the
+     * translation for us, used by CreateDirectory CreateFile etc. but looking
+     * at the dissasembly it grabs the loader lock! why does it need
+     * to do that? is it to translate . or ..?, better just to do the 
+     * translation here where we know what's going on */
+    if (name[0] == '\\') {
+        name += 1; /* eat the first \ */
+        if (name[0] == '\\') {
+            if (name[1] == '?') {
+                /* is \\?\UNC\server or \\?\c:\ type,
+                 * chop off the \\?\ and we'll check for the UNC later */
+                ASSERT_CURIOSITY(name[2] == '\\' && "create file invalid name");
+                /* safety check, don't go beyond end of string */
+                if (name[2] != '\0') {
+                    name += 3;
+                } else {
+                    return false;
+                } 
+            } else {
+                /* is \\server type */
+                is_UNC = true;
+            }
+        } else {
+            /* is \??\UNC\server for \??\c:\ type
+             * chop off the \??\ and we'll check for the UNC later */
+            ASSERT_CURIOSITY(name[0] == '?' && name[1] == '?' && name[2] == '\\' &&
+                             "create file invalid name");
+            /* safety check, don't go beyond end of string */
+            if (name[0] != '\0' && name[1] != '\0' && name[2] != '\0') {
+                name += 3;
+            } else {
+                return false;
+            } 
+        }
+        if (!is_UNC) {
+            /* we've eaten the initial \\?\ or \??\ check for UNC */
+            if ((name[0] == 'U' || name[0] == 'u') &&
+                (name[1] == 'N' || name[1] == 'n') &&
+                (name[2] == 'C' || name[2] == 'c')) {
+                /* is \??\UNC\server or \\?\UNC\server type, chop of the UNC 
+                 * (we'll re-add below)
+                 * NOTE '/' is not a legal separator for a \??\ or \\?\ path */
+                ASSERT_CURIOSITY(name[3] == '\\' && "create file invalid name");
+                is_UNC = true;
+                name += 3;
+            } else {
+                /* is \??\c:\ or \\?\c:\ type,
+                 * NOTE '/' is not a legal separator for a \??\ or \\?\ path */
+                ASSERT_CURIOSITY(name[1] == ':' && name[2] == '\\' &&
+                                 "create file invalid name");
+            }
+        }
+    } else {
+        /* is c:\ type, NOTE case 9329 c:/ is also legal */
+        ASSERT_CURIOSITY(name[1] == ':' && (name[2] == '/' || name[2] == '\\') &&
+                         "create file invalid name");
+    }
+
+    /* should now have either ("c:\" and !is_UNC) or ("\server" and is_UNC) */ 
+    snwprintf(buf, buf_len, L"\\??\\%ls%hs",
+              is_UNC ? L"UNC" : L"", name);
+    buf[buf_len-1] = L'\0';
+    /* change / to \ */
+    for (i = 0; i < buf_len; i++) { 
+        if (L'/' == buf[i]) 
+            buf[i] = L'\\';
+    }
+    return true;
+}
+
+static file_t
+os_internal_create_file(const char *fname, bool is_dir, ACCESS_MASK rights, uint sharing,
+                        uint create_disposition)
+{
+    wchar_t buf[MAX_PATH];
+    if (!convert_to_NT_file_path(buf, fname, BUFFER_SIZE_ELEMENTS(buf)))
+        return INVALID_FILE;
+    NULL_TERMINATE_BUFFER(buf); /* be paranoid */
+    return create_file(buf, is_dir,
+                       rights, sharing, create_disposition, true);
+}
+
+static bool
+os_internal_create_file_test(const char *fname, bool is_dir, ACCESS_MASK rights,
+                             uint sharing, uint create_disposition)
+{
+    HANDLE file = os_internal_create_file(fname, is_dir, rights, sharing,
+                                          create_disposition);
+    if (INVALID_FILE == file) {
+        return false;
+    }
+    os_close(file);
+    return true;
+}
+
+bool
+os_file_exists(const char *fname, bool is_dir)
+{
+    return os_internal_create_file_test(fname, is_dir, 0, FILE_SHARE_READ, FILE_OPEN);
+}
+
+/* Returns true and sets 'size' of file on success; returns false on failure.
+ * Note: This size is different from the allocation size of the file, which can
+ * be larger or smaller (if file compression is turned on - case 8272).
+ */
+bool
+os_get_file_size(const char *file, uint64 *size)
+{
+    wchar_t filename[MAXIMUM_PATH + 1];
+    FILE_NETWORK_OPEN_INFORMATION file_info;
+
+    ASSERT(file != NULL && size != NULL); 
+    if (file == NULL || size == NULL)
+        return false;
+
+    /* See FIXME in os_internal_create_file() about prepending \??\ to the path
+     * directly. 
+     */
+     /* FIXME: case 9182 this won't work for remote files */
+     _snwprintf(filename, BUFFER_SIZE_ELEMENTS(filename), L"\\??\\%hs", file);
+     NULL_TERMINATE_BUFFER(filename);
+     if (query_full_attributes_file(filename, &file_info)) {
+        ASSERT(sizeof(*size) == sizeof(file_info.EndOfFile.QuadPart));
+        *size = file_info.EndOfFile.QuadPart;
+        return true;
+     }
+     return false;
+}
+
+bool
+os_get_file_size_by_handle(IN HANDLE file_handle,
+                           uint64 *end_of_file /* OUT */)
+{
+    FILE_STANDARD_INFORMATION standard_info;
+    NTSTATUS res = nt_query_file_info(file_handle,
+                                      &standard_info,
+                                      sizeof(standard_info),
+                                      FileStandardInformation);
+    /* should always be able to get this */
+    ASSERT(NT_SUCCESS(res) && "bad file handle?");
+    if (!NT_SUCCESS(res)) {
+        return false;
+    }
+ 
+    *end_of_file = standard_info.EndOfFile.QuadPart;
+    return true;
+}
+
+bool
+os_set_file_size(IN HANDLE file_handle,
+                 uint64 end_of_file)
+{
+    NTSTATUS res;
+    FILE_END_OF_FILE_INFORMATION file_end_info;
+    ASSERT_CURIOSITY(end_of_file != 0);
+    file_end_info.EndOfFile.QuadPart = end_of_file;
+    res = nt_set_file_info(file_handle,
+                           &file_end_info,
+                           sizeof(file_end_info),
+                           FileEndOfFileInformation);
+    ASSERT(NT_SUCCESS(res) && "can't set size: bad handle?");
+    return NT_SUCCESS(res);
+}
+
+/* returns available and total quota for the current thread's user (if
+ * impersonated), as well as total available on the volume 
+ * Note that any valid handle on the volume can be used.
+ */
+bool
+os_get_disk_free_space(IN HANDLE file_handle,
+                       OUT uint64 *AvailableQuotaBytes OPTIONAL,
+                       OUT uint64 *TotalQuotaBytes OPTIONAL,
+                       OUT uint64 *TotalVolumeBytes OPTIONAL)
+{
+    /* FIXME: considering that we don't usually care about the actual
+     * bytes available on the volume, we may use just
+     * FILE_FS_SIZE_INFORMATION instead of FILE_FS_FULL_SIZE_INFORMATION
+     * case 9000: need to check if both are available on NT
+     */
+
+    /* Windows Driver Kit: Installable File System Drivers ::
+     * FILE_FS_FULL_SIZE_INFORMATION
+     *
+     * "The size of the buffer passed ... must be at least sizeof
+     * (FILE_FS_FULL_SIZE_INFORMATION).  This structure must be
+     * aligned on a LONGLONG (8-byte) boundary.  "
+     *
+     * Although on XP SP2 this call succeeds even on a non-aligned
+     * value, to be sure we'll follow the recommendation.
+     */
+
+    FILE_FS_FULL_SIZE_INFORMATION unaligned_fs_full_size[2];
+    FILE_FS_FULL_SIZE_INFORMATION *FileFsFullSize = 
+        (FILE_FS_FULL_SIZE_INFORMATION *)
+        ALIGN_FORWARD(unaligned_fs_full_size, sizeof(LONGLONG));
+    uint64 BytesPerUnit;
+    NTSTATUS res;
+
+    ASSERT(sizeof(LONGLONG) < sizeof(FILE_FS_FULL_SIZE_INFORMATION));
+    ASSERT(ALIGNED(FileFsFullSize, sizeof(LONGLONG)));
+    res = nt_query_volume_info(file_handle,
+                               FileFsFullSize,
+                               sizeof(*FileFsFullSize),
+                               FileFsFullSizeInformation);
+    if (!NT_SUCCESS(res))
+        return false;
+
+    BytesPerUnit = FileFsFullSize->SectorsPerAllocationUnit *
+        FileFsFullSize->BytesPerSector;
+    if (AvailableQuotaBytes != NULL) {
+        *AvailableQuotaBytes = FileFsFullSize->
+            CallerAvailableAllocationUnits.QuadPart * BytesPerUnit;
+    }
+    if (TotalQuotaBytes != NULL) {
+        *TotalQuotaBytes = FileFsFullSize->
+            TotalAllocationUnits.QuadPart * BytesPerUnit;
+    }
+    if (TotalVolumeBytes != NULL) {
+        *TotalVolumeBytes = FileFsFullSize->
+            ActualAvailableAllocationUnits.QuadPart * BytesPerUnit;
+    }
+
+    return true;
+}
+
+/* NYI: os_copy_file - copies a portion of a file onto another.  Note
+ * that if new_file is non-empty we are overwriting only the
+ * appropriate subregion. os_copy_file() can be used as a full file
+ * copy (with offset 0 in both files).  With an offset os_copy_file()
+ * can be used to overwrite the portions of a file that are not mapped
+ * in memory or are suffixes not at all covered by the PE format.
+ */
+/* NOTE: cf CopyFileEx which also claims to be doing something special
+ * to preserve OLE structured storage?
+ *
+ * NOTE: we do don't support NTFS alternate data streams,
+ * e.g. downloaded.dll:Zone.Identifier since we would expect that any
+ * checks by say Software Restriction Policies are done on the
+ * original file, not on what we really open.
+ *
+ * NOTE we don't preserve extended attributes, file attributes.  If we
+ * care to have these should see see
+ * kernel32!CreateFile(,hTemplateFile) which supplies file attributes
+ * and extended attributes for the new file.
+ *
+ * Note we don't preserve security attributes - see
+ * shell32!SHFileOperation if we need this.
+ *
+ * We don't deal in any way with encrypted files - they are opened
+ * raw.  FIXME: may want to at least make sure that encrypted files
+ * aren't shared.
+ *
+ * FIXME: testing: doublecheck compressed file offsets are properly
+ * used - test both encrypted and compressed folders.
+ */
+bool
+os_copy_file(HANDLE new_file, HANDLE original_file,
+             uint64 new_file_offset, uint64 original_file_offset)
+{
+    /* we don't care to have the fastest filecopy implementation
+     * current uses are rare enough.  See p.64 and 02 FileCopy from
+     * Richter&Clark if a fast one is needed. */
+
+    /* Note that NTFS will make the calls synchronously */
+    /* FIXME: it may be useful to set the expected total file size
+     * right away with os_set_file_size(), but that should be done
+     * only in case the current size is smaller (e.g. we shouldn't
+     * truncate if trying to overwrite a subsection ) */
+    ASSERT_NOT_IMPLEMENTED(false);
+    return false;
+}
+
+bool
+os_create_dir(const char *fname, create_directory_flags_t create_dir_flags)
+{
+    bool require_new = TEST(CREATE_DIR_REQUIRE_NEW, create_dir_flags);
+    bool force_owner = TEST(CREATE_DIR_FORCE_OWNER, create_dir_flags);
+
+    /* case 9057 note that hard links are only between files but not directories */
+    /* upcoming symlinks can be between either, for consistency should
+     * always require_new,
+     * FIXME: not all current users do this properly
+     */
+    return os_internal_create_file_test(fname, true, 0, FILE_SHARE_READ,
+                                        (require_new ? FILE_CREATE : 
+                                                      FILE_OPEN_IF) |
+                                        (force_owner ? FILE_DISPOSITION_SET_OWNER : 0));
+}
+
+file_t
+os_open_directory(const char *fname, int os_open_flags)
+{
+    uint sharing = FILE_SHARE_READ
+        /* case 10255: allow persisted cache file renaming in
+         * directory */
+        | FILE_SHARE_WRITE;
+    uint access = READ_CONTROL;
+
+    /* FIXME: only 0 is allowed by create_file for now */
+    if (TEST(OS_OPEN_READ, os_open_flags))
+        access |= FILE_GENERIC_READ;
+
+    return os_internal_create_file(fname, true, access, sharing,
+                                   FILE_OPEN);
+}
+
+/* FIXME : investigate difference between GENERIC_* and FILE_GENERIC_*
+ * both seem to work as expected (and CreateFile uses the GENERIC_* while
+ * the ddk uses FILE_GENERIC_*) but they resolve differently, some confusion.
+ * ntddk.h has GENERIC_* as a single bit flag while FILE_GENERIC_* is
+ * a combination including FILE_{READ,WRITE}_DATA, so going with the latter.
+ */
+file_t
+os_open(const char *fname, int os_open_flags)
+{
+    uint access = 0;
+    /* FIXME case 8865: should default be no sharing? */
+    uint sharing = FILE_SHARE_READ;
+
+    if (TEST(OS_EXECUTE, os_open_flags))
+        access |= FILE_GENERIC_EXECUTE;
+    if (TEST(OS_OPEN_READ, os_open_flags))
+        access |= FILE_GENERIC_READ;
+
+    if (TEST(OS_SHARE_DELETE, os_open_flags))
+        sharing |= FILE_SHARE_DELETE;
+
+    if (!TEST(OS_OPEN_WRITE, os_open_flags))
+        return os_internal_create_file(fname, false, access, sharing, FILE_OPEN);
+
+    /* clients are allowed to open the file however they want, xref PR 227737 */
+    ASSERT_CURIOSITY_ONCE((TEST(OS_OPEN_REQUIRE_NEW, os_open_flags)
+                           IF_CLIENT_INTERFACE( ||
+                             !IS_INTERNAL_STRING_OPTION_EMPTY(client_lib))) &&
+                          "symlink risk PR 213492");
+    
+    return os_internal_create_file(fname, false,
+                                   access |
+                                   (TEST(OS_OPEN_APPEND, os_open_flags) ? 
+                                    /* FILE_GENERIC_WRITE minus
+                                     * FILE_WRITE_DATA, so we get auto-append
+                                     */
+                                    (STANDARD_RIGHTS_WRITE | FILE_APPEND_DATA |
+                                     FILE_WRITE_ATTRIBUTES | FILE_WRITE_EA) :
+                                    FILE_GENERIC_WRITE),
+                                   sharing,
+                                   (TEST(OS_OPEN_REQUIRE_NEW, os_open_flags) ? 
+                                       FILE_CREATE : 
+                                       (TEST(OS_OPEN_APPEND, os_open_flags) ? 
+                                           FILE_OPEN_IF : 
+                                           FILE_OVERWRITE_IF)) |
+                                   (TEST(OS_OPEN_FORCE_OWNER, os_open_flags) ? 
+                                    FILE_DISPOSITION_SET_OWNER : 0));
+}
+
+void
+os_close(file_t f)
+{
+    close_handle(f);
+}
+
+/* We take in size_t count to match linux, but Nt{Read,Write}File only
+ * takes in a ULONG (==uint), though they return a ULONG_PTR (size_t)
+ */
+ssize_t
+os_write(file_t f, const void *buf, size_t count)
+{
+    /* file_t is HANDLE opened with CreateFile */
+    size_t written = 0;
+    ssize_t out = -1;
+    bool ok;
+    if (f == INVALID_FILE)
+        return out;
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(count)));
+    ok = write_file(f, buf, (uint) count, NULL, &written);
+    if (ok) {
+        ASSERT(written <= INT_MAX && written <= count);
+        out = (ssize_t)written;
+    } else {
+        ASSERT(written == 0);
+    }
+    return out;
+}
+
+/* We take in size_t count to match linux, but Nt{Read,Write}File only
+ * takes in a ULONG (==uint), though they return a ULONG_PTR (size_t)
+ */
+ssize_t
+os_read(file_t f, void *buf, size_t count)
+{
+    /* file_t is HANDLE opened with CreateFile */
+    size_t read = 0;
+    ssize_t out = -1;
+    bool ok;
+    if (f == INVALID_FILE)
+        return out;
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(count)));
+    ok = read_file(f, buf, (uint) count, NULL, &read);
+    if (ok) {
+        ASSERT(read <= INT_MAX && read <= count);
+        out = (ssize_t)read;
+    } else {
+        ASSERT(read == 0);
+    }
+    return out;
+}
+
+void
+os_flush(file_t f)
+{
+    bool ok = flush_file_buffers(f);
+}
+
+/* seek the current file position to offset bytes from origin, return true if succesful */
+bool
+os_seek(file_t f, int64 offset, int origin)
+{
+    FILE_POSITION_INFORMATION info;
+    NTSTATUS res;
+    int64 abs_offset = offset;
+
+    switch (origin) {
+    case OS_SEEK_SET:
+        break;
+    case OS_SEEK_CUR:
+        {
+            int64 cur_pos = os_tell(f);
+            ASSERT(cur_pos != -1 && "bad file handle?"); /* shouldn't fail */
+            abs_offset += cur_pos;
+        }
+        break;
+    case OS_SEEK_END:
+        {
+            uint64 file_size = 0;
+            bool res = os_get_file_size_by_handle(f, &file_size);
+            ASSERT(res && "bad file handle?"); /* shouldn't fail */
+            abs_offset += file_size;
+        }
+        break;
+    default:
+        ASSERT(false && "os_seek: invalid origin");
+        return false;
+    }
+
+    info.CurrentByteOffset.QuadPart = abs_offset;
+    res = nt_set_file_info(f, &info, sizeof(info), FilePositionInformation);
+
+    /* can fail if invalid seek (past end of read only file for ex.) */
+    return NT_SUCCESS(res);
+}
+
+/* return the current file position, -1 on failure */
+int64
+os_tell(file_t f)
+{
+    FILE_POSITION_INFORMATION info;
+    NTSTATUS res = nt_query_file_info(f, &info, sizeof(info), FilePositionInformation);
+
+    /* should always be able to get this */
+    ASSERT(NT_SUCCESS(res) && "bad file handle?");
+    if (!NT_SUCCESS(res)) {
+        return -1;
+    }
+ 
+    return info.CurrentByteOffset.QuadPart;
+}
+
+/* Tries to delete a file that may be mapped in by this or another process.
+ * We use FILE_DELETE_ON_CLOSE, which works only on SEC_COMMIT, not on SEC_IMAGE.
+ * There is no known way to immediately delete a mapped-in SEC_IMAGE file.
+ * Xref case 9964.
+ */
+bool
+os_delete_mapped_file(const char *filename)
+{
+    NTSTATUS res;
+    HANDLE hf;
+    FILE_DISPOSITION_INFORMATION file_dispose_info;
+    bool deleted = false;
+    wchar_t wname[MAX_FILE_NAME_LENGTH];
+
+    if (!convert_to_NT_file_path(wname, filename,
+                                 BUFFER_SIZE_ELEMENTS(wname)))
+        return false;
+    NULL_TERMINATE_BUFFER(wname); /* be paranoid */
+
+    res = nt_create_file(&hf, wname, NULL, 0, SYNCHRONIZE | DELETE,
+                         FILE_ATTRIBUTE_NORMAL,
+                         FILE_SHARE_DELETE | /* if already deleted */
+                         FILE_SHARE_READ,
+                         FILE_OPEN,
+                         FILE_SYNCHRONOUS_IO_NONALERT 
+                         | FILE_DELETE_ON_CLOSE
+                         /* This should open a handle on a symlink rather
+                          * than its target, and avoid other reparse code.
+                          * Otherwise the FILE_DELETE_ON_CLOSE would cause
+                          * us to delete the target of a symlink!
+                          * FIXME: fully test this: case 10067
+                          */
+                         | FILE_OPEN_REPARSE_POINT);
+    if (!NT_SUCCESS(res)) {
+        LOG(GLOBAL, LOG_NT, 2,
+            "os_delete_mapped_file: unable to open handle to %s: "PFX"\n",
+            filename, res);
+        return false;
+    }
+
+    /* Try to delete immediately.  If the file is mapped in, this will fail
+     * with STATUS_CANNOT_DELETE 0xc0000121.
+     */
+    file_dispose_info.DeleteFile = TRUE;
+    res = nt_set_file_info(hf, &file_dispose_info, sizeof(file_dispose_info),
+                           FileDispositionInformation);
+    if (NT_SUCCESS(res))
+        deleted = true;
+    else {
+        LOG(GLOBAL, LOG_NT, 2,
+            "os_delete_mapped_file: unable to mark for deletion %s: "PFX"\n",
+            filename, res);
+        /* continue on */
+    }
+    close_handle(hf);
+    if (!deleted) {
+        /* We can't accurately tell if FILE_DELETE_ON_CLOSE worked but we can try to
+         * open and assume nobody created a new file of the same name.
+         */
+        res = nt_create_file(&hf, wname, NULL, 0, SYNCHRONIZE,
+                             FILE_ATTRIBUTE_NORMAL,
+                             FILE_SHARE_DELETE | FILE_SHARE_READ, FILE_OPEN,
+                             FILE_SYNCHRONOUS_IO_NONALERT | FILE_OPEN_REPARSE_POINT);
+        LOG(GLOBAL, LOG_NT, 2,
+            "os_delete_mapped_file: opening after close %s: "PFX"\n", filename, res);
+        if (NT_SUCCESS(res))
+            close_handle(hf);
+        else if (res == STATUS_DELETE_PENDING ||
+                 res == STATUS_OBJECT_NAME_NOT_FOUND) {
+            deleted = true;
+        } else
+            ASSERT_CURIOSITY(false && "unable to confirm close-on-delete");
+    }
+    /* FIXME case 10048: if failure here, schedule for smss-on-boot deletion */
+    return deleted;
+}
+
+bool
+os_delete_file(const wchar_t *file_name,
+               HANDLE directory_handle)
+{
+    NTSTATUS res;
+    HANDLE hf;
+    FILE_DISPOSITION_INFORMATION file_dispose_info;
+
+    res = nt_create_module_file(&hf, file_name, 
+                                directory_handle,
+                                DELETE,
+                                FILE_ATTRIBUTE_NORMAL,
+                                FILE_SHARE_DELETE | /* if already deleted */
+                                FILE_SHARE_READ |
+                                FILE_SHARE_WRITE,
+                                FILE_OPEN,
+                                0);
+    /* note that FILE_DELETE_ON_CLOSE will act on the target of a
+     * symbolic link (in Longhorn), while we want to act on the link
+     * itself
+     */
+
+    /* this is expected to be called only when a file is in the way */
+    ASSERT_CURIOSITY(NT_SUCCESS(res) && "can't open for deletion");
+    if (!NT_SUCCESS(res))
+        return false;
+
+    file_dispose_info.DeleteFile = TRUE;
+    res = nt_set_file_info(hf,
+                           &file_dispose_info,
+                           sizeof(file_dispose_info),
+                           FileDispositionInformation);
+    /* close regardless of success */
+    close_handle(hf);
+    ASSERT_CURIOSITY(NT_SUCCESS(res) && "couldn't mark for deletion");
+    /* file may have sections mapped (the usual case for DLLs in ASLR cache) */
+    /* we don't expect to be deleting files that are in use by others */
+
+    /* if we had the only handle, the file should be deleted by now */
+    return NT_SUCCESS(res);
+}
+
+/* We take in orig_name instead of a file handle so that we can abstract
+ * away the privileges required to rename a file when opening the handle.
+ * We also do not take in a rootdir handle to be parallel to the linux
+ * system call, so caller must specify full path.
+ * This will not rename a file across volumes.
+ *
+ * see os_rename_file_in_directory() for a Win32-specific interface
+ */
+bool
+os_rename_file(const char *orig_name, const char *new_name, bool replace)
+{
+    file_t fd = INVALID_FILE;
+    NTSTATUS res;
+    FILE_RENAME_INFORMATION info;
+    wchar_t worig[MAX_FILE_NAME_LENGTH];
+
+    if (!convert_to_NT_file_path(info.FileName, new_name,
+                                 BUFFER_SIZE_ELEMENTS(info.FileName)))
+        return false;
+    NULL_TERMINATE_BUFFER(info.FileName); /* be paranoid */
+
+    /* We could use os_open if we added OS_DELETE => DELETE+FILE_OPEN,
+     * but then we couldn't rename directories; ditto for create_file,
+     * so we directly call nt_create_file.
+     */
+    if (!convert_to_NT_file_path(worig, orig_name,
+                                 BUFFER_SIZE_ELEMENTS(worig)))
+        return false;
+    NULL_TERMINATE_BUFFER(worig); /* be paranoid */
+    res = nt_create_file(&fd, worig, NULL, 0, DELETE | SYNCHRONIZE,
+                         FILE_ATTRIBUTE_NORMAL,
+                         /* need F_S_READ if currently open w/ F_S_READ */
+                         FILE_SHARE_READ | FILE_SHARE_DELETE,
+                         FILE_OPEN, /* FILE_SUPERSEDE fails */
+                         /* no FILE_{NON_,}DIRECTORY_FILE */
+                         FILE_SYNCHRONOUS_IO_NONALERT);
+    if (!NT_SUCCESS(res) || fd == INVALID_FILE) {
+        LOG(GLOBAL, LOG_NT, 2,
+            "os_rename_file: unable to open handle to %s: "PFX"\n", orig_name, res);
+        return false;
+    }
+
+    /* I tried three rename options with NtSetFileInformation:
+     * 1) set FileRenameInformation: works on FAT, NTFS, all platforms
+     * 2) set FileNameInformation: not allowed; only for get
+     * 3) set FileShortNameInformation: I couldn't get this to work, but
+     *    was probably missing some privilege; but, only available on NTFS XP+
+     */
+    info.ReplaceIfExists = (BOOLEAN) replace;
+    info.RootDirectory = NULL;
+    IF_X64(ASSERT_TRUNCATE(info.FileNameLength, uint,
+                           wcslen(info.FileName) * sizeof(wchar_t)));
+    info.FileNameLength = (uint) (wcslen(info.FileName) * sizeof(wchar_t));
+    res = nt_set_file_info(fd, &info, sizeof(info), FileRenameInformation);
+    /* Renaming will fail if a file handle (other than this one) is open */
+    if (!NT_SUCCESS(res)) {
+        LOG(GLOBAL, LOG_NT, 2,
+            "os_rename_file: NtSetFileInformation error "PFX"\n", res);
+    }
+    close_handle(fd);
+    return NT_SUCCESS(res);
+}
+
+/* similar to os_rename_file(), but more geared to Windows users
+ * We take in orig_name instead of a file handle, so that we can abstract
+ * away the privileges required to rename a file when opening the handle.
+ * Note however, that any other handle must be closed before calling.
+ * Both names are relative to rootdir handle, since renaming files in
+ * same directory is our primary use.
+ */
+bool
+os_rename_file_in_directory(IN HANDLE rootdir,
+                            const wchar_t *orig_name,
+                            const wchar_t *new_name,
+                            bool replace)
+{
+    file_t fd = INVALID_FILE;
+    NTSTATUS res;
+    FILE_RENAME_INFORMATION info;
+
+    res = nt_create_file(&fd, orig_name, rootdir, 
+                         0, DELETE | SYNCHRONIZE,
+                         FILE_ATTRIBUTE_NORMAL,
+                         /* need F_S_READ if currently open w/ F_S_READ */
+                         FILE_SHARE_READ | FILE_SHARE_DELETE,
+                         FILE_OPEN, /* FILE_SUPERSEDE fails */
+                         /* no FILE_{NON_,}DIRECTORY_FILE */
+                         FILE_SYNCHRONOUS_IO_NONALERT);
+    if (!NT_SUCCESS(res) || fd == INVALID_FILE) {
+        LOG(GLOBAL, LOG_NT, 2,
+            "os_rename_file: unable to open handle to %s: "PFX"\n", orig_name, res);
+        return false;
+    }
+
+    info.ReplaceIfExists = (BOOLEAN) replace;
+    info.RootDirectory = rootdir;
+    wcsncpy(info.FileName, new_name,
+            BUFFER_SIZE_ELEMENTS(info.FileName));
+    NULL_TERMINATE_BUFFER(info.FileName); /* be paranoid */
+    IF_X64(ASSERT_TRUNCATE(info.FileNameLength, uint,
+                           wcslen(info.FileName) * sizeof(wchar_t)));
+    info.FileNameLength = (uint) (wcslen(info.FileName) * sizeof(wchar_t));
+    res = nt_set_file_info(fd, &info, sizeof(info), FileRenameInformation);
+    /* Renaming will fail if a file handle (other than this one) is open */
+    if (!NT_SUCCESS(res)) {
+        LOG(GLOBAL, LOG_NT, 2,
+            "os_rename_file_in_directory: NtSetFileInformation error "PFX"\n", res);
+    }
+    close_handle(fd);
+    return NT_SUCCESS(res);
+}
+
+byte *
+os_map_file(file_t f, size_t size, uint64 offs, app_pc addr, uint prot,
+            bool copy_on_write)
+{
+    NTSTATUS res;
+    HANDLE section;
+    byte *map = addr;
+    size_t view_size = size;
+    uint osprot = memprot_to_osprot(prot);
+    LARGE_INTEGER li_offs;
+    li_offs.QuadPart = offs;
+
+    if (copy_on_write) {
+        /* Ask for COW for both the section and the view, though we should only
+         * need it for the view (except on win98, according to Richter p604)
+         */
+        osprot = osprot_add_writecopy(osprot);
+    }
+    res = nt_create_section(&section,
+                            SECTION_ALL_ACCESS, /* FIXME: maybe less privileges needed */
+                            NULL, /* full file size, even if partial view map */
+                            osprot,
+                            /* can only be SEC_IMAGE if a PE file */
+                            /* FIXME: SEC_RESERVE shouldn't work w/ COW yet
+                             * it did in my test */
+                            SEC_COMMIT,
+                            f,
+                            /* process private - no security needed */
+                            /* object name attributes */
+                            NULL /* unnamed */, 0, NULL, NULL);
+    if (!NT_SUCCESS(res)) {
+        LOG(GLOBAL, LOG_NT, 2, "os_map_file: NtCreateSection error "PFX"\n", res);
+        return NULL;
+    }
+    /* FIXME case 9642: support requesting a particular base address so
+     * we can randomize, make adjacent to vmheap, etc.
+     */
+    res = nt_map_view_of_section(section, /* 0 */
+                                 NT_CURRENT_PROCESS, /* 1 */
+                                 &map, /* 2 */
+                                 0, /* 3 */
+                                 0 /* not page-file-backed */, /* 4 */
+                                 &li_offs, /* 5 */
+                                 (PSIZE_T) &view_size, /* 6 */
+                                 ViewUnmap /* FIXME: expose? */, /* 7 */
+                                 0 /* no special top-down or anything */, /* 8 */
+                                 osprot); /* 9 */
+    /* We do not need to keep the section handle open */
+    close_handle(section);
+    if (!NT_SUCCESS(res)) {
+        LOG(GLOBAL, LOG_NT, 2, "os_map_file: NtMapViewOfSection error "PFX"\n", res);
+        return NULL;
+    }
+    return map;
+}
+
+bool
+os_unmap_file(byte *map, size_t size/*unused*/)
+{
+    int res = nt_unmap_view_of_section(NT_CURRENT_PROCESS, map);
+    return NT_SUCCESS(res);
+}
+
+/* FIXME : should check context flags, what if only integer or only control! */
+/* Translates the context cxt for the given thread trec
+ * Like any instance where a thread_record_t is used by a thread other than its
+ * owner, the caller must hold the thread_initexit_lock to ensure that it
+ * remains valid.
+ * Requires thread trec is at_safe_spot().
+ */
+bool
+translate_context(thread_record_t *trec, CONTEXT *cxt, bool restore_memory)
+{
+    dr_mcontext_t mc;
+    bool res;
+    /* ensure we have eip and esp */
+    ASSERT(TESTALL(CONTEXT_CONTROL/*2 bits so ALL*/, cxt->ContextFlags));
+    /* really we should have the full state */
+    ASSERT(TESTALL(CONTEXT_DR_STATE, cxt->ContextFlags));
+    context_to_mcontext(&mc, cxt);
+    res = translate_mcontext(trec, &mc, restore_memory);
+    if (res)
+        mcontext_to_context(cxt, &mc);
+    return res;
+}
+
+/* be careful about args: for windows different versions have different offsets
+ * see SYSCALL_PARAM_OFFSET in win32/os.c
+ */
+static void
+set_mcontext_for_syscall(dcontext_t *dcontext, int sys_enum,
+#ifdef X64
+                         reg_t arg1, reg_t arg2, reg_t arg3
+#else
+                         reg_t sys_arg
+#endif
+                         )
+{
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+#ifdef X64
+    LOG(THREAD, LOG_SYSCALLS, 2,
+        "issue_last_system_call_from_app(0x%x, "PFX" "PFX" "PFX")\n",
+        syscalls[sys_enum], arg1, arg2, arg3);
+#else
+    LOG(THREAD, LOG_SYSCALLS, 2,
+        "issue_last_system_call_from_app(0x%x, "PFX")\n", syscalls[sys_enum], sys_arg);
+#endif
+
+    mc->xax = syscalls[sys_enum];
+    if (get_syscall_method() == SYSCALL_METHOD_WOW64) {
+        mc->xcx = wow64_index[sys_enum];
+    }
+#ifdef X64
+    mc->xcx = arg1;
+    mc->xdx = arg2;
+    mc->r8 = arg3;
+#else
+    mc->xdx = sys_arg;
+#endif
+}
+
+/* raise an exception in the application context */
+/* FIXME : see os_forge_exception's call of this function for issues */
+void
+os_raise_exception(dcontext_t *dcontext, 
+                   EXCEPTION_RECORD* pexcrec, CONTEXT* pcontext)
+{
+#ifdef X64
+    set_mcontext_for_syscall(dcontext, SYS_RaiseException,
+                             (reg_t)pexcrec, (reg_t)pcontext, (reg_t)true);
+#else
+    /* ZwRaiseException arguments */
+    struct _raise_exception_arguments_t {
+        PEXCEPTION_RECORD ExceptionRecord;
+        PCONTEXT Context;
+        DWORD SearchFrames;
+    } raise_exception_arguments = {pexcrec, pcontext, true};
+    /* NOTE this struct stays on dstack when the syscall is executed! */
+
+    /* args are on our stack so offset bytes are valid, we won't return
+     * here so is ok if os clobbers them, though it won't since natively
+     * they hold return addresses */
+    reg_t arg_pointer = (reg_t)
+        ((ptr_uint_t)&raise_exception_arguments) - SYSCALL_PARAM_OFFSET();
+
+    set_mcontext_for_syscall(dcontext, SYS_RaiseException, arg_pointer),
+#endif
+    issue_last_system_call_from_app(dcontext);
+    ASSERT_NOT_REACHED();
+}
+
+/***************************************************************************
+ * CORE DUMPS
+ */
+/* all static vars here are not persistent across cache execution, so unprot */
+START_DATA_SECTION(NEVER_PROTECTED_SECTION, "w");
+
+static char dump_core_buf[256] VAR_IN_SECTION(NEVER_PROTECTED_SECTION)
+     = {0,}; /* protected by dumpcore_lock */
+static char dump_core_file_name[MAXIMUM_PATH] VAR_IN_SECTION(NEVER_PROTECTED_SECTION)
+     = {0,}; /* protected by dumpcore_lock */
+
+static void
+os_dump_core_segment_info(file_t file, HANDLE h, ULONG selector, const char *name)
+{
+    NTSTATUS res;
+    DESCRIPTOR_TABLE_ENTRY entry = {0,};
+
+    entry.Selector = selector;
+    res = query_seg_descriptor(h, &entry);
+    /* This feature from PR 212905 does not work on x64 b/c there is no
+     * support for the underlying system call: we get STATUS_NOT_IMPLEMENTED.
+     */
+    if (NT_SUCCESS(res)) {
+        snprintf(dump_core_buf, BUFFER_SIZE_ELEMENTS(dump_core_buf),
+                 "%s=0x%04x (0x%08x 0x%08x)\n", name, entry.Selector,
+                 /* print the raw bits in the descriptor */
+                 *((PULONG)&entry.Descriptor), *(((PULONG)&entry.Descriptor)+1));
+    } else {
+        snprintf(dump_core_buf, BUFFER_SIZE_ELEMENTS(dump_core_buf),
+                 "%s=0x%04x\n", name, entry.Selector);
+    }        
+
+    NULL_TERMINATE_BUFFER(dump_core_buf);
+    os_write(file, dump_core_buf, strlen(dump_core_buf));
+}
+
+static void
+os_dump_core_dump_thread(file_t file, thread_id_t tid, TEB *teb, HANDLE h,
+                         int handle_rights, CONTEXT *cxt,
+                         dcontext_t *dcontext)
+{
+    app_pc win32_start_addr = 0;
+
+    /* for x64, FIXME PR 249988: need to coordinate w/ ldmp.c */
+
+    snprintf(dump_core_buf, BUFFER_SIZE_ELEMENTS(dump_core_buf),
+             "Thread="PFX"\nTEB="PFX"\n"
+             "HandleRights=0x%08x\n"
+             "Eax="PFX", Ebx="PFX", Ecx="PFX", Edx="PFX"\n"
+             "Esi="PFX", Edi="PFX", Esp="PFX", Ebp="PFX"\n"
+             "EFlags="PFX", Eip="PFX"\n",
+             tid, teb, handle_rights,
+             cxt->CXT_XAX, cxt->CXT_XBX, cxt->CXT_XCX, cxt->CXT_XDX, cxt->CXT_XSI, 
+             cxt->CXT_XDI, cxt->CXT_XSP, cxt->CXT_XBP, cxt->CXT_XFLAGS, cxt->CXT_XIP);
+    NULL_TERMINATE_BUFFER(dump_core_buf);
+    os_write(file, dump_core_buf, strlen(dump_core_buf));
+
+    /* print segment selectors and associated descriptors */
+    os_dump_core_segment_info(file, h, cxt->SegCs, "Cs");
+    os_dump_core_segment_info(file, h, cxt->SegSs, "Ss");
+    os_dump_core_segment_info(file, h, cxt->SegDs, "Ds");
+    os_dump_core_segment_info(file, h, cxt->SegEs, "Es");
+    os_dump_core_segment_info(file, h, cxt->SegFs, "Fs");
+    os_dump_core_segment_info(file, h, cxt->SegGs, "Gs");
+
+    /* Print the win32 start address.  This is saved away in the
+     * dcontext when the thread is created.
+     */
+    if (dcontext != NULL) {
+        win32_start_addr = dcontext->win32_start_addr;
+    }
+    /* if the dcontext is unavailable, use the syscall */
+    else {
+        NTSTATUS res = query_win32_start_addr(h, &win32_start_addr);
+        ASSERT(NT_SUCCESS(res) && "failed to obtain win32 start address");
+    }
+    snprintf(dump_core_buf, BUFFER_SIZE_ELEMENTS(dump_core_buf),
+             "Win32StartAddr="PFX"\n", win32_start_addr);
+    NULL_TERMINATE_BUFFER(dump_core_buf);
+    os_write(file, dump_core_buf, strlen(dump_core_buf));
+}
+
+#pragma warning( push )
+/* warning is from GET_OWN_CONTEXT: flow in/out of asm code suppresses global opt */
+#pragma warning( disable : 4740)
+static void
+os_dump_core_live_dump(const char *msg)
+{
+    /* like the dump_core_buf, all the locals are protected by the
+     * dumpcore_lock and are static to save stack space (CONTEXT is quite
+     * sizable) */
+    static file_t dmp_file VAR_IN_SECTION(NEVER_PROTECTED_SECTION) = INVALID_FILE;
+    static thread_record_t *tr VAR_IN_SECTION(NEVER_PROTECTED_SECTION) = NULL;
+    static thread_record_t *my_tr VAR_IN_SECTION(NEVER_PROTECTED_SECTION) = NULL;
+    static int i VAR_IN_SECTION(NEVER_PROTECTED_SECTION) = 0;
+    static thread_id_t my_id VAR_IN_SECTION(NEVER_PROTECTED_SECTION) = 0;
+    static bool have_all_threads_lock VAR_IN_SECTION(NEVER_PROTECTED_SECTION) = false;
+    static PBYTE pb VAR_IN_SECTION(NEVER_PROTECTED_SECTION) = NULL;
+    static MEMORY_BASIC_INFORMATION mbi VAR_IN_SECTION(NEVER_PROTECTED_SECTION) = {0,};
+    static CONTEXT cxt VAR_IN_SECTION(NEVER_PROTECTED_SECTION) = {0,};
+    DEBUG_DECLARE(static bool suspend_failures VAR_IN_SECTION(NEVER_PROTECTED_SECTION)
+                  = false;)
+
+    /* initialize */
+    pb = NULL;
+    have_all_threads_lock = false;
+    my_id = get_thread_id();
+    my_tr = NULL;
+    /* We should eventually add xmm regs to ldmp and use CONTEXT_DR_STATE here
+     * (xref PR 264138) */
+    cxt.ContextFlags = CONTEXT_CONTROL|CONTEXT_INTEGER|CONTEXT_SEGMENTS;
+    
+    /* get logfile */
+    /* use no option synch for syslogs to avoid grabbing locks and risking
+     * deadlock, caller should have synchronized already anyways */
+    if (!get_unique_logfile(".ldmp", dump_core_file_name, 
+                            sizeof(dump_core_file_name), false, &dmp_file) || 
+        dmp_file == INVALID_FILE) {
+        SYSLOG_INTERNAL_NO_OPTION_SYNCH(SYSLOG_WARNING, 
+                                        "Unable to open core dump file");
+        return;
+    }
+
+    /* Write message */
+    if (msg != NULL) {
+        size_t length = strlen(msg);
+        /* we start with length of message to make parsing easier */
+        snprintf(dump_core_buf, BUFFER_SIZE_ELEMENTS(dump_core_buf),
+                 PFX"\n", length+1 /* +1 for the \n */);
+        NULL_TERMINATE_BUFFER(dump_core_buf);
+        os_write(dmp_file, dump_core_buf, strlen(dump_core_buf));
+        os_write(dmp_file, msg, length);
+        os_write(dmp_file, "\n", 1);
+    }
+    
+    /* synch with all threads */
+    /* Don't use get_list_of_threads, it grabs a lock and allocates memory
+     * both of which might be dangerous on this path, instead walk table
+     * by hand (we try to grab the neccesary locks, but we will go ahead 
+     * and walk the table if we can't FIXME)
+     * FIXME : share with dynamo.c */
+    /* Try to grab locks,
+     * NOTE os_dump_core, already turned off deadlock_avoidance for us */
+#ifdef DEADLOCK_AVOIDANCE
+    /* ref case 4174, deadlock avoidance will assert if we try to grab a lock
+     * we already own, even if its only a trylock and even if the option is
+     * turned off! We hack around it here */
+    if (all_threads_lock.owner == get_thread_id()) {
+        LOG(GLOBAL, LOG_ALL, 1, 
+            "WARNING : live dump, faulting thread already owns the all_threads lock, let's hope things are consistent\n");
+    } else {
+#endif
+        for(i=0; i < 100 /* arbitrary num */; i++) {
+            if (mutex_trylock(&all_threads_lock)) {
+                have_all_threads_lock = true;
+                break;
+            } else {
+                thread_yield();
+            }
+        }
+        DODEBUG({
+            if (!have_all_threads_lock) {
+                LOG(GLOBAL, LOG_ALL, 1, 
+                    "WARNING : live dump unable to grab all_threads lock, "
+                    "continuing without it\n");
+            }
+        });
+#ifdef DEADLOCK_AVOIDANCE
+    }
+#endif
+    
+    /* print out peb address */
+    snprintf(dump_core_buf, BUFFER_SIZE_ELEMENTS(dump_core_buf),
+             "PEB="PFX"\n", get_own_peb());
+    NULL_TERMINATE_BUFFER(dump_core_buf);
+    os_write(dmp_file, dump_core_buf, strlen(dump_core_buf));
+    
+    /* print out DR address */
+    snprintf(dump_core_buf, BUFFER_SIZE_ELEMENTS(dump_core_buf),
+             "dynamorio.dll="PFX"\n", get_dynamorio_dll_start());
+    NULL_TERMINATE_BUFFER(dump_core_buf);
+    os_write(dmp_file, dump_core_buf, strlen(dump_core_buf));
+
+    /* for all threads, suspend and dump context */
+    /* FIXME : do we care about segment, sse, float, or debug registers? */
+    /* Do current thread first, first get thread record */
+    if (all_threads != NULL) {
+        for (i = 0; i < HASHTABLE_SIZE(ALL_THREADS_HASH_BITS); i++) {
+            for (tr = all_threads[i]; tr != NULL; tr = tr->next) {
+                if (tr->id == my_id)
+                    my_tr = tr;
+            }
+        }
+    }
+    GET_OWN_CONTEXT(&cxt);
+    os_dump_core_dump_thread(dmp_file, my_id, get_own_teb(), NT_CURRENT_THREAD,
+                             my_tr != NULL ? 
+                              nt_get_handle_access_rights(my_tr->handle) : 0, 
+                             &cxt,
+                             my_tr != NULL ? my_tr->dcontext : NULL);
+
+    /* now walk all threads, skipping current thread */
+    if (all_threads != NULL) {
+        for (i = 0; i < HASHTABLE_SIZE(ALL_THREADS_HASH_BITS); i++) {
+            for (tr = all_threads[i]; tr != NULL; tr = tr->next) {
+                if (tr->id != my_id) {
+                    ACCESS_MASK handle_rights = 
+                        nt_get_handle_access_rights(tr->handle);
+                    TEB *teb_addr = get_teb(tr->handle);
+                    DEBUG_DECLARE(bool res = )
+                        thread_suspend(tr);
+                    /* we can't assert here (could infinite loop) */
+                    DODEBUG({ suspend_failures = suspend_failures || !res; });
+                    if (thread_get_context(tr, &cxt)) {
+                        os_dump_core_dump_thread(dmp_file, tr->id, teb_addr, 
+                                                 tr->handle, handle_rights, &cxt,
+                                                 tr->dcontext);
+                    } else {
+                        snprintf(dump_core_buf, 
+                                 BUFFER_SIZE_ELEMENTS(dump_core_buf),
+                                 "Thread=0x%08x\nTEB="PFX"\n"
+                                 "HandleRights=0x%08x\n"
+                                 "<error state not available>\n\n",
+                                 tr->id, teb_addr, handle_rights);
+                        NULL_TERMINATE_BUFFER(dump_core_buf);
+                        os_write(dmp_file, dump_core_buf, strlen(dump_core_buf));
+                    }
+                }
+            }
+        }
+    } else {
+        char *msg = "<error all threads list is already freed>";
+        os_write(dmp_file, msg, strlen(msg));
+        /* FIXME : if other threads are active (say in the case of detaching)
+         * walking the memory below could be racy, what if another thread
+         * frees some chunk of memory while we are copying it! Just live with
+         * the race for now. */
+    }
+
+    /* dump memory */
+    /* FIXME : print_ldr_data() ? */
+    while (query_virtual_memory(pb, &mbi, sizeof(mbi)) == sizeof(mbi)) {
+        snprintf(dump_core_buf, BUFFER_SIZE_ELEMENTS(dump_core_buf), "\n"
+                 "BaseAddress="PFX"\n"
+                 "AllocationBase="PFX"\n"
+                 "AllocationProtect=0x%08x %s\n"
+                 "RegionSize=0x%08x\n"
+                 "State=0x%08x %s\n"
+                 "Protect=0x%08x %s\n"
+                 "Type=0x%08x %s\n",
+                 mbi.BaseAddress,
+                 mbi.AllocationBase,
+                 mbi.AllocationProtect, prot_string(mbi.AllocationProtect),
+                 mbi.RegionSize,
+                 mbi.State, mem_state_string(mbi.State),
+                 mbi.Protect, prot_string(mbi.Protect),
+                 mbi.Type, mem_type_string(mbi.Type));
+        NULL_TERMINATE_BUFFER(dump_core_buf);
+        os_write(dmp_file, dump_core_buf, strlen(dump_core_buf));
+        
+        if (mbi.State == MEM_COMMIT && !TEST(PAGE_GUARD, mbi.Protect) &&
+            prot_is_readable(mbi.Protect)) {
+            os_write(dmp_file, mbi.BaseAddress, mbi.RegionSize);
+        }
+            
+        if (pb + mbi.RegionSize < pb)
+            break;
+        pb += mbi.RegionSize;
+    }
+
+    /* dump handles */
+    {
+        /* see Nebbett examples 1.2 and 2.1, may not be able to do this 
+         * in the general case one methodolgy requires the debug privilege 
+         * the other requires that a global flag is set at boot time 
+         * FIXME */
+    }
+    
+    /* end dump, forensics file will have call stacks and module list */
+    /* unsynch with threads */
+    if (all_threads != NULL) {
+        for (i = 0; i < HASHTABLE_SIZE(ALL_THREADS_HASH_BITS); i++) {
+            for (tr = all_threads[i]; tr != NULL; tr = tr->next) {
+                if (tr->id != my_id) {
+                    /* we assume that if a suspend failed, the corresponding
+                     * resume will also fail -- o/w we could end up resuming
+                     * a thread that a caller suspended!
+                     */
+                    DEBUG_DECLARE(bool res = )
+                        thread_resume(tr);
+                    /* we can't assert here (could infinite loop) */
+                    DODEBUG({ suspend_failures = suspend_failures || !res; });
+                }
+            }
+        }
+    }
+    
+    /* cleanup */
+    if (have_all_threads_lock)
+        mutex_unlock(&all_threads_lock);
+    close_file(dmp_file);
+
+    /* write an event indicating the file was created */
+    SYSLOG_NO_OPTION_SYNCH(SYSLOG_INFORMATION, LDMP, 
+                           3, get_application_name(), get_application_pid(), 
+                           dump_core_file_name);
+
+    DODEBUG({
+        if (suspend_failures) {
+            SYSLOG_INTERNAL_NO_OPTION_SYNCH(SYSLOG_ERROR,
+                "suspend/resume failures during ldmp creation");
+        }
+    });
+}
+#pragma warning( pop )
+
+#ifdef INTERNAL
+static void
+os_dump_core_external_dump()
+{
+    /* static buffers save stack space, this is do-once anyway, protected by
+     * dumpcore_lock from os_dump_core() */
+    static char oncrash_var[MAXIMUM_PATH] VAR_IN_SECTION(NEVER_PROTECTED_SECTION) = {0,};
+    static wchar_t oncrash_cmdline[MAXIMUM_PATH] VAR_IN_SECTION(NEVER_PROTECTED_SECTION)
+        = {0,};
+    static wchar_t oncrash_exe[MAXIMUM_PATH] VAR_IN_SECTION(NEVER_PROTECTED_SECTION)
+        = {0,};
+    
+    /* the ONCRASH key tells us exactly what to launch, with our pid appended */
+    int retval = get_parameter(L_IF_WIN(DYNAMORIO_VAR_ONCRASH), oncrash_var,
+                               sizeof(oncrash_var));
+    if (IS_GET_PARAMETER_SUCCESS(retval)) {
+        HANDLE child;
+        /* ASSUMPTION: no spaces in exe name, should be ok since only developers will
+         * specify a name for this key, everyone else will use tools
+         */
+        char *c = strchr(oncrash_var, ' ');
+        if (c == NULL)
+            c = oncrash_var + strlen(oncrash_var);
+        ASSERT(c - oncrash_var < sizeof(oncrash_exe)/sizeof(wchar_t));
+        snwprintf(oncrash_exe, c - oncrash_var, L"%hs", oncrash_var);
+        oncrash_exe[c - oncrash_var] = L'\0';
+        
+        snwprintf(oncrash_cmdline, sizeof(oncrash_cmdline)/sizeof(wchar_t),
+                  L"%hs %hs", oncrash_var, get_application_pid());
+        NULL_TERMINATE_BUFFER(oncrash_cmdline);
+        
+        SYSLOG_INTERNAL_INFO("Thread %d dumping core via \"%ls\"", get_thread_id(),
+                             oncrash_cmdline);
+        
+        child = create_process(oncrash_exe, oncrash_cmdline);
+        
+        if (child != INVALID_HANDLE_VALUE) {
+            /* wait for child to exit
+             * FIXME: this makes ntsd have to do a 30-second wait to break in!
+             * plus it causes drwtsn32 to hang, then timeout and kill us
+             * w/o producing a dump file -- and only the header on the log file
+             * BUT, if we don't do this, we only get dumps for -kill_thread!
+             */
+            nt_wait_event_with_timeout(child, INFINITE_WAIT);
+            close_handle(child);
+        } else
+            SYSLOG_INTERNAL_WARNING("Unable to dump core via \"%ls\"", oncrash_cmdline);
+    } else {
+        SYSLOG_INTERNAL_WARNING("Unable to dump core due to missing parameter");
+    }
+}
+#endif /* INTERNAL */
+
+void
+os_dump_core(const char *msg)
+{
+    static thread_id_t current_dumping_thread_id VAR_IN_SECTION(NEVER_PROTECTED_SECTION)
+        = 0;
+    thread_id_t current_id = get_thread_id();
+#ifdef DEADLOCK_AVOIDANCE
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    thread_locks_t * old_thread_owned_locks = NULL;
+#endif
+
+    if (current_id == current_dumping_thread_id)
+        return; /* avoid infinite loop */
+
+    /* FIXME : A failure in the mutex_lock or mutex_unlock of the 
+     * dump_core_lock could lead to an infinite recursion, also a failure while
+     * holding the eventlog_lock would lead to a deadlock at the syslog in
+     * livedump (but we would likely deadlock later anyways), all other 
+     * recursion/deadlock cases should be handled by the above check */
+
+#ifdef DEADLOCK_AVOIDANCE
+    /* first turn off deadlock avoidance for this thread (needed for live dump
+     * to try to grab all_threads and thread_initexit locks) */ 
+    if (dcontext != NULL) {
+        old_thread_owned_locks = dcontext->thread_owned_locks;
+        dcontext->thread_owned_locks = NULL;
+    }
+#endif
+
+    /* only allow one thread to dumpcore at a time, also protects static 
+     * buffers and current_dumping_thread_id */
+    mutex_lock(&dump_core_lock);
+    current_dumping_thread_id = current_id;
+
+    if (DYNAMO_OPTION(live_dump)) {
+        os_dump_core_live_dump(msg);
+    }
+    
+#ifdef INTERNAL
+    /* not else-if, allow to be composable */
+    if (DYNAMO_OPTION(external_dump)) {
+        os_dump_core_external_dump();
+    }
+#endif
+    
+    current_dumping_thread_id = 0;
+    mutex_unlock(&dump_core_lock);
+
+#ifdef DEADLOCK_AVOIDANCE
+    /* restore deadlock avoidance for this thread */
+    if (dcontext != NULL) {
+        dcontext->thread_owned_locks = old_thread_owned_locks;
+    }
+#endif
+}
+
+/* back to normal section */
+END_DATA_SECTION()
+/***************************************************************************/
+
+/***************************************************************************/
+/* detaching routines */
+
+/* not static only for a few asserts in other files */
+bool doing_detach = false;
+
+static bool internal_detach = false;
+
+/* Handle any outstanding callbacks.
+ *
+ * For sysenter system calls the kernel callback return returns to a known fixed
+ * location that does a ret.  To regain control we have overwritten the return
+ * address on the stack to point back to the after syscall location and need to restore
+ * the original target here.
+ *
+ * For all other types of system calls the kernel will return the instruction after
+ * the system call which is in our generated code.  We allocate a piece of thread
+ * shared code here followed by an array of thread private detach_callback_stack_ts and
+ * an array of the callback return addresses.  We redirect all after syscall locations
+ * to that shared code which then dispatches on thread_id to find the proper
+ * detach_callback_stack_t, get the right return address from it and then jmp to it.
+ *
+ * Returns true if there are outstanding non-sysenter callbacks.
+ */
+static bool
+detach_helper_handle_callbacks(int num_threads, thread_record_t **threads,
+                               bool *cleanup_tpc /* array of size num_threads */)
+{
+    int i, num_threads_with_callbacks = 0, num_stacked_callbacks = 0;
+
+    /* First walk counts the number of threads with outstanding callbacks and the number
+     * of stacked callbacks (and also fixes the stack for sysenter system calls) so we
+     * now how much memory to allocate for non-sysenter system calls. */
+    for (i = 0; i < num_threads; i++) {
+        dcontext_t *dcontext = threads[i]->dcontext;
+        cleanup_tpc[i] = true; /* default to clean up */
+        if (dcontext->prev_unused != NULL && dcontext->prev_unused->valid) {
+            dcontext_t *tmp_dc = dcontext->prev_unused;
+            int count = 0;
+            LOG(GLOBAL, LOG_ALL, 1, 
+                "Detach : thread %d has stacked callbacks\n", threads[i]->id);
+            do {
+                count++;
+                LOG(GLOBAL, LOG_ALL, 1, "callback %d has ret pc "PFX"\n", 
+                    count, POST_SYSCALL_PC(tmp_dc));
+                ASSERT(POST_SYSCALL_PC(tmp_dc) != NULL &&
+                       !is_dynamo_address(POST_SYSCALL_PC(tmp_dc)));
+                if (get_syscall_method() == SYSCALL_METHOD_SYSENTER && 
+                    INTERNAL_OPTION(detach_fix_sysenter_on_stack)) {
+                    /* Fix up our stack modifications. Since the kernel returns to a
+                     * fixed location this is all we need to do to restore app state.
+                     * Note that shared syscall saves xsp for us, so xsp should be
+                     * correct. */
+                    ASSERT(*((app_pc *)get_mcontext(tmp_dc)->xsp) == 
+                           after_do_syscall_code(dcontext) ||
+                           *((app_pc *)get_mcontext(tmp_dc)->xsp) ==
+                           after_shared_syscall_code(dcontext));
+                    /* fix return address */
+                    LOG(GLOBAL, LOG_ALL, 1,
+                        "callback %d patching stack address "PFX" from "PFX" to "PFX"\n",
+                        get_mcontext(tmp_dc)->xsp, *((app_pc *)get_mcontext(tmp_dc)->xsp),
+                        POST_SYSCALL_PC(tmp_dc));
+                    *((app_pc *)get_mcontext(tmp_dc)->xsp) = POST_SYSCALL_PC(tmp_dc);
+                    if (DYNAMO_OPTION(sygate_sysenter)) {
+                        *((app_pc *)(get_mcontext(tmp_dc)->xsp+XSP_SZ)) =
+                            dcontext->sysenter_storage;
+                    }
+                }
+                tmp_dc = tmp_dc->prev_unused;
+            } while (tmp_dc != NULL && tmp_dc->valid);
+            num_threads_with_callbacks++;
+            num_stacked_callbacks += count;
+            /* can't free thread private syscall code if not SYSENTER since kernel
+             * will return to there */
+            cleanup_tpc[i] = (get_syscall_method() == SYSCALL_METHOD_SYSENTER && 
+                              INTERNAL_OPTION(detach_fix_sysenter_on_stack));
+            LOG(GLOBAL, LOG_ALL, 1, 
+                "Detach : thread %d had %d stacked callbacks\n", threads[i]->id, count);
+        } else {
+            /* no saved callback state, done with this thread */
+            LOG(GLOBAL, LOG_ALL, 1, 
+                "Detach : thread %d has no stacked callbacks\n", threads[i]->id);
+        }
+    }
+    
+    /* Second walk (only needed for non-sysenter systemcalls).  Allocate and populate
+     * the callback dispatch code and data structures. */
+    if (num_stacked_callbacks > 0 &&
+        (get_syscall_method() != SYSCALL_METHOD_SYSENTER ||
+         !INTERNAL_OPTION(detach_fix_sysenter_on_stack))) {
+        /* callback handling buf layout
+         * {
+         *   byte dispatch_code[DETACH_CALLBACK_CODE_SIZE];
+         *   detach_callback_stack_t per_thread[num_threads_with_callbacks]
+         *   app_pc callback_addrs[num_stacked_callbacks]
+         * }
+         * Not a real struct since variable size arrays. Note that nothing requires the
+         * above elements to be in that order (or even in the same allocation).  We
+         * allocate them together to save memory since we must leak this. FIXME - find
+         * a way to free the allocation once we are finished with it. */
+        int callback_buf_size = DETACH_CALLBACK_CODE_SIZE +
+            num_threads_with_callbacks * sizeof(detach_callback_stack_t) +
+            num_stacked_callbacks * sizeof(app_pc);
+        /* FIXME - this should (along with any do/shared syscall containing gencode) be
+         * allocated outside of our vmmheap so that we can free the vmmheap reservation
+         * on detach. */
+        byte *callback_buf = (byte *)heap_mmap(callback_buf_size);
+        detach_callback_stack_t *per_thread =
+            (detach_callback_stack_t *)(callback_buf + DETACH_CALLBACK_CODE_SIZE);
+        app_pc *callback_addrs = (app_pc *)(&per_thread[num_threads_with_callbacks]);
+        int j = 0; /* per_thread index */
+
+        emit_detach_callback_code(GLOBAL_DCONTEXT, callback_buf, per_thread);
+#ifdef X64 /* we only emit shared/do_syscall in shared_code on 64-bit */
+        arch_patch_syscall(GLOBAL_DCONTEXT, callback_buf); /* patch the shared syscalls */
+#endif
+
+        for (i = 0; i < num_threads; i++) {
+            dcontext_t *dcontext = threads[i]->dcontext;
+            if (dcontext->prev_unused != NULL && dcontext->prev_unused->valid) {
+                dcontext_t *tmp_dc = dcontext->prev_unused;
+                
+                arch_patch_syscall(dcontext, callback_buf);
+                emit_detach_callback_final_jmp(dcontext, &(per_thread[j]));
+                per_thread[j].callback_addrs = callback_addrs;
+                per_thread[j].tid = dcontext->owning_thread;
+                per_thread[j].count = 0;
+                
+                /* NOTE - we are walking the stacked dcontexts in reverse order
+                 * (see callback.c, the last dcontext is considered the top of the
+                 * stack).  This is ok since our emitted code expects this. */
+                do {
+                    *callback_addrs++ = POST_SYSCALL_PC(tmp_dc);
+                    ASSERT((byte *)callback_addrs - (byte *)per_thread <=
+                           callback_buf_size);
+                    per_thread[j].count++;
+                    tmp_dc = tmp_dc->prev_unused;
+                } while (tmp_dc != NULL && tmp_dc->valid);
+                
+                j++;
+            }
+        }
+        ASSERT(j == num_threads_with_callbacks);
+        return true;
+    }
+    return false;
+}
+
+/* note not transparent while suspending since suspend count 
+ * will be different, (and number of threads) */
+/* FIXME : ? right now give each thread private code its own top heap_mmap so 
+ * that can be left behind, is this too much of a hit, otherwise ok? */
+void
+detach_helper(int detach_type)
+{
+    thread_record_t **threads;
+    dcontext_t *my_dcontext = get_thread_private_dcontext();
+    int i, num_threads, my_thread_index = -1;
+    thread_id_t my_id;
+    bool res;
+    bool *cleanup_tpc;
+    bool translate_cxt;
+    bool detach_stacked_callbacks;
+    CONTEXT cxt;
+    DEBUG_DECLARE(bool ok;)
+
+    /* Caller (generic_nudge_handler) should have already checked these and
+     * verified the nudge is valid. */
+    ASSERT(dynamo_initialized && !dynamo_exited && my_dcontext != NULL);
+    if (!dynamo_initialized || dynamo_exited || my_dcontext == NULL) {
+        return;
+    }
+    
+    /* enter DR after we have the detach lock for DEADLOCK_AVOIDANCE LIFO 
+     * FIXME: for self-protection, though, we'll need this earlier so we
+     * can write to the detach lock!
+     */
+    ENTERING_DR();
+    
+    /* dynamo_detaching_flag is not really a lock, 
+       and since no one ever waits on it we can't deadlock on it either */
+    if (!atomic_compare_exchange(&dynamo_detaching_flag,
+                                 LOCK_FREE_STATE, LOCK_SET_STATE)) {
+        EXITING_DR();
+        return;
+    }
+    /* we'll need to unprotect for exit cleanup
+     * FIXME: more secure to not do this until we've synched, but then need
+     * alternative prot for doing_detach and init_apc_go_native*
+     */
+    SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+
+    doing_detach = true;
+    
+    if (!internal_detach && !SYNCHRONIZE_DYNAMIC_OPTION(allow_detach)) {
+        doing_detach = false;
+        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+        dynamo_detaching_flag = LOCK_FREE_STATE;
+        SYSLOG_INTERNAL_ERROR("Detach called without the allow_detach option set");
+        EXITING_DR();
+        return;
+    }
+
+    ASSERT(dynamo_initialized);
+    ASSERT(!dynamo_exited);
+    cxt.ContextFlags = CONTEXT_DR_STATE;
+    my_id = get_thread_id();
+
+    ASSERT(detach_type < DETACH_NORMAL_TYPE || 
+           ((my_dcontext != NULL && my_dcontext->whereami == WHERE_FCACHE) ||
+            /* If detaching in thin_client/hotp_only mode, must only be WHERE_APP!  */
+            (RUNNING_WITHOUT_CODE_CACHE() && my_dcontext->whereami == WHERE_APP)));
+
+    LOG(GLOBAL, LOG_ALL, 1, "Detach : thread %d starting\n", my_id);
+    SYSLOG(SYSLOG_INFORMATION, INFO_DETACHING, 2, get_application_name(), 
+           get_application_pid());
+
+    /* synch with flush */
+    if (my_dcontext != NULL)
+        enter_threadexit(my_dcontext);
+
+    /* signal to go native at APC init here, set pause first so that threads
+     * will wait till we are ready for them to go native 
+     * (after ntdll unpatching)*/
+    /* note to avoid races these must be set in this order! */
+    init_apc_go_native_pause = true;
+    init_apc_go_native = true;
+    /* see FIXME below about threads caught between the lock and initialization,
+     * this just reduces the risk */
+    thread_yield();
+
+#ifdef CLIENT_INTERFACE
+    /* make sure client nudges are finished */
+    wait_for_outstanding_nudges();
+#endif
+
+    /* suspend all dynamo controlled threads at safe locations */
+    DEBUG_DECLARE(ok =)
+        synch_with_all_threads(THREAD_SYNCH_SUSPENDED_VALID_MCONTEXT, &threads, 
+                               /* Case 6821: allow other synch-all-thread uses that beat
+                                * us to not wait on us.  We still have a problem
+                                * if we go first since we must xfer other threads.
+                                */
+                               &num_threads, THREAD_SYNCH_NO_LOCKS_NO_XFER,
+                               /* if we fail to suspend a thread (e.g., privilege
+                                * problems) ignore it. FIXME: retry instead? */
+                               THREAD_SYNCH_SUSPEND_FAILURE_IGNORE);
+    ASSERT(ok);
+    /* now we own the thread_initexit_lock */
+    /* NOTE : we will release the locks grabbed in synch_with_all_threads
+     * below after cleaning up all the threads in case we will need to grab
+     * it during process exit cleanup */
+    ASSERT(mutex_testlock(&all_threads_synch_lock) && 
+           mutex_testlock(&thread_initexit_lock));
+
+#ifdef HOT_PATCHING_INTERFACE
+    /* In hotp_only mode, we must remove patches when detaching; we don't want 
+     * to leave in all our hooks and detach; that will definitely crash the app.
+     */
+    if (DYNAMO_OPTION(hotp_only))
+        hotp_only_detach_helper();
+#endif
+    /* FIXME : NYI now all we know about are suspended, should do safety check for 
+     * additional threads here, race condition may be threads that were passed the 
+     * init_apc lock, but not yet initialized and so didn't show up on list */
+    
+    /* FIXME : if we hooked the image entry point and haven't unhooked it yet
+     * need to do so now, can tell from callback hack since see thread with
+     * LOST_CONTROL_AT_CALLBACK in the under_dynamo_control bool */
+    {
+        bool did_unhook = false;
+        for (i = 0; i < num_threads; i++) {
+            if (threads[i]->under_dynamo_control == UNDER_DYN_HACK) {
+                LOG(GLOBAL, LOG_ALL, 1, 
+                    "Detach : unpatching image entry point (from thread %d)\n",
+                    threads[i]->id);
+                ASSERT(!did_unhook); /* should only happen once, at most! */
+                did_unhook = true;
+                remove_image_entry_trampoline();
+            }
+        }
+        if (!did_unhook) {
+            /* case 9347/9475 if detaching before we have taken over the primary thread */
+            if (dr_injected_secondary_thread && !dr_late_injected_primary_thread) {
+                LOG(GLOBAL, LOG_ALL, 1, 
+                    "Detach : unpatching image entry point (from primary)\n");
+                did_unhook = true;
+                /* note that primary thread is unknown and therefore not suspended */
+                remove_image_entry_trampoline();
+            }
+        }
+    }
+
+    /* unpatch ntdll.dll, revert memory protections */
+    LOG(GLOBAL, LOG_ALL, 1, 
+        "Detach : about to unpatch ntdll.dll and fix memory permissions\n");
+    /* FIXME : will go ahead and check option, though detach probably won't work with
+     * noasynch anyways */
+    if (!INTERNAL_OPTION(noasynch)) {
+        callback_interception_unintercept();
+    }
+    if (!DYNAMO_OPTION(thin_client))
+        revert_memory_regions();
+    LOG(GLOBAL, LOG_ALL, 1, 
+        "Detach : unpatched ntdll.dll and fixed memory permissions\n");
+
+    /* release APC init lock, let any threads waiting there go native */
+    LOG(GLOBAL, LOG_ALL, 1, "Detach : Releasing init_apc_go_native_pause\n");
+    init_apc_go_native_pause = false;
+
+    /* perform exit tasks that require full thread data structs */
+    dynamo_process_exit_with_thread_info();
+
+    /* note that no dynamorio code will be run by any other thread than this 
+     * one from now on once APC's blocked at lock clear the method and go 
+     * native */
+    LOG(GLOBAL, LOG_ALL, 1, "Detach : starting to translate contexts\n");
+
+    /* prefer not to do thread cleanup here as can be slow and we are blocking
+     * the whole process, cleanup after resumed, need shadow list to know
+     * if should free thread private code or not
+     */
+    cleanup_tpc = (bool *)global_heap_alloc(num_threads*sizeof(bool)
+                                            HEAPACCT(ACCT_OTHER));
+
+    /* Handle any outstanding callbacks */
+    detach_stacked_callbacks =
+        detach_helper_handle_callbacks(num_threads, threads, cleanup_tpc);
+
+    /* translate current context */
+    for (i = 0; i < num_threads; i++) {
+        dcontext_t *dcontext = threads[i]->dcontext;
+        translate_cxt = true;
+        /* note is safe to check via id since no new thread records have been
+         * created since threads was grabbed */
+        if (threads[i]->id == my_id) {
+            my_thread_index = i;
+            continue;
+        }
+        res = thread_get_context(threads[i], &cxt);
+        ASSERT(res);
+        /* FIXME : callback UNDER_DYN_HACK hack again */
+        if (threads[i]->under_dynamo_control == UNDER_DYN_HACK) {
+            LOG(GLOBAL, LOG_ALL, 1, 
+                "Detach : thread %d running natively since lost control at callback "
+                "return and have not regained it, no need to translate context\n", 
+                threads[i]->id);
+            /* We don't expect to be at do_syscall (and therefore require translation
+             * even though native) since we should've re-taken over by then. */
+            ASSERT(!is_at_do_syscall(dcontext, (app_pc)cxt.CXT_XIP,
+                                     (byte *)cxt.CXT_XSP));
+            translate_cxt = false;
+        }
+        if (translate_cxt) {
+            LOG(GLOBAL, LOG_ALL, 1, 
+                "Detach : recreating address for "PFX"\n", cxt.CXT_XIP);
+            /* fine to call this for native thread, will return cxt right back */
+            res = translate_context(threads[i], &cxt, true/*restore memory*/);
+            ASSERT(res);
+            if (!threads[i]->under_dynamo_control) {
+                LOG(GLOBAL, LOG_ALL, 1, 
+                    "Detach : thread %d already running natively\n", 
+                    threads[i]->id);
+                /* we do need to restore the app ret addr, for native_exec */
+                if (!DYNAMO_OPTION(thin_client) && DYNAMO_OPTION(native_exec) &&
+                    !vmvector_empty(native_exec_areas)) {
+                    /* We store the retaddr location that we clobbered in
+                     * native_exec_retloc.  We don't currently follow callbacks
+                     * so we don't have to do this while walking the callback
+                     * stack below, just for this dcontext.  We also only have a
+                     * single location since we don't re-takeover while native on
+                     * an APC or an exception.
+                     */
+                    app_pc *esp = (app_pc *) threads[i]->dcontext->native_exec_retloc;
+                    app_pc real_retaddr = threads[i]->dcontext->native_exec_retval;
+
+                    /* In hotp_only mode, a thread can be !under_dynamo_control 
+                     * and have no native_exec_retloc.  For hotp_only, there 
+                     * should be no need to restore a return value on the stack 
+                     * as the thread has been native from the start and not 
+                     * half-way through as it would in the regular hot patching 
+                     * mode, i.e., with the code cache.  See case 7681. 
+                     */
+#ifdef HOT_PATCHING_INTERFACE
+                    if (esp == NULL) {
+                        ASSERT(DYNAMO_OPTION(hotp_only));
+                        ASSERT(real_retaddr == NULL);
+                    } else {
+                        ASSERT(!DYNAMO_OPTION(hotp_only));
+#endif
+                        ASSERT(esp != NULL && 
+                               *esp == (app_pc)back_from_native);
+                        ASSERT(real_retaddr != NULL);
+                        *esp = real_retaddr;
+#ifdef HOT_PATCHING_INTERFACE
+                    }
+#endif
+                }
+            }
+            /* handle special case of vsyscall, need to hack the return address
+             * on the stack as part of the translation */
+            if (get_syscall_method() == SYSCALL_METHOD_SYSENTER &&
+                cxt.CXT_XIP == (ptr_uint_t) vsyscall_after_syscall) {
+                ASSERT(get_os_version() >= WINDOWS_VERSION_XP);
+                /* handle special case of vsyscall */
+                /* case 5441 Sygate hack means after_syscall will be at
+                 * esp+4 (esp will point to sysenter_ret_address in ntdll) */
+                if (*(cache_pc *)(cxt.CXT_XSP+
+                                  (DYNAMO_OPTION(sygate_sysenter) ? XSP_SZ : 0)) ==
+                    after_do_syscall_code(dcontext) ||
+                    *(cache_pc *)(cxt.CXT_XSP+
+                                  (DYNAMO_OPTION(sygate_sysenter) ? XSP_SZ : 0)) ==
+                    after_shared_syscall_code(dcontext)) {
+                    LOG(GLOBAL, LOG_ALL, 1, 
+                        "Detach : thread %d suspended at vsysall with ret to after "
+                        "shared syscall, fixing up by changing ret to "PFX"\n", 
+                        threads[i]->id, POST_SYSCALL_PC(dcontext));
+                    /* need to restore sysenter_storage for Sygate hack */
+                    if (DYNAMO_OPTION(sygate_sysenter))
+                        *(app_pc *)(cxt.CXT_XSP+XSP_SZ) = dcontext->sysenter_storage;
+                    *(app_pc *)cxt.CXT_XSP = POST_SYSCALL_PC(dcontext);
+                } else {
+                    LOG(GLOBAL, LOG_ALL, 1,
+                        "Detach, thread %d suspended at vsyscall with ret to "
+                        "unknown addr, must be running native!\n",
+                        threads[i]->id);
+                }
+            }
+            LOG(GLOBAL, LOG_ALL, 1, 
+                "Detach : pc = "PFX" for thread %d\n", cxt.CXT_XIP, threads[i]->id);
+            ASSERT(!is_dynamo_address((app_pc)cxt.CXT_XIP)&&
+                   !in_fcache((app_pc)cxt.CXT_XIP));
+            /* FIXME case 7457: if the thread is suspended after it
+             * received a fault but before the kernel copied the faulting
+             * context to the user mode structures for the handler, it could
+             * result in a codemod exception that wouldn't happen natively!
+             */
+            /* FIXME: switch to using set_synched_thread_context() once we address
+             * the context storage issue
+             */
+            res = thread_set_context(threads[i], &cxt);
+            ASSERT(res);
+        }
+
+#ifdef CLIENT_INTERFACE
+        /* If this is a client-owned thread then there is no app state to return it to
+         * so we kill it here.  Note - we won't kill threads that have returned from
+         * the client nudge routine, but the exit path there doesn't do anything that
+         * would interfere with rest of the detach cleanup. */
+        if (IS_CLIENT_THREAD(dcontext)) {
+            bool terminated = nt_terminate_thread(dcontext->thread_record->handle, 0);
+            ASSERT(terminated); /* Should always be able to terminate. */
+        }
+#endif
+        /* resume thread */
+        LOG(GLOBAL, LOG_ALL, 1, 
+            "Detach : thread %d is being resumed in native context\n", 
+            threads[i]->id);
+        res = thread_resume(threads[i]);
+        ASSERT(res);
+    }
+
+    if (detach_type == DETACH_BAD_STATE_NO_CLEANUP) {
+        SYSLOG_INTERNAL_WARNING("finished detaching, skipping cleanup");
+        /* do a quick exit, skipping all cleanup except eventlog */
+        eventlog_fast_exit();
+        /* We don't even unload our dll since it's no longer required to unload
+         * our dll for proper tools function. */
+        /* FIXME : since we reached detach_helper via a clean call out of the
+         * cache, if we return we will return back into the cache! It would be
+         * cleaner for the thread to die by returning from its start function,
+         * but to avoid complications we just kill it here. */
+        /* NOTE - ref case 4923 (2k3sp1 doesn't free the LdrLock when the owning
+         * thread dies unlike earlier versions).  With the fix for that case we
+         * should no longer be holding any application locks at this point. */
+        nt_terminate_thread(NT_CURRENT_THREAD, 0);
+        ASSERT_NOT_REACHED();
+        return;
+    }
+    
+    /* assert that we found the index of the detaching thread in the 
+     * threads thread_record_t array */
+    ASSERT(detach_type < DETACH_NORMAL_TYPE || my_thread_index != -1);
+    for (i = 0; i < num_threads; i++) {
+        /* clean up threads, including us, but do us last in case cleanup
+         * routines call is_self_* style routines */
+        if (i != my_thread_index) {
+            if (cleanup_tpc[i]) {
+                LOG(GLOBAL, LOG_ALL, 1, 
+                    "Detach : cleaning up thread %d, including its TPC\n", threads[i]->id);
+                dynamo_other_thread_exit(threads[i], false);
+            } else {
+                LOG(GLOBAL, LOG_ALL, 1,
+                    "Detach : cleaning up thread %d, but not its TPC\n",
+                    threads[i]->id);
+                dynamo_other_thread_exit(threads[i], true);
+            }
+        }
+    }
+    /* now free the detaching thread's dcontext */
+    if (my_thread_index != -1)
+        dynamo_other_thread_exit(threads[my_thread_index], false);
+
+    /* free list of threads and cleanup_tpc */
+    global_heap_free(cleanup_tpc, num_threads*sizeof(bool) HEAPACCT(ACCT_OTHER));
+    end_synch_with_all_threads(threads, num_threads, false/*no resume*/);
+
+    /* FIXME : NYI check that any threads waiting at APC have left dynamo code 
+     * and interception code (will be cleaned up in shared_exit), 
+     * potential race condition with unloading the dll, what if is suspended? 
+     */
+    thread_yield();
+
+    LOG(GLOBAL, LOG_ALL, 1, 
+        "Detach :  Last message from detach, about to clean up some more memory and unload\n");
+    SYSLOG_INTERNAL_INFO("Detaching from process, entering final cleanup");
+    /* call dynamo exit routines */
+    res = dynamo_shared_exit(detach_stacked_callbacks); 
+    ASSERT(res == SUCCESS);
+
+    /* we can free the initstack, it can't be our stack, we are specially created thread */
+    stack_free(initstack, DYNAMORIO_STACK_SIZE);
+    dynamo_initialized = false;
+
+    /* FIXME : ? have we freed all space, released all handles */
+    
+    /* CHECK: by now EXITING_DR should have silently happened */
+
+    /* FIXME : unload dll, be able to have thread continue etc. */
+
+    /* FIXME : since we reached detach_helper via a clean call out of the
+     * cache, if we return we will return back into the cache! It would be
+     * cleaner for the thread to die by returning from its start function,
+     * but to avoid complications we just kill it here. */
+    /* NOTE - ref case 4923 (2k3sp1 doesn't free the LdrLock when the owning
+     * thread dies unlike earlier versions).  With the fix for that case we
+     * should no longer be holding any application locks at this point. */
+    nt_terminate_thread(NT_CURRENT_THREAD, 0);
+    ASSERT_NOT_REACHED();
+    return;
+}
+
+/* FIXME : we create a thread to do the detaching, and all other dlls will
+ * be notifed of its creation by dll_thread_attach, is a transparency issue. */
+/* sets detach in motion and then returns */
+void
+detach_internal()
+{
+    SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+    internal_detach = true;
+    /* we go ahead and re-protect though detach thread will soon un-prot */
+    SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+    LOG(GLOBAL, LOG_ALL, 1, "Starting detach\n");
+    nudge_internal(NUDGE_GENERIC(detach), NULL, 0 /* ignored */);
+    LOG(GLOBAL, LOG_ALL, 1, "Created detach thread\n");
+}
+
+/* mcontext must be valid, including the pc field (native) and app_errno
+ * must not be holding any locks */
+/* sets detach in motion and never returns */
+void
+detach_internal_synch()
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    detach_internal();
+    /* to be safe with flush */
+    enter_threadexit(dcontext);
+    /* make sure we spin forever */
+    adjust_wait_at_safe_spot(dcontext, 1);
+    check_wait_at_safe_spot(dcontext, THREAD_SYNCH_VALID_MCONTEXT);
+}
+
+bool
+is_thread_currently_native(thread_record_t *tr)
+{
+    return (!tr->under_dynamo_control ||
+            tr->under_dynamo_control == UNDER_DYN_HACK);
+}
+
+/* contended path of mutex operations */
+
+static contention_event_t
+mutex_get_contended_event(contention_event_t *contended_event, EVENT_TYPE event_type)
+{
+    contention_event_t ret = *contended_event;
+    if (ret == CONTENTION_EVENT_NOT_CREATED)
+    {
+        contention_event_t new_event;
+        bool not_yet_created;
+        /* not signaled */
+        /* EVENT_ALL_ACCESS, although observed access mask of 0x100003 (SYNCHRONIZE|0x3) */
+        new_event = nt_create_event(event_type); 
+
+        not_yet_created =
+            atomic_compare_exchange_ptr((ptr_uint_t*)contended_event, 
+                                        (ptr_uint_t)CONTENTION_EVENT_NOT_CREATED,
+                                        (ptr_uint_t)new_event);
+        if (not_yet_created) {
+            /* we were first to create it */
+            ret = new_event;
+        } else {
+            /* already created by someone else */
+            ret = *contended_event;
+            close_handle(new_event);
+        }
+    }
+    ASSERT(ret != CONTENTION_EVENT_NOT_CREATED);
+    return ret;
+}
+
+/* common wrapper that also attempts to detect deadlocks */
+static void
+os_wait_event(event_t e _IF_CLIENT_INTERFACE(bool set_safe_for_synch)
+              _IF_CLIENT_INTERFACE(dcontext_t *dcontext))
+{
+    wait_status_t res;
+    bool reported_timeout = false;
+
+    KSTART(wait_event);
+    /* we allow using this in release builds as well */
+    if (DYNAMO_OPTION(deadlock_timeout) > 0) {
+        LARGE_INTEGER timeout;
+        timeout.QuadPart= - ((int)DYNAMO_OPTION(deadlock_timeout)) * 
+            TIMER_UNITS_PER_MILLISECOND;
+#ifdef CLIENT_INTERFACE
+        /* if set_safe_for_synch dcontext must be non-NULL */
+        ASSERT(!set_safe_for_synch || dcontext != NULL);
+        if (set_safe_for_synch)
+            dcontext->client_data->client_thread_safe_for_synch = true;
+#endif
+        res = nt_wait_event_with_timeout(e, &timeout /* debug timeout */);
+#ifdef CLIENT_INTERFACE
+        if (set_safe_for_synch)
+            dcontext->client_data->client_thread_safe_for_synch = false;
+#endif
+        if (res == WAIT_SIGNALED) {
+            KSTOP(wait_event);
+            return;             /* all went well */
+        }
+        ASSERT(res == WAIT_TIMEDOUT);
+        /* We could use get_own_peb()->BeingDebugged to determine whether
+         * there was a debugger, but we can't just ignore this. It's better
+         * to explicitly overwrite the hidden DO_ONCE variable from a debugging
+         * session if this is getting in the way.
+         */
+        /* FIXME - instead of DO_ONCE we may want a named static variable that
+         * we can access easily from the debugger. */
+        DO_ONCE({
+            reported_timeout = true;
+            report_dynamorio_problem(NULL, DUMPCORE_TIMEOUT, NULL, NULL,
+                                     "Timeout expired - 1st wait, possible deadlock (or you were debugging)");
+            /* do a 2nd wait so we can get two dumps to compare for progress */
+            /* FIXME - use shorter timeout for the 2nd wait? */
+            res = nt_wait_event_with_timeout(e, &timeout /* debug timeout */);
+            if (res == WAIT_SIGNALED) {
+                /* 2nd wait succeeded! We must not have been really deadlocked.
+                 * Syslog a warning to ignore the first ldmp and continue. */
+                /* FIXME - should we reset the DO_ONCE now? */
+                /* FIXME - should this be a report_dynamorio_problem or some
+                 * such so is more useful in release builds? */
+                SYSLOG_INTERNAL_WARNING("WARNING - 2nd wait after deadlock timeout expired succeeded! Not really deadlocked.");
+                KSTOP(wait_event);
+                return;
+            }
+            ASSERT(res == WAIT_TIMEDOUT);
+            report_dynamorio_problem(NULL, DUMPCORE_TIMEOUT, NULL, NULL,
+                                     "Timeout expired - 2nd wait, possible deadlock (or you were debugging)");
+        });
+    }
+    /* fallback to waiting forever */
+#ifdef CLIENT_INTERFACE
+    if (set_safe_for_synch)
+        dcontext->client_data->client_thread_safe_for_synch = true;
+#endif
+    res = nt_wait_event_with_timeout(e, INFINITE_WAIT);
+#ifdef CLIENT_INTERFACE
+    if (set_safe_for_synch)
+        dcontext->client_data->client_thread_safe_for_synch = false;
+#endif
+    ASSERT(res == WAIT_SIGNALED);
+    if (reported_timeout) {
+        /* Our wait eventually succeeded so not truely a deadlock.  Syslog a
+         * warning to that effect. */
+        /* FIXME - should we reset the DO_ONCE now? */
+        /* FIXME - should this be a report_dynamorio_problem or some
+         * such so is more useful in release builds? */
+        SYSLOG_INTERNAL_WARNING("WARNING - Final wait after reporting deadlock timeout expired succeeded! Not really deadlocked.");
+    }
+    KSTOP(wait_event);
+}
+
+void
+mutex_wait_contended_lock(mutex_t *lock)
+{
+    contention_event_t event = 
+        mutex_get_contended_event(&lock->contended_event, SynchronizationEvent);
+#ifdef CLIENT_INTERFACE
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    bool set_safe_for_sync =
+        (dcontext != NULL && IS_CLIENT_THREAD(dcontext) &&
+         (mutex_t *)dcontext->client_data->client_grab_mutex == lock);
+    ASSERT(!set_safe_for_sync || dcontext != NULL);
+#endif
+    os_wait_event(event _IF_CLIENT_INTERFACE(set_safe_for_sync)
+                  _IF_CLIENT_INTERFACE(dcontext));
+    /* the event was signaled, and this thread was released,
+       the auto-reset event is again nonsignaled for all other threads to wait on
+    */
+}
+
+void
+mutex_notify_released_lock(mutex_t *lock)
+{
+    contention_event_t event = 
+        mutex_get_contended_event(&lock->contended_event, SynchronizationEvent);
+    nt_set_event(event);
+}
+
+void
+rwlock_wait_contended_writer(read_write_lock_t *rwlock)
+{
+    contention_event_t event = 
+        mutex_get_contended_event(&rwlock->writer_waiting_readers, SynchronizationEvent);
+    os_wait_event(event _IF_CLIENT_INTERFACE(false) _IF_CLIENT_INTERFACE(NULL));
+    /* the event was signaled, and this thread was released,
+       the auto-reset event is again nonsignaled for all other threads to wait on
+    */
+}
+
+void
+rwlock_notify_writer(read_write_lock_t *rwlock)
+{
+    contention_event_t event = 
+        mutex_get_contended_event(&rwlock->writer_waiting_readers, SynchronizationEvent);
+    nt_set_event(event);
+}
+
+/* The current implementation uses auto events and will wake up only a
+   single reader.  We then expect each of them to wake up any other ones
+   by properly counting.
+*/
+void
+rwlock_wait_contended_reader(read_write_lock_t *rwlock)
+{
+    contention_event_t notify_readers = 
+        mutex_get_contended_event(&rwlock->readers_waiting_writer, SynchronizationEvent);
+    os_wait_event(notify_readers _IF_CLIENT_INTERFACE(false) _IF_CLIENT_INTERFACE(NULL));
+    /* the event was signaled, and only a single threads waiting on this event are released,
+       if this was indeed the last reader
+    */
+}
+
+void
+rwlock_notify_readers(read_write_lock_t *rwlock)
+{
+    contention_event_t notify_readers = 
+        mutex_get_contended_event(&rwlock->readers_waiting_writer, SynchronizationEvent);
+    /* this will wake up only one since we're using an auto event */
+    nt_set_event(notify_readers);
+}
+
+/***************************************************************************/
+
+event_t
+create_event()
+{
+    return nt_create_event(SynchronizationEvent);
+}
+
+void
+destroy_event(event_t e)
+{
+    nt_close_event(e);
+}
+
+void
+signal_event(event_t e)
+{
+    nt_set_event(e);
+}
+
+void
+reset_event(event_t e)
+{
+    /* should be used only for manual events (NotificationEvent) */
+    nt_clear_event(e);
+}
+
+void
+wait_for_event(event_t e)
+{
+    os_wait_event(e _IF_CLIENT_INTERFACE(false) _IF_CLIENT_INTERFACE(NULL));
+}
+
+timestamp_t
+get_timer_frequency()
+{
+    LARGE_INTEGER ignore_tsc;
+    LARGE_INTEGER freq;
+    timestamp_t processor_speed;
+
+    nt_query_performance_counter(&ignore_tsc, /* not optional */
+                                 &freq);
+    DOLOG(2, LOG_ALL, {
+        timestamp_t tsc;
+        RDTSC_LL(tsc);
+
+        LOG(GLOBAL, LOG_ALL, 2, 
+            "Starting RDTSC: "UINT64_FORMAT_STRING
+            " nt_query_performance_counter: "UINT64_FORMAT_STRING
+            " freq:" UINT64_FORMAT_STRING "\n",
+            tsc, ignore_tsc.QuadPart, freq.QuadPart);
+    });
+
+    processor_speed = freq.QuadPart / 1000; /* convert to KHz */
+    /* case 2937 - windows sometimes is using RTC */
+    if (processor_speed < 500*1000 /* considering 500 MHz too low for a modern machine */) {
+        processor_speed = 2937*1000;
+        LOG(GLOBAL, LOG_ALL, 1, "get_timer_frequency: OS is using RTC!  Reported speed is bogus.\n");
+    }
+    return processor_speed;
+}
+
+uint
+os_random_seed()
+{
+    LARGE_INTEGER tsc_or_rtc;   
+    uint seed = (uint) get_thread_id();
+    seed ^= query_time_millis();
+
+    /* safer to use than RDTSC, since it defaults to real time clock
+     * if TSC is not available, either one is good enough for randomness
+     */
+    nt_query_performance_counter(&tsc_or_rtc, NULL);
+    seed ^= tsc_or_rtc.LowPart;
+    seed ^= tsc_or_rtc.HighPart;
+
+    LOG(GLOBAL, LOG_ALL, 1, "os_random_seed: %d\n", seed);
+    return seed;
+}
+
+void
+early_inject_init()
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    module_handle_t mod;
+    bool under_dr_save;
+    where_am_i_t whereami_save;
+    wchar_t buf[MAX_PATH];
+    int os_version = get_os_version();
+    GET_NTDLL(LdrLoadDll, (IN PCWSTR PathToFile OPTIONAL,
+                           IN PULONG Flags OPTIONAL,
+                           IN PUNICODE_STRING ModuleFileName,
+                           OUT PHANDLE ModuleHandle));
+    ASSERT(dcontext != NULL);
+
+    early_inject_location = DYNAMO_OPTION(early_inject_location);
+
+    /* check for option override of the address */
+    if (DYNAMO_OPTION(early_inject_location) == INJECT_LOCATION_LdrCustom) {
+        early_inject_address = (app_pc)DYNAMO_OPTION(early_inject_address);
+        ASSERT(early_inject_address != NULL);
+        LOG(GLOBAL, LOG_TOP, 1,
+            "early_inject using option provided address "PFX" at location %d\n",
+            early_inject_address, early_inject_location);
+        return;
+    }
+
+    /* We only need to figure out the address for Ldr* locations */
+    if (!INJECT_LOCATION_IS_LDR(early_inject_location)) {
+        LOG(GLOBAL, LOG_TOP, 1,
+            "early_inject is using location %d, no need to find address\n",
+            early_inject_location);
+        return;
+    }
+
+    /* Figure out which location we're using, keep in synch with
+     * LdrpLoadImportModule check in options.c */
+    if (DYNAMO_OPTION(early_inject_location) == INJECT_LOCATION_LdrDefault) {
+        LOG(GLOBAL, LOG_TOP, 2,
+            "early_inject using default ldr location for this os_ver\n");
+        switch (os_version) {
+        case WINDOWS_VERSION_NT:
+            /* LdrpImportModule is best but we can't find that address
+             * automatically since one of the stack frames we need to walk
+             * for it doesn't use frame ptrs (we can get LdrpLoadDll though),
+             * LdrpLoadDll seems to work fairly well, but won't get us in til
+             * after some of the static dlls are loaded. */
+            /* if someone provided a location for us go ahead and use that on
+             * the presumption they're providing LdrpLoadImportModule for us.
+             */
+            if (DYNAMO_OPTION(early_inject_address != 0)) {
+                early_inject_address =
+                    (app_pc)DYNAMO_OPTION(early_inject_address);
+                LOG(GLOBAL, LOG_TOP, 1,
+                    "early_inject using option provided address "PFX" at location %d\n",
+                    early_inject_address, early_inject_location);
+                return;
+            }
+            /* Case 7806, on some NT machines LdrpLoadDll causes problems
+             * while on others it doesn't.  Just turn off early injection
+             * on NT for now (LdrpLoadDll wasn't giving very good aslr support
+             * anyways and isn't a desktop target).  FIXME - we could just
+             * hardcode a table of LdrpLoadImportModule addresses for NT since
+             * we don't expect Microsoft to release any more patches for it.
+             */
+            options_make_writable();
+            dynamo_options.early_inject = false;
+            options_restore_readonly();
+            return;
+        case WINDOWS_VERSION_2000:
+            /* LdrpImportModule is best, LdrpLoadDll kind of works but won't
+             * get us in til after most of the static dlls are loaded */ 
+            early_inject_location = INJECT_LOCATION_LdrpLoadImportModule;;
+            break;
+        case WINDOWS_VERSION_XP:
+            /* LdrpLoadDll is best, LdrpLoadImportModule also works but it
+             * misses the load of kernel32 */
+            early_inject_location = INJECT_LOCATION_LdrpLoadDll;
+            break;
+        case WINDOWS_VERSION_2003:
+        case WINDOWS_VERSION_VISTA:
+            /* LdrLoadDll is best but LdrpLoadDll seems to work just as well
+             * (FIXME would it be better just to use that so matches XP?),
+             * LdrpLoadImportModule also works but it misses the load of
+             * kernel32. */
+            early_inject_location = INJECT_LOCATION_LdrLoadDll;
+            break;
+        default:
+            /* is prob. a newer windows version so the 2003 location is the
+             * most likely to work */
+            early_inject_location = INJECT_LOCATION_LdrLoadDll;
+            ASSERT_NOT_REACHED();
+        }
+    }
+    ASSERT(early_inject_location != INJECT_LOCATION_LdrDefault);
+    LOG(GLOBAL, LOG_TOP, 1,
+        "early_inject is using location %d, finding address\n",
+        early_inject_location);
+
+    /* check if we already have the right address */
+    if (dr_early_injected &&
+        INJECT_LOCATION_IS_LDR_NON_DEFAULT(early_inject_location) &&
+        early_inject_location == dr_early_injected_location
+        /* don't use parent's address if stress option set */
+        && !(INTERNAL_OPTION(early_inject_stress_helpers) && 
+             early_inject_location == INJECT_LOCATION_LdrpLoadImportModule)) {
+        /* We've got the right address to use already (from parent) */
+        early_inject_address = parent_early_inject_address;
+        ASSERT(early_inject_address != NULL);
+        ASSERT(early_inject_location != INJECT_LOCATION_LdrLoadDll ||
+               early_inject_address == (app_pc)LdrLoadDll);
+        LOG(GLOBAL, LOG_TOP, 1,
+            "early_inject using parent supplied address "PFX"\n",
+            early_inject_address);
+        return;
+    }
+
+    switch (early_inject_location) {
+    case INJECT_LOCATION_LdrLoadDll:
+        early_inject_address = (app_pc)LdrLoadDll;
+        break;
+    case INJECT_LOCATION_LdrpLoadDll:
+        /* If we were early injected have to have already gotten this address
+         * from parent as our DllMain stack walk will have gotten the wrong
+         * locations (during process init the Ldr delays calling DllMains
+         * until all static dlls are loaded unless GetProcAddress is called
+         * on the dll first, in that case its DllMain is called from there
+         * not LdrpLoadDll as we expect). */
+        /* FIXME - we could use a helper dll to get this, but it won't work
+         * when early_injected for the same reason dr's DllMain walk doesn't.
+         * Maybe there's some flag we can pass to the Ldr to tell it to call
+         * the DllMain right away (could then use it when trampoline loads
+         * dr dll).  Other option is we could wait and use the helper dll
+         * once the Ldr is in a state where it will do what we expect
+         * (the image entry point would qualify, though we could prob. find
+         * somewhere earlier then that, say when we see the execution of the
+         * DllMain of one of the non ntdll system dlls or something).  That
+         * said in the product I expect any given platform (let alone machine)
+         * to always use the same inject location. */
+        ASSERT_NOT_IMPLEMENTED(!dr_early_injected && "process early injected"
+                               "at non LdrpLoadDll location is configured to"
+                               "use LdrpLoadDll location which is NYI");
+        if (os_version == WINDOWS_VERSION_NT)
+            early_inject_address = ldrpLoadDll_address_NT;
+        else
+            early_inject_address = ldrpLoadDll_address_not_NT;
+        break;
+    case INJECT_LOCATION_LdrpLoadImportModule:
+        /* We use helper dlls to determine this address at runtime.  We pretend
+         * to be a native_exec thread and load drearlyhelper1.dll which
+         * statically links to drearlyhelper2.dll.  We watch for the
+         * NtMapViewOfSection call that loads drearlyhelper2.dll in
+         * syscall_while_native. At that point we expect the stack to look
+         * like this:
+         *   (in NtMapViewOfSection)
+         *   ntdll!LdrpMapDll
+         *   ntdll!LdrpLoadImportModule (what we want)
+         * After that don't really care (is one of the Ldrp*ImportDescriptor*
+         * routines. So we walk the stack back and get the desired address.
+         */
+        ASSERT(DYNAMO_OPTION(native_exec_syscalls));
+        LOG(GLOBAL, LOG_ALL, 1,
+            "early_inject using helper dlls to find LdrpLoadImportModule\n");
+
+        /* Pretend to be native, so Ki & Ldr hooks don't bother us. NOTE that
+         * since we're still pre dynamo_initialized no other threads can be
+         * running in dr code (so we're ok with the synch routines which could
+         * otherwise be a problem since we're still on the appstack at this
+         * point so could pass at_safe_spot while we were native). Hotpatch
+         * nudge dll loading does the same trick. This does assume that,
+         * like hotpatch nudge, we aren't running on the dstack as that
+         * will be clobbered. Alt. we could remove the KSTATS issue and
+         * the stack restriction by special casing this thread in
+         * syscall_while_native (just let all system calls run natively except
+         * MapViewOfSection which we do there so we can check the result). */
+        ASSERT(!is_currently_on_dstack(dcontext));
+        under_dr_save = dcontext->thread_record->under_dynamo_control;
+        dcontext->thread_record->under_dynamo_control = false;
+        whereami_save = dcontext->whereami;
+        /* FIXME - this is an ugly hack to get the kstack in a form compatible
+         * with dispatch for processing the native exec syscalls we'll hit
+         * while loading the helper dll (hotpatch has a similar issue but
+         * lucks out with having a compatible stack).  Shouldn't mess things
+         * up too much though.  We do have to use non-matching stops so not
+         * sure how accurate these times will be (should be tiny anyways)
+         * should poke around dispatch sometime and figure out some way to
+         * do this nicer. */
+        KSTART(dispatch_num_exits);
+        KSTART(dispatch_num_exits);
+        
+        string_option_read_lock();
+        snwprintf(buf, BUFFER_SIZE_ELEMENTS(buf), L"%hs",
+                  DYNAMO_OPTION(early_inject_helper_dll));
+        NULL_TERMINATE_BUFFER(buf);
+        string_option_read_unlock();
+        /* load the helper library, post syscall hook will fill in
+         * ldrpLoadImportModule_address for us */
+        early_inject_load_helper_dcontext = dcontext;
+        /* FIXME : if we are early_injected and the load fails because either
+         * of the helper dlls don't exist/can't be found the Ldr treats
+         * that as a process init failure and aborts the process.  Wonder if
+         * there's a flag we can pass to the Ldr to tell it not to do that.
+         * Anyways, in normal usage we expect to use the parent's address when
+         * early_injected (would only fail to do so if the parent was using
+         * a different inject_location which would be unexpected in a product
+         * configuration). */
+        EXITING_DR();
+        /* FIXME - we are making the assumption (currently true) that our
+         * load_library() & free_library() routines themselves don't write to
+         * any self protected regions, if changes we may need special versions
+         * here. */
+        mod = load_library(buf);
+        if (mod != NULL) {
+            free_library(mod);
+        }
+        ENTERING_DR();
+
+        /* clean up & restore state */
+        dcontext->whereami = whereami_save;
+        early_inject_load_helper_dcontext = NULL;
+        dcontext->thread_record->under_dynamo_control = under_dr_save;
+        /* Undo the kstack hack (see comment above) */
+        KSTOP_NOT_MATCHING_NOT_PROPAGATED(dispatch_num_exits);
+        KSTOP_NOT_PROPAGATED(dispatch_num_exits);
+
+        ASSERT(mod != NULL && ldrpLoadImportModule_address != NULL &&
+               "check that drearlyhelp*.dlls are installed");
+
+        /* FIXME - should we do anything if the address isn't found for some
+         * reason (most likely would be the helper dlls didn't exist/couldn't
+         * be found)?  Could choose to fall back to another os version
+         * appropriate location. As is, in release build we'd just fail to
+         * follow children when we couldn't find the address (see FIXME in
+         * inject_into_process()).  I expect QA is going to run into this
+         * occasionally (esp. till nodemgr etc. handle the helper dlls), so
+         * can we do anything to make things easier/more apparent for them? */
+        early_inject_address = ldrpLoadImportModule_address;
+        break;
+    default:
+        ASSERT_NOT_REACHED();
+    }
+    
+    /* FIXME - if failed to get address for any reason and we were early
+     * injected, we could fall back to parent's address. */
+    ASSERT(early_inject_address != NULL);
+    /* Since we are using a non-overriden Ldr* location can assert that
+     * early_inject_address is in ntdll */
+    ASSERT(get_allocation_base(early_inject_address) == get_ntdll_base());
+    LOG(GLOBAL, LOG_TOP, 1, "early_inject found address "PFX" to use\n",
+        early_inject_location);
+}
+
+#define SECURITY_MAX_SID_STRING_SIZE                            \
+    (2 + MAX_DWORD_STRING_LENGTH + 1 + MAX_DWORD_STRING_LENGTH  \
+     + (MAX_DWORD_STRING_LENGTH * SID_MAX_SUB_AUTHORITIES) + 1)
+    /* S-SID_REVISION- + IdentifierAuthority- + subauthorities- + NULL */
+
+static
+const char*
+get_process_SID_string()
+{
+    static char process_SID[SECURITY_MAX_SID_STRING_SIZE];
+    if (!process_SID[0]) {
+        wchar_t sid_string[SECURITY_MAX_SID_STRING_SIZE];
+        /* FIXME: we only need to query NtOpenProcessToken, but we'll
+         * assume that this function is called early enough before any
+         * impersonation could have taken place and NtOpenThreadToken
+         */
+        get_current_user_SID(sid_string, sizeof(sid_string));
+
+        snprintf(process_SID, BUFFER_SIZE_ELEMENTS(process_SID), "%ls",
+                 sid_string);
+        NULL_TERMINATE_BUFFER(process_SID);
+    }
+    return process_SID;
+}
+
+static
+const PSID
+get_Everyone_SID()
+{
+    static PSID everyone_SID = NULL;
+    static UCHAR everyone_buf[LengthRequiredSID(1)];
+
+    if (everyone_SID == NULL) {
+        SID_IDENTIFIER_AUTHORITY world = SECURITY_WORLD_SID_AUTHORITY;
+        everyone_SID = (PSID)everyone_buf;
+        initialize_known_SID(&world, SECURITY_WORLD_RID,
+                             everyone_SID);
+    }
+    return everyone_SID;
+}
+
+/* default owner SID for created objects */
+static
+const PSID
+get_process_owner_SID()
+{
+    static PSID owner_SID = NULL;
+    /* owner SID will be self-referenced in TOKEN_OWNER */
+    static UCHAR owner_buf[SECURITY_MAX_SID_SIZE + sizeof(TOKEN_OWNER)];
+
+    if (owner_SID == NULL) {
+        PTOKEN_OWNER powner = (PTOKEN_OWNER)owner_buf;
+        NTSTATUS res;
+        ASSERT(!dynamo_initialized); /* .data still writable */
+        /* initialization expected with os_user_directory() */
+        res = get_primary_owner_token(powner, sizeof(owner_buf));
+        ASSERT(NT_SUCCESS(res));
+           
+        if (!NT_SUCCESS(res)) {
+            /* while we don't expect to fail even once, we better fail
+             * all the time, otherwise we'll crash later when writing
+             * to owner_buf */
+            return NULL;
+        }
+        owner_SID = powner->Owner;
+    }
+    /* static buffer, no need to deallocate */
+    return owner_SID;
+}
+
+static bool
+os_validate_owner_equals(HANDLE file_or_directory_handle, PSID expected_owner)
+{
+    /* see comments in os_current_user_directory() */
+    /* when this scheme would work.
+     *
+     * Note that we only allow files used by initial process, so we
+     * must memoize initial SID.
+     */
+
+    /* Note on Unix this scheme doesn't work - anyone can chown(2)
+     * a directory or file to pretend to be created by the victim
+     * - we can only ask a trusted component to create a directory
+     * writable only by the corresponding user.  On Windows,
+     * however, chown() requries restore or TCB privileges.
+     * therefore it doesn't present a privilege escalation route.
+     */
+
+    /* FIXME: If we do allow anyone to create their own directory,
+     * then we'd have to verify it wasn't created by somebody else -
+     * after we open a file we should validate that we are its
+     * rightful owner (and we'll assume we have maintained the correct
+     * ACLs) to maintain that nobody else could have had write access
+     * to the file.
+     */
+
+    /* Note that we assume that TokenUser == TokenOwner, so all
+     * created files' owner will be the current user (in addition to
+     * being readable by the current user).  We also assume that the
+     * cache\ directory is on the local system.
+     * FIXME: case 10884 we can't assume that, we have to create our files explicitly
+     *
+     * (FIXME: unclear whether Machine account will be available for
+     * us on the network for services)
+     */
+
+    /* FIXME: having a open handle to the directory instead of
+     * concatenating strings would allow us to do the check only on
+     * the directory, and not on the files.  We only need to make sure
+     * there are no TOCTOU races: no symbolic links allowed, and
+     * that directories cannot be renamed or deleted.
+     */
+
+    /* just owner */
+    UCHAR sd_buf[SECURITY_MAX_SID_SIZE + sizeof(SECURITY_DESCRIPTOR)];
+    PSECURITY_DESCRIPTOR sd = (PSECURITY_DESCRIPTOR)sd_buf;
+    /* it is really SECURITY_DESCRIPTOR_RELATIVE */
+    PSID owner;
+    DWORD actual_sd_length;
+    NTSTATUS res;
+
+    /* This buffer must be aligned on a 4-byte boundary. */
+    ASSERT(ALIGNED(sd, sizeof(DWORD)));
+
+    /* FIXME: unlike SIDs which we can bound, there is no good bound
+     * for a complete SD.  We need to ensure that only one SID would
+     * be returned to us here.
+     */
+
+    /* We need READ_CONTROL access to the file_or_directory_handle */
+    res = nt_query_security_object(file_or_directory_handle,
+                                   OWNER_SECURITY_INFORMATION,
+                                   sd,
+                                   sizeof(sd_buf),
+                                   &actual_sd_length);
+    if (!NT_SUCCESS(res)) {
+        if (res == STATUS_ACCESS_DENIED) {
+            ASSERT_CURIOSITY(false && "verify handle allows READ_CONTROL");
+        }
+        return false;
+    }
+    ASSERT(actual_sd_length < sizeof(sd_buf));
+
+    if (get_owner_sd(sd, &owner)) {
+        /* FIXME: on Vista services using restricted SIDs may require
+         * obtaining the SID that we can use for creating files */
+
+        if (!equal_sid(owner, expected_owner)) {
+            /* !sid poi(owner) */
+            LOG(GLOBAL, LOG_TOP, 1,
+                "os_validate_owner_equals: owner not matching expected_owner\n");
+
+            return false;
+        }
+        return true;
+    }
+
+    ASSERT_NOT_REACHED();
+    return false;
+}
+
+/* Recommended that callers check ownership of a file that is
+ * guaranteed to not be writable.
+ */
+bool
+os_filesystem_supports_ownership(HANDLE file_or_directory_handle)
+{
+    /* Can we verify we are on FAT32 in a documented way to be certain? */
+
+    /* Currently done by checking if cache\ directory is Owned by
+     * Everyone - which certainly should only happen on FAT32.
+     */
+
+    /* FIXME: Alternatively we can test for support for file
+     * ID/reference, since creation by file reference is only
+     * supported on NTFS
+     */
+
+    /* either FAT32 or we have a proper owner */
+    if (os_validate_owner_equals(file_or_directory_handle, get_Everyone_SID())) {
+        /* On FAT32 :
+         * 0:000> !sid poi(owner)
+         * SID is: S-1-1-0 Everyone
+         *
+         * We assume that a malicious user cannot set the SID to
+         * Everyone.  Although Everyone is not the same as Anonymous
+         * Logon S-1-5-7, just in case malware can run as Everyone and
+         * creates a file we cannot decide we're on FAT32 just based
+         * on this for files that.
+         */
+
+        SYSLOG_INTERNAL_WARNING_ONCE("cache root directory is on FAT32, no security\n");
+                
+        return false;
+    } else {
+        /* we have a real owner - presumably NTFS */
+        return true;
+    }
+    return false;
+}
+
+/* opens the cache\ directory that should be modifed only by trusted users */
+/* and is used by both ASLR and persistent cache trusted producers */
+HANDLE
+open_trusted_cache_root_directory(void)
+{
+    char base_directory[MAXIMUM_PATH];
+    wchar_t wbuf[MAXIMUM_PATH];
+    HANDLE directory_handle;
+    int retval;
+
+    retval = get_parameter(L_IF_WIN(DYNAMORIO_VAR_CACHE_ROOT),
+                           base_directory, sizeof(base_directory));
+    if (IS_GET_PARAMETER_FAILURE(retval) || 
+        (strchr(base_directory, DIRSEP) == NULL &&
+         strchr(base_directory, ALT_DIRSEP) == NULL)) {
+        SYSLOG_INTERNAL_ERROR(" %s not set!\n", DYNAMORIO_VAR_CACHE_ROOT);
+        return INVALID_HANDLE_VALUE;
+    }
+    NULL_TERMINATE_BUFFER(base_directory);
+
+    if (!convert_to_NT_file_path(wbuf, base_directory, BUFFER_SIZE_ELEMENTS(wbuf)))
+        return INVALID_HANDLE_VALUE;
+
+    /* the cache root directory is supposed to be created by nodemgr
+     * and owned by Administrators, and the directory ACL should not
+     * allow changes.  We should not create one if it doesn't exist,
+     * even if we did we wouldn't have the correct ACLs for its
+     * children.
+     */
+    directory_handle = create_file(wbuf, true /* is_dir */, 
+                                   READ_CONTROL /* generic rights */,
+                                   FILE_SHARE_READ
+                                   /* case 10255: allow persisted cache files
+                                    * in same directory */
+                                   | FILE_SHARE_WRITE,
+                                   FILE_OPEN, true);
+    if (directory_handle == INVALID_HANDLE_VALUE) {
+        SYSLOG_INTERNAL_ERROR("%s=%s is invalid!",
+                              DYNAMORIO_VAR_CACHE_ROOT, base_directory);
+    }
+
+    return directory_handle;
+}
+
+bool
+os_user_directory_supports_ownership()
+{
+    /* should evaluate early so no need for .data unprotection */
+    static int user_directory_has_ownership = -1;  /* not evaluated yet */
+    /* note using explicit int, to not rely on bool true values */
+    if (user_directory_has_ownership < 0) {
+        if (DYNAMO_OPTION(validate_owner_dir) || 
+            DYNAMO_OPTION(validate_owner_file)) {
+            HANDLE root_handle = open_trusted_cache_root_directory();
+            /* Note that if root_handle is INVALID_HANDLE_VALUE we
+             * don't care about user_directory_has_ownership, it is
+             * undefined.  Since all users that verify ownership
+             * construct paths based on this directory, they should
+             * all fail and we don't really care.  We assume that this
+             * directory is created with correct privileges, so if
+             * anyone controls the registry key or can create the
+             * directory we have lost already.  (Interestingly,
+             * nt_query_security_object() returns current user for
+             * owner of -1, and so os_filesystem_supports_ownership()
+             * does return true instead.)
+             */
+            if (os_filesystem_supports_ownership(root_handle))
+                user_directory_has_ownership = 1;
+            else 
+                user_directory_has_ownership = 0;
+            close_handle(root_handle);
+        } else {
+            user_directory_has_ownership = 0; /* nobody cares whether it supports */
+        }
+    }
+    return (user_directory_has_ownership == 1);
+}
+
+/* validate we are the rightful owner */
+/* Note. we assume all calls to os_validate_owner_equals are on the same volume as
+ * DYNAMORIO_VAR_CACHE_ROOT.
+ * Handle needs to have READ_CONTROL access (FILE_GENERIC_READ provides that).
+ */
+bool
+os_validate_user_owned(HANDLE file_or_directory_handle)
+{
+    /* note that Creator and Owner don't have to match, but we expect
+     * that we'll be creating new files with current token as owner
+     */
+    PSID process_SID = get_process_primary_SID();
+    /* Note we only trust the primary token!  If we are impersonating,
+     * we also need ACLs allowing us to open other files created by
+     * the primary token */
+
+    if (os_validate_owner_equals(file_or_directory_handle, process_SID)) {
+        return true;
+    }
+    if (!os_user_directory_supports_ownership()) {
+        /* Although on FAT32 there is no owner (or any other ACLs), we
+         * get as owner Everyone.  Since file ACLs are unsupported by
+         * file system on the system drive (where we install), we can
+         * assume that privilege escalation is irrelevant for this
+         * host.
+         */
+        /* nobody really cares about this owner validation on FAT32 */
+        ASSERT(os_validate_owner_equals(file_or_directory_handle, get_Everyone_SID()));
+        return true;
+    }
+
+    ASSERT_CURIOSITY(false && "unauthorized user tried to forge our files");
+    return false;
+}
+
+/* append per-user directory name to provided directory_prefix,
+ * and optionally create a new one if possible
+ *
+ * Note 'current' is actually the primary process token: we currently
+ * allow only read-only access for impersonated threads.
+ */
+bool
+os_current_user_directory(char *directory_prefix /* INOUT */,
+                          uint directory_len,
+                          bool create)
+{
+    char *directory = directory_prefix;
+    char *dirend = directory_prefix + strlen(directory_prefix);
+    snprintf(dirend, directory_len - (dirend - directory_prefix), "%c%s",
+             DIRSEP, get_process_SID_string());
+    directory_prefix[directory_len - 1] = '\0';
+
+    directory = directory_prefix;
+    LOG(GLOBAL, LOG_CACHE, 2, "\tper-user dir is %s\n", directory);
+    DODEBUG({
+        if (!equal_sid(get_process_owner_SID(),
+                       get_process_primary_SID())) {
+            LOG(GLOBAL, LOG_CACHE, 1, 
+                "Default owner is not current user, we must be an Administrator?\n");
+            /* FIXME: we could try to really check */
+        }
+    });
+
+    /* Note that if an application impersonates threads, data for a
+     * single application will be spread across different users secure
+     * storage locations.  This may be a vulnerability - if a secure
+     * server loads a DLL while impersonated we may be erroneously
+     * using (without validation) a DLL controlled by lower privilege.
+     * Delay-loaded DLLs may provide such unexpected DLL loads.
+     *
+     * ACLs: We may want to leave files readable by Everyone - allows
+     * any impersonated threads to read files in the directory of the
+     * original process token.  (Note that Anonymous token belongs to
+     * Everyone).  World readable files also allow us to share files
+     * produced by TCB services.  Yet, for stronger security against
+     * local privilege exploits, there is some value in not allowing
+     * anyone else to read our persistent files - the layout may be
+     * useful to attackers; and general need to know principle:
+     * normally other process don't need to read these.
+     */
+
+    /* FIXME: Of course, at beginning we want be dealing with
+     * impersonation at all, but we should try to detect it here if we
+     * fail to open a directory due to impersonated thread.
+     */
+
+    /* create directory if it doesn't exist */
+    /* check for existence first so we can require new during creation */
+    if (!os_file_exists(directory, true/*is dir*/) && create) {
+        /* CREATE_DIR_FORCE_OWNER case 10884 - NoDefaultAdminOwner -
+         * the default owner doesn't have to be the current user, if
+         * member of Administrators.  Therefore we specify our own
+         * SecurityDescriptor.Owner when creating a file so that we
+         * don't use SE_OWNER_DEFAULTED, but we still want a default
+         * DACL and we don't care about group.
+         */
+
+        /* FIXME: we should ensure we do not follow symlinks! */
+        if (!os_create_dir(directory, CREATE_DIR_REQUIRE_NEW | CREATE_DIR_FORCE_OWNER)) {
+            LOG(GLOBAL, LOG_CACHE, 2, 
+                "\terror creating per-user dir %s\n", directory);
+
+            /* FIXME: currently this is expected for the 4.2 ACLs */
+            /* Note SYSLOG can be just a Warning since we will still
+             * run correctly without persistence. */
+            SYSLOG_INTERNAL_ERROR_ONCE("Persistent cache per-user needed.\n"
+                                       "mkdir \"%s\"\n"
+                                       "cacls \"%s\" /E /G username:F",
+                                       /* Note cacls needs a real user
+                                        * name, while subinacl does take SIDs */
+                                       directory, directory);
+            return false;
+        } else {
+            LOG(GLOBAL, LOG_CACHE, 2, 
+                "\tcreated per-user dir %s\n", directory);
+        }
+    }
+    
+    /* FIXME: case 8812 if the cache\ directory inheritable ACLs are
+     * setup accordingly we should be able to automatically create a
+     * our own per-user folder, without dealing with forging ACLs
+     * here, and without asking a trusted components to create it for
+     * us.
+     
+     * currently each user MUST call os_validate_user_owned()
+     * before trusting a file, or if a directory handle is
+     * guaranteed to be open at all times such that renaming is
+     * disallowed, then only the directory needs to be validated
+     */
+
+    return true;
+}
+
+/* checks for compatibility OS specific options, returns true if
+ * modified the value of any options to make them compatible
+ */
+bool
+os_check_option_compatibility(void)
+{
+    bool changed_options = false;
+    bool os_has_aslr = get_os_version() >= (int)INTERNAL_OPTION(os_aslr_version);
+    /* ASLR introduced in Vista Beta2, but we support only RTM+ so
+     * WINDOWS_VERSION_VISTA */
+
+    if (!os_has_aslr)
+        return false;
+
+    if (TEST(OS_ASLR_DISABLE_PCACHE_ALL, DYNAMO_OPTION(os_aslr))) {
+        /* completely disable pcache */
+
+        /* enabled by -desktop, but can be enabled independently as well */
+        if (DYNAMO_OPTION(coarse_enable_freeze)) {
+            dynamo_options.coarse_enable_freeze = false;
+            changed_options = true;
+        }
+        if (DYNAMO_OPTION(coarse_freeze_at_unload)) {
+            dynamo_options.coarse_freeze_at_unload = false;
+            changed_options = true;
+        }
+        if (DYNAMO_OPTION(use_persisted)) {
+            dynamo_options.use_persisted = false;
+            changed_options = true;
+        }
+        if (changed_options)
+            SYSLOG_INTERNAL_WARNING_ONCE("pcache completely disabled, Vista+");
+    }
+
+    /* note dynamorio.dll is not marked as ASLR friendly so we keep
+     * using our own -aslr_dr */
+    if (TEST(OS_ASLR_DISABLE_PCACHE_ALL, DYNAMO_OPTION(os_aslr))) {
+        /* completely disable ASLR */
+        /* enabled by -client, but can be enabled independently as well */
+        if (DYNAMO_OPTION(aslr) != 0) {
+            dynamo_options.aslr = 0;
+            changed_options = true;
+            SYSLOG_INTERNAL_WARNING_ONCE("ASLR completely disabled, Vista+");
+        }
+        if (DYNAMO_OPTION(aslr_cache) != 0) {
+            dynamo_options.aslr_cache = 0;
+            changed_options = true;
+        }
+    }
+    ASSERT(os_has_aslr);
+    return changed_options;
+}
diff --git a/core/win32/os_exports.h b/core/win32/os_exports.h
new file mode 100644
index 000000000..147110b8d
--- /dev/null
+++ b/core/win32/os_exports.h
@@ -0,0 +1,443 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * os_exports.h - Win32 specific exported declarations
+ */
+
+#ifndef _OS_EXPORTS_H_
+#define _OS_EXPORTS_H_ 1
+
+/* CONTEXT field name changes: before including arch_exports.h, for cxt_seg_t! */
+#ifdef X64
+typedef ushort cxt_seg_t;
+# define CXT_XIP Rip
+# define CXT_XAX Rax
+# define CXT_XCX Rcx
+# define CXT_XDX Rdx
+# define CXT_XBX Rbx
+# define CXT_XSP Rsp
+# define CXT_XBP Rbp
+# define CXT_XSI Rsi
+# define CXT_XDI Rdi
+/* It looks like both CONTEXT.Xmm0 and CONTEXT.FltSave.XmmRegisters[0] are filled in.
+ * We use the latter so that we don't have to hardcode the index.
+ */
+# define CXT_XMM(cxt, idx) ((dr_xmm_t*)&((cxt)->FltSave.XmmRegisters[idx]))
+/* they kept the 32-bit EFlags field; sure, the upper 32 bits of Rflags
+ * are undefined right now, but doesn't seem very forward-thinking. */
+# define CXT_XFLAGS EFlags
+#else
+typedef DWORD cxt_seg_t;
+# define CXT_XIP Eip
+# define CXT_XAX Eax
+# define CXT_XCX Ecx
+# define CXT_XDX Edx
+# define CXT_XBX Ebx
+# define CXT_XSP Esp
+# define CXT_XBP Ebp
+# define CXT_XSI Esi
+# define CXT_XDI Edi
+# define CXT_XFLAGS EFlags
+/* This is not documented, but CONTEXT.ExtendedRegisters looks like fxsave layout.
+ * Presumably there are no processors that have SSE but not FXSR
+ * (we ASSERT on that in proc_init()).
+ */
+# define FXSAVE_XMM0_OFFSET 160
+# define CXT_XMM(cxt, idx) \
+    ((dr_xmm_t*)&((cxt)->ExtendedRegisters[FXSAVE_XMM0_OFFSET + (idx)*16]))
+#endif
+
+#include "os_shared.h"
+#include "arch_exports.h"       /* for dr_mcontext_t */
+#include "aslr.h"               /* for aslr_context */
+
+/* you can rely on these increasing with later versions */
+#define WINDOWS_VERSION_VISTA  60
+#define WINDOWS_VERSION_2003   52
+#define WINDOWS_VERSION_XP     51
+#define WINDOWS_VERSION_2000   50
+#define WINDOWS_VERSION_NT     40
+int get_os_version(void);
+
+/* TEB offsets
+ * we'd like to use offsetof(TEB, field) but that would require
+ * everyone to include ntdll.h, and wouldn't work for inline assembly,
+ * so we hardcode the fields we need here.  We check vs offsetof() in os_init().
+ */
+enum {
+#ifdef X64
+    EXCEPTION_LIST_TIB_OFFSET = 0x000,
+    TOP_STACK_TIB_OFFSET      = 0x008,
+    BASE_STACK_TIB_OFFSET     = 0x010,
+    FIBER_DATA_TIB_OFFSET     = 0x020,
+    SELF_TIB_OFFSET           = 0x030,
+    PID_TIB_OFFSET            = 0x040,
+    TID_TIB_OFFSET            = 0x048,
+    ERRNO_TIB_OFFSET          = 0x068,
+    WOW64_TIB_OFFSET          = 0x100,
+    PEB_TIB_OFFSET            = 0x060,
+#else
+    EXCEPTION_LIST_TIB_OFFSET = 0x00,
+    TOP_STACK_TIB_OFFSET      = 0x04,
+    BASE_STACK_TIB_OFFSET     = 0x08,
+    FIBER_DATA_TIB_OFFSET     = 0x10,
+    SELF_TIB_OFFSET           = 0x18,
+    PID_TIB_OFFSET            = 0x20,
+    TID_TIB_OFFSET            = 0x24,
+    ERRNO_TIB_OFFSET          = 0x34,
+    WOW64_TIB_OFFSET          = 0xC0,
+    PEB_TIB_OFFSET            = 0x30,
+#endif
+};
+
+#ifdef X64
+# define SEG_TLS SEG_GS
+#else
+# define SEG_TLS SEG_FS /* x86 and WOW64 */
+#endif
+
+static inline void *
+get_tls(ushort tls_offs)
+{
+    return (void *) IF_X64_ELSE(__readgsqword,__readfsdword)(tls_offs);
+}
+
+static inline void
+set_tls(ushort tls_offs, void *value)
+{
+    IF_X64_ELSE(__writegsqword,__writefsdword)(tls_offs, (ptr_uint_t) value);
+}
+
+/* If this changes our persisted caches may all fail.
+ * We assert that this matches SYSTEM_BASIC_INFORMATION.AllocationGranularity
+ * in get_system_basic_info().
+ */
+#define OS_ALLOC_GRANULARITY    64*1024
+#define MAP_FILE_VIEW_ALIGNMENT OS_ALLOC_GRANULARITY
+
+/* Used to flush a thread's stack prior to rest of exit routines.  Caller
+ * is required to own the thread_initexit_lock when calling this routine. */
+void
+os_thread_stack_exit(dcontext_t *dcontext);
+
+int debugbox(char *msg);
+int os_countdown_messagebox(char *message, int time_in_milliseconds);
+
+/* raise an exception in the application context */
+void os_raise_exception(dcontext_t *dcontext, 
+                        EXCEPTION_RECORD* pexcrec, CONTEXT* pcontext);
+bool exception_frame_chain_depth(dcontext_t *dcontext);
+
+/* PR 263338: we have to pad for alignment */
+#define CONTEXT_HEAP_SIZE(cxt) (sizeof(cxt) IF_X64(+8)/*heap is 8-aligned already*/)
+#define CONTEXT_HEAP_SIZE_OPAQUE (CONTEXT_HEAP_SIZE(CONTEXT))
+
+bool
+thread_get_context(thread_record_t *tr, CONTEXT *context);
+
+bool
+thread_set_context(thread_record_t *tr, CONTEXT *context);
+
+/* To move a var into one of our special self-protected sections, in
+ * addition to declaring a var between START_DATA_SECTION and
+ * END_DATA_SECTION you must initialize it to something!
+ * cl has no var attributes, only pragmas to indicate current section, so
+ * have to ensure that your var goes into proper one of data, bss, and const
+ * by manipulating its initialization -- fortunately, any explicit initialization,
+ * even to 0, causes cl to put it into data and not bss.
+ */
+/* Use special C99 operator _Pragma to generate a pragma from a macro */
+#if _MSC_VER <= 1200 /* FIXME: __pragma may work w/ vc6: then don't need #if */
+# define ACTUAL_PRAGMA(p) _Pragma ( #p )
+#else
+# define ACTUAL_PRAGMA(p) __pragma ( p )
+#endif
+#define START_DATA_SECTION(name, prot) ACTUAL_PRAGMA( data_seg(name) )
+#define VAR_IN_SECTION(name) /* nothing */
+#define END_DATA_SECTION() ACTUAL_PRAGMA( data_seg() )
+
+#ifdef DEBUG
+void print_dynamo_regions(void);
+#endif
+
+app_pc get_image_entry(void);
+bool check_for_image_entry(app_pc bb_start);
+bool reached_image_entry_yet(void);
+
+size_t get_allocation_size(byte *pc, byte **base_pc);
+byte *get_allocation_base(byte *pc);
+void mark_page_as_guard(byte *pc);
+
+char *get_dynamorio_library_path(void);
+
+void merge_writecopy_pages(app_pc start, app_pc end);
+
+bool is_pid_me(process_id_t pid);
+bool is_phandle_me(HANDLE phandle);
+
+extern bool intercept_asynch;
+extern bool intercept_callbacks;
+extern process_id_t win32_pid;
+extern void *peb_ptr; /* not exposing type in case not including ntdll.h */
+
+extern app_pc vsyscall_page_start;
+/* pc kernel will claim app is at while in syscall */
+extern app_pc vsyscall_after_syscall;
+/* pc of the end of the syscall instr itself */
+extern app_pc vsyscall_syscall_end_pc;
+/* IF_X64(ASSERT_NOT_IMPLEMENTED(false)) -- need to update */
+#define VSYSCALL_PAGE_START_BOOTSTRAP_VALUE ((app_pc)(ptr_uint_t) 0x7ffe0000)
+#define VSYSCALL_BOOTSTRAP_ADDR ((app_pc)(ptr_uint_t) 0x7ffe0300)
+#define VSYSCALL_AFTER_SYSCALL_BOOTSTRAP_VALUE ((app_pc)(ptr_uint_t) 0x7ffe0304)
+
+/* ref case 5217 - for Sygate compatibility have to execute int's out of
+ * ntdll.dll. This will hold the target to use (points to int 2e ret 0 in
+ * NtYieldExecution). */
+extern app_pc int_syscall_address;
+/* ref case 5441 - for Sygate compatibility the return address for sysenter
+ * system calls needs to be in ntdll.dll for some platforms.  This will point
+ * to a ret 0 in ntdll (NtYieldExecution). */
+extern app_pc sysenter_ret_address;
+
+bool ignorable_system_call(int num);
+bool optimizable_system_call(int num);
+#ifdef DEBUG
+void check_syscall_numbers(dcontext_t *dcontext);
+#endif
+bool is_cb_return_syscall(dcontext_t *dcontext);
+
+#ifdef WINDOWS_PC_SAMPLE
+file_t profile_file;
+mutex_t profile_dump_lock;
+
+typedef struct _profile_t {
+    void *start;
+    void *end;
+    uint bucket_shift;
+    uint *buffer;
+    size_t buffer_size; /* in bytes */
+    bool enabled;
+    dcontext_t *dcontext;
+    HANDLE handle;
+} profile_t;
+
+/* dcontext optional, allocates global if NULL, otherwise local to that dcontext*/
+profile_t *
+create_profile(void *start, void *end, uint bucket_shift, dcontext_t *dcontext);
+
+void
+free_profile(profile_t *profile);
+
+void
+start_profile(profile_t *profile);
+
+void
+stop_profile(profile_t *profile);
+
+void
+dump_profile(file_t file, profile_t *profile);
+
+void
+dump_profile_range(file_t file, profile_t *profile, byte *start, byte *end);
+
+uint
+sum_profile_range(profile_t *profile, byte *start, byte *end);
+
+uint
+sum_profile(profile_t *profile);
+
+void
+reset_profile(profile_t *profile);
+#endif
+
+enum {
+    /* via -hide_from_query_mem controls what dr does when the app does
+     * a query virtual memory call on the dynmorio.dll base */
+    HIDE_FROM_QUERY_TYPE_PROTECT   = 0x1, /* to PRIVATE, NO_ACCESS */
+    HIDE_FROM_QUERY_BASE_SIZE      = 0x2, /* shift reported allocation a page, 
+                                           * and expand size to whole dll */
+    HIDE_FROM_QUERY_RETURN_INVALID = 0x4, /* return STATUS_INVALID_ADDRESS to
+                                           * the app */
+};
+
+/* flags for DYNAMO_OPTION(tls_flags) */
+enum tls_flags {
+    TLS_FLAG_BITMAP_TOP_DOWN  = 0x1, /* when set use last available
+                                      * TLS slots, otherwise use first
+                                      * just the way TlsAlloc would */
+    TLS_FLAG_CACHE_LINE_START = 0x2, /* when set the first should
+                                      * start at a cache line,
+                                      * otherwise as long as all
+                                      * entries should fit order doesn't matter */
+    TLS_FLAG_BITMAP_FILL      = 0x4, /* FIXME: NYI: reserve slots
+                                      * unused due to alignment,
+                                      * should be needed only for
+                                      * aligned bottom up xref case
+                                      * 6770 SQL 2005
+                                      */
+};
+
+/* flags for DYNAMO_OPTION(os_aslr) */
+enum {
+    /* disable completely our ASLR and persistent caches for all modules */
+    OS_ASLR_DISABLE_ASLR_ALL      = 0x01, /* disable ASLR DLL,stack,heap */
+    OS_ASLR_DISABLE_PCACHE_ALL    = 0x02, /* disable pcache generation and use */
+
+    /* note that after section mapping we can read from header whether
+     * OS would have randomized base
+     */
+    OS_ASLR_DISABLE_ASLR_DETECT   = 0x10, /* NYI case 8225 */
+    OS_ASLR_DISABLE_PCACHE_DETECT = 0x20, /* NYI case 8225 */
+};
+
+enum {
+    /* Does not override attack handling options (i.e. kill_thread etc. still
+     * do their thing) only detaches if the we were going to kill the 
+     * process */
+    DETACH_UNHANDLED_VIOLATION   = 0x01, /* FIXME : separate A, B, C etc.? */
+    /* Subset of DETACH_UNHANDLED_VIOLATION, detaches if we see an unsupported
+     * module */
+    DETACH_UNSUPPORTED_MODULE    = 0x02,
+    
+    /* Anything below this line is unsafe and will likely fail */
+    /* FIXME : this detaches on any internal process terminate, including from
+     * a security violation (which we may want to allow to kill the process, as
+     * opposed to an internal error in future), in future may also want to 
+     * further break it up into internal_exception, assertion, etc. */
+    DETACH_ON_TERMINATE          = 0x010,
+    /* safer then w/cleanup, leaves dr memory behind */
+    DETACH_ON_TERMINATE_NO_CLEAN = 0x020,
+
+    /* **** FIMXE : following NYI **** */
+    /* don't kill faulting thread, make a best guess at its app context 
+     * note this could turn the kill thread on the faulting thread to a 
+     * throw exception if we get the context wrong */
+    DETACH_ON_TERMINATE_NO_KILL  = 0x040,
+    /* the following two options try to help prevent hangs when detaching on
+     * terminate, but there are hanging scenarios that aren't covered */
+    /* tries to avoid deadlocking by proactively freeing some locks (if held)
+     * very unsafe */
+    DETACH_ON_TERMINATE_NO_LOCKS = 0x080,
+    /* tries to detect an infinite loop in the detach synchronization
+     * routines and kills the process in that scenario */
+    DETACH_ON_TERMINATE_NO_HANG  = 0x100,
+};
+
+/* mcontext must be valid, including the pc field (native) and app_errno
+ * must not be holding any locks */
+/* sets detach in motion and never returns */
+void detach_internal_synch(void);
+void detach_internal(void);
+enum {
+    DETACH_NORMAL_TYPE          =  0,
+    DETACH_BAD_STATE            = -1,
+    DETACH_BAD_STATE_NO_CLEANUP = -2,
+};
+void detach_helper(int detach_type); /* needs to be exported for nudge.c */
+extern bool doing_detach;
+
+void early_inject_init(void);
+
+/* in module.c */
+app_pc get_module_preferred_base_safe(app_pc pc);
+app_pc get_module_preferred_base(app_pc pc);
+ssize_t get_module_preferred_base_delta(app_pc pc);
+bool in_same_module(app_pc target, app_pc source);
+void print_modules_safe(file_t f, bool dump_xml);
+void print_modules_ldrlist_and_ourlist(file_t f, bool dump_xml, bool conservative);
+/* FIXME: rename this to get_module_path, cf. get_module_short_name() */
+void get_module_name(app_pc, char *buf, int max_chars);
+bool is_module_patch_region(dcontext_t *dcontext, app_pc start, app_pc end,
+                            bool conservative);
+bool is_IAT(app_pc start, app_pc end, bool page_align,
+            app_pc *iat_start, app_pc *iat_end);
+bool is_in_IAT(app_pc addr);
+bool
+get_IAT_section_bounds(app_pc module_base, app_pc *iat_start, app_pc *iat_end);
+bool os_module_store_IAT_code(app_pc addr);
+bool os_module_cmp_IAT_code(app_pc addr);
+bool os_module_free_IAT_code(app_pc addr);
+void print_module_section_info(file_t file, app_pc addr);
+/* here rather than os_shared.h since this is win32-specific */
+/* Returns true if addr is in a xdata section and if so returns in sec_start and sec_end
+ * the bounds of the section containing addr (MERGED with adjacent xdata sections). 
+ * sec_start and sec_end are optional. */
+bool is_in_xdata_section(app_pc module_base, app_pc addr,
+                         app_pc *start_pc, app_pc *end_pc);
+thread_id_t get_loader_lock_owner(void);
+bool module_pc_section_lookup(app_pc module_base, app_pc pc,
+                              IMAGE_SECTION_HEADER *section_out);
+
+/* in signal.c */
+void start_itimer(dcontext_t *dcontext);
+void stop_itimer(void);
+
+/* in callback.c */
+dcontext_t *
+get_prev_swapped_dcontext(dcontext_t *dcontext);
+void callback_interception_init(void);
+void callback_interception_unintercept(void);
+void callback_interception_exit(void);
+void set_asynch_interception(thread_id_t tid, bool intercept);
+bool intercept_asynch_for_thread(thread_id_t tid, bool intercept_unknown);
+bool intercept_asynch_for_self(bool intercept_unknown);
+bool
+is_in_interception_buffer(byte *pc);
+bool
+is_syscall_trampoline(byte *pc);
+app_pc get_app_pc_from_intercept_pc(byte *pc);
+bool is_intercepted_app_pc(app_pc pc, byte **interception_pc);
+
+/* in inject_shared.c */
+#include "inject_shared.h"
+
+/* in ntdll.c, exported through here */
+void syscalls_init(void);
+void syscalls_init_options_read(void);
+uint query_time_millis(void);
+int get_last_error(void);
+void set_last_error(int error);
+HANDLE get_stderr_handle(void);
+HANDLE get_stdout_handle(void);
+/* used in certain asserts in x86/interp.c otherwise should be in os_private.h */
+bool use_ki_syscall_routines(void);
+
+wchar_t *get_application_cmdline(void);
+const char *
+get_application_short_unqualified_name(void);
+
+#endif /* _OS_EXPORTS_H_ */
diff --git a/core/win32/os_private.h b/core/win32/os_private.h
new file mode 100644
index 000000000..228651de9
--- /dev/null
+++ b/core/win32/os_private.h
@@ -0,0 +1,601 @@
+/* **********************************************************
+ * Copyright (c) 2005-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2005-2007 Determina Corp. */
+
+/*
+ * os_private.h - declarations shared among os-specific files, but not
+ * exported to the rest of the code
+ */
+
+#ifndef _OS_PRIVATE_H_
+#define _OS_PRIVATE_H_ 1
+
+#include "drmarker.h"
+
+/* in os.c *********************************************************/
+#define GLOBAL_NT_PREFIX L"\\??\\"
+/* FIXME: used for converting Dos to NT paths */
+
+/* thread-local data that's os-private, for modularity and easy sharing across
+ * callbacks
+ */
+typedef struct _os_thread_data_t {
+    /* Store stack info at thread startup since an attack or inadvertent write
+     * could clobber the teb fields storing this info.  Also, on NT and 2k the
+     * stack is freed in process during kernel32!ExitThread (which uses some
+     * unused TEB space as the stack to free oringinal stack and exit the
+     * the thread) so we mark teb_stack_no_longer_valid when we see the free
+     * (which we watch for). */
+    byte *stack_base;
+    byte *stack_top;
+    bool teb_stack_no_longer_valid;
+} os_thread_data_t;
+
+/* pc values delimiting dynamo dll image */
+extern app_pc dynamo_dll_start;
+extern app_pc dynamo_dll_end;
+
+extern dcontext_t *early_inject_load_helper_dcontext;
+
+bool
+is_first_thread_in_new_process(HANDLE process_handle, CONTEXT *cxt);
+
+void
+maybe_inject_into_process(dcontext_t *dcontext, HANDLE process_handle,
+                          CONTEXT *cxt);
+
+bool
+translate_context(thread_record_t *trec, CONTEXT *cxt, bool restore_memory);
+
+void
+dump_mbi(file_t file, MEMORY_BASIC_INFORMATION *mbi, bool dump_xml);
+
+void
+dump_mbi_addr(file_t file, app_pc target, bool dump_xml);
+
+void
+set_reached_image_entry(void);
+
+bool
+prot_is_writable(uint prot);
+
+bool
+prot_is_executable(uint osprot);
+
+uint
+get_current_protection(byte *pc);
+
+/* translate platform independent protection bits to native flags */
+int
+memprot_to_osprot(uint prot);
+
+/* translate native flags to platform independent protection bits */
+int
+osprot_to_memprot(uint prot);
+
+int
+process_mmap(dcontext_t *dcontext, app_pc pc, size_t size, bool add);
+
+void
+check_for_ldrpLoadImportModule(byte *base, uint *ebp);
+
+/* in syscall.c *********************************************************/
+
+/* this points to a windows-version-specific syscall array */
+extern int *syscalls;
+#define SYSCALL_NOT_PRESENT 0xffffffff
+/* this points to a windows-version-specific WOW table index array */
+extern int *wow64_index;
+
+extern const int windows_vista_sp1_x64_syscalls[];
+extern const int windows_vista_sp1_syscalls[];
+extern const int windows_vista_sp0_x64_syscalls[];
+extern const int windows_vista_sp0_syscalls[];
+extern const int windows_2003_syscalls[];
+extern const int windows_XP_x64_syscalls[];
+extern const int windows_XP_wow64_index[];
+extern const int windows_XP_syscalls[];
+extern const int windows_2000_syscalls[];
+extern const int windows_NT_sp3_syscalls[];
+extern const int windows_NT_sp0_syscalls[];
+extern const int windows_NT_sp4_syscalls[];
+
+/* for x64 this is the # of args */
+extern const uint syscall_argsz[];
+
+extern const char * const syscall_names[];
+
+extern bool init_apc_go_native_pause;
+extern bool init_apc_go_native;
+
+#ifdef DEBUG
+void check_syscall_array_sizes(void);
+#endif
+
+void
+windows_version_init(void);
+
+enum {
+#define SYSCALL(name, act, nargs, arg32, vista_sp1_x64, vista_sp1, vista_sp0_x64, \
+                vista_sp0, tk3, xp64, wow64, xp, tk, ntsp4, ntsp3, ntsp0)         \
+    SYS_##name,
+#include "syscallx.h"
+#undef SYSCALL
+    SYS_MAX,
+};
+
+/* the offset from edx of the parameters to a system call, our current 
+ * (FIXME - also potentially unreliable, since really is a function of os 
+ * version and processor type) check is by the system entry method, 
+ * if it's int then offset is 0, if it's sysenter or syscall then offset is 8 
+ * will also have it default to 0 since I think 2k uses int regardless of 
+ * proccesor type */
+/* FIXME - if we are really paranoid then we should ensure that the offset 
+ * holds the return values the os would expect (i.e. the ntdll wrapper return
+ * address for XP/2003), also if used before we know the syscall method will
+ * default to 0! */
+#define SYSCALL_PARAM_MAX_OFFSET        (2*XSP_SZ) /* offset for real arguments */
+#ifdef X64
+/* retaddr, then args */
+# define SYSCALL_PARAM_OFFSET() (XSP_SZ)
+#else
+/* as done on WinXP, syscalls have extra slots before real params */
+/* edx is 4 less than on 2000, plus there's an extra call to provide
+ * return address for sysenter, so we have to skip 2 slots:
+ */
+# define SYSCALL_PARAM_OFFSET()                          \
+    ((get_syscall_method() == SYSCALL_METHOD_SYSCALL || \
+      get_syscall_method() == SYSCALL_METHOD_SYSENTER)  \
+     ? SYSCALL_PARAM_MAX_OFFSET : 0)
+#endif
+
+static inline reg_t *
+sys_param_addr(dcontext_t *dcontext, reg_t *param_base, int num)
+{
+#ifdef X64
+    /* we force-inline get_mcontext() and so don't take it as a param */
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    switch (num) {
+    case 0: return &mc->xcx;
+    case 1: return &mc->xdx;
+    case 2: return &mc->r8;
+    case 3: return &mc->r9;
+    default: return ((reg_t *)param_base) + num;
+    }
+#else
+    return ((reg_t *)param_base) + num;
+#endif
+}
+
+static inline reg_t
+sys_param(dcontext_t *dcontext, reg_t *param_base, int num)
+{
+    return *sys_param_addr(dcontext, param_base, num);
+}
+
+static inline reg_t
+postsys_param(dcontext_t *dcontext, reg_t *param_base, int num)
+{
+#ifdef X64
+    switch (num) {
+    /* Register params are volatile so we save in dcontext in pre-syscall */
+    case 0: return dcontext->sys_param0;
+    case 1: return dcontext->sys_param1;
+    case 2: return dcontext->sys_param2;
+    case 3: return dcontext->sys_param3;
+    default: return *sys_param_addr(dcontext, param_base, num);
+    }
+#else
+    return *sys_param_addr(dcontext, param_base, num);
+#endif
+}
+
+void
+init_syscall_trampolines(void);
+
+void
+exit_syscall_trampolines(void);
+
+bool
+os_get_file_size_by_handle(IN HANDLE file_handle,
+                           uint64 *end_of_file);
+bool
+os_set_file_size(IN HANDLE file_handle,
+                 uint64 end_of_file);
+
+/* use os_rename_file() for cross-platform uses */
+bool
+os_rename_file_in_directory(IN HANDLE rootdir,
+                            const wchar_t *orig_name,
+                            const wchar_t *new_name,
+                            bool replace);
+
+/* in callback.c ***************************************************/
+
+#define INTERCEPTION_CODE_SIZE IF_X64_ELSE(6*4096,4*4096)
+
+void
+callback_init(void);
+
+void
+callback_exit(void);
+
+dr_marker_t*
+get_drmarker(void);
+
+#define UNDER_DYN_HACK 0xabcd
+
+byte *
+intercept_syscall_wrapper(byte **ptgt_pc /* IN/OUT */, 
+                          intercept_function_t prof_func,
+                          void *callee_arg, after_intercept_action_t action_after,
+                          app_pc *skip_syscall_pc /* OUT */,
+                          byte **orig_bytes_pc /* OUT */,
+                          byte *fpo_stack_adjustment /* OUT OPTIONAL */);
+byte *
+insert_trampoline(byte *tgt_pc, intercept_function_t prof_func, void *callee_arg,
+                  bool assume_esp, after_intercept_action_t action_after,
+                  bool cti_safe_to_ignore);
+
+void
+remove_trampoline(byte *our_pc, byte *tgt_pc);
+
+void 
+remove_image_entry_trampoline(void);
+
+void
+callback_start_return(int app_errno, dr_mcontext_t *mc);
+
+void
+intercept_nt_continue(CONTEXT *cxt, int flag);
+
+void
+intercept_nt_setcontext(dcontext_t *dcontext, CONTEXT *cxt);
+
+/* methods of taking over */
+#define INTERCEPT_POINT(point) point,
+#define INTERCEPT_ALL_POINTS                      \
+    /* when dr_preinjected=false, not used */     \
+    INTERCEPT_POINT(INTERCEPT_EXPLICIT_INJECT)    \
+    /* otherwise we are in one of these */        \
+    INTERCEPT_POINT(INTERCEPT_PREINJECT)          \
+    INTERCEPT_POINT(INTERCEPT_IMAGE_ENTRY)        \
+    INTERCEPT_POINT(INTERCEPT_LOAD_DLL)           \
+    INTERCEPT_POINT(INTERCEPT_UNLOAD_DLL)         \
+    /* asynch prior to image entry */             \
+    INTERCEPT_POINT(INTERCEPT_EARLY_ASYNCH)       \
+    /* syscall trampoline prior to image entry */ \
+    INTERCEPT_POINT(INTERCEPT_SYSCALL)
+typedef enum {
+    INTERCEPT_ALL_POINTS
+} retakeover_point_t;
+#undef INTERCEPT_POINT
+
+void
+retakeover_after_native(thread_record_t *tr, retakeover_point_t where);
+
+void
+context_to_mcontext(dr_mcontext_t *mcontext, CONTEXT* cxt);
+
+void
+mcontext_to_context(CONTEXT* cxt, dr_mcontext_t *mcontext);
+
+#ifdef DEBUG
+void dump_context_info(CONTEXT *context, file_t file, bool all);
+#endif
+
+/* PR 264138: we need to preserve xmm0-5 for x64 and wow64.
+ * These flags must be used for any CONTEXT that is being used
+ * to set a dr_mcontext_t for executing by DR, or if the CONTEXT
+ * will be passed to nt_set_context() and the thread in question
+ * will execute DR code in between.
+ * Although winnt.h mentions CONTEXT_MMX_REGISTERS, there is no such
+ * constant: they must mean CONTEXT_FLOATING_POINT.
+ * We allow non-core inject.c to use this by not using
+ * preserve_xmm_caller_saved() and just ignoring underlying SSE support: so we
+ * have some duplication of logic, but it's messy to get preserve_xmm_caller_saved()
+ * into arch_exports.h as NT_CURRENT_PROCESS is not defined yet, and
+ * non-core modules don't link with proc.c.
+ * Since this affects only what we request from the kernel, asking
+ * for floating point w/o underlying sse support is not a problem.
+ */
+#define CONTEXT_XMM_FLAG IF_X64_ELSE(CONTEXT_FLOATING_POINT, CONTEXT_EXTENDED_REGISTERS)
+#define CONTEXT_PRESERVE_XMM IF_X64_ELSE(true, is_wow64_process(NT_CURRENT_PROCESS))
+#define CONTEXT_DR_STATE (CONTEXT_INTEGER | CONTEXT_CONTROL | \
+                          (CONTEXT_PRESERVE_XMM ? CONTEXT_XMM_FLAG : 0U))
+
+enum {
+      EXCEPTION_INFORMATION_READ_EXECUTE_FAULT = 0,
+      /* On non-NX capable machines Read and Execute faults are not
+       * differentiated */
+      EXCEPTION_INFORMATION_WRITE_FAULT        = 1,
+      EXCEPTION_INFORMATION_EXECUTE_FAULT      = 8, /* case 5879 */
+      /* Only on NX enabled machines Execute faults are differentiated */
+};
+
+bool
+should_inject_into_process(dcontext_t *dcontext, HANDLE process_handle,
+                           int *rununder_mask, /* OPTIONAL OUT */
+                           inject_setting_mask_t *inject_settings /* OPTIONAL OUT */);
+
+/* in inject.c (also used by dynamo library) ***********************/
+
+/* Note : inject_init calls get_module_handle and therefore should be called
+ * during dynamo initialization to avoid race conditions on the loader lock */
+void
+inject_init(void); /* must be called prior to inject_into_thread(void) */
+
+bool
+inject_into_thread(HANDLE phandle, CONTEXT *cxt, HANDLE thandle,
+                   char *dynamo_path);
+
+/* inject_location values come form the INJECT_LOCATION_* enum is os_shared.h */
+bool
+inject_into_new_process(HANDLE phandle, char *dynamo_path, bool map,
+                        uint inject_location, void *inject_address);
+
+
+/* in <arch.s> (x86.asm for us) ************************************/
+
+void 
+internal_dynamo_start(void);
+
+void
+cleanup_after_interp(void);
+
+void
+callback_dynamo_start(void);
+
+void
+nt_continue_dynamo_start(void);
+
+/* x86.asm custom routine used only for check_for_modified_code() */
+void call_modcode_alt_stack(dcontext_t *dcontext, EXCEPTION_RECORD *pExcptRec,
+                            CONTEXT *cxt, app_pc target, uint flags,
+                            bool using_initstack);
+
+/* x86.asm routine used for injection */
+void load_dynamo(void);
+
+/* in eventlog.c ***************************************************/
+
+void
+eventlog_init(void);
+
+void
+eventlog_fast_exit(void);
+void
+eventlog_slow_exit(void);
+
+
+/* in module.c *****************************************************/
+
+#ifdef X64
+#ifndef __IMAGE_UNWIND_INFO
+#define __IMAGE_UNWIND_INFO
+#pragma warning(push)
+#pragma warning(disable : 4214) /* allow byte-sized bitfields */
+/* These definitions are needed to parse exception handlers to add to the RCT
+ * table as part of PR 250395.  These definitions aren't found in any header
+ * files.  These seem to be coming directly from internal sources.  I saw a
+ * definition for RUNTIME_FUNCTION in winternal.h which was identical to
+ * IMAGE_RUNTIME_FUNCTION_ENTRY, which under a big comment block saying that it
+ * is for internal windows use only and might change from release to release, so
+ * using externally visible ones and declaring those that aren't available.
+ *
+ * These are based on the Microsoft specifications and suggested
+ * C defines at http://msdn2.microsoft.com/en-us/library/ssa62fwe(VS.80).aspx
+ * Since these are not in any header files we use our own style conventions.
+ */
+typedef enum _unwind_opcode_t {
+    UWOP_PUSH_NONVOL = 0, /* info == register number */
+    UWOP_ALLOC_LARGE,     /* no info, alloc size in next 2 slots */
+    UWOP_ALLOC_SMALL,     /* info == size of allocation / 8 - 1 */
+    UWOP_SET_FPREG,       /* no info, FP = RSP + UNWIND_INFO.FPRegOffset*16 */
+    UWOP_SAVE_NONVOL,     /* info == register number, offset in next slot */
+    UWOP_SAVE_NONVOL_FAR, /* info == register number, offset in next 2 slots */
+    UWOP_SAVE_XMM128,     /* info == XMM reg number, offset in next slot */
+    UWOP_SAVE_XMM128_FAR, /* info == XMM reg number, offset in next 2 slots */
+    UWOP_PUSH_MACHFRAME   /* info == 0: no error-code, 1: error-code */
+} unwind_opcode_t;
+
+typedef union _unwind_code_t {
+    struct {
+        byte CodeOffset;
+        byte UnwindOp :4;
+        byte OpInfo   :4;
+    };
+    USHORT FrameOffset;
+} unwind_code_t;
+
+#define UNW_FLAG_EHANDLER  0x01
+#define UNW_FLAG_UHANDLER  0x02
+#define UNW_FLAG_CHAININFO 0x04
+
+typedef struct _unwind_info_t {
+    byte Version      :3;
+    byte Flags        :5;
+    byte SizeOfProlog;
+    byte CountOfCodes;
+    byte FrameRegister:4;
+    byte FrameOffset  :4;
+    unwind_code_t UnwindCode[1];
+#if 0 /* variable-length tail of struct */
+    /* MSDN uses "((CountOfCodes + 1) & ~1)" which is just align-forward-2,
+     * used b/c the UnwindCode array must always have an even capacity */
+    unwind_code_t MoreUnwindCode[ALIGN_FORWARD(CountOfCodes, 2) - 1];
+    union {
+        OPTIONAL ULONG ExceptionHandler;
+        OPTIONAL ULONG FunctionEntry;
+    };
+    OPTIONAL ULONG ExceptionData[];
+#endif
+} unwind_info_t;
+
+/* Address of field that's after the unwind code data */
+#define UNWIND_INFO_PTR_ADDR(info) \
+    ((void *)&(info)->UnwindCode[ALIGN_FORWARD((info)->CountOfCodes, 2)])
+
+/* Field that's after the unwind code data, treated as an RVA */
+#define UNWIND_INFO_PTR_RVA(info) (*(uint*)UNWIND_INFO_PTR_ADDR(info))
+
+/* ExceptionData field (2nd one after the unwind code data) */
+#define UNWIND_INFO_DATA_ADDR(info) (((uint*)UNWIND_INFO_PTR_ADDR(info))+1)
+#define UNWIND_INFO_DATA_RVA(info) (*(((uint*)UNWIND_INFO_PTR_ADDR(info))+1))
+
+/* ExceptionData takes this form.  It is inlined according to my observation
+ * but it may instead be pointed at by an RVA.
+ */
+typedef struct _scope_table_t {
+    ULONG Count;
+    struct {
+        ULONG BeginAddress;
+        ULONG EndAddress;
+        ULONG HandlerAddress;
+        ULONG JumpTarget;
+    } ScopeRecord[1];
+} scope_table_t;
+
+#pragma warning(pop)
+#endif /* __IMAGE_UNWIND_INFO */
+#endif /* X64 */
+
+bool
+is_readable_pe_base(app_pc base);
+
+bool
+get_module_info_pe(const app_pc module_base, uint *checksum, uint *timestamp,
+                   size_t *image_size, char **pe_name, size_t *code_size);
+
+/* Use get_module_short_name for arbitrary pcs -- only call this if
+ * you KNOW this is the base addr of a non-executable module, as it
+ * bypasses some safety checks in get_module_short_name to avoid 4
+ * system calls.
+ */
+char*
+get_dll_short_name(app_pc base_addr);
+
+/* Use get_module_short_name in general */
+const char *
+get_module_short_name_uncached(dcontext_t *dcontext, app_pc base, bool at_map
+                               HEAPACCT(which_heap_t which));
+
+app_pc
+get_module_entry(app_pc module_base);
+
+uint
+get_module_characteristics(app_pc module_base);
+
+bool
+module_has_cor20_header(app_pc module_base);
+
+bool
+module_is_32bit(app_pc module_base);
+
+bool
+module_is_64bit(app_pc module_base);
+
+#ifdef DEBUG
+
+enum {SYMBOLS_LOGLEVEL = 1};
+
+int
+loaded_modules_exports(void);
+
+int
+remove_module_info(app_pc start, size_t size); 
+
+int
+add_module_info(app_pc base_addr, size_t size);
+
+void
+module_cleanup(void);
+
+void
+module_info_exit(void);
+
+#endif /* DEBUG */
+
+bool
+module_file_relocatable(app_pc module_base);
+
+bool module_rebase(app_pc module_base, size_t module_size,
+                   ssize_t relocation_delta);
+bool module_dump_pe_file(HANDLE new_file,
+                         app_pc module_base, size_t module_size);
+
+bool
+module_contents_compare(app_pc original_module_base,
+                        app_pc suspect_module_base,
+                        size_t matching_module_size,
+                        bool relocated,
+                        ssize_t relocation_delta,
+                        size_t validation_section_prefix);
+
+bool
+module_make_writable(app_pc module_base, size_t module_size);
+
+bool
+aslr_compare_header(app_pc original_module_base, size_t original_header_len,
+                    app_pc suspect_module_base);
+
+
+/* in aslr.c */
+const wchar_t *
+get_file_short_name(IN HANDLE file_handle, IN OUT FILE_NAME_INFORMATION *name_info);
+
+void
+aslr_set_last_section_file_name(dcontext_t *dcontext, const wchar_t *short_name);
+
+/* in os.c */
+bool
+safe_write(void *base, size_t size, const void *in_buf);
+
+/* Note that we should keep an eye for potential additional qualifier
+ * flags.  Alternatively we may simply mask off ~0xff to allow for any
+ * future flags added here.
+ */
+#define PAGE_PROTECTION_QUALIFIERS (PAGE_GUARD | PAGE_NOCACHE | PAGE_WRITECOMBINE)
+
+char *
+prot_string(uint prot);
+
+/* FIXME: should we try to alert any dynamo running the other process?
+ * Refer new instances to Case 68
+ */
+#define IPC_ALERT(...) SYSLOG_INTERNAL_WARNING_ONCE("IPC ALERT " __VA_ARGS__)
+
+const PSID
+get_process_primary_SID(void);
+
+#endif /* _OS_PRIVATE_H_ */
diff --git a/core/win32/pre_inject.c b/core/win32/pre_inject.c
new file mode 100644
index 000000000..11e62052e
--- /dev/null
+++ b/core/win32/pre_inject.c
@@ -0,0 +1,501 @@
+/* **********************************************************
+ * Copyright (c) 2001-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2001 Hewlett-Packard Company */
+
+/*
+ * pre_inject.c - small shared library injected into every process
+ *   checks to see if dynamo should be loaded in to take control
+ *
+ * DYNAMORIO_AUTOINJECT:
+ *   points to the dynamorio.dll library to load
+ *   (uses this instead of fixed path off DYNAMORIO_HOME to make
+ *   it easy to switch between libraries used systemwide)
+ *
+ * see inject_shared.c for discussion of variables used to 
+ *  determine injection
+ */
+
+/*
+ * N.B.: if using the user32 registry key to inject systemwide,
+ * only routines from kernel32.dll may be called (not even ones from
+ * user32.dll will work)
+ */
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include <tchar.h>
+#include <stdio.h>
+/* we share ntdll.c w/ the main library */
+#include "globals_shared.h"
+#include "ntdll.h"
+#include "inject_shared.h"
+#include "drmarker.h"
+
+/* case 191729:
+ * Avoid warning C4996: '_snwprintf' was declared deprecated 
+ * We do not want to use the suggested replacement _snwprintf_s as it comes
+ * from msvcr80.dll (though we could statically link w/ the libc version
+ * I suppose).
+ */
+#pragma warning( disable : 4996 )
+
+/* allow converting between data and function pointers */
+#pragma warning(disable : 4055)
+
+/* FIXME : assert stuff, internal error, display_message duplicated from 
+ * pre_inject, share? */
+
+/* for asserts, copied from utils.h */
+#ifdef assert
+# undef assert
+#endif
+/* avoid mistake of lower-case assert */
+#define assert assert_no_good_use_ASSERT_instead
+void internal_error(char *file, int line, char *msg);
+#ifdef DEBUG
+# ifdef INTERNAL
+#   define ASSERT(x)         if (!(x)) internal_error(__FILE__, __LINE__, #x)
+# else 
+#   define ASSERT(x)         if (!(x)) internal_error(__FILE__, __LINE__, "")
+# endif /* INTERNAL */
+#else
+# define ASSERT(x)         ((void) 0)
+#endif
+
+/* in ntdll.c */
+extern char *get_application_name(void);
+extern char *get_application_pid(void);
+
+/* for convenience, duplicated from utils.h, FIXME share */
+#define BUFFER_SIZE_BYTES(buf)      sizeof(buf)
+#define BUFFER_SIZE_ELEMENTS(buf)   (BUFFER_SIZE_BYTES(buf) / sizeof(buf[0]))
+#define BUFFER_LAST_ELEMENT(buf)    buf[BUFFER_SIZE_ELEMENTS(buf) - 1]
+#define NULL_TERMINATE_BUFFER(buf)  BUFFER_LAST_ELEMENT(buf) = 0
+
+/* can only set to 1 for debug builds, unless also set in inject_shared.c */
+/* must turn on VERBOSE in inject_shared.c as well since we're now
+ * using display_verbose_message() -- FIXME: link them automatically */
+#define VERBOSE 0
+
+#if VERBOSE
+/* in inject_shared.c: must turn on VERBOSE=1 there as well */
+void display_verbose_message(char *format, ...);
+# define VERBOSE_MESSAGE(...) display_verbose_message(__VA_ARGS__)
+#else
+# define VERBOSE_MESSAGE(...)
+#endif
+
+static void
+display_error_helper(wchar_t *msg) 
+{
+    wchar_t title_buf[MAX_PATH + 64];
+    _snwprintf(title_buf, BUFFER_SIZE_ELEMENTS(title_buf),
+              L_PRODUCT_NAME L" Notice: %hs(%hs)", 
+              get_application_name(), get_application_pid());
+    NULL_TERMINATE_BUFFER(title_buf);
+    nt_messagebox(msg, title_buf);
+}
+
+void
+internal_error(char *file, int line, char *expr)
+{
+#ifdef INTERNAL
+# define FILENAME_LENGTH L""
+#else 
+/* truncate file name to first character */
+# define FILENAME_LENGTH L".1"   
+#endif
+    wchar_t buf[512];
+    _snwprintf(buf, BUFFER_SIZE_ELEMENTS(buf),
+              L"Preinject Error %" FILENAME_LENGTH L"hs:%d %hs\n", 
+              file, line, expr);
+    NULL_TERMINATE_BUFFER(buf);
+    display_error_helper(buf);
+    TerminateProcess(GetCurrentProcess(), (uint)-1);
+}
+
+#ifdef DEBUG
+void 
+display_error(char *msg)
+{
+    wchar_t buf[512];
+    _snwprintf(buf, BUFFER_SIZE_ELEMENTS(buf), L"%hs", msg);
+    NULL_TERMINATE_BUFFER(buf);
+    display_error_helper(buf);
+}
+#endif /* DEBUG */
+
+typedef int (*int_func_t) ();
+typedef void (*void_func_t) ();
+
+#ifndef X64
+/* in x86/pre_inject.asm */
+extern int
+switch_modes_and_load(void *ntdll64_LdrLoadDll, UNICODE_STRING_64 *lib, HANDLE *result);
+
+/* in x86/pre_inject.asm */
+extern int
+switch_modes_and_call(void_func_t func, void *arg);
+
+static HANDLE
+load_library_64(const char *path)
+{
+    HANDLE ntdll64;
+    HANDLE result;
+    int success;
+    byte *ntdll64_LoadLibrary;
+
+    /* We hand-build our UNICODE_STRING_64 rather than jumping through
+     * hoops to call ntdll64's RtlInitUnicodeString */
+    UNICODE_STRING_64 us;
+    wchar_t wpath[MAXIMUM_PATH + 1];
+    _snwprintf(wpath, BUFFER_SIZE_ELEMENTS(wpath), L"%S", path);
+    NULL_TERMINATE_BUFFER(wpath);
+
+    ASSERT((wcslen(wpath) + 1) * sizeof(wchar_t) <= USHRT_MAX);
+    us.Length = (USHORT) wcslen(wpath) * sizeof(wchar_t);
+    /* If not >= 2 bytes larger then STATUS_INVALID_PARAMETER ((NTSTATUS)0xC000000DL) */
+    us.MaximumLength = (USHORT) (wcslen(wpath) + 1) * sizeof(wchar_t);
+    us.Buffer = wpath;
+    us.Buffer_hi = 0;
+
+    ntdll64 = get_module_handle_64(L"ntdll.dll");
+    if (ntdll64 == NULL)
+        return NULL;
+    VERBOSE_MESSAGE("Found ntdll64 at 0x%08x %s\n", ntdll64, path);
+    /* There is no kernel32 so we use LdrLoadDll.
+     * 32-bit GetProcAddress is doing some header checks and fails,
+     * Our 32-bit get_proc_address does work though.
+     */
+    ntdll64_LoadLibrary = (byte *) get_proc_address_64(ntdll64, "LdrLoadDll");
+    VERBOSE_MESSAGE("Found ntdll64!LdrLoadDll at 0x%08x\n", ntdll64_LoadLibrary);
+    if (ntdll64_LoadLibrary == NULL)
+        return NULL;
+
+    success = switch_modes_and_load(ntdll64_LoadLibrary, &us, &result);
+    VERBOSE_MESSAGE("Loaded at 0x%08x with success 0x%08x\n", result, success);
+    if (success >= 0)
+        return result;
+    else
+        return NULL;
+}
+
+static void
+free_library_64(HANDLE lib)
+{
+    void_func_t ntdll64_LdrUnloadDll;
+    int res;
+    HANDLE ntdll64= get_module_handle_64(L"ntdll.dll");
+    if (ntdll64 == NULL)
+        return;
+    ntdll64_LdrUnloadDll = (void_func_t) get_proc_address_64(ntdll64, "LdrUnloadDll");
+    res = switch_modes_and_call(ntdll64_LdrUnloadDll, (void *)lib);
+    ASSERT(res >= 0);
+}
+#endif
+
+static bool
+load_dynamorio_lib(IF_NOT_X64(bool x64_in_wow64))
+{
+    HMODULE dll = NULL;
+    char path[MAX_PATH];
+#ifdef DEBUG
+    char msg[3*MAX_PATH];
+#endif
+    int retval = -1; /* failure */
+#ifndef X64
+    bool wow64 = is_wow64_process(NT_CURRENT_PROCESS);
+    if (x64_in_wow64) {
+        ASSERT(wow64);
+        retval = get_parameter_64(L_DYNAMORIO_VAR_AUTOINJECT, path, MAX_PATH);
+    } else
+#endif
+        retval = get_parameter(L_DYNAMORIO_VAR_AUTOINJECT, path, MAX_PATH);
+    if (IS_GET_PARAMETER_SUCCESS(retval)) {
+        dr_marker_t mark;
+        VERBOSE_MESSAGE("Loading \"%hs\"", path);
+        /* The read_and_verify_dr_marker is the canonical check for dr in a
+         * process, we double check against GetModuleHandle here just to be
+         * extra safe (in case dr failed to initialize before).  Note that
+         * GetModuleHandle won't find dr's dll if we implement certian -hide
+         * or early_injection proposals. */
+        if (read_and_verify_dr_marker(GetCurrentProcess(), &mark) !=
+            DR_MARKER_FOUND &&
+            GetModuleHandle(DYNAMORIO_LIBRARY_NAME) == NULL
+#ifndef X64 /* these ifdefs are rather ugly: just export all routines in x64 builds? */
+            && /* check for 64-bit as well */
+            (!wow64 || read_and_verify_dr_marker_64(GetCurrentProcess(), &mark) !=
+             DR_MARKER_FOUND)
+            /* FIXME PR 251677: need 64-bit early injection to fully test
+             * read_and_verify_dr_marker_64
+             */
+#endif
+            ) {
+            /* OK really going to load dr now, verify that we are injecting
+             * early enough (i.e. user32.dll is statically linked).  This 
+             * presumes preinject is only used with app_init injection which is
+             * currently the case. FIXME - should we also check_sole_thread
+             * here?  We can't really handle more then one thread when dr is
+             * loading, but this can happen with early remote injected threads
+             * many of which (CTRL) are relatively harmless.
+             */
+            LDR_MODULE *mod = get_ldr_module_by_name(L"user32.dll");
+            ASSERT(mod != NULL);
+            /* The ldr uses -1 as the load count for statically linked dlls
+             * (signals to not bother to keep track of the load count/never
+             * unload).  It doesn't appear to ever use this value for non-
+             * statically linked dlls (including user32.dll if late loaded).
+             */
+            if (mod->LoadCount == -1) {
+#ifndef X64
+                if (x64_in_wow64)
+                    dll = load_library_64(path);
+                else
+#endif
+                    dll = LoadLibrary(path);
+            } else {
+                /* FIXME - would be really nice to communicate this back to
+                 * the controller. */
+#ifdef DEBUG
+                _snprintf(msg, BUFFER_SIZE_ELEMENTS(msg), 
+                          PRODUCT_NAME" Error: improper injection - "PRODUCT_NAME" (%s) can't inject into process %s (%s) (user32.dll not statically linked)\n", 
+                          path, get_application_name(), get_application_pid());
+                NULL_TERMINATE_BUFFER(msg);
+                display_error(msg);
+#endif
+            }
+        } else {
+            /* notify failure only in debug builds, otherwise just return */
+#ifdef DEBUG
+            /* with early injection this becomes even more likely */
+            if (read_and_verify_dr_marker(GetCurrentProcess(), &mark) ==
+                DR_MARKER_FOUND
+# ifndef X64
+                ||
+                (wow64 && read_and_verify_dr_marker_64(GetCurrentProcess(), &mark) ==
+                DR_MARKER_FOUND)
+# endif
+                ) {
+                /* ok, early injection should always beat this */
+#if VERBOSE
+                /* can't readily tell what was expected */
+                _snprintf(msg, BUFFER_SIZE_ELEMENTS(msg), 
+                          PRODUCT_NAME" ok if early injection, otherwise ERROR: double injection, "PRODUCT_NAME" (%s) is already loaded in process %s (%s), continuing\n", 
+                          path, get_application_name(), get_application_pid());
+                NULL_TERMINATE_BUFFER(msg);
+                display_error(msg);
+#endif /* VERBOSE */
+            } else {
+                /* if GetModuleHandle finds us but we don't have a marker
+                 * we may have failed somehow */
+                _snprintf(msg, BUFFER_SIZE_ELEMENTS(msg), 
+                          PRODUCT_NAME" Error: failed injection, "PRODUCT_NAME" (%s) is loaded but not initialized in process %s (%s), continuing\n", 
+                          path, get_application_name(), get_application_pid());
+                NULL_TERMINATE_BUFFER(msg);
+                display_error(msg);
+            }
+#endif /* DEBUG */
+            return false;
+        }
+    } else
+        path[0] = 0;
+    if (dll == NULL) {
+#ifdef DEBUG
+        int err = GetLastError();
+        _snprintf(msg, BUFFER_SIZE_ELEMENTS(msg), 
+                  PRODUCT_NAME" Error %d loading %s\n", err, path);
+        NULL_TERMINATE_BUFFER(msg);
+        display_error(msg);
+#endif
+        return false;
+    } else {
+        int_func_t init_func;
+        void_func_t take_over_func;
+        int res;
+#ifndef X64
+        if (x64_in_wow64) {
+            init_func = (int_func_t) get_proc_address_64(dll, "dynamorio_app_init");
+            take_over_func = (void_func_t)
+                get_proc_address_64(dll, "dynamorio_app_take_over");
+            VERBOSE_MESSAGE("dynamorio_app_init: 0x%08x; dynamorio_app_take_over: 0x%08x\n",
+                            init_func, take_over_func);
+        } else {
+#endif
+            init_func = (int_func_t) GetProcAddress(dll, "dynamorio_app_init");
+            take_over_func = (void_func_t) GetProcAddress(dll, "dynamorio_app_take_over");
+#ifndef X64
+        }
+#endif
+        if (init_func == NULL || take_over_func == NULL) {
+            /* unload the library so that it's clear DR is not in control
+             * (o/w the DR library is in the process and it's not clear
+             * what's going on)
+             */
+#ifndef X64
+            if (x64_in_wow64)
+                free_library_64(dll);
+            else
+#endif
+                FreeLibrary(dll);
+#ifdef DEBUG
+            display_error("Error getting "PRODUCT_NAME" functions\n");
+#endif
+            return false;
+        }
+        VERBOSE_MESSAGE("about to inject dynamorio");
+#ifndef X64
+        if (x64_in_wow64)
+            res = switch_modes_and_call(init_func, NULL);
+        else
+#endif
+            res = (*init_func)();
+        VERBOSE_MESSAGE("dynamorio_app_init() returned %d\n", res);
+#ifndef X64
+        if (x64_in_wow64)
+            switch_modes_and_call(take_over_func, NULL);
+        else
+#endif
+            (*take_over_func)();
+        VERBOSE_MESSAGE("inside "PRODUCT_NAME" now\n");
+    }
+    return true;
+}
+
+static
+int
+parameters_present(IF_NOT_X64(bool x64_in_wow64))
+{
+    char path[MAX_PATH];
+    int retval;
+
+    /* We should do some sanity checking on our parameters, 
+       to make sure we can really inject in applications.
+       War story: When renaming the product from DynamoRIO to SecureCore 
+       we'd start injecting and then failing to load a dll for all apps. 
+    */
+#ifndef X64
+    if (x64_in_wow64) {
+        ASSERT(is_wow64_process(NT_CURRENT_PROCESS));
+        retval = get_parameter_64(L_DYNAMORIO_VAR_AUTOINJECT, path, MAX_PATH);
+    } else
+#endif
+        retval = get_parameter(L_DYNAMORIO_VAR_AUTOINJECT, path, MAX_PATH);
+    if (IS_GET_PARAMETER_SUCCESS(retval)) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+/* forward declaration */
+BOOL APIENTRY 
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved);
+
+void
+process_attach()
+{
+    int rununder_mask;
+    int should_inject;
+    bool takeover = true;
+#if VERBOSE
+    int len;
+    char exename[MAX_PATH];
+#endif
+    /* FIXME: append to event log to indicate we're in the address space */
+    VERBOSE_MESSAGE("inside preinject dll\n");
+
+    ntdll_init();
+
+#if VERBOSE
+    len = GetModuleFileName(NULL, exename, MAX_PATH);
+    ASSERT(len > 0);
+#endif
+#if 0 /* PR 314367: re-enable once it all works */
+#ifndef X64
+    /* PR 253431: one method of injecting 64-bit DR into a WOW64 process is
+     * via 32-bit AppInit drpreinject.
+     * x64 configuration takes precedence over wow64.
+     */
+    if (is_wow64_process(NT_CURRENT_PROCESS)) {
+        should_inject = systemwide_should_inject_64(NULL, &rununder_mask);
+        if (((INJECT_TRUE & should_inject) != 0) &&
+            ((INJECT_EXPLICIT & should_inject) == 0) &&
+            !is_safe_mode() &&
+            parameters_present(true)) {
+            VERBOSE_MESSAGE("<"PRODUCT_NAME" is taking over process %d (%s) as x64>\n",
+                            GetCurrentProcessId(), exename);
+            check_for_run_once(NULL, rununder_mask);
+            /* we commit to x64 takeover based on there being a positive
+             * rununder setting and an AUTOINJECT entry.  if the AUTOINJECT
+             * turns out to be invalid, we'll try the 32-bit.
+             */
+            takeover = !load_dynamorio_lib(true);
+        }
+    }
+#endif
+#endif /* 0 */
+    if (takeover) {
+        should_inject = systemwide_should_inject(NULL, &rununder_mask);
+        if (((INJECT_TRUE & should_inject) == 0) ||
+            ((INJECT_EXPLICIT & should_inject) != 0) ||
+            is_safe_mode() ||
+            !parameters_present(IF_NOT_X64(false))) {
+            /* not taking over */
+            VERBOSE_MESSAGE(PRODUCT_NAME" is NOT taking over process %d (%s)\n",
+                            GetCurrentProcessId(), exename);
+        } else {
+            /* yes, load in dynamo to take over! */
+            VERBOSE_MESSAGE("<"PRODUCT_NAME" is taking over process %d (%s)>\n",
+                            GetCurrentProcessId(), exename);
+            check_for_run_once(NULL, rununder_mask);
+            load_dynamorio_lib(IF_NOT_X64(false));
+        }
+    }
+    ntdll_exit();
+}
+
+/* DLL entry point is in x86/pre_inject.asm */
+BOOL APIENTRY
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved);
+
+/* A dummy exported routine just so the linker will give us an export directory
+ * in the pe.  An export directory is needed to find the pe_name for a dll and
+ * it's nice to be able to do so for at least our own dlls. This doesn't
+ * increase the size of drpreinject.dll. */
+__declspec(dllexport) void
+dr_dummy_function()
+{
+    /* nothing */
+}
diff --git a/core/win32/resources.rc b/core/win32/resources.rc
new file mode 100644
index 000000000..f5206e1f1
--- /dev/null
+++ b/core/win32/resources.rc
@@ -0,0 +1,160 @@
+// **********************************************************
+// Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+// **********************************************************
+
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// 
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+// 
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the following disclaimer in the documentation
+//   and/or other materials provided with the distribution.
+// 
+// * Neither the name of VMware, Inc. nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+// DAMAGE.
+
+// Copyright (c) 2003-2007 Determina Corp.
+
+// resources.rc
+//
+// adds Version information to a dll
+// that can be used by the installer (and by forgetful engineers in Explorer
+// file properties), assumes that FILE_NAME, FILE_DESCRIPTION and FILE_TYPE are defined
+// externally to this file 
+
+#include "winres.h"
+
+#ifdef INCLUDE_EVENTS
+# include "events.rc"
+#endif
+
+// get PRODUCT_NAME, etc, constants
+#include "globals_shared.h"
+
+#ifndef FILE_NAME
+# error FILE_NAME not defined
+#endif
+
+#ifndef FILE_DESCRIPTION
+# error FILE_DESCRIPTION not defined
+#endif
+
+#if !defined(FILE_TYPE) || (FILE_TYPE != VFT_DLL && FILE_TYPE != VFT_APP)
+# error FILE_TYPE not defined
+#endif
+
+#ifndef VERSION_COMMA_DELIMITED
+# define VERSION_COMMA_DELIMITED 4,2,0
+#endif
+// resources may also expect a comma separated string
+#define VERSION_DOT_STRING STRINGIFY(VERSION_COMMA_DELIMITED)
+
+#ifndef BUILD_NUMBER
+# define BUILD_NUMBER 0
+#endif
+
+#define BUILD_NUMBER_STR STRINGIFY(BUILD_NUMBER)
+
+#ifdef INTERNAL
+# include "defines.h"
+# define COMMENT_STR DYNAMORIO_DEFINES
+#else
+# define COMMENT_STR ""
+#endif
+
+#ifdef DEBUG
+# define FILE_DEBUG_FLAG | VS_FF_DEBUG
+#else
+# define FILE_DEBUG_FLAG
+#endif
+
+#ifdef PRERELEASE_BUILD
+# define FILE_PRERELEASE_FLAG | VS_FF_PRERELEASE
+#else
+# define FILE_PRERELEASE_FLAG
+#endif
+
+#ifdef PRIVATE_BUILD
+# define FILE_PRIVATE_FLAG | VS_FF_PRIVATEBUILD
+# ifndef PRIVATE_BUILD_DESC
+#  define PRIVATE_BUILD_DESC ""
+# endif
+#else
+# define FILE_PRIVATE_FLAG
+# ifdef PRIVATE_BUILD_DESC
+#  error /* private desc without flag */
+# else
+#  define PRIVATE_BUILD_DESC ""
+# endif
+#endif
+
+#ifdef SPECIAL_BUILD
+# define FILE_SPECIAL_FLAG | VS_FF_SPECIALBUILD
+# ifndef SPECIAL_BUILD_DESC
+#  define SPECIAL_BUILD_DESC ""
+# endif
+#else
+# define FILE_SPECIAL_FLAG
+# ifdef SPECIAL_BUILD_DESC
+#  error /* special desc without flag */
+# else
+#  define SPECIAL_BUILD_DESC ""
+# endif
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Version
+//
+
+LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
+
+VS_VERSION_INFO VERSIONINFO
+ FILEVERSION VERSION_COMMA_DELIMITED,BUILD_NUMBER
+ PRODUCTVERSION VERSION_COMMA_DELIMITED,BUILD_NUMBER
+ FILEFLAGSMASK 0x3fL
+ FILEFLAGS 0x0L FILE_DEBUG_FLAG FILE_PRERELEASE_FLAG FILE_PRIVATE_FLAG FILE_SPECIAL_FLAG
+ FILEOS 0x4L
+ FILETYPE FILE_TYPE
+ FILESUBTYPE 0x0L
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+        BLOCK "040904b0"
+        BEGIN
+// CHECK: make sure it is OK not to put the terminating null here
+#define ASCIIZ // original contents was in the form "string \0"
+            VALUE "Comments", COMMENT_STR ASCIIZ
+            VALUE "CompanyName", COMPANY_LONG_NAME ASCIIZ
+            VALUE "FileDescription", FILE_DESCRIPTION ASCIIZ
+            VALUE "InternalName", FILE_NAME ASCIIZ
+            VALUE "LegalCopyright", "Copyright (C) " COMPANY_LONG_NAME " 2003-2008" ASCIIZ
+            VALUE "LegalTrademarks", "" ASCIIZ
+            VALUE "OriginalFilename", FILE_NAME ASCIIZ
+            VALUE "PrivateBuild", PRIVATE_BUILD_DESC ASCIIZ
+            VALUE "ProductName", PRODUCT_NAME ASCIIZ
+            VALUE "FileVersion", VERSION_DOT_STRING "," BUILD_NUMBER_STR ASCIIZ
+            VALUE "ProductVersion", VERSION_DOT_STRING "," BUILD_NUMBER_STR ASCIIZ
+            VALUE "SpecialBuild", SPECIAL_BUILD_DESC ASCIIZ
+        END
+    END
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Translation", 0x409, 1200
+    END
+END
diff --git a/core/win32/syscall.c b/core/win32/syscall.c
new file mode 100644
index 000000000..62932465f
--- /dev/null
+++ b/core/win32/syscall.c
@@ -0,0 +1,3274 @@
+/* **********************************************************
+ * Copyright (c) 2006-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2006-2007 Determina Corp. */
+
+/*
+ * syscall.c - win32-specific system call handling routines
+ */
+
+#include "globals.h"
+#include "fragment.h"
+#include "ntdll.h"
+#include "os_private.h"
+#include "aslr.h"
+#include "instrument.h"
+#include "synch.h"
+
+/* this points to one of the os-version-specific system call # arrays below */
+int *syscalls = NULL;
+/* this points to one of the os-version-specific wow64 argument conversion arrays */
+int *wow64_index = NULL;
+
+/* Ref case 5217 - for Sygate compatibility we indirect int 2e system
+ * calls through the int_syscall_address (which after syscalls_init()
+ * will point to an int 2e, ret 0 in ntdll.dll. This is, for all intents
+ * and purposes, a function pointer that will be set only once early
+ * during app init, so we keep it here with the options to leverage their
+ * protection. */ 
+app_pc int_syscall_address = NULL;
+/* Ref case 5441 - for Sygate compatibility we fake our return address from
+ * sysenter system calls (they sometimes verify) to this address which will
+ * (by default) point to a ret 0 in ntdll.dll.  This is, for all intents and
+ * purposes, a function pointer that will be set only once early during app
+ * init, so we keep it here with the options to leverage their protection. */
+app_pc sysenter_ret_address = NULL;
+
+/* Snapshots are relatively heavyweight, so we do not take them on every memory
+ * system call.  On the other hand, if we only did them when we dumped
+ * stats, we'd miss large memory allocations that were freed prior
+ * to the next stats dump (which can be far between if not much new code
+ * is being executed).  Thus, we do them whenever we print stats and on
+ * every memory operation larger than this threshold:
+ */
+#define SNAPSHOT_THRESHOLD (16*PAGE_SIZE)
+
+/*******************************************************/
+
+const char * const syscall_names[] = {
+#define SYSCALL(name, act, nargs, arg32, vista_sp1_x64, vista_sp1, vista_sp0_x64, \
+                vista_sp0, tk3, xp64, wow64, xp, tk, ntsp4, ntsp3, ntsp0) \
+    "Nt"#name,
+#include "syscallx.h"
+#undef SYSCALL
+};
+const int windows_vista_sp1_x64_syscalls[] = {
+#define SYSCALL(name, act, nargs, arg32, vista_sp1_x64, vista_sp1, vista_sp0_x64, \
+                vista_sp0, tk3, xp64, wow64, xp, tk, ntsp4, ntsp3, ntsp0) \
+    vista_sp1_x64,
+#include "syscallx.h"
+#undef SYSCALL
+};
+const int windows_vista_sp1_syscalls[] = {
+#define SYSCALL(name, act, nargs, arg32, vista_sp1_x64, vista_sp1, vista_sp0_x64, \
+                vista_sp0, tk3, xp64, wow64, xp, tk, ntsp4, ntsp3, ntsp0) \
+    vista_sp1,
+#include "syscallx.h"
+#undef SYSCALL
+};
+const int windows_vista_sp0_x64_syscalls[] = {
+#define SYSCALL(name, act, nargs, arg32, vista_sp1_x64, vista_sp1, vista_sp0_x64, \
+                vista_sp0, tk3, xp64, wow64, xp, tk, ntsp4, ntsp3, ntsp0) \
+    vista_sp0_x64,
+#include "syscallx.h"
+#undef SYSCALL
+};
+const int windows_vista_sp0_syscalls[] = {
+#define SYSCALL(name, act, nargs, arg32, vista_sp1_x64, vista_sp1, vista_sp0_x64, \
+                vista_sp0, tk3, xp64, wow64, xp, tk, ntsp4, ntsp3, ntsp0) \
+    vista_sp0,
+#include "syscallx.h"
+#undef SYSCALL
+};
+const int windows_2003_syscalls[] = {
+#define SYSCALL(name, act, nargs, arg32, vista_sp1_x64, vista_sp1, vista_sp0_x64, \
+                vista_sp0, tk3, xp64, wow64, xp, tk, ntsp4, ntsp3, ntsp0) \
+    tk3,
+#include "syscallx.h"
+#undef SYSCALL
+};
+const int windows_XP_x64_syscalls[] = {
+#define SYSCALL(name, act, nargs, arg32, vista_sp1_x64, vista_sp1, vista_sp0_x64, \
+                vista_sp0, tk3, xp64, wow64, xp, tk, ntsp4, ntsp3, ntsp0) \
+    xp64,
+#include "syscallx.h"
+#undef SYSCALL
+};
+const int windows_XP_wow64_index[] = {
+#define SYSCALL(name, act, nargs, arg32, vista_sp1_x64, vista_sp1, vista_sp0_x64, \
+                vista_sp0, tk3, xp64, wow64, xp, tk, ntsp4, ntsp3, ntsp0) \
+    wow64,
+#include "syscallx.h"
+#undef SYSCALL
+};
+const int windows_XP_syscalls[] = {
+#define SYSCALL(name, act, nargs, arg32, vista_sp1_x64, vista_sp1, vista_sp0_x64, \
+                vista_sp0, tk3, xp64, wow64, xp, tk, ntsp4, ntsp3, ntsp0) \
+    xp,
+#include "syscallx.h"
+#undef SYSCALL
+};
+const int windows_2000_syscalls[] = {
+#define SYSCALL(name, act, nargs, arg32, vista_sp1_x64, vista_sp1, vista_sp0_x64, \
+                vista_sp0, tk3, xp64, wow64, xp, tk, ntsp4, ntsp3, ntsp0) \
+    tk,
+#include "syscallx.h"
+#undef SYSCALL
+};
+const int windows_NT_sp4_syscalls[] = {
+#define SYSCALL(name, act, nargs, arg32, vista_sp1_x64, vista_sp1, vista_sp0_x64, \
+                vista_sp0, tk3, xp64, wow64, xp, tk, ntsp4, ntsp3, ntsp0) \
+    ntsp4,
+#include "syscallx.h"
+#undef SYSCALL
+};
+/* for SP3 (and maybe SP2 or SP1 -- haven't checked those) */
+const int windows_NT_sp3_syscalls[] = {
+#define SYSCALL(name, act, nargs, arg32, vista_sp1_x64, vista_sp1, vista_sp0_x64, \
+                vista_sp0, tk3, xp64, wow64, xp, tk, ntsp4, ntsp3, ntsp0) \
+    ntsp3,
+#include "syscallx.h"
+#undef SYSCALL
+};
+/* for SP0 (and maybe SP2 or SP1 -- haven't checked those) */
+const int windows_NT_sp0_syscalls[] = {
+#define SYSCALL(name, act, nargs, arg32, vista_sp1_x64, vista_sp1, vista_sp0_x64, \
+                vista_sp0, tk3, xp64, wow64, xp, tk, ntsp4, ntsp3, ntsp0) \
+    ntsp0,
+#include "syscallx.h"
+#undef SYSCALL
+};
+
+/* for x64 this is the # of args */
+const uint syscall_argsz[] = {
+#ifdef X64
+# define SYSCALL(name, act, nargs, arg32, vista_sp1_x64, vista_sp1, vista_sp0_x64, \
+                 vista_sp0, tk3, xp64, wow64, xp, tk, ntsp4, ntsp3, ntsp0) \
+    nargs,
+#else
+# define SYSCALL(name, act, nargs, arg32, vista_sp1_x64, vista_sp1, vista_sp0_x64, \
+                 vista_sp0, tk3, xp64, wow64, xp, tk, ntsp4, ntsp3, ntsp0) \
+    arg32,
+#endif
+#include "syscallx.h"
+#undef SYSCALL
+};
+
+/* FIXME: currently whether a syscall needs action or not can't be
+ * dynamically changed since this flag is used early on by
+ * intercept_native_syscall() */
+static const int syscall_requires_action[] = {
+#define SYSCALL(name, act, nargs, arg32, vista_sp1_x64, vista_sp1, vista_sp0_x64, \
+                vista_sp0, tk3, xp64, wow64, xp, tk, ntsp4, ntsp3, ntsp0) \
+    act,
+#include "syscallx.h"
+#undef SYSCALL
+};
+
+/* used to intercept syscalls while native */
+static byte *syscall_trampoline_pc[SYS_MAX];
+static app_pc syscall_trampoline_skip_pc[SYS_MAX];
+static app_pc syscall_trampoline_hook_pc[SYS_MAX];
+static app_pc syscall_trampoline_copy_pc[SYS_MAX];
+
+#ifdef GBOP
+/* GBOP stack adjustment - currently either always 0 or always 4 for
+ * vsyscall calls, but may need to be a more general array in case
+ * HOOKED_TRAMPOLINE_HOOK_DEEPER allows different offsets 
+ * FIXME: case 7127 this can be compressed further, if really only a bitmask
+ * see intercept_syscall_wrapper
+ */
+static byte syscall_trampoline_gbop_fpo_offset[SYS_MAX];
+#endif /* GBOP */
+
+/****************************************************************************/
+
+/* System call interception: put any special handling here
+ * Arguments come from the pusha right before the call
+ * Win32 syscall: int 0x2e, number is in eax, address of start of params
+ * on user stack is in edx
+ *
+ * WinXP uses sysenter instruction and does a call to it since sysenter
+ * doesn't store return info -- instead sysexit (called from kernel) grabs
+ * continuation pc from edx.  So the callee, same one used by all syscalls,
+ * puts esp in edx so that kernel just has to dereference it.
+ * Actually, on closer examination, it looks like the kernel sends control
+ * directly to 0x7ffe0304, which does a ret to get back to the ret after
+ * the call %edx -- since the 0x7ffe0304 ret executes natively we can't tell
+ * the difference, but we should be aware of it!  If this is true, why bother
+ * filling in edx for sysenter?  Seems like the kernel must be hardcoding it
+ * with 0x7ffe0304.  
+ * FIXME: think about whether want to
+ * insert a trampoline (and risk clobbering entry point after the ret)
+ * instead of the current method of clobbering the return address
+ *
+ * Here are some win2000 examples (from ntdll.dll):
+NtSetContextThread:
+  77F97BFA: B8 BA 00 00 00     mov         eax,0BAh
+  77F97BFF: 8D 54 24 04        lea         edx,[esp+4]
+  77F97C03: CD 2E              int         2Eh
+  77F97C05: C2 08 00           ret         8
+this is the only one that does not immediately have a ret, though it
+does ret after a jump, some poorly chosen "optimization":
+NtContinue:
+  77F82872: B8 1C 00 00 00     mov         eax,1Ch
+  77F82877: 8D 54 24 04        lea         edx,[esp+4]
+  77F8287B: CD 2E              int         2Eh
+  77F8287D: E9 82 74 01 00     jmp         77F99D04
+  77F99D04: C2 08 00           ret         8
+ *
+ * WinXP example:
+NtOpenKey:
+  0x77f7eb23   b8 77 00 00 00       mov    $0x00000077 -> %eax 
+  0x77f7eb28   ba 00 03 fe 7f       mov    $0x7ffe0300 -> %edx 
+  0x77f7eb2d   ff d2                call   %edx 
+  0x7ffe0300   8b d4                mov    %esp -> %edx 
+  0x7ffe0302   0f 34                sysenter 
+  0x7ffe0304   c3                   ret    %esp (%esp) -> %esp 
+  0x77f7eb2f   c2 0c 00             ret    $0x000c %esp (%esp) -> %esp
+ */
+
+
+/* the win32ksys calls are all above 0x1000, only Zw/Nt* are below */
+#define MAX_NTOSKRNL_SYSCALL_NUM  0x1000
+
+bool
+ignorable_system_call(int num)
+{
+    /* FIXME: this should really be a complete list of ignorable calls,
+     * just ntoskrnl ones that we understand, to avoid surprises
+     * with added calls?
+     */
+    /* FIXME: switch to a bit vector?
+     * we may want an inverted bit vector instead (inw2k p.123  - lower 12 bits) 
+     * there are 285 syscalls on xp  - let's say we support 320
+     * instead of the 40 ints (160 bytes) and a loop we're using now, 
+     * we can grab 40 bytes for 320 syscalls and do the bit extraction  
+     * precomputing from this table will be easy
+     */
+    /* FIXME : it looks like most file IO/creation syscalls are alertable
+     * ref bug 2520, should be added to non-ignorable */
+    /* FIXME : we just return false for all system calls, to be safe we should
+     * really be checking for known ignoreable system calls rather then the reverse,
+     * see syscallx.h for old enumeration. */
+    return false;
+}
+
+bool
+optimizable_system_call(int num)
+{
+    if (INTERNAL_OPTION(shared_eq_ignore))
+        return ignorable_system_call(num);
+    else {
+
+        int i;
+
+        /* FIXME: switch to a bit vector, just as for the syscalls array? */
+        for (i = 0; i < SYS_MAX; i++)
+            if (num == syscalls[i])
+                return !syscall_requires_action[i];
+
+        /* If the syscall isn't in the array, DR doesn't care about it. */
+        return true;
+    }
+}
+
+/* The trampoline handler called for ntdll syscall wrappers that we
+ * care about, so that we can act on them while native_exec-ing
+ */
+after_intercept_action_t
+syscall_while_native(app_state_at_intercept_t *state)
+{
+    int sysnum = (int) (ptr_int_t) state->callee_arg;
+    /* FIXME : if dr calls through ntdll functions that are hooked by a third
+     * party (say Sygate's sysfer.dll) then they could perform syscalls that
+     * would get us here.  Most of the time we'll be ok, but if the current
+     * thread is under_dyn_hack or native_exec we might try to process the
+     * system call or takeover, neither of which is safe.  Currently we avoid
+     * calling through nt wrappers that sysfer.dll hooks (doing system call
+     * internally instead).  This also applies if we call our own hooks, which
+     * we avoid in a similar manner.
+     */
+    /* Returning AFTER_INTERCEPT_LET_GO will perform the syscall natively,
+     * while AFTER_INTERCEPT_LET_GO_ALT_DYN will skip it.  Modify the register
+     * arguments to change the returned state, note that the stack will have
+     * to be popped once (modify reg_esp) to match up the returns.
+     */
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    IF_X64(ASSERT_TRUNCATE(int, int, (ptr_int_t)state->callee_arg));
+    /* N.B.: if any intercepted syscalls are used by DR from ntdll, rather
+     * than custom wrappers, then a recursion-avoidance check here would
+     * be required to avoid infinite loop on error here!
+     */
+    STATS_INC(num_syscall_trampolines);
+    if (dcontext == NULL) {
+        /* unknown thread */
+        return AFTER_INTERCEPT_LET_GO; /* do syscall natively */
+    } else if (dcontext->thread_record->under_dynamo_control == UNDER_DYN_HACK) {
+        /* this trampoline is our ticket to taking control again prior
+         * to the image entry point
+         * we often hit this on NtAllocateVirtualMemory from HeapCreate for
+         * the next dll init after the cb ret where we lost control
+         */
+        STATS_INC(num_syscall_trampolines_retakeover);
+        LOG(THREAD, LOG_SYSCALLS, 1,
+            "syscall_while_native: retakeover in %s after native cb return lost control\n",
+            syscall_names[sysnum]);
+        retakeover_after_native(dcontext->thread_record, INTERCEPT_SYSCALL);
+        return AFTER_INTERCEPT_TAKE_OVER; /* syscall under DR */ 
+    } else if (!dcontext->thread_record->under_dynamo_control
+               /* xref PR 230836 */
+               IF_CLIENT_INTERFACE(&& !IS_CLIENT_THREAD(dcontext))) {
+        /* assumption is that any known native thread is one we control in general,
+         * just not right now while in a native_exec_list dll */
+        STATS_INC(num_syscall_trampolines_native);
+        LOG(THREAD, LOG_SYSCALLS, 1,
+            "NATIVE system call %s\n", syscall_names[sysnum]);
+        DOLOG(IF_DGCDIAG_ELSE(1, 2), LOG_SYSCALLS, {
+            dump_callstack(*((byte **)state->mc.xsp) /*retaddr*/,
+                           (app_pc) state->mc.xbp,
+                           THREAD, DUMP_NOT_XML);
+        });
+
+#ifdef GBOP
+        /* case 7127 - validate GBOP on syscalls that are already hooked for
+         * hotp_only on native_exec 
+         */
+        if (DYNAMO_OPTION(gbop) != GBOP_DISABLED) {
+            /* FIXME: case 7127: should enforce here GBOP_WHEN_NATIVE_EXEC if we
+             * want to apply for -hotp_only but not for native_exec.
+             * Today we always validate.
+             */
+
+            /* FIXME: case 7127: for -exclude_gbop_list need to check a flag
+             * whether this ntdll!Nt* hook has been excluded
+             */
+            /* state->xsp is the wishful thinking after syscall
+             * address, instead of the original one -
+             * intercept_syscall_wrapper() keeps the relevant
+             * FPO information: 4 on XP SP2+, or 0 earlier
+             */
+            gbop_validate_and_act(state, 
+                                  /* adjust ESP */
+                                  syscall_trampoline_gbop_fpo_offset[sysnum], 
+                                  syscall_trampoline_hook_pc[sysnum]);
+            /* if the routine at all returns it passed the GBOP checks */
+
+            /* FIXME: case 7127: may want alternative handling 
+             * and for system calls returning an error of some kind 
+             * like STATUS_INVALID_ADDRESS or STATUS_BUFFER_OVERFLOW
+             * may be a somewhat useful attack handling alternative
+             */
+
+            /* FIXME: case 7127 for completeness should be able to add
+             * this check to the regular DR syscalls where we'll be at
+             * the PC calling sysenter, not necessarily at the start
+             * of a function.  Though other than uniform testing it
+             * won't serve much else.  There we'll have to match the
+             * correct FPO offset at the syscall as well.
+             */
+        }
+#endif /* GBOP */
+        /* Notes on handling syscalls for native threads:
+         * 
+         * FIXME: make sure each syscall handler can handle this thread being native,
+         * as well as target being native.  E.g., will a native thread terminating
+         * itself hit any assertion about not coming back under DR control first?
+         * Another example, will GetCxt fail trying to translate a native thread's
+         * context?
+         * FIXME: what about asynch event while in syscall?  none of ones we
+         * intercept are alertable?
+         * FIXME: exception during pre-syscall sequence can cause us to miss
+         * the go-native trigger!
+         *
+         * Be careful with cache consistency events -- we assume in general that
+         * code executed natively is never mixed with code executed under DR, in
+         * both execution and manipulation, and we try to have _all_ DGC-using
+         * dlls listed in the native_exec_list.  We do handle write faults from
+         * cache consistency in native threads, so we'll have correct behavior,
+         * but we don't want a performance hit from in-cache DGC slowing down
+         * from-native DGC b/c they share memory and it keeps bouncing from RO to
+         * RW -- that's a big reason we're going native in the first place!  For
+         * handling app memory-changing syscalls, we don't mark new code as
+         * read-only until executed from, so in the common case we should not
+         * incur any cost from cache consistency while native.
+         */
+        /* Invoke normal DR syscall-handling by calling dispatch() with a
+         * linkstub_t marked just like those for fragments ending in syscalls.
+         * (We cannot return to the trampoline tail for asynch_take_over() since
+         * it will clobber out next_tag and last_exit and will execute the jmp
+         * back to the syscall under DR, requiring a more intrusive way of going
+         * native afterward.)  Normal handling may skip the syscall or do
+         * whatever, but we expect it to not change control flow (we don't
+         * intercept those while threads are native) and to come out of the
+         * cache and continue on with the next_tag that we set here, which is a
+         * special stopping point routine of ours that causes DR to go native @
+         * the pc we store in dcontext->native_exec_postsyscall.
+         */
+        dcontext->next_tag = BACK_TO_NATIVE_AFTER_SYSCALL;
+        /* start_pc is the take-over pc that will jmp to the syscall instr, while
+         * we need the post-syscall pc, which we stored when generating the trampoline
+         */
+        ASSERT(syscall_trampoline_skip_pc[sysnum] != NULL);
+        dcontext->native_exec_postsyscall = syscall_trampoline_skip_pc[sysnum];
+        ASSERT(dcontext->whereami == WHERE_APP);
+        dcontext->whereami = WHERE_TRAMPOLINE;
+        set_last_exit(dcontext, (linkstub_t *) get_native_exec_syscall_linkstub());
+        /* assumption: no special cleanup from tail of trampoline needed */
+        transfer_to_dispatch(dcontext, state->app_errno, &state->mc);
+        ASSERT_NOT_REACHED();
+    }
+
+    /* This routine tries to handle syscalls from DR, but will fail in some
+     * cases (if the current thread has certain under_dynamo_control values) --
+     * so we use our own custom wrapper rather than go through ntdll when we
+     * expect going through wrapper to reach here (FIXME should do this for
+     * all system calls). */
+    DODEBUG({
+        /* Unfortunately we use various ntdll routines (most notably Ldr*)
+         * that may be hooked (hook code could do anything including making
+         * system calls).  Also some the of the ntdll Rtl routines we
+         * import may be similarly ill behaved (though we don't believe any
+         * of the currently used ones are problematic). Also calling
+         * through Sygate hooks may reach here.
+         */
+        SYSLOG_INTERNAL_WARNING_ONCE("syscall_while_native: using %s - maybe hooked?\n",
+                                     syscall_names[sysnum]);
+    });
+    STATS_INC(num_syscall_trampolines_DR);
+    LOG(THREAD, LOG_SYSCALLS, 1,
+        "WARNING: syscall_while_native: syscall from DR %s\n",
+        syscall_names[sysnum]);
+    return AFTER_INTERCEPT_LET_GO; /* do syscall natively */
+}
+
+
+static inline bool
+intercept_syscall_for_thin_client(int SYSnum)
+{
+    if (SYSnum == SYS_CreateThread ||
+        SYSnum == SYS_CreateProcess ||
+        SYSnum == SYS_CreateProcessEx ||
+        SYSnum == SYS_CreateUserProcess ||
+        SYSnum == SYS_TerminateThread ||    /* Case 9079. */
+        /* case 8866: for -early_inject we must intercept NtMapViewOfSection */
+        (DYNAMO_OPTION(early_inject) && SYSnum == SYS_MapViewOfSection)) {
+        return true;
+    }
+    return false;
+}
+
+static inline bool
+intercept_native_syscall(int SYSnum)
+{
+    /* Don't hook all syscalls for thin_client. */
+    if (DYNAMO_OPTION(thin_client) && !intercept_syscall_for_thin_client(SYSnum))
+        return false;
+
+    if (!syscall_requires_action[SYSnum] || syscalls[SYSnum] == SYSCALL_NOT_PRESENT)
+        return false;
+    /* ignore control transfer system calls:
+     * 1) NtCallbackReturn (assume the corresponding cb was native as well,
+     *                      else we have big problems!  we could detect
+     *                      by stacking up info on native cbs, if nobody ever
+     *                      did an int 2b natively...not worth it for now)
+     * 2) NtContinue
+     * 3) NtCreateThread  
+     *     Ref case 5295 - Sygate hooks this nt wrapper differently then the
+     *     others (@ 2nd instruction).  We only need to hook CreateThread
+     *     system call for follow children from native exec threads anyways, so
+     *     is easiest to just skip this one and live without that ability.
+     * 4) NtWriteVirtualMemory:
+     *    Case 9156/9103: we don't hook it to avoid removing
+     *    our own GBOP hook, until we actually implement acting on it (case 8321)
+     *
+     * We do NOT ignore SetContextThread or suspension/resumption, since
+     * the target could be in DR!
+     */
+    if (SYSnum == SYS_CallbackReturn ||
+        SYSnum == SYS_Continue ||
+        (!DYNAMO_OPTION(native_exec_hook_create_thread) &&
+         SYSnum == SYS_CreateThread) ||
+        SYSnum == SYS_WriteVirtualMemory)
+        return false;
+    return true;
+}
+
+void
+init_syscall_trampolines(void)
+{
+    int i;
+    HMODULE h = (HMODULE)get_ntdll_base();
+    ASSERT(DYNAMO_OPTION(native_exec_syscalls));
+    for (i = 0; i < SYS_MAX; i++) {
+        if (intercept_native_syscall(i)) {
+            byte *fpo_adjustment = NULL;
+#ifdef GBOP
+            fpo_adjustment = &syscall_trampoline_gbop_fpo_offset[i];
+#endif
+
+            syscall_trampoline_hook_pc[i] = (app_pc)get_proc_address(h, syscall_names[i]);
+            syscall_trampoline_pc[i] =
+                /* FIXME: would like to use static references to entry points -- yet,
+                 * set of those we care about varies dynamically by platform, and
+                 * we cannot include a pointer to a 2003-only Nt* entry point and
+                 * avoid a loader link error on 2000, right?
+                 * For now just using get_proc_address!
+                 */
+                intercept_syscall_wrapper(&syscall_trampoline_hook_pc[i],
+                                          syscall_while_native,
+                                          (void *) (ptr_int_t) i /* callee arg */,
+                                          AFTER_INTERCEPT_DYNAMIC_DECISION,
+                                          /* must store the skip_pc for the new dispatch()
+                                           * to know where to go after handling from DR --
+                                           * this is simpler than having trampoline
+                                           * pass it in as an arg to syscall_while_native
+                                           * or trying to decode it.
+                                           */
+                                          &syscall_trampoline_skip_pc[i],
+                                          /* Returns a pointer to a copy of the original
+                                           * first 5 bytes for removing the trampoline
+                                           * later. Excepting hook chaining situations
+                                           * this could just simply be the same as the
+                                           * returned syscall_trampoline_pc. */
+                                          &syscall_trampoline_copy_pc[i],
+                                          fpo_adjustment);
+        }
+    }
+}
+
+void
+exit_syscall_trampolines(void)
+{
+    int i;
+    ASSERT(DYNAMO_OPTION(native_exec_syscalls));
+    for (i = 0; i < SYS_MAX; i++) {
+        if (intercept_native_syscall(i)) {
+            ASSERT(syscall_trampoline_pc[i] != NULL &&
+                   syscall_trampoline_copy_pc[i] != NULL &&
+                   syscall_trampoline_hook_pc[i] != NULL);
+            remove_trampoline(syscall_trampoline_copy_pc[i],
+                              syscall_trampoline_hook_pc[i]);
+        }
+        DEBUG_DECLARE(else ASSERT(syscall_trampoline_pc[i] == NULL));
+    }
+}
+
+#ifdef DEBUG
+void
+check_syscall_array_sizes()
+{
+    ASSERT(sizeof(windows_2003_syscalls) == sizeof(windows_2000_syscalls));
+    ASSERT(sizeof(windows_XP_syscalls) == sizeof(windows_2000_syscalls));
+    ASSERT(sizeof(windows_NT_sp4_syscalls) == sizeof(windows_2000_syscalls));
+    ASSERT(sizeof(windows_NT_sp3_syscalls) == sizeof(windows_2000_syscalls));
+    ASSERT(sizeof(windows_NT_sp0_syscalls) == sizeof(windows_2000_syscalls));
+    ASSERT(sizeof(windows_2000_syscalls)/sizeof(windows_2000_syscalls[0]) == 
+           sizeof(syscall_requires_action)/sizeof(syscall_requires_action[0]));
+    ASSERT(sizeof(windows_2000_syscalls)/sizeof(windows_2000_syscalls[0]) == 
+           sizeof(syscall_names)/sizeof(syscall_names[0]));
+}
+
+/* verify that syscall numbers match our static lists in an attempt to catch
+ * changes to syscall interface across Windows patches and service packs
+ */
+void
+check_syscall_numbers(dcontext_t *dcontext)
+{
+    int i;
+    int sysnum;
+    byte *addr;
+    module_handle_t h = get_ntdll_base();
+    ASSERT(h != NULL && h != INVALID_HANDLE_VALUE);
+    LOG(GLOBAL, LOG_SYSCALLS, 4, "check_syscall_numbers: ntdll @ "PFX"\n", h);
+    for (i = 0; i < SYS_MAX; i++) {
+        if (syscalls[i] == SYSCALL_NOT_PRESENT)
+            continue;
+        addr = (byte *)get_proc_address(h, syscall_names[i]);
+        ASSERT(addr != NULL);
+        LOG(GLOBAL, LOG_SYSCALLS, 4,
+            "\tsyscall 0x%x %s: addr "PFX"\n", i, syscall_names[i], addr);
+        sysnum = decode_syscall_num(dcontext, addr);
+        /* because of Sygate hooks can't assert sysnum is valid here */
+        if (sysnum >= 0 && sysnum != syscalls[i]) {
+            SYSLOG_INTERNAL_ERROR("syscall %s is really 0x%x not 0x%x\n",
+                                  syscall_names[i], sysnum, syscalls[i]);
+            syscalls[i] = sysnum;
+            /* of course is much too late to fix if we already used via
+             * NT_SYSCALL */
+        }
+    }
+}
+#endif
+
+/* adjust region to page boundaries, since Windows lets you pass
+ * non-aligned values, unlike Linux
+ * e.g. a two byte cross-page request will result in a two page region
+ */
+static inline void
+align_page_boundary(dcontext_t *dcontext, 
+                    app_pc *base /* IN OUT */, size_t *size/* IN OUT */)
+{
+    if (!ALIGNED(*base, PAGE_SIZE) || !ALIGNED(*size, PAGE_SIZE)) {
+        /* need to cover all pages overlapping the region [base, base + size) */
+        *size = ALIGN_FORWARD(*base+*size, PAGE_SIZE) - PAGE_START(*base);
+        *base = (app_pc) PAGE_START(*base);
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+            "\talign_page_boundary => base="PFX" size="PIFX"\n",  *base, *size);
+    }
+}
+
+/* verifies whether target process is being created, presumably as a
+ * child of the current process
+ */
+bool
+is_newly_created_process(HANDLE process_handle)
+{
+    uint remote_ldr_data;
+    /* We check based on - trait 3) PEB.Ldr
+     *    The Ldr entry is created by the running process itself later */
+
+    /* ATTIC - rejected traits
+     * trait 1) it doesn't have any threads created
+     * Seems overly expensive to have no easy alternative to
+     * NtQuerySystemInformation to tell there are no threads created
+     * in the process, should use to verify new process since that
+     * should be the rare case
+     * FIXME: could at least store the last created pid and a flag indicating if
+     * its thread has been created and use that as an auxiliary check
+     *
+     * May be easier to check the PEB
+     * trait 2) PEB.ProcessParameters
+     * The process parameters are available only after they have been created,
+     * (in fact a good trait that a process without them has just been created,
+     * yet they are created at the time the first thread's stack is needed.
+     *
+     * NOTE - in Vista traits 1 and 2 are no longer valid for this
+     * purpose.  NtCreateUserProcess creates the first thread and sets up
+     * the process parameters in addition to creating the process.  However this
+     * is only used for aslr_stack so doesn't really matter that much. Trait 3
+     * (the one we use) should still work anyways (and cover anyone using the legacy
+     * native interface NtCreateProcess to create the process).
+     */
+
+    DODEBUG({
+        /* dead end approach, this code can be removed*/
+
+        /* invalid trait 4: shouldn't have many handles open
+         * Attempted using NtQueryInformationProcess
+         * ProcessHandleCount which is usually 1 on XP at the time
+         * a new process is created, if it holds on all platforms
+         * 
+         * Note unfortunately this cannot be counted on, since handles may
+         * be inherited - and processes created by cygwin do inherit a lot
+         * of handles.
+         */
+        ulong remote_process_handle_count;
+        NTSTATUS res = get_process_handle_count(process_handle, 
+                                                &remote_process_handle_count);
+        if (NT_SUCCESS(res)) {
+            LOG(GLOBAL, LOG_ALL, 2, 
+                "is_newly_created_process: process "IDFMT" has %d handles -> %s\n",
+                process_id_from_handle(process_handle),
+                remote_process_handle_count,
+                remote_process_handle_count == 1 ? "NEW" : "maybe new");
+        }
+    });
+
+    remote_ldr_data = get_remote_process_ldr_status(process_handle);
+    if (remote_ldr_data >= 0) {
+        LOG(GLOBAL, LOG_ALL, 1, 
+            "is_newly_created_process: process "IDFMT" PEB->Ldr = %s\n",
+            process_id_from_handle(process_handle),
+            remote_ldr_data != 0 ? "initialized" : "NULL -> new process");
+        
+        return (remote_ldr_data == 0); /* new process */
+    } else {
+        /* xref case 9800 - can happen if the app handle lacks the rights we
+         * need (in which case isn't a new process since the handle used then has
+         * full rights). Get handle rights in local since won't be available in an
+         * ldmp. */
+        DEBUG_DECLARE(ACCESS_MASK rights = nt_get_handle_access_rights(process_handle);)
+        ASSERT_CURIOSITY(get_os_version() == WINDOWS_VERSION_VISTA &&
+                         "xref case 9800, is_newly_created_process failure");
+    }
+
+    return false;
+}
+
+/* FIXME : For int/syscall we can just subtract 2 from the post syscall pc but for
+ * sysenter we do the post-syscall ret native and therefore we've lost the 
+ * address of the actual syscall, but we are only going to use this for 
+ * certain ntdll system calls so is almost certainly the ntdll sysenter.  As
+ * a hack for now we just use the address of the first system call we saw
+ * (which should be ntdll's), this is good enough for detach and prob. good
+ * enough for app GetThreadContext (we could just use 0x7ffe0302 but it moved 
+ * on xp sp2) */
+#define SYSCALL_PC(dc) \
+ ((get_syscall_method() == SYSCALL_METHOD_INT ||     \
+   get_syscall_method() == SYSCALL_METHOD_SYSCALL) ? \
+  (ASSERT(SYSCALL_LENGTH == INT_LENGTH),             \
+   POST_SYSCALL_PC(dc) - INT_LENGTH) : \
+  (get_syscall_method() == SYSCALL_METHOD_WOW64 ? \
+   (POST_SYSCALL_PC(dc) - CTI_FAR_ABS_LENGTH) : \
+   get_app_sysenter_addr()))
+
+/* since always coming from dispatch now, only need to set mcontext */
+#define SET_RETURN_VAL(dc, val) \
+  get_mcontext(dc)->xax = (reg_t) (val);
+
+/***************************************************************************
+ * PRE SYSTEM CALL
+ *
+ * FIXME: should we pass mcontext to these routines to avoid
+ * the get_mcontext() call and derefs?
+ * => now we're forcing the inline of get_mcontext() so should be fine
+ */
+
+static reg_t *
+pre_system_call_param_base(dr_mcontext_t *mc)
+{
+#ifdef X64
+    reg_t *param_base = (reg_t *) mc->xsp;
+#else
+    reg_t *param_base = (reg_t *) mc->xdx;
+#endif
+    param_base += (SYSCALL_PARAM_OFFSET() / sizeof(reg_t));
+    return param_base;
+}
+
+/* NtCreateProcess, NtCreateProcessEx */
+static void
+presys_CreateProcess(dcontext_t *dcontext, reg_t *param_base, bool ex)
+{
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    HANDLE *process_handle = (HANDLE *) sys_param(dcontext, param_base, 0);
+    uint access_mask = (uint) sys_param(dcontext, param_base, 1);
+    uint attributes  = (uint) sys_param(dcontext, param_base, 2);
+    uint inherit_from_process = (uint) sys_param(dcontext, param_base, 3);
+    bool inherit_handles_only = (uint) sys_param(dcontext, param_base, 4);
+    HANDLE section_handle = (HANDLE) sys_param(dcontext, param_base, 5);
+    HANDLE debug_handle = (HANDLE) sys_param(dcontext, param_base, 6);
+    HANDLE exception_handle = (HANDLE) sys_param(dcontext, param_base, 7);
+
+    if (ex) {
+        /* according to metasploit, others type as HANDLE unknown etc. */
+        uint job_member_level = (uint) sys_param(dcontext, param_base, 8);
+    }
+
+    /* Case 9173: guard against pid reuse.  Better in post after success
+     * check but not a big deal.
+     * We don't do this on CreateThread b/c is_newly_created_process() is still
+     * true after the first thread (one fix is to store the last created pid and
+     * a flag indicating if its thread has been created and use that as an auxiliary
+     * check in is_newly_created_process())
+     */
+    dcontext->aslr_context.last_child_padded = 0;
+
+    DOLOG(1, LOG_SYSCALLS, {
+        app_pc base = (app_pc) get_section_address(section_handle);
+        /* we will inject in post_syscall or when the first thread is about
+         * to be created */
+        LOG(THREAD, LOG_SYSCALLS, IF_DGCDIAG_ELSE(1, 2),
+            "syscall: NtCreateProcess section @"PFX"\n", base);
+        DOLOG(1, LOG_SYSCALLS, {
+            char buf[MAXIMUM_PATH];
+            get_module_name(base, buf, sizeof(buf));
+            if (buf[0] != '\0')
+                LOG(THREAD, LOG_SYSCALLS, 2,
+                    "\tNtCreateProcess for module %s\n", buf);
+        });
+    });
+}
+
+#ifdef DEBUG
+/* NtCreateUserProcess */
+static void
+presys_CreateUserProcess(dcontext_t *dcontext, reg_t *param_base)
+{
+    /* New in Vista, here's what I got reverse engineering
+     * NtCreateUserProcess (11 args, using windows types)
+     *
+     * NtCreateUserProcess (
+     *   OUT PHANDLE ProcessHandle,
+     *   OUT PHANDLE ThreadHandle,
+     *   IN ACCESS_MASK ProcDesiredAccess,
+     *   IN ACCESS_MASK ThreadDesiredAccess,
+     *   IN POBJECT_ATTRIBUTES ProcObjectAttributes,
+     *   IN POBJECT_ATTRIBUTES ThreadObjectAttributes,
+     *   IN uint? unknown,  [ observed 0x4 ]
+     *   IN BOOL CreateSuspended, [ refers to the thread not the process ]
+     *   IN PRTL_USER_PROCESS_PARAMETERS Params,
+     *   INOUT proc_stuff proc,
+     *   INOUT create_proc_thread_info_t *thread [ see ntdll.h ])
+     * CreateProcess hardcodes 0x2000000 (== MAXIMUM_ALLOWED) for both
+     * ACCESS_MASK arguments.  I've only observed NULL (== default) for the
+     * OBJECT_ATTRIBUTES arguments so they are a bit of a guess, but they
+     * need to be here somewhere and based on error codes I know they are
+     * ptr arguments so seems quite likely esp. given the arg layout.
+     *
+     * where proc_stuff { \\ speculative - the 64bit differences are odd and imply
+     *                    \\ more then just size changes
+     *   size_t struct_size, [observed 0x48 (0x58 for 64bit)]  \\ prob. sizeof(proc_stuff)
+     *   ptr_uint_t unknown_p2,       \\ OUT
+     *   ptr_uint_t unknown_p3,       \\ IN/OUT
+     *   OUT HANDLE file_handle, [exe file handle]
+     *   OUT HANDLE section_handle, [exe section handle]
+     *   uint32 unknown_p6,           \\ OUT
+     *   uint32 unknown_p7,           \\ OUT
+     *   uint32 unknown_p8,           \\ OUT
+     *   uint32 unknown_p9,           \\ OUT
+     * #ifndef X64
+     *   uint32 unknown_p10,          \\ OUT
+     * #endif
+     *   OUT PEB *new_proc_peb,
+     *   uint32 unknown_p12_p17[6],   \\ OUT
+     * #ifndef X64
+     *   uint32 unknown_p18,          \\ OUT
+     * #endif
+     * }
+     */
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    ACCESS_MASK proc_access_mask = (uint) sys_param(dcontext, param_base, 2);
+    ACCESS_MASK thread_access_mask = (uint) sys_param(dcontext, param_base, 3);
+    bool create_suspended = (bool) sys_param(dcontext, param_base, 7);
+    create_proc_thread_info_t *thread_stuff = (void *) sys_param(dcontext, param_base, 10);
+    ASSERT(get_os_version() == WINDOWS_VERSION_VISTA);
+
+    /* might need these in post, note CreateProcess appears to hardcode them */
+    ASSERT_CURIOSITY(proc_access_mask == MAXIMUM_ALLOWED);
+    ASSERT_CURIOSITY(thread_access_mask == MAXIMUM_ALLOWED);
+    ASSERT_CURIOSITY(create_suspended);
+    /* FIXME - NYI - if any of the above curiosities don't hold we should
+     * change them here and then fixup as needed in post. */
+
+    /* Potentially dangerous deref of app ptr, but is only for debug logging */
+    ASSERT(thread_stuff != NULL && thread_stuff->nt_path_to_exe.buffer != NULL);
+    LOG(THREAD, LOG_SYSCALLS, 1, "syscall: NtCreateUserProcess presys %.*S\n",
+        MIN(MAXIMUM_PATH, thread_stuff->nt_path_to_exe.buffer_size),
+        (wchar_t *)thread_stuff->nt_path_to_exe.buffer);
+}
+#endif /* DEBUG */
+
+/* NtCreateThread */
+static void
+presys_CreateThread(dcontext_t *dcontext, reg_t *param_base)
+{
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    HANDLE *thread_handle= (HANDLE *) sys_param(dcontext, param_base, 0);
+    uint access_mask   = (uint) sys_param(dcontext, param_base, 1);
+    uint attributes    = (uint) sys_param(dcontext, param_base, 2);
+    HANDLE process_handle= (HANDLE) sys_param(dcontext, param_base, 3);
+    uint *client_id    = (uint*) sys_param(dcontext, param_base, 4);
+    CONTEXT *cxt         = (CONTEXT *) sys_param(dcontext, param_base, 5);
+    USER_STACK *stack    = (USER_STACK *) sys_param(dcontext, param_base, 6);
+    bool   suspended     = (uint) sys_param(dcontext, param_base, 7);
+    DEBUG_DECLARE(process_id_t pid = process_id_from_handle(process_handle);)
+    LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, IF_DGCDIAG_ELSE(1, 2),
+        "syscall: NtCreateThread pid="PFX" suspended=%d\n",
+        pid, suspended);
+    LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, 2,
+        "\tstack: "PFX" "PFX" "PFX" "PFX" "PFX"\n",
+        stack->FixedStackBase, stack->FixedStackLimit,
+        stack->ExpandableStackBase, stack->ExpandableStackLimit,
+        stack->ExpandableStackBottom);
+    /* According to Nebbett, in eax is the win32 start address
+     * (stored in ThreadQuerySetWin32StartAddress slot, though that
+     * is reused by the os, so might not be the same later) and eax is used
+     * by the thread start kernel32 thunk.  It also appears from the thunk 
+     * that the argument to the thread start function is in ebx */
+    LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, 2,
+        "\tesp="PFX", xip="PFX"\n\tstart address "PFX" with arg "PFX"\n",
+        cxt->CXT_XSP, cxt->CXT_XIP, cxt->CXT_XAX, cxt->CXT_XBX);
+    DOLOG(2, LOG_SYSCALLS|LOG_THREADS, {
+        char buf[MAXIMUM_PATH];
+        print_symbolic_address((app_pc)cxt->CXT_XAX, buf, sizeof(buf), false);
+        LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, 2, 
+            "\tsymbol info for start address : %s\n", buf);
+    });
+    /* if not early injecting, we will unsafely modify cxt (for late follow
+     * children) FIXME */
+    ASSERT(cxt != NULL);
+    maybe_inject_into_process(dcontext, process_handle, cxt);
+}
+
+#ifdef DEBUG
+/* NtCreateThreadEx */
+static void
+presys_CreateThreadEx(dcontext_t *dcontext, reg_t *param_base)
+{
+    /* New in Vista, here's what I got reverse engineering NtCreateThreadEx
+     * (11 args, using windows types)
+     *
+     * NtCreateThreadEx (
+     *   OUT PHANDLE ThreadHandle,
+     *   IN ACCESS_MASK DesiredAccess,
+     *   IN POBJECT_ATTRIBUTES ObjectAttributes,
+     *   IN HANDLE ProcessHandle,
+     *   IN LPTHREAD_START_ROUTINE Win32StartAddress,
+     *   IN LPVOID StartParameter,
+     *   IN BOOL CreateSuspended,
+     *   IN uint unknown, [ CreateThread hardcodes to 0 ]
+     *   IN SIZE_T StackCommitSize,
+     *   IN SIZE_T StackReserveSize,
+     *   INOUT create_thread_info_t *thread_info [ see ntdll.h ])
+     */
+    DEBUG_DECLARE(dr_mcontext_t *mc = get_mcontext(dcontext);)
+    DEBUG_DECLARE(HANDLE process_handle = (HANDLE) sys_param(dcontext, param_base, 3);)
+    DEBUG_DECLARE(byte *start_addr = (byte *) sys_param(dcontext, param_base, 4);)
+    DEBUG_DECLARE(void *start_parameter = (void *) sys_param(dcontext, param_base, 5);)
+    DEBUG_DECLARE(bool create_suspended = (bool) sys_param(dcontext, param_base, 6);)
+    DEBUG_DECLARE(process_id_t pid = process_id_from_handle(process_handle);)
+    ASSERT(get_os_version() == WINDOWS_VERSION_VISTA);
+
+    LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, 2,
+        "syscall: NtCreateThread pid="PFX" suspended=%d\n"
+        "\tstart_addr="PFX" arg="PFX"\n",
+        pid, create_suspended, start_addr, start_parameter);
+    DOLOG(2, LOG_SYSCALLS|LOG_THREADS, {
+        char buf[MAXIMUM_PATH];
+        print_symbolic_address(start_addr, buf, sizeof(buf), false);
+        LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, 2, 
+            "\tsymbol info for start address : %s\n", buf);
+    });
+}
+#endif
+
+/* NtTerminateProcess */
+static bool /* returns whether to execute syscall */
+presys_TerminateProcess(dcontext_t *dcontext, reg_t *param_base)
+{
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    HANDLE process_handle = (HANDLE) sys_param(dcontext, param_base, 0);
+    NTSTATUS exit_status = (NTSTATUS) sys_param(dcontext, param_base, 1);
+    LOG(THREAD, LOG_SYSCALLS, 1,
+        "syscall: NtTerminateProcess handle="PFX" pid=%d exit=%d\n",
+        process_handle,
+        process_id_from_handle((process_handle == 0) ? NT_CURRENT_PROCESS : process_handle),
+        exit_status);
+    if (process_handle == 0) {
+        NTSTATUS return_val;
+        thread_record_t **threads;
+        int num_threads;
+        dr_mcontext_t mcontext;
+        DEBUG_DECLARE(bool ok;)
+        /* this thread won't be terminated! */
+        LOG(THREAD, LOG_SYSCALLS, 2, "terminating all other threads, not this one\n");
+        copy_mcontext(mc, &mcontext);
+        mc->pc = SYSCALL_PC(dcontext);
+
+#ifdef CLIENT_INTERFACE
+        /* make sure client nudges are finished */
+        wait_for_outstanding_nudges();
+#endif
+
+        /* FIXME : issues with cleaning up here what if syscall fails */
+        DEBUG_DECLARE(ok =)
+            synch_with_all_threads(THREAD_SYNCH_SUSPENDED_AND_CLEANED, 
+                                   &threads, &num_threads, 
+                                   /* Case 6821: while we're ok to be detached, we're
+                                    * not ok to be reset since we won't have the
+                                    * last_exit flag set for coming back here (plus
+                                    * our kstats get off since we didn't yet enter
+                                    * the cache)
+                                    */
+                                   THREAD_SYNCH_VALID_MCONTEXT_NO_XFER,
+                                   /* if we fail to suspend a thread (e.g., privilege
+                                    * problems) ignore it. FIXME: retry instead? */
+                                   THREAD_SYNCH_SUSPEND_FAILURE_IGNORE);
+        ASSERT(ok);
+        ASSERT(threads == NULL && num_threads == 0); /* We asked for CLEANED */
+        copy_mcontext(&mcontext, mc);
+
+        /* we hold the initexit lock at this point, but we cannot release
+         * it, b/c a new thread waiting on it could start initializing and
+         * then we'd issue the syscall and kill it while it's holding our
+         * lock, causing a deadlock when the subsequent process-terminating
+         * syscall comes in! (==case 4243) So, we hold the lock to issue
+         * the syscall, safest to do syscall right here rather than going
+         * back to handle_system_call()
+         */
+        return_val = nt_terminate_process_for_app(process_handle, exit_status);
+        SET_RETURN_VAL(dcontext, return_val);
+        LOG(THREAD, LOG_SYSCALLS, 2,
+            "\tNtTerminateProcess("PFX", "PFX") => "PIFX" on behalf of app\n",
+            process_handle, exit_status, return_val);
+
+        end_synch_with_all_threads(threads, num_threads, false/*no resume*/);
+            
+        return false; /* do not execute syscall -- we already did it */
+    } else if (is_phandle_me((process_handle == 0) ? NT_CURRENT_PROCESS : process_handle)) {
+        /* case 10338: we don't synchall here for faster shutdown, but we have
+         * to try and not crash any other threads.  FIXME: if it's rare to get here
+         * w/ > 1 thread perhaps we should do the synchall.
+         */
+        LOG(THREAD, LOG_SYSCALLS, 2, "\tterminating process w/ %d running thread(s)\n",
+            get_num_threads());
+        KSTOP(pre_syscall);
+        KSTOP(num_exits_dir_syscall);
+        /* FIXME: what if syscall returns w/ STATUS_PROCESS_IS_TERMINATING? */
+        cleanup_and_terminate(dcontext, syscalls[SYS_TerminateProcess],
+                              IF_X64_ELSE(mc->xcx, mc->xdx),
+                              mc->xdx, true /* entire process */);
+    }
+    return true;
+}
+
+/* NtTerminateThread */
+static void
+presys_TerminateThread(dcontext_t *dcontext, reg_t *param_base)
+{
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    HANDLE thread_handle = (HANDLE) sys_param(dcontext, param_base, 0);
+    /* need to determine which thread is being terminated
+     * it's harder than you'd think -- we can get its handle but
+     * the handle may have been duplicated, no way to test
+     * equivalence, we have to get the thread id
+     */
+    thread_id_t tid;
+    thread_record_t *tr = thread_lookup(get_thread_id());
+    ASSERT(tr != NULL);
+    if (thread_handle == 0)
+        thread_handle = NT_CURRENT_THREAD;
+    tid = thread_id_from_handle(thread_handle);
+    LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, 1, "syscall: NtTerminateThread tid=%d\n", tid);
+
+    if (tid == 0xFFFFFFFF) {
+        /* probably invalid handle, do nothing for now */
+        /* FIXME: case 2573 about adding ASSERT_CURIOSITY replacing the ASSERT we had */
+    } else if (tid != tr->id) {
+        dr_mcontext_t mcontext;
+        DEBUG_DECLARE(thread_synch_result_t synch_res;)
+
+        copy_mcontext(mc, &mcontext);
+        mc->pc = SYSCALL_PC(dcontext);
+          
+        /* Fixme : issues with cleaning up here, what if syscall fails */
+        DEBUG_DECLARE(synch_res =)
+            synch_with_thread(tid, true, false, THREAD_SYNCH_VALID_MCONTEXT, 
+                              THREAD_SYNCH_SUSPENDED_AND_CLEANED,
+                              /* if we fail to suspend a thread (e.g., privilege
+                               * problems) ignore it. FIXME: retry instead? */
+                              THREAD_SYNCH_SUSPEND_FAILURE_IGNORE);
+        ASSERT(synch_res == THREAD_SYNCH_RESULT_SUCCESS ||
+               /* App could be calling on already exited thread (xref 8125)
+                * or thread could have exited while we were synching.
+                * FIXME - check is racy since for dr purposes the thread is
+                * considered exited just before it is signaled, but is ok
+                * for an assert. */
+               is_thread_exited(thread_handle) == THREAD_EXITED ||
+               !is_pid_me(process_id_from_thread_handle(thread_handle)));
+        copy_mcontext(&mcontext, mc);
+    } else {
+        /* case 9347 - racy early thread, yet primary is not yet 'known' */
+        /* we should evaluate dr_late_injected_primary_thread before
+         * get_num_threads()
+         */
+        bool secondary = dr_injected_secondary_thread && 
+            !dr_late_injected_primary_thread;
+
+        bool exitproc = !secondary && (get_num_threads() == 1 && !dynamo_exited);
+        /* this should really be check_sole_thread() */
+        /* FIXME: case 9461 - we may not control all threads,
+         * the syscall may fail and may not be allowed to kill last thread
+         */
+
+        if (secondary) {
+            SYSLOG_INTERNAL_WARNING("secondary thread terminating, primary not ready\n");
+            ASSERT(!exitproc);
+            ASSERT(!check_sole_thread());
+        }
+        ASSERT(!exitproc || check_sole_thread());
+
+        KSTOP(pre_syscall);
+        KSTOP(num_exits_dir_syscall);
+        cleanup_and_terminate(dcontext, syscalls[SYS_TerminateThread],
+                              IF_X64_ELSE(mc->xcx, mc->xdx),
+                              mc->xdx, exitproc);
+    }
+}
+
+/* NtSetContextThread */
+static bool
+presys_SetContextThread(dcontext_t *dcontext, reg_t *param_base)
+{
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    HANDLE thread_handle = (HANDLE) sys_param(dcontext, param_base, 0);
+    CONTEXT *cxt = (CONTEXT *) sys_param(dcontext, param_base, 1);
+    thread_id_t tid = thread_id_from_handle(thread_handle);
+    bool intercept = true;
+    bool execute_syscall = true;
+    /* FIXME : we are going to read and write to cxt, which may be unsafe */
+    ASSERT(tid != 0xFFFFFFFF);
+    LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, IF_DGCDIAG_ELSE(1, 2),
+        "syscall: NtSetContextThread handle="PFX" tid=%d cxt->Xip="PFX"\n",
+        thread_handle, tid, cxt->CXT_XIP);
+    mutex_lock(&thread_initexit_lock); /* need lock to lookup thread */
+    if (intercept_asynch_for_thread(tid, false/*no unknown threads*/)) {
+        dr_mcontext_t mcontext;
+        thread_record_t *tr = thread_lookup(tid);
+        CONTEXT *my_cxt;
+        NTSTATUS res;
+        const thread_synch_state_t desired_state = THREAD_SYNCH_VALID_MCONTEXT;
+        DEBUG_DECLARE(thread_synch_result_t synch_res;)
+        ASSERT(tr != NULL);
+        SELF_PROTECT_LOCAL(tr->dcontext, WRITABLE);
+        /* now ensure target thread is at a safe point when it gets reset */
+        copy_mcontext(mc, &mcontext);
+        mc->pc = SYSCALL_PC(dcontext);
+            
+        DEBUG_DECLARE(synch_res =)
+            synch_with_thread(tid, true, true, desired_state, 
+                              THREAD_SYNCH_SUSPENDED_VALID_MCONTEXT,
+                              /* if we fail to suspend a thread (e.g., privilege
+                               * problems) ignore it. FIXME: retry instead? */
+                              THREAD_SYNCH_SUSPEND_FAILURE_IGNORE);
+        ASSERT(synch_res == THREAD_SYNCH_RESULT_SUCCESS);
+        copy_mcontext(&mcontext, mc);
+        if (!TESTALL(CONTEXT_CONTROL/*2 bits so ALL*/, cxt->ContextFlags)) {
+            /* app didn't request pc so we'd better get it now.
+             * FIXME: this isn't transparent as we have to clobber
+             * fields in the app cxt: should restore in post-syscall.
+             */
+            CONTEXT alt_cxt;
+            alt_cxt.ContextFlags = CONTEXT_DR_STATE;
+            STATS_INC(num_app_setcontext_no_control);
+            if (thread_get_context(tr, &alt_cxt) &&
+                translate_context(tr, &alt_cxt, true/*set memory*/)) {
+                LOG(THREAD, LOG_SYSCALLS, 2, "no CONTROL flag on original cxt:\n");
+                DOLOG(3, LOG_SYSCALLS, { dump_context_info(cxt, THREAD, true); });
+                cxt->ContextFlags |= CONTEXT_CONTROL;
+                cxt->CXT_XIP = alt_cxt.CXT_XIP;
+                cxt->CXT_XFLAGS = alt_cxt.CXT_XFLAGS;
+                cxt->CXT_XSP = alt_cxt.CXT_XSP;
+                cxt->CXT_XBP = alt_cxt.CXT_XBP;
+                IF_X64(ASSERT_NOT_IMPLEMENTED(false)); /* Rbp not part of CONTROL */
+                cxt->SegCs = alt_cxt.SegCs;
+                cxt->SegSs = alt_cxt.SegSs;
+                LOG(THREAD, LOG_SYSCALLS, 3, "changed cxt:\n");
+                DOLOG(3, LOG_SYSCALLS, { dump_context_info(cxt, THREAD, true); });
+                /* don't care about other regs -- if app didn't
+                 * specify CONTEXT_INTEGER that's fine
+                 */
+            } else {
+                /* just don't intercept: could crash us in middle of mangled
+                 * sequence once we start translating there and treating them
+                 * as safe spots, but for now will be ok.
+                 */
+                intercept = false;
+                ASSERT_NOT_REACHED();
+            }
+        }
+        if (intercept) {
+            /* modify the being-set cxt so that we retain control */
+            intercept_nt_setcontext(tr->dcontext, cxt);
+            LOG(THREAD, LOG_SYSCALLS, 3, "final cxt passed to syscall:\n");
+            DOLOG(3, LOG_SYSCALLS, { dump_context_info(cxt, THREAD, true); });
+        }
+        /* nt_continue_dynamo_start path assumes target is !couldbelinking
+         * all synch_with_thread synch points should be, we check here
+         */
+        ASSERT(!is_couldbelinking(tr->dcontext));
+        if (TEST(THREAD_SET_CONTEXT, nt_get_handle_access_rights(thread_handle))) {
+            /* Case 10101: a thread waiting at check_wait_at_safe_spot can't
+             * be directly setcontext-ed so we explicitly do the context
+             * set request here and skip the system call.
+             * A waiting thread does NtContinue and so bypasses permission issues,
+             * so we explicitly check for setcontext permission.
+             * We have to make a copy since the app could de-allocate or modify
+             * cxt before a waiting thread examines it.
+             */
+            DEBUG_DECLARE(bool ok;)
+#ifdef X64
+            /* PR 263338: we need to align to 16 on x64.  Heap is 8-byte aligned. */
+            byte *cxt_alloc;
+#endif
+            my_cxt = global_heap_alloc(CONTEXT_HEAP_SIZE(*my_cxt) HEAPACCT(ACCT_OTHER));
+#ifdef X64
+            cxt_alloc = (byte *) cxt;
+            if (!ALIGNED(cxt, 16)) {
+                ASSERT(ALIGNED(cxt, 8));
+                cxt = (CONTEXT *) ( ((app_pc)cxt)+8 );
+            }
+            ASSERT(ALIGNED(cxt, 16));
+#endif
+            *my_cxt = *cxt;
+            /* my_cxt is freed by set_synched_thread_context() or target thread */
+            DEBUG_DECLARE(ok = )
+                set_synched_thread_context(tr, NULL, (void *) my_cxt,
+                                           CONTEXT_HEAP_SIZE(*my_cxt), desired_state
+                                           _IF_X64(cxt_alloc) _IF_WINDOWS(&res));
+            /* We just tested permissions, but could be bad handle, etc.
+             * FIXME: if so and thread was waiting we have transparency violation
+             */
+            ASSERT_CURIOSITY(ok);
+            SET_RETURN_VAL(tr->dcontext, res);
+            /* must wake up thread so it can go to nt_continue_dynamo_start */
+            nt_thread_resume(tr->handle, NULL);
+            execute_syscall = false;
+        } else {
+            /* we expect the system call to fail */
+            DODEBUG({ tr->dcontext->expect_last_syscall_to_fail = true; });
+        }
+        SELF_PROTECT_LOCAL(tr->dcontext, READONLY);
+    }
+    mutex_unlock(&thread_initexit_lock);
+    return execute_syscall;
+}
+
+/* Assumes mc is app state prior to system call.
+ * Returns true iff system call is a callback return that does transfer control
+ * (xref case 10579).
+ */
+bool
+is_cb_return_syscall(dcontext_t *dcontext)
+{
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    if (mc->xax == (reg_t) syscalls[SYS_CallbackReturn]) {
+        reg_t *param_base = pre_system_call_param_base(mc);
+        if ((NTSTATUS)sys_param(dcontext, param_base, 2) != STATUS_CALLBACK_POP_STACK)
+            return true;
+    }
+    return false;
+}
+
+/* NtCallbackReturn */
+static void
+presys_CallbackReturn(dcontext_t *dcontext, reg_t *param_base)
+{
+    /* args are:
+     *   IN PVOID Result OPTIONAL, IN ULONG ResultLength, IN NTSTATUS Status
+     * same args go to int 2b (my theory anyway), where they are passed in
+     * eax, ecx, and edx.  if KiUserCallbackDispatcher returns, it leaves
+     * eax w/ result value of callback, and zeros out ecx and edx, then int 2b.
+     * people doing the int 2b in user32 set ecx and edx to what they want, then
+     * call a routine that simply pulls first arg into eax and then does int 2b.
+     */
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    NTSTATUS status = (NTSTATUS) sys_param(dcontext, param_base, 2);
+    if (status == STATUS_CALLBACK_POP_STACK) {
+        /* case 10579: this status code instructs the kernel to only
+         * pop the stack and not transfer control there */
+        LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, IF_DGCDIAG_ELSE(1, 2),
+            "syscall: NtCallbackReturn STATUS_CALLBACK_POP_STACK\n");
+    } else {
+        /* NtCallbackReturn returns from callback via a syscall, and it
+         * requires us to restore the prev dcontext immediately prior
+         * to the syscall (want to use current dcontext in prior instructions
+         * in shared_syscall).
+         * N.B.: this means that the return from the call to pre_system_call
+         * uses a different dcontext than the setup for the call!
+         * the popa and popf will be ok -- old dstack is still in esp, isn't
+         * restored, isn't deleted by swapping to new dcontext.
+         * The problem is the restore of the app's esp -- so we fix that by
+         * having the clean call to pre_system_call store and restore app's esp
+         * from a special nonswapped dcontext slot.
+         */
+        LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, IF_DGCDIAG_ELSE(1, 2),
+            "syscall: NtCallbackReturn\n");
+        callback_start_return(0, mc);
+    }
+}
+
+static void
+check_for_stack_free(dcontext_t *dcontext, byte *base, size_t size)
+{
+    /* Ref case 5518 - on some versions of windows the thread stack is freed
+     * in process.  So we watch here for the free to keep from removing again
+     * at thread exit. */
+    os_thread_data_t *ostd = (os_thread_data_t *) dcontext->os_field;
+    ASSERT(dcontext == get_thread_private_dcontext());
+    if (base == ostd->stack_base) {
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+            "Thread's os stack is being freed\n");
+        ASSERT(base + size == ostd->stack_top);
+        /* only seen the in process free on 2k and NT */
+        ASSERT_CURIOSITY(get_os_version() <= WINDOWS_VERSION_2000);
+        /* When we've seen it happen (in kernel32!ExitThread), ExitThread uses
+         * a chunk of the TEB as the stack while freeing and calling 
+         * NtTerminate. */
+        ASSERT_CURIOSITY((byte *)get_mcontext(dcontext)->xsp >=
+                         (byte *)get_own_teb() &&
+                         (byte *)get_mcontext(dcontext)->xsp <
+                         ((byte *)get_own_teb()) + PAGE_SIZE);
+        /* FIXME - Instead of saying the teb stack is no longer valid, we could
+         * instead change the bounds to be the TEB region.  Other users could
+         * then always we assert we have something valid set. Is slightly
+         * greater dependence on observed behavior though. */
+        ostd->teb_stack_no_longer_valid = true;
+        ostd->stack_base = NULL;
+        ostd->stack_top = NULL;
+    }
+}
+
+#ifdef PROGRAM_SHEPHERDING
+/* NtAllocateVirtualMemory */
+static void
+presys_AllocateVirtualMemory(dcontext_t *dcontext, reg_t *param_base)
+{
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    HANDLE process_handle = (HANDLE) sys_param(dcontext, param_base, 0);
+    void **pbase = (void **) sys_param(dcontext, param_base, 1);
+    uint type = (uint) sys_param(dcontext, param_base, 4);
+    uint prot = (uint) sys_param(dcontext, param_base, 5);
+    app_pc base;
+    if (is_phandle_me(process_handle) && TEST(MEM_COMMIT, type) &&
+        TESTALL(PAGE_EXECUTE_READWRITE, prot)) {
+        /* executable_if_alloc policy says we only add a region to the future
+         * list if it is committed rwx with no prior reservation.
+         * - if a base is passed and MEM_RESERVE is not set, there must be a prior
+         *   reservation
+         * - if a base is passed and MEM_RESERVE is set, do a query to see if
+         *   reservation existed before
+         * - if no base is passed, there was no reservation
+         */
+        /* unfortunately no way to avoid syscall to check readability
+         * (unless have try...except)
+         */
+        if (safe_read(pbase, sizeof(base), &base)) {
+            dcontext->alloc_no_reserve = 
+                (base == NULL ||
+                 (TEST(MEM_RESERVE, type) && !get_memory_info(base, NULL, NULL, NULL)));
+            /* FIXME: can one MEM_RESERVE an address previously
+             * MEM_RESERVEd - at least on XP that's not allowed */ 
+        }
+    } else if (TEST(ASLR_STACK, DYNAMO_OPTION(aslr)) &&
+               !is_phandle_me(process_handle) && 
+               TEST(MEM_RESERVE, type)
+               && is_newly_created_process(process_handle)) {
+        /* pre-processing of remote NtAllocateVirtualMemory reservation */
+        /* Case 9173: ignore allocations with a requested base.  These may come
+         * after we've inserted our pad (is_newly_created_process() isn't
+         * perfect), but may also come before, and we do not want to cause
+         * interop issues.  We could instead try to adjust our pad to not cause
+         * their alloc to fail, but may end up eliminating any security
+         * advantage anyway.
+         */
+        if (safe_read(pbase, sizeof(base), &base)) {
+            if (base == NULL) {
+                /* FIXME: make the above check stronger */
+                ASSERT_CURIOSITY(prot == PAGE_READWRITE);
+                /* this is just a reservation, so can be anything */
+                
+                /* currently not following child flags, so maybe is almost always */
+                /* NOTE - on vista we should only ever get here if someone is using
+                 * the legacy NtCreateProcess native api (vs NtCreateUserProcess) or
+                 * the app is injecting memory into a new process before it's started
+                 * initializing itself. */
+                ASSERT_CURIOSITY(get_os_version() != WINDOWS_VERSION_VISTA);
+                aslr_maybe_pad_stack(dcontext, process_handle);
+            } else {
+                DODEBUG({
+                    if (process_id_from_handle(process_handle) !=
+                        dcontext->aslr_context.last_child_padded) {
+                        SYSLOG_INTERNAL_WARNING_ONCE("aslr stack: allowing alloc prior "
+                                                     "to pad");
+                    }
+                });
+            }
+        }
+    }
+}
+#endif /* PROGRAM_SHEPHERDING */
+
+/* NtFreeVirtualMemory */
+static void
+presys_FreeVirtualMemory(dcontext_t *dcontext, reg_t *param_base)
+{
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    HANDLE process_handle = (HANDLE) sys_param(dcontext, param_base, 0);
+    void **pbase = (void **) sys_param(dcontext, param_base, 1);
+    size_t *psize = (size_t *) sys_param(dcontext, param_base, 2);
+    uint type = (uint) sys_param(dcontext, param_base, 3);
+    app_pc base;
+    size_t size;
+        
+    /* check for common argument problems, apps tend to screw this call
+     * up a lot (who cares about a memory leak, esp. at process exit) */
+    /* ref case 3536, 545, 4046 */
+    if (!safe_read(pbase, sizeof(base), &base) || base == NULL ||
+        !safe_read(psize, sizeof(size), &size) || 
+        !(type == MEM_RELEASE || type == MEM_DECOMMIT)) {
+        /* we expect the system call to fail */
+        DODEBUG(dcontext->expect_last_syscall_to_fail = true;);
+        return;
+    }
+
+    if (!is_phandle_me(process_handle)) {
+        IPC_ALERT("ERROR: FreeVirtualMemory %s "PFX" "PIFX" on another process\n",
+                  type == MEM_DECOMMIT ? "MEM_DECOMMIT" : "MEM_RELEASE",
+                  base, size);
+        return;
+    }
+
+    if ((type == MEM_DECOMMIT && size == 0) || (type == MEM_RELEASE)) {
+        app_pc real_base;
+        /* whole region being freed, we must look up size, ignore psize
+         * msdn and Nebbet claim that you need *psize == 0 for MEM_RELEASE
+         * but that doesn't seem to be true on all platforms */
+
+        /* 2K+: if base is anywhere on the first page of region this succeeds,
+         * and doesn't otherwise.
+         * NT: base must be the actual base.
+         */
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+            "syscall: NtFreeVirtualMemory type=%s region base="PFX" size="PIFX"\n",
+            type == MEM_DECOMMIT ? "MEM_DECOMMIT" : "MEM_RELEASE",
+            base, size);
+
+        size = get_allocation_size(base, &real_base);
+        ASSERT(ALIGNED(real_base, PAGE_SIZE));
+        /* if region has been already been freed */
+        if (((app_pc) ALIGN_BACKWARD(base, PAGE_SIZE) != real_base) ||
+            (get_os_version() == WINDOWS_VERSION_NT && base != real_base)) {
+            /* we expect the system call to fail 
+             * with (NTSTATUS) 0xc000009f -
+             * "Virtual memory cannot be freed as base address is not
+             * the base of the region and a region size of zero was
+             * specified"
+             */
+            LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                "syscall: NtFreeVirtualMemory base="PFX", size="PIFX" invalid base\n",
+                base, size);
+            DODEBUG(dcontext->expect_last_syscall_to_fail = true;);
+            return;
+        }
+        /* make sure we use correct region base address, */
+        /* otherwise we'll free an extra page */
+        base = real_base;
+        ASSERT(real_base != NULL && "already freed");
+    }
+
+    DODEBUG({
+        /* FIXME: this shouldn't be DODEBUG since we need to handle syscall failure */
+        if (type == MEM_DECOMMIT && size != 0) {
+            size_t real_size = get_allocation_size(base, NULL);
+            if ((app_pc)ALIGN_BACKWARD(base, PAGE_SIZE) + real_size < base + size) {
+                /* we expect the system call to fail with
+                 * (NTSTATUS) 0xc000001a - "Virtual memory cannot be freed."
+                 */
+                DODEBUG(dcontext->expect_last_syscall_to_fail = true;);
+                LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                    "syscall: NtFreeVirtualMemory base="PFX", size="PIFX
+                    " too large should fail \n", base, size);
+                return;
+            }
+        }
+    });
+
+
+    LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+        "syscall: NtFreeVirtualMemory base="PFX" size="PIFX"\n",
+        base, size);
+    DOLOG(1, LOG_SYSCALLS|LOG_VMAREAS, {
+        char buf[MAXIMUM_PATH];
+        get_module_name(base, buf, sizeof(buf));
+        if (buf[0] != '\0') {
+            LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+                "\tNtFreeVirtualMemory called on module %s\n", buf);
+            ASSERT_CURIOSITY(false && "NtFreeVirtualMemory called on module");
+            /* should switch to PE name and then can do this at loglevel 0 */
+        }
+    });
+    DOLOG(1, LOG_MEMSTATS, {
+        /* snapshots are heavyweight, so do rarely */
+        if (size > SNAPSHOT_THRESHOLD)
+            mem_stats_snapshot();
+    });
+        
+    align_page_boundary(dcontext, &base, &size);
+    ASSERT_BUG_NUM(4511, ALIGNED(base, PAGE_SIZE) && ALIGNED(size, PAGE_SIZE));
+    /* ref case 5518 - we need to keep track if the thread stack is freed */
+    if (type == MEM_RELEASE) {
+        check_for_stack_free(dcontext, base, size);
+    }
+    if (type == MEM_RELEASE && 
+        TEST(ASLR_HEAP_FILL, DYNAMO_OPTION(aslr))) {
+        /* We free our allocation before the application
+         * reservation is released.  Not a critical failure if the
+         * application free fails but we have freed our pad.  Also
+         * avoids fragmentation if a racy allocation.
+         */
+        aslr_pre_process_free_virtual_memory(dcontext, base, size);
+        /* note we handle the untracked stack free in
+         * os_thread_stack_exit() */
+    }
+
+    app_memory_deallocation(dcontext, base, size, 
+                            false /* don't own thread_initexit_lock */,
+                            false /* not image */);
+}
+
+/* NtProtectVirtualMemory */
+static bool /* returns whether to execute syscall */
+presys_ProtectVirtualMemory(dcontext_t *dcontext, reg_t *param_base)
+{
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    HANDLE process_handle = (HANDLE) sys_param(dcontext, param_base, 0);
+    void **pbase = (void **) sys_param(dcontext, param_base, 1);
+    size_t *psize = (size_t *) sys_param(dcontext, param_base, 2);
+    uint prot = (uint) sys_param(dcontext, param_base, 3);
+    uint *oldprot = (uint *) sys_param(dcontext, param_base, 4);
+    app_pc base;
+    size_t size;
+    uint old_memprot = MEMPROT_NONE; /* for SUBSET_APP_MEM_PROT_CHANGE 
+                                      * or PRETEND_APP_MEM_PROT_CHANGE */
+    uint subset_memprot = MEMPROT_NONE; /* for SUBSET_APP_MEM_PROT_CHANGE */
+        
+    if (!safe_read(pbase, sizeof(base), &base) || 
+        !safe_read(psize, sizeof(size), &size)) {
+        /* we expect the system call to fail */
+        DODEBUG(dcontext->expect_last_syscall_to_fail = true;);
+        return true;
+    }
+
+    LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+        "syscall: NtProtectVirtualMemory process="PFX" base="PFX" size="
+        PIFX" prot=%s 0x%x\n", process_handle, base, size, prot_string(prot), prot);
+    if (is_phandle_me(process_handle)) {
+        uint res;
+        /* go to page boundaries, since windows lets you pass non-aligned
+         * values, unlike Linux
+         */
+        /* FIXME: use align_page_boundary(dcontext, &base, &size) instead */
+        if (!ALIGNED(base, PAGE_SIZE) || !ALIGNED(base+size, PAGE_SIZE)) {
+            /* need to cover all pages between base and base + size */
+            size = ALIGN_FORWARD(base+size, PAGE_SIZE) - PAGE_START(base);
+            base = (app_pc) PAGE_START(base);
+            LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+                "\tpage boundaries => base="PFX" size="PIFX"\n",  base, size);
+        }
+        DOLOG(1, LOG_SYSCALLS|LOG_VMAREAS, {
+            char module_name[MAX_MODNAME_INTERNAL];
+            if (os_get_module_name_buf(base, module_name,
+                                       BUFFER_SIZE_ELEMENTS(module_name)) > 0) {
+                LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+                    "\tNtProtectVirtualMemory called on module %s\n", module_name);
+            }
+        });
+#ifdef DGC_DIAGNOSTICS
+        DOLOG(1, LOG_VMAREAS, {
+            dump_callstack(POST_SYSCALL_PC(dcontext),
+                           (app_pc) mc->xbp, THREAD,
+                           DUMP_NOT_XML);
+        });
+#endif
+        res = app_memory_protection_change(dcontext, base, size, 
+                                           osprot_to_memprot(prot),
+                                           &subset_memprot,
+                                           &old_memprot);
+        if (res != DO_APP_MEM_PROT_CHANGE) {
+            /* from experimentation it seems to return 
+             * STATUS_CONFLICTING_ADDRESSES
+             * rather than STATUS_NOT_COMMITTED for invalid memory
+             */
+            if (res == FAIL_APP_MEM_PROT_CHANGE) {
+                SET_RETURN_VAL(dcontext, STATUS_CONFLICTING_ADDRESSES);
+            } else if (res == PRETEND_APP_MEM_PROT_CHANGE ||
+                       res == SUBSET_APP_MEM_PROT_CHANGE) {
+                /*
+                 * FIXME: is alternative of letting it go through and undoing in
+                 * post-handler simpler and safer (here we have to emulate kernel
+                 * behavior), if we remove +w flag to avoid other-thread issues?
+                 */
+                uint pretend_oldprot;
+                uint old_osprot = PAGE_NOACCESS;
+                SET_RETURN_VAL(dcontext, STATUS_SUCCESS);
+
+                if (res == SUBSET_APP_MEM_PROT_CHANGE) {
+                    uint subset_osprot = 
+                        osprot_replace_memprot(prot, subset_memprot);
+                
+                    /* we explicitly make our system call.  Although in this
+                     * case we could change the application arguments as
+                     * well, in general it is not nice to the application
+                     * to change IN arguments.
+                     */
+                    bool ok = 
+                        nt_remote_protect_virtual_memory(process_handle,
+                                                         base, size, 
+                                                         subset_osprot, &old_osprot);
+                    /* using app's handle in case it has different rights that current thread */
+                    ASSERT_CURIOSITY(process_handle == NT_CURRENT_PROCESS);
+                    ASSERT_CURIOSITY(ok);
+                    /* we'll keep going anyways as if it would have worked */
+                } else {
+                    ASSERT_NOT_TESTED();
+                    ASSERT(res == PRETEND_APP_MEM_PROT_CHANGE);
+                    /* pretend it worked but don't execute system call */
+                    old_osprot = get_current_protection(base);
+                }
+
+                /* Today we base on the current actual flags
+                 * (old_osprot), and preserve WRITECOPY and other
+                 * unlikely original flags.
+                 *
+                 * We should be using our value for what the correct
+                 * view of the application memory should be.  case
+                 * 10437 we should be able to transparently carry the
+                 * original protection flags across multiple calls to
+                 * NtProtectVirtualMemory.
+                 */
+                pretend_oldprot = osprot_replace_memprot(old_osprot,
+                                                         old_memprot);
+
+                /* have to set OUT vars properly */
+                /* size and base were already aligned up above */
+                ASSERT(ALIGNED(size, PAGE_SIZE));
+                ASSERT(ALIGNED(base, PAGE_SIZE));
+                LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                    "skipping NtProtectVirtualMemory, returning base="PFX", size="
+                    PIFX", oldprot=%s 0x%x\n",
+                    base, size, prot_string(pretend_oldprot), pretend_oldprot);
+
+                /* FIXME: we really should be _probing_ these writes
+                 * to make sure not targeting DR addresses when
+                 * PROTECT_FROM_APP
+                 */
+                safe_write(oldprot, sizeof(pretend_oldprot), &pretend_oldprot);
+                safe_write(pbase, sizeof(base), &base);
+                safe_write(psize, sizeof(size), &size);
+            } else {
+                ASSERT_NOT_REACHED();
+            }
+
+            return false; /* do not execute system call */
+        } else {
+            /* FIXME: ASSERT here that have not modified size unless using, e.g. fix_unsafe_hooker */
+        }
+    } else {
+        /* FIXME: should we try to alert any dynamo running the other process?
+         */
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+            "WARNING: ProtectVirtualMemory called on process "PFX" %d\n",
+            process_handle, process_id_from_handle(process_handle));
+        /* this actually happens (e.g., in calc.exe's winhlp popups)
+         * so don't die here with IPC_ALERT
+         */
+    }
+    return true;
+}
+
+/* NtMapViewOfSection */
+static void
+presys_MapViewOfSection(dcontext_t *dcontext, reg_t *param_base)
+{
+    DODEBUG({
+        dr_mcontext_t *mc = get_mcontext(dcontext);
+        HANDLE section_handle = (HANDLE) sys_param(dcontext, param_base, 0);
+        /* trying to make sure we're tracking properly all section
+         * handles
+         * 
+         * Unfortunately SHELL32!SHChangeRegistration_Create seems
+         * to be using sections to communicate with explorer.exe
+         * and sends a message via sending a duplicate section
+         * handle, and likely receives a message back in a
+         * similarly duplicated handle from the other process.
+         * Hard to match that particular call so cannot keep a
+         * CURIOSITY here.
+         * 
+         * Note we also wouldn't like some global handle being used by
+         * different threads as well, or any other unusually nested
+         * use of NtCreateSection/NtOpenSection before NtMapViewOfSection.
+         */
+        if (!(section_handle == dcontext->aslr_context.
+              last_app_section_handle || 
+              section_handle == dcontext->aslr_context.
+              randomized_section_handle)) {
+            LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS,
+                1, "syscall: NtMapViewOfSection unusual section mapping");
+        }
+    });
+
+    /* no pre-processing needed except for ASLR */
+    if (TESTANY(ASLR_DLL|ASLR_MAPPED, DYNAMO_OPTION(aslr))) {
+        aslr_pre_process_mapview(dcontext);
+    }
+}
+
+/* NtUnmapViewOfSection */
+static void
+presys_UnmapViewOfSection(dcontext_t *dcontext, reg_t *param_base)
+{
+    /* This is what actually removes a dll from memory */
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    HANDLE process_handle = (HANDLE) sys_param(dcontext, param_base, 0);
+    app_pc base = (app_pc) sys_param(dcontext, param_base, 1);
+    app_pc real_base;
+    size_t size = get_allocation_size(base, &real_base);
+    MEMORY_BASIC_INFORMATION mbi;
+    LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+        "syscall: NtUnmapViewOfSection "PFX" size="PIFX"\n", base, size);
+        
+    if (!is_phandle_me(process_handle)) {
+        IPC_ALERT("ERROR: UnmapViewOfSection on another process\n");
+        return;
+    }
+
+    /* check for args we expect to fail, ref case 545, 3697, on east coast
+     * xp server shell32 dllmain process attach calls kernel32 
+     * CreateActCtxW which ends up calling this with an unaligned pointer
+     * into private memory (which is suspicously just a few bytes under
+     * the base address of a recently freed mapped region) */
+    /* Don't worry about the query_virtual_memory cost, we are already 
+     * doing a ton of them for the get_allocation_size and process_mmap
+     * calls */
+    if (query_virtual_memory(base, &mbi, sizeof(mbi)) != sizeof(mbi) ||
+        (mbi.Type != MEM_IMAGE && mbi.Type != MEM_MAPPED)) {
+        DODEBUG(dcontext->expect_last_syscall_to_fail = true;);
+        return;
+    }
+    /* people don't always call with the actual base address (see east 
+     * coast xp server (sp1) whose uxtheme.dll CThemeSignature::
+     * CalculateHash always calls this with base+0x130, is hardcoded in 
+     * the assembly).  OS doesn't seem to care as the syscall still 
+     * succeeds. */
+    if (base != mbi.AllocationBase) {
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+            "syscall: NtUnmapViewOfSection real base is "PFX"\n",
+            mbi.AllocationBase);
+        base = mbi.AllocationBase;
+    }
+        
+    DOLOG(1, LOG_MEMSTATS, {
+        /* snapshots are heavyweight, so do rarely */
+        if (size > SNAPSHOT_THRESHOLD)
+            mem_stats_snapshot();
+    });
+    RSTATS_INC(num_app_munmaps);
+
+    /* we have to mark before any policy processing gets started */
+
+    /* FIXME: we could also allow MEM_MAPPED areas here, since .B
+     * policies may in fact allow such to be executable areas, but
+     * since we can keep track of only one, focusing on MEM_IMAGE only
+     */
+    if (DYNAMO_OPTION(unloaded_target_exception) && 
+        mbi.Type == MEM_IMAGE) {
+        mark_unload_start(base, size);
+    }
+
+    if (TESTANY(ASLR_DLL|ASLR_MAPPED, DYNAMO_OPTION(aslr))) {
+        aslr_pre_process_unmapview(dcontext, base, size);
+    }
+    process_mmap(dcontext, base, size, false/*unmap*/);
+}
+
+/* NtFlushInstructionCache */
+static void
+presys_FlushInstructionCache(dcontext_t *dcontext, reg_t *param_base)
+{
+    /* This syscall is from the days when Windows ran on multiple
+     * architectures, but many apps still use it
+     */
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    HANDLE process_handle = (HANDLE) sys_param(dcontext, param_base, 0);
+    app_pc base = (app_pc) sys_param(dcontext, param_base, 1);
+    size_t size = (size_t) sys_param(dcontext, param_base, 2);
+#ifdef PROGRAM_SHEPHERDING
+    uint prot;
+#endif
+    /* base can be NULL, in which case size is meaningless
+     * loader calls w/ NULL & 0 on rebasing -- means entire icache?
+     */
+    LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+        "syscall: NtFlushInstructionCache "PFX" size="PIFX"\n", base, size);
+    if (base == NULL)
+        return;
+    if (is_phandle_me(process_handle)) {
+#ifdef DGC_DIAGNOSTICS
+        DOLOG(1, LOG_VMAREAS, {
+            dump_callstack(POST_SYSCALL_PC(dcontext),
+                           (app_pc) mc->xbp, THREAD,
+                           DUMP_NOT_XML);
+        });
+#endif
+#ifdef PROGRAM_SHEPHERDING
+        prot = osprot_to_memprot(get_current_protection(base));
+        app_memory_flush(dcontext, base, size, prot);
+#endif
+    } else {
+        /* FIXME: should we try to alert any dynamo running the other process?
+         * no reason to ASSERT here, not critical like alloc/dealloc in other process
+         */
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+            "WARNING: NtFlushInstructionCache on another process\n");
+    }
+}
+
+/* NtCreateSection */
+static void
+presys_CreateSection(dcontext_t *dcontext, reg_t *param_base)
+{
+    /* a section is an object that can be mmapped */
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    HANDLE *section_handle = (HANDLE*) sys_param(dcontext, param_base, 0);
+    uint access_mask = (uint) sys_param(dcontext, param_base, 1);
+    POBJECT_ATTRIBUTES obj = (POBJECT_ATTRIBUTES) sys_param(dcontext, param_base, 2);
+    void *size = (void *) sys_param(dcontext, param_base, 3);
+    uint protect = (uint) sys_param(dcontext, param_base, 4);
+    uint attributes = (uint) sys_param(dcontext, param_base, 5);
+    HANDLE file_handle = (HANDLE) sys_param(dcontext, param_base, 6);
+    LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+        "syscall: NtCreateSection protect 0x%x, attributes 0x%x\n", protect, attributes);
+
+    DODEBUG({
+        if (obj != NULL && obj->ObjectName != NULL) {
+            DEBUG_DECLARE(char buf[MAXIMUM_PATH];)
+            /* convert name from unicode to ansi */
+            wchar_t *name = obj->ObjectName->Buffer;
+            _snprintf(buf, BUFFER_SIZE_ELEMENTS(buf), "%S", name);
+            NULL_TERMINATE_BUFFER(buf);
+            LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+                "syscall: NtCreateSection %s\n", buf);
+        } else {
+            LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+                "syscall: NtCreateSection\n");
+        }
+    });
+}
+
+#ifdef DEBUG
+/* NtOpenFile */
+static void
+presys_OpenFile(dcontext_t *dcontext, reg_t *param_base)
+{
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    HANDLE *file_handle = (HANDLE*) sys_param(dcontext, param_base, 0);
+    uint access_mask = (uint) sys_param(dcontext, param_base, 1);
+    POBJECT_ATTRIBUTES obj = (POBJECT_ATTRIBUTES) sys_param(dcontext, param_base, 2);
+    void *status = (void *) sys_param(dcontext, param_base, 3);
+    uint share = (uint) sys_param(dcontext, param_base, 4);
+    uint options = (uint) sys_param(dcontext, param_base, 5);
+    if (obj != NULL) {
+        /* convert name from unicode to ansi */
+        char buf[MAXIMUM_PATH];
+        wchar_t *name = obj->ObjectName->Buffer;
+        _snprintf(buf, BUFFER_SIZE_ELEMENTS(buf), "%S", name);
+        NULL_TERMINATE_BUFFER(buf);
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+            "syscall: NtOpenFile %s\n", buf);
+    } else {
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+            "syscall: NtOpenFile\n");
+    }
+}
+#endif
+
+/* WARNING: flush_fragments_and_remove_region assumes that pre and post system
+ * call handlers do not examine or modify fcache or its fragments in any
+ * way except for calling flush_fragments_and_remove_region!
+ */
+bool
+pre_system_call(dcontext_t *dcontext)
+{
+    bool execute_syscall = true;
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    int sysnum = (int) mc->xax;
+    reg_t *param_base = pre_system_call_param_base(mc);
+    where_am_i_t old_whereami = dcontext->whereami;
+    dcontext->whereami = WHERE_SYSCALL_HANDLER;
+    IF_X64(ASSERT_TRUNCATE(sysnum, int, mc->xax));
+    DODEBUG(dcontext->expect_last_syscall_to_fail = false;);
+
+    KSTART(pre_syscall);
+    RSTATS_INC(pre_syscall);
+    DOSTATS({
+        if (ignorable_system_call(sysnum))
+            STATS_INC(pre_syscall_ignorable);
+    });
+    LOG(THREAD, LOG_SYSCALLS, 2, "system call: sysnum = "PFX", param_base = "PFX"\n",
+        sysnum, param_base);
+
+#ifdef DEBUG
+    DOLOG(2, LOG_SYSCALLS, {
+          dump_mcontext(mc, THREAD, false/*not xml*/);
+    });
+    /* we can't pass other than a numeric literal anymore */
+    LOG(THREAD, LOG_SYSCALLS, 3, "\tparam 0: "PFX"\n", sys_param(dcontext, param_base, 0));
+    LOG(THREAD, LOG_SYSCALLS, 3, "\tparam 1: "PFX"\n", sys_param(dcontext, param_base, 1));
+    LOG(THREAD, LOG_SYSCALLS, 3, "\tparam 2: "PFX"\n", sys_param(dcontext, param_base, 2));
+    LOG(THREAD, LOG_SYSCALLS, 3, "\tparam 3: "PFX"\n", sys_param(dcontext, param_base, 3));
+    LOG(THREAD, LOG_SYSCALLS, 3, "\tparam 4: "PFX"\n", sys_param(dcontext, param_base, 4));
+    LOG(THREAD, LOG_SYSCALLS, 3, "\tparam 5: "PFX"\n", sys_param(dcontext, param_base, 5));
+    LOG(THREAD, LOG_SYSCALLS, 3, "\tparam 6: "PFX"\n", sys_param(dcontext, param_base, 6));
+    LOG(THREAD, LOG_SYSCALLS, 3, "\tparam 7: "PFX"\n", sys_param(dcontext, param_base, 7));
+    LOG(THREAD, LOG_SYSCALLS, 3, "\tparam 8: "PFX"\n", sys_param(dcontext, param_base, 8));
+    DOLOG(3, LOG_SYSCALLS, {
+        /* ebp isn't in mcontext right now, so pass ebp */
+        dump_callstack(POST_SYSCALL_PC(dcontext), (app_pc) mc->xbp, THREAD,
+                       DUMP_NOT_XML);
+    });
+#endif
+
+    /* save key register values for post_system_call (they get clobbered
+     * in syscall itself)
+     * FIXME: our new stateless asynch handling means that these values
+     * are wrong when we finally return to an interrupted syscall, so post-processing
+     * looks at the wrong system call!
+     * Fortunately it always looks at NtContinue, and we haven't yet implemented
+     * NtContinue failure
+     * We need fields analogous to asynch_target: asynch_sys_num and
+     * asynch_param_base.  Unlike callbacks only one outstanding return-to point
+     * can exist.  Let's do this when we go and make our syscall failure handling
+     * more robust.  (This is case 1501)
+     */
+    dcontext->sys_num = sysnum;
+    dcontext->sys_param_base = param_base;
+#ifdef X64
+    /* save params that are in registers */
+    dcontext->sys_param0 = sys_param(dcontext, param_base, 0);
+    dcontext->sys_param1 = sys_param(dcontext, param_base, 1);
+    dcontext->sys_param2 = sys_param(dcontext, param_base, 2);
+    dcontext->sys_param3 = sys_param(dcontext, param_base, 3);
+#endif
+
+    if (sysnum == syscalls[SYS_Continue]) {
+        CONTEXT *cxt = (CONTEXT *) sys_param(dcontext, param_base, 0);
+        /* FIXME : we are going to read and write to cxt, which may be unsafe */
+        int flag = (int) sys_param(dcontext, param_base, 1);
+        LOG(THREAD, LOG_SYSCALLS|LOG_ASYNCH, IF_DGCDIAG_ELSE(1, 2),
+            "syscall: NtContinue cxt->Xip="PFX" flag="PFX"\n",
+            cxt->CXT_XIP, flag);
+        intercept_nt_continue(cxt, flag);
+    }
+    else if (sysnum == syscalls[SYS_CallbackReturn]) {
+        presys_CallbackReturn(dcontext, param_base);
+    }
+    else if (sysnum == syscalls[SYS_SetContextThread]) {
+        execute_syscall = presys_SetContextThread(dcontext, param_base);
+    }
+    else if (sysnum == syscalls[SYS_CreateProcess]) {
+        presys_CreateProcess(dcontext, param_base, false/*!Ex*/);
+    }
+    else if (sysnum == syscalls[SYS_CreateProcessEx]) {
+        presys_CreateProcess(dcontext, param_base, true/*Ex*/);
+    }
+#ifdef DEBUG
+    else if (sysnum == syscalls[SYS_CreateUserProcess]) {
+        presys_CreateUserProcess(dcontext, param_base);
+    }
+#endif
+    else if (sysnum == syscalls[SYS_CreateThread]) {
+        presys_CreateThread(dcontext, param_base);
+    }
+#ifdef DEBUG
+    else if (sysnum == syscalls[SYS_CreateThreadEx]) {
+        presys_CreateThreadEx(dcontext, param_base);
+    }
+#endif
+    else if (sysnum == syscalls[SYS_SuspendThread]) {
+        HANDLE thread_handle= (HANDLE) sys_param(dcontext, param_base, 0);
+        thread_id_t tid = thread_id_from_handle(thread_handle);
+        LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, IF_DGCDIAG_ELSE(1, 2),
+            "syscall: NtSuspendThread tid=%d\n", tid);
+        if (SELF_PROTECT_ON_CXT_SWITCH) {
+            /* This thread must make it back out of the cache for post-syscall
+             * processing, regardless of what locks target thread holds at
+             * suspension point, so we have to turn off our cxt switch hooks
+             * (see case 4942)
+             */
+            dcontext->ignore_enterexit = true;
+        }
+    }
+#ifdef DEBUG
+    else if (sysnum == syscalls[SYS_ResumeThread]) {
+        HANDLE thread_handle= (HANDLE) sys_param(dcontext, param_base, 0);
+        thread_id_t tid = thread_id_from_handle(thread_handle);
+        LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, IF_DGCDIAG_ELSE(1, 2),
+            "syscall: NtResumeThread tid=%d\n", tid);
+    }
+    else if (sysnum == syscalls[SYS_AlertResumeThread]) {
+        HANDLE thread_handle= (HANDLE) sys_param(dcontext, param_base, 0);
+        thread_id_t tid = thread_id_from_handle(thread_handle);
+        LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, IF_DGCDIAG_ELSE(1, 2),
+            "syscall: NtAlertResumeThread tid=%d\n", tid);
+    }
+#endif
+    else if (sysnum == syscalls[SYS_TerminateProcess]) {
+        execute_syscall = presys_TerminateProcess(dcontext, param_base);
+    }
+    else if (sysnum == syscalls[SYS_TerminateThread]) {
+        presys_TerminateThread(dcontext, param_base);
+    }
+#ifdef PROGRAM_SHEPHERDING
+    else if (sysnum == syscalls[SYS_AllocateVirtualMemory]) {
+        presys_AllocateVirtualMemory(dcontext, param_base);
+    }
+#endif
+    else if (sysnum == syscalls[SYS_FreeVirtualMemory]) {
+        KSTART(pre_syscall_free);
+        presys_FreeVirtualMemory(dcontext, param_base);
+        KSTOP(pre_syscall_free);
+    }
+    else if (sysnum == syscalls[SYS_ProtectVirtualMemory]) {
+        KSTART(pre_syscall_protect);
+        execute_syscall = presys_ProtectVirtualMemory(dcontext, param_base);
+        KSTOP(pre_syscall_protect);
+    }
+    else if (sysnum == syscalls[SYS_WriteVirtualMemory]) {
+        /* FIXME NYI: case 8321: need to watch for cache consistency 
+         * FIXME case 9103: note that we don't hook this for native_exec yet
+         */
+    }
+    else if (sysnum == syscalls[SYS_MapViewOfSection]) {
+        presys_MapViewOfSection(dcontext, param_base);
+    }
+    else if (sysnum == syscalls[SYS_UnmapViewOfSection]) {
+        KSTART(pre_syscall_unmap);
+        presys_UnmapViewOfSection(dcontext, param_base);
+        KSTOP(pre_syscall_unmap);
+    }
+    else if (sysnum == syscalls[SYS_FlushInstructionCache]) {
+        presys_FlushInstructionCache(dcontext, param_base);
+    }
+    else if (sysnum == syscalls[SYS_CreateSection]) {
+        presys_CreateSection(dcontext, param_base);
+    }
+#ifdef DEBUG
+    /* FIXME: move this stuff to an strace-like client, not needed
+     * for core DynamoRIO (at least not that we know of)
+     */
+    else if (sysnum == syscalls[SYS_OpenFile]) {
+        presys_OpenFile(dcontext, param_base);
+    }
+#endif
+    /* Address Windowing Extensions (win2k only):
+     * swap pieces of memory in and out of virtual address space 
+     * => we must intercept when virtual addresses could point to something new
+     */
+    else if (sysnum == syscalls[SYS_FreeUserPhysicalPages]) {
+        HANDLE process_handle = (HANDLE) sys_param(dcontext, param_base, 0);
+        uint *num_pages = (uint *) sys_param(dcontext, param_base, 1);
+        uint *page_frame_nums = (uint *) sys_param(dcontext, param_base, 2);
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, IF_DGCDIAG_ELSE(1, 2),
+            "syscall: NtFreeUserPhysicalPages %d pages\n", num_pages);
+        /* FIXME: need to know base if currently mapped, must
+         * record every mapping to do so
+         */
+        SYSLOG_INTERNAL_WARNING_ONCE(PRODUCT_NAME" is using un-supported "
+                                     "Address Windowing Extensions");
+    }
+    else if (sysnum == syscalls[SYS_MapUserPhysicalPages]) {
+        app_pc base = (app_pc) sys_param(dcontext, param_base, 0);
+        uint *pnum_pages = (uint *) sys_param(dcontext, param_base, 1);
+        uint *page_frame_nums = (uint *) sys_param(dcontext, param_base, 2);
+        uint num_pages;
+        if (safe_read(pnum_pages, sizeof(num_pages), &num_pages)) {
+            LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, IF_DGCDIAG_ELSE(1, 2),
+                "syscall: NtMapUserPhysicalPages "PFX" pages=%d\n",
+                base, num_pages);
+            base = proc_get_containing_page(base);
+            app_memory_deallocation(dcontext, base, num_pages*PAGE_SIZE,
+                                    false /* don't own thread_initexit_lock */,
+                                    false /* not image */);
+        } else {
+            DODEBUG(dcontext->expect_last_syscall_to_fail = true;);
+            goto exit_pre_system_call;
+        }
+    }
+
+ exit_pre_system_call:
+    dcontext->whereami = old_whereami;
+    KSTOP(pre_syscall);
+    return execute_syscall;
+}
+
+/***************************************************************************
+ * POST SYSTEM CALL
+ */
+
+/* NtCreateUserProcess */
+static void
+postsys_CreateUserProcess(dcontext_t *dcontext, reg_t *param_base, bool success)
+{
+    /* See notes in presys_CreateUserProcess for information on signature
+     * of NtCreateUserProcess. */
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    HANDLE *process_handle = (HANDLE *) postsys_param(dcontext, param_base, 0);
+    bool create_suspended = (bool) postsys_param(dcontext, param_base, 7);
+    /* FIXME should have type for this */
+    DEBUG_DECLARE(create_proc_thread_info_t *thread_stuff =
+                  (create_proc_thread_info_t *) postsys_param(dcontext, param_base, 10);)
+    ASSERT(get_os_version() == WINDOWS_VERSION_VISTA);
+
+    LOG(THREAD, LOG_SYSCALLS, 1, "syscall: NtCreateUserProcess => "PIFX"\n", mc->xax);
+    DOLOG(1, LOG_SYSCALLS, {
+        if (success) {
+            PCLIENT_ID client_id;
+            ASSERT(thread_stuff != NULL && thread_stuff->client_id.buffer != NULL);
+            /* Potentially dangerous deref of app ptr,
+             * but is only for debug logging */
+            client_id = (PCLIENT_ID)thread_stuff->client_id.buffer;
+            LOG(THREAD, LOG_SYSCALLS, 1,
+                "syscall: NtCreateUserProcess created process "PIFX" "
+                "with main thread "PIFX"\n",
+                client_id->UniqueProcess, client_id->UniqueThread);
+        }
+    });
+
+    if (success) {
+        /* FIXME - deref of app ptr process_handle, but success so should be ok. */
+        HANDLE proc_handle = *process_handle;
+        ACCESS_MASK rights = nt_get_handle_access_rights(proc_handle);
+        
+        if (TESTALL(PROCESS_VM_OPERATION|PROCESS_VM_READ|
+                    PROCESS_VM_WRITE|PROCESS_QUERY_INFORMATION, rights)) {
+            if (create_suspended) {
+                maybe_inject_into_process(dcontext, proc_handle, NULL);
+            } else {
+                LOG(THREAD, LOG_SYSCALLS, 1,
+                    "syscall: NtCreateUserProcess first thread not suspended "
+                    "can't safely follow children.\n");
+                ASSERT_NOT_IMPLEMENTED(create_suspended);
+                /* FIXME - NYI - should change in pre and resume the thread
+                 * after we inject. */
+            }
+        } else {
+            LOG(THREAD, LOG_SYSCALLS, 1,
+                "syscall: NtCreateUserProcess unable to get sufficient rights"
+                " to follow children\n");
+            /* This happens for Vista protected processes (drm). xref 8485 */
+            /* FIXME - could check against executable file name from
+             * thread_stuff to see if this was a process we're configured to
+             * protect. */
+            SYSLOG_INTERNAL_WARNING("Insufficient permissions to examine "
+                                    "child process\n");
+        }
+        /* Case 9173: guard against pid reuse */
+        dcontext->aslr_context.last_child_padded = 0;
+    }
+}
+
+/* NtGetContextThread */
+static void
+postsys_GetContextThread(dcontext_t *dcontext, reg_t *param_base, bool success)
+{
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    HANDLE thread_handle = (HANDLE) postsys_param(dcontext, param_base, 0);
+    CONTEXT *cxt = (CONTEXT *) postsys_param(dcontext, param_base, 1);
+    thread_record_t *trec;
+    thread_id_t tid = thread_id_from_handle(thread_handle);
+    CONTEXT alt_cxt;
+    CONTEXT *xlate_cxt;
+    LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, 1,
+        "syscall: NtGetContextThread handle="PFX" (tid=%d) flags=0x%x"
+        " cxt->Xip="PFX" => "PIFX"\n",
+        thread_handle, tid, cxt->ContextFlags, cxt->CXT_XIP, mc->xax);
+    if (!success)
+        return;
+
+    /* FIXME : we are going to read/write the context argument which is
+     * potentially unsafe, since success it must have been readable when
+     * at the os call, but there could always be multi-thread races */
+        
+    /* so trec remains valid, we are !could_be_linking */
+    mutex_lock(&thread_initexit_lock);
+    trec = thread_lookup(tid);
+    if (trec == NULL) {
+        /* this can occur if the target thread hasn't been scheduled yet
+         * and therefore we haven't initalized it yet, (scheduled for 
+         * fixing), OR if the thread is in another process (FIXME : IPC)
+         * for either case we do nothing for now
+         */
+        DODEBUG({
+            process_id_t pid = process_id_from_thread_handle(thread_handle);
+            if (!is_pid_me(pid)) {
+                IPC_ALERT("Warning: NtGetContextThread called on thread "
+                          "tid="PFX" in different process, pid="PFX,
+                          tid, pid);
+            } else {
+                SYSLOG_INTERNAL_WARNING_ONCE("Warning: NtGetContextThread "
+                                             "called on unknown thread "
+                                             PFX, tid);
+            }
+        });
+        LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, 2, 
+            "NtGetContextThread on unknown thread %d\n", tid);
+    } else {
+        /* FIXME : the following routine (and the routines it calls
+         * namely recreate_app_state) require that trec thread be
+         * suspended at a consistent spot, but we could have that the
+         * trec thread is not suspended (get_thread_context doesn't
+         * require it!), should we check the suspend count?
+         */
+
+        xlate_cxt = cxt;
+        if (!TESTALL(CONTEXT_DR_STATE, cxt->ContextFlags)) {
+            STATS_INC(num_app_getcontext_no_control);
+            /* we need esp and eip, plus all regs + xmm, to translate the machine state.
+             * no further permissions are needed to acquire them so we
+             * get our own context w/ them.
+             */
+            alt_cxt.ContextFlags = CONTEXT_DR_STATE;
+            if (!thread_get_context(trec, &alt_cxt)) {
+                ASSERT_NOT_REACHED();
+                /* FIXME: just don't translate -- right now won't hurt us since
+                 * we don't translate other than the pc anyway.
+                 */
+                return;
+            } else
+                xlate_cxt = &alt_cxt;
+        }
+
+        SELF_PROTECT_LOCAL(trec->dcontext, WRITABLE);
+        /* PR 214962: since we are not relocating the target thread, we do NOT
+         * want to restore memory.  This is no less transparent, because
+         * this thread could read the target thread's memory at any time anyway.
+         */
+        if (!translate_context(trec, xlate_cxt, false/*leave memory alone*/)) {
+            /* FIXME: can get here native if GetThreadContext on
+             * an un-suspended thread, but then API says result is 
+             * undefined so just pass anything reasonable 
+             * PLUS, need to handle unknown (unscheduled yet) thread -- 
+             * passing native should be fine
+             */
+            SYSLOG_INTERNAL_WARNING("NtGetContextThread called for thread not in translatable spot");
+            LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, 1,
+                "ERROR: NtGetContextThread called for thread not in translatable spot\n");
+        } else if (xlate_cxt != cxt) {
+            /* copy the fields we may have changed that app requested */
+            ASSERT(!TESTALL(CONTEXT_DR_STATE, cxt->ContextFlags));
+            if (TESTALL(CONTEXT_CONTROL/*2 bits so ALL*/, cxt->ContextFlags)) {
+                cxt->CXT_XIP = xlate_cxt->CXT_XIP;
+                cxt->CXT_XFLAGS = xlate_cxt->CXT_XFLAGS;
+                cxt->CXT_XSP = xlate_cxt->CXT_XSP;
+#ifndef X64
+                cxt->CXT_XBP = xlate_cxt->CXT_XBP;
+#endif
+            }
+            if (TESTALL(CONTEXT_INTEGER/*2 bits so ALL*/, cxt->ContextFlags)) {
+                cxt->CXT_XAX = xlate_cxt->CXT_XAX;
+                cxt->CXT_XBX = xlate_cxt->CXT_XBX;
+                cxt->CXT_XCX = xlate_cxt->CXT_XCX;
+                cxt->CXT_XDX = xlate_cxt->CXT_XDX;
+                cxt->CXT_XSI = xlate_cxt->CXT_XSI;
+                cxt->CXT_XDI = xlate_cxt->CXT_XDI;
+#ifdef X64
+                cxt->CXT_XBP = xlate_cxt->CXT_XBP;
+                cxt->R8  = xlate_cxt->R8;
+                cxt->R9  = xlate_cxt->R9;
+                cxt->R10 = xlate_cxt->R10;
+                cxt->R11 = xlate_cxt->R11;
+                cxt->R12 = xlate_cxt->R12;
+                cxt->R13 = xlate_cxt->R13;
+                cxt->R14 = xlate_cxt->R14;
+                cxt->R15 = xlate_cxt->R15;
+#endif
+            }
+            if (TESTALL(CONTEXT_XMM_FLAG, cxt->ContextFlags) && preserve_xmm_caller_saved()) {
+                /* PR 264138 */
+                memcpy(CXT_XMM(cxt, 0), CXT_XMM(xlate_cxt, 0), XMM_SAVED_SIZE);
+            }
+        }
+        SELF_PROTECT_LOCAL(trec->dcontext, READONLY);
+    }
+    mutex_unlock(&thread_initexit_lock);
+}
+
+/* NtSuspendThread */
+static void
+postsys_SuspendThread(dcontext_t *dcontext, reg_t *param_base, bool success)
+{
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    HANDLE thread_handle= (HANDLE) postsys_param(dcontext, param_base, 0);
+    /* ignoring 2nd argument (OUT PULONG PreviousSuspendCount OPTIONAL) */
+    thread_id_t tid = thread_id_from_handle(thread_handle);
+    process_id_t pid;
+
+    LOG(THREAD, LOG_SYSCALLS|LOG_THREADS, 1,
+        "syscall: NtSuspendThread tid=%d => "PIFX"\n", tid, mc->xax);
+    if (SELF_PROTECT_ON_CXT_SWITCH) {
+        /* No matter what, restore ignore to default value */
+        dcontext->ignore_enterexit = false;
+    }
+    /* if we suspended ourselves then skip synchronization, 
+     * already resumed, FIXME : what if someone else resumes the thread
+     * while we are trying to synch with it */
+    if (!success || tid == get_thread_id())
+        return;
+        
+    pid = process_id_from_thread_handle(thread_handle);
+    if (!is_pid_me(pid)) {
+        /* (FIXME : IPC) */
+        IPC_ALERT("Warning: SuspendThread called on thread in "
+                  "different process, pid="PFX, pid);
+        return;
+    }
+
+    /* as optimization check if at good spot already before resuming for
+     * synch, use trylocks in case suspended thread is holding any locks */
+    if (mutex_trylock(&thread_initexit_lock)) {
+        if (!mutex_testlock(&all_threads_lock)) {
+            CONTEXT cxt;
+            thread_record_t *tr;
+            cxt.ContextFlags = CONTEXT_DR_STATE;
+            /* know thread isn't holding any of the locks we will need */
+            LOG(THREAD, LOG_SYNCH, 2, 
+                "SuspendThread got necessary locks to test if thread %d suspended at good spot without resuming\n", 
+                tid); 
+            tr = thread_lookup(tid);
+            if (tr == NULL) {
+                /* Could be unknown thread, a thread just starting up or
+                 * a thread that is in the process of exiting. 
+                 * synch_with_thread will take care of the last case at
+                 * least so we fall through to that. */
+            } else if (thread_get_context(tr, &cxt)) {
+                dr_mcontext_t mc;
+                context_to_mcontext(&mc, &cxt);
+                SELF_PROTECT_LOCAL(tr->dcontext, WRITABLE);
+                if (at_safe_spot(tr, &mc, 
+                                 THREAD_SYNCH_SUSPENDED_VALID_MCONTEXT)) {
+                    /* suspended at good spot, skip synch */
+                    mutex_unlock(&thread_initexit_lock);
+                    LOG(THREAD, LOG_SYNCH, 2, 
+                        "SuspendThread suspended thread %d at good place\n", 
+                        tid);
+                    SELF_PROTECT_LOCAL(tr->dcontext, READONLY);
+                    return;
+                }
+                SELF_PROTECT_LOCAL(tr->dcontext, READONLY);
+            }
+        } else {
+            LOG(THREAD, LOG_SYNCH, 2, 
+                "SuspendThread couldn't get all_threads_lock to test if thread %d at good spot without resuming\n", 
+                tid); 
+        }
+        mutex_unlock(&thread_initexit_lock);
+    } else {
+        LOG(THREAD, LOG_SYNCH, 2, 
+            "SuspendThread couldn't get thread_initexit_lock to test if thread %d at good spot without resuming\n", 
+            tid); 
+    }
+    LOG(THREAD, LOG_SYNCH, 2, 
+        "SuspendThread resuming suspended thread %d for synch routine\n",
+        tid);
+        
+    /* resume for synch */
+    nt_thread_resume(thread_handle, NULL);
+
+    /* do synch */
+    {
+        dr_mcontext_t mcontext;
+        thread_synch_result_t synch_res;
+        copy_mcontext(mc, &mcontext);
+        mc->pc = POST_SYSCALL_PC(dcontext);
+        
+        /* we hold the initexit lock for case 9489, see comment below in failure
+         * to synch path for details why */
+        if (DYNAMO_OPTION(suspend_on_synch_failure_for_app_suspend))
+            mutex_lock(&thread_initexit_lock);
+        synch_res =
+            synch_with_thread(tid, true /* block */,
+                              /* initexit lock status */
+                              DYNAMO_OPTION(suspend_on_synch_failure_for_app_suspend),
+                              THREAD_SYNCH_VALID_MCONTEXT, 
+                              THREAD_SYNCH_SUSPENDED_VALID_MCONTEXT,
+                              /* if we fail to suspend a thread (e.g., privilege
+                               * problems) ignore it. FIXME: retry instead? */
+                              THREAD_SYNCH_SUSPEND_FAILURE_IGNORE);
+        if (synch_res != THREAD_SYNCH_RESULT_SUCCESS) {
+            /* xref case 9488 - we failed to synch, could be we exceeded our loop count
+             * for some reason, we lack GetContext permission (or the apps handle has
+             * suspend and ours doesn't somehow), or could be an unknown thread. FIXME - 
+             * we suspend the thread so the app doesn't get screwed up (it expects a
+             * suspended thread) at the risk of possibly deadlocking DR if it holds
+             * one of our locks etc. */
+            /* If the thread is unknown everything might be ok, could be a thread that's
+             * almost exited (should be fine though app might get slightly screwy result
+             * if it calls get context, e.g. an eip in our dll) or a new thread that
+             * hasn't yet initialized (see case 9489, should also be fine since we hold
+             * the initexit lock so the thread can't have gone anywhere since the
+             * synch_with_thread checks). NOTE - SetEvent appears to do the sensible
+             * thing when an auto-reset event that has a suspended thread waiting on it
+             * is signaled (the new thread could be waiting on the initexit lock), i.e.
+             * leave the event signaled for someone else to grab. */
+            /* Full ASSERT if thread is known (always bad to fail then), curiosity 
+             * instead if thread is unknown (since expected to be ok). */
+            ASSERT(thread_lookup(tid) == NULL); /* i.e. thread not known */
+            /* The suspend.c unit test can hit this regularly on (via suspend new thread)
+             * though we expect it to be unusual in normal applications. Same thing with
+             * detach_test.exe and threadinjection.exe. */
+            ASSERT_CURIOSITY_ONCE((thread_lookup(tid) != NULL || /* thread known */
+                                   EXEMPT_TEST("suspend.exe;detach_test.exe;"
+                                               "threadinjection.exe")) &&
+                                  "app suspending unknown thread");
+            if (DYNAMO_OPTION(suspend_on_synch_failure_for_app_suspend)) {
+                /* thread may already be exited in which case this will fail */
+                DEBUG_DECLARE(bool res = )nt_thread_suspend(thread_handle, NULL);
+                ASSERT(res || is_thread_exited(thread_handle) == THREAD_EXITED);
+            }
+        }
+        if (DYNAMO_OPTION(suspend_on_synch_failure_for_app_suspend))
+            mutex_unlock(&thread_initexit_lock);
+        
+        /* FIXME - if the thread exited we should prob. change the return value to
+         * the app to a failure value. Only an assert_curiosity for now to see if any
+         * apps suspend threads while the threads are exiting and if so what they expect
+         * to happen. */
+        ASSERT_CURIOSITY(is_thread_exited(thread_handle) == THREAD_NOT_EXITED);
+
+        copy_mcontext(&mcontext, mc);
+    }
+}
+
+/* NtAllocateVirtualMemory */
+static void
+postsys_AllocateVirtualMemory(dcontext_t *dcontext, reg_t *param_base, bool success)
+{
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    HANDLE process_handle = (HANDLE) postsys_param(dcontext, param_base, 0);
+    void **pbase = (void **) postsys_param(dcontext, param_base, 1);
+    uint zerobits = (uint) postsys_param(dcontext, param_base, 2);
+    size_t *psize = (size_t *) postsys_param(dcontext, param_base, 3);
+    uint type = (uint) postsys_param(dcontext, param_base, 4);
+    uint prot = (uint) postsys_param(dcontext, param_base, 5);
+    app_pc base;
+    size_t size;
+    if (!success)
+        return;
+    /* we assume that since syscall succeeded these dereferences are safe 
+     * FIXME : could be multi-thread races though */
+    base = *((app_pc *)pbase);
+    size = *psize;
+    LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, prot_is_executable(prot) ? 1U : 2U,
+        "syscall: NtAllocateVirtualMemory%s%s%s@"PFX" sz="PIFX" prot=%s 0x%x => 0x%x\n",
+        is_phandle_me(process_handle) ? "" : " IPC",
+        TEST(MEM_RESERVE, type) ? " reserve" : "",
+        TEST(MEM_COMMIT, type) ?  " commit  " : " ",
+        base, size, prot_string(prot), prot, mc->xax);
+    DOLOG(1, LOG_MEMSTATS, {
+        /* snapshots are heavyweight, so do rarely */
+        if (size > SNAPSHOT_THRESHOLD && is_phandle_me(process_handle))
+            mem_stats_snapshot();
+    });
+
+    if (TEST(ASLR_HEAP_FILL, DYNAMO_OPTION(aslr)) && 
+        is_phandle_me(process_handle)) {
+        /* We allocate our padding after the application region is
+         * succesfully reserved.  FIXME: assuming that one cannot
+         * pass MEM_RESERVE|MEM_COMMIT on an already reserved
+         * region.  Yet note one can MEM_COMMIT a region that has
+         * been committed already.  Note that it is OK to pass
+         * MEM_COMMIT with original base set to NULL, and then the
+         * allocation will act as MEM_RESERVE|MEM_COMMIT!  One
+         * can't pass MEM_COMMIT that with non-zero base on a
+         * region that hasn't been reserved before.  We want to
+         * make sure we pad only an amount corresponding to the
+         * new reservations.  (Currently we only pad immediately
+         * after an allocation but that may change.)
+         */
+
+        /* FIXME: case 6287 we should TEST(MEM_RESERVE, type) if
+         * allocation has just been reserved, or if pre_syscall
+         * base was NULL for a MEM_COMMIT.  Currently a pad is
+         * reserved only in case immediate region has not been
+         * reserved, so we're ok to attempt to pad even
+         * a MEM_COMMIT with an existing reservation
+         */
+        aslr_post_process_allocate_virtual_memory(dcontext, base, size);
+    }
+
+    if (!TEST(MEM_COMMIT, type)) {
+        /* MEM_RESERVE only: protection bits are meaningless, we do nothing
+         * MEM_RESET: we do not need to flush on a reset, since whatever is there
+         * cannot be changed without writing to it!
+         * the subsequent commit to the already committed region will work fine.
+         */
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2, "not committing, so ignorable\n");
+        return;
+    }
+    if (is_phandle_me(process_handle)) {
+#ifdef DGC_DIAGNOSTICS
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, TEST(MEM_COMMIT, type) ? 1U : 2U,
+            "syscall: NtAllocateVirtualMemory%s%s@"PFX" sz="PIFX" prot=%s 0x%x => 0x%x\n",
+            TEST(MEM_RESERVE, type) ? " reserve" : "",
+            TEST(MEM_COMMIT, type) ?  " commit  " : " ",
+            base, size, prot_string(prot), prot, mc->xax);
+        DOLOG(1, LOG_VMAREAS, {
+            dump_callstack(POST_SYSCALL_PC(dcontext),
+                           (app_pc) mc->xbp, THREAD, DUMP_NOT_XML);
+        });
+#endif
+        app_memory_allocation(dcontext, base, size, osprot_to_memprot(prot),
+                              false/*not image*/
+                              _IF_DEBUG("NtAllocateVirtualMemory"));
+#ifdef DGC_DIAGNOSTICS
+        DOLOG(3, LOG_VMAREAS, {
+            /* make all heap RO in attempt to view generation of DGC */
+            if (!is_address_on_stack(dcontext, base) && prot_is_writable(prot)) {
+                /* new thread stack: reserve big region, commit 2 pages, then mark
+                 * 1 page as PAGE_GUARD.  strangely thread gets resumed sometimes
+                 * before see PAGE_GUARD prot, so instead of tracking that we have
+                 * a hack to guess if this is a thread stack:
+                 */
+                IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+                if (size == 0x2000 && ((ptr_uint_t)base & 0xf0000000) == 0x00000000 &&
+                    prot == PAGE_READWRITE) {
+                    LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                        "Guessing "PFX"-"PFX" is thread stack\n",
+                        base, base+size);
+                } else {
+                    LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                        "Making "PFX"-"PFX" 0x%x unwritable\n",
+                        base, base+size, prot);
+                    make_unwritable(base, size);
+                }
+            }
+        });
+#endif
+    } else {
+        /* FIXME: should we try to alert any dynamo running the other process?
+         */
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+            "WARNING: NtAllocateVirtualMemory for process "PFX" %d\n",
+            process_handle, process_id_from_handle(process_handle));
+        DODEBUG({
+            if (prot_is_executable(prot)) {
+                IPC_ALERT("NtAllocateVirtualMemory for process "PFX" %d prot=%s\n",
+                          process_handle, process_id_from_handle(process_handle), 
+                          prot_string(prot));
+            }
+        });
+                              
+        /* This actually happens in calc's help defn popup!
+         * FIXME: we need IPC!  Plus need to queue up msgs to child dynamo,
+         * for calc it did NtCreateProcess, NtAllocateVirtualMemory, then
+         * the NtCreateThread that triggers our fork injection!
+         * don't die with IPC_ALERT
+         */
+    }
+}
+
+/* NtQueryVirtualMemory */
+static void
+postsys_QueryVirtualMemory(dcontext_t *dcontext, reg_t *param_base, bool success)
+{
+    /* we intercept this for transparency wrt the executable regions
+     * that we mark as read-only
+     */
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    HANDLE process_handle = (HANDLE) postsys_param(dcontext, param_base, 0);
+    app_pc base = (app_pc) postsys_param(dcontext, param_base, 1);
+    uint class = (uint) postsys_param(dcontext, param_base, 2);
+    MEMORY_BASIC_INFORMATION *mbi =
+        (MEMORY_BASIC_INFORMATION *) postsys_param(dcontext, param_base, 3);
+    size_t infolen = (size_t) postsys_param(dcontext, param_base, 4);
+    size_t *returnlen = (size_t *) postsys_param(dcontext, param_base, 5);
+    LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, IF_DGCDIAG_ELSE(1, 2),
+        "syscall: NtQueryVirtualMemory base="PFX" => "PIFX"\n", base, mc->xax);
+    if (!success)
+        return;
+    /* FIXME : since success we assume that all argument dereferences are
+     * safe though there could always be multi-thread races */
+    if (is_phandle_me(process_handle)) {
+        if (class == MemoryBasicInformation) {
+            /* see if asking about an executable area we made read-only */
+            if (is_executable_area_writable(base) ||
+                (USING_PRETEND_WRITABLE() &&
+                 is_pretend_writable_address(base))) {
+                /* pretend area is writable */
+                uint flags =
+                    mbi->Protect & ~PAGE_PROTECTION_QUALIFIERS;
+                LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+                    "WARNING: Query to now-readonly executable area, pretending writable\n");
+                if (flags == PAGE_READONLY) {
+                    mbi->Protect &= ~PAGE_READONLY;
+                    mbi->Protect |= PAGE_READWRITE;
+                } else if (flags == PAGE_EXECUTE_READ) {
+                    mbi->Protect &= ~PAGE_EXECUTE_READ;
+                    mbi->Protect |= PAGE_EXECUTE_READWRITE;
+                } else {
+                    LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                        "ERROR: Query to now-readonly executable area w/ bad flags %s\n",
+                        prot_string(mbi->Protect));
+                    SYSLOG_INTERNAL_INFO("ERROR: Query to now-readonly executable area w/ bad flags\n");
+                }
+            } else if (is_dynamo_address(base)) {
+                LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                    "WARNING: QueryVM to DR memory "PFX"\n", base);
+                if (base == dynamo_dll_start && mbi != NULL &&
+                    DYNAMO_OPTION(hide_from_query) != 0) {
+                    /* pretend area is un-allocated */
+                    LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                        "WARNING: QueryVM to DR DLL "PFX", pretending not a dll\n", base);
+                    if (TEST(HIDE_FROM_QUERY_TYPE_PROTECT,
+                             DYNAMO_OPTION(hide_from_query))) {
+                        mbi->Type = MEM_PRIVATE; /* not image! */
+                        mbi->Protect = PAGE_NOACCESS;
+                    }
+                    /* now do an off-by-1 to fool any calls to GetModuleFileName
+                     * (it doesn't turn into a syscall)
+                     * FIXME: app could still use a snapshot to get list 
+                     * of modules, but that is covered by -hide
+                     */
+                    if (TEST(HIDE_FROM_QUERY_BASE_SIZE,
+                             DYNAMO_OPTION(hide_from_query))) {
+                        mbi->AllocationBase = ((app_pc)mbi->AllocationBase)
+                            + PAGE_SIZE;
+                        mbi->BaseAddress = ((app_pc)mbi->BaseAddress) 
+                            + PAGE_SIZE;
+                        /* skip over the other regions in our dll -- ok to 
+                         * be PAGE_SIZE off, better to be beyond than 
+                         * return too small and have caller incrementing
+                         * only and ignoring bases!
+                         */
+                        mbi->RegionSize = (dynamo_dll_end - dynamo_dll_start);
+                    }
+                    /* note that returning STATUS_INVALID_ADDRESS is too
+                     * extreme of a solution, so this is off by default */
+                    if (TEST(HIDE_FROM_QUERY_RETURN_INVALID,
+                             DYNAMO_OPTION(hide_from_query))) {
+                        /* FIXME: SET_RETURN_VAL bug 5068 had return val as 0
+                         * Need to re-test this with this actual return val
+                         */
+                        SET_RETURN_VAL(dcontext, STATUS_INVALID_ADDRESS);
+                    }
+                }
+            }
+        }
+    } else {
+        IPC_ALERT("Warning: QueryVirtualMemory on another process\n");
+    }
+}
+
+static void
+postsys_create_or_open_section(dcontext_t *dcontext,
+                               HANDLE *unsafe_section_handle, HANDLE file_handle)
+{
+    if (DYNAMO_OPTION(track_module_filenames)) {
+        /* Case 1272: keep file name around to use for module identification */
+        FILE_NAME_INFORMATION name_info; /* note: large struct */
+        const wchar_t *short_name;
+        if (file_handle == INVALID_HANDLE_VALUE)
+            short_name = NULL;
+        else
+            short_name = get_file_short_name(file_handle, &name_info);
+        
+        /* We keep only last section, so we need the handle to see if it matches
+         * when we hit the mapview */
+        safe_read(unsafe_section_handle,
+                  sizeof(dcontext->aslr_context.last_app_section_handle),
+                  &dcontext->aslr_context.last_app_section_handle);
+        
+        aslr_set_last_section_file_name(dcontext, short_name);
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+            "\tNt{Create,Open}Section: sec handle "PIFX", file name %s\n",
+            dcontext->aslr_context.last_app_section_handle,
+            dcontext->aslr_context.last_section_file_name == NULL ? "<null>" :
+            dcontext->aslr_context.last_section_file_name);
+    } else {
+        DODEBUG({
+            dcontext->aslr_context.
+                last_app_section_handle = *unsafe_section_handle;
+        });
+    }
+}
+
+/* NtCreateSection */
+static void
+postsys_CreateSection(dcontext_t *dcontext, reg_t *param_base, bool success)
+{
+    /* a section is an object that can be mmapped */
+    HANDLE *unsafe_section_handle = (HANDLE*) postsys_param(dcontext, param_base, 0);
+    uint access_mask = (uint) postsys_param(dcontext, param_base, 1);
+    POBJECT_ATTRIBUTES obj = (POBJECT_ATTRIBUTES) postsys_param(dcontext, param_base, 2);
+    void *size = (void *) postsys_param(dcontext, param_base, 3);
+    uint protect = (uint) postsys_param(dcontext, param_base, 4);
+    uint attributes = (uint) postsys_param(dcontext, param_base, 5);
+    HANDLE file_handle = (HANDLE) postsys_param(dcontext, param_base, 6);
+    LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+        "syscall: NtCreateSection protect 0x%x, attributes 0x%x\n", protect, attributes);
+    if (!success)
+        return;
+
+    postsys_create_or_open_section(dcontext, unsafe_section_handle, file_handle);
+
+    if (TEST(ASLR_DLL, DYNAMO_OPTION(aslr))) {
+        if (TEST(SEC_IMAGE, attributes)) {
+            if (aslr_post_process_create_or_open_section(dcontext,
+                                                         true, /* create */
+                                                         file_handle,
+                                                         unsafe_section_handle)) {
+                LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+                    "syscall: ASLR: NtCreateSection replaced with new section "PIFX"\n",
+                    *unsafe_section_handle);
+            } else {
+                /* leaving as is */
+            }
+        } else {
+            /* ignoring SEC_COMMIT mappings - since SEC_COMMIT is
+             * default it doesn't need to be set */
+        }
+    }
+}
+
+/* NtOpenSection */
+static void
+postsys_OpenSection(dcontext_t *dcontext, reg_t *param_base, bool success)
+{
+    /* a section is an object that can be mmapped, here opened by object name */
+    HANDLE *unsafe_section_handle = (HANDLE*) postsys_param(dcontext, param_base, 0);
+    uint access_mask = (uint) postsys_param(dcontext, param_base, 1);
+    POBJECT_ATTRIBUTES obj_attr = (POBJECT_ATTRIBUTES)
+        postsys_param(dcontext, param_base, 2);
+    HANDLE new_file_handle = INVALID_HANDLE_VALUE;
+    if (!success) {
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+            "syscall: NtOpenSection, failed, access 0x%x\n", 
+            access_mask);
+        return;
+    }
+
+    LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+        "syscall: NtOpenSection opened sh "PIFX", access_mask 0x%x\n", 
+        *unsafe_section_handle, access_mask);
+
+    /* FIXME: currently we only use short names for -track_module_filenames,
+     * so we only need to use obj_attr->ObjectName->Buffer and don't need
+     * to call aslr_recreate_known_dll_file() at all, but I'm assuming
+     * we may add full path to module list in the future.
+     */
+    if ((DYNAMO_OPTION(track_module_filenames) ||
+         (TEST(ASLR_DLL, DYNAMO_OPTION(aslr)) && 
+          TEST(ASLR_SHARED_CONTENTS, DYNAMO_OPTION(aslr_cache)))) &&
+        obj_attr != NULL) {
+        /* need to identify KnownDlls here */
+        /* FIXME: NtOpenSection doesn't give us section attributes,
+         * and we can't even query them - the only reasonable solution is to
+         * match the directory handle 
+         *
+         * FIXME: case 9032 about possibly duplicating the handle if
+         * that is any faster than any other syscalls we're making here
+         */
+        /* FIXME: we could restrict the check to potential DLLs
+         * based on access_mask, although most users use
+         * SECTION_ALL_ACCESS */
+        HANDLE root_directory = NULL;
+        bool ok = safe_read(&obj_attr->RootDirectory, 
+                            sizeof(root_directory), &root_directory);
+        if (ok && root_directory != NULL && 
+            aslr_is_handle_KnownDlls(root_directory)) {
+
+            aslr_recreate_known_dll_file(obj_attr, 
+                                         &new_file_handle);
+
+            if (TEST(ASLR_DLL, DYNAMO_OPTION(aslr)) && 
+                TEST(ASLR_SHARED_CONTENTS, DYNAMO_OPTION(aslr_cache))) {
+                if (aslr_post_process_create_or_open_section(dcontext,
+                                                             false, /* open */
+                                                             /* recreated file */
+                                                             new_file_handle,
+                                                             unsafe_section_handle)) {
+                    LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+                        "syscall: ASLR: NtOpenSection replaced with new section "PIFX"\n", 
+                        *unsafe_section_handle);
+                } else {
+                    /* leaving as is */
+                }
+            }
+            close_handle(new_file_handle);
+        } else {
+            /* nothing */
+        }
+    }
+    /* Even if we didn't get the name we need to set the last section handle */
+    if (DYNAMO_OPTION(track_module_filenames)) {
+        postsys_create_or_open_section(dcontext, unsafe_section_handle,
+                                       new_file_handle);
+    }
+}
+
+/* NtMapViewOfSection */
+static void
+postsys_MapViewOfSection(dcontext_t *dcontext, reg_t *param_base, bool success)
+{
+    /* This is what actually allocates a dll into memory */
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    HANDLE section_handle;  /* 0 */
+    HANDLE process_handle;  /* 1 */
+    void **pbase_unsafe;    /* 2 */
+    uint zerobits;          /* 3 */
+    size_t commit_size;     /* 4 */
+    LARGE_INTEGER *section_offs; /* 5 */
+    size_t *view_size;      /* 6 */
+    uint inherit_disposition; /* 7 */
+    uint type;              /* 8 */
+    uint prot;              /* 9 */
+
+    size_t size;            /* safe dereferenced values */
+    app_pc base;
+
+    /* only process if we acted on this call in aslr_pre_process_mapview */
+    if (dcontext->aslr_context.sys_aslr_clobbered) {
+        aslr_post_process_mapview(dcontext);
+        /* preceding call sets mcontext.xax so re-evaluate */
+        success = NT_SUCCESS(mc->xax);
+        /* reevaluate all system call OUT arguments, since they
+         * may have changed in aslr_post_process_mapview()!
+         */
+
+        /* FIXME: registers may not necessarily match state of
+         * mangled system call, but we assume only state->mc.xax matters
+         */
+    }
+
+    if (!success) { 
+        prot = (uint) postsys_param(dcontext, param_base, 9);
+        LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+            "syscall: failed NtMapViewOfSection prot=%s => "PFX"\n",
+            prot_string(prot), mc->xax);
+
+        return;
+    }
+
+    section_handle = (HANDLE) postsys_param(dcontext, param_base, 0);
+    process_handle = (HANDLE) postsys_param(dcontext, param_base, 1);
+    pbase_unsafe = (void *) postsys_param(dcontext, param_base, 2);
+    zerobits = (uint) postsys_param(dcontext, param_base, 3);
+    commit_size = (size_t) postsys_param(dcontext, param_base, 4);
+    section_offs = (LARGE_INTEGER *) postsys_param(dcontext, param_base, 5);
+    view_size = (size_t *) postsys_param(dcontext, param_base, 6);
+    inherit_disposition = (uint) postsys_param(dcontext, param_base, 7);
+    type = (uint) postsys_param(dcontext, param_base, 8);
+    prot = (uint) postsys_param(dcontext, param_base, 9);
+
+    /* we assume that since syscall succeeded these dereferences are safe 
+     * FIXME : could always be multi-thread races though */
+    size = *view_size; /* ignore commit_size? */
+    base = *((app_pc *)pbase_unsafe);
+
+    LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 1,
+        "syscall: NtMapViewOfSection "PFX" size="PIFX" prot=%s => "PFX"\n",
+        base, size, prot_string(prot), mc->xax);
+
+    if (is_phandle_me(process_handle)) {
+        /* Check if we are looking for LdrpLoadImportModule address */
+        if (dcontext == early_inject_load_helper_dcontext) {
+            check_for_ldrpLoadImportModule(base, (uint *) mc->xbp); 
+        }
+        DOLOG(1, LOG_MEMSTATS, {
+            /* snapshots are heavyweight, so do rarely */
+            if (size > SNAPSHOT_THRESHOLD)
+                mem_stats_snapshot();
+        });
+#ifdef DGC_DIAGNOSTICS
+        DOLOG(1, LOG_VMAREAS, {
+            dump_callstack(POST_SYSCALL_PC(dcontext),
+                           (app_pc) mc->xbp, THREAD,
+                           DUMP_NOT_XML);
+        });
+#endif
+        RSTATS_INC(num_app_mmaps);
+        if (!DYNAMO_OPTION(thin_client)) {
+            if (DYNAMO_OPTION(track_module_filenames) &&
+                dcontext->aslr_context.last_app_section_handle != section_handle) {
+                aslr_free_last_section_file_name(dcontext);
+                STATS_INC(map_section_mismatch);
+                /* FIXME case 9028: use mapping rather than just last-seen to avoid */
+                SYSLOG_INTERNAL_WARNING_ONCE("app section "PIFX" vs map "PIFX" mismatch",
+                                         dcontext->aslr_context.last_app_section_handle,
+                                         section_handle);
+            } else {
+                LOG(THREAD, LOG_SYSCALLS|LOG_VMAREAS, 2,
+                    "\tNtMapViewOfSection: sec handle "PIFX" == file name %s\n",
+                    dcontext->aslr_context.last_app_section_handle,
+                    dcontext->aslr_context.last_section_file_name == NULL ? "<null>" :
+                    dcontext->aslr_context.last_section_file_name);
+            }
+            /* Else, we use the stored filename.  Local kernel handle
+             * reuse won't be a problem here even though we're not watching
+             * close() b/c we'll see the re-open, but we could have a
+             * handle closed and a duplicate one passed via IPC that we would
+             * think was the same: FIXME.
+             */
+            process_mmap(dcontext, base, size, true/*map*/);
+        }
+    } else {
+        IPC_ALERT("WARNING: MapViewOfSection on another process\n");
+    }
+}
+
+
+/* NtUnmapViewOfSection */
+static void
+postsys_UnmapViewOfSection(dcontext_t *dcontext, reg_t *param_base, bool success)
+{
+    /* This is what actually removes a dll from memory */
+    HANDLE process_handle = (HANDLE) postsys_param(dcontext, param_base, 0);
+#ifdef DEBUG
+    if (dcontext->expect_last_syscall_to_fail) {
+        ASSERT(!success);
+    } else { 
+        /* FIXME : try to recover if the syscall fails, could re-walk this
+         * region but that gets us in trouble with the stateful policies */
+        ASSERT_CURIOSITY(success || !is_phandle_me(process_handle));
+    }
+#endif
+    /* note that if expected this to fail we wouldn't have really registered,
+     * but we don't keep track in release builds
+     */
+    if (DYNAMO_OPTION(unloaded_target_exception) && 
+        is_phandle_me(process_handle)) {
+        app_pc base = (app_pc) postsys_param(dcontext, param_base, 1);
+        /* We always mark end of unmap no matter what the original
+         * section really was.  FIXME: Note we can't get the real_base
+         * of the allocation, unless we keep it in dcontext from
+         * presys_UnmapViewOfSection, but we don't really need it in
+         * release build.  We don't care about success or !success
+         * either.  Note that this means that if a MEM_MAPPED UnMap
+         * ends before an overlapping MEM_IMAGE UnMap, we will mark
+         * end too early.
+         */
+        mark_unload_end(base);
+    }
+}
+
+/* NOTE : no locks can be grabbed on the path to SuspendThread handling code */
+void post_system_call(dcontext_t *dcontext)
+{
+    /* registers have been clobbered, so grab key values that were
+     * saved in pre_system_call
+     */
+    int sysnum = dcontext->sys_num;
+    reg_t *param_base = dcontext->sys_param_base;
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    bool success = NT_SUCCESS(mc->xax);
+    where_am_i_t old_whereami = dcontext->whereami;
+    KSTART(post_syscall);
+    dcontext->whereami = WHERE_SYSCALL_HANDLER;
+
+#ifdef CLIENT_INTERFACE 
+    instrument_post_syscall(dcontext, sysnum);
+    /* re-set in case client changed.  client can't change sysnum. */
+    success = NT_SUCCESS(mc->xax);
+#endif
+
+    LOG(THREAD, LOG_SYSCALLS, 2,
+        "post syscall: sysnum="PFX", params @"PFX", result="PFX"\n",
+        sysnum, param_base, mc->xax);
+
+    if (sysnum == syscalls[SYS_GetContextThread]) {
+        postsys_GetContextThread(dcontext, param_base, success);
+    }
+    else if (sysnum == syscalls[SYS_SuspendThread]) {
+        postsys_SuspendThread(dcontext, param_base, success);
+    }
+    else if (sysnum == syscalls[SYS_SetContextThread]) {
+        HANDLE thread_handle = (HANDLE) postsys_param(dcontext, param_base, 0);
+        thread_id_t tid = thread_id_from_handle(thread_handle);
+        ASSERT(tid != 0xFFFFFFFF);
+        /* FIXME : we modified the passed in context, we should restore it
+         * to app state (same for SYS_Continue though is more difficult there)
+         */
+        mutex_lock(&thread_initexit_lock); /* need lock to lookup thread */
+        if (intercept_asynch_for_thread(tid, false/*no unknown threads*/)) {
+            /* Case 10101: we shouldn't get here since we now skip the system call,
+             * unless it should fail for permission issues */
+            ASSERT(dcontext->expect_last_syscall_to_fail);
+            /* must wake up thread so it can go to nt_continue_dynamo_start */
+            nt_thread_resume(thread_handle, NULL);
+        }
+        mutex_unlock(&thread_initexit_lock); /* need lock to lookup thread */
+    }
+    else if (sysnum == syscalls[SYS_AllocateVirtualMemory]) {
+        KSTART(post_syscall_alloc);
+        postsys_AllocateVirtualMemory(dcontext, param_base, success);
+        KSTOP(post_syscall_alloc);
+    } 
+    else if (sysnum == syscalls[SYS_QueryVirtualMemory]) {
+        postsys_QueryVirtualMemory(dcontext, param_base, success);
+    }
+    else if (sysnum == syscalls[SYS_CreateSection]) {
+        postsys_CreateSection(dcontext, param_base, success);
+    }
+    else if (sysnum == syscalls[SYS_OpenSection]) {
+        postsys_OpenSection(dcontext, param_base, success);
+    }
+    else if (sysnum == syscalls[SYS_MapViewOfSection]) {
+        KSTART(post_syscall_map);
+        postsys_MapViewOfSection(dcontext, param_base, success);
+        KSTOP(post_syscall_map);
+    }
+    else if (sysnum == syscalls[SYS_CreateProcess]) {
+        HANDLE *process_handle = (HANDLE *) postsys_param(dcontext, param_base, 0);
+        uint access_mask = (uint) postsys_param(dcontext, param_base, 1);
+        uint attributes  = (uint) postsys_param(dcontext, param_base, 2);
+        uint inherit_from= (uint) postsys_param(dcontext, param_base, 3);
+        bool   inherit     = (uint) postsys_param(dcontext, param_base, 4);
+        HANDLE section_handle = (HANDLE) postsys_param(dcontext, param_base, 5);
+        HANDLE debug_handle = (HANDLE) postsys_param(dcontext, param_base, 6);
+        HANDLE exception_handle = (HANDLE) postsys_param(dcontext, param_base, 7);
+
+        DOLOG(1, LOG_SYSCALLS, {
+            app_pc base = (app_pc) get_section_address(section_handle);
+            
+            LOG(THREAD, LOG_SYSCALLS, IF_DGCDIAG_ELSE(1, 2),
+                "syscall post: NtCreateProcess section @"PFX"\n", base);
+        });
+
+        maybe_inject_into_process(dcontext, *process_handle, NULL);
+    }
+    else if (sysnum == syscalls[SYS_CreateProcessEx]) {
+        HANDLE *process_handle = (HANDLE *) postsys_param(dcontext, param_base, 0);
+        uint access_mask = (uint) postsys_param(dcontext, param_base, 1);
+        uint attributes  = (uint) postsys_param(dcontext, param_base, 2);
+        uint inherit_from= (uint) postsys_param(dcontext, param_base, 3);
+        bool   inherit     = (uint) postsys_param(dcontext, param_base, 4);
+        HANDLE section_handle = (HANDLE) postsys_param(dcontext, param_base, 5);
+        HANDLE debug_handle = (HANDLE) postsys_param(dcontext, param_base, 6);
+        HANDLE exception_handle = (HANDLE) postsys_param(dcontext, param_base, 7);
+
+        /* according to metasploit, others type as HANDLE unknown etc. */
+        uint job_member_level = (uint) postsys_param(dcontext, param_base, 8);
+
+        DOLOG(1, LOG_SYSCALLS, {
+            app_pc base = (app_pc) get_section_address(section_handle);
+            
+            LOG(THREAD, LOG_SYSCALLS, IF_DGCDIAG_ELSE(1, 2),
+                "syscall: NtCreateProcessEx section @"PFX"\n", base);
+        });
+
+        maybe_inject_into_process(dcontext, *process_handle, NULL);
+    }
+    else if (sysnum == syscalls[SYS_CreateUserProcess]) {
+        postsys_CreateUserProcess(dcontext, param_base, success);
+    } else if (sysnum == syscalls[SYS_UnmapViewOfSection]) {
+        postsys_UnmapViewOfSection(dcontext, param_base, success);
+#ifdef DEBUG
+    /* Check to see if any system calls for which we did non-reversible
+     * processing in pre_system_call() failed. FIXME : handle failure
+     * cases as needed */
+    /* FIXME : because of our stateless apc handling we can't check 
+     * SYS_Continue for success (all syscalls interupted by an APC will
+     * look like a continue at post)
+     */
+    } else if (sysnum == syscalls[SYS_CallbackReturn]) {
+        /* should never get here, also ref case 4121, except for
+         * STATUS_CALLBACK_POP_STACK (case 10579) */
+        ASSERT_CURIOSITY((NTSTATUS)postsys_param(dcontext, param_base, 2) ==
+                         STATUS_CALLBACK_POP_STACK);
+        /* FIXME: should provide a routine to swap the dcontexts back so we can
+         * handle any future cases like case 10579 */
+    } else if (sysnum == syscalls[SYS_TerminateProcess]) { 
+        HANDLE process_handle = (HANDLE) postsys_param(dcontext, param_base, 0);
+        NTSTATUS exit_status = (NTSTATUS) postsys_param(dcontext, param_base, 1);
+        /* FIXME : no way to recover if syscall fails and handle is 0 or us */
+        /* Don't allow success && handle == us since we should never get here
+         * in that case */
+        ASSERT((process_handle == 0 && success) || 
+               !is_phandle_me(process_handle));
+    } else if (sysnum == syscalls[SYS_TerminateThread]) {
+        HANDLE thread_handle = (HANDLE) postsys_param(dcontext, param_base, 0);
+        ASSERT(thread_handle != 0); /* 0 => current thread */
+        if (thread_handle != 0) {
+            thread_id_t tid = thread_id_from_handle(thread_handle);
+            process_id_t pid = process_id_from_thread_handle(thread_handle);
+            ASSERT(tid != get_thread_id()); /* not current thread */
+            /* FIXME : if is thread in this process and syscall fails then
+             * no way to recover since we already cleaned up the thread */
+            /* Don't allow success && handle == us since we should never get 
+             * here in that case */
+            ASSERT(success || 
+                   tid == 0xFFFFFFFF /* prob. bad/incorrect type handle */ ||
+                   is_thread_exited(thread_handle) == THREAD_EXITED ||
+                   !is_pid_me(pid));
+            if (success && !is_pid_me(pid)) {
+                IPC_ALERT("Warning: NtTerminateThread on thread tid="PFX" "
+                          "in other process pid="PFX, tid, pid);
+            }
+        }
+    } else if (sysnum == syscalls[SYS_CreateThread]) {
+        HANDLE process_handle= (HANDLE) postsys_param(dcontext, param_base, 3);
+        CONTEXT *cxt         = (CONTEXT *) postsys_param(dcontext, param_base, 5);
+        /* FIXME : we are going to read cxt, this is potentially unsafe */
+        if (is_first_thread_in_new_process(process_handle, cxt)) {
+            /* we might have tried to inject into the process with this
+             * new thread, assert curiosity to see if this ever fails */
+            ASSERT_CURIOSITY(success);
+        }
+    } else if (sysnum == syscalls[SYS_FreeVirtualMemory]) {
+        HANDLE process_handle = (HANDLE) postsys_param(dcontext, param_base, 0);
+        void **pbase = (void **) postsys_param(dcontext, param_base, 1);
+        size_t *psize = (size_t *) postsys_param(dcontext, param_base, 2);
+        uint type = (uint) postsys_param(dcontext, param_base, 3);
+        if (dcontext->expect_last_syscall_to_fail) {
+            ASSERT(!success);
+        } else { 
+            /* FIXME : try to recover if the syscall fails, could re-walk this
+             * region but that gets us in trouble with the stateful policies */
+            ASSERT_CURIOSITY_ONCE(success || !is_phandle_me(process_handle));;
+        }
+    } else if (sysnum == syscalls[SYS_ProtectVirtualMemory]) {
+        HANDLE process_handle = (HANDLE) postsys_param(dcontext, param_base, 0);
+        if (dcontext->expect_last_syscall_to_fail) {
+            ASSERT(!success);
+        } else { 
+            /* FIXME : try to recover if the syscall fails, could re-walk this
+             * region but that gets us in trouble with the stateful policies */
+            ASSERT_CURIOSITY(success || !is_phandle_me(process_handle));
+        }
+    } else if (sysnum == syscalls[SYS_FlushInstructionCache]) {
+        HANDLE process_handle = (HANDLE) postsys_param(dcontext, param_base, 0);
+        /* Even if this system call fails, doesn't affect our correctness, but
+         * lets see if this ever fails, slight false negative risk if it does */
+        ASSERT_CURIOSITY(success || !is_phandle_me(process_handle));
+    } else if (sysnum == syscalls[SYS_MapUserPhysicalPages]) {
+        HANDLE process_handle = (HANDLE) postsys_param(dcontext, param_base, 0);
+        /* Even if this system call fails, doesn't affect our correctness, but
+         * lets see if this ever fails, slight false negative risk if it does */
+        if (dcontext->expect_last_syscall_to_fail) {
+            ASSERT(!success);
+        } else { 
+            ASSERT_CURIOSITY(success || !is_phandle_me(process_handle));
+        }
+#endif /* DEBUG */
+    }
+
+    /* stats lock grabbing ok here, any synch with suspended threads taken
+     * care of already */
+    RSTATS_INC(post_syscall);
+    DOSTATS({
+        if (ignorable_system_call(sysnum))
+            STATS_INC(post_syscall_ignorable);
+    });
+    dcontext->whereami = old_whereami;
+    KSTOP(post_syscall);
+}
+
+/***************************************************************************
+ * SYSTEM CALL API
+ */
+
+#ifdef CLIENT_INTERFACE
+DR_API
+reg_t
+dr_syscall_get_param(void *drcontext, int param_num)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    /* if we supported this from post-syscall we would need to
+     * get dcontext->sys_param_base() and call postsys_param() -- but
+     * then it would be confusing vs client checking its set param
+     */
+    reg_t *param_base = pre_system_call_param_base(mc);
+    CLIENT_ASSERT(dcontext->client_data->in_pre_syscall,
+                  "dr_syscall_get_param() can only be called from pre-syscall event");
+    return sys_param(dcontext, param_base, param_num);
+}
+
+DR_API
+void
+dr_syscall_set_param(void *drcontext, int param_num, reg_t new_value)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    reg_t *param_base;
+    CLIENT_ASSERT(dcontext->client_data->in_pre_syscall ||
+                  dcontext->client_data->in_post_syscall,
+                  "dr_syscall_set_param() can only be called from a syscall event");
+    if (dcontext->client_data->in_pre_syscall)
+        param_base = pre_system_call_param_base(mc);
+    else
+        param_base = dcontext->sys_param_base;
+    *sys_param_addr(dcontext, param_base, param_num) = new_value;
+}
+
+DR_API
+reg_t
+dr_syscall_get_result(void *drcontext)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(dcontext->client_data->in_post_syscall,
+                  "dr_syscall_get_result() can only be called from post-syscall event");
+    return get_mcontext(dcontext)->xax;
+}
+
+DR_API
+void
+dr_syscall_set_result(void *drcontext, reg_t value)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(dcontext->client_data->in_pre_syscall ||
+                  dcontext->client_data->in_post_syscall,
+                  "dr_syscall_set_result() can only be called from a syscall event");
+    SET_RETURN_VAL(dcontext, value);
+}
+
+DR_API
+void
+dr_syscall_set_sysnum(void *drcontext, int new_num)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    CLIENT_ASSERT(dcontext->client_data->in_pre_syscall ||
+                  dcontext->client_data->in_post_syscall,
+                  "dr_syscall_set_sysnum() can only be called from a syscall event");
+    mc->xax = new_num;
+}
+
+DR_API
+void
+dr_syscall_invoke_another(void *drcontext)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    dr_mcontext_t *mc = get_mcontext(dcontext);
+    CLIENT_ASSERT(dcontext->client_data->in_post_syscall,
+                  "dr_syscall_invoke_another() can only be called from post-syscall event");
+    LOG(THREAD, LOG_SYSCALLS, 2, "invoking additional syscall on client request\n");
+    dcontext->client_data->invoke_another_syscall = true;
+    if (get_syscall_method() == SYSCALL_METHOD_SYSENTER) {
+        /* Since we're not regaining control immediately after sysenter,
+         * need to push regain-control retaddr on stack, and then copy esp to edx.
+         */
+        mc->xsp -= XSP_SZ;
+        /* put the post-call-to-vsyscall address, currently in asynch_target, back
+         * on stack, and set asynch_target back to post-sysenter pc (will be put
+         * into next_tag back in handle_post_system_call())
+         */
+        *((app_pc *)mc->xsp) = dcontext->asynch_target;
+        dcontext->asynch_target = vsyscall_syscall_end_pc;
+        mc->xdx = mc->xsp;
+    }
+    else if (get_syscall_method() == SYSCALL_METHOD_WOW64) {
+        /* perform: lea edx,[esp+0x4] */
+        mc->xdx = mc->xsp + XSP_SZ;
+    }
+# ifdef X64
+    else if (get_syscall_method() == SYSCALL_METHOD_SYSCALL) {
+        /* we could instead have sys_param_addr() use r10, like we do on linux */
+        mc->r10 = mc->xcx;
+    }
+# endif
+}
+#endif /* CLIENT_INTERFACE */
+
diff --git a/core/win32/syscallx.h b/core/win32/syscallx.h
new file mode 100644
index 000000000..ffd959dbf
--- /dev/null
+++ b/core/win32/syscallx.h
@@ -0,0 +1,182 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2007 Determina Corp. */
+
+/*
+ * syscallx.h
+ *
+ * System call support due to the varying system call numbers across platforms.
+ * To have one binary work on multiple Windows we can't have one set of constants.
+ *
+ * Usage:
+ * #define SYSCALL(name, act, arg, vista_sp1_x64, vista_sp1, vista_sp0_x64,
+ *                 vista_sp0, 2k3, xp64, wow64, xp, 2k, ntsp4, ntsp3, ntsp0)
+ * #include "syscallx.h"
+ * #undef SYSCALL
+ *
+ */
+
+/* NOTE - Vista Beta 2 has different syscall numbers than Vista final,
+ * the #s here are for for Vista Final (see cvs code attic for Beta
+ * 2). 
+ */ 
+
+/* We expect x64 2003 and x64 XP to have the same system call numbers but
+ * that has not been verified
+ */
+
+/* for NT SP4, SP5, SP6, and SP6a 
+ * Metasploit's table claims SP4 has additional syscalls, though our
+ * investigation disagrees (see case 5616) -- even if so they are appended 
+ * and so don't affect the numbering of any of these.
+ */
+
+/* Column descriptions:
+ * action?    == does DR need to take action when the app issues this system call?
+ * nargs      == number of arguments on x64; should we assume this is always arg32/4?
+ * arg32      == argument size in bytes on x86
+ * wow64      == index into argument conversion routines (see case 3922)
+ * all others == system call number for that windows version
+ */
+
+/* FIXME: MS sometimes changes argsz between OS versions (see vista case 6853 for
+ * some examples) instead of adding an Ex version: if that happens to any of the
+ * syscalls we care about we'll have to augment this table.
+ */
+
+/* shorter name for the table */
+#define NONE SYSCALL_NOT_PRESENT
+/* to make some system calls actionable only in DEBUG (because all we do is log) */
+#define ACTION_LOG IF_DEBUG_ELSE(true, false)
+/*                                                       vista         vista
+/*                                                        x64   vista   x64   vista
+ *      Name                       action?  nargs arg32   sp1    sp1    sp0    sp0    2003   xp64 wow64     xp  2000 ntsp4 ntsp3 ntsp0 */
+SYSCALL(Continue,                     true,     2, 0x08, 0x040, 0x037, 0x040, 0x037, 0x022, 0x040,    0, 0x020, 0x1c, 0x13, 0x13, 0x13)
+SYSCALL(CallbackReturn,               true,     3, 0x0c, 0x002, 0x02b, 0x002, 0x02b, 0x016, 0x002,    0, 0x014, 0x13, 0x0b, 0x0b, 0x0b)
+SYSCALL(SetContextThread,             true,     2, 0x08, 0x149, 0x121, 0x14f, 0x125, 0x0dd, 0x0f6,    0, 0x0d5, 0xba, 0x99, 0x99, 0x98)
+SYSCALL(GetContextThread,             true,     2, 0x08, 0x0c7, 0x097, 0x0c9, 0x097, 0x059, 0x09d,    0, 0x055, 0x49, 0x3c, 0x3c, 0x3c)
+SYSCALL(CreateProcess,                true,     8, 0x20, 0x0a0, 0x048, 0x0a2, 0x048, 0x031, 0x082,    0, 0x02f, 0x29, 0x1f, 0x1f, 0x1f)
+SYSCALL(CreateProcessEx,              true,     9, 0x24, 0x04a, 0x049, 0x04a, 0x049, 0x032, 0x04a,    0, 0x030, NONE, NONE, NONE, NONE)
+SYSCALL(CreateUserProcess,            true,    11, 0x2c, 0x0aa, 0x17f, 0x0ac, 0x185,  NONE,  NONE, NONE,  NONE, NONE, NONE, NONE, NONE)
+SYSCALL(TerminateProcess,             true,     2, 0x08, 0x029, 0x14e, 0x029, 0x152, 0x10a, 0x029,    0, 0x101, 0xe0, 0xbb, 0xbb, 0xba)
+SYSCALL(CreateThread,                 true,     8, 0x20, 0x04b, 0x04e, 0x04b, 0x04e, 0x037, 0x04b,    0, 0x035, 0x2e, 0x24, 0x24, 0x24)
+SYSCALL(CreateThreadEx,               true,    11, 0x2c, 0x0a5, 0x17e, 0x0a7, 0x184,  NONE,  NONE, NONE,  NONE, NONE, NONE, NONE, NONE)
+SYSCALL(TerminateThread,              true,     2, 0x08, 0x050, 0x14f, 0x050, 0x153, 0x10b, 0x050,    0, 0x102, 0xe1, 0xbc, 0xbc, 0xbb)
+SYSCALL(SuspendThread,                true,     2, 0x08, 0x172, 0x14b, 0x179, 0x14f, 0x107, 0x118, 0x07, 0x0fe, 0xdd, 0xb9, 0xb9, 0xb8)
+SYSCALL(AllocateVirtualMemory,        true,     6, 0x18, 0x015, 0x012, 0x015, 0x012, 0x012, 0x015,    0, 0x011, 0x10, 0x0a, 0x0a, 0x0a)
+SYSCALL(FreeVirtualMemory,            true,     4, 0x10, 0x01b, 0x093, 0x01b, 0x093, 0x057, 0x01b,    0, 0x053, 0x47, 0x3a, 0x3a, 0x3a)
+SYSCALL(ProtectVirtualMemory,         true,     5, 0x14, 0x04d, 0x0d2, 0x04d, 0x0d2, 0x08f, 0x04d,    0, 0x089, 0x77, 0x60, 0x60, 0x60)
+SYSCALL(QueryVirtualMemory,           true,     6, 0x18, 0x020, 0x0fd, 0x020, 0x0fd, 0x0ba, 0x020,    0, 0x0b2, 0x9c, 0x81, 0x81, 0x81)
+SYSCALL(WriteVirtualMemory,           true,     5, 0x14, 0x037, 0x166, 0x037, 0x16a, 0x11f, 0x037,    0, 0x115, 0xf0, 0xcb, 0xcb, 0xc9)
+SYSCALL(MapViewOfSection,             true,    10, 0x28, 0x025, 0x0b1, 0x025, 0x0b1, 0x071, 0x025,    0, 0x06c, 0x5d, 0x49, 0x49, 0x49)
+SYSCALL(UnmapViewOfSection,           true,     2, 0x08, 0x027, 0x15c, 0x027, 0x160, 0x115, 0x027,    0, 0x10b, 0xe7, 0xc2, 0xc2, 0xc1)
+SYSCALL(FlushInstructionCache,        true,     3, 0x0c, 0x0bf, 0x08d, 0x0c1, 0x08d, 0x052, 0x098, 0x0c, 0x04e, 0x42, 0x36, 0x36, 0x36)
+SYSCALL(FreeUserPhysicalPages,        true,     3, 0x0c, 0x0c4, 0x092, 0x0c6, 0x092, 0x056, 0x09c,    0, 0x052, 0x46, NONE, NONE, NONE)
+SYSCALL(MapUserPhysicalPages,         true,     3, 0x0c, 0x0e4, 0x0af, 0x0e7, 0x0af, 0x06f, 0x0b2, 0x0a, 0x06a, 0x5b, NONE, NONE, NONE)
+    /* FIXME: processing needed only for
+     * ASLR_SHARED, but can't be made dynamic */
+SYSCALL(OpenSection,                  true,     3, 0x0c, 0x034, 0x0c5, 0x034, 0x0c5, 0x083, 0x034,    0, 0x07d, 0x6c, 0x56, 0x56, 0x56)
+SYSCALL(CreateSection,                true,     7, 0x1c, 0x047, 0x04b, 0x047, 0x04b, 0x034, 0x047,    0, 0x032, 0x2b, 0x21, 0x21, 0x21)
+#ifdef DEBUG                                                                                                    
+    /* FIXME: move this stuff to an strace-like
+     * client, not needed for core DynamoRIO (at
+     * least not that we know of)
+     */
+SYSCALL(AlertResumeThread,       ACTION_LOG,    2, 0x08, 0x06a, 0x00d, 0x06a, 0x00d, 0x00d, 0x069, 0x07, 0x00c, 0x0b, 0x06, 0x06, 0x06)
+SYSCALL(OpenFile,                ACTION_LOG,    6, 0x18, 0x030, 0x0ba, 0x030, 0x0ba, 0x07a, 0x030,    0, 0x074, 0x64, 0x4f, 0x4f, 0x4f)
+#endif                                                                 
+    /* These ones are here for DR's own use */
+SYSCALL(ResumeThread,            ACTION_LOG,    2, 0x08, 0x04f, 0x11a, 0x04f, 0x119, 0x0d6, 0x04f, 0x07, 0x0ce, 0xb5, 0x96, 0x96, 0x95)
+SYSCALL(TestAlert,                    false,    0,    0, 0x175, 0x150, 0x17c, 0x154, 0x10c, 0x11b, 0x02, 0x103, 0xe2, 0xbd, 0xbd, 0xbc)
+SYSCALL(RaiseException,               false,    3, 0x0c, 0x126, 0x100, 0x12b, 0x100, 0x0bd, 0x0e1,    0, 0x0b5, 0x9f, 0x84, 0x84, 0x84)
+SYSCALL(CreateFile,                   false,   11, 0x2c, 0x052, 0x03c, 0x052, 0x03c, 0x027, 0x052,    0, 0x025, 0x20, 0x17, 0x17, 0x17)
+SYSCALL(CreateKey,                    false,    7, 0x1c, 0x01a, 0x040, 0x01a, 0x040, 0x02b, 0x01a,    0, 0x029, 0x23, 0x19, 0x19, 0x19)
+SYSCALL(OpenKey,                      false,    3, 0x0c, 0x00f, 0x0bd, 0x00f, 0x0bd, 0x07d, 0x00f,    0, 0x077, 0x67, 0x51, 0x51, 0x51)
+SYSCALL(SetInformationFile,           false,    5, 0x14, 0x024, 0x12d, 0x024, 0x131, 0x0e9, 0x024,    0, 0x0e0, 0xc2, 0xa1, 0xa1, 0xa0)
+SYSCALL(SetValueKey,                  false,    6, 0x18, 0x05d, 0x144, 0x05d, 0x148, 0x100, 0x05d,    0, 0x0f7, 0xd7, 0xb3, 0xb3, 0xb2)
+
+#undef NONE
+#undef ACTION_LOG
+
+
+
+/* Attic - there is little point in continuing to update the syscall numbers
+ * below since they are only used for ignorable system calls which is
+ * terminally broken anyways. */
+#if 0
+    /* we don't intercept these syscalls for correctness or security policies,
+     * but we need to come back to a non-cache point for them since they
+     * are alertable and callbacks can be delivered during the syscall.
+     * this list came from this filter of the syscall names:
+     *   grep 'Alert|Wait' | grep -v CreateWaitablePort
+     *   (NtContinue is already up above)
+     *   plus grep 'Alert|Wait' in ntdll.h (ZwDelayExecution)
+     */
+    /* FIXME - don't think all of these are in fact alertable, plus many file
+     * io syscalls are alertable depending on the options passed when the file
+     * handle was created.  There also may other alertable system calls we
+     * don't know about and since we don't even use ignore syscalls (with no
+     * plan to bring it back) having them in our arrays doesn't serve much
+     * purpose. */
+    /* FIXME: NT4.0 (only) has two more exported *Wait* routines,
+     * NtSetHighWaitLowThread and NtSetLowWaitHighThread, that each
+     * have a gap in the system call numbering, but in the ntdll dump
+     * they look like int 2b and 2c, resp.  Now, Inside Win2K lists int 2c as
+     * "KiSetLowWaitHighThread" (though 2b is what we expect, even on NT,
+     * KiCallbackReturn).  Nebbett has more pieces of the story:
+     * "three of the four entry points purporting to refer to this system
+     * service actually invoke a different routine", int 2c/2b, which does not
+     * do what this call is supposed to do -- only NTOSKRNL!NtSet* does the
+     * right thing.  Thus, we don't bother to intercept, as we will never
+     * see those system calls, right?
+     */
+/*                                                       vista  vista
+ *      Name                       action?  nargs arg32   sp1    sp0    2003   xp64 wow64     xp  2000 ntsp4 ntsp3 ntsp0 */
+SYSCALL(AlertThread,                  false,    1, 0x04, 0x00e, 0x00e, 0x00e, 0x06a, 0x03, 0x00d, 0x0c, 0x07, 0x07, 0x07)
+SYSCALL(DelayExecution,               false,    2, 0x08, 0x076, 0x076, 0x03d, 0x031, 0x06, 0x03b, 0x32, 0x27, 0x27, 0x27)
+SYSCALL(ReplyWaitReceivePort,         false,    4, 0x10, 0x10f, 0x10e, 0x0cb, 0x008,    0, 0x0c3, 0xab, 0x90, 0x90, 0x8f)
+SYSCALL(ReplyWaitReceivePortEx,       false,    5, 0x14, 0x110, 0x10f, 0x0cc, 0x028,    0, 0x0c4, 0xac, NONE, NONE, NONE)
+SYSCALL(ReplyWaitReplyPort,           false,    2, 0x08, 0x111, 0x110, 0x0cd, 0x0e8,    0, 0x0c5, 0xad, 0x91, 0x91, 0x90)
+SYSCALL(ReplyWaitSendChannel,         false,    3, 0x0c,  NONE,  NONE,  NONE,  NONE, NONE,  NONE, 0xf4, 0xcf, 0xd0, 0xce)
+SYSCALL(RequestWaitReplyPort,         false,    3, 0x0c, 0x114, 0x113, 0x0d0, 0x01f,    0, 0x0c8, 0xb0, 0x93, 0x93, 0x92)
+SYSCALL(SendWaitReplyChannel,         false,    4, 0x10,  NONE,  NONE,  NONE,  NONE, NONE,  NONE, 0xf5, 0xd0, 0xd1, 0xcf)
+SYSCALL(SetHighWaitLowEventPair,      false,    1, 0x04, 0x12b, 0x12f, 0x0e7, 0x0fe, 0x03, 0x0de, 0xc1, 0x9f, 0x9f, 0x9e)
+SYSCALL(SetLowWaitHighEventPair,      false,    1, 0x04, 0x138, 0x13c, 0x0f4, 0x107, 0x03, 0x0eb, 0xcc, 0xaa, 0xaa, 0xa9)
+SYSCALL(SignalAndWaitForSingleObject, false,    4, 0x10, 0x147, 0x14b, 0x103, 0x114, 0x13, 0x0fa, 0xda, 0xb6, 0xb6, 0xb5)
+SYSCALL(WaitForDebugEvent,            false,    4, 0x10, 0x15e, 0x162, 0x117, 0x124,    0, 0x10d, NONE, NONE, NONE, NONE)
+SYSCALL(WaitForKeyedEvent,            false,    4, 0x10, 0x16b, 0x16f, 0x124, 0x125, 0x15, 0x11a, NONE, NONE, NONE, NONE)
+SYSCALL(WaitForMultipleObjects,       false,    5, 0x14, 0x15f, 0x163, 0x118, 0x058, 0x1d, 0x10e, 0xe9, 0xc4, 0xc4, 0xc3)
+SYSCALL(WaitForSingleObject,          false,    3, 0x0c, 0x160, 0x164, 0x119, 0x001, 0x0d, 0x10f, 0xea, 0xc5, 0xc5, 0xc4)
+SYSCALL(WaitHighEventPair,            false,    1, 0x04, 0x161, 0x165, 0x11a, 0x126, 0x03, 0x110, 0xeb, 0xc6, 0xc6, 0xc5)
+SYSCALL(WaitLowEventPair,             false,    1, 0x04, 0x162, 0x166, 0x11b, 0x127, 0x03, 0x111, 0xec, 0xc7, 0xc7, 0xc6)
+#endif /* if 0 */
diff --git a/core/x86/Makefile b/core/x86/Makefile
new file mode 100644
index 000000000..1b0e66d49
--- /dev/null
+++ b/core/x86/Makefile
@@ -0,0 +1,45 @@
+# **********************************************************
+# Copyright (c) 2004-2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# Copyright (c) 2004-2007 Determina Corp.
+
+# local submodule Makefile wrapper
+#   keep in sync with the copies in win32 and linux
+# execute any rule using the top level makefile
+
+# You can actually override any rule here, 
+# but currently there is no reason to do so
+
+all:
+
+# for any action we go to the master Makefile
+%: force
+	@$(MAKE) -C .. $@
+force: ;
diff --git a/core/x86/arch.c b/core/x86/arch.c
new file mode 100644
index 000000000..227452574
--- /dev/null
+++ b/core/x86/arch.c
@@ -0,0 +1,4114 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * arch.c - x86 architecture specific routines
+ */
+
+#include "globals.h"
+#include "link.h"
+#include "fragment.h"
+
+#include "arch.h"
+#include "instr.h"
+#include "instr_create.h"
+#include "decode.h"
+#include "decode_fast.h"
+#include "fcache.h"
+#include "proc.h"
+#include "instrument.h"
+
+#include <string.h> /* for memcpy */
+
+#if defined(DEBUG) || defined(INTERNAL)
+# include "disassemble.h"
+#endif
+
+/* in x86.asm */
+#ifdef RETURN_STACK
+void return_lookup(void);
+void unlinked_return(void);
+#endif
+
+/* in interp.c */
+void interp_init(void);
+void interp_exit(void);
+
+/* Thread-shared generated routines.
+ * We don't allocate the shared_code statically so that we can mark it 
+ * executable.
+ */
+generated_code_t *shared_code = NULL;
+#ifdef X64
+/* PR 282576: For WOW64 processes we need context switches that swap between 64-bit
+ * mode and 32-bit mode when executing 32-bit code cache code, as well as
+ * 32-bit-targeted IBL routines for performance.
+ */
+generated_code_t *shared_code_x86 = NULL;
+#endif
+
+static int syscall_method = SYSCALL_METHOD_UNINITIALIZED;
+byte *app_sysenter_instr_addr = NULL;
+#ifdef LINUX
+static bool sysenter_hook_failed = false;
+#endif
+
+/* static functions forward references */
+static byte *
+emit_ibl_routines(dcontext_t *dcontext, generated_code_t *code,
+                  byte *pc, byte *fcache_return_pc,
+                  ibl_source_fragment_type_t source_fragment_type,
+                  bool thread_shared,
+                  bool target_trace_table,
+                  ibl_code_t ibl_code[]);
+
+static byte *
+emit_syscall_routines(dcontext_t *dcontext, generated_code_t *code, byte *pc,
+                      bool thread_shared);
+
+int
+reg_spill_tls_offs(reg_id_t reg)
+{
+    switch (reg) {
+    case REG_XAX: return TLS_XAX_SLOT;
+    case REG_XBX: return TLS_XBX_SLOT;
+    case REG_XCX: return TLS_XCX_SLOT;
+    case REG_XDX: return TLS_XDX_SLOT;
+    }
+    /* don't assert if another reg passed: used on random regs looking for spills */
+    return -1;
+}
+
+#ifdef INTERNAL
+/* routine can be used for dumping both thread private and the thread shared routines */
+static void
+dump_emitted_routines(dcontext_t *dcontext, file_t file,
+                      const char *code_description, 
+                      generated_code_t *code, byte *emitted_pc)
+{
+    byte *last_pc;
+
+    print_file(file, "%s routines created:\n", code_description);
+    {
+        last_pc = code->gen_start_pc;
+        do {
+            const char *ibl_brtype;
+            const char *ibl_name =
+                get_ibl_routine_name(dcontext, (cache_pc)last_pc, &ibl_brtype);
+
+# ifdef WINDOWS
+            /* must test first, as get_ibl_routine_name will think "bb_ibl_indjmp" */
+            if (last_pc == code->unlinked_shared_syscall)
+                print_file(file, "unlinked_shared_syscall:\n");
+            else if (last_pc == code->shared_syscall)
+                print_file(file, "shared_syscall:\n");
+            else
+# endif
+            if (ibl_name)
+                print_file(file, "%s_%s:\n", ibl_name, ibl_brtype);
+            else if (last_pc == code->fcache_enter)
+                print_file(file, "fcache_enter:\n");
+            else if (last_pc == code->fcache_return)
+                print_file(file, "fcache_return:\n");
+# ifdef RETURN_STACK
+            else if (last_pc == code->return_lookup)
+                print_file(file, "return_lookup:\n");
+            else if (last_pc == code->unlinked_return)
+                print_file(file, "unlinked_return:\n");
+# endif
+            else if (last_pc == code->do_syscall)
+                print_file(file, "do_syscall:\n");
+# ifdef WINDOWS
+            else if (last_pc == code->fcache_enter_indirect)
+                print_file(file, "fcache_enter_indirect:\n");
+            else if (last_pc == code->do_callback_return)
+                print_file(file, "do_callback_return:\n");
+# else
+            else if (last_pc == code->do_int_syscall)
+                print_file(file, "do_int_syscall:\n");
+            else if (last_pc == code->do_clone_syscall)
+                print_file(file, "do_clone_syscall:\n");
+# endif
+# ifdef LINUX
+            else if (last_pc == code->new_thread_dynamo_start)
+                print_file(file, "new_thread_dynamo_start:\n");
+# endif
+# ifdef TRACE_HEAD_CACHE_INCR
+            else if (last_pc == code->trace_head_incr)
+                print_file(file, "trace_head_incr:\n");
+# endif
+            else if (last_pc == code->reset_exit_stub)
+                print_file(file, "reset_exit_stub:\n");
+            else if (last_pc == code->fcache_return_coarse)
+                print_file(file, "fcache_return_coarse:\n");
+            else if (last_pc == code->trace_head_return_coarse)
+                print_file(file, "trace_head_return_coarse:\n");
+            last_pc = disassemble_with_bytes(dcontext, last_pc, file);
+        } while (last_pc < emitted_pc);
+        print_file(file, "%s routines size: "SSZFMT" / "SSZFMT"\n\n", 
+                   code_description, emitted_pc - code->gen_start_pc,
+                   code->commit_end_pc - code->gen_start_pc);
+    }
+}
+
+void
+dump_emitted_routines_to_file(dcontext_t *dcontext, const char *filename,
+                              const char *label, generated_code_t *code,
+                              byte *stop_pc)
+{
+    file_t file = open_log_file(filename, NULL, 0);
+    if (file != INVALID_FILE) {
+        /* FIXME: we currently miss later patches for table & mask, but
+         * that only changes a few immeds
+         */
+        dump_emitted_routines(dcontext, file, label, code, stop_pc);
+        os_close(file);
+    } else
+        ASSERT_NOT_REACHED();            
+}
+#endif /* INTERNAL */
+
+/*** functions exported to src directory ***/
+
+byte *
+code_align_forward(byte *pc, size_t alignment)
+{
+    byte *new_pc = (byte *) ALIGN_FORWARD(pc, alignment);
+    DODEBUG({
+        SET_TO_NOPS(pc, new_pc - pc);
+    });
+    return new_pc;
+}
+
+static byte *
+move_to_start_of_cache_line(byte *pc)
+{
+    return code_align_forward(pc, proc_get_cache_line_size());
+}
+
+/* The real size of generated code we need varies by cache line size and
+ * options like inlining of ibl code.  We also generate different routines
+ * for thread-private and thread-shared.  So, we dynamically extend the size
+ * as we generate.  Currently our max is under 5 pages.
+ */
+#define GENCODE_RESERVE_SIZE (5*PAGE_SIZE)
+
+#define GENCODE_COMMIT_SIZE \
+    ((size_t)(ALIGN_FORWARD(sizeof(generated_code_t), PAGE_SIZE) + PAGE_SIZE))
+
+static byte *
+check_size_and_cache_line(generated_code_t *code, byte *pc)
+{
+    /* Assumption: no single emit uses more than a page.
+     * We keep an extra page at all times and release it at the end.
+     */
+    byte *next_pc = move_to_start_of_cache_line(pc);
+    if ((byte *)ALIGN_FORWARD(pc, PAGE_SIZE) + PAGE_SIZE > code->commit_end_pc) {
+        ASSERT(code->commit_end_pc + PAGE_SIZE <= ((byte *)code) + GENCODE_RESERVE_SIZE);
+        heap_mmap_extend_commitment(code->commit_end_pc, PAGE_SIZE);
+        code->commit_end_pc += PAGE_SIZE;
+    }
+    return next_pc;
+}
+
+static void
+release_final_page(generated_code_t *code)
+{
+    /* FIXME: have heap_mmap not allocate a guard page, and use our
+     * extra for that page, to use one fewer total page of address space.
+     */
+    size_t leftover = (ptr_uint_t)code->commit_end_pc -
+        ALIGN_FORWARD(code->gen_end_pc, PAGE_SIZE);
+    ASSERT(code->commit_end_pc >= (byte *) ALIGN_FORWARD(code->gen_end_pc, PAGE_SIZE));
+    ASSERT(ALIGNED(code->commit_end_pc, PAGE_SIZE));
+    ASSERT(ALIGNED(leftover, PAGE_SIZE));
+    if (leftover > 0) {
+        heap_mmap_retract_commitment(code->commit_end_pc - leftover, leftover);
+        code->commit_end_pc -= leftover;
+    }
+    LOG(THREAD_GET, LOG_EMIT, 1,
+        "Generated code "PFX": %d header, "SZFMT" gen, "SZFMT" commit/%d reserve\n", 
+        code, sizeof(*code), code->gen_end_pc - code->gen_start_pc,
+        (ptr_uint_t)code->commit_end_pc - (ptr_uint_t)code, GENCODE_RESERVE_SIZE);
+}
+
+static void
+shared_gencode_init(IF_X64_ELSE(bool x86_mode, void))
+{
+    generated_code_t *gencode;
+    ibl_branch_type_t branch_type;
+    byte *pc;
+#ifdef X64
+    fragment_t *fragment;
+#endif
+
+    gencode = heap_mmap_reserve(GENCODE_RESERVE_SIZE, GENCODE_COMMIT_SIZE);
+    /* we would return gencode and let caller assign, but emit routines
+     * that this routine calls query the shared vars so we set here
+     */
+#ifdef X64
+    if (x86_mode)
+        shared_code_x86 = gencode;
+    else
+#endif
+        shared_code = gencode;
+    memset(gencode, 0, sizeof(*gencode));
+
+    IF_X64(gencode->x86_mode = x86_mode);
+    /* Generated code immediately follows struct */
+    gencode->gen_start_pc = ((byte *)gencode) + sizeof(*gencode);
+    gencode->commit_end_pc = ((byte *)gencode) + GENCODE_COMMIT_SIZE;
+    for (branch_type = IBL_BRANCH_TYPE_START; 
+         branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+        gencode->trace_ibl[branch_type].initialized = false;
+        gencode->bb_ibl[branch_type].initialized = false;
+        gencode->coarse_ibl[branch_type].initialized = false;
+        /* cache the mode so we can pass just the ibl_code_t around */
+        IF_X64(gencode->trace_ibl[branch_type].x86_mode = x86_mode);
+        IF_X64(gencode->bb_ibl[branch_type].x86_mode = x86_mode);
+        IF_X64(gencode->coarse_ibl[branch_type].x86_mode = x86_mode);
+    }
+
+    pc = gencode->gen_start_pc;
+    pc = check_size_and_cache_line(gencode, pc);
+    gencode->fcache_enter = pc;
+    pc = emit_fcache_enter_shared(GLOBAL_DCONTEXT, gencode, pc);
+    pc = check_size_and_cache_line(gencode, pc);
+    gencode->fcache_return = pc;
+    pc = emit_fcache_return_shared(GLOBAL_DCONTEXT, gencode, pc);
+    if (DYNAMO_OPTION(coarse_units)) {
+        pc = check_size_and_cache_line(gencode, pc);
+        gencode->fcache_return_coarse = pc;
+        pc = emit_fcache_return_coarse(GLOBAL_DCONTEXT, gencode, pc);
+        pc = check_size_and_cache_line(gencode, pc);
+        gencode->trace_head_return_coarse = pc;
+        pc = emit_trace_head_return_coarse(GLOBAL_DCONTEXT, gencode, pc);
+    }
+#ifdef WINDOWS_PC_SAMPLE
+    gencode->fcache_enter_return_end = pc;
+#endif
+
+    /* PR 244737: thread-private uses shared gencode on x64.
+     * Should we set the option instead? */
+    if (IF_X64(!DYNAMO_OPTION(disable_traces) ||)
+        DYNAMO_OPTION(shared_trace_ibl_routine)) {
+        /* expected to be false for private trace IBL routine  */
+        pc = emit_ibl_routines(GLOBAL_DCONTEXT, gencode,
+                               pc, gencode->fcache_return, 
+                               DYNAMO_OPTION(shared_traces) ?
+                               IBL_TRACE_SHARED : IBL_TRACE_PRIVATE, /* source_fragment_type */
+                               /* thread_shared */
+                               IF_X64_ELSE(true, DYNAMO_OPTION(shared_trace_ibl_routine)),
+                               true, /* target_trace_table */
+                               gencode->trace_ibl);
+    }
+    if (IF_X64_ELSE(true, DYNAMO_OPTION(shared_bbs))) {
+        pc = emit_ibl_routines(GLOBAL_DCONTEXT, gencode,
+                               pc, gencode->fcache_return,
+                               IBL_BB_SHARED, /* source_fragment_type */
+                               /* thread_shared */
+                               IF_X64_ELSE(true, SHARED_FRAGMENTS_ENABLED()),
+                               !DYNAMO_OPTION(bb_ibl_targets), /* target_trace_table */
+                               gencode->bb_ibl);
+    }
+    if (DYNAMO_OPTION(coarse_units)) {
+        pc = emit_ibl_routines(GLOBAL_DCONTEXT, gencode, pc,
+                               /* ibl routines use regular fcache_return */
+                               gencode->fcache_return, 
+                               IBL_COARSE_SHARED, /* source_fragment_type */
+                               /* thread_shared */
+                               IF_X64_ELSE(true, SHARED_FRAGMENTS_ENABLED()),
+                               !DYNAMO_OPTION(bb_ibl_targets), /*target_trace_table*/
+                               gencode->coarse_ibl);
+    }
+                                   
+#ifdef WINDOWS_PC_SAMPLE
+    gencode->ibl_routines_end = pc;
+#endif
+#if defined(WINDOWS) && !defined(X64)
+    /* no dispatch needed on x64 since syscall routines are thread-shared */
+    if (DYNAMO_OPTION(shared_fragment_shared_syscalls)) {
+        pc = check_size_and_cache_line(gencode, pc);
+        gencode->shared_syscall = pc;
+        pc = emit_shared_syscall_dispatch(GLOBAL_DCONTEXT, pc);
+        pc = check_size_and_cache_line(gencode, pc);
+        gencode->unlinked_shared_syscall = pc;
+        pc = emit_unlinked_shared_syscall_dispatch(GLOBAL_DCONTEXT, pc);
+        LOG(GLOBAL, LOG_EMIT, 3,
+            "shared_syscall_dispatch: linked "PFX", unlinked "PFX"\n",
+            gencode->shared_syscall, gencode->unlinked_shared_syscall);
+    }
+#endif
+
+#ifdef LINUX
+    /* must create before emit_do_clone_syscall() in emit_syscall_routines() */
+    pc = check_size_and_cache_line(gencode, pc);
+    gencode->new_thread_dynamo_start = pc;
+    pc = emit_new_thread_dynamo_start(GLOBAL_DCONTEXT, pc);
+#endif
+
+#ifdef X64
+# ifdef WINDOWS
+    /* plain fcache_enter indirects through edi, and next_tag is in tls,
+     * so we don't need a separate routine for callback return
+     */
+    gencode->fcache_enter_indirect = gencode->fcache_enter;
+    gencode->shared_syscall_code.x86_mode = x86_mode;
+# endif
+    /* PR 284029: for now we assume there are no syscalls in x86 code */
+    if (IF_X64_ELSE(!x86_mode, true)) {
+        /* PR 244737: syscall routines are all shared */
+        pc = emit_syscall_routines(GLOBAL_DCONTEXT, gencode, pc, true/*thread-shared*/);
+    }
+
+    /* since we always have a shared fcache_return we can make reset stub shared */
+    gencode->reset_exit_stub = pc;
+    fragment = linkstub_fragment(GLOBAL_DCONTEXT, (linkstub_t *) get_reset_linkstub());
+    if (gencode->x86_mode)
+        fragment = empty_fragment_mark_x86(fragment);
+    /* reset exit stub should look just like a direct exit stub */
+    pc += insert_exit_stub_other_flags
+        (GLOBAL_DCONTEXT, fragment,
+         (linkstub_t *) get_reset_linkstub(), pc, LINK_DIRECT);
+#elif defined(LINUX) && defined(HAVE_TLS)
+    /* PR 212570: we need a thread-shared do_syscall for our vsyscall hook */
+    /* PR 361894: we don't support sysenter if no TLS */
+    ASSERT(gencode->do_syscall == NULL);
+    pc = check_size_and_cache_line(gencode, pc);
+    gencode->do_syscall = pc;
+    pc = emit_do_syscall(GLOBAL_DCONTEXT, gencode, pc, gencode->fcache_return,
+                         true/*shared*/, false, &gencode->do_syscall_offs);
+#endif
+
+#ifdef TRACE_HEAD_CACHE_INCR
+    pc = check_size_and_cache_line(gencode, pc);
+    gencode->trace_head_incr = pc;
+    pc = emit_trace_head_incr_shared(GLOBAL_DCONTEXT, pc, gencode->fcache_return);
+#endif
+
+    ASSERT(pc < gencode->commit_end_pc);
+    gencode->gen_end_pc = pc;
+    release_final_page(gencode);
+
+    DOLOG(3, LOG_EMIT, {
+        dump_emitted_routines(GLOBAL_DCONTEXT, GLOBAL,
+                              IF_X64_ELSE(x86_mode ? "thread-shared x86" :
+                                          "thread-shared", "thread-shared"),
+                              gencode, pc);
+    });
+#ifdef INTERNAL
+    if (INTERNAL_OPTION(gendump)) {
+        dump_emitted_routines_to_file(GLOBAL_DCONTEXT, "gencode-shared",
+                                      IF_X64_ELSE(x86_mode ? "thread-shared x86" :
+                                                  "thread-shared", "thread-shared"),
+                                      gencode, pc);
+    }
+#endif
+#ifdef WINDOWS_PC_SAMPLE
+    if (dynamo_options.profile_pcs && 
+        dynamo_options.prof_pcs_gencode >= 2 && 
+        dynamo_options.prof_pcs_gencode <= 32) {
+        gencode->profile = 
+            create_profile(gencode->gen_start_pc, pc,
+                           dynamo_options.prof_pcs_gencode, NULL);
+        start_profile(gencode->profile);
+    } else 
+        gencode->profile = NULL;
+#endif
+
+    gencode->writable = true;
+    protect_generated_code(gencode, READONLY);
+}
+
+/* arch-specific initializations */
+void
+arch_init()
+{
+    ASSERT(sizeof(opnd_t) == EXPECTED_SIZEOF_OPND);
+    /* ensure our flag sharing is done properly */
+    ASSERT(LINK_FINAL_INSTR_SHARED_FLAG < INSTR_FIRST_NON_LINK_SHARED_FLAG);
+    ASSERT_TRUNCATE(byte, byte, OPSZ_LAST_ENUM);
+    DODEBUG({ reg_check_reg_fixer(); });
+
+    /* Verify that the structures used for a register spill area and to hold IBT
+     * table addresses & masks for IBL code are laid out as expected. We expect
+     * the spill area to be at offset 0 within the container struct and for the
+     * table address/mask pair array to follow immediately after the spill area.
+     */
+    /* FIXME These can be converted into compile-time checks as follows:
+     *
+     *    lookup_table_access_t table[
+     *       (offsetof(local_state_extended_t, spill_space) == 0 &&
+     *        offsetof(local_state_extended_t, table_space) ==
+     *           sizeof(spill_state_t)) ? IBL_BRANCH_TYPE_END : -1 ];
+     *
+     * This isn't self-descriptive, though, so it's not being used right now
+     * (xref case 7097).
+     */
+    ASSERT(offsetof(local_state_extended_t, spill_space) == 0);
+    ASSERT(offsetof(local_state_extended_t, table_space) == sizeof(spill_state_t));
+#ifdef WINDOWS
+    /* syscall_init() should have already set the syscall_method so go ahead
+     * and create the globlal_do_syscall now */
+    ASSERT(syscall_method != SYSCALL_METHOD_UNINITIALIZED);
+#endif
+
+    /* Ensure we have no unexpected padding inside structs that include
+     * dr_mcontext_t (app_state_at_intercept_t and dcontext_t) */
+    ASSERT(offsetof(dr_mcontext_t, pc) + sizeof(byte*) == sizeof(dr_mcontext_t));
+    ASSERT(offsetof(app_state_at_intercept_t, mc) ==
+           offsetof(app_state_at_intercept_t, app_errno) + sizeof(int)
+           IF_X64(+ 4/*padding*/));
+    /* Try to catch errors in x86.asm offsets for dcontext_t */
+    ASSERT(sizeof(unprotected_context_t) == sizeof(dr_mcontext_t) + 
+           IF_WINDOWS_ELSE(IF_X64_ELSE(8, 4), 8));
+
+    interp_init();
+
+#ifdef CHECK_RETURNS_SSE2
+    if (proc_has_feature(FEATURE_SSE2)) {
+        FATAL_USAGE_ERROR(CHECK_RETURNS_SSE2_REQUIRES_SSE2, 2, 
+                          get_application_name(), get_application_pid());
+    }
+#endif
+
+    if (USE_SHARED_GENCODE()) {
+        /* thread-shared generated code */
+        /* Assumption: no single emit uses more than a page.
+         * We keep an extra page at all times and release it at the end.
+         * FIXME: have heap_mmap not allocate a guard page, and use our
+         * extra for that page, to use one fewer total page of address space.
+         */
+        ASSERT(GENCODE_COMMIT_SIZE < GENCODE_RESERVE_SIZE);
+
+        shared_gencode_init(IF_X64(false/*x64 mode*/));
+#if defined(X64) && defined(WINDOWS)
+        /* FIXME: usually LOL64 has only 32-bit code (kernel has 32-bit syscall
+         * interface) but for mixed modes how would we know?  We'd have to make
+         * this be initialized lazily on first occurrence.
+         */
+        if (is_wow64_process(NT_CURRENT_PROCESS)) {
+            shared_gencode_init(true/*x86 mode*/);
+        }
+#endif
+    }
+}
+
+#ifdef WINDOWS_PC_SAMPLE
+static void
+arch_extract_profile(dcontext_t *dcontext _IF_X64(gencode_mode_t mode))
+{
+    generated_code_t *tpc = get_emitted_routines_code(dcontext _IF_X64(mode));
+    thread_id_t tid = dcontext == GLOBAL_DCONTEXT ? 0 : dcontext->owning_thread;
+    /* we may not have x86 gencode */
+    ASSERT(tpc != NULL IF_X64(|| mode == GENCODE_X86));
+    if (tpc != NULL && tpc->profile != NULL) {
+
+        ibl_branch_type_t branch_type;
+        int sum;
+
+        protect_generated_code(tpc, WRITABLE);
+
+        stop_profile(tpc->profile);
+        mutex_lock(&profile_dump_lock);
+
+        /* Print the thread id so even if it has no hits we can
+         * count the # total threads. */
+        print_file(profile_file, "Profile for thread "IDFMT"\n", tid);
+        sum = sum_profile_range(tpc->profile, tpc->fcache_enter,
+                                tpc->fcache_enter_return_end);
+        if (sum > 0) {
+            print_file(profile_file, "\nDumping cache enter/exit code profile "
+                       "(thread "IDFMT")\n%d hits\n", tid, sum);
+            dump_profile_range(profile_file, tpc->profile, tpc->fcache_enter,
+                               tpc->fcache_enter_return_end);
+        }
+
+        /* Break out the IBL code by trace/BB and opcode types. */
+        for (branch_type = IBL_BRANCH_TYPE_START; 
+             branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+
+            byte *start;
+            byte *end;
+
+            if (tpc->trace_ibl[branch_type].initialized) {
+                start = tpc->trace_ibl[branch_type].indirect_branch_lookup_routine;
+                end = start + tpc->trace_ibl[branch_type].ibl_routine_length;
+                sum = sum_profile_range(tpc->profile, start, end);
+                if (sum > 0) {
+                    print_file(profile_file, "\nDumping trace IBL code %s profile "
+                               "(thread "IDFMT")\n%d hits\n",
+                               get_branch_type_name(branch_type), tid, sum);
+                    dump_profile_range(profile_file, tpc->profile, start, end);
+                }
+            }
+            if (tpc->bb_ibl[branch_type].initialized) {
+                start = tpc->bb_ibl[branch_type].indirect_branch_lookup_routine;
+                end = start + tpc->bb_ibl[branch_type].ibl_routine_length;
+                sum = sum_profile_range(tpc->profile, start, end);
+                if (sum > 0) {
+                    print_file(profile_file, "\nDumping BB IBL code %s profile "
+                               "(thread "IDFMT")\n%d hits\n",
+                               get_branch_type_name(branch_type), tid, sum);
+                    dump_profile_range(profile_file, tpc->profile, start, end);
+                }
+            }
+            if (tpc->coarse_ibl[branch_type].initialized) {
+                start = tpc->coarse_ibl[branch_type].indirect_branch_lookup_routine;
+                end = start + tpc->coarse_ibl[branch_type].ibl_routine_length;
+                sum = sum_profile_range(tpc->profile, start, end);
+                if (sum > 0) {
+                    print_file(profile_file, "\nDumping coarse IBL code %s profile "
+                               "(thread "IDFMT")\n%d hits\n",
+                               get_branch_type_name(branch_type), tid, sum);
+                    dump_profile_range(profile_file, tpc->profile, start, end);
+                }
+            }
+        }
+
+        sum = sum_profile_range(tpc->profile, tpc->ibl_routines_end,
+                                tpc->profile->end);
+        if (sum > 0) {
+            print_file(profile_file, "\nDumping generated code profile "
+                       "(thread "IDFMT")\n%d hits\n", tid, sum);
+            dump_profile_range(profile_file, tpc->profile,
+                               tpc->ibl_routines_end, tpc->profile->end);
+        }
+
+        mutex_unlock(&profile_dump_lock);
+        free_profile(tpc->profile);
+        tpc->profile = NULL;
+    }
+}
+
+void
+arch_profile_exit()
+{
+    if (USE_SHARED_GENCODE()) {
+        arch_extract_profile(GLOBAL_DCONTEXT _IF_X64(GENCODE_X64));
+        IF_X64(arch_extract_profile(GLOBAL_DCONTEXT _IF_X64(GENCODE_X86)));
+    }
+}
+#endif /* WINDOWS_PC_SAMPLE */
+
+/* arch-specific atexit cleanup */
+void
+arch_exit(IF_WINDOWS_ELSE_NP(bool detach_stacked_callbacks, void))
+{
+    /* we only need to unprotect shared_code for profile extraction
+     * so we do it there to also cover the fast exit path
+     */
+#ifdef WINDOWS_PC_SAMPLE
+    arch_profile_exit();
+#endif
+    /* on x64 we have syscall routines in the shared code so can't free if detaching */
+    if (IF_WINDOWS(IF_X64(!detach_stacked_callbacks &&)) shared_code != NULL) {
+        heap_munmap(shared_code, GENCODE_RESERVE_SIZE);
+    }
+#ifdef X64
+    if (shared_code_x86 != NULL)
+        heap_munmap(shared_code_x86, GENCODE_RESERVE_SIZE);
+#endif
+    interp_exit();
+}
+
+static byte *
+emit_ibl_routine_and_template(dcontext_t *dcontext, generated_code_t *code,
+                              byte *pc, 
+                              byte *fcache_return_pc,
+                              bool target_trace_table,
+                              bool inline_ibl_head,
+                              bool thread_shared,
+                              ibl_branch_type_t branch_type,
+                              ibl_source_fragment_type_t source_type,
+                              ibl_code_t *ibl_code)
+{
+    pc = check_size_and_cache_line(code, pc);
+    ibl_code->initialized = true;
+    ibl_code->indirect_branch_lookup_routine = pc;
+    ibl_code->ibl_head_is_inlined = inline_ibl_head;
+    ibl_code->thread_shared_routine = thread_shared;
+    ibl_code->branch_type = branch_type;
+    ibl_code->source_fragment_type = source_type;
+
+    pc = emit_indirect_branch_lookup(dcontext, code, pc, fcache_return_pc,
+                                     target_trace_table, inline_ibl_head,
+                                     ibl_code);
+    if (inline_ibl_head) {
+        /* create the inlined ibl template */
+        pc = check_size_and_cache_line(code, pc);
+        pc = emit_inline_ibl_stub(dcontext, pc, ibl_code, target_trace_table);
+    }
+
+    return pc;
+}
+
+static byte *
+emit_ibl_routines(dcontext_t *dcontext, generated_code_t *code, byte *pc, 
+                  byte *fcache_return_pc,
+                  ibl_source_fragment_type_t source_fragment_type,
+                  bool thread_shared,
+                  bool target_trace_table,
+                  ibl_code_t ibl_code_routines[])
+{
+    ibl_branch_type_t branch_type;
+    /* emit separate routines for each branch type 
+       The goal is to have routines that target different fragment tables
+       so that we can control for example return targets for RAC,
+       or we can control inlining if some branch types have better hit ratios.
+
+       Currently it only gives us better stats.
+     */
+    /*
+       N.B.: shared fragments requires -atomic_inlined_linking in order to
+       inline ibl lookups, but not for private since they're unlinked by another thread
+       flushing but not linked by anyone but themselves.
+    */
+
+    bool inline_ibl_head =  (IS_IBL_TRACE(source_fragment_type)) ?
+        DYNAMO_OPTION(inline_trace_ibl) : DYNAMO_OPTION(inline_bb_ibl);
+
+    for (branch_type = IBL_BRANCH_TYPE_START; 
+         branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+#ifdef HASHTABLE_STATISTICS
+        /* ugly asserts but we'll stick with uints to save space */
+        IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint
+                      (GET_IBL_TARGET_TABLE(branch_type, target_trace_table)
+                       + offsetof(ibl_table_t, unprot_stats))));
+        ibl_code_routines[branch_type].unprot_stats_offset = (uint)
+            GET_IBL_TARGET_TABLE(branch_type, target_trace_table)
+            + offsetof(ibl_table_t, unprot_stats);
+        IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint
+                      (GET_IBL_TARGET_TABLE(branch_type, target_trace_table)
+                       + offsetof(ibl_table_t, entry_stats_to_lookup_table))));
+        ibl_code_routines[branch_type].entry_stats_to_lookup_table_offset = (uint)
+            GET_IBL_TARGET_TABLE(branch_type, target_trace_table)
+            + offsetof(ibl_table_t, entry_stats_to_lookup_table);
+        IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint
+                      (offsetof(unprot_ht_statistics_t, trace_ibl_stats[branch_type]))));
+        IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint
+                      (offsetof(unprot_ht_statistics_t, bb_ibl_stats[branch_type]))));
+        ibl_code_routines[branch_type].hashtable_stats_offset = (uint)
+            ((IS_IBL_TRACE(source_fragment_type)) ? 
+             offsetof(unprot_ht_statistics_t, trace_ibl_stats[branch_type])
+             : offsetof(unprot_ht_statistics_t, bb_ibl_stats[branch_type]));
+#endif
+        pc = emit_ibl_routine_and_template(dcontext, code, pc, 
+                                           fcache_return_pc,
+                                           target_trace_table,
+                                           inline_ibl_head, thread_shared,
+                                           branch_type, source_fragment_type,
+                                           &ibl_code_routines[branch_type]);
+    }
+    return pc;
+}
+
+static byte *
+emit_syscall_routines(dcontext_t *dcontext, generated_code_t *code, byte *pc,
+                      bool thread_shared)
+{
+#ifdef HASHTABLE_STATISTICS
+    /* Stats for the syscall IBLs (note it is also using the trace hashtable, and it never hits!) */
+# ifdef WINDOWS
+    /* ugly asserts but we'll stick with uints to save space */
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint
+                  (GET_IBL_TARGET_TABLE(IBL_SHARED_SYSCALL, true)
+                   + offsetof(ibl_table_t, unprot_stats))));
+    code->shared_syscall_code.unprot_stats_offset = (uint)
+        GET_IBL_TARGET_TABLE(IBL_SHARED_SYSCALL, true)
+        + offsetof(ibl_table_t, unprot_stats);
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint
+                  (GET_IBL_TARGET_TABLE(IBL_SHARED_SYSCALL, true)
+                   + offsetof(ibl_table_t, entry_stats_to_lookup_table))));
+    code->shared_syscall_code.entry_stats_to_lookup_table_offset = (uint)
+        GET_IBL_TARGET_TABLE(IBL_SHARED_SYSCALL, true)
+        + offsetof(ibl_table_t, entry_stats_to_lookup_table);
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint
+                  (offsetof(unprot_ht_statistics_t, shared_syscall_hit_stats))));
+    code->shared_syscall_code.hashtable_stats_offset = (uint)
+        offsetof(unprot_ht_statistics_t, shared_syscall_hit_stats);
+# endif /* WINDOWS */
+#endif /* HASHTABLE_STATISTICS */
+
+#ifdef WINDOWS
+    pc = check_size_and_cache_line(code, pc);
+    code->do_callback_return = pc;
+    pc = emit_do_callback_return(dcontext, pc, code->fcache_return, thread_shared);
+    if (DYNAMO_OPTION(shared_syscalls)) {
+
+        ibl_code_t *ibl_code;
+
+        if (DYNAMO_OPTION(disable_traces)) {
+            ibl_code = DYNAMO_OPTION(shared_bbs) ?
+                &SHARED_GENCODE(code->x86_mode)->bb_ibl[IBL_SHARED_SYSCALL] :
+                &code->bb_ibl[IBL_SHARED_SYSCALL];
+        }
+        else if (DYNAMO_OPTION(shared_traces)) {
+            ibl_code = &SHARED_GENCODE(code->x86_mode)->trace_ibl[IBL_SHARED_SYSCALL];
+        }
+        else {
+            ibl_code = &code->trace_ibl[IBL_SHARED_SYSCALL];
+        }
+
+        pc = check_size_and_cache_line(code, pc);
+        code->unlinked_shared_syscall = pc;
+        pc = emit_shared_syscall(dcontext, code, pc,
+                                 &code->shared_syscall_code,
+                                 &code->shared_syscall_code.ibl_patch,
+                                 ibl_code->indirect_branch_lookup_routine,
+                                 ibl_code->unlinked_ibl_entry,
+                                 !DYNAMO_OPTION(disable_traces), /* target_trace_table */
+                                 /* Only a single copy of shared syscall is
+                                  * emitted and afterwards it performs an IBL.
+                                  * Since both traces and BBs execute shared
+                                  * syscall (when trace building isn't disabled),
+                                  * we can't target the trace IBT table; otherwise,
+                                  * we'd miss marking secondary trace heads after
+                                  * a post-trace IBL misses. More comments are
+                                  * co-located with emit_shared_syscall().
+                                  */
+                                 DYNAMO_OPTION(disable_traces) ?
+                                 DYNAMO_OPTION(inline_bb_ibl) :
+                                 DYNAMO_OPTION(inline_trace_ibl), /* inline_ibl_head */
+                                 ibl_code->thread_shared_routine, /* thread_shared */
+                                 &code->shared_syscall);
+        code->end_shared_syscall = pc;
+        /* Lookup at end of shared_syscall should be able to go to bb or trace,
+         * unrestricted (will never be an exit from a trace so no secondary trace
+         * restrictions) -- currently only traces supported so using the trace_ibl
+         * is OK.
+         */
+    }
+    pc = check_size_and_cache_line(code, pc);
+    code->do_syscall = pc;
+    pc = emit_do_syscall(dcontext, code, pc, code->fcache_return, thread_shared,
+                         false, &code->do_syscall_offs);
+#else /* LINUX */
+    pc = check_size_and_cache_line(code, pc);
+    code->do_syscall = pc;
+    pc = emit_do_syscall(dcontext, code, pc, code->fcache_return, thread_shared,
+                         false, &code->do_syscall_offs);
+    pc = check_size_and_cache_line(code, pc);
+    code->do_int_syscall = pc;
+    pc = emit_do_syscall(dcontext, code, pc, code->fcache_return, thread_shared,
+                         true/*force int*/, &code->do_int_syscall_offs);
+    pc = check_size_and_cache_line(code, pc);
+    code->do_clone_syscall = pc;
+    pc = emit_do_clone_syscall(dcontext, code, pc, code->fcache_return, thread_shared,
+                               &code->do_clone_syscall_offs);
+#endif /* LINUX */
+    
+    return pc;
+}
+
+void
+arch_thread_init(dcontext_t *dcontext)
+{
+    byte *pc;
+    generated_code_t *code;
+    ibl_branch_type_t branch_type;
+
+    /* Simplest to have a real dcontext for emitting the selfmod code
+     * and finding the patch offsets so we do it on 1st thread init */
+    static bool selfmod_init = false;
+    if (!selfmod_init) {
+        ASSERT(!dynamo_initialized); /* .data +w */
+        selfmod_init = true;
+        set_selfmod_sandbox_offsets(dcontext);
+    }
+
+    ASSERT_CURIOSITY(proc_is_cache_aligned(get_local_state())
+                     IF_WINDOWS(|| DYNAMO_OPTION(tls_align != 0)));
+
+#ifdef X64
+    /* PR 244737: thread-private uses only shared gencode on x64 */
+    ASSERT(dcontext->private_code == NULL);
+    return;
+#endif
+
+    /* For detach on windows need to use a seperate mmap so we can leave this
+     * memory around in case of outstanding callbacks when we detach.  Without
+     * detach or on linux could just use one of our heaps (which would save
+     * a little space, (would then need to coordinate with arch_thread_exit)
+     */
+    ASSERT(GENCODE_COMMIT_SIZE < GENCODE_RESERVE_SIZE);
+    /* case 9474; share allocation unit w/ thread-private stack */
+    code = heap_mmap_reserve_post_stack(dcontext,
+                                        GENCODE_RESERVE_SIZE, GENCODE_COMMIT_SIZE);
+    ASSERT(code != NULL);
+    /* FIXME case 6493: if we split private from shared, remove this
+     * memset since we will no longer have a bunch of fields we don't use
+     */
+    memset(code, 0, sizeof(*code));
+    /* Generated code immediately follows struct */
+    code->gen_start_pc = ((byte *)code) + sizeof(*code);
+    code->commit_end_pc = ((byte *)code) + GENCODE_COMMIT_SIZE;
+    for (branch_type = IBL_BRANCH_TYPE_START; 
+         branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+        code->trace_ibl[branch_type].initialized = false;
+        code->bb_ibl[branch_type].initialized = false;
+        code->coarse_ibl[branch_type].initialized = false;
+    }
+
+    dcontext->private_code = (void *) code;
+
+    pc = code->gen_start_pc;
+    pc = check_size_and_cache_line(code, pc);
+    code->fcache_enter = pc;
+    pc = emit_fcache_enter(dcontext, code, pc);
+    pc = check_size_and_cache_line(code, pc);
+    code->fcache_return = pc;
+    pc = emit_fcache_return(dcontext, code, pc);;
+#ifdef WINDOWS_PC_SAMPLE
+    code->fcache_enter_return_end = pc;
+#endif
+
+    /* Currently all ibl routines target the trace hashtable 
+       and we don't yet support basic blocks as targets of an IBL.
+       However, having separate routines at least enables finer control
+       over the indirect exit stubs.  
+       This way we have inlined IBL stubs for trace but not in basic blocks.
+
+       TODO: After separating the IBL routines, now we can retarget them to separate
+       hashtables (or alternatively chain several IBL routines together).
+       From trace ib exits we can only go to {traces}, so no change here.
+         (when we exit to a basic block we need to mark as a trace head)
+       From basic block ib exits we should be able to go to {traces + bbs - traceheads}
+          (for the tracehead bbs we actually have to increment counters.
+       From shared_syscall we should be able to go to {traces + bbs}.
+
+       TODO: we also want to have separate routines per indirect branch types to enable
+       the restricted control transfer policies to be efficiently enforced.
+    */
+    if (!DYNAMO_OPTION(disable_traces) && DYNAMO_OPTION(shared_trace_ibl_routine)) {
+        if (!DYNAMO_OPTION(shared_traces)) {
+            /* copy all bookkeeping information from shared_code into thread private 
+               needed by get_ibl_routine*() */
+            ibl_branch_type_t branch_type;
+            for (branch_type = IBL_BRANCH_TYPE_START; 
+                 branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+                code->trace_ibl[branch_type] =
+                    SHARED_GENCODE(code->x86_mode)->trace_ibl[branch_type];
+            }
+        } /* FIXME: no private traces supported right now w/ -shared_traces */
+    } else if (PRIVATE_TRACES_ENABLED()) {
+        /* shared_trace_ibl_routine should be false for private (performance test only) */
+        pc = emit_ibl_routines(dcontext, code, pc, code->fcache_return, 
+                               IBL_TRACE_PRIVATE, /* source_fragment_type */
+                               DYNAMO_OPTION(shared_trace_ibl_routine), /* thread_shared */
+                               true, /* target_trace_table */
+                               code->trace_ibl);
+    }
+    pc = emit_ibl_routines(dcontext, code, pc, code->fcache_return, 
+                           IBL_BB_PRIVATE, /* source_fragment_type */
+                           /* need thread-private for selfmod regardless of sharing */
+                           false, /* thread_shared */
+                           !DYNAMO_OPTION(bb_ibl_targets), /* target_trace_table */
+                           code->bb_ibl);
+#ifdef WINDOWS_PC_SAMPLE
+    code->ibl_routines_end = pc;
+#endif
+
+#if defined(LINUX) && !defined(HAVE_TLS)
+    /* for HAVE_TLS we use the shared version; w/o TLS we don't
+     * make any shared routines (PR 361894)
+     */
+    /* must create before emit_do_clone_syscall() in emit_syscall_routines() */
+    pc = check_size_and_cache_line(code, pc);
+    code->new_thread_dynamo_start = pc;
+    pc = emit_new_thread_dynamo_start(dcontext, pc);
+#endif
+
+#ifdef RETURN_STACK
+    /* unlinked_return comes first */
+    pc = check_size_and_cache_line(code, pc);
+    code->return_lookup = pc;
+    pc = emit_return_lookup(dcontext, pc,
+                            code->indirect_branch_lookup,
+                            code->unlinked_ib_lookup,
+                            &code->unlinked_return);
+#endif
+#ifdef WINDOWS
+    pc = check_size_and_cache_line(code, pc);
+    code->fcache_enter_indirect = pc;
+    pc = emit_fcache_enter_indirect(dcontext, code, pc, code->fcache_return);
+#endif
+    pc = emit_syscall_routines(dcontext, code, pc, false/*thread-private*/);
+#ifdef TRACE_HEAD_CACHE_INCR
+    pc = check_size_and_cache_line(code, pc);
+    code->trace_head_incr = pc;
+    pc = emit_trace_head_incr(dcontext, pc, code->fcache_return);
+#endif
+#ifdef CHECK_RETURNS_SSE2_EMIT
+    /* PR 248210: unsupported feature on x64: need to move to thread-shared gencode
+     * if want to support it.
+     */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    pc = check_size_and_cache_line(code, pc);
+    code->pextrw = pc;
+    pc = emit_pextrw(dcontext, pc);
+    pc = check_size_and_cache_line(code, pc);
+    code->pinsrw = pc;
+    pc = emit_pinsrw(dcontext, pc);
+#endif
+    code->reset_exit_stub = pc;
+    /* reset exit stub should look just like a direct exit stub */
+    pc += insert_exit_stub_other_flags(dcontext,
+                                       linkstub_fragment(dcontext, (linkstub_t *)
+                                                         get_reset_linkstub()),
+                                       (linkstub_t *) get_reset_linkstub(),
+                                       pc, LINK_DIRECT);
+    ASSERT(pc < code->commit_end_pc);
+    code->gen_end_pc = pc;
+    release_final_page(code);
+
+    DOLOG(3, LOG_EMIT, {
+        dump_emitted_routines(dcontext, THREAD, "thread-private", code, pc);
+    });
+#ifdef INTERNAL
+    if (INTERNAL_OPTION(gendump)) {
+        dump_emitted_routines_to_file(dcontext, "gencode-private", "thread-private",
+                                      code, pc);
+    }
+#endif
+#ifdef WINDOWS_PC_SAMPLE
+    if (dynamo_options.profile_pcs && dynamo_options.prof_pcs_gencode >= 2 && 
+        dynamo_options.prof_pcs_gencode <= 32) {
+        code->profile = create_profile(code->gen_start_pc, pc, 
+                                       dynamo_options.prof_pcs_gencode, NULL);
+        start_profile(code->profile);
+    } else 
+        code->profile = NULL;
+#endif
+
+    code->writable = true;
+    /* For SELFPROT_GENCODE we don't make unwritable until after we patch,
+     * though for hotp_only we don't patch.
+     */
+#ifdef HOT_PATCHING_INTERFACE
+    if (DYNAMO_OPTION(hotp_only)) 
+#endif
+        protect_generated_code(code, READONLY);
+}
+
+#ifdef WINDOWS_PC_SAMPLE
+void
+arch_thread_profile_exit(dcontext_t *dcontext)
+{
+    arch_extract_profile(dcontext _IF_X64(GENCODE_FROM_DCONTEXT));
+}
+#endif
+
+void
+arch_thread_exit(dcontext_t *dcontext _IF_WINDOWS(bool detach_stacked_callbacks))
+{
+#ifdef X64
+    /* PR 244737: thread-private uses only shared gencode on x64 */
+    ASSERT(dcontext->private_code == NULL);
+    return;
+#endif
+    /* We only need to unprotect private_code for profile extraction
+     * so we do it there to also cover the fast exit path.
+     * Also note that for detach w/ stacked callbacks arch_patch_syscall()
+     * will have already unprotected.
+     */
+#ifdef WINDOWS
+    if (!detach_stacked_callbacks && !DYNAMO_OPTION(thin_client)) {
+#endif
+        /* ensure we didn't miss the init patch and leave it writable! */
+        ASSERT(!TEST(SELFPROT_GENCODE, DYNAMO_OPTION(protect_mask)) ||
+               !((generated_code_t *)dcontext->private_code)->writable);
+#ifdef WINDOWS
+    }
+#endif
+#ifdef WINDOWS_PC_SAMPLE
+    arch_thread_profile_exit(dcontext);
+#endif
+#ifdef WINDOWS
+    if (!detach_stacked_callbacks)
+#endif
+        heap_munmap_post_stack(dcontext, dcontext->private_code, GENCODE_RESERVE_SIZE);
+}
+
+#ifdef WINDOWS
+/* Patch syscall routines for detach */
+static void
+arch_patch_syscall_common(dcontext_t *dcontext, byte *target _IF_X64(gencode_mode_t mode))
+{
+    generated_code_t *code = get_emitted_routines_code(dcontext _IF_X64(mode));
+    if (code != NULL && (!is_shared_gencode(code) || dcontext == GLOBAL_DCONTEXT)) {
+        /* ensure we didn't miss the init patch and leave it writable! */
+        ASSERT(!TEST(SELFPROT_GENCODE, DYNAMO_OPTION(protect_mask)) || !code->writable);
+        /* this is only done for detach, so no need to re-protect */
+        protect_generated_code(code, WRITABLE);
+        emit_patch_syscall(dcontext, target _IF_X64(mode));
+    }
+}
+void
+arch_patch_syscall(dcontext_t *dcontext, byte *target)
+{
+    if (dcontext == GLOBAL_DCONTEXT) {
+        arch_patch_syscall_common(GLOBAL_DCONTEXT, target _IF_X64(GENCODE_X64));
+        IF_X64(arch_patch_syscall_common(GLOBAL_DCONTEXT, target _IF_X64(GENCODE_X86)));
+    } else
+        arch_patch_syscall_common(GLOBAL_DCONTEXT, target _IF_X64(GENCODE_FROM_DCONTEXT));
+}
+#endif
+
+void 
+update_generated_hashtable_access(dcontext_t *dcontext)
+{
+    update_indirect_branch_lookup(dcontext);
+}
+
+void
+protect_generated_code(generated_code_t *code, bool writable)
+{
+    if (TEST(SELFPROT_GENCODE, DYNAMO_OPTION(protect_mask)) &&
+        code->writable != writable) {
+        byte *genstart = (byte *)PAGE_START(code->gen_start_pc);
+        if (!writable) {
+            ASSERT(code->writable);
+            code->writable = writable;
+        }
+        STATS_INC(gencode_prot_changes);
+        change_protection(genstart, code->commit_end_pc - genstart,
+                          writable);
+        if (writable) {
+            ASSERT(!code->writable);
+            code->writable = writable;
+        }
+    }
+}
+
+ibl_source_fragment_type_t
+get_source_fragment_type(dcontext_t *dcontext, uint fragment_flags)
+{
+    if (TEST(FRAG_IS_TRACE, fragment_flags)) {
+        return (TEST(FRAG_SHARED, fragment_flags)) ? IBL_TRACE_SHARED : IBL_TRACE_PRIVATE;
+    } else if (TEST(FRAG_COARSE_GRAIN, fragment_flags)) {
+        ASSERT(TEST(FRAG_SHARED, fragment_flags));
+        return IBL_COARSE_SHARED;
+    } else {
+        return (TEST(FRAG_SHARED, fragment_flags)) ? IBL_BB_SHARED : IBL_BB_PRIVATE;
+    }
+}
+
+#ifdef WINDOWS
+bool 
+is_shared_syscall_routine(dcontext_t *dcontext, cache_pc pc)
+{
+    if (DYNAMO_OPTION(shared_fragment_shared_syscalls)) {
+        return (pc == (cache_pc) shared_code->shared_syscall
+                || pc == (cache_pc) shared_code->unlinked_shared_syscall)
+            IF_X64(|| (shared_code_x86 != NULL &&
+                       (pc == (cache_pc) shared_code_x86->shared_syscall
+                        || pc == (cache_pc) shared_code_x86->unlinked_shared_syscall)));
+    }
+    else {
+        generated_code_t *code = THREAD_GENCODE(dcontext);
+        return (code != NULL && (pc == (cache_pc) code->shared_syscall
+                                 || pc == (cache_pc) code->unlinked_shared_syscall));
+    }
+}
+#endif
+
+bool 
+is_indirect_branch_lookup_routine(dcontext_t *dcontext, cache_pc pc)
+{
+#ifdef RETURN_STACK
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+    if (pc == (cache_pc) code->return_lookup
+        || pc == (cache_pc) code->unlinked_return)
+        return true;
+#endif /* RETURN_STACK */
+    
+#ifdef WINDOWS
+    if (is_shared_syscall_routine(dcontext, pc))
+        return true;
+#endif
+    /* we only care if it is found */
+    return get_ibl_routine_type_ex(dcontext, pc, NULL _IF_X64(NULL));
+}
+
+/* Promotes the current ibl routine from IBL_BB* to IBL_TRACE* preserving other properties */
+/* There seems to be no need for the opposite transformation */
+cache_pc
+get_trace_ibl_routine(dcontext_t *dcontext, cache_pc current_entry)
+{
+    ibl_type_t ibl_type = {0};
+    DEBUG_DECLARE(bool is_ibl = )
+        get_ibl_routine_type(dcontext, current_entry, &ibl_type);
+
+    ASSERT(is_ibl);
+    ASSERT(IS_IBL_BB(ibl_type.source_fragment_type));
+    
+    return
+#ifdef WINDOWS
+        DYNAMO_OPTION(shared_syscalls) &&
+        is_shared_syscall_routine(dcontext, current_entry) ? current_entry :
+#endif
+        get_ibl_routine(dcontext, ibl_type.link_state, 
+                        (ibl_type.source_fragment_type == IBL_BB_PRIVATE) ?
+                        IBL_TRACE_PRIVATE : IBL_TRACE_SHARED,
+                        ibl_type.branch_type);
+}
+
+/* Shifts the current ibl routine from IBL_BB_SHARED to IBL_BB_PRIVATE,
+ * preserving other properties.
+ * There seems to be no need for the opposite transformation 
+ */
+cache_pc
+get_private_ibl_routine(dcontext_t *dcontext, cache_pc current_entry)
+{
+    ibl_type_t ibl_type = {0};
+    DEBUG_DECLARE(bool is_ibl = )
+        get_ibl_routine_type(dcontext, current_entry, &ibl_type);
+
+    ASSERT(is_ibl);
+    ASSERT(IS_IBL_BB(ibl_type.source_fragment_type));
+    
+    return get_ibl_routine(dcontext, ibl_type.link_state, 
+                           IBL_BB_PRIVATE, ibl_type.branch_type);
+}
+
+/* Shifts the current ibl routine from IBL_BB_PRIVATE to IBL_BB_SHARED,
+ * preserving other properties.
+ * There seems to be no need for the opposite transformation 
+ */
+cache_pc
+get_shared_ibl_routine(dcontext_t *dcontext, cache_pc current_entry)
+{
+    ibl_type_t ibl_type = {0};
+    DEBUG_DECLARE(bool is_ibl = )
+        get_ibl_routine_type(dcontext, current_entry, &ibl_type);
+
+    ASSERT(is_ibl);
+    ASSERT(IS_IBL_BB(ibl_type.source_fragment_type));
+    
+    return get_ibl_routine(dcontext, ibl_type.link_state, 
+                           IBL_BB_SHARED, ibl_type.branch_type);
+}
+
+/* gets the corresponding routine to current_entry but matching whether
+ * FRAG_IS_TRACE and FRAG_SHARED are set in flags
+ */
+cache_pc
+get_alternate_ibl_routine(dcontext_t *dcontext, cache_pc current_entry,
+                          uint flags)
+{
+    ibl_type_t ibl_type = {0};
+    IF_X64(gencode_mode_t mode = GENCODE_FROM_DCONTEXT;)
+    DEBUG_DECLARE(bool is_ibl = )
+        get_ibl_routine_type_ex(dcontext, current_entry, &ibl_type _IF_X64(&mode));
+    ASSERT(is_ibl);
+#ifdef WINDOWS
+    /* shared_syscalls does not change currently
+     * FIXME: once we support targeting both private and shared syscall
+     * we will need to change sharing here
+     */
+    if (DYNAMO_OPTION(shared_syscalls) &&
+        is_shared_syscall_routine(dcontext, current_entry))
+        return current_entry;
+#endif
+    return get_ibl_routine_ex(dcontext, ibl_type.link_state, 
+                              get_source_fragment_type(dcontext, flags),
+                              ibl_type.branch_type _IF_X64(mode));
+}
+
+cache_pc
+get_linked_entry(dcontext_t *dcontext, cache_pc unlinked_entry)
+{
+    ibl_type_t ibl_type = {0};
+    IF_X64(gencode_mode_t mode = GENCODE_FROM_DCONTEXT;)
+    DEBUG_DECLARE(bool is_ibl = )
+        get_ibl_routine_type_ex(dcontext, unlinked_entry, &ibl_type _IF_X64(&mode));
+    ASSERT(is_ibl && IS_IBL_UNLINKED(ibl_type.link_state));
+
+#ifdef WINDOWS
+    if (unlinked_entry == unlinked_shared_syscall_routine_ex(dcontext _IF_X64(mode))) {
+        return shared_syscall_routine_ex(dcontext _IF_X64(mode));
+    }
+#endif
+
+    return get_ibl_routine_ex(dcontext,
+                              /* for -unsafe_ignore_eflags_{ibl,trace} the trace cmp
+                               * entry and unlink are both identical, so we may mix
+                               * them up but will have no problems */
+                              IF_X64(ibl_type.link_state == IBL_TRACE_CMP_UNLINKED ?
+                                     IBL_TRACE_CMP :) IBL_LINKED, 
+                              ibl_type.source_fragment_type, ibl_type.branch_type
+                              _IF_X64(mode));
+}
+
+#ifdef X64
+cache_pc
+get_trace_cmp_entry(dcontext_t *dcontext, cache_pc linked_entry)
+{
+    ibl_type_t ibl_type = {0};
+    DEBUG_DECLARE(bool is_ibl = )
+        get_ibl_routine_type(dcontext, linked_entry, &ibl_type);
+    IF_WINDOWS(ASSERT(linked_entry != shared_syscall_routine(dcontext)));
+    ASSERT(is_ibl && ibl_type.link_state == IBL_LINKED);
+    return get_ibl_routine(dcontext, IBL_TRACE_CMP, 
+                           ibl_type.source_fragment_type, ibl_type.branch_type);
+}
+#endif
+
+cache_pc
+get_unlinked_entry(dcontext_t *dcontext, cache_pc linked_entry)
+{
+    ibl_type_t ibl_type = {0};
+    IF_X64(gencode_mode_t mode = GENCODE_FROM_DCONTEXT;)
+    DEBUG_DECLARE(bool is_ibl = )
+        get_ibl_routine_type_ex(dcontext, linked_entry, &ibl_type _IF_X64(&mode));
+    ASSERT(is_ibl && IS_IBL_LINKED(ibl_type.link_state));
+
+#ifdef WINDOWS
+    if (linked_entry == shared_syscall_routine_ex(dcontext _IF_X64(mode)))
+        return unlinked_shared_syscall_routine_ex(dcontext _IF_X64(mode));
+#endif
+    return get_ibl_routine_ex(dcontext,
+                              IF_X64(ibl_type.link_state == IBL_TRACE_CMP ?
+                                     IBL_TRACE_CMP_UNLINKED :) IBL_UNLINKED, 
+                              ibl_type.source_fragment_type, ibl_type.branch_type
+                              _IF_X64(mode));
+}
+
+static bool
+in_generated_shared_routine(dcontext_t *dcontext, cache_pc pc)
+{
+    if (USE_SHARED_GENCODE()) {
+        return (pc >= (cache_pc)(shared_code->gen_start_pc) &&
+                pc < (cache_pc)(shared_code->commit_end_pc))
+            IF_X64(|| (shared_code_x86 != NULL &&
+                       pc >= (cache_pc)(shared_code_x86->gen_start_pc) &&
+                       pc < (cache_pc)(shared_code_x86->commit_end_pc)))
+            ;
+    }
+    return false;
+}
+
+bool
+in_generated_routine(dcontext_t *dcontext, cache_pc pc)
+{
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+
+    return ((pc >= (cache_pc)(code->gen_start_pc) &&
+            pc < (cache_pc)(code->commit_end_pc))
+            || in_generated_shared_routine(dcontext, pc));
+    /* FIXME: what about inlined IBL stubs */
+}
+
+bool 
+in_context_switch_code(dcontext_t *dcontext, cache_pc pc)
+{
+    return (pc >= (cache_pc)fcache_enter_routine(dcontext) &&
+            /* get last emitted routine */
+            pc <= get_ibl_routine(dcontext, IBL_LINKED, IBL_SOURCE_TYPE_END-1,
+                                  IBL_BRANCH_TYPE_START));
+    /* FIXME: too hacky, should have an extra field for PC profiling */
+}
+
+bool
+in_indirect_branch_lookup_code(dcontext_t *dcontext, cache_pc pc)
+{
+    ibl_source_fragment_type_t source_fragment_type;
+    ibl_branch_type_t branch_type;
+
+    for (source_fragment_type = IBL_SOURCE_TYPE_START; 
+         source_fragment_type < IBL_SOURCE_TYPE_END; 
+         source_fragment_type++) {
+        for (branch_type = IBL_BRANCH_TYPE_START; 
+             branch_type < IBL_BRANCH_TYPE_END; 
+             branch_type++) {
+            if (pc >= get_ibl_routine(dcontext, IBL_LINKED, source_fragment_type, branch_type) &&
+                pc <  get_ibl_routine(dcontext, IBL_UNLINKED, source_fragment_type, branch_type)) 
+                return true;
+        }
+    }
+    return false;                /* not an IBL */
+    /* FIXME: what about inlined IBL stubs */
+}
+
+fcache_enter_func_t
+fcache_enter_routine(dcontext_t *dcontext)
+{
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+    return (fcache_enter_func_t) convert_data_to_function(code->fcache_enter);
+}
+
+/* exported to dispatch.c */
+fcache_enter_func_t 
+get_fcache_enter_private_routine(dcontext_t *dcontext)
+{
+    return fcache_enter_routine(dcontext);
+}
+
+cache_pc
+get_reset_exit_stub(dcontext_t *dcontext)
+{
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+    return (cache_pc) code->reset_exit_stub;
+}
+
+cache_pc
+get_do_syscall_entry(dcontext_t *dcontext)
+{
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+    return (cache_pc) code->do_syscall;
+}
+
+#ifdef WINDOWS
+fcache_enter_func_t
+get_fcache_enter_indirect_routine(dcontext_t *dcontext)
+{
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+    return (fcache_enter_func_t) convert_data_to_function(code->fcache_enter_indirect);
+}
+cache_pc
+get_do_callback_return_entry(dcontext_t *dcontext)
+{
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+    return (cache_pc) code->do_callback_return;
+}
+#else
+/* PR 286922: we need an int syscall even when vsyscall is sys{call,enter} */
+cache_pc
+get_do_int_syscall_entry(dcontext_t *dcontext)
+{
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+    return (cache_pc) code->do_int_syscall;
+}
+
+cache_pc
+get_do_clone_syscall_entry(dcontext_t *dcontext)
+{
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+    return (cache_pc) code->do_clone_syscall;
+}
+#endif
+
+cache_pc
+fcache_return_routine(dcontext_t *dcontext)
+{
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+    return (cache_pc) code->fcache_return;
+}
+
+cache_pc
+fcache_return_routine_ex(dcontext_t *dcontext _IF_X64(gencode_mode_t mode))
+{
+    generated_code_t *code = get_emitted_routines_code(dcontext _IF_X64(mode));
+    return (cache_pc) code->fcache_return;
+}
+
+cache_pc
+fcache_return_coarse_routine(IF_X64_ELSE(gencode_mode_t mode, void))
+{
+    generated_code_t *code = get_shared_gencode(GLOBAL_DCONTEXT _IF_X64(mode));
+    ASSERT(DYNAMO_OPTION(coarse_units));
+    if (code == NULL)
+        return NULL;
+    else
+        return (cache_pc) code->fcache_return_coarse;
+}
+
+cache_pc
+trace_head_return_coarse_routine(IF_X64_ELSE(gencode_mode_t mode, void))
+{
+    generated_code_t *code = get_shared_gencode(GLOBAL_DCONTEXT _IF_X64(mode));
+    ASSERT(DYNAMO_OPTION(coarse_units));
+    if (code == NULL)
+        return NULL;
+    else
+        return (cache_pc) code->trace_head_return_coarse;
+}
+
+/* returns false if target is not an IBL routine.
+ * if type is not NULL it is set to the type of the found routine.
+ * if mode_out is NULL, dcontext cannot be GLOBAL_DCONTEXT.
+ * if mode_out is not NULL, it is set to which mode the found routine is in.
+ */
+bool
+get_ibl_routine_type_ex(dcontext_t *dcontext, cache_pc target, ibl_type_t *type
+                        _IF_X64(gencode_mode_t *mode_out))
+{
+    ibl_entry_point_type_t link_state;
+    ibl_source_fragment_type_t source_fragment_type;
+    ibl_branch_type_t branch_type;
+#ifdef X64
+    gencode_mode_t mode;
+#endif
+
+    /* An up-front range check. Many calls into this routine are with addresses
+     * outside of the IBL code or the generated_code_t in which IBL resides.
+     * For all of those cases, this quick up-front check saves the expense of
+     * examining all of the different IBL entry points.
+     */
+    if ((shared_code == NULL ||
+         target < shared_code->gen_start_pc ||
+         target >= shared_code->gen_end_pc)
+        IF_X64(&& (shared_code_x86 == NULL ||
+                   target < shared_code_x86->gen_start_pc ||
+                   target >= shared_code_x86->gen_end_pc))) {
+        if (dcontext == GLOBAL_DCONTEXT ||
+            /* PR 244737: thread-private uses shared gencode on x64 */
+            IF_X64(true ||)
+            target < ((generated_code_t *)dcontext->private_code)->gen_start_pc ||
+            target >= ((generated_code_t *)dcontext->private_code)->gen_end_pc)
+            return false;
+    }
+
+    /* a decent compiler should inline these nested loops */
+    /* iterate in order <linked, unlinked> */
+    for (link_state = IBL_LINKED; 
+         /* keep in mind we need a signed comparison when going downwards */
+         (int)link_state >= (int)IBL_UNLINKED; link_state-- ) { 
+        /* it is OK to compare to IBL_BB_PRIVATE even when !SHARED_FRAGMENTS_ENABLED() */
+        for (source_fragment_type = IBL_SOURCE_TYPE_START; 
+             source_fragment_type < IBL_SOURCE_TYPE_END; 
+             source_fragment_type++) {
+            for (branch_type = IBL_BRANCH_TYPE_START; 
+                 branch_type < IBL_BRANCH_TYPE_END; 
+                 branch_type++) {
+#ifdef X64
+                for (mode = GENCODE_X64; mode <= GENCODE_X86; mode++) {
+#endif
+                    if (target == get_ibl_routine_ex(dcontext, link_state,
+                                                     source_fragment_type,
+                                                     branch_type _IF_X64(mode))) {
+                        if (type) {
+                            type->link_state = link_state;
+                            type->source_fragment_type = source_fragment_type;
+                            type->branch_type = branch_type;
+                        }
+#ifdef X64
+                        if (mode_out != NULL)
+                            *mode_out = mode;
+#endif
+                        return true;
+                    }
+#ifdef X64
+                }
+#endif
+            }
+        }
+    }
+#ifdef WINDOWS
+    if (is_shared_syscall_routine(dcontext, target)) {
+        if (type != NULL) {
+            type->branch_type = IBL_SHARED_SYSCALL;
+            type->source_fragment_type = DEFAULT_IBL_BB();
+#ifdef X64
+            for (mode = GENCODE_X64; mode <= GENCODE_X86; mode++) {
+#endif
+                if (target == unlinked_shared_syscall_routine_ex(dcontext _IF_X64(mode)))
+                    type->link_state = IBL_UNLINKED;
+                else IF_X64(if (target ==
+                                shared_syscall_routine_ex(dcontext _IF_X64(mode))))
+                    type->link_state = IBL_LINKED;
+#ifdef X64
+                else
+                    continue;
+                if (mode_out != NULL)
+                    *mode_out = mode;
+                break;
+            }
+#endif
+        }        
+        return true;
+    }
+#endif
+
+    return false;                /* not an IBL */
+}
+
+bool
+get_ibl_routine_type(dcontext_t *dcontext, cache_pc target, ibl_type_t *type)
+{
+    IF_X64(ASSERT(dcontext != GLOBAL_DCONTEXT)); /* should call get_ibl_routine_type_ex */
+    return get_ibl_routine_type_ex(dcontext, target, type _IF_X64(NULL));
+}
+
+/* returns false if target is not an IBL template
+   if type is not NULL it is set to the type of the found routine 
+*/
+static bool
+get_ibl_routine_template_type(dcontext_t *dcontext, cache_pc target, ibl_type_t *type
+                              _IF_X64(gencode_mode_t *mode_out))
+{
+    ibl_source_fragment_type_t source_fragment_type;
+    ibl_branch_type_t branch_type;
+#ifdef X64
+    gencode_mode_t mode;
+#endif
+
+    for (source_fragment_type = IBL_SOURCE_TYPE_START; 
+         source_fragment_type < IBL_SOURCE_TYPE_END; 
+         source_fragment_type++) {
+        for (branch_type = IBL_BRANCH_TYPE_START; 
+             branch_type < IBL_BRANCH_TYPE_END; 
+             branch_type++) {
+#ifdef X64
+            for (mode = GENCODE_X64; mode <= GENCODE_X86; mode++) {
+#endif
+                if (target == get_ibl_routine_template(dcontext, source_fragment_type,
+                                                       branch_type _IF_X64(mode))) {
+                    if (type) {
+                        type->link_state = IBL_TEMPLATE;
+                        type->source_fragment_type = source_fragment_type;
+                        type->branch_type = branch_type;
+#ifdef X64
+                        if (mode_out != NULL)
+                            *mode_out = mode;
+#endif
+                    }
+                    return true;
+#ifdef X64
+                }
+#endif
+            }
+        }
+    }
+    return false;                /* not an IBL template */
+}
+
+const char *
+get_branch_type_name(ibl_branch_type_t branch_type)
+{
+    static const char *const ibl_brtype_names[IBL_BRANCH_TYPE_END] = 
+        {"ret", "indcall", "indjmp"};
+    return ibl_brtype_names[branch_type];
+}
+
+ibl_branch_type_t
+get_ibl_branch_type(instr_t *instr)
+{
+    ASSERT(instr_is_mbr(instr));
+
+    if (instr_is_return(instr))
+        return IBL_RETURN;
+    else if (instr_is_call_indirect(instr))
+        return IBL_INDCALL;
+    else
+        return IBL_INDJMP;
+}
+
+
+/* returns a symbolic name if target is an IBL routine or an IBL template,
+ * otherwise returns NULL 
+ */
+const char *
+get_ibl_routine_name(dcontext_t *dcontext, cache_pc target, const char **ibl_brtype_name)
+{
+    static const char *const
+        ibl_routine_names IF_X64([2]) [IBL_SOURCE_TYPE_END][IBL_LINK_STATE_END] = {
+        IF_X64({)
+        {"shared_unlinked_bb_ibl", "shared_delete_bb_ibl",
+         IF_X64_("shared_bb_cmp") IF_X64_("shared_bb_cmp_unlinked")
+         "shared_bb_ibl", "shared_bb_ibl_template"},
+        {"shared_unlinked_trace_ibl", "shared_delete_trace_ibl",
+         IF_X64_("shared_trace_cmp") IF_X64_("shared_trace_cmp_unlinked")
+         "shared_trace_ibl", "shared_trace_ibl_template"},
+        {"private_unlinked_bb_ibl", "private_delete_bb_ibl",
+         IF_X64_("private_bb_cmp") IF_X64_("private_bb_cmp_unlinked")
+         "private_bb_ibl", "private_bb_ibl_template"},
+        {"private_unlinked_trace_ibl", "private_delete_trace_ibl",
+         IF_X64_("private_trace_cmp") IF_X64_("private_trace_cmp_unlinked")
+         "private_trace_ibl", "private_trace_ibl_template"},
+        {"shared_unlinked_coarse_ibl", "shared_delete_coarse_ibl",
+         IF_X64_("shared_coarse_trace_cmp") IF_X64_("shared_coarse_trace_cmp_unlinked")
+         "shared_coarse_ibl", "shared_coarse_ibl_template"},
+#ifdef X64
+        /* PR 282576: for WOW64 processes we have separate x86 routines */
+        }, {
+        {"x86_shared_unlinked_bb_ibl", "x86_shared_delete_bb_ibl",
+         IF_X64_("x86_shared_bb_cmp") IF_X64_("x86_shared_bb_cmp_unlinked")
+         "x86_shared_bb_ibl", "x86_shared_bb_ibl_template"},
+        {"x86_shared_unlinked_trace_ibl", "x86_shared_delete_trace_ibl",
+         IF_X64_("x86_shared_trace_cmp") IF_X64_("x86_shared_trace_cmp_unlinked")
+         "x86_shared_trace_ibl", "x86_shared_trace_ibl_template"},
+        {"x86_private_unlinked_bb_ibl", "x86_private_delete_bb_ibl",
+         IF_X64_("x86_private_bb_cmp") IF_X64_("x86_private_bb_cmp_unlinked")
+         "x86_private_bb_ibl", "x86_private_bb_ibl_template"},
+        {"x86_private_unlinked_trace_ibl", "x86_private_delete_trace_ibl",
+         IF_X64_("x86_private_trace_cmp") IF_X64_("x86_private_trace_cmp_unlinked")
+         "x86_private_trace_ibl", "x86_private_trace_ibl_template"},
+        {"x86_shared_unlinked_coarse_ibl", "x86_shared_delete_coarse_ibl",
+         IF_X64_("x86_shared_coarse_trace_cmp")
+         IF_X64_("x86_shared_coarse_trace_cmp_unlinked")
+         "x86_shared_coarse_ibl", "x86_shared_coarse_ibl_template"},
+        }
+#endif
+    };
+    ibl_type_t ibl_type;
+#ifdef X64
+    gencode_mode_t mode;
+#endif
+    if (!get_ibl_routine_type_ex(dcontext, target, &ibl_type _IF_X64(&mode))) {
+        /* not an IBL routine */
+        if (!get_ibl_routine_template_type(dcontext, target, &ibl_type _IF_X64(&mode))) {
+            return NULL;                /* not an IBL template either */
+        }
+    }
+    /* ibl_type is valid and will give routine or template name, and qualifier */
+    
+    *ibl_brtype_name = get_branch_type_name(ibl_type.branch_type);
+    return ibl_routine_names IF_X64([mode])
+        [ibl_type.source_fragment_type][ibl_type.link_state];
+}
+
+static inline
+ibl_code_t*
+get_ibl_routine_code_internal(dcontext_t *dcontext, 
+                              ibl_source_fragment_type_t source_fragment_type, 
+                              ibl_branch_type_t branch_type
+                              _IF_X64(gencode_mode_t mode))
+{
+#ifdef X64
+    if ((mode == GENCODE_X86 ||
+         (mode == GENCODE_FROM_DCONTEXT && dcontext != GLOBAL_DCONTEXT &&
+          dcontext->x86_mode)) &&
+        shared_code_x86 == NULL)
+        return NULL;
+#endif
+    switch (source_fragment_type) {
+    case IBL_BB_SHARED:
+        if (!DYNAMO_OPTION(shared_bbs))
+            return NULL;
+        return &(get_shared_gencode(dcontext _IF_X64(mode))->bb_ibl[branch_type]);
+    case IBL_BB_PRIVATE:
+        return &(get_emitted_routines_code(dcontext _IF_X64(mode))->bb_ibl[branch_type]);
+    case IBL_TRACE_SHARED: 
+        if (!DYNAMO_OPTION(shared_traces))
+            return NULL;
+        return &(get_shared_gencode(dcontext _IF_X64(mode))->trace_ibl[branch_type]);
+    case IBL_TRACE_PRIVATE:
+        return &(get_emitted_routines_code(dcontext _IF_X64(mode))
+                 ->trace_ibl[branch_type]);
+    case IBL_COARSE_SHARED:
+        if (!DYNAMO_OPTION(coarse_units))
+            return NULL;
+        return &(get_shared_gencode(dcontext _IF_X64(mode))->coarse_ibl[branch_type]);
+    default:
+        ASSERT_NOT_REACHED();
+    }
+    ASSERT_NOT_REACHED();
+    return NULL;
+}
+
+
+cache_pc 
+get_ibl_routine_ex(dcontext_t *dcontext, ibl_entry_point_type_t entry_type,
+                   ibl_source_fragment_type_t source_fragment_type,
+                   ibl_branch_type_t branch_type _IF_X64(gencode_mode_t mode))
+{
+    ibl_code_t *ibl_code = 
+        get_ibl_routine_code_internal(dcontext, 
+                                      source_fragment_type, branch_type _IF_X64(mode));
+    if (ibl_code == NULL || !ibl_code->initialized)
+        return NULL;
+    switch (entry_type) {
+    case IBL_LINKED:
+        return (cache_pc) ibl_code->indirect_branch_lookup_routine;
+    case IBL_UNLINKED:
+        return (cache_pc) ibl_code->unlinked_ibl_entry;
+    case IBL_DELETE:
+        return (cache_pc) ibl_code->target_delete_entry;
+#ifdef X64
+    case IBL_TRACE_CMP:
+        return (cache_pc) ibl_code->trace_cmp_entry;
+    case IBL_TRACE_CMP_UNLINKED:
+        return (cache_pc) ibl_code->trace_cmp_unlinked;
+#endif
+    default:
+        ASSERT_NOT_REACHED();
+    }
+    return NULL;
+}
+
+cache_pc 
+get_ibl_routine(dcontext_t *dcontext, ibl_entry_point_type_t entry_type,
+                ibl_source_fragment_type_t source_fragment_type,
+                ibl_branch_type_t branch_type)
+{
+    return get_ibl_routine_ex(dcontext, entry_type, source_fragment_type,
+                              branch_type _IF_X64(GENCODE_FROM_DCONTEXT));
+}
+
+cache_pc 
+get_ibl_routine_template(dcontext_t *dcontext, 
+                         ibl_source_fragment_type_t source_fragment_type, 
+                         ibl_branch_type_t branch_type
+                         _IF_X64(gencode_mode_t mode))
+{
+    ibl_code_t *ibl_code = get_ibl_routine_code_internal
+        (dcontext, source_fragment_type, branch_type _IF_X64(mode));
+    if (ibl_code == NULL || !ibl_code->initialized)
+        return NULL;
+    return ibl_code->inline_ibl_stub_template;
+}
+
+/* Convert FRAG_TABLE_* flags to FRAG_* flags */
+/* FIXME This seems more appropriate in fragment.c but since there's no
+ * need for the functionality there, we place it here and inline it. We
+ * can move it if other pieces need the functionality later.
+ */
+static inline uint
+table_flags_to_frag_flags(dcontext_t *dcontext, ibl_table_t *table)
+{
+    uint flags = 0;
+    if (TEST(FRAG_TABLE_TARGET_SHARED, table->table_flags))
+        flags |= FRAG_SHARED;
+    if (TEST(FRAG_TABLE_TRACE, table->table_flags))
+        flags |= FRAG_IS_TRACE;
+    /* We want to make sure that any updates to FRAG_TABLE_* flags
+     * are reflected in this routine. */
+    ASSERT_NOT_IMPLEMENTED(!TESTANY(~(FRAG_TABLE_INCLUSIVE_HIERARCHY |
+                                      FRAG_TABLE_IBL_TARGETED |
+                                      FRAG_TABLE_TARGET_SHARED |
+                                      FRAG_TABLE_SHARED |
+                                      FRAG_TABLE_TRACE |
+                                      FRAG_TABLE_PERSISTENT |
+                                      HASHTABLE_USE_ENTRY_STATS |
+                                      HASHTABLE_ALIGN_TABLE),
+                                    table->table_flags));
+    return flags;
+}
+
+/* Derive the PC of an entry point that aids in atomic hashtable deletion.
+ * FIXME: Once we can correlate from what table the fragment is being
+ * deleted and therefore type of the corresponding IBL routine, we can
+ * widen the interface and be more precise about which entry point
+ * is returned, i.e., specify something other than IBL_GENERIC.
+ */
+cache_pc
+get_target_delete_entry_pc(dcontext_t *dcontext, ibl_table_t *table)
+{
+    /*
+     * A shared IBL routine makes sure any registers restored on the
+     * miss path are all saved in the current dcontext - as well as
+     * copying the ECX in both TLS scratch and dcontext, so it is OK
+     * to simply return the thread private routine.  We have
+     * proven that they are functionally equivalent (all data in the
+     * shared lookup is fs indirected to the private dcontext)
+     *
+     * FIXME: we can in fact use a global delete_pc entry point that
+     * is the unlinked path of a shared_ibl_not_found, just like we
+     * could share all routines. Since it doesn't matter much for now
+     * we can also return the slightly more efficient private
+     * ibl_not_found path.
+     */
+    uint frag_flags = table_flags_to_frag_flags(dcontext, table);
+
+    ASSERT(dcontext != GLOBAL_DCONTEXT);
+
+    return (cache_pc) get_ibl_routine(dcontext, IBL_DELETE,
+                                      get_source_fragment_type(dcontext,
+                                                               frag_flags),
+                                      table->branch_type);
+}
+
+ibl_code_t *
+get_ibl_routine_code_ex(dcontext_t *dcontext, ibl_branch_type_t branch_type,
+                        uint fragment_flags _IF_X64(gencode_mode_t mode))
+{
+    ibl_source_fragment_type_t source_fragment_type =
+        get_source_fragment_type(dcontext, fragment_flags);
+
+    ibl_code_t *ibl_code =
+        get_ibl_routine_code_internal(dcontext, source_fragment_type, branch_type
+                                      _IF_X64(mode));
+    ASSERT(ibl_code != NULL);
+    return ibl_code;
+}
+
+ibl_code_t *
+get_ibl_routine_code(dcontext_t *dcontext, ibl_branch_type_t branch_type,
+                     uint fragment_flags)
+{
+    return get_ibl_routine_code_ex(dcontext, branch_type, fragment_flags
+                                   _IF_X64(dcontext == GLOBAL_DCONTEXT ?
+                                           MODE_OVERRIDE(FRAG_IS_32(fragment_flags)) :
+                                           GENCODE_FROM_DCONTEXT));
+}
+
+#ifdef RETURN_STACK
+cache_pc
+return_lookup_routine(dcontext_t *dcontext)
+{
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+    return (cache_pc) code->return_lookup;
+}
+
+cache_pc
+unlinked_return_routine(dcontext_t *dcontext)
+{
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+    return (cache_pc) code->unlinked_return;
+}
+#endif /* RETURN_STACK */
+
+
+#ifdef WINDOWS
+/* FIXME We support a private and shared fragments simultaneously targeting
+ * shared syscall -- -shared_fragment_shared_syscalls must be on and both
+ * fragment types target the entry point in shared_code. We could optimize
+ * the private fragment->shared syscall path (case 8025).
+ */
+/* PR 282576: These separate routines are ugly, but less ugly than adding param to
+ * the main routines, which are called in many places and usually passed a
+ * non-global dcontext; also less ugly than adding GLOBAL_DCONTEXT_X86.
+ */
+cache_pc
+shared_syscall_routine_ex(dcontext_t *dcontext _IF_X64(gencode_mode_t mode))
+{
+    generated_code_t *code = DYNAMO_OPTION(shared_fragment_shared_syscalls) ?
+        get_shared_gencode(dcontext _IF_X64(mode)) :
+        get_emitted_routines_code(dcontext _IF_X64(mode));
+    if (code == NULL)
+        return NULL;
+    else
+        return (cache_pc) code->shared_syscall;
+}
+
+cache_pc
+shared_syscall_routine(dcontext_t *dcontext)
+{
+    return shared_syscall_routine_ex(dcontext _IF_X64(GENCODE_FROM_DCONTEXT));
+}
+
+cache_pc
+unlinked_shared_syscall_routine_ex(dcontext_t *dcontext _IF_X64(gencode_mode_t mode))
+{
+    generated_code_t *code = DYNAMO_OPTION(shared_fragment_shared_syscalls) ?
+        get_shared_gencode(dcontext _IF_X64(mode)) :
+        get_emitted_routines_code(dcontext _IF_X64(mode));
+    if (code == NULL)
+        return NULL;
+    else
+        return (cache_pc) code->unlinked_shared_syscall;
+}
+
+cache_pc
+unlinked_shared_syscall_routine(dcontext_t *dcontext)
+{
+    return unlinked_shared_syscall_routine_ex(dcontext _IF_X64(GENCODE_FROM_DCONTEXT));
+}
+
+cache_pc
+after_shared_syscall_code(dcontext_t *dcontext)
+{
+    return after_shared_syscall_code_ex(dcontext  _IF_X64(GENCODE_FROM_DCONTEXT));
+}
+
+cache_pc
+after_shared_syscall_code_ex(dcontext_t *dcontext _IF_X64(gencode_mode_t mode))
+{
+    generated_code_t *code = get_emitted_routines_code(dcontext _IF_X64(mode));
+    ASSERT(code != NULL);
+    return (cache_pc) (code->unlinked_shared_syscall + code->sys_syscall_offs);
+}
+
+cache_pc
+after_shared_syscall_addr(dcontext_t *dcontext)
+{
+    ASSERT(get_syscall_method() != SYSCALL_METHOD_UNINITIALIZED);
+    if (DYNAMO_OPTION(sygate_int) &&
+        get_syscall_method() == SYSCALL_METHOD_INT)
+        return (int_syscall_address + INT_LENGTH /* sizeof int 2e */);
+    else
+        return after_shared_syscall_code(dcontext);
+}
+
+/* These are Windows-only since Linux needs to disambiguate its two
+ * versions of do_syscall
+ */
+cache_pc
+after_do_syscall_code(dcontext_t *dcontext)
+{
+    return after_do_syscall_code_ex(dcontext  _IF_X64(GENCODE_FROM_DCONTEXT));
+}
+
+cache_pc
+after_do_syscall_code_ex(dcontext_t *dcontext _IF_X64(gencode_mode_t mode))
+{
+    generated_code_t *code = get_emitted_routines_code(dcontext _IF_X64(mode));
+    ASSERT(code != NULL);
+    return (cache_pc) (code->do_syscall + code->do_syscall_offs);
+}
+
+cache_pc
+after_do_syscall_addr(dcontext_t *dcontext)
+{
+    ASSERT(get_syscall_method() != SYSCALL_METHOD_UNINITIALIZED);
+    if (DYNAMO_OPTION(sygate_int) &&
+        get_syscall_method() == SYSCALL_METHOD_INT)
+        return (int_syscall_address + INT_LENGTH /* sizeof int 2e */);
+    else
+        return after_do_syscall_code(dcontext);
+}
+#else
+cache_pc
+after_do_shared_syscall_addr(dcontext_t *dcontext)
+{
+    /* PR 212570: return the thread-shared do_syscall used for vsyscall hook */
+    generated_code_t *code = get_emitted_routines_code(GLOBAL_DCONTEXT
+                                                       _IF_X64(GENCODE_X64));
+    IF_X64(ASSERT_NOT_REACHED()); /* else have to worry about GENCODE_X86 */
+    ASSERT(code != NULL);
+    ASSERT(code->do_syscall != NULL);
+    return (cache_pc) (code->do_syscall + code->do_syscall_offs);
+}
+
+bool
+is_after_do_syscall_addr(dcontext_t *dcontext, cache_pc pc)
+{
+    generated_code_t *code = get_emitted_routines_code(dcontext
+                                                       _IF_X64(GENCODE_FROM_DCONTEXT));
+    ASSERT(code != NULL);
+    return (pc == (cache_pc) (code->do_syscall + code->do_syscall_offs) ||
+            pc == (cache_pc) (code->do_int_syscall + code->do_int_syscall_offs));
+}
+#endif
+
+bool
+is_after_syscall_address(dcontext_t *dcontext, cache_pc pc)
+{
+#ifdef WINDOWS 
+    if (pc == after_shared_syscall_addr(dcontext))
+        return true;
+    if (pc == after_do_syscall_addr(dcontext))
+        return true;
+    return false;
+#else
+    return is_after_do_syscall_addr(dcontext, pc);
+#endif
+    /* NOTE - we ignore global_do_syscall since that's only used in special
+     * circumstances and is not something the callers (recreate_app_state)
+     * really know how to handle. */
+}
+
+#ifdef LINUX
+/* PR 212290: can't be static code in x86.asm since it can't be PIC */
+cache_pc
+get_new_thread_start(dcontext_t *dcontext _IF_X64(gencode_mode_t mode))
+{
+#ifdef HAVE_TLS
+    /* for HAVE_TLS we use the shared version; w/o TLS we don't
+     * make any shared routines (PR 361894)
+     */
+    dcontext = GLOBAL_DCONTEXT;
+#endif
+    generated_code_t *gen = get_emitted_routines_code(dcontext _IF_X64(mode));
+    return gen->new_thread_dynamo_start;
+}
+#endif
+
+#ifdef TRACE_HEAD_CACHE_INCR
+cache_pc
+trace_head_incr_routine(dcontext_t *dcontext)
+{
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+    return (cache_pc) code->trace_head_incr;
+}
+#endif
+
+#ifdef CHECK_RETURNS_SSE2_EMIT
+cache_pc
+get_pextrw_entry(dcontext_t *dcontext)
+{
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+    return (cache_pc) code->pextrw;
+}
+
+cache_pc
+get_pinsrw_entry(dcontext_t *dcontext)
+{
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+    return (cache_pc) code->pinsrw;
+}
+#endif
+
+/* exported beyond x86/ */
+fcache_enter_func_t 
+get_fcache_enter_shared_routine(dcontext_t *dcontext)
+{
+    return fcache_enter_shared_routine(dcontext);
+}
+
+fcache_enter_func_t
+fcache_enter_shared_routine(dcontext_t *dcontext)
+{
+    ASSERT(USE_SHARED_GENCODE());
+    return (fcache_enter_func_t)
+        convert_data_to_function(SHARED_GENCODE_MATCH_THREAD(dcontext)->fcache_enter);
+}
+
+cache_pc
+fcache_return_shared_routine(IF_X64_ELSE(gencode_mode_t mode, void))
+{
+    generated_code_t *code = get_shared_gencode(GLOBAL_DCONTEXT _IF_X64(mode));
+    ASSERT(USE_SHARED_GENCODE());
+    if (code == NULL)
+        return NULL;
+    else
+        return code->fcache_return;
+}
+
+#ifdef TRACE_HEAD_CACHE_INCR
+cache_pc
+trace_head_incr_shared_routine(IF_X64_ELSE(gencode_mode_t mode, void))
+{
+    generated_code_t *code = get_shared_gencode(GLOBAL_DCONTEXT _IF_X64(mode));
+    ASSERT(USE_SHARED_GENCODE());
+    if (code == NULL)
+        return NULL;
+    else
+        return code->trace_head_incr;
+}
+#endif
+
+/* get the fcache target for the next code cache entry */
+cache_pc
+get_fcache_target(dcontext_t *dcontext)
+{
+    /* we used to use mcontext.pc, but that's in the writable
+     * portion of the dcontext, and so for self-protection we use the
+     * next_tag slot, which is protected
+     */
+    return dcontext->next_tag;
+}
+
+/* set the fcache target for the next code cache entry */
+void
+set_fcache_target(dcontext_t *dcontext, cache_pc value)
+{
+    /* we used to use mcontext.pc, but that's in the writable
+     * portion of the dcontext, and so for self-protection we use the
+     * next_tag slot, which is protected
+     */
+    dcontext->next_tag = value;
+    /* set eip as well to complete mcontext state */
+    get_mcontext(dcontext)->pc = value;
+}
+
+/***************************************************************************
+ * FAULT TRANSLATION
+ *
+ * Current status:
+ * After PR 214962 and PR 267260, we properly translate indirect branch
+ * mangling and client modifications.
+ * FIXME: However, we still do not properly translate for:
+ * - PR 267764: translate selfmod-mangled code
+ * - PR 303413: properly translate native_exec and windows sysenter mangling faults
+ * - PR 208037: flushed fragments (need -safe_translate_flushed)
+ * - PR 213251: hot patch fragments (b/c nudge can change whether patched => 
+ *     should store translations for all hot patch fragments) 
+ */
+
+typedef struct _translate_walk_t {
+    /* The context we're translating */
+    dr_mcontext_t *mc;
+    /* The code cache span of the containing fragment */
+    byte *start_cache;
+    byte *end_cache;
+    /* PR 263407: Track registers spilled since the last cti, for
+     * restoring indirect branch and rip-rel spills
+     */
+    bool reg_spilled[REG_SPILL_NUM];
+    bool reg_tls[REG_SPILL_NUM];
+    /* PR 267260: Track our own mangle-inserted pushes and pops, for
+     * restoring state in the middle of our indirect branch mangling.
+     * This is the adjustment in the forward direction.
+     */
+    int xsp_adjust;
+    /* Track whether we've seen an instr for which we can't relocate */
+    bool unsupported_mangle;
+    /* Are we currently in a mangle region */
+    bool in_mangle_region;
+    /* What is the translation target of the current mangle region */
+    app_pc translation;
+} translate_walk_t;
+
+static void
+translate_walk_init(translate_walk_t *walk, byte *start_cache, byte *end_cache,
+                    dr_mcontext_t *mc)
+{
+    memset(walk, 0, sizeof(*walk));
+    walk->mc = mc;
+    walk->start_cache = start_cache;
+    walk->end_cache = end_cache;
+}
+
+static inline bool
+instr_check_xsp_mangling(dcontext_t *dcontext, instr_t *inst, int *xsp_adjust)
+{
+    ASSERT(xsp_adjust != NULL);
+    if (instr_get_opcode(inst) == OP_push ||
+        instr_get_opcode(inst) == OP_push_imm) {
+        LOG(THREAD_GET, LOG_INTERP, 4, "\tstate track: push or push_imm\n");
+        *xsp_adjust -= opnd_size_in_bytes(opnd_get_size(instr_get_dst(inst, 1)));
+    } else if (instr_get_opcode(inst) == OP_pop) {
+        LOG(THREAD_GET, LOG_INTERP, 4, "\tstate track: pop\n");
+        *xsp_adjust += opnd_size_in_bytes(opnd_get_size(instr_get_src(inst, 1)));
+    }
+    /* 1st part of push emulation from insert_push_retaddr */
+    else if (instr_get_opcode(inst) == OP_lea &&
+             opnd_get_reg(instr_get_dst(inst, 0)) == REG_XSP &&
+             opnd_get_base(instr_get_src(inst, 0)) == REG_XSP &&
+             opnd_get_index(instr_get_src(inst, 0)) == REG_NULL) {
+        LOG(THREAD_GET, LOG_INTERP, 4, "\tstate track: lea xsp adjust\n");
+        *xsp_adjust += opnd_get_disp(instr_get_src(inst, 0));
+    }
+    /* 2nd part of push emulation from insert_push_retaddr */
+    else if (instr_get_opcode(inst) == OP_mov_st &&
+             opnd_is_base_disp(instr_get_dst(inst, 0)) &&
+             opnd_get_base(instr_get_dst(inst, 0)) == REG_XSP &&
+             opnd_get_index(instr_get_dst(inst, 0)) == REG_NULL) {
+        LOG(THREAD_GET, LOG_INTERP, 4, "\tstate track: store to stack\n");
+        /* nothing to track: paired lea is what we undo */
+    }
+    /* retrieval of target for call* or jmp* */
+    else if ((instr_get_opcode(inst) == OP_movzx &&
+              reg_overlap(opnd_get_reg(instr_get_dst(inst, 0)), REG_XCX)) ||
+             (instr_get_opcode(inst) == OP_mov_ld &&
+              reg_overlap(opnd_get_reg(instr_get_dst(inst, 0)), REG_XCX))) {
+        LOG(THREAD_GET, LOG_INTERP, 4, "\tstate track: ib tgt to *cx\n");
+        /* nothing: our xcx spill restore will undo */
+    }
+    /* part of pop emulation for iretd/lretd in x64 mode */
+    else if (instr_get_opcode(inst) == OP_mov_ld &&
+             opnd_is_base_disp(instr_get_src(inst, 0)) &&
+             opnd_get_base(instr_get_src(inst, 0)) == REG_XSP &&
+             opnd_get_index(instr_get_src(inst, 0)) == REG_NULL) {
+        LOG(THREAD_GET, LOG_INTERP, 4, "\tstate track: load from stack\n");
+        /* nothing to track: paired lea is what we undo */
+    }
+    /* part of data16 ret.  once we have cs preservation (PR 271317) we'll
+     * need to not fail when walking over a movzx to a pop cs (right now we
+     * do not read the stack for the pop cs).
+     */
+    else if (instr_get_opcode(inst) == OP_movzx &&
+             opnd_get_reg(instr_get_dst(inst, 0)) == REG_CX) {
+        LOG(THREAD_GET, LOG_INTERP, 4, "\tstate track: movzx to cx\n");
+        /* nothing: our xcx spill restore will undo */
+    }
+    /* fake pop of cs for iret */
+    else if (instr_get_opcode(inst) == OP_add &&
+             opnd_is_reg(instr_get_dst(inst, 0)) &&
+             opnd_get_reg(instr_get_dst(inst, 0)) == REG_XSP &&
+             opnd_is_immed_int(instr_get_src(inst, 0))) {
+        LOG(THREAD_GET, LOG_INTERP, 4, "\tstate track: add to xsp\n");
+        ASSERT(CHECK_TRUNCATE_TYPE_int(opnd_get_immed_int(instr_get_src(inst, 0))));
+        *xsp_adjust += (int) opnd_get_immed_int(instr_get_src(inst, 0));
+    }
+    /* popf for iret */
+    else if (instr_get_opcode(inst) == OP_popf) {
+        LOG(THREAD_GET, LOG_INTERP, 4, "\tstate track: popf\n");
+        *xsp_adjust += opnd_size_in_bytes(opnd_get_size(instr_get_src(inst, 1)));
+    } else {
+        return false;
+    }
+    return true;
+}
+
+static void
+translate_walk_track(dcontext_t *tdcontext, instr_t *inst, translate_walk_t *walk)
+{
+    reg_id_t reg, r;
+    bool spill, spill_tls;
+
+    /* Two mangle regions can be adjacent: distinguish by translation field */
+    if (walk->in_mangle_region &&
+        (!instr_is_our_mangling(inst) ||
+         instr_get_translation(inst) != walk->translation)) {
+        /* We assume our manglings are local and contiguous: once out of a
+         * mangling region, we're good to go again */
+        walk->in_mangle_region = false;
+        walk->unsupported_mangle = false;
+        walk->xsp_adjust = 0;
+        for (r = 0; r < REG_SPILL_NUM; r++) {
+            /* we should have seen a restore for every spill, unless at
+             * fragment-ending jump to ibl, which shouldn't come here
+             */
+            ASSERT(!walk->reg_spilled[r]);
+            walk->reg_spilled[r] = false; /* be paranoid */
+        }
+    }
+
+    if (instr_is_our_mangling(inst)) {
+        if (!walk->in_mangle_region) {
+            walk->in_mangle_region = true;
+            walk->translation = instr_get_translation(inst);
+        } else
+            ASSERT(walk->translation == instr_get_translation(inst));
+        /* PR 302951: we recognize a clean call by its NULL translation.
+         * We do not track any stack or spills: we assume we will only
+         * fault on an argument that references app memory, in which case
+         * we restore to the dr_mcontext_t on the stack.
+         */
+        if (walk->translation == NULL) {
+            DOLOG(4, LOG_INTERP, {
+                loginst(get_thread_private_dcontext(), 4, inst,
+                        "\tin clean call arg region");
+            });
+            return;
+        }
+        /* PR 263407: track register values that we've spilled.  We assume
+         * that spilling to non-canonical slots only happens in ibl or
+         * context switch code: never in app code mangling.  Since a client
+         * might add ctis (non-linear code) and its own spills, we track
+         * register spills only within our own mangling code (for
+         * post-mangling traces (PR 306163) we require that the client
+         * handle all translation if it modifies our mangling regions:
+         * we'll provide a query routine instr_is_DR_mangling()): our
+         * spills are all local anyway, except
+         * for selfmod, which is NYI (FIXME: PR 267764: we should at least
+         * return failure today for selfmod thread relocation since won't
+         * match push/pop checks below).  Our trace cmp is the only
+         * instance (besides selfmod) where we have a cti in our mangling,
+         * but it doesn't affect our linearity assumption.  We assume we
+         * have no entry points in between a spill and a restore.  Our
+         * mangling goes in last (for regular bbs and traces; see
+         * comment above for post-mangling traces), and so for local
+         * spills like rip-rel and ind branches this is fine.
+         */
+        if (instr_is_cti(inst) &&
+            /* Do not reset for a 32-bit trace-cmp jecxz or jmp, since ecx
+             * needs to be restored (won't fault, but for thread
+             * relocation)
+             */
+            ((instr_get_opcode(inst) != OP_jecxz && instr_get_opcode(inst) != OP_jmp) ||
+             /* Rather than check for trace, just ignore exit jumps, which
+              * won't mess up linearity here. For stored translation info we
+              * don't have meta-flags so we can't use instr_is_exit_cti(). */
+             (instr_get_opcode(inst) == OP_jmp &&
+              (!opnd_is_pc(instr_get_target(inst)) ||
+               (opnd_get_pc(instr_get_target(inst)) >= walk->start_cache &&
+                opnd_get_pc(instr_get_target(inst)) < walk->end_cache))))) {
+            /* reset for non-exit non-trace-jecxz cti (i.e., selfmod cti) */
+            for (r = 0; r < REG_SPILL_NUM; r++)
+                walk->reg_spilled[r] = false;
+        }
+        if (instr_is_reg_spill_or_restore(tdcontext, inst, &spill_tls, &spill, &reg)) {
+            r = reg - REG_START_SPILL;
+            /* if a restore whose spill was before a cti, ignore */
+            if (spill || walk->reg_spilled[r]) {
+                /* ensure restores and spills are properly paired up */
+                ASSERT((spill && !walk->reg_spilled[r]) ||
+                       (!spill && walk->reg_spilled[r]));
+                ASSERT(spill || walk->reg_tls[r] == spill_tls);
+                walk->reg_spilled[r] = spill;
+                walk->reg_tls[r] = spill_tls;
+                LOG(THREAD_GET, LOG_INTERP, 5,
+                    "\tspill update: %s %s %s\n", spill ? "spill" : "restore",
+                    spill_tls ? "tls" : "mcontext", reg_names[reg]);
+            }
+        }
+        /* PR 267260: Track our own mangle-inserted pushes and pops, for
+         * restoring state on an app fault in the middle of our indirect
+         * branch mangling.  We only need to support instrs added up until
+         * the last one that could have an app fault, as we can fail when
+         * called to translate for thread relocation: thus we ignore
+         * syscall mangling.
+         *
+         * The main scenarios are:
+         * 
+         * 1) call*: "spill ecx; mov->ecx; push retaddr": 
+         *    ecx restore handled above
+         * 2) far direct call: "push cs; push retaddr"
+         *    if fail on 2nd push need to undo 1st push
+         * 3) far call*: "spill ecx; tgt->ecx; push cs; push retaddr"
+         *    if fail on 1st push, restore ecx (above); 2nd push, also undo 1st push
+         * 4) iret: "pop eip; pop cs; pop eflags; (pop rsp; pop ss)"
+         *    if fail on non-initial pop, undo earlier pops
+         * 5) lret: "pop eip; pop cs"
+         *    if fail on non-initial pop, undo earlier pops
+         *
+         * FIXME: some of these push/pops are simulated (we simply adjust
+         * esp or do nothing), so we're not truly fault-transparent.
+         */
+        else if (instr_check_xsp_mangling(tdcontext, inst, &walk->xsp_adjust)) {
+            /* walk->xsp_adjust is now adjusted */
+        }
+        /* We do not support restoring state at arbitrary points for thread
+         * relocation (a performance issue, not a correctness one): if not a
+         * spill, restore, push, or pop, we will not properly translate.
+         * For an exit jmp for a simple ret we could relocate: but better not to
+         * for a call, since we've modified the stack w/ a push, so we fail on
+         * all exit jmps.
+         */
+        else {
+            DOLOG(4, LOG_INTERP,
+                  loginst(get_thread_private_dcontext(), 4,
+                          inst, "unsupported mangle instr"););
+            walk->unsupported_mangle = true;
+        }
+    }
+}
+
+static bool
+translate_walk_good_state(dcontext_t *tdcontext, translate_walk_t *walk,
+                          app_pc translate_pc)
+{
+    return (!walk->unsupported_mangle ||
+            /* If we're at the instr AFTER the mangle region, we're ok */
+            (walk->in_mangle_region && translate_pc != walk->translation));
+}
+
+static void
+translate_walk_restore(dcontext_t *tdcontext, translate_walk_t *walk,
+                       app_pc translate_pc)
+{
+    reg_id_t r;
+
+    if (translate_pc != walk->translation) {
+        /* When we walk we update only each instr we pass.  If we're
+         * now sitting at the instr AFTER the mangle region, we do
+         * NOT want to adjust xsp, since we're not translating to
+         * before that instr.  We should not have any outstanding spills.
+         */
+        LOG(THREAD_GET, LOG_INTERP, 2,
+            "\ttranslation "PFX" is post-walk "PFX" so not fixing xsp\n",
+            translate_pc, walk->translation);
+        DODEBUG({
+            for (r = 0; r < REG_SPILL_NUM; r++)
+                ASSERT(!walk->reg_spilled[r]);
+        });
+        return;
+    }
+
+    /* PR 263407: restore register values that are currently in spill slots
+     * for ind branches or rip-rel mangling.
+     * FIXME: for rip-rel loads, we may have clobbered the destination
+     * already, and won't be able to restore it: but that's a minor issue.
+     */
+    for (r = 0; r < REG_SPILL_NUM; r++) {
+        if (walk->reg_spilled[r]) {
+            reg_id_t reg = r + REG_START_SPILL;
+            reg_t value;
+            if (walk->reg_tls[r]) {
+                value = *(reg_t *)(((byte*)&tdcontext->local_state->spill_space) +
+                                   reg_spill_tls_offs(reg));
+            } else {
+                value = reg_get_value(reg, get_mcontext(tdcontext));
+            }
+            LOG(THREAD_GET, LOG_INTERP, 2,
+                "\trestoring spilled %s to "PFX"\n", reg_names[reg], value);
+            STATS_INC(recreate_spill_restores);
+            reg_set_value(reg, walk->mc, value);
+        }
+    }
+    /* PR 267260: Restore stack-adjust mangling of ctis.
+     * FIXME: we do NOT undo writes to the stack, so we're not completely
+     * transparent.  If we ever do restore memory, we'll want to pass in
+     * the restore_memory param.
+     */
+    if (walk->xsp_adjust != 0) {
+        walk->mc->xsp -= walk->xsp_adjust; /* negate to undo */
+        LOG(THREAD_GET, LOG_INTERP, 2,
+            "\tundoing push/pop by %d: xsp now "PFX"\n",
+            walk->xsp_adjust, walk->mc->xsp);
+    }
+}
+
+static void
+translate_restore_clean_call(dcontext_t *tdcontext, translate_walk_t *walk)
+{
+    /* PR 302951: we recognize a clean call by its combination of
+     * our-mangling and NULL translation.
+     * We restore to the dr_mcontext_t that was pushed on the stack.
+     */
+    LOG(THREAD_GET, LOG_INTERP, 2, "\ttranslating clean call arg crash\n");
+    dr_get_mcontext(tdcontext, walk->mc, NULL/*errno should be unchanged*/);
+    /* walk->mc->pc will be fixed up by caller */
+
+    /* PR 306410: up to caller to shift signal or SEH frame from dstack
+     * to app stack.  We naturally do that already for linux b/c we always
+     * have an alternate signal handling stack, but for Windows it takes
+     * extra work.
+     */
+}
+
+/* Returns a success code, but makes a best effort regardless.
+ * If just_pc is true, only recreates pc.
+ * Modifies mc with the recreated state.
+ * The caller must ensure tdcontext remains valid.
+ */
+/* Use THREAD_GET instead of THREAD so log messages go to calling thread */
+static recreate_success_t
+recreate_app_state_from_info(dcontext_t *tdcontext, const translation_info_t *info, 
+                             byte *start_cache, byte *end_cache,
+                             dr_mcontext_t *mc, bool just_pc _IF_DEBUG(uint flags))
+{
+    byte *answer = NULL;
+    byte *cpc, *prev_cpc;
+    cache_pc target_cache = mc->pc;
+    uint i;
+    bool contig = true, ours = false;
+    recreate_success_t res = (just_pc ? RECREATE_SUCCESS_PC : RECREATE_SUCCESS_STATE);
+    instr_t instr;
+    translate_walk_t walk;
+    translate_walk_init(&walk, start_cache, end_cache, mc);
+    instr_init(tdcontext, &instr);
+
+    ASSERT(info != NULL);
+    ASSERT(end_cache >= start_cache);
+
+    LOG(THREAD_GET, LOG_INTERP, 3,
+        "recreate_app : looking for "PFX" in frag @ "PFX" (tag "PFX")\n",
+        target_cache, start_cache, info->translation[0].app);
+    DOLOG(3, LOG_INTERP, {
+        translation_info_print(info, start_cache, THREAD_GET);
+    });
+
+    /* Strategy: walk through cache instrs, updating current app translation
+     * as we go along from the info table.  The table records only
+     * translations at change points and must interpolate between them, using
+     * either a stride of 0 if the previous translation entry is marked
+     * "identical" or a stride equal to the instruction length as we decode
+     * from the cache if the previous entry is !identical=="contiguous".
+     */
+
+    cpc = start_cache;
+    ASSERT(cpc - start_cache == info->translation[0].cache_offs);
+    i = 0;
+    while (cpc < end_cache) {
+        /* we can go beyond the end of the table: then use the last point */
+        if (i < info->num_entries &&
+            cpc - start_cache >= info->translation[i].cache_offs) {
+            /* We hit a change point: new app translation target */
+            answer = info->translation[i].app;
+            contig = !TEST(TRANSLATE_IDENTICAL, info->translation[i].flags);
+            ours = TEST(TRANSLATE_OUR_MANGLING, info->translation[i].flags);
+            i++;
+        }
+
+        if (cpc >= target_cache) {
+            /* we found the target to translate */
+            ASSERT(cpc == target_cache);
+            if (cpc > target_cache) { /* in debug will hit assert 1st */
+                LOG(THREAD_GET, LOG_INTERP, 2,
+                    "recreate_app -- WARNING: cache pc "PFX" != "PFX"\n",
+                    cpc, target_cache);
+                res = RECREATE_FAILURE; /* try to restore, but return false */
+            }
+            break;
+        }
+
+        /* PR 263407/PR 268372: we need to decode to instr level to track register
+         * values that we've spilled, and watch for ctis.  So far we don't need
+         * enough to justify a full decode_fragment().
+         */
+        instr_reset(tdcontext, &instr);
+        prev_cpc = cpc;
+        cpc = decode(tdcontext, cpc, &instr);
+        instr_set_our_mangling(&instr, ours);
+        translate_walk_track(tdcontext, &instr, &walk);
+
+        /* advance translation by the stride: either instr length or 0 */
+        if (contig)
+            answer += (cpc - prev_cpc);
+        /* else, answer stays put */
+    }
+    /* should always find xlation */
+    ASSERT(cpc < end_cache);
+    instr_free(tdcontext, &instr);
+
+    if (answer == NULL || !translate_walk_good_state(tdcontext, &walk, answer)) {
+        /* PR 214962: we're either in client meta-code (NULL translation) or
+         * post-app-fault in our own manglings: we shouldn't get an app
+         * fault in either case, so it's ok to fail, and neither is a safe
+         * spot for thread relocation.  For client meta-code we could split
+         * synch view (since we can get the app state consistent, just not
+         * the client state) from synch relocate, but that would require
+         * synchall re-architecting and may not be a noticeable perf win
+         * (should spend enough time at syscalls that will hit safe spot in
+         * reasonable time).
+         */
+        /* PR 302951: our clean calls do show up here and have full state */
+        if (answer == NULL && ours)
+            translate_restore_clean_call(tdcontext, &walk);
+        else
+            res = RECREATE_SUCCESS_PC; /* failed on full state, but pc good */
+        /* should only happen for thread synch, not a fault */
+        ASSERT(res == RECREATE_SUCCESS_STATE /* clean call */ ||
+               tdcontext != get_thread_private_dcontext() ||
+               INTERNAL_OPTION(stress_recreate_pc) ||
+               /* we can currently fail for selfmod (PR 267764) and flushed (PR 208037)
+                * (and hotpatch, native_exec, and sysenter: but too rare to check) */
+               TEST(FRAG_SELFMOD_SANDBOXED, flags) ||
+               TEST(FRAG_WAS_DELETED, flags));
+        if (answer == NULL) {
+            /* use next instr's translation.  skip any further meta-instrs regions. */
+            for (; i < info->num_entries; i++) {
+                if (info->translation[i].app != NULL)
+                    break;
+            }
+            ASSERT(i < info->num_entries);
+            if (i < info->num_entries)
+                answer = info->translation[i].app;;
+            ASSERT(answer != NULL);
+        }
+    }
+
+    if (!just_pc)
+        translate_walk_restore(tdcontext, &walk, answer);
+    LOG(THREAD_GET, LOG_INTERP, 2,
+        "recreate_app -- found ok pc "PFX"\n", answer);
+    mc->pc = answer;
+    return res;
+}
+
+/* Returns a success code, but makes a best effort regardless.
+ * If just_pc is true, only recreates pc.
+ * Modifies mc with the recreated state.
+ * The caller must ensure tdcontext remains valid.
+ */
+/* Use THREAD_GET instead of THREAD so log messages go to calling thread */
+static recreate_success_t
+recreate_app_state_from_ilist(dcontext_t *tdcontext, instrlist_t *ilist,
+                              byte *start_app, byte *start_cache, byte *end_cache,
+                              dr_mcontext_t *mc, bool just_pc _IF_DEBUG(uint flags))
+{
+    byte *answer = NULL;
+    byte *cpc, *prev_bytes;
+    instr_t *inst, *prev_ok;
+    cache_pc target_cache = mc->pc;
+    recreate_success_t res = (just_pc ? RECREATE_SUCCESS_PC : RECREATE_SUCCESS_STATE);
+    translate_walk_t walk;
+
+    LOG(THREAD_GET, LOG_INTERP, 3,
+        "recreate_app : looking for "PFX" in frag @ "PFX" (tag "PFX")\n",
+        target_cache, start_cache, start_app);
+
+    DOLOG(5, LOG_INTERP, {
+        instrlist_disassemble(tdcontext, 0, ilist, THREAD_GET);
+    });
+
+    /* walk ilist, incrementing cache pc by each instr's length until
+     * cache pc equals target, then look at original address of
+     * current instr, which is set by routines in mangle except for
+     * cti_short_rewrite.
+     */
+    cpc = start_cache;
+    /* since asking for the length will encode to a buffer, we cannot
+     * walk backwards at all.  thus we keep track of the previous instr
+     * with valid original bytes.
+     */
+    prev_ok = NULL;
+    prev_bytes = NULL;
+
+    translate_walk_init(&walk, start_cache, end_cache, mc);
+
+    for (inst = instrlist_first(ilist); inst; inst = instr_get_next(inst)) {
+        int len = instr_length(tdcontext, inst);
+
+        /* All we care about is that we are not going to skip over a
+         * bundle of app instructions. 
+         */
+        ASSERT(!instr_is_level_0(inst));
+
+        /* Case 4531, 4344: raw instructions being up-decoded can have
+         * their translation fields clobbered so we don't want any of those.
+         * (We used to have raw jecxz and nop instrs.)
+         * FIXME: if bb associated with this instr was hot patched, then
+         * the inserted raw instructions can trigger this assert.  Part of
+         * fix for case 5981.  In that case, this would be harmless.
+         */
+        ASSERT_CURIOSITY(instr_operands_valid(inst));
+
+        /* PR 332437: skip label instrs.  Nobody should expect setting
+         * a label's translation field to have any effect, and we
+         * don't need to explicitly split our mangling regions at
+         * labels so no reason to call translate_walk_track().
+         *
+         * We also skip all other length 0 instrs.  That would
+         * include un-encodable instrs, which we wouldn't have output,
+         * and so we should skip here in case the very next instr that we
+         * did encode had the real fault.
+         */
+        if (len == 0)
+            continue;
+
+        /* note this will be exercised for all instructions up to the answer */
+#ifndef CLIENT_INTERFACE
+# ifdef INTERNAL
+        ASSERT(instr_get_translation(inst) != NULL || DYNAMO_OPTION(optimize));
+# else
+        ASSERT(instr_get_translation(inst) != NULL);
+# endif
+#endif
+
+        LOG(THREAD_GET, LOG_INTERP, 5, "cache pc "PFX" vs "PFX"\n", 
+            cpc, target_cache);
+        if (cpc >= target_cache) {
+            if (cpc > target_cache) {
+                if (cpc == start_cache) {
+                    /* Prefix instructions are not added to recreate_fragment_ilist()
+                     * FIXME: we should do so, and then we can at least restore
+                     * our spills, just in case.
+                     */
+                    LOG(THREAD_GET, LOG_INTERP, 2,
+                        "recreate_app -- cache pc "PFX" != "PFX", "
+                        "assuming a prefix instruction\n", cpc, target_cache);
+                    res = RECREATE_SUCCESS_PC; /* failed on full state, but pc good */
+                    /* should only happen for thread synch, not a fault */
+                    ASSERT(tdcontext != get_thread_private_dcontext() ||
+                           INTERNAL_OPTION(stress_recreate_pc));
+                } else {
+                    LOG(THREAD_GET, LOG_INTERP, 2,
+                        "recreate_app -- WARNING: cache pc "PFX" != "PFX", "
+                        "probably prefix instruction\n", cpc, target_cache);
+                    res = RECREATE_FAILURE; /* try to restore, but return false */
+                }
+            }
+            if (instr_get_translation(inst) == NULL) {
+                /* Clients are supposed to leave their meta instrs with
+                 * NULL translations.  (DR may hit this assert for
+                 * -optimize but we need to fix that by setting translation
+                 * for all our optimizations.)  We assume we will never
+                 * get an app fault here, so we fail if asked for full state
+                 * since although we can get full app state we can't relocate
+                 * in the middle of client meta code.
+                 */
+                ASSERT(!instr_ok_to_mangle(inst));
+                /* PR 302951: our clean calls do show up here and have full state */
+                if (instr_is_our_mangling(inst))
+                    translate_restore_clean_call(tdcontext, &walk);
+                else
+                    res = RECREATE_SUCCESS_PC; /* failed on full state, but pc good */
+                /* should only happen for thread synch, not a fault */
+                ASSERT(instr_is_our_mangling(inst) /* PR 302951 */ ||
+                       tdcontext != get_thread_private_dcontext() ||
+                       INTERNAL_OPTION(stress_recreate_pc));
+                if (prev_ok == NULL) {
+                    answer = start_app;
+                    LOG(THREAD_GET, LOG_INTERP, 2,
+                        "recreate_app -- WARNING: guessing start pc "PFX"\n", answer);
+                } else {
+                    answer = prev_bytes;
+                    LOG(THREAD_GET, LOG_INTERP, 2,
+                        "recreate_app -- WARNING: guessing after prev "
+                        "translation (pc "PFX")\n", answer);
+                    DOLOG(2, LOG_INTERP, loginst(get_thread_private_dcontext(),
+                                                 2, prev_ok, "\tprev instr"););
+                }
+            } else {
+                answer = instr_get_translation(inst);
+                if (translate_walk_good_state(tdcontext, &walk, answer)) {
+                    LOG(THREAD_GET, LOG_INTERP, 2,
+                        "recreate_app -- found valid state pc "PFX"\n", answer);
+                } else {
+                    res = RECREATE_SUCCESS_PC; /* failed on full state, but pc good */
+                    /* should only happen for thread synch, not a fault */
+                    ASSERT(tdcontext != get_thread_private_dcontext() ||
+                           INTERNAL_OPTION(stress_recreate_pc) ||
+                           /* we can currently fail for selfmod (PR 267764) and
+                            * flushed (PR 208037) (and hotpatch, native_exec, and
+                            * sysenter: but too rare to check) */
+                           TEST(FRAG_SELFMOD_SANDBOXED, flags) ||
+                           TEST(FRAG_WAS_DELETED, flags));
+                    LOG(THREAD_GET, LOG_INTERP, 2,
+                        "recreate_app -- not able to fully recreate "
+                        "context, pc is in added instruction from mangling\n");
+                }
+            }
+            if (!just_pc)
+                translate_walk_restore(tdcontext, &walk, answer);
+            LOG(THREAD_GET, LOG_INTERP, 2,
+                "recreate_app -- found ok pc "PFX"\n", answer);
+            mc->pc = answer;
+            return res;
+        }
+        /* we only use translation pointers, never just raw bit pointers */
+        if (instr_get_translation(inst) != NULL) {
+            prev_ok = inst;
+            DOLOG(5, LOG_INTERP, loginst(get_thread_private_dcontext(), 
+                                         5, prev_ok, "\tok instr"););
+            prev_bytes = instr_get_translation(inst);
+            /* we really want the pc after the translation target since we'll 
+             * use this if we pass up the target without hitting it
+             */
+            /* FIXME: do we need to check for readability first? 
+             * in normal usage all translation targets should have been decoded
+             * already while building the bb ilist
+             */
+            prev_bytes = decode_next_pc(tdcontext, prev_bytes);
+        }
+
+        translate_walk_track(tdcontext, inst, &walk);
+
+        cpc += len;
+    }
+
+    /* ERROR! */
+    LOG(THREAD_GET, LOG_INTERP, 1,
+        "ERROR: recreate_app : looking for "PFX" in frag @ "PFX" "
+        "(tag "PFX")\n", target_cache, start_cache, start_app);
+    DOLOG(1, LOG_INTERP, {
+        instrlist_disassemble(tdcontext, 0, ilist, THREAD_GET);
+    });
+    ASSERT_NOT_REACHED();
+    if (just_pc) {
+        /* just guess */
+        mc->pc = answer;
+    }
+    return RECREATE_FAILURE;
+}
+
+/* The esp in mcontext must either be valid or NULL (if null will be unable to 
+ * recreate on XP and 03 at vsyscall_after_syscall and on sygate 2k at after syscall).
+ * Returns true if successful.  Whether successful or not, attempts to modify
+ * mcontext with recreated state. If just_pc only translates the pc
+ * (this is more likely to succeed)
+ */
+/* Use THREAD_GET instead of THREAD so log messages go to calling thread */
+/* Also see NOTEs at recreate_app_state() about lock usage, and lack of full stack
+ * translation. */
+static recreate_success_t
+recreate_app_state_internal(dcontext_t *tdcontext, dr_mcontext_t *mcontext,
+                            bool just_pc, fragment_t *owning_f, bool restore_memory)
+{
+    recreate_success_t res = (just_pc ? RECREATE_SUCCESS_PC : RECREATE_SUCCESS_STATE);
+#ifdef WINDOWS
+    if (get_syscall_method() == SYSCALL_METHOD_SYSENTER && 
+        mcontext->pc == vsyscall_after_syscall &&
+        mcontext->xsp != 0) {
+        ASSERT(get_os_version() >= WINDOWS_VERSION_XP);
+        /* case 5441 sygate hack means ret addr to after_syscall will be at
+         * esp+4 (esp will point to ret in ntdll.dll) for sysenter */
+        /* FIXME - should we check that esp is readable? */
+        if (is_after_syscall_address(tdcontext, *(cache_pc *)
+                                     (mcontext->xsp+
+                                      (DYNAMO_OPTION(sygate_sysenter) ? 4 : 0)))) {
+            /* no translation needed, ignoring sysenter stack hacks */
+            LOG(THREAD_GET, LOG_INTERP|LOG_SYNCH, 2, 
+                "recreate_app no translation needed (at vsyscall)\n");
+            return res;
+        } else {
+            /* this is a dynamo system call! */
+            LOG(THREAD_GET, LOG_INTERP|LOG_SYNCH, 2, 
+                "recreate_app at dynamo system call\n");
+            return RECREATE_FAILURE;
+        }
+    }
+#else
+    if (get_syscall_method() == SYSCALL_METHOD_SYSENTER && 
+        /* Even when the main syscall method is sysenter, we also have a
+         * do_int_syscall and do_clone_syscall that use int, so check both.
+         * Note that we don't modify the stack, so once we do sysenter syscalls
+         * inlined in the cache (PR 288101) we'll need some mechanism to
+         * distinguish those: but for now if a sysenter instruction is used it
+         * has to be do_syscall since DR's own syscalls are ints.
+         */
+        (mcontext->pc == vsyscall_sysenter_return_pc ||
+         is_after_do_syscall_addr(tdcontext, mcontext->pc))) {
+        LOG(THREAD_GET, LOG_INTERP|LOG_SYNCH, 2, 
+            "recreate_app no translation needed (at syscall)\n");
+        return res;
+    }
+#endif
+    else if (is_after_syscall_address(tdcontext, mcontext->pc) &&
+             does_syscall_ret_to_callsite()) {
+            /* suspended inside kernel at syscall
+             * all registers have app values for syscall */
+            LOG(THREAD_GET, LOG_INTERP|LOG_SYNCH, 2, 
+                "recreate_app pc = after_syscall, translating\n");
+#ifdef WINDOWS
+            if (DYNAMO_OPTION(sygate_int) &&
+                get_syscall_method() == SYSCALL_METHOD_INT) {
+                if ((app_pc)mcontext->xsp == NULL)
+                    return RECREATE_FAILURE;
+                /* dr system calls will have the same after_syscall address when
+                 * sygate hack are in effect so need to check top of stack to see
+                 * if we are returning to dr or do/share syscall (generated
+                 * routines) */
+                if (!in_generated_routine(tdcontext, *(app_pc *)mcontext->xsp)) {
+                    /* this must be a dynamo system call! */
+                    LOG(THREAD_GET, LOG_INTERP|LOG_SYNCH, 2, 
+                        "recreate_app at dynamo system call\n");
+                    return RECREATE_FAILURE;
+                }
+                ASSERT(*(app_pc *)mcontext->xsp == 
+                       after_do_syscall_code(tdcontext) ||
+                       *(app_pc *)mcontext->xsp == 
+                       after_shared_syscall_code(tdcontext));
+                if (!just_pc) {
+                    /* This is an int system call and since for sygate
+                     * compatibility we redirect those with a call to an ntdll.dll
+                     * int 2e ret 0 we need to pop the stack once to match app. */
+                    mcontext->xsp += XSP_SZ; /* pop the stack */
+                }
+            }
+#endif
+            mcontext->pc = POST_SYSCALL_PC(tdcontext);
+            return res;
+    } else if (mcontext->pc == get_reset_exit_stub(tdcontext)) {
+        LOG(THREAD_GET, LOG_INTERP|LOG_SYNCH, 2, 
+            "recreate_app at reset exit stub => using next_tag "PFX"\n",
+            tdcontext->next_tag);
+        /* context is completely native except the pc */
+        mcontext->pc = tdcontext->next_tag;
+        return res;
+    } else if (in_generated_routine(tdcontext, mcontext->pc)) {
+        LOG(THREAD_GET, LOG_INTERP|LOG_SYNCH, 2, 
+            "recreate_app state at untranslatable address in "
+            "generated routines for thread "IDFMT"\n", tdcontext->owning_thread);
+        return RECREATE_FAILURE;
+    } else if (in_fcache(mcontext->pc)) {
+        /* FIXME: what if pc is in separate direct stub??? 
+         * do we have to read the &l from the stub to find linkstub_t and thus
+         * fragment_t owner?
+         */
+        /* NOTE - only at this point is it safe to grab locks other then the
+         * fcache_unit_areas.lock */
+        linkstub_t *l;
+        cache_pc cti_pc;
+        instrlist_t *ilist = NULL;
+        fragment_t *f = owning_f;
+        bool alloc = false;
+        IF_X64(bool old_mode;)
+
+        /* Rather than storing a mapping table, we re-build the fragment
+         * containing the code cache pc whenever we can.  For pending-deletion
+         * fragments we can't do that and have to store the info, due to our
+         * weak consistency flushing where the app code may have changed
+         * before we get here (case 3559).
+         */
+
+        /* Check whether we have a fragment w/ stored translations before
+         * asking to recreate the ilist
+         */
+        if (f == NULL)
+            f = fragment_pclookup_with_linkstubs(tdcontext, mcontext->pc, &alloc);
+
+        /* Whether a bb or trace, this routine will recreate the entire ilist. */
+        if (f == NULL)
+            ilist = recreate_fragment_ilist(tdcontext, mcontext->pc, &f, &alloc,
+                                            true/*mangle*/ _IF_CLIENT(true/*client*/));
+        else if (FRAGMENT_TRANSLATION_INFO(f) == NULL) {
+            /* NULL for pc indicates that f is valid */
+            bool new_alloc;
+            ilist = recreate_fragment_ilist(tdcontext, NULL, &f, &new_alloc,
+                                            true/*mangle*/ _IF_CLIENT(true/*client*/));
+            ASSERT(owning_f == NULL || f == owning_f);
+            ASSERT(!new_alloc);
+        }
+        if (ilist == NULL && (f == NULL || FRAGMENT_TRANSLATION_INFO(f) == NULL)) {
+            /* It is problematic if this routine fails.  Many places assume that 
+             * recreate_app_pc() will work.
+             */
+            ASSERT(!INTERNAL_OPTION(safe_translate_flushed));
+            res = RECREATE_FAILURE;
+            goto recreate_app_state_done;
+        }
+        
+        LOG(THREAD_GET, LOG_INTERP, 2,
+            "recreate_app : pc is in F%d("PFX")%s\n", f->id, f->tag,
+            ((f->flags & FRAG_IS_TRACE) != 0)?" (trace)":"");
+        
+        DOLOG(2, LOG_SYNCH, {
+            if (ilist != NULL) {
+                LOG(THREAD_GET, LOG_SYNCH, 2, "ilist for recreation:\n");
+                instrlist_disassemble(tdcontext, f->tag, ilist, THREAD_GET);
+            }
+        });
+        
+        /* if pc is in an exit stub, we find the corresponding exit instr */
+        cti_pc = NULL;
+        for (l = FRAGMENT_EXIT_STUBS(f); l; l = LINKSTUB_NEXT_EXIT(l)) {
+            if (EXIT_HAS_LOCAL_STUB(l->flags, f->flags)) {
+                /* FIXME: as computing the stub pc becomes more expensive,
+                 * should perhaps check fragment_body_end_pc() or something
+                 * that only does one stub check up front, and then find the
+                 * exact stub if pc is beyond the end of the body.
+                 */
+                if (mcontext->pc < EXIT_STUB_PC(tdcontext, f, l))
+                    break;
+                cti_pc = EXIT_CTI_PC(f, l);
+            }
+        }
+        if (cti_pc != NULL) {
+            /* target is inside an exit stub!
+             * new target: the exit cti, not its stub
+             */
+            if (!just_pc) {
+                /* FIXME : translate from exit stub */
+                LOG(THREAD_GET, LOG_INTERP|LOG_SYNCH, 2, 
+                    "recreate_app_helper -- can't full recreate state, pc "PFX" "
+                    "is in exit stub\n", mcontext->pc);
+                res = RECREATE_SUCCESS_PC; /* failed on full state, but pc good */
+                goto recreate_app_state_done;
+            }
+            LOG(THREAD_GET, LOG_INTERP|LOG_SYNCH, 2,
+                "\ttarget "PFX" is inside an exit stub, looking for its cti "
+                " "PFX"\n", mcontext->pc, cti_pc);
+            mcontext->pc = cti_pc;
+        }
+       
+        /* Recreate in same mode as original fragment */
+        IF_X64(old_mode = set_x86_mode(tdcontext, FRAG_IS_32(f->flags)));
+
+        /* now recreate the state */
+        if (ilist == NULL) {
+            ASSERT(f != NULL && FRAGMENT_TRANSLATION_INFO(f) != NULL);
+            ASSERT(!TEST(FRAG_WAS_DELETED, f->flags) ||
+                   INTERNAL_OPTION(safe_translate_flushed));
+            res = recreate_app_state_from_info(tdcontext, FRAGMENT_TRANSLATION_INFO(f),
+                                               (byte *) f->start_pc,
+                                               (byte *) f->start_pc + f->size,
+                                               mcontext, just_pc _IF_DEBUG(f->flags));
+            STATS_INC(recreate_via_stored_info);
+        } else {
+            res = recreate_app_state_from_ilist(tdcontext, ilist, (byte *) f->tag,
+                                                (byte *) FCACHE_ENTRY_PC(f),
+                                                (byte *) f->start_pc + f->size,
+                                                mcontext, just_pc _IF_DEBUG(f->flags));
+            STATS_INC(recreate_via_app_ilist);
+        }
+        IF_X64(set_x86_mode(tdcontext, old_mode));
+
+#ifdef STEAL_REGISTER
+        /* FIXME: conflicts w/ PR 263407 reg spill tracking */
+        ASSERT_NOT_IMPLEMENTED(false && "conflicts w/ reg spill tracking");
+        if (!just_pc) {
+            /* get app's value of edi */
+            mc->xdi = get_mcontext(tdcontext)->xdi;
+        }
+#endif
+#ifdef CLIENT_INTERFACE
+        if (res != RECREATE_FAILURE) {
+            /* PR 214962: if the client has a restore callback, invoke it to
+             * fix up the state (and pc).
+             */
+            instrument_restore_state(tdcontext, f->tag, mcontext, restore_memory,
+                                     !TESTANY(FRAG_WAS_DELETED|FRAG_SELFMOD_SANDBOXED,
+                                              f->flags));
+        }
+#endif
+
+    recreate_app_state_done:
+        /* free the instrlist_t elements */
+        if (ilist != NULL)
+            instrlist_clear_and_destroy(tdcontext, ilist);
+        if (alloc) {
+            ASSERT(f != NULL);
+            fragment_free(tdcontext, f);
+        }
+        return res;
+    } else {
+        /* handle any other cases, in DR etc. */
+        return RECREATE_FAILURE;
+    }
+}
+
+/* Assumes that pc is a pc_recreatable place (i.e. in_fcache(), though could do
+ * syscalls with esp, also see FIXME about separate stubs in 
+ * recreate_app_state_internal()), ASSERTs otherwise.
+ * If caller knows which fragment pc belongs to, caller should pass in f
+ * to avoid work and avoid lock rank issues as pclookup acquires shared_cache_lock;
+ * else, pass in NULL for f.
+ * NOTE - If called by a thread other than the tdcontext owner, caller must 
+ * ensure tdcontext remains valid.  Caller also must ensure that it is safe to
+ * allocate memory from tdcontext (for instr routines), i.e. caller owns 
+ * tdcontext or the owner of tdcontext is suspended.  Also if tdcontext is
+ * !couldbelinking then caller must own the thread_initexit_lock in case
+ * recreate_fragment_ilist() is called.
+ * NOTE - If this function is unable to translate the pc, but the pc is
+ * in_fcache() then there is an assert curiosity and the function returns NULL.
+ * This can happen only from the pc being in a fragment that is pending 
+ * deletion (ref case 3559 others).  Most callers don't check the returned
+ * value and wouldn't have a way to recover even if they did. FIXME
+ */
+/* Use THREAD_GET instead of THREAD so log messages go to calling thread */
+app_pc
+recreate_app_pc(dcontext_t *tdcontext, cache_pc pc, fragment_t *f)
+{
+    dr_mcontext_t mc;
+    recreate_success_t res;
+
+    LOG(THREAD_GET, LOG_INTERP, 2,
+        "recreate_app_pc -- translating from pc="PFX"\n", pc);
+
+    memset(&mc, 0, sizeof(mc)); /* ensures esp is NULL */
+    mc.pc = pc;
+
+    res = recreate_app_state_internal(tdcontext, &mc, true, f, false);
+    if (res != RECREATE_SUCCESS_PC) {
+        ASSERT(res != RECREATE_SUCCESS_STATE); /* shouldn't return that for just_pc */
+        ASSERT(in_fcache(pc)); /* Make sure caller didn't screw up */
+        /* We were unable to translate the pc, most likely because the
+         * pc is in a fragment that is pending deletion FIXME, most callers
+         * aren't able to recover! */
+        ASSERT_CURIOSITY(res && "Unable to translate pc");
+        mc.pc = NULL;
+    }
+    
+    LOG(THREAD_GET, LOG_INTERP, 2,
+        "recreate_app_pc -- translation is "PFX"\n", mc.pc);
+    
+    return mc.pc;
+}
+
+/* Translates the code cache state in mcontext into what it would look like
+ * in the original application.
+ * If it fails altogether, returns RECREATE_FAILURE, but still provides
+ * a best-effort translation.
+ * If it fails to restore the full machine state, but does restore the pc,
+ * returns RECREATE_SUCCESS_PC.
+ * If it successfully restores the full machine state, 
+ * returns RECREATE_SUCCESS_STATE.  Only for full success does it
+ * consider the restore_memory parameter, which, if true, requests restoration
+ * of any memory values that were shifted (primarily due to clients) (otherwise,
+ * only the passed-in mcontext is modified). If restore_memory is
+ * true, the caller should always relocate the translated thread, as
+ * it may not execute properly if left at its current location (it
+ * could be in the middle of client code in the cache).
+ *
+ * FIXME: does not undo stack mangling for sysenter
+ */
+/* NOTE - Can be called with a thread suspended at an arbitrary place by synch 
+ * routines so must not call mutex_lock (or call a function that does) unless 
+ * the synch routines have checked that lock.  Currently only fcache_unit_areas.lock 
+ * is used (for in_fcache in recreate_app_state_internal()) 
+ * (if in_fcache succeeds then assumes other locks won't be a problem). 
+ * NOTE - If called by a thread other than the tdcontext owner, caller must 
+ * ensure tdcontext remains valid.  Caller also must ensure that it is safe to
+ * allocate memory from tdcontext (for instr routines), i.e. caller owns 
+ * tdcontext or the owner of tdcontext is suspended.  Also if tdcontext is
+ * !couldbelinking then caller must own the thread_initexit_lock in case
+ * recreate_fragment_ilist() is called.  We assume that when tdcontext is
+ * not the calling thread, this is a thread synch request, and is NOT from
+ * an app fault (PR 267260)!
+ */
+/* Use THREAD_GET instead of THREAD so log messages go to calling thread */
+recreate_success_t
+recreate_app_state(dcontext_t *tdcontext, dr_mcontext_t *mcontext,
+                   bool restore_memory)
+{
+    recreate_success_t res;
+
+#ifdef DEBUG
+    if (stats->loglevel >= 2 && (stats->logmask & LOG_SYNCH) != 0) {
+        LOG(THREAD_GET, LOG_SYNCH, 2,
+            "recreate_app_state -- translating from:\n");
+        dump_mcontext(mcontext, THREAD_GET, DUMP_NOT_XML);
+    }
+#endif
+
+    res = recreate_app_state_internal(tdcontext, mcontext, false, NULL, restore_memory);
+        
+#ifdef DEBUG
+    if (res) {
+        if (stats->loglevel >= 2 && (stats->logmask & LOG_SYNCH) != 0) {
+            LOG(THREAD_GET, LOG_SYNCH, 2,
+                "recreate_app_state -- translation is:\n");
+            dump_mcontext(mcontext, THREAD_GET, DUMP_NOT_XML);
+        }
+    } else {
+        LOG(THREAD_GET, LOG_SYNCH, 2,
+            "recreate_app_state -- unable to translate\n");
+    }
+#endif
+
+    return res;
+}
+
+static inline uint
+translation_info_alloc_size(uint num_entries)
+{
+    return (sizeof(translation_info_t) + sizeof(translation_entry_t)*num_entries);
+}
+
+/* we save space by inlining the array with the struct holding the length */
+static translation_info_t *
+translation_info_alloc(dcontext_t *dcontext, uint num_entries)
+{
+    /* we need to use global heap since pending-delete fragments become
+     * shared entities
+     */
+    translation_info_t *info =
+        global_heap_alloc(translation_info_alloc_size(num_entries) HEAPACCT(ACCT_OTHER));
+    info->num_entries = num_entries;
+    return info;
+}
+
+void
+translation_info_free(dcontext_t *dcontext, translation_info_t *info)
+{
+    global_heap_free(info, translation_info_alloc_size(info->num_entries)
+                     HEAPACCT(ACCT_OTHER));
+}
+
+static inline void
+set_translation(dcontext_t *dcontext, translation_entry_t **entries,
+                uint *num_entries, uint entry,
+                ushort cache_offs, app_pc app, bool identical, bool our_mangling)
+{
+    if (entry >= *num_entries) {
+        /* alloc new arrays 2x as big */
+        *entries = global_heap_realloc(*entries, *num_entries,
+                                       *num_entries*2, sizeof(translation_entry_t)
+                                       HEAPACCT(ACCT_OTHER));
+        *num_entries *= 2;
+    }
+    ASSERT(entry < *num_entries);
+    (*entries)[entry].cache_offs = cache_offs;
+    (*entries)[entry].app = app;
+    (*entries)[entry].flags = 0;
+    if (identical)
+        (*entries)[entry].flags |= TRANSLATE_IDENTICAL;
+    if (our_mangling)
+        (*entries)[entry].flags |= TRANSLATE_OUR_MANGLING;
+    LOG(THREAD, LOG_FRAGMENT, 4, "\tset_translation: %d +%5d => "PFX" %s%s\n",
+        entry, cache_offs, app, identical ? "identical" : "contiguous",
+        our_mangling ? " ours" : "");
+}
+
+void
+translation_info_print(const translation_info_t *info, cache_pc start, file_t file)
+{
+    uint i;
+    ASSERT(info != NULL);
+    ASSERT(file != INVALID_FILE);
+    print_file(file, "translation info "PFX"\n", info);
+    for (i=0; i<info->num_entries; i++) {
+        print_file(file, "\t%d +%5d == "PFX" => "PFX" %s%s\n",
+                   i, info->translation[i].cache_offs,
+                   start + info->translation[i].cache_offs,
+                   info->translation[i].app,
+                   TEST(TRANSLATE_IDENTICAL, info->translation[i].flags) ?
+                   "identical" : "contiguous",
+                   TEST(TRANSLATE_OUR_MANGLING, info->translation[i].flags) ?
+                   " ours" : "");
+    }
+}
+
+/* With our weak flushing consistency we must store translation info
+ * for any fragment that may outlive its original app code (case
+ * 3559).  Here we store actual translation info.  An alternative is
+ * to store elided jmp information and a copy of the source memory,
+ * but that takes more memory for all but the smallest fragments.  A
+ * better alternative is to reliably de-mangle, which would require
+ * only elided jmp information.
+ */
+translation_info_t *
+record_translation_info(dcontext_t *dcontext, fragment_t *f, instrlist_t *existing_ilist)
+{
+    translation_entry_t *entries;
+    uint num_entries;
+    translation_info_t *info = NULL;
+    instrlist_t *ilist;
+    instr_t *inst;
+    uint i;
+    uint last_len = 0;
+    bool last_contig;
+    app_pc last_translation = NULL;
+    cache_pc cpc;
+    
+    LOG(THREAD, LOG_FRAGMENT, 3, "record_translation_info: F%d("PFX")."PFX"\n",
+        f->id, f->tag, f->start_pc);
+
+    if (existing_ilist != NULL)
+        ilist = existing_ilist;
+    else if (TEST(FRAG_SELFMOD_SANDBOXED, f->flags)) {
+        /* If f is selfmod, app code may have changed (we see this w/ code
+         * on the stack later flushed w/ os_thread_stack_exit(), though in that
+         * case we don't expect it to be executed again), so we do a special
+         * recreate from the selfmod copy.
+         * Since selfmod is straight-line code we can rebuild from cache and
+         * offset each translation entry
+         */
+        cache_pc selfmod_copy = FRAGMENT_SELFMOD_COPY_PC(f);
+        ASSERT(!TEST(FRAG_IS_TRACE, f->flags));
+        ASSERT(!TEST(FRAG_HAS_DIRECT_CTI, f->flags));
+        /* We must build our ilist w/o calling check_thread_vm_area(), as it will
+         * freak out that we are decoding DR memory.
+         */
+        ilist = recreate_bb_ilist(dcontext, selfmod_copy, FRAG_SELFMOD_SANDBOXED,
+                                  NULL, NULL,
+                                  false/*don't check vm areas!*/, true/*mangle*/
+                                  _IF_CLIENT(true/*call client*/)
+                                  _IF_CLIENT(false/*!for_trace*/));
+        ASSERT(ilist != NULL); /* shouldn't fail: our own code is always readable! */
+        for (inst = instrlist_first(ilist); inst; inst = instr_get_next(inst)) {
+            app_pc app = instr_get_translation(inst);
+            if (app != NULL)
+                instr_set_translation(inst, app - selfmod_copy + f->tag);
+        }
+    } else {
+        /* Must re-build fragment and record translation info for each instr.
+         * Whether a bb or trace, this routine will recreate the entire ilist.
+         */
+        ilist = recreate_fragment_ilist(dcontext, NULL, &f, NULL,
+                                        true/*mangle*/ _IF_CLIENT(true/*client*/));
+    }
+    ASSERT(ilist != NULL);
+    DOLOG(3, LOG_FRAGMENT, {
+        LOG(THREAD, LOG_FRAGMENT, 3, "ilist for recreation:\n");
+        instrlist_disassemble(dcontext, f->tag, ilist, THREAD);
+    });
+
+    /* To avoid two passes we do one pass and store into a large-enough array.
+     * We then copy the results into a just-right-sized array.  A typical bb
+     * requires 2 entries, one for its body of straight-line code and one for
+     * the inserted jmp at the end, so we start w/ that to avoid copying in
+     * the common case.  FIXME: optimization: instead of every bb requiring a
+     * final entry for the inserted jmp, have recreate_ know about it and cut
+     * in half the typical storage reqts.
+     */
+#   define NUM_INITIAL_TRANSLATIONS 2
+    num_entries = NUM_INITIAL_TRANSLATIONS;
+    entries = HEAP_ARRAY_ALLOC(GLOBAL_DCONTEXT, translation_entry_t,
+                               NUM_INITIAL_TRANSLATIONS, ACCT_OTHER, PROTECTED);
+
+    i = 0;
+    cpc = (byte *) FCACHE_ENTRY_PC(f);
+    if (fragment_prefix_size(f->flags) > 0) {
+        ASSERT(f->start_pc < cpc);
+        set_translation(dcontext, &entries, &num_entries, i, 0,
+                        f->tag, true/*identical*/, true/*our mangling*/);
+        last_translation = f->tag;
+        last_contig = false;
+        i++;
+    } else {
+        ASSERT(f->start_pc == cpc);
+        last_contig = true; /* we create 1st entry on 1st loop iter */
+    }
+    for (inst = instrlist_first(ilist); inst; inst = instr_get_next(inst)) {
+        app_pc app = instr_get_translation(inst);
+        uint prev_i = i;
+#ifndef CLIENT_INTERFACE
+# ifdef INTERNAL
+        ASSERT(app != NULL || DYNAMO_OPTION(optimize));
+# else
+        ASSERT(app != NULL);
+# endif
+#endif
+        /* Should only be NULL for meta-code added by a client.
+         * We preserve the NULL so our translation routines know to not
+         * let this be a thread relocation point
+         */
+        /* PR 302951: clean call args are instr_is_our_mangling so no assert for that */
+        ASSERT(app != NULL || !instr_ok_to_mangle(inst));
+        /* see whether we need a new entry, or the current stride (contig
+         * or identical) holds
+         */
+        if (last_contig) {
+            if ((i == 0 && (app == NULL || instr_is_our_mangling(inst)))
+                || app == last_translation) {
+                /* we are now in an identical region */
+                /* Our incremental discovery can cause us to add a new
+                 * entry of one type that on the next instr we discover
+                 * can optimally be recorded as the other type.  Here we
+                 * hit an app pc shift whose target needs an identical
+                 * entry: so rather than a contig followed by identical,
+                 * we can get away with a single identical.
+                 * Example: "x x+1 y y", where we use an identical for the
+                 * first y instead of the contig that we initially guessed
+                 * at b/c we assumed it was an elision.
+                 */
+                if (i > 0 && entries[i-1].cache_offs == cpc - last_len - f->start_pc) {
+                    /* convert prev contig into identical */
+                    ASSERT(!TEST(TRANSLATE_IDENTICAL, entries[i-1].flags));
+                    entries[i-1].flags |= TRANSLATE_IDENTICAL;
+                    LOG(THREAD, LOG_FRAGMENT, 3,
+                        "\tchanging %d to identical\n", i-1);
+                } else {
+                    set_translation(dcontext, &entries, &num_entries, i,
+                                    (ushort) (cpc - f->start_pc),
+                                    app, true/*identical*/, instr_is_our_mangling(inst));
+                    i++;
+                }
+                last_contig = false;
+            } else if ((i == 0 && app != NULL && !instr_is_our_mangling(inst))
+                       || app != last_translation + last_len) {
+                /* either 1st loop iter w/ app instr & no prefix, or else probably
+                 * a follow-ubr, so create a new contig entry
+                 */
+                set_translation(dcontext, &entries, &num_entries, i,
+                                (ushort) (cpc - f->start_pc),
+                                app, false/*contig*/, instr_is_our_mangling(inst));
+                last_contig = true;
+                i++;
+            } /* else, contig continues */
+        } else {
+            if (app != last_translation) {
+                /* no longer in an identical region */
+                ASSERT(i > 0);
+                /* If we have translations "x x+1 x+1 x+2 x+3" we can more
+                 * efficiently encode with a new contig entry at the 2nd
+                 * x+1 rather than an identical entry there followed by a
+                 * contig entry for x+2.
+                 */
+                if (app == last_translation + last_len &&
+                    entries[i-1].cache_offs == cpc - last_len - f->start_pc) {
+                    /* convert prev identical into contig */
+                    ASSERT(TEST(TRANSLATE_IDENTICAL, entries[i-1].flags));
+                    entries[i-1].flags &= ~TRANSLATE_IDENTICAL;
+                    LOG(THREAD, LOG_FRAGMENT, 3,
+                        "\tchanging %d to contig\n", i-1);
+                } else {
+                    /* probably a follow-ubr, so create a new contig entry */
+                    set_translation(dcontext, &entries, &num_entries, i,
+                                    (ushort) (cpc - f->start_pc), app, false/*contig*/,
+                                    instr_is_our_mangling(inst));
+                    last_contig = true;
+                    i++;
+                }
+            }
+        }
+        last_translation = app;
+
+        /* We need to make a new entry if the our-mangling flag changed */
+        if (i > 0 && i == prev_i && instr_is_our_mangling(inst) !=
+            TEST(TRANSLATE_OUR_MANGLING, entries[i-1].flags)) {
+            /* our manglings are usually identical */
+            bool identical = instr_is_our_mangling(inst);
+            set_translation(dcontext, &entries, &num_entries, i,
+                            (ushort) (cpc - f->start_pc),
+                            app, identical, instr_is_our_mangling(inst));
+            last_contig = !identical;
+            i++;
+        }
+        last_len = instr_length(dcontext, inst);
+        cpc += last_len;
+        ASSERT(CHECK_TRUNCATE_TYPE_ushort(cpc - f->start_pc));
+    }
+    /* exit stubs can be examined after app code is gone, so we don't need
+     * to store any info on them here
+     */
+    
+    /* free the instrlist_t elements */
+    if (existing_ilist == NULL)
+        instrlist_clear_and_destroy(dcontext, ilist);
+
+    /* now copy into right-sized array */
+    info = translation_info_alloc(dcontext, i);
+    memcpy(info->translation, entries, sizeof(translation_entry_t) * i);
+    HEAP_ARRAY_FREE(GLOBAL_DCONTEXT, entries, translation_entry_t,
+                    num_entries, ACCT_OTHER, PROTECTED);
+
+    STATS_INC(translations_computed);
+
+    DOLOG(3, LOG_INTERP, {
+        translation_info_print(info, f->start_pc, THREAD);
+    });
+    
+    return info;
+}
+
+#ifdef INTERNAL
+void
+stress_test_recreate_state(dcontext_t *dcontext, fragment_t *f, instrlist_t *ilist)
+{
+    dr_mcontext_t mc;
+    bool res;
+    cache_pc cpc;
+    instr_t *in;
+    static const reg_t STRESS_XSP_INIT = 0x08000000; /* arbitrary */
+    bool success_so_far = true;
+    bool inside_mangle_region = false;
+    bool spill_xcx_outstanding = false;
+    reg_id_t reg;
+    bool spill;
+    int xsp_adjust = 0;
+    app_pc mangle_translation = f->tag;
+
+    LOG(THREAD, LOG_INTERP, 3, "Testing restoring state fragment #%d\n",
+        GLOBAL_STAT(num_fragments));
+
+    if (TEST(FRAG_IS_TRACE, f->flags)) {
+        /* decode_fragment() does not set the our-mangling bits, nor the
+         * translation fields (to distinguish back-to-back mangling
+         * regions): not ideal to test using part of what we're testing but
+         * better than nothing
+         */
+        ilist = recreate_fragment_ilist(dcontext, NULL, &f, NULL, true/*mangle*/
+                                        _IF_CLIENT(true/*call client*/));
+    }
+
+    cpc = FCACHE_ENTRY_PC(f);
+    for (in = instrlist_first(ilist); in != NULL;
+         cpc += instr_length(dcontext, in), in = instr_get_next(in)) {
+        /* PR 267260: we're only testing mangling regions.
+         * FIXME: also verify rip-relative mangling translation
+         */
+        if (inside_mangle_region &&
+            (!instr_is_our_mangling(in) ||
+             /* handle adjacent mangle regions */
+             (TEST(FRAG_IS_TRACE, f->flags) /* we have translation only for traces */ &&
+              mangle_translation != instr_get_translation(in)))) {
+            /* reset */
+            LOG(THREAD, LOG_INTERP, 3, "  out of mangling region\n");
+            inside_mangle_region = false;
+            xsp_adjust = 0;
+            success_so_far = true;
+            spill_xcx_outstanding = false;
+            /* go ahead and fall through and ensure we succeed w/ 0 xsp adjust */
+        }
+        if (instr_is_our_mangling(in)) {
+            if (!inside_mangle_region) {
+                inside_mangle_region = true;
+                LOG(THREAD, LOG_INTERP, 3, "  entering mangling region\n");
+                mangle_translation = instr_get_translation(in);
+            } else {
+                ASSERT(!TEST(FRAG_IS_TRACE, f->flags) ||
+                       mangle_translation == instr_get_translation(in));
+            }
+
+            mc.xcx = (reg_t) get_tls(os_tls_offset
+                                     ((ushort)reg_spill_tls_offs(REG_XCX))) + 1;
+            mc.xsp = STRESS_XSP_INIT;
+            mc.pc = cpc;
+            LOG(THREAD, LOG_INTERP, 3,
+                "  restoring cpc="PFX", xsp="PFX"\n", mc.pc, mc.xsp);
+            res = recreate_app_state(dcontext, &mc, false/*just registers*/);
+            LOG(THREAD, LOG_INTERP, 3,
+                "  restored res=%d pc="PFX", xsp="PFX" vs "PFX", xcx="PFX" vs "PFX"\n",
+                res, mc.pc, mc.xsp, STRESS_XSP_INIT -/*negate*/ xsp_adjust,
+                mc.xcx, get_tls(os_tls_offset((ushort)reg_spill_tls_offs(REG_XCX))));
+            /* We should only have failures at tail end of mangle regions.
+             * No instrs after a failing instr should touch app memory.
+             */
+            ASSERT(success_so_far /* ok to fail */ ||
+                   (!res &&
+                    (instr_is_reg_spill_or_restore(dcontext, in, NULL, NULL, NULL) ||
+                     (!instr_reads_memory(in) && !instr_writes_memory(in)))));
+
+            /* check that xsp and xcx are adjusted properly */
+            ASSERT(mc.xsp == STRESS_XSP_INIT -/*negate*/ xsp_adjust);
+            ASSERT(!spill_xcx_outstanding ||
+                   mc.xcx == (reg_t)
+                   get_tls(os_tls_offset((ushort)reg_spill_tls_offs(REG_XCX))));
+
+            if (success_so_far && !res)
+                success_so_far = false;
+            instr_check_xsp_mangling(dcontext, in, &xsp_adjust);
+            if (xsp_adjust != 0)
+                LOG(THREAD, LOG_INTERP, 3, "  xsp_adjust=%d\n", xsp_adjust);
+            if (instr_is_reg_spill_or_restore(dcontext, in, NULL, &spill, &reg) &&
+                reg == REG_XCX)
+                spill_xcx_outstanding = spill;
+        }
+    }
+    if (TEST(FRAG_IS_TRACE, f->flags)) {
+        instrlist_clear_and_destroy(dcontext, ilist);
+    }
+}
+#endif /* INTERNAL */
+
+/* END TRANSLATION CODE ***********************************************************/
+
+
+/* For 32-bit linux apps on 64-bit kernels we assume that all syscalls that
+ * we use this for are ok w/ int (i.e., we don't need a sys{call,enter} version).
+ */
+byte *
+get_global_do_syscall_entry()
+{
+    int method = get_syscall_method();
+    if (method == SYSCALL_METHOD_INT) {
+#ifdef WINDOWS
+        if (DYNAMO_OPTION(sygate_int))
+            return (byte *)global_do_syscall_sygate_int;
+        else
+#endif
+            return (byte *)global_do_syscall_int;
+    } else if (method == SYSCALL_METHOD_SYSENTER) {
+#ifdef WINDOWS
+        if (DYNAMO_OPTION(sygate_sysenter))
+            return (byte *)global_do_syscall_sygate_sysenter;
+        else
+            return (byte *)global_do_syscall_sysenter;
+#else
+        return (byte *)global_do_syscall_int;
+#endif
+    }
+#ifdef WINDOWS
+    else if (method == SYSCALL_METHOD_WOW64)
+        return (byte *)global_do_syscall_wow64;
+#endif
+    else if (method == SYSCALL_METHOD_SYSCALL) {
+#ifdef X64
+        return (byte *)global_do_syscall_syscall;
+#else
+# ifdef WINDOWS
+        ASSERT_NOT_IMPLEMENTED(false && "PR 205898: 32-bit syscall on Windows NYI");
+# else
+        return (byte *)global_do_syscall_int;
+# endif
+#endif
+    } else {
+#ifdef LINUX
+        /* PR 205310: we sometimes have to execute syscalls before we
+         * see an app syscall: for a signal default action, e.g.
+         */
+        return (byte *)global_do_syscall_int;
+#else
+        ASSERT_NOT_REACHED();
+#endif
+    }
+    return NULL;
+}
+
+/* used only by cleanup_and_terminate to avoid the sysenter 
+ * sygate hack version */
+byte *
+get_cleanup_and_terminate_global_do_syscall_entry()
+{
+    /* see note above: for 32-bit linux apps we use int.
+     * xref PR 332427 as well where sysenter causes a crash
+     * if called from cleanup_and_terminate() where ebp is
+     * left pointing to the old freed stack.
+     */
+#if defined(WINDOWS) || defined(X64)
+    if (get_syscall_method() == SYSCALL_METHOD_SYSENTER)
+        return (byte *)global_do_syscall_sysenter;
+    else
+#endif
+#ifdef WINDOWS
+    if (get_syscall_method() == SYSCALL_METHOD_WOW64)
+        return (byte *)global_do_syscall_wow64_index0;
+    else
+#endif
+        return get_global_do_syscall_entry();
+}
+
+#ifdef LINUX
+/* PR 212570: for sysenter support we need to regain control after the
+ * kernel sets eip to a hardcoded user-mode address on the vsyscall page.
+ * The vsyscall code layout is as follows:
+ *     0xffffe400 <__kernel_vsyscall+0>:       push   %ecx
+ *     0xffffe401 <__kernel_vsyscall+1>:       push   %edx
+ *     0xffffe402 <__kernel_vsyscall+2>:       push   %ebp
+ *     0xffffe403 <__kernel_vsyscall+3>:       mov    %esp,%ebp
+ *     0xffffe405 <__kernel_vsyscall+5>:       sysenter 
+ *   nops for alignment of return point:
+ *     0xffffe407 <__kernel_vsyscall+7>:       nop    
+ *     0xffffe408 <__kernel_vsyscall+8>:       nop    
+ *     0xffffe409 <__kernel_vsyscall+9>:       nop    
+ *     0xffffe40a <__kernel_vsyscall+10>:      nop    
+ *     0xffffe40b <__kernel_vsyscall+11>:      nop    
+ *     0xffffe40c <__kernel_vsyscall+12>:      nop    
+ *     0xffffe40d <__kernel_vsyscall+13>:      nop    
+ *   system call restart point:
+ *     0xffffe40e <__kernel_vsyscall+14>:      jmp    0xffffe403 <__kernel_vsyscall+3>
+ *   system call normal return point:
+ *     0xffffe410 <__kernel_vsyscall+16>:      pop    %ebp
+ *     0xffffe411 <__kernel_vsyscall+17>:      pop    %edx
+ *     0xffffe412 <__kernel_vsyscall+18>:      pop    %ecx
+ *     0xffffe413 <__kernel_vsyscall+19>:      ret    
+ *
+ * For randomized vsyscall page locations we can mark the page +w and
+ * write to it.  For now, for simplicity, we focus only on that case;
+ * for vsyscall page at un-reachable 0xffffe000 we bail out and use
+ * ints for now (perf hit but works).  PR 288330 covers leaving
+ * as sysenters.
+ *
+ * There are either nops or garbage after the ret, so we clobber one
+ * byte past the ret to put in a rel32 jmp (an alternative is to do
+ * rel8 jmp into the nop area and have a rel32 jmp there).  We
+ * cleverly copy the 4 bytes of displaced code into the nop area, so
+ * that 1) we don't have to allocate any memory and 2) we don't have
+ * to do any extra work in dispatch, which will naturally go to the
+ * post-system-call-instr pc.
+ *
+ * Using a hook is much simpler than clobbering the retaddr, which is what
+ * Windows does and then has to spend a lot of effort juggling transparency
+ * and control on asynch in/out events.
+ */
+
+#define VSYS_DISPLACED_LEN 4
+
+static bool
+hook_vsyscall(dcontext_t *dcontext)
+{
+    bool res = true;
+    instr_t instr;
+    byte *pc;
+    uint num_nops = 0;
+    uint prot;
+
+    ASSERT(DATASEC_WRITABLE(DATASEC_RARELY_PROT));
+    IF_X64(ASSERT_NOT_REACHED()); /* no sysenter support on x64 */
+    ASSERT(vsyscall_page_start != NULL && vsyscall_syscall_end_pc != NULL);
+
+    instr_init(dcontext, &instr);
+    pc = vsyscall_syscall_end_pc;
+    do {
+        instr_reset(dcontext, &instr);
+        pc = decode(dcontext, pc, &instr);
+        if (instr_is_nop(&instr))
+            num_nops++;
+    } while (instr_is_nop(&instr));
+    vsyscall_sysenter_return_pc = pc;
+    ASSERT(instr_get_opcode(&instr) == OP_jmp_short);
+
+    /* We fail if the pattern looks different */
+#define CHECK(x) do {                                 \
+    if (!(x)) {                                       \
+        ASSERT(false && "vsyscall pattern mismatch"); \
+        res = false;                                  \
+        goto hook_vsyscall_return;                    \
+    }                                                 \
+} while (0);
+
+    CHECK(num_nops >= VSYS_DISPLACED_LEN);
+
+    /* Only now that we've set vsyscall_sysenter_return_pc do we check writability */
+    if (!DYNAMO_OPTION(hook_vsyscall)) {
+        res = false;
+        goto hook_vsyscall_return;
+    }
+    get_memory_info(vsyscall_page_start, NULL, NULL, &prot);
+    if (!TEST(MEMPROT_WRITE, prot)) {
+        res = set_protection(vsyscall_page_start, PAGE_SIZE, prot|MEMPROT_WRITE);
+        if (!res)
+            goto hook_vsyscall_return;
+    }
+
+    LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1,
+        "Hooking vsyscall page @ "PFX"\n", vsyscall_sysenter_return_pc);
+
+    /* The 5 bytes we'll clobber: */
+    instr_reset(dcontext, &instr);
+    pc = decode(dcontext, pc, &instr);
+    CHECK(instr_get_opcode(&instr) == OP_pop);
+    instr_reset(dcontext, &instr);
+    pc = decode(dcontext, pc, &instr);
+    CHECK(instr_get_opcode(&instr) == OP_pop);
+    instr_reset(dcontext, &instr);
+    pc = decode(dcontext, pc, &instr);
+    CHECK(instr_get_opcode(&instr) == OP_pop);
+    instr_reset(dcontext, &instr);
+    pc = decode(dcontext, pc, &instr);
+    CHECK(instr_get_opcode(&instr) == OP_ret);
+    /* Sometimes the next byte is a nop, sometimes it's non-code */
+    ASSERT(*pc == RAW_OPCODE_nop || *pc == 0);
+
+    /* FIXME: at some point we should pull out all the hook code from
+     * callback.c into an os-neutral location.  For now, this hook
+     * is very special-case and simple.
+     */
+
+    /* For thread synch, the datasec prot lock will serialize us (FIXME: do this at
+     * init time instead, when see [vdso] page in maps file?)
+     */
+
+    CHECK(pc - vsyscall_sysenter_return_pc == VSYS_DISPLACED_LEN);
+    ASSERT(pc + 1/*nop*/ - vsyscall_sysenter_return_pc == JMP_LONG_LENGTH);
+    CHECK(num_nops >= pc - vsyscall_sysenter_return_pc);
+    memcpy(vsyscall_syscall_end_pc, vsyscall_sysenter_return_pc,
+           /* we don't copy the 5th byte to preserve nop for nice disassembly */
+           pc - vsyscall_sysenter_return_pc);
+    insert_relative_jump(vsyscall_sysenter_return_pc,
+                         /* we require a thread-shared fcache_return */
+                         after_do_shared_syscall_addr(dcontext),
+                         NOT_HOT_PATCHABLE);
+
+    if (!TEST(MEMPROT_WRITE, prot)) {
+        /* we don't override res here since not much point in not using the
+         * hook once its in if we failed to re-protect: we're going to have to
+         * trust the app code here anyway */
+        DEBUG_DECLARE(bool ok =)
+            set_protection(vsyscall_page_start, PAGE_SIZE, prot);
+        ASSERT(ok);
+    }
+ hook_vsyscall_return:
+    instr_free(dcontext, &instr);
+    return res;
+#undef CHECK
+}
+
+bool
+unhook_vsyscall(void)
+{
+    uint prot;
+    bool res;
+    uint len = VSYS_DISPLACED_LEN;
+    if (get_syscall_method() != SYSCALL_METHOD_SYSENTER)
+        return false;
+    ASSERT(!sysenter_hook_failed);
+    ASSERT(vsyscall_sysenter_return_pc != NULL);
+    ASSERT(vsyscall_syscall_end_pc != NULL);
+    get_memory_info(vsyscall_page_start, NULL, NULL, &prot);
+    if (!TEST(MEMPROT_WRITE, prot)) {
+        res = set_protection(vsyscall_page_start, PAGE_SIZE, prot|MEMPROT_WRITE);
+        if (!res)
+            return false;
+    }
+    memcpy(vsyscall_sysenter_return_pc, vsyscall_syscall_end_pc, len);
+    /* we do not restore the 5th (junk/nop) byte (we never copied it) */
+    memset(vsyscall_syscall_end_pc, RAW_OPCODE_nop, len);
+    if (!TEST(MEMPROT_WRITE, prot)) {
+        res = set_protection(vsyscall_page_start, PAGE_SIZE, prot);
+        ASSERT(res);
+    }
+    return true;
+}
+#endif
+
+void
+check_syscall_method(dcontext_t *dcontext, instr_t *instr)
+{
+    int new_method = SYSCALL_METHOD_UNINITIALIZED;
+    if (instr_get_opcode(instr) == OP_int)
+        new_method = SYSCALL_METHOD_INT;
+    else if (instr_get_opcode(instr) == OP_sysenter)
+        new_method = SYSCALL_METHOD_SYSENTER;
+    else if (instr_get_opcode(instr) == OP_syscall)
+        new_method = SYSCALL_METHOD_SYSCALL;
+#ifdef WINDOWS
+    else if (instr_get_opcode(instr) == OP_call_ind)
+        new_method = SYSCALL_METHOD_WOW64;
+#endif
+    else
+        ASSERT_NOT_REACHED();
+
+    if (new_method == SYSCALL_METHOD_SYSENTER ||
+        IF_X64_ELSE(false, new_method == SYSCALL_METHOD_SYSCALL)) {
+        DO_ONCE({
+            /* FIXME: DO_ONCE will unprot and reprot, and here we unprot again */
+            SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+            /* FIXME : using the raw-bits as the app pc for the instr is
+             * not really supported, but places in monitor assume it as well */
+            ASSERT(instr_raw_bits_valid(instr) && 
+                   !instr_has_allocated_bits(instr));
+            /* Some places (such as clean_syscall_wrapper) assume that only int system
+             * calls are used in older versions of windows. */
+            IF_WINDOWS(ASSERT(get_os_version() > WINDOWS_VERSION_2000 &&
+                              "Expected int syscall method on NT and 2000"));
+            /* Used by SYSCALL_PC in win32/os.c for non int system calls */
+            IF_WINDOWS(app_sysenter_instr_addr = instr_get_raw_bits(instr));
+            /* we expect, only on XP and later or on recent linux kernels, 
+             * indirected syscalls through a certain page, which we record here
+             * FIXME: don't allow anyone to make this region writable?
+             */
+            /* FIXME : we need to verify that windows lays out all of the 
+             * syscall stuff as expected on AMD chips: xref PR 205898.
+             */
+            /* FIXME: bootstrapping problem...would be nicer to read ahead and find
+             * syscall before needing to know about page it's on, but for now we just
+             * check if our initial assignments were correct
+             */
+            vsyscall_syscall_end_pc = instr_get_raw_bits(instr) +
+                instr_length(dcontext, instr);
+            IF_WINDOWS({
+                /* for XP sp0,1 (but not sp2) and 03 fixup boostrap values */
+                if (vsyscall_page_start ==  VSYSCALL_PAGE_START_BOOTSTRAP_VALUE) {
+                    vsyscall_page_start = (app_pc) PAGE_START(instr_get_raw_bits(instr));
+                    ASSERT(vsyscall_page_start == VSYSCALL_PAGE_START_BOOTSTRAP_VALUE);
+                }
+                if (vsyscall_after_syscall == VSYSCALL_AFTER_SYSCALL_BOOTSTRAP_VALUE) {
+                    /* for XP sp0,1 and 03 the ret is immediately after the 
+                     * sysenter instruction */
+                    vsyscall_after_syscall = instr_get_raw_bits(instr) +
+                        instr_length(dcontext, instr);
+                    ASSERT(vsyscall_after_syscall ==
+                           VSYSCALL_AFTER_SYSCALL_BOOTSTRAP_VALUE);
+                }
+            });
+            /* For linux, we should have found "[vdso]" in the maps file */
+            IF_LINUX(ASSERT(vsyscall_page_start != NULL &&
+                            vsyscall_page_start ==
+                            (app_pc) PAGE_START(instr_get_raw_bits(instr))));
+            LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 2,
+                "Found vsyscall @ "PFX" => page "PFX", post "PFX"\n",
+                instr_get_raw_bits(instr), vsyscall_page_start,
+                IF_WINDOWS_ELSE(vsyscall_after_syscall, vsyscall_syscall_end_pc));
+            /* make sure system call numbers match */
+            IF_WINDOWS(DODEBUG({ check_syscall_numbers(dcontext); }));
+            SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+        });
+    } else {
+#ifdef WINDOWS
+        DO_ONCE({
+            /* FIXME: DO_ONCE will unprot and reprot, and here we unprot again */
+            SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+            /* Close vsyscall page hole.
+             * FIXME: the vsyscall page can still be in use and contain int:
+             * though I have yet to see that case where the page is not marked rx.
+             * On linux the vsyscall page is reached via "call *%gs:0x10", but
+             * sometimes that call ends up at /lib/ld-2.3.4.so:_dl_sysinfo_int80
+             * instead (which is the case when the vsyscall page is marked with no
+             * permissions).
+             */
+            LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 2,
+                "Closing vsyscall page hole (int @ "PFX") => page "PFX", post "PFX"\n",
+                instr_get_translation(instr), vsyscall_page_start,
+                IF_WINDOWS_ELSE(vsyscall_after_syscall, vsyscall_syscall_end_pc));
+            vsyscall_page_start = NULL;
+            vsyscall_after_syscall = NULL;
+            ASSERT_CURIOSITY(new_method != SYSCALL_METHOD_WOW64 ||
+                             (get_os_version() > WINDOWS_VERSION_XP &&
+                              is_wow64_process(NT_CURRENT_PROCESS) &&
+                              "Unexpected WOW64 syscall method"));
+            /* make sure system call numbers match */
+            DODEBUG({ check_syscall_numbers(dcontext); });
+            SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+        });
+#else
+        /* On Linux we can't clear vsyscall_page_start as the app will often use both
+         * inlined int and vsyscall sysenter system calls. We handle fixing up for that
+         * in the next ifdef. */
+#endif
+    }
+
+#ifdef LINUX
+    if (new_method != get_syscall_method() &&
+        /* PR 286922: for linux, vsyscall method trumps occasional use of int.  We
+         * update do_syscall for the vsyscall method, and use do_int_syscall for any
+         * int uses. */
+        (new_method != SYSCALL_METHOD_INT ||
+         (get_syscall_method() != SYSCALL_METHOD_SYSENTER &&
+          get_syscall_method() != SYSCALL_METHOD_SYSCALL))) {
+        ASSERT(get_syscall_method() == SYSCALL_METHOD_UNINITIALIZED ||
+               get_syscall_method() == SYSCALL_METHOD_INT);
+        if (new_method == SYSCALL_METHOD_SYSENTER) {
+# ifndef HAVE_TLS
+            /* PR 361894: we use TLS for our vsyscall hook (PR 212570) */
+            FATAL_USAGE_ERROR(SYSENTER_NOT_SUPPORTED, 2, 
+                              get_application_name(), get_application_pid());
+# endif
+            /* Hook the sysenter continuation point so we don't lose control */
+            if (!sysenter_hook_failed && !hook_vsyscall(dcontext)) {
+                /* PR 212570: for now we bail out to using int;
+                 * for performance we should clobber the retaddr and
+                 * keep the sysenters.
+                 */
+                SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+                sysenter_hook_failed = true;
+                SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+                LOG(GLOBAL, LOG_SYSCALLS|LOG_VMAREAS, 1,
+                    "Unable to hook vsyscall page; falling back on int\n");
+            }
+            if (sysenter_hook_failed)
+                new_method = SYSCALL_METHOD_INT;
+        }
+        if (get_syscall_method() == SYSCALL_METHOD_UNINITIALIZED ||
+            new_method != get_syscall_method()) {
+            set_syscall_method(new_method);
+            /* update the places we have emitted syscalls: do_*syscall */
+            update_syscalls(dcontext);
+        }
+    }
+#else
+    /* we assume only single method; else need multiple do_syscalls */
+    ASSERT(new_method == get_syscall_method());
+#endif
+}
+
+int
+get_syscall_method(void)
+{
+    return syscall_method;
+}
+
+/* Does the syscall instruction always return to the invocation point? */
+bool
+does_syscall_ret_to_callsite(void)
+{
+    return (syscall_method == SYSCALL_METHOD_INT || 
+            syscall_method == SYSCALL_METHOD_SYSCALL
+            IF_WINDOWS(|| syscall_method == SYSCALL_METHOD_WOW64));
+}
+
+void
+set_syscall_method(int method)
+{
+    ASSERT(syscall_method == SYSCALL_METHOD_UNINITIALIZED
+           IF_LINUX(|| syscall_method == SYSCALL_METHOD_INT/*PR 286922*/));
+    syscall_method = method;
+}
+
+#ifdef LINUX
+/* PR 313715: If we fail to hook the vsyscall page (xref PR 212570, PR 288330)
+ * we fall back on int, but we have to tweak syscall param #5 (ebp)
+ */
+bool
+should_syscall_method_be_sysenter(void)
+{
+    return sysenter_hook_failed;
+}
+#endif
+
+/* returns the address of the first app syscall instruction we saw (see hack
+ * in win32/os.c that uses this for PRE_SYSCALL_PC, not for general use */
+byte *
+get_app_sysenter_addr()
+{
+    /* FIXME : would like to assert that this has been initialized, but interp
+     * bb_process_convertible_indcall() will use it before we initialize it. */
+    return app_sysenter_instr_addr;
+}
+
+#ifdef NATIVE_RETURN
+app_pc
+ret_tgt_cache_to_app(dcontext_t *dcontext, cache_pc pc)
+{
+    fragment_t wrapper;
+    fragment_t *tgt;
+    linkstub_t *tgt_l;
+    if (!in_fcache(pc)) {
+        return NULL;
+    }
+    LOG(THREAD, LOG_DISPATCH, 3, "ret_tgt_cache_to_app: cache pc "PFX"\n", pc);
+    /* expects pc to point to the jmp after a call
+     * the jmp is an exit cti that targets the fragment for the
+     * app return address
+     */
+    tgt = fragment_pclookup(dcontext, pc, &wrapper);
+    if (tgt == NULL) {
+        /* HACK: fragment may have been "deleted" by removing from htable
+         * find what is pointed to -- must be exit stub
+         */
+        uint offs = *((uint*)(pc+1));
+        /* +5+offs = stub, +6 = &linkstub_t */
+        cache_pc lpc = pc + 5 + offs + 6;
+        /* assumption: it's unlinked...else we'll be off at top of tgt frag! */
+        ASSERT(*(lpc-1) == 0xb8);
+        tgt_l = *((linkstub_t **) lpc);
+        LOG(THREAD, LOG_DISPATCH, 3,
+            "\ttgt is NULL, got intra-stub pc "PFX" (pc+0x%x+11)-> l = "PFX"\n",
+            lpc, offs, tgt_l);
+    } else {
+        tgt_l = FRAGMENT_EXIT_STUBS(tgt);
+        ASSERT(tgt_l != NULL);
+        while (EXIT_CTI_PC(tgt, tgt_l) != pc) {
+            tgt_l = LINKSTUB_NEXT_EXIT(tgt_l);
+            ASSERT(tgt_l != NULL);
+        }
+    }
+    ASSERT(tgt_l != NULL);
+    LOG(THREAD, LOG_DISPATCH, 3, "\tcache pc "PFX" -> app ret addr "PFX"\n",
+        pc, EXIT_TARGET_TAG(dcontext, tgt, tgt_l));
+    return EXIT_TARGET_TAG(dcontext, tgt, tgt_l);
+}
+#endif
+
+void
+copy_mcontext(dr_mcontext_t *src, dr_mcontext_t *dst)
+{
+    /* FIXME: do we need this? */
+    *dst = *src;
+}
+
+/* dumps the context */
+void
+dump_mcontext(dr_mcontext_t *context, file_t f, bool dump_xml)
+{
+    print_file(f, dump_xml ? 
+               "\t<dr_mcontext_t value=\"@"PFX"\""
+               "\n\t\txax=\""PFX"\"\n\t\txbx=\""PFX"\""
+               "\n\t\txcx=\""PFX"\"\n\t\txdx=\""PFX"\""
+               "\n\t\txsi=\""PFX"\"\n\t\txdi=\""PFX"\""
+               "\n\t\txbp=\""PFX"\"\n\t\txsp=\""PFX"\""
+#ifdef X64
+               "\n\t\tr8=\""PFX"\"\n\t\tr9=\""PFX"\""
+               "\n\t\tr10=\""PFX"\"\n\t\tr11=\""PFX"\""
+               "\n\t\tr12=\""PFX"\"\n\t\tr13=\""PFX"\""
+               "\n\t\tr14=\""PFX"\"\n\t\tr15=\""PFX"\""
+#endif
+               : 
+               "dr_mcontext_t @"PFX"\n"
+               "\txax = "PFX"\n\txbx = "PFX"\n\txcx = "PFX"\n\txdx = "PFX"\n"
+               "\txsi = "PFX"\n\txdi = "PFX"\n\txbp = "PFX"\n\txsp = "PFX"\n"
+#ifdef X64
+               "\tr8  = "PFX"\n\tr9  = "PFX"\n\tr10 = "PFX"\n\tr11 = "PFX"\n"
+               "\tr12 = "PFX"\n\tr13 = "PFX"\n\tr14 = "PFX"\n\tr15 = "PFX"\n"
+#endif
+               ,
+               context, 
+               context->xax, context->xbx, context->xcx, context->xdx,
+               context->xsi, context->xdi, context->xbp, context->xsp 
+#ifdef X64
+               , context->r8,  context->r9,  context->r10,  context->r11,
+               context->r12, context->r13, context->r14,  context->r15
+#endif
+               );
+    if (preserve_xmm_caller_saved()) {
+        int i, j;
+        for (i=0; i<NUM_XMM_SAVED; i++) {
+            print_file(f, dump_xml ? "\t\txmm%d= \"0x" : "\txmm%d= 0x", i);
+            /* This would be simpler if we had uint64 fields in dr_xmm_t but
+             * that complicates our struct layouts */
+            for (j = 0; j < 4; j++) {
+                print_file(f, "%08x", context->xmm[i].u32[j]);
+            }
+            print_file(f, dump_xml ? "\"\n" : "\n");
+        }
+    }
+    print_file(f, dump_xml ? 
+               "\n\t\teflags=\""PFX"\"\n\t\tpc=\""PFX"\" />\n"
+               :
+               "\teflags = "PFX"\n\tpc     = "PFX"\n",
+               context->xflags, context->pc);
+}
+
+
+#ifdef PROFILE_RDTSC
+/* This only works on Pentium I or later */
+#ifdef LINUX
+__inline__ uint64 get_time()
+{
+    uint64 x;
+    __asm__ volatile (".byte 0x0f, 0x31" : "=A" (x));
+    return x;
+}
+#else /* WINDOWS */
+uint64 get_time()
+{
+    return __rdtsc(); /* compiler intrinsic */
+}
+#endif
+#endif /* PROFILE_RDTSC */
+
+
+#ifdef DEBUG
+bool
+is_ibl_routine_type(dcontext_t *dcontext, cache_pc target, ibl_branch_type_t branch_type)
+{
+    ibl_type_t ibl_type;
+    DEBUG_DECLARE(bool is_ibl = )
+        get_ibl_routine_type_ex(dcontext, target, &ibl_type _IF_X64(NULL));
+    ASSERT(is_ibl);
+    return (branch_type == ibl_type.branch_type);
+}
+#endif /* DEBUG */
diff --git a/core/x86/arch.h b/core/x86/arch.h
new file mode 100644
index 000000000..9743fc88c
--- /dev/null
+++ b/core/x86/arch.h
@@ -0,0 +1,803 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/* file "arch.h" -- internal x86-specific definitions
+ *
+ * References: 
+ *   "Intel Architecture Software Developer's Manual", 1999.
+ */
+
+#ifndef X86_ARCH_H
+#define X86_ARCH_H
+
+#include <stddef.h> /* for offsetof */
+#include "instr.h" /* for reg_id_t */
+
+/* FIXME: check on all platforms: these are for Fedora 8 and XP SP2
+ * Keep in synch w/ defines in pre_inject_asm.asm
+ */
+#define CS32_SELECTOR 0x23
+#define CS64_SELECTOR 0x33
+
+/* dcontext_t field offsets 
+ * N.B.: DO NOT USE offsetof(dcontext_t) anywhere else if passing to the
+ * dcontext operand construction routines!
+ * Otherwise we will have issues w/ the upcontext offset game below
+ */
+
+/* offs is not raw offset, but includes upcontext size, so we
+ * can tell unprotected from normal!
+ * unprotected are raw 0..sizeof(unprotected_context_t)
+ * protected are raw + sizeof(unprotected_context_t)
+ * (see the instr.c routines for dcontext instr building)
+ * FIXME: we could get rid of this hack if unprotected_context_t == dr_mcontext_t
+ */
+#define PROT_OFFS         (sizeof(unprotected_context_t))
+#define MC_OFFS           (offsetof(unprotected_context_t, mcontext))
+
+#define XAX_OFFSET        ((MC_OFFS) + (offsetof(dr_mcontext_t, xax)))
+#define XBX_OFFSET        ((MC_OFFS) + (offsetof(dr_mcontext_t, xbx)))
+#define XCX_OFFSET        ((MC_OFFS) + (offsetof(dr_mcontext_t, xcx)))
+#define XDX_OFFSET        ((MC_OFFS) + (offsetof(dr_mcontext_t, xdx)))
+#define XSI_OFFSET        ((MC_OFFS) + (offsetof(dr_mcontext_t, xsi)))
+#define XDI_OFFSET        ((MC_OFFS) + (offsetof(dr_mcontext_t, xdi)))
+#define XBP_OFFSET        ((MC_OFFS) + (offsetof(dr_mcontext_t, xbp)))
+#define XSP_OFFSET        ((MC_OFFS) + (offsetof(dr_mcontext_t, xsp)))
+#define XFLAGS_OFFSET     ((MC_OFFS) + (offsetof(dr_mcontext_t, xflags)))
+#define PC_OFFSET         ((MC_OFFS) + (offsetof(dr_mcontext_t, pc)))
+#ifdef X64
+# define R8_OFFSET        ((MC_OFFS) + (offsetof(dr_mcontext_t, r8)))
+# define R9_OFFSET        ((MC_OFFS) + (offsetof(dr_mcontext_t, r9)))
+# define R10_OFFSET       ((MC_OFFS) + (offsetof(dr_mcontext_t, r10)))
+# define R11_OFFSET       ((MC_OFFS) + (offsetof(dr_mcontext_t, r11)))
+# define R12_OFFSET       ((MC_OFFS) + (offsetof(dr_mcontext_t, r12)))
+# define R13_OFFSET       ((MC_OFFS) + (offsetof(dr_mcontext_t, r13)))
+# define R14_OFFSET       ((MC_OFFS) + (offsetof(dr_mcontext_t, r14)))
+# define R15_OFFSET       ((MC_OFFS) + (offsetof(dr_mcontext_t, r15)))
+#endif
+#define XMM_OFFSET        ((MC_OFFS) + (offsetof(dr_mcontext_t, xmm)))
+
+#define ERRNO_OFFSET      (offsetof(unprotected_context_t, errno))
+#define AT_SYSCALL_OFFSET (offsetof(unprotected_context_t, at_syscall))
+
+#define NEXT_TAG_OFFSET        ((PROT_OFFS)+offsetof(dcontext_t, next_tag))
+#define LAST_EXIT_OFFSET       ((PROT_OFFS)+offsetof(dcontext_t, last_exit))
+#define DSTACK_OFFSET          ((PROT_OFFS)+offsetof(dcontext_t, dstack))
+#define NATIVE_EXEC_RETVAL_OFFSET ((PROT_OFFS)+offsetof(dcontext_t, native_exec_retval))
+#define NATIVE_EXEC_RETLOC_OFFSET ((PROT_OFFS)+offsetof(dcontext_t, native_exec_retloc))
+#ifdef RETURN_STACK
+# define RSTACK_OFFSET         ((PROT_OFFS)+offsetof(dcontext_t, rstack))
+# define TOP_OF_RSTACK_OFFSET  ((PROT_OFFS)+offsetof(dcontext_t, top_of_rstack))
+#endif
+#define APP_ERRNO_OFFSET       ((PROT_OFFS)+offsetof(dcontext_t, app_errno))
+
+#define FRAGMENT_FIELD_OFFSET  ((PROT_OFFS)+offsetof(dcontext_t, fragment_field))
+#define PRIVATE_CODE_OFFSET    ((PROT_OFFS)+offsetof(dcontext_t, private_code))
+
+#ifdef WINDOWS
+# define NONSWAPPED_SCRATCH_OFFSET  ((PROT_OFFS)+offsetof(dcontext_t, nonswapped_scratch))
+#endif
+
+#ifdef TRACE_HEAD_CACHE_INCR 
+# define TRACE_HEAD_PC_OFFSET  ((PROT_OFFS)+offsetof(dcontext_t, trace_head_pc))
+#endif
+
+#ifdef NATIVE_RETURN_CALLDEPTH
+# define CALL_DEPTH_OFFSET     ((PROT_OFFS)+offsetof(dcontext_t, call_depth))
+#endif
+
+#ifdef WINDOWS
+# define SYSENTER_STORAGE_OFFSET ((PROT_OFFS)+offsetof(dcontext_t, sysenter_storage))
+# define IGNORE_ENTEREXIT_OFFSET ((PROT_OFFS)+offsetof(dcontext_t, ignore_enterexit))
+#endif
+
+#ifdef CLIENT_INTERFACE
+# define CLIENT_DATA_OFFSET    ((PROT_OFFS)+offsetof(dcontext_t, client_data))
+#endif
+
+#define COARSE_IB_SRC_OFFSET   ((PROT_OFFS)+offsetof(dcontext_t, coarse_exit.src_tag))
+#define COARSE_DIR_EXIT_OFFSET ((PROT_OFFS)+offsetof(dcontext_t, coarse_exit.dir_exit))
+
+int
+reg_spill_tls_offs(reg_id_t reg);
+
+/* Xref the partially overlapping CONTEXT_PRESERVE_XMM */
+static inline bool
+preserve_xmm_caller_saved(void)
+{
+    /* PR 264138: we must preserve xmm0-5 if on a 64-bit Windows kernel.
+     * PR 302107: we must preserve xmm0-15 for 64-bit Linux apps.
+     * PR 306394: for now we do not preserve xmm0-7, which are technically
+     * caller-saved, for 32-bit Windows or Linux (including LOL64).
+     */
+    return IF_X64_ELSE(true,
+                       IF_WINDOWS_ELSE(is_wow64_process(NT_CURRENT_PROCESS), false)) &&
+        proc_has_feature(FEATURE_SSE) /* do xmm registers exist? */;
+}
+
+typedef enum {
+    IBL_UNLINKED,
+    IBL_DELETE,
+#ifdef X64
+    /* PR 257963: trace inline cmp has separate entries b/c it saves flags */
+    IBL_TRACE_CMP,
+    IBL_TRACE_CMP_UNLINKED,
+#endif
+    IBL_LINKED,
+    
+    IBL_TEMPLATE, /* a template is presumed to be always linked */
+    IBL_LINK_STATE_END
+} ibl_entry_point_type_t;
+
+/* we should allow for all {{bb,trace} x {ret,ind call, ind jmp} x {shared, private}} */
+/* combinations of routines which are in turn  x {unlinked, linked} */
+typedef enum {
+    /* FIXME: have a separate flag for private vs shared */
+    IBL_BB_SHARED, 
+    IBL_SOURCE_TYPE_START = IBL_BB_SHARED,
+    IBL_TRACE_SHARED,
+    IBL_BB_PRIVATE,
+    IBL_TRACE_PRIVATE,
+    IBL_COARSE_SHARED, /* no coarse private, for now */
+    IBL_SOURCE_TYPE_END
+} ibl_source_fragment_type_t;
+
+#define DEFAULT_IBL_BB() \
+    (DYNAMO_OPTION(shared_bbs) ? IBL_BB_SHARED : IBL_BB_PRIVATE)
+#define DEFAULT_IBL_TRACE() \
+    (DYNAMO_OPTION(shared_traces) ? IBL_TRACE_SHARED : IBL_TRACE_PRIVATE)
+#define IS_IBL_BB(ibltype) \
+    ((ibltype) == IBL_BB_PRIVATE || (ibltype) == IBL_BB_SHARED)
+#define IS_IBL_TRACE(ibltype) \
+    ((ibltype) == IBL_TRACE_PRIVATE || (ibltype) == IBL_TRACE_SHARED)
+#define IS_IBL_LINKED(ibltype) \
+    ((ibltype) == IBL_LINKED IF_X64(|| (ibltype) == IBL_TRACE_CMP))
+#define IS_IBL_UNLINKED(ibltype) \
+    ((ibltype) == IBL_UNLINKED IF_X64(|| (ibltype) == IBL_TRACE_CMP_UNLINKED))
+
+#define IBL_FRAG_FLAGS(ibl_code) \
+    (IS_IBL_TRACE((ibl_code)->source_fragment_type) ? FRAG_IS_TRACE : 0)
+
+typedef struct
+{
+    /* these could be bit fields, if needed */
+    ibl_entry_point_type_t link_state;
+    ibl_source_fragment_type_t source_fragment_type;
+    ibl_branch_type_t branch_type;
+} ibl_type_t;
+
+#ifdef X64
+/* PR 282576: With shared_code_x86, GLOBAL_DCONTEXT no longer specifies
+ * a unique generated_code_t.  Rather than add GLOBAL_DCONTEXT_X86 everywhere,
+ * we add mode parameters to a handful of routines that take in GLOBAL_DCONTEXT.
+ */
+typedef enum {
+    GENCODE_X64 = 0,
+    GENCODE_X86,
+    GENCODE_FROM_DCONTEXT,
+} gencode_mode_t;
+# define MODE_OVERRIDE(x86_mode) ((x86_mode) ? GENCODE_X86 : GENCODE_X64)
+# define SHARED_GENCODE(x86_mode) \
+    get_shared_gencode(GLOBAL_DCONTEXT, MODE_OVERRIDE(x86_mode))
+# define SHARED_GENCODE_MATCH_THREAD(dc) get_shared_gencode(dc, GENCODE_FROM_DCONTEXT)
+# define THREAD_GENCODE(dc) get_emitted_routines_code(dc, GENCODE_FROM_DCONTEXT)
+#else
+# define SHARED_GENCODE(b) get_shared_gencode(GLOBAL_DCONTEXT)
+# define THREAD_GENCODE(dc) get_emitted_routines_code(dc)
+# define SHARED_GENCODE_MATCH_THREAD(dc) get_shared_gencode(dc)
+#endif
+
+cache_pc get_ibl_routine_ex(dcontext_t *dcontext, ibl_entry_point_type_t entry_type,
+                            ibl_source_fragment_type_t source_fragment_type,
+                            ibl_branch_type_t branch_type _IF_X64(gencode_mode_t mode));
+cache_pc get_ibl_routine(dcontext_t *dcontext, ibl_entry_point_type_t entry_type,
+                         ibl_source_fragment_type_t source_fragment_type, 
+                         ibl_branch_type_t branch_type);
+cache_pc get_ibl_routine_template(dcontext_t *dcontext, 
+                                  ibl_source_fragment_type_t source_fragment_type, 
+                                  ibl_branch_type_t branch_type
+                                  _IF_X64(gencode_mode_t mode));
+bool get_ibl_routine_type(dcontext_t *dcontext, cache_pc target, ibl_type_t *type);
+bool get_ibl_routine_type_ex(dcontext_t *dcontext, cache_pc target, ibl_type_t *type
+                             _IF_X64(gencode_mode_t *mode_out));
+const char *get_ibl_routine_name(dcontext_t *dcontext, cache_pc target, 
+                                 const char **ibl_brtype_name);
+cache_pc get_trace_ibl_routine(dcontext_t *dcontext, cache_pc current_entry);
+cache_pc get_private_ibl_routine(dcontext_t *dcontext, cache_pc current_entry);
+cache_pc get_shared_ibl_routine(dcontext_t *dcontext, cache_pc current_entry);
+cache_pc get_alternate_ibl_routine(dcontext_t *dcontext, cache_pc current_entry,
+                                   uint flags);
+
+ibl_source_fragment_type_t
+get_source_fragment_type(dcontext_t *dcontext, uint fragment_flags);
+
+const char *get_target_delete_entry_name(dcontext_t *dcontext,
+                                         cache_pc target, 
+                                         const char **ibl_brtype_name);
+
+#define GET_IBL_TARGET_TABLE(branch_type, target_trace_table)               \
+    ((target_trace_table) ? offsetof(per_thread_t, trace_ibt[(branch_type)]) : \
+     offsetof(per_thread_t, bb_ibt[(branch_type)]))
+
+
+#ifdef RETURN_STACK
+cache_pc return_lookup_routine(dcontext_t *dcontext);
+cache_pc unlinked_return_routine(dcontext_t *dcontext);
+#endif
+#ifdef WINDOWS
+/* PR 282576: These separate routines are ugly, but less ugly than adding param to
+ * after_shared_syscall_code(), which is called in many places and usually passed a
+ * non-global dcontext; also less ugly than adding GLOBAL_DCONTEXT_X86.
+ */
+cache_pc
+shared_syscall_routine_ex(dcontext_t *dcontext _IF_X64(gencode_mode_t mode));
+cache_pc
+unlinked_shared_syscall_routine_ex(dcontext_t *dcontext _IF_X64(gencode_mode_t mode));
+cache_pc shared_syscall_routine(dcontext_t *dcontext);
+cache_pc unlinked_shared_syscall_routine(dcontext_t *dcontext);
+#endif
+#ifdef TRACE_HEAD_CACHE_INCR
+cache_pc trace_head_incr_routine(dcontext_t *dcontext);
+cache_pc trace_head_incr_shared_routine(IF_X64(gencode_mode_t mode));
+#endif
+
+/* in mangle.c but not exported to non-x86 files */
+void mangle_insert_clone_code(dcontext_t *dcontext, instrlist_t *ilist,
+                              instr_t *instr, instr_t *pc_to_ecx, bool skip
+                              _IF_X64(gencode_mode_t mode));
+void
+set_selfmod_sandbox_offsets(dcontext_t *dcontext);
+uint
+insert_push_all_registers(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+                          bool stack_align16);
+void
+insert_pop_all_registers(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+                          bool stack_align16);
+bool
+parameters_stack_padded(void);
+/* Inserts a complete call to callee with the passed-in arguments */
+void
+insert_meta_call_vargs(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+                       bool clean_call, void *callee, uint num_args, va_list ap);
+void
+insert_get_mcontext_base(dcontext_t *dcontext, instrlist_t *ilist, 
+                         instr_t *where, reg_id_t reg);
+uint
+prepare_for_clean_call(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr);
+void
+cleanup_after_clean_call(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr);
+void convert_to_near_rel(dcontext_t *dcontext, instr_t *instr);
+void convert_to_near_rel_meta(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr);
+int find_syscall_num(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr);
+bool insert_selfmod_sandbox(dcontext_t *dcontext, instrlist_t *ilist, uint flags,
+                            app_pc start_pc, app_pc end_pc, /* end is open */
+                            bool record_translation, bool for_cache);
+
+/* offsets within local_state_t used for specific scratch purposes */
+enum {
+    /* ok for this guy to overlap w/ others since he is pre-cache */
+    FCACHE_ENTER_TARGET_SLOT    = TLS_XAX_SLOT,
+    /* FIXME: put register name in each enum name to avoid conflicts
+     * when mixed with raw slot names?
+     */
+    /* ok for the next_tag and direct_stub to overlap as next_tag is
+     * used for sysenter shared syscall mangling, which uses an
+     * indirect stub.
+     */
+    MANGLE_NEXT_TAG_SLOT        = TLS_XAX_SLOT,
+    DIRECT_STUB_SPILL_SLOT      = TLS_XAX_SLOT,
+    INDIRECT_STUB_SPILL_SLOT    = TLS_XBX_SLOT,
+    MANGLE_XCX_SPILL_SLOT       = TLS_XCX_SLOT,
+    /* FIXME: edi is used as the base, yet I labeled this slot for edx
+     * since it's next in the progression -- change one or the other?
+     * (this is case 5239)
+     */
+    DCONTEXT_BASE_SPILL_SLOT    = TLS_XDX_SLOT,
+    PREFIX_XAX_SPILL_SLOT       = TLS_XAX_SLOT,
+#ifdef HASHTABLE_STATISTICS
+    HTABLE_STATS_SPILL_SLOT     = TLS_HTABLE_STATS_SLOT,
+#endif
+};
+
+void mangle(dcontext_t *dcontext, instrlist_t *ilist, uint flags,
+            bool mangle_calls, bool record_translation);
+
+/* in interp.c but not exported to non-x86 files */
+bool must_not_be_inlined(app_pc pc);
+
+/* A simple linker to give us indirection for patching after relocating structures */
+typedef struct patch_entry_t {
+    union {
+        instr_t *instr;       /* used before instructions are encoded */
+        size_t   offset;      /* offset in instruction stream */
+    } where;
+    ptr_uint_t value_location_offset; /* location containing value to be updated */
+                 /* offset from dcontext->fragment_field (usually pt->trace.field),
+                  * or an absolute address */
+    ushort patch_flags; /* whether to use the address of location or its value */
+    short  instr_offset; /* desired offset within instruction, 
+                            negative offsets are from end of instruction */
+} patch_entry_t;
+
+enum {
+    MAX_PATCH_ENTRIES = 
+#ifdef HASHTABLE_STATISTICS
+6 +     /* will need more only for statistics */
+#endif  
+    7, /* we use 5 normally, 7 w/ -atomic_inlined_linking and inlining */
+    /* Patch entry flags */
+    /* Patch offset entries for dynamic updates from input variables */
+    PATCH_TAKE_ADDRESS      = 0x01, /* use computed address if set, value at address otherwise */
+    PATCH_PER_THREAD        = 0x02, /* address is relative to the per_thread_t thread local field */
+    PATCH_UNPROT_STAT       = 0x04, /* address is (unprot_ht_statistics_t offs << 16) | (stats offs) */
+    /* Patch offset markers update an output variable in encode_with_patch_list */
+    PATCH_MARKER            = 0x08, /* if set use only as a static marker */
+    PATCH_ASSEMBLE_ABSOLUTE = 0x10, /* if set retrieve an absolute pc into given target address, 
+                                      otherwise relative to start pc */
+    PATCH_OFFSET_VALID      = 0x20, /* if set use patch_entry_t.where.offset;
+                                     * else patch_entry_t.where.instr */
+    PATCH_UINT_SIZED        = 0x40, /* if set value is uint-sized; else pointer-sized */
+};
+
+typedef enum {
+    PATCH_TYPE_ABSOLUTE     = 0x0, /* link with absolute address, updated dynamically */
+    PATCH_TYPE_INDIRECT_XDI = 0x1, /* linked with indirection through EDI, no updates */
+    PATCH_TYPE_INDIRECT_FS  = 0x2, /* linked with indirection through FS, no updates */
+} patch_list_type_t;
+
+typedef struct patch_list_t {
+    ushort num_relocations;
+    ushort /* patch_list_type_t */ type;
+    patch_entry_t entry[MAX_PATCH_ENTRIES];
+} patch_list_t;
+
+void
+init_patch_list(patch_list_t *patch, patch_list_type_t type);
+
+void
+add_patch_marker(patch_list_t *patch, instr_t *instr, ushort patch_flags,
+                 short instr_offset, ptr_uint_t *target_offset /* OUT */);
+
+int
+encode_with_patch_list(dcontext_t *dcontext, patch_list_t *patch, 
+                       instrlist_t *ilist, cache_pc start_pc);
+
+/* Defines book-keeping structures needed for an indirect branch lookup routine */
+typedef struct ibl_code_t {
+    bool initialized:1; /* currently only used for ibl routines */
+    bool thread_shared_routine:1;
+    bool ibl_head_is_inlined:1;
+    byte *indirect_branch_lookup_routine;
+#ifdef X64
+    /* PR 257963: trace inline cmp has already saved eflags */
+    byte *trace_cmp_entry;
+    byte *trace_cmp_unlinked;
+    bool x86_mode; /* Is this code for 32-bit (x86 mode)? */
+#endif
+    byte *unlinked_ibl_entry;
+    byte *target_delete_entry;
+    uint ibl_routine_length;
+    /* offsets into ibl routine */
+    patch_list_t ibl_patch;
+    ibl_branch_type_t branch_type;
+    ibl_source_fragment_type_t source_fragment_type;
+
+    /* bookkeeping for the inlined ibl stub template, if inlining */
+    byte *inline_ibl_stub_template;
+    patch_list_t ibl_stub_patch;
+    uint inline_stub_length;
+    /* for atomic_inlined_linking we store the linkstub twice so need to update
+     * two offsets */
+    uint inline_linkstub_first_offs;
+    uint inline_linkstub_second_offs;
+    uint inline_unlink_offs;
+    uint inline_linkedjmp_offs;
+    uint inline_unlinkedjmp_offs;
+
+#ifdef HASHTABLE_STATISTICS
+    /* need two offsets to get to stats, since in unprotected memory */
+    uint unprot_stats_offset; 
+    uint hashtable_stats_offset; 
+    /* e.g. offsetof(per_thread_t, trace) + offsetof(ibl_table_t, bb_ibl_stats) */
+    /* Note hashtable statistics are associated with the hashtable for easier use when sharing IBL routines */
+    uint entry_stats_to_lookup_table_offset; /* offset to (entry_stats - lookup_table)  */
+#endif 
+} ibl_code_t;
+    
+/* Each thread needs its own copy of these routines, but not all
+ * routines here are created in a thread-private: we could save space
+ * by splitting into two separate structs.
+ *
+ * On x64, we only have thread-shared generated routines,
+ * including do_syscall and shared_syscall and detach's post-syscall
+ * continuation (PR 244737).
+ */
+typedef struct _generated_code_t {
+    byte *fcache_enter;
+    byte *fcache_return;
+#ifdef WINDOWS_PC_SAMPLE
+    byte *fcache_enter_return_end;
+#endif
+
+    ibl_code_t trace_ibl[IBL_BRANCH_TYPE_END];
+    ibl_code_t bb_ibl[IBL_BRANCH_TYPE_END];
+    ibl_code_t coarse_ibl[IBL_BRANCH_TYPE_END];
+#ifdef WINDOWS_PC_SAMPLE
+    byte *ibl_routines_end;
+#endif
+
+#ifdef RETURN_STACK
+    byte *return_lookup;
+    byte *unlinked_return;
+#endif
+#ifdef WINDOWS
+    /* for the shared_syscalls option */
+    ibl_code_t shared_syscall_code;
+    byte *shared_syscall;
+    byte *unlinked_shared_syscall;
+    byte *end_shared_syscall; /* just marks end */
+    /* N.B.: these offsets are from the start of unlinked_shared_syscall,
+     * not from shared_syscall (which is later)!!!
+     */
+    /* offsets into shared_syscall routine */
+    uint sys_syscall_offs;
+    /* where to patch to unlink end of syscall thread-wide */
+    uint sys_unlink_offs;
+#endif
+    byte *do_syscall;
+    uint do_syscall_offs; /* offs of pc after actual syscall instr */
+#ifdef WINDOWS
+    byte *fcache_enter_indirect;
+    byte *do_callback_return;
+#else
+    /* PR 286922: we both need an int and a sys{call,enter} do-syscall for
+     * 32-bit apps on 64-bit kernels.  do_syscall is whatever is in
+     * vsyscall, while do_int_syscall is hardcoded to use OP_int.
+     */
+    byte *do_int_syscall;
+    uint do_int_syscall_offs; /* offs of pc after actual syscall instr */
+    byte *do_clone_syscall;
+    uint do_clone_syscall_offs; /* offs of pc after actual syscall instr */
+#endif
+#ifdef LINUX
+    /* PR 212290: can't be static code in x86.asm since it can't be PIC */
+    byte *new_thread_dynamo_start;
+#endif
+#ifdef TRACE_HEAD_CACHE_INCR
+    byte *trace_head_incr;
+#endif
+#ifdef CHECK_RETURNS_SSE2
+    byte *pextrw;
+    byte *pinsrw;
+#endif
+#ifdef WINDOWS_PC_SAMPLE
+    profile_t *profile;
+#endif
+    /* For control redirection from a syscall.
+     * We could make this shared-only and save some space, if we
+     * generated a shared fcache_return in all-private-fragment configs.
+     */
+    byte *reset_exit_stub;
+
+    /* Coarse-grain fragments don't have linkstubs and need custom routines.
+     * Direct exits use entrance stubs that record the target app pc,
+     * while coarse indirect stubs record the source cache cti.
+     */
+    /* FIXME: these two return routines are only needed in the global struct */
+    byte *fcache_return_coarse;
+    byte *trace_head_return_coarse;
+
+    bool writable;
+#ifdef X64
+    bool x86_mode; /* Is this code for 32-bit (x86 mode)? */
+#endif
+
+    /* We store the start of the generated code for simplicity even
+     * though it is always right after this struct; if we really need
+     * to shrink 4 bytes we can remove this field and replace w/
+     * ((char *)TPC_ptr) + sizeof(generated_code_t)
+     */
+    byte *gen_start_pc; /* start of generated code */
+    byte *gen_end_pc; /* end of generated code */
+    byte *commit_end_pc; /* end of committed region */
+    /* generated code follows, ends at gen_end_pc < commit_end_pc */
+} generated_code_t;
+
+/* thread-private generated code */
+fcache_enter_func_t fcache_enter_routine(dcontext_t *dcontext);
+cache_pc fcache_return_routine(dcontext_t *dcontext);
+cache_pc fcache_return_routine_ex(dcontext_t *dcontext _IF_X64(gencode_mode_t mode));
+
+/* thread-shared generated code */
+byte * emit_fcache_enter_shared(dcontext_t *dcontext, generated_code_t *code, byte *pc);
+byte * emit_fcache_return_shared(dcontext_t *dcontext, generated_code_t *code, byte *pc);
+fcache_enter_func_t fcache_enter_shared_routine(dcontext_t *dcontext);
+/* the fcache_return routines are queried by get_direct_exit_target and need more
+ * direct control than the dcontext
+ */
+cache_pc fcache_return_shared_routine(IF_X64(gencode_mode_t mode));
+
+/* coarse-grain generated code */
+byte * emit_fcache_return_coarse(dcontext_t *dcontext, generated_code_t *code, byte *pc);
+byte * emit_trace_head_return_coarse(dcontext_t *dcontext, generated_code_t *code,
+                                     byte *pc);
+cache_pc fcache_return_coarse_routine(IF_X64(gencode_mode_t mode));
+cache_pc trace_head_return_coarse_routine(IF_X64(gencode_mode_t mode));
+
+void protect_generated_code(generated_code_t *code, bool writable);
+
+extern generated_code_t *shared_code;
+#ifdef X64
+extern generated_code_t *shared_code_x86;
+#endif
+
+static inline bool
+is_shared_gencode(generated_code_t *code)
+{
+    if (code == NULL) /* since shared_code_x86 in particular can be NULL */
+        return false;
+#ifdef X64
+    return code == shared_code_x86 || code == shared_code;
+#else
+    return code == shared_code;
+#endif
+}
+
+static inline generated_code_t *
+get_shared_gencode(dcontext_t *dcontext _IF_X64(gencode_mode_t mode))
+{
+#ifdef X64
+    ASSERT(mode != GENCODE_FROM_DCONTEXT || dcontext != GLOBAL_DCONTEXT
+           IF_INTERNAL(IF_CLIENT_INTERFACE(|| dynamo_exited)));
+# if defined(INTERNAL) || defined(CLIENT_INTERFACE)
+    /* PR 302344: this is here only for tracedump_origins */
+    if (dynamo_exited && mode == GENCODE_FROM_DCONTEXT && dcontext == GLOBAL_DCONTEXT)
+        return (get_x86_mode(dcontext) ? shared_code_x86 : shared_code);
+# endif
+    return (mode == GENCODE_X86 ||
+            (mode == GENCODE_FROM_DCONTEXT && dcontext->x86_mode)) ?
+        shared_code_x86 : shared_code;
+#else
+    return shared_code;
+#endif
+}
+
+/* PR 244737: thread-private uses shared gencode on x64 */
+#define USE_SHARED_GENCODE_ALWAYS() IF_X64_ELSE(true, false)
+/* PR 212570: on linux we need a thread-shared do_syscall for our vsyscall hook,
+ * if we have TLS and support sysenter (PR 361894) 
+ */
+#define USE_SHARED_GENCODE()                                         \
+    (USE_SHARED_GENCODE_ALWAYS() || IF_LINUX(IF_HAVE_TLS_ELSE(true, false) ||) \
+     SHARED_FRAGMENTS_ENABLED() || DYNAMO_OPTION(shared_trace_ibl_routine))
+
+/* returns the thread private code or GLOBAL thread shared code */
+static inline generated_code_t*
+get_emitted_routines_code(dcontext_t *dcontext _IF_X64(gencode_mode_t mode))
+{
+    generated_code_t *code;
+    /* This routine exists only because GLOBAL_DCONTEXT is not a real dcontext
+     * structure. Still, useful to wrap all references to private_code. */
+    /* PR 244737: thread-private uses only shared gencode on x64 */
+    /* PR 253431: to distinguish shared x86 gencode from x64 gencode, a dcontext
+     * must be passed in; use get_shared_gencode() for x64 builds */
+    IF_X64(ASSERT(mode != GENCODE_FROM_DCONTEXT || dcontext != GLOBAL_DCONTEXT));
+    if (USE_SHARED_GENCODE_ALWAYS() ||
+        (USE_SHARED_GENCODE() && dcontext == GLOBAL_DCONTEXT)) {
+        code = get_shared_gencode(dcontext _IF_X64(mode));
+    } else {
+        ASSERT(dcontext != GLOBAL_DCONTEXT);
+        /* NOTE thread private code entry points may also refer to shared
+         * routines */
+        code = (generated_code_t *) dcontext->private_code;
+    }
+    return code;
+}
+
+ibl_code_t *get_ibl_routine_code(dcontext_t *dcontext, ibl_branch_type_t branch_type,
+                                 uint fragment_flags);
+ibl_code_t *get_ibl_routine_code_ex(dcontext_t *dcontext, ibl_branch_type_t branch_type,
+                                    uint fragment_flags _IF_X64(gencode_mode_t mode));
+
+/* in emit_utils.c but not exported to non-x86 files */
+byte * emit_inline_ibl_stub(dcontext_t *dcontext, byte *pc, 
+                            ibl_code_t *ibl_code, bool target_trace_table);
+byte * emit_fcache_enter(dcontext_t *dcontext, generated_code_t *code, byte *pc);
+byte * emit_fcache_return(dcontext_t *dcontext, generated_code_t *code, byte *pc);
+byte * emit_indirect_branch_lookup(dcontext_t *dcontext, generated_code_t *code, byte *pc,
+                                   byte *fcache_return_pc,
+                                   bool target_trace_table,
+                                   bool inline_ibl_head,
+                                   ibl_code_t *ibl_code);
+void update_indirect_branch_lookup(dcontext_t *dcontext);
+#ifdef RETURN_STACK
+byte * emit_return_lookup(dcontext_t *dcontext, byte *pc,
+                          byte *indirect_branch_lookup_pc,
+                          byte *unlinked_ib_lookup_pc,
+                          byte **return_lookup_pc);
+#endif
+
+#ifndef WINDOWS
+void update_syscalls(dcontext_t *dcontext);
+#endif
+
+#ifdef WINDOWS
+/* FIXME If we widen the interface any further, do we want to use an options
+ * struct or OR-ed flags to replace the bool args? */
+byte * emit_shared_syscall(dcontext_t *dcontext, generated_code_t *code, byte *pc,
+                           ibl_code_t *ibl_code,
+                           patch_list_t *patch,
+                           byte *ind_br_lookup_pc,
+                           byte *unlinked_ib_lookup_pc,
+                           bool target_trace_table,
+                           bool inline_ibl_head,
+                           bool thread_shared,
+                           byte **shared_syscall_pc);
+
+byte *
+emit_shared_syscall_dispatch(dcontext_t *dcontext, byte *pc);
+
+byte *
+emit_unlinked_shared_syscall_dispatch(dcontext_t *dcontext, byte *pc);
+
+void
+preinsert_get_last_error(dcontext_t *dcontext, instrlist_t *ilist, instr_t *next,
+                         reg_id_t reg_result);
+void
+preinsert_set_last_error(dcontext_t *dcontext, instrlist_t *ilist, instr_t *next,
+                         reg_id_t reg_errno /* saved errno */);
+void emit_patch_syscall(dcontext_t *dcontext, byte *target _IF_X64(gencode_mode_t mode));
+#endif /* WINDOWS */
+
+byte * emit_do_syscall(dcontext_t *dcontext, generated_code_t *code, byte *pc,
+                       byte *fcache_return_pc, bool thread_shared, bool force_int,
+                       uint *syscall_offs /*OUT*/);
+
+#ifdef WINDOWS
+/* PR 282576: These separate routines are ugly, but less ugly than adding param to
+ * the main routines, which are called in many places and usually passed a
+ * non-global dcontext; also less ugly than adding GLOBAL_DCONTEXT_X86.
+ */
+cache_pc
+after_shared_syscall_code_ex(dcontext_t *dcontext _IF_X64(gencode_mode_t mode));
+cache_pc
+after_do_syscall_code_ex(dcontext_t *dcontext _IF_X64(gencode_mode_t mode));
+
+byte * emit_fcache_enter_indirect(dcontext_t *dcontext, generated_code_t *code, byte *pc,
+                                  byte *fcache_return_pc);
+byte * emit_do_callback_return(dcontext_t *dcontext, byte *pc,
+                               byte *fcache_return_pc, bool thread_shared);
+#else
+byte * emit_do_clone_syscall(dcontext_t *dcontext, generated_code_t *code, byte *pc,
+                             byte *fcache_return_pc, bool thread_shared,
+                             uint *syscall_offs /*OUT*/);
+#endif
+
+#ifdef LINUX
+byte * 
+emit_new_thread_dynamo_start(dcontext_t *dcontext, byte *pc);
+
+cache_pc get_new_thread_start(dcontext_t *dcontext _IF_X64(gencode_mode_t mode));
+#endif
+
+#ifdef TRACE_HEAD_CACHE_INCR
+byte *emit_trace_head_incr(dcontext_t *dcontext, byte *pc,
+                           byte *fcache_return_pc);
+byte * 
+emit_trace_head_incr_shared(dcontext_t *dcontext, byte *pc, byte *fcache_return_pc);
+#endif
+
+void
+insert_save_eflags(dcontext_t *dcontext, instrlist_t *ilist, instr_t *where,
+                   uint flags, bool tls, bool absolute);
+void
+insert_restore_eflags(dcontext_t *dcontext, instrlist_t *ilist, instr_t *where,
+                      uint flags, bool tls, bool absolute);
+
+instr_t * create_syscall_instr(dcontext_t *dcontext);
+
+void
+append_shared_get_dcontext(dcontext_t *dcontext, instrlist_t *ilist, bool save_xdi);
+
+void
+append_shared_restore_dcontext_reg(dcontext_t *dcontext, instrlist_t *ilist);
+
+/* in optimize.c */
+instr_t *find_next_self_loop(dcontext_t *dcontext, app_pc tag, instr_t *instr);
+void replace_inst(dcontext_t *dcontext, instrlist_t *ilist, instr_t *old, instr_t *new);
+void remove_redundant_loads(dcontext_t *dcontext, app_pc tag,
+                            instrlist_t *trace);
+void remove_dead_code(dcontext_t *dcontext, app_pc tag, 
+                      instrlist_t *trace);
+
+#ifdef CHECK_RETURNS_SSE2
+/* in retcheck.c */
+void check_return_handle_call(dcontext_t *dcontext, instrlist_t *ilist, instr_t *next);
+void check_return_handle_return(dcontext_t *dcontext, instrlist_t *ilist, instr_t *next);
+void check_return_too_deep(dcontext_t *dcontext,
+                           volatile int errno, volatile reg_t eflags,
+                           volatile reg_t reg_edi, volatile reg_t reg_esi,
+                           volatile reg_t reg_ebp, volatile reg_t reg_esp,
+                           volatile reg_t reg_ebx, volatile reg_t reg_edx,
+                           volatile reg_t reg_ecx, volatile reg_t reg_eax);
+void check_return_too_shallow(dcontext_t *dcontext,
+                              volatile int errno, volatile reg_t eflags,
+                              volatile reg_t reg_edi, volatile reg_t reg_esi,
+                              volatile reg_t reg_ebp, volatile reg_t reg_esp,
+                              volatile reg_t reg_ebx, volatile reg_t reg_edx,
+                              volatile reg_t reg_ecx, volatile reg_t reg_eax);
+void check_return_ra_mangled(dcontext_t *dcontext,
+                             volatile int errno, volatile reg_t eflags,
+                             volatile reg_t reg_edi, volatile reg_t reg_esi,
+                             volatile reg_t reg_ebp, volatile reg_t reg_esp,
+                             volatile reg_t reg_ebx, volatile reg_t reg_edx,
+                             volatile reg_t reg_ecx, volatile reg_t reg_eax);
+#endif
+
+/* experimental native execution feature */
+/* in x86_code.c */
+void entering_native(void);
+
+#ifdef LINUX
+void new_thread_setup(dr_mcontext_t *mc);
+#endif
+
+void
+get_xmm_vals(dr_mcontext_t *mc);
+
+/* from x86.asm */
+/* Note these have specialized calling conventions and shouldn't be called from
+ * C code (see comments in x86.asm). */
+void global_do_syscall_sysenter(void);
+void global_do_syscall_int(void);
+void global_do_syscall_sygate_int(void);
+void global_do_syscall_sygate_sysenter(void);
+# ifdef WINDOWS
+void global_do_syscall_wow64(void);
+void global_do_syscall_wow64_index0(void);
+# endif
+#ifdef X64
+void global_do_syscall_syscall(void);
+#endif
+void get_xmm_caller_saved(dr_xmm_t *xmm_caller_saved_buf);
+
+/* in encode.c */
+byte *instr_encode_ignore_reachability(dcontext_t *dcontext_t, instr_t *instr, byte *pc);
+byte *instr_encode_check_reachability(dcontext_t *dcontext_t, instr_t *instr, byte *pc);
+byte *copy_and_re_relativize_raw_instr(dcontext_t *dcontext, instr_t *instr, byte *dst_pc);
+
+#endif /* X86_ARCH_H */
+
diff --git a/core/x86/arch_exports.h b/core/x86/arch_exports.h
new file mode 100644
index 000000000..ced6bdec6
--- /dev/null
+++ b/core/x86/arch_exports.h
@@ -0,0 +1,1812 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/* file "arch_exports.h" -- x86-specific exported declarations
+ *
+ * References: 
+ *   "Intel Architecture Software Developer's Manual", 1999.
+ */
+
+#ifndef _ARCH_EXPORTS_H_
+#define _ARCH_EXPORTS_H_ 1
+
+/* stack slot width */
+#define XSP_SZ (sizeof(reg_t))
+
+/* PR 264138: we must preserve xmm0-5 if on a 64-bit kernel */
+#define XMM_REG_SIZE  16
+#define XMM_SLOTS_SIZE  (NUM_XMM_SLOTS*XMM_REG_SIZE)
+#define XMM_SAVED_SIZE  (NUM_XMM_SAVED*XMM_REG_SIZE)
+
+typedef enum {
+    IBL_NONE = -1,
+    /* N.B.: order determines which table is on 2nd cache line in local_state_t */
+    IBL_RETURN = 0, /* returns lookup routine has stricter requirements */
+    IBL_BRANCH_TYPE_START = IBL_RETURN,
+    IBL_INDCALL,
+    IBL_INDJMP,
+    IBL_GENERIC           = IBL_INDJMP, /* currently least restrictive */
+    /* can double if a generic lookup is needed 
+       FIXME: remove this and add names for specific needs */
+    IBL_SHARED_SYSCALL    = IBL_GENERIC,
+    IBL_BRANCH_TYPE_END
+} ibl_branch_type_t;
+
+#define IBL_HASH_FUNC_OFFSET_MAX IF_X64_ELSE(4,3)
+
+struct _fragment_entry_t; /* in fragment.h */
+struct _ibl_table_t; /* in fragment.h */
+
+/* Scratch space and state required to be easily accessible from
+ * in-cache indirect branch lookup routines, store in thread-local storage.
+ * Goal is to get it all in one cache line: currently, though, it is
+ * 2 lines on 32-byte-line machines, with the call* and jmp* tables and the
+ * hashtable stats spilling onto the 2nd line.
+ * Even all on one line, shared ibl has a load vs private ibl's hardcoded immed...
+ *
+ * FIXME: to avoid splitting the mcontext for now these scratch
+ * fs:slots are used in fcache, but copied to the
+ * mcontext on transitions.  see case 3701
+ */
+typedef struct _lookup_table_access_t {
+    ptr_uint_t hash_mask;
+    struct _fragment_entry_t *lookuptable;
+} lookup_table_access_t;
+
+typedef struct _table_stat_state_t {
+    /* Organized in mask-table pairs to get both fields for a particular table
+     * on the same cache line.
+     */
+    /* FIXME We can play w/ordering these fields differently or if TLS space is
+     * crunched keeping a subset of them in TLS.
+     * For example, the ret_trace & indcall_trace tables could be heavily used
+     * but if the indjmp table isn't, it might make sense to put the ret_bb
+     * table's fields into TLS since ret_bb is likely to be the most heavily
+     * used for BB2BB IBL.
+     */
+    lookup_table_access_t table[IBL_BRANCH_TYPE_END];
+    /* FIXME: should allocate this separately so that release and
+     * DEBUG builds have the same layout especially when backward
+     * aligned entry */
+#ifdef HASHTABLE_STATISTICS
+    uint stats;
+#endif
+} table_stat_state_t;
+
+/* All spill slots are grouped in a separate struct because with
+ * -no_ibl_table_in_tls, only these slots are mapped to TLS (and the
+ * table address/mask pairs are not).
+ */
+typedef struct _spill_state_t {
+    /* Four registers are used in the indirect branch lookup routines */
+    reg_t xax, xbx, xcx, xdx;    /* general-purpose registers */
+    /* FIXME: move this below the tables to fit more on cache line */
+    dcontext_t *dcontext;
+} spill_state_t;
+
+typedef struct _local_state_t {
+    spill_state_t spill_space;
+} local_state_t;
+
+typedef struct _local_state_extended_t {
+    spill_state_t spill_space;
+    table_stat_state_t table_space;
+} local_state_extended_t;
+
+/* local_state_[extended_]t is allocated in os-specific thread-local storage (TLS),
+ * accessible off of fs:.  But, the actual segment offset varies, so
+ * os_tls_offset() must be used to obtain an fs: offset from a slot.
+ */
+#define TLS_XAX_SLOT             ((ushort)offsetof(spill_state_t, xax))
+#define TLS_XBX_SLOT             ((ushort)offsetof(spill_state_t, xbx))
+#define TLS_XCX_SLOT             ((ushort)offsetof(spill_state_t, xcx))
+#define TLS_XDX_SLOT             ((ushort)offsetof(spill_state_t, xdx))
+#define TLS_DCONTEXT_SLOT        ((ushort)offsetof(spill_state_t, dcontext))
+
+#define TABLE_OFFSET             (offsetof(local_state_extended_t, table_space))
+#define TLS_MASK_SLOT(btype)     ((ushort)(TABLE_OFFSET                         \
+                                  + offsetof(table_stat_state_t, table[btype])  \
+                                  + offsetof(lookup_table_access_t, hash_mask)))
+#define TLS_TABLE_SLOT(btype)    ((ushort)(TABLE_OFFSET                         \
+                                  + offsetof(table_stat_state_t, table[btype])  \
+                                  + offsetof(lookup_table_access_t, lookuptable)))
+  
+#ifdef HASHTABLE_STATISTICS
+# define TLS_HTABLE_STATS_SLOT   ((ushort)(offsetof(local_state_extended_t,     \
+                                                    table_space)                \
+                                  + offsetof(table_stat_state_t, stats)))
+#endif
+
+#define TLS_NUM_SLOTS                                  \
+   (DYNAMO_OPTION(ibl_table_in_tls) ?                  \
+    sizeof(local_state_extended_t) / sizeof(void *) :  \
+    sizeof(local_state_t) / sizeof(void *))
+
+#ifdef WINDOWS
+# define DETACH_CALLBACK_CODE_SIZE 256
+# define DETACH_CALLBACK_FINAL_JMP_SIZE 32
+
+/* For detach - stores callback continuation pcs and is used to dispatch to them after
+ * we detach. We have one per a thread (with stacked callbacks) stored in an array. */
+typedef struct _detach_callback_stack_t {
+    thread_id_t tid; /* thread tid */
+    ptr_uint_t count;   /* number of saved post-syscall continuation pcs */
+    app_pc *callback_addrs;   /* location of array of saved continuation pcs */
+    reg_t xax_save; /* spill slot for post-syscall code */
+    reg_t xbx_save; /* spill slot for post-syscall code */
+    reg_t xcx_save; /* spill slot for post-syscall code */
+    app_pc target;  /* temp slot for post-syscall code */
+    /* we need some private code to do the actual jmp */
+    byte code_buf[DETACH_CALLBACK_FINAL_JMP_SIZE];
+} detach_callback_stack_t;
+
+void
+arch_patch_syscall(dcontext_t *dcontext, byte *target);
+byte *
+emit_detach_callback_code(dcontext_t *dcontext, byte *buf,
+                          detach_callback_stack_t *callback_state);
+void
+emit_detach_callback_final_jmp(dcontext_t *dcontext,
+                               detach_callback_stack_t *callback_state);
+#endif
+
+/* We use this to ensure that linking and unlinking is atomic with respect
+ * to a thread in the cache, this is needed for our current flushing 
+ * implementation.  Note that linking and unlinking are only atomic with 
+ * respect to a thread in the cache not with respect to a thread in dynamorio 
+ * (which can see linking flags etc.)
+ */ 
+/* see bug 524 for additional notes, reproduced here :
+ * there is no way to do a locked mov, have to use an xchg or similar which is 
+ * a larger performance penalty (not really an issue), note that xchg implies
+ * lock, so no need for the lock prefix below
+ *
+ * further Intel's documentation is a little weird on the issue of 
+ * cross-modifying code (see IA32 volume 3 7-2 through 7-7), "Locked 
+ * instructions should not be used to insure that data written can be fetched 
+ * as instructions" and "Locked operations are atomic with respect to all other
+ * memory operations and all externally visible events.  Only instruction fetch
+ * and page table access can pass locked instructions", (pass?) however it does
+ * note that the current versions of P6 family, pentium 4, xeon, pentium and 
+ * 486 allow data written by locked instructions to be fetched as instructions.
+ * In the cross-modifying code section, however, it gives a (horrible for us) 
+ * algorithm to ensure cross-modifying code is compliant with current and 
+ * future versions of IA-32 then says that "the use of this option is not 
+ * required for programs intended to run on the 486, but is recommended to 
+ * insure compatibility with pentium 4, xeon, P6 family and pentium 
+ * processors", so my take home is that it works now, but don't have any 
+ * expectations for the future - FIXME - */
+/* Ref case 3628, case 4397, empirically this only works for code where the
+ * entire offset being written is within a cache line, so we can't use a locked
+ * instruction to ensure atomicity */
+#define PAD_JMPS_ALIGNMENT \
+    (INTERNAL_OPTION(pad_jmps_set_alignment) != 0 ? \
+     INTERNAL_OPTION(pad_jmps_set_alignment) : proc_get_cache_line_size())
+#ifdef WINDOWS
+/* note that the microsoft compiler will not enregister variables across asm
+ * blocks that touch those registers, so don't need to worry about clobbering
+ * eax and ebx */
+# define ATOMIC_4BYTE_WRITE(target, value, hot_patch) do {          \
+    /* test that we aren't crossing a cache line boundary */        \
+    DODEBUG({                                                       \
+        ASSERT(sizeof(value) == 4);                                 \
+        if (hot_patch && CROSSES_ALIGNMENT(target, 4, PAD_JMPS_ALIGNMENT)) { \
+            STATS_INC(unaligned_patches);                           \
+            ASSERT(!DYNAMO_OPTION(pad_jmps));                       \
+        }                                                           \
+    });                                                             \
+    _InterlockedExchange((volatile LONG *)target, value);           \
+  } while (0)
+# ifdef X64
+#  define ATOMIC_8BYTE_WRITE(target, value, hot_patch) do {         \
+    /* test that we aren't crossing a cache line boundary */        \
+    DODEBUG({                                                       \
+        ASSERT(sizeof(value) == 8);                                 \
+        /* Not currently used to write code */                      \
+        ASSERT_CURIOSITY(!hot_patch);                               \
+        if (hot_patch && CROSSES_ALIGNMENT(target, 8, PAD_JMPS_ALIGNMENT)) { \
+            STATS_INC(unaligned_patches);                           \
+            ASSERT(!DYNAMO_OPTION(pad_jmps));                       \
+        }                                                           \
+    });                                                             \
+    _InterlockedExchange64((volatile __int64 *)target, (__int64)value); \
+  } while (0)
+# endif
+
+/* We use intrinsics since they eliminated inline asm support for x64.
+ * FIXME: these intrinsics all use xadd even when no return value is needed!
+ *   We assume these aren't performance-critical enough to care.
+ *   If we do change to not have return value, need to change static inlines below.
+ * Even if these turn into callouts, they should be reentrant.
+ */
+# define ATOMIC_INC_int(var) _InterlockedIncrement((volatile LONG *)&(var))
+# ifdef X64 /* 64-bit intrinsics only avail on x64 */
+#  define ATOMIC_INC_int64(var) _InterlockedIncrement64((volatile __int64 *)&(var))
+# endif
+# define ATOMIC_INC(type, var) ATOMIC_INC_##type(var)
+# define ATOMIC_DEC_int(var) _InterlockedDecrement((volatile LONG *)&(var))
+# ifdef X64 /* 64-bit intrinsics only avail on x64 */
+#  define ATOMIC_DEC_int64(var) _InterlockedDecrement64((volatile __int64 *)&(var))
+# endif
+# define ATOMIC_DEC(type, var) ATOMIC_DEC_##type(var)
+/* Note that there is no x86/x64 _InterlockedAdd: only for IA64 */
+# define ATOMIC_ADD_int(var, value) \
+    _InterlockedExchangeAdd((volatile LONG *)&(var), value)
+# ifdef X64 /* 64-bit intrinsics only avail on x64 */
+#  define ATOMIC_ADD_int64(var, value) \
+    _InterlockedExchangeAdd64((volatile __int64 *)&(var), value)
+# endif
+# define ATOMIC_COMPARE_EXCHANGE_int(var, compare, exchange) \
+    _InterlockedCompareExchange((volatile LONG *)&(var), exchange, compare)
+# define ATOMIC_COMPARE_EXCHANGE_int64(var, compare, exchange) \
+    _InterlockedCompareExchange64((volatile __int64 *)&(var), exchange, compare)
+# define ATOMIC_COMPARE_EXCHANGE ATOMIC_COMPARE_EXCHANGE_int
+# define ATOMIC_ADD(type, var, val) ATOMIC_ADD_##type(var, val)
+# ifdef X64
+#  define ATOMIC_ADD_PTR(type, var, val) ATOMIC_ADD_int64(var, val)
+#  define ATOMIC_COMPARE_EXCHANGE_PTR ATOMIC_COMPARE_EXCHANGE_int64
+# else
+#  define ATOMIC_ADD_PTR(type, var, val) ATOMIC_ADD_int(var, val)
+#  define ATOMIC_COMPARE_EXCHANGE_PTR ATOMIC_COMPARE_EXCHANGE_int
+# endif
+# define SPINLOCK_PAUSE() _mm_pause() /* PAUSE = 0xf3 0x90 = repz nop */
+# define RDTSC_LL(var) (var = __rdtsc())
+# define SERIALIZE_INSTRUCTIONS() do { \
+        int cpuid_res_local[4];        \
+        __cpuid(cpuid_res_local, 0);   \
+    }
+/* no intrinsic available, and no inline asm support, so we have x86.asm routine */
+byte * get_frame_ptr(void);
+byte * get_stack_ptr(void);
+# define GET_FRAME_PTR(var) (var = get_frame_ptr())
+# define GET_STACK_PTR(var) (var = get_stack_ptr())
+
+/* returns true if result value is zero */
+static inline bool atomic_inc_and_test(volatile int *var) {
+    return (ATOMIC_INC(int, *(var)) == 0);
+}
+/* returns true if initial value was zero */
+static inline bool atomic_dec_and_test(volatile int *var) {
+    return (ATOMIC_DEC(int, *(var)) == -1);
+}
+/* returns true if result value is zero */
+static inline bool atomic_dec_becomes_zero(volatile int *var) {
+    return (ATOMIC_DEC(int, *(var)) == 0);
+}
+/* returns true if var was equal to compare */
+static inline bool atomic_compare_exchange_int(volatile int *var,
+                                           int compare, int exchange) {
+    return (ATOMIC_COMPARE_EXCHANGE_int(*(var), compare, exchange) == (compare));
+}
+static inline bool atomic_compare_exchange_int64(volatile int64 *var,
+                                                 int64 compare, int64 exchange) {
+    return ((ptr_int_t)ATOMIC_COMPARE_EXCHANGE_int64(*(var), compare, exchange) == 
+            (compare));
+}
+/* atomically adds value to memory location var and returns the sum */
+static inline int atomic_add_exchange_int(volatile int *var, int value) {
+    return ((value) + ATOMIC_ADD(int, *(var), value));
+}
+# ifdef X64 /* 64-bit intrinsics only avail on x64 */
+static inline int64 atomic_add_exchange_int64(volatile int64 *var, int64 value) {
+    return ((value) + ATOMIC_ADD(int64, *(var), value));
+}
+# endif
+# define atomic_add_exchange atomic_add_exchange_int
+
+#else /* LINUX */
+# define ATOMIC_4BYTE_WRITE(target, value, hot_patch) do {           \
+    /* test that we aren't crossing a cache line boundary */         \
+    DODEBUG({                                                        \
+        ASSERT(sizeof(value) == 4);                                  \
+        if (hot_patch && CROSSES_ALIGNMENT(target, 4, proc_get_cache_line_size())) { \
+            STATS_INC(unaligned_patches);                            \
+            ASSERT(!DYNAMO_OPTION(pad_jmps));                        \
+        }                                                            \
+    });                                                              \
+    __asm__ __volatile__("xchgl (%0), %1" : : "r" (target), "r" (value) : "memory"); \
+  } while (0)
+# ifdef X64
+#  define ATOMIC_8BYTE_WRITE(target, value, hot_patch) do {         \
+    /* test that we aren't crossing a cache line boundary */         \
+    DODEBUG({                                                        \
+        ASSERT(sizeof(value) == 8);                                  \
+        /* Not currently used to write code */                      \
+        ASSERT_CURIOSITY(!hot_patch);                               \
+        if (hot_patch && CROSSES_ALIGNMENT(target, 8, proc_get_cache_line_size())) { \
+            STATS_INC(unaligned_patches);                            \
+            ASSERT(!DYNAMO_OPTION(pad_jmps));                        \
+        }                                                            \
+    });                                                              \
+    __asm__ __volatile__("xchgq (%0), %1" : : "r" (target), "r" (value) : "memory"); \
+  } while (0)
+# endif
+# define ATOMIC_INC_suffix(suffix, var) \
+    __asm__ __volatile__("lock inc" suffix " %0" : "=m" (var) : : "memory")
+# define ATOMIC_INC_int(var) ATOMIC_INC_suffix("l", var)
+# define ATOMIC_INC_int64(var) ATOMIC_INC_suffix("q", var)
+# define ATOMIC_INC(type, var) ATOMIC_INC_##type(var)
+# define ATOMIC_DEC_suffix(suffix, var) \
+    __asm__ __volatile__("lock dec" suffix " %0" : "=m" (var) : : "memory")
+# define ATOMIC_DEC_int(var) ATOMIC_DEC_suffix("l", var)
+# define ATOMIC_DEC_int64(var) ATOMIC_DEC_suffix("q", var)
+# define ATOMIC_DEC(type, var) ATOMIC_DEC_##type(var)
+# define ATOMIC_ADD_suffix(suffix, var, value)                 \
+   __asm__ __volatile__("lock add" suffix " %1, %0"            \
+                        : "=m" (var) : "r" (value) : "memory")
+# define ATOMIC_ADD_int(var, val) ATOMIC_ADD_suffix("l", var, val)
+# define ATOMIC_ADD_int64(var, val) ATOMIC_ADD_suffix("q", var, val)
+# define ATOMIC_ADD(type, var, val) ATOMIC_ADD_##type(var, val)
+# ifdef X64
+#  define ATOMIC_ADD_PTR(type, var, val) ATOMIC_ADD_int64(var, val)
+# else
+#  define ATOMIC_ADD_PTR(type, var, val) ATOMIC_ADD_int(var, val)
+# endif
+/* Not safe for general use, just for atomic_add_exchange(), undefed below */
+# define ATOMIC_ADD_EXCHANGE_suffix(suffix, var, value, result) \
+   __asm__ __volatile__("lock xadd" suffix " %1, %0"            \
+                        : "=m" (*var), "=r" (result) : "1" (value) : "memory")
+# define ATOMIC_ADD_EXCHANGE_int(var, val, res) \
+    ATOMIC_ADD_EXCHANGE_suffix("l", var, val, res)
+# define ATOMIC_ADD_EXCHANGE_int64(var, val, res) \
+    ATOMIC_ADD_EXCHANGE_suffix("q", var, val, res)
+# define ATOMIC_COMPARE_EXCHANGE_suffix(suffix, var, compare, exchange) \
+   __asm__ __volatile__ ("lock cmpxchg" suffix " %2,%0"         \
+                         : "=m" (var)                           \
+                         : "a" (compare), "r" (exchange)        \
+                         : "memory")
+# define ATOMIC_COMPARE_EXCHANGE_int(var, compare, exchange) \
+    ATOMIC_COMPARE_EXCHANGE_suffix("l", var, compare, exchange)
+# define ATOMIC_COMPARE_EXCHANGE_int64(var, compare, exchange) \
+    ATOMIC_COMPARE_EXCHANGE_suffix("q", var, compare, exchange)
+# define ATOMIC_COMPARE_EXCHANGE ATOMIC_COMPARE_EXCHANGE_int
+# ifdef X64
+#  define ATOMIC_COMPARE_EXCHANGE_PTR ATOMIC_COMPARE_EXCHANGE_int64
+# else
+#  define ATOMIC_COMPARE_EXCHANGE_PTR ATOMIC_COMPARE_EXCHANGE
+# endif
+# define SPINLOCK_PAUSE()   __asm__ __volatile__("pause")
+# define RDTSC_LL(llval)                        \
+    __asm__ __volatile__                        \
+    ("rdtsc" : "=A" (llval))
+# define SERIALIZE_INSTRUCTIONS()                                       \
+    __asm__ __volatile__                                                \
+    ("xor %%eax, %%eax; cpuid" : : : "eax", "ebx", "ecx", "edx");
+# define GET_FRAME_PTR(var) asm("mov %%"IF_X64_ELSE("rbp","ebp")", %0" : "=m"(var))
+# define GET_STACK_PTR(var) asm("mov %%"IF_X64_ELSE("rsp","esp")", %0" : "=m"(var))
+
+# define SET_FLAG(cc, flag) __asm__ __volatile__("set"#cc " %0" :"=qm" (flag) )
+# define SET_IF_NOT_ZERO(flag) SET_FLAG(nz, flag)
+# define SET_IF_NOT_LESS(flag) SET_FLAG(nl, flag)
+
+/* Atomically increments *var by 1
+ * Returns true if the resulting value is zero, otherwise returns false
+ */ 
+static inline bool atomic_inc_and_test(volatile int *var)
+{
+    unsigned char c;
+
+    ATOMIC_INC(int, *var);
+    /* flags should be set according to resulting value, now we convert that back to C */
+    SET_IF_NOT_ZERO(c);
+    /* FIXME: we add an extra memory reference to a local, 
+       although we could put the return value in EAX ourselves */
+    return c == 0;
+}
+
+/* Atomically decrements *var by 1
+ * Returns true if the initial value was zero, otherwise returns false
+ */ 
+static inline bool atomic_dec_and_test(volatile int *var)
+{
+    unsigned char c;
+
+    ATOMIC_DEC(int, *var);
+    /* result should be set according to value before change, now we convert that back to C */
+    SET_IF_NOT_LESS(c);
+    /* FIXME: we add an extra memory reference to a local, 
+       although we could put the return value in EAX ourselves */
+    return c == 0;
+}
+
+/* Atomically decrements *var by 1
+ * Returns true if the resulting value is zero, otherwise returns false
+ */ 
+static inline bool atomic_dec_becomes_zero(volatile int *var)
+{
+    unsigned char c;
+
+    ATOMIC_DEC(int, *var);
+    /* result should be set according to value after change, now we convert that back to C */
+    SET_IF_NOT_ZERO(c);
+    /* FIXME: we add an extra memory reference to a local, 
+       although we could put the return value in EAX ourselves */
+    return c == 0;
+}
+
+
+/* returns true if var was equal to compare, and now is equal to exchange, 
+   otherwise returns false
+ */ 
+static inline bool atomic_compare_exchange_int(volatile int *var,
+                                               int compare, int exchange)
+{
+    unsigned char c;
+    ATOMIC_COMPARE_EXCHANGE(*var, compare, exchange);
+    /* ZF is set if matched, all other flags are as if a normal compare happened */
+    /* we convert ZF value back to C */
+    SET_IF_NOT_ZERO(c);
+    /* FIXME: we add an extra memory reference to a local, 
+       although we could put the return value in EAX ourselves */
+    return c == 0;
+}
+
+/* returns true if var was equal to compare, and now is equal to exchange, 
+   otherwise returns false
+ */ 
+static inline bool atomic_compare_exchange_int64(volatile int64 *var,
+                                                 int64 compare,
+                                                 int64 exchange)
+{
+    unsigned char c;
+    ATOMIC_COMPARE_EXCHANGE_int64(*var, compare, exchange);
+    /* ZF is set if matched, all other flags are as if a normal compare happened */
+    /* we convert ZF value back to C */
+    SET_IF_NOT_ZERO(c);
+    /* FIXME: we add an extra memory reference to a local, 
+       although we could put the return value in EAX ourselves */
+    return c == 0;
+}
+
+/* atomically adds value to memory location var and returns the sum */
+static inline int atomic_add_exchange_int(volatile int *var, int value)
+{
+    int temp;
+    ATOMIC_ADD_EXCHANGE_int(var, value, temp);
+    return (temp + value);
+}
+static inline int64 atomic_add_exchange_int64(volatile int64 *var, int64 value)
+{
+    int64 temp;
+    ATOMIC_ADD_EXCHANGE_int64(var, value, temp);
+    return (temp + value);
+}
+# define atomic_add_exchange atomic_add_exchange_int
+# undef ATOMIC_ADD_EXCHANGE_suffix
+# undef ATOMIC_ADD_EXCHANGE_int
+# undef ATOMIC_ADD_EXCHANGE_int64
+
+
+#endif /* LINUX */
+
+#define atomic_compare_exchange atomic_compare_exchange_int
+#ifdef X64
+# define atomic_compare_exchange_ptr(v, c, e) \
+     atomic_compare_exchange_int64((volatile int64 *)(v), (int64)(c), (int64)(e))
+# define ATOMIC_ADDR_WRITE ATOMIC_8BYTE_WRITE
+#else
+# define atomic_compare_exchange_ptr(v, c, e) \
+     atomic_compare_exchange_int((volatile int *)(v), (int)(c), (int)(e))
+# define ATOMIC_ADDR_WRITE ATOMIC_4BYTE_WRITE
+#endif
+
+#define ATOMIC_MAX_int(type, maxvar, curvar) do {               \
+        type atomic_max__maxval;                                \
+        type atomic_max__curval = (curvar);                     \
+        ASSERT(sizeof(int) == sizeof(maxvar));                  \
+        ASSERT(sizeof(type) == sizeof(maxvar));                 \
+        ASSERT(sizeof(type) == sizeof(curvar));                 \
+        do {                                                    \
+            atomic_max__maxval = (maxvar);                      \
+        } while (atomic_max__maxval < atomic_max__curval &&     \
+                 !atomic_compare_exchange_int((int*)&(maxvar),      \
+                                              atomic_max__maxval,   \
+                                              atomic_max__curval)); \
+    } while (0)
+
+#ifdef X64 /* 64-bit intrinsics only avail on x64 */
+# define ATOMIC_MAX_int64(type, maxvar, curvar) do {            \
+        type atomic_max__maxval;                                \
+        type atomic_max__curval = (curvar);                     \
+        ASSERT(sizeof(int64) == sizeof(maxvar));                \
+        ASSERT(sizeof(type) == sizeof(maxvar));                 \
+        ASSERT(sizeof(type) == sizeof(curvar));                 \
+        do {                                                    \
+            atomic_max__maxval = (maxvar);                      \
+        } while (atomic_max__maxval < atomic_max__curval &&     \
+                 !atomic_compare_exchange_int64((int64*)&(maxvar),    \
+                                                atomic_max__maxval,   \
+                                                atomic_max__curval)); \
+    } while (0)
+#endif
+
+#define ATOMIC_MAX(type, maxvar, curvar) ATOMIC_MAX_##type(type, maxvar, curvar)
+
+#define DEBUGGER_INTERRUPT_BYTE 0xcc
+
+/* if hot_patch is true:
+ *   The write that inserts the relative target is done atomically so this
+ *   function is safe with respect to a thread executing the code containing 
+ *   this target, presuming that the code in both the before and after states
+ *   is valid 
+ */
+byte *
+insert_relative_target(byte *pc, cache_pc target, bool hot_patch);
+
+byte *
+insert_relative_jump(byte *pc, cache_pc target, bool hot_patch);
+
+/* in arch.c */
+
+#ifdef PROFILE_RDTSC
+#ifdef LINUX
+/* This only works on Pentium I or later */
+__inline__ uint64 get_time();
+#else /* WINDOWS */
+/* This only works on Pentium I or later */
+uint64 get_time(void);
+#endif
+#endif
+
+void arch_init(void);
+void arch_exit(IF_WINDOWS_ELSE_NP(bool detach_stacked_callbacks, void));
+void arch_thread_init(dcontext_t *dcontext);
+void arch_thread_exit(dcontext_t *dcontext _IF_WINDOWS(bool detach_stacked_callbacks));
+#if defined(WINDOWS_PC_SAMPLE) && !defined(DEBUG) 
+/* for sampling fast exit path */
+void arch_thread_profile_exit(dcontext_t *dcontext);
+void arch_profile_exit(void);
+#endif
+
+byte *
+code_align_forward(byte *pc, size_t alignment);
+
+enum {
+    /* Each translation entry represents a sequence of instructions.
+     * If TRANSLATE_IDENTICAL is set, that sequence of instructions
+     * share the same translation ("identical" == stride 0); otherwise, the
+     * translation should advance by instruction length with distance
+     * from the head ("contiguous" == stride of instr length).
+     * Generally identical instrs are mangled sequences that we inserted,
+     * though currently the table creation scheme doesn't follow that
+     * on all contig-identical borders.
+     */
+    TRANSLATE_IDENTICAL      = 0x0001, /* otherwise contiguous */
+    TRANSLATE_OUR_MANGLING   = 0x0002, /* added by our own mangling (PR 267260) */
+}; /* no typedef b/c we need ushort not int */
+
+/* Translation table entry (case 3559).
+ * PR 299783: for now we only support pc translation, not full arbitrary reg
+ * state mappings, which aren't needed for DR but may be nice for clients.
+ */
+typedef struct _translation_entry_t {
+    /* offset from fragment start_pc */
+    ushort cache_offs;
+    /* TRANSLATE_ flags */
+    ushort flags;
+    app_pc app;
+} translation_entry_t;
+
+/* Translation table that records info for translating cache pc to app
+ * pc without reading app memory (used when it is unsafe to do so).
+ * The table records only translations at change points, so the
+ * recreater must interpolate between them, using either a stride of 0
+ * if the previous translation entry is marked "identical" or a stride
+ * equal to the instruction length as we decode from the cache if the
+ * previous entry is !identical=="contiguous".
+ */
+typedef struct _translation_info_t {
+    uint num_entries;
+    /* an array of num_entries elements */
+    translation_entry_t translation[1]; /* variable-sized */
+} translation_info_t;
+
+/* PR 244737: all generated code is thread-shared on x64 */
+#define IS_SHARED_SYSCALL_THREAD_SHARED IF_X64_ELSE(true, false)
+
+/* state translation for faults and thread relocation */
+app_pc recreate_app_pc(dcontext_t *tdcontext, cache_pc pc, fragment_t *f);
+
+typedef enum {
+    RECREATE_FAILURE,
+    RECREATE_SUCCESS_PC,
+    RECREATE_SUCCESS_STATE,
+} recreate_success_t;
+
+recreate_success_t 
+recreate_app_state(dcontext_t *tdcontext, dr_mcontext_t *mcontext, bool restore_memory);
+
+void translation_info_free(dcontext_t *tdcontext, translation_info_t *info);
+translation_info_t *record_translation_info(dcontext_t *dcontext, fragment_t *f,
+                                            instrlist_t *ilist);
+void translation_info_print(const translation_info_t *info, cache_pc start, file_t file);
+#ifdef INTERNAL
+void stress_test_recreate_state(dcontext_t *dcontext, fragment_t *f, instrlist_t *ilist);
+#endif
+
+bool is_indirect_branch_lookup_routine(dcontext_t *dcontext, cache_pc pc);
+bool in_generated_routine(dcontext_t *dcontext, cache_pc pc);
+bool in_context_switch_code(dcontext_t *dcontext, cache_pc pc);
+bool in_indirect_branch_lookup_code(dcontext_t *dcontext, cache_pc pc);
+cache_pc get_fcache_target(dcontext_t *dcontext);
+void set_fcache_target(dcontext_t *dcontext, cache_pc value);
+void copy_mcontext(dr_mcontext_t *src, dr_mcontext_t *dst);
+void dump_dr_mcontext(dr_mcontext_t *context, file_t f, bool dump_xml);
+void dump_mcontext(dr_mcontext_t *context, file_t f, bool dump_xml);
+const char *get_branch_type_name(ibl_branch_type_t branch_type);
+ibl_branch_type_t get_ibl_branch_type(instr_t *instr);
+
+/* Return the entry point for a routine with which an
+ * atomic hashtable delete can be performed for the given fragment.
+ */
+cache_pc
+get_target_delete_entry_pc(dcontext_t *dcontext,
+                           struct _ibl_table_t *table);
+cache_pc get_reset_exit_stub(dcontext_t *dcontext);
+
+typedef linkstub_t * (* fcache_enter_func_t) (dcontext_t *dcontext);
+fcache_enter_func_t get_fcache_enter_private_routine(dcontext_t *dcontext);
+
+cache_pc get_unlinked_entry(dcontext_t *dcontext, cache_pc linked_entry);
+cache_pc get_linked_entry(dcontext_t *dcontext, cache_pc unlinked_entry);
+#ifdef X64
+cache_pc get_trace_cmp_entry(dcontext_t *dcontext, cache_pc linked_entry);
+#endif
+
+cache_pc get_do_syscall_entry(dcontext_t *dcontext);
+#ifdef WINDOWS
+fcache_enter_func_t get_fcache_enter_indirect_routine(dcontext_t *dcontext);
+cache_pc get_do_callback_return_entry(dcontext_t *dcontext);
+#else
+cache_pc get_do_int_syscall_entry(dcontext_t *dcontext);
+cache_pc get_do_clone_syscall_entry(dcontext_t *dcontext);
+#endif
+byte * get_global_do_syscall_entry(void);
+
+/* NOTE - because of the sygate int 2e hack, after_do_syscall_addr and
+ * after_shared_syscall_addr in fact both check for the same address with
+ * int system calls, so can't use them to disambiguate between the two. */
+#ifdef WINDOWS
+/* For int system calls
+ *  - addr is the instruction immediately after the int 2e
+ *  - code is the addr of our code that handles post int 2e
+ * For non int system calls addr and code will be the same (ret addr), this
+ * is just for working around our Sygate int 2e in ntdll hack (5217) */
+cache_pc after_shared_syscall_addr(dcontext_t *dcontext);
+cache_pc after_shared_syscall_code(dcontext_t *dcontext);
+bool is_shared_syscall_routine(dcontext_t *dcontext, cache_pc pc);
+/* For int system calls
+ *  - addr is the instruction immediately after the int 2e
+ *  - code is the addr of our code that handles post int 2e
+ * For non int system calls addr and code will be the same (ret addr), this
+ * is just for working around our Sygate int 2e in ntdll hack (5217) */
+cache_pc after_do_syscall_addr(dcontext_t *dcontext);
+cache_pc after_do_syscall_code(dcontext_t *dcontext);
+#else
+cache_pc after_do_shared_syscall_addr(dcontext_t *dcontext);
+bool is_after_do_syscall_addr(dcontext_t *dcontext, cache_pc pc);
+#endif
+
+bool is_after_syscall_address(dcontext_t *dcontext, cache_pc pc);
+
+void update_generated_hashtable_access(dcontext_t *dcontext);
+fcache_enter_func_t get_fcache_enter_shared_routine(dcontext_t *dcontext);
+
+/* Method of performing system call.
+ * We assume that only one method is in use, except for 32-bit applications
+ * on 64-bit linux kernels, which use both sys{enter,call} on the vsyscall
+ * page and inlined int (PR 286922).
+ * For these apps, DR itself and global_do_syscall use int, but we
+ * have both a do_syscall for the vsyscall and a separate do_int_syscall
+ * (we can't use the vsyscall for some system calls like clone: we could
+ * potentially use int for everything if we fixed up the syscall args).
+ * The method set in that case is the vsyscall method.
+ */
+enum {
+    SYSCALL_METHOD_UNINITIALIZED,
+    SYSCALL_METHOD_INT,
+    SYSCALL_METHOD_SYSENTER,
+    SYSCALL_METHOD_SYSCALL,
+#ifdef WINDOWS
+    SYSCALL_METHOD_WOW64,
+#endif
+};
+#ifdef LINUX
+enum { SYSCALL_METHOD_LONGEST_INSTR = 2 }; /* to ensure safe patching */
+#endif
+void check_syscall_method(dcontext_t *dcontext, instr_t *instr);
+int get_syscall_method(void);
+/* Does the syscall instruction always return to the invocation point? */
+bool does_syscall_ret_to_callsite(void);
+void set_syscall_method(int method);
+#ifdef LINUX
+bool should_syscall_method_be_sysenter(void);
+#endif
+/* returns the address of the first app syscall instruction we saw (see hack
+ * in win32/os.c that uses this for PRE_SYSCALL_PC, not for general use */
+byte *get_app_sysenter_addr(void); 
+
+#ifdef NATIVE_RETURN
+/* HACK for NATIVE_RETURN and ib-opt.c, to translate cache ret targets
+ * to app targets for cmp,je pairs
+ *
+ * pc is really a cache_pc!  But we don't export that type for
+ * CLIENT_INTERFACE
+ *
+ * case 10543: Removed this from the API for the DR community release.
+ */
+app_pc
+ret_tgt_cache_to_app(dcontext_t *dcontext, app_pc pc);
+#endif
+
+/* in x86.asm */
+void call_switch_stack(dcontext_t *dcontext, byte *stack, void (*func) (dcontext_t *),
+                       bool free_initstack, bool return_on_return);
+void unexpected_return(void);
+void go_native(dcontext_t *dcontext);
+void cleanup_and_terminate(dcontext_t *dcontext, int sysnum,
+                           ptr_uint_t sys_arg1, ptr_uint_t sys_arg2, bool exitproc);
+#ifdef CLIENT_INTERFACE
+void cleanup_and_terminate_client_thread(int sysnum, byte *stack_base,
+                                         ptr_uint_t sys_arg1,
+                                         ptr_uint_t sys_arg2);
+#endif
+bool cpuid_supported(void);
+void our_cpuid(int res[4], int eax);
+#ifdef WINDOWS
+int dynamorio_syscall_int2e(int sysnum, ...);
+int dynamorio_syscall_sysenter(int sysnum, ...);
+int dynamorio_syscall_sygate_int2e(int sysnum, ...);
+int dynamorio_syscall_sygate_sysenter(int sysnum, ...);
+# ifdef X64
+int dynamorio_syscall_syscall(int sysnum, ...);
+# endif
+int dynamorio_syscall_wow64(int sysnum, ...);
+void get_segments_defg(cxt_seg_t *ds, cxt_seg_t *es, cxt_seg_t *fs, cxt_seg_t *gs);
+void get_own_context_helper(CONTEXT *cxt);
+void dr_fxsave(byte *buf_aligned);
+void dr_fnsave(byte *buf_aligned);
+void dr_fxrstor(byte *buf_aligned);
+void dr_frstor(byte *buf_aligned);
+#ifdef STACK_GUARD_PAGE
+/* PR203701: If the dstack is exhausted we'll use this function to
+ * call internal_exception_info() with a separate exception stack.
+ */
+void call_intr_excpt_alt_stack(dcontext_t *dcontext, EXCEPTION_RECORD *pExcptRec,
+                               CONTEXT *cxt, byte *stack);
+#endif
+#else /* LINUX */
+long dynamorio_syscall(uint sysnum, uint num_args, ...);
+void dynamorio_sigreturn(void);
+# ifndef X64
+void dynamorio_nonrt_sigreturn(void);
+# endif
+#endif
+void back_from_native(void);
+DEBUG_DECLARE(void debug_infinite_loop(); /* handy cpu eating infinite loop */)
+
+/* in proc.c -- everything in proc.h is exported so just include it here */
+#include "proc.h"
+
+/* in disassemble.c */
+#ifdef DEBUG /* because uses logfile */
+void disassemble_fragment(dcontext_t *dcontext, fragment_t *f, bool just_header);
+void disassemble_app_bb(dcontext_t *dcontext, app_pc tag, file_t outfile);
+/* dumps callstack for ebp stored in mcontext */
+void dump_mcontext_callstack(dcontext_t *dcontext);
+#endif
+/* dumps callstack for current pc and ebp */
+void dump_dr_callstack(file_t outfile);
+/* user-specified ebp */
+void dump_callstack(app_pc pc, app_pc ebp, file_t outfile, bool dump_xml);
+
+#if defined(INTERNAL) || defined(DEBUG) || defined(CLIENT_INTERFACE)
+void disassemble_fragment_header(dcontext_t *dcontext, fragment_t *f, file_t outfile);
+void disassemble_fragment_body(dcontext_t *dcontext, fragment_t *f, file_t outfile);
+void disassemble_app_bb(dcontext_t *dcontext, app_pc tag, file_t outfile);
+
+/* DR_API EXPORT TOFILE dr_ir_instrlist.h */
+DR_API
+/**
+ * Prints each instruction in \p ilist in sequence to \p outfile.
+ * The default is to use AT&T-style syntax, unless the \ref op_syntax_intel
+ * "-syntax_intel" runtime option is specified.
+ */
+void 
+instrlist_disassemble(dcontext_t *dcontext, app_pc tag,
+                      instrlist_t *ilist, file_t outfile);
+#endif /* INTERNAL || DEBUG || CLIENT_INTERFACE */
+
+/* in emit_utils.c */
+
+static inline bool
+use_addr_prefix_on_short_disp(void)
+{
+    /* -ibl_addr_prefix => addr prefix everywhere */
+    return (DYNAMO_OPTION(ibl_addr_prefix) || 
+            /* PR 212807, PR 209709: addr prefix is noticeably worse
+             * on Pentium M, Core, and Core2.
+             * It's better on Pentium 4 and Pentium D.
+             *
+             * Note that this variation by processor type does not need to
+             * be stored in pcaches b/c either way works and the size is
+             * not assumed (except for prefixes: but coarse_units doesn't
+             * support prefixes in general).
+             */
+            /* P4 and PD */
+            (proc_get_family() == FAMILY_PENTIUM_4 ||
+             /* PPro, P2, P3, but not PM */
+             (proc_get_family() == FAMILY_PENTIUM_3 &&
+              (proc_get_model() <= 8 ||
+               proc_get_model() == 10 ||
+               proc_get_model() == 11))));
+    /* FIXME: should similarly remove addr prefixes from hardcoded
+     * emits in emit_utils.c, except in cases where space is more
+     * important than speed.
+     * FIXME: case 5231 long term solution should properly choose 
+     * - ibl - speed
+     * - prefixes - speed/space?
+     * - app code - preserverd since we normally don't need to reencode,
+     *              unless it is a CTI that goes through FS - should be preserved too
+     * - direct stubs - space
+     * - indirect stubs - speed/space?
+     * - enter/exit - speed?
+     * - interception routines - speed?
+     */
+}
+
+/* Merge w/ _LENGTH enum below? */
+/* not ifdef X64 to simplify code */
+#define SIZE64_MOV_XAX_TO_TLS         8
+#define SIZE64_MOV_XBX_TO_TLS         9
+#define SIZE64_MOV_PTR_IMM_TO_XAX    10
+#define SIZE64_MOV_PTR_IMM_TO_TLS   (12*2) /* high and low 32 bits separately */
+#define SIZE32_MOV_XAX_TO_TLS         5
+#define SIZE32_MOV_XBX_TO_TLS         6
+#define SIZE32_MOV_XAX_TO_TLS_DISP32  6
+#define SIZE32_MOV_XBX_TO_TLS_DISP32  7
+#define SIZE32_MOV_XAX_TO_ABS         5
+#define SIZE32_MOV_XBX_TO_ABS         6
+#define SIZE32_MOV_PTR_IMM_TO_XAX     5
+#define SIZE32_MOV_PTR_IMM_TO_TLS    10
+
+#ifdef X64
+# define FRAG_IS_32(flags) (TEST(FRAG_32_BIT, (flags)))
+#else
+# define FRAG_IS_32(flags) true
+#endif
+
+#define SIZE_MOV_XAX_TO_TLS(flags, require_addr16) \
+    (FRAG_IS_32(flags) ? \
+     ((require_addr16 || use_addr_prefix_on_short_disp()) ?   \
+      SIZE32_MOV_XAX_TO_TLS : SIZE32_MOV_XAX_TO_TLS_DISP32) : \
+      SIZE64_MOV_XAX_TO_TLS)
+#define SIZE_MOV_XBX_TO_TLS(flags, require_addr16) \
+    (FRAG_IS_32(flags) ? \
+     ((require_addr16 || use_addr_prefix_on_short_disp()) ?   \
+      SIZE32_MOV_XBX_TO_TLS : SIZE32_MOV_XBX_TO_TLS_DISP32) : \
+      SIZE64_MOV_XBX_TO_TLS)
+#define SIZE_MOV_PTR_IMM_TO_XAX(flags) \
+    (FRAG_IS_32(flags) ? SIZE32_MOV_PTR_IMM_TO_XAX : SIZE64_MOV_PTR_IMM_TO_XAX)
+
+/* size of restore ecx prefix */
+#define XCX_IN_TLS(flags) (DYNAMO_OPTION(private_ib_in_tls) || TEST(FRAG_SHARED, (flags)))
+#define FRAGMENT_BASE_PREFIX_SIZE(flags) \
+    (XCX_IN_TLS(flags) ? SIZE_MOV_XBX_TO_TLS(flags, false) : SIZE32_MOV_XBX_TO_ABS)
+
+/* exported for DYNAMO_OPTION(separate_private_stubs)
+ * FIXME: find better way to export -- would use global var accessed
+ * by macro, but easiest to have as static initializer for heap bucket
+ */
+/* for -thread_private, we're relying on the fact that
+ * SIZE32_MOV_XAX_TO_TLS == SIZE32_MOV_XAX_TO_ABS, and that
+ * x64 always uses tls
+ */
+#define DIRECT_EXIT_STUB_SIZE32 \
+    (SIZE32_MOV_XAX_TO_TLS + SIZE32_MOV_PTR_IMM_TO_XAX + JMP_LONG_LENGTH)
+#define DIRECT_EXIT_STUB_SIZE64 \
+    (SIZE64_MOV_XAX_TO_TLS + SIZE64_MOV_PTR_IMM_TO_XAX + JMP_LONG_LENGTH)
+#define DIRECT_EXIT_STUB_SIZE(flags) \
+    (FRAG_IS_32(flags) ? DIRECT_EXIT_STUB_SIZE32 : DIRECT_EXIT_STUB_SIZE64)
+
+/* coarse-grain stubs use a store directly to memory so they can
+ * link through the stub and not mess up app state.
+ * 1st instr example:
+ *   67 64 c7 06 e0 0e 02 99 4e 7d  addr16 mov $0x7d4e9902 -> %fs:0x0ee0 
+ * 64-bit is split into high and low dwords:
+ *   65 c7 04 25 20 16 00 00 02 99 4e 7d  mov $0x7d4e9902 -> %gs:0x1620 
+ *   65 c7 04 25 24 16 00 00 00 00 00 00  mov $0x00000000 -> %gs:0x1624 
+ * both of these exact sequences are assumed in entrance_stub_target_tag()
+ * and coarse_indirect_stub_jmp_target().
+ */
+#define STUB_COARSE_DIRECT_SIZE32  (SIZE32_MOV_PTR_IMM_TO_TLS + JMP_LONG_LENGTH)
+#define STUB_COARSE_DIRECT_SIZE64  (SIZE64_MOV_PTR_IMM_TO_TLS + JMP_LONG_LENGTH)
+#define STUB_COARSE_DIRECT_SIZE(flags) \
+    (FRAG_IS_32(flags) ? STUB_COARSE_DIRECT_SIZE32 : STUB_COARSE_DIRECT_SIZE64)
+
+/* writes nops into the address range */
+#define SET_TO_NOPS(addr, size) memset(addr, 0x90, size)
+/* writes debugbreaks into the address range */
+#define SET_TO_DEBUG(addr, size) memset(addr, 0xcc, size)
+/* check if region is SET_TO_NOP */
+#define IS_SET_TO_NOP(addr, size) is_region_memset_to_char(addr, size, 0x90)
+/* check if region is SET_TO_DEBUG */
+#define IS_SET_TO_DEBUG(addr, size) is_region_memset_to_char(addr, size, 0xcc)
+
+/* offset of the patchable region from the end of a cti */
+#define CTI_PATCH_OFFSET 4
+/* size of the patch to a cti */
+#define CTI_PATCH_SIZE 4
+
+/* offset of the patchable region from the end of a stub */
+#define EXIT_STUB_PATCH_OFFSET 4
+/* size of the patch to a stub */
+#define EXIT_STUB_PATCH_SIZE 4
+
+/* the most bytes we'll need to shift a patchable location for -pad_jmps */
+#define MAX_PAD_SIZE 3
+
+/* evaluates to true if region crosses at most 1 padding boundary */
+#define WITHIN_PAD_REGION(lower, upper) \
+    ((upper)-(lower) <= PAD_JMPS_ALIGNMENT)
+
+#define STATS_PAD_JMPS_ADD(flags, stat, val) DOSTATS({    \
+    if (TEST(FRAG_SHARED, (flags))) {                     \
+        if (TEST(FRAG_IS_TRACE, (flags)))                 \
+            STATS_ADD(pad_jmps_shared_trace_##stat, val); \
+        else                                              \
+            STATS_ADD(pad_jmps_shared_bb_##stat, val);    \
+    }                                                     \
+    else if (TEST(FRAG_IS_TRACE, (flags)))                \
+        STATS_ADD(pad_jmps_trace_##stat, val);            \
+    else if (TEST(FRAG_TEMP_PRIVATE, (flags)))            \
+        STATS_ADD(pad_jmps_temp_##stat, val);             \
+    else                                                  \
+        STATS_ADD(pad_jmps_bb_##stat, val);               \
+})
+
+bool 
+is_exit_cti_stub_patchable(dcontext_t *dcontext, instr_t *inst, 
+                           uint frag_flags);
+
+uint
+extend_trace_pad_bytes(fragment_t *add_frag);
+
+uint 
+patchable_exit_cti_align_offs(dcontext_t *dcontext, instr_t *inst, cache_pc pc);
+
+bool 
+is_patchable_exit_stub(dcontext_t *dcontext, linkstub_t *l, fragment_t *f);
+
+uint 
+bytes_for_exitstub_alignment(dcontext_t *dcontext, linkstub_t *l, fragment_t *f,
+                             byte *startpc);
+
+byte * 
+pad_for_exitstub_alignment(dcontext_t *dcontext, linkstub_t *l, fragment_t *f, 
+                           byte *startpc);
+
+void 
+remove_nops_from_ilist(dcontext_t *dcontext, instrlist_t *ilist 
+                       _IF_DEBUG(bool recreating));
+
+uint 
+nop_pad_ilist(dcontext_t *dcontext, fragment_t *f, instrlist_t *ilist, bool emitting);
+
+bool 
+is_exit_cti_patchable(dcontext_t *dcontext, instr_t *inst, uint frag_flags);
+
+int exit_stub_size(dcontext_t *dcontext, cache_pc target, uint flags);
+
+int insert_exit_stub(dcontext_t *dcontext, fragment_t *f,
+                     linkstub_t *l, cache_pc stub_pc);
+int insert_exit_stub_other_flags(dcontext_t *dcontext,  fragment_t *f, linkstub_t *l,
+                                 cache_pc stub_pc, ushort l_flags);
+
+int
+linkstub_unlink_entry_offset(dcontext_t *dcontext, fragment_t *f, linkstub_t *l);
+
+cache_pc
+indirect_linkstub_stub_pc(dcontext_t *dcontext, fragment_t *f, linkstub_t *l);
+
+cache_pc
+indirect_linkstub_target(dcontext_t *dcontext, fragment_t *f, linkstub_t *l);
+
+/* based on machine state, returns which of l1 and l2 must have been taken */
+linkstub_t *
+linkstub_cbr_disambiguate(dcontext_t *dcontext, fragment_t *f,
+                          linkstub_t *l1, linkstub_t *l2);
+
+cache_pc
+cbr_fallthrough_exit_cti(cache_pc prev_cti_pc);
+
+/* for use with patch_branch and insert_relative target */
+enum {
+    NOT_HOT_PATCHABLE=false,
+    HOT_PATCHABLE=true
+};
+void patch_branch(cache_pc branch_pc, cache_pc target_pc, bool hot_patch);
+bool link_direct_exit(dcontext_t *dcontext, fragment_t *f, linkstub_t *l,
+                      fragment_t *targetf, bool hot_patch);
+void unlink_direct_exit(dcontext_t *dcontext, fragment_t *f, linkstub_t *l);
+void link_indirect_exit(dcontext_t *dcontext, fragment_t *f, linkstub_t *l,
+                        bool hot_patch);
+void unlink_indirect_exit(dcontext_t *dcontext, fragment_t *f, linkstub_t *l);
+void insert_fragment_prefix(dcontext_t *dcontext, fragment_t *f);
+int fragment_prefix_size(uint flags);
+void update_indirect_exit_stub(dcontext_t *dcontext, fragment_t *f, linkstub_t *l);
+#ifdef PROFILE_RDTSC
+uint profile_call_size(void);
+void insert_profile_call(cache_pc start_pc);
+void finalize_profile_call(dcontext_t *dcontext, fragment_t *f);
+#endif
+int decode_syscall_num(dcontext_t *dcontext, byte *entry);
+#ifdef WINDOWS
+void link_shared_syscall(dcontext_t *dcontext);
+void unlink_shared_syscall(dcontext_t *dcontext);
+#endif
+
+/* Coarse-grain fragment support */
+cache_pc
+entrance_stub_jmp(cache_pc stub);
+cache_pc
+entrance_stub_jmp_target(cache_pc stub);
+cache_pc
+entrance_stub_target_tag(cache_pc stub);
+bool
+coarse_is_indirect_stub(cache_pc stub);
+cache_pc
+coarse_indirect_stub_jmp_target(cache_pc stub);
+uint
+coarse_indirect_stub_size(coarse_info_t *info);
+bool
+coarse_is_entrance_stub(cache_pc stub);
+bool
+coarse_is_trace_head(cache_pc stub);
+bool
+entrance_stub_linked(cache_pc stub, coarse_info_t *info /*OPTIONAL*/);
+void
+link_entrance_stub(dcontext_t *dcontext, cache_pc stub, cache_pc tgt,
+                   bool hot_patch, coarse_info_t *info /*OPTIONAL*/);
+void
+unlink_entrance_stub(dcontext_t *dcontext, cache_pc stub, uint flags,
+                     coarse_info_t *info /*OPTIONAL*/);
+cache_pc
+entrance_stub_from_cti(cache_pc cti);
+
+uint
+coarse_exit_prefix_size(uint flags);
+
+byte *
+emit_coarse_exit_prefix(dcontext_t *dcontext, byte *pc, coarse_info_t *info);
+
+/* Update info pointer in exit prefixes */
+void
+patch_coarse_exit_prefix(dcontext_t *dcontext, coarse_info_t *info);
+
+enum {
+    MAX_INSTR_LENGTH = 17,
+    /* size of 32-bit-offset jcc instr, assuming it has no
+     * jcc branch hint!
+     */
+    CBR_LONG_LENGTH  = 6,
+    JMP_LONG_LENGTH  = 5,
+    JMP_SHORT_LENGTH = 2,
+    CBR_SHORT_REWRITE_LENGTH = 9, /* FIXME: use this in mangle.c */
+    RET_0_LENGTH     = 1,
+    PUSH_IMM32_LENGTH = 5,
+
+    /* size of 32-bit call and jmp instructions w/o prefixes. */
+    CTI_IND1_LENGTH    = 2, /* FF D6             call esi                      */
+    CTI_IND2_LENGTH    = 3, /* FF 14 9E          call dword ptr [esi+ebx*4]    */
+    CTI_IND3_LENGTH    = 4, /* FF 54 B3 08       call dword ptr [ebx+esi*4+8]  */
+    CTI_DIRECT_LENGTH  = 5, /* E8 9A 0E 00 00    call 7C8024CB                 */
+    CTI_IAT_LENGTH     = 6, /* FF 15 38 10 80 7C call dword ptr ds:[7C801038h] */
+    CTI_FAR_ABS_LENGTH = 7, /* 9A 1B 07 00 34 39 call 0739:3400071B            */
+                            /* 07                                              */
+
+    INT_LENGTH = 2,
+    SYSCALL_LENGTH = 2,
+};
+
+#define REL32_REACHABLE_OFFS(offs) ((offs) <= INT_MAX && (offs) >= INT_MIN)
+/* source should be the end of a rip-relative-referencing instr */
+#define REL32_REACHABLE(source, target) REL32_REACHABLE_OFFS((target) - (source))
+
+/* If code_buf points to a jmp rel32 returns true and returns the target of
+ * the jmp in jmp_target as if was located at app_loc. */
+bool
+is_jmp_rel32(byte *code_buf, app_pc app_loc, app_pc *jmp_target /* OUT */);
+
+/* If code_buf points to a jmp rel8 returns true and returns the target of
+ * the jmp in jmp_target as if was located at app_loc. */
+bool
+is_jmp_rel8(byte *code_buf, app_pc app_loc, app_pc *jmp_target /* OUT */);
+
+/* in interp.c */
+
+/* An upper bound on instructions added to a bb when added to a trace,
+ * which is of course highest for the case of indirect branch mangling.
+ * Normal lea, jecxz, lea is 14, NATIVE_RETURN can get above 20,
+ * but this should cover everything, fine to be well above, this is
+ * only used to keep below the maximum trace size for the next bb,
+ * we calculate the exact size in fixup_last_cti().
+ *
+ * For x64 we have to increase this (PR 333576 hit this):
+ *    +19   L4  65 48 89 0c 25 10 00 mov    %rcx -> %gs:0x10
+ *              00 00
+ *    +28   L4  48 8b c8             mov    %rax -> %rcx
+ *    +31   L4  e9 1b e2 f6 ff       jmp    $0x00000000406536e0 <shared_bb_ibl_indjmp>
+ *    (+36)
+ *   =>
+ *    +120  L0  65 48 89 0c 25 10 00 mov    %rcx -> %gs:0x10
+ *              00 00
+ *    +129  L0  48 8b c8             mov    %rax -> %rcx
+ *    +132  L3  65 48 a3 00 00 00 00 mov    %rax -> %gs:0x00
+ *              00 00 00 00
+ *    +143  L3  48 b8 23 24 93 28 00 mov    $0x0000000028932423 -> %rax
+ *              00 00 00
+ *    +153  L3  65 48 a3 08 00 00 00 mov    %rax -> %gs:0x08
+ *              00 00 00 00
+ *    +164  L3  9f                   lahf   -> %ah
+ *    +165  L3  0f 90 c0             seto   -> %al
+ *    +168  L3  65 48 3b 0c 25 08 00 cmp    %rcx %gs:0x08
+ *              00 00
+ *    +177  L4  0f 85 a9 d7 f6 ff    jnz    $0x000000004065312f <shared_trace_cmp_indjmp>
+ *    +183  L3  65 48 8b 0c 25 10 00 mov    %gs:0x10 -> %rcx
+ *              00 00
+ *    +192  L3  04 7f                add    $0x7f %al -> %al
+ *    +194  L3  9e                   sahf   %ah
+ *    +195  L3  65 48 a1 00 00 00 00 mov    %gs:0x00 -> %rax
+ *              00 00 00 00
+ *    +206
+ *   
+ *    (36-19)=17 vs (206-120)=86 => 69 bytes.  was 65 bytes prior to PR 209709!
+ *    usually 3 bytes smaller since don't need to restore eflags.
+ */
+#define TRACE_CTI_MANGLE_SIZE_UPPER_BOUND 72
+
+fragment_t *
+build_basic_block_fragment(dcontext_t *dcontext, app_pc start_pc,
+                           uint initial_flags, bool linked, bool visible
+                           _IF_CLIENT(bool for_trace)
+                           _IF_CLIENT(instrlist_t **unmangled_ilist));
+
+void interp(dcontext_t *dcontext);
+uint extend_trace(dcontext_t *dcontext, fragment_t *f, linkstub_t *prev_l);
+int append_trace_speculate_last_ibl(dcontext_t *dcontext, instrlist_t *trace,
+                                    app_pc speculate_next_tag, bool record_translation);
+
+uint
+forward_eflags_analysis(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr);
+
+/* Converts instr_t EFLAGS_ flags to corresponding fragment_t FRAG_ flags,
+ * assuming that the instr_t flags correspond to the start of the fragment_t
+ */
+uint
+instr_eflags_to_fragment_eflags(uint instr_eflags);
+
+
+instrlist_t * 
+decode_fragment(dcontext_t *dcontext, fragment_t *f, byte *buf, /*OUT*/uint *bufsz,
+                uint target_flags, /*OUT*/uint *dir_exits, /*OUT*/uint *indir_exits);
+
+instrlist_t *
+decode_fragment_exact(dcontext_t *dcontext, fragment_t *f, byte *buf,
+                      /*IN/OUT*/uint *bufsz, uint target_flags,
+                      /*OUT*/uint *dir_exits, /*OUT*/uint *indir_exits);
+
+fragment_t * copy_fragment(dcontext_t *dcontext, fragment_t *f, bool replace);
+void shift_ctis_in_fragment(dcontext_t *dcontext, fragment_t *f, ssize_t shift,
+                            cache_pc start, cache_pc end, size_t old_size);
+#ifdef PROFILE_RDTSC
+void add_profile_call(dcontext_t *dcontext);
+#endif
+app_pc emulate(dcontext_t *dcontext, app_pc pc, dr_mcontext_t *mc);
+
+typedef struct {
+    app_pc region_start;
+    app_pc region_end;
+    app_pc start_pc;
+    app_pc min_pc;
+    app_pc max_pc;
+    app_pc bb_end;
+    bool contiguous;
+    bool overlap;
+} overlap_info_t;
+
+instrlist_t * build_app_bb_ilist(dcontext_t *dcontext, byte *start_pc, file_t outf);
+
+void
+bb_build_abort(dcontext_t *dcontext, bool clean_vmarea);
+
+/* Builds an instrlist_t as though building a bb from pc.
+ * Use recreate_fragment_ilist() for building an instrlist_t for a fragment.
+ * If check_vm_area is false, Does NOT call check_thread_vm_area()!
+ *   Make sure you know it will terminate at the right spot.  It does
+ *   check selfmod and native_exec for elision, but otherwise will
+ *   follow ubrs to the limit.  Currently used for
+ *   record_translation_info() (case 3559).
+ */
+instrlist_t * recreate_bb_ilist(dcontext_t *dcontext, byte *pc,  uint flags,
+                                uint *res_flags, uint *res_exit_type,
+                                bool check_vm_area, bool mangle
+                                _IF_CLIENT(bool call_client)
+                                _IF_CLIENT(bool for_trace));
+
+instrlist_t *
+recreate_fragment_ilist(dcontext_t *dcontext, byte *pc,
+                        /*IN/OUT*/fragment_t **f_res, /*OUT*/bool *alloc,
+                        bool mangle _IF_CLIENT(bool call_client));
+
+app_pc find_app_bb_end(dcontext_t *dcontext, byte *start_pc, uint flags);
+bool app_bb_overlaps(dcontext_t *dcontext, byte *start_pc, uint flags,
+                       byte *region_start, byte *region_end, overlap_info_t *info_res);
+
+/* pass in name if you already have it, else this routine looks it up from modbase */
+bool
+on_native_exec_list(app_pc modbase, const char *name);
+
+/* in decode.c, needed here for ref in arch.h */
+/* DR_API EXPORT TOFILE dr_ir_utils.h */
+/* DR_API EXPORT BEGIN */
+#ifdef X64
+/* DR_API EXPORT END */
+DR_API
+/**
+ * The decode and encode routines use a per-thread persistent flag that
+ * indicates whether to treat code as 32-bit (x86) or 64-bit (x64).  This
+ * routine sets that flag to the indicated value and returns the old value.  Be
+ * sure to restore the old value prior to any further application execution to
+ * avoid problems in mis-interpreting application code.
+ *
+ * \note For 64-bit DR builds only.
+ */
+bool
+set_x86_mode(dcontext_t *dcontext, bool x86);
+
+DR_API
+/**
+ * The decode and encode routines use a per-thread persistent flag that
+ * indicates whether to treat code as 32-bit (x86) or 64-bit (x64).  This
+ * routine returns the value of that flag.
+ *
+ * \note For 64-bit DR builds only.
+ */
+bool
+get_x86_mode(dcontext_t *dcontext);
+/* DR_API EXPORT BEGIN */
+#endif
+/* DR_API EXPORT END */
+
+/* in instr.c */
+/* DR_API EXPORT TOFILE dr_ir_instr.h */
+
+/* DR_API EXPORT BEGIN */
+/****************************************************************************
+ * INSTR ROUTINES
+ */
+/**
+ * @file dr_ir_instr.h
+ * @brief Functions to create and manipulate instructions.
+ */
+
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Returns an initialized instr_t allocated on the thread-local heap. 
+ * Sets the x86/x64 mode of the returned instr_t to the mode of dcontext.
+ */
+instr_t*
+instr_create(dcontext_t *dcontext);
+
+DR_API
+/** Initializes \p instr.
+ * Sets the x86/x64 mode of \p instr to the mode of dcontext.
+ */
+void 
+instr_init(dcontext_t *dcontext, instr_t *instr);
+
+DR_API
+/** 
+ * Deallocates all memory that was allocated by \p instr.  This
+ * includes raw bytes allocated by instr_allocate_raw_bits() and
+ * operands allocated by instr_set_num_opnds().  Does not deallocate
+ * the storage for \p instr itself.
+ */
+void 
+instr_free(dcontext_t *dcontext, instr_t *instr);
+
+DR_API
+/** 
+ * Performs both instr_free() and instr_init().
+ * \p instr must have been initialized.
+ */
+void 
+instr_reset(dcontext_t *dcontext, instr_t *instr);
+
+DR_API
+/** 
+ * Frees all dynamically allocated storage that was allocated by \p instr,
+ * except for allocated bits.
+ * Also zeroes out \p instr's fields, except for raw bit fields,
+ * whether \p instr is instr_ok_to_mangle(), and the x86 mode of \p instr.
+ * \p instr must have been initialized.
+ */
+void 
+instr_reuse(dcontext_t *dcontext, instr_t *instr);
+
+DR_API
+/** 
+ * Performs instr_free() and then deallocates the thread-local heap
+ * storage for \p instr.
+ */
+void 
+instr_destroy(dcontext_t *dcontext, instr_t *instr);
+
+DR_API
+/** 
+ * Returns the next instr_t in the instrlist_t that contains \p instr.
+ * \note The next pointer for an instr_t is inside the instr_t data
+ * structure itself, making it impossible to have on instr_t in 
+ * two different InstrLists (but removing the need for an extra data
+ * structure for each element of the instrlist_t).
+ */
+instr_t*
+instr_get_next(instr_t *instr);
+
+DR_API
+/** Returns the previous instr_t in the instrlist_t that contains \p instr. */
+instr_t*
+instr_get_prev(instr_t *instr);
+
+DR_API
+/** Sets the next field of \p instr to point to \p next. */
+void 
+instr_set_next(instr_t *instr, instr_t *next);
+
+DR_API
+/** Sets the prev field of \p instr to point to \p prev. */
+void 
+instr_set_prev(instr_t *instr, instr_t *prev);
+
+DR_API
+/** 
+ * Gets the value of the user-controlled note field in \p instr.
+ * \note Important: is also used when emitting for targets that are other
+ * instructions, so make sure to clear or set appropriately the note field
+ * prior to emitting.
+ */
+void *
+instr_get_note(instr_t *instr);
+
+DR_API
+/** Sets the user-controlled note field in \p instr to \p value. */
+void 
+instr_set_note(instr_t *instr, void *value);
+
+DR_API
+/** Return the taken target pc of the (direct branch) instruction. */
+app_pc 
+instr_get_branch_target_pc(instr_t *cti_instr);
+
+DR_API
+/** Set the taken target pc of the (direct branch) instruction. */
+void 
+instr_set_branch_target_pc(instr_t *cti_instr, app_pc pc);
+
+DR_API
+/**
+ * Returns true iff \p instr is a conditional branch, unconditional branch,
+ * or indirect branch with a program address target (NOT an instr_t address target)
+ * and \p instr is ok to mangle.
+ */
+#ifdef UNSUPPORTED_API
+/**
+ * This routine does NOT try to decode an opcode in a Level 1 or Level
+ * 0 routine, and can thus be called on Level 0 routines.  
+ */
+#endif
+bool 
+instr_is_exit_cti(instr_t *instr);
+
+DR_API
+/** Return true iff \p instr's opcode is OP_int, OP_into, or OP_int3. */
+bool 
+instr_is_interrupt(instr_t *instr);
+
+#ifdef UNSUPPORTED_API
+DR_API
+/**
+ * Returns true iff \p instr has been marked as targeting the prefix of its
+ * target fragment.
+ *
+ * Some code manipulations need to store a target address in a
+ * register and then jump there, but need the register to be restored
+ * as well.  DR provides a single-instruction prefix that is
+ * placed on all fragments (basic blocks as well as traces) that
+ * restores ecx.  It is on traces for internal DR use.  To have
+ * it added to basic blocks as well, call
+ * dr_add_prefixes_to_basic_blocks() during initialization.
+ */
+bool
+instr_branch_targets_prefix(instr_t *instr);
+
+DR_API
+/**
+ * If \p val is true, indicates that \p instr's target fragment should be
+ *   entered through its prefix, which restores ecx.
+ * If \p val is false, indicates that \p instr should target the normal entry
+ *   point and not the prefix.
+ *
+ * Some code manipulations need to store a target address in a
+ * register and then jump there, but need the register to be restored
+ * as well.  DR provides a single-instruction prefix that is
+ * placed on all fragments (basic blocks as well as traces) that
+ * restores ecx.  It is on traces for internal DR use.  To have
+ * it added to basic blocks as well, call
+ * dr_add_prefixes_to_basic_blocks() during initialization.
+ */
+void
+instr_branch_set_prefix_target(instr_t *instr, bool val);
+#endif /* UNSUPPORTED_API */
+
+DR_UNS_API
+/** 
+ * Returns true iff \p instr has been marked as a selfmod check failure
+ * exit.
+ */
+bool
+instr_branch_selfmod_exit(instr_t *instr);
+
+DR_UNS_API
+/**
+ * If \p val is true, indicates that \p instr is a selfmod check failure exit.
+ * If \p val is false, indicates otherwise.
+ */
+void
+instr_branch_set_selfmod_exit(instr_t *instr, bool val);
+
+DR_API
+/**
+ * Return true iff \p instr is not a meta-instruction
+ * (see instr_set_ok_to_mangle() for more information).
+ */
+bool 
+instr_ok_to_mangle(instr_t *instr);
+
+DR_API
+/**
+ * Sets \p instr to "ok to mangle" if \p val is true and "not ok to
+ * mangle" if \p val is false.  An instruction that is "not ok to
+ * mangle" is treated by DR as a "meta-instruction", distinct from
+ * normal application instructions, and is not mangled in any way.
+ * This is necessary to have DR not create an exit stub for a direct
+ * jump.  All non-meta instructions that are added to basic blocks or
+ * traces should have their translation fields set (via
+ * #instr_set_translation(), or the convenience routine
+ * #instr_set_meta_no_translation()) when recreating state at a fault;
+ * meta instructions should not fault and are not considered
+ * application instructions but rather added instrumentation code (see
+ * #dr_register_bb_event() for further information on recreating).
+ */
+void 
+instr_set_ok_to_mangle(instr_t *instr, bool val);
+
+DR_API
+/**
+ * A convenience routine that calls both
+ * #instr_set_ok_to_mangle (instr, false) and
+ * #instr_set_translation (instr, NULL).
+ */
+void 
+instr_set_meta_no_translation(instr_t *instr);
+
+DR_API
+/** Return true iff \p instr is to be emitted into the cache. */
+bool 
+instr_ok_to_emit(instr_t *instr);
+
+DR_API
+/**
+ * Set \p instr to "ok to emit" if \p val is true and "not ok to emit"
+ * if \p val is false.  An instruction that should not be emitted is
+ * treated normally by DR for purposes of exits but is not placed into
+ * the cache.  It is used for final jumps that are to be elided.
+ */
+void 
+instr_set_ok_to_emit(instr_t *instr, bool val);
+
+#ifdef CUSTOM_EXIT_STUBS
+DR_API
+/**
+ * If \p instr is not an exit cti, does nothing.
+ * If \p instr is an exit cti, sets \p stub to be custom exit stub code
+ * that will be inserted in the exit stub prior to the normal exit
+ * stub code.  If \p instr already has custom exit stub code, that
+ * existing instrlist_t is cleared and destroyed (using current thread's
+ * context).  (If \p stub is NULL, any existing stub code is NOT destroyed.)
+ * The creator of the instrlist_t containing \p instr is
+ * responsible for destroying stub.
+ * \note Custom exit stubs containing control transfer instructions to
+ * other instructions inside a fragment besides the custom stub itself
+ * are not fully supported in that they will not be decoded from the
+ * cache properly as having instr_t targets.
+ */
+void
+instr_set_exit_stub_code(instr_t *instr, instrlist_t *stub);
+
+DR_API
+/**
+ * Returns the custom exit stub code instruction list that has been
+ * set for this instruction.  If none exists, returns NULL.
+ */
+instrlist_t *
+instr_exit_stub_code(instr_t *instr);
+#endif
+
+DR_API
+/**
+ * Returns the length of \p instr.
+ * As a side effect, if instr_ok_to_mangle(instr) and \p instr's raw bits
+ * are invalid, encodes \p instr into bytes allocated with
+ * instr_allocate_raw_bits(), after which instr is marked as having
+ * valid raw bits.
+ */
+int 
+instr_length(dcontext_t *dcontext, instr_t *instr);
+
+/* also in instr.c, but not exported */
+void instr_shift_raw_bits(instr_t *instr, ssize_t offs);
+uint instr_branch_type(instr_t *cti_instr);
+int instr_exit_branch_type(instr_t *instr);
+void instr_exit_branch_set_type(instr_t *instr, uint type);
+
+/* in encode.c */
+/* DR_API EXPORT TOFILE dr_ir_instr.h */
+DR_API
+/** Returns true iff \p instr can be encoding as a valid IA-32 instruction. */
+bool 
+instr_is_encoding_possible(instr_t *instr);
+
+DR_API
+/** 
+ * Encodes \p instr into the memory at \p pc.
+ * Uses the x86/x64 mode stored in instr, not the mode of the current thread.
+ * Returns the pc after the encoded instr, or NULL if the encoding failed.
+ * If instr is a cti with an instr_t target, the note fields of instr and
+ * of the target must be set with the respective offsets of each instr_t!
+ * (instrlist_encode does this automatically, if the target is in the list).
+ */
+byte * 
+instr_encode(dcontext_t *dcontext, instr_t *instr, byte *pc);
+
+/* DR_API EXPORT TOFILE dr_ir_instrlist.h */
+DR_API
+/** 
+ * Encodes each instruction in \p ilist in turn in contiguous memory starting
+ * at \p pc.  Returns the pc after all of the encodings, or NULL if any one
+ * of the encodings failed.
+ * Uses the x86/x64 mode stored in each instr, not the mode of the current thread.
+ * In order for instr_t operands to be encoded properly,
+ * \p has_instr_jmp_targets must be true.  If \p has_instr_jmp_targets is true,
+ * the note field of each instr_t in ilist will be overwritten, and if any
+ * instr_t targets are not in \p ilist, they must have their note fields set with
+ * their offsets relative to pc.
+ */
+byte *
+instrlist_encode(dcontext_t *dcontext, instrlist_t *ilist, byte *pc,
+                 bool has_instr_jmp_targets);
+
+/* in mangle.c */
+void insert_clean_call_with_arg_jmp_if_ret_true(dcontext_t *dcontext, instrlist_t *ilist,
+        instr_t *instr, void *callee, int arg, app_pc jmp_tag, instr_t *jmp_instr);
+void
+insert_push_immed_ptrsz(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+                    ptr_int_t val);
+#ifdef LINUX
+void mangle_clone_code(dcontext_t *dcontext, byte *pc, bool skip);
+void mangle_syscall_code(dcontext_t *dcontext, fragment_t *f, byte *pc, bool skip);
+#endif
+void finalize_selfmod_sandbox(dcontext_t *dcontext, fragment_t *f);
+
+/* in retcheck.c */
+#ifdef CHECK_RETURNS_SSE2
+void finalize_return_check(dcontext_t *dcontext, fragment_t *f);
+#endif
+#ifdef RETURN_AFTER_CALL
+void add_return_target(dcontext_t *dcontext, app_pc instr_pc, instr_t *instr);
+int ret_after_call_check(dcontext_t *dcontext, app_pc target_addr, app_pc src_addr);
+bool is_observed_call_site(dcontext_t *dcontext, app_pc retaddr);
+#endif
+
+/* in optimize.c */
+void optimize_trace(dcontext_t *dcontext, app_pc tag, instrlist_t *trace);
+#ifdef DEBUG
+void print_optimization_stats(void); 
+#endif
+
+#ifdef SIDELINE
+/* exact overlap with sideline.h */
+#include "sideline.h"
+#endif
+
+#include "link.h"
+/* convert link flags to ibl_branch_type_t */
+static inline ibl_branch_type_t
+extract_branchtype(ushort linkstub_flags)
+{
+    if (TEST(LINK_RETURN, linkstub_flags))
+        return IBL_RETURN;
+    if (TEST(LINK_CALL, linkstub_flags))
+        return IBL_INDCALL;
+    if (TEST(LINK_JMP, linkstub_flags))
+        return IBL_INDJMP;
+    ASSERT_NOT_REACHED();
+    return IBL_GENERIC;
+}
+
+/* convert ibl_branch_type_t to LINK_ flags */
+static inline uint
+ibltype_to_linktype(ibl_branch_type_t ibltype)
+{
+    if (ibltype == IBL_RETURN)
+        return LINK_INDIRECT | LINK_RETURN;
+    if (ibltype == IBL_INDCALL)
+        return LINK_INDIRECT | LINK_CALL;
+    if (ibltype == IBL_INDJMP)
+        return LINK_INDIRECT | LINK_JMP;
+    ASSERT_NOT_REACHED();
+    return 0;
+}
+
+#ifdef DEBUG
+bool is_ibl_routine_type(dcontext_t *dcontext, cache_pc target, ibl_branch_type_t branch_type);
+#endif /* DEBUG */
+
+/* This completely optimizable routine is the only place where we
+ * allow a data pointer to be converted to a function pointer to allow
+ * better type-checking for the rest of our C code
+ */
+/* on x86 function pointers and data pointers are interchangeable */
+
+static inline 
+generic_func_t
+convert_data_to_function(void *data_ptr)
+{
+#ifdef WINDOWS
+#  pragma warning(push)
+#  pragma warning(disable : 4055)
+#endif
+    return (generic_func_t)data_ptr;
+#ifdef WINDOWS
+#  pragma warning(pop)
+#endif
+}
+
+/* Our version of setjmp & longjmp.  Currently used only for handling hot patch
+ * exceptions and to implement an internal generic try-catch mechanism
+ * later on (case 1800).
+ * We could use a dr_mcontext_t here, but that has 4 extra fields that aren't
+ * used here.  TODO: we could use it and live with the wastage?
+ * Espcially in light of the merging from PR 218131.
+ *
+ * edx & eax need not be saved because they are scratch registers in a 
+ * call, i.e., caller-save;  they are used to return values from functions.  
+ * As longjmp is implemented as return from setjmp, eax & edx need not be saved.
+ */
+typedef struct dr_jmp_buf_t {
+    reg_t xbx;
+    reg_t xcx;
+    reg_t xdi;
+    reg_t xsi;
+    reg_t xbp;
+    reg_t xsp;
+    reg_t xip;
+#ifdef X64
+    /* optimization: can we trust callee-saved regs r8,r9,r10,r11 and not save them? */
+    reg_t r8, r9, r10, r11, r12, r13, r14, r15;
+#endif
+#ifdef LINUX
+    /* our longjmp() is siglongjmp() */
+    kernel_sigset_t sigmask;
+#endif
+} dr_jmp_buf_t;
+/* in x86.asm */
+int 
+dr_longjmp(dr_jmp_buf_t *buf, int val);
+int 
+dr_setjmp(dr_jmp_buf_t *buf);
+
+#ifdef LINUX
+/* PR 206278: for try/except we need to save the signal mask */
+void dr_setjmp_sigmask(dr_jmp_buf_t *buf);
+# define DR_SETJMP(buf) (dr_setjmp_sigmask(buf), dr_setjmp(buf))
+#else 
+# define DR_SETJMP(buf) (dr_setjmp(buf))
+#endif
+
+#define DR_LONGJMP(buf, val)          \
+    do {                              \
+        ASSERT(val !=0);              \
+        dr_longjmp(buf, val);         \
+    } while (0)
+
+
+/* Macros to access application function parameters.
+ * These assume that mc->rsp points at the return address (i.e., we're
+ * at function entry).
+ * Compare the SYS_PARAM* macros and REGPARM* enum: some duplication there.
+ * Note that if a param is 32 bits we must ignore the top 32 bits of
+ * its stack slot (Since passed via "mov dword" instead of "push", top
+ * bits are garbage.)
+ */
+#ifdef X64
+# ifdef WINDOWS
+#  define APP_PARAM_0(mc) (mc)->xcx
+#  define APP_PARAM_1(mc) (mc)->xdx
+#  define APP_PARAM_2(mc) (mc)->r8
+#  define APP_PARAM_3(mc) (mc)->r9
+#  define APP_PARAM_4(mc) (*(((reg_t *)((mc)->xsp)) + 5))
+#  define APP_PARAM_5(mc) (*(((reg_t *)((mc)->xsp)) + 6))
+#  define APP_PARAM_6(mc) (*(((reg_t *)((mc)->xsp)) + 7))
+#  define APP_PARAM_7(mc) (*(((reg_t *)((mc)->xsp)) + 8))
+#  define APP_PARAM_8(mc) (*(((reg_t *)((mc)->xsp)) + 9))
+#  define APP_PARAM_9(mc) (*(((reg_t *)((mc)->xsp)) + 10))
+#  define APP_PARAM_10(mc) (*(((reg_t *)((mc)->xsp)) + 11))
+# else
+#  define APP_PARAM_0(mc) (mc)->xdi
+#  define APP_PARAM_1(mc) (mc)->xsi
+#  define APP_PARAM_2(mc) (mc)->rdx
+#  define APP_PARAM_3(mc) (mc)->rcx
+#  define APP_PARAM_4(mc) (mc)->r8
+#  define APP_PARAM_5(mc) (mc)->r9
+#  define APP_PARAM_6(mc) (*(((reg_t *)((mc)->xsp)) + 1))
+#  define APP_PARAM_7(mc) (*(((reg_t *)((mc)->xsp)) + 2))
+#  define APP_PARAM_8(mc) (*(((reg_t *)((mc)->xsp)) + 3))
+#  define APP_PARAM_9(mc) (*(((reg_t *)((mc)->xsp)) + 4))
+#  define APP_PARAM_10(mc) (*(((reg_t *)((mc)->xsp)) + 5))
+# endif
+/* only takes integer literals */
+# define APP_PARAM(mc, offs) APP_PARAM_##offs(mc)
+#else
+/* only takes integer literals */
+# define APP_PARAM(mc, offs) (*(((reg_t *)((mc)->xsp)) + (offs) + 1))
+#endif
+
+
+#endif /* _ARCH_EXPORTS_H_ */
diff --git a/core/x86/asm_defines.asm b/core/x86/asm_defines.asm
new file mode 100644
index 000000000..04c4ead9b
--- /dev/null
+++ b/core/x86/asm_defines.asm
@@ -0,0 +1,316 @@
+/* **********************************************************
+ * Copyright (c) 2008-2009 VMware, Inc.  All rights reserved.
+ * ********************************************************** */
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifndef _ASM_DEFINES_ASM_
+#define _ASM_DEFINES_ASM_ 1
+
+/* Preprocessor macro definitions shared among all .asm files.
+ * Since cpp macros can't generate newlines we have a later
+ * script replace @N@ for us.
+ */
+
+/****************************************************/
+#if defined(ASSEMBLE_WITH_GAS)
+# define START_FILE .text
+# define END_FILE /* nothing */
+# define DECLARE_FUNC(symbol) \
+.align 0 @N@\
+.global symbol @N@\
+.hidden symbol @N@\
+.type symbol, %function
+# define DECLARE_EXPORTED_FUNC(symbol) \
+.align 0 @N@\
+.global symbol @N@\
+.type symbol, %function
+# define END_FUNC(symbol) /* nothing */
+# define DECLARE_GLOBAL(symbol) \
+.global symbol @N@\
+.hidden symbol
+# define GLOBAL_LABEL(label) label
+# define ADDRTAKEN_LABEL(label) label
+# define WORD word ptr
+# define DWORD dword ptr
+# define QWORD qword ptr
+# ifdef X64
+/* w/o the rip, gas won't use rip-rel and adds relocs that ld trips over */
+#  define SYMREF(sym) [rip + sym]
+# else
+#  define SYMREF(sym) [sym]
+# endif
+# define HEX(n) 0x##n
+# define SEGMEM(seg,mem) [seg:mem]
+# define DECL_EXTERN(symbol) /* nothing */
+/* include newline so we can put multiple on one line */
+# define RAW(n) .byte HEX(n) @N@
+# define DECLARE_FUNC_SEH(symbol) DECLARE_FUNC(symbol)
+# define PUSH_SEH(reg) push reg
+# define PUSH_NONCALLEE_SEH(reg) push reg
+# define END_PROLOG /* nothing */
+/* PR 212290: avoid text relocations.
+ * @GOT returns the address and is for extern vars; @GOTOFF gets the value.
+ * Note that using ## to paste => 
+ *   "error: pasting "initstack_mutex" and "@" does not give a valid preprocessing token"
+ * but whitespace separation seems fine.
+ */
+# define ADDR_VIA_GOT(base, sym) [sym @GOT + base]
+# define VAR_VIA_GOT(base, sym) [sym @GOTOFF + base]
+/****************************************************/
+#elif defined(ASSEMBLE_WITH_MASM)
+# ifdef X64
+#  define START_FILE .CODE
+# else
+#  define START_FILE \
+.686 /* default is 8086! need 686 for sysenter */ @N@\
+.XMM /* needed for fnclex and fxrstor */ @N@\
+.MODEL flat, c @N@\
+ASSUME fs:_DATA @N@\
+.CODE
+# endif
+# define END_FILE END
+/* we don't seem to need EXTERNDEF or GLOBAL */
+# define DECLARE_FUNC(symbol) symbol PROC
+# define DECLARE_EXPORTED_FUNC(symbol) symbol PROC
+# define END_FUNC(symbol) symbol ENDP
+# define DECLARE_GLOBAL(symbol) 
+# define GLOBAL_LABEL(label) 
+# define ADDRTAKEN_LABEL(label) label
+# define WORD word ptr
+# define DWORD dword ptr
+# define QWORD qword ptr
+/* ml64 uses rip-rel automatically */
+# define SYMREF(sym) [sym]
+# define HEX(n) 0##n##h
+# define SEGMEM(seg,mem) seg:[mem]
+# define DECL_EXTERN(symbol) EXTERN symbol:PROC
+/* include newline so we can put multiple on one line */
+# define RAW(n) DB HEX(n) @N@
+# ifdef X64
+/* 64-bit SEH directives */
+/* Declare non-leaf function (adjusts stack; makes calls; saves non-volatile regs): */  
+#  define DECLARE_FUNC_SEH(symbol) symbol PROC FRAME
+/* Push a non-volatile register in prolog: */  
+#  define PUSH_SEH(reg) push reg @N@ .pushreg reg
+/* Push a volatile register or an immed in prolog: */  
+#  define PUSH_NONCALLEE_SEH(reg) push reg @N@ .allocstack 8
+#  define END_PROLOG .endprolog
+# else
+#  define DECLARE_FUNC_SEH(symbol) DECLARE_FUNC(symbol)
+#  define PUSH_SEH(reg) push reg
+#  define PUSH_NONCALLEE_SEH(reg) push reg
+#  define END_PROLOG /* nothing */
+# endif
+/****************************************************/
+#elif defined(ASSEMBLE_WITH_NASM)
+# define START_FILE SECTION .text
+# define END_FILE /* nothing */
+# define DECLARE_FUNC(symbol) global symbol
+# define DECLARE_EXPORTED_FUNC(symbol) global symbol
+# define END_FUNC(symbol) /* nothing */
+# define DECLARE_GLOBAL(symbol) global symbol
+# define GLOBAL_LABEL(label) label
+# define WORD word
+# define DWORD dword
+# define QWORD qword
+# define SYMREF(sym) [sym]
+# define HEX(n) 0x##n
+# define SEGMEM(seg,mem) [seg:mem]
+# define DECL_EXTERN(symbol) EXTERN symbol
+# define RAW(n) error_not_implemented
+# define DECLARE_FUNC_SEH(symbol) DECLARE_FUNC(symbol)
+# define PUSH_SEH(reg) push reg
+# define PUSH_NONCALLEE_SEH(reg) push reg
+# define END_PROLOG /* nothing */
+/****************************************************/
+#else
+# error Unknown assembler
+#endif
+
+/****************************************************/
+/* Macros for writing cross-platform 64-bit + 32-bit code */
+#ifdef X64
+# define REG_XAX rax
+# define REG_XBX rbx
+# define REG_XCX rcx
+# define REG_XDX rdx
+# define REG_XSI rsi
+# define REG_XDI rdi
+# define REG_XBP rbp
+# define REG_XSP rsp
+# define SEG_TLS gs /* keep in sync w/ {linux,win32}/os_exports.h defines */
+# ifdef WINDOWS
+/* Arguments are passed in: rcx, rdx, r8, r9, then on stack right-to-left, but
+ * leaving space on stack for the 1st 4.
+ */
+#  define ARG1 rcx
+#  define ARG2 rdx
+#  define ARG3 r8
+#  define ARG4 r9
+#  define ARG5 QWORD [40 + esp] /* includes ret addr */
+#  define ARG6 QWORD [48 + esp]
+#  define ARG5_NORETADDR QWORD [32 + esp]
+# else
+/* Arguments are passed in: rdi, rsi, rdx, rcx, r8, r9, then on stack right-to-left,
+ * without leaving any space on stack for the 1st 6.
+ */
+#  define ARG1 rdi
+#  define ARG2 rsi
+#  define ARG3 rdx
+#  define ARG4 rcx
+#  define ARG5 r8
+#  define ARG5_NORETADDR ARG5
+#  define ARG6 r9
+# endif
+# define ARG_SZ 8
+# define PTRSZ QWORD
+#else /* x86 */
+# define REG_XAX eax
+# define REG_XBX ebx
+# define REG_XCX ecx
+# define REG_XDX edx
+# define REG_XSI esi
+# define REG_XDI edi
+# define REG_XBP ebp
+# define REG_XSP esp
+# define SEG_TLS fs /* keep in sync w/ {linux,win32}/os_exports.h defines */
+# define ARG_SZ 4
+# define PTRSZ DWORD
+/* Arguments are passed on stack right-to-left. */
+# define ARG1 DWORD [4 + esp] /* includes ret addr */
+# define ARG2 DWORD [8 + esp] 
+# define ARG3 DWORD [12 + esp]
+# define ARG4 DWORD [16 + esp]
+# define ARG5 DWORD [20 + esp]
+# define ARG6 DWORD [24 + esp]
+#endif
+
+#ifdef X64
+#  define PUSHF   pushfq
+#  define POPF    popfq
+#  ifdef WINDOWS
+#    define STACK_PAD(tot, gt4) \
+        lea      REG_XSP, [-32 - ARG_SZ*gt4 + REG_XSP]
+#    define STACK_UNPAD(tot, gt4) \
+        lea      REG_XSP, [32 + ARG_SZ*gt4 + REG_XSP]
+/* we split these out just to avoid nop lea for x64 linux */
+#    define STACK_PAD_LE4 STACK_PAD(0/*doesn't matter*/, 0)
+#    define STACK_UNPAD_LE4(tot) STACK_UNPAD(tot, 0)
+#  else
+#    define STACK_PAD(tot, gt4) \
+        lea      REG_XSP, [-ARG_SZ*gt4 + REG_XSP]
+#    define STACK_UNPAD(tot, gt4) \
+        lea      REG_XSP, [ARG_SZ*gt4 + REG_XSP]
+#    define STACK_PAD_LE4 /* nothing */
+#    define STACK_UNPAD_LE4(tot) /* nothing */
+#  endif
+#  define SETARG(argreg, p) \
+        mov      argreg, p
+#else
+#  define PUSHF   pushfd
+#  define POPF    popfd
+#  define STACK_PAD(tot, gt4) /* nothing */
+#  define STACK_UNPAD(tot, gt4) \
+        lea      REG_XSP, [ARG_SZ*tot + REG_XSP]
+#  define STACK_PAD_LE4 /* nothing */
+#  define STACK_UNPAD_LE4(tot) STACK_UNPAD(tot, 0)
+/* SETARG usage is order-dependent on 32-bit.  we could avoid that
+ * by having STACK_PAD allocate the stack space and do a mov here.
+ */
+#  define SETARG(argreg, p) \
+        push     p
+#endif
+/* CALLC* are for C calling convention callees only.
+ * Caller must ensure that if params are passed in regs there are no conflicts.
+ * Caller can rely on us storing each parameter in reverse order.
+ * For x64, caller must arrange for 16-byte alignment at end of arg setup.
+ */
+#define CALLC0(callee)     \
+        STACK_PAD_LE4   @N@\
+        call     callee @N@\
+        STACK_UNPAD_LE4(0)
+#define CALLC1(callee, p1)    \
+        STACK_PAD_LE4      @N@\
+        SETARG(ARG1, p1)   @N@\
+        call     callee    @N@\
+        STACK_UNPAD_LE4(1)
+#define CALLC2(callee, p1, p2)    \
+        STACK_PAD_LE4          @N@\
+        SETARG(ARG2, p2)       @N@\
+        SETARG(ARG1, p1)       @N@\
+        call     callee        @N@\
+        STACK_UNPAD_LE4(2)
+#define CALLC3(callee, p1, p2, p3)    \
+        STACK_PAD_LE4              @N@\
+        SETARG(ARG3, p3)           @N@\
+        SETARG(ARG2, p2)           @N@\
+        SETARG(ARG1, p1)           @N@\
+        call     callee            @N@\
+        STACK_UNPAD_LE4(3)
+#define CALLC4(callee, p1, p2, p3, p4)    \
+        STACK_PAD_LE4                  @N@\
+        SETARG(ARG4, p4)               @N@\
+        SETARG(ARG3, p3)               @N@\
+        SETARG(ARG2, p2)               @N@\
+        SETARG(ARG1, p1)               @N@\
+        call     callee                @N@\
+        STACK_UNPAD_LE4(4)
+#define CALLC5(callee, p1, p2, p3, p4, p5)    \
+        STACK_PAD(5, 1)                    @N@\
+        SETARG(ARG5_NORETADDR, p5)         @N@\
+        SETARG(ARG4, p4)                   @N@\
+        SETARG(ARG3, p3)                   @N@\
+        SETARG(ARG2, p2)                   @N@\
+        SETARG(ARG1, p1)                   @N@\
+        call     callee                    @N@\
+        STACK_UNPAD(5, 1)
+
+/* For stdcall callees */
+#ifdef X64
+# define CALLWIN0 CALLC0
+# define CALLWIN1 CALLC1
+# define CALLWIN2 CALLC2
+#else
+# define CALLWIN0(callee)     \
+        STACK_PAD_LE4   @N@\
+        call     callee
+# define CALLWIN1(callee, p1)    \
+        STACK_PAD_LE4      @N@\
+        SETARG(ARG1, p1)   @N@\
+        call     callee
+# define CALLWIN2(callee, p1, p2)    \
+        STACK_PAD_LE4          @N@\
+        SETARG(ARG2, p2)       @N@\
+        SETARG(ARG1, p1)       @N@\
+        call     callee
+#endif
+
+
+#endif /* _ASM_DEFINES_ASM_ */
diff --git a/core/x86/decode.c b/core/x86/decode.c
new file mode 100644
index 000000000..e1b864e3f
--- /dev/null
+++ b/core/x86/decode.c
@@ -0,0 +1,1754 @@
+/* **********************************************************
+ * Copyright (c) 2000-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/* decode.c -- a full x86 decoder */
+
+#include "globals.h"
+#include "arch.h"
+#include "instr.h"
+#include "decode.h"
+#include "decode_fast.h"
+#include <string.h> /* for memcpy */
+
+/*
+ * FIXME
+ *
+ * 1) several opcodes gdb thinks bad/not bad -- changes to ISA?
+ * 2) why does gdb use Ex, Ed, & Ev for all float opcodes w/ modrm < 0xbf?
+ *    a float instruction's modrm cannot specify a register, right?
+ *    sizes are single-real => d, double-real => q, extended-real = 90 bits,
+ *    14/28 bytes, and 98/108 bytes!
+ *    should address sizes all be 'v'?
+ * 3) there don't seem to be any immediate float values...?!?
+ * 4) fld (0xd9 0xc0-7) in Table A-10 has 2 operands in different order
+ *    than expected, plus asm prints using just 2nd one
+ * 5) I don't see T...is there a T?  gdb has it for 0f26mov
+ */
+
+/* N.B.: must justify each assert, since we do not want to assert on a bad
+ * instruction -- we want to fail gracefully and have the caller deal with it
+ */
+
+#ifdef DEBUG
+/* case 10450: give messages to clients */
+/* we can't undef ASSERT b/c of DYNAMO_OPTION */
+# undef ASSERT_TRUNCATE
+# undef ASSERT_BITFIELD_TRUNCATE
+# undef ASSERT_NOT_REACHED
+# define ASSERT_TRUNCATE DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_BITFIELD_TRUNCATE DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_NOT_REACHED DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+#endif
+
+#ifdef X64
+/* PR 302344: used for shared traces -tracedump_origins where we
+ * need to change the mode but we have no dcontext
+ */
+static bool initexit_x86_mode = DEFAULT_X86_MODE;
+
+/*
+ * The decode and encode routines use a per-thread persistent flag that
+ * indicates whether to treat code as 32-bit (x86) or 64-bit (x64).  This
+ * routine sets that flag to the indicated value and returns the old value.  Be
+ * sure to restore the old value prior to any further application execution to
+ * avoid problems in mis-interpreting application code.
+ */
+bool
+set_x86_mode(dcontext_t *dcontext, bool x86)
+{
+    bool old_mode;
+    /* We would disallow but some early init routines need to use global heap */
+    if (dcontext == GLOBAL_DCONTEXT)
+        dcontext = get_thread_private_dcontext();
+    if (dcontext == NULL) {
+        ASSERT(!dynamo_initialized || dynamo_exited);
+        old_mode = initexit_x86_mode;
+        initexit_x86_mode = x86;
+    } else {
+        old_mode = dcontext->x86_mode;
+        dcontext->x86_mode = x86;
+    }
+    return old_mode;
+}
+
+/*
+ * The decode and encode routines use a per-thread persistent flag that
+ * indicates whether to treat code as 32-bit (x86) or 64-bit (x64).  This
+ * routine returns the value of that flag.
+ */
+bool
+get_x86_mode(dcontext_t *dcontext)
+{
+    /* We would disallow but some early init routines need to use global heap */
+    if (dcontext == GLOBAL_DCONTEXT)
+        dcontext = get_thread_private_dcontext();
+    if (dcontext == NULL) {
+        ASSERT(!dynamo_initialized || dynamo_exited);
+        return initexit_x86_mode;
+    } else
+        return dcontext->x86_mode;
+}
+#endif
+
+/****************************************************************************
+ * All code below based on tables in the ``Intel Architecture Software
+ * Developer's Manual,'' Volume 2: Instruction Set Reference, 2001.
+ */
+
+#ifdef DEBUG /* currently only used in ASSERTs */
+static bool
+is_variable_size(opnd_size_t sz)
+{
+    switch (sz) {
+    case OPSZ_2_short1:
+    case OPSZ_4_short2:
+    case OPSZ_4x8:
+    case OPSZ_4x8_short2:
+    case OPSZ_4x8_short2xi8:
+    case OPSZ_4_short2xi4:
+    case OPSZ_4_rex8_short2:
+    case OPSZ_4_rex8:
+    case OPSZ_6_irex10_short4:
+    case OPSZ_6x10:
+    case OPSZ_8_short2:
+    case OPSZ_8_short4:
+    case OPSZ_28_short14:
+    case OPSZ_108_short94:
+    case OPSZ_1_reg4:
+    case OPSZ_2_reg4:
+    case OPSZ_4_reg16:
+        return true;
+    default:
+        return false;
+    }
+}
+#endif
+
+opnd_size_t
+resolve_var_reg_size(opnd_size_t sz, bool is_reg)
+{
+    switch (sz) {
+    case OPSZ_1_reg4: return (is_reg ? OPSZ_4 : OPSZ_1);
+    case OPSZ_2_reg4: return (is_reg ? OPSZ_4 : OPSZ_2);
+    case OPSZ_4_reg16: return (is_reg ? OPSZ_16 : OPSZ_4);
+    }
+    return sz;
+}
+
+/* Like all our code, we assume cs specifies default data and address sizes.
+ * This routine assumes the size varies by data, NOT by address!
+ */
+opnd_size_t
+resolve_variable_size(decode_info_t *di/*IN: x86_mode, prefixes*/,
+                      opnd_size_t sz, bool is_reg)
+{
+    switch (sz) {
+    case OPSZ_2_short1: return (TEST(PREFIX_DATA, di->prefixes) ? OPSZ_1 : OPSZ_2);
+    case OPSZ_4_short2: return (TEST(PREFIX_DATA, di->prefixes) ? OPSZ_2 : OPSZ_4);
+    case OPSZ_4x8: return (X64_MODE(di) ? OPSZ_8 : OPSZ_4);
+    case OPSZ_4x8_short2:
+        return (TEST(PREFIX_DATA, di->prefixes) ? OPSZ_2 :
+                (X64_MODE(di) ? OPSZ_8 : OPSZ_4));
+    case OPSZ_4x8_short2xi8:
+        return (X64_MODE(di) ? (proc_get_vendor() == VENDOR_INTEL ? OPSZ_8 :
+                              (TEST(PREFIX_DATA, di->prefixes) ? OPSZ_2 : OPSZ_8)) :
+                (TEST(PREFIX_DATA, di->prefixes) ? OPSZ_2 : OPSZ_4));
+    case OPSZ_4_short2xi4:
+        return (X64_MODE(di) && proc_get_vendor() == VENDOR_INTEL ? OPSZ_4 :
+                (TEST(PREFIX_DATA, di->prefixes) ? OPSZ_2 : OPSZ_4));
+    case OPSZ_4_rex8_short2: /* rex.w trumps data prefix */
+        return (TEST(PREFIX_REX_W, di->prefixes) ? OPSZ_8 :
+                (TEST(PREFIX_DATA, di->prefixes) ? OPSZ_2 : OPSZ_4));
+    case OPSZ_4_rex8: return (TEST(PREFIX_REX_W, di->prefixes) ? OPSZ_8 : OPSZ_4);
+    case OPSZ_6_irex10_short4: /* rex.w trumps data prefix, but is ignored on AMD */
+        ASSERT_CURIOSITY(!TEST(PREFIX_REX_W, di->prefixes));
+        return ((TEST(PREFIX_REX_W, di->prefixes) && proc_get_vendor() != VENDOR_AMD) ?
+                OPSZ_10 : (TEST(PREFIX_DATA, di->prefixes) ? OPSZ_4 : OPSZ_6));
+    case OPSZ_6x10: return (X64_MODE(di) ? OPSZ_10 : OPSZ_6);
+    case OPSZ_8_short2: return (TEST(PREFIX_DATA, di->prefixes) ? OPSZ_2 : OPSZ_8);
+    case OPSZ_8_short4: return (TEST(PREFIX_DATA, di->prefixes) ? OPSZ_4 : OPSZ_8);
+    case OPSZ_28_short14:
+        return (TEST(PREFIX_DATA, di->prefixes) ?  OPSZ_14 : OPSZ_28);
+    case OPSZ_108_short94:
+        return (TEST(PREFIX_DATA, di->prefixes) ?  OPSZ_94 : OPSZ_108);
+    case OPSZ_1_reg4:
+    case OPSZ_2_reg4:
+    case OPSZ_4_reg16:
+        return resolve_var_reg_size(sz, is_reg);
+    }
+    return sz;
+}
+
+opnd_size_t
+resolve_variable_size_dc(dcontext_t *dcontext, uint prefixes, opnd_size_t sz, bool is_reg)
+{
+    decode_info_t di;
+    IF_X64(di.x86_mode = get_x86_mode(dcontext));
+    di.prefixes = prefixes;
+    return resolve_variable_size(&di, sz, is_reg);
+}
+
+opnd_size_t
+resolve_addr_size(decode_info_t *di/*IN: x86_mode, prefixes*/)
+{
+    if (TEST(PREFIX_ADDR, di->prefixes))
+        return (X64_MODE(di) ? OPSZ_4 : OPSZ_2);
+    else
+        return (X64_MODE(di) ? OPSZ_8 : OPSZ_4);
+}
+
+opnd_size_t
+indir_var_reg_size(int optype)
+{
+    switch (optype) {
+    case TYPE_INDIR_VAR_XREG: return OPSZ_VARSTACK;
+    case TYPE_INDIR_VAR_XIREG: return OPSZ_4x8_short2xi8;
+    case TYPE_INDIR_VAR_REG: return OPSZ_REXVARSTACK;
+    default: CLIENT_ASSERT(false, "internal error: indir_var_reg_size invalid param");
+    }
+    return OPSZ_VARSTACK;
+}
+
+/****************************************************************************
+ * Reading all bytes of instruction
+ */
+
+static byte *
+read_immed(byte *pc, decode_info_t *di, opnd_size_t size, ptr_int_t *result)
+{
+    size = resolve_variable_size(di, size, false);
+
+    /* all data immediates are sign-extended.  we use the compiler's casts with
+     * signed types to do our sign extensions for us.
+     */
+    switch (size) {
+    case OPSZ_1:
+        *result = (ptr_int_t) (char) *pc; /* sign-extend */
+        pc++;
+        break;
+    case OPSZ_2:
+        *result = (ptr_int_t) *((short*)pc); /* sign-extend */
+        pc += 2;
+        break;
+    case OPSZ_4:
+        *result = (ptr_int_t) *((int*)pc); /* sign-extend */
+        pc += 4;
+        break;
+    case OPSZ_8:
+        CLIENT_ASSERT(X64_MODE(di), "decode immediate: invalid size");
+        CLIENT_ASSERT(sizeof(ptr_int_t) == 8, "decode immediate: internal size error");
+        *result = *((ptr_int_t*)pc);
+        pc += 8;
+        break;
+    default:
+        /* called internally w/ instr_info_t fields or hardcoded values,
+         * so ok to assert */
+        CLIENT_ASSERT(false, "decode immediate: unknown size");
+    }
+    return pc;
+}
+
+/* reads any trailing immed bytes */
+static byte *
+read_operand(byte *pc, decode_info_t *di, byte optype, opnd_size_t opsize)
+{
+    ptr_int_t val = 0;
+    opnd_size_t size = opsize;
+    switch (optype) {
+    case TYPE_A:
+        {
+            CLIENT_ASSERT(!X64_MODE(di), "x64 has no type A instructions");
+#ifdef IA32_ON_IA64
+            /* somewhat hacked dispatch on size */
+            if (opsize == OPSZ_4_short2) {
+                pc = read_immed(pc, di, opsize, &val);
+                break;
+            }
+#endif
+            /* ok b/c only instr_info_t fields passed */
+            CLIENT_ASSERT(opsize == OPSZ_6_irex10_short4, "decode A operand error");
+            if (TEST(PREFIX_DATA, di->prefixes)) {
+                /* 4-byte immed */
+                pc = read_immed(pc, di, OPSZ_4, &val);
+#ifdef X64
+                if (!X64_MODE(di)) {
+                    /* we do not want the sign extension that read_immed() applied */
+                    val &= (ptr_int_t) 0x00000000ffffffff;
+                }
+#endif
+                /* ok b/c only instr_info_t fields passed */
+                CLIENT_ASSERT(di->size_immed == OPSZ_NA &&
+                              di->size_immed2 == OPSZ_NA, "decode A operand error");
+                di->size_immed = resolve_variable_size(di, opsize, false);
+                ASSERT(di->size_immed == OPSZ_4);
+                di->immed = val;
+            } else {
+                /* 6-byte immed */
+                ptr_int_t val2 = 0;
+                /* little-endian: segment comes last */
+                pc = read_immed(pc, di, OPSZ_4, &val2);
+                pc = read_immed(pc, di, OPSZ_2, &val);
+#ifdef X64
+                if (!X64_MODE(di)) {
+                    /* we do not want the sign extension that read_immed() applied */
+                    val2 &= (ptr_int_t) 0x00000000ffffffff;
+                }
+#endif
+                /* ok b/c only instr_info_t fields passed */
+                CLIENT_ASSERT(di->size_immed == OPSZ_NA &&
+                              di->size_immed2 == OPSZ_NA, "decode A operand error");
+                di->size_immed = resolve_variable_size(di, opsize, false);
+                ASSERT(di->size_immed == OPSZ_6);
+                di->size_immed2 = resolve_variable_size(di, opsize, false);
+                di->immed = val;
+                di->immed2 = val2;
+            }
+            return pc;
+        }
+    case TYPE_I:
+        {
+            pc = read_immed(pc, di, opsize, &val);
+            break;
+        }
+    case TYPE_J:
+        {
+            byte *end_pc;
+            pc = read_immed(pc, di, opsize, &val);
+            if (di->orig_pc != di->start_pc) {
+                CLIENT_ASSERT(di->start_pc != NULL,
+                              "internal decode error: start pc not set");
+                end_pc = di->orig_pc + (pc - di->start_pc);
+            } else
+                end_pc = pc;
+            /* convert from relative offset to absolute target pc */
+            val = ((ptr_int_t)end_pc) + val;
+            if ((!X64_MODE(di) || proc_get_vendor() != VENDOR_INTEL) &&
+                TEST(PREFIX_DATA, di->prefixes)) {
+                /* need to clear upper 16 bits */
+                val &= (ptr_int_t) 0x0000ffff;
+            } /* for x64 Intel, always 64-bit addr ("f64" in Intel table) */
+            break;
+        }
+    case TYPE_O:
+        {
+            /* no modrm byte, offset follows directly.  this is address-sized,
+             * so 64-bit for x64, and addr prefix affects it. */
+            size = resolve_addr_size(di);
+            pc = read_immed(pc, di, size, &val);
+            if (TEST(PREFIX_ADDR, di->prefixes)) {
+                /* need to clear upper bits */
+                if (X64_MODE(di))
+                    val &= (ptr_int_t) 0xffffffff;
+                else
+                    val &= (ptr_int_t) 0x0000ffff;
+            }
+#ifdef X64
+            if (!X64_MODE(di)) {
+                /* we do not want the sign extension that read_immed() applied */
+                val &= (ptr_int_t) 0x00000000ffffffff;
+            }
+#endif
+            break;
+        }
+    default:
+        return pc;
+    }
+    if (di->size_immed == OPSZ_NA) {
+        di->size_immed = size;
+        di->immed = val;
+    } else {
+        /* ok b/c only instr_info_t fields passed */
+        CLIENT_ASSERT(di->size_immed2 == OPSZ_NA, "decode operand error");
+        di->size_immed2 = size;
+        di->immed2 = val;
+    }
+    return pc;
+}
+
+/* reads the modrm byte and any following sib and offset bytes */
+static byte *
+read_modrm(byte *pc, decode_info_t *di)
+{
+    byte modrm = *pc;
+    pc++;
+    di->modrm = modrm;
+    di->mod = (byte)((modrm >> 6) & 0x3); /* top 2 bits */
+    di->reg = (byte)((modrm >> 3) & 0x7); /* middle 3 bits */
+    di->rm  = (byte)(modrm & 0x7);        /* bottom 3 bits */
+
+    /* addr16 displacement */
+    if (!X64_MODE(di) && TEST(PREFIX_ADDR, di->prefixes)) {
+        di->has_sib = false;
+        if ((di->mod == 0 && di->rm == 6) || di->mod == 2) {
+            /* 2-byte disp */
+            di->has_disp = true;
+            if (di->mod == 0 && di->rm == 6) {
+                /* treat absolute addr as unsigned */
+                di->disp = (int) *((ushort *)pc); /* zero-extend */
+            } else {
+                /* treat relative addr as signed */
+                di->disp = (int) *((short *)pc); /* sign-extend */
+            }
+            pc += 2;
+        } else if (di->mod == 1) {
+            /* 1-byte disp */
+            di->has_disp = true;
+            di->disp = (int) (char) *pc; /* sign-extend */
+            pc++;
+        } else {
+            di->has_disp = false;
+        }
+    } else {
+        /* 32-bit, which sometimes has a SIB */
+        if (di->rm == 4 && di->mod != 3) {
+            /* need SIB */
+            byte sib = *pc;
+            pc++;
+            di->has_sib = true;
+            di->scale = (byte)((sib >> 6) & 0x3); /* top 2 bits */
+            di->index = (byte)((sib >> 3) & 0x7); /* middle 3 bits */
+            di->base  = (byte)(sib & 0x7);        /* bottom 3 bits */
+        } else {
+            di->has_sib = false;
+        }
+
+        /* displacement */
+        if ((di->mod == 0 && di->rm == 5) ||
+            (di->has_sib && di->mod == 0 && di->base == 5) ||
+            di->mod == 2) {
+            /* 4-byte disp */
+            di->has_disp = true;
+            di->disp = *((int *)pc);
+            IF_X64(di->disp_abs = pc); /* used to set instr->rip_rel_pos */
+            pc += 4;
+        } else if (di->mod == 1) {
+            /* 1-byte disp */
+            di->has_disp = true;
+            di->disp = (int) (char) *pc; /* sign-extend */
+            pc++;
+        } else {
+            di->has_disp = false;
+        }
+    }
+    return pc;
+}
+
+/* Disassembles the instruction at pc into the data structures ret_info
+ * and di.  Does NOT set or read di->len.
+ * Returns a pointer to the pc of the next instruction.
+ * If just_opcode is true, does not decode the immeds and returns NULL
+ * (you must call decode_next_pc to get the next pc, but that's faster
+ * than decoding the immeds)
+ * Returns NULL on an invalid instruction
+ */
+static byte *
+read_instruction(byte *pc, byte *orig_pc,
+                 const instr_info_t **ret_info, decode_info_t *di,
+                 bool just_opcode _IF_DEBUG(bool report_invalid))
+{
+    DEBUG_DECLARE(byte *post_suffix_pc = NULL;)
+    byte first_instr_byte;
+    const instr_info_t *info;
+    /* these 3 prefixes may be part of opcode: */
+    bool data_prefix = false;
+    bool rep_prefix = false;
+    bool repne_prefix = false;
+
+    /* initialize di */
+    /* though we only need di->start_pc for full decode rip-rel (and
+     * there only post-read_instruction()) and decode_from_copy(), and
+     * di->orig_pc only for decode_from_copy(), we assume that
+     * high-perf decoding uses decode_cti() and live w/ the extra
+     * writes here for decode_opcode() and decode_eflags_usage().
+     */
+    di->start_pc = pc;
+    di->orig_pc = orig_pc;
+    di->size_immed = OPSZ_NA;
+    di->size_immed2 = OPSZ_NA;
+    di->seg_override = REG_NULL;
+    /* FIXME: set data and addr sizes to current mode
+     * for now I assume always 32-bit mode (or 64 for X64_MODE(di))!
+     */
+    di->prefixes = 0;
+    
+    do {
+        first_instr_byte = *pc;
+        pc++;
+        info = &first_byte[first_instr_byte];
+        if (info->type == X64_EXT) {
+            /* discard old info, get new one */
+            info = &x64_extensions[info->code][X64_MODE(di) ? 1 : 0];
+        }
+        if (info->type == PREFIX) {
+            if (TESTANY(PREFIX_REX_ALL, di->prefixes)) {
+                /* rex.* must come after all other prefixes (including those that are
+                 * part of the opcode, xref PR 271878): so discard them if before
+                 * matching the behavior of decode_sizeof().  This in effect nops
+                 * improperly placed rex prefixes which (xref PR 241563 and Intel Manual
+                 * 2A 2.2.1) is the correct thing to do. NOTE - windbg shows early bytes
+                 * as ??, objdump as their prefix names, separate from the next instr.
+                 */
+                di->prefixes &= ~PREFIX_REX_ALL;
+            }
+            if (info->code == PREFIX_REP) {
+                /* see if used as part of opcode before considering prefix */
+                rep_prefix = true;
+            } else if (info->code == PREFIX_REPNE) {
+                /* see if used as part of opcode before considering prefix */
+                repne_prefix = true;
+            } else if (REG_START_SEGMENT <= info->code &&
+                       info->code <= REG_STOP_SEGMENT) {
+                CLIENT_ASSERT_TRUNCATE(di->seg_override, byte, info->code,
+                                       "decode error: invalid segment override");
+                di->seg_override = (byte) info->code;
+            } else if (info->code == PREFIX_DATA) {
+                /* see if used as part of opcode before considering prefix */
+                data_prefix = true;
+            } else if (TESTANY(PREFIX_REX_ALL | PREFIX_ADDR | PREFIX_LOCK,
+                               info->code)) {
+                di->prefixes |= info->code;
+            }
+        } else
+            break;
+    } while (true);
+
+    if (info->type == ESCAPE) {
+        /* discard first byte, move to second */
+        first_instr_byte = *pc; /* really 2nd, just reusing var */
+        pc++;
+        info = &second_byte[first_instr_byte];
+    }
+    if (info->type == ESCAPE_3BYTE_38 ||
+        info->type == ESCAPE_3BYTE_3a) {
+        /* discard second byte, move to third */
+        first_instr_byte = *pc; /* really 3rd, just reusing var */
+        pc++;
+        if (info->type == ESCAPE_3BYTE_38)
+            info = &third_byte_38[third_byte_38_index[first_instr_byte]];
+        else
+            info = &third_byte_3a[third_byte_3a_index[first_instr_byte]];
+    }
+
+    /* all FLOAT_EXT and PREFIX_EXT (except nop & pause) and EXTENSION need modrm,
+     * get it now 
+     */
+    if ((info->flags & HAS_MODRM) != 0)
+        pc = read_modrm(pc, di);
+
+    if (info->type == FLOAT_EXT) {
+        if (di->modrm <= 0xbf) {
+            int offs = (first_instr_byte - 0xd8) * 8 + di->reg;
+            info = &float_low_modrm[offs];
+        } else {
+            int offs1 = (first_instr_byte - 0xd8);
+            int offs2 = di->modrm - 0xc0;
+            info = &float_high_modrm[offs1][offs2];
+        }
+    }
+    else if (info->type == PREFIX_EXT) {
+        /* discard old info, get new one */
+        int code = (int) info->code;
+        int idx = (rep_prefix?1 :(data_prefix?2 :(repne_prefix?3 :0)));
+        info = &prefix_extensions[code][idx];
+        if (rep_prefix)
+            rep_prefix = false;
+        else if (data_prefix)
+            data_prefix = false;
+        else if (repne_prefix)
+            repne_prefix = false;
+        if (info->type == REX_EXT) {
+            /* discard old info, get new one */
+            int code = (int) info->code;
+            /* currently indexed by rex.b only */
+            int idx = (TEST(PREFIX_REX_B, di->prefixes) ? 1 : 0);
+            info = &rex_extensions[code][idx];
+        }
+    }
+    else if (info->type == REP_EXT) {
+        /* discard old info, get new one */
+        int code = (int) info->code;
+        int idx = (rep_prefix ? 2 : 0);
+        info = &rep_extensions[code][idx];
+        if (rep_prefix)
+            rep_prefix = false;
+    }
+    else if (info->type == REPNE_EXT) {
+        /* discard old info, get new one */
+        int code = (int) info->code;
+        int idx = (rep_prefix? 2 : (repne_prefix? 4 :0));
+        info = &repne_extensions[code][idx];
+        rep_prefix = false;
+        repne_prefix = false;
+    }
+    else if (info->type == EXTENSION) {
+        /* discard old info, get new one */
+        info = &extensions[info->code][di->reg];
+        /* absurd cases of using prefix on top of reg opcode extension
+         * (pslldq, psrldq)
+         */
+        if (info->type == PREFIX_EXT) {
+            /* discard old info, get new one */
+            int code = (int) info->code;
+            int idx = (rep_prefix?1 :(data_prefix?2 :(repne_prefix?3 :0)));
+            info = &prefix_extensions[code][idx];
+            if (rep_prefix)
+                rep_prefix = false;
+            else if (data_prefix)
+                data_prefix = false;
+            else if (repne_prefix)
+                repne_prefix = false;
+        } else if (info->type == MOD_EXT) {
+            info = &mod_extensions[info->code][(di->mod==3) ? 1 : 0];
+            /* Yes, we have yet another layer, thanks to Intel's poor choice
+             * in opcodes -- why didn't they fill out the PREFIX_EXT space?
+             */
+            if (info->type == RM_EXT) {
+                info = &rm_extensions[info->code][di->rm];
+            }
+        }
+    }
+    else if (info->type == SUFFIX_EXT) {
+        /* Discard old info, get new one for complete opcode, which includes
+         * a suffix byte where an immed would be (yes, ugly!).
+         * We should have already read in the modrm (+ sib).
+         */
+        CLIENT_ASSERT(TEST(HAS_MODRM, info->flags), "decode error on 3DNow instr");
+        info = &suffix_extensions[suffix_index[*pc]];
+        pc++;
+        DEBUG_DECLARE(post_suffix_pc = pc;)
+    }
+
+    if (TEST(REQUIRES_PREFIX, info->flags)) {
+        byte required = (byte)(info->opcode >> 24);
+        bool *prefix_var = NULL;
+        CLIENT_ASSERT(info->opcode > 0xffffff, "decode error in SSSE3/SSE4 instr");
+        if (required == DATA_PREFIX_OPCODE)
+            prefix_var = &data_prefix;
+        else if (required == REPNE_PREFIX_OPCODE)
+            prefix_var = &repne_prefix;
+        else if (required == REP_PREFIX_OPCODE)
+            prefix_var = &rep_prefix;
+        else
+            CLIENT_ASSERT(false, "internal required-prefix error");
+        if (prefix_var == NULL || !*prefix_var) {
+            /* Invalid instr.  TODO: have processor w/ SSE4, confirm that
+             * an exception really is raised.
+             */
+            info = NULL;
+        } else
+            *prefix_var = false;
+    }
+
+    /* at this point should be an instruction, so type should be an OP_ constant */
+    if (info == NULL || info->type < OP_FIRST || info->type > OP_LAST ||
+        (X64_MODE(di) && TEST(X64_INVALID, info->flags)) ||
+        (!X64_MODE(di) && TEST(X86_INVALID, info->flags))) {
+        /* invalid instruction: up to caller to decide what to do with it */
+        /* FIXME case 10672: provide a runtime option to specify new
+         * instruction formats */
+        DODEBUG({
+            if (report_invalid) {
+                SYSLOG_INTERNAL_WARNING_ONCE("Invalid opcode encountered");
+                if (info != NULL && info->type == INVALID) {
+                    LOG(THREAD_GET, LOG_ALL, 1, "Invalid opcode @"PFX": 0x%x\n",
+                        di->start_pc, info->opcode);
+                } else {
+                    int i;
+                    dcontext_t *dcontext = get_thread_private_dcontext();
+                    IF_X64(bool old_mode = set_x86_mode(dcontext, di->x86_mode);)
+                    int sz = decode_sizeof(dcontext, di->start_pc, NULL _IF_X64(NULL));
+                    IF_X64(set_x86_mode(dcontext, old_mode));
+                    LOG(THREAD_GET, LOG_ALL, 1, "Error decoding "PFX" == ", di->start_pc);
+                    for (i=0; i<sz; i++) {
+                        LOG(THREAD_GET, LOG_ALL, 1, "0x%x ", *(di->start_pc+i));
+                    }
+                    LOG(THREAD_GET, LOG_ALL, 1, "\n");
+                }
+            }
+        });
+        *ret_info = &invalid_instr;
+        return NULL;
+    }
+        
+#ifdef INTERNAL
+    DODEBUG({ /* rep & repne should have been completely handled by now */
+        /* processor will typically ignore extra prefixes, but we log this internally
+         * in case it's our decode messing up instead of weird app instrs
+         */
+        if (report_invalid &&
+            ((rep_prefix &&
+              /* case 6861: AMD64 opt: "rep ret" used if br tgt or after cbr */
+              (pc != di->start_pc+2 || *(di->start_pc+1) != RAW_OPCODE_ret))
+             || repne_prefix)) {
+            char bytes[17*3];
+            int i;
+            dcontext_t *dcontext = get_thread_private_dcontext();
+            IF_X64(bool old_mode = set_x86_mode(dcontext, di->x86_mode);)
+            int sz = decode_sizeof(dcontext, di->start_pc, NULL _IF_X64(NULL));
+            IF_X64(set_x86_mode(dcontext, old_mode));
+            CLIENT_ASSERT(sz <= 17, "decode rep/repne error: unsupported opcode?");
+            for (i=0; i<sz; i++)
+                snprintf(&bytes[i*3], 3, "%02x ", *(di->start_pc+i));
+            bytes[sz*3-1] = '\0'; /* -1 to kill trailing space */
+            SYSLOG_INTERNAL_WARNING_ONCE("spurious rep/repne prefix @"PFX" (%s): "
+                                         "decoding error?", di->start_pc, bytes);
+        }
+    });
+#endif
+
+    /* if just want opcode, stop here!  faster for caller to
+     * separately call decode_next_pc than for us to decode immeds!  
+     */
+    if (just_opcode) {
+        *ret_info = info;
+        return NULL;
+    }
+
+    if (data_prefix) {
+        /* prefix was not part of opcode, it's a real prefix */
+        di->prefixes |= PREFIX_DATA;
+    }
+    
+    /* read any trailing immediate bytes */
+    if (info->dst1_type != TYPE_NONE)
+        pc = read_operand(pc, di, info->dst1_type, info->dst1_size);
+    if (info->dst2_type != TYPE_NONE)
+        pc = read_operand(pc, di, info->dst2_type, info->dst2_size);
+    if (info->src1_type != TYPE_NONE)
+        pc = read_operand(pc, di, info->src1_type, info->src1_size);
+    if (info->src2_type != TYPE_NONE)
+        pc = read_operand(pc, di, info->src2_type, info->src2_size);
+    if (info->src3_type != TYPE_NONE)
+        pc = read_operand(pc, di, info->src3_type, info->src3_size);
+
+    if (info->type == SUFFIX_EXT) {
+        /* Shouldn't be any more bytes (immed bytes) read after the modrm+suffix! */
+        DODEBUG({CLIENT_ASSERT(pc == post_suffix_pc, "decode error on 3DNow instr");});
+    }
+
+    /* return values */
+    *ret_info = info;
+    return pc;
+}
+
+/****************************************************************************
+ * Full decoding
+ */
+
+/* Caller must check for rex.{r,b} extensions before calling this routine */
+static reg_id_t
+reg8_alternative(decode_info_t *di, reg_id_t reg, uint prefixes)
+{
+    if (X64_MODE(di) && reg >= REG_START_x86_8 && reg <= REG_STOP_x86_8 &&
+        TESTANY(PREFIX_REX_ALL, prefixes)) {
+        /* for x64, if any rex prefix exists, we use SPL...SDL instead of
+         * AH..BH (this seems to be the only use of 0x40 == PREFIX_REX_GENERAL)
+         */
+        return (reg - REG_START_x86_8 + REG_START_x64_8);
+    }
+    return reg;
+}
+
+/* which register within modrm we're decoding */
+typedef enum {
+    DECODE_REG_REG,
+    DECODE_REG_BASE,
+    DECODE_REG_INDEX,
+    DECODE_REG_RM,
+} decode_reg_t;
+
+/* Pass in the raw opsize, NOT a size passed through resolve_variable_size(),
+ * to avoid allowing OPSZ_6_irex10_short4 w/ data16
+ */
+static reg_id_t
+decode_reg(decode_reg_t which_reg, decode_info_t *di, byte optype, opnd_size_t opsize)
+{
+    bool extend = false;
+    byte reg = 0;
+    switch (which_reg) {
+    case DECODE_REG_REG:
+        reg = di->reg;   extend = X64_MODE(di) && TEST(PREFIX_REX_R, di->prefixes); break;
+    case DECODE_REG_BASE:
+        reg = di->base;  extend = X64_MODE(di) && TEST(PREFIX_REX_B, di->prefixes); break;
+    case DECODE_REG_INDEX:
+        reg = di->index; extend = X64_MODE(di) && TEST(PREFIX_REX_X, di->prefixes); break;
+    case DECODE_REG_RM:
+        reg = di->rm;    extend = X64_MODE(di) && TEST(PREFIX_REX_B, di->prefixes); break;
+    default:
+        CLIENT_ASSERT(false, "internal unknown reg error");
+    }
+
+    switch (optype) {
+    case TYPE_P:
+    case TYPE_Q:
+    case TYPE_P_MODRM:
+        return (REG_START_MMX + reg); /* no x64 extensions */
+    case TYPE_V:
+    case TYPE_W:
+    case TYPE_V_MODRM:
+        return (extend? (REG_START_XMM + 8 + reg) : (REG_START_XMM + reg));
+    case TYPE_S:
+        if (reg >= 6)
+            return REG_NULL;
+        return (REG_START_SEGMENT + reg);
+    case TYPE_C:
+        return (extend? (REG_START_CR + 8 + reg) : (REG_START_CR + reg));
+    case TYPE_D:
+        return (extend? (REG_START_DR + 8 + reg) : (REG_START_DR + reg));
+    case TYPE_E:
+    case TYPE_G:
+    case TYPE_R:
+    case TYPE_M:
+    case TYPE_INDIR_E:
+    case TYPE_FLOATMEM:
+        /* GPR: fall-through since variable subset of full register */
+        break;
+    default:
+        CLIENT_ASSERT(false, "internal unknown reg error");
+    }
+
+    /* Do not allow a register for 'p' or 'a' types.  FIXME: maybe *_far_ind_* should
+     * use TYPE_INDIR_M instead of TYPE_INDIR_E?  What other things are going to turn
+     * into asserts or crashes instead of invalid instrs based on events as fragile
+     * as these decode routines moving sizes around?
+     */
+    if (opsize != OPSZ_6_irex10_short4 && opsize != OPSZ_8_short4)
+        opsize = resolve_variable_size(di, opsize, true);
+
+    switch (opsize) {
+    case OPSZ_1:
+        if (extend)
+            return (REG_START_8 + 8 + reg);
+        else
+            return reg8_alternative(di, REG_START_8 + reg, di->prefixes);
+    case OPSZ_2:
+        return (extend? (REG_START_16 + 8 + reg) : (REG_START_16 + reg));
+    case OPSZ_4:
+        return (extend? (REG_START_32 + 8 + reg) : (REG_START_32 + reg));
+    case OPSZ_8:
+        return (extend? (REG_START_64 + 8 + reg) : (REG_START_64 + reg));
+    case OPSZ_6:
+    case OPSZ_6_irex10_short4:
+    case OPSZ_8_short4:
+        /* invalid: no register of size p */
+        return REG_NULL;
+    default:
+        /* ok to assert since params controlled by us */
+        CLIENT_ASSERT(false, "decode error: unknown register size");
+        return REG_NULL;
+    }
+}
+
+static bool
+decode_modrm(decode_info_t *di, byte optype, opnd_size_t opsize,
+             opnd_t *reg_opnd, opnd_t *rm_opnd)
+{
+    /* for x64, addr prefix affects only base/index and truncates final addr:
+     * modrm + sib table is the same
+     */
+    bool addr16 = !X64_MODE(di) && TEST(PREFIX_ADDR, di->prefixes);
+
+    if (reg_opnd != NULL) {
+        reg_id_t reg = decode_reg(DECODE_REG_REG, di, optype, opsize);
+        if (reg == REG_NULL)
+            return false;
+        *reg_opnd = opnd_create_reg(reg);
+    }
+
+    if (rm_opnd != NULL) {
+        reg_id_t base_reg = REG_NULL;
+        int disp = 0;
+        reg_id_t index_reg = REG_NULL;
+        int scale = 0;
+        char memtype = TYPE_M;
+        opnd_size_t memsize = resolve_addr_size(di);
+        bool encode_zero_disp, force_full_disp;
+        if (di->has_disp)
+            disp = di->disp;
+        else
+            disp = 0;
+        if (di->has_sib) {
+            CLIENT_ASSERT(!addr16,
+                          "decode error: x86 addr16 cannot have a SIB byte");
+            if (di->index == 4 &&
+                /* rex.x enables r12 as index */
+                (!X64_MODE(di) || !TEST(PREFIX_REX_X, di->prefixes))) {
+                /* no scale/index */
+                index_reg = REG_NULL;
+            } else {
+                index_reg = decode_reg(DECODE_REG_INDEX, di, memtype, memsize);
+                if (index_reg == REG_NULL) {
+                    CLIENT_ASSERT(false, "decode error: internal modrm decode error");
+                    return false;
+                }
+                if (di->scale == 0)
+                    scale = 1;
+                else if (di->scale == 1)
+                    scale = 2;
+                else if (di->scale == 2)
+                    scale = 4;
+                else if (di->scale == 3)
+                    scale = 8;
+            }
+            if (di->base == 5 && di->mod == 0) {
+                /* no base */
+                base_reg = REG_NULL;
+            } else {
+                base_reg = decode_reg(DECODE_REG_BASE, di, memtype, memsize);
+                if (base_reg == REG_NULL) {
+                    CLIENT_ASSERT(false, "decode error: internal modrm decode error");
+                    return false;
+                }
+            }
+        } else {
+            if ((!addr16 && di->mod == 0 && di->rm == 5) ||
+                (addr16 && di->mod == 0 && di->rm == 6)) {
+                /* just absolute displacement, or rip-relative for x64 */
+#ifdef X64
+                if (X64_MODE(di)) {
+                    /* rip-relative: convert from relative offset to absolute target pc */
+                    byte *addr;
+                    CLIENT_ASSERT(di->start_pc != NULL,
+                                  "internal decode error: start pc not set");
+                    if (di->orig_pc != di->start_pc)
+                        addr = di->orig_pc + di->len + di->disp;
+                    else
+                        addr = di->start_pc + di->len + di->disp;
+                    if (TEST(PREFIX_ADDR, di->prefixes)) {
+                        /* Need to clear upper 32 bits.
+                         * Debuggers do not display this truncation, though
+                         * both Intel and AMD manuals describe it.
+                         * I did verify it w/ actual execution.
+                         */
+                        ASSERT_NOT_TESTED();
+                        addr = (byte *) ((ptr_uint_t)addr & 0xffffffff);
+                    }
+                    *rm_opnd = opnd_create_far_rel_addr
+                        (di->seg_override, (void *) addr,
+                         resolve_variable_size(di, opsize, false));
+                    return true;
+                } else
+#endif
+                    base_reg = REG_NULL;
+                index_reg = REG_NULL;
+            } else if (di->mod == 3) {
+                /* register */
+                reg_id_t rm_reg = decode_reg(DECODE_REG_RM, di, optype, opsize);
+                if (rm_reg == REG_NULL) /* no assert since happens, e.g., ff d9 */
+                    return false;
+                else {
+                    *rm_opnd = opnd_create_reg(rm_reg);
+                    return true;
+                }
+            } else {
+                /* non-sib reg-based memory address */
+                if (addr16) {
+                    /* funny order requiring custom decode */
+                    switch (di->rm) {
+                    case 0: base_reg = REG_BX; index_reg = REG_SI; scale = 1; break;
+                    case 1: base_reg = REG_BX; index_reg = REG_DI; scale = 1; break;
+                    case 2: base_reg = REG_BP; index_reg = REG_SI; scale = 1; break;
+                    case 3: base_reg = REG_BP; index_reg = REG_DI; scale = 1; break;
+                    case 4: base_reg = REG_SI; break;
+                    case 5: base_reg = REG_DI; break;
+                    case 6: base_reg = REG_BP;
+                        CLIENT_ASSERT(di->mod != 0,
+                                      "decode error: %bp cannot have mod 0");
+                        break;
+                    case 7: base_reg = REG_BX; break;
+                    default: CLIENT_ASSERT(false, "decode error: unknown modrm rm");
+                        break;
+                    }
+                } else {
+                    /* single base reg */
+                    base_reg = decode_reg(DECODE_REG_RM, di, memtype, memsize);
+                    if (base_reg == REG_NULL) {
+                        CLIENT_ASSERT(false,
+                                      "decode error: internal modrm decode error");
+                        return false;
+                    }
+                }
+            }
+           
+        }
+        /* We go ahead and preserve the force bools if the original really had a 0
+         * disp; up to user to unset bools when changing disp value (FIXME: should
+         * we auto-unset on first mod?)
+         */
+        encode_zero_disp = di->has_disp && disp == 0 &&
+            /* there is no bp base without a disp */
+            (!addr16 || base_reg != REG_BP);
+        force_full_disp = di->has_disp && disp >= INT8_MIN && disp <= INT8_MAX &&
+            di->mod == 2;
+        if (di->seg_override != REG_NULL) {
+            *rm_opnd = opnd_create_far_base_disp_ex
+                (di->seg_override, base_reg, index_reg, scale, disp,
+                 resolve_variable_size(di, opsize, false),
+                 encode_zero_disp, force_full_disp,
+                 TEST(PREFIX_ADDR, di->prefixes));
+        } else {
+            /* Note that OP_{jmp,call}_far_ind does NOT have a far base disp
+             * operand: it is a regular base disp containing 6 bytes that
+             * specify a segment selector and address.  The opcode must be
+             * examined to know how to interpret those 6 bytes.
+             */
+            *rm_opnd = opnd_create_base_disp_ex
+                (base_reg, index_reg, scale, disp,
+                 resolve_variable_size(di, opsize, false),
+                 encode_zero_disp, force_full_disp,
+                 TEST(PREFIX_ADDR, di->prefixes));
+        }
+    }
+    return true;
+}
+
+static ptr_int_t
+get_immed(decode_info_t *di, opnd_size_t opsize)
+{
+    ptr_int_t val = 0;
+    if (di->size_immed == OPSZ_NA) {
+        /* ok b/c only instr_info_t fields passed */
+        CLIENT_ASSERT(di->size_immed2 != OPSZ_NA, "decode immediate size error");
+        val = di->immed2;
+        di->size_immed2 = OPSZ_NA; /* mark as used up */
+    } else {
+        /* ok b/c only instr_info_t fields passed */
+        CLIENT_ASSERT(di->size_immed != OPSZ_NA, "decode immediate size error");
+        val = di->immed;
+        di->size_immed = OPSZ_NA; /* mark as used up */
+    }
+    return val;
+}
+
+/* Also takes in reg8 for TYPE_REG_EX mov_imm */
+reg_id_t
+resolve_var_reg(decode_info_t *di/*IN: x86_mode, prefixes*/,
+                reg_id_t reg32, bool addr, bool can_shrink
+                _IF_X64(bool default_64) _IF_X64(bool can_grow)
+                _IF_X64(bool extendable))
+{
+#ifdef X64
+    if (extendable && X64_MODE(di) && di->prefixes != 0/*optimization*/) {
+        /* Note that Intel's table 3-1 on +r possibilities is incorrect:
+         * it lists rex.r, while Table 2-4 lists rex.b which is correct.
+         */
+        if (TEST(PREFIX_REX_B, di->prefixes))
+            reg32 = reg32 + 8;
+        else
+            reg32 = reg8_alternative(di, reg32, di->prefixes);
+    }
+#endif
+
+    if (addr) {
+#ifdef X64
+        if (X64_MODE(di)) {
+            CLIENT_ASSERT(default_64, "addr-based size must be default 64");
+            if (!can_shrink || !TEST(PREFIX_ADDR, di->prefixes))
+                return reg_32_to_64(reg32);
+            /* else leave 32 (it's addr32 not addr16) */
+        } else
+#endif
+            if (can_shrink && TEST(PREFIX_ADDR, di->prefixes))
+                return reg_32_to_16(reg32);
+    } else {
+#ifdef X64
+        /* rex.w trumps data prefix */
+        if (X64_MODE(di) &&
+            ((can_grow && TEST(PREFIX_REX_W, di->prefixes)) ||
+             (default_64 && (!can_shrink || !TEST(PREFIX_DATA, di->prefixes)))))
+            return reg_32_to_64(reg32);
+        else
+#endif
+            if (can_shrink && TEST(PREFIX_DATA, di->prefixes))
+                return reg_32_to_16(reg32);
+    }
+    return reg32;
+}
+
+static reg_id_t
+ds_seg(decode_info_t *di)
+{
+    if (di->seg_override != REG_NULL) {
+#ifdef X64
+        /* Although the AMD docs say that es,cs,ss,ds prefixes are NOT treated as
+         * segment override prefixes and instead as NULL prefixes, Intel docs do not
+         * say that, and both gdb and windbg disassemble as though the prefixes are
+         * taking effect.  We therefore do not suppress those prefixes.
+         */
+#endif
+        return di->seg_override;
+    }
+    return SEG_DS;
+}
+
+static bool
+decode_operand(decode_info_t *di, byte optype, opnd_size_t opsize, opnd_t *opnd)
+{
+    /* resolving here, for non-reg, makes for simpler code: though the
+     * most common types don't need this.
+     */
+    opnd_size_t ressize = resolve_variable_size(di, opsize, false);
+    switch (optype) {
+    case TYPE_NONE: 
+        *opnd = opnd_create_null();
+        return true;
+    case TYPE_REG:
+        *opnd = opnd_create_reg(opsize);
+        return true;
+    case TYPE_VAR_REG:
+        *opnd = opnd_create_reg(resolve_var_reg
+                                (di, opsize, false/*!addr*/, true/*shrinkable*/
+                                 _IF_X64(false/*d32*/) _IF_X64(true/*growable*/)
+                                 _IF_X64(false/*!extendable*/)));
+        return true;
+    case TYPE_VARZ_REG:
+        *opnd = opnd_create_reg(resolve_var_reg
+                                (di, opsize, false/*!addr*/, true/*shrinkable*/
+                                 _IF_X64(false/*d32*/) _IF_X64(false/*!growable*/)
+                                 _IF_X64(false/*!extendable*/)));
+        return true;
+    case TYPE_VAR_XREG:
+        *opnd = opnd_create_reg(resolve_var_reg
+                                (di, opsize, false/*!addr*/, true/*shrinkable*/
+                                 _IF_X64(true/*d64*/) _IF_X64(false/*!growable*/)
+                                 _IF_X64(false/*!extendable*/)));
+        return true;
+    case TYPE_VAR_ADDR_XREG:
+        *opnd = opnd_create_reg(resolve_var_reg
+                                (di, opsize, true/*addr*/, true/*shrinkable*/
+                                 _IF_X64(true/*d64*/) _IF_X64(false/*!growable*/)
+                                 _IF_X64(false/*!extendable*/)));
+        return true;
+    case TYPE_REG_EX:
+        *opnd = opnd_create_reg(resolve_var_reg
+                                (di, opsize, false/*!addr*/, false/*!shrink*/
+                                 _IF_X64(false/*d32*/) _IF_X64(false/*!growable*/)
+                                 _IF_X64(true/*extendable*/)));
+        return true;
+    case TYPE_VAR_REG_EX:
+        *opnd = opnd_create_reg(resolve_var_reg
+                                (di, opsize, false/*!addr*/, true/*shrinkable*/
+                                 _IF_X64(false/*d32*/) _IF_X64(true/*growable*/)
+                                 _IF_X64(true/*extendable*/)));
+        return true;
+    case TYPE_VAR_XREG_EX:
+        *opnd = opnd_create_reg(resolve_var_reg
+                                (di, opsize, false/*!addr*/, true/*shrinkable*/
+                                 _IF_X64(true/*d64*/) _IF_X64(false/*!growable*/)
+                                 _IF_X64(true/*extendable*/)));
+        return true;
+    case TYPE_VAR_REGX_EX:
+        *opnd = opnd_create_reg(resolve_var_reg
+                                (di, opsize, false/*!addr*/, false/*!shrink*/
+                                 _IF_X64(false/*d64*/) _IF_X64(true/*growable*/)
+                                 _IF_X64(true/*extendable*/)));
+        return true;
+    case TYPE_FLOATMEM:
+    case TYPE_M:
+        /* ensure referencing memory */
+        if (di->mod >= 3)
+            return false;
+        /* fall through */
+    case TYPE_E:
+    case TYPE_Q:
+    case TYPE_W:
+        return decode_modrm(di, optype, opsize, NULL, opnd);
+    case TYPE_R:
+    case TYPE_P_MODRM:
+    case TYPE_V_MODRM:
+        /* ensure referencing a register */
+        if (di->mod != 3)
+            return false;
+        return decode_modrm(di, optype, opsize, NULL, opnd);
+    case TYPE_G:
+    case TYPE_P:
+    case TYPE_V:
+    case TYPE_S:
+    case TYPE_C:
+    case TYPE_D:
+        return decode_modrm(di, optype, opsize, opnd, NULL);
+    case TYPE_I:
+        *opnd = opnd_create_immed_int(get_immed(di, opsize), ressize);
+        return true;
+    case TYPE_1:
+        CLIENT_ASSERT(opsize == OPSZ_0, "internal decode inconsistency");
+        *opnd = opnd_create_immed_int(1, ressize);
+        return true;
+    case TYPE_FLOATCONST:
+        CLIENT_ASSERT(opsize == OPSZ_0, "internal decode inconsistency");
+        /* FIXME: we don't modify any floating point state, but we
+         * could cleanup the binary representation of 0.0f is 0x0
+         */
+        *opnd = opnd_create_immed_float(0.);
+        return true;
+    case TYPE_J:
+        if (di->seg_override == SEG_JCC_NOT_TAKEN ||
+            di->seg_override == SEG_JCC_TAKEN) {
+            /* SEG_DS - taken,     pt */
+            /* SEG_CS - not taken, pn */
+            /* starting from RH9 I see code using this */
+            LOG(THREAD_GET, LOG_EMIT, 5, "disassemble: branch hint %s:\n",
+                di->seg_override == SEG_JCC_TAKEN ? "pt" : "pn");
+            if (di->seg_override == SEG_JCC_NOT_TAKEN)
+                di->prefixes |= PREFIX_JCC_NOT_TAKEN;
+            else
+                di->prefixes |= PREFIX_JCC_TAKEN;
+            di->seg_override = REG_NULL;
+            STATS_INC(num_branch_hints);
+        }
+        /* just ignore other segment prefixes -- don't assert */
+        *opnd = opnd_create_pc((app_pc)get_immed(di, opsize));
+        return true;
+    case TYPE_A: 
+        {
+#ifdef IA32_ON_IA64
+            if (opsize == OPSZ_4_short2) {
+                if (di->seg_override == SEG_CS || di->seg_override == SEG_DS) {
+                    /* branch hint! just delete it? FIXME! */
+                    di->seg_override = REG_NULL;
+                    ASSERT_CURIOSITY(false); /* see if this ever happens */
+                }
+                /* just ignore other segment prefixes -- don't assert */
+                *opnd = opnd_create_pc((app_pc)get_immed(di, opsize));
+                return true;
+            }
+#endif
+            /* ok since instr_info_t fields */
+            CLIENT_ASSERT(!X64_MODE(di), "x64 has no type A instructions");
+            CLIENT_ASSERT(opsize == OPSZ_6_irex10_short4, "decode A operand error");
+            /* just ignore segment prefixes -- don't assert */
+            if (TEST(PREFIX_DATA, di->prefixes)) {
+                /* 4-byte immed */
+                ptr_int_t val = get_immed(di, opsize);
+                *opnd = opnd_create_far_pc
+                    ((ushort) (((ptr_int_t)val & 0xffff0000) >> 16),
+                     (app_pc)((ptr_int_t)val & 0x0000ffff));
+            } else {
+                /* 6-byte immed */
+                /* ok since instr_info_t fields */
+                CLIENT_ASSERT(di->size_immed == OPSZ_6 &&
+                              di->size_immed2 == OPSZ_6,
+                              "decode A operand 6-byte immed error");
+                ASSERT(CHECK_TRUNCATE_TYPE_short(di->immed));
+                *opnd = opnd_create_far_pc((ushort)(short)di->immed, (app_pc) di->immed2);
+                di->size_immed = OPSZ_NA;
+                di->size_immed2 = OPSZ_NA;
+            }
+            return true;
+        }
+    case TYPE_O:
+        {
+            /* no modrm byte, offset follows directly */
+            ptr_int_t immed = get_immed(di, resolve_addr_size(di));
+            *opnd = opnd_create_far_abs_addr(di->seg_override, (void *) immed, ressize);
+            return true;
+        }
+    case TYPE_X:
+        /* this means the memory address DS:(E)SI */
+        if (!X64_MODE(di) && TEST(PREFIX_ADDR, di->prefixes))
+            *opnd = opnd_create_far_base_disp(ds_seg(di), REG_SI, REG_NULL,0,0,ressize);
+        else if (!X64_MODE(di) || TEST(PREFIX_ADDR, di->prefixes))
+            *opnd = opnd_create_far_base_disp(ds_seg(di), REG_ESI, REG_NULL,0,0,ressize);
+        else
+            *opnd = opnd_create_far_base_disp(ds_seg(di), REG_RSI, REG_NULL,0,0,ressize);
+        return true;
+    case TYPE_Y:
+        /* this means the memory address ES:(E)DI */
+        if (!X64_MODE(di) && TEST(PREFIX_ADDR, di->prefixes))
+            *opnd = opnd_create_far_base_disp(SEG_ES, REG_DI, REG_NULL, 0, 0, ressize);
+        else if (!X64_MODE(di) || TEST(PREFIX_ADDR, di->prefixes))
+            *opnd = opnd_create_far_base_disp(SEG_ES, REG_EDI, REG_NULL, 0, 0, ressize);
+        else
+            *opnd = opnd_create_far_base_disp(SEG_ES, REG_RDI, REG_NULL, 0, 0, ressize);
+        return true;
+    case TYPE_XLAT:
+        /* this means the memory address DS:(E)BX+AL */
+        if (!X64_MODE(di) && TEST(PREFIX_ADDR, di->prefixes))
+            *opnd = opnd_create_far_base_disp(ds_seg(di), REG_BX, REG_AL,1,0,ressize);
+        else if (!X64_MODE(di) || TEST(PREFIX_ADDR, di->prefixes))
+            *opnd = opnd_create_far_base_disp(ds_seg(di), REG_EBX, REG_AL,1,0,ressize);
+        else
+            *opnd = opnd_create_far_base_disp(ds_seg(di), REG_RBX, REG_AL,1,0,ressize);
+        return true;
+    case TYPE_MASKMOVQ:
+        /* this means the memory address DS:(E)DI */
+        if (!X64_MODE(di) && TEST(PREFIX_ADDR, di->prefixes))
+            *opnd = opnd_create_far_base_disp(ds_seg(di), REG_DI, REG_NULL,0,0,ressize);
+        else if (!X64_MODE(di) || TEST(PREFIX_ADDR, di->prefixes))
+            *opnd = opnd_create_far_base_disp(ds_seg(di), REG_EDI, REG_NULL,0,0,ressize);
+        else
+            *opnd = opnd_create_far_base_disp(ds_seg(di), REG_RDI, REG_NULL,0,0,ressize);
+        return true;
+    case TYPE_INDIR_REG:
+        /* FIXME: how know data size?  for now just use reg size: our only use
+         * of this does not have a varying hardcoded reg, fortunately. */
+        *opnd = opnd_create_base_disp(opsize, REG_NULL, 0, 0, reg_get_size(opsize));
+        return true;
+    case TYPE_INDIR_VAR_XREG: /* indirect reg varies by addr16 not data16, base is 4x8,
+                               * opsize varies by data16 */
+    case TYPE_INDIR_VAR_REG: /* indirect reg varies by addr16 not data16, base is 4x8,
+                              * opsize varies by rex and data16 */
+    case TYPE_INDIR_VAR_XIREG: /* indirect reg varies by addr16 not data16, base is 4x8,
+                                * opsize varies by data16 except on 64-bit Intel */
+        {
+            reg_id_t reg =
+                resolve_var_reg(di, opsize, true/*addr*/, true/*shrinkable*/
+                                _IF_X64(true/*d64*/) _IF_X64(false/*!growable*/)
+                                _IF_X64(false/*!extendable*/));
+            /* FIXME - xref case 10541/PR 214976, this type is overused and could
+             * have different sizes. Right now TYPE_INDIR_VAR_XREG is used for
+             * push/pop/pushf/popf/call (for which
+             * OPSZ_VARSTACK == OPSZ_4x8_short2 is right), and
+             * TYPE_INDIR_VAR_XIREG is used for ret/ret_imm, which are
+             * correct.  However, TYPE_INDIR_VAR_XREG is also used
+             * for pusha/popa/call_far/enter/leave/ret_far/int which have
+             * different sizes, may access memory down instead of up from the address
+             * or at an offset to the address.  Currently iret, far ret, and
+             * far call use TPYE_INDIR_VAR_REG, and they do multiple pushes/pops
+             * as well (iret does 3 (or 5) pops of this size
+             * depending on the operand size (3 for 16 or 32-bit, 5 for 64-bit)).
+             * NOTE - needs to match size in opnd_type_ok() and instr_create.h */
+            *opnd = opnd_create_base_disp (reg, REG_NULL, 0, 0,
+                                           resolve_variable_size
+                                           (di, indir_var_reg_size(optype),
+                                            false/*not reg*/));
+            return true;
+        }
+    case TYPE_INDIR_E:
+        /* FIXME: how mark as indirect? 
+         * in current usage decode_modrm will be treated as indirect, becoming
+         * a base_disp operand, vs. an immed, which becomes a pc operand
+         * besides, Ap is just as indirect as i_Ep!
+         */
+        return decode_operand(di, TYPE_E, opsize, opnd);
+    default:
+        /* ok to assert, types coming only from instr_info_t */
+        CLIENT_ASSERT(false, "decode error: unknown operand type");
+    }
+    return false;
+}
+
+/****************************************************************************
+ * Exported routines
+ */
+
+/* Decodes only enough of the instruction at address pc to determine
+ * its eflags usage, which is returned in usage as EFLAGS_ constants
+ * or'ed together.
+ * This corresponds to halfway between Level 1 and Level 2: a Level 1 decoding
+ * plus eflags information (usually only at Level 2).
+ * Returns the address of the next byte after the decoded instruction.
+ * Returns NULL on decoding an invalid instruction.
+ *
+ * N.B.: an instruction that has an "undefined" effect on eflags is considered
+ * to write to eflags.  This is fine since programs shouldn't be reading
+ * eflags after an undefined modification to them, but a weird program that
+ * relies on some undefined eflag thing might behave differently under dynamo
+ * than not!
+ */
+byte *
+decode_eflags_usage(dcontext_t *dcontext, byte *pc, uint *usage)
+{
+    const instr_info_t *info;
+    decode_info_t di;
+    IF_X64(di.x86_mode = get_x86_mode(dcontext));
+
+    /* don't decode immeds, instead use decode_next_pc, it's faster */
+    read_instruction(pc, pc, &info, &di, true /* just opcode */ _IF_DEBUG(true));
+    *usage = info->eflags;
+    pc = decode_next_pc(dcontext, pc);
+    /* failure handled fine -- we'll go ahead and return the NULL */
+
+    return pc;
+}
+
+/* Decodes the opcode and eflags usage of instruction at address pc
+ * into instr.
+ * This corresponds to a Level 2 decoding.
+ * Assumes that instr is already initialized, but uses the x86/x64 mode
+ * for the current thread rather than that set in instr.
+ * If caller is re-using same instr struct over multiple decodings,
+ * should call instr_reset or instr_reuse.
+ * Returns the address of the next byte after the decoded instruction.
+ * Returns NULL on decoding an invalid instruction.
+ */
+byte *
+decode_opcode(dcontext_t *dcontext, byte *pc, instr_t *instr)
+{
+    const instr_info_t *info;
+    decode_info_t di;
+    int sz;
+#ifdef X64
+    /* PR 251479: we need to know about all rip-relative addresses.
+     * Since change/setting raw bits invalidates, we must set this
+     * on every return. */
+    uint rip_rel_pos;
+#endif
+    IF_X64(di.x86_mode = instr_get_x86_mode(instr));
+    /* when pass true to read_instruction it doesn't decode immeds,
+     * so have to call decode_next_pc, but that ends up being faster
+     * than decoding immeds!
+     */
+    read_instruction(pc, pc, &info, &di, true /* just opcode */
+                     _IF_DEBUG(!TEST(INSTR_IGNORE_INVALID, instr->flags)));
+    sz = decode_sizeof(dcontext, pc, NULL _IF_X64(&rip_rel_pos));
+    IF_X64(instr_set_x86_mode(instr, get_x86_mode(dcontext)));
+    instr_set_opcode(instr, info->type);
+    /* read_instruction sets opcode to OP_INVALID for illegal instr.
+     * decode_sizeof will return 0 for _some_ illegal instrs, so we
+     * check it first since it's faster than instr_valid, but we have to
+     * also check instr_valid to catch all illegal instrs.
+     */
+    if (sz == 0 || !instr_valid(instr)) {
+        CLIENT_ASSERT(!instr_valid(instr), "decode_opcode: invalid instr");
+        return NULL;
+    }
+    instr->eflags = info->eflags;
+    instr_set_eflags_valid(instr, true);
+    /* operands are NOT set */
+    instr_set_operands_valid(instr, false);
+    /* raw bits are valid though and crucial for encoding */
+    instr_set_raw_bits(instr, pc, sz);
+    /* must set rip_rel_pos after setting raw bits */
+    IF_X64(instr_set_rip_rel_pos(instr, rip_rel_pos));
+    return pc + sz;
+}
+
+#ifdef DEBUG
+/* PR 215143: we must resolve variable sizes at decode time */
+static bool
+check_is_variable_size(opnd_t op)
+{
+    if (opnd_is_memory_reference(op) ||
+        /* reg_get_size() fails on fp registers since no OPSZ for them */
+        (opnd_is_reg(op) && !reg_is_fp(opnd_get_reg(op))))
+        return !is_variable_size(opnd_get_size(op));
+    /* else no legitimate size to check */
+    return true;
+}
+#endif
+
+/* Decodes the instruction at address pc into instr, filling in the
+ * instruction's opcode, eflags usage, prefixes, and operands.
+ * This corresponds to a Level 3 decoding.
+ * Assumes that instr is already initialized, but uses the x86/x64 mode
+ * for the current thread rather than that set in instr.
+ * If caller is re-using same instr struct over multiple decodings,
+ * should call instr_reset or instr_reuse.
+ * Returns the address of the next byte after the decoded instruction.
+ * Returns NULL on decoding an invalid instruction.
+ */
+static byte *
+decode_common(dcontext_t *dcontext, byte *pc, byte *orig_pc, instr_t *instr)
+{
+    const instr_info_t *info;
+    decode_info_t di;
+    byte *next_pc;
+    int instr_num_dsts = 0, instr_num_srcs = 0;
+    opnd_t dsts[8];
+    opnd_t srcs[8];
+
+    CLIENT_ASSERT(instr->opcode == OP_INVALID || instr->opcode == OP_UNDECODED,
+                  "decode: instr is already decoded, may need to call instr_reset()");
+
+    IF_X64(di.x86_mode = get_x86_mode(dcontext));
+    next_pc = read_instruction(pc, orig_pc, &info, &di, false /* not just opcode,
+                                                                 decode operands too */
+                               _IF_DEBUG(!TEST(INSTR_IGNORE_INVALID, instr->flags)));
+    instr_set_opcode(instr, info->type);
+    IF_X64(instr_set_x86_mode(instr, di.x86_mode));
+    /* failure up to this point handled fine -- we set opcode to OP_INVALID */
+    if (next_pc == NULL) {
+        LOG(THREAD, LOG_INTERP, 3, "decode: invalid instr at "PFX"\n", pc);
+        CLIENT_ASSERT(!instr_valid(instr), "decode: invalid instr");
+        return NULL;
+    }
+    instr->eflags = info->eflags;
+    instr_set_eflags_valid(instr, true);
+    /* since we don't use set_src/set_dst we must explicitly say they're valid */
+    instr_set_operands_valid(instr, true);
+    /* read_instruction doesn't set di.len since only needed for rip-relative opnds */
+    IF_X64(CLIENT_ASSERT_TRUNCATE(di.len, int, next_pc - pc,
+                                  "internal truncation error"));
+    di.len = (int) (next_pc - pc);
+
+    instr->prefixes |= di.prefixes;
+
+    /* operands */
+    do {
+        if (info->dst1_type != TYPE_NONE) {
+            if (!decode_operand(&di, info->dst1_type, info->dst1_size,
+                                &(dsts[instr_num_dsts++])))
+                goto decode_invalid;
+            ASSERT(check_is_variable_size(dsts[instr_num_dsts-1]));
+        }
+        if (info->dst2_type != TYPE_NONE) {
+            if (!decode_operand(&di, info->dst2_type, info->dst2_size,
+                                &(dsts[instr_num_dsts++])))
+                goto decode_invalid;
+            ASSERT(check_is_variable_size(dsts[instr_num_dsts-1]));
+        }
+        if (info->src1_type != TYPE_NONE) {
+            if (!decode_operand(&di, info->src1_type, info->src1_size,
+                                &(srcs[instr_num_srcs++])))
+                goto decode_invalid;
+            ASSERT(check_is_variable_size(srcs[instr_num_srcs-1]));
+        }
+        if (info->src2_type != TYPE_NONE) {
+            if (!decode_operand(&di, info->src2_type, info->src2_size,
+                                &(srcs[instr_num_srcs++])))
+                goto decode_invalid;
+            ASSERT(check_is_variable_size(srcs[instr_num_srcs-1]));
+        }
+        if (info->src3_type != TYPE_NONE) {
+            if (!decode_operand(&di, info->src3_type, info->src3_size,
+                                &(srcs[instr_num_srcs++])))
+                goto decode_invalid;
+            ASSERT(check_is_variable_size(srcs[instr_num_srcs-1]));
+        }
+        /* extra operands:
+         * we take advantage of the fact that all instructions that need extra
+         * operands have only one encoding, so the code field points to instr_info_t
+         * structures containing the extra operands
+         */
+        if ((info->flags & HAS_EXTRA_OPERANDS) != 0) {
+            if ((info->flags & EXTRAS_IN_CODE_FIELD) != 0)
+                info = (const instr_info_t *)(info->code);
+            else /* extra operands are in next entry */
+                info = info + 1;
+        } else
+            break;
+    } while (true);
+
+    /* some operands add to di.prefixes so we copy again */
+    instr->prefixes |= di.prefixes;
+    if (di.seg_override == SEG_FS)
+        instr->prefixes |= PREFIX_SEG_FS;
+    if (di.seg_override == SEG_GS)
+        instr->prefixes |= PREFIX_SEG_GS;
+
+    /* now copy operands into their real slots */
+    instr_set_num_opnds(dcontext, instr, instr_num_dsts, instr_num_srcs);
+    if (instr_num_dsts > 0) {
+        memcpy(instr->dsts, dsts, instr_num_dsts*sizeof(opnd_t));
+    }
+    if (instr_num_srcs > 0) {
+        /* remember that src0 is static */
+        instr->src0 = srcs[0];
+        if (instr_num_srcs > 1) {
+            memcpy(instr->srcs, &(srcs[1]), (instr_num_srcs-1)*sizeof(opnd_t));
+        }
+    }
+
+    /* check for invalid prefixes that depend on operand types */
+    if (TEST(PREFIX_LOCK, di.prefixes)) {
+        /* check for invalid opcode, list on p3-397 of IA-32 vol 2 */
+        switch (instr_get_opcode(instr)) {
+        case OP_add: case OP_adc: case OP_and: case OP_btc: case OP_btr: case OP_bts:
+        case OP_cmpxchg: case OP_cmpxchg8b: case OP_dec: case OP_inc: case OP_neg:
+        case OP_not: case OP_or: case OP_sbb: case OP_sub: case OP_xor: case OP_xadd:
+        case OP_xchg: {
+            /* still illegal unless dest is mem op rather than src */
+            CLIENT_ASSERT(instr->num_dsts > 0, "internal lock prefix check error");
+            if (!opnd_is_memory_reference(instr->dsts[0])) {
+                LOG(THREAD, LOG_INTERP, 3, "decode: invalid lock prefix at "PFX"\n", pc);
+                goto decode_invalid;
+            }
+            break;
+        }
+        default: {
+            LOG(THREAD, LOG_INTERP, 3, "decode: invalid lock prefix at "PFX"\n", pc);
+            goto decode_invalid;
+        }
+        }
+    }
+
+    if (orig_pc != pc) {
+        /* We do not want to copy when encoding and condone an invalid
+         * relative target
+         */
+        instr_set_raw_bits_valid(instr, false);
+        instr_set_translation(instr, orig_pc);
+    } else {
+        /* we set raw bits AFTER setting all srcs and dsts b/c setting
+         * a src or dst marks instr as having invalid raw bits
+         */
+        IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(next_pc - pc)));
+        instr_set_raw_bits(instr, pc, (uint)(next_pc - pc));
+#ifdef X64
+        if (X64_MODE(&di) && TEST(HAS_MODRM, info->flags) && di.mod == 0 && di.rm == 5) {
+            CLIENT_ASSERT(di.disp_abs > di.start_pc, "decode: internal rip-rel error");
+            CLIENT_ASSERT(CHECK_TRUNCATE_TYPE_int(di.disp_abs - di.start_pc),
+                          "decode: internal rip-rel error");
+            /* must do this AFTER setting raw bits to avoid being invalidated */
+            instr_set_rip_rel_pos(instr, (int) (di.disp_abs - di.start_pc));
+        }
+#endif
+    }
+
+    return next_pc;
+
+ decode_invalid:
+    instr_set_operands_valid(instr, false);
+    instr_set_opcode(instr, OP_INVALID);
+    return NULL;
+}
+
+byte *
+decode(dcontext_t *dcontext, byte *pc, instr_t *instr)
+{
+    return decode_common(dcontext, pc, pc, instr);
+}
+
+byte *
+decode_from_copy(dcontext_t *dcontext, byte *copy_pc, byte *orig_pc, instr_t *instr)
+{
+    return decode_common(dcontext, copy_pc, orig_pc, instr);
+}
+
+const instr_info_t *
+get_next_instr_info(const instr_info_t * info)
+{
+    return (const instr_info_t *)(info->code);
+}
+
+byte 
+decode_first_opcode_byte(int opcode)
+{
+    const instr_info_t * info = op_instr[opcode];
+    return (byte)((info->opcode & 0x00ff0000) >> 16);
+}
+
+#ifdef DECODE_UNIT_TEST
+# include "instr_create.h"
+
+/* FIXME: Tried putting this inside a separate unit-decode.c file, but
+ *        required creating a unit-decode_table.c file.  Since the
+ *        infrastructure is not fully set up, currently leaving this here
+ * FIXME: beef up to check if something went wrong
+ */
+static bool
+unit_check_decode_ff_opcode() {
+    static int do_once = 0;
+    instr_t instr;
+    byte modrm, sib;
+    byte raw_bytes[] = { 0xff, 0x0, 0x0, 
+                         0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
+                         0xab, 0xbc, 0xcd, 0xde, 0xef, 0xfa,
+                       };
+    app_pc next_pc = NULL;
+
+    for (modrm = 0x0; modrm < 0xff; modrm++) {
+        raw_bytes[1] = modrm;
+        for(sib = 0x0; sib < 0xff; sib++) {
+            raw_bytes[2] = sib;
+
+            /* set up instr for decode_opcode */
+            instr_init(GLOBAL_DCONTEXT, &instr);
+            instr.bytes = raw_bytes;
+            instr.length = 15;
+            instr_set_raw_bits_valid(&instr, true);
+            instr_set_operands_valid(&instr, false);
+
+            next_pc = decode_opcode(GLOBAL_DCONTEXT, instr.bytes, &instr);
+            if (next_pc != NULL &&
+                instr.opcode != OP_INVALID && instr.opcode != OP_UNDECODED) {
+                print_file(STDERR, "## %02x %02x %02x len=%d\n",
+                    instr.bytes[0], instr.bytes[1], instr.bytes[2], instr.length);
+            }
+        }
+    }
+    return 0;
+}
+
+/* Standalone building is still broken so I tested this by calling
+ * from a real DR build.
+ */
+#define CHECK_ENCODE_OPCODE(dcontext, instr, pc, opc, ...) \
+    instr = INSTR_CREATE_##opc(dcontext, ## __VA_ARGS__);  \
+    instr_encode(dcontext, instr, pc);                 \
+    instr_reset(dcontext, instr);                      \
+    decode(dcontext, pc, instr);                       \
+    /* FIXME: use EXPECT */                            \
+    CLIENT_ASSERT(instr_get_opcode(instr) == OP_##opc, "unit test"); \
+    instr_destroy(dcontext, instr);
+
+/* FIXME: case 8212: add checks for every single instr type */
+static bool
+unit_check_sse3()
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    byte buf[32];
+    instr_t *instr;
+    CHECK_ENCODE_OPCODE(dcontext, instr, buf, mwait);
+    CHECK_ENCODE_OPCODE(dcontext, instr, buf, monitor);
+    CHECK_ENCODE_OPCODE(dcontext, instr, buf, haddpd,
+                        opnd_create_reg(REG_XMM7), opnd_create_reg(REG_XMM2));
+    CHECK_ENCODE_OPCODE(dcontext, instr, buf, haddps,
+                        opnd_create_reg(REG_XMM7), opnd_create_reg(REG_XMM2));
+    CHECK_ENCODE_OPCODE(dcontext, instr, buf, hsubpd,
+                        opnd_create_reg(REG_XMM7), opnd_create_reg(REG_XMM2));
+    CHECK_ENCODE_OPCODE(dcontext, instr, buf, hsubps,
+                        opnd_create_reg(REG_XMM7), opnd_create_reg(REG_XMM2));
+    CHECK_ENCODE_OPCODE(dcontext, instr, buf, addsubpd,
+                        opnd_create_reg(REG_XMM7), opnd_create_reg(REG_XMM2));
+    CHECK_ENCODE_OPCODE(dcontext, instr, buf, addsubps,
+                        opnd_create_reg(REG_XMM7), opnd_create_reg(REG_XMM2));
+    CHECK_ENCODE_OPCODE(dcontext, instr, buf, lddqu,
+                        opnd_create_reg(REG_XMM7),
+                        opnd_create_base_disp(REG_NULL, REG_NULL, 0, 0, OPSZ_16));
+    CHECK_ENCODE_OPCODE(dcontext, instr, buf, movsldup,
+                        opnd_create_reg(REG_XMM7), opnd_create_reg(REG_XMM2));
+    CHECK_ENCODE_OPCODE(dcontext, instr, buf, movshdup,
+                        opnd_create_reg(REG_XMM7), opnd_create_reg(REG_XMM2));
+    CHECK_ENCODE_OPCODE(dcontext, instr, buf, movddup,
+                        opnd_create_reg(REG_XMM7), opnd_create_reg(REG_XMM2));
+    /* not sse3 but I fixed it at same time so here to test */
+    CHECK_ENCODE_OPCODE(dcontext, instr, buf, cmpxchg8b,
+                        opnd_create_base_disp(REG_NULL, REG_NULL, 0, 0, OPSZ_8));
+    return true;
+}
+
+int main()
+{
+    bool res;
+    standalone_init();
+    res = unit_check_sse3();
+    res = unit_check_decode_ff_opcode() && res;
+    return res;
+}
+
+#endif /* DECODE_UNIT_TEST */
diff --git a/core/x86/decode.h b/core/x86/decode.h
new file mode 100644
index 000000000..e4ea59798
--- /dev/null
+++ b/core/x86/decode.h
@@ -0,0 +1,714 @@
+/* **********************************************************
+ * Copyright (c) 2000-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/* file "decode.h" */
+
+#ifndef DECODE_H
+#define DECODE_H
+
+
+/* Prefix constants.  1st 5 are used in bitfields, rest are not.
+ * We assume that the PREFIX_ constants are invalid as pointers for
+ * our use in instr_info_t.code.
+ */
+/* DR_API EXPORT TOFILE dr_ir_instr.h */
+/* DR_API EXPORT BEGIN */
+
+/****************************************************************************
+ * instr_t prefixes
+ *
+ * Note that prefixes that change the data or address size, or that
+ * specify a different base segment, are not specified on a
+ * whole-instruction level, but rather on individual operands (of
+ * course with multiple operands they must all match).
+ * The rep and repne prefixes are encoded directly into the opcodes.
+ * 
+ */
+#define PREFIX_LOCK           0x1 /**< Makes the instruction's memory accesses atomic. */
+#define PREFIX_JCC_NOT_TAKEN  0x2 /**< Branch hint: conditional branch is taken. */
+#define PREFIX_JCC_TAKEN      0x4 /**< Branch hint: conditional branch is not taken. */
+
+/* DR_API EXPORT END */
+
+/* These are used only in the decoding tables.  We decode the
+ * information into the operands.
+ * For encoding these properties are specified in the operands,
+ * with our encoder auto-adding the appropriate prefixes.
+ */
+#define PREFIX_DATA           0x0008
+#define PREFIX_ADDR           0x0010
+#define PREFIX_REX_W          0x0020
+#define PREFIX_REX_R          0x0040
+#define PREFIX_REX_X          0x0080
+#define PREFIX_REX_B          0x0100
+#define PREFIX_REX_GENERAL    0x0200 /* 0x40: only matters for SPL...SDL vs AH..BH */
+#define PREFIX_REX_ALL        (PREFIX_REX_W|PREFIX_REX_R|PREFIX_REX_X|PREFIX_REX_B|\
+                               PREFIX_REX_GENERAL)
+#define PREFIX_SIZE_SPECIFIERS (PREFIX_DATA|PREFIX_ADDR|PREFIX_REX_ALL)
+
+/* Unused except in decode tables (we encode the prefix into the opcodes) */
+#define PREFIX_REP            0x0400
+#define PREFIX_REPNE          0x0800
+
+/* PREFIX_SEG_* is set by decode or decode_cti and is only a hint
+ * to the caller.  Is ignored by encode in favor of the segment 
+ * reg specified in the applicable opnds.  We rely on it being set during 
+ * bb building.
+ */
+#define PREFIX_SEG_FS         0x1000
+#define PREFIX_SEG_GS         0x2000
+
+/* We encode some prefixes in the operands themselves, such that we shouldn't
+ * consider the whole-instr_t flags when considering equality of Instrs
+ */
+#define PREFIX_SIGNIFICANT (PREFIX_LOCK|PREFIX_JCC_TAKEN|PREFIX_JCC_TAKEN)
+
+/* branch hints show up as segment modifiers */
+#define SEG_JCC_NOT_TAKEN     SEG_CS
+#define SEG_JCC_TAKEN         SEG_DS
+
+/*
+ o get rid of sI?  I sign-extend all the Ib's on decoding, many are
+   actually sign-extended that aren't marked sIb
+ o had to add size to immed_int and base_disp, don't have it for indir_reg!
+   that's fine for encoding, aren't duplicate instrs differing on size of
+   indir reg data, but should set it properly for later IR passes...
+ o rewrite emit.c so does emit_inst only once (instr_length() does it!)
+*/
+
+/* bits used to encode info in instr_info_t's opcode field */
+enum {
+    OPCODE_TWOBYTES = 0x00000010,
+    OPCODE_REG      = 0x00000020,
+    OPCODE_MODRM    = 0x00000040,
+    OPCODE_SUFFIX   = 0x00000080,
+    OPCODE_THREEBYTES=0x00000008,
+};
+
+/* instr_info_t: each table entry is one of these
+ * For reading all bytes of instruction, only need to know:
+ * 1) prefixes + opcode boundary
+ * 2) whether to read modrm byte, from modrm get sib and disp
+ * 3) whether to read immed bytes (types A, I, sI, J, and O)
+ * The rest of the types are for interpretation only
+ *
+ * We have room for 2 destinations and 3 sources
+ *   Instrs with 2 dests include push, pop, edx:eax, xchg, rep lods
+ * Exceptions:
+ *   pusha = 8 sources
+ *   popa = 8 destinations
+ *   enter = 3 dests, 4 srcs (incl. 2 immeds)
+ *   cpuid = 4 dests
+ *   cmpxchg8b = 3 dests, 5 srcs
+ *   movs = 3 dests, rest of string instrs w/ rep/repne have 3+ dests
+ * Represent exceptions as sequence of instrs?
+ *   All 5 exceptions have only one form, can use list field to
+ *   point into separate table that holds types of extra operands.
+ *
+ * Separate immed field and only 2 srcs?
+ * enter has 2 immeds, all other instrs have <=1
+ * there are a number of instrs that need 3 srcs even ignoring immeds!
+ *   (pusha,shld,shrd,cmpxchg,cmpxhchg8b,rep outs,rep cmps, rep scas)
+ * => no separate immed field
+ *
+ * FIXMEs:
+ * lea = computes addr, doesn't touch mem!  how encode?
+ * in & out, ins & outs: are I/O ports in memory?!?
+ * Should we model fp stack changes?!?
+ * All i_eSP really read 0x4(esp) or some other offset, depending on
+ *   if take esp value before or after instrs, etc.
+ * punpck*, pshuf*: say "reads entire x-bits, only uses half of them"
+ *   table is inconsistent in this: for punpck* it gives a memory size of x/2
+ *   (b/c Intel's table gives that), for pshuf* (there are others too I think)
+ *   gives size x (not present in Intel's table)
+ */
+typedef struct instr_info_t {
+    int type; /* an OP_ constant or special type code below */
+    /* opcode: split into bytes
+     * 0th (ms) = prefix byte, if byte 3's 1st nibble's bit 3 and bit 4 are both NOT set;
+     *            modrm byte, if  byte 3's 1st nibble's bit 3 IS set.
+     *            suffix byte, if  byte 3's 1st nibble's bit 4 IS set.
+     * 1st = 1st byte of opcode
+     * 2nd = 2nd byte of opcode (if there are 2)
+     * 3rd (ls) = split into nibbles
+     *   1st nibble (ms) = if bit 1 (OPCODE_TWOBYTES) set, opcode has 2 bytes
+     *                     if bit 2 (OPCODE_REG) set, opcode has /n
+     *                     if bit 3 (OPCODE_MODRM) set, opcode based on entire modrm
+     *                       that modrm is stored as the byte 0
+     *                     if bit 4 (OPCODE_SUFFIX) set, opcode based on suffix byte
+     *                       that byte is stored as the byte 0
+     *                     FIXME: so we do not support an instr that has an opcode
+     *                     dependent on both a prefix and the entire modrm or suffix!
+     *   2nd nibble (ls) = bits 1-3 hold /n for OPCODE_REG
+     *                     if bit 4 (OPCODE_THREEBYTES) is set, the opcode has
+     *                       3 bytes, with the first being an implied 0x0f (so
+     *                       the 2nd byte is stored as "1st" and 3rd as "2nd").
+     */
+    uint opcode;
+    const char *name;
+    /* operands */
+    byte dst1_type;  opnd_size_t dst1_size;
+    byte dst2_type;  opnd_size_t dst2_size;
+    byte src1_type;  opnd_size_t src1_size;
+    byte src2_type;  opnd_size_t src2_size;
+    byte src3_type;  opnd_size_t src3_size;
+    byte flags; /* modrm and extra operand flags */
+    uint eflags; /* combination of read & write flags from instr.h */
+    ptr_int_t code; /* for PREFIX: one of the PREFIX_ constants, or SEG_ constant
+                     * for EXTENSION and *_EXT: index into extensions table
+                     * for OP_: pointer to next entry of that opcode
+                     *   may also point to extra operand table
+                     */
+} instr_info_t;
+
+
+/* classification of instruction bytes up to modrm/disp/immed
+ * these constants are used for instr_info_t.type field
+ */
+enum {
+    /* not a valid opcode */
+    INVALID = OP_LAST + 1,
+    /* prefix byte */
+    PREFIX,
+    /* 0x0f = two-byte escape code */
+    ESCAPE,
+    /* floating point instruction escape code */
+    FLOAT_EXT,
+    /* opcode extension via reg field of modrm */
+    EXTENSION,
+    /* 2-byte instructions differing by presence of 0xf3/0x66/0xf2 prefixes */
+    PREFIX_EXT,
+    /* (rep prefix +) 1-byte-opcode string instruction */
+    REP_EXT,
+    /* (repne prefix +) 1-byte-opcode string instruction */
+    REPNE_EXT,
+    /* 2-byte instructions differing by mod bits of modrm */
+    MOD_EXT,
+    /* 2-byte instructions differing by rm bits of modrm */
+    RM_EXT,
+    /* 2-byte instructions whose opcode also depends on a suffix byte */
+    SUFFIX_EXT,
+    /* instructions that vary based on whether in 64-bit mode or not */
+    X64_EXT,
+    /* 3-byte opcodes beginning 0x0f 0x38 (SSSE3 and SSE4) */
+    ESCAPE_3BYTE_38,
+    /* 3-byte opcodes beginning 0x0f 0x3a (SSE4) */
+    ESCAPE_3BYTE_3a,
+    /* instructions differing if a rex prefix is present */
+    REX_EXT,
+    /* else, from OP_ enum */
+};
+
+/* modrm/extra operands flags == single byte only! */
+#define HAS_MODRM             0x01   /* else, no modrm */
+#define HAS_EXTRA_OPERANDS    0x02   /* else, <= 2 dsts, <= 3 srcs */
+/* if HAS_EXTRA_OPERANDS: */
+#define EXTRAS_IN_CODE_FIELD  0x04   /* next instr_info_t pointed to by code field */
+/* rather than split out into little tables of 32-bit vs OP_INVALID, we use a
+ * flag to indicate opcodes that are invalid in particular modes:
+ */
+#define X86_INVALID           0x08
+#define X64_INVALID           0x10
+/* to avoid needing a single-valid-entry subtable in prefix_extensions */
+#define REQUIRES_PREFIX       0x20
+
+/* instr_info_t is used for table entries, it holds info that is constant
+ * for all instances of an instruction.
+ * All variable information is kept in this struct, which is used for
+ * decoding and encoding.
+ */
+typedef struct decode_info_t {
+    /* Holds address and data size prefixes, as well as the prefixes
+     * that are shared as-is with instr_t (PREFIX_SIGNIFICANT).
+     * We assume we're in the default mode (32-bit or 64-bit,
+     * depending on our build) and that the address and data size
+     * prefixes can be treated as absolute.
+     */
+    uint prefixes;
+    byte seg_override; /* REG enum of seg, REG_NULL if none */
+    /* modrm info */
+    byte modrm;
+    byte mod;
+    byte reg;
+    byte rm;
+    bool has_sib;
+    byte scale;
+    byte index;
+    byte base;
+    bool has_disp;
+    int disp;
+    /* immed info */
+    opnd_size_t size_immed;
+    opnd_size_t size_immed2;
+    ptr_int_t immed;
+    ptr_int_t immed2; /* this additional field could be 32-bit on all platforms */
+    /* These two fields are only used when decoding rip-relative data refs */
+    byte *start_pc;
+    uint len;
+    /* This field is only used when encoding rip-relative data refs.
+     * To save space we could make it a union with disp.
+     */
+    byte *disp_abs;
+#ifdef X64
+    /* Since the mode when an instr_t is involved is per-instr rather than
+     * per-dcontext we have our own field here instead of passing dcontext around.
+     * It's up to the caller to set this field to match either the instr_t
+     * or the dcontext_t field.
+     */
+    bool x86_mode;
+#endif
+    /* PR 302353: support decoding as though somewhere else */
+    byte *orig_pc;
+} decode_info_t;
+
+
+/* N.B.: if you change the type or size enums, change the string names for
+ * them, kept in encode.c
+ */
+
+/* operand types have 2 parts, type and size */
+enum {
+    /* operand types */
+    TYPE_NONE,
+    TYPE_A, /* immediate that is absolute address */
+    TYPE_C, /* reg of modrm selects control reg */
+    TYPE_D, /* reg of modrm selects debug reg */
+    TYPE_E, /* modrm selects reg or mem addr */
+    /* I don't use type F, I have eflags info in separate field */
+    TYPE_G, /* reg of modrm selects register */
+    TYPE_I, /* immediate */
+    TYPE_J, /* immediate that is relative offset of EIP */
+    TYPE_M, /* modrm select mem addr */
+    TYPE_O, /* immediate that is memory offset */
+    TYPE_P, /* reg of modrm selects MMX */
+    TYPE_Q, /* modrm selects MMX or mem addr */
+    TYPE_R, /* modrm selects register */
+    TYPE_S, /* reg of modrm selects segment register */
+    TYPE_V, /* reg of modrm selects XMM */
+    TYPE_W, /* modrm selects XMM or mem addr */
+    TYPE_X, /* DS:(RE)(E)SI */
+    TYPE_Y, /* ES:(RE)(E)SDI */
+    TYPE_P_MODRM,  /* == Intel 'N': modrm selects MMX */
+    TYPE_V_MODRM,  /* == Intel 'U': modrm selects XMM */
+    TYPE_1,
+    TYPE_FLOATCONST,
+    TYPE_XLAT,     /* DS:(RE)(E)BX+AL */
+    TYPE_MASKMOVQ, /* DS:(RE)(E)DI */
+    TYPE_FLOATMEM,
+    TYPE_REG,     /* hardcoded register */
+    TYPE_VAR_REG, /* hardcoded register, default 32 bits, but can be
+                   * 16 w/ data prefix or 64 w/ rex.w: equivalent of Intel 'v'
+                   * == like OPSZ_4_rex8_short2 */
+    TYPE_VARZ_REG, /* hardcoded register, default 32 bits, but can be
+                    * 16 w/ data prefix: equivalent of Intel 'z'
+                    * == like OPSZ_4_short2 */
+    TYPE_VAR_XREG, /* hardcoded register, default 32/64 bits depending on mode,
+                    * but can be 16 w/ data prefix: equivalent of Intel 'd64'
+                    * == like OPSZ_4x8_short2 */
+    TYPE_VAR_ADDR_XREG, /* hardcoded register, default 32/64 bits depending on mode,
+                         * but can be 16/32 w/ addr prefix: equivalent of Intel 'd64' */
+    /* For x64 extensions (Intel '+r.') where rex.r can select an extended
+     * register (r8-r15): we could try to add a flag that modifies the above
+     * register types, but we'd have to stick it inside some stolen bits.  For
+     * simplicity, we just make each combination a separate type:
+     */
+    TYPE_REG_EX,      /* like TYPE_REG but extendable. used for mov_imm 8-bit immed */
+    TYPE_VAR_REG_EX,  /* like TYPE_VAR_REG (OPSZ_4_rex8_short2) but extendable.
+                       * used for xchg and mov_imm 'v' immed. */
+    TYPE_VAR_XREG_EX, /* like TYPE_VAR_XREG (OPSZ_4x8_short2) but extendable.
+                       * used for pop and push. */
+    TYPE_VAR_REGX_EX, /* hardcoded register, default 32 bits, but can be 64 w/ rex.w,
+                       * and extendable.  used for bswap. 
+                       * == OPSZ_4_rex8 */
+    TYPE_INDIR_E,
+    TYPE_INDIR_REG,
+    TYPE_INDIR_VAR_XREG, /* indirected register that varies (by addr prefix),
+                          * with a base of 32/64 depending on the mode;
+                          * indirected size varies with data prefix */
+    TYPE_INDIR_VAR_REG, /* indirected register that varies (by addr prefix),
+                         * with a base of 32/64;
+                         * indirected size varies with data and rex prefixes */
+    TYPE_INDIR_VAR_XIREG, /* indirected register that varies (by addr prefix),
+                           * with a base of 32/64 depending on the mode;
+                           * indirected size varies w/ data prefix, except 64-bit Intel */
+};
+
+
+/* PR 225845: Our IR does not try to specify the format of the operands or the
+ * addressing mode in opnd_t.size: only the size.  Our decode table uses the
+ * Intel opcode table "type" fields, and we used to use them for opnd_t.size.
+ * They do say more than just the size, but in core code we use the TYPE_ to
+ * tell us any formatting we need to know about, and we've always treated
+ * identical sizes with different formatting identically: we do not distinguish
+ * 128-bit packed doubles from 128-bit packed floats, e.g.  Would any client
+ * want that distinction?  There are enough subtleties in the ISA that
+ * dispatching by opcode is probably going to be necessary for the client anyway
+ * (e.g., maskmovq only writes selected bytes).  Furthermore, many of the
+ * distinctions in the OPSZ_ constants apply only to registers, with such
+ * distinctions having no way to be specified when constructing an operand as we
+ * do not use the size field for register operand types (we only use it for
+ * immediates and memory references): to be complete in supplying formatting
+ * information we would want to use that field.  Decision: we're only going to
+ * provide size information.
+ */
+
+/* DR_API EXPORT TOFILE dr_ir_opnd.h */
+/* DR_API EXPORT BEGIN */
+
+/* Memory operand sizes (with Intel's corresponding size names noted).
+ * For register operands, the REG_ constants are used, which implicitly
+ * state a size (e.g., REG_CX is 2 bytes).
+ * Use the type opnd_size_t for these values (we avoid typedef-ing the
+ * enum, as its storage size is compiler-specific).  opnd_size_t is a
+ * byte, so the largest value here needs to be <= 255.
+ */
+enum {
+    /* register enum values are used for TYPE_*REG */
+    OPSZ_NA = REG_LAST_ENUM+1, /**< Sentinel value: not a valid size. */ /* = 139 */
+    OPSZ_0,  /**< Intel 'm': "sizeless": used for both start addresses
+              * (lea, invlpg) and implicit constants (rol, fldl2e, etc.) */
+    OPSZ_1,  /**< Intel 'b': 1 byte */
+    OPSZ_2,  /**< Intel 'w': 2 bytes */
+    OPSZ_4,  /**< Intel 'd','si': 4 bytes */
+    OPSZ_6,  /**< Intel 'p','s': 6 bytes */
+    OPSZ_8,  /**< Intel 'q','pi': 8 bytes */
+    OPSZ_10, /**< Intel 's' 64-bit, or double extended precision floating point
+              * (latter used by fld, fstp, fbld, fbstp) */
+    OPSZ_16, /**< Intel 'dq','ps','pd','ss','sd': 16 bytes */
+    OPSZ_14, /**< FPU operating environment with short data size (fldenv, fnstenv) */
+    OPSZ_28, /**< FPU operating environment with normal data size (fldenv, fnstenv) */
+    OPSZ_94,  /**< FPU state with short data size (fnsave, frstor) */
+    OPSZ_108, /**< FPU state with normal data size (fnsave, frstor) */
+    OPSZ_512, /**< FPU, MMX, XMM state (fxsave, fxrstor) */
+    /**
+     * The following sizes (OPSZ_*_short*) vary according to the cs segment and the
+     * operand size prefix.  This IR assumes that the cs segment is set to the
+     * default operand size.  The operand size prefix then functions to shrink the
+     * size.  The IR does not explicitly mark the prefix; rather, a shortened size is
+     * requested in the operands themselves, with the IR adding the prefix at encode
+     * time.  Normally the fixed sizes above should be used rather than these
+     * variable sizes, which are used internally by the IR and should only be
+     * externally specified when building an operand in order to be flexible and
+     * allow other operands to decide the size for the instruction (the prefix
+     * applies to the entire instruction).
+     */
+    OPSZ_2_short1, /**< Intel 'c': 2/1 bytes ("2/1" means 2 bytes normally, but if
+                    * another operand requests a short size then this size can
+                    * accomodate by shifting to its short size, which is 1 byte). */
+    OPSZ_4_short2, /**< Intel 'z': 4/2 bytes */
+    OPSZ_4_rex8_short2, /**< Intel 'v': 8/4/2 bytes */
+    OPSZ_4_rex8,   /**< Intel 'd/q' (like 'v' but never 2 bytes). */
+    OPSZ_6_irex10_short4, /**< Intel 'p': On Intel processors this is 10/6/4 bytes for
+                           * segment selector + address.  On AMD processors this is
+                           * 6/4 bytes for segment selector + address (rex is ignored). */
+    OPSZ_8_short2, /**< partially resolved 4x8_short2 */
+    OPSZ_8_short4, /**< Intel 'a': pair of 4_short2 (bound) */
+    OPSZ_28_short14, /**< FPU operating env variable data size (fldenv, fnstenv) */
+    OPSZ_108_short94, /**< FPU state with variable data size (fnsave, frstor) */
+    /** Varies by 32-bit versus 64-bit processor mode. */
+    OPSZ_4x8,  /**< Full register size with no variation by prefix.
+                *   Used for control and debug register moves. */
+    OPSZ_6x10, /**< Intel 's': 6-byte (10-byte for 64-bit mode) table base + limit */
+    /** 
+     * Stack operands not only vary by operand size specifications but also
+     * by 32-bit versus 64-bit processor mode.
+     */
+    OPSZ_4x8_short2, /**< Intel 'v'/'d64' for stack operations.
+                      * Also 64-bit address-size specified operands, which are
+                      * short4 rather than short2 in 64-bit mode (but short2 in
+                      * 32-bit mode).
+                      * Note that this IR does not distinguish multiple stack
+                      * operations; dispatch by opcode must be used:
+                      *   X2 = far call/far ret
+                      *   X3 = int/iret
+                      *   X8 = pusha/popa
+                      *   X* = enter (dynamically varying amount)
+                      * Note that stack operations may also modify the stack
+                      * pointer prior to accessing the top of the stack, so
+                      * for example "(esp)" may in fact be "4(esp)" depending
+                      * on the opcode.
+                      */
+    OPSZ_4x8_short2xi8, /**< Intel 'f64': 4_short2 for 32-bit, 8_short2 for 64-bit AMD,
+                         *   always 8 for 64-bit Intel */
+    OPSZ_4_short2xi4,   /**< Intel 'f64': 4_short2 for 32-bit or 64-bit AMD,
+                         *   always 4 for 64-bit Intel */
+    /**
+     * The following sizes differ based on whether the modrm chooses a
+     * register or memory.
+     */
+    OPSZ_1_reg4,  /**< Intel Rd/Mb: zero-extends if reg; used by pextrb */
+    OPSZ_2_reg4,  /**< Intel Rd/Mw: zero-extends if reg; used by pextrw */
+    OPSZ_4_reg16, /**< Intel Udq/Md: sub-xmm but we consider that whole xmm;
+                   *   used by insertps. */
+    OPSZ_LAST,
+};
+
+#ifdef X64
+# define OPSZ_PTR OPSZ_8       /**< Operand size for pointer values. */
+# define OPSZ_STACK OPSZ_8     /**< Operand size for stack push/pop operand sizes. */
+#else
+# define OPSZ_PTR OPSZ_4       /**< Operand size for pointer values. */
+# define OPSZ_STACK OPSZ_4     /**< Operand size for stack push/pop operand sizes. */
+#endif
+#define OPSZ_VARSTACK OPSZ_4x8_short2 /**< Operand size for prefix-varying stack
+                                       * push/pop operand sizes. */
+#define OPSZ_REXVARSTACK OPSZ_4_rex8_short2 /* Operand size for prefix/rex-varying
+                                             * stack push/pop like operand sizes. */
+
+#define OPSZ_ret OPSZ_4x8_short2xi8 /**< Operand size for ret instruction. */
+#define OPSZ_call OPSZ_ret         /**< Operand size for push portion of call. */
+
+/* Convenience defines for specific opcodes */
+#define OPSZ_lea OPSZ_0              /**< Operand size for lea memory reference. */
+#define OPSZ_invlpg OPSZ_0           /**< Operand size for invlpg memory reference. */
+#define OPSZ_xlat OPSZ_1             /**< Operand size for xlat memory reference. */
+#define OPSZ_clflush OPSZ_1          /**< Operand size for clflush memory reference. */
+#define OPSZ_prefetch OPSZ_1         /**< Operand size for prefetch memory references. */
+#define OPSZ_lgdt OPSZ_6x10          /**< Operand size for lgdt memory reference. */
+#define OPSZ_sgdt OPSZ_6x10          /**< Operand size for sgdt memory reference. */
+#define OPSZ_lidt OPSZ_6x10          /**< Operand size for lidt memory reference. */
+#define OPSZ_sidt OPSZ_6x10          /**< Operand size for sidt memory reference. */
+#define OPSZ_bound OPSZ_8_short4     /**< Operand size for bound memory reference. */
+#define OPSZ_maskmovq OPSZ_8         /**< Operand size for maskmovq memory reference. */
+#define OPSZ_maskmovdqu OPSZ_16      /**< Operand size for maskmovdqu memory reference. */
+#define OPSZ_fldenv OPSZ_28_short14  /**< Operand size for fldenv memory reference. */
+#define OPSZ_fnstenv OPSZ_28_short14 /**< Operand size for fnstenv memory reference. */
+#define OPSZ_fnsave OPSZ_108_short94 /**< Operand size for fnsave memory reference. */
+#define OPSZ_frstor OPSZ_108_short94 /**< Operand size for frstor memory reference. */
+#define OPSZ_fxsave OPSZ_512         /**< Operand size for fxsave memory reference. */
+#define OPSZ_fxrstor OPSZ_512        /**< Operand size for fxrstor memory reference. */
+/* DR_API EXPORT END */
+
+enum {
+    /* OPSZ_ constants not exposed to the user */
+    OPSZ_4_of_8 = OPSZ_LAST,  /* 32 bits, but can be half of MMX register */
+    OPSZ_4_of_16, /* 32 bits, but can be part of XMM register */
+    OPSZ_8_of_16, /* 64 bits, but can be half of XMM register */
+    OPSZ_LAST_ENUM /* note last is NOT inclusive */
+};
+
+#ifdef X64
+# define OPSZ_STATS OPSZ_8
+#else
+# define OPSZ_STATS OPSZ_4
+#endif
+
+#define MODRM_BYTE(mod, reg, rm) ((byte) (((mod) << 6) | ((reg) << 3) | (rm)))
+
+/* in decode.c, not exported to non-x86 files */
+opnd_size_t resolve_var_reg_size(opnd_size_t sz, bool is_reg);
+opnd_size_t resolve_variable_size(decode_info_t *di/*IN: x86_mode, prefixes*/,
+                                  opnd_size_t sz, bool is_reg);
+opnd_size_t resolve_variable_size_dc(dcontext_t *dcontext,
+                                     uint prefixes, opnd_size_t sz, bool is_reg);
+/* Also takes in reg8 for TYPE_REG_EX mov_imm */
+reg_id_t resolve_var_reg(decode_info_t *di/*IN: x86_mode, prefixes*/,
+                         reg_id_t reg32, bool addr, bool can_shrink
+                         _IF_X64(bool default_64) _IF_X64(bool can_grow)
+                         _IF_X64(bool extendable));
+opnd_size_t resolve_addr_size(decode_info_t *di/*IN: x86_mode, prefixes*/);
+opnd_size_t indir_var_reg_size(int optype);
+
+/* in encode.c, not exported to non-x86 files */
+const instr_info_t * get_encoding_info(instr_t *instr);
+const instr_info_t * instr_info_extra_opnds(const instr_info_t *info);
+byte instr_info_opnd_type(const instr_info_t *info, bool src, uint num);
+
+extern const instr_info_t invalid_instr;
+
+/* DR_API EXPORT TOFILE dr_ir_utils.h */
+
+/* exported routines */
+
+#define DEFAULT_X86_MODE IF_X64_ELSE(false, true)
+/* for decode_info_t */
+#define X64_MODE(di) IF_X64_ELSE(!(di)->x86_mode, false)
+/* for dcontext_t */
+#define X64_MODE_DC(dc) IF_X64_ELSE(!get_x86_mode(dc), false)
+
+
+DR_API
+/**
+ * Decodes only enough of the instruction at address \p pc to determine
+ * its eflags usage, which is returned in \p usage as EFLAGS_ constants
+ * or'ed together.
+ * Returns the address of the next byte after the decoded instruction.
+ * Returns NULL on decoding an invalid instruction.
+ */
+#ifdef UNSUPPORTED_API
+/**
+ * This corresponds to halfway between Level 1 and Level 2: a Level 1 decoding
+ * plus eflags information (usually only at Level 2).
+ */
+#endif
+byte *
+decode_eflags_usage(dcontext_t *dcontext, byte *pc, uint *usage);
+
+DR_UNS_API
+/**
+ * Decodes the opcode and eflags usage of instruction at address \p pc
+ * into \p instr.
+ * The instruction's raw bits are set to valid and pointed at \p pc
+ * (xref instr_get_raw_bits()).
+ * Assumes that \p instr is already initialized, and uses the x86/x64 mode
+ * set for it rather than the current thread's mode!
+ * If caller is re-using same instr_t struct over multiple decodings,
+ * caller should call instr_reset() or instr_reuse().
+ * Returns the address of the next byte after the decoded instruction.
+ * Returns NULL on decoding an invalid instr and sets opcode to OP_INVALID.
+ */
+byte *
+decode_opcode(dcontext_t *dcontext, byte *pc, instr_t *instr);
+
+DR_API
+/**
+ * Decodes the instruction at address \p pc into \p instr, filling in the
+ * instruction's opcode, eflags usage, prefixes, and operands.
+ * The instruction's raw bits are set to valid and pointed at \p pc
+ * (xref instr_get_raw_bits()).
+ * Assumes that \p instr is already initialized, but uses the x86/x64 mode
+ * for the thread \p dcontext rather than that set in instr.
+ * If caller is re-using same instr_t struct over multiple decodings,
+ * caller should call instr_reset() or instr_reuse().
+ * Returns the address of the next byte after the decoded instruction.
+ * Returns NULL on decoding an invalid instr and sets opcode to OP_INVALID.
+ */
+#ifdef UNSUPPORTED_API
+/**
+ * This corresponds to a Level 3 decoding.
+ */
+#endif
+byte *
+decode(dcontext_t *dcontext, byte *pc, instr_t *instr);
+
+DR_API
+/**
+ * Decodes the instruction at address \p copy_pc into \p instr as though
+ * it were located at address \p orig_pc.  Any pc-relative operands have
+ * their values calculated as though the instruction were actually at
+ * \p orig_pc, though that address is never de-referenced.
+ * The instruction's raw bits are not valid, but its translation field
+ * (see instr_get_translation()) is set to \p orig_pc.
+ * The instruction's opcode, eflags usage, prefixes, and operands are
+ * all filled in.
+ * Assumes that \p instr is already initialized, but uses the x86/x64 mode
+ * for the thread \p dcontext rather than that set in instr.
+ * If caller is re-using same instr_t struct over multiple decodings,
+ * caller should call instr_reset() or instr_reuse().
+ * Returns the address of the next byte after the decoded instruction
+ * copy at \p copy_pc.
+ * Returns NULL on decoding an invalid instr and sets opcode to OP_INVALID.
+ */
+#ifdef UNSUPPORTED_API
+/**
+ * This corresponds to a Level 3 decoding.
+ */
+#endif
+byte *
+decode_from_copy(dcontext_t *dcontext, byte *copy_pc, byte *orig_pc, instr_t *instr);
+
+#ifdef CLIENT_INTERFACE
+/* decode_as_bb() is defined in interp.c, but declared here so it will
+ * be listed next to the other decode routines in the API headers.
+ */
+DR_API
+/**
+ * Client routine to decode instructions at an arbitrary app address,
+ * following all the rules that DynamoRIO follows internally for
+ * terminating basic blocks.  Note that DynamoRIO does not validate
+ * that \p start_pc is actually the first instruction of a basic block.
+ * \note Caller is reponsible for freeing the list and its instrs!
+ */
+instrlist_t *
+decode_as_bb(void *drcontext, byte *start_pc);
+
+/* decode_trace() is also in interp.c */
+DR_API
+/**
+ * Decodes the trace with tag \p tag, and returns an instrlist_t of
+ * the instructions comprising that fragment.  If \p tag is not a
+ * valid tag for an existing trace, the routine returns NULL.  Clients
+ * can use dr_trace_exists_at() to determine whether the trace exists.
+ * \note Unlike the instruction list presented by the trace event, the
+ * list here does not include any existing client modifications.  If
+ * client-modified instructions are needed, it is the responsibility
+ * of the client to record or recreate that list itself.
+ * \note This routine does not support decoding thread-private traces
+ * created by other than the calling thread.
+ */
+instrlist_t *
+decode_trace(void *drcontext, void *tag);
+#endif
+
+const struct instr_info_t * get_next_instr_info(const instr_info_t * info);
+
+DR_API
+/**
+ * Given an OP_ constant, returns the first byte of its opcode when
+ * encoded as an IA-32 instruction.
+ */
+byte 
+decode_first_opcode_byte(int opcode);
+
+/* DR_API EXPORT TOFILE dr_ir_opcodes.h */
+
+/* exported tables */
+extern const instr_info_t first_byte[];
+extern const instr_info_t second_byte[];
+extern const instr_info_t extensions[][8];
+extern const instr_info_t prefix_extensions[][4];
+extern const instr_info_t mod_extensions[][2];
+extern const instr_info_t rm_extensions[][8];
+extern const instr_info_t x64_extensions[][2];
+extern const instr_info_t rex_extensions[][2];
+extern const byte third_byte_38_index[256];
+extern const byte third_byte_3a_index[256];
+extern const instr_info_t third_byte_38[];
+extern const instr_info_t third_byte_3a[];
+extern const instr_info_t rep_extensions[][4];
+extern const instr_info_t repne_extensions[][6];
+extern const instr_info_t float_low_modrm[];
+extern const instr_info_t float_high_modrm[][64];
+extern const byte suffix_index[256];
+extern const instr_info_t suffix_extensions[];
+extern const instr_info_t extra_operands[];
+/* table that translates opcode enums into pointers into above tables */
+extern const instr_info_t * const op_instr[];
+/* for debugging: printing out types and sizes */
+extern const char * const type_names[];
+extern const char * const size_names[];
+
+#endif /* DECODE_H */
diff --git a/core/x86/decode_fast.c b/core/x86/decode_fast.c
new file mode 100644
index 000000000..cd5698795
--- /dev/null
+++ b/core/x86/decode_fast.c
@@ -0,0 +1,1208 @@
+/* **********************************************************
+ * Copyright (c) 2001-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2001 Hewlett-Packard Company */
+
+/* decode_fast.c -- a partial but fast x86 decoder */
+
+#include "globals.h"
+#include "decode_fast.h"
+#include "link.h"
+#include "arch.h"
+#include "instr.h"
+#include "instr_create.h"
+#include "decode.h"
+#include "disassemble.h"
+
+#ifdef DEBUG
+/* case 10450: give messages to clients */
+# undef ASSERT /* N.B.: if have issues w/ DYNAMO_OPTION, re-instate */
+# undef ASSERT_TRUNCATE
+# undef ASSERT_BITFIELD_TRUNCATE
+# undef ASSERT_NOT_REACHED
+# define ASSERT DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_TRUNCATE DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_BITFIELD_TRUNCATE DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_NOT_REACHED DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+#endif
+
+/* This file contains tables and functions that help decode x86
+   instructions so that we can determine the length of the decode
+   instruction.  All code below based on tables in the ``Intel
+   Architecture Software Developer's Manual,'' Volume 2: Instruction
+   Set Reference, 1999. 
+   This decoder assumes that we are running in 32-bit, flat-address mode.  
+*/
+
+/* NOTE that all of the tables in this file are indexed by the (primary
+   or secondary) opcode byte.  The upper opcode nibble defines the rows,
+   starting with 0 at the top.  The lower opcode nibble defines the
+   columns, starting with 0 at left. */
+
+/* Data table for fixed part of an x86 instruction.  The table is
+   indexed by the 1st (primary) opcode byte.  Zero entries are
+   reserved opcodes. */
+static const byte fixed_length[256] = {
+    1,1,1,1, 2,5,1,1, 1,1,1,1, 2,5,1,1,  /* 0 */
+    1,1,1,1, 2,5,1,1, 1,1,1,1, 2,5,1,1,  /* 1 */
+    1,1,1,1, 2,5,1,1, 1,1,1,1, 2,5,1,1,  /* 2 */
+    1,1,1,1, 2,5,1,1, 1,1,1,1, 2,5,1,1,  /* 3 */
+                                                
+    1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1,  /* 4 */
+    1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1,  /* 5 */
+    1,1,1,1, 1,1,1,1, 5,5,2,2, 1,1,1,1,  /* 6 */
+    2,2,2,2, 2,2,2,2, 2,2,2,2, 2,2,2,2,  /* 7 */
+                                                
+    2,5,2,2, 1,1,1,1, 1,1,1,1, 1,1,1,1,  /* 8 */
+    1,1,1,1, 1,1,1,1, 1,1,7,1, 1,1,1,1,  /* 9 */
+    5,5,5,5, 1,1,1,1, 2,5,1,1, 1,1,1,1,  /* A */
+    2,2,2,2, 2,2,2,2, 5,5,5,5, 5,5,5,5,  /* B */
+                                                
+    2,2,3,1, 1,1,2,5, 4,1,3,1, 1,2,1,1,  /* C */
+    1,1,1,1, 2,2,1,1, 1,1,1,1, 1,1,1,1,  /* D */
+    2,2,2,2, 2,2,2,2, 5,5,7,2, 1,1,1,1,  /* E */
+    1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1   /* F */
+    /* f6 and f7 OP_test immeds are handled specially in decode_sizeof() */
+};
+
+/* Data table for fixed immediate part of an x86 instruction that
+   depends upon the existence of an operand-size byte.  The table is
+   indexed by the 1st (primary) opcode byte.  Entries with non-zero
+   values indicate opcodes with a variable-length immediate field.  We
+   use this table if we've seen a operand-size prefix byte to adjust
+   the fixed_length from dword to word. 
+ */
+static const signed char immed_adjustment[256] = {
+     0, 0, 0, 0,  0,-2, 0, 0,  0, 0, 0, 0,  0,-2, 0, 0,  /* 0 */
+     0, 0, 0, 0,  0,-2, 0, 0,  0, 0, 0, 0,  0,-2, 0, 0,  /* 1 */
+     0, 0, 0, 0,  0,-2, 0, 0,  0, 0, 0, 0,  0,-2, 0, 0,  /* 2 */
+     0, 0, 0, 0,  0,-2, 0, 0,  0, 0, 0, 0,  0,-2, 0, 0,  /* 3 */
+                                                                
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 4 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 5 */
+     0, 0, 0, 0,  0, 0, 0, 0, -2,-2, 0, 0,  0, 0, 0, 0,  /* 6 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 7 */
+                                                                
+     0,-2, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 8 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0,-2, 0,  0, 0, 0, 0,  /* 9 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0,-2, 0, 0,  0, 0, 0, 0,  /* A */
+     0, 0, 0, 0,  0, 0, 0, 0, -2,-2,-2,-2, -2,-2,-2,-2,  /* B */
+                                                                
+     0, 0, 0, 0,  0, 0, 0,-2,  0, 0, 0, 0,  0, 0, 0, 0,  /* C */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* D */
+     0, 0, 0, 0,  0, 0, 0, 0, -2,-2,-2,-2,  0, 0, 0, 0,  /* E */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0   /* F */
+};
+
+/* Data table for fixed immediate part of an x86 instruction that
+ * depends upon the existence of an address-size byte.  The table is
+ * indexed by the 1st (primary) opcode byte.
+ * The value here is doubled for x64 mode.
+ */
+static const signed char disp_adjustment[256] = {
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 0 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 1 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 2 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 3 */
+                                                                
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 4 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 5 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 6 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 7 */
+                                                                
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 8 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 9 */
+    -2,-2,-2,-2,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* A */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* B */
+                                                                
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* C */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* D */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* E */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0   /* F */
+};
+
+#ifdef X64
+/* Data table for immediate adjustments that only apply when
+ * in x64 mode.  We fit two types of adjustments in here:
+ * default-size adjustments (positive numbers) and rex.w-prefix-based
+ * adjustments (negative numbers, to be made positive when applied).
+ */
+static const char x64_adjustment[256] = {
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 0 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 1 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 2 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 3 */
+                                                                
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 4 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 5 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 6 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 7 */
+                                                                
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 8 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 9 */
+     4, 4, 4, 4,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* A */
+     0, 0, 0, 0,  0, 0, 0, 0, -4,-4,-4,-4, -4,-4,-4,-4,  /* B */
+                                                                
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* C */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* D */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* E */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0   /* F */
+};
+#endif
+
+/* Prototypes for the functions that calculate the variable
+ * part of the x86 instruction length. */
+static int sizeof_modrm(dcontext_t *dcontext, byte *pc, bool addr16
+                        _IF_X64(byte **rip_rel_pc));
+static int sizeof_fp_op(dcontext_t *dcontext, byte *pc, bool addr16
+                        _IF_X64(byte **rip_rel_pc));
+static int sizeof_escape(dcontext_t *dcontext, byte *pc, bool addr16
+                         _IF_X64(byte **rip_rel_pc));
+
+enum {
+    VARLEN_NONE,
+    VARLEN_MODRM,
+    VARLEN_FP_OP,
+    VARLEN_ESCAPE, /* 2-byte opcodes */
+    VARLEN_3BYTE_38_ESCAPE, /* 3-byte opcodes 0f 38 */
+    VARLEN_3BYTE_3A_ESCAPE, /* 3-byte opcodes 0f 3a */
+};
+
+/* Some macros to make the following table look better. */
+#define m VARLEN_MODRM
+#define f VARLEN_FP_OP
+#define e VARLEN_ESCAPE
+
+/* Data table indicating what function to use to calculate
+   the variable part of the x86 instruction.  This table
+   is indexed by the primary opcode.  */
+static const byte variable_length[256] = {
+    m,m,m,m, 0,0,0,0, m,m,m,m, 0,0,0,e,   /* 0 */
+    m,m,m,m, 0,0,0,0, m,m,m,m, 0,0,0,0,   /* 1 */
+    m,m,m,m, 0,0,0,0, m,m,m,m, 0,0,0,0,   /* 2 */
+    m,m,m,m, 0,0,0,0, m,m,m,m, 0,0,0,0,   /* 3 */
+                                                 
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,   /* 4 */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,   /* 5 */
+    0,0,m,m, 0,0,0,0, 0,m,0,m, 0,0,0,0,   /* 6 */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,   /* 7 */
+                                                 
+    m,m,m,m, m,m,m,m, m,m,m,m, m,m,m,m,   /* 8 */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,   /* 9 */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,   /* A */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,   /* B */
+                                                 
+    m,m,0,0, m,m,m,m, 0,0,0,0, 0,0,0,0,   /* C */
+    m,m,m,m, 0,0,0,0, f,f,f,f, f,f,f,f,   /* D */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,   /* E */
+    0,0,0,0, 0,0,m,m, 0,0,0,0, 0,0,m,m    /* F */
+};
+
+/* eliminate the macros */
+#undef m
+#undef f
+#undef e
+
+
+/* Data table for the additional fixed part of a two-byte opcode.
+ * This table is indexed by the 2nd opcode byte.  Zero entries are
+ * reserved/bad opcodes. 
+ * N.B.: none of these (except IA32_ON_IA64) need adjustment
+ * for data16 or addr16.
+ */
+static const byte escape_fixed_length[256] = {
+    1,1,1,1, 0,1,1,1, 1,1,0,1, 0,1,1,2,  /* 0 */ /* 0f0f has extra suffix opcode byte */
+    1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1,  /* 1 */
+    1,1,1,1, 0,0,0,0, 1,1,1,1, 1,1,1,1,  /* 2 */
+    1,1,1,1, 1,1,0,0, 1,0,1,0, 0,0,0,0,  /* 3 */ 
+    1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1,  /* 4 */
+    1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1,  /* 5 */
+    1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1,  /* 6 */
+    2,2,2,2, 1,1,1,1, 1,1,0,0, 1,1,1,1,  /* 7 */
+                                                
+    5,5,5,5, 5,5,5,5, 5,5,5,5, 5,5,5,5,  /* 8 */
+    1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1,  /* 9 */
+    1,1,1,1, 2,1,0,0, 1,1,1,1, 2,1,1,1,  /* A */
+#ifdef IA32_ON_IA64
+    /* change is the 5, could also be 3 depending on which mode we are */
+    /* FIXME : no modrm byte so the standard variable thing won't work */
+    /* (need a escape_disp_adjustment table) */
+    1,1,1,1, 1,1,1,1, 5,1,2,1, 1,1,1,1,  /* B */
+#else
+    1,1,1,1, 1,1,1,1, 1,1,2,1, 1,1,1,1,  /* B */
+#endif
+                
+    1,1,2,1, 2,2,2,1, 1,1,1,1, 1,1,1,1,  /* C */
+    1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1,  /* D */
+    1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1,  /* E */
+    1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,0   /* F */
+    /* 0f78 has immeds depending on prefixes: handled in decode_sizeof() */
+};
+
+
+/* Some macros to make the following table look better. */
+#define m VARLEN_MODRM
+#define e1 VARLEN_3BYTE_38_ESCAPE
+#define e2 VARLEN_3BYTE_3A_ESCAPE
+
+/* Data table indicating what function to use to calcuate
+   the variable part of the escaped x86 instruction.  This table
+   is indexed by the 2nd opcode byte.  */
+static const byte escape_variable_length[256] = {
+    m,m,m,m, 0,0,0,0, 0,0,0,0, 0,m,0,m, /* 0 */
+    m,m,m,m, m,m,m,m, m,m,m,m, m,m,m,m, /* 1 */
+    m,m,m,m, 0,0,0,0, m,m,m,m, m,m,m,m, /* 2 */
+    0,0,0,0, 0,0,0,0, e1,0,e2,0, 0,0,0,0, /* 3 */
+
+    m,m,m,m, m,m,m,m, m,m,m,m, m,m,m,m, /* 4 */
+    m,m,m,m, m,m,m,m, m,m,m,m, m,m,m,m, /* 5 */
+    m,m,m,m, m,m,m,m, m,m,m,m, m,m,m,m, /* 6 */
+    m,m,m,m, m,m,m,0, m,m,0,0, m,m,m,m, /* 7 */
+
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, /* 8 */
+    m,m,m,m, m,m,m,m, m,m,m,m, m,m,m,m, /* 9 */
+    0,0,0,m, m,m,0,0, 0,0,0,m, m,m,m,m, /* A */
+#ifdef IA32_ON_IA64
+    m,m,m,m, m,m,m,m, 0,0,m,m, m,m,m,m, /* B */
+#else
+    m,m,m,m, m,m,m,m, m,0,m,m, m,m,m,m, /* B */
+#endif
+    m,m,m,m, m,m,m,m, 0,0,0,0, 0,0,0,0, /* C */
+    m,m,m,m, m,m,m,m, m,m,m,m, m,m,m,m, /* D */
+    m,m,m,m, m,m,m,m, m,m,m,m, m,m,m,m, /* E */
+    m,m,m,m, m,m,m,m, m,m,m,m, m,m,m,0  /* F */
+};
+
+/* eliminate the macros */
+#undef m
+#undef e
+
+/* Data table for the additional fixed part of a three-byte opcode 0f 38.
+ * This table is indexed by the 3rd opcode byte.  Zero entries are
+ * reserved/bad opcodes.
+ * N.B.: ALL of these have modrm bytes, and NONE of these need adjustment for data16
+ * or addr16.
+ */
+#if 0 /* to be robust wrt future additions we assume all entries are 1 */
+static const byte threebyte_38_fixed_length[256] = {
+    1,1,1,1, 1,1,1,1, 1,1,1,1, 0,0,0,0,  /* 0 */
+    1,0,0,0, 1,1,0,1, 0,0,0,0, 1,1,1,0,  /* 1 */
+    1,1,1,1, 1,1,0,0, 1,1,1,1, 0,0,0,0,  /* 2 */
+    1,1,1,1, 1,1,0,1, 1,1,1,1, 1,1,1,1,  /* 3 */ 
+    1,1,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* 4 */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* 5 */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* 6 */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* 7 */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* 8 */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* 9 */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* A */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* B */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* C */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* D */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* E */
+    1,1,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0   /* F */
+};
+#endif
+/* Three-byte 0f 3a: all are assumed to have a 1-byte immediate as well! */
+#if 0 /* to be robust wrt future additions we assume all entries are 1 */
+static const byte threebyte_3a_fixed_length[256] = {
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,1,  /* 0 */
+    0,0,0,0, 1,1,1,1, 1,1,1,1, 1,1,1,0,  /* 1 */
+    1,1,1,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* 2 */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* 3 */ 
+    0,1,1,1, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* 4 */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* 5 */
+    1,1,1,1, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* 6 */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* 7 */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* 8 */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* 9 */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* A */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* B */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* C */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* D */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,  /* E */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0   /* F */
+};
+#endif
+
+/* Returns the length of the instruction at pc.
+ * If num_prefixes is non-NULL, returns the number of prefix bytes.
+ * If rip_rel_pos is non-NULL, returns the offset into the instruction
+ * of a rip-relative addressing displacement (for data only: ignores
+ * control-transfer relative addressing), or 0 if none.
+ * May return 0 size for certain invalid instructions
+ */
+int
+decode_sizeof(dcontext_t *dcontext, byte *start_pc, int *num_prefixes
+              _IF_X64(uint *rip_rel_pos))
+{
+    byte *pc = start_pc;
+    uint opc = (uint)*pc;
+    int sz = 0;
+    ushort varlen;
+    bool word_operands = false; /* data16 */
+    bool qword_operands = false; /* rex.w */
+    bool addr16 = false; /* really "addr32" for x64 mode */
+    bool found_prefix = true;
+    bool rep_prefix = false;
+    byte reg_opcode;    /* reg_opcode field of modrm byte */
+#ifdef X64
+    byte *rip_rel_pc = NULL;
+#endif
+
+    /* Check for prefix byte(s) */
+    while (found_prefix) {
+        /* NOTE - rex prefixes must come after all other prefixes (including
+         * prefixes that are part of the opcode xref PR 271878).  We match
+         * read_instruction() in considering pre-prefix rex bytes as part of
+         * the following instr, event when ignored, rather then treating them
+         * as invalid.  This in effect nops improperly placed rex prefixes which
+         * (xref PR 241563 and Intel Manual 2A 2.2.1) is the correct thing to do.
+         * Rex prefixes are 0x40-0x4f; >=0x48 has rex.w bit set.
+         */
+        if (X64_MODE_DC(dcontext) && opc >= REX_PREFIX_BASE_OPCODE &&
+            opc <= (REX_PREFIX_BASE_OPCODE | REX_PREFIX_ALL_OPFLAGS)) {
+            if (opc >= (REX_PREFIX_BASE_OPCODE | REX_PREFIX_W_OPFLAG)) {
+                qword_operands = true;
+                if (word_operands)
+                    word_operands = false; /* rex.w trumps data16 */
+            } /* else, doesn't affect instr size */
+            opc = (uint)*(++pc);
+            sz += 1;
+        } else {
+            switch (opc) {
+            case 0x66:    /* operand size */
+                /* rex.w before other prefixes is a nop */
+                if (qword_operands)
+                    qword_operands = false;
+                word_operands = true;
+                opc = (uint)*(++pc);
+                sz += 1;
+                break;
+            case 0xf2: case 0xf3: /* REP */
+                rep_prefix = true;
+                /* fall through */
+            case 0xf0:          /* LOCK */
+            case 0x64: case 0x65: /* segment overrides */
+            case 0x26: case 0x36:
+            case 0x2e: case 0x3e:
+                opc = (uint)*(++pc);
+                sz += 1;
+                break;
+            case 0x67:
+                addr16 = true;
+                opc = (uint)*(++pc);
+                sz += 1;
+                /* up to caller to check for addr prefix! */
+                break;
+            default:
+                found_prefix = false;
+            }
+        }
+    }
+    if (num_prefixes != NULL)
+        *num_prefixes = sz;
+    if (word_operands)
+        sz += immed_adjustment[opc]; /* no adjustment for 2-byte escapes */
+    if (addr16) {  /* no adjustment for 2-byte escapes */
+        if (X64_MODE_DC(dcontext)) /* from 64 bits down to 32 bits */
+            sz += 2*disp_adjustment[opc];
+        else /* from 32 bits down to 16 bits */
+            sz += disp_adjustment[opc];
+    }
+#ifdef X64
+    if (X64_MODE_DC(dcontext)) {
+        int adj64 = x64_adjustment[opc];
+        if (adj64 > 0) /* default size adjustment */
+            sz += adj64;
+        else if (qword_operands)
+            sz += -adj64; /* negative indicates prefix, not default, adjust */
+        /* else, no adjustment */
+    }
+#endif
+
+    /* opc now really points to opcode */
+    sz += fixed_length[opc];
+    varlen = variable_length[opc];
+
+    /* for a valid instr, sz must be > 0 here, but we don't want to assert
+     * since we need graceful failure
+     */
+
+    if (varlen == VARLEN_MODRM)
+        sz += sizeof_modrm(dcontext, pc+1, addr16 _IF_X64(&rip_rel_pc));
+    else if (varlen == VARLEN_ESCAPE) {
+        sz += sizeof_escape(dcontext, pc+1, addr16 _IF_X64(&rip_rel_pc));
+        /* special case: Intel and AMD added size-differing prefix-dependent instrs! */
+        if (*(pc+1) == 0x78) {
+            if (word_operands || rep_prefix) {
+                /* extrq, insertq: 2 1-byte immeds */
+                sz += 2;
+            } /* else, vmread, w/ no immeds */
+        }
+    } else if (varlen == VARLEN_FP_OP)
+        sz += sizeof_fp_op(dcontext, pc+1, addr16 _IF_X64(&rip_rel_pc));
+    else
+        CLIENT_ASSERT(varlen == VARLEN_NONE, "internal decoding error");
+
+    /* special case that doesn't fit the mold (of course one had to exist) */
+    reg_opcode = (byte) (((*(pc + 1)) & 0x38) >> 3);
+    if (opc == 0xf6 && reg_opcode == 0) {
+        sz += 1;        /* TEST Eb,ib -- add size of immediate */
+    } else if (opc == 0xf7 && reg_opcode == 0) {
+        if (word_operands)
+            sz += 2;    /* TEST Ew,iw -- add size of immediate */
+        else
+            sz += 4;    /* TEST El,il -- add size of immediate */
+    }
+
+#ifdef X64
+    if (rip_rel_pos != NULL) {
+        if (rip_rel_pc != NULL) {
+            CLIENT_ASSERT(X64_MODE_DC(dcontext),
+                          "decode_sizeof: invalid non-x64 rip_rel instr");
+            CLIENT_ASSERT(CHECK_TRUNCATE_TYPE_uint(rip_rel_pc - start_pc),
+                          "decode_sizeof: unknown rip_rel instr type");
+            *rip_rel_pos = (uint) (rip_rel_pc - start_pc);
+        } else
+            *rip_rel_pos = 0;
+    }
+#endif
+
+    return sz;
+}
+
+
+/* Two-byte opcode map (Tables A-4 and A-5).  You use this routine
+ * when you have identified the primary opcode as 0x0f.  You pass this
+ * routine the next byte to determine the number of extra bytes in the
+ * entire instruction. 
+ * May return 0 size for certain invalid instructions.
+ */
+static int
+sizeof_escape(dcontext_t *dcontext, byte *pc, bool addr16 _IF_X64(byte **rip_rel_pc))
+{
+    uint opc = (uint)*pc;
+    int sz = escape_fixed_length[opc];
+    ushort varlen = escape_variable_length[opc];
+
+    /* for a valid instr, sz must be > 0 here, but we don't want to assert
+     * since we need graceful failure
+     */
+
+    if (varlen == VARLEN_MODRM)
+        return sz + sizeof_modrm(dcontext, pc+1, addr16 _IF_X64(rip_rel_pc));
+    else if (varlen == VARLEN_3BYTE_38_ESCAPE) {
+        opc = *(++pc);
+        /* so far all 3-byte instrs have modrm bytes */
+        /* to be robust for future additions we don't actually
+         * use the threebyte_38_fixed_length[opc] entry and assume 1 */
+        return sz + 1 + sizeof_modrm(dcontext, pc+1, addr16 _IF_X64(rip_rel_pc));
+    }
+    else if (varlen == VARLEN_3BYTE_3A_ESCAPE) {
+        opc = *(++pc);
+        /* so far all 0f 3a 3-byte instrs have modrm bytes and 1-byte immeds */
+        /* to be robust for future additions we don't actually
+         * use the threebyte_3a_fixed_length[opc] entry and assume 1 */
+        return sz + 1 + sizeof_modrm(dcontext, pc+1, addr16 _IF_X64(rip_rel_pc)) + 1;
+    }
+    else
+        CLIENT_ASSERT(varlen == VARLEN_NONE, "internal decoding error");
+
+    return sz;
+}
+
+
+/* 32-bit addressing forms with the ModR/M Byte (Table 2-2).  You call
+ * this routine with the byte following the primary opcode byte when you
+ * know that the operation's next byte is a ModR/M byte.  This routine
+ * passes back the size of the Eaddr specification in bytes based on the
+ * following encoding of Table 2-2.
+ *
+ *   Mod        R/M
+ *        0 1 2 3 4 5 6 7
+ *    0   1 1 1 1 * 5 1 1
+ *    1   2 2 2 2 3 2 2 2
+ *    2   5 5 5 5 6 5 5 5
+ *    3   1 1 1 1 1 1 1 1
+ *   where (*) is 6 if base==5 and 2 otherwise.
+ */
+static int
+sizeof_modrm(dcontext_t *dcontext, byte *pc, bool addr16 _IF_X64(byte **rip_rel_pc))
+{
+    int l = 0;          /* return value for sizeof(eAddr) */
+
+    uint modrm = (uint)*pc;
+    int r_m = modrm & 0x7;
+    uint mod = modrm >> 6;
+    uint sib;
+
+#ifdef X64
+    if (rip_rel_pc != NULL && X64_MODE_DC(dcontext) && mod == 0 && r_m == 5) {
+        *rip_rel_pc = pc + 1; /* no sib: next 4 bytes are disp */
+    }
+#endif
+
+    if (addr16 && !X64_MODE_DC(dcontext)) {
+        if (mod == 1)
+            return 2; /* modrm + disp8 */
+        else if (mod == 2)
+            return 3; /* modrm + disp16 */
+        else if (mod == 3)
+            return 1; /* just modrm */
+        else {
+            CLIENT_ASSERT(mod == 0, "internal decoding error on addr16 prefix");
+            if (r_m == 6)
+                return 3; /* modrm + disp16 */
+            else
+                return 1; /* just modrm */
+        }
+        CLIENT_ASSERT(false, "internal decoding error on addr16 prefix");
+    }
+    
+    /* for x64, addr16 simply truncates the computed address: there is
+     * no change in disp sizes */
+    
+    if (mod == 3)       /* register operand */
+        return 1;
+
+    switch (mod) {      /* memory or immediate operand */
+      case 0: l = (r_m == 5) ? 5 : 1; break;
+      case 1: l = 2; break;
+      case 2: l = 5; break;
+    }
+    if (r_m == 4) {
+        l += 1;         /* adjust for sib byte */
+        sib = (uint)(*(pc+1));
+        if ((sib & 0x7) == 5) {
+            if (mod == 0)
+                l += 4; /* disp32(,index,s) */
+        }
+    }   
+
+    return l;
+}
+
+/* General floating-point instruction formats (Table B-22).  You use
+ * this routine when you have identified the primary opcode as one in
+ * the range 0xb8 through 0xbf.  You pass this routine the next byte
+ * to determine the number of extra bytes in the entire
+ * instruction. */
+static int
+sizeof_fp_op(dcontext_t *dcontext, byte *pc, bool addr16 _IF_X64(byte **rip_rel_pc))
+{
+    if (*pc > 0xbf)
+        return 1;       /* entire ModR/M byte is an opcode extension */
+
+    /* fp opcode in reg/opcode field */
+    return sizeof_modrm(dcontext, pc, addr16 _IF_X64(rip_rel_pc));
+}
+
+
+/* Table indicating "interesting" instructions, i.e., ones we
+ * would like to decode.  Currently these are control-transfer
+ * instructions and interrupts.
+ * This table is indexed by the 1st (primary) opcode byte.
+ * A 0 indicates we are not interested, a 1 that we are.
+ * A 2 indicates a second opcode byte exists, a 3 indicates an opcode
+ * extension is present in the modrm byte.
+ */
+static const byte interesting[256] = {
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,2,
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+    1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1, /* jcc_short */
+
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+    0,0,0,0, 0,0,0,0, 0,0,1,0, 0,0,0,0, /* call_far */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+
+    0,0,1,1, 0,0,0,0, 0,0,1,1, 1,1,1,1, /* ret*, int* */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
+    1,1,1,1, 0,0,0,0, 1,1,1,1, 0,0,0,0, /* loop*, call, jmp* */
+    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,3,
+};
+
+
+/* Table indicating operations on the lower 6 eflags (CF,PF,AF,ZF,SF,OF)
+ * This table is indexed by the 1st (primary) opcode byte.
+ * We use the eflags constants from instr.h.
+ * We ignore writing some of the 6 as a conservative simplification.
+ * Also note that for some groups we assign values to invalid opcodes
+ * just for simplicity
+ */
+#define x     0
+#define RC   EFLAGS_READ_CF
+#define RP   EFLAGS_READ_PF
+#define RZ   EFLAGS_READ_ZF
+#define RS   EFLAGS_READ_SF
+#define RO   EFLAGS_READ_OF
+#define R6   EFLAGS_READ_6
+#define RB   (EFLAGS_READ_CF|EFLAGS_READ_ZF)
+#define RL   (EFLAGS_READ_SF|EFLAGS_READ_OF)
+#define RE   (EFLAGS_READ_SF|EFLAGS_READ_OF|EFLAGS_READ_ZF)
+#define R5O  (EFLAGS_READ_6 & (~EFLAGS_READ_OF))
+#define WC   EFLAGS_WRITE_CF
+#define WZ   EFLAGS_WRITE_ZF
+#define W6   EFLAGS_WRITE_6
+#define W5   (EFLAGS_WRITE_6 & (~EFLAGS_WRITE_CF))
+#define W5O  (EFLAGS_WRITE_6 & (~EFLAGS_WRITE_OF))
+#define BC   (EFLAGS_WRITE_6|EFLAGS_READ_CF)
+#define BA   (EFLAGS_WRITE_6|EFLAGS_READ_AF)
+#define BD   (EFLAGS_WRITE_6|EFLAGS_READ_CF|EFLAGS_READ_AF)
+#define B6   (EFLAGS_WRITE_6|EFLAGS_READ_6)
+#define EFLAGS_6_ESCAPE  -1
+#define EFLAGS_6_SPECIAL -2
+#define E EFLAGS_6_ESCAPE
+#define S EFLAGS_6_SPECIAL
+
+static const int eflags_6[256] = {
+    W6,W6,W6,W6, W6,W6, x, x, W6,W6,W6,W6, W6,W6, x, E, /* 0 */
+    BC,BC,BC,BC, BC,BC, x, x, BC,BC,BC,BC, BC,BC, x, x, /* 1 */
+    W6,W6,W6,W6, W6,W6, x,BD, W6,W6,W6,W6, W6,W6, x,BD, /* 2 */
+    W6,W6,W6,W6, W6,W6, x,BA, W6,W6,W6,W6, W6,W6, x,BA, /* 3 */
+
+    W5,W5,W5,W5, W5,W5,W5,W5, W5,W5,W5,W5, W5,W5,W5,W5, /* 4 */
+     x, x, x, x,  x, x, x, x,  x, x, x, x,  x, x, x, x, /* 5 */
+     x, x, x,WZ,  x, x, x, x,  x,W6, x,W6,  x, x, x, x, /* 6 */
+    RO,RO,RC,RC, RZ,RZ,RB,RB, RS,RS,RP,RP, RL,RL,RE,RE, /* 7 */
+
+     S, S, S, S, W6,W6, x, x,  x, x, x, x,  x, x, x, x, /* 8 */
+     x, x, x, x,  x, x, x, x,  x, x, x, x, R6,W6,W5O,R5O, /* 9 */
+     x, x, x, x,  x, x,W6,W6, W6,W6, x, x,  x, x,W6,W6, /* A */
+     x, x, x, x,  x, x, x, x,  x, x, x, x,  x, x, x, x, /* B */
+
+     S, S, x, x,  x, x, x, x,  x, x, x, x, R6,R6,R6,W6, /* C */
+     S, S, S, S, W6,W6, x, x,  x, x, S, S,  x, x, x, S, /* D */
+    RZ,RZ, x, x,  x, x, x, x,  x, x, x, x,  x, x, x, x, /* E */
+     x, x, x, x,  x,WC, S, S, WC,WC, x, x,  x, x, S, S, /* F */
+};
+
+/* Same as eflags_6 table, but for 2nd byte of 0x0f extension opcodes
+ */
+static const int escape_eflags_6[256] = {
+     x, x,WZ,WZ,  x, x, x, x,  x, x, x, x,  x, x, x, x, /* 0 */
+     x, x, x, x,  x, x, x, x,  x, x, x, x,  x, x, x, x, /* 1 */
+    W6,W6,W6,W6,  x, x, x, x,  x, x, x, x,  x, x,W6,W6, /* 2 */
+     x, x, x, x,  x, x, x, x,  x, x, x, x,  x, x, x, x, /* 3 */
+
+    RO,RO,RC,RC, RZ,RZ,RB,RB, RS,RS,RP,RP, RL,RL,RE,RE, /* 4 */
+     x, x, x, x,  x, x, x, x,  x, x, x, x,  x, x, x, x, /* 5 */
+     x, x, x, x,  x, x, x, x,  x, x, x, x,  x, x, x, x, /* 6 */
+     x, x, x, x,  x, x, x, x,  x, x, x, x,  x, x, x, x, /* 7 */
+
+    RO,RO,RC,RC, RZ,RZ,RB,RB, RS,RS,RP,RP, RL,RL,RE,RE, /* 8 */
+    RO,RO,RC,RC, RZ,RZ,RB,RB, RS,RS,RP,RP, RL,RL,RE,RE, /* 9 */
+     x, x, x,W6, W6,W6, x, x,  x, x,W6,W6, W6,W6, x,W6, /* A */
+    W6,W6, x,W6,  x, x, x, x,  x, x,W6,W6, W6,W6, x, x, /* B */
+
+    W6,W6, x, x,  x, x, x,WZ,  x, x, x, x,  x, x, x, x, /* C */
+     x, x, x, x,  x, x, x, x,  x, x, x, x,  x, x, x, x, /* D */
+     x, x, x, x,  x, x, x, x,  x, x, x, x,  x, x, x, x, /* E */
+     x, x, x, x,  x, x, x, x,  x, x, x, x,  x, x, x, x, /* F */
+};
+#undef x
+#undef RC
+#undef RP
+#undef RZ
+#undef RS
+#undef RO
+#undef R6
+#undef RB
+#undef RL
+#undef RE
+#undef R5O
+#undef WC
+#undef WZ
+#undef W6
+#undef W5
+#undef W5O
+#undef BC
+#undef BA
+#undef BD
+#undef B6
+#undef E
+#undef S
+
+
+/* This routine converts a signed 8-bit offset into a target pc.  The
+ * formal parameter pc should point to the beginning of the branch
+ * instruction containing the offset and having length len in bytes. 
+ * The x86 architecture calculates offsets from the beginning of the
+ * instruction following the branch. */
+static app_pc
+convert_8bit_offset(byte *pc, byte offset, uint len)
+{
+    return ((app_pc)pc) + (((int)(offset << 24)) >> 24) + len;
+}
+
+/* Decodes only enough of the instruction at address pc to determine
+ * its size, its effects on the 6 arithmetic eflags, and whether it is
+ * a control-transfer instruction.  If it is, the operands fields of
+ * instr are filled in.  If not, only the raw bits fields of instr are
+ * filled in.  This corresponds to a Level 3 decoding for control
+ * transfer instructions but a Level 1 decoding plus arithmetic eflags
+ * information for all other instructions.
+ *
+ * Fills in the PREFIX_SEG_GS and PREFIX_SEG_FS prefix flags for all instrs.
+ * Does NOT fill in any other prefix flags unless this is a cti instr
+ * and the flags affect the instr.
+ *
+ * Assumes that instr is already initialized, but uses the x86/x64 mode
+ * for the current thread rather than that set in instr.
+ * If caller is re-using same instr struct over multiple decodings,
+ * should call instr_reset or instr_reuse.
+ * Returns the address of the byte following the instruction.  
+ * Returns NULL on decoding an invalid instr and sets opcode to OP_INVALID.
+ */
+byte *
+decode_cti(dcontext_t *dcontext, byte *pc, instr_t *instr)
+{
+    byte byte0, byte1;
+    byte *start_pc = pc;
+
+    /* find and remember the instruction and its size */
+    int prefixes;
+    /* next two needed for eflags analysis */
+    int eflags;
+    int i;
+    byte modrm = 0;    /* used only for EFLAGS_6_SPECIAL */
+#ifdef X64
+    /* PR 251479: we need to know about all rip-relative addresses.
+     * Since change/setting raw bits invalidates, we must set this
+     * on every return. */
+    uint rip_rel_pos;
+#endif
+    int sz = decode_sizeof(dcontext, pc, &prefixes _IF_X64(&rip_rel_pos));
+    if (sz == 0) {
+        /* invalid instruction! */
+        instr_set_opcode(instr, OP_INVALID);
+        return NULL;
+    }
+    instr_set_opcode(instr, OP_UNDECODED);
+    IF_X64(instr_set_x86_mode(instr, get_x86_mode(dcontext)));
+
+    /* we call instr_set_raw_bits on every return from here, not up
+     * front, because any instr_set_src, instr_set_dst, or
+     * instr_set_opcode will kill original bits state */
+
+    /* Fill in SEG_FS and SEG_GS override prefixes, ignore rest for now.
+     * We rely on having these set during bb building.
+     * FIXME - could be done in decode_sizeof which is already walking these
+     * bytes, but would need to complicate its interface and prefixes are
+     * fairly rare to begin with. */
+    if (prefixes > 0) {
+        for (i = 0; i < prefixes; i++, pc++) {
+            switch (*pc) {
+            case FS_SEG_OPCODE:
+                instr_set_prefix_flag(instr, PREFIX_SEG_FS);
+                break;
+            case GS_SEG_OPCODE:
+                instr_set_prefix_flag(instr, PREFIX_SEG_GS);
+                break;
+            default:
+                break;
+            }
+        }
+    }
+
+    byte0 = *pc;
+    byte1 = *(pc + 1);
+
+    /* eflags analysis
+     * we do this even if -unsafe_ignore_eflags b/c it doesn't cost that
+     * much and we can use the analysis to detect any bb that reads a flag
+     * prior to writing it
+     */
+    eflags = eflags_6[byte0];
+    if (eflags == EFLAGS_6_ESCAPE) {
+        eflags = escape_eflags_6[byte1];
+        if (eflags == EFLAGS_6_SPECIAL)
+            modrm = *(pc + 2);
+    } else if (eflags == EFLAGS_6_SPECIAL) {
+        modrm = byte1;
+    }
+    if (eflags == EFLAGS_6_SPECIAL) {
+        /* a number of cases exist beyond the ability of 2 tables
+         * to distinguish
+         */
+        int opc_ext = (modrm >> 3) & 7; /* middle 3 bits */
+        if (byte0 <= 0x84) {
+            /* group 1* (80-83): all W6 except /2,/3=B */
+            if (opc_ext == 2 || opc_ext == 3)
+                eflags = EFLAGS_WRITE_6|EFLAGS_READ_CF;
+            else
+                eflags = EFLAGS_WRITE_6;
+        } else if (byte0 <= 0xd3) {
+            /* group 2* (c0,c1,d0-d3): /0,/1=WC|WO, /2,/3=WC|WO|RC, /4,/5,/7=W6 */
+            if (opc_ext == 0 || opc_ext == 1)
+                eflags = EFLAGS_WRITE_CF|EFLAGS_WRITE_OF;
+            else if (opc_ext == 2 || opc_ext == 3)
+                eflags = EFLAGS_WRITE_CF|EFLAGS_WRITE_OF|EFLAGS_READ_CF;
+            else if (opc_ext == 4 || opc_ext == 5 || opc_ext == 7)
+                eflags = EFLAGS_WRITE_6;
+            else
+                eflags = 0;
+        } else if (byte0 <= 0xdf) {
+            /* floats: dac0-dadf and dbc0-dbdf = RC|RP|RZ */
+            if ((byte0 == 0xda || byte0 == 0xdb) &&
+                modrm >= 0xc0 && modrm <= 0xdf)
+                eflags = EFLAGS_READ_CF|EFLAGS_READ_PF|EFLAGS_READ_ZF;
+            /* floats: dbe8-dbf7 and dfe8-dff7 = WC|WP|WZ */
+            else if ((byte0 == 0xdb || byte0 == 0xdf) &&
+                     modrm >= 0xe8 && modrm <= 0xf7)
+                eflags = EFLAGS_WRITE_CF|EFLAGS_WRITE_PF|EFLAGS_WRITE_ZF;
+            else
+                eflags = 0;
+        } else if (byte0 <= 0xf7) {
+            /* group 3a (f6) & 3b (f7): all W except /2 (OP_not) */
+            if (opc_ext == 2)
+                eflags = 0;
+            else
+                eflags = EFLAGS_WRITE_6;
+        } else {
+            /* group 4 (fe) & 5 (ff): /0,/1=W5 */
+            if (opc_ext == 0 || opc_ext == 1)
+                eflags = EFLAGS_WRITE_6 & (~EFLAGS_WRITE_CF);
+            else
+                eflags = 0;
+        }
+    }
+    instr->eflags = eflags;
+    instr_set_arith_flags_valid(instr, true);
+
+    if (interesting[byte0] == 0) {
+        /* assumption: opcode already OP_UNDECODED */
+        /* assumption: operands are already marked invalid (instr was reset) */
+        instr_set_raw_bits(instr, start_pc, sz);
+        IF_X64(instr_set_rip_rel_pos(instr, rip_rel_pos));
+        return (start_pc + sz);
+    }
+
+    /* FIXME: would further "interesting" table produce any noticeable
+     * performance improvement?
+     */
+
+    if (prefixes > 0) {
+        /* prefixes are rare on ctis
+         * rather than handle them all here, just do full decode
+         * FIXME: if we start to see more and more jcc branch hints we
+         * may change our minds here!  This is case 211206/6749.
+         */
+        if (decode(dcontext, start_pc, instr) == NULL)
+            return NULL;
+        else
+            return (start_pc + sz);
+    }
+
+#ifdef FOOL_CPUID
+    /* for fooling program into thinking hardware is different than it is */
+    if (byte0==0x0f && byte1==0xa2) { /* cpuid */
+        instr_set_opcode(instr, OP_cpuid);
+        /* don't bother to set dsts/srcs */
+        instr_set_operands_valid(instr, false);
+        instr_set_raw_bits(instr, start_pc, sz);
+        IF_X64(instr_set_rip_rel_pos(instr, rip_rel_pos));
+        return (start_pc + sz);
+    }
+#endif
+
+    /* prefixes won't make a difference for 8-bit-offset jumps */
+
+    if (byte0 == 0xeb) {                /* jmp_short */
+        app_pc tgt = convert_8bit_offset(pc, byte1, 2);
+        instr_set_opcode(instr, OP_jmp_short);
+        instr_set_num_opnds(dcontext, instr, 0, 1);
+        instr_set_target(instr, opnd_create_pc(tgt));
+        instr_set_raw_bits(instr, start_pc, sz);
+        IF_X64(instr_set_rip_rel_pos(instr, rip_rel_pos));
+        return (pc + 2);
+    }
+
+    if ((byte0 & 0xf0) == 0x70) {       /* jcc_short */
+        /* 2-byte pc-relative jumps with an 8-bit displacement */
+        app_pc tgt = convert_8bit_offset(pc, byte1, 2);
+        /* Set the instr's opcode field.  Relies on special ordering
+         * in opcode enum. */
+        instr_set_opcode(instr, OP_jo_short + (byte0 & 0x0f));
+
+        /* calculate the branch's target address */
+        instr_set_num_opnds(dcontext, instr, 0, 1);
+        instr_set_target(instr, opnd_create_pc(tgt));
+
+        instr_set_raw_bits(instr, start_pc, sz);
+        IF_X64(instr_set_rip_rel_pos(instr, rip_rel_pos));
+        return (pc + 2);
+    }
+
+    if (byte0 == 0xe8) {                /* call */
+        int offset = *((int *)(pc + 1));
+        app_pc tgt = pc + offset + 5;
+        instr_set_opcode(instr, OP_call);
+        instr_set_num_opnds(dcontext, instr, 2, 2);
+        instr_set_target(instr, opnd_create_pc(tgt));
+        instr_set_src(instr, 1, opnd_create_reg(REG_XSP));
+        instr_set_dst(instr, 0, opnd_create_reg(REG_XSP));
+        instr_set_dst(instr, 1, opnd_create_base_disp
+                      (REG_XSP, REG_NULL, 0, 0,
+                       resolve_variable_size_dc(dcontext, 0, OPSZ_call, false)));
+        instr_set_raw_bits(instr, start_pc, sz);
+        IF_X64(instr_set_rip_rel_pos(instr, rip_rel_pos));
+        return (pc + 5);
+    }
+
+    if (byte0 == 0xe9) {                /* jmp */
+        int offset = *((int *)(pc + 1));
+        app_pc tgt = pc + offset + 5;
+        instr_set_opcode(instr, OP_jmp);
+        instr_set_num_opnds(dcontext, instr, 0, 1);
+        instr_set_target(instr, opnd_create_pc(tgt));
+        instr_set_raw_bits(instr, start_pc, sz);
+        IF_X64(instr_set_rip_rel_pos(instr, rip_rel_pos));
+        return (pc + 5);
+    }
+
+    if ((byte0 == 0x0f) && ((byte1 & 0xf0) == 0x80)) { /* jcc */
+        /* 6-byte pc-relative jumps with a 32-bit displacement */
+        /* calculate the branch's target address */
+        int offset = *((int *)(pc + 2));
+        app_pc tgt = pc + offset + 6;
+        /* Set the instr's opcode field.  Relies on special ordering
+         * in opcode enum. */
+        instr_set_opcode(instr, OP_jo + (byte1 & 0x0f));
+
+        instr_set_num_opnds(dcontext, instr, 0, 1);
+        instr_set_target(instr, opnd_create_pc(tgt));
+
+        instr_set_raw_bits(instr, start_pc, sz);
+        IF_X64(instr_set_rip_rel_pos(instr, rip_rel_pos));
+        return (pc + 6);
+    }
+
+    if (byte0 == 0xff) {                /* check for indirect calls/branches */
+        /* dispatch based on bits 5,4,3 in mod_rm byte */
+        uint opc = (byte1 >> 3) & 0x7;
+
+        if (opc >= 2 && opc <= 5) {
+            /* this is an indirect jump or call */
+            /* we care about the operands and prefixes, so just do the full decode
+             */
+            if (decode(dcontext, start_pc, instr) == NULL)
+                return NULL;
+            else
+                return (start_pc + sz);
+        }
+        /* otherwise it wasn't an indirect branch so continue */
+    }
+
+    if ((byte0 & 0xf0) == 0xc0) {       /* check for returns */
+        byte nibble1 = (byte) (byte0 & 0x0f);
+        switch (nibble1) {
+        case 2:   /* ret w/ 2-byte immed */
+        case 0xa: /* far ret w/ 2-byte immed */
+            /* we bailed out to decode() earlier if any prefixes */
+            CLIENT_ASSERT(prefixes == 0, "decode_cti: internal prefix error");
+            instr_set_opcode(instr, nibble1 == 2 ? OP_ret : OP_ret_far);
+            instr_set_num_opnds(dcontext, instr, 1, 3);
+            instr_set_dst(instr, 0, opnd_create_reg(REG_XSP));
+            instr_set_src(instr, 0,
+                          opnd_create_immed_int(*((short*)(pc+1)), OPSZ_2));
+            instr_set_src(instr, 1, opnd_create_reg(REG_XSP));
+            instr_set_src(instr, 2, opnd_create_base_disp
+                          (REG_XSP, REG_NULL, 0, 0,
+                           resolve_variable_size_dc(dcontext, 0,
+                                                    nibble1 == 2 ? OPSZ_ret :
+                                                    OPSZ_REXVARSTACK, false)));
+            instr_set_raw_bits(instr, start_pc, sz);
+            IF_X64(instr_set_rip_rel_pos(instr, rip_rel_pos));
+            return (pc + 3);
+        case 3:         /* ret w/ no immed */
+            instr_set_opcode(instr, OP_ret);
+            instr_set_raw_bits(instr, start_pc, sz);
+            IF_X64(instr_set_rip_rel_pos(instr, rip_rel_pos));
+            /* we don't set any operands and leave to an up-decode for that */
+            return (pc + 1);
+        case 0xb: /* far ret w/ no immed */
+            instr_set_opcode(instr, OP_ret_far);
+            instr_set_raw_bits(instr, start_pc, sz);
+            IF_X64(instr_set_rip_rel_pos(instr, rip_rel_pos));
+            /* we don't set any operands and leave to an up-decode for that */
+            return (pc + 1);
+        }
+        /* otherwise it wasn't a return so continue */
+    }
+
+    if ((byte0 & 0xf0) == 0xe0) {       /* check for a funny 8-bit branch */
+        byte nibble1 = (byte) (byte0 & 0x0f);
+
+        /* determine the opcode */
+        if (nibble1 == 0) {             /* loopne */
+            instr_set_opcode(instr, OP_loopne);
+        } else if (nibble1 == 1) {      /* loope */
+            instr_set_opcode(instr, OP_loope);
+        } else if (nibble1 == 2) {      /* loop */
+            instr_set_opcode(instr, OP_loop);
+        } else if (nibble1 == 3) {      /* jecxz */
+            instr_set_opcode(instr, OP_jecxz);
+        } else if (nibble1 == 10) {     /* jmp_far */
+            /* we need prefix info (data size controls immediate offset size),
+             * this is rare so go ahead and do full decode
+             */
+            if (decode(dcontext, start_pc, instr) == NULL)
+                return NULL;
+            else
+                return (start_pc + sz);
+        }
+        if (instr_opcode_valid(instr)) {
+            /* calculate the branch's target address */
+            app_pc tgt = convert_8bit_offset(pc, byte1, 2);
+            /* all (except jmp far) use ecx as a source */
+            instr_set_num_opnds(dcontext, instr, 0, 2);
+            /* if we made it here, no addr prefix, so REG_XCX not REG_ECX or REG_CX */
+            CLIENT_ASSERT(prefixes == 0, "decoding internal inconsistency");
+            instr_set_src(instr, 1, opnd_create_reg(REG_XCX));
+            instr_set_target(instr, opnd_create_pc(tgt));
+            instr_set_raw_bits(instr, start_pc, sz);
+            IF_X64(instr_set_rip_rel_pos(instr, rip_rel_pos));
+            return (pc + 2);
+        }
+        /* otherwise it wasn't a funny 8-bit cbr so continue */
+    }
+
+    if (byte0 == 0x9a) {        /* check for far-absolute calls */
+        /* we need prefix info, this is rare so we do a full decode
+         */
+        if (decode(dcontext, start_pc, instr) == NULL)
+            return NULL;
+        else
+            return (start_pc + sz);
+    }
+
+    /* both win32 and linux want to know about interrupts */
+    if (byte0 == 0xcd) { /* int */
+        instr_set_opcode(instr, OP_int);
+        instr_set_num_opnds(dcontext, instr, 2, 2);
+        instr_set_dst(instr, 0, opnd_create_reg(REG_XSP));
+        instr_set_dst(instr, 1,
+                      opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_4));
+        instr_set_src(instr, 0, opnd_create_immed_int((char)byte1, OPSZ_1));
+        instr_set_src(instr, 1, opnd_create_reg(REG_XSP));
+        instr_set_raw_bits(instr, start_pc, sz);
+        IF_X64(instr_set_rip_rel_pos(instr, rip_rel_pos));
+        return (pc + 2);
+    }
+    /* sys{enter,exit,call,ret} */
+    if (byte0 == 0x0f &&
+        (byte1 == 0x34 || byte1 == 0x35 || byte1 == 0x05 || byte1 == 0x07)) {
+        if (byte1 == 0x34) {
+            instr_set_opcode(instr, OP_sysenter);
+            instr_set_num_opnds(dcontext, instr, 1, 0);
+            instr_set_dst(instr, 0, opnd_create_reg(REG_XSP));
+        } else if (byte1 == 0x34) {
+            instr_set_opcode(instr, OP_sysexit);
+            instr_set_num_opnds(dcontext, instr, 1, 0);
+            instr_set_dst(instr, 0, opnd_create_reg(REG_XSP));
+        } else if (byte1 == 0x05) {
+            instr_set_opcode(instr, OP_syscall);
+            instr_set_num_opnds(dcontext, instr, 1, 0);
+            instr_set_dst(instr, 0, opnd_create_reg(REG_XCX));
+        } else if (byte1 == 0x07) {
+            instr_set_opcode(instr, OP_sysret);
+            instr_set_num_opnds(dcontext, instr, 0, 0);
+        }
+        instr_set_raw_bits(instr, start_pc, sz);
+        IF_X64(instr_set_rip_rel_pos(instr, rip_rel_pos));
+        return (pc + 2);
+    }
+    /* iret */
+    if (byte0 == 0xcf) {
+        instr_set_opcode(instr, OP_iret);
+        instr_set_raw_bits(instr, start_pc, sz);
+        IF_X64(instr_set_rip_rel_pos(instr, rip_rel_pos));
+        return (pc + 1);
+    }
+
+    /* all non-pc-relative instructions */
+    /* assumption: opcode already OP_UNDECODED */
+    instr_set_raw_bits(instr, start_pc, sz);
+    IF_X64(instr_set_rip_rel_pos(instr, rip_rel_pos));
+    /* assumption: operands are already marked invalid (instr was reset) */
+    return (start_pc + sz);
+}
+
+/* Returns a pointer to the pc of the next instruction
+ * Returns NULL on decoding an invalid instruction.
+ */
+byte *
+decode_next_pc(dcontext_t *dcontext, byte *pc)
+{
+    int sz = decode_sizeof(dcontext, pc, NULL _IF_X64(NULL));
+    if (sz == 0)
+        return NULL;
+    else
+        return pc + sz;
+}
+
+/* Decodes the size of the instruction at address pc and points instr
+ * at the raw bits for the instruction.
+ * This corresponds to a Level 1 decoding.
+ * Assumes that instr is already initialized, but uses the x86/x64 mode
+ * for the current thread rather than that set in instr.
+ * If caller is re-using same instr struct over multiple decodings,
+ * should call instr_reset or instr_reuse.
+ * Returns the address of the next byte after the decoded instruction.
+ * Returns NULL on decoding an invalid instr and sets opcode to OP_INVALID.
+ */
+byte *
+decode_raw(dcontext_t *dcontext, byte *pc, instr_t *instr)
+{
+    int sz = decode_sizeof(dcontext, pc, NULL _IF_X64(NULL));
+    IF_X64(instr_set_x86_mode(instr, get_x86_mode(dcontext)));
+    if (sz == 0) {
+        /* invalid instruction! */
+        instr_set_opcode(instr, OP_INVALID);
+        return NULL;
+    }
+    instr_set_opcode(instr, OP_UNDECODED);
+    instr_set_raw_bits(instr, pc, sz);
+    /* assumption: operands are already marked invalid (instr was reset) */
+    return (pc + sz);
+}
+
diff --git a/core/x86/decode_fast.h b/core/x86/decode_fast.h
new file mode 100644
index 000000000..5c4323e1a
--- /dev/null
+++ b/core/x86/decode_fast.h
@@ -0,0 +1,123 @@
+/* **********************************************************
+ * Copyright (c) 2001-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2001 Hewlett-Packard Company */
+
+/* file "decode_fast.h" */
+
+#ifndef DECODE_FAST_H
+#define DECODE_FAST_H
+
+/* DR_API EXPORT TOFILE dr_ir_utils.h */
+DR_API
+/** 
+ * Decodes only enough of the instruction at address \p pc to determine its size.
+ * Returns that size.
+ * If \p num_prefixes is non-NULL, returns the number of prefix bytes.
+ * If \p rip_rel_pos is non-NULL, returns the offset into the instruction
+ * of a rip-relative addressing displacement (for data only: ignores
+ * control-transfer relative addressing), or 0 if none.
+ * May return 0 size for certain invalid instructions.
+ */
+int 
+decode_sizeof(dcontext_t *dcontext, byte *pc, int *num_prefixes
+              _IF_X64(uint *rip_rel_pos));
+
+DR_API
+/** 
+ * Decodes only enough of the instruction at address \p pc to determine its size.
+ * Returns the address of the byte following the instruction.
+ * Returns NULL on decoding an invalid instruction.
+ */
+#ifdef UNSUPPORTED_API
+/**
+ * This corresponds to a Level 1 decoding.
+ */
+#endif
+byte *
+decode_next_pc(dcontext_t *dcontext, byte *pc);
+
+DR_UNS_API
+/** 
+ * Decodes the size of the instruction at address \p pc.
+ * The instruction's raw bits are set to valid and pointed at \p pc
+ * (xref instr_get_raw_bits()).
+ * Assumes that \p instr is already initialized, but uses the x86/x64 mode
+ * for the thread \p dcontext rather than that set in instr.
+ * If caller is re-using same instr struct over multiple decodings,
+ * caller should call instr_reset() or instr_reuse().
+ * Returns the address of the next byte after the decoded instruction.
+ * Returns NULL on decoding an invalid instr and sets opcode to OP_INVALID.
+ */
+#ifdef UNSUPPORTED_API
+/* This corresponds to a Level 1 decoding. */
+#endif
+byte *
+decode_raw(dcontext_t *dcontext, byte *pc, instr_t *instr);
+
+DR_UNS_API
+/** 
+ * Decodes only enough of the instruction at address \p pc to determine
+ * its size, its effects on the 6 arithmetic eflags, and whether it is
+ * a control-transfer instruction.  If it is, the operands fields of
+ * \p instr are filled in.  If not, only the raw bits fields of \p instr are
+ * filled in. 
+ *
+ * Fills in the PREFIX_SEG_GS and PREFIX_SEG_FS prefix flags for all instrs.
+ * Does NOT fill in any other prefix flags unless this is a cti instr
+ * and the flags affect the instr.
+ *
+ * For x64, calls instr_set_rip_rel_pos().  Thus, if the raw bytes are
+ * not modified prior to encode time, any rip-relative offset will be
+ * automatically re-relativized (though encoding will fail if the new
+ * encode location cannot reach the original target).
+ *
+ * Assumes that \p instr is already initialized, but uses the x86/x64 mode
+ * for the thread \p dcontext rather than that set in instr.
+ * If caller is re-using same instr struct over multiple decodings,
+ * caller should call instr_reset() or instr_reuse().
+ * Returns the address of the byte following the instruction.  
+ * Returns NULL on decoding an invalid instr and sets opcode to OP_INVALID.
+ */
+#ifdef UNSUPPORTED_API
+/* This corresponds to a Level 3 decoding for control transfer
+ * instructions but a Level 1 decoding plus arithmetic eflags
+ * information for all other instructions.
+ */
+#endif
+byte *
+decode_cti(dcontext_t *dcontext, byte *pc, instr_t *instr);
+
+
+#endif /* DECODE_FAST_H */
diff --git a/core/x86/decode_table.c b/core/x86/decode_table.c
new file mode 100644
index 000000000..89d991132
--- /dev/null
+++ b/core/x86/decode_table.c
@@ -0,0 +1,4079 @@
+/* **********************************************************
+ * Copyright (c) 2001-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2001 Hewlett-Packard Company */
+
+/* decode_table.c -- tables for decoding x86 instructions
+ */
+
+#include "globals.h" /* need this to include decode.h (uint, etc.) */
+#include "arch.h"    /* need this to include decode.h (byte, etc. */
+#include "instr.h" /* for REG_ constants */
+#include "decode.h"
+
+/****************************************************************************
+ * All code below based on tables in the ``Intel Architecture Software
+ * Developer's Manual,'' Volume 2: Instruction Set Reference, 2001.
+ * Updated with information from later Intel manuals and AMD manuals.
+ * 
+ * I added many new types not present in the Intel tables: see decode.h
+ *
+ * I don't list %eflags as a source or dest operand, but the particular
+ * flags written are encoded.
+ */
+
+/****************************************************************************
+ * Operand pointers into tables
+ * When there are multiple encodings of an opcode, this points to the first
+ * entry in a linked list.
+ * This array corresponds with the enum in instr.h
+ * IF YOU CHANGE ONE YOU MUST CHANGE THE OTHER
+ */
+const instr_info_t * const op_instr[] =
+{
+    /* OP_INVALID */   NULL,
+    /* OP_UNDECODED */ NULL,
+    /* OP_CONTD   */   NULL,
+    /* OP_LABEL   */   NULL,
+
+    /* OP_add     */   &first_byte[0x05],
+    /* OP_or      */   &first_byte[0x0d],
+    /* OP_adc     */   &first_byte[0x15],
+    /* OP_sbb     */   &first_byte[0x1d],
+    /* OP_and     */   &first_byte[0x25],
+    /* OP_daa     */   &first_byte[0x27],
+    /* OP_sub     */   &first_byte[0x2d],
+    /* OP_das     */   &first_byte[0x2f],
+    /* OP_xor     */   &first_byte[0x35],
+    /* OP_aaa     */   &first_byte[0x37],
+    /* OP_cmp     */   &first_byte[0x3d],
+    /* OP_aas     */   &first_byte[0x3f],
+    /* OP_inc     */   &x64_extensions[0][0],
+    /* OP_dec     */   &x64_extensions[8][0],
+    /* OP_push    */   &first_byte[0x50],
+    /* OP_push_imm*/   &first_byte[0x68],
+    /* OP_pop     */   &first_byte[0x58],
+    /* OP_pusha   */   &first_byte[0x60],
+    /* OP_popa    */   &first_byte[0x61],
+    /* OP_bound   */   &first_byte[0x62],
+    /* OP_arpl    */   &x64_extensions[16][0],
+    /* OP_imul    */   &extensions[10][5],
+
+    /* OP_jo_short    */   &first_byte[0x70],
+    /* OP_jno_short   */   &first_byte[0x71],
+    /* OP_jb_short    */   &first_byte[0x72],
+    /* OP_jnb_short   */   &first_byte[0x73],
+    /* OP_jz_short    */   &first_byte[0x74],
+    /* OP_jnz_short   */   &first_byte[0x75],
+    /* OP_jbe_short   */   &first_byte[0x76],
+    /* OP_jnbe_short  */   &first_byte[0x77],
+    /* OP_js_short    */   &first_byte[0x78],
+    /* OP_jns_short   */   &first_byte[0x79],
+    /* OP_jp_short    */   &first_byte[0x7a],
+    /* OP_jnp_short   */   &first_byte[0x7b],
+    /* OP_jl_short    */   &first_byte[0x7c],
+    /* OP_jnl_short   */   &first_byte[0x7d],
+    /* OP_jle_short   */   &first_byte[0x7e],
+    /* OP_jnle_short  */   &first_byte[0x7f],
+
+    /* OP_call          */   &first_byte[0xe8],
+    /* OP_call_ind      */   &extensions[12][2],
+    /* OP_call_far      */   &first_byte[0x9a],
+    /* OP_call_far_ind  */   &extensions[12][3],
+    /* OP_jmp           */   &first_byte[0xe9],
+    /* OP_jmp_short     */   &first_byte[0xeb],
+    /* OP_jmp_ind       */   &extensions[12][4],
+    /* OP_jmp_far       */   &first_byte[0xea],
+    /* OP_jmp_far_ind   */   &extensions[12][5],
+
+    /* OP_loopne  */   &first_byte[0xe0],
+    /* OP_loope   */   &first_byte[0xe1],
+    /* OP_loop    */   &first_byte[0xe2],
+    /* OP_jecxz   */   &first_byte[0xe3],
+
+    /* point ld & st at eAX & al instrs, they save 1 byte (no modrm),
+     * hopefully time taken considering them doesn't offset that */
+    /* OP_mov_ld     */   &first_byte[0xa1],
+    /* OP_mov_st     */   &first_byte[0xa3],
+    /* PR 250397: store of immed is mov_st not mov_imm, even though can be immed->reg,
+     * which we address by sharing part of the mov_st template chain */
+    /* OP_mov_imm    */   &first_byte[0xb8],
+    /* OP_mov_seg    */   &first_byte[0x8e],
+    /* OP_mov_priv   */   &second_byte[0x20],
+
+    /* OP_test    */   &first_byte[0xa9],
+    /* OP_lea     */   &first_byte[0x8d],
+    /* OP_xchg    */   &first_byte[0x91],
+    /* OP_cwde    */   &first_byte[0x98],
+    /* OP_cdq     */   &first_byte[0x99],
+    /* OP_fwait   */   &first_byte[0x9b],
+    /* OP_pushf   */   &first_byte[0x9c],
+    /* OP_popf    */   &first_byte[0x9d],
+    /* OP_sahf    */   &first_byte[0x9e],
+    /* OP_lahf    */   &first_byte[0x9f],
+
+    /* OP_ret      */   &first_byte[0xc2],
+    /* OP_ret_far  */   &first_byte[0xca],
+
+    /* OP_les     */   &first_byte[0xc4],
+    /* OP_lds     */   &first_byte[0xc5],
+    /* OP_enter   */   &first_byte[0xc8],
+    /* OP_leave   */   &first_byte[0xc9],
+    /* OP_int3    */   &first_byte[0xcc],
+    /* OP_int     */   &first_byte[0xcd],
+    /* OP_into    */   &first_byte[0xce],
+    /* OP_iret    */   &first_byte[0xcf],
+    /* OP_aam     */   &first_byte[0xd4],
+    /* OP_aad     */   &first_byte[0xd5],
+    /* OP_xlat    */   &first_byte[0xd7],
+    /* OP_in      */   &first_byte[0xe5],
+    /* OP_out     */   &first_byte[0xe7],
+    /* OP_hlt     */   &first_byte[0xf4],
+    /* OP_cmc     */   &first_byte[0xf5],
+    /* OP_clc     */   &first_byte[0xf8],
+    /* OP_stc     */   &first_byte[0xf9],
+    /* OP_cli     */   &first_byte[0xfa],
+    /* OP_sti     */   &first_byte[0xfb],
+    /* OP_cld     */   &first_byte[0xfc],
+    /* OP_std     */   &first_byte[0xfd],
+
+
+    /* OP_lar         */   &second_byte[0x02],
+    /* OP_lsl         */   &second_byte[0x03],
+    /* OP_syscall     */   &second_byte[0x05],
+    /* OP_clts        */   &second_byte[0x06],
+    /* OP_sysret      */   &second_byte[0x07],
+    /* OP_invd        */   &second_byte[0x08],
+    /* OP_wbinvd      */   &second_byte[0x09],
+    /* OP_ud2a        */   &second_byte[0x0b],
+    /* OP_nop_modrm   */   &second_byte[0x1f],
+    /* OP_movntps     */   &prefix_extensions[11][0],
+    /* OP_movntpd     */   &prefix_extensions[11][2],
+    /* OP_wrmsr       */   &second_byte[0x30],
+    /* OP_rdtsc       */   &second_byte[0x31],
+    /* OP_rdmsr       */   &second_byte[0x32],
+    /* OP_rdpmc       */   &second_byte[0x33],
+    /* OP_sysenter    */   &second_byte[0x34],
+    /* OP_sysexit     */   &second_byte[0x35],
+
+    /* OP_cmovo       */   &second_byte[0x40],
+    /* OP_cmovno      */   &second_byte[0x41],
+    /* OP_cmovb       */   &second_byte[0x42],
+    /* OP_cmovnb      */   &second_byte[0x43],
+    /* OP_cmovz       */   &second_byte[0x44],
+    /* OP_cmovnz      */   &second_byte[0x45],
+    /* OP_cmovbe      */   &second_byte[0x46],
+    /* OP_cmovnbe     */   &second_byte[0x47],
+    /* OP_cmovs       */   &second_byte[0x48],
+    /* OP_cmovns      */   &second_byte[0x49],
+    /* OP_cmovp       */   &second_byte[0x4a],
+    /* OP_cmovnp      */   &second_byte[0x4b],
+    /* OP_cmovl       */   &second_byte[0x4c],
+    /* OP_cmovnl      */   &second_byte[0x4d],
+    /* OP_cmovle      */   &second_byte[0x4e],
+    /* OP_cmovnle     */   &second_byte[0x4f],
+
+    /* OP_punpcklbw   */   &prefix_extensions[32][0],
+    /* OP_punpcklwd   */   &prefix_extensions[33][0],
+    /* OP_punpckldq   */   &prefix_extensions[34][0],
+    /* OP_packsswb    */   &prefix_extensions[35][0],
+    /* OP_pcmpgtb     */   &prefix_extensions[36][0],
+    /* OP_pcmpgtw     */   &prefix_extensions[37][0],
+    /* OP_pcmpgtd     */   &prefix_extensions[38][0],
+    /* OP_packuswb    */   &prefix_extensions[39][0],
+    /* OP_punpckhbw   */   &prefix_extensions[40][0],
+    /* OP_punpckhwd   */   &prefix_extensions[41][0],
+    /* OP_punpckhdq   */   &prefix_extensions[42][0],
+    /* OP_packssdw    */   &prefix_extensions[43][0],
+    /* OP_punpcklqdq  */   &prefix_extensions[44][2],
+    /* OP_punpckhqdq  */   &prefix_extensions[45][2],
+    /* OP_movd        */   &prefix_extensions[46][0],
+    /* OP_movq        */   &prefix_extensions[112][0],
+    /* OP_movdqu      */   &prefix_extensions[112][1], 
+    /* OP_movdqa      */   &prefix_extensions[112][2],
+    /* OP_pshufw      */   &prefix_extensions[47][0],
+    /* OP_pshufd      */   &prefix_extensions[47][2],
+    /* OP_pshufhw     */   &prefix_extensions[47][1],
+    /* OP_pshuflw     */   &prefix_extensions[47][3],
+    /* OP_pcmpeqb     */   &prefix_extensions[48][0],
+    /* OP_pcmpeqw     */   &prefix_extensions[49][0],
+    /* OP_pcmpeqd     */   &prefix_extensions[50][0],
+    /* OP_emms        */   &second_byte[0x77],
+
+    /* OP_jo      */   &second_byte[0x80],
+    /* OP_jno     */   &second_byte[0x81],
+    /* OP_jb      */   &second_byte[0x82],
+    /* OP_jnb     */   &second_byte[0x83],
+    /* OP_jz      */   &second_byte[0x84],
+    /* OP_jnz     */   &second_byte[0x85],
+    /* OP_jbe     */   &second_byte[0x86],
+    /* OP_jnbe    */   &second_byte[0x87],
+    /* OP_js      */   &second_byte[0x88],
+    /* OP_jns     */   &second_byte[0x89],
+    /* OP_jp      */   &second_byte[0x8a],
+    /* OP_jnp     */   &second_byte[0x8b],
+    /* OP_jl      */   &second_byte[0x8c],
+    /* OP_jnl     */   &second_byte[0x8d],
+    /* OP_jle     */   &second_byte[0x8e],
+    /* OP_jnle    */   &second_byte[0x8f],
+
+    /* OP_seto        */   &second_byte[0x90],
+    /* OP_setno       */   &second_byte[0x91],
+    /* OP_setb        */   &second_byte[0x92],
+    /* OP_setnb       */   &second_byte[0x93],
+    /* OP_setz        */   &second_byte[0x94],
+    /* OP_setnz       */   &second_byte[0x95],
+    /* OP_setbe       */   &second_byte[0x96],
+    /* OP_setnbe        */   &second_byte[0x97],
+    /* OP_sets        */   &second_byte[0x98],
+    /* OP_setns       */   &second_byte[0x99],
+    /* OP_setp        */   &second_byte[0x9a],
+    /* OP_setnp       */   &second_byte[0x9b],
+    /* OP_setl        */   &second_byte[0x9c],
+    /* OP_setnl       */   &second_byte[0x9d],
+    /* OP_setle       */   &second_byte[0x9e],
+    /* OP_setnle        */   &second_byte[0x9f],
+
+    /* OP_cpuid       */   &second_byte[0xa2],
+    /* OP_bt          */   &second_byte[0xa3],
+    /* OP_shld        */   &second_byte[0xa4],
+    /* OP_rsm         */   &second_byte[0xaa],
+    /* OP_bts         */   &second_byte[0xab],
+    /* OP_shrd        */   &second_byte[0xac],
+    /* OP_cmpxchg     */   &second_byte[0xb1],
+    /* OP_lss         */   &second_byte[0xb2],
+    /* OP_btr         */   &second_byte[0xb3],
+    /* OP_lfs         */   &second_byte[0xb4],
+    /* OP_lgs         */   &second_byte[0xb5],
+    /* OP_movzx       */   &second_byte[0xb7],
+    /* OP_ud2b        */   &second_byte[0xb9],
+    /* OP_btc         */   &second_byte[0xbb],
+    /* OP_bsf         */   &second_byte[0xbc],
+    /* OP_bsr         */   &prefix_extensions[136][0],
+    /* OP_movsx       */   &second_byte[0xbf],
+    /* OP_xadd        */   &second_byte[0xc1],
+    /* OP_movnti      */   &second_byte[0xc3],
+    /* OP_pinsrw      */   &prefix_extensions[53][0],
+    /* OP_pextrw      */   &prefix_extensions[54][0],
+    /* OP_bswap       */   &second_byte[0xc8],
+    /* OP_psrlw       */   &prefix_extensions[56][0],
+    /* OP_psrld       */   &prefix_extensions[57][0],
+    /* OP_psrlq       */   &prefix_extensions[58][0],
+    /* OP_paddq       */   &prefix_extensions[59][0],
+    /* OP_pmullw      */   &prefix_extensions[60][0],
+    /* OP_pmovmskb    */   &prefix_extensions[62][0],
+    /* OP_psubusb     */   &prefix_extensions[63][0],
+    /* OP_psubusw     */   &prefix_extensions[64][0],
+    /* OP_pminub      */   &prefix_extensions[65][0],
+    /* OP_pand        */   &prefix_extensions[66][0],
+    /* OP_paddusb     */   &prefix_extensions[67][0],
+    /* OP_paddusw     */   &prefix_extensions[68][0],
+    /* OP_pmaxub      */   &prefix_extensions[69][0],
+    /* OP_pandn       */   &prefix_extensions[70][0],
+    /* OP_pavgb       */   &prefix_extensions[71][0],
+    /* OP_psraw       */   &prefix_extensions[72][0],
+    /* OP_psrad       */   &prefix_extensions[73][0],
+    /* OP_pavgw       */   &prefix_extensions[74][0],
+    /* OP_pmulhuw     */   &prefix_extensions[75][0],
+    /* OP_pmulhw      */   &prefix_extensions[76][0],
+    /* OP_movntq      */   &prefix_extensions[78][0],
+    /* OP_movntdq     */   &prefix_extensions[78][2],
+    /* OP_psubsb      */   &prefix_extensions[79][0],
+    /* OP_psubsw      */   &prefix_extensions[80][0],
+    /* OP_pminsw      */   &prefix_extensions[81][0],
+    /* OP_por         */   &prefix_extensions[82][0],
+    /* OP_paddsb      */   &prefix_extensions[83][0],
+    /* OP_paddsw      */   &prefix_extensions[84][0],
+    /* OP_pmaxsw      */   &prefix_extensions[85][0],
+    /* OP_pxor        */   &prefix_extensions[86][0],
+    /* OP_psllw       */   &prefix_extensions[87][0],
+    /* OP_pslld       */   &prefix_extensions[88][0],
+    /* OP_psllq       */   &prefix_extensions[89][0],
+    /* OP_pmuludq     */   &prefix_extensions[90][0],
+    /* OP_pmaddwd     */   &prefix_extensions[91][0],
+    /* OP_psadbw      */   &prefix_extensions[92][0],
+    /* OP_maskmovq    */   &prefix_extensions[93][0],
+    /* OP_maskmovdqu  */   &prefix_extensions[93][2],
+    /* OP_psubb       */   &prefix_extensions[94][0],
+    /* OP_psubw       */   &prefix_extensions[95][0],
+    /* OP_psubd       */   &prefix_extensions[96][0],
+    /* OP_psubq       */   &prefix_extensions[97][0],
+    /* OP_paddb       */   &prefix_extensions[98][0],
+    /* OP_paddw       */   &prefix_extensions[99][0],
+    /* OP_paddd       */   &prefix_extensions[100][0],
+    /* OP_psrldq      */   &prefix_extensions[101][2],
+    /* OP_pslldq      */   &prefix_extensions[102][2],
+
+
+    /* OP_rol          */   &extensions[ 4][0],
+    /* OP_ror          */   &extensions[ 4][1],
+    /* OP_rcl          */   &extensions[ 4][2],
+    /* OP_rcr          */   &extensions[ 4][3],
+    /* OP_shl          */   &extensions[ 4][4],
+    /* OP_shr          */   &extensions[ 4][5],
+    /* OP_sar          */   &extensions[ 4][7],
+    /* OP_not          */   &extensions[10][2],
+    /* OP_neg          */   &extensions[10][3],
+    /* OP_mul          */   &extensions[10][4],
+    /* OP_div          */   &extensions[10][6],
+    /* OP_idiv         */   &extensions[10][7],
+    /* OP_sldt         */   &extensions[13][0],
+    /* OP_str          */   &extensions[13][1],
+    /* OP_lldt         */   &extensions[13][2],
+    /* OP_ltr          */   &extensions[13][3],
+    /* OP_verr         */   &extensions[13][4],
+    /* OP_verw         */   &extensions[13][5],
+    /* OP_sgdt         */   &mod_extensions[0][0],
+    /* OP_sidt         */   &mod_extensions[1][0],
+    /* OP_lgdt         */   &extensions[14][2],
+    /* OP_lidt         */   &extensions[14][3],
+    /* OP_smsw         */   &extensions[14][4],
+    /* OP_lmsw         */   &extensions[14][6],
+    /* OP_invlpg       */   &mod_extensions[2][0],
+    /* OP_cmpxchg8b    */   &extensions[16][1],
+    /* OP_fxsave       */   &extensions[22][0],
+    /* OP_fxrstor      */   &extensions[22][1],
+    /* OP_ldmxcsr      */   &extensions[22][2],
+    /* OP_stmxcsr      */   &extensions[22][3],
+    /* OP_lfence       */   &extensions[22][5],
+    /* OP_mfence       */   &extensions[22][6],
+    /* OP_clflush      */   &mod_extensions[3][0],
+    /* OP_sfence       */   &mod_extensions[3][1],
+    /* OP_prefetchnta  */   &extensions[23][0],
+    /* OP_prefetcht0   */   &extensions[23][1],
+    /* OP_prefetcht1   */   &extensions[23][2],
+    /* OP_prefetcht2   */   &extensions[23][3],
+    /* OP_prefetch     */   &extensions[24][0],
+    /* OP_prefetchw    */   &extensions[24][1],
+
+
+    /* OP_movups     */   &prefix_extensions[ 0][0],
+    /* OP_movss      */   &prefix_extensions[ 0][1],
+    /* OP_movupd     */   &prefix_extensions[ 0][2],
+    /* OP_movsd      */   &prefix_extensions[ 0][3],
+    /* OP_movlps     */   &prefix_extensions[ 2][0],
+    /* OP_movlpd     */   &prefix_extensions[ 2][2],
+    /* OP_unpcklps   */   &prefix_extensions[ 4][0],
+    /* OP_unpcklpd   */   &prefix_extensions[ 4][2],
+    /* OP_unpckhps   */   &prefix_extensions[ 5][0],
+    /* OP_unpckhpd   */   &prefix_extensions[ 5][2],
+    /* OP_movhps     */   &prefix_extensions[ 6][0],
+    /* OP_movhpd     */   &prefix_extensions[ 6][2],
+    /* OP_movaps     */   &prefix_extensions[ 8][0],
+    /* OP_movapd     */   &prefix_extensions[ 8][2],
+    /* OP_cvtpi2ps   */   &prefix_extensions[10][0],
+    /* OP_cvtsi2ss   */   &prefix_extensions[10][1],
+    /* OP_cvtpi2pd   */   &prefix_extensions[10][2],
+    /* OP_cvtsi2sd   */   &prefix_extensions[10][3],
+    /* OP_cvttps2pi  */   &prefix_extensions[12][0],
+    /* OP_cvttss2si  */   &prefix_extensions[12][1],
+    /* OP_cvttpd2pi  */   &prefix_extensions[12][2],
+    /* OP_cvttsd2si  */   &prefix_extensions[12][3],
+    /* OP_cvtps2pi   */   &prefix_extensions[13][0],
+    /* OP_cvtss2si   */   &prefix_extensions[13][1],
+    /* OP_cvtpd2pi   */   &prefix_extensions[13][2],
+    /* OP_cvtsd2si   */   &prefix_extensions[13][3],
+    /* OP_ucomiss    */   &prefix_extensions[14][0],
+    /* OP_ucomisd    */   &prefix_extensions[14][2],
+    /* OP_comiss     */   &prefix_extensions[15][0],
+    /* OP_comisd     */   &prefix_extensions[15][2],
+    /* OP_movmskps   */   &prefix_extensions[16][0],
+    /* OP_movmskpd   */   &prefix_extensions[16][2],
+    /* OP_sqrtps     */   &prefix_extensions[17][0],
+    /* OP_sqrtss     */   &prefix_extensions[17][1],
+    /* OP_sqrtpd     */   &prefix_extensions[17][2],
+    /* OP_sqrtsd     */   &prefix_extensions[17][3],
+    /* OP_rsqrtps    */   &prefix_extensions[18][0],
+    /* OP_rsqrtss    */   &prefix_extensions[18][1],
+    /* OP_rcpps      */   &prefix_extensions[19][0],
+    /* OP_rcpss      */   &prefix_extensions[19][1],
+    /* OP_andps      */   &prefix_extensions[20][0],
+    /* OP_andpd      */   &prefix_extensions[20][2],
+    /* OP_andnps     */   &prefix_extensions[21][0],
+    /* OP_andnpd     */   &prefix_extensions[21][2],
+    /* OP_orps       */   &prefix_extensions[22][0],
+    /* OP_orpd       */   &prefix_extensions[22][2],
+    /* OP_xorps      */   &prefix_extensions[23][0],
+    /* OP_xorpd      */   &prefix_extensions[23][2],
+    /* OP_addps      */   &prefix_extensions[24][0],
+    /* OP_addss      */   &prefix_extensions[24][1],
+    /* OP_addpd      */   &prefix_extensions[24][2],
+    /* OP_addsd      */   &prefix_extensions[24][3],
+    /* OP_mulps      */   &prefix_extensions[25][0],
+    /* OP_mulss      */   &prefix_extensions[25][1],
+    /* OP_mulpd      */   &prefix_extensions[25][2],
+    /* OP_mulsd      */   &prefix_extensions[25][3],
+    /* OP_cvtps2pd   */   &prefix_extensions[26][0],
+    /* OP_cvtss2sd   */   &prefix_extensions[26][1],
+    /* OP_cvtpd2ps   */   &prefix_extensions[26][2],
+    /* OP_cvtsd2ss   */   &prefix_extensions[26][3],
+    /* OP_cvtdq2ps   */   &prefix_extensions[27][0],
+    /* OP_cvttps2dq  */   &prefix_extensions[27][1],
+    /* OP_cvtps2dq   */   &prefix_extensions[27][2],
+    /* OP_subps      */   &prefix_extensions[28][0],
+    /* OP_subss      */   &prefix_extensions[28][1],
+    /* OP_subpd      */   &prefix_extensions[28][2],
+    /* OP_subsd      */   &prefix_extensions[28][3],
+    /* OP_minps      */   &prefix_extensions[29][0],
+    /* OP_minss      */   &prefix_extensions[29][1],
+    /* OP_minpd      */   &prefix_extensions[29][2],
+    /* OP_minsd      */   &prefix_extensions[29][3],
+    /* OP_divps      */   &prefix_extensions[30][0],
+    /* OP_divss      */   &prefix_extensions[30][1],
+    /* OP_divpd      */   &prefix_extensions[30][2],
+    /* OP_divsd      */   &prefix_extensions[30][3],
+    /* OP_maxps      */   &prefix_extensions[31][0],
+    /* OP_maxss      */   &prefix_extensions[31][1],
+    /* OP_maxpd      */   &prefix_extensions[31][2],
+    /* OP_maxsd      */   &prefix_extensions[31][3],
+    /* OP_cmpps      */   &prefix_extensions[52][0],
+    /* OP_cmpss      */   &prefix_extensions[52][1],
+    /* OP_cmppd      */   &prefix_extensions[52][2],
+    /* OP_cmpsd      */   &prefix_extensions[52][3],
+    /* OP_shufps     */   &prefix_extensions[55][0],
+    /* OP_shufpd     */   &prefix_extensions[55][2],
+    /* OP_cvtdq2pd   */   &prefix_extensions[77][1],
+    /* OP_cvttpd2dq  */   &prefix_extensions[77][2],
+    /* OP_cvtpd2dq   */   &prefix_extensions[77][3],
+    /* OP_nop        */   &rex_extensions[0][0],
+    /* OP_pause      */   &prefix_extensions[103][1],
+
+    /* OP_ins         */   &rep_extensions[1][0],
+    /* OP_rep_ins     */   &rep_extensions[1][2],
+    /* OP_outs        */   &rep_extensions[3][0],
+    /* OP_rep_outs    */   &rep_extensions[3][2],
+    /* OP_movs        */   &rep_extensions[5][0],
+    /* OP_rep_movs    */   &rep_extensions[5][2],
+    /* OP_stos        */   &rep_extensions[7][0],
+    /* OP_rep_stos    */   &rep_extensions[7][2],
+    /* OP_lods        */   &rep_extensions[9][0],
+    /* OP_rep_lods    */   &rep_extensions[9][2],
+    /* OP_cmps        */   &repne_extensions[1][0],
+    /* OP_rep_cmps    */   &repne_extensions[1][2],
+    /* OP_repne_cmps  */   &repne_extensions[1][4],
+    /* OP_scas        */   &repne_extensions[3][0],
+    /* OP_rep_scas    */   &repne_extensions[3][2],
+    /* OP_repne_scas  */   &repne_extensions[3][4],
+
+
+    /* OP_fadd     */   &float_low_modrm[0x00],
+    /* OP_fmul     */   &float_low_modrm[0x01],
+    /* OP_fcom     */   &float_low_modrm[0x02],
+    /* OP_fcomp    */   &float_low_modrm[0x03],
+    /* OP_fsub     */   &float_low_modrm[0x04],
+    /* OP_fsubr    */   &float_low_modrm[0x05],
+    /* OP_fdiv     */   &float_low_modrm[0x06],
+    /* OP_fdivr    */   &float_low_modrm[0x07],
+    /* OP_fld      */   &float_low_modrm[0x08],
+    /* OP_fst      */   &float_low_modrm[0x0a],
+    /* OP_fstp     */   &float_low_modrm[0x0b],
+    /* OP_fldenv   */   &float_low_modrm[0x0c],
+    /* OP_fldcw    */   &float_low_modrm[0x0d],
+    /* OP_fnstenv  */   &float_low_modrm[0x0e],
+    /* OP_fnstcw   */   &float_low_modrm[0x0f],
+    /* OP_fiadd    */   &float_low_modrm[0x10],
+    /* OP_fimul    */   &float_low_modrm[0x11],
+    /* OP_ficom    */   &float_low_modrm[0x12],
+    /* OP_ficomp   */   &float_low_modrm[0x13],
+    /* OP_fisub    */   &float_low_modrm[0x14],
+    /* OP_fisubr   */   &float_low_modrm[0x15],
+    /* OP_fidiv    */   &float_low_modrm[0x16],
+    /* OP_fidivr   */   &float_low_modrm[0x17],
+    /* OP_fild     */   &float_low_modrm[0x18],
+    /* OP_fist     */   &float_low_modrm[0x1a],
+    /* OP_fistp    */   &float_low_modrm[0x1b],
+    /* OP_frstor   */   &float_low_modrm[0x2c],
+    /* OP_fnsave   */   &float_low_modrm[0x2e],
+    /* OP_fnstsw   */   &float_low_modrm[0x2f],
+
+    /* OP_fbld     */   &float_low_modrm[0x3c],
+    /* OP_fbstp    */   &float_low_modrm[0x3e],
+
+
+    /* OP_fxch      */   &float_high_modrm[1][0x08],
+    /* OP_fnop      */   &float_high_modrm[1][0x10],
+    /* OP_fchs      */   &float_high_modrm[1][0x20],
+    /* OP_fabs      */   &float_high_modrm[1][0x21],
+    /* OP_ftst      */   &float_high_modrm[1][0x24],
+    /* OP_fxam      */   &float_high_modrm[1][0x25],
+    /* OP_fld1      */   &float_high_modrm[1][0x28],
+    /* OP_fldl2t    */   &float_high_modrm[1][0x29],
+    /* OP_fldl2e    */   &float_high_modrm[1][0x2a],
+    /* OP_fldpi     */   &float_high_modrm[1][0x2b],
+    /* OP_fldlg2    */   &float_high_modrm[1][0x2c],
+    /* OP_fldln2    */   &float_high_modrm[1][0x2d],
+    /* OP_fldz      */   &float_high_modrm[1][0x2e],
+    /* OP_f2xm1     */   &float_high_modrm[1][0x30],
+    /* OP_fyl2x     */   &float_high_modrm[1][0x31],
+    /* OP_fptan     */   &float_high_modrm[1][0x32],
+    /* OP_fpatan    */   &float_high_modrm[1][0x33],
+    /* OP_fxtract   */   &float_high_modrm[1][0x34],
+    /* OP_fprem1    */   &float_high_modrm[1][0x35],
+    /* OP_fdecstp   */   &float_high_modrm[1][0x36],
+    /* OP_fincstp   */   &float_high_modrm[1][0x37],
+    /* OP_fprem     */   &float_high_modrm[1][0x38],
+    /* OP_fyl2xp1   */   &float_high_modrm[1][0x39],
+    /* OP_fsqrt     */   &float_high_modrm[1][0x3a],
+    /* OP_fsincos   */   &float_high_modrm[1][0x3b],
+    /* OP_frndint   */   &float_high_modrm[1][0x3c],
+    /* OP_fscale    */   &float_high_modrm[1][0x3d],
+    /* OP_fsin      */   &float_high_modrm[1][0x3e],
+    /* OP_fcos      */   &float_high_modrm[1][0x3f],
+    /* OP_fcmovb    */   &float_high_modrm[2][0x00],
+    /* OP_fcmove    */   &float_high_modrm[2][0x08],
+    /* OP_fcmovbe   */   &float_high_modrm[2][0x10],
+    /* OP_fcmovu    */   &float_high_modrm[2][0x18],
+    /* OP_fucompp   */   &float_high_modrm[2][0x29],
+    /* OP_fcmovnb   */   &float_high_modrm[3][0x00],
+    /* OP_fcmovene  */   &float_high_modrm[3][0x08],
+    /* OP_fcmovnbe  */   &float_high_modrm[3][0x10],
+    /* OP_fcmovnu   */   &float_high_modrm[3][0x18],
+    /* OP_fnclex    */   &float_high_modrm[3][0x22],
+    /* OP_fninit    */   &float_high_modrm[3][0x23],
+    /* OP_fucomi    */   &float_high_modrm[3][0x28],
+    /* OP_fcomi     */   &float_high_modrm[3][0x30],
+    /* OP_ffree     */   &float_high_modrm[5][0x00],
+    /* OP_fucom     */   &float_high_modrm[5][0x20],
+    /* OP_fucomp    */   &float_high_modrm[5][0x28],
+    /* OP_faddp     */   &float_high_modrm[6][0x00],
+    /* OP_fmulp     */   &float_high_modrm[6][0x08],
+    /* OP_fcompp    */   &float_high_modrm[6][0x19],
+    /* OP_fsubrp    */   &float_high_modrm[6][0x20],
+    /* OP_fsubp     */   &float_high_modrm[6][0x28],
+    /* OP_fdivrp    */   &float_high_modrm[6][0x30],
+    /* OP_fdivp     */   &float_high_modrm[6][0x38],
+    /* OP_fucomip   */   &float_high_modrm[7][0x28],
+    /* OP_fcomip    */   &float_high_modrm[7][0x30],
+
+    /* SSE3 instructions */
+    /* OP_fisttp      */   &float_low_modrm[0x29],
+    /* OP_haddpd      */   &prefix_extensions[114][2],
+    /* OP_haddps      */   &prefix_extensions[114][3],
+    /* OP_hsubpd      */   &prefix_extensions[115][2],
+    /* OP_hsubps      */   &prefix_extensions[115][3],
+    /* OP_addsubpd    */   &prefix_extensions[116][2],
+    /* OP_addsubps    */   &prefix_extensions[116][3],
+    /* OP_lddqu       */   &prefix_extensions[117][3],
+    /* OP_monitor     */    &rm_extensions[1][0],
+    /* OP_mwait       */    &rm_extensions[1][1],
+    /* OP_movsldup    */   &prefix_extensions[ 2][1],
+    /* OP_movshdup    */   &prefix_extensions[ 6][1],
+    /* OP_movddup     */   &prefix_extensions[ 2][3],
+
+    /* 3D-Now! instructions */
+    /* OP_femms         */   &second_byte[0x0e],
+    /* OP_unknown_3dnow */   &suffix_extensions[0],
+    /* OP_pavgusb       */   &suffix_extensions[1],
+    /* OP_pfadd         */   &suffix_extensions[2],
+    /* OP_pfacc         */   &suffix_extensions[3],
+    /* OP_pfcmpge       */   &suffix_extensions[4],
+    /* OP_pfcmpgt       */   &suffix_extensions[5],
+    /* OP_pfcmpeq       */   &suffix_extensions[6],
+    /* OP_pfmin         */   &suffix_extensions[7],
+    /* OP_pfmax         */   &suffix_extensions[8],
+    /* OP_pfmul         */   &suffix_extensions[9],
+    /* OP_pfrcp         */   &suffix_extensions[10],
+    /* OP_pfrcpit1      */   &suffix_extensions[11],
+    /* OP_pfrcpit2      */   &suffix_extensions[12],
+    /* OP_pfrsqrt       */   &suffix_extensions[13],
+    /* OP_pfrsqit1      */   &suffix_extensions[14],
+    /* OP_pmulhrw       */   &suffix_extensions[15],
+    /* OP_pfsub         */   &suffix_extensions[16],
+    /* OP_pfsubr        */   &suffix_extensions[17],
+    /* OP_pi2fd         */   &suffix_extensions[18],
+    /* OP_pf2id         */   &suffix_extensions[19],
+    /* OP_pi2fw         */   &suffix_extensions[20],
+    /* OP_pf2iw         */   &suffix_extensions[21],
+    /* OP_pfnacc        */   &suffix_extensions[22],
+    /* OP_pfpnacc       */   &suffix_extensions[23],
+    /* OP_pswapd        */   &suffix_extensions[24],
+
+    /* SSSE3 */
+    /* OP_pshufb        */   &prefix_extensions[118][0],
+    /* OP_phaddw        */   &prefix_extensions[119][0],
+    /* OP_phaddd        */   &prefix_extensions[120][0],
+    /* OP_phaddsw       */   &prefix_extensions[121][0],
+    /* OP_pmaddubsw     */   &prefix_extensions[122][0],
+    /* OP_phsubw        */   &prefix_extensions[123][0],
+    /* OP_phsubd        */   &prefix_extensions[124][0],
+    /* OP_phsubsw       */   &prefix_extensions[125][0],
+    /* OP_psignb        */   &prefix_extensions[126][0],
+    /* OP_psignw        */   &prefix_extensions[127][0],
+    /* OP_psignd        */   &prefix_extensions[128][0],
+    /* OP_pmulhrsw      */   &prefix_extensions[129][0],
+    /* OP_pabsb         */   &prefix_extensions[130][0],
+    /* OP_pabsw         */   &prefix_extensions[131][0],
+    /* OP_pabsd         */   &prefix_extensions[132][0],
+    /* OP_palignr       */   &prefix_extensions[133][0],
+
+    /* SSE4 (incl AMD and Intel-specific extensions */
+    /* OP_popcnt        */   &second_byte[0xb8],
+    /* OP_movntss       */   &prefix_extensions[11][1],
+    /* OP_movntsd       */   &prefix_extensions[11][3],
+    /* OP_extrq         */   &prefix_extensions[134][2],
+    /* OP_insertq       */   &prefix_extensions[134][3],
+    /* OP_lzcnt         */   &prefix_extensions[136][1],
+    /* OP_pblendvb      */   &third_byte_38[16],
+    /* OP_blendvps      */   &third_byte_38[17],
+    /* OP_blendvpd      */   &third_byte_38[18],
+    /* OP_ptest         */   &third_byte_38[19],
+    /* OP_pmovsxbw      */   &third_byte_38[20],
+    /* OP_pmovsxbd      */   &third_byte_38[21],
+    /* OP_pmovsxbq      */   &third_byte_38[22],
+    /* OP_pmovsxdw      */   &third_byte_38[23],
+    /* OP_pmovsxwq      */   &third_byte_38[24],
+    /* OP_pmovsxdq      */   &third_byte_38[25],
+    /* OP_pmuldq        */   &third_byte_38[26],
+    /* OP_pcmpeqq       */   &third_byte_38[27],
+    /* OP_movntdqa      */   &third_byte_38[28],
+    /* OP_packusdw      */   &third_byte_38[29],
+    /* OP_pmovzxbw      */   &third_byte_38[30],
+    /* OP_pmovzxbd      */   &third_byte_38[31],
+    /* OP_pmovzxbq      */   &third_byte_38[32],
+    /* OP_pmovzxdw      */   &third_byte_38[33],
+    /* OP_pmovzxwq      */   &third_byte_38[34],
+    /* OP_pmovzxdq      */   &third_byte_38[35],
+    /* OP_pcmpgtq       */   &third_byte_38[36],
+    /* OP_pminsb        */   &third_byte_38[37],
+    /* OP_pminsd        */   &third_byte_38[38],
+    /* OP_pminuw        */   &third_byte_38[39],
+    /* OP_pminud        */   &third_byte_38[40],
+    /* OP_pmaxsb        */   &third_byte_38[41],
+    /* OP_pmaxsd        */   &third_byte_38[42],
+    /* OP_pmaxuw        */   &third_byte_38[43],
+    /* OP_pmaxud        */   &third_byte_38[44],
+    /* OP_pmulld        */   &third_byte_38[45],
+    /* OP_phminposuw    */   &third_byte_38[46],
+    /* OP_crc32         */   &third_byte_38[48],
+    /* OP_pextrb        */   &third_byte_3a[2],
+    /* OP_pextrd        */   &third_byte_3a[4],
+    /* OP_extractps     */   &third_byte_3a[5],
+    /* OP_roundps       */   &third_byte_3a[6],
+    /* OP_roundpd       */   &third_byte_3a[7],
+    /* OP_roundss       */   &third_byte_3a[8],
+    /* OP_roundsd       */   &third_byte_3a[9],
+    /* OP_blendps       */   &third_byte_3a[10],
+    /* OP_blendpd       */   &third_byte_3a[11],
+    /* OP_pblendw       */   &third_byte_3a[12],
+    /* OP_pinsrb        */   &third_byte_3a[13],
+    /* OP_insertps      */   &third_byte_3a[14],
+    /* OP_pinsrd        */   &third_byte_3a[15],
+    /* OP_dpps          */   &third_byte_3a[16],
+    /* OP_dppd          */   &third_byte_3a[17],
+    /* OP_mpsadbw       */   &third_byte_3a[18],
+    /* OP_pcmpestrm     */   &third_byte_3a[19],
+    /* OP_pcmpestri     */   &third_byte_3a[20],
+    /* OP_pcmpistrm     */   &third_byte_3a[21],
+    /* OP_pcmpistri     */   &third_byte_3a[22],
+
+    /* x64 */
+    /* OP_movsxd        */   &x64_extensions[16][1],
+    /* OP_swapgs        */   &rm_extensions[2][0],
+
+    /* VMX */
+    /* OP_vmcall        */   &rm_extensions[0][1],
+    /* OP_vmlaunch      */   &rm_extensions[0][2],
+    /* OP_vmresume      */   &rm_extensions[0][3],
+    /* OP_vmxoff        */   &rm_extensions[0][4],
+    /* OP_vmptrst       */   &extensions[16][7],
+    /* OP_vmptrld       */   &prefix_extensions[137][0],
+    /* OP_vmxon         */   &prefix_extensions[137][1],
+    /* OP_vmclear       */   &prefix_extensions[137][2],
+    /* OP_vmread        */   &prefix_extensions[134][0],
+    /* OP_vmwrite       */   &prefix_extensions[135][0],
+
+    /* undocumented */
+    /* OP_int1          */   &first_byte[0xf1],
+    /* OP_salc          */   &first_byte[0xd6],
+    /* OP_ffreep        */   &float_high_modrm[7][0x00],
+
+    /* Keep these at the end so that ifdefs don't change internal enum values */
+#ifdef IA32_ON_IA64
+    /* OP_jmpe      */   &extensions[13][6],
+    /* OP_jmpe_abs  */   &second_byte[0xb8],
+#endif
+};
+
+
+/****************************************************************************
+ * Macros to make tables legible
+ */
+
+/* Jb is defined in dynamo.h, undefine it for this file */
+#undef Jb
+
+#define xx  TYPE_NONE, OPSZ_NA
+
+/* from Intel tables, using our corresponding OPSZ constants */
+#define Ap  TYPE_A, OPSZ_6_irex10_short4 /* NOTE - not legal for 64-bit instructions */
+#define Cr  TYPE_C, OPSZ_4x8
+#define Dr  TYPE_D, OPSZ_4x8
+#define Eb  TYPE_E, OPSZ_1
+#define Ew  TYPE_E, OPSZ_2
+#define Ev  TYPE_E, OPSZ_4_rex8_short2
+#define Esv TYPE_E, OPSZ_4x8_short2 /* "stack v", or "d64" in Intel tables */
+#define Ed  TYPE_E, OPSZ_4
+#define Ep  TYPE_E, OPSZ_6_irex10_short4
+#define Ed_q TYPE_E, OPSZ_4_rex8
+#define Rd_Mb TYPE_E, OPSZ_1_reg4
+#define Rd_Mw TYPE_E, OPSZ_2_reg4
+#define Gb  TYPE_G, OPSZ_1
+#define Gw  TYPE_G, OPSZ_2
+#define Gv  TYPE_G, OPSZ_4_rex8_short2
+#define Gz  TYPE_G, OPSZ_4_short2
+#define Gd  TYPE_G, OPSZ_4
+#define Gd_q TYPE_G, OPSZ_4_rex8
+#define Ib  TYPE_I, OPSZ_1
+#define Iw  TYPE_I, OPSZ_2
+#define Iv  TYPE_I, OPSZ_4_rex8_short2
+#define Iz  TYPE_I, OPSZ_4_short2
+#define Jb  TYPE_J, OPSZ_1
+#define Jz  TYPE_J, OPSZ_4_short2xi4
+#define Ma  TYPE_M, OPSZ_8_short4
+#define Mp  TYPE_M, OPSZ_6_irex10_short4
+#define Ms  TYPE_M, OPSZ_6x10
+#define Ob  TYPE_O, OPSZ_1
+#define Ov  TYPE_O, OPSZ_4_rex8_short2
+#define Pd  TYPE_P, OPSZ_4
+#define Pq  TYPE_P, OPSZ_8
+#define Pd_q TYPE_P, OPSZ_4_of_8
+#define Ppi TYPE_P, OPSZ_8
+#define Nq  TYPE_P_MODRM, OPSZ_8
+#define Qd  TYPE_Q, OPSZ_4
+#define Qq  TYPE_Q, OPSZ_8
+#define Qpi TYPE_Q, OPSZ_8
+#define Rr  TYPE_R, OPSZ_4x8
+#define Sw  TYPE_S, OPSZ_2
+#define Vq  TYPE_V, OPSZ_8
+#define Vdq TYPE_V, OPSZ_16
+#define Vd_dq TYPE_V, OPSZ_4_of_16
+#define Vq_dq TYPE_V, OPSZ_8_of_16
+#define Vps TYPE_V, OPSZ_16
+#define Vpd TYPE_V, OPSZ_16
+#define Vss TYPE_V, OPSZ_16
+#define Vsd TYPE_V, OPSZ_16
+#define Ups TYPE_V_MODRM, OPSZ_16
+#define Upd TYPE_V_MODRM, OPSZ_16
+#define Udq TYPE_V_MODRM, OPSZ_16
+#define Wq  TYPE_W, OPSZ_8
+#define Wdq TYPE_W, OPSZ_16
+#define Wq_dq TYPE_W, OPSZ_8_of_16
+#define Wps TYPE_W, OPSZ_16
+#define Wpd TYPE_W, OPSZ_16
+#define Wss TYPE_W, OPSZ_16
+#define Wsd TYPE_W, OPSZ_16
+#define Udq_Md TYPE_W, OPSZ_4_reg16
+#define Xb  TYPE_X, OPSZ_1
+#define Xv  TYPE_X, OPSZ_4_rex8_short2
+#define Xz  TYPE_X, OPSZ_4_short2
+#define Yb  TYPE_Y, OPSZ_1
+#define Yv  TYPE_Y, OPSZ_4_rex8_short2
+#define Yz  TYPE_Y, OPSZ_4_short2
+
+/* my own codes
+ * size m = 32 or 16 bit depending on addr size attribute
+ * B=ds:eDI, Z=xlat's mem, K=float in mem, i_==indirect 
+ */
+#define Mb  TYPE_M, OPSZ_1
+#define Md  TYPE_M, OPSZ_4
+#define Md_q  TYPE_M, OPSZ_4_rex8
+#define Mw  TYPE_M, OPSZ_2
+#define Mm  TYPE_M, OPSZ_lea
+#define Me  TYPE_M, OPSZ_512
+#define Mps  TYPE_M, OPSZ_16
+#define Mpd  TYPE_M, OPSZ_16
+#define Mss  TYPE_M, OPSZ_16
+#define Msd  TYPE_M, OPSZ_16
+#define Mq  TYPE_M, OPSZ_8
+#define Mdq  TYPE_M, OPSZ_16
+#define Zb  TYPE_XLAT, OPSZ_1
+#define Bq  TYPE_MASKMOVQ, OPSZ_8
+#define Bdq  TYPE_MASKMOVQ, OPSZ_16
+#define Kw  TYPE_FLOATMEM, OPSZ_2
+#define Kd  TYPE_FLOATMEM, OPSZ_4
+#define Kq  TYPE_FLOATMEM, OPSZ_8
+#define Kx  TYPE_FLOATMEM, OPSZ_10
+#define Ky  TYPE_FLOATMEM, OPSZ_28_short14 /* _14_ if data16 */
+#define Kz  TYPE_FLOATMEM, OPSZ_108_short94 /* _98_ if data16 */
+#define i_dx  TYPE_INDIR_REG, REG_DX
+#define i_Ev  TYPE_INDIR_E, OPSZ_4_rex8_short2
+#define i_Exi  TYPE_INDIR_E, OPSZ_4x8_short2xi8
+#define i_Ep  TYPE_INDIR_E, OPSZ_6_irex10_short4
+#define i_vSP TYPE_INDIR_VAR_REG, REG_ESP
+#define i_xSP TYPE_INDIR_VAR_XREG, REG_ESP
+#define i_iSP TYPE_INDIR_VAR_XIREG, REG_ESP
+#define i_xBP TYPE_INDIR_VAR_XREG, REG_EBP
+#define c1  TYPE_1, OPSZ_0
+#define cF  TYPE_FLOATCONST, OPSZ_0
+
+#ifdef IA32_ON_IA64
+#define Av TYPE_A, OPSZ_4_short2
+#endif
+
+/* registers that are base 32 but vary down or up */
+#define eAX TYPE_VAR_REG, REG_EAX
+#define eCX TYPE_VAR_REG, REG_ECX
+#define eDX TYPE_VAR_REG, REG_EDX
+#define eBX TYPE_VAR_REG, REG_EBX
+#define eSP TYPE_VAR_REG, REG_ESP
+#define eBP TYPE_VAR_REG, REG_EBP
+#define eSI TYPE_VAR_REG, REG_ESI
+#define eDI TYPE_VAR_REG, REG_EDI
+
+/* registers that are base 32 and can vary down but not up */
+#define zAX TYPE_VARZ_REG, REG_EAX
+#define zCX TYPE_VARZ_REG, REG_ECX
+#define zDX TYPE_VARZ_REG, REG_EDX
+#define zBX TYPE_VARZ_REG, REG_EBX
+#define zSP TYPE_VARZ_REG, REG_ESP
+#define zBP TYPE_VARZ_REG, REG_EBP
+#define zSI TYPE_VARZ_REG, REG_ESI
+#define zDI TYPE_VARZ_REG, REG_EDI
+
+/* registers whose base matches the mode, and can vary down but not up.
+ * we use the 32-bit versions but expand in resolve_var_reg()
+ */
+#define xAX TYPE_VAR_XREG, REG_EAX
+#define xCX TYPE_VAR_XREG, REG_ECX
+#define xDX TYPE_VAR_XREG, REG_EDX
+#define xBX TYPE_VAR_XREG, REG_EBX
+#define xSP TYPE_VAR_XREG, REG_ESP
+#define xBP TYPE_VAR_XREG, REG_EBP
+#define xSI TYPE_VAR_XREG, REG_ESI
+#define xDI TYPE_VAR_XREG, REG_EDI
+
+/* esp modifications in push/pop instrs operate on esp vs. sp
+ * wrt addr16, not data16, so we have to special-case those esp operands
+ */
+#define axSP TYPE_VAR_ADDR_XREG, REG_ESP
+#define axBP TYPE_VAR_ADDR_XREG, REG_EBP
+/* jecxz and loop* use addr16 */
+#define axCX TYPE_VAR_ADDR_XREG, REG_ECX
+/* string ops use addr16 */
+#define axSI TYPE_VAR_ADDR_XREG, REG_ESI
+#define axDI TYPE_VAR_ADDR_XREG, REG_EDI
+
+/* 8-bit implicit registers (not from modrm) that can be exteded via rex.r */
+#define al_x TYPE_REG_EX, REG_AL
+#define cl_x TYPE_REG_EX, REG_CL
+#define dl_x TYPE_REG_EX, REG_DL
+#define bl_x TYPE_REG_EX, REG_BL
+#define ah_x TYPE_REG_EX, REG_AH
+#define ch_x TYPE_REG_EX, REG_CH
+#define dh_x TYPE_REG_EX, REG_DH
+#define bh_x TYPE_REG_EX, REG_BH
+
+/* 4_rex8_short2 implicit registers (not from modrm) that can be exteded via rex.r */
+#define eAX_x TYPE_VAR_REG_EX, REG_EAX
+#define eCX_x TYPE_VAR_REG_EX, REG_ECX
+#define eDX_x TYPE_VAR_REG_EX, REG_EDX
+#define eBX_x TYPE_VAR_REG_EX, REG_EBX
+#define eSP_x TYPE_VAR_REG_EX, REG_ESP
+#define eBP_x TYPE_VAR_REG_EX, REG_EBP
+#define eSI_x TYPE_VAR_REG_EX, REG_ESI
+#define eDI_x TYPE_VAR_REG_EX, REG_EDI
+
+/* 4x8_short2 implicit registers (not from modrm) that can be exteded via rex.r */
+#define xAX_x TYPE_VAR_XREG_EX, REG_EAX
+#define xCX_x TYPE_VAR_XREG_EX, REG_ECX
+#define xDX_x TYPE_VAR_XREG_EX, REG_EDX
+#define xBX_x TYPE_VAR_XREG_EX, REG_EBX
+#define xSP_x TYPE_VAR_XREG_EX, REG_ESP
+#define xBP_x TYPE_VAR_XREG_EX, REG_EBP
+#define xSI_x TYPE_VAR_XREG_EX, REG_ESI
+#define xDI_x TYPE_VAR_XREG_EX, REG_EDI
+
+/* 4_rex8 implicit registers (not from modrm) that can be exteded via rex.r */
+#define uAX_x TYPE_VAR_REGX_EX, REG_EAX
+#define uCX_x TYPE_VAR_REGX_EX, REG_ECX
+#define uDX_x TYPE_VAR_REGX_EX, REG_EDX
+#define uBX_x TYPE_VAR_REGX_EX, REG_EBX
+#define uSP_x TYPE_VAR_REGX_EX, REG_ESP
+#define uBP_x TYPE_VAR_REGX_EX, REG_EBP
+#define uSI_x TYPE_VAR_REGX_EX, REG_ESI
+#define uDI_x TYPE_VAR_REGX_EX, REG_EDI
+
+#define ax TYPE_REG, REG_AX
+#define cx TYPE_REG, REG_CX
+#define dx TYPE_REG, REG_DX
+#define bx TYPE_REG, REG_BX
+#define sp TYPE_REG, REG_SP
+#define bp TYPE_REG, REG_BP
+#define si TYPE_REG, REG_SI
+#define di TYPE_REG, REG_DI
+
+#define al TYPE_REG, REG_AL
+#define cl TYPE_REG, REG_CL
+#define dl TYPE_REG, REG_DL
+#define bl TYPE_REG, REG_BL
+#define ah TYPE_REG, REG_AH
+#define ch TYPE_REG, REG_CH
+#define dh TYPE_REG, REG_DH
+#define bh TYPE_REG, REG_BH
+
+#define eax TYPE_REG, REG_EAX
+#define ecx TYPE_REG, REG_ECX
+#define edx TYPE_REG, REG_EDX
+#define ebx TYPE_REG, REG_EBX
+#define esp TYPE_REG, REG_ESP
+#define ebp TYPE_REG, REG_EBP
+#define esi TYPE_REG, REG_ESI
+#define edi TYPE_REG, REG_EDI
+
+#define xsp TYPE_REG, REG_XSP
+#define xcx TYPE_REG, REG_XCX
+
+#define cs  TYPE_REG, SEG_CS
+#define ss  TYPE_REG, SEG_SS
+#define ds  TYPE_REG, SEG_DS
+#define es  TYPE_REG, SEG_ES
+#define fs  TYPE_REG, SEG_FS
+#define gs  TYPE_REG, SEG_GS
+
+#define st0 TYPE_REG, REG_ST0
+#define st1 TYPE_REG, REG_ST1
+#define st2 TYPE_REG, REG_ST2
+#define st3 TYPE_REG, REG_ST3
+#define st4 TYPE_REG, REG_ST4
+#define st5 TYPE_REG, REG_ST5
+#define st6 TYPE_REG, REG_ST6
+#define st7 TYPE_REG, REG_ST7
+
+#define xmm0 TYPE_REG, REG_XMM0
+
+/* flags */
+#define no       0
+#define mrm      HAS_MODRM
+#define xop      (HAS_EXTRA_OPERANDS|EXTRAS_IN_CODE_FIELD)
+#define mrm_xop  (HAS_MODRM|HAS_EXTRA_OPERANDS|EXTRAS_IN_CODE_FIELD)
+#define xop_next (HAS_EXTRA_OPERANDS)
+#define i64      X64_INVALID
+#define o64      X86_INVALID
+#define reqp     REQUIRES_PREFIX
+
+/* eflags */
+#define x     0
+#define fRC   EFLAGS_READ_CF
+#define fRP   EFLAGS_READ_PF
+#define fRA   EFLAGS_READ_AF
+#define fRZ   EFLAGS_READ_ZF
+#define fRS   EFLAGS_READ_SF
+#define fRT   EFLAGS_READ_TF
+#define fRI   EFLAGS_READ_IF
+#define fRD   EFLAGS_READ_DF
+#define fRO   EFLAGS_READ_OF
+#define fRN   EFLAGS_READ_NT
+#define fRR   EFLAGS_READ_RF
+#define fRX   EFLAGS_READ_ALL
+#define fR6   EFLAGS_READ_6
+#define fWC   EFLAGS_WRITE_CF
+#define fWP   EFLAGS_WRITE_PF
+#define fWA   EFLAGS_WRITE_AF
+#define fWZ   EFLAGS_WRITE_ZF
+#define fWS   EFLAGS_WRITE_SF
+#define fWT   EFLAGS_WRITE_TF
+#define fWI   EFLAGS_WRITE_IF
+#define fWD   EFLAGS_WRITE_DF
+#define fWO   EFLAGS_WRITE_OF
+#define fWN   EFLAGS_WRITE_NT
+#define fWR   EFLAGS_WRITE_RF
+#define fWX   EFLAGS_WRITE_ALL
+#define fW6   EFLAGS_WRITE_6
+/* flags affected by OP_int*
+ * FIXME: should we add AC and VM flags?
+ */
+#define fINT  (fRX|fWT|fWN|fWI|fWR)
+
+/* for constructing linked lists of table entries */
+#define NA 0
+#define END_LIST  0
+#define tfb (ptr_int_t)&first_byte
+#define tsb (ptr_int_t)&second_byte
+#define tex (ptr_int_t)&extensions
+#define t38 (ptr_int_t)&third_byte_38
+#define t3a (ptr_int_t)&third_byte_3a
+#define tpe (ptr_int_t)&prefix_extensions
+#define tre (ptr_int_t)&rep_extensions
+#define tne (ptr_int_t)&repne_extensions
+#define tfl (ptr_int_t)&float_low_modrm
+#define tfh (ptr_int_t)&float_high_modrm
+#define exop (ptr_int_t)&extra_operands
+#define t64e (ptr_int_t)&x64_extensions
+#define trex (ptr_int_t)&rex_extensions
+
+/* point at this when you need a canonical invalid instr 
+ * type is OP_INVALID so can be copied to instr->opcode
+ */
+const instr_info_t invalid_instr =
+    {OP_INVALID,  0x000000, "(bad)", xx, xx, xx, xx, xx, no, x, NA};
+
+/****************************************************************************
+ * One-byte opcodes
+ * This is from Tables A-2 & A-3
+ */
+const instr_info_t first_byte[] = {
+    /* {op/type, op encoding, name, dst1, dst2, src1, src2, src3, modrm?, eflags, code} */
+    /* 00 */
+    {OP_add,  0x000000, "add",  Eb, xx, Gb, Eb, xx, mrm, fW6, tex[1][0]},
+    {OP_add,  0x010000, "add",  Ev, xx, Gv, Ev, xx, mrm, fW6, tfb[0x00]},
+    {OP_add,  0x020000, "add",  Gb, xx, Eb, Gb, xx, mrm, fW6, tfb[0x01]},
+    {OP_add,  0x030000, "add",  Gv, xx, Ev, Gv, xx, mrm, fW6, tfb[0x02]},
+    {OP_add,  0x040000, "add",  al, xx, Ib, al, xx, no,  fW6, tfb[0x03]},
+    {OP_add,  0x050000, "add", eAX, xx, Iz, eAX, xx, no,  fW6, tfb[0x04]},
+    {OP_push, 0x060000, "push", axSP, i_xSP, es, axSP, xx, i64, x, tfb[0x0e]},
+    {OP_pop,  0x070000, "pop", es, axSP, axSP, i_xSP, xx, i64, x, tsb[0xa1]},
+    /* 08 */
+    {OP_or,  0x080000, "or",  Eb, xx, Gb, Eb, xx, mrm, fW6, tex[1][1]},
+    {OP_or,  0x090000, "or",  Ev, xx, Gv, Ev, xx, mrm, fW6, tfb[0x08]},
+    {OP_or,  0x0a0000, "or",  Gb, xx, Eb, Gb, xx, mrm, fW6, tfb[0x09]},
+    {OP_or,  0x0b0000, "or",  Gv, xx, Ev, Gv, xx, mrm, fW6, tfb[0x0a]},
+    {OP_or,  0x0c0000, "or",  al, xx, Ib, al, xx, no,  fW6, tfb[0x0b]},
+    {OP_or,  0x0d0000, "or", eAX, xx, Iz, eAX, xx, no,  fW6, tfb[0x0c]},
+    {OP_push,0x0e0000, "push", axSP, i_xSP, cs, axSP, xx, i64, x, tfb[0x16]},
+    {ESCAPE, 0x0f0000, "(escape)", xx, xx, xx, xx, xx, no, x, NA},
+    /* 10 */
+    {OP_adc,  0x100000, "adc",  Eb, xx, Gb, Eb, xx, mrm, (fW6|fRC), tex[1][2]},
+    {OP_adc,  0x110000, "adc",  Ev, xx, Gv, Ev, xx, mrm, (fW6|fRC), tfb[0x10]},
+    {OP_adc,  0x120000, "adc",  Gb, xx, Eb, Gb, xx, mrm, (fW6|fRC), tfb[0x11]},
+    {OP_adc,  0x130000, "adc",  Gv, xx, Ev, Gv, xx, mrm, (fW6|fRC), tfb[0x12]},
+    {OP_adc,  0x140000, "adc",  al, xx, Ib, al, xx, no,  (fW6|fRC), tfb[0x13]},
+    {OP_adc,  0x150000, "adc", eAX, xx, Iz, eAX, xx, no,  (fW6|fRC), tfb[0x14]},
+    {OP_push, 0x160000, "push", axSP, i_xSP, ss, axSP, xx, i64, x, tfb[0x1e]},
+    {OP_pop,  0x170000, "pop", ss, axSP, axSP, i_xSP, xx, i64, x, tfb[0x1f]},
+    /* 18 */
+    {OP_sbb,  0x180000, "sbb",  Eb, xx, Gb, Eb, xx, mrm, (fW6|fRC), tex[1][3]},
+    {OP_sbb,  0x190000, "sbb",  Ev, xx, Gv, Ev, xx, mrm, (fW6|fRC), tfb[0x18]},
+    {OP_sbb,  0x1a0000, "sbb",  Gb, xx, Eb, Gb, xx, mrm, (fW6|fRC), tfb[0x19]},
+    {OP_sbb,  0x1b0000, "sbb",  Gv, xx, Ev, Gv, xx, mrm, (fW6|fRC), tfb[0x1a]},
+    {OP_sbb,  0x1c0000, "sbb",  al, xx, Ib, al, xx, no,  (fW6|fRC), tfb[0x1b]},
+    {OP_sbb,  0x1d0000, "sbb", eAX, xx, Iz, eAX, xx, no,  (fW6|fRC), tfb[0x1c]},
+    {OP_push, 0x1e0000, "push", axSP, i_xSP, ds, axSP, xx, i64, x, tsb[0xa0]},
+    {OP_pop,  0x1f0000, "pop", ds, axSP, axSP, i_xSP, xx, i64, x, tfb[0x07]},
+    /* 20 */
+    {OP_and,  0x200000, "and",  Eb, xx, Gb, Eb, xx, mrm, fW6, tex[1][4]},
+    {OP_and,  0x210000, "and",  Ev, xx, Gv, Ev, xx, mrm, fW6, tfb[0x20]},
+    {OP_and,  0x220000, "and",  Gb, xx, Eb, Gb, xx, mrm, fW6, tfb[0x21]},
+    {OP_and,  0x230000, "and",  Gv, xx, Ev, Gv, xx, mrm, fW6, tfb[0x22]},
+    {OP_and,  0x240000, "and",  al, xx, Ib, al, xx, no,  fW6, tfb[0x23]},
+    {OP_and,  0x250000, "and", eAX, xx, Iz, eAX, xx, no,  fW6, tfb[0x24]},
+    {PREFIX,  0x260000, "es", xx, xx, xx, xx, xx, no, x, SEG_ES},
+    {OP_daa,  0x270000, "daa", al, xx, al, xx, xx, i64, (fW6|fRC|fRA), END_LIST},
+    /* 28 */
+    {OP_sub,  0x280000, "sub",  Eb, xx, Gb, Eb, xx, mrm, fW6, tex[1][5]},
+    {OP_sub,  0x290000, "sub",  Ev, xx, Gv, Ev, xx, mrm, fW6, tfb[0x28]},
+    {OP_sub,  0x2a0000, "sub",  Gb, xx, Eb, Gb, xx, mrm, fW6, tfb[0x29]},
+    {OP_sub,  0x2b0000, "sub",  Gv, xx, Ev, Gv, xx, mrm, fW6, tfb[0x2a]},
+    {OP_sub,  0x2c0000, "sub",  al, xx, Ib, al, xx, no,  fW6, tfb[0x2b]},
+    {OP_sub,  0x2d0000, "sub", eAX, xx, Iz, eAX, xx, no,  fW6, tfb[0x2c]},
+    {PREFIX,  0x2e0000, "cs", xx, xx, xx, xx, xx, no, x, SEG_CS},
+    {OP_das,  0x2f0000, "das", al, xx, al, xx, xx, i64, (fW6|fRC|fRA), END_LIST},
+    /* 30 */
+    {OP_xor,  0x300000, "xor",  Eb, xx, Gb, Eb, xx, mrm, fW6, tex[1][6]},
+    {OP_xor,  0x310000, "xor",  Ev, xx, Gv, Ev, xx, mrm, fW6, tfb[0x30]},
+    {OP_xor,  0x320000, "xor",  Gb, xx, Eb, Gb, xx, mrm, fW6, tfb[0x31]},
+    {OP_xor,  0x330000, "xor",  Gv, xx, Ev, Gv, xx, mrm, fW6, tfb[0x32]},
+    {OP_xor,  0x340000, "xor",  al, xx, Ib, al, xx, no,  fW6, tfb[0x33]},
+    {OP_xor,  0x350000, "xor", eAX, xx, Iz, eAX, xx, no,  fW6, tfb[0x34]},
+    {PREFIX,  0x360000, "ss", xx, xx, xx, xx, xx, no, x, SEG_SS},
+    {OP_aaa,  0x370000, "aaa", ax, xx, ax, xx, xx, i64, (fW6|fRA), END_LIST},
+    /* 38 */
+    {OP_cmp,  0x380000, "cmp", xx, xx,  Eb, Gb, xx, mrm, fW6, tex[1][7]},
+    {OP_cmp,  0x390000, "cmp", xx, xx,  Ev, Gv, xx, mrm, fW6, tfb[0x38]},
+    {OP_cmp,  0x3a0000, "cmp", xx, xx,  Gb, Eb, xx, mrm, fW6, tfb[0x39]},
+    {OP_cmp,  0x3b0000, "cmp", xx, xx,  Gv, Ev, xx, mrm, fW6, tfb[0x3a]},
+    {OP_cmp,  0x3c0000, "cmp", xx, xx,  al, Ib, xx, no,  fW6, tfb[0x3b]},
+    {OP_cmp,  0x3d0000, "cmp", xx, xx, eAX, Iz, xx, no,  fW6, tfb[0x3c]},
+    {PREFIX,  0x3e0000, "ds", xx, xx, xx, xx, xx, no, x, SEG_DS},
+    {OP_aas,  0x3f0000, "aas", ax, xx, ax, xx, xx, i64, (fW6|fRA), END_LIST},
+    /* 40 */
+    {X64_EXT, 0x400000, "(x64_ext 0)", xx, xx, xx, xx, xx, no, x, 0},
+    {X64_EXT, 0x410000, "(x64_ext 1)", xx, xx, xx, xx, xx, no, x, 1},
+    {X64_EXT, 0x420000, "(x64_ext 2)", xx, xx, xx, xx, xx, no, x, 2},
+    {X64_EXT, 0x430000, "(x64_ext 3)", xx, xx, xx, xx, xx, no, x, 3},
+    {X64_EXT, 0x440000, "(x64_ext 4)", xx, xx, xx, xx, xx, no, x, 4},
+    {X64_EXT, 0x450000, "(x64_ext 5)", xx, xx, xx, xx, xx, no, x, 5},
+    {X64_EXT, 0x460000, "(x64_ext 6)", xx, xx, xx, xx, xx, no, x, 6},
+    {X64_EXT, 0x470000, "(x64_ext 7)", xx, xx, xx, xx, xx, no, x, 7},
+    /* 48 */
+    {X64_EXT, 0x480000, "(x64_ext 8)", xx, xx, xx, xx, xx, no, x, 8},
+    {X64_EXT, 0x490000, "(x64_ext 9)", xx, xx, xx, xx, xx, no, x, 9},
+    {X64_EXT, 0x4a0000, "(x64_ext 10)", xx, xx, xx, xx, xx, no, x, 10},
+    {X64_EXT, 0x4b0000, "(x64_ext 11)", xx, xx, xx, xx, xx, no, x, 11},
+    {X64_EXT, 0x4c0000, "(x64_ext 12)", xx, xx, xx, xx, xx, no, x, 12},
+    {X64_EXT, 0x4d0000, "(x64_ext 13)", xx, xx, xx, xx, xx, no, x, 13},
+    {X64_EXT, 0x4e0000, "(x64_ext 14)", xx, xx, xx, xx, xx, no, x, 14},
+    {X64_EXT, 0x4f0000, "(x64_ext 15)", xx, xx, xx, xx, xx, no, x, 15},
+    /* 50 */
+    {OP_push,  0x500000, "push", axSP, i_xSP, xAX_x, axSP, xx, no, x, tfb[0x51]},
+    {OP_push,  0x510000, "push", axSP, i_xSP, xCX_x, axSP, xx, no, x, tfb[0x52]},
+    {OP_push,  0x520000, "push", axSP, i_xSP, xDX_x, axSP, xx, no, x, tfb[0x53]},
+    {OP_push,  0x530000, "push", axSP, i_xSP, xBX_x, axSP, xx, no, x, tfb[0x54]},
+    {OP_push,  0x540000, "push", axSP, i_xSP, xSP_x, axSP, xx, no, x, tfb[0x55]},
+    {OP_push,  0x550000, "push", axSP, i_xSP, xBP_x, axSP, xx, no, x, tfb[0x56]},
+    {OP_push,  0x560000, "push", axSP, i_xSP, xSI_x, axSP, xx, no, x, tfb[0x57]},
+    {OP_push,  0x570000, "push", axSP, i_xSP, xDI_x, axSP, xx, no, x, tex[12][6]},
+    /* 58 */
+    {OP_pop,  0x580000, "pop", xAX_x, axSP, axSP, i_xSP, xx, no, x, tfb[0x59]},
+    {OP_pop,  0x590000, "pop", xCX_x, axSP, axSP, i_xSP, xx, no, x, tfb[0x5a]},
+    {OP_pop,  0x5a0000, "pop", xDX_x, axSP, axSP, i_xSP, xx, no, x, tfb[0x5b]},
+    {OP_pop,  0x5b0000, "pop", xBX_x, axSP, axSP, i_xSP, xx, no, x, tfb[0x5c]},
+    {OP_pop,  0x5c0000, "pop", xSP_x, axSP, axSP, i_xSP, xx, no, x, tfb[0x5d]},
+    {OP_pop,  0x5d0000, "pop", xBP_x, axSP, axSP, i_xSP, xx, no, x, tfb[0x5e]},
+    {OP_pop,  0x5e0000, "pop", xSI_x, axSP, axSP, i_xSP, xx, no, x, tfb[0x5f]},
+    {OP_pop,  0x5f0000, "pop", xDI_x, axSP, axSP, i_xSP, xx, no, x, tex[26][0]},
+    /* 60 */
+    {OP_pusha, 0x600000, "pusha", axSP, i_xSP, axSP, eAX, eBX, xop|i64, x, exop[0x00]},
+    {OP_popa,  0x610000, "popa", axSP, eAX, axSP, i_xSP, xx, xop|i64, x, exop[0x02]},
+    {OP_bound, 0x620000, "bound", xx, xx, Gv, Ma, xx, mrm|i64, x, END_LIST},
+    {X64_EXT,  0x630000, "(x64_ext 16)", xx, xx, xx, xx, xx, no, x, 16},
+    {PREFIX, 0x640000, "fs", xx, xx, xx, xx, xx, no, x, SEG_FS},
+    {PREFIX, 0x650000, "gs", xx, xx, xx, xx, xx, no, x, SEG_GS},
+    {PREFIX, 0x660000, "data size", xx, xx, xx, xx, xx, no, x, PREFIX_DATA},
+    {PREFIX, 0x670000, "addr size", xx, xx, xx, xx, xx, no, x, PREFIX_ADDR},
+    /* 68 */
+    {OP_push_imm, 0x680000, "push", axSP, i_xSP, Iz, axSP, xx, no, x, tfb[0x6a]},
+    {OP_imul,  0x690000, "imul", Gv, xx, Ev, Iz, xx, mrm, fW6, tfb[0x6b]},
+    {OP_push_imm, 0x6a0000, "push", axSP, i_xSP, Ib, axSP, xx, no, x, END_LIST},/* sign-extend to push 2/4/8 bytes */
+    {OP_imul,  0x6b0000, "imul", Gv, xx, Ev, Ib, xx, mrm, fW6, END_LIST},
+    {REP_EXT,  0x6c0000, "((rep) ins)", Yb, xx, i_dx, xx, xx, no, fRD, 0},
+    {REP_EXT,  0x6d0000, "((rep) ins)", Yz, xx, i_dx, xx, xx, no, fRD, 1},
+    {REP_EXT,  0x6e0000, "((rep) outs)", i_dx, xx, Xb, xx, xx, no, fRD, 2},
+    {REP_EXT,  0x6f0000, "((rep) outs)", i_dx, xx, Xz, xx, xx, no, fRD, 3},
+    /* 70 */
+    {OP_jo_short,  0x700000, "jo",  xx, xx, Jb, xx, xx, no, fRO, END_LIST},
+    {OP_jno_short, 0x710000, "jno", xx, xx, Jb, xx, xx, no, fRO, END_LIST},
+    {OP_jb_short,  0x720000, "jb",  xx, xx, Jb, xx, xx, no, fRC, END_LIST},
+    {OP_jnb_short, 0x730000, "jnb", xx, xx, Jb, xx, xx, no, fRC, END_LIST},
+    {OP_jz_short,  0x740000, "jz",  xx, xx, Jb, xx, xx, no, fRZ, END_LIST},
+    {OP_jnz_short, 0x750000, "jnz", xx, xx, Jb, xx, xx, no, fRZ, END_LIST},
+    {OP_jbe_short, 0x760000, "jbe", xx, xx, Jb, xx, xx, no, (fRC|fRZ), END_LIST},
+    {OP_jnbe_short,0x770000, "jnbe",xx, xx, Jb, xx, xx, no, (fRC|fRZ), END_LIST},
+    /* 78 */
+    {OP_js_short,  0x780000, "js",  xx, xx, Jb, xx, xx, no, fRS, END_LIST},
+    {OP_jns_short, 0x790000, "jns", xx, xx, Jb, xx, xx, no, fRS, END_LIST},
+    {OP_jp_short,  0x7a0000, "jp",  xx, xx, Jb, xx, xx, no, fRP, END_LIST},
+    {OP_jnp_short, 0x7b0000, "jnp", xx, xx, Jb, xx, xx, no, fRP, END_LIST},
+    {OP_jl_short,  0x7c0000, "jl",  xx, xx, Jb, xx, xx, no, (fRS|fRO), END_LIST},
+    {OP_jnl_short, 0x7d0000, "jnl", xx, xx, Jb, xx, xx, no, (fRS|fRO), END_LIST},
+    {OP_jle_short, 0x7e0000, "jle", xx, xx, Jb, xx, xx, no, (fRS|fRO|fRZ), END_LIST},
+    {OP_jnle_short,0x7f0000, "jnle",xx, xx, Jb, xx, xx, no, (fRS|fRO|fRZ), END_LIST},
+    /* 80 */
+    {EXTENSION, 0x800000, "(group 1a)", Eb, xx, Ib, xx, xx, mrm, x, 0},
+    {EXTENSION, 0x810000, "(group 1b)", Ev, xx, Iz, xx, xx, mrm, x, 1},
+    {EXTENSION, 0x820000, "(group 1c*)", Ev, xx, Ib, xx, xx, mrm|i64, x, 25}, /* PR 235092: gnu tools (gdb, objdump) think this is a bad opcode but windbg and the hardware disagree */
+    {EXTENSION, 0x830000, "(group 1c)", Ev, xx, Ib, xx, xx, mrm, x, 2},
+    {OP_test,  0x840000, "test", xx, xx, Eb, Gb, xx, mrm, fW6, tex[10][0]},
+    {OP_test,  0x850000, "test", xx, xx, Ev, Gv, xx, mrm, fW6, tfb[0x84]},
+    {OP_xchg,  0x860000, "xchg", Eb, Gb, Eb, Gb, xx, mrm, x, END_LIST},
+    {OP_xchg,  0x870000, "xchg", Ev, Gv, Ev, Gv, xx, mrm, x, tfb[0x86]},
+    /* 88 */
+    {OP_mov_st,  0x880000, "mov", Eb, xx, Gb, xx, xx, mrm, x, tex[18][0]},
+    {OP_mov_st,  0x890000, "mov", Ev, xx, Gv, xx, xx, mrm, x, tfb[0x88]},
+    {OP_mov_ld,  0x8a0000, "mov", Gb, xx, Eb, xx, xx, mrm, x, END_LIST},
+    {OP_mov_ld,  0x8b0000, "mov", Gv, xx, Ev, xx, xx, mrm, x, tfb[0x8a]},
+    {OP_mov_seg, 0x8c0000, "mov", Ev, xx, Sw, xx, xx, mrm, x, END_LIST},
+    {OP_lea,  0x8d0000, "lea", Gv, xx, Mm, xx, xx, mrm, x, END_LIST}, /* Intel has just M */
+    {OP_mov_seg, 0x8e0000, "mov", Sw, xx, Ev, xx, xx, mrm, x, tfb[0x8c]},
+    {EXTENSION, 0x8f0000, "(group 1d)", xx, xx, xx, xx, xx, mrm, x, 26},
+    /* 90 */
+    {PREFIX_EXT, 0x900000, "(prefix ext 103)", xx, xx, xx, xx, xx, no, x, 103},
+    {OP_xchg, 0x910000, "xchg", eCX_x, eAX, eCX_x, eAX, xx, no, x, tfb[0x92]},
+    {OP_xchg, 0x920000, "xchg", eDX_x, eAX, eDX_x, eAX, xx, no, x, tfb[0x93]},
+    {OP_xchg, 0x930000, "xchg", eBX_x, eAX, eBX_x, eAX, xx, no, x, tfb[0x94]},
+    {OP_xchg, 0x940000, "xchg", eSP_x, eAX, eSP_x, eAX, xx, no, x, tfb[0x95]},
+    {OP_xchg, 0x950000, "xchg", eBP_x, eAX, eBP_x, eAX, xx, no, x, tfb[0x96]},
+    {OP_xchg, 0x960000, "xchg", eSI_x, eAX, eSI_x, eAX, xx, no, x, tfb[0x97]},
+    {OP_xchg, 0x970000, "xchg", eDI_x, eAX, eDI_x, eAX, xx, no, x, tfb[0x87]},
+    /* 98 */
+    {OP_cwde, 0x980000, "cwde", eAX, xx, ax, xx, xx, no, x, END_LIST},/*16-bit=="cbw", src is al not ax; FIXME: newer gdb calls it "cwtl"?!?*/
+    /* PR 354096: does not write to ax/eax/rax: sign-extends into dx/edx/rdx */
+    {OP_cdq,  0x990000, "cdq", eDX, xx, eAX, xx, xx, no, x, END_LIST},/*16-bit=="cwd";64-bit=="cqo"*/
+    {OP_call_far, 0x9a0000, "lcall",  axSP, i_vSP, Ap, axSP, xx, i64, x, END_LIST},
+    {OP_fwait, 0x9b0000, "fwait", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pushf, 0x9c0000, "pushf", axSP, i_xSP, axSP, xx, xx, no, fRX, END_LIST},
+    {OP_popf,  0x9d0000, "popf", axSP, xx, axSP, i_xSP, xx, no, fWX, END_LIST},
+    {OP_sahf,  0x9e0000, "sahf", xx, xx, ah, xx, xx, no, (fW6&(~fWO)), END_LIST},
+    {OP_lahf,  0x9f0000, "lahf", ah, xx, xx, xx, xx, no, (fR6&(~fRO)), END_LIST},
+    /* a0 */
+    {OP_mov_ld,  0xa00000, "mov", al, xx, Ob, xx, xx, no, x, tfb[0x8b]},
+    {OP_mov_ld,  0xa10000, "mov", eAX, xx, Ov, xx, xx, no, x, tfb[0xa0]},
+    {OP_mov_st,  0xa20000, "mov", Ob, xx, al, xx, xx, no, x, tfb[0x89]},
+    {OP_mov_st,  0xa30000, "mov", Ov, xx, eAX, xx, xx, no, x, tfb[0xa2]},
+    {REP_EXT, 0xa40000, "((rep) movs)", Yb, xx, Xb, xx, xx, no, fRD, 4},
+    {REP_EXT, 0xa50000, "((rep) movs)", Yv, xx, Xv, xx, xx, no, fRD, 5},
+    {REPNE_EXT, 0xa60000, "((rep/ne) cmps)", Xb, xx, Yb, xx, xx, no, (fW6|fRD), 0},
+    {REPNE_EXT, 0xa70000, "((rep/ne) cmps)", Xv, xx, Yv, xx, xx, no, (fW6|fRD), 1},
+    /* a8 */
+    {OP_test,  0xa80000, "test", xx, xx,  al, Ib, xx, no, fW6, tfb[0x85]},
+    {OP_test,  0xa90000, "test", xx, xx, eAX, Iz, xx, no, fW6, tfb[0xa8]},
+    {REP_EXT, 0xaa0000, "((rep) stos)", Yb, xx, al, xx, xx, no, fRD, 6},
+    {REP_EXT, 0xab0000, "((rep) stos)", Yv, xx, eAX, xx, xx, no, fRD, 7},
+    {REP_EXT, 0xac0000, "((rep) lods)", al, xx, Xb, xx, xx, no, fRD, 8},
+    {REP_EXT, 0xad0000, "((rep) lods)", eAX, xx, Xv, xx, xx, no, fRD, 9},
+    {REPNE_EXT, 0xae0000, "((rep/ne) scas)", al, xx, Yb, xx, xx, no, (fW6|fRD), 2},
+    {REPNE_EXT, 0xaf0000, "((rep/ne) scas)", eAX, xx, Yv, xx, xx, no, (fW6|fRD), 3},
+    /* b0 */
+    {OP_mov_imm, 0xb00000, "mov", al_x, xx, Ib, xx, xx, no, x, tfb[0xb1]},
+    {OP_mov_imm, 0xb10000, "mov", cl_x, xx, Ib, xx, xx, no, x, tfb[0xb2]},
+    {OP_mov_imm, 0xb20000, "mov", dl_x, xx, Ib, xx, xx, no, x, tfb[0xb3]},
+    {OP_mov_imm, 0xb30000, "mov", bl_x, xx, Ib, xx, xx, no, x, tfb[0xb4]},
+    {OP_mov_imm, 0xb40000, "mov", ah_x, xx, Ib, xx, xx, no, x, tfb[0xb5]},
+    {OP_mov_imm, 0xb50000, "mov", ch_x, xx, Ib, xx, xx, no, x, tfb[0xb6]},
+    {OP_mov_imm, 0xb60000, "mov", dh_x, xx, Ib, xx, xx, no, x, tfb[0xb7]},
+    /* PR 250397: we point at the tail end of the mov_st templates */
+    {OP_mov_imm, 0xb70000, "mov", bh_x, xx, Ib, xx, xx, no, x, tex[18][0]},
+    /* b8 */
+    {OP_mov_imm, 0xb80000, "mov", eAX_x, xx, Iv, xx, xx, no, x, tfb[0xb9]},
+    {OP_mov_imm, 0xb90000, "mov", eCX_x, xx, Iv, xx, xx, no, x, tfb[0xba]},
+    {OP_mov_imm, 0xba0000, "mov", eDX_x, xx, Iv, xx, xx, no, x, tfb[0xbb]},
+    {OP_mov_imm, 0xbb0000, "mov", eBX_x, xx, Iv, xx, xx, no, x, tfb[0xbc]},
+    {OP_mov_imm, 0xbc0000, "mov", eSP_x, xx, Iv, xx, xx, no, x, tfb[0xbd]},
+    {OP_mov_imm, 0xbd0000, "mov", eBP_x, xx, Iv, xx, xx, no, x, tfb[0xbe]},
+    {OP_mov_imm, 0xbe0000, "mov", eSI_x, xx, Iv, xx, xx, no, x, tfb[0xbf]},
+    {OP_mov_imm, 0xbf0000, "mov", eDI_x, xx, Iv, xx, xx, no, x, tfb[0xb0]},
+    /* c0 */
+    {EXTENSION, 0xc00000, "(group 2a)", Eb, xx, Ib, xx, xx, mrm, x, 3},
+    {EXTENSION, 0xc10000, "(group 2b)", Ev, xx, Ib, xx, xx, mrm, x, 4},
+    {OP_ret,  0xc20000, "ret", axSP, xx, Iw, axSP, i_iSP, no, x, tfb[0xc3]},
+    {OP_ret,  0xc30000, "ret", axSP, xx, axSP, i_iSP, xx, no, x, END_LIST},
+    {OP_les,  0xc40000, "les", Gz, es, Mp, xx, xx, mrm|i64, x, END_LIST},
+    {OP_lds,  0xc50000, "lds", Gz, ds, Mp, xx, xx, mrm|i64, x, END_LIST},
+    {EXTENSION, 0xc60000, "(group 11a)", Eb, xx, Ib, xx, xx, mrm, x, 17},
+    {EXTENSION, 0xc70000, "(group 11b)", Ev, xx, Iz, xx, xx, mrm, x, 18},
+    /* c8 */
+    {OP_enter,  0xc80000, "enter", axSP, i_xSP, Iw, Ib, axSP, xop, x, exop[0x05]},
+    {OP_leave,  0xc90000, "leave", axSP, axBP, axBP, axSP, i_xBP, no, x, END_LIST},
+    {OP_ret_far,  0xca0000, "lret", axSP, xx, Iw, axSP, i_vSP, no, x, tfb[0xcb]},
+    {OP_ret_far,  0xcb0000, "lret", axSP, xx, axSP, i_vSP, xx, no, x, END_LIST},
+    /* FIXME: int & iret push & pop on user stack, right?!? */
+    {OP_int3, 0xcc0000, "int3", axSP, i_xSP, axSP, xx, xx, no, fINT, END_LIST},
+    {OP_int,  0xcd0000, "int",  axSP, i_xSP, Ib, axSP, xx, no, fINT, END_LIST},
+    {OP_into, 0xce0000, "into", axSP, i_xSP, axSP, xx, xx, i64, fINT, END_LIST},
+    {OP_iret, 0xcf0000, "iret", axSP, xx, axSP, i_vSP, xx, no, fWX, END_LIST},
+    /* d0 */
+    {EXTENSION, 0xd00000, "(group 2c)", Eb, xx, c1,  xx, xx, mrm, x, 5},
+    {EXTENSION, 0xd10000, "(group 2d)", Ev, xx, c1,  xx, xx, mrm, x, 6},
+    {EXTENSION, 0xd20000, "(group 2e)", Eb, xx, cl, xx, xx, mrm, x, 7},
+    {EXTENSION, 0xd30000, "(group 2f)", Ev, xx, cl, xx, xx, mrm, x, 8},
+    {OP_aam,  0xd40000, "aam", ax, xx, Ib, ax, xx, i64, fW6, END_LIST},
+    {OP_aad,  0xd50000, "aad", ax, xx, Ib, ax, xx, i64, fW6, END_LIST},
+    {OP_salc,  0xd60000, "salc", al, xx, xx, xx, xx, i64, fRC, END_LIST},/*undocumented*/
+    {OP_xlat,  0xd70000, "xlat", al, xx, Zb, xx, xx, no, x, END_LIST},
+    /* d8 */
+    {FLOAT_EXT, 0xd80000, "(float)", xx, xx, xx, xx, xx, mrm, x, NA},/* all floats need modrm */
+    {FLOAT_EXT, 0xd90000, "(float)", xx, xx, xx, xx, xx, mrm, x, NA},
+    {FLOAT_EXT, 0xda0000, "(float)", xx, xx, xx, xx, xx, mrm, x, NA},
+    {FLOAT_EXT, 0xdb0000, "(float)", xx, xx, xx, xx, xx, mrm, x, NA},
+    {FLOAT_EXT, 0xdc0000, "(float)", xx, xx, xx, xx, xx, mrm, x, NA},
+    {FLOAT_EXT, 0xdd0000, "(float)", xx, xx, xx, xx, xx, mrm, x, NA},
+    {FLOAT_EXT, 0xde0000, "(float)", xx, xx, xx, xx, xx, mrm, x, NA},
+    {FLOAT_EXT, 0xdf0000, "(float)", xx, xx, xx, xx, xx, mrm, x, NA},
+    /* e0 */
+    {OP_loopne,0xe00000, "loopne", axCX, xx, Jb, axCX, xx, no, fRZ, END_LIST},
+    {OP_loope, 0xe10000, "loope",  axCX, xx, Jb, axCX, xx, no, fRZ, END_LIST},
+    {OP_loop,  0xe20000, "loop",   axCX, xx, Jb, axCX, xx, no, x, END_LIST},
+    {OP_jecxz, 0xe30000, "jecxz",  xx, xx, Jb, axCX, xx, no, x, END_LIST},/*16-bit=="jcxz",64-bit="jrcxz"*/
+    /* FIXME: in & out access "I/O ports", are these memory addresses?
+     * if so, change Ib to Ob and change dx to i_dx (move to dest for out)
+     */
+    {OP_in,  0xe40000, "in", al, xx, Ib, xx, xx, no, x, tfb[0xed]},
+    {OP_in,  0xe50000, "in", zAX, xx, Ib, xx, xx, no, x, tfb[0xe4]},
+    {OP_out,  0xe60000, "out", xx, xx, Ib, al, xx, no, x, tfb[0xef]},
+    {OP_out,  0xe70000, "out", xx, xx, Ib, zAX, xx, no, x, tfb[0xe6]},
+    /* e8 */
+    {OP_call,     0xe80000, "call",  axSP, i_iSP, Jz, axSP, xx, no, x, END_LIST},
+    {OP_jmp,       0xe90000, "jmp", xx, xx, Jz, xx, xx, no, x, END_LIST},
+    {OP_jmp_far,   0xea0000, "ljmp", xx, xx, Ap, xx, xx, i64, x, END_LIST},
+    {OP_jmp_short, 0xeb0000, "jmp", xx, xx, Jb, xx, xx, no, x, END_LIST},
+    {OP_in,  0xec0000, "in", al, xx, dx, xx, xx, no, x, END_LIST},
+    {OP_in,  0xed0000, "in", zAX, xx, dx, xx, xx, no, x, tfb[0xec]},
+    {OP_out,  0xee0000, "out", xx, xx, al, dx, xx, no, x, END_LIST},
+    {OP_out,  0xef0000, "out", xx, xx, zAX, dx, xx, no, x, tfb[0xee]},
+    /* f0 */
+    {PREFIX, 0xf00000, "lock", xx, xx, xx, xx, xx, no, x, PREFIX_LOCK},
+    /* Also called OP_icebp.  Undocumented.  I'm assuming looks like OP_int* */
+    {OP_int1, 0xf10000, "int1", axSP, i_xSP, axSP, xx, xx, no, fINT, END_LIST},
+    {PREFIX, 0xf20000, "repne", xx, xx, xx, xx, xx, no, x, PREFIX_REPNE},
+    {PREFIX, 0xf30000, "rep", xx, xx, xx, xx, xx, no, x, PREFIX_REP},
+    {OP_hlt,  0xf40000, "hlt", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_cmc,  0xf50000, "cmc", xx, xx, xx, xx, xx, no, fWC, END_LIST},
+    {EXTENSION, 0xf60000, "(group 3a)", Eb, xx, xx, xx, xx, mrm, x, 9},
+    {EXTENSION, 0xf70000, "(group 3b)", Ev, xx, xx, xx, xx, mrm, x, 10},
+    /* f8 */
+    {OP_clc,  0xf80000, "clc", xx, xx, xx, xx, xx, no, fWC, END_LIST},
+    {OP_stc,  0xf90000, "stc", xx, xx, xx, xx, xx, no, fWC, END_LIST},
+    {OP_cli,  0xfa0000, "cli", xx, xx, xx, xx, xx, no, fWI, END_LIST},
+    {OP_sti,  0xfb0000, "sti", xx, xx, xx, xx, xx, no, fWI, END_LIST},
+    {OP_cld,  0xfc0000, "cld", xx, xx, xx, xx, xx, no, fWD, END_LIST},
+    {OP_std,  0xfd0000, "std", xx, xx, xx, xx, xx, no, fWD, END_LIST},
+    {EXTENSION, 0xfe0000, "(group 4)", xx, xx, xx, xx, xx, mrm, x, 11},
+    {EXTENSION, 0xff0000, "(group 5)", xx, xx, xx, xx, xx, mrm, x, 12},
+};
+
+/****************************************************************************
+ * Two-byte opcodes
+ * This is from Tables A-4 & A-5
+ */
+const instr_info_t second_byte[] = {
+  /* 00 */
+  {EXTENSION, 0x0f0010, "(group 6)", xx, xx, xx, xx, xx, mrm, x, 13},
+  {EXTENSION, 0x0f0110, "(group 7)", xx, xx, xx, xx, xx, mrm, x, 14},
+  {OP_lar, 0x0f0210, "lar", Gv, xx, Ew, xx, xx, mrm, fWZ, END_LIST},
+  {OP_lsl, 0x0f0310, "lsl", Gv, xx, Ew, xx, xx, mrm, fWZ, END_LIST},
+  {INVALID, 0x0f0410, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  /* FIXME: writes ss and cs, we don't care, right? */
+  {OP_syscall, 0x0f0510, "syscall", xcx, xx, xx, xx, xx, no, x, NA}, /* AMD/x64 only */
+  {OP_clts, 0x0f0610, "clts", xx, xx, xx, xx, xx, no, x, END_LIST},
+  /* FIXME: writes ss and cs, we don't care, right? */
+  {OP_sysret, 0x0f0710, "sysret", xx, xx, xx, xx, xx, no, x, NA}, /* AMD/x64 only */
+  /* 08 */
+  {OP_invd, 0x0f0810, "invd", xx, xx, xx, xx, xx, no, x, END_LIST},
+  {OP_wbinvd, 0x0f0910, "wbinvd", xx, xx, xx, xx, xx, no, x, END_LIST},
+  {INVALID, 0x0f0a10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  {OP_ud2a, 0x0f0b10, "ud2a", xx, xx, xx, xx, xx, no, x, END_LIST}, /* "undefined instr" instr */
+  {INVALID, 0x0f0c10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  {EXTENSION, 0x0f0d10, "(group amd)", xx, xx, xx, xx, xx, mrm, x, 24}, /* AMD only */
+  {OP_femms, 0x0f0e10, "femms", xx, xx, xx, xx, xx, no, x, END_LIST},
+  {SUFFIX_EXT, 0x0f0f10, "(group 3DNow!)", xx, xx, xx, xx, xx, mrm, x, 0},
+  /* 10 */
+  {PREFIX_EXT, 0x0f1010, "(prefix ext 0)", xx, xx, xx, xx, xx, mrm, x, 0},
+  {PREFIX_EXT, 0x0f1110, "(prefix ext 1)", xx, xx, xx, xx, xx, mrm, x, 1},
+  {PREFIX_EXT, 0x0f1210, "(prefix ext 2)", xx, xx, xx, xx, xx, mrm, x, 2},
+  {PREFIX_EXT, 0x0f1310, "(prefix ext 3)", xx, xx, xx, xx, xx, mrm, x, 3},
+  {PREFIX_EXT, 0x0f1410, "(prefix ext 4)", xx, xx, xx, xx, xx, mrm, x, 4},
+  {PREFIX_EXT, 0x0f1510, "(prefix ext 5)", xx, xx, xx, xx, xx, mrm, x, 5},
+  {PREFIX_EXT, 0x0f1610, "(prefix ext 6)", xx, xx, xx, xx, xx, mrm, x, 6},
+  {PREFIX_EXT, 0x0f1710, "(prefix ext 7)", xx, xx, xx, xx, xx, mrm, x, 7},
+  /* 18 */
+  {EXTENSION, 0x0f1810, "(group 16)", xx, xx, xx, xx, xx, mrm, x, 23},
+  /* xref case 9862/PR 214297 : 0f19-0f1e are "HINT_NOP": valid on P6+.
+   * we treat them the same as 0f1f but do not put on encoding chain.
+   * The operand is ignored but to support encoding it we must list it.
+   * FIXME PR 305265: should treat as NOT a mem/reg read for analysis routines. */
+  {OP_nop_modrm, 0x0f1910, "nop", xx, xx, Ed, xx, xx, mrm, x, END_LIST},
+  {OP_nop_modrm, 0x0f1a10, "nop", xx, xx, Ed, xx, xx, mrm, x, END_LIST},
+  {OP_nop_modrm, 0x0f1b10, "nop", xx, xx, Ed, xx, xx, mrm, x, END_LIST},
+  {OP_nop_modrm, 0x0f1c10, "nop", xx, xx, Ed, xx, xx, mrm, x, END_LIST},
+  {OP_nop_modrm, 0x0f1d10, "nop", xx, xx, Ed, xx, xx, mrm, x, END_LIST},
+  {OP_nop_modrm, 0x0f1e10, "nop", xx, xx, Ed, xx, xx, mrm, x, END_LIST},
+  {OP_nop_modrm, 0x0f1f10, "nop", xx, xx, Ed, xx, xx, mrm, x, END_LIST},
+  /* 20 */
+  {OP_mov_priv, 0x0f2010, "mov", Rr, xx, Cr, xx, xx, mrm, fW6, tsb[0x21]},
+  {OP_mov_priv, 0x0f2110, "mov", Rr, xx, Dr, xx, xx, mrm, fW6, tsb[0x22]},
+  {OP_mov_priv, 0x0f2210, "mov", Cr, xx, Rr, xx, xx, mrm, fW6, tsb[0x23]},
+  {OP_mov_priv, 0x0f2310, "mov", Dr, xx, Rr, xx, xx, mrm, fW6, END_LIST},
+  {INVALID, 0x0f2410, "(bad)", xx, xx, xx, xx, xx, no, x, NA}, /* FIXME: gdb thinks ok! */
+  {INVALID, 0x0f2510, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  {INVALID, 0x0f2610, "(bad)", xx, xx, xx, xx, xx, no, x, NA}, /* FIXME: gdb thinks ok! */
+  {INVALID, 0x0f2710, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  /* 28 */
+  {PREFIX_EXT, 0x0f2810, "(prefix ext 8)", xx, xx, xx, xx, xx, mrm, x, 8},
+  {PREFIX_EXT, 0x0f2910, "(prefix ext 9)", xx, xx, xx, xx, xx, mrm, x, 9},
+  {PREFIX_EXT, 0x0f2a10, "(prefix ext 10)", xx, xx, xx, xx, xx, mrm, x, 10},
+  {PREFIX_EXT, 0x0f2b10, "(prefix ext 11)", xx, xx, xx, xx, xx, mrm, x, 11},
+  {PREFIX_EXT, 0x0f2c10, "(prefix ext 12)", xx, xx, xx, xx, xx, mrm, x, 12},
+  {PREFIX_EXT, 0x0f2d10, "(prefix ext 13)", xx, xx, xx, xx, xx, mrm, x, 13},
+  {PREFIX_EXT, 0x0f2e10, "(prefix ext 14)", xx, xx, xx, xx, xx, mrm, x, 14},
+  {PREFIX_EXT, 0x0f2f10, "(prefix ext 15)", xx, xx, xx, xx, xx, mrm, x, 15},
+  /* 30 */
+  {OP_wrmsr, 0x0f3010, "wrmsr", xx, xx, edx, eax, ecx, no, x, END_LIST},
+  {OP_rdtsc, 0x0f3110, "rdtsc", edx, eax, xx, xx, xx, no, x, END_LIST},
+  {OP_rdmsr, 0x0f3210, "rdmsr", edx, eax, ecx, xx, xx, no, x, END_LIST},
+  {OP_rdpmc, 0x0f3310, "rdpmc", edx, eax, ecx, xx, xx, no, x, END_LIST},
+  /* FIXME: sysenter writes cs and ss, but we don't care, right? */
+  {OP_sysenter, 0x0f3410, "sysenter", xsp, xx, xx, xx, xx, no, x, END_LIST},
+  /* FIXME: sysexit writes cs and ss and reads ecx, but we don't care, right? */
+  {OP_sysexit, 0x0f3510, "sysexit", xsp, xx, xx, xx, xx, no, x, END_LIST},
+  {INVALID, 0x0f3610, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  {INVALID, 0x0f3710, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  /* 38 */
+  {ESCAPE_3BYTE_38, 0x0f3810, "(3byte 38)", xx, xx, xx, xx, xx, no, x, NA},
+  {INVALID, 0x0f3910, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  {ESCAPE_3BYTE_3a, 0x0f3a10, "(3byte 3a)", xx, xx, xx, xx, xx, no, x, NA},
+  {INVALID, 0x0f3b10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  {INVALID, 0x0f3c10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  {INVALID, 0x0f3d10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  {INVALID, 0x0f3e10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  {INVALID, 0x0f3f10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  /* 40 */
+  /* cmovxx does not always write dest -- model that as a read of dest */
+  {OP_cmovo,  0x0f4010, "cmovo",  Gv, xx, Ev, Gv, xx, mrm, fRO, END_LIST},
+  {OP_cmovno, 0x0f4110, "cmovno", Gv, xx, Ev, Gv, xx, mrm, fRO, END_LIST},
+  {OP_cmovb,  0x0f4210, "cmovb",  Gv, xx, Ev, Gv, xx, mrm, fRC, END_LIST},
+  {OP_cmovnb, 0x0f4310, "cmovnb", Gv, xx, Ev, Gv, xx, mrm, fRC, END_LIST},
+  {OP_cmovz,  0x0f4410, "cmovz",  Gv, xx, Ev, Gv, xx, mrm, fRZ, END_LIST},
+  {OP_cmovnz, 0x0f4510, "cmovnz", Gv, xx, Ev, Gv, xx, mrm, fRZ, END_LIST},
+  {OP_cmovbe, 0x0f4610, "cmovbe", Gv, xx, Ev, Gv, xx, mrm, (fRC|fRZ), END_LIST},
+  {OP_cmovnbe,0x0f4710, "cmovnbe",Gv, xx, Ev, Gv, xx, mrm, (fRC|fRZ), END_LIST},
+  /* 48 */
+  {OP_cmovs,  0x0f4810, "cmovs",  Gv, xx, Ev, Gv, xx, mrm, fRS, END_LIST},
+  {OP_cmovns, 0x0f4910, "cmovns", Gv, xx, Ev, Gv, xx, mrm, fRS, END_LIST},
+  {OP_cmovp,  0x0f4a10, "cmovp",  Gv, xx, Ev, Gv, xx, mrm, fRP, END_LIST},
+  {OP_cmovnp, 0x0f4b10, "cmovnp", Gv, xx, Ev, Gv, xx, mrm, fRP, END_LIST},
+  {OP_cmovl,  0x0f4c10, "cmovl",  Gv, xx, Ev, Gv, xx, mrm, (fRS|fRO), END_LIST},
+  {OP_cmovnl, 0x0f4d10, "cmovnl", Gv, xx, Ev, Gv, xx, mrm, (fRS|fRO), END_LIST},
+  {OP_cmovle, 0x0f4e10, "cmovle", Gv, xx, Ev, Gv, xx, mrm, (fRS|fRO|fRZ), END_LIST},
+  {OP_cmovnle,0x0f4f10, "cmovnle",Gv, xx, Ev, Gv, xx, mrm, (fRS|fRO|fRZ), END_LIST},
+  /* 50 */
+  {PREFIX_EXT, 0x0f5010, "(prefix ext 16)", xx, xx, xx, xx, xx, mrm, x, 16},
+  {PREFIX_EXT, 0x0f5110, "(prefix ext 17)", xx, xx, xx, xx, xx, mrm, x, 17},
+  {PREFIX_EXT, 0x0f5210, "(prefix ext 18)", xx, xx, xx, xx, xx, mrm, x, 18},
+  {PREFIX_EXT, 0x0f5310, "(prefix ext 19)", xx, xx, xx, xx, xx, mrm, x, 19},
+  {PREFIX_EXT, 0x0f5410, "(prefix ext 20)", xx, xx, xx, xx, xx, mrm, x, 20},
+  {PREFIX_EXT, 0x0f5510, "(prefix ext 21)", xx, xx, xx, xx, xx, mrm, x, 21},
+  {PREFIX_EXT, 0x0f5610, "(prefix ext 22)", xx, xx, xx, xx, xx, mrm, x, 22},
+  {PREFIX_EXT, 0x0f5710, "(prefix ext 23)", xx, xx, xx, xx, xx, mrm, x, 23},
+  /* 58 */
+  {PREFIX_EXT, 0x0f5810, "(prefix ext 24)", xx, xx, xx, xx, xx, mrm, x, 24},
+  {PREFIX_EXT, 0x0f5910, "(prefix ext 25)", xx, xx, xx, xx, xx, mrm, x, 25},
+  {PREFIX_EXT, 0x0f5a10, "(prefix ext 26)", xx, xx, xx, xx, xx, mrm, x, 26},
+  {PREFIX_EXT, 0x0f5b10, "(prefix ext 27)", xx, xx, xx, xx, xx, mrm, x, 27},
+  {PREFIX_EXT, 0x0f5c10, "(prefix ext 28)", xx, xx, xx, xx, xx, mrm, x, 28},
+  {PREFIX_EXT, 0x0f5d10, "(prefix ext 29)", xx, xx, xx, xx, xx, mrm, x, 29},
+  {PREFIX_EXT, 0x0f5e10, "(prefix ext 30)", xx, xx, xx, xx, xx, mrm, x, 30},
+  {PREFIX_EXT, 0x0f5f10, "(prefix ext 31)", xx, xx, xx, xx, xx, mrm, x, 31},
+  /* 60 */
+  {PREFIX_EXT, 0x0f6010, "(prefix ext 32)", xx, xx, xx, xx, xx, mrm, x, 32},
+  {PREFIX_EXT, 0x0f6110, "(prefix ext 33)", xx, xx, xx, xx, xx, mrm, x, 33},
+  {PREFIX_EXT, 0x0f6210, "(prefix ext 34)", xx, xx, xx, xx, xx, mrm, x, 34},
+  {PREFIX_EXT, 0x0f6310, "(prefix ext 35)", xx, xx, xx, xx, xx, mrm, x, 35},
+  {PREFIX_EXT, 0x0f6410, "(prefix ext 36)", xx, xx, xx, xx, xx, mrm, x, 36},
+  {PREFIX_EXT, 0x0f6510, "(prefix ext 37)", xx, xx, xx, xx, xx, mrm, x, 37},
+  {PREFIX_EXT, 0x0f6610, "(prefix ext 38)", xx, xx, xx, xx, xx, mrm, x, 38},
+  {PREFIX_EXT, 0x0f6710, "(prefix ext 39)", xx, xx, xx, xx, xx, mrm, x, 39},
+  /* 68 */
+  {PREFIX_EXT, 0x0f6810, "(prefix ext 40)", xx, xx, xx, xx, xx, mrm, x, 40},
+  {PREFIX_EXT, 0x0f6910, "(prefix ext 41)", xx, xx, xx, xx, xx, mrm, x, 41},
+  {PREFIX_EXT, 0x0f6a10, "(prefix ext 42)", xx, xx, xx, xx, xx, mrm, x, 42},
+  {PREFIX_EXT, 0x0f6b10, "(prefix ext 43)", xx, xx, xx, xx, xx, mrm, x, 43},
+  {PREFIX_EXT, 0x0f6c10, "(prefix ext 44)", xx, xx, xx, xx, xx, mrm, x, 44},
+  {PREFIX_EXT, 0x0f6d10, "(prefix ext 45)", xx, xx, xx, xx, xx, mrm, x, 45},
+  {PREFIX_EXT, 0x0f6e10, "(prefix ext 46)", xx, xx, xx, xx, xx, mrm, x, 46},
+  {PREFIX_EXT, 0x0f6f10, "(prefix ext 112)", xx, xx, xx, xx, xx, mrm, x, 112},
+  /* 70 */
+  {PREFIX_EXT, 0x0f7010, "(prefix ext 47)", xx, xx, xx, xx, xx, mrm, x, 47},
+  {EXTENSION, 0x0f7110, "(group 12)", xx, xx, xx, xx, xx, mrm, x, 19},
+  {EXTENSION, 0x0f7210, "(group 13)", xx, xx, xx, xx, xx, mrm, x, 20},
+  {EXTENSION, 0x0f7310, "(group 14)", xx, xx, xx, xx, xx, mrm, x, 21},
+  {PREFIX_EXT, 0x0f7410, "(prefix ext 48)", xx, xx, xx, xx, xx, mrm, x, 48},
+  {PREFIX_EXT, 0x0f7510, "(prefix ext 49)", xx, xx, xx, xx, xx, mrm, x, 49},
+  {PREFIX_EXT, 0x0f7610, "(prefix ext 50)", xx, xx, xx, xx, xx, mrm, x, 50},
+  {OP_emms, 0x0f7710, "emms", xx, xx, xx, xx, xx, no, x, END_LIST},
+  /* 78 */
+  {PREFIX_EXT, 0x0f7810, "(prefix ext 134)", xx, xx, xx, xx, xx, mrm, x, 134},
+  {PREFIX_EXT, 0x0f7910, "(prefix ext 135)", xx, xx, xx, xx, xx, mrm, x, 135},
+  {INVALID, 0x0f7a10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  {INVALID, 0x0f7b10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  {PREFIX_EXT, 0x0f7c10, "(prefix ext 114)", xx, xx, xx, xx, xx, mrm, x, 114},
+  {PREFIX_EXT, 0x0f7d10, "(prefix ext 115)", xx, xx, xx, xx, xx, mrm, x, 115},
+  {PREFIX_EXT, 0x0f7e10, "(prefix ext 51)", xx, xx, xx, xx, xx, mrm, x, 51},
+  {PREFIX_EXT, 0x0f7f10, "(prefix ext 113)", xx, xx, xx, xx, xx, mrm, x, 113},
+  /* 80 */
+  {OP_jo,  0x0f8010, "jo",  xx, xx, Jz, xx, xx, no, fRO, END_LIST},
+  {OP_jno, 0x0f8110, "jno", xx, xx, Jz, xx, xx, no, fRO, END_LIST},
+  {OP_jb,  0x0f8210, "jb",  xx, xx, Jz, xx, xx, no, fRC, END_LIST},
+  {OP_jnb, 0x0f8310, "jnb", xx, xx, Jz, xx, xx, no, fRC, END_LIST},
+  {OP_jz,  0x0f8410, "jz",  xx, xx, Jz, xx, xx, no, fRZ, END_LIST},
+  {OP_jnz, 0x0f8510, "jnz", xx, xx, Jz, xx, xx, no, fRZ, END_LIST},
+  {OP_jbe, 0x0f8610, "jbe", xx, xx, Jz, xx, xx, no, (fRC|fRZ), END_LIST},
+  {OP_jnbe,0x0f8710, "jnbe",xx, xx, Jz, xx, xx, no, (fRC|fRZ), END_LIST},
+  /* 88 */
+  {OP_js,  0x0f8810, "js",  xx, xx, Jz, xx, xx, no, fRS, END_LIST},
+  {OP_jns, 0x0f8910, "jns", xx, xx, Jz, xx, xx, no, fRS, END_LIST},
+  {OP_jp,  0x0f8a10, "jp",  xx, xx, Jz, xx, xx, no, fRP, END_LIST},
+  {OP_jnp, 0x0f8b10, "jnp", xx, xx, Jz, xx, xx, no, fRP, END_LIST},
+  {OP_jl,  0x0f8c10, "jl",  xx, xx, Jz, xx, xx, no, (fRS|fRO), END_LIST},
+  {OP_jnl, 0x0f8d10, "jnl", xx, xx, Jz, xx, xx, no, (fRS|fRO), END_LIST},
+  {OP_jle, 0x0f8e10, "jle", xx, xx, Jz, xx, xx, no, (fRS|fRO|fRZ), END_LIST},
+  {OP_jnle,0x0f8f10, "jnle",xx, xx, Jz, xx, xx, no, (fRS|fRO|fRZ), END_LIST},
+  /* 90 */
+  {OP_seto,  0x0f9010, "seto",  Eb, xx, xx, xx, xx, mrm, fRO, END_LIST},
+  {OP_setno, 0x0f9110, "setno", Eb, xx, xx, xx, xx, mrm, fRO, END_LIST},
+  {OP_setb,  0x0f9210, "setb",  Eb, xx, xx, xx, xx, mrm, fRC, END_LIST},
+  {OP_setnb, 0x0f9310, "setnb", Eb, xx, xx, xx, xx, mrm, fRC, END_LIST},
+  {OP_setz,  0x0f9410, "setz",  Eb, xx, xx, xx, xx, mrm, fRZ, END_LIST},
+  {OP_setnz, 0x0f9510, "setnz", Eb, xx, xx, xx, xx, mrm, fRZ, END_LIST},
+  {OP_setbe, 0x0f9610, "setbe", Eb, xx, xx, xx, xx, mrm, (fRC|fRZ), END_LIST},
+  {OP_setnbe,0x0f9710, "setnbe",Eb, xx, xx, xx, xx, mrm, (fRC|fRZ), END_LIST},
+  /* 98 */
+  {OP_sets,  0x0f9810, "sets",  Eb, xx, xx, xx, xx, mrm, fRS, END_LIST},
+  {OP_setns, 0x0f9910, "setns", Eb, xx, xx, xx, xx, mrm, fRS, END_LIST},
+  {OP_setp,  0x0f9a10, "setp",  Eb, xx, xx, xx, xx, mrm, fRP, END_LIST},
+  {OP_setnp, 0x0f9b10, "setnp", Eb, xx, xx, xx, xx, mrm, fRP, END_LIST},
+  {OP_setl,  0x0f9c10, "setl",  Eb, xx, xx, xx, xx, mrm, (fRS|fRO), END_LIST},
+  {OP_setnl, 0x0f9d10, "setnl", Eb, xx, xx, xx, xx, mrm, (fRS|fRO), END_LIST},
+  {OP_setle, 0x0f9e10, "setle", Eb, xx, xx, xx, xx, mrm, (fRS|fRO|fRZ), END_LIST},
+  {OP_setnle,0x0f9f10, "setnle",Eb, xx, xx, xx, xx, mrm, (fRS|fRO|fRZ), END_LIST},
+  /* a0 */
+  {OP_push, 0x0fa010, "push", axSP, i_xSP, fs, axSP, xx, no, x, tsb[0xa8]},
+  {OP_pop,  0x0fa110, "pop", fs, axSP, axSP, i_xSP, xx, no, x, tsb[0xa9]},
+  {OP_cpuid, 0x0fa210, "cpuid", eax, ebx, eax, xx, xx, xop, x, exop[0x06]},
+  {OP_bt,   0x0fa310, "bt",   xx, xx, Ev, Gv, xx, mrm, fW6, tex[15][4]},
+  {OP_shld, 0x0fa410, "shld", Ev, xx, Gv, Ib, Ev, mrm, fW6, tsb[0xa5]},
+  {OP_shld, 0x0fa510, "shld", Ev, xx, Gv, cl, Ev, mrm, fW6, END_LIST},
+  {INVALID, 0x0fa610, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  {INVALID, 0x0fa710, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  /* a8 */
+  {OP_push, 0x0fa810, "push", axSP, i_xSP, gs, axSP, xx, no, x, END_LIST},
+  {OP_pop,  0x0fa910, "pop", gs, axSP, axSP, i_xSP, xx, no, x, END_LIST},
+  {OP_rsm,  0x0faa10, "rsm", xx, xx, xx, xx, xx, no, fWX, END_LIST},
+  {OP_bts,  0x0fab10, "bts", Ev, xx, Gv, Ev, xx, mrm, fW6, tex[15][5]},
+  {OP_shrd, 0x0fac10, "shrd", Ev, xx, Gv, Ib, Ev, mrm, fW6, tsb[0xad]},
+  {OP_shrd, 0x0fad10, "shrd", Ev, xx, Gv, cl, Ev, mrm, fW6, END_LIST},
+  {EXTENSION, 0x0fae10, "(group 15)", xx, xx, xx, xx, xx, mrm, x, 22},
+  {OP_imul, 0x0faf10, "imul", Gv, xx, Ev, Gv, xx, mrm, fW6, tfb[0x69]},
+  /* b0 */
+  {OP_cmpxchg, 0x0fb010, "cmpxchg", Eb, al, Gb, Eb, al, mrm, fW6, END_LIST},
+  {OP_cmpxchg, 0x0fb110, "cmpxchg", Ev, eAX, Gv, Ev, eAX, mrm, fW6, tsb[0xb0]},
+  {OP_lss, 0x0fb210, "lss", Gv, ss, Mp, xx, xx, mrm, x, END_LIST},
+  {OP_btr, 0x0fb310, "btr", Ev, xx, Gv, Ev, xx, mrm, fW6, tex[15][6]},
+  {OP_lfs, 0x0fb410, "lfs", Gv, fs, Mp, xx, xx, mrm, x, END_LIST},
+  {OP_lgs, 0x0fb510, "lgs", Gv, gs, Mp, xx, xx, mrm, x, END_LIST},
+  {OP_movzx, 0x0fb610, "movzx", Gv, xx, Eb, xx, xx, mrm, x, END_LIST},
+  {OP_movzx, 0x0fb710, "movzx", Gv, xx, Ew, xx, xx, mrm, x, tsb[0xb6]},
+  /* b8 */
+#ifdef IA32_ON_IA64
+  /* FIXME : unsure about this encoding */
+  /* had to define type Av for this to work, see decode.c/above etc.  */
+  /* is jump to absolute pc address, not realitve like all other jumps */
+  /* is like the pointer type except no segement change, no modrm byte */
+  /* should be 0x0fb800? no! 01 signals to encoder is 2 byte instruction */
+  {OP_jmpe_abs,  0x0fb810, "jmpe", xx, xx, Av, xx, xx, no, x, END_LIST}, 
+#else
+  {OP_popcnt, 0x0fb810, "popcnt", Gv, xx, Ev, xx, xx, mrm, fW6, END_LIST},
+#endif
+  /* This is Group 10, but all identical (ud2b) so no reason to split opcode by /reg */
+  {OP_ud2b, 0x0fb910, "ud2b", xx, xx, xx, xx, xx, no, x, END_LIST},
+  {EXTENSION, 0x0fba10, "(group 8)", xx, xx, xx, xx, xx, mrm, x, 15},
+  {OP_btc, 0x0fbb10, "btc", Ev, xx, Gv, Ev, xx, mrm, fW6, tex[15][7]},
+  {OP_bsf, 0x0fbc10, "bsf", Gv, xx, Ev, xx, xx, mrm, fW6, END_LIST},
+  {PREFIX_EXT, 0x0fbd10, "(prefix ext 136)", xx, xx, xx, xx, xx, mrm, x, 136},
+  {OP_movsx, 0x0fbe10, "movsx", Gv, xx, Eb, xx, xx, mrm, x, END_LIST},
+  {OP_movsx, 0x0fbf10, "movsx", Gv, xx, Ew, xx, xx, mrm, x, tsb[0xbe]},
+  /* c0 */
+  {OP_xadd, 0x0fc010, "xadd", Eb, Gb, Eb, Gb, xx, mrm, fW6, END_LIST},
+  {OP_xadd, 0x0fc110, "xadd", Ev, Gv, Ev, Gv, xx, mrm, fW6, tsb[0xc0]},
+  {PREFIX_EXT, 0x0fc210, "(prefix ext 52)", xx, xx, xx, xx, xx, mrm, x, 52},
+  {OP_movnti, 0x0fc310, "movnti", Md_q, xx, Gd_q, xx, xx, mrm, x, END_LIST},
+  {PREFIX_EXT, 0x0fc410, "(prefix ext 53)", xx, xx, xx, xx, xx, mrm, x, 53},
+  {PREFIX_EXT, 0x0fc510, "(prefix ext 54)", xx, xx, xx, xx, xx, mrm, x, 54},
+  {PREFIX_EXT, 0x0fc610, "(prefix ext 55)", xx, xx, xx, xx, xx, mrm, x, 55},
+  {EXTENSION, 0x0fc710, "(group 9)", xx, xx, xx, xx, xx, mrm, x, 16},
+  /* c8 */
+  {OP_bswap, 0x0fc810, "bswap", uAX_x, xx, uAX_x, xx, xx, no, x, tsb[0xc9]},
+  {OP_bswap, 0x0fc910, "bswap", uCX_x, xx, uCX_x, xx, xx, no, x, tsb[0xca]},
+  {OP_bswap, 0x0fca10, "bswap", uDX_x, xx, uDX_x, xx, xx, no, x, tsb[0xcb]},
+  {OP_bswap, 0x0fcb10, "bswap", uBX_x, xx, uBX_x, xx, xx, no, x, tsb[0xcc]},
+  {OP_bswap, 0x0fcc10, "bswap", uSP_x, xx, uSP_x, xx, xx, no, x, tsb[0xcd]},
+  {OP_bswap, 0x0fcd10, "bswap", uBP_x, xx, uBP_x, xx, xx, no, x, tsb[0xce]},
+  {OP_bswap, 0x0fce10, "bswap", uSI_x, xx, uSI_x, xx, xx, no, x, tsb[0xcf]},
+  {OP_bswap, 0x0fcf10, "bswap", uDI_x, xx, uDI_x, xx, xx, no, x, END_LIST},
+  /* d0 */
+  {PREFIX_EXT, 0x0fd010, "(prefix ext 116)", xx, xx, xx, xx, xx, mrm, x, 116},
+  {PREFIX_EXT, 0x0fd110, "(prefix ext 56)", xx, xx, xx, xx, xx, mrm, x, 56},
+  {PREFIX_EXT, 0x0fd210, "(prefix ext 57)", xx, xx, xx, xx, xx, mrm, x, 57},
+  {PREFIX_EXT, 0x0fd310, "(prefix ext 58)", xx, xx, xx, xx, xx, mrm, x, 58},
+  {PREFIX_EXT, 0x0fd410, "(prefix ext 59)", xx, xx, xx, xx, xx, mrm, x, 59},
+  {PREFIX_EXT, 0x0fd510, "(prefix ext 60)", xx, xx, xx, xx, xx, mrm, x, 60},
+  {PREFIX_EXT, 0x0fd610, "(prefix ext 61)", xx, xx, xx, xx, xx, mrm, x, 61},
+  {PREFIX_EXT, 0x0fd710, "(prefix ext 62)", xx, xx, xx, xx, xx, mrm, x, 62},
+  /* d8 */
+  {PREFIX_EXT, 0x0fd810, "(prefix ext 63)", xx, xx, xx, xx, xx, mrm, x, 63},
+  {PREFIX_EXT, 0x0fd910, "(prefix ext 64)", xx, xx, xx, xx, xx, mrm, x, 64},
+  {PREFIX_EXT, 0x0fda10, "(prefix ext 65)", xx, xx, xx, xx, xx, mrm, x, 65},
+  {PREFIX_EXT, 0x0fdb10, "(prefix ext 66)", xx, xx, xx, xx, xx, mrm, x, 66},
+  {PREFIX_EXT, 0x0fdc10, "(prefix ext 67)", xx, xx, xx, xx, xx, mrm, x, 67},
+  {PREFIX_EXT, 0x0fdd10, "(prefix ext 68)", xx, xx, xx, xx, xx, mrm, x, 68},
+  {PREFIX_EXT, 0x0fde10, "(prefix ext 69)", xx, xx, xx, xx, xx, mrm, x, 69},
+  {PREFIX_EXT, 0x0fdf10, "(prefix ext 70)", xx, xx, xx, xx, xx, mrm, x, 70},
+  /* e0 */
+  {PREFIX_EXT, 0x0fe010, "(prefix ext 71)", xx, xx, xx, xx, xx, mrm, x, 71},
+  {PREFIX_EXT, 0x0fe110, "(prefix ext 72)", xx, xx, xx, xx, xx, mrm, x, 72},
+  {PREFIX_EXT, 0x0fe210, "(prefix ext 73)", xx, xx, xx, xx, xx, mrm, x, 73},
+  {PREFIX_EXT, 0x0fe310, "(prefix ext 74)", xx, xx, xx, xx, xx, mrm, x, 74},
+  {PREFIX_EXT, 0x0fe410, "(prefix ext 75)", xx, xx, xx, xx, xx, mrm, x, 75},
+  {PREFIX_EXT, 0x0fe510, "(prefix ext 76)", xx, xx, xx, xx, xx, mrm, x, 76},
+  {PREFIX_EXT, 0x0fe610, "(prefix ext 77)", xx, xx, xx, xx, xx, mrm, x, 77},
+  {PREFIX_EXT, 0x0fe710, "(prefix ext 78)", xx, xx, xx, xx, xx, mrm, x, 78},
+  /* e8 */
+  {PREFIX_EXT, 0x0fe810, "(prefix ext 79)", xx, xx, xx, xx, xx, mrm, x, 79},
+  {PREFIX_EXT, 0x0fe910, "(prefix ext 80)", xx, xx, xx, xx, xx, mrm, x, 80},
+  {PREFIX_EXT, 0x0fea10, "(prefix ext 81)", xx, xx, xx, xx, xx, mrm, x, 81},
+  {PREFIX_EXT, 0x0feb10, "(prefix ext 82)", xx, xx, xx, xx, xx, mrm, x, 82},
+  {PREFIX_EXT, 0x0fec10, "(prefix ext 83)", xx, xx, xx, xx, xx, mrm, x, 83},
+  {PREFIX_EXT, 0x0fed10, "(prefix ext 84)", xx, xx, xx, xx, xx, mrm, x, 84},
+  {PREFIX_EXT, 0x0fee10, "(prefix ext 85)", xx, xx, xx, xx, xx, mrm, x, 85},
+  {PREFIX_EXT, 0x0fef10, "(prefix ext 86)", xx, xx, xx, xx, xx, mrm, x, 86},
+  /* f0 */
+  {PREFIX_EXT, 0x0ff010, "(prefix ext 117)", xx, xx, xx, xx, xx, mrm, x, 117},
+  {PREFIX_EXT, 0x0ff110, "(prefix ext 87)", xx, xx, xx, xx, xx, mrm, x, 87},
+  {PREFIX_EXT, 0x0ff210, "(prefix ext 88)", xx, xx, xx, xx, xx, mrm, x, 88},
+  {PREFIX_EXT, 0x0ff310, "(prefix ext 89)", xx, xx, xx, xx, xx, mrm, x, 89},
+  {PREFIX_EXT, 0x0ff410, "(prefix ext 90)", xx, xx, xx, xx, xx, mrm, x, 90},
+  {PREFIX_EXT, 0x0ff510, "(prefix ext 91)", xx, xx, xx, xx, xx, mrm, x, 91},
+  {PREFIX_EXT, 0x0ff610, "(prefix ext 92)", xx, xx, xx, xx, xx, mrm, x, 92},
+  {PREFIX_EXT, 0x0ff710, "(prefix ext 93)", xx, xx, xx, xx, xx, mrm, x, 93},
+  /* f8 */
+  {PREFIX_EXT, 0x0ff810, "(prefix ext 94)", xx, xx, xx, xx, xx, mrm, x, 94},
+  {PREFIX_EXT, 0x0ff910, "(prefix ext 95)", xx, xx, xx, xx, xx, mrm, x, 95},
+  {PREFIX_EXT, 0x0ffa10, "(prefix ext 96)", xx, xx, xx, xx, xx, mrm, x, 96},
+  {PREFIX_EXT, 0x0ffb10, "(prefix ext 97)", xx, xx, xx, xx, xx, mrm, x, 97},
+  {PREFIX_EXT, 0x0ffc10, "(prefix ext 98)", xx, xx, xx, xx, xx, mrm, x, 98},
+  {PREFIX_EXT, 0x0ffd10, "(prefix ext 99)", xx, xx, xx, xx, xx, mrm, x, 99},
+  {PREFIX_EXT, 0x0ffe10, "(prefix ext 100)", xx, xx, xx, xx, xx, mrm, x, 100},
+  {INVALID, 0x0fff10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+};
+
+/****************************************************************************
+ * Opcode extensions
+ * This is from Table A-6
+ */
+const instr_info_t extensions[][8] = {
+  /* group 1a -- first opcode byte 80: all assumed to have Ib */
+  { /* extensions[0] */
+    {OP_add, 0x800020, "add", Eb, xx, Ib, Eb, xx, mrm, fW6,  tex[25][0]},
+    {OP_or,  0x800021, "or",  Eb, xx, Ib, Eb, xx, mrm, fW6,  tex[25][1]},
+    {OP_adc, 0x800022, "adc", Eb, xx, Ib, Eb, xx, mrm, (fW6|fRC), tex[25][2]},
+    {OP_sbb, 0x800023, "sbb", Eb, xx, Ib, Eb, xx, mrm, (fW6|fRC), tex[25][3]},
+    {OP_and, 0x800024, "and", Eb, xx, Ib, Eb, xx, mrm, fW6,  tex[25][4]},
+    {OP_sub, 0x800025, "sub", Eb, xx, Ib, Eb, xx, mrm, fW6,  tex[25][5]},
+    {OP_xor, 0x800026, "xor", Eb, xx, Ib, Eb, xx, mrm, fW6,  tex[25][6]},
+    {OP_cmp, 0x800027, "cmp", xx, xx, Eb, Ib, xx, mrm, fW6,  tex[25][7]},
+ },
+  /* group 1b -- first opcode byte 81: all assumed to have Iz */
+  { /* extensions[1] */
+    {OP_add, 0x810020, "add", Ev, xx, Iz, Ev, xx, mrm, fW6,  tex[2][0]},
+    {OP_or,  0x810021, "or",  Ev, xx, Iz, Ev, xx, mrm, fW6,  tex[2][1]},
+    {OP_adc, 0x810022, "adc", Ev, xx, Iz, Ev, xx, mrm, (fW6|fRC), tex[2][2]},
+    {OP_sbb, 0x810023, "sbb", Ev, xx, Iz, Ev, xx, mrm, (fW6|fRC), tex[2][3]},
+    {OP_and, 0x810024, "and", Ev, xx, Iz, Ev, xx, mrm, fW6,  tex[2][4]},
+    {OP_sub, 0x810025, "sub", Ev, xx, Iz, Ev, xx, mrm, fW6,  tex[2][5]},
+    {OP_xor, 0x810026, "xor", Ev, xx, Iz, Ev, xx, mrm, fW6,  tex[2][6]},
+    {OP_cmp, 0x810027, "cmp", xx, xx, Ev, Iz, xx, mrm, fW6,  tex[2][7]},
+ },
+  /* group 1c -- first opcode byte 83 (for 82, see below "group 1c*"):
+   * all assumed to have Ib */
+  { /* extensions[2] */
+    {OP_add, 0x830020, "add", Ev, xx, Ib, Ev, xx, mrm, fW6,  tex[0][0]},
+    {OP_or,  0x830021, "or",  Ev, xx, Ib, Ev, xx, mrm, fW6,  tex[0][1]},
+    {OP_adc, 0x830022, "adc", Ev, xx, Ib, Ev, xx, mrm, (fW6|fRC), tex[0][2]},
+    {OP_sbb, 0x830023, "sbb", Ev, xx, Ib, Ev, xx, mrm, (fW6|fRC), tex[0][3]},
+    {OP_and, 0x830024, "and", Ev, xx, Ib, Ev, xx, mrm, fW6,  tex[0][4]},
+    {OP_sub, 0x830025, "sub", Ev, xx, Ib, Ev, xx, mrm, fW6,  tex[0][5]},
+    {OP_xor, 0x830026, "xor", Ev, xx, Ib, Ev, xx, mrm, fW6,  tex[0][6]},
+    {OP_cmp, 0x830027, "cmp", xx, xx, Ev, Ib, xx, mrm, fW6,  tex[0][7]},
+ },
+  /* group 2a -- first opcode byte c0: all assumed to have Ib */
+  { /* extensions[3] */
+    {OP_rol, 0xc00020, "rol", Eb, xx, Ib, Eb, xx, mrm, (fWC|fWO),  tex[5][0]},
+    {OP_ror, 0xc00021, "ror", Eb, xx, Ib, Eb, xx, mrm, (fWC|fWO),  tex[5][1]},
+    {OP_rcl, 0xc00022, "rcl", Eb, xx, Ib, Eb, xx, mrm, (fRC|fWC|fWO), tex[5][2]},
+    {OP_rcr, 0xc00023, "rcr", Eb, xx, Ib, Eb, xx, mrm, (fRC|fWC|fWO), tex[5][3]},
+    {OP_shl, 0xc00024, "shl", Eb, xx, Ib, Eb, xx, mrm, fW6,  tex[5][4]},
+    {OP_shr, 0xc00025, "shr", Eb, xx, Ib, Eb, xx, mrm, fW6,  tex[5][5]},
+    /* PR 332254: /6 is an alias for /4; we do not add to encoding chain though */
+    {OP_shl, 0xc00026, "shl", Eb, xx, Ib, Eb, xx, mrm, fW6,  END_LIST},
+    {OP_sar, 0xc00027, "sar", Eb, xx, Ib, Eb, xx, mrm, fW6,  tex[5][7]},
+ },
+  /* group 2b -- first opcode byte c1: all assumed to have Ib */
+  { /* extensions[4] */
+    {OP_rol, 0xc10020, "rol", Ev, xx, Ib, Ev, xx, mrm, (fWC|fWO),  tex[6][0]},
+    {OP_ror, 0xc10021, "ror", Ev, xx, Ib, Ev, xx, mrm, (fWC|fWO),  tex[6][1]},
+    {OP_rcl, 0xc10022, "rcl", Ev, xx, Ib, Ev, xx, mrm, (fRC|fWC|fWO), tex[6][2]},
+    {OP_rcr, 0xc10023, "rcr", Ev, xx, Ib, Ev, xx, mrm, (fRC|fWC|fWO), tex[6][3]},
+    {OP_shl, 0xc10024, "shl", Ev, xx, Ib, Ev, xx, mrm, fW6,  tex[6][4]},
+    {OP_shr, 0xc10025, "shr", Ev, xx, Ib, Ev, xx, mrm, fW6,  tex[6][5]},
+    /* PR 332254: /6 is an alias for /4; we do not add to encoding chain though */
+    {OP_shl, 0xc10026, "shl", Ev, xx, Ib, Ev, xx, mrm, fW6,  END_LIST},
+    {OP_sar, 0xc10027, "sar", Ev, xx, Ib, Ev, xx, mrm, fW6,  tex[6][7]},
+ },
+  /* group 2c -- first opcode byte d0 */
+  { /* extensions[5] */
+    {OP_rol, 0xd00020, "rol", Eb, xx, c1, Eb, xx, mrm, (fWC|fWO),  tex[8][0]},
+    {OP_ror, 0xd00021, "ror", Eb, xx, c1, Eb, xx, mrm, (fWC|fWO),  tex[8][1]},
+    {OP_rcl, 0xd00022, "rcl", Eb, xx, c1, Eb, xx, mrm, (fRC|fWC|fWO), tex[8][2]},
+    {OP_rcr, 0xd00023, "rcr", Eb, xx, c1, Eb, xx, mrm, (fRC|fWC|fWO), tex[8][3]},
+    {OP_shl, 0xd00024, "shl", Eb, xx, c1, Eb, xx, mrm, fW6,  tex[8][4]},
+    {OP_shr, 0xd00025, "shr", Eb, xx, c1, Eb, xx, mrm, fW6,  tex[8][5]},
+    /* PR 332254: /6 is an alias for /4; we do not add to encoding chain though */
+    {OP_shl, 0xd00026, "shl", Eb, xx, c1, Eb, xx, mrm, fW6,  END_LIST},
+    {OP_sar, 0xd00027, "sar", Eb, xx, c1, Eb, xx, mrm, fW6,  tex[8][7]},
+ },
+  /* group 2d -- first opcode byte d1 */
+  { /* extensions[6] */
+    {OP_rol, 0xd10020, "rol", Ev, xx, c1, Ev, xx, mrm, (fWC|fWO),  tex[3][0]},
+    {OP_ror, 0xd10021, "ror", Ev, xx, c1, Ev, xx, mrm, (fWC|fWO),  tex[3][1]},
+    {OP_rcl, 0xd10022, "rcl", Ev, xx, c1, Ev, xx, mrm, (fRC|fWC|fWO), tex[3][2]},
+    {OP_rcr, 0xd10023, "rcr", Ev, xx, c1, Ev, xx, mrm, (fRC|fWC|fWO), tex[3][3]},
+    {OP_shl, 0xd10024, "shl", Ev, xx, c1, Ev, xx, mrm, fW6,  tex[3][4]},
+    {OP_shr, 0xd10025, "shr", Ev, xx, c1, Ev, xx, mrm, fW6,  tex[3][5]},
+    /* PR 332254: /6 is an alias for /4; we do not add to encoding chain though */
+    {OP_shl, 0xd10026, "shl", Ev, xx, c1, Ev, xx, mrm, fW6,  END_LIST},
+    {OP_sar, 0xd10027, "sar", Ev, xx, c1, Ev, xx, mrm, fW6,  tex[3][7]},
+ },
+  /* group 2e -- first opcode byte d2 */
+  { /* extensions[7] */
+    {OP_rol, 0xd20020, "rol", Eb, xx, cl, Eb, xx, mrm, (fWC|fWO),  END_LIST},
+    {OP_ror, 0xd20021, "ror", Eb, xx, cl, Eb, xx, mrm, (fWC|fWO),  END_LIST},
+    {OP_rcl, 0xd20022, "rcl", Eb, xx, cl, Eb, xx, mrm, (fRC|fWC|fWO), END_LIST},
+    {OP_rcr, 0xd20023, "rcr", Eb, xx, cl, Eb, xx, mrm, (fRC|fWC|fWO), END_LIST},
+    {OP_shl, 0xd20024, "shl", Eb, xx, cl, Eb, xx, mrm, fW6,  END_LIST},
+    {OP_shr, 0xd20025, "shr", Eb, xx, cl, Eb, xx, mrm, fW6,  END_LIST},
+    /* PR 332254: /6 is an alias for /4; we do not add to encoding chain though */
+    {OP_shl, 0xd20026, "shl", Eb, xx, cl, Eb, xx, mrm, fW6,  END_LIST},
+    {OP_sar, 0xd20027, "sar", Eb, xx, cl, Eb, xx, mrm, fW6,  END_LIST},
+ },
+  /* group 2f -- first opcode byte d3 */
+  { /* extensions[8] */
+    {OP_rol, 0xd30020, "rol", Ev, xx, cl, Ev, xx, mrm, (fWC|fWO),  tex[7][0]},
+    {OP_ror, 0xd30021, "ror", Ev, xx, cl, Ev, xx, mrm, (fWC|fWO),  tex[7][1]},
+    {OP_rcl, 0xd30022, "rcl", Ev, xx, cl, Ev, xx, mrm, (fRC|fWC|fWO), tex[7][2]},
+    {OP_rcr, 0xd30023, "rcr", Ev, xx, cl, Ev, xx, mrm, (fRC|fWC|fWO), tex[7][3]},
+    {OP_shl, 0xd30024, "shl", Ev, xx, cl, Ev, xx, mrm, fW6,  tex[7][4]},
+    {OP_shr, 0xd30025, "shr", Ev, xx, cl, Ev, xx, mrm, fW6,  tex[7][5]},
+    /* PR 332254: /6 is an alias for /4; we do not add to encoding chain though */
+    {OP_shl, 0xd30026, "shl", Ev, xx, cl, Ev, xx, mrm, fW6,  END_LIST},
+    {OP_sar, 0xd30027, "sar", Ev, xx, cl, Ev, xx, mrm, fW6,  tex[7][7]},
+ },
+  /* group 3a -- first opcode byte f6 */
+  { /* extensions[9] */
+    {OP_test, 0xf60020, "test", xx, xx, Eb, Ib, xx, mrm, fW6, END_LIST},
+    /* PR 332254: /1 is an alias for /0; we do not add to encoding chain though */
+    {OP_test, 0xf60021, "test", xx, xx, Eb, Ib, xx, mrm, fW6, END_LIST},
+    {OP_not,  0xf60022, "not", Eb, xx, Eb, xx, xx, mrm, x, END_LIST},
+    {OP_neg,  0xf60023, "neg", Eb, xx, Eb, xx, xx, mrm, fW6, END_LIST},
+    {OP_mul,  0xf60024, "mul", ax, xx, Eb, al, xx, mrm, fW6, END_LIST},
+    {OP_imul, 0xf60025, "imul", ax, xx, Eb, al, xx, mrm, fW6, tsb[0xaf]},
+    {OP_div,  0xf60026, "div", ah, al, Eb, ax, xx, mrm, fW6, END_LIST},
+    {OP_idiv, 0xf60027, "idiv", ah, al, Eb, ax, xx, mrm, fW6, END_LIST},
+ },
+  /* group 3b -- first opcode byte f7 */
+  { /* extensions[10] */
+    {OP_test, 0xf70020, "test", xx,  xx, Ev, Iz, xx, mrm, fW6, tex[9][0]},
+    /* PR 332254: /1 is an alias for /0; we do not add to encoding chain though */
+    {OP_test, 0xf70021, "test", xx,  xx, Ev, Iz, xx, mrm, fW6, END_LIST},
+    {OP_not,  0xf70022, "not", Ev,  xx, Ev, xx, xx, mrm, x, tex[9][2]},
+    {OP_neg,  0xf70023, "neg", Ev,  xx, Ev, xx, xx, mrm, fW6, tex[9][3]},
+    {OP_mul,  0xf70024, "mul",   eDX, eAX, Ev, eAX, xx, mrm, fW6, tex[9][4]},
+    {OP_imul, 0xf70025, "imul",  eDX, eAX, Ev, eAX, xx, mrm, fW6, tex[9][5]},
+    {OP_div,  0xf70026, "div",   eDX, eAX, Ev, eDX, eAX, mrm, fW6, tex[9][6]},
+    {OP_idiv, 0xf70027, "idiv",  eDX, eAX, Ev, eDX, eAX, mrm, fW6, tex[9][7]},
+ },
+  /* group 4 (first byte fe) */
+  { /* extensions[11] */
+    {OP_inc, 0xfe0020, "inc", Eb, xx, Eb, xx, xx, mrm, (fW6&(~fWC)), END_LIST},
+    {OP_dec, 0xfe0021, "dec", Eb, xx, Eb, xx, xx, mrm, (fW6&(~fWC)), END_LIST},
+    {INVALID, 0xfe0022, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xfe0023, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xfe0024, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xfe0025, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xfe0026, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xfe0027, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+ },
+  /* group 5 (first byte ff) */
+  { /* extensions[12] */
+    {OP_inc, 0xff0020, "inc", Ev, xx, Ev, xx, xx, mrm, (fW6&(~fWC)), tex[11][0]},
+    {OP_dec, 0xff0021, "dec", Ev, xx, Ev, xx, xx, mrm, (fW6&(~fWC)), tex[11][1]},
+    {OP_call_ind,     0xff0022, "call",  axSP, i_iSP, i_Exi, axSP, xx, mrm, x, END_LIST},
+    /* Note how a far call's stack operand size matches far ret rather than call */
+    {OP_call_far_ind, 0xff0023, "lcall",  axSP, i_vSP, i_Ep, axSP, xx, mrm, x, END_LIST},
+    {OP_jmp_ind,      0xff0024, "jmp",  xx, xx, i_Exi, xx, xx, mrm, x, END_LIST},
+    {OP_jmp_far_ind,  0xff0025, "ljmp",  xx, xx, i_Ep, xx, xx, mrm, x, END_LIST},
+    {OP_push, 0xff0026, "push", axSP, i_xSP, Esv, axSP, xx, mrm, x, tfb[0x06]},
+    {INVALID, 0xff0027, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+ },
+  /* group 6 (first bytes 0f 00) */
+  { /* extensions[13] */
+    {OP_sldt, 0x0f0030, "sldt", Ew, xx, xx, xx, xx, mrm, x, END_LIST},
+    {OP_str,  0x0f0031, "str", Ew, xx, xx, xx, xx, mrm, x, END_LIST},
+    {OP_lldt, 0x0f0032, "lldt", xx, xx, Ew, xx, xx, mrm, x, END_LIST},
+    {OP_ltr,  0x0f0033, "ltr", xx, xx, Ew, xx, xx, mrm, x, END_LIST},
+    {OP_verr, 0x0f0034, "verr", xx, xx, Ew, xx, xx, mrm, fWZ, END_LIST},
+    {OP_verw, 0x0f0035, "verw", xx, xx, Ew, xx, xx, mrm, fWZ, END_LIST},
+#ifdef IA32_ON_IA64
+    {OP_jmpe, 0x0f0036, "jmpe", xx, xx, i_Ev, xx, xx, mrm, x, END_LIST},
+#else
+    {INVALID, 0x0f0036, "(bad)",xx, xx, xx, xx, xx, no, x, NA},
+#endif
+    {INVALID, 0x0f0037, "(bad)",xx, xx, xx, xx, xx, no, x, NA},
+ },
+  /* group 7 (first bytes 0f 01) */
+  { /* extensions[14] */
+    {MOD_EXT, 0x0f0130, "(group 7 mod ext 0)", xx, xx, xx, xx, xx, no, x, 0},
+    {MOD_EXT, 0x0f0131, "(group 7 mod ext 1)", xx, xx, xx, xx, xx, no, x, 1},
+    {OP_lgdt, 0x0f0132, "lgdt",  xx, xx, Ms, xx, xx, mrm, x, END_LIST},
+    {OP_lidt, 0x0f0133, "lidt",  xx, xx, Ms, xx, xx, mrm, x, END_LIST},
+    {OP_smsw, 0x0f0134, "smsw",  Ew, xx, xx, xx, xx, mrm, x, END_LIST},
+    {INVALID, 0x0f0135, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_lmsw, 0x0f0136, "lmsw",  xx, xx, Ew, xx, xx, mrm, x, END_LIST},
+    {MOD_EXT, 0x0f0137, "(group 7 mod ext 2)", xx, xx, xx, xx, xx, no, x, 2},
+  },
+  /* group 8 (first bytes 0f ba): all assumed to have Ib */
+  { /* extensions[15] */
+    {INVALID, 0x0fba30, "(bad)",xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x0fba31, "(bad)",xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x0fba32, "(bad)",xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x0fba33, "(bad)",xx, xx, xx, xx, xx, no, x, NA},
+    {OP_bt,  0x0fba34, "bt",    xx, xx, Ev, Ib, xx, mrm, fW6, END_LIST},
+    {OP_bts, 0x0fba35, "bts", Ev, xx, Ib, Ev, xx, mrm, fW6, END_LIST},
+    {OP_btr, 0x0fba36, "btr", Ev, xx, Ib, Ev, xx, mrm, fW6, END_LIST},
+    {OP_btc, 0x0fba37, "btc", Ev, xx, Ib, Ev, xx, mrm, fW6, END_LIST},
+  },
+  /* group 9 (first bytes 0f c7) */
+  { /* extensions[16] */
+    {INVALID, 0x0fc730, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_cmpxchg8b, 0x0fc731, "cmpxchg8b", Mq, eAX, Mq, eAX, eDX, mrm_xop, fWZ, exop[0x07]},/*"cmpxchg16b" w/ rex.w*/
+    {INVALID, 0x0fc732, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x0fc733, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x0fc734, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x0fc735, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {PREFIX_EXT, 0x0fc736, "(prefix ext 137)", xx, xx, xx, xx, xx, no, x, 137},
+    {OP_vmptrst, 0x0fc737, "vmptrst", Mq, xx, xx, xx, xx, mrm|o64, x, END_LIST},
+  },
+  /* group 10 is all ud2b and is not used by us since identical */
+  /* group 11a (first byte c6) */
+  { /* extensions[17] */
+    {OP_mov_st, 0xc60020, "mov", Eb, xx, Ib, xx, xx, mrm, x, END_LIST},
+    {INVALID, 0xc60021, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xc60022, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xc60023, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xc60024, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xc60025, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xc60026, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xc60027, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* group 11b (first byte c7) */
+  { /* extensions[18] */
+    /* PR 250397: be aware that mov_imm shares this tail end of mov_st templates */
+    {OP_mov_st, 0xc70020, "mov", Ev, xx, Iz, xx, xx, mrm, x, tex[17][0]},
+    {INVALID, 0xc70021, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xc70022, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xc70023, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xc70024, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xc70025, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xc70026, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xc70027, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* group 12 (first bytes 0f 71): all assumed to have Ib */
+  { /* extensions[19] */
+    {INVALID, 0x0f7130, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x0f7131, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {PREFIX_EXT, 0x0f7132, "(prefix ext 104)", xx, xx, xx, xx, xx, no, x, 104},
+    {INVALID, 0x0f7133, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {PREFIX_EXT, 0x0f7134, "(prefix ext 105)", xx, xx, xx, xx, xx, no, x, 105},
+    {INVALID, 0x0f7135, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {PREFIX_EXT, 0x0f7136, "(prefix ext 106)", xx, xx, xx, xx, xx, no, x, 106},
+    {INVALID, 0x0f7137, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+ },
+  /* group 13 (first bytes 0f 72): all assumed to have Ib */
+  { /* extensions[20] */
+    {INVALID, 0x0f7230, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x0f7231, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {PREFIX_EXT, 0x0f7232, "(prefix ext 107)", xx, xx, xx, xx, xx, no, x, 107},
+    {INVALID, 0x0f7233, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {PREFIX_EXT, 0x0f7234, "(prefix ext 108)", xx, xx, xx, xx, xx, no, x, 108},
+    {INVALID, 0x0f7235, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {PREFIX_EXT, 0x0f7236, "(prefix ext 109)", xx, xx, xx, xx, xx, no, x, 109},
+    {INVALID, 0x0f7237, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+ },
+  /* group 14 (first bytes 0f 73): all assumed to have Ib */
+  { /* extensions[21] */
+    {INVALID, 0x0f7330, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x0f7331, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {PREFIX_EXT, 0x0f7332, "(prefix ext 110)", xx, xx, xx, xx, xx, no, x, 110},
+    {PREFIX_EXT, 0x0f7333, "(prefix ext 101)", xx, xx, xx, xx, xx, no, x, 101},
+    {INVALID, 0x0f7334, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x0f7335, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {PREFIX_EXT, 0x0f7336, "(prefix ext 111)", xx, xx, xx, xx, xx, no, x, 111},
+    {PREFIX_EXT, 0x0f7337, "(prefix ext 102)", xx, xx, xx, xx, xx, no, x, 102},
+ },
+  /* group 15 (first bytes 0f ae) */
+  { /* extensions[22] */
+    /* Intel tables imply they may add opcodes in the mod=3 (non-mem) space in future */
+    {OP_fxsave,  0x0fae30, "fxsave",  Me, xx, xx, xx, xx, mrm, x, END_LIST},
+    {OP_fxrstor, 0x0fae31, "fxrstor", xx, xx, Me, xx, xx, mrm, x, END_LIST},
+    {OP_ldmxcsr, 0x0fae32, "ldmxcsr", xx, xx, Md, xx, xx, mrm, x, END_LIST},
+    {OP_stmxcsr, 0x0fae33, "stmxcsr", Md, xx, xx, xx, xx, mrm, x, END_LIST},
+    {INVALID, 0x0fae34, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_lfence, 0x0fae35, "lfence", xx, xx, xx, xx, xx, mrm, x, END_LIST},/*FIXME PR 239920: newer gdb thinks it's "lfence (bad)"?*/
+    {OP_mfence, 0x0fae36, "mfence", xx, xx, xx, xx, xx, mrm, x, END_LIST},
+    {MOD_EXT,   0x0fae37, "(group 7 mod ext 3)", xx, xx, xx, xx, xx, no, x, 3},
+ },
+  /* group 16 (first bytes 0f 18) */
+  { /* extensions[23] */
+    /* Intel tables imply they may add opcodes in the mod=3 (non-mem) space in future */
+    {OP_prefetchnta, 0x0f1830, "prefetchnta", xx, xx, Mb, xx, xx, mrm, x, END_LIST},
+    {OP_prefetcht0,  0x0f1831, "prefetcht0",  xx, xx, Mb, xx, xx, mrm, x, END_LIST},
+    {OP_prefetcht1,  0x0f1832, "prefetcht1",  xx, xx, Mb, xx, xx, mrm, x, END_LIST},
+    {OP_prefetcht2,  0x0f1833, "prefetcht2",  xx, xx, Mb, xx, xx, mrm, x, END_LIST},
+    {INVALID, 0x0f1834, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x0f1835, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x0f1836, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x0f1837, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+ },
+  /* group AMD (first bytes 0f 0d) */
+  { /* extensions[24] */
+    {OP_prefetch,  0x0f0d30, "prefetch",  xx, xx, Mb, xx, xx, mrm, x, END_LIST},
+    {OP_prefetchw, 0x0f0d31, "prefetchw", xx, xx, Mb, xx, xx, mrm, x, END_LIST},
+    {INVALID, 0x0f0d32, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x0f0d33, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x0f0d34, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x0f0d35, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x0f0d36, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x0f0d37, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* group 1c* -- first opcode byte 82
+   * see PR 235092 for the discrepancies in what 0x82 should be: empirically
+   * and according to recent Intel manuals it matches 0x80, not 0x83 (as old
+   * Intel manuals implied) or invalid (as gnu tools claim).
+   * not linked into any encode chain.
+   */
+  { /* extensions[25]: all assumed to have Ib */
+    {OP_add, 0x820020, "add", Eb, xx, Ib, Eb, xx, mrm|i64, fW6,  END_LIST},
+    {OP_or,  0x820021, "or",  Eb, xx, Ib, Eb, xx, mrm|i64, fW6,  END_LIST},
+    {OP_adc, 0x820022, "adc", Eb, xx, Ib, Eb, xx, mrm|i64, (fW6|fRC), END_LIST},
+    {OP_sbb, 0x820023, "sbb", Eb, xx, Ib, Eb, xx, mrm|i64, (fW6|fRC), END_LIST},
+    {OP_and, 0x820024, "and", Eb, xx, Ib, Eb, xx, mrm|i64, fW6,  END_LIST},
+    {OP_sub, 0x820025, "sub", Eb, xx, Ib, Eb, xx, mrm|i64, fW6,  END_LIST},
+    {OP_xor, 0x820026, "xor", Eb, xx, Ib, Eb, xx, mrm|i64, fW6,  END_LIST},
+    {OP_cmp, 0x820027, "cmp", xx, xx, Eb, Ib, xx, mrm|i64, fW6,  END_LIST},
+  },
+  /* group 1d (Intel now calling Group 1A) -- first opcode byte 8f */
+  { /* extensions[26] */
+    {OP_pop,  0x8f0020, "pop", Esv, axSP, axSP, i_xSP, xx, mrm, x, tfb[0x17]},
+    {INVALID, 0x8f0021, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x8f0022, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x8f0023, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x8f0024, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x8f0025, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x8f0026, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0x8f0027, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+};
+
+/****************************************************************************
+ * Two-byte instructions that differ depending on presence of 
+ * prefixes, indexed in this order: 
+ *   none, 0xf3, 0x66, 0xf2
+ */
+const instr_info_t prefix_extensions[][4] = {
+  /* prefix extension 0 */
+  {
+    {OP_movups, 0x0f1010, "movups", Vps, xx, Wps, xx, xx, mrm, x, tpe[1][0]},
+    {OP_movss,  0xf30f1010, "movss",  Vss, xx, Wss, xx, xx, mrm, x, tpe[1][1]},
+    {OP_movupd, 0x660f1010, "movupd", Vpd, xx, Wpd, xx, xx, mrm, x, tpe[1][2]},
+    {OP_movsd,  0xf20f1010, "movsd",  Vsd, xx, Wsd, xx, xx, mrm, x, tpe[1][3]},
+  },
+  /* prefix extension 1 */
+  {
+    {OP_movups, 0x0f1110, "movups", Wps, xx, Vps, xx, xx, mrm, x, END_LIST},
+    {OP_movss,  0xf30f1110, "movss",  Wss, xx, Vss, xx, xx, mrm, x, END_LIST},
+    {OP_movupd, 0x660f1110, "movupd", Wpd, xx, Vpd, xx, xx, mrm, x, END_LIST},
+    {OP_movsd,  0xf20f1110, "movsd",  Wsd, xx, Vsd, xx, xx, mrm, x, END_LIST},
+  },
+  /* prefix extension 2 */
+  {
+    {OP_movlps, 0x0f1210, "movlps", Vq_dq, xx, Mq, xx, xx, mrm, x, tpe[3][0]}, /*"movhlps" if reg-reg */
+    {OP_movsldup, 0xf30f1210, "movsldup", Vps, xx, Wps, xx, xx, mrm, x, END_LIST},
+    {OP_movlpd, 0x660f1210, "movlpd", Vq_dq, xx, Mq, xx, xx, mrm, x, tpe[3][1]}, 
+    {OP_movddup, 0xf20f1210, "movddup", Vpd, xx, Wq, xx, xx, mrm, x, END_LIST},
+  },
+  /* prefix extension 3 */
+  {
+    {OP_movlps, 0x0f1310, "movlps", Mq, xx, Vq_dq, xx, xx, mrm, x, END_LIST},
+    {INVALID, 0x00000000, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_movlpd, 0x660f1310, "movlpd", Mq, xx, Vq_dq, xx, xx, mrm, x, END_LIST},
+    {INVALID, 0x00000000, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 4 */
+  {
+    {OP_unpcklps, 0x0f1410, "unpcklps", Vps, xx, Wq, Vps, xx, mrm, x, END_LIST},
+    {INVALID, 0x00000000, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_unpcklpd, 0x660f1410, "unpcklpd", Vpd, xx, Wq, Vpd, xx, mrm, x, END_LIST},
+    {INVALID, 0x00000000, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 5 */
+  {
+    {OP_unpckhps, 0x0f1510, "unpckhps", Vps, xx, Wq, Vps, xx, mrm, x, END_LIST},
+    {INVALID, 0x00000000, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_unpckhpd, 0x660f1510, "unpckhpd", Vpd, xx, Wq, Vpd, xx, mrm, x, END_LIST},
+    {INVALID, 0x00000000, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 6 */
+  {
+    {OP_movhps, 0x0f1610, "movhps", Vq_dq, xx, Mq, xx, xx, mrm, x, tpe[7][0]}, /*"movlhps" if reg-reg */
+    {OP_movshdup, 0xf30f1610, "movshdup", Vps, xx, Wps, xx, xx, mrm, x, END_LIST},
+    {OP_movhpd, 0x660f1610, "movhpd", Vq_dq, xx, Mq, xx, xx, mrm, x, tpe[7][1]},
+    {INVALID, 0x00000000, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 7 */
+  {
+    {OP_movhps, 0x0f1710, "movhps", Mq, xx, Vq_dq, xx, xx, mrm, x, END_LIST},
+    {INVALID, 0x00000000, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_movhpd, 0x660f1710, "movhpd", Mq, xx, Vq_dq, xx, xx, mrm, x, END_LIST},
+    {INVALID, 0x00000000, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 8 */
+  {
+    {OP_movaps, 0x0f2810, "movaps", Vps, xx, Wps, xx, xx, mrm, x, tpe[9][0]},
+    {INVALID, 0x00000000, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_movapd, 0x660f2810, "movapd", Vpd, xx, Wpd, xx, xx, mrm, x, tpe[9][1]},
+    {INVALID, 0x00000000, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 9 */
+  {
+    {OP_movaps, 0x0f2910, "movaps", Wps, xx, Vps, xx, xx, mrm, x, END_LIST},
+    {INVALID, 0x00000000, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_movapd, 0x660f2910, "movapd", Wpd, xx, Vpd, xx, xx, mrm, x, END_LIST},
+    {INVALID, 0x00000000, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 10 */
+  {
+    {OP_cvtpi2ps,  0x0f2a10, "cvtpi2ps", Vps, xx, Qq, xx, xx, mrm, x, END_LIST},
+    {OP_cvtsi2ss, 0xf30f2a10, "cvtsi2ss", Vss, xx, Ed_q, xx, xx, mrm, x, END_LIST},
+    {OP_cvtpi2pd, 0x660f2a10, "cvtpi2pd", Vpd, xx, Qq, xx, xx, mrm, x, END_LIST},
+    {OP_cvtsi2sd, 0xf20f2a10, "cvtsi2sd", Vsd, xx, Ed_q, xx, xx, mrm, x, END_LIST},
+  },
+  /* prefix extension 11 */
+  {
+    {OP_movntps,   0x0f2b10, "movntps", Mps, xx, Vps, xx, xx, mrm, x, END_LIST},
+    {OP_movntss, 0xf30f2b10, "movntss", Mss, xx, Vss, xx, xx, mrm, x, END_LIST},
+    {OP_movntpd, 0x660f2b10, "movntpd", Mpd, xx, Vpd, xx, xx, mrm, x, END_LIST},
+    {OP_movntsd, 0xf20f2b10, "movntsd", Msd, xx, Vsd, xx, xx, mrm, x, END_LIST},
+  },
+  /* prefix extension 12 */
+  {
+    {OP_cvttps2pi, 0x0f2c10, "cvttps2pi", Pq, xx, Wps, xx, xx, mrm, x, END_LIST},
+    {OP_cvttss2si, 0xf30f2c10, "cvttss2si", Gd, xx, Wss, xx, xx, mrm, x, END_LIST},
+    {OP_cvttpd2pi, 0x660f2c10, "cvttpd2pi", Pq, xx, Wpd, xx, xx, mrm, x, END_LIST},
+    {OP_cvttsd2si, 0xf20f2c10, "cvttsd2si", Gd, xx, Wsd, xx, xx, mrm, x, END_LIST},
+  },
+  /* prefix extension 13 */
+  {
+    {OP_cvtps2pi, 0x0f2d10, "cvtps2pi", Pq, xx, Wps, xx, xx, mrm, x, END_LIST},
+    {OP_cvtss2si, 0xf30f2d10, "cvtss2si", Gd_q, xx, Wss, xx, xx, mrm, x, END_LIST},
+    {OP_cvtpd2pi, 0x660f2d10, "cvtpd2pi", Pq, xx, Wpd, xx, xx, mrm, x, END_LIST},
+    {OP_cvtsd2si, 0xf20f2d10, "cvtsd2si", Gd_q, xx, Wsd, xx, xx, mrm, x, END_LIST},
+  },
+  /* prefix extension 14 */
+  {
+    {OP_ucomiss, 0x0f2e10, "ucomiss", xx, xx, Vss, Wss, xx, mrm, fW6, END_LIST},
+    {INVALID, 0xf30f2e10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_ucomisd, 0x660f2e10, "ucomisd", xx, xx, Vsd, Wsd, xx, mrm, fW6, END_LIST},
+    {INVALID, 0xf20f2e10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 15 */
+  {
+    {OP_comiss,  0x0f2f10, "comiss",  xx, xx, Vps, Wps, xx, mrm, fW6, END_LIST},
+    {INVALID, 0xf30f2f10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_comisd,  0x660f2f10, "comisd",  xx, xx, Vpd, Wpd, xx, mrm, fW6, END_LIST},
+    {INVALID, 0xf20f2f10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 16 */
+  {
+    {OP_movmskps, 0x0f5010, "movmskps", Gd, xx, Ups, xx, xx, mrm, x, END_LIST},
+    {INVALID, 0xf30f5010, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_movmskpd, 0x660f5010, "movmskpd", Gd, xx, Upd, xx, xx, mrm, x, END_LIST},
+    {INVALID, 0xf20f5010, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 17 */
+  {
+    {OP_sqrtps, 0x0f5110, "sqrtps", Vps, xx, Wps, xx, xx, mrm, x, END_LIST},
+    {OP_sqrtss, 0xf30f5110, "sqrtss", Vss, xx, Wss, xx, xx, mrm, x, END_LIST},
+    {OP_sqrtpd, 0x660f5110, "sqrtpd", Vpd, xx, Wpd, xx, xx, mrm, x, END_LIST},
+    {OP_sqrtsd, 0xf20f5110, "sqrtsd", Vsd, xx, Wsd, xx, xx, mrm, x, END_LIST},
+  },
+  /* prefix extension 18 */
+  {
+    {OP_rsqrtps, 0x0f5210, "rsqrtps", Vps, xx, Wps, xx, xx, mrm, x, END_LIST},
+    {OP_rsqrtss, 0xf30f5210, "rsqrtss", Vss, xx, Wss, xx, xx, mrm, x, END_LIST},
+    {INVALID, 0x660f5210, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xf20f5210, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 19 */
+  {
+    {OP_rcpps, 0x0f5310, "rcpps", Vps, xx, Wps, xx, xx, mrm, x, END_LIST},
+    {OP_rcpss, 0xf30f5310, "rcpss", Vss, xx, Wss, xx, xx, mrm, x, END_LIST},
+    {INVALID, 0x660f5310, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xf20f5310, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 20 */
+  {
+    {OP_andps,  0x0f5410, "andps",  Vps, xx, Wps, Vps, xx, mrm, x, END_LIST},
+    {INVALID, 0xf30f5410, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_andpd,  0x660f5410, "andpd",  Vpd, xx, Wpd, Vpd, xx, mrm, x, END_LIST},
+    {INVALID, 0xf20f5410, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 21 */
+  {
+    {OP_andnps, 0x0f5510, "andnps", Vps, xx, Wps, Vps, xx, mrm, x, END_LIST},
+    {INVALID, 0xf30f5510, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_andnpd, 0x660f5510, "andnpd", Vpd, xx, Wpd, Vpd, xx, mrm, x, END_LIST},
+    {INVALID, 0xf20f5510, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 22 */
+  {
+    {OP_orps,   0x0f5610, "orps",   Vps, xx, Wps, Vps, xx, mrm, x, END_LIST},
+    {INVALID, 0xf30f5610, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_orpd,   0x660f5610, "orpd",   Vpd, xx, Wpd, Vpd, xx, mrm, x, END_LIST},
+    {INVALID, 0xf20f5610, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 23 */
+  {
+    {OP_xorps,  0x0f5710, "xorps",  Vps, xx, Wps, Vps, xx, mrm, x, END_LIST},
+    {INVALID, 0xf30f5710, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_xorpd,  0x660f5710, "xorpd",  Vpd, xx, Wpd, Vpd, xx, mrm, x, END_LIST},
+    {INVALID, 0xf20f5710, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 24 */
+  {
+    {OP_addps, 0x0f5810, "addps", Vps, xx, Wps, Vps, xx, mrm, x, END_LIST},
+    {OP_addss, 0xf30f5810, "addss", Vss, xx, Wss, Vss, xx, mrm, x, END_LIST},
+    {OP_addpd, 0x660f5810, "addpd", Vpd, xx, Wpd, Vpd, xx, mrm, x, END_LIST},
+    {OP_addsd, 0xf20f5810, "addsd", Vsd, xx, Wsd, Vsd, xx, mrm, x, END_LIST},
+  },
+  /* prefix extension 25 */
+  {
+    {OP_mulps, 0x0f5910, "mulps", Vps, xx, Wps, Vps, xx, mrm, x, END_LIST},
+    {OP_mulss, 0xf30f5910, "mulss", Vss, xx, Wss, Vss, xx, mrm, x, END_LIST},
+    {OP_mulpd, 0x660f5910, "mulpd", Vpd, xx, Wpd, Vpd, xx, mrm, x, END_LIST},
+    {OP_mulsd, 0xf20f5910, "mulsd", Vsd, xx, Wsd, Vsd, xx, mrm, x, END_LIST},
+  },
+  /* prefix extension 26 */
+  {
+    {OP_cvtps2pd, 0x0f5a10, "cvtps2pd", Vpd, xx, Wps, xx, xx, mrm, x, END_LIST},
+    {OP_cvtss2sd, 0xf30f5a10, "cvtss2sd", Vsd, xx, Wss, xx, xx, mrm, x, END_LIST},
+    {OP_cvtpd2ps, 0x660f5a10, "cvtpd2ps", Vps, xx, Wpd, xx, xx, mrm, x, END_LIST},
+    {OP_cvtsd2ss, 0xf20f5a10, "cvtsd2ss", Vss, xx, Wsd, xx, xx, mrm, x, END_LIST},
+  },
+  /* prefix extension 27 */
+  {
+    {OP_cvtdq2ps, 0x0f5b10, "cvtdq2ps", Vps, xx, Wq, xx, xx, mrm, x, END_LIST},
+    {OP_cvttps2dq, 0xf30f5b10, "cvttps2dq", Vq, xx, Wps, xx, xx, mrm, x, END_LIST},
+    {OP_cvtps2dq, 0x660f5b10, "cvtps2dq", Vq, xx, Wps, xx, xx, mrm, x, END_LIST},
+    {INVALID, 0xf20f5b10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 28 */
+  {
+    {OP_subps, 0x0f5c10, "subps", Vps, xx, Wps, Vps, xx, mrm, x, END_LIST},
+    {OP_subss, 0xf30f5c10, "subss", Vss, xx, Wss, Vss, xx, mrm, x, END_LIST},
+    {OP_subpd, 0x660f5c10, "subpd", Vpd, xx, Wpd, Vpd, xx, mrm, x, END_LIST},
+    {OP_subsd, 0xf20f5c10, "subsd", Vsd, xx, Wsd, Vsd, xx, mrm, x, END_LIST},
+  },
+  /* prefix extension 29 */
+  {
+    {OP_minps, 0x0f5d10, "minps", Vps, xx, Wps, Vps, xx, mrm, x, END_LIST},
+    {OP_minss, 0xf30f5d10, "minss", Vss, xx, Wss, Vss, xx, mrm, x, END_LIST},
+    {OP_minpd, 0x660f5d10, "minpd", Vpd, xx, Wpd, Vpd, xx, mrm, x, END_LIST},
+    {OP_minsd, 0xf20f5d10, "minsd", Vsd, xx, Wsd, Vsd, xx, mrm, x, END_LIST},
+  },
+  /* prefix extension 30 */
+  {
+    {OP_divps, 0x0f5e10, "divps", Vps, xx, Wps, Vps, xx, mrm, x, END_LIST},
+    {OP_divss, 0xf30f5e10, "divss", Vss, xx, Wss, Vss, xx, mrm, x, END_LIST},
+    {OP_divpd, 0x660f5e10, "divpd", Vpd, xx, Wpd, Vpd, xx, mrm, x, END_LIST},
+    {OP_divsd, 0xf20f5e10, "divsd", Vsd, xx, Wsd, Vsd, xx, mrm, x, END_LIST},
+  },
+  /* prefix extension 31 */
+  {
+    {OP_maxps, 0x0f5f10, "maxps", Vps, xx, Wps, Vps, xx, mrm, x, END_LIST},
+    {OP_maxss, 0xf30f5f10, "maxss", Vss, xx, Wss, Vss, xx, mrm, x, END_LIST},
+    {OP_maxpd, 0x660f5f10, "maxpd", Vpd, xx, Wpd, Vpd, xx, mrm, x, END_LIST},
+    {OP_maxsd, 0xf20f5f10, "maxsd", Vsd, xx, Wsd, Vsd, xx, mrm, x, END_LIST},
+  },
+  /* prefix extension 32 */
+  {
+    {OP_punpcklbw,   0x0f6010, "punpcklbw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[32][2]},
+    {INVALID,      0xf30f6010, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_punpcklbw, 0x660f6010, "punpcklbw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf20f6010, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 33 */
+  {
+    {OP_punpcklwd,   0x0f6110, "punpcklwd", Pq, xx, Qq, Pq, xx, mrm, x, tpe[33][2]},
+    {INVALID,      0xf30f6110, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_punpcklwd, 0x660f6110, "punpcklwd", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf20f6110, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 34 */
+  {
+    {OP_punpckldq,   0x0f6210, "punpckldq", Pq, xx, Qq, Pq, xx, mrm, x, tpe[34][2]},
+    {INVALID,      0xf30f6210, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_punpckldq, 0x660f6210, "punpckldq", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf20f6210, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 35 */
+  {
+    {OP_packsswb,   0x0f6310, "packsswb", Pq, xx, Qq, Pq, xx, mrm, x, tpe[35][2]},
+    {INVALID,     0xf30f6310, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_packsswb, 0x660f6310, "packsswb", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,     0xf20f6310, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 36 */
+  {
+    {OP_pcmpgtb,   0x0f6410, "pcmpgtb", Pq, xx, Qq, Pq, xx, mrm, x, tpe[36][2]},
+    {INVALID,    0xf30f6410, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pcmpgtb, 0x660f6410, "pcmpgtb", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20f6410, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 37 */
+  {
+    {OP_pcmpgtw,   0x0f6510, "pcmpgtw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[37][2]},
+    {INVALID,    0xf30f6510, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pcmpgtw, 0x660f6510, "pcmpgtw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20f6510, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 38 */
+  {
+    {OP_pcmpgtd,   0x0f6610, "pcmpgtd", Pq, xx, Qq, Pq, xx, mrm, x, tpe[38][2]},
+    {INVALID,    0xf30f6610, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pcmpgtd, 0x660f6610, "pcmpgtd", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20f6610, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 39 */
+  {
+    {OP_packuswb,   0x0f6710, "packuswb", Pq, xx, Qq, Pq, xx, mrm, x, tpe[39][2]},
+    {INVALID,     0xf30f6710, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_packuswb, 0x660f6710, "packuswb", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,     0xf20f6710, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 40 */
+  {
+    {OP_punpckhbw,   0x0f6810, "punpckhbw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[40][2]},
+    {INVALID,      0xf30f6810, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_punpckhbw, 0x660f6810, "punpckhbw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf20f6810, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 41 */
+  {
+    {OP_punpckhwd,   0x0f6910, "punpckhwd", Pq, xx, Qq, Pq, xx, mrm, x, tpe[41][2]},
+    {INVALID,      0xf30f6910, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_punpckhwd, 0x660f6910, "punpckhwd", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf20f6910, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 42 */
+  {
+    {OP_punpckhdq,   0x0f6a10, "punpckhdq", Pq, xx, Qq, Pq, xx, mrm, x, tpe[42][2]},
+    {INVALID,      0xf30f6a10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_punpckhdq, 0x660f6a10, "punpckhdq", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf20f6a10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 43 */
+  {
+    {OP_packssdw,   0x0f6b10, "packssdw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[43][2]},
+    {INVALID,     0xf30f6b10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_packssdw, 0x660f6b10, "packssdw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,     0xf20f6b10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 44 */
+  {
+    {INVALID,         0x0f6c10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {INVALID,       0xf30f6c10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_punpcklqdq, 0x660f6c10, "punpcklqdq", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,       0xf20f6c10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 45 */
+  {
+    {INVALID,         0x0f6d10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {INVALID,       0xf30f6d10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_punpckhqdq, 0x660f6d10, "punpckhqdq", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,       0xf20f6d10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 46 */
+  {
+    {OP_movd,   0x0f6e10, "movd", Pd_q, xx, Ed_q, xx, xx, mrm, x, tpe[46][2]},
+    {INVALID, 0xf30f6e10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_movd, 0x660f6e10, "movd", Vd_dq, xx, Ed_q, xx, xx, mrm, x, tpe[51][0]},
+    {INVALID, 0xf20f6e10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 47: all assumed to have Ib */
+  {
+    {OP_pshufw,   0x0f7010, "pshufw",   Pq, xx, Qq, Ib, xx, mrm, x, END_LIST},
+    {OP_pshufhw, 0xf30f7010, "pshufhw", Vdq, xx, Wdq, Ib, xx, mrm, x, END_LIST},
+    {OP_pshufd,  0x660f7010, "pshufd",  Vdq, xx, Wdq, Ib, xx, mrm, x, END_LIST},
+    {OP_pshuflw, 0xf20f7010, "pshuflw", Vdq, xx, Wdq, Ib, xx, mrm, x, END_LIST},
+  },
+  /* prefix extension 48 */
+  {
+    {OP_pcmpeqb,   0x0f7410, "pcmpeqb", Pq, xx, Qq, Pq, xx, mrm, x, tpe[48][2]},
+    {INVALID,    0xf30f7410, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pcmpeqb, 0x660f7410, "pcmpeqb", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20f7410, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 49 */
+  {
+    {OP_pcmpeqw,   0x0f7510, "pcmpeqw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[49][2]},
+    {INVALID,    0xf30f7510, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pcmpeqw, 0x660f7510, "pcmpeqw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20f7510, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 50 */
+  {
+    {OP_pcmpeqd,   0x0f7610, "pcmpeqd", Pq, xx, Qq, Pq, xx, mrm, x, tpe[50][2]},
+    {INVALID,    0xf30f7610, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pcmpeqd, 0x660f7610, "pcmpeqd", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20f7610, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 51 */
+  {
+    {OP_movd,   0x0f7e10, "movd", Ed_q, xx, Pd_q, xx, xx, mrm, x, tpe[51][2]},
+    {OP_movq, 0xf30f7e10, "movq", Vq_dq, xx, Wq_dq, xx, xx, mrm, x, tpe[61][2]},
+    {OP_movd, 0x660f7e10, "movd", Ed_q, xx, Vd_dq, xx, xx, mrm, x, END_LIST},
+    {INVALID, 0xf20f7e10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 52: all assumed to have Ib */
+  {
+    {OP_cmpps, 0x0fc210, "cmpps", Vps, xx, Wps, Ib, Vps, mrm, x, END_LIST},
+    {OP_cmpss, 0xf30fc210, "cmpss", Vss, xx, Wss, Ib, Vss, mrm, x, END_LIST},
+    {OP_cmppd, 0x660fc210, "cmppd", Vpd, xx, Wpd, Ib, Vpd, mrm, x, END_LIST},
+    {OP_cmpsd, 0xf20fc210, "cmpsd", Vsd, xx, Wsd, Ib, Vsd, mrm, x, END_LIST},
+  },
+  /* prefix extension 53: all assumed to have Ib */
+  { /* note that gnu tools print immed first: pinsrw $0x0,(%esp),%xmm0 */
+    {OP_pinsrw,   0x0fc410, "pinsrw", Pq, xx, Ed, Ib, xx, mrm, x, tpe[53][2]},
+    {INVALID,   0xf30fc410, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pinsrw, 0x660fc410, "pinsrw", Vdq, xx, Ed, Ib, xx, mrm, x, END_LIST},
+    {INVALID,   0xf20fc410, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 54: all assumed to have Ib */
+  { /* note that gnu tools print immed first: pextrw $0x7,%xmm7,%edx */
+    {OP_pextrw,   0x0fc510, "pextrw", Gd, xx, Nq, Ib, xx, mrm, x, tpe[54][2]},
+    {INVALID,   0xf30fc510, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pextrw, 0x660fc510, "pextrw", Gd, xx, Udq, Ib, xx, mrm, x, t3a[3]},
+    {INVALID,   0xf20fc510, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 55: all assumed to have Ib */
+  {
+    {OP_shufps, 0x0fc610, "shufps", Vps, xx, Wps, Ib, Vps, mrm, x, END_LIST},
+    {INVALID, 0xf30fc610, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_shufpd, 0x660fc610, "shufpd", Vpd, xx, Wpd, Ib, Vpd, mrm, x, END_LIST},
+    {INVALID, 0xf20fc610, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 56 */
+  {
+    {OP_psrlw,   0x0fd110, "psrlw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[56][2]},
+    {INVALID,  0xf30fd110, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_psrlw, 0x660fd110, "psrlw", Vdq, xx, Wdq, Vdq, xx, mrm, x, tpe[104][0]},
+    {INVALID,  0xf20fd110, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 57 */
+  {
+    {OP_psrld,   0x0fd210, "psrld", Pq, xx, Qq, Pq, xx, mrm, x, tpe[57][2]},
+    {INVALID,  0xf30fd210, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_psrld, 0x660fd210, "psrld", Vdq, xx, Wdq, Vdq, xx, mrm, x, tpe[107][0]},
+    {INVALID,  0xf20fd210, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 58 */
+  {
+    {OP_psrlq,   0x0fd310, "psrlq", Pq, xx, Qq, Pq, xx, mrm, x, tpe[58][2]},
+    {INVALID,  0xf30fd310, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_psrlq, 0x660fd310, "psrlq", Vdq, xx, Wdq, Vdq, xx, mrm, x, tpe[110][0]},
+    {INVALID,  0xf20fd310, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 59 */
+  {
+    {OP_paddq,   0x0fd410, "paddq", Pq, xx, Qq, Pq, xx, mrm, x, tpe[59][2]},
+    {INVALID,  0xf30fd410, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_paddq, 0x660fd410, "paddq", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,  0xf20fd410, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 60 */
+  {
+    {OP_pmullw,   0x0fd510, "pmullw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[60][2]},
+    {INVALID,   0xf30fd510, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_pmullw, 0x660fd510, "pmullw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,   0xf20fd510, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 61 */
+  {
+    {INVALID,   0x0fd610, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID, 0xf30fd610, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_movq, 0x660fd610, "movq", Wq_dq, xx, Vq_dq, xx, xx, mrm, x, END_LIST},
+    {INVALID, 0xf20fd610, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 62 */
+  {
+    {OP_pmovmskb,   0x0fd710, "pmovmskb", Gd, xx, Nq, xx, xx, mrm, x, tpe[62][2]},
+    {INVALID,     0xf30fd710, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_pmovmskb, 0x660fd710, "pmovmskb", Gd, xx, Udq, xx, xx, mrm, x, END_LIST},
+    {INVALID,     0xf20fd710, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 63 */
+  {
+    {OP_psubusb,   0x0fd810, "psubusb", Pq, xx, Qq, Pq, xx, mrm, x, tpe[63][2]},
+    {INVALID,    0xf30fd810, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_psubusb, 0x660fd810, "psubusb", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fd810, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 64 */
+  {
+    {OP_psubusw,   0x0fd910, "psubusw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[64][2]},
+    {INVALID,    0xf30fd910, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_psubusw, 0x660fd910, "psubusw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fd910, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 65 */
+  {
+    {OP_pminub,   0x0fda10, "pminub", Pq, xx, Qq, Pq, xx, mrm, x, tpe[65][2]},
+    {INVALID,    0xf30fda10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pminub, 0x660fda10, "pminub", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fda10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 66 */
+  {
+    {OP_pand,   0x0fdb10, "pand", Pq, xx, Qq, Pq, xx, mrm, x, tpe[66][2]},
+    {INVALID,    0xf30fdb10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pand, 0x660fdb10, "pand", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fdb10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 67 */
+  {
+    {OP_paddusb,   0x0fdc10, "paddusb", Pq, xx, Qq, Pq, xx, mrm, x, tpe[67][2]},
+    {INVALID,    0xf30fdc10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_paddusb, 0x660fdc10, "paddusb", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fdc10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 68 */
+  {
+    {OP_paddusw,   0x0fdd10, "paddusw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[68][2]},
+    {INVALID,    0xf30fdd10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_paddusw, 0x660fdd10, "paddusw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fdd10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 69 */
+  {
+    {OP_pmaxub,   0x0fde10, "pmaxub", Pq, xx, Qq, Pq, xx, mrm, x, tpe[69][2]},
+    {INVALID,    0xf30fde10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pmaxub, 0x660fde10, "pmaxub", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fde10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 70 */
+  {
+    {OP_pandn,   0x0fdf10, "pandn", Pq, xx, Qq, Pq, xx, mrm, x, tpe[70][2]},
+    {INVALID,    0xf30fdf10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pandn, 0x660fdf10, "pandn", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fdf10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 71 */
+  {
+    {OP_pavgb,   0x0fe010, "pavgb", Pq, xx, Qq, Pq, xx, mrm, x, tpe[71][2]},
+    {INVALID,    0xf30fe010, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pavgb, 0x660fe010, "pavgb", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fe010, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 72 */
+  {
+    {OP_psraw,   0x0fe110, "psraw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[72][2]},
+    {INVALID,    0xf30fe110, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_psraw, 0x660fe110, "psraw", Vdq, xx, Wdq, Vdq, xx, mrm, x, tpe[105][0]},
+    {INVALID,    0xf20fe110, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 73 */
+  {
+    {OP_psrad,   0x0fe210, "psrad", Pq, xx, Qq, Pq, xx, mrm, x, tpe[73][2]},
+    {INVALID,    0xf30fe210, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_psrad, 0x660fe210, "psrad", Vdq, xx, Wdq, Vdq, xx, mrm, x, tpe[108][0]},
+    {INVALID,    0xf20fe210, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 74 */
+  {
+    {OP_pavgw,   0x0fe310, "pavgw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[74][2]},
+    {INVALID,    0xf30fe310, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pavgw, 0x660fe310, "pavgw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fe310, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 75 */
+  {
+    {OP_pmulhuw,   0x0fe410, "pmulhuw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[75][2]},
+    {INVALID,    0xf30fe410, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pmulhuw, 0x660fe410, "pmulhuw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fe410, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 76 */
+  {
+    {OP_pmulhw,   0x0fe510, "pmulhw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[76][2]},
+    {INVALID,    0xf30fe510, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pmulhw, 0x660fe510, "pmulhw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fe510, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 77 */
+  {
+    {INVALID, 0x0fe610, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_cvtdq2pd, 0xf30fe610, "cvtdq2pd",  Vpd, xx, Wq, xx, xx, mrm, x, END_LIST},
+    {OP_cvttpd2dq,0x660fe610, "cvttpd2dq", Vq, xx, Wpd, xx, xx, mrm, x, END_LIST},
+    {OP_cvtpd2dq, 0xf20fe610, "cvtpd2dq",  Vq, xx, Wpd, xx, xx, mrm, x, END_LIST},
+  },
+  /* prefix extension 78 */
+  {
+    {OP_movntq,    0x0fe710, "movntq",  Mq, xx, Pq, xx, xx, mrm, x, END_LIST},
+    {INVALID,    0xf30fe710, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_movntdq, 0x660fe710, "movntdq", Mdq, xx, Vdq, xx, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fe710, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 79 */
+  {
+    {OP_psubsb,   0x0fe810, "psubsb", Pq, xx, Qq, Pq, xx, mrm, x, tpe[79][2]},
+    {INVALID,    0xf30fe810, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_psubsb, 0x660fe810, "psubsb", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fe810, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 80 */
+  {
+    {OP_psubsw,   0x0fe910, "psubsw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[80][2]},
+    {INVALID,    0xf30fe910, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_psubsw, 0x660fe910, "psubsw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fe910, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 81 */
+  {
+    {OP_pminsw,   0x0fea10, "pminsw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[81][2]},
+    {INVALID,    0xf30fea10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pminsw, 0x660fea10, "pminsw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fea10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 82 */
+  {
+    {OP_por,   0x0feb10, "por", Pq, xx, Qq, Pq, xx, mrm, x, tpe[82][2]},
+    {INVALID,    0xf30feb10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_por, 0x660feb10, "por", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20feb10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 83 */
+  {
+    {OP_paddsb,   0x0fec10, "paddsb", Pq, xx, Qq, Pq, xx, mrm, x, tpe[83][2]},
+    {INVALID,    0xf30fec10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_paddsb, 0x660fec10, "paddsb", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fec10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 84 */
+  {
+    {OP_paddsw,   0x0fed10, "paddsw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[84][2]},
+    {INVALID,    0xf30fed10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_paddsw, 0x660fed10, "paddsw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fed10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 85 */
+  {
+    {OP_pmaxsw,   0x0fee10, "pmaxsw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[85][2]},
+    {INVALID,    0xf30fee10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pmaxsw, 0x660fee10, "pmaxsw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fee10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 86 */
+  {
+    {OP_pxor,   0x0fef10, "pxor", Pq, xx, Qq, Pq, xx, mrm, x, tpe[86][2]},
+    {INVALID,    0xf30fef10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pxor, 0x660fef10, "pxor", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20fef10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 87 */
+  {
+    {OP_psllw,   0x0ff110, "psllw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[87][2]},
+    {INVALID,    0xf30ff110, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_psllw, 0x660ff110, "psllw", Vdq, xx, Wdq, Vdq, xx, mrm, x, tpe[106][0]},
+    {INVALID,    0xf20ff110, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 88 */
+  {
+    {OP_pslld,   0x0ff210, "pslld", Pq, xx, Qq, Pq, xx, mrm, x, tpe[88][2]},
+    {INVALID,    0xf30ff210, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pslld, 0x660ff210, "pslld", Vdq, xx, Wdq, Vdq, xx, mrm, x, tpe[109][0]},
+    {INVALID,    0xf20ff210, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 89 */
+  {
+    {OP_psllq,   0x0ff310, "psllq", Pq, xx, Qq, Pq, xx, mrm, x, tpe[89][2]},
+    {INVALID,    0xf30ff310, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_psllq, 0x660ff310, "psllq", Vdq, xx, Wdq, Vdq, xx, mrm, x, tpe[111][0]},
+    {INVALID,    0xf20ff310, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 90 */
+  {
+    {OP_pmuludq,   0x0ff410, "pmuludq", Pq, xx, Qq, Pq, xx, mrm, x, tpe[90][2]},
+    {INVALID,    0xf30ff410, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pmuludq, 0x660ff410, "pmuludq", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20ff410, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 91 */
+  {
+    {OP_pmaddwd,   0x0ff510, "pmaddwd", Pq, xx, Qq, Pq, xx, mrm, x, tpe[91][2]},
+    {INVALID,    0xf30ff510, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_pmaddwd, 0x660ff510, "pmaddwd", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20ff510, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 92 */
+  {
+    {OP_psadbw,   0x0ff610, "psadbw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[92][2]},
+    {INVALID,    0xf30ff610, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_psadbw, 0x660ff610, "psadbw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20ff610, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 93 */
+  { 
+    {OP_maskmovq,     0x0ff710, "maskmovq", Bq, xx, Pq, Nq, xx, mrm, x, END_LIST}, /* Intel table says "Ppi, Qpi" */
+    {INVALID,       0xf30ff710, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_maskmovdqu, 0x660ff710, "maskmovdqu", Bdq, xx, Vdq, Udq, xx, mrm, x, END_LIST},
+    {INVALID,       0xf20ff710, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 94 */
+  {
+    {OP_psubb,   0x0ff810, "psubb", Pq, xx, Qq, Pq, xx, mrm, x, tpe[94][2]},
+    {INVALID,    0xf30ff810, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_psubb, 0x660ff810, "psubb", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20ff810, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 95 */
+  {
+    {OP_psubw,   0x0ff910, "psubw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[95][2]},
+    {INVALID,    0xf30ff910, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_psubw, 0x660ff910, "psubw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20ff910, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 96 */
+  {
+    {OP_psubd,   0x0ffa10, "psubd", Pq, xx, Qq, Pq, xx, mrm, x, tpe[96][2]},
+    {INVALID,    0xf30ffa10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_psubd, 0x660ffa10, "psubd", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20ffa10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 97 */
+  {
+    {OP_psubq,   0x0ffb10, "psubq", Pq, xx, Qq, Pq, xx, mrm, x, tpe[97][2]},
+    {INVALID,  0xf30ffb10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_psubq, 0x660ffb10, "psubq", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,  0xf20ffb10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 98 */
+  {
+    {OP_paddb,   0x0ffc10, "paddb", Pq, xx, Qq, Pq, xx, mrm, x, tpe[98][2]},
+    {INVALID,    0xf30ffc10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_paddb, 0x660ffc10, "paddb", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20ffc10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 99 */
+  {
+    {OP_paddw,   0x0ffd10, "paddw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[99][2]},
+    {INVALID,    0xf30ffd10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_paddw, 0x660ffd10, "paddw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20ffd10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 100 */
+  {
+    {OP_paddd,   0x0ffe10, "paddd", Pq, xx, Qq, Pq, xx, mrm, x, tpe[100][2]},
+    {INVALID,    0xf30ffe10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+    {OP_paddd, 0x660ffe10, "paddd", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,    0xf20ffe10, "(bad)", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 101: all assumed to have Ib */
+  {
+    {INVALID,     0x0f7333, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,   0xf30f7333, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_psrldq, 0x660f7333, "psrldq", Udq, xx, Ib, Udq, xx, mrm, x, END_LIST},
+    {INVALID,   0xf20f7333, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 102: all assumed to have Ib */
+  {
+    {INVALID,     0x0f7337, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,   0xf30f7337, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_pslldq, 0x660f7337, "pslldq", Udq, xx, Ib, Udq, xx, mrm, x, END_LIST},
+    {INVALID,   0xf20f7337, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 103 */
+  {
+    {REX_EXT,  0x900000, "(rex ext 0)", xx, xx, xx, xx, xx, no, x, 0},
+    {OP_pause,0xf3900000, "pause", xx, xx, xx, xx, xx, no, x, END_LIST},
+    /* we chain these even though encoding won't find them */
+    {OP_nop, 0x66900000, "nop", xx, xx, xx, xx, xx, no, x, tpe[103][3]},
+    /* windbg displays as "repne nop" */
+    {OP_nop, 0xf2900000, "nop", xx, xx, xx, xx, xx, no, x, END_LIST},
+  },
+  /* prefix extension 104: all assumed to have Ib */
+  {
+    /* Intel tables imply they may add opcodes in the mod<3 (mem) space in future */
+    {OP_psrlw,    0x0f7132, "psrlw", Nq, xx, Ib, Nq, xx, mrm, x, tpe[104][2]},
+    {INVALID,   0xf30f7132, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_psrlw,  0x660f7132, "psrlw", Udq, xx, Ib, Udq, xx, mrm, x, END_LIST},
+    {INVALID,   0xf20f7132, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 105: all assumed to have Ib */
+  {
+    /* Intel tables imply they may add opcodes in the mod<3 (mem) space in future */
+    {OP_psraw,    0x0f7134, "psraw", Nq, xx, Ib, Nq, xx, mrm, x, tpe[105][2]},
+    {INVALID,   0xf30f7134, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_psraw,  0x660f7134, "psraw", Udq, xx, Ib, Udq, xx, mrm, x, END_LIST},
+    {INVALID,   0xf20f7134, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 106: all assumed to have Ib */
+  {
+    /* Intel tables imply they may add opcodes in the mod<3 (mem) space in future */
+    {OP_psllw,    0x0f7136, "psllw", Nq, xx, Ib, Nq, xx, mrm, x, tpe[106][2]},
+    {INVALID,   0xf30f7136, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_psllw,  0x660f7136, "psllw", Udq, xx, Ib, Udq, xx, mrm, x, END_LIST},
+    {INVALID,   0xf20f7136, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 107: all assumed to have Ib */
+  {
+    /* Intel tables imply they may add opcodes in the mod<3 (mem) space in future */
+    {OP_psrld,    0x0f7232, "psrld", Nq, xx, Ib, Nq, xx, mrm, x, tpe[107][2]},
+    {INVALID,   0xf30f7232, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_psrld,  0x660f7232, "psrld", Udq, xx, Ib, Udq, xx, mrm, x, END_LIST},
+    {INVALID,   0xf20f7232, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 108: all assumed to have Ib */
+  {
+    /* Intel tables imply they may add opcodes in the mod<3 (mem) space in future */
+    {OP_psrad,    0x0f7234, "psrad", Nq, xx, Ib, Nq, xx, mrm, x, tpe[108][2]},
+    {INVALID,   0xf30f7234, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_psrad,  0x660f7234, "psrad", Udq, xx, Ib, Udq, xx, mrm, x, END_LIST},
+    {INVALID,   0xf20f7234, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 109: all assumed to have Ib */
+  {
+    /* Intel tables imply they may add opcodes in the mod<3 (mem) space in future */
+    {OP_pslld,    0x0f7236, "pslld", Nq, xx, Ib, Nq, xx, mrm, x, tpe[109][2]},
+    {INVALID,   0xf30f7236, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_pslld,  0x660f7236, "pslld", Udq, xx, Ib, Udq, xx, mrm, x, END_LIST},
+    {INVALID,   0xf20f7236, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 110: all assumed to have Ib */
+  {
+    /* Intel tables imply they may add opcodes in the mod<3 (mem) space in future */
+    {OP_psrlq,    0x0f7332, "psrlq", Nq, xx, Ib, Nq, xx, mrm, x, tpe[110][2]},
+    {INVALID,   0xf30f7332, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_psrlq,  0x660f7332, "psrlq", Udq, xx, Ib, Udq, xx, mrm, x, END_LIST},
+    {INVALID,   0xf20f7332, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 111: all assumed to have Ib */
+  {
+    /* Intel tables imply they may add opcodes in the mod<3 (mem) space in future */
+    {OP_psllq,    0x0f7336, "psllq", Nq, xx, Ib, Nq, xx, mrm, x, tpe[111][2]},
+    {INVALID,   0xf30f7336, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_psllq,  0x660f7336, "psllq", Udq, xx, Ib, Udq, xx, mrm, x, END_LIST},
+    {INVALID,   0xf20f7336, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 112 */
+  {
+    {OP_movq,     0x0f6f10, "movq", Pq, xx, Qq, xx, xx, mrm, x, tpe[113][0]},
+    {OP_movdqu, 0xf30f6f10, "movdqu", Vdq, xx, Wdq, xx, xx, mrm, x, tpe[113][1]},
+    {OP_movdqa, 0x660f6f10, "movdqa", Vdq, xx, Wdq, xx, xx, mrm, x, tpe[113][2]},
+    {INVALID,   0xf20f6f10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 113 */
+  {
+    {OP_movq,     0x0f7f10, "movq", Qq, xx, Pq, xx, xx, mrm, x, tpe[51][1]},
+    {OP_movdqu, 0xf30f7f10, "movdqu", Wdq, xx, Vdq, xx, xx, mrm, x, END_LIST},
+    {OP_movdqa, 0x660f7f10, "movdqa", Wdq, xx, Vdq, xx, xx, mrm, x, END_LIST},
+    {INVALID,   0xf20f7f10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  /* prefix extension 114 */
+  {
+    {INVALID,     0x0f7c10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,   0xf30f7c10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_haddpd, 0x660f7c10, "haddpd", Vpd, xx, Wpd, Vpd, xx, mrm, x, END_LIST},
+    {OP_haddps, 0xf20f7c10, "haddps", Vps, xx, Wps, Vps, xx, mrm, x, END_LIST},
+  },
+  /* prefix extension 115 */
+  {
+    {INVALID,     0x0f7d10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,   0xf30f7d10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_hsubpd, 0x660f7d10, "hsubpd", Vpd, xx, Wpd, Vpd, xx, mrm, x, END_LIST},
+    {OP_hsubps, 0xf20f7d10, "hsubps", Vps, xx, Wps, Vps, xx, mrm, x, END_LIST},
+  },
+  /* prefix extension 116 */
+  {
+    {INVALID,     0x0fd010, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,   0xf30fd010, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_addsubpd, 0x660fd010, "addsubpd", Vpd, xx, Wpd, Vpd, xx, mrm, x, END_LIST},
+    {OP_addsubps, 0xf20fd010, "addsubps", Vps, xx, Wps, Vps, xx, mrm, x, END_LIST},
+  },
+  /* prefix extension 117 */
+  {
+    {INVALID,     0x0ff010, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,   0xf30ff010, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,   0x660ff010, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_lddqu,  0xf20ff010, "lddqu", Vdq, xx, Mdq, xx, xx, mrm, x, END_LIST},
+  },
+  /***************************************************
+   * SSSE3
+   */
+  { /* prefix extension 118 */
+    {OP_pshufb,     0x380018, "pshufb",   Pq, xx, Qq, xx, xx, mrm, x, tpe[118][2]},
+    {INVALID,     0xf3380018, "(bad)",    xx, xx, xx, xx, xx, no, x, NA},
+    {OP_pshufb,   0x66380018, "pshufb",   Vdq, xx, Wdq, xx, xx, mrm, x, END_LIST},
+    {INVALID,     0xf2380018, "(bad)",    xx, xx, xx, xx, xx, no, x, NA},
+  },
+  { /* prefix extension 119 */
+    {OP_phaddw,      0x380118, "phaddw",  Pq, xx, Qq, Pq, xx, mrm, x, tpe[119][2]},
+    {INVALID,      0xf3380118, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+    {OP_phaddw,    0x66380118, "phaddw",  Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf2380118, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+  },
+  { /* prefix extension 120 */
+    {OP_phaddd,      0x380218, "phaddd",  Pq, xx, Qq, Pq, xx, mrm, x, tpe[120][2]},
+    {INVALID,      0xf3380218, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+    {OP_phaddd,    0x66380218, "phaddd",  Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf2380218, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+  },
+  { /* prefix extension 121 */
+    {OP_phaddsw,     0x380318, "phaddsw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[121][2]},
+    {INVALID,      0xf3380318, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+    {OP_phaddsw,   0x66380318, "phaddsw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf2380318, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+  },
+  { /* prefix extension 122 */
+    {OP_pmaddubsw,   0x380418, "pmaddubsw",Pq, xx, Qq, Pq, xx, mrm, x, tpe[122][2]},
+    {INVALID,      0xf3380418, "(bad)",    xx, xx, xx, xx, xx, no, x, NA},
+    {OP_pmaddubsw, 0x66380418, "pmaddubsw",Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf2380418, "(bad)",    xx, xx, xx, xx, xx, no, x, NA},
+  },
+  { /* prefix extension 123 */
+    {OP_phsubw,      0x380518, "phsubw",  Pq, xx, Qq, Pq, xx, mrm, x, tpe[123][2]},
+    {INVALID,      0xf3380518, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+    {OP_phsubw,    0x66380518, "phsubw",  Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf2380518, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+  },
+  { /* prefix extension 124 */
+    {OP_phsubd,      0x380618, "phsubd",  Pq, xx, Qq, Pq, xx, mrm, x, tpe[124][2]},
+    {INVALID,      0xf3380618, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+    {OP_phsubd,    0x66380618, "phsubd",  Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf2380618, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+  },
+  { /* prefix extension 125 */
+    {OP_phsubsw,     0x380718, "phsubsw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[125][2]},
+    {INVALID,      0xf3380718, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+    {OP_phsubsw,   0x66380718, "phsubsw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf2380718, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+  },
+  { /* prefix extension 126 */
+    {OP_psignb,      0x380818, "psignb",  Pq, xx, Qq, Pq, xx, mrm, x, tpe[126][2]},
+    {INVALID,      0xf3380818, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+    {OP_psignb,    0x66380818, "psignb",  Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf2380818, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+  },
+  { /* prefix extension 127 */
+    {OP_psignw,      0x380918, "psignw",  Pq, xx, Qq, Pq, xx, mrm, x, tpe[127][2]},
+    {INVALID,      0xf3380918, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+    {OP_psignw,    0x66380918, "psignw",  Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf2380918, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+  },
+  { /* prefix extension 128 */
+    {OP_psignd,      0x380a18, "psignd",  Pq, xx, Qq, Pq, xx, mrm, x, tpe[128][2]},
+    {INVALID,      0xf3380a18, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+    {OP_psignd,    0x66380a18, "psignd",  Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf2380a18, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+  },
+  { /* prefix extension 129 */
+    {OP_pmulhrsw,    0x380b18, "pmulhrsw", Pq, xx, Qq, Pq, xx, mrm, x, tpe[129][2]},
+    {INVALID,      0xf3380b18, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+    {OP_pmulhrsw,  0x66380b18, "pmulhrsw", Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf2380b18, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+  },
+  { /* prefix extension 130 */
+    {OP_pabsb,       0x381c18, "pabsb",   Pq, xx, Qq, Pq, xx, mrm, x, tpe[130][2]},
+    {INVALID,      0xf3381c18, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+    {OP_pabsb,     0x66381c18, "pabsb",   Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf2381c18, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+  },
+  { /* prefix extension 131 */
+    {OP_pabsw,       0x381d18, "pabsw",   Pq, xx, Qq, Pq, xx, mrm, x, tpe[131][2]},
+    {INVALID,      0xf3381d18, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+    {OP_pabsw,     0x66381d18, "pabsw",   Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf2381d18, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+  },
+  { /* prefix extension 132 */
+    {OP_pabsd,       0x381e18, "pabsd",   Pq, xx, Qq, Pq, xx, mrm, x, tpe[132][2]},
+    {INVALID,      0xf3381e18, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+    {OP_pabsd,     0x66381e18, "pabsd",   Vdq, xx, Wdq, Vdq, xx, mrm, x, END_LIST},
+    {INVALID,      0xf2381e18, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+  },
+  { /* prefix extension 133: all assumed to have Ib */
+    {OP_palignr,     0x3a0f18, "palignr", Pq, xx, Qq, Ib, Pq, mrm, x, tpe[133][2]},
+    {INVALID,      0xf33a0f18, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+    {OP_palignr,   0x663a0f18, "palignr", Vdq, xx, Wdq, Ib, Vdq, mrm, x, END_LIST},
+    {INVALID,      0xf23a0f18, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+  },
+  { /* prefix extension 134 */
+    {OP_vmread,      0x0f7810, "vmread",  Ed_q, xx, Gd_q, xx, xx, mrm|o64, x, END_LIST},
+    {INVALID,      0xf30f7810, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+    /* FIXME PR 338279: this is listed as /0 but I'm not going to chain it into
+     * the reg extensions table until I can verify, since gdb thinks it
+     * does NOT need /0.  Waiting for a processor that actually supports it.
+     * It's ok for DR proper to think a non-cti instr is valid when really it's not,
+     * though for our decoding library use we should get it right.
+     */
+    {OP_extrq,     0x660f7810, "extrq",   Udq, xx, Ib, Ib, xx, mrm, x, tpe[135][2]},
+    /* FIXME: is src or dst Udq? */
+    {OP_insertq,   0xf20f7810, "insertq", Vdq, xx, Udq, Ib, Ib, mrm, x, tpe[135][3]},
+  },
+  { /* prefix extension 135 */
+    {OP_vmwrite,     0x0f7910, "vmwrite", Gd_q, xx, Ed_q, xx, xx, mrm|o64, x, END_LIST},
+    {INVALID,      0xf30f7910, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+    /* FIXME: is src or dst Udq? */
+    {OP_extrq,     0x660f7910, "extrq",   Vdq, xx, Udq, xx, xx, mrm, x, END_LIST},
+    {OP_insertq,   0xf20f7910, "insertq", Vdq, xx, Udq, xx, xx, mrm, x, END_LIST},
+  },
+  { /* prefix extension 136 */
+    {OP_bsr,         0x0fbd10, "bsr",     Gv, xx, Ev, xx, xx, mrm, fW6, END_LIST},
+    {OP_lzcnt,     0xf30fbd10, "lzcnt",   Gv, xx, Ev, xx, xx, mrm, fW6, END_LIST},
+    {INVALID,      0x660fbd10, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,      0xf20fbd10, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+  },
+  { /* prefix extension 137 */
+    {OP_vmptrld,     0x0fc736, "vmptrld", xx, xx, Mq, xx, xx, mrm|o64, x, END_LIST},
+    {OP_vmxon,     0xf30fc736, "vmxon",   xx, xx, Mq, xx, xx, mrm|o64, x, END_LIST},
+    {OP_vmclear,   0x660fc736, "vmclear", Mq, xx, xx, xx, xx, mrm|o64, x, END_LIST},
+    {INVALID,      0xf20fc736, "(bad)",   xx, xx, xx, xx, xx, no, x, NA},
+  },
+};
+
+/****************************************************************************
+ * Instructions that differ depending on mod and rm bits in modrm byte
+ * For mod, entry 0 is all mem ref mod values (0,1,2) while entry 1 is 3.
+ * For the mem ref, we give just one of the 3 possible modrm bytes
+ * (we only use it when encoding so we don't need all 3).
+ */
+const instr_info_t mod_extensions[][2] = {
+  { /* mod extension 0 */
+    {OP_sgdt, 0x000f0170, "sgdt", Ms, xx, xx, xx, xx, mrm, x, END_LIST},
+    {RM_EXT,  0x0f0171, "(group 7 mod + rm ext 0)", xx, xx, xx, xx, xx, mrm, x, 0},
+  },
+  { /* mod extension 1 */
+    {OP_sidt, 0x080f0171, "sidt",  Ms, xx, xx, xx, xx, mrm, x, END_LIST},
+    {RM_EXT,  0x0f0171, "(group 7 mod + rm ext 1)", xx, xx, xx, xx, xx, mrm, x, 1},
+  },
+  { /* mod extension 2 */
+    {OP_invlpg, 0x380f0177, "invlpg", xx, xx, Mm, xx, xx, mrm, x, END_LIST},
+    {RM_EXT,    0x0f0177, "(group 7 mod + rm ext 2)", xx, xx, xx, xx, xx, mrm, x, 2},
+  },
+  { /* mod extension 3 */
+    {OP_clflush, 0x380fae77, "clflush", xx, xx, Mb, xx, xx, mrm, x, END_LIST},
+    {OP_sfence,  0xf80fae77, "sfence",  xx, xx, xx, xx, xx, mrm, x, END_LIST},
+  },
+};
+
+/* Naturally all of these have modrm bytes even if they have no explicit operands */
+const instr_info_t rm_extensions[][8] = {
+  { /* rm extension 0 */
+    {INVALID,   0x0f0131, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {OP_vmcall,   0xc10f0171, "vmcall",   xx, xx, xx, xx, xx, mrm|o64, x, END_LIST},
+    {OP_vmlaunch, 0xc20f0171, "vmlaunch", xx, xx, xx, xx, xx, mrm|o64, x, END_LIST},
+    {OP_vmresume, 0xc30f0171, "vmresume", xx, xx, xx, xx, xx, mrm|o64, x, END_LIST},
+    {OP_vmxoff,   0xc40f0171, "vmxoff",   xx, xx, xx, xx, xx, mrm|o64, x, END_LIST},
+    {INVALID,   0x0f0131, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,   0x0f0131, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,   0x0f0131, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  { /* rm extension 1 */
+    {OP_monitor, 0xc80f0171, "monitor",  xx, xx, eax, ecx, edx, mrm, x, END_LIST},
+    {OP_mwait,   0xc90f0171, "mwait",  xx, xx, eax, ecx, xx, mrm, x, END_LIST},
+    {INVALID,   0x0f0131, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,   0x0f0131, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,   0x0f0131, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,   0x0f0131, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,   0x0f0131, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,   0x0f0131, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+  { /* rm extension 2 */
+    {OP_swapgs, 0xf80f0177, "swapgs", xx, xx, xx, xx, xx, mrm|o64, x, END_LIST},
+    {INVALID,   0x0f0131, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,   0x0f0131, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,   0x0f0131, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,   0x0f0131, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,   0x0f0131, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+    {INVALID,   0x0f0131, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+  },
+};
+
+/****************************************************************************
+ * Instructions that differ depending on whether in 64-bit mode
+ */
+
+const instr_info_t x64_extensions[][2] = {
+  {    /* x64_ext 0 */
+    {OP_inc,  0x400000, "inc", zAX, xx, zAX, xx, xx, no, (fW6&(~fWC)), t64e[1][0]},
+    {PREFIX,  0x400000, "rex", xx, xx, xx, xx, xx, no, x, PREFIX_REX_GENERAL},
+  }, { /* x64_ext 1 */
+    {OP_inc,  0x410000, "inc", zCX, xx, zCX, xx, xx, no, (fW6&(~fWC)), t64e[2][0]},
+    {PREFIX,  0x410000, "rex.b", xx, xx, xx, xx, xx, no, x, PREFIX_REX_B},
+  }, { /* x64_ext 2 */
+    {OP_inc,  0x420000, "inc", zDX, xx, zDX, xx, xx, no, (fW6&(~fWC)), t64e[3][0]},
+    {PREFIX,  0x420000, "rex.x", xx, xx, xx, xx, xx, no, x, PREFIX_REX_X},
+  }, { /* x64_ext 3 */
+    {OP_inc,  0x430000, "inc", zBX, xx, zBX, xx, xx, no, (fW6&(~fWC)), t64e[4][0]},
+    {PREFIX,  0x430000, "rex.xb", xx, xx, xx, xx, xx, no, x, PREFIX_REX_X|PREFIX_REX_B},
+  }, { /* x64_ext 4 */
+    {OP_inc,  0x440000, "inc", zSP, xx, zSP, xx, xx, no, (fW6&(~fWC)), t64e[5][0]},
+    {PREFIX,  0x440000, "rex.r", xx, xx, xx, xx, xx, no, x, PREFIX_REX_R},
+  }, { /* x64_ext 5 */
+    {OP_inc,  0x450000, "inc", zBP, xx, zBP, xx, xx, no, (fW6&(~fWC)), t64e[6][0]},
+    {PREFIX,  0x450000, "rex.rb", xx, xx, xx, xx, xx, no, x, PREFIX_REX_R|PREFIX_REX_B},
+  }, { /* x64_ext 6 */
+    {OP_inc,  0x460000, "inc", zSI, xx, zSI, xx, xx, no, (fW6&(~fWC)), t64e[7][0]},
+    {PREFIX,  0x460000, "rex.rx", xx, xx, xx, xx, xx, no, x, PREFIX_REX_R|PREFIX_REX_X},
+  }, { /* x64_ext 7 */
+    {OP_inc,  0x470000, "inc", zDI, xx, zDI, xx, xx, no, (fW6&(~fWC)), tex[12][0]},
+    {PREFIX,  0x470000, "rex.rxb", xx, xx, xx, xx, xx, no, x, PREFIX_REX_R|PREFIX_REX_X|PREFIX_REX_B},
+  }, { /* x64_ext 8 */
+    {OP_dec,  0x480000, "dec", zAX, xx, zAX, xx, xx, no, (fW6&(~fWC)), t64e[9][0]},
+    {PREFIX,  0x480000, "rex.w", xx, xx, xx, xx, xx, no, x, PREFIX_REX_W},
+  }, { /* x64_ext 9 */
+    {OP_dec,  0x490000, "dec", zCX, xx, zCX, xx, xx, no, (fW6&(~fWC)), t64e[10][0]},
+    {PREFIX,  0x490000, "rex.wb", xx, xx, xx, xx, xx, no, x, PREFIX_REX_W|PREFIX_REX_B},
+  }, { /* x64_ext 10 */
+    {OP_dec,  0x4a0000, "dec", zDX, xx, zDX, xx, xx, no, (fW6&(~fWC)), t64e[11][0]},
+    {PREFIX,  0x4a0000, "rex.wx", xx, xx, xx, xx, xx, no, x, PREFIX_REX_W|PREFIX_REX_X},
+  }, { /* x64_ext 11 */
+    {OP_dec,  0x4b0000, "dec", zBX, xx, zBX, xx, xx, no, (fW6&(~fWC)), t64e[12][0]},
+    {PREFIX,  0x4b0000, "rex.wxb", xx, xx, xx, xx, xx, no, x, PREFIX_REX_W|PREFIX_REX_X|PREFIX_REX_B},
+  }, { /* x64_ext 12 */
+    {OP_dec,  0x4c0000, "dec", zSP, xx, zSP, xx, xx, no, (fW6&(~fWC)), t64e[13][0]},
+    {PREFIX,  0x4c0000, "rex.wr", xx, xx, xx, xx, xx, no, x, PREFIX_REX_W|PREFIX_REX_R},
+  }, { /* x64_ext 13 */
+    {OP_dec,  0x4d0000, "dec", zBP, xx, zBP, xx, xx, no, (fW6&(~fWC)), t64e[14][0]},
+    {PREFIX,  0x4d0000, "rex.wrb", xx, xx, xx, xx, xx, no, x, PREFIX_REX_W|PREFIX_REX_R|PREFIX_REX_B},
+  }, { /* x64_ext 14 */
+    {OP_dec,  0x4e0000, "dec", zSI, xx, zSI, xx, xx, no, (fW6&(~fWC)), t64e[15][0]},
+    {PREFIX,  0x4e0000, "rex.wrx", xx, xx, xx, xx, xx, no, x, PREFIX_REX_W|PREFIX_REX_R|PREFIX_REX_X},
+  }, { /* x64_ext 15 */
+    {OP_dec,  0x4f0000, "dec", zDI, xx, zDI, xx, xx, no, (fW6&(~fWC)), tex[12][1]},
+    {PREFIX,  0x4f0000, "rex.wrxb", xx, xx, xx, xx, xx, no, x, PREFIX_REX_W|PREFIX_REX_R|PREFIX_REX_X|PREFIX_REX_B},
+  }, { /* x64_ext 16 */
+    {OP_arpl,   0x630000, "arpl", Ew, xx, Gw, xx, xx, mrm|i64, fWZ, END_LIST},
+    {OP_movsxd, 0x630000, "movsxd", Gv, xx, Ed, xx, xx, mrm|o64, x, END_LIST},
+  },
+};
+
+/****************************************************************************
+ * Instructions that differ depending on whether a rex prefix is present.
+ * Our only entry right now depends on rex.b in particular so our
+ * table is indexed by that: index 0 is for no rex.b.
+ */
+const instr_info_t rex_extensions[][2] = {
+  { /* rex extension 0 */
+    {OP_nop,  0x900000, "nop", xx, xx, xx, xx, xx, no, x, tpe[103][2]},
+    /* For decoding we avoid needing new operand types by only getting
+     * here if rex.b is set.  For encode, we would need either a hack
+     * flag REQUIRES_REX_B or a new operand type for registers that
+     * must be extended (could also try to list r8 instead of eax but
+     * have to make sure all decode/encode routines can handle that as most
+     * assume the registers listed here are 32-bit base): that's too
+     * much effort for a corner case that we're not 100% certain works on
+     * all x64 processors, so we just don't list in the encoding chain.
+     */
+    {OP_xchg, 0x900000, "xchg", eAX_x, eAX, eAX_x, eAX, xx, o64, x, END_LIST},
+  },
+};
+
+/****************************************************************************
+ * 3-byte-opcode instructions: 0x0f 0x38 and 0x0f 0x3a.
+ * SSSE3 and SSE4.
+ *
+ * FIXME: if they add more 2nd byte possibilities, we could switch to one
+ * large table here and one extension type with indices into which subtable.
+ * For now we have two separate tables.
+ *
+ * N.B.: if any are added here that do not take modrm bytes, or whose
+ * size can vary based on data16 or addr16, we need to modify our
+ * decode_fast table assumptions!
+ *
+ * Many of these only come in Vdq,Wdq forms, yet still require the 0x66 prefix.
+ * Rather than waste space in the prefix_extensions table for 4 entries 3 of which
+ * are invalid, and need another layer of lookup, we use the new REQUIRES_PREFIX
+ * flag ("reqp").
+ *
+ * Since large parts of the opcode space are empty, we save space by having a
+ * table of 256 indices instead of 256 instr_info_t structs.
+ */
+const byte third_byte_38_index[256] = {
+  /* 0  1  2  3   4  5  6  7   8  9  A  B   C  D  E  F */
+     1, 2, 3, 4,  5, 6, 7, 8,  9,10,11,12,  0, 0, 0, 0,  /* 0 */
+    16, 0, 0, 0, 17,18, 0,19,  0, 0, 0, 0, 13,14,15, 0,  /* 1 */
+    20,21,22,23, 24,25, 0, 0, 26,27,28,29,  0, 0, 0, 0,  /* 2 */
+    30,31,32,33, 34,35, 0,36, 37,38,39,40, 41,42,43,44,  /* 3 */
+    45,46, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 4 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 5 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 6 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 7 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 8 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 9 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* A */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* B */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* C */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* D */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* E */
+    47,48, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0   /* F */
+};
+const instr_info_t third_byte_38[] = {
+  {INVALID,     0x38ff18, "(bad)", xx, xx, xx, xx, xx, no, x, NA},              /* 0*/
+  /**** SSSE3 ****/
+  {PREFIX_EXT,  0x380018,   "(prefix ext 118)", xx, xx, xx, xx, xx, mrm, x, 118},/* 1*/
+  {PREFIX_EXT,  0x380118,   "(prefix ext 119)", xx, xx, xx, xx, xx, mrm, x, 119},/* 2*/
+  {PREFIX_EXT,  0x380218,   "(prefix ext 120)", xx, xx, xx, xx, xx, mrm, x, 120},/* 3*/
+  {PREFIX_EXT,  0x380318,   "(prefix ext 121)", xx, xx, xx, xx, xx, mrm, x, 121},/* 4*/
+  {PREFIX_EXT,  0x380418,   "(prefix ext 122)", xx, xx, xx, xx, xx, mrm, x, 122},/* 5*/
+  {PREFIX_EXT,  0x380518,   "(prefix ext 123)", xx, xx, xx, xx, xx, mrm, x, 123},/* 6*/
+  {PREFIX_EXT,  0x380618,   "(prefix ext 124)", xx, xx, xx, xx, xx, mrm, x, 124},/* 7*/
+  {PREFIX_EXT,  0x380718,   "(prefix ext 125)", xx, xx, xx, xx, xx, mrm, x, 125},/* 8*/
+  {PREFIX_EXT,  0x380818,   "(prefix ext 126)", xx, xx, xx, xx, xx, mrm, x, 126},/* 9*/
+  {PREFIX_EXT,  0x380918,   "(prefix ext 127)", xx, xx, xx, xx, xx, mrm, x, 127},/*10*/
+  {PREFIX_EXT,  0x380a18,   "(prefix ext 128)", xx, xx, xx, xx, xx, mrm, x, 128},/*11*/
+  {PREFIX_EXT,  0x380b18,   "(prefix ext 129)", xx, xx, xx, xx, xx, mrm, x, 129},/*12*/
+  {PREFIX_EXT,  0x381c18,   "(prefix ext 130)", xx, xx, xx, xx, xx, mrm, x, 130},/*13*/
+  {PREFIX_EXT,  0x381d18,   "(prefix ext 131)", xx, xx, xx, xx, xx, mrm, x, 131},/*14*/
+  {PREFIX_EXT,  0x381e18,   "(prefix ext 132)", xx, xx, xx, xx, xx, mrm, x, 132},/*15*/
+  /**** SSE4 ****/
+  {OP_pblendvb, 0x66381018, "pblendvb", Vdq, xx, Wdq,xmm0,xx, mrm|reqp, x, END_LIST},/*16*/
+  {OP_blendvps, 0x66381418, "blendvps", Vdq, xx, Wdq,xmm0,xx, mrm|reqp, x, END_LIST},/*17*/
+  {OP_blendvpd, 0x66381518, "blendvpd", Vdq, xx, Wdq,xmm0,xx, mrm|reqp, x, END_LIST},/*18*/
+  {OP_ptest,    0x66381718, "ptest",    xx, xx,  Vdq,Wdq, xx, mrm|reqp, fW6, END_LIST},/*19*/
+  /* 20 */
+  {OP_pmovsxbw, 0x66382018, "pmovsxbw", Vdq, xx, Wdq, xx, xx, mrm|reqp, x, END_LIST},/*20*/
+  {OP_pmovsxbd, 0x66382118, "pmovsxbd", Vdq, xx, Wdq, xx, xx, mrm|reqp, x, END_LIST},/*21*/
+  {OP_pmovsxbq, 0x66382218, "pmovsxbq", Vdq, xx, Wdq, xx, xx, mrm|reqp, x, END_LIST},/*22*/
+  {OP_pmovsxdw, 0x66382318, "pmovsxwd", Vdq, xx, Wdq, xx, xx, mrm|reqp, x, END_LIST},/*23*/
+  {OP_pmovsxwq, 0x66382418, "pmovsxwq", Vdq, xx, Wdq, xx, xx, mrm|reqp, x, END_LIST},/*24*/
+  {OP_pmovsxdq, 0x66382518, "pmovsxdq", Vdq, xx, Wdq, xx, xx, mrm|reqp, x, END_LIST},/*25*/
+  {OP_pmuldq,   0x66382818, "pmuldq",   Vdq, xx, Wdq,Vdq, xx, mrm|reqp, x, END_LIST},/*26*/
+  {OP_pcmpeqq,  0x66382918, "pcmpeqq",  Vdq, xx, Wdq,Vdq, xx, mrm|reqp, x, END_LIST},/*27*/
+  {OP_movntdqa, 0x66382a18, "movntdqa", Mdq, xx, Vdq, xx, xx, mrm|reqp, x, END_LIST},/*28*/
+  {OP_packusdw, 0x66382b18, "packusdw", Vdq, xx, Wdq,Vdq, xx, mrm|reqp, x, END_LIST},/*29*/
+  /* 30 */
+  {OP_pmovzxbw, 0x66383018, "pmovzxbw", Vdq, xx, Wdq, xx, xx, mrm|reqp, x, END_LIST},/*30*/
+  {OP_pmovzxbd, 0x66383118, "pmovzxbd", Vdq, xx, Wdq, xx, xx, mrm|reqp, x, END_LIST},/*31*/
+  {OP_pmovzxbq, 0x66383218, "pmovzxbq", Vdq, xx, Wdq, xx, xx, mrm|reqp, x, END_LIST},/*32*/
+  {OP_pmovzxdw, 0x66383318, "pmovzxwd", Vdq, xx, Wdq, xx, xx, mrm|reqp, x, END_LIST},/*33*/
+  {OP_pmovzxwq, 0x66383418, "pmovzxwq", Vdq, xx, Wdq, xx, xx, mrm|reqp, x, END_LIST},/*34*/
+  {OP_pmovzxdq, 0x66383518, "pmovzxdq", Vdq, xx, Wdq, xx, xx, mrm|reqp, x, END_LIST},/*35*/
+  {OP_pcmpgtq,  0x66383718, "pcmpgtq",  Vdq, xx, Wdq,Vdq, xx, mrm|reqp, x, END_LIST},/*36*/
+  {OP_pminsb,   0x66383818, "pminsb",   Vdq, xx, Wdq,Vdq, xx, mrm|reqp, x, END_LIST},/*37*/
+  {OP_pminsd,   0x66383918, "pminsd",   Vdq, xx, Wdq,Vdq, xx, mrm|reqp, x, END_LIST},/*38*/
+  {OP_pminuw,   0x66383a18, "pminuw",   Vdq, xx, Wdq,Vdq, xx, mrm|reqp, x, END_LIST},/*39*/
+  {OP_pminud,   0x66383b18, "pminud",   Vdq, xx, Wdq,Vdq, xx, mrm|reqp, x, END_LIST},/*40*/
+  {OP_pmaxsb,   0x66383c18, "pmaxsb",   Vdq, xx, Wdq,Vdq, xx, mrm|reqp, x, END_LIST},/*41*/
+  {OP_pmaxsd,   0x66383d18, "pmaxsd",   Vdq, xx, Wdq,Vdq, xx, mrm|reqp, x, END_LIST},/*42*/
+  {OP_pmaxuw,   0x66383e18, "pmaxuw",   Vdq, xx, Wdq,Vdq, xx, mrm|reqp, x, END_LIST},/*43*/
+  {OP_pmaxud,   0x66383f18, "pmaxud",   Vdq, xx, Wdq,Vdq, xx, mrm|reqp, x, END_LIST},/*44*/
+  /* 40 */
+  {OP_pmulld,   0x66384018, "pmulld",   Vdq, xx, Wdq,Vdq, xx, mrm|reqp, x, END_LIST},/*45*/
+  {OP_phminposuw,0x66384118,"phminposuw",Vdq,xx, Wdq, xx, xx, mrm|reqp, x, END_LIST},/*46*/
+  /* f0 */
+  {OP_crc32, 0xf238f018, "crc32", Gv, xx, Eb, Gv, xx, mrm|reqp, x, END_LIST},        /*47*/
+  {OP_crc32, 0xf238f118, "crc32", Gv, xx, Ev, Gv, xx, mrm|reqp, x, t38[47]},         /*48*/
+};
+
+const byte third_byte_3a_index[256] = {
+  /* 0  1  2  3   4  5  6  7   8  9  A  B   C  D  E  F */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 1,  /* 0 */
+     0, 0, 0, 0,  2, 3, 4, 5,  6, 7, 8, 9, 10,11,12, 0,  /* 1 */
+    13,14,15, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 2 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 3 */
+     0,16,17,18,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 4 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 5 */
+    19,20,21,22,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 6 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 7 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 8 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 9 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* A */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* B */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* C */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* D */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* E */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0   /* F */
+};
+const instr_info_t third_byte_3a[] = {
+  {INVALID,     0x3aff18, "(bad)", xx, xx, xx, xx, xx, no, x, NA},                 /* 0*/
+  /**** SSSE3 ****/
+  {PREFIX_EXT,  0x3a0f18, "(prefix ext 133)", xx, xx, xx, xx, xx, mrm, x, 133},    /* 1*/
+  /**** SSE4 ****/
+  {OP_pextrb,   0x663a1418, "pextrb", Rd_Mb, xx, Vdq, Ib, xx, mrm|reqp, x, END_LIST},/* 2*/
+  {OP_pextrw,   0x663a1518, "pextrw", Rd_Mw, xx, Vdq, Ib, xx, mrm|reqp, x, END_LIST},/* 3*/
+  {OP_pextrd,   0x663a1618, "pextrd",  Ed_q, xx, Vdq, Ib, xx, mrm|reqp, x, END_LIST},/* 4*//*"pextrq" with rex.w*/
+  {OP_extractps,0x663a1718, "extractps", Ed, xx, Vdq, Ib, xx, mrm|reqp, x, END_LIST},/* 5*/
+  {OP_roundps,  0x663a1818, "roundps",  Vdq, xx, Wdq, Ib, xx, mrm|reqp, x, END_LIST},/* 6*/
+  {OP_roundpd,  0x663a1918, "roundpd",  Vdq, xx, Wdq, Ib, xx, mrm|reqp, x, END_LIST},/* 7*/
+  {OP_roundss,  0x663a1a18, "roundss",  Vss, xx, Wss, Ib, xx, mrm|reqp, x, END_LIST},/* 8*/
+  {OP_roundsd,  0x663a1b18, "roundsd",  Vsd, xx, Wsd, Ib, xx, mrm|reqp, x, END_LIST},/* 9*/
+  {OP_blendps,  0x663a1c18, "blendps",  Vdq, xx, Wdq, Ib, xx, mrm|reqp, x, END_LIST},/*10*/
+  {OP_blendpd,  0x663a1d18, "blendps",  Vdq, xx, Wdq, Ib, xx, mrm|reqp, x, END_LIST},/*11*/
+  {OP_pblendw,  0x663a1e18, "pblendw",  Vdq, xx, Wdq, Ib, xx, mrm|reqp, x, END_LIST},/*12*/
+  /* 20 */
+  /* pinsrb only writes one byte of the dest: but we don't distinguish that
+   * (OPSZ_4_of_16 is there for instrs that also take gprs: this one only takes xmm) */
+  {OP_pinsrb,   0x663a2018, "pinsrb",   Vdq, xx, Eb,  Ib, xx, mrm|reqp, x, END_LIST},/*13*/
+  {OP_insertps, 0x663a2118, "insertps", Vdq,xx,Udq_Md,Ib, xx, mrm|reqp, x, END_LIST},/*14*/
+  {OP_pinsrd,   0x663a2218, "pinsrd",   Vdq, xx, Ed_q,Ib, xx, mrm|reqp, x, END_LIST},/*15*//*"pinsrq" with rex.w*/
+  /* 40 */
+  {OP_dpps,     0x663a4118, "dpps",     Vdq, xx, Eb,  Ib, Vdq,mrm|reqp, x, END_LIST},/*16*/
+  {OP_dppd,     0x663a4218, "dppd",     Vdq, xx, Eb,  Ib, Vdq,mrm|reqp, x, END_LIST},/*17*/
+  {OP_mpsadbw,  0x663a4318, "mpsadbw",  Vdq, xx, Eb,  Ib, Vdq,mrm|reqp, x, END_LIST},/*18*/
+  /* 60 */
+  {OP_pcmpestrm,0x663a6018, "pcmpestrm",xmm0, xx, Vdq, Wdq, Ib, mrm|reqp, fW6, exop[8]},/*19*/
+  {OP_pcmpestri,0x663a6118, "pcmpestri",ecx, xx, Vdq, Wdq, Ib, mrm|reqp, fW6, exop[9]},/*20*/
+  {OP_pcmpistrm,0x663a6218, "pcmpistrm",xmm0, xx, Vdq, Wdq, Ib, mrm|reqp, fW6, END_LIST},/*21*/
+  {OP_pcmpistri,0x663a6318, "pcmpistri",ecx, xx, Vdq, Wdq, Ib, mrm|reqp, fW6, END_LIST},/*22*/
+};
+
+/****************************************************************************
+ * String instructions that differ depending on rep/repne prefix
+ */
+const instr_info_t rep_extensions[][4] = {
+    /* FIXME: ins and outs access "I/O ports", are these memory addresses?
+     * if so, change Ib to Ob and change dx to i_dx (move to dest for outs)
+     */
+  { /* rep extension 0 */
+    {OP_ins,      0x6c0000, "ins",       Yb, axDI, dx, axDI, xx, no, fRD, END_LIST},
+    {INVALID,   0x00000000, "(bad)",  xx, xx, xx, xx, xx, no, x, NA},
+    {OP_rep_ins,  0xf36c0000, "rep ins", Yb, axDI, dx, axDI, eCX, xop_next, fRD, END_LIST}, 
+    {OP_CONTD,  0xf36c0000, "rep ins", eCX, xx, xx, xx, xx, no, fRD, END_LIST},
+  },
+  { /* rep extension 1 */
+    {OP_ins,      0x6d0000, "ins",       Yz, axDI, dx, axDI, xx, no, fRD, tre[0][0]},
+    {INVALID,   0x00000000, "(bad)",  xx, xx, xx, xx, xx, no, x, NA},
+    {OP_rep_ins,  0xf36d0000, "rep ins", Yz, axDI, dx, axDI, eCX, xop_next, fRD, tre[0][2]},
+    {OP_CONTD,  0xf36d0000, "rep ins", eCX, xx, xx, xx, xx, no, fRD, END_LIST},
+  },
+  { /* rep extension 2 */
+    {OP_outs,      0x6e0000, "outs",       axSI, xx, Xb, dx, axSI, no, fRD, END_LIST},
+    {INVALID,   0x00000000, "(bad)",  xx, xx, xx, xx, xx, no, x, NA},
+    {OP_rep_outs,  0xf36e0000, "rep outs", axSI, eCX, Xb, dx, axSI, xop_next, fRD, END_LIST},
+    {OP_CONTD,  0xf36e0000, "rep outs", xx, xx, eCX, xx, xx, no, fRD, END_LIST},
+  },
+  { /* rep extension 3 */
+    {OP_outs,      0x6f0000, "outs",       axSI, xx, Xz, dx, axSI, no, fRD, tre[2][0]},
+    {INVALID,   0x00000000, "(bad)",  xx, xx, xx, xx, xx, no, x, NA},
+    {OP_rep_outs,  0xf36f0000, "rep outs", axSI, eCX, Xz, dx, axSI, xop_next, fRD, tre[2][2]},
+    {OP_CONTD,  0xf36f0000, "rep outs", xx, xx, eCX, xx, xx, no, fRD, END_LIST},
+  },
+  { /* rep extension 4 */
+    {OP_movs,      0xa40000, "movs",       Yb, axSI, Xb, axSI, axDI, xop_next, fRD, END_LIST},
+    {OP_CONTD,      0xa40000, "movs",       axDI, xx, xx, xx, xx, no, fRD, END_LIST},
+    {OP_rep_movs,  0xf3a40000, "rep movs", Yb, axSI, Xb, axSI, axDI, xop_next, fRD, END_LIST},
+    {OP_CONTD,  0xf3a40000, "rep movs", axDI, eCX, eCX, xx, xx, no, fRD, END_LIST},
+  },
+  { /* rep extension 5 */
+    {OP_movs,      0xa50000, "movs",       Yv, axSI, Xv, axSI, axDI, xop_next, fRD, tre[4][0]},
+    {OP_CONTD,      0xa50000, "movs",       axDI, xx, xx, xx, xx, no, fRD, END_LIST},
+    {OP_rep_movs,  0xf3a50000, "rep movs", Yv, axSI, Xv, axSI, axDI, xop_next, fRD, tre[4][2]},
+    {OP_CONTD,  0xf3a50000, "rep movs", axDI, eCX, eCX, xx, xx, no, fRD, END_LIST},
+  },
+  { /* rep extension 6 */
+    {OP_stos,      0xaa0000, "stos",       Yb, axDI, al, axDI, xx, no, fRD, END_LIST},
+    {INVALID,   0x00000000, "(bad)",  xx, xx, xx, xx, xx, no, x, NA},
+    {OP_rep_stos,  0xf3aa0000, "rep stos", Yb, axDI, al, axDI, eCX, xop_next, fRD, END_LIST},
+    {OP_CONTD,  0xf3aa0000, "rep stos", eCX, xx, xx, xx, xx, no, fRD, END_LIST},
+  },
+  { /* rep extension 7 */
+    {OP_stos,      0xab0000, "stos",       Yv, axDI, eAX, axDI, xx, no, fRD, tre[6][0]},
+    {INVALID,   0x00000000, "(bad)",  xx, xx, xx, xx, xx, no, x, NA},
+    {OP_rep_stos,  0xf3ab0000, "rep stos", Yv, axDI, eAX, axDI, eCX, xop_next, fRD, tre[6][2]},
+    {OP_CONTD,  0xf3ab0000, "rep stos", eCX, xx, xx, xx, xx, no, fRD, END_LIST},
+  },
+  { /* rep extension 8 */
+    {OP_lods,      0xac0000, "lods",       al, axSI, Xb, axSI, xx, no, fRD, END_LIST},
+    {INVALID,   0x00000000, "(bad)",  xx, xx, xx, xx, xx, no, x, NA},
+    {OP_rep_lods,  0xf3ac0000, "rep lods", al, axSI, Xb, axSI, eCX, xop_next, fRD, END_LIST},
+    {OP_CONTD,  0xf3ac0000, "rep lods", eCX, xx, xx, xx, xx, no, fRD, END_LIST},
+  },
+  { /* rep extension 9 */
+    {OP_lods,      0xad0000, "lods",       eAX, axSI, Xv, axSI, xx, no, fRD, tre[8][0]},
+    {INVALID,   0x00000000, "(bad)",  xx, xx, xx, xx, xx, no, x, NA},
+    {OP_rep_lods,  0xf3ad0000, "rep lods", eAX, axSI, Xv, axSI, eCX, xop_next, fRD, tre[8][2]},
+    {OP_CONTD,  0xf3ad0000, "rep lods", eCX, xx, xx, xx, xx, no, fRD, END_LIST},
+  },
+};
+
+const instr_info_t repne_extensions[][6] = {
+  { /* repne extension 0 */
+    {OP_cmps,       0xa60000, "cmps",         axSI, axDI, Xb, Yb, axSI, xop_next, (fW6|fRD), END_LIST},
+    {OP_CONTD,      0xa60000, "cmps",         xx, xx, axDI, xx, xx, no, (fW6|fRD), END_LIST},
+    {OP_rep_cmps,   0xf3a60000, "rep cmps",   axSI, axDI, Xb, Yb, axSI, xop_next, (fW6|fRD), END_LIST},
+    {OP_CONTD,      0xf3a60000, "rep cmps",   eCX, xx, axDI, eCX, xx, no, (fW6|fRD), END_LIST},
+    {OP_repne_cmps, 0xf2a60000, "repne cmps", axSI, axDI, Xb, Yb, axSI, xop_next, (fW6|fRD), END_LIST},
+    {OP_CONTD,      0xf2a60000, "repne cmps", eCX, xx, axDI, eCX, xx, no, (fW6|fRD), END_LIST},
+  },
+  { /* repne extension 1 */
+    {OP_cmps,       0xa70000, "cmps",         axSI, axDI, Xv, Yv, axSI, xop_next, (fW6|fRD), tne[0][0]},
+    {OP_CONTD,      0xa70000, "cmps",         xx, xx, axDI, xx, xx, no, (fW6|fRD), END_LIST},
+    {OP_rep_cmps,   0xf3a70000, "rep cmps",   axSI, axDI, Xv, Yv, axSI, xop_next, (fW6|fRD), tne[0][2]},
+    {OP_CONTD,      0xf3a70000, "rep cmps",   eCX, xx, axDI, eCX, xx, no, (fW6|fRD), END_LIST},
+    {OP_repne_cmps, 0xf2a70000, "repne cmps", axSI, axDI, Xv, Yv, axSI, xop_next, (fW6|fRD), tne[0][4]},
+    {OP_CONTD,      0xf2a70000, "repne cmps", eCX, xx, axDI, eCX, xx, no, (fW6|fRD), END_LIST},
+  },
+  { /* repne extension 2 */
+    {OP_scas,       0xae0000, "scas",         axDI, xx, Yb, al, axDI, no, (fW6|fRD), END_LIST},
+    {INVALID,   0x00000000, "(bad)",  xx, xx, xx, xx, xx, no, x, NA},
+    {OP_rep_scas,   0xf3ae0000, "rep scas",   axDI, eCX, Yb, al, axDI, xop_next, (fW6|fRD), END_LIST},
+    {OP_CONTD,      0xf3ae0000, "rep scas",   xx, xx, eCX, xx, xx, no, (fW6|fRD), END_LIST},
+    {OP_repne_scas, 0xf2ae0000, "repne scas", axDI, eCX, Yb, al, axDI, xop_next, (fW6|fRD), END_LIST},
+    {OP_CONTD,      0xf2ae0000, "repne scas", xx, xx, eCX, xx, xx, no, (fW6|fRD), END_LIST},
+  },
+  { /* repne extension 3 */
+    {OP_scas,       0xaf0000, "scas",         axDI, xx, Yv, eAX, axDI, no, (fW6|fRD), tne[2][0]},
+    {INVALID,   0x00000000, "(bad)",  xx, xx, xx, xx, xx, no, x, NA},
+    {OP_rep_scas,   0xf3af0000, "rep scas",   axDI, eCX, Yv, eAX, axDI, xop_next, (fW6|fRD), tne[2][2]},
+    {OP_CONTD,      0xf3af0000, "rep scas",   xx, xx, eCX, xx, xx, no, (fW6|fRD), END_LIST},
+    {OP_repne_scas, 0xf2af0000, "repne scas", axDI, eCX, Yv, eAX, axDI, xop_next, (fW6|fRD), tne[2][4]},
+    {OP_CONTD,      0xf2af0000, "repne scas", xx, xx, eCX, xx, xx, no, (fW6|fRD), END_LIST},
+  }
+};
+
+/****************************************************************************
+ * Float instructions with ModR/M from 0x00 to 0xbf
+ * This is from Tables A-7, A-9, A-11, A-13, A-15, A-17, A-19, A-21
+ * I've added my own symbol '+' to indicate a float, and:
+ *   'x' to indicate extended real (80 bits)
+ *   'y' to indicate 14/28 byte value in memory
+ *   'z' to indicate 98/108 byte value in memory
+ */
+/* FIXME: I ignore fp stack changes, should we model that? */
+const instr_info_t float_low_modrm[] = {
+  /* d8 */
+  {OP_fadd,  0xd80020, "fadd",  st0, xx, Kd, st0, xx, mrm, x, tfl[0x20]}, /* 00 */
+  {OP_fmul,  0xd80021, "fmul",  st0, xx, Kd, st0, xx, mrm, x, tfl[0x21]},
+  {OP_fcom,  0xd80022, "fcom",  xx, xx, Kd, st0, xx, mrm, x, tfl[0x22]},
+  {OP_fcomp, 0xd80023, "fcomp", xx, xx, Kd, st0, xx, mrm, x, tfl[0x23]},
+  {OP_fsub,  0xd80024, "fsub",  st0, xx, Kd, st0, xx, mrm, x, tfl[0x24]},
+  {OP_fsubr, 0xd80025, "fsubr", st0, xx, Kd, st0, xx, mrm, x, tfl[0x25]},
+  {OP_fdiv,  0xd80026, "fdiv",  st0, xx, Kd, st0, xx, mrm, x, tfl[0x26]},
+  {OP_fdivr, 0xd80027, "fdivr", st0, xx, Kd, st0, xx, mrm, x, tfl[0x27]},
+  /*  d9 */
+  {OP_fld,    0xd90020, "fld",    st0, xx, Kd, xx, xx, mrm, x, tfl[0x1d]}, /* 08 */
+  {INVALID,   0xd90021, "(bad)",  xx, xx, xx, xx, xx, no, x, NA},
+  {OP_fst,    0xd90022, "fst",    Kd, xx, st0, xx, xx, mrm, x, tfl[0x2a]},
+  {OP_fstp,   0xd90023, "fstp",   Kd, xx, st0, xx, xx, mrm, x, tfl[0x1f]},
+  {OP_fldenv, 0xd90024, "fldenv", xx, xx, Ky, xx, xx, mrm, x, END_LIST},
+  {OP_fldcw,  0xd90025, "fldcw",  xx, xx, Kw, xx, xx, mrm, x, END_LIST},
+  {OP_fnstenv, 0xd90026, "fnstenv", Ky, xx, xx, xx, xx, mrm, x, END_LIST},/*FIXME: w/ preceding fwait instr, this is "fstenv"*/
+  {OP_fnstcw,  0xd90027, "fnstcw",  Kw, xx, xx, xx, xx, mrm, x, END_LIST},/*FIXME: w/ preceding fwait instr, this is "fstcw"*/
+  /* da */
+  {OP_fiadd,  0xda0020, "fiadd",  st0, xx, Md, st0, xx, mrm, x, tfl[0x30]}, /* 10 */
+  {OP_fimul,  0xda0021, "fimul",  st0, xx, Md, st0, xx, mrm, x, tfl[0x31]},
+  {OP_ficom,  0xda0022, "ficom",  st0, xx, Md, st0, xx, mrm, x, tfl[0x32]},
+  {OP_ficomp, 0xda0023, "ficomp", st0, xx, Md, st0, xx, mrm, x, tfl[0x33]},
+  {OP_fisub,  0xda0024, "fisub",  st0, xx, Md, st0, xx, mrm, x, tfl[0x34]},
+  {OP_fisubr, 0xda0025, "fisubr", st0, xx, Md, st0, xx, mrm, x, tfl[0x35]},
+  {OP_fidiv,  0xda0026, "fidiv",  st0, xx, Md, st0, xx, mrm, x, tfl[0x36]},
+  {OP_fidivr, 0xda0027, "fidivr", st0, xx, Md, st0, xx, mrm, x, tfl[0x37]},
+  /* db */
+  {OP_fild,  0xdb0020, "fild",  st0, xx, Md, xx, xx, mrm, x, tfl[0x38]}, /* 18 */
+  {OP_fisttp, 0xdb0021, "fisttp",  Md, xx, st0, xx, xx, no, x, tfl[0x39]},
+  {OP_fist,  0xdb0022, "fist",  Md, xx, st0, xx, xx, mrm, x, tfl[0x3a]},
+  {OP_fistp, 0xdb0023, "fistp", Md, xx, st0, xx, xx, mrm, x, tfl[0x3b]},
+  {INVALID,  0xdb0024, "(bad)",  xx, xx, xx, xx, xx, no, x, NA},
+  {OP_fld,   0xdb0025, "fld",   st0, xx, Kx, xx, xx, mrm, x, tfl[0x28]},
+  {INVALID,  0xdb0026, "(bad)",  xx, xx, xx, xx, xx, no, x, NA},
+  {OP_fstp,  0xdb0027, "fstp",  Kx, xx, st0, xx, xx, mrm, x, tfl[0x2b]},
+  /* dc */
+  {OP_fadd,  0xdc0020, "fadd",  st0, xx, Kq, st0, xx, mrm, x, tfh[0][0x00]}, /* 20 */
+  {OP_fmul,  0xdc0021, "fmul",  st0, xx, Kq, st0, xx, mrm, x, tfh[0][0x08]},
+  {OP_fcom,  0xdc0022, "fcom",  xx, xx, Kq, st0, xx, mrm, x, tfh[0][0x10]},
+  {OP_fcomp, 0xdc0023, "fcomp", xx, xx, Kq, st0, xx, mrm, x, tfh[0][0x18]},
+  {OP_fsub,  0xdc0024, "fsub",  st0, xx, Kq, st0, xx, mrm, x, tfh[0][0x20]},
+  {OP_fsubr, 0xdc0025, "fsubr", st0, xx, Kq, st0, xx, mrm, x, tfh[0][0x28]},
+  {OP_fdiv,  0xdc0026, "fdiv",  st0, xx, Kq, st0, xx, mrm, x, tfh[0][0x30]},
+  {OP_fdivr, 0xdc0027, "fdivr", st0, xx, Kq, st0, xx, mrm, x, tfh[0][0x38]},
+  /* dd */
+  {OP_fld,   0xdd0020, "fld",    st0, xx, Kq, xx, xx, mrm, x, tfh[1][0x00]}, /* 28 */
+  {OP_fisttp, 0xdd0021, "fisttp",  Mq, xx, st0, xx, xx, no, x, tfl[0x19]},
+  {OP_fst,   0xdd0022, "fst",    Kq, xx, st0, xx, xx, mrm, x, tfh[5][0x10]},
+  {OP_fstp,  0xdd0023, "fstp",   Kq, xx, st0, xx, xx, mrm, x, tfh[5][0x18]},
+  {OP_frstor,0xdd0024, "frstor", xx, xx, Kz, xx, xx, mrm, x, END_LIST},
+  {INVALID,  0xdd0025, "(bad)",  xx, xx, xx, xx, xx, no, x, NA},
+  {OP_fnsave, 0xdd0026, "fnsave",  Kz, xx, xx, xx, xx, mrm, x, END_LIST},/*FIXME:w/ preceding fwait instr, this is "fsave"*/
+  {OP_fnstsw, 0xdd0027, "fnstsw",  Kw, xx, xx, xx, xx, mrm, x, tfh[7][0x20]},/*FIXME:w/ preceding fwait instr, this is "fstsw"*/
+  /* de */
+  {OP_fiadd,  0xde0020, "fiadd",  st0, xx, Kw, st0, xx, mrm, x, END_LIST}, /* 30 */
+  {OP_fimul,  0xde0021, "fimul",  st0, xx, Kw, st0, xx, mrm, x, END_LIST},
+  {OP_ficom,  0xde0022, "ficom",  xx, xx, Kw, st0, xx, mrm, x, END_LIST},
+  {OP_ficomp, 0xde0023, "ficomp", xx, xx, Kw, st0, xx, mrm, x, END_LIST},
+  {OP_fisub,  0xde0024, "fisub",  st0, xx, Kw, st0, xx, mrm, x, END_LIST},
+  {OP_fisubr, 0xde0025, "fisubr", st0, xx, Kw, st0, xx, mrm, x, END_LIST},
+  {OP_fidiv,  0xde0026, "fidiv",  st0, xx, Kw, st0, xx, mrm, x, END_LIST},
+  {OP_fidivr, 0xde0027, "fidivr", st0, xx, Kw, st0, xx, mrm, x, END_LIST},
+  /* df */
+  {OP_fild,   0xdf0020, "fild",    st0, xx, Kw, xx, xx, mrm, x, tfl[0x3d]}, /* 38 */
+  {OP_fisttp, 0xdf0021, "fisttp",  Mw, xx, st0, xx, xx, no, x, END_LIST},
+  {OP_fist,   0xdf0022, "fist",    Kw, xx, st0, xx, xx, mrm, x, END_LIST},
+  {OP_fistp,  0xdf0023, "fistp",   Kw, xx, st0, xx, xx, mrm, x, tfl[0x3f]},
+  {OP_fbld,   0xdf0024, "fbld",    st0, xx, Kx, xx, xx, mrm, x, END_LIST},
+  {OP_fild,   0xdf0025, "fild",    st0, xx, Kq, xx, xx, mrm, x, END_LIST},
+  {OP_fbstp,  0xdf0026, "fbstp",   Kx, xx, st0, xx, xx, mrm, x, END_LIST},
+  {OP_fistp,  0xdf0027, "fistp",   Kq, xx, st0, xx, xx, mrm, x, END_LIST},
+};
+
+/****************************************************************************
+ * Float instructions with ModR/M above 0xbf
+ * This is from Tables A-8, A-10, A-12, A-14, A-16, A-18, A-20, A-22
+ */
+const instr_info_t float_high_modrm[][64] = {
+    { /* d8 = [0] */
+        {OP_fadd, 0xd8c010, "fadd", st0, xx, st0, st0, xx, mrm, x, tfh[0][0x01]}, /* c0 = [0x00] */
+        {OP_fadd, 0xd8c110, "fadd", st0, xx, st1, st0, xx, mrm, x, tfh[0][0x02]},
+        {OP_fadd, 0xd8c210, "fadd", st0, xx, st2, st0, xx, mrm, x, tfh[0][0x03]},
+        {OP_fadd, 0xd8c310, "fadd", st0, xx, st3, st0, xx, mrm, x, tfh[0][0x04]},
+        {OP_fadd, 0xd8c410, "fadd", st0, xx, st4, st0, xx, mrm, x, tfh[0][0x05]},
+        {OP_fadd, 0xd8c510, "fadd", st0, xx, st5, st0, xx, mrm, x, tfh[0][0x06]},
+        {OP_fadd, 0xd8c610, "fadd", st0, xx, st6, st0, xx, mrm, x, tfh[0][0x07]},
+        {OP_fadd, 0xd8c710, "fadd", st0, xx, st7, st0, xx, mrm, x, tfh[4][0x00]},
+        {OP_fmul, 0xd8c810, "fmul", st0, xx, st0, st0, xx, mrm, x, tfh[0][0x09]}, /* c8 = [0x08] */
+        {OP_fmul, 0xd8c910, "fmul", st0, xx, st1, st0, xx, mrm, x, tfh[0][0x0a]},
+        {OP_fmul, 0xd8ca10, "fmul", st0, xx, st2, st0, xx, mrm, x, tfh[0][0x0b]},
+        {OP_fmul, 0xd8cb10, "fmul", st0, xx, st3, st0, xx, mrm, x, tfh[0][0x0c]},
+        {OP_fmul, 0xd8cc10, "fmul", st0, xx, st4, st0, xx, mrm, x, tfh[0][0x0d]},
+        {OP_fmul, 0xd8cd10, "fmul", st0, xx, st5, st0, xx, mrm, x, tfh[0][0x0e]},
+        {OP_fmul, 0xd8ce10, "fmul", st0, xx, st6, st0, xx, mrm, x, tfh[0][0x0f]},
+        {OP_fmul, 0xd8cf10, "fmul", st0, xx, st7, st0, xx, mrm, x, tfh[4][0x08]},
+        {OP_fcom, 0xd8d010, "fcom", xx, xx, st0, st0, xx, mrm, x, tfh[0][0x11]}, /* d0 = [0x10] */
+        {OP_fcom, 0xd8d110, "fcom", xx, xx, st0, st1, xx, mrm, x, tfh[0][0x12]},
+        {OP_fcom, 0xd8d210, "fcom", xx, xx, st0, st2, xx, mrm, x, tfh[0][0x13]},
+        {OP_fcom, 0xd8d310, "fcom", xx, xx, st0, st3, xx, mrm, x, tfh[0][0x14]},
+        {OP_fcom, 0xd8d410, "fcom", xx, xx, st0, st4, xx, mrm, x, tfh[0][0x15]},
+        {OP_fcom, 0xd8d510, "fcom", xx, xx, st0, st5, xx, mrm, x, tfh[0][0x16]},
+        {OP_fcom, 0xd8d610, "fcom", xx, xx, st0, st6, xx, mrm, x, tfh[0][0x17]},
+        {OP_fcom, 0xd8d710, "fcom", xx, xx, st0, st7, xx, mrm, x, END_LIST},
+        {OP_fcomp, 0xd8d810, "fcomp", xx, xx, st0, st0, xx, mrm, x, tfh[0][0x19]}, /* d8 = [0x18] */
+        {OP_fcomp, 0xd8d910, "fcomp", xx, xx, st0, st1, xx, mrm, x, tfh[0][0x1a]},
+        {OP_fcomp, 0xd8da10, "fcomp", xx, xx, st0, st2, xx, mrm, x, tfh[0][0x1b]},
+        {OP_fcomp, 0xd8db10, "fcomp", xx, xx, st0, st3, xx, mrm, x, tfh[0][0x1c]},
+        {OP_fcomp, 0xd8dc10, "fcomp", xx, xx, st0, st4, xx, mrm, x, tfh[0][0x1d]},
+        {OP_fcomp, 0xd8dd10, "fcomp", xx, xx, st0, st5, xx, mrm, x, tfh[0][0x1e]},
+        {OP_fcomp, 0xd8de10, "fcomp", xx, xx, st0, st6, xx, mrm, x, tfh[0][0x1f]},
+        {OP_fcomp, 0xd8df10, "fcomp", xx, xx, st0, st7, xx, mrm, x, END_LIST},
+        {OP_fsub, 0xd8e010, "fsub", st0, xx, st0, st0, xx, mrm, x, tfh[0][0x21]}, /* e0 = [0x20] */
+        {OP_fsub, 0xd8e110, "fsub", st0, xx, st1, st0, xx, mrm, x, tfh[0][0x22]},
+        {OP_fsub, 0xd8e210, "fsub", st0, xx, st2, st0, xx, mrm, x, tfh[0][0x23]},
+        {OP_fsub, 0xd8e310, "fsub", st0, xx, st3, st0, xx, mrm, x, tfh[0][0x24]},
+        {OP_fsub, 0xd8e410, "fsub", st0, xx, st4, st0, xx, mrm, x, tfh[0][0x25]},
+        {OP_fsub, 0xd8e510, "fsub", st0, xx, st5, st0, xx, mrm, x, tfh[0][0x26]},
+        {OP_fsub, 0xd8e610, "fsub", st0, xx, st6, st0, xx, mrm, x, tfh[0][0x27]},
+        {OP_fsub, 0xd8e710, "fsub", st0, xx, st7, st0, xx, mrm, x, tfh[4][0x28]},
+        {OP_fsubr, 0xd8e810, "fsubr", st0, xx, st0, st0, xx, mrm, x, tfh[0][0x29]}, /* e8 = [0x28] */
+        {OP_fsubr, 0xd8e910, "fsubr", st0, xx, st1, st0, xx, mrm, x, tfh[0][0x2a]},
+        {OP_fsubr, 0xd8ea10, "fsubr", st0, xx, st2, st0, xx, mrm, x, tfh[0][0x2b]},
+        {OP_fsubr, 0xd8eb10, "fsubr", st0, xx, st3, st0, xx, mrm, x, tfh[0][0x2c]},
+        {OP_fsubr, 0xd8ec10, "fsubr", st0, xx, st4, st0, xx, mrm, x, tfh[0][0x2d]},
+        {OP_fsubr, 0xd8ed10, "fsubr", st0, xx, st5, st0, xx, mrm, x, tfh[0][0x2e]},
+        {OP_fsubr, 0xd8ee10, "fsubr", st0, xx, st6, st0, xx, mrm, x, tfh[0][0x2f]},
+        {OP_fsubr, 0xd8ef10, "fsubr", st0, xx, st7, st0, xx, mrm, x, tfh[4][0x20]},
+        {OP_fdiv, 0xd8f010, "fdiv", st0, xx, st0, st0, xx, mrm, x, tfh[0][0x31]}, /* f0 = [0x30] */
+        {OP_fdiv, 0xd8f110, "fdiv", st0, xx, st1, st0, xx, mrm, x, tfh[0][0x32]},
+        {OP_fdiv, 0xd8f210, "fdiv", st0, xx, st2, st0, xx, mrm, x, tfh[0][0x33]},
+        {OP_fdiv, 0xd8f310, "fdiv", st0, xx, st3, st0, xx, mrm, x, tfh[0][0x34]},
+        {OP_fdiv, 0xd8f410, "fdiv", st0, xx, st4, st0, xx, mrm, x, tfh[0][0x35]},
+        {OP_fdiv, 0xd8f510, "fdiv", st0, xx, st5, st0, xx, mrm, x, tfh[0][0x36]},
+        {OP_fdiv, 0xd8f610, "fdiv", st0, xx, st6, st0, xx, mrm, x, tfh[0][0x37]},
+        {OP_fdiv, 0xd8f710, "fdiv", st0, xx, st7, st0, xx, mrm, x, tfh[4][0x38]},
+        {OP_fdivr, 0xd8f810, "fdivr", st0, xx, st0, st0, xx, mrm, x, tfh[0][0x39]}, /* f8 = [0x38] */
+        {OP_fdivr, 0xd8f910, "fdivr", st0, xx, st1, st0, xx, mrm, x, tfh[0][0x3a]},
+        {OP_fdivr, 0xd8fa10, "fdivr", st0, xx, st2, st0, xx, mrm, x, tfh[0][0x3b]},
+        {OP_fdivr, 0xd8fb10, "fdivr", st0, xx, st3, st0, xx, mrm, x, tfh[0][0x3c]},
+        {OP_fdivr, 0xd8fc10, "fdivr", st0, xx, st4, st0, xx, mrm, x, tfh[0][0x3d]},
+        {OP_fdivr, 0xd8fd10, "fdivr", st0, xx, st5, st0, xx, mrm, x, tfh[0][0x3e]},
+        {OP_fdivr, 0xd8fe10, "fdivr", st0, xx, st6, st0, xx, mrm, x, tfh[0][0x3f]},
+        {OP_fdivr, 0xd8ff10, "fdivr", st0, xx, st7, st0, xx, mrm, x, tfh[4][0x30]},
+   },
+    { /* d9 = [1] */
+        {OP_fld, 0xd9c010, "fld", st0, xx, st0, xx, xx, mrm, x, tfh[1][0x01]}, /* c0 = [0x00] */
+        {OP_fld, 0xd9c110, "fld", st0, xx, st1, xx, xx, mrm, x, tfh[1][0x02]},
+        {OP_fld, 0xd9c210, "fld", st0, xx, st2, xx, xx, mrm, x, tfh[1][0x03]},
+        {OP_fld, 0xd9c310, "fld", st0, xx, st3, xx, xx, mrm, x, tfh[1][0x04]},
+        {OP_fld, 0xd9c410, "fld", st0, xx, st4, xx, xx, mrm, x, tfh[1][0x05]},
+        {OP_fld, 0xd9c510, "fld", st0, xx, st5, xx, xx, mrm, x, tfh[1][0x06]},
+        {OP_fld, 0xd9c610, "fld", st0, xx, st6, xx, xx, mrm, x, tfh[1][0x07]},
+        {OP_fld, 0xd9c710, "fld", st0, xx, st7, xx, xx, mrm, x, END_LIST},
+        {OP_fxch, 0xd9c810, "fxch", st0, st0, st0, st0, xx, mrm, x, tfh[1][0x09]}, /* c8 = [0x08] */
+        {OP_fxch, 0xd9c910, "fxch", st0, st1, st0, st1, xx, mrm, x, tfh[1][0x0a]},
+        {OP_fxch, 0xd9ca10, "fxch", st0, st2, st0, st2, xx, mrm, x, tfh[1][0x0b]},
+        {OP_fxch, 0xd9cb10, "fxch", st0, st3, st0, st3, xx, mrm, x, tfh[1][0x0c]},
+        {OP_fxch, 0xd9cc10, "fxch", st0, st4, st0, st4, xx, mrm, x, tfh[1][0x0d]},
+        {OP_fxch, 0xd9cd10, "fxch", st0, st5, st0, st5, xx, mrm, x, tfh[1][0x0e]},
+        {OP_fxch, 0xd9ce10, "fxch", st0, st6, st0, st6, xx, mrm, x, tfh[1][0x0f]},
+        {OP_fxch, 0xd9cf10, "fxch", st0, st7, st0, st7, xx, mrm, x, END_LIST},
+        {OP_fnop, 0xd9d010, "fnop", xx, xx, xx, xx, xx, mrm, x, END_LIST}, /* d0 = [0x10] */
+        {INVALID, 0xd9d110, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xd9d210, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xd9d310, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xd9d410, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xd9d510, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xd9d610, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xd9d710, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        /* Undocumented.  On sandpile.org as "fstp1".  We assume an alias for fstp
+         * and do not include in the encode chain. 
+         */
+        {OP_fstp, 0xd9d810, "fstp", st0, xx, st0, xx, xx, mrm, x, END_LIST}, /* d8 = [0x18] */
+        {OP_fstp, 0xd9d910, "fstp", st1, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fstp, 0xd9da10, "fstp", st2, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fstp, 0xd9db10, "fstp", st3, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fstp, 0xd9dc10, "fstp", st4, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fstp, 0xd9dd10, "fstp", st5, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fstp, 0xd9de10, "fstp", st6, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fstp, 0xd9df10, "fstp", st7, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fchs,   0xd9e010, "fchs",   st0, xx, st0, xx, xx, mrm, x, END_LIST}, /* e0 = [0x20] */
+        {OP_fabs,   0xd9e110, "fabs",   st0, xx, st0, xx, xx, mrm, x, END_LIST},
+        {INVALID,   0xd9e210, "(bad)",  xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID,   0xd9e310, "(bad)",  xx, xx, xx, xx, xx, no, x, NA},
+        {OP_ftst,   0xd9e410, "ftst",   st0, xx, cF, xx, xx, mrm, x, END_LIST},
+        {OP_fxam,   0xd9e510, "fxam",   xx, xx, st0, xx, xx, mrm, x, END_LIST},
+        {INVALID,   0xd9e610, "(bad)",  xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID,   0xd9e710, "(bad)",  xx, xx, xx, xx, xx, no, x, NA},
+        {OP_fld1,   0xd9e810, "fld1",   st0, xx, cF, xx, xx, mrm, x, END_LIST}, /* e8 = [0x28] */
+        {OP_fldl2t, 0xd9e910, "fldl2t", st0, xx, cF, xx, xx, mrm, x, END_LIST},
+        {OP_fldl2e, 0xd9ea10, "fldl2e", st0, xx, cF, xx, xx, mrm, x, END_LIST},
+        {OP_fldpi,  0xd9eb10, "fldpi",  st0, xx, cF, xx, xx, mrm, x, END_LIST},
+        {OP_fldlg2, 0xd9ec10, "fldlg2", st0, xx, cF, xx, xx, mrm, x, END_LIST},
+        {OP_fldln2, 0xd9ed10, "fldln2", st0, xx, cF, xx, xx, mrm, x, END_LIST},
+        {OP_fldz,   0xd9ee10, "fldz",   st0, xx, cF, xx, xx, mrm, x, END_LIST},
+        {INVALID,   0xd9ef10, "(bad)",  xx, xx, xx, xx, xx, no, x, NA},
+        {OP_f2xm1,  0xd9f010, "f2xm1",  st0, xx, st0, xx, xx, mrm, x, END_LIST}, /* f0 = [0x30] */
+        {OP_fyl2x,  0xd9f110, "fyl2x",  st0, st1, st0, st1, xx, mrm, x, END_LIST},
+        {OP_fptan,  0xd9f210, "fptan",  st0, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fpatan, 0xd9f310, "fpatan", st0, st1, st0, st1, xx, mrm, x, END_LIST},
+        {OP_fxtract,0xd9f410, "fxtract",st0, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fprem1, 0xd9f510, "fprem1", st0, st1, st0, st1, xx, mrm, x, END_LIST},
+        {OP_fdecstp,0xd9f610, "fdecstp", xx, xx, xx, xx, xx, mrm, x, END_LIST},
+        {OP_fincstp,0xd9f710, "fincstp", xx, xx, xx, xx, xx, mrm, x, END_LIST},
+        {OP_fprem,  0xd9f810, "fprem",  st0, st1, st0, st1, xx, mrm, x, END_LIST}, /* f8 = [0x38] */
+        {OP_fyl2xp1,0xd9f910, "fyl2xp1",st0, st1, st0, st1, xx, mrm, x, END_LIST},
+        {OP_fsqrt,  0xd9fa10, "fsqrt",  st0, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fsincos,0xd9fb10, "fsincos",st0, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_frndint,0xd9fc10, "frndint",st0, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fscale, 0xd9fd10, "fscale", st0, xx, st1, st0, xx, mrm, x, END_LIST},
+        {OP_fsin,   0xd9fe10, "fsin",   st0, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fcos,   0xd9ff10, "fcos",   st0, xx, st0, xx, xx, mrm, x, END_LIST},
+   },
+    { /* da = [2] */
+        {OP_fcmovb, 0xdac010, "fcmovb", st0, xx, st0, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x01]}, /* c0 = [0x00] */
+        {OP_fcmovb, 0xdac110, "fcmovb", st0, xx, st1, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x02]},
+        {OP_fcmovb, 0xdac210, "fcmovb", st0, xx, st2, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x03]},
+        {OP_fcmovb, 0xdac310, "fcmovb", st0, xx, st3, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x04]},
+        {OP_fcmovb, 0xdac410, "fcmovb", st0, xx, st4, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x05]},
+        {OP_fcmovb, 0xdac510, "fcmovb", st0, xx, st5, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x06]},
+        {OP_fcmovb, 0xdac610, "fcmovb", st0, xx, st6, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x07]},
+        {OP_fcmovb, 0xdac710, "fcmovb", st0, xx, st7, xx, xx, mrm, (fRC|fRP|fRZ), END_LIST},
+        {OP_fcmove, 0xdac810, "fcmove", st0, xx, st0, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x09]}, /* c8 = [0x08] */
+        {OP_fcmove, 0xdac910, "fcmove", st0, xx, st1, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x0a]},
+        {OP_fcmove, 0xdaca10, "fcmove", st0, xx, st2, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x0b]},
+        {OP_fcmove, 0xdacb10, "fcmove", st0, xx, st3, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x0c]},
+        {OP_fcmove, 0xdacc10, "fcmove", st0, xx, st4, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x0d]},
+        {OP_fcmove, 0xdacd10, "fcmove", st0, xx, st5, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x0e]},
+        {OP_fcmove, 0xdace10, "fcmove", st0, xx, st6, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x0f]},
+        {OP_fcmove, 0xdacf10, "fcmove", st0, xx, st7, xx, xx, mrm, (fRC|fRP|fRZ), END_LIST},
+        {OP_fcmovbe, 0xdad010, "fcmovbe", st0, xx, st0, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x11]}, /* d0 = [0x10] */
+        {OP_fcmovbe, 0xdad110, "fcmovbe", st0, xx, st1, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x12]},
+        {OP_fcmovbe, 0xdad210, "fcmovbe", st0, xx, st2, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x13]},
+        {OP_fcmovbe, 0xdad310, "fcmovbe", st0, xx, st3, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x14]},
+        {OP_fcmovbe, 0xdad410, "fcmovbe", st0, xx, st4, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x15]},
+        {OP_fcmovbe, 0xdad510, "fcmovbe", st0, xx, st5, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x16]},
+        {OP_fcmovbe, 0xdad610, "fcmovbe", st0, xx, st6, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x17]},
+        {OP_fcmovbe, 0xdad710, "fcmovbe", st0, xx, st7, xx, xx, mrm, (fRC|fRP|fRZ), END_LIST},
+        {OP_fcmovu, 0xdad810, "fcmovu", st0, xx, st0, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x19]}, /* d8 = [0x18] */
+        {OP_fcmovu, 0xdad910, "fcmovu", st0, xx, st1, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x1a]},
+        {OP_fcmovu, 0xdada10, "fcmovu", st0, xx, st2, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x1b]},
+        {OP_fcmovu, 0xdadb10, "fcmovu", st0, xx, st3, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x1c]},
+        {OP_fcmovu, 0xdadc10, "fcmovu", st0, xx, st4, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x1d]},
+        {OP_fcmovu, 0xdadd10, "fcmovu", st0, xx, st5, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x1e]},
+        {OP_fcmovu, 0xdade10, "fcmovu", st0, xx, st6, xx, xx, mrm, (fRC|fRP|fRZ), tfh[2][0x1f]},
+        {OP_fcmovu, 0xdadf10, "fcmovu", st0, xx, st7, xx, xx, mrm, (fRC|fRP|fRZ), END_LIST},
+        {INVALID, 0xdae010, "(bad)", xx, xx, xx, xx, xx, no, x, NA}, /* e0 = [0x20] */
+        {INVALID, 0xdae110, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdae210, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdae310, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdae410, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdae510, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdae610, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdae710, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdae810, "(bad)", xx, xx, xx, xx, xx, no, x, NA}, /* e8 = [0x28] */
+        {OP_fucompp, 0xdae910, "fucompp", xx, xx, st0, st1, xx, mrm, x, END_LIST},
+        {INVALID, 0xdaea10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdaeb10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdaec10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdaed10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdaee10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdaef10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdaf010, "(bad)", xx, xx, xx, xx, xx, no, x, NA}, /* f0 = [0x30] */
+        {INVALID, 0xdaf110, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdaf210, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdaf310, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdaf410, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdaf510, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdaf610, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdaf710, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdaf810, "(bad)", xx, xx, xx, xx, xx, no, x, NA}, /* f8 = [0x38] */
+        {INVALID, 0xdaf910, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdafa10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdafb10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdafc10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdafd10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdafe10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdaff10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+   },
+    { /* db = [3] */
+        {OP_fcmovnb, 0xdbc010, "fcmovnb", st0, xx, st0, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x01]}, /* c0 = [0x00] */
+        {OP_fcmovnb, 0xdbc110, "fcmovnb", st0, xx, st1, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x02]},
+        {OP_fcmovnb, 0xdbc210, "fcmovnb", st0, xx, st2, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x03]},
+        {OP_fcmovnb, 0xdbc310, "fcmovnb", st0, xx, st3, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x04]},
+        {OP_fcmovnb, 0xdbc410, "fcmovnb", st0, xx, st4, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x05]},
+        {OP_fcmovnb, 0xdbc510, "fcmovnb", st0, xx, st5, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x06]},
+        {OP_fcmovnb, 0xdbc610, "fcmovnb", st0, xx, st6, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x07]},
+        {OP_fcmovnb, 0xdbc710, "fcmovnb", st0, xx, st7, xx, xx, mrm, (fRC|fRP|fRZ), END_LIST},
+        {OP_fcmovene, 0xdbc810, "fcmovene", st0, xx, st0, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x09]}, /* c8 = [0x08] */
+        {OP_fcmovene, 0xdbc910, "fcmovene", st0, xx, st1, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x0a]},
+        {OP_fcmovene, 0xdbca10, "fcmovene", st0, xx, st2, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x0b]},
+        {OP_fcmovene, 0xdbcb10, "fcmovene", st0, xx, st3, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x0c]},
+        {OP_fcmovene, 0xdbcc10, "fcmovene", st0, xx, st4, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x0d]},
+        {OP_fcmovene, 0xdbcd10, "fcmovene", st0, xx, st5, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x0e]},
+        {OP_fcmovene, 0xdbce10, "fcmovene", st0, xx, st6, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x0f]},
+        {OP_fcmovene, 0xdbcf10, "fcmovene", st0, xx, st7, xx, xx, mrm, (fRC|fRP|fRZ), END_LIST},
+        {OP_fcmovnbe, 0xdbd010, "fcmovnbe", st0, xx, st0, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x12]}, /* d0 = [0x10] */
+        {OP_fcmovnbe, 0xdbd110, "fcmovnbe", st0, xx, st1, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x12]},
+        {OP_fcmovnbe, 0xdbd210, "fcmovnbe", st0, xx, st2, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x13]},
+        {OP_fcmovnbe, 0xdbd310, "fcmovnbe", st0, xx, st3, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x14]},
+        {OP_fcmovnbe, 0xdbd410, "fcmovnbe", st0, xx, st4, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x15]},
+        {OP_fcmovnbe, 0xdbd510, "fcmovnbe", st0, xx, st5, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x16]},
+        {OP_fcmovnbe, 0xdbd610, "fcmovnbe", st0, xx, st6, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x17]},
+        {OP_fcmovnbe, 0xdbd710, "fcmovnbe", st0, xx, st7, xx, xx, mrm, (fRC|fRP|fRZ), END_LIST},
+        {OP_fcmovnu, 0xdbd810, "fcmovnu", st0, xx, st0, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x19]}, /* d8 = [0x18] */
+        {OP_fcmovnu, 0xdbd910, "fcmovnu", st0, xx, st1, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x1a]},
+        {OP_fcmovnu, 0xdbda10, "fcmovnu", st0, xx, st2, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x1b]},
+        {OP_fcmovnu, 0xdbdb10, "fcmovnu", st0, xx, st3, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x1c]},
+        {OP_fcmovnu, 0xdbdc10, "fcmovnu", st0, xx, st4, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x1d]},
+        {OP_fcmovnu, 0xdbdd10, "fcmovnu", st0, xx, st5, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x1e]},
+        {OP_fcmovnu, 0xdbde10, "fcmovnu", st0, xx, st6, xx, xx, mrm, (fRC|fRP|fRZ), tfh[3][0x1f]},
+        {OP_fcmovnu, 0xdbdf10, "fcmovnu", st0, xx, st7, xx, xx, mrm, (fRC|fRP|fRZ), END_LIST},
+        {INVALID, 0xdbe010, "(bad)", xx, xx, xx, xx, xx, no, x, NA}, /* e0 = [0x20] */
+        {INVALID, 0xdbe110, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {OP_fnclex, 0xdbe210, "fnclex", xx, xx, xx, xx, xx, mrm, x, END_LIST},/*FIXME: w/ preceding fwait instr, called "fclex"*/
+        {OP_fninit, 0xdbe310, "fninit", xx, xx, xx, xx, xx, mrm, x, END_LIST},/*FIXME: w/ preceding fwait instr, called "finit"*/
+        {INVALID, 0xdbe410, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdbe510, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdbe610, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdbe710, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {OP_fucomi, 0xdbe810, "fucomi", xx, xx, st0, st0, xx, mrm, (fWC|fWP|fWZ), tfh[3][0x29]}, /* e8 = [0x28] */
+        {OP_fucomi, 0xdbe910, "fucomi", xx, xx, st0, st1, xx, mrm, (fWC|fWP|fWZ), tfh[3][0x2a]},
+        {OP_fucomi, 0xdbea10, "fucomi", xx, xx, st0, st2, xx, mrm, (fWC|fWP|fWZ), tfh[3][0x2b]},
+        {OP_fucomi, 0xdbeb10, "fucomi", xx, xx, st0, st3, xx, mrm, (fWC|fWP|fWZ), tfh[3][0x2c]},
+        {OP_fucomi, 0xdbec10, "fucomi", xx, xx, st0, st4, xx, mrm, (fWC|fWP|fWZ), tfh[3][0x2d]},
+        {OP_fucomi, 0xdbed10, "fucomi", xx, xx, st0, st5, xx, mrm, (fWC|fWP|fWZ), tfh[3][0x2e]},
+        {OP_fucomi, 0xdbee10, "fucomi", xx, xx, st0, st6, xx, mrm, (fWC|fWP|fWZ), tfh[3][0x2f]},
+        {OP_fucomi, 0xdbef10, "fucomi", xx, xx, st0, st7, xx, mrm, (fWC|fWP|fWZ), END_LIST},
+        {OP_fcomi, 0xdbf010, "fcomi", xx, xx, st0, st0, xx, mrm, (fWC|fWP|fWZ), tfh[3][0x31]}, /* f0 = [0x30] */
+        {OP_fcomi, 0xdbf110, "fcomi", xx, xx, st0, st1, xx, mrm, (fWC|fWP|fWZ), tfh[3][0x32]},
+        {OP_fcomi, 0xdbf210, "fcomi", xx, xx, st0, st2, xx, mrm, (fWC|fWP|fWZ), tfh[3][0x33]},
+        {OP_fcomi, 0xdbf310, "fcomi", xx, xx, st0, st3, xx, mrm, (fWC|fWP|fWZ), tfh[3][0x34]},
+        {OP_fcomi, 0xdbf410, "fcomi", xx, xx, st0, st4, xx, mrm, (fWC|fWP|fWZ), tfh[3][0x35]},
+        {OP_fcomi, 0xdbf510, "fcomi", xx, xx, st0, st5, xx, mrm, (fWC|fWP|fWZ), tfh[3][0x36]},
+        {OP_fcomi, 0xdbf610, "fcomi", xx, xx, st0, st6, xx, mrm, (fWC|fWP|fWZ), tfh[3][0x37]},
+        {OP_fcomi, 0xdbf710, "fcomi", xx, xx, st0, st7, xx, mrm, (fWC|fWP|fWZ), END_LIST},
+        {INVALID, 0xdbf810, "(bad)", xx, xx, xx, xx, xx, no, x, NA}, /* f8 = [0x38] */
+        {INVALID, 0xdbf910, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdbfa10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdbfb10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdbfc10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdbfd10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdbfe10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdbff10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+   },
+    { /* dc = [4] */ 
+        {OP_fadd, 0xdcc010, "fadd", st0, xx, st0, st0, xx, mrm, x, tfh[4][0x01]}, /* c0 = [0x00] */
+        {OP_fadd, 0xdcc110, "fadd", st1, xx, st0, st1, xx, mrm, x, tfh[4][0x02]},
+        {OP_fadd, 0xdcc210, "fadd", st2, xx, st0, st2, xx, mrm, x, tfh[4][0x03]},
+        {OP_fadd, 0xdcc310, "fadd", st3, xx, st0, st3, xx, mrm, x, tfh[4][0x04]},
+        {OP_fadd, 0xdcc410, "fadd", st4, xx, st0, st4, xx, mrm, x, tfh[4][0x05]},
+        {OP_fadd, 0xdcc510, "fadd", st5, xx, st0, st5, xx, mrm, x, tfh[4][0x06]},
+        {OP_fadd, 0xdcc610, "fadd", st6, xx, st0, st6, xx, mrm, x, tfh[4][0x07]},
+        {OP_fadd, 0xdcc710, "fadd", st7, xx, st0, st7, xx, mrm, x, END_LIST},
+        {OP_fmul, 0xdcc810, "fmul", st0, xx, st0, st0, xx, mrm, x, tfh[4][0x09]}, /* c8 = [0x08] */
+        {OP_fmul, 0xdcc910, "fmul", st1, xx, st0, st1, xx, mrm, x, tfh[4][0x0a]},
+        {OP_fmul, 0xdcca10, "fmul", st2, xx, st0, st2, xx, mrm, x, tfh[4][0x0b]},
+        {OP_fmul, 0xdccb10, "fmul", st3, xx, st0, st3, xx, mrm, x, tfh[4][0x0c]},
+        {OP_fmul, 0xdccc10, "fmul", st4, xx, st0, st4, xx, mrm, x, tfh[4][0x0d]},
+        {OP_fmul, 0xdccd10, "fmul", st5, xx, st0, st5, xx, mrm, x, tfh[4][0x0e]},
+        {OP_fmul, 0xdcce10, "fmul", st6, xx, st0, st6, xx, mrm, x, tfh[4][0x0f]},
+        {OP_fmul, 0xdccf10, "fmul", st7, xx, st0, st7, xx, mrm, x, END_LIST},
+        /* Undocumented.  On sandpile.org as "fcom2".  We assume an alias for fcom
+         * and do not include in the encode chain. 
+         */
+        {OP_fcom, 0xdcd010, "fcom", xx, xx, st0, st0, xx, mrm, x, END_LIST}, /* d0 = [0x10] */
+        {OP_fcom, 0xdcd110, "fcom", xx, xx, st0, st1, xx, mrm, x, END_LIST},
+        {OP_fcom, 0xdcd210, "fcom", xx, xx, st0, st2, xx, mrm, x, END_LIST},
+        {OP_fcom, 0xdcd310, "fcom", xx, xx, st0, st3, xx, mrm, x, END_LIST},
+        {OP_fcom, 0xdcd410, "fcom", xx, xx, st0, st4, xx, mrm, x, END_LIST},
+        {OP_fcom, 0xdcd510, "fcom", xx, xx, st0, st5, xx, mrm, x, END_LIST},
+        {OP_fcom, 0xdcd610, "fcom", xx, xx, st0, st6, xx, mrm, x, END_LIST},
+        {OP_fcom, 0xdcd710, "fcom", xx, xx, st0, st7, xx, mrm, x, END_LIST},
+        /* Undocumented.  On sandpile.org as "fcomp3".  We assume an alias for fcomp
+         * and do not include in the encode chain. 
+         */
+        {OP_fcomp, 0xdcd810, "fcomp", xx, xx, st0, st0, xx, mrm, x, END_LIST}, /* d8 = [0x18] */
+        {OP_fcomp, 0xdcd910, "fcomp", xx, xx, st0, st1, xx, mrm, x, END_LIST},
+        {OP_fcomp, 0xdcda10, "fcomp", xx, xx, st0, st2, xx, mrm, x, END_LIST},
+        {OP_fcomp, 0xdcdb10, "fcomp", xx, xx, st0, st3, xx, mrm, x, END_LIST},
+        {OP_fcomp, 0xdcdc10, "fcomp", xx, xx, st0, st4, xx, mrm, x, END_LIST},
+        {OP_fcomp, 0xdcdd10, "fcomp", xx, xx, st0, st5, xx, mrm, x, END_LIST},
+        {OP_fcomp, 0xdcde10, "fcomp", xx, xx, st0, st6, xx, mrm, x, END_LIST},
+        {OP_fcomp, 0xdcdf10, "fcomp", xx, xx, st0, st7, xx, mrm, x, END_LIST},
+        {OP_fsubr, 0xdce010, "fsubr", st0, xx, st0, st0, xx, mrm, x, tfh[4][0x21]}, /* e0 = [0x20] */
+        {OP_fsubr, 0xdce110, "fsubr", st1, xx, st0, st1, xx, mrm, x, tfh[4][0x22]},
+        {OP_fsubr, 0xdce210, "fsubr", st2, xx, st0, st2, xx, mrm, x, tfh[4][0x23]},
+        {OP_fsubr, 0xdce310, "fsubr", st3, xx, st0, st3, xx, mrm, x, tfh[4][0x24]},
+        {OP_fsubr, 0xdce410, "fsubr", st4, xx, st0, st4, xx, mrm, x, tfh[4][0x25]},
+        {OP_fsubr, 0xdce510, "fsubr", st5, xx, st0, st5, xx, mrm, x, tfh[4][0x26]},
+        {OP_fsubr, 0xdce610, "fsubr", st6, xx, st0, st6, xx, mrm, x, tfh[4][0x27]},
+        {OP_fsubr, 0xdce710, "fsubr", st7, xx, st0, st7, xx, mrm, x, END_LIST},
+        {OP_fsub, 0xdce810, "fsub", st0, xx, st0, st0, xx, mrm, x, tfh[4][0x29]}, /* e8 = [0x28] */
+        {OP_fsub, 0xdce910, "fsub", st1, xx, st0, st1, xx, mrm, x, tfh[4][0x2a]},
+        {OP_fsub, 0xdcea10, "fsub", st2, xx, st0, st2, xx, mrm, x, tfh[4][0x2b]},
+        {OP_fsub, 0xdceb10, "fsub", st3, xx, st0, st3, xx, mrm, x, tfh[4][0x2c]},
+        {OP_fsub, 0xdcec10, "fsub", st4, xx, st0, st4, xx, mrm, x, tfh[4][0x2d]},
+        {OP_fsub, 0xdced10, "fsub", st5, xx, st0, st5, xx, mrm, x, tfh[4][0x2e]},
+        {OP_fsub, 0xdcee10, "fsub", st6, xx, st0, st6, xx, mrm, x, tfh[4][0x2f]},
+        {OP_fsub, 0xdcef10, "fsub", st7, xx, st0, st7, xx, mrm, x, END_LIST},
+        {OP_fdivr, 0xdcf010, "fdivr", st0, xx, st0, st0, xx, mrm, x, tfh[4][0x31]}, /* f0 = [0x30] */
+        {OP_fdivr, 0xdcf110, "fdivr", st1, xx, st0, st1, xx, mrm, x, tfh[4][0x32]},
+        {OP_fdivr, 0xdcf210, "fdivr", st2, xx, st0, st2, xx, mrm, x, tfh[4][0x33]},
+        {OP_fdivr, 0xdcf310, "fdivr", st3, xx, st0, st3, xx, mrm, x, tfh[4][0x34]},
+        {OP_fdivr, 0xdcf410, "fdivr", st4, xx, st0, st4, xx, mrm, x, tfh[4][0x35]},
+        {OP_fdivr, 0xdcf510, "fdivr", st5, xx, st0, st5, xx, mrm, x, tfh[4][0x36]},
+        {OP_fdivr, 0xdcf610, "fdivr", st6, xx, st0, st6, xx, mrm, x, tfh[4][0x37]},
+        {OP_fdivr, 0xdcf710, "fdivr", st7, xx, st0, st7, xx, mrm, x, END_LIST},
+        {OP_fdiv, 0xdcf810, "fdiv", st0, xx, st0, st0, xx, mrm, x, tfh[4][0x39]}, /* f8 = [0x38] */
+        {OP_fdiv, 0xdcf910, "fdiv", st1, xx, st0, st1, xx, mrm, x, tfh[4][0x3a]},
+        {OP_fdiv, 0xdcfa10, "fdiv", st2, xx, st0, st2, xx, mrm, x, tfh[4][0x3b]},
+        {OP_fdiv, 0xdcfb10, "fdiv", st3, xx, st0, st3, xx, mrm, x, tfh[4][0x3c]},
+        {OP_fdiv, 0xdcfc10, "fdiv", st4, xx, st0, st4, xx, mrm, x, tfh[4][0x3d]},
+        {OP_fdiv, 0xdcfd10, "fdiv", st5, xx, st0, st5, xx, mrm, x, tfh[4][0x3e]},
+        {OP_fdiv, 0xdcfe10, "fdiv", st6, xx, st0, st6, xx, mrm, x, tfh[4][0x3f]},
+        {OP_fdiv, 0xdcff10, "fdiv", st7, xx, st0, st7, xx, mrm, x, END_LIST},
+   },
+    { /* dd = [5] */
+        {OP_ffree, 0xddc010, "ffree", st0, xx, xx, xx, xx, mrm, x, tfh[5][0x01]}, /* c0 = [0x00] */
+        {OP_ffree, 0xddc110, "ffree", st1, xx, xx, xx, xx, mrm, x, tfh[5][0x02]},
+        {OP_ffree, 0xddc210, "ffree", st2, xx, xx, xx, xx, mrm, x, tfh[5][0x03]},
+        {OP_ffree, 0xddc310, "ffree", st3, xx, xx, xx, xx, mrm, x, tfh[5][0x04]},
+        {OP_ffree, 0xddc410, "ffree", st4, xx, xx, xx, xx, mrm, x, tfh[5][0x05]},
+        {OP_ffree, 0xddc510, "ffree", st5, xx, xx, xx, xx, mrm, x, tfh[5][0x06]},
+        {OP_ffree, 0xddc610, "ffree", st6, xx, xx, xx, xx, mrm, x, tfh[5][0x07]},
+        {OP_ffree, 0xddc710, "ffree", st7, xx, xx, xx, xx, mrm, x, END_LIST},
+        /* Undocumented.  On sandpile.org as "fxch4".  We assume an alias for fxch
+         * and do not include in the encode chain. 
+         */
+        {OP_fxch, 0xddc810, "fxch", st0, st0, st0, st0, xx, mrm, x, END_LIST}, /* c8 = [0x08] */
+        {OP_fxch, 0xddc910, "fxch", st0, st1, st0, st1, xx, mrm, x, END_LIST},
+        {OP_fxch, 0xddca10, "fxch", st0, st2, st0, st2, xx, mrm, x, END_LIST},
+        {OP_fxch, 0xddcb10, "fxch", st0, st3, st0, st3, xx, mrm, x, END_LIST},
+        {OP_fxch, 0xddcc10, "fxch", st0, st4, st0, st4, xx, mrm, x, END_LIST},
+        {OP_fxch, 0xddcd10, "fxch", st0, st5, st0, st5, xx, mrm, x, END_LIST},
+        {OP_fxch, 0xddce10, "fxch", st0, st6, st0, st6, xx, mrm, x, END_LIST},
+        {OP_fxch, 0xddcf10, "fxch", st0, st7, st0, st7, xx, mrm, x, END_LIST},
+        {OP_fst, 0xddd010, "fst", st0, xx, st0, xx, xx, mrm, x, tfh[5][0x11]}, /* d0 = [0x10] */
+        {OP_fst, 0xddd110, "fst", st1, xx, st0, xx, xx, mrm, x, tfh[5][0x12]},
+        {OP_fst, 0xddd210, "fst", st2, xx, st0, xx, xx, mrm, x, tfh[5][0x13]},
+        {OP_fst, 0xddd310, "fst", st3, xx, st0, xx, xx, mrm, x, tfh[5][0x14]},
+        {OP_fst, 0xddd410, "fst", st4, xx, st0, xx, xx, mrm, x, tfh[5][0x15]},
+        {OP_fst, 0xddd510, "fst", st5, xx, st0, xx, xx, mrm, x, tfh[5][0x16]},
+        {OP_fst, 0xddd610, "fst", st6, xx, st0, xx, xx, mrm, x, tfh[5][0x17]},
+        {OP_fst, 0xddd710, "fst", st7, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fstp, 0xddd810, "fstp", st0, xx, st0, xx, xx, mrm, x, tfh[5][0x19]}, /* d8 = [0x18] */
+        {OP_fstp, 0xddd910, "fstp", st1, xx, st0, xx, xx, mrm, x, tfh[5][0x1a]},
+        {OP_fstp, 0xddda10, "fstp", st2, xx, st0, xx, xx, mrm, x, tfh[5][0x1b]},
+        {OP_fstp, 0xdddb10, "fstp", st3, xx, st0, xx, xx, mrm, x, tfh[5][0x1c]},
+        {OP_fstp, 0xdddc10, "fstp", st4, xx, st0, xx, xx, mrm, x, tfh[5][0x1d]},
+        {OP_fstp, 0xdddd10, "fstp", st5, xx, st0, xx, xx, mrm, x, tfh[5][0x1e]},
+        {OP_fstp, 0xddde10, "fstp", st6, xx, st0, xx, xx, mrm, x, tfh[5][0x1f]},
+        {OP_fstp, 0xdddf10, "fstp", st7, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fucom, 0xdde010, "fucom", xx, xx, st0, st0, xx, mrm, x, tfh[5][0x21]}, /* e0 = [0x20] */
+        {OP_fucom, 0xdde110, "fucom", xx, xx, st1, st0, xx, mrm, x, tfh[5][0x22]},
+        {OP_fucom, 0xdde210, "fucom", xx, xx, st2, st0, xx, mrm, x, tfh[5][0x23]},
+        {OP_fucom, 0xdde310, "fucom", xx, xx, st3, st0, xx, mrm, x, tfh[5][0x24]},
+        {OP_fucom, 0xdde410, "fucom", xx, xx, st4, st0, xx, mrm, x, tfh[5][0x25]},
+        {OP_fucom, 0xdde510, "fucom", xx, xx, st5, st0, xx, mrm, x, tfh[5][0x26]},
+        {OP_fucom, 0xdde610, "fucom", xx, xx, st6, st0, xx, mrm, x, tfh[5][0x27]},
+        {OP_fucom, 0xdde710, "fucom", xx, xx, st7, st0, xx, mrm, x, END_LIST},
+        {OP_fucomp, 0xdde810, "fucomp", xx, xx, st0, st0, xx, mrm, x, tfh[5][0x29]}, /* e8 = [0x28] */
+        {OP_fucomp, 0xdde910, "fucomp", xx, xx, st1, st0, xx, mrm, x, tfh[5][0x2a]},
+        {OP_fucomp, 0xddea10, "fucomp", xx, xx, st2, st0, xx, mrm, x, tfh[5][0x2b]},
+        {OP_fucomp, 0xddeb10, "fucomp", xx, xx, st3, st0, xx, mrm, x, tfh[5][0x2c]},
+        {OP_fucomp, 0xddec10, "fucomp", xx, xx, st4, st0, xx, mrm, x, tfh[5][0x2d]},
+        {OP_fucomp, 0xdded10, "fucomp", xx, xx, st5, st0, xx, mrm, x, tfh[5][0x2e]},
+        {OP_fucomp, 0xddee10, "fucomp", xx, xx, st6, st0, xx, mrm, x, tfh[5][0x2f]},
+        {OP_fucomp, 0xddef10, "fucomp", xx, xx, st7, st0, xx, mrm, x, END_LIST},
+        {INVALID, 0xddf010, "(bad)", xx, xx, xx, xx, xx, no, x, NA}, /* f0 = [0x30] */
+        {INVALID, 0xddf110, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xddf210, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xddf310, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xddf410, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xddf510, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xddf610, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xddf710, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xddf810, "(bad)", xx, xx, xx, xx, xx, no, x, NA}, /* f8 = [0x38] */
+        {INVALID, 0xddf910, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xddfa10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xddfb10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xddfc10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xddfd10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xddfe10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xddff10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+   },
+    { /* de = [6]*/
+        {OP_faddp, 0xdec010, "faddp", st0, xx, st0, st0, xx, mrm, x, tfh[6][0x01]}, /* c0 = [0x00] */
+        {OP_faddp, 0xdec110, "faddp", st1, xx, st0, st1, xx, mrm, x, tfh[6][0x02]},
+        {OP_faddp, 0xdec210, "faddp", st2, xx, st0, st2, xx, mrm, x, tfh[6][0x03]},
+        {OP_faddp, 0xdec310, "faddp", st3, xx, st0, st3, xx, mrm, x, tfh[6][0x04]},
+        {OP_faddp, 0xdec410, "faddp", st4, xx, st0, st4, xx, mrm, x, tfh[6][0x05]},
+        {OP_faddp, 0xdec510, "faddp", st5, xx, st0, st5, xx, mrm, x, tfh[6][0x06]},
+        {OP_faddp, 0xdec610, "faddp", st6, xx, st0, st6, xx, mrm, x, tfh[6][0x07]},
+        {OP_faddp, 0xdec710, "faddp", st7, xx, st0, st7, xx, mrm, x, END_LIST},
+        {OP_fmulp, 0xdec810, "fmulp", st0, xx, st0, st0, xx, mrm, x, tfh[6][0x09]}, /* c8 = [0x08] */
+        {OP_fmulp, 0xdec910, "fmulp", st1, xx, st0, st1, xx, mrm, x, tfh[6][0x0a]},
+        {OP_fmulp, 0xdeca10, "fmulp", st2, xx, st0, st2, xx, mrm, x, tfh[6][0x0b]},
+        {OP_fmulp, 0xdecb10, "fmulp", st3, xx, st0, st3, xx, mrm, x, tfh[6][0x0c]},
+        {OP_fmulp, 0xdecc10, "fmulp", st4, xx, st0, st4, xx, mrm, x, tfh[6][0x0d]},
+        {OP_fmulp, 0xdecd10, "fmulp", st5, xx, st0, st5, xx, mrm, x, tfh[6][0x0e]},
+        {OP_fmulp, 0xdece10, "fmulp", st6, xx, st0, st6, xx, mrm, x, tfh[6][0x0f]},
+        {OP_fmulp, 0xdecf10, "fmulp", st7, xx, st0, st7, xx, mrm, x, END_LIST},
+        /* Undocumented.  On sandpile.org as "fcomp5".  We assume an alias for fcomp
+         * and do not include in the encode chain. 
+         */
+        {OP_fcomp, 0xded010, "fcomp", xx, xx, st0, st0, xx, mrm, x, END_LIST}, /* d0 = [0x10] */
+        {OP_fcomp, 0xded110, "fcomp", xx, xx, st0, st1, xx, mrm, x, END_LIST},
+        {OP_fcomp, 0xded210, "fcomp", xx, xx, st0, st2, xx, mrm, x, END_LIST},
+        {OP_fcomp, 0xded310, "fcomp", xx, xx, st0, st3, xx, mrm, x, END_LIST},
+        {OP_fcomp, 0xded410, "fcomp", xx, xx, st0, st4, xx, mrm, x, END_LIST},
+        {OP_fcomp, 0xded510, "fcomp", xx, xx, st0, st5, xx, mrm, x, END_LIST},
+        {OP_fcomp, 0xded610, "fcomp", xx, xx, st0, st6, xx, mrm, x, END_LIST},
+        {OP_fcomp, 0xded710, "fcomp", xx, xx, st0, st7, xx, mrm, x, END_LIST},
+        {INVALID, 0xded810, "(bad)", xx, xx, xx, xx, xx, no, x, NA}, /* d8 = [0x18] */
+        {OP_fcompp, 0xded910, "fcompp", xx, xx, st0, st1, xx, mrm, x, END_LIST},
+        {INVALID, 0xdeda10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdedb10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdedc10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdedd10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdede10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdedf10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {OP_fsubrp, 0xdee010, "fsubrp", st0, xx, st0, st0, xx, mrm, x, tfh[6][0x21]}, /* e0 = [0x20] */
+        {OP_fsubrp, 0xdee110, "fsubrp", st1, xx, st0, st1, xx, mrm, x, tfh[6][0x22]},
+        {OP_fsubrp, 0xdee210, "fsubrp", st2, xx, st0, st2, xx, mrm, x, tfh[6][0x23]},
+        {OP_fsubrp, 0xdee310, "fsubrp", st3, xx, st0, st3, xx, mrm, x, tfh[6][0x24]},
+        {OP_fsubrp, 0xdee410, "fsubrp", st4, xx, st0, st4, xx, mrm, x, tfh[6][0x25]},
+        {OP_fsubrp, 0xdee510, "fsubrp", st5, xx, st0, st5, xx, mrm, x, tfh[6][0x26]},
+        {OP_fsubrp, 0xdee610, "fsubrp", st6, xx, st0, st6, xx, mrm, x, tfh[6][0x27]},
+        {OP_fsubrp, 0xdee710, "fsubrp", st7, xx, st0, st7, xx, mrm, x, END_LIST},
+        {OP_fsubp, 0xdee810, "fsubp", st0, xx, st0, st0, xx, mrm, x, tfh[6][0x29]}, /* e8 = [0x28] */
+        {OP_fsubp, 0xdee910, "fsubp", st1, xx, st0, st1, xx, mrm, x, tfh[6][0x2a]},
+        {OP_fsubp, 0xdeea10, "fsubp", st2, xx, st0, st2, xx, mrm, x, tfh[6][0x2b]},
+        {OP_fsubp, 0xdeeb10, "fsubp", st3, xx, st0, st3, xx, mrm, x, tfh[6][0x2c]},
+        {OP_fsubp, 0xdeec10, "fsubp", st4, xx, st0, st4, xx, mrm, x, tfh[6][0x2d]},
+        {OP_fsubp, 0xdeed10, "fsubp", st5, xx, st0, st5, xx, mrm, x, tfh[6][0x2e]},
+        {OP_fsubp, 0xdeee10, "fsubp", st6, xx, st0, st6, xx, mrm, x, tfh[6][0x2f]},
+        {OP_fsubp, 0xdeef10, "fsubp", st7, xx, st0, st7, xx, mrm, x, END_LIST},
+        {OP_fdivrp, 0xdef010, "fdivrp", st0, xx, st0, st0, xx, mrm, x, tfh[6][0x31]}, /* f0 = [0x30] */
+        {OP_fdivrp, 0xdef110, "fdivrp", st1, xx, st0, st1, xx, mrm, x, tfh[6][0x32]},
+        {OP_fdivrp, 0xdef210, "fdivrp", st2, xx, st0, st2, xx, mrm, x, tfh[6][0x33]},
+        {OP_fdivrp, 0xdef310, "fdivrp", st3, xx, st0, st3, xx, mrm, x, tfh[6][0x34]},
+        {OP_fdivrp, 0xdef410, "fdivrp", st4, xx, st0, st4, xx, mrm, x, tfh[6][0x35]},
+        {OP_fdivrp, 0xdef510, "fdivrp", st5, xx, st0, st5, xx, mrm, x, tfh[6][0x36]},
+        {OP_fdivrp, 0xdef610, "fdivrp", st6, xx, st0, st6, xx, mrm, x, tfh[6][0x37]},
+        {OP_fdivrp, 0xdef710, "fdivrp", st7, xx, st0, st7, xx, mrm, x, END_LIST},
+        {OP_fdivp, 0xdef810, "fdivp", st0, xx, st0, st0, xx, mrm, x, tfh[6][0x39]}, /* f8 = [0x38] */
+        {OP_fdivp, 0xdef910, "fdivp", st1, xx, st0, st1, xx, mrm, x, tfh[6][0x3a]},
+        {OP_fdivp, 0xdefa10, "fdivp", st2, xx, st0, st2, xx, mrm, x, tfh[6][0x3b]},
+        {OP_fdivp, 0xdefb10, "fdivp", st3, xx, st0, st3, xx, mrm, x, tfh[6][0x3c]},
+        {OP_fdivp, 0xdefc10, "fdivp", st4, xx, st0, st4, xx, mrm, x, tfh[6][0x3d]},
+        {OP_fdivp, 0xdefd10, "fdivp", st5, xx, st0, st5, xx, mrm, x, tfh[6][0x3e]},
+        {OP_fdivp, 0xdefe10, "fdivp", st6, xx, st0, st6, xx, mrm, x, tfh[6][0x3f]},
+        {OP_fdivp, 0xdeff10, "fdivp", st7, xx, st0, st7, xx, mrm, x, END_LIST},
+   },
+    { /* df = [7] */
+        /* Undocumented by Intel, but is on p152 of "AMD Athlon
+         * Processor x86 Code Optimization Guide."
+         */
+        {OP_ffreep, 0xdfc010, "ffreep", st0, xx, xx, xx, xx, mrm, x, tfh[7][0x01]}, /* c0 = [0x00] */
+        {OP_ffreep, 0xdfc110, "ffreep", st1, xx, xx, xx, xx, mrm, x, tfh[7][0x02]},
+        {OP_ffreep, 0xdfc210, "ffreep", st2, xx, xx, xx, xx, mrm, x, tfh[7][0x03]},
+        {OP_ffreep, 0xdfc310, "ffreep", st3, xx, xx, xx, xx, mrm, x, tfh[7][0x04]},
+        {OP_ffreep, 0xdfc410, "ffreep", st4, xx, xx, xx, xx, mrm, x, tfh[7][0x05]},
+        {OP_ffreep, 0xdfc510, "ffreep", st5, xx, xx, xx, xx, mrm, x, tfh[7][0x06]},
+        {OP_ffreep, 0xdfc610, "ffreep", st6, xx, xx, xx, xx, mrm, x, tfh[7][0x07]},
+        {OP_ffreep, 0xdfc710, "ffreep", st7, xx, xx, xx, xx, mrm, x, END_LIST},
+        /* Undocumented.  On sandpile.org as "fxch7".  We assume an alias for fxch
+         * and do not include in the encode chain. 
+         */
+        {OP_fxch, 0xdfc810, "fxch", st0, st0, st0, st0, xx, mrm, x, END_LIST}, /* c8 = [0x08] */
+        {OP_fxch, 0xdfc910, "fxch", st0, st1, st0, st1, xx, mrm, x, END_LIST},
+        {OP_fxch, 0xdfca10, "fxch", st0, st2, st0, st2, xx, mrm, x, END_LIST},
+        {OP_fxch, 0xdfcb10, "fxch", st0, st3, st0, st3, xx, mrm, x, END_LIST},
+        {OP_fxch, 0xdfcc10, "fxch", st0, st4, st0, st4, xx, mrm, x, END_LIST},
+        {OP_fxch, 0xdfcd10, "fxch", st0, st5, st0, st5, xx, mrm, x, END_LIST},
+        {OP_fxch, 0xdfce10, "fxch", st0, st6, st0, st6, xx, mrm, x, END_LIST},
+        {OP_fxch, 0xdfcf10, "fxch", st0, st7, st0, st7, xx, mrm, x, END_LIST},
+        /* Undocumented.  On sandpile.org as "fstp8".  We assume an alias for fstp
+         * and do not include in the encode chain. 
+         */
+        {OP_fstp, 0xdfd010, "fstp", st0, xx, st0, xx, xx, mrm, x, END_LIST}, /* d0 = [0x10] */
+        {OP_fstp, 0xdfd110, "fstp", st1, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fstp, 0xdfd210, "fstp", st2, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fstp, 0xdfd310, "fstp", st3, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fstp, 0xdfd410, "fstp", st4, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fstp, 0xdfd510, "fstp", st5, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fstp, 0xdfd610, "fstp", st6, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fstp, 0xdfd710, "fstp", st7, xx, st0, xx, xx, mrm, x, END_LIST},
+        /* Undocumented.  On sandpile.org as "fstp9".  We assume an alias for fstp
+         * and do not include in the encode chain. 
+         */
+        {OP_fstp, 0xdfd810, "fstp", st0, xx, st0, xx, xx, mrm, x, END_LIST}, /* d8 = [0x18] */
+        {OP_fstp, 0xdfd910, "fstp", st1, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fstp, 0xdfda10, "fstp", st2, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fstp, 0xdfdb10, "fstp", st3, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fstp, 0xdfdc10, "fstp", st4, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fstp, 0xdfdd10, "fstp", st5, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fstp, 0xdfde10, "fstp", st6, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fstp, 0xdfdf10, "fstp", st7, xx, st0, xx, xx, mrm, x, END_LIST},
+        {OP_fnstsw, 0xdfe010, "fnstsw", ax, xx, xx, xx, xx, mrm, x, END_LIST}, /* e0 = [0x20] */ /*FIXME:w/ preceding fwait instr, this is "fstsw"*/
+        {INVALID, 0xdfe110, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdfe210, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdfe310, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdfe410, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdfe510, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdfe610, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdfe710, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {OP_fucomip, 0xdfe810, "fucomip", xx, xx, st0, st0, xx, mrm, (fWC|fWP|fWZ), tfh[7][0x29]}, /* e8 = [0x28] */
+        {OP_fucomip, 0xdfe910, "fucomip", xx, xx, st0, st1, xx, mrm, (fWC|fWP|fWZ), tfh[7][0x2a]},
+        {OP_fucomip, 0xdfea10, "fucomip", xx, xx, st0, st2, xx, mrm, (fWC|fWP|fWZ), tfh[7][0x2b]},
+        {OP_fucomip, 0xdfeb10, "fucomip", xx, xx, st0, st3, xx, mrm, (fWC|fWP|fWZ), tfh[7][0x2c]},
+        {OP_fucomip, 0xdfec10, "fucomip", xx, xx, st0, st4, xx, mrm, (fWC|fWP|fWZ), tfh[7][0x2d]},
+        {OP_fucomip, 0xdfed10, "fucomip", xx, xx, st0, st5, xx, mrm, (fWC|fWP|fWZ), tfh[7][0x2e]},
+        {OP_fucomip, 0xdfee10, "fucomip", xx, xx, st0, st6, xx, mrm, (fWC|fWP|fWZ), tfh[7][0x2f]},
+        {OP_fucomip, 0xdfef10, "fucomip", xx, xx, st0, st7, xx, mrm, (fWC|fWP|fWZ), END_LIST},
+        {OP_fcomip, 0xdff010, "fcomip", xx, xx, st0, st0, xx, mrm, (fWC|fWP|fWZ), tfh[7][0x31]}, /* f0 = [0x30] */
+        {OP_fcomip, 0xdff110, "fcomip", xx, xx, st0, st1, xx, mrm, (fWC|fWP|fWZ), tfh[7][0x32]},
+        {OP_fcomip, 0xdff210, "fcomip", xx, xx, st0, st2, xx, mrm, (fWC|fWP|fWZ), tfh[7][0x33]},
+        {OP_fcomip, 0xdff310, "fcomip", xx, xx, st0, st3, xx, mrm, (fWC|fWP|fWZ), tfh[7][0x34]},
+        {OP_fcomip, 0xdff410, "fcomip", xx, xx, st0, st4, xx, mrm, (fWC|fWP|fWZ), tfh[7][0x35]},
+        {OP_fcomip, 0xdff510, "fcomip", xx, xx, st0, st5, xx, mrm, (fWC|fWP|fWZ), tfh[7][0x36]},
+        {OP_fcomip, 0xdff610, "fcomip", xx, xx, st0, st6, xx, mrm, (fWC|fWP|fWZ), tfh[7][0x37]},
+        {OP_fcomip, 0xdff710, "fcomip", xx, xx, st0, st7, xx, mrm, (fWC|fWP|fWZ), END_LIST},
+        {INVALID, 0xdff810, "(bad)", xx, xx, xx, xx, xx, no, x, NA}, /* f8 = [0x38] */
+        {INVALID, 0xdff910, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdffa10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdffb10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdffc10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdffd10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdffe10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+        {INVALID, 0xdfff10, "(bad)", xx, xx, xx, xx, xx, no, x, NA},
+   },
+};
+
+/****************************************************************************
+ * Suffix extensions: 3DNow! and 3DNow!+ 
+ * Since there are only 24 of them, we save space by having a
+ * table of 256 indices instead of 256 instr_info_t structs.
+ */
+const byte suffix_index[256] = {
+  /* 0  1  2  3   4  5  6  7   8  9  A  B   C  D  E  F */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0, 20,18, 0, 0,  /* 0 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0, 21,19, 0, 0,  /* 1 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 2 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 3 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 4 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 5 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 6 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* 7 */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0,22, 0,  0, 0,23, 0,  /* 8 */
+     4, 0, 0, 0,  7, 0,10,13,  0, 0,16, 0,  0, 0, 2, 0,  /* 9 */
+     5, 0, 0, 0,  8, 0,11,14,  0, 0,17, 0,  0, 0, 3, 0,  /* A */
+     6, 0, 0, 0,  9, 0,12,15,  0, 0, 0,24,  0, 0, 0, 1,  /* B */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* C */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* D */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  /* E */
+     0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0,  0, 0, 0, 0   /* F */
+};
+const instr_info_t suffix_extensions[] = {
+    /* Rather than forging an exception let's anticipate future additions: we know
+     * (pretty sure anyway) that they'll have the same length and operand structure.
+     * Won't encode properly from Level 4 but that's ok.
+     */
+    {OP_unknown_3dnow, 0x000f0f90, "unknown 3DNow",
+                                          Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/* 0*/
+    {OP_pavgusb , 0xbf0f0f90, "pavgusb",  Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/* 1*/
+    {OP_pfadd   , 0x9e0f0f90, "pfadd",    Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/* 2*/
+    {OP_pfacc   , 0xae0f0f90, "pfacc",    Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/* 3*/
+    {OP_pfcmpge , 0x900f0f90, "pfcmpge",  Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/* 4*/
+    {OP_pfcmpgt , 0xa00f0f90, "pfcmpgt",  Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/* 5*/
+    {OP_pfcmpeq , 0xb00f0f90, "pfcmpeq",  Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/* 6*/
+    {OP_pfmin   , 0x940f0f90, "pfmin"  ,  Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/* 7*/
+    {OP_pfmax   , 0xa40f0f90, "pfmax"  ,  Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/* 8*/
+    {OP_pfmul   , 0xb40f0f90, "pfmul"  ,  Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/* 9*/
+    {OP_pfrcp   , 0x960f0f90, "pfrcp"  ,  Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/*10*/
+    {OP_pfrcpit1, 0xa60f0f90, "pfrcpit1", Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/*11*/
+    {OP_pfrcpit2, 0xb60f0f90, "pfrcpit2", Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/*12*/
+    {OP_pfrsqrt , 0x970f0f90, "pfrsqrt",  Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/*13*/
+    {OP_pfrsqit1, 0xa70f0f90, "pfrsqit1", Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/*14*/
+    {OP_pmulhrw , 0xb70f0f90, "pmulhrw",  Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/*15*/
+    {OP_pfsub   , 0x9a0f0f90, "pfsub"  ,  Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/*16*/
+    {OP_pfsubr  , 0xaa0f0f90, "pfsubr" ,  Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/*17*/
+    {OP_pi2fd   , 0x0d0f0f90, "pi2fd"  ,  Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/*18*/
+    {OP_pf2id   , 0x1d0f0f90, "pf2id",    Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/*19*/
+    {OP_pi2fw   , 0x0c0f0f90, "pi2fw"  ,  Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/*20*/
+    {OP_pf2iw   , 0x1c0f0f90, "pf2iw",    Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/*21*/
+    {OP_pfnacc  , 0x8a0f0f90, "pfnacc" ,  Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/*22*/
+    {OP_pfpnacc , 0x8e0f0f90, "pfpnacc",  Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/*23*/
+    {OP_pswapd  , 0xbb0f0f90, "pswapd" ,  Pq, xx, Qq, Pq, xx, mrm, x, END_LIST},/*24*/
+};
+
+/****************************************************************************
+ * To handle more than 2 dests or 3 sources we chain on extra instructions.
+ * All cases where we have extra operands are single-encoding-only instructions,
+ * so we use the list field to point to here.
+ * N.B.: the size of this table is hardcoded in decode.c.
+ * Also, only implicit operands are in these instruction extensions!!!
+ */
+const instr_info_t extra_operands[] =
+{
+    /* 0x00 */
+    {OP_CONTD, 0x000000, "<pusha cont'd>", xx, xx, eCX, eDX, eBP, xop, x, exop[0x01]},
+    {OP_CONTD, 0x000000, "<pusha cont'd>", xx, xx, eSI, eDI, xx, no, x, END_LIST},
+    /* 0x02 */
+    {OP_CONTD, 0x000000, "<popa cont'd>", eBX, eCX, xx, xx, xx, xop, x, exop[0x03]},
+    {OP_CONTD, 0x000000, "<popa cont'd>", eDX, eBP, xx, xx, xx, xop, x, exop[0x04]},
+    {OP_CONTD, 0x000000, "<popa cont'd>", eSI, eDI, xx, xx, xx, no, x, END_LIST},
+    /* 0x05 */
+    {OP_CONTD, 0x000000, "<enter cont'd>", axBP, xx, axBP, xx, xx, no, x, END_LIST},
+    /* 0x06 */
+    {OP_CONTD, 0x000000, "<cpuid cont'd>", ecx, edx, xx, xx, xx, no, x, END_LIST},
+    /* 0x07 */
+    {OP_CONTD, 0x000000, "<cmpxchg8b cont'd>", eDX, xx, eCX, eBX, xx, mrm, fWZ, END_LIST},
+    {OP_pcmpestrm,0x663a6018, "pcmpestrm", xx, xx, eax, edx, xx, mrm|reqp, fW6, END_LIST},
+    {OP_pcmpestri,0x663a6018, "pcmpestri", xx, xx, eax, edx, xx, mrm|reqp, fW6, END_LIST},
+};
+
diff --git a/core/x86/disassemble.c b/core/x86/disassemble.c
new file mode 100644
index 000000000..ec570699e
--- /dev/null
+++ b/core/x86/disassemble.c
@@ -0,0 +1,1330 @@
+/* **********************************************************
+ * Copyright (c) 2001-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2001 Hewlett-Packard Company */
+
+/* disassemble.c -- printing of x86 instructions
+ *
+ * Note that when printing out instructions:
+ * Uses AT&T syntax, NOT Intel syntax, unless -syntax_intel is specified.
+ * See the info pages for "as" for details on the differences.
+ *   From gdb: "The enter and bound instructions are printed with operands
+ *   in the same order as the intel book; everything else is printed in
+ *   reverse order."
+ */
+
+/*
+ * FIXME
+ *
+ * 1) I print "%st(0),%st(1)", gdb prints "%st,%st(1)"
+ * 2) I print movzx, gdb prints movzw (with an 'l' suffix tacked on)
+ * 3) gdb says bound and leave are to be printed "Intel order", not AT&T ?!?
+ * 4) add ATT-style 'l' & other suffixes based on op size
+ *    I have a set_suffix function, I'm not sure when the suffix is really
+ *    added by gdb...only when size not obvious from registers, etc.
+ * 5) print symbolic addresses
+ *    gdb: opcodes/i386-dis.c calls print_address_func, which points to
+ *    print_address in gdb/printcmd.c, which calls print_address_symbolic
+ *    in the same file.  We can get fragment tags and exit stubs through
+ *    our own data structures, so no reason to load symbol table and play
+ *    tricks like adding each fragment to it.
+ */
+
+#include "globals.h"
+#include "arch.h"
+#include "instr.h"
+#include "decode.h"
+#include "decode_fast.h"
+#include "disassemble.h"
+#include <string.h>
+
+/* these are only needed for symbolic address lookup: */
+#include "fragment.h" /* for fragment_pclookup */
+#include "link.h" /* for linkstub lookup */
+
+#include "fcache.h" /* for in_fcache */
+
+#if defined(INTERNAL) || defined(DEBUG) || defined(CLIENT_INTERFACE) 
+
+#ifdef DEBUG
+/* case 10450: give messages to clients */
+/* we can't undef ASSERT b/c of DYNAMO_OPTION */
+# undef ASSERT_TRUNCATE
+# undef ASSERT_BITFIELD_TRUNCATE
+# undef ASSERT_NOT_REACHED
+# define ASSERT_TRUNCATE DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_BITFIELD_TRUNCATE DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_NOT_REACHED DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+#endif
+
+/****************************************************************************
+ * Printing of instructions
+ */
+
+static inline const char *
+immed_prefix(void)
+{
+    return (DYNAMO_OPTION(syntax_intel) ? "" : "$");
+}
+
+static inline const char *
+postop_suffix(void)
+{
+    return (DYNAMO_OPTION(syntax_intel) ? "" : " ");
+}
+
+static void
+reg_disassemble(file_t outfile, reg_id_t reg, const char *prefix, const char *suffix)
+{
+    print_file(outfile, DYNAMO_OPTION(syntax_intel) ? "%s%s%s" : "%s%%%s%s",
+               prefix, reg_names[reg], suffix);
+}
+
+static void
+opnd_mem_disassemble_prefix(dcontext_t *dcontext, opnd_t opnd, file_t outfile)
+{
+    if (DYNAMO_OPTION(syntax_intel)) {
+        int sz =  opnd_size_in_bytes(opnd_get_size(opnd));
+        if (sz == 1)
+            print_file(outfile, "byte ptr [");
+        else if (sz == 2)
+            print_file(outfile, "word ptr [");
+        else if (sz == 4)
+            print_file(outfile, "dword ptr [");
+        else if (sz == 8)
+            print_file(outfile, "qword ptr [");
+        else if (sz == 10)
+            print_file(outfile, "tbyte ptr [");
+        else /* assume size implied by opcode */
+            print_file(outfile, "[");
+    }
+
+}
+
+static void
+opnd_base_disp_disassemble(dcontext_t *dcontext, opnd_t opnd, file_t outfile)
+{
+    reg_id_t seg = opnd_get_segment(opnd);
+    reg_id_t base = opnd_get_base(opnd);
+    int disp = opnd_get_disp(opnd);
+    reg_id_t index = opnd_get_index(opnd);
+    int scale = opnd_get_scale(opnd);
+
+    opnd_mem_disassemble_prefix(dcontext, opnd, outfile);
+
+    if (seg != REG_NULL)
+        reg_disassemble(outfile, seg, "", ":");
+
+    if (DYNAMO_OPTION(syntax_intel)) {
+        if (base != REG_NULL)
+            reg_disassemble(outfile, base, "", "");
+        if (index != REG_NULL) {
+            reg_disassemble(outfile, index, base == REG_NULL ? "" : "+", "");
+            if (scale > 1)
+                print_file(outfile, "*%d", scale);
+        }
+    }
+
+    if (disp != 0 || (base == REG_NULL && index == REG_NULL) ||
+        opnd_is_disp_encode_zero(opnd)) {
+        if (DYNAMO_OPTION(syntax_intel)) {
+            /* windbg negates if top byte is 0xff 
+             * for x64 udis86 negates if at all negative
+             */
+            if (IF_X64_ELSE(disp < 0, (disp & 0xff000000) == 0xff000000)) {
+                disp = -disp;
+                print_file(outfile, "-");
+            } else if (base != REG_NULL || index != REG_NULL)
+                print_file(outfile, "+");
+        }
+        if (disp >= INT8_MIN && disp <= INT8_MAX &&
+            !opnd_is_disp_force_full(opnd))
+            print_file(outfile, "0x%02x", disp);
+        else if (opnd_is_disp_short_addr(opnd))                    
+            print_file(outfile, "0x%04x", disp);
+        else /* there are no 64-bit displacements */
+            print_file(outfile, "0x%08x", disp);
+    }
+
+    if (!DYNAMO_OPTION(syntax_intel)) {
+        if (base != REG_NULL || index != REG_NULL) {
+            print_file(outfile, "(");
+            if (base != REG_NULL)
+                reg_disassemble(outfile, base, "", "");
+            if (index != REG_NULL) {
+                reg_disassemble(outfile, index, ",", "");
+                if (scale != 0)
+                    print_file(outfile, ",%d", scale);
+            }
+            print_file(outfile, ")");
+        }
+    }
+    
+    if (DYNAMO_OPTION(syntax_intel))
+        print_file(outfile, "]");
+    print_file(outfile, "%s", postop_suffix());
+}
+
+void
+opnd_disassemble(dcontext_t *dcontext, opnd_t opnd, file_t outfile)
+{
+    switch (opnd.kind) {
+    case NULL_kind:
+        break;
+    case IMMED_INTEGER_kind:
+        {
+            opnd_size_t sz = opnd_get_size(opnd);
+            /* PR 327775: when we don't know other operands we truncate.
+             * We rely on instr_disassemble to temporarily change operand
+             * size to sign-extend to match the size of adjacent operands.
+             */
+            if (sz == OPSZ_1 || sz == OPSZ_0) {
+                print_file(outfile, "%s0x%02x%s", immed_prefix(),
+                           (uint)((byte)opnd_get_immed_int(opnd)), postop_suffix());
+            } else if (sz == OPSZ_2) {
+                print_file(outfile, "%s0x%04x%s", immed_prefix(),
+                           (uint)((unsigned short)opnd_get_immed_int(opnd)),
+                           postop_suffix());
+            } else if (sz == OPSZ_4) {
+                print_file(outfile, "%s0x%08x%s", immed_prefix(),
+                           (uint)opnd_get_immed_int(opnd), postop_suffix());
+            } else {
+                print_file(outfile, "%s0x"ZHEX64_FORMAT_STRING"%s", immed_prefix(),
+                           opnd_get_immed_int(opnd), postop_suffix());
+            }
+        }
+        break;
+    case IMMED_FLOAT_kind:
+        {
+            /* need to save floating state around float printing */
+            PRESERVE_FLOATING_POINT_STATE({
+                uint top; uint bottom;
+                char *sign;
+                double_print(opnd_get_immed_float(opnd), 6, 
+                             &top, &bottom, &sign);
+                print_file(outfile, "%s%s%u.%.6u%s", immed_prefix(), sign, top, bottom,
+                           postop_suffix());
+            });
+            break;
+        }
+    case PC_kind:
+        {
+            app_pc target = opnd_get_pc(opnd);
+            bool printed = false;
+
+            /* symbolic addresses */
+            if (ENTER_DR_HOOK != NULL && target == (app_pc) ENTER_DR_HOOK) {
+                print_file(outfile, "$"PFX" <enter_dynamorio_hook> ", target);
+                printed = true;
+            } else if (EXIT_DR_HOOK != NULL && target == (app_pc) EXIT_DR_HOOK) {
+                print_file(outfile, "$"PFX" <exit_dynamorio_hook> ", target);
+                printed = true;
+            } else if (dcontext != NULL && dynamo_initialized
+# if defined(CLIENT_INTERFACE) || defined(STANDALONE_UNIT_TEST)
+                       && !standalone_library
+# endif
+                       ) {
+                const char *gencode_routine = NULL;
+                const char *ibl_brtype;
+                const char *ibl_name =
+                    get_ibl_routine_name(dcontext, target, &ibl_brtype);
+                if (ibl_name == NULL && in_coarse_stub_prefixes(target) &&
+                    *target == JMP_OPCODE) {
+                    ibl_name = get_ibl_routine_name(dcontext,
+                                                    PC_RELATIVE_TARGET(target+1),
+                                                    &ibl_brtype);
+                }
+# ifdef WINDOWS
+                /* must test first, as get_ibl_routine_name will think "bb_ibl_indjmp" */
+                if (dcontext != GLOBAL_DCONTEXT) {
+                    if (target == shared_syscall_routine(dcontext))
+                        gencode_routine = "shared_syscall";
+                    else if (target == unlinked_shared_syscall_routine(dcontext))
+                        gencode_routine = "unlinked_shared_syscall";
+                } else {
+                    if (target == shared_syscall_routine_ex(dcontext _IF_X64(GENCODE_X64)))
+                        gencode_routine = "shared_syscall";
+                    else if (target == unlinked_shared_syscall_routine_ex
+                             (dcontext _IF_X64(GENCODE_X64)))
+                        gencode_routine = "unlinked_shared_syscall";
+#  ifdef X64
+                    else if (target == shared_syscall_routine_ex
+                             (dcontext _IF_X64(GENCODE_X86)))
+                        gencode_routine = "x86_shared_syscall";
+                    else if (target == unlinked_shared_syscall_routine_ex
+                             (dcontext _IF_X64(GENCODE_X86)))
+                        gencode_routine = "x86_unlinked_shared_syscall";
+#  endif
+                }
+# endif
+                if (ibl_name) {
+                    /* can't use gencode_routine since need two strings here */
+                    print_file(outfile, "$"PFX" <%s_%s>", target, ibl_name, ibl_brtype);
+                    printed = true;
+                } else if (SHARED_FRAGMENTS_ENABLED() && target ==
+                           fcache_return_shared_routine(IF_X64(GENCODE_X64)))
+                    gencode_routine = "fcache_return";
+# ifdef X64
+                else if (SHARED_FRAGMENTS_ENABLED() && target ==
+                         fcache_return_shared_routine(IF_X64(GENCODE_X86)))
+                    gencode_routine = "x86_fcache_return";
+# endif
+                else if (dcontext != GLOBAL_DCONTEXT &&
+                         target == fcache_return_routine(dcontext))
+                    gencode_routine = "fcache_return";
+                else if (DYNAMO_OPTION(coarse_units)) {
+                    if (target == fcache_return_coarse_prefix(target, NULL) ||
+                        target == fcache_return_coarse_routine(IF_X64(GENCODE_X64)))
+                        gencode_routine = "fcache_return_coarse";
+                    else if (target == trace_head_return_coarse_prefix(target, NULL) ||
+                             target == trace_head_return_coarse_routine
+                             (IF_X64(GENCODE_X64)))
+                        gencode_routine = "trace_head_return_coarse";
+# ifdef X64
+                    else if (target == fcache_return_coarse_prefix(target, NULL) ||
+                             target == fcache_return_coarse_routine(IF_X64(GENCODE_X86)))
+                        gencode_routine = "x86_fcache_return_coarse";
+                    else if (target == trace_head_return_coarse_prefix(target, NULL) ||
+                             target == trace_head_return_coarse_routine
+                             (IF_X64(GENCODE_X86)))
+                        gencode_routine = "x86_trace_head_return_coarse";
+# endif
+                }
+#ifdef RETURN_STACK
+                else if (target == return_lookup_routine(dcontext))
+                    gencode_routine = "return_lookup";
+                else if (target == unlinked_return_routine(dcontext))
+                    gencode_routine = "unlinked_return";
+#endif
+#ifdef PROFILE_RDTSC
+                else if ((void *)target == profile_fragment_enter)
+                    gencode_routine = "profile_fragment_enter";
+#endif
+#ifdef TRACE_HEAD_CACHE_INCR
+                else if ((void *)target == trace_head_incr_routine(dcontext))
+                    gencode_routine = "trace_head_incr";
+#endif
+
+                if (gencode_routine != NULL) {
+                    print_file(outfile, "$"PFX" <%s> ", target, gencode_routine);
+                    printed = true;
+                } else if (!printed && fragment_initialized(dcontext)) {
+                    /* see if target is in a fragment */
+                    bool alloc = false;
+                    fragment_t *fragment;
+#ifdef DEBUG
+                    fragment_t wrapper;
+                    /* Unfortunately our fast lookup by fcache unit has lock
+                     * ordering issues which we get around by using the htable
+                     * method, though that won't find invisible fragments
+                     * (FIXME: for those could perhaps pass in a pointer).
+                     * For !DEADLOCK_AVOIDANCE, OWN_MUTEX's conservative imprecision
+                     * is fine.
+                     */
+                    if ((SHARED_FRAGMENTS_ENABLED() && OWN_MUTEX(&change_linking_lock))
+                        /* HACK to avoid recursion if the pclookup invokes
+                         * decode_fragment() (for coarse target) and it then invokes
+                         * disassembly
+                         */
+                        IF_DEBUG(|| (dcontext != GLOBAL_DCONTEXT &&
+                                     dcontext->in_opnd_disassemble))) {
+                        fragment = fragment_pclookup_by_htable(dcontext, (void *)target,
+                                                               &wrapper);
+                    } else {
+                        bool prev_flag = false;
+                        if (dcontext != GLOBAL_DCONTEXT) {
+                            prev_flag = dcontext->in_opnd_disassemble;
+                            dcontext->in_opnd_disassemble = true; 
+                        }
+#endif /* shouldn't be any logging so no disasm in the middle of sensitive ops */
+                        fragment = fragment_pclookup_with_linkstubs(dcontext, target,
+                                                                    &alloc);
+#ifdef DEBUG
+                        if (dcontext != GLOBAL_DCONTEXT)
+                            dcontext->in_opnd_disassemble = prev_flag;
+                    }
+#endif
+                    if (fragment != NULL) {
+                        if (FCACHE_ENTRY_PC(fragment) == (cache_pc)target ||
+                            FCACHE_PREFIX_ENTRY_PC(fragment) == (cache_pc)target ||
+                            FCACHE_IBT_ENTRY_PC(fragment) == (cache_pc)target) {
+#ifdef DEBUG
+                            print_file(outfile, "$"PFX" <fragment %d> ",
+                                       target, fragment->id);
+#else
+                            print_file(outfile, "$"PFX" <fragment "PFX"> ",
+                                       target, fragment->tag);
+#endif
+                            printed = true;
+                        } else if (!TEST(FRAG_FAKE, fragment->flags)) {
+                            /* check exit stubs */
+                            linkstub_t *ls;
+                            int ls_num = 0;
+                            CLIENT_ASSERT(!TEST(FRAG_FAKE, fragment->flags),
+                                          "opnd_disassemble: invalid target");
+                            for (ls=FRAGMENT_EXIT_STUBS(fragment); ls; ls=LINKSTUB_NEXT_EXIT(ls)) {
+                                if (target == EXIT_STUB_PC(dcontext, fragment, ls)) {
+                                    print_file(outfile, "$"PFX" <exit stub %d> ",
+                                               target, ls_num);
+                                    printed = true;
+                                    break;
+                                }
+                                ls_num++;
+                            }
+                        }
+                        if (alloc)
+                            fragment_free(dcontext, fragment);
+                    } else if (coarse_is_entrance_stub(target)) {
+                        print_file(outfile, "$"PFX" <entrance stub for "PFX"> ",
+                                   target, entrance_stub_target_tag(target));
+                        printed = true;
+                    }                        
+                }
+            } else if (dynamo_initialized && !SHARED_FRAGMENTS_ENABLED()
+# if defined(CLIENT_INTERFACE) || defined(STANDALONE_UNIT_TEST)
+                       && !standalone_library
+# endif
+                       ) {
+                    print_file(outfile, "NULL DCONTEXT! ");
+            }
+            if (!printed) {
+                print_file(outfile, "%s"PFX"%s", immed_prefix(), target,
+                           postop_suffix());
+            }
+            break;
+        }
+    case FAR_PC_kind:
+        /* constant is selector and not a SEG_constant */
+        print_file(outfile, "0x%04x:"PFX"%s",
+                   (ushort)opnd_get_segment_selector(opnd), opnd_get_pc(opnd),
+                   postop_suffix());
+        break;
+    case INSTR_kind:
+        print_file(outfile, "instr_ptr%s", postop_suffix());
+        break;
+    case FAR_INSTR_kind:
+        /* constant is selector and not a SEG_constant */
+        print_file(outfile, "0x%04x:"PFX"%s",
+                   (ushort)opnd_get_segment_selector(opnd), opnd_get_pc(opnd),
+                   postop_suffix());
+        print_file(outfile, "instr_ptr ");
+        break;
+    case REG_kind:
+        reg_disassemble(outfile, opnd_get_reg(opnd), "", postop_suffix());
+        break;
+    case BASE_DISP_kind:
+        opnd_base_disp_disassemble(dcontext, opnd, outfile);
+        break;
+#ifdef X64
+    case REL_ADDR_kind:
+        print_file(outfile, "<rel> ");
+        /* fall-through */
+    case ABS_ADDR_kind:
+        opnd_mem_disassemble_prefix(dcontext, opnd, outfile);
+        if (opnd_get_segment(opnd) != REG_NULL)
+            reg_disassemble(outfile, opnd_get_segment(opnd), "", ":");
+        print_file(outfile, PFX"%s%s", opnd_get_addr(opnd),
+                   DYNAMO_OPTION(syntax_intel) ? "]" : "",
+                   postop_suffix());
+        break;
+#endif
+    default:
+        print_file(outfile, "UNKNOWN OPERAND TYPE %d\n", opnd.kind);
+        CLIENT_ASSERT(false, "opnd_disassemble: invalid opnd type");
+    }
+}
+
+/* Disassembles the instruction at pc and prints the result to outfile
+ * Returns a pointer to the pc of the next instruction.
+ * Returns NULL if the instruction at pc is invalid.
+ */
+static byte *
+internal_disassemble(dcontext_t *dcontext, byte *pc, byte *orig_pc, file_t outfile,
+                     bool with_pc, bool with_bytes, const char *extra_bytes_prefix)
+{
+    int i, sz = 0, extra_sz = 0;
+    byte * next_pc;
+    instr_t instr;
+    bool valid = true;
+
+    instr_init(dcontext, &instr);
+    if (orig_pc != pc)
+        next_pc = decode_from_copy(dcontext, pc, orig_pc, &instr);
+    else
+        next_pc = decode(dcontext, pc, &instr);
+    if (next_pc == NULL) {
+        valid = false;
+        /* HACK: if decode_fast thinks it knows size use that */
+        next_pc = decode_next_pc(dcontext, pc);
+    }
+    if (next_pc == NULL) {
+        valid = false;
+        /* last resort: arbitrarily pick 4 bytes */
+        next_pc = pc + 4;
+    }
+    
+    if (with_pc)
+        print_file(outfile, "  "PFX" ", orig_pc);
+
+    if (with_bytes) {
+        sz = (int) (next_pc - pc);
+        if (sz > 7) {
+            extra_sz = sz - 7;
+            sz = 7;
+        } else
+            extra_sz = 0;
+        for (i = 0; i < sz; i++)
+            print_file(outfile, " %02x", *(pc + i));
+        if (!instr_valid(&instr)) {
+            print_file(outfile, "...?? ");
+            sz += 2;
+        }
+        for (i = sz; i < 7; i++)
+            print_file(outfile, "   ");
+        print_file(outfile, " ");
+    }
+
+    instr_disassemble(dcontext, &instr, outfile);
+
+    print_file(outfile, "\n");
+
+    if (with_bytes) {
+        if (extra_sz > 0) {
+            if (with_pc) {
+                print_file(outfile, IF_X64_ELSE("%21s","%13s")"%s"," ",
+                           extra_bytes_prefix);
+            } else
+                print_file(outfile, "    %s", extra_bytes_prefix);
+            for (i = 0; i < extra_sz; i++)
+                print_file(outfile, " %02x", *(pc + 7 + i));
+            print_file(outfile, "\n");
+        }
+    }
+
+    instr_free(dcontext, &instr);
+
+    return (valid ? next_pc : NULL);
+}
+
+/****************************************************************************
+ * Exported routines
+ */
+
+/* Disassembles the instruction at pc and prints the result to outfile
+ * Returns a pointer to the pc of the next instruction.
+ * Returns NULL if the instruction at pc is invalid.
+ */
+byte *
+disassemble(dcontext_t *dcontext, byte *pc, file_t outfile)
+{
+    return internal_disassemble(dcontext, pc, pc, outfile, true, false, "");
+}
+
+/* Disassembles a single instruction and prints its pc and
+ * bytes then the disassembly.
+ * Returns the pc of the next instruction.
+ * If the instruction at pc is invalid, guesses a size!
+ * This is b/c we internally use that feature, but I don't want to export it:
+ * so this unexported routine maintains it, and we don't have to change all
+ * our call sites to check for NULL.
+ */
+byte *
+disassemble_with_bytes(dcontext_t *dcontext, byte *pc, file_t outfile)
+{
+    byte *next_pc = internal_disassemble(dcontext, pc, pc, outfile, true, true, "");
+    if (next_pc == NULL) {
+        next_pc = decode_next_pc(dcontext, pc);
+        if (next_pc == NULL)
+            next_pc = pc + 4; /* guess size */
+    }
+    return next_pc;
+}
+
+/* Disassembles a single instruction, optionally printing its pc (if show_pc)
+ * and its raw bytes (show_bytes) beforehand.
+ * Returns the pc of the next instruction.
+ * FIXME: vs disassemble_with_bytes -- didn't want to update all callers
+ * so leaving, though should probably remove.
+ * Returns NULL if the instruction at pc is invalid.
+ */
+byte *
+disassemble_with_info(dcontext_t *dcontext, byte *pc, file_t outfile,
+                      bool show_pc, bool show_bytes)
+{
+    return internal_disassemble(dcontext, pc, pc, outfile, show_pc, show_bytes, "");
+}
+
+/*
+ * Decodes the instruction at address \p copy_pc as though
+ * it were located at address \p orig_pc, and then prints the
+ * instruction to file \p outfile.
+ * Prior to the instruction the address \p orig_pc is printed if \p show_pc and the raw
+ * bytes are printed if \p show_bytes.
+ * Returns the address of the subsequent instruction after the copy at
+ * \p copy_pc, or NULL if the instruction at \p copy_pc is invalid.
+ */
+byte *
+disassemble_from_copy(dcontext_t *dcontext, byte *copy_pc, byte *orig_pc,
+                      file_t outfile, bool show_pc, bool show_bytes)
+{
+    return internal_disassemble(dcontext, copy_pc, orig_pc, outfile,
+                                show_pc, show_bytes, "");
+}
+
+static bool
+instr_implicit_reg(instr_t *instr)
+{
+    /* instrs that have multiple encodings whose reg opnds are always implicit */
+    switch (instr_get_opcode(instr)) {
+    case OP_ins: case OP_rep_ins:
+    case OP_outs: case OP_rep_outs:
+    case OP_movs: case OP_rep_movs:
+    case OP_stos: case OP_rep_stos:
+    case OP_lods: case OP_rep_lods:
+    case OP_cmps: case OP_rep_cmps: case OP_repne_cmps:
+    case OP_scas: case OP_rep_scas: case OP_repne_scas:
+        return true;
+    }
+    return false;
+}
+
+static bool
+opnd_disassemble_intel(dcontext_t *dcontext, file_t outfile, instr_t *instr,
+                       byte optype, opnd_t opnd, bool prev, bool multiple_encodings)
+{
+    switch (optype) {
+    case TYPE_REG:
+    case TYPE_VAR_REG:
+    case TYPE_VARZ_REG:
+    case TYPE_VAR_XREG:
+    case TYPE_REG_EX:
+    case TYPE_VAR_REG_EX:
+    case TYPE_VAR_XREG_EX:
+    case TYPE_VAR_REGX_EX:
+        /* we do want to print implicit operands for opcode-decides-register
+         * instrs like inc-reg and pop-reg, but not for say lahf, aaa, or cdq.
+         */
+        if (!multiple_encodings || instr_implicit_reg(instr) ||
+            /* if has implicit st0 then don't print it */
+            (opnd_get_reg(opnd) == REG_ST0 && instr_memory_reference_size(instr) > 0))
+            return false;
+        /* else fall through */
+    case TYPE_A:
+    case TYPE_C:
+    case TYPE_D:
+    case TYPE_E:
+    case TYPE_G:
+    case TYPE_I:
+    case TYPE_J:
+    case TYPE_M:
+    case TYPE_O:
+    case TYPE_P:
+    case TYPE_Q:
+    case TYPE_R:
+    case TYPE_S:
+    case TYPE_V:
+    case TYPE_W:
+    case TYPE_P_MODRM:
+    case TYPE_V_MODRM:
+    case TYPE_FLOATMEM:
+    case TYPE_1:
+        if (prev)
+            print_file(outfile, ", ");
+        opnd_disassemble(dcontext, opnd, outfile);
+        return true;
+    case TYPE_X:
+    case TYPE_XLAT:
+    case TYPE_MASKMOVQ:
+        if (opnd_get_segment(opnd) != SEG_DS) {
+            /* FIXME: really we should put before opcode */
+            if (prev)
+                print_file(outfile, ", ");
+            reg_disassemble(outfile, opnd_get_segment(opnd), "", postop_suffix());
+            return true;
+        }
+    default:
+        /* implicit operand */
+        return false;
+    }
+    return false;
+}
+
+static void
+instr_disassemble_opnds_intel(dcontext_t *dcontext, instr_t *instr, file_t outfile)
+{
+    /* We need to find the non-implicit operands */
+    const instr_info_t *info;
+    int i;
+    byte optype, dst0type = TYPE_NONE, dst1type = TYPE_NONE;
+    opnd_t opnd;
+    bool prev = false, multiple_encodings = false;
+
+    info = instr_get_instr_info(instr);
+    if (info != NULL && get_next_instr_info(info) != NULL &&
+        !TESTALL(HAS_EXTRA_OPERANDS | EXTRAS_IN_CODE_FIELD, info->flags))
+        multiple_encodings = true;
+
+    info = get_encoding_info(instr);
+    if (info == NULL) {
+        print_file(outfile, "<INVALID>");
+        return;
+    }
+    for (i=0; i<instr_num_dsts(instr); i++) {
+        bool printing;
+        opnd = instr_get_dst(instr, i);
+        optype = instr_info_opnd_type(info, false/*dst*/, i);
+        printing = opnd_disassemble_intel(dcontext, outfile, instr, optype, opnd, prev,
+                                          multiple_encodings);
+        if (printing) {
+            /* w/o the "printing" check we suppress "push esp" => "push" */
+            if (i == 0)
+                dst0type = optype;
+            else if (i == 1)
+                dst1type = optype;
+        }
+        prev = printing || prev;
+    }
+    for (i=0; i<instr_num_srcs(instr); i++) {
+        opnd = instr_get_src(instr, i);
+        optype = instr_info_opnd_type(info, true/*src*/, i);
+        /* PR 312458: still not matching Intel-style tools like windbg or udis86:
+         * we need to suppress certain implicit operands, such as:
+         * - div dx, ax
+         * - imul ax
+         * - idiv edx, eax
+         * - in al
+         */
+
+        /* Don't re-do src==dst of ALU ops */
+        if ((optype != dst0type && optype != dst1type) ||
+            /* Don't suppress 2nd of st* if FP ALU */
+            (i == 0 && opnd_is_reg(opnd) && reg_is_fp(opnd_get_reg(opnd)))) {
+            prev = opnd_disassemble_intel(dcontext, outfile, instr, optype, opnd, prev,
+                                          multiple_encodings)
+                || prev;
+        }
+    }
+}
+
+static const char *
+instr_opcode_name(instr_t *instr, const instr_info_t *info)
+{
+    if (DYNAMO_OPTION(syntax_intel)) {
+        switch (instr_get_opcode(instr)) {
+        /* remove "l" prefix */
+        case OP_call_far: return "call";
+        case OP_call_far_ind: return "call";
+        case OP_jmp_far: return "jmp";
+        case OP_jmp_far_ind: return "jmp";
+        case OP_ret_far: return "retf";
+        }
+    }
+#ifdef X64
+    if (!instr_get_x86_mode(instr) && instr_get_opcode(instr) == OP_jecxz) {
+        return "jrcxz";
+    }
+#endif
+    return info->name;
+}
+
+static const char *
+instr_opcode_name_suffix(instr_t *instr)
+{
+    if (DYNAMO_OPTION(syntax_intel)) {
+        /* add "b" or "d" suffix */
+        switch (instr_get_opcode(instr)) {
+        case OP_pushf: case OP_popf:
+        case OP_pusha: case OP_popa:
+        case OP_iret: case OP_xlat:
+        case OP_ins: case OP_rep_ins:
+        case OP_outs: case OP_rep_outs:
+        case OP_movs: case OP_rep_movs:
+        case OP_stos: case OP_rep_stos:
+        case OP_lods: case OP_rep_lods:
+        case OP_cmps: case OP_rep_cmps: case OP_repne_cmps:
+        case OP_scas: case OP_rep_scas: case OP_repne_scas:
+            {
+                uint sz = instr_memory_reference_size(instr);
+                if (sz == 1)
+                    return "b";
+                else if (sz == 2)
+                    return "w";
+                else if (sz == 4)
+                    return "d";
+                else if (sz == 8)
+                    return "q";
+            }
+        }
+    }
+    return "";
+}
+
+/*
+ * Prints the instruction instr to file outfile. 
+ * Does not print addr16 or data16 prefixes for other than just-decoded instrs,
+ * and does not check that the instruction has a valid encoding.
+ * Prints each operand with leading zeros indicating the size.
+ */
+void
+instr_disassemble(dcontext_t *dcontext, instr_t *instr, file_t outfile)
+{
+    int i, sz;
+    const instr_info_t * info;
+    const char * name;
+    int name_width = 6;
+    if (!instr_valid(instr)) {
+        print_file(outfile, "<INVALID>");
+        return;
+    } else if (instr_is_label(instr)) {
+        print_file(outfile, "<label>");
+        return;
+    } else if (instr_opcode_valid(instr)) {
+        info = instr_get_instr_info(instr);
+        name = instr_opcode_name(instr, info);
+    } else
+        name = "<RAW>";
+
+    if ((instr->prefixes & PREFIX_LOCK) != 0)
+        print_file(outfile, "lock ");
+    /* Note that we do not try to figure out data16 or addr16 prefixes
+     * if they are not already set from a recent decode;
+     * we don't want to enforce a valid encoding at this point.
+     *
+     * To walk the operands and find addr16, we'd need to look for
+     * opnd_is_disp_short_addr() as well as push/pop of REG_SP, jecxz/loop* of
+     * REG_CX, or string ops, maskmov*, or xlat of REG_DI or REG_SI.
+     * For data16, we'd look for 16-bit reg or OPSZ_2 immed or base_disp.
+     */
+    if (!DYNAMO_OPTION(syntax_intel)) {
+        if (TEST(PREFIX_DATA, instr->prefixes))
+            print_file(outfile, "data16 ");
+        if (TEST(PREFIX_ADDR, instr->prefixes))
+            print_file(outfile, X64_MODE_DC(dcontext) ? "addr32 " : "addr16 ");
+#if 0 /* disabling for PR 256226 */
+        if (TEST(PREFIX_REX_W, instr->prefixes))
+            print_file(outfile, "rex.w ");
+#endif
+    }
+
+    print_file(outfile, "%s%s", name, instr_opcode_name_suffix(instr));
+
+    if (TEST(PREFIX_JCC_TAKEN, instr->prefixes)) {
+        print_file(outfile, ",pt ");
+        name_width -= 3;
+    } else if (TEST(PREFIX_JCC_NOT_TAKEN, instr->prefixes)) {
+        print_file(outfile, ",pn ");
+        name_width -= 3;
+    } else
+        print_file(outfile, " ");
+
+    /* FIXME: AT&T suffix */
+
+    IF_X64(CLIENT_ASSERT(CHECK_TRUNCATE_TYPE_int(strlen(name)),
+                         "instr_disassemble: internal truncation error"));
+    sz = (int) strlen(name) + (int) strlen(instr_opcode_name_suffix(instr));
+    for (i=sz; i<name_width; i++)
+        print_file(outfile, " ");
+
+    /* operands */
+    if (!instr_operands_valid(instr)) {
+        /* we could decode the raw bits, but caller should if they want that */
+        byte *raw = instr_get_raw_bits(instr);
+        uint len = instr_length(dcontext, instr);
+        byte *b;
+        print_file(outfile, "<raw "PFX"-"PFX" ==", raw, raw + len);
+        for (b = raw; b < raw + len && b < raw + 9; b++)
+            print_file(outfile, " %02x", *b);
+        if (len > 9)
+            print_file(outfile, " ...");
+        print_file(outfile, ">");
+        return;
+    }
+
+    if (DYNAMO_OPTION(syntax_intel)) {
+        instr_disassemble_opnds_intel(dcontext, instr, outfile);
+        return;
+    }
+
+    for (i=0; i<instr_num_srcs(instr); i++) {
+        opnd_t src = instr_get_src(instr, i);
+
+        /* PR 327775: force operand to sign-extend if all other operands
+         * are of a larger and identical-to-each-other size (since we
+         * don't want to extend immeds used in stores) and are not
+         * multimedia registers (since immeds there are always indices).
+         */
+        if (opnd_is_immed_int(src)) {
+            opnd_size_t opsz = OPSZ_NA;
+            bool resize = true;
+            int j;
+            for (j=0; j<instr_num_srcs(instr); j++) {
+                if (j != i) {
+                    if (opnd_is_reg(instr_get_src(instr, j)) &&
+                        !reg_is_gpr(opnd_get_reg(instr_get_src(instr, j)))) {
+                        resize = false;
+                        break;
+                    }
+                    if (opsz == OPSZ_NA)
+                        opsz = opnd_get_size(instr_get_src(instr, j));
+                    else if (opsz != opnd_get_size(instr_get_src(instr, j))) {
+                        resize = false;
+                        break;
+                    }
+                }
+            }
+            for (j=0; j<instr_num_dsts(instr); j++) {
+                if (j != i) {
+                    if (opnd_is_reg(instr_get_dst(instr, j)) &&
+                        !reg_is_gpr(opnd_get_reg(instr_get_dst(instr, j)))) {
+                        resize = false;
+                        break;
+                    }
+                    if (opsz == OPSZ_NA)
+                        opsz = opnd_get_size(instr_get_dst(instr, j));
+                    else if (opsz != opnd_get_size(instr_get_dst(instr, j))) {
+                        resize = false;
+                        break;
+                    }
+                }
+            }
+            if (resize && opsz != OPSZ_NA && !instr_is_interrupt(instr))
+                opnd_set_size(&src, opsz);
+        }
+
+        opnd_disassemble(dcontext, src, outfile);
+    }
+    if (instr_num_dsts(instr) > 0) {
+        print_file(outfile, "-> ");
+        for (i=0; i<instr_num_dsts(instr); i++)
+            opnd_disassemble(dcontext, instr_get_dst(instr, i), outfile);
+    }
+}
+
+static inline char*
+exit_stub_type_desc(dcontext_t *dcontext, fragment_t *f, linkstub_t *l)
+{
+    if (LINKSTUB_DIRECT(l->flags)) {
+        if (TEST(LINK_CALL, l->flags))
+            return "call";
+        if (TEST(LINK_JMP, l->flags))
+            return "jmp/jcc";
+        return "fall-through/speculated/IAT";
+        /* FIXME: mark these appropriately */
+    } else {
+        CLIENT_ASSERT(LINKSTUB_INDIRECT(l->flags), "invalid exit stub");
+        if (TEST(LINK_RETURN, l->flags))
+            return "ret";
+        if (TEST(LINK_CALL, l->flags))
+            return "indcall";
+        if (TEST(LINK_JMP, l->flags))
+            return "indjmp";
+#ifdef WINDOWS
+        if (is_shared_syscall_routine(dcontext, EXIT_TARGET_TAG(dcontext, f, l)))
+            return "shared_syscall";
+#endif
+    }
+    CLIENT_ASSERT(false, "unknown exit stub type");
+    return "<unknown>";
+}
+                       
+/* Disassemble and pretty-print the generated code for fragment f.
+ * Header and body control whether header info and code itself are printed
+ */
+static void
+common_disassemble_fragment(dcontext_t *dcontext,
+                            fragment_t *f_in, file_t outfile, bool header, bool body)
+{
+    cache_pc entry_pc, prefix_pc;
+    cache_pc pc;
+    cache_pc body_end_pc;
+    cache_pc end_pc;
+    linkstub_t *l;
+    int exit_num = 0;
+#ifdef PROFILE_RDTSC
+    cache_pc profile_end = 0;
+#endif
+    bool alloc;
+    fragment_t *f = f_in;
+
+    if (header) {
+#ifdef DEBUG
+        print_file(outfile, "Fragment %d, tag "PFX", flags 0x%x, %s%s%ssize %d%s%s:\n",
+                   f->id,
+#else
+        print_file(outfile, "Fragment tag "PFX", flags 0x%x, %s%s%ssize %d%s%s:\n",
+#endif
+                   f->tag, f->flags,
+                   TEST(FRAG_COARSE_GRAIN, f->flags) ? "coarse, " : "",
+                   (TEST(FRAG_SHARED, f->flags) ? "shared, " : 
+                    (SHARED_FRAGMENTS_ENABLED() ?
+                     (TEST(FRAG_TEMP_PRIVATE, f->flags) ? "private temp, " : "private, ") : "")),
+                   (TEST( FRAG_IS_TRACE, f->flags)) ? "trace, " : 
+                   (TEST(FRAG_IS_TRACE_HEAD, f->flags)) ? "tracehead, " : "", 
+                   f->size,
+                   (TEST(FRAG_CANNOT_BE_TRACE, f->flags)) ?", cannot be trace":"",
+                   (TEST(FRAG_MUST_END_TRACE, f->flags)) ?", must end trace":"",
+                   (TEST(FRAG_CANNOT_DELETE, f->flags)) ?", cannot delete":"");
+
+        DOLOG(2, LOG_SYMBOLS, { /* FIXME: this affects non-logging uses... dump_traces, etc. */
+            char symbolbuf[MAXIMUM_SYMBOL_LENGTH];
+            print_symbolic_address(f->tag, symbolbuf, sizeof(symbolbuf), false);
+            print_file(outfile, "\t%s\n", symbolbuf);
+        });
+    }
+
+    if (!body)
+        return;
+
+    if (body && TEST(FRAG_FAKE, f->flags)) {
+        alloc = true;
+        f = fragment_recreate_with_linkstubs(dcontext, f_in);
+    } else {
+        alloc = false;
+    }
+    end_pc = f->start_pc + f->size;
+    body_end_pc = fragment_body_end_pc(dcontext, f);
+    entry_pc = FCACHE_ENTRY_PC(f);
+    prefix_pc = FCACHE_PREFIX_ENTRY_PC(f);
+    pc = FCACHE_IBT_ENTRY_PC(f);
+    if (pc != entry_pc) {
+        if (pc != prefix_pc) {
+            /* indirect branch target prefix exists */
+            print_file(outfile, "  -------- indirect branch target entry: --------\n");
+        }
+        while (pc < entry_pc) {
+            if (pc == prefix_pc) {
+                print_file(outfile, "  -------- prefix entry: --------\n");
+            }
+            pc = (cache_pc) disassemble_with_bytes(dcontext, (byte *)pc, outfile);
+        }
+        print_file(outfile, "  -------- normal entry: --------\n");
+    }
+
+    CLIENT_ASSERT(pc == entry_pc, "disassemble_fragment: invalid prefix");
+
+#ifdef PROFILE_RDTSC
+    if (dynamo_options.profile_times && (f->flags & FRAG_IS_TRACE) != 0) {
+        int sz = profile_call_size();
+        profile_end = pc + sz;
+        if (stats->loglevel < 3) {
+            /* don't print profile stuff to save space */
+            print_file(outfile, "  "PFX"..."PFX" = profile code\n",
+                       pc, (pc+sz-1));
+            pc += sz;
+        } else {
+            /* print profile stuff, but delineate it: */
+            print_file(outfile, "  -------- profile call: --------\n");
+        }
+    }
+#endif
+
+    while (pc < body_end_pc) {
+        pc = (cache_pc) disassemble_with_bytes(dcontext, (byte *)pc, outfile);
+#ifdef PROFILE_RDTSC
+        if (dynamo_options.profile_times &&
+            (f->flags & FRAG_IS_TRACE) != 0 &&
+            pc == profile_end) {
+            print_file(outfile, "  -------- end profile call -----\n");
+        }
+#endif
+    }
+
+    for (l = FRAGMENT_EXIT_STUBS(f); l; l = LINKSTUB_NEXT_EXIT(l)) {
+        cache_pc next_stop_pc;
+        linkstub_t *nxt;
+        /* store fragment pc since we don't want to walk forward in fragment */
+        cache_pc frag_pc = pc;
+        print_file(outfile, "  -------- exit stub %d: -------- <target: "PFX"> type: %s\n",
+                   exit_num, EXIT_TARGET_TAG(dcontext, f, l),
+                   exit_stub_type_desc(dcontext, f, l));
+        if (!EXIT_HAS_LOCAL_STUB(l->flags, f->flags)) {
+            if (EXIT_STUB_PC(dcontext, f, l) != NULL) {
+                pc = EXIT_STUB_PC(dcontext, f, l);
+                next_stop_pc = pc + linkstub_size(dcontext, f, l);
+            } else if (TEST(FRAG_COARSE_GRAIN, f->flags)) {
+                cache_pc cti_pc = EXIT_CTI_PC(f, l);
+                if (cti_pc == end_pc) {
+                    /* must be elided final jmp */
+                    print_file(outfile, "  <no final jmp since elided>\n");
+                    print_file(outfile, "  <no stub since linked and frozen>\n");
+                    CLIENT_ASSERT(pc == end_pc, "disassemble_fragment: invalid end");
+                    next_stop_pc = end_pc;
+                } else {
+                    pc = entrance_stub_from_cti(cti_pc);
+                    if (coarse_is_entrance_stub(pc)) {
+                        next_stop_pc = pc + linkstub_size(dcontext, f, l);
+                    } else {
+                        CLIENT_ASSERT(in_fcache(pc),
+                                      "disassemble_fragment: invalid exit stub");
+                        print_file(outfile, "  <no stub since linked and frozen>\n");
+                        next_stop_pc = pc;
+                    }
+                }
+            } else {
+                if (TEST(LINK_SEPARATE_STUB, l->flags))
+                    print_file(outfile, "  <no stub created since linked>\n");
+                else if (!EXIT_HAS_STUB(l->flags, f->flags))
+                    print_file(outfile, "  <no stub needed: -no_indirect_stubs>\n");
+                else
+                    CLIENT_ASSERT(false, "disassemble_fragment: invalid exit stub");
+                next_stop_pc = pc;
+            }
+        } else {
+            for (nxt = LINKSTUB_NEXT_EXIT(l); nxt != NULL; nxt = LINKSTUB_NEXT_EXIT(nxt)) {
+                if (EXIT_HAS_LOCAL_STUB(nxt->flags, f->flags))
+                    break;
+            }
+            if (nxt != NULL)
+                next_stop_pc = EXIT_STUB_PC(dcontext, f, nxt);
+            else
+                next_stop_pc =  pc + linkstub_size(dcontext, f, l);
+            CLIENT_ASSERT(next_stop_pc != NULL, "disassemble_fragment: invalid stubs");
+        }
+        while (pc < next_stop_pc) {
+            pc = (cache_pc) disassemble_with_bytes(dcontext, (byte *)pc, outfile);
+        }
+        /* point pc back at tail of fragment code if it was off in separate stub land */
+        if (TEST(LINK_SEPARATE_STUB, l->flags))
+            pc = frag_pc;
+        exit_num++;
+    }
+
+    if (TEST(FRAG_SELFMOD_SANDBOXED, f->flags)) {
+        DOSTATS({ /* skip stored sz */ end_pc -= sizeof(uint); });
+        print_file(outfile, "  -------- original code (from "PFX"-"PFX") -------- \n",
+                   f->tag, (f->tag + (end_pc - pc)));
+        while (pc < end_pc) {
+            pc = (cache_pc) disassemble_with_bytes(dcontext, (byte *)pc, outfile);
+        }
+    }
+
+    if (alloc)
+        fragment_free(dcontext, f);
+}
+
+#ifdef DEBUG /* because uses dcontext->logfile */
+void
+disassemble_fragment(dcontext_t *dcontext, fragment_t *f, bool just_header)
+{
+    if ((stats->logmask & LOG_EMIT) != 0) {
+        common_disassemble_fragment(dcontext, f, THREAD,
+                                    true, !just_header);
+        if (!just_header)
+            LOG(THREAD, LOG_EMIT, 1, "\n");
+    }
+}
+#endif /* DEBUG */
+
+void
+disassemble_fragment_header(dcontext_t *dcontext, fragment_t *f, file_t outfile)
+{
+    common_disassemble_fragment(dcontext, f, outfile, true, false);
+}
+
+void
+disassemble_fragment_body(dcontext_t *dcontext, fragment_t *f, file_t outfile)
+{
+    common_disassemble_fragment(dcontext, f, outfile, false, true);
+}
+
+void
+disassemble_app_bb(dcontext_t *dcontext, app_pc tag, file_t outfile)
+{
+    instrlist_t *ilist = build_app_bb_ilist(dcontext, tag, outfile);
+    instrlist_clear_and_destroy(dcontext, ilist);
+}
+
+/* Two entry points to the disassembly routines: */
+
+void
+instrlist_disassemble(dcontext_t *dcontext,
+                      app_pc tag, instrlist_t *ilist, file_t outfile)
+{
+    int len, sz;
+    instr_t *instr;
+    byte *addr;
+    byte *next_addr;
+    byte bytes[64];     /* scratch array for encoding instrs */
+    int level;
+    int offs = 0;
+    /* we want to print out the decode level each instr is at, so we have to
+     * do a little work
+     */
+
+    print_file(outfile, "TAG  "PFX"\n", tag);
+
+    for (instr = instrlist_first(ilist); instr; instr = instr_get_next(instr)) {
+        DOLOG(5, LOG_ALL, {
+            if (instr_raw_bits_valid(instr)) {
+                print_file(outfile, " <raw "PFX"-"PFX">::\n",
+                           instr_get_raw_bits(instr),
+                           instr_get_raw_bits(instr) + instr_length(dcontext, instr));
+            }
+            if (instr_get_translation(instr) != NULL) {
+                print_file(outfile, " <translation "PFX">::\n",
+                           instr_get_translation(instr));
+            }
+        });
+        if (instr_needs_encoding(instr)) {
+            byte *nxt_pc;
+            level = 4;
+            /* encode instr and then output as BINARY */
+            nxt_pc = instr_encode_ignore_reachability(dcontext, instr, bytes);
+            ASSERT(nxt_pc != NULL);
+            len = (int) (nxt_pc - bytes);
+            addr = bytes;
+            CLIENT_ASSERT(len < 64, "instrlist_disassemble: too-long instr");
+        } else {
+            addr = instr_get_raw_bits(instr);
+            len = instr_length(dcontext, instr);
+            if (instr_operands_valid(instr))
+                level = 3;
+            else if (instr_opcode_valid(instr))
+                level = 2;
+            else if (decode_sizeof(dcontext, addr, NULL _IF_X64(NULL)) == len)
+                level = 1;
+            else
+                level = 0;
+        }
+
+        /* Print out individual instructions.  Remember that multiple
+         * instructions may be packed into a single instr.
+         */
+        while (len) {
+            print_file(outfile, " +%-4d L%d ", offs, level);
+            next_addr = internal_disassemble(dcontext, addr, addr, outfile, false, true,
+                                             "      ");
+            if (next_addr == NULL)
+                break;
+            sz = (int) (next_addr - addr);
+            CLIENT_ASSERT(sz <= len, "instrlist_disassemble: invalid length");
+            len -= sz;
+            addr += sz;
+            offs += sz;
+        }
+#ifdef DEBUG
+        if (stats->loglevel >= 5)
+            print_file(outfile, "---- multi-instr boundary ----\n");
+#endif
+
+#ifdef CUSTOM_EXIT_STUBS
+        /* custom exit stub? */
+        if (instr_is_exit_cti(instr) && instr_ok_to_mangle(instr)) {
+            instrlist_t * custom = instr_exit_stub_code(instr);
+            if (custom != NULL) {
+                print_file(outfile, "\t=> custom exit stub code:\n");
+                instrlist_disassemble(dcontext, instr_get_branch_target_pc(instr),
+                                      custom, outfile);
+            }
+        }
+#endif
+
+    }
+
+    print_file(outfile, "END "PFX"\n\n", tag);
+}
+
+#endif /* INTERNAL || CLIENT_INTERFACE */
+
+
+static void
+internal_dump_callstack(app_pc cur_pc, app_pc ebp, file_t outfile, bool dump_xml)
+{
+    ptr_uint_t *pc = (ptr_uint_t *) ebp;
+    int num = 0;
+    LOG_DECLARE(char symbolbuf[MAXIMUM_SYMBOL_LENGTH];)
+    char *symbol_name = "";
+
+    if (cur_pc != NULL) {
+        DOLOG(1, LOG_SYMBOLS, {
+            print_symbolic_address(cur_pc, symbolbuf, sizeof(symbolbuf), false);
+            symbol_name = symbolbuf;
+        });
+        print_file(outfile, dump_xml ? 
+                   "\t<current_pc=\""PFX"\" name=\"%s\" />\n" : 
+                   "\tcurrent pc = "PFX"   %s\n",
+                   cur_pc, symbol_name);
+    }
+
+    while (pc != NULL &&
+           is_readable_without_exception_query_os((byte *)pc, 8)) {
+        DOLOG(1, LOG_SYMBOLS, {
+            print_symbolic_address((app_pc)*(pc+1), symbolbuf, sizeof(symbolbuf), false);
+            symbol_name = symbolbuf;
+        });
+        print_file(outfile, dump_xml ?
+                   "\t\t<frame ptr=\""PFX"\" parent=\""PFX"\" "
+                   "ret=\""PFX"\" name=\"%s\" />\n" :
+                   "\tframe ptr "PFX" => parent "PFX", ret = "PFX"   %s\n",
+                   pc, *pc, *(pc+1), symbol_name);
+        num++;
+        /* yes I've seen weird recursive cases before */
+        if (pc == (ptr_uint_t *) *pc || num > 100)
+            break;
+        pc = (ptr_uint_t *) *pc;
+    }
+}
+
+void
+dump_callstack(app_pc pc, app_pc ebp, file_t outfile, bool dump_xml)
+{
+    print_file(outfile, dump_xml ? "\t<call-stack>\n" : "Call stack:\n");
+    internal_dump_callstack(pc, ebp, outfile, dump_xml);
+    if (dump_xml)
+        print_file(outfile, "\t</call-stack>\n");
+}
+
+#ifdef DEBUG
+void
+dump_mcontext_callstack(dcontext_t *dcontext)
+{
+    LOG(THREAD, LOG_ALL, 1, "Call stack:\n");
+    internal_dump_callstack((app_pc)get_mcontext(dcontext)->pc,
+                            (app_pc)get_mcontext(dcontext)->xbp, THREAD,
+                            DUMP_NOT_XML);
+}
+#endif
+
+void
+dump_dr_callstack(file_t outfile)
+{
+    /* Since we're in DR we can't just clobber the saved app fields --
+     * so we save them first
+     */
+    app_pc our_ebp = 0;
+    GET_FRAME_PTR(our_ebp);
+    LOG(outfile, LOG_ALL, 1, "DynamoRIO call stack:\n");
+    internal_dump_callstack(NULL /* don't care about cur pc */, our_ebp,
+                            outfile, DUMP_NOT_XML);
+}
diff --git a/core/x86/disassemble.h b/core/x86/disassemble.h
new file mode 100644
index 000000000..cabbcfd5f
--- /dev/null
+++ b/core/x86/disassemble.h
@@ -0,0 +1,126 @@
+/* **********************************************************
+ * Copyright (c) 2001-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2001 Hewlett-Packard Company */
+
+/* file "disassemble.h" */
+
+#ifndef DISASSEMBLE_H
+#define DISASSEMBLE_H
+
+#if defined(INTERNAL) || defined(DEBUG) || defined(CLIENT_INTERFACE)
+
+/* DR_API EXPORT TOFILE dr_ir_utils.h */
+DR_API
+/** 
+ * Decodes and then prints the instruction at address \p pc to file \p outfile.
+ * The default is to use AT&T-style syntax, unless the \ref op_syntax_intel
+ * "-syntax_intel" runtime option is specified.
+ * Returns the address of the subsequent instruction, or NULL if the instruction
+ * at \p pc is invalid.
+ */
+byte *
+disassemble(dcontext_t *dcontext, byte *pc, file_t outfile);
+
+DR_UNS_API /* deprecated from interface */
+/**
+ * Decodes and then prints the instruction at address \p pc to file \p outfile.
+ * Prior to the instruction the address and raw bytes of the instruction
+ * are printed.
+ * The default is to use AT&T-style syntax, unless the \ref op_syntax_intel
+ * "-syntax_intel" runtime option is specified.
+ * Returns the address of the subsequent instruction, or a guess if the instruction
+ * at \p pc is invalid.
+ */
+byte *
+disassemble_with_bytes(dcontext_t *dcontext, byte *pc, file_t outfile);
+
+DR_API
+/**
+ * Decodes and then prints the instruction at address \p pc to file \p outfile.
+ * Prior to the instruction the address is printed if \p show_pc and the raw
+ * bytes are printed if \p show_bytes.
+ * The default is to use AT&T-style syntax, unless the \ref op_syntax_intel
+ * "-syntax_intel" runtime option is specified.
+ * Returns the address of the subsequent instruction, or NULL if the instruction
+ * at \p pc is invalid.
+ */
+byte *
+disassemble_with_info(dcontext_t *dcontext, byte *pc, file_t outfile,
+                      bool show_pc, bool show_bytes);
+
+DR_API
+/**
+ * Decodes the instruction at address \p copy_pc as though
+ * it were located at address \p orig_pc, and then prints the
+ * instruction to file \p outfile.
+ * Prior to the instruction the address \p orig_pc is printed if \p show_pc and the raw
+ * bytes are printed if \p show_bytes.
+ * The default is to use AT&T-style syntax, unless the \ref op_syntax_intel
+ * "-syntax_intel" runtime option is specified.
+ * Returns the address of the subsequent instruction after the copy at
+ * \p copy_pc, or NULL if the instruction at \p copy_pc is invalid.
+ */
+byte *
+disassemble_from_copy(dcontext_t *dcontext, byte *copy_pc, byte *orig_pc,
+                      file_t outfile, bool show_pc, bool show_bytes);
+
+
+/* DR_API EXPORT TOFILE dr_ir_instr.h */
+DR_API
+/**
+ * Prints the instruction \p instr to file \p outfile. 
+ * Does not print address-size or data-size prefixes for other than
+ * just-decoded instrs, and does not check that the instruction has a
+ * valid encoding.  Prints each operand with leading zeros indicating
+ * the size.
+ * The default is to use AT&T-style syntax, unless the \ref op_syntax_intel
+ * "-syntax_intel" runtime option is specified.
+ */
+void 
+instr_disassemble(dcontext_t *dcontext, instr_t *instr, file_t outfile);
+
+/* DR_API EXPORT TOFILE dr_ir_opnd.h */
+DR_API
+/**
+ * Prints the operand \p opnd to file \p outfile.
+ * The default is to use AT&T-style syntax, unless the \ref op_syntax_intel
+ * "-syntax_intel" runtime option is specified.
+ */
+void 
+opnd_disassemble(dcontext_t *dcontext, opnd_t opnd, file_t outfile);
+
+#endif /* INTERNAL || DEBUG || CLIENT_INTERFACE */
+
+#endif /* DISASSEMBLE_H */
diff --git a/core/x86/emit_utils.c b/core/x86/emit_utils.c
new file mode 100644
index 000000000..7d5b79e00
--- /dev/null
+++ b/core/x86/emit_utils.c
@@ -0,0 +1,7476 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/* file "emit_utils.c" 
+ * The Pentium processors maintain cache consistency in hardware, so we don't
+ * worry about getting stale cache entries.
+ */
+
+#include "globals.h"
+#include "link.h"
+#include "fragment.h"
+#include "fcache.h"
+#include "emit.h"
+
+#include "arch.h"
+#include "instr.h"
+#include "instr_create.h"
+#include "instrument.h" /* for dr_insert_call() */
+#include "proc.h"
+#include <string.h> /* for memcpy */
+#include "instrlist.h"
+#include "decode.h"
+#include "decode_fast.h"
+#ifdef DEBUG
+#include "disassemble.h"
+#endif
+#include <limits.h> /* for UCHAR_MAX */
+#include "perscache.h"
+
+/* fragment_t fields */
+#define FRAGMENT_TAG_OFFS        (offsetof(fragment_t, tag))
+/* CAUTION: if TAG_OFFS changes from 0, must change indirect exit stub! */
+#define FRAGMENT_START_PC_OFFS   (offsetof(fragment_t, start_pc))
+#define FRAGMENT_COUNTER_OFFS    (offsetof(fragment_t, hot_counter))
+#define FRAGMENT_PREFIX_SIZE_OFFS (offsetof(fragment_t, prefix_size))
+
+#define HASHLOOKUP_TAG_OFFS       (offsetof(fragment_entry_t, tag_fragment))
+#define HASHLOOKUP_START_PC_OFFS       (offsetof(fragment_entry_t, start_pc_fragment))
+
+#ifdef HASHTABLE_STATISTICS
+#  define HASHLOOKUP_STAT_OFFS(event) (offsetof(hashtable_statistics_t, event##_stat))
+#endif
+
+#ifdef TRACE_HEAD_CACHE_INCR 
+/* linkstub_t field */
+#  define LINKSTUB_TARGET_FRAG_OFFS  (offsetof(direct_linkstub_t, target_fragment))
+#endif
+
+#ifdef PROFILE_LINKCOUNT
+#  define LINKSTUB_COUNT_OFFS        (offsetof(linkstub_t, count))
+#endif
+
+/* macro to make it easier to get offsets of fields that are in
+ * a different memory space with self-protection
+ */
+#define UNPROT_OFFS(dcontext, offs)                         \
+    (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask) ? \
+     (((ptr_uint_t)((dcontext)->upcontext.separate_upcontext)) + (offs)) :  \
+     (((ptr_uint_t)(dcontext)) + (offs)))
+
+/* N.B.: I decided to not keep supporting DCONTEXT_IN_EDI
+ * If we really want it later we can add it, it's a pain to keep
+ * maintaining it with every change here
+ */
+#ifdef DCONTEXT_IN_EDI
+#error DCONTEXT_IN_EDI Not Implemented
+#endif
+
+/* make code more readable by shortening long lines 
+ * we mark all as meta to avoid client interface asserts
+ */
+#define POST instrlist_meta_postinsert
+#define PRE  instrlist_meta_preinsert
+#define APP  instrlist_append
+
+/* in x86.asm */
+#ifdef RETURN_STACK
+void return_lookup(void);
+void unlinked_return(void);
+void end_return_lookup(void);
+#endif
+
+/* we are sharing bbs w/o ibs -- we assume that a bb
+ * w/ a direct branch cannot have an ib and thus is shared
+ */
+#ifdef TRACE_HEAD_CACHE_INCR
+/* incr routine can't tell whether coming from shared bb
+ * or non-shared fragment (such as a trace) so must always
+ * use shared stubs
+ */
+#  define FRAG_DB_SHARED(flags) true
+#else
+#  define FRAG_DB_SHARED(flags) (TEST(FRAG_SHARED, (flags)))
+#endif
+
+/**
+ ** CAUTION!
+ **
+ ** The following definitions and routines are highly dependent upon
+ ** definitions made in x86.asm.  Do NOT change any constants or code
+ ** without first consulting that file.
+ **
+ **/
+
+/***************************************************************************
+ ***************************************************************************
+ ** EXIT STUB
+ **
+ ** WARNING: all exit stubs must support atomic linking and unlinking,
+ ** meaning a link/unlink operation must involve a single store!
+ ** There is an exception: a first-time link (detected using a sentinel
+ ** LINKCOUNT_NEVER_LINKED_SENTINEL placed where the unlinked entry
+ ** code will go once linked) does not need to be atomic.
+ **/
+
+/*
+direct branch exit_stub:
+   5x8  mov   %xax, xax_offs(&dcontext) or tls
+  #if defined(PROFILE_LINKCOUNT)  (PR 248210: x64 not supported)
+  | 1   lahf
+  | 3   seto  %al
+  |#if !defined(LINKCOUNT_64_BITS)
+  | 6   inc   l->count 
+  |#else
+  | 7   add   $1,l->count
+  | 7   adc   $0,l->count+4
+  |#endif
+  | 2   add   $0x7f,%al
+  | 1   sahf
+  #endif
+   5x10 mov   &linkstub, %xax
+    5   jmp   target addr
+#if defined(PROFILE_LINKCOUNT)  (PR 248210: x64 not supported)
+|unlinked entry point:
+|    5   movl  %eax, eax_offs(&dcontext)
+|    5   movl  &linkstub, %eax
+|    5   jmp   fcache_return
+|
+|  Notes: we link/unlink by modifying the 1st jmp to either target unlinked
+|  entry point or the target fragment.  When we link for the first time
+|  we try to remove the eflags save/restore, shifting the 1st jmp up (the
+|  space between it and unlinked entry just becomes junk).
+#endif
+
+indirect branch exit_stub (only used if -indirect_stubs):
+   6x9  mov   %xbx, xbx_offs(&dcontext) or tls
+   5x11 mov   &linkstub, %xbx
+    5   jmp   indirect_branch_lookup
+
+indirect branches use xbx so that the flags can be saved into xax using
+the lahf instruction!
+xref PR 249775 on lahf support on x64.
+
+for PROFILE_LINKCOUNT, the count increment is performed inside the
+hashtable lookup (in both linked and unlinked paths) both since the flags
+are saved there for the linked path and to save space in stubs
+
+also see emit_inline_ibl_stub() below
+
+*/
+
+/* DIRECT_EXIT_STUB_SIZE is in arch_exports.h */
+#define STUB_DIRECT_SIZE(flags) DIRECT_EXIT_STUB_SIZE(flags)
+
+/* for -thread_private, we're relying on the fact that
+ * SIZE32_MOV_XBX_TO_TLS == SIZE32_MOV_XBX_TO_ABS, and that
+ * x64 always uses tls
+ */
+#define STUB_INDIRECT_SIZE32 \
+    (SIZE32_MOV_XBX_TO_TLS + SIZE32_MOV_PTR_IMM_TO_XAX + JMP_LONG_LENGTH)
+#define STUB_INDIRECT_SIZE64 \
+    (SIZE64_MOV_XBX_TO_TLS + SIZE64_MOV_PTR_IMM_TO_XAX + JMP_LONG_LENGTH)
+#define STUB_INDIRECT_SIZE(flags) \
+    (FRAG_IS_32(flags) ? STUB_INDIRECT_SIZE32 : STUB_INDIRECT_SIZE64)
+
+/* STUB_COARSE_DIRECT_SIZE is in arch_exports.h */
+#define STUB_COARSE_INDIRECT_SIZE(flags) (STUB_INDIRECT_SIZE(flags))
+
+#ifndef LINKCOUNT_64_BITS
+#  define LINKCOUNT_INCSIZE (6)
+#else
+#  define LINKCOUNT_INCSIZE (7+7)
+#endif
+#define LINKCOUNT_EFLAGS_SAVE (3+1)
+#define LINKCOUNT_EFLAGS_RESTORE (2+1)
+#define LINKCOUNT_FLAGSIZE (LINKCOUNT_EFLAGS_SAVE + LINKCOUNT_EFLAGS_RESTORE)
+
+#define LINKCOUNT_DIRECT_EXTRA(flags) \
+    (LINKCOUNT_INCSIZE + LINKCOUNT_FLAGSIZE + STUB_DIRECT_SIZE(flags))
+#define LINKCOUNT_UNLINKED_ENTRY(flags) \
+    (LINKCOUNT_INCSIZE + LINKCOUNT_FLAGSIZE + STUB_DIRECT_SIZE(flags))
+
+/* used to distinguish a never-linked direct exit -- once linked this
+ * will be replaced by the beginning of the unlink entry point, which is
+ * a save of xax, which will never look like this.  we choose nops to
+ * avoid complicating our disassembly routines.
+ */
+#define LINKCOUNT_NEVER_LINKED_SENTINEL 0x90909090
+
+/* Return size in bytes required for an exit stub with specified
+ * target and FRAG_ flags
+ */
+int
+exit_stub_size(dcontext_t *dcontext, cache_pc target, uint flags)
+{
+    if (TEST(FRAG_COARSE_GRAIN, flags)) {
+        /* For coarse: bb building points at bb ibl, and then insert_exit_stub
+         * changes that to the appropriate coarse prefix.  So the emit() calls to
+         * this routine pass in a real ibl.  But any later calls, e.g. for
+         * disassembly, that ask linkstub_size() will call EXIT_TARGET_TAG() which
+         * calls indirect_linkstub_target() which returns get_coarse_ibl_prefix():
+         * which then is not recognized as indirect by this routine!
+         * Note that coarse_indirect_stub_jmp_target() derefs the prefix:
+         * should we require callers who have stub pc to call that instead of us
+         * de-referencing?
+         */
+        target = coarse_deref_ibl_prefix(dcontext, target);
+    }
+    if (is_indirect_branch_lookup_routine(dcontext, target)) {
+        /* indirect branch */
+
+        /* FIXME: Since we don't have the stub flags we'll lookup the
+         * target routine's template in a very roundabout fashion here
+         * by dispatching on the ibl_routine entry point
+         */
+        ibl_code_t *ibl_code;
+
+        ibl_type_t ibl_type;
+        IF_X64(gencode_mode_t mode;)
+        DEBUG_DECLARE(bool is_ibl = )
+            get_ibl_routine_type_ex(dcontext, target, &ibl_type _IF_X64(&mode));
+        ASSERT(is_ibl);
+        IF_X64(ASSERT(mode == MODE_OVERRIDE(FRAG_IS_32(flags))));
+        ibl_code = get_ibl_routine_code_ex(dcontext, ibl_type.branch_type, flags
+                                           _IF_X64(mode));
+        
+        if (!EXIT_HAS_STUB(ibltype_to_linktype(ibl_code->branch_type),
+                           IBL_FRAG_FLAGS(ibl_code)))
+            return 0;
+
+        if (TEST(FRAG_COARSE_GRAIN, flags)) {
+#ifdef WINDOWS
+            ASSERT(!is_shared_syscall_routine(dcontext, target));
+#endif
+            /* keep in synch w/ coarse_indirect_stub_size() */
+            return (STUB_COARSE_INDIRECT_SIZE(flags));
+        }
+
+#ifdef WINDOWS
+        if (is_shared_syscall_routine(dcontext, target)) {
+            return INTERNAL_OPTION(shared_syscalls_fastpath) ? 5 :
+                STUB_INDIRECT_SIZE(flags);
+        }
+#endif
+
+        if (ibl_code->ibl_head_is_inlined) 
+            return ibl_code->inline_stub_length;
+        else
+            return (STUB_INDIRECT_SIZE(flags));
+    } else {
+        /* direct branch */
+        if (TEST(FRAG_COARSE_GRAIN, flags))
+            return (STUB_COARSE_DIRECT_SIZE(flags));
+#ifdef PROFILE_LINKCOUNT
+        if (dynamo_options.profile_counts && (flags & FRAG_IS_TRACE) != 0)
+            return (STUB_DIRECT_SIZE(flags) + LINKCOUNT_DIRECT_EXTRA(flags));
+        else {
+#endif
+            return (STUB_DIRECT_SIZE(flags));
+#ifdef PROFILE_LINKCOUNT
+        }
+#endif
+    }
+}
+
+/* The write that inserts the relative target is done atomically so this
+ * function is safe with respect to a thread executing the code containing 
+ * this target, presuming that the code in both the before and after states
+ * is valid.
+ * For x64 this routine only works for 32-bit reachability.  If further
+ * reach is needed the caller must use indirection.  Xref PR 215395.
+ */
+byte *
+insert_relative_target(byte *pc, cache_pc target, bool hot_patch)
+{
+    /* insert 4-byte pc-relative offset from the beginning of the next instruction
+     */
+    int value = (int)(ptr_int_t)(target - pc - 4);
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(target - pc - 4)));
+    ATOMIC_4BYTE_WRITE(pc, value, hot_patch);
+    pc += 4;
+    return pc;
+}
+
+byte *
+insert_relative_jump(byte *pc, cache_pc target, bool hot_patch)
+{
+    ASSERT(pc != NULL);
+    *pc = JMP_OPCODE;
+    pc++;
+    return insert_relative_target(pc, target, hot_patch);
+}
+
+/* make sure to keep in sync w/ instr_raw_is_tls_spill() */
+static cache_pc
+insert_spill_or_restore(dcontext_t *dcontext, cache_pc pc, uint flags, bool spill,
+                        bool shared, reg_id_t reg, ushort tls_offs,
+                        uint dc_offs, bool require_addr16)
+{
+    DEBUG_DECLARE(cache_pc start_pc = pc;)
+    byte opcode = ((reg == REG_XAX) ?
+                   (spill ? MOV_XAX2MEM_OPCODE : MOV_MEM2XAX_OPCODE) :
+                   (spill ? MOV_REG2MEM_OPCODE : MOV_MEM2REG_OPCODE));
+#ifdef X64
+    /* for x64, shared and private fragments all use tls, even for 32-bit code */
+    shared = true;
+    if (!FRAG_IS_32(flags)) {
+        /* mov %rbx, gs:os_tls_offset(tls_offs) */
+        if (reg == REG_XAX) {
+            /* shorter to use 0xa3 w/ addr32 prefix than 0x89/0x8b w/ sib byte */
+            /* FIXME: PR 209709: test perf and remove if outweighs space */
+            *pc = ADDR_PREFIX_OPCODE; pc++;
+        }
+        *pc = TLS_SEG_OPCODE; pc++;
+        *pc = REX_PREFIX_BASE_OPCODE | REX_PREFIX_W_OPFLAG; pc++;
+        *pc = opcode; pc++;
+        if (reg != REG_XAX) {
+            /* 0x1c for rbx, 0x0c for rcx, 0x04 for rax */
+            *pc = MODRM_BYTE(0/*mod*/, reg_get_bits(reg), 4/*rm*/); pc++;
+            *pc = SIB_DISP32; pc++;   /* sib byte: abs addr */
+        }
+        *((uint *)pc) = (uint) os_tls_offset(tls_offs); pc+=4;
+    } else
+#endif /* X64 */
+    if (shared) {
+        /* mov %ebx, fs:os_tls_offset(tls_offs) */
+        /* trying hard to keep the size of the stub 5 for eax, 6 else */
+        /* FIXME: case 5231 when staying on trace space is better,
+         * when going through this to the IBL routine speed asks for
+         * not adding the prefix.
+         */
+        bool addr16 = (require_addr16 || use_addr_prefix_on_short_disp());
+        if (addr16) {
+            *pc = ADDR_PREFIX_OPCODE; 
+            pc++;
+        }
+        *pc = TLS_SEG_OPCODE; pc++;
+        *pc = opcode; pc++;
+        if (reg != REG_XAX) {
+            /* 0x1e for ebx, 0x0e for ecx, 0x06 for eax
+             * w/o addr16 those are 0x1d, 0x0d, 0x05
+             */
+            *pc = MODRM_BYTE(0/*mod*/, reg_get_bits(reg), addr16 ? 6 : 5/*rm*/);
+            pc++;
+        }
+        if (addr16) {
+            *((ushort *)pc) = os_tls_offset(tls_offs);
+            pc+=2;
+        } else {
+            *((uint *)pc) = os_tls_offset(tls_offs);
+            pc+=4;
+        }
+    } else {
+        /* mov %ebx,((int)&dcontext)+dc_offs */
+        *pc = opcode; pc++;
+        if (reg != REG_XAX) {
+            /* 0x1d for ebx, 0x0d for ecx, 0x05 for eax */
+            *pc = MODRM_BYTE(0/*mod*/, reg_get_bits(reg), 5/*rm*/); pc++;
+        }
+        IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+        *((uint *)pc) = (uint)(ptr_uint_t) UNPROT_OFFS(dcontext, dc_offs); pc+=4;
+    }
+    ASSERT(IF_X64_ELSE(false, !shared) || (pc - start_pc) == 
+           (reg == REG_XAX ? SIZE_MOV_XAX_TO_TLS(flags, require_addr16) :
+            SIZE_MOV_XBX_TO_TLS(flags, require_addr16)));
+    ASSERT(IF_X64_ELSE(false, !shared) || !spill || reg == REG_XAX ||
+           instr_raw_is_tls_spill(start_pc, reg, tls_offs));
+    return pc;
+}
+
+/* instr_raw_is_tls_spill() matches the exact sequence of bytes inserted here */
+static byte *
+insert_jmp_to_ibl(byte *pc, fragment_t *f, linkstub_t *l, cache_pc exit_target,
+                  dcontext_t *dcontext)
+{
+#ifdef WINDOWS
+    bool spill_xbx_to_fs =
+        FRAG_DB_SHARED(f->flags) ||
+        (is_shared_syscall_routine(dcontext, exit_target) &&
+         DYNAMO_OPTION(shared_fragment_shared_syscalls));
+#else
+    bool spill_xbx_to_fs = FRAG_DB_SHARED(f->flags);
+#endif
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    /* we use XBX to hold the linkstub pointer for IBL routines, 
+       note that direct stubs use XAX for linkstub pointer */
+#ifdef WINDOWS
+    if (INTERNAL_OPTION(shared_syscalls_fastpath)
+        && is_shared_syscall_routine(dcontext, exit_target)) {
+        /* jmp <exit_target> */
+        *pc = 0xe9; pc++;
+        pc = insert_relative_target(pc, exit_target, NOT_HOT_PATCHABLE);
+        return pc;
+    } else
+#endif
+        pc = insert_spill_or_restore(dcontext, pc, f->flags, true/*spill*/,
+                                     spill_xbx_to_fs, REG_XBX,
+                                     INDIRECT_STUB_SPILL_SLOT, XBX_OFFSET,
+                                     true);
+
+    /* mov $linkstub_ptr,%xbx */
+#ifdef X64
+    if (!FRAG_IS_32(f->flags))
+        *pc = REX_PREFIX_BASE_OPCODE | REX_PREFIX_W_OPFLAG; pc++;
+#endif
+    *pc = MOV_IMM2XBX_OPCODE; pc++;
+#ifdef WINDOWS
+    if (DYNAMO_OPTION(shared_syscalls)
+        && is_shared_syscall_routine(dcontext, exit_target)) {
+        /* FIXME We could reduce mem usage by not allocating a linkstub for
+         * this exit since it's never referenced.
+         */
+        LOG(THREAD, LOG_LINKS, 4,
+            "\tF%d using %s shared syscalls link stub\n",
+            f->id, TEST(FRAG_IS_TRACE, f->flags) ? "trace" :
+            "bb");
+        l = (linkstub_t *) (TEST(FRAG_IS_TRACE, f->flags) ?
+                          get_shared_syscalls_trace_linkstub() :
+                          get_shared_syscalls_bb_linkstub());
+    }
+#endif
+    if (TEST(FRAG_COARSE_GRAIN, f->flags)) {
+        /* FIXME: once we separate these we should switch to 15-byte w/
+         * store-to-mem instead of in a spilled xbx, to use same
+         * slots as coarse direct stubs
+         */
+        /* There is no linkstub_t so we store source tag instead */
+        *((ptr_uint_t *)pc) = (ptr_uint_t)f->tag; pc += sizeof(f->tag);
+        /* FIXME: once we separate the indirect stubs out we will need
+         * a 15-byte stub .  For that we should simply store the
+         * source cti directly into a tls slot.  For now though we inline
+         * the stubs and spill xbx.
+         */
+    } else {
+        *((ptr_uint_t *)pc) = (ptr_uint_t)l; pc += sizeof(l);
+    }
+    /* jmp <exit_target> */
+    pc = insert_relative_jump(pc, exit_target, NOT_HOT_PATCHABLE);
+    return pc;
+}
+
+#ifdef PROFILE_LINKCOUNT /**************************************/
+static byte *
+insert_linkcount_inc(byte *pc, linkstub_t *l)
+{
+    /* increment counter l->count: */
+#ifndef LINKCOUNT_64_BITS
+    /* incl &(l->count) */
+    *pc = 0xff; pc++;
+    *pc = 0x05; pc++;
+    /* PR 248210: PROFILE_LINKCOUNT is unsupported on x64 */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    *((uint *)pc) = (uint) &(l->count); pc+=4;
+#else /* 64-bit */
+    /* addl $0x1,&(l->count) */
+    *pc = 0x83; pc++;
+    *pc = 0x05; pc++;
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    *((uint *)pc) = (uint) &(l->count); pc+=4;
+    *pc = 0x01; pc++;
+    /* adcl $0x0,&(l->count)+4 */
+    *pc = 0x83; pc++;
+    *pc = 0x15; pc++;
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    *((uint *)pc) = 4 + (uint)(&(l->count)); pc+=4;
+    *pc = 0x00; pc++;
+#endif /* 64-bit */
+    return pc;
+}
+
+/* FIXME: keep in synch w/ insert_save_eflags */
+static byte *
+insert_linkcount_saveflags(byte *pc, dcontext_t *dcontext, uint flags)
+{
+    DEBUG_DECLARE(cache_pc original_pc = pc;)
+    /* lahf */
+    *pc = LAHF_OPCODE; pc++;
+    /* 0f 90 c0   seto %al */
+    *pc = SETO_OPCODE_1; pc++;
+    *pc = SETO_OPCODE_2; pc++;
+    *pc = 0xc0; pc++;
+    ASSERT(pc - original_pc == LINKCOUNT_EFLAGS_SAVE);
+    return pc;
+}
+
+/* FIXME: keep in synch w/ insert_restore_eflags */
+static byte *
+insert_linkcount_restoreflags(byte *pc, dcontext_t *dcontext, uint flags)
+{
+    DEBUG_DECLARE(cache_pc original_pc = pc;)
+    /* 04 7f   add $0x7f,%al */
+    *pc = ADD_AL_OPCODE; pc++;
+    *pc = 0x7f; pc++;
+    /* sahf */
+    *pc = SAHF_OPCODE; pc++;
+    ASSERT(pc - original_pc == LINKCOUNT_EFLAGS_RESTORE);
+    return pc;
+}
+#endif /* PROFILE_LINKCOUNT **************************************/
+
+static bool
+is_patchable_exit_stub_helper(dcontext_t *dcontext, cache_pc ltarget,
+                              ushort lflags, uint fflags)
+{
+    if (LINKSTUB_INDIRECT(lflags)) {
+        /*indirect */
+        if (!DYNAMO_OPTION(indirect_stubs))
+            return false;
+#ifdef NATIVE_RETURN
+        ASSERT_NOT_IMPLEMENTED(false); /* don't bother */
+#endif
+        if (
+#ifdef WINDOWS
+            !is_shared_syscall_routine(dcontext, ltarget) && 
+#endif
+            get_ibl_routine_code(dcontext, extract_branchtype(lflags), fflags)
+                ->ibl_head_is_inlined) {
+            return !DYNAMO_OPTION(atomic_inlined_linking);
+        } else {
+            return true;
+        }      
+    } else {
+        /* direct */
+        ASSERT(LINKSTUB_DIRECT(lflags));
+#if defined(PROFILE_LINKCOUNT) || defined(TRACE_HEAD_CACHE_INCR)
+        return true;
+#else
+        return false;
+#endif
+    }
+}
+
+bool
+is_patchable_exit_stub(dcontext_t *dcontext, linkstub_t *l, fragment_t *f)
+{
+    return is_patchable_exit_stub_helper(dcontext, EXIT_TARGET_TAG(dcontext, f, l),
+                                         l->flags, f->flags);
+}
+
+bool
+is_exit_cti_stub_patchable(dcontext_t *dcontext, instr_t *inst, uint frag_flags)
+{
+    app_pc target;
+    /* we figure out what the linkstub flags should be 
+     * N.B.: we have to be careful to match the LINKSTUB_ macros
+     */
+    ushort lflags = (ushort) instr_exit_branch_type(inst);
+    ASSERT_TRUNCATE(lflags, ushort, instr_exit_branch_type(inst));
+    ASSERT(instr_is_exit_cti(inst));
+    target = instr_get_branch_target_pc(inst);
+    if (is_indirect_branch_lookup_routine(dcontext, target)) {
+        lflags |= LINK_INDIRECT;
+    } else {
+        lflags |= LINK_DIRECT;
+    }
+    return is_patchable_exit_stub_helper(dcontext, target, lflags, frag_flags);
+}
+
+uint
+bytes_for_exitstub_alignment(dcontext_t *dcontext, linkstub_t *l, fragment_t *f, byte *startpc)
+{
+    if (is_patchable_exit_stub(dcontext, l, f)) {
+        /* assumption - we only hot patch the ending jmp of the exit stub 
+         * (and that exit stub size returns the right values) */
+        ptr_uint_t shift = ALIGN_SHIFT_SIZE
+            (startpc +
+             exit_stub_size(dcontext, EXIT_TARGET_TAG(dcontext, f, l), f->flags) - 
+             EXIT_STUB_PATCH_OFFSET, 
+             EXIT_STUB_PATCH_SIZE, PAD_JMPS_ALIGNMENT);
+#ifdef PROFILE_LINKCOUNT
+        /* assumption doesn't hold because of the optimize ... */
+        /* FIXME : once this is implemented re-enable the ifdefed out stats 
+         * in emit_fragment_common */
+        ASSERT_NOT_IMPLEMENTED(false);
+#endif
+        IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(shift)));
+        return (uint) shift;
+    }
+    return 0;
+}
+
+/* Returns an upper bound on the number of bytes that will be needed to add
+ * this fragment to a trace */
+uint
+extend_trace_pad_bytes(fragment_t *add_frag)
+{
+    /* FIXME : this is a poor estimate, we could do better by looking at the
+     * linkstubs and checking if we are inlining ibl, but since this is just
+     * used by monitor.c for a max size check should be fine to overestimate
+     * we'll just end up with slightly shorter max size traces */
+    /* we don't trace through traces in normal builds, so don't worry about
+     * number of exits (FIXME this also assumes bbs don't trace through 
+     * conditional or indirect branches) */
+    ASSERT_NOT_IMPLEMENTED(!TEST(FRAG_IS_TRACE, add_frag->flags));
+    /* Also, if -pad_jmps_shift_bb we assume that we don't need to remove
+     * any nops from fragments added to traces since there shouldn't be any if
+     * we only add bbs (nop_pad_ilist has an assert that verifies we don't add
+     * any nops to bbs when -pad_jmps_shift_bb without marking as CANNOT_BE_TRACE,
+     * so here we also verify that we only add bbs) - Xref PR 215179, LINUX syscall
+     * fence exits and CLIENT_INTERFACE added/moved exits can lead to bbs with
+     * additional hot_patchable locations.  We mark such bb fragments as CANNOT_BE_TRACE
+     * in nop_pad_ilist() if -pad_jmps_mark_no_trace is set or assert otherwise to avoid
+     * various difficulties so should not see them here. */
+    /* A standard bb has at most 2 patchable locations (ends in conditional or ends
+     * in indirect that is promoted to inlined). */
+    return 2*MAX_PAD_SIZE;
+}
+
+/* return startpc shifted by the necessary bytes to pad patchable jmps of the
+ * exit stub to proper alignment */
+byte *
+pad_for_exitstub_alignment(dcontext_t *dcontext, linkstub_t *l, fragment_t *f, byte *startpc)
+{
+    uint shift;
+    ASSERT(PAD_FRAGMENT_JMPS(f->flags)); /* shouldn't call this otherwise */
+    
+    shift = bytes_for_exitstub_alignment(dcontext, l, f, startpc);
+    if (shift > 0) {
+        /* Pad with 1 byte instructions so looks nice in debuggers.
+         * decode_fragment also checks for this as a sanity check.  Note, 
+         * while these instructions can never be reached, they will be decoded
+         * by shift fcache pointers so must put something valid here. */
+        SET_TO_DEBUG(startpc, shift);
+        startpc += shift;
+        STATS_PAD_JMPS_ADD(f->flags, num_shifted_stubs, 1);
+        STATS_PAD_JMPS_ADD(f->flags, shifted_stub_bytes, shift);
+    } else {
+        STATS_PAD_JMPS_ADD(f->flags, num_stubs_no_shift, 1);
+    }
+    return startpc;
+}
+
+/* Only used if -no_pad_jmps_shift_{bb,trace}. FIXME this routine is expensive (the
+ * instr_expand) and we may end up removing app nops (an optimizations but
+ * not really what we're after here). */
+void
+remove_nops_from_ilist(dcontext_t *dcontext, instrlist_t *ilist _IF_DEBUG(bool recreating))
+{
+    instr_t *inst, *next_inst;
+
+    for (inst = instrlist_first(ilist); inst != NULL; inst = next_inst) {
+        /* FIXME : expensive, just expand instr before cti, function not used
+         * if -no_pad_jmps_shift_{bb,trace} */
+        inst = instr_expand(dcontext, ilist, inst);
+        next_inst = instr_get_next(inst);
+        if (instr_is_nop(inst)) {
+            instrlist_remove(ilist, inst);
+            DOSTATS({
+                if (!recreating) {
+                    STATS_INC(num_nops_removed);
+                    STATS_ADD(num_nop_bytes_removed, instr_length(dcontext, inst));
+                }
+            });
+            instr_destroy(dcontext, inst);
+        }
+    }
+}
+
+
+/* inserts any nop padding needed to ensure patchable branch offsets don't 
+ * cross cache line boundaries.  If emitting sets the offset field of all 
+ * instructions, else sets the translation for the added nops (for 
+ * recreating). If emitting and -pad_jmps_shift_{bb,trace} returns the number
+ * of bytes to shift the start_pc by (this avoids putting a nop before the 
+ * first exit cti) else returns 0. */
+uint
+nop_pad_ilist(dcontext_t *dcontext, fragment_t *f, instrlist_t *ilist, bool emitting)
+{
+    instr_t *inst;
+    uint offset = 0;
+    int first_patch_offset = -1;
+    uint start_shift = 0;
+    /* if emitting prefix_size isn't set up yet */
+    cache_pc starting_pc = f->start_pc + fragment_prefix_size(f->flags);
+    DODEBUG({ASSERT(emitting || f->prefix_size == fragment_prefix_size(f->flags));});
+
+    ASSERT(PAD_FRAGMENT_JMPS(f->flags)); /* shouldn't call this otherwise */
+
+    for (inst = instrlist_first(ilist); inst != NULL; inst = instr_get_next(inst)) {
+        /* don't support non exit cti patchable instructions yet */
+        ASSERT_NOT_IMPLEMENTED(!TEST(INSTR_HOT_PATCHABLE, inst->flags));
+        if (instr_is_exit_cti(inst)) {
+            /* see if we need to be able to patch this instruction */
+            if (is_exit_cti_patchable(dcontext, inst, f->flags)) {
+                /* see if we are crossing a cache line, offset is the start of 
+                 * the next instr here so need to see if the dword ending at 
+                 * offset crosses a cache line */
+                uint nop_length = 
+                    patchable_exit_cti_align_offs(dcontext, inst,
+                                                  starting_pc+offset);
+                if (first_patch_offset < 0)
+                    first_patch_offset = offset; 
+                if (nop_length > 0) { 
+                    /* crosses cache line, nop pad */
+                    /* instead of inserting a nop, shift the starting pc if
+                     * we are within 1 cache line of the first patchable offset
+                     * (this covers the case of a conditional branch which 
+                     * mangles to two patchable exits and is still safe since
+                     * they are less then a cache line apart) */
+                    if (PAD_JMPS_SHIFT_START(f->flags) &&
+                        offset + instr_length(dcontext, inst) - first_patch_offset < 
+                        PAD_JMPS_ALIGNMENT) {
+                        ASSERT(start_shift == 0); /* should only shift once */
+                        start_shift = nop_length;
+                        /* adjust the starting_pc, all previously checked
+                         * instructions should be fine since we are still 
+                         * within the same cache line as the first patchable 
+                         * offset */
+                        starting_pc += nop_length;
+                    } else {
+                        instr_t *nop_inst = INSTR_CREATE_nopNbyte(dcontext, nop_length);
+
+                        /* We expect bbs to never need this if
+                         * -pad_jmps_shift_bb except on LINUX (signal fence exit) and
+                         * CLIENT_INTERFACE (client can re-arrange fragment) where we
+                         * -pad_jmps_mark_no_trace.  We rely on having not
+                         * inserted any nops into traceable bbs in interp (so that we
+                         * don't have to remove them when we build the trace).
+                         * FIXME add tracing support so we don't have to mark
+                         * CANNOT_BE_TRACE (see PR 215179). */
+                        if (emitting && DYNAMO_OPTION(pad_jmps_mark_no_trace) && 
+                            !TEST(FRAG_IS_TRACE, f->flags)) {
+                            f->flags |= FRAG_CANNOT_BE_TRACE;
+                            STATS_INC(pad_jmps_mark_no_trace);
+                        } else {
+                            ASSERT(TEST(FRAG_IS_TRACE, f->flags) ||
+                                   TEST(FRAG_CANNOT_BE_TRACE, f->flags) ||
+                                   !INTERNAL_OPTION(pad_jmps_shift_bb));
+                        }
+                        instrlist_preinsert(ilist, inst, nop_inst);
+                        /* sanity check */
+                        ASSERT((int)nop_length == instr_length(dcontext, nop_inst));
+                        if (emitting) {
+                            /* fixup offsets */
+                            instr_set_note(nop_inst, (void *)(ptr_uint_t)offset);
+                            /* only inc stats for emitting, not for recreating */
+                            STATS_PAD_JMPS_ADD(f->flags, num_nops, 1);
+                            STATS_PAD_JMPS_ADD(f->flags, nop_bytes, nop_length);
+                        }
+                        /* set translation whether emitting or not */
+                        instr_set_translation(nop_inst, instr_get_translation(inst));
+                        instr_set_our_mangling(nop_inst, true);
+                        offset += nop_length;
+                    }
+                    /* sanity check that we fixed the alignment */
+                    ASSERT(patchable_exit_cti_align_offs(dcontext, inst,
+                                                         starting_pc+offset) == 0);
+                } else {
+                    DOSTATS({
+                        /* only inc stats for emitting, not for recreating */
+                        if (emitting)
+                            STATS_PAD_JMPS_ADD(f->flags, 
+                                               num_no_pad_exits, 1);
+                    });
+                }
+            }
+        }
+        if (emitting)
+            instr_set_note(inst, (void *)(ptr_uint_t)offset); /* used by instr_encode */
+        offset += instr_length(dcontext, inst);
+    }
+#ifdef CUSTOM_EXIT_STUBS
+    ASSERT_NOT_IMPLEMENTED(false);
+#endif
+    return start_shift;
+}
+
+/* for the hashtable lookup inlined into exit stubs, the
+ * lookup routine is encoded earlier into a template,
+ * (in the routine emit_inline_ibl_stub(), below)
+ * which we copy here and fix up the linkstub ptr for.
+ * when the hashtable changes, the mask and table are
+ * updated in update_indirect_exit_stub(), below.
+ */
+static byte *
+insert_inlined_ibl(dcontext_t *dcontext, fragment_t *f, linkstub_t *l, byte *pc,
+                   cache_pc unlinked_exit_target, uint flags)
+{
+    ibl_code_t *ibl_code = 
+        get_ibl_routine_code(dcontext, 
+                             extract_branchtype(l->flags), f->flags);
+    byte *start_pc = pc;
+    cache_pc linked_exit_target = 
+        get_linked_entry(dcontext, unlinked_exit_target);
+
+    /* PR 248207: haven't updated the inlining to be x64-compliant yet */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    ASSERT(ibl_code->ibl_head_is_inlined);
+    ASSERT(EXIT_HAS_STUB(l->flags, f->flags));
+    memcpy(start_pc, ibl_code->inline_ibl_stub_template, ibl_code->inline_stub_length);
+
+    /* exit should be unlinked initially */
+    patch_branch(EXIT_CTI_PC(f, l), start_pc + ibl_code->inline_unlink_offs,
+                 NOT_HOT_PATCHABLE);
+
+    if (DYNAMO_OPTION(indirect_stubs)) {
+        /* fixup linked/unlinked targets */
+        if (DYNAMO_OPTION(atomic_inlined_linking)) {
+            insert_relative_target(start_pc + ibl_code->inline_linkedjmp_offs, 
+                                   linked_exit_target, NOT_HOT_PATCHABLE);
+            insert_relative_target(start_pc + ibl_code->inline_unlinkedjmp_offs, 
+                                   unlinked_exit_target, NOT_HOT_PATCHABLE);
+        } else {
+            insert_relative_target(start_pc + ibl_code->inline_linkedjmp_offs,
+                                   unlinked_exit_target, NOT_HOT_PATCHABLE);
+        }
+        /* set the linkstub ptr */
+        pc = start_pc + ibl_code->inline_linkstub_first_offs;
+        IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+        *((uint *)pc) = (uint)(ptr_uint_t)l;
+        if (DYNAMO_OPTION(atomic_inlined_linking)) {
+            pc = start_pc + ibl_code->inline_linkstub_second_offs;
+            IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+            *((uint *)pc) = (uint)(ptr_uint_t)l;
+        }
+    } else {
+        insert_relative_target(start_pc + ibl_code->inline_linkedjmp_offs,
+                               linked_exit_target, NOT_HOT_PATCHABLE);
+        insert_relative_target(start_pc + ibl_code->inline_unlink_offs
+                               + 1 /* skip jmp opcode: see emit_inline_ibl_stub FIXME */, 
+                               unlinked_exit_target, NOT_HOT_PATCHABLE);
+    }
+
+    return start_pc + ibl_code->inline_stub_length;
+}
+
+static cache_pc
+get_direct_exit_target(dcontext_t *dcontext, uint flags)
+{
+    if (FRAG_DB_SHARED(flags)) {
+        if (TEST(FRAG_COARSE_GRAIN, flags)) {
+            /* note that entrance stubs should target their unit's prefix,
+             * who will then target this routine
+             */
+            return fcache_return_coarse_routine(IF_X64(MODE_OVERRIDE(FRAG_IS_32(flags))));
+        } else
+            return fcache_return_shared_routine(IF_X64(MODE_OVERRIDE(FRAG_IS_32(flags))));
+    } else {
+        return fcache_return_routine_ex(dcontext
+                                        _IF_X64(MODE_OVERRIDE(FRAG_IS_32(flags))));
+    }
+}
+
+static cache_pc
+insert_save_xax(dcontext_t *dcontext, cache_pc pc, uint flags,
+                bool shared, ushort tls_offs, bool require_addr16)
+{
+    return insert_spill_or_restore(dcontext, pc, flags, true/*spill*/,
+                                   shared, REG_XAX, tls_offs, XAX_OFFSET,
+                                   require_addr16);
+}
+
+/* restore xax in a stub or a fragment prefix */
+static cache_pc
+insert_restore_xax(dcontext_t *dcontext, cache_pc pc, uint flags,
+                   bool shared, ushort tls_offs, bool require_addr16)
+{
+    return insert_spill_or_restore(dcontext, pc, flags, false/*restore*/,
+                                   shared, REG_XAX, tls_offs, XAX_OFFSET,
+                                   require_addr16);
+}
+
+/* Emit code for the exit stub at stub_pc.  Return the size of the
+ * emitted code in bytes.  This routine assumes that the caller will
+ * take care of any cache synchronization necessary (though none is
+ * necessary on the Pentium).
+ * The stub is unlinked initially, except coarse grain indirect exits,
+ * which are always linked.
+ */
+int
+insert_exit_stub_other_flags(dcontext_t *dcontext, fragment_t *f,
+                             linkstub_t *l, cache_pc stub_pc, ushort l_flags)
+{
+    byte *pc = (byte *)stub_pc;
+    cache_pc exit_target;
+    bool indirect = false;
+#ifdef PROFILE_LINKCOUNT
+    bool linkcount = (dynamo_options.profile_counts && (f->flags & FRAG_IS_TRACE) != 0);
+#endif
+    bool can_inline = true;
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+
+    /* select the correct exit target */
+    if (LINKSTUB_DIRECT(l_flags)) {
+        if (TEST(FRAG_COARSE_GRAIN, f->flags)) {
+            /* need to target the fcache return prefix */
+            exit_target = fcache_return_coarse_prefix(stub_pc, NULL);
+            ASSERT(exit_target != NULL);
+        } else
+            exit_target = get_direct_exit_target(dcontext, f->flags);
+    } else {
+        ASSERT(LINKSTUB_INDIRECT(l_flags));
+        /* caller shouldn't call us if no stub */
+        ASSERT(EXIT_HAS_STUB(l_flags, f->flags));
+        if (TEST(FRAG_COARSE_GRAIN, f->flags)) {
+            /* need to target the ibl prefix */
+            exit_target = get_coarse_ibl_prefix(dcontext, stub_pc,
+                                                extract_branchtype(l_flags));
+            ASSERT(exit_target != NULL);
+        } else {
+            /* initially, stub should be unlinked */
+            exit_target = get_unlinked_entry(dcontext,
+                                             EXIT_TARGET_TAG(dcontext, f, l));
+        }
+        indirect = true;
+# ifdef WINDOWS
+        can_inline = (exit_target != unlinked_shared_syscall_routine(dcontext));
+# endif
+        if (can_inline) {
+            ibl_code_t *ibl_code = 
+                get_ibl_routine_code(dcontext, 
+                                     extract_branchtype(l_flags), f->flags);
+            if (!ibl_code->ibl_head_is_inlined)
+                can_inline = false;
+        }
+    }
+
+    if (indirect && can_inline) {
+        pc = insert_inlined_ibl(dcontext, f, l, pc, exit_target, f->flags);
+        IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(pc - stub_pc)));
+        return (int) (pc - stub_pc);
+    }
+
+    if (indirect) {
+        pc = insert_jmp_to_ibl(pc, f, l, exit_target, dcontext);
+    } else if (TEST(FRAG_COARSE_GRAIN, f->flags)) {
+        /* This is an entrance stub.  It may be executed even when linked,
+         * so we store target info to memory instead of a register.
+         * The exact bytes used here are assumed by entrance_stub_target_tag().
+         */
+#ifdef X64
+        if (!FRAG_IS_32(f->flags)) {
+            app_pc tgt = EXIT_TARGET_TAG(dcontext, f, l);
+            /* Both entrance_stub_target_tag() and coarse_indirect_stub_jmp_target()
+             * assume that the addr prefix is present for 32-bit but not 64-bit.
+             */
+            /* since we have no 8-byte-immed-to-memory, we split into two pieces */
+            *pc = TLS_SEG_OPCODE; pc++;
+            *pc = MOV_IMM2MEM_OPCODE; pc++;
+            *pc = MODRM_BYTE(0/*mod*/, 0/*reg*/, 4/*rm*/); pc++; /* => no base, w/ sib */
+            *pc = SIB_DISP32; pc++; /* just disp32 */
+            /* low 32 bits */
+            *((uint *)pc) = (uint) os_tls_offset(DIRECT_STUB_SPILL_SLOT); pc+=4;
+            *((uint *)pc) = (uint)(ptr_uint_t) tgt; pc += 4;
+            
+            *pc = TLS_SEG_OPCODE; pc++;
+            *pc = MOV_IMM2MEM_OPCODE; pc++;
+            *pc = MODRM_BYTE(0/*mod*/, 0/*reg*/, 4/*rm*/); pc++; /* => no base, w/ sib */
+            *pc = SIB_DISP32; pc++; /* just disp32 */
+            /* high 32 bits */
+            *((uint *)pc) = 4 + (uint) os_tls_offset(DIRECT_STUB_SPILL_SLOT); pc+=4;
+            *((uint *)pc) = (uint)(((ptr_uint_t)tgt) >> 32); pc += 4;
+        } else {
+#endif /* X64 */
+            /* We must be at or below 15 bytes so we require addr16. 
+             * Both entrance_stub_target_tag() and coarse_indirect_stub_jmp_target()
+             * assume that the addr prefix is present for 32-bit but not 64-bit.
+             */
+            /* addr16 mov <target>, fs:<dir-stub-spill> */
+            /* FIXME: PR 209709: test perf and remove if outweighs space */
+            *pc = ADDR_PREFIX_OPCODE; pc++;
+            *pc = TLS_SEG_OPCODE; pc++;
+            *pc = MOV_IMM2MEM_OPCODE; pc++;
+            *pc = MODRM16_DISP16; pc++;
+            *((ushort *)pc) = os_tls_offset(DIRECT_STUB_SPILL_SLOT); pc+=2;
+            *((uint *)pc) = (uint)(ptr_uint_t) EXIT_TARGET_TAG(dcontext, f, l); pc += 4;
+#ifdef X64
+        }
+#endif /* X64 */
+        /* jmp to exit target */
+        pc = insert_relative_jump(pc, exit_target, NOT_HOT_PATCHABLE);
+    } else {
+        /* direct branch */
+
+        /* we use XAX to hold the linkstub pointer before we get to fcache_return, 
+           note that indirect stubs use XBX for linkstub pointer */
+        pc = insert_save_xax(dcontext, pc, f->flags, FRAG_DB_SHARED(f->flags),
+                             DIRECT_STUB_SPILL_SLOT, true);
+
+#ifdef PROFILE_LINKCOUNT
+        if (linkcount) {
+            pc = insert_linkcount_saveflags(pc, dcontext, f->flags);
+            pc = insert_linkcount_inc(pc, l);
+            pc = insert_linkcount_restoreflags(pc, dcontext, f->flags);
+        }
+#endif
+        /* mov $linkstub_ptr,%xax */
+#ifdef X64
+        if (!FRAG_IS_32(f->flags))
+            *pc = REX_PREFIX_BASE_OPCODE | REX_PREFIX_W_OPFLAG; pc++;
+#endif
+        *pc = MOV_IMM2XAX_OPCODE; pc++;
+        *((ptr_uint_t *)pc) = (ptr_uint_t)l; pc += sizeof(l);
+        /* jmp to exit target */
+        pc = insert_relative_jump(pc, exit_target, NOT_HOT_PATCHABLE);
+
+#ifdef PROFILE_LINKCOUNT
+        if (linkcount) {
+            /* this is where the unlinked entry point goes, but to
+             * distinguish a never-linked stub we place a sentinel here
+             */
+            ASSERT(pc == stub_pc + LINKCOUNT_DIRECT_EXTRA(f->flags));
+            *((uint *)pc) = LINKCOUNT_NEVER_LINKED_SENTINEL;
+            pc += sizeof(uint);
+            /* nop out rest of mem to play nice with the disassembler
+             * (we're profile linkcount so presumably going to output) */
+            memset(pc, 0x90 /* NOP */, STUB_DIRECT_SIZE(f->flags) - sizeof(uint));
+            pc += STUB_DIRECT_SIZE(f->flags) - sizeof(uint);
+        }
+#endif
+    }
+
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int(pc - stub_pc)));
+    return (int) (pc - stub_pc);
+}
+
+int
+insert_exit_stub(dcontext_t *dcontext, fragment_t *f,
+                 linkstub_t *l, cache_pc stub_pc)
+{
+    return insert_exit_stub_other_flags(dcontext, f, l, stub_pc, l->flags);
+}
+
+static cache_pc
+exit_cti_disp_pc(cache_pc branch_pc)
+{
+    cache_pc byte_ptr = branch_pc;
+    byte opcode = *byte_ptr;
+    uint length = 0;
+
+    if (opcode == RAW_PREFIX_jcc_taken || opcode == RAW_PREFIX_jcc_not_taken) {
+        length++;
+        byte_ptr++;
+        opcode = *byte_ptr;
+        /* branch hints are only valid with jcc instrs */
+        ASSERT(opcode == RAW_OPCODE_jcc_byte1);
+    }
+
+    if (opcode >= RAW_OPCODE_loop_start && opcode <= RAW_OPCODE_loop_end) {
+        /* assume that this is a mangled jcxz/loop*
+         * target pc is in last 4 bytes of "9-byte instruction"
+         */
+        length += CTI_SHORT_REWRITE_LENGTH;
+    } else if (opcode == RAW_OPCODE_jcc_byte1) {
+        /* 2-byte opcode, 6-byte instruction, except for branch hint */
+        ASSERT(*(byte_ptr+1) >= RAW_OPCODE_jcc_byte2_start &&
+               *(byte_ptr+1) <= RAW_OPCODE_jcc_byte2_end);
+        length += CBR_LONG_LENGTH;
+    } else {
+        /* 1-byte opcode, 5-byte instruction */
+#if (defined(RETURN_STACK) || defined(HOT_PATCHING_INTERFACE))
+        ASSERT(opcode == RAW_OPCODE_jmp || opcode == RAW_OPCODE_call);
+#else
+        ASSERT(opcode == RAW_OPCODE_jmp);
+#endif
+        length += JMP_LONG_LENGTH;
+    }
+    return branch_pc + length - 4; /* disp is 4 even on x64 */
+}
+
+/* Patch the (direct) branch at branch_pc so it branches to target_pc 
+ * The write that actually patches the branch is done atomically so this
+ * function is safe with respect to a thread executing this branch presuming
+ * that both the before and after targets are valid
+ */
+void
+patch_branch(cache_pc branch_pc, cache_pc target_pc, bool hot_patch)
+{
+    cache_pc byte_ptr = exit_cti_disp_pc(branch_pc);
+    insert_relative_target(byte_ptr, target_pc, hot_patch);
+}
+
+#ifdef PROFILE_LINKCOUNT
+static byte *
+change_linkcount_target(byte *pc, app_pc target)
+{
+    /* Once we've linked once, we modify the jmp at the end of the
+     * link code in the stub to either jmp to the unlinked entry
+     * (which has no counter inc code of its own, that's why the exit
+     * jmp doesn't go straight there) or to the target.
+     * To find the jmp, watch first opcode to determine which state
+     * stub is in (depending on whether had to save eflags or not).
+     */
+    if (*pc == 0xff || *pc == 0x83) { /* inc/add is 1st instr */
+        pc += LINKCOUNT_INCSIZE + 1;
+    } else {
+        IF_X64(ASSERT_NOT_IMPLEMENTED(false)); /* need to pass in flags */
+        pc += LINKCOUNT_INCSIZE + LINKCOUNT_FLAGSIZE + STUB_DIRECT_SIZE(FRAG_32_BIT) - 4;
+    }
+    pc = insert_relative_target(pc, target, HOT_PATCHABLE);
+    return pc;
+}
+
+static void
+optimize_linkcount_stub(dcontext_t *dcontext, fragment_t *f,
+                        linkstub_t *l, fragment_t *targetf)
+{
+    /* first-time link: try to remove eflags save/restore */
+# ifdef CUSTOM_EXIT_STUBS
+    byte *stub_pc = (byte *) EXIT_FIXED_STUB_PC(dcontext, f, l);
+# else
+    byte *stub_pc = (byte *) EXIT_STUB_PC(dcontext, f, l);
+# endif
+    byte *pc = stub_pc;
+    bool remove_eflags_save = false;
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    ASSERT(LINKSTUB_DIRECT(l->flags));
+
+    if (!INTERNAL_OPTION(unsafe_ignore_eflags_prefix)) {
+        remove_eflags_save = TEST(FRAG_WRITES_EFLAGS_6, targetf->flags);
+    }
+    else {
+        /* scan through code at target fragment, stop scanning at 1st branch */
+        uint eflags = 0;
+        cache_pc end_pc = EXIT_CTI_PC(f, FRAGMENT_EXIT_STUBS(targetf));
+        byte *fpc = (byte *) FCACHE_ENTRY_PC(targetf);
+        /* for simplicity, stop at first instr that touches the flags */
+        while (eflags == 0 && fpc != NULL && ((cache_pc)fpc) < end_pc) {
+            fpc = decode_eflags_usage(dcontext, fpc, &eflags);
+        }
+        remove_eflags_save =
+            (eflags & (EFLAGS_WRITE_6|EFLAGS_READ_6)) == EFLAGS_WRITE_6;
+    }
+    if (remove_eflags_save) {
+        /* the 6 flags modified by add and adc are written before 
+         * they're read -> don't need to save eflags!
+         *
+         * I tried replacing lahf & sahf w/ nops, it's noticeably
+         * faster to not have the nops, so redo the increment:
+         */
+        pc = insert_linkcount_inc(pc, l);
+        *pc = 0xe9; pc++;
+        pc = insert_relative_target(pc, FCACHE_ENTRY_PC(targetf),
+                                    NOT_HOT_PATCHABLE);
+        /* Fill out with nops till the unlinked entry point so disassembles
+         * nicely for logfile (we're profile linkcount so presumably going
+         * to dump this). */
+        while (pc < (stub_pc + LINKCOUNT_DIRECT_EXTRA(f->flags))) {
+            *pc = 0x90; pc++;  /* nop */
+        }
+    } else {
+        /* keep eflags save & restore -- need to keep save of eax
+         * so skip all that now, go to right before store of &l into eax
+         */
+        pc += LINKCOUNT_DIRECT_EXTRA(f->flags) - 5 - 5;
+        /* need to insert a restore of eax -- luckily it perfectly
+         * overwrites the store of &l into eax, FIXME - dangerous
+         * though, if we ever drop the addr16 flag on a shared restore the
+         * instruction will be 6 bytes and our hardcoded 5 above will
+         * lead to a crash (should trigger assert below at least).
+         */
+        pc = insert_restore_xax(dcontext, pc, f->flags, FRAG_DB_SHARED(f->flags),
+                                DIRECT_STUB_SPILL_SLOT, true);
+        ASSERT(pc == stub_pc + LINKCOUNT_DIRECT_EXTRA(f->flags) - 5);
+        /* now add jmp */
+        *pc = 0xe9; pc++;
+        pc = insert_relative_target(pc, FCACHE_ENTRY_PC(targetf),
+                                    NOT_HOT_PATCHABLE);
+    }
+
+    /* we need to replace our never-linked sentinel w/ the real
+     * unlinked entry point.
+     */
+    ASSERT(pc == stub_pc + LINKCOUNT_DIRECT_EXTRA(f->flags));
+    pc = stub_pc + LINKCOUNT_DIRECT_EXTRA(f->flags);
+    ASSERT(*((uint *)pc) == LINKCOUNT_NEVER_LINKED_SENTINEL);
+    pc = insert_save_xax(dcontext, pc, f->flags, FRAG_DB_SHARED(f->flags),
+                         DIRECT_STUB_SPILL_SLOT, true); 
+    /* mov $linkstub_ptr,%xax */
+    *pc = MOV_IMM2XAX_OPCODE; pc++;
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    *((uint *)pc) = (uint)l; pc += 4;
+    /* jmp to target */
+    *pc = JMP_OPCODE; pc++;
+    pc = insert_relative_target(pc, get_direct_exit_target(dcontext, f->flags),
+                                NOT_HOT_PATCHABLE);
+}
+#endif /* PROFILE_LINKCOUNT */
+
+/* Checks patchable exit cti for proper alignment for patching. If it's
+ * properly aligned returns 0, else returns the number of bytes it would
+ * need to be forward shifted to be properly aligned */
+uint
+patchable_exit_cti_align_offs(dcontext_t *dcontext, instr_t *inst, cache_pc pc)
+{
+    /* all our exit cti's currently use 4 byte offsets */
+    /* FIXME : would be better to use a instr_is_cti_long or some such
+     * also should check for addr16 flag (we shouldn't have any prefixes) */
+    ASSERT((instr_is_cti(inst) && !instr_is_cti_short(inst) && 
+            !TESTANY(~(PREFIX_JCC_TAKEN|PREFIX_JCC_NOT_TAKEN), instr_get_prefixes(inst)))
+            || instr_is_cti_short_rewrite(inst, NULL));
+    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint
+                  (ALIGN_SHIFT_SIZE(pc + instr_length(dcontext, inst) - CTI_PATCH_SIZE,
+                                    CTI_PATCH_SIZE, PAD_JMPS_ALIGNMENT))));
+    return (uint) ALIGN_SHIFT_SIZE(pc + instr_length(dcontext, inst) - CTI_PATCH_SIZE,
+                                   CTI_PATCH_SIZE, PAD_JMPS_ALIGNMENT);
+}
+
+/* Returns true if the exit cti is ever dynamically modified */
+bool
+is_exit_cti_patchable(dcontext_t *dcontext, instr_t *inst, uint frag_flags)
+{
+    app_pc target;
+    if (TEST(FRAG_COARSE_GRAIN, frag_flags)) {
+        /* Case 8647: coarse grain fragment bodies always link through stubs
+         * until frozen, so their ctis are never patched except at freeze time
+         * when we suspend the world.
+         */
+        ASSERT(!TEST(FRAG_IS_TRACE, frag_flags));
+        return false;
+    }
+    ASSERT(instr_is_exit_cti(inst));
+    target = instr_get_branch_target_pc(inst);
+    if (is_indirect_branch_lookup_routine(dcontext, target)) {
+#ifdef NATIVE_RETURN
+        ASSERT_NOT_IMPLEMENTED(false); /* not going to bother */
+#endif
+        /* whether has an inline stub or not, cti is always
+         * patched if -no_indirect_stubs
+         */
+        if (!DYNAMO_OPTION(indirect_stubs))
+            return true;
+#ifdef WINDOWS
+        if (target != shared_syscall_routine(dcontext)) {
+#endif
+            return get_ibl_routine_code(dcontext, 
+                     extract_branchtype((ushort)instr_exit_branch_type(inst)),
+                                        frag_flags)->ibl_head_is_inlined;
+#ifdef WINDOWS
+        }
+        return false;
+#endif
+    } else {
+        /* direct exit */
+#ifdef PROFILE_LINKCOUNT
+        if (DYNAMO_OPTION(profile_counts) && TEST(FRAG_IS_TRACE, frag_flags)) {
+# ifdef CUSTOM_EXIT_STUBS
+            return true;
+# else
+            return false;
+# endif
+        }
+#endif
+        if (instr_branch_selfmod_exit(inst))
+            return false;
+        return true;
+    }
+}
+
+/* returns true if exit cti no longer points at stub
+ * (certain situations, like profiling or TRACE_HEAD_CACHE_INCR, go
+ * through the stub even when linked)
+ */
+bool
+link_direct_exit(dcontext_t *dcontext, fragment_t *f, linkstub_t *l, fragment_t *targetf,
+                 bool hot_patch)
+{
+#if defined(PROFILE_LINKCOUNT) || defined(TRACE_HEAD_CACHE_INCR)
+# ifdef CUSTOM_EXIT_STUBS
+    byte *stub_pc = (byte *) (EXIT_FIXED_STUB_PC(dcontext, f, l));
+# else
+    byte *stub_pc = (byte *) (EXIT_STUB_PC(dcontext, f, l));
+# endif
+#endif
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    ASSERT(LINKSTUB_DIRECT(l->flags));
+    STATS_INC(num_direct_links);
+
+#ifdef PROFILE_LINKCOUNT
+    if (dynamo_options.profile_counts && TEST(FRAG_IS_TRACE, f->flags)) {
+        /* do not change the exit jmp, instead change the stub itself */
+        if (*((uint *)(stub_pc + LINKCOUNT_DIRECT_EXTRA(f->flags))) ==
+            LINKCOUNT_NEVER_LINKED_SENTINEL) {
+            /* this is not atomic, but that's ok, it's first-time only */
+            /* FIXME - this assumption is so not safe with shared cache
+             * since we add to table and link incoming before linking outgoing
+             */
+            optimize_linkcount_stub(dcontext, f, l, targetf);
+# ifdef CUSTOM_EXIT_STUBS
+            /* FIXME: want flag that says whether should go through custom
+             * only when unlinked, or always!
+             * For now we assume only when unlinked:
+             */
+            /* skip custom code */
+            patch_branch(EXIT_CTI_PC(f, l), stub_pc,
+                         TEST(FRAG_SHARED, f->flags) ? hot_patch : NOT_HOT_PATCHABLE);
+# endif
+        } else {
+# ifdef CUSTOM_EXIT_STUBS
+            /* FIXME: want flag that says whether should go through custom
+             * only when unlinked, or always!
+             * For now we assume only when unlinked:
+             */
+            /* skip custom code */
+            patch_branch(EXIT_CTI_PC(f, l), stub_pc, hot_patch);
+# endif
+            change_linkcount_target(stub_pc, FCACHE_ENTRY_PC(targetf));
+        }
+# ifdef TRACE_HEAD_CACHE_INCR
+        /* yes, we wait for linkcount to do its thing and then we change it --
+         * but to make it more efficient will make this already ungainly
+         * code even harder to read
+         */
+        /* FIXME - atomicity issues? */
+        if ((targetf->flags & FRAG_IS_TRACE_HEAD) != 0) {
+            /* after optimized inc, jmp to unlinked code, but change its final
+             * jmp to go to incr routine
+             */
+            change_linkcount_target(stub_pc, stub_pc + LINKCOUNT_UNLINKED_ENTRY(f->flags));
+            LOG(THREAD, LOG_LINKS, 4,
+                "\tlinking F%d."PFX" to incr routine b/c F%d is trace head\n",
+                f->id, EXIT_CTI_PC(f, l), targetf->id);
+            patch_branch(stub_pc + LINKCOUNT_UNLINKED_ENTRY(f->flags) + 10,
+                         trace_head_incr_routine(dcontext), hot_patch);
+        }
+# endif
+        return false; /* going through stub */
+    }
+#endif /* PROFILE_LINKCOUNT */
+
+#ifdef TRACE_HEAD_CACHE_INCR
+    if ((targetf->flags & FRAG_IS_TRACE_HEAD) != 0) {
+        LOG(THREAD, LOG_LINKS, 4,
+            "\tlinking F%d."PFX" to incr routine b/c F%d is trace head\n",
+            f->id, EXIT_CTI_PC(f, l), targetf->id);
+        /* FIXME: more efficient way than multiple calls to get size-5? */
+        ASSERT(linkstub_size(dcontext, f, l) == DIRECT_EXIT_STUB_SIZE(f->flags));
+        patch_branch(stub_pc + DIRECT_EXIT_STUB_SIZE(f->flags) - 5,
+                     trace_head_incr_routine(dcontext), hot_patch);
+        return false; /* going through stub */
+    }
+#endif
+
+    /* change jmp target to point to the passed-in target */
+#ifdef UNSUPPORTED_API
+    if ((l->flags & LINK_TARGET_PREFIX) != 0) {
+        /* want to target just the xcx restore, not the eflags restore
+         * (only ibl targets eflags restore)
+         */
+        patch_branch(EXIT_CTI_PC(f, l), FCACHE_PREFIX_ENTRY_PC(targetf), 
+                     hot_patch);
+    } else
+#endif
+        patch_branch(EXIT_CTI_PC(f, l), FCACHE_ENTRY_PC(targetf), hot_patch);
+    return true; /* do not need stub anymore */
+}
+
+void
+unlink_direct_exit(dcontext_t *dcontext, fragment_t *f, linkstub_t *l)
+{
+    cache_pc stub_pc = (cache_pc) EXIT_STUB_PC(dcontext, f, l);
+#ifdef TRACE_HEAD_CACHE_INCR
+    direct_linkstub_t *dl = (direct_linkstub_t *) l;
+#endif
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    ASSERT(LINKSTUB_DIRECT(l->flags));
+
+#ifdef PROFILE_LINKCOUNT
+    if (dynamo_options.profile_counts && TEST(FRAG_IS_TRACE, f->flags)) {
+        byte *pc;
+        if (*((uint *)(stub_pc + LINKCOUNT_DIRECT_EXTRA(f->flags))) ==
+            LINKCOUNT_NEVER_LINKED_SENTINEL) {
+            /* never been linked, don't go pointing at the uninitialized
+             * unlink entry point -- just return, initial state is fine
+             */
+            return;
+        }
+# ifdef CUSTOM_EXIT_STUBS
+        pc = (byte *) (EXIT_FIXED_STUB_PC(dcontext, f, l));
+        stub_pc = (cache_pc) pc;
+        /* FIXME: want flag that says whether should go through custom
+         * only when unlinked, or always! Also is racy with 2nd branch patch.
+         * For now we assume only when unlinked.
+         */
+        /* go through custom code again */
+        patch_branch(EXIT_CTI_PC(f, l), stub_pc, HOT_PATCHABLE);
+# else
+        pc = (byte *) stub_pc;
+# endif
+# ifdef TRACE_HEAD_CACHE_INCR
+        if (dl->target_fragment != NULL) { /* HACK to tell if targeted trace head */
+            /* make unlinked jmp go back to fcache_return */
+            patch_branch(pc + LINKCOUNT_UNLINKED_ENTRY(f->flags) + 10,
+                         get_direct_exit_target(dcontext, f->flags),
+                         HOT_PATCHABLE);
+        } else
+# endif
+            /* make jmp after incr go to unlinked entry */
+            change_linkcount_target(pc, stub_pc + LINKCOUNT_UNLINKED_ENTRY(f->flags));
+        return;
+    }
+#endif
+
+#ifdef TRACE_HEAD_CACHE_INCR
+    if (dl->target_fragment != NULL) { /* HACK to tell if targeted trace head */
+# ifdef CUSTOM_EXIT_STUBS
+        byte *pc = (byte *) (EXIT_FIXED_STUB_PC(dcontext, f, l));
+# else
+        byte *pc = (byte *) (EXIT_STUB_PC(dcontext, f, l));
+# endif
+        /* FIXME: more efficient way than multiple calls to get size-5? */
+        ASSERT(linkstub_size(dcontext, f, l) == DIRECT_EXIT_STUB_SIZE(f->flags));
+        patch_branch(pc + DIRECT_EXIT_STUB_SIZE(f->flags) - 5,
+                     get_direct_exit_target(dcontext, f->flags),
+                     HOT_PATCHABLE);
+    }
+#endif
+
+    /* change jmp target to point to top of exit stub */
+    patch_branch(EXIT_CTI_PC(f, l), stub_pc, HOT_PATCHABLE);
+}
+
+/* NOTE : for inlined indirect branches linking is !NOT! atomic with respect
+ * to a thread executing in the cache unless using the atomic_inlined_linking
+ * option (unlike unlinking)
+ */
+void
+link_indirect_exit(dcontext_t *dcontext, fragment_t *f, linkstub_t *l, bool hot_patch)
+{
+    uint stub_size;
+    app_pc exit_target, cur_target;
+    app_pc target_tag = EXIT_TARGET_TAG(dcontext, f, l);
+    byte *pc;
+    /* w/ indirect exits now having their stub pcs computed based
+     * on the cti targets, we must calculate them at a consistent
+     * state (we do have multi-stage modifications for inlined stubs)
+     */
+    byte *stub_pc = (byte *) EXIT_STUB_PC(dcontext, f, l);
+#ifdef CUSTOM_EXIT_STUBS
+    byte *fixed_stub_pc = (byte *) EXIT_FIXED_STUB_PC(dcontext, f, l);
+#endif
+
+    ASSERT(!TEST(FRAG_COARSE_GRAIN, f->flags));
+
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    ASSERT(LINKSTUB_INDIRECT(l->flags));
+    /* target is always the same, so if it's already linked, this is a nop */
+    if ((l->flags & LINK_LINKED) != 0) {
+        STATS_INC(num_indirect_already_linked);
+        return;
+    }
+    STATS_INC(num_indirect_links);
+
+#ifdef NATIVE_RETURN
+    if (l->ret_pc != 0 && *(l->ret_pc) == 0x90) {
+        /* if ind branch is a ret, need to re-instate it over the nops */
+        /* assume either 0xc2 or 0xc3, i.e., no far rets! */
+        /* see if ret has immed
+         * ASSUMPTION: no nop right after ret -- shouldn't be, that's
+         * where we put our "save xdx, save eflags, pop xdx" code
+         */
+        if (*(l->ret_pc+1) == 0x90) {
+            /* yes, this ret has an immed.
+             * we can restore the ret's immed value (if any) by looking at the
+             * add instr, which is always right before the jmp
+             * => ASSUMPTION: client doesn't add stuff bet. add and jmp to ibl!
+             *   0x4224c3da   83 c4 04             add    $0x04 %esp -> %esp 
+             * if 2-byte immed, add will be something like "81 c4 i1 i2"
+             */
+            byte op;
+            *(l->ret_pc) = 0xc2; /* ret */
+            LOG(THREAD, LOG_LINKS, 4, "re-instating ret imm over nop\n");
+            op = *(EXIT_CTI_PC(f, l) - 3);
+            /* 3 back: either 0x83 or 0xc4 */
+            if (op == 0xc4) {
+                /* copy 2-byte immed */
+                *((unsigned short *)(l->ret_pc+1)) =
+                    *((unsigned short *)(EXIT_CTI_PC(f, l) - 2));
+            } else {
+                ASSERT(op == 0x83);
+                /* copy 1-byte immed, extend to 2-byte */
+                *(l->ret_pc+1) = *(EXIT_CTI_PC(f, l) - 1);
+                *(l->ret_pc+2) = (byte) 0;
+            }
+        } else {
+            /* 1 byte ret */
+            *(l->ret_pc) = 0xc3; /* ret */
+            LOG(THREAD, LOG_LINKS, 4, "re-instating ret over nop\n");
+        }
+    }
+#endif
+
+# ifdef WINDOWS
+    if (!is_shared_syscall_routine(dcontext, target_tag))
+# endif
+    {
+        ibl_code_t *ibl_code = 
+            get_ibl_routine_code(dcontext, 
+                                 extract_branchtype(l->flags), f->flags);
+
+        if (ibl_code->ibl_head_is_inlined) {
+            /* need to make branch target the top of the exit stub */
+            patch_branch(EXIT_CTI_PC(f, l), stub_pc, hot_patch);
+            if (DYNAMO_OPTION(atomic_inlined_linking)) {
+                return;
+            }
+        }
+    }
+
+    if (DYNAMO_OPTION(indirect_stubs)) {
+        /* go to start of 5-byte jump instruction at end of exit stub */
+        stub_size = exit_stub_size(dcontext, target_tag, f->flags);
+# ifdef CUSTOM_EXIT_STUBS
+        pc = fixed_stub_pc + stub_size - 5;
+# else
+        pc = stub_pc + stub_size - 5;
+# endif
+    } else {
+        /* cti goes straight to ibl, and must be a jmp, not jcc,
+         * except for -unsafe_ignore_eflags_trace stay-on-trace cmp,jne
+         */
+        pc = EXIT_CTI_PC(f, l);
+        /* for x64, or -unsafe_ignore_eflags_trace, a trace may have a jne to the stub */
+        if (*pc == JNE_OPCODE_1) {
+            ASSERT(TEST(FRAG_IS_TRACE, f->flags));
+#ifndef X64
+            ASSERT(INTERNAL_OPTION(unsafe_ignore_eflags_trace));
+#endif
+            /* FIXME: share this code w/ common path below: 1 opcode byte vs 2 */
+            /* get absolute address of target */
+            cur_target = (app_pc) PC_RELATIVE_TARGET(pc+2);
+            exit_target = get_linked_entry(dcontext, cur_target);
+            pc += 2; /* skip jne opcode */
+            pc = insert_relative_target(pc, exit_target, hot_patch);
+            return;
+        } else {
+            ASSERT(*pc == JMP_OPCODE);
+        }
+    }
+    /* get absolute address of target */
+    cur_target = (app_pc) PC_RELATIVE_TARGET(pc+1);
+    exit_target = get_linked_entry(dcontext, cur_target);
+    pc++; /* skip jmp opcode */
+    pc = insert_relative_target(pc, exit_target, hot_patch);
+}
+
+int
+linkstub_unlink_entry_offset(dcontext_t *dcontext, fragment_t *f, linkstub_t *l)
+{
+    ibl_code_t *ibl_code;
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    if (!LINKSTUB_INDIRECT(l->flags))
+        return 0;
+#ifdef WINDOWS
+    if (is_shared_syscall_routine(dcontext, EXIT_TARGET_TAG(dcontext, f, l)))
+        return 0;
+#endif
+    ibl_code = get_ibl_routine_code(dcontext, extract_branchtype(l->flags), f->flags);
+    if (ibl_code->ibl_head_is_inlined)
+        return ibl_code->inline_unlink_offs;
+    else
+        return 0;
+}
+
+cache_pc
+indirect_linkstub_stub_pc(dcontext_t *dcontext, fragment_t *f, linkstub_t *l)
+{
+    cache_pc cti = EXIT_CTI_PC(f, l);
+    /* decode the cti: it should be a relative jmp to the stub */
+    cache_pc stub;
+    if (!EXIT_HAS_STUB(l->flags, f->flags))
+        return NULL;
+    /* for x64, or -unsafe_ignore_eflags_trace, a trace may have a jne to the stub */
+    if (*cti == JNE_OPCODE_1) {
+        ASSERT(TEST(FRAG_IS_TRACE, f->flags));
+#ifndef X64
+        ASSERT(INTERNAL_OPTION(unsafe_ignore_eflags_trace));
+#endif
+        stub = (cache_pc) PC_RELATIVE_TARGET(cti+2/*opcode bytes*/);
+    } else {
+        if (*cti == JMP_OPCODE) {
+            stub = (cache_pc) PC_RELATIVE_TARGET(cti+1/*opcode byte*/);
+        } else {
+            /* case 6532/10987: frozen coarse has no jmp to stub */
+            ASSERT(TEST(FRAG_COARSE_GRAIN, f->flags));
+            ASSERT(coarse_is_indirect_stub(cti));
+            stub = cti;
+        }
+    }
+    ASSERT(stub >= cti && (stub - cti) <= MAX_FRAGMENT_SIZE);
+    if (!TEST(LINK_LINKED, l->flags)) {
+        /* the unlink target is not always the start of the stub */
+        stub -= linkstub_unlink_entry_offset(dcontext, f, l);
+        /* FIXME: for -no_indirect_stubs we could point exit cti directly
+         * at unlink ibl routine if we could find the stub target for
+         * linking here...should consider storing stub pc for ind exits
+         * for that case to save 5 bytes in the inlined stub
+         */
+    }
+    return stub;
+}
+
+cache_pc
+indirect_linkstub_target(dcontext_t *dcontext, fragment_t *f, linkstub_t *l)
+{
+    ASSERT(LINKSTUB_INDIRECT(l->flags));
+    ASSERT(!TESTANY(LINK_NI_SYSCALL_ALL, l->flags));
+#ifdef WINDOWS
+    if (EXIT_TARGETS_SHARED_SYSCALL(l->flags)) {
+        /* currently this is the only way to distinguish shared_syscall
+         * exit from other indirect exits and from other exits in
+         * a fragment containing ignorable or non-ignorable syscalls
+         */
+        ASSERT(TEST(FRAG_HAS_SYSCALL, f->flags));
+        return shared_syscall_routine_ex(dcontext
+                                         _IF_X64(MODE_OVERRIDE(FRAG_IS_32(f->flags))));
+    }
+#endif
+    if (TEST(FRAG_COARSE_GRAIN, f->flags)) {
+        /* Need to target the ibl prefix.  Passing in cti works as well as stub,
+         * and avoids a circular dependence where linkstub_unlink_entry_offset()
+         * call this routine to get the target and then this routine asks for
+         * the stub which calls linkstub_unlink_entry_offset()...
+         */
+        return get_coarse_ibl_prefix(dcontext, EXIT_CTI_PC(f, l),
+                                     extract_branchtype(l->flags));
+    } else {
+        return get_ibl_routine_ex(dcontext,
+                                  IF_X64(TEST(LINK_TRACE_CMP, l->flags) ? IBL_TRACE_CMP :)
+                                  IBL_LINKED,
+                                  get_source_fragment_type(dcontext, f->flags),
+                                  extract_branchtype(l->flags)
+                                  _IF_X64(MODE_OVERRIDE(FRAG_IS_32(f->flags))));
+    }
+}
+
+/* based on machine state, returns which of cbr l1 and fall-through l2
+ * must have been taken
+ */
+linkstub_t *
+linkstub_cbr_disambiguate(dcontext_t *dcontext, fragment_t *f,
+                          linkstub_t *l1, linkstub_t *l2)
+{
+    instr_t instr;
+    linkstub_t *taken;
+    instr_init(dcontext, &instr);
+    decode(dcontext, EXIT_CTI_PC(f, l1), &instr);
+    ASSERT(instr_is_cbr(&instr));
+    if (instr_cbr_taken(&instr, get_mcontext(dcontext), false/*post-state*/))
+        taken = l1;
+    else
+        taken = l2;
+    instr_free(dcontext, &instr);
+    return taken;
+}
+
+/* since we now support branch hints on long cbrs, we need to do a little
+ * decoding to find their length
+ */
+cache_pc
+cbr_fallthrough_exit_cti(cache_pc prev_cti_pc)
+{
+    if (*prev_cti_pc == RAW_PREFIX_jcc_taken ||
+        *prev_cti_pc == RAW_PREFIX_jcc_not_taken)
+        prev_cti_pc++;
+    return (prev_cti_pc + CBR_LONG_LENGTH);
+}
+
+/* This is an atomic operation with respect to a thread executing in the
+ * cache (barring ifdef NATIVE_RETURN), for inlined indirect exits the
+ * unlinked path of the ibl routine detects the race condition between the
+ * two patching writes and handles it appropriately unless using the
+ * atomic_inlined_linking option in which case there is only one patching
+ * write (since tail is duplicated) */
+void
+unlink_indirect_exit(dcontext_t *dcontext, fragment_t *f, linkstub_t *l)
+{
+    uint stub_size;
+    cache_pc exit_target, cur_target;
+    app_pc target_tag = EXIT_TARGET_TAG(dcontext, f, l);
+    byte *pc;
+    ibl_code_t *ibl_code = NULL;
+    /* w/ indirect exits now having their stub pcs computed based
+     * on the cti targets, we must calculate them at a consistent
+     * state (we do have multi-stage modifications for inlined stubs)
+     */
+#ifdef CUSTOM_EXIT_STUBS
+    byte *fixed_stub_pc = (byte *) EXIT_FIXED_STUB_PC(dcontext, f, l);
+#else
+    byte *stub_pc = (byte *) EXIT_STUB_PC(dcontext, f, l);
+#endif
+    ASSERT(!TEST(FRAG_COARSE_GRAIN, f->flags));
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    ASSERT(LINKSTUB_INDIRECT(l->flags));
+    /* target is always the same, so if it's already unlinked, this is a nop */
+    if (!TEST(LINK_LINKED, l->flags))
+        return;
+
+#ifdef NATIVE_RETURN
+    if (l->ret_pc != 0) {
+        /* if ind branch is a ret, need to overwrite the ret itself w/ a nop
+         * hard to find the ret (would have to decode backwards!)
+         * so we store ret_pc in the linkstub.
+         */
+        byte op;
+        /* ret may be 1 or 3 bytes */
+        op = *(l->ret_pc);
+        /* assume either 0xc2 or 0xc3, i.e., no far rets! */
+        if (op == 0xc2) {
+            /* 3 bytes */
+            *(l->ret_pc) = 0x90; /* nop */
+            *(l->ret_pc+1) = 0x90; /* nop */
+            *(l->ret_pc+2) = 0x90; /* nop */
+            LOG(THREAD, LOG_LINKS, 4, "overwriting ret immed w/ nops\n");
+        } else {
+            ASSERT(op == 0xc3);
+            /* 1 byte */
+            *(l->ret_pc) = 0x90; /* nop */
+            LOG(THREAD, LOG_LINKS, 4, "overwriting ret w/ nop\n");
+        }
+    }
+#endif
+
+# ifdef WINDOWS
+    if (!is_shared_syscall_routine(dcontext, target_tag))
+# endif
+        ibl_code = get_ibl_routine_code(dcontext, 
+                                        extract_branchtype(l->flags), f->flags);
+
+    if ((!DYNAMO_OPTION(atomic_inlined_linking) && DYNAMO_OPTION(indirect_stubs)) ||
+# ifdef WINDOWS
+        target_tag == shared_syscall_routine_ex
+        (dcontext _IF_X64(MODE_OVERRIDE(FRAG_IS_32(f->flags)))) ||
+# endif
+        /* FIXME: for -no_indirect_stubs and inlined ibl, we'd like to directly
+         * target the unlinked ibl entry but we don't yet -- see FIXME in
+         * emit_inline_ibl_stub()
+         */
+        !ibl_code->ibl_head_is_inlined) {
+
+        if (DYNAMO_OPTION(indirect_stubs)) {
+            /* go to start of 5-byte jump instruction at end of exit stub */
+            stub_size = exit_stub_size(dcontext, target_tag, f->flags);
+# ifdef CUSTOM_EXIT_STUBS
+            pc = fixed_stub_pc + stub_size - 5;
+# else
+            pc = stub_pc + stub_size - 5;
+# endif
+        } else {
+            /* cti goes straight to ibl, and must be a jmp, not jcc */
+            pc = EXIT_CTI_PC(f, l);
+            /* for x64, or -unsafe_ignore_eflags_trace, a trace may have a jne */
+            if (*pc == JNE_OPCODE_1) {
+                ASSERT(TEST(FRAG_IS_TRACE, f->flags));
+#ifndef X64
+                ASSERT(INTERNAL_OPTION(unsafe_ignore_eflags_trace));
+#endif
+                pc++; /* 2-byte opcode, skip 1st here */
+            } else
+                ASSERT (*pc == JMP_OPCODE);
+        }
+        cur_target = (cache_pc) PC_RELATIVE_TARGET(pc+1);
+        exit_target = get_unlinked_entry(dcontext, cur_target);
+        pc++; /* skip jmp opcode */
+        pc = insert_relative_target(pc, exit_target, HOT_PATCHABLE);   
+    }
+
+    /* To maintain atomicity with respect to executing thread, must unlink
+     * the ending jmp (above) first so that the unlinked path can detect the
+     * race condition case */
+# ifdef WINDOWS
+    /* faster than is_shared_syscall_routine() since only linked target can get here
+     * yet inconsistent
+     */
+    if (target_tag != shared_syscall_routine_ex
+        (dcontext _IF_X64(MODE_OVERRIDE(FRAG_IS_32(f->flags)))))
+# endif
+    {
+        /* need to make branch target the unlink entry point inside exit stub */
+        if (ibl_code->ibl_head_is_inlined) {
+# ifdef CUSTOM_EXIT_STUBS
+            /* FIXME: now custom code will be skipped!!! */
+            cache_pc target = fixed_stub_pc;
+# else
+            cache_pc target = stub_pc;
+# endif
+
+            /* now add offset of unlinked entry */
+            target += ibl_code->inline_unlink_offs;
+            patch_branch(EXIT_CTI_PC(f, l), target, HOT_PATCHABLE);
+        }
+    }
+}
+
+/*******************************************************************************
+ * COARSE-GRAIN FRAGMENT SUPPORT
+ */
+
+cache_pc
+entrance_stub_jmp(cache_pc stub)
+{
+#ifdef X64
+    if (*stub == 0x65)
+        return (stub + STUB_COARSE_DIRECT_SIZE64 - JMP_LONG_LENGTH);
+    /* else, 32-bit stub */
+#endif
+    return (stub + STUB_COARSE_DIRECT_SIZE32 - JMP_LONG_LENGTH);
+}
+
+/* Returns whether stub is an entrance stub as opposed to a fragment
+ * or a coarse indirect stub.  FIXME: if we separate coarse indirect
+ * stubs from bodies we'll need to put them somewhere else, or fix up
+ * decode_fragment() to be able to distinguish them in some other way
+ * like first instruction tls slot.
+ */
+bool
+coarse_is_entrance_stub(cache_pc stub)
+{
+    bool res = false;
+    /* FIXME: case 10334: pass in info and if non-NULL avoid lookup here? */
+    coarse_info_t *info = get_stub_coarse_info(stub);
+    if (info != NULL) {
+        res = ALIGNED(stub, coarse_stub_alignment(info)) &&
+            *entrance_stub_jmp(stub) == JMP_OPCODE;
+        DODEBUG({
+            if (res) {
+                cache_pc tgt = entrance_stub_jmp_target(stub);
+                ASSERT(!in_fcache(stub));
+                ASSERT(tgt == trace_head_return_coarse_prefix(stub, info) ||
+                       tgt == fcache_return_coarse_prefix(stub, info) ||
+                       /* another fragment */
+                       in_fcache(tgt));
+            }
+        });
+    }
+    return res;
+}
+
+/* FIXME: case 10334: pass in info? */
+bool
+coarse_is_trace_head(cache_pc stub)
+{
+    if (coarse_is_entrance_stub(stub)) {
+        cache_pc tgt = entrance_stub_jmp_target(stub);
+        /* FIXME: could see if tgt is a jmp and deref and cmp to
+         * trace_head_return_coarse_routine() to avoid the vmvector
+         * lookup required to find the prefix
+         */
+        return tgt == trace_head_return_coarse_prefix(stub, NULL);
+    }
+    return false;
+}
+
+cache_pc
+entrance_stub_jmp_target(cache_pc stub)
+{
+    cache_pc jmp = entrance_stub_jmp(stub);
+    cache_pc tgt;
+    ASSERT(jmp != NULL);
+    tgt = (cache_pc) PC_RELATIVE_TARGET(jmp+1);
+    ASSERT(*jmp == JMP_OPCODE);
+    return tgt;
+}
+
+cache_pc
+entrance_stub_target_tag(cache_pc stub)
+{
+    cache_pc jmp = entrance_stub_jmp(stub);
+    /* find the immed that is put into tls: at end of pre-jmp instr */
+#ifdef X64
+    /* To identify whether 32-bit: we could look up the coarse_info_t
+     * this is part of but that's expensive so we check whether the
+     * tls offset has 2 high byte 0's (we always use addr16 for 32-bit).
+     * 32-bit:
+     *   67 64 c7 06 e0 0e 02 99 4e 7d  addr16 mov $0x7d4e9902 -> %fs:0x0ee0 
+     * 64-bit is split into high and low dwords:
+     *   65 c7 04 25 20 16 00 00 02 99 4e 7d  mov $0x7d4e9902 -> %gs:0x1620 
+     *   65 c7 04 25 24 16 00 00 00 00 00 00  mov $0x00000000 -> %gs:0x1624
+     * both are followed by a direct jmp.
+     */
+    if (*((ushort *)(jmp-6)) == 0) { /* 64-bit has 2 0's for high 2 bytes of tls offs */
+        ptr_uint_t high32 = (ptr_uint_t) *((uint *)(jmp-4));
+        ptr_uint_t low32 = (ptr_uint_t)
+            *((uint *)(jmp - (SIZE64_MOV_PTR_IMM_TO_TLS/2) - 4));
+        return (cache_pc) ((high32 << 32) | low32);
+    } /* else fall-through to 32-bit case */
+#endif
+    return *((cache_pc *)(jmp-4));
+}
+
+bool
+coarse_is_indirect_stub(cache_pc pc)
+{
+    /* match insert_jmp_to_ibl */
+    return instr_raw_is_tls_spill(pc, REG_XBX, INDIRECT_STUB_SPILL_SLOT);
+}
+
+cache_pc
+coarse_indirect_stub_jmp_target(cache_pc stub)
+{
+    cache_pc prefix_tgt, tgt;
+    cache_pc jmp;
+    size_t stub_size;
+#ifdef X64
+    /* See the stub sequences in entrance_stub_target_tag(): 32-bit always has
+     * an addr prefix while 64-bit does not
+     */
+    /* FIXME: PR 209709: test perf and remove if outweighs space */
+    if (*stub == ADDR_PREFIX_OPCODE)
+        stub_size = STUB_COARSE_INDIRECT_SIZE(FRAG_32_BIT);
+    else /* default */
+#endif
+        stub_size = STUB_COARSE_INDIRECT_SIZE(0);
+    jmp = stub + stub_size - JMP_LONG_LENGTH;
+    ASSERT(*jmp == JMP_OPCODE);
+    prefix_tgt = (cache_pc) PC_RELATIVE_TARGET(jmp+1);
+    ASSERT(*prefix_tgt == JMP_OPCODE);
+    tgt = (cache_pc) PC_RELATIVE_TARGET(prefix_tgt+1);
+    return tgt;
+}
+
+uint
+coarse_indirect_stub_size(coarse_info_t *info)
+{
+    /* Keep in synch w/ exit_stub_size().  We export this separately since
+     * it's difficult to get the target to pass to exit_stub_size().
+     */
+    return STUB_COARSE_INDIRECT_SIZE(COARSE_32_FLAG(info));
+}
+
+/* Passing in stub's info avoids a vmvector lookup */
+bool
+entrance_stub_linked(cache_pc stub, coarse_info_t *info /*OPTIONAL*/)
+{
+    /* entrance stubs are of two types:
+     * - targeting trace heads: always point to trace_head_return_coarse,
+     *   whether target exists or not, so are always unlinked;
+     * - targeting non-trace-heads: if linked, point to fragment; if unlinked,
+     *   point to fcache_return_coarse
+     */
+    cache_pc tgt = entrance_stub_jmp_target(stub);
+    /* FIXME: do vmvector just once instead of for each call */
+    return (tgt != trace_head_return_coarse_prefix(stub, info) &&
+            tgt != fcache_return_coarse_prefix(stub, info));
+}
+
+/* Returns whether it had to change page protections */
+static bool
+patch_coarse_branch(cache_pc stub, cache_pc tgt, bool hot_patch,
+                    coarse_info_t *info /*OPTIONAL*/)
+{
+    bool stubs_readonly = false;
+    bool stubs_restore = false;
+    if (DYNAMO_OPTION(persist_protect_stubs)) {
+        if (info == NULL)
+            info = get_stub_coarse_info(stub);
+        ASSERT(info != NULL);
+        if (info->stubs_readonly) {
+            stubs_readonly = true;
+            stubs_restore = true;
+            /* if we don't preserve mapped-in COW state the protection change
+             * will fail (case 10570)
+             */
+            make_copy_on_writable((byte *)PAGE_START(entrance_stub_jmp(stub)),
+                                  /* stub jmp can't cross page boundary (can't
+                                   * cross cache line in fact) */
+                                  PAGE_SIZE);
+            if (DYNAMO_OPTION(persist_protect_stubs_limit) > 0) {
+                info->stubs_write_count++;
+                if (info->stubs_write_count >
+                    DYNAMO_OPTION(persist_protect_stubs_limit)) {
+                    SYSLOG_INTERNAL_WARNING_ONCE("pcache stubs over write limit");
+                    STATS_INC(pcache_unprot_over_limit);
+                    stubs_restore = false;
+                    info->stubs_readonly = false;
+                }
+            }
+        }
+    }
+    patch_branch(entrance_stub_jmp(stub), tgt, HOT_PATCHABLE);
+    if (stubs_restore)
+        make_unwritable((byte *)PAGE_START(entrance_stub_jmp(stub)), PAGE_SIZE);
+    return stubs_readonly;
+}
+
+/* Passing in stub's info avoids a vmvector lookup */
+void
+link_entrance_stub(dcontext_t *dcontext, cache_pc stub, cache_pc tgt,
+                   bool hot_patch, coarse_info_t *info /*OPTIONAL*/)
+{
+    ASSERT(DYNAMO_OPTION(coarse_units));
+    ASSERT(self_owns_recursive_lock(&change_linking_lock));
+    LOG(THREAD, LOG_LINKS, 5, "link_entrance_stub "PFX"\n", stub);
+    if (patch_coarse_branch(stub, tgt, hot_patch, info))
+        STATS_INC(pcache_unprot_link);
+    /* We check this afterward since this link may be what makes it consistent
+     * FIXME: pass in arg to not check target?  Then call before and after */
+    ASSERT(coarse_is_entrance_stub(stub));
+}
+
+/* Passing in stub's info avoids a vmvector lookup */
+void
+unlink_entrance_stub(dcontext_t *dcontext, cache_pc stub, uint flags,
+                     coarse_info_t *info /*OPTIONAL*/)
+{
+    cache_pc tgt;
+    ASSERT(DYNAMO_OPTION(coarse_units));
+    ASSERT(coarse_is_entrance_stub(stub));
+    ASSERT(self_owns_recursive_lock(&change_linking_lock));
+    LOG(THREAD, LOG_LINKS, 5,
+        "unlink_entrance_stub "PFX"\n", stub);
+    if (TESTANY(FRAG_IS_TRACE_HEAD|FRAG_IS_TRACE, flags))
+        tgt = trace_head_return_coarse_prefix(stub, info);
+    else
+        tgt = fcache_return_coarse_prefix(stub, info);
+    if (patch_coarse_branch(stub, tgt, HOT_PATCHABLE, info))
+        STATS_INC(pcache_unprot_unlink);
+}
+
+cache_pc
+entrance_stub_from_cti(cache_pc cti)
+{
+    cache_pc disp = exit_cti_disp_pc(cti);
+    cache_pc tgt = (cache_pc) PC_RELATIVE_TARGET(disp);
+    return tgt;
+}
+
+/*******************************************************************************/
+
+/* Patch list support routines */
+void
+init_patch_list(patch_list_t *patch, patch_list_type_t type)
+{
+    patch->num_relocations = 0;
+    ASSERT_TRUNCATE(patch->type, ushort, type);
+    patch->type = (ushort) type;
+}
+
+/* add an instruction to patch list and address of location for future updates */
+/* Use the type checked wrappers add_patch_entry or add_patch_marker */
+static void
+add_patch_entry_internal(patch_list_t *patch, instr_t *instr, ushort patch_flags, 
+                         short instruction_offset,
+                         ptr_uint_t value_location_offset)
+{
+    uint i = patch->num_relocations;
+
+    ASSERT(patch->num_relocations < MAX_PATCH_ENTRIES);
+    /* Since in debug build we have the extra slots for stats, it's important
+     * to provide a useful release build message
+     */
+    if (patch->num_relocations >= MAX_PATCH_ENTRIES) {
+        SYSLOG_CUSTOM_NOTIFY(SYSLOG_CRITICAL, MSG_EXCEPTION, 4,
+                             "Maximum patch entries exceeded",
+                             get_application_name(), get_application_pid(), 
+                             "<maxpatch>", "Maximum patch entries exceeded");
+        os_terminate(get_thread_private_dcontext(), TERMINATE_PROCESS);
+        ASSERT_NOT_REACHED();
+    }
+
+    LOG(THREAD_GET, LOG_EMIT, 4, 
+        "add_patch_entry[%d] value_location_offset="PFX"\n", i,
+        value_location_offset);
+
+    patch->entry[i].where.instr = instr;
+    patch->entry[i].patch_flags = patch_flags;
+    patch->entry[i].value_location_offset = value_location_offset;
+    patch->entry[i].instr_offset = instruction_offset; 
+
+    patch->num_relocations++;
+}
+
+/* add an instruction to patch list and address of location for future updates */
+static inline void
+add_patch_entry(patch_list_t *patch, instr_t *instr, ushort patch_flags,
+                ptr_uint_t value_location_offset)
+{
+    add_patch_entry_internal(patch, instr, patch_flags, -4 /* offset of imm32 argument */,
+                             value_location_offset);
+}
+
+/* add an instruction to patch list to retrieve its offset later.
+   Takes an instruction and an offset within the instruction.
+   Result: The offset within an encoded instruction stream will
+   be stored in target_offset by encode_with_patch_list 
+*/
+void
+add_patch_marker(patch_list_t *patch, instr_t *instr, ushort patch_flags,
+                 short instr_offset, ptr_uint_t *target_offset /* OUT */) 
+{
+    add_patch_entry_internal(patch, instr, (ushort) (patch_flags | PATCH_MARKER),
+                             instr_offset, (ptr_uint_t) target_offset);
+}
+
+/* remove PATCH_MARKER entries since not needed for dynamic updates */
+static INLINE_ONCE void
+remove_assembled_patch_markers(dcontext_t *dcontext, patch_list_t *patch)
+{
+    ushort i=0, j=0;
+
+    /* we can remove the PATCH_MARKER entries after encoding, 
+       and so patch_emitted_code won't even need to check for PATCH_MARKER
+    */
+
+    while (j < patch->num_relocations) {
+        if (TEST(PATCH_MARKER, patch->entry[j].patch_flags)) {
+            LOG(THREAD, LOG_EMIT, 4,
+                "remove_assembled_patch_markers: removing marker %d\n", j);
+        } else {
+            patch->entry[i] = patch->entry[j];
+            i++;
+        }
+
+        j++;
+    }
+
+    LOG(THREAD, LOG_EMIT, 3, "remove_assembled_patch_markers: relocations %d, left only %d\n", 
+        patch->num_relocations, i);
+    patch->num_relocations = i;
+}
+
+
+/* Indirect all instructions instead of later patching */
+static void
+relocate_patch_list(dcontext_t *dcontext, patch_list_t *patch, 
+                    instrlist_t *ilist)
+{
+    instr_t *inst;
+    uint cur = 0;
+    LOG(THREAD, LOG_EMIT, 3, "relocate_patch_list ["PFX"]\n", patch);
+
+    /* go through the instructions and "relocate" by indirectly using XDI */
+    for (inst = instrlist_first(ilist); inst; inst = instr_get_next(inst)) {
+        if (cur < patch->num_relocations && 
+            inst == patch->entry[cur].where.instr) {
+            ASSERT(!TEST(PATCH_OFFSET_VALID, patch->entry[cur].patch_flags));
+
+            if (!TEST(PATCH_MARKER, patch->entry[cur].patch_flags)) {
+                opnd_t opnd;
+                ASSERT(instr_num_srcs(inst) > 0);
+                opnd = instr_get_src(inst, 0);
+                    
+                DOLOG(4, LOG_EMIT, {
+                    LOG(THREAD, LOG_EMIT, 2,
+                        "encode_with_patch_list: patch_entry_t[%d] before update \n");
+                    instr_disassemble(dcontext, inst, THREAD);
+                    LOG(THREAD, LOG_EMIT, 2, "\n");
+                });
+                /* we assume that per_thread_t will be in XDI, 
+                   and the displacement is in value_location_offset */
+                IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int
+                              (patch->entry[cur].value_location_offset)));
+                if (opnd_is_near_base_disp(opnd)) {
+                    /* indirect through XDI and update displacement */
+                    opnd_set_disp(&opnd, (int) patch->entry[cur].value_location_offset);
+                    opnd_replace_reg(&opnd, REG_NULL, REG_XDI);
+                } else if (opnd_is_immed_int(opnd)) {
+                    /* indirect through XDI and set displacement */
+                    /* converting AND $0x00003fff, %xcx -> %xcx 
+                       into       AND  mask(%xdi), %xcx -> %xcx 
+                    */
+                    opnd = opnd_create_base_disp
+                        (REG_XDI, REG_NULL, 0,
+                         (int) patch->entry[cur].value_location_offset, OPSZ_4);
+                }
+
+                instr_set_src(inst, 0, opnd);
+                DOLOG(3, LOG_EMIT, {
+                    LOG(THREAD, LOG_EMIT, 2,
+                        "encode_with_patch_list: patch_entry_t[%d] after update \n");
+                    instr_disassemble(dcontext, inst, THREAD);
+                    LOG(THREAD, LOG_EMIT, 2, "\n");
+                });
+            }
+            cur++;
+        }
+    }
+}
+
+/* Updates patch list with offsets in assembled instruction list */
+/* Cf: instrlist_encode which does not support a patch list */
+/* Returns length of emitted code */
+int
+encode_with_patch_list(dcontext_t *dcontext, patch_list_t *patch, 
+                       instrlist_t *ilist, cache_pc start_pc)
+{
+    instr_t *inst;
+    uint len;
+    uint cur;
+    cache_pc pc = start_pc;
+
+    ASSERT(patch->num_relocations < MAX_PATCH_ENTRIES);
+
+    if (patch->type == PATCH_TYPE_INDIRECT_XDI) {
+        relocate_patch_list(dcontext, patch, ilist);
+    }
+
+    /* now encode the instructions */
+    /* must set note fields first with offset */
+    len = 0;
+    for (inst = instrlist_first(ilist); inst; inst = instr_get_next(inst)) {
+        instr_set_note(inst, (void *)(ptr_uint_t)len);
+        len += instr_length(dcontext, inst);
+    }
+
+    cur = 0;
+    /* after instruction list is assembled we collect the offsets */
+    for (inst = instrlist_first(ilist); inst; inst = instr_get_next(inst)) {
+        short offset_in_instr = patch->entry[cur].instr_offset;
+        byte *nxt_pc = instr_encode(dcontext, inst, pc);
+        ASSERT(nxt_pc != NULL);
+        len = (int) (nxt_pc - pc);
+        pc = nxt_pc;
+
+        if (cur < patch->num_relocations && 
+            inst == patch->entry[cur].where.instr) {
+            ASSERT(!TEST(PATCH_OFFSET_VALID, patch->entry[cur].patch_flags));
+
+            /* support positive offsets from beginning and negative - from end of instruction */
+            if (offset_in_instr < 0) {
+                /* grab offset offset_in_instr bytes from the end of instruction */
+                /* most commonly -4 for a 32bit immediate  */
+                patch->entry[cur].where.offset = 
+                    ((pc + offset_in_instr) - start_pc);
+            } else {
+                /* grab offset after skipping offset_in_instr from beginning of instruction */
+                patch->entry[cur].where.offset =
+                    ((pc - len + offset_in_instr) - start_pc);
+            }
+            patch->entry[cur].patch_flags |= PATCH_OFFSET_VALID;
+
+            LOG(THREAD, LOG_EMIT, 4,
+                "encode_with_patch_list: patch_entry_t[%d] offset="PFX"\n",
+                cur, patch->entry[cur].where.offset);
+
+            if (TEST(PATCH_MARKER, patch->entry[cur].patch_flags)) {
+                /* treat value_location_offset as an output argument 
+                   and store there the computed offset, 
+                */
+                ptr_uint_t *output_value = (ptr_uint_t *)
+                    patch->entry[cur].value_location_offset;
+                ptr_uint_t output_offset = patch->entry[cur].where.offset;
+                if (TEST(PATCH_ASSEMBLE_ABSOLUTE, patch->entry[cur].patch_flags)) {
+                    ASSERT(!TEST(PATCH_UINT_SIZED, patch->entry[cur].patch_flags));
+                    output_offset += (ptr_uint_t)start_pc;
+                }
+                if (TEST(PATCH_UINT_SIZED, patch->entry[cur].patch_flags)) {
+                    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(output_offset)));
+                    *((uint *)output_value) = (uint) output_offset;
+                } else
+                    *output_value = output_offset;
+            }
+
+            LOG(THREAD, LOG_EMIT, 4, 
+                "encode_with_patch_list [%d] extras patch_flags=0x%x value_offset="
+                PFX"\n", cur, patch->entry[cur].patch_flags,
+                patch->entry[cur].value_location_offset);
+            cur++;
+        }
+    }
+
+    /* assuming patchlist is in the same order as ilist, we should have seen all */
+    LOG(THREAD, LOG_EMIT, 4, "cur %d, num %d", cur, patch->num_relocations);
+    ASSERT(cur == patch->num_relocations);
+
+    remove_assembled_patch_markers(dcontext, patch);
+    ASSERT(CHECK_TRUNCATE_TYPE_int(pc - start_pc));
+    return (int)(pc - start_pc);
+}
+
+#ifdef DEBUG
+void
+print_patch_list(patch_list_t *patch)
+{
+    uint i;
+    LOG(THREAD_GET, LOG_EMIT, 4, "patch="PFX" num_relocations=%d\n", 
+        patch, patch->num_relocations);
+
+    for(i=0; i<patch->num_relocations; i++) {
+        ASSERT(TEST(PATCH_OFFSET_VALID, patch->entry[i].patch_flags));
+        LOG(THREAD_GET, LOG_EMIT, 4, 
+            "patch_list [%d] offset="PFX" patch_flags=%d value_offset="PFX"\n", i, 
+            patch->entry[i].where.offset, 
+            patch->entry[i].patch_flags, 
+            patch->entry[i].value_location_offset);
+    }
+}
+
+# ifdef INTERNAL
+/* disassembles code adding patch list labels */
+static void
+disassemble_with_annotations(dcontext_t *dcontext, patch_list_t *patch, 
+                             byte *start_pc, byte *end_pc)
+{
+    byte *pc = start_pc;
+    uint cur = 0;
+
+    do {
+        if (cur < patch->num_relocations && 
+            pc >= start_pc + patch->entry[cur].where.offset) {
+            ASSERT(TEST(PATCH_OFFSET_VALID, patch->entry[cur].patch_flags));
+            /* this is slightly off - we'll mark next instruction, 
+               but is good enough for this purpose */
+            LOG(THREAD, LOG_EMIT, 2, "%d:", cur);
+            cur++;
+        } else {
+            LOG(THREAD, LOG_EMIT, 2, "  ");
+        }
+
+        pc = disassemble_with_bytes(dcontext, pc, THREAD);       
+    } while (pc < end_pc);
+    LOG(THREAD, LOG_EMIT, 2, "\n");
+}
+# endif
+#endif
+
+/* updates emitted code according to patch list */
+static void
+patch_emitted_code(dcontext_t *dcontext, patch_list_t *patch, byte *start_pc)
+{
+    uint i;
+    /* FIXME: can get this as a patch list entry through indirection */
+    per_thread_t *pt = (per_thread_t *) dcontext->fragment_field;
+    ASSERT(dcontext != GLOBAL_DCONTEXT && dcontext != NULL);
+
+    LOG(THREAD, LOG_EMIT, 2, "patch_emitted_code start_pc="PFX" pt="PFX"\n", 
+        start_pc);
+    if (patch->type != PATCH_TYPE_ABSOLUTE) {
+        LOG(THREAD, LOG_EMIT, 2,
+            "patch_emitted_code type=%d indirected, nothing to patch\n", patch->type);
+        /* FIXME: propagate the check earlier to save the extraneous calls 
+           to update_indirect_exit_stub and update_indirect_branch_lookup
+        */
+        return;
+    }
+    DOLOG(4, LOG_EMIT, {
+        print_patch_list(patch);
+    });
+    for(i=0; i<patch->num_relocations; i++) {
+        byte *pc = start_pc + patch->entry[i].where.offset;
+        /* value address, (think for example of pt->trace.hash_mask) */
+        ptr_uint_t value;
+        char *vaddr = NULL;
+        if (TEST(PATCH_PER_THREAD, patch->entry[i].patch_flags)) {
+            vaddr = (char *)pt + patch->entry[i].value_location_offset;
+        } else if (TEST(PATCH_UNPROT_STAT, patch->entry[i].patch_flags)) {
+            /* separate the two parts of the stat */
+            uint unprot_offs = (uint) (patch->entry[i].value_location_offset) >> 16;
+            uint field_offs = (uint) (patch->entry[i].value_location_offset) & 0xffff;
+            IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint
+                          (patch->entry[i].value_location_offset)));
+            vaddr = (*((char **)((char *)pt + unprot_offs))) + field_offs;
+            LOG(THREAD, LOG_EMIT, 4, 
+                "patch_emitted_code [%d] value "PFX" => 0x%x 0x%x => "PFX"\n",
+                i, patch->entry[i].value_location_offset, unprot_offs, field_offs, vaddr);
+        } 
+        else
+            ASSERT_NOT_REACHED();
+        ASSERT(TEST(PATCH_OFFSET_VALID, patch->entry[i].patch_flags));
+        ASSERT(!TEST(PATCH_MARKER, patch->entry[i].patch_flags));
+
+        if (!TEST(PATCH_TAKE_ADDRESS, patch->entry[i].patch_flags)) {
+            /* use value pointed by computed address */
+            if (TEST(PATCH_UINT_SIZED, patch->entry[i].patch_flags))
+                value = (ptr_uint_t) *((uint *)vaddr);
+            else
+                value = *(ptr_uint_t*)vaddr;
+        } else {
+            ASSERT(!TEST(PATCH_UINT_SIZED, patch->entry[i].patch_flags));
+            value = (ptr_uint_t)vaddr;   /* use computed address */
+        }
+
+        LOG(THREAD, LOG_EMIT, 4, 
+            "patch_emitted_code [%d] offset="PFX" patch_flags=%d value_offset="PFX
+            " vaddr="PFX" value="PFX"\n", i, 
+            patch->entry[i].where.offset, patch->entry[i].patch_flags, 
+            patch->entry[i].value_location_offset, vaddr, value);
+        if (TEST(PATCH_UINT_SIZED, patch->entry[i].patch_flags)) {
+            IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(value)));
+            *((uint*)pc) = (uint) value;
+        } else
+            *((ptr_uint_t *)pc) = value;
+        LOG(THREAD, LOG_EMIT, 4,
+            "patch_emitted_code: updated pc *"PFX" = "PFX"\n", pc, value);
+    }
+
+    STATS_INC(emit_patched_fragments);
+    DOSTATS({
+        /* PR 217008: avoid gcc warning from truncation assert in XSTATS_ADD_DC */
+        int tmp_num = patch->num_relocations;
+        STATS_ADD(emit_patched_relocations, tmp_num);
+    });
+    LOG(THREAD, LOG_EMIT, 4, "patch_emitted_code done\n");
+}
+
+#define GET_IB_FTABLE(ibl_code,target_trace_table,field)                   \
+  (GET_IBL_TARGET_TABLE((ibl_code)->branch_type, (target_trace_table))     \
+   + offsetof(ibl_table_t, field))
+
+/* Updates an indirect branch exit stub with the latest hashtable mask
+ * and hashtable address
+ * See also update_indirect_branch_lookup
+ */
+void
+update_indirect_exit_stub(dcontext_t *dcontext, fragment_t *f, linkstub_t *l)
+{
+    generated_code_t *code = get_emitted_routines_code
+        (dcontext _IF_X64(MODE_OVERRIDE(FRAG_IS_32(f->flags))));
+# ifdef CUSTOM_EXIT_STUBS
+    byte *start_pc = (byte *) EXIT_FIXED_STUB_PC(dcontext, f, l);
+# else
+    byte *start_pc = (byte *) EXIT_STUB_PC(dcontext, f, l);
+# endif
+    ibl_branch_type_t branch_type;
+
+    ASSERT(linkstub_owned_by_fragment(dcontext, f, l));
+    ASSERT(LINKSTUB_INDIRECT(l->flags));
+    ASSERT(EXIT_HAS_STUB(l->flags, f->flags));
+    /* Shared use indirection so no patching needed -- caller should check */
+    ASSERT(!TEST(FRAG_SHARED, f->flags));
+#ifdef WINDOWS
+    /* Do not touch shared_syscall */
+    if (EXIT_TARGET_TAG(dcontext, f, l) ==
+        shared_syscall_routine_ex(dcontext _IF_X64(MODE_OVERRIDE(FRAG_IS_32(f->flags)))))
+        return;
+#endif
+    branch_type = extract_branchtype(l->flags);
+
+    LOG(THREAD, LOG_EMIT, 4, "update_indirect_exit_stub: f->tag="PFX"\n", 
+        f->tag);
+
+    if (DYNAMO_OPTION(disable_traces) && !code->bb_ibl[branch_type].ibl_head_is_inlined) {
+        return;
+    }
+        
+    if (TEST(FRAG_IS_TRACE, f->flags)) {
+        ASSERT(code->trace_ibl[branch_type].ibl_head_is_inlined);
+        patch_emitted_code(dcontext, &code->trace_ibl[branch_type].ibl_stub_patch, start_pc);
+    } else {
+        ASSERT(code->bb_ibl[branch_type].ibl_head_is_inlined);
+        patch_emitted_code(dcontext, &code->bb_ibl[branch_type].ibl_stub_patch, start_pc);
+    }
+}
+
+/*###########################################################################
+ *
+ * fragment_t Prefixes
+ *
+ * Two types: indirect branch target, which restores eflags and xcx, and
+ * normal prefix, which just restores xcx
+ */
+
+/* Indirect Branch Target Prefix
+ * We have 3 different prefixes: one if we don't need to restore eflags, one
+ * if we need to restore just using sahf, and one if we also need to restore
+ * the overflow flag OF.
+ *
+ * FIXME: currently we cache-align the prefix, not the normal
+ * entry point...if prefix gets much longer, might want to add
+ * nops to get normal entry cache-aligned?
+ */
+
+/* for now all ibl targets must use same scratch locations: tls or not, no mixture */
+/* PR 244737: x64 always uses tls even if all-private */
+#define IBL_EFLAGS_IN_TLS() (IF_X64_ELSE(true, SHARED_IB_TARGETS()))
+
+#define RESTORE_XAX_PREFIX(flags) \
+    (IBL_EFLAGS_IN_TLS() ? SIZE_MOV_XAX_TO_TLS(flags, false) : SIZE32_MOV_XAX_TO_ABS)
+#define PREFIX_BASE(flags) \
+    (RESTORE_XAX_PREFIX(flags) + FRAGMENT_BASE_PREFIX_SIZE(flags))
+
+enum {PREFIX_SIZE_RESTORE_OF = 2, /* add $0x7f, %al  */
+      PREFIX_SIZE_FIVE_EFLAGS = 1,    /* SAHF */
+};
+
+/* use indirect branch target prefix? */
+static inline bool
+use_ibt_prefix(uint flags)
+{
+    /* when no traces, all bbs use IBT prefix */
+    /* FIXME: currently to allow bb2bb we simply have a prefix on all BB's
+     * should experiment with a shorter prefix for targetting BBs
+     * by restoring the flags in the IBL routine,
+     * or even jump through memory to avoid having the register restore prefix
+     * Alternatively, we can reemit a fragment only once it is known to be an IBL target,
+     * assuming the majority will be reached with an IB when they are first built.  
+     * (Simplest counterexample is of a return from a function with no arguments
+     * called within a conditional, but the cache compaction of not having
+     * prefixes on all bb's may offset this double emit).
+     * All of these are covered by case 147.
+     */
+    return (IS_IBL_TARGET(flags) &&
+            /* coarse bbs (and fine in presence of coarse) do not support prefixes */
+            !(DYNAMO_OPTION(coarse_units) &&
+              !TEST(FRAG_IS_TRACE, flags) &&
+              DYNAMO_OPTION(bb_ibl_targets)));
+}
+
+static inline bool
+ibl_use_target_prefix(ibl_code_t *ibl_code)
+{
+   return !(DYNAMO_OPTION(coarse_units) &&
+            /* If coarse units are enabled we need to have no prefix
+             * for both fine and coarse bbs
+             */
+            ((ibl_code->source_fragment_type == IBL_COARSE_SHARED &&
+              DYNAMO_OPTION(bb_ibl_targets)) ||
+             (IS_IBL_BB(ibl_code->source_fragment_type) &&
+              /* FIXME case 147/9636: if -coarse_units -bb_ibl_targets
+               * but traces are enabled, we won't put prefixes on regular
+               * bbs but will assume we have them here!  We don't support
+               * that combination yet.  When we do this routine should return
+               * another bit of info: whether to do two separate lookups.
+               */
+              DYNAMO_OPTION(disable_traces) && DYNAMO_OPTION(bb_ibl_targets))));
+}
+
+int
+fragment_prefix_size(uint flags)
+{
+    if (use_ibt_prefix(flags)) {
+        bool use_eflags_restore = TEST(FRAG_IS_TRACE, flags) ?
+            !DYNAMO_OPTION(trace_single_restore_prefix) :
+            !DYNAMO_OPTION(bb_single_restore_prefix);
+        /* The common case is !INTERNAL_OPTION(unsafe_ignore_eflags*) so
+         * PREFIX_BASE(flags) is defined accordingly, and we subtract from it to
+         * get the correct value when the option is on.
+         */
+        if (INTERNAL_OPTION(unsafe_ignore_eflags_prefix)) {
+            if (INTERNAL_OPTION(unsafe_ignore_eflags_ibl)) {
+                ASSERT(PREFIX_BASE(flags) - RESTORE_XAX_PREFIX(flags) >= 0);
+                return PREFIX_BASE(flags) - RESTORE_XAX_PREFIX(flags);
+            } else {
+                /* still need to restore xax, just don't restore eflags */
+                return PREFIX_BASE(flags);
+            }
+        }
+        if (!use_eflags_restore)
+            return PREFIX_BASE(flags) - RESTORE_XAX_PREFIX(flags);
+        if (TEST(FRAG_WRITES_EFLAGS_6, flags)) /* no flag restoration needed */
+            return PREFIX_BASE(flags);
+        else if (TEST(FRAG_WRITES_EFLAGS_OF, flags)) /* no OF restoration needed */
+            return (PREFIX_BASE(flags) + PREFIX_SIZE_FIVE_EFLAGS);
+        else /* must restore all 6 flags */
+            if (INTERNAL_OPTION(unsafe_ignore_overflow)) {
+                /* do not restore OF */
+                return (PREFIX_BASE(flags) + PREFIX_SIZE_FIVE_EFLAGS);
+            } else {
+                return (PREFIX_BASE(flags) + PREFIX_SIZE_RESTORE_OF +
+                        PREFIX_SIZE_FIVE_EFLAGS);
+            }
+    } else {
+#ifdef CLIENT_INTERFACE
+        if (dynamo_options.bb_prefixes)
+            return FRAGMENT_BASE_PREFIX_SIZE(flags);
+        else
+#endif
+            return 0;
+    }
+}
+
+/* See SAVE_TO_DC_OR_TLS() in mangle.c for the save xcx code */
+static cache_pc
+insert_restore_xcx(dcontext_t *dcontext, cache_pc pc, uint flags, bool require_addr16)
+{
+    /* shared fragment prefixes use tls, private use mcontext
+     * this works b/c the shared ibl copies the app xcx to both places!
+     * private_ib_in_tls option makes all prefixes use tls
+     */
+    return insert_spill_or_restore(dcontext, pc, flags, false/*restore*/,
+                                   XCX_IN_TLS(flags), REG_XCX,
+                                   MANGLE_XCX_SPILL_SLOT, XCX_OFFSET,
+                                   require_addr16);
+}
+
+void
+insert_fragment_prefix(dcontext_t *dcontext, fragment_t *f)
+{
+    byte *pc = (byte *) f->start_pc;
+    bool insert_eflags_xax_restore = TEST(FRAG_IS_TRACE, f->flags) ?
+        !DYNAMO_OPTION(trace_single_restore_prefix) :
+        !DYNAMO_OPTION(bb_single_restore_prefix);
+    ASSERT(f->prefix_size == 0); /* shouldn't be any prefixes yet */
+    if (use_ibt_prefix(f->flags)) {
+        if ((!INTERNAL_OPTION(unsafe_ignore_eflags_prefix) ||
+             !INTERNAL_OPTION(unsafe_ignore_eflags_ibl)) &&
+            insert_eflags_xax_restore) {
+            if (!INTERNAL_OPTION(unsafe_ignore_eflags_prefix) &&
+                !TEST(FRAG_WRITES_EFLAGS_6, f->flags)) {
+                if (!TEST(FRAG_WRITES_EFLAGS_OF, f->flags)
+                    && !INTERNAL_OPTION(unsafe_ignore_overflow)) {
+                    DEBUG_DECLARE(byte *restore_of_prefix_pc = pc;)
+                    /* must restore OF
+                     * we did a seto on %al, so we restore OF by adding 0x7f to
+                     * %al (7f not ff b/c add only sets OF for signed operands,
+                     * sets CF for uint)
+                     */
+                    STATS_INC(num_oflag_prefix_restore);
+
+                    /* 04 7f   add $0x7f,%al */
+                    *pc = ADD_AL_OPCODE; pc++;
+                    *pc = 0x7f; pc++;
+
+                    ASSERT(pc - restore_of_prefix_pc == PREFIX_SIZE_RESTORE_OF);
+                }
+            
+                /* restore other 5 flags w/ sahf */
+                *pc = SAHF_OPCODE; pc++;
+                ASSERT(PREFIX_SIZE_FIVE_EFLAGS == 1);
+            }
+            /* restore xax */
+            pc = insert_restore_xax(dcontext, pc, f->flags, IBL_EFLAGS_IN_TLS(),
+                                    PREFIX_XAX_SPILL_SLOT, false);
+        }
+        
+        pc = insert_restore_xcx(dcontext, pc, f->flags, false);
+        
+        /* set normal entry point to be next pc */
+        ASSERT_TRUNCATE(f->prefix_size, byte, ((cache_pc) pc) - f->start_pc);
+        f->prefix_size = (byte)(((cache_pc) pc) - f->start_pc);
+    } else {
+#ifdef CLIENT_INTERFACE
+        if (dynamo_options.bb_prefixes) {
+            pc = insert_restore_xcx(dcontext, pc, f->flags, false);
+
+            /* set normal entry point to be next pc */
+            ASSERT_TRUNCATE(f->prefix_size, byte, ((cache_pc) pc) - f->start_pc);
+            f->prefix_size = (byte)(((cache_pc) pc) - f->start_pc);
+        } /* else, no prefix */
+#endif
+    }
+    /* make sure emitted size matches size we requested */
+    ASSERT(f->prefix_size == fragment_prefix_size(f->flags));
+}
+
+#ifdef PROFILE_RDTSC
+/***************************************************************************
+ ***************************************************************************
+ ** PROFILING USING RDTSC
+ **
+ **/
+/*
+We want the profile code to not count towards fragment times.
+So we stop time as quickly as possible, in assembly here instead of
+in the profile_fragment_enter function, and start time again as late
+as possible:
+    mov %eax, eax_offset(dcontext) # save eax
+    mov %edx, edx_offset(dcontext) # save edx
+    rdtsc                          # stop time
+    switch to dynamo stack         
+    pushfl                         # save eflags (call will clobber)
+    mov %ecx, ecx_offset(dcontext) # save ecx
+    pushl %edx                     # pass time as arg
+    pushl %eax                    
+    pushil &fragment_address       # pass &frag as arg
+    call profile_fragment_enter    # 
+    addl $0xc, %esp                # clean up args
+    popl %ecx                      # restore ecx 
+    popfl                          # restore eflags
+    restore app stack     
+    rdtsc                          # start time
+    movl %eax, start_time_OFFS(dcontext)   # store time value 
+    movl %edx, 4+start_time_OFFS(dcontext) # store time value
+    mov eax_offset(dcontext), %eax   # restore eax
+    mov edx_offset(dcontext), %edx   # restore edx 
+    mov ecx_offset(dcontext), %ecx   # restore ecx 
+*/
+
+static uint profile_call_length = 0;
+static int profile_call_fragment_offset = 0;
+static int profile_call_call_offset = 0;
+static byte profile_call_buf[128];
+static dcontext_t *buffer_dcontext;
+static void build_profile_call_buffer(void);
+
+uint
+profile_call_size()
+{
+    if (profile_call_length == 0)
+        build_profile_call_buffer();
+    return profile_call_length;
+}
+
+/* if insert_profile_call emits its code into the trace buffer, this
+ * routine must be called once the fragment is created and the code is
+ * in the fcache
+ */
+void
+finalize_profile_call(dcontext_t *dcontext, fragment_t *f)
+{
+    byte *start_pc = (byte *) FCACHE_ENTRY_PC(f);
+    byte *pc;
+    byte *prev_pc;
+    instr_t instr;
+    instr_init(dcontext, &instr);
+
+    /* fill in address of owning fragment now that that fragment exists */
+    pc = start_pc + profile_call_fragment_offset;
+    /* PR 248210: unsupported feature on x64 */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    *((int *)pc) = (uint)f;
+
+    /* fill in call's proper pc-relative offset now that code is
+     * in its final location in fcache
+     */
+    pc = start_pc + profile_call_call_offset;
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    *((int *)pc) = (int)&profile_fragment_enter - (int)pc - 4;
+
+    /* must fix up all dcontext references to point to the right dcontext */
+    pc = start_pc;
+    do {
+        prev_pc = pc;
+        instr_reset(dcontext, &instr);
+        pc = decode(dcontext, pc, &instr);
+        ASSERT(instr_valid(&instr)); /* our own code! */
+        /* look for loads and stores that reference buffer_dcontext */
+        if (instr_get_opcode(&instr) == OP_mov_ld &&
+            opnd_is_near_base_disp(instr_get_src(&instr, 0)) &&
+            opnd_get_base(instr_get_src(&instr, 0)) == REG_NULL &&
+            opnd_get_index(instr_get_src(&instr, 0)) == REG_NULL) {
+            /* if not really dcontext value, update_ will return old value */
+            instr_set_src(&instr, 0, 
+                          update_dcontext_address(instr_get_src(&instr, 0),
+                                                  buffer_dcontext, dcontext));
+        }
+        else if (instr_get_opcode(&instr) == OP_mov_st &&
+                 opnd_is_near_base_disp(instr_get_dst(&instr, 0)) &&
+                 opnd_get_base(instr_get_dst(&instr, 0)) == REG_NULL &&
+                 opnd_get_index(instr_get_dst(&instr, 0)) == REG_NULL) {
+            /* if not really dcontext value, update_ will return old value */
+            instr_set_dst(&instr, 0, 
+                          update_dcontext_address(instr_get_dst(&instr, 0),
+                                                  buffer_dcontext, dcontext));
+        }
+        if (!instr_raw_bits_valid(&instr)) {
+            DEBUG_DECLARE(byte *nxt_pc;)
+            DEBUG_DECLARE(nxt_pc = ) instr_encode(dcontext, &instr, prev_pc);
+            ASSERT(nxt_pc != NULL);
+        }
+    } while (pc < start_pc + profile_call_length);
+    instr_free(dcontext, &instr);
+}
+
+
+void
+insert_profile_call(cache_pc start_pc)
+{
+    if (profile_call_length == 0)
+        build_profile_call_buffer();
+    memcpy((void *)start_pc, profile_call_buf, profile_call_length);
+    /* if thread-private, we change to proper dcontext when finalizing */
+}
+
+
+/* This routine builds the profile call code using the instr_t
+ * abstraction, then emits it into a buffer to be saved.
+ * The code can then be directly copied whenever needed.
+ * Assumption: this thread's dcontext must have been created
+ * before calling this function.
+ */
+static void
+build_profile_call_buffer()
+{
+    byte *pc, *nxt_pc;
+    instrlist_t ilist;
+    instr_t *inst;
+    int start_time_offs;
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    ASSERT(dcontext != NULL);
+    /* remember dcontext for easy replacement when finalizing: */
+    buffer_dcontext = dcontext;
+
+    /* we require a dcontext to find this offset because it may
+     * or may not be pushed to a quadword boundary, making it
+     * hard to hardcode it
+     */
+    start_time_offs = (int)(&(dcontext->start_time)) - (int)dcontext;
+
+    /* initialize the ilist */
+    instrlist_init(&ilist);
+
+    APP(&ilist, instr_create_save_to_dcontext(dcontext, REG_EAX, XAX_OFFSET));
+    APP(&ilist, instr_create_save_to_dcontext(dcontext, REG_EDX, XDX_OFFSET));
+
+    /* get time = rdtsc */
+    APP(&ilist, INSTR_CREATE_rdtsc(dcontext));
+
+    /* swap to dstack */
+    APP(&ilist, instr_create_save_to_dcontext(dcontext, REG_ESP, XSP_OFFSET));
+    APP(&ilist, instr_create_restore_dynamo_stack(dcontext));
+
+    /* finish saving caller-saved registers
+     * The profile_fragment_enter function will save the callee-saved
+     * regs (ebx, ebp, esi, edi) and will restore ebp and esp, but we need
+     * to explicitly save eax, ecx, and edx
+     */
+    APP(&ilist, instr_create_save_to_dcontext(dcontext, REG_ECX, XCX_OFFSET));
+
+    /* save eflags (call will clobber) */
+    APP(&ilist, INSTR_CREATE_RAW_pushf(dcontext));
+
+#ifdef WINDOWS
+    /* must preserve the LastErrorCode (if the profile procedure
+     * calls a Win32 API routine it could overwrite the app's error code)
+     * currently this is done in the profile routine itself --
+     * if you want to move it here, look at the code in profile.c
+     */
+#endif
+
+    /* push time as 2nd argument for call */
+    APP(&ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_EDX)));
+    APP(&ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_EAX)));
+
+    /* push fragment address as 1st argument for call
+     * fragment isn't built yet, we fill it in in finalize_profile_call 
+     */
+    APP(&ilist, INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32(0)));
+
+    /* call near rel: 4-byte pc-relative offset from start of next instr
+     * we don't have that offset now so we fill it in later (in
+     * finalize_profile_call)
+     */
+    APP(&ilist, INSTR_CREATE_call(dcontext, opnd_create_pc(NULL)));
+
+    /* pop arguments: addl $0xc, %esp */
+    APP(&ilist,
+        INSTR_CREATE_add(dcontext, opnd_create_reg(REG_ESP), OPND_CREATE_INT8(0xc)));
+
+    /* restore eflags */
+    APP(&ilist, INSTR_CREATE_RAW_popf(dcontext));
+
+    /* restore caller-saved registers */
+    APP(&ilist, instr_create_restore_from_dcontext(dcontext, REG_ECX, XCX_OFFSET));
+
+    /* restore app stack */
+    APP(&ilist, instr_create_restore_from_dcontext(dcontext, REG_ESP, XSP_OFFSET));
+
+    /* get start time = rdtsc */
+    APP(&ilist, INSTR_CREATE_rdtsc(dcontext));
+
+    /* copy start time into dcontext */
+    APP(&ilist, instr_create_save_to_dcontext(dcontext, REG_EAX, start_time_offs));
+    APP(&ilist, instr_create_save_to_dcontext(dcontext, REG_EDX, start_time_offs+4));
+
+    /* finish restoring caller-saved registers */
+    APP(&ilist, instr_create_restore_from_dcontext(dcontext, REG_EDX, XDX_OFFSET));
+    APP(&ilist, instr_create_restore_from_dcontext(dcontext, REG_EAX, XAX_OFFSET));
+
+    /* now encode the instructions */
+    pc = profile_call_buf;
+    for (inst = instrlist_first(&ilist); inst; inst = instr_get_next(inst)) {
+        if (instr_is_call_direct(inst)) {
+            /* push_immed was just before us, so fragment address
+             * starts 4 bytes before us:
+             */
+            profile_call_fragment_offset = (int) (pc - 4 - profile_call_buf);
+            /* call opcode is 1 byte, offset is next: */
+            profile_call_call_offset = (int) (pc + 1 - profile_call_buf);
+        }
+        /* we have no jumps with instr_t targets so we don't need to set note
+         * field in order to use instr_encode
+         */
+        nxt_pc = instr_encode(dcontext, inst, (void*)pc);
+        ASSERT(nxt_pc != NULL);
+        profile_call_length += nxt_pc - pc;
+        pc = nxt_pc;
+        ASSERT(profile_call_length < 128);
+    }
+
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+}
+
+#endif /* PROFILE_RDTSC */
+
+#ifdef WINDOWS
+/* LastError preservation routines, used in fcache_enter/return and shared_syscall */
+
+/* inlined versions of save/restore last error by reading of TIB */
+/* If our inlined version fails on a later version of windows 
+   should verify [GS]etLastError matches the disassembly below.
+*/
+void
+preinsert_set_last_error(dcontext_t *dcontext, instrlist_t *ilist, instr_t *next,
+                         reg_id_t reg_errno /* saved errno */)
+{
+/* Win2000: kernel32!SetLastError: */
+/*   77E87671: 55                 push        ebp */
+/*   77E87672: 8B EC              mov         ebp,esp */
+/*   77E87674: 64 A1 18 00 00 00  mov         eax,fs:[00000018] */
+/*   77E8767A: 8B 4D 08           mov         ecx,dword ptr [ebp+8] */
+/*   77E8767D: 89 48 34           mov         dword ptr [eax+34h],ecx */
+/*   77E87680: 5D                 pop         ebp */
+/*   77E87681: C2 04 00           ret         4 */
+
+/* Win2003: ntdll!RtlSetLastWin32Error: optimized to */
+/*   77F45BB4: 64 A1 18 00 00 00  mov         eax,fs:[00000018] */
+/*   77F45BBA: 8B 4C 24 04        mov         ecx,dword ptr [esp+4] */
+/*   77F45BBE: 89 48 34           mov         dword ptr [eax+34h],ecx */
+/*   77F45BC1: C2 04 00           ret         4 */
+
+    /* See InsideWin2k, p. 329 SelfAddr fs:[18h] simply has the linear address of the TIB 
+       while we're interested only in LastError which is at fs:[34h] */
+
+    /* Therefore all we need is a single instruction! */
+    /* 64 a1 34 00 00 00  mov         dword ptr fs:[34h],errno_register */
+
+    /* Overall savings: 7 instructions, 5 data words */
+    /* Given the opportunity for optimization of the above code
+       we should definitely do it if this is on the critical path */
+
+    PRE(ilist, next,
+        INSTR_CREATE_mov_st(dcontext,
+                            opnd_create_far_base_disp(SEG_TLS, REG_NULL, REG_NULL, 0, 
+                                                      ERRNO_TIB_OFFSET, OPSZ_4),
+                            opnd_create_reg(reg_errno)));
+}
+
+void
+preinsert_get_last_error(dcontext_t *dcontext, instrlist_t *ilist, instr_t *next,
+                         reg_id_t reg_result /* gets errno */)
+{
+    /*kernel32!GetLastError:*/
+    /*   77E87684: 64 A1 18 00 00 00  mov         eax,fs:[00000018] */
+    /*   77E8768A: 8B 40 34           mov         eax,dword ptr [eax+34h] */
+    /*   77E8768D: C3                 ret */
+    /* see comments in preinsert_set_last_error */
+
+    /* All we need is a single instruction: */
+    /*  77F45BBE: 89 48 34           mov         reg_result, dword ptr fs:[34h] */
+
+    PRE(ilist, next, 
+        INSTR_CREATE_mov_ld(dcontext,
+                            opnd_create_reg(reg_result),
+                            opnd_create_far_base_disp(SEG_TLS, REG_NULL, REG_NULL, 0, 
+                                                      ERRNO_TIB_OFFSET, OPSZ_4)
+                            ));
+}
+#endif /* WINDOWS */
+
+
+/***************************************************************************/
+/*             THREAD-PRIVATE/SHARED ROUTINE GENERATION                    */
+/***************************************************************************/
+
+/* macros shared by fcache_enter and fcache_return
+ * in order to generate both thread-private code that uses absolute
+ * addressing and thread-shared or dcontext-shared code that uses
+ * xdi (and xsi) for addressing.
+ * The via_reg macros now auto-magically pick the opnd size from the
+ * target register and so work with more than just pointer-sized values.
+ */
+/* PR 244737: even thread-private fragments use TLS on x64.  We accomplish
+ * that at the caller site, so we should never see an "absolute" request.
+ */
+#define RESTORE_FROM_DC(dc, reg, offs) \
+    ((absolute) ? (IF_X64_(ASSERT_NOT_IMPLEMENTED(false))                     \
+                   instr_create_restore_from_dcontext((dc), (reg), (offs))) : \
+     instr_create_restore_from_dc_via_reg((dc), REG_NULL, (reg), (offs)))
+/* Note the magic absolute boolean that callers are expected to have declared */
+#define SAVE_TO_DC(dc, reg, offs) \
+    ((absolute) ? (IF_X64_(ASSERT_NOT_IMPLEMENTED(false))                \
+                   instr_create_save_to_dcontext((dc), (reg), (offs))) : \
+     instr_create_save_to_dc_via_reg((dc), REG_NULL, (reg), (offs)))
+
+#define OPND_TLS_FIELD(offs) opnd_create_tls_slot(os_tls_offset(offs))
+
+#define OPND_TLS_FIELD_SZ(offs, sz) \
+    opnd_create_sized_tls_slot(os_tls_offset(offs), sz)
+
+#define SAVE_TO_TLS(dc, reg, offs) \
+    instr_create_save_to_tls(dc, reg, offs)
+#define RESTORE_FROM_TLS(dc, reg, offs) \
+    instr_create_restore_from_tls(dc, reg, offs)
+
+#define OPND_DC_FIELD(absolute, dcontext, sz, offs) \
+    ((absolute) ? (IF_X64_(ASSERT_NOT_IMPLEMENTED(false))                \
+                   opnd_create_dcontext_field_sz(dcontext, (offs), (sz))) : \
+     opnd_create_dcontext_field_via_reg_sz((dcontext), REG_NULL, (offs), (sz)))
+
+/* Export this in instr.h if it becomes useful elsewhere */
+#ifdef X64
+# ifdef WINDOWS
+#  define OPND_ARG1  opnd_create_reg(REG_RCX)
+# else
+#  define OPND_ARG1  opnd_create_reg(REG_RDI)
+# endif
+#else
+# define OPND_ARG1   OPND_CREATE_MEM32(REG_ESP, 4)
+#endif
+
+/* Our context switch to and from the fragment cache are arranged such
+ * that there is no persistent state kept on the dstack, allowing us to
+ * start with a clean slate on exiting the cache.  This eliminates the
+ * need to protect our dstack from inadvertent or malicious writes.
+ *
+ * We do not bother to save any DynamoRIO state, even the eflags.  We clear
+ * them in fcache_return, assuming that a cleared state is always the
+ * proper value (df is never set across the cache, etc.)
+ *
+ * fcache_enter(dcontext_t *dcontext)
+ *   Used by dispatch to begin execution in fcache at dcontext->next_tag
+
+    if (!absolute)
+        mov    ARG1,%xdi # dcontext param
+      if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask))
+        RESTORE_FROM_UPCONTEXT PROT_OFFSET,%xsi
+      endif
+    endif  
+
+        # restore app's error code (32 bits even on x64)
+    .ifdef WINDOWS
+        RESTORE_FROM_DCONTEXT app_errno_OFFSET,%eax
+        mov     %eax, fs:ERRNO_TIB_OFFSET
+    .else
+        RESTORE_FROM_DCONTEXT app_errno_OFFSET,%eax
+        SAVE_TO_DCONTEXT %eax,errno_OFFSET
+    .endif
+
+    if (!absolute)
+        # put target somewhere we can be absolute about
+        RESTORE_FROM_UPCONTEXT next_tag_OFFSET,%xax
+      if (shared)
+        mov  %xax,fs:xax_OFFSET
+      endif
+    endif
+
+    if (EXIT_DR_HOOK != NULL && !dcontext->ignore_enterexit)
+      if (!absolute)
+        push    %xdi
+        push    %xsi
+      else
+        # support for skipping the hook
+        RESTORE_FROM_UPCONTEXT ignore_enterexit_OFFSET,%edi
+        cmpl    %edi,0
+        jnz     post_hook
+      endif
+        call    EXIT_DR_HOOK # for x64 windows, reserve 32 bytes stack space for call
+      if (!absolute)
+        pop    %xsi
+        pop    %xdi
+      endif
+    endif
+
+    post_hook:        
+
+        # restore the original register state
+        RESTORE_FROM_UPCONTEXT xflags_OFFSET,%xax
+        push    %xax
+        popf            # restore eflags temporarily using dstack
+    if preserve_xmm_caller_saved
+        RESTORE_FROM_UPCONTEXT xmm_OFFSET+0*16,%xmm0
+        RESTORE_FROM_UPCONTEXT xmm_OFFSET+1*16,%xmm1
+        RESTORE_FROM_UPCONTEXT xmm_OFFSET+2*16,%xmm2
+        RESTORE_FROM_UPCONTEXT xmm_OFFSET+3*16,%xmm3
+        RESTORE_FROM_UPCONTEXT xmm_OFFSET+4*16,%xmm4
+        RESTORE_FROM_UPCONTEXT xmm_OFFSET+5*16,%xmm5
+    endif
+    ifdef X64
+        RESTORE_FROM_UPCONTEXT r8_OFFSET,%r8
+        RESTORE_FROM_UPCONTEXT r9_OFFSET,%r9
+        RESTORE_FROM_UPCONTEXT r10_OFFSET,%r10
+        RESTORE_FROM_UPCONTEXT r11_OFFSET,%r11
+        RESTORE_FROM_UPCONTEXT r12_OFFSET,%r12
+        RESTORE_FROM_UPCONTEXT r13_OFFSET,%r13
+        RESTORE_FROM_UPCONTEXT r14_OFFSET,%r14
+        RESTORE_FROM_UPCONTEXT r15_OFFSET,%r15
+    endif
+        RESTORE_FROM_UPCONTEXT xax_OFFSET,%xax
+        RESTORE_FROM_UPCONTEXT xbx_OFFSET,%xbx
+        RESTORE_FROM_UPCONTEXT xcx_OFFSET,%xcx
+        RESTORE_FROM_UPCONTEXT xdx_OFFSET,%xdx
+    if (absolute || !TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask))
+        RESTORE_FROM_UPCONTEXT xsi_OFFSET,%xsi
+    endif
+    if (absolute || TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask))
+        RESTORE_FROM_UPCONTEXT xdi_OFFSET,%xdi
+    endif
+        RESTORE_FROM_UPCONTEXT xbp_OFFSET,%xbp
+        RESTORE_FROM_UPCONTEXT xsp_OFFSET,%xsp
+    if (!absolute)
+      if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask))
+        RESTORE_FROM_UPCONTEXT xsi_OFFSET,%xsi
+      else
+        RESTORE_FROM_UPCONTEXT xdi_OFFSET,%xdi
+      endif
+    endif
+
+    ifdef X64 and (target is x86 mode)    
+        # we can't indirect through a register since we couldn't restore
+        # the high bits (PR 283152)
+        mov gencode-jmp86-value, fs:xbx_OFFSET
+        far jmp to next instr, stored w/ 32-bit cs selector in fs:xbx_OFFSET
+    endif
+
+        # jump indirect through dcontext->next_tag, set by dispatch()
+    if (absolute)    
+        JUMP_VIA_DCONTEXT next_tag_OFFSET
+    else
+      if (shared)
+        jmp *fs:xax_OFFSET
+      else
+        JUMP_VIA_DCONTEXT nonswapped_scratch_OFFSET
+      endif
+    endif
+
+        # now executing in fcache
+ */
+static byte *
+emit_fcache_enter_common(dcontext_t *dcontext, generated_code_t *code, byte *pc,
+                         bool absolute, bool shared)
+{
+    int len;
+    instrlist_t ilist;
+    patch_list_t patch;
+#ifdef X64
+    byte *jmp86_store_addr = NULL;
+    byte *jmp86_target_addr = NULL;
+#endif
+    instr_t *post_hook = INSTR_CREATE_label(dcontext);
+
+    init_patch_list(&patch, absolute ? PATCH_TYPE_ABSOLUTE : PATCH_TYPE_INDIRECT_XDI);
+    instrlist_init(&ilist);
+
+    /* no support for absolute addresses on x64: we always use tls */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(!absolute && shared));
+
+    if (!absolute) {
+        /* grab gen routine's parameter dcontext and put it into edi */
+        APP(&ilist, INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XDI), OPND_ARG1));
+        if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask))
+            APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XSI, PROT_OFFS));
+    }
+
+    /* restore app's error code */
+#ifdef WINDOWS
+    APP(&ilist, RESTORE_FROM_DC(dcontext, REG_EAX, APP_ERRNO_OFFSET));
+    preinsert_set_last_error(dcontext, &ilist, NULL, REG_EAX);
+#else
+    APP(&ilist, RESTORE_FROM_DC(dcontext, REG_EAX, APP_ERRNO_OFFSET));
+    APP(&ilist, SAVE_TO_DC(dcontext, REG_EAX, ERRNO_OFFSET));
+#endif
+        
+    if (!absolute) {
+        /* put target into special slot that we can be absolute about */
+        APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XAX, NEXT_TAG_OFFSET));
+        if (shared) {
+            APP(&ilist, SAVE_TO_TLS(dcontext, REG_XAX, FCACHE_ENTER_TARGET_SLOT));
+        } else {
+#ifdef WINDOWS
+            /* absolute into main dcontext (not one in edi) */
+            APP(&ilist, instr_create_save_to_dcontext(dcontext, REG_XAX,
+                                                     NONSWAPPED_SCRATCH_OFFSET));
+#else
+            /* no special scratch slot! */
+            ASSERT_NOT_IMPLEMENTED(false);
+#endif
+        }
+    }
+
+    if (EXIT_DR_HOOK != NULL) {
+        /* if absolute, don't bother to save any regs around the call */ 
+        if (!absolute) {
+            /* save xdi and xsi around call.
+             * for x64, they're supposed to be callee-saved on windows,
+             * but not linux (though we could move to r12-r15 on linux
+             * instead of pushing them).
+             */
+            APP(&ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XDI)));
+            APP(&ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XSI)));
+        } 
+#ifdef WINDOWS
+        else {
+            /* For thread-private (used for syscalls), don't call if
+             * dcontext->ignore_enterexit.  This is a perf hit to check: could
+             * instead have a space hit via a separate routine.  This is only
+             * needed right now for NtSuspendThread handling (see case 4942).
+             */
+            APP(&ilist, RESTORE_FROM_DC(dcontext, REG_EDI, IGNORE_ENTEREXIT_OFFSET));
+            /* P4 opt guide says to use test to cmp reg with 0: shorter instr */
+            APP(&ilist, INSTR_CREATE_test(dcontext, opnd_create_reg(REG_EDI),
+                                         opnd_create_reg(REG_EDI)));
+            APP(&ilist, INSTR_CREATE_jcc(dcontext, OP_jnz, opnd_create_instr(post_hook)));
+        }
+#endif
+        /* make sure to use dr_insert_call() rather than a raw OP_call instr,
+         * since x64 windows requires 32 bytes of stack space even w/ no args,
+         * and we don't want anyone clobbering our pushed registers!
+         */
+        dr_insert_call((void *)dcontext, &ilist, NULL/*append*/, (void *)EXIT_DR_HOOK, 0);
+        if (!absolute) {
+            /* save edi and esi around call */
+            APP(&ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_XSI)));
+            APP(&ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_XDI)));
+        }
+    }
+
+    /* restore the original register state */
+
+    APP(&ilist, post_hook/*label*/);
+    APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XAX, XFLAGS_OFFSET));
+    APP(&ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XAX)));
+    /* restore eflags temporarily using dstack */
+    APP(&ilist, INSTR_CREATE_RAW_popf(dcontext));
+    if (preserve_xmm_caller_saved()) {
+        /* PR 264138: we must preserve xmm0-5 if on a 64-bit kernel.
+         * Rather than try and optimize we save/restore on every cxt
+         * sw.  The xmm field is aligned, so we can use movdqa/movaps,
+         * though movdqu is stated to be as fast as movdqa when aligned:
+         * but if so, why have two versions?  Is it only loads and not stores
+         * for which that is true?  => PR 266305.
+         * It's not clear that movdqa is any faster (and its opcode is longer):
+         * movdqa and movaps are listed as the same latency and throughput in
+         * the AMD optimization guide.  Yet examples of fast memcpy online seem
+         * to use movdqa when sse2 is available.
+         * Note that mov[au]p[sd] and movdq[au] are functionally equivalent.
+         */
+        int i;
+        uint opcode = (proc_has_feature(FEATURE_SSE2) ? OP_movdqa : OP_movaps);
+        ASSERT(proc_has_feature(FEATURE_SSE));
+        for (i=0; i<NUM_XMM_SAVED; i++) {
+            APP(&ilist, instr_create_1dst_1src
+                (dcontext, opcode, opnd_create_reg(REG_XMM0 + (reg_id_t)i),
+                 OPND_DC_FIELD(absolute, dcontext, OPSZ_16, XMM_OFFSET + i*XMM_REG_SIZE)));
+        }
+    }
+#ifdef X64
+    APP(&ilist, RESTORE_FROM_DC(dcontext, REG_R8, R8_OFFSET));
+    APP(&ilist, RESTORE_FROM_DC(dcontext, REG_R9, R9_OFFSET));
+    APP(&ilist, RESTORE_FROM_DC(dcontext, REG_R10, R10_OFFSET));
+    APP(&ilist, RESTORE_FROM_DC(dcontext, REG_R11, R11_OFFSET));
+    APP(&ilist, RESTORE_FROM_DC(dcontext, REG_R12, R12_OFFSET));
+    APP(&ilist, RESTORE_FROM_DC(dcontext, REG_R13, R13_OFFSET));
+    APP(&ilist, RESTORE_FROM_DC(dcontext, REG_R14, R14_OFFSET));
+    APP(&ilist, RESTORE_FROM_DC(dcontext, REG_R15, R15_OFFSET));
+#endif
+    APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XAX, XAX_OFFSET));
+    APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XBX, XBX_OFFSET));
+    APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XCX, XCX_OFFSET));
+    APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XDX, XDX_OFFSET));
+    /* must restore esi last */
+    if (absolute || !TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask))
+        APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XSI, XSI_OFFSET));
+    /* must restore edi last */
+    if (absolute || TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask))
+        APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XDI, XDI_OFFSET));
+    APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XBP, XBP_OFFSET));
+    APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XSP, XSP_OFFSET));
+    /* must restore esi last */
+    if (!absolute) {
+        if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask))
+            APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XSI, XSI_OFFSET));
+        else
+            APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XDI, XDI_OFFSET));
+    }
+
+#ifdef X64
+    if (code->x86_mode) {
+        instr_t *label = INSTR_CREATE_label(dcontext);
+        instr_t *store;
+        /* We must use an indirect jmp (far direct are illegal in x64) and
+         * we can't indirect through a register since we couldn't restore the
+         * high bits (PR 283152) so we write the 6-byte far address to TLS.
+         */
+        APP(&ilist, INSTR_CREATE_mov_st(dcontext,
+                                        OPND_TLS_FIELD_SZ(TLS_XBX_SLOT, OPSZ_2),
+                                        OPND_CREATE_INT16((ushort)CS32_SELECTOR)));
+        /* AMD only supports 32-bit address for far jmp */
+        store = INSTR_CREATE_mov_st(dcontext,
+                                    OPND_TLS_FIELD_SZ(TLS_XBX_SLOT+2, OPSZ_4),
+                                    OPND_CREATE_INT32(0/*placeholder*/));
+        APP(&ilist, store);
+        APP(&ilist, INSTR_CREATE_jmp_far_ind(dcontext,
+                                             OPND_TLS_FIELD_SZ(TLS_XBX_SLOT, OPSZ_6)));
+        APP(&ilist, label);
+        /* We need a patch that involves two instrs, which is not supported,
+         * so we get both addresses involved into local vars and do the patch
+         * by hand after emitting.
+         */
+        add_patch_marker(&patch, store, PATCH_ASSEMBLE_ABSOLUTE,
+                         -4 /* 4 bytes from end */, (ptr_uint_t*)&jmp86_store_addr);
+        add_patch_marker(&patch, label, PATCH_ASSEMBLE_ABSOLUTE,
+                         0 /* start of label */, (ptr_uint_t*)&jmp86_target_addr);
+    }
+#endif
+
+    /* Jump indirect through next_tag.  Dispatch set this value with
+     * where we want to go next in the fcache_t.
+     */
+    if (absolute) {
+        APP(&ilist, instr_create_jump_via_dcontext(dcontext, NEXT_TAG_OFFSET));
+    } else {
+        if (shared) {
+            /* next_tag placed into tls slot earlier in this routine */
+            APP(&ilist, INSTR_CREATE_jmp_ind(dcontext, 
+                                             OPND_TLS_FIELD(FCACHE_ENTER_TARGET_SLOT)));
+
+        } else {
+#ifdef WINDOWS
+            /* FIXME: we could just use tls, right?  no real need for the "shared"
+             * parameter? 
+             */
+            /* need one absolute ref using main dcontext (not one in edi):
+             * it's the final jmp, using the special slot we set up earlier 
+             */
+            APP(&ilist, instr_create_jump_via_dcontext(dcontext,
+                                                      NONSWAPPED_SCRATCH_OFFSET));
+#else
+            /* no special scratch slot! */
+            ASSERT_NOT_IMPLEMENTED(false);
+#endif
+        }
+    }
+
+    /* now encode the instructions */
+    len = encode_with_patch_list(dcontext, &patch, &ilist, pc);
+    ASSERT(len != 0);
+
+#ifdef X64
+    if (code->x86_mode) {
+        /* Put the absolute address in place */
+        ASSERT(jmp86_target_addr != NULL && jmp86_store_addr != NULL);
+        ASSERT(CHECK_TRUNCATE_TYPE_uint((ptr_uint_t)jmp86_target_addr));
+        *((uint *)jmp86_store_addr) = (uint)(ptr_uint_t)jmp86_target_addr;
+    }
+#endif
+
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+
+    return pc + len;
+}
+
+byte *
+emit_fcache_enter(dcontext_t *dcontext, generated_code_t *code, byte *pc)
+{
+    return emit_fcache_enter_common(dcontext, code, pc,
+                                    true/*absolute*/, false/*!shared*/);
+}
+
+/* Generate a shared prologue for grabbing the dcontext into XDI 
+
+   TODO: Should be used by fcache_return and shared IBL routines,
+   but for now some assumptions are not quite the same.
+
+   Only assumption is that xcx cannot be touched (IBL expects looked up address)
+       if save_xdi we assume DCONTEXT_BASE_SPILL_SLOT can be clobbered
+
+   OUTPUT: xdi contains dcontext
+       if save_xdi DCONTEXT_BASE_SPILL_SLOT will contain saved value
+   FIXME: xdx is the spill slot -- switch over to xdx as base reg?
+       Have to measure perf effect first (case 5239)
+
+    00:   mov xdi, tls_slot_scratch2   64 89 3d 0c 0f 00 00 mov    %edi -> %fs:0xf0c 
+    07:   mov tls_slot_dcontext, xdi   64 8b 3d 14 0f 00 00 mov    %fs:0xf14 -> %edi 
+  if TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)
+     ASSERT_NOT_TESTED
+  endif
+*/
+void
+append_shared_get_dcontext(dcontext_t *dcontext, instrlist_t *ilist, bool save_xdi)
+{
+    /* needed to support grabbing the dcontext w/ shared cache */
+    if (save_xdi) {
+        APP(ilist, SAVE_TO_TLS(dcontext, REG_XDI, DCONTEXT_BASE_SPILL_SLOT));
+    }
+    APP(ilist, RESTORE_FROM_TLS(dcontext, REG_XDI, TLS_DCONTEXT_SLOT));
+    if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)) {
+        bool absolute = false;
+        /* PR 224798: we could avoid extra indirection by storing
+         * unprotected_context_t in TLS_DCONTEXT_SLOT instead of dcontext_t
+         */
+        ASSERT_NOT_TESTED();
+        /* we'd need a 3rd slot in order to nicely get unprot ptr into esi
+         * we can do it w/ only 2 slots by clobbering dcontext ptr
+         * (we could add base reg info to RESTORE_FROM_DC/SAVE_TO_DC and go
+         * straight through esi to begin w/ and subtract one instr (xchg)
+         */
+        APP(ilist, RESTORE_FROM_DC(dcontext, REG_XDI, PROT_OFFS));
+        APP(ilist, INSTR_CREATE_xchg(dcontext, opnd_create_reg(REG_XSI),
+                                     opnd_create_reg(REG_XDI)));
+        APP(ilist, SAVE_TO_DC(dcontext, REG_XDI, XSI_OFFSET));
+        APP(ilist, RESTORE_FROM_TLS(dcontext, REG_XDI, TLS_DCONTEXT_SLOT));
+    }
+}
+
+
+/* restore XDI through TLS */
+void
+append_shared_restore_dcontext_reg(dcontext_t *dcontext, instrlist_t *ilist)
+{
+    APP(ilist, RESTORE_FROM_TLS(dcontext, REG_XDI, DCONTEXT_BASE_SPILL_SLOT));
+}
+
+/*
+# fcache_return: context switch back to DynamoRIO.
+# Invoked via
+#     a) from the fcache via a fragment exit stub,
+#     b) from indirect_branch_lookup().
+# Invokes dispatch() with a clean dstack.
+# Assumptions:
+#     1) app's value in xax already saved in dcontext.
+#     2) xax holds the linkstub ptr 
+
+    ifdef X64 and (source is x86 mode)    
+        far direct jmp to next instr w/ 64-bit switch
+    endif
+
+    if (!absolute)
+        mov  %xdi,fs:xdx_OFFSET
+        mov  fs:dcontext,%xdi
+      if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask))
+        RESTORE_FROM_DCONTEXT PROT_OFFSET,%xdi
+        xchg   %xsi,%xdi
+        SAVE_TO_UPCONTEXT %xdi,xsi_OFFSET
+        mov    fs:dcontext,%xdi
+      endif
+        # get xax and xdi into their real slots, via xbx
+        SAVE_TO_UPCONTEXT %xbx,xbx_OFFSET
+        mov    fs:xax_OFFSET,%xbx
+        SAVE_TO_UPCONTEXT %xbx,xax_OFFSET
+        mov    fs:xdx_OFFSET,%xbx
+        SAVE_TO_UPCONTEXT %xbx,xdi_OFFSET
+    endif
+
+        # save the current register state to context->regs
+        # xax already in context
+
+    if (absolute)
+        SAVE_TO_UPCONTEXT %xbx,xbx_OFFSET
+    endif
+        SAVE_TO_UPCONTEXT %xcx,xcx_OFFSET
+        SAVE_TO_UPCONTEXT %xdx,xdx_OFFSET
+    if (absolute || !TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask))
+        SAVE_TO_UPCONTEXT %xsi,xsi_OFFSET
+    endif
+    if (absolute)
+        SAVE_TO_UPCONTEXT %xdi,xdi_OFFSET
+    endif
+        SAVE_TO_UPCONTEXT %xbp,xbp_OFFSET
+        SAVE_TO_UPCONTEXT %xsp,xsp_OFFSET
+    ifdef X64
+        SAVE_TO_UPCONTEXT %r8,r8_OFFSET
+        SAVE_TO_UPCONTEXT %r9,r9_OFFSET
+        SAVE_TO_UPCONTEXT %r10,r10_OFFSET
+        SAVE_TO_UPCONTEXT %r11,r11_OFFSET
+        SAVE_TO_UPCONTEXT %r12,r12_OFFSET
+        SAVE_TO_UPCONTEXT %r13,r13_OFFSET
+        SAVE_TO_UPCONTEXT %r14,r14_OFFSET
+        SAVE_TO_UPCONTEXT %r15,r15_OFFSET
+    endif
+    if preserve_xmm_caller_saved
+        SAVE_TO_UPCONTEXT %xmm0,xmm_OFFSET+0*16
+        SAVE_TO_UPCONTEXT %xmm1,xmm_OFFSET+1*16
+        SAVE_TO_UPCONTEXT %xmm2,xmm_OFFSET+2*16
+        SAVE_TO_UPCONTEXT %xmm3,xmm_OFFSET+3*16
+        SAVE_TO_UPCONTEXT %xmm4,xmm_OFFSET+4*16
+        SAVE_TO_UPCONTEXT %xmm5,xmm_OFFSET+5*16
+    endif
+
+        # switch to clean dstack
+        RESTORE_FROM_DCONTEXT dstack_OFFSET,%xsp
+
+        # now save eflags -- too hard to do without a stack!
+        pushf           # push eflags on stack
+        pop     %xbx    # grab eflags value
+        SAVE_TO_UPCONTEXT %xbx,xflags_OFFSET # save eflags value
+
+        # clear eflags now to avoid app's eflags messing up our ENTER_DR_HOOK
+        # FIXME: this won't work at CPL0 if we ever run there!
+        push  0
+        popf
+
+    if (ENTER_DR_HOOK != NULL && !dcontext->ignore_enterexit)
+        # don't bother to save any registers around call except for xax
+        # and xcx, which holds next_tag
+        push    %xcx
+      if (!absolute)
+        push    %xdi
+        push    %xsi
+      endif
+        push    %xax
+      if (absolute)
+        # support for skipping the hook (note: 32-bits even on x64)
+        RESTORE_FROM_UPCONTEXT ignore_enterexit_OFFSET,%edi
+        cmp     %edi,0
+        jnz     post_hook
+      endif
+        # for x64 windows, reserve 32 bytes stack space for call prior to call
+        call    ENTER_DR_HOOK
+    post_hook:
+        pop     %xax
+      if (!absolute)
+        pop     %xsi
+        pop     %xdi
+      endif
+        pop     %xcx
+    endif
+        
+        # save last_exit, currently in eax, into dcontext->last_exit
+        SAVE_TO_DCONTEXT %xax,last_exit_OFFSET
+
+    .ifdef WINDOWS
+        # save app's error code (ok to use stack, it's dynamo's)
+        movl    fs:ERRNO_TIB_OFFSET,%eax
+        SAVE_TO_DCONTEXT %eax,app_errno_OFFSET
+    .else
+        # copy shared errno into app storage slot
+        RESTORE_FROM_DCONTEXT errno_OFFSET,%ebx
+        SAVE_TO_DCONTEXT %ebx,app_errno_OFFSET
+    .endif
+
+    .ifdef SIDELINE
+        # clear cur-trace field so we don't think cur trace is still running
+        movl    $0, _sideline_trace
+    .endif
+
+        # call central dispatch routine w/ dcontext as an argument
+    if (absolute)
+        push    <dcontext>
+    else
+        push     %xdi  # for x64, mov %xdi, ARG1
+    endif
+        call    dispatch # for x64 windows, reserve 32 bytes stack space for call
+        # dispatch() shouldn't return!
+        jmp     unexpected_return
+*/
+/* N.B.: this routine is used to generate both the regular fcache_return
+ * and a slightly different copy that is used for the miss/unlinked paths
+ * for indirect_branch_lookup for self-protection.
+ * ibl_end should be true only for that end of the lookup routine.
+ *
+ * If linkstub != NULL, used for coarse fragments, this routine assumes that:
+ * - app xax is still in %xax
+ * - next target pc is in DIRECT_STUB_SPILL_SLOT tls
+ * - linkstub is the linkstub_t to pass back to dispatch
+ * - if coarse_info:
+ *   - app xcx is in MANGLE_XCX_SPILL_SLOT
+ *   - source coarse info is in %xcx
+ */
+static bool
+append_fcache_return_common(dcontext_t *dcontext, generated_code_t *code,
+                            instrlist_t *ilist, bool ibl_end,
+                            bool absolute, bool shared, linkstub_t *linkstub,
+                            bool coarse_info)
+{
+    bool instr_targets = false;
+
+    /* no support for absolute addresses on x64: we always use tls */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(!absolute && shared));
+
+    /* currently linkstub is only used for coarse-grain exits */
+    ASSERT(linkstub == NULL || !absolute);
+
+#ifdef X64
+    if (code->x86_mode) {
+        instr_t *label = INSTR_CREATE_label(dcontext);
+        APP(ilist, INSTR_CREATE_jmp_far
+            (dcontext, opnd_create_far_instr(CS64_SELECTOR, label)));
+        APP(ilist, label);
+    }
+#endif
+
+    if (!absolute) {
+        /* only support non-absolute w/ shared cache */
+        ASSERT_NOT_IMPLEMENTED(shared);
+        /* xax is in 1 scratch slot, so we have to use a 2nd scratch
+         * slot in order to get dcontext into xdi
+         */
+        APP(ilist, SAVE_TO_TLS(dcontext, REG_XDI, DCONTEXT_BASE_SPILL_SLOT));
+        APP(ilist, RESTORE_FROM_TLS(dcontext, REG_XDI, TLS_DCONTEXT_SLOT));
+        if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)) {
+            /* we'd need a 3rd slot in order to nicely get unprot ptr into xsi
+             * we can do it w/ only 2 slots by clobbering dcontext ptr
+             * (we could add base reg info to RESTORE_FROM_DC/SAVE_TO_DC and go
+             * straight through xsi to begin w/ and subtract one instr (xchg)
+             */
+            ASSERT_NOT_TESTED();
+            APP(ilist, RESTORE_FROM_DC(dcontext, REG_XDI, PROT_OFFS));
+            APP(ilist, INSTR_CREATE_xchg(dcontext, opnd_create_reg(REG_XSI),
+                                         opnd_create_reg(REG_XDI)));
+            APP(ilist, SAVE_TO_DC(dcontext, REG_XDI, XSI_OFFSET));
+            APP(ilist, RESTORE_FROM_TLS(dcontext, REG_XDI, TLS_DCONTEXT_SLOT));
+        }
+        /* get xax and xdi into their real slots, via xbx */
+        APP(ilist, SAVE_TO_DC(dcontext, REG_XBX, XBX_OFFSET));
+        APP(ilist, RESTORE_FROM_TLS(dcontext, REG_XBX, DIRECT_STUB_SPILL_SLOT));
+        if (linkstub != NULL) {
+            /* app xax is still in %xax, src info is in %xcx, while target pc
+             * is now in %xbx
+             */
+            APP(ilist, SAVE_TO_DC(dcontext, REG_XAX, XAX_OFFSET));
+            APP(ilist, SAVE_TO_DC(dcontext, REG_XBX, NEXT_TAG_OFFSET));
+            APP(ilist, INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XAX),
+                                            OPND_CREATE_INTPTR((ptr_int_t)linkstub)));
+            if (coarse_info) {
+                APP(ilist, SAVE_TO_DC(dcontext, REG_XCX, COARSE_DIR_EXIT_OFFSET));
+                APP(ilist, RESTORE_FROM_TLS(dcontext, REG_XCX, MANGLE_XCX_SPILL_SLOT));
+            }
+        } else {
+            APP(ilist, SAVE_TO_DC(dcontext, REG_XBX, XAX_OFFSET));
+        }
+        APP(ilist, RESTORE_FROM_TLS(dcontext, REG_XBX, DCONTEXT_BASE_SPILL_SLOT));
+        APP(ilist, SAVE_TO_DC(dcontext, REG_XBX, XDI_OFFSET));
+    }
+
+    /* save the current register state to context->regs
+     * xax already in context
+     */
+    if (!ibl_end) {
+        /* for ibl_end, xbx and xcx are already in their dcontext slots */
+        if (absolute) /* else xbx saved above */
+            APP(ilist, SAVE_TO_DC(dcontext, REG_XBX, XBX_OFFSET));
+        APP(ilist, SAVE_TO_DC(dcontext, REG_XCX, XCX_OFFSET));
+    }
+    APP(ilist, SAVE_TO_DC(dcontext, REG_XDX, XDX_OFFSET));
+    if (absolute || !TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask))
+        APP(ilist, SAVE_TO_DC(dcontext, REG_XSI, XSI_OFFSET));
+    if (absolute) /* else xdi saved above */
+        APP(ilist, SAVE_TO_DC(dcontext, REG_XDI, XDI_OFFSET));
+    APP(ilist, SAVE_TO_DC(dcontext, REG_XBP, XBP_OFFSET));
+    APP(ilist, SAVE_TO_DC(dcontext, REG_XSP, XSP_OFFSET));
+#ifdef X64
+    APP(ilist, SAVE_TO_DC(dcontext, REG_R8, R8_OFFSET));
+    APP(ilist, SAVE_TO_DC(dcontext, REG_R9, R9_OFFSET));
+    APP(ilist, SAVE_TO_DC(dcontext, REG_R10, R10_OFFSET));
+    APP(ilist, SAVE_TO_DC(dcontext, REG_R11, R11_OFFSET));
+    APP(ilist, SAVE_TO_DC(dcontext, REG_R12, R12_OFFSET));
+    APP(ilist, SAVE_TO_DC(dcontext, REG_R13, R13_OFFSET));
+    APP(ilist, SAVE_TO_DC(dcontext, REG_R14, R14_OFFSET));
+    APP(ilist, SAVE_TO_DC(dcontext, REG_R15, R15_OFFSET));
+#endif
+    if (preserve_xmm_caller_saved()) {
+        /* PR 264138: we must preserve xmm0-5 if on a 64-bit kernel.
+         * Rather than try and optimize we save/restore on every cxt sw.
+         * The xmm field is aligned, so we can use movdqa/movaps.
+         * See comments in fcache_enter about performance, though here
+         * we have stores instead of loads.
+         */
+        int i;
+        uint opcode = (proc_has_feature(FEATURE_SSE2) ? OP_movdqa : OP_movaps);
+        ASSERT(proc_has_feature(FEATURE_SSE));
+        for (i=0; i<NUM_XMM_SAVED; i++) {
+            APP(ilist, instr_create_1dst_1src
+                (dcontext, opcode,
+                 OPND_DC_FIELD(absolute, dcontext, OPSZ_16, XMM_OFFSET + i*XMM_REG_SIZE),
+                 opnd_create_reg(REG_XMM0 + (reg_id_t)i)));
+        }
+    }
+
+    /* Switch to a clean dstack as part of our scheme to avoid state kept
+     * unprotected across cache executions.
+     * FIXME: this isn't perfect: we switch to the dstack BEFORE we call
+     * the entrance hook that will be used to coordinate other threads,
+     * so if our hook suspends all other threads to protect vs cross-thread
+     * attacks, the dstack is not perfectly protected.
+     */
+    APP(ilist, RESTORE_FROM_DC(dcontext, REG_XSP, DSTACK_OFFSET));
+
+    /* now save eflags -- too hard to do without a stack! */    
+    APP(ilist, INSTR_CREATE_RAW_pushf(dcontext));
+    APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_XBX)));
+    APP(ilist, SAVE_TO_DC(dcontext, REG_XBX, XFLAGS_OFFSET));
+                
+    /* clear eflags now to avoid app's eflags (namely an app std)
+     * messing up our ENTER_DR_HOOK
+     */
+    /* on x64 a push immed is sign-extended to 64-bit */
+    APP(ilist, INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT8(0)));
+    APP(ilist, INSTR_CREATE_RAW_popf(dcontext));
+
+    if (ENTER_DR_HOOK != NULL) {
+        /* xax is only reg we need to save around the call.
+         * we could move to a callee-saved register instead of pushing.
+         */
+        instr_t *post_hook = INSTR_CREATE_label(dcontext);
+        if (ibl_end) {
+            /* also save xcx, which holds next_tag */
+            APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XCX)));
+        }
+        if (!absolute) {
+            /* save xdi and xsi around call */
+            APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XDI)));
+            APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XSI)));
+        }
+        APP(ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XAX)));
+#ifdef WINDOWS
+        if (absolute) {
+            /* For thread-private (used for syscalls), don't call if
+             * dcontext->ignore_enterexit.  This is a perf hit to check: could
+             * instead have a space hit via a separate routine.  This is only
+             * needed right now for NtSuspendThread handling (see case 4942).
+             */
+            APP(ilist, RESTORE_FROM_DC(dcontext, REG_EDI, IGNORE_ENTEREXIT_OFFSET));
+            /* P4 opt guide says to use test to cmp reg with 0: shorter instr */
+            APP(ilist, INSTR_CREATE_test(dcontext, opnd_create_reg(REG_EDI),
+                                         opnd_create_reg(REG_EDI)));
+            APP(ilist, INSTR_CREATE_jcc(dcontext, OP_jnz, opnd_create_instr(post_hook)));
+            instr_targets = true;
+        }
+#endif
+        /* make sure to use dr_insert_call() rather than a raw OP_call instr,
+         * since x64 windows requires 32 bytes of stack space even w/ no args,
+         * and we don't want anyone clobbering our pushed registers!
+         */
+        dr_insert_call((void *)dcontext, ilist, NULL/*append*/,
+                       (void *)ENTER_DR_HOOK, 0);
+        APP(ilist, post_hook/*label*/);
+        APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_XAX)));
+        if (!absolute) {
+            /* save xdi and xsi around call */
+            APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_XSI)));
+            APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_XDI)));
+        }
+        if (ibl_end) {
+            APP(ilist, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_XCX)));
+            
+            /* now we can store next tag */
+            APP(ilist, SAVE_TO_DC(dcontext, REG_XCX, NEXT_TAG_OFFSET));
+        }
+    }
+
+    /* save last_exit, currently in xax, into dcontext->last_exit */
+    APP(ilist, SAVE_TO_DC(dcontext, REG_XAX, LAST_EXIT_OFFSET));
+
+#ifdef WINDOWS
+    /* save app's error code */
+# ifdef KERNEL32_ERRNO
+    preinsert_get_last_error(dcontext, ilist, NULL, REG_EAX);
+    APP(ilist, SAVE_TO_DC(dcontext, REG_EAX, APP_ERRNO_OFFSET));
+# else
+    preinsert_get_last_error(dcontext, ilist, NULL, REG_EBX);
+    APP(ilist, SAVE_TO_DC(dcontext, REG_EBX, APP_ERRNO_OFFSET));
+# endif
+#else
+    /* copy shared errno into app storage slot */
+    APP(ilist, RESTORE_FROM_DC(dcontext, REG_EBX, ERRNO_OFFSET));
+    APP(ilist, SAVE_TO_DC(dcontext, REG_EBX, APP_ERRNO_OFFSET));
+#endif
+
+#ifdef SIDELINE
+    if (dynamo_options.sideline) {
+        /* clear cur-trace field so we don't think cur trace is still running */
+        /* PR 248210: unsupported feature on x64 */
+        IF_X64(ASSERT_NOT_IMPLEMENTED(false)); /* PR 244737: fix abs address */
+        APP(ilist,
+            INSTR_CREATE_mov_st(dcontext,
+                                OPND_CREATE_MEM32(REG_NULL, (int)&sideline_trace),
+                                OPND_CREATE_INT32(0)));
+    }
+#endif
+
+    /* call central dispatch routine */
+    /* for x64 linux we could optimize and avoid the "mov rdi, rdi" */
+    dr_insert_call((void *)dcontext, ilist, NULL/*append*/,
+                   (void *)dispatch, 1,
+                   absolute ?
+                   OPND_CREATE_INTPTR((ptr_int_t)dcontext) : opnd_create_reg(REG_XDI));
+
+    /* dispatch() shouldn't return! */
+    APP(ilist, INSTR_CREATE_jmp(dcontext, opnd_create_pc((app_pc)unexpected_return)));
+
+    return instr_targets;
+}
+
+byte *
+emit_fcache_return(dcontext_t *dcontext, generated_code_t *code, byte *pc)
+{
+    bool instr_targets;
+    instrlist_t ilist;
+    instrlist_init(&ilist);
+    instr_targets = append_fcache_return_common(dcontext, code, &ilist,
+                                                false/*!ibl_end*/,
+                                                true/*absolute*/, false/*!shared*/,
+                                                NULL, false/*not coarse*/);
+    /* now encode the instructions */
+    pc = instrlist_encode(dcontext, &ilist, pc, instr_targets);
+    ASSERT(pc != NULL);
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+    return pc;
+}
+
+byte *
+emit_fcache_enter_shared(dcontext_t *dcontext, generated_code_t *code, byte *pc)
+{
+    return emit_fcache_enter_common(dcontext, code, pc,
+                                    false/*through xdi*/, true/*shared*/);
+}
+
+byte *
+emit_fcache_return_shared(dcontext_t *dcontext, generated_code_t *code, byte *pc)
+{
+    instrlist_t ilist;
+    instrlist_init(&ilist);
+    append_fcache_return_common(dcontext, code, &ilist, false/*!ibl_end*/,
+                                false/*through xdi*/, true/*shared*/,
+                                NULL, false/*not coarse*/);
+    /* now encode the instructions */
+    pc = instrlist_encode(dcontext, &ilist, pc, false /* no instr targets */);
+    ASSERT(pc != NULL);
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+    return pc;
+}
+
+byte *
+emit_fcache_return_coarse(dcontext_t *dcontext, generated_code_t *code, byte *pc)
+{
+    linkstub_t *linkstub = (linkstub_t *) get_coarse_exit_linkstub();
+    instrlist_t ilist;
+    instrlist_init(&ilist);
+    append_fcache_return_common(dcontext, code, &ilist, false/*!ibl_end*/,
+                                false/*through xdi*/, true/*shared*/,
+                                linkstub, true/*coarse info in xcx*/);
+    /* now encode the instructions */
+    pc = instrlist_encode(dcontext, &ilist, pc, false /* no instr targets */);
+    ASSERT(pc != NULL);
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+    return pc;
+}
+
+byte *
+emit_trace_head_return_coarse(dcontext_t *dcontext, generated_code_t *code, byte *pc)
+{
+    /* Could share tail end of coarse_fcache_return instead of duplicating */
+    linkstub_t *linkstub = (linkstub_t *) get_coarse_trace_head_exit_linkstub();
+    instrlist_t ilist;
+    instrlist_init(&ilist);
+    append_fcache_return_common(dcontext, code, &ilist, false/*!ibl_end*/,
+                                false/*through xdi*/, true/*shared*/,
+                                linkstub, false/*no coarse info*/);
+    /* now encode the instructions */
+    pc = instrlist_encode(dcontext, &ilist, pc, false /* no instr targets */);
+    ASSERT(pc != NULL);
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+    return pc;
+}
+
+/* Our coarse entrance stubs have several advantages, such as eliminating
+ * future fragments, but their accompanying lazy linking does need source
+ * information that is not available in each stub.  We instead have an
+ * unlinked entrance stub target a per-unit prefix that records the source
+ * unit.  We can then search within the unit to identify the actual source
+ * entrance stub, which is enough for lazy linking (but does not find the
+ * unique source tag: case 8565).  This also gives us a single indirection
+ * point in the form of the prefix at which to patch the fcache_return target.
+ * We also place in the prefix indirection points for trace head cache exit and
+ * the 3 coarse ibl targets, to keep the cache read-only and (again) make it
+ * easier to patch when persisting/sharing.
+ */
+uint
+coarse_exit_prefix_size(uint flags)
+{
+    /* FIXME: would be nice to use size calculated in emit_coarse_exit_prefix(),
+     * but we need to know size before we emit and would have to do a throwaway
+     * emit, or else set up a template to be patched w/ specific info field.
+     * Also we'd have to unprot .data as we don't access this until post-init.
+     */
+    return SIZE_MOV_XBX_TO_TLS(flags, true) + SIZE_MOV_PTR_IMM_TO_XAX(flags)
+        + 5*JMP_LONG_LENGTH;
+}
+
+byte *
+emit_coarse_exit_prefix(dcontext_t *dcontext, byte *pc, coarse_info_t *info)
+{
+    byte *ibl;
+    DEBUG_DECLARE(byte *start_pc = pc;)
+    instrlist_t ilist;
+    patch_list_t patch;
+    instr_t *fcache_ret_prefix;
+#ifdef X64
+    gencode_mode_t mode = MODE_OVERRIDE(FRAG_IS_32(COARSE_32_FLAG(info)));
+#endif
+
+    instrlist_init(&ilist);
+    init_patch_list(&patch, PATCH_TYPE_INDIRECT_FS);
+
+    /* prefix looks like this, using xcx instead of xbx just to make
+     * the fcache_return code simpler (as it already uses xbx early),
+     * and using the info as we're doing per-cache and not per-unit:
+     *
+     *   fcache_return_coarse_prefix:
+     *   6/9 mov  %xcx, MANGLE_XCX_SPILL_SLOT
+     *  5/10 mov  <info ptr>, %xcx
+     *     5 jmp fcache_return_coarse
+     *   trace_head_return_coarse_prefix:
+     *     5 jmp trace_head_return_coarse
+     *       (if -disable_traces, it jmps to fcache_return_coarse_prefix instead)
+     *   coarse_ibl_ret_prefix:
+     *     5 jmp coarse_ibl_ret
+     *   coarse_ibl_call_prefix:
+     *     5 jmp coarse_ibl_call
+     *   coarse_ibl_jmp_prefix:
+     *     5 jmp coarse_ibl_jmp
+     *
+     * We assume that info ptr is at
+     *   trace_head_return_prefix - JMP_LONG_LENGTH - 4
+     * in patch_coarse_exit_prefix().
+     * We assume that the ibl prefixes are nothing but jmps in
+     * coarse_indirect_stub_jmp_target() so we can recover the ibl type.
+     *
+     * FIXME case 9647: on P4 our jmp->jmp sequence will be
+     * elided, but on Core we may want to switch to a jmp*, though
+     * since we have no register for a base ptr we'd need a reloc
+     * entry for every single stub
+     */
+    /* entrance stub has put target_tag into xax-slot so we use xcx-slot */
+    ASSERT(DIRECT_STUB_SPILL_SLOT != MANGLE_XCX_SPILL_SLOT);
+
+#ifdef X64
+    if (TEST(PERSCACHE_X86_32, info->flags)) {
+        fcache_ret_prefix = SAVE_TO_TLS(dcontext, REG_ECX, MANGLE_XCX_SPILL_SLOT);
+        APP(&ilist, fcache_ret_prefix);
+        /* We assume all our data structures are <4GB which is guaranteed for
+         * WOW64 processes.
+         */
+        ASSERT(CHECK_TRUNCATE_TYPE_int((ptr_int_t)info));
+        APP(&ilist, INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_ECX),
+                                         OPND_CREATE_INT32((int)(ptr_int_t)info)));
+    } else { /* default code */
+#endif
+        fcache_ret_prefix = SAVE_TO_TLS(dcontext, REG_XCX, MANGLE_XCX_SPILL_SLOT);
+        APP(&ilist, fcache_ret_prefix);
+        APP(&ilist, INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XCX),
+                                         OPND_CREATE_INTPTR((ptr_int_t)info)));
+#ifdef X64
+    }
+#endif
+    APP(&ilist, INSTR_CREATE_jmp(dcontext,
+        opnd_create_pc(get_direct_exit_target(dcontext,
+                                              FRAG_SHARED | FRAG_COARSE_GRAIN |
+                                              COARSE_32_FLAG(info)))));
+
+    if (DYNAMO_OPTION(disable_traces)) {
+        /* trace_t heads need to store the info ptr for lazy linking */
+        APP(&ilist, INSTR_CREATE_jmp(dcontext, opnd_create_instr(fcache_ret_prefix)));
+    } else {
+        APP(&ilist, INSTR_CREATE_jmp
+            (dcontext, opnd_create_pc(trace_head_return_coarse_routine(IF_X64(mode)))));
+    }
+    add_patch_marker(&patch, instrlist_last(&ilist), PATCH_ASSEMBLE_ABSOLUTE,
+                     0 /* start of instr */,
+                     (ptr_uint_t*)&info->trace_head_return_prefix);
+
+    ibl = get_ibl_routine_ex(dcontext, IBL_LINKED,
+                             get_source_fragment_type(dcontext,
+                                                      FRAG_SHARED | FRAG_COARSE_GRAIN),
+                             IBL_RETURN _IF_X64(mode));
+    APP(&ilist, INSTR_CREATE_jmp(dcontext, opnd_create_pc(ibl)));
+    add_patch_marker(&patch, instrlist_last(&ilist), PATCH_ASSEMBLE_ABSOLUTE,
+                     0 /* start of instr */, (ptr_uint_t*)&info->ibl_ret_prefix);
+
+    ibl = get_ibl_routine_ex(dcontext, IBL_LINKED,
+                             get_source_fragment_type(dcontext,
+                                                      FRAG_SHARED | FRAG_COARSE_GRAIN),
+                             IBL_INDCALL _IF_X64(mode));
+    APP(&ilist, INSTR_CREATE_jmp(dcontext, opnd_create_pc(ibl)));
+    add_patch_marker(&patch, instrlist_last(&ilist), PATCH_ASSEMBLE_ABSOLUTE,
+                     0 /* start of instr */, (ptr_uint_t*)&info->ibl_call_prefix);
+
+    ibl = get_ibl_routine_ex(dcontext, IBL_LINKED,
+                             get_source_fragment_type(dcontext,
+                                                      FRAG_SHARED | FRAG_COARSE_GRAIN),
+                             IBL_INDJMP _IF_X64(mode));
+    APP(&ilist, INSTR_CREATE_jmp(dcontext, opnd_create_pc(ibl)));
+    add_patch_marker(&patch, instrlist_last(&ilist), PATCH_ASSEMBLE_ABSOLUTE,
+                     0 /* start of instr */, (ptr_uint_t*)&info->ibl_jmp_prefix);
+
+    /* now encode the instructions */
+    pc += encode_with_patch_list(dcontext, &patch, &ilist, pc);
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+    ASSERT((size_t)(pc - start_pc) == coarse_exit_prefix_size(COARSE_32_FLAG(info)));
+
+    DOLOG(3, LOG_EMIT, {
+        byte *dpc = start_pc;
+        LOG(GLOBAL, LOG_EMIT, 3, "\nprefixes for coarse unit %s:\n", info->module);
+        do {
+            if (dpc == info->fcache_return_prefix)
+                LOG(GLOBAL, LOG_EMIT, 3, "fcache_return_coarse_prefix:\n");
+            else if (dpc == info->trace_head_return_prefix)
+                LOG(GLOBAL, LOG_EMIT, 3, "trace_head_return_coarse_prefix:\n");
+            else if (dpc == info->ibl_ret_prefix)
+                LOG(GLOBAL, LOG_EMIT, 3, "ibl_coarse_ret_prefix:\n");
+            else if (dpc == info->ibl_call_prefix)
+                LOG(GLOBAL, LOG_EMIT, 3, "ibl_coarse_call_prefix:\n");
+            else if (dpc == info->ibl_jmp_prefix)
+                LOG(GLOBAL, LOG_EMIT, 3, "ibl_coarse_jmp_prefix:\n");
+            dpc = disassemble_with_bytes(dcontext, dpc, GLOBAL);
+        } while (dpc < pc);
+        LOG(GLOBAL, LOG_EMIT, 3, "\n");
+    });
+
+    return pc;
+}
+
+/* Update info pointer in exit prefixes */
+void
+patch_coarse_exit_prefix(dcontext_t *dcontext, coarse_info_t *info)
+{
+    ptr_uint_t *pc = (ptr_uint_t *)
+        (info->trace_head_return_prefix - JMP_LONG_LENGTH - sizeof(info));
+    *pc = (ptr_uint_t) info;
+}
+
+
+/* saves the eflags
+ * uses the xax slot, either in TLS
+ * memory if tls is true; else using mcontext accessed using absolute address if
+ * absolute is true, else off xdi.
+ * MUST NOT clobber xax between this call and the restore call!
+ */
+void
+insert_save_eflags(dcontext_t *dcontext, instrlist_t *ilist, instr_t *where,
+                   uint flags, bool tls, bool absolute)
+{
+    /* no support for absolute addresses on x64: we always use tls/reg */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(!absolute));
+
+    if (TEST(FRAG_WRITES_EFLAGS_6, flags)) /* no flag save needed */
+        return;
+    /* save the flags */
+    /*>>>    SAVE_TO_TLS/UPCONTEXT %xax,xax_tls_slot/xax_OFFSET  */
+    /*>>>    lahf                                                */
+    /*>>>    seto        %al                                     */
+    /* for shared ibl targets we put eflags in tls -- else, we use mcontext,
+     * either absolute address or indirected via xdi as specified by absolute param
+     */
+    if (tls) {
+        /* We need to save this in an easy location for the prefixes
+         * to restore from.  FIXME: This can be much more streamlined
+         * if TLS_SLOT_SCRATCH1 was the XAX spill slot for everyone
+         */
+        /* FIXME: since the prefixes are trying to be smart now
+         * based on shared/privateness of the fragment, we also need
+         * to know what would the target do if shared.
+         */
+        /*>>>    SAVE_TO_TLS %xax,xax_tls_slot                       */
+        PRE(ilist, where, SAVE_TO_TLS(dcontext, REG_XAX, PREFIX_XAX_SPILL_SLOT));
+    } else {
+        /*>>>    SAVE_TO_UPCONTEXT %xax,xax_OFFSET                       */
+        PRE(ilist, where, SAVE_TO_DC(dcontext, REG_XAX, XAX_OFFSET));
+    }
+    PRE(ilist, where, INSTR_CREATE_lahf(dcontext));
+    if (!TEST(FRAG_WRITES_EFLAGS_OF, flags)
+        && !INTERNAL_OPTION(unsafe_ignore_overflow)) { /* OF needs saving */
+        /* Move OF flags into the OF flag spill slot. */
+        PRE(ilist, where,
+            INSTR_CREATE_setcc(dcontext, OP_seto, opnd_create_reg(REG_AL)));
+    }
+}
+
+/* restores eflags from xax and the xax app value from the xax slot, either in
+ * TLS memory if tls is true; else using mcontext accessed using absolute
+ * address if absolute is true, else off xdi.
+ * also restores xax
+ */
+void
+insert_restore_eflags(dcontext_t *dcontext, instrlist_t *ilist, instr_t *where,
+                      uint flags, bool tls, bool absolute)
+{
+    /* no support for absolute addresses on x64: we always use tls/reg */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(!absolute));
+
+    if (TEST(FRAG_WRITES_EFLAGS_6, flags)) /* no flag save was done */
+        return;
+    if (!TEST(FRAG_WRITES_EFLAGS_OF, flags) &&  /* OF was saved */
+        !INTERNAL_OPTION(unsafe_ignore_overflow)) { 
+        /* restore OF using add that overflows and sets OF if OF was on when we did seto */
+        PRE(ilist, where,
+            INSTR_CREATE_add(dcontext, opnd_create_reg(REG_AL), OPND_CREATE_INT8(0x7f)));
+    }
+    /* restore other 5 flags (still in xax at this point) */
+    /*>>>    sahf                                                    */
+    PRE(ilist, where, INSTR_CREATE_sahf(dcontext));
+    /* restore xax */
+    if (tls) {
+        PRE(ilist, where, RESTORE_FROM_TLS(dcontext, REG_XAX, PREFIX_XAX_SPILL_SLOT));
+    } else {
+        /*>>>    RESTORE_FROM_UPCONTEXT xax_OFFSET,%xax                  */
+        PRE(ilist, where, RESTORE_FROM_DC(dcontext, REG_XAX, XAX_OFFSET));
+    }
+}
+
+#ifdef HASHTABLE_STATISTICS
+/* note that arch_thread_init is called before fragment_thread_init, so these need to be updated */
+/* When used in a thread-shared routine, this routine clobbers XDI. The
+ * caller should spill & restore it or rematerialize it as needed. */
+/* NOTE - this routine does NOT save the eflags, which will be clobbered by the
+ * inc */
+static void
+append_increment_counter(dcontext_t *dcontext, instrlist_t *ilist, 
+                         ibl_code_t *ibl_code, patch_list_t *patch,
+                         reg_id_t entry_register, /* register indirect (XCX) or NULL */
+                         uint counter_offset, /* adjusted to unprot_ht_statistics_t if no entry_register */
+                         reg_id_t scratch_register)
+{
+    instr_t *counter;
+    bool absolute = !ibl_code->thread_shared_routine;
+    /* no support for absolute addresses on x64: we always use tls/reg */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(!absolute));
+
+    if (!INTERNAL_OPTION(hashtable_ibl_stats)) 
+        return;
+
+    LOG(THREAD, LOG_EMIT, 3, "append_increment_counter: hashtable_stats_offset=0x%x counter_offset=0x%x\n",
+        ibl_code->hashtable_stats_offset, counter_offset);
+
+    if (entry_register == REG_NULL) {
+        /* adjust offset within a unprot_ht_statistics_t structure */
+        counter_offset += ibl_code->hashtable_stats_offset; 
+    }
+
+    if (!absolute) {
+        opnd_t counter_opnd;
+        
+        /* get dcontext in register (xdi) */
+        append_shared_get_dcontext(dcontext, ilist, false /* dead register */);
+        /* XDI now has dcontext */
+        APP(ilist, INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XDI),
+                                       OPND_DC_FIELD(absolute, dcontext, OPSZ_PTR, 
+                                                     FRAGMENT_FIELD_OFFSET)));
+
+        /* XDI now has per_thread_t structure */
+        /* an extra step here: find the unprot_stats field in the fragment_table_t
+         * could avoid for protect_mask==0 if we always had a copy
+         * in the per_thread_t struct -- see fragment.h, not worth it
+         */
+        if (entry_register != REG_NULL) {
+            APP(ilist, INSTR_CREATE_mov_ld
+                (dcontext, opnd_create_reg(REG_XDI),
+                 OPND_CREATE_MEMPTR(REG_XDI,
+                                    ibl_code->entry_stats_to_lookup_table_offset)));
+            /* XDI should now have (entry_stats - lookup_table) value,
+             * so we need [xdi+xcx] to get an entry reference
+             */
+            counter_opnd = opnd_create_base_disp(REG_XDI, entry_register, 1,
+                                                 counter_offset, OPSZ_4);
+        } else {
+            APP(ilist, INSTR_CREATE_mov_ld
+                (dcontext, opnd_create_reg(REG_XDI),
+                 OPND_CREATE_MEMPTR(REG_XDI, ibl_code->unprot_stats_offset)));
+            /* XDI now has unprot_stats structure */
+            counter_opnd = OPND_CREATE_MEM32(REG_XDI, counter_offset);
+        }
+
+        counter = INSTR_CREATE_inc(dcontext, counter_opnd);
+        APP(ilist, counter);
+    } else {
+        /* TAKE_ADDRESS will in fact add the necessary base to the statistics structure, 
+           hence no explicit indirection needed here */
+        opnd_t counter_opnd = OPND_CREATE_MEMPTR(entry_register, counter_offset);
+        counter = INSTR_CREATE_inc(dcontext, counter_opnd);
+        /* hack to get both this table's unprot offset and the specific stat's offs */
+        ASSERT(counter_offset < USHRT_MAX);
+        if (entry_register != REG_NULL) {
+            /* although we currently don't use counter_offset, it doesn't hurt to support as well */
+            ASSERT(ibl_code->entry_stats_to_lookup_table_offset < USHRT_MAX);
+            add_patch_entry(patch, counter, PATCH_UNPROT_STAT|PATCH_TAKE_ADDRESS, 
+                            (ibl_code->entry_stats_to_lookup_table_offset << 16) | counter_offset);
+        } else {
+            ASSERT(ibl_code->unprot_stats_offset < USHRT_MAX);
+            add_patch_entry(patch, counter, PATCH_UNPROT_STAT|PATCH_TAKE_ADDRESS, 
+                            (ibl_code->unprot_stats_offset << 16) | counter_offset);
+        }
+        APP(ilist, counter);
+    }
+}
+#endif /* HASHTABLE_STATISTICS */
+
+#ifdef INTERNAL
+/* add a slowdown loop to measure if a routine is likely to be on a critical path */
+/* note that FLAGS are clobbered */
+static void
+append_empty_loop(dcontext_t *dcontext, instrlist_t *ilist, 
+                  uint iterations, reg_id_t scratch_register)
+{
+    instr_t *initloop;
+    instr_t *loop;
+    /*       mov     ebx, iterations */
+    /* loop: dec     ebx */
+    /*       jnz loop */
+    ASSERT(REG_NULL != scratch_register);
+    
+    initloop = INSTR_CREATE_mov_imm(dcontext,
+                                    opnd_create_reg(scratch_register),
+                                    OPND_CREATE_INT32(iterations));
+    loop = INSTR_CREATE_dec(dcontext, opnd_create_reg(scratch_register));
+    APP(ilist, initloop);
+    APP(ilist, loop);
+    APP(ilist, INSTR_CREATE_jcc(dcontext, OP_jnz_short, opnd_create_instr(loop)));
+}
+#endif /* INTERNAL */
+
+/* what we do on a hit in the hashtable */
+/* Restore XBX saved from the indirect exit stub insert_jmp_to_ibl() */
+/* Indirect jump through hashtable entry pointed to by XCX */
+static inline void
+append_ibl_found(dcontext_t *dcontext, instrlist_t *ilist, 
+                 ibl_code_t *ibl_code, patch_list_t *patch,
+                 uint start_pc_offset,
+                 bool collision,
+                 bool only_spill_state_in_tls, /* if true, no table info in TLS;
+                                                * indirection off of XDI is used */
+                 bool restore_eflags,
+                 instr_t **fragment_found)
+{
+    bool absolute = !ibl_code->thread_shared_routine;
+    bool target_prefix = true;
+    /* eflags and xcx are restored in the target's prefix */
+    /* if thread private routine */
+    /*>>>    RESTORE_FROM_UPCONTEXT xbx_OFFSET,%xbx                  */
+    /*>>>    jmp     *FRAGMENT_START_PC_OFFS(%xcx)                   */
+    instr_t *inst = NULL;
+
+    /* no support for absolute addresses on x64: we always use tls/reg */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(!absolute));
+
+    if (absolute) {
+        inst = RESTORE_FROM_DC(dcontext, REG_XBX, XBX_OFFSET);
+    }
+
+    if (!ibl_use_target_prefix(ibl_code)) {
+        target_prefix = false;
+        restore_eflags = true;
+    }
+
+#ifdef HASHTABLE_STATISTICS
+    if (INTERNAL_OPTION(hashtable_ibl_stats) ||
+        INTERNAL_OPTION(hashtable_ibl_entry_stats)) {
+        if (!absolute && !only_spill_state_in_tls) {
+            /* XDI holds app state, not a ptr to dcontext+<some offset> */
+            APP(ilist, SAVE_TO_TLS(dcontext, REG_XDI, HTABLE_STATS_SPILL_SLOT));
+        }
+        append_increment_counter(dcontext, ilist, ibl_code, patch,
+                                 REG_NULL,
+                                 HASHLOOKUP_STAT_OFFS(hit),
+                                 REG_XBX);
+        if (collision) {
+            append_increment_counter(dcontext, ilist, ibl_code, patch,
+                                     REG_NULL,
+                                     HASHLOOKUP_STAT_OFFS(collision_hit),
+                                     REG_XBX);
+        }
+        if (INTERNAL_OPTION(hashtable_ibl_entry_stats)) {
+            /* &lookup_table[i] - should allow access to &entry_stats[i] */
+            append_increment_counter(dcontext, ilist, ibl_code, patch,
+                                     REG_XCX, 
+                                     offsetof(fragment_stat_entry_t, hits),
+                                     REG_XBX);
+        }
+        if (!absolute && !only_spill_state_in_tls)
+            APP(ilist, RESTORE_FROM_TLS(dcontext, REG_XDI, HTABLE_STATS_SPILL_SLOT));
+    }
+#endif /* HASHTABLE_STATISTICS */
+
+#ifdef INTERNAL
+    if (INTERNAL_OPTION(slowdown_ibl_found)) {
+        /* add a loop here */
+        append_empty_loop(dcontext, ilist, INTERNAL_OPTION(slowdown_ibl_found),
+                          REG_XBX /* dead */);
+    }
+#endif /* INTERNAL */
+
+    if (restore_eflags) {
+        insert_restore_eflags(dcontext, ilist, NULL, 0,
+                              IBL_EFLAGS_IN_TLS(), absolute);
+    }
+    if (!target_prefix) {
+        /* We're going to clobber the xax slot */
+        ASSERT(restore_eflags);
+        /* For target_delete support with no prefix, since we're
+         * clobbering all the registers here, we must save something;
+         * We save the tag, rather than the table entry, to avoid an
+         * extra load to get the tag in target_delete:
+         *     <save    %xbx to xax slot>  # put tag in xax slot for target_delete
+         */
+        if (absolute) {
+            APP(ilist, SAVE_TO_DC(dcontext, REG_XBX, XAX_OFFSET));
+        } else {
+            APP(ilist, SAVE_TO_TLS(dcontext, REG_XBX, DIRECT_STUB_SPILL_SLOT));
+        }
+    }
+    if (absolute) {
+        /* restore XBX through dcontext */
+        APP(ilist, inst);
+    } else {
+        /* restore XBX through INDIRECT_STUB_SPILL_SLOT */
+        APP(ilist, RESTORE_FROM_TLS(dcontext, REG_XBX, INDIRECT_STUB_SPILL_SLOT));
+        DODEBUG({
+            if (!SHARED_IB_TARGETS())
+                ASSERT(only_spill_state_in_tls);
+        });
+    }
+    if (only_spill_state_in_tls) {
+        /* If TLS doesn't hold table info, XDI was used for indirection.
+         * Restore XDI through DCONTEXT_BASE_SPILL_SLOT */
+        append_shared_restore_dcontext_reg(dcontext, ilist);
+    }
+
+    if (target_prefix) {
+        /* FIXME: do we want this?  seems to be a problem, I'm disabling:
+         * ASSERT(!collision || start_pc_offset == FRAGMENT_START_PC_OFFS);  // c \imply FRAGMENT
+         */
+        APP(ilist, INSTR_CREATE_jmp_ind(dcontext,
+                                        OPND_CREATE_MEMPTR(REG_XCX, start_pc_offset)));
+    } else {
+        /* There is no prefix so we must restore all and jmp through memory:
+         *     mov      start_pc_offset(%xcx), %xcx
+         *     <save    %xcx to xbx slot>  # put target in xbx slot for later jmp
+         *     <restore %xcx from xcx slot>
+         *     jmp*     <xbx slot>
+         */
+        APP(ilist, INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XCX),
+                                       OPND_CREATE_MEMPTR(REG_XCX, start_pc_offset)));
+        if (absolute) {
+            APP(ilist, SAVE_TO_DC(dcontext, REG_XCX, XBX_OFFSET));
+            if (XCX_IN_TLS(0/*!FRAG_SHARED*/))
+                APP(ilist, RESTORE_FROM_TLS(dcontext, REG_XCX, MANGLE_XCX_SPILL_SLOT));
+            else
+                APP(ilist, RESTORE_FROM_DC(dcontext, REG_XCX, XCX_OFFSET));
+            APP(ilist, INSTR_CREATE_jmp_ind(dcontext,
+                                            OPND_DC_FIELD(absolute, dcontext, OPSZ_PTR,
+                                                          XBX_OFFSET)));
+        } else {
+            APP(ilist, SAVE_TO_TLS(dcontext, REG_XCX, INDIRECT_STUB_SPILL_SLOT));
+            APP(ilist, RESTORE_FROM_TLS(dcontext, REG_XCX, MANGLE_XCX_SPILL_SLOT));
+            APP(ilist, INSTR_CREATE_jmp_ind(dcontext,
+                                            OPND_TLS_FIELD(INDIRECT_STUB_SPILL_SLOT)));
+        }
+    }
+
+    if (fragment_found != NULL)
+        *fragment_found = inst;
+}
+
+#ifdef PROFILE_LINKCOUNT
+/* assumes linkstub in ebx, can handle NULL linkstub pointer
+ * if clobber_eflags is true, assumes can kill eflags
+ * else, ASSUMES CAN KILL ECX
+ * both require a non null next instruction
+ */
+void
+append_linkcount_incr(dcontext_t *dcontext, instrlist_t *ilist, bool clobber_eflags,
+                      instr_t *next)
+{
+    /* PR 248210: unsupported feature on x64 */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    ASSERT(next != NULL);
+    if (clobber_eflags) {
+        /*>>>    testl   ebx,ebx */
+        /*>>>    je      next:  */
+        /* P4 opt guide says to use test to cmp reg with 0: shorter instr */
+        APP(ilist, INSTR_CREATE_test(dcontext, opnd_create_reg(REG_EBX), 
+                                     opnd_create_reg(REG_EBX)));
+        APP(ilist, INSTR_CREATE_jcc_short(dcontext, OP_je, 
+                                          opnd_create_instr(next)));
+# ifdef LINKCOUNT_64_BITS
+        /*>>>    addl    $1,count_offs(ebx == &linkstub)                 */
+        /*>>>    adcl    $0,count_offs+4(ebx == &linkstub)               */
+        APP(ilist, INSTR_CREATE_add(dcontext,
+                                    OPND_CREATE_MEM32(REG_EBX, LINKSTUB_COUNT_OFFS),
+                                    OPND_CREATE_INT8(1)));
+        APP(ilist, INSTR_CREATE_adc(dcontext,
+                                    OPND_CREATE_MEM32(REG_EBX, LINKSTUB_COUNT_OFFS+4),
+                                    OPND_CREATE_INT8(0)));
+# else
+        /*>>>    incl    count_offs(ebx == &linkstub)                    */
+        APP(ilist, INSTR_CREATE_inc(dcontext,
+                                    OPND_CREATE_MEM32(REG_EBX, LINKSTUB_COUNT_OFFS)));
+# endif
+    } else {
+# ifdef LINKCOUNT_64_BITS
+        instr_t *carry;
+# endif
+        /*>>>    movl    %ebx, %ecx */
+        /*>>>    jecxz   next:      */
+        APP(ilist, INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_ECX),
+                                       opnd_create_reg(REG_EBX)));
+        APP(ilist, INSTR_CREATE_jecxz(dcontext, opnd_create_instr(next)));
+        /*>>>    movl    count_offs(%ebx == &linkstub), %ecx              */
+        /*>>>    lea     1(%ecx), %ecx                                    */
+        /*>>>    movl    %ecx, count_offs(%ebx == &linkstub), %ecx        */
+        APP(ilist, INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_ECX),
+                               OPND_CREATE_MEM32(REG_EBX, LINKSTUB_COUNT_OFFS)));
+        APP(ilist, INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ECX),
+                            opnd_create_base_disp(REG_ECX, REG_NULL, 0, 1, OPSZ_lea)));
+        APP(ilist, INSTR_CREATE_mov_st(dcontext, 
+                               OPND_CREATE_MEM32(REG_EBX, LINKSTUB_COUNT_OFFS),
+                               opnd_create_reg(REG_ECX)));
+# ifdef LINKCOUNT_64_BITS
+        /*>>>    jecxz   carry                                            */
+        /*>>>    jmp     nocarry                                          */
+        /*>>>  carry:                                                     */
+        /*>>>    movl    count_offs+4(%ebx == &linkstub), %ecx            */
+        /*>>>    lea     1(%ecx), %ecx                                    */
+        /*>>>    movl    %ecx, count_offs+4(%ebx == &linkstub), %ecx      */
+        /*>>>  nocarry:                                                   */
+        carry = INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_ECX),
+                                    OPND_CREATE_MEM32(REG_EBX, LINKSTUB_COUNT_OFFS));
+        APP(ilist, INSTR_CREATE_jecxz(dcontext, opnd_create_instr(carry)));
+        APP(ilist, INSTR_CREATE_jmp(dcontext, opnd_create_instr(next)));
+        APP(ilist, carry);
+        APP(ilist, INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ECX),
+                            opnd_create_base_disp(REG_ECX, REG_NULL, 0, 1, OPSZ_lea)));
+        APP(ilist, INSTR_CREATE_mov_st(dcontext, 
+                               OPND_CREATE_MEM32(REG_EBX, LINKSTUB_COUNT_OFFS),
+                               opnd_create_reg(REG_ECX)));
+# endif
+    }
+}
+#endif
+
+/* When inline_ibl_head, this emits the inlined lookup for the exit stub.
+ *   Only assumption is that xcx = effective address of indirect branch
+ * Else, this emits the top of the shared lookup routine, which assumes:
+ *   1) xbx = &linkstub
+ *   2) xcx = effective address of indirect branch
+ * Assumes that a jne_short is sufficient to reach miss_tgt.
+ * Returns pointers to three instructions, for use in calculating offsets
+ * and in pointing jmps inside the ibl head.
+ * It's fine to pass NULL if you're not interested in them.
+ */
+static void
+append_ibl_head(dcontext_t *dcontext, instrlist_t *ilist, 
+                ibl_code_t *ibl_code, patch_list_t *patch,
+                instr_t **fragment_found, instr_t **compare_tag_inst,
+                instr_t **post_eflags_save,
+                opnd_t miss_tgt, bool miss_8bit,
+                bool target_trace_table,
+                bool inline_ibl_head)
+{
+    instr_t *mask, *table = NULL, *compare_tag = NULL, *after_linkcount;
+    opnd_t mask_opnd;
+    bool absolute = !ibl_code->thread_shared_routine;
+    bool table_in_tls = SHARED_IB_TARGETS() &&
+        (target_trace_table || SHARED_BB_ONLY_IB_TARGETS()) &&
+        DYNAMO_OPTION(ibl_table_in_tls);
+    uint hash_to_address_factor;
+    /* Use TLS only for spilling app state -- registers & flags */
+    bool only_spill_state_in_tls = !absolute && !table_in_tls;
+
+    /* no support for absolute addresses on x64: we always use tls/reg */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(!absolute));
+
+#ifndef X64    
+    /* For x64 we need this after the cmp post-eflags entry; for x86, it's
+     * needed before for thread-private eflags save.
+     */
+    if (only_spill_state_in_tls) {
+        /* grab dcontext in XDI for thread shared routine */
+        append_shared_get_dcontext(dcontext, ilist, true /* save xdi to scratch */);
+    }
+#endif
+    if (!INTERNAL_OPTION(unsafe_ignore_eflags_ibl)) {
+        /* There are ways to generate IBL that doesn't touch the EFLAGS -- see
+         * case 7169. We're not using any of those techniques, so we save the
+         * flags.
+         */
+         insert_save_eflags(dcontext, ilist, NULL, 0, IBL_EFLAGS_IN_TLS(),
+                            absolute); 
+    }
+    /* PR 245832: x64 trace cmp saves flags so we need an entry point post-flags-save */
+    if (post_eflags_save != NULL) {
+        *post_eflags_save = INSTR_CREATE_label(dcontext);
+        APP(ilist, *post_eflags_save);
+    }
+#ifdef X64    
+    /* See comments above */
+    if (only_spill_state_in_tls) {
+        /* grab dcontext in XDI for thread shared routine */
+        append_shared_get_dcontext(dcontext, ilist, true /* save xdi to scratch */);
+    }
+#endif
+    if (inline_ibl_head || !DYNAMO_OPTION(indirect_stubs)) {
+        /*>>>    SAVE_TO_UPCONTEXT %xbx,xbx_OFFSET                       */
+        if (absolute)
+            after_linkcount = SAVE_TO_DC(dcontext, REG_XBX, XBX_OFFSET);
+        else
+            after_linkcount = SAVE_TO_TLS(dcontext, REG_XBX, TLS_XBX_SLOT);
+    } else {
+        /* create scratch register: re-use xbx, it holds linkstub ptr,
+         * don't need to restore it on hit!  save to **xdi** slot so as
+         * to not overwrite linkstub ptr */
+        /*>>>    SAVE_TO_UPCONTEXT %xbx,xdi_OFFSET                       */
+        if (absolute) 
+            after_linkcount = SAVE_TO_DC(dcontext, REG_XBX, XDI_OFFSET);
+        else if (table_in_tls) /* xdx is the free slot for tls */
+            after_linkcount = SAVE_TO_TLS(dcontext, REG_XBX, TLS_XDX_SLOT);
+        else /* the xdx slot already holds %xdi so use the mcontext */
+            after_linkcount = SAVE_TO_DC(dcontext, REG_XBX, XDI_OFFSET);
+    }
+#ifdef PROFILE_LINKCOUNT
+    ASSERT_NOT_IMPLEMENTED(!inline_ibl_head);
+    /* not supported for inline_ibl_head, so we can assume xbx==&linkstub 
+     * FIXME: this ignores whether fragment is trace or not --
+     * everybody has the count field, we just end up incr-ing it for bbs that
+     * nobody looks at
+     */
+    if (dynamo_options.profile_counts &&
+        IS_IBL_TRACE(ibl_code->source_fragment_type)) {
+        append_linkcount_incr(dcontext, ilist, true /* can clobber eflags */, 
+                              after_linkcount);
+    }
+#endif /* PROFILE_LINKCOUNT */
+    APP(ilist, after_linkcount);
+    if (ibl_code->thread_shared_routine && !DYNAMO_OPTION(private_ib_in_tls)) {
+        /* copy app xcx currently in tls slot into mcontext slot, so that we
+         * can work with both tls and mcontext prefixes
+         * do not need this if using all-tls (private_ib_in_tls option)
+         */
+        /* xbx is now dead, just saved it */
+        APP(ilist, RESTORE_FROM_TLS(dcontext, REG_XBX, MANGLE_XCX_SPILL_SLOT));
+        APP(ilist, SAVE_TO_DC(dcontext, REG_XBX, XCX_OFFSET));
+    }
+    /* make a copy of the tag for hashing
+     * keep original in xbx, hash will be in xcx
+     *>>>    mov     %xcx,%xbx                                       */
+    APP(ilist, INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XBX),
+                                   opnd_create_reg(REG_XCX)));
+
+    if (only_spill_state_in_tls) {
+        /* grab the per_thread_t into XDI - can't use SAVE_TO_DC after this */
+        /* >>> mov  %xdi, fragment_field(%xdi) */
+        /*   TODO:  make this an 8bit offset, currently it is a 32bit one
+             8b bf 94 00 00 00    mov    0x94(%xdi) -> %xdi 
+        */
+        APP(ilist, INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XDI),
+                                       OPND_DC_FIELD(absolute, dcontext, OPSZ_PTR, 
+                                                     FRAGMENT_FIELD_OFFSET)));
+        /* TODO: should have a flag that SAVE_TO_DC can ASSERT(valid_DC_in_reg) */
+    }
+    /* hash function = (tag & mask) */
+    if (!absolute && table_in_tls) {
+        /* mask is in tls */
+        mask_opnd = OPND_TLS_FIELD(TLS_MASK_SLOT(ibl_code->branch_type));
+    } else if (!absolute) {
+        ASSERT(only_spill_state_in_tls);
+        /* this is an offset in per_thread_t so should fit in 32 bits */
+        IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int
+                      (GET_IB_FTABLE(ibl_code, target_trace_table, hash_mask))));
+        mask_opnd =
+            opnd_create_base_disp(REG_XDI, REG_NULL, 0,
+                                  (int) GET_IB_FTABLE(ibl_code, target_trace_table,
+                                                      hash_mask), OPSZ_PTR);
+    } else {
+        /* mask not created yet, use 0x3fff for now 
+         * if we did need to support an immediate for x64, we could
+         * just use the lower 32 bits and let them be sign-extended.
+         *>>>    andl    $0x3fff,%xcx                                    */
+        mask_opnd = opnd_create_immed_int(0x3fff, OPSZ_4);
+    }
+    mask = INSTR_CREATE_and(dcontext, opnd_create_reg(REG_XCX), mask_opnd);
+    APP(ilist, mask);
+    if (absolute) {
+        add_patch_entry(patch, mask, PATCH_PER_THREAD,
+                        (ptr_uint_t) GET_IB_FTABLE(ibl_code, target_trace_table,
+                                                   hash_mask));
+    }
+
+    /* load from lookup hash table tag and start_pc */
+    /* simply   cmp     BOGUS_HASH_TABLE(,%xcx,8),%xcx   # tag */
+    /*          jne     next_fragment    */
+    /*          jmp     *FRAGMENT_START_PC_OFFS(4,%xdx,3)# pc */
+    /* or better yet */
+    /*  lea     BOGUS_HASH_TABLE(,%xcx,8),%xcx   # xcx  = &lookuptable[hash]    */
+    /*  cmp     HASHLOOKUP_TAG_OFFS(%xcx),%xbx   # tag           _cache line 1_ */
+    /*  jne     next_fragment    */
+    /*  jmp     *HASHLOOKUP_START_PC_OFFS(%xcx)  # pc            _cache line 1_ */
+
+    /* *>>>    lea    BOGUS_HASH_TABLE(,%xcx,8),%xcx                  
+       not created yet, use 0 */
+
+    if (only_spill_state_in_tls) {
+        /* grab the corresponding table or lookuptable for trace into XDI */
+        /* >>> mov  %xdi, lookuptable(%xdi) */
+        /* 8b 7f 40             mov    0x40(%xdi) -> %xdi 
+        */
+
+        instr_t *table_in_xdi;
+        /* this is an offset in per_thread_t so should fit in 32 bits */
+        IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int
+                      (GET_IB_FTABLE(ibl_code, target_trace_table, table))));
+        table_in_xdi =
+            INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XDI),
+                                opnd_create_base_disp(REG_XDI, REG_NULL, 0,
+                                                      (int)
+                                                      GET_IB_FTABLE(ibl_code,
+                                                                    target_trace_table,
+                                                                    table),
+                                                      OPSZ_PTR));
+        /* lookuptable can still be reloaded from XDI later at sentinel_check */
+        APP(ilist, table_in_xdi);
+    }
+    
+    if (absolute) {
+        ASSERT(sizeof(fragment_entry_t) == 8); /* x64 not supported */
+        if (HASHTABLE_IBL_OFFSET(ibl_code->branch_type) <= IBL_HASH_FUNC_OFFSET_MAX) {
+            /* multiply by 16,8,4,2 or 1 respectively when we offset 0,1,2,3,4 bits */
+            IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint
+                          (sizeof(fragment_entry_t) / 
+                           (size_t)(1 << HASHTABLE_IBL_OFFSET(ibl_code->branch_type)))));
+            hash_to_address_factor = (uint) sizeof(fragment_entry_t) / 
+                (1 << HASHTABLE_IBL_OFFSET(ibl_code->branch_type));
+        } else {
+            /* FIXME: we'll need to shift right a few more bits */
+            /* >>>   shrl  factor-3, %xcx */
+            ASSERT_NOT_IMPLEMENTED(false);
+            hash_to_address_factor = 1;
+        }
+        /* FIXME: there is no good way to ASSERT that the table we're looking up
+         * is using the correct hash_mask_offset
+         */
+        
+        /* FIXME: case 4893: three ADD's are faster than one LEA - if IBL
+         * head is not inlined we may want to try that advice 
+         */
+        /* FIXME: case 4893 when hash_mask_offset==3 we can use a better encoding
+         * since we don't need an index register we can switch to a non-SIB encoding
+         * so that instead of 7 bytes we have 6 byte encoding going through the fast decoder
+         * 8d 0c 0d 5039721c   lea     xcx,[1c723950+xcx]   ; currently
+         * 8d 89 __ 5039721c   lea     xcx,[xcx+0x1c723950] ; shorter
+         */
+        table = INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XCX),
+                                 opnd_create_base_disp(REG_NULL, REG_XCX, 
+                                                       hash_to_address_factor, 0, OPSZ_lea));
+        add_patch_entry(patch, table, PATCH_PER_THREAD,
+                        GET_IB_FTABLE(ibl_code, target_trace_table, table));
+        APP(ilist, table);
+    } else { /* !absolute && (table_in_tls  || only_spill_state_in_tls) */
+        /* we have the base added separately already, so we skip the lea and
+         * use faster and smaller add sequences for our shift
+         */
+        uint i;
+        opnd_t table_opnd;
+        
+        ASSERT_NOT_IMPLEMENTED(HASHTABLE_IBL_OFFSET(ibl_code->branch_type) <=
+                               IBL_HASH_FUNC_OFFSET_MAX);
+        /* are 4 adds faster than 1 lea, for x64? */
+        for (i = IBL_HASH_FUNC_OFFSET_MAX;
+             i > HASHTABLE_IBL_OFFSET(ibl_code->branch_type); i--) {
+            APP(ilist, INSTR_CREATE_add(dcontext, opnd_create_reg(REG_XCX),
+                                        opnd_create_reg(REG_XCX)));
+        }
+        /* we separately add the lookuptable base since we'd need an extra register
+         * to do it in combination with the shift:
+         *    add   fs:lookuptable,%xcx -> %xcx
+         * or if the table addr is in %xdi:
+         *    add   %xdi,%xcx -> %xcx
+         */
+        table_opnd = table_in_tls ?
+            OPND_TLS_FIELD(TLS_TABLE_SLOT(ibl_code->branch_type)) /* addr in TLS */:
+            opnd_create_reg(REG_XDI) /* addr in %xdi */;
+        APP(ilist,
+            INSTR_CREATE_add(dcontext, opnd_create_reg(REG_XCX), table_opnd));
+    }
+
+    /* compare tags, empty slot is not 0, instead is a constant frag w/ tag 0 
+     *>>>    cmp     HASHLOOKUP_TAG_OFFS(%xcx),%xbx                    */ 
+    compare_tag = INSTR_CREATE_cmp(dcontext,
+                                   OPND_CREATE_MEMPTR(REG_XCX, HASHLOOKUP_TAG_OFFS),
+                                   opnd_create_reg(REG_XBX));
+    APP(ilist, compare_tag);
+
+    /*>>>    jne     next_fragment                                   */
+    if (miss_8bit)
+        APP(ilist, INSTR_CREATE_jcc(dcontext, OP_jne_short, miss_tgt));
+    else
+        APP(ilist, INSTR_CREATE_jcc(dcontext, OP_jne, miss_tgt));
+
+#ifdef X64
+    if (ibl_code->x86_mode) {
+        /* Currently we're using the x64 table, so we have to ensure the top
+         * bits are 0 before we declare it a match (xref PR 283895).
+         */
+        APP(ilist, INSTR_CREATE_test(dcontext,
+                                     OPND_CREATE_MEMPTR(REG_XCX, HASHLOOKUP_TAG_OFFS+4),
+                                     OPND_CREATE_MEMPTR(REG_XCX, HASHLOOKUP_TAG_OFFS+4)));
+        if (miss_8bit)
+            APP(ilist, INSTR_CREATE_jcc(dcontext, OP_jne_short, miss_tgt));
+        else
+            APP(ilist, INSTR_CREATE_jcc(dcontext, OP_jne, miss_tgt));
+    }
+#endif
+
+#define HEAD_START_PC_OFFS HASHLOOKUP_START_PC_OFFS
+    append_ibl_found(dcontext, ilist, ibl_code, patch, HEAD_START_PC_OFFS,
+                     false, only_spill_state_in_tls,
+                     target_trace_table ?
+                     DYNAMO_OPTION(trace_single_restore_prefix) :
+                     DYNAMO_OPTION(bb_single_restore_prefix),
+                     fragment_found);
+#undef HEAD_START_PC_OFFS
+
+    if (compare_tag_inst != NULL)
+        *compare_tag_inst = compare_tag;
+}
+
+/* create the inlined ibl exit stub template
+ *
+hit path (shared_syscall remains as before):
+  if (!INTERNAL_OPTION(unsafe_ignore_eflags_ibl)) {
+  | 5   movl  %eax,eax_OFFSET
+  | 1   lahf
+  | 3   seto  %al
+  }
+    6   movl  %ebx, ebx_offs(&dcontext)
+    2   movl  %ecx,%ebx                 # tag in ecx, hash will be in ebx
+    6   andl  $0x3fff,%ecx              # hash the tag
+    7   movl  ftable(,%ecx,4),%ecx      # ecx = ftable[hash]
+        # empty slot is not 0, instead is a constant frag w/ tag 0
+    2   cmpl  FRAGMENT_TAG_OFFS(%ecx),%ebx
+    2   jne   miss # if !DYNAMO_OPTION(indirect_stubs), jne ibl
+    6   movl  ebx_offs(&dcontext),%ebx
+    3   jmp   *FRAGMENT_START_PC_OFFS(%ecx)
+unlinked entry point into stub:
+ if (!DYNAMO_OPTION(indirect_stubs)) {
+     5  jmp   unlinked_ib_lookup  # we can eliminate this if we store stub pc
+ } else {
+  if (DYNAMO_OPTION(atomic_inlined_linking)) {
+        # duplicate miss path so linking can be atomic
+    10  movl  &linkstub, edi_offs(&dcontext)
+    5   jmp   unlinked_ib_lookup
+  } else {
+        # set flag in ecx (bottom byte = 0x1) so that unlinked path can
+        # detect race condition during unlinking
+    6   movl  %ecx, ebx_offs(&dcontext)
+    2   movb  $0x1, %ecx
+  }
+miss:
+    10  movl  &linkstub, edi_offs(&dcontext)
+    5   jmp   indirect_branch_lookup/(if !atomic_inlined_linking)unlinked_ib_lookup
+ }
+*/
+byte *
+emit_inline_ibl_stub(dcontext_t *dcontext, byte *pc, 
+                     ibl_code_t *ibl_code, bool target_trace_table)
+{
+    /* careful -- we're called in middle of setting up code fields, so don't go
+     * reading any without making sure they're initialized first
+     */
+    instrlist_t ilist;
+    instr_t *miss = NULL, *unlink, *after_unlink = NULL;
+
+    patch_list_t *patch =  &ibl_code->ibl_stub_patch;
+    byte *unlinked_ibl_pc = ibl_code->unlinked_ibl_entry;
+    byte *linked_ibl_pc = ibl_code->indirect_branch_lookup_routine;
+
+    bool absolute = !ibl_code->thread_shared_routine;
+
+    /* PR 248207: haven't updated the inlining to be x64-compliant yet.
+     * Keep in mind PR 257963: trace inline cmp needs separate entry. */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    /* no support for absolute addresses on x64: we always use tls/reg */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(!absolute));
+
+    ibl_code->inline_ibl_stub_template = pc;
+    ibl_code->ibl_head_is_inlined = true;
+
+    /* initialize the ilist and the patch list */
+    instrlist_init(&ilist);
+    /* FIXME: for !absolute need to optimize to PATCH_TYPE_INDIRECT_FS */
+    init_patch_list(patch, absolute ? PATCH_TYPE_ABSOLUTE : PATCH_TYPE_INDIRECT_XDI);
+
+    /*
+        <head>
+unlinked entry point into stub:
+  if (DYNAMO_OPTION(atomic_inlined_linking)) {
+        # duplicate miss path so linking can be atomic
+    10  movl  &linkstub, edi_offs(&dcontext)
+    5   jmp   unlinked_ib_lookup
+  } else {
+        # set flag in ecx (bottom byte = 0x1) so that unlinked path can
+        # detect race condition during unlinking
+    6   movl  %ecx, ebx_offs(&dcontext)
+    2   movb  $0x1, %ecx
+  }
+miss:
+    10  movl  &linkstub, edi_offs(&dcontext)
+    5   jmp   indirect_branch_lookup/(if !atomic_inlined_linking)unlinked_ib_lookup
+    */
+
+    if (DYNAMO_OPTION(indirect_stubs)) {
+        if (absolute) {
+            miss = INSTR_CREATE_mov_st(dcontext,
+                                       opnd_create_dcontext_field(dcontext, XDI_OFFSET),
+                                       OPND_CREATE_INT32(0));
+        } else {
+            miss = INSTR_CREATE_mov_st(dcontext,
+                                       OPND_TLS_FIELD(TLS_XDX_SLOT),
+                                       OPND_CREATE_INT32(0));
+        }
+        append_ibl_head(dcontext, &ilist, ibl_code, patch, NULL, NULL, NULL,
+                        opnd_create_instr(miss), true/*miss can have 8-bit offs*/,
+                        target_trace_table, true /* inline of course */ );     
+
+        /*>>>    SAVE_TO_UPCONTEXT %ebx,ebx_OFFSET                       */
+        /*>>>    SAVE_TO_UPCONTEXT &linkstub,edx_OFFSET                  */
+        /*>>>    jmp     unlinked_ib_lookup                              */
+        if (DYNAMO_OPTION(atomic_inlined_linking)) {
+            if (absolute) {
+                unlink = SAVE_TO_DC(dcontext, REG_EBX, XBX_OFFSET);
+                after_unlink =
+                    INSTR_CREATE_mov_st(dcontext,
+                                        opnd_create_dcontext_field(dcontext, 
+                                                                   XDI_OFFSET),
+                                        OPND_CREATE_INT32(0));
+            } else {
+                unlink = SAVE_TO_TLS(dcontext, REG_EBX, TLS_XBX_SLOT);
+                after_unlink = INSTR_CREATE_mov_st(dcontext,
+                                                   OPND_TLS_FIELD(TLS_XDX_SLOT),
+                                                   OPND_CREATE_INT32(0));
+            }
+            APP(&ilist, unlink);
+            APP(&ilist, after_unlink);
+            APP(&ilist, INSTR_CREATE_jmp(dcontext, 
+                                         opnd_create_pc(unlinked_ibl_pc)));
+        } else {
+            if (absolute)
+                unlink = SAVE_TO_DC(dcontext, REG_ECX, XBX_OFFSET);
+            else
+                unlink = SAVE_TO_TLS(dcontext, REG_ECX, TLS_XBX_SLOT);
+            APP(&ilist, unlink);
+            APP(&ilist, INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_CL), 
+                                             OPND_CREATE_INT8(1)));
+        }
+        APP(&ilist, miss);
+        APP(&ilist,
+            INSTR_CREATE_jmp(dcontext, 
+                             opnd_create_pc(DYNAMO_OPTION(atomic_inlined_linking) ? 
+                                            linked_ibl_pc : unlinked_ibl_pc)));
+    
+        add_patch_marker(patch, unlink, PATCH_UINT_SIZED /* pc relative */,
+                         0 /* beginning of instruction */,
+                         (ptr_uint_t*)&ibl_code->inline_unlink_offs);
+
+        if (DYNAMO_OPTION(atomic_inlined_linking)) {
+            add_patch_marker(patch, after_unlink, PATCH_UINT_SIZED /* pc relative */,
+                             -4 /* grab last 4 bytes of instructions */,
+                             (ptr_uint_t*)&ibl_code->inline_linkstub_second_offs);
+            add_patch_marker(patch, instr_get_prev(miss),
+                             PATCH_UINT_SIZED /* pc relative */,
+                             1 /* skip jmp opcode */,
+                             (ptr_uint_t*)&ibl_code->inline_unlinkedjmp_offs);
+        }
+        add_patch_marker(patch, miss, PATCH_UINT_SIZED /* pc relative */, 
+                         -4 /* grab offsets that are last 4 bytes of instructions */,
+                         (ptr_uint_t*)&ibl_code->inline_linkstub_first_offs);
+        add_patch_marker(patch, instrlist_last(&ilist),
+                         PATCH_UINT_SIZED /* pc relative */, 
+                         1 /* skip jmp opcode */,
+                         (ptr_uint_t*)&ibl_code->inline_linkedjmp_offs);
+    } else {
+        instr_t *cmp;
+        append_ibl_head(dcontext, &ilist, ibl_code, patch, NULL, &cmp, NULL,
+                        opnd_create_pc(linked_ibl_pc), false/*miss needs 32-bit offs*/,
+                        target_trace_table, true /* inline of course */ );     
+        /* FIXME: we'd like to not have this jmp at all and instead have the cti
+         * go to the inlined stub when linked and straight to the unlinked ibl
+         * entry when unlinked but we haven't put in the support in the link
+         * routines (they all assume they can find the unlinked from the current
+         * target in a certain manner).
+         */
+        unlink = INSTR_CREATE_jmp(dcontext, opnd_create_pc(unlinked_ibl_pc));
+        APP(&ilist, unlink);
+        /* FIXME: w/ private traces and htable stats we have a patch entry
+         * inserted inside app_ibl_head (in append_ibl_found) at a later instr
+         * than the miss instr.  To fix, we must either put the miss patch point
+         * in the middle of the array and shift it over to keep it sorted, or
+         * enable patch-encode to handle out-of-order entries (we could mark
+         * this with a flag).
+         */
+#ifdef HASHTABLE_STATISTICS
+        ASSERT_NOT_IMPLEMENTED(!absolute || !INTERNAL_OPTION(hashtable_ibl_stats));
+#endif
+        /* FIXME: cleaner to have append_ibl_head pass back miss instr */
+        add_patch_marker(patch, instr_get_next(cmp), PATCH_UINT_SIZED /* pc relative */, 
+                         2 /* skip jne opcode */,
+                         (ptr_uint_t*)&ibl_code->inline_linkedjmp_offs);
+        /* FIXME: we would add a patch for inline_unlinkedjmp_offs at unlink+1,
+         * but encode_with_patch_list asserts, wanting 1 patch per instr, in order
+         */
+        add_patch_marker(patch, unlink, PATCH_UINT_SIZED /* pc relative */,
+                         0 /* beginning of instruction */,
+                         (ptr_uint_t*)&ibl_code->inline_unlink_offs);
+    }
+
+    ibl_code->inline_stub_length = encode_with_patch_list(dcontext, patch, &ilist, pc);
+
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+    return pc + ibl_code->inline_stub_length;
+}
+
+
+/* FIXME: case 5232 where this should really be smart - 
+ * for now always using jmp rel32 with statistics
+ *
+ * Use with caution where jmp_short would really work in release - no
+ * ASSERTs to help you.
+ */
+#ifdef HASHTABLE_STATISTICS
+#  define INSTR_CREATE_jmp_smart INSTR_CREATE_jmp
+#else
+#  define INSTR_CREATE_jmp_smart INSTR_CREATE_jmp_short
+#endif
+
+/*
+# indirect_branch_lookup
+#.If the lookup succeeds, control jumps to the fcache target; otherwise
+# it sets up for and jumps to fcache_return.  
+
+# when we unlink an indirect branch we go through the cleanup part of
+# this lookup routine that takes us straight to fcache_return.
+
+# We assume dynamo is NOT in trace creation mode (which would require
+# going back to dynamo here).  We assume that when a fragment is
+# unlinked its indirect branch exit stubs are redirected to the
+# unlinked_* labels below.  Note that even if you did come in here in
+# trace creation mode, and we didn't go back to dynamo here, the
+# current trace would have ended now (b/c next fragment is a trace),
+# so we'd end up possibly adding erroneous fragments to the end of
+# the trace but the indirect branch check would ensure they were never
+# executed.
+
+# N.B.: a number of optimizations of the miss path are possible by making
+# it separate from the unlink path
+*/
+/* Must have a valid fcache return pc prior to calling this function! */
+byte *
+emit_indirect_branch_lookup(dcontext_t *dcontext, generated_code_t *code, byte *pc,
+                            byte *fcache_return_pc,
+                            bool target_trace_table,
+                            bool inline_ibl_head,
+                            ibl_code_t *ibl_code /* IN/OUT */)
+{
+    instrlist_t ilist;
+    instr_t *fragment_not_found, *unlinked = INSTR_CREATE_label(dcontext);
+    instr_t *target_delete_entry;
+    instr_t *post_linkcount;
+    patch_list_t *patch = &ibl_code->ibl_patch;
+    bool absolute = !ibl_code->thread_shared_routine;
+    bool table_in_tls = SHARED_IB_TARGETS() &&
+        (target_trace_table || SHARED_BB_ONLY_IB_TARGETS()) &&
+        DYNAMO_OPTION(ibl_table_in_tls);
+    /* Use TLS only for spilling app state -- registers & flags */
+    bool only_spill_state_in_tls = !absolute && !table_in_tls;
+#ifdef HASHTABLE_STATISTICS
+    /* save app XDI since inc routine uses it */
+    bool save_xdi = !absolute && table_in_tls;
+#endif
+    instr_t *fragment_found;
+    instr_t *compare_tag = NULL;
+    instr_t *sentinel_check;
+    /* for IBL_COARSE_SHARED and !DYNAMO_OPTION(indirect_stubs) */
+    const linkstub_t *linkstub = NULL;
+
+    instr_t *next_fragment_nochasing = 
+        INSTR_CREATE_cmp(dcontext,
+                         OPND_CREATE_MEMPTR(REG_XCX, HASHLOOKUP_TAG_OFFS),
+                         OPND_CREATE_INT8(0));
+
+    /* no support for absolute addresses on x64: we always use tls/reg */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(!absolute));
+
+    if (ibl_code->source_fragment_type == IBL_COARSE_SHARED ||
+        !DYNAMO_OPTION(indirect_stubs)) {
+        linkstub =
+            get_ibl_sourceless_linkstub(ibltype_to_linktype(ibl_code->branch_type),
+                                        IBL_FRAG_FLAGS(ibl_code));
+    }
+
+    /* When the target_delete_entry is reached, all registers contain
+     * app state, except for those restored in a prefix. We need to massage
+     * the state so that it looks like the fragment_not_found -- IBL miss -
+     * path, so we need to restore %xbx. See more on the target_delete_entry
+     * below, where the instr is added to the ilist.
+     */
+    target_delete_entry = absolute ?
+        instr_create_save_to_dcontext(dcontext, REG_XBX, XBX_OFFSET) :
+        SAVE_TO_TLS(dcontext, REG_XBX, INDIRECT_STUB_SPILL_SLOT);
+    
+    fragment_not_found = INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XCX),
+                                             opnd_create_reg(REG_XBX));
+
+    /* initialize the ilist */
+    instrlist_init(&ilist);
+    init_patch_list(patch, absolute ? PATCH_TYPE_ABSOLUTE : PATCH_TYPE_INDIRECT_XDI);
+
+    LOG(THREAD, LOG_EMIT, 3,
+        "emit_indirect_branch_lookup: pc="PFX" fcache_return_pc="PFX"\n"
+        "target_trace_table=%d inline_ibl_head=%d absolute=%d\n",
+        pc, fcache_return_pc, target_trace_table, inline_ibl_head, absolute);
+
+    if (inline_ibl_head) {
+        /* entry point is next_fragment, expects
+         * 1) xbx = effective address of indirect branch
+         * 2) xcx = &fragment
+         * 3) xdx_slot = &linkstub
+         */
+    } else {
+        /* entry point: expects
+         * 1) xbx = &linkstub if DYNAMO_OPTION(indirect_stubs),
+         *          or src tag if DYNAMO_OPTION(coarse_units)
+         * 2) xcx = effective address of indirect branch
+         */
+#ifdef X64
+        instr_t *trace_cmp_entry;
+#endif
+        append_ibl_head(dcontext, &ilist, ibl_code, patch, &fragment_found, &compare_tag,
+                        IF_X64_ELSE(&trace_cmp_entry, NULL),
+                        opnd_create_instr(next_fragment_nochasing),
+                        true/*miss can have 8-bit offs*/, target_trace_table,
+                        inline_ibl_head);
+#ifdef X64
+        if (IS_IBL_TRACE(ibl_code->source_fragment_type) && !code->x86_mode) {
+            /* if -unsafe_ignore_eflags_ibl this will equal regular entry */
+            add_patch_marker(patch, trace_cmp_entry, PATCH_ASSEMBLE_ABSOLUTE, 
+                             0 /* beginning of instruction */, 
+                             (ptr_uint_t*)&ibl_code->trace_cmp_entry);
+        }
+#endif
+    }
+    /* next_fragment_nochasing: */
+    /*>>>    cmp     $0, HASHLOOKUP_TAG_OFFS(%xcx)                   */
+    APP(&ilist, next_fragment_nochasing);
+
+    /* forward reference to sentinel_check */
+    if (INTERNAL_OPTION(ibl_sentinel_check)) {
+        /* sentinel_check: */
+        /* check if at table end sentinel */
+        /* One solution would be to compare xcx to
+         * &lookuptable[ftable->capacity-1] (sentinel) while it would
+         * work great for thread private IBL routines where we can
+         * hardcode the address.  
+         *  >>>a)   cmp     %xcx, HASHLOOKUP_SENTINEL_ADDR
+                                  ;; &lookuptable[ftable->capacity-1] (sentinel)
+         * For shared routines currently we'd need to walk a few
+         * pointers - we could just put that one TLS to avoid pointer
+         * chasing.  Yet if we are to have even one extra memory load
+         * anyways, it is easier to just store a special start_pc to
+         * compare instead
+         *  >>>b)   cmp     4x8(%xcx), HASHLOOKUP_SENTINEL_PC
+         * Where the expectation is that null_fragment=(0,0) while
+         * sentinel_fragment=(0,1) For simplicity we just use b) even
+         * in private IBL routines.
+         */
+        /* we can use 8-bit immed, will be sign-expanded before cmp */
+        ASSERT((int)(ptr_int_t)HASHLOOKUP_SENTINEL_START_PC <= INT8_MAX &&
+               (int)(ptr_int_t)HASHLOOKUP_SENTINEL_START_PC >= INT8_MIN);
+        sentinel_check = INSTR_CREATE_cmp(dcontext,
+                                          OPND_CREATE_MEMPTR(REG_XCX, HASHLOOKUP_START_PC_OFFS),
+                                          OPND_CREATE_INT8((int)(ptr_int_t)
+                                                           HASHLOOKUP_SENTINEL_START_PC));
+    } else {
+        /* sentinel handled in C code 
+         * just exit back to dispatch  
+         */
+        sentinel_check = fragment_not_found;
+    }
+
+    /*>>>    je      sentinel_check                                  */
+    /* FIXME: je_short ends up not reaching target for shared inline! */
+    APP(&ilist,
+        INSTR_CREATE_jcc(dcontext, OP_je_short, opnd_create_instr(sentinel_check)));
+
+    /* For open address hashing xcx = &lookuptable[h]; to get &lt[h+1] just add 8x16
+     *   add xcx, 8x16  # no wrap around check, instead rely on a nulltag sentinel entry 
+     * alternative method of rehashing xbx+4x8 or without checks is also not efficient
+     */
+# ifdef HASHTABLE_STATISTICS
+    if (INTERNAL_OPTION(hashtable_ibl_stats)) {
+        if (save_xdi)
+            APP(&ilist, SAVE_TO_TLS(dcontext, REG_XDI, HTABLE_STATS_SPILL_SLOT));
+        append_increment_counter(dcontext, &ilist, ibl_code, patch,
+                                 REG_NULL,
+                                 HASHLOOKUP_STAT_OFFS(collision),
+                                 REG_NULL); /* no registers dead */
+        if (save_xdi)
+            APP(&ilist, RESTORE_FROM_TLS(dcontext, REG_XDI, HTABLE_STATS_SPILL_SLOT));
+    }
+# endif
+    APP(&ilist, 
+        INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XCX),
+                         opnd_create_base_disp(REG_XCX, REG_NULL, 0,
+                                               sizeof(fragment_entry_t), OPSZ_lea)));
+
+    if (inline_ibl_head) {
+        compare_tag = INSTR_CREATE_cmp(dcontext,
+                                       OPND_CREATE_MEMPTR(REG_XCX, HASHLOOKUP_TAG_OFFS),
+                                       opnd_create_reg(REG_XBX));
+        APP(&ilist, compare_tag);
+
+        /*  TODO: check whether the static predictor can help here */
+        /*  P4OG:2-18 "Use prefix 3E (DS) for taken and 2E (CS) for not taken cbr" (DS == PREFIX_DATA) */
+        APP(&ilist,
+            INSTR_CREATE_jcc(dcontext, OP_jne_short, opnd_create_instr(next_fragment_nochasing)));
+
+        append_ibl_found(dcontext, &ilist, ibl_code, patch,
+                         HASHLOOKUP_START_PC_OFFS, true, only_spill_state_in_tls,
+                         target_trace_table ?
+                         DYNAMO_OPTION(trace_single_restore_prefix) :
+                         DYNAMO_OPTION(bb_single_restore_prefix),
+                         NULL);
+    } else {
+        /* case 5232: use INSTR_CREATE_jmp_smart, since release builds can use a short jump */
+        APP(&ilist, INSTR_CREATE_jmp_smart(dcontext, opnd_create_instr(compare_tag)));
+    }
+
+    if (INTERNAL_OPTION(ibl_sentinel_check)) {
+        /* check if at table end sentinel */
+        APP(&ilist, sentinel_check);
+
+        /* not found, if not at end of table sentinel fragment  */
+        /*>>>    jne      fragment_not_found                        */
+        APP(&ilist,
+            INSTR_CREATE_jcc(dcontext, OP_jne_short, opnd_create_instr(fragment_not_found)));
+
+# ifdef HASHTABLE_STATISTICS
+        if (INTERNAL_OPTION(hashtable_ibl_stats)) {
+            if (save_xdi)
+                APP(&ilist, SAVE_TO_TLS(dcontext, REG_XDI, HTABLE_STATS_SPILL_SLOT));
+            append_increment_counter(dcontext, &ilist, ibl_code, patch,
+                                     REG_NULL,
+                                     HASHLOOKUP_STAT_OFFS(overwrap),
+                                     REG_NULL); /* no registers dead */
+            if (save_xdi) {
+                APP(&ilist,
+                    RESTORE_FROM_TLS(dcontext, REG_XDI, HTABLE_STATS_SPILL_SLOT));
+            }
+        }
+# endif
+
+        /* should overwrap to beginning of table, if at end of table sentinel */
+        /* for private table */
+        /*>>>    mov     BOGUS_HASH_TABLE -> %xcx  ; &lookuptable[0]   */
+
+        /* for shared table - table address should still be preserved in XDI
+         *        mov    %xdi -> %xcx        ; xdi == &lookuptable[0] 
+         */
+        if (absolute) {
+            /* lookuptable is a patchable immediate */
+            enum {BOGUS_HASH_TABLE = 0xabcdabcd};
+            instr_t *table = INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XCX),
+                                                OPND_CREATE_INT32(BOGUS_HASH_TABLE));
+            add_patch_entry(patch, table, PATCH_PER_THREAD, 
+                            GET_IB_FTABLE(ibl_code, target_trace_table, table));
+            APP(&ilist, table);
+        } else if (table_in_tls) {
+            /* grab lookuptable from tls */
+            APP(&ilist, RESTORE_FROM_TLS(dcontext, REG_XCX,
+                                         TLS_TABLE_SLOT(ibl_code->branch_type)));
+        } else {
+# ifdef HASHTABLE_STATISTICS
+            if (INTERNAL_OPTION(hashtable_ibl_stats)) {
+                /* The hash stats inc routine clobbers XDI so we need to reload
+                 * it and then reload per_thread_t* and then the table*. */
+                append_shared_get_dcontext(dcontext, &ilist, false);
+                APP(&ilist,
+                    INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XDI),
+                                        OPND_DC_FIELD(absolute, dcontext, OPSZ_PTR, 
+                                                      FRAGMENT_FIELD_OFFSET)));
+                /* We could load directly into XCX but since hash stats are on,
+                 * we assume that this isn't a performance-sensitive run and
+                 * opt for code simplicity by rematerializing XDI.
+                 */
+                /* this is an offset in per_thread_t so should fit in 32 bits */
+                IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_int
+                              (GET_IB_FTABLE(ibl_code, target_trace_table, table))));
+                APP(&ilist,
+                    INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XDI),
+                                        opnd_create_base_disp
+                                        (REG_XDI, REG_NULL, 0,
+                                         (int) GET_IB_FTABLE(ibl_code,
+                                                             target_trace_table, table),
+                                         OPSZ_PTR)));
+            }
+# endif
+            /* XDI should still point to lookuptable[0] */
+            APP(&ilist, INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XCX),
+                                            opnd_create_reg(REG_XDI)));
+        }
+
+        /*>>>    jmp    compare_tag                                 */
+        /* FIXME: should fit in a jmp_short for release builds */
+        /* case 5232: use INSTR_CREATE_jmp_smart here */
+        APP(&ilist, INSTR_CREATE_jmp(dcontext, opnd_create_instr(compare_tag)));
+    } 
+
+    /* there is no fall-through through here: we insert separate entry points here */
+
+#ifdef X64
+    if (IS_IBL_TRACE(ibl_code->source_fragment_type) && !code->x86_mode) {
+        if (DYNAMO_OPTION(unsafe_ignore_eflags_trace)) { /*==unsafe_ignore_eflags_ibl */
+            /* trace_cmp link and unlink entries are identical to regular */
+            add_patch_marker(patch, unlinked, PATCH_ASSEMBLE_ABSOLUTE, 
+                             0 /* beginning of instruction */, 
+                             (ptr_uint_t*)&ibl_code->trace_cmp_unlinked);
+        } else if (inline_ibl_head) {
+            /* For inlining we can't reuse the eflags restore below, so
+             * we insert our own
+             */
+            instr_t *trace_cmp_unlinked = INSTR_CREATE_label(dcontext);
+            APP(&ilist, trace_cmp_unlinked);
+            add_patch_marker(patch, trace_cmp_unlinked, PATCH_ASSEMBLE_ABSOLUTE, 
+                             0 /* beginning of instruction */, 
+                             (ptr_uint_t*)&ibl_code->trace_cmp_unlinked);
+            insert_restore_eflags(dcontext, &ilist, NULL, 0, true/*tls*/, false/*!abs*/);
+            APP(&ilist, INSTR_CREATE_jmp(dcontext, opnd_create_instr(unlinked)));
+        }
+    }
+#endif
+
+    /****************************************************************************
+     * target_delete_entry
+     */
+
+    /*>>>  target_delete_entry:                                     */
+    /* This entry point aids in atomic hashtable deletion. A        */
+    /* hashtable entry's start_pc_fragment is redirected to here    */
+    /* when the entry's fragment is being deleted. It's a prefix to */
+    /* the fragment_not_found path and so leads to a cache exit.    */
+    /* The regular not_found path skips over these instructions,    */
+    /* directly to the fragment_not_found entry.                    */
+    /*                                                              */
+    /* If coming from a shared ibl, xbx is NOT in the mcontext, which our
+     * miss path restore assumes -- so we put it there now.  If coming from
+     * a private ibl or a no-prefix-target ibl, this is simply a redundant
+     * store.  Xref case 4649.
+     */
+    /*>>>    SAVE_TO_UPCONTEXT %xbx,xbx_OFFSET                       */
+    APP(&ilist, target_delete_entry);
+
+    if (linkstub == NULL) {
+        /* If coming from an inlined ibl, the linkstub was not stored, so
+         * we use a special linkstub_t in the last_exit "slot" (xdi / tls xdx) for any
+         * source (xref case 4635).
+         * Rare enough that should be ok, and everyone, including trace building,
+         * can handle it.  Although w/ an unknown last_exit the trace builder has to
+         * assume the final exit was taken, that's only bad when ending in a cbr,
+         * and when that's the case won't end up here (have to have -inline_bb_ibl
+         * to get here, since we only add bbs to traces).
+         */
+#ifdef X64
+        /* xbx is now dead so we can use it */
+        APP(&ilist, INSTR_CREATE_mov_imm
+            (dcontext, opnd_create_reg(REG_XBX),
+             OPND_CREATE_INTPTR((ptr_int_t)get_ibl_deleted_linkstub())));
+#endif
+        if (absolute) {
+            APP(&ilist, instr_create_save_immed_to_dcontext
+                (dcontext, (int)(ptr_int_t) get_ibl_deleted_linkstub(), XDI_OFFSET));
+        } else if (table_in_tls) {
+            APP(&ilist, INSTR_CREATE_mov_st
+                (dcontext, OPND_TLS_FIELD(TLS_XDX_SLOT),
+                 IF_X64_ELSE(opnd_create_reg(REG_XBX),
+                             OPND_CREATE_INTPTR((ptr_int_t)get_ibl_deleted_linkstub()))));
+        } else {
+            append_shared_get_dcontext(dcontext, &ilist, true); /* doesn't touch xbx */
+            APP(&ilist, INSTR_CREATE_mov_st
+                (dcontext, OPND_DC_FIELD(absolute, dcontext, OPSZ_PTR, XDI_OFFSET),
+                 IF_X64_ELSE(opnd_create_reg(REG_XBX),
+                             OPND_CREATE_INTPTR((ptr_int_t)get_ibl_deleted_linkstub()))));
+            append_shared_restore_dcontext_reg(dcontext, &ilist);
+        }
+    } /* else later will fill in fake linkstub anyway.
+       * FIXME: for -no_indirect_stubs, is this source of add_ibl curiosities on IIS?
+       * but one at least was a post-syscall!
+       */
+
+    /* load the tag value from the table ptr in xcx into xbx, so    */
+    /* that it gets shuffled into xcx by the following instruction  */
+    /*>>>    mov     (%xcx), %xbx                                   */
+    if (!ibl_use_target_prefix(ibl_code)) {
+        /* case 9688: for no-prefix-target ibl we stored the tag in xax slot
+         * in the hit path.  we also restored eflags already.
+         */
+        if (absolute) {
+            APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XBX, XAX_OFFSET));
+        } else {
+            APP(&ilist, RESTORE_FROM_TLS(dcontext, REG_XBX, DIRECT_STUB_SPILL_SLOT));
+        }
+        if (!INTERNAL_OPTION(unsafe_ignore_eflags_ibl)) {
+            /* save flags so we can re-use miss path below */
+            insert_save_eflags(dcontext, &ilist, NULL, 0,
+                               IBL_EFLAGS_IN_TLS(), absolute);
+        }
+    } else {
+        APP(&ilist,
+            INSTR_CREATE_mov_ld(dcontext, 
+                                opnd_create_reg(REG_XBX),
+                                OPND_CREATE_MEMPTR(REG_XCX, FRAGMENT_TAG_OFFS)));
+    }
+
+    /* Add to the patch list right away; hashtable stats could be added
+     * further later, so if we don't add now the patch ordering becomes
+     * confused.
+     */
+    add_patch_marker(patch, target_delete_entry, PATCH_ASSEMBLE_ABSOLUTE, 
+                     0 /* beginning of instruction */, 
+                     (ptr_uint_t*)&ibl_code->target_delete_entry);
+
+    /****************************************************************************
+     * fragment_not_found
+     */
+
+    /* put target back in xcx to match regular unlinked path, 
+     * the unlinked inlined indirect branch race condition case 
+     * also comes here (if !atomic_inlined_linking) */
+    /*>>>  fragment_not_found:
+     *>>>    mov     %xbx, %xcx                                      */
+    APP(&ilist, fragment_not_found);
+
+    /* This counter will also get the unlink inline indirect branch race
+     * condition cases (if !atomic_inlined_linking), but that should almost 
+     * never happen so don't worry about it screwing up the count */
+#ifdef HASHTABLE_STATISTICS
+    if (INTERNAL_OPTION(hashtable_ibl_stats)) {
+        if (save_xdi)
+            APP(&ilist, SAVE_TO_TLS(dcontext, REG_XDI, HTABLE_STATS_SPILL_SLOT));
+        append_increment_counter(dcontext, &ilist, ibl_code, patch,
+                                 REG_NULL,
+                                 HASHLOOKUP_STAT_OFFS(miss),
+                                 REG_XBX); /* xbx dead */
+        if (save_xdi)
+            APP(&ilist, RESTORE_FROM_TLS(dcontext, REG_XDI, HTABLE_STATS_SPILL_SLOT));
+    }
+#endif
+    if (only_spill_state_in_tls) {
+        /* get dcontext in register (xdi) */
+        append_shared_get_dcontext(dcontext, &ilist, false /* xdi is dead */);
+    }
+
+    /* for inlining we must restore flags prior to xbx restore: but when not
+     * inlining we reverse them so that trace_cmp entry can come in at the restore */
+    if (inline_ibl_head) {
+        if (!INTERNAL_OPTION(unsafe_ignore_eflags_ibl)) {
+            /* restore flags + xax (which we only need so save below works) */
+            insert_restore_eflags(dcontext, &ilist, NULL, 0, IBL_EFLAGS_IN_TLS(),
+                                  absolute);
+        }
+        APP(&ilist, unlinked);
+    }
+    if (DYNAMO_OPTION(indirect_stubs)) {
+        /* restore scratch xbx from **xdi / tls xdx ** offset */
+        /*>>>    RESTORE_FROM_UPCONTEXT xdi_OFFSET,%xbx                  */
+        if (absolute) {
+            APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XBX, XDI_OFFSET));
+        } else if (table_in_tls) {
+            APP(&ilist, RESTORE_FROM_TLS(dcontext, REG_XBX, TLS_XDX_SLOT));
+        } else {
+            ASSERT(only_spill_state_in_tls);
+            APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XBX, XDI_OFFSET));
+        }
+    } else {
+        /* restore xbx */
+        if (absolute)
+            APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XBX, XBX_OFFSET));
+        else
+            APP(&ilist, RESTORE_FROM_TLS(dcontext, REG_XBX, INDIRECT_STUB_SPILL_SLOT));
+    }
+
+    if (only_spill_state_in_tls) {
+        append_shared_restore_dcontext_reg(dcontext, &ilist);
+    }
+
+    if (!inline_ibl_head) {
+        /* see above: when not inlining we do eflags restore after xbx restore */
+#ifdef X64
+        if (IS_IBL_TRACE(ibl_code->source_fragment_type) && !code->x86_mode &&
+            !DYNAMO_OPTION(unsafe_ignore_eflags_trace)) {
+            instr_t *trace_cmp_unlinked = INSTR_CREATE_label(dcontext);
+            APP(&ilist, trace_cmp_unlinked);
+            add_patch_marker(patch, trace_cmp_unlinked, PATCH_ASSEMBLE_ABSOLUTE, 
+                             0 /* beginning of instruction */, 
+                             (ptr_uint_t*)&ibl_code->trace_cmp_unlinked);
+        }
+#endif
+        if (!INTERNAL_OPTION(unsafe_ignore_eflags_ibl)) {
+            /* restore flags + xax (which we only need so save below works) */
+            insert_restore_eflags(dcontext, &ilist, NULL, 0, IBL_EFLAGS_IN_TLS(),
+                                  absolute);
+        }
+        APP(&ilist, unlinked);
+    }
+
+    if (!absolute) {
+        append_shared_get_dcontext(dcontext, &ilist, true /* save register */);
+    }
+    /* note we are now saving XAX to the dcontext - no matter where it
+     * was saved before for saving and restoring eflags.  FIXME: in
+     * some incarnations of this routine it is redundant, yet this is
+     * the slow path anyways
+     */
+    APP(&ilist, SAVE_TO_DC(dcontext, REG_XAX, XAX_OFFSET));
+
+    /* Indirect exit stub: we have app XBX in slot1, and linkstub in XBX */
+    /*   fcache_return however is geared for direct exit stubs which uses XAX */
+    /*   app XBX is properly restored  */
+    /*   XAX gets the linkstub_ptr */
+    /*   app XAX is saved in slot1 */
+    /* FIXME: this all can be cleaned up at the cost of an extra byte in 
+       direct exit stubs to use XBX */
+
+    if (ibl_code->source_fragment_type == IBL_COARSE_SHARED) {
+        /* Coarse-grain uses the src tag plus sourceless but type-containing
+         * fake linkstubs.  Here we put the src from xbx into its special slot.
+         */
+        ASSERT(DYNAMO_OPTION(coarse_units));
+        APP(&ilist, SAVE_TO_DC(dcontext, REG_XBX, COARSE_IB_SRC_OFFSET));
+        ASSERT(linkstub != NULL);
+    }
+
+    if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)) {
+        /* in order to write next_tag we must unprotect -- and we don't
+         * have safe stack yet!  so we duplicate fcache_return code here,
+         * but we keep xcx w/ next tag around until we can store it as next_tag.
+         */
+        /* need to save xax (was never saved before) */
+        /*>>>    SAVE_TO_UPCONTEXT %xax,xax_OFFSET                  */
+        /* put &linkstub where dispatch expects it */
+        /*>>>    mov     %xbx,%xax                                       */
+        if (linkstub == NULL) {
+            APP(&ilist, INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XAX),
+                                            opnd_create_reg(REG_XBX)));
+        } else {
+            APP(&ilist, INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XAX),
+                                             OPND_CREATE_INTPTR((ptr_int_t)linkstub)));
+        }
+        append_fcache_return_common(dcontext, code, &ilist, true/*ibl end*/, absolute,
+                                    false/*!shared*/, NULL, false/*no coarse info*/);
+    } else {
+        /* set up for fcache_return: save xax, put xcx in next_tag, &linkstub in xax */
+        /*>>>    SAVE_TO_UPCONTEXT %xax,xax_OFFSET                       */
+        /*>>>    SAVE_TO_DCONTEXT %xcx,next_tag_OFFSET                   */
+        /*>>>    mov     %xbx,%xax                                       */
+        /*>>>    RESTORE_FROM_UPCONTEXT xbx_OFFSET,%xbx                  */
+        /*>>>    RESTORE_FROM_UPCONTEXT xcx_OFFSET,%xcx                  */
+        /*>>>    jmp     _fcache_return                                  */
+        APP(&ilist, SAVE_TO_DC(dcontext, REG_XCX, NEXT_TAG_OFFSET));
+
+        if (linkstub == NULL) {
+            post_linkcount = INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XAX),
+                                                 opnd_create_reg(REG_XBX));
+        } else {
+            /* there is no exit-specific stub -- we use a generic one here */
+            post_linkcount = INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XAX),
+                                                  OPND_CREATE_INTPTR((ptr_int_t)linkstub));
+        }
+#ifdef PROFILE_LINKCOUNT
+        /* not supported for inline_ibl_head, so we can assume xbx==&linkstub
+         * assumes xcx is free */
+        if (dynamo_options.profile_counts &&
+            IS_IBL_TRACE(ibl_code->source_fragment_type)) {
+            append_linkcount_incr(dcontext, &ilist, false/*preserve eflags*/,
+                                  post_linkcount);
+        }
+#endif
+        APP(&ilist, post_linkcount);
+
+        if (absolute) {
+            if (DYNAMO_OPTION(indirect_stubs))
+                APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XBX, XBX_OFFSET));
+            /* else, xbx never spilled */
+        } else /* table_in_tls || only_spill_state_in_tls */ {
+            if (DYNAMO_OPTION(indirect_stubs)) {
+                /* restore XBX from EXIT_STUB_SPILL_SLOT */
+                APP(&ilist, RESTORE_FROM_TLS(dcontext, REG_XBX,
+                                             INDIRECT_STUB_SPILL_SLOT));
+            } /* else, xbx never spilled */
+            /* now need to juggle with app XAX to be in DIRECT_STUB_SPILL_SLOT, using XCX */
+            APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XCX, XAX_OFFSET)); 
+            APP(&ilist, SAVE_TO_TLS(dcontext, REG_XCX, DIRECT_STUB_SPILL_SLOT));
+            /* DIRECT_STUB_SPILL_SLOT has XAX value as needed for fcache_return_shared */
+        }
+        
+        /* We need to restore XCX from TLS for shared IBL routines, but from
+         * mcontext for private IBL routines (unless private_ib_in_tls is set)
+         */
+        if (ibl_code->thread_shared_routine || DYNAMO_OPTION(private_ib_in_tls)) {
+            APP(&ilist, RESTORE_FROM_TLS(dcontext, REG_XCX, MANGLE_XCX_SPILL_SLOT));
+        } else {
+            APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XCX, XCX_OFFSET));            
+        }
+        
+        if (!absolute) {
+            /* restore from scratch the XDI register even if fcache_return will
+             * save it again */
+            append_shared_restore_dcontext_reg(dcontext, &ilist);
+        }
+        /* pretending we came from a direct exit stub - linkstub in XAX, all
+         * other app registers restored */
+        APP(&ilist, INSTR_CREATE_jmp(dcontext, opnd_create_pc(fcache_return_pc)));
+    }
+    
+    if (inline_ibl_head && !DYNAMO_OPTION(atomic_inlined_linking)) {
+        /* #ifdef HASHTABLE_STATISTICS
+         * >>> race_condition_inc:
+         * >>>   #note that eflags are already saved in this path
+         * >>>   <inc_stat>
+         * >>>   jmp fragment_not_found
+         * #endif
+         * >>>   #detect unlink path flag to check for unlinking race condition
+         * >>>   #must be eflags safe, they are prob. not saved yet
+         * >>> unlinked:
+         * >>>   movzx %cl, %xcx
+         * >>>   # xcx now holds 1 in the unlink case, and the zero extended
+         * >>>   # lower byte of a pointer into the hashtable in the race
+         * >>>   # condition case (since our pointers into the hashtable are
+         * >>>   # aligned this can't be 1), the loop will jmp if xcx != 1
+         * #ifdef HASHTABLE_STATISTICS
+         * >>>   loop race_condition_inc #race condition handling path
+         * #else
+         * >>>   loop fragment_not_found  #race condition handling path
+         * #endif
+         * >>>   #normal unlink path
+         * >>>   RESTORE_FROM_UPCONTEXT xbx_offset, %xcx
+         * >>>   SAVE_TO_UPCONTEXT %xbx, xbx_offset
+         * >>>   jmp old_unlinked
+         */
+        instr_t *old_unlinked_target = instr_get_next(unlinked);
+        instr_t *race_target = fragment_not_found;
+#ifdef HASHTABLE_STATISTICS
+        if (INTERNAL_OPTION(hashtable_ibl_stats)) {
+            race_target = instrlist_last(&ilist);
+            if (save_xdi)
+                APP(&ilist, SAVE_TO_TLS(dcontext, REG_XDI, HTABLE_STATS_SPILL_SLOT));
+            append_increment_counter(dcontext, &ilist, ibl_code, patch,
+                                     REG_NULL,
+                                     HASHLOOKUP_STAT_OFFS(race_condition),
+                                     REG_XCX);
+            if (save_xdi) {
+                APP(&ilist,
+                    RESTORE_FROM_TLS(dcontext, REG_XDI, HTABLE_STATS_SPILL_SLOT));
+            }
+            APP(&ilist,
+                INSTR_CREATE_jmp_short(dcontext, opnd_create_instr(fragment_not_found)));
+        }
+        race_target = instr_get_next(race_target);
+#endif
+        instrlist_remove(&ilist, unlinked);
+        APP(&ilist, unlinked);
+        APP(&ilist, INSTR_CREATE_movzx(dcontext, opnd_create_reg(REG_XCX),
+                                       opnd_create_reg(REG_CL)));
+        add_patch_marker(patch, unlinked, PATCH_ASSEMBLE_ABSOLUTE, 
+                         0 /* beginning of instruction */, 
+                         (ptr_uint_t*)&ibl_code->unlinked_ibl_entry);
+        /* subtract 1 from xcx and jmp if !=0 (race condition case) */
+        APP(&ilist, INSTR_CREATE_loop(dcontext, 
+                                      opnd_create_instr(race_target)));
+#ifdef HASHTABLE_STATISTICS
+        if (INTERNAL_OPTION(hashtable_ibl_stats)) {
+            if (save_xdi)
+                APP(&ilist, SAVE_TO_TLS(dcontext, REG_XDI, HTABLE_STATS_SPILL_SLOT));
+            append_increment_counter(dcontext, &ilist, ibl_code, patch,
+                                     REG_NULL,
+                                     HASHLOOKUP_STAT_OFFS(unlinked_count),
+                                     REG_XCX);
+            if (save_xdi) {
+                APP(&ilist,
+                    RESTORE_FROM_TLS(dcontext, REG_XDI, HTABLE_STATS_SPILL_SLOT));
+            }
+            else if (only_spill_state_in_tls) /* restore app %xdi */
+                append_shared_restore_dcontext_reg(dcontext, &ilist);
+        }
+#endif
+        if (absolute) {
+            APP(&ilist, RESTORE_FROM_DC(dcontext, REG_XCX, XBX_OFFSET));
+            APP(&ilist, SAVE_TO_DC(dcontext, REG_XBX, XBX_OFFSET));
+        } else {
+            APP(&ilist, RESTORE_FROM_TLS(dcontext, REG_XCX, XBX_OFFSET));
+            APP(&ilist, SAVE_TO_TLS(dcontext, REG_XBX, XBX_OFFSET));
+        }
+        APP(&ilist, INSTR_CREATE_jmp_short(dcontext, opnd_create_instr(old_unlinked_target)));
+    } else {
+        /* get a patch marker at the instruction the label is at: */
+#ifdef HASHTABLE_STATISTICS
+        if (INTERNAL_OPTION(hashtable_ibl_stats)) {
+            instr_t *old_unlinked = instr_get_next(unlinked);
+            instrlist_remove(&ilist, unlinked);
+            APP(&ilist, unlinked);
+            add_patch_marker(patch, unlinked, PATCH_ASSEMBLE_ABSOLUTE, 
+                             0 /* beginning of instruction */, 
+                             (ptr_uint_t*)&ibl_code->unlinked_ibl_entry);
+            /* FIXME: for x64 -thread_private we enter here from trace,
+             * and not from top of ibl, so we must save xdi.  Is this true
+             * for all cases of only_spill_state_in_tls with !save_xdi?
+             * Maybe should be saved in append_increment_counter's call to
+             * append_shared_get_dcontext() instead.
+             */
+            if (IF_X64_ELSE(true, save_xdi)) {
+                APP(&ilist,
+                    SAVE_TO_TLS(dcontext, REG_XDI, HTABLE_STATS_SPILL_SLOT));
+            }
+            /* have to save eflags before increment, saving eflags clobbers
+             * xax slot, but that should be dead here */
+            insert_save_eflags(dcontext, &ilist, NULL, 0, !absolute, absolute);
+            append_increment_counter(dcontext, &ilist, ibl_code, patch,
+                                     REG_NULL,
+                                     HASHLOOKUP_STAT_OFFS(unlinked_count),
+                                     REG_XBX);
+            insert_restore_eflags(dcontext, &ilist, NULL, 0,
+                                  !absolute, absolute);
+            if (IF_X64_ELSE(true, save_xdi)) {
+                APP(&ilist,
+                    RESTORE_FROM_TLS(dcontext, REG_XDI, HTABLE_STATS_SPILL_SLOT));
+            }
+            else if (only_spill_state_in_tls) {
+                /* we didn't care that XDI got clobbered since it was spilled
+                 * at the entry point into the IBL routine but we do need to
+                 * restore app state now.
+                 */
+                append_shared_restore_dcontext_reg(dcontext, &ilist);
+            }
+            /* In a PROFILE_LINKCOUNT LINKCOUNT_64_BITS build with
+             * -prof_counts on, this jmp is too far for a jmp_short on a shared
+             * ibl.  We just use a regular jmp since this code isn't even in a
+             * release build. FIXME - would be a good place for a real
+             * jmp_smart opcode xref case 5232. */
+            APP(&ilist, INSTR_CREATE_jmp(dcontext, opnd_create_instr(old_unlinked)));
+        } else 
+#endif
+        {
+            add_patch_marker(patch, unlinked, PATCH_ASSEMBLE_ABSOLUTE, 
+                             0 /* beginning of instruction */, 
+                             (ptr_uint_t*)&ibl_code->unlinked_ibl_entry);
+        }
+    }
+
+#ifdef X64
+    if (code->x86_mode) {
+        /* We currently have x86 code parsing the regular x64 table (PR 283895 covers
+         * using an x86 table, for both full correctness and performance: for now
+         * we have no way to detect a source in one mode jumping to a target built in
+         * another mode w/o a mode switch, but that would be an app error anyway).
+         * Rather than complicating the REG_X* defines used above we have a post-pass
+         * that shrinks all the registers and all the INTPTR immeds.
+         * The other two changes we need are performed up above:
+         *   1) cmp top bits to 0 for match
+         *   2) no trace_cmp entry points
+         * Note that we're punting on PR 283152: we go ahead and clobber the top bits
+         * of all our scratch registers.
+         */
+        instr_t *in;
+        for (in = instrlist_first(&ilist); in != NULL; in = instr_get_next(in)) {
+            instr_shrink_to_32_bits(in);
+        }
+    }
+#endif
+
+    ibl_code->ibl_routine_length = encode_with_patch_list(dcontext, patch, &ilist, pc);
+
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+
+    return pc + ibl_code->ibl_routine_length;
+}
+
+static inline void
+update_ibl_routine(dcontext_t *dcontext, ibl_code_t *ibl_code)
+{
+    if (!ibl_code->initialized)
+        return;
+    patch_emitted_code(dcontext, &ibl_code->ibl_patch, 
+                       ibl_code->indirect_branch_lookup_routine);
+    DOLOG(2, LOG_EMIT, {
+        const char *ibl_name;
+        const char *ibl_brtype;
+        ibl_name = get_ibl_routine_name(dcontext, 
+                                        ibl_code->indirect_branch_lookup_routine,
+                                        &ibl_brtype);
+        LOG(THREAD, LOG_EMIT, 2, "Just updated indirect branch lookup\n%s_%s:\n",
+            ibl_name, ibl_brtype);
+        disassemble_with_annotations(dcontext, &ibl_code->ibl_patch, 
+                                     ibl_code->indirect_branch_lookup_routine,
+                                     ibl_code->indirect_branch_lookup_routine + ibl_code->ibl_routine_length);
+    });
+
+    if (ibl_code->ibl_head_is_inlined) {
+        patch_emitted_code(dcontext, &ibl_code->ibl_stub_patch, ibl_code->inline_ibl_stub_template);
+        DOLOG(2, LOG_EMIT, {
+            const char *ibl_name;
+            const char *ibl_brtype;
+            ibl_name = get_ibl_routine_name(dcontext, 
+                                            ibl_code->indirect_branch_lookup_routine,
+                                            &ibl_brtype);
+            LOG(THREAD, LOG_EMIT, 2, "Just updated inlined stub indirect branch lookup\n%s_template_%s:\n",
+                ibl_name, ibl_brtype);
+            disassemble_with_annotations(dcontext, &ibl_code->ibl_stub_patch, ibl_code->inline_ibl_stub_template,
+                                         ibl_code->inline_ibl_stub_template + ibl_code->inline_stub_length);
+        });
+    }
+}
+
+void
+update_indirect_branch_lookup(dcontext_t *dcontext)
+{
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+
+    ibl_branch_type_t branch_type;
+#ifdef X64
+    ASSERT(is_shared_gencode(code));
+    return; /* nothing to do: routines are all thread-shared */
+#endif
+    protect_generated_code(code, WRITABLE);
+    for (branch_type = IBL_BRANCH_TYPE_START; branch_type < IBL_BRANCH_TYPE_END; branch_type++) {
+        update_ibl_routine(dcontext, &code->bb_ibl[branch_type]);
+        if (PRIVATE_TRACES_ENABLED() && !DYNAMO_OPTION(shared_trace_ibl_routine))
+            update_ibl_routine(dcontext, &code->trace_ibl[branch_type]);
+    }
+#ifdef WINDOWS
+    /* update mask and table in inlined ibl at end of syscall routine */
+    if (DYNAMO_OPTION(shared_syscalls)) {
+        patch_emitted_code(dcontext, &code->shared_syscall_code.ibl_patch,
+                           code->unlinked_shared_syscall);
+        DOLOG(2, LOG_EMIT, {
+            LOG(THREAD, LOG_EMIT, 2, "Just updated shared syscall routine:\n");       
+            disassemble_with_annotations(dcontext,
+                                         &code->shared_syscall_code.ibl_patch,
+                                         code->unlinked_shared_syscall,
+                                         code->end_shared_syscall);
+        });
+    }
+#endif
+    protect_generated_code(code, READONLY);
+}
+
+#ifdef RETURN_STACK /********************************************************/
+
+byte *
+emit_return_lookup(dcontext_t *dcontext, byte *pc,
+                   byte *indirect_branch_lookup_pc,
+                   byte *unlinked_ib_lookup_pc,
+                   byte **unlinked_return_pc)
+{
+    instr_t instr;
+    byte *prev_pc;
+    uint len;
+    byte *end_pc;
+    int past_ret = 0;
+    instr_init(dcontext, &instr);
+
+    /* PR 248210: unsupported feature on x64 */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+
+    /* first copy the assembly-code version */
+    ASSERT_TRUNCATE(len, uint,
+                    (ptr_uint_t)end_return_lookup - (ptr_uint_t)return_lookup);
+    len = (uint) ((ptr_uint_t)end_return_lookup - (ptr_uint_t)return_lookup);
+    memcpy((void *)pc, (void *)return_lookup, len);
+    end_pc = pc + len;
+
+    /* set unlinked entry point */
+    ASSERT_TRUNCATE(len, uint,
+                    (ptr_uint_t)unlinked_return - (ptr_uint_t)return_lookup);
+    len = (uint) ((ptr_uint_t)unlinked_return - (ptr_uint_t)return_lookup);
+    *unlinked_return_pc = pc + len;
+
+    /* now touch up addresses & offsets
+     */
+    do {
+        prev_pc = pc;
+        instr_reset(dcontext, &instr);
+        pc = decode(dcontext, pc, &instr);
+        ASSERT(instr_valid(&instr)); /* our own code! */
+        /* note we cannot look for pc targets, we'd have to decode the
+         * static code for that, so we assume the final jmps (past the
+         * ret) go to unlinked_ib_lookup and indirect_branch_lookup
+         */
+        if (past_ret == 1 && instr_is_ubr(&instr)) {
+            instr_set_target(&instr, opnd_create_pc(unlinked_ib_lookup_pc));
+            past_ret = 2;
+        }
+        else if (past_ret == 2 && instr_is_ubr(&instr)) {
+            instr_set_target(&instr, opnd_create_pc(indirect_branch_lookup_pc));
+        }
+        else if (instr_get_opcode(&instr) == OP_mov_ld &&
+                 opnd_is_near_base_disp(instr_get_src(&instr, 0)) &&
+                 opnd_get_base(instr_get_src(&instr, 0)) == REG_NULL &&
+                 opnd_get_index(instr_get_src(&instr, 0)) == REG_NULL) {
+            /* if not really dcontext value, update_ will return old value */
+            instr_set_src(&instr, 0, 
+                          update_dcontext_address(instr_get_src(&instr, 0),
+                                                  &shared_dcontext, dcontext));
+        }
+        else if (instr_get_opcode(&instr) == OP_mov_st &&
+                 opnd_is_near_base_disp(instr_get_dst(&instr, 0)) &&
+                 opnd_get_base(instr_get_dst(&instr, 0)) == REG_NULL &&
+                 opnd_get_index(instr_get_dst(&instr, 0)) == REG_NULL) {
+            /* if not really dcontext value, update_ will return old value */
+            instr_set_dst(&instr, 0, 
+                          update_dcontext_address(instr_get_dst(&instr, 0),
+                                                  &shared_dcontext, dcontext));
+        }
+        else if (instr_get_opcode(&instr) == OP_cmp &&
+                 opnd_is_near_base_disp(instr_get_src(&instr, 0)) &&
+                 opnd_get_base(instr_get_src(&instr, 0)) == REG_NULL &&
+                 opnd_get_index(instr_get_src(&instr, 0)) == REG_NULL) {
+            /* if not really dcontext value, update_ will return old value */
+            instr_set_src(&instr, 0, 
+                          update_dcontext_address(instr_get_src(&instr, 0),
+                                                  &shared_dcontext, dcontext));
+        } else if (instr_is_return(&instr))
+            past_ret = 1;
+        if (!instr_raw_bits_valid(&instr)) {
+            byte *nxt_pc = instr_encode(dcontext, &instr, prev_pc);
+            /* instructions must not change size! */
+            ASSERT(nxt_pc != NULL && nxt_pc == pc);
+        }
+    } while (pc < end_pc);
+
+    instr_free(dcontext, &instr);
+
+    return end_pc;
+}
+
+#endif /* RETURN_STACK */ /*************************************************/
+
+static instr_t *
+create_int_syscall_instr(dcontext_t *dcontext)
+{
+#ifdef WINDOWS
+    /* On windows should already be initialized by syscalls_init() */
+    ASSERT(get_syscall_method() != SYSCALL_METHOD_UNINITIALIZED);
+    /* int $0x2e */
+    if (DYNAMO_OPTION(sygate_int)) {
+        /* ref case 5217, we call to an existing int in NtYieldExecution
+         * to avoid tripping up Sygate. */
+        return INSTR_CREATE_call(dcontext, opnd_create_pc(int_syscall_address));
+    } else {
+        return INSTR_CREATE_int(dcontext, opnd_create_immed_int((char)0x2e, OPSZ_1));
+    }
+#else
+    /* if uninitialized just guess int, we'll patch up later */
+    /* int $0x80 */
+    return INSTR_CREATE_int(dcontext, opnd_create_immed_int((char)0x80, OPSZ_1));
+#endif
+}
+
+instr_t *
+create_syscall_instr(dcontext_t *dcontext)
+{
+    int method = get_syscall_method();
+    if (method == SYSCALL_METHOD_INT || method == SYSCALL_METHOD_UNINITIALIZED) {
+        return create_int_syscall_instr(dcontext);
+    } else if (method == SYSCALL_METHOD_SYSENTER) {
+        return INSTR_CREATE_sysenter(dcontext);
+    } else if (method == SYSCALL_METHOD_SYSCALL) {
+        return INSTR_CREATE_syscall(dcontext);
+    }
+#ifdef WINDOWS
+    else if (method == SYSCALL_METHOD_WOW64) {
+        /* call *fs:0xc0 */
+        return INSTR_CREATE_call_ind(dcontext,
+                                     opnd_create_far_base_disp(SEG_FS, REG_NULL,
+                                                               REG_NULL, 0,
+                                                               WOW64_TIB_OFFSET,
+                                                               OPSZ_4_short2));
+    }
+#endif
+    else {
+        ASSERT_NOT_REACHED();
+        return NULL;
+    }
+}
+
+#ifdef WINDOWS
+
+/* All system call instructions turn into a jump to an exit stub that
+ * jumps here, with the xsi slot in dcontext (or the mangle-next-tag tls slot
+ * for -shared_fragment_shared_syscalls) containing the return address
+ * after the original system call instr, and xbx containing the linkstub ptr.
+ *
+ * Unlinked version of shared_syscall is needed, even though syscalls are
+ * not part of traces (we unlink for other reasons, like flushing or
+ * in-trace replacement).
+ * To make unlinked entry point, have to make completely separate routine
+ * that calls unlinked_ibl instead of indirect_branch_lookup, or else
+ * common linked case needs an extra conditional.  I chose the latter
+ * approach.  I figure an extra load and jecxz won't be noticeable.
+ * Another reason is that this approach means there is a single system
+ * call instruction to check for suspended threads at, instead of two.
+ * To make the jecxz match forward-not-taken I actually add another store
+ * on the linked path.
+ * FIXME: is this a perf hit that makes it worth the code complexity
+ * of two syscall routines?
+ * FIXME: The 'target_trace_table' indicates whether the trace or BB IBT
+ * table should be targetted. If BB2BB IBL is used (when trace building is
+ * not disabled), then both traces and BBs use the same shared syscall.
+ * (We emit only one.) So we can't target the BB table since that would
+ * result in missed opportunities to mark secondary trace heads (trace->BB
+ * IB transitions after shared syscall). So for BB2BB IBL this could be
+ * a perf hit, but not a regression compared to not using BB2BB IBL. More
+ * comments below in the routine.
+ *
+_unlinked_shared_syscall:
+        SAVE_TO_UPCONTEXT $0,xax_OFFSET # flag: use unlinked ibl; xcx tls if all_shared
+        jmp skip_linked
+_shared_syscall:
+        SAVE_TO_UPCONTEXT $1,xax_OFFSET # flag: use regular ibl; xcx tls if all_shared
+skip_linked:
+        .ifdef SIDELINE
+        # clear cur-trace field so we don't think cur trace is still running
+        mov     $0, _sideline_trace
+        .endif
+
+        .if all_shared
+        SAVE_TO_TLS xdi, xdi_offset
+        RESTORE_FROM_TLS xdi, dcontext_offset
+        .endif
+
+        .if !all_shared && DYNAMO_OPTION(shared_fragment_shared_syscalls)
+          .if !sysenter_syscall_method
+              LOAD_FROM_TLS MANGLE_NEXT_TAG_SLOT,%xdi
+              SAVE_TO_UPCONTEXT %xdi,xsi_OFFSET
+          .endif
+        RESTORE_FROM_TLS xdi_OFFSET
+        .endif
+
+        # make registers have app values for interrupt
+        .if !INTERNAL_OPTION(shared_syscalls_fastpath)
+        SAVE_TO_UPCONTEXT %xbx,xdi_OFFSET # save linkstub ptr
+           .if all_shared
+           # get next_tag (from xcx tls slot) into upcontext, for callback dcontext swap
+           RESTORE_FROM_TLS xbx, mangle_next_tag_slot
+           SAVE_TO_UPCONTEXT xbx, xsi_OFFSET
+           .endif
+           # %xbx is stored in TLS if shared fragments can target shared syscall
+           .if DYNAMO_OPTION(shared_fragment_shared_syscalls)
+           LOAD_FROM_TLS INDIRECT_STUB_SPILL_SLOT,%xbx # restore app's xbx
+           .else
+           RESTORE_FROM_UPCONTEXT xbx_OFFSET,%xbx # restore app's xbx
+           .endif
+        .endif
+
+        .if sysenter_syscall_method
+        pop     xsi_OFFSET
+        push    <after-syscall-address>
+        .endif
+
+        # even if !DYNAMO_OPTION(syscalls_synch_flush) must set for reset
+        movl 1, at_syscall_OFFSET # indicate to flusher we're in a syscall
+
+        .if all_shared
+        SAVE_TO_UPCONTEXT  xdi, xdi_offset
+        RESTORE_FROM_TLS xdi, xdi_offset
+        .endif
+
+        # system call itself
+        int     $0x2e
+        # kernel may decide to run a callback here...but when we come
+        #   back we can't tell the difference
+        
+        .if all_shared
+        RESTORE_FROM_TLS xdi, dcontext_offset
+        .endif
+
+        # even if !DYNAMO_OPTION(syscalls_synch_flush) must clear for cbret
+        movl 0, at_syscall_OFFSET # indicate to flusher/dispatch we're done w/ syscall
+
+        # assume interrupt could have changed register values
+        .if !inline_ibl_head # else, saved inside inlined ibl
+           # for shared_fragment_shared_syscalls = true, absolute != true
+           .if !DYNAMO_OPTION(shared_fragment_shared_syscalls)
+           SAVE_TO_UPCONTEXT %xbx,xbx_OFFSET
+           .endif
+           .if !absolute
+           SAVE_TO_TLS %xbx,INDIRECT_STUB_SPILL_SLOT
+           .endif
+           .if !INTERNAL_OPTION(shared_syscalls_fastpath)
+           RESTORE_FROM_UPCONTEXT xdi_OFFSET,%xbx # bring back linkstub ptr
+           .endif
+        .endif
+
+        # now set up for indirect_branch_lookup
+        .if !DYNAMO_OPTION(shared_fragment_shared_syscalls)
+        SAVE_TO_UPCONTEXT %xcx,xcx_OFFSET
+        .endif
+        .if !absolute && !all_shared
+        SAVE_TO_TLS %xcx,MANGLE_XCX_SPILL_SLOT
+        .endif
+
+        .if all_shared
+        xchg  xcx-tls, xcx # get link/unlink flag, and save app xcx, at once
+          .if x64
+           mov   ecx,ecx # clear top 32 bits of flag
+           .endif
+        .else
+        RESTORE_FROM_UPCONTEXT xax_OFFSET,%xcx # get link/unlink flag
+        .endif
+        
+        # patch point: jecxz -> jmp for shared_syscall unlink
+        jecxz unlink
+
+        .if INTERNAL_OPTION(shared_syscalls_fastpath)
+        mov     shared-syscalls-bb-linkstub, %xbx # set linkstub ptr
+           .if inline_ibl_head
+           SAVE_TO_UPCONTEXT %xbx,xdi_OFFSET # save linkstub ptr
+           .endif
+        .endif
+
+        # linked code
+        RESTORE_FROM_UPCONTEXT xsi_OFFSET,%xcx # bring back return address
+        .if !inline_ibl_head
+        jmp     _indirect_branch_lookup
+        .else
+        # inline ibl lookup head here! (don't need unlink/miss, already did
+        #   that work, miss goes straight to ibl routine)
+        .endif
+
+unlink:
+        # unlinked code
+        RESTORE_FROM_UPCONTEXT xsi_OFFSET,%xcx # bring back return address
+        .if !inline_ibl_head
+        mov  @shared_syscall_unlinked_linkstub,%xbx
+        .else
+           .if absolute
+           SAVE_TO_UPCONTEXT @shared_syscall_unlinked_linkstub,xdi_OFFSET
+           .else
+           SAVE_TO_TLS @shared_syscall_unlinked_linkstub,INDIRECT_STUB_SPILL_SLOT
+           .endif
+           .if !DYNAMO_OPTION(atomic_inlined_linking)
+           SAVE_TO_UPCONTEXT %xcx,xbx_offset
+           movb  $0x1, %cl
+           .else
+           SAVE_TO_UPCONTEXT %xbx,xbx_OFFSET # could have changed in kernel
+           .endif
+        .endif
+
+        jmp     _unlinked_ib_lookup
+ */
+byte *
+emit_shared_syscall(dcontext_t *dcontext, generated_code_t *code, byte *pc,
+                    ibl_code_t *ibl_code,
+                    patch_list_t *patch,
+                    byte *ind_br_lookup_pc, byte *unlinked_ib_lookup_pc,
+                    bool target_trace_table,
+                    bool inline_ibl_head,
+                    bool thread_shared,
+                    byte **shared_syscall_pc)
+{
+    instrlist_t ilist;
+    byte *start_pc = pc;
+    instr_t *syscall; /* remember after-syscall pc b/c often suspended there */
+    /* relative labels */
+    instr_t *linked, *jecxz, *unlink, *skip_syscall = NULL;
+    bool absolute = !thread_shared;
+    uint after_syscall_ptr = 0;
+    uint syscall_method = get_syscall_method();
+    instr_t *adjust_tos;
+    /* thread_shared indicates whether ibl is thread-shared: this bool indicates
+     * whether this routine itself is all thread-shared */
+    bool all_shared = IF_X64_ELSE(true, false); /* PR 244737 */
+
+    /* no support for absolute addresses on x64: we always use tls */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(!absolute));
+    /* x64 always shares shared_syscall fragments */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(DYNAMO_OPTION(shared_fragment_shared_syscalls)));
+    /* PR 248207: haven't updated the inlining to be x64-compliant yet */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(!inline_ibl_head));
+
+    /* PR 284029: for now we assume there are no syscalls in x86 code.
+     * To support them we need to update this routine, emit_do_syscall*,
+     * and emit_detach_callback_code().
+     */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(!ibl_code->x86_mode));
+
+    /* ibl_code was not initialized by caller */
+    ibl_code->thread_shared_routine = thread_shared;
+    ibl_code->branch_type = IBL_SHARED_SYSCALL;
+
+    /* initialize the ilist */
+    instrlist_init(&ilist);
+    init_patch_list(patch, absolute ? PATCH_TYPE_ABSOLUTE : PATCH_TYPE_INDIRECT_XDI);
+    /* We should generate some thread-shared code when
+     * shared_fragment_shared_syscalls=true. */
+    DODEBUG({
+        if (DYNAMO_OPTION(shared_fragment_shared_syscalls))
+            ASSERT(!absolute);
+    });
+    LOG(THREAD, LOG_EMIT, 3,
+        "emit_shared_syscall: pc="PFX" patch="PFX
+        " inline_ibl_head=%d thread shared=%d\n",
+        pc, patch, inline_ibl_head, thread_shared);
+
+    /* FIXME: could save space by storing a single byte, and using movzx into ecx
+     * below before the jecxz
+     */
+    if (all_shared) {
+        /* xax and xbx tls slots are taken so we use xcx */
+        linked = INSTR_CREATE_mov_st(dcontext,
+                                     OPND_TLS_FIELD_SZ(MANGLE_XCX_SPILL_SLOT, OPSZ_4),
+                                     OPND_CREATE_INT32(1));
+    } else
+        linked = instr_create_save_immed_to_dcontext(dcontext, 1, XAX_OFFSET);
+    APP(&ilist, linked);
+    add_patch_marker(patch, instrlist_first(&ilist), PATCH_ASSEMBLE_ABSOLUTE, 
+                     0 /* beginning of instruction */, (ptr_uint_t*)shared_syscall_pc);
+
+# ifdef SIDELINE
+    if (dynamo_options.sideline) {
+        /* clear cur-trace field so we don't think cur trace is still running */
+        APP(&ilist, INSTR_CREATE_mov_st
+            (dcontext, OPND_CREATE_ABSMEM((void *)&sideline_trace, OPSZ_4),
+             OPND_CREATE_INT32(0)));
+    }
+# endif
+
+    if (all_shared) {
+        /* load %xdi w/ dcontext */
+        append_shared_get_dcontext(dcontext, &ilist, true/*save xdi*/);
+    }
+
+    /* for all-shared we move next tag from tls down below once xbx is dead */
+    if (!all_shared && DYNAMO_OPTION(shared_fragment_shared_syscalls)) {
+        if (syscall_method != SYSCALL_METHOD_SYSENTER) {
+            /* Move the next tag field from TLS into the proper slot. */
+            APP(&ilist,
+                INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XDI),
+                                    opnd_create_tls_slot(os_tls_offset(MANGLE_NEXT_TAG_SLOT))));
+            APP(&ilist,
+                instr_create_save_to_dcontext(dcontext, REG_XDI, XSI_OFFSET));
+        }
+        /* restore app %xdi */
+        append_shared_restore_dcontext_reg(dcontext, &ilist);
+    }
+
+    /* put linkstub ptr in slot such that when inlined it will be
+     * in the right place in case of a miss */
+    if (!INTERNAL_OPTION(shared_syscalls_fastpath) && DYNAMO_OPTION(indirect_stubs)) {
+        /* even if inline_ibl_head and !absolute, we must put into mcontext
+         * here since tls is not saved on callback stack
+         */
+        if (all_shared) {
+            APP(&ilist,
+                instr_create_save_to_dc_via_reg(dcontext, REG_NULL/*default*/,
+                                                REG_XBX, XDI_OFFSET));
+        } else {
+            APP(&ilist,
+                instr_create_save_to_dcontext(dcontext, REG_XBX, XDI_OFFSET));
+        }
+    } else {
+        /* FIXME: for -no_indirect_stubs, we need our own complete ibl
+         * here in order to use our own linkstub_t.  For now we just use
+         * a trace jmp* linkstub_t from the ibl we target, making every
+         * post-non-ignorable-syscall fragment a trace head.
+         */
+    }
+
+    if (all_shared) {
+        /* move next_tag from tls into dcontext, for callback dcontext swap,
+         * using dead xbx */
+        if (!DYNAMO_OPTION(indirect_stubs)) {
+            /* xbx isn't dead */
+            APP(&ilist, instr_create_save_to_tls
+                (dcontext, REG_XBX, INDIRECT_STUB_SPILL_SLOT));
+        }
+        APP(&ilist,
+            instr_create_restore_from_tls(dcontext, REG_XBX, MANGLE_NEXT_TAG_SLOT));
+        APP(&ilist,
+            instr_create_save_to_dc_via_reg(dcontext, REG_NULL/*default*/,
+                                            REG_XBX, XSI_OFFSET));
+        if (!DYNAMO_OPTION(indirect_stubs)) {
+            /* restore xbx */
+            APP(&ilist, instr_create_restore_from_tls
+                (dcontext, REG_XBX, INDIRECT_STUB_SPILL_SLOT));
+        }
+    }
+
+    /* make registers have app values for the interrupt */
+    /* restore app's xbx (if we went through a stub to get here) */
+    if (!INTERNAL_OPTION(shared_syscalls_fastpath) && DYNAMO_OPTION(indirect_stubs)) {
+        if (DYNAMO_OPTION(shared_fragment_shared_syscalls)) {
+            APP(&ilist,
+                INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XBX),
+                                    opnd_create_tls_slot(os_tls_offset(INDIRECT_STUB_SPILL_SLOT))));
+        }
+        else {
+            APP(&ilist,
+                instr_create_restore_from_dcontext(dcontext, REG_XBX, XBX_OFFSET));
+        }
+    }
+    if (syscall_method == SYSCALL_METHOD_SYSENTER) {
+        /* PR 248210: not bothering to make x64-ready: if we do, be sure to pop into
+         * next-tag tls */
+        IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+        /* For sysenter, mangle pushed the next tag onto the stack,
+         * so we pop it into the xsi slot and push the [to-be-patched]
+         * after-syscall address.
+         */
+        /* We have to save xsp in case a callback is delivered and we later detach
+         * (since detach expects the callback dcontext xsp to be correct). xref 9889 */
+        APP(&ilist, instr_create_save_to_dcontext(dcontext, REG_XSP, XSP_OFFSET));
+        APP(&ilist,
+            INSTR_CREATE_pop(dcontext,
+                             opnd_create_dcontext_field(dcontext, XSI_OFFSET)));
+        adjust_tos = INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32(0));
+        APP(&ilist, adjust_tos);
+        add_patch_marker(patch, adjust_tos, PATCH_ASSEMBLE_ABSOLUTE,
+                         1 /* offset of imm field */,
+                         (ptr_uint_t *)&after_syscall_ptr);
+    }
+    /* even if !DYNAMO_OPTION(syscalls_synch_flush) must set for reset */
+    ASSERT(!TEST(SELFPROT_DCONTEXT, DYNAMO_OPTION(protect_mask)));
+    if (all_shared) {
+        /* readers of at_syscall are ok w/ us not quite having xdi restored yet */
+        APP(&ilist, INSTR_CREATE_mov_st
+            (dcontext,
+             opnd_create_dcontext_field_via_reg_sz(dcontext, REG_NULL/*default*/,
+                                                   AT_SYSCALL_OFFSET, OPSZ_4),
+             OPND_CREATE_INT32(1)));
+        /* restore app %xdi */
+        append_shared_restore_dcontext_reg(dcontext, &ilist);
+    } else
+        APP(&ilist, instr_create_save_immed_to_dcontext(dcontext, 1, AT_SYSCALL_OFFSET));
+
+    if (DYNAMO_OPTION(sygate_sysenter) &&
+        get_syscall_method() == SYSCALL_METHOD_SYSENTER) {
+        /* PR 248210: not bothering to make x64-ready */
+        IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+        /* case 5441 hack - set up stack so first return address points to ntdll
+         * Won't worry about arithmetic eflags since no one should care about
+         * those at a syscall, will preserve other regs though. */
+        /* FIXME - what is the perf impact of these extra 5 instructions, we can
+         * prob. do better. */
+        /* note we assume xsp == xdx (if doesn't we already have prob. ref 
+         * case 5461) */
+        /* current state
+         *     xsi_slot = next_pc
+         *     xsp -> after_shared_syscall
+         *      +4 -> app value1
+         * desired state
+         *     sysenter_storage_slot = app_value1
+         *     xsp -> sysenter_ret_address (ntdll ret)
+         *      +4 -> after_shared_syscall
+         */
+        /* NOTE - the stack mangling must match that of handle_system_call()
+         * and intercept_nt_continue() as not all routines looking at the stack
+         * differentiate. */
+        /* pop stack leaving old value (after_shared_syscall) in place */
+        APP(&ilist, INSTR_CREATE_add(dcontext, opnd_create_reg(REG_XSP),
+                                     OPND_CREATE_INT8(4)));
+        APP(&ilist,
+            INSTR_CREATE_pop(dcontext,
+                             opnd_create_dcontext_field(dcontext,
+                                                        SYSENTER_STORAGE_OFFSET)));
+        /* instead of pulling in the existing stack value we could just patch in
+         * the after syscall imm */
+        /* see intel docs, source calculated before xsp dec'ed so we're pushing two
+         * stack slots up into the next slot up */
+        APP(&ilist, INSTR_CREATE_push(dcontext, OPND_CREATE_MEM32(REG_XSP, -8)));
+        APP(&ilist, INSTR_CREATE_push_imm
+            (dcontext, OPND_CREATE_INTPTR((ptr_int_t)sysenter_ret_address)));
+    }
+
+    /* syscall itself */
+    APP(&ilist, create_syscall_instr(dcontext));
+    syscall = instrlist_last(&ilist);
+
+    if (DYNAMO_OPTION(sygate_sysenter) &&
+        get_syscall_method() == SYSCALL_METHOD_SYSENTER) {
+        /* PR 248210: not bothering to make x64-ready */
+        IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+        /* case 5441 hack - we popped an extra stack slot, need to fill with saved
+         * app value */
+        APP(&ilist,
+            INSTR_CREATE_push(dcontext,
+                              opnd_create_dcontext_field(dcontext,
+                                                         SYSENTER_STORAGE_OFFSET)));
+    }
+
+    /* Now that all instructions from the linked entry point up to and
+     * including the syscall have been added, prepend the unlinked path
+     * instructions. We wait until the syscall has been added because when
+     * shared_syscalls_fastpath = true and "int 2e" syscalls are used, the
+     * target of the unlinked path's jmp is the syscall itself.
+     */
+    /* these two in reverse order since prepended */
+    instrlist_prepend(&ilist, INSTR_CREATE_jmp
+                      (dcontext, opnd_create_instr(instr_get_next(linked))));
+    if (all_shared) {
+        /* xax and xbx tls slots are taken so we use xcx */
+        instrlist_prepend(&ilist, INSTR_CREATE_mov_st
+                          (dcontext, 
+                           /* simpler to do 4 bytes even on x64 */
+                           OPND_TLS_FIELD_SZ(MANGLE_XCX_SPILL_SLOT, OPSZ_4),
+                           OPND_CREATE_INT32(0)));
+    } else {
+        instrlist_prepend(&ilist,
+                          instr_create_save_immed_to_dcontext(dcontext, 0, XAX_OFFSET));
+    }
+
+    /* even if !DYNAMO_OPTION(syscalls_synch_flush) must clear for cbret */
+    if (all_shared) {
+        /* readers of at_syscall are ok w/ us spilling xdi first */
+        append_shared_get_dcontext(dcontext, &ilist, true/*save xdi*/);
+        APP(&ilist, INSTR_CREATE_mov_st
+            (dcontext, 
+             opnd_create_dcontext_field_via_reg_sz(dcontext, REG_NULL/*default*/,
+                                                   AT_SYSCALL_OFFSET, OPSZ_4),
+             OPND_CREATE_INT32(0)));
+    } else
+        APP(&ilist, instr_create_save_immed_to_dcontext(dcontext, 0, AT_SYSCALL_OFFSET));
+
+    if (!inline_ibl_head && DYNAMO_OPTION(indirect_stubs)) {
+        /* FIXME Can we remove the write to the mcontext for the !absolute
+         * case? Initial tests w/notepad crashed when doing so -- we should
+         * look deeper.
+         */
+        /* save app's xbx (assume interrupt could have changed it) */
+        /* Remember, shared_fragment_shared_syscalls=true means absolute=false,
+         * so for shared_fragment_shared_syscalls=true %xbx is saved in
+         * the !absolute "if" that follows.
+         */
+        if (!DYNAMO_OPTION(shared_fragment_shared_syscalls)) {
+            APP(&ilist,
+                instr_create_save_to_dcontext(dcontext, REG_XBX, XBX_OFFSET));
+        }
+        if (!absolute) {
+            /* save xbx in TLS so that downstream code can find it */
+            APP(&ilist, SAVE_TO_TLS(dcontext, REG_XBX, INDIRECT_STUB_SPILL_SLOT));
+        }
+        if (!INTERNAL_OPTION(shared_syscalls_fastpath)) {
+            if (all_shared) {
+                APP(&ilist,
+                    instr_create_restore_from_dc_via_reg(dcontext, REG_NULL/*default*/,
+                                                         REG_XBX, XDI_OFFSET));
+            } else {
+                APP(&ilist,
+                    instr_create_restore_from_dcontext(dcontext, REG_XBX, XDI_OFFSET));
+            }
+        }
+    } /* if inlined, xbx will be saved inside inlined ibl; if no indirect stubs,
+       * xbx will be saved in the ibl routine, or not at all if unlinked
+       */
+
+    /* set up for indirect_branch_lookup */
+    /* save app's xcx */
+    if (!DYNAMO_OPTION(shared_fragment_shared_syscalls))
+        APP(&ilist, instr_create_save_to_dcontext(dcontext, REG_XCX, XCX_OFFSET));
+    /* FIXME Can we remove the write to the mcontext for the !absolute
+     * case, as suggested above? */
+    if (!absolute && !all_shared/*done later*/) {
+        /* save xcx in TLS */
+        APP(&ilist, SAVE_TO_TLS(dcontext, REG_XCX, MANGLE_XCX_SPILL_SLOT));
+    }
+
+    if (!INTERNAL_OPTION(shared_syscalls_fastpath)) {
+        if (inline_ibl_head && DYNAMO_OPTION(indirect_stubs)) {
+            /* Need to move linkstub ptr from mcontext->xdi into tls.
+             * We couldn't put it directly there pre-syscall b/c tls
+             * is not saved on callback stack!
+             * We do this now to take advantage of xcx being dead.
+             */
+            APP(&ilist,
+                instr_create_restore_from_dcontext(dcontext, REG_XCX, XDI_OFFSET));
+            APP(&ilist, SAVE_TO_TLS(dcontext, REG_XCX, TLS_XDX_SLOT));
+        }
+    }
+
+    /* get link flag */
+    if (all_shared) {
+        /* we duplicate the restore from dc and restore of xdi on the link
+         * and unlink paths, rather than putting next_tag back into tls here
+         * (can't rely on that tls slot persisting over syscall w/ callbacks)
+         */
+        unlink = instr_create_restore_from_dc_via_reg
+            (dcontext, REG_NULL/*default*/, REG_XCX, XSI_OFFSET);
+        /* we stored 4 bytes so get 4 bytes back; save app xcx at same time */
+        APP(&ilist, INSTR_CREATE_xchg
+            (dcontext, OPND_TLS_FIELD(MANGLE_XCX_SPILL_SLOT), opnd_create_reg(REG_XCX)));
+# ifdef X64
+        /* clear top 32 bits */
+        APP(&ilist, INSTR_CREATE_mov_st
+            (dcontext, opnd_create_reg(REG_ECX), opnd_create_reg(REG_ECX)));
+# endif
+        /* app xdi is restored later after we've restored next_tag from xsi slot */
+    } else {
+        unlink = instr_create_restore_from_dcontext(dcontext, REG_XCX, XSI_OFFSET);
+        APP(&ilist, instr_create_restore_from_dcontext(dcontext, REG_XCX, XAX_OFFSET));
+    }
+    jecxz = INSTR_CREATE_jecxz(dcontext, opnd_create_instr(unlink));
+    APP(&ilist, jecxz);
+    /* put linkstub ptr in xbx */
+    if (INTERNAL_OPTION(shared_syscalls_fastpath) && DYNAMO_OPTION(indirect_stubs)) {
+        APP(&ilist, INSTR_CREATE_mov_imm
+            (dcontext, opnd_create_reg(REG_XBX),
+             OPND_CREATE_INTPTR((ptr_int_t)get_shared_syscalls_bb_linkstub())));
+        /* put linkstub ptr in slot such that when inlined it will be
+         * in the right place in case of a miss */
+        if (inline_ibl_head) {
+            if (absolute) {
+                APP(&ilist,
+                    instr_create_save_to_dcontext(dcontext, REG_XBX, XDI_OFFSET));
+            } else {
+                APP(&ilist, SAVE_TO_TLS(dcontext, REG_XBX, TLS_XDX_SLOT));
+            }
+        }
+    } /* else case is up above to use dead xcx reg */
+
+    /* Add a patch marker once we know that there's an instr in the ilist
+     * after the syscall. */
+    add_patch_marker(patch, instr_get_next(syscall) /* take addr of next instr */, 
+                     PATCH_UINT_SIZED /* pc relative */,
+                     0 /* beginning of instruction */,
+                     (ptr_uint_t*)&code->sys_syscall_offs);
+    add_patch_marker(patch, jecxz, PATCH_UINT_SIZED /* pc relative */,
+                     0 /* point at opcode of jecxz */,
+                     (ptr_uint_t*)&code->sys_unlink_offs);
+
+    /* put return address in xcx (was put in xsi slot by mangle.c, or in tls
+     * by mangle.c and into xsi slot before syscall for all_shared) */
+    if (all_shared) {
+        /* we duplicate the restore from dc and restore of xdi on the link
+         * and unlink paths: see note above */
+        APP(&ilist, instr_create_restore_from_dc_via_reg
+            (dcontext, REG_NULL/*default*/, REG_XCX, XSI_OFFSET));
+        /* restore app %xdi */
+        append_shared_restore_dcontext_reg(dcontext, &ilist);
+    } else
+        APP(&ilist, instr_create_restore_from_dcontext(dcontext, REG_XCX, XSI_OFFSET));
+
+    /* FIXME As noted in the routine's header comments, shared syscall targets
+     * the trace [IBT] table when both traces and BBs could be using it (when
+     * trace building is not disabled). Ideally, we want traces to target the
+     * trace table and BBs to target the BB table (when BB2BB IBL is on, that is).
+     * Since the BB IBT table usually holds non-trace head BBs as well as traces
+     * (including traces is option controlled), using it will doubtless lead to
+     * higher IBL hit rate, though it's unclear if there would be a visible
+     * impact on performance. Since BBs and traces use different fake linkstubs
+     * when executing thru shared syscall, we can detect what the last fragment
+     * was and conditionally jump to the ideal IBL routine.
+     *
+     * Since the EFLAGS at this point hold app state, we'd need to save/restore
+     * them prior to executing the IBL code if we used a 'cmp' followed by cond.
+     * branch. Or we could save the EFLAGS and jump to a new entry point in the
+     * IBL, one just after the 'seto'. (We'd have to move any load of %xdi
+     * with the dcontext to just below the 'seto'.)
+     *
+     * We could avoid conditional code altogether if both inline_trace_ibl
+     * and inline_bb_ibl are false. Instead of passing fake linkstub addresses
+     * from a fragment exit stub through shared syscall, we could pass the
+     * address of the IBL routine to jump to -- BB IBL for BBs and trace IBL
+     * for traces. Shared syscall would do an indirect jump to reach the proper
+     * routine. On an IBL miss, the address is passed through to dispatch, which
+     * can convert the address into the appropriate fake linkstub address (check
+     * if the address is within emitted code and equals either BB or trace IBL.)
+     * Since an address is being passed around and saved to the dcontext during
+     * syscalls, some of which could be relatively long, this is a security
+     * hole.
+     */
+    if (!inline_ibl_head) {
+        APP(&ilist, INSTR_CREATE_jmp(dcontext, opnd_create_pc(ind_br_lookup_pc)));
+    } else {
+        append_ibl_head(dcontext, &ilist, ibl_code, patch, NULL, NULL, NULL,
+                        opnd_create_pc(ind_br_lookup_pc),
+                        false/*miss cannot have 8-bit offs*/, 
+                        target_trace_table,
+                        inline_ibl_head);
+    }
+
+    /* unlink path (there can be no fall-through) */
+    APP(&ilist, unlink);
+    if (all_shared) {
+        append_shared_restore_dcontext_reg(dcontext, &ilist);
+    }
+    /* When traversing the unlinked entry path, since IBL is bypassed
+     * control reaches dispatch, and the target is (usually) added to the IBT
+     * table. But since the unlinked path was used, the target may already be
+     * present in the table so the add attempt is unnecessary and triggers an
+     * ASSERT in fragment_add_ibl_target().
+     *
+     * The add attempt is bypassed by moving an unlinked linkstub ptr into the
+     * correct place -- for inlined IBL, the %xdi slot, otherwise, %xbx. This will
+     * identify exits from the unlinked path. The stub's flags are set to 0
+     * to bypass the add IBL target attempt.
+     */
+    if (!inline_ibl_head) {
+        if (DYNAMO_OPTION(indirect_stubs)) {
+            APP(&ilist, INSTR_CREATE_mov_imm
+                (dcontext, opnd_create_reg(REG_XBX),
+                 OPND_CREATE_INTPTR((ptr_int_t)get_shared_syscalls_unlinked_linkstub())));
+        }
+    }
+    else {
+        if (absolute) {
+            APP(&ilist, instr_create_save_immed_to_dcontext
+                (dcontext, (int)(ptr_int_t)get_shared_syscalls_unlinked_linkstub(),
+                 XDI_OFFSET));
+        }
+        else {
+            APP(&ilist, INSTR_CREATE_mov_st
+                (dcontext, OPND_TLS_FIELD(TLS_XDX_SLOT),
+                 OPND_CREATE_INTPTR((ptr_int_t)get_shared_syscalls_unlinked_linkstub())));
+        }
+        if (!DYNAMO_OPTION(atomic_inlined_linking)) {
+            /* we need to duplicate the emit_inline_ibl_stub unlinking race 
+             * condition detection code here, before we jump to unlink
+             */
+            /*
+             * # set flag in xcx (bottom byte = 0x1) so that unlinked path can
+             * # detect race condition during unlinking
+             * 2   movb  $0x1, %cl
+            */
+            /* we expect target saved in xbx_offset */
+            if (absolute) {
+                APP(&ilist,
+                    instr_create_save_to_dcontext(dcontext, REG_XCX, XBX_OFFSET));
+            } else
+                APP(&ilist, SAVE_TO_TLS(dcontext, REG_XCX, TLS_XBX_SLOT));
+            APP(&ilist, INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_CL),
+                                             OPND_CREATE_INT8(1)));
+        } else {
+            /* xbx could have changed in kernel, unlink expects it saved */
+            if (absolute) {
+                APP(&ilist,
+                    instr_create_save_to_dcontext(dcontext, REG_XBX, XBX_OFFSET));
+            } else
+                APP(&ilist, SAVE_TO_TLS(dcontext, REG_XBX, TLS_XBX_SLOT));
+        }
+    }
+    APP(&ilist, INSTR_CREATE_jmp(dcontext, opnd_create_pc(unlinked_ib_lookup_pc)));
+
+    pc += encode_with_patch_list(dcontext, patch, &ilist, pc);
+    if (syscall_method == SYSCALL_METHOD_SYSENTER) {
+        ASSERT(after_syscall_ptr != 0);
+        IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+        *((uint *)(ptr_uint_t) after_syscall_ptr) =
+            (uint)(ptr_uint_t) (code->unlinked_shared_syscall + code->sys_syscall_offs);
+    }
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+
+    return pc;
+}
+
+
+static byte *
+emit_dispatch_template(dcontext_t *dcontext, byte *pc, uint offset)
+{
+    instrlist_t ilist;
+
+    /* PR 244737: we don't use this for x64 b/c syscall routines are thread-shared */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+
+    /* initialize the ilist */
+    instrlist_init(&ilist);
+
+    /* load %edi w/the dcontext */
+    append_shared_get_dcontext(dcontext, &ilist, true);
+
+    /* load the generated_code_t address */
+    APP(&ilist, INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_EDI),
+                                    OPND_DC_FIELD(false, dcontext, OPSZ_PTR,
+                                                  PRIVATE_CODE_OFFSET)));
+
+    /* jump thru the address in the offset */
+    APP(&ilist, INSTR_CREATE_jmp_ind(dcontext, OPND_CREATE_MEM32(REG_EDI, offset)));
+
+    pc = instrlist_encode(dcontext, &ilist, pc, false /* no instr targets */);
+    ASSERT(pc != NULL);
+
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+
+    return pc;
+}
+
+byte *
+emit_shared_syscall_dispatch(dcontext_t *dcontext, byte *pc)
+{
+    return emit_dispatch_template(dcontext, pc,
+                                  offsetof(generated_code_t, shared_syscall));
+}
+
+byte *
+emit_unlinked_shared_syscall_dispatch(dcontext_t *dcontext, byte *pc)
+{
+    return emit_dispatch_template(dcontext, pc, offsetof(generated_code_t,
+                                                         unlinked_shared_syscall));
+}
+
+/* Links the shared_syscall routine to go directly to the indirect branch
+ * lookup routine.
+ * If it is already linked, does nothing.
+ * Assumes caller takes care of any synchronization if this is called
+ * from other than the owning thread!
+ */
+/* NOTE the link/unlink of shared syscall is atomic w/respect to threads in the
+ * cache since is only single byte write (always atomic). */
+static void
+link_shared_syscall_common(generated_code_t *code)
+{
+    /* strategy: change "jmp unlink" back to "jecxz unlink" */
+    cache_pc pc;
+    if (code == NULL) /* shared_code_x86 */
+        return;
+    pc = code->unlinked_shared_syscall + code->sys_unlink_offs;
+    if (*pc != JECXZ_OPCODE) {
+        protect_generated_code(code, WRITABLE);
+        ASSERT(*pc == JMP_SHORT_OPCODE);
+        *pc = JECXZ_OPCODE;
+        protect_generated_code(code, READONLY);
+    }
+}
+
+void
+link_shared_syscall(dcontext_t *dcontext)
+{
+    ASSERT(IS_SHARED_SYSCALL_THREAD_SHARED || dcontext != GLOBAL_DCONTEXT);
+    if (dcontext == GLOBAL_DCONTEXT) {
+        link_shared_syscall_common(SHARED_GENCODE(false));
+        IF_X64(link_shared_syscall_common(SHARED_GENCODE(true)));
+    } else 
+        link_shared_syscall_common(THREAD_GENCODE(dcontext));
+}
+
+/* Unlinks the shared_syscall routine so it goes back to dispatch after
+ * the system call itself.
+ * If it is already unlinked, does nothing.
+ * Assumes caller takes care of any synchronization if this is called
+ * from other than the owning thread!
+ */
+static void
+unlink_shared_syscall_common(generated_code_t *code)
+{
+    /* strategy: change "jecxz unlink" to "jmp unlink" */
+    cache_pc pc;
+    if (code == NULL) /* shared_code_x86 */
+        return;
+    pc = code->unlinked_shared_syscall + code->sys_unlink_offs;
+    if (*pc != JMP_SHORT_OPCODE) {
+        protect_generated_code(code, WRITABLE);
+        ASSERT(*pc == JECXZ_OPCODE);
+        *pc = JMP_SHORT_OPCODE;
+        protect_generated_code(code, READONLY);
+    }
+}
+
+void
+unlink_shared_syscall(dcontext_t *dcontext)
+{
+    ASSERT(IS_SHARED_SYSCALL_THREAD_SHARED || dcontext != GLOBAL_DCONTEXT);
+    if (dcontext == GLOBAL_DCONTEXT) {
+        unlink_shared_syscall_common(SHARED_GENCODE(false));
+        IF_X64(unlink_shared_syscall_common(SHARED_GENCODE(true)));
+    } else 
+        unlink_shared_syscall_common(THREAD_GENCODE(dcontext));
+}
+
+#endif /* defined(WINDOWS) ****************************/
+
+#ifdef WINDOWS
+/* used by detach, this inlines the callback stack so that we can detach 
+ *
+ * we spill xax and xbx to the PID and TID (respectively) TLS slots until we find
+ * the thread private state at which point we switch to using it for spilling.  We
+ * use the TID slot (as opposed to the PEB slot that callback.c uses) because we need
+ * to get the TID anyways.
+ *
+ * note the counter walks backwards through the array of saved address (they are 
+ * stored in reverse order)
+ *
+ * FIXME - we clobber eflags, but those should be dead after a system call anyways.
+ *
+ * From emit_patch_syscall()
+ * after_shared_syscall:
+ *   jmp _after_do_syscall
+ *
+ * after_do_syscall:
+ *   mov xax -> PID in TEB
+ *   mov &callback_buf -> xax
+ *   jmp xax
+ *
+ *
+ * From emit_detach_callback_code()
+ * // xax is currently saved in PID slot of TEB
+ *  callback_buf:
+ *   xchg xbx, TID in TEB  // store xbx and get TID
+ *   mov &callback_state -> xax  //the array of detach_callback_stack_t
+ *  match_tid:
+ *   cmp xbx, thread_id_offset(xax)
+ *   je match_found
+ *   add xax, sizeof(detach_callback_stack_t)
+ *   jmp match_tid  // Note - infinite loop till find or crash (not clear what else to do)
+ *  match_found:  // xax now holds ptr to the detach_callback_stack_t for this thread
+ *   xchg xbx, TID in TEB  // restore tid & xbx
+ *   mov xbx -> xbx_save_offset(xax)
+ *   mov PID -> xbx
+ *   xchg xbx, PID in TEB  // restore pid, saved xax now in xbx
+ *   mov xbx -> xax_save_offset(xax)
+ *   mov xcx -> xcx_save_offset(xax)
+ *   mov count_offset(xax) -> xbx  // need count in regsiter for addr calculation below
+ *   sub xbx, 1
+ *   mov xbx -> count_offset(xax)
+ *   mov callback_addrs_offset(xax) -> xcx
+ *   mov (xcx + xbx*sizeof(app_pc)) -> xcx // xcx now holds the xip we need to go to
+ *   mov xcx -> target_offset(xax)
+ *   mov xcx_save_offset(xax) -> xcx
+ *   mov xbx_save_offset(xax) -> xbx
+ *   lea code_buf_offset(xax) -> xax
+ *   jmp xax
+ *
+ 214f1000 6764871e2400     xchg    fs:[0024],ebx
+ 214f1006 b800114f21       mov     eax,0x214f1100
+ 214f100b 3b18             cmp     ebx,[eax]
+ 214f100d 0f8408000000     je      214f101b
+ 214f1013 83c03c           add     eax,0x3c
+ 214f1016 e9f0ffffff       jmp     214f100b
+ 214f101b 6764871e2400     xchg    fs:[0024],ebx
+ 214f1021 895810           mov     [eax+0x10],ebx
+ 214f1024 bb5c040000       mov     ebx,0x45c
+ 214f1029 6764871e2000     xchg    fs:[0020],ebx
+ 214f102f 89580c           mov     [eax+0xc],ebx
+ 214f1032 894814           mov     [eax+0x14],ecx
+ 214f1035 8b5804           mov     ebx,[eax+0x4]
+ 214f1038 83eb01           sub     ebx,0x1
+ 214f103b 895804           mov     [eax+0x4],ebx
+ 214f103e 8b4808           mov     ecx,[eax+0x8]
+ 214f1041 8b0c99           mov     ecx,[ecx+ebx*4]
+ 214f1044 894818           mov     [eax+0x18],ecx
+ 214f1047 8b4814           mov     ecx,[eax+0x14]
+ 214f104a 8b5810           mov     ebx,[eax+0x10]
+ 214f104d 8d401c           lea     eax,[eax+0x1c]
+ 214f1050 ffe0             jmp     eax
+ *
+ *
+ * From emit_detach_callback_final_jmp()
+ * _detach_callback_stack_t.code_buf (thread private)
+ *   mov (xax_save_offset) -> xax
+ *   jmp *target
+ *
+ 214f111c a10c114f21       mov     eax,[214f110c]   
+ 214f1121 ff2518114f21     jmp     dword ptr [214f1118]
+ */
+byte *
+emit_detach_callback_code(dcontext_t *dcontext, byte *buf,
+                          detach_callback_stack_t *callback_state)
+{
+    byte *pc = buf;
+    instrlist_t ilist;
+    instr_t *match_tid = INSTR_CREATE_label(dcontext),
+        *match_found = INSTR_CREATE_label(dcontext);
+
+    /* PR 284029: for now we assume there are no syscalls in x86 code, so
+     * we do not need to generate an x86 version
+     */
+
+    /* initialize the ilist */
+    instrlist_init(&ilist);
+
+    /* create instructions */
+    APP(&ilist, INSTR_CREATE_xchg(dcontext, opnd_create_tls_slot(TID_TIB_OFFSET),
+                                  opnd_create_reg(REG_XBX)));
+    APP(&ilist, INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XAX),
+                                     OPND_CREATE_INTPTR((ptr_uint_t)callback_state)));
+    APP(&ilist, match_tid);
+    /* FIXME - we clobber eflags.  We don't anticipate that being a problem on callback
+     * returns since syscalls clobber eflags too. */
+    APP(&ilist, INSTR_CREATE_cmp
+        (dcontext, opnd_create_reg(REG_XBX),
+         OPND_CREATE_MEMPTR(REG_XAX, offsetof(detach_callback_stack_t, tid))));
+    APP(&ilist, INSTR_CREATE_jcc_short(dcontext, OP_je, opnd_create_instr(match_found)));
+    APP(&ilist, INSTR_CREATE_add(dcontext, opnd_create_reg(REG_XAX),
+                                 OPND_CREATE_INT_32OR8(sizeof(detach_callback_stack_t))));
+    APP(&ilist, INSTR_CREATE_jmp(dcontext, opnd_create_instr(match_tid)));
+    APP(&ilist, match_found);
+    /* found matching tid ptr is in xax
+     * spill registers into local slots and restore TEB fields */
+    APP(&ilist, INSTR_CREATE_xchg(dcontext, opnd_create_tls_slot(TID_TIB_OFFSET),
+                                  opnd_create_reg(REG_XBX)));
+    APP(&ilist, INSTR_CREATE_mov_st
+        (dcontext,
+         OPND_CREATE_MEMPTR(REG_XAX, offsetof(detach_callback_stack_t, xbx_save)),
+         opnd_create_reg(REG_XBX)));
+    APP(&ilist, INSTR_CREATE_mov_imm
+        (dcontext, opnd_create_reg(REG_XBX),
+         OPND_CREATE_INTPTR((ptr_uint_t)get_process_id())));
+    APP(&ilist, INSTR_CREATE_xchg(dcontext, opnd_create_tls_slot(PID_TIB_OFFSET),
+                                  opnd_create_reg(REG_XBX)));
+    APP(&ilist, INSTR_CREATE_mov_st
+        (dcontext,
+         OPND_CREATE_MEMPTR(REG_XAX, offsetof(detach_callback_stack_t, xax_save)),
+         opnd_create_reg(REG_XBX)));
+    APP(&ilist, INSTR_CREATE_mov_st
+        (dcontext,
+         OPND_CREATE_MEMPTR(REG_XAX, offsetof(detach_callback_stack_t, xcx_save)),
+         opnd_create_reg(REG_XCX)));
+    /* now find the right address and move it into target while updating the
+     * thread private count */
+    APP(&ilist, INSTR_CREATE_mov_ld
+        (dcontext, opnd_create_reg(REG_XBX),
+         OPND_CREATE_MEMPTR(REG_XAX, offsetof(detach_callback_stack_t, count))));
+    /* see earlier comment on clobbering eflags */
+    APP(&ilist, INSTR_CREATE_sub(dcontext, opnd_create_reg(REG_XBX),
+                                 OPND_CREATE_INT8(1)));
+    APP(&ilist, INSTR_CREATE_mov_st
+        (dcontext,
+         OPND_CREATE_MEMPTR(REG_XAX, offsetof(detach_callback_stack_t, count)),
+         opnd_create_reg(REG_XBX)));
+    APP(&ilist, INSTR_CREATE_mov_ld
+        (dcontext, opnd_create_reg(REG_XCX),
+         OPND_CREATE_MEMPTR(REG_XAX, offsetof(detach_callback_stack_t, callback_addrs))));
+    APP(&ilist, INSTR_CREATE_mov_ld
+        (dcontext, opnd_create_reg(REG_XCX),
+         opnd_create_base_disp(REG_XCX, REG_XBX, sizeof(app_pc), 0, OPSZ_PTR)));
+    APP(&ilist, INSTR_CREATE_mov_st
+        (dcontext,
+         OPND_CREATE_MEMPTR(REG_XAX, offsetof(detach_callback_stack_t, target)),
+         opnd_create_reg(REG_XCX)));
+    APP(&ilist, INSTR_CREATE_mov_ld
+        (dcontext, opnd_create_reg(REG_XCX),
+         OPND_CREATE_MEMPTR(REG_XAX, offsetof(detach_callback_stack_t, xcx_save))));
+    APP(&ilist, INSTR_CREATE_mov_ld
+        (dcontext, opnd_create_reg(REG_XBX),
+         OPND_CREATE_MEMPTR(REG_XAX, offsetof(detach_callback_stack_t, xbx_save))));
+    APP(&ilist, INSTR_CREATE_lea
+        (dcontext, opnd_create_reg(REG_XAX),
+         OPND_CREATE_MEM_lea(REG_XAX, REG_NULL, 0,
+                             offsetof(detach_callback_stack_t, code_buf))));
+    APP(&ilist, INSTR_CREATE_jmp_ind(dcontext, opnd_create_reg(REG_XAX)));
+
+    /* now encode the instructions */
+    pc = instrlist_encode(dcontext, &ilist, pc, true /* instr targets */);
+    ASSERT(pc != NULL);
+    ASSERT(pc - buf < DETACH_CALLBACK_CODE_SIZE);
+
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+
+    return pc;
+}
+
+void
+emit_detach_callback_final_jmp(dcontext_t *dcontext,
+                               detach_callback_stack_t *callback_state)
+{
+    byte *pc = callback_state->code_buf;
+    instrlist_t ilist;
+
+    /* initialize the ilist */
+    instrlist_init(&ilist);
+
+    /* restore eax and jmp target */
+    APP(&ilist, INSTR_CREATE_mov_ld
+        (dcontext, opnd_create_reg(REG_XAX),
+         OPND_CREATE_ABSMEM(&(callback_state->xax_save), OPSZ_PTR)));
+    APP(&ilist, INSTR_CREATE_jmp_ind
+        (dcontext, OPND_CREATE_ABSMEM(&(callback_state->target), OPSZ_PTR)));
+
+    /* now encode the instructions */
+    pc = instrlist_encode(dcontext, &ilist, pc, true /* instr targets */);
+    ASSERT(pc != NULL);
+    ASSERT(pc - callback_state->code_buf < DETACH_CALLBACK_FINAL_JMP_SIZE);
+
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+}
+
+
+void
+emit_patch_syscall(dcontext_t *dcontext, byte *target _IF_X64(gencode_mode_t mode))
+{
+    byte *pc = after_do_syscall_code_ex(dcontext _IF_X64(mode));
+    instrlist_t ilist;
+
+    if (DYNAMO_OPTION(shared_syscalls)) {
+        /* Simply patch shared_syscall to jump to after_do_syscall. Only
+         * one array of callback stack addresses is needed -- a return from
+         * a callback entered from shared_syscall will jump to the patched
+         * after_do_syscall and fetch the correct address off of our
+         * callback stack copy. It "just works".
+         */
+        instr_t *instr = INSTR_CREATE_jmp(dcontext, opnd_create_pc(pc));
+        DEBUG_DECLARE(byte *nxt_pc =)
+            instr_encode(dcontext, instr,
+                         after_shared_syscall_code_ex(dcontext _IF_X64(mode)));
+        ASSERT(nxt_pc != NULL);
+        /* check that there was room - shared_syscall should be before do_syscall
+         * anything between them is dead at this point */
+        ASSERT(after_shared_syscall_code_ex(dcontext _IF_X64(mode)) < pc && nxt_pc < pc);
+        instr_destroy(dcontext, instr);
+        LOG(THREAD, LOG_EMIT, 2,
+            "Finished patching shared syscall routine for detach -- patch "PFX
+            " to jump to "PFX"\n", after_shared_syscall_code(dcontext), pc);
+    }
+
+    /* initialize the ilist */
+    instrlist_init(&ilist);
+
+    /* patch do_syscall to jmp to target */
+    /* Note that on 64-bit target may not be reachable in which case we need to inline
+     * the first register spill here so we can jmp reg. We go ahead and the spill here
+     * and jmp through reg for 32-bit as well for consistency. */
+    APP(&ilist, INSTR_CREATE_mov_st(dcontext, opnd_create_tls_slot(PID_TIB_OFFSET),
+                                    opnd_create_reg(REG_XAX)));
+    APP(&ilist, INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XAX),
+                                     OPND_CREATE_INTPTR((ptr_uint_t)target)));
+    APP(&ilist, INSTR_CREATE_jmp_ind(dcontext, opnd_create_reg(REG_XAX)));
+
+    /* now encode the instructions */
+    pc = instrlist_encode(dcontext, &ilist, pc, true /* instr targets */);
+    ASSERT(pc != NULL);
+    /* ASSERT that there was enough space after the system call (everything after
+     * do_syscall should be dead at this point). */
+    ASSERT(pc <= get_emitted_routines_code(dcontext _IF_X64(mode))->commit_end_pc);
+
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+}
+#endif /* WINDOWS */
+
+/* this routine performs a single system call instruction and then returns
+ * to dynamo via fcache_return
+ */
+static byte * 
+emit_do_syscall_common(dcontext_t *dcontext, generated_code_t *code,
+                       byte *pc, byte *fcache_return_pc,
+                       bool handle_clone, bool thread_shared, bool force_int,
+                       uint *syscall_offs /*OUT*/)
+{
+    instrlist_t ilist;
+    instr_t *syscall;
+#ifdef LINUX
+    instr_t *post_syscall;
+#endif
+
+#if defined(LINUX) && !defined(X64)
+    /* PR 286922: 32-bit clone syscall cannot use vsyscall: must be int */
+    if (handle_clone)
+        force_int = true;
+#endif
+    if (force_int)
+        syscall = create_int_syscall_instr(dcontext);
+    else
+        syscall = create_syscall_instr(dcontext);
+
+    /* PR 284029: for now we assume there are no syscalls in x86 code.
+     */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(!code->x86_mode));
+
+    ASSERT(syscall_offs != NULL);
+    *syscall_offs = instr_length(dcontext, syscall);
+
+    /* initialize the ilist */
+    instrlist_init(&ilist);
+
+    /* system call itself -- using same method we've observed OS using */
+    APP(&ilist, syscall);
+#ifdef LINUX
+    if (get_syscall_method() == SYSCALL_METHOD_UNINITIALIZED) {
+        /* Since we lazily find out the method, but emit these routines
+         * up front, we have to leave room for the longest syscall method.
+         * This used to the 6-byte LOL64 call* but we now walk into that
+         * call* (PR 286922).  Not much of a perf worry, but if we
+         * ever have proactive syscall determination on linux we should
+         * remove these nops.
+         */
+        ASSERT(instr_length(dcontext, instrlist_last(&ilist)) == 2);
+        if (SYSCALL_METHOD_LONGEST_INSTR == 6) {
+            /* we could add 4-byte nop support but I'm too lazy */
+            APP(&ilist, INSTR_CREATE_nop3byte(dcontext));
+            APP(&ilist, INSTR_CREATE_nop1byte(dcontext));
+        } else
+            ASSERT_NOT_IMPLEMENTED(instr_length(dcontext, instrlist_last(&ilist)) ==
+                                   SYSCALL_METHOD_LONGEST_INSTR);
+    }
+    post_syscall = instrlist_last(&ilist);
+#endif
+
+    /* go to fcache return -- use special syscall linkstub */
+    /* in case it returns: go to fcache return -- use 0 as &linkstub */
+    if (thread_shared)
+        APP(&ilist, instr_create_save_to_tls(dcontext, REG_XAX, TLS_XAX_SLOT));
+    else
+        APP(&ilist, instr_create_save_to_dcontext(dcontext, REG_EAX, XAX_OFFSET));
+    APP(&ilist, INSTR_CREATE_mov_imm
+        (dcontext, opnd_create_reg(REG_XAX),
+         OPND_CREATE_INTPTR((ptr_int_t)get_syscall_linkstub())));
+    APP(&ilist, INSTR_CREATE_jmp(dcontext, opnd_create_pc(fcache_return_pc)));
+
+#ifdef LINUX
+    if (handle_clone) {
+        /* put in clone code, and make sure to target it.
+         * do it here since it assumes an instr after the syscall exists.
+         * assume that the next pc after the int is in CLONE_SCRATCH_REG_REG
+         * (see handle_system_call() in dispatch for more info)
+         */
+        instr_t *pc_to_ecx =
+            INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XCX),
+                                opnd_create_reg(CLONE_SCRATCH_REG_REG));
+        mangle_insert_clone_code(dcontext, &ilist, post_syscall,
+                                 pc_to_ecx, false /*do not skip*/
+                                 _IF_X64(MODE_OVERRIDE(code->x86_mode)));
+    }
+#endif
+
+    /* now encode the instructions */
+    pc = instrlist_encode(dcontext, &ilist, pc,
+#ifdef LINUX
+                          handle_clone /* instr targets */
+#else
+                          false /* no instr targets */
+#endif
+                          );
+    ASSERT(pc != NULL);
+
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+
+    return pc;
+}
+
+#ifdef WINDOWS
+/* like fcache_enter but indirects the dcontext passed in through edi */
+byte * 
+emit_fcache_enter_indirect(dcontext_t *dcontext, generated_code_t *code,
+                           byte *pc, byte *fcache_return_pc)
+{
+    return emit_fcache_enter_common(dcontext, code, pc,
+                                    false/*indirect*/, false/*!shared*/);
+}
+
+/* This routine performs an int 2b, which maps to NtCallbackReturn, and then returns
+ * to dynamo via fcache_return (though it won't reach there)
+ */
+byte * 
+emit_do_callback_return(dcontext_t *dcontext, byte *pc, byte *fcache_return_pc,
+                        bool thread_shared)
+{
+    instrlist_t ilist;
+
+    /* initialize the ilist */
+    instrlist_init(&ilist);
+
+    /* interrupt 2b */
+    APP(&ilist, INSTR_CREATE_int(dcontext, opnd_create_immed_int(0x2b, OPSZ_1)));
+
+    /* in case it returns: go to fcache return -- use 0 as &linkstub */
+    if (thread_shared)
+        APP(&ilist, instr_create_save_to_tls(dcontext, REG_XAX, TLS_XAX_SLOT));
+    else
+        APP(&ilist, instr_create_save_to_dcontext(dcontext, REG_EAX, XAX_OFFSET));
+    /* for x64 we rely on sign-extension to fill out rax */
+    APP(&ilist, INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_EAX),
+                                     OPND_CREATE_INT32(0)));
+    APP(&ilist, INSTR_CREATE_jmp(dcontext, opnd_create_pc(fcache_return_pc)));
+
+    /* now encode the instructions */
+    pc = instrlist_encode(dcontext, &ilist, pc, false /* no instr targets */);
+    ASSERT(pc != NULL);
+
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+
+    return pc;
+}
+#else /* !WINDOWS => LINUX */
+byte * 
+emit_do_clone_syscall(dcontext_t *dcontext, generated_code_t *code, byte *pc,
+                      byte *fcache_return_pc, bool thread_shared,
+                      uint *syscall_offs /*OUT*/)
+{
+    return emit_do_syscall_common(dcontext, code, pc, fcache_return_pc,
+                                  true, thread_shared, false, syscall_offs);
+}
+#endif /* LINUX */
+
+byte * 
+emit_do_syscall(dcontext_t *dcontext, generated_code_t *code, byte *pc,
+                byte *fcache_return_pc, bool thread_shared, bool force_int,
+                uint *syscall_offs /*OUT*/)
+{
+    pc = emit_do_syscall_common(dcontext, code, pc, fcache_return_pc,
+                                false, thread_shared, force_int, syscall_offs);
+    return pc;
+}
+
+#ifndef WINDOWS
+/* updates first syscall instr it finds with the new method of syscall */
+static void
+update_syscall(dcontext_t *dcontext, byte *pc)
+{
+    LOG_DECLARE(byte *start_pc = pc;)
+    byte *prev_pc;
+    instr_t instr;
+    instr_init(dcontext, &instr);
+
+    do {
+        prev_pc = pc;
+        instr_reset(dcontext, &instr);
+        pc = decode_cti(dcontext, pc, &instr);
+        ASSERT(pc != NULL); /* this our own code we're decoding, should be valid */
+        if (instr_is_syscall(&instr)) {
+            instr_t *newinst = create_syscall_instr(dcontext);
+            byte *nxt_pc = instr_encode(dcontext, newinst, prev_pc);
+            /* instruction must not change size! */
+            ASSERT(nxt_pc != NULL);
+            if (nxt_pc != pc) {
+                pc = nxt_pc;
+                byte *stop_pc = prev_pc + SYSCALL_METHOD_LONGEST_INSTR;
+                ASSERT(nxt_pc <= stop_pc);
+                while (pc < stop_pc) {
+                    /* we could add >3-byte nop support but I'm too lazy */
+                    int noplen = MIN(stop_pc - pc, 3);
+                    instr_t *nop = instr_create_nbyte_nop(dcontext, noplen, true);
+                    pc = instr_encode(dcontext, nop, pc);
+                    ASSERT(pc != NULL);
+                    instr_destroy(dcontext, nop);
+                }
+            }
+            instr_destroy(dcontext, newinst);
+            break;
+        }
+        ASSERT(pc - prev_pc < 128);
+    } while (1);
+    
+    instr_free(dcontext, &instr);
+
+    DOLOG(3, LOG_EMIT, {
+        LOG(THREAD, LOG_EMIT, 3, "Just updated syscall routine:\n");
+        prev_pc = pc;
+        pc = start_pc;
+        do {
+            pc = disassemble_with_bytes(dcontext, pc, THREAD);       
+        } while (pc < prev_pc + 1); /* +1 to get next instr */
+        LOG(THREAD, LOG_EMIT, 3, "  ...\n");
+    });
+}
+
+void
+update_syscalls(dcontext_t *dcontext)
+{
+    byte *pc;
+    pc = get_do_syscall_entry(dcontext);
+    update_syscall(dcontext, pc);
+#ifdef X64
+    /* PR 286922: for 32-bit, we do NOT update the clone syscall as it
+     * always uses int (since can't use call to vsyscall when swapping
+     * stacks!)
+     */
+    pc = get_do_clone_syscall_entry(dcontext);
+    update_syscall(dcontext, pc);
+#endif
+}
+#endif /* !WINDOWS */
+
+/* Returns -1 on failure */
+int
+decode_syscall_num(dcontext_t *dcontext, byte *entry)
+{
+    byte *pc;
+    int syscall = -1;
+    instr_t instr;
+    ASSERT(entry != NULL);
+    instr_init(dcontext, &instr);
+    pc = entry;
+    LOG(GLOBAL, LOG_EMIT, 3, "decode_syscall_num "PFX"\n", entry);
+    while (true) {
+        DOLOG(3, LOG_EMIT, { disassemble_with_bytes(dcontext, pc, GLOBAL); });
+        instr_reset(dcontext, &instr);
+        pc = decode(dcontext, pc, &instr);
+        if (pc == NULL)
+            break; /* give up gracefully */
+        /* we do not handle control transfer instructions! */
+        if (instr_is_cti(&instr)) {
+#ifdef WINDOWS /* since no interception code buffer to check on linux */
+            if (DYNAMO_OPTION(native_exec_syscalls) && instr_is_ubr(&instr)) {
+                /* probably our own trampoline, follow it
+                 * ASSUMPTION: mov eax is the instr that jmp targets
+                 * Today native_exec syscall hooking doesn't use landing pads
+                 * to reach trampolines.  When that is done as part of PR
+                 * 245169, this should check landing_pad_areas first (see fix
+                 * for PR 250294 to see how it is done).
+                 */
+                pc = opnd_get_pc(instr_get_target(&instr));
+                if (!is_in_interception_buffer(pc)) {
+                    break;  /* give up gracefully */
+                } /* else, carry on at pc */
+            } else
+#endif
+                break; /* give up gracefully */
+        }
+        if (instr_num_dsts(&instr) > 0 &&
+            opnd_is_reg(instr_get_dst(&instr, 0)) &&
+            opnd_get_reg(instr_get_dst(&instr, 0)) == REG_XAX) {
+            if (instr_get_opcode(&instr) == OP_mov_imm) {
+                IF_X64(ASSERT_TRUNCATE(int, int,
+                                       opnd_get_immed_int(instr_get_src(&instr, 0))));
+                syscall = (int) opnd_get_immed_int(instr_get_src(&instr, 0));
+                LOG(GLOBAL, LOG_EMIT, 3, "\tfound syscall num: 0x%x\n", syscall);
+                break;
+            } else
+                break; /* give up gracefully */
+        }
+    }
+    instr_free(dcontext, &instr);
+    return syscall;
+}
+
+/* If code_buf points to a jmp rel32 returns true and returns the target of
+ * the jmp in jmp_target as if was located at app_loc. */
+bool
+is_jmp_rel32(byte *code_buf, app_pc app_loc, app_pc *jmp_target /* OUT */)
+{
+    if (*code_buf == JMP_OPCODE) {
+        if (jmp_target != NULL) {
+            *jmp_target = app_loc + JMP_LONG_LENGTH +
+                *(int *)(code_buf+1);
+        }
+        return true;
+    }
+    return false;
+}
+
+/* If code_buf points to a jmp rel8 returns true and returns the target of
+ * the jmp in jmp_target as if was located at app_loc. */
+bool
+is_jmp_rel8(byte *code_buf, app_pc app_loc, app_pc *jmp_target /* OUT */)
+{
+    if (*code_buf == JMP_SHORT_OPCODE) {
+        if (jmp_target != NULL) {
+            *jmp_target = app_loc + JMP_SHORT_LENGTH +
+                *(char *)(code_buf+1);
+        }
+        return true;
+    }
+    return false;
+}
+
+#ifdef LINUX
+/* PR 212290: can't be static code in x86.asm since it can't be PIC */
+/*
+ * new_thread_dynamo_start - for initializing a new thread created
+ * via the clone system call.
+ * assumptions:
+ *   1) app's xcx is in xax.
+ *   2) xcx contains next app pc, which is not 0.
+ *   3) app's xax should contain 0.
+ */
+byte * 
+emit_new_thread_dynamo_start(dcontext_t *dcontext, byte *pc)
+{
+    instrlist_t ilist;
+    instr_t *try_again = INSTR_CREATE_label(dcontext);
+    instr_t *have_lock = INSTR_CREATE_label(dcontext);
+    uint offset;
+
+    /* initialize the ilist */
+    instrlist_init(&ilist);
+
+    APP(&ilist, try_again);
+    /* We used to xchg ecx w/ initstack_mutex, since it's non-zero, but
+     * we can lose it if two threads are here simultaneously (PR 207903).
+     * Plus, for 64-bit, initstack_mutex.lock_requests is 32-bit,
+     * so we'd end up clobbering the top 32 bits of rcx w/ an xchg.
+     * So, we have to spill a register.
+     * FIXME PR 207903: this is still racy: we need a thread-private spill slot.
+     * We could fix w/ CLONE_SETTLS (only kernel 2.5.32+ though) (PR 285898).
+     */
+    APP(&ilist, INSTR_CREATE_mov_st
+        (dcontext, OPND_CREATE_ABSMEM((void *)&initstack_app_xsp, OPSZ_PTR),
+         opnd_create_reg(REG_XCX)));
+    APP(&ilist, INSTR_CREATE_mov_imm
+        (dcontext, opnd_create_reg(REG_XCX), OPND_CREATE_INT32(1)));
+    APP(&ilist, INSTR_CREATE_xchg
+        (dcontext, OPND_CREATE_ABSMEM((void *)&initstack_mutex, OPSZ_PTR),
+         opnd_create_reg(REG_XCX)));
+    APP(&ilist, INSTR_CREATE_jecxz(dcontext, opnd_create_instr(have_lock)));
+    /* improve spin-loop perf on P4 */
+    APP(&ilist, INSTR_CREATE_pause(dcontext));
+    /* try again -- too few free regs to call sleep() */
+    APP(&ilist, INSTR_CREATE_jecxz(dcontext, opnd_create_instr(try_again)));
+
+    APP(&ilist, have_lock);
+    /* put app xcx into xcx , and next pc in xax */
+    APP(&ilist, INSTR_CREATE_mov_ld
+        (dcontext, opnd_create_reg(REG_XCX), opnd_create_reg(REG_XAX)));
+    APP(&ilist, INSTR_CREATE_mov_ld
+        (dcontext, opnd_create_reg(REG_XAX),
+         OPND_CREATE_ABSMEM((void *)&initstack_app_xsp, OPSZ_PTR)));
+    /* save app's esp */
+    APP(&ilist, INSTR_CREATE_mov_st
+        (dcontext, OPND_CREATE_ABSMEM((void *)&initstack_app_xsp, OPSZ_PTR),
+         opnd_create_reg(REG_XSP)));
+    /* now use initstack as temporary stack */
+    APP(&ilist, INSTR_CREATE_mov_ld
+        (dcontext, opnd_create_reg(REG_XSP),
+         OPND_CREATE_ABSMEM((void *)&initstack, OPSZ_PTR)));
+
+    /* grab exec state and pass as param in a dr_mcontext_t struct
+     * new_thread_setup will restore real app xsp and xax
+     * we emulate x86.asm's PUSH_DR_MCONTEXT(REG_XAX) (for dr_mcontext_t.pc)
+     */
+    APP(&ilist, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XAX)));
+    APP(&ilist, INSTR_CREATE_pushf(dcontext));
+    offset = insert_push_all_registers(dcontext, &ilist, NULL, IF_X64_ELSE(true, false));
+    ASSERT(ALIGNED(offset, 16));
+    /* put pre-push xsp into dr_mcontext_t.xsp slot */
+    ASSERT(offset + 2*XSP_SZ == sizeof(dr_mcontext_t));
+    APP(&ilist, INSTR_CREATE_lea
+        (dcontext, opnd_create_reg(REG_XAX),
+         OPND_CREATE_MEM_lea(REG_XSP, REG_NULL, 0, sizeof(dr_mcontext_t))));
+    APP(&ilist, INSTR_CREATE_mov_st
+        (dcontext, OPND_CREATE_MEMPTR(REG_XSP, offsetof(dr_mcontext_t, xsp)),
+         opnd_create_reg(REG_XAX)));
+
+    /* We avoid get_thread_id syscall in get_thread_private_dcontext()
+     * by clearing the segment register here (cheaper check than syscall)
+     * (xref PR 192231).  If we crash prior to this point though, the
+     * signal handler will get the wrong dcontext, but that's a small window.
+     * See comments in get_thread_private_dcontext() for alternatives.
+     */
+    APP(&ilist, INSTR_CREATE_mov_imm
+        (dcontext, opnd_create_reg(REG_AX), OPND_CREATE_INT16(0)));
+    APP(&ilist, INSTR_CREATE_mov_seg
+        (dcontext, opnd_create_reg(SEG_TLS), opnd_create_reg(REG_AX)));
+    /* stack grew down, so dr_mcontext_t at tos */
+    APP(&ilist, INSTR_CREATE_lea
+        (dcontext, opnd_create_reg(REG_XAX),
+         OPND_CREATE_MEM_lea(REG_XSP, REG_NULL, 0, 0)));
+
+    dr_insert_call(dcontext, &ilist, NULL, (void *)new_thread_setup,
+                   1, opnd_create_reg(REG_XAX));
+
+    /* should not return */
+    APP(&ilist, INSTR_CREATE_jmp(dcontext, opnd_create_pc((app_pc)unexpected_return)));
+
+    /* now encode the instructions */
+    pc = instrlist_encode(dcontext, &ilist, pc, true /* instr targets */);
+    ASSERT(pc != NULL);
+
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+
+    return pc;
+}
+#endif /* LINUX */
+
+#ifdef TRACE_HEAD_CACHE_INCR
+/* trace_t heads come here instead of back to dynamo to have their counters
+ * incremented.
+ */
+byte * 
+emit_trace_head_incr(dcontext_t *dcontext, byte *pc, byte *fcache_return_pc)
+{
+    /*  save ecx
+        save eax->xbx slot
+        mov target_fragment_offs(eax), eax
+        movzx counter_offs(eax), ecx
+        lea 1(ecx), ecx                 # increment counter
+        mov data16 cx, counter_offs(eax)
+        lea -hot_threshold(ecx), ecx    # compare to hot_threshold
+        jecxz is_hot
+        mov start_pc_offs(eax), ecx
+        movzx prefix_size_offs(eax), eax
+        lea (ecx,eax,1), ecx
+        mov ecx, trace_head_pc_offs + dcontext   # special slot to avoid target prefix
+        restore ecx
+        restore eax
+        jmp * trace_head_pc_offs + dcontext
+      is_hot:
+        restore ebx slot to eax         # put &l into eax
+        restore ecx
+        jmp fcache_return
+     */
+    instrlist_t ilist;
+    instr_t *is_hot =
+        instr_create_restore_from_dcontext(dcontext, REG_EAX, XBX_OFFSET);
+    instr_t *in;
+
+    /* PR 248210: unsupported feature on x64 */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+
+    instrlist_init(&ilist);
+    APP(&ilist, instr_create_save_to_dcontext(dcontext, REG_ECX, XCX_OFFSET));
+    if (DYNAMO_OPTION(shared_bbs)) {
+        /* HACK to get shared exit stub, which puts eax into fs:scratch1, to work
+         * w/ thread-private THCI: we pull eax out of the tls slot and into mcontext.
+         * This requires that all direct stubs for cti that can link to trace
+         * heads use the shared stub -- so if traces can link to trace heads, their
+         * exits must use the shared stubs, even if the traces are thread-private.
+         */
+        APP(&ilist, RESTORE_FROM_TLS(dcontext, REG_ECX, EXIT_STUB_SPILL_SLOT));
+        APP(&ilist, instr_create_save_to_dcontext(dcontext, REG_ECX, XAX_OFFSET));
+    }
+    APP(&ilist, instr_create_save_to_dcontext(dcontext, REG_EAX, XBX_OFFSET));
+    APP(&ilist, INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_EAX),
+                            OPND_CREATE_MEM32(REG_EAX, LINKSTUB_TARGET_FRAG_OFFS)));
+    ASSERT_NOT_IMPLEMENTED(false && "must handle LINKSTUB_CBR_FALLTHROUGH case by calculating target tag")
+    APP(&ilist, INSTR_CREATE_movzx(dcontext, opnd_create_reg(REG_ECX),
+                                   opnd_create_base_disp(REG_EAX, REG_NULL, 0,
+                                                         FRAGMENT_COUNTER_OFFS, OPSZ_2)));
+    APP(&ilist, INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ECX),
+                              opnd_create_base_disp(REG_ECX, REG_NULL, 0, 1, OPSZ_lea)));
+    /* data16 prefix is set auto-magically */
+    APP(&ilist, INSTR_CREATE_mov_st(dcontext,
+                                    opnd_create_base_disp(REG_EAX, REG_NULL, 0,
+                                                          FRAGMENT_COUNTER_OFFS, OPSZ_2),
+                                    opnd_create_reg(REG_CX)));
+    APP(&ilist, INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ECX),
+                              opnd_create_base_disp(REG_ECX, REG_NULL, 0,
+                                            -((int)INTERNAL_OPTION(trace_threshold)), OPSZ_lea)));
+    APP(&ilist, INSTR_CREATE_jecxz(dcontext, opnd_create_instr(is_hot)));
+    APP(&ilist, INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_ECX),
+                         OPND_CREATE_MEM32(REG_EAX, FRAGMENT_START_PC_OFFS)));
+    APP(&ilist, INSTR_CREATE_movzx(dcontext, opnd_create_reg(REG_EAX),
+                                   opnd_create_base_disp(REG_EAX, REG_NULL, 0,
+                                                         FRAGMENT_PREFIX_SIZE_OFFS, OPSZ_1)));
+    APP(&ilist, INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ECX),
+                                 opnd_create_base_disp(REG_ECX, REG_EAX, 1, 0, OPSZ_lea)));
+    APP(&ilist, instr_create_save_to_dcontext(dcontext, REG_ECX, TRACE_HEAD_PC_OFFSET));
+    APP(&ilist, instr_create_restore_from_dcontext(dcontext, REG_ECX, XCX_OFFSET));
+    APP(&ilist, instr_create_restore_from_dcontext(dcontext, REG_EAX, XAX_OFFSET));
+    APP(&ilist, INSTR_CREATE_jmp_ind(dcontext,
+                          opnd_create_dcontext_field(dcontext, TRACE_HEAD_PC_OFFSET)));
+    APP(&ilist, is_hot);
+    APP(&ilist, instr_create_restore_from_dcontext(dcontext, REG_ECX, XCX_OFFSET));
+    APP(&ilist, INSTR_CREATE_jmp(dcontext, opnd_create_pc(fcache_return_pc)));
+
+    /* now encode the instructions */
+    pc = instrlist_encode(dcontext, &ilist, pc, true /* instr targets */);
+    ASSERT(pc != NULL);
+
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+
+    return pc;
+}
+
+byte * 
+emit_trace_head_incr_shared(dcontext_t *dcontext, byte *pc, byte *fcache_return_pc)
+{
+    ASSERT_NOT_IMPLEMENTED(false);
+}
+
+#endif /* TRACE_HEAD_CACHE_INCR */
diff --git a/core/x86/encode.c b/core/x86/encode.c
new file mode 100644
index 000000000..4f4a65a6b
--- /dev/null
+++ b/core/x86/encode.c
@@ -0,0 +1,2270 @@
+/* **********************************************************
+ * Copyright (c) 2001-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2001 Hewlett-Packard Company */
+
+/* encode.c -- an x86 encoder */
+
+#include "globals.h"
+#include "arch.h"
+#include "instr.h"
+#include "decode.h"
+#include "disassemble.h"
+#include "decode_fast.h"
+
+#include <string.h> /* memcpy, memset */
+
+#ifdef DEBUG
+/* case 10450: give messages to clients */
+/* we can't undef ASSERT b/c of DYNAMO_OPTION */
+# undef ASSERT_TRUNCATE
+# undef ASSERT_BITFIELD_TRUNCATE
+# undef ASSERT_NOT_REACHED
+# define ASSERT_TRUNCATE DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_BITFIELD_TRUNCATE DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_NOT_REACHED DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+#endif
+
+/* level at which encoding attempts get dumped out...lots of logging! */
+#define ENC_LEVEL 6
+
+const char * const type_names[] = {
+    "TYPE_NONE",
+    "TYPE_A", /* immediate that is absolute address */
+    "TYPE_C", /* reg of modrm selects control reg */
+    "TYPE_D", /* reg of modrm selects debug reg */
+    "TYPE_E", /* modrm selects reg or mem addr */
+    "TYPE_G", /* reg of modrm selects register */
+    "TYPE_I", /* immediate */
+    "TYPE_J", /* immediate that is relative offset of EIP */
+    "TYPE_M", /* modrm select mem addr */
+    "TYPE_O", /* immediate that is memory offset */
+    "TYPE_P", /* reg of modrm selects MMX */
+    "TYPE_Q", /* modrm selects MMX or mem addr */
+    "TYPE_R", /* mod of modrm selects register */
+    "TYPE_S", /* reg of modrm selects segment register */
+    "TYPE_V", /* reg of modrm selects XMM */
+    "TYPE_W", /* modrm selects XMM or mem addr */
+    "TYPE_X", /* DS:(RE)(E)SI */
+    "TYPE_Y", /* ES:(RE)(E)SDI */
+    "TYPE_P_MODRM",  /* mod of modrm selects MMX */
+    "TYPE_V_MODRM",  /* mod of modrm selects XMM */
+    "TYPE_1",
+    "TYPE_FLOATCONST",
+    "TYPE_XLAT",     /* DS:(RE)(E)BX+AL */
+    "TYPE_MASKMOVQ", /* DS:(RE)(E)DI */
+    "TYPE_FLOATMEM",
+    "TYPE_REG",
+    "TYPE_VAR_REG",
+    "TYPE_VARZ_REG",
+    "TYPE_VAR_XREG",
+    "TYPE_VAR_ADDR_XREG",
+    "TYPE_REG_EX",
+    "TYPE_VAR_REG_EX",
+    "TYPE_VAR_XREG_EX",
+    "TYPE_VAR_REGX_EX",
+    "TYPE_INDIR_E",
+    "TYPE_INDIR_REG",
+    "TYPE_INDIR_VAR_XREG",
+    "TYPE_INDIR_VAR_REG",
+    "TYPE_INDIR_VAR_XIREG",
+};
+
+/* order corresponds to enum of REG_ and SEG_ constants */
+const char * const * const reg_names = size_names;
+
+const char * const size_names[] = {
+    "<NULL>",
+    "rax",  "rcx",  "rdx",  "rbx",  "rsp",  "rbp",  "rsi",  "rdi",
+    "r8",   "r9",   "r10",  "r11",  "r12",  "r13",  "r14",  "r15",
+    "eax",  "ecx",  "edx",  "ebx",  "esp",  "ebp",  "esi",  "edi",
+    "r8d",  "r9d",  "r10d", "r11d", "r12d", "r13d", "r14d", "r15d",
+    "ax",   "cx",   "dx",   "bx",   "sp",   "bp",   "si",   "di",
+    "r8w",  "r9w",  "r10w", "r11w", "r12w", "r13w", "r14w", "r15w",
+    "al",   "cl",   "dl",   "bl",   "ah",   "ch",   "dh",   "bh",
+    "r8l",  "r9l",  "r10l", "r11l", "r12l", "r13l", "r14l", "r15l",
+    "spl",  "bpl",  "sil",  "dil",
+    "mm0",  "mm1",  "mm2",  "mm3",  "mm4",  "mm5",  "mm6",  "mm7",
+    "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7",
+    "xmm8", "xmm9", "xmm10","xmm11","xmm12","xmm13","xmm14","xmm15",
+    "st0",  "st1",  "st2",  "st3",  "st4",  "st5",  "st6",  "st7",
+    "es",   "cs",   "ss",   "ds",   "fs",   "gs",
+    "dr0",  "dr1",  "dr2",  "dr3",  "dr4",  "dr5",  "dr6",  "dr7",
+    "dr8",  "dr9",  "dr10", "dr11", "dr12", "dr13", "dr14", "dr15", 
+    "cr0",  "cr1",  "cr2",  "cr3",  "cr4",  "cr5",  "cr6",  "cr7", 
+    "cr8",  "cr9",  "cr10", "cr11", "cr12", "cr13", "cr14", "cr15",
+    "<invalid>",
+    "OPSZ_NA", 
+    "OPSZ_lea",
+    "OPSZ_1",
+    "OPSZ_2",
+    "OPSZ_4",
+    "OPSZ_6",
+    "OPSZ_8",
+    "OPSZ_10",
+    "OPSZ_16",
+    "OPSZ_14",
+    "OPSZ_28",
+    "OPSZ_94",
+    "OPSZ_108",
+    "OPSZ_512",
+    "OPSZ_2_short1",
+    "OPSZ_4_short2",
+    "OPSZ_4_rex8_short2",
+    "OPSZ_4_rex8",
+    "OPSZ_6_irex10_short4",
+    "OPSZ_8_short2",
+    "OPSZ_8_short4",
+    "OPSZ_28_short14",
+    "OPSZ_108_short94",
+    "OPSZ_6x10",
+    "OPSZ_4x8",
+    "OPSZ_4x8_short2",
+    "OPSZ_4x8_short2xi8",
+    "OPSZ_4_short2xi4",
+    "OPSZ_4_of_8",
+    "OPSZ_4_of_16",
+    "OPSZ_8_of_16",
+    "OPSZ_1_reg4",
+    "OPSZ_2_reg4",
+    "OPSZ_4_reg16",
+};
+
+/***************************************************************************
+ * Functions to see if instr operands match instr_info_t template
+ */
+
+static bool
+type_instr_uses_reg_bits(int type)
+{
+    switch (type) {
+    case TYPE_C:
+    case TYPE_D: 
+    case TYPE_G:
+    case TYPE_P: 
+    case TYPE_S:
+    case TYPE_V: 
+        return true;
+    default:
+        return false;
+    }
+}
+
+static bool
+type_uses_modrm_bits(int type)
+{
+    switch (type) {
+    case TYPE_E:
+    case TYPE_M: 
+    case TYPE_Q:
+    case TYPE_R: 
+    case TYPE_W:
+    case TYPE_INDIR_E:
+    case TYPE_P_MODRM:
+    case TYPE_V_MODRM:
+        return true;
+    default:
+        return false;
+    }
+}
+
+/* Helper routine that sets/checks rex.w or data prefix, if necessary, for
+ * variable-sized OPSZ_ constants that the user asks for.  We try to be flexible
+ * setting/checking only enough prefix flags to ensure that the final template size
+ * is one of the possible sizes in the request.
+ */
+static bool
+size_ok_varsz(decode_info_t *di/*prefixes field is IN/OUT; x86_mode is IN*/,
+              opnd_size_t size_op, opnd_size_t size_template,
+              uint prefix_data_addr)
+{
+    /* FIXME: this code is getting long and complex: is there a better way?
+     * Any way to resolve these var sizes further first?  Doesn't seem like it.
+     */
+    /* if identical sizes we shouldn't be called */
+    CLIENT_ASSERT(size_op != size_template, "size_ok_varsz: internal decoding error");
+    switch (size_op) {
+    case OPSZ_2_short1:
+        if (size_template == OPSZ_2 || size_template == OPSZ_1)
+            return true; /* will take prefix or no prefix */
+        if (size_template == OPSZ_4_short2 || size_template == OPSZ_8_short2) {
+            di->prefixes |= prefix_data_addr;
+            return true;
+        }
+        if (size_template == OPSZ_4_rex8_short2) {
+            if (TEST(PREFIX_REX_W, di->prefixes))
+                return false; /* rex.w trumps data prefix */
+            di->prefixes |= prefix_data_addr;
+            return true;
+        }
+        return false;
+    case OPSZ_4_short2:
+        if (size_template == OPSZ_4 || size_template == OPSZ_2)
+            return true; /* will take prefix or no prefix */
+        if (size_template == OPSZ_4_rex8_short2 || size_template == OPSZ_4_rex8)
+            return !TEST(PREFIX_REX_W, di->prefixes);
+        if (size_template == OPSZ_8_short2 || size_template == OPSZ_8_short4) {
+            di->prefixes |= prefix_data_addr;
+            return true;
+        }
+        if (size_template == OPSZ_6_irex10_short4 &&
+            (proc_get_vendor() == VENDOR_AMD || !TEST(PREFIX_REX_W, di->prefixes))) {
+            di->prefixes |= prefix_data_addr;
+            return true;
+        }
+        return false;
+    case OPSZ_4_rex8_short2:
+        if (size_template == OPSZ_4_short2 || size_template == OPSZ_4_rex8 ||
+            size_template == OPSZ_8_short2 || size_template == OPSZ_8_short4 ||
+            size_template == OPSZ_2 || size_template == OPSZ_4 || size_template == OPSZ_8)
+            return true; /* will take prefix or no prefix */
+        if (size_template == OPSZ_6_irex10_short4 &&
+            (proc_get_vendor() == VENDOR_AMD || !TEST(PREFIX_REX_W, di->prefixes))) {
+            di->prefixes |= prefix_data_addr;
+            return true;
+        }
+        return false;
+    case OPSZ_4_rex8:
+        if (size_template == OPSZ_8_short4 || size_template == OPSZ_4 ||
+            size_template == OPSZ_8)
+            return true; /* will take prefix or no prefix */
+        if (size_template == OPSZ_4_short2 || size_template == OPSZ_4_rex8_short2 ||
+            size_template == OPSZ_8_short2)
+            return !TEST(prefix_data_addr, di->prefixes);
+        if (size_template == OPSZ_6_irex10_short4 &&
+            (proc_get_vendor() == VENDOR_AMD || !TEST(PREFIX_REX_W, di->prefixes))) {
+            di->prefixes |= prefix_data_addr;
+            return true;
+        }
+        return false;
+    case OPSZ_6_irex10_short4:
+        if (size_template == OPSZ_6 || size_template == OPSZ_4 ||
+            (size_template == OPSZ_10 && proc_get_vendor() != VENDOR_AMD))
+            return true; /* will take prefix or no prefix */
+        if (size_template == OPSZ_4_short2)
+            return !TEST(prefix_data_addr, di->prefixes);
+        if (size_template == OPSZ_4_rex8_short2)
+            return !TESTANY(prefix_data_addr|PREFIX_REX_W, di->prefixes);
+        if (size_template == OPSZ_4_rex8)
+            return !TEST(PREFIX_REX_W, di->prefixes);
+        if (size_template == OPSZ_8_short4) {
+            di->prefixes |= prefix_data_addr;
+            return true;
+        }
+        return false;
+    case OPSZ_8_short2:
+        if (size_template == OPSZ_8 || size_template == OPSZ_2)
+            return true; /* will take prefix or no prefix */
+        if (size_template == OPSZ_4_short2) {
+            di->prefixes |= prefix_data_addr;
+            return true;
+        }
+        if (size_template == OPSZ_4_rex8_short2) {
+            if (TEST(prefix_data_addr, di->prefixes))
+                return true; /* Already shrinking so ok */
+            /* FIXME - ambiguous on 64-bit (could widen to 8 or shrink to 2).
+             * We choose to widen by default for 64-bit as that seems the more likely
+             * usage, but whatever choice we make here could conflict with a later
+             * operand and lead to encoding failure even if there was a possible match. */
+            if (X64_MODE(di))
+                di->prefixes |= PREFIX_REX_W;
+            else
+                di->prefixes |= prefix_data_addr;
+            return true;
+        }
+        if (X64_MODE(di) && size_template == OPSZ_4_rex8) {
+            di->prefixes |= PREFIX_REX_W;
+            return true;
+        }
+        if (size_template == OPSZ_8_short4)
+            return !TEST(prefix_data_addr, di->prefixes);
+        return false;
+    case OPSZ_8_short4:
+        if (size_template == OPSZ_4_rex8 || size_template == OPSZ_8 ||
+            size_template == OPSZ_4)
+            return true; /* will take prefix or no prefix */
+        if (size_template == OPSZ_4_short2 || size_template == OPSZ_4_rex8_short2 ||
+            size_template == OPSZ_8_short2)
+            return !TEST(prefix_data_addr, di->prefixes);
+        if (size_template == OPSZ_6_irex10_short4 &&
+            (proc_get_vendor() == VENDOR_AMD || !TEST(PREFIX_REX_W, di->prefixes))) {
+            di->prefixes |= prefix_data_addr;
+            return true;
+        }
+        return false;
+    case OPSZ_28_short14:
+        if (size_template == OPSZ_28 || size_template == OPSZ_14)
+            return true; /* will take prefix or no prefix */
+        return false;
+    case OPSZ_108_short94:
+        if (size_template == OPSZ_108 || size_template == OPSZ_94)
+            return true; /* will take prefix or no prefix */
+        return false;
+    default:
+        CLIENT_ASSERT(false, "size_ok_varsz() internal decoding error (invalid size)");
+        break;
+    }
+    return false;
+}
+
+static opnd_size_t
+resolve_var_x64_size(decode_info_t *di/*x86_mode is IN*/,
+                     opnd_size_t sz, bool addr_short4)
+{
+    /* Resolve what we can based purely on x64 and addr_short4
+     * (gets rid of all NxM sizes), as well as vendor where the size
+     * differences are static.  FIXME - could also resolve rex
+     * availability and vendor rex-varying sizes here,
+     * but not without adding more types that would make
+     * size_ok routines more complicated.  */
+    switch (sz) {
+    case OPSZ_4x8: return (X64_MODE(di) ? OPSZ_8 : OPSZ_4);
+    case OPSZ_4_short2xi4: return (X64_MODE(di) && proc_get_vendor() == VENDOR_INTEL ?
+                                  OPSZ_4 : OPSZ_4_short2);
+    case OPSZ_4x8_short2: return (X64_MODE(di) ? 
+                                  (addr_short4 ? OPSZ_8_short4 : OPSZ_8_short2) :
+                                  OPSZ_4_short2);
+    case OPSZ_4x8_short2xi8: return (X64_MODE(di) ? (proc_get_vendor() == VENDOR_INTEL ?
+                                                  OPSZ_8 : OPSZ_8_short2) :
+                                    OPSZ_4_short2);
+    case OPSZ_6x10: return (X64_MODE(di) ? OPSZ_10 : OPSZ_6);
+    }
+    return sz;
+}
+
+/* Caller should resolve the OPSZ_*_reg* sizes prior to calling this
+ * routine, as here we don't know the operand types
+ * Note that this routine modifies prefixes, so it is not idempotent; the
+ * prefixes are stateful and are kept around as each operand is checked to
+ * ensure the later ones are ok w/ prefixes needed for the earlier ones.
+ */
+static bool
+size_ok(decode_info_t *di/*prefixes field is IN/OUT; x86_mode is IN*/,
+        opnd_size_t size_op, opnd_size_t size_template, bool addr)
+{
+    uint prefix_data_addr = (addr ? PREFIX_ADDR : PREFIX_DATA);
+    /* for OPSZ_4x8_short2, does the addr prefix select 4 instead of 2 bytes? */
+    bool addr_short4 = X64_MODE(di) && addr;
+    /* Assumption: the only addr-specified operands that can be short
+     * are OPSZ_4x8_short2 and OPSZ_4x8_short2xi8
+     */
+    CLIENT_ASSERT(!addr || size_template == OPSZ_4x8_short2xi8 ||
+                  size_template == OPSZ_4x8_short2, "internal prefix assumption error");
+    size_template = resolve_var_x64_size(di, size_template, addr_short4);
+    size_op = resolve_var_x64_size(di, size_op, addr_short4);
+    /* all NxM sizes should be resolved (size_op is checked in the switch statement as
+     * these values will hit the default assert) */
+    CLIENT_ASSERT(size_template != OPSZ_6x10 && size_template != OPSZ_4x8_short2 &&
+                  size_template != OPSZ_4x8_short2xi8 &&
+                  size_template != OPSZ_4_short2xi4 && size_template != OPSZ_4x8,
+                  "internal encoding error in size_ok()");
+
+    /* First set/check rex.w or data prefix, if necessary
+     * if identical size then don't need to set or check anything */
+    if (size_op != size_template) { 
+        switch (size_op) {
+        case OPSZ_1:
+            if (size_template == OPSZ_2_short1) {
+                di->prefixes |= prefix_data_addr;
+                return true;
+            }
+            return false;
+        case OPSZ_2:
+            if (size_template == OPSZ_2_short1)
+                return !TEST(prefix_data_addr, di->prefixes);
+            if (size_template == OPSZ_4_short2 || size_template == OPSZ_8_short2) {
+                di->prefixes |= prefix_data_addr;
+                return true;
+            }
+            if (size_template == OPSZ_4_rex8_short2) {
+                if (TEST(PREFIX_REX_W, di->prefixes))
+                    return false; /* rex.w trumps data prefix */
+                di->prefixes |= prefix_data_addr;
+                return true;
+            }
+            return false;
+        case OPSZ_4:
+            if (size_template == OPSZ_4_short2)
+                return !TEST(prefix_data_addr, di->prefixes);
+            if (size_template == OPSZ_4_rex8_short2)
+                return !TESTANY(prefix_data_addr|PREFIX_REX_W, di->prefixes);
+            if (size_template == OPSZ_4_rex8)
+                return !TEST(PREFIX_REX_W, di->prefixes);
+            if (size_template == OPSZ_6_irex10_short4) {
+                if (TEST(PREFIX_REX_W, di->prefixes) && proc_get_vendor() != VENDOR_AMD)
+                    return false; /* rex.w trumps data prefix */
+                di->prefixes |= prefix_data_addr;
+                return true;
+            }
+            if (size_template == OPSZ_8_short4) {
+                di->prefixes |= prefix_data_addr;
+                return true;
+            }
+            return false;
+        case OPSZ_6:
+            if (size_template == OPSZ_6_irex10_short4)
+                return !TEST(prefix_data_addr, di->prefixes) &&
+                    (!TEST(PREFIX_REX_W, di->prefixes) || proc_get_vendor() == VENDOR_AMD);
+            return false;
+        case OPSZ_8:
+            if (X64_MODE(di) &&
+                (size_template == OPSZ_4_rex8 || size_template == OPSZ_4_rex8_short2)) {
+                di->prefixes |= PREFIX_REX_W; /* rex.w trumps data prefix */
+                return true;
+            }
+            if (size_template == OPSZ_8_short4 || size_template == OPSZ_8_short2)
+                return !TEST(prefix_data_addr, di->prefixes);
+            return false;
+        case OPSZ_10:
+            if (X64_MODE(di) && size_template == OPSZ_6_irex10_short4 &&
+                proc_get_vendor() != VENDOR_AMD) {
+                di->prefixes |= PREFIX_REX_W; /* rex.w trumps data prefix */
+                return true;
+            }
+            return false;
+        case OPSZ_16:
+            return false; /* no matching varsz, must be exact match */
+        case OPSZ_14:
+            if (size_template == OPSZ_28_short14) {
+                di->prefixes |= prefix_data_addr;
+                return true;
+            }
+            return false;
+        case OPSZ_28:
+            if (size_template == OPSZ_28_short14)
+                return !TEST(prefix_data_addr, di->prefixes);
+            return false;
+        case OPSZ_94:
+            if (size_template == OPSZ_108_short94) {
+                di->prefixes |= prefix_data_addr;
+                return true;
+            }
+            return false;
+        case OPSZ_108:
+            if (size_template == OPSZ_108_short94)
+                return !TEST(prefix_data_addr, di->prefixes);
+            return false;
+        case OPSZ_512:
+            return false; /* no variable sizes match, need identical request */
+        /* We do support variable-sized requests */
+        case OPSZ_2_short1:
+        case OPSZ_4_short2:
+        case OPSZ_4_rex8_short2:
+        case OPSZ_4_rex8:
+        case OPSZ_6_irex10_short4:
+        case OPSZ_8_short2:
+        case OPSZ_8_short4:
+        case OPSZ_28_short14:
+        case OPSZ_108_short94:
+            return size_ok_varsz(di, size_op, size_template, prefix_data_addr);
+        case OPSZ_1_reg4:
+        case OPSZ_2_reg4:
+        case OPSZ_4_reg16:
+            CLIENT_ASSERT(false, "error: cannot pass OPSZ_*_reg* to size_ok()");
+            return false;
+        case OPSZ_4_of_8:
+        case OPSZ_4_of_16:
+        case OPSZ_8_of_16:
+        case OPSZ_0:
+            /* handled below */
+            break;
+        default:
+            CLIENT_ASSERT(false, "error: unhandled OPSZ_ in size_ok()");
+            return false;
+        }
+    }
+
+    /* prefix doesn't come into play below here: do a direct comparison */
+
+    switch (size_op) {
+    case OPSZ_4_of_8: 
+    case OPSZ_4_of_16: 
+        size_op = OPSZ_4;
+        break;
+    case OPSZ_8_of_16: 
+        size_op = OPSZ_8;
+        break;
+    }
+
+    switch (size_template) {
+    case OPSZ_4_of_8: 
+    case OPSZ_4_of_16: 
+        size_template = OPSZ_4;
+        break;
+    case OPSZ_8_of_16: 
+        size_template = OPSZ_8;
+        break;
+    }
+
+    DOLOG(4, LOG_EMIT, {
+        if (size_op != size_template) {
+            LOG(THREAD_GET, LOG_EMIT, ENC_LEVEL, "size_ok: %s != %s\n",
+                size_names[size_op], size_names[size_template]);
+        }
+    });
+    return (size_op == size_template);
+}
+
+/* We assume size_ok() is called ahead of time to check whether a prefix
+ * is needed.
+ */
+static bool
+immed_size_ok(decode_info_t *di/*prefixes field is IN/OUT; x86_mode is IN*/,
+              ptr_int_t immed, opnd_size_t opsize)
+{
+    opsize = resolve_variable_size(di, opsize, false);
+    switch (opsize) {
+    case OPSZ_1:
+        return (immed >= INT8_MIN && immed <= INT8_MAX);
+    case OPSZ_2: /* unsigned max is 65535 */
+        return (immed >= INT16_MIN && immed <= INT16_MAX);
+#ifndef X64
+    case OPSZ_4:
+        return true;
+#else
+    case OPSZ_4:
+        return (immed >= INT32_MIN && immed <= INT32_MAX);
+    case OPSZ_8:
+        return true;
+#endif
+    default:
+        CLIENT_ASSERT(false, "encode error: immediate has unknown size");
+        return false;
+    }
+}
+
+/* prefixes that aren't set by size_ok */
+static bool
+reg_set_ext_prefixes(decode_info_t *di/*prefixes field is IN/OUT; x86_mode is IN*/,
+                     reg_id_t reg, uint which_rex)
+{
+#ifdef X64
+    if (reg >= REG_START_x64_8 && reg <= REG_STOP_x64_8) {
+        /* alternates to AH-BH that are specified via any rex prefix */
+        if (!TESTANY(PREFIX_REX_ALL, di->prefixes))
+            di->prefixes |= PREFIX_REX_GENERAL;
+    } else if (reg_is_extended(reg))
+        di->prefixes |= which_rex;
+#endif
+    return true; /* for use in && series */
+}
+
+static bool
+reg_size_ok(decode_info_t *di/*prefixes field is IN/OUT; x86_mode is IN*/,
+            reg_id_t reg, int optype, opnd_size_t opsize, bool addr)
+{
+    /* It's ok to have an operand of type mmx or xmm but half-size (e.g., movddup) */ 
+    if (opsize == OPSZ_4_of_8 &&
+        (optype == TYPE_P || optype == TYPE_Q || optype == TYPE_P_MODRM))
+        return (reg >= REG_START_MMX && reg <= REG_STOP_MMX);
+    if ((opsize == OPSZ_4_of_16 || opsize == OPSZ_8_of_16) &&
+        (optype == TYPE_V || optype == TYPE_V_MODRM || optype == TYPE_W))
+        return (reg >= REG_START_XMM && reg <= REG_STOP_XMM);
+    /* We assume that only type p uses OPSZ_6_irex10_short4: w/ data16, even though it's
+     * 4 bytes and would fit in a register, this is invalid.
+     */
+    if (opsize == OPSZ_6_irex10_short4)
+        return false; /* no register of size p */
+    return size_ok(di, reg_get_size(reg), resolve_var_reg_size(opsize, true), addr);
+}
+
+static bool
+reg_rm_selectable(reg_id_t reg)
+{
+    /* assumption: GPR registers (of all sizes) and mmx and xmm are all in a row */
+    return (reg >= REG_START_64 && reg <= REG_STOP_XMM);
+}
+
+static bool
+mem_size_ok(decode_info_t *di/*prefixes field is IN/OUT; x86_mode is IN*/,
+            opnd_t opnd, int optype, opnd_size_t opsize)
+{
+    /* TODO PR 238624: support instr targets as pc-relative? */
+    if (!opnd_is_memory_reference(opnd))
+        return false;
+    if (opnd_is_base_disp(opnd) && opnd_is_disp_short_addr(opnd))
+        di->prefixes |= PREFIX_ADDR;
+    return (size_ok(di, opnd_get_size(opnd), opsize, false/*!addr*/) &&
+            (IF_X64(!opnd_is_base_disp(opnd) ||)
+             opnd_get_base(opnd) == REG_NULL ||
+             reg_size_ok(di, opnd_get_base(opnd), TYPE_M, OPSZ_4x8_short2,
+                         true/*addr*/)) &&
+            (IF_X64(!opnd_is_base_disp(opnd) ||)
+             opnd_get_index(opnd) == REG_NULL ||
+             reg_size_ok(di, opnd_get_index(opnd), TYPE_M, OPSZ_4x8_short2,
+                         true/*addr*/)));
+}
+
+static bool
+opnd_type_ok(decode_info_t *di/*prefixes field is IN/OUT; x86_mode is IN*/,
+             opnd_t opnd, int optype, opnd_size_t opsize)
+{
+    DOLOG(ENC_LEVEL, LOG_EMIT, {
+        dcontext_t *dcontext = get_thread_private_dcontext();
+        LOG(THREAD, LOG_EMIT, ENC_LEVEL, "opnd_type_ok on operand ");
+        opnd_disassemble(dcontext, opnd, THREAD);
+        if (!opnd_is_pc(opnd) && !opnd_is_instr(opnd)) {
+            LOG(THREAD, LOG_EMIT, ENC_LEVEL, "with size %s (%d bytes)\n",
+                size_names[opnd_get_size(opnd)],
+                opnd_size_in_bytes(opnd_get_size(opnd)));
+        }
+        LOG(THREAD, LOG_EMIT, ENC_LEVEL, "\tvs. template type %s with size %s (%d bytes)\n",
+            type_names[optype], size_names[opsize],
+            opnd_size_in_bytes(opsize));
+    });
+    switch (optype) {
+    case TYPE_NONE: 
+        return opnd_is_null(opnd);
+    case TYPE_REG:
+        return (opnd_is_reg(opnd) && opnd_get_reg(opnd) == opsize);
+    case TYPE_VAR_REG:
+        return (opnd_is_reg(opnd) &&
+                reg_size_ok(di, opnd_get_reg(opnd), optype, OPSZ_4_rex8_short2, 
+                            false/*!addr*/) &&
+                opnd_get_reg(opnd) == resolve_var_reg(di, opsize, false, true
+                                                      _IF_X64(false) _IF_X64(true)
+                                                      _IF_X64(false/*!extendable*/)));
+    case TYPE_VARZ_REG:
+        return (opnd_is_reg(opnd) &&
+                reg_size_ok(di, opnd_get_reg(opnd), optype, OPSZ_4_short2,
+                            false/*!addr*/) &&
+                opnd_get_reg(opnd) == resolve_var_reg(di, opsize, false, true
+                                                      _IF_X64(false) _IF_X64(false)
+                                                      _IF_X64(false/*!extendable*/)));
+    case TYPE_VAR_XREG:
+        return (opnd_is_reg(opnd) &&
+                reg_size_ok(di, opnd_get_reg(opnd), optype, OPSZ_4x8_short2,
+                            false/*!addr*/) &&
+                opnd_get_reg(opnd) == resolve_var_reg(di, opsize, false, true
+                                                      _IF_X64(true) _IF_X64(true)
+                                                      _IF_X64(false/*!extendable*/)));
+    case TYPE_VAR_ADDR_XREG:
+        return (opnd_is_reg(opnd) &&
+                reg_size_ok(di, opnd_get_reg(opnd), optype, OPSZ_4x8_short2,
+                            true/*addr*/) &&
+                opnd_get_reg(opnd) == resolve_var_reg(di, opsize, true, true
+                                                      _IF_X64(true) _IF_X64(false)
+                                                      _IF_X64(false/*!extendable*/)));
+    case TYPE_REG_EX:
+        return (opnd_is_reg(opnd) &&
+                reg_size_ok(di, opnd_get_reg(opnd), optype, reg_get_size(opsize),
+                            false/*!addr*/) &&
+                reg_set_ext_prefixes(di, opnd_get_reg(opnd), PREFIX_REX_B) && 
+                opnd_get_reg(opnd) == resolve_var_reg(di, opsize, false, false
+                                                      _IF_X64(false) _IF_X64(false)
+                                                      _IF_X64(true/*extendable*/)));
+    case TYPE_VAR_REG_EX:
+        return (opnd_is_reg(opnd) &&
+                reg_size_ok(di, opnd_get_reg(opnd), optype, OPSZ_4_rex8_short2,
+                            false/*!addr*/) &&
+                reg_set_ext_prefixes(di, opnd_get_reg(opnd), PREFIX_REX_B) && 
+                opnd_get_reg(opnd) == resolve_var_reg(di, opsize, false, true
+                                                      _IF_X64(false) _IF_X64(true)
+                                                      _IF_X64(true/*extendable*/)));
+    case TYPE_VAR_XREG_EX:
+        return (opnd_is_reg(opnd) &&
+                reg_size_ok(di, opnd_get_reg(opnd), optype, OPSZ_4x8_short2,
+                            false/*!addr*/) &&
+                reg_set_ext_prefixes(di, opnd_get_reg(opnd), PREFIX_REX_B) && 
+                opnd_get_reg(opnd) == resolve_var_reg(di, opsize, false, true
+                                                      _IF_X64(true) _IF_X64(true)
+                                                      _IF_X64(true/*extendable*/)));
+    case TYPE_VAR_REGX_EX:
+        return (opnd_is_reg(opnd) &&
+                reg_size_ok(di, opnd_get_reg(opnd), optype, OPSZ_4_rex8,
+                            false/*!addr*/) &&
+                reg_set_ext_prefixes(di, opnd_get_reg(opnd), PREFIX_REX_B) && 
+                opnd_get_reg(opnd) == resolve_var_reg(di, opsize, false, false
+                                                      _IF_X64(false) _IF_X64(true)
+                                                      _IF_X64(true/*extendable*/)));
+    case TYPE_FLOATMEM:
+    case TYPE_M:
+        return mem_size_ok(di, opnd, optype, opsize);
+    case TYPE_E:
+    case TYPE_Q:
+    case TYPE_W: 
+    case TYPE_INDIR_E:
+        return (mem_size_ok(di, opnd, optype, opsize) || 
+                (opnd_is_reg(opnd) &&
+                 reg_size_ok(di, opnd_get_reg(opnd), optype, opsize, false/*!addr*/) &&
+                 reg_rm_selectable(opnd_get_reg(opnd))));
+    case TYPE_G:
+    case TYPE_P:
+    case TYPE_V:
+    case TYPE_R:
+    case TYPE_P_MODRM:
+    case TYPE_V_MODRM:
+        /* We are able to rule out segment registers b/c they should use TYPE_S
+         * (OP_mov_seg) or hardcoded (push cs) (if we don't rule them out
+         * they can match a 16-bit GPR slot by size alone); CR and DR also
+         * have separate types (TYPE_C and TYPE_D).
+         */
+        return (opnd_is_reg(opnd) &&
+                reg_size_ok(di, opnd_get_reg(opnd), optype, opsize,
+                            false/*!addr*/) &&
+                reg_rm_selectable(opnd_get_reg(opnd))); /* reg, not rm, but see above */
+    case TYPE_C:
+        return (opnd_is_reg(opnd) &&
+                reg_size_ok(di, opnd_get_reg(opnd), optype, opsize,
+                            false/*!addr*/) &&
+                opnd_get_reg(opnd) >= REG_START_CR && opnd_get_reg(opnd) <= REG_STOP_CR);
+    case TYPE_D:
+        return (opnd_is_reg(opnd) &&
+                reg_size_ok(di, opnd_get_reg(opnd), optype, opsize,
+                            false/*!addr*/) &&
+                opnd_get_reg(opnd) >= REG_START_DR && opnd_get_reg(opnd) <= REG_STOP_DR);
+    case TYPE_S:
+        return (opnd_is_reg(opnd) &&
+                opnd_get_reg(opnd) >= REG_START_SEGMENT &&
+                opnd_get_reg(opnd) <= REG_STOP_SEGMENT);
+    case TYPE_I:
+        /* we allow instr: it means 4/8-byte immed equal to pc of instr */
+        return ((opnd_is_near_instr(opnd) &&
+                 size_ok(di, OPSZ_PTR, opsize, false/*!addr*/)) ||
+                (opnd_is_immed_int(opnd) &&
+                 size_ok(di, opnd_get_size(opnd), opsize, false/*!addr*/) &&
+                 immed_size_ok(di, opnd_get_immed_int(opnd), opsize)));
+    case TYPE_1:
+        /* FIXME (xref PR 229127): Ib vs c1: if say "1, OPSZ_1" will NOT match c1 and
+         * will get the Ib version: do we want to match c1?  What if they really want
+         * an immed byte in the encoding?  OTOH, we do match constant registers
+         * automatically w/ no control from the user.
+         * Currently, we document in instr_create.h that the user must
+         * specify OPSZ_0 in order to get c1.
+         */
+        return (opnd_is_immed_int(opnd) && opnd_get_immed_int(opnd) == 1 &&
+                size_ok(di, opnd_get_size(opnd), opsize, false/*!addr*/));
+    case TYPE_FLOATCONST:
+        return (opnd_is_immed_float(opnd)); /* FIXME: is actual float const decoded? */
+    case TYPE_J:
+        /* FIXME PR 225937: support 16-bit data16 immediates */
+        /* FIXME: need relative pc offset to test immed_size_ok, but all we have
+         * here is absolute pc or instr: but we don't auto-select opcode, and opcode
+         * selects immed size (except for data16 which we don't support),
+         * so we don't need to choose among templates now: we'll complain
+         * at emit time if we have reachability issues. */
+        return (opnd_is_near_pc(opnd) || opnd_is_near_instr(opnd));
+    case TYPE_A: 
+        {
+            CLIENT_ASSERT(!X64_MODE(di), "x64 has no type A instructions");
+#ifdef IA32_ON_IA64
+            if (opsize != OPSZ_6_irex10_short4) {
+                return (opnd_is_near_instr(opnd) ||
+                        (opnd_is_near_pc(opnd) &&
+                         immed_size_ok(di, (uint)opnd_get_pc(opnd), opsize)));
+            }
+#endif
+            return (opnd_is_far_pc(opnd) || opnd_is_far_instr(opnd));
+        }
+    case TYPE_O:
+        return (opnd_is_abs_addr(opnd) &&
+                size_ok(di, opnd_get_size(opnd), opsize, false/*!addr*/));
+    case TYPE_X:
+        /* this means the memory address DS:(RE)(E)SI */
+        if (opnd_is_far_base_disp(opnd)) {
+            reg_id_t base = opnd_get_base(opnd);
+            /* reg_size_ok will set PREFIX_ADDR if necessary */
+            return (reg_size_ok(di, base, optype, OPSZ_4x8_short2, true/*addr*/) &&
+                    reg_is_segment(opnd_get_segment(opnd)) &&
+                    base == resolve_var_reg(di, REG_ESI, true, true
+                                            _IF_X64(true) _IF_X64(false)
+                                            _IF_X64(false)) &&
+                    opnd_get_index(opnd) == REG_NULL &&
+                    opnd_get_disp(opnd) == 0 &&
+                    size_ok(di, opnd_get_size(opnd), opsize, false/*!addr*/));
+        } else {
+            return false;
+        }
+    case TYPE_Y:
+        /* this means the memory address ES:(RE)(E)DI */
+        if (opnd_is_far_base_disp(opnd)) {
+            reg_id_t base = opnd_get_base(opnd);
+            /* reg_size_ok will set PREFIX_ADDR if necessary */
+            return (reg_size_ok(di, base, optype, OPSZ_4x8_short2, true/*addr*/) &&
+                    opnd_get_segment(opnd) == SEG_ES &&
+                    base == resolve_var_reg(di, REG_EDI, true, true
+                                            _IF_X64(true) _IF_X64(false)
+                                            _IF_X64(false)) &&
+                    opnd_get_index(opnd) == REG_NULL &&
+                    opnd_get_disp(opnd) == 0 &&
+                    size_ok(di, opnd_get_size(opnd), opsize, false/*!addr*/));
+        } else {
+            return false;
+        }
+    case TYPE_XLAT:
+        /* this means the memory address DS:(RE)(E)BX+AL */
+        if (opnd_is_far_base_disp(opnd)) {
+            reg_id_t base = opnd_get_base(opnd);
+            /* reg_size_ok will set PREFIX_ADDR if necessary */
+            return (reg_size_ok(di, base, optype, OPSZ_4x8_short2, true/*addr*/) &&
+                    reg_is_segment(opnd_get_segment(opnd)) &&
+                    base == resolve_var_reg(di, REG_EBX, true, true
+                                            _IF_X64(true) _IF_X64(false)
+                                            _IF_X64(false)) &&
+                    opnd_get_index(opnd) == REG_AL &&
+                    opnd_get_scale(opnd) == 1 &&
+                    opnd_get_disp(opnd) == 0 &&
+                    size_ok(di, opnd_get_size(opnd), opsize, false/*!addr*/));
+        } else {
+            return false;
+        }
+    case TYPE_MASKMOVQ:
+        /* this means the memory address DS:(RE)(E)DI */
+        if (opnd_is_far_base_disp(opnd)) {
+            reg_id_t base = opnd_get_base(opnd);
+            /* reg_size_ok will set PREFIX_ADDR if necessary */
+            return (reg_size_ok(di, base, optype, OPSZ_4x8_short2, true/*addr*/) &&
+                    reg_is_segment(opnd_get_segment(opnd)) &&
+                    base == resolve_var_reg(di, REG_EDI, true, true
+                                            _IF_X64(true) _IF_X64(false)
+                                            _IF_X64(false)) &&
+                    opnd_get_index(opnd) == REG_NULL &&
+                    opnd_get_disp(opnd) == 0 &&
+                    size_ok(di, opnd_get_size(opnd), opsize, false/*!addr*/));
+        } else {
+            return false;
+        }
+    case TYPE_INDIR_REG:
+        /* far_ ok */
+        return (opnd_is_base_disp(opnd) &&
+                opnd_get_base(opnd) == opsize &&
+                opnd_get_index(opnd) == REG_NULL &&
+                opnd_get_disp(opnd) == 0 &&
+                /* FIXME: how know data size?  for now just use reg size... */
+                size_ok(di, opnd_get_size(opnd), reg_get_size(opsize), false/*!addr*/));
+    case TYPE_INDIR_VAR_XREG: /* indirect reg that varies (by addr16), base is 4x8,
+                               * opsize that varies by data16 */
+    case TYPE_INDIR_VAR_REG: /* indrect reg that varies (by addr16), base is 4x8,
+                              * opsize that varies by rex & data16 */
+    case TYPE_INDIR_VAR_XIREG: /* indrect reg that varies (by addr16), base is 4x8,
+                                * opsize that varies by data16 except on 64-bit Intel */
+        if (opnd_is_base_disp(opnd)) {
+            reg_id_t base = opnd_get_base(opnd);
+            /* reg_size_ok will set PREFIX_ADDR if 16-bit reg is asked for.
+             * these are all specified as 32-bit, so we hardcode OPSZ_VARSTACK
+             * for reg_size_ok.
+             * to generalize we'll want opsize_var_size(reg_get_size(opsize)) or sthg.
+             * also xref case 214976/10541.
+             */
+            CLIENT_ASSERT(reg_get_size(opsize) == OPSZ_4, "internal decoding error");
+            return (reg_size_ok(di, base, optype, OPSZ_VARSTACK, true/*addr*/) &&
+                    base == resolve_var_reg(di, opsize, true,
+                                            true _IF_X64(true) _IF_X64(false)
+                                            _IF_X64(false)) &&
+                    opnd_get_index(opnd) == REG_NULL &&
+                    opnd_get_disp(opnd) == 0 &&
+                    /* FIXME: xref PR 214976: this is used for some instructions that
+                     * aren't actually OPSZ_VARSTACK.  NOTE - needs to match size in
+                     * decode_operand() and instr_create.h. */
+                    size_ok(di, opnd_get_size(opnd), indir_var_reg_size(optype),
+                            false/*!addr*/));
+        } else {
+            return false;
+        }
+    default:
+        CLIENT_ASSERT(false, "encode error: unknown operand type");
+        return false;
+    }
+}
+
+const instr_info_t *
+instr_info_extra_opnds(const instr_info_t *info)
+{
+    if (TEST(HAS_EXTRA_OPERANDS, info->flags)) {
+        if (TEST(EXTRAS_IN_CODE_FIELD, info->flags))
+            return (const instr_info_t *)(info->code);
+        else /* extra operands are in next entry */
+            return (info + 1);
+    }
+    return NULL;
+}
+
+/* macro for speed so we don't have to pass opnds around */
+#define TEST_OPND(di, iitype, iisize, iinum, inst_num, get_op)   \
+    if (iitype != TYPE_NONE) {                                   \
+        if (inst_num < iinum)                                    \
+            return false;                                        \
+        if (!opnd_type_ok(di, get_op, iitype, iisize))           \
+            return false;                                        \
+        if (type_instr_uses_reg_bits(iitype)) {                  \
+            if (!opnd_is_null(using_reg_bits) &&                 \
+                !opnd_same(using_reg_bits, get_op))              \
+                return false;                                    \
+            using_reg_bits = get_op;                             \
+        } else if (type_uses_modrm_bits(iitype)) {               \
+            if (!opnd_is_null(using_modrm_bits) &&               \
+                !opnd_same(using_modrm_bits, get_op))            \
+                return false;                                    \
+            using_modrm_bits = get_op;                           \
+        }                                                        \
+    } else if (inst_num >= iinum)                                \
+        return false;
+
+/* May be called a 2nd time to check size prefix consistency.
+ * FIXME optimization: in 2nd pass we only need to call opnd_type_ok()
+ * and don't need to check reg, modrm, numbers, etc.
+ */
+static bool
+encoding_possible_pass(decode_info_t *di, instr_t *in, const instr_info_t * ii)
+{
+#ifdef DEBUG
+    dcontext_t *dcontext = get_thread_private_dcontext();
+#endif
+    /* make sure multiple operands aren't using same modrm bits */
+    opnd_t using_reg_bits = opnd_create_null();
+    opnd_t using_modrm_bits = opnd_create_null();
+
+    /* for efficiency we separately test 2 dsts, 3 srcs */
+    TEST_OPND(di, ii->dst1_type, ii->dst1_size, 1, in->num_dsts, instr_get_dst(in, 0));
+    TEST_OPND(di, ii->dst2_type, ii->dst2_size, 2, in->num_dsts, instr_get_dst(in, 1));
+    TEST_OPND(di, ii->src1_type, ii->src1_size, 1, in->num_srcs, instr_get_src(in, 0));
+    TEST_OPND(di, ii->src2_type, ii->src2_size, 2, in->num_srcs, instr_get_src(in, 1));
+    TEST_OPND(di, ii->src3_type, ii->src3_size, 3, in->num_srcs, instr_get_src(in, 2));
+
+    if ((ii->flags & HAS_EXTRA_OPERANDS) != 0) {
+        /* extra operands to test! */
+        int offs = 1;
+        ii = instr_info_extra_opnds(ii);
+        LOG(THREAD, LOG_EMIT, ENC_LEVEL, "encoding_possible extra operands\n");
+        do {
+            LOG(THREAD, LOG_EMIT, ENC_LEVEL,
+                "encoding possible checking extra operands for "PFX"\n",
+                ii->opcode);
+            CLIENT_ASSERT(ii->type == OP_CONTD,
+                          "encode error: extra operand template mismatch");
+
+            TEST_OPND(di, ii->dst1_type, ii->dst1_size, (offs*2 + 1), in->num_dsts,
+                      instr_get_dst(in, (offs*2 + 0)));
+            TEST_OPND(di, ii->dst2_type, ii->dst2_size, (offs*2 + 2), in->num_dsts,
+                      instr_get_dst(in, (offs*2 + 1)));
+            TEST_OPND(di, ii->src1_type, ii->src1_size, (offs*3 + 1), in->num_srcs,
+                      instr_get_src(in, (offs*3 + 0)));
+            TEST_OPND(di, ii->src2_type, ii->src2_size, (offs*3 + 2), in->num_srcs,
+                      instr_get_src(in, (offs*3 + 1)));
+            TEST_OPND(di, ii->src3_type, ii->src3_size, (offs*3 + 3), in->num_srcs,
+                      instr_get_src(in, (offs*3 + 2)));
+            offs++;
+            ii = instr_info_extra_opnds(ii);
+        } while (ii != NULL);
+    }
+
+    return true;
+}
+
+/* Does not check operands beyond 2 dsts and 3 srcs! 
+ * Modifies in's prefixes to reflect whether operand or data size
+ * prefixes are required.
+ * Assumes caller has set di->x86_mode (i.e., ignores in's mode).
+ */
+static bool
+encoding_possible(decode_info_t *di, instr_t *in, const instr_info_t * ii)
+{
+#ifdef DEBUG
+    dcontext_t *dcontext = get_thread_private_dcontext();
+#endif
+    if (ii == NULL || in == NULL)
+        return false;
+    LOG(THREAD, LOG_EMIT, ENC_LEVEL, "\nencoding_possible on "PFX"\n", ii->opcode);
+
+    if (TEST(IF_X64_ELSE(X64_INVALID, X86_INVALID), ii->flags))
+        return false;
+
+    /* For size prefixes we use the di prefix field since that's what
+     * the decode.c routines use; we transfer to the instr's prefix field
+     * when done.  The first
+     * operand that would need a prefix to match its template sets the
+     * prefixes.  Rather than force operands that don't want prefixes
+     * to say so (thus requiring a 3-value field: uninitialized,
+     * prefix, and no-prefix, and extra work in the common case) we
+     * instead do a 2nd pass if any operand wanted a prefix.
+     * If an operand wants no prefix and the flag is set, the match fails.
+     * I.e., first pass: does anyone want a prefix?  If so, 2nd pass: does
+     * everyone want a prefix?  We also re-check the immed sizes on the 2nd
+     * pass.
+     *
+     * If an operand specifies a variable-sized size, it will take on either of
+     * the default size or the prefix size.
+     */
+    di->prefixes &= ~PREFIX_SIZE_SPECIFIERS;
+    if (!encoding_possible_pass(di, in, ii))
+        return false;
+    if (TESTANY(PREFIX_SIZE_SPECIFIERS, di->prefixes)) {
+        LOG(THREAD, LOG_EMIT, ENC_LEVEL, "\tflags needed: "PFX"\n", in->prefixes);
+        if (!encoding_possible_pass(di, in, ii))
+            return false;
+    }
+    LOG(THREAD, LOG_EMIT, ENC_LEVEL, "\ttemplate match w/ flags: "PFX"\n", in->prefixes);
+    return true;
+}
+
+/* exported, looks at all possible instr_info_t templates
+ */
+bool
+instr_is_encoding_possible(instr_t *instr)
+{
+    const instr_info_t * info = get_encoding_info(instr);
+    return (info != NULL);
+}
+
+/* looks at all possible instr_info_t templates, returns first match
+ * returns NULL if no encoding is possible
+ */
+const instr_info_t *
+get_encoding_info(instr_t *instr)
+{
+    const instr_info_t * info = instr_get_instr_info(instr);
+    decode_info_t di = {0};
+    IF_X64(di.x86_mode = instr_get_x86_mode(instr));
+
+    while (!encoding_possible(&di, instr, info)) {
+        info = get_next_instr_info(info);
+        /* stop when hit end of list or when hit extra operand tables (OP_CONTD) */
+        if (info == NULL || info->opcode == OP_CONTD) {
+            return NULL;
+        }
+    }
+    return info;
+}
+
+/* num is 0-based */
+byte
+instr_info_opnd_type(const instr_info_t *info, bool src, uint num)
+{
+    if (num < 0) {
+        CLIENT_ASSERT(false, "internal decode error");
+        return TYPE_NONE;
+    }
+    if ((src && num >= 3) || (!src && num >= 2)) {
+        const instr_info_t *nxt = instr_info_extra_opnds(info);
+        if (nxt == NULL) {
+            CLIENT_ASSERT(false, "internal decode error");
+            return TYPE_NONE;
+        } else
+            return instr_info_opnd_type(nxt, src, src ? (num - 3) : (num - 2));
+    } else {
+        if (src) {
+            if (num == 0)
+                return info->src1_type;
+            else if (num == 1)
+                return info->src2_type;
+            else if (num == 2)
+                return info->src3_type;
+            else {
+                CLIENT_ASSERT(false, "internal decode error");
+                return TYPE_NONE;
+            }
+        } else {
+            if (num == 0)
+                return info->dst1_type;
+            else if (num == 1)
+                return info->dst2_type;
+            else {
+                CLIENT_ASSERT(false, "internal decode error");
+                return TYPE_NONE;
+            }
+        }
+    }
+    return TYPE_NONE;
+}
+
+/***************************************************************************
+ * Actual encoding
+ */
+
+static byte *
+encode_immed(decode_info_t * di, byte *pc)
+{
+    /* 1st or 2nd immed? */
+    ptr_int_t val;
+    opnd_size_t size;
+    if (di->size_immed != OPSZ_NA) {
+        /* do we need to pc-relativize a target pc? */
+        if (di->size_immed == OPSZ_512) { /* special code */
+            int len;
+            size = di->size_immed2; /* TYPE_J put real size there */
+            di->size_immed2 = OPSZ_NA;
+            if ((size == OPSZ_4_short2 && !TEST(PREFIX_DATA, di->prefixes)) ||
+                size == OPSZ_4)
+                len = 4;
+            else if (size == OPSZ_4_short2 && TEST(PREFIX_DATA, di->prefixes))
+                len = 2;
+            else if (size == OPSZ_1)
+                len = 1;
+            else {
+                len = 0; /* avoid compiler warning */
+                CLIENT_ASSERT(false, "encode error: immediate has unknown size");
+            }
+            /* offset is from start of next instruction */
+            val = di->immed - ((ptr_int_t)pc + len);
+        } else if (di->size_immed == OPSZ_10) { /* another code */
+            /* this code means that the immed holds not the absolute pc but
+             * the offset not counting the instruction length
+             */
+            int len;
+            size = di->size_immed2; /* TYPE_J put real size there */
+            di->size_immed2 = OPSZ_NA;
+            if ((size == OPSZ_4_short2 && !TEST(PREFIX_DATA, di->prefixes)) ||
+                size == OPSZ_4)
+                len = 4;
+            else if (size == OPSZ_4_short2 && TEST(PREFIX_DATA, di->prefixes))
+                len = 2;
+            else if (size == OPSZ_1)
+                len = 1;
+            else {
+                len = 0; /* avoid compiler warning */
+                CLIENT_ASSERT(false, "encode error: immediate has unknown size");
+            }
+            /* just need to subtract the total instr length from the offset */
+            /* HACK: di->modrm was set with the number of instruction bytes
+             * prior to this immed
+             */
+            val = di->immed - (len + di->modrm);
+        } else if (di->size_immed == OPSZ_28_short14) { /* another code */
+            /* this code means that the immed holds not the absolute pc but
+             * the offset
+             */
+            size = di->size_immed2; /* TYPE_I put real size there */
+            di->size_immed2 = OPSZ_NA;
+            CLIENT_ASSERT((size == OPSZ_4_short2 && !TEST(PREFIX_DATA, di->prefixes)) ||
+                          size == OPSZ_4,
+                          "encode error: immediate has invalid size");
+            /* we want val to be pc of target instr 
+             * immed is the difference between us and target
+             * HACK: di->modrm was set with the number of instruction bytes
+             * prior to this immed
+             */
+            val = di->immed + (ptr_int_t)pc - di->modrm;
+        } else {
+            val = di->immed;
+            size = di->size_immed;
+        }
+        di->size_immed = OPSZ_NA; /* mark as used */
+    } else {
+        CLIENT_ASSERT(di->size_immed2 != OPSZ_NA,
+                      "encode error: immediate has invalid size");
+        val = di->immed2;
+        size = di->size_immed2;
+        di->size_immed2 = OPSZ_NA; /* mark as used */
+    }
+
+    /* variable-sized */
+    size = resolve_variable_size(di, size, false);
+
+    switch (size) {
+    case OPSZ_1:
+        *pc = (byte) (val);
+        pc += 1;
+        break;
+    case OPSZ_2:
+        *((short *)pc) = (short) (val);
+        pc += 2;
+        break;
+    case OPSZ_4:
+        *((int *)pc) = (int) val;
+        pc += 4;
+        break;
+#ifdef X64
+    case OPSZ_8:
+        *((int64 *)pc) = val;
+        pc += 8;
+        break;
+#endif
+    case OPSZ_6:
+        CLIENT_ASSERT(di->size_immed2 == size,
+                      "encode error: immediate has invalid size OPSZ_6");
+        di->size_immed2 = OPSZ_NA;
+        *((short *)pc) = (short) (di->immed);
+        pc += 2;
+        *((int *)pc) = (int) di->immed2;
+        pc += 4;
+        break;
+    default:
+        LOG(THREAD_GET, LOG_EMIT, 1, "ERROR: encode_immed: unhandled size: %d\n", size);
+        CLIENT_ASSERT(false, "encode error: immediate has unknown size");
+    }
+    return pc;
+}
+
+static void
+encode_reg_ext_prefixes(decode_info_t *di, reg_id_t reg, uint which_rex)
+{
+#ifdef X64
+    reg_set_ext_prefixes(di, reg, which_rex);
+#endif
+}
+
+#ifdef X64
+static void
+encode_rel_addr(decode_info_t * di, opnd_t opnd)
+{
+    /* Unlike TYPE_J and TYPE_I, who use immed values, can assume
+     * there are no other immeds, and have encode_immed complete
+     * the pc relativization once the final pc is known, we have
+     * to use a different mechanism as we're dealing with a disp
+     * and can have other immeds.
+     * We simply have instr_encode check for this exact modrrm
+     * and use a new field disp_abs to store our target.
+     */
+    CLIENT_ASSERT(opnd_is_rel_addr(opnd),
+                  "encode error: invalid type for pc-relativization");
+    di->has_sib = false;
+    di->mod = 0;
+    di->rm = 5;
+    di->has_disp = true;
+    di->disp_abs = (byte *) opnd_get_addr(opnd);
+    /* PR 253327: since we have no explicit request for addr32, we
+     * deduce it here, w/ a conservative range estimate of instr length.
+     * However, we consult use_addr_prefix_on_short_disp() first, which will
+     * probably disallow for most x64 processors for performance reasons.
+     */
+    if (use_addr_prefix_on_short_disp() &&
+        (ptr_uint_t)di->disp_abs <= INT_MAX &&
+        (!REL32_REACHABLE(di->start_pc + MAX_INSTR_LENGTH, di->disp_abs) ||
+         !REL32_REACHABLE(di->start_pc + 4, di->disp_abs)))
+        di->prefixes |= PREFIX_ADDR;
+}
+#endif
+
+static void
+encode_base_disp(decode_info_t * di, opnd_t opnd)
+{
+    reg_id_t base, index;
+    int scale, disp;
+    /* in 64-bit mode, addr prefix simply truncates registers and final address */
+    bool addr16 = !X64_MODE(di) && TEST(PREFIX_ADDR, di->prefixes);
+    base = opnd_get_base(opnd);
+    index = opnd_get_index(opnd);
+    scale = opnd_get_scale(opnd);
+    disp = opnd_get_disp(opnd);
+
+    /* user can use opnd_create_abs_addr() but it will internally be a base-disp
+     * if its disp is 32-bit: if it's larger it has to be TYPE_O and not get here!
+     */
+    CLIENT_ASSERT(opnd_is_base_disp(opnd), "encode error: operand type mismatch");
+    if (di->mod < 5) {
+        /* mod, rm, & sib have already been set, probably b/c
+         * we have a src that equals a dst.
+         * just exit.
+         */
+        return;
+    }
+
+    if (base == REG_NULL && index == REG_NULL) {
+        /* absolute displacement */
+        if (!addr16 && di->seg_override != REG_NULL &&
+            ((!X64_MODE(di) && disp >= INT16_MIN && disp <= INT16_MAX) ||
+             (X64_MODE(di) && disp >= INT32_MIN && disp <= INT32_MAX)) &&
+            !opnd_is_disp_force_full(opnd)) {
+            /* already have segment prefix, so adding addr16 prefix
+             * won't make worse (already in slow decoder on processor), so try to
+             * reduce size: unless on newer microarch: see comments in
+             * use_addr_prefix_on_short_disp().
+             * if a client doesn't want this happening to a patch-later value, should
+             * use a large bogus value that won't trigger this, or
+             * specify force_full_disp.
+             */
+            if (!X64_MODE(di) && /* disp always 32-bit for x64 */
+                use_addr_prefix_on_short_disp()) {
+                di->prefixes |= PREFIX_ADDR; /* for 16-bit disp */
+                addr16 = true;
+            }
+        }
+        if (X64_MODE(di)) {
+            /* need a sib byte to do abs (not rip-relative) */
+            di->mod = 0;
+            di->rm = 4;
+            di->has_sib = true;
+            di->scale = 0;
+            di->index = 4;
+            di->base = 5;
+            di->has_disp = true;
+            di->disp = disp;
+            /* if rex.x is set we'll have r12 instead of no base */
+            CLIENT_ASSERT(!TEST(PREFIX_REX_X, di->prefixes),
+                          "encode error: for x64 cannot encode abs addr w/ rex.x");
+        } else {
+            di->has_sib = false;
+            di->mod = 0;
+            di->rm = (byte) ((addr16) ? 6 : 5);
+            di->has_disp = true;
+            di->disp = disp;
+        }
+    } else {
+        if (disp == 0 &&
+            /* must use 8-bit disp for 0x0(%ebp) or 0x0(%r13) */
+            ((!addr16 && base != REG_EBP /* x64 w/ addr prefix => ebp */
+              IF_X64(&& base != REG_RBP && base != REG_R13 && base != REG_R13D)) ||
+             /* must use 8-bit disp for 0x0(%bp) */
+             (addr16 && (base != REG_BP || index != REG_NULL))) &&
+            !opnd_is_disp_encode_zero(opnd)) {
+            /* no disp */
+            di->mod = 0;
+            di->has_disp = false;
+        } else if (disp >= INT8_MIN && disp <= INT8_MAX &&
+                   !opnd_is_disp_force_full(opnd)) {
+            /* 8-bit disp */
+            di->mod = 1;
+            di->has_disp = true;
+            di->disp = disp;
+        } else {
+            /* 32/16-bit disp */
+            di->mod = 2;
+            di->has_disp = true;
+            di->disp = disp;
+        }
+        if (addr16) {
+            di->has_sib = false;
+            if (base == REG_BX && index == REG_SI)
+                di->rm = 0;
+            else if (base == REG_BX && index == REG_DI)
+                di->rm = 1;
+            else if (base == REG_BP && index == REG_SI)
+                di->rm = 2;
+            else if (base == REG_BP && index == REG_DI)
+                di->rm = 3;
+            else if (base == REG_SI && index == REG_NULL)
+                di->rm = 4;
+            else if (base == REG_DI && index == REG_NULL)
+                di->rm = 5;
+            else if (base == REG_BP && index == REG_NULL)
+                di->rm = 6;
+            else if (base == REG_BX && index == REG_NULL)
+                di->rm = 7;
+            else {
+                CLIENT_ASSERT(false, "encode error: invalid 16-bit base+index");
+                di->rm = 0;
+            }
+        } else if (index == REG_NULL && base != REG_ESP /* x64 w/ addr prefix => esp */
+                   IF_X64(&& base != REG_RSP && base != REG_R12 && base != REG_R12D)) {
+            /* don't need SIB byte */
+            di->has_sib = false;
+            encode_reg_ext_prefixes(di, base, PREFIX_REX_B);
+            di->rm = reg_get_bits(base);
+        } else {
+            /* need SIB byte */
+            di->has_sib = true;
+            di->rm = 4;
+            if (index == REG_NULL) {
+                di->index = 4;
+                di->scale = 0; /* does it matter?!? */
+            } else {
+                /* note that r13 can be an index register */
+                CLIENT_ASSERT(index != REG_ESP IF_X64(&& index != REG_RSP),
+                              "encode error: xsp cannot be an index register");
+                CLIENT_ASSERT(reg_is_32bit(index) || (X64_MODE(di) && reg_is_64bit(index)),
+                              "encode error: index must be general-purpose register");
+                encode_reg_ext_prefixes(di, index, PREFIX_REX_X);
+                if (X64_MODE(di) && reg_is_32bit(index))
+                    di->prefixes |= PREFIX_ADDR;
+                di->index = reg_get_bits(index);
+                switch (scale) {
+                case 1: di->scale = 0; break;
+                case 2: di->scale = 1; break;
+                case 4: di->scale = 2; break;
+                case 8: di->scale = 3; break;
+                default: CLIENT_ASSERT(false, "encode error: invalid scale");
+                }
+            }
+            if (base == REG_NULL) {
+                di->base = 5;
+                di->mod = 0;
+                di->has_disp = true;
+                di->disp = disp;
+            } else {
+                /* can't do nodisp(ebp) or nodisp(r13) */
+                CLIENT_ASSERT(di->mod != 0 ||
+                              (base != REG_EBP
+                               IF_X64(&& base != REG_RBP &&
+                                      base != REG_R13 && base != REG_R13D)),
+                              "encode error: xbp/r13 base must have disp");
+                encode_reg_ext_prefixes(di, base, PREFIX_REX_B);
+                if (X64_MODE(di) && reg_is_32bit(base)) {
+                    CLIENT_ASSERT(index == REG_NULL ||
+                                  (reg_is_32bit(index) && TEST(PREFIX_ADDR, di->prefixes)),
+                                  "encode error: index and base must be same width");
+                    di->prefixes |= PREFIX_ADDR;
+                }
+                di->base = reg_get_bits(base);
+            }
+        }
+    }
+}
+
+static void
+set_immed(decode_info_t *di, ptr_int_t val, opnd_size_t opsize)
+{
+    if (di->size_immed == OPSZ_NA) {
+        di->immed = val;
+        di->size_immed = opsize;
+    } else {
+        CLIENT_ASSERT(di->size_immed2 == OPSZ_NA,
+                      "encode error: >4-byte immed encoding error");
+        di->immed2 = val;
+        di->size_immed2 = opsize;
+    }
+}
+
+static void
+encode_operand(decode_info_t *di, int optype, opnd_size_t opsize, opnd_t opnd)
+{
+    switch (optype) {
+    case TYPE_NONE: 
+    case TYPE_REG:
+    case TYPE_VAR_REG:
+    case TYPE_VARZ_REG:
+    case TYPE_VAR_XREG:
+    case TYPE_VAR_ADDR_XREG:
+    case TYPE_1:
+    case TYPE_FLOATCONST:
+    case TYPE_INDIR_REG:
+    case TYPE_INDIR_VAR_XREG:
+    case TYPE_INDIR_VAR_REG:
+    case TYPE_INDIR_VAR_XIREG:
+        return;
+    case TYPE_REG_EX:
+    case TYPE_VAR_REG_EX:
+    case TYPE_VAR_XREG_EX:
+    case TYPE_VAR_REGX_EX:
+        encode_reg_ext_prefixes(di, opnd_get_reg(opnd), PREFIX_REX_B);
+        return;
+    case TYPE_FLOATMEM:
+    case TYPE_M:
+        CLIENT_ASSERT(opnd_is_memory_reference(opnd),
+                      "encode error: M operand must be mem ref");
+        /* fall through */
+    case TYPE_INDIR_E:
+    case TYPE_E:
+    case TYPE_Q:
+    case TYPE_W:
+    case TYPE_R: /* we already ensured this is a reg, not memory */
+    case TYPE_P_MODRM: /* we already ensured this is a reg, not memory */
+    case TYPE_V_MODRM: /* we already ensured this is a reg, not memory */
+        if (opnd_is_memory_reference(opnd)) {
+            if (opnd_is_far_memory_reference(opnd)) {
+                di->seg_override = opnd_get_segment(opnd);
+                /* should be just a SEG_ constant */
+                CLIENT_ASSERT(di->seg_override >= REG_START_SEGMENT &&
+                              di->seg_override <= REG_STOP_SEGMENT,
+                              "encode error: invalid segment override");
+            }
+            /* TODO PR 238624: support instr targets as pc-relative? */
+#ifdef X64
+            if (opnd_is_rel_addr(opnd))
+                encode_rel_addr(di, opnd);
+            else
+#endif
+                encode_base_disp(di, opnd);
+        } else {
+            CLIENT_ASSERT(opnd_is_reg(opnd),
+                          "encode error: modrm not selecting mem but not selecting reg");
+            if (di->mod < 5) {
+                /* already set (by a dst equal to src, probably) */
+                CLIENT_ASSERT(di->mod == 3 && di->rm == reg_get_bits(opnd_get_reg(opnd)),
+                              "encode error: modrm mismatch");
+                return;
+            }
+            di->mod = 3;
+            encode_reg_ext_prefixes(di, opnd_get_reg(opnd), PREFIX_REX_B);
+            di->rm = reg_get_bits(opnd_get_reg(opnd));
+        }
+        return;
+    case TYPE_G:
+    case TYPE_P:
+    case TYPE_V:
+    case TYPE_S:
+    case TYPE_C:
+    case TYPE_D:
+        {
+            CLIENT_ASSERT(opnd_is_reg(opnd), "encode error: operand must be a register");
+            if (di->reg < 8) {
+                /* already set (by a dst equal to src, probably) */
+                CLIENT_ASSERT(di->reg == reg_get_bits(opnd_get_reg(opnd)),
+                              "encode error: modrm mismatch");
+                return;
+            }
+            encode_reg_ext_prefixes(di, opnd_get_reg(opnd), PREFIX_REX_R);
+            di->reg = reg_get_bits(opnd_get_reg(opnd));
+            return;
+        }
+    case TYPE_I:
+        if (opnd_is_near_instr(opnd)) {
+            /* allow instr as immed, that means we want to put in the 4/8-byte
+             * pc of target instr as the immed
+             * This only works if the instr has no other immeds!
+             */
+            instr_t *target_instr = opnd_get_instr(opnd);
+            ptr_uint_t target = (ptr_uint_t)target_instr->note - di->immed;
+            /* We don't know the encode pc yet, so we put it in as pc-relative and
+             * fix it up later
+             */
+            set_immed(di, (ptr_uint_t)target, opsize);
+            /* this immed is pc-relative except it needs to have the
+             * instruction length subtracted from it -- we indicate that
+             * like this:
+             */
+            CLIENT_ASSERT(di->size_immed2 == OPSZ_NA,
+                          "encode error: immed size already set");
+            di->size_immed2 = opsize;
+            /* And now we ask to be adjusted to become an absolute pc: */
+            di->size_immed = OPSZ_28_short14; /* == immed needs +pc */
+        } else {
+            CLIENT_ASSERT(opnd_is_immed_int(opnd), "encode error: opnd not immed int");
+            set_immed(di, opnd_get_immed_int(opnd), opsize);
+        }
+        return;
+    case TYPE_J:
+        {
+            ptr_uint_t target;
+            /* Since we don't know pc values right now, we convert
+             * from an absolute pc to a relative offset in encode_immed.
+             * Here we simply set the immed to the absolute pc target.
+             */
+            if (opnd_is_near_instr(opnd)) {
+                /* assume the note fields have been set with relative offsets
+                 * from some start pc, and that our caller put our note in
+                 * di->immed
+                 */
+                instr_t *target_instr = opnd_get_instr(opnd);
+                target = (ptr_uint_t)target_instr->note - di->immed;
+                /* target is now a pc-relative target, so we can encode as is */
+                set_immed(di, target, opsize);
+                /* this immed is pc-relative except it needs to have the
+                 * instruction length subtracted from it -- we indicate that
+                 * like this:
+                 */
+                CLIENT_ASSERT(di->size_immed2 == OPSZ_NA,
+                              "encode error: immed size already set");
+                di->size_immed2 = opsize;
+                di->size_immed = OPSZ_10; /* == immed needs -length */
+            } else {
+                CLIENT_ASSERT(opnd_is_near_pc(opnd), "encode error: opnd not pc");
+                target = (ptr_uint_t) opnd_get_pc(opnd);
+                set_immed(di, target, opsize);
+                /* here's how we indicate that this immed needs to
+                 * be pc-relativized, we take advantage of the fact that all
+                 * TYPE_J do not have other immeds in the same instruction:
+                 */
+                CLIENT_ASSERT(di->size_immed2 == OPSZ_NA,
+                              "encode error: immed size already set");
+                di->size_immed2 = opsize;
+                di->size_immed = OPSZ_512; /* == immed needs relativizing */
+            }
+            return;
+        }
+    case TYPE_A: 
+        {
+            ptr_int_t offs;
+
+#ifdef IA32_ON_IA64
+            ptr_uint_t target;
+            if (opsize == OPSZ_4_short2) {
+                if (opnd_is_near_instr(opnd)) {
+                    /* assume the note fields have been set with relative offsets
+                     * from some start pc, and that our caller put our note in
+                     * di->immed
+                     */
+                    instr_t *target_instr = opnd_get_instr(opnd);
+                    target = (ptr_uint_t)target_instr->note;
+                    /* target is absolute address of instr ready to go */
+                    set_immed(di, target, opsize);
+                } else {
+                    CLIENT_ASSERT(opnd_is_near_pc(opnd), "encode error: opnd not pc");
+                    target = (ptr_uint_t) opnd_get_pc(opnd);
+                    set_immed(di, target, opsize);
+                }
+                return;
+            }
+#endif
+            
+            CLIENT_ASSERT(!X64_MODE(di), "x64 has no type A instructions");
+            CLIENT_ASSERT(opsize == OPSZ_6_irex10_short4 || opsize == OPSZ_6 ||
+                          opsize == OPSZ_4 ||
+                          (opsize == OPSZ_10 && proc_get_vendor() != VENDOR_AMD),
+                          "encode error: A operand size mismatch");
+            if (opnd_is_far_instr(opnd)) {
+                /* assume the note fields have been set with relative offsets
+                 * from some start pc, and that our caller put our note in
+                 * di->immed
+                 */
+                instr_t *target_instr = opnd_get_instr(opnd);
+                ptr_uint_t target = (ptr_uint_t) target_instr->note;
+                /* target is absolute address of instr ready to go */
+                set_immed(di, target, opsize);
+            } else {
+                CLIENT_ASSERT(opnd_is_far_pc(opnd),
+                              "encode error: A operand must be far pc or far instr");
+                /* FIXME PR 225937: allow client to specify whether data16 or not
+                 * instead of auto-adding the prefix if offset is small
+                 */
+                offs = (ptr_int_t) opnd_get_pc(opnd);
+                if (offs >= INT16_MIN && offs <= INT16_MAX) {
+                    int val;
+                    di->prefixes |= PREFIX_DATA;
+                    val = (opnd_get_segment_selector(opnd)<<16) | ((short) offs);
+                    set_immed(di, val, OPSZ_4);
+                } else {
+                    CLIENT_ASSERT(di->size_immed == OPSZ_NA &&
+                                  di->size_immed2 == OPSZ_NA,
+                                  "encode error: A operand size mismatch");
+                    di->immed = opnd_get_segment_selector(opnd);
+                    di->immed2 = offs;
+                    di->size_immed = OPSZ_6;
+                    di->size_immed2 = OPSZ_6;
+                }
+            }
+            return;
+        }
+    case TYPE_O:
+        {
+            ptr_int_t addr;
+            CLIENT_ASSERT(opnd_is_abs_addr(opnd),
+                          "encode error: O operand must be absolute mem ref");
+            addr = (ptr_int_t) opnd_get_addr(opnd);
+            if (opnd_is_far_abs_addr(opnd)) {
+                di->seg_override = opnd_get_segment(opnd);
+                /* should be just a SEG_ constant */
+                CLIENT_ASSERT(di->seg_override >= REG_START_SEGMENT &&
+                              di->seg_override <= REG_STOP_SEGMENT,
+                              "encode error: invalid segment override");
+                if ((!X64_MODE(di) && addr >= INT16_MIN && addr <= INT16_MAX) ||
+                    (X64_MODE(di) && addr >= INT32_MIN && addr <= INT32_MAX)) {
+                    /* same optimization as in encode_base_disp -- see comments there */
+                    if (use_addr_prefix_on_short_disp()) {
+                        di->prefixes |= PREFIX_ADDR;
+                    }
+                }
+            }
+            set_immed(di, addr, resolve_addr_size(di));
+            return;
+        }
+
+    /* assume that opnd_type_ok has already been called --
+     * nothing to do unless has an override, these are implicit operands */
+    case TYPE_X: /* this means the memory address DS:(RE)(E)SI */
+    case TYPE_XLAT: /* this means the memory address DS:(RE)(E)BX+AL */
+    case TYPE_MASKMOVQ: /* this means the memory address DS:(RE)(E)DI */
+        if (opnd_get_segment(opnd) != SEG_DS)
+            di->seg_override = opnd_get_segment(opnd);
+        return;
+    case TYPE_Y: /* this means the memory address ES:(RE)(E)DI */
+        return; /* no override possible */
+
+    default:
+        CLIENT_ASSERT(false, "encode error: unknown operand type");
+    }
+}
+
+/* special-case (==fast) encoder for cti instructions 
+ * this routine cannot handle indirect branches or rets or far jmp/call;
+ * it can handle loop/jecxz but it does NOT check for data16!
+ */
+static byte *
+encode_cti(instr_t *instr, byte *start_pc, bool check_reachable
+           _IF_DEBUG(bool assert_reachable))
+{
+    byte *pc = start_pc;
+    const instr_info_t * info = instr_get_instr_info(instr);
+    opnd_t opnd;
+    ptr_uint_t target;
+
+    if (instr->prefixes != 0) {
+        if (TEST(PREFIX_JCC_TAKEN, instr->prefixes)) {
+            *pc = RAW_PREFIX_jcc_taken;
+            pc++;
+        } else if (TEST(PREFIX_JCC_NOT_TAKEN, instr->prefixes)) {
+            *pc = RAW_PREFIX_jcc_not_taken;
+            pc++;
+        }
+        /* assumption: no 16-bit targets */
+        CLIENT_ASSERT(!TESTANY(~(PREFIX_JCC_TAKEN|PREFIX_JCC_NOT_TAKEN),
+                               instr->prefixes),
+                      "encode cti error: non-branch-hint prefixes not supported");
+    }
+
+    /* output opcode */
+    /* first opcode byte */
+    *pc = (byte)((info->opcode & 0x00ff0000) >> 16); 
+    pc++;
+    /* second opcode byte, if there is one */
+    if (TEST(OPCODE_TWOBYTES, info->opcode)) {
+        *pc = (byte)((info->opcode & 0x0000ff00) >> 8); 
+        pc++;
+    }
+    ASSERT(!TEST(OPCODE_THREEBYTES, info->opcode)); /* no cti has 3 opcode bytes */
+
+    /* we assume only one operand: 1st src == jump target, but we do
+     * not check that, for speed
+     */
+    opnd = instr_get_target(instr);
+    if (opnd_is_near_pc(opnd)) {
+        target = (ptr_uint_t) opnd_get_pc(opnd);
+    } else if (opnd_is_near_instr(opnd)) {
+        instr_t *in = opnd_get_instr(opnd);
+        target = (ptr_uint_t)start_pc + ((ptr_uint_t)in->note - (ptr_uint_t)instr->note);
+    } else {
+        target = 0; /* avoid compiler warning */
+        CLIENT_ASSERT(false, "encode_cti error: opnd must be near pc or near instr");
+    }
+
+    if (instr_is_cti_short(instr)) {
+        /* 8-bit offset */
+        ptr_int_t offset;
+        /* handled w/ mangled bytes */
+        CLIENT_ASSERT(!instr_is_cti_short_rewrite(instr, NULL),
+                      "encode_cti error: jecxz/loop already mangled");
+        /* offset is from start of next instr */
+        offset = target - ((ptr_int_t)pc + 1);
+        if (check_reachable && !(offset >= INT8_MIN && offset <= INT8_MAX)) {
+            CLIENT_ASSERT(!assert_reachable,
+                          "encode_cti error: target beyond 8-bit reach");
+            return NULL;
+        }
+        *((char *)pc) = (char) offset;
+        pc++;
+    } else {
+        /* 32-bit offset */
+        /* offset is from start of next instr */
+        ptr_int_t offset = target - ((ptr_int_t)pc + 4);
+#ifdef X64
+        if (check_reachable && !REL32_REACHABLE_OFFS(offset)) {
+            CLIENT_ASSERT(!assert_reachable,
+                          "encode_cti error: target beyond 32-bit reach");
+            return NULL;
+        }
+#endif
+        *((int *)pc) = (int) offset;
+        pc += 4;
+    }
+    return pc;
+}
+
+/* PR 251479: support general re-relativization.
+ * Takes in a level 0-3 instruction and encodes it by copying its
+ * raw bytes to dst_pc. For x64, if it is marked as having a rip-relative 
+ * displacement, that displacement is re-relativized to reach
+ * its current target from the encoded location.
+ * Returns NULL on failure to encode (due to reachability).
+ */
+byte *
+copy_and_re_relativize_raw_instr(dcontext_t *dcontext, instr_t *instr, byte *dst_pc)
+{
+    byte *orig_dst_pc = dst_pc;
+    ASSERT(instr_raw_bits_valid(instr));
+    /* FIXME PR 253447: if want to support ctis as well, need
+     * instr->rip_rel_disp_sz and need to set both for non-x64 as well
+     * in decode_sizeof(): or only in decode_cti()?
+     */
+    /* For PR 251646 we have special support for mangled jecxz/loop* */
+    if (instr_is_cti_short_rewrite(instr, NULL)) {
+        app_pc target;
+        CLIENT_ASSERT(opnd_is_pc(instr_get_target(instr)),
+                      "cti_short_rewrite: must have pc target");
+        target = opnd_get_pc(instr_get_target(instr));
+        memcpy(dst_pc, instr->bytes, instr->length - 4);
+        dst_pc += instr->length - 4;
+        if (!REL32_REACHABLE(dst_pc + 4, target)) {
+            CLIENT_ASSERT(false, "mangled jecxz/loop*: target out of 32-bit reach");
+            return NULL;
+        }
+        *((int *)dst_pc) = (int) (target - (dst_pc + 4));
+    }
+#ifdef X64
+    /* We test the flag directly to support cases where the raw bits are
+     * being set by private_instr_encode() */
+    else if (instr_rip_rel_valid(instr) && instr_get_rip_rel_pos(instr) > 0) {
+        /* x64 4-byte rip-relative data address displacement */
+        byte *target;
+        ptr_int_t new_offs;
+        bool addr32 = false;
+        uint rip_rel_pos = instr_get_rip_rel_pos(instr); /* disp offs within instr */
+        DEBUG_DECLARE(bool ok;)
+        ASSERT(!instr_is_level_0(instr));
+        DEBUG_DECLARE(ok =) instr_get_rel_addr_target(instr, &target);
+        ASSERT(ok);
+        new_offs = target - (dst_pc + instr->length);
+        /* PR 253327: we don't record whether addr32 so we have to deduce it now */
+        if ((ptr_uint_t)target <= INT_MAX) {
+            int num_prefixes;
+            int i;
+            IF_X64(bool old_mode = set_x86_mode(dcontext, instr_get_x86_mode(instr));)
+            decode_sizeof(dcontext, instr->bytes, &num_prefixes, NULL);
+            IF_X64(set_x86_mode(dcontext, old_mode));
+            for (i = 0; i < num_prefixes; i++) {
+                if (*(instr->bytes+i) == ADDR_PREFIX_OPCODE) {
+                    addr32 = true;
+                    break;
+                }
+            }
+        }
+        if (!addr32 && !REL32_REACHABLE_OFFS(new_offs)) {
+            /* unreachable: not clear whether routing through register here
+             * is worth the complexities of the length changing, so for now
+             * we fail and rely on caller to do a conservative estimate
+             * of reachability and transform this instruction before encoding.
+             */
+            CLIENT_ASSERT(false, "encoding failed re-relativizing rip-relative "
+                          "address whose target is unreachable");
+            return NULL;
+        }
+        memcpy(dst_pc, instr->bytes, rip_rel_pos);
+        dst_pc += rip_rel_pos;
+        *((int *)dst_pc) = (int) new_offs;
+        if (rip_rel_pos + 4U < instr->length) {
+            /* suffix byte */
+            memcpy(dst_pc + 4, instr->bytes + rip_rel_pos + 4,
+                   instr->length - (rip_rel_pos + 4));
+        }
+    } else
+#endif
+        memcpy(dst_pc, instr->bytes, instr->length);
+    return orig_dst_pc + instr->length;
+}
+
+/* Encodes instrustion instr.  The parameter pc points
+ * to the address of this instruction in the fragment cache.
+ * Checks for and fixes pc-relative instructions.  
+ * N.B: if instr is a jump with an instr_t target, the caller MUST set the note
+ * field in the target instr_t prior to calling instr_encode on the jump instr.
+ * 
+ * Returns the pc after the encoded instr, or NULL if the instruction cannot be encoded.  
+ * Note that if instr_is_label(instr) will encoded as a 0-byte instruction.
+ * If a pc-relative operand cannot reach its target:
+ *   If reachable == NULL, we assert and encoding fails (returning NULL);
+ *   Else, encoding continues, and *reachable is set to false.
+ * Else, if reachable != NULL, *reachable is set to true.
+ */
+static byte *
+instr_encode_common(dcontext_t *dcontext, instr_t *instr, byte *pc,
+                    bool check_reachable _IF_DEBUG(bool assert_reachable))
+{
+    const instr_info_t * info;
+    decode_info_t di;
+
+    /* pointer to and into the instruction binary */
+    byte *cache_pc = pc;
+    byte *field_ptr = cache_pc;
+    byte *disp_relativize_at = NULL;
+    uint opc;
+
+    /* first handle the already-encoded instructions */
+    if (instr_raw_bits_valid(instr)) {
+        CLIENT_ASSERT(check_reachable, "internal encode error: cannot encode raw "
+                      "bits and ignore reachability");
+        /* copy raw bits, possibly re-relativizing */
+        return copy_and_re_relativize_raw_instr(dcontext, instr, cache_pc);
+    }
+    CLIENT_ASSERT(instr_operands_valid(instr), "instr_encode error: operands invalid");
+    opc = instr_get_opcode(instr);
+    if ((instr_is_cbr(instr) &&
+         (!instr_is_cti_loop(instr) ||
+          /* no data16 */
+          reg_is_32bit(opnd_get_reg(instr_get_src(instr, 1))))) ||
+        /* no indirect or far */
+        opc == OP_jmp_short || opc == OP_jmp || opc == OP_call) {
+        if (!TESTANY(~(PREFIX_JCC_TAKEN|PREFIX_JCC_NOT_TAKEN), instr->prefixes)) {
+            /* encode_cti cannot handle funny prefixes or indirect branches or rets */
+            return encode_cti(instr, pc, check_reachable _IF_DEBUG(assert_reachable));
+        }
+    } 
+
+    /* else really encode */
+    info = instr_get_instr_info(instr);
+    if (info == NULL) {
+        CLIENT_ASSERT(instr_is_label(instr), "instr_encode: invalid instr");
+        return (instr_is_label(instr) ? pc : NULL);
+    }
+
+    /* first, walk through instr list to find format that matches
+     * this instr's operands
+     */
+    DOLOG(ENC_LEVEL, LOG_EMIT, { loginst(dcontext, 1, instr, "\n--- encoding"); });
+
+    memset(&di, 0, sizeof(decode_info_t));
+    IF_X64(di.x86_mode = instr_get_x86_mode(instr));
+    /* while only PREFIX_SIGNIFICANT should be set by the user, internally
+     * we set di.prefixes to communicate size prefixes between
+     * opnd_type_ok() and here, first clearing out the size specifiers
+     * in encoding_possible().
+     */
+    di.prefixes = instr->prefixes;
+
+    while (!encoding_possible(&di, instr, info)) {
+        LOG(THREAD, LOG_EMIT, ENC_LEVEL, "\tencoding for 0x%x no good...\n", info->opcode);
+        info = get_next_instr_info(info);
+        /* stop when hit end of list or when hit extra operand tables (OP_CONTD) */
+        if (info == NULL || info->opcode == OP_CONTD) {
+#ifdef DEBUG
+            LOG(THREAD, LOG_EMIT, 1, "ERROR: Could not find encoding for: ");
+            instr_disassemble(dcontext, instr, THREAD);
+            LOG(THREAD, LOG_EMIT, 1, "\n");
+#endif
+            CLIENT_ASSERT(false, "instr_encode error: no encoding found");
+            /* FIXME: since labels (case 4468) have a legal length 0
+             * we may want to return a separate status code for failure.
+             */
+            return NULL;
+        }
+    }
+
+    /* fill out the other fields of di */
+    di.start_pc = cache_pc; /* only needed for PR 253327 addr32 rip-relative */
+
+    di.size_immed = OPSZ_NA;
+    di.size_immed2 = OPSZ_NA;
+    /* these (illegal) values indicate uninitialization */
+    di.reg = 8;
+    di.mod = 5;
+
+    /* prefixes */
+    di.seg_override = REG_NULL; /* operands will fill in */
+
+    /* operands 
+     * we can ignore extra operands here, since all extra operands
+     * are hardcoded
+     */
+    if (info->dst1_type != TYPE_NONE)
+        encode_operand(&di, info->dst1_type, info->dst1_size, instr_get_dst(instr, 0));
+    if (info->dst2_type != TYPE_NONE)
+        encode_operand(&di, info->dst2_type, info->dst2_size, instr_get_dst(instr, 1));
+    if (info->src1_type != TYPE_NONE) {
+        if (opnd_is_instr(instr_get_src(instr, 0))) {
+            /* special case: target is instr
+             * need to pass this instr's note in
+             */
+            di.immed = (ptr_int_t) instr->note;
+        }
+        encode_operand(&di, info->src1_type, info->src1_size, instr_get_src(instr, 0));
+    }
+    if (info->src2_type != TYPE_NONE)
+        encode_operand(&di, info->src2_type, info->src2_size, instr_get_src(instr, 1));
+    if (info->src3_type != TYPE_NONE)
+        encode_operand(&di, info->src3_type, info->src3_size, instr_get_src(instr, 2));
+    if (di.mod == 5 && di.reg < 8) { /* mod may never be set (e.g., OP_extrq) */
+        /* follow lead of below where we set to all 1's */
+        di.mod = 3;
+        CLIENT_ASSERT(di.rm == 0, "internal error: mod not set but rm was");
+        di.rm = 7;
+    }
+
+    /* finally, do the actual bit writing */
+
+    /* output the prefix byte(s) */
+    if (di.prefixes != 0) {
+        if (TEST(PREFIX_LOCK, di.prefixes)) {
+            *field_ptr = RAW_PREFIX_lock;
+            field_ptr++;
+        }
+        if (TEST(PREFIX_JCC_TAKEN, di.prefixes)) {
+            *field_ptr = RAW_PREFIX_jcc_taken;
+            field_ptr++;
+        } else if (TEST(PREFIX_JCC_NOT_TAKEN, di.prefixes)) {
+            *field_ptr = RAW_PREFIX_jcc_not_taken;
+            field_ptr++;
+        }
+    }
+    if (TEST(PREFIX_DATA, di.prefixes)) {
+        *field_ptr = DATA_PREFIX_OPCODE;
+        field_ptr++;
+    }
+    /* N.B.: we assume the order of 0x67 <seg> in coarse_is_indirect_stub()
+     * and instr_is_tls_xcx_spill()
+     */
+    if (TEST(PREFIX_ADDR, di.prefixes)) {
+        *field_ptr = ADDR_PREFIX_OPCODE;
+        field_ptr++;
+    }
+    if (di.seg_override != REG_NULL) {
+        switch (di.seg_override) {
+        case SEG_ES: *field_ptr = 0x26; break;
+        case SEG_CS: *field_ptr = 0x2e; break;
+        case SEG_SS: *field_ptr = 0x36; break;
+        case SEG_DS: *field_ptr = 0x3e; break;
+        case SEG_FS: *field_ptr = 0x64; break;
+        case SEG_GS: *field_ptr = 0x65; break;
+        default: CLIENT_ASSERT(false, "instr_encode error: unknown segment prefix");
+        }
+        field_ptr++;
+    }
+
+    /* output the opcode required prefix byte (if needed) */
+    if (info->opcode > 0xffffff &&
+        /* if OPCODE_{MODRM,SUFFIX} there can be no prefix-opcode byte */
+        !TESTANY(OPCODE_MODRM|OPCODE_SUFFIX, info->opcode)) {
+        /* prefix byte is part of opcode */
+        *field_ptr = (byte)(info->opcode >> 24); 
+        field_ptr++;
+    }
+
+    /* NOTE - the rex prefix must be the last prefix (even if the other prefix is
+     * part of the opcode). Xref PR 271878. */
+    if (TESTANY(PREFIX_REX_ALL, di.prefixes)) {
+        byte rexval = REX_PREFIX_BASE_OPCODE;
+        if (TEST(PREFIX_REX_W, di.prefixes))
+            rexval |= REX_PREFIX_W_OPFLAG;
+        if (TEST(PREFIX_REX_R, di.prefixes))
+            rexval |= REX_PREFIX_R_OPFLAG;
+        if (TEST(PREFIX_REX_X, di.prefixes))
+            rexval |= REX_PREFIX_X_OPFLAG;
+        if (TEST(PREFIX_REX_B, di.prefixes))
+            rexval |= REX_PREFIX_B_OPFLAG;
+        *field_ptr = rexval;
+        field_ptr++;
+    }
+
+    /* output the opcode byte(s) (opcode required prefixes are handled above) */
+    if (TEST(OPCODE_THREEBYTES, info->opcode)) {
+        /* implied initial opcode byte */
+        *field_ptr = 0x0f;
+        field_ptr++;
+    }
+    /* first opcode byte */
+    *field_ptr = (byte)((info->opcode & 0x00ff0000) >> 16); 
+    field_ptr++;
+    /* second opcode byte, if there is one */
+    if (TEST(OPCODE_TWOBYTES, info->opcode)) {
+        *field_ptr = (byte)((info->opcode & 0x0000ff00) >> 8); 
+        field_ptr++;
+    }
+    /* /n: part of opcode is in reg of modrm byte */
+    if (TEST(OPCODE_REG, info->opcode)) {
+        CLIENT_ASSERT(di.reg == 8,
+                      "instr_encode error: /n opcode inconsistency"); /* unset */
+        di.reg = (byte) (info->opcode & 0x00000007);
+        if (di.mod == 5) {
+            /* modrm only used for opcode
+             * mod and rm are arbitrary: seem to be set to all 1's by compilers
+             */
+            di.mod = 3;
+            di.rm = 7;
+        }
+    }
+    /* opcode depends on entire modrm byte */
+    if (TEST(OPCODE_MODRM, info->opcode)) {
+        /* modrm is encoded in prefix byte */
+        *field_ptr = (byte)(info->opcode >> 24); 
+        field_ptr++;
+        di.mod = 5; /* prevent modrm output from opnds below */
+    }
+
+    /* output modrm byte(s) */
+    if (di.mod != 5) {
+        byte modrm;
+        if (di.reg == 8) /* if never set, set to 0 */
+            di.reg = 0;
+        CLIENT_ASSERT(di.mod <= 0x3 && di.reg <= 0x7 && di.rm <= 0x7,
+                      "encode error: invalid modrm");
+        modrm = MODRM_BYTE(di.mod, di.reg, di.rm);
+        *field_ptr = modrm;
+        field_ptr++;
+        if (di.has_sib) {
+            byte sib = (byte) ((di.scale << 6) | (di.index << 3) | (di.base));
+            CLIENT_ASSERT(di.scale <= 0x3 && di.index <= 0x7 && di.base <= 0x7,
+                          "encode error: invalid scale/index/base");
+            *field_ptr = sib;
+            field_ptr++;
+        }
+        if (di.has_disp) {
+            if (di.mod == 1) {
+                *field_ptr = (byte) di.disp;
+                field_ptr++;
+            } else if (!X64_MODE(&di) && TEST(PREFIX_ADDR, di.prefixes)) {
+                CLIENT_ASSERT_TRUNCATE(*((short *)field_ptr), ushort, di.disp,
+                                       "encode error: modrm disp too large for 16-bit");
+                *((short *)field_ptr) = (ushort) di.disp;
+                field_ptr+=2;
+            } else {
+                if (X64_MODE(&di) && di.mod == 0 && di.rm == 5) {
+                    /* pc-relative, but we don't know size of immeds yet */
+                    disp_relativize_at = field_ptr;
+                } else {
+                    *((int *)field_ptr) = di.disp;
+                }
+                field_ptr+=4;
+            }
+        }
+    }
+
+    /* output immed byte(s) */
+    /* HACK: to tell an instr target of a control transfer instruction
+     * our length, store into di.modrm the bytes so far
+     */
+    CLIENT_ASSERT_TRUNCATE(di.modrm, byte, (field_ptr - cache_pc),
+                           "encode error: instr too long");
+    di.modrm = (byte) (field_ptr - cache_pc);
+    if (di.size_immed != OPSZ_NA)
+        field_ptr = encode_immed(&di, field_ptr);
+    if (di.size_immed2 != OPSZ_NA)
+        field_ptr = encode_immed(&di, field_ptr);
+
+    /* suffix opcode */
+    if (TEST(OPCODE_SUFFIX, info->opcode)) {
+        /* none of these have immeds, currently (and presumably never will have) */
+        ASSERT_CURIOSITY(di.size_immed == OPSZ_NA && di.size_immed2 == OPSZ_NA);
+        /* modrm is encoded in prefix byte */
+        *field_ptr = (byte)(info->opcode >> 24); 
+        field_ptr++;
+    }
+
+    if (disp_relativize_at != NULL) {
+        CLIENT_ASSERT(X64_MODE(&di), "encode error: no rip-relative in x86 mode!");
+        if (check_reachable &&
+            !CHECK_TRUNCATE_TYPE_int(di.disp_abs - field_ptr) &&
+            /* PR 253327: we auto-add addr prefix for out-of-reach low tgt */
+            (!TEST(PREFIX_ADDR, di.prefixes) || (ptr_uint_t)di.disp_abs > INT_MAX)) {
+            CLIENT_ASSERT(!assert_reachable,
+                          "encode error: rip-relative reference out of 32-bit reach");
+            return NULL;
+        }
+        *((int *)disp_relativize_at) = (int) (di.disp_abs - field_ptr);
+        /* in case caller is caching these bits (in particular,
+         * private_instr_encode()), set rip_rel_pos */
+        CLIENT_ASSERT(CHECK_TRUNCATE_TYPE_byte(disp_relativize_at - di.start_pc),
+                      "internal encode error: rip-relative instr pos too large");
+        IF_X64(instr_set_rip_rel_pos(instr, (byte)(disp_relativize_at - di.start_pc)));
+    }
+
+#if DEBUG_DISABLE /* turn back on if want to debug */
+    if (stats->loglevel >= 3) {
+        byte *pc = cache_pc;
+        LOG(THREAD, LOG_EMIT, 3, "instr_encode on: ");
+        instr_disassemble(dcontext, instr, THREAD);
+        LOG(THREAD, LOG_EMIT, 3, " : ");
+        do {
+            LOG(THREAD, LOG_EMIT, 3, " %02x", *pc);
+            pc++;
+        } while (pc < field_ptr);
+        LOG(THREAD, LOG_EMIT, 3, "\n");
+    }
+    if (0 && instr_raw_bits_valid(instr)) {
+        /* encoding in original bits */
+        int i;
+        int len = (field_ptr - cache_pc);
+        byte ours, orig;
+        for (i=0; i<len; i++) {
+            ours = *(cache_pc+i);
+            orig = *(((byte *)instr_get_raw_bits(instr))+i);
+            if (ours != orig) {
+                /* special cases: difference is ok, not an error: */
+                if (i==0 && ours==0x66 && (orig==0xf2||orig==0xf3) &&
+                    /* allow rep/repne & data prefixes to have order swapped */
+                    *(cache_pc+i+1)==orig &&
+                    *(((byte *)instr_get_raw_bits(instr))+i+1)==ours) {
+                    i++; /* skip next */
+                } else if (i==0 && 
+                           ((ours==0x3a && orig==0x38) ||
+                            (ours==0x3b && orig==0x39) ||
+                            (ours==0x03 && orig==0x01) ||
+                            (ours==0x33 && orig==0x31))) {
+                    /* Gv, Ev vs Ev, Gv equivalent when both opnds regs */
+                    i++; /* skip next */
+                } else if ((i==1 || i==2) && 
+                           ((ours | 0x3c)==0x3c) && 
+                           ((orig | 0x7c)==0x7c) && 
+                           *(((byte *)instr_get_raw_bits(instr))+i+2)==0x0) {
+                    /* mod=0, rm=4, no disp same as mod=1, rm=4, disp=0 */
+                    /* I see these for (%esp) */
+                    i=len; /* skip rest */
+                } else if (i==0 && ours==0xd8 && orig==0xdc &&
+                    *(cache_pc+i+1)==0xc0 &&
+                    *(((byte *)instr_get_raw_bits(instr))+i+1)==0xc0) {
+                    /* fadd st0, st0 => st0: two ways to do it */
+                    i++; /* skip rest */
+                } else {
+                    LOG(THREAD, LOG_EMIT, 3, 
+                        "Error: encoding does not match original bits!\n");
+                    for (i=0; i<len; i++) {
+                        ours = *(cache_pc+i);
+                        LOG(THREAD, LOG_EMIT, 3, " %02x", ours);
+                    }
+                    LOG(THREAD, LOG_EMIT, 3, " vs. ");
+                    for (i=0; i<len; i++) {
+                        orig = *(((byte *)instr_get_raw_bits(instr))+i);
+                        LOG(THREAD, LOG_EMIT, 3, " %02x", orig);
+                    }
+                    LOG(THREAD, LOG_EMIT, 3, "\n");
+                    CLIENT_ASSERT(false, "instr_encode error: encode != raw bits");
+                }
+            }
+        }
+    }
+#endif
+
+    return field_ptr;
+}
+
+/* completely ignores reachability failures */
+byte *
+instr_encode_ignore_reachability(dcontext_t *dcontext, instr_t *instr, byte *pc)
+{
+    return instr_encode_common(dcontext, instr, pc, false _IF_DEBUG(false));
+}
+
+/* just like instr_encode but doesn't assert on reachability failures */
+byte *
+instr_encode_check_reachability(dcontext_t *dcontext, instr_t *instr, byte *pc)
+{
+    return instr_encode_common(dcontext, instr, pc, true _IF_DEBUG(false));
+}
+
+byte *
+instr_encode(dcontext_t *dcontext, instr_t *instr, byte *pc)
+{
+    return instr_encode_common(dcontext, instr, pc, true _IF_DEBUG(true));
+}
+
+/* If has_instr_jmp_targets is true, this routine trashes the note field
+ * of each instr_t to store the offset in order to properly encode
+ * the relative pc for an instr_t jump target
+ */
+byte *
+instrlist_encode(dcontext_t *dcontext, instrlist_t *ilist, byte *pc,
+                 bool has_instr_jmp_targets)
+{
+    instr_t *inst;
+    int len = 0;
+    if (has_instr_jmp_targets) {
+        /* must set note fields first with offset */
+        for (inst = instrlist_first(ilist); inst; inst = instr_get_next(inst)) {
+            instr_set_note(inst, (void *)(ptr_int_t)len);
+            len += instr_length(dcontext, inst);
+        }
+    }
+    for (inst = instrlist_first(ilist); inst; inst = instr_get_next(inst)) {
+        pc = instr_encode(dcontext, inst, pc);
+        if (pc == NULL)
+            return NULL;
+    }
+    return pc;
+}
+
diff --git a/core/x86/instr.c b/core/x86/instr.c
new file mode 100644
index 000000000..670c35abd
--- /dev/null
+++ b/core/x86/instr.c
@@ -0,0 +1,5163 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/* file "instr.c" -- x86-specific IR utilities
+ */
+
+#include "globals.h"
+#include "instr.h"
+#include "arch.h"
+#include "link.h"
+#include "decode.h"
+#include "decode_fast.h"
+#include "instr_create.h"
+
+#include <string.h> /* for memcpy */
+
+#ifdef DEBUG
+# include "disassemble.h"
+#endif
+
+#ifdef DEBUG
+/* case 10450: give messages to clients */
+# undef ASSERT /* N.B.: if have issues w/ DYNAMO_OPTION, re-instate */
+# undef ASSERT_TRUNCATE
+# undef ASSERT_BITFIELD_TRUNCATE
+# undef ASSERT_NOT_REACHED
+# define ASSERT DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_TRUNCATE DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_BITFIELD_TRUNCATE DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_NOT_REACHED DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+#endif
+
+/*************************
+ ***       opnd_t        ***
+ *************************/
+
+/* predicates */
+bool opnd_is_valid(opnd_t opnd)
+{
+    return opnd.kind < LAST_kind;
+}
+bool opnd_is_null(opnd_t opnd) { return opnd.kind == NULL_kind; }
+bool opnd_is_reg(opnd_t opnd) { return opnd.kind == REG_kind; }
+bool opnd_is_immed(opnd_t opnd) { return opnd.kind == IMMED_INTEGER_kind ||
+                               opnd.kind == IMMED_FLOAT_kind; }
+bool opnd_is_immed_int(opnd_t opnd) { return opnd.kind == IMMED_INTEGER_kind; }
+bool opnd_is_immed_float(opnd_t opnd) { return opnd.kind == IMMED_FLOAT_kind; }
+bool opnd_is_pc(opnd_t opnd) {
+    return opnd.kind == PC_kind || opnd.kind == FAR_PC_kind; 
+}
+bool opnd_is_near_pc(opnd_t opnd) { return opnd.kind == PC_kind; }
+bool opnd_is_far_pc(opnd_t opnd) { return opnd.kind == FAR_PC_kind; }
+bool opnd_is_instr(opnd_t opnd) {
+    return opnd.kind == INSTR_kind || opnd.kind == FAR_INSTR_kind;
+}
+bool opnd_is_near_instr(opnd_t opnd) { return opnd.kind == INSTR_kind; }
+bool opnd_is_far_instr(opnd_t opnd) { return opnd.kind == FAR_INSTR_kind; }
+bool opnd_is_base_disp(opnd_t opnd) { return opnd.kind == BASE_DISP_kind; }
+bool opnd_is_near_base_disp(opnd_t opnd) {
+    return opnd.kind == BASE_DISP_kind && opnd.seg.segment == REG_NULL; 
+}
+bool opnd_is_far_base_disp(opnd_t opnd) {
+    return opnd.kind == BASE_DISP_kind && opnd.seg.segment != REG_NULL;
+}
+
+#ifdef X64
+bool opnd_is_rel_addr(opnd_t opnd) { return opnd.kind == REL_ADDR_kind; }
+bool opnd_is_near_rel_addr(opnd_t opnd) {
+    return opnd.kind == REL_ADDR_kind && opnd.seg.segment == REG_NULL; 
+}
+bool opnd_is_far_rel_addr(opnd_t opnd) {
+    return opnd.kind == REL_ADDR_kind && opnd.seg.segment != REG_NULL; 
+}
+#endif
+
+/* We allow overlap between ABS_ADDR_kind and BASE_DISP_kind w/ no base or index */
+static bool
+opnd_is_abs_base_disp(opnd_t opnd) {
+    return (opnd_is_base_disp(opnd) && opnd_get_base(opnd) == REG_NULL &&
+            opnd_get_index(opnd) == REG_NULL);
+}
+bool opnd_is_abs_addr(opnd_t opnd) {
+    return IF_X64(opnd.kind == ABS_ADDR_kind ||) opnd_is_abs_base_disp(opnd);
+}
+bool opnd_is_near_abs_addr(opnd_t opnd) {
+    return opnd_is_abs_addr(opnd) && opnd.seg.segment == REG_NULL; 
+}
+bool opnd_is_far_abs_addr(opnd_t opnd) {
+    return opnd_is_abs_addr(opnd) && opnd.seg.segment != REG_NULL; 
+}
+
+bool
+opnd_is_reg_32bit(opnd_t opnd)
+{
+    if (opnd_is_reg(opnd))
+        return reg_is_32bit(opnd_get_reg(opnd));
+    return false;
+}
+
+bool
+reg_is_32bit(reg_id_t reg)
+{
+    return (reg >= REG_START_32 && reg <= REG_STOP_32);
+}
+
+bool
+opnd_is_reg_64bit(opnd_t opnd)
+{
+    if (opnd_is_reg(opnd))
+        return reg_is_64bit(opnd_get_reg(opnd));
+    return false;
+}
+
+bool
+reg_is_64bit(reg_id_t reg)
+{
+    return (reg >= REG_START_64 && reg <= REG_STOP_64);
+}
+
+bool
+opnd_is_reg_pointer_sized(opnd_t opnd)
+{
+    if (opnd_is_reg(opnd))
+        return reg_is_pointer_sized(opnd_get_reg(opnd));
+    return false;
+}
+
+bool
+reg_is_pointer_sized(reg_id_t reg)
+{
+#ifdef X64
+    return (reg >= REG_START_64 && reg <= REG_STOP_64);
+#else
+    return (reg >= REG_START_32 && reg <= REG_STOP_32);
+#endif
+}
+
+/* null operands */
+
+opnd_t
+opnd_create_null(void)
+{
+    opnd_t opnd;
+    opnd.kind = NULL_kind;
+    return opnd;
+}
+
+/* register operands */
+
+opnd_t
+opnd_create_reg(reg_id_t r)
+{
+    opnd_t opnd;
+    CLIENT_ASSERT(r < REG_LAST_VALID_ENUM, "opnd_create_reg: invalid register");
+    opnd.kind = REG_kind;
+    opnd.value.reg = r;
+    return opnd;
+}
+
+reg_id_t
+opnd_get_reg(opnd_t opnd)
+{
+    CLIENT_ASSERT(opnd_is_reg(opnd), "opnd_get_reg called on non-reg opnd");
+    return opnd.value.reg;
+}
+
+
+opnd_size_t
+opnd_get_size(opnd_t opnd)
+{
+    switch(opnd.kind) {
+    case REG_kind: 
+        return reg_get_size(opnd_get_reg(opnd));
+    case IMMED_INTEGER_kind:
+    case BASE_DISP_kind:
+#ifdef X64
+    case REL_ADDR_kind: 
+    case ABS_ADDR_kind: 
+#endif
+        return opnd.size;
+    default:
+        CLIENT_ASSERT(false, "opnd_get_size: unknown opnd type");
+        return OPSZ_NA;
+    }
+}
+
+void
+opnd_set_size(opnd_t *opnd, opnd_size_t newsize)
+{
+    switch(opnd->kind) {
+    case IMMED_INTEGER_kind:
+    case BASE_DISP_kind:
+#ifdef X64
+    case REL_ADDR_kind: 
+    case ABS_ADDR_kind: 
+#endif
+        opnd->size = newsize;
+        return;
+    default:
+        CLIENT_ASSERT(false, "opnd_set_size: unknown opnd type");
+    }
+}
+
+/* immediate operands */
+
+opnd_t
+opnd_create_immed_int(ptr_int_t i, opnd_size_t size)
+{
+    opnd_t opnd;
+    opnd.kind = IMMED_INTEGER_kind;
+    CLIENT_ASSERT(size < OPSZ_LAST_ENUM, "opnd_create_immed_int: invalid size");
+    opnd.size = size;
+    opnd.value.immed_int = i;
+    DODEBUG({
+        uint sz = opnd_size_in_bytes(size);
+        if (sz == 1) {
+            CLIENT_ASSERT(CHECK_TRUNCATE_TYPE_sbyte(i),
+                          "opnd_create_immed_int: value too large for 8-bit size");
+        } else if (sz == 2) {
+            CLIENT_ASSERT(CHECK_TRUNCATE_TYPE_short(i),
+                          "opnd_create_immed_int: value too large for 16-bit size");
+        } else if (sz == 4) {
+            CLIENT_ASSERT(CHECK_TRUNCATE_TYPE_int(i),
+                          "opnd_create_immed_int: value too large for 32-bit size");
+        }
+    });
+    return opnd;
+}
+
+opnd_t
+opnd_create_immed_float(float i)
+{
+    opnd_t opnd;
+    opnd.kind = IMMED_FLOAT_kind;
+    /* note that manipulating floats is dangerous - see case 4360 
+     * this copy shouldn't require any fp state, though
+     */
+    opnd.value.immed_float = i;
+    /* currently only used for implicit constants that have no size */
+    opnd.size = OPSZ_0;
+    return opnd;
+}
+
+ptr_int_t
+opnd_get_immed_int(opnd_t opnd)
+{
+    CLIENT_ASSERT(opnd_is_immed_int(opnd), "opnd_get_immed_int called on non-immed-int");
+    return opnd.value.immed_int;
+}
+
+/* NOTE: requires caller to be under PRESERVE_FLOATING_POINT_STATE */
+float
+opnd_get_immed_float(opnd_t opnd)
+{
+    CLIENT_ASSERT(opnd_is_immed_float(opnd),
+                  "opnd_get_immed_float called on non-immed-float");
+    /* note that manipulating floats is dangerous - see case 4360 
+     * this return shouldn't require any fp state, though
+     */
+    return opnd.value.immed_float;
+}
+
+
+/* address operands */
+
+opnd_t
+opnd_create_pc(app_pc pc)
+{
+    opnd_t opnd;
+    opnd.kind = PC_kind;
+    opnd.value.pc = pc;
+    return opnd;
+}
+
+/* N.B.: seg_selector is a segment selector, not a SEG_ constant */
+opnd_t
+opnd_create_far_pc(ushort seg_selector, app_pc pc)
+{
+    opnd_t opnd;
+    opnd.kind = FAR_PC_kind;
+    opnd.seg.far_pc_seg_selector = seg_selector;
+    opnd.value.pc = pc;
+    return opnd;
+}
+
+opnd_t
+opnd_create_instr(instr_t *instr)
+{
+    opnd_t opnd;
+    opnd.kind = INSTR_kind;
+    opnd.value.instr = instr;
+    return opnd;
+}
+
+opnd_t
+opnd_create_far_instr(ushort seg_selector, instr_t *instr)
+{
+    opnd_t opnd;
+    opnd.kind = FAR_INSTR_kind;
+    opnd.seg.far_pc_seg_selector = seg_selector;
+    opnd.value.instr = instr;
+    return opnd;
+}
+
+app_pc
+opnd_get_pc(opnd_t opnd)
+{
+    if (opnd_is_pc(opnd))
+        return opnd.value.pc;
+    else {
+        SYSLOG_INTERNAL_ERROR("opnd type is %d", opnd.kind);
+        CLIENT_ASSERT(false, "opnd_get_pc called on non-pc");
+        return NULL;
+    }
+}
+
+ushort
+opnd_get_segment_selector(opnd_t opnd)
+{
+    if (opnd_is_far_pc(opnd) || opnd_is_far_instr(opnd)) {
+        /* FIXME: segment selectors are 16-bit values */
+        return opnd.seg.far_pc_seg_selector;
+    }
+    CLIENT_ASSERT(false, "opnd_get_segment_selector called on invalid opnd type");
+    return REG_INVALID;
+}
+
+instr_t *
+opnd_get_instr(opnd_t opnd)
+{
+    CLIENT_ASSERT(opnd_is_instr(opnd), "opnd_get_instr called on non-instr");
+    return opnd.value.instr;
+}
+
+
+/* Base+displacement+scaled index operands */
+
+opnd_t
+opnd_create_base_disp_ex(reg_id_t base_reg, reg_id_t index_reg, int scale, int disp,
+                         opnd_size_t size, bool encode_zero_disp, bool force_full_disp,
+                         bool disp_short_addr)
+{
+    return opnd_create_far_base_disp_ex(REG_NULL, base_reg, index_reg, scale, disp, size,
+                                        encode_zero_disp, force_full_disp,
+                                        disp_short_addr);
+}
+
+opnd_t
+opnd_create_base_disp(reg_id_t base_reg, reg_id_t index_reg, int scale, int disp,
+                      opnd_size_t size)
+{
+    return opnd_create_far_base_disp_ex(REG_NULL, base_reg, index_reg, scale, disp, size,
+                                        false, false, false);
+}
+
+opnd_t
+opnd_create_far_base_disp_ex(reg_id_t seg, reg_id_t base_reg, reg_id_t index_reg,
+                             int scale, int disp, opnd_size_t size,
+                             bool encode_zero_disp, bool force_full_disp,
+                             bool disp_short_addr)
+{
+    opnd_t opnd;
+    opnd.kind = BASE_DISP_kind;
+    CLIENT_ASSERT(size < OPSZ_LAST_ENUM, "opnd_create_*base_disp*: invalid size");
+    opnd.size = size;
+    CLIENT_ASSERT(scale <= 8, "opnd_create_*base_disp*: invalid scale");
+    CLIENT_ASSERT(index_reg == REG_NULL || scale > 0,
+                  "opnd_create_*base_disp*: index requires scale");
+    CLIENT_ASSERT(seg == REG_NULL ||
+                  (seg >= REG_START_SEGMENT && seg <= REG_STOP_SEGMENT),
+                  "opnd_create_*base_disp*: invalid segment");
+    CLIENT_ASSERT(base_reg <= REG_LAST_ENUM, "opnd_create_*base_disp*: invalid base");
+    CLIENT_ASSERT(index_reg <= REG_LAST_ENUM, "opnd_create_*base_disp*: invalid index");
+    CLIENT_ASSERT_BITFIELD_TRUNCATE(SCALE_SPECIFIER_BITS, scale,
+                                    "opnd_create_*base_disp*: invalid scale");
+    opnd.seg.segment = seg;
+    opnd.value.base_disp.base_reg = base_reg;
+    opnd.value.base_disp.index_reg = index_reg;
+    opnd.value.base_disp.scale = (byte) scale;
+    opnd.value.base_disp.disp = disp;
+    opnd.value.base_disp.encode_zero_disp = (byte) encode_zero_disp;
+    opnd.value.base_disp.force_full_disp = (byte) force_full_disp;
+    opnd.value.base_disp.disp_short_addr = (byte) disp_short_addr;
+    return opnd;
+}
+
+opnd_t
+opnd_create_far_base_disp(reg_id_t seg, reg_id_t base_reg, reg_id_t index_reg, int scale,
+                          int disp, opnd_size_t size)
+{
+    return opnd_create_far_base_disp_ex(seg, base_reg, index_reg, scale, disp, size,
+                                        false, false, false);
+}
+
+reg_id_t
+opnd_get_base(opnd_t opnd)
+{
+    if (opnd_is_base_disp(opnd))
+        return opnd.value.base_disp.base_reg;
+#ifdef X64
+    if (opnd_is_abs_addr(opnd))
+        return REG_NULL;
+#endif
+    CLIENT_ASSERT(false, "opnd_get_base called on invalid opnd type");
+    return REG_INVALID;
+}
+
+int
+opnd_get_disp(opnd_t opnd)
+{
+    if (opnd_is_base_disp(opnd))
+        return opnd.value.base_disp.disp;
+    CLIENT_ASSERT(false, "opnd_get_disp called on invalid opnd type");
+    return REG_INVALID;
+}
+
+bool
+opnd_is_disp_encode_zero(opnd_t opnd)
+{
+    if (opnd_is_base_disp(opnd))
+        return opnd.value.base_disp.encode_zero_disp;
+    CLIENT_ASSERT(false, "opnd_is_disp_encode_zero called on invalid opnd type");
+    return false;
+}
+
+bool
+opnd_is_disp_force_full(opnd_t opnd)
+{
+    if (opnd_is_base_disp(opnd))
+        return opnd.value.base_disp.force_full_disp;
+    CLIENT_ASSERT(false, "opnd_is_disp_force_full called on invalid opnd type");
+    return false;
+}
+
+bool
+opnd_is_disp_short_addr(opnd_t opnd)
+{
+    if (opnd_is_base_disp(opnd))
+        return opnd.value.base_disp.disp_short_addr;
+    CLIENT_ASSERT(false, "opnd_is_disp_short_addr called on invalid opnd type");
+    return false;
+}
+
+void
+opnd_set_disp(opnd_t *opnd, int disp)
+{
+    if (opnd_is_base_disp(*opnd))
+        opnd->value.base_disp.disp = disp;
+    else
+        CLIENT_ASSERT(false, "opnd_set_disp called on invalid opnd type");
+}
+
+void
+opnd_set_disp_ex(opnd_t *opnd, int disp, bool encode_zero_disp, bool force_full_disp,
+                 bool disp_short_addr)
+{
+    if (opnd_is_base_disp(*opnd)) {
+        opnd->value.base_disp.encode_zero_disp = (byte) encode_zero_disp;
+        opnd->value.base_disp.force_full_disp = (byte) force_full_disp;
+        opnd->value.base_disp.disp_short_addr = (byte) disp_short_addr;
+        opnd->value.base_disp.disp = disp;
+    } else
+        CLIENT_ASSERT(false, "opnd_set_disp_ex called on invalid opnd type"); 
+}
+
+reg_id_t
+opnd_get_index(opnd_t opnd)
+{
+    if (opnd_is_base_disp(opnd))
+        return opnd.value.base_disp.index_reg;
+    CLIENT_ASSERT(false, "opnd_get_index called on invalid opnd type");
+    return REG_INVALID;
+}
+
+int
+opnd_get_scale(opnd_t opnd)
+{
+    if (opnd_is_base_disp(opnd))
+        return opnd.value.base_disp.scale;
+#ifdef X64
+    if (opnd_is_abs_addr(opnd))
+        return REG_NULL;
+#endif
+    CLIENT_ASSERT(false, "opnd_get_scale called on invalid opnd type");
+    return -1;
+}
+
+reg_id_t
+opnd_get_segment(opnd_t opnd)
+{
+    if (opnd_is_base_disp(opnd)
+        IF_X64(|| opnd_is_abs_addr(opnd) || opnd_is_rel_addr(opnd)))
+        return opnd.seg.segment;
+    CLIENT_ASSERT(false, "opnd_get_segment called on invalid opnd type");
+    return REG_INVALID;
+}
+
+opnd_t 
+opnd_create_abs_addr(void *addr, opnd_size_t data_size)
+{
+    return opnd_create_far_abs_addr(REG_NULL, addr, data_size);
+}
+
+opnd_t 
+opnd_create_far_abs_addr(reg_id_t seg, void *addr, opnd_size_t data_size)
+{
+    /* PR 253327: For x64, there's no way to create 0xa0-0xa3 w/ addr
+     * prefix since we'll make a base-disp instead: but our IR is
+     * supposed to be at a higher abstraction level anyway, though w/
+     * the sib byte the base-disp ends up being one byte longer.
+     */
+    if (IF_X64_ELSE((ptr_uint_t)addr <= UINT_MAX, true)) {
+        CLIENT_ASSERT(CHECK_TRUNCATE_TYPE_uint((ptr_uint_t)addr),
+                      "internal error: abs addr too large");
+        return opnd_create_far_base_disp(seg, REG_NULL, REG_NULL, 0,
+                                         (int)(ptr_int_t)addr, data_size);
+    } 
+#ifdef X64
+    else {
+        opnd_t opnd;
+        opnd.kind = ABS_ADDR_kind;
+        CLIENT_ASSERT(data_size < OPSZ_LAST_ENUM, "opnd_create_base_disp: invalid size");
+        opnd.size = data_size;
+        CLIENT_ASSERT(seg == REG_NULL ||
+                      (seg >= REG_START_SEGMENT && seg <= REG_STOP_SEGMENT),
+                      "opnd_create_far_abs_addr: invalid segment");
+        opnd.seg.segment = seg;
+        opnd.value.addr = addr;
+        return opnd;
+    }
+#endif
+}
+
+#ifdef X64
+opnd_t 
+opnd_create_rel_addr(void *addr, opnd_size_t data_size)
+{
+    return opnd_create_far_rel_addr(REG_NULL, addr, data_size);
+}
+
+/* PR 253327: We represent rip-relative w/ an address-size prefix
+ * (i.e., 32 bits instead of 64) as simply having the top 32 bits of
+ * "addr" zeroed out.  This means that we never encode an address
+ * prefix, and we if one already exists in the raw bits we have to go
+ * looking for it at encode time.
+ */
+opnd_t 
+opnd_create_far_rel_addr(reg_id_t seg, void *addr, opnd_size_t data_size)
+{
+    opnd_t opnd;
+    opnd.kind = REL_ADDR_kind;
+    CLIENT_ASSERT(data_size < OPSZ_LAST_ENUM, "opnd_create_base_disp: invalid size");
+    opnd.size = data_size;
+    CLIENT_ASSERT(seg == REG_NULL ||
+                  (seg >= REG_START_SEGMENT && seg <= REG_STOP_SEGMENT),
+                  "opnd_create_far_rel_addr: invalid segment");
+    opnd.seg.segment = seg;
+    opnd.value.addr = addr;
+    return opnd;
+}
+#endif
+
+void *
+opnd_get_addr(opnd_t opnd)
+{
+    /* check base-disp first since opnd_is_abs_addr() says yes for it */
+    if (opnd_is_abs_base_disp(opnd))
+        return (void *)(ptr_uint_t) opnd_get_disp(opnd);
+#ifdef X64
+    if (opnd_is_rel_addr(opnd) || opnd_is_abs_addr(opnd))
+        return opnd.value.addr;
+#endif
+    CLIENT_ASSERT(false, "opnd_get_addr called on invalid opnd type");
+    return NULL;
+}
+
+bool
+opnd_is_memory_reference(opnd_t opnd)
+{
+    return (opnd_is_base_disp(opnd)
+            IF_X64(|| opnd_is_abs_addr(opnd) || opnd_is_rel_addr(opnd)));
+}
+
+bool
+opnd_is_far_memory_reference(opnd_t opnd)
+{
+    return (opnd_is_far_base_disp(opnd)
+            IF_X64(|| opnd_is_far_abs_addr(opnd) || opnd_is_far_rel_addr(opnd)));
+}
+
+bool
+opnd_is_near_memory_reference(opnd_t opnd)
+{
+    return (opnd_is_near_base_disp(opnd)
+            IF_X64(|| opnd_is_near_abs_addr(opnd) || opnd_is_near_rel_addr(opnd)));
+}
+
+int
+opnd_num_regs_used(opnd_t opnd)
+{
+    switch (opnd.kind) {
+    case NULL_kind:
+    case IMMED_INTEGER_kind:
+    case IMMED_FLOAT_kind:
+    case PC_kind:
+    case FAR_PC_kind:
+    case INSTR_kind:
+    case FAR_INSTR_kind:
+        return 0;
+
+    case REG_kind: 
+        return 1;
+
+    case BASE_DISP_kind: 
+        return (((opnd_get_base(opnd)==REG_NULL) ? 0 : 1) + 
+                ((opnd_get_index(opnd)==REG_NULL) ? 0 : 1) +
+                ((opnd_get_segment(opnd)==REG_NULL) ? 0 : 1));
+
+#ifdef X64
+    case REL_ADDR_kind: 
+    case ABS_ADDR_kind: 
+        return ((opnd_get_segment(opnd) == REG_NULL) ? 0 : 1);
+#endif
+    default: 
+        CLIENT_ASSERT(false, "opnd_num_regs_used called on invalid opnd type"); 
+        return 0;
+    }    
+}
+
+reg_id_t
+opnd_get_reg_used(opnd_t opnd, int index)
+{
+    switch (opnd.kind) {
+    case NULL_kind:
+    case IMMED_INTEGER_kind:
+    case IMMED_FLOAT_kind:
+    case PC_kind:
+    case FAR_PC_kind:
+        CLIENT_ASSERT(false, "opnd_get_reg_used called on invalid opnd type");
+        return REG_NULL;
+
+    case REG_kind: 
+        if (index == 0)
+            return opnd_get_reg(opnd);
+        else {
+            CLIENT_ASSERT(false, "opnd_get_reg_used called on invalid opnd type");
+            return REG_NULL;
+        }
+
+    case BASE_DISP_kind: 
+        if (index == 0) {
+            if (opnd_get_base(opnd) != REG_NULL)
+                return opnd_get_base(opnd);
+            else if (opnd_get_index(opnd) != REG_NULL)
+                return opnd_get_index(opnd);
+            else
+                return opnd_get_segment(opnd);
+        } else if (index == 1) {
+            if (opnd_get_index(opnd) != REG_NULL)
+                return opnd_get_index(opnd);
+            else
+                return opnd_get_segment(opnd);
+        } else if (index == 2)
+            return opnd_get_segment(opnd);
+        else {
+            CLIENT_ASSERT(false, "opnd_get_reg_used called on invalid opnd type"); 
+            return REG_NULL;
+        }
+
+#ifdef X64
+    case REL_ADDR_kind: 
+    case ABS_ADDR_kind: 
+        if (index == 0)
+            return opnd_get_segment(opnd);
+        else {
+            /* We only assert if beyond the number possible: not if beyond the
+             * number present.  Should we assert on the latter?
+             */
+            CLIENT_ASSERT(false, "opnd_get_reg_used called on invalid opnd type"); 
+            return REG_NULL;
+        }
+#endif
+
+    default: 
+        CLIENT_ASSERT(false, "opnd_get_reg_used called on invalid opnd type"); 
+        return REG_NULL;
+    }    
+}
+
+/***************************************************************************/
+/* utility routines */
+
+const reg_id_t regparms[] = {
+#ifdef X64
+    REGPARM_0, REGPARM_1, REGPARM_2, REGPARM_3, 
+# ifdef LINUX
+    REGPARM_4, REGPARM_5,
+# endif
+#endif
+    REG_INVALID
+};
+
+/* Maps sub-registers to their containing register. */
+const reg_id_t reg_fixer[]={
+    REG_NULL,
+    REG_XAX,  REG_XCX,  REG_XDX,  REG_XBX,  REG_XSP,  REG_XBP,  REG_XSI,  REG_XDI,
+    REG_R8,   REG_R9,   REG_R10,  REG_R11,  REG_R12,  REG_R13,  REG_R14,  REG_R15,
+    REG_XAX,  REG_XCX,  REG_XDX,  REG_XBX,  REG_XSP,  REG_XBP,  REG_XSI,  REG_XDI,
+    REG_R8,   REG_R9,   REG_R10,  REG_R11,  REG_R12,  REG_R13,  REG_R14,  REG_R15,
+    REG_XAX,  REG_XCX,  REG_XDX,  REG_XBX,  REG_XSP,  REG_XBP,  REG_XSI,  REG_XDI,
+    REG_R8,   REG_R9,   REG_R10,  REG_R11,  REG_R12,  REG_R13,  REG_R14,  REG_R15,
+    REG_XAX,  REG_XCX,  REG_XDX,  REG_XBX,  REG_XAX,  REG_XCX,  REG_XDX,  REG_XBX, 
+    REG_R8,   REG_R9,   REG_R10,  REG_R11,  REG_R12,  REG_R13,  REG_R14,  REG_R15,
+    REG_XSI,  REG_XDI,  REG_XSP,  REG_XBP,
+    REG_MM0,  REG_MM1,  REG_MM2,  REG_MM3,  REG_MM4,  REG_MM5,  REG_MM6,  REG_MM7,
+    REG_XMM0, REG_XMM1, REG_XMM2, REG_XMM3, REG_XMM4, REG_XMM5, REG_XMM6, REG_XMM7,
+    REG_XMM8, REG_XMM9, REG_XMM10,REG_XMM11,REG_XMM12,REG_XMM13,REG_XMM14,REG_XMM15,
+    REG_ST0,  REG_ST1,  REG_ST2,  REG_ST3,  REG_ST4,  REG_ST5,  REG_ST6,  REG_ST7,
+    SEG_ES,   SEG_CS,   SEG_SS,   SEG_DS,   SEG_FS,   SEG_GS,
+    REG_DR0,  REG_DR1,  REG_DR2,  REG_DR3,  REG_DR4,  REG_DR5,  REG_DR6,  REG_DR7,
+    REG_DR8,  REG_DR9,  REG_DR10, REG_DR11, REG_DR12, REG_DR13, REG_DR14, REG_DR15, 
+    REG_CR0,  REG_CR1,  REG_CR2,  REG_CR3,  REG_CR4,  REG_CR5,  REG_CR6,  REG_CR7, 
+    REG_CR8,  REG_CR9,  REG_CR10, REG_CR11, REG_CR12, REG_CR13, REG_CR14, REG_CR15,
+};
+
+#ifdef DEBUG
+void
+reg_check_reg_fixer(void)
+{
+    /* ignore REG_INVALID, so should equal REG_LAST_ENUM */
+    CLIENT_ASSERT(sizeof(reg_fixer)/sizeof(reg_fixer[0]) == REG_LAST_ENUM,
+                  "internal register enum error");
+}
+#endif
+
+/* 
+   opnd_uses_reg is now changed so that it does consider 8/16 bit
+   register overlaps.  i think this change is OK and correct, but not
+   sure.  as far as I'm aware, only my optimization stuff and the
+   register stealing code (which is now not used, right?) relies on
+   this code ==> but we now export it via CI API */
+
+bool 
+opnd_uses_reg(opnd_t opnd, reg_id_t reg)
+{
+    if (reg == REG_NULL)
+        return false;
+    switch (opnd.kind) {
+    case NULL_kind:
+    case IMMED_INTEGER_kind:
+    case IMMED_FLOAT_kind:
+    case PC_kind:
+    case FAR_PC_kind:
+    case INSTR_kind:
+    case FAR_INSTR_kind:
+        return false;
+
+    case REG_kind: 
+        return (reg_fixer[reg] == reg_fixer[opnd_get_reg(opnd)]);
+
+    case BASE_DISP_kind: 
+        return (reg_fixer[reg] == reg_fixer[opnd_get_base(opnd)] || 
+                reg_fixer[reg] == reg_fixer[opnd_get_index(opnd)] ||
+                reg_fixer[reg] == reg_fixer[opnd_get_segment(opnd)]);
+
+#ifdef X64
+    case REL_ADDR_kind:
+    case ABS_ADDR_kind:
+        return (reg_fixer[reg] == reg_fixer[opnd_get_segment(opnd)]);
+#endif
+
+    default: 
+        CLIENT_ASSERT(false, "opnd_uses_reg: unknown opnd type"); 
+        return false;
+    }    
+}
+
+bool 
+opnd_replace_reg(opnd_t *opnd, reg_id_t old_reg, reg_id_t new_reg)
+{
+    switch (opnd->kind) {
+    case NULL_kind:
+    case IMMED_INTEGER_kind:
+    case IMMED_FLOAT_kind:
+    case PC_kind:
+    case FAR_PC_kind:
+    case INSTR_kind:
+    case FAR_INSTR_kind:
+        return false;
+
+    case REG_kind: 
+        if (old_reg == opnd_get_reg(*opnd)) {
+            *opnd = opnd_create_reg(new_reg);
+            return true;
+        }
+        return false;
+                
+    case BASE_DISP_kind:
+        {
+            reg_id_t ob = opnd_get_base(*opnd);
+            reg_id_t oi = opnd_get_index(*opnd);
+            reg_id_t os = opnd_get_segment(*opnd);
+            opnd_size_t size = opnd_get_size(*opnd);
+            if (old_reg == ob || old_reg == oi || old_reg == os) {
+                reg_id_t b = (old_reg == ob) ? new_reg : ob;
+                reg_id_t i = (old_reg == oi) ? new_reg : oi;
+                reg_id_t s = (old_reg == os) ? new_reg : os;
+                int sc = opnd_get_scale(*opnd);
+                int d = opnd_get_disp(*opnd);
+                *opnd = opnd_create_far_base_disp_ex(s, b, i, sc, d, size,
+                                                     opnd_is_disp_encode_zero(*opnd),
+                                                     opnd_is_disp_force_full(*opnd),
+                                                     opnd_is_disp_short_addr(*opnd));
+                return true;
+            }
+        }
+        return false;
+
+#ifdef X64
+    case REL_ADDR_kind:
+        if (old_reg == opnd_get_segment(*opnd)) {
+            *opnd = opnd_create_far_rel_addr(new_reg, opnd_get_addr(*opnd),
+                                             opnd_get_size(*opnd));
+            return true;
+        }
+        return false;
+
+    case ABS_ADDR_kind:
+        if (old_reg == opnd_get_segment(*opnd)) {
+            *opnd = opnd_create_far_abs_addr(new_reg, opnd_get_addr(*opnd),
+                                             opnd_get_size(*opnd));
+            return true;
+        }
+        return false;
+#endif
+                
+    default: 
+        CLIENT_ASSERT(false, "opnd_replace_reg: invalid opnd type"); 
+        return false;
+    }    
+}
+
+/* this is not conservative -- only considers two memory references to
+ * be the same if their constituent components (registers, displacement)
+ * are the same.
+ * different from opnd_same b/c this routine ignores data size!
+ */
+bool opnd_same_address(opnd_t op1, opnd_t op2)
+{
+    if (op1.kind != op2.kind)
+        return false;
+    if (!opnd_is_memory_reference(op1) || !opnd_is_memory_reference(op2))
+    if (opnd_get_segment(op1) != opnd_get_segment(op2))
+        return false;
+    if (opnd_is_base_disp(op1)) {
+        if (!opnd_is_base_disp(op2))
+            return false;
+        if (opnd_get_base(op1) != opnd_get_base(op2))
+            return false;
+        if (opnd_get_index(op1) != opnd_get_index(op2))
+            return false;
+        if (opnd_get_scale(op1) != opnd_get_scale(op2))
+            return false;
+        if (opnd_get_disp(op1) != opnd_get_disp(op2))
+            return false;
+    } else {
+#ifdef X64        
+        CLIENT_ASSERT(opnd_is_abs_addr(op1) || opnd_is_rel_addr(op1),
+                      "internal type error in opnd_same_address");
+        if (opnd_get_addr(op1) != opnd_get_addr(op2))
+            return false;
+#else
+        CLIENT_ASSERT(false, "internal type error in opnd_same_address");
+#endif
+    }
+
+    /* we ignore size */
+
+    return true;
+}
+
+static bool
+opnd_same_sizes_ok(opnd_size_t s1, opnd_size_t s2, bool is_reg)
+{
+    opnd_size_t s1_default, s2_default;
+    decode_info_t di;
+    if (s1 == s2)
+        return true;
+    /* This routine is used for variable sizes in INSTR_CREATE macros so we
+     * check whether the default size matches.  If we need to do more
+     * then we'll have to hook into encode's size resolution to resolve all
+     * operands with each other's constraints at the instr level before coming here.
+     */
+    IF_X64(di.x86_mode = false);
+    di.prefixes = 0;
+    s1_default = resolve_variable_size(&di, s1, is_reg);
+    s2_default = resolve_variable_size(&di, s2, is_reg);
+    return (s1_default == s2_default);
+}
+
+bool opnd_same(opnd_t op1, opnd_t op2)
+{
+    if (op1.kind!=op2.kind)
+        return false;
+    else if (!opnd_same_sizes_ok(op1.size, op2.size, opnd_is_reg(op1)) &&
+             (opnd_is_immed_int(op1) ||
+              opnd_is_memory_reference(op1)))
+        return false;
+    /* If we could rely on unused bits being 0 could avoid dispatch on type.
+     * Presumably not on critical path, though, so not bothering to try and
+     * asssert that those bits are 0.
+     */
+    switch (op1.kind) {
+    case NULL_kind:
+        return true;
+    case IMMED_INTEGER_kind:
+        return op1.value.immed_int == op2.value.immed_int;
+    case IMMED_FLOAT_kind:
+        /* HACK to avoid generating floating point instrs:
+         * we assume a float is 32 bit just like an int
+         */
+        return op1.value.immed_int == op2.value.immed_int;
+    case PC_kind:
+        return op1.value.pc == op2.value.pc;
+    case FAR_PC_kind:
+        return (op1.seg.far_pc_seg_selector == op2.seg.far_pc_seg_selector && 
+                op1.value.pc == op2.value.pc);
+    case INSTR_kind:
+    case FAR_INSTR_kind:
+        return op1.value.instr == op2.value.instr;
+    case REG_kind: 
+        return op1.value.reg == op2.value.reg;
+    case BASE_DISP_kind:
+        return (op1.seg.segment == op2.seg.segment && 
+                op1.value.base_disp.base_reg == op2.value.base_disp.base_reg &&
+                op1.value.base_disp.index_reg == op2.value.base_disp.index_reg &&
+                op1.value.base_disp.scale == op2.value.base_disp.scale &&
+                op1.value.base_disp.disp == op2.value.base_disp.disp &&
+                op1.value.base_disp.encode_zero_disp ==
+                op2.value.base_disp.encode_zero_disp &&
+                op1.value.base_disp.force_full_disp ==
+                op2.value.base_disp.force_full_disp &&
+                /* disp_short_addr only matters if no registers are set */
+                (((op1.value.base_disp.base_reg != REG_NULL ||
+                   op1.value.base_disp.index_reg != REG_NULL) &&
+                  (op2.value.base_disp.base_reg != REG_NULL ||
+                   op2.value.base_disp.index_reg != REG_NULL)) ||
+                 op1.value.base_disp.disp_short_addr ==
+                 op2.value.base_disp.disp_short_addr));
+#ifdef X64
+    case REL_ADDR_kind:
+    case ABS_ADDR_kind:
+        return (op1.seg.segment == op2.seg.segment && 
+                op1.value.addr == op2.value.addr);
+#endif
+    default: 
+        CLIENT_ASSERT(false, "opnd_same: invalid opnd type"); 
+        return false;
+    }    
+}
+
+bool opnd_share_reg(opnd_t op1, opnd_t op2)
+{
+    switch (op1.kind) {
+    case NULL_kind:
+    case IMMED_INTEGER_kind:
+    case IMMED_FLOAT_kind:
+    case PC_kind:
+    case FAR_PC_kind:
+    case INSTR_kind:
+    case FAR_INSTR_kind:
+        return false;
+    case REG_kind: 
+        return opnd_uses_reg(op2, opnd_get_reg(op1));
+    case BASE_DISP_kind:
+        return (opnd_uses_reg(op2, opnd_get_base(op1)) ||
+                opnd_uses_reg(op2, opnd_get_index(op1)) ||
+                opnd_uses_reg(op2, opnd_get_segment(op1)));
+#ifdef X64
+    case REL_ADDR_kind:
+    case ABS_ADDR_kind:
+        return (opnd_uses_reg(op2, opnd_get_segment(op1)));
+#endif
+    default: 
+        CLIENT_ASSERT(false, "opnd_share_reg: invalid opnd type"); 
+        return false;
+    }    
+}
+
+static bool
+range_overlap(ptr_uint_t a1, ptr_uint_t a2, size_t s1, size_t s2)
+{
+    ptr_uint_t min, max;
+    size_t min_plus;
+    if (a1 < a2) {
+        min = a1;
+        min_plus = s1;
+        max = a2;
+    } else {
+        min = a2;
+        min_plus = s2;
+        max = a1;
+    }
+    return (min + min_plus > max); /* open-ended */
+}
+
+/* Returns true if def, considered as a write, affects use.
+ * Is conservative, so if both def and use are memory references,
+ * will return true unless it can disambiguate them.
+ */
+bool opnd_defines_use(opnd_t def, opnd_t use)
+{
+    switch (def.kind) {
+    case NULL_kind:
+    case IMMED_INTEGER_kind:
+    case IMMED_FLOAT_kind:
+    case PC_kind:
+    case FAR_PC_kind:
+    case INSTR_kind:
+    case FAR_INSTR_kind:
+        return false;
+    case REG_kind: 
+        return opnd_uses_reg(use, opnd_get_reg(def));
+    case BASE_DISP_kind:
+        if (!opnd_is_memory_reference(use))
+            return false;
+#ifdef X64
+        if (!opnd_is_base_disp(use))
+            return true;
+#endif
+        /* try to disambiguate the two memory references 
+         * for now, only consider identical regs and different disp
+         */
+        if (opnd_get_base(def) != opnd_get_base(use))
+            return true;
+        if (opnd_get_index(def) != opnd_get_index(use))
+            return true;
+        if (opnd_get_scale(def) != opnd_get_scale(use))
+            return true;
+        if (opnd_get_segment(def) != opnd_get_segment(use))
+            return true;
+        /* everything is identical, now make sure disps don't overlap */
+        return range_overlap(opnd_get_disp(def), opnd_get_disp(use),
+                             opnd_size_in_bytes(opnd_get_size(def)),
+                             opnd_size_in_bytes(opnd_get_size(use)));
+#ifdef X64
+    case REL_ADDR_kind:
+    case ABS_ADDR_kind:
+        if (!opnd_is_memory_reference(use))
+            return false;
+        if (opnd_is_base_disp(use))
+            return true;
+        if (opnd_get_segment(def) != opnd_get_segment(use))
+            return true;
+        return range_overlap((ptr_uint_t)opnd_get_addr(def),
+                             (ptr_uint_t)opnd_get_addr(use),
+                             opnd_size_in_bytes(opnd_get_size(def)),
+                             opnd_size_in_bytes(opnd_get_size(use)));
+#endif
+    default: 
+        CLIENT_ASSERT(false, "opnd_defines_use: invalid opnd type"); 
+        return false;
+    }    
+}
+
+uint
+opnd_size_in_bytes(opnd_size_t size)
+{
+    /* allow REG_ constants, convert them to OPSZ_ constants */
+    if (size <= REG_LAST_ENUM)
+        size = reg_get_size(size);
+    switch (size) {
+    case OPSZ_0:
+        return 0;
+    case OPSZ_1:
+    case OPSZ_1_reg4: /* mem size */
+        return 1;
+    case OPSZ_2_short1: /* default size */
+    case OPSZ_2:
+    case OPSZ_2_reg4: /* mem size */
+        return 2;
+    case OPSZ_4_of_8:
+    case OPSZ_4_of_16:
+    case OPSZ_4_short2: /* default size */
+#ifndef X64
+    case OPSZ_4x8: /* default size */
+    case OPSZ_4x8_short2: /* default size */
+    case OPSZ_4x8_short2xi8: /* default size */
+#endif
+    case OPSZ_4_short2xi4: /* default size */
+    case OPSZ_4_rex8_short2: /* default size */
+    case OPSZ_4_rex8:
+    case OPSZ_4:
+    case OPSZ_4_reg16: /* mem size */
+        return 4;
+    case OPSZ_6_irex10_short4: /* default size */
+    case OPSZ_6:
+        return 6;
+    case OPSZ_8_of_16:
+    case OPSZ_8_short2:
+    case OPSZ_8_short4:
+    case OPSZ_8:
+#ifdef X64
+    case OPSZ_4x8: /* default size */
+    case OPSZ_4x8_short2: /* default size */
+    case OPSZ_4x8_short2xi8: /* default size */
+#endif
+        return 8;
+    case OPSZ_16:
+        return 16;
+    case OPSZ_6x10:
+        /* table base + limit; w/ addr16, different format, but same total footprint */
+        return IF_X64_ELSE(6, 10);
+    case OPSZ_10:
+        return 10;
+    case OPSZ_14:
+        return 14;
+    case OPSZ_28_short14: /* default size */
+    case OPSZ_28:
+        return 28;
+    case OPSZ_94:
+        return 94;
+    case OPSZ_108_short94: /* default size */
+    case OPSZ_108:
+        return 108;
+    case OPSZ_512:
+        return 512;
+    default:
+        CLIENT_ASSERT(false, "opnd_size_in_bytes: invalid opnd type");
+        return 0;
+    }
+}
+
+/* shrinks all 32-bit registers in opnd to 16 bits.  also shrinks the size of
+ * immed ints and mem refs from OPSZ_4 to OPSZ_2.
+ */
+opnd_t
+opnd_shrink_to_16_bits(opnd_t opnd)
+{
+    int i;
+    for (i=0; i<opnd_num_regs_used(opnd); i++) {
+        reg_id_t reg = opnd_get_reg_used(opnd, i);
+        if (reg >= REG_START_32 && reg <= REG_STOP_32) {
+            opnd_replace_reg(&opnd, reg, reg_32_to_16(reg));
+        }
+    }
+    if ((opnd_is_immed_int(opnd) || opnd_is_memory_reference(opnd)) &&
+        opnd_get_size(opnd) == OPSZ_4)
+        opnd_set_size(&opnd, OPSZ_2);
+    return opnd;
+}
+
+#ifdef X64
+/* shrinks all 64-bit registers in opnd to 32 bits.  also shrinks the size of
+ * immed ints and mem refs from OPSZ_8 to OPSZ_4.
+ */
+opnd_t
+opnd_shrink_to_32_bits(opnd_t opnd)
+{
+    int i;
+    for (i=0; i<opnd_num_regs_used(opnd); i++) {
+        reg_id_t reg = opnd_get_reg_used(opnd, i);
+        if (reg >= REG_START_64 && reg <= REG_STOP_64) {
+            opnd_replace_reg(&opnd, reg, reg_64_to_32(reg));
+        }
+    }
+    if ((opnd_is_immed_int(opnd) || opnd_is_memory_reference(opnd)) &&
+        opnd_get_size(opnd) == OPSZ_8)
+        opnd_set_size(&opnd, OPSZ_4);
+    return opnd;
+}
+
+#endif
+
+static reg_t
+reg_get_value_helper(reg_id_t reg, dr_mcontext_t *mc)
+{
+    CLIENT_ASSERT(reg_is_pointer_sized(reg),
+                  "reg_get_value_helper(): internal error non-ptr sized reg");
+    if (reg == REG_NULL)
+        return 0;
+
+    return *(reg_t *)((byte *)mc + opnd_get_reg_mcontext_offs(reg));
+}
+
+DR_API
+/* Returns the value of the register reg, selected from the passed-in
+ * register values.
+ */
+reg_t
+reg_get_value(reg_id_t reg, dr_mcontext_t *mc)
+{
+    if (reg == REG_NULL)
+        return 0;
+#ifdef X64
+    if (reg >= REG_START_64 && reg <= REG_STOP_64)
+        return reg_get_value_helper(reg, mc);
+    if (reg >= REG_START_32 && reg <= REG_STOP_32) {
+        reg_t val = reg_get_value_helper(reg_fixer[reg], mc);
+        return (val & 0x00000000ffffffff);
+    }
+#else
+    if (reg >= REG_START_32 && reg <= REG_STOP_32) {
+        return reg_get_value_helper(reg, mc);
+    }
+#endif
+    if (reg >= REG_START_8 && reg <= REG_STOP_8) {
+        reg_t val = reg_get_value_helper(reg_fixer[reg], mc);
+        if (reg >= REG_AH && reg <= REG_BH)
+            return ((val & 0x0000ff00) >> 8);
+        else /* all others are the lower 8 bits */
+            return (val & 0x000000ff);
+    }
+    if (reg >= REG_START_16 && reg <= REG_STOP_16) {
+        reg_t val = reg_get_value_helper(reg_fixer[reg], mc);
+        return (val & 0x0000ffff);
+    }
+    /* mmx, xmm, and segment cannot be part of address 
+     * if want to use this routine for more than just effective address
+     * calculations, need to pass in mmx/xmm state, or need to grab it
+     * here
+     */
+    CLIENT_ASSERT(false, "reg_get_value: unsupported register");
+    return 0;
+}
+
+DR_API
+/* Sets the register reg in the passed in mcontext to value.  Currently only works
+ * with ptr sized registers. FIXME - handle other sized registers. */
+void
+reg_set_value(reg_id_t reg, dr_mcontext_t *mc, reg_t value)
+{
+    CLIENT_ASSERT(reg_is_pointer_sized(reg),
+                  "reg_get_value_helper(): internal error non-ptr sized reg");
+
+    if (reg == REG_NULL)
+        return;
+
+    *(reg_t *)((byte *)mc + opnd_get_reg_mcontext_offs(reg)) = value;
+}
+
+DR_API
+/* Returns the effective address of opnd, computed using the passed-in
+ * register values.  If opnd is a far address, ignores that aspect
+ * except for fs: references on Windows.
+ */
+app_pc
+opnd_compute_address(opnd_t opnd, dr_mcontext_t *mc)
+{
+    reg_id_t base, index;
+    int scale, disp;
+    ptr_uint_t seg_base = 0;
+    ptr_uint_t addr = 0;
+    CLIENT_ASSERT(opnd_is_memory_reference(opnd),
+                  "opnd_compute_address: must pass memory reference");
+    if (opnd_is_far_base_disp(opnd)) {
+#ifdef X86
+        seg_base = (ptr_uint_t) get_segment_base(opnd_get_segment(opnd));
+        if (seg_base == POINTER_MAX) /* failure */
+            seg_base = 0;
+#endif
+    }
+#ifdef X64
+    if (opnd_is_abs_addr(opnd) || opnd_is_rel_addr(opnd)) {
+        return (app_pc) opnd_get_addr(opnd) + seg_base;
+    }
+#endif
+    addr = seg_base;
+    base = opnd_get_base(opnd);
+    index = opnd_get_index(opnd);
+    scale = opnd_get_scale(opnd);
+    disp = opnd_get_disp(opnd);
+    logopnd(get_thread_private_dcontext(), 4, opnd, "opnd_compute_address for");
+    addr += reg_get_value(base, mc);
+    LOG(THREAD_GET, LOG_ALL, 4, "\tbase => "PFX"\n", addr);
+    addr += scale * reg_get_value(index, mc);
+    LOG(THREAD_GET, LOG_ALL, 4, "\tindex,scale => "PFX"\n", addr);
+    /* FIXME PR 332730: should disp with no base or index be unsigned
+     * (only matters for x64 or seg_base != 0)?  Certainly not allowed
+     * to subtract from non-zero seg_base but not clear whether it
+     * wraps or what.
+     */
+    addr += disp;
+    LOG(THREAD_GET, LOG_ALL, 4, "\tdisp => "PFX"\n", addr);
+    return (app_pc) addr;
+}
+
+/***************************************************************************
+ ***      Register utility functions
+ ***************************************************************************/
+
+const char *
+get_register_name(reg_id_t reg)
+{
+    return reg_names[reg];
+}
+
+reg_id_t
+reg_to_pointer_sized(reg_id_t reg)
+{
+    return reg_fixer[reg];
+}
+
+reg_id_t
+reg_32_to_16(reg_id_t reg)
+{
+    CLIENT_ASSERT(reg >= REG_START_32 && reg <= REG_STOP_32,
+                  "reg_32_to_16: passed non-32-bit reg");
+    return (reg - REG_START_32) + REG_START_16;
+}
+
+reg_id_t
+reg_32_to_8(reg_id_t reg)
+{
+    reg_id_t r8;
+    CLIENT_ASSERT(reg >= REG_START_32 && reg <= REG_STOP_32,
+                  "reg_32_to_16: passed non-32-bit reg");
+    r8 = (reg - REG_START_32) + REG_START_8;
+    if (r8 >= REG_START_x86_8 && r8 <= REG_STOP_x86_8) {
+#ifdef X64
+        r8 += (REG_START_x64_8 - REG_START_x86_8);
+#else
+        r8 = REG_NULL;
+#endif
+    }
+    return r8;
+}
+
+#ifdef X64
+reg_id_t
+reg_32_to_64(reg_id_t reg)
+{
+    CLIENT_ASSERT(reg >= REG_START_32 && reg <= REG_STOP_32,
+                  "reg_32_to_64: passed non-32-bit reg");
+    return (reg - REG_START_32) + REG_START_64;
+}
+
+reg_id_t
+reg_64_to_32(reg_id_t reg)
+{
+    CLIENT_ASSERT(reg >= REG_START_64 && reg <= REG_STOP_64,
+                  "reg_64_to_32: passed non-64-bit reg");
+    return (reg - REG_START_64) + REG_START_32;
+}
+
+bool
+reg_is_extended(reg_id_t reg)
+{
+    /* Note that we do consider spl, bpl, sil, and dil to be "extended" */
+    return ((reg >= REG_START_64+8  && reg <= REG_STOP_64) ||
+            (reg >= REG_START_32+8  && reg <= REG_STOP_32) ||
+            (reg >= REG_START_16+8  && reg <= REG_STOP_16) ||
+            (reg >= REG_START_8+8   && reg <= REG_STOP_8) ||
+            (reg >= REG_START_x64_8 && reg <= REG_STOP_x64_8) ||
+            (reg >= REG_START_XMM+8 && reg <= REG_STOP_XMM) ||
+            (reg >= REG_START_DR+8  && reg <= REG_STOP_DR) ||
+            (reg >= REG_START_CR+8  && reg <= REG_STOP_CR));
+}
+#endif
+
+reg_id_t
+reg_32_to_opsz(reg_id_t reg, opnd_size_t sz)
+{
+    CLIENT_ASSERT(reg >= REG_START_32 && reg <= REG_STOP_32,
+                  "reg_32_to_opsz: passed non-32-bit reg");
+    if (sz == OPSZ_4)
+        return reg;
+    else if (sz == OPSZ_2)
+        return reg_32_to_16(reg);
+    else if (sz == OPSZ_1)
+        return reg_32_to_8(reg);
+#ifdef X64
+    else if (sz == OPSZ_8)
+        return reg_32_to_64(reg);
+#endif
+    else
+        CLIENT_ASSERT(false, "reg_32_to_opsz: invalid size parameter");
+    return reg;
+}
+
+int
+reg_parameter_num(reg_id_t reg)
+{
+    int r;
+    for (r = 0; r < NUM_REGPARM; r++) {
+        if (reg == regparms[r])
+            return r;
+    }
+    return -1;
+}
+
+int
+opnd_get_reg_dcontext_offs(reg_id_t reg)
+{
+    switch (reg) {
+    case REG_XAX: return XAX_OFFSET;
+    case REG_XBX: return XBX_OFFSET;
+    case REG_XCX: return XCX_OFFSET;
+    case REG_XDX: return XDX_OFFSET;
+    case REG_XSP: return XSP_OFFSET;
+    case REG_XBP: return XBP_OFFSET;
+    case REG_XSI: return XSI_OFFSET;
+    case REG_XDI: return XDI_OFFSET;
+#ifdef X64
+    case REG_R8:  return  R8_OFFSET;
+    case REG_R9:  return  R9_OFFSET;
+    case REG_R10: return R10_OFFSET;
+    case REG_R11: return R11_OFFSET;
+    case REG_R12: return R12_OFFSET;
+    case REG_R13: return R13_OFFSET;
+    case REG_R14: return R14_OFFSET;
+    case REG_R15: return R15_OFFSET;
+#endif
+    default: CLIENT_ASSERT(false, "opnd_get_reg_dcontext_offs: invalid reg");
+        return -1;
+    }
+}
+
+int
+opnd_get_reg_mcontext_offs(reg_id_t reg)
+{
+    return opnd_get_reg_dcontext_offs(reg) - MC_OFFS;
+}
+
+bool
+reg_overlap(reg_id_t r1, reg_id_t r2)
+{
+    if (r1 == REG_NULL || r2 == REG_NULL)
+        return false;
+    /* The XH registers do NOT overlap with the XL registers; else, the
+     * reg_fixer is the answer.
+     */
+    if ((r1 >= REG_START_8HL && r1 <= REG_STOP_8HL) &&
+        (r2 >= REG_START_8HL && r2 <= REG_STOP_8HL) &&
+        r1 != r2)
+        return false;
+    return (reg_fixer[r1] == reg_fixer[r2]);
+}
+
+/* returns the register's representation as 3 bits in a modrm byte,
+ * callers do not expect it to fail 
+ */
+enum {REG_INVALID_BITS = 0x0}; /* returns a valid register nevertheless */
+byte
+reg_get_bits(reg_id_t reg)
+{
+#ifdef X64
+    if (reg >= REG_START_64 && reg <= REG_STOP_64)
+        return (byte) ((reg - REG_START_64) % 8);
+#endif
+    if (reg >= REG_START_32 && reg <= REG_STOP_32)
+        return (byte) ((reg - REG_START_32) % 8);
+    if (reg >= REG_START_8 && reg <= REG_R15L)
+        return (byte) ((reg - REG_START_8) % 8);
+#ifdef X64
+    if (reg >= REG_START_x64_8 && reg <= REG_STOP_x64_8) /* alternates to AH-BH */
+        return (byte) ((reg - REG_START_x64_8 + 4) % 8);
+#endif
+    if (reg >= REG_START_16 && reg <= REG_STOP_16)
+        return (byte) ((reg - REG_START_16) % 8);
+    if (reg >= REG_START_MMX && reg <= REG_STOP_MMX)
+        return (byte) ((reg - REG_START_MMX) % 8);
+    if (reg >= REG_START_XMM && reg <= REG_STOP_XMM)
+        return (byte) ((reg - REG_START_XMM) % 8);
+    if (reg >= REG_START_SEGMENT && reg <= REG_STOP_SEGMENT)
+        return (byte) ((reg - REG_START_SEGMENT) % 8);
+    if (reg >= REG_START_DR && reg <= REG_STOP_DR)
+        return (byte) ((reg - REG_START_DR) % 8);
+    if (reg >= REG_START_CR && reg <= REG_STOP_CR)
+        return (byte) ((reg - REG_START_CR) % 8);
+    CLIENT_ASSERT(false, "reg_get_bits: invalid register");
+    return REG_INVALID_BITS; /* callers don't expect a failure - return some value */
+}
+
+/* returns the OPSZ_ field appropriate for the register */
+opnd_size_t
+reg_get_size(reg_id_t reg)
+{
+#ifdef X64
+    if (reg >= REG_START_64 && reg <= REG_STOP_64)
+        return OPSZ_8;
+#endif
+    if (reg >= REG_START_32 && reg <= REG_STOP_32)
+        return OPSZ_4;
+    if (reg >= REG_START_8 && reg <= REG_STOP_8)
+        return OPSZ_1;
+#ifdef X64
+    if (reg >= REG_START_x64_8 && reg <= REG_STOP_x64_8) /* alternates to AH-BH */
+        return OPSZ_1;
+#endif
+    if (reg >= REG_START_16 && reg <= REG_STOP_16)
+        return OPSZ_2;
+    if (reg >= REG_START_MMX && reg <= REG_STOP_MMX)
+        return OPSZ_8;
+    if (reg >= REG_START_XMM && reg <= REG_STOP_XMM)
+        return OPSZ_16;
+    if (reg >= REG_START_SEGMENT && reg <= REG_STOP_SEGMENT)
+        return OPSZ_2;
+    if (reg >= REG_START_DR && reg <= REG_STOP_DR)
+        return IF_X64_ELSE(OPSZ_8, OPSZ_4);
+    if (reg >= REG_START_CR && reg <= REG_STOP_CR)
+        return IF_X64_ELSE(OPSZ_8, OPSZ_4);
+    CLIENT_ASSERT(false, "reg_get_size: invalid register");
+    return OPSZ_NA;
+}
+
+/*************************
+ ***       instr_t       ***
+ *************************/
+
+/* returns an empty instr_t object */
+instr_t*
+instr_create(dcontext_t *dcontext)
+{
+    instr_t *instr = (instr_t*) heap_alloc(dcontext, sizeof(instr_t) HEAPACCT(ACCT_IR));
+    /* everything initializes to 0, even flags, to indicate
+     * an uninitialized instruction */
+    memset((void *)instr, 0, sizeof(instr_t));
+    IF_X64(instr_set_x86_mode(instr, get_x86_mode(dcontext)));
+    return instr;
+}
+
+/* deletes the instr_t object with handle "inst" and frees its storage */
+void
+instr_destroy(dcontext_t *dcontext, instr_t *instr)
+{
+    instr_free(dcontext, instr);
+
+    /* CAUTION: assumes that instr is not part of any instrlist */
+    heap_free(dcontext, instr, sizeof(instr_t) HEAPACCT(ACCT_IR));
+}
+
+/* returns a clone of orig, but with next and prev fields set to NULL */
+instr_t *
+instr_clone(dcontext_t *dcontext, instr_t *orig)
+{
+    instr_t *instr = (instr_t*) heap_alloc(dcontext, sizeof(instr_t) HEAPACCT(ACCT_IR));
+    memcpy((void *)instr, (void *)orig, sizeof(instr_t));
+    instr->next = NULL;
+    instr->prev = NULL;
+
+    /* PR 214962: clients can see some of our mangling
+     * (dr_insert_mbr_instrumentation(), traces), but don't let the flag
+     * mark other client instrs, which could mess up state translation
+     */
+    instr_set_our_mangling(instr, false);
+
+    if ((orig->flags & INSTR_RAW_BITS_ALLOCATED) != 0) {
+        /* instr length already set from memcpy */
+        instr->bytes = (byte *) heap_alloc(dcontext, instr->length
+                                           HEAPACCT(ACCT_IR));
+        memcpy((void *)instr->bytes, (void *)orig->bytes, instr->length);
+    }
+#ifdef CUSTOM_EXIT_STUBS
+    if ((orig->flags & INSTR_HAS_CUSTOM_STUB) != 0) {
+        /* HACK: dsts is used to store list */
+        instrlist_t *existing = (instrlist_t *) orig->dsts;
+        CLIENT_ASSERT(existing != NULL, "instr_clone: src has inconsistent custom stub");
+        instr->dsts = (opnd_t *) instrlist_clone(dcontext, existing);
+    } 
+    else /* disable normal dst cloning */
+#endif
+    if (orig->dsts != NULL) {
+        instr->dsts = (opnd_t *) heap_alloc(dcontext, instr->num_dsts*sizeof(opnd_t)
+                                          HEAPACCT(ACCT_IR));
+        memcpy((void *)instr->dsts, (void *)orig->dsts,
+               instr->num_dsts*sizeof(opnd_t));
+    }
+    if (orig->srcs != NULL) {
+        instr->srcs = (opnd_t *) heap_alloc(dcontext,
+                                          (instr->num_srcs-1)*sizeof(opnd_t)
+                                          HEAPACCT(ACCT_IR));
+        memcpy((void *)instr->srcs, (void *)orig->srcs,
+               (instr->num_srcs-1)*sizeof(opnd_t));
+    }
+    /* make sure nobody expects us to clone custom data structs pointed to by note */
+    CLIENT_ASSERT(orig->note == 0, "instr_clone: cloning note field not supported");
+    return instr;
+}
+
+/* zeroes out the fields of instr */
+void
+instr_init(dcontext_t *dcontext, instr_t *instr)
+{
+    /* everything initializes to 0, even flags, to indicate
+     * an uninitialized instruction */
+    memset((void *)instr, 0, sizeof(instr_t));
+    IF_X64(instr_set_x86_mode(instr, get_x86_mode(dcontext)));
+}
+
+/* Frees all dynamically allocated storage that was allocated by instr */
+void 
+instr_free(dcontext_t *dcontext, instr_t *instr)
+{
+    if ((instr->flags & INSTR_RAW_BITS_ALLOCATED) != 0) {
+        heap_free(dcontext, instr->bytes, instr->length HEAPACCT(ACCT_IR));
+        instr->bytes = NULL;
+        instr->flags &= ~INSTR_RAW_BITS_ALLOCATED;
+    }
+#ifdef CUSTOM_EXIT_STUBS
+    if ((instr->flags & INSTR_HAS_CUSTOM_STUB) != 0) {
+        /* HACK: dsts is used to store list */
+        instrlist_t *existing = (instrlist_t *) instr->dsts;
+        CLIENT_ASSERT(existing != NULL, "instr_free: custom stubs inconsistent");
+        instrlist_clear_and_destroy(dcontext, existing);
+        instr->dsts = NULL;
+    }
+#endif
+    if (instr->dsts != NULL) {
+        heap_free(dcontext, instr->dsts, instr->num_dsts*sizeof(opnd_t)
+                  HEAPACCT(ACCT_IR));
+        instr->dsts = NULL;
+        instr->num_dsts = 0;
+    }
+    if (instr->srcs != NULL) {
+        /* remember one src is static, rest are dynamic */
+        heap_free(dcontext, instr->srcs, (instr->num_srcs-1)*sizeof(opnd_t)
+                  HEAPACCT(ACCT_IR));
+        instr->srcs = NULL;
+        instr->num_srcs = 0;
+    }
+}
+
+/* Returns number of bytes of heap used by instr */
+int
+instr_mem_usage(instr_t *instr)
+{
+    int usage = 0;
+    if ((instr->flags & INSTR_RAW_BITS_ALLOCATED) != 0) {
+        usage += instr->length;
+    }
+#ifdef CUSTOM_EXIT_STUBS
+    if ((instr->flags & INSTR_HAS_CUSTOM_STUB) != 0) {
+        /* HACK: dsts is used to store list */
+        instrlist_t *il = (instrlist_t *) instr->dsts;
+        instr_t *in;
+        CLIENT_ASSERT(il != NULL, "instr_mem_usage: custom stubs inconsistent");
+        for (in = instrlist_first(il); in != NULL; in = instr_get_next(in))
+            usage += instr_mem_usage(in);
+    }
+#endif
+    if (instr->dsts != NULL) {
+        usage += instr->num_dsts*sizeof(opnd_t);
+    }
+    if (instr->srcs != NULL) {
+        /* remember one src is static, rest are dynamic */
+        usage += (instr->num_srcs-1)*sizeof(opnd_t);
+    }
+    usage += sizeof(instr_t);
+    return usage;
+}
+
+
+/* Frees all dynamically allocated storage that was allocated by instr
+ * Also zeroes out instr's fields
+ * This instr must have been initialized before!
+ */
+void 
+instr_reset(dcontext_t *dcontext, instr_t *instr)
+{
+    instr_free(dcontext, instr);
+    instr_init(dcontext, instr);
+}
+
+/* Frees all dynamically allocated storage that was allocated by instr,
+ * except for allocated raw bits.
+ * Also zeroes out instr's fields, except for raw bit fields and next and prev
+ * fields, whether instr is ok to mangle, and instr's x86 mode.
+ * Use this routine when you want to decode more information into the
+ * same instr_t stucture.
+ * This instr must have been initialized before!
+ */
+void 
+instr_reuse(dcontext_t *dcontext, instr_t *instr)
+{
+    byte *bits = NULL;
+    uint len = 0;
+    bool alloc = false;
+    bool mangle = instr_ok_to_mangle(instr);
+#ifdef X64
+    bool x86_mode = instr_get_x86_mode(instr);
+    uint rip_rel_pos = instr_rip_rel_valid(instr) ? instr->rip_rel_pos : 0;
+#endif
+    instr_t *next = instr->next;
+    instr_t *prev = instr->prev;
+    if (instr_raw_bits_valid(instr)) {
+        if (instr_has_allocated_bits(instr)) {
+            /* pretend has no allocated bits to prevent freeing of them */
+            instr->flags &= ~INSTR_RAW_BITS_ALLOCATED;
+            alloc = true;
+        }
+        bits = instr->bytes;
+        len = instr->length;
+    }
+    instr_free(dcontext, instr);
+    instr_init(dcontext, instr);
+    /* now re-add them */
+    instr->next = next;
+    instr->prev = prev;
+    if (bits != NULL) {
+        instr->bytes = bits;
+        instr->length = len;
+        /* assume that the bits are now valid and the operands are not
+         * (operand and eflags flags are already unset from init)
+         */
+        instr->flags |= INSTR_RAW_BITS_VALID;
+        if (alloc)
+            instr->flags |= INSTR_RAW_BITS_ALLOCATED;
+    }
+#ifdef X64
+    /* preserve across the up-decode */
+    instr_set_x86_mode(instr, x86_mode);
+    if (rip_rel_pos > 0)
+        instr_set_rip_rel_pos(instr, rip_rel_pos);
+#endif
+    if (!mangle)
+        instr->flags |= INSTR_DO_NOT_MANGLE;
+}
+
+instr_t *
+instr_build(dcontext_t *dcontext, int opcode, int instr_num_dsts, int instr_num_srcs)
+{
+    instr_t *instr = instr_create(dcontext);
+    instr_set_opcode(instr, opcode);
+    instr_set_num_opnds(dcontext, instr, instr_num_dsts, instr_num_srcs);
+    return instr;
+}
+
+instr_t *
+instr_build_bits(dcontext_t *dcontext, int opcode, uint num_bytes)
+{
+    instr_t *instr = instr_create(dcontext);
+    instr_set_opcode(instr, opcode);
+    instr_allocate_raw_bits(dcontext, instr, num_bytes);
+    return instr;
+}
+
+
+/* encodes to buffer, then returns length.
+ * needed for things we must have encoding for: length and eflags.
+ * if !always_cache, only caches the encoding if instr_ok_to_mangle();
+ * if always_cache, the caller should invalidate the cache when done.
+ */
+static int
+private_instr_encode(dcontext_t *dcontext, instr_t *instr, bool always_cache)
+{
+    /* we cannot use a stack buffer for encoding since our stack on x64 linux
+     * can be too far to reach from our heap
+     */
+    byte *buf = heap_alloc(dcontext, 32 /* max instr length is 17 bytes */
+                           HEAPACCT(ACCT_IR));
+    uint len;
+    byte *nxt = instr_encode_check_reachability(dcontext, instr, buf);
+    bool valid_to_cache = true;
+    if (nxt == NULL) {
+        nxt = instr_encode_ignore_reachability(dcontext, instr, buf);
+        if (nxt == NULL) {
+            SYSLOG_INTERNAL_WARNING("cannot encode %s\n", op_instr[instr->opcode]->name);
+            heap_free(dcontext, buf, 32 HEAPACCT(ACCT_IR));
+            return 0;
+        }
+        /* if unreachable, we can't cache, since re-relativization won't work */
+        valid_to_cache = false;
+    }
+    len = (int) (nxt - buf);    
+    CLIENT_ASSERT(len > 0 || instr_is_label(instr),
+                  "encode instr for length/eflags error: zero length");
+    CLIENT_ASSERT(len < 32, "encode instr for length/eflags error: instr too long");
+    ASSERT_CURIOSITY(len >= 0 && len < 18);
+
+    /* do not cache encoding if mangle is false, that way we can have
+     * non-cti-instructions that are pc-relative.
+     * we also cannot cache if a rip-relative operand is unreachable.
+     * we can cache if a rip-relative operand is present b/c instr_encode()
+     * sets instr_set_rip_rel_pos() for us.
+     */
+    if (len > 0 &&
+        ((valid_to_cache && instr_ok_to_mangle(instr)) ||
+         always_cache /*caller will use then invalidate*/)) {
+        bool valid = instr_operands_valid(instr);
+#ifdef X64
+        /* we can't call instr_rip_rel_valid() b/c the raw bytes are not yet
+         * set up: we rely on instr_encode() setting instr->rip_rel_pos and
+         * the valid flag, even though raw bytes weren't there at the time.
+         * we rely on the INSTR_RIP_REL_VALID flag being invalidated whenever
+         * the raw bits are.
+         */
+        bool rip_rel_valid = TEST(INSTR_RIP_REL_VALID, instr->flags);
+#endif
+        byte *tmp;
+        CLIENT_ASSERT(!instr_raw_bits_valid(instr),
+                      "encode instr: bit validity error"); /* else shouldn't get here */
+        instr_allocate_raw_bits(dcontext, instr, len);
+        /* we use a hack in order to take advantage of
+         * copy_and_re_relativize_raw_instr(), which copies from instr->bytes
+         * using rip-rel-calculating routines that also use instr->bytes.
+         */
+        tmp = instr->bytes;
+        instr->bytes = buf;
+#ifdef X64
+        instr_set_rip_rel_valid(instr, rip_rel_valid);
+#endif
+        copy_and_re_relativize_raw_instr(dcontext, instr, tmp);
+        instr->bytes = tmp;
+        instr_set_operands_valid(instr, valid);
+    }
+    heap_free(dcontext, buf, 32 HEAPACCT(ACCT_IR));
+    return len;
+}
+
+int
+instr_get_opcode(instr_t *instr)
+{
+    if (instr->opcode == OP_UNDECODED)
+        instr_decode_opcode(get_thread_private_dcontext(), instr);
+    return instr->opcode;
+}
+
+static inline void
+instr_being_modified(instr_t *instr, bool raw_bits_valid)
+{
+    if (!raw_bits_valid) {
+        /* if we're modifying the instr, don't use original bits to encode! */
+        instr_set_raw_bits_valid(instr, false);
+    }
+    /* PR 214962: if client changes our mangling, un-mark to avoid bad translation */
+    instr_set_our_mangling(instr, false);
+}
+
+void
+instr_set_opcode(instr_t *instr, int opcode)
+{
+    instr->opcode = opcode;
+    /* if we're modifying opcode, don't use original bits to encode! */
+    instr_being_modified(instr, false/*raw bits invalid*/);
+    /* do not assume operands are valid, they are separate from opcode,
+     * but if opcode is invalid operands shouldn't be valid
+     */
+    CLIENT_ASSERT((opcode != OP_INVALID && opcode != OP_UNDECODED) ||
+                  !instr_operands_valid(instr),
+                  "instr_set_opcode: operand-opcode validity mismatch");
+}
+
+/* Returns true iff instr's opcode is NOT OP_INVALID.
+ * Not to be confused with an invalid opcode, which can be OP_INVALID or
+ * OP_UNDECODED.  OP_INVALID means an instruction with no valid fields:
+ * raw bits (may exist but do not correspond to a valid instr), opcode,
+ * eflags, or operands.  It could be an uninitialized
+ * instruction or the result of decoding an invalid sequence of bytes.
+ */
+bool 
+instr_valid(instr_t *instr)
+{
+    return (instr->opcode != OP_INVALID);
+}
+
+DR_API
+/* Get the original application PC of the instruction if it exists. */
+app_pc
+instr_get_app_pc(instr_t *instr)
+{
+    return instr_get_translation(instr);
+}
+
+/* Returns true iff instr's opcode is valid.  If the opcode is not
+ * OP_INVALID or OP_UNDECODED it is assumed to be valid.  However, calling
+ * instr_get_opcode() will attempt to decode an OP_UNDECODED opcode, hence the
+ * purpose of this routine.  
+ */
+bool 
+instr_opcode_valid(instr_t *instr)
+{
+    return (instr->opcode != OP_INVALID && instr->opcode != OP_UNDECODED);
+}
+
+
+const instr_info_t * 
+instr_get_instr_info(instr_t *instr)
+{
+    return op_instr[instr_get_opcode(instr)];
+}
+
+const instr_info_t * 
+get_instr_info(int opcode)
+{
+    return op_instr[opcode];
+}
+
+int
+instr_num_srcs(instr_t *instr)
+{
+    if ((instr->flags & INSTR_OPERANDS_VALID) == 0)
+        instr_decode(get_thread_private_dcontext(), instr);
+    return instr->num_srcs;
+}
+
+int
+instr_num_dsts(instr_t *instr)
+{
+    if ((instr->flags & INSTR_OPERANDS_VALID) == 0)
+        instr_decode(get_thread_private_dcontext(), instr);
+    return instr->num_dsts;
+}
+
+/* Returns the pos-th source operand of instr.
+ * If instr's operands are not decoded, goes ahead and decodes them.
+ * Assumes that instr is a single instr (i.e., NOT Level 0).
+ */
+opnd_t
+instr_get_src(instr_t *instr, uint pos)
+{
+    if ((instr->flags & INSTR_OPERANDS_VALID) == 0)
+        instr_decode(get_thread_private_dcontext(), instr);
+    CLIENT_ASSERT(pos >= 0 && pos < instr->num_srcs, "instr_get_src: ordinal invalid");
+    /* remember that src0 is static, rest are dynamic */
+    if (pos == 0)
+        return instr->src0;
+    else
+        return instr->srcs[pos-1];
+}
+
+/* returns the dst opnd at position pos in instr */
+opnd_t
+instr_get_dst(instr_t *instr, uint pos)
+{
+    if ((instr->flags & INSTR_OPERANDS_VALID) == 0)
+        instr_decode(get_thread_private_dcontext(), instr);
+    CLIENT_ASSERT(pos >= 0 && pos < instr->num_dsts, "instr_get_dst: ordinal invalid");
+    return instr->dsts[pos];
+}
+
+/* allocates storage for instr_num_srcs src operands and instr_num_dsts dst operands
+ * assumes that instr is currently all zeroed out!
+ */
+void
+instr_set_num_opnds(dcontext_t *dcontext,
+                    instr_t *instr, int instr_num_dsts, int instr_num_srcs)
+{
+    if (instr_num_dsts > 0) {
+        CLIENT_ASSERT(instr->num_dsts == 0 && instr->dsts == NULL,
+                      "instr_set_num_opnds: dsts are already set");
+        CLIENT_ASSERT_TRUNCATE(instr->num_dsts, byte, instr_num_dsts,
+                               "instr_set_num_opnds: too many dsts");
+        instr->num_dsts = (byte) instr_num_dsts;
+        instr->dsts = (opnd_t *) heap_alloc(dcontext, instr_num_dsts*sizeof(opnd_t)
+                                          HEAPACCT(ACCT_IR));
+    }
+    if (instr_num_srcs > 0) {
+        /* remember that src0 is static, rest are dynamic */
+        if (instr_num_srcs > 1) {
+            CLIENT_ASSERT(instr->num_srcs <= 1 && instr->srcs == NULL,
+                          "instr_set_num_opnds: srcs are already set");
+            instr->srcs = (opnd_t *) heap_alloc(dcontext, (instr_num_srcs-1)*sizeof(opnd_t)
+                                              HEAPACCT(ACCT_IR));
+        }
+        CLIENT_ASSERT_TRUNCATE(instr->num_srcs, byte, instr_num_srcs,
+                               "instr_set_num_opnds: too many srcs");
+        instr->num_srcs = (byte) instr_num_srcs;
+    }
+    instr_being_modified(instr, false/*raw bits invalid*/);
+    /* assume all operands are valid */
+    instr_set_operands_valid(instr, true);
+}
+
+/* sets the src opnd at position pos in instr */
+void
+instr_set_src(instr_t *instr, uint pos, opnd_t opnd)
+{
+    CLIENT_ASSERT(pos >= 0 && pos < instr->num_srcs, "instr_set_src: ordinal invalid");
+    /* remember that src0 is static, rest are dynamic */
+    if (pos == 0)
+        instr->src0 = opnd;
+    else
+        instr->srcs[pos-1] = opnd;
+    /* if we're modifying operands, don't use original bits to encode! */
+    instr_being_modified(instr, false/*raw bits invalid*/);
+    /* assume all operands are valid */
+    instr_set_operands_valid(instr, true);
+}
+
+/* sets the dst opnd at position pos in instr */
+void
+instr_set_dst(instr_t *instr, uint pos, opnd_t opnd)
+{
+    CLIENT_ASSERT(pos >= 0 && pos < instr->num_dsts, "instr_set_dst: ordinal invalid");
+    instr->dsts[pos] = opnd;
+    /* if we're modifying operands, don't use original bits to encode! */
+    instr_being_modified(instr, false/*raw bits invalid*/);
+    /* assume all operands are valid */
+    instr_set_operands_valid(instr, true);
+}
+
+/* These next two functions assume that, if an instr_t has a target
+   field, the target is kept in the 0th src location. */
+opnd_t
+instr_get_target(instr_t *instr)
+{
+    if ((instr->flags & INSTR_OPERANDS_VALID) == 0)
+        instr_decode(get_thread_private_dcontext(), instr);
+    CLIENT_ASSERT(instr_is_cti(instr), "instr_get_target called on non-cti");
+    CLIENT_ASSERT(instr->num_srcs >= 1, "instr_get_target: instr has no sources");
+    return instr->src0;
+}
+
+void
+instr_set_target(instr_t *instr, opnd_t target)
+{
+    CLIENT_ASSERT(instr->num_srcs >= 1, "instr_set_target: instr has no sources");
+    instr->src0 = target;
+    /* if we're modifying operands, don't use original bits to encode,
+     * except for jecxz/loop*
+     */
+    instr_being_modified(instr, instr_is_cti_short_rewrite(instr, NULL));
+    /* assume all operands are valid */
+    instr_set_operands_valid(instr, true);
+}
+
+instr_t *
+instr_set_prefix_flag(instr_t *instr, uint prefix)
+{
+    instr->prefixes |= prefix;
+    instr_being_modified(instr, false/*raw bits invalid*/);
+    return instr;
+}
+
+bool
+instr_get_prefix_flag(instr_t *instr, uint prefix)
+{
+    return ((instr->prefixes & prefix) != 0);
+}
+
+void
+instr_set_prefixes(instr_t *instr, uint prefixes)
+{
+    instr->prefixes = prefixes;
+    instr_being_modified(instr, false/*raw bits invalid*/);
+}
+
+uint
+instr_get_prefixes(instr_t *instr)
+{
+    return instr->prefixes;
+}
+
+#ifdef X64
+/*
+ * Each instruction stores whether it should be interpreted in 32-bit
+ * (x86) or 64-bit (x64) mode.  This routine sets the mode for \p instr.
+ */
+void
+instr_set_x86_mode(instr_t *instr, bool x86)
+{
+    if (x86)
+        instr->flags |= INSTR_X86_MODE;
+    else
+        instr->flags &= ~INSTR_X86_MODE;
+}
+
+/*
+ * Each instruction stores whether it should be interpreted in 32-bit
+ * (x86) or 64-bit (x64) mode.  This routine returns the mode for \p instr.
+ */
+bool 
+instr_get_x86_mode(instr_t *instr)
+{
+    return TEST(INSTR_X86_MODE, instr->flags);
+}
+#endif
+
+#ifdef UNSUPPORTED_API
+/* Returns true iff instr has been marked as targeting the prefix of its
+ * target fragment.
+ *
+ * Some code manipulations need to store a target address in a
+ * register and then jump there, but need the register to be restored
+ * as well.  DR provides a single-instruction prefix that is
+ * placed on all fragments (basic blocks as well as traces) that
+ * restores ecx.  It is on traces for internal DR use.  To have
+ * it added to basic blocks as well, call
+ * dr_add_prefixes_to_basic_blocks() during initialization.
+ */
+bool
+instr_branch_targets_prefix(instr_t *instr)
+{
+    return ((instr->flags & INSTR_BRANCH_TARGETS_PREFIX) != 0);
+}
+
+/* If val is true, indicates that instr's target fragment should be
+ *   entered through its prefix, which restores ecx.
+ * If val is false, indicates that instr should target the normal entry
+ *   point and not the prefix.
+ *
+ * Some code manipulations need to store a target address in a
+ * register and then jump there, but need the register to be restored
+ * as well.  DR provides a single-instruction prefix that is
+ * placed on all fragments (basic blocks as well as traces) that
+ * restores ecx.  It is on traces for internal DR use.  To have
+ * it added to basic blocks as well, call
+ * dr_add_prefixes_to_basic_blocks() during initialization.
+ */
+void
+instr_branch_set_prefix_target(instr_t *instr, bool val)
+{
+    if (val)
+        instr->flags |= INSTR_BRANCH_TARGETS_PREFIX;
+    else
+        instr->flags &= ~INSTR_BRANCH_TARGETS_PREFIX;
+}
+#endif /* UNSUPPORTED_API */
+
+/* Returns true iff instr has been marked as a selfmod check failure exit
+ */
+bool
+instr_branch_selfmod_exit(instr_t *instr)
+{
+    return ((instr->flags & INSTR_BRANCH_SELFMOD_EXIT) != 0);
+}
+
+/* If val is true, indicates that instr is a selfmod check failure exit
+ * If val is false, indicates otherwise
+ */
+void
+instr_branch_set_selfmod_exit(instr_t *instr, bool val)
+{
+    if (val)
+        instr->flags |= INSTR_BRANCH_SELFMOD_EXIT;
+    else
+        instr->flags &= ~INSTR_BRANCH_SELFMOD_EXIT;
+}
+
+/* Returns the type of the original indirect branch of an exit
+ */
+int
+instr_exit_branch_type(instr_t *instr)
+{
+    return instr->flags & EXIT_CTI_TYPES;
+}
+
+/* Set type of indirect branch exit
+ */
+void
+instr_exit_branch_set_type(instr_t *instr, uint type)
+{
+    /* set only expected flags */
+    type &= EXIT_CTI_TYPES;
+    instr->flags &= ~EXIT_CTI_TYPES;
+    instr->flags |= type;
+}
+
+bool
+instr_ok_to_mangle(instr_t *instr)
+{
+    return ((instr->flags & INSTR_DO_NOT_MANGLE) == 0);
+}
+
+void
+instr_set_ok_to_mangle(instr_t *instr, bool val)
+{
+    if (val)
+        instr->flags &= ~INSTR_DO_NOT_MANGLE;
+    else
+        instr->flags |= INSTR_DO_NOT_MANGLE;
+}
+
+/* convenience routine */
+void
+instr_set_meta_no_translation(instr_t *instr)
+{
+    instr_set_ok_to_mangle(instr, false);
+    instr_set_translation(instr, NULL);
+}
+
+bool
+instr_ok_to_emit(instr_t *instr)
+{
+    CLIENT_ASSERT(instr != NULL, "instr_ok_to_emit: passed NULL");
+    return (!TEST(INSTR_DO_NOT_EMIT, instr->flags));
+}
+
+void
+instr_set_ok_to_emit(instr_t *instr, bool val)
+{
+    CLIENT_ASSERT(instr != NULL, "instr_set_ok_to_emit: passed NULL");
+    if (val)
+        instr->flags &= ~INSTR_DO_NOT_EMIT;
+    else
+        instr->flags |= INSTR_DO_NOT_EMIT;
+}
+
+#ifdef CUSTOM_EXIT_STUBS
+/* If instr is not an exit cti, does nothing.  
+ * If instr is an exit cti, sets stub to be custom exit stub code
+ * that will be inserted in the exit stub prior to the normal exit
+ * stub code.  If instr already has custom exit stub code, that
+ * existing instrlist_t is cleared and destroyed (using current thread's
+ * context).  (If stub is NULL, any existing stub code is NOT destroyed.)
+ * The creator of the instrlist_t containing instr is
+ * responsible for destroying stub.  
+ */
+void
+instr_set_exit_stub_code(instr_t *instr, instrlist_t *stub)
+{
+    /* HACK: dsts array is NULL, so we use the dsts pointer
+     * FIXME: put checks in set_num_opnds, etc. that may overwrite this?
+     * FIXME: make separate call to clear existing stubs?
+     * having it not clear for stub==NULL a little hacky
+     */
+    CLIENT_ASSERT(instr_is_cbr(instr) || instr_is_ubr(instr),
+                  "instr_set_exit_stub_code called on non-exit");
+    CLIENT_ASSERT(instr->num_dsts == 0, "instr_set_exit_stub_code: instr invalid");
+    if (stub != NULL && (instr->flags & INSTR_HAS_CUSTOM_STUB) != 0) {
+        /* delete existing */
+        instrlist_t *existing = (instrlist_t *) instr->dsts;
+        instrlist_clear_and_destroy(get_thread_private_dcontext(), existing);
+    }
+    if (stub == NULL) {
+        instr->flags &= ~INSTR_HAS_CUSTOM_STUB;
+        instr->dsts = NULL;
+    } else {
+        instr->flags |= INSTR_HAS_CUSTOM_STUB;
+        instr->dsts = (opnd_t *) stub;
+    }
+}
+
+/* Returns the custom exit stub code instruction list that has been
+ * set for this instruction.  If none exists, returns NULL.
+ */
+instrlist_t *
+instr_exit_stub_code(instr_t *instr)
+{
+    if (!instr_is_cbr(instr) && !instr_is_ubr(instr))
+        return NULL;
+    if (opnd_is_far_pc(instr_get_target(instr)))
+        return NULL;
+    if ((instr->flags & INSTR_HAS_CUSTOM_STUB) == 0)
+        return NULL;
+    return (instrlist_t *) instr->dsts;
+}
+#endif
+
+uint
+instr_get_eflags(instr_t *instr)
+{
+    if ((instr->flags & INSTR_EFLAGS_VALID) == 0) {
+        bool encoded = false;
+        dcontext_t *dcontext = get_thread_private_dcontext();
+        IF_X64(bool old_mode;)
+        /* we assume we cannot trust the opcode independently of operands */
+        if (instr_needs_encoding(instr)) {
+            int len;
+            encoded = true;
+            len = private_instr_encode(dcontext, instr, true/*cache*/);
+            if (len == 0) {
+                if (!instr_is_label(instr))
+                    CLIENT_ASSERT(false, "instr_get_eflags: invalid instr");
+                return 0;
+            }
+        }
+        IF_X64(old_mode = set_x86_mode(dcontext, instr_get_x86_mode(instr)));
+        decode_eflags_usage(dcontext, instr_get_raw_bits(instr), &instr->eflags);
+        IF_X64(set_x86_mode(dcontext, old_mode));
+        if (encoded) {
+            /* if private_instr_encode passed us back whether it's valid
+             * to cache (i.e., non-meta instr that can reach) we could skip
+             * this invalidation for such cases */
+            instr_free_raw_bits(dcontext, instr);
+            CLIENT_ASSERT(!instr_raw_bits_valid(instr), "internal encoding buf error");
+        }
+        /* even if decode fails, set valid to true -- ok?  FIXME */
+        instr_set_eflags_valid(instr, true);
+    }
+    return instr->eflags;
+}
+
+DR_API
+/* Returns the eflags usage of instructions with opcode "opcode",
+ * as EFLAGS_ constants or'ed together 
+ */
+uint 
+instr_get_opcode_eflags(int opcode)
+{
+    /* assumption: all encoding of an opcode have same eflags behavior! */
+    const instr_info_t *info = get_instr_info(opcode);
+    return info->eflags;
+}
+
+uint
+instr_get_arith_flags(instr_t *instr)
+{
+    if ((instr->flags & INSTR_EFLAGS_6_VALID) == 0) {
+        /* just get info on all the flags */
+        return instr_get_eflags(instr);
+    }
+    return instr->eflags;
+}
+
+bool
+instr_eflags_valid(instr_t *instr)
+{
+    return ((instr->flags & INSTR_EFLAGS_VALID) != 0);
+}
+
+void
+instr_set_eflags_valid(instr_t *instr, bool valid)
+{
+    if (valid) {
+        instr->flags |= INSTR_EFLAGS_VALID;
+        instr->flags |= INSTR_EFLAGS_6_VALID;
+    } else {
+        /* assume that arith flags are also invalid */
+        instr->flags &= ~INSTR_EFLAGS_VALID;
+        instr->flags &= ~INSTR_EFLAGS_6_VALID;
+    }
+}
+
+/* Returns true iff instr's arithmetic flags (the 6 bottom eflags) are
+ * up to date */
+bool 
+instr_arith_flags_valid(instr_t *instr)
+{
+    return ((instr->flags & INSTR_EFLAGS_6_VALID) != 0);
+}
+
+/* Sets instr's arithmetic flags (the 6 bottom eflags) to be valid if
+ * valid is true, invalid otherwise */
+void 
+instr_set_arith_flags_valid(instr_t *instr, bool valid)
+{
+    if (valid)
+        instr->flags |= INSTR_EFLAGS_6_VALID;
+    else {
+        instr->flags &= ~INSTR_EFLAGS_VALID;
+        instr->flags &= ~INSTR_EFLAGS_6_VALID;
+    }
+}
+
+void
+instr_set_operands_valid(instr_t *instr, bool valid)
+{
+    if (valid)
+        instr->flags |= INSTR_OPERANDS_VALID;
+    else
+        instr->flags &= ~INSTR_OPERANDS_VALID;
+}
+
+/* N.B.: this routine sets the "raw bits are valid" flag */
+void
+instr_set_raw_bits(instr_t *instr, byte *addr, uint length)
+{
+    if ((instr->flags & INSTR_RAW_BITS_ALLOCATED) != 0) {
+        /* this does happen, when up-decoding an instr using its
+         * own raw bits, so let it happen, but make sure allocated
+         * bits aren't being lost
+         */
+        CLIENT_ASSERT(addr == instr->bytes && length == instr->length,
+                      "instr_set_raw_bits: bits already there, but different");
+    }
+    if (!instr_valid(instr))
+        instr_set_opcode(instr, OP_UNDECODED);
+    instr->flags |= INSTR_RAW_BITS_VALID;
+    instr->bytes = addr;
+    instr->length = length;
+#ifdef X64
+    instr_set_rip_rel_valid(instr, false); /* relies on original raw bits */
+#endif
+}
+
+/* this is sort of a hack, used to allow dynamic reallocation of
+ * the trace buffer, which requires shifting the addresses of all
+ * the trace Instrs since they point into the old buffer
+ */
+void
+instr_shift_raw_bits(instr_t *instr, ssize_t offs)
+{
+    if ((instr->flags & INSTR_RAW_BITS_VALID) != 0)
+        instr->bytes += offs;
+#ifdef X64
+    instr_set_rip_rel_valid(instr, false); /* relies on original raw bits */
+#endif
+}
+
+/* moves the instruction from USE_ORIGINAL_BITS state to a 
+ * needs-full-encoding state
+ */
+void
+instr_set_raw_bits_valid(instr_t *instr, bool valid)
+{
+    if (valid)
+        instr->flags |= INSTR_RAW_BITS_VALID;
+    else {
+        instr->flags &= ~INSTR_RAW_BITS_VALID;
+        /* DO NOT set bytes to NULL or length to 0, we still want to be
+         * able to point at the original instruction for use in translating
+         * addresses for exception/signal handlers
+         * Also do not de-allocate allocated bits
+         */
+#ifdef X64
+        instr_set_rip_rel_valid(instr, false);
+#endif
+    }
+}
+
+bool
+instr_operands_valid(instr_t *instr)
+{
+    return ((instr->flags & INSTR_OPERANDS_VALID) != 0);
+}
+
+bool
+instr_raw_bits_valid(instr_t *instr)
+{
+    return ((instr->flags & INSTR_RAW_BITS_VALID) != 0);
+}
+
+bool
+instr_has_allocated_bits(instr_t *instr)
+{
+    return ((instr->flags & INSTR_RAW_BITS_ALLOCATED) != 0);
+}
+
+bool
+instr_needs_encoding(instr_t *instr)
+{
+    return ((instr->flags & INSTR_RAW_BITS_VALID) == 0);
+}
+
+void
+instr_free_raw_bits(dcontext_t *dcontext, instr_t *instr)
+{
+    if ((instr->flags & INSTR_RAW_BITS_ALLOCATED) == 0)
+        return;
+    heap_free(dcontext, instr->bytes, instr->length HEAPACCT(ACCT_IR));
+    instr->flags &= ~INSTR_RAW_BITS_VALID;
+    instr->flags &= ~INSTR_RAW_BITS_ALLOCATED;
+}
+
+/* creates array of bytes to store raw bytes of an instr into
+ * (original bits are read-only)
+ * initializes array to the original bits!
+ */
+void
+instr_allocate_raw_bits(dcontext_t *dcontext, instr_t *instr, uint num_bytes)
+{
+    byte *original_bits = NULL;
+    if ((instr->flags & INSTR_RAW_BITS_VALID) != 0)
+        original_bits = instr->bytes;
+    if ((instr->flags & INSTR_RAW_BITS_ALLOCATED) == 0 ||
+        instr->length != num_bytes) {
+        byte * new_bits = (byte *) heap_alloc(dcontext, num_bytes HEAPACCT(ACCT_IR));
+        if (original_bits != NULL) {
+            /* copy original bits into modified bits so can just modify
+             * a few and still have all info in one place
+             */
+            memcpy(new_bits, original_bits,
+                   (num_bytes < instr->length) ? num_bytes : instr->length);
+        }
+        if ((instr->flags & INSTR_RAW_BITS_ALLOCATED) != 0)
+            instr_free_raw_bits(dcontext, instr);
+        instr->bytes = new_bits;
+        instr->length = num_bytes;
+    }
+    /* assume that the bits are now valid and the operands are not */
+    instr->flags |= INSTR_RAW_BITS_VALID;
+    instr->flags |= INSTR_RAW_BITS_ALLOCATED;
+    instr->flags &= ~INSTR_OPERANDS_VALID;
+    instr->flags &= ~INSTR_EFLAGS_VALID;
+#ifdef X64
+    instr_set_rip_rel_valid(instr, false); /* relies on original raw bits */
+#endif
+}
+
+instr_t * 
+instr_set_translation(instr_t *instr, app_pc addr)
+{
+#ifdef WINDOWS
+    /* The first jump (to the trampoline) in a landing pad is never interpreted
+     * (see must_not_be_elided(), so we come here only for the second jump
+     * (back to the instruction after the hook, which is always 32-bit.  For
+     * this we set the translation address to the same as the instruction after
+     * the hook, so we get the jump target and use it. */
+    if (vmvector_overlap(landing_pad_areas, addr, addr + 1)) {
+        
+        CLIENT_ASSERT(*addr == JMP_REL32_OPCODE,
+                      "eliding wrong jmp in the landing pad");
+        addr = (addr + 5) + *(int *)(addr + 1); /* end of jump + rel addr */
+        CLIENT_ASSERT(!is_in_interception_buffer(addr),
+                      "eliding wrong jmp in the landing pad");
+    }
+
+    if (is_in_interception_buffer(addr))
+        addr = get_app_pc_from_intercept_pc(addr);
+#endif
+    instr->translation = addr;
+    return instr;
+}
+
+app_pc
+instr_get_translation(instr_t *instr)
+{
+    return instr->translation;
+}
+
+/* This makes it safe to keep an instr around indefinitely when an instrs raw
+ * bits point into the cache. It allocates memory local to the instr to hold
+ * a copy of the raw bits. If this was not done the original raw bits could 
+ * be deleted at some point.  This is neccesary if you want to keep an instr
+ * around for a long time (for clients, beyond returning from the call that 
+ * gave you the instr)
+ */
+void
+instr_make_persistent(dcontext_t *dcontext, instr_t *instr)
+{
+    if ((instr->flags & INSTR_RAW_BITS_VALID) != 0 &&
+        (instr->flags & INSTR_RAW_BITS_ALLOCATED) == 0) {
+        instr_allocate_raw_bits(dcontext, instr, instr->length);
+    }
+}
+
+byte *
+instr_get_raw_bits(instr_t *instr)
+{
+    return instr->bytes;
+}
+
+/* returns the pos-th instr byte */
+byte
+instr_get_raw_byte(instr_t *instr, uint pos)
+{
+    CLIENT_ASSERT(pos >= 0 && pos < instr->length && instr->bytes != NULL,
+                  "instr_get_raw_byte: ordinal invalid, or no raw bits");
+    return instr->bytes[pos];
+}
+
+/* returns the 4 bytes starting at position pos */
+uint 
+instr_get_raw_word(instr_t *instr, uint pos)
+{
+    CLIENT_ASSERT(pos >= 0 && pos+3 < instr->length && instr->bytes != NULL,
+                  "instr_get_raw_word: ordinal invalid, or no raw bits");
+    return *((uint *)(instr->bytes + pos));
+}
+
+/* Sets the pos-th instr byte by storing the unsigned
+ * character value in the pos-th slot
+ * Must call instr_allocate_raw_bits before calling this function
+ * (original bits are read-only!)
+ */
+void
+instr_set_raw_byte(instr_t *instr, uint pos, byte val)
+{
+    CLIENT_ASSERT((instr->flags & INSTR_RAW_BITS_ALLOCATED) != 0,
+                  "instr_set_raw_byte: no raw bits");
+    CLIENT_ASSERT(pos >= 0 && pos < instr->length && instr->bytes != NULL,
+                  "instr_set_raw_byte: ordinal invalid, or no raw bits");
+    instr->bytes[pos] = (byte) val;
+#ifdef X64
+    instr_set_rip_rel_valid(instr, false); /* relies on original raw bits */
+#endif
+}
+
+/* Copies num_bytes bytes from start into the mangled bytes
+ * array of instr.
+ * Must call instr_allocate_raw_bits before calling this function.
+ */
+void 
+instr_set_raw_bytes(instr_t *instr, byte *start, uint num_bytes)
+{
+    CLIENT_ASSERT((instr->flags & INSTR_RAW_BITS_ALLOCATED) != 0,
+                  "instr_set_raw_bytes: no raw bits");
+    CLIENT_ASSERT(num_bytes <= instr->length && instr->bytes != NULL,
+                  "instr_set_raw_bytes: ordinal invalid, or no raw bits");
+    memcpy(instr->bytes, start, num_bytes);
+#ifdef X64
+    instr_set_rip_rel_valid(instr, false); /* relies on original raw bits */
+#endif
+}
+
+/* Stores 32-bit value word in positions pos through pos+3 in
+ * modified_bits.
+ * Must call instr_allocate_raw_bits before calling this function.
+ */
+void
+instr_set_raw_word(instr_t *instr, uint pos, uint word)
+{
+    CLIENT_ASSERT((instr->flags & INSTR_RAW_BITS_ALLOCATED) != 0,
+                  "instr_set_raw_word: no raw bits");
+    CLIENT_ASSERT(pos >= 0 && pos+3 < instr->length && instr->bytes != NULL,
+                  "instr_set_raw_word: ordinal invalid, or no raw bits");
+    *((uint *)(instr->bytes+pos)) = word;
+#ifdef X64
+    instr_set_rip_rel_valid(instr, false); /* relies on original raw bits */
+#endif
+}
+
+/* set the note field of instr to value */
+void 
+instr_set_note(instr_t *instr, void *value)
+{
+    instr->note = value;
+}
+
+/* return the note field of instr */
+void *
+instr_get_note(instr_t *instr)
+{
+    return instr->note;
+}
+
+/* return instr->next */
+instr_t*
+instr_get_next(instr_t *instr)
+{
+    return instr->next;
+}
+
+/* return instr->prev */
+instr_t*
+instr_get_prev(instr_t *instr)
+{
+    return instr->prev;
+}
+
+/* set instr->next to next */
+void
+instr_set_next(instr_t *instr, instr_t *next)
+{
+    instr->next = next;
+}
+
+/* set instr->prev to prev */
+void
+instr_set_prev(instr_t *instr, instr_t *prev)
+{
+    instr->prev = prev;
+}
+
+int
+instr_length(dcontext_t *dcontext, instr_t *instr)
+{
+    if (!instr_needs_encoding(instr))
+        return instr->length;
+
+    /* hardcode length for cti */
+    switch (instr_get_opcode(instr)) {
+    case OP_jmp:
+    case OP_call:
+        return 5;
+    case OP_jb: case OP_jnb: case OP_jbe: case OP_jnbe:
+    case OP_jl: case OP_jnl: case OP_jle: case OP_jnle:
+    case OP_jo: case OP_jno: case OP_jp: case OP_jnp:
+    case OP_js: case OP_jns: case OP_jz: case OP_jnz:
+        return 6 + ((TEST(PREFIX_JCC_TAKEN, instr_get_prefixes(instr)) ||
+                     TEST(PREFIX_JCC_NOT_TAKEN, instr_get_prefixes(instr))) ? 1 : 0);
+    case OP_jb_short: case OP_jnb_short: case OP_jbe_short: case OP_jnbe_short:
+    case OP_jl_short: case OP_jnl_short: case OP_jle_short: case OP_jnle_short:
+    case OP_jo_short: case OP_jno_short: case OP_jp_short: case OP_jnp_short:
+    case OP_js_short: case OP_jns_short: case OP_jz_short: case OP_jnz_short:
+        return 2 + ((TEST(PREFIX_JCC_TAKEN, instr_get_prefixes(instr)) ||
+                     TEST(PREFIX_JCC_NOT_TAKEN, instr_get_prefixes(instr))) ? 1 : 0);
+        /* alternative names (e.g., OP_jae_short) are equivalent, 
+         * so don't need to list them */
+    case OP_jmp_short: 
+        return 2;
+    case OP_jecxz:
+    case OP_loop:
+    case OP_loope:
+    case OP_loopne: 
+        if (opnd_get_reg(instr_get_src(instr, 1)) != REG_XCX)
+            return 3; /* need addr prefix */
+        else
+            return 2;
+    case OP_LABEL:
+        return 0;
+    }
+
+    /* else, encode to find length */
+    return private_instr_encode(dcontext, instr, false/*don't need to cache*/);
+}
+
+/***********************************************************************/
+/* decoding routines */
+
+/* If instr is at Level 0 (i.e., a bundled group of instrs as raw bits),
+ * expands instr into a sequence of Level 1 instrs using decode_raw() which
+ * are added in place to ilist.
+ * Returns the replacement of instr, if any expansion is performed
+ * (in which case the old instr is destroyed); otherwise returns
+ * instr unchanged.
+ * If encounters an invalid instr, stops expanding at that instr, and keeps
+ * instr in the ilist pointing to the invalid bits as an invalid instr.
+ */
+instr_t *
+instr_expand(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr)
+{
+    /* Sometimes deleting instr but sometimes not (when return early)
+     * is painful -- so we go to the trouble of re-using instr
+     * for the first expanded instr
+     */
+    instr_t *newinstr, *firstinstr = NULL;
+    int remaining_bytes, cur_inst_len;
+    byte *curbytes, *newbytes;
+    IF_X64(bool old_mode;)
+
+    /* make it easy for iterators: handle NULL
+     * assume that if opcode is valid, is at Level 2, so not a bundle
+     * do not expand meta-instrs -- FIXME: is that the right thing to do?
+     */
+    if (instr == NULL || instr_opcode_valid(instr) || !instr_ok_to_mangle(instr) ||
+        /* if an invalid instr (not just undecoded) do not try to expand */
+        !instr_valid(instr))
+        return instr;
+
+    DOLOG(4, LOG_ALL, { loginst(dcontext, 4, instr, "instr_expand"); });
+
+    /* decode routines use dcontext mode, but we want instr mode */
+    IF_X64(old_mode = set_x86_mode(dcontext, instr_get_x86_mode(instr)));
+
+    /* never have opnds but not opcode */
+    CLIENT_ASSERT(!instr_operands_valid(instr), "instr_expand: opnds are already valid");
+    CLIENT_ASSERT(instr_raw_bits_valid(instr), "instr_expand: raw bits are invalid");
+    curbytes = instr->bytes;
+    if ((uint)decode_sizeof(dcontext, curbytes, NULL _IF_X64(NULL)) == instr->length) {
+        IF_X64(set_x86_mode(dcontext, old_mode));
+        return instr; /* Level 1 */
+    }
+    
+    remaining_bytes = instr->length;
+    while (remaining_bytes > 0) {
+        /* insert every separated instr into list */
+        newinstr = instr_create(dcontext);
+        newbytes = decode_raw(dcontext, curbytes, newinstr);
+        if (newbytes == NULL) {
+            /* invalid instr -- stop expanding, point instr at remaining bytes */
+            instr_set_raw_bits(instr, curbytes, remaining_bytes);
+            instr_set_opcode(instr, OP_INVALID);
+            if (firstinstr == NULL)
+                firstinstr = instr;
+            instr_destroy(dcontext, newinstr);
+            IF_X64(set_x86_mode(dcontext, old_mode));
+            return firstinstr;
+        }
+        DOLOG(4, LOG_ALL, { loginst(dcontext, 4, newinstr, "\tjust expanded into"); });
+
+        /* CAREFUL of what you call here -- don't call anything that
+         * auto-upgrades instr to Level 2, it will fail on Level 0 bundles!
+         */
+
+        if (instr_has_allocated_bits(instr) &&
+            !instr_is_cti_short_rewrite(newinstr, curbytes)) {
+            /* make sure to have our own copy of any allocated bits
+             * before we destroy the original instr
+             */
+            IF_X64(CLIENT_ASSERT(CHECK_TRUNCATE_TYPE_uint(newbytes - curbytes),
+                                 "instr_expand: internal truncation error"));
+            instr_allocate_raw_bits(dcontext, newinstr, (uint)(newbytes - curbytes));
+        }
+        
+        /* special case: for cti_short, do not fully decode the
+         * constituent instructions, leave as a bundle.
+         * the instr will still have operands valid.
+         */
+        if (instr_is_cti_short_rewrite(newinstr, curbytes)) {
+            newbytes = remangle_short_rewrite(dcontext, newinstr, curbytes, 0);
+        } else if (instr_is_cti_short(newinstr)) {
+            /* make sure non-mangled short ctis, which are generated by
+             * us and never left there from app's, are not marked as exit ctis
+             */
+            instr_set_ok_to_mangle(newinstr, false);
+        }
+
+        IF_X64(CLIENT_ASSERT(CHECK_TRUNCATE_TYPE_int(newbytes - curbytes),
+                             "instr_expand: internal truncation error"));
+        cur_inst_len = (int) (newbytes - curbytes);
+        remaining_bytes -= cur_inst_len;
+        curbytes = newbytes;
+        
+        instrlist_preinsert(ilist, instr, newinstr);
+        if (firstinstr == NULL)
+            firstinstr = newinstr;
+    }
+    
+    /* delete original instr from list */
+    instrlist_remove(ilist, instr);
+    instr_destroy(dcontext, instr);
+    
+    CLIENT_ASSERT(firstinstr != NULL, "instr_expand failure");
+    IF_X64(set_x86_mode(dcontext, old_mode));
+    return firstinstr;
+}
+
+bool
+instr_is_level_0(instr_t *instr) 
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    IF_X64(bool old_mode;)
+    /* assume that if opcode is valid, is at Level 2, so not a bundle
+     * do not expand meta-instrs -- FIXME: is that the right to do? */
+    if (instr == NULL || instr_opcode_valid(instr) || !instr_ok_to_mangle(instr) ||
+        /* if an invalid instr (not just undecoded) do not try to expand */
+        !instr_valid(instr))
+        return false;
+
+    /* never have opnds but not opcode */
+    CLIENT_ASSERT(!instr_operands_valid(instr),
+                  "instr_is_level_0: opnds are already valid");
+    CLIENT_ASSERT(instr_raw_bits_valid(instr),
+                  "instr_is_level_0: raw bits are invalid");
+    IF_X64(old_mode = set_x86_mode(dcontext, instr_get_x86_mode(instr)));
+    if ((uint)decode_sizeof(dcontext, instr->bytes, NULL _IF_X64(NULL)) ==
+        instr->length) {
+        IF_X64(set_x86_mode(dcontext, old_mode));
+        return false; /* Level 1 */
+    }
+    IF_X64(set_x86_mode(dcontext, old_mode));
+    return true;
+}
+
+/* If the next instr is at Level 0 (i.e., a bundled group of instrs as raw bits),
+ * expands it into a sequence of Level 1 instrs using decode_raw() which
+ * are added in place to ilist.  Then returns the new next instr.
+ */
+instr_t *
+instr_get_next_expanded(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr)
+{
+    instr_expand(dcontext, ilist, instr_get_next(instr));
+    return instr_get_next(instr);
+}
+
+/* If the prev instr is at Level 0 (i.e., a bundled group of instrs as raw bits),
+ * expands it into a sequence of Level 1 instrs using decode_raw() which
+ * are added in place to ilist.  Then returns the new prev instr.
+ */
+instr_t *
+instr_get_prev_expanded(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr)
+{
+    instr_expand(dcontext, ilist, instr_get_prev(instr));
+    return instr_get_prev(instr);
+}
+
+/* If the first instr is at Level 0 (i.e., a bundled group of instrs as raw bits),
+ * expands it into a sequence of Level 1 instrs using decode_raw() which
+ * are added in place to ilist.  Then returns the new first instr.
+ */
+instr_t *
+instrlist_first_expanded(dcontext_t *dcontext, instrlist_t *ilist)
+{
+    instr_expand(dcontext, ilist, instrlist_first(ilist));
+    return instrlist_first(ilist);
+}
+
+
+/* If the last instr is at Level 0 (i.e., a bundled group of instrs as raw bits),
+ * expands it into a sequence of Level 1 instrs using decode_raw() which
+ * are added in place to ilist.  Then returns the new last instr.
+ */
+instr_t *
+instrlist_last_expanded(dcontext_t *dcontext, instrlist_t *ilist)
+{
+    instr_expand(dcontext, ilist, instrlist_last(ilist));
+    return instrlist_last(ilist);
+}
+
+/* If instr is not already at the level of decode_cti, decodes enough
+ * from the raw bits pointed to by instr to bring it to that level.
+ * Assumes that instr is a single instr (i.e., NOT Level 0).
+ *
+ * decode_cti decodes only enough of instr to determine
+ * its size, its effects on the 6 arithmetic eflags, and whether it is
+ * a control-transfer instruction.  If it is, the operands fields of
+ * instr are filled in.  If not, only the raw bits fields of instr are
+ * filled in.  This corresponds to a Level 3 decoding for control
+ * transfer instructions but a Level 1 decoding plus arithmetic eflags
+ * information for all other instructions.
+ */
+void
+instr_decode_cti(dcontext_t *dcontext, instr_t *instr)
+{
+    /* if arith flags are missing but otherwise decoded, who cares,
+     * next get_arith_flags() will fill it in
+     */
+    if (!instr_opcode_valid(instr) ||
+        (instr_is_cti(instr) && !instr_operands_valid(instr))) {
+        byte *next_pc;
+        /* decode_cti() will use the dcontext mode, but we want the instr mode */
+        IF_X64(bool old_mode = set_x86_mode(dcontext, instr_get_x86_mode(instr));)
+        DEBUG_DECLARE(int old_len = instr->length;)
+        CLIENT_ASSERT(instr_raw_bits_valid(instr),
+                      "instr_decode_cti: raw bits are invalid");
+        instr_reuse(dcontext, instr);
+        next_pc = decode_cti(dcontext, instr->bytes, instr);
+        IF_X64(set_x86_mode(dcontext, old_mode));
+        /* ok to be invalid, let caller deal with it */
+        CLIENT_ASSERT(next_pc == NULL || (next_pc - instr->bytes == old_len),
+                      "instr_decode_cti requires a Level 1 or higher instruction");
+    }
+}
+
+/* If instr is not already at the level of decode_opcode, decodes enough
+ * from the raw bits pointed to by instr to bring it to that level.
+ * Assumes that instr is a single instr (i.e., NOT Level 0).
+ *
+ * decode_opcode decodes the opcode and eflags usage of the instruction.
+ * This corresponds to a Level 2 decoding.
+ */
+void
+instr_decode_opcode(dcontext_t *dcontext, instr_t *instr)
+{
+    if (!instr_opcode_valid(instr)) {
+        byte *next_pc;
+#ifdef X64
+        bool rip_rel_valid = instr_rip_rel_valid(instr);
+        /* decode_opcode() will use the dcontext mode, but we want the instr mode */
+        bool old_mode = set_x86_mode(dcontext, instr_get_x86_mode(instr));
+#endif
+        DEBUG_DECLARE(int old_len = instr->length;)
+        CLIENT_ASSERT(instr_raw_bits_valid(instr),
+                      "instr_decode_opcode: raw bits are invalid");
+        instr_reuse(dcontext, instr);
+        next_pc = decode_opcode(dcontext, instr->bytes, instr);
+#ifdef X64
+        set_x86_mode(dcontext, old_mode);
+        /* decode_opcode sets raw bits which invalidates rip_rel, but
+         * it should still be valid on an up-decode of the opcode */
+        if (rip_rel_valid)
+            instr_set_rip_rel_pos(instr, instr->rip_rel_pos);
+#endif
+        /* ok to be invalid, let caller deal with it */
+        CLIENT_ASSERT(next_pc == NULL || (next_pc - instr->bytes == old_len),
+                      "instr_decode_opcode requires a Level 1 or higher instruction");
+    }
+}
+
+/* If instr is not already fully decoded, decodes enough
+ * from the raw bits pointed to by instr to bring it Level 3.
+ * Assumes that instr is a single instr (i.e., NOT Level 0).
+ */
+void
+instr_decode(dcontext_t *dcontext, instr_t *instr)
+{
+    if (!instr_operands_valid(instr)) {
+        byte *next_pc;
+#ifdef X64
+        bool rip_rel_valid = instr_rip_rel_valid(instr);
+        /* decode() will use the current dcontext mode, but we want the instr mode */
+        bool old_mode = set_x86_mode(dcontext, instr_get_x86_mode(instr));
+#endif
+        DEBUG_DECLARE(int old_len = instr->length;)
+        CLIENT_ASSERT(instr_raw_bits_valid(instr), "instr_decode: raw bits are invalid");
+        instr_reuse(dcontext, instr);
+        next_pc = decode(dcontext, instr_get_raw_bits(instr), instr);
+#ifdef X64
+        set_x86_mode(dcontext, old_mode);
+        /* decode sets raw bits which invalidates rip_rel, but
+         * it should still be valid on an up-decode */
+        if (rip_rel_valid)
+            instr_set_rip_rel_pos(instr, instr->rip_rel_pos);
+#endif
+        /* ok to be invalid, let caller deal with it */
+        CLIENT_ASSERT(next_pc == NULL || (next_pc - instr->bytes == old_len),
+                      "instr_decode requires a Level 1 or higher instruction");
+    }
+}
+
+/* Brings all instrs in ilist up to the decode_cti level, and
+ * hooks up intra-ilist cti targets to use instr_t targets, by
+ * matching pc targets to each instruction's raw bits.
+ *
+ * decode_cti decodes only enough of instr to determine
+ * its size, its effects on the 6 arithmetic eflags, and whether it is
+ * a control-transfer instruction.  If it is, the operands fields of
+ * instr are filled in.  If not, only the raw bits fields of instr are
+ * filled in.  This corresponds to a Level 3 decoding for control
+ * transfer instructions but a Level 1 decoding plus arithmetic eflags
+ * information for all other instructions.
+ */
+void 
+instrlist_decode_cti(dcontext_t *dcontext, instrlist_t *ilist)
+{
+    instr_t *instr;
+
+    LOG(THREAD, LOG_ALL, 3, "\ninstrlist_decode_cti\n");
+
+    DOLOG(4, LOG_ALL, {
+        LOG(THREAD, LOG_ALL, 4, "beforehand:\n");
+        instrlist_disassemble(dcontext, 0, ilist, THREAD);
+    });
+
+    /* just use the expanding iterator to get to Level 1, then decode cti */
+    for (instr = instrlist_first_expanded(dcontext, ilist);
+         instr != NULL;
+         instr = instr_get_next_expanded(dcontext, ilist, instr)) {
+        /* if arith flags are missing but otherwise decoded, who cares,
+         * next get_arith_flags() will fill it in
+         */
+        if (!instr_opcode_valid(instr) ||
+            (instr_is_cti(instr) && !instr_operands_valid(instr))) {
+            DOLOG(4, LOG_ALL, { 
+                loginst(dcontext, 4, instr, "instrlist_decode_cti: about to decode");
+            });
+            instr_decode_cti(dcontext, instr);
+            DOLOG(4, LOG_ALL, { loginst(dcontext, 4, instr, "\tjust decoded"); });
+        }
+    }
+
+    /* must fix up intra-ilist cti's to have instr_t targets
+     * assumption: all intra-ilist cti's have been marked as do-not-mangle,
+     * plus all targets have their raw bits already set
+     */
+    for (instr = instrlist_first(ilist); instr != NULL;
+         instr = instr_get_next(instr)) {
+        /* N.B.: if we change exit cti's to have instr_t targets, we have to
+         * change other modules like emit to handle that!
+         * FIXME
+         */
+        if (!instr_is_exit_cti(instr) &&
+            instr_opcode_valid(instr) && /* decode_cti only filled in cti opcodes */
+            instr_is_cti(instr) &&
+            instr_num_srcs(instr) > 0 && opnd_is_near_pc(instr_get_src(instr, 0))) {
+            instr_t *tgt;
+            DOLOG(4, LOG_ALL, {
+                loginst(dcontext, 4, instr, "instrlist_decode_cti: found cti w/ pc target");
+            });
+            for (tgt = instrlist_first(ilist); tgt != NULL; tgt = instr_get_next(tgt)) {
+                DOLOG(4, LOG_ALL, { loginst(dcontext, 4, tgt, "\tchecking"); });
+                LOG(THREAD, LOG_INTERP|LOG_OPTS, 4, "\t\taddress is "PFX"\n",
+                    instr_get_raw_bits(tgt));
+                if (opnd_get_pc(instr_get_target(instr)) == instr_get_raw_bits(tgt)) {
+                    /* cti targets this instr */
+                    app_pc bits = 0;
+                    int len = 0;
+                    if (instr_raw_bits_valid(instr)) {
+                        bits = instr_get_raw_bits(instr);
+                        len = instr_length(dcontext, instr);
+                    }
+                    instr_set_target(instr, opnd_create_instr(tgt));
+                    if (bits != 0)
+                        instr_set_raw_bits(instr, bits, len);
+                    DOLOG(4, LOG_ALL, { loginst(dcontext, 4, tgt, "\tcti targets this"); });
+                    break;
+                }
+            }
+        }
+    }
+
+    DOLOG(4, LOG_ALL, { 
+        LOG(THREAD, LOG_ALL, 4, "afterward:\n");
+        instrlist_disassemble(dcontext, 0, ilist, THREAD);
+    });
+    LOG(THREAD, LOG_ALL, 4, "done with instrlist_decode_cti\n");
+}
+
+/****************************************************************************/
+/* utility routines */
+
+void 
+loginst(dcontext_t *dcontext, uint level, instr_t *instr, char *string)
+{
+    DOLOG(level, LOG_ALL, {
+        LOG(THREAD, LOG_ALL, level, "%s: ", string);
+        instr_disassemble(dcontext,instr,THREAD);
+        LOG(THREAD, LOG_ALL, level,"\n");
+    });
+}
+
+void 
+logopnd(dcontext_t *dcontext, uint level, opnd_t opnd, char *string) 
+{
+    DOLOG(level, LOG_ALL, {
+        LOG(THREAD, LOG_ALL, level, "%s: ", string);
+        opnd_disassemble(dcontext, opnd, THREAD);
+        LOG(THREAD, LOG_ALL, level,"\n");
+    });
+}
+
+
+void
+logtrace(dcontext_t *dcontext, uint level, instrlist_t *trace, char *string)
+{
+    DOLOG(level, LOG_ALL, {
+        instr_t *inst;
+        instr_t *next_inst;
+        LOG(THREAD, LOG_ALL, level, "%s:\n", string);
+        for (inst = instrlist_first(trace); inst != NULL; inst = next_inst) {
+            next_inst = instr_get_next(inst);
+            instr_disassemble(dcontext, inst, THREAD);
+            LOG(THREAD, LOG_ALL, level, "\n");
+        }
+        LOG(THREAD, LOG_ALL, level, "\n");
+    });
+}
+
+/* Shrinks all registers not used as addresses, and all immed int and
+ * address sizes, to 16 bits
+ */
+void
+instr_shrink_to_16_bits(instr_t *instr)
+{
+    int i;
+    opnd_t opnd;
+    const instr_info_t * info;
+    byte optype;
+    CLIENT_ASSERT(instr_operands_valid(instr), "instr_shrink_to_16_bits: invalid opnds");
+    info = get_encoding_info(instr);
+    for (i=0; i<instr_num_dsts(instr); i++) {
+        opnd = instr_get_dst(instr, i);
+        /* some non-memory references vary in size by addr16, not data16:
+         * e.g., the edi/esi inc/dec of string instrs
+         */
+        optype = instr_info_opnd_type(info, false/*dst*/, i);
+        if (!opnd_is_memory_reference(opnd) &&
+            optype != TYPE_VAR_ADDR_XREG &&
+            optype != TYPE_INDIR_VAR_XREG &&
+            optype != TYPE_INDIR_VAR_REG &&
+            optype != TYPE_INDIR_VAR_XIREG) {
+            instr_set_dst(instr, i, opnd_shrink_to_16_bits(opnd));
+        }
+    }
+    for (i=0; i<instr_num_srcs(instr); i++) {
+        opnd = instr_get_src(instr, i);
+        optype = instr_info_opnd_type(info, true/*dst*/, i);
+        if (!opnd_is_memory_reference(opnd) &&
+            optype != TYPE_VAR_ADDR_XREG &&
+            optype != TYPE_INDIR_VAR_XREG &&
+            optype != TYPE_INDIR_VAR_REG &&
+            optype != TYPE_INDIR_VAR_XIREG) {
+            instr_set_src(instr, i, opnd_shrink_to_16_bits(opnd));
+        }
+    }
+}
+
+#ifdef X64
+/* Shrinks all registers, including addresses, and all immed int and
+ * address sizes, to 32 bits 
+ */
+void
+instr_shrink_to_32_bits(instr_t *instr)
+{
+    int i;
+    opnd_t opnd;
+    CLIENT_ASSERT(instr_operands_valid(instr), "instr_shrink_to_32_bits: invalid opnds");
+    for (i=0; i<instr_num_dsts(instr); i++) {
+        opnd = instr_get_dst(instr, i);
+        if (opnd_is_immed_int(opnd)) {
+            CLIENT_ASSERT(opnd_get_immed_int(opnd) <= INT_MAX,
+                          "instr_shrink_to_32_bits: immed int will be truncated");
+            instr_set_dst(instr, i, opnd_shrink_to_32_bits(opnd));
+        }
+    }
+    for (i=0; i<instr_num_srcs(instr); i++) {
+        instr_set_src(instr, i, opnd_shrink_to_32_bits(instr_get_src(instr, i)));
+    }
+}
+#endif
+
+bool
+instr_uses_reg(instr_t *instr, reg_id_t reg)
+{
+    return (instr_reg_in_dst(instr,reg)||instr_reg_in_src(instr,reg));
+}
+
+bool instr_reg_in_dst(instr_t *instr, reg_id_t reg)
+{
+    int i;
+    for (i=0; i<instr_num_dsts(instr); i++)
+        if (opnd_uses_reg(instr_get_dst(instr, i), reg))
+            return true;
+    return false;
+}
+
+bool instr_reg_in_src(instr_t *instr, reg_id_t reg)
+{
+    int i;
+    for (i =0; i<instr_num_srcs(instr); i++)
+        if (opnd_uses_reg(instr_get_src(instr, i), reg))
+            return true;
+    return false;
+}
+
+/* checks regs in dest base-disp but not dest reg */
+bool
+instr_reads_from_reg(instr_t *instr, reg_id_t reg)
+{
+    int i;
+    opnd_t opnd;
+
+    if (instr_reg_in_src(instr, reg))
+        return true;
+
+    for (i=0; i<instr_num_dsts(instr); i++) {
+        opnd = instr_get_dst(instr, i);
+        if (!opnd_is_reg(opnd) && opnd_uses_reg(opnd, reg))
+            return true;
+    }
+    return false;
+}
+
+/* this checks sub-registers */
+bool instr_writes_to_reg(instr_t *instr, reg_id_t reg)
+{
+    int i;
+    opnd_t opnd;
+
+    for (i=0; i<instr_num_dsts(instr); i++) {
+        opnd=instr_get_dst(instr, i);
+        if (opnd_is_reg(opnd)&&(reg_fixer[opnd_get_reg(opnd)]==reg_fixer[reg]))
+            return true;
+    }
+    return false;
+}
+
+/* in this func, it must be the exact same register, not a sub reg. ie. eax!=ax */
+bool instr_writes_to_exact_reg(instr_t *instr, reg_id_t reg)
+{
+    int i;
+    opnd_t opnd;
+
+    for (i=0; i<instr_num_dsts(instr); i++) {
+        opnd=instr_get_dst(instr, i);
+        if (opnd_is_reg(opnd)&&(opnd_get_reg(opnd)==reg))
+            return true;
+    }
+    return false;
+}
+
+bool instr_replace_src_opnd(instr_t *instr, opnd_t old_opnd, opnd_t new_opnd)
+{
+    int srcs,a;
+    
+    srcs=instr_num_srcs(instr);
+    
+    for (a=0;a<srcs;a++) {
+        if (opnd_same(instr_get_src(instr,a),old_opnd)||
+            opnd_same_address(instr_get_src(instr,a),old_opnd)) {
+            instr_set_src(instr,a,new_opnd);
+            return true;
+        }
+    }
+    return false;
+}
+
+
+bool instr_same(instr_t *inst1,instr_t *inst2)
+{
+    int dsts,srcs,a;
+
+    if (instr_get_opcode(inst1)!=instr_get_opcode(inst2))
+        return false;
+
+    if ((srcs=instr_num_srcs(inst1))!=instr_num_srcs(inst2))
+        return false;
+    for (a=0;a<srcs;a++) {
+        if (!opnd_same(instr_get_src(inst1,a),instr_get_src(inst2,a)))
+            return false;
+    }
+
+    if ((dsts=instr_num_dsts(inst1))!=instr_num_dsts(inst2))
+        return false;
+    for (a=0;a<dsts;a++) {
+        if (!opnd_same(instr_get_dst(inst1,a),instr_get_dst(inst2,a)))
+            return false;
+    }
+
+    /* We encode some prefixes in the operands themselves, such that
+     * we shouldn't consider the whole-instr_t flags when considering
+     * equality of Instrs
+     */
+    if ((instr_get_prefixes(inst1) & PREFIX_SIGNIFICANT) !=
+        (instr_get_prefixes(inst2) & PREFIX_SIGNIFICANT))
+        return false;
+
+#ifdef X64
+    if (instr_get_x86_mode(inst1) != instr_get_x86_mode(inst2))
+        return false;
+#endif
+
+    return true;
+}
+
+bool instr_reads_memory(instr_t *instr)
+{
+    int a;
+    opnd_t curop;
+
+    /* lea has a mem_ref source operand, but doesn't actually read */
+    if (instr_get_opcode(instr) == OP_lea)
+        return false;
+
+    for (a=0; a<instr_num_srcs(instr); a++) {
+        curop = instr_get_src(instr,a);
+        if (opnd_is_memory_reference(curop)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+bool instr_writes_memory(instr_t *instr)
+{
+    int a;
+    opnd_t curop;
+    for (a=0; a<instr_num_dsts(instr); a++) {
+        curop = instr_get_dst(instr,a);
+        if (opnd_is_memory_reference(curop)) {
+            return true;
+        }
+    }
+    return false;
+}
+
+#ifdef X64
+/* PR 251479: support general re-relativization.  If INSTR_RIP_REL_VALID is set and
+ * the raw bits are valid, instr->rip_rel_pos is assumed to hold the offset into the
+ * instr of a 32-bit rip-relative displacement, which is used to re-relativize during
+ * encoding.  We only use this for level 1-3 instrs, and we invalidate it if the raw
+ * bits are modified at all.
+ * For caching the encoded bytes of a Level 4 instr, instr_encode() sets
+ * the rip_rel_pos field and flag without setting the raw bits valid:
+ * private_instr_encode() then sets the raw bits, after examing the rip rel flag
+ * by itself.  Thus, we must invalidate the rip rel flag when we invalidate
+ * raw bits: we can't rely just on the raw bits invalidation.
+ * There can only be one rip-relative operand per instruction.
+ */
+bool
+instr_rip_rel_valid(instr_t *instr)
+{
+    return instr_raw_bits_valid(instr) && TEST(INSTR_RIP_REL_VALID, instr->flags);
+}
+
+void
+instr_set_rip_rel_valid(instr_t *instr, bool valid)
+{
+    if (valid)
+        instr->flags |= INSTR_RIP_REL_VALID;
+    else
+        instr->flags &= ~INSTR_RIP_REL_VALID;
+}
+
+uint
+instr_get_rip_rel_pos(instr_t *instr)
+{
+    return instr->rip_rel_pos;
+}
+
+void
+instr_set_rip_rel_pos(instr_t *instr, uint pos)
+{
+    CLIENT_ASSERT_TRUNCATE(instr->rip_rel_pos, byte, pos,
+                           "instr_set_rip_rel_pos: offs must be <= 256");
+    instr->rip_rel_pos = (byte) pos;
+    instr_set_rip_rel_valid(instr, true);
+}
+
+bool
+instr_get_rel_addr_target(instr_t *instr, app_pc *target)
+{
+    int i;
+    opnd_t curop;
+    if (!instr_valid(instr))
+        return false;
+    /* PR 251479: we support rip-rel info in level 1 instrs */
+    if (instr_rip_rel_valid(instr)) {
+        if (instr_get_rip_rel_pos(instr) > 0) {
+            if (target != NULL)
+                *target = instr->bytes + instr->length +
+                    *((int *)(instr->bytes + instr_get_rip_rel_pos(instr)));
+            return true;
+        } else
+            return false;
+    }
+    /* else go to level 3 operands */
+    for (i=0; i<instr_num_dsts(instr); i++) {
+        curop = instr_get_dst(instr, i);
+        if (opnd_is_rel_addr(curop)) {
+            if (target != NULL)
+                *target = opnd_get_addr(curop);
+            return true;
+        }
+    }
+    for (i=0; i<instr_num_srcs(instr); i++) {
+        curop = instr_get_src(instr, i);
+        if (opnd_is_rel_addr(curop)) {
+            if (target != NULL)
+                *target = opnd_get_addr(curop);
+            return true;
+        }
+    }
+    return false;
+}
+
+bool
+instr_has_rel_addr_reference(instr_t *instr)
+{
+    return instr_get_rel_addr_target(instr, NULL);
+}
+
+int
+instr_get_rel_addr_dst_idx(instr_t *instr)
+{
+    int i;
+    opnd_t curop;
+    if (!instr_valid(instr))
+        return -1;
+    /* must go to level 3 operands */
+    for (i=0; i<instr_num_dsts(instr); i++) {
+        curop = instr_get_dst(instr, i);
+        if (opnd_is_rel_addr(curop))
+            return i;
+    }
+    return -1;
+}
+
+int
+instr_get_rel_addr_src_idx(instr_t *instr)
+{
+    int i;
+    opnd_t curop;
+    if (!instr_valid(instr))
+        return -1;
+    /* must go to level 3 operands */
+    for (i=0; i<instr_num_srcs(instr); i++) {
+        curop = instr_get_src(instr, i);
+        if (opnd_is_rel_addr(curop))
+            return i;
+    }
+    return -1;
+}
+#endif /* X64 */
+
+bool
+instr_is_our_mangling(instr_t *instr)
+{
+    return TEST(INSTR_OUR_MANGLING, instr->flags);
+}
+
+void
+instr_set_our_mangling(instr_t *instr, bool ours)
+{
+    if (ours)
+        instr->flags |= INSTR_OUR_MANGLING;
+    else
+        instr->flags &= ~INSTR_OUR_MANGLING;
+}
+
+DR_API
+/* Returns NULL if none of instr's operands is a memory reference.
+ * Otherwise, returns the effective address of the first memory operand
+ * when the operands are considered in this order: destinations and then
+ * sources.  The address is computed using the passed-in registers.
+ */
+app_pc
+instr_compute_address(instr_t *instr, dr_mcontext_t *mc)
+{
+    /* for string instr, even w/ rep prefix, assume want value at point of
+     * register snapshot passed in
+     */
+    int a;
+    opnd_t curop = {0};
+    bool found = false;
+    for (a=0; a<instr_num_dsts(instr); a++) {
+        curop = instr_get_dst(instr,a);
+        if (opnd_is_memory_reference(curop)) {
+            found = true;
+            break;
+        }
+    }
+    if (!found) {
+        /* lea has a mem_ref source operand, but doesn't actually read */
+        if (instr_get_opcode(instr) == OP_lea)
+            return NULL;
+        for (a=0; a<instr_num_srcs(instr); a++) {
+            curop = instr_get_src(instr,a);
+            if (opnd_is_memory_reference(curop)) {
+                found = true;
+                break;
+            }
+        }
+    }
+    if (!found)
+        return NULL;
+    return opnd_compute_address(curop, mc);
+}
+
+/* Calculates the size, in bytes, of the memory read or write of instr
+ * If instr does not reference memory, or is invalid, returns 0
+ */
+uint
+instr_memory_reference_size(instr_t *instr)
+{
+    int i;
+    if (!instr_valid(instr))
+        return 0;
+    for (i=0; i<instr_num_dsts(instr); i++) {
+        if (opnd_is_memory_reference(instr_get_dst(instr, i))) {
+            return opnd_size_in_bytes(opnd_get_size(instr_get_dst(instr, i)));
+        }
+    }
+    for (i=0; i<instr_num_srcs(instr); i++) {
+        if (opnd_is_memory_reference(instr_get_src(instr, i))) {
+            return opnd_size_in_bytes(opnd_get_size(instr_get_src(instr, i)));
+        }
+    }
+    return 0;
+}
+
+/* Calculates the size, in bytes, of the memory read or write of
+ * the instr at pc.
+ * Returns the pc of the following instr.
+ * If the instr at pc does not reference memory, or is invalid, 
+ * returns NULL.
+ */
+app_pc
+decode_memory_reference_size(dcontext_t *dcontext, app_pc pc, uint *size_in_bytes)
+{
+    app_pc next_pc;
+    instr_t instr;
+    instr_init(dcontext, &instr);
+    next_pc = decode(dcontext, pc, &instr);
+    if (!instr_valid(&instr))
+        return NULL;
+    CLIENT_ASSERT(size_in_bytes != NULL, "decode_memory_reference_size: passed NULL");
+    *size_in_bytes = instr_memory_reference_size(&instr);
+    instr_free(dcontext, &instr);
+    return next_pc;
+}
+
+/* return the branch type of the (branch) inst */
+uint
+instr_branch_type(instr_t *cti_instr)
+{
+    switch (instr_get_opcode(cti_instr)) {
+    case OP_call: case OP_call_far:
+        return LINK_DIRECT|LINK_CALL;     /* unconditional */
+    case OP_jmp_short: case OP_jmp: case OP_jmp_far:
+        return LINK_DIRECT|LINK_JMP;     /* unconditional */
+    case OP_ret: case OP_ret_far: case OP_iret:
+        return LINK_INDIRECT|LINK_RETURN;
+    case OP_jmp_ind: case OP_jmp_far_ind: 
+        return LINK_INDIRECT|LINK_JMP;
+    case OP_call_ind: case OP_call_far_ind:
+        return LINK_INDIRECT|LINK_CALL;
+    case OP_jb_short: case OP_jnb_short: case OP_jbe_short: case OP_jnbe_short:
+    case OP_jl_short: case OP_jnl_short: case OP_jle_short: case OP_jnle_short:
+    case OP_jo_short: case OP_jno_short: case OP_jp_short: case OP_jnp_short:
+    case OP_js_short: case OP_jns_short: case OP_jz_short: case OP_jnz_short:
+        /* alternative names (e.g., OP_jae_short) are equivalent, 
+         * so don't need to list them */
+    case OP_jecxz: case OP_loop: case OP_loope: case OP_loopne: 
+    case OP_jb: case OP_jnb: case OP_jbe: case OP_jnbe:
+    case OP_jl: case OP_jnl: case OP_jle: case OP_jnle:
+    case OP_jo: case OP_jno: case OP_jp: case OP_jnp:
+    case OP_js: case OP_jns: case OP_jz: case OP_jnz:
+        return LINK_DIRECT|LINK_JMP;     /* conditional */
+    default:
+        LOG(THREAD_GET, LOG_ALL, 0, "branch_type: unknown opcode: %d\n",
+            instr_get_opcode(cti_instr));
+        CLIENT_ASSERT(false, "instr_branch_type: unknown opcode");
+    }
+
+    return LINK_INDIRECT;
+}
+
+DR_API
+/* return the taken target pc of the (direct branch) inst */
+app_pc
+instr_get_branch_target_pc(instr_t *cti_instr)
+{
+    CLIENT_ASSERT(opnd_is_pc(instr_get_target(cti_instr)),
+                  "instr_branch_target_pc: target not pc");
+    return opnd_get_pc(instr_get_target(cti_instr));
+}
+
+DR_API
+/* set the taken target pc of the (direct branch) inst */
+void
+instr_set_branch_target_pc(instr_t *cti_instr, app_pc pc)
+{
+    opnd_t op = opnd_create_pc(pc);
+    instr_set_target(cti_instr, op);
+}
+
+/* An exit CTI is a control-transfer instruction whose target
+ * is a pc (and not an instr_t pointer).  This routine assumes
+ * that no other input operands exist in a CTI.
+ * An undecoded instr cannot be an exit cti.
+ * This routine does NOT try to decode an opcode in a Level 1 or Level
+ * 0 routine, and can thus be called on Level 0 routines.  
+ */
+bool
+instr_is_exit_cti(instr_t *instr)
+{
+    if (!instr_operands_valid(instr) || /* implies !opcode_valid */
+        !instr_ok_to_mangle(instr) ||
+        !instr_is_cti(instr) ||
+        instr_is_call(instr) ||
+        instr_is_return(instr))
+        return false;
+    return (instr_num_srcs(instr) > 0 &&
+             /* far pc should only happen for mangle's call to here */
+            opnd_is_pc(instr_get_src(instr, 0)));
+}
+
+bool 
+instr_is_mov(instr_t *instr)
+{
+    int opc = instr_get_opcode(instr);
+    return (opc == OP_mov_st ||
+            opc == OP_mov_ld ||
+            opc == OP_mov_imm ||
+            opc == OP_mov_seg ||
+            opc == OP_mov_priv);
+}
+
+bool 
+instr_is_call(instr_t *instr)
+{
+    int opc = instr_get_opcode(instr);
+    return (opc == OP_call ||
+            opc == OP_call_far ||
+            opc == OP_call_ind ||
+            opc == OP_call_far_ind);
+}
+
+bool 
+instr_is_call_direct(instr_t *instr)
+{
+    int opc = instr_get_opcode(instr);
+    return (opc == OP_call || opc == OP_call_far);
+}
+
+bool 
+instr_is_call_indirect(instr_t *instr)
+{
+    int opc = instr_get_opcode(instr);
+    return (opc == OP_call_ind || opc == OP_call_far_ind);
+}
+
+bool
+instr_is_return(instr_t *instr)
+{
+    int opc = instr_get_opcode(instr);
+    return (opc == OP_ret || opc == OP_ret_far || opc == OP_iret);
+}
+
+/*** WARNING!  The following rely on ordering of opcodes! ***/
+
+bool
+instr_is_cbr(instr_t *instr)      /* conditional branch */
+{
+    int opc = instr_get_opcode(instr);
+    return ((opc >= OP_jo && opc <= OP_jnle) ||
+            (opc >= OP_jo_short && opc <= OP_jnle_short) ||
+            (opc >= OP_loopne && opc <= OP_jecxz));
+}
+
+bool
+instr_is_mbr(instr_t *instr)      /* multi-way branch */
+{
+    int opc = instr_get_opcode(instr);
+    return (opc == OP_jmp_ind ||
+            opc == OP_call_ind ||
+            opc == OP_ret ||
+            opc == OP_jmp_far_ind ||
+            opc == OP_call_far_ind ||
+            opc == OP_ret_far ||
+            opc == OP_iret);
+}
+
+bool
+instr_is_far_cti(instr_t *instr) /* target address has a segment and offset */
+{
+    int opc = instr_get_opcode(instr);
+    return (opc == OP_jmp_far ||
+            opc == OP_call_far ||
+            opc == OP_jmp_far_ind ||
+            opc == OP_call_far_ind ||
+            opc == OP_ret_far ||
+            opc == OP_iret);
+}
+
+bool
+instr_is_far_abs_cti(instr_t *instr)
+{
+    int opc = instr_get_opcode(instr);
+    return (opc == OP_jmp_far || opc == OP_call_far);
+}
+
+bool
+instr_is_ubr(instr_t *instr)      /* unconditional branch */
+{
+    int opc = instr_get_opcode(instr);
+    return (opc == OP_jmp ||
+            opc == OP_jmp_short ||
+            opc == OP_jmp_far);
+}
+
+bool 
+instr_is_cti(instr_t *instr)      /* any control-transfer instruction */
+{
+    return (instr_is_cbr(instr) || instr_is_mbr(instr) || instr_is_ubr(instr) ||
+            instr_is_call(instr));
+}
+
+/* This routine does NOT decode the cti of instr if the raw bits are valid,
+ * since all short ctis have single-byte opcodes and so just grabbing the first
+ * byte can tell if instr is a cti short
+ */
+bool 
+instr_is_cti_short(instr_t *instr)
+{
+    int opc;
+    if (instr_opcode_valid(instr)) /* 1st choice: set opcode */
+        opc = instr_get_opcode(instr);
+    else if (instr_raw_bits_valid(instr)) { /* 2nd choice: 1st byte */
+        /* get raw opcode
+         * FIXME: figure out which callers really rely on us not
+         * up-decoding here -- if nobody then just do the
+         * instr_get_opcode() and get rid of all this
+         */
+        opc = (int) *(instr_get_raw_bits(instr));
+        return (opc == RAW_OPCODE_jmp_short ||
+                (opc >= RAW_OPCODE_jcc_short_start &&
+                 opc <= RAW_OPCODE_jcc_short_end) ||
+                (opc >= RAW_OPCODE_loop_start && opc <= RAW_OPCODE_loop_end));
+    } else /* ok, fine, decode opcode */
+        opc = instr_get_opcode(instr);
+    return (opc == OP_jmp_short ||
+            (opc >= OP_jo_short && opc <= OP_jnle_short) ||
+            (opc >= OP_loopne && opc <= OP_jecxz));
+}
+
+bool 
+instr_is_cti_loop(instr_t *instr)
+{
+    int opc = instr_get_opcode(instr);
+    /* only looking for loop* and jecxz */
+    return (opc >= OP_loopne && opc <= OP_jecxz);
+}
+
+/* Checks whether instr is a jecxz/loop* that was originally an app instruction.
+ * All such app instructions are mangled into a jecxz/loop*,jmp_short,jmp sequence.
+ * If pc != NULL, pc is expected to point the the beginning of the encoding of
+ * instr, and the following instructions are assumed to be encoded in sequence
+ * after instr.
+ * Otherwise, the encoding is expected to be found in instr's allocated bits.
+ * This routine does NOT decode instr to the opcode level.
+ */
+bool
+instr_is_cti_short_rewrite(instr_t *instr, byte *pc)
+{
+    /* ASSUMPTION: all app jecxz/loop* are converted to the pattern
+     * (jecxz/loop*,jmp_short,jmp), and all jecxz/loop* generated by DynamoRIO
+     * DO NOT MATCH THAT PATTERN.
+     *
+     * For clients, I belive we're robust in the presence of a client adding a
+     * pattern that matches ours exactly: decode_fragment() won't think it's an
+     * exit cti if it's in a fine-grained fragment where we have Linkstubs.  Since
+     * bb building marks as non-coarse if a client adds any cti at all (meta or
+     * not), we're protected there.  The other uses of remangle are in perscache,
+     * which is only for coarse once again (coarse in general has a hard time
+     * finding exit ctis: case 8711/PR 213146), and instr_expand(), which shouldn't
+     * be used in the presence of clients w/ bb hooks.
+     * Note that we now help clients make jecxz/loop transformations that look
+     * just like ours: instr_convert_short_meta_jmp_to_long() (PR 266292).
+     */
+    if (pc == NULL) {
+        if (!instr_has_allocated_bits(instr) || instr->length != CTI_SHORT_REWRITE_LENGTH)
+            return false;
+        pc = instr_get_raw_bits(instr);
+    }
+    if (instr_opcode_valid(instr)) {
+        int opc = instr_get_opcode(instr);
+        if (opc < OP_loopne || opc > OP_jecxz)
+            return false;
+    }
+    else {
+        /* don't require decoding to opcode level */
+        int raw_opc = (int) *(pc);
+        if (raw_opc < RAW_OPCODE_loop_start || raw_opc > RAW_OPCODE_loop_end)
+            return false;
+    }
+    /* now check remaining undecoded bytes */
+    if (*(pc+2) != decode_first_opcode_byte(OP_jmp_short))
+        return false;
+    if (*(pc+4) != decode_first_opcode_byte(OP_jmp))
+        return false;
+    return true;
+}
+
+
+bool
+instr_is_interrupt(instr_t *instr)
+{
+    int opc = instr_get_opcode(instr);
+    return (opc == OP_int || opc == OP_int3 || opc == OP_into);
+}
+
+int
+instr_get_interrupt_number(instr_t *instr)
+{
+    CLIENT_ASSERT(instr_get_opcode(instr) == OP_int,
+                  "instr_get_interrupt_number: instr not interrupt");
+    if (instr_operands_valid(instr))
+        return (int) opnd_get_immed_int(instr_get_src(instr, 0));
+    else if (instr_raw_bits_valid(instr))
+        return instr_get_raw_byte(instr, 1);
+    else {
+        CLIENT_ASSERT(false, "instr_get_interrupt_number: invalid instr");
+        return 0;
+    }
+}
+
+bool
+instr_is_syscall(instr_t *instr)
+{
+    int opc = instr_get_opcode(instr);
+    /* FIXME: Intel processors treat "syscall" as invalid in 32-bit mode;
+     * do we need to treat it specially? */
+    if (opc == OP_sysenter || opc == OP_syscall)
+        return true;
+    if (opc == OP_int) {
+        int num = instr_get_interrupt_number(instr);
+#ifdef WINDOWS
+        return ((byte) num == 0x2e);
+#else
+        return ((byte) num == 0x80);
+#endif
+    }
+#ifdef WINDOWS
+    /* PR 240258 (WOW64): consider this a syscall */
+    if (instr_is_wow64_syscall(instr))
+        return true;
+#endif
+    return false;
+}
+
+/* looks for mov_imm and mov_st and xor w/ src==dst,
+ * returns the constant they set their dst to
+ */
+bool
+instr_is_mov_constant(instr_t *instr, ptr_int_t *value)
+{
+    int opc = instr_get_opcode(instr);
+    if (opc == OP_xor) {
+        if (opnd_same(instr_get_src(instr, 0), instr_get_dst(instr, 0))) {
+            *value = 0;
+            return true;
+        } else
+            return false;
+    } else if (opc == OP_mov_imm || opc == OP_mov_st) {
+        opnd_t op = instr_get_src(instr, 0);
+        if (opnd_is_immed_int(op)) {
+            *value = opnd_get_immed_int(op);
+            return true;
+        } else
+            return false;
+    }
+    return false;
+}
+
+bool instr_is_prefetch(instr_t *instr)
+{
+    int opcode=instr_get_opcode(instr);
+
+    if ((opcode==OP_prefetchnta)||
+        (opcode==OP_prefetcht0)||
+        (opcode==OP_prefetcht1)||
+        (opcode==OP_prefetcht2))
+        return true;
+
+    return false;
+}
+
+bool 
+instr_is_floating(instr_t *instr)
+{
+    int op = instr_get_opcode(instr);
+    /* WARNING -- assumes things about order of OP_ constants */
+    if (op >= OP_fadd && op <= OP_fcomip)
+        return true;
+    return false;
+}
+
+bool
+opcode_is_mmx(int op)
+{
+    /* WARNING -- assumes things about order of OP_ constants */
+    return ((op >= OP_punpcklbw && op <= OP_packssdw) || /* both */
+            (op >= OP_movd && op <= OP_movq) || /* both */
+            op == OP_pshufw || /* mmx */
+            (op >= OP_pcmpeqb && op <= OP_pcmpeqd) || /* both */
+            op == OP_emms || /* mmx */
+            (op >= OP_pinsrw && op <= OP_pmulhw && op != OP_bswap) || /* both */
+            (op >= OP_psubsb && op <= OP_psadbw) || /* both */
+            (op >= OP_psubb && op <= OP_paddd) || /* both */
+            op == OP_fxsave || op == OP_fxrstor); /* both */
+}
+
+bool 
+opcode_is_sse_or_sse2(int op)
+{
+    /* WARNING -- assumes things about order of OP_ constants */
+    return (op == OP_movntps || op == OP_movntpd || /* sse */
+            (op >= OP_punpcklbw && op <= OP_packssdw) || /* both */
+            (op >= OP_punpcklqdq && op <= OP_punpckhqdq) || /* sse */
+            (op >= OP_movd && op <= OP_movq) || /* both */
+            (op >= OP_pshufd && op <= OP_pshuflw) || /* sse */
+            (op >= OP_pcmpeqb && op <= OP_pcmpeqd) || /* both */
+            op == OP_movnti || /* sse */
+            (op >= OP_pinsrw && op <= OP_pmulhw && op != OP_bswap) || /* both */
+            op == OP_movntq || op == OP_movntdq || /* sse */
+            (op >= OP_psubsb && op <= OP_psadbw) || /* both */
+            op == OP_maskmovq || /* introduced in sse, operates on mmx */
+            op == OP_maskmovdqu || /* sse */
+            (op >= OP_psubb && op <= OP_paddd) || /* both */
+            (op >= OP_psrldq && op <= OP_pslldq) || /* sse */
+            op == OP_fxsave || op == OP_fxrstor || /* both */
+            (op >= OP_ldmxcsr && op <= OP_prefetcht2) || /* sse */
+            (op >= OP_movups && op <= OP_cvtpd2dq) || /* sse */
+            op == OP_pause); /* sse2 */
+}
+
+bool
+type_is_sse(int type)
+{
+    return (type == TYPE_V || type == TYPE_W || type == TYPE_V_MODRM);
+}
+
+bool 
+instr_is_mmx(instr_t *instr)
+{
+    int op = instr_get_opcode(instr);
+    if (opcode_is_mmx(op)) {
+        if (opcode_is_sse_or_sse2(op)) {
+            const instr_info_t * info;
+            CLIENT_ASSERT(instr_operands_valid(instr), "instr_is_mmx: invalid opnds");
+            info = get_encoding_info(instr);
+            if (type_is_sse(info->dst1_type) || type_is_sse(info->dst2_type) ||
+                type_is_sse(info->src1_type) || type_is_sse(info->src2_type) ||
+                type_is_sse(info->src3_type))
+                return false;
+        }
+        return true;
+    }
+    return false;
+}
+
+bool 
+instr_is_sse_or_sse2(instr_t *instr)
+{
+    int op = instr_get_opcode(instr);
+    if (opcode_is_sse_or_sse2(op)) {
+        if (opcode_is_mmx(op)) {
+        }
+        return true;
+    }
+    return false;
+}
+
+bool
+instr_is_mov_imm_to_tos(instr_t *instr)
+{
+    return instr_opcode_valid(instr) &&
+        instr_get_opcode(instr) == OP_mov_st &&
+        (opnd_is_immed(instr_get_src(instr, 0)) ||
+         opnd_is_near_instr(instr_get_src(instr, 0))) &&
+        opnd_is_near_base_disp(instr_get_dst(instr, 0)) &&
+        opnd_get_base(instr_get_dst(instr, 0)) == REG_ESP &&
+        opnd_get_index(instr_get_dst(instr, 0)) == REG_NULL &&
+        opnd_get_disp(instr_get_dst(instr, 0)) == 0;
+}
+
+/* Returns true iff instr is a label meta-instruction */
+bool 
+instr_is_label(instr_t *instr)
+{
+    return instr_opcode_valid(instr) && instr_get_opcode(instr) == OP_LABEL;
+}
+
+/* Given a cbr, change the opcode (and potentially branch hint
+ * prefixes) to that of the inverted branch condition.
+ */
+void
+instr_invert_cbr(instr_t *instr)
+{
+    int opc = instr_get_opcode(instr);
+    CLIENT_ASSERT(instr_is_cbr(instr), "instr_invert_cbr: instr not a cbr");
+    if (instr_is_cti_short_rewrite(instr, NULL)) {
+        /* these all look like this:
+                     jcxz cx_zero
+                     jmp-short cx_nonzero
+            cx_zero: jmp foo
+            cx_nonzero:
+         */
+        if (instr_get_raw_byte(instr, 1) == 2) {
+            CLIENT_ASSERT(instr_get_raw_byte(instr, 3) == 5,
+                          "instr_invert_cbr: cti_short_rewrite is corrupted");
+            /* swap targets of the short jumps: */
+            instr_set_raw_byte(instr, 1, (byte)7); /* target cx_nonzero */
+            instr_set_raw_byte(instr, 3, (byte)0); /* target next instr, cx_zero */
+            /* with inverted logic we don't need jmp-short but we keep it in
+             * case we get inverted again */
+        } else {
+            /* re-invert */
+            CLIENT_ASSERT(instr_get_raw_byte(instr, 1) == 7 &&
+                          instr_get_raw_byte(instr, 3) == 0,
+                          "instr_invert_cbr: cti_short_rewrite is corrupted");
+            instr_set_raw_byte(instr, 1, (byte)2);
+            instr_set_raw_byte(instr, 3, (byte)5);
+        }
+    } else if (opc >= OP_jo && opc <= OP_jnle) {
+        switch (opc) {
+        case OP_jb:   opc = OP_jnb; break;
+        case OP_jnb:  opc = OP_jb; break;
+        case OP_jbe:  opc = OP_jnbe; break;
+        case OP_jnbe: opc = OP_jbe; break;
+        case OP_jl:   opc = OP_jnl; break;
+        case OP_jnl:  opc = OP_jl; break;
+        case OP_jle:  opc = OP_jnle; break;
+        case OP_jnle: opc = OP_jle; break;
+        case OP_jo:   opc = OP_jno; break;
+        case OP_jno:  opc = OP_jo; break;
+        case OP_jp:   opc = OP_jnp; break;
+        case OP_jnp:  opc = OP_jp; break;
+        case OP_js:   opc = OP_jns; break;
+        case OP_jns:  opc = OP_js; break;
+        case OP_jz:   opc = OP_jnz; break;
+        case OP_jnz:  opc = OP_jz; break;
+        }
+        instr_set_opcode(instr, opc);
+        /* reverse any branch hint */
+        if (TEST(PREFIX_JCC_TAKEN, instr_get_prefixes(instr))) {
+            instr->prefixes &= ~PREFIX_JCC_TAKEN;
+            instr->prefixes |= PREFIX_JCC_NOT_TAKEN;
+        } else if (TEST(PREFIX_JCC_NOT_TAKEN, instr_get_prefixes(instr))) {
+            instr->prefixes &= ~PREFIX_JCC_NOT_TAKEN;
+            instr->prefixes |= PREFIX_JCC_TAKEN;
+        }
+    } else
+        CLIENT_ASSERT(false, "instr_invert_cbr: unknown opcode");
+}
+
+DR_API
+/**
+ * PR 266292:
+ * Assumes that instr is a meta instruction (!instr_ok_to_mangle())
+ * and an instr_is_cti_short() (8-bit reach).  Converts instr's opcode
+ * to a long form (32-bit reach).  If instr's opcode is OP_loop* or
+ * OP_jecxz, converts it to a sequence of multiple instructions (which
+ * is different from instr_is_cti_short_rewrite()).  Each added instruction
+ * is marked !instr_ok_to_mangle().
+ * \note DR automatically converts non-meta short ctis to long form.
+ */
+void 
+instr_convert_short_meta_jmp_to_long(dcontext_t *dcontext, instrlist_t *ilist,
+                                     instr_t *instr)
+{
+    CLIENT_ASSERT(!instr_ok_to_mangle(instr),
+                  "instr_convert_short_meta_jmp_to_long: instr is not meta");
+    CLIENT_ASSERT(instr_is_cti_short(instr),
+                  "instr_convert_short_meta_jmp_to_long: instr is not a short cti");
+    if (instr_ok_to_mangle(instr) || !instr_is_cti_short(instr))
+        return;
+    convert_to_near_rel_meta(dcontext, ilist, instr);
+}
+
+/* Given a machine state, returns whether or not the cbr instr would be taken
+ * if the state is before execution (pre == true) or after (pre == false).
+ */
+bool
+instr_cbr_taken(instr_t *instr, dr_mcontext_t *mcontext, bool pre)
+{
+    CLIENT_ASSERT(instr_is_cbr(instr), "instr_cbr_taken: instr not a cbr");
+    if (instr_is_cti_loop(instr)) {
+        uint opc = instr_get_opcode(instr);
+        switch (opc) {
+        case OP_loop:
+            return (mcontext->xcx != (pre ? 1U : 0U));
+        case OP_loope:
+            return (TEST(EFLAGS_ZF, mcontext->xflags) &&
+                    mcontext->xcx != (pre ? 1U : 0U));
+        case OP_loopne:
+            return (!TEST(EFLAGS_ZF, mcontext->xflags) &&
+                    mcontext->xcx != (pre ? 1U : 0U));
+        case OP_jecxz:
+            return (mcontext->xcx == 0U);
+        default:
+            CLIENT_ASSERT(false, "instr_cbr_taken: unknown opcode");
+            return false;
+        }
+    }
+    return instr_jcc_taken(instr, mcontext->xflags);
+}
+
+/* Given eflags, returns whether or not the conditional branch instr would be taken */
+bool
+instr_jcc_taken(instr_t *instr, reg_t eflags)
+{
+    uint opc = instr_get_opcode(instr);
+    CLIENT_ASSERT(instr_is_cbr(instr) && !instr_is_cti_loop(instr),
+                  "instr_jcc_taken: instr not a non-jecxz/loop-cbr");
+    switch (opc) {
+    case OP_jo: case OP_jo_short:
+        return TEST(EFLAGS_OF, eflags);
+    case OP_jno: case OP_jno_short:
+        return !TEST(EFLAGS_OF, eflags);
+    case OP_jb: case OP_jb_short: 
+        return TEST(EFLAGS_CF, eflags);
+    case OP_jnb: case OP_jnb_short:
+        return !TEST(EFLAGS_CF, eflags);
+    case OP_jz: case OP_jz_short: 
+        return TEST(EFLAGS_ZF, eflags);
+    case OP_jnz: case OP_jnz_short:
+        return !TEST(EFLAGS_ZF, eflags);
+    case OP_jbe: case OP_jbe_short:
+        return TESTANY(EFLAGS_CF | EFLAGS_ZF, eflags);
+    case OP_jnbe: case OP_jnbe_short:
+        return !TESTANY(EFLAGS_CF | EFLAGS_ZF, eflags);
+    case OP_js: case OP_js_short: 
+        return TEST(EFLAGS_SF, eflags);
+    case OP_jns: case OP_jns_short:
+        return !TEST(EFLAGS_SF, eflags);
+    case OP_jp: case OP_jp_short: 
+        return TEST(EFLAGS_PF, eflags);
+    case OP_jnp: case OP_jnp_short:
+        return !TEST(EFLAGS_PF, eflags);
+    case OP_jl: case OP_jl_short: 
+        return (TEST(EFLAGS_SF, eflags) != TEST(EFLAGS_OF, eflags));
+    case OP_jnl: case OP_jnl_short:
+        return (TEST(EFLAGS_SF, eflags) == TEST(EFLAGS_OF, eflags));
+    case OP_jle: case OP_jle_short:
+        return (TEST(EFLAGS_ZF, eflags) ||
+                TEST(EFLAGS_SF, eflags) != TEST(EFLAGS_OF, eflags));
+    case OP_jnle: case OP_jnle_short:
+        return (!TEST(EFLAGS_ZF, eflags) &&
+                TEST(EFLAGS_SF, eflags) == TEST(EFLAGS_OF, eflags));
+    default:
+        CLIENT_ASSERT(false, "instr_jcc_taken: unknown opcode");
+        return false;
+    }
+}
+
+
+bool
+instr_uses_fp_reg(instr_t *instr)
+{
+    int a;
+    opnd_t curop;
+    for (a=0; a<instr_num_dsts(instr); a++) {
+        curop = instr_get_dst(instr,a);
+        if (opnd_is_reg(curop) && reg_is_fp(opnd_get_reg(curop)))
+            return true;
+        else if (opnd_is_memory_reference(curop)) {
+            if (reg_is_fp(opnd_get_base(curop)))
+                return true;
+            else if (reg_is_fp(opnd_get_index(curop)))
+                return true;
+        }
+    }
+    
+    for (a=0; a<instr_num_srcs(instr); a++) {
+        curop = instr_get_src(instr,a);
+        if (opnd_is_reg(curop) && reg_is_fp(opnd_get_reg(curop)))
+            return true;
+        else if (opnd_is_memory_reference(curop)) {
+            if (reg_is_fp(opnd_get_base(curop)))
+                return true;
+            else if (reg_is_fp(opnd_get_index(curop)))
+                return true;
+        }
+    }
+    return false;
+}
+
+bool
+reg_is_gpr(reg_id_t reg)
+{
+    return (reg >= REG_RAX && reg <= REG_DIL);
+}
+
+bool
+reg_is_segment(reg_id_t reg)
+{
+    return (reg >= SEG_ES && reg <= SEG_GS);
+}
+
+bool
+reg_is_xmm(reg_id_t reg)
+{
+    return (reg>=REG_START_XMM && reg<=REG_STOP_XMM);
+}
+
+bool
+reg_is_mmx(reg_id_t reg)
+{
+    return (reg>=REG_START_MMX && reg<=REG_STOP_MMX);
+}
+
+bool
+reg_is_fp(reg_id_t reg)
+{
+    return (reg>=REG_START_FLOAT && reg<=REG_STOP_FLOAT);
+}
+
+/***********************************************************************
+ * instr_t creation routines
+ * To use 16-bit data sizes, must call set_prefix after creating instr
+ * To support this, all relevant registers must be of eAX form!
+ * FIXME: how do that?
+ * will an all-operand replacement work, or do some instrs have some
+ * var-size regs but some const-size also?
+ *
+ * FIXME: what if want eflags or modrm info on constructed instr?!?
+ *
+ * FIXME: pusha/popa/leave/enter/interrupt i_eSP below are left as OPSZ_4_short2
+ * but their size really varies...OPSZ_4_short2 is needed now to satisfy the encoder,
+ * xref case 10541 on giving these their correct size.
+ *
+ * fld pushes onto top of stack, call that writing to ST0 or ST7?
+ * f*p pops the stack -- not modeled at all!
+ * should floating point constants be doubles, not floats?!?
+ *
+ * opcode complaints:
+ * OP_imm vs. OP_st
+ * OP_ret: build routines have to separate ret_imm and ret_far_imm
+ * others, see FIXME's in instr_create.h
+ */
+
+instr_t * 
+instr_create_0dst_0src(dcontext_t *dcontext, int opcode)
+{
+    instr_t *in = instr_build(dcontext, opcode, 0, 0);
+    return in;
+}
+
+instr_t * 
+instr_create_0dst_1src(dcontext_t *dcontext, int opcode, opnd_t src)
+{
+    instr_t *in = instr_build(dcontext, opcode, 0, 1);
+    instr_set_src(in, 0, src);
+    return in;
+}
+
+instr_t * 
+instr_create_0dst_2src(dcontext_t *dcontext, int opcode,
+                       opnd_t src1, opnd_t src2)
+{
+    instr_t *in = instr_build(dcontext, opcode, 0, 2);
+    instr_set_src(in, 0, src1);
+    instr_set_src(in, 1, src2);
+    return in;
+}
+
+instr_t * 
+instr_create_0dst_3src(dcontext_t *dcontext, int opcode,
+                       opnd_t src1, opnd_t src2, opnd_t src3)
+{
+    instr_t *in = instr_build(dcontext, opcode, 0, 3);
+    instr_set_src(in, 0, src1);
+    instr_set_src(in, 1, src2);
+    instr_set_src(in, 2, src3);
+    return in;
+}
+
+instr_t * 
+instr_create_1dst_0src(dcontext_t *dcontext, int opcode, opnd_t dst)
+{
+    instr_t *in = instr_build(dcontext, opcode, 1, 0);
+    instr_set_dst(in, 0, dst);
+    return in;
+}
+
+instr_t * 
+instr_create_1dst_1src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst, opnd_t src)
+{
+    instr_t *in = instr_build(dcontext, opcode, 1, 1);
+    instr_set_dst(in, 0, dst);
+    instr_set_src(in, 0, src);
+    return in;
+}
+
+instr_t * 
+instr_create_1dst_2src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst, opnd_t src1, opnd_t src2)
+{
+    instr_t *in = instr_build(dcontext, opcode, 1, 2);
+    instr_set_dst(in, 0, dst);
+    instr_set_src(in, 0, src1);
+    instr_set_src(in, 1, src2);
+    return in;
+}
+
+instr_t * 
+instr_create_1dst_3src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst, opnd_t src1, opnd_t src2, opnd_t src3)
+{
+    instr_t *in = instr_build(dcontext, opcode, 1, 3);
+    instr_set_dst(in, 0, dst);
+    instr_set_src(in, 0, src1);
+    instr_set_src(in, 1, src2);
+    instr_set_src(in, 2, src3);
+    return in;
+}
+
+instr_t * 
+instr_create_1dst_5src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst, opnd_t src1, opnd_t src2, opnd_t src3,
+                       opnd_t src4, opnd_t src5)
+{
+    instr_t *in = instr_build(dcontext, opcode, 1, 5);
+    instr_set_dst(in, 0, dst);
+    instr_set_src(in, 0, src1);
+    instr_set_src(in, 1, src2);
+    instr_set_src(in, 2, src3);
+    instr_set_src(in, 3, src4);
+    instr_set_src(in, 4, src5);
+    return in;
+}
+
+instr_t * 
+instr_create_2dst_0src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2)
+{
+    instr_t *in = instr_build(dcontext, opcode, 2, 0);
+    instr_set_dst(in, 0, dst1);
+    instr_set_dst(in, 1, dst2);
+    return in;
+}
+
+instr_t * 
+instr_create_2dst_1src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2, opnd_t src)
+{
+    instr_t *in = instr_build(dcontext, opcode, 2, 1);
+    instr_set_dst(in, 0, dst1);
+    instr_set_dst(in, 1, dst2);
+    instr_set_src(in, 0, src);
+    return in;
+}
+
+instr_t * 
+instr_create_2dst_2src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2, opnd_t src1, opnd_t src2)
+{
+    instr_t *in = instr_build(dcontext, opcode, 2, 2);
+    instr_set_dst(in, 0, dst1);
+    instr_set_dst(in, 1, dst2);
+    instr_set_src(in, 0, src1);
+    instr_set_src(in, 1, src2);
+    return in;
+}
+
+instr_t * 
+instr_create_2dst_3src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2, opnd_t src1, opnd_t src2, opnd_t src3)
+{
+    instr_t *in = instr_build(dcontext, opcode, 2, 3);
+    instr_set_dst(in, 0, dst1);
+    instr_set_dst(in, 1, dst2);
+    instr_set_src(in, 0, src1);
+    instr_set_src(in, 1, src2);
+    instr_set_src(in, 2, src3);
+    return in;
+}
+
+instr_t * 
+instr_create_2dst_4src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2,
+                       opnd_t src1, opnd_t src2, opnd_t src3, opnd_t src4)
+{
+    instr_t *in = instr_build(dcontext, opcode, 2, 4);
+    instr_set_dst(in, 0, dst1);
+    instr_set_dst(in, 1, dst2);
+    instr_set_src(in, 0, src1);
+    instr_set_src(in, 1, src2);
+    instr_set_src(in, 2, src3);
+    instr_set_src(in, 3, src4);
+    return in;
+}
+
+instr_t * 
+instr_create_3dst_0src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2, opnd_t dst3)
+{
+    instr_t *in = instr_build(dcontext, opcode, 3, 0);
+    instr_set_dst(in, 0, dst1);
+    instr_set_dst(in, 1, dst2);
+    instr_set_dst(in, 2, dst3);
+    return in;
+}
+
+instr_t * 
+instr_create_3dst_3src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2, opnd_t dst3,
+                       opnd_t src1, opnd_t src2, opnd_t src3)
+{
+    instr_t *in = instr_build(dcontext, opcode, 3, 3);
+    instr_set_dst(in, 0, dst1);
+    instr_set_dst(in, 1, dst2);
+    instr_set_dst(in, 2, dst3);
+    instr_set_src(in, 0, src1);
+    instr_set_src(in, 1, src2);
+    instr_set_src(in, 2, src3);
+    return in;
+}
+
+instr_t * 
+instr_create_3dst_4src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2, opnd_t dst3,
+                       opnd_t src1, opnd_t src2, opnd_t src3, opnd_t src4)
+{
+    instr_t *in = instr_build(dcontext, opcode, 3, 4);
+    instr_set_dst(in, 0, dst1);
+    instr_set_dst(in, 1, dst2);
+    instr_set_dst(in, 2, dst3);
+    instr_set_src(in, 0, src1);
+    instr_set_src(in, 1, src2);
+    instr_set_src(in, 2, src3);
+    instr_set_src(in, 3, src4);
+    return in;
+}
+
+instr_t * 
+instr_create_3dst_5src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2, opnd_t dst3,
+                       opnd_t src1, opnd_t src2, opnd_t src3,
+                       opnd_t src4, opnd_t src5)
+{
+    instr_t *in = instr_build(dcontext, opcode, 3, 5);
+    instr_set_dst(in, 0, dst1);
+    instr_set_dst(in, 1, dst2);
+    instr_set_dst(in, 2, dst3);
+    instr_set_src(in, 0, src1);
+    instr_set_src(in, 1, src2);
+    instr_set_src(in, 2, src3);
+    instr_set_src(in, 3, src4);
+    instr_set_src(in, 4, src5);
+    return in;
+}
+
+instr_t * 
+instr_create_4dst_1src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2, opnd_t dst3, opnd_t dst4,
+                       opnd_t src)
+{
+    instr_t *in = instr_build(dcontext, opcode, 4, 1);
+    instr_set_dst(in, 0, dst1);
+    instr_set_dst(in, 1, dst2);
+    instr_set_dst(in, 2, dst3);
+    instr_set_dst(in, 3, dst4);
+    instr_set_src(in, 0, src);
+    return in;
+}
+
+instr_t * 
+instr_create_4dst_4src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2, opnd_t dst3, opnd_t dst4,
+                       opnd_t src1, opnd_t src2, opnd_t src3, opnd_t src4)
+{
+    instr_t *in = instr_build(dcontext, opcode, 4, 4);
+    instr_set_dst(in, 0, dst1);
+    instr_set_dst(in, 1, dst2);
+    instr_set_dst(in, 2, dst3);
+    instr_set_dst(in, 3, dst4);
+    instr_set_src(in, 0, src1);
+    instr_set_src(in, 1, src2);
+    instr_set_src(in, 2, src3);
+    instr_set_src(in, 3, src4);
+    return in;
+}
+
+instr_t *
+instr_create_popa(dcontext_t *dcontext)
+{
+    instr_t *in = instr_build(dcontext, OP_popa, 8, 2);
+    instr_set_dst(in, 0, opnd_create_reg(REG_ESP));
+    instr_set_dst(in, 1, opnd_create_reg(REG_EAX));
+    instr_set_dst(in, 2, opnd_create_reg(REG_EBX));
+    instr_set_dst(in, 3, opnd_create_reg(REG_ECX));
+    instr_set_dst(in, 4, opnd_create_reg(REG_EDX));
+    instr_set_dst(in, 5, opnd_create_reg(REG_EBP));
+    instr_set_dst(in, 6, opnd_create_reg(REG_ESI));
+    instr_set_dst(in, 7, opnd_create_reg(REG_EDI));
+    instr_set_src(in, 0, opnd_create_reg(REG_ESP));
+    /* FIXME - size is wrong, xref case 10541 */
+    instr_set_src(in, 1, opnd_create_base_disp(REG_ESP, REG_NULL, 0, 0, OPSZ_4_short2));
+    return in;
+}
+
+instr_t *
+instr_create_pusha(dcontext_t *dcontext)
+{
+    instr_t *in = instr_build(dcontext, OP_pusha, 2, 8);
+    instr_set_dst(in, 0, opnd_create_reg(REG_ESP));
+    /* FIXME - size is wrong, xref case 10541 */
+    instr_set_dst(in, 1, opnd_create_base_disp(REG_ESP, REG_NULL, 0, 0, OPSZ_4_short2));
+    instr_set_src(in, 0, opnd_create_reg(REG_ESP));
+    instr_set_src(in, 1, opnd_create_reg(REG_EAX));
+    instr_set_src(in, 2, opnd_create_reg(REG_EBX));
+    instr_set_src(in, 3, opnd_create_reg(REG_ECX));
+    instr_set_src(in, 4, opnd_create_reg(REG_EDX));
+    instr_set_src(in, 5, opnd_create_reg(REG_EBP));
+    instr_set_src(in, 6, opnd_create_reg(REG_ESI));
+    instr_set_src(in, 7, opnd_create_reg(REG_EDI));
+    return in;
+}
+
+/****************************************************************************/
+/* build instructions from raw bits
+ * convention: give them OP_UNDECODED opcodes
+ */
+
+instr_t *
+instr_create_raw_1byte(dcontext_t *dcontext, byte byte1)
+{
+    instr_t *in = instr_build_bits(dcontext, OP_UNDECODED, 1);
+    instr_set_raw_byte(in, 0, byte1);
+    return in;
+}
+
+instr_t *
+instr_create_raw_2bytes(dcontext_t *dcontext, byte byte1,
+                        byte byte2)
+{
+    instr_t *in = instr_build_bits(dcontext, OP_UNDECODED, 2);
+    instr_set_raw_byte(in, 0, byte1);
+    instr_set_raw_byte(in, 1, byte2);
+    return in;
+}
+
+instr_t *
+instr_create_raw_3bytes(dcontext_t *dcontext, byte byte1,
+                        byte byte2, byte byte3)
+{
+    instr_t *in = instr_build_bits(dcontext, OP_UNDECODED, 3);
+    instr_set_raw_byte(in, 0, byte1);
+    instr_set_raw_byte(in, 1, byte2);
+    instr_set_raw_byte(in, 2, byte3);
+    return in;
+}
+
+instr_t *
+instr_create_raw_4bytes(dcontext_t *dcontext, byte byte1,
+                        byte byte2, byte byte3,
+                        byte byte4)
+{
+    instr_t *in = instr_build_bits(dcontext, OP_UNDECODED, 4);
+    instr_set_raw_byte(in, 0, byte1);
+    instr_set_raw_byte(in, 1, byte2);
+    instr_set_raw_byte(in, 2, byte3);
+    instr_set_raw_byte(in, 3, byte4);
+    return in;
+}
+
+instr_t *
+instr_create_raw_5bytes(dcontext_t *dcontext, byte byte1,
+                        byte byte2, byte byte3,
+                        byte byte4, byte byte5)
+{
+    instr_t *in = instr_build_bits(dcontext, OP_UNDECODED, 5);
+    instr_set_raw_byte(in, 0, byte1);
+    instr_set_raw_byte(in, 1, byte2);
+    instr_set_raw_byte(in, 2, byte3);
+    instr_set_raw_byte(in, 3, byte4);
+    instr_set_raw_byte(in, 4, byte5);
+    return in;
+}
+
+instr_t *
+instr_create_raw_6bytes(dcontext_t *dcontext, byte byte1,
+                        byte byte2, byte byte3,
+                        byte byte4, byte byte5,
+                        byte byte6)
+{
+    instr_t *in = instr_build_bits(dcontext, OP_UNDECODED, 6);
+    instr_set_raw_byte(in, 0, byte1);
+    instr_set_raw_byte(in, 1, byte2);
+    instr_set_raw_byte(in, 2, byte3);
+    instr_set_raw_byte(in, 3, byte4);
+    instr_set_raw_byte(in, 4, byte5);
+    instr_set_raw_byte(in, 5, byte6);
+    return in;
+}
+
+instr_t *
+instr_create_raw_7bytes(dcontext_t *dcontext, byte byte1,
+                        byte byte2, byte byte3,
+                        byte byte4, byte byte5,
+                        byte byte6, byte byte7)
+{
+    instr_t *in = instr_build_bits(dcontext, OP_UNDECODED, 7);
+    instr_set_raw_byte(in, 0, byte1);
+    instr_set_raw_byte(in, 1, byte2);
+    instr_set_raw_byte(in, 2, byte3);
+    instr_set_raw_byte(in, 3, byte4);
+    instr_set_raw_byte(in, 4, byte5);
+    instr_set_raw_byte(in, 5, byte6);
+    instr_set_raw_byte(in, 6, byte7);
+    return in;
+}
+
+instr_t *
+instr_create_raw_8bytes(dcontext_t *dcontext, byte byte1,
+                        byte byte2, byte byte3,
+                        byte byte4, byte byte5,
+                        byte byte6, byte byte7,
+                        byte byte8)
+{
+    instr_t *in = instr_build_bits(dcontext, OP_UNDECODED, 8);
+    instr_set_raw_byte(in, 0, byte1);
+    instr_set_raw_byte(in, 1, byte2);
+    instr_set_raw_byte(in, 2, byte3);
+    instr_set_raw_byte(in, 3, byte4);
+    instr_set_raw_byte(in, 4, byte5);
+    instr_set_raw_byte(in, 5, byte6);
+    instr_set_raw_byte(in, 6, byte7);
+    instr_set_raw_byte(in, 7, byte8);
+    return in;
+}
+
+instr_t *
+instr_create_nbyte_nop(dcontext_t *dcontext, uint num_bytes, bool raw)
+{
+    CLIENT_ASSERT(num_bytes != 0, "instr_create_nbyte_nop: 0 bytes passed");
+    CLIENT_ASSERT(num_bytes <= 3, "instr_create_nbyte_nop: > 3 bytes not supported");
+    if (raw) {
+        switch(num_bytes) {
+        case 1 :
+            return INSTR_CREATE_RAW_nop1byte(dcontext);
+        case 2 :
+            return INSTR_CREATE_RAW_nop2byte(dcontext);
+        case 3 :
+            return INSTR_CREATE_RAW_nop3byte(dcontext);
+        }
+    } else {
+        switch(num_bytes) {
+        case 1 :
+            return INSTR_CREATE_nop1byte(dcontext);
+        case 2:
+            return INSTR_CREATE_nop2byte(dcontext);
+        case 3:
+            return INSTR_CREATE_nop3byte(dcontext);
+        }
+    }
+    CLIENT_ASSERT(false, "instr_create_nbyte_nop: invalid parameters");
+    return NULL;
+}
+
+/* Borrowed from optimize.c, prob. belongs here anyways, could make it more
+ * specific to the ones we create above, but know it works as is FIXME */
+/* return true if this instr is a nop, does not check for all types of nops 
+ * since there are many, these seem to be the most common */
+bool 
+instr_is_nop(instr_t *inst)
+{
+    /* FIXME : could check raw bits for 0x90 to avoid the decoding if raw */
+    int opcode = instr_get_opcode(inst);
+    if (opcode == OP_nop)
+        return true;
+    if ((opcode == OP_mov_ld || opcode == OP_mov_st) && 
+        opnd_same(instr_get_src(inst, 0), instr_get_dst(inst, 0))
+        /* for 64-bit, targeting a 32-bit register zeroes the top bits => not a nop! */
+        IF_X64(&& (instr_get_x86_mode(inst) ||
+                   !opnd_is_reg(instr_get_dst(inst, 0)) ||
+                   reg_get_size(opnd_get_reg(instr_get_dst(inst, 0))) != OPSZ_4)))
+        return true;
+    if (opcode == OP_xchg && opnd_same(instr_get_dst(inst, 0), instr_get_dst(inst, 1))
+        /* for 64-bit, targeting a 32-bit register zeroes the top bits => not a nop! */
+        IF_X64(&& (instr_get_x86_mode(inst) ||
+                   opnd_get_size(instr_get_dst(inst, 0)) != OPSZ_4)))
+        return true;
+    if (opcode == OP_lea &&
+        opnd_is_base_disp(instr_get_src(inst, 0)) /* x64: rel, abs aren't base-disp */ &&
+        opnd_get_disp(instr_get_src(inst, 0)) == 0 && 
+        ((opnd_get_base(instr_get_src(inst, 0)) == opnd_get_reg(instr_get_dst(inst, 0)) &&
+          opnd_get_index(instr_get_src(inst, 0)) == REG_NULL) ||
+         (opnd_get_index(instr_get_src(inst, 0)) == opnd_get_reg(instr_get_dst(inst, 0)) && 
+          opnd_get_base(instr_get_src(inst, 0)) == REG_NULL && 
+          opnd_get_scale(instr_get_src(inst, 0)) == 1)))
+        return true;
+    return false;
+}
+
+/****************************************************************************/
+/* dcontext convenience routines */
+
+static opnd_t
+dcontext_opnd_common(dcontext_t *dcontext, bool absolute, reg_id_t basereg,
+                     int offs, opnd_size_t size)
+{
+    IF_X64(ASSERT_NOT_IMPLEMENTED(!absolute));
+    /* offs is not raw offset, but includes upcontext size, so we
+     * can tell unprotected from normal
+     */
+    if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask) &&
+        offs < sizeof(unprotected_context_t)) {
+        return opnd_create_base_disp(absolute ? REG_NULL :
+                                     ((basereg == REG_NULL) ? REG_XSI : basereg),
+                                     REG_NULL, 0,
+                                     ((int)(ptr_int_t)(absolute ?
+                                            dcontext->upcontext.separate_upcontext : 0))
+                                     + offs, size);
+    } else {
+        if (offs >= sizeof(unprotected_context_t))
+            offs -= sizeof(unprotected_context_t);
+        return opnd_create_base_disp(absolute ? REG_NULL :
+                                     ((basereg == REG_NULL) ? REG_XDI : basereg),
+                                     REG_NULL, 0,
+                                     ((int)(ptr_int_t)
+                                      (absolute ? dcontext : 0)) + offs, size);
+    }
+}
+
+opnd_t
+opnd_create_dcontext_field_sz(dcontext_t *dcontext, int offs, opnd_size_t sz)
+{
+    return dcontext_opnd_common(dcontext, true, REG_NULL, offs, sz);
+}
+
+opnd_t
+opnd_create_dcontext_field(dcontext_t *dcontext, int offs)
+{
+    return dcontext_opnd_common(dcontext, true, REG_NULL, offs, OPSZ_PTR);
+}
+
+/* use basereg==REG_NULL to get default (xdi, or xsi for upcontext) */
+opnd_t
+opnd_create_dcontext_field_via_reg_sz(dcontext_t *dcontext, reg_id_t basereg,
+                                      int offs, opnd_size_t sz)
+{
+    return dcontext_opnd_common(dcontext, false, basereg, offs, sz);
+}
+
+/* use basereg==REG_NULL to get default (xdi, or xsi for upcontext) */
+opnd_t
+opnd_create_dcontext_field_via_reg(dcontext_t *dcontext, reg_id_t basereg, int offs)
+{
+    return dcontext_opnd_common(dcontext, false, basereg, offs, OPSZ_PTR);
+}
+
+opnd_t
+opnd_create_dcontext_field_byte(dcontext_t *dcontext, int offs)
+{
+    return dcontext_opnd_common(dcontext, true, REG_NULL, offs, OPSZ_1);
+}
+
+instr_t *
+instr_create_restore_from_dcontext(dcontext_t *dcontext, reg_id_t reg, int offs)
+{
+    opnd_t memopnd = opnd_create_dcontext_field(dcontext, offs);
+    /* use movd for xmm/mmx */
+    if (reg_is_xmm(reg) || reg_is_mmx(reg))
+        return INSTR_CREATE_movd(dcontext, opnd_create_reg(reg), memopnd);
+    else
+        return INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(reg), memopnd);
+}
+
+instr_t *
+instr_create_save_to_dcontext(dcontext_t *dcontext, reg_id_t reg, int offs)
+{
+    opnd_t memopnd = opnd_create_dcontext_field(dcontext, offs);
+    CLIENT_ASSERT(dcontext != GLOBAL_DCONTEXT,
+                  "instr_create_save_to_dcontext: invalid dcontext");
+    /* use movd for xmm/mmx */
+    if (reg_is_xmm(reg) || reg_is_mmx(reg))
+        return INSTR_CREATE_movd(dcontext, memopnd, opnd_create_reg(reg));
+    else
+        return INSTR_CREATE_mov_st(dcontext, memopnd, opnd_create_reg(reg));
+}
+
+/* Use basereg==REG_NULL to get default (xdi, or xsi for upcontext) 
+ * Auto-magically picks the mem opnd size to match reg if it's a GPR.
+ */
+instr_t *
+instr_create_restore_from_dc_via_reg(dcontext_t *dcontext, reg_id_t basereg,
+                                     reg_id_t reg, int offs)
+{
+    /* use movd for xmm/mmx, and OPSZ_PTR */
+    if (reg_is_xmm(reg) || reg_is_mmx(reg)) {
+        opnd_t memopnd = opnd_create_dcontext_field_via_reg(dcontext, basereg, offs);
+        return INSTR_CREATE_movd(dcontext, opnd_create_reg(reg), memopnd);
+    } else {
+        opnd_t memopnd = opnd_create_dcontext_field_via_reg_sz
+            (dcontext, basereg, offs, reg_get_size(reg));
+        return INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(reg), memopnd);
+    }
+}
+
+/* Use basereg==REG_NULL to get default (xdi, or xsi for upcontext) 
+ * Auto-magically picks the mem opnd size to match reg if it's a GPR.
+ */
+instr_t *
+instr_create_save_to_dc_via_reg(dcontext_t *dcontext, reg_id_t basereg,
+                                reg_id_t reg, int offs)
+{
+    /* use movd for xmm/mmx, and OPSZ_PTR */
+    if (reg_is_xmm(reg) || reg_is_mmx(reg)) {
+        opnd_t memopnd = opnd_create_dcontext_field_via_reg(dcontext, basereg, offs);
+        return INSTR_CREATE_movd(dcontext, memopnd, opnd_create_reg(reg));
+    } else {
+        opnd_t memopnd = opnd_create_dcontext_field_via_reg_sz
+            (dcontext, basereg, offs, reg_get_size(reg));
+        return INSTR_CREATE_mov_st(dcontext, memopnd, opnd_create_reg(reg));
+    }
+}
+
+instr_t *
+instr_create_save_immed_to_dcontext(dcontext_t *dcontext, int immed, int offs)
+{
+    opnd_t memopnd = opnd_create_dcontext_field(dcontext, offs);
+    /* PR 244737: thread-private scratch space needs to fixed for x64 */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    return INSTR_CREATE_mov_st(dcontext, memopnd, OPND_CREATE_INT32(immed));
+}
+
+instr_t *
+instr_create_jump_via_dcontext(dcontext_t *dcontext, int offs)
+{
+    opnd_t memopnd = opnd_create_dcontext_field(dcontext, offs);
+    return INSTR_CREATE_jmp_ind(dcontext, memopnd);
+}
+
+/* there is no corresponding save routine since we no longer support
+ * keeping state on the stack while code other than our own is running
+ * (in the same thread)
+ */
+instr_t *
+instr_create_restore_dynamo_stack(dcontext_t *dcontext)
+{
+    return instr_create_restore_from_dcontext(dcontext, REG_ESP, DSTACK_OFFSET);
+}
+
+#ifdef RETURN_STACK
+instr_t *
+instr_create_restore_dynamo_return_stack(dcontext_t *dcontext)
+{
+    return instr_create_restore_from_dcontext(dcontext, REG_ESP,
+                                              TOP_OF_RSTACK_OFFSET);
+}
+
+instr_t *
+instr_create_save_dynamo_return_stack(dcontext_t *dcontext)
+{
+    return instr_create_save_to_dcontext(dcontext, REG_ESP,
+                                         TOP_OF_RSTACK_OFFSET);
+}
+#endif
+
+opnd_t
+update_dcontext_address(opnd_t op, dcontext_t *old_dcontext,
+                        dcontext_t *new_dcontext)
+{
+    int offs;
+    CLIENT_ASSERT(opnd_is_near_base_disp(op) &&
+           opnd_get_base(op) == REG_NULL &&
+           opnd_get_index(op) == REG_NULL, "update_dcontext_address: invalid opnd");
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    offs = opnd_get_disp(op) - (uint)(ptr_uint_t)old_dcontext;
+    if (offs >= 0 && offs < sizeof(dcontext_t)) {
+        /* don't pass raw offset, add in upcontext size */
+        offs += sizeof(unprotected_context_t);
+        return opnd_create_dcontext_field(new_dcontext, offs);
+    }
+    /* some fields are in a separate memory region! */
+    else {
+        CLIENT_ASSERT(TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask),
+                      "update_dcontext_address: inconsistent layout");
+        IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+        offs = opnd_get_disp(op) -
+            (uint)(ptr_uint_t)(old_dcontext->upcontext.separate_upcontext);
+        if (offs >= 0 && offs < sizeof(unprotected_context_t)) {
+            /* raw offs is what we want for upcontext */
+            return opnd_create_dcontext_field(new_dcontext, offs);
+        }
+    }
+    /* not a dcontext offset: just return original value */
+    return op;
+}
+
+opnd_t
+opnd_create_tls_slot(int offs)
+{
+    return opnd_create_sized_tls_slot(offs, OPSZ_PTR);
+}
+
+opnd_t
+opnd_create_sized_tls_slot(int offs, opnd_size_t size)
+{
+    /* We do not request disp_short_addr or force_full_disp, letting
+     * encode_base_disp() choose whether to use the 0x67 addr prefix
+     * (assuming offs is small).
+     */
+    return opnd_create_far_base_disp(SEG_TLS, REG_NULL, REG_NULL, 0, offs, size);
+}
+
+/* make sure to keep in sync w/ emit_utils.c's insert_spill_or_restore() */
+bool
+instr_raw_is_tls_spill(byte *pc, reg_id_t reg, ushort offs)
+{
+    ASSERT_NOT_IMPLEMENTED(reg != REG_XAX);
+#ifdef X64
+    /* match insert_jmp_to_ibl */
+    if     (*pc == TLS_SEG_OPCODE && 
+            *(pc+1) == (REX_PREFIX_BASE_OPCODE | REX_PREFIX_W_OPFLAG) &&
+            *(pc+2) == MOV_REG2MEM_OPCODE &&
+            /* 0x1c for ebx, 0x0c for ecx, 0x04 for eax */
+            *(pc+3) == MODRM_BYTE(0/*mod*/, reg_get_bits(reg), 4/*rm*/) &&
+            *(pc+4) == 0x25 &&
+            *((uint*)(pc+5)) == (uint) os_tls_offset(offs))
+        return true;
+    /* we also check for 32-bit.  we could take in flags and only check for one
+     * version, but we're not worried about false positives.
+     */
+#endif
+    /* looking for:   67 64 89 1e e4 0e    addr16 mov    %ebx -> %fs:0xee4   */
+    /* ASSUMPTION: when addr16 prefix is used, prefix order is fixed */
+    return (*pc == ADDR_PREFIX_OPCODE && 
+            *(pc+1) == TLS_SEG_OPCODE && 
+            *(pc+2) == MOV_REG2MEM_OPCODE &&
+            /* 0x1e for ebx, 0x0e for ecx, 0x06 for eax */
+            *(pc+3) == MODRM_BYTE(0/*mod*/, reg_get_bits(reg), 6/*rm*/) &&
+            *((ushort*)(pc+4)) == os_tls_offset(offs)) ||
+        /* PR 209709: allow for no addr16 prefix */
+        (*pc == TLS_SEG_OPCODE && 
+         *(pc+1) == MOV_REG2MEM_OPCODE &&
+         /* 0x1e for ebx, 0x0e for ecx, 0x06 for eax */
+         *(pc+2) == MODRM_BYTE(0/*mod*/, reg_get_bits(reg), 6/*rm*/) &&
+         *((uint*)(pc+4)) == os_tls_offset(offs));
+}
+
+/* this routine may upgrade a level 1 instr */
+static bool
+instr_check_tls_spill_restore(instr_t *instr, bool *spill, reg_id_t *reg, int *offs)
+{
+    opnd_t regop, memop;
+    CLIENT_ASSERT(instr != NULL,
+                  "internal error: tls spill/restore check: NULL argument");
+    if (instr_get_opcode(instr) == OP_mov_st) {
+        regop = instr_get_src(instr, 0);
+        memop = instr_get_dst(instr, 0);
+        if (spill != NULL)
+            *spill = true;
+    } else if (instr_get_opcode(instr) == OP_mov_ld) {
+        regop = instr_get_dst(instr, 0);
+        memop = instr_get_src(instr, 0);
+        if (spill != NULL)
+            *spill = false;
+    } else if (instr_get_opcode(instr) == OP_xchg) {
+        /* we use xchg to restore in dr_insert_mbr_instrumentation */
+        regop = instr_get_src(instr, 0);
+        memop = instr_get_dst(instr, 0);
+        if (spill != NULL)
+            *spill = false;
+    } else
+        return false;
+    if (opnd_is_far_base_disp(memop) &&
+        opnd_get_segment(memop) == SEG_TLS && 
+        opnd_is_abs_base_disp(memop) &&
+        opnd_is_reg(regop)) {
+        if (reg != NULL)
+            *reg = opnd_get_reg(regop);
+        if (offs != NULL)
+            *offs = opnd_get_disp(memop);
+        return true;
+    }
+    return false;
+}
+
+/* if instr is level 1, does not upgrade it and instead looks at raw bits,
+ * to support identification w/o ruining level 0 in decode_fragment, etc.
+ */
+bool
+instr_is_tls_spill(instr_t *instr, reg_id_t reg, ushort offs)
+{
+    reg_id_t check_reg;
+    int check_disp;
+    bool spill;
+    return (instr_check_tls_spill_restore(instr, &spill, &check_reg, &check_disp) &&
+            spill && check_reg == reg && check_disp == os_tls_offset(offs));
+}
+
+/* if instr is level 1, does not upgrade it and instead looks at raw bits,
+ * to support identification w/o ruining level 0 in decode_fragment, etc.
+ */
+bool
+instr_is_tls_xcx_spill(instr_t *instr)
+{
+    if (instr_raw_bits_valid(instr)) {
+        /* avoid upgrading instr */
+        return instr_raw_is_tls_spill(instr_get_raw_bits(instr),
+                                      REG_ECX, MANGLE_XCX_SPILL_SLOT);
+    } else
+        return instr_is_tls_spill(instr, REG_ECX, MANGLE_XCX_SPILL_SLOT);
+}
+
+/* this routine may upgrade a level 1 instr */
+static bool
+instr_check_mcontext_spill_restore(dcontext_t *dcontext, instr_t *instr,
+                                   bool *spill, reg_id_t *reg, int *offs)
+{
+#ifdef X64
+    /* PR 244737: we always use tls for x64 */
+    return false;
+#else
+    opnd_t regop, memop;
+    if (instr_get_opcode(instr) == OP_mov_st) {
+        regop = instr_get_src(instr, 0);
+        memop = instr_get_dst(instr, 0);
+        if (spill != NULL)
+            *spill = true;
+    } else if (instr_get_opcode(instr) == OP_mov_ld) {
+        regop = instr_get_dst(instr, 0);
+        memop = instr_get_src(instr, 0);
+        if (spill != NULL)
+            *spill = false;
+    } else if (instr_get_opcode(instr) == OP_xchg) {
+        /* we use xchg to restore in dr_insert_mbr_instrumentation */
+        regop = instr_get_src(instr, 0);
+        memop = instr_get_dst(instr, 0);
+        if (spill != NULL)
+            *spill = false;
+    } else
+        return false;
+    if (opnd_is_near_base_disp(memop) &&
+        opnd_is_abs_base_disp(memop) &&
+        opnd_is_reg(regop)) {
+        byte *pc = (byte *) opnd_get_disp(memop);
+        byte *mc = (byte *) get_mcontext(dcontext);
+        if (pc >= mc && pc < mc + sizeof(dr_mcontext_t)) {
+            if (reg != NULL)
+                *reg = opnd_get_reg(regop);
+            if (offs != NULL)
+                *offs = pc - (byte *)dcontext;
+            return true;
+        }
+    }
+    return false;
+#endif
+}
+
+bool
+instr_is_reg_spill_or_restore(dcontext_t *dcontext, instr_t *instr,
+                              bool *tls, bool *spill, reg_id_t *reg)
+{
+    int check_disp;
+    reg_id_t myreg;
+    CLIENT_ASSERT(instr != NULL, "internal error: NULL argument");
+    if (reg == NULL)
+        reg = &myreg;
+    if (instr_check_tls_spill_restore(instr, spill, reg, &check_disp)) {
+        int offs = reg_spill_tls_offs(*reg);
+        if (offs != -1 && check_disp == os_tls_offset((ushort)offs)) {
+            if (tls != NULL)
+                *tls = true;
+            return true;
+        }
+    }
+    if (dcontext != GLOBAL_DCONTEXT &&
+        instr_check_mcontext_spill_restore(dcontext, instr, spill,
+                                           reg, &check_disp)) {
+        int offs = opnd_get_reg_dcontext_offs(reg_fixer[*reg]);
+        if (offs != -1 && check_disp == offs) {
+            if (tls != NULL)
+                *tls = false;
+            return true;
+        }
+    }
+    return false;
+}
+
+/* N.B. : client meta routines (dr_insert_* etc.) should never use anything other
+ * then TLS_XAX_SLOT unless the client has specified a slot to use as we let the
+ * client use the rest. */
+instr_t *
+instr_create_save_to_tls(dcontext_t *dcontext, reg_id_t reg, ushort offs)
+{
+    return INSTR_CREATE_mov_st(dcontext, opnd_create_tls_slot(os_tls_offset(offs)), 
+                               opnd_create_reg(reg));
+}
+
+instr_t *
+instr_create_restore_from_tls(dcontext_t *dcontext, reg_id_t reg, ushort offs)
+{
+    return INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(reg), 
+                               opnd_create_tls_slot(os_tls_offset(offs)));
+}
+
+#ifdef X64
+/* Returns NULL if pc is not the start of a rip-rel lea.
+ * If it could be, returns the address it refers to (which we assume is
+ * never NULL).
+ */
+byte *
+instr_raw_is_rip_rel_lea(byte *pc, byte *read_end)
+{
+    /* PR 215408: look for "lea reg, [rip+disp]"
+     * We assume no extraneous prefixes, and we require rex.w, though not strictly
+     * necessary for say WOW64 or other known-lower-4GB situations
+     */
+    if (pc + 7 <= read_end) {
+        if (*(pc+1) == RAW_OPCODE_lea &&
+            (TESTALL(REX_PREFIX_BASE_OPCODE | REX_PREFIX_W_OPFLAG, *pc) &&
+             !TESTANY(~(REX_PREFIX_BASE_OPCODE | REX_PREFIX_ALL_OPFLAGS), *pc)) &&
+            /* does mod==0 and rm==5? */
+            ((*(pc+2)) | MODRM_BYTE(0,7,0)) == MODRM_BYTE(0,7,5)) {
+            return pc + 7 + *(int*)(pc+3);
+        }
+    }
+    return NULL;
+}
+#endif
+
+#ifdef WINDOWS
+bool
+instr_is_wow64_syscall(instr_t *instr)
+{
+    opnd_t tgt;
+    if (!is_wow64_process(NT_CURRENT_PROCESS) ||
+        instr_get_opcode(instr) != OP_call_ind)
+        return false;
+    CLIENT_ASSERT(get_syscall_method() == SYSCALL_METHOD_WOW64,
+                  "wow64 system call inconsistency");
+    tgt = instr_get_target(instr);
+    return (opnd_is_far_base_disp(tgt) &&
+            opnd_get_segment(tgt) == SEG_FS &&
+            opnd_get_base(tgt) == REG_NULL &&
+            opnd_get_index(tgt) == REG_NULL &&
+            opnd_get_disp(tgt) == WOW64_TIB_OFFSET);
+}
+#endif
diff --git a/core/x86/instr.h b/core/x86/instr.h
new file mode 100644
index 000000000..6c8a157a0
--- /dev/null
+++ b/core/x86/instr.h
@@ -0,0 +1,3650 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/* file "instr.h" -- x86-specific instr_t definitions and utilities */
+
+#ifndef _INSTR_H_
+#define _INSTR_H_ 1
+
+#ifdef WINDOWS
+/* disabled warning for
+ *   "nonstandard extension used : bit field types other than int"
+ * so we can use bitfields on our now-byte-sized reg_id_t type in opnd_t.
+ */
+# pragma warning( disable : 4214)
+#endif
+
+/* can't include decode.h, it includes us, just declare struct */
+struct instr_info_t;
+
+/* The machine-specific IR consists of instruction lists,
+   instructions, and operands.  You can find the instrlist_t stuff in
+   the upper-level directory (shared infrastructure).  The
+   declarations and interface functions (which insulate the system
+   from the specifics of each constructs implementation) for opnd_t and
+   instr_t appear below. */
+
+
+/*************************
+ ***       opnd_t        ***
+ *************************/
+
+/* DR_API EXPORT TOFILE dr_ir_opnd.h */
+/* DR_API EXPORT BEGIN */
+/****************************************************************************
+ * OPERAND ROUTINES
+ */
+/**
+ * @file dr_ir_opnd.h
+ * @brief Functions and defines to create and manipulate instruction operands.
+ */
+
+#ifdef AVOID_API_EXPORT
+/* We encode this enum plus the OPSZ_ extensions in bytes, so we can have
+ * at most 256 total REG_ plus OPSZ_ values.  Currently there are 165-odd.
+ * Decoder assumes 32-bit, 16-bit, and 8-bit are in specific order
+ * corresponding to modrm encodings.
+ * We also assume that the SEG_ constants are invalid as pointers for
+ * our use in instr_info_t.code.
+ * Also, reg_names array in encode.c corresponds to this enum order.
+ * Plus, reg_fixer array in instr.c.
+ * Lots of optimizations assume same ordering of registers among
+ * 32, 16, and 8  i.e. eax same position (first) in each etc.
+ * reg_rm_selectable() assumes the GPR registers, mmx, and xmm are all in a row.
+ */
+#endif
+enum {
+#ifdef AVOID_API_EXPORT
+    /* compiler gives weird errors for "REG_NONE" */
+    /* PR 227381: genapi.pl auto-inserts doxygen comments for lines without any! */
+#endif
+    REG_NULL, /**< Sentinel value indicating no register, for address modes. */
+    /* 64-bit general purpose */
+    REG_RAX,  REG_RCX,  REG_RDX,  REG_RBX,  REG_RSP,  REG_RBP,  REG_RSI,  REG_RDI,
+    REG_R8,   REG_R9,   REG_R10,  REG_R11,  REG_R12,  REG_R13,  REG_R14,  REG_R15,
+    /* 32-bit general purpose */
+    REG_EAX,  REG_ECX,  REG_EDX,  REG_EBX,  REG_ESP,  REG_EBP,  REG_ESI,  REG_EDI,
+    REG_R8D,  REG_R9D,  REG_R10D, REG_R11D, REG_R12D, REG_R13D, REG_R14D, REG_R15D,
+    /* 16-bit general purpose */
+    REG_AX,   REG_CX,   REG_DX,   REG_BX,   REG_SP,   REG_BP,   REG_SI,   REG_DI,
+    REG_R8W,  REG_R9W,  REG_R10W, REG_R11W, REG_R12W, REG_R13W, REG_R14W, REG_R15W,
+    /* 8-bit general purpose */
+    REG_AL,   REG_CL,   REG_DL,   REG_BL,   REG_AH,   REG_CH,   REG_DH,   REG_BH,
+    REG_R8L,  REG_R9L,  REG_R10L, REG_R11L, REG_R12L, REG_R13L, REG_R14L, REG_R15L,
+    REG_SPL,  REG_BPL,  REG_SIL,  REG_DIL,
+    /* 64-BIT MMX */
+    REG_MM0,  REG_MM1,  REG_MM2,  REG_MM3,  REG_MM4,  REG_MM5,  REG_MM6,  REG_MM7,
+    /* 128-BIT XMM */
+    REG_XMM0, REG_XMM1, REG_XMM2, REG_XMM3, REG_XMM4, REG_XMM5, REG_XMM6, REG_XMM7,
+    REG_XMM8, REG_XMM9, REG_XMM10,REG_XMM11,REG_XMM12,REG_XMM13,REG_XMM14,REG_XMM15,
+    /* floating point registers */
+    REG_ST0,  REG_ST1,  REG_ST2,  REG_ST3,  REG_ST4,  REG_ST5,  REG_ST6,  REG_ST7,
+    /* segments (order from "Sreg" description in Intel manual) */
+    SEG_ES,   SEG_CS,   SEG_SS,   SEG_DS,   SEG_FS,   SEG_GS,
+    /* debug & control registers (privileged access only; 8-15 for future processors) */
+    REG_DR0,  REG_DR1,  REG_DR2,  REG_DR3,  REG_DR4,  REG_DR5,  REG_DR6,  REG_DR7,
+    REG_DR8,  REG_DR9,  REG_DR10, REG_DR11, REG_DR12, REG_DR13, REG_DR14, REG_DR15, 
+    /* cr9-cr15 do not yet exist on current x64 hardware */
+    REG_CR0,  REG_CR1,  REG_CR2,  REG_CR3,  REG_CR4,  REG_CR5,  REG_CR6,  REG_CR7, 
+    REG_CR8,  REG_CR9,  REG_CR10, REG_CR11, REG_CR12, REG_CR13, REG_CR14, REG_CR15,
+    REG_INVALID, /**< Sentinel value indicating an invalid register. */
+};
+/* we avoid typedef-ing the enum, as its storage size is compiler-specific */
+typedef byte reg_id_t; /* contains a REG_ enum value */
+typedef byte opnd_size_t; /* contains a REG_ or OPSZ_ enum value */
+
+/* Platform-independent full-register specifiers */
+#ifdef X64
+# define REG_XAX REG_RAX  /**< Platform-independent way to refer to rax/eax. */
+# define REG_XCX REG_RCX  /**< Platform-independent way to refer to rcx/ecx. */
+# define REG_XDX REG_RDX  /**< Platform-independent way to refer to rdx/edx. */
+# define REG_XBX REG_RBX  /**< Platform-independent way to refer to rbx/ebx. */
+# define REG_XSP REG_RSP  /**< Platform-independent way to refer to rsp/esp. */
+# define REG_XBP REG_RBP  /**< Platform-independent way to refer to rbp/ebp. */
+# define REG_XSI REG_RSI  /**< Platform-independent way to refer to rsi/esi. */
+# define REG_XDI REG_RDI  /**< Platform-independent way to refer to rdi/edi. */
+#else
+# define REG_XAX REG_EAX  /**< Platform-independent way to refer to rax/eax. */
+# define REG_XCX REG_ECX  /**< Platform-independent way to refer to rcx/ecx. */
+# define REG_XDX REG_EDX  /**< Platform-independent way to refer to rdx/edx. */
+# define REG_XBX REG_EBX  /**< Platform-independent way to refer to rbx/ebx. */
+# define REG_XSP REG_ESP  /**< Platform-independent way to refer to rsp/esp. */
+# define REG_XBP REG_EBP  /**< Platform-independent way to refer to rbp/ebp. */
+# define REG_XSI REG_ESI  /**< Platform-independent way to refer to rsi/esi. */
+# define REG_XDI REG_EDI  /**< Platform-independent way to refer to rdi/edi. */
+#endif
+
+/* DR_API EXPORT END */
+/* indexed by enum */
+extern const char * const * const reg_names;
+extern const reg_id_t reg_fixer[];
+/* DR_API EXPORT BEGIN */
+
+#define REG_START_64      REG_RAX   /**< Start of 64-bit general register enum values. */
+#define REG_STOP_64       REG_R15   /**< End of 64-bit general register enum values. */  
+#define REG_START_32      REG_EAX   /**< Start of 32-bit general register enum values. */
+#define REG_STOP_32       REG_R15D  /**< End of 32-bit general register enum values. */  
+#define REG_START_16      REG_AX    /**< Start of 16-bit general register enum values. */
+#define REG_STOP_16       REG_R15W  /**< End of 16-bit general register enum values. */  
+#define REG_START_8       REG_AL    /**< Start of 8-bit general register enum values. */
+#define REG_STOP_8        REG_DIL   /**< End of 8-bit general register enum values. */  
+#define REG_START_8HL     REG_AL    /**< Start of 8-bit high-low register enum values. */
+#define REG_STOP_8HL      REG_BH    /**< End of 8-bit high-low register enum values. */  
+#define REG_START_x86_8   REG_AH    /**< Start of 8-bit x86-only register enum values. */
+#define REG_STOP_x86_8    REG_BH    /**< Stop of 8-bit x86-only register enum values. */
+#define REG_START_x64_8   REG_SPL   /**< Start of 8-bit x64-only register enum values. */
+#define REG_STOP_x64_8    REG_DIL   /**< Stop of 8-bit x64-only register enum values. */
+#define REG_START_MMX     REG_MM0   /**< Start of mmx register enum values. */
+#define REG_STOP_MMX      REG_MM7   /**< End of mmx register enum values. */  
+#define REG_START_XMM     REG_XMM0  /**< Start of xmm register enum values. */
+#define REG_STOP_XMM      REG_XMM15 /**< End of xmm register enum values. */  
+#define REG_START_FLOAT   REG_ST0   /**< Start of floating-point-register enum values. */
+#define REG_STOP_FLOAT    REG_ST7   /**< End of floating-point-register enum values. */  
+#define REG_START_SEGMENT SEG_ES    /**< Start of segment register enum values. */
+#define REG_STOP_SEGMENT  SEG_GS    /**< End of segment register enum values. */  
+#define REG_START_DR      REG_DR0   /**< Start of debug register enum values. */
+#define REG_STOP_DR       REG_DR15  /**< End of debug register enum values. */  
+#define REG_START_CR      REG_CR0   /**< Start of control register enum values. */
+#define REG_STOP_CR       REG_CR15  /**< End of control register enum values. */  
+#define REG_LAST_VALID_ENUM REG_CR15  /**< Last valid register enum value. */
+#define REG_LAST_ENUM     REG_INVALID /**< Last register enum value. */
+/* DR_API EXPORT END */
+#define REG_START_SPILL   REG_XAX
+#define REG_STOP_SPILL    REG_XDI
+#define REG_SPILL_NUM     (REG_STOP_SPILL - REG_START_SPILL + 1)
+
+#define REG_SPECIFIER_BITS 8
+#define SCALE_SPECIFIER_BITS 4
+
+#define INT8_MIN   SCHAR_MIN
+#define INT8_MAX   SCHAR_MAX
+#define INT16_MIN  SHRT_MIN
+#define INT16_MAX  SHRT_MAX
+#define INT32_MIN  INT_MIN
+#define INT32_MAX  INT_MAX
+
+/* typedef is in globals.h */
+struct _opnd_t {
+    byte kind;
+    /* size field only used for immed_ints and addresses
+     * it holds a OPSZ_ field from decode.h 
+     * we need it so we can pick the proper instruction form for
+     * encoding -- an alternative would be to split all the opcodes
+     * up into different data size versions.
+     */
+    opnd_size_t size;
+    /* To avoid increasing our union beyond 64 bits, we store additional data
+     * needed for x64 operand types here in the alignment padding.
+     */
+    union {
+        ushort far_pc_seg_selector; /* FAR_PC_kind and FAR_INSTR_kind */
+        /* We could fit segment in value.base_disp but more consistent here */
+        reg_id_t segment : REG_SPECIFIER_BITS; /* BASE_DISP_kind, REL_ADDR_kind,
+                                                * and ABS_ADDR_kind */
+    } seg;
+    union {
+        /* all are 64 bits or less */
+        /* NULL_kind has no value */
+        ptr_int_t immed_int;   /* IMMED_INTEGER_kind */
+        float immed_float;     /* IMMED_FLOAT_kind */
+        /* PR 225937: today we provide no way of specifying a 16-bit immediate
+         * (encoded as a data16 prefix, which also implies a 16-bit EIP,
+         * making it only useful for far pcs)
+         */
+        app_pc pc;             /* PC_kind and FAR_PC_kind */
+        /* For FAR_PC_kind and FAR_INSTR_kind, we use pc/instr, and keep the
+         * segment selector (which is NOT a SEG_constant) in far_pc_seg_selector
+         * above, to save space.
+         */
+        instr_t *instr;         /* INSTR_kind and FAR_INSTR_kind */
+        reg_id_t reg;           /* REG_kind */
+        struct {
+            int disp;
+            reg_id_t base_reg : REG_SPECIFIER_BITS;
+            reg_id_t index_reg : REG_SPECIFIER_BITS;
+            /* to get cl to not align to 4 bytes we can't use uint here
+             * when we have reg_id_t elsewhere: it won't combine them
+             * (gcc will). alternative is all uint and no reg_id_t. */
+            byte scale : SCALE_SPECIFIER_BITS;
+            byte/*bool*/ encode_zero_disp : 1;
+            byte/*bool*/ force_full_disp : 1; /* don't use 8-bit even w/ 8-bit value */
+            byte/*bool*/ disp_short_addr : 1; /* 16-bit (32 in x64) addr (disp-only) */
+        } base_disp;            /* BASE_DISP_kind */
+        void *addr;             /* REL_ADDR_kind and ABS_ADDR_kind */
+    } value;
+};
+
+/* We assert that our fields are packed properly in arch_init().
+ * We could use #pragma pack to shrink x64 back down to 12 bytes (it's at 16
+ * b/c the struct is aligned to its max field align which is 8), but
+ * probably not much gain since in either case it's passed/returned as a pointer
+ * and the temp memory allocated is 16-byte aligned (on Windows; for
+ * Linux it is passed in two consecutive registers I believe, but
+ * still 12 bytes vs 16 makes no difference).
+ */
+#define EXPECTED_SIZEOF_OPND (3*sizeof(uint) IF_X64(+4/*struct size padding*/))
+
+/* x86 operand kinds */
+enum {
+    NULL_kind,
+    IMMED_INTEGER_kind,
+    IMMED_FLOAT_kind,
+    PC_kind,
+    INSTR_kind,
+    REG_kind,
+    BASE_DISP_kind, /* optional SEG_ reg + base reg + scaled index reg + disp */
+    FAR_PC_kind,    /* a segment is specified as a selector value */
+    FAR_INSTR_kind, /* a segment is specified as a selector value */
+#ifdef X64
+    REL_ADDR_kind,  /* pc-relative address: x64 only */
+    ABS_ADDR_kind,  /* 64-bit absolute address: x64 only */
+#endif
+    LAST_kind,      /* sentinal; not a valid opnd kind */
+};
+
+/* functions to build an operand */
+
+DR_API
+/** Returns an empty operand. */
+opnd_t 
+opnd_create_null(void);
+
+DR_API
+/** Returns a register operand (\p r must be a REG_ constant). */
+opnd_t 
+opnd_create_reg(reg_id_t r);
+
+DR_API
+/** 
+ * Returns an immediate integer operand with value \p i and size
+ * \p data_size; \p data_size must be a OPSZ_ constant.
+ */
+opnd_t 
+opnd_create_immed_int(ptr_int_t i, opnd_size_t data_size);
+
+DR_API
+/** Returns an immediate float operand with value \p f. */
+opnd_t 
+opnd_create_immed_float(float f);
+
+DR_API
+/** Returns a program address operand with value \p pc. */
+opnd_t 
+opnd_create_pc(app_pc pc);
+
+DR_API
+/**
+ * Returns a far program address operand with value \p seg_selector:pc.
+ * \p seg_selector is a segment selector, not a SEG_ constant.
+ */
+opnd_t 
+opnd_create_far_pc(ushort seg_selector, app_pc pc);
+
+DR_API
+/** Returns an instr_t pointer address with value \p instr. */
+opnd_t 
+opnd_create_instr(instr_t *instr);
+
+DR_API
+/**
+ * Returns a far instr_t pointer address with value \p seg_selector:instr.
+ * \p seg_selector is a segment selector, not a SEG_ constant.
+ */
+opnd_t 
+opnd_create_far_instr(ushort seg_selector, instr_t *instr);
+
+DR_API
+/** 
+ * Returns a memory reference operand that refers to the address:
+ * - disp(base_reg, index_reg, scale)
+ *
+ * or, in other words,
+ * - base_reg + index_reg*scale + disp
+ *
+ * The operand has data size data_size (must be a OPSZ_ constant).
+ * Both \p base_reg and \p index_reg must be REG_ constants.
+ * \p scale must be either 1, 2, 4, or 8.
+ */
+opnd_t 
+opnd_create_base_disp(reg_id_t base_reg, reg_id_t index_reg, int scale, int disp,
+                      opnd_size_t data_size);
+
+DR_API
+/** 
+ * Returns a memory reference operand that refers to the address:
+ * - disp(base_reg, index_reg, scale)
+ *
+ * or, in other words,
+ * - base_reg + index_reg*scale + disp
+ *
+ * The operand has data size \p data_size (must be a OPSZ_ constant).
+ * Both \p base_reg and \p index_reg must be REG_ constants.
+ * \p scale must be either 1, 2, 4, or 8.
+ * Gives control over encoding optimizations:
+ * -# If \p encode_zero_disp, a zero value for disp will not be omitted;
+ * -# If \p force_full_disp, a small value for disp will not occupy only one byte.
+ * -# If \p disp_short_addr, short (16-bit for 32-bit mode, 32-bit for
+ *    64-bit mode) addressing will be used (note that this normally only
+ *    needs to be specified for an absolute address; otherwise, simply
+ *    use the desired short registers for base and/or index).
+ *
+ * (Both of those are false when using opnd_create_base_disp()).
+ */
+opnd_t
+opnd_create_base_disp_ex(reg_id_t base_reg, reg_id_t index_reg, int scale,
+                         int disp, opnd_size_t size,
+                         bool encode_zero_disp, bool force_full_disp,
+                         bool disp_short_addr);
+
+DR_API
+/** 
+ * Returns a far memory reference operand that refers to the address:
+ * - seg : disp(base_reg, index_reg, scale)
+ *
+ * or, in other words,
+ * - seg : base_reg + index_reg*scale + disp
+ *
+ * The operand has data size \p data_size (must be a OPSZ_ constant).
+ * \p seg must be a SEG_ constant.
+ * Both \p base_reg and \p index_reg must be REG_ constants.
+ * \p scale must be either 1, 2, 4, or 8.
+ */
+opnd_t 
+opnd_create_far_base_disp(reg_id_t seg, reg_id_t base_reg, reg_id_t index_reg, int scale,
+                          int disp, opnd_size_t data_size);
+
+DR_API
+/** 
+ * Returns a far memory reference operand that refers to the address:
+ * - seg : disp(base_reg, index_reg, scale)
+ *
+ * or, in other words,
+ * - seg : base_reg + index_reg*scale + disp
+ *
+ * The operand has data size \p data_size (must be a OPSZ_ constant).
+ * \p seg must be a SEG_ constant.
+ * Both \p base_reg and \p index_reg must be REG_ constants.
+ * scale must be either 1, 2, 4, or 8.
+ * Gives control over encoding optimizations:
+ * -# If \p encode_zero_disp, a zero value for disp will not be omitted;
+ * -# If \p force_full_disp, a small value for disp will not occupy only one byte.
+ * -# If \p disp_short_addr, short (16-bit for 32-bit mode, 32-bit for
+ *    64-bit mode) addressing will be used (note that this normally only
+ *    needs to be specified for an absolute address; otherwise, simply
+ *    use the desired short registers for base and/or index).
+ *
+ * (Both of those are false when using opnd_create_far_base_disp()).
+ */
+opnd_t
+opnd_create_far_base_disp_ex(reg_id_t seg, reg_id_t base_reg, reg_id_t index_reg,
+                             int scale, int disp, opnd_size_t size,
+                             bool encode_zero_disp, bool force_full_disp,
+                             bool disp_short_addr);
+
+DR_API
+/**
+ * Returns a memory reference operand that refers to the address \p addr.
+ * The operand has data size \p data_size (must be a OPSZ_ constant).
+ *
+ * If \p addr <= 2^32 (which is always true in 32-bit mode), this routine
+ * is equivalent to
+ * opnd_create_base_disp(REG_NULL, REG_NULL, 0, (int)addr, data_size).
+ *
+ * Otherwise, this routine creates a separate operand type with an
+ * absolute 64-bit memory address.  Note that such an operand can only be
+ * used as a load or store from or to the rax register.
+ */
+opnd_t 
+opnd_create_abs_addr(void *addr, opnd_size_t data_size);
+
+DR_API
+/**
+ * Returns a memory reference operand that refers to the address
+ * \p seg: \p addr.
+ * The operand has data size \p data_size (must be a OPSZ_ constant).
+ *
+ * If \p addr <= 2^32 (which is always true in 32-bit mode), this routine
+ * is equivalent to
+ * opnd_create_far_base_disp(seg, REG_NULL, REG_NULL, 0, (int)addr, data_size).
+ *
+ * Otherwise, this routine creates a separate operand type with an
+ * absolute 64-bit memory address.  Note that such an operand can only be
+ * used as a load or store from or to the rax register.
+ */
+opnd_t 
+opnd_create_far_abs_addr(reg_id_t seg, void *addr, opnd_size_t data_size);
+
+/* DR_API EXPORT BEGIN */
+#ifdef X64
+/* DR_API EXPORT END */
+DR_API
+/**
+ * Returns a memory reference operand that refers to the address \p
+ * addr, but will be encoded as a pc-relative address.  At emit time,
+ * if \p addr is out of reach of a 32-bit signed displacement from the
+ * next instruction, encoding will fail.
+ *
+ * DR guarantees that all of its code caches and heap are within the
+ * same 2GB memory region.  DR also loads client libraries within
+ * 32-bit reachability of its code caches and heap.  This means that
+ * any static data or code in a client library, or any data allocated
+ * using DR's API, is guaranteed to be reachable from code cache code.
+ *
+ * The operand has data size data_size (must be a OPSZ_ constant).
+ *
+ * To represent a 32-bit address (i.e., what an address size prefix
+ * indicates), simply zero out the top 32 bits of the address before
+ * passing it to this routine.
+ *
+ * \note For 64-bit DR builds only.
+ */
+opnd_t 
+opnd_create_rel_addr(void *addr, opnd_size_t data_size);
+
+DR_API
+/**
+ * Returns a memory reference operand that refers to the address \p
+ * seg : \p addr, but will be encoded as a pc-relative address.  It is
+ * up to the caller to ensure that the resulting address is reachable
+ * via a 32-bit signed displacement from the next instruction at emit
+ * time.
+ *
+ * DR guarantees that all of its code caches and heap are within the
+ * same 2GB memory region.  DR also loads client libraries within
+ * 32-bit reachability of its code caches and heap.  This means that
+ * any static data or code in a client library, or any data allocated
+ * using DR's API, is guaranteed to be reachable from code cache code.
+ *
+ * The operand has data size \p data_size (must be a OPSZ_ constant).
+ *
+ * To represent a 32-bit address (i.e., what an address size prefix
+ * indicates), simply zero out the top 32 bits of the address before
+ * passing it to this routine.
+ *
+ * \note For 64-bit DR builds only.
+ */
+opnd_t 
+opnd_create_far_rel_addr(reg_id_t seg, void *addr, opnd_size_t data_size);
+/* DR_API EXPORT BEGIN */
+#endif
+/* DR_API EXPORT END */
+
+/* predicate functions */
+
+/* Check if the operand kind and size fields are valid */
+bool
+opnd_is_valid(opnd_t opnd);
+
+DR_API
+/** Returns true iff \p opnd is an empty operand. */
+bool 
+opnd_is_null(opnd_t opnd);
+
+DR_API
+/** Returns true iff \p opnd is a register operand. */
+bool 
+opnd_is_reg(opnd_t opnd);
+
+DR_API
+/** Returns true iff \p opnd is an immediate (integer or float) operand. */
+bool 
+opnd_is_immed(opnd_t opnd);
+
+DR_API
+/** Returns true iff \p opnd is an immediate integer operand. */
+bool 
+opnd_is_immed_int(opnd_t opnd);
+
+DR_API
+/** Returns true iff \p opnd is an immediate float operand. */
+bool 
+opnd_is_immed_float(opnd_t opnd);
+
+DR_API
+/** Returns true iff \p opnd is a (near or far) program address operand. */
+bool 
+opnd_is_pc(opnd_t opnd);
+
+DR_API
+/** Returns true iff \p opnd is a near (i.e., default segment) program address operand. */
+bool 
+opnd_is_near_pc(opnd_t opnd);
+
+DR_API
+/** Returns true iff \p opnd is a far program address operand. */
+bool 
+opnd_is_far_pc(opnd_t opnd);
+
+DR_API
+/** Returns true iff \p opnd is a (near or far) instr_t pointer address operand. */
+bool 
+opnd_is_instr(opnd_t opnd);
+
+DR_API
+/** Returns true iff \p opnd is a near instr_t pointer address operand. */
+bool 
+opnd_is_near_instr(opnd_t opnd);
+
+DR_API
+/** Returns true iff \p opnd is a far instr_t pointer address operand. */
+bool 
+opnd_is_far_instr(opnd_t opnd);
+
+DR_API
+/** Returns true iff \p opnd is a (near or far) base+disp memory reference operand. */
+bool 
+opnd_is_base_disp(opnd_t opnd);
+
+DR_API
+/**
+ * Returns true iff \p opnd is a near (i.e., default segment) base+disp memory
+ * reference operand.
+ */
+bool 
+opnd_is_near_base_disp(opnd_t opnd);
+
+DR_API
+/** Returns true iff \p opnd is a far base+disp memory reference operand. */
+bool 
+opnd_is_far_base_disp(opnd_t opnd);
+
+DR_API
+/** 
+ * Returns true iff \p opnd is a (near or far) absolute address operand.
+ * Returns true for both base-disp operands with no base or index and
+ * 64-bit non-base-disp absolute address operands. 
+ */
+bool 
+opnd_is_abs_addr(opnd_t opnd);
+
+DR_API
+/** 
+ * Returns true iff \p opnd is a near (i.e., default segment) absolute address operand.
+ * Returns true for both base-disp operands with no base or index and
+ * 64-bit non-base-disp absolute address operands. 
+ */
+bool 
+opnd_is_near_abs_addr(opnd_t opnd);
+
+DR_API
+/** 
+ * Returns true iff \p opnd is a far absolute address operand.
+ * Returns true for both base-disp operands with no base or index and
+ * 64-bit non-base-disp absolute address operands. 
+ */
+bool 
+opnd_is_far_abs_addr(opnd_t opnd);
+
+/* DR_API EXPORT BEGIN */
+#ifdef X64
+/* DR_API EXPORT END */
+DR_API
+/**
+ * Returns true iff \p opnd is a (near or far) pc-relative memory reference operand. 
+ *
+ * \note For 64-bit DR builds only.
+ */
+bool 
+opnd_is_rel_addr(opnd_t opnd);
+
+DR_API
+/**
+ * Returns true iff \p opnd is a near (i.e., default segment) pc-relative memory
+ * reference operand. 
+ *
+ * \note For 64-bit DR builds only.
+ */
+bool 
+opnd_is_near_rel_addr(opnd_t opnd);
+
+DR_API
+/**
+ * Returns true iff \p opnd is a far pc-relative memory reference operand. 
+ *
+ * \note For 64-bit DR builds only.
+ */
+bool 
+opnd_is_far_rel_addr(opnd_t opnd);
+/* DR_API EXPORT BEGIN */
+#endif
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Returns true iff \p opnd is a (near or far) memory reference operand
+ * of any type: base-disp, absolute address, or pc-relative address.
+ *
+ * \note For 64-bit DR builds only.
+ */
+bool 
+opnd_is_memory_reference(opnd_t opnd);
+
+DR_API
+/**
+ * Returns true iff \p opnd is a far memory reference operand
+ * of any type: base-disp, absolute address, or pc-relative address.
+ */
+bool
+opnd_is_far_memory_reference(opnd_t opnd);
+
+DR_API
+/**
+ * Returns true iff \p opnd is a near memory reference operand
+ * of any type: base-disp, absolute address, or pc-relative address.
+ */
+bool
+opnd_is_near_memory_reference(opnd_t opnd);
+
+/* accessor functions */
+
+DR_API
+/** 
+ * Return the data size of \p opnd as a OPSZ_ constant.
+ * Assumes \p opnd is a register, immediate integer, or memory reference.
+ * If \p opnd is a register returns the result of opnd_reg_get_size()
+ * called on the REG_ constant.
+ * Returns OPSZ_NA if \p opnd does not have a valid size.
+ */
+opnd_size_t
+opnd_get_size(opnd_t opnd);
+
+DR_API
+/** 
+ * Sets the data size of \p opnd.
+ * Assumes \p opnd is an immediate integer or a memory reference.
+ */
+void   
+opnd_set_size(opnd_t *opnd, opnd_size_t newsize);
+
+DR_API
+/** 
+ * Assumes \p opnd is a register operand.
+ * Returns the register it refers to (a REG_ constant).
+ */
+reg_id_t  
+opnd_get_reg(opnd_t opnd);
+
+DR_API
+/* Assumes opnd is an immediate integer, returns its value. */
+ptr_int_t
+opnd_get_immed_int(opnd_t opnd);
+
+DR_API
+/** Assumes \p opnd is an immediate float, returns its value. */
+float  
+opnd_get_immed_float(opnd_t opnd);
+
+DR_API
+/** Assumes \p opnd is a (near or far) program address, returns its value. */
+app_pc 
+opnd_get_pc(opnd_t opnd);
+
+DR_API
+/** 
+ * Assumes \p opnd is a far program address.
+ * Returns \p opnd's segment, a segment selector (not a SEG_ constant).
+ */
+ushort    
+opnd_get_segment_selector(opnd_t opnd);
+
+DR_API
+/** Assumes \p opnd is an instr_t pointer, returns its value. */
+instr_t*
+opnd_get_instr(opnd_t opnd);
+
+DR_API
+/**
+ * Assumes \p opnd is a (near or far) base+disp memory reference.  Returns the base
+ * register (a REG_ constant).
+ */
+reg_id_t
+opnd_get_base(opnd_t opnd);
+
+DR_API
+/**
+ * Assumes \p opnd is a (near or far) base+disp memory reference.
+ * Returns the displacement. 
+ */
+int 
+opnd_get_disp(opnd_t opnd);
+
+DR_API
+/** 
+ * Assumes \p opnd is a (near or far) base+disp memory reference; returns whether
+ * encode_zero_disp has been specified for \p opnd.
+ */
+bool
+opnd_is_disp_encode_zero(opnd_t opnd);
+
+DR_API
+/** 
+ * Assumes \p opnd is a (near or far) base+disp memory reference; returns whether
+ * force_full_disp has been specified for \p opnd.
+ */
+bool
+opnd_is_disp_force_full(opnd_t opnd);
+
+DR_API
+/** 
+ * Assumes \p opnd is a (near or far) base+disp memory reference; returns whether
+ * disp_short_addr has been specified for \p opnd.
+ */
+bool
+opnd_is_disp_short_addr(opnd_t opnd);
+
+DR_API
+/** 
+ * Assumes \p opnd is a (near or far) base+disp memory reference.
+ * Returns the index register (a REG_ constant).
+ */
+reg_id_t
+opnd_get_index(opnd_t opnd);
+
+DR_API
+/** Assumes \p opnd is a (near or far) base+disp memory reference.  Returns the scale. */
+int 
+opnd_get_scale(opnd_t opnd);
+
+DR_API
+/** 
+ * Assumes \p opnd is a (near or far) memory reference of any type.
+ * Returns \p opnd's segment (a SEG_ constant), or REG_NULL if it is a near
+ * memory reference.
+ */
+reg_id_t    
+opnd_get_segment(opnd_t opnd);
+
+DR_API
+/** 
+ * Assumes \p opnd is a (near or far) absolute or pc-relative memory reference,
+ * or a base+disp memory reference with no base or index register.
+ * Returns \p opnd's absolute address (which will be pc-relativized on encoding
+ * for pc-relative memory references).
+ */
+void *
+opnd_get_addr(opnd_t opnd);
+
+DR_API
+/** 
+ * Returns the number of registers referred to by \p opnd. This will only
+ * be non-zero for register operands and memory references.
+ */
+int 
+opnd_num_regs_used(opnd_t opnd);
+
+DR_API
+/** 
+ * Used in conjunction with opnd_num_regs_used(), this routine can be used
+ * to iterate through all registers used by \p opnd.
+ * The index values begin with 0 and proceed through opnd_num_regs_used(opnd)-1.
+ */
+reg_id_t
+opnd_get_reg_used(opnd_t opnd, int index);
+
+/* utility functions */
+
+#ifdef DEBUG
+void
+reg_check_reg_fixer(void);
+#endif
+
+DR_API
+/** 
+ * Assumes that \p reg is a REG_ 32-bit register constant.
+ * Returns the string name for \p reg.
+ */
+const char *
+get_register_name(reg_id_t reg);
+
+DR_API
+/** 
+ * Assumes that \p reg is a REG_ 32-bit register constant.
+ * Returns the 16-bit version of \p reg.
+ */
+reg_id_t
+reg_32_to_16(reg_id_t reg);
+
+DR_API
+/** 
+ * Assumes that \p reg is a REG_ 32-bit register constant.
+ * Returns the 8-bit version of \p reg (the least significant byte:
+ * REG_AL instead of REG_AH if passed REG_EAX, e.g.).  For 32-bit DR
+ * builds, returns REG_NULL if passed REG_ESP, REG_EBP, REG_ESI, or
+ * REG_EDI.
+ */
+reg_id_t
+reg_32_to_8(reg_id_t reg);
+
+/* DR_API EXPORT BEGIN */
+#ifdef X64
+/* DR_API EXPORT END */
+DR_API
+/** 
+ * Assumes that \p reg is a REG_ 32-bit register constant.
+ * Returns the 64-bit version of \p reg.
+ *
+ * \note For 64-bit DR builds only.
+ */
+reg_id_t
+reg_32_to_64(reg_id_t reg);
+
+DR_API
+/** 
+ * Assumes that \p reg is a REG_ 64-bit register constant.
+ * Returns the 32-bit version of \p reg.
+ *
+ * \note For 64-bit DR builds only.
+ */
+reg_id_t
+reg_64_to_32(reg_id_t reg);
+
+DR_API
+/** 
+ * Returns true iff \p reg refers to an extended register only available
+ * in 64-bit mode and not in 32-bit mode (e.g., R8-R15, XMM8-XMM15, etc.)
+ *
+ * \note For 64-bit DR builds only.
+ */
+bool 
+reg_is_extended(reg_id_t reg);
+/* DR_API EXPORT BEGIN */
+#endif
+/* DR_API EXPORT END */
+
+DR_API
+/** 
+ * Assumes that \p reg is a REG_ 32-bit register constant.
+ * If \p sz == OPSZ_2, returns the 16-bit version of \p reg.
+ * For 64-bit versions of this library, if \p sz == OPSZ_8, returns 
+ * the 64-bit version of \p reg.
+ */
+reg_id_t
+reg_32_to_opsz(reg_id_t reg, opnd_size_t sz);
+
+DR_API
+/**  
+ * Assumes that \p reg is a REG_ register constant.
+ * If reg is used as part of the calling convention, returns which
+ * parameter ordinal it matches (0-based); otherwise, returns -1.
+ */
+int
+reg_parameter_num(reg_id_t reg);
+
+DR_API
+/** 
+ * Assumes that \p reg is a REG_ constant.
+ * Returns true iff it refers to a General Purpose Register,
+ * i.e., rax, rcx, rdx, rbx, rsp, rbp, rsi, rdi, or a subset.
+ */
+bool
+reg_is_gpr(reg_id_t reg);
+
+DR_API
+/** 
+ * Assumes that \p reg is a REG_ constant.
+ * Returns true iff it refers to a segment (i.e., it's really a SEG_
+ * constant).
+ */
+bool
+reg_is_segment(reg_id_t reg);
+
+DR_API
+/** 
+ * Assumes that \p reg is a REG_ constant.
+ * Returns true iff it refers to an xmm (128-bit SSE/SSE2) register.
+ */
+bool 
+reg_is_xmm(reg_id_t reg);
+
+DR_API
+/** 
+ * Assumes that \p reg is a REG_ constant.
+ * Returns true iff it refers to an mmx (64-bit) register.
+ */
+bool 
+reg_is_mmx(reg_id_t reg);
+
+DR_API
+/** 
+ * Assumes that \p reg is a REG_ constant.
+ * Returns true iff it refers to a floating-point register.
+ */
+bool 
+reg_is_fp(reg_id_t reg);
+
+DR_API
+/** 
+ * Assumes that \p reg is a REG_ constant.
+ * Returns true iff it refers to a 32-bit general-purpose register.
+ */
+bool 
+reg_is_32bit(reg_id_t reg);
+
+DR_API
+/** 
+ * Returns true iff \p opnd is a register operand that refers to a 32-bit
+ * general-purpose register.
+ */
+bool 
+opnd_is_reg_32bit(opnd_t opnd);
+
+DR_API
+/** 
+ * Assumes that \p reg is a REG_ constant.
+ * Returns true iff it refers to a 64-bit general-purpose register.
+ */
+bool 
+reg_is_64bit(reg_id_t reg);
+
+DR_API
+/** 
+ * Returns true iff \p opnd is a register operand that refers to a 64-bit
+ * general-purpose register.
+ */
+bool 
+opnd_is_reg_64bit(opnd_t opnd);
+
+DR_API
+/** 
+ * Assumes that \p reg is a REG_ constant.
+ * Returns true iff it refers to a pointer-sized general-purpose register.
+ */
+bool 
+reg_is_pointer_sized(reg_id_t reg);
+
+DR_API
+/** 
+ * Assumes that \p reg is a REG_ 32-bit register constant.
+ * Returns the pointer-sized version of \p reg.
+ */
+reg_id_t
+reg_to_pointer_sized(reg_id_t reg);
+
+DR_API
+/** 
+ * Returns true iff \p opnd is a register operand that refers to a
+ * pointer-sized general-purpose register.
+ */
+bool 
+opnd_is_reg_pointer_sized(opnd_t opnd);
+
+/* not exported */
+int
+opnd_get_reg_dcontext_offs(reg_id_t reg);
+
+int
+opnd_get_reg_mcontext_offs(reg_id_t reg);
+
+DR_API
+/** 
+ * Assumes that \p r1 and \p r2 are both REG_ constants.
+ * Returns true iff \p r1's register overlaps \p r2's register
+ * (e.g., if \p r1 == REG_AX and \p r2 == REG_EAX).
+ */
+bool 
+reg_overlap(reg_id_t r1, reg_id_t r2);
+
+DR_API
+/** 
+ * Assumes that \p reg is a REG_ constant.
+ * Returns \p reg's representation as 3 bits in a modrm byte
+ * (the 3 bits are the lower-order bits in the return value).
+ */
+byte
+reg_get_bits(reg_id_t reg);
+
+DR_API
+/** 
+ * Assumes that \p reg is a REG_ constant.
+ * Returns the OPSZ_ constant corresponding to the register size.
+ * Returns OPSZ_NA if reg is not a REG_ constant.
+ */
+opnd_size_t 
+reg_get_size(reg_id_t reg);
+
+DR_API
+/** 
+ * Assumes that \p reg is a REG_ constant.
+ * Returns true iff \p opnd refers to reg directly or refers to a register
+ * that overlaps \p reg (e.g., REG_AX overlaps REG_EAX).
+ */
+bool 
+opnd_uses_reg(opnd_t opnd, reg_id_t reg);
+
+DR_API
+/** Set the displacement of a memory reference operand \p opnd to \p disp. */
+void
+opnd_set_disp(opnd_t *opnd, int disp);
+
+DR_API
+/** 
+ * Set the displacement and encoding controls of a memory reference operand:
+ * - If \p encode_zero_disp, a zero value for \p disp will not be omitted;
+ * - If \p force_full_disp, a small value for \p disp will not occupy only one byte.
+ * -# If \p disp_short_addr, short (16-bit for 32-bit mode, 32-bit for
+ *    64-bit mode) addressing will be used (note that this normally only
+ *    needs to be specified for an absolute address; otherwise, simply
+ *    use the desired short registers for base and/or index).
+ */
+void
+opnd_set_disp_ex(opnd_t *opnd, int disp, bool encode_zero_disp, bool force_full_disp,
+                 bool disp_short_addr);
+
+DR_API
+/** 
+ * Assumes that both \p old_reg and \p new_reg are REG_ constants.
+ * Replaces all occurrences of \p old_reg in \p *opnd with \p new_reg.
+ */
+bool 
+opnd_replace_reg(opnd_t *opnd, reg_id_t old_reg, reg_id_t new_reg);
+
+DR_API
+/** Returns true iff \p op1 and \p op2 are indistinguishable. 
+ *  If either uses variable operand sizes, the default size is assumed.
+ */
+bool 
+opnd_same(opnd_t op1,opnd_t op2);
+
+DR_API
+/** 
+ * Returns true iff \p op1 and \p op2 are both memory references and they
+ * are indistinguishable, ignoring data size.
+ */
+bool 
+opnd_same_address(opnd_t op1,opnd_t op2);
+
+DR_API
+/** 
+ * Returns true iff there exists some register that is refered to (directly
+ * or overlapping) by both \p op1 and \p op2.
+ */
+bool 
+opnd_share_reg(opnd_t op1, opnd_t op2);
+
+DR_API
+/** 
+ * Returns true iff \p def, considered as a write, affects \p use.
+ * Is conservative, so if both \p def and \p use are memory references,
+ * will return true unless it can disambiguate them based on their
+ * registers and displacement.
+ */
+bool 
+opnd_defines_use(opnd_t def, opnd_t use);
+
+DR_API
+/** 
+ * Assumes \p size is a OPSZ_ or a REG_ constant.
+ * If \p size is a REG_ constant, first calls reg_get_size(\p size)
+ * to get a OPSZ_ constant.
+ * Returns the number of bytes the OPSZ_ constant represents.
+ */
+uint 
+opnd_size_in_bytes(opnd_size_t size);
+
+DR_API
+/** 
+ * Shrinks all 32-bit registers in \p opnd to their 16-bit versions.  
+ * Also shrinks the size of immediate integers and memory references from
+ * OPSZ_4 to OPSZ_2.
+ */
+opnd_t 
+opnd_shrink_to_16_bits(opnd_t opnd);
+
+/* DR_API EXPORT BEGIN */
+#ifdef X64
+/* DR_API EXPORT END */
+DR_API
+/** 
+ * Shrinks all 64-bit registers in \p opnd to their 32-bit versions.  
+ * Also shrinks the size of immediate integers and memory references from
+ * OPSZ_8 to OPSZ_4.
+ *
+ * \note For 64-bit DR builds only.
+ */
+opnd_t 
+opnd_shrink_to_32_bits(opnd_t opnd);
+/* DR_API EXPORT BEGIN */
+#endif
+/* DR_API EXPORT END */
+
+DR_API
+/** 
+ * Returns the value of the register \p reg, selected from the passed-in
+ * register values.
+ */
+reg_t
+reg_get_value(reg_id_t reg, dr_mcontext_t *mc);
+
+DR_API
+/**
+ * Sets the register \p reg in the passed in mcontext \p mc to \p value.
+ * \note Current release is limited to setting pointer-sized registers only
+ * (no sub-registers).
+ */
+void
+reg_set_value(reg_id_t reg, dr_mcontext_t *mc, reg_t value);
+
+DR_API
+/** 
+ * Returns the effective address of \p opnd, computed using the passed-in 
+ * register values.  If \p opnd is a far address, ignores that aspect
+ * except for fs: references on Windows.
+ */
+app_pc
+opnd_compute_address(opnd_t opnd, dr_mcontext_t *mc);
+
+
+/*************************
+ ***       instr_t       ***
+ *************************/
+
+/* instr_t structure
+ * An instruction represented by instr_t can be in a number of states,
+ * depending on whether it points to raw bits that are valid,
+ * whether its operand and opcode fields are up to date, and whether
+ * its eflags field is up to date.
+ * Invariant: if opcode == OP_UNDECODED, raw bits should be valid.
+ *            if opcode == OP_INVALID, raw bits may point to real bits,
+ *              but they are not a valid instruction stream.
+ *
+ * CORRESPONDENCE WITH CGO LEVELS
+ * Level 0 = raw bits valid, !opcode_valid, decode_sizeof(instr) != instr->len
+ *   opcode_valid is equivalent to opcode != OP_INVALID && opcode != OP_UNDECODED
+ * Level 1 = raw bits valid, !opcode_valid, decode_sizeof(instr) == instr->len
+ * Level 2 = raw bits valid, opcode_valid, !operands_valid
+ *   (eflags info is auto-derived on demand so not an issue)
+ * Level 3 = raw bits valid, operands valid
+ *   (we assume that if operands_valid then opcode_valid)
+ * Level 4 = !raw bits valid, operands valid
+ *
+ * Independent of these is whether its raw bits were allocated for
+ * the instr or not.
+ */
+
+/* flags */
+enum {
+    /* these first flags are shared with the LINK_ flags and are
+     * used to pass on info to link stubs 
+     */
+    /* used to determine type of indirect branch for exits */
+    INSTR_DIRECT_EXIT           = LINK_DIRECT,
+    INSTR_INDIRECT_EXIT         = LINK_INDIRECT,
+    INSTR_RETURN_EXIT           = LINK_RETURN,
+    INSTR_CALL_EXIT             = LINK_CALL,
+    INSTR_JMP_EXIT              = LINK_JMP,
+    /* marks an indirect jmp preceded by a call (== a PLT-style ind call) */
+    INSTR_IND_JMP_PLT_EXIT      = LINK_IND_JMP_PLT,
+    INSTR_BRANCH_SELFMOD_EXIT   = LINK_SELFMOD_EXIT,
+#ifdef UNSUPPORTED_API
+    INSTR_BRANCH_TARGETS_PREFIX = LINK_TARGET_PREFIX,
+#endif
+#ifdef X64
+    /* PR 257963: since we don't store targets of ind branches, we need a flag
+     * so we know whether this is a trace cmp exit, which has its own ibl entry
+     */
+    INSTR_TRACE_CMP_EXIT        = LINK_TRACE_CMP,
+#endif
+#ifdef WINDOWS
+    INSTR_CALLBACK_RETURN       = LINK_CALLBACK_RETURN,
+#else
+    INSTR_NI_SYSCALL_INT        = LINK_NI_SYSCALL_INT,
+#endif
+    INSTR_NI_SYSCALL            = LINK_NI_SYSCALL,
+    INSTR_NI_SYSCALL_ALL        = LINK_NI_SYSCALL_ALL,
+    /* meta-flag */
+    EXIT_CTI_TYPES              = (INSTR_DIRECT_EXIT | INSTR_INDIRECT_EXIT |
+                                   INSTR_RETURN_EXIT | INSTR_CALL_EXIT |     
+                                   INSTR_JMP_EXIT | INSTR_IND_JMP_PLT_EXIT |
+                                   INSTR_BRANCH_SELFMOD_EXIT |
+#ifdef UNSUPPORTED_API
+                                   INSTR_BRANCH_TARGETS_PREFIX |
+#endif
+#ifdef X64
+                                   INSTR_TRACE_CMP_EXIT |
+#endif
+#ifdef WINDOWS
+                                   INSTR_CALLBACK_RETURN |
+#else
+                                   INSTR_NI_SYSCALL_INT |
+#endif
+                                   INSTR_NI_SYSCALL),
+
+    /* instr_t-internal flags (not shared with LINK_) */
+    INSTR_OPERANDS_VALID        = 0x00010000,
+    /* meta-flag */
+    INSTR_FIRST_NON_LINK_SHARED_FLAG = INSTR_OPERANDS_VALID,
+    INSTR_EFLAGS_VALID          = 0x00020000,
+    INSTR_EFLAGS_6_VALID        = 0x00040000,
+    INSTR_RAW_BITS_VALID        = 0x00080000,
+    INSTR_RAW_BITS_ALLOCATED    = 0x00100000,
+    INSTR_DO_NOT_MANGLE         = 0x00200000,
+    INSTR_HAS_CUSTOM_STUB       = 0x00400000,
+    /* used to indicate that an indirect call can be treated as a direct call */
+    INSTR_IND_CALL_DIRECT       = 0x00800000,
+#ifdef WINDOWS
+    /* used to indicate that a syscall should be executed via shared syscall */
+    INSTR_SHARED_SYSCALL        = 0x01000000,
+    /* 0x02000000 -- used to be INSTR_SYSENTER_CALL_PAIR */
+#endif
+    /* Signifies that this instruction may need to be hot patched and should
+     * therefore not cross a cache line. It is not necessary to set this for
+     * exit cti's or linkstubs since it is mainly intended for clients etc. 
+     * Handling of this flag is not yet implemented */
+    INSTR_HOT_PATCHABLE         = 0x04000000,
+#ifdef DEBUG
+    /* case 9151: only report invalid instrs for normal code decoding */
+    INSTR_IGNORE_INVALID        = 0x08000000,
+#endif
+    /* Currently used for frozen coarse fragments with final jmps and
+     * jmps to ib stubs that are elided: we need the jmp instr there
+     * to build the linkstub_t but we do not want to emit it. */
+    INSTR_DO_NOT_EMIT           = 0x10000000,
+    /* PR 251479: re-relativization support: is instr->rip_rel_pos valid? */
+    INSTR_RIP_REL_VALID         = 0x20000000,
+#ifdef X64
+    /* PR 278329: each instr stores its own x64/x86 mode */
+    INSTR_X86_MODE              = 0x40000000,
+#endif
+    /* PR 267260: distinguish our own mangling from client-added instrs */
+    INSTR_OUR_MANGLING          = 0x80000000,
+};
+
+/* FIXME: could shrink prefixes, eflags, opcode, and flags fields
+ * this struct isn't a memory bottleneck though b/c it isn't persistent
+ */
+struct _instr_t {
+    /* flags contains the constants defined above */
+    uint    flags;
+
+    /* raw bits of length length are pointed to by the bytes field */
+    byte    *bytes;
+    uint    length;
+
+    /* translation target for this instr */
+    app_pc  translation;
+
+    uint    opcode;
+
+#ifdef X64
+    /* PR 251479: offset into instr's raw bytes of rip-relative 4-byte displacement */
+    byte    rip_rel_pos;
+#endif
+
+    /* we dynamically allocate dst and src arrays b/c x86 instrs can have
+     * up to 8 of each of them, but most have <=2 dsts and <=3 srcs, and we
+     * use this struct for un-decoded instrs too
+     */
+    byte    num_dsts;
+    byte    num_srcs;
+
+    opnd_t    *dsts;
+    /* for efficiency everyone has a 1st src opnd, since we often just
+     * decode jumps, which all have a single source (==target)
+     * yes this is an extra 10 bytes, but the whole struct is still < 64 bytes!
+     */
+    opnd_t    src0;
+    opnd_t    *srcs; /* this array has 2nd src and beyond */
+    uint    prefixes; /* data size, addr size, or lock prefix info */
+    uint    eflags;   /* contains EFLAGS_ bits, but amount of info varies
+                       * depending on how instr was decoded/built */
+
+    /* this field is for the use of passes as an annotation.
+     * it is also used to hold the offset of an instruction when encoding
+     * pc-relative instructions.
+     */
+    void *note;
+
+    /* fields for building instructions into instruction lists */
+    instr_t   *prev;
+    instr_t   *next;
+
+}; /* instr_t */
+
+/* DR_API EXPORT TOFILE dr_ir_instr.h */
+
+/* functions to inspect and manipulate the fields of an instr_t 
+ * NB: a number of instr_ routines are declared in arch_exports.h.
+ */
+
+DR_API
+/** Returns number of bytes of heap used by \p instr. */
+int
+instr_mem_usage(instr_t *instr);
+
+DR_API
+/**
+ * Returns a copy of \p orig with separately allocated memory for
+ * operands and raw bytes if they were present in \p orig.
+ */
+instr_t *
+instr_clone(dcontext_t *dcontext, instr_t *orig);
+
+DR_API
+/**
+ * Convenience routine: calls
+ * - instr_create(dcontext)
+ * - instr_set_opcode(opcode)
+ * - instr_set_num_opnds(dcontext, instr, num_dsts, num_srcs)
+ *
+ * and returns the resulting instr_t.
+ */
+instr_t *
+instr_build(dcontext_t *dcontext, int opcode, int num_dsts, int num_srcs);
+
+DR_API
+/**
+ * Convenience routine: calls 
+ * - instr_create(dcontext)
+ * - instr_set_opcode(instr, opcode)
+ * - instr_allocate_raw_bits(dcontext, instr, num_bytes)
+ *
+ * and returns the resulting instr_t.
+ */
+instr_t *
+instr_build_bits(dcontext_t *dcontext, int opcode, uint num_bytes);
+
+DR_API
+/**
+ * Returns true iff \p instr's opcode is NOT OP_INVALID.
+ * Not to be confused with an invalid opcode, which can be OP_INVALID or
+ * OP_UNDECODED.  OP_INVALID means an instruction with no valid fields:
+ * raw bits (may exist but do not correspond to a valid instr), opcode,
+ * eflags, or operands.  It could be an uninitialized
+ * instruction or the result of decoding an invalid sequence of bytes.
+ */
+bool 
+instr_valid(instr_t *instr);
+
+DR_API
+/** Get the original application PC of \p instr if it exists. */
+app_pc
+instr_get_app_pc(instr_t *instr);
+
+DR_API
+/** Returns \p instr's opcode (an OP_ constant). */
+int 
+instr_get_opcode(instr_t *instr);
+
+DR_API
+/** Assumes \p opcode is an OP_ constant and sets it to be instr's opcode. */
+void 
+instr_set_opcode(instr_t *instr, int opcode);
+
+const struct instr_info_t * 
+instr_get_instr_info(instr_t *instr);
+
+const struct instr_info_t *
+get_instr_info(int opcode);
+
+DR_API
+/**
+ * Returns the number of source operands of \p instr.
+ *
+ * \note Addressing registers used in destination memory references
+ * (i.e., base, index, or segment registers) are not separately listed
+ * as source operands.
+ */
+int 
+instr_num_srcs(instr_t *instr);
+
+DR_API
+/**
+ * Returns the number of destination operands of \p instr.
+ */
+int 
+instr_num_dsts(instr_t *instr);
+
+DR_API
+/**
+ * Assumes that \p instr has been initialized but does not have any
+ * operands yet.  Allocates storage for \p num_srcs source operands
+ * and \p num_dsts destination operands.
+ */
+void 
+instr_set_num_opnds(dcontext_t *dcontext, instr_t *instr, int num_dsts, int num_srcs);
+
+DR_API
+/**
+ * Returns \p instr's source operand at position \p pos (0-based).
+ */
+opnd_t 
+instr_get_src(instr_t *instr, uint pos);
+
+DR_API
+/**
+ * Returns \p instr's destination operand at position \p pos (0-based).
+ */
+opnd_t 
+instr_get_dst(instr_t *instr, uint pos);
+
+DR_API
+/**
+ * Sets \p instr's source operand at position \p pos to be \p opnd.
+ * Also calls instr_set_raw_bits_valid(\p instr, false) and
+ * instr_set_operands_valid(\p instr, true).
+ */
+void 
+instr_set_src(instr_t *instr, uint pos, opnd_t opnd);
+
+DR_API
+/**
+ * Sets \p instr's destination operand at position \p pos to be \p opnd.
+ * Also calls instr_set_raw_bits_valid(\p instr, false) and
+ * instr_set_operands_valid(\p instr, true).
+ */
+void 
+instr_set_dst(instr_t *instr, uint pos, opnd_t opnd);
+
+DR_API
+/**
+ * Assumes that \p cti_instr is a control transfer instruction
+ * Returns the first source operand of \p cti_instr (its target).
+ */
+opnd_t 
+instr_get_target(instr_t *cti_instr);
+
+DR_API
+/**
+ * Assumes that \p cti_instr is a control transfer instruction.
+ * Sets the first source operand of \p cti_instr to be \p target.
+ * Also calls instr_set_raw_bits_valid(\p instr, false) and
+ * instr_set_operands_valid(\p instr, true).
+ */
+void 
+instr_set_target(instr_t *cti_instr, opnd_t target);
+
+DR_API
+/** Returns true iff \p instr's operands are up to date. */
+bool 
+instr_operands_valid(instr_t *instr);
+
+DR_API
+/** Sets \p instr's operands to be valid if \p valid is true, invalid otherwise. */
+void 
+instr_set_operands_valid(instr_t *instr, bool valid);
+
+DR_API
+/**
+ * Returns true iff \p instr's opcode is valid.
+ * If the opcode is ever set to other than OP_INVALID or OP_UNDECODED it is assumed
+ * to be valid.  However, calling instr_get_opcode() will attempt to
+ * decode a valid opcode, hence the purpose of this routine.
+ */
+bool 
+instr_opcode_valid(instr_t *instr);
+
+/******************************************************************
+ * Eflags validity is not exported!  It's hidden.  Calling get_eflags or
+ * get_arith_flags will make them valid if they're not.
+ */
+
+bool 
+instr_arith_flags_valid(instr_t *instr);
+
+/* Sets instr's arithmetic flags (the 6 bottom eflags) to be valid if
+ * valid is true, invalid otherwise. */
+void 
+instr_set_arith_flags_valid(instr_t *instr, bool valid);
+
+/* Returns true iff instr's eflags are up to date. */
+bool 
+instr_eflags_valid(instr_t *instr);
+
+/* Sets instr's eflags to be valid if valid is true, invalid otherwise. */
+void 
+instr_set_eflags_valid(instr_t *instr, bool valid);
+
+DR_API
+/** Returns \p instr's eflags use as EFLAGS_ constants or'ed together. */
+uint 
+instr_get_eflags(instr_t *instr);
+
+DR_API
+/** Returns the eflags usage of instructions with opcode \p opcode,
+ * as EFLAGS_ constants or'ed together.
+ */
+uint 
+instr_get_opcode_eflags(int opcode);
+
+DR_API
+/**
+ * Returns \p instr's arithmetic flags (bottom 6 eflags) use 
+ * as EFLAGS_ constants or'ed together.
+ * If \p instr's eflags behavior has not been calculated yet or is
+ * invalid, the entire eflags use is calculated and returned (not
+ * just the arithmetic flags).
+ */
+uint 
+instr_get_arith_flags(instr_t *instr);
+
+/*
+ ******************************************************************/
+
+DR_API
+/**
+ * Assumes that \p instr does not currently have any raw bits allocated.
+ * Sets \p instr's raw bits to be \p length bytes starting at \p addr.
+ * Does not set the operands invalid.
+ */
+void 
+instr_set_raw_bits(instr_t *instr, byte * addr, uint length);
+
+DR_API
+/** Sets \p instr's raw bits to be valid if \p valid is true, invalid otherwise. */
+void 
+instr_set_raw_bits_valid(instr_t *instr, bool valid);
+
+DR_API
+/** Returns true iff \p instr's raw bits are a valid encoding of instr. */
+bool 
+instr_raw_bits_valid(instr_t *instr);
+
+DR_API
+/** Returns true iff \p instr has its own allocated memory for raw bits. */
+bool 
+instr_has_allocated_bits(instr_t *instr);
+
+DR_API
+/** Returns true iff \p instr's raw bits are not a valid encoding of \p instr. */
+bool 
+instr_needs_encoding(instr_t *instr);
+
+DR_API
+/**
+ * Allocates \p num_bytes of memory for \p instr's raw bits.
+ * If \p instr currently points to raw bits, the allocated memory is
+ * initialized with the bytes pointed to.
+ * \p instr is then set to point to the allocated memory.
+ */
+void 
+instr_allocate_raw_bits(dcontext_t *dcontext, instr_t *instr, uint num_bytes);
+
+DR_API
+/**
+ * Sets the translation pointer for \p instr, used to recreate the
+ * application address corresponding to this instruction.  When adding
+ * or modifying instructions that are to be considered application
+ * instructions (i.e., non meta-instructions: see
+ * #instr_ok_to_mangle), the translation should always be set.  Pick
+ * the application address that if executed will be equivalent to
+ * restarting \p instr.
+ * Returns the supplied \p instr (for easy chaining).  Use
+ * #instr_get_app_pc to see the current value of the translation.
+ */
+instr_t * 
+instr_set_translation(instr_t *instr, app_pc addr);
+
+DR_UNS_API
+/**
+ * If the translation pointer is set for \p instr, returns that
+ * else returns NULL.
+ * \note The translation pointer is not automatically set when
+ * decoding instructions from raw bytes (via decode(), e.g.); it is
+ * set for instructions in instruction lists generated by DR (see
+ * dr_register_bb_event()).
+ * 
+ */
+app_pc 
+instr_get_translation(instr_t *instr);
+
+DR_API
+/**
+ * Calling this function with \p instr makes it safe to keep the
+ * instruction around indefinitely when its raw bits point into the
+ * cache.  The function allocates memory local to \p instr to hold a
+ * copy of the raw bits. If this was not done, the original raw bits
+ * could be deleted at some point.  Making an instruction persistent
+ * is neccesary if you want to keep it beyond returning from the call
+ * that produced the instruction.
+ */
+void
+instr_make_persistent(dcontext_t *dcontext, instr_t *instr);
+
+DR_API
+/**
+ * Assumes that \p instr's raw bits are valid.
+ * Returns a pointer to \p instr's raw bits.
+ * \note A freshly-decoded instruction has valid raw bits that point to the
+ * address from which it was decoded.
+ */
+byte *
+instr_get_raw_bits(instr_t *instr);
+
+DR_API
+/** If \p instr has raw bits allocated, frees them. */
+void
+instr_free_raw_bits(dcontext_t *dcontext, instr_t *instr);
+
+DR_API
+/**
+ * Assumes that \p instr's raw bits are valid and have > \p pos bytes.
+ * Returns a pointer to \p instr's raw byte at position \p pos (beginning with 0).
+ */
+byte 
+instr_get_raw_byte(instr_t *instr, uint pos);
+
+DR_API
+/**
+ * Assumes that \p instr's raw bits are valid and allocated by \p instr
+ * and have > \p pos bytes.
+ * Sets instr's raw byte at position \p pos (beginning with 0) to the value \p byte.
+ */
+void 
+instr_set_raw_byte(instr_t *instr, uint pos, byte byte);
+
+DR_API
+/**
+ * Assumes that \p instr's raw bits are valid and allocated by \p instr
+ * and have >= num_bytes bytes.
+ * Copies the \p num_bytes beginning at start to \p instr's raw bits.
+ */
+void 
+instr_set_raw_bytes(instr_t *instr, byte *start, uint num_bytes);
+
+DR_API
+/**
+ * Assumes that \p instr's raw bits are valid and allocated by \p instr
+ * and have > pos+3 bytes.
+ * Sets the 4 bytes beginning at position \p pos (0-based) to the value word.
+ */
+void 
+instr_set_raw_word(instr_t *instr, uint pos, uint word);
+
+DR_API
+/**
+ * Assumes that \p instr's raw bits are valid and have > \p pos + 3 bytes.
+ * Returns the 4 bytes beginning at position \p pos (0-based).
+ */
+uint 
+instr_get_raw_word(instr_t *instr, uint pos);
+
+DR_API
+/**
+ * Assumes that \p prefix is a PREFIX_ constant.
+ * Ors \p instr's prefixes with \p prefix.
+ * Returns the supplied instr (for easy chaining).
+ */
+instr_t *
+instr_set_prefix_flag(instr_t *instr, uint prefix);
+
+DR_API
+/**
+ * Assumes that \p prefix is a PREFIX_ constant.
+ * Returns true if \p instr's prefixes contain the flag \p prefix.
+ */
+bool 
+instr_get_prefix_flag(instr_t *instr, uint prefix);
+
+/* NOT EXPORTED because we want to limit a client to seeing only the
+ * handful of PREFIX_ flags we're exporting.
+ * Assumes that prefix is a group of PREFIX_ constants or-ed together.
+ * Sets instr's prefixes to be exactly those flags in prefixes.
+ */
+void 
+instr_set_prefixes(instr_t *instr, uint prefixes);
+
+/* NOT EXPORTED because we want to limit a client to seeing only the
+ * handful of PREFIX_ flags we're exporting.
+ * Returns instr's prefixes as PREFIX_ constants or-ed together.
+ */
+uint 
+instr_get_prefixes(instr_t *instr);
+
+/* DR_API EXPORT BEGIN */
+#ifdef X64
+/* DR_API EXPORT END */
+DR_API
+/**
+ * Each instruction stores whether it should be interpreted in 32-bit
+ * (x86) or 64-bit (x64) mode.  This routine sets the mode for \p instr.
+ *
+ * \note For 64-bit DR builds only.
+ */
+void
+instr_set_x86_mode(instr_t *instr, bool x86);
+
+DR_API
+/**
+ * Returns true if \p instr is an x86 instruction (32-bit) and false
+ * if \p instr is an x64 instruction (64-bit).
+ *
+ * \note For 64-bit DR builds only.
+ */
+bool 
+instr_get_x86_mode(instr_t *instr);
+/* DR_API EXPORT BEGIN */
+#endif
+/* DR_API EXPORT END */
+
+/***********************************************************************/
+/* decoding routines */
+
+DR_UNS_API
+/**
+ * If instr is at Level 0 (i.e., a bundled group of instrs as raw bits),
+ * expands instr into a sequence of Level 1 instrs using decode_raw() which
+ * are added in place to ilist.
+ * Returns the replacement of instr, if any expansion is performed
+ * (in which case the old instr is destroyed); otherwise returns
+ * instr unchanged.
+ */
+instr_t *
+instr_expand(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr);
+
+DR_UNS_API
+/**
+ * Returns true if instr is at Level 0 (i.e. a bundled group of instrs
+ * as raw bits).
+ */
+bool
+instr_is_level_0(instr_t *inst);
+
+DR_UNS_API
+/**
+ * If the next instr is at Level 0 (i.e., a bundled group of instrs as raw bits),
+ * expands it into a sequence of Level 1 instrs using decode_raw() which
+ * are added in place to ilist.  Then returns the new next instr.
+ */
+instr_t *
+instr_get_next_expanded(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr);
+
+DR_UNS_API
+/**
+ * If the prev instr is at Level 0 (i.e., a bundled group of instrs as raw bits),
+ * expands it into a sequence of Level 1 instrs using decode_raw() which
+ * are added in place to ilist.  Then returns the new prev instr.
+ */
+instr_t *
+instr_get_prev_expanded(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr);
+
+DR_UNS_API
+/**
+ * If instr is not already at the level of decode_cti, decodes enough
+ * from the raw bits pointed to by instr to bring it to that level.
+ * Assumes that instr is a single instr (i.e., NOT Level 0).
+ *
+ * decode_cti decodes only enough of instr to determine
+ * its size, its effects on the 6 arithmetic eflags, and whether it is
+ * a control-transfer instruction.  If it is, the operands fields of
+ * instr are filled in.  If not, only the raw bits fields of instr are
+ * filled in.  This corresponds to a Level 3 decoding for control
+ * transfer instructions but a Level 1 decoding plus arithmetic eflags
+ * information for all other instructions.
+ */
+void
+instr_decode_cti(dcontext_t *dcontext, instr_t *instr);
+
+DR_UNS_API
+/**
+ * If instr is not already at the level of decode_opcode, decodes enough
+ * from the raw bits pointed to by instr to bring it to that level.
+ * Assumes that instr is a single instr (i.e., NOT Level 0).
+ *
+ * decode_opcode decodes the opcode and eflags usage of the instruction.
+ * This corresponds to a Level 2 decoding.
+ */
+void
+instr_decode_opcode(dcontext_t *dcontext, instr_t *instr);
+
+DR_UNS_API
+/**
+ * If instr is not already fully decoded, decodes enough
+ * from the raw bits pointed to by instr to bring it Level 3.
+ * Assumes that instr is a single instr (i.e., NOT Level 0).
+ */
+void
+instr_decode(dcontext_t *dcontext, instr_t *instr);
+
+/* DR_API EXPORT TOFILE dr_ir_instrlist.h */
+DR_UNS_API
+/**
+ * If the first instr is at Level 0 (i.e., a bundled group of instrs as raw bits),
+ * expands it into a sequence of Level 1 instrs using decode_raw() which
+ * are added in place to ilist.  Then returns the new first instr.
+ */
+instr_t*
+instrlist_first_expanded(dcontext_t *dcontext, instrlist_t *ilist);
+
+DR_UNS_API
+/**
+ * If the last instr is at Level 0 (i.e., a bundled group of instrs as raw bits),
+ * expands it into a sequence of Level 1 instrs using decode_raw() which
+ * are added in place to ilist.  Then returns the new last instr.
+ */
+instr_t*
+instrlist_last_expanded(dcontext_t *dcontext, instrlist_t *ilist);
+
+DR_UNS_API
+/**
+ * Brings all instrs in ilist up to the decode_cti level, and
+ * hooks up intra-ilist cti targets to use instr_t targets, by
+ * matching pc targets to each instruction's raw bits.
+ *
+ * decode_cti decodes only enough of instr to determine
+ * its size, its effects on the 6 arithmetic eflags, and whether it is
+ * a control-transfer instruction.  If it is, the operands fields of
+ * instr are filled in.  If not, only the raw bits fields of instr are
+ * filled in.  This corresponds to a Level 3 decoding for control
+ * transfer instructions but a Level 1 decoding plus arithmetic eflags
+ * information for all other instructions.
+ */
+void 
+instrlist_decode_cti(dcontext_t *dcontext, instrlist_t *ilist);
+/* DR_API EXPORT TOFILE dr_ir_instr.h */
+
+/***********************************************************************/
+/* utility functions */
+
+DR_API
+/**
+ * Shrinks all registers not used as addresses, and all immed integer and
+ * address sizes, to 16 bits.
+ * Does not shrink REG_ESI or REG_EDI used in string instructions.
+ */
+void 
+instr_shrink_to_16_bits(instr_t *instr);
+
+/* DR_API EXPORT BEGIN */
+#ifdef X64
+/* DR_API EXPORT END */
+DR_API
+/**
+ * Shrinks all registers, including addresses, and all immed integer and
+ * address sizes, to 32 bits.
+ *
+ * \note For 64-bit DR builds only.
+ */
+void 
+instr_shrink_to_32_bits(instr_t *instr);
+/* DR_API EXPORT BEGIN */
+#endif
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Assumes that \p reg is a REG_ constant.
+ * Returns true iff at least one of \p instr's operands references a
+ * register that overlaps \p reg.
+ */
+bool 
+instr_uses_reg(instr_t *instr, reg_id_t reg);
+
+DR_API
+/**
+ * Returns true iff at least one of \p instr's operands references a floating
+ * point register.
+ */
+bool 
+instr_uses_fp_reg(instr_t *instr);
+
+DR_API
+/**
+ * Assumes that \p reg is a REG_ constant.
+ * Returns true iff at least one of \p instr's source operands references \p reg.
+ *
+ * \note Use instr_reads_from_reg() to also consider addressing
+ * registers in destination operands.
+ */
+bool 
+instr_reg_in_src(instr_t *instr, reg_id_t reg);
+
+DR_API
+/**
+ * Assumes that \p reg is a REG_ constant.
+ * Returns true iff at least one of \p instr's destination operands references \p reg.
+ */
+bool 
+instr_reg_in_dst(instr_t *instr, reg_id_t reg);
+
+DR_API
+/**
+ * Assumes that \p reg is a REG_ constant.
+ * Returns true iff at least one of \p instr's destination operands is
+ * a register operand for a register that overlaps \p reg.
+ */
+bool 
+instr_writes_to_reg(instr_t *instr, reg_id_t reg);
+
+DR_API
+/**
+ * Assumes that reg is a REG_ constant.
+ * Returns true iff at least one of instr's operands reads
+ * from a register that overlaps reg (checks both source operands
+ * and addressing registers used in destination operands).
+ */
+bool 
+instr_reads_from_reg(instr_t *instr, reg_id_t reg);
+
+DR_API 
+/**
+ * Assumes that \p reg is a REG_ constant.
+ * Returns true iff at least one of \p instr's destination operands is
+ * the same register (not enough to just overlap) as \p reg.
+ */
+bool
+instr_writes_to_exact_reg(instr_t *instr, reg_id_t reg);
+
+DR_API
+/**
+ * Replaces all instances of \p old_opnd in \p instr's source operands with
+ * \p new_opnd (uses opnd_same() to detect sameness).
+ */
+bool 
+instr_replace_src_opnd(instr_t *instr, opnd_t old_opnd, opnd_t new_opnd);
+
+DR_API
+/**
+ * Returns true iff \p instr1 and \p instr2 have the same opcode, prefixes,
+ * and source and destination operands (uses opnd_same() to compare the operands).
+ */
+bool 
+instr_same(instr_t *instr1, instr_t *instr2);
+
+DR_API
+/** Returns true iff any of \p instr's source operands is a memory reference. */
+bool 
+instr_reads_memory(instr_t *instr);
+
+DR_API
+/** Returns true iff any of \p instr's destination operands is a memory reference. */
+bool 
+instr_writes_memory(instr_t *instr);
+
+/* DR_API EXPORT BEGIN */
+#ifdef X64
+/* DR_API EXPORT END */
+DR_API
+/**
+ * Returns true iff any of \p instr's operands is a rip-relative memory reference. 
+ *
+ * \note For 64-bit DR builds only.
+ */
+bool 
+instr_has_rel_addr_reference(instr_t *instr);
+
+DR_API
+/**
+ * If any of \p instr's operands is a rip-relative memory reference, returns the
+ * address that reference targets.  Else returns false.
+ *
+ * \note For 64-bit DR builds only.
+ */
+bool
+instr_get_rel_addr_target(instr_t *instr, /*OUT*/ app_pc *target);
+
+DR_API
+/**
+ * If any of \p instr's destination operands is a rip-relative memory
+ * reference, returns the operand position.  If there is no such
+ * destination operand, returns -1.
+ *
+ * \note For 64-bit DR builds only.
+ */
+int
+instr_get_rel_addr_dst_idx(instr_t *instr);
+
+DR_API
+/**
+ * If any of \p instr's source operands is a rip-relative memory
+ * reference, returns the operand position.  If there is no such
+ * source operand, returns -1.
+ *
+ * \note For 64-bit DR builds only.
+ */
+int
+instr_get_rel_addr_src_idx(instr_t *instr);
+
+/* We're not exposing the low-level rip_rel_pos routines directly to clients,
+ * who should only use this level 1-3 feature via decode_cti + encode.
+ */
+
+/* Returns true iff instr's raw bits are valid and the offset within
+ * those raw bits of a rip-relative displacement is set (to 0 if there
+ * is no such displacement).
+ */
+bool
+instr_rip_rel_valid(instr_t *instr);
+
+/* Sets whether instr's rip-relative displacement offset is valid. */
+void
+instr_set_rip_rel_valid(instr_t *instr, bool valid);
+
+/* Assumes that instr_rip_rel_valid() is true.
+ * Returns the offset within the encoded bytes of instr of the
+ * displacement used for rip-relative addressing; returns 0
+ * if instr has no rip-relative operand.
+ * There can be at most one rip-relative operand in one instruction.
+ */
+uint
+instr_get_rip_rel_pos(instr_t *instr);
+
+/* Sets the offset within instr's encoded bytes of instr's
+ * rip-relative displacement (the offset should be 0 if there is no
+ * rip-relative operand) and marks it valid.  \p pos must be less
+ * than 256.
+ */
+void
+instr_set_rip_rel_pos(instr_t *instr, uint pos);
+/* DR_API EXPORT BEGIN */
+#endif /* X64 */
+/* DR_API EXPORT END */
+
+/* not exported: for PR 267260 */
+bool
+instr_is_our_mangling(instr_t *instr);
+
+/* Sets whether instr came from our mangling. */
+void
+instr_set_our_mangling(instr_t *instr, bool ours);
+
+
+DR_API
+/**
+ * Returns NULL if none of \p instr's operands is a memory reference.
+ * Otherwise, returns the effective address of the first memory operand
+ * when the operands are considered in this order: destinations and then
+ * sources.  The address is computed using the passed-in registers.
+ */
+app_pc
+instr_compute_address(instr_t *instr, dr_mcontext_t *mc);
+
+DR_API
+/**
+ * Calculates the size, in bytes, of the memory read or write of \p instr.
+ * If \p instr does not reference memory, or is invalid, returns 0.
+ * If \p instr is a repeated string instruction, considers only one iteration.
+ */
+uint
+instr_memory_reference_size(instr_t *instr);
+
+/* DR_API EXPORT TOFILE dr_ir_utils.h */
+/* DR_API EXPORT BEGIN */
+
+/***************************************************************************
+ * DECODE / DISASSEMBLY ROUTINES
+ */
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Calculates the size, in bytes, of the memory read or write of
+ * the instr at \p pc.  If the instruction is a repeating string instruction,
+ * considers only one iteration.
+ * Returns the pc of the following instruction.
+ * If the instruction at \p pc does not reference memory, or is invalid, 
+ * returns NULL.
+ */
+app_pc
+decode_memory_reference_size(dcontext_t *dcontext, app_pc pc, uint *size_in_bytes);
+
+/* DR_API EXPORT TOFILE dr_ir_instr.h */
+DR_API
+/**
+ * Returns true iff \p instr is an IA-32 "mov" instruction: either OP_mov_st,
+ * OP_mov_ld, OP_mov_imm, OP_mov_seg, or OP_mov_priv.
+ */
+bool 
+instr_is_mov(instr_t *instr);
+
+DR_API
+/**
+ * Returns true iff \p instr's opcode is OP_call, OP_call_far, OP_call_ind,
+ * or OP_call_far_ind.
+ */
+bool 
+instr_is_call(instr_t *instr);
+
+DR_API
+/** Returns true iff \p instr's opcode is OP_call or OP_call_far. */
+bool 
+instr_is_call_direct(instr_t *instr);
+
+DR_API
+/** Returns true iff \p instr's opcode is OP_call_ind or OP_call_far_ind. */
+bool 
+instr_is_call_indirect(instr_t *instr);
+
+DR_API
+/** Returns true iff \p instr's opcode is OP_ret, OP_ret_far, or OP_iret. */
+bool 
+instr_is_return(instr_t *instr);
+
+DR_API
+/**
+ * Returns true iff \p instr is a control transfer instruction of any kind
+ * This includes OP_jcc, OP_jcc_short, OP_loop*, OP_jecxz, OP_call*, and OP_jmp*.
+ */
+bool 
+instr_is_cti(instr_t *instr);
+
+DR_API
+/**
+ * Returns true iff \p instr is a control transfer instruction that takes an
+ * 8-bit offset: OP_loop*, OP_jecxz, OP_jmp_short, or OP_jcc_short
+ */
+#ifdef UNSUPPORTED_API
+/**
+ * This routine does NOT try to decode an opcode in a Level 1 or Level
+ * 0 routine, and can thus be called on Level 0 routines.  
+ */
+#endif
+bool 
+instr_is_cti_short(instr_t *instr);
+
+DR_API
+/** Returns true iff \p instr is one of OP_loop* or OP_jecxz. */
+bool 
+instr_is_cti_loop(instr_t *instr);
+
+DR_API
+/**
+ * Returns true iff \p instr's opcode is OP_loop* or OP_jecxz and instr has
+ * been transformed to a sequence of instruction that will allow a 32-bit
+ * offset.
+ * If \p pc != NULL, \p pc is expected to point the the beginning of the encoding of
+ * \p instr, and the following instructions are assumed to be encoded in sequence
+ * after \p instr.
+ * Otherwise, the encoding is expected to be found in \p instr's allocated bits.
+ */
+#ifdef UNSUPPORTED_API
+/**
+ * This routine does NOT try to decode an opcode in a Level 1 or Level
+ * 0 routine, and can thus be called on Level 0 routines.  
+ */
+#endif
+bool 
+instr_is_cti_short_rewrite(instr_t *instr, byte *pc);
+
+byte *
+remangle_short_rewrite(dcontext_t *dcontext, instr_t *instr, byte *pc, app_pc target);
+
+DR_API
+/**
+ * Returns true iff \p instr is a conditional branch: OP_jcc, OP_jcc_short,
+ * OP_loop*, or OP_jecxz.
+ */
+bool 
+instr_is_cbr(instr_t *instr);
+
+DR_API
+/**
+ * Returns true iff \p instr is a multi-way (indirect) branch: OP_jmp_ind,
+ * OP_call_ind, OP_ret, OP_jmp_far_ind, OP_call_far_ind, OP_ret_far, or
+ * OP_iret.
+ */
+bool 
+instr_is_mbr(instr_t *instr);
+
+DR_API
+/**
+ * Returns true iff \p instr is an unconditional direct branch: OP_jmp,
+ * OP_jmp_short, or OP_jmp_far.
+ */
+bool 
+instr_is_ubr(instr_t *instr);
+
+DR_API
+/**
+ * Returns true iff \p instr is a far control transfer instruction: OP_jmp_far,
+ * OP_call_far, OP_jmp_far_ind, OP_call_far_ind, OP_ret_far, or OP_iret.
+ */
+bool 
+instr_is_far_cti(instr_t *instr);
+
+DR_API
+/** Returns true if \p instr is an absolute call or jmp that is far. */
+bool 
+instr_is_far_abs_cti(instr_t *instr);
+
+DR_API
+/**
+ * Returns true iff \p instr is used to implement system calls: OP_int with a
+ * source operand of 0x80 on linux or 0x2e on windows, or OP_sysenter,
+ * or OP_syscall, or #instr_is_wow64_syscall() for WOW64.
+ */
+bool 
+instr_is_syscall(instr_t *instr);
+
+/* DR_API EXPORT BEGIN */
+#ifdef WINDOWS 
+/* DR_API EXPORT END */
+DR_API
+/**
+ * Returns true iff \p instr is the indirect transfer from the 32-bit
+ * ntdll.dll to the wow64 system call emulation layer.  This
+ * instruction will also return true for instr_is_syscall, as well as
+ * appear as an indirect call, so clients modifying indirect calls may
+ * want to avoid modifying this type.
+ *
+ * \note Windows-only
+ */
+bool
+instr_is_wow64_syscall(instr_t *instr);
+/* DR_API EXPORT BEGIN */
+#endif
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Returns true iff \p instr is a prefetch instruction: OP_prefetchnta,
+ * OP_prefetchnt0, OP_prefetchnt1, OP_prefetchnt2, OP_prefetch, or
+ * OP_prefetchw.
+ */
+bool 
+instr_is_prefetch(instr_t *instr);
+
+DR_API
+/**
+ * Tries to identify common cases of moving a constant into either a
+ * register or a memory address.
+ * Returns true and sets \p *value to the constant being moved for the following
+ * cases: mov_imm, mov_st, and xor where the source equals the destination.
+ */
+bool 
+instr_is_mov_constant(instr_t *instr, ptr_int_t *value);
+
+DR_API
+/** Returns true iff \p instr is a floating point instruction. */
+bool 
+instr_is_floating(instr_t *instr);
+
+DR_API
+/** Returns true iff \p instr is part of Intel's MMX instructions. */
+bool 
+instr_is_mmx(instr_t *instr);
+
+DR_API
+/** Returns true iff \p instr is part of Intel's SSE or SSE2 instructions. */
+bool 
+instr_is_sse_or_sse2(instr_t *instr);
+
+DR_API
+/** Returns true iff \p instr is a "mov $imm -> (%esp)". */
+bool 
+instr_is_mov_imm_to_tos(instr_t *instr);
+
+DR_API
+/** Returns true iff \p instr is a label meta-instruction. */
+bool 
+instr_is_label(instr_t *instr);
+
+DR_API
+/**
+ * Assumes that \p instr's opcode is OP_int and that either \p instr's
+ * operands or its raw bits are valid.
+ * Returns the first source operand if \p instr's operands are valid,
+ * else if \p instr's raw bits are valid returns the first raw byte.
+ */
+int 
+instr_get_interrupt_number(instr_t *instr);
+
+DR_API
+/**
+ * Assumes that \p instr is a conditional branch instruction
+ * Reverses the logic of \p instr's conditional
+ * e.g., changes OP_jb to OP_jnb.
+ * Works on cti_short_rewrite as well.
+ */
+void 
+instr_invert_cbr(instr_t *instr);
+
+DR_API
+/**
+ * Assumes that instr is a meta instruction (!instr_ok_to_mangle())
+ * and an instr_is_cti_short() (8-bit reach).  Converts instr's opcode
+ * to a long form (32-bit reach).  If instr's opcode is OP_loop* or
+ * OP_jecxz, converts it to a sequence of multiple instructions (which
+ * is different from instr_is_cti_short_rewrite()).  Each added instruction
+ * is marked !instr_ok_to_mangle().
+ * \note DR automatically converts non-meta short ctis to long form.
+ */
+void 
+instr_convert_short_meta_jmp_to_long(dcontext_t *dcontext, instrlist_t *ilist,
+                                     instr_t *instr);
+
+DR_API
+/** 
+ * Given \p eflags, returns whether or not the conditional branch, \p
+ * instr, would be taken.
+ */
+bool
+instr_jcc_taken(instr_t *instr, reg_t eflags);
+
+/* Given a machine state, returns whether or not the cbr instr would be taken
+ * if the state is before execution (pre == true) or after (pre == false).
+ * (not exported since machine state isn't)
+ */
+bool
+instr_cbr_taken(instr_t *instr, dr_mcontext_t *mcontext, bool pre);
+
+/* utility routines that are in optimize.c */
+opnd_t 
+instr_get_src_mem_access(instr_t *instr);
+
+void 
+loginst(dcontext_t *dcontext, uint level, instr_t *instr, char *string);
+
+void 
+logopnd(dcontext_t *dcontext, uint level, opnd_t opnd, char *string);
+
+DR_API
+/**
+ * Returns true if \p instr is one of a class of common nops.
+ * currently checks:
+ * - xchg reg, reg
+ * - mov reg, reg
+ * - lea reg, (reg)
+ */
+bool 
+instr_is_nop(instr_t *instr);
+
+DR_UNS_API
+/**
+ * Convenience routine to create a nop of a certain size.  If \p raw
+ * is true, sets raw bytes rather than filling in the operands or opcode.
+ */
+instr_t *
+instr_create_nbyte_nop(dcontext_t *dcontext, uint num_bytes, bool raw);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated on the
+ * thread-local heap with opcode \p opcode and no sources or destinations.
+ */
+instr_t *
+instr_create_0dst_0src(dcontext_t *dcontext, int opcode);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated on the
+ * thread-local heap with opcode \p opcode and a single source (\p src).
+ */
+instr_t *
+instr_create_0dst_1src(dcontext_t *dcontext, int opcode,
+                       opnd_t src);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated on the
+ * thread-local heap with opcode \p opcode and two sources (\p src1, \p src2).
+ */
+instr_t *
+instr_create_0dst_2src(dcontext_t *dcontext, int opcode,
+                       opnd_t src1, opnd_t src2);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated
+ * on the thread-local heap with opcode \p opcode and three sources
+ * (\p src1, \p src2, \p src3).
+ */
+instr_t * 
+instr_create_0dst_3src(dcontext_t *dcontext, int opcode,
+                       opnd_t src1, opnd_t src2, opnd_t src3);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated on the
+ * thread-local heap with opcode \p opcode and one destination (\p dst).
+ */
+instr_t *
+instr_create_1dst_0src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated on the
+ * thread-local heap with opcode \p opcode, one destination(\p dst),
+ * and one source (\p src).
+ */
+instr_t *
+instr_create_1dst_1src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst, opnd_t src);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated on the
+ * thread-local heap with opcode \p opcode, one destination (\p dst),
+ * and two sources (\p src1, \p src2).
+ */
+instr_t *
+instr_create_1dst_2src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst, opnd_t src1, opnd_t src2);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated on the
+ * thread-local heap with opcode \p opcode, one destination (\p dst),
+ * and three sources (\p src1, \p src2, \p src3).
+ */
+instr_t *
+instr_create_1dst_3src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst, opnd_t src1, opnd_t src2, opnd_t src3);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated on the
+ * thread-local heap with opcode \p opcode, one destination (\p dst),
+ * and five sources (\p src1, \p src2, \p src3, \p src4, \p src5).
+ */
+instr_t * 
+instr_create_1dst_5src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst, opnd_t src1, opnd_t src2, opnd_t src3,
+                       opnd_t src4, opnd_t src5);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated on the
+ * thread-local heap with opcode \p opcode, two destinations (\p dst1, \p dst2)
+ * and no sources.
+ */
+instr_t *
+instr_create_2dst_0src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated on the
+ * thread-local heap with opcode \p opcode, two destinations (\p dst1, \p dst2)
+ * and one source (\p src).
+ */
+instr_t *
+instr_create_2dst_1src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2, opnd_t src);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated on the
+ * thread-local heap with opcode \p opcode, two destinations (\p dst1, \p dst2)
+ * and two sources (\p src1, \p src2).
+ */
+instr_t *
+instr_create_2dst_2src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2, opnd_t src1, opnd_t src2);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated on the
+ * thread-local heap with opcode \p opcode, two destinations (\p dst1, \p dst2)
+ * and three sources (\p src1, \p src2, \p src3).
+ */
+instr_t *
+instr_create_2dst_3src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2,
+                       opnd_t src1, opnd_t src2, opnd_t src3);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated on the
+ * thread-local heap with opcode \p opcode, two destinations (\p dst1, \p dst2)
+ * and four sources (\p src1, \p src2, \p src3, \p src4).
+ */
+instr_t *
+instr_create_2dst_4src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2,
+                       opnd_t src1, opnd_t src2, opnd_t src3, opnd_t src4);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated
+ * on the thread-local heap with opcode \p opcode, three destinations
+ * (\p dst1, \p dst2, \p dst3) and no sources.
+ */
+instr_t *
+instr_create_3dst_0src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2, opnd_t dst3);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated
+ * on the thread-local heap with opcode \p opcode, three destinations
+ * (\p dst1, \p dst2, \p dst3) and three sources 
+ * (\p src1, \p src2, \p src3).
+ */
+instr_t *
+instr_create_3dst_3src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2, opnd_t dst3,
+                       opnd_t src1, opnd_t src2, opnd_t src3);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated
+ * on the thread-local heap with opcode \p opcode, three destinations
+ * (\p dst1, \p dst2, \p dst3) and four sources 
+ * (\p src1, \p src2, \p src3, \p src4).
+ */
+instr_t *
+instr_create_3dst_4src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2, opnd_t dst3,
+                       opnd_t src1, opnd_t src2, opnd_t src3, opnd_t src4);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated
+ * on the thread-local heap with opcode \p opcode, three destinations
+ * (\p dst1, \p dst2, \p dst3) and five sources
+ * (\p src1, \p src2, \p src3, \p src4, \p src5).
+ */
+instr_t *
+instr_create_3dst_5src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2, opnd_t dst3,
+                       opnd_t src1, opnd_t src2, opnd_t src3,
+                       opnd_t src4, opnd_t src5);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated
+ * on the thread-local heap with opcode \p opcode, four destinations
+ * (\p dst1, \p dst2, \p dst3, \p dst4) and 1 source (\p src).
+ */
+instr_t *
+instr_create_4dst_1src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2, opnd_t dst3, opnd_t dst4,
+                       opnd_t src);
+
+DR_API
+/**
+ * Convenience routine that returns an initialized instr_t allocated
+ * on the thread-local heap with opcode \p opcode, four destinations
+ * (\p dst1, \p dst2, \p dst3, \p dst4) and four sources
+ * (\p src1, \p src2, \p src3, \p src4).
+ */
+instr_t *
+instr_create_4dst_4src(dcontext_t *dcontext, int opcode,
+                       opnd_t dst1, opnd_t dst2, opnd_t dst3, opnd_t dst4,
+                       opnd_t src1, opnd_t src2, opnd_t src3, opnd_t src4);
+
+DR_API
+/** Convenience routine that returns an initialized instr_t for OP_popa. */
+instr_t *
+instr_create_popa(dcontext_t *dcontext);
+
+DR_API
+/** Convenience routine that returns an initialized instr_t for OP_pusha. */
+instr_t *
+instr_create_pusha(dcontext_t *dcontext);
+
+/* build instructions from raw bits */
+
+DR_UNS_API
+/**
+ * Convenience routine that returns an initialized instr_t with invalid operands
+ * and allocated raw bits with 1 byte (byte).
+ */
+instr_t *
+instr_create_raw_1byte(dcontext_t *dcontext, byte byte);
+
+DR_UNS_API
+/**
+ * Convenience routine that returns an initialized instr_t with invalid operands
+ * and allocated raw bits with 2 bytes (byte1, byte2).
+ */
+instr_t *
+instr_create_raw_2bytes(dcontext_t *dcontext, byte byte1, byte byte2);
+
+DR_UNS_API
+/**
+ * Convenience routine that returns an initialized instr_t with invalid operands
+ * and allocated raw bits with 3 bytes (byte1, byte2, byte3).
+ */
+instr_t *
+instr_create_raw_3bytes(dcontext_t *dcontext, byte byte1,
+                        byte byte2, byte byte3);
+
+DR_UNS_API
+/**
+ * Convenience routine that returns an initialized instr_t with invalid operands
+ * and allocated raw bits with 4 bytes (byte1, byte2, byte3, byte4).
+ */
+instr_t *
+instr_create_raw_4bytes(dcontext_t *dcontext, byte byte1,
+                        byte byte2, byte byte3, byte byte4);
+
+DR_UNS_API
+/**
+ * Convenience routine that returns an initialized instr_t with invalid operands
+ * and allocated raw bits with 5 bytes (byte1, byte2, byte3, byte4, byte5).
+ */
+instr_t *
+instr_create_raw_5bytes(dcontext_t *dcontext, byte byte1,
+                        byte byte2, byte byte3, byte byte4, byte byte5);
+
+DR_UNS_API
+/**
+ * Convenience routine that returns an initialized instr_t with invalid operands
+ * and allocated raw bits with 6 bytes (byte1, byte2, byte3, byte4, byte5, byte6).
+ */
+instr_t *
+instr_create_raw_6bytes(dcontext_t *dcontext, byte byte1, byte byte2,
+                        byte byte3, byte byte4, byte byte5, byte byte6);
+
+DR_UNS_API
+/**
+ * Convenience routine that returns an initialized instr_t with invalid operands
+ * and allocated raw bits with 7 bytes (byte1, byte2, byte3, byte4, byte5, byte6,
+ * byte7).
+ */
+instr_t *
+instr_create_raw_7bytes(dcontext_t *dcontext, byte byte1, byte byte2,
+                        byte byte3, byte byte4, byte byte5,
+                        byte byte6, byte byte7);
+
+DR_UNS_API
+/**
+ * Convenience routine that returns an initialized instr_t with invalid operands
+ * and allocated raw bits with 7 bytes (byte1, byte2, byte3, byte4, byte5, byte6,
+ * byte7, byte8).
+ */
+instr_t *
+instr_create_raw_8bytes(dcontext_t *dcontext, byte byte1, byte byte2,
+                        byte byte3, byte byte4, byte byte5,
+                        byte byte6, byte byte7, byte byte8);
+
+opnd_t opnd_create_dcontext_field(dcontext_t *dcontext, int offs);
+opnd_t opnd_create_dcontext_field_byte(dcontext_t *dcontext, int offs);
+opnd_t opnd_create_dcontext_field_sz(dcontext_t *dcontext, int offs, opnd_size_t sz);
+instr_t * instr_create_save_to_dcontext(dcontext_t *dcontext, reg_id_t reg, int offs);
+instr_t * instr_create_save_immed_to_dcontext(dcontext_t *dcontext, int immed, int offs);
+instr_t * instr_create_restore_from_dcontext(dcontext_t *dcontext, reg_id_t reg, int offs);
+
+/* basereg, if left as REG_NULL, is assumed to be xdi (xsi for upcontext) */
+opnd_t
+opnd_create_dcontext_field_via_reg_sz(dcontext_t *dcontext, reg_id_t basereg,
+                                      int offs, opnd_size_t sz);
+opnd_t opnd_create_dcontext_field_via_reg(dcontext_t *dcontext, reg_id_t basereg,
+                                        int offs);
+
+instr_t * instr_create_save_to_dc_via_reg(dcontext_t *dcontext, reg_id_t basereg,
+                                        reg_id_t reg, int offs);
+instr_t * instr_create_restore_from_dc_via_reg(dcontext_t *dcontext, reg_id_t basereg,
+                                             reg_id_t reg, int offs);
+
+instr_t * instr_create_jump_via_dcontext(dcontext_t *dcontext, int offs);
+instr_t * instr_create_save_dynamo_stack(dcontext_t *dcontext);
+instr_t * instr_create_restore_dynamo_stack(dcontext_t *dcontext);
+#ifdef RETURN_STACK
+instr_t * instr_create_restore_dynamo_return_stack(dcontext_t *dcontext);
+instr_t * instr_create_save_dynamo_return_stack(dcontext_t *dcontext);
+#endif
+opnd_t update_dcontext_address(opnd_t op, dcontext_t *old_dcontext,
+                             dcontext_t *new_dcontext);
+opnd_t opnd_create_tls_slot(int offs);
+/* For size, use a OPSZ_ value from decode.h, typically OPSZ_1 or OPSZ_4 */
+opnd_t opnd_create_sized_tls_slot(int offs, opnd_size_t size);
+bool instr_raw_is_tls_spill(byte *pc, reg_id_t reg, ushort offs);
+bool instr_is_tls_spill(instr_t *instr, reg_id_t reg, ushort offs);
+bool instr_is_tls_xcx_spill(instr_t *instr);
+bool
+instr_is_reg_spill_or_restore(dcontext_t *dcontext, instr_t *instr,
+                              bool *tls, bool *spill, reg_id_t *reg);
+
+/* N.B. : client meta routines (dr_insert_* etc.) should never use anything other
+ * then TLS_XAX_SLOT unless the client has specified a slot to use as we let the
+ * client use the rest. */
+instr_t * instr_create_save_to_tls(dcontext_t *dcontext, reg_id_t reg, ushort offs);
+instr_t * instr_create_restore_from_tls(dcontext_t *dcontext, reg_id_t reg, ushort offs);
+
+#ifdef X64
+byte *
+instr_raw_is_rip_rel_lea(byte *pc, byte *read_end);
+#endif
+
+/* DR_API EXPORT TOFILE dr_ir_instr.h */
+/* DR_API EXPORT BEGIN */
+
+
+/****************************************************************************
+ * EFLAGS
+ */
+/* we only care about these 11 flags, and mostly only about the first 6
+ * we consider an undefined effect on a flag to be a write
+ */
+#define EFLAGS_READ_CF   0x00000001 /**< Reads CF (Carry Flag). */             
+#define EFLAGS_READ_PF   0x00000002 /**< Reads PF (Parity Flag). */            
+#define EFLAGS_READ_AF   0x00000004 /**< Reads AF (Auxiliary Carry Flag). */   
+#define EFLAGS_READ_ZF   0x00000008 /**< Reads ZF (Zero Flag). */              
+#define EFLAGS_READ_SF   0x00000010 /**< Reads SF (Sign Flag). */              
+#define EFLAGS_READ_TF   0x00000020 /**< Reads TF (Trap Flag). */              
+#define EFLAGS_READ_IF   0x00000040 /**< Reads IF (Interrupt Enable Flag). */  
+#define EFLAGS_READ_DF   0x00000080 /**< Reads DF (Direction Flag). */         
+#define EFLAGS_READ_OF   0x00000100 /**< Reads OF (Overflow Flag). */          
+#define EFLAGS_READ_NT   0x00000200 /**< Reads NT (Nested Task). */            
+#define EFLAGS_READ_RF   0x00000400 /**< Reads RF (Resume Flag). */            
+#define EFLAGS_WRITE_CF  0x00000800 /**< Writes CF (Carry Flag). */             
+#define EFLAGS_WRITE_PF  0x00001000 /**< Writes PF (Parity Flag). */            
+#define EFLAGS_WRITE_AF  0x00002000 /**< Writes AF (Auxiliary Carry Flag). */   
+#define EFLAGS_WRITE_ZF  0x00004000 /**< Writes ZF (Zero Flag). */              
+#define EFLAGS_WRITE_SF  0x00008000 /**< Writes SF (Sign Flag). */              
+#define EFLAGS_WRITE_TF  0x00010000 /**< Writes TF (Trap Flag). */              
+#define EFLAGS_WRITE_IF  0x00020000 /**< Writes IF (Interrupt Enable Flag). */  
+#define EFLAGS_WRITE_DF  0x00040000 /**< Writes DF (Direction Flag). */         
+#define EFLAGS_WRITE_OF  0x00080000 /**< Writes OF (Overflow Flag). */          
+#define EFLAGS_WRITE_NT  0x00100000 /**< Writes NT (Nested Task). */            
+#define EFLAGS_WRITE_RF  0x00200000 /**< Writes RF (Resume Flag). */            
+
+#define EFLAGS_READ_ALL  0x000007ff /**< Reads all flags. */    
+#define EFLAGS_WRITE_ALL 0x003ff800 /**< Writes all flags. */   
+/* 6 most common flags ("arithmetic flags"): CF, PF, AF, ZF, SF, OF */
+/** Reads all 6 arithmetic flags (CF, PF, AF, ZF, SF, OF). */ 
+#define EFLAGS_READ_6    0x0000011f
+/** Writes all 6 arithmetic flags (CF, PF, AF, ZF, SF, OF). */ 
+#define EFLAGS_WRITE_6   0x0008f800
+
+/** Converts an EFLAGS_WRITE_* value to the corresponding EFLAGS_READ_* value. */
+#define EFLAGS_WRITE_TO_READ(x) ((x) >> 11)
+/** Converts an EFLAGS_READ_* value to the corresponding EFLAGS_WRITE_* value. */
+#define EFLAGS_READ_TO_WRITE(x) ((x) << 11)
+
+/**
+ * The actual bits in the eflags register that we care about:\n<pre>
+ *   11 10  9  8  7  6  5  4  3  2  1  0
+ *   OF DF       SF ZF    AF    PF    CF  </pre>
+ */
+enum {
+    EFLAGS_CF = 0x00000001, /**< The bit in the eflags register of CF (Carry Flag). */
+    EFLAGS_PF = 0x00000004, /**< The bit in the eflags register of PF (Parity Flag). */
+    EFLAGS_AF = 0x00000010, /**< The bit in the eflags register of AF (Aux Carry Flag). */
+    EFLAGS_ZF = 0x00000040, /**< The bit in the eflags register of ZF (Zero Flag). */
+    EFLAGS_SF = 0x00000080, /**< The bit in the eflags register of SF (Sign Flag). */
+    EFLAGS_OF = 0x00000800, /**< The bit in the eflags register of OF (Overflow Flag). */
+};
+
+/* DR_API EXPORT END */
+
+/* even on x64, displacements are 32 bits, so we keep the "int" type and 4-byte size */
+#define PC_RELATIVE_TARGET(addr) ( *((int *)(addr)) + (addr) + 4 )
+
+enum {
+    RAW_OPCODE_nop             = 0x90,
+    RAW_OPCODE_jmp_short       = 0xeb,
+    RAW_OPCODE_call            = 0xe8,
+    RAW_OPCODE_ret             = 0xc3,
+    RAW_OPCODE_jmp             = 0xe9,
+    RAW_OPCODE_push_imm32      = 0x68,
+    RAW_OPCODE_jcc_short_start = 0x70,
+    RAW_OPCODE_jcc_short_end   = 0x7f,
+    RAW_OPCODE_jcc_byte1       = 0x0f,
+    RAW_OPCODE_jcc_byte2_start = 0x80,
+    RAW_OPCODE_jcc_byte2_end   = 0x8f,
+    RAW_OPCODE_loop_start      = 0xe0,
+    RAW_OPCODE_loop_end        = 0xe3,
+    RAW_OPCODE_lea             = 0x8d,
+    RAW_PREFIX_jcc_not_taken   = 0x2e,
+    RAW_PREFIX_jcc_taken       = 0x3e,
+    RAW_PREFIX_lock            = 0xf0,
+};
+
+enum { /* FIXME: vs RAW_OPCODE_* enum */
+    FS_SEG_OPCODE        = 0x64,
+    GS_SEG_OPCODE        = 0x65,
+
+    /* For Windows, we piggyback on native TLS via gs for x64 and fs for x86.
+     * For Linux, we steal a segment register, and so use fs for x86 (where
+     * pthreads uses gs) and gs for x64 (where pthreads uses fs) (presumably
+     * to avoid conflicts w/ wine).
+     */
+#ifdef X64
+    TLS_SEG_OPCODE       = GS_SEG_OPCODE,
+#else
+    TLS_SEG_OPCODE       = FS_SEG_OPCODE,
+#endif
+
+    DATA_PREFIX_OPCODE   = 0x66,
+    ADDR_PREFIX_OPCODE   = 0x67,
+    REPNE_PREFIX_OPCODE  = 0xf2,
+    REP_PREFIX_OPCODE    = 0xf3,
+    REX_PREFIX_BASE_OPCODE = 0x40,
+    REX_PREFIX_W_OPFLAG    = 0x8,
+    REX_PREFIX_R_OPFLAG    = 0x4,
+    REX_PREFIX_X_OPFLAG    = 0x2,
+    REX_PREFIX_B_OPFLAG    = 0x1,
+    REX_PREFIX_ALL_OPFLAGS = 0xf,
+    MOV_REG2MEM_OPCODE   = 0x89,
+    MOV_MEM2REG_OPCODE   = 0x8b,
+    MOV_XAX2MEM_OPCODE   = 0xa3, /* no ModRm */
+    MOV_MEM2XAX_OPCODE   = 0xa1, /* no ModRm */
+    MOV_IMM2XAX_OPCODE   = 0xb8, /* no ModRm */
+    MOV_IMM2XBX_OPCODE   = 0xbb, /* no ModRm */
+    MOV_IMM2MEM_OPCODE   = 0xc7, /* has ModRm */
+    JECXZ_OPCODE         = 0xe3,
+    JMP_SHORT_OPCODE     = 0xeb,
+    JMP_OPCODE           = 0xe9,
+    JNE_OPCODE_1         = 0x0f,
+    SAHF_OPCODE          = 0x9e,
+    LAHF_OPCODE          = 0x9f,
+    SETO_OPCODE_1        = 0x0f,
+    SETO_OPCODE_2        = 0x90,
+    ADD_AL_OPCODE        = 0x04,
+    INC_MEM32_OPCODE_1   = 0xff, /* has /0 as well */
+    MODRM16_DISP16       = 0x06, /* see vol.2 Table 2-1 for modR/M */
+    SIB_DISP32           = 0x25, /* see vol.2 Table 2-1 for modR/M */
+};
+
+/* length of our mangling of jecxz/loop* */
+#define CTI_SHORT_REWRITE_LENGTH 9
+
+/* This should be kept in sync w/ the defines in x86/x86.asm */
+enum {
+#ifdef X64
+# ifdef LINUX
+    /* SysV ABI calling convention */
+    NUM_REGPARM          = 6,
+    REGPARM_0            = REG_RDI,
+    REGPARM_1            = REG_RSI,
+    REGPARM_2            = REG_RDX,
+    REGPARM_3            = REG_RCX,
+    REGPARM_4            = REG_R8,
+    REGPARM_5            = REG_R9,
+    REGPARM_MINSTACK     = 0,
+    REDZONE_SIZE         = 128,
+# else
+    /* Intel/Microsoft calling convention */
+    NUM_REGPARM          = 4,
+    REGPARM_0            = REG_RCX,
+    REGPARM_1            = REG_RDX,
+    REGPARM_2            = REG_R8,
+    REGPARM_3            = REG_R9,
+    REGPARM_MINSTACK     = 4*sizeof(XSP_SZ),
+    REDZONE_SIZE         = 0,
+# endif
+    /* In fact, for Windows the stack pointer is supposed to be
+     * 16-byte aligned at all times except in a prologue or epilogue.
+     * The prologue will always adjust by 16*n+8 since push of retaddr
+     * always makes stack pointer not 16-byte aligned.
+     */
+    REGPARM_END_ALIGN    = 16,
+#else
+    NUM_REGPARM          = 0,
+    REGPARM_MINSTACK     = 0,
+    REDZONE_SIZE         = 0,
+    REGPARM_END_ALIGN    = sizeof(XSP_SZ),
+#endif
+};
+extern const reg_id_t regparms[];
+
+/* DR_API EXPORT TOFILE dr_ir_opcodes.h */
+/* DR_API EXPORT BEGIN */
+
+/****************************************************************************
+ * OPCODES
+ */
+/**
+ * @file dr_ir_opcodes.h
+ * @brief Instruction opcode constants.
+ */
+#ifdef AVOID_API_EXPORT
+/*
+ * This enum corresponds with the array in decode_table.c
+ * IF YOU CHANGE ONE YOU MUST CHANGE THE OTHER
+ * The Perl script tools/x86opnums.pl is useful for re-numbering these
+ * if you add or delete in the middle (feed it the array from decode_table.c)
+ */
+#endif
+/** Opcode constants for use in the instr_t data structure. */
+enum {
+/*   0 */     OP_INVALID,  /* NULL, */ /**< Indicates an invalid instr_t. */
+/*   1 */     OP_UNDECODED,/* NULL, */ /**< Indicates an undecoded instr_t. */
+/*   2 */     OP_CONTD,    /* NULL, */ /**< Used internally only. */
+/*   3 */     OP_LABEL,    /* NULL, */ /**< A label is used for instr_t branch targets. */
+
+/*   4 */     OP_add,      /* &first_byte[0x05], */ /**< add opcode */
+/*   5 */     OP_or,       /* &first_byte[0x0d], */ /**< or opcode */
+/*   6 */     OP_adc,      /* &first_byte[0x15], */ /**< adc opcode */
+/*   7 */     OP_sbb,      /* &first_byte[0x1d], */ /**< sbb opcode */
+/*   8 */     OP_and,      /* &first_byte[0x25], */ /**< and opcode */
+/*   9 */     OP_daa,      /* &first_byte[0x27], */ /**< daa opcode */
+/*  10 */     OP_sub,      /* &first_byte[0x2d], */ /**< sub opcode */
+/*  11 */     OP_das,      /* &first_byte[0x2f], */ /**< das opcode */
+/*  12 */     OP_xor,      /* &first_byte[0x35], */ /**< xor opcode */
+/*  13 */     OP_aaa,      /* &first_byte[0x37], */ /**< aaa opcode */
+/*  14 */     OP_cmp,      /* &first_byte[0x3d], */ /**< cmp opcode */
+/*  15 */     OP_aas,      /* &first_byte[0x3f], */ /**< aas opcode */
+/*  16 */     OP_inc,      /* &x64_extensions[0][0], */ /**< inc opcode */
+/*  17 */     OP_dec,      /* &x64_extensions[8][0], */ /**< dec opcode */
+/*  18 */     OP_push,     /* &first_byte[0x50], */ /**< push opcode */
+/*  19 */     OP_push_imm, /* &first_byte[0x68], */ /**< push_imm opcode */
+/*  20 */     OP_pop,      /* &first_byte[0x58], */ /**< pop opcode */
+/*  21 */     OP_pusha,    /* &first_byte[0x60], */ /**< pusha opcode */
+/*  22 */     OP_popa,     /* &first_byte[0x61], */ /**< popa opcode */
+/*  23 */     OP_bound,    /* &first_byte[0x62], */ /**< bound opcode */
+/*  24 */     OP_arpl,     /* &x64_extensions[16][0], */ /**< arpl opcode */
+/*  25 */     OP_imul,     /* &extensions[10][5], */ /**< imul opcode */
+
+/*  26 */     OP_jo_short,     /* &first_byte[0x70], */ /**< jo_short opcode */
+/*  27 */     OP_jno_short,    /* &first_byte[0x71], */ /**< jno_short opcode */
+/*  28 */     OP_jb_short,     /* &first_byte[0x72], */ /**< jb_short opcode */
+/*  29 */     OP_jnb_short,    /* &first_byte[0x73], */ /**< jnb_short opcode */
+/*  30 */     OP_jz_short,     /* &first_byte[0x74], */ /**< jz_short opcode */
+/*  31 */     OP_jnz_short,    /* &first_byte[0x75], */ /**< jnz_short opcode */
+/*  32 */     OP_jbe_short,    /* &first_byte[0x76], */ /**< jbe_short opcode */
+/*  33 */     OP_jnbe_short,   /* &first_byte[0x77], */ /**< jnbe_short opcode */
+/*  34 */     OP_js_short,     /* &first_byte[0x78], */ /**< js_short opcode */
+/*  35 */     OP_jns_short,    /* &first_byte[0x79], */ /**< jns_short opcode */
+/*  36 */     OP_jp_short,     /* &first_byte[0x7a], */ /**< jp_short opcode */
+/*  37 */     OP_jnp_short,    /* &first_byte[0x7b], */ /**< jnp_short opcode */
+/*  38 */     OP_jl_short,     /* &first_byte[0x7c], */ /**< jl_short opcode */
+/*  39 */     OP_jnl_short,    /* &first_byte[0x7d], */ /**< jnl_short opcode */
+/*  40 */     OP_jle_short,    /* &first_byte[0x7e], */ /**< jle_short opcode */
+/*  41 */     OP_jnle_short,   /* &first_byte[0x7f], */ /**< jnle_short opcode */
+
+/*  42 */     OP_call,           /* &first_byte[0xe8], */ /**< call opcode */
+/*  43 */     OP_call_ind,       /* &extensions[12][2], */ /**< call_ind opcode */
+/*  44 */     OP_call_far,       /* &first_byte[0x9a], */ /**< call_far opcode */
+/*  45 */     OP_call_far_ind,   /* &extensions[12][3], */ /**< call_far_ind opcode */
+/*  46 */     OP_jmp,            /* &first_byte[0xe9], */ /**< jmp opcode */
+/*  47 */     OP_jmp_short,      /* &first_byte[0xeb], */ /**< jmp_short opcode */
+/*  48 */     OP_jmp_ind,        /* &extensions[12][4], */ /**< jmp_ind opcode */
+/*  49 */     OP_jmp_far,        /* &first_byte[0xea], */ /**< jmp_far opcode */
+/*  50 */     OP_jmp_far_ind,    /* &extensions[12][5], */ /**< jmp_far_ind opcode */
+
+/*  51 */     OP_loopne,   /* &first_byte[0xe0], */ /**< loopne opcode */
+/*  52 */     OP_loope,    /* &first_byte[0xe1], */ /**< loope opcode */
+/*  53 */     OP_loop,     /* &first_byte[0xe2], */ /**< loop opcode */
+/*  54 */     OP_jecxz,    /* &first_byte[0xe3], */ /**< jecxz opcode */
+
+    /* point ld & st at eAX & al instrs, they save 1 byte (no modrm),
+     * hopefully time taken considering them doesn't offset that */
+/*  55 */     OP_mov_ld,      /* &first_byte[0xa1], */ /**< mov_ld opcode */
+/*  56 */     OP_mov_st,      /* &first_byte[0xa3], */ /**< mov_st opcode */
+    /* note that store of immed is mov_st not mov_imm, even though can be immed->reg,
+     * which we address by sharing part of the mov_st template chain */
+/*  57 */     OP_mov_imm,     /* &first_byte[0xb8], */ /**< mov_imm opcode */
+/*  58 */     OP_mov_seg,     /* &first_byte[0x8e], */ /**< mov_seg opcode */
+/*  59 */     OP_mov_priv,    /* &second_byte[0x20], */ /**< mov_priv opcode */
+
+/*  60 */     OP_test,     /* &first_byte[0xa9], */ /**< test opcode */
+/*  61 */     OP_lea,      /* &first_byte[0x8d], */ /**< lea opcode */
+/*  62 */     OP_xchg,     /* &first_byte[0x91], */ /**< xchg opcode */
+/*  63 */     OP_cwde,     /* &first_byte[0x98], */ /**< cwde opcode */
+/*  64 */     OP_cdq,      /* &first_byte[0x99], */ /**< cdq opcode */
+/*  65 */     OP_fwait,    /* &first_byte[0x9b], */ /**< fwait opcode */
+/*  66 */     OP_pushf,    /* &first_byte[0x9c], */ /**< pushf opcode */
+/*  67 */     OP_popf,     /* &first_byte[0x9d], */ /**< popf opcode */
+/*  68 */     OP_sahf,     /* &first_byte[0x9e], */ /**< sahf opcode */
+/*  69 */     OP_lahf,     /* &first_byte[0x9f], */ /**< lahf opcode */
+
+/*  70 */     OP_ret,       /* &first_byte[0xc2], */ /**< ret opcode */
+/*  71 */     OP_ret_far,   /* &first_byte[0xca], */ /**< ret_far opcode */
+
+/*  72 */     OP_les,      /* &first_byte[0xc4], */ /**< les opcode */
+/*  73 */     OP_lds,      /* &first_byte[0xc5], */ /**< lds opcode */
+/*  74 */     OP_enter,    /* &first_byte[0xc8], */ /**< enter opcode */
+/*  75 */     OP_leave,    /* &first_byte[0xc9], */ /**< leave opcode */
+/*  76 */     OP_int3,     /* &first_byte[0xcc], */ /**< int3 opcode */
+/*  77 */     OP_int,      /* &first_byte[0xcd], */ /**< int opcode */
+/*  78 */     OP_into,     /* &first_byte[0xce], */ /**< into opcode */
+/*  79 */     OP_iret,     /* &first_byte[0xcf], */ /**< iret opcode */
+/*  80 */     OP_aam,      /* &first_byte[0xd4], */ /**< aam opcode */
+/*  81 */     OP_aad,      /* &first_byte[0xd5], */ /**< aad opcode */
+/*  82 */     OP_xlat,     /* &first_byte[0xd7], */ /**< xlat opcode */
+/*  83 */     OP_in,       /* &first_byte[0xe5], */ /**< in opcode */
+/*  84 */     OP_out,      /* &first_byte[0xe7], */ /**< out opcode */
+/*  85 */     OP_hlt,      /* &first_byte[0xf4], */ /**< hlt opcode */
+/*  86 */     OP_cmc,      /* &first_byte[0xf5], */ /**< cmc opcode */
+/*  87 */     OP_clc,      /* &first_byte[0xf8], */ /**< clc opcode */
+/*  88 */     OP_stc,      /* &first_byte[0xf9], */ /**< stc opcode */
+/*  89 */     OP_cli,      /* &first_byte[0xfa], */ /**< cli opcode */
+/*  90 */     OP_sti,      /* &first_byte[0xfb], */ /**< sti opcode */
+/*  91 */     OP_cld,      /* &first_byte[0xfc], */ /**< cld opcode */
+/*  92 */     OP_std,      /* &first_byte[0xfd], */ /**< std opcode */
+
+
+/*  93 */     OP_lar,          /* &second_byte[0x02], */ /**< lar opcode */
+/*  94 */     OP_lsl,          /* &second_byte[0x03], */ /**< lsl opcode */
+/*  95 */     OP_syscall,      /* &second_byte[0x05], */ /**< syscall opcode */
+/*  96 */     OP_clts,         /* &second_byte[0x06], */ /**< clts opcode */
+/*  97 */     OP_sysret,       /* &second_byte[0x07], */ /**< sysret opcode */
+/*  98 */     OP_invd,         /* &second_byte[0x08], */ /**< invd opcode */
+/*  99 */     OP_wbinvd,       /* &second_byte[0x09], */ /**< wbinvd opcode */
+/* 100 */     OP_ud2a,         /* &second_byte[0x0b], */ /**< ud2a opcode */
+/* 101 */     OP_nop_modrm,    /* &second_byte[0x1f], */ /**< nop_modrm opcode */
+/* 102 */     OP_movntps,      /* &prefix_extensions[11][0], */ /**< movntps opcode */
+/* 103 */     OP_movntpd,      /* &prefix_extensions[11][2], */ /**< movntpd opcode */
+/* 104 */     OP_wrmsr,        /* &second_byte[0x30], */ /**< wrmsr opcode */
+/* 105 */     OP_rdtsc,        /* &second_byte[0x31], */ /**< rdtsc opcode */
+/* 106 */     OP_rdmsr,        /* &second_byte[0x32], */ /**< rdmsr opcode */
+/* 107 */     OP_rdpmc,        /* &second_byte[0x33], */ /**< rdpmc opcode */
+/* 108 */     OP_sysenter,     /* &second_byte[0x34], */ /**< sysenter opcode */
+/* 109 */     OP_sysexit,      /* &second_byte[0x35], */ /**< sysexit opcode */
+
+/* 110 */     OP_cmovo,        /* &second_byte[0x40], */ /**< cmovo opcode */
+/* 111 */     OP_cmovno,       /* &second_byte[0x41], */ /**< cmovno opcode */
+/* 112 */     OP_cmovb,        /* &second_byte[0x42], */ /**< cmovb opcode */
+/* 113 */     OP_cmovnb,       /* &second_byte[0x43], */ /**< cmovnb opcode */
+/* 114 */     OP_cmovz,        /* &second_byte[0x44], */ /**< cmovz opcode */
+/* 115 */     OP_cmovnz,       /* &second_byte[0x45], */ /**< cmovnz opcode */
+/* 116 */     OP_cmovbe,       /* &second_byte[0x46], */ /**< cmovbe opcode */
+/* 117 */     OP_cmovnbe,      /* &second_byte[0x47], */ /**< cmovnbe opcode */
+/* 118 */     OP_cmovs,        /* &second_byte[0x48], */ /**< cmovs opcode */
+/* 119 */     OP_cmovns,       /* &second_byte[0x49], */ /**< cmovns opcode */
+/* 120 */     OP_cmovp,        /* &second_byte[0x4a], */ /**< cmovp opcode */
+/* 121 */     OP_cmovnp,       /* &second_byte[0x4b], */ /**< cmovnp opcode */
+/* 122 */     OP_cmovl,        /* &second_byte[0x4c], */ /**< cmovl opcode */
+/* 123 */     OP_cmovnl,       /* &second_byte[0x4d], */ /**< cmovnl opcode */
+/* 124 */     OP_cmovle,       /* &second_byte[0x4e], */ /**< cmovle opcode */
+/* 125 */     OP_cmovnle,      /* &second_byte[0x4f], */ /**< cmovnle opcode */
+
+/* 126 */     OP_punpcklbw,    /* &prefix_extensions[32][0], */ /**< punpcklbw opcode */
+/* 127 */     OP_punpcklwd,    /* &prefix_extensions[33][0], */ /**< punpcklwd opcode */
+/* 128 */     OP_punpckldq,    /* &prefix_extensions[34][0], */ /**< punpckldq opcode */
+/* 129 */     OP_packsswb,     /* &prefix_extensions[35][0], */ /**< packsswb opcode */
+/* 130 */     OP_pcmpgtb,      /* &prefix_extensions[36][0], */ /**< pcmpgtb opcode */
+/* 131 */     OP_pcmpgtw,      /* &prefix_extensions[37][0], */ /**< pcmpgtw opcode */
+/* 132 */     OP_pcmpgtd,      /* &prefix_extensions[38][0], */ /**< pcmpgtd opcode */
+/* 133 */     OP_packuswb,     /* &prefix_extensions[39][0], */ /**< packuswb opcode */
+/* 134 */     OP_punpckhbw,    /* &prefix_extensions[40][0], */ /**< punpckhbw opcode */
+/* 135 */     OP_punpckhwd,    /* &prefix_extensions[41][0], */ /**< punpckhwd opcode */
+/* 136 */     OP_punpckhdq,    /* &prefix_extensions[42][0], */ /**< punpckhdq opcode */
+/* 137 */     OP_packssdw,     /* &prefix_extensions[43][0], */ /**< packssdw opcode */
+/* 138 */     OP_punpcklqdq,   /* &prefix_extensions[44][2], */ /**< punpcklqdq opcode */
+/* 139 */     OP_punpckhqdq,   /* &prefix_extensions[45][2], */ /**< punpckhqdq opcode */
+/* 140 */     OP_movd,         /* &prefix_extensions[46][0], */ /**< movd opcode */
+/* 141 */     OP_movq,         /* &prefix_extensions[112][0], */ /**< movq opcode */
+/* 142 */     OP_movdqu,       /* &prefix_extensions[112][1],  */ /**< movdqu opcode */
+/* 143 */     OP_movdqa,       /* &prefix_extensions[112][2], */ /**< movdqa opcode */
+/* 144 */     OP_pshufw,       /* &prefix_extensions[47][0], */ /**< pshufw opcode */
+/* 145 */     OP_pshufd,       /* &prefix_extensions[47][2], */ /**< pshufd opcode */
+/* 146 */     OP_pshufhw,      /* &prefix_extensions[47][1], */ /**< pshufhw opcode */
+/* 147 */     OP_pshuflw,      /* &prefix_extensions[47][3], */ /**< pshuflw opcode */
+/* 148 */     OP_pcmpeqb,      /* &prefix_extensions[48][0], */ /**< pcmpeqb opcode */
+/* 149 */     OP_pcmpeqw,      /* &prefix_extensions[49][0], */ /**< pcmpeqw opcode */
+/* 150 */     OP_pcmpeqd,      /* &prefix_extensions[50][0], */ /**< pcmpeqd opcode */
+/* 151 */     OP_emms,         /* &second_byte[0x77], */ /**< emms opcode */
+
+/* 152 */     OP_jo,       /* &second_byte[0x80], */ /**< jo opcode */
+/* 153 */     OP_jno,      /* &second_byte[0x81], */ /**< jno opcode */
+/* 154 */     OP_jb,       /* &second_byte[0x82], */ /**< jb opcode */
+/* 155 */     OP_jnb,      /* &second_byte[0x83], */ /**< jnb opcode */
+/* 156 */     OP_jz,       /* &second_byte[0x84], */ /**< jz opcode */
+/* 157 */     OP_jnz,      /* &second_byte[0x85], */ /**< jnz opcode */
+/* 158 */     OP_jbe,      /* &second_byte[0x86], */ /**< jbe opcode */
+/* 159 */     OP_jnbe,     /* &second_byte[0x87], */ /**< jnbe opcode */
+/* 160 */     OP_js,       /* &second_byte[0x88], */ /**< js opcode */
+/* 161 */     OP_jns,      /* &second_byte[0x89], */ /**< jns opcode */
+/* 162 */     OP_jp,       /* &second_byte[0x8a], */ /**< jp opcode */
+/* 163 */     OP_jnp,      /* &second_byte[0x8b], */ /**< jnp opcode */
+/* 164 */     OP_jl,       /* &second_byte[0x8c], */ /**< jl opcode */
+/* 165 */     OP_jnl,      /* &second_byte[0x8d], */ /**< jnl opcode */
+/* 166 */     OP_jle,      /* &second_byte[0x8e], */ /**< jle opcode */
+/* 167 */     OP_jnle,     /* &second_byte[0x8f], */ /**< jnle opcode */
+
+/* 168 */     OP_seto,         /* &second_byte[0x90], */ /**< seto opcode */
+/* 169 */     OP_setno,        /* &second_byte[0x91], */ /**< setno opcode */
+/* 170 */     OP_setb,         /* &second_byte[0x92], */ /**< setb opcode */
+/* 171 */     OP_setnb,        /* &second_byte[0x93], */ /**< setnb opcode */
+/* 172 */     OP_setz,         /* &second_byte[0x94], */ /**< setz opcode */
+/* 173 */     OP_setnz,        /* &second_byte[0x95], */ /**< setnz opcode */
+/* 174 */     OP_setbe,        /* &second_byte[0x96], */ /**< setbe opcode */
+/* 175 */     OP_setnbe,         /* &second_byte[0x97], */ /**< setnbe opcode */
+/* 176 */     OP_sets,         /* &second_byte[0x98], */ /**< sets opcode */
+/* 177 */     OP_setns,        /* &second_byte[0x99], */ /**< setns opcode */
+/* 178 */     OP_setp,         /* &second_byte[0x9a], */ /**< setp opcode */
+/* 179 */     OP_setnp,        /* &second_byte[0x9b], */ /**< setnp opcode */
+/* 180 */     OP_setl,         /* &second_byte[0x9c], */ /**< setl opcode */
+/* 181 */     OP_setnl,        /* &second_byte[0x9d], */ /**< setnl opcode */
+/* 182 */     OP_setle,        /* &second_byte[0x9e], */ /**< setle opcode */
+/* 183 */     OP_setnle,         /* &second_byte[0x9f], */ /**< setnle opcode */
+
+/* 184 */     OP_cpuid,        /* &second_byte[0xa2], */ /**< cpuid opcode */
+/* 185 */     OP_bt,           /* &second_byte[0xa3], */ /**< bt opcode */
+/* 186 */     OP_shld,         /* &second_byte[0xa4], */ /**< shld opcode */
+/* 187 */     OP_rsm,          /* &second_byte[0xaa], */ /**< rsm opcode */
+/* 188 */     OP_bts,          /* &second_byte[0xab], */ /**< bts opcode */
+/* 189 */     OP_shrd,         /* &second_byte[0xac], */ /**< shrd opcode */
+/* 190 */     OP_cmpxchg,      /* &second_byte[0xb1], */ /**< cmpxchg opcode */
+/* 191 */     OP_lss,          /* &second_byte[0xb2], */ /**< lss opcode */
+/* 192 */     OP_btr,          /* &second_byte[0xb3], */ /**< btr opcode */
+/* 193 */     OP_lfs,          /* &second_byte[0xb4], */ /**< lfs opcode */
+/* 194 */     OP_lgs,          /* &second_byte[0xb5], */ /**< lgs opcode */
+/* 195 */     OP_movzx,        /* &second_byte[0xb7], */ /**< movzx opcode */
+/* 196 */     OP_ud2b,         /* &second_byte[0xb9], */ /**< ud2b opcode */
+/* 197 */     OP_btc,          /* &second_byte[0xbb], */ /**< btc opcode */
+/* 198 */     OP_bsf,          /* &second_byte[0xbc], */ /**< bsf opcode */
+/* 199 */     OP_bsr,          /* &prefix_extensions[136][0], */ /**< bsr opcode */
+/* 200 */     OP_movsx,        /* &second_byte[0xbf], */ /**< movsx opcode */
+/* 201 */     OP_xadd,         /* &second_byte[0xc1], */ /**< xadd opcode */
+/* 202 */     OP_movnti,       /* &second_byte[0xc3], */ /**< movnti opcode */
+/* 203 */     OP_pinsrw,       /* &prefix_extensions[53][0], */ /**< pinsrw opcode */
+/* 204 */     OP_pextrw,       /* &prefix_extensions[54][0], */ /**< pextrw opcode */
+/* 205 */     OP_bswap,        /* &second_byte[0xc8], */ /**< bswap opcode */
+/* 206 */     OP_psrlw,        /* &prefix_extensions[56][0], */ /**< psrlw opcode */
+/* 207 */     OP_psrld,        /* &prefix_extensions[57][0], */ /**< psrld opcode */
+/* 208 */     OP_psrlq,        /* &prefix_extensions[58][0], */ /**< psrlq opcode */
+/* 209 */     OP_paddq,        /* &prefix_extensions[59][0], */ /**< paddq opcode */
+/* 210 */     OP_pmullw,       /* &prefix_extensions[60][0], */ /**< pmullw opcode */
+/* 211 */     OP_pmovmskb,     /* &prefix_extensions[62][0], */ /**< pmovmskb opcode */
+/* 212 */     OP_psubusb,      /* &prefix_extensions[63][0], */ /**< psubusb opcode */
+/* 213 */     OP_psubusw,      /* &prefix_extensions[64][0], */ /**< psubusw opcode */
+/* 214 */     OP_pminub,       /* &prefix_extensions[65][0], */ /**< pminub opcode */
+/* 215 */     OP_pand,         /* &prefix_extensions[66][0], */ /**< pand opcode */
+/* 216 */     OP_paddusb,      /* &prefix_extensions[67][0], */ /**< paddusb opcode */
+/* 217 */     OP_paddusw,      /* &prefix_extensions[68][0], */ /**< paddusw opcode */
+/* 218 */     OP_pmaxub,       /* &prefix_extensions[69][0], */ /**< pmaxub opcode */
+/* 219 */     OP_pandn,        /* &prefix_extensions[70][0], */ /**< pandn opcode */
+/* 220 */     OP_pavgb,        /* &prefix_extensions[71][0], */ /**< pavgb opcode */
+/* 221 */     OP_psraw,        /* &prefix_extensions[72][0], */ /**< psraw opcode */
+/* 222 */     OP_psrad,        /* &prefix_extensions[73][0], */ /**< psrad opcode */
+/* 223 */     OP_pavgw,        /* &prefix_extensions[74][0], */ /**< pavgw opcode */
+/* 224 */     OP_pmulhuw,      /* &prefix_extensions[75][0], */ /**< pmulhuw opcode */
+/* 225 */     OP_pmulhw,       /* &prefix_extensions[76][0], */ /**< pmulhw opcode */
+/* 226 */     OP_movntq,       /* &prefix_extensions[78][0], */ /**< movntq opcode */
+/* 227 */     OP_movntdq,      /* &prefix_extensions[78][2], */ /**< movntdq opcode */
+/* 228 */     OP_psubsb,       /* &prefix_extensions[79][0], */ /**< psubsb opcode */
+/* 229 */     OP_psubsw,       /* &prefix_extensions[80][0], */ /**< psubsw opcode */
+/* 230 */     OP_pminsw,       /* &prefix_extensions[81][0], */ /**< pminsw opcode */
+/* 231 */     OP_por,          /* &prefix_extensions[82][0], */ /**< por opcode */
+/* 232 */     OP_paddsb,       /* &prefix_extensions[83][0], */ /**< paddsb opcode */
+/* 233 */     OP_paddsw,       /* &prefix_extensions[84][0], */ /**< paddsw opcode */
+/* 234 */     OP_pmaxsw,       /* &prefix_extensions[85][0], */ /**< pmaxsw opcode */
+/* 235 */     OP_pxor,         /* &prefix_extensions[86][0], */ /**< pxor opcode */
+/* 236 */     OP_psllw,        /* &prefix_extensions[87][0], */ /**< psllw opcode */
+/* 237 */     OP_pslld,        /* &prefix_extensions[88][0], */ /**< pslld opcode */
+/* 238 */     OP_psllq,        /* &prefix_extensions[89][0], */ /**< psllq opcode */
+/* 239 */     OP_pmuludq,      /* &prefix_extensions[90][0], */ /**< pmuludq opcode */
+/* 240 */     OP_pmaddwd,      /* &prefix_extensions[91][0], */ /**< pmaddwd opcode */
+/* 241 */     OP_psadbw,       /* &prefix_extensions[92][0], */ /**< psadbw opcode */
+/* 242 */     OP_maskmovq,     /* &prefix_extensions[93][0], */ /**< maskmovq opcode */
+/* 243 */     OP_maskmovdqu,   /* &prefix_extensions[93][2], */ /**< maskmovdqu opcode */
+/* 244 */     OP_psubb,        /* &prefix_extensions[94][0], */ /**< psubb opcode */
+/* 245 */     OP_psubw,        /* &prefix_extensions[95][0], */ /**< psubw opcode */
+/* 246 */     OP_psubd,        /* &prefix_extensions[96][0], */ /**< psubd opcode */
+/* 247 */     OP_psubq,        /* &prefix_extensions[97][0], */ /**< psubq opcode */
+/* 248 */     OP_paddb,        /* &prefix_extensions[98][0], */ /**< paddb opcode */
+/* 249 */     OP_paddw,        /* &prefix_extensions[99][0], */ /**< paddw opcode */
+/* 250 */     OP_paddd,        /* &prefix_extensions[100][0], */ /**< paddd opcode */
+/* 251 */     OP_psrldq,       /* &prefix_extensions[101][2], */ /**< psrldq opcode */
+/* 252 */     OP_pslldq,       /* &prefix_extensions[102][2], */ /**< pslldq opcode */
+
+
+/* 253 */     OP_rol,           /* &extensions[ 4][0], */ /**< rol opcode */
+/* 254 */     OP_ror,           /* &extensions[ 4][1], */ /**< ror opcode */
+/* 255 */     OP_rcl,           /* &extensions[ 4][2], */ /**< rcl opcode */
+/* 256 */     OP_rcr,           /* &extensions[ 4][3], */ /**< rcr opcode */
+/* 257 */     OP_shl,           /* &extensions[ 4][4], */ /**< shl opcode */
+/* 258 */     OP_shr,           /* &extensions[ 4][5], */ /**< shr opcode */
+/* 259 */     OP_sar,           /* &extensions[ 4][7], */ /**< sar opcode */
+/* 260 */     OP_not,           /* &extensions[10][2], */ /**< not opcode */
+/* 261 */     OP_neg,           /* &extensions[10][3], */ /**< neg opcode */
+/* 262 */     OP_mul,           /* &extensions[10][4], */ /**< mul opcode */
+/* 263 */     OP_div,           /* &extensions[10][6], */ /**< div opcode */
+/* 264 */     OP_idiv,          /* &extensions[10][7], */ /**< idiv opcode */
+/* 265 */     OP_sldt,          /* &extensions[13][0], */ /**< sldt opcode */
+/* 266 */     OP_str,           /* &extensions[13][1], */ /**< str opcode */
+/* 267 */     OP_lldt,          /* &extensions[13][2], */ /**< lldt opcode */
+/* 268 */     OP_ltr,           /* &extensions[13][3], */ /**< ltr opcode */
+/* 269 */     OP_verr,          /* &extensions[13][4], */ /**< verr opcode */
+/* 270 */     OP_verw,          /* &extensions[13][5], */ /**< verw opcode */
+/* 271 */     OP_sgdt,          /* &mod_extensions[0][0], */ /**< sgdt opcode */
+/* 272 */     OP_sidt,          /* &mod_extensions[1][0], */ /**< sidt opcode */
+/* 273 */     OP_lgdt,          /* &extensions[14][2], */ /**< lgdt opcode */
+/* 274 */     OP_lidt,          /* &extensions[14][3], */ /**< lidt opcode */
+/* 275 */     OP_smsw,          /* &extensions[14][4], */ /**< smsw opcode */
+/* 276 */     OP_lmsw,          /* &extensions[14][6], */ /**< lmsw opcode */
+/* 277 */     OP_invlpg,        /* &mod_extensions[2][0], */ /**< invlpg opcode */
+/* 278 */     OP_cmpxchg8b,     /* &extensions[16][1], */ /**< cmpxchg8b opcode */
+/* 279 */     OP_fxsave,        /* &extensions[22][0], */ /**< fxsave opcode */
+/* 280 */     OP_fxrstor,       /* &extensions[22][1], */ /**< fxrstor opcode */
+/* 281 */     OP_ldmxcsr,       /* &extensions[22][2], */ /**< ldmxcsr opcode */
+/* 282 */     OP_stmxcsr,       /* &extensions[22][3], */ /**< stmxcsr opcode */
+/* 283 */     OP_lfence,        /* &extensions[22][5], */ /**< lfence opcode */
+/* 284 */     OP_mfence,        /* &extensions[22][6], */ /**< mfence opcode */
+/* 285 */     OP_clflush,       /* &mod_extensions[3][0], */ /**< clflush opcode */
+/* 286 */     OP_sfence,        /* &mod_extensions[3][1], */ /**< sfence opcode */
+/* 287 */     OP_prefetchnta,   /* &extensions[23][0], */ /**< prefetchnta opcode */
+/* 288 */     OP_prefetcht0,    /* &extensions[23][1], */ /**< prefetcht0 opcode */
+/* 289 */     OP_prefetcht1,    /* &extensions[23][2], */ /**< prefetcht1 opcode */
+/* 290 */     OP_prefetcht2,    /* &extensions[23][3], */ /**< prefetcht2 opcode */
+/* 291 */     OP_prefetch,      /* &extensions[24][0], */ /**< prefetch opcode */
+/* 292 */     OP_prefetchw,     /* &extensions[24][1], */ /**< prefetchw opcode */
+
+
+/* 293 */     OP_movups,      /* &prefix_extensions[ 0][0], */ /**< movups opcode */
+/* 294 */     OP_movss,       /* &prefix_extensions[ 0][1], */ /**< movss opcode */
+/* 295 */     OP_movupd,      /* &prefix_extensions[ 0][2], */ /**< movupd opcode */
+/* 296 */     OP_movsd,       /* &prefix_extensions[ 0][3], */ /**< movsd opcode */
+/* 297 */     OP_movlps,      /* &prefix_extensions[ 2][0], */ /**< movlps opcode */
+/* 298 */     OP_movlpd,      /* &prefix_extensions[ 2][2], */ /**< movlpd opcode */
+/* 299 */     OP_unpcklps,    /* &prefix_extensions[ 4][0], */ /**< unpcklps opcode */
+/* 300 */     OP_unpcklpd,    /* &prefix_extensions[ 4][2], */ /**< unpcklpd opcode */
+/* 301 */     OP_unpckhps,    /* &prefix_extensions[ 5][0], */ /**< unpckhps opcode */
+/* 302 */     OP_unpckhpd,    /* &prefix_extensions[ 5][2], */ /**< unpckhpd opcode */
+/* 303 */     OP_movhps,      /* &prefix_extensions[ 6][0], */ /**< movhps opcode */
+/* 304 */     OP_movhpd,      /* &prefix_extensions[ 6][2], */ /**< movhpd opcode */
+/* 305 */     OP_movaps,      /* &prefix_extensions[ 8][0], */ /**< movaps opcode */
+/* 306 */     OP_movapd,      /* &prefix_extensions[ 8][2], */ /**< movapd opcode */
+/* 307 */     OP_cvtpi2ps,    /* &prefix_extensions[10][0], */ /**< cvtpi2ps opcode */
+/* 308 */     OP_cvtsi2ss,    /* &prefix_extensions[10][1], */ /**< cvtsi2ss opcode */
+/* 309 */     OP_cvtpi2pd,    /* &prefix_extensions[10][2], */ /**< cvtpi2pd opcode */
+/* 310 */     OP_cvtsi2sd,    /* &prefix_extensions[10][3], */ /**< cvtsi2sd opcode */
+/* 311 */     OP_cvttps2pi,   /* &prefix_extensions[12][0], */ /**< cvttps2pi opcode */
+/* 312 */     OP_cvttss2si,   /* &prefix_extensions[12][1], */ /**< cvttss2si opcode */
+/* 313 */     OP_cvttpd2pi,   /* &prefix_extensions[12][2], */ /**< cvttpd2pi opcode */
+/* 314 */     OP_cvttsd2si,   /* &prefix_extensions[12][3], */ /**< cvttsd2si opcode */
+/* 315 */     OP_cvtps2pi,    /* &prefix_extensions[13][0], */ /**< cvtps2pi opcode */
+/* 316 */     OP_cvtss2si,    /* &prefix_extensions[13][1], */ /**< cvtss2si opcode */
+/* 317 */     OP_cvtpd2pi,    /* &prefix_extensions[13][2], */ /**< cvtpd2pi opcode */
+/* 318 */     OP_cvtsd2si,    /* &prefix_extensions[13][3], */ /**< cvtsd2si opcode */
+/* 319 */     OP_ucomiss,     /* &prefix_extensions[14][0], */ /**< ucomiss opcode */
+/* 320 */     OP_ucomisd,     /* &prefix_extensions[14][2], */ /**< ucomisd opcode */
+/* 321 */     OP_comiss,      /* &prefix_extensions[15][0], */ /**< comiss opcode */
+/* 322 */     OP_comisd,      /* &prefix_extensions[15][2], */ /**< comisd opcode */
+/* 323 */     OP_movmskps,    /* &prefix_extensions[16][0], */ /**< movmskps opcode */
+/* 324 */     OP_movmskpd,    /* &prefix_extensions[16][2], */ /**< movmskpd opcode */
+/* 325 */     OP_sqrtps,      /* &prefix_extensions[17][0], */ /**< sqrtps opcode */
+/* 326 */     OP_sqrtss,      /* &prefix_extensions[17][1], */ /**< sqrtss opcode */
+/* 327 */     OP_sqrtpd,      /* &prefix_extensions[17][2], */ /**< sqrtpd opcode */
+/* 328 */     OP_sqrtsd,      /* &prefix_extensions[17][3], */ /**< sqrtsd opcode */
+/* 329 */     OP_rsqrtps,     /* &prefix_extensions[18][0], */ /**< rsqrtps opcode */
+/* 330 */     OP_rsqrtss,     /* &prefix_extensions[18][1], */ /**< rsqrtss opcode */
+/* 331 */     OP_rcpps,       /* &prefix_extensions[19][0], */ /**< rcpps opcode */
+/* 332 */     OP_rcpss,       /* &prefix_extensions[19][1], */ /**< rcpss opcode */
+/* 333 */     OP_andps,       /* &prefix_extensions[20][0], */ /**< andps opcode */
+/* 334 */     OP_andpd,       /* &prefix_extensions[20][2], */ /**< andpd opcode */
+/* 335 */     OP_andnps,      /* &prefix_extensions[21][0], */ /**< andnps opcode */
+/* 336 */     OP_andnpd,      /* &prefix_extensions[21][2], */ /**< andnpd opcode */
+/* 337 */     OP_orps,        /* &prefix_extensions[22][0], */ /**< orps opcode */
+/* 338 */     OP_orpd,        /* &prefix_extensions[22][2], */ /**< orpd opcode */
+/* 339 */     OP_xorps,       /* &prefix_extensions[23][0], */ /**< xorps opcode */
+/* 340 */     OP_xorpd,       /* &prefix_extensions[23][2], */ /**< xorpd opcode */
+/* 341 */     OP_addps,       /* &prefix_extensions[24][0], */ /**< addps opcode */
+/* 342 */     OP_addss,       /* &prefix_extensions[24][1], */ /**< addss opcode */
+/* 343 */     OP_addpd,       /* &prefix_extensions[24][2], */ /**< addpd opcode */
+/* 344 */     OP_addsd,       /* &prefix_extensions[24][3], */ /**< addsd opcode */
+/* 345 */     OP_mulps,       /* &prefix_extensions[25][0], */ /**< mulps opcode */
+/* 346 */     OP_mulss,       /* &prefix_extensions[25][1], */ /**< mulss opcode */
+/* 347 */     OP_mulpd,       /* &prefix_extensions[25][2], */ /**< mulpd opcode */
+/* 348 */     OP_mulsd,       /* &prefix_extensions[25][3], */ /**< mulsd opcode */
+/* 349 */     OP_cvtps2pd,    /* &prefix_extensions[26][0], */ /**< cvtps2pd opcode */
+/* 350 */     OP_cvtss2sd,    /* &prefix_extensions[26][1], */ /**< cvtss2sd opcode */
+/* 351 */     OP_cvtpd2ps,    /* &prefix_extensions[26][2], */ /**< cvtpd2ps opcode */
+/* 352 */     OP_cvtsd2ss,    /* &prefix_extensions[26][3], */ /**< cvtsd2ss opcode */
+/* 353 */     OP_cvtdq2ps,    /* &prefix_extensions[27][0], */ /**< cvtdq2ps opcode */
+/* 354 */     OP_cvttps2dq,   /* &prefix_extensions[27][1], */ /**< cvttps2dq opcode */
+/* 355 */     OP_cvtps2dq,    /* &prefix_extensions[27][2], */ /**< cvtps2dq opcode */
+/* 356 */     OP_subps,       /* &prefix_extensions[28][0], */ /**< subps opcode */
+/* 357 */     OP_subss,       /* &prefix_extensions[28][1], */ /**< subss opcode */
+/* 358 */     OP_subpd,       /* &prefix_extensions[28][2], */ /**< subpd opcode */
+/* 359 */     OP_subsd,       /* &prefix_extensions[28][3], */ /**< subsd opcode */
+/* 360 */     OP_minps,       /* &prefix_extensions[29][0], */ /**< minps opcode */
+/* 361 */     OP_minss,       /* &prefix_extensions[29][1], */ /**< minss opcode */
+/* 362 */     OP_minpd,       /* &prefix_extensions[29][2], */ /**< minpd opcode */
+/* 363 */     OP_minsd,       /* &prefix_extensions[29][3], */ /**< minsd opcode */
+/* 364 */     OP_divps,       /* &prefix_extensions[30][0], */ /**< divps opcode */
+/* 365 */     OP_divss,       /* &prefix_extensions[30][1], */ /**< divss opcode */
+/* 366 */     OP_divpd,       /* &prefix_extensions[30][2], */ /**< divpd opcode */
+/* 367 */     OP_divsd,       /* &prefix_extensions[30][3], */ /**< divsd opcode */
+/* 368 */     OP_maxps,       /* &prefix_extensions[31][0], */ /**< maxps opcode */
+/* 369 */     OP_maxss,       /* &prefix_extensions[31][1], */ /**< maxss opcode */
+/* 370 */     OP_maxpd,       /* &prefix_extensions[31][2], */ /**< maxpd opcode */
+/* 371 */     OP_maxsd,       /* &prefix_extensions[31][3], */ /**< maxsd opcode */
+/* 372 */     OP_cmpps,       /* &prefix_extensions[52][0], */ /**< cmpps opcode */
+/* 373 */     OP_cmpss,       /* &prefix_extensions[52][1], */ /**< cmpss opcode */
+/* 374 */     OP_cmppd,       /* &prefix_extensions[52][2], */ /**< cmppd opcode */
+/* 375 */     OP_cmpsd,       /* &prefix_extensions[52][3], */ /**< cmpsd opcode */
+/* 376 */     OP_shufps,      /* &prefix_extensions[55][0], */ /**< shufps opcode */
+/* 377 */     OP_shufpd,      /* &prefix_extensions[55][2], */ /**< shufpd opcode */
+/* 378 */     OP_cvtdq2pd,    /* &prefix_extensions[77][1], */ /**< cvtdq2pd opcode */
+/* 379 */     OP_cvttpd2dq,   /* &prefix_extensions[77][2], */ /**< cvttpd2dq opcode */
+/* 380 */     OP_cvtpd2dq,    /* &prefix_extensions[77][3], */ /**< cvtpd2dq opcode */
+/* 381 */     OP_nop,         /* &prefix_extensions[103][0], */ /**< nop opcode */
+/* 382 */     OP_pause,       /* &prefix_extensions[103][1], */ /**< pause opcode */
+
+/* 383 */     OP_ins,          /* &rep_extensions[1][0], */ /**< ins opcode */
+/* 384 */     OP_rep_ins,      /* &rep_extensions[1][2], */ /**< rep_ins opcode */
+/* 385 */     OP_outs,         /* &rep_extensions[3][0], */ /**< outs opcode */
+/* 386 */     OP_rep_outs,     /* &rep_extensions[3][2], */ /**< rep_outs opcode */
+/* 387 */     OP_movs,         /* &rep_extensions[5][0], */ /**< movs opcode */
+/* 388 */     OP_rep_movs,     /* &rep_extensions[5][2], */ /**< rep_movs opcode */
+/* 389 */     OP_stos,         /* &rep_extensions[7][0], */ /**< stos opcode */
+/* 390 */     OP_rep_stos,     /* &rep_extensions[7][2], */ /**< rep_stos opcode */
+/* 391 */     OP_lods,         /* &rep_extensions[9][0], */ /**< lods opcode */
+/* 392 */     OP_rep_lods,     /* &rep_extensions[9][2], */ /**< rep_lods opcode */
+/* 393 */     OP_cmps,         /* &repne_extensions[1][0], */ /**< cmps opcode */
+/* 394 */     OP_rep_cmps,     /* &repne_extensions[1][2], */ /**< rep_cmps opcode */
+/* 395 */     OP_repne_cmps,   /* &repne_extensions[1][4], */ /**< repne_cmps opcode */
+/* 396 */     OP_scas,         /* &repne_extensions[3][0], */ /**< scas opcode */
+/* 397 */     OP_rep_scas,     /* &repne_extensions[3][2], */ /**< rep_scas opcode */
+/* 398 */     OP_repne_scas,   /* &repne_extensions[3][4], */ /**< repne_scas opcode */
+
+
+/* 399 */     OP_fadd,      /* &float_low_modrm[0x00], */ /**< fadd opcode */
+/* 400 */     OP_fmul,      /* &float_low_modrm[0x01], */ /**< fmul opcode */
+/* 401 */     OP_fcom,      /* &float_low_modrm[0x02], */ /**< fcom opcode */
+/* 402 */     OP_fcomp,     /* &float_low_modrm[0x03], */ /**< fcomp opcode */
+/* 403 */     OP_fsub,      /* &float_low_modrm[0x04], */ /**< fsub opcode */
+/* 404 */     OP_fsubr,     /* &float_low_modrm[0x05], */ /**< fsubr opcode */
+/* 405 */     OP_fdiv,      /* &float_low_modrm[0x06], */ /**< fdiv opcode */
+/* 406 */     OP_fdivr,     /* &float_low_modrm[0x07], */ /**< fdivr opcode */
+/* 407 */     OP_fld,       /* &float_low_modrm[0x08], */ /**< fld opcode */
+/* 408 */     OP_fst,       /* &float_low_modrm[0x0a], */ /**< fst opcode */
+/* 409 */     OP_fstp,      /* &float_low_modrm[0x0b], */ /**< fstp opcode */
+/* 410 */     OP_fldenv,    /* &float_low_modrm[0x0c], */ /**< fldenv opcode */
+/* 411 */     OP_fldcw,     /* &float_low_modrm[0x0d], */ /**< fldcw opcode */
+/* 412 */     OP_fnstenv,   /* &float_low_modrm[0x0e], */ /**< fnstenv opcode */
+/* 413 */     OP_fnstcw,    /* &float_low_modrm[0x0f], */ /**< fnstcw opcode */
+/* 414 */     OP_fiadd,     /* &float_low_modrm[0x10], */ /**< fiadd opcode */
+/* 415 */     OP_fimul,     /* &float_low_modrm[0x11], */ /**< fimul opcode */
+/* 416 */     OP_ficom,     /* &float_low_modrm[0x12], */ /**< ficom opcode */
+/* 417 */     OP_ficomp,    /* &float_low_modrm[0x13], */ /**< ficomp opcode */
+/* 418 */     OP_fisub,     /* &float_low_modrm[0x14], */ /**< fisub opcode */
+/* 419 */     OP_fisubr,    /* &float_low_modrm[0x15], */ /**< fisubr opcode */
+/* 420 */     OP_fidiv,     /* &float_low_modrm[0x16], */ /**< fidiv opcode */
+/* 421 */     OP_fidivr,    /* &float_low_modrm[0x17], */ /**< fidivr opcode */
+/* 422 */     OP_fild,      /* &float_low_modrm[0x18], */ /**< fild opcode */
+/* 423 */     OP_fist,      /* &float_low_modrm[0x1a], */ /**< fist opcode */
+/* 424 */     OP_fistp,     /* &float_low_modrm[0x1b], */ /**< fistp opcode */
+/* 425 */     OP_frstor,    /* &float_low_modrm[0x2c], */ /**< frstor opcode */
+/* 426 */     OP_fnsave,    /* &float_low_modrm[0x2e], */ /**< fnsave opcode */
+/* 427 */     OP_fnstsw,    /* &float_low_modrm[0x2f], */ /**< fnstsw opcode */
+
+/* 428 */     OP_fbld,      /* &float_low_modrm[0x3c], */ /**< fbld opcode */
+/* 429 */     OP_fbstp,     /* &float_low_modrm[0x3e], */ /**< fbstp opcode */
+
+
+/* 430 */     OP_fxch,       /* &float_high_modrm[1][0x08], */ /**< fxch opcode */
+/* 431 */     OP_fnop,       /* &float_high_modrm[1][0x10], */ /**< fnop opcode */
+/* 432 */     OP_fchs,       /* &float_high_modrm[1][0x20], */ /**< fchs opcode */
+/* 433 */     OP_fabs,       /* &float_high_modrm[1][0x21], */ /**< fabs opcode */
+/* 434 */     OP_ftst,       /* &float_high_modrm[1][0x24], */ /**< ftst opcode */
+/* 435 */     OP_fxam,       /* &float_high_modrm[1][0x25], */ /**< fxam opcode */
+/* 436 */     OP_fld1,       /* &float_high_modrm[1][0x28], */ /**< fld1 opcode */
+/* 437 */     OP_fldl2t,     /* &float_high_modrm[1][0x29], */ /**< fldl2t opcode */
+/* 438 */     OP_fldl2e,     /* &float_high_modrm[1][0x2a], */ /**< fldl2e opcode */
+/* 439 */     OP_fldpi,      /* &float_high_modrm[1][0x2b], */ /**< fldpi opcode */
+/* 440 */     OP_fldlg2,     /* &float_high_modrm[1][0x2c], */ /**< fldlg2 opcode */
+/* 441 */     OP_fldln2,     /* &float_high_modrm[1][0x2d], */ /**< fldln2 opcode */
+/* 442 */     OP_fldz,       /* &float_high_modrm[1][0x2e], */ /**< fldz opcode */
+/* 443 */     OP_f2xm1,      /* &float_high_modrm[1][0x30], */ /**< f2xm1 opcode */
+/* 444 */     OP_fyl2x,      /* &float_high_modrm[1][0x31], */ /**< fyl2x opcode */
+/* 445 */     OP_fptan,      /* &float_high_modrm[1][0x32], */ /**< fptan opcode */
+/* 446 */     OP_fpatan,     /* &float_high_modrm[1][0x33], */ /**< fpatan opcode */
+/* 447 */     OP_fxtract,    /* &float_high_modrm[1][0x34], */ /**< fxtract opcode */
+/* 448 */     OP_fprem1,     /* &float_high_modrm[1][0x35], */ /**< fprem1 opcode */
+/* 449 */     OP_fdecstp,    /* &float_high_modrm[1][0x36], */ /**< fdecstp opcode */
+/* 450 */     OP_fincstp,    /* &float_high_modrm[1][0x37], */ /**< fincstp opcode */
+/* 451 */     OP_fprem,      /* &float_high_modrm[1][0x38], */ /**< fprem opcode */
+/* 452 */     OP_fyl2xp1,    /* &float_high_modrm[1][0x39], */ /**< fyl2xp1 opcode */
+/* 453 */     OP_fsqrt,      /* &float_high_modrm[1][0x3a], */ /**< fsqrt opcode */
+/* 454 */     OP_fsincos,    /* &float_high_modrm[1][0x3b], */ /**< fsincos opcode */
+/* 455 */     OP_frndint,    /* &float_high_modrm[1][0x3c], */ /**< frndint opcode */
+/* 456 */     OP_fscale,     /* &float_high_modrm[1][0x3d], */ /**< fscale opcode */
+/* 457 */     OP_fsin,       /* &float_high_modrm[1][0x3e], */ /**< fsin opcode */
+/* 458 */     OP_fcos,       /* &float_high_modrm[1][0x3f], */ /**< fcos opcode */
+/* 459 */     OP_fcmovb,     /* &float_high_modrm[2][0x00], */ /**< fcmovb opcode */
+/* 460 */     OP_fcmove,     /* &float_high_modrm[2][0x08], */ /**< fcmove opcode */
+/* 461 */     OP_fcmovbe,    /* &float_high_modrm[2][0x10], */ /**< fcmovbe opcode */
+/* 462 */     OP_fcmovu,     /* &float_high_modrm[2][0x18], */ /**< fcmovu opcode */
+/* 463 */     OP_fucompp,    /* &float_high_modrm[2][0x29], */ /**< fucompp opcode */
+/* 464 */     OP_fcmovnb,    /* &float_high_modrm[3][0x00], */ /**< fcmovnb opcode */
+/* 465 */     OP_fcmovene,   /* &float_high_modrm[3][0x08], */ /**< fcmovene opcode */
+/* 466 */     OP_fcmovnbe,   /* &float_high_modrm[3][0x10], */ /**< fcmovnbe opcode */
+/* 467 */     OP_fcmovnu,    /* &float_high_modrm[3][0x18], */ /**< fcmovnu opcode */
+/* 468 */     OP_fnclex,     /* &float_high_modrm[3][0x22], */ /**< fnclex opcode */
+/* 469 */     OP_fninit,     /* &float_high_modrm[3][0x23], */ /**< fninit opcode */
+/* 470 */     OP_fucomi,     /* &float_high_modrm[3][0x28], */ /**< fucomi opcode */
+/* 471 */     OP_fcomi,      /* &float_high_modrm[3][0x30], */ /**< fcomi opcode */
+/* 472 */     OP_ffree,      /* &float_high_modrm[5][0x00], */ /**< ffree opcode */
+/* 473 */     OP_fucom,      /* &float_high_modrm[5][0x20], */ /**< fucom opcode */
+/* 474 */     OP_fucomp,     /* &float_high_modrm[5][0x28], */ /**< fucomp opcode */
+/* 475 */     OP_faddp,      /* &float_high_modrm[6][0x00], */ /**< faddp opcode */
+/* 476 */     OP_fmulp,      /* &float_high_modrm[6][0x08], */ /**< fmulp opcode */
+/* 477 */     OP_fcompp,     /* &float_high_modrm[6][0x19], */ /**< fcompp opcode */
+/* 478 */     OP_fsubrp,     /* &float_high_modrm[6][0x20], */ /**< fsubrp opcode */
+/* 479 */     OP_fsubp,      /* &float_high_modrm[6][0x28], */ /**< fsubp opcode */
+/* 480 */     OP_fdivrp,     /* &float_high_modrm[6][0x30], */ /**< fdivrp opcode */
+/* 481 */     OP_fdivp,      /* &float_high_modrm[6][0x38], */ /**< fdivp opcode */
+/* 482 */     OP_fucomip,    /* &float_high_modrm[7][0x28], */ /**< fucomip opcode */
+/* 483 */     OP_fcomip,     /* &float_high_modrm[7][0x30], */ /**< fcomip opcode */
+
+    /* SSE3 instructions */
+/* 484 */     OP_fisttp,       /* &float_low_modrm[0x29], */ /**< fisttp opcode */
+/* 485 */     OP_haddpd,       /* &prefix_extensions[114][2], */ /**< haddpd opcode */
+/* 486 */     OP_haddps,       /* &prefix_extensions[114][3], */ /**< haddps opcode */
+/* 487 */     OP_hsubpd,       /* &prefix_extensions[115][2], */ /**< hsubpd opcode */
+/* 488 */     OP_hsubps,       /* &prefix_extensions[115][3], */ /**< hsubps opcode */
+/* 489 */     OP_addsubpd,     /* &prefix_extensions[116][2], */ /**< addsubpd opcode */
+/* 490 */     OP_addsubps,     /* &prefix_extensions[116][3], */ /**< addsubps opcode */
+/* 491 */     OP_lddqu,        /* &prefix_extensions[117][3], */ /**< lddqu opcode */
+/* 492 */     OP_monitor,      /* &rm_extensions[1][0], */ /**< monitor opcode */
+/* 493 */     OP_mwait,        /* &rm_extensions[1][1], */ /**< mwait opcode */
+/* 494 */     OP_movsldup,     /* &prefix_extensions[ 2][1], */ /**< movsldup opcode */
+/* 495 */     OP_movshdup,     /* &prefix_extensions[ 6][1], */ /**< movshdup opcode */
+/* 496 */     OP_movddup,      /* &prefix_extensions[ 2][3], */ /**< movddup opcode */
+
+    /* 3D-Now! instructions */
+/* 497 */     OP_femms,          /* &second_byte[0x0e], */ /**< femms opcode */
+/* 498 */     OP_unknown_3dnow,  /* &suffix_extensions[0], */ /**< unknown_3dnow opcode */
+/* 499 */     OP_pavgusb,        /* &suffix_extensions[1], */ /**< pavgusb opcode */
+/* 500 */     OP_pfadd,          /* &suffix_extensions[2], */ /**< pfadd opcode */
+/* 501 */     OP_pfacc,          /* &suffix_extensions[3], */ /**< pfacc opcode */
+/* 502 */     OP_pfcmpge,        /* &suffix_extensions[4], */ /**< pfcmpge opcode */
+/* 503 */     OP_pfcmpgt,        /* &suffix_extensions[5], */ /**< pfcmpgt opcode */
+/* 504 */     OP_pfcmpeq,        /* &suffix_extensions[6], */ /**< pfcmpeq opcode */
+/* 505 */     OP_pfmin,          /* &suffix_extensions[7], */ /**< pfmin opcode */
+/* 506 */     OP_pfmax,          /* &suffix_extensions[8], */ /**< pfmax opcode */
+/* 507 */     OP_pfmul,          /* &suffix_extensions[9], */ /**< pfmul opcode */
+/* 508 */     OP_pfrcp,          /* &suffix_extensions[10], */ /**< pfrcp opcode */
+/* 509 */     OP_pfrcpit1,       /* &suffix_extensions[11], */ /**< pfrcpit1 opcode */
+/* 510 */     OP_pfrcpit2,       /* &suffix_extensions[12], */ /**< pfrcpit2 opcode */
+/* 511 */     OP_pfrsqrt,        /* &suffix_extensions[13], */ /**< pfrsqrt opcode */
+/* 512 */     OP_pfrsqit1,       /* &suffix_extensions[14], */ /**< pfrsqit1 opcode */
+/* 513 */     OP_pmulhrw,        /* &suffix_extensions[15], */ /**< pmulhrw opcode */
+/* 514 */     OP_pfsub,          /* &suffix_extensions[16], */ /**< pfsub opcode */
+/* 515 */     OP_pfsubr,         /* &suffix_extensions[17], */ /**< pfsubr opcode */
+/* 516 */     OP_pi2fd,          /* &suffix_extensions[18], */ /**< pi2fd opcode */
+/* 517 */     OP_pf2id,          /* &suffix_extensions[19], */ /**< pf2id opcode */
+/* 518 */     OP_pi2fw,          /* &suffix_extensions[20], */ /**< pi2fw opcode */
+/* 519 */     OP_pf2iw,          /* &suffix_extensions[21], */ /**< pf2iw opcode */
+/* 520 */     OP_pfnacc,         /* &suffix_extensions[22], */ /**< pfnacc opcode */
+/* 521 */     OP_pfpnacc,        /* &suffix_extensions[23], */ /**< pfpnacc opcode */
+/* 522 */     OP_pswapd,         /* &suffix_extensions[24], */ /**< pswapd opcode */
+
+    /* SSSE3 */
+/* 523 */     OP_pshufb,         /* &prefix_extensions[118][0], */ /**< pshufb opcode */
+/* 524 */     OP_phaddw,         /* &prefix_extensions[119][0], */ /**< phaddw opcode */
+/* 525 */     OP_phaddd,         /* &prefix_extensions[120][0], */ /**< phaddd opcode */
+/* 526 */     OP_phaddsw,        /* &prefix_extensions[121][0], */ /**< phaddsw opcode */
+/* 527 */     OP_pmaddubsw,      /* &prefix_extensions[122][0], */ /**< pmaddubsw opcode */
+/* 528 */     OP_phsubw,         /* &prefix_extensions[123][0], */ /**< phsubw opcode */
+/* 529 */     OP_phsubd,         /* &prefix_extensions[124][0], */ /**< phsubd opcode */
+/* 530 */     OP_phsubsw,        /* &prefix_extensions[125][0], */ /**< phsubsw opcode */
+/* 531 */     OP_psignb,         /* &prefix_extensions[126][0], */ /**< psignb opcode */
+/* 532 */     OP_psignw,         /* &prefix_extensions[127][0], */ /**< psignw opcode */
+/* 533 */     OP_psignd,         /* &prefix_extensions[128][0], */ /**< psignd opcode */
+/* 534 */     OP_pmulhrsw,       /* &prefix_extensions[129][0], */ /**< pmulhrsw opcode */
+/* 535 */     OP_pabsb,          /* &prefix_extensions[130][0], */ /**< pabsb opcode */
+/* 536 */     OP_pabsw,          /* &prefix_extensions[131][0], */ /**< pabsw opcode */
+/* 537 */     OP_pabsd,          /* &prefix_extensions[132][0], */ /**< pabsd opcode */
+/* 538 */     OP_palignr,        /* &prefix_extensions[133][0], */ /**< palignr opcode */
+
+    /* SSE4 (incl AMD and Intel-specific extensions */
+/* 539 */     OP_popcnt,         /* &second_byte[0xb8], */ /**< popcnt opcode */
+/* 540 */     OP_movntss,        /* &prefix_extensions[11][1], */ /**< movntss opcode */
+/* 541 */     OP_movntsd,        /* &prefix_extensions[11][3], */ /**< movntsd opcode */
+/* 542 */     OP_extrq,          /* &prefix_extensions[134][2], */ /**< extrq opcode */
+/* 543 */     OP_insertq,        /* &prefix_extensions[134][3], */ /**< insertq opcode */
+/* 544 */     OP_lzcnt,          /* &prefix_extensions[136][1], */ /**< lzcnt opcode */
+/* 545 */     OP_pblendvb,       /* &third_byte_38[16], */ /**< pblendvb opcode */
+/* 546 */     OP_blendvps,       /* &third_byte_38[17], */ /**< blendvps opcode */
+/* 547 */     OP_blendvpd,       /* &third_byte_38[18], */ /**< blendvpd opcode */
+/* 548 */     OP_ptest,          /* &third_byte_38[19], */ /**< ptest opcode */
+/* 549 */     OP_pmovsxbw,       /* &third_byte_38[20], */ /**< pmovsxbw opcode */
+/* 550 */     OP_pmovsxbd,       /* &third_byte_38[21], */ /**< pmovsxbd opcode */
+/* 551 */     OP_pmovsxbq,       /* &third_byte_38[22], */ /**< pmovsxbq opcode */
+/* 552 */     OP_pmovsxdw,       /* &third_byte_38[23], */ /**< pmovsxdw opcode */
+/* 553 */     OP_pmovsxwq,       /* &third_byte_38[24], */ /**< pmovsxwq opcode */
+/* 554 */     OP_pmovsxdq,       /* &third_byte_38[25], */ /**< pmovsxdq opcode */
+/* 555 */     OP_pmuldq,         /* &third_byte_38[26], */ /**< pmuldq opcode */
+/* 556 */     OP_pcmpeqq,        /* &third_byte_38[27], */ /**< pcmpeqq opcode */
+/* 557 */     OP_movntdqa,       /* &third_byte_38[28], */ /**< movntdqa opcode */
+/* 558 */     OP_packusdw,       /* &third_byte_38[29], */ /**< packusdw opcode */
+/* 559 */     OP_pmovzxbw,       /* &third_byte_38[30], */ /**< pmovzxbw opcode */
+/* 560 */     OP_pmovzxbd,       /* &third_byte_38[31], */ /**< pmovzxbd opcode */
+/* 561 */     OP_pmovzxbq,       /* &third_byte_38[32], */ /**< pmovzxbq opcode */
+/* 562 */     OP_pmovzxdw,       /* &third_byte_38[33], */ /**< pmovzxdw opcode */
+/* 563 */     OP_pmovzxwq,       /* &third_byte_38[34], */ /**< pmovzxwq opcode */
+/* 564 */     OP_pmovzxdq,       /* &third_byte_38[35], */ /**< pmovzxdq opcode */
+/* 565 */     OP_pcmpgtq,        /* &third_byte_38[36], */ /**< pcmpgtq opcode */
+/* 566 */     OP_pminsb,         /* &third_byte_38[37], */ /**< pminsb opcode */
+/* 567 */     OP_pminsd,         /* &third_byte_38[38], */ /**< pminsd opcode */
+/* 568 */     OP_pminuw,         /* &third_byte_38[39], */ /**< pminuw opcode */
+/* 569 */     OP_pminud,         /* &third_byte_38[40], */ /**< pminud opcode */
+/* 570 */     OP_pmaxsb,         /* &third_byte_38[41], */ /**< pmaxsb opcode */
+/* 571 */     OP_pmaxsd,         /* &third_byte_38[42], */ /**< pmaxsd opcode */
+/* 572 */     OP_pmaxuw,         /* &third_byte_38[43], */ /**< pmaxuw opcode */
+/* 573 */     OP_pmaxud,         /* &third_byte_38[44], */ /**< pmaxud opcode */
+/* 574 */     OP_pmulld,         /* &third_byte_38[45], */ /**< pmulld opcode */
+/* 575 */     OP_phminposuw,     /* &third_byte_38[46], */ /**< phminposuw opcode */
+/* 576 */     OP_crc32,          /* &third_byte_38[48], */ /**< crc32 opcode */
+/* 577 */     OP_pextrb,         /* &third_byte_3a[2], */ /**< pextrb opcode */
+/* 578 */     OP_pextrd,         /* &third_byte_3a[4], */ /**< pextrd opcode */
+/* 579 */     OP_extractps,      /* &third_byte_3a[5], */ /**< extractps opcode */
+/* 580 */     OP_roundps,        /* &third_byte_3a[6], */ /**< roundps opcode */
+/* 581 */     OP_roundpd,        /* &third_byte_3a[7], */ /**< roundpd opcode */
+/* 582 */     OP_roundss,        /* &third_byte_3a[8], */ /**< roundss opcode */
+/* 583 */     OP_roundsd,        /* &third_byte_3a[9], */ /**< roundsd opcode */
+/* 584 */     OP_blendps,        /* &third_byte_3a[10], */ /**< blendps opcode */
+/* 585 */     OP_blendpd,        /* &third_byte_3a[11], */ /**< blendpd opcode */
+/* 586 */     OP_pblendw,        /* &third_byte_3a[12], */ /**< pblendw opcode */
+/* 587 */     OP_pinsrb,         /* &third_byte_3a[13], */ /**< pinsrb opcode */
+/* 588 */     OP_insertps,       /* &third_byte_3a[14], */ /**< insertps opcode */
+/* 589 */     OP_pinsrd,         /* &third_byte_3a[15], */ /**< pinsrd opcode */
+/* 590 */     OP_dpps,           /* &third_byte_3a[16], */ /**< dpps opcode */
+/* 591 */     OP_dppd,           /* &third_byte_3a[17], */ /**< dppd opcode */
+/* 592 */     OP_mpsadbw,        /* &third_byte_3a[18], */ /**< mpsadbw opcode */
+/* 593 */     OP_pcmpestrm,      /* &third_byte_3a[19], */ /**< pcmpestrm opcode */
+/* 594 */     OP_pcmpestri,      /* &third_byte_3a[20], */ /**< pcmpestri opcode */
+/* 595 */     OP_pcmpistrm,      /* &third_byte_3a[21], */ /**< pcmpistrm opcode */
+/* 596 */     OP_pcmpistri,      /* &third_byte_3a[22], */ /**< pcmpistri opcode */
+
+    /* x64 */
+/* 597 */     OP_movsxd,         /* &x64_extensions[16][1], */ /**< movsxd opcode */
+/* 598 */     OP_swapgs,         /* &rm_extensions[2][0], */ /**< swapgs opcode */
+
+    /* VMX */
+/* 599 */     OP_vmcall,         /* &rm_extensions[0][1], */ /**< vmcall opcode */
+/* 600 */     OP_vmlaunch,       /* &rm_extensions[0][2], */ /**< vmlaunch opcode */
+/* 601 */     OP_vmresume,       /* &rm_extensions[0][3], */ /**< vmresume opcode */
+/* 602 */     OP_vmxoff,         /* &rm_extensions[0][4], */ /**< vmxoff opcode */
+/* 603 */     OP_vmptrst,        /* &extensions[16][7], */ /**< vmptrst opcode */
+/* 604 */     OP_vmptrld,        /* &prefix_extensions[137][0], */ /**< vmptrld opcode */
+/* 605 */     OP_vmxon,          /* &prefix_extensions[137][1], */ /**< vmxon opcode */
+/* 606 */     OP_vmclear,        /* &prefix_extensions[137][2], */ /**< vmclear opcode */
+/* 607 */     OP_vmread,         /* &prefix_extensions[134][0], */ /**< vmread opcode */
+/* 608 */     OP_vmwrite,        /* &prefix_extensions[135][0], */ /**< vmwrite opcode */
+
+    /* undocumented */
+/* 609 */     OP_int1,           /* &first_byte[0xf1], */ /**< int1 opcode */
+/* 610 */     OP_salc,           /* &first_byte[0xd6], */ /**< salc opcode */
+/* 611 */     OP_ffreep,         /* &float_high_modrm[7][0x00], */ /**< ffreep opcode */
+
+    /* Keep these at the end so that ifdefs don't change internal enum values */
+#ifdef IA32_ON_IA64
+/* 612 */     OP_jmpe,       /* &extensions[13][6], */ /**< jmpe opcode */
+/* 613 */     OP_jmpe_abs,   /* &second_byte[0xb8], */ /**< jmpe_abs opcode */
+#endif
+
+    OP_FIRST = OP_add, /**< First real opcode. */
+#ifdef IA32_ON_IA64
+    OP_LAST = OP_jmpe_abs, /**< Last real opcode. */
+#else
+    OP_LAST = OP_ffreep,   /**< Last real opcode. */
+#endif
+};
+
+#ifdef IA32_ON_IA64
+/* redefine instead of if else so works with genapi.pl script */
+#define OP_LAST OP_jmpe_abs
+#endif
+
+/* alternative names */
+/* we do not equate the fwait+op opcodes
+ *   fstsw, fstcw, fstenv, finit, fclex
+ * for us that has to be a sequence of instructions: a separate fwait
+ */
+/* 16-bit versions that have different names */
+#define OP_cbw        OP_cwde /**< Alternative opcode name for 16-bit version. */
+#define OP_cwd        OP_cdq /**< Alternative opcode name for 16-bit version. */
+#define OP_jcxz       OP_jecxz /**< Alternative opcode name for 16-bit version. */
+/* 64-bit versions that have different names */
+#define OP_jrcxz      OP_jecxz     /**< Alternative opcode name for 64-bit version. */
+#define OP_cmpxchg16b OP_cmpxchg8b /**< Alternative opcode name for 64-bit version. */
+#define OP_pextrq     OP_pextrd    /**< Alternative opcode name for 64-bit version. */
+#define OP_pinsrq     OP_pinsrd    /**< Alternative opcode name for 64-bit version. */
+/* reg-reg version has different name */
+#define OP_movhlps    OP_movlps /**< Alternative opcode name for reg-reg version. */
+#define OP_movlhps    OP_movhps /**< Alternative opcode name for reg-reg version. */
+/* condition codes */
+#define OP_jae_short  OP_jnb_short  /**< Alternative opcode name. */
+#define OP_jnae_short OP_jb_short   /**< Alternative opcode name. */
+#define OP_ja_short   OP_jnbe_short /**< Alternative opcode name. */
+#define OP_jna_short  OP_jbe_short  /**< Alternative opcode name. */
+#define OP_je_short   OP_jz_short   /**< Alternative opcode name. */
+#define OP_jne_short  OP_jnz_short  /**< Alternative opcode name. */
+#define OP_jge_short  OP_jnl_short  /**< Alternative opcode name. */
+#define OP_jg_short   OP_jnle_short /**< Alternative opcode name. */
+#define OP_jae  OP_jnb        /**< Alternative opcode name. */
+#define OP_jnae OP_jb         /**< Alternative opcode name. */
+#define OP_ja   OP_jnbe       /**< Alternative opcode name. */
+#define OP_jna  OP_jbe        /**< Alternative opcode name. */
+#define OP_je   OP_jz         /**< Alternative opcode name. */
+#define OP_jne  OP_jnz        /**< Alternative opcode name. */
+#define OP_jge  OP_jnl        /**< Alternative opcode name. */
+#define OP_jg   OP_jnle       /**< Alternative opcode name. */
+#define OP_setae  OP_setnb    /**< Alternative opcode name. */
+#define OP_setnae OP_setb     /**< Alternative opcode name. */
+#define OP_seta   OP_setnbe   /**< Alternative opcode name. */
+#define OP_setna  OP_setbe    /**< Alternative opcode name. */
+#define OP_sete   OP_setz     /**< Alternative opcode name. */
+#define OP_setne  OP_setnz    /**< Alternative opcode name. */
+#define OP_setge  OP_setnl    /**< Alternative opcode name. */
+#define OP_setg   OP_setnle   /**< Alternative opcode name. */
+#define OP_cmovae  OP_cmovnb  /**< Alternative opcode name. */
+#define OP_cmovnae OP_cmovb   /**< Alternative opcode name. */
+#define OP_cmova   OP_cmovnbe /**< Alternative opcode name. */
+#define OP_cmovna  OP_cmovbe  /**< Alternative opcode name. */
+#define OP_cmove   OP_cmovz   /**< Alternative opcode name. */
+#define OP_cmovne  OP_cmovnz  /**< Alternative opcode name. */
+#define OP_cmovge  OP_cmovnl  /**< Alternative opcode name. */
+#define OP_cmovg   OP_cmovnle /**< Alternative opcode name. */
+/* undocumented opcodes */
+#define OP_icebp OP_int1
+#define OP_setalc OP_salc
+
+/****************************************************************************/
+/* DR_API EXPORT END */
+
+#endif /* _INSTR_H_ */
diff --git a/core/x86/instr_create.h b/core/x86/instr_create.h
new file mode 100644
index 000000000..efc953dc1
--- /dev/null
+++ b/core/x86/instr_create.h
@@ -0,0 +1,2487 @@
+/* **********************************************************
+ * Copyright (c) 2002-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+
+#ifndef _INSTR_CREATE_H_
+#define _INSTR_CREATE_H_ 1
+
+/* DR_API EXPORT TOFILE dr_ir_macros.h */
+/* DR_API EXPORT BEGIN */
+/**
+ * @file dr_ir_macros.h
+ * @brief Instruction creation convenience macros.
+ *
+ * All macros assume default data and address sizes.  For the most part these
+ * macros do not support building non-default address or data size
+ * versions; for that, simply duplicate the macro's body, replacing the
+ * SIZE and/or hardcoded registers with smaller versions (the IR does
+ * not support cs segments with non-default sizes where the default
+ * size requires instruction prefixes).  For shrinking data sizes, see
+ * the instr_shrink_to_16_bits() routine.
+ */
+
+#include <math.h> /* for floating-point math constants */
+
+/* instruction modification convenience routines */
+/**
+ * Add the lock prefix to an instruction. For example:
+ * instr_t *lock_inc_instr = LOCK(INSTR_CREATE_inc(....));
+ */
+#define LOCK(instr_ptr) instr_set_prefix_flag((instr_ptr), PREFIX_LOCK)
+/**
+ * Set the translation field for an instruction. For example:
+ * instr_t *pushf_instr = INSTR_XL8(INSTR_CREATE_pushf(drcontext), addr);
+ */
+#define INSTR_XL8(instr_ptr, app_addr) instr_set_translation((instr_ptr), (app_addr))
+
+/* operand convenience routines for common cases */
+/** Create a base+disp 8-byte operand. */
+#define OPND_CREATE_MEM64(base_reg, disp) \
+  opnd_create_base_disp(base_reg, REG_NULL, 0, disp, OPSZ_8)
+/** Create a base+disp 4-byte operand. */
+#define OPND_CREATE_MEM32(base_reg, disp) \
+  opnd_create_base_disp(base_reg, REG_NULL, 0, disp, OPSZ_4)
+/** Create a base+disp 2-byte operand. */
+#define OPND_CREATE_MEM16(base_reg, disp) \
+  opnd_create_base_disp(base_reg, REG_NULL, 0, disp, OPSZ_2)
+/** Create a base+disp 1-byte operand. */
+#define OPND_CREATE_MEM8(base_reg, disp) \
+  opnd_create_base_disp(base_reg, REG_NULL, 0, disp, OPSZ_1)
+#ifdef X64
+/** Create a base+disp pointer-sized operand. */
+# define OPND_CREATE_MEMPTR OPND_CREATE_MEM64
+/**
+ * Create an absolute address operand encoded as pc-relative.
+ * Encoding will fail if addr is out of 32-bit-signed-displacement reach.
+ */
+# define OPND_CREATE_ABSMEM(addr, size) \
+  opnd_create_rel_addr(addr, size)
+#else
+/** Create a base+disp pointer-sized operand. */
+# define OPND_CREATE_MEMPTR OPND_CREATE_MEM32
+/** Create an absolute address operand. */
+# define OPND_CREATE_ABSMEM(addr, size) \
+  opnd_create_abs_addr(addr, size)
+#endif 
+
+#ifdef X64
+/** Create an 8-byte immediate integer operand. */
+#define OPND_CREATE_INT64(val) opnd_create_immed_int((ptr_int_t)(val), OPSZ_8)
+/** Create a pointer-sized immediate integer operand. */
+# define OPND_CREATE_INTPTR OPND_CREATE_INT64
+#else
+/** Create a pointer-sized immediate integer operand. */
+# define OPND_CREATE_INTPTR OPND_CREATE_INT32
+#endif
+/** Create a 4-byte immediate integer operand. */
+#define OPND_CREATE_INT32(val) opnd_create_immed_int((ptr_int_t)(val), OPSZ_4)
+/** Create a 2-byte immediate integer operand. */
+#define OPND_CREATE_INT16(val) opnd_create_immed_int((ptr_int_t)(val), OPSZ_2)
+/** Create a 1-byte immediate integer operand. */
+#define OPND_CREATE_INT8(val) opnd_create_immed_int((ptr_int_t)(val), OPSZ_1)
+/**
+ * Create a 1-byte immediate interger operand if val will fit, else create a 4-byte
+ * immediate integer operand.
+ */
+#define OPND_CREATE_INT_32OR8(val) ((val) <= INT8_MAX && (ptr_int_t)(val) >= INT8_MIN ? \
+    OPND_CREATE_INT8(val) : OPND_CREATE_INT32(val))
+/**
+ * Create a 1-byte immediate interger operand if val will fit, else create a 2-byte
+ * immediate integer operand.
+ */
+#define OPND_CREATE_INT_16OR8(val) ((val) <= INT8_MAX && (ptr_int_t)(val) >= INT8_MIN ? \
+    OPND_CREATE_INT8(val) : OPND_CREATE_INT16(val))
+
+
+/* operand convenience routines for specific opcodes with odd sizes */
+/** Create a memory reference operand appropriately sized for OP_lea. */
+#define OPND_CREATE_MEM_lea(base, index, scale, disp) \
+    opnd_create_base_disp(base, index, scale, disp, OPSZ_lea)
+/** Create a memory reference operand appropriately sized for OP_invlpg. */
+#define OPND_CREATE_MEM_invlpg(base, index, scale, disp) \
+    opnd_create_base_disp(base, index, scale, disp, OPSZ_invlpg)
+/** Create a memory reference operand appropriately sized for OP_clflush. */
+#define OPND_CREATE_MEM_clflush(base, index, scale, disp) \
+    opnd_create_base_disp(base, index, scale, disp, OPSZ_clflush)
+/** Create a memory reference operand appropriately sized for OP_prefetch*. */
+#define OPND_CREATE_MEM_prefetch(base, index, scale, disp) \
+    opnd_create_base_disp(base, index, scale, disp, OPSZ_prefetch)
+/** Create a memory reference operand appropriately sized for OP_lgdt. */
+#define OPND_CREATE_MEM_lgdt(base, index, scale, disp) \
+    opnd_create_base_disp(base, index, scale, disp, OPSZ_lgdt)
+/** Create a memory reference operand appropriately sized for OP_sgdt. */
+#define OPND_CREATE_MEM_sgdt(base, index, scale, disp) \
+    opnd_create_base_disp(base, index, scale, disp, OPSZ_sgdt)
+/** Create a memory reference operand appropriately sized for OP_lidt. */
+#define OPND_CREATE_MEM_lidt(base, index, scale, disp) \
+    opnd_create_base_disp(base, index, scale, disp, OPSZ_lidt)
+/** Create a memory reference operand appropriately sized for OP_sidt. */
+#define OPND_CREATE_MEM_sidt(base, index, scale, disp) \
+    opnd_create_base_disp(base, index, scale, disp, OPSZ_sidt)
+/** Create a memory reference operand appropriately sized for OP_bound. */
+#define OPND_CREATE_MEM_bound(base, index, scale, disp) \
+    opnd_create_base_disp(base, index, scale, disp, OPSZ_bound)
+/** Create a memory reference operand appropriately sized for OP_fldenv. */
+#define OPND_CREATE_MEM_fldenv(base, index, scale, disp) \
+    opnd_create_base_disp(base, index, scale, disp, OPSZ_fldenv)
+/** Create a memory reference operand appropriately sized for OP_fnstenv. */
+#define OPND_CREATE_MEM_fnstenv(base, index, scale, disp) \
+    opnd_create_base_disp(base, index, scale, disp, OPSZ_fnstenv)
+/** Create a memory reference operand appropriately sized for OP_fnsave. */
+#define OPND_CREATE_MEM_fnsave(base, index, scale, disp) \
+    opnd_create_base_disp(base, index, scale, disp, OPSZ_fnsave)
+/** Create a memory reference operand appropriately sized for OP_frstor. */
+#define OPND_CREATE_MEM_frstor(base, index, scale, disp) \
+    opnd_create_base_disp(base, index, scale, disp, OPSZ_frstor)
+/** Create a memory reference operand appropriately sized for OP_fxsave. */
+#define OPND_CREATE_MEM_fxsave(base, index, scale, disp) \
+    opnd_create_base_disp(base, index, scale, disp, OPSZ_fxsave)
+/** Create a memory reference operand appropriately sized for OP_fxrstor. */
+#define OPND_CREATE_MEM_fxrstor(base, index, scale, disp) \
+    opnd_create_base_disp(base, index, scale, disp, OPSZ_fxrstor)
+
+/* Macros for building instructions, one for each opcode.
+ * Each INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * The macro parameter types, encoded by name, are:
+ *   dc = DR Context*
+ *   op = uint = opcode
+ *   s  = opnd_t = source operand
+ *   i  = opnd_t = source operand that is an immediate
+ *   ri = opnd_t = source operand that can be a register or an immediate
+ *   t  = opnd_t = source operand that is a jump target
+ *   m  = opnd_t = source operand that can only reference memory
+ *   f  = opnd_t = floating point register operand
+ *   d  = opnd_t = destination operand
+ */
+
+/* no-operand instructions */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx, automatically
+ * supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ */
+#define INSTR_CREATE_fwait(dc)    instr_create_0dst_0src((dc), OP_fwait)
+#define INSTR_CREATE_hlt(dc)      instr_create_0dst_0src((dc), OP_hlt)
+#define INSTR_CREATE_cmc(dc)      instr_create_0dst_0src((dc), OP_cmc)
+#define INSTR_CREATE_clc(dc)      instr_create_0dst_0src((dc), OP_clc)
+#define INSTR_CREATE_stc(dc)      instr_create_0dst_0src((dc), OP_stc)
+#define INSTR_CREATE_cli(dc)      instr_create_0dst_0src((dc), OP_cli)
+#define INSTR_CREATE_sti(dc)      instr_create_0dst_0src((dc), OP_sti)
+#define INSTR_CREATE_cld(dc)      instr_create_0dst_0src((dc), OP_cld)
+#define INSTR_CREATE_std(dc)      instr_create_0dst_0src((dc), OP_std)
+#define INSTR_CREATE_clts(dc)     instr_create_0dst_0src((dc), OP_clts)
+#define INSTR_CREATE_invd(dc)     instr_create_0dst_0src((dc), OP_invd)
+#define INSTR_CREATE_wbinvd(dc)   instr_create_0dst_0src((dc), OP_wbinvd)
+#define INSTR_CREATE_ud2a(dc)     instr_create_0dst_0src((dc), OP_ud2a)
+#define INSTR_CREATE_emms(dc)     instr_create_0dst_0src((dc), OP_emms)
+#define INSTR_CREATE_rsm(dc)      instr_create_0dst_0src((dc), OP_rsm)
+#define INSTR_CREATE_ud2b(dc)     instr_create_0dst_0src((dc), OP_ud2b)
+#define INSTR_CREATE_lfence(dc)   instr_create_0dst_0src((dc), OP_lfence)
+#define INSTR_CREATE_mfence(dc)   instr_create_0dst_0src((dc), OP_mfence)
+#define INSTR_CREATE_sfence(dc)   instr_create_0dst_0src((dc), OP_sfence)
+#define INSTR_CREATE_nop(dc)      instr_create_0dst_0src((dc), OP_nop)
+#define INSTR_CREATE_pause(dc)    instr_create_0dst_0src((dc), OP_pause)
+#define INSTR_CREATE_fnop(dc)     instr_create_0dst_0src((dc), OP_fnop)
+#define INSTR_CREATE_fdecstp(dc)  instr_create_0dst_0src((dc), OP_fdecstp)
+#define INSTR_CREATE_fincstp(dc)  instr_create_0dst_0src((dc), OP_fincstp)
+#define INSTR_CREATE_fnclex(dc)   instr_create_0dst_0src((dc), OP_fnclex)
+#define INSTR_CREATE_fninit(dc)   instr_create_0dst_0src((dc), OP_fninit)
+#define INSTR_CREATE_sysret(dc)   instr_create_0dst_0src((dc), OP_sysret)
+#define INSTR_CREATE_femms(dc)    instr_create_0dst_0src((dc), OP_femms)
+#define INSTR_CREATE_swapgs(dc)   instr_create_0dst_0src((dc), OP_swapgs)
+#define INSTR_CREATE_vmcall(dc)   instr_create_0dst_0src((dc), OP_vmcall)
+#define INSTR_CREATE_vmlaunch(dc) instr_create_0dst_0src((dc), OP_vmlaunch)
+#define INSTR_CREATE_vmresume(dc) instr_create_0dst_0src((dc), OP_vmresume)
+#define INSTR_CREATE_vmxoff(dc)   instr_create_0dst_0src((dc), OP_vmxoff)
+/* @} */ /* end doxygen group */
+/**
+ * Creates an instr_t with opcode OP_LABEL.  An OP_LABEL instruction can be used as a
+ * jump or call instr_t target, and when emitted it will take no space in the
+ * resulting machine code.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ */
+#define INSTR_CREATE_label(dc)    instr_create_0dst_0src((dc), OP_LABEL)
+
+/* no destination, 1 source */
+/**
+ * Creates an instr_t for a short conditional branch instruction with the given
+ * opcode and target operand.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param op The OP_xxx opcode for the conditional branch, which should be
+ * in the range [OP_jo_short, OP_jnle_short].
+ * \param t The opnd_t target operand for the instruction, which can be either
+ * a pc (opnd_create_pc()) or an instr_t (opnd_create_instr()).  Be sure to
+ * ensure that the limited reach of this short branch will reach the target
+ * (a pc operand is not suitable for most uses unless you know precisely where
+ * this instruction will be encoded).
+ */
+#define INSTR_CREATE_jcc_short(dc, op, t) \
+  instr_create_0dst_1src((dc), (op), (t))
+/**
+ * Creates an instr_t for a conditional branch instruction with the given opcode
+ * and target operand.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param op The OP_xxx opcode for the conditional branch, which should be
+ * in the range [OP_jo, OP_jnle].
+ * \param t The opnd_t target operand for the instruction, which can be either
+ * a pc (opnd_create_pc()) or an instr_t (opnd_create_instr()).
+ */
+#define INSTR_CREATE_jcc(dc, op, t) \
+  instr_create_0dst_1src((dc), (op), (t))
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param t The opnd_t target operand for the instruction, which can be either
+ * a pc (opnd_create_pc()) or an instr_t (opnd_create_instr()).
+ */
+#define INSTR_CREATE_jmp(dc, t) \
+  instr_create_0dst_1src((dc), OP_jmp, (t))
+#define INSTR_CREATE_jmp_short(dc, t) \
+  instr_create_0dst_1src((dc), OP_jmp_short, (t))
+/* @} */ /* end doxygen group */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param t The opnd_t target operand for the instruction, which should be
+ * a memory reference created with opnd_create_base_disp().
+ */
+#define INSTR_CREATE_jmp_ind(dc, t) \
+  instr_create_0dst_1src((dc), OP_jmp_ind, (t))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param t The opnd_t target operand for the instruction, which should be
+ * a far pc operand created with opnd_create_far_pc().
+ */
+#define INSTR_CREATE_jmp_far(dc, t) \
+  instr_create_0dst_1src((dc), OP_jmp_far, (t))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param t The opnd_t target operand for the instruction, which should be
+ * a far memory reference created with opnd_create_far_base_disp().
+ */
+#define INSTR_CREATE_jmp_far_ind(dc, t) \
+  instr_create_0dst_1src((dc), OP_jmp_far_ind, (t))
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param s The opnd_t explicit source operand for the instruction.
+ */
+#define INSTR_CREATE_lldt(dc, s) \
+  instr_create_0dst_1src((dc), OP_lldt, (s))
+#define INSTR_CREATE_ltr(dc, s) \
+  instr_create_0dst_1src((dc), OP_ltr, (s))
+#define INSTR_CREATE_verr(dc, s) \
+  instr_create_0dst_1src((dc), OP_verr, (s))
+#define INSTR_CREATE_verw(dc, s) \
+  instr_create_0dst_1src((dc), OP_verw, (s))
+#define INSTR_CREATE_vmptrld(dc, s) \
+  instr_create_0dst_1src((dc), OP_vmptrld, (s))
+#define INSTR_CREATE_vmxon(dc, s) \
+  instr_create_0dst_1src((dc), OP_vmxon, (s))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param s The opnd_t explicit source operand for the instruction, which can be
+ * created with OPND_CREATE_MEM_lgdt() to get the appropriate operand size.
+ */
+#define INSTR_CREATE_lgdt(dc, s) \
+  instr_create_0dst_1src((dc), OP_lgdt, (s))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param s The opnd_t explicit source operand for the instruction, which can be
+ * created with OPND_CREATE_MEM_lidt() to get the appropriate operand size.
+ */
+#define INSTR_CREATE_lidt(dc, s) \
+  instr_create_0dst_1src((dc), OP_lidt, (s))
+#define INSTR_CREATE_lmsw(dc, s) \
+  instr_create_0dst_1src((dc), OP_lmsw, (s))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param s The opnd_t explicit source operand for the instruction, which can be
+ * created with OPND_CREATE_MEM_invlpg() to get the appropriate operand size.
+ */
+#define INSTR_CREATE_invlpg(dc, s) \
+  instr_create_0dst_1src((dc), OP_invlpg, (s))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param s The opnd_t explicit source operand for the instruction, which can be
+ * created with OPND_CREATE_MEM_fxrstor() to get the appropriate operand size.
+ */
+#define INSTR_CREATE_fxrstor(dc, s) \
+  instr_create_0dst_1src((dc), OP_fxrstor, (s))
+#define INSTR_CREATE_ldmxcsr(dc, s) \
+  instr_create_0dst_1src((dc), OP_ldmxcsr, (s))
+#define INSTR_CREATE_nop_modrm(dc, s) \
+  instr_create_0dst_1src((dc), OP_nop_modrm, (s))
+/* @} */ /* end doxygen group */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param s The opnd_t explicit source operand for the instruction, which can be
+ * created with OPND_CREATE_MEM_prefetch() to get the appropriate operand size.
+ */
+#define INSTR_CREATE_prefetchnta(dc, s) \
+  instr_create_0dst_1src((dc), OP_prefetchnta, (s))
+#define INSTR_CREATE_prefetcht0(dc, s) \
+  instr_create_0dst_1src((dc), OP_prefetcht0, (s))
+#define INSTR_CREATE_prefetcht1(dc, s) \
+  instr_create_0dst_1src((dc), OP_prefetcht1, (s))
+#define INSTR_CREATE_prefetcht2(dc, s) \
+  instr_create_0dst_1src((dc), OP_prefetcht2, (s))
+#define INSTR_CREATE_prefetch(dc, s) \
+  instr_create_0dst_1src((dc), OP_prefetch, (s))
+#define INSTR_CREATE_prefetchw(dc, s) \
+  instr_create_0dst_1src((dc), OP_prefetchw, (s))
+/* @} */ /* end doxygen group */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param s The opnd_t explicit source operand for the instruction, which can be
+ * created with OPND_CREATE_MEM_clflush() to get the appropriate operand size.
+ */
+#define INSTR_CREATE_clflush(dc, s) \
+  instr_create_0dst_1src((dc), OP_clflush, (s))
+
+#ifdef IA32_ON_IA64
+/* DR_API EXPORT BEGIN */
+#define INSTR_CREATE_jmpe(dc, t) \
+  instr_create_0dst_1src((dc), OP_jmpe, (t))
+#define INSTR_CREATE_jmpe_abs(dc, t) \
+  instr_create_0dst_1src((dc), OP_jmpe_abs, (t))
+#endif
+
+/* floating-point */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param m The opnd_t explicit destination operand for the instruction, which can be
+ * created with OPND_CREATE_MEM_fldenv() to get the appropriate operand size.
+ */
+#define INSTR_CREATE_fldenv(dc, m) \
+  instr_create_0dst_1src((dc), OP_fldenv, (m))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param m The opnd_t explicit destination operand for the instruction, which must
+ * be a memory reference (opnd_create_base_disp() or opnd_create_far_base_disp()).
+ */
+#define INSTR_CREATE_fldcw(dc, m) \
+  instr_create_0dst_1src((dc), OP_fldcw, (m))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param m The opnd_t explicit destination operand for the instruction, which can be
+ * created with OPND_CREATE_MEM_frstor() to get the appropriate operand size.
+ */
+#define INSTR_CREATE_frstor(dc, m) \
+  instr_create_0dst_1src((dc), OP_frstor, (m))
+
+/* no destination, 1 implicit source */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx, automatically
+ * supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ */
+#define INSTR_CREATE_fxam(dc) \
+  instr_create_0dst_1src((dc), OP_fxam, opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_sahf(dc) \
+  instr_create_0dst_1src((dc), OP_sahf, opnd_create_reg(REG_AH))
+/* @} */ /* end doxygen group */
+
+/* no destination, 2 sources */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param s1 The opnd_t first source operand for the instruction.
+ * \param s2 The opnd_t second source operand for the instruction.
+ */
+#define INSTR_CREATE_cmp(dc, s1, s2) \
+  instr_create_0dst_2src((dc), OP_cmp, (s1), (s2))
+#define INSTR_CREATE_test(dc, s1, s2) \
+  instr_create_0dst_2src((dc), OP_test, (s1), (s2))
+#define INSTR_CREATE_ptest(dc, s1, s2) \
+  instr_create_0dst_2src((dc), OP_ptest, (s1), (s2))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param s1 The opnd_t first source operand for the instruction.
+ * \param s2 The opnd_t second source operand for the instruction, which can
+ * be created with OPND_CREATE_MEM_bound() to get the appropriate operand size.
+ */
+#define INSTR_CREATE_bound(dc, s1, s2) \
+  instr_create_0dst_2src((dc), OP_bound, (s1), (s2))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param s The opnd_t first source operand for the instruction.
+ * \param ri The opnd_t second source operand for the instruction, which can
+ * be either a register or an immediate integer.
+ */
+#define INSTR_CREATE_bt(dc, s, ri) \
+  instr_create_0dst_2src((dc), OP_bt, (s), (ri))
+#define INSTR_CREATE_ucomiss(dc, s1, s2) \
+  instr_create_0dst_2src((dc), OP_ucomiss, (s1), (s2))
+#define INSTR_CREATE_ucomisd(dc, s1, s2) \
+  instr_create_0dst_2src((dc), OP_ucomisd, (s1), (s2))
+#define INSTR_CREATE_comiss(dc, s1, s2) \
+  instr_create_0dst_2src((dc), OP_comiss, (s1), (s2))
+#define INSTR_CREATE_comisd(dc, s1, s2) \
+  instr_create_0dst_2src((dc), OP_comisd, (s1), (s2))
+/* @} */ /* end doxygen group */
+
+/* no destination, 2 sources: 1 implicit */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param t The opnd_t target operand for the instruction, which can be either
+ * a pc (opnd_create_pc()) or an instr_t (opnd_create_instr()).
+ */
+#define INSTR_CREATE_jecxz(dc, t) \
+  instr_create_0dst_2src((dc), OP_jecxz, (t), opnd_create_reg(REG_XCX))
+/**
+ * Creates an instr_t for an OP_jecxz instruction that uses cx instead of ecx
+ * (there is no separate OP_jcxz).
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param t The opnd_t target operand for the instruction, which can be either
+ * a pc (opnd_create_pc()) or an instr_t (opnd_create_instr()).
+ */
+#define INSTR_CREATE_jcxz(dc, t) \
+  instr_create_0dst_2src((dc), OP_jecxz, (t), opnd_create_reg(REG_CX))
+
+/* no destination, 2 sources */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * Creates an instr_t for an OP_out instruction with a source of al
+ * (INSTR_CREATE_out_1()) or eax (INSTR_CREATE_out_4()) and dx.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ */
+#define INSTR_CREATE_out_1(dc) \
+  instr_create_0dst_2src((dc), OP_out, opnd_create_reg(REG_AL), opnd_create_reg(REG_DX))
+#define INSTR_CREATE_out_4(dc) \
+  instr_create_0dst_2src((dc), OP_out, opnd_create_reg(REG_EAX), opnd_create_reg(REG_DX))
+/* @} */ /* end doxygen group */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * Creates an instr_t for an OP_out instruction with a source of al
+ * (INSTR_CREATE_out_1_imm()) or eax (INSTR_CREATE_out_4_imm()) and an immediate.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param i The opnd_t explicit source operand for the instruction, which must be an
+ * immediate integer (opnd_create_immed_int()).
+ */
+#define INSTR_CREATE_out_1_imm(dc, i) \
+  instr_create_0dst_2src((dc), OP_out, (i), opnd_create_reg(REG_AL))
+#define INSTR_CREATE_out_4_imm(dc, i) \
+  instr_create_0dst_2src((dc), OP_out, (i), opnd_create_reg(REG_EAX))
+/* @} */ /* end doxygen group */
+
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx, automatically
+ * supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ */
+/* no destination, 2 implicit sources */
+#define INSTR_CREATE_mwait(dc) \
+  instr_create_0dst_2src((dc), OP_mwait, opnd_create_reg(REG_EAX), \
+      opnd_create_reg(REG_ECX))
+/* no destination, 3 implicit sources */
+#define INSTR_CREATE_wrmsr(dc) \
+  instr_create_0dst_3src((dc), OP_wrmsr, opnd_create_reg(REG_EDX), \
+    opnd_create_reg(REG_EAX), opnd_create_reg(REG_ECX))
+#define INSTR_CREATE_monitor(dc) \
+  instr_create_0dst_3src((dc), OP_monitor, opnd_create_reg(REG_EAX), \
+      opnd_create_reg(REG_ECX), opnd_create_reg(REG_EDX))
+/* @} */ /* end doxygen group */
+
+/* floating-point */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the
+ * given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param s The opnd_t explicit source operand for the instruction, which must
+ * be one of the following:
+ * -# A floating point register (opnd_create_reg()).
+ * -# A memory reference (opnd_create_base_disp() or opnd_create_far_base_disp()).
+ * The other (implicit) source operand is #REG_ST0.
+ */
+#define INSTR_CREATE_fcom(dc, s) \
+  instr_create_0dst_2src((dc), OP_fcom, (s), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_fcomp(dc, s) \
+  instr_create_0dst_2src((dc), OP_fcomp, (s), opnd_create_reg(REG_ST0))
+/* @} */ /* end doxygen group */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the
+ * given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param f The opnd_t explicit source operand for the instruction, which must
+ * be a floating point register (opnd_create_reg()).
+ * The other (implicit) source operand is #REG_ST0.
+ */
+#define INSTR_CREATE_fcomi(dc, f) \
+  instr_create_0dst_2src((dc), OP_fcomi, opnd_create_reg(REG_ST0), (f))
+#define INSTR_CREATE_fcomip(dc, f) \
+  instr_create_0dst_2src((dc), OP_fcomip, opnd_create_reg(REG_ST0), (f))
+#define INSTR_CREATE_fucomi(dc, f) \
+  instr_create_0dst_2src((dc), OP_fucomi, opnd_create_reg(REG_ST0), (f))
+#define INSTR_CREATE_fucomip(dc, f) \
+  instr_create_0dst_2src((dc), OP_fucomip, opnd_create_reg(REG_ST0), (f))
+#define INSTR_CREATE_fucom(dc, f) \
+  instr_create_0dst_2src((dc), OP_fucom, opnd_create_reg(REG_ST0), (f))
+#define INSTR_CREATE_fucomp(dc, f) \
+  instr_create_0dst_2src((dc), OP_fucomp, opnd_create_reg(REG_ST0), (f))
+/* @} */ /* end doxygen group */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx,
+ * automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ */
+#define INSTR_CREATE_fucompp(dc) \
+  instr_create_0dst_2src((dc), OP_fucompp, opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST1))
+#define INSTR_CREATE_fcompp(dc) \
+  instr_create_0dst_2src((dc), OP_fcompp, opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST1))
+/* @} */ /* end doxygen group */
+
+/* 1 destination, no sources */
+/**
+ * Creats an instr_t for a conditional set instruction with the given opcode
+ * and destination operand.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param op The OP_xxx opcode for the instruction, which should be in the range
+ * [OP_seto, OP_setnle].
+ * \param d The opnd_t destination operand for the instruction.
+ */
+#define INSTR_CREATE_setcc(dc, op, d) \
+  instr_create_1dst_0src((dc), (op), (d))
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ */
+#define INSTR_CREATE_sldt(dc, d) \
+  instr_create_1dst_0src((dc), OP_sldt, (d))
+#define INSTR_CREATE_str(dc, d) \
+  instr_create_1dst_0src((dc), OP_str, (d))
+#define INSTR_CREATE_vmptrst(dc, d) \
+  instr_create_1dst_0src((dc), OP_vmptrst, (d))
+#define INSTR_CREATE_vmclear(dc, d) \
+  instr_create_1dst_0src((dc), OP_vmclear, (d))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction, which can
+ * be created with OPND_CREATE_MEM_sgdt() to get the appropriate operand size.
+ */
+#define INSTR_CREATE_sgdt(dc, d) \
+  instr_create_1dst_0src((dc), OP_sgdt, (d))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction, which can
+ * be created with OPND_CREATE_MEM_sidt() to get the appropriate operand size.
+ */
+#define INSTR_CREATE_sidt(dc, d) \
+  instr_create_1dst_0src((dc), OP_sidt, (d))
+#define INSTR_CREATE_smsw(dc, d) \
+  instr_create_1dst_0src((dc), OP_smsw, (d))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction, which can
+ * be created with OPND_CREATE_MEM_fxsave() to get the appropriate operand size.
+ */
+#define INSTR_CREATE_fxsave(dc, d) \
+  instr_create_1dst_0src((dc), OP_fxsave, (d))
+#define INSTR_CREATE_stmxcsr(dc, d) \
+  instr_create_1dst_0src((dc), OP_stmxcsr, (d))
+/* @} */ /* end doxygen group */
+
+/* floating-point */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param m The opnd_t explicit destination operand for the instruction, which must
+ * be a memory reference (opnd_create_base_disp() or opnd_create_far_base_disp()).
+ */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param m The opnd_t explicit destination operand for the instruction, which can
+ * be created with OPND_CREATE_MEM_fnstenv() to get the appropriate operand size.
+ */
+#define INSTR_CREATE_fnstenv(dc, m) \
+  instr_create_1dst_0src((dc), OP_fnstenv, (m))
+#define INSTR_CREATE_fnstcw(dc, m) \
+  instr_create_1dst_0src((dc), OP_fnstcw, (m))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param m The opnd_t explicit destination operand for the instruction, which can
+ * be created with OPND_CREATE_MEM_fnsave() to get the appropriate operand size.
+ */
+#define INSTR_CREATE_fnsave(dc, m) \
+  instr_create_1dst_0src((dc), OP_fnsave, (m))
+#define INSTR_CREATE_fnstsw(dc, m) \
+  instr_create_1dst_0src((dc), OP_fnstsw, (m))
+/* @} */ /* end doxygen group */
+
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param f The opnd_t explicit destination operand for the instruction, which must
+ * be a floating point register (opnd_create_reg()).
+ */
+#define INSTR_CREATE_ffree(dc, f) \
+  instr_create_1dst_0src((dc), OP_ffree, (f))
+#define INSTR_CREATE_ffreep(dc, f) \
+  instr_create_1dst_0src((dc), OP_ffreep, (f))
+
+/* 1 implicit destination, no sources */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx, automatically
+ * supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ */
+#define INSTR_CREATE_lahf(dc) \
+  instr_create_1dst_0src((dc), OP_lahf, opnd_create_reg(REG_AH))
+#define INSTR_CREATE_sysenter(dc) \
+  instr_create_1dst_0src((dc), OP_sysenter, opnd_create_reg(REG_XSP))
+#define INSTR_CREATE_sysexit(dc) \
+  instr_create_1dst_0src((dc), OP_sysexit, opnd_create_reg(REG_XSP))
+#define INSTR_CREATE_syscall(dc) \
+  instr_create_1dst_0src((dc), OP_syscall, opnd_create_reg(REG_XCX))
+#define INSTR_CREATE_salc(dc) \
+  instr_create_1dst_0src((dc), OP_salc, opnd_create_reg(REG_AL))
+/* @} */ /* end doxygen group */
+
+/* 1 destination, 1 source */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ * \param s The opnd_t explicit source operand for the instruction.
+ */
+#define INSTR_CREATE_arpl(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_arpl, (d), (s))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ * \param s The opnd_t explicit source operand for the instruction, which can be
+ * created with OPND_CREATE_MEM_lea() to get the appropriate operand size.
+ */
+#define INSTR_CREATE_lea(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_lea, (d), (s))
+#define INSTR_CREATE_mov_ld(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_mov_ld, (d), (s))
+#define INSTR_CREATE_mov_st(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_mov_st, (d), (s))
+#define INSTR_CREATE_mov_imm(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_mov_imm, (d), (s))
+#define INSTR_CREATE_mov_seg(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_mov_seg, (d), (s))
+#define INSTR_CREATE_mov_priv(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_mov_priv, (d), (s))
+#define INSTR_CREATE_lar(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_lar, (d), (s))
+#define INSTR_CREATE_lsl(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_lsl, (d), (s))
+#define INSTR_CREATE_movntps(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movntps, (d), (s))
+#define INSTR_CREATE_movntpd(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movntpd, (d), (s))
+#define INSTR_CREATE_movd(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movd, (d), (s))
+#define INSTR_CREATE_movq(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movq, (d), (s))
+#define INSTR_CREATE_movdqu(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movdqu, (d), (s))
+#define INSTR_CREATE_movdqa(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movdqa, (d), (s))
+#define INSTR_CREATE_movzx(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movzx, (d), (s))
+#define INSTR_CREATE_movsx(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movsx, (d), (s))
+#define INSTR_CREATE_bsf(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_bsf, (d), (s))
+#define INSTR_CREATE_bsr(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_bsr, (d), (s))
+#define INSTR_CREATE_pmovmskb(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_pmovmskb, (d), (s))
+#define INSTR_CREATE_movups(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movups, (d), (s))
+#define INSTR_CREATE_movss(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movss, (d), (s))
+#define INSTR_CREATE_movupd(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movupd, (d), (s))
+#define INSTR_CREATE_movsd(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movsd, (d), (s))
+#define INSTR_CREATE_movlps(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movlps, (d), (s))
+#define INSTR_CREATE_movlpd(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movlpd, (d), (s))
+#define INSTR_CREATE_movhps(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movhps, (d), (s))
+#define INSTR_CREATE_movhpd(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movhpd, (d), (s))
+#define INSTR_CREATE_movaps(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movaps, (d), (s))
+#define INSTR_CREATE_movapd(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movapd, (d), (s))
+#define INSTR_CREATE_cvtpi2ps(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvtpi2ps, (d), (s))
+#define INSTR_CREATE_cvtsi2ss(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvtsi2ss, (d), (s))
+#define INSTR_CREATE_cvtpi2pd(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvtpi2pd, (d), (s))
+#define INSTR_CREATE_cvtsi2sd(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvtsi2sd, (d), (s))
+#define INSTR_CREATE_cvttps2pi(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvttps2pi, (d), (s))
+#define INSTR_CREATE_cvttss2si(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvttss2si, (d), (s))
+#define INSTR_CREATE_cvttpd2pi(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvttpd2pi, (d), (s))
+#define INSTR_CREATE_cvttsd2si(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvttsd2si, (d), (s))
+#define INSTR_CREATE_cvtps2pi(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvtps2pi, (d), (s))
+#define INSTR_CREATE_cvtss2si(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvtss2si, (d), (s))
+#define INSTR_CREATE_cvtpd2pi(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvtpd2pi, (d), (s))
+#define INSTR_CREATE_cvtsd2si(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvtsd2si, (d), (s))
+#define INSTR_CREATE_cvtps2pd(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvtps2pd, (d), (s))
+#define INSTR_CREATE_cvtss2sd(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvtss2sd, (d), (s))
+#define INSTR_CREATE_cvtpd2ps(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvtpd2ps, (d), (s))
+#define INSTR_CREATE_cvtsd2ss(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvtsd2ss, (d), (s))
+#define INSTR_CREATE_cvtdq2ps(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvtdq2ps, (d), (s))
+#define INSTR_CREATE_cvttps2dq(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvttps2dq, (d), (s))
+#define INSTR_CREATE_cvtps2dq(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvtps2dq, (d), (s))
+#define INSTR_CREATE_cvtdq2pd(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvtdq2pd, (d), (s))
+#define INSTR_CREATE_cvttpd2dq(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvttpd2dq, (d), (s))
+#define INSTR_CREATE_cvtpd2dq(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_cvtpd2dq, (d), (s))
+#define INSTR_CREATE_movmskps(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movmskps, (d), (s))
+#define INSTR_CREATE_movmskpd(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movmskpd, (d), (s))
+#define INSTR_CREATE_sqrtps(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_sqrtps, (d), (s))
+#define INSTR_CREATE_sqrtss(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_sqrtss, (d), (s))
+#define INSTR_CREATE_sqrtpd(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_sqrtpd, (d), (s))
+#define INSTR_CREATE_sqrtsd(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_sqrtsd, (d), (s))
+#define INSTR_CREATE_rsqrtps(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_rsqrtps, (d), (s))
+#define INSTR_CREATE_rsqrtss(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_rsqrtss, (d), (s))
+#define INSTR_CREATE_rcpps(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_rcpps, (d), (s))
+#define INSTR_CREATE_rcpss(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_rcpss, (d), (s))
+#define INSTR_CREATE_lddqu(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_lddqu, (d), (s))
+#define INSTR_CREATE_movsldup(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movsldup, (d), (s))
+#define INSTR_CREATE_movshdup(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movshdup, (d), (s))
+#define INSTR_CREATE_movddup(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movddup, (d), (s))
+#define INSTR_CREATE_pshufb(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_pshufb, (d), (s))
+#define INSTR_CREATE_popcnt(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_popcnt, (d), (s))
+#define INSTR_CREATE_movntss(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movntss, (d), (s))
+#define INSTR_CREATE_movntsd(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movntsd, (d), (s))
+#define INSTR_CREATE_movntq(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movntq, (d), (s))
+#define INSTR_CREATE_movntdq(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movntdq, (d), (s))
+#define INSTR_CREATE_movnti(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movnti, (d), (s))
+#define INSTR_CREATE_lzcnt(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_lzcnt, (d), (s))
+#define INSTR_CREATE_pmovsxbw(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_pmovsxbw, (d), (s))
+#define INSTR_CREATE_pmovsxbd(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_pmovsxbd, (d), (s))
+#define INSTR_CREATE_pmovsxbq(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_pmovsxbq, (d), (s))
+#define INSTR_CREATE_pmovsxdw(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_pmovsxdw, (d), (s))
+#define INSTR_CREATE_pmovsxwq(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_pmovsxwq, (d), (s))
+#define INSTR_CREATE_pmovsxdq(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_pmovsxdq, (d), (s))
+#define INSTR_CREATE_movntdqa(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movntdqa, (d), (s))
+#define INSTR_CREATE_pmovzxbw(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_pmovzxbw, (d), (s))
+#define INSTR_CREATE_pmovzxbd(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_pmovzxbd, (d), (s))
+#define INSTR_CREATE_pmovzxbq(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_pmovzxbq, (d), (s))
+#define INSTR_CREATE_pmovzxdw(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_pmovzxdw, (d), (s))
+#define INSTR_CREATE_pmovzxwq(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_pmovzxwq, (d), (s))
+#define INSTR_CREATE_pmovzxdq(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_pmovzxdq, (d), (s))
+#define INSTR_CREATE_phminposuw(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_phminposuw, (d), (s))
+#define INSTR_CREATE_vmread(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_vmread, (d), (s))
+#define INSTR_CREATE_vmwrite(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_vmwrite, (d), (s))
+#define INSTR_CREATE_movsxd(dc, d, s) \
+  instr_create_1dst_1src((dc), OP_movsxd, (d), (s))
+/* @} */ /* end doxygen group */
+
+/* 1 destination, 1 implicit source */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ */
+#define INSTR_CREATE_inc(dc, d) \
+  instr_create_1dst_1src((dc), OP_inc, (d), (d))
+#define INSTR_CREATE_dec(dc, d) \
+  instr_create_1dst_1src((dc), OP_dec, (d), (d))
+/* FIXME: check that d is a 32-bit reg? */
+#define INSTR_CREATE_bswap(dc, d) \
+  instr_create_1dst_1src((dc), OP_bswap, (d), (d))
+#define INSTR_CREATE_not(dc, d) \
+  instr_create_1dst_1src((dc), OP_not, (d), (d))
+#define INSTR_CREATE_neg(dc, d) \
+  instr_create_1dst_1src((dc), OP_neg, (d), (d))
+/* @} */ /* end doxygen group */
+
+/* 1 implicit destination, 1 implicit source */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx, automatically
+ * supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ */
+#define INSTR_CREATE_cdq(dc) \
+  instr_create_1dst_1src((dc), OP_cdq, opnd_create_reg(REG_EDX), opnd_create_reg(REG_EAX))
+#define INSTR_CREATE_daa(dc) \
+  instr_create_1dst_1src((dc), OP_daa, opnd_create_reg(REG_AL), opnd_create_reg(REG_AL))
+#define INSTR_CREATE_das(dc) \
+  instr_create_1dst_1src((dc), OP_das, opnd_create_reg(REG_AL), opnd_create_reg(REG_AL))
+#define INSTR_CREATE_aaa(dc) \
+  instr_create_1dst_1src((dc), OP_aaa, opnd_create_reg(REG_AX), opnd_create_reg(REG_AX))
+#define INSTR_CREATE_aas(dc) \
+  instr_create_1dst_1src((dc), OP_aas, opnd_create_reg(REG_AX), opnd_create_reg(REG_AX))
+#define INSTR_CREATE_cwde(dc) \
+  instr_create_1dst_1src((dc), OP_cwde, opnd_create_reg(REG_EAX), opnd_create_reg(REG_AX))
+#define INSTR_CREATE_xlat(dc) \
+  instr_create_1dst_1src((dc), OP_xlat, opnd_create_reg(REG_AL), \
+    opnd_create_far_base_disp(SEG_DS, REG_XBX, REG_AL, 1, 0, OPSZ_xlat))
+/* @} */ /* end doxygen group */
+
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * Creates an instr_t for an OP_in instruction with a source of al
+ * (INSTR_CREATE_in_1()) or eax (INSTR_CREATE_in_4()) and dx.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ */
+#define INSTR_CREATE_in_1(dc) \
+  instr_create_1dst_1src((dc), OP_in, opnd_create_reg(REG_AL), opnd_create_reg(REG_DX))
+#define INSTR_CREATE_in_4(dc) \
+  instr_create_1dst_1src((dc), OP_in, opnd_create_reg(REG_EAX), opnd_create_reg(REG_DX))
+/* @} */ /* end doxygen group */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * Creates an instr_t for an OP_in instruction with a source of al
+ * (INSTR_CREATE_in_1_imm()) or eax (INSTR_CREATE_in_4_imm()) and an immediate.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param i The opnd_t explicit source operand for the instruction, which must be an
+ * immediate integer (opnd_create_immed_int()).
+ */
+#define INSTR_CREATE_in_1_imm(dc, i) \
+  instr_create_1dst_1src((dc), OP_in, opnd_create_reg(REG_AL), (i))
+#define INSTR_CREATE_in_4_imm(dc, i) \
+  instr_create_1dst_1src((dc), OP_in, opnd_create_reg(REG_EAX), (i))
+/* @} */ /* end doxygen group */
+
+/* floating-point */
+/**
+ * Creats an instr_t for a conditional move instruction with the given opcode
+ * and destination operand.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param op The OP_xxx opcode for the instruction, which should be in the range
+ * [OP_fcmovb, OP_fcmovnu], excluding OP_fucompp.
+ * \param f The opnd_t explicit source operand for the instruction, which must
+ * be a floating point register (opnd_create_reg()).
+ */
+#define INSTR_CREATE_fcmovcc(dc, op, f) \
+  instr_create_1dst_1src((dc), (op), opnd_create_reg(REG_ST0), (f))
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the
+ * given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction, which must
+ * be one of the following:
+ * -# A floating point register (opnd_create_reg()).
+ * -# A memory reference (opnd_create_base_disp() or opnd_create_far_base_disp()).
+ */
+#define INSTR_CREATE_fst(dc, d) \
+  instr_create_1dst_1src((dc), OP_fst, (d), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_fstp(dc, d) \
+  instr_create_1dst_1src((dc), OP_fstp, (d), opnd_create_reg(REG_ST0))
+/* @} */ /* end doxygen group */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the
+ * given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param s The opnd_t explicit source operand for the instruction, which must
+ * be one of the following:
+ * -# A floating point register (opnd_create_reg()).
+ * -# A memory reference (opnd_create_base_disp() or opnd_create_far_base_disp()).
+ */
+#define INSTR_CREATE_fld(dc, s) \
+  instr_create_1dst_1src((dc), OP_fld, opnd_create_reg(REG_ST0), (s))
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx_mem macro creates an instr_t with opcode OP_xxx and
+ * the given explicit memory operand, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param m The opnd_t explicit destination operand for the instruction, which must be
+ * a memory reference (opnd_create_base_disp() or opnd_create_far_base_disp()).
+ */
+#define INSTR_CREATE_fist(dc, m) \
+  instr_create_1dst_1src((dc), OP_fist, (m), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_fistp(dc, m) \
+  instr_create_1dst_1src((dc), OP_fistp, (m), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_fisttp(dc, m) \
+  instr_create_1dst_1src((dc), OP_fisttp, (m), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_fbstp(dc, m) \
+  instr_create_1dst_1src((dc), OP_fbstp, (m), opnd_create_reg(REG_ST0))
+/* @} */ /* end doxygen group */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx_mem macro creates an instr_t with opcode OP_xxx and
+ * the given explicit memory operand, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param m The opnd_t explicit source operand for the instruction, which must be
+ * a memory reference (opnd_create_base_disp() or opnd_create_far_base_disp()).
+ */
+#define INSTR_CREATE_fild(dc, m) \
+  instr_create_1dst_1src((dc), OP_fild, opnd_create_reg(REG_ST0), (m))
+#define INSTR_CREATE_fbld(dc, m) \
+  instr_create_1dst_1src((dc), OP_fbld, opnd_create_reg(REG_ST0), (m))
+/* @} */ /* end doxygen group */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx, automatically
+ * supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ */
+#define INSTR_CREATE_fchs(dc) \
+  instr_create_1dst_1src((dc), OP_fchs, opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_fabs(dc) \
+  instr_create_1dst_1src((dc), OP_fabs, opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_ftst(dc) \
+  instr_create_1dst_1src((dc), OP_ftst, opnd_create_reg(REG_ST0), opnd_create_immed_float(0.0f))
+#define INSTR_CREATE_fld1(dc) \
+  instr_create_1dst_1src((dc), OP_fld1, opnd_create_reg(REG_ST0), opnd_create_immed_float(1.0f))
+/* FIXME: do we really want these constants here? Should they be floats or doubles? */
+#define INSTR_CREATE_fldl2t(dc) \
+  instr_create_1dst_1src((dc), OP_fldl2t, opnd_create_reg(REG_ST0), \
+  opnd_create_immed_float(M_LN10/M_LN2))
+#define INSTR_CREATE_fldl2e(dc) \
+  instr_create_1dst_1src((dc), OP_fldl2e, opnd_create_reg(REG_ST0), \
+  opnd_create_immed_float(1.0f/M_LN2))
+#define INSTR_CREATE_fldpi(dc) \
+  instr_create_1dst_1src((dc), OP_fldpi, opnd_create_reg(REG_ST0), \
+  opnd_create_immed_float(M_PI))
+#define INSTR_CREATE_fldlg2(dc) \
+  instr_create_1dst_1src((dc), OP_fldlg2, opnd_create_reg(REG_ST0), \
+  opnd_create_immed_float(M_LN2/M_LN10))
+#define INSTR_CREATE_fldln2(dc) \
+  instr_create_1dst_1src((dc), OP_fldln2, opnd_create_reg(REG_ST0), \
+  opnd_create_immed_float(M_LN2))
+#define INSTR_CREATE_fldz(dc) \
+  instr_create_1dst_1src((dc), OP_fldz, opnd_create_reg(REG_ST0), opnd_create_immed_float(0.0f))
+#define INSTR_CREATE_f2xm1(dc) \
+  instr_create_1dst_1src((dc), OP_f2xm1, opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_fptan(dc) \
+  instr_create_1dst_1src((dc), OP_fptan, opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_fxtract(dc) \
+  instr_create_1dst_1src((dc), OP_fxtract, opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_fsqrt(dc) \
+  instr_create_1dst_1src((dc), OP_fsqrt, opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_fsincos(dc) \
+  instr_create_1dst_1src((dc), OP_fsincos, opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_frndint(dc) \
+  instr_create_1dst_1src((dc), OP_frndint, opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_fsin(dc) \
+  instr_create_1dst_1src((dc), OP_fsin, opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_fcos(dc) \
+  instr_create_1dst_1src((dc), OP_fcos, opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST0))
+
+#define INSTR_CREATE_fscale(dc) \
+  instr_create_1dst_2src((dc), OP_fscale, opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST1), \
+    opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_fyl2x(dc) \
+  instr_create_2dst_2src((dc), OP_fyl2x, opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST1), \
+    opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST1))
+#define INSTR_CREATE_fyl2xp1(dc) \
+  instr_create_2dst_2src((dc), OP_fyl2xp1, opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST1), \
+    opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST1))
+#define INSTR_CREATE_fpatan(dc) \
+  instr_create_2dst_2src((dc), OP_fpatan, opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST1), \
+    opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST1))
+#define INSTR_CREATE_fprem(dc) \
+  instr_create_2dst_2src((dc), OP_fprem, opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST1), \
+    opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST1))
+#define INSTR_CREATE_fprem1(dc) \
+  instr_create_2dst_2src((dc), OP_fprem1, opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST1), \
+    opnd_create_reg(REG_ST0), opnd_create_reg(REG_ST1))
+/* @} */ /* end doxygen group */
+
+/* 1 destination, 2 sources */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ * \param s The opnd_t explicit source operand for the instruction.
+ * \param i The opnd_t explicit second source operand for the instruction, which
+ * must be an immediate integer (opnd_create_immed_int()).
+ */
+#define INSTR_CREATE_pshufw(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_pshufw, (d), (s), (i))
+#define INSTR_CREATE_pshufd(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_pshufd, (d), (s), (i))
+#define INSTR_CREATE_pshufhw(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_pshufhw, (d), (s), (i))
+#define INSTR_CREATE_pshuflw(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_pshuflw, (d), (s), (i))
+#define INSTR_CREATE_pinsrw(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_pinsrw, (d), (s), (i))
+#define INSTR_CREATE_pextrw(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_pextrw, (d), (s), (i))
+/* SSE4 */
+#define INSTR_CREATE_pextrb(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_pextrb, (d), (s), (i))
+#define INSTR_CREATE_pextrd(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_pextrd, (d), (s), (i))
+#define INSTR_CREATE_extractps(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_extractps, (d), (s), (i))
+#define INSTR_CREATE_roundps(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_roundps, (d), (s), (i))
+#define INSTR_CREATE_roundpd(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_roundpd, (d), (s), (i))
+#define INSTR_CREATE_roundss(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_roundss, (d), (s), (i))
+#define INSTR_CREATE_roundsd(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_roundsd, (d), (s), (i))
+#define INSTR_CREATE_blendps(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_blendps, (d), (s), (i))
+#define INSTR_CREATE_blendpd(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_blendpd, (d), (s), (i))
+#define INSTR_CREATE_pblendw(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_pblendw, (d), (s), (i))
+#define INSTR_CREATE_pinsrb(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_pinsrb, (d), (s), (i))
+#define INSTR_CREATE_insertps(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_insertps, (d), (s), (i))
+#define INSTR_CREATE_pinsrd(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_pinsrd, (d), (s), (i))
+/* @} */ /* end doxygen group */
+
+/* 1 destination, 2 sources: 1 explicit, 1 implicit */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ * \param s The opnd_t explicit source operand for the instruction.
+ */
+#define INSTR_CREATE_add(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_add, (d), (s), (d))
+#define INSTR_CREATE_or(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_or,  (d), (s), (d))
+#define INSTR_CREATE_adc(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_adc, (d), (s), (d))
+#define INSTR_CREATE_sbb(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_sbb, (d), (s), (d))
+#define INSTR_CREATE_and(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_and, (d), (s), (d))
+#define INSTR_CREATE_sub(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_sub, (d), (s), (d))
+#define INSTR_CREATE_xor(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_xor, (d), (s), (d))
+#define INSTR_CREATE_punpcklbw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_punpcklbw, (d), (s), (d))
+#define INSTR_CREATE_punpcklwd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_punpcklwd, (d), (s), (d))
+#define INSTR_CREATE_punpckldq(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_punpckldq, (d), (s), (d))
+#define INSTR_CREATE_packsswb(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_packsswb, (d), (s), (d))
+#define INSTR_CREATE_pcmpgtb(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pcmpgtb, (d), (s), (d))
+#define INSTR_CREATE_pcmpgtw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pcmpgtw, (d), (s), (d))
+#define INSTR_CREATE_pcmpgtd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pcmpgtd, (d), (s), (d))
+#define INSTR_CREATE_packuswb(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_packuswb, (d), (s), (d))
+#define INSTR_CREATE_punpckhbw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_punpckhbw, (d), (s), (d))
+#define INSTR_CREATE_punpckhwd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_punpckhwd, (d), (s), (d))
+#define INSTR_CREATE_punpckhdq(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_punpckhdq, (d), (s), (d))
+#define INSTR_CREATE_packssdw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_packssdw, (d), (s), (d))
+#define INSTR_CREATE_punpcklqdq(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_punpcklqdq, (d), (s), (d))
+#define INSTR_CREATE_punpckhqdq(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_punpckhqdq, (d), (s), (d))
+#define INSTR_CREATE_pcmpeqb(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pcmpeqb, (d), (s), (d))
+#define INSTR_CREATE_pcmpeqw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pcmpeqw, (d), (s), (d))
+#define INSTR_CREATE_pcmpeqd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pcmpeqd, (d), (s), (d))
+#define INSTR_CREATE_psrlw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psrlw, (d), (s), (d))
+#define INSTR_CREATE_psrld(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psrld, (d), (s), (d))
+#define INSTR_CREATE_psrlq(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psrlq, (d), (s), (d))
+#define INSTR_CREATE_paddq(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_paddq, (d), (s), (d))
+#define INSTR_CREATE_pmullw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pmullw, (d), (s), (d))
+#define INSTR_CREATE_psubusb(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psubusb, (d), (s), (d))
+#define INSTR_CREATE_psubusw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psubusw, (d), (s), (d))
+#define INSTR_CREATE_pminub(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pminub, (d), (s), (d))
+#define INSTR_CREATE_pand(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pand, (d), (s), (d))
+#define INSTR_CREATE_paddusb(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_paddusb, (d), (s), (d))
+#define INSTR_CREATE_paddusw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_paddusw, (d), (s), (d))
+#define INSTR_CREATE_pmaxub(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pmaxub, (d), (s), (d))
+#define INSTR_CREATE_pandn(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pandn, (d), (s), (d))
+#define INSTR_CREATE_pavgb(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pavgb, (d), (s), (d))
+#define INSTR_CREATE_psraw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psraw, (d), (s), (d))
+#define INSTR_CREATE_psrad(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psrad, (d), (s), (d))
+#define INSTR_CREATE_pavgw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pavgw, (d), (s), (d))
+#define INSTR_CREATE_pmulhuw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pmulhuw, (d), (s), (d))
+#define INSTR_CREATE_pmulhw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pmulhw, (d), (s), (d))
+#define INSTR_CREATE_psubsb(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psubsb, (d), (s), (d))
+#define INSTR_CREATE_psubsw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psubsw, (d), (s), (d))
+#define INSTR_CREATE_pminsw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pminsw, (d), (s), (d))
+#define INSTR_CREATE_por(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_por, (d), (s), (d))
+#define INSTR_CREATE_paddsb(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_paddsb, (d), (s), (d))
+#define INSTR_CREATE_paddsw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_paddsw, (d), (s), (d))
+#define INSTR_CREATE_pmaxsw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pmaxsw, (d), (s), (d))
+#define INSTR_CREATE_pxor(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pxor, (d), (s), (d))
+#define INSTR_CREATE_psllw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psllw, (d), (s), (d))
+#define INSTR_CREATE_pslld(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pslld, (d), (s), (d))
+#define INSTR_CREATE_psllq(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psllq, (d), (s), (d))
+#define INSTR_CREATE_pmuludq(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pmuludq, (d), (s), (d))
+#define INSTR_CREATE_pmaddwd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pmaddwd, (d), (s), (d))
+#define INSTR_CREATE_psadbw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psadbw, (d), (s), (d))
+#define INSTR_CREATE_psubb(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psubb, (d), (s), (d))
+#define INSTR_CREATE_psubw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psubw, (d), (s), (d))
+#define INSTR_CREATE_psubd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psubd, (d), (s), (d))
+#define INSTR_CREATE_psubq(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psubq, (d), (s), (d))
+#define INSTR_CREATE_paddb(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_paddb, (d), (s), (d))
+#define INSTR_CREATE_paddw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_paddw, (d), (s), (d))
+#define INSTR_CREATE_paddd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_paddd, (d), (s), (d))
+#define INSTR_CREATE_psrldq(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psrldq, (d), (s), (d))
+#define INSTR_CREATE_pslldq(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pslldq, (d), (s), (d))
+#define INSTR_CREATE_unpcklps(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_unpcklps, (d), (s), (d))
+#define INSTR_CREATE_unpcklpd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_unpcklpd, (d), (s), (d))
+#define INSTR_CREATE_unpckhps(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_unpckhps, (d), (s), (d))
+#define INSTR_CREATE_unpckhpd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_unpckhpd, (d), (s), (d))
+#define INSTR_CREATE_andps(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_andps, (d), (s), (d))
+#define INSTR_CREATE_andpd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_andpd, (d), (s), (d))
+#define INSTR_CREATE_andnps(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_andnps, (d), (s), (d))
+#define INSTR_CREATE_andnpd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_andnpd, (d), (s), (d))
+#define INSTR_CREATE_orps(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_orps, (d), (s), (d))
+#define INSTR_CREATE_orpd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_orpd, (d), (s), (d))
+#define INSTR_CREATE_xorps(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_xorps, (d), (s), (d))
+#define INSTR_CREATE_xorpd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_xorpd, (d), (s), (d))
+#define INSTR_CREATE_addps(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_addps, (d), (s), (d))
+#define INSTR_CREATE_addss(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_addss, (d), (s), (d))
+#define INSTR_CREATE_addpd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_addpd, (d), (s), (d))
+#define INSTR_CREATE_addsd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_addsd, (d), (s), (d))
+#define INSTR_CREATE_mulps(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_mulps, (d), (s), (d))
+#define INSTR_CREATE_mulss(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_mulss, (d), (s), (d))
+#define INSTR_CREATE_mulpd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_mulpd, (d), (s), (d))
+#define INSTR_CREATE_mulsd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_mulsd, (d), (s), (d))
+#define INSTR_CREATE_subps(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_subps, (d), (s), (d))
+#define INSTR_CREATE_subss(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_subss, (d), (s), (d))
+#define INSTR_CREATE_subpd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_subpd, (d), (s), (d))
+#define INSTR_CREATE_subsd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_subsd, (d), (s), (d))
+#define INSTR_CREATE_minps(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_minps, (d), (s), (d))
+#define INSTR_CREATE_minss(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_minss, (d), (s), (d))
+#define INSTR_CREATE_minpd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_minpd, (d), (s), (d))
+#define INSTR_CREATE_minsd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_minsd, (d), (s), (d))
+#define INSTR_CREATE_divps(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_divps, (d), (s), (d))
+#define INSTR_CREATE_divss(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_divss, (d), (s), (d))
+#define INSTR_CREATE_divpd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_divpd, (d), (s), (d))
+#define INSTR_CREATE_divsd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_divsd, (d), (s), (d))
+#define INSTR_CREATE_maxps(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_maxps, (d), (s), (d))
+#define INSTR_CREATE_maxss(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_maxss, (d), (s), (d))
+#define INSTR_CREATE_maxpd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_maxpd, (d), (s), (d))
+#define INSTR_CREATE_maxsd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_maxsd, (d), (s), (d))
+/* SSE3 */
+#define INSTR_CREATE_haddpd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_haddpd, (d), (s), (d))
+#define INSTR_CREATE_haddps(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_haddps, (d), (s), (d))
+#define INSTR_CREATE_hsubpd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_hsubpd, (d), (s), (d))
+#define INSTR_CREATE_hsubps(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_hsubps, (d), (s), (d))
+#define INSTR_CREATE_addsubpd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_addsubpd, (d), (s), (d))
+#define INSTR_CREATE_addsubps(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_addsubps, (d), (s), (d))
+/* 3D-Now */
+#define INSTR_CREATE_pavgusb(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pavgusb, (d), (s), (d))
+#define INSTR_CREATE_pfadd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pfadd, (d), (s), (d))
+#define INSTR_CREATE_pfacc(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pfacc, (d), (s), (d))
+#define INSTR_CREATE_pfcmpge(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pfcmpge, (d), (s), (d))
+#define INSTR_CREATE_pfcmpgt(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pfcmpgt, (d), (s), (d))
+#define INSTR_CREATE_pfcmpeq(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pfcmpeq, (d), (s), (d))
+#define INSTR_CREATE_pfmin(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pfmin, (d), (s), (d))
+#define INSTR_CREATE_pfmax(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pfmax, (d), (s), (d))
+#define INSTR_CREATE_pfmul(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pfmul, (d), (s), (d))
+#define INSTR_CREATE_pfrcp(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pfrcp, (d), (s), (d))
+#define INSTR_CREATE_pfrcpit1(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pfrcpit1, (d), (s), (d))
+#define INSTR_CREATE_pfrcpit2(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pfrcpit2, (d), (s), (d))
+#define INSTR_CREATE_pfrsqrt(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pfrsqrt, (d), (s), (d))
+#define INSTR_CREATE_pfrsqit1(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pfrsqit1, (d), (s), (d))
+#define INSTR_CREATE_pmulhrw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pmulhrw, (d), (s), (d))
+#define INSTR_CREATE_pfsub(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pfsub, (d), (s), (d))
+#define INSTR_CREATE_pfsubr(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pfsubr, (d), (s), (d))
+#define INSTR_CREATE_pi2fd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pi2fd, (d), (s), (d))
+#define INSTR_CREATE_pf2id(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pf2id, (d), (s), (d))
+#define INSTR_CREATE_pi2fw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pi2fw, (d), (s), (d))
+#define INSTR_CREATE_pf2iw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pf2iw, (d), (s), (d))
+#define INSTR_CREATE_pfnacc(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pfnacc, (d), (s), (d))
+#define INSTR_CREATE_pfpnacc(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pfpnacc, (d), (s), (d))
+#define INSTR_CREATE_pswapd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pswapd, (d), (s), (d))
+/* SSSE3 */
+#define INSTR_CREATE_phaddw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_phaddw, (d), (s), (d))
+#define INSTR_CREATE_phaddd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_phaddd, (d), (s), (d))
+#define INSTR_CREATE_phaddsw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_phaddsw, (d), (s), (d))
+#define INSTR_CREATE_pmaddubsw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pmaddubsw, (d), (s), (d))
+#define INSTR_CREATE_phsubw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_phsubw, (d), (s), (d))
+#define INSTR_CREATE_phsubd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_phsubd, (d), (s), (d))
+#define INSTR_CREATE_phsubsw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_phsubsw, (d), (s), (d))
+#define INSTR_CREATE_psignb(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psignb, (d), (s), (d))
+#define INSTR_CREATE_psignw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psignw, (d), (s), (d))
+#define INSTR_CREATE_psignd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_psignd, (d), (s), (d))
+#define INSTR_CREATE_pmulhrsw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pmulhrsw, (d), (s), (d))
+#define INSTR_CREATE_pabsb(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pabsb, (d), (s), (d))
+#define INSTR_CREATE_pabsw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pabsw, (d), (s), (d))
+#define INSTR_CREATE_pabsd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pabsd, (d), (s), (d))
+/* SSE4 */
+#define INSTR_CREATE_pblendvb(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pblendvb, (d), (s), opnd_create_reg(REG_XMM0))
+#define INSTR_CREATE_blendvps(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_blendvps, (d), (s), opnd_create_reg(REG_XMM0))
+#define INSTR_CREATE_blendvpd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_blendvpd, (d), (s), opnd_create_reg(REG_XMM0))
+#define INSTR_CREATE_crc32(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_crc32, (d), (s), (d))
+#define INSTR_CREATE_packusdw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_packusdw, (d), (s), (d))
+#define INSTR_CREATE_pcmpeqq(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pcmpeqq, (d), (s), (d))
+#define INSTR_CREATE_pcmpgtq(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pcmpgtq, (d), (s), (d))
+#define INSTR_CREATE_pminsb(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pminsb, (d), (s), (d))
+#define INSTR_CREATE_pminsd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pminsd, (d), (s), (d))
+#define INSTR_CREATE_pminuw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pminuw, (d), (s), (d))
+#define INSTR_CREATE_pminud(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pminud, (d), (s), (d))
+#define INSTR_CREATE_pmaxsb(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pmaxsb, (d), (s), (d))
+#define INSTR_CREATE_pmaxsd(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pmaxsd, (d), (s), (d))
+#define INSTR_CREATE_pmaxuw(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pmaxuw, (d), (s), (d))
+#define INSTR_CREATE_pmaxud(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pmaxud, (d), (s), (d))
+#define INSTR_CREATE_pmuldq(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pmuldq, (d), (s), (d))
+#define INSTR_CREATE_pmulld(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_pmulld, (d), (s), (d))
+/* @} */ /* end doxygen group */
+
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ * \param ri The opnd_t explicit source operand for the instruction, which can
+ * be a register (opnd_create_reg()) or an immediate integer (opnd_create_immed_int()).
+ */
+#define INSTR_CREATE_bts(dc, d, ri) \
+  instr_create_1dst_2src((dc), OP_bts, (d), (ri), (d))
+#define INSTR_CREATE_btr(dc, d, ri) \
+  instr_create_1dst_2src((dc), OP_btr, (d), (ri), (d))
+#define INSTR_CREATE_btc(dc, d, ri) \
+  instr_create_1dst_2src((dc), OP_btc, (d), (ri), (d))
+/* @} */ /* end doxygen group */
+
+/**
+ * Creats an instr_t for a conditional move instruction with the given opcode
+ * and destination operand.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param op The OP_xxx opcode for the instruction, which should be in the range
+ * [OP_cmovo, OP_cmovnle].
+ * \param d The opnd_t explicit destination operand for the instruction.
+ * \param s The opnd_t explicit source operand for the instruction.
+ */
+#define INSTR_CREATE_cmovcc(dc, op, d, s) \
+  instr_create_1dst_2src((dc), (op), (d), (s), (d))
+
+/**
+ * This INSTR_CREATE_xxx_imm macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands. The _imm
+ * suffix distinguishes between alternative forms of the same opcode: this
+ * form takes an explicit immediate.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ * \param s The opnd_t explicit source operand for the instruction.
+ * \param i The opnd_t explicit second source operand for the instruction, which
+ * must be an immediate integer (opnd_create_immed_int()).
+ */
+#define INSTR_CREATE_imul_imm(dc, d, s, i) \
+  instr_create_1dst_2src((dc), OP_imul, (d), (s), (i))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ * \param s The opnd_t explicit source operand for the instruction.
+ */
+#define INSTR_CREATE_imul(dc, d, s) \
+  instr_create_1dst_2src((dc), OP_imul, (d), (s), (d))
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx, INSTR_CREATE_xxx_1, or INSTR_CREATE_xxx_4 macro creates an
+ * instr_t with opcode OP_xxx and the given explicit operands, automatically
+ * supplying any implicit operands.    The _1 or _4 suffixes distinguish between
+ * alternative forms of the same opcode (1 and 4 identify the operand size).
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param s The opnd_t explicit source operand for the instruction.
+ */
+#define INSTR_CREATE_imul_1(dc, s) \
+  instr_create_1dst_2src((dc), OP_imul, opnd_create_reg(REG_AX), (s), opnd_create_reg(REG_AL))
+#define INSTR_CREATE_imul_4(dc, s) \
+  instr_create_2dst_2src((dc), OP_imul, opnd_create_reg(REG_EDX), opnd_create_reg(REG_EAX), \
+    (s), opnd_create_reg(REG_EAX))
+#define INSTR_CREATE_mul_1(dc, s) \
+  instr_create_1dst_2src((dc), OP_mul, opnd_create_reg(REG_AX), (s), opnd_create_reg(REG_AL))
+#define INSTR_CREATE_mul_4(dc, s) \
+  instr_create_2dst_2src((dc), OP_mul, opnd_create_reg(REG_EDX), opnd_create_reg(REG_EAX), \
+    (s), opnd_create_reg(REG_EAX))
+#define INSTR_CREATE_div_1(dc, s) \
+  instr_create_2dst_2src((dc), OP_div, opnd_create_reg(REG_AH), opnd_create_reg(REG_AL), (s), opnd_create_reg(REG_AX))
+#define INSTR_CREATE_div_4(dc, s) \
+  instr_create_2dst_3src((dc), OP_div, opnd_create_reg(REG_EDX), opnd_create_reg(REG_EAX), \
+    (s), opnd_create_reg(REG_EDX), opnd_create_reg(REG_EAX))
+#define INSTR_CREATE_idiv_1(dc, s) \
+  instr_create_2dst_2src((dc), OP_idiv, opnd_create_reg(REG_AH), opnd_create_reg(REG_AL), (s), opnd_create_reg(REG_AX))
+#define INSTR_CREATE_idiv_4(dc, s) \
+  instr_create_2dst_3src((dc), OP_idiv, opnd_create_reg(REG_EDX), opnd_create_reg(REG_EAX), \
+    (s), opnd_create_reg(REG_EDX), opnd_create_reg(REG_EAX))
+/* @} */ /* end doxygen group */
+
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the
+ * given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ * \param ri The opnd_t explicit source operand for the instruction, which must
+ * be one of the following:
+ * -# The register cl (#opnd_create_reg(#REG_CL));
+ * -# An immediate integer (opnd_create_immed_int()) of size #OPSZ_1;
+ * -# An immediate integer with value 1 and size #OPSZ_0
+ * (#opnd_create_immed_int(1, #OPSZ_0)), which will become an implicit operand
+ * (whereas #opnd_create_immed_int(1, #OPSZ_1) will be encoded explicitly).
+ */
+#define INSTR_CREATE_rol(dc, d, ri) \
+  instr_create_1dst_2src((dc), OP_rol, (d), (ri), (d))
+#define INSTR_CREATE_ror(dc, d, ri) \
+  instr_create_1dst_2src((dc), OP_ror, (d), (ri), (d))
+#define INSTR_CREATE_rcl(dc, d, ri) \
+  instr_create_1dst_2src((dc), OP_rcl, (d), (ri), (d))
+#define INSTR_CREATE_rcr(dc, d, ri) \
+  instr_create_1dst_2src((dc), OP_rcr, (d), (ri), (d))
+#define INSTR_CREATE_shl(dc, d, ri) \
+  instr_create_1dst_2src((dc), OP_shl, (d), (ri), (d))
+#define INSTR_CREATE_shr(dc, d, ri) \
+  instr_create_1dst_2src((dc), OP_shr, (d), (ri), (d))
+#define INSTR_CREATE_sar(dc, d, ri) \
+  instr_create_1dst_2src((dc), OP_sar, (d), (ri), (d))
+/* @} */ /* end doxygen group */
+
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param s1 The opnd_t first source operand for the instruction.
+ * \param s2 The opnd_t second source operand for the instruction.
+ */
+#define INSTR_CREATE_maskmovq(dc, s1, s2) \
+  instr_create_1dst_2src((dc), OP_maskmovq, \
+    opnd_create_far_base_disp(SEG_DS, REG_XDI, REG_NULL, 0, 0, OPSZ_maskmovq), \
+   (s1), (s2))
+#define INSTR_CREATE_maskmovdqu(dc, s1, s2) \
+  instr_create_1dst_2src((dc), OP_maskmovdqu, \
+    opnd_create_far_base_disp(SEG_DS, REG_XDI, REG_NULL, 0, 0, OPSZ_maskmovdqu), \
+    (s1), (s2))
+/* @} */ /* end doxygen group */
+
+/* floating-point */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given operands, automatically supplying any further implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param f The opnd_t destination (and implicit source) operand for the
+ * instruction, which must be a floating point register (opnd_create_reg()).
+ * \param s The opnd_t source (and non-destination) operand for the
+ * instruction, which must be one of the following:
+ * -# A floating point register (opnd_create_reg()).
+ * -# A memory reference (opnd_create_base_disp() or opnd_create_far_base_disp()),
+ *    in which case the destination \p f must be #REG_ST0.
+ */
+#define INSTR_CREATE_fadd(dc, f, s) \
+  instr_create_1dst_2src((dc), OP_fadd, (f), (s), (f))
+#define INSTR_CREATE_fmul(dc, f, s) \
+  instr_create_1dst_2src((dc), OP_fmul, (f), (s), (f))
+#define INSTR_CREATE_fdiv(dc, f, s) \
+  instr_create_1dst_2src((dc), OP_fdiv, (f), (s), (f))
+#define INSTR_CREATE_fdivr(dc, f, s) \
+  instr_create_1dst_2src((dc), OP_fdivr, (f), (s), (f))
+#define INSTR_CREATE_fsub(dc, f, s) \
+  instr_create_1dst_2src((dc), OP_fsub, (f), (s), (f))
+#define INSTR_CREATE_fsubr(dc, f, s) \
+  instr_create_1dst_2src((dc), OP_fsubr, (f), (s), (f))
+/* @} */ /* end doxygen group */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with
+ * opcode OP_xxx and the given explicit register operand, automatically
+ * supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param f The opnd_t explicit destination + source operand for the instruction, which 
+ * must be a floating point register (opnd_create_reg()).
+ */
+#define INSTR_CREATE_faddp(dc, f) \
+  instr_create_1dst_2src((dc), OP_faddp, (f), opnd_create_reg(REG_ST0), (f))
+#define INSTR_CREATE_fmulp(dc, f) \
+  instr_create_1dst_2src((dc), OP_fmulp, (f), opnd_create_reg(REG_ST0), (f))
+#define INSTR_CREATE_fdivp(dc, f) \
+  instr_create_1dst_2src((dc), OP_fdivp, (f), opnd_create_reg(REG_ST0), (f))
+#define INSTR_CREATE_fdivrp(dc, f) \
+  instr_create_1dst_2src((dc), OP_fdivrp, (f), opnd_create_reg(REG_ST0), (f))
+#define INSTR_CREATE_fsubp(dc, f) \
+  instr_create_1dst_2src((dc), OP_fsubp, (f), opnd_create_reg(REG_ST0), (f))
+#define INSTR_CREATE_fsubrp(dc, f) \
+  instr_create_1dst_2src((dc), OP_fsubrp, (f), opnd_create_reg(REG_ST0), (f))
+/* @} */ /* end doxygen group */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit memory operand, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param m The opnd_t explicit source operand for the instruction, which must be
+ * a memory reference (opnd_create_base_disp() or opnd_create_far_base_disp()).
+ */
+#define INSTR_CREATE_fiadd(dc, m) \
+  instr_create_1dst_2src((dc), OP_fiadd, opnd_create_reg(REG_ST0), (m), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_fimul(dc, m) \
+  instr_create_1dst_2src((dc), OP_fimul, opnd_create_reg(REG_ST0), (m), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_fidiv(dc, m) \
+  instr_create_1dst_2src((dc), OP_fidiv, opnd_create_reg(REG_ST0), (m), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_fidivr(dc, m) \
+  instr_create_1dst_2src((dc), OP_fidivr, opnd_create_reg(REG_ST0), (m), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_fisub(dc, m) \
+  instr_create_1dst_2src((dc), OP_fisub, opnd_create_reg(REG_ST0), (m), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_fisubr(dc, m) \
+  instr_create_1dst_2src((dc), OP_fisubr, opnd_create_reg(REG_ST0), (m), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_ficom(dc, m) \
+  instr_create_1dst_2src((dc), OP_ficom, opnd_create_reg(REG_ST0), (m), opnd_create_reg(REG_ST0))
+#define INSTR_CREATE_ficomp(dc, m) \
+  instr_create_1dst_2src((dc), OP_ficomp, opnd_create_reg(REG_ST0), (m), opnd_create_reg(REG_ST0))
+/* @} */ /* end doxygen group */
+
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the
+ * given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ * \param r The opnd_t explicit source operand for the instruction, which
+ * must be an xmm register (opnd_create_reg()).
+ */
+#define INSTR_CREATE_extrq(dc, d, r) \
+  instr_create_1dst_1src((dc), OP_extrq, (d), (r))
+/**
+ * This INSTR_CREATE_xxx_imm macro creates an instr_t with opcode OP_xxx and the
+ * given explicit operands, automatically supplying any implicit operands. The _imm
+ * suffix distinguishes between alternative forms of the same opcode: this
+ * form takes explicit immediates.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ * \param i1 The opnd_t explicit first source operand for the instruction, which
+ * must be an immediate integer (opnd_create_immed_int()).
+ * \param i2 The opnd_t explicit second source operand for the instruction, which
+ * must be an immediate integer (opnd_create_immed_int()).
+ */
+#define INSTR_CREATE_extrq_imm(dc, d, i1, i2) \
+  instr_create_1dst_2src((dc), OP_extrq, (d), (i1), (i2))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the
+ * given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ * \param r The opnd_t explicit source operand for the instruction, which
+ * must be an xmm register (opnd_create_reg()).
+ */
+#define INSTR_CREATE_insertq(dc, d, r) \
+  instr_create_1dst_1src((dc), OP_insertq, (d), (r))
+/**
+ * This INSTR_CREATE_xxx_imm macro creates an instr_t with opcode OP_xxx and the
+ * given explicit operands, automatically supplying any implicit operands. The _imm
+ * suffix distinguishes between alternative forms of the same opcode: this
+ * form takes explicit immediates.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ * \param r The opnd_t explicit first source operand for the instruction, which
+ * must be an xmm register (opnd_create_reg()).
+ * \param i1 The opnd_t explicit second source operand for the instruction, which
+ * must be an immediate integer (opnd_create_immed_int()).
+ * \param i2 The opnd_t explicit third source operand for the instruction, which
+ * must be an immediate integer (opnd_create_immed_int()).
+ */
+#define INSTR_CREATE_insertq_imm(dc, d, r, i1, i2) \
+  instr_create_1dst_3src((dc), OP_insertq, (d), (r), (i1), (i2))
+
+/* 1 implicit destination, 2 sources: 1 explicit, 1 implicit */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param i The opnd_t explicit source operand for the instruction, which
+ * must be an immediate integer (opnd_create_immed_int()).
+ */
+#define INSTR_CREATE_aam(dc, i) \
+  instr_create_1dst_2src((dc), OP_aam, opnd_create_reg(REG_AX), (i), opnd_create_reg(REG_AX))
+#define INSTR_CREATE_aad(dc, i) \
+  instr_create_1dst_2src((dc), OP_aad, opnd_create_reg(REG_AX), (i), opnd_create_reg(REG_AX))
+/* @} */ /* end doxygen group */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param t The opnd_t target operand for the instruction, which can be either
+ * a pc (opnd_create_pc()) or an instr_t (opnd_create_instr()).
+ */
+#define INSTR_CREATE_loopne(dc, t) \
+  instr_create_1dst_2src((dc), OP_loopne, opnd_create_reg(REG_XCX), (t), opnd_create_reg(REG_XCX))
+#define INSTR_CREATE_loope(dc, t) \
+  instr_create_1dst_2src((dc), OP_loope, opnd_create_reg(REG_XCX), (t), opnd_create_reg(REG_XCX))
+#define INSTR_CREATE_loop(dc, t) \
+  instr_create_1dst_2src((dc), OP_loop, opnd_create_reg(REG_XCX), (t), opnd_create_reg(REG_XCX))
+/* @} */ /* end doxygen group */
+
+/* 1 implicit destination, 2 implicit sources */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx, automatically
+ * supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ */
+#define INSTR_CREATE_popf(dc) \
+  instr_create_1dst_2src((dc), OP_popf, opnd_create_reg(REG_XSP), \
+    opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_STACK))
+#define INSTR_CREATE_ret(dc) \
+  instr_create_1dst_2src((dc), OP_ret, opnd_create_reg(REG_XSP), \
+    opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_ret))
+/* FIXME - size is wrong, xref PR 214976/10541 */
+/* WARNING: actually performs multiple stack operations (not reflected in size) */
+/* FIXME - using OPSZ_STACK to get 64-bit size (32-bit is default for x64) */
+#define INSTR_CREATE_ret_far(dc) \
+  instr_create_1dst_2src((dc), OP_ret_far, opnd_create_reg(REG_XSP), \
+    opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_STACK))
+/* FIXME - size is wrong, xref PR 214976/10541 */
+/* WARNING: actually performs multiple stack operations (not reflected in size) */
+/* FIXME - must use OPSZ_STACK (OPSZ_REXVARSTACK is ambiguous => iretd in 64-bit mode) */
+#define INSTR_CREATE_iret(dc) \
+  instr_create_1dst_2src((dc), OP_iret, opnd_create_reg(REG_XSP), \
+    opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_STACK))
+/* @} */ /* end doxygen group */
+
+/* 1 destination, 3 sources: 1 implicit */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the
+ * given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ * \param s The opnd_t explicit source operand for the instruction.
+ * \param ri The opnd_t explicit source operand for the instruction, which must
+ * be one of the following:
+ * -# The register cl (#opnd_create_reg(#REG_CL));
+ * -# An immediate integer (opnd_create_immed_int()) of size #OPSZ_1;
+ */
+#define INSTR_CREATE_shld(dc, d, s, ri) \
+  instr_create_1dst_3src((dc), OP_shld, (d), (s), (ri), (d))
+#define INSTR_CREATE_shrd(dc, d, s, ri) \
+  instr_create_1dst_3src((dc), OP_shrd, (d), (s), (ri), (d))
+/* @} */ /* end doxygen group */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the
+ * given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ * \param s The opnd_t explicit source operand for the instruction.
+ * \param i The opnd_t explicit second source operand for the instruction, which
+ * must be an immediate integer (opnd_create_immed_int()).
+ */
+#define INSTR_CREATE_shufps(dc, d, s, i ) \
+  instr_create_1dst_3src((dc), OP_shufps, (d), (s), (i), (d))
+#define INSTR_CREATE_shufpd(dc, d, s, i) \
+  instr_create_1dst_3src((dc), OP_shufpd, (d), (s), (i), (d))
+#define INSTR_CREATE_cmpps(dc, d, s, i) \
+  instr_create_1dst_3src((dc), OP_cmpps, (d), (s), (i), (d))
+#define INSTR_CREATE_cmpss(dc, d, s, i) \
+  instr_create_1dst_3src((dc), OP_cmpss, (d), (s), (i), (d))
+#define INSTR_CREATE_cmppd(dc, d, s, i) \
+  instr_create_1dst_3src((dc), OP_cmppd, (d), (s), (i), (d))
+#define INSTR_CREATE_cmpsd(dc, d, s, i) \
+  instr_create_1dst_3src((dc), OP_cmpsd, (d), (s), (i), (d))
+#define INSTR_CREATE_palignr(dc, d, s, i) \
+  instr_create_1dst_3src((dc), OP_palignr, (d), (s), (i), (d))
+#define INSTR_CREATE_dpps(dc, d, s, i) \
+  instr_create_1dst_3src((dc), OP_dpps, (d), (s), (i), (d))
+#define INSTR_CREATE_dppd(dc, d, s, i) \
+  instr_create_1dst_3src((dc), OP_dppd, (d), (s), (i), (d))
+#define INSTR_CREATE_mpsadbw(dc, d, s, i) \
+  instr_create_1dst_3src((dc), OP_mpsadbw, (d), (s), (i), (d))
+/* @} */ /* end doxygen group */
+
+/* 1 implicit destination, 3 sources */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the
+ * given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param s1 The opnd_t explicit first source operand for the instruction.
+ * \param s2 The opnd_t explicit second source operand for the instruction.
+ * \param i The opnd_t explicit third source operand for the instruction, which
+ * must be an immediate integer (opnd_create_immed_int()).
+ */
+#define INSTR_CREATE_pcmpistrm(dc, s1, s2, i) \
+  instr_create_1dst_3src((dc), OP_pcmpistrm, opnd_create_reg(REG_XMM0), (s1), (s2), (i))
+#define INSTR_CREATE_pcmpistri(dc, s1, s2, i) \
+  instr_create_1dst_3src((dc), OP_pcmpistri, opnd_create_reg(REG_ECX), (s1), (s2), (i))
+/* @} */ /* end doxygen group */
+
+/* 1 implicit destination, 3 sources: 2 implicit */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx_imm macro creates an instr_t with opcode OP_xxx and the
+ * given explicit operands, automatically supplying any implicit operands. The _imm
+ * suffix distinguishes between alternative forms of the same opcode: these
+ * forms take an explicit immediate.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param i The opnd_t explicit second source operand for the instruction, which
+ * must be an immediate integer (opnd_create_immed_int()).
+ */
+#define INSTR_CREATE_ret_imm(dc, i) \
+  instr_create_1dst_3src((dc), OP_ret, opnd_create_reg(REG_XSP), (i), \
+    opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_ret))
+/* FIXME - size is wrong, xref PR 214976/10541 */
+/* WARNING: actually performs multiple stack operations (not reflected in size) */
+/* FIXME - using OPSZ_STACK to get 64-bit size (32-bit is default for x64) */
+#define INSTR_CREATE_ret_far_imm(dc, i) \
+  instr_create_1dst_3src((dc), OP_ret_far, opnd_create_reg(REG_XSP), (i), \
+    opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_STACK))
+/* @} */ /* end doxygen group */
+
+/* 1 implicit destination, 5 sources: 2 implicit */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the
+ * given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param s1 The opnd_t explicit first source operand for the instruction.
+ * \param s2 The opnd_t explicit second source operand for the instruction.
+ * \param i The opnd_t explicit third source operand for the instruction, which
+ * must be an immediate integer (opnd_create_immed_int()).
+ */
+#define INSTR_CREATE_pcmpestrm(dc, s1, s2, i) \
+  instr_create_1dst_5src((dc), OP_pcmpestrm, opnd_create_reg(REG_XMM0), \
+    (s1), (s2), (i), opnd_create_reg(REG_EAX), opnd_create_reg(REG_EDX))
+#define INSTR_CREATE_pcmpestri(dc, s1, s2, i) \
+  instr_create_1dst_5src((dc), OP_pcmpestri, opnd_create_reg(REG_ECX), \
+    (s1), (s2), (i), opnd_create_reg(REG_EAX), opnd_create_reg(REG_EDX))
+/* @} */ /* end doxygen group */
+
+/* 2 implicit destinations, no sources */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx, automatically
+ * supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ */
+#define INSTR_CREATE_rdtsc(dc) \
+  instr_create_2dst_0src((dc), OP_rdtsc, opnd_create_reg(REG_EDX), opnd_create_reg(REG_EAX))
+
+/* 2 destinations: 1 implicit, 1 source */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ * \param s The opnd_t explicit source operand for the instruction.
+ */
+#define INSTR_CREATE_lds(dc, d, s) \
+  instr_create_2dst_1src((dc), OP_lds, (d), opnd_create_reg(SEG_DS), (s))
+#define INSTR_CREATE_lss(dc, d, s) \
+  instr_create_2dst_1src((dc), OP_lss, (d), opnd_create_reg(SEG_SS), (s))
+#define INSTR_CREATE_les(dc, d, s) \
+  instr_create_2dst_1src((dc), OP_les, (d), opnd_create_reg(SEG_ES), (s))
+#define INSTR_CREATE_lfs(dc, d, s) \
+  instr_create_2dst_1src((dc), OP_lfs, (d), opnd_create_reg(SEG_FS), (s))
+#define INSTR_CREATE_lgs(dc, d, s) \
+  instr_create_2dst_1src((dc), OP_lgs, (d), opnd_create_reg(SEG_GS), (s))
+/* @} */ /* end doxygen group */
+
+/* 2 implicit destinations, 1 implicit source */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx, automatically
+ * supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ */
+#define INSTR_CREATE_pushf(dc) \
+  instr_create_2dst_1src((dc), OP_pushf, opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_STACK), \
+    opnd_create_reg(REG_XSP))
+/* FIXME - size is wrong, xref PR 214976/10541 */
+/* WARNING: actually performs multiple stack operations (not reflected in size) */
+#define INSTR_CREATE_int3(dc) \
+  instr_create_2dst_1src((dc), OP_int3, opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_STACK), \
+    opnd_create_reg(REG_XSP))
+/* FIXME - size is wrong, xref PR 214976/10541 */
+/* WARNING: actually performs multiple stack operations (not reflected in size) */
+#define INSTR_CREATE_into(dc) \
+  instr_create_2dst_1src((dc), OP_into, opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_STACK), \
+    opnd_create_reg(REG_XSP))
+/* FIXME - size is wrong, xref PR 214976/10541 */
+/* WARNING: actually performs multiple stack operations (not reflected in size) */
+#define INSTR_CREATE_int1(dc) \
+  instr_create_2dst_1src((dc), OP_int1, opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_STACK), \
+    opnd_create_reg(REG_XSP))
+#define INSTR_CREATE_rdmsr(dc) \
+  instr_create_2dst_1src((dc), OP_rdmsr, opnd_create_reg(REG_EDX), opnd_create_reg(REG_EAX), \
+    opnd_create_reg(REG_ECX))
+#define INSTR_CREATE_rdpmc(dc) \
+  instr_create_2dst_1src((dc), OP_rdpmc, opnd_create_reg(REG_EDX), opnd_create_reg(REG_EAX), \
+    opnd_create_reg(REG_ECX))
+/* @} */ /* end doxygen group */
+
+/* 2 destinations: 1 implicit, 2 sources */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ */
+#define INSTR_CREATE_pop(dc, d) \
+  instr_create_2dst_2src((dc), OP_pop, (d), opnd_create_reg(REG_XSP), \
+    opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_VARSTACK))
+
+/* 2 destinations: 1 implicit, 2 sources: 1 implicit */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ * \param s The opnd_t explicit source operand for the instruction.
+ */
+#define INSTR_CREATE_xchg(dc, d, s) \
+  instr_create_2dst_2src((dc), OP_xchg, (d), (s), (d), (s))
+#define INSTR_CREATE_xadd(dc, d, s) \
+  instr_create_2dst_2src((dc), OP_xadd, (d), (s), (d), (s))
+/* @} */ /* end doxygen group */
+
+/* string instructions */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx_1 or INSTR_CREATE_xxx_4 macro creates an instr_t with opcode
+ * OP_xxx, automatically supplying any implicit operands.  The _1 or _4 suffixes
+ * distinguish between alternative forms of the same opcode (1 and 4 identify the
+ * operand size).
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ */
+#define INSTR_CREATE_ins_1(dc) \
+  instr_create_2dst_2src((dc), OP_ins, \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_reg(REG_XDI), opnd_create_reg(REG_DX), opnd_create_reg(REG_XDI))
+#define INSTR_CREATE_ins_4(dc) \
+  instr_create_2dst_2src((dc), OP_ins, \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_reg(REG_XDI), opnd_create_reg(REG_DX), opnd_create_reg(REG_XDI))
+#define INSTR_CREATE_stos_1(dc) \
+  instr_create_2dst_2src((dc), OP_stos, \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_reg(REG_XDI), opnd_create_reg(REG_AL), opnd_create_reg(REG_XDI))
+#define INSTR_CREATE_stos_4(dc) \
+  instr_create_2dst_2src((dc), OP_stos, \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_reg(REG_XDI), opnd_create_reg(REG_XAX), opnd_create_reg(REG_XDI))
+#define INSTR_CREATE_lods_1(dc) \
+  instr_create_2dst_2src((dc), OP_lods, opnd_create_reg(REG_AL), opnd_create_reg(REG_XSI), \
+    opnd_create_far_base_disp(SEG_DS, REG_XSI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_reg(REG_XSI))
+#define INSTR_CREATE_lods_4(dc) \
+  instr_create_2dst_2src((dc), OP_lods, opnd_create_reg(REG_XAX), opnd_create_reg(REG_XSI), \
+    opnd_create_far_base_disp(SEG_DS, REG_XSI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_reg(REG_XSI)) 
+#define INSTR_CREATE_movs_1(dc) \
+  instr_create_3dst_3src((dc), OP_movs, \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI), \
+    opnd_create_far_base_disp(SEG_DS, REG_XSI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI))
+#define INSTR_CREATE_movs_4(dc) \
+  instr_create_3dst_3src((dc), OP_movs, \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI), \
+    opnd_create_far_base_disp(SEG_DS, REG_XSI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI))
+#define INSTR_CREATE_rep_ins_1(dc) \
+  instr_create_3dst_3src((dc), OP_rep_ins, \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX), \
+    opnd_create_reg(REG_DX), opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX))
+#define INSTR_CREATE_rep_ins_4(dc) \
+  instr_create_3dst_3src((dc), OP_rep_ins, \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX), \
+    opnd_create_reg(REG_DX), opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX))
+#define INSTR_CREATE_rep_stos_1(dc) \
+  instr_create_3dst_3src((dc), OP_rep_stos, \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX), \
+    opnd_create_reg(REG_AL), opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX))
+#define INSTR_CREATE_rep_stos_4(dc) \
+  instr_create_3dst_3src((dc), OP_rep_stos, \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX), \
+    opnd_create_reg(REG_EAX), opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX))
+#define INSTR_CREATE_rep_lods_1(dc) \
+  instr_create_3dst_3src((dc), OP_rep_lods, opnd_create_reg(REG_AL), opnd_create_reg(REG_XSI), \
+    opnd_create_reg(REG_ECX), \
+    opnd_create_far_base_disp(SEG_DS, REG_XSI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_reg(REG_XSI), opnd_create_reg(REG_ECX))
+#define INSTR_CREATE_rep_lods_4(dc) \
+  instr_create_3dst_3src((dc), OP_rep_lods, opnd_create_reg(REG_EAX), opnd_create_reg(REG_XSI), \
+    opnd_create_reg(REG_ECX), \
+    opnd_create_far_base_disp(SEG_DS, REG_XSI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_reg(REG_XSI), opnd_create_reg(REG_ECX))
+#define INSTR_CREATE_rep_movs_1(dc) \
+  instr_create_4dst_4src((dc), OP_rep_movs, \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX), \
+    opnd_create_far_base_disp(SEG_DS, REG_XSI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX))
+#define INSTR_CREATE_rep_movs_4(dc) \
+  instr_create_4dst_4src((dc), OP_rep_movs, \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX), \
+    opnd_create_far_base_disp(SEG_DS, REG_XSI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX))
+#define INSTR_CREATE_outs_1(dc) \
+  instr_create_1dst_3src((dc), OP_outs, opnd_create_reg(REG_XSI), \
+    opnd_create_far_base_disp(SEG_DS, REG_XSI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_reg(REG_DX), opnd_create_reg(REG_XSI))
+#define INSTR_CREATE_outs_4(dc) \
+  instr_create_1dst_3src((dc), OP_outs, opnd_create_reg(REG_XSI), \
+    opnd_create_far_base_disp(SEG_DS, REG_XSI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_reg(REG_DX), opnd_create_reg(REG_XSI))
+#define INSTR_CREATE_cmps_1(dc) \
+  instr_create_2dst_4src((dc), OP_cmps, opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI), \
+    opnd_create_far_base_disp(SEG_DS, REG_XSI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI))
+#define INSTR_CREATE_cmps_4(dc) \
+  instr_create_2dst_4src((dc), OP_cmps, opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI), \
+    opnd_create_far_base_disp(SEG_DS, REG_XSI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI))
+#define INSTR_CREATE_scas_1(dc) \
+  instr_create_1dst_3src((dc), OP_scas, opnd_create_reg(REG_XDI), \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_reg(REG_AL), opnd_create_reg(REG_XDI))
+#define INSTR_CREATE_scas_4(dc) \
+  instr_create_1dst_3src((dc), OP_scas, opnd_create_reg(REG_XDI), \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_reg(REG_EAX), opnd_create_reg(REG_XDI))
+#define INSTR_CREATE_rep_outs_1(dc) \
+  instr_create_2dst_4src((dc), OP_rep_outs, opnd_create_reg(REG_XSI), opnd_create_reg(REG_ECX), \
+    opnd_create_far_base_disp(SEG_DS, REG_XSI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_reg(REG_DX), opnd_create_reg(REG_XSI), opnd_create_reg(REG_ECX))
+#define INSTR_CREATE_rep_outs_4(dc) \
+  instr_create_2dst_4src((dc), OP_rep_outs, opnd_create_reg(REG_XSI), opnd_create_reg(REG_ECX), \
+    opnd_create_far_base_disp(SEG_DS, REG_XSI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_reg(REG_DX), opnd_create_reg(REG_XSI), opnd_create_reg(REG_ECX))
+#define INSTR_CREATE_rep_cmps_1(dc) \
+  instr_create_3dst_5src((dc), OP_rep_cmps, opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI), \
+    opnd_create_reg(REG_ECX), \
+    opnd_create_far_base_disp(SEG_DS, REG_XSI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX))
+#define INSTR_CREATE_rep_cmps_4(dc) \
+  instr_create_3dst_5src((dc), OP_rep_cmps, opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI), \
+    opnd_create_reg(REG_ECX), \
+    opnd_create_far_base_disp(SEG_DS, REG_XSI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX))
+#define INSTR_CREATE_repne_cmps_1(dc) \
+  instr_create_3dst_5src((dc), OP_repne_cmps, opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI), \
+    opnd_create_reg(REG_ECX), \
+    opnd_create_far_base_disp(SEG_DS, REG_XSI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX))
+#define INSTR_CREATE_repne_cmps_4(dc) \
+  instr_create_3dst_5src((dc), OP_repne_cmps, opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI), \
+    opnd_create_reg(REG_ECX), \
+    opnd_create_far_base_disp(SEG_DS, REG_XSI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_reg(REG_XSI), opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX))
+#define INSTR_CREATE_rep_scas_1(dc) \
+  instr_create_2dst_4src((dc), OP_rep_scas, opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX), \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_reg(REG_AL), opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX))
+#define INSTR_CREATE_rep_scas_4(dc) \
+  instr_create_2dst_4src((dc), OP_rep_scas, opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX), \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_reg(REG_EAX), opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX))
+#define INSTR_CREATE_repne_scas_1(dc) \
+  instr_create_2dst_4src((dc), OP_repne_scas, opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX), \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_1), \
+    opnd_create_reg(REG_AL), opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX))
+#define INSTR_CREATE_repne_scas_4(dc) \
+  instr_create_2dst_4src((dc), OP_repne_scas, opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX), \
+    opnd_create_far_base_disp(SEG_ES, REG_XDI, REG_NULL, 0, 0, OPSZ_4_rex8_short2), \
+    opnd_create_reg(REG_EAX), opnd_create_reg(REG_XDI), opnd_create_reg(REG_ECX))
+/* @} */ /* end doxygen group */
+
+/* floating point */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit register operand, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param f The opnd_t explicit source operand for the instruction, which must
+ * be a floating point register (opnd_create_reg()).
+ */
+#define INSTR_CREATE_fxch(dc, f) \
+  instr_create_2dst_2src((dc), OP_fxch, opnd_create_reg(REG_ST0), (f), \
+    opnd_create_reg(REG_ST0), (f))
+
+/* 2 destinations, 2 sources: 1 implicit */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param t The opnd_t target operand for the instruction, which can be either
+ * a pc (opnd_create_pc()) or an instr_t (opnd_create_instr()).
+ */
+#define INSTR_CREATE_call(dc, t) \
+  instr_create_2dst_2src((dc), OP_call, opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_STACK), \
+    (t), opnd_create_reg(REG_XSP))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param t The opnd_t target operand for the instruction, which should be
+ * a memory reference created with opnd_create_base_disp().
+ */
+#define INSTR_CREATE_call_ind(dc, t) \
+  instr_create_2dst_2src((dc), OP_call_ind, opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_STACK), \
+    (t), opnd_create_reg(REG_XSP))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param t The opnd_t target operand for the instruction, which should be
+ * a far pc operand created with opnd_create_far_pc().
+ */
+/* FIXME - size is wrong, xref PR 214976/10541 */
+/* WARNING: actually performs multiple stack operations (not reflected in size) */
+#define INSTR_CREATE_call_far(dc, t) \
+  instr_create_2dst_2src((dc), OP_call_far, opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_STACK), \
+    (t), opnd_create_reg(REG_XSP))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param t The opnd_t target operand for the instruction, which should be
+ * a far memory reference created with opnd_create_far_base_disp().
+ */
+/* FIXME - size is wrong, xref PR 214976/10541 */
+/* WARNING: actually performs multiple stack operations (not reflected in size) */
+/* FIXME - unlike iret/ret_far, OPSZ_4 is typical desired size => OPSZ_REXVARSTACK */
+#define INSTR_CREATE_call_far_ind(dc, t) \
+  instr_create_2dst_2src((dc), OP_call_far_ind, opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_REXVARSTACK), \
+    (t), opnd_create_reg(REG_XSP))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and
+ * the given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param s The opnd_t explicit source operand for the instruction.
+ */
+#define INSTR_CREATE_push(dc, s) \
+  instr_create_2dst_2src((dc), OP_push, opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_VARSTACK), \
+    (s), opnd_create_reg(REG_XSP))
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the
+ * given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param i The opnd_t explicit second source operand for the instruction, which
+ * must be an immediate integer (opnd_create_immed_int()).
+ */
+#define INSTR_CREATE_push_imm(dc, i) \
+  instr_create_2dst_2src((dc), OP_push_imm, opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_VARSTACK), \
+    (i), opnd_create_reg(REG_XSP))
+
+/* 2 implicit destinations, 2 sources: 1 implicit */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the
+ * given explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param i The opnd_t explicit second source operand for the instruction, which
+ * must be an immediate integer (opnd_create_immed_int()).
+ */
+/* FIXME - size is wrong, xref PR 214976/10541 */
+/* WARNING: actually performs multiple stack operations (not reflected in size) */
+#define INSTR_CREATE_int(dc, i) \
+  instr_create_2dst_2src((dc), OP_int, opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_STACK), \
+    (i), opnd_create_reg(REG_XSP))
+
+/* 2 destinations: 1 implicit, 3 sources: 1 implicit */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx_1 or INSTR_CREATE_xxx_4 macro creates an
+ * instr_t with opcode OP_xxx and the given explicit operands, automatically
+ * supplying any implicit operands.    The _1 or _4 suffixes distinguish between
+ * alternative forms of the same opcode (1 and 4 identify the operand size).
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ * \param s The opnd_t explicit source operand for the instruction.
+ */
+#define INSTR_CREATE_cmpxchg_1(dc, d, s) \
+  instr_create_2dst_3src((dc), OP_cmpxchg, (d), opnd_create_reg(REG_AL), (s), (d), \
+    opnd_create_reg(REG_AL))
+#define INSTR_CREATE_cmpxchg_4(dc, d, s) \
+  instr_create_2dst_3src((dc), OP_cmpxchg, (d), opnd_create_reg(REG_EAX), (s), (d), \
+    opnd_create_reg(REG_EAX))
+/* @} */ /* end doxygen group */
+
+/* 2 implicit destinations, 3 implicit sources */
+#define INSTR_CREATE_leave(dc) \
+  instr_create_2dst_3src((dc), OP_leave, opnd_create_reg(REG_XSP), \
+    opnd_create_reg(REG_XBP), opnd_create_reg(REG_XBP), opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XBP, REG_NULL, 0, 0, OPSZ_STACK))
+
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx, automatically
+ * supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ */
+/* 2 implicit destinations, 8 implicit sources */
+/* FIXME - size is wrong, xref PR 214976/10541 */
+/* WARNING: actually performs multiple stack operations (not reflected in size) */
+#define INSTR_CREATE_pusha(dc)  instr_create_pusha((dc))
+
+/* 3 implicit destinations, 1 source */
+#define INSTR_CREATE_cpuid(dc) \
+  instr_create_4dst_1src((dc), OP_cpuid, opnd_create_reg(REG_EAX), \
+    opnd_create_reg(REG_EBX), opnd_create_reg(REG_ECX), opnd_create_reg(REG_EDX), opnd_create_reg(REG_EAX))
+/* @} */ /* end doxygen group */
+
+/* 3 destinations: 2 implicit, 5 implicit sources */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param d The opnd_t explicit destination operand for the instruction.
+ */
+#define INSTR_CREATE_cmpxchg8b(dc, d) \
+  instr_create_3dst_5src((dc), OP_cmpxchg8b, (d), opnd_create_reg(REG_EAX), \
+    opnd_create_reg(REG_EDX), (d), opnd_create_reg(REG_EAX), opnd_create_reg(REG_EDX), \
+    opnd_create_reg(REG_ECX), opnd_create_reg(REG_EBX))
+
+/* 3 implicit destinations, 4 sources: 2 implicit */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx and the given
+ * explicit operands, automatically supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param i16 The opnd_t explicit second source operand for the instruction, which
+ * must be an immediate integer (opnd_create_immed_int()) of OPSZ_2.
+ * \param i8 The opnd_t explicit second source operand for the instruction, which
+ * must be an immediate integer (opnd_create_immed_int()) of OPSZ_1.
+ */
+/* FIXME - size is wrong, xref PR 214976/10541 */
+/* WARNING: actually performs multiple stack operations (not reflected in size) */
+#define INSTR_CREATE_enter(dc, i16, i8) \
+  instr_create_3dst_4src((dc), OP_enter, opnd_create_reg(REG_XSP), \
+    opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_STACK), \
+    opnd_create_reg(REG_XBP), \
+    (i16), (i8), opnd_create_reg(REG_XSP), opnd_create_reg(REG_XBP))
+
+/* 8 implicit destinations, 2 implicit sources */
+/**
+ * This INSTR_CREATE_xxx macro creates an instr_t with opcode OP_xxx, automatically
+ * supplying any implicit operands.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ */
+/* FIXME - size is wrong, xref PR 214976/10541 */
+/* WARNING: actually performs multiple stack operations (not reflected in size) */
+#define INSTR_CREATE_popa(dc)   instr_create_popa((dc))
+
+/****************************************************************************/
+
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * Convenience routine for nop of certain size.  We choose edi as working register
+ * for multibyte nops (seems least likely to impact performance: Microsoft uses it
+ * and DR used to steal it).
+ * Note that Intel now recommends a different set of multi-byte nops,
+ * but we stick with these as our tools (mainly windbg) don't understand
+ * the OP_nop_modrm encoding (though should work on PPro+).
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ */
+#define INSTR_CREATE_nop1byte(dc) INSTR_CREATE_nop(dc)
+#define INSTR_CREATE_nop2byte(dc) INSTR_CREATE_nop2byte_reg(dc, REG_XDI)
+#define INSTR_CREATE_nop3byte(dc) INSTR_CREATE_nop3byte_reg(dc, REG_XDI)
+/* @} */ /* end doxygen group */
+/* @{ */ /* doxygen start group; w/ DISTRIBUTE_GROUP_DOC=YES, one comment suffices. */
+/**
+ * Convenience routine for nop of certain size.
+ * Note that Intel now recommends a different set of multi-byte nops,
+ * but we stick with these as our tools (mainly windbg) don't understand
+ * the OP_nop_modrm encoding (though should work on PPro+).
+ * AMD recommends 0x66 0x66 ... 0x90 for older processors.
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param reg A reg_id_t (NOT opnd_t) to use as source and destination.
+ * For 64-bit mode, use a 64-bit register, but NOT rbp or rsp for the 3-byte form.
+ */
+#ifdef X64
+static inline instr_t *
+INSTR_CREATE_nop2byte_reg(dcontext_t *dcontext, reg_id_t reg)
+{
+    /* 32-bit register target zeroes out the top bits, so we use the Intel
+     * and AMD recommended 0x66 0x90 */
+    instr_t *in = instr_build_bits(dcontext, OP_nop, 2);
+    instr_set_raw_byte(in, 0, 0x66);
+    instr_set_raw_byte(in, 1, 0x90);
+    instr_set_operands_valid(in, true);
+    return in;
+}
+#else
+# define INSTR_CREATE_nop2byte_reg(dc, reg) \
+    INSTR_CREATE_mov_st(dc, opnd_create_reg(reg), opnd_create_reg(reg))
+#endif
+#ifdef X64
+/* lea's target is 32-bit but address register is 64: so we eliminate the
+ * displacement and put in rex.w 
+ */
+# define INSTR_CREATE_nop3byte_reg(dc, reg)     \
+    INSTR_CREATE_lea(dc, opnd_create_reg(reg), OPND_CREATE_MEM_lea(reg, REG_NULL, 0, 0)) 
+#else
+# define INSTR_CREATE_nop3byte_reg(dc, reg)     \
+    INSTR_CREATE_lea(dc, opnd_create_reg(reg), \
+                     opnd_create_base_disp_ex(reg, REG_NULL, 0, 0, OPSZ_lea, \
+                                              true/*encode 0*/, false, false)) 
+#endif
+/* @} */ /* end doxygen group */
+#ifndef UNSUPPORTED_API
+/* DR_API EXPORT END */
+#endif
+
+/**
+ * Convenience routine for nop of certain size.
+ * Note that Intel now recommends a different set of multi-byte nops,
+ * but we stick with these as our tools (mainly windbg) don't understand
+ * the OP_nop_modrm encoding (though should work on PPro+).
+ * \param dc The void * dcontext used to allocate memory for the instr_t.
+ * \param n  The number of bytes in the encoding of the nop.
+ */
+#define INSTR_CREATE_nopNbyte(dc, n) instr_create_nbyte_nop(dc, n, false)
+
+/* convenience routines for when you only need raw bits */
+#define INSTR_CREATE_RAW_pushf(dc) instr_create_raw_1byte(dc, 0x9c)
+#define INSTR_CREATE_RAW_popf(dc)  instr_create_raw_1byte(dc, 0x9d)
+#define INSTR_CREATE_RAW_pusha(dc) instr_create_raw_1byte(dc, 0x60)
+#define INSTR_CREATE_RAW_popa(dc)  instr_create_raw_1byte(dc, 0x61)
+#define INSTR_CREATE_RAW_nop(dc)   instr_create_raw_1byte(dc, 0x90)
+#define INSTR_CREATE_RAW_nop1byte(dc) INSTR_CREATE_RAW_nop(dc)
+#ifdef X64
+# define INSTR_CREATE_RAW_nop2byte(dc) instr_create_raw_2bytes(dc, 0x66, 0x90)
+# define INSTR_CREATE_RAW_nop3byte(dc) instr_create_raw_3bytes(dc, 0x48, 0x8d, 0x3f)
+#else
+# define INSTR_CREATE_RAW_nop2byte(dc) instr_create_raw_2bytes(dc, 0x8b, 0xff)
+# define INSTR_CREATE_RAW_nop3byte(dc) instr_create_raw_3bytes(dc, 0x8d, 0x7f, 0x00)
+#endif
+#define INSTR_CREATE_RAW_nopNbyte(dc, n) instr_create_nbyte_nop(dc, n, true)
+#ifdef UNSUPPORTED_API
+/* DR_API EXPORT END */
+#endif
+
+#endif /* _INSTR_CREATE_H_ */
+
diff --git a/core/x86/instrument.c b/core/x86/instrument.c
new file mode 100644
index 000000000..72a7d348a
--- /dev/null
+++ b/core/x86/instrument.c
@@ -0,0 +1,4121 @@
+/* **********************************************************
+ * Copyright (c) 2002-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+/*
+ * instrument.c - interface for instrumentation
+ */
+
+#include "globals.h"   /* just to disable warning C4206 about an empty file */
+
+
+#include "instrument.h"
+#include "instrlist.h"
+#include "arch.h"
+#include "instr.h"
+#include "instr_create.h"
+#include "decode.h"
+#include "disassemble.h"
+#include "fragment.h"
+#include "emit.h"
+#include "link.h"
+#include "monitor.h" /* for mark_trace_head */
+#include <string.h> /* for strstr */
+#include <stdarg.h> /* for varargs */
+#ifdef WINDOWS
+# include "nudge.h" /* for nudge_internal() */
+#endif
+
+#ifdef CLIENT_INTERFACE
+/* in utils.c, not exported to everyone */
+extern void do_file_write(file_t f, const char *fmt, va_list ap);
+
+#ifdef DEBUG
+/* case 10450: give messages to clients */
+/* we can't undef ASSERT b/c of DYNAMO_OPTION */
+# undef ASSERT_TRUNCATE
+# undef ASSERT_BITFIELD_TRUNCATE
+# undef ASSERT_NOT_REACHED
+# define ASSERT_TRUNCATE DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_BITFIELD_TRUNCATE DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_NOT_REACHED DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+#endif
+
+/* PR 200065: User passes us the shared library, we look up "dr_init",
+ * and call it.  From there, the client can register which events it
+ * wishes to receive.
+ */
+#define INSTRUMENT_INIT_NAME "dr_init"
+
+/* PR 250952: version check 
+ * If changing this, don't forget to update:
+ * - lib/dr_defines.h _USES_DR_VERSION_
+ * - api/docs/footer.html
+ */
+#define USES_DR_VERSION_NAME "_USES_DR_VERSION_"
+/* should we expose this for use in samples/tracedump.c? */
+#define OLDEST_COMPATIBLE_VERSION  96 /* 0.9.6 == 1.0.0 through 1.2.0 */
+/* The 3rd version number, the bugfix/patch number, should not affect
+ * compatibility, so our version check number simply uses:
+ *   major*100 + minor
+ * Which gives us room for 100 minor versions per major.
+ */
+#define NEWEST_COMPATIBLE_VERSION 103 /* 1.3.x */
+
+/* Store the unique not-part-of-version build number (the version
+ * BUILD_NUMBER is limited to 64K and is not guaranteed to be unique)
+ * somewhere accessible at a customer site.  We could alternatively
+ * pull it out of our DYNAMORIO_DEFINES string.
+ */
+DR_API const char *unique_build_number = STRINGIFY(UNIQUE_BUILD_NUMBER);
+
+/* Acquire when registering or unregistering event callbacks */
+DECLARE_CXTSWPROT_VAR(static mutex_t callback_registration_lock,
+                      INIT_LOCK_FREE(callback_registration_lock));
+
+/* Structures for maintaining lists of event callbacks */
+typedef void (*callback_t)(void);
+typedef struct _callback_list_t {
+    callback_t *callbacks;      /* array of callback functions */
+    size_t num;                 /* number of callbacks registered */
+    size_t size;                /* allocated space (may be larger than num) */
+} callback_list_t;
+
+/* This is a little convoluted.  The following is a macro to iterate
+ * over a list of callbacks and call each function.  We use a macro
+ * instead of a function so we can pass the function type and perform
+ * a typecast.  We need to copy the callback list before iterating to
+ * support the possibility of one callback unregistering another and
+ * messing up the list while we're iterating.  We'll optimize the case
+ * for 5 or fewer registered callbacks and stack-allocate the temp
+ * list.  Otherwise, we'll heap-allocate the temp.
+ *
+ * We consider the first registered callback to have the highest
+ * priority and call it last.  If we gave the last registered callback
+ * the highest priority, a client could re-register a routine to
+ * increase its priority.  That seems a little weird.
+ */
+#define FAST_COPY_SIZE 5
+#define call_all_ret(ret, retop, postop, vec, type, ...)                \
+    do {                                                                \
+        size_t idx, num;                                                \
+        mutex_lock(&callback_registration_lock);                        \
+        num = (vec).num;                                                \
+        if (num == 0) {                                                 \
+            mutex_unlock(&callback_registration_lock);                  \
+        }                                                               \
+        else if (num <= FAST_COPY_SIZE) {                               \
+            callback_t tmp[FAST_COPY_SIZE];                             \
+            memcpy(tmp, (vec).callbacks, num * sizeof(callback_t));     \
+            mutex_unlock(&callback_registration_lock);                  \
+            for (idx=0; idx<num; idx++) {                               \
+                ret retop (((type)tmp[num-idx-1])(__VA_ARGS__)) postop; \
+            }                                                           \
+        }                                                               \
+        else {                                                          \
+            callback_t *tmp = HEAP_ARRAY_ALLOC(GLOBAL_DCONTEXT, callback_t, num, \
+                                               ACCT_OTHER, UNPROTECTED); \
+            memcpy(tmp, (vec).callbacks, num * sizeof(callback_t));     \
+            mutex_unlock(&callback_registration_lock);                  \
+            for (idx=0; idx<num; idx++) {                               \
+                ret retop ((type)tmp[num-idx-1])(__VA_ARGS__) postop;   \
+            }                                                           \
+            HEAP_ARRAY_FREE(GLOBAL_DCONTEXT, tmp, callback_t, num,      \
+                            ACCT_OTHER, UNPROTECTED);                    \
+        }                                                               \
+    } while (0)
+
+/* It's less error-prone if we just have one call_all macro.  We'll
+ * reuse call_all_ret above for callbacks that don't have a return
+ * value by assigning to a dummy var.  Note that this means we'll
+ * have to pass an int-returning type to call_all()
+ */
+#define call_all(vec, type, ...)                        \
+    do {                                                \
+        int dummy;                                      \
+        call_all_ret(dummy, =, , vec, type, __VA_ARGS__); \
+    } while (0)
+
+/* Lists of callbacks for each event type.  Note that init and nudge
+ * callback lists are kept in the client_lib_t data structure below.
+ * We could store all lists on a per-client basis, but we can iterate
+ * over these lists slightly more efficiently if we store all
+ * callbacks for a specific event in a single list.
+ */
+static callback_list_t exit_callbacks = {0,};
+static callback_list_t thread_init_callbacks = {0,};
+static callback_list_t thread_exit_callbacks = {0,};
+#ifdef LINUX
+static callback_list_t fork_init_callbacks = {0,};
+#endif
+static callback_list_t bb_callbacks = {0,};
+static callback_list_t trace_callbacks = {0,};
+#ifdef CUSTOM_TRACES
+static callback_list_t end_trace_callbacks = {0,};
+#endif
+static callback_list_t fragdel_callbacks = {0,};
+static callback_list_t restore_state_callbacks = {0,};
+static callback_list_t module_load_callbacks = {0,};
+static callback_list_t module_unload_callbacks = {0,};
+static callback_list_t filter_syscall_callbacks = {0,};
+static callback_list_t pre_syscall_callbacks = {0,};
+static callback_list_t post_syscall_callbacks = {0,};
+#ifdef WINDOWS
+static callback_list_t exception_callbacks = {0,};
+#else
+static callback_list_t signal_callbacks = {0,};
+#endif
+#ifdef PROGRAM_SHEPHERDING
+static callback_list_t security_violation_callbacks = {0,};
+#endif
+
+/* An array of client libraries.  We use a static array instead of a
+ * heap-allocated list so we can load the client libs before
+ * initializing DR's heap.
+ */
+typedef struct _client_lib_t {
+    client_id_t id;
+    char path[MAXIMUM_PATH];
+    /* PR 366195: dlopen() handle truly is opaque: != start */
+    shlib_handle_t lib;
+    app_pc start;
+    app_pc end;
+    char options[MAX_OPTION_LENGTH];
+    /* We need to associate nudge events with a specific client so we
+     * store that list here in the client_lib_t instead of using a
+     * single global list.
+     */
+    callback_list_t nudge_callbacks;
+} client_lib_t;
+
+/* these should only be modified prior to instrument_init(), since no
+ * readers of the client_libs array (event handlers, etc.) use synch
+ */
+static client_lib_t client_libs[MAX_CLIENT_LIBS] = {{0,}};
+static size_t num_client_libs = 0;
+
+#ifdef WINDOWS
+/* used for nudge support */
+static bool block_client_owned_threads = false;
+DECLARE_CXTSWPROT_VAR(static int num_client_owned_threads, 0);
+/* protects block_client_owned_threads and incrementing num_client_owned_threads */
+DECLARE_CXTSWPROT_VAR(static mutex_t client_nudge_count_lock,
+                      INIT_LOCK_FREE(client_nudge_count_lock));
+#endif
+
+/****************************************************************************/
+/* INTERNAL ROUTINES */
+
+static void
+add_callback(callback_list_t *vec, void (*func)(void), bool unprotect)
+{
+    if (func == NULL) {
+        CLIENT_ASSERT(false, "trying to register a NULL callback");
+        return;
+    }
+
+    mutex_lock(&callback_registration_lock);
+    /* Although we're receiving a pointer to a callback_list_t, we're
+     * usually modifying a static var.
+     */
+    if (unprotect) {
+        SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+    }
+
+    /* We may already have an open slot since we allocate in twos and
+     * because we don't bother to free the storage when we remove the
+     * callback.  Check and only allocate if necessary.
+     */
+    if (vec->num == vec->size) {
+        callback_t *tmp = HEAP_ARRAY_ALLOC
+            (GLOBAL_DCONTEXT, callback_t, vec->size + 2, /* Let's allocate 2 */
+             ACCT_OTHER, UNPROTECTED);
+
+        if (tmp == NULL) {
+            CLIENT_ASSERT(false, "out of memory: can't register callback");
+            mutex_unlock(&callback_registration_lock);        
+            return;
+        }
+
+        if (vec->callbacks != NULL) {
+            memcpy(tmp, vec->callbacks, vec->num * sizeof(callback_t));
+            HEAP_ARRAY_FREE(GLOBAL_DCONTEXT, vec->callbacks, callback_t, vec->size,
+                        ACCT_OTHER, UNPROTECTED);
+        }
+        vec->callbacks = tmp;
+        vec->size += 2;
+    }
+
+    vec->callbacks[vec->num] = func;
+    vec->num++;
+
+    if (unprotect) {
+        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+    }
+    mutex_unlock(&callback_registration_lock);
+}
+
+static bool
+remove_callback(callback_list_t *vec, void (*func)(void), bool unprotect)
+{
+    size_t i;
+    bool found = false;
+
+    if (func == NULL) {
+        CLIENT_ASSERT(false, "trying to unregister a NULL callback");
+        return false;
+    }
+
+    mutex_lock(&callback_registration_lock);
+    /* Although we're receiving a pointer to a callback_list_t, we're
+     * usually modifying a static var.
+     */
+    if (unprotect) {
+        SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+    }
+
+    for (i=0; i<vec->num; i++) {
+        if (vec->callbacks[i] == func) {
+            size_t j;
+            /* shift down the entries on the tail */
+            for (j=i; j<vec->num-1; j++) {
+                vec->callbacks[j] = vec->callbacks[j+1];
+            }
+
+            vec->num -= 1;
+            found = true;
+            break;
+        }
+    }
+
+    SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+    if (unprotect) {
+        mutex_unlock(&callback_registration_lock);
+    }
+
+    return found;
+}
+
+/* This should only be called prior to instrument_init(),
+ * since no readers of the client_libs array use synch
+ * and since this routine assumes .data is writable.
+ */
+static void
+add_client_lib(char *path, char *id_str, char *options)
+{
+    client_id_t id;
+    shlib_handle_t client_lib;
+    DEBUG_DECLARE(size_t i);
+
+    ASSERT(!dynamo_initialized);
+
+    /* if ID not specified, we'll default to 0 */
+    id = (id_str == NULL) ? 0 : strtoul(id_str, NULL, 16);
+
+#ifdef DEBUG
+    /* Check for conflicting IDs */
+    for (i=0; i<num_client_libs; i++) {
+        CLIENT_ASSERT(client_libs[i].id != id, "Clients have the same ID");
+    }
+#endif
+
+    if (num_client_libs == MAX_CLIENT_LIBS) {
+        CLIENT_ASSERT(false, "Max number of clients reached");
+        return;
+    }
+
+    client_lib = load_shared_library(path);
+    if (client_lib == NULL) {
+        char msg[MAXIMUM_PATH*4];
+        char err[MAXIMUM_PATH*2];
+        shared_library_error(err, BUFFER_SIZE_ELEMENTS(err));
+        snprintf(msg, BUFFER_SIZE_ELEMENTS(msg),
+                 "\n\tError opening instrumentation library %s:\n\t%s",
+                 path, err);
+        NULL_TERMINATE_BUFFER(msg);
+
+        /* PR 232490 - malformed library names or incorrect
+         * permissions shouldn't blow up an app in release builds as
+         * they may happen at customer sites with a third party
+         * client.
+         */
+        CLIENT_ASSERT(false, msg);
+    }
+    else {
+        /* PR 250952: version check */
+        int *uses_dr_version = (int *)
+            lookup_library_routine(client_lib, USES_DR_VERSION_NAME);
+        if (uses_dr_version == NULL ||
+            *uses_dr_version < OLDEST_COMPATIBLE_VERSION ||
+            *uses_dr_version > NEWEST_COMPATIBLE_VERSION) {
+            /* not a fatal usage error since we want release build to continue */
+            CLIENT_ASSERT(false, 
+                          "client library is incompatible with this version of DR");
+            SYSLOG(SYSLOG_WARNING, CLIENT_VERSION_INCOMPATIBLE, 2, 
+                   get_application_name(), get_application_pid());
+        }
+        else {
+            size_t idx = num_client_libs++;
+            DEBUG_DECLARE(bool ok;)
+            client_libs[idx].id = id;
+            client_libs[idx].lib = client_lib;
+            DEBUG_DECLARE(ok =)
+                shared_library_bounds(client_lib, (byte *) uses_dr_version,
+                                      &client_libs[idx].start, &client_libs[idx].end);
+            ASSERT(ok);
+
+            LOG(GLOBAL, LOG_INTERP, 1, "loaded %s at "PFX"-"PFX"\n",
+                path, client_libs[idx].start, client_libs[idx].end);
+#ifdef X64
+            request_region_be_heap_reachable(client_libs[idx].start,
+                                             client_libs[idx].end -
+                                             client_libs[idx].start);
+#endif
+            strncpy(client_libs[idx].path, path,
+                    BUFFER_SIZE_ELEMENTS(client_libs[idx].path));
+            NULL_TERMINATE_BUFFER(client_libs[idx].path);
+
+            if (options != NULL) {
+                strncpy(client_libs[idx].options, options,
+                        BUFFER_SIZE_ELEMENTS(client_libs[idx].options));
+                NULL_TERMINATE_BUFFER(client_libs[idx].options);
+            }
+
+            /* We'll look up dr_init and call it in instrument_init */
+        }
+    }
+}
+
+void
+instrument_load_client_libs(void)
+{
+    if (!IS_INTERNAL_STRING_OPTION_EMPTY(client_lib)) {
+        char buf[MAX_LIST_OPTION_LENGTH];
+        char *path;
+
+        string_option_read_lock();
+        strncpy(buf, INTERNAL_OPTION(client_lib), BUFFER_SIZE_ELEMENTS(buf));
+        string_option_read_unlock();
+        NULL_TERMINATE_BUFFER(buf);
+
+        /* We're expecting path;ID;options triples */
+        path = buf;
+        do {
+            char *id = NULL;
+            char *options = NULL;
+            char *next_path = NULL;
+
+            id = strstr(path, ";");
+            if (id != NULL) {
+                id[0] = '\0';
+                id++;
+
+                options = strstr(id, ";");
+                if (options != NULL) {
+                    options[0] = '\0';
+                    options++;
+
+                    next_path = strstr(options, ";");
+                    if (next_path != NULL) {
+                        next_path[0] = '\0';
+                        next_path++;
+                    }
+                }
+            }
+
+            add_client_lib(path, id, options);
+            path = next_path;
+        } while (path != NULL);
+    }
+}
+
+void
+instrument_init(void)
+{
+    /* Iterate over the client libs and call each dr_init */
+    size_t i;
+    for (i=0; i<num_client_libs; i++) {
+        void (*init)(client_id_t) = (void (*)(client_id_t))
+            (lookup_library_routine(client_libs[i].lib, INSTRUMENT_INIT_NAME));
+        /* Since the user has to register all other events, it
+         * doesn't make sense to provide the -client_lib
+         * option for a module that doesn't export dr_init.
+         */
+        CLIENT_ASSERT(init != NULL,
+                      "client library does not export a dr_init routine");
+        (*init)(client_libs[i].id);
+    }
+
+    /* If the client just registered the module-load event, let's
+     * assume it wants to be informed of *all* modules and tell it
+     * which modules are already loaded.  If the client registers the
+     * event later, it will need to use the module iterator routines
+     * to retrieve currently loaded modules.  We use the dr_module_iterator
+     * exposed to the client to avoid locking issues.
+     */
+    if (module_load_callbacks.num > 0) {
+        dr_module_iterator_t *mi = dr_module_iterator_start();
+        while (dr_module_iterator_hasnext(mi)) {
+            module_data_t *data = dr_module_iterator_next(mi);
+            instrument_module_load(data, true /*already loaded*/);
+            dr_free_module_data(data);
+        }
+        dr_module_iterator_stop(mi);
+    }
+
+    /* We now initialize the 1st thread before coming here, so we can
+     * hand the client a dcontext; so we need to specially generate
+     * the thread init event now.  An alternative is to have
+     * dr_get_global_drcontext(), but that's extra complexity for no
+     * real reason.
+     */
+    if (thread_init_callbacks.num > 0) {
+        instrument_thread_init(get_thread_private_dcontext());
+    }
+}
+
+#ifdef DEBUG
+void
+free_callback_list(callback_list_t *vec)
+{
+    if (vec->callbacks != NULL) {
+        HEAP_ARRAY_FREE(GLOBAL_DCONTEXT, vec->callbacks, callback_t, vec->size,
+                        ACCT_OTHER, UNPROTECTED);
+        vec->callbacks = NULL;
+    }
+    vec->size = 0;
+    vec->num = 0;
+}
+
+void free_all_callback_lists()
+{
+    free_callback_list(&exit_callbacks);
+    free_callback_list(&thread_init_callbacks);
+    free_callback_list(&thread_exit_callbacks);
+#ifdef LINUX
+    free_callback_list(&fork_init_callbacks);
+#endif
+    free_callback_list(&bb_callbacks);
+    free_callback_list(&trace_callbacks);
+#ifdef CUSTOM_TRACES
+    free_callback_list(&end_trace_callbacks);
+#endif
+    free_callback_list(&fragdel_callbacks);
+    free_callback_list(&restore_state_callbacks);
+    free_callback_list(&module_load_callbacks);
+    free_callback_list(&module_unload_callbacks);
+    free_callback_list(&filter_syscall_callbacks);
+    free_callback_list(&pre_syscall_callbacks);
+    free_callback_list(&post_syscall_callbacks);
+#ifdef WINDOWS
+    free_callback_list(&exception_callbacks);
+#else
+    free_callback_list(&signal_callbacks);
+#endif
+#ifdef PROGRAM_SHEPHERDING
+    free_callback_list(&security_violation_callbacks);
+#endif
+}
+#endif /* DEBUG */
+
+void
+instrument_exit(void)
+{
+    DEBUG_DECLARE(size_t i);
+    /* Note - currently own initexit lock when this is called (see PR 227619). */
+
+    call_all(exit_callbacks, int (*)(),
+             /* It seems the compiler is confused if we pass no var args
+              * to the call_all macro.  Bogus NULL arg */
+             NULL);
+
+#ifdef DEBUG
+    /* Unload all client libs and free any allocated storage */
+    for (i=0; i<num_client_libs; i++) {
+        free_callback_list(&client_libs[i].nudge_callbacks);
+        unload_shared_library(client_libs[i].lib);
+    }
+
+    free_all_callback_lists();
+#endif
+
+#ifdef WINDOWS
+    DELETE_LOCK(client_nudge_count_lock);
+#endif
+    DELETE_LOCK(callback_registration_lock);
+}
+
+bool
+is_in_client_lib(app_pc addr)
+{
+    /* NOTE: we use this routine for detecting exceptions in
+     * clients. If we add a callback on that event we'll have to be
+     * sure to deliver it only to the right client.
+     */
+    size_t i;
+    for (i=0; i<num_client_libs; i++) {
+        if ((addr >= (app_pc)client_libs[i].start) &&
+            (addr < client_libs[i].end)) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+void
+dr_register_exit_event(void (*func)(void))
+{
+    add_callback(&exit_callbacks, (void (*)(void))func, true);
+}
+
+bool
+dr_unregister_exit_event(void (*func)(void))
+{
+    return remove_callback(&exit_callbacks, (void (*)(void))func, true);
+}
+
+void
+dr_register_bb_event(dr_emit_flags_t (*func)
+                     (void *drcontext, void *tag, instrlist_t *bb,
+                      bool for_trace, bool translating))
+{
+    if (!INTERNAL_OPTION(code_api)) {
+        CLIENT_ASSERT(false, "asking for bb event when code_api is disabled");
+        return;
+    }
+
+    add_callback(&bb_callbacks, (void (*)(void))func, true);
+}
+
+bool
+dr_unregister_bb_event(dr_emit_flags_t (*func)
+                       (void *drcontext, void *tag, instrlist_t *bb,
+                        bool for_trace, bool translating))
+{
+    return remove_callback(&bb_callbacks, (void (*)(void))func, true);
+}
+
+void
+dr_register_trace_event(dr_emit_flags_t (*func)
+                        (void *drcontext, void *tag, instrlist_t *trace,
+                         bool translating))
+{
+    if (!INTERNAL_OPTION(code_api)) {
+        CLIENT_ASSERT(false, "asking for trace event when code_api is disabled");
+        return;
+    }
+
+    add_callback(&trace_callbacks, (void (*)(void))func, true);
+}
+
+bool
+dr_unregister_trace_event(dr_emit_flags_t (*func)
+                          (void *drcontext, void *tag, instrlist_t *trace,
+                           bool translating))
+{
+    return remove_callback(&trace_callbacks, (void (*)(void))func, true);
+}
+
+#ifdef CUSTOM_TRACES
+void
+dr_register_end_trace_event(dr_custom_trace_action_t (*func)
+                            (void *drcontext, void *tag, void *next_tag))
+{
+    if (!INTERNAL_OPTION(code_api)) {
+        CLIENT_ASSERT(false, "asking for end-trace event when code_api is disabled");
+        return;
+    }
+
+    add_callback(&end_trace_callbacks, (void (*)(void))func, true);
+}
+
+bool
+dr_unregister_end_trace_event(dr_custom_trace_action_t
+                              (*func)(void *drcontext, void *tag, void *next_tag))
+{
+    return remove_callback(&end_trace_callbacks, (void (*)(void))func, true);
+}
+#endif
+
+void
+dr_register_delete_event(void (*func)(void *drcontext, void *tag))
+{
+    if (!INTERNAL_OPTION(code_api)) {
+        CLIENT_ASSERT(false, "asking for delete event when code_api is disabled");
+        return;
+    }
+
+    add_callback(&fragdel_callbacks, (void (*)(void))func, true);
+}
+
+bool
+dr_unregister_delete_event(void (*func)(void *drcontext, void *tag))
+{
+    return remove_callback(&fragdel_callbacks, (void (*)(void))func, true);
+}
+
+void
+dr_register_restore_state_event(void (*func)
+                                (void *drcontext, void *tag, dr_mcontext_t *mcontext,
+                                 bool restore_memory, bool app_code_consistent))
+{
+    if (!INTERNAL_OPTION(code_api)) {
+        CLIENT_ASSERT(false, "asking for restore state event when code_api is disabled");
+        return;
+    }
+
+    add_callback(&restore_state_callbacks, (void (*)(void))func, true);
+}
+
+bool
+dr_unregister_restore_state_event(void (*func)
+                                  (void *drcontext, void *tag, dr_mcontext_t *mcontext,
+                                   bool restore_memory, bool app_code_consistent))
+{
+    return remove_callback(&restore_state_callbacks, (void (*)(void))func, true);
+}
+
+void
+dr_register_thread_init_event(void (*func)(void *drcontext))
+{
+    add_callback(&thread_init_callbacks, (void (*)(void))func, true);
+}
+
+bool
+dr_unregister_thread_init_event(void (*func)(void *drcontext))
+{
+    return remove_callback(&thread_init_callbacks, (void (*)(void))func, true);
+}
+
+void
+dr_register_thread_exit_event(void (*func)(void *drcontext))
+{
+    add_callback(&thread_exit_callbacks, (void (*)(void))func, true);
+}
+
+bool
+dr_unregister_thread_exit_event(void (*func)(void *drcontext))
+{
+    return remove_callback(&thread_exit_callbacks, (void (*)(void))func, true);
+}
+
+#ifdef LINUX
+void
+dr_register_fork_init_event(void (*func)(void *drcontext))
+{
+    add_callback(&fork_init_callbacks, (void (*)(void))func, true);
+}
+
+bool
+dr_unregister_fork_init_event(void (*func)(void *drcontext))
+{
+    return remove_callback(&fork_init_callbacks, (void (*)(void))func, true);
+}    
+#endif
+
+void
+dr_register_module_load_event(void (*func)(void *drcontext, const module_data_t *info,
+                                           bool loaded))
+{
+    add_callback(&module_load_callbacks, (void (*)(void))func, true);
+}
+
+bool
+dr_unregister_module_load_event(void (*func)(void *drcontext, const module_data_t *info,
+                                             bool loaded))
+{
+    return remove_callback(&module_load_callbacks, (void (*)(void))func, true);
+}
+
+void
+dr_register_module_unload_event(void (*func)(void *drcontext,
+                                             const module_data_t *info))
+{
+    add_callback(&module_unload_callbacks, (void (*)(void))func, true);
+}
+
+bool
+dr_unregister_module_unload_event(void (*func)(void *drcontext,
+                                               const module_data_t *info))
+{
+    return remove_callback(&module_unload_callbacks, (void (*)(void))func, true);
+}
+
+#ifdef WINDOWS
+void
+dr_register_exception_event(void (*func)(void *drcontext, dr_exception_t *excpt))
+{
+    add_callback(&exception_callbacks, (void (*)(void))func, true);
+}
+
+bool
+dr_unregister_exception_event(void (*func)(void *drcontext, dr_exception_t *excpt))
+{
+    return remove_callback(&exception_callbacks, (void (*)(void))func, true);
+}
+#else
+void
+dr_register_signal_event(dr_signal_action_t (*func)
+                         (void *drcontext, dr_siginfo_t *siginfo))
+{
+    add_callback(&signal_callbacks, (void (*)(void))func, true);
+}
+
+bool
+dr_unregister_signal_event(dr_signal_action_t (*func)
+                           (void *drcontext, dr_siginfo_t *siginfo))
+{
+    return remove_callback(&signal_callbacks, (void (*)(void))func, true);
+}
+#endif /* WINDOWS */
+
+void
+dr_register_filter_syscall_event(bool (*func)(void *drcontext, int sysnum))
+{
+    add_callback(&filter_syscall_callbacks, (void (*)(void))func, true);
+}
+
+bool
+dr_unregister_filter_syscall_event(bool (*func)(void *drcontext, int sysnum))
+{
+    return remove_callback(&filter_syscall_callbacks, (void (*)(void))func, true);
+}
+
+void
+dr_register_pre_syscall_event(bool (*func)(void *drcontext, int sysnum))
+{
+    add_callback(&pre_syscall_callbacks, (void (*)(void))func, true);
+}
+
+bool
+dr_unregister_pre_syscall_event(bool (*func)(void *drcontext, int sysnum))
+{
+    return remove_callback(&pre_syscall_callbacks, (void (*)(void))func, true);
+}
+
+void
+dr_register_post_syscall_event(void (*func)(void *drcontext, int sysnum))
+{
+    add_callback(&post_syscall_callbacks, (void (*)(void))func, true);
+}
+
+bool
+dr_unregister_post_syscall_event(void (*func)(void *drcontext, int sysnum))
+{
+    return remove_callback(&post_syscall_callbacks, (void (*)(void))func, true);
+}
+
+#ifdef PROGRAM_SHEPHERDING
+void
+dr_register_security_event(void (*func)(void *drcontext, void *source_tag,
+                                        app_pc source_pc, app_pc target_pc,
+                                        dr_security_violation_type_t violation,
+                                        dr_mcontext_t *mcontext,
+                                        dr_security_violation_action_t *action))
+{
+    add_callback(&security_violation_callbacks, (void (*)(void))func, true);
+}           
+
+bool
+dr_unregister_security_event(void (*func)(void *drcontext, void *source_tag,
+                                          app_pc source_pc, app_pc target_pc,
+                                          dr_security_violation_type_t violation,
+                                          dr_mcontext_t *mcontext,
+                                          dr_security_violation_action_t *action))
+{
+    return remove_callback(&security_violation_callbacks, (void (*)(void))func, true);
+}           
+#endif
+
+#ifdef WINDOWS
+void
+dr_register_nudge_event(void (*func)(void *drcontext, uint64 argument), client_id_t id)
+{
+    size_t i;
+    for (i=0; i<num_client_libs; i++) {
+        if (client_libs[i].id == id) {
+            add_callback(&client_libs[i].nudge_callbacks, (void (*)(void))func,
+                         /* the nudge callback list is stored on the heap, so
+                          * we don't need to unprotect the .data section when
+                          * we update the list */
+                         false);
+            return;
+        }
+    }
+
+    CLIENT_ASSERT(false, "dr_register_nudge_event: invalid client ID");
+}
+
+bool
+dr_unregister_nudge_event(void (*func)(void *drcontext, uint64 argument), client_id_t id)
+{
+    size_t i;
+    for (i=0; i<num_client_libs; i++) {
+        if (client_libs[i].id == id) {
+            return remove_callback(&client_libs[i].nudge_callbacks, (void (*)(void))func,
+                                   /* the nudge callback list is stored on the heap, so
+                                    * we don't need to unprotect the .data section when
+                                    * we update the list */
+                                   false);
+        }
+    }
+
+    CLIENT_ASSERT(false, "dr_unregister_nudge_event: invalid client ID");
+    return false;
+}
+
+bool
+dr_nudge_client(client_id_t client_id, uint64 argument)
+{
+    size_t i;
+
+    for (i=0; i<num_client_libs; i++) {
+        if (client_libs[i].id == client_id) {
+            if (client_libs[i].nudge_callbacks.num == 0) {
+                CLIENT_ASSERT(false, "dr_nudge_client: no nudge handler registered");
+                return false;
+            }
+            return nudge_internal(NUDGE_GENERIC(client), argument, client_id);
+        }
+    }
+
+    return false;
+}
+#endif /* WINDOWS */
+
+void
+instrument_thread_init(dcontext_t *dcontext)
+{
+    /* Note that we're called twice: once prior to instrument_init()
+     * (PR 216936) to set up the dcontext client field, and once
+     * after instrument_init() to call the client event.
+     */
+    if (dcontext->client_data == NULL) {
+        dcontext->client_data = HEAP_TYPE_ALLOC(dcontext, client_data_t,
+                                                ACCT_OTHER, UNPROTECTED);
+        memset(dcontext->client_data, 0x0, sizeof(client_data_t));
+
+#ifdef CLIENT_SIDELINE
+        ASSIGN_INIT_LOCK_FREE(dcontext->client_data->sideline_mutex, sideline_mutex);
+        ASSIGN_INIT_LOCK_FREE(dcontext->client_data->sideline_heap_lock,
+                              sideline_heap_lock);
+#endif
+    }
+
+    call_all(thread_init_callbacks, int (*)(void *), (void *)dcontext);
+}
+
+#ifdef LINUX
+void
+instrument_fork_init(dcontext_t *dcontext)
+{
+    call_all(fork_init_callbacks, int (*)(void *), (void *)dcontext);
+}
+#endif
+
+void
+instrument_thread_exit(dcontext_t *dcontext)
+{
+    client_todo_list_t *todo;
+    client_flush_req_t *flush;
+
+    /* Note - currently own initexit lock when this is called (see PR 227619). */
+    call_all(thread_exit_callbacks, int (*)(void *), (void *)dcontext);
+
+#ifdef CLIENT_SIDELINE
+    DELETE_LOCK(dcontext->client_data->sideline_mutex);
+    DELETE_LOCK(dcontext->client_data->sideline_heap_lock);
+#endif
+
+    /* could be heap space allocated for the todo list */
+    todo = dcontext->client_data->to_do;
+    while (todo != NULL) {
+        client_todo_list_t *next_todo = todo->next;
+        if (todo->ilist != NULL) {
+            instrlist_clear_and_destroy(dcontext, todo->ilist);
+        }
+        HEAP_TYPE_FREE(dcontext, todo, client_todo_list_t, ACCT_OTHER, UNPROTECTED);
+        todo = next_todo;
+    }
+
+    /* could be heap space allocated for the flush list */
+    flush = dcontext->client_data->flush_list;
+    while (flush != NULL) {
+        client_flush_req_t *next_flush = flush->next;
+        HEAP_TYPE_FREE(dcontext, flush, client_flush_req_t, ACCT_OTHER, UNPROTECTED);
+        flush = next_flush;
+    }        
+
+    HEAP_TYPE_FREE(dcontext, dcontext->client_data, client_data_t,
+                   ACCT_OTHER, UNPROTECTED);
+    dcontext->client_data = NULL; /* for mutex_wait_contended_lock() */
+}
+
+bool
+dr_bb_hook_exists(void)
+{
+    return (bb_callbacks.num > 0);
+}
+
+bool
+dr_trace_hook_exists(void)
+{
+    return (trace_callbacks.num > 0);
+}
+
+bool
+dr_fragment_deleted_hook_exists(void)
+{
+    return (fragdel_callbacks.num > 0);
+}
+
+bool
+dr_end_trace_hook_exists(void)
+{
+    return (end_trace_callbacks.num > 0);
+}
+
+static bool
+hide_tag_from_client(app_pc tag)
+{
+#ifdef WINDOWS
+    /* Case 10009: Basic blocks that consist of a single jump into the
+     * interception buffer should be obscured from clients.  Clients
+     * will see the displaced code, so we'll provide the address of this 
+     * block if the client asks for the address of the displaced code.
+     *
+     * Note that we assume the jump is the first instruction in the
+     * BB for any blocks that jump to the interception buffer.
+     */
+    if (is_intercepted_app_pc(tag, NULL) ||
+        /* Landing pads that exist between hook points and the trampolines
+         * shouldn't be seen by the client too.  PR 250294.
+         */
+        vmvector_overlap(landing_pad_areas, tag, tag + 1) ||
+        /* PR 219351: if we lose control on a callback and get it back on
+         * one of our syscall trampolines, we'll appear at the jmp out of
+         * the interception buffer to the int/sysenter instruction.  The
+         * problem is that our syscall trampolines, unlike our other
+         * intercepted code, are hooked earlier than the real action point
+         * and we have displaced app code at the start of the interception
+         * buffer: we hook at the wrapper entrance and return w/ a jmp to
+         * the sysenter/int instr.  When creating bbs at the start we hack
+         * it to make it look like there is no hook.  But on retaking control
+         * we end up w/ this jmp out that won't be solved w/ our normal
+         * mechanism for other hook jmp-outs: so we just suppress and the
+         * client next sees the post-syscall bb.  It already saw a gap.
+         */
+        is_syscall_trampoline(tag))
+        return true;
+#endif
+    return false;
+}
+
+#ifdef DEBUG
+/* PR 214962: client must set translation fields */
+static void
+check_ilist_translations(instrlist_t *ilist)
+{
+    /* Ensure client set the translation field for all non-meta
+     * instrs, even if it didn't return DR_EMIT_STORE_TRANSLATIONS
+     * (since we may decide ourselves to store)
+     */
+    instr_t *in;
+    for (in = instrlist_first(ilist); in != NULL; in = instr_get_next(in)) {
+        if (instr_ok_to_mangle(in)) {
+            DOLOG(LOG_INTERP, 1, {
+                if (instr_get_translation(in) == NULL)
+                    loginst(get_thread_private_dcontext(), 1, in, "translation is NULL");
+            });
+            CLIENT_ASSERT(instr_get_translation(in) != NULL,
+                          "translation field must be set for every non-meta instruction");
+        } else {
+            /* The meta instr could indeed not affect app state, but
+             * better I think to assert and make them put in an
+             * empty restore event callback in that case. */
+            DOLOG(LOG_INTERP, 1, {
+                if (instr_get_translation(in) != NULL &&
+                    !instr_is_our_mangling(in) &&
+                    restore_state_callbacks.num == 0)
+                    loginst(get_thread_private_dcontext(), 1, in, "translation != NULL");
+            });
+            CLIENT_ASSERT(instr_get_translation(in) == NULL ||
+                          instr_is_our_mangling(in) ||
+                          restore_state_callbacks.num > 0,
+                          /* FIXME: if multiple clients, we need to check that this
+                           * particular client has the callback: but we have
+                           * no way to do that other than looking at library
+                           * bounds...punting for now */
+                          "a meta instr should not have its translation field "
+                          "set without also having a restore_state callback");
+        }
+    }
+}
+#endif
+
+/* Returns true if the bb hook is called */
+bool
+instrument_basic_block(dcontext_t *dcontext, app_pc tag, instrlist_t *bb,
+                       bool for_trace, bool translating, dr_emit_flags_t *emitflags)
+{
+    dr_emit_flags_t ret = DR_EMIT_DEFAULT;
+    
+    /* return false if no BB hooks are registered */
+    if (bb_callbacks.num == 0)
+        return false;
+
+    if (hide_tag_from_client(tag))
+        return false;
+
+    /* do not expand or up-decode the instrlist, client gets to choose
+     * whether and how to do that
+     */
+
+#ifdef DEBUG
+    LOG(THREAD, LOG_INTERP, 3, "\ninstrument_basic_block ******************\n");
+    LOG(THREAD, LOG_INTERP, 3, "\nbefore instrumentation:\n");
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_INTERP) != 0)
+        instrlist_disassemble(dcontext, tag, bb, THREAD);
+#endif
+
+    /* Note - currently we are couldbelinking and hold the
+     * bb_building lock when this is called (see PR 227619).
+     */
+    /* We or together the return values */
+    call_all_ret(ret, |=, , bb_callbacks,
+                 int (*) (void *, void *, instrlist_t *, bool, bool),
+                 (void *)dcontext, (void *)tag, bb, for_trace, translating);
+    if (emitflags != NULL)
+        *emitflags = ret;
+    DODEBUG({ check_ilist_translations(bb); });
+
+#ifdef DEBUG
+    LOG(THREAD, LOG_INTERP, 3, "\nafter instrumentation:\n");
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_INTERP) != 0)
+        instrlist_disassemble(dcontext, tag, bb, THREAD);
+#endif
+
+    return true;
+}
+
+/* Give the user the completely mangled and optimized trace just prior
+ * to emitting into code cache, user gets final crack at it
+ */
+dr_emit_flags_t
+instrument_trace(dcontext_t *dcontext, app_pc tag, instrlist_t *trace,
+                 bool translating)
+{
+    dr_emit_flags_t ret = DR_EMIT_DEFAULT;
+#ifdef UNSUPPORTED_API
+    instr_t *instr;
+#endif
+    if (trace_callbacks.num == 0)
+        return DR_EMIT_DEFAULT;
+
+    /* do not expand or up-decode the instrlist, client gets to choose
+     * whether and how to do that
+     */
+
+#ifdef DEBUG
+    LOG(THREAD, LOG_INTERP, 3, "\ninstrument_trace ******************\n");
+    LOG(THREAD, LOG_INTERP, 3, "\nbefore instrumentation:\n");
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_INTERP) != 0)
+        instrlist_disassemble(dcontext, tag, trace, THREAD);
+#endif
+
+    /* We always pass Level 3 instrs to the client, since we no longer
+     * expose the expansion routines.
+     */
+#ifdef UNSUPPORTED_API
+    for (instr = instrlist_first_expanded(dcontext, trace);
+         instr != NULL;
+         instr = instr_get_next_expanded(dcontext, trace, instr)) {
+        instr_decode(dcontext, instr);
+    }
+    /* ASSUMPTION: all ctis are already at Level 3, so we don't have
+     * to do a separate pass to fix up intra-list targets like
+     * instrlist_decode_cti() does
+     */
+#endif
+
+    /* We or together the return values */
+    call_all_ret(ret, |=, , trace_callbacks,
+                 int (*)(void *, void *, instrlist_t *, bool),
+                 (void *)dcontext, (void *)tag, trace, translating);
+
+    DODEBUG({ check_ilist_translations(trace); });
+
+#ifdef DEBUG
+    LOG(THREAD, LOG_INTERP, 3, "\nafter instrumentation:\n");
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_INTERP) != 0)
+        instrlist_disassemble(dcontext, tag, trace, THREAD);
+#endif
+
+    return ret;
+}
+
+/* Notify user when a fragment is deleted from the cache
+ * FIXME PR 242544: how does user know whether this is a shadowed copy or the
+ * real thing?  The user might free memory that shouldn't be freed!
+ */
+void
+instrument_fragment_deleted(dcontext_t *dcontext, app_pc tag, uint flags)
+{
+    if (fragdel_callbacks.num == 0)
+        return;
+
+#ifdef WINDOWS
+    /* Case 10009: We don't call the basic block hook for blocks that
+     * are jumps to the interception buffer, so we'll hide them here
+     * as well.
+     */
+    if (!TEST(FRAG_IS_TRACE, flags) && hide_tag_from_client(tag))
+        return;
+#endif
+
+    /* PR 243008: we don't expose GLOBAL_DCONTEXT, so change to NULL.
+     * Our comments warn the user about this.
+     */
+    if (dcontext == GLOBAL_DCONTEXT)
+        dcontext = NULL;
+
+    call_all(fragdel_callbacks, int (*)(void *, void *),
+             (void *)dcontext, (void *)tag);
+}
+
+void
+instrument_restore_state(dcontext_t *dcontext, app_pc tag, dr_mcontext_t *mc,
+                         bool restore_memory, bool app_code_consistent)
+{
+    if (restore_state_callbacks.num == 0)
+        return;
+
+    call_all(restore_state_callbacks,
+             int (*)(void *, void *, dr_mcontext_t *, bool, bool),
+             (void *)dcontext, (void *)tag, mc, restore_memory, app_code_consistent);
+}
+
+#ifdef CUSTOM_TRACES
+/* Ask whether to end trace prior to adding next_tag fragment.
+ * Return values:
+ *   CUSTOM_TRACE_DR_DECIDES = use standard termination criteria
+ *   CUSTOM_TRACE_END_NOW    = end trace
+ *   CUSTOM_TRACE_CONTINUE   = do not end trace
+ */
+dr_custom_trace_action_t 
+instrument_end_trace(dcontext_t *dcontext, app_pc trace_tag, app_pc next_tag)
+{
+    dr_custom_trace_action_t ret = CUSTOM_TRACE_DR_DECIDES;
+
+    if (end_trace_callbacks.num == 0)
+        return ret;
+
+    /* Highest priority callback decides how to end the trace (see
+     * call_all_ret implementation)
+     */
+    call_all_ret(ret, =, , end_trace_callbacks, int (*)(void *, void *, void *),
+                 (void *)dcontext, (void *)trace_tag, (void *)next_tag);
+
+    return ret;
+}
+#endif
+
+
+static module_data_t *
+create_and_initialize_module_data(app_pc start, app_pc end, app_pc entry_point,
+                                  uint flags, const module_names_t *names
+#ifdef WINDOWS
+                                  , version_number_t file_version,
+                                  version_number_t product_version,
+                                  uint checksum, uint timestamp,
+                                  size_t mod_size
+#endif
+                                  )
+{
+    module_data_t *copy = (module_data_t *)
+        HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, module_data_t, ACCT_OTHER, UNPROTECTED);
+    memset(copy, 0, sizeof(module_data_t));
+
+    copy->start = start;
+    copy->end = end;
+    copy->entry_point = entry_point;
+    copy->flags = flags;
+
+    if (names->module_name != NULL)
+        copy->names.module_name = dr_strdup(names->module_name HEAPACCT(ACCT_OTHER));
+    if (names->file_name != NULL)
+        copy->names.file_name = dr_strdup(names->file_name HEAPACCT(ACCT_OTHER));
+#ifdef WINDOWS
+    if (names->exe_name != NULL)
+        copy->names.exe_name = dr_strdup(names->exe_name HEAPACCT(ACCT_OTHER));
+    if (names->rsrc_name != NULL)
+        copy->names.rsrc_name = dr_strdup(names->rsrc_name HEAPACCT(ACCT_OTHER));
+
+    copy->file_version = file_version;
+    copy->product_version = product_version;
+    copy->checksum = checksum;
+    copy->timestamp = timestamp;
+    copy->module_internal_size = mod_size;
+#endif
+    return copy;
+}
+
+module_data_t *
+copy_module_area_to_module_data(const module_area_t *area)
+{
+    if (area == NULL)
+        return NULL;
+
+    return create_and_initialize_module_data(area->start, area->end, area->entry_point,
+                                             0, &area->names
+#ifdef WINDOWS
+                                             , area->os_data.file_version,
+                                             area->os_data.product_version,
+                                             area->os_data.checksum,
+                                             area->os_data.timestamp,
+                                             area->os_data.module_internal_size
+#endif
+                                             );
+}
+
+DR_API
+/* Makes a copy of a module_data_t for returning to the client.  We return a copy so
+ * we don't have to hold the module areas list lock while in the client (xref PR 225020).
+ * Note - dr_data is allowed to be NULL. */
+module_data_t *
+dr_copy_module_data(const module_data_t *data)
+{
+    if (data == NULL)
+        return NULL;
+
+    return create_and_initialize_module_data(data->start, data->end, data->entry_point,
+                                             0, &data->names
+#ifdef WINDOWS
+                                             , data->file_version,
+                                             data->product_version,
+                                             data->checksum, data->timestamp,
+                                             data->module_internal_size
+#endif
+                                             );
+}
+
+DR_API
+/* Used to free a module_data_t created by dr_copy_module_data() */
+void
+dr_free_module_data(module_data_t *data)
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+
+    if (data == NULL)
+        return;
+
+    if (dcontext != NULL && data == dcontext->client_data->no_delete_mod_data) {
+        CLIENT_ASSERT(false, "dr_free_module_data: don\'t free module_data passed to "
+                      "the image load or image unload event callbacks.");
+        return;
+    }
+
+    free_module_names(&data->names HEAPACCT(ACCT_OTHER));
+
+    HEAP_TYPE_FREE(GLOBAL_DCONTEXT, data, module_data_t, ACCT_OTHER, UNPROTECTED);
+}
+
+/* Notify user when a module is loaded */
+void
+instrument_module_load(module_data_t *data, bool previously_loaded)
+{
+    /* Note - during DR initialization this routine is called before we've set up a
+     * dcontext for the main thread and before we've called instrument_init.  It's okay
+     * since there's no way a callback will be registered and we'll return immediately. */
+    dcontext_t *dcontext;
+
+    if (module_load_callbacks.num == 0)
+        return;
+
+    dcontext = get_thread_private_dcontext();
+
+    /* client shouldn't delete this */
+    dcontext->client_data->no_delete_mod_data = data;
+
+    call_all(module_load_callbacks, int (*)(void *, module_data_t *, bool),
+             (void *)dcontext, data, previously_loaded);
+
+    dcontext->client_data->no_delete_mod_data = NULL;
+}    
+
+/* Notify user when a module is unloaded */
+void
+instrument_module_unload(module_data_t *data)
+{
+    dcontext_t *dcontext;
+
+    if (module_unload_callbacks.num == 0)
+        return;
+
+    dcontext = get_thread_private_dcontext();
+    /* client shouldn't delete this */
+    dcontext->client_data->no_delete_mod_data = data;
+
+    call_all(module_unload_callbacks, int (*)(void *, module_data_t *),
+             (void *)dcontext, data);
+
+    dcontext->client_data->no_delete_mod_data = NULL;
+}
+
+/* returns whether this sysnum should be intercepted */
+bool
+instrument_filter_syscall(dcontext_t *dcontext, int sysnum)
+{
+    bool ret = false;
+    /* if client does not filter then we don't intercept anything */
+    if (filter_syscall_callbacks.num == 0)
+        return ret;
+    /* if any client wants to intercept, then we intercept */
+    call_all_ret(ret, =, || ret, filter_syscall_callbacks, bool (*)(void *, int),
+                 (void *)dcontext, sysnum);
+    return ret;
+}
+
+/* returns whether this syscall should execute */
+bool
+instrument_pre_syscall(dcontext_t *dcontext, int sysnum)
+{
+    bool exec = true;
+    dcontext->client_data->in_pre_syscall = true;
+    /* clear flag from dr_syscall_invoke_another() */
+    dcontext->client_data->invoke_another_syscall = false;
+    if (pre_syscall_callbacks.num > 0) {
+        /* skip syscall if any client wants to skip it */
+        call_all_ret(exec, = exec &&, , pre_syscall_callbacks,
+                     bool (*)(void *, int), (void *)dcontext, sysnum);
+    }
+    dcontext->client_data->in_pre_syscall = false;
+    return exec;
+}
+
+void
+instrument_post_syscall(dcontext_t *dcontext, int sysnum)
+{
+    if (post_syscall_callbacks.num == 0)
+        return;
+    dcontext->client_data->in_post_syscall = true;
+    call_all(post_syscall_callbacks, int (*)(void *, int),
+             (void *)dcontext, sysnum);
+    dcontext->client_data->in_post_syscall = false;
+}
+
+bool
+instrument_invoke_another_syscall(dcontext_t *dcontext)
+{
+    return dcontext->client_data->invoke_another_syscall;
+}
+
+#ifdef WINDOWS
+/* Notify user of exceptions.  Note: not called for RaiseException */
+void
+instrument_exception(dcontext_t *dcontext, dr_exception_t *exception)
+{
+    call_all(exception_callbacks, int (*)(void *, dr_exception_t *),
+             (void *)dcontext, exception);
+}
+#else
+dr_signal_action_t
+instrument_signal(dcontext_t *dcontext, dr_siginfo_t *siginfo)
+{
+    dr_signal_action_t ret = DR_SIGNAL_DELIVER;
+    /* Highest priority callback decides what to do with signal.
+     * If we get rid of DR_SIGNAL_BYPASS we could change to a bool and
+     * then only deliver to app if nobody suppresses.
+     */
+    call_all_ret(ret, =, , signal_callbacks,
+                 dr_signal_action_t (*)(void *, dr_siginfo_t *),
+                 (void *)dcontext, siginfo);
+    return ret;
+}
+
+bool
+dr_signal_hook_exists(void)
+{
+    return (signal_callbacks.num > 0);
+}
+#endif /* WINDOWS */
+
+#ifdef PROGRAM_SHEPHERDING
+/* Notify user when a security violation is detected */
+void
+instrument_security_violation(dcontext_t *dcontext, app_pc target_pc,
+                              security_violation_t violation, action_type_t *action)
+{
+    dr_security_violation_type_t dr_violation;
+    dr_security_violation_action_t dr_action, dr_action_original;
+    app_pc source_pc = NULL;
+    fragment_t *last;
+
+    if (security_violation_callbacks.num == 0)
+        return;
+
+    /* FIXME - the source_tag, source_pc, and context can all be incorrect if the
+     * violation ends up occuring in the middle of a bb we're building.  See case
+     * 7380 which we should fix in interp.c.
+     */
+    
+    /* Obtain the source addr to pass to the client.  xref case 285 --
+     * we're using the more heavy-weight solution 2) here, but that
+     * should be okay since we already have the overhead of calling
+     * into the client. */
+    last = dcontext->last_fragment;
+    if (!TEST(FRAG_FAKE, last->flags)) {
+        cache_pc pc = EXIT_CTI_PC(last, dcontext->last_exit);
+        source_pc = recreate_app_pc(dcontext, pc, last);
+    }
+    /* FIXME - set pc field of dr_mcontext_t.  We'll probably want it
+     * for thread start and possibly apc/callback events as well.
+     */
+
+    switch (violation) {
+      case STACK_EXECUTION_VIOLATION:
+          dr_violation = DR_RCO_STACK_VIOLATION;
+          break;
+      case HEAP_EXECUTION_VIOLATION:
+          dr_violation = DR_RCO_HEAP_VIOLATION;
+          break;
+      case RETURN_TARGET_VIOLATION:
+          dr_violation = DR_RCT_RETURN_VIOLATION;
+          break;
+      case RETURN_DIRECT_RCT_VIOLATION:
+          ASSERT(false); /* Not a client fault, should be NOT_REACHED(). */
+          dr_violation = DR_UNKNOWN_VIOLATION;
+          break;
+      case INDIRECT_CALL_RCT_VIOLATION:
+          dr_violation = DR_RCT_INDIRECT_CALL_VIOLATION;
+          break;
+      case INDIRECT_JUMP_RCT_VIOLATION:
+          dr_violation = DR_RCT_INDIRECT_JUMP_VIOLATION;
+          break;
+      default:
+          ASSERT(false); /* Not a client fault, should be NOT_REACHED(). */
+          dr_violation = DR_UNKNOWN_VIOLATION;
+          break;
+    }
+
+    switch (*action) {
+      case ACTION_TERMINATE_PROCESS:
+          dr_action = DR_VIOLATION_ACTION_KILL_PROCESS;
+          break;
+      case ACTION_CONTINUE:
+          dr_action = DR_VIOLATION_ACTION_CONTINUE;
+          break;
+      case ACTION_TERMINATE_THREAD:
+          dr_action = DR_VIOLATION_ACTION_KILL_THREAD;
+          break;
+      case ACTION_THROW_EXCEPTION:
+          dr_action = DR_VIOLATION_ACTION_THROW_EXCEPTION;
+          break;
+      default:
+          ASSERT(false); /* Not a client fault, should be NOT_REACHED(). */
+          dr_action = DR_VIOLATION_ACTION_CONTINUE;
+          break;
+    }
+    dr_action_original = dr_action;
+
+    /* NOTE - last->tag should be valid here (even if the frag is fake since the
+     * coarse wrappers set the tag).  FIXME - for traces we really want the bb tag not
+     * the trace tag, should get that. Of course the only real reason we pass source
+     * tag is because we can't always give a valid source_pc. */
+
+    /* Note that the last registered function gets the final crack at
+     * changing the action.
+     */
+    call_all(security_violation_callbacks,
+             int (*)(void *, void *, app_pc, app_pc, dr_security_violation_type_t,
+                      dr_mcontext_t *, dr_security_violation_action_t *),
+             (void *)dcontext, last->tag, source_pc, target_pc,
+             dr_violation, get_mcontext(dcontext), &dr_action);
+
+    if (dr_action != dr_action_original) {
+        switch(dr_action) {
+        case DR_VIOLATION_ACTION_KILL_PROCESS:
+            *action = ACTION_TERMINATE_PROCESS;
+            break;
+        case DR_VIOLATION_ACTION_KILL_THREAD:
+            *action = ACTION_TERMINATE_THREAD;
+            break;
+        case DR_VIOLATION_ACTION_THROW_EXCEPTION:
+            *action = ACTION_THROW_EXCEPTION;
+            break;
+        case DR_VIOLATION_ACTION_CONTINUE_CHANGED_CONTEXT:
+            /* FIXME - not safe to implement till case 7380 is fixed. */
+            CLIENT_ASSERT(false, "action DR_VIOLATION_ACTION_CONTINUE_CHANGED_CONTEXT "
+                          "not yet supported.");
+            /* note - no break, fall through */
+        case DR_VIOLATION_ACTION_CONTINUE:
+            *action = ACTION_CONTINUE;
+            break;
+        default:
+            CLIENT_ASSERT(false, "Security violation event callback returned invalid "
+                          "action value.");
+        }
+    }
+}
+#endif
+
+#ifdef WINDOWS
+/* Notify the client of a nudge. */
+void
+instrument_nudge(dcontext_t *dcontext, client_id_t id, uint64 arg)
+{
+    size_t i;
+
+    ASSERT(dcontext != NULL && dcontext != GLOBAL_DCONTEXT &&
+           dcontext == get_thread_private_dcontext());
+    /* synch_with_all_threads and flush API assume that client nudge threads
+     * hold no dr locks and are !couldbelinking while in client lib code */
+    ASSERT_OWN_NO_LOCKS();
+    ASSERT(!is_couldbelinking(dcontext));
+
+    /* find the client the nudge is intended for */
+    for (i=0; i<num_client_libs; i++) {
+        if (client_libs[i].id == id) {
+            break;
+        }
+    }
+
+    if (i == num_client_libs || client_libs[i].nudge_callbacks.num == 0)
+        return;
+
+    /* count the number of nudge events so we can make sure they're
+     * all finished before exiting
+     */
+    mutex_lock(&client_nudge_count_lock);
+    if (block_client_owned_threads) {
+        /* FIXME - would be nice if there was a way to let the external agent know that
+         * the nudge event wasn't delivered (but this only happens when the process
+         * is detaching or exiting). */
+        mutex_unlock(&client_nudge_count_lock);
+        return;
+    }
+
+    /* atomic to avoid locking around the dec */
+    ATOMIC_INC(int, num_client_owned_threads);
+    mutex_unlock(&client_nudge_count_lock);
+
+    /* We need to mark this as a client controlled thread for synch_with_all_threads
+     * and otherwise treat it as native.  Xref PR 230836 on what to do if this
+     * thread hits native_exec_syscalls hooks. */
+    dcontext->client_data->is_client_thread = true;
+    dcontext->thread_record->under_dynamo_control = false;
+
+    call_all(client_libs[i].nudge_callbacks, int (*)(void *, uint64), 
+             (void *)dcontext, arg);
+
+    dcontext->thread_record->under_dynamo_control = true;
+    dcontext->client_data->is_client_thread = false;
+
+    ATOMIC_DEC(int, num_client_owned_threads);
+}
+
+/* wait for all nudges to finish */
+void
+wait_for_outstanding_nudges()
+{
+    /* block any new nudge threads from starting */
+    mutex_lock(&client_nudge_count_lock);
+    SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+    block_client_owned_threads = true;
+    SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+
+    DOLOG(1, LOG_TOP, {
+        if (num_client_owned_threads > 0) {
+            LOG(GLOBAL, LOG_TOP, 1,
+                "Waiting for %d nudges to finish - app is about to kill all threads "
+                "except the current one./n", num_client_owned_threads);
+        }
+    });
+
+    while (num_client_owned_threads > 0) {
+        /* yield with lock released to allow nudges to finish */
+        mutex_unlock(&client_nudge_count_lock);
+        dr_thread_yield();
+        mutex_lock(&client_nudge_count_lock);
+    }
+    mutex_unlock(&client_nudge_count_lock);
+}
+#endif /* WINDOWS */
+
+/****************************************************************************/
+/* EXPORTED ROUTINES */
+
+DR_API 
+/* Creates a DR context that can be used in a standalone program.
+ * WARNING: this context cannot be used as the drcontext for a thread 
+ * running under DR control!  It is only for standalone programs that 
+ * wish to use DR as a library of disassembly, etc. routines.
+ */
+void *
+dr_standalone_init(void)
+{
+    dcontext_t *dcontext = standalone_init();
+    return (void *) dcontext;
+}
+
+DR_API
+/* Aborts the process immediately */
+void
+dr_abort(void)
+{
+    os_terminate(NULL, TERMINATE_PROCESS);
+}
+
+DR_API
+/* Returns true if all \DynamoRIO caches are thread private. */
+bool
+dr_using_all_private_caches(void)
+{
+    return !SHARED_FRAGMENTS_ENABLED();
+}
+
+DR_API
+/* Returns the option string passed along with a client path via DR's
+ * -client_lib option.
+ */
+const char *
+dr_get_options(client_id_t id)
+{
+    size_t i;
+    for (i=0; i<num_client_libs; i++) {
+        if (client_libs[i].id == id) {
+            return client_libs[i].options;
+        }
+    }
+
+    CLIENT_ASSERT(false, "dr_get_options(): invalid client id");
+    return NULL;
+}
+
+DR_API 
+/* Returns the path to the client library.  Client must pass its ID */
+const char *
+dr_get_client_path(client_id_t id)
+{
+    size_t i;
+    for (i=0; i<num_client_libs; i++) {
+        if (client_libs[i].id == id) {
+            return client_libs[i].path;
+        }
+    }
+
+    CLIENT_ASSERT(false, "dr_get_client_path(): invalid client id");
+    return NULL;
+}
+
+DR_API const char *
+dr_get_application_name(void)
+{
+#ifdef LINUX
+    return get_application_name();
+#else
+    return get_application_short_unqualified_name();
+#endif
+}
+
+DR_API process_id_t
+dr_get_process_id(void)
+{
+    /* FIXME PR 226085: internally we need to reconcile these types; for now just
+     * getting the client-exposed type set so we can change it internally
+     * w/ no API changes
+     */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    return (process_id_t) get_process_id();
+}
+
+#ifdef WINDOWS
+DR_API
+bool
+dr_is_wow64(void)
+{
+    return is_wow64_process(NT_CURRENT_PROCESS);
+}
+#endif
+
+DR_API
+/* Retrieves the current time */
+void
+dr_get_time(dr_time_t *time)
+{
+#ifdef LINUX
+    CLIENT_ASSERT(false, "dr_get_time NYI on linux");
+#else
+    SYSTEMTIME st;
+    query_system_time(&st);
+    time->year = st.wYear;
+    time->month = st.wMonth;
+    time->day_of_week = st.wDayOfWeek;
+    time->day = st.wDay;
+    time->hour = st.wHour;
+    time->minute = st.wMinute;
+    time->second = st.wSecond;
+    time->milliseconds = st.wMilliseconds;
+#endif 
+}
+
+DR_API 
+/* Allocates memory from DR's memory pool specific to the
+ * thread associated with drcontext.
+ */
+void *
+dr_thread_alloc(void *drcontext, size_t size)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    return heap_alloc(dcontext, size HEAPACCT(ACCT_IR));
+}
+
+DR_API 
+/* Frees thread-specific memory allocated by dr_thread_alloc.
+ * size must be the same size passed to dr_thread_alloc.
+ */
+void
+dr_thread_free(void *drcontext, void *mem, size_t size)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(drcontext != NULL, "dr_thread_free: drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_thread_free: drcontext is invalid");
+    heap_free(dcontext, mem, size HEAPACCT(ACCT_IR));
+}
+
+DR_API 
+/* Allocates memory from DR's global memory pool.
+ */
+void *
+dr_global_alloc(size_t size)
+{
+    return global_heap_alloc(size HEAPACCT(ACCT_OTHER));
+}
+
+DR_API 
+/* Frees memory allocated by dr_global_alloc.
+ * size must be the same size passed to dr_global_alloc.
+ */
+void
+dr_global_free(void *mem, size_t size)
+{
+    global_heap_free(mem, size HEAPACCT(ACCT_OTHER));
+}
+
+DR_API 
+/* PR 352427: API routine to allocate executable memory */
+void *
+dr_nonheap_alloc(size_t size, uint prot)
+{
+    return heap_mmap_ex(size, size, prot, false/*no guard pages*/);
+}
+
+DR_API 
+void
+dr_nonheap_free(void *mem, size_t size)
+{
+    heap_munmap_ex(mem, size, false/*no guard pages*/);
+}
+
+#ifdef LINUX
+DR_API
+/* With ld's -wrap option, we can supply a replacement for malloc.
+ * This routine allocates memory from DR's global memory pool.  Unlike
+ * dr_global_alloc(), however, we store the size of the allocation in
+ * the first few bytes so __wrap_free() can retrieve it.
+ */
+void *
+__wrap_malloc(size_t size)
+{
+    void *mem;
+    ASSERT(sizeof(size_t) >= HEAP_ALIGNMENT);
+    size += sizeof(size_t);
+    mem = global_heap_alloc(size HEAPACCT(ACCT_OTHER));
+    if (mem == NULL) {
+        CLIENT_ASSERT(false, "malloc failed: out of memory");
+        return NULL;
+    }
+    *((size_t *)mem) = size;
+    return mem + sizeof(size_t);
+}
+
+DR_API
+/* With ld's -wrap option, we can supply a replacement for realloc.
+ * This routine allocates memory from DR's global memory pool.  Unlike
+ * dr_global_alloc(), however, we store the size of the allocation in
+ * the first few bytes so __wrap_free() can retrieve it.
+ */
+void *
+__wrap_realloc(void *mem, size_t size)
+{
+    void *buf = NULL;
+    if (size > 0) {
+        buf = __wrap_malloc(size);
+        if (buf != NULL) {
+            size_t old_size = *((size_t *)(mem - sizeof(size_t)));
+            size_t min_size = MIN(old_size, size);
+            memcpy(buf, mem, min_size);
+        }
+    }
+    __wrap_free(mem);
+    return buf;
+}
+
+DR_API
+/* With ld's -wrap option, we can supply a replacement for free.  This
+ * routine frees memory allocated by __wrap_alloc and expects the
+ * allocation size to be available in the few bytes before 'mem'.
+ */
+void
+__wrap_free(void *mem)
+{
+    /* PR 200203: must_not_be_inlined() is assuming this routine calls
+     * no other DR routines besides global_heap_free!
+     */
+    if (mem != NULL) {
+        mem -= sizeof(size_t);
+        global_heap_free(mem, *((size_t *)mem) HEAPACCT(ACCT_OTHER));
+    }
+}
+#endif
+
+DR_API
+bool
+dr_memory_protect(void *base, size_t size, uint new_prot)
+{
+    /* We do allow the client to modify DR memory, for allocating a
+     * region and later making it unwritable.  We should probably
+     * allow modifying ntdll, since our general model is to trust the
+     * client and let it shoot itself in the foot, but that would require
+     * passing in extra args to app_memory_protection_change() to ignore
+     * the patch_proof_list: and maybe it is safer to disallow client
+     * from putting hooks in ntdll.
+     */
+    if (!dynamo_vm_area_overlap(base, ((byte *)base) + size)) {
+        uint mod_prot = new_prot;
+        uint res = app_memory_protection_change(get_thread_private_dcontext(),
+                                                base, size, new_prot, &mod_prot, NULL);
+        if (res != DO_APP_MEM_PROT_CHANGE) {
+            if (res == FAIL_APP_MEM_PROT_CHANGE ||
+                res == PRETEND_APP_MEM_PROT_CHANGE) {
+                return false;
+            } else {
+                /* SUBSET_APP_MEM_PROT_CHANGE should only happen for
+                 * PROGRAM_SHEPHERDING.  FIXME: not sure how common
+                 * this will be: for now we just fail.
+                 */
+                return false;
+            }
+        }
+        CLIENT_ASSERT(mod_prot == new_prot, "internal error on dr_memory_protect()");
+    }
+    return set_protection(base, size, new_prot);
+}
+
+DR_API
+/* checks to see that all bytes with addresses from pc to pc+size-1 
+ * are readable and that reading from there won't generate an exception.
+ */
+bool
+dr_memory_is_readable(const byte *pc, size_t size)
+{
+    return is_readable_without_exception(pc, size);
+}
+
+DR_API
+/* OS neutral memory query for clients, just wrapper around our get_memroy_info().
+ * FIXME - do something about executable areas we made non-writable - see PR 198873 */
+bool
+dr_query_memory(const byte *pc, byte **base_pc, size_t *size, uint *prot)
+{
+#ifdef LINUX
+    /* xref PR 246897 - the cached all memory list won't include the client lib
+     * mappings and appears to be inaccurate at times. For now we use the from
+     * os version instead (even though it's slower). FIXME */
+    return get_memory_info_from_os(pc, base_pc, size, prot);
+#else 
+    return get_memory_info(pc, base_pc, size, prot);
+#endif
+}
+
+#ifdef WINDOWS
+DR_API
+/* Calls NtQueryVirtualMemory.
+ * FIXME - do something about executable areas we made non-writable - see PR 198873 */
+size_t
+dr_virtual_query(const byte *pc, MEMORY_BASIC_INFORMATION *mbi, size_t mbi_size)
+{
+    return query_virtual_memory(pc, mbi, mbi_size);
+}
+#endif
+
+DR_API
+/* Wrapper around our safe_read. Xref P4 198875, placeholder till we have try/except.
+ * FIXME - the Linux version isn't actually safe - see PR 208562. */
+bool
+dr_safe_read(const void *base, size_t size, void *out_buf, size_t *bytes_read)
+{
+    return safe_read_ex(base, size, out_buf, bytes_read);
+}
+
+DR_API
+/* Wrapper around our safe_write. Xref P4 198875, placeholder till we have try/except.
+ * FIXME - the Linux version isn't actually safe - see PR 208562. */
+bool
+dr_safe_write(void *base, size_t size, const void *in_buf, size_t *bytes_written)
+{
+    return safe_write_ex(base, size, in_buf, bytes_written);
+}
+
+DR_API 
+/* Initializes a mutex
+ */
+void *
+dr_mutex_create(void)
+{
+    void *mutex = (void *)HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, mutex_t, 
+                                          ACCT_OTHER, UNPROTECTED);
+    ASSIGN_INIT_LOCK_FREE(*((mutex_t *) mutex), dr_client_mutex);
+    return mutex;
+}
+
+DR_API
+/* Deletes mutex
+ */
+void
+dr_mutex_destroy(void *mutex)
+{
+    /* Delete mutex so locks_not_closed()==0 test in dynamo.c passes */
+    DELETE_LOCK(*((mutex_t *) mutex));
+    HEAP_TYPE_FREE(GLOBAL_DCONTEXT, (mutex_t *)mutex, mutex_t, ACCT_OTHER, UNPROTECTED);
+}
+
+DR_API 
+/* Locks mutex
+ */
+void
+dr_mutex_lock(void *mutex)
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    /* set client_grab_mutex so that we now to set client_thread_safe_for_synch
+     * around the actual wait for the lock */
+    if (IS_CLIENT_THREAD(dcontext))
+        dcontext->client_data->client_grab_mutex = mutex;
+    mutex_lock((mutex_t *) mutex);
+    if (IS_CLIENT_THREAD(dcontext))
+        dcontext->client_data->client_grab_mutex = NULL;
+}
+
+DR_API 
+/* Unlocks mutex
+ */
+void
+dr_mutex_unlock(void *mutex)
+{
+    mutex_unlock((mutex_t *) mutex);
+}
+
+DR_API 
+/* Tries once to grab the lock, returns whether or not successful
+ */
+bool
+dr_mutex_trylock(void *mutex)
+{
+    return mutex_trylock((mutex_t *) mutex);
+}
+
+DR_API
+/* Looks up the module data containing pc.  Returns NULL if not found.
+ * Returned module_data_t must be freed with dr_free_module_data(). */
+module_data_t *
+dr_lookup_module(byte *pc)
+{
+    module_area_t *area;
+    module_data_t *client_data;
+    os_get_module_info_lock();
+    area = module_pc_lookup(pc);
+    client_data = copy_module_area_to_module_data(area);
+    os_get_module_info_unlock();
+    return client_data;
+}
+
+DR_API
+/* Looks up the module with name matching name (ignoring case).  Returns NULL if not
+ * found. Returned module_data_t must be freed with dr_free_module_data(). */
+module_data_t *
+dr_lookup_module_by_name(const char *name)
+{
+    /* We have no quick way of doing this since our module list is indexed by pc. We
+     * could use get_module_handle() but that's dangerous to call at arbitrary times,
+     * so we just walk our full list here. */
+    module_iterator_t *mi = module_iterator_start();
+    CLIENT_ASSERT((name != NULL), "dr_lookup_module_info_by_name: null name");
+    while (module_iterator_hasnext(mi)) {
+        module_area_t *area = module_iterator_next(mi);
+        module_data_t *client_data;
+        if (strcasecmp(GET_MODULE_NAME(&area->names), name) == 0) {
+            client_data = copy_module_area_to_module_data(area);
+            module_iterator_stop(mi);
+            return client_data;
+        }
+    }
+    module_iterator_stop(mi);
+    return NULL;
+}
+
+typedef struct _client_mod_iterator_list_t {
+    module_data_t *info;
+    struct _client_mod_iterator_list_t *next;
+} client_mod_iterator_list_t;
+
+typedef struct {
+    client_mod_iterator_list_t *current;
+    client_mod_iterator_list_t *full_list;
+} client_mod_iterator_t;
+
+DR_API
+/* Initialize a new client module iterator. */
+dr_module_iterator_t
+dr_module_iterator_start(void)
+{
+    client_mod_iterator_t *client_iterator = (client_mod_iterator_t *)
+        HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, client_mod_iterator_t, ACCT_OTHER, UNPROTECTED);
+    module_iterator_t *dr_iterator = module_iterator_start();
+
+    memset(client_iterator, 0, sizeof(*client_iterator)); 
+    while (module_iterator_hasnext(dr_iterator)) {
+        module_area_t *area = module_iterator_next(dr_iterator);
+        client_mod_iterator_list_t *list = (client_mod_iterator_list_t *)
+            HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, client_mod_iterator_list_t, ACCT_OTHER,
+                            UNPROTECTED);
+
+        ASSERT(area != NULL);
+        list->info = copy_module_area_to_module_data(area);
+        list->next = NULL;
+        if (client_iterator->current == NULL) {
+            client_iterator->current = list;
+            client_iterator->full_list = client_iterator->current;
+        } else {
+            client_iterator->current->next = list;
+            client_iterator->current = client_iterator->current->next;
+        }
+    }
+    module_iterator_stop(dr_iterator);
+    client_iterator->current = client_iterator->full_list;
+
+    return (dr_module_iterator_t)client_iterator;
+}
+
+DR_API
+/* Returns true if there is another loaded module in the iterator. */
+bool
+dr_module_iterator_hasnext(dr_module_iterator_t *mi)
+{
+    CLIENT_ASSERT((mi != NULL), "dr_module_iterator_hasnext: null iterator");
+    return ((client_mod_iterator_t *)mi)->current != NULL;
+}
+
+DR_API
+/* Retrieves the module_data_t for the next loaded module in the iterator. */
+module_data_t *
+dr_module_iterator_next(dr_module_iterator_t *mi)
+{
+    module_data_t *data;
+    client_mod_iterator_t *ci = (client_mod_iterator_t *)mi;
+    CLIENT_ASSERT((mi != NULL), "dr_module_iterator_next: null iterator");
+    CLIENT_ASSERT((ci->current != NULL), "dr_module_iterator_next: has no next, use "
+                  "dr_module_iterator_hasnext() first");
+    if (ci->current == NULL)
+        return NULL;
+    data = ci->current->info;
+    ci->current = ci->current->next;
+    return data;
+}
+
+DR_API 
+/* Free the module iterator. */
+void
+dr_module_iterator_stop(dr_module_iterator_t *mi)
+{
+    client_mod_iterator_t *ci = (client_mod_iterator_t *)mi;
+    CLIENT_ASSERT((mi != NULL), "dr_module_iterator_stop: null iterator");
+
+    /* free module_data_t's we didn't give to the client */
+    while (ci->current != NULL) {
+        dr_free_module_data(ci->current->info);
+        ci->current = ci->current->next;
+    }
+
+    ci->current = ci->full_list;
+    while (ci->current != NULL) {
+        client_mod_iterator_list_t *next = ci->current->next;
+        HEAP_TYPE_FREE(GLOBAL_DCONTEXT, ci->current, client_mod_iterator_list_t,
+                       ACCT_OTHER, UNPROTECTED);
+        ci->current = next;
+    }
+    HEAP_TYPE_FREE(GLOBAL_DCONTEXT, ci, client_mod_iterator_t, ACCT_OTHER, UNPROTECTED);
+}
+
+DR_API
+/* Get the name dr uses for this module. */
+const char *
+dr_module_preferred_name(const module_data_t *data)
+{
+    if (data == NULL)
+        return NULL;
+
+    return GET_MODULE_NAME(&data->names);
+}
+
+#ifdef WINDOWS
+
+DR_API
+/* If pc is within a section of module lib returns true and (optionally) a copy of
+ * the IMAGE_SECTION_HEADER in section_out.  If pc is not within a section of the
+ * module mod return false. */
+bool
+dr_lookup_module_section(module_handle_t lib, byte *pc, IMAGE_SECTION_HEADER *section_out)
+{
+    CLIENT_ASSERT((lib != NULL), "dr_lookup_module_section: null module_handle_t");
+    return module_pc_section_lookup((app_pc)lib, pc, section_out);
+}
+
+DR_API
+/* Returns the entry point of the function with the given name in the module
+ * with the given handle. */
+generic_func_t
+dr_get_proc_address(module_handle_t lib, const char *name)
+{
+    return get_proc_address(lib, name);
+}
+
+#endif
+
+DR_API
+/* Creates a new directory.  Fails if the directory already exists
+ * or if it can't be created.
+ */
+bool
+dr_create_dir(const char *fname)
+{
+    return os_create_dir(fname, CREATE_DIR_REQUIRE_NEW);
+}
+
+#ifdef WINDOWS
+DR_API
+/* Checks existence of a directory. */
+bool
+dr_directory_exists(const char *fname)
+{
+    return os_file_exists(fname, true);
+}
+
+DR_API
+/* Checks for the existence of a file. */
+bool
+dr_file_exists(const char *fname)
+{
+    return os_file_exists(fname, false);
+}
+#endif
+
+DR_API 
+/* Opens a file in the mode specified by mode_flags.
+ * Returns INVALID_FILE if unsuccessful
+ */
+file_t
+dr_open_file(const char *fname, uint mode_flags)
+{
+    uint flags = 0;
+
+    if (TEST(DR_FILE_WRITE_REQUIRE_NEW, mode_flags)) {
+        flags |= OS_OPEN_WRITE | OS_OPEN_REQUIRE_NEW;
+    }
+    if (TEST(DR_FILE_WRITE_APPEND, mode_flags)) {
+        CLIENT_ASSERT((flags == 0), "dr_open_file: multiple write modes selected"); 
+        flags |= OS_OPEN_WRITE | OS_OPEN_APPEND;
+    }   
+    if (TEST(DR_FILE_WRITE_OVERWRITE, mode_flags)) {
+        CLIENT_ASSERT((flags == 0), "dr_open_file: multiple write modes selected"); 
+        flags |= OS_OPEN_WRITE;
+    }
+
+    if (TEST(DR_FILE_READ, mode_flags))
+        flags |= OS_OPEN_READ;
+
+    if (TEST(DR_FILE_ALLOW_LARGE, mode_flags))
+        flags |= OS_OPEN_ALLOW_LARGE;
+
+    CLIENT_ASSERT((flags != 0), "dr_open_file: no mode selected"); 
+    return os_open(fname, flags);
+}
+
+DR_API 
+/* Closes file f
+ */
+void
+dr_close_file(file_t f)
+{
+    os_close(f);
+}
+
+DR_API 
+/* Flushes any buffers for file f
+ */
+void
+dr_flush_file(file_t f)
+{
+    os_flush(f);
+}
+
+DR_API 
+/* Writes count bytes from buf to f.
+ * Returns the actual number written.
+ */
+ssize_t
+dr_write_file(file_t f, const void *buf, size_t count)
+{
+    return os_write(f, buf, count);
+}
+
+DR_API 
+/* Reads up to count bytes from f into buf.
+ * Returns the actual number read.
+ */
+ssize_t
+dr_read_file(file_t f, void *buf, size_t count)
+{
+    return os_read(f, buf, count);
+}
+
+DR_API
+/* sets the current file position for file f to offset bytes from the specified origin
+ * returns true if succesful */
+bool
+dr_file_seek(file_t f, int64 offset, int origin)
+{
+    CLIENT_ASSERT(origin == DR_SEEK_SET || origin == DR_SEEK_CUR || origin == DR_SEEK_END,
+                  "dr_file_seek: invalid origin value");
+    return os_seek(f, offset, origin);
+}
+
+DR_API
+/* gets the current file position for file f in bytes from start of file */
+int64
+dr_file_tell(file_t f)
+{
+    return os_tell(f);
+}
+
+DR_API
+void
+dr_log(void *drcontext, uint mask, uint level, const char *fmt, ...)
+{
+#ifdef DEBUG
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    va_list ap;
+    if (stats != NULL &&
+        ((stats->logmask & mask) == 0 ||
+         stats->loglevel < level))
+        return;
+    va_start(ap, fmt);
+    if (dcontext != NULL)
+        do_file_write(dcontext->logfile, fmt, ap);
+    else
+        do_file_write(main_logfile, fmt, ap);
+    va_end(ap);
+#else
+    return; /* no logging if not debug */
+#endif
+}
+
+DR_API 
+/* Returns the log file for the drcontext thread.
+ * If drcontext is NULL, returns the main log file.
+ */
+file_t
+dr_get_logfile(void *drcontext)
+{
+#ifdef DEBUG
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    if (dcontext != NULL)
+        return dcontext->logfile;
+    else
+        return main_logfile;
+#else
+    return INVALID_FILE;
+#endif
+}
+
+DR_API 
+/* Returns true iff the -stderr_mask runtime option is non-zero, indicating
+ * that the user wants notification messages printed to stderr.
+ */
+bool
+dr_is_notify_on(void)
+{
+    return (dynamo_options.stderr_mask != 0);
+}
+
+#ifdef WINDOWS
+DR_API file_t 
+dr_get_stdout_file(void)
+{
+    return get_stdout_handle();
+}
+
+DR_API file_t
+dr_get_stderr_file(void)
+{
+    return get_stderr_handle();
+}
+#endif
+
+#ifdef PROGRAM_SHEPHERDING
+
+DR_API void
+dr_write_forensics_report(void *dcontext, file_t file,
+                          dr_security_violation_type_t violation,
+                          dr_security_violation_action_t action,
+                          const char *violation_name)
+{
+    security_violation_t sec_violation;
+    action_type_t sec_action;
+
+    switch (violation) {
+      case DR_RCO_STACK_VIOLATION:
+          sec_violation = STACK_EXECUTION_VIOLATION;
+          break;
+      case DR_RCO_HEAP_VIOLATION:
+          sec_violation = HEAP_EXECUTION_VIOLATION;
+          break;
+      case DR_RCT_RETURN_VIOLATION:
+          sec_violation = RETURN_TARGET_VIOLATION;
+          break;
+      case DR_RCT_INDIRECT_CALL_VIOLATION:
+          sec_violation = INDIRECT_CALL_RCT_VIOLATION;
+          break;
+      case DR_RCT_INDIRECT_JUMP_VIOLATION:
+          sec_violation = INDIRECT_JUMP_RCT_VIOLATION;
+          break;
+      default:
+          CLIENT_ASSERT(false, "dr_write_forensics_report does not support "
+                        "DR_UNKNOWN_VIOLATION or invalid violation types");
+          return;
+    }
+
+    switch (action) {
+      case DR_VIOLATION_ACTION_KILL_PROCESS:
+          sec_action = ACTION_TERMINATE_PROCESS;
+          break;
+      case DR_VIOLATION_ACTION_CONTINUE:
+      case DR_VIOLATION_ACTION_CONTINUE_CHANGED_CONTEXT:
+          sec_action = ACTION_CONTINUE;
+          break;
+      case DR_VIOLATION_ACTION_KILL_THREAD:
+          sec_action = ACTION_TERMINATE_THREAD;
+          break;
+      case DR_VIOLATION_ACTION_THROW_EXCEPTION:
+          sec_action = ACTION_THROW_EXCEPTION;
+          break;
+      default:
+          CLIENT_ASSERT(false, "dr_write_forensics_report invalid action selection");
+          return;
+    }
+
+    /* FIXME - could use a better message. */
+    append_diagnostics(file, action_message[sec_action], violation_name, sec_violation);
+}
+
+#endif /* PROGRAM_SHEPHERDING */
+
+#ifdef WINDOWS
+DR_API void
+dr_messagebox(const char *fmt, ...)
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    char msg[MAX_LOG_LENGTH];
+    wchar_t wmsg[MAX_LOG_LENGTH];
+    va_list ap;
+    va_start(ap, fmt);
+    vsnprintf(msg, BUFFER_SIZE_ELEMENTS(msg), fmt, ap);
+    NULL_TERMINATE_BUFFER(msg);
+    snwprintf(wmsg, BUFFER_SIZE_ELEMENTS(wmsg), L"%S", msg);
+    NULL_TERMINATE_BUFFER(wmsg);
+    if (IS_CLIENT_THREAD(dcontext))
+        dcontext->client_data->client_thread_safe_for_synch = true;
+    nt_messagebox(wmsg, L"Notice");
+    if (IS_CLIENT_THREAD(dcontext))
+        dcontext->client_data->client_thread_safe_for_synch = false;
+    va_end(ap);
+}
+#endif
+
+DR_API void
+dr_printf(const char *fmt, ...)
+{
+    va_list ap;
+    va_start(ap, fmt);
+    do_file_write(STDOUT, fmt, ap);
+    va_end(ap);
+}
+
+DR_API void 
+dr_fprintf(file_t f, const char *fmt, ...)
+{
+    va_list ap;
+    va_start(ap, fmt);
+    do_file_write(f, fmt, ap);
+    va_end(ap);
+}
+
+DR_API int
+/* We have to temporarily suspend our snprintf->_snprintf define */
+#undef snprintf
+snprintf(char *buf, size_t max, const char *fmt, ...)
+#ifdef WINDOWS
+# define snprintf _snprintf /* restore the define */
+#else
+# define snprintf our_snprintf /* restore the define */
+#endif
+{
+    /* in io.c */
+    extern int our_vsnprintf(char *s, size_t max, const char *fmt, va_list ap);
+    int res;
+    va_list ap;
+    va_start(ap, fmt);
+    /* PR 219380: we use our_vsnprintf instead of ntdll._vsnprintf b/c the
+     * latter does not support floating point (while ours does not support
+     * wide chars: but we also forward _snprintf to ntdll for clients).
+     */
+    res = our_vsnprintf(buf, max, fmt, ap);
+    va_end(ap);
+    /* Normalize Linux behavior to match Windows */
+    if ((size_t)res > max)
+        res = -1;
+    return res;
+}
+
+DR_API void 
+dr_print_instr(void *drcontext, file_t f, instr_t *instr, const char *msg)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(drcontext != NULL, "dr_print_instr: drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_print_instr: drcontext is invalid");
+    dr_fprintf(f, "%s "PFX" ", msg, instr_get_translation(instr));
+    instr_disassemble(dcontext, instr, f);
+    dr_fprintf(f, "\n");
+}
+
+DR_API void 
+dr_print_opnd(void *drcontext, file_t f, opnd_t opnd, const char *msg)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(drcontext != NULL, "dr_print_opnd: drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_print_opnd: drcontext is invalid");
+    dr_fprintf(f, "%s ", msg);
+    opnd_disassemble(dcontext, opnd, f);
+    dr_fprintf(f, "\n");
+}
+
+/***************************************************************************
+ * Thread support
+ */
+
+DR_API 
+/* Returns the DR context of the current thread */
+void *
+dr_get_current_drcontext(void)
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    return (void *) dcontext;
+}
+
+DR_API thread_id_t
+dr_get_thread_id(void *drcontext)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(drcontext != NULL, "dr_get_thread_id: drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_get_thread_id: drcontext is invalid");
+    return dcontext->owning_thread;
+}
+
+DR_API void *
+dr_get_tls_field(void *drcontext)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(drcontext != NULL, "dr_get_tls_field: drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_get_tls_field: drcontext is invalid");
+    return dcontext->client_data->user_field;
+}
+
+DR_API void 
+dr_set_tls_field(void *drcontext, void *value)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(drcontext != NULL, "dr_set_tls_field: drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_set_tls_field: drcontext is invalid");
+    dcontext->client_data->user_field = value;
+}
+
+DR_API
+/* Current thread gives up its time quantum. */
+void
+dr_thread_yield(void)
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    if (IS_CLIENT_THREAD(dcontext))
+        dcontext->client_data->client_thread_safe_for_synch = true;
+    thread_yield();
+    if (IS_CLIENT_THREAD(dcontext))
+        dcontext->client_data->client_thread_safe_for_synch = false;
+}
+
+/* for now tied to sideline feature PR 222812, not implemented on Linux */
+# if defined(CLIENT_SIDELINE) && defined(WINDOWS)
+DR_API
+/* Current thread sleeps for time_ms milliseconds. */
+void
+dr_sleep(int time_ms)
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    LARGE_INTEGER liDueTime;
+    if (IS_CLIENT_THREAD(dcontext))
+        dcontext->client_data->client_thread_safe_for_synch = true;
+    /* FIXME - add an os agnostic os_sleep() at some point when we need it on Linux */
+    liDueTime.QuadPart= - time_in_milliseconds * TIMER_UNITS_PER_MILLISECOND;
+    nt_sleep(&liDueTime);
+    if (IS_CLIENT_THREAD(dcontext))
+        dcontext->client_data->client_thread_safe_for_synch = false;
+}
+# endif
+
+#endif /* CLIENT_INTERFACE */
+
+DR_API
+/* Inserts inst as a non-application instruction into ilist prior to "where" */
+void
+instrlist_meta_preinsert(instrlist_t *ilist, instr_t *where, instr_t *inst)
+{
+    instr_set_ok_to_mangle(inst, false);
+    instrlist_preinsert(ilist, where, inst);
+}
+
+DR_API
+/* Inserts inst as a non-application instruction into ilist after "where" */
+void
+instrlist_meta_postinsert(instrlist_t *ilist, instr_t *where, instr_t *inst)
+{
+    instr_set_ok_to_mangle(inst, false);
+    instrlist_postinsert(ilist, where, inst);
+}
+
+DR_API
+/* Inserts inst as a non-application instruction onto the end of ilist */
+void
+instrlist_meta_append(instrlist_t *ilist, instr_t *inst)
+{
+    instr_set_ok_to_mangle(inst, false);
+    instrlist_append(ilist, inst);
+}
+
+/* dr_insert_* are used by general DR */
+
+/* Inserts a complete call to callee with the passed-in arguments */
+void
+dr_insert_call(void *drcontext, instrlist_t *ilist, instr_t *where,
+               void *callee, uint num_args, ...)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    va_list ap;
+    CLIENT_ASSERT(drcontext != NULL, "dr_insert_call: drcontext cannot be NULL");
+    /* we don't check for GLOBAL_DCONTEXT since DR internally calls this */
+    va_start(ap, num_args);
+    insert_meta_call_vargs(dcontext, ilist, where, false/*not clean*/,
+                           callee, num_args, ap);
+    va_end(ap);
+}
+
+/* Inserts a complete call to callee with the passed-in arguments, wrapped
+ * by an app save and restore.
+ * If "save_fpstate" is true, saves the fp/mmx/sse state.
+ *
+ * NOTE : this routine clobbers TLS_XAX_SLOT and the XSP mcontext slot via
+ * dr_prepare_for_clean_call(). We guarantee to clients that all other slots
+ * (except the XAX mcontext slot) will remain untouched.
+ */
+void
+dr_insert_clean_call(void *drcontext, instrlist_t *ilist, instr_t *where,
+                     void *callee, bool save_fpstate, uint num_args, ...)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    uint dstack_offs, pad = 0;
+    size_t buf_sz = 0;
+    va_list ap;
+    CLIENT_ASSERT(drcontext != NULL, "dr_insert_clean_call: drcontext cannot be NULL");
+    /* we don't check for GLOBAL_DCONTEXT since DR internally calls this */
+    va_start(ap, num_args);
+    dstack_offs = dr_prepare_for_call(dcontext, ilist, where);
+#ifdef X64
+    /* PR 218790: we assume that dr_prepare_for_call() leaves stack 16-byte
+     * aligned, which is what insert_meta_call_vargs requires. */
+    CLIENT_ASSERT(ALIGNED(dstack_offs, 16), "internal error: bad stack alignment");
+#endif
+    if (save_fpstate) {
+        /* save on the stack: xref PR 202669 on clients using more stack */
+        buf_sz = proc_fpstate_save_size();
+        /* we need 16-byte-alignment */
+        pad = ALIGN_FORWARD_UINT(dstack_offs, 16) - dstack_offs;
+        IF_X64(CLIENT_ASSERT(CHECK_TRUNCATE_TYPE_int(buf_sz + pad),
+                             "dr_insert_clean_call: internal truncation error"));
+        MINSERT(ilist, where, INSTR_CREATE_sub(dcontext, opnd_create_reg(REG_XSP),
+                                               OPND_CREATE_INT32((int)(buf_sz + pad))));
+        dr_insert_save_fpstate(drcontext, ilist, where,
+                               opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0,
+                                                     OPSZ_512));
+    }
+
+    /* PR 302951: restore state if clean call args reference app memory.
+     * We use a hack here: this is the only instance where we mark as our-mangling
+     * but do not have a translation target set, which indicates to the restore
+     * routines that this is a clean call.  If the client adds instrs in the middle
+     * translation will fail; if the client modifies any instr, the our-mangling
+     * flag will disappear and translation will fail.
+     */
+    instrlist_set_our_mangling(ilist, true);
+    insert_meta_call_vargs(dcontext, ilist, where, true/*clean*/, callee, num_args, ap);
+    instrlist_set_our_mangling(ilist, false);
+
+    if (save_fpstate) {
+        dr_insert_restore_fpstate(drcontext, ilist, where,
+                                  opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0,
+                                                        OPSZ_512));
+        MINSERT(ilist, where, INSTR_CREATE_add(dcontext, opnd_create_reg(REG_XSP),
+                                               OPND_CREATE_INT32(buf_sz + pad)));
+    }
+    dr_cleanup_after_call(dcontext, ilist, where, 0);
+    va_end(ap);
+}
+
+/* Utility routine for inserting a clean call to an instrumentation routine
+ * Returns the size of the data stored on the DR stack (in case the caller
+ * needs to align the stack pointer).  XSP and XAX are modified by this call.
+ * 
+ * NOTE : this routine clobbers TLS_XAX_SLOT and the XSP mcontext slot via
+ * prepare_for_clean_call(). We guarantee to clients that all other slots
+ * (except the XAX mcontext slot) will remain untouched.
+ */
+DR_API uint
+dr_prepare_for_call(void *drcontext, instrlist_t *ilist, instr_t *where)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    instr_t *in = (where == NULL) ? instrlist_last(ilist) : instr_get_prev(where);
+    uint dstack_offs;
+    CLIENT_ASSERT(drcontext != NULL, "dr_prepare_for_call: drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_prepare_for_call: drcontext is invalid");
+    dstack_offs = prepare_for_clean_call(dcontext, ilist, where);
+    /* now go through and mark inserted instrs as meta */
+    if (in == NULL)
+        in = instrlist_first(ilist);
+    else
+        in = instr_get_next(in);
+    while (in != where) {
+        instr_set_ok_to_mangle(in, false);
+        in = instr_get_next(in);
+    }
+    return dstack_offs;
+}
+
+DR_API void
+dr_cleanup_after_call(void *drcontext, instrlist_t *ilist, instr_t *where,
+                       uint sizeof_param_area)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    instr_t *in = (where == NULL) ? instrlist_last(ilist) : instr_get_prev(where);
+    CLIENT_ASSERT(drcontext != NULL, "dr_cleanup_after_call: drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_cleanup_after_call: drcontext is invalid");
+    if (sizeof_param_area > 0) {
+        /* clean up the parameter area */
+        CLIENT_ASSERT(sizeof_param_area <= 127,
+                      "dr_cleanup_after_call: sizeof_param_area must be <= 127");
+        /* mark it meta down below */
+        instrlist_preinsert(ilist, where,
+            INSTR_CREATE_add(dcontext, opnd_create_reg(REG_XSP),
+                             OPND_CREATE_INT8(sizeof_param_area)));
+    }
+    cleanup_after_clean_call(dcontext, ilist, where);
+    /* now go through and mark inserted instrs as meta */
+    if (in == NULL)
+        in = instrlist_first(ilist);
+    else
+        in = instr_get_next(in);
+    while (in != where) {
+        instr_set_ok_to_mangle(in, false);
+        in = instr_get_next(in);
+    }
+}
+
+#ifdef CLIENT_INTERFACE
+DR_API void
+dr_swap_to_clean_stack(void *drcontext, instrlist_t *ilist, instr_t *where)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(drcontext != NULL, "dr_swap_to_clean_stack: drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_swap_to_clean_stack: drcontext is invalid");
+
+    /* PR 219620: For thread-shared, we need to get the dcontext
+     * dynamically rather than use the constant passed in here.
+     */
+    if (SHARED_FRAGMENTS_ENABLED()) {
+        MINSERT(ilist, where, instr_create_save_to_tls
+                (dcontext, REG_XAX, TLS_XAX_SLOT));
+        insert_get_mcontext_base(dcontext, ilist, where, REG_XAX);
+        /* save app xsp, and then bring in dstack to xsp */
+        MINSERT(ilist, where, instr_create_save_to_dc_via_reg
+                (dcontext, REG_XAX, REG_XSP, XSP_OFFSET));
+        /* DSTACK_OFFSET isn't within the upcontext so if it's separate this won't
+         * work right.  FIXME - the dcontext accessing routines are a mess of shared
+         * vs. no shared support, separate context vs. no separate context support etc. */
+        ASSERT_NOT_IMPLEMENTED(!TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask));
+        MINSERT(ilist, where, instr_create_restore_from_dc_via_reg
+                (dcontext, REG_XAX, REG_XSP, DSTACK_OFFSET));
+        MINSERT(ilist, where, instr_create_restore_from_tls
+                (dcontext, REG_XAX, TLS_XAX_SLOT));
+    }
+    else {
+        MINSERT(ilist, where, instr_create_save_to_dcontext
+                (dcontext, REG_XSP, XSP_OFFSET));
+        MINSERT(ilist, where, instr_create_restore_dynamo_stack(dcontext));
+    }
+}
+
+DR_API void
+dr_restore_app_stack(void *drcontext, instrlist_t *ilist, instr_t *where)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(drcontext != NULL, "dr_restore_app_stack: drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_restore_app_stack: drcontext is invalid");
+    /* restore stack */
+    if (SHARED_FRAGMENTS_ENABLED()) {
+        /* use the register we're about to clobber as scratch space */        
+        insert_get_mcontext_base(dcontext, ilist, where, REG_XSP);
+        MINSERT(ilist, where, instr_create_restore_from_dc_via_reg
+                (dcontext, REG_XSP, REG_XSP, XSP_OFFSET));
+    } else {
+        MINSERT(ilist, where, instr_create_restore_from_dcontext
+                (dcontext, REG_XSP, XSP_OFFSET));
+    }
+}
+
+#define SPILL_SLOT_TLS_MAX 2
+#define NUM_TLS_SPILL_SLOTS (SPILL_SLOT_TLS_MAX + 1)
+#define NUM_SPILL_SLOTS (SPILL_SLOT_MAX + 1)
+/* The three tls slots we make available to clients.  We reserve TLS_XAX_SLOT for our
+ * own use in dr convenience routines. Note the +1 is because the max is an array index
+ * (so zero based) while array size is number of slots.  We don't need to +1 in 
+ * SPILL_SLOT_MC_REG because subtracting SPILL_SLOT_TLS_MAX already accounts for it. */
+static const ushort SPILL_SLOT_TLS_OFFS[NUM_TLS_SPILL_SLOTS] =
+    { TLS_XDX_SLOT, TLS_XCX_SLOT, TLS_XBX_SLOT };
+/* The dcontext reg slots we make available to clients.  We reserve XAX and XSP for
+ * our own use in dr convenience routines. */
+static const reg_id_t SPILL_SLOT_MC_REG[NUM_SPILL_SLOTS - NUM_TLS_SPILL_SLOTS] = { 
+#ifdef X64
+    REG_R15, REG_R14, REG_R13, REG_R12, REG_R11, REG_R10, REG_R9, REG_R8,
+#endif
+    REG_XDI, REG_XSI, REG_XBP, REG_XDX, REG_XCX, REG_XBX };
+
+DR_API void 
+dr_save_reg(void *drcontext, instrlist_t *ilist, instr_t *where, reg_id_t reg,
+            dr_spill_slot_t slot)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(drcontext != NULL, "dr_save_reg: drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_save_reg: drcontext is invalid");
+    CLIENT_ASSERT(slot <= SPILL_SLOT_MAX,
+                  "dr_save_reg: invalid spill slot selection");
+
+    CLIENT_ASSERT(reg_is_pointer_sized(reg),
+                  "dr_save_reg requires pointer-sized gpr");
+
+    if (slot <= SPILL_SLOT_TLS_MAX) {
+        ushort offs = os_tls_offset(SPILL_SLOT_TLS_OFFS[slot]);
+        MINSERT(ilist, where,
+                INSTR_CREATE_mov_st(dcontext, opnd_create_tls_slot(offs),
+                                    opnd_create_reg(reg)));
+    } else {
+        reg_id_t reg_slot = SPILL_SLOT_MC_REG[slot - NUM_TLS_SPILL_SLOTS];
+        int offs = opnd_get_reg_dcontext_offs(reg_slot);
+        if (SHARED_FRAGMENTS_ENABLED()) {
+            /* PR 219620: For thread-shared, we need to get the dcontext
+             * dynamically rather than use the constant passed in here.
+             */
+            reg_id_t tmp = (reg == REG_XAX) ? REG_XBX : REG_XAX;
+            
+            MINSERT(ilist, where, instr_create_save_to_tls
+                    (dcontext, tmp, TLS_XAX_SLOT));
+
+            insert_get_mcontext_base(dcontext, ilist, where, tmp);
+
+            MINSERT(ilist, where, instr_create_save_to_dc_via_reg
+                    (dcontext, tmp, reg, offs));
+
+            MINSERT(ilist, where, instr_create_restore_from_tls
+                    (dcontext, tmp, TLS_XAX_SLOT));
+        } else {
+            MINSERT(ilist, where, instr_create_save_to_dcontext(dcontext, reg, offs));
+        }
+    }
+}
+
+/* if want to save 8 or 16-bit reg, must pass in containing ptr-sized reg! */
+DR_API void 
+dr_restore_reg(void *drcontext, instrlist_t *ilist, instr_t *where, reg_id_t reg,
+               dr_spill_slot_t slot)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(drcontext != NULL, "dr_restore_reg: drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_restore_reg: drcontext is invalid");
+    CLIENT_ASSERT(slot <= SPILL_SLOT_MAX,
+                  "dr_restore_reg: invalid spill slot selection");
+
+    CLIENT_ASSERT(reg_is_pointer_sized(reg),
+                  "dr_restore_reg requires a pointer-sized gpr");
+
+    if (slot <= SPILL_SLOT_TLS_MAX) {
+        ushort offs = os_tls_offset(SPILL_SLOT_TLS_OFFS[slot]);
+        MINSERT(ilist, where,
+                INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(reg),
+                                    opnd_create_tls_slot(offs)));
+    } else {
+        reg_id_t reg_slot = SPILL_SLOT_MC_REG[slot - NUM_TLS_SPILL_SLOTS];
+        int offs = opnd_get_reg_dcontext_offs(reg_slot);
+        if (SHARED_FRAGMENTS_ENABLED()) {
+            /* PR 219620: For thread-shared, we need to get the dcontext
+             * dynamically rather than use the constant passed in here.
+             */
+            /* use the register we're about to clobber as scratch space */
+            insert_get_mcontext_base(dcontext, ilist, where, reg);
+
+            MINSERT(ilist, where, instr_create_restore_from_dc_via_reg
+                    (dcontext, reg, reg, offs));
+        } else {
+            MINSERT(ilist, where,
+                    instr_create_restore_from_dcontext(dcontext, reg, offs));
+        }
+    }
+}
+
+DR_API dr_spill_slot_t
+dr_max_opnd_accessible_spill_slot()
+{
+    if (SHARED_FRAGMENTS_ENABLED())
+        return SPILL_SLOT_TLS_MAX;
+    else
+        return SPILL_SLOT_MAX;
+}
+
+/* creates an opnd to access spill slot slot, slot must be <=
+ * dr_max_opnd_accessible_spill_slot() */
+opnd_t
+dr_reg_spill_slot_opnd(void *drcontext, dr_spill_slot_t slot)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(drcontext != NULL, "dr_reg_spill_slot_opnd: drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_reg_spill_slot_opnd: drcontext is invalid");
+    CLIENT_ASSERT(slot <= dr_max_opnd_accessible_spill_slot(),
+                  "dr_reg_spill_slot_opnd: slot must be less than "
+                  "dr_max_opnd_accessible_spill_slot()");
+
+    if (slot <= SPILL_SLOT_TLS_MAX) {
+        ushort offs = os_tls_offset(SPILL_SLOT_TLS_OFFS[slot]);
+        return opnd_create_tls_slot(offs);
+    } else {
+        reg_id_t reg_slot = SPILL_SLOT_MC_REG[slot - NUM_TLS_SPILL_SLOTS];
+        int offs = opnd_get_reg_dcontext_offs(reg_slot);
+        ASSERT(!SHARED_FRAGMENTS_ENABLED()); /* client assert above should catch */
+        return opnd_create_dcontext_field(dcontext, offs);
+    }
+}
+
+DR_API
+/* used to read a saved register spill slot from a clean call or a restore_state_event */
+reg_t
+dr_read_saved_reg(void *drcontext, dr_spill_slot_t slot)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(drcontext != NULL, "dr_read_saved_reg: drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_read_saved_reg: drcontext is invalid");
+    CLIENT_ASSERT(slot <= SPILL_SLOT_MAX,
+                  "dr_read_saved_reg: invalid spill slot selection");
+
+    /* FIXME - should we allow clients to read other threads saved registers? It's not
+     * as dangerous as write, but I can't think of a usage scenario where you'd want to
+     * Seems more likely to be a bug.  */
+    CLIENT_ASSERT(dcontext == get_thread_private_dcontext(),
+                  "dr_read_saved_reg(): drcontext does not belong to current thread");
+    
+    if (slot <= SPILL_SLOT_TLS_MAX) {
+        ushort offs = SPILL_SLOT_TLS_OFFS[slot];
+        return *(reg_t *)(((byte *)&dcontext->local_state->spill_space) + offs);
+    } else {
+        reg_id_t reg_slot = SPILL_SLOT_MC_REG[slot - NUM_TLS_SPILL_SLOTS];
+        return reg_get_value(reg_slot, get_mcontext(dcontext));
+    }
+}
+
+DR_API
+/* used to write a saved register spill slot from a clean call */
+void
+dr_write_saved_reg(void *drcontext, dr_spill_slot_t slot, reg_t value)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(drcontext != NULL, "dr_write_saved_reg: drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_write_saved_reg: drcontext is invalid");
+    CLIENT_ASSERT(slot <= SPILL_SLOT_MAX,
+                  "dr_write_saved_reg: invalid spill slot selection");
+
+    /* FIXME - should we allow clients to write to other threads saved registers?
+     * I can't think of a usage scenario where that would be correct, seems much more
+     * likely to be a difficult to diagnose bug that crashes the app or dr.  */
+    CLIENT_ASSERT(dcontext == get_thread_private_dcontext(),
+                  "dr_write_saved_reg(): drcontext does not belong to current thread");
+
+    if (slot <= SPILL_SLOT_TLS_MAX) {
+        ushort offs = SPILL_SLOT_TLS_OFFS[slot];
+        *(reg_t *)(((byte *)&dcontext->local_state->spill_space) + offs) = value;
+    } else {
+        reg_id_t reg_slot = SPILL_SLOT_MC_REG[slot - NUM_TLS_SPILL_SLOTS];
+        reg_set_value(reg_slot, get_mcontext(dcontext), value);
+    }
+}
+
+DR_API
+/**
+ * Inserts into ilist prior to "where" instruction(s) to read into the
+ * general-purpose full-size register reg from the user-controlled drcontext
+ * field for this thread.
+ */
+void
+dr_insert_read_tls_field(void *drcontext, instrlist_t *ilist, instr_t *where,
+                               reg_id_t reg)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(drcontext != NULL,
+                  "dr_insert_read_tls_field: drcontext cannot be NULL");
+    CLIENT_ASSERT(reg_is_pointer_sized(reg),
+                  "must use a pointer-sized general-purpose register");
+    if (SHARED_FRAGMENTS_ENABLED()) {
+        /* For thread-shared, since reg must be general-purpose we can
+         * use it as a base pointer (repeatedly).  Plus it's already dead.
+         */
+        MINSERT(ilist, where, instr_create_restore_from_tls
+                (dcontext, reg, TLS_DCONTEXT_SLOT));
+        MINSERT(ilist, where, instr_create_restore_from_dc_via_reg
+                (dcontext, reg, reg, CLIENT_DATA_OFFSET));
+        MINSERT(ilist, where, INSTR_CREATE_mov_ld
+                (dcontext, opnd_create_reg(reg),
+                 OPND_CREATE_MEMPTR(reg, offsetof(client_data_t, user_field))));
+    } else {
+        MINSERT(ilist, where, INSTR_CREATE_mov_ld
+                (dcontext, opnd_create_reg(reg),
+                 OPND_CREATE_ABSMEM(&dcontext->client_data->user_field, OPSZ_PTR)));
+    }
+}
+
+DR_API
+/**
+ * Inserts into ilist prior to "where" instruction(s) to write the
+ * general-purpose full-size register reg to the user-controlled drcontext field
+ * for this thread.
+ */
+void
+dr_insert_write_tls_field(void *drcontext, instrlist_t *ilist, instr_t *where,
+                                reg_id_t reg)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(drcontext != NULL,
+                  "dr_insert_write_tls_field: drcontext cannot be NULL");
+    CLIENT_ASSERT(reg_is_pointer_sized(reg),
+                  "must use a pointer-sized general-purpose register");
+    if (SHARED_FRAGMENTS_ENABLED()) {
+        reg_id_t spill = REG_XAX;
+        if (reg == spill) /* don't need sub-reg test b/c we know it's pointer-sized */
+            spill = REG_XDI;
+        MINSERT(ilist, where, instr_create_save_to_tls(dcontext, spill, TLS_XAX_SLOT));
+        MINSERT(ilist, where, instr_create_restore_from_tls
+                (dcontext, spill, TLS_DCONTEXT_SLOT));
+        MINSERT(ilist, where, instr_create_restore_from_dc_via_reg
+                (dcontext, spill, spill, CLIENT_DATA_OFFSET));
+        MINSERT(ilist, where, INSTR_CREATE_mov_st
+                (dcontext, OPND_CREATE_MEMPTR(spill,
+                                              offsetof(client_data_t, user_field)),
+                 opnd_create_reg(reg)));
+        MINSERT(ilist, where,
+                instr_create_restore_from_tls(dcontext, spill, TLS_XAX_SLOT));
+    } else {
+        MINSERT(ilist, where, INSTR_CREATE_mov_st
+                (dcontext, OPND_CREATE_ABSMEM
+                 (&dcontext->client_data->user_field, OPSZ_PTR),
+                 opnd_create_reg(reg)));
+    }
+}
+
+DR_API void 
+dr_save_arith_flags(void *drcontext, instrlist_t *ilist, instr_t *where,
+                    dr_spill_slot_t slot)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(drcontext != NULL, "dr_save_arith_flags: drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_save_arith_flags: drcontext is invalid");
+    CLIENT_ASSERT(slot <= SPILL_SLOT_MAX,
+                  "dr_save_arith_flags: invalid spill slot selection");
+
+    /* flag saving code:
+     *   save eax
+     *   lahf
+     *   seto al
+     */
+    dr_save_reg(drcontext, ilist, where, REG_XAX, slot);
+    MINSERT(ilist, where, INSTR_CREATE_lahf(dcontext));
+    MINSERT(ilist, where,
+            INSTR_CREATE_setcc(dcontext, OP_seto, opnd_create_reg(REG_AL)));
+}
+
+DR_API void 
+dr_restore_arith_flags(void *drcontext, instrlist_t *ilist, instr_t *where,
+                       dr_spill_slot_t slot)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    CLIENT_ASSERT(drcontext != NULL, "dr_restore_arith_flags: drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_restore_arith_flags: drcontext is invalid");
+    CLIENT_ASSERT(slot <= SPILL_SLOT_MAX,
+                  "dr_restore_arith_flags: invalid spill slot selection");
+
+    /* flag restoring code:
+     *   add 0x7f,%al
+     *   sahf
+     *   restore eax
+     */
+    /* do an add such that OF will be set only if seto set
+     * the MSB of saveto to 1
+     */
+    MINSERT(ilist, where,
+            INSTR_CREATE_add(dcontext, opnd_create_reg(REG_AL), OPND_CREATE_INT8(0x7f)));
+    MINSERT(ilist, where, INSTR_CREATE_sahf(dcontext));
+    dr_restore_reg(drcontext, ilist, where, REG_XAX, slot);
+}
+
+/* providing functionality of old -instr_calls and -instr_branches flags
+ *
+ * NOTE : this routine clobbers TLS_XAX_SLOT and the XSP mcontext slot via
+ * dr_insert_clean_call(). We guarantee to clients that all other slots
+ * (except the XAX mcontext slot) will remain untouched.
+ */
+DR_API void
+dr_insert_call_instrumentation(void *drcontext, instrlist_t *ilist, instr_t *instr,
+                               void *callee)
+{
+    ptr_uint_t target, address;
+    CLIENT_ASSERT(drcontext != NULL,
+                  "dr_insert_call_instrumentation: drcontext cannot be NULL");
+    address = (ptr_uint_t) instr_get_translation(instr);
+    /* dr_insert_ubr_instrumentation() uses this function */
+    CLIENT_ASSERT(instr_is_call(instr) || instr_is_ubr(instr),
+                  "dr_insert_{ubr,call}_instrumentation must be applied to a ubr");
+    CLIENT_ASSERT(address != 0,
+                  "dr_insert_{ubr,call}_instrumentation: can't determine app address");
+    if (opnd_is_pc(instr_get_target(instr))) {
+        if (opnd_is_far_pc(instr_get_target(instr))) {
+            /* FIXME: handle far pc */
+            CLIENT_ASSERT(false,
+                          "dr_insert_{ubr,call}_instrumentation: far pc not supported");
+        }
+        /* In release build for far pc keep going assuming 0 base */
+        target = (ptr_uint_t) opnd_get_pc(instr_get_target(instr));
+    }
+    else if (opnd_is_instr(instr_get_target(instr))) {
+        instr_t *tgt = opnd_get_instr(instr_get_target(instr));
+        target = (ptr_uint_t) instr_get_translation(tgt);
+        CLIENT_ASSERT(target != 0,
+                      "dr_insert_{ubr,call}_instrumentation: unknown target");
+        if (opnd_is_far_instr(instr_get_target(instr))) {
+            /* FIXME: handle far instr */
+            CLIENT_ASSERT(false,
+                          "dr_insert_{ubr,call}_instrumentation: far instr not supported");
+        }
+    } else {
+        CLIENT_ASSERT(false, "dr_insert_{ubr,call}_instrumentation: unknown target");
+        target = 0;
+    }
+
+    dr_insert_clean_call(drcontext, ilist, instr, callee, false/*no fpstate*/, 2,
+                         /* address of call is 1st parameter */
+                         OPND_CREATE_INTPTR(address),
+                         /* call target is 2nd parameter */
+                         OPND_CREATE_INTPTR(target));
+}
+
+/* NOTE : this routine clobbers TLS_XAX_SLOT and the XSP mcontext slot via
+ * dr_insert_clean_call(). We guarantee to clients that all other slots
+ * (except the XAX mcontext slot) will remain untouched. Since we need another
+ * tls spill slot in this routine we require the caller to give us one. */
+DR_API void
+dr_insert_mbr_instrumentation(void *drcontext, instrlist_t *ilist, instr_t *instr,
+                              void *callee, dr_spill_slot_t scratch_slot)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    ptr_uint_t address = (ptr_uint_t) instr_get_translation(instr);
+    opnd_t tls_opnd;
+    instr_t *newinst;
+
+    /* PR 214051: dr_insert_mbr_instrumentation() broken with -indcall2direct */
+    CLIENT_ASSERT(!DYNAMO_OPTION(indcall2direct),
+                  "dr_insert_mbr_instrumentation not supported with -opt_speed");
+    CLIENT_ASSERT(drcontext != NULL,
+                  "dr_insert_mbr_instrumentation: drcontext cannot be NULL");
+    CLIENT_ASSERT(address != 0,
+                  "dr_insert_mbr_instrumentation: can't determine app address");
+    CLIENT_ASSERT(instr_is_mbr(instr),
+                  "dr_insert_mbr_instrumentation must be applied to an mbr");
+
+    /* We need a TLS spill slot to use.  We can use any tls slot that is opnd
+     * accessible. */
+    CLIENT_ASSERT(scratch_slot <= dr_max_opnd_accessible_spill_slot(),
+                  "dr_insert_mbr_instrumentation: scratch_slot must be less than "
+                  "dr_max_opnd_accessible_spill_slot()");
+
+    /* PR 240265: we disallow clients to add post-mbr instrumentation, so we
+     * avoid doing that here even though it's a little less efficient since
+     * our mbr mangling will re-grab the target.
+     * We could keep it post-mbr and mark it w/ a special flag so we allow
+     * our own but not clients' instrumentation post-mbr: but then we
+     * hit post-syscall issues for wow64 where post-mbr equals post-syscall
+     * (PR 240258: though we might solve that some other way).
+     */
+
+    /* Note that since we're using a client exposed slot we know it will be
+     * preserved across the clean call. */
+    tls_opnd = dr_reg_spill_slot_opnd(drcontext, scratch_slot);
+    newinst = INSTR_CREATE_mov_st(dcontext, tls_opnd, opnd_create_reg(REG_XCX));
+
+    /* PR 214962: ensure we'll properly translate the de-ref of app
+     * memory by marking the spill and de-ref as INSTR_OUR_MANGLING.
+     */
+    instr_set_our_mangling(newinst, true);
+    MINSERT(ilist, instr, newinst);            
+
+    if (instr_is_return(instr)) {
+        /* the retaddr operand is always the final source for all OP_ret* instrs */
+        opnd_t retaddr = instr_get_src(instr, instr_num_srcs(instr) - 1);
+        opnd_size_t sz = opnd_get_size(retaddr);
+        /* even for far ret and iret, retaddr is at TOS */
+        newinst = instr_create_1dst_1src(dcontext, sz == OPSZ_2 ? OP_movzx : OP_mov_ld,
+                                         opnd_create_reg(REG_XCX), retaddr);
+    } else {
+        /* call* or jmp* */
+        opnd_t src = instr_get_src(instr, 0);
+        opnd_size_t sz = opnd_get_size(src);
+        reg_id_t reg_target = REG_XCX;
+        /* if a far cti, we can't fit it into a register: asserted above.
+         * in release build we'll get just the address here.
+         */
+        if (instr_is_far_cti(instr)) {
+            if (sz == OPSZ_10) {
+                sz = OPSZ_8;
+                reg_target = REG_RCX;
+            } else if (sz == OPSZ_6) {
+                sz = OPSZ_4;
+                reg_target = REG_ECX;
+            } else /* target has OPSZ_4 */ {
+                sz = OPSZ_2;
+                reg_target = REG_XCX; /* we use movzx below */
+            }
+            opnd_set_size(&src, sz);
+        }
+        newinst = instr_create_1dst_1src(dcontext, sz == OPSZ_2 ? OP_movzx : OP_mov_ld,
+                                         opnd_create_reg(reg_target), src);
+    }
+    instr_set_our_mangling(newinst, true);
+    MINSERT(ilist, instr, newinst);            
+
+    /* Now we want the true app state saved, for dr_get_mcontext().
+     * We specially recognize our OP_xchg as a restore in
+     * instr_is_reg_spill_or_restore().
+     */
+    MINSERT(ilist, instr,
+            INSTR_CREATE_xchg(dcontext, tls_opnd, opnd_create_reg(REG_XCX)));
+
+    dr_insert_clean_call(drcontext, ilist, instr, callee, false/*no fpstate*/, 2,
+                         /* address of mbr is 1st param */
+                         OPND_CREATE_INTPTR(address),
+                         /* indirect target (in tls, xchg-d from ecx) is 2nd param */
+                         tls_opnd);
+}
+
+/* NOTE : this routine clobbers TLS_XAX_SLOT and the XSP mcontext slot via
+ * dr_insert_clean_call(). We guarantee to clients that all other slots
+ * (except the XAX mcontext slot) will remain untouched. */
+DR_API void
+dr_insert_cbr_instrumentation(void *drcontext, instrlist_t *ilist, instr_t *instr,
+                              void *callee)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    ptr_uint_t address, target;
+    int opc;
+    instr_t *app_flags_ok;
+    CLIENT_ASSERT(drcontext != NULL,
+                  "dr_insert_cbr_instrumentation: drcontext cannot be NULL");
+    address = (ptr_uint_t) instr_get_translation(instr);
+    CLIENT_ASSERT(address != 0,
+                  "dr_insert_cbr_instrumentation: can't determine app address");
+    CLIENT_ASSERT(instr_is_cbr(instr),
+                  "dr_insert_cbr_instrumentation must be applied to a cbr");
+    CLIENT_ASSERT(opnd_is_near_pc(instr_get_target(instr)) ||
+                  opnd_is_near_instr(instr_get_target(instr)),
+                  "dr_insert_cbr_instrumentation: target opnd must be a near pc or "
+                  "near instr");
+    if (opnd_is_near_pc(instr_get_target(instr)))
+        target = (ptr_uint_t) opnd_get_pc(instr_get_target(instr));
+    else if (opnd_is_near_instr(instr_get_target(instr))) {
+        instr_t *tgt = opnd_get_instr(instr_get_target(instr));
+        target = (ptr_uint_t) instr_get_translation(tgt);
+        CLIENT_ASSERT(target != 0, "dr_insert_cbr_instrumentation: unknown target");
+    } else {
+        CLIENT_ASSERT(false, "dr_insert_cbr_instrumentation: unknown target");
+        target = 0;
+    }
+
+    app_flags_ok = instr_get_prev(instr);
+    dr_insert_clean_call(drcontext, ilist, instr, callee, false/*no fpstate*/, 3,
+                         /* push address of mbr onto stack as 1st parameter */
+                         OPND_CREATE_INTPTR(address),
+                         /* target is 2nd parameter */
+                         OPND_CREATE_INTPTR(target),
+                         /* branch direction (put in ebx below) is 3rd parameter */
+                         opnd_create_reg(REG_XBX));
+
+    /* calculate whether branch taken or not
+     * since the clean call mechanism clobbers eflags, we
+     * must insert our checks prior to that clobbering.
+     * since we do it AFTER the pusha, we don't have to save; but, we
+     * can't use a param that's part of any calling convention b/c w/
+     * PR 250976 our clean call will get it from the pusha.
+     * ebx is a good choice.
+     */
+    /* We expect:
+       mov    0x400e5e34 -> %esp 
+       pusha  %esp %eax %ebx %ecx %edx %ebp %esi %edi -> %esp (%esp) 
+       pushf  %esp -> %esp (%esp) 
+       push   $0x00000000 %esp -> %esp (%esp) 
+       popf   %esp (%esp) -> %esp 
+       mov    0x400e5e40 -> %eax 
+       push   %eax %esp -> %esp (%esp) 
+     * We also assume all clean call instrs are expanded.
+     */
+    if (app_flags_ok == NULL)
+        app_flags_ok = instrlist_first(ilist);
+    while (!instr_opcode_valid(app_flags_ok) ||
+           instr_get_opcode(app_flags_ok) != OP_popf) {
+        app_flags_ok = instr_get_next(app_flags_ok);
+        CLIENT_ASSERT(app_flags_ok != NULL, 
+                      "dr_insert_cbr_instrumentation: cannot find eflags save");
+    }
+    /* put our code before the popf */
+
+    opc = instr_get_opcode(instr);
+    if (opc == OP_jecxz || opc == OP_loop || opc == OP_loope || opc == OP_loopne) {
+        /* for 8-bit cbrs w/ multiple conditions and state, simpler to
+         * simply execute them -- they're rare so shouldn't be a perf hit.
+         * after all, ecx is saved, can clobber it.
+         * we do:
+         *               loop/jecxz taken
+         *    not_taken: mov 0, ebx
+         *               jmp done
+         *    taken:     mov 1, ebx
+         *    done:
+         */
+        instr_t *branch = instr_clone(dcontext, instr);
+        instr_t *not_taken =
+            INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_EBX),
+                                 OPND_CREATE_INT32(0));
+        instr_t *taken =
+            INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_EBX),
+                                 OPND_CREATE_INT32(1));
+        instr_set_target(branch, opnd_create_instr(taken));
+        /* client-added meta instrs should not have translation set */
+        instr_set_translation(branch, NULL);
+        MINSERT(ilist, app_flags_ok, branch);
+        MINSERT(ilist, app_flags_ok, not_taken);
+        MINSERT(ilist, app_flags_ok,
+                INSTR_CREATE_jmp_short(dcontext, opnd_create_instr(app_flags_ok)));
+        MINSERT(ilist, app_flags_ok, taken);
+    } else {
+        /* build a setcc equivalent of instr's jcc operation
+         * WARNING: this relies on order of OP_ enum!
+         */
+        opc = instr_get_opcode(instr);
+        if (opc <= OP_jnle_short)
+            opc += (OP_jo - OP_jo_short);
+        CLIENT_ASSERT(opc >= OP_jo && opc <= OP_jnle,
+                      "dr_insert_cbr_instrumentation: unknown opcode");
+        opc = opc - OP_jo + OP_seto;
+        MINSERT(ilist, app_flags_ok,
+                INSTR_CREATE_setcc(dcontext, opc, opnd_create_reg(REG_BL)));
+        /* movzx ebx <- bl */
+        MINSERT(ilist, app_flags_ok,
+                INSTR_CREATE_movzx(dcontext, opnd_create_reg(REG_EBX),
+                                   opnd_create_reg(REG_BL)));
+    }
+
+    /* now branch dir is in ebx and will be passed to clean call */
+}
+
+DR_API void
+dr_insert_ubr_instrumentation(void *drcontext, instrlist_t *ilist, instr_t *instr,
+                              void *callee)
+{
+    /* same as call */
+    dr_insert_call_instrumentation(drcontext, ilist, instr, callee);
+}
+
+DR_API bool
+dr_mcontext_xmm_fields_valid(void)
+{
+    return preserve_xmm_caller_saved();
+}
+
+#endif /* CLIENT_INTERFACE */
+/* dr_get_mcontext() needed for translating clean call arg errors */
+
+DR_API void
+dr_get_mcontext(void *drcontext, dr_mcontext_t *context, int *app_errno)
+{
+    char *state;
+    dcontext_t *dcontext = (dcontext_t *)drcontext;
+    CLIENT_ASSERT(!TEST(SELFPROT_DCONTEXT, DYNAMO_OPTION(protect_mask)),
+                  "DR context protection NYI");
+    CLIENT_ASSERT(context != NULL, "invalid context");
+
+#ifdef CLIENT_INTERFACE
+    /* PR 207947: support mcontext access from syscall events */
+    if (dcontext->client_data->in_pre_syscall ||
+        dcontext->client_data->in_post_syscall) {
+        *context = *get_mcontext(dcontext);
+        if (app_errno != NULL)
+            *app_errno = dcontext->app_errno;
+        return;
+    }
+#endif
+
+    /* dr_prepare_for_call() puts the machine context on the dstack
+     * with pusha and pushf, but only fills in xmm values for
+     * preserve_xmm_caller_saved(): however, we tell the client that the xmm
+     * fields are not valid otherwise.  so, we just have to copy the
+     * state from the dstack.
+     */
+    state = (char *)dcontext->dstack - sizeof(dr_mcontext_t);
+    *context = *((dr_mcontext_t *)state);
+    if (app_errno != NULL) {
+        state -= sizeof(int);
+        *app_errno = *((int *)state);
+    }
+
+    /* esp is a dstack value -- get the app stack's esp from the dcontext */
+    context->xsp = get_mcontext(dcontext)->xsp;
+
+    /* FIXME: should we set the pc field? */
+}
+
+#ifdef CLIENT_INTERFACE
+DR_API void
+dr_set_mcontext(void *drcontext, dr_mcontext_t *context, const int *app_errno)
+{
+    char *state;
+    dcontext_t *dcontext = (dcontext_t *)drcontext;
+    CLIENT_ASSERT(!TEST(SELFPROT_DCONTEXT, DYNAMO_OPTION(protect_mask)),
+                  "DR context protection NYI");
+    CLIENT_ASSERT(context != NULL, "invalid context");
+
+    /* PR 207947: support mcontext access from syscall events */
+    if (dcontext->client_data->in_pre_syscall ||
+        dcontext->client_data->in_post_syscall) {
+        *get_mcontext(dcontext) = *context;
+        if (app_errno != NULL)
+            dcontext->app_errno = *app_errno;
+        return;
+    }
+
+    /* copy the machine context to the dstack area created with 
+     * dr_prepare_for_call().  note that xmm0-5 copied there
+     * will override any save_fpstate xmm values, as desired.
+     */
+    state = (char *)dcontext->dstack - sizeof(dr_mcontext_t);
+    *((dr_mcontext_t *)state) = *context;
+    if (app_errno != NULL) {
+        state -= sizeof(int);
+        *((int *)state) = *app_errno;
+    }
+
+    /* esp will be restored from a field in the dcontext */
+    get_mcontext(dcontext)->xsp = context->xsp;
+
+    /* FIXME: should we support setting the pc field? */
+}
+
+DR_API
+void
+dr_redirect_execution(dr_mcontext_t *mcontext, int app_errno)
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+
+    /* PR 352429: squash current trace.
+     * FIXME: will clients use this so much that this will be a perf issue?
+     * samples/cbr doesn't hit this even at -trace_threshold 1
+     */
+    if (is_building_trace(dcontext)) {
+        LOG(THREAD, LOG_INTERP, 1, "squashing trace-in-progress\n");
+        trace_abort(dcontext);
+    }
+
+    dcontext->next_tag = mcontext->pc;
+    dcontext->whereami = WHERE_FCACHE;
+    set_last_exit(dcontext, (linkstub_t *)get_client_linkstub());
+    transfer_to_dispatch(dcontext, app_errno, mcontext);
+}
+
+/***************************************************************************
+ * ADAPTIVE OPTIMIZATION SUPPORT
+ * *Note for non owning thread support (i.e. sideline) all methods assume
+ * the dcontext valid, the client will have to insure this with a lock
+ * on thread_exit!!
+ *
+ * *need way for side thread to get a dcontext to use for logging and mem
+ * alloc, before do that should think more about mem alloc in/for adaptive
+ * routines
+ *
+ * *made local mem alloc by side thread safe (see heap.c)
+ *
+ * *loging not safe if not owning thread?
+ */
+
+DR_API
+/* Schedules the fragment to be deleted.  Once this call is completed,
+ * an existing executing fragment is allowed to complete, but control
+ * will not enter the fragment again before it is deleted.
+ * 
+ * NOTE: this comment used to say, "after deletion, control may still 
+ * reach the fragment by indirect branch.".  We believe this is now only
+ * true for shared fragments, which are not currently supported.
+ */
+bool
+dr_delete_fragment(void *drcontext, void *tag) 
+{
+    dcontext_t *dcontext = (dcontext_t *)drcontext;
+    fragment_t *f;
+    bool deletable = false;
+    CLIENT_ASSERT(!SHARED_FRAGMENTS_ENABLED(),
+                  "dr_delete_fragment() only valid with -thread_private");
+    CLIENT_ASSERT(drcontext != NULL, "dr_delete_fragment(): drcontext cannot be NULL");
+#ifdef CLIENT_SIDELINE
+    mutex_lock(&(dcontext->client_data->sideline_mutex));
+    fragment_get_fragment_delete_mutex(dcontext);
+#else
+    CLIENT_ASSERT(drcontext == get_thread_private_dcontext(),
+                  "dr_delete_fragment(): drcontext does not belong to current thread");
+#endif
+    f = fragment_lookup(dcontext, tag);
+    if (f != NULL && (f->flags & FRAG_CANNOT_DELETE) == 0) {
+        client_todo_list_t * todo = HEAP_TYPE_ALLOC(dcontext, client_todo_list_t, ACCT_OTHER, UNPROTECTED);
+        client_todo_list_t * iter = dcontext->client_data->to_do;
+        todo->next = NULL;
+        todo->ilist = NULL;
+        todo->tag = tag;
+        if (iter == NULL)
+            dcontext->client_data->to_do = todo;
+        else {
+            while (iter->next != NULL)
+                iter = iter->next;
+            iter->next = todo;
+        }
+        deletable = true;
+        /* unlink fragment so will return to dynamo and delete.
+         * Do not remove the fragment from the hashtable --
+         * we need to be able to look up the fragment when 
+         * inspecting the to_do list in dispatch.
+         */
+        if ((f->flags & FRAG_LINKED_INCOMING) != 0)
+            unlink_fragment_incoming(dcontext, f);
+        fragment_remove_from_ibt_tables(dcontext, f, false);
+    }
+#ifdef CLIENT_SIDELINE
+    fragment_release_fragment_delete_mutex(dcontext);
+    mutex_unlock(&(dcontext->client_data->sideline_mutex));
+#endif
+    return deletable;
+}
+
+DR_API 
+/* Schedules the fragment at 'tag' for replacement.  Once this call is 
+ * completed, an existing executing fragment is allowed to complete, 
+ * but control will not enter the fragment again before it is replaced.
+ * 
+ * NOTE: this comment used to say, "after replacement, control may still 
+ * reach the fragment by indirect branch.".  We believe this is now only
+ * true for shared fragments, which are not currently supported.
+ *
+ * Takes control of the ilist and all responsibility for deleting it and the
+ * instrs inside of it.  The client should not keep, use, reference, etc. the
+ * instrlist or any of the instrs it contains after they are passed in. 
+ */
+bool
+dr_replace_fragment(void *drcontext, void *tag, instrlist_t *ilist)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    bool frag_found;
+    fragment_t * f;
+    CLIENT_ASSERT(!SHARED_FRAGMENTS_ENABLED(),
+                  "dr_replace_fragment() only valid with -thread_private");
+    CLIENT_ASSERT(drcontext != NULL, "dr_replace_fragment(): drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_replace_fragment: drcontext is invalid");
+#ifdef CLIENT_SIDELINE
+    mutex_lock(&(dcontext->client_data->sideline_mutex));
+    fragment_get_fragment_delete_mutex(dcontext);
+#else
+    CLIENT_ASSERT(drcontext == get_thread_private_dcontext(),
+                  "dr_replace_fragment(): drcontext does not belong to current thread");
+#endif
+    f = fragment_lookup(dcontext, tag);
+    frag_found = (f != NULL);
+    if (frag_found) {
+        client_todo_list_t * iter = dcontext->client_data->to_do;
+        client_todo_list_t * todo = HEAP_TYPE_ALLOC(dcontext, client_todo_list_t, ACCT_OTHER, UNPROTECTED);
+        todo->next = NULL;
+        todo->ilist = ilist;
+        todo->tag = tag;
+        if (iter == NULL)
+            dcontext->client_data->to_do = todo;
+        else {
+            while (iter->next != NULL)
+                iter = iter->next;
+            iter->next = todo;
+        }
+        /* unlink fragment so will return to dynamo and replace for next time
+         * its executed
+         */
+        if ((f->flags & FRAG_LINKED_INCOMING) != 0)
+            unlink_fragment_incoming(dcontext, f);
+        fragment_remove_from_ibt_tables(dcontext, f, false);
+    }
+#ifdef CLIENT_SIDELINE
+    fragment_release_fragment_delete_mutex(dcontext);
+    mutex_unlock(&(dcontext->client_data->sideline_mutex));
+#endif
+    return frag_found;
+}
+
+#ifdef UNSUPPORTED_API
+/* FIXME - doesn't work with shared fragments.  Consider removing since dr_flush_region
+ * and dr_delay_flush_region give us most of this functionality. */
+DR_API
+/* Flushes all fragments containing 'flush_tag', or the entire code
+ * cache if flush_tag is NULL.  'curr_tag' must specify the tag of the
+ * currently-executing fragment.  If curr_tag is NULL, flushing can be
+ * delayed indefinitely.  Note that flushing is performed across all
+ * threads, but other threads may continue to execute fragments 
+ * containing 'curr_tag' until those fragments finish.
+ */
+void dr_flush_fragments(void *drcontext, void *curr_tag, void *flush_tag)
+{
+    client_flush_req_t *iter, *flush;
+    dcontext_t *dcontext = (dcontext_t *)drcontext;
+
+    /* We want to unlink the currently executing fragment so we'll
+     * force a context switch to DR.  That way, we'll perform the
+     * flush as soon as possible.  Unfortunately, the client may not
+     * know the tag of the current trace.  Therefore, we unlink all
+     * fragments in the region.
+     *
+     * Note that we aren't unlinking or ibl-invalidating (i.e., making
+     * unreachable) any fragments in other threads containing curr_tag
+     * until the delayed flush happens in enter_nolinking().
+     */
+    if (curr_tag != NULL)
+        vm_area_unlink_incoming(dcontext, (app_pc)curr_tag);
+
+    flush = HEAP_TYPE_ALLOC(dcontext, client_flush_req_t, ACCT_OTHER, UNPROTECTED);
+    flush->flush_callback = NULL;
+    if (flush_tag == NULL) {
+        flush->start = UNIVERSAL_REGION_BASE;
+        flush->size = UNIVERSAL_REGION_SIZE;
+    } else {
+        flush->start = (app_pc)flush_tag;
+        flush->size = 1;
+    }
+    flush->next = NULL;
+
+    iter = dcontext->client_data->flush_list;
+    if (iter == NULL) {
+        dcontext->client_data->flush_list = flush;
+    }
+    else {
+        while (iter->next != NULL)
+            iter = iter->next;
+        iter->next = flush;
+    }
+}
+#endif /* UNSUPPORTED_API */
+
+DR_API
+/* Flush all fragments that contain code from the region [start, start+size).
+ * Uses a synchall flush to guarantee that no execution occurs out of the fragments
+ * flushed once this returns. Requires caller to be holding no locks (dr or client) and
+ * to be !couldbelinking (xref PR 199115, 227619). Caller must use
+ * dr_redirect_execution() to return to the cache. */
+bool
+dr_flush_region(app_pc start, size_t size)
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    ASSERT(dcontext != NULL);
+
+    /* Flush requires !couldbelinking. FIXME - not all event callbacks to the client are
+     * !couldbelinking (see PR 227619) restricting where this routine can be used. */
+    CLIENT_ASSERT(!is_couldbelinking(dcontext), "dr_flush_region: called from an event "
+                  "callback that doesn't support calling this routine; see header file "
+                  "for restrictions.");
+    /* Flush requires caller to hold no locks that might block a couldbelinking thread
+     * (which includes almost all dr locks).  FIXME - some event callbacks are holding
+     * dr locks (see PR 227619) so can't call this routine.  Since we are going to use
+     * a synchall flush, holding client locks is disallowed too (could block a thread
+     * at an unsafe spot for synch). */
+    CLIENT_ASSERT(thread_owns_no_locks(dcontext), "dr_flush_region: caller owns a client "
+                  "lock or was called from an event callback that doesn't support "
+                  "calling this routine; see header file for restrictions.");
+    CLIENT_ASSERT(size != 0, "dr_flush_region: 0 is invalid size for flush");
+
+    /* release build check of requirements, as many as possible at least */
+    if (size == 0 || is_couldbelinking(dcontext))
+        return false;
+
+    if (!executable_vm_area_executed_from(start, start + size))
+        return true;
+
+    flush_fragments_from_region(dcontext, start, size, true/*force synchall*/);
+
+    return true;
+}
+
+DR_API
+/* Flush all fragments that contain code from the region [start, start+size).
+ * Uses an unlink flush which guarantees that no thread will enter a fragment that was
+ * flushed once this returns (threads already in a flushed fragment will continue).
+ * Requires caller to be holding no locks (dr or client) and to be !couldbelinking
+ * (xref PR 199115, 227619). */
+bool
+dr_unlink_flush_region(app_pc start, size_t size)
+{
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    ASSERT(dcontext != NULL);
+
+    /* This routine won't work with coarse_units */
+    CLIENT_ASSERT(!DYNAMO_OPTION(coarse_units),
+                  /* as of now, coarse_units are always disabled with -thread_private. */
+                  "dr_unlink_flush_region is not supported with -opt_memory unless -thread_private or -enable_full_api is also specified");
+
+    /* Flush requires !couldbelinking. FIXME - not all event callbacks to the client are
+     * !couldbelinking (see PR 227619) restricting where this routine can be used. */
+    CLIENT_ASSERT(!is_couldbelinking(dcontext), "dr_flush_region: called from an event "
+                  "callback that doesn't support calling this routine, see header file "
+                  "for restrictions.");
+    /* Flush requires caller to hold no locks that might block a couldbelinking thread
+     * (which includes almost all dr locks).  FIXME - some event callbacks are holding
+     * dr locks (see PR 227619) so can't call this routine.  FIXME - some event callbacks
+     * are couldbelinking (see PR 227619) so can't allow the caller to hold any client
+     * locks that could block threads in one of those events (otherwise we don't need
+     * to care about client locks) */
+    CLIENT_ASSERT(thread_owns_no_locks(dcontext), "dr_flush_region: caller owns a client "
+                  "lock or was called from an event callback that doesn't support "
+                  "calling this routine, see header file for restrictions.");
+    CLIENT_ASSERT(size != 0, "dr_unlink_flush_region: 0 is invalid size for flush");
+
+    /* release build check of requirements, as many as possible at least */
+    if (size == 0 || is_couldbelinking(dcontext))
+        return false;
+
+    if (!executable_vm_area_executed_from(start, start + size))
+        return true;
+
+    flush_fragments_from_region(dcontext, start, size, false/*don't force synchall*/);
+
+    return true;
+}
+
+DR_API
+/* Flush all fragments that contain code from the region [start, start+size) at the next
+ * convenient time.  Unlike dr_flush_region() this routine has no restrictions on lock
+ * or couldbelinking status; the downside is that the delay till the flush actually
+ * occurs is unbounded (FIXME - we could do something safely here to try to speed it
+ * up like unlinking shared_syscall etc.), but should occur before any new code is
+ * executed or any nudges are processed. */
+bool
+dr_delay_flush_region(app_pc start, size_t size, uint flush_id,
+                      void (*flush_completion_callback) (int flush_id))
+{
+    client_flush_req_t *flush;
+
+    if (size == 0) {
+        CLIENT_ASSERT(false, "dr_delay_flush_region: 0 is invalid size for flush");
+        return false;
+    }
+
+    /* FIXME - would be nice if we could check the requirements and call
+     * dr_unlink_flush_region() here if it's safe. Is difficult to detect non-dr locks
+     * that could block a couldbelinking thread though. */
+
+    flush = HEAP_TYPE_ALLOC(GLOBAL_DCONTEXT, client_flush_req_t, ACCT_OTHER, UNPROTECTED);
+    memset(flush, 0x0, sizeof(client_flush_req_t));
+    flush->start = (app_pc)start;
+    flush->size = size;
+    flush->flush_id = flush_id;
+    flush->flush_callback = flush_completion_callback;
+
+    mutex_lock(&client_flush_request_lock);
+    flush->next = client_flush_requests;
+    client_flush_requests = flush;
+    mutex_unlock(&client_flush_request_lock);
+
+    return true;
+}
+
+DR_API
+/* returns whether or not there is a fragment in the drcontext fcache at tag
+ */
+bool
+dr_fragment_exists_at(void *drcontext, void *tag)
+{
+    dcontext_t *dcontext = (dcontext_t *)drcontext;
+    fragment_t *f;
+#ifdef CLIENT_SIDELINE
+    fragment_get_fragment_delete_mutex(dcontext);
+#endif
+    f = fragment_lookup(dcontext, tag);
+#ifdef CLIENT_SIDELINE
+    fragment_release_fragment_delete_mutex(dcontext);
+#endif
+    return f != NULL;
+}
+
+DR_API
+bool
+dr_bb_exists_at(void *drcontext, void *tag)
+{
+    dcontext_t *dcontext = (dcontext_t *)drcontext;
+    fragment_t *f = fragment_lookup(dcontext, tag);
+    if (f != NULL && !TEST(FRAG_IS_TRACE, f->flags)) {
+        return true;
+    }
+
+    return false;
+}
+
+DR_API 
+/* Looks up the fragment associated with the application pc tag.
+ * If not found, returns 0.
+ * If found, returns the total size occupied in the cache by the fragment.
+ */
+uint
+dr_fragment_size(void *drcontext, void *tag)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    fragment_t *f;
+    int size = 0;
+    CLIENT_ASSERT(drcontext != NULL, "dr_fragment_size: drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_fragment_size: drcontext is invalid");
+#ifdef CLIENT_SIDELINE
+    /* used to check to see if owning thread, if so don't need lock */
+    /* but the check for owning thread more expensive then just getting lock */
+    /* to check if owner get_thread_id() == dcontext->owning_thread */
+    fragment_get_fragment_delete_mutex(dcontext);
+#endif
+    f = fragment_lookup(dcontext, tag);
+    if (f == NULL)
+        size = 0;
+    else
+        size = f->size;
+#ifdef CLIENT_SIDELINE
+    fragment_release_fragment_delete_mutex(dcontext);
+#endif
+    return size;
+}
+
+DR_API 
+/* Retrieves the application PC of a fragment */
+app_pc
+dr_fragment_app_pc(void *tag)
+{
+#ifdef WINDOWS
+    /* Only the returning (second) jump in a landing pad is interpreted, thus
+     * visible to a client.  The first jump is filtered out by
+     * must_not_be_elided().  The second jump will always be a 32-bit rel
+     * returning after the hook point (i.e., not the interception buffer).
+     */
+    if (vmvector_overlap(landing_pad_areas, (app_pc)tag, (app_pc)tag + 1)) {
+        ASSERT(*((byte *)tag) == JMP_REL32_OPCODE);
+        /* End of jump + relative address. */
+        tag = ((app_pc)tag + 5) + *(int *)((app_pc)tag + 1);
+        ASSERT(!is_in_interception_buffer(tag));
+    }
+
+    if (is_in_interception_buffer(tag))
+        tag = get_app_pc_from_intercept_pc(tag);
+    CLIENT_ASSERT(tag != NULL, "dr_fragment_app_pc shouldn't be NULL");
+
+    if (DYNAMO_OPTION(hide)) {
+#endif
+        CLIENT_ASSERT(!is_dynamo_address(tag), "dr_fragment_app_pc shouldn't be DR pc");
+#ifdef WINDOWS
+    } /* without -hide our DllMain routine ends up in the cache (xref PR 223120) */
+#endif
+    return tag;
+}
+
+
+/***************************************************************************
+ * CUSTOM TRACES SUPPORT
+ * *could use a method to unmark a trace head, would be nice if DR
+ * notified the client when it marked a trace head and gave the client a
+ * chance to override its decision
+ */
+
+DR_API
+/* Marks the fragment associated with the application pc tag as
+ * a trace head.  The fragment need not exist yet -- once it is
+ * created it will be marked as a trace head.
+ *
+ * DR associates a counter with a trace head and once it
+ * passes the -hot_threshold parameter, DR begins building
+ * a trace.  Before each fragment is added to the trace, DR
+ * calls the client routine dr_end_trace to determine whether
+ * to end the trace.  (dr_end_trace will be called both for
+ * standard DR traces and for client-defined traces.)
+ *
+ * Note, some fragments are unsuitable for trace heads. DR will 
+ * ignore attempts to mark such fragments as trace heads and will return
+ * false. If the client marks a fragment that doesn't exist yet as a trace
+ * head and DR later determines that the fragment is unsuitable for
+ * a trace head it will unmark the fragment as a trace head without 
+ * notifying the client.
+ *
+ * Returns true if the target fragment is marked as a trace head.
+ *
+ * If coarse, headness depends on path: currently this will only have
+ * links from tag's coarse unit unlinked.
+ */
+bool /* FIXME: dynamorio_app_init returns an int! */
+dr_mark_trace_head(void *drcontext, void *tag)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    fragment_t *f;
+    fragment_t coarse_f;
+    bool success = true;
+    CLIENT_ASSERT(drcontext != NULL, "dr_mark_trace_head: drcontext cannot be NULL");
+    CLIENT_ASSERT(drcontext != GLOBAL_DCONTEXT,
+                  "dr_mark_trace_head: drcontext is invalid");
+#ifdef CLIENT_SIDELINE
+    /* used to check to see if owning thread, if so don't need lock */
+    /* but the check for owning thread more expensive then just getting lock */
+    /* to check if owner get_thread_id() == dcontext->owning_thread */
+    fragment_get_fragment_delete_mutex(dcontext);
+#endif
+    f = fragment_lookup_fine_and_coarse(dcontext, tag, &coarse_f, NULL);
+    if (f == NULL) {
+        future_fragment_t *fut;
+        /* make the lookup and add atomic */
+        SHARED_FLAGS_RECURSIVE_LOCK(FRAG_SHARED, acquire, change_linking_lock);
+        fut = fragment_lookup_future(dcontext, tag);
+        if (fut == NULL) {
+            /* need to create a future fragment */
+            fut = fragment_create_and_add_future(dcontext, tag, FRAG_IS_TRACE_HEAD);
+        } else {
+            /* don't call mark_trace_head, it will try to do some linking */
+            fut->flags |= FRAG_IS_TRACE_HEAD;
+        }
+        SHARED_FLAGS_RECURSIVE_LOCK(FRAG_SHARED, release, change_linking_lock);
+#ifndef CLIENT_SIDELINE
+        LOG(THREAD, LOG_MONITOR, 2,
+            "Client mark trace head : will mark fragment as trace head when built "
+            ": address "PFX"\n", tag);
+#endif
+    } else {
+        /* check precluding conditions */
+        if (TEST(FRAG_IS_TRACE, f->flags)) {
+#ifndef CLIENT_SIDELINE
+            LOG(THREAD, LOG_MONITOR, 2,
+                "Client mark trace head : not marking as trace head, is already "
+                "a trace : address "PFX"\n", tag);
+#endif
+            success = false;
+        } else if (TEST(FRAG_CANNOT_BE_TRACE, f->flags)) {
+#ifndef CLIENT_SIDELINE
+            LOG(THREAD, LOG_MONITOR, 2,
+                "Client mark trace head : not marking as trace head, particular "
+                "fragment cannot be trace head : address "PFX"\n", tag);
+#endif
+            success = false;
+        } else if (TEST(FRAG_IS_TRACE_HEAD, f->flags)) {
+#ifndef CLIENT_SIDELINE
+            LOG(THREAD, LOG_MONITOR, 2,
+                "Client mark trace head : fragment already marked as trace head : "
+                "address "PFX"\n", tag);
+#endif
+            success =  true;
+        } else {
+            /* if reach here is all right to mark as trace head */
+            SHARED_FLAGS_RECURSIVE_LOCK(f->flags, acquire, change_linking_lock);
+            mark_trace_head(dcontext, f, NULL, NULL);
+            SHARED_FLAGS_RECURSIVE_LOCK(f->flags, release, change_linking_lock);
+#ifndef CLIENT_SIDELINE
+            LOG(THREAD, LOG_MONITOR, 3,
+                "Client mark trace head : just marked as trace head : address "PFX"\n",
+                tag);
+#endif
+        }
+    }
+#ifdef CLIENT_SIDELINE
+    fragment_release_fragment_delete_mutex(dcontext);
+#endif
+    return success;
+}
+
+DR_API
+/* Checks to see if the fragment (or future fragment) in the drcontext
+ * fcache at tag is marked as a trace head
+ */
+bool
+dr_trace_head_at(void *drcontext, void *tag) 
+{
+    dcontext_t *dcontext = (dcontext_t *)drcontext;
+    fragment_t *f;
+    bool trace_head;
+#ifdef CLIENT_SIDELINE
+    fragment_get_fragment_delete_mutex(dcontext);
+#endif
+    f = fragment_lookup(dcontext, tag);
+    if (f != NULL)
+        trace_head = (f->flags & FRAG_IS_TRACE_HEAD) != 0;
+    else {
+        future_fragment_t *fut = fragment_lookup_future(dcontext, tag);
+        if (fut != NULL)
+            trace_head = (fut->flags & FRAG_IS_TRACE_HEAD) != 0;
+        else
+            trace_head = false; 
+    }
+#ifdef CLIENT_SIDELINE
+    fragment_release_fragment_delete_mutex(dcontext);
+#endif
+    return trace_head;
+}
+
+DR_API
+/* checks to see that if there is a trace in the drcontext fcache at tag 
+ */
+bool
+dr_trace_exists_at(void *drcontext, void *tag)
+{
+    dcontext_t *dcontext = (dcontext_t *)drcontext;
+    fragment_t *f;
+    bool trace;
+#ifdef CLIENT_SIDELINE
+    fragment_get_fragment_delete_mutex(dcontext);
+#endif
+    f = fragment_lookup(dcontext, tag);
+    if (f != NULL)
+        trace = (f->flags & FRAG_IS_TRACE) != 0;
+    else
+        trace = false;
+#ifdef CLIENT_SIDELINE
+    fragment_release_fragment_delete_mutex(dcontext);
+#endif
+    return trace;
+}
+
+#ifdef UNSUPPORTED_API
+DR_API 
+/* All basic blocks created after this routine is called will have a prefix
+ * that restores the ecx register.  Exit ctis can be made to target this prefix
+ * instead of the normal entry point by using the instr_branch_set_prefix_target()
+ * routine.
+ * WARNING: this routine should almost always be called during client
+ * initialization, since having a mixture of prefixed and non-prefixed basic
+ * blocks can lead to trouble.
+ */
+void
+dr_add_prefixes_to_basic_blocks(void)
+{
+    if (DYNAMO_OPTION(coarse_units)) {
+        /* coarse_units doesn't support prefixes in general.
+         * the variation by addr prefix according to processor type
+         * is also not stored in pcaches.
+         */
+        CLIENT_ASSERT(false,
+                      "dr_add_prefixes_to_basic_blocks() not supported with -opt_memory");
+    }
+    options_make_writable();
+    dynamo_options.bb_prefixes = true;
+    options_restore_readonly();
+}
+#endif /* UNSUPPORTED_API */
+
+#endif /* CLIENT_INTERFACE */
+
diff --git a/core/x86/instrument.h b/core/x86/instrument.h
new file mode 100644
index 000000000..352af1084
--- /dev/null
+++ b/core/x86/instrument.h
@@ -0,0 +1,2847 @@
+/* **********************************************************
+ * Copyright (c) 2002-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+/*
+ * instrument.h - interface for instrumentation
+ */
+
+#ifndef _INSTRUMENT_H_
+#define _INSTRUMENT_H_ 1
+
+#include "globals.h"
+#include "module_shared.h"
+#include "arch.h"
+#include "instr.h"
+
+/* xref _USES_DR_VERSION_ in dr_api.h (PR 250952) and compatibility
+ * check in instrument.c (OLDEST_COMPATIBLE_VERSION, etc.).
+ * this is defined outside of CLIENT_INTERFACE b/c it's used for
+ * a general tracedump version as well.
+ */
+#define CURRENT_API_VERSION 103 /* 1.3.x */
+
+#ifdef CLIENT_INTERFACE
+
+/* DR_API EXPORT TOFILE dr_events.h */
+/* DR_API EXPORT BEGIN */
+
+/**************************************************
+ * ROUTINES TO REGISTER EVENT CALLBACKS
+ */
+/**
+ * @file dr_events.h
+ * @brief Event callback registration routines.
+ */
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Registers a callback function for the process exit event.  DR calls
+ * \p func when the process exits.
+ *
+ * On Linux, SYS_execve does NOT result in an exit event, but it WILL
+ * result in the client library being reloaded and its dr_init()
+ * routine being called.
+ */
+void
+dr_register_exit_event(void (*func)(void));
+
+DR_API
+/**
+ * Unregister a callback function for the process exit event.
+ * \return true if unregistration is successful and false if it is not
+ * (e.g., \p func was not registered).
+ */
+bool
+dr_unregister_exit_event(void (*func)(void));
+
+/* DR_API EXPORT TOFILE dr_events.h */
+/* DR_API EXPORT BEGIN */
+/**
+ * Flags controlling the behavior of basic blocks and traces when emitted
+ * into the code cache.  For multiple clients, the flags returned by each
+ * client are or-ed together.
+ */
+typedef enum {
+    /** Emit as normal. */
+    DR_EMIT_DEFAULT              =    0,
+    /** 
+     * Store translation information at emit time rather than calling
+     * the basic block or trace event later to recreate the
+     * information.  Note that even if a standalone basic block has
+     * stored translations, if when it is added to a trace it does not
+     * request storage (and the trace callback also does not request
+     * storage) then the basic block callback may still be called to
+     * translate for the trace.
+     *
+     * \sa #dr_register_bb_event()
+     */
+    DR_EMIT_STORE_TRANSLATIONS   = 0x01,
+} dr_emit_flags_t;
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Registers a callback function for the basic block event.  DR calls
+ * \p func before inserting a new basic block into the code cache.
+ * When adding a basic block to a new trace, DR calls \p func again
+ * with \p for_trace set to true, giving the client the opportunity to
+ * keep its same instrumentation in the trace, or to change it.  The
+ * original basic block's instrumentation is unchanged by whatever
+ * action is taken in the \p for_trace call.
+ *
+ * DR may call \p func again if it needs to translate from code cache
+ * addresses back to application addresses, which happens on faulting
+ * instructions as well as in certain situations involving suspended
+ * threads or forcibly relocated threads.  The \p translating
+ * parameter distinguishes the two types of calls and is further
+ * explained below.
+ *
+ * - \p drcontext is a pointer to the input program's machine context.
+ * Clients should not inspect or modify the context; it is provided as
+ * an opaque pointer (i.e., <tt>void *</tt>) to be passed to API
+ * routines that require access to this internal data.
+ * drcontext is specific to the current thread, but in normal
+ * configurations the basic block being created is thread-shared: thus,
+ * when allocating data structures with the same lifetime as the
+ * basic block, usually global heap (#dr_global_alloc()) is a better
+ * choice than heap tied to the thread that happened to first create
+ * the basic block (#dr_thread_alloc()).  Thread-private heap is fine
+ * for temporary structures such as instr_t and instrlist_t.
+ *
+ * - \p tag is a unique identifier for the basic block fragment.
+ * - \p bb is a pointer to the list of instructions that comprise the
+ * basic block.  Clients can examine, manipulate, or completely
+ * replace the instructions in the list.
+ *
+ * - \p translating indicates whether this callback is for basic block
+ * creation (false) or is for address translation (true).  This is
+ * further explained below.
+ *
+ * \return a #dr_emit_flags_t flag.
+ *
+ * The user is free to inspect and modify the block before it
+ * executes, but must adhere to the following restrictions:
+ * - If there is more than one non-meta branch, only the last can be
+ * conditional.
+ * - A non-meta conditional branch or direct call must be the final
+ * instruction in the block.
+ * - There can only be one indirect branch (call, jump, or return) in
+ * a basic block, and it must be the final instruction in the
+ * block.
+ * - The exit control-flow of a block ending in a system call or
+ * int instruction cannot be changed, nor can instructions be inserted
+ * after the system call or int instruction itself, unless
+ * the system call or int instruction is removed entirely.
+ * - The number of an interrupt cannot be changed.  (Note that the
+ * parameter to a system call, normally kept in the eax register, can
+ * be freely changed in a basic block: but not in a trace.)
+ * - A system call or interrupt instruction can only be added
+ * if it satisfies the above constraints: i.e., if it is the final
+ * instruction in the block and the only system call or interrupt.
+ * - The block's application source code (as indicated by the
+ * translation targets, set by #instr_set_translation()) must remain
+ * within the original bounds of the block (the one exception to this
+ * is that a jump can translate to its target).  Otherwise, DR's cache
+ * consistency algorithms cannot guarantee to properly invalidate the
+ * block if the source application code is modified.  To send control
+ * to other application code regions, truncate the block and use a
+ * direct jump to target the desired address, which will then
+ * materialize in the subsequent block, rather than embedding the
+ * desired instructions in this block.
+ * - There is a limit on the size of a basic block in the code cache.
+ * DR performs its own modifications, especially on memory writes for
+ * cache consistency of self-modifying (or false sharing) code
+ * regions.  If an assert fires in debug build indicating a limit was
+ * reached, either truncate blocks or use the -max_bb_instrs runtime
+ * option to ask DR to make them smaller.
+ *
+ * To support transparent fault handling, DR must translate a fault in the
+ * code cache into a fault at the corresponding application address.  DR
+ * must also be able to translate when a suspended thread is examined by
+ * the application or by DR itself for internal synchronization purposes.
+ * If the client is only adding observational instrumentation (i.e., meta
+ * instructions: see #instr_set_ok_to_mangle()) (which should not fault) and
+ * is not modifying, reordering, or removing application instructions,
+ * these details can be ignored.  In that case the client should return
+ * #DR_EMIT_DEFAULT and set up its basic block callback to be deterministic
+ * and idempotent.  If the client is performing modifications, then in
+ * order for DR to properly translate a code cache address the client must
+ * use #instr_set_translation() in the basic block creation callback to set
+ * the corresponding application address (the address that should be
+ * presented to the application as the faulting address, or the address
+ * that should be restarted after a suspend) for each modified instruction
+ * and each added non-meta instruction (see #instr_set_ok_to_mangle()).
+ *
+ * There are two methods for using the translated addresses:
+ *
+ * -# Return #DR_EMIT_STORE_TRANSLATIONS from the basic block creation
+ *    callback.  DR will then store the translation addresses and use
+ *    the stored information on a fault.  The basic block callback for
+ *    \p tag will not be called with \p translating set to true.  Note
+ *    that unless #DR_EMIT_STORE_TRANSLATIONS is also returned for \p
+ *    for_trace calls (or #DR_EMIT_STORE_TRANSLATIONS is returned in
+ *    the trace callback), each constituent block comprising the trace
+ *    will need to be re-created with both \p for_trace and \p
+ *    translating set to true.  Storing translations uses additional
+ *    memory that can be significant: up to 20% in some cases, as it
+ *    prevents DR from using its simple data structures and forces it
+ *    to fall back to its complex, corner-case design.  This is why DR
+ *    does not store all translations by default.
+ * -# Return #DR_EMIT_DEFAULT from the basic block creation callback.
+ *    DR will then call the callback again during fault translation
+ *    with \p translating set to true.  All modifications to \p bb
+ *    that were performed on the creation callback must be repeated on
+ *    the translating callback.  This option is only posible when
+ *    basic block modifications are deterministic and idempotent, but
+ *    it saves memory.  Naturally, global state changes triggered by
+ *    block creation should be wrapped in checks for \p translating
+ *    being false.  Even in this case, #instr_set_translation() should
+ *    be called for non-meta instructions even when \p translating is
+ *    false, as DR may decide to store the translations at creation
+ *    time for reasons of its own.
+ *
+ * Furthermore, if the client's modifications change any part of the
+ * machine state besides the program counter, the client should use
+ * #dr_register_restore_state_event() to restore the registers and
+ * application memory to their original application values.
+ *
+ * For meta instructions that do not reference application memory
+ * (i.e., they should not fault), leave the translation field as NULL.
+ * A NULL value instructs DR to use the subsequent non-meta
+ * instruction's translation as the application address, and to fail
+ * when translating the full state.  Since the full state will only be
+ * needed when relocating a thread (as stated, there will not be a
+ * fault here), failure indicates that this is not a valid relocation
+ * point, and DR's thread synchronization scheme will use another
+ * spot.  If the translation field is set to a non-NULL value, the
+ * client should be willing to also restore the rest of the machine
+ * state at that point (restore spilled registers, etc.) via
+ * #dr_register_restore_state_event().  This is necessary for meta
+ * instructions that reference application memory.  DR takes care of
+ * such potentially-faulting instructions added by its own API
+ * routines (#dr_insert_clean_call() arguments that reference
+ * application data, #dr_insert_mbr_instrumentation()'s read of
+ * application indirect branch data, etc.)
+ *
+ * \note In order to present a more straightforward code stream to clients,
+ * this release of DR disables several internal optimizations.  As a result,
+ * some applications may see a performance degradation.  Applications making
+ * heavy use of system calls are the most likely to be affected.
+ * Future releases may allow clients some control over performance versus
+ * visibility.
+ *
+ * \note If multiple clients are present, the instruction list for a
+ * basic block passed to earlier-registered clients will contain the
+ * instrumentation and modifications put in place by later-registered
+ * clients.
+ * 
+ * \note Basic blocks can be deleted due to hitting capacity limits or
+ * cache consistency events (when the source application code of a
+ * basic block is modified).  In that case, the client will see a new
+ * basic block callback if the block is then executed again after
+ * deletion.  The deletion event (#dr_register_delete_event()) will be
+ * raised at deletion time.
+ *
+ * \note If the -thread_private runtime option is specified, clients
+ * should expect to see duplicate tags for separate threads, albeit
+ * with different dcrcontext values.  Additionally, DR employs a
+ * cache-sizing algorithm for thread private operation that
+ * proactively deletes fragments.  Even with thread-shared caches
+ * enabled, however, certain situations cause DR to emit
+ * thread-private basic blocks (e.g., self-modifying code).  In this
+ * case, clients should be prepared to see duplicate tags without an
+ * intermediate deletion.
+ */
+void
+dr_register_bb_event(dr_emit_flags_t (*func)
+                     (void *drcontext, void *tag, instrlist_t *bb,
+                      bool for_trace, bool translating));
+
+DR_API
+/**
+ * Unregister a callback function for the basic block event.
+ * \return true if unregistration is successful and false if it is not
+ * (e.g., \p func was not registered).
+ *
+ * \note We do not recommend unregistering for the basic block event
+ * unless it aways returned #DR_EMIT_STORE_TRANSLATIONS (including
+ * when \p for_trace is true, or if the client has a trace creation
+ * callback that returns #DR_EMIT_STORE_TRANSLATIONS).  Unregistering
+ * can prevent proper state translation on a later fault or other
+ * translation event for this basic block or for a trace that includes
+ * this basic block.  Instead of unregistering, turn the event
+ * callback into a nop.
+ */
+bool
+dr_unregister_bb_event(dr_emit_flags_t (*func)
+                       (void *drcontext, void *tag, instrlist_t *bb,
+                        bool for_trace, bool translating));
+
+DR_API
+/**
+ * Registers a callback function for the trace event.  DR calls \p func
+ * before inserting a new trace into the code cache.  DR may call \p func
+ * again if it needs to translate from code cache addresses back to
+ * application addresses, which happens on faulting instructions as well as
+ * in certain situations involving suspended threads or forcibly relocated
+ * threads.  The \p translating parameter distinguishes the two types of
+ * calls and behaves identically to the same parameter in the basic
+ * block callback: see #dr_register_bb_event() for further details.
+ *
+ * Traces are not built if the -disable_traces runtime option
+ * is specified.
+ *
+ * - \p drcontext is a pointer to the input program's machine context.
+ * Clients should not inspect or modify the context; it is provided as
+ * an opaque pointer (i.e., <tt>void *</tt>) to be passed to API
+ * routines that require access to this internal data.
+ * - \p tag is a unique identifier for the trace fragment.
+ * - \p trace is a pointer to the list of instructions that comprise the
+ * trace.
+ * - \p translating indicates whether this callback is for trace creation
+ * (false) or is for fault address recreation (true).  This is further
+ * explained below.
+ *
+ * \return a #dr_emit_flags_t flag.
+ *
+ * The user is free to inspect and modify the trace before it
+ * executes, with certain restrictions on introducing control-flow
+ * that include those for basic blocks (see dr_register_bb_event()).
+ * Additional restrictions unique to traces also apply:
+ * - Only one non-meta direct branch that targets the subsequent block
+ *   in the trace can be present in each block.
+ * - Each block must end with a non-meta control transfer.
+ * - The parameter to a system call, normally kept in the eax register, 
+ *   cannot be changed.
+ * - A system call or interrupt instruction cannot be added.
+ *
+ * If hitting a size limit due to extensive instrumentation, reduce
+ * the -max_trace_bbs option to start with a smaller trace.
+ *
+ * The basic block restrictions on modifying application source code
+ * apply to traces as well.  If the user wishes to change which basic
+ * blocks comprise the trace, either the
+ * #dr_register_end_trace_event() should be used or the \p for_trace
+ * basic block callbacks should modify their continuation addresses
+ * via direct jumps.
+ *
+ * All of the comments for #dr_register_bb_event() regarding
+ * transparent fault handling and state translation apply to the trace
+ * callback as well.  Please read those comments carefully.
+ *
+ * \note As each basic block is added to a new trace, the basic block
+ * callback (see #dr_register_bb_event()) is called with its \p
+ * for_trace parameter set to true.  In order to preserve basic block
+ * instrumentation inside of traces, a client need only act
+ * identically with respect to the \p for_trace parameter; it can
+ * ignore the trace event if its goal is to place instrumentation
+ * on all code.
+ *
+ * \note Certain control flow modifications applied to a basic block
+ * can prevent it from becoming part of a trace: e.g., adding
+ * additional non-meta control transfers.
+ * 
+ * \note If multiple clients are present, the instruction list for a
+ * trace passed to earlier-registered clients will contain the
+ * instrumentation and modifications put in place by later-registered
+ * clients; similarly for each constituent basic block.
+ *
+ * \note Traces can be deleted due to hitting capacity limits or cache
+ * consistency events (when the source application code of a trace is
+ * modified).  In that case, the client will see a new trace callback
+ * if a new trace containing that code is created again after
+ * deletion.  The deletion event (#dr_register_delete_event()) will be
+ * raised at deletion time.
+ */
+void
+dr_register_trace_event(dr_emit_flags_t (*func)
+                        (void *drcontext, void *tag, instrlist_t *trace,
+                         bool translating));
+
+DR_API
+/**
+ * Unregister a callback function for the trace event.
+ * \return true if unregistration is successful and false if it is not
+ * (e.g., \p func was not registered).
+ *
+ * \note We do not recommend unregistering for the trace event unless it
+ * always returned #DR_EMIT_STORE_TRANSLATIONS, as doing so can prevent
+ * proper state translation on a later fault or other translation event.
+ * Instead of unregistering, turn the event callback into a nop.
+ */
+bool
+dr_unregister_trace_event(dr_emit_flags_t (*func)
+                          (void *drcontext, void *tag, instrlist_t *trace,
+                           bool translating));
+
+#ifdef CUSTOM_TRACES
+/* DR_API EXPORT BEGIN */
+
+/**
+ * DR will call the end trace event if it is registered prior to
+ * adding each basic block to a trace being generated.  The return
+ * value of the event callback should be from the
+ * dr_custom_trace_action_t enum.
+ *
+ * \note DR treats CUSTOM_TRACE_CONTINUE as an advisement only.  Certain
+ * fragments are not suitable to be included in a trace and if DR runs
+ * into one it will end the trace regardless of what the client returns
+ * through the event callback.
+ */
+typedef enum {
+    CUSTOM_TRACE_DR_DECIDES,
+    CUSTOM_TRACE_END_NOW,
+    CUSTOM_TRACE_CONTINUE
+} dr_custom_trace_action_t;
+
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Registers a callback function for the end-trace event.  DR calls \p
+ * func before extending a trace with a new basic block.  The \p func
+ * should return one of the #dr_custom_trace_action_t enum values.
+ */
+void
+dr_register_end_trace_event(dr_custom_trace_action_t (*func)
+                            (void *drcontext, void *tag, void *next_tag));
+
+DR_API
+/**
+ * Unregister a callback function for the end-trace event.
+ * \return true if unregistration is successful and false if it is not
+ * (e.g., \p func was not registered).
+ */
+bool
+dr_unregister_end_trace_event(dr_custom_trace_action_t (*func)
+                              (void *drcontext, void *tag, void *next_tag));
+#endif
+
+DR_API
+/**
+ * Registers a callback function for the fragment deletion event.  DR
+ * calls \p func whenever it removes a fragment from the code cache.
+ * \note drcontext may be NULL when thread-shared fragments are being
+ * deleted during process exit.  For this reason, thread-private
+ * heap should not be used for data structures intended to be freed
+ * at thread-shared fragment deletion.
+ */
+void
+dr_register_delete_event(void (*func)(void *drcontext, void *tag));
+
+DR_API
+/**
+ * Unregister a callback function for the fragment deletion event.
+ * \return true if unregistration is successful and false if it is not
+ * (e.g., \p func was not registered).
+ */
+bool
+dr_unregister_delete_event(void (*func)(void *drcontext, void *tag));
+
+DR_API
+/**
+ * Registers a callback function for the machine state restoration event.
+ * DR calls \p func whenever it needs to translate a code cache machine
+ * context from the code cache to its corresponding original application
+ * context.  DR needs to translate when instructions fault in the cache as
+ * well as when a suspended thread is examined or relocated for internal
+ * purposes.
+ *
+ * If a client is only adding instrumentation (meta-code: see
+ * #instr_ok_to_mangle()) that does not reference application memory,
+ * and is not reordering or removing application instructions, then it
+ * need not register for this event.  If, however, a client is
+ * modifying application code or is adding code that can fault, the
+ * client must be capable of restoring the original context.
+ *
+ * When DR needs to translate a code cache context, DR recreates the
+ * faulting instruction's containing fragment, storing translation
+ * information along the way, by calling the basic block and/or trace event
+ * callbacks with the \p translating parameter set to true.  DR uses the
+ * recreated code to identify the application instruction (\p mcontext.pc)
+ * corresponding to the faulting code cache instruction.  If the client
+ * asked to store translation information by returning
+ * #DR_EMIT_STORE_TRANSLATIONS from the basic block or trace event
+ * callback, then this step of re-calling the event callback is skipped and
+ * the stored value is used as the application address (\p mcontext.pc).
+ *
+ * DR then calls the fault state restoration event to allow the client
+ * to restore the registers and application memory to their proper
+ * values as they would have appeared if the original application code
+ * had been executed up to the \p mcontext.pc instruction.  Memory
+ * should only be restored if the \p restore_memory parameter is true;
+ * if it is false, DR may only be querying for the address (\p
+ * mcontext.pc) or register state and may not relocate this thread.
+ *
+ * The \p app_code_consistent parameter indicates whether the original
+ * application code containing the instruction being translated is
+ * guaranteed to still be in the same state it was when the code was
+ * placed in the code cache.  This guarantee varies depending on the
+ * type of cache consistency being used by DR.
+ *
+ * The client can update \p mcontext.pc in this callback.
+ */
+void
+dr_register_restore_state_event(void (*func)
+                                (void *drcontext, void *tag, dr_mcontext_t *mcontext,
+                                 bool restore_memory, bool app_code_consistent));
+
+DR_API
+/**
+ * Unregister a callback function for the fault state restoration event.
+ * \return true if unregistration is successful and false if it is not
+ * (e.g., \p func was not registered).
+ */
+bool
+dr_unregister_restore_state_event(void (*func)
+                                  (void *drcontext, void *tag, dr_mcontext_t *mcontext,
+                                   bool restore_memory, bool app_code_consistent));
+
+DR_API 
+/**
+ * Registers a callback function for the thread initialization event.
+ * DR calls \p func whenever the application creates a new thread.
+ */
+void
+dr_register_thread_init_event(void (*func)(void *drcontext));
+
+DR_API
+/**
+ * Unregister a callback function for the thread initialization event.
+ * \return true if unregistration is successful and false if it is not
+ * (e.g., \p func was not registered).
+ */
+bool
+dr_unregister_thread_init_event(void (*func)(void *drcontext));
+
+DR_API
+/**
+ * Registers a callback function for the thread exit event.  DR calls
+ * \p func whenever an application thread exits.
+ */
+void
+dr_register_thread_exit_event(void (*func)(void *drcontext));
+
+DR_API
+/**
+ * Unregister a callback function for the thread exit event.
+ * \return true if unregistration is successful and false if it is not
+ * (e.g., \p func was not registered).
+ */
+bool
+dr_unregister_thread_exit_event(void (*func)(void *drcontext));
+
+#ifdef LINUX
+DR_API
+/**
+ * Registers a callback function for the fork event.  DR calls \p func
+ * whenever the application forks a new process.
+ * \note Valid on Linux only.
+ */
+void
+dr_register_fork_init_event(void (*func)(void *drcontext));
+
+DR_API
+/**
+ * Unregister a callback function for the fork event.
+ * \return true if unregistration is successful and false if it is not
+ * (e.g., \p func was not registered).
+ */
+bool
+dr_unregister_fork_init_event(void (*func)(void *drcontext));
+#endif
+
+DR_API
+/**
+ * Registers a callback function for the module load event.  DR calls
+ * \p func whenever the application loads a module.  The \p loaded
+ * parameter indicates whether the module is about to be loaded (the
+ * normal case) or is already loaded (if the module was already there
+ * at the time DR initialized). \note The client should be aware that
+ * if the module is being loaded it may not be fully processed by the
+ * loader (relocating, rebinding and on Linux segment remapping may
+ * have not yet occured). n\note The module_data_t \p *info passed
+ * to the callback routine is valid only for the duration of the
+ * callback and should not be freed; a persistent copy can be made with
+ * dr_copy_module_data().
+ */
+void
+dr_register_module_load_event(void (*func)(void *drcontext, const module_data_t *info,
+                                           bool loaded));
+
+DR_API
+/**
+ * Unregister a callback for the module load event.
+ * \return true if unregistration is successful and false if it is not
+ * (e.g., \p func was not registered).
+ */
+bool
+dr_unregister_module_load_event(void (*func)(void *drcontext, const module_data_t *info,
+                                             bool loaded));
+
+DR_API
+/**
+ * Registers a callback function for the module unload event.  DR
+ * calls \p func whenever the application unloads a module.
+ * \note The module_data_t \p *info passed to
+ * the callback routine is valid only for the duration of the callback
+ * and should not be freed; a persistent copy can be made with
+ * dr_copy_module_data().
+ */
+void
+dr_register_module_unload_event(void (*func)(void *drcontext,
+                                             const module_data_t *info));
+
+DR_API
+/**
+ * Unregister a callback function for the module unload event.
+ * \return true if unregistration is successful and false if it is not
+ * (e.g., \p func was not registered).
+ */
+bool
+dr_unregister_module_unload_event(void (*func)(void *drcontext,
+                                               const module_data_t *info));
+
+/* DR_API EXPORT BEGIN */
+#ifdef WINDOWS
+/* DR_API EXPORT END */
+
+/* DR_API EXPORT BEGIN */
+/**
+ * Data structure passed with an exception event.  Contains the
+ * machine context and the Win32 exception record.
+ */
+typedef struct _dr_exception_t {
+    dr_mcontext_t mcontext;   /**< Machine context at exception point. */
+    EXCEPTION_RECORD *record; /**< Win32 exception record. */
+} dr_exception_t;
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Registers a callback function for the exception event.  DR calls \p
+ * func whenever the application throws an exception.  \note Only
+ * valid on Windows. \note The function is not called for
+ * RaiseException.
+ */
+void
+dr_register_exception_event(void (*func)(void *drcontext, dr_exception_t *excpt));
+
+DR_API
+/**
+ * Unregister a callback function for the exception event.
+ * \return true if unregistration is successful and false if it is not
+ * (e.g., \p func was not registered).
+ */
+bool
+dr_unregister_exception_event(void (*func)(void *drcontext, dr_exception_t *excpt));
+/* DR_API EXPORT BEGIN */
+#endif /* WINDOWS */
+/* DR_API EXPORT END */
+
+
+DR_API
+/**
+ * Registers a callback function for the syscall filter event.  DR
+ * calls \p func to decide whether to invoke the syscall events for
+ * each system call site encountered with a statically-determinable
+ * system call number.  If \p func returns true, the pre-syscall
+ * (dr_register_pre_syscall_event()) and post-syscall
+ * (dr_register_post_syscall_event()) events will be invoked.
+ * Otherwise, the events may or may not occur, depending on whether DR
+ * itself needs to intercept them and whether the system call number
+ * is statically determinable.  System call number determination can
+ * depend on whether the -opt_speed option is enabled.  If a system
+ * call number is not determinable, the filter event will not be
+ * called, but the pre and post events will be called.
+ *
+ * Intercepting every system call can be detrimental to performance
+ * for certain types of applications.  Filtering provides for greater
+ * performance by letting uninteresting system calls execute without
+ * interception overhead.
+ */
+void
+dr_register_filter_syscall_event(bool (*func)(void *drcontext, int sysnum));
+
+DR_API
+/**
+ * Unregister a callback function for the syscall filter event.
+ * \return true if unregistration is successful and false if it is not
+ * (e.g., \p func was not registered).
+ */
+bool
+dr_unregister_filter_syscall_event(bool (*func)(void *drcontext, int sysnum));
+
+DR_API
+/**
+ * Registers a callback function for the pre-syscall event.  DR calls
+ * \p func whenever the application is about to invoke a system call,
+ * if any client asked for that system call number to be intercepted
+ * via the filter event (dr_register_filter_syscall_event()).
+ *
+ * The application parameters to the system call can be viewed with
+ * dr_syscall_get_param() and set with dr_syscall_set_param().  The
+ * system call number can also be changed with
+ * dr_syscall_set_sysnum().
+ *
+ * The application's machine state can be accessed and set with
+ * dr_get_mcontext() and dr_set_mcontext().  Changing registers in
+ * this way overlaps with system call parameter changes on some
+ * platforms.  On Linux, for SYS_clone, client changes to the ebp/rbp
+ * register will be ignored by the clone child.
+ * 
+ * If \p func returns true, the application's system call is invoked
+ * normally; if \p func returns false, the system call is skipped.  If
+ * it is skipped, the return value can be set with
+ * dr_syscall_set_result().  If the system call is skipped, there will
+ * not be a post-syscall event.
+ */
+void
+dr_register_pre_syscall_event(bool (*func)(void *drcontext, int sysnum));
+
+DR_API
+/**
+ * Unregister a callback function for the pre-syscall event.
+ * \return true if unregistration is successful and false if it is not
+ * (e.g., \p func was not registered).
+ */
+bool
+dr_unregister_pre_syscall_event(bool (*func)(void *drcontext, int sysnum));
+
+DR_API
+/**
+ * Registers a callback function for the post-syscall event.  DR calls
+ * \p func whenever the application just finished invoking a system
+ * call, if any client asked for that system call number to be
+ * intercepted via the filter event
+ * (dr_register_filter_syscall_event()) or if DR itself needs to
+ * intercept the system call.  The result of the system call can be
+ * modified with dr_syscall_set_result().
+ *
+ * System calls that change control flow or terminate the current
+ * thread or process typically do not have a post-syscall event.
+ * These include SYS_exit, SYS_exit_group, SYS_execve, SYS_sigreturn,
+ * and SYS_rt_sigreturn on Linux, and NtTerminateThread,
+ * NtTerminateProcess (depending on the parameters), NtCallbackReturn,
+ * and NtContinue on Windows.
+ *
+ * The application's machine state can be accessed and set with
+ * dr_get_mcontext() and dr_set_mcontext().
+ * 
+ * Additional system calls may be invoked by calling
+ * dr_syscall_invoke_another() prior to returning from the
+ * post-syscall event callback.  The system call to be invoked should
+ * be specified with dr_syscall_set_sysnum(), and its parameters can
+ * be set with dr_syscall_set_param().
+ */
+void
+dr_register_post_syscall_event(void (*func)(void *drcontext, int sysnum));
+
+DR_API
+/**
+ * Unregister a callback function for the post-syscall event.
+ * \return true if unregistration is successful and false if it is not
+ * (e.g., \p func was not registered).
+ */
+bool
+dr_unregister_post_syscall_event(void (*func)(void *drcontext, int sysnum));
+
+
+/* DR_API EXPORT BEGIN */
+
+#ifdef LINUX
+/* DR_API EXPORT END */
+
+/* FIXME: for PR 304708 I originally included siginfo_t in
+ * dr_siginfo_t.  But can we really trust siginfo_t to be identical on
+ * all supported platforms?  Esp. once we start supporting VMKUW,
+ * MacOS, etc.  I'm removing it for now.  None of my samples need it,
+ * and in my experience its fields are unreliable in any case.
+ * PR 371370 covers re-adding it if users request it.
+ * Xref PR 371339: we will need to not include it through signal.h but
+ * instead something like this:
+ *   #  define __need_siginfo_t
+ *   #  include <bits/siginfo.h>
+ */
+/* DR_API EXPORT BEGIN */
+/**
+ * Data structure passed with a signal event.  Contains the machine
+ * context at the signal interruption point and other signal
+ * information.
+ */
+typedef struct _dr_siginfo_t {
+    int sig;                /**< The signal number. */
+    void *drcontext;        /**< The context of the thread receiving the signal. */
+    dr_mcontext_t mcontext; /**< The machine state at the signal interruption point. */
+} dr_siginfo_t;
+
+/**
+ * Return value of client signal event callback, determining how DR will
+ * proceed with the signal.
+ */
+typedef enum {
+    /** Deliver signal to the application as normal. */
+    DR_SIGNAL_DELIVER,
+    /** Suppress signal as though it never happened. */
+    DR_SIGNAL_SUPPRESS,
+    /**
+     * Deliver signal according to the default SIG_DFL action, as would
+     * happen if the application had no handler.
+     */
+    DR_SIGNAL_BYPASS,
+    /**
+     * Do not deliver the signal.  Instead, redirect control to the
+     * application state specified in dr_siginfo_t.mcontext.
+     */
+    DR_SIGNAL_REDIRECT,
+} dr_signal_action_t;
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Requests that DR call the provided callback function \p func whenever a
+ * signal is received by any application thread.  The return value of \p
+ * func determines whether DR delivers the signal to the application.  The
+ * callback function will be called even if the application has no handler
+ * or has registered a SIG_IGN or SIG_DFL handler.
+ *
+ * Modifications to the fields of \p siginfo->mcontext will be
+ * propagated to the application if it has a handler for the signal.
+ *
+ * DR raises a signal event only when about to devlier a signal to the
+ * application.  Thus, if the application has blocked a signal, the
+ * corresponding signal event will not occur until the application unblocks
+ * the signal, even if such a signal is delivered by the kernel.  DR will
+ * not raise a signal event for a SIGSEGV or SIGBUS raised by a client code
+ * fault rather than the application.  Use dr_safe_read() or
+ * dr_safe_write() to prevent such faults.
+ * 
+ * \note Only valid on Linux.
+ *
+ * \note DR always requests SA_SIGINFO for all signals.
+ *
+ * \note This version of DR does not intercept the signals SIGCONT,
+ * SIGSTOP, SIGTSTP, SIGTTIN, or SIGTTOU.  Future versions should add
+ * support for these signals.
+ *
+ * \note If the client uses signals for its own communication it should set
+ * a flag to distinguish its own uses of signals from the application's
+ * use.  Races where the two are re-ordered should not be problematic.
+ */
+void
+dr_register_signal_event(dr_signal_action_t (*func)
+                         (void *drcontext, dr_siginfo_t *siginfo));
+
+DR_API
+/**
+ * Unregister a callback function for the signal event.
+ * \return true if unregistration is successful and false if it is not
+ * (e.g., \p func was not registered).
+ */
+bool
+dr_unregister_signal_event(dr_signal_action_t (*func)
+                           (void *drcontext, dr_siginfo_t *siginfo));
+/* DR_API EXPORT BEGIN */
+#endif /* LINUX */
+/* DR_API EXPORT END */
+
+
+/****************************************************************************
+ * SECURITY SUPPORT
+ */
+
+#ifdef PROGRAM_SHEPHERDING
+/* DR_API EXPORT BEGIN */
+
+/** Types of security violations that can be received at a security violation event
+ *  callback.
+ *  - DR_RCO_* :
+ *      A violation of the Restricted Code Origins policies.  The target address is not
+ *      in an allowed execution area.
+ *      - DR_RCO_STACK_VIOLATION - The target address is on the current thread's stack.
+ *      - DR_RCO_HEAP_VIOLATION - The target address is not on the current thread's stack.
+ *  - DR_RCT_* :
+ *      A violation of the Restricted Control Transfer policies.  The transition from the
+ *      source address to the target address is not allowed.
+ *      - DR_RCT_RETURN_VIOLATION - The transition from source_pc to target_pc is via a
+ *        return instruction.  The target address does not follow an executed call
+ *        instruction and is not exempted.
+ *      - DR_RCT_INDIRECT_CALL_VIOLATION - The transition from source_pc to target_pc is
+ *        via an indirect call instruction.
+ *      - DR_RCT_INDIRECT_JUMP_VIOLATION - The transition from source_pc to target_pc is
+ *        via an indirect jmp instruction.
+ *  - DR_UNKNOWN_VIOLATION :
+ *      An unknown violation type, the client shouldn't expect to see this.
+ */
+typedef enum {
+    DR_RCO_STACK_VIOLATION,
+    DR_RCO_HEAP_VIOLATION,
+    DR_RCT_RETURN_VIOLATION,
+    DR_RCT_INDIRECT_CALL_VIOLATION,
+    DR_RCT_INDIRECT_JUMP_VIOLATION,
+    DR_UNKNOWN_VIOLATION,
+} dr_security_violation_type_t;
+
+/** Types of remediations available at a security violation event callback.
+ *  - DR_VIOLATION_ACTION_CONTINUE :
+ *       Continue application execution as if no violation occurred. Use this if the
+ *       violation is determined to be a false positive.
+ *  - DR_VIOLATION_ACTION_CONTINUE_CHANGED_CONTEXT :
+ *       Continue application execution after applying any changes made to the mcontext.
+ *       Use this to fix up the application's state and continue execution.
+ *  - DR_VIOLATION_ACTION_KILL_PROCESS :
+ *       Immediately kills the process.  This is the safest course of action to take
+ *       when faced with possibly corrupt application state, but availability concerns
+ *       may dictate using one of the other choices, since they can be less disruptive.
+ *  - DR_VIOLATION_ACTION_KILL_THREAD :
+ *       Immediately kills the thread that caused the violation (the current thread).
+ *       If the current thread is part of a pool of worker threads kept by the application
+ *       then it's likely the application will recover gracefully.  If the thread is
+ *       responsible for a particular function within the application (such as a
+ *       particular service within an svchost process) then the application may continue
+ *       with only that functionality lost.  Note that no cleanup of the thread's state
+ *       is preformed (application locks it owns are not released and, for Windows NT and
+ *       2000 its stack is not freed).  However, the client will still receive the thread
+ *       exit event for this thread.
+ *  - DR_VIOLATION_ACTION_THROW_EXCEPTION :
+ *       Causes the application to receive an unreadable memory execution exception in
+ *       the thread that caused the violation (the current thread).  The exception will
+ *       appear to originate from an application attempt to execute from the target
+ *       address.  If the application has good exception handling it may recover
+ *       gracefully.
+ */
+typedef enum {
+    DR_VIOLATION_ACTION_CONTINUE,
+    DR_VIOLATION_ACTION_CONTINUE_CHANGED_CONTEXT,
+    DR_VIOLATION_ACTION_KILL_PROCESS,
+    DR_VIOLATION_ACTION_KILL_THREAD,
+    DR_VIOLATION_ACTION_THROW_EXCEPTION,
+} dr_security_violation_action_t;
+
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Registers a callback function for the security violation event.  DR
+ * calls \p func whenever it intercepts a security violation.  Clients
+ * can override the default remediation by changing \p action.  If
+ * multiple callbacks are registered, the callback registered last has
+ * final control over the action.  \note source_pc can be NULL if DR
+ * fails to recreate the source pc.
+ */
+void
+dr_register_security_event(void (*func)(void *drcontext, void *source_tag,
+                                        app_pc source_pc, app_pc target_pc,
+                                        dr_security_violation_type_t violation,
+                                        dr_mcontext_t *mcontext,
+                                        dr_security_violation_action_t *action));
+
+DR_API
+/**
+ * Unregister a callback function for the security violation event.
+ * \return true if unregistration is successful and false if it is not
+ * (e.g., \p func was not registered).
+ */
+bool
+dr_unregister_security_event(void (*func)(void *drcontext, void *source_tag,
+                                          app_pc source_pc, app_pc target_pc,
+                                          dr_security_violation_type_t violation,
+                                          dr_mcontext_t *mcontext,
+                                          dr_security_violation_action_t *action));
+#endif /* PROGRAM_SHEPHERDING */
+
+/* DR_API EXPORT BEGIN */
+#ifdef WINDOWS
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Registers a callback function for nudge events.  External entities
+ * can nudge a process through the dr_nudge_process() API routine.  DR
+ * then calls \p func whenever the current process receives the
+ * nudge. The nudge event is delivered in a new non-application
+ * thread.  Callers must specify the target client by passing the client
+ * ID that was provided in dr_init(). 
+ */
+void
+dr_register_nudge_event(void (*func)(void *drcontext, uint64 argument), client_id_t id);
+
+DR_API
+/**
+ * Unregister a callback function for the nudge event.
+ * \return true if unregistration is successful and false if it is not
+ * (e.g., \p func was not registered).
+ */
+bool
+dr_unregister_nudge_event(void (*func)(void *drcontext, uint64 argument), client_id_t id);
+
+DR_API
+/**
+ * Triggers an asynchronous nudge event in the current process.  The callback
+ * function registered with dr_register_nudge_event() will be called with the
+ * supplied \p argument in a new non-application thread.
+ * \note Currently Windows only.
+ * \note Not yet supported for 32-bit processes running on 64-bit windows (WOW64).
+ */
+bool
+dr_nudge_client(client_id_t id, uint64 argument);
+
+/* DR_API EXPORT BEGIN */
+#endif /* WINDOWS */
+/* DR_API EXPORT END */
+
+void instrument_load_client_libs(void);
+void instrument_init(void);
+void instrument_exit(void);
+bool is_in_client_lib(app_pc addr);
+void instrument_thread_init(dcontext_t *dcontext);
+void instrument_thread_exit(dcontext_t *dcontext);
+#ifdef LINUX
+void instrument_fork_init(dcontext_t *dcontext);
+#endif
+bool instrument_basic_block(dcontext_t *dcontext, app_pc tag, instrlist_t *bb,
+                            bool for_trace, bool translating,
+                            dr_emit_flags_t *emitflags);
+dr_emit_flags_t instrument_trace(dcontext_t *dcontext, app_pc tag, instrlist_t *trace,
+                                 bool translating);
+#ifdef CUSTOM_TRACES
+dr_custom_trace_action_t instrument_end_trace(dcontext_t *dcontext, app_pc trace_tag,
+                                           app_pc next_tag);
+#endif
+void instrument_fragment_deleted(dcontext_t *dcontext, app_pc tag, uint flags);
+void instrument_restore_state(dcontext_t *dcontext, app_pc tag, dr_mcontext_t *mc,
+                              bool restore_memory, bool app_code_consistent);
+
+module_data_t * copy_module_area_to_module_data(const module_area_t *area);
+void instrument_module_load(module_data_t *data, bool previously_loaded);
+void instrument_module_unload(module_data_t *data);
+
+/* returns whether this sysnum should be intercepted */
+bool instrument_filter_syscall(dcontext_t *dcontext, int sysnum);
+/* returns whether this syscall should execute */
+bool instrument_pre_syscall(dcontext_t *dcontext, int sysnum);
+void instrument_post_syscall(dcontext_t *dcontext, int sysnum);
+bool instrument_invoke_another_syscall(dcontext_t *dcontext);
+
+#ifdef WINDOWS
+void instrument_exception(dcontext_t *dcontext, dr_exception_t *exception);
+void instrument_nudge(dcontext_t *dcontext, client_id_t id, uint64 arg);
+void instrument_client_thread_termination(void);
+void wait_for_outstanding_nudges(void);
+#else
+dr_signal_action_t instrument_signal(dcontext_t *dcontext, dr_siginfo_t *siginfo);
+bool dr_signal_hook_exists(void);
+#endif
+#ifdef PROGRAM_SHEPHERDING
+void instrument_security_violation(dcontext_t *dcontext, app_pc target_pc,
+                                   security_violation_t violation, action_type_t *action);
+#endif
+bool dr_bb_hook_exists(void);
+bool dr_trace_hook_exists(void);
+bool dr_fragment_deleted_hook_exists(void);
+bool dr_end_trace_hook_exists(void);
+
+/* DR_API EXPORT TOFILE dr_tools.h */
+/* DR_API EXPORT BEGIN */
+
+/**************************************************
+ * TOP-LEVEL ROUTINES
+ */
+/**
+ * @file dr_tools.h
+ * @brief Main API routines, including transparency support.
+ */
+/* DR_API EXPORT END */
+
+DR_API 
+/**
+ * Creates a DR context that can be used in a standalone program.
+ * \warning This context cannot be used as the drcontext for a thread 
+ * running under DR control!  It is only for standalone programs that 
+ * wish to use DR as a library of disassembly, etc. routines.
+ */
+void *
+dr_standalone_init(void);
+
+/* DR_API EXPORT BEGIN */
+
+/**************************************************
+ * UTILITY ROUTINES
+ */
+
+#ifdef WINDOWS
+# define DR_ASSERT_MSG(x, msg) \
+    ((void)((!(x)) ? \
+        (dr_messagebox("ASSERT FAILURE: %s:%d: %s (%s)", __FILE__,  __LINE__, #x, msg),\
+         dr_abort(), 0) : 0))
+#else
+# define DR_ASSERT_MSG(x, msg) \
+    ((void)((!(x)) ? \
+        (dr_fprintf(STDERR, "ASSERT FAILURE: %s:%d: %s (%s)", \
+                     __FILE__,  __LINE__, #x, msg), \
+         dr_abort(), 0) : 0))
+#endif
+
+#define DR_ASSERT(x) DR_ASSERT_MSG(x, "")
+
+/* DR_API EXPORT END */
+
+DR_API
+/** Returns true if all \DynamoRIO caches are thread private. */
+bool
+dr_using_all_private_caches(void);
+
+DR_API
+/** 
+ * Returns the client-specific option string specified at client
+ * registration.  \p client_id is the client ID passed to dr_init().
+ */
+const char *
+dr_get_options(client_id_t client_id);
+
+DR_API 
+/**
+ * Returns the client library name and path that were originally specified
+ * to load the library.  If the resulting string is longer than #MAXIMUM_PATH
+ * it will be truncated.  \p client_id is the client ID passed to a client's
+ * dr_init() function.
+ */
+const char *
+dr_get_client_path(client_id_t client_id);
+
+DR_API 
+/** Returns the image name (without path) of the current application. */
+const char *
+dr_get_application_name(void);
+
+DR_API 
+/** Returns the process id of the current process. */
+process_id_t
+dr_get_process_id(void);
+
+#ifdef WINDOWS
+DR_API
+/** 
+ * Returns true if this process is a 32-bit process operating on a
+ * 64-bit Windows kernel, known as Windows-On-Windows-64, or WOW64.
+ * Returns false otherwise.
+ */
+bool
+dr_is_wow64(void);
+#endif
+
+/* DR_API EXPORT BEGIN */
+
+/** Structure written by dr_get_time() to specify the current time. */
+typedef struct {
+    uint year;         /**< */
+    uint month;        /**< */
+    uint day_of_week;  /**< */
+    uint day;          /**< */
+    uint hour;         /**< */
+    uint minute;       /**< */
+    uint second;       /**< */
+    uint milliseconds; /**< */
+} dr_time_t;
+/* DR_API EXPORT END */
+
+DR_API
+/** Retrieves the current time. */
+void
+dr_get_time(dr_time_t *time);
+
+DR_API
+/** Aborts the process immediately. */
+void
+dr_abort(void);
+
+/* DR_API EXPORT BEGIN */
+
+/**************************************************
+ * APPLICATION-INDEPENDENT MEMORY ALLOCATION
+ */
+/* DR_API EXPORT END */
+
+DR_API 
+/**
+ * Allocates \p size bytes of memory from DR's memory pool specific to the
+ * thread associated with \p drcontext.
+ */
+void *
+dr_thread_alloc(void *drcontext, size_t size);
+
+DR_API 
+/**
+ * Frees thread-specific memory allocated by dr_thread_alloc().
+ * \p size must be the same as that passed to dr_thread_alloc().
+ */
+void
+dr_thread_free(void *drcontext, void *mem, size_t size);
+
+DR_API 
+/** Allocates \p size bytes of memory from DR's global memory pool. */
+void *
+dr_global_alloc(size_t size);
+
+DR_API 
+/**
+ * Frees memory allocated by dr_global_alloc().
+ * \p size must be the same as that passed to dr_global_alloc().
+ */
+void
+dr_global_free(void *mem, size_t size);
+
+DR_API 
+/**
+ * Allocates \p size bytes of memory as a separate allocation from DR's
+ * heap, allowing for separate protection.
+ * The \p prot protection should use the DR_MEMPROT_READ,
+ * DR_MEMPROT_WRITE, and DR_MEMPROT_EXEC bits.
+ * When creating a region to hold dynamically generated code, use
+ * this routine in order to create executable memory.
+ */
+void *
+dr_nonheap_alloc(size_t size, uint prot);
+
+DR_API 
+/**
+ * Frees memory allocated by dr_nonheap_alloc().
+ * \p size must be the same as that passed to dr_nonheap_alloc().
+ */
+void
+dr_nonheap_free(void *mem, size_t size);
+
+#ifdef LINUX
+DR_API
+/** 
+ * Allocates memory from DR's global memory pool, but mimics the
+ * behavior of malloc.  Memory must be freed with __wrap_free().  The
+ * __wrap routines are intended to be used with ld's -wrap option to
+ * replace a client's use of malloc, realloc, and free with internal
+ * versions that allocate memory from DR's private pool.  With -wrap,
+ * clients can link to libraries that allocate heap memory without
+ * interfering with application allocations.
+ * \note Currently Linux only.
+ */
+void *
+__wrap_malloc(size_t size);
+
+DR_API
+/** 
+ * Reallocates memory from DR's global memory pool, but mimics the
+ * behavior of realloc.  Memory must be freed with __wrap_free().  The
+ * __wrap routines are intended to be used with ld's -wrap option; see
+ * __wrap_malloc() for more information.
+ * \note Currently Linux only.
+ */
+void *
+__wrap_realloc(void *mem, size_t size);
+
+DR_API
+/**
+ * Frees memory from DR's global memory pool.  Memory must have been
+ * allocated with __wrap_malloc(). The __wrap routines are intended to
+ * be used with ld's -wrap option; see __wrap_malloc() for more
+ * information.
+ * \note Currently Linux only.
+ */
+void
+__wrap_free(void *mem);
+#endif
+
+/* DR_API EXPORT BEGIN */
+
+/**************************************************
+ * MEMORY QUERY/ACCESS ROUTINES
+ */
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Checks to see that all bytes with addresses in the range [\p pc, \p pc + \p size - 1]
+ * are readable and that reading from that range won't generate an exception (see also
+ * dr_safe_read()).
+ * \note Nothing guarantees that the memory will stay readable for any length of time.
+ */
+bool
+dr_memory_is_readable(const byte *pc, size_t size);
+
+/* NOTE - keep these defines in sync with their counterparts in os_shared.h.
+ * The extra BEGIN END is to get spacing nice. Once we have more control over the layout
+ * of the API header files share with os_shared.h. */
+/* DR_API EXPORT BEGIN */
+/* DR_API EXPORT END */
+/* DR_API EXPORT BEGIN */
+#define DR_MEMPROT_NONE  0x00 /**< No read, write, or execute privileges. */
+#define DR_MEMPROT_READ  0x01 /**< Read privileges. */
+#define DR_MEMPROT_WRITE 0x02 /**< Write privileges. */
+#define DR_MEMPROT_EXEC  0x04 /**< Execute privileges. */
+/* DR_API EXPORT END */
+
+/* FIXME - this is a real view of memory including changes made for dr cache consistency,
+ * but what we really want to show the client is the apps view of memory (which would
+ * requires fixing correcting the view and fixing up exceptions for areas we made read
+ * only) - see PR 198873 */
+DR_API
+/**
+ * An os neutral method for querying a memory address. Returns true
+ * iff a memory region containing \p pc is found.  If found additional
+ * information about the memory region is returned in the optional out
+ * arguments \p base_pc, \p size, and \p prot where \p base_pc is the
+ * start address of the memory region continaing \p pc, \p size is the
+ * size of said memory region and \p prot is an ORed combination of
+ * DR_MEMPROT_* flags describing its current protection.
+ */
+bool
+dr_query_memory(const byte *pc, byte **base_pc, size_t *size, uint *prot);
+
+/* DR_API EXPORT BEGIN */
+#ifdef WINDOWS 
+/* DR_API EXPORT END */
+/* NOTE - see fixme for dr_query_memory - PR 198873. */
+DR_API
+/**
+ * Equivalent to the win32 API function VirtualQuery(), see it for a description of
+ * arguments and return values.  \note Windows-only.
+ */
+size_t
+dr_virtual_query(const byte *pc, MEMORY_BASIC_INFORMATION *mbi, size_t mbi_size);
+/* DR_API EXPORT BEGIN */
+#endif
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Safely reads \p size bytes from address \p base into buffer \p
+ * out_buf.  Reading is done without the possibility of an exception
+ * occuring.  Optionally returns the actual number of bytes copied
+ * into \p bytes_read.  Returns true if successful.  
+ */
+bool
+dr_safe_read(const void *base, size_t size, void *out_buf, size_t *bytes_read);
+
+DR_API
+/**
+ * Safely writes \p size bytes from buffer \p in_buf to address \p
+ * base.  Writing is done without the possiblity of an exception
+ * occuring.  Optionally returns the actual number of bytes copied
+ * into \p bytes_written.  Returns true if successful.
+ */
+bool
+dr_safe_write(void *base, size_t size, const void *in_buf, size_t *bytes_written);
+
+DR_API
+/**
+ * Modifies the memory protections of the region from \p start through
+ * \p start + \p size.  Modification of memory allocated by DR or of
+ * the DR or client libraries themselves is allowed under the
+ * assumption that the client knows what it is doing.  Modification of
+ * the ntdll.dll library on Windows is not allowed.  Returns true if
+ * successful.
+ */
+bool
+dr_memory_protect(void *base, size_t size, uint new_prot);
+
+/* DR_API EXPORT BEGIN */
+
+/**************************************************
+ * SIMPLE MUTEX SUPPORT
+ */
+/* DR_API EXPORT END */
+
+DR_API 
+/** 
+ * Initializes a mutex. 
+ *
+ * Warning: there are restrictions on when DR-provided mutexes, and
+ * locks in general, can be held by a client: no lock should be held
+ * while application code is executing in the code cache.  Locks can
+ * be used while inside client code reached from clean calls out of
+ * the code cache, but they must be released before returning to the
+ * cache.  Failing to follow these restrictions can lead to deadlocks.
+ */
+void *
+dr_mutex_create(void);
+
+DR_API
+/** Deletes \p mutex. */
+void
+dr_mutex_destroy(void *mutex);
+
+DR_API 
+/** Locks \p mutex.  Waits until the mutex is successfully held. */
+void
+dr_mutex_lock(void *mutex);
+
+DR_API 
+/** Unlocks \p mutex.  Asserts that mutex is currently locked. */
+void
+dr_mutex_unlock(void *mutex);
+
+DR_API 
+/** Tries once to lock \p mutex, returns whether or not successful. */
+bool
+dr_mutex_trylock(void *mutex);
+
+/* DR_API EXPORT BEGIN */
+/**************************************************
+ * MODULE INFORMATION ROUTINES
+ */
+
+/** For dr_module_iterator_* interface */
+typedef void * dr_module_iterator_t;
+
+#ifdef AVOID_API_EXPORT
+/* We always give copies of the module_area_t information to clients (in the form
+ * of a module_data_t defined below) to avoid locking issues (see PR 225020). */
+#endif
+/**
+ * Holds information about a loaded module. \note On Linux the start address can be
+ * cast to an Elf32_Ehdr or Elf64_Ehdr. \note On Windows the start address can be cast to
+ * an IMAGE_DOS_HEADER for use in finding the IMAGE_NT_HEADER and its OptionalHeader.
+ * The OptionalHeader can be used to walk the module sections (among other things).
+ * See WINNT.H. \note When accesing any memory inside the module (including header fields)
+ * user is responsible for guarding against corruption and the possibility of the module
+ * being unmapped.
+ */
+struct _module_data_t {
+    union {
+        app_pc start; /**< starting address of this module */
+        module_handle_t handle; /**< module_handle for use with dr_get_proc_address() */
+    } ; /* anonymous union of start address and module handle */
+    app_pc end; /**< ending address of this module */
+
+    app_pc entry_point; /**< entry point for this module as specified in the headers */
+
+    uint flags; /**< Reserved, set to 0 */
+
+    module_names_t names; /**< struct containing name(s) for this module; use
+                           * dr_module_preferred_name() to get the preferred name for
+                           * this module */
+
+    char *full_path; /**< full path to the file backing this module;
+                      * currently not implemented (always NULL) */
+
+#ifdef WINDOWS
+    version_number_t file_version; /**< file version number from .rsrc section */
+    version_number_t product_version; /**< product version number from .rsrc section */
+    uint checksum; /**< module checksum from the PE headers */
+    uint timestamp; /**< module timestamp from the PE headers */
+    size_t module_internal_size; /**< module internal size (from PE headers SizeOfImage) */
+#else
+    /* No Linux specific fields at this time. */
+#endif
+#ifdef AVOID_API_EXPORT
+    /* FIXME - add path PR 307636 */
+    /* FIXME: PR 215890: ELF64 size? Anything else? */
+    /* We can add additional fields to the end without breaking compatibility */
+#endif
+};
+
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Looks up the module containing \p pc.  If a module containing \p pc is found returns
+ * a module_data_t describing that module else returns NULL.  Can be used to
+ * obtain a module_handle_t for dr_lookup_module_section().
+ * \note Returned module_data_t must be freed with dr_free_module_data().
+ */
+module_data_t *
+dr_lookup_module(byte *pc);
+
+DR_API
+/**
+ * Looks up the module with name \p name ignoring case.  If an exact name match is found
+ * returns a module_data_t describing that module else returns NULL.  User must call
+ * dr_free_module_data() on the returned module_data_t once finished. Can be used to
+ * obtain a module_handle_t for dr_get_proc_address().
+ * \note Returned module_data_t must be freed with dr_free_module_data().
+ */
+module_data_t *
+dr_lookup_module_by_name(const char *name);
+
+DR_API
+/**
+ * Initialize a new module iterator.  The returned module iterator contains a snapshot
+ * of the modules loaded at the time it was created.  Use dr_module_iterator_hasnext()
+ * and dr_module_iterator_next() to walk the loaded modules.  Call
+ * dr_module_iterator_stop() when finished to release the iterator. \note The iterator
+ * does not prevent modules from being loaded or unloaded while the iterator is being
+ * walked.
+ */
+dr_module_iterator_t
+dr_module_iterator_start(void);
+
+DR_API
+/**
+ * Returns true if there is another loaded module in the iterator.
+ */
+bool
+dr_module_iterator_hasnext(dr_module_iterator_t *mi);
+
+DR_API
+/**
+ * Retrieves the module_data_t for the next loaded module in the iterator. User must call
+ * dr_free_module_data() on the returned module_data_t once finished.
+ * \note Returned module_data_t must be freed with dr_free_module_data().
+ */
+module_data_t *
+dr_module_iterator_next(dr_module_iterator_t *mi);
+
+DR_API 
+/**
+ * User should call this routine to free the module iterator.
+ */
+void
+dr_module_iterator_stop(dr_module_iterator_t *mi);
+
+DR_API
+/**
+ * Makes a copy of \p data.  Copy must be freed with dr_free_module_data().
+ * Useful for making persistent copies of module_data_t's recieved as part of 
+ * image load and unload event callbacks.
+ */
+module_data_t *
+dr_copy_module_data(const module_data_t *data);
+
+DR_API
+/**
+ * Frees a module_data_t returned by dr_module_iterator_next(), dr_lookup_module(),
+ * dr_lookup_module_by_name(), or dr_copy_module_data(). \note Should NOT be used with
+ * a module_data_t obtained as part of a module load or unload event.
+ */
+void
+dr_free_module_data(module_data_t *data);
+
+DR_API
+/**
+ * Returns the preferred name for the module described by \p data from
+ * \p data->module_names.
+ */
+const char *
+dr_module_preferred_name(const module_data_t *data);
+
+/* DR_API EXPORT BEGIN */
+#ifdef WINDOWS
+/* DR_API EXPORT END */
+DR_API
+/**
+ * Optionally also
+ * returns whether \p pc is within a section within the module in \p section_found and
+ * information about that section in \p section_out. \note Not yet available on Linux.
+ * \note Returned module_data_t must be freed with dr_free_module_data().
+ */
+bool
+dr_lookup_module_section(module_handle_t lib,
+                         byte *pc, IMAGE_SECTION_HEADER *section_out);
+
+DR_API
+/**
+ * Returns the entry point of the function with the given name in the module
+ * with the given base. Returns NULL on failure.
+ * \note Currently Windows only.
+ */
+generic_func_t
+dr_get_proc_address(module_handle_t lib, const char *name);
+
+/* DR_API EXPORT BEGIN */
+#endif /* WINDOWS */
+/* DR_API EXPORT END */
+
+
+/* DR_API EXPORT BEGIN */
+/**************************************************
+ * SYSTEM CALL PROCESSING ROUTINES
+ */
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Usable only from a pre-syscall (dr_register_pre_syscall_event()) 
+ * event.  Returns the value of system call parameter number \p param_num.
+ */
+reg_t
+dr_syscall_get_param(void *drcontext, int param_num);
+
+DR_API
+/**
+ * Usable only from a pre-syscall (dr_register_pre_syscall_event())
+ * event, or from a post-syscall (dr_register_post_syscall_event())
+ * event when also using dr_syscall_invoke_another().  Sets the value
+ * of system call parameter number \p param_num to \p new_value.
+ */
+void
+dr_syscall_set_param(void *drcontext, int param_num, reg_t new_value);
+
+DR_API
+/**
+ * Usable only from a post-syscall (dr_register_post_syscall_event())
+ * event.  Returns the return value of the system call that will be
+ * presented to the application.
+ */
+reg_t
+dr_syscall_get_result(void *drcontext);
+
+DR_API
+/**
+ * Usable only from a pre-syscall (dr_register_pre_syscall_event()) or
+ * post-syscall (dr_register_post_syscall_event()) event.
+ * For pre-syscall, should only be used when skipping the system call.
+ * This sets the return value of the system call that the application sees
+ * to \p value.
+ */
+void
+dr_syscall_set_result(void *drcontext, reg_t value);
+
+DR_API
+/**
+ * Usable only from a pre-syscall (dr_register_pre_syscall_event())
+ * event, or from a post-syscall (dr_register_post_syscall_event())
+ * event when also using dr_syscall_invoke_another().  Sets the system
+ * call number of the system call about to be invoked to \p new_num.
+ */
+void
+dr_syscall_set_sysnum(void *drcontext, int new_num);
+
+DR_API
+/**
+ * Usable only from a post-syscall (dr_register_post_syscall_event())
+ * event.  An additional system call will be invoked immediately,
+ * using the current values of the parameters, which can be set with
+ * dr_syscall_set_param().  The system call to be invoked should be
+ * specified with dr_syscall_set_sysnum().
+ *
+ * Use this routine with caution.  Especially on Windows, care must be
+ * taken if the application is expected to continue afterward.  When
+ * system call parameters are stored on the stack, modifying them can
+ * result in incorrect application behavior, particularly when setting
+ * more parameters than were present in the original system call,
+ * which will result in corruption of the application stack.
+ *
+ * On Windows, when the first system call is interruptible
+ * (alertable), the additional system call may be delayed.
+ *
+ * DR will set key registers such as r10 for 64-bit or xdx for
+ * sysenter or WOW64 system calls.  However, DR will not set ecx for
+ * WOW64; that is up to the client.
+ */
+void
+dr_syscall_invoke_another(void *drcontext);
+
+/**************************************************
+ * PLATFORM-INDEPENDENT FILE SUPPORT
+ *
+ * Since a FILE cannot be used outside of the DLL it was created in,
+ * we have to use HANDLE on Windows.
+ * We hide the distinction behind the file_t type
+ */
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Creates a new directory.  Fails if the directory already exists
+ * or if it can't be created.
+ */
+bool
+dr_create_dir(const char *fname);
+
+/* DR_API EXPORT BEGIN */
+#ifdef WINDOWS 
+/* DR_API EXPORT END */
+ /* FIXME - implement Linux versions */
+DR_API
+/** Checks for the existence of a directory. \note Windows only. */
+bool
+dr_directory_exists(const char *fname);
+
+DR_API
+/** Checks the existence of a file. \note Windows only. */
+bool
+dr_file_exists(const char *fname);
+/* DR_API EXPORT BEGIN */
+#endif
+/* DR_API EXPORT END */
+
+/* The extra BEGIN END is to get spacing nice. */
+/* DR_API EXPORT BEGIN */
+/* DR_API EXPORT END */
+/* DR_API EXPORT BEGIN */
+/* flags for use with dr_open_file() */
+/** Open with read access. */
+#define DR_FILE_READ               0x1
+/** Open with write access, but do not open if the file already exists. */
+#define DR_FILE_WRITE_REQUIRE_NEW  0x2
+/**
+ * Open with write access.  If the file already exists, set the file position to the
+ * end of the file.
+ */
+#define DR_FILE_WRITE_APPEND       0x4
+/**
+ * Open with write access.  If the file already exists, truncate the
+ * file to zero length.
+ */
+#define DR_FILE_WRITE_OVERWRITE    0x8
+/**
+ * Open with large (>2GB) file support.  Only applicable on 32-bit Linux.
+ * \note DR's log files and tracedump files are all created with this flag.
+ */
+#define DR_FILE_ALLOW_LARGE       0x10
+/* DR_API EXPORT END */
+
+DR_API 
+/**
+ * Opens the file \p fname.  On Windows, \p fname must be an absolute path.
+ * If no such file exists then one is created.
+ * The file access mode is set by the \p mode_flags argument which is drawn from
+ * the DR_FILE_* defines ORed together.  Returns INVALID_FILE if unsuccessful.
+ * \note No more then one write mode flag can be specified.
+ */
+file_t
+dr_open_file(const char *fname, uint mode_flags);
+
+DR_API 
+/** Closes file \p f. */
+void
+dr_close_file(file_t f);
+
+DR_API 
+/** Flushes any buffers for file \p f. */
+void
+dr_flush_file(file_t f);
+
+DR_API
+/**
+ * Writes \p count bytes from \p buf to file \p f.  
+ * Returns the actual number written.
+ */
+ssize_t
+dr_write_file(file_t f, const void *buf, size_t count);
+
+DR_API 
+/**
+ * Reads up to \p count bytes from file \p f into \p buf.
+ * Returns the actual number read.
+ */
+ssize_t
+dr_read_file(file_t f, void *buf, size_t count);
+
+/* NOTE - keep in synch with OS_SEEK_* in os_shared.h and SEEK_* from Linux headers.
+ * The extra BEGIN END is to get spacing nice. Once we have more control over the layout
+ * of the API header files share with os_shared.h. */
+/* DR_API EXPORT BEGIN */
+/* DR_API EXPORT END */
+/* DR_API EXPORT BEGIN */
+/* For use with dr_file_seek(), specifies the origin at which to apply the offset. */
+#define DR_SEEK_SET 0  /**< start of file */
+#define DR_SEEK_CUR 1  /**< current file position */
+#define DR_SEEK_END 2  /**< end of file */
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Sets the current file position for file \p f to \p offset bytes
+ * from the specfied origin, where \p origin is one of the DR_SEEK_*
+ * values.  Returns true if successful.
+ */
+bool
+dr_file_seek(file_t f, int64 offset, int origin);
+
+DR_API
+/**
+ * Returns the current position for the file \p f in bytes from the start of the file.
+ * Returns -1 on an error.
+ */
+int64
+dr_file_tell(file_t f);
+
+/* TODO add delete_file, rename/move_file, copy_file, get_file_size, truncate_file etc.
+ * All should be easy though at some point should perhaps tell people to just use the raw
+ * systemcalls, esp for linux where they're documented and let them provide their own
+ * wrappers. */
+
+/* DR_API EXPORT BEGIN */
+
+/**************************************************
+ * PRINTING
+ */
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Writes to DR's log file for the thread with drcontext \p drcontext if the current
+ * loglevel is >= \p level and the current \p logmask & \p mask != 0.
+ * The mask constants are below.
+ * Logging is disabled for the release build.
+ * If \p drcontext is NULL, writes to the main log file.
+ */
+void
+dr_log(void *drcontext, uint mask, uint level, const char *fmt, ...);
+
+/* hack to get these constants in the right place in dr_ir_api.h */
+#if 0 /* now skip a line to prevent gen_api.pl from removing this */
+
+/* DR_API EXPORT BEGIN */
+
+/* The log mask constants */
+#define LOG_NONE           0x00000000  /**< Log no data. */                              
+#define LOG_STATS          0x00000001  /**< Log per-thread and global statistics. */     
+#define LOG_TOP            0x00000002  /**< Log top-level information. */                
+#define LOG_THREADS        0x00000004  /**< Log data related to threads. */              
+#define LOG_SYSCALLS       0x00000008  /**< Log data related to system calls. */         
+#define LOG_ASYNCH         0x00000010  /**< Log data related to signals/callbacks/etc. */
+#define LOG_INTERP         0x00000020  /**< Log data related to app interpretation. */   
+#define LOG_EMIT           0x00000040  /**< Log data related to emitting code. */        
+#define LOG_LINKS          0x00000080  /**< Log data related to linking code. */         
+#define LOG_CACHE          0x00000100  /**< Log data related to code cache management. */
+#define LOG_FRAGMENT       0x00000200  /**< Log data related to app code fragments. */   
+#define LOG_DISPATCH       0x00000400  /**< Log data on every context switch dispatch. */
+#define LOG_MONITOR        0x00000800  /**< Log data related to trace building. */       
+#define LOG_HEAP           0x00001000  /**< Log data related to memory managment. */     
+#define LOG_VMAREAS        0x00002000  /**< Log data related to address space regions. */
+#define LOG_SYNCH          0x00004000  /**< Log data related to synchronization. */      
+#define LOG_MEMSTATS       0x00008000  /**< Log data related to memory statistics. */    
+#define LOG_OPTS           0x00010000  /**< Log data related to optimizations. */        
+#define LOG_SIDELINE       0x00020000  /**< Log data related to sideline threads. */ 
+#define LOG_SYMBOLS        0x00040000  /**< Log data related to app symbols. */ 
+#define LOG_RCT            0x00080000  /**< Log data related to indirect transfers. */ 
+#define LOG_NT             0x00100000  /**< Log data related to Windows Native API. */ 
+#define LOG_HOT_PATCHING   0x00200000  /**< Log data related to hot patching. */ 
+#define LOG_HTABLE         0x00400000  /**< Log data related to hash tables. */ 
+#define LOG_MODULEDB       0x00800000  /**< Log data related to the module database. */ 
+#define LOG_ALL            0x00ffffff  /**< Log all data. */
+/* DR_API EXPORT END */
+#endif
+
+
+DR_API 
+/**
+ * Returns the log file for the thread with drcontext \p drcontext.
+ * If \p drcontext is NULL, returns the main log file.
+ */
+file_t
+dr_get_logfile(void *drcontext);
+
+DR_API 
+/**
+ * Returns true iff the -stderr_mask runtime option is non-zero, indicating
+ * that the user wants notification messages printed to stderr.
+ */
+bool
+dr_is_notify_on(void);
+
+DR_API
+/** Returns a handle to stdout. */
+file_t 
+dr_get_stdout_file(void);
+
+DR_API
+/** Returns a handle to stderr. */
+file_t
+dr_get_stderr_file(void);
+
+#ifdef PROGRAM_SHEPHERDING
+DR_API
+/** Writes a security violation forensics report to the supplied file. The forensics 
+ *  report will include detailed information about the source and target addresses of the
+ *  violation as well as information on the current thread, process, and machine.  The
+ *  forensics report is generated in an xml block described by dr_forensics-1.0.dtd.
+ *  The encoding used is iso-8859-1.
+ *  
+ *  The dcontext, violation, and action arguments are supplied by the security violation
+ *  event callback.  The file argument is the file to write the forensics report to and
+ *  the violation_name argument is a supplied name for the violation.
+ */
+void
+dr_write_forensics_report(void *dcontext, file_t file,
+                          dr_security_violation_type_t violation,
+                          dr_security_violation_action_t action,
+                          const char *violation_name);
+#endif /* PROGRAM_SHEPHERDING */
+
+/* DR_API EXPORT BEGIN */
+#ifdef WINDOWS 
+/* DR_API EXPORT END */
+DR_API
+/** 
+ * Displays a message in a pop-up window. \note Windows only. \note On Windows Vista
+ * most Windows services are unable to display message boxes.
+ */
+void
+dr_messagebox(const char *fmt, ...);
+/* DR_API EXPORT BEGIN */
+#endif
+/* DR_API EXPORT END */
+
+DR_API 
+/**
+ * Stdout printing that won't interfere with the
+ * application's own printing.  Currently non-buffered.
+ * \note On Windows, this routine does not support printing floating
+ * point values.  Use snprintf() instead.
+ */
+void 
+dr_printf(const char *fmt, ...);
+
+DR_API
+/**
+ * Printing to a file that won't interfere with the
+ * application's own printing.  Currently non-buffered.
+ * \note On Windows, this routine does not support printing floating
+ * point values.  Use snprintf() instead.
+ */
+void
+dr_fprintf(file_t f, const char *fmt, ...);
+
+/* We have to temporarily suspend our snprintf->_snprintf define */
+#undef snprintf
+DR_API
+/**
+ * Utility routine to print a formatted message to a string.  Will not
+ * print more than max characters.  If successful, returns the number
+ * of characters printed, not including the terminating null
+ * character.  If the number of characters to write equals max, then
+ * the caller is responsible for supplying a terminating null
+ * character.  If the number of characters to write exceeds max, then
+ * max characters are written and -1 is returned.  If an error
+ * occurs, a negative value is returned.
+ * \note This routine does not support printing wide characters: use
+ * _snprintf() instead (though _snprintf() does not support printing floating
+ * point values).
+ */
+int
+snprintf(char *buf, size_t max, const char *fmt, ...);
+#ifdef WINDOWS
+# define snprintf _snprintf /* restore the define */
+#else
+# define snprintf our_snprintf /* restore the define */
+#endif
+
+DR_API 
+/** Prints \p msg followed by the instruction \p instr to file \p f. */
+void 
+dr_print_instr(void *drcontext, file_t f, instr_t *instr, const char *msg);
+
+DR_API 
+/** Prints \p msg followed by the operand \p opnd to file \p f. */
+void 
+dr_print_opnd(void *drcontext, file_t f, opnd_t opnd, const char *msg);
+
+/* DR_API EXPORT BEGIN */
+
+/**************************************************
+ * THREAD SUPPORT
+ */
+/* DR_API EXPORT END */
+
+DR_API 
+/**
+ * Returns the DR context of the current thread. 
+ * This routine may return NULL during process exit.
+ */
+void *
+dr_get_current_drcontext(void);
+
+DR_API 
+/** Returns the thread id of the thread with drcontext \p drcontext. */
+thread_id_t 
+dr_get_thread_id(void *drcontext);
+
+DR_API 
+/**
+ * Returns the user-controlled thread-local-storage field.  To
+ * generate an instruction sequence that reads the drcontext field
+ * inline in the code cache, use dr_insert_read_tls_field().
+ */
+void *
+dr_get_tls_field(void *drcontext);
+
+DR_API 
+/** 
+ * Sets the user-controlled thread-local-storage field.  To
+ * generate an instruction sequence that reads the drcontext field
+ * inline in the code cache, use dr_insert_write_tls_field().
+ */
+void 
+dr_set_tls_field(void *drcontext, void *value);
+
+/* PR 222812: due to issues in supporting client thread synchronization
+ * and other complexities we are using nudges for simple push-i/o and
+ * saving thread creation for sideline usage scenarios.
+ */
+/* These are implemented in win32/os.c.  For Linux we should use
+ * create_thread() in x86/sideline.c, with a different stack freeing model.
+ */
+#if defined(CLIENT_SIDELINE) && defined(WINDOWS)
+DR_API
+/**
+ * Creates a new thread that is marked as a non-application thread (i.e.,
+ * DR will let it run natively and not execute its code from the code cache).
+ * The thread will NOT terminate automatically simply by returning from
+ * \p func: instead it must call dr_terminate_thread() to exit and to
+ * free its stack.
+ *
+ * \note Thread creation via this routine is not yet fully transparent:
+ * -# All dlls will be notifed of thread creation by DLL_THREAD_ATTACH.
+ * -# The thread will show up in the list of application threads if
+ *    the operating system is queried about threads.
+ *
+ * \note Windows-only in the current implementation.
+ */
+bool
+dr_create_client_thread(void (*func)(void *param), void *arg,
+                        size_t stack_reserve, size_t stack_commit);
+
+DR_API
+/**
+ * Must only be called by a thread created via dr_create_client_thread().
+ * Frees its own stack and terminates itself.  This routine does not return
+ * when successful and returns false when unsuccessful.
+ *
+ * \note Windows-only in the current implementation.
+ */
+bool
+dr_terminate_client_thread(void);
+
+DR_API
+/** Current thread sleeps for \p time_ms milliseconds. */
+void
+dr_sleep(int time_ms);
+
+#endif /* defined(CLIENT_SIDELINE) && defined(WINDOWS) */
+
+DR_API
+/** Current thread gives up its time quantum. */
+void
+dr_thread_yield(void);
+
+/* DR_API EXPORT TOFILE dr_ir_utils.h */
+/* DR_API EXPORT BEGIN */
+
+/**************************************************
+ * CODE TRANSFORMATION UTILITIES
+ */
+/**
+ * @file dr_ir_utils.h
+ * @brief Code transformation utilities.
+ */
+
+/**
+ * An enum of spill slots to use with dr_save_reg(), dr_restore_reg(),
+ * dr_save_arith_flags(), dr_restore_arith_flags() and
+ * dr_insert_mbr_instrumentation().  Values stored in spill slots remain
+ * valid only until the next non-meta (i.e. application) instruction.  Spill slots
+ * can be accessed/modifed during clean calls and restore_state_events (see
+ * dr_register_restore_state_event()) with dr_read_saved_reg() and
+ * dr_write_saved_reg().
+ *
+ * Spill slots <= dr_max_opnd_accessible_spill_slot() can be directly accessed
+ * from client inserted instructions with dr_reg_spill_slot_opnd().
+ *
+ * \note Some spill slots may be faster to access than others.  Currently spill
+ * slots 1-3 are significantly faster to access than the others when running
+ * without -thread_private.  When running with -thread_private all spill slots
+ * are expected to have similar performance.  This is subject to change in future
+ * releases, but clients may assume that smaller numbered spill slots are faster
+ * or the same cost to access as larger numbered spill slots.
+ *
+ * \note The number of spill slots may change in future releases.
+ */
+typedef enum {
+    SPILL_SLOT_1  =  0,  /** spill slot for register save/restore routines */
+    SPILL_SLOT_2  =  1,  /** spill slot for register save/restore routines */
+    SPILL_SLOT_3  =  2,  /** spill slot for register save/restore routines */
+    SPILL_SLOT_4  =  3,  /** spill slot for register save/restore routines */
+    SPILL_SLOT_5  =  4,  /** spill slot for register save/restore routines */
+    SPILL_SLOT_6  =  5,  /** spill slot for register save/restore routines */
+    SPILL_SLOT_7  =  6,  /** spill slot for register save/restore routines */
+    SPILL_SLOT_8  =  7,  /** spill slot for register save/restore routines */
+    SPILL_SLOT_9  =  8,  /** spill slot for register save/restore routines */
+#ifdef X64
+    SPILL_SLOT_10 =  9,  /** spill slot for register save/restore routines
+                          * \note x64 only */
+    SPILL_SLOT_11 = 10,  /** spill slot for register save/restore routines
+                          * \note x64 only */
+    SPILL_SLOT_12 = 11,  /** spill slot for register save/restore routines
+                          * \note x64 only */
+    SPILL_SLOT_13 = 12,  /** spill slot for register save/restore routines
+                          * \note x64 only */
+    SPILL_SLOT_14 = 13,  /** spill slot for register save/restore routines
+                          * \note x64 only */
+    SPILL_SLOT_15 = 14,  /** spill slot for register save/restore routines
+                          * \note x64 only */
+    SPILL_SLOT_16 = 15,  /** spill slot for register save/restore routines
+                          * \note x64 only */
+    SPILL_SLOT_17 = 16,  /** spill slot for register save/restore routines
+                          * \note x64 only */
+    SPILL_SLOT_MAX = SPILL_SLOT_17 /** Enum value of the last register save/restore
+                                    *  spill slot. */
+#else
+    SPILL_SLOT_MAX = SPILL_SLOT_9 /** Enum value of the last register save/restore
+                                   *  spill slot. */
+#endif
+} dr_spill_slot_t;
+/* DR_API EXPORT END */
+
+DR_API
+/** 
+ * Inserts into \p ilist prior to \p where meta-instruction(s) to save the
+ * register \p reg in the spill slot \p slot.  See dr_restore_reg(). Use
+ * dr_read_saved_reg() and dr_write_saved_reg() to access spill slots from clean
+ * calls and restore_state_events (see dr_register_restore_state_event()).
+ * \note The stored value remains available only until the next non-meta (i.e.
+ * application) instruction. Use dr_insert_write_tls_field() and 
+ * dr_insert_read_tls_field() for a persistent (but more costly to access)
+ * thread-local-storage location.
+ */
+void 
+dr_save_reg(void *drcontext, instrlist_t *ilist, instr_t *where, reg_id_t reg,
+            dr_spill_slot_t slot);
+
+DR_API
+/** 
+ * Inserts into \p ilist prior to \p where meta-instruction(s) to restore the 
+ * register \p reg from the spill slot \p slot.  See dr_save_reg() for notes on
+ * lifetime and alternative access to spill slots.
+ */
+void 
+dr_restore_reg(void *drcontext, instrlist_t *ilist, instr_t *where, reg_id_t reg,
+               dr_spill_slot_t slot);
+
+DR_API
+/**
+ * Returns the largest dr_spill_slot_t that can be accessed with an opnd_t from
+ * dr_reg_spill_slot_opnd().
+ */
+dr_spill_slot_t
+dr_max_opnd_accessible_spill_slot(void);
+
+DR_API
+/**
+ * Returns an opnd_t that directly accesses the spill slot \p slot. Only slots
+ * <= dr_max_opnd_accessible_spill_slot() can be used with this routine.
+ * \note \p slot must be <= dr_max_opnd_accessible_spill_slot()
+ */
+opnd_t
+dr_reg_spill_slot_opnd(void *drcontext, dr_spill_slot_t slot);
+
+DR_API
+/**
+ * Can be used from a clean call or a restore_state_event (see
+ * dr_register_restore_state_event()) to see the value saved in spill slot
+ * \p slot by dr_save_reg().
+ */
+reg_t
+dr_read_saved_reg(void *drcontext, dr_spill_slot_t slot);
+
+DR_API
+/**
+ * Can be used from a clean call to modify the value saved in the spill slot 
+ * \p slot by dr_save_reg() such that a later dr_restore_reg() will see the
+ * new value.
+ * 
+ * \note This routine should only be used during a clean call out of the
+ * cache.  Use at any other time could corrupt application or \DynamoRIO
+ * state.
+ */
+void
+dr_write_saved_reg(void *drcontext, dr_spill_slot_t slot, reg_t value);
+
+DR_API
+/** 
+ * Inserts into \p ilist prior to \p where meta-instruction(s) to save the 6
+ * arithmetic flags into xax after first saving xax to the spill slot \p slot.
+ * This is equivalent to dr_save_reg() of xax to \p slot followed by lahf and
+ * seto al instructions.  See dr_restore_arith_flags().
+ *
+ * \note At completion of the inserted instructions the saved flags are in the
+ * xax register.  The xax register should not be modified after using this
+ * routine unless it is first saved (and later restored prior to
+ * using dr_restore_arith_flags()).
+ */
+void 
+dr_save_arith_flags(void *drcontext, instrlist_t *ilist, instr_t *where,
+                    dr_spill_slot_t slot);
+
+DR_API
+/** 
+ * Inserts into \p ilist prior to \p where meta-instruction(s) to restore the 6
+ * arithmetic flags, assuming they were saved using dr_save_arith_flags() with
+ * slot \p slot and that xax holds the same value it did after the save.
+ */
+void 
+dr_restore_arith_flags(void *drcontext, instrlist_t *ilist, instr_t *where,
+                       dr_spill_slot_t slot);
+
+/* FIXME PR 315333: add routine that scans ahead to see if need to save eflags.  See 
+ * forward_eflags_analysis(). */
+
+/* FIXME PR 315327: add routines to save, restore and access from C code xmm registers
+ * from our dcontext slots.  Not clear we really need to since we can't do it all
+ * that much faster than the client can already with read/write tls field (only one
+ * extra load) or (if -thread_private) absolute addresses.
+ */
+
+DR_API
+/**
+ * Inserts into \p ilist prior to \p where meta-instruction(s) to read into the
+ * general-purpose full-size register \p reg from the user-controlled drcontext
+ * field for this thread.  Reads from the same field as dr_get_tls_field().
+ */
+void
+dr_insert_read_tls_field(void *drcontext, instrlist_t *ilist, instr_t *where,
+                         reg_id_t reg);
+
+DR_API
+/**
+ * Inserts into \p ilist prior to \p where meta-instruction(s) to write the
+ * general-purpose full-size register \p reg to the user-controlled drcontext field
+ * for this thread.  Writes to the same field as dr_set_tls_field().
+ */
+void
+dr_insert_write_tls_field(void *drcontext, instrlist_t *ilist, instr_t *where,
+                          reg_id_t reg);
+
+#endif /* CLIENT_INTERFACE (need the next few functions for hot patching) */
+
+/* to make our own code shorter */
+#define MINSERT instrlist_meta_preinsert
+
+DR_API
+/** Inserts \p instr as a non-application instruction into \p ilist prior to \p where. */
+void 
+instrlist_meta_preinsert(instrlist_t *ilist, instr_t *where, instr_t *instr);
+
+DR_API
+/** Inserts \p instr as a non-application instruction into \p ilist after \p where. */
+void
+instrlist_meta_postinsert(instrlist_t *ilist, instr_t *where, instr_t *instr);
+
+DR_API
+/** Inserts \p instr as a non-application instruction onto the end of ilist */
+void
+instrlist_meta_append(instrlist_t *ilist, instr_t *instr);
+
+/* dr_insert_* are used by general DR */
+
+/* FIXME PR 213600: for clean call args that reference memory the
+ * client may prefer to receive the fault itself rather than it being treated
+ * as an app exception (xref PR 302951).
+ */
+DR_API
+/**
+ * Inserts into \p ilist prior to \p where meta-instruction(s) to save state
+ * for a call, switch to this thread's DR stack, set up the passed-in
+ * parameters, make a call to \p callee, clean up the parameters, and
+ * then restore the saved state.
+ *
+ * The callee must use the standard C calling convention that matches the
+ * underlying 32-bit or 64-bit binary interface convention ("cdecl"). Other
+ * calling conventions, such as "fastcall" and "stdcall", are not supported.
+ *
+ * Stores the application state information on the DR stack, where it can
+ * be accessed from \c callee using dr_get_mcontext() and modified using
+ * dr_set_mcontext().
+ *
+ * If \p save_fpstate is true, preserves the fp/mmx/sse state on the DR stack.
+ * Note that it is relatively expensive to save this state (on the
+ * order of 200 cycles) and that it typically takes 512 bytes to store
+ * it (see proc_fpstate_save_size()).
+ *
+ * DR does support translating a fault in an argument (e.g., an
+ * argument that references application memory); such a fault will be
+ * treated as an application exception.
+ *
+ * \note The stack used to save state and call \p callee is limited to
+ * 20KB by default; this can be changed with the -stack_size DR runtime
+ * parameter.  This stack cannot be used to store state that persists
+ * beyond \c callee's return point.
+ *
+ * \note This routine only supports passing arguments that are
+ * integers or pointers of a size equal to the register size: i.e., no
+ * floating-point, multimedia, or aggregate data types.
+ * The routine also supports immediate integers that are smaller than
+ * the register size, and for 64-bit mode registers or memory references that
+ * are OPSZ_4.
+ *
+ * \note For 64-bit mode, passing arguments that use calling
+ * convention registers (for Windows, RCX, RDX, R8, R9; for Linux,
+ * RDI, RSI, RDX, RCX, R8 and R9) are supported but may incur
+ * additional stack usage.
+ *
+ * \note For 64-bit mode, if a 32-bit immediate integer is specified as an
+ * argument and it has its top bit set, we assume it is intended to be
+ * sign-extended to 64-bits; otherwise we zero-extend it.
+ *
+ * \note For 64-bit mode, variable-sized argument operands may not work
+ * properly.
+ *
+ * \note Arguments that reference sub-register portions of REG_XSP are
+ * not supported (full REG_XSP is supported).
+ */
+void 
+dr_insert_clean_call(void *drcontext, instrlist_t *ilist, instr_t *where,
+                     void *callee, bool save_fpstate, uint num_args, ...);
+
+DR_API
+/**
+ * Inserts into \p ilist prior to \p where meta-instruction(s) to set
+ * up the passed-in parameters, make a call to \p callee, and clean up
+ * the parameters.
+ *
+ * The callee must use the standard C calling convention that matches the
+ * underlying 32-bit or 64-bit binary interface convention ("cdecl"). Other
+ * calling conventions, such as "fastcall" and "stdcall", are not supported.
+ *
+ * This routine uses the existing stack.  In 64-bit mode, this routine assumes
+ * that the stack pointer is currently 16-byte aligned.
+ *
+ * The application state is NOT saved or restored (use dr_prepare_for_call()
+ * and dr_cleanup_after_call(), or replace this routine with dr_insert_clean_call()).
+ * The parameter set-up may write to registers if the calling convention so
+ * dictates.  The registers are NOT saved beforehand (to do so, use
+ * dr_insert_clean_call()).
+ *
+ * It is up to the caller of this routine to preserve caller-saved registers.
+ *
+ * DR does not support translating a fault in an argument.  For fault
+ * transparency, the client must perform the translation (see
+ * #dr_register_restore_state_event()), or use
+ * #dr_insert_clean_call().
+ *
+ * \note This routine only supports passing arguments that are
+ * integers or pointers of a size equal to the register size: i.e., no
+ * floating-point, multimedia, or aggregate data types.
+ * The routine also supports immediate integers that are smaller than
+ * the register size, and for 64-bit mode registers or memory references that
+ * are OPSZ_4.
+ *
+ * \note For 64-bit mode, passing arguments that use calling
+ * convention registers (for Windows, RCX, RDX, R8, R9; for Linux,
+ * RDI, RSI, RDX, RCX, R8 and R9) are supported but may incur
+ * additional stack usage.
+ *
+ * \note For 64-bit mode, if a 32-bit immediate integer is specified as an
+ * argument and it has its top bit set, we assume it is intended to be
+ * sign-extended to 64-bits; otherwise we zero-extend it.
+ *
+ * \note For 64-bit mode, variable-sized argument operands may not work
+ * properly.
+ *
+ * \note Arguments that reference REG_XSP are not supported in 64-bit mode.
+ */
+void 
+dr_insert_call(void *drcontext, instrlist_t *ilist, instr_t *where,
+               void *callee, uint num_args, ...);
+
+DR_API
+/**
+ * Inserts into \p ilist prior to \p where meta-instruction(s) to save state for a call.
+ * Stores the application state information on the DR stack.
+ * Returns the size of the data stored on the DR stack (in case the caller
+ * needs to align the stack pointer).
+ *
+ * \warning This routine does NOT save the fp/mmx/sse state: to do that the
+ * instrumentation routine should call proc_save_fpstate() to save and
+ * then proc_restore_fpstate() to restore (or use dr_insert_clean_call()).
+ *
+ * \note The preparation modifies the REG_XSP and REG_XAX registers
+ * (after saving them).  Use dr_insert_clean_call() instead if an
+ * argument to the subsequent call that references REG_XAX is
+ * desired.
+ *
+ * \note The stack used to save the state is limited to
+ * 20KB by default; this can be changed with the -stack_size DR runtime
+ * parameter.  This stack cannot be used to store state that persists
+ * beyond a single clean call, code cache execution, or probe callback
+ * function execution.
+ */
+uint
+dr_prepare_for_call(void *drcontext, instrlist_t *ilist, instr_t *instr);
+
+DR_API
+/** 
+ * Inserts into \p ilist prior to \p where meta-instruction(s) to restore state
+ * after a call.
+ */
+void 
+dr_cleanup_after_call(void *drcontext, instrlist_t *ilist, instr_t *where,
+                      uint sizeof_param_area);
+
+#ifdef CLIENT_INTERFACE
+DR_API
+/**
+ * Inserts into \p ilist prior to \p where meta-instruction(s) to save the current
+ * esp and switch to this thread's DR stack.
+ * \note The DR stack is limited to 20KB by default; this can be changed with
+ * the -stack_size DR runtime parameter.  This stack cannot be used to store
+ * state that persists beyond a single clean call, code cache execution,
+ * or probe callback function execution.
+ */
+void
+dr_swap_to_clean_stack(void *drcontext, instrlist_t *ilist, instr_t *where);
+
+DR_API
+/**
+ * Inserts into \p ilist prior to \p where meta-instruction(s) to restore into
+ * esp the value saved by dr_swap_to_dr_stack().
+ */
+void
+dr_restore_app_stack(void *drcontext, instrlist_t *ilist, instr_t *where);
+
+/* providing functionality of old -instr_calls and -instr_branches flags */
+DR_API
+/**
+ * Assumes that \p instr is a near call.
+ * Inserts into \p ilist prior to \p instr instruction(s) to call callee passing
+ * two arguments:
+ * -# address of call instruction (caller)
+ * -# target address of call (callee)
+ */
+void 
+dr_insert_call_instrumentation(void *drcontext, instrlist_t *ilist,
+                               instr_t *instr, void *callee);
+
+DR_API
+/**
+ * Assumes that \p instr is an indirect branch.
+ * Inserts into \p ilist prior to \p instr instruction(s) to call callee passing
+ * two arguments:
+ * -# address of branch instruction
+ * -# target address of branch
+ * \note Only the address portion of a far indirect branch is considered.
+ * \note \p scratch_slot must be <= dr_max_opnd_accessible_spill_slot(). \p scratch_slot
+ * is used internally to this routine and will be clobbered.
+ */
+#ifdef AVOID_API_EXPORT
+/* If we re-enable -opt_speed (or -indcall2direct directly) we should add back:
+ * \note This routine is not supported when the -opt_speed option is specified.
+ */
+#endif
+void 
+dr_insert_mbr_instrumentation(void *drcontext, instrlist_t *ilist,
+                              instr_t *instr, void *callee, dr_spill_slot_t scratch_slot);
+
+DR_API
+/**
+ * Assumes that \p instr is a conditional branch
+ * Inserts into \p ilist prior to \p instr instruction(s) to call callee passing
+ * three arguments:
+ * -# address of branch instruction
+ * -# target address of branch
+ * -# 0 if the branch is not taken, 1 if it is taken
+ */
+void 
+dr_insert_cbr_instrumentation(void *drcontext, instrlist_t *ilist,
+                              instr_t *instr, void *callee);
+
+/* FIXME: will never see any ubrs! */
+DR_API
+/**
+ * Assumes that \p instr is a direct, near, unconditional branch.
+ * Inserts into \p ilist prior to \p instr instruction(s) to call callee passing
+ * two arguments:
+ * -# address of branch instruction
+ * -# target address of branch
+ *
+ * \warning Basic block eliding is controlled by -max_elide_jmp.  If that
+ * option is set to non-zero, ubrs may never be seen.
+ */
+void 
+dr_insert_ubr_instrumentation(void *drcontext, instrlist_t *ilist,
+                              instr_t *instr, void *callee);
+
+DR_API
+/**
+ * Returns true if the xmm0 through xmm5 for Windows, or xmm0 through
+ * xmm15 for Linux, fields in dr_mcontext_t are valid for this process
+ * (i.e., whether this process is 64-bit or WOW64, and the processor
+ * supports SSE).
+ */
+bool
+dr_mcontext_xmm_fields_valid(void);
+
+#endif /* CLIENT_INTERFACE */
+/* dr_get_mcontext() needed for translating clean call arg errors */
+
+DR_API
+/**
+ * Copies the machine context captured at dr_insert_clean_call() or
+ * dr_prepare_for_call() or at a pre- or post-syscall event
+ * (dr_register_pre_syscall_event(), dr_register_post_syscall_event())
+ * to \p context.  Does NOT copy the pc field.
+ * If \p app_errno is non-NULL copies the saved application error code
+ * (value of GetLastError() on Windows or errno on Linux) to \p app_errno.
+ * \note NUM_XMM_SLOTS in the dr_mcontext_t.xmm array are filled in, but
+ * only if dr_mcontext_fields_valid() returns true.
+ * \note The context is the context saved at the dr_insert_clean_call() or
+ * dr_prepare_for_call() points.  It does not correct for any registers saved
+ * with dr_save_reg().  To access registers saved with dr_save_reg() from a
+ * clean call use dr_read_saved_reg().
+ */
+void
+dr_get_mcontext(void *drcontext, dr_mcontext_t *context, int *app_errno);
+
+#ifdef CLIENT_INTERFACE
+DR_API
+/**
+ * Sets the machine context to \p context, which is restored at the
+ * cleanup stage of dr_insert_clean_call() or in
+ * dr_cleanup_after_call() or after a pre- or post-syscall event
+ * (dr_register_pre_syscall_event(),
+ * dr_register_post_syscall_event()).  Ignores the pc
+ * field. if \p app_errno is non-NULL sets the application error code (value
+ * of GetLastError() on Windows or errno on Linux) to be restored as well.
+ * \note The xmm0 through xmm5 fields are only set for 64-bit or WOW64
+ * processes where the underlying processor supports SSE.  For
+ * dr_insert_clean_call() that requested \p save_fpstate, the xmm0
+ * through xmm5 values set here override that saved state.
+ * Use dr_mcontext_xmm_fields_valid() to determine whether the fields are valid.
+ */
+void
+dr_set_mcontext(void *drcontext, dr_mcontext_t *context, const int *app_errno);
+
+/* FIXME - combine with dr_set_mcontext()?  Implementation wise it's nice to split the
+ * two since handling the pc with dr_set_mcontext() would complicate the clean call 
+ * handling. But perhaps would be nicer from an interface perspective to combine them. */
+DR_API
+/**
+ * Immediately resumes application execution from a clean call out of the cache (see
+ * dr_insert_clean_call() or dr_prepare_for_call()) or an exception event with the
+ * state specified in \p mcontext (including pc, and including the xmm0 through xmm5
+ * values if dr_mcontext_xmm_fields_valid() returns true) and the application error
+ * code (value of GetLastError() on Windows or errno on Linux) specified by \p app_errno.
+ *
+ * \note dr_get_mcontext() can be used to get the register state (except pc) and the
+ * \p app_errno value saved in dr_insert_clean_call() or dr_prepare_for_call()
+ * \note If floating point state was saved by dr_prepare_for_call() or
+ * dr_insert_clean_call() it is not restored (other than xmm0 through xmm5, if
+ * dr_mcontext_xmm_fields_valid()).  The caller should instead manually save and
+ * restore the floating point state with proc_save_fpstate() and proc_restore_fpstate()
+ * if necessary.
+ * \note If the caller wishes to set any other state (such as xmm registers that are
+ * not part of the mcontext) they may do so by just setting that state in the current
+ * thread before making this call.
+ * \note This routine may only be called from a clean call from the cache. It can not be
+ * called from any registered event callback.
+ * \note This routine doesn't return.
+ */
+void
+dr_redirect_execution(dr_mcontext_t *mcontext, int app_errno);
+
+/* DR_API EXPORT TOFILE dr_tools.h */
+/* DR_API EXPORT BEGIN */
+
+/****************************************************************************
+ * ADAPTIVE OPTIMIZATION SUPPORT
+ */
+/* DR_API EXPORT END */
+
+/* xref PR 199115 and PR 237461: We decided to make the replace and
+ * delete routines valid for -thread_private only.  Both routines
+ * replace for the current thread and leave the other threads
+ * unmodified.  The rationale is that we expect these routines will be
+ * primarily useful for optimization, where a client wants to modify a
+ * fragment specific to one thread.
+ */
+DR_API
+/**
+ * Replaces the fragment with tag \p tag with the instructions in \p
+ * ilist.  This routine is only valid with the -thread_private option;
+ * it replaces the fragment for the current thread only.  After
+ * replacement, the existing fragment is allowed to complete if
+ * currently executing.  For example, a clean call replacing the
+ * currently executing fragment will safely return to the existing
+ * code.  Subsequent executions will use the new instructions.
+ *
+ * \note The routine takes control of \p ilist and all responsibility
+ * for deleting it.  The client should not keep, use, or reference,
+ * the instrlist or any of the instrs it contains after passing.
+ *
+ * \note This routine supports replacement for the current thread
+ * only.  \p drcontext must be from the current thread and must
+ * be the drcontext used to create the instruction list.
+ *
+ * \return false if the fragment does not exist and true otherwise.
+ */
+bool
+dr_replace_fragment(void *drcontext, void *tag, instrlist_t *ilist);
+
+DR_API
+/**
+ * Deletes the fragment with tag \p tag.  This routine is only valid
+ * with the -thread_private option; it deletes the fragment in the
+ * current thread only.  After deletion, the existing fragment is
+ * allowed to complete execution.  For example, a clean call deleting
+ * the currently executing fragment will safely return to the existing
+ * code.  Subsequent executions will cause \DynamoRIO to reconstruct
+ * the fragment, and therefore call the appropriate fragment-creation
+ * event hook, if registered.
+ *
+ * \note This routine supports deletion for the current thread
+ * only.  \p drcontext must be from the current thread and must
+ * be the drcontext used to create the instruction list.
+ *
+ * \return false if the fragment does not exist and true otherwise.
+ */
+bool
+dr_delete_fragment(void *drcontext, void *tag);
+
+/* FIXME - xref PR 227619 - some other event handler are safe (image_load/unload for*
+ * example) which we could note here. */
+DR_API
+/**
+ * Flush all fragments containing any code from the region [\p start, \p start + \p size).
+ * Once this routine returns no execution will occur out of the fragments flushed.
+ * This routine may only be called during a clean call from the cache or from a nudge
+ * event handler.  It may not be called from any other event callback.  No locks can
+ * held when calling this routine.  If called from a clean call, caller can NOT return
+ * to the cache (the fragment that was called out of may have been flushed even if it
+ * doesn't apparently overlap the flushed region). Instead the caller must call
+ * dr_redirect_execution() after this routine to continue execution.  Returns true if
+ * successful.
+ *
+ * \note This routine may not be called from any registered event callback other then
+ * the nudge event; clean calls out of the cache may call this routine.
+ * \note If called from a clean call, caller must continue execution by calling
+ * dr_redirect_execution() after this routine, as the fragment containing the callout may
+ * have been flushed. The context and app_errno to use can be obtained via
+ * dr_get_mcontext() with the exception of the pc to continue at which must be passed as
+ * an argument to the callout (see instr_get_app_pc()) or otherwise determined.
+ * \note This routine may not be called while any locks are held that could block a thread
+ * processing a registered event callback or cache callout.
+ * \note dr_delay_flush_region() has fewer restrictions on use, but is less synchronous.
+ * \note Use \p size == 1 to flush fragments containing the instruction at address 
+ * \p start. A flush of \p size == 0 is not allowed.
+ * \note As currently implemented, dr_delay_flush_region() with no completion callback
+ * routine specified can be substantially more performant. 
+ */
+bool
+dr_flush_region(app_pc start, size_t size);
+
+/* FIXME - get rid of the no locks requirement by making event callbacks !couldbelinking
+ * and no dr locks (see PR 227619) so that client locks owned by this thread can't block
+ * any couldbelinking thread.  FIXME - would be nice to make this available for 
+ * windows since there's less of a performance hit than using synch_all flushing, but
+ * with coarse_units can't tell if we need a synch all flush or not and that confuses
+ * the interface a lot. FIXME - xref PR 227619 - some other event handler are safe
+ * (image_load/unload for example) which we could note here. */
+/* FIXME - add a completion callback (see vm_area_check_shared_pending()). */
+/* FIXME - could enable on windows when -thread_private since no coarse then. */
+DR_API
+/**
+ * Flush all fragments containing any code from the region [\p start, \p start + \p size).
+ * Control will not enter a fragment containing code from the region after this returns,
+ * but a thread already in such a fragment will finish out the fragment.  This includes
+ * the current thread if this is called from a clean call that returns to the cache.
+ * This routine may only be called during a clean call from the cache or from a nudge
+ * event handler.  It may not be called from any other event callback.  No locks can be
+ * held when calling this routine.  Returns true if successful.
+ *
+ * \note This routine may not be called from any registered event callback other then
+ * the nudge event; clean calls out of the cache may call this routine.
+ * \note This routine may not be called while any locks are held that could block a thread
+ * processing a registered event callback or cache callout.
+ * \note dr_delay_flush_region() has fewer restrictions on use, but is less synchronous.
+ * \note Use \p size == 1 to flush fragments containing the instruction at address 
+ * \p start. A flush of \p size == 0 is not allowed.
+ * \note This routine is only available with either the -thread_private
+ * or -enable_full_api options.  It is not available when -opt_memory is specified.
+ */
+bool
+dr_unlink_flush_region(app_pc start, size_t size);
+
+/* FIXME - can we better bound when the flush will happen?  Maybe unlink shared syscalls
+ * or similar or check the queue in more locations?  Should always hit the flush before
+ * executing new code in the cache, and I think we'll always hit it before a nudge is
+ * processed too.  Could trigger a nudge, or do this in a nudge, but that's rather 
+ * expensive. */
+DR_API
+/**
+ * Request a flush of all fragments containing code from the region 
+ * [\p start, \p start + \p size).  The flush will be performed at the next safe 
+ * point in time (usually before any new code is added to the cache after this 
+ * routine is called). If \p flush_completion_callback is non-NULL, it will be 
+ * called with the \p flush_id provided to this routine when the flush completes, 
+ * after which no execution will occur out of the fragments flushed. Returns true 
+ * if the flush was successfully queued.
+ *
+ * \note dr_flush_region() and dr_unlink_flush_region() can give stronger guarantees on
+ * when the flush will occur, but have more restrictions on use.
+ * \note Use \p size == 1 to flush fragments containing the instruction at address 
+ * \p start.  A flush of \p size == 0 is not allowed.
+ * \note As currently implemented there may be a performance penalty for requesting a
+ * \p flush_completion_callback; for most performant usage set 
+ * \p flush_completion_callback to NULL.
+ */
+bool
+dr_delay_flush_region(app_pc start, size_t size, uint flush_id,
+                      void (*flush_completion_callback) (int flush_id));
+
+DR_API
+/** Returns whether or not there is a fragment in code cache with tag \p tag. */
+bool
+dr_fragment_exists_at(void *drcontext, void *tag);
+
+DR_API
+/**
+ * Returns true if a basic block with tag \p tag exists in the code cache.
+ */
+bool
+dr_bb_exists_at(void *drcontext, void *tag);
+
+DR_API 
+/**
+ * Looks up the fragment with tag \p tag.
+ * If not found, returns 0.
+ * If found, returns the total size occupied in the cache by the fragment.
+ */
+uint
+dr_fragment_size(void *drcontext, void *tag);
+
+DR_API 
+/** Retrieves the application PC of a fragment with tag \p tag. */
+app_pc
+dr_fragment_app_pc(void *tag);
+
+#ifdef CUSTOM_TRACES
+/* DR_API EXPORT BEGIN */
+
+/****************************************************************************
+ * CUSTOM TRACE SUPPORT
+ */
+
+/* DR_API EXPORT END */
+
+DR_API
+/**
+ * Marks the fragment associated with tag \p tag as
+ * a trace head.  The fragment need not exist yet -- once it is
+ * created it will be marked as a trace head.
+ *
+ * DR associates a counter with a trace head and once it
+ * passes the -hot_threshold parameter, DR begins building
+ * a trace.  Before each fragment is added to the trace, DR
+ * calls the client's end_trace callback to determine whether
+ * to end the trace.  (The callback will be called both for
+ * standard DR traces and for client-defined traces.)
+ *
+ * \note Some fragments are unsuitable for trace heads. DR will 
+ * ignore attempts to mark such fragments as trace heads and will return
+ * false. If the client marks a fragment that doesn't exist yet as a trace
+ * head and DR later determines that the fragment is unsuitable for
+ * a trace head it will unmark the fragment as a trace head without 
+ * notifying the client.
+ *
+ * \note Some fragments' notion of trace heads is dependent on
+ * which previous block targets them.  For these fragments, calling
+ * this routine will only mark as a trace head for targets from
+ * the same memory region.
+ *
+ * Returns true if the target fragment is marked as a trace head.
+ */
+bool
+dr_mark_trace_head(void *drcontext, void *tag);
+
+DR_API
+/**
+ * Checks to see if the fragment (or future fragment) with tag \p tag
+ * is marked as a trace head.
+ */
+bool
+dr_trace_head_at(void *drcontext, void *tag);
+
+DR_API
+/** Checks to see that if there is a trace in the code cache at tag \p tag. */
+bool
+dr_trace_exists_at(void *drcontext, void *tag);
+#endif /* CUSTOM_TRACES */
+
+#ifdef UNSUPPORTED_API
+DR_API 
+/**
+ * All basic blocks created after this routine is called will have a prefix
+ * that restores the ecx register.  Exit ctis can be made to target this prefix
+ * instead of the normal entry point by using the
+ * instr_branch_set_prefix_target() routine.
+ * \warning This routine should almost always be called during client
+ * initialization, since having a mixture of prefixed and non-prefixed basic
+ * blocks can lead to trouble.
+ */
+void
+dr_add_prefixes_to_basic_blocks(void);
+#endif /* UNSUPPORTED_API */
+
+#endif /* CLIENT_INTERFACE */
+
+/****************************************************************************
+ * proc.c routines exported here due to proc.h being in arch_exports.h
+ * which is included in places where opnd_t isn't a complete type.
+ * These are used for dr_insert_clean_call() and thus are not just CLIENT_INTERFACE.
+ */
+/* DR_API EXPORT TOFILE dr_proc.h */
+DR_API
+/**
+ * Inserts into \p ilist prior to \p where meta-instruction(s) to save the
+ * floating point state into the 16-byte-aligned buffer referred to by
+ * \p buf, which must be 512 bytes for processors with the FXSR
+ * feature, and 108 bytes for those without (where this routine does
+ * not support 16-bit operand sizing).  \p buf should have size of
+ * OPSZ_512; this routine will automatically adjust it to OPSZ_108 if
+ * necessary.  \note proc_fpstate_save_size() can be used to determine
+ * the particular size needed.
+ */
+void
+dr_insert_save_fpstate(void *drcontext, instrlist_t *ilist, instr_t *where,
+                       opnd_t buf);
+
+DR_API
+/**
+ * Inserts into \p ilist prior to \p where meta-instruction(s) to restore the
+ * floating point state from the 16-byte-aligned buffer referred to by
+ * buf, which must be 512 bytes for processors with the FXSR feature,
+ * and 108 bytes for those without (where this routine does not
+ * support 16-bit operand sizing).  \p buf should have size of
+ * OPSZ_512; this routine will automatically adjust it to OPSZ_108 if
+ * necessary.  \note proc_fpstate_save_size() can be used to determine
+ * the particular size needed.
+ */
+void
+dr_insert_restore_fpstate(void *drcontext, instrlist_t *ilist, instr_t *where,
+                          opnd_t buf);
+
+#endif /* _INSTRUMENT_H_ */
diff --git a/core/x86/interp.c b/core/x86/interp.c
new file mode 100644
index 000000000..86ab821b7
--- /dev/null
+++ b/core/x86/interp.c
@@ -0,0 +1,6949 @@
+/* **********************************************************
+ * Copyright (c) 2001-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2001 Hewlett-Packard Company */
+
+/*
+ * interp.c - interpreter used for native trace selection
+ */
+
+#include "globals.h"
+#include "link.h"
+#include "fragment.h"
+#include "emit.h"
+#include "dispatch.h"
+#include "instrlist.h"
+#include "fcache.h"
+#include "monitor.h" /* for trace_abort and monitor_data_t */
+#include "arch.h"
+#include "instr.h"
+#include "instr_create.h"
+#include "decode.h"
+#include "decode_fast.h"
+#include "disassemble.h"
+#include <string.h> /* for memcpy */
+#include "instrument.h"
+#include "hotpatch.h"
+#ifdef RETURN_AFTER_CALL
+# include "rct.h"
+#endif
+#ifdef WINDOWS
+# include "ntdll.h" /* for EXCEPTION_REGISTRATION */
+# include "nudge.h" /* for generic_nudge_target() address */
+#endif
+#include "perscache.h"
+
+#ifdef CHECK_RETURNS_SSE2
+#include <setjmp.h> /* for warning when see libc setjmp */
+#endif
+
+enum { DIRECT_XFER_LENGTH = 5 };
+
+/* forward declarations */
+static void process_nops_for_trace(dcontext_t *dcontext, instrlist_t *ilist, uint flags
+                                   _IF_DEBUG(bool recreating));
+static int fixup_last_cti(dcontext_t *dcontext, instrlist_t *trace,
+                          app_pc next_tag, uint next_flags, uint trace_flags,
+                          fragment_t *prev_f, linkstub_t *prev_l,
+                          bool record_translation, uint *num_exits_deleted/*OUT*/,
+                          /* If non-NULL, only looks inside trace between these two */
+                          instr_t *start_instr, instr_t *end_instr);
+static bool can_use_mangle_trace(void);
+bool mangle_trace(dcontext_t *dcontext, instrlist_t *ilist, monitor_data_t *md);
+
+/* we use a branch limit of 1 to make it easier for the trace
+ * creation mechanism to stitch basic blocks together
+ */
+#define BRANCH_LIMIT 1
+
+/* we limit total bb size to handle cases like infinite loop or sequence
+ * of calls.
+ * also, we have a limit on fragment body sizes, which should be impossible
+ * to break since x86 instrs are max 17 bytes and we only modify ctis.
+ * Although...selfmod mangling does really expand fragments!
+ * -selfmod_max_writes helps for selfmod bbs (case 7893/7909).
+ * System call mangling is also large, for degenerate cases like tests/linux/infinite.
+ * PR 215217: also client additions: we document and assert.
+ * FIXME: need better way to know how big will get, b/c we can construct
+ * cases that will trigger the size assertion!
+ */
+/* define replaced by -max_bb_instrs option */
+
+/* exported so micro routines can assert whether held */
+DECLARE_CXTSWPROT_VAR(mutex_t bb_building_lock, INIT_LOCK_FREE(bb_building_lock));
+
+#ifdef INTERNAL
+file_t bbdump_file = INVALID_FILE;
+#endif
+
+/* initialization */
+void
+interp_init()
+{
+#ifdef INTERNAL
+    if (INTERNAL_OPTION(bbdump_tags)) {
+        bbdump_file = open_log_file("bbs", NULL, 0);
+        ASSERT(bbdump_file != INVALID_FILE);
+    }
+#endif
+}
+
+#ifdef CUSTOM_TRACES_RET_REMOVAL
+# ifdef DEBUG
+/* don't bother with adding lock */
+static int num_rets_removed;
+# endif
+#endif
+
+/* cleanup */
+void
+interp_exit()
+{
+#ifdef INTERNAL
+    if (INTERNAL_OPTION(bbdump_tags)) {
+        os_close(bbdump_file);
+    }
+#endif
+    DELETE_LOCK(bb_building_lock);
+
+    LOG(GLOBAL, LOG_INTERP|LOG_STATS, 1, "Total application code seen: %d KB\n",
+        GLOBAL_STAT(app_code_seen)/1024);
+#ifdef CUSTOM_TRACES_RET_REMOVAL
+# ifdef DEBUG
+    LOG(GLOBAL, LOG_INTERP|LOG_STATS, 1, "Total rets removed: %d\n",
+        num_rets_removed);
+# endif
+#endif
+}
+
+/****************************************************************************
+ ****************************************************************************
+ *
+ * B A S I C   B L O C K   B U I L D I N G
+ */
+
+/* we have a lot of data to pass around so we package it in this struct
+ * so we can have separate routines for readability
+ */
+typedef struct {
+    /* in */
+    app_pc start_pc;
+    bool app_interp;         /* building bb to interp app, as opposed to for pc
+                              * translation or figuring out what pages a bb touches? */
+    bool for_cache;          /* should vmareas be updated? */
+    bool mangle_ilist;       /* should bb ilist be mangled? */
+    bool record_translation; /* store translation info for each instr_t? */
+    bool has_bb_building_lock; /* usually ==for_cache; used for aborting bb building */
+    file_t outf;               /* send disassembly and notes to a file? 
+                              * we use this mainly for dumping trace origins */
+#ifdef CLIENT_INTERFACE
+    bool pass_to_client;     /* pass to client, if a bb hook exists;
+                              * we store this up front to avoid race conditions
+                              * between full_decode setting and hook calling time.
+                              */
+    bool post_client;        /* has the client already processed the bb? */
+    bool for_trace;          /* PR 299808: we tell client if building a trace */
+#endif
+
+    /* in and out */
+    overlap_info_t *overlap_info; /* if non-null, records overlap information here;
+                                   * caller must initialize region_start and region_end */
+
+    /* out */
+    instrlist_t *ilist;
+    uint flags;
+    void *vmlist;
+    app_pc end_pc;
+    bool native_exec;        /* replace cur ilist with a native_exec version */
+#ifdef CLIENT_INTERFACE
+    instrlist_t **unmangled_ilist; /* PR 299808: clone ilist pre-mangling */
+#endif
+
+    /* internal usage only */
+    bool full_decode;        /* decode every instruction into a separate instr_t? */
+    bool follow_direct;      /* elide unconditional branches? */
+    bool check_vm_area;      /* whether to call check_thread_vm_area() */
+    uint num_elide_jmp;
+    uint num_elide_call;
+    app_pc last_page;
+    app_pc cur_pc;
+    app_pc instr_start;
+    app_pc checked_end;       /* end of current vmarea checked */
+    cache_pc exit_target;     /* fall-through target of final instr */
+    uint exit_type;           /* indirect branch type  */
+#ifdef LINUX
+    bool invalid_instr_hack;
+#endif
+    instr_t *instr;             /* the current instr */
+    int eflags;
+    DEBUG_DECLARE(bool initialized;)
+} build_bb_t;
+
+/* forward decl */
+static inline bool
+bb_process_syscall(dcontext_t *dcontext, build_bb_t *bb);
+
+static void
+init_build_bb(build_bb_t *bb, app_pc start_pc, bool app_interp, bool for_cache,
+              bool mangle_ilist, bool record_translation, file_t outf, uint known_flags,
+              overlap_info_t *overlap_info)
+{
+    memset(bb, 0, sizeof(*bb));
+    bb->check_vm_area = true;
+    bb->start_pc = start_pc;
+    bb->app_interp = app_interp;
+    bb->for_cache = for_cache;
+    bb->mangle_ilist = mangle_ilist;
+    bb->record_translation = record_translation;
+    bb->outf = outf;
+    bb->overlap_info = overlap_info;
+    bb->follow_direct = !TEST(FRAG_SELFMOD_SANDBOXED, known_flags);
+    bb->flags = known_flags;
+    DODEBUG(bb->initialized = true;);
+}
+
+static void
+reset_overlap_info(dcontext_t *dcontext, build_bb_t *bb)
+{
+    bb->overlap_info->start_pc = bb->start_pc;
+    bb->overlap_info->min_pc = bb->start_pc;
+    bb->overlap_info->max_pc = bb->start_pc;
+    bb->overlap_info->contiguous = true;
+    bb->overlap_info->overlap = false;
+}
+
+static void
+update_overlap_info(dcontext_t *dcontext, build_bb_t *bb, app_pc new_pc, bool jmp)
+{
+    if (new_pc < bb->overlap_info->min_pc)
+        bb->overlap_info->min_pc = new_pc;
+    if (new_pc > bb->overlap_info->max_pc)
+        bb->overlap_info->max_pc = new_pc;
+    /* we get called at end of all contiguous intervals, so ignore jmps */
+    LOG(THREAD, LOG_ALL, 5, "\t    app_bb_overlaps "PFX".."PFX" %s\n",
+        bb->last_page, new_pc, jmp?"jmp":"");
+    if (!bb->overlap_info->overlap && !jmp) {
+        /* contiguous interval: prev_pc..new_pc (open-ended) */
+        if (bb->last_page < bb->overlap_info->region_end &&
+            new_pc > bb->overlap_info->region_start) {
+            LOG(THREAD_GET, LOG_ALL, 5, "\t    it overlaps!\n");
+            bb->overlap_info->overlap = true;
+        }
+    }
+    if (bb->overlap_info->contiguous && jmp)
+        bb->overlap_info->contiguous = false;
+}
+
+#ifdef DEBUG
+# define BBPRINT(bb, level, ...) do {                     \
+    LOG(THREAD, LOG_INTERP, level, __VA_ARGS__);          \
+    if (bb->outf != INVALID_FILE && bb->outf != (THREAD)) \
+        print_file(bb->outf, __VA_ARGS__);                \
+} while (0);
+#else
+# ifdef INTERNAL
+#  define BBPRINT(bb, level, ...) do {                     \
+     if (bb->outf != INVALID_FILE)                         \
+         print_file(bb->outf, __VA_ARGS__);                \
+   } while (0);
+# else
+#  define BBPRINT(bb, level, ...) /* nothing */
+# endif
+#endif
+
+#ifdef WINDOWS
+extern void intercept_load_dll(void);
+extern void intercept_unload_dll(void);
+# ifdef INTERNAL
+extern void DllMainThreadAttach(void);
+# endif
+#endif
+
+/* forward declarations */
+static bool
+mangle_bb_ilist(dcontext_t *dcontext, build_bb_t *bb);
+
+static void
+build_native_exec_bb(dcontext_t *dcontext, build_bb_t *bb);
+
+static bool
+at_native_exec_gateway(dcontext_t *dcontext, app_pc start
+                       _IF_DEBUG(bool xfer_target));
+
+#ifdef DEBUG
+static void
+report_native_module(dcontext_t *dcontext, app_pc modpc);
+#endif
+
+/* return true if pc is a call target that should NOT be inlined */
+#if defined(DEBUG) || !defined(WINDOWS)
+/* cl.exe non-debug won't let other modules use it if inlined */
+inline
+#endif 
+bool
+must_not_be_inlined(app_pc pc)
+{
+    return (
+#ifdef INTERNAL
+            !dynamo_options.inline_calls
+#else
+            0
+#endif
+#ifdef WINDOWS
+            || pc == (app_pc)intercept_load_dll
+            || pc == (app_pc)intercept_unload_dll
+            /* we're guaranteed to have direct calls to the next routine since our
+             * own DllMain calls it! */
+# ifdef INTERNAL
+            || pc == (app_pc) DllMainThreadAttach
+# endif
+            /* check for nudge handling escape from cache */
+            || (pc == (app_pc)generic_nudge_handler)
+#else
+            /* PR 200203: long-term we want to control loading of client
+             * libs, but for now we have to let the loader call _fini()
+             * in the client, which may end up calling __wrap_free().
+             * It's simpler to let those be interpreted and make a native
+             * call to the real heap routine here as this is a direct
+             * call whereas we'd need native_exec for the others:
+             */
+            || pc == (app_pc)global_heap_free
+#endif
+        );
+}
+
+/* return true if pc is a direct jmp target that should NOT be elided and followed */
+static inline bool
+must_not_be_elided(app_pc pc)
+{
+#ifdef WINDOWS
+    /* Allow only the return jump in the landing pad to be elided, as we
+     * interpret the return path from trampolines.  The forward jump leads to
+     * the trampoline and shouldn't be elided. */
+    if (vmvector_overlap(landing_pad_areas, pc, pc + 1)) {
+        /* Ok, is it the return jump, which is always 32-bit rel and the target
+         * is not in the interception buffer?  If so, elide. */
+        if (*pc == JMP_REL32_OPCODE &&
+            !is_in_interception_buffer(PC_RELATIVE_TARGET(pc + 1))) {
+            return false;
+        } else {
+            return true;
+        }
+    }
+#endif
+    return (0
+#ifdef WINDOWS
+            /* we insert trampolines by adding direct jmps to our interception code buffer
+             * we don't want to interpret the code in that buffer, as it may swap to the
+             * dstack and mess up a return-from-fcache.
+             * N.B.: if use this routine anywhere else, pay attention to the
+             * hack for is_syscall_trampoline() in the use here!
+             */
+            || (is_in_interception_buffer(pc))
+#else /* LINUX */
+#endif
+            );
+}
+
+#ifdef DR_APP_EXPORTS
+/* This function allows automatically injected dynamo to ignore
+ * dynamo API routines that would really mess things up
+ */
+inline bool
+must_escape_from(app_pc pc)
+{
+    /* if ever find ourselves at top of one of these, immediately issue
+     * a ret instruction...haven't set up frame yet so stack fine, only
+     * problem is return value, go ahead and overwrite xax, it's caller-saved
+     * FIXME: is this ok?
+     */
+    /* Note that we can't just look for direct calls to these functions
+     * because of stubs, etc. that end up doing indirect jumps to them!
+     */
+    bool res = false
+#ifdef DR_APP_EXPORTS
+        || (automatic_startup &&
+            (pc == (app_pc)dynamorio_app_init ||
+             pc == (app_pc)dr_app_start ||
+             pc == (app_pc)dynamo_thread_init ||
+             pc == (app_pc)dynamorio_app_exit ||
+             /* dr_app_stop is a nop already */
+             pc == (app_pc)dynamo_thread_exit))
+#endif
+        ;
+#ifdef DEBUG
+    if (res) {
+# ifdef DR_APP_EXPORTS
+        LOG(THREAD_GET, LOG_INTERP, 3, "must_escape_from: found ");
+        if (pc == (app_pc)dynamorio_app_init)
+            LOG(THREAD_GET, LOG_INTERP, 3, "dynamorio_app_init\n");
+        else if (pc == (app_pc)dr_app_start)
+            LOG(THREAD_GET, LOG_INTERP, 3, "dr_app_start\n");
+        /* FIXME: are dynamo_thread_* still needed hered? */
+        else if (pc == (app_pc)dynamo_thread_init)
+            LOG(THREAD_GET, LOG_INTERP, 3, "dynamo_thread_init\n");
+        else if (pc ==  (app_pc)dynamorio_app_exit)
+            LOG(THREAD_GET, LOG_INTERP, 3, "dynamorio_app_exit\n");
+        else if (pc ==  (app_pc)dynamo_thread_exit)
+            LOG(THREAD_GET, LOG_INTERP, 3, "dynamo_thread_exit\n");
+# endif
+    }
+#endif
+
+    return res;
+}
+#endif /* DR_APP_EXPORTS */
+
+/* Perform checks such as looking for dynamo stopping points and bad places
+ * to be.  We assume we only have to check after control transfer instructions,
+ * i.e., we assume that all of these conditions are procedures that are only
+ * entered by calling or jumping, never falling through.
+ */
+static inline bool
+check_for_stopping_point(dcontext_t *dcontext, build_bb_t *bb)
+{
+#ifdef DR_APP_EXPORTS
+    if (must_escape_from(bb->cur_pc)) {
+        BBPRINT(bb, 3, "interp: emergency exit from "PFX"\n", bb->cur_pc);
+        /* if ever find ourselves at top of one of these, immediately issue
+         * a ret instruction...haven't set up frame yet so stack fine, only
+         * problem is return value, go ahead and overwrite xax, it's
+         * caller-saved.
+         * FIXME: is this ok?
+         */
+        /* move 0 into xax -- our functions return 0 to indicate success */
+        instrlist_append(bb->ilist, /* x64 will zero-extend to rax */
+                         INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_EAX),
+                                              OPND_CREATE_INT32(0)));
+        /* insert a ret instruction */
+        instrlist_append(bb->ilist, INSTR_CREATE_ret(dcontext));
+        /* should this be treated as a real return? */
+        bb->exit_type |= LINK_INDIRECT | LINK_RETURN;
+        ASSERT_NOT_IMPLEMENTED(false);
+        bb->exit_target = get_ibl_routine(dcontext, IBL_LINKED, DEFAULT_IBL_BB(), IBL_RETURN);
+        return true;
+    }
+#endif /* DR_APP_EXPORTS */
+
+#ifdef CHECK_RETURNS_SSE2
+    if (bb->cur_pc == (app_pc)longjmp) {
+        SYSLOG_INTERNAL_WARNING("encountered longjmp, which will cause ret mismatch!");
+    }
+#endif
+
+    return is_stopping_point(dcontext, bb->cur_pc);
+}
+
+/* Arithmetic eflags analysis to see if sequence of instrs reads an
+ * arithmetic flag prior to writing it.
+ * Usage: first initialize status to 0 and eflags_6 to 0.
+ * Then call this routine for each instr in sequence, assigning result to status.
+ * eflags_6 holds flags written and read so far.
+ * Uses these flags, defined in instr.h, as status values:
+ *   EFLAGS_WRITE_6   = writes all 6 flags before reading any
+ *   EFLAGS_WRITE_OF  = writes OF before reading it
+ *   EFLAGS_READ_6    = reads some of 6 before writing
+ *   EFLAGS_READ_OF   = reads OF before writing OF
+ *   0                = no information yet
+ */
+static inline int
+eflags_analysis(instr_t *instr, int status, uint *eflags_6)
+{
+    uint e6 = *eflags_6; /* local copy */
+    uint e6_w2r = EFLAGS_WRITE_TO_READ(e6);
+    uint instr_eflags = instr_get_arith_flags(instr);
+
+    /* Keep going until result is non-zero, also keep going if
+     * result is writes to OF to see if later writes to rest of flags
+     * before reading any, and keep going if reads one of the 6 to see
+     * if later writes to OF before reading it.
+     */
+    if (instr_eflags == 0 || status == EFLAGS_READ_OF || status == EFLAGS_WRITE_6)
+        return status;
+    /* we ignore interrupts */
+    if ((instr_eflags & EFLAGS_READ_6) != 0 &&
+        (!instr_opcode_valid(instr) || !instr_is_interrupt(instr))) {
+        /* store the flags we're reading */
+        e6 |= (instr_eflags & EFLAGS_READ_6);
+        *eflags_6 = e6;
+        if ((e6_w2r | (instr_eflags & EFLAGS_READ_6)) != e6_w2r) {
+            /* we're reading a flag that has not been written yet */
+            status = EFLAGS_READ_6; /* some read before all written */
+            LOG(THREAD_GET, LOG_INTERP, 4, "\treads flag before writing it!\n");
+            if ((instr_eflags & EFLAGS_READ_OF) != 0 && (e6 & EFLAGS_WRITE_OF) == 0) {
+                status = EFLAGS_READ_OF; /* reads OF before writing! */
+                LOG(THREAD_GET, LOG_INTERP, 4, "\t  reads OF prior to writing it!\n");
+            }
+        }
+    } else if ((instr_eflags & EFLAGS_WRITE_6) != 0) {
+        /* store the flags we're writing */
+        e6 |= (instr_eflags & EFLAGS_WRITE_6);
+        *eflags_6 = e6;
+        /* check if all written but none read yet */
+        if ((e6 & EFLAGS_WRITE_6) == EFLAGS_WRITE_6 && (e6 & EFLAGS_READ_6) == 0) {
+            status = EFLAGS_WRITE_6; /* all written before read */
+            LOG(THREAD_GET, LOG_INTERP, 4, "\twrote all 6 flags now!\n");
+        }
+        /* check if at least OF was written but not read */
+        else if ((e6 & EFLAGS_WRITE_OF) != 0 && (e6 & EFLAGS_READ_OF) == 0) {
+            status = EFLAGS_WRITE_OF; /* OF written before read */
+            LOG(THREAD_GET, LOG_INTERP, 4, "\twrote overflow flag before reading it!\n");
+        }
+    }
+    return status;
+}
+
+
+/* check origins of code for several purposes:
+ * 1) we need list of areas where this thread's fragments come
+ *    from, for faster flushing on munmaps
+ * 2) also for faster flushing, each vmarea has a list of fragments
+ * 3) we need to mark as read-only any writable region that
+ *    has a fragment come from it, to handle self-modifying code
+ * 4) for PROGRAM_SHEPHERDING restricted code origins for security
+ * 5) for restricted execution environments: not letting bb cross regions
+ */
+
+/*
+  FIXME CASE 7380:
+  since report security violation before execute off bad page, can be
+  false positive due to:
+  - a faulting instruction in middle of bb would have prevented
+    getting there
+  - ignorable syscall in middle
+  - self-mod code would have ended bb sooner than bad page
+
+  One solution is to have check_thread_vm_area() return false and have
+  bb building stop at checked_end if a violation will occur when we
+  get there.  Then we only raise the violation once building a bb
+  starting there.
+ */
+
+static inline void
+check_new_page_start(dcontext_t *dcontext, build_bb_t *bb)
+{
+    DEBUG_DECLARE(bool ok;)
+    if (!bb->check_vm_area)
+        return;
+    DEBUG_DECLARE(ok =) check_thread_vm_area(dcontext, bb->start_pc, bb->start_pc,
+                                             (bb->for_cache ? &bb->vmlist : NULL),
+                                             &bb->flags, &bb->checked_end,
+                                             false/*!xfer*/);
+    ASSERT(ok); /* cannot return false on non-xfer */
+    bb->last_page = bb->start_pc;
+    if (bb->overlap_info != NULL)
+        reset_overlap_info(dcontext, bb);
+}
+
+/* Walk forward in straight line from prev_pc to new_pc.  
+ * FIXME: with checked_end we don't need to call this on every contig end
+ * while bb building like we used to.  Should revisit the overlap info and
+ * walk_app_bb reasons for keeping those contig() calls and see if we can
+ * optimize them away for bb building at least.
+ */
+static inline void
+check_new_page_contig(dcontext_t *dcontext, build_bb_t *bb, app_pc new_pc)
+{
+    if (!bb->check_vm_area)
+        return;
+    if (bb->overlap_info != NULL)
+        update_overlap_info(dcontext, bb, new_pc, false/*not jmp*/);
+    if (bb->checked_end == NULL) {
+        ASSERT(new_pc == bb->start_pc);
+    } else if (new_pc > bb->checked_end) {
+        DEBUG_DECLARE(bool ok =)
+            check_thread_vm_area(dcontext, new_pc, bb->start_pc,
+                                 (bb->for_cache ? &bb->vmlist : NULL),
+                                 &bb->flags, &bb->checked_end,
+                                 false/*!xfer*/);
+        ASSERT(ok); /* cannot return false on non-xfer */
+    }
+    DOLOG(4, LOG_INTERP, {
+        if (PAGE_START(bb->last_page) != PAGE_START(new_pc))
+            LOG(THREAD, LOG_INTERP, 4, "page boundary crossed\n");
+    });
+    bb->last_page = new_pc; /* update even if not new page, for walk_app_bb */
+}
+
+/* Direct cti from prev_pc to new_pc */
+static bool
+check_new_page_jmp(dcontext_t *dcontext, build_bb_t *bb, app_pc new_pc)
+{
+    /* cur sandboxing doesn't handle direct cti
+     * not good enough to only check this at top of interp -- could walk contig
+     * from non-selfmod to selfmod page, and then do a direct cti, which
+     * check_thread_vm_area would allow (no flag changes on direct cti)!
+     * also not good enough to put this check in check_thread_vm_area, as that
+     * only checks across pages.
+     */
+    if ((bb->flags & FRAG_SELFMOD_SANDBOXED) != 0)
+        return false;         
+    if (PAGE_START(bb->last_page) != PAGE_START(new_pc))
+        LOG(THREAD, LOG_INTERP, 4, "page boundary crossed\n");
+    /* do not walk into a native exec dll (we assume not currently there,
+     * though could happen if bypass a gateway -- even then this is a feature
+     * to allow getting back to native ASAP)
+     * FIXME: we could assume that such direct calls only
+     * occur from DGC, and rely on check_thread_vm_area to disallow,
+     * as an (unsafe) optimization
+     */
+    if (DYNAMO_OPTION(native_exec) &&
+        DYNAMO_OPTION(native_exec_dircalls) &&
+        !vmvector_empty(native_exec_areas) &&
+        vmvector_overlap(native_exec_areas, new_pc, new_pc+1))
+        return false;
+    if (!bb->check_vm_area)
+        return true;
+    /* need to check this even if an intra-page jmp b/c we allow sub-page vm regions */
+    if (!check_thread_vm_area(dcontext, new_pc, bb->start_pc,
+                              (bb->for_cache ? &bb->vmlist : NULL),
+                              &bb->flags, &bb->checked_end, true/*xfer*/))
+        return false;
+    if (bb->overlap_info != NULL)
+        update_overlap_info(dcontext, bb, new_pc, true/*jmp*/);
+    bb->flags |= FRAG_HAS_DIRECT_CTI;
+    bb->last_page = new_pc; /* update even if not new page, for walk_app_bb */
+    return true;
+}
+
+static inline void
+bb_process_invalid_instr(dcontext_t *dcontext, build_bb_t *bb)
+{
+    
+    /* invalid instr: end bb BEFORE the instr, we'll throw exception if we
+     * reach the instr itself
+     */
+    LOG(THREAD, LOG_INTERP, 2, "interp: invalid instr at "PFX"\n", bb->instr_start);
+    /* This routine is called by more than just bb builder, also used
+     * for recreating state, so check bb->app_interp parameter to find out
+     * if building a real app bb to be executed
+     */
+    if (bb->app_interp && bb->instr_start == bb->start_pc) {
+        /* This is first instr in bb so it will be executed for sure and
+         * we need to generate an invalid instruction exception.
+         * A benefit of being first instr is that the state is easy
+         * to translate.
+         */
+#ifdef WINDOWS
+        /* Copying the invalid bytes and having the processor generate
+         * the exception would be cleaner in every way except our fear
+         * of a new processor making those bytes valid and us inadvertently
+         * executing the unexamined instructions afterward, since we do not
+         * know the proper amount of bytes to copy.  Copying is cleaner
+         * since Windows splits invalid instructions into different cases,
+         * an invalid lock prefix and maybe some other distinctions
+         * (it's all interrupt 6 to the processor), and it is hard to
+         * duplicate Windows' behavior in our forged exception.
+         */
+        /* FIXME case 10672: provide a runtime option to specify new
+         * instruction formats to avoid this app exception */
+        ASSERT(dcontext->bb_build_info == bb);
+        bb_build_abort(dcontext, true/*clean vm area*/);
+        /* FIXME : we use illegal instruction here, even though we 
+         * know windows uses different exception codes for different
+         * types of invalid instructions (for ex. STATUS_INVALID_LOCK
+         * _SEQUENCE for lock prefix on a jmp instruction)
+         */
+        if (TEST(DUMPCORE_FORGE_ILLEGAL_INST, DYNAMO_OPTION(dumpcore_mask)))
+            os_dump_core("Warning: Encountered Illegal Instruction");
+        os_forge_exception(bb->instr_start, ILLEGAL_INSTRUCTION_EXCEPTION);
+        ASSERT_NOT_REACHED();
+#else
+        /* FIXME: Linux hack until we have a real os_forge_exception implementation:
+         * copy the bytes and have the process generate the exception.
+         * Once remove this, also disable check at top of insert_selfmod_sandbox
+         * FIXME PR 307880: we now have a preliminary
+         * os_forge_exception impl, but I'm leaving this hack until
+         * we're more comfortable w/ our forging.
+         */
+        uint sz;
+        instrlist_append(bb->ilist, bb->instr);
+        /* pretend raw bits valid to get it encoded
+         * For now we just do 17 bytes, being wary of unreadable pages.
+         * FIXME: better solution is to have decoder guess at length (if
+         * ok opcode just bad lock prefix or something know length, if
+         * bad opcode just bytes up until know it's bad).
+         */
+        if (!is_readable_without_exception(bb->instr_start, MAX_INSTR_LENGTH)) {
+            app_pc nxt_page = (app_pc) ALIGN_FORWARD(bb->instr_start, PAGE_SIZE);
+            sz = nxt_page - bb->instr_start;
+        } else {
+            sz = MAX_INSTR_LENGTH;
+        }
+        bb->cur_pc += sz; /* just in case, should have a non-self target */
+        ASSERT(bb->cur_pc > bb->instr_start); /* else still a self target */
+        instr_set_raw_bits(bb->instr, bb->instr_start, sz);
+        bb->invalid_instr_hack = true;
+#endif
+    } else {
+        instr_destroy(dcontext, bb->instr);
+        bb->instr = NULL;
+    }
+}
+
+/* returns true to indicate "elide and continue" and false to indicate "end bb now"
+ * should be used both for converted indirect jumps and 
+ * FIXME: for direct jumps by bb_process_ubr
+ */
+static inline bool
+follow_direct_jump(dcontext_t *dcontext, build_bb_t *bb, 
+                   app_pc target)
+{
+    if (bb->follow_direct &&
+        bb->num_elide_jmp < DYNAMO_OPTION(max_elide_jmp) &&
+        (DYNAMO_OPTION(elide_back_jmps) || bb->cur_pc <= target)) {
+        check_new_page_contig(dcontext, bb, bb->cur_pc-1);
+        if (check_new_page_jmp(dcontext, bb, target)) {
+            /* Elide unconditional branch and follow target */
+            bb->num_elide_jmp++;
+            STATS_INC(total_elided_jmps);
+            STATS_TRACK_MAX(max_elided_jmps, bb->num_elide_jmp);
+            bb->cur_pc = target;
+            BBPRINT(bb, 4, "        continuing at target "PFX"\n", bb->cur_pc);
+
+            return true;        /* keep bb going */
+        } else {
+            BBPRINT(bb, 3, "        NOT following jmp from "PFX" to "PFX"\n",
+                    bb->instr_start, target);
+        }
+    } else {
+        BBPRINT(bb, 3, "   NOT attempting to follow jump from "PFX" to "PFX"\n",
+                bb->instr_start, target);
+    }
+    return false;               /* stop bb */
+}
+
+/* returns true to indicate "elide and continue" and false to indicate "end bb now" */
+static inline bool
+bb_process_ubr(dcontext_t *dcontext, build_bb_t *bb)
+{
+    app_pc tgt = (byte *) opnd_get_pc(instr_get_target(bb->instr));
+    BBPRINT(bb, 4, "interp: direct jump at "PFX"\n", bb->instr_start);
+    if (must_not_be_elided(tgt)) {
+#ifdef WINDOWS
+        if (is_syscall_trampoline(tgt)) {
+            /* HACK to avoid entering the syscall trampoline that is meant
+             * only for native syscalls -- we replace the jmp with the
+             * original app mov immed that it replaced
+             */
+            BBPRINT(bb, 3,
+                    "interp: replacing syscall trampoline @"PFX" w/ orig mov @"PFX"\n",
+                    bb->instr_start, tgt);
+            instr_reset(dcontext, bb->instr);
+
+            /* leave bb->cur_pc unchanged */
+            decode(dcontext, tgt, bb->instr);
+            /* ASSUMPTION: syscall trampoline puts hooked instruction
+             * (usually mov_imm but can be lea if hooked_deeper) here */
+            ASSERT(instr_get_opcode(bb->instr) == OP_mov_imm ||
+                   (instr_get_opcode(bb->instr) == OP_lea &&
+                    DYNAMO_OPTION(native_exec_hook_conflict) == 
+                    HOOKED_TRAMPOLINE_HOOK_DEEPER));
+            instrlist_append(bb->ilist, bb->instr);
+            /* translation should point to the trampoline at the
+             * original application address
+             */
+            if (bb->record_translation)
+                instr_set_translation(bb->instr, bb->instr_start);
+            if (instr_get_opcode(bb->instr) == OP_lea) {
+                app_pc translation = bb->instr_start + 
+                    instr_length(dcontext, bb->instr);
+                ASSERT_CURIOSITY(instr_length(dcontext, bb->instr) == 4);
+                /* we hooked deep need to add the int 2e instruction */
+                /* can't use create_syscall_instr because of case 5217 hack */
+                ASSERT(get_syscall_method() == SYSCALL_METHOD_INT);
+                bb->instr = INSTR_CREATE_int(dcontext,
+                                             opnd_create_immed_int((char)0x2e,
+                                                                   OPSZ_1));
+                if (bb->record_translation)
+                    instr_set_translation(bb->instr, translation);
+                ASSERT(instr_is_syscall(bb->instr) &&
+                       instr_get_opcode(bb->instr) == OP_int);
+                instrlist_append(bb->ilist, bb->instr);
+                return bb_process_syscall(dcontext, bb);
+            }
+            return true; /* keep bb going */
+        }
+#endif 
+        BBPRINT(bb, 3, "interp: NOT following jmp to "PFX"\n", tgt);
+        /* add instruction to instruction list */
+        instrlist_append(bb->ilist, bb->instr);
+        /* indicate that relative target must be
+         * re-encoded, and that it is not an exit cti
+         */
+        instr_set_ok_to_mangle(bb->instr, false);
+        instr_set_raw_bits_valid(bb->instr, false);
+        /* Case 8711: coarse-grain can't handle non-exit cti */
+        bb->flags &= ~FRAG_COARSE_GRAIN;
+        STATS_INC(coarse_prevent_cti);
+        return false; /* end bb now */
+    } else {
+        if (bb->follow_direct &&
+            bb->num_elide_jmp < DYNAMO_OPTION(max_elide_jmp) &&
+            (DYNAMO_OPTION(elide_back_jmps) || bb->cur_pc <= tgt)) {
+            check_new_page_contig(dcontext, bb, bb->cur_pc-1);
+            if (check_new_page_jmp(dcontext, bb, tgt)) {
+                /* Elide unconditional branch and follow target */
+                bb->num_elide_jmp++;
+                STATS_INC(total_elided_jmps);
+                STATS_TRACK_MAX(max_elided_jmps, bb->num_elide_jmp);
+                bb->cur_pc = tgt;
+                BBPRINT(bb, 4, "        continuing at target "PFX"\n", bb->cur_pc);
+                /* pretend never saw this ubr: delete instr, then continue */
+                instr_destroy(dcontext, bb->instr);
+                bb->instr = NULL;
+                return true; /* keep bb going */
+            } else {
+                BBPRINT(bb, 3, "        NOT following direct jmp from "PFX" to "PFX"\n",
+                        bb->instr_start, tgt);
+            }
+        }
+        /* End this bb now */
+        bb->exit_target = opnd_get_pc(instr_get_target(bb->instr));
+        instrlist_append(bb->ilist, bb->instr);
+        return false; /* end bb */
+    }
+    return true; /* keep bb going */
+}
+
+
+/* returns true if call is elided,
+ * and false if not following due to hitting a limit or other reason */
+static bool
+follow_direct_call(dcontext_t *dcontext, build_bb_t *bb, app_pc callee)
+{
+    /* FIXME: This code should be reused in bb_process_convertible_indcall()
+     * and in bb_process_call_direct()
+     */
+    if (bb->follow_direct &&
+        bb->num_elide_call < DYNAMO_OPTION(max_elide_call) &&
+        (DYNAMO_OPTION(elide_back_calls) || bb->cur_pc <= callee)) {
+        check_new_page_contig(dcontext, bb, bb->cur_pc-1);
+        if (check_new_page_jmp(dcontext, bb, callee)) {
+            bb->num_elide_call++;
+            STATS_INC(total_elided_calls);
+            STATS_TRACK_MAX(max_elided_calls, bb->num_elide_call);
+            bb->cur_pc = callee;
+
+            BBPRINT(bb, 4, "   continuing in callee at "PFX"\n", bb->cur_pc);
+            return true;        /* keep bb going in callee */
+        } else {
+            BBPRINT(bb, 3,
+                    "   NOT following direct (or converted) call from "PFX" to "PFX"\n",
+                    bb->instr_start, callee);
+        }
+    }
+    else {
+        BBPRINT(bb, 3, "   NOT attempting to follow call from "PFX" to "PFX"\n",
+                bb->instr_start, callee);
+    }    
+    return false; /* stop bb */
+}
+
+/* returns true to indicate "elide and continue" and false to indicate "end bb now" */
+static inline bool
+bb_process_call_direct(dcontext_t *dcontext, build_bb_t *bb)
+{
+    byte *callee = (byte *)opnd_get_pc(instr_get_target(bb->instr));
+# ifdef CUSTOM_TRACES_RET_REMOVAL
+    if (callee == bb->instr_start + 5) {
+        LOG(THREAD, LOG_INTERP, 4, "found call to next instruction\n");
+    } else
+        dcontext->num_calls++;
+# endif
+    STATS_INC(num_all_calls);
+    BBPRINT(bb, 4, "interp: direct call at "PFX"\n", bb->instr_start);
+    if (must_not_be_inlined(callee)) {
+        BBPRINT(bb, 3, "interp: NOT inlining call to "PFX"\n", callee);
+        /* Case 8711: coarse-grain can't handle non-exit cti.
+         * If we allow this fragment to be coarse we must kill the freeze
+         * nudge thread!
+         */
+        bb->flags &= ~FRAG_COARSE_GRAIN;
+        STATS_INC(coarse_prevent_cti);
+        return true; /* keep bb going, w/o inlining call */
+    } else {
+        /* FIXME: use follow_direct_call() */
+        if (bb->follow_direct &&
+            bb->num_elide_call < DYNAMO_OPTION(max_elide_call) &&
+            (DYNAMO_OPTION(elide_back_calls) || bb->cur_pc <= callee)) {
+            check_new_page_contig(dcontext, bb, bb->cur_pc-1);
+            if (check_new_page_jmp(dcontext, bb, callee)) {
+                bb->num_elide_call++;
+                STATS_INC(total_elided_calls);
+                STATS_TRACK_MAX(max_elided_calls, bb->num_elide_call);
+                bb->cur_pc = callee;
+                BBPRINT(bb, 4, "      continuing in callee at "PFX"\n", bb->cur_pc);
+                return true; /* keep bb going */
+            } else {
+                BBPRINT(bb, 3, "        NOT following direct call from "PFX" to "PFX"\n",
+                        bb->instr_start, callee);
+            }
+        }
+        /* End this bb now */
+        bb->exit_target = callee;
+        return false; /* end bb now */
+    }
+    return true; /* keep bb going */
+}
+
+#ifdef WINDOWS
+/* Walk up from the bb->instr and verify that the preceding instructions
+ * match the pattern that we expect to precede a sysenter. */
+static instr_t *
+bb_verify_sysenter_pattern(dcontext_t *dcontext, build_bb_t *bb)
+{
+    /* Walk back up 2 instructions and verify that there's a
+     * "call (%xdx); mov %xsp -> %xdx" or "call %xdx; mov %xsp -> %xdx"
+     * just prior to the sysenter.
+     * We use "xsp" and "xdx" to be ready for x64 sysenter though we don't
+     * expect to see it.
+     */
+    instr_t *instr = instr_get_prev_expanded(dcontext, bb->ilist, bb->instr);
+
+    /* FIXME Relax the pattern matching on the "mov; call" pair so that small
+     * changes in the register dataflow and call construct are tolerated. */
+
+    /* Did we find a "mov %xsp -> %xdx"? */
+    if (!(instr != NULL && instr_get_opcode(instr) == OP_mov_ld &&
+          instr_num_srcs(instr) == 1 && instr_num_dsts(instr) == 1 &&
+          opnd_is_reg(instr_get_dst(instr, 0)) &&
+          opnd_get_reg(instr_get_dst(instr, 0)) == REG_XDX &&
+          opnd_is_reg(instr_get_src(instr, 0)) &&
+          opnd_get_reg(instr_get_src(instr, 0)) == REG_XSP)) {
+        BBPRINT(bb, 3, "bb_verify_sysenter_pattern -- mov didn't match\n");
+        return NULL;
+    }
+    instr = instr_get_prev_expanded(dcontext, bb->ilist, instr);
+
+    /* Did we find a "call (%xdx) or "call %xdx" that's already marked
+     * for ind->direct call conversion? */
+    if (!(instr != NULL && TEST(INSTR_IND_CALL_DIRECT, instr->flags) &&
+          instr_is_call_indirect(instr) &&
+          /* The 2nd src operand should always be %xsp. */
+          opnd_is_reg(instr_get_src(instr, 1)) &&
+          opnd_get_reg(instr_get_src(instr, 1)) == REG_XSP &&
+          /* Match 'call (%xdx)' for post-SP2. */
+          ((opnd_is_near_base_disp(instr_get_src(instr, 0)) &&
+            opnd_get_base(instr_get_src(instr, 0)) == REG_XDX &&
+            opnd_get_disp(instr_get_src(instr, 0)) == 0) ||
+           /* Match 'call %xdx' for pre-SP2. */
+           (opnd_is_reg(instr_get_src(instr, 0)) &&
+            opnd_get_reg(instr_get_src(instr, 0)) == REG_XDX)))) {
+        BBPRINT(bb, 3, "bb_verify_sysenter_pattern -- call didn't match\n");
+        return NULL;
+    }
+
+    return instr;
+}
+
+/* Only used for the Borland SEH exemption. */
+/* FIXME - we can't really tell a push from a pop since both are typically a
+ * mov to fs:[0], but double processing doesn't hurt. */
+/* NOTE we don't see dynamic SEH frame pushes, we only see the first SEH push
+ * per mov -> fs:[0] instruction in the app.  So we don't see modified in place
+ * handler addresses (see at_Borland_SEH_rct_exemption()) or handler addresses
+ * that are passed into a shared routine that sets up the frame (not yet seen,
+ * note that MS dlls that have a _SEH_prolog hardcode the handler address in
+ * the _SEH_prolog routine, only the data is passed in).
+ */
+static void
+bb_process_SEH_push(dcontext_t *dcontext, build_bb_t *bb, void *value)
+{
+    if (value == NULL || value == (void *)PTR_UINT_MINUS_1) {
+        /* could be popping off the last frame (leaving -1) of the SEH stack */
+        STATS_INC(num_endlist_SEH_write);
+        ASSERT_CURIOSITY(value != NULL);
+        return;
+    }
+    LOG(THREAD, LOG_INTERP, 3, "App moving "PFX" to fs:[0]\n", value);
+# ifdef RETURN_AFTER_CALL
+    if (DYNAMO_OPTION(borland_SEH_rct)) {
+        /* xref case 5752, the Borland compiler SEH implementation uses a push
+         * imm ret motif for fall through to the finally of a try finally block
+         * (very similar to what the Microsoft NT at_SEH_rct_exception() is
+         * doing). The layout will always look like this :
+         *     push e:  (imm32)  (e should be in the .E/.F table)
+         *  a:
+         *     ...
+         *  b: ret
+         *  c: jmp rel32         (c should be in the .E/.F table)
+         *  d: jmp a:  (rel8/32)
+         *     ... (usually nothing)
+         *  e:
+         * (where ret at b is targeting e, or a valid after call).  The 
+         * exception dispatcher calls c (the SEH frame has c as the handler)
+         * which jmps to the exception handler which, in turn, calls d to
+         * execute the finally block.  Fall through is as shown above. So,
+         * we see a .E violation for the handlers call to d and a .C violation
+         * for the fall trough case of the ret @ b targeting e.  We may also
+         * see a .E violation for a call to a as sometimes the handler computes
+         * the target of the jmp @ d an passes that to a different exception
+         * handler.
+         *
+         * For try-except we see the following layout :
+         *           I've only seen jmp ind in the case that led to needing
+         *           at_Borland_SEH_rct_exemption() to be added, not that
+         *           it makes any difference.
+         *    [ jmp z:  (rel8/32) || (rarely) ret || (very rarely) jmp ind]
+         * x: jmp rel32          (x should be in the .E/.F table)  
+         * y:
+         *    ...
+         *    call rel32
+         * [z: ... || ret ]
+         * Though there may be other optimized layouts (the ret instead of the
+         * jmp z: is one such) so we may not want to rely on anything other
+         * then x y.  The exception dispatcher calls x (the SEH frame has x as
+         * the handler) which jmps to the exception handler which, in turn,
+         * jmps to y to execute the except block.  We see a .F violation from
+         * the handler's jmp to y.  at_Borland_SEH_rct_exemption() covers a
+         * case where the address of x (and thus y) in an existing SEH frame
+         * is changed in place instead of popping and pushing a new frame.
+         *
+         * All addresses (rel and otherwise) should be in the same module.  So
+         * we need to recognize the patter and add d:/y: to the .E/.F table
+         * as well as a: (sometimes the handler calculates the target of d and
+         * passes that up to a higher level routine, though I don't see the
+         * point) and add e: to the .C table.
+         *
+         * It would be preferable to handle these exemptions reactively at
+         * the violation point, but unfortunately, by the time we get to the
+         * violation the SEH frame information has been popped off the stack
+         * and is lost, so we have to do it pre-emptively here (pattern
+         * matching at violation time has proven to difficult in the face of
+         * certain compiler optimizations). See at_Borland_SEH_rct_exemption()
+         * in callback.c, that could handle all ind branches to y and ind calls
+         * to d (see below) at an acceptable level of security if we desired.
+         * Handling the ret @ b to e reactively would require the ability to
+         * recreate the exact src cti (so we can use the addr of the ret to
+         * pattern match) at the violation point (something that can't always
+         * currently be done, reset flushing etc.).  Handling the ind call to
+         * a (which I've never acutally seen, though I've seen the address
+         * computed and it looks like it could likely be hit) reactively is
+         * more tricky. Prob. the only way to handle that is to allow .E/.F
+         * transistions to any address after a push imm32 of an address in the
+         * same module, but that might be too permissive.  FIXME - should still
+         * revisit doing the exemptions reactively at some point, esp. once we
+         * can reliably get the src cti.
+         */
+
+        extern bool seen_Borland_SEH; /* set for callback.c */
+        /* First read in the SEH frame, this is the observed structure and
+         * the first two fields (which are all that we use) are constrained by
+         * ntdll exception dispatcher (see EXCEPTION_REGISTRATION decleration
+         * in ntdll.h). */
+        /* FIXME - could just use EXCEPTION_REGISTRATION period since all we
+         * need is the handler address and it would allow simpler curiosity
+         * [see 8181] below.  If, as is expected, other options make use of
+         * this routine we'll probably have one shared get of the SEH frame
+         * anyways. */
+        typedef struct _borland_seh_frame_t {
+            EXCEPTION_REGISTRATION reg;
+            reg_t xbp; /* not used by us */
+        } borland_seh_frame_t;
+        borland_seh_frame_t frame;
+        /* will hold [b,e] or [x-1,y] */
+        byte target_buf[RET_0_LENGTH + 2 * JMP_LONG_LENGTH];
+        app_pc handler_jmp_target = NULL;
+
+        if (!safe_read(value, sizeof(frame), &frame)) {
+            /* We already checked for NULL and -1 above so this should be
+             * a valid SEH frame. Xref 8181, borland_seh_frame_t struct is
+             * bigger then EXCEPTION_REGISTRATION (which is all that is 
+             * required) so verify smaller size is readable. */
+            ASSERT_CURIOSITY(sizeof(EXCEPTION_REGISTRATION) < sizeof(frame) &&
+                             safe_read(value, sizeof(EXCEPTION_REGISTRATION),
+                                       &frame));
+            goto post_borland;
+        }
+        /* frame.reg.handler is c or y, read extra prior bytes to look for b */
+        if (!safe_read((app_pc)frame.reg.handler - RET_0_LENGTH,
+                       sizeof(target_buf), target_buf)) {
+            goto post_borland;
+        }
+        if (is_jmp_rel32(&target_buf[RET_0_LENGTH], (app_pc)frame.reg.handler,
+                         &handler_jmp_target)) {
+            /* we have a possible match, now do the more expensive checking */
+            app_pc base;
+            LOG(THREAD, LOG_INTERP, 3,
+                "Read possible borland SEH frame @"PFX"\n\t"
+                "next="PFX" handler="PFX" xbp="PFX"\n\t",
+                value, frame.reg.prev, frame.reg.handler, frame.xbp);
+            DOLOG(3, LOG_INTERP, {
+                dump_buffer_as_bytes(THREAD, target_buf, sizeof(target_buf), 0);
+            });
+            /* optimize check if we've already processed this frame once */
+            if ((DYNAMO_OPTION(rct_ind_jump) != OPTION_DISABLED ||
+                 DYNAMO_OPTION(rct_ind_call) != OPTION_DISABLED) &&
+                rct_ind_branch_target_lookup(dcontext,
+                                             (app_pc)frame.reg.handler +
+                                             JMP_LONG_LENGTH)) {
+                /* we already processed this SEH frame once, this is prob. a
+                 * frame pop, no need to continue */
+                STATS_INC(num_borland_SEH_dup_frame);
+                LOG(THREAD, LOG_INTERP, 3,
+                    "Processing duplicate Borland SEH frame\n");
+                goto post_borland;
+            }
+            base = get_module_base((app_pc)frame.reg.handler);
+            STATS_INC(num_borland_SEH_initial_match);
+            /* Perf opt, we use the cheaper get_allocation_base() below instead
+             * of get_module_base().  We are checking the result against a
+             * known module base (base) so no need to duplicate the is module
+             * check.  FIXME - the checks prob. aren't even neccesary given the
+             * later is_in_code_section checks. Xref case 8171. */
+            /* FIXME - (perf) we could cache the region from the first
+             * is_in_code_section() call and check against that before falling
+             * back on is_in_code_section in case of multiple code sections. */
+            if (base != NULL &&
+                get_allocation_base(handler_jmp_target) == base &&
+                get_allocation_base(bb->instr_start) == base &&
+                /* FIXME - with -rct_analyze_at_load we should be able to
+                 * verify that frame->handler (x: c:) is on the .E/.F
+                 * table already. We could also try to match known pre x:
+                 * post y: patterns. */
+                is_in_code_section(base, bb->instr_start, NULL, NULL) &&
+                is_in_code_section(base, handler_jmp_target, NULL, NULL) &&
+                is_range_in_code_section(base, (app_pc)frame.reg.handler,
+                                         (app_pc)frame.reg.handler+JMP_LONG_LENGTH+1,
+                                         NULL, NULL)) {
+                app_pc finally_target;
+                byte push_imm_buf[PUSH_IMM32_LENGTH];
+                DEBUG_DECLARE(bool ok;)
+                /* we have a match, add handler+JMP_LONG_LENGTH (y: d:)
+                 * to .E/.F table */
+                STATS_INC(num_borland_SEH_try_match);
+                LOG(THREAD, LOG_INTERP, 2,
+                    "Found Borland SEH frame adding "PFX" to .E/.F table\n",
+                    (app_pc)frame.reg.handler+JMP_LONG_LENGTH);
+                if ((DYNAMO_OPTION(rct_ind_jump) != OPTION_DISABLED ||
+                     DYNAMO_OPTION(rct_ind_call) != OPTION_DISABLED)) {
+                    mutex_lock(&rct_module_lock);
+                    rct_add_valid_ind_branch_target(dcontext,
+                                                    (app_pc)frame.reg.handler +
+                                                    JMP_LONG_LENGTH);
+                    mutex_unlock(&rct_module_lock);
+                }
+                /* we set this as an enabler for another exemption in 
+                 * callback .C, see notes there */
+                if (!seen_Borland_SEH) {
+                    SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+                    seen_Borland_SEH = true;
+                    SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+                }
+                /* case 8648: used to decide which RCT entries to persist */
+                DEBUG_DECLARE(ok =) os_module_set_flag(base, MODULE_HAS_BORLAND_SEH);
+                ASSERT(ok);
+                /* look for .C addresses for try finally */
+                if (target_buf[0] == RAW_OPCODE_ret &&
+                    (is_jmp_rel32(&target_buf[RET_0_LENGTH+JMP_LONG_LENGTH],
+                                  (app_pc)frame.reg.handler+JMP_LONG_LENGTH,
+                                  &finally_target) ||
+                     is_jmp_rel8(&target_buf[RET_0_LENGTH+JMP_LONG_LENGTH],
+                                 (app_pc)frame.reg.handler+JMP_LONG_LENGTH,
+                                 &finally_target)) &&
+                    safe_read(finally_target - sizeof(push_imm_buf), 
+                              sizeof(push_imm_buf), push_imm_buf) &&
+                    push_imm_buf[0] == RAW_OPCODE_push_imm32) {
+                    app_pc push_val = *(app_pc *)&push_imm_buf[1];
+                    /* do a few more, expensive, sanity checks */
+                    /* FIXME - (perf) see earlier note on get_allocation_base()
+                     * and is_in_code_section() usage. */
+                    if (get_allocation_base(finally_target) == base &&
+                        is_in_code_section(base, finally_target, NULL, NULL) &&
+                        get_allocation_base(push_val) == base &&
+                        /* FIXME - could also check that push_val is in
+                         * .E/.F table, at least for -rct_analyze_at_load */
+                        is_in_code_section(base, push_val, NULL, NULL)) {
+                        /* Full match, add push_val (e:) to the .C table
+                         * and finally_target (a:) to the .E/.F table */
+                        STATS_INC(num_borland_SEH_finally_match);
+                        LOG(THREAD, LOG_INTERP, 2,
+                            "Found Borland SEH finally frame adding "PFX" to"
+                            " .C table and "PFX" to .E/.F table\n",
+                            push_val, finally_target);
+                        if ((DYNAMO_OPTION(rct_ind_jump) != OPTION_DISABLED ||
+                             DYNAMO_OPTION(rct_ind_call) != OPTION_DISABLED)) {
+                            mutex_lock(&rct_module_lock);
+                            rct_add_valid_ind_branch_target(dcontext,
+                                                            finally_target);
+                            mutex_unlock(&rct_module_lock);
+                        }
+                        if (DYNAMO_OPTION(ret_after_call)) {
+                            fragment_add_after_call(dcontext, push_val);
+                        }
+                    } else {
+                        ASSERT_CURIOSITY(false &&
+                                         "partial borland seh finally match");
+                    }
+                }
+            }
+        }
+    }
+ post_borland:
+# endif /* RETURN_AFTER_CALL */
+    return;
+}
+
+/* helper routine for bb_process_fs_ref
+ * return true if bb should be continued, false if it shouldn't  */
+static bool
+bb_process_fs_ref_opnd(dcontext_t *dcontext, build_bb_t *bb, opnd_t dst,
+                       bool *is_to_fs0)
+{
+    ASSERT(is_to_fs0 != NULL);
+    *is_to_fs0 = false;
+    if (opnd_is_far_base_disp(dst) && /* FIXME - check size? */
+        opnd_get_segment(dst) == SEG_FS) {
+        /* is a write to fs:[*] */
+        if (bb->instr_start != bb->start_pc) {
+            /* Not first instruction in the bb, end bb before this
+             * instruction, so we can see it as the first instruction of a
+             * new bb where we can use the register state. */
+            /* As is, always ending the bb here has a mixed effect on mem usage
+             * with default options.  We do end up with slightly more bb's
+             * (and associated bookeeping costs), but frequently with MS dlls
+             * we reduce code cache dupliaction from jmp/call ellision 
+             * (_SEH_[Pro,Epi]log otherwise ends up frequently duplicated for
+             * instance). */
+            /* FIXME - we must stop the bb here even if there's already
+             * a bb built for the next instruction, as we have to have
+             * reproducible bb building for recreate app state.  We should
+             * only get here through code duplication (typically jmp/call
+             * inlining, though can also be through multiple entry points into
+             * the same block of non cti instructions). */ 
+            /* FIXME - this is getting common enough to be a shared routine */
+            instr_destroy(dcontext, bb->instr);
+            bb->instr = NULL;
+            bb->cur_pc = bb->instr_start;
+            return false; /* stop bb */
+        }
+        /* Only process the push if building a new bb for cache, can't check
+         * this any earlier since have to preserve bb building/ending behavior
+         * even when not for cache (for recreation etc.). */
+        if (bb->app_interp) {
+            /* check is write to fs:[0] */
+            if (opnd_compute_address(dst, get_mcontext(dcontext)) == NULL) {
+                /* we have new mov to fs:[0] */
+                *is_to_fs0 = true;
+            }
+        }
+    }
+    return true;
+}
+
+/* While currently only used for Borland SEH exemptions, this analysis could
+ * also be helpful for other SEH tasks (xref case 5824). */
+static bool
+bb_process_fs_ref(dcontext_t *dcontext, build_bb_t *bb)
+{
+    ASSERT(DYNAMO_OPTION(process_SEH_push) &&
+           instr_get_prefix_flag(bb->instr, PREFIX_SEG_FS));
+
+    /* If this is the first instruction of a bb for the cache we
+     * want to fully decode it, check if it's pushing an SEH frame
+     * and, if so, pass it to the SEH checking routines (currently
+     * just used for the Borland SEH rct handling).  If this is not
+     * the first instruction of the bb then we want to stop the bb
+     * just before this instruction so that when we do process this
+     * instruction it will be the first in the bb (allowing us to
+     * use the register state). */
+    if (!bb->full_decode) {
+        instr_decode(dcontext, bb->instr);
+        /* is possible this is an invalid instr that made it through the fast
+         * decode, FIXME is there a better way to handle this? */
+        if (!instr_valid(bb->instr)) {
+            ASSERT_NOT_TESTED();
+            if (bb->cur_pc == NULL)
+                bb->cur_pc = bb->instr_start;
+            bb_process_invalid_instr(dcontext, bb);
+            return false; /* stop bb */
+        }
+        ASSERT(instr_get_prefix_flag(bb->instr, PREFIX_SEG_FS));
+    }
+    /* expect to see only simple mov's to fs:[0] for new SEH frames
+     * FIXME - might we see other types we'd want to intercept? 
+     * do we want to proccess pop instructions (usually just for removing
+     * a frame)? */
+    if (instr_get_opcode(bb->instr) == OP_mov_st) {
+        bool is_to_fs0;
+        opnd_t dst = instr_get_dst(bb->instr, 0);
+        if (!bb_process_fs_ref_opnd(dcontext, bb, dst, &is_to_fs0))
+            return false; /* end bb */
+        /* Only process the push if building a new bb for cache, can't check
+         * this any earlier since have to preserve bb building/ending behavior
+         * even when not for cache (for recreation etc.). */
+        if (bb->app_interp) {
+            if (is_to_fs0) {
+                ptr_int_t value = 0;
+                opnd_t src = instr_get_src(bb->instr, 0);
+                if (opnd_is_immed_int(src)) {
+                    value = opnd_get_immed_int(src);
+                } else if (opnd_is_reg(src)) {
+                    value = reg_get_value(opnd_get_reg(src), get_mcontext(dcontext));
+                } else {
+                    ASSERT_NOT_REACHED();
+                }
+                STATS_INC(num_SEH_pushes_processed);
+                LOG(THREAD, LOG_INTERP, 3, "found mov to fs:[0] @ "PFX"\n",
+                    bb->instr_start);
+                bb_process_SEH_push(dcontext, bb, (void *)value);
+            } else {
+                STATS_INC(num_fs_movs_not_SEH);
+            }
+        }
+    }
+# if defined(DEBUG) && defined(INTERNAL)
+    else if (INTERNAL_OPTION(check_for_SEH_push)) {
+        /* Debug build Sanity check that we aren't missing SEH frame pushes */
+        int i;
+        int num_dsts = instr_num_dsts(bb->instr);
+        for (i = 0; i < num_dsts; i++) {
+            bool is_to_fs0;
+            opnd_t dst = instr_get_dst(bb->instr, i);
+            if (!bb_process_fs_ref_opnd(dcontext, bb, dst, &is_to_fs0)) {
+                STATS_INC(num_process_SEH_bb_early_terminate_debug);
+                return false; /* end bb */
+            }
+            /* common case is pop instructions to fs:[0] when popping an
+             * SEH frame stored on tos */
+            if (is_to_fs0) {
+                if (instr_get_opcode(bb->instr) == OP_pop) {
+                    LOG(THREAD, LOG_INTERP, 4,
+                        "found pop to fs:[0] @ "PFX"\n", bb->instr_start);
+                    STATS_INC(num_process_SEH_pop_fs0);
+                } else {
+                    /* an unexpected SEH frame push */
+                    LOG(THREAD, LOG_INTERP, 1,
+                        "found unexpected write to fs:[0] @"PFX"\n",
+                        bb->instr_start);
+                    DOLOG(1, LOG_INTERP, {
+                        loginst(dcontext, 1, bb->instr, "");
+                    }); 
+                    ASSERT_CURIOSITY(!is_to_fs0);
+                }
+            }
+        }
+    }
+# endif
+    return true; /* continue bb */
+}
+#endif /* win32 */
+
+/* Returns true to indicate that ignorable syscall processing is completed
+ * with *continue_bb indicating if the bb should be continued or not.
+ * When returning false, continue_bb isn't pertinent.
+ */
+static bool
+bb_process_ignorable_syscall(dcontext_t *dcontext, build_bb_t *bb,
+                             int sysnum, bool *continue_bb)
+{
+    STATS_INC(ignorable_syscalls);
+    BBPRINT(bb, 3, "found ignorable system call 0x%04x\n", sysnum);
+#ifdef WINDOWS
+    if (get_syscall_method() != SYSCALL_METHOD_SYSENTER) {
+        DODEBUG({
+            if (get_syscall_method() == SYSCALL_METHOD_WOW64)
+                ASSERT_NOT_TESTED();
+        });
+        if (continue_bb != NULL)
+            *continue_bb = true;
+        return true;
+    }
+    else {
+        /* Can we continue interp after the sysenter at the instruction
+         * after the call to sysenter? */
+        instr_t *call = bb_verify_sysenter_pattern(dcontext, bb);
+
+        if (call != NULL) {
+            /* If we're continuing code discovery at the after-call address,
+             * change the cur_pc to continue at the after-call addr. This is
+             * safe since the preceding call is in the fragment and
+             * %xsp/(%xsp) hasn't changed since the call. Obviously, we assume
+             * that the sysenter breaks control flow in fashion such any
+             * instruction that follows it isn't reached by DR.
+             */
+            if (DYNAMO_OPTION(ignore_syscalls_follow_sysenter)) {
+                bb->cur_pc =
+                    instr_get_raw_bits(call) + instr_length(dcontext, call);
+                if (continue_bb != NULL)
+                    *continue_bb = true;
+                return true;
+            }
+            else {
+                /* End this bb now. We set the exit target so that control
+                 * skips the vsyscall 'ret' that's executed natively after the
+                 * syscall and ends up at the correct place.
+                 */
+                /* FIXME Assigning exit_target causes the fragment to end 
+                 * with a direct exit stub to the after-call address, which
+                 * is fine. If bb->exit_target < bb->start_pc, the future
+                 * fragment for exit_target is marked as a trace head which
+                 * isn't intended. A potentially undesirable side effect
+                 * is that exit_target's fragment can't be included in
+                 * trace for start_pc.
+                 */
+                bb->exit_target =
+                    instr_get_raw_bits(call) + instr_length(dcontext, call);
+                if (continue_bb != NULL)
+                    *continue_bb = false;
+                return true;
+            }
+        }
+        STATS_INC(ignorable_syscalls_failed_sysenter_pattern);
+        /* Pattern match failed but the syscall is ignorable so maybe we
+         * can try shared syscall? */
+        /* Decrement the stat to prevent double counting. We rarely expect to hit
+         * this case. */
+        STATS_DEC(ignorable_syscalls);
+        return false;
+    }
+#else
+    if (continue_bb != NULL)
+        *continue_bb = true;
+    return true;
+#endif
+}
+
+#ifdef WINDOWS
+/* Process a syscall that is executed via shared syscall. */
+static void
+bb_process_shared_syscall(dcontext_t *dcontext, build_bb_t *bb, int sysnum)
+{
+    ASSERT(DYNAMO_OPTION(shared_syscalls));
+    DODEBUG({
+        if (ignorable_system_call(sysnum))
+            STATS_INC(ignorable_syscalls);
+        else
+            STATS_INC(optimizable_syscalls);
+    });
+    BBPRINT(bb, 3, "found %soptimizable system call 0x%04x\n",
+            INTERNAL_OPTION(shared_eq_ignore) ? "ignorable-" : "",
+            sysnum);
+
+    LOG(THREAD, LOG_INTERP, 3,
+        "ending bb at syscall & NOT removing the interrupt itself\n");
+
+    /* Mark the instruction as pointing to shared syscall */
+    bb->instr->flags |= INSTR_SHARED_SYSCALL;
+    /* this block must be the last one in a trace */
+    bb->flags |= FRAG_MUST_END_TRACE;
+    /* we redirect all optimizable syscalls to a single shared piece of code.
+     * Once a fragment reaches the shared syscall code, it can be safely
+     * deleted, for example, if the thread is interupted for a callback and
+     * DR needs to delete fragments for cache management.
+     *
+     * Note that w/shared syscall, syscalls can be executed from TWO
+     * places -- shared_syscall and do_syscall.
+     */
+    bb->exit_target = shared_syscall_routine(dcontext);
+    /* make sure translation for ending jmp ends up right, mangle will
+     * remove this instruction, so set to NULL so translation does the
+     * right thing */
+    bb->instr = NULL;
+}
+#endif
+
+static bool
+bb_process_non_ignorable_syscall(dcontext_t *dcontext, build_bb_t *bb,
+                                 int sysnum)
+{
+    BBPRINT(bb, 3, "found non-ignorable system call 0x%04x\n", sysnum);
+    STATS_INC(non_ignorable_syscalls);
+    bb->exit_type |= LINK_NI_SYSCALL;
+    /* destroy the interrupt instruction */
+    LOG(THREAD, LOG_INTERP, 3,
+        "ending bb at syscall & removing the interrupt itself\n");
+    /* Indicate that this is a non-ignorable syscall so mangle will remove */
+#ifdef LINUX
+    if (instr_get_opcode(bb->instr) == OP_int) {
+        bb->exit_type |= LINK_NI_SYSCALL_INT;
+        bb->instr->flags |= INSTR_NI_SYSCALL_INT;
+    } else
+#endif
+        bb->instr->flags |= INSTR_NI_SYSCALL;
+    /* Set instr to NULL in order to get translation of exit cti correct. */
+    bb->instr = NULL;
+    /* this block must be the last one in a trace */
+    bb->flags |= FRAG_MUST_END_TRACE;
+    return false; /* end bb now */
+}
+
+/* returns true to indicate "continue bb" and false to indicate "end bb now" */
+static inline bool
+bb_process_syscall(dcontext_t *dcontext, build_bb_t *bb)
+{
+    int sysnum;
+#ifdef CLIENT_INTERFACE
+    /* PR 307284: for simplicity do syscall/int processing post-client.
+     * We give up on inlining but we can still use ignorable/shared syscalls
+     * and trace continuation.
+     */
+    if (bb->pass_to_client && !bb->post_client)
+        return false;
+#endif
+#ifdef DGC_DIAGNOSTICS
+    if (TEST(FRAG_DYNGEN, bb->flags) && !is_dyngen_vsyscall(bb->instr_start)) {
+        LOG(THREAD, LOG_INTERP, 1, "WARNING: syscall @ "PFX" in dyngen code!\n",
+            bb->instr_start);
+    }
+#endif
+    BBPRINT(bb, 4, "interp: syscall @ "PFX"\n", bb->instr_start);
+    check_syscall_method(dcontext, bb->instr);
+    bb->flags |= FRAG_HAS_SYSCALL;
+    /* if we can identify syscall number and it is an ignorable syscall,
+     * we let bb keep going, else we end bb and flag it
+     */
+    sysnum = find_syscall_num(dcontext, bb->ilist, bb->instr);
+    BBPRINT(bb, 3, "syscall # is %d\n", sysnum);
+#ifdef CLIENT_INTERFACE 
+    if (instrument_filter_syscall(dcontext, sysnum)) {
+        BBPRINT(bb, 3, "client asking to intercept => pretending syscall # %d is -1\n",
+                sysnum);
+        sysnum = -1;
+    }
+#endif
+    if (sysnum > -1 &&
+        DYNAMO_OPTION(ignore_syscalls) && 
+        ignorable_system_call(sysnum)
+        /* PR 288101: On Linux we do not yet support inlined sysenter instrs as we
+         * do not have in-cache support for the post-sysenter continuation: we rely
+         * for now on very simple sysenter handling where dispatch uses asynch_target
+         * to know where to go next.
+         */
+        IF_LINUX(&& instr_get_opcode(bb->instr) != OP_sysenter)) {
+
+        bool continue_bb;
+
+        if (bb_process_ignorable_syscall(dcontext, bb, sysnum, &continue_bb)) {
+            if (!DYNAMO_OPTION(inline_ignored_syscalls))
+                continue_bb = false;
+            return continue_bb;
+        }
+    }
+#ifdef WINDOWS
+    if (sysnum > -1 && DYNAMO_OPTION(shared_syscalls) &&
+        optimizable_system_call(sysnum)) {
+        bb_process_shared_syscall(dcontext, bb, sysnum);
+        return false;
+    }
+#endif
+    
+    /* Fall thru and handle as a non-ignorable syscall. */
+    return bb_process_non_ignorable_syscall(dcontext, bb, sysnum);
+}
+
+/* Case 3922: for wow64 we treat "call *fs:0xc0" as a system call.
+ * Only sets continue_bb if it returns true.
+ */
+static bool
+bb_process_indcall_syscall(dcontext_t *dcontext, build_bb_t *bb,
+                           bool *continue_bb)
+{
+    ASSERT(continue_bb != NULL);
+#ifdef WINDOWS
+    if (instr_is_wow64_syscall(bb->instr)) {
+        /* we could check the preceding instrs but we don't bother */
+        *continue_bb = bb_process_syscall(dcontext, bb);
+        return true;
+    }
+#endif
+    return false;
+}
+
+/* returns true to indicate "continue bb" and false to indicate "end bb now" */
+static inline bool
+bb_process_interrupt(dcontext_t *dcontext, build_bb_t *bb)
+{
+#if defined(DEBUG) || defined(INTERNAL) || defined(WINDOWS)
+    int num = instr_get_interrupt_number(bb->instr);
+#endif
+#ifdef CLIENT_INTERFACE
+    /* PR 307284: for simplicity do syscall/int processing post-client.
+     * We give up on inlining but we can still use ignorable/shared syscalls
+     * and trace continuation.
+     */
+    if (bb->pass_to_client && !bb->post_client)
+        return false;
+#endif
+    BBPRINT(bb, 3, "int 0x%x @ "PFX"\n", num, bb->instr_start);
+#ifdef WINDOWS
+    if (num == 0x2b) {
+        /* interrupt 0x2B signals return from callback */
+        /* end block here and come back to dynamo to perform interrupt */
+        bb->exit_type |= LINK_CALLBACK_RETURN;
+        BBPRINT(bb, 3, "ending bb at cb ret & removing the interrupt itself\n");
+        /* Set instr to NULL in order to get translation of exit cti
+         * correct.  mangle will destroy the instruction */
+        bb->instr = NULL;
+        bb->flags |= FRAG_MUST_END_TRACE;
+        STATS_INC(num_int2b);
+        return false;
+    } else {
+        SYSLOG_INTERNAL_INFO_ONCE("non-syscall, non-int2b 0x%x @ "PFX" from "PFX,
+                                  num, bb->instr_start, bb->start_pc);
+    }
+#endif /* WINDOWS */
+    return true;
+}
+
+/* If the current instr in the BB is an indirect call that can be converted into a
+ * direct call, process it and return true, else, return false.
+ * FIXME PR 288327: put in linux call* to vsyscall page
+ */
+static bool
+bb_process_convertible_indcall(dcontext_t *dcontext, build_bb_t *bb)
+{
+    /* We perform several levels of checking, each increasingly more stringent
+     * and expensive, with a false return should any fail.
+     */
+    instr_t *instr;
+    opnd_t src0;
+    instr_t *call_instr;
+    int call_src_reg;
+    app_pc callee;
+    bool vsyscall = false;
+
+    /* Check if this BB can be extended and the instr is a (near) indirect call */
+    if (instr_get_opcode(bb->instr) != OP_call_ind)
+        return false;
+
+    /* Check if we have a "mov <imm> -> %reg; call %reg" or a
+     * "mov <imm> -> %reg; call (%reg)" pair. First check for the call.
+     */
+    /* The 'if' conditions are broken up to make the code more readable
+     * while #ifdef-ing the WINDOWS case. It's still ugly though.
+     */
+    instr = bb->instr;
+    if (!(
+#ifdef WINDOWS
+          /* Match 'call (%xdx)' for a post-SP2 indirect call to sysenter. */
+          (opnd_is_near_base_disp(instr_get_src(instr, 0)) &&
+           opnd_get_base(instr_get_src(instr, 0)) == REG_XDX &&
+           opnd_get_disp(instr_get_src(instr, 0)) == 0) ||
+#endif
+          /* Match 'call %reg'. */
+          opnd_is_reg(instr_get_src(instr, 0))))
+        return false;
+
+    /* If there's no CTI in the BB, we can check if there are 5+ preceding
+     * bytes and if they could hold a "mov" instruction.
+     */
+    if (!TEST(FRAG_HAS_DIRECT_CTI, bb->flags) &&
+        bb->instr_start - 5 >= bb->start_pc) {
+
+        byte opcode = *((byte *) bb->instr_start - 5);
+
+        /* Check the opcode. Do we see a "mov ... -> %reg"?  Valid opcodes are in
+         * the 0xb8-0xbf range (Intel IA-32 ISA ref, v.2) and specify the
+         * destination register, i.e., 0xb8 means that %xax is the destination.
+         */
+        if (opcode < 0xb8 || opcode > 0xbf)
+            return false;
+    }
+
+    /* Check the previous instruction -- is it really a "mov"? */
+    src0 = instr_get_src(instr, 0);
+    call_instr = instr;
+    instr = instr_get_prev_expanded(dcontext, bb->ilist, bb->instr);
+    call_src_reg = opnd_is_near_base_disp(src0) ? opnd_get_base(src0) :
+        opnd_get_reg(src0);
+    if (instr == NULL || instr_get_opcode(instr) != OP_mov_imm ||
+        opnd_get_reg(instr_get_dst(instr, 0)) != call_src_reg)
+        return false;
+
+    /* For the general case, we don't try to optimize a call
+     * thru memory -- just check that the call uses a register.
+     */
+    callee = NULL;
+    if (opnd_is_reg(src0)) {
+        /* Extract the target address. */
+        callee = (app_pc) opnd_get_immed_int(instr_get_src(instr, 0));
+#ifdef WINDOWS
+# ifdef PROGRAM_SHEPHERDING
+        /* FIXME - is checking for on vsyscall page better or is checking == to 
+         * VSYSCALL_BOOTSTRAP_ADDR? Both are hacky. */
+        if (is_dyngen_vsyscall((app_pc)opnd_get_immed_int(instr_get_src(instr, 0)))) {
+            LOG(THREAD, LOG_INTERP, 4, "Pre-SP2 style indirect call "
+                "to sysenter found at "PFX"\n", bb->instr_start);
+            STATS_INC(num_sysenter_indcalls);
+            vsyscall = true;
+            ASSERT(opnd_get_immed_int(instr_get_src(instr, 0)) ==
+                   (ptr_int_t)VSYSCALL_BOOTSTRAP_ADDR);
+            ASSERT(!use_ki_syscall_routines()); /* double check our determination */
+        }
+        else
+# endif
+#endif
+            STATS_INC(num_convertible_indcalls);
+    }
+#ifdef WINDOWS
+    /* Match the "call (%xdx)" to sysenter case for SP2-patched os's. Memory at
+     * address VSYSCALL_BOOTSTRAP_ADDR (0x7ffe0300) holds the address of
+     * KiFastSystemCall or (FIXME - not handled) on older platforms KiIntSystemCall.
+     * FIXME It's unsavory to hard-code 0x7ffe0300, but the constant has little
+     * context in an SP2 os. It's a hold-over from pre-SP2.
+     */
+    else if (get_syscall_method() == SYSCALL_METHOD_SYSENTER
+             && call_src_reg == REG_XDX
+             && opnd_get_immed_int(instr_get_src(instr, 0)) == 
+             (ptr_int_t)VSYSCALL_BOOTSTRAP_ADDR) {
+        /* Extract the target address. We expect that the memory read using the
+         * value in the immediate field is ok as it's the vsyscall page
+         * which 1) cannot be made unreadable and 2) cannot be made writable so
+         * the stored value will not change. Of course, it's possible that the
+         * os could change the page contents.
+         */
+        callee = (app_pc)
+            *((ptr_uint_t *) opnd_get_immed_int(instr_get_src(instr, 0)));
+        if (get_app_sysenter_addr() == NULL) {
+            /* For the first call* we've yet to decode an app syscall, yet we
+             * cannot have later recreations have differing behavior, so we must
+             * handle that case (even though it doesn't matter performance-wise
+             * as the first call* is usually in runtime init code that's
+             * executed once).  So we do a raw byte compare to:
+             *   ntdll!KiFastSystemCall:
+             *   7c82ed50 8bd4             mov     xdx,xsp
+             *   7c82ed52 0f34             sysenter
+             */
+            uint raw;
+            if (!safe_read(callee, sizeof(raw), &raw) || raw != 0x340fd48b)
+                callee = NULL;
+        } else {
+            /* The callee should be a 2 byte "mov %xsp -> %xdx" followed by the
+             * sysenter -- check the sysenter's address as 2 bytes past the callee.
+             */
+            if (callee + 2 != get_app_sysenter_addr())
+                callee = NULL;
+        }
+        vsyscall = (callee != NULL);
+        ASSERT(use_ki_syscall_routines()); /* double check our determination */
+        DODEBUG({
+            if (callee == NULL)
+                ASSERT_CURIOSITY(false && "call* to vsyscall unexpected mismatch");
+            else {
+                LOG(THREAD, LOG_INTERP, 4, "Post-SP2 style indirect call "
+                    "to sysenter found at "PFX"\n", bb->instr_start);
+                STATS_INC(num_sysenter_indcalls);
+            }
+        });
+    }
+#endif
+
+    /* Check if register dataflow matched and we were able to extract
+     * the callee address.
+     */
+    if (callee == NULL)
+        return false;
+
+    if (vsyscall) {
+        /* Case 8917: abandon coarse-grainness in favor of performance */
+        bb->flags &= ~FRAG_COARSE_GRAIN;
+        STATS_INC(coarse_prevent_indcall);
+    }
+
+    LOG(THREAD, LOG_INTERP, 4, "interp: possible convertible"
+        " indirect call from "PFX" to "PFX"\n",
+        bb->instr_start, callee);
+
+    if (must_not_be_inlined(callee)) {
+        BBPRINT(bb, 3, "   NOT inlining indirect call to "PFX"\n", callee);
+        /* Case 8711: coarse-grain can't handle non-exit cti */
+        bb->flags &= ~FRAG_COARSE_GRAIN;
+        STATS_INC(coarse_prevent_cti);
+        ASSERT_CURIOSITY_ONCE(!vsyscall && "leaving call* to vsyscall");
+        return true; /* keep bb going, w/o inlining call */
+    }
+
+    if (bb->follow_direct &&
+        bb->num_elide_call < DYNAMO_OPTION(max_elide_call) &&
+        (DYNAMO_OPTION(elide_back_calls) || bb->cur_pc <= callee)) {
+        check_new_page_contig(dcontext, bb, bb->cur_pc-1);
+        /* FIXME This is identical to the code for evaluating a
+         * direct call's callee. If such code appears in another
+         * (3rd) place, we should outline it.
+         * FIXME: use follow_direct_call()
+         */
+        if (vsyscall) {
+            /* As a flag to allow our xfer from now-non-coarse to coarse
+             * (for vsyscall-in-ntdll) we pre-emptively mark as has-syscall.
+             */
+            ASSERT(!TEST(FRAG_HAS_SYSCALL, bb->flags));
+            bb->flags |= FRAG_HAS_SYSCALL;
+        }
+        if (check_new_page_jmp(dcontext, bb, callee)) {
+            if (vsyscall) /* Restore */
+                bb->flags &= ~FRAG_HAS_SYSCALL;
+            bb->num_elide_call++;
+            STATS_INC(total_elided_calls);
+            STATS_TRACK_MAX(max_elided_calls, bb->num_elide_call);
+            bb->cur_pc = callee;
+            /* FIXME: when using follow_direct_call don't forget to set this */
+            call_instr->flags |= INSTR_IND_CALL_DIRECT;
+            BBPRINT(bb, 4, "   continuing in callee at "PFX"\n", bb->cur_pc);
+            return true; /* keep bb going */
+        }
+        if (vsyscall) {
+            /* Case 8917: Restore, just in case, though we certainly expect to have
+             * this flag set as soon as we decode a few more instrs and hit the
+             * syscall itself -- but for pre-sp2 we currently could be elsewhere on
+             * the same page, so let's be safe here.
+             */
+            bb->flags &= ~FRAG_HAS_SYSCALL;
+        }
+    }
+    /* FIXME: we're also not converting to a direct call - was this intended? */
+    BBPRINT(bb, 3, "   NOT following indirect call from "PFX" to "PFX"\n",
+            bb->instr_start, callee);
+    DODEBUG({
+        if (vsyscall) {
+            DO_ONCE({
+                /* Case 9095: don't complain so loudly if user asked for no elision */
+                if (DYNAMO_OPTION(max_elide_call) <= 2)
+                    SYSLOG_INTERNAL_WARNING("leaving call* to vsyscall");
+                else
+                    ASSERT_CURIOSITY(false && "leaving call* to vsyscall");
+            });
+        }
+    });;
+    return false; /* stop bb */
+}
+
+/* if we make the IAT sections unreadable we will need to map to proper location */
+static inline app_pc
+read_from_IAT(app_pc iat_reference)
+{
+    /* FIXME: we should have looked up where the real IAT should be at
+     * the time of checking whether is_in_IAT
+     */
+    return *(app_pc*) iat_reference;
+}
+
+/* returns whether target is an IAT of a module that we convert.  Note
+ * users still have to check the referred to value to verify targeting
+ * a native module.
+ */
+static bool
+is_targeting_convertible_IAT(dcontext_t *dcontext, instr_t *instr,
+                             app_pc *iat_reference /* OUT */)
+{
+    /* FIXME: we could give up on optimizing a particular module,
+     * if too many writes to its IAT are found,
+     * even 1 may be too much to handle!
+     */
+    
+    /* We only allow constant address, 
+     * any registers used for effective address calculation 
+     * can not be guaranteed to be constant dynamically.
+     */
+    /* FIXME: yet a 'call %reg' if that value is an export would be a
+     * good sign that we should go backwards and look for a possible
+     * mov IAT[func] -> %reg and then optimize that as well - case 1948
+     */
+
+    app_pc memory_reference = NULL;
+    opnd_t opnd = instr_get_target(instr);
+
+    LOG(THREAD, LOG_INTERP, 4, "is_targeting_convertible_IAT: ");
+
+    /* A typical example of a proper call
+     * ff 15 8810807c     call    dword ptr [kernel32+0x1088 (7c801088)]
+     * where
+     * [7c801088] =  7c90f04c ntdll!RtlAnsiStringToUnicodeString
+     *
+     * The ModR/M byte for a displacement only with no SIB should be
+     * 15 for CALL, 25 for JMP, (no far versions for IAT)
+     */
+    if (opnd_is_near_base_disp(opnd)) {
+        /* FIXME PR 253930: pattern-match x64 IAT calls */
+        IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+        memory_reference = (app_pc)(ptr_uint_t)opnd_get_disp(opnd);
+
+        /* now should check all other fields */
+        if (opnd_get_base(opnd) != REG_NULL || 
+            opnd_get_index(opnd) != REG_NULL) {
+            /* this is not a pure memory reference, can't be IAT */
+            return false;
+        }
+        ASSERT(opnd_get_scale(opnd) == 0);
+    } else {
+        return false;
+    }
+
+    LOG(THREAD, LOG_INTERP, 3, "is_targeting_convertible_IAT: memory_reference "PFX"\n",
+        memory_reference);
+
+    /* FIXME: if we'd need some more additional structures those can
+     * be looked up in a separate hashtable based on the IAT base, or
+     * we'd have to extend the vmareas with custom fields
+     */
+    ASSERT(DYNAMO_OPTION(IAT_convert));
+    if (vmvector_overlap(IAT_areas, memory_reference, memory_reference+1)) {
+        /* IAT has to be in the same module as current instruction,
+         * but even in the unlikely reference by address from another
+         * module there is really no problem, so not worth checking
+         */
+        ASSERT_CURIOSITY(get_module_base(instr->bytes) == get_module_base(memory_reference));
+
+        /* FIXME: now that we know it is in IAT/GOT,
+         * we have to READ the contents and return that 
+         * safely to the caller so they can convert accordingly
+         */
+
+        /* FIXME: we would want to add the IAT section to the vmareas
+         * of a region that has a converted block.  Then on a write to
+         * IAT we can flush efficiently only blocks affected by a
+         * particular module, for a first hack though flushing
+         * everything on a hooker will do.
+         */
+        *iat_reference = memory_reference;
+        return true;
+    } else {
+        /* plain global function
+         * e.g. ntdll!RtlUnicodeStringToAnsiString+0x4c:
+         * ff15c009917c call dword ptr [ntdll!RtlAllocateStringRoutine (7c9109c0)]
+         */
+        return false;
+    }
+}
+
+/* If the current instr in the BB is an indirect call through IAT that
+ * can be converted into a direct call, process it and return true,
+ * else, return false.
+ */
+static bool
+bb_process_IAT_convertible_indjmp(dcontext_t *dcontext, build_bb_t *bb,
+                                  bool *elide_continue)
+{
+    app_pc iat_reference;
+    app_pc target;
+    ASSERT(DYNAMO_OPTION(IAT_convert));
+
+    /* Check if the instr is a (near) indirect jump */
+    if (instr_get_opcode(bb->instr) != OP_jmp_ind) {
+        ASSERT_CURIOSITY(false && "far ind jump");
+        return false;           /* not matching, stop bb */
+    }
+
+    if (!is_targeting_convertible_IAT(dcontext, bb->instr, 
+                                      &iat_reference)) {
+        DOSTATS({
+            if (TEST(INSTR_IND_JMP_PLT_EXIT, bb->exit_type)) {
+                /* see how often we mark as likely a PLT a JMP which in
+                 * fact is not going through IAT 
+                 */
+                STATS_INC(num_indirect_jumps_PLT_not_IAT);
+                LOG(THREAD, LOG_INTERP, 3,
+                    "bb_process_IAT_convertible_indjmp: indirect jmp not PLT instr="
+                    PFX"\n", bb->instr->bytes);
+            }
+        });
+        
+        return false;           /* not matching, stop bb */
+    }
+
+    target = read_from_IAT(iat_reference);
+
+    DOLOG(4, LOG_INTERP, {
+        char name[MAXIMUM_SYMBOL_LENGTH];
+        print_symbolic_address(target, name, sizeof(name), false);
+        LOG(THREAD, LOG_INTERP, 4, 
+            "bb_process_IAT_convertible_indjmp: target="PFX" %s\n", target, name);
+    });
+
+    STATS_INC(num_indirect_jumps_IAT);
+    DOSTATS({
+        if (!TEST(INSTR_IND_JMP_PLT_EXIT, bb->exit_type)) {
+            /* count any other known uses for an indirect jump to go
+             * through the IAT other than PLT uses, although a block
+             * reaching max_elide_call would prevent the above
+             * match */
+            STATS_INC(num_indirect_jumps_IAT_not_PLT);
+            /* FIXME: case 6459 for further inquiry */
+            LOG(THREAD, LOG_INTERP, 4, 
+                "bb_process_IAT_convertible_indjmp: indirect jmp not PLT target="PFX"\n", 
+                target);
+        }
+    });
+
+    if (must_not_be_elided(target)) {
+        ASSERT_NOT_TESTED();
+        BBPRINT(bb, 3,
+                "   NOT inlining indirect jmp to must_not_be_elided "PFX"\n", target);
+        return false; /* do not convert indirect jump, will stop bb */
+    }
+
+    /* Verify not targeting native exec DLLs, note that the IATs of
+     * any module may have imported a native DLL.  Note it may be
+     * possible to optimize with a range check on IAT subregions, but
+     * this check isn't much slower.
+     */
+
+    /* IAT_elide should definitely not touch native_exec modules.
+     * 
+     * FIXME: we also prevent IAT_convert from optimizing imports in
+     * native_exec_list DLLs, although we could let that convert to a
+     * direct jump and require native_exec_dircalls to be always on to
+     * intercept those jmps.
+     */
+    if (DYNAMO_OPTION(native_exec) &&
+        vmvector_overlap(native_exec_areas, target, target+1)) {
+        BBPRINT(bb, 3,
+                "   NOT inlining indirect jump to native exec module "PFX"\n", target);
+        STATS_INC(num_indirect_jumps_IAT_native);
+        return false;           /* do not convert indirect jump, stop bb */
+    }
+
+    /* mangle mostly as such as direct jumps would be mangled in
+     * bb_process_ubr(dcontext, bb) but note bb->instr has already
+     * been appended so has to reverse some of its actions
+     */
+    
+    /* pretend never saw an indirect JMP, we'll either add a new
+       direct JMP or we'll just continue in target */
+    instrlist_remove(bb->ilist, bb->instr);    /* bb->instr has been appended already */
+    instr_destroy(dcontext, bb->instr);
+    bb->instr = NULL;
+
+    if (DYNAMO_OPTION(IAT_elide)) {
+        /* try to elide just as a direct jmp would have been elided */
+
+        /* We could have used follow_direct_call instead since
+         * commonly this really is a disguised CALL*. Yet for PLT use
+         * of the form of CALL PLT[foo]; JMP* IAT[foo] we would have
+         * already counted the CALL.  If we have tail call elimination
+         * that converts a CALL* into a JMP* it is also OK to treat as
+         * a JMP instead of a CALL just as if sharing tails.
+         */
+        if (follow_direct_jump(dcontext, bb, target)) {
+            LOG(THREAD, LOG_INTERP, 4, 
+                "bb_process_IAT_convertible_indjmp: eliding jmp* target="PFX"\n", 
+                target);
+
+            STATS_INC(num_indirect_jumps_IAT_elided);
+            *elide_continue = true; /* do not stop bb */
+            return true;        /* converted indirect to direct */
+        }
+    }
+    /* otherwise convert to direct jump without eliding */
+
+    /* we set bb->instr to NULL so unlike bb_process_ubr
+     * we get the final exit_target added by build_bb_ilist
+     * FIXME: case 85: which will work only when we're using bb->mangle_ilist
+     * FIXME: what are callers supposed to see when we do NOT mangle?
+     */
+
+    LOG(THREAD, LOG_INTERP, 4, 
+        "bb_process_IAT_convertible_indjmp: converting jmp* target="PFX"\n", 
+        target);
+
+    STATS_INC(num_indirect_jumps_IAT_converted);
+    /* end basic block with a direct JMP to target */
+    bb->exit_target = target;   
+    *elide_continue = false;    /* matching, but should stop bb */
+    return true;               /* matching */
+}
+
+/* Returns true if the current instr in the BB is an indirect call
+ * through IAT that can be converted into a direct call, process it
+ * and sets elide_continue.  Otherwise function return false.
+ * OUT elide_continue is set when bb building should continue in target,
+ * and not set when bb building should be stopped.
+ */
+static bool
+bb_process_IAT_convertible_indcall(dcontext_t *dcontext, build_bb_t *bb,
+                                   bool *elide_continue)
+{
+    app_pc iat_reference;
+    app_pc target;
+    ASSERT(DYNAMO_OPTION(IAT_convert));
+
+    /* FIXME: the code structure is the same as
+     * bb_process_IAT_convertible_indjmp, could fuse the two
+     */
+
+    /* We perform several levels of checking, each increasingly more stringent
+     * and expensive, with a false return should any fail.
+     */
+
+    /* Check if the instr is a (near) indirect call */
+    if (instr_get_opcode(bb->instr) != OP_call_ind) {
+        ASSERT_CURIOSITY(false && "far call");
+        return false;           /* not matching, stop bb */
+    }
+
+    if (!is_targeting_convertible_IAT(dcontext, bb->instr,
+                                      &iat_reference)) {
+        return false;           /* not matching, stop bb */
+    }
+    target = read_from_IAT(iat_reference);
+    DOLOG(4, LOG_INTERP, {
+        char name[MAXIMUM_SYMBOL_LENGTH];
+        print_symbolic_address(target, name, sizeof(name), false);
+        LOG(THREAD, LOG_INTERP, 4, 
+            "bb_process_IAT_convertible_indcall: target="PFX" %s\n", target, name);
+    });
+    STATS_INC(num_indirect_calls_IAT);
+    
+    /* mangle mostly as such as direct calls are mangled with
+     * bb_process_call_direct(dcontext, bb)
+     */
+
+    if (must_not_be_inlined(target)) {
+        ASSERT_NOT_TESTED();
+        BBPRINT(bb, 3,
+                "   NOT inlining indirect call to must_not_be_inlined "PFX"\n", target);
+        return false; /* do not convert indirect call, stop bb */
+    }
+
+    /* Verify not targeting native exec DLLs, note that the IATs of
+     * any module may have imported a native DLL.  Note it may be
+     * possible to optimize with a range check on IAT subregions, but
+     * this check isn't much slower.
+     */
+    if (DYNAMO_OPTION(native_exec) &&
+        vmvector_overlap(native_exec_areas, target, target+1)) {
+        BBPRINT(bb, 3,
+                "   NOT inlining indirect call to native exec module "PFX"\n", target);
+        STATS_INC(num_indirect_calls_IAT_native);
+        return false;           /* do not convert indirect call, stop bb */
+    }
+
+    /* mangle_indirect_call and calculate return address as of
+     * bb->instr and will remove bb->instr 
+     * FIXME: it would have been
+     * better to replace in instrlist with a direct call and have
+     * mangle_{in,}direct_call use other than the raw bytes, but this for now does the job.
+     */
+    bb->instr->flags |= INSTR_IND_CALL_DIRECT;
+
+    if (DYNAMO_OPTION(IAT_elide)) {
+        /* try to elide just as a direct call would have been elided */
+        if (follow_direct_call(dcontext, bb, target)) {
+            LOG(THREAD, LOG_INTERP, 4, 
+                "bb_process_IAT_convertible_indcall: eliding call* flags=0x%08x target="
+                PFX"\n", bb->instr->flags, target);
+
+            STATS_INC(num_indirect_calls_IAT_elided);
+            *elide_continue = true; /* do not stop bb */
+            return true;        /* converted indirect to direct */
+        }
+    }
+    /* otherwise convert to direct call without eliding */
+
+    LOG(THREAD, LOG_INTERP, 4, 
+        "bb_process_IAT_convertible_indcall: converting call* flags=0x%08x target="PFX
+        "\n", bb->instr->flags, target);
+
+    STATS_INC(num_indirect_calls_IAT_converted);
+    /* bb->instr has been appended already, and will get removed by
+     * mangle_indirect_call.  We don't need to set to NULL, since this
+     * instr is a CTI and the final jump's translation target should
+     * still be the original indirect call.
+     */
+    bb->exit_target = target;
+    /* end basic block with a direct CALL to target.  With default
+     * options it should get mangled to a PUSH; JMP
+     */
+    *elide_continue = false;    /* matching, but should stop bb */
+    return true;                /* converted indirect to direct */
+}
+
+static bool
+instr_will_be_exit_cti(instr_t *inst)
+{
+    /* can't use instr_is_exit_cti() on pre-mangled instrs */
+    return (instr_ok_to_mangle(inst) &&
+            instr_is_cti(inst) &&
+            (!instr_is_call_direct(inst) ||
+             !must_not_be_inlined(instr_get_branch_target_pc(inst)))
+            /* PR 239470: ignore wow64 syscall, which is an ind call */
+            IF_WINDOWS(&& !instr_is_wow64_syscall(inst)));
+}
+
+#ifdef CLIENT_INTERFACE
+/* PR 215217: check syscall restrictions */
+static bool
+client_check_syscall(instrlist_t *ilist, instr_t *inst,
+                     bool *found_syscall, bool *found_int)
+{
+    /* We do consider the wow64 call* a syscall here (it is both
+     * a syscall and a call*: PR 240258).
+     */
+    if (instr_is_syscall(inst) || instr_get_opcode(inst) == OP_int) {
+        if (instr_is_syscall(inst) && found_syscall != NULL)
+            *found_syscall = true;
+        /* Xref PR 313869 - we should be ignoring int 3 here. */
+        if (instr_get_opcode(inst) == OP_int && found_int != NULL)
+            *found_int = true;
+        /* For linux an ignorable syscall is not a problem.  Our
+         * pre-syscall-exit jmp is added post client mangling so should
+         * be robust.
+         * FIXME: now that we have -no_inline_ignored_syscalls should
+         * we assert on ignorable also?  Probably we'd have to have
+         * an exception for the middle of a trace?
+         */
+        if (IF_LINUX(TEST(INSTR_NI_SYSCALL, inst->flags))
+            /* PR 243391: only block-ending interrupt 2b matters */
+            IF_WINDOWS(instr_is_syscall(inst) ||
+                       ((instr_get_opcode(inst) == OP_int &&
+                         instr_get_interrupt_number(inst) == 0x2b)))) {
+            /* This check means we shouldn't hit the exit_type flags
+             * check below but we leave it in place in case we add
+             * other flags in future
+             */
+            if (inst != instrlist_last(ilist)) {
+                CLIENT_ASSERT(false, "a syscall or interrupt must terminate the block");
+                return false;
+            }
+            /* should we forcibly delete the subsequent instrs?
+             * or the client has to deal w/ bad behavior in release build?
+             */
+        }
+    }
+    return true;
+}
+
+/* Pass bb to client, and afterward check for criteria we require and rescan for
+ * eflags and other flags that might have changed.
+ */
+static void
+client_process_bb(dcontext_t *dcontext, build_bb_t *bb)
+{
+    dr_emit_flags_t emitflags = DR_EMIT_DEFAULT;
+    instr_t *inst;
+    bool found_exit_cti = false;
+    bool found_syscall = false;
+    bool found_int = false;
+    instr_t *last_app_instr = NULL;
+
+    /* This routine is called by more than just bb builder, also used
+     * for recreating state, so only call if caller requested it
+     * (usually that coincides w/ bb->app_interp being set, but not
+     * when recreating state on a fault (PR 214962)).
+     * FIXME: hot patches shouldn't be injected during state recreations;
+     *        does predicating on bb->app_interp take care of this issue?
+     */
+    if (!bb->pass_to_client)
+        return;
+
+    /* Call the bb creation callback(s) */
+    if (!instrument_basic_block(dcontext, (app_pc) bb->start_pc, bb->ilist,
+                                bb->for_trace, !bb->app_interp, &emitflags)) {
+        /* although no callback was called we must process syscalls/ints (PR 307284) */
+    }
+
+    bb->post_client = true;
+
+    /* FIXME: instrumentor may totally mess us up -- our flags
+     * or syscall info might be wrong.  xref PR 215217
+     */
+
+    /* PR 215217, PR 240265:
+     * We need to check for client changes that require a new exit
+     * target.  We can't practically analyze the instrlist to decipher
+     * the exit, so we'll search backwards and require that the last 
+     * cti is the exit cti.  Typically, the last instruction in the 
+     * block should be the exit.  Post-mbr and post-syscall positions
+     * are particularly fragile, as our mangling code sets state up for
+     * the exit that could be messed up by instrs inserted after the
+     * mbr/syscall.  We thus disallow such instrs (except for
+     * dr_insert_mbr_instrumentation()).  xref cases 10503, 10782, 10784
+     *
+     * Here's what we support:
+     * - more than one exit cti; all but the last must be a ubr
+     * - an exit cbr or call must be the final instr in the block
+     * - only one mbr; must be the final instr in the block and the exit target
+     * - clients can't change the exit of blocks ending in a syscall
+     *   (or int), and the syscall must be the final instr in the block;
+     *   client can, however, remove the syscall and then add a different exit
+     * - client can't add a translation target that's outside of the original
+     *   source code bounds, or else our cache consistency breaks down
+     *   (the one exception to this is that a jump can translate to its target)
+     */
+
+    /* we set to NULL to have a default of fall-through */
+    bb->exit_target = NULL;
+    bb->exit_type = 0;
+
+    /* N.B.: we're walking backward */
+    for (inst = instrlist_last(bb->ilist); inst != NULL; inst = instr_get_prev(inst)) {
+
+        if (!instr_opcode_valid(inst))
+            continue;
+
+        if (instr_is_cti(inst) && inst != instrlist_last(bb->ilist)) {
+            /* PR 213005: coarse_units can't handle added ctis (meta or not)
+             * since decode_fragment(), used for state recreation, can't
+             * distinguish from exit cti.
+             */
+            bb->flags &= ~FRAG_COARSE_GRAIN;
+            STATS_INC(coarse_prevent_client);
+        }
+
+        if (!instr_ok_to_mangle(inst))
+            continue;
+
+        /* in case bb was truncated, find last non-meta fall-through */
+        if (last_app_instr == NULL)
+            last_app_instr = inst;
+    
+        /* PR 215217: client should not add new source code regions, else our
+         * cache consistency (both page prot and selfmod) will fail
+         */
+        CLIENT_ASSERT((instr_get_translation(inst) >= bb->start_pc &&
+                       instr_get_translation(inst) < bb->cur_pc) ||
+                      (instr_is_ubr(inst) && opnd_is_pc(instr_get_target(inst)) &&
+                       instr_get_translation(inst) == opnd_get_pc(instr_get_target(inst)))
+                      /* the displaced code and jmp return from intercept buffer
+                       * has translation fields set to hooked app routine */
+                      IF_WINDOWS(|| dr_fragment_app_pc(bb->start_pc) != bb->start_pc),
+                      "block's app sources (instr_set_translation() targets) "
+                      "must remain within original bounds");
+
+        /* PR 307284: we didn't process syscalls and ints pre-client
+         * so do so now to get bb->flags and bb->exit_type set
+         */
+        if (instr_is_syscall(inst) || instr_get_opcode(inst) == OP_int) {
+            instr_t *tmp = bb->instr;
+            bb->instr = inst;
+            if (instr_is_syscall(bb->instr))
+                bb_process_syscall(dcontext, bb);
+            else if (instr_get_opcode(bb->instr) == OP_int) /* non-syscall int */
+                bb_process_interrupt(dcontext, bb);
+            if (inst != instrlist_last(bb->ilist))
+                bb->instr = tmp;
+        }
+
+        /* ensure syscall/int2b terminates block */
+        client_check_syscall(bb->ilist, inst, &found_syscall, &found_int);
+
+        if (instr_will_be_exit_cti(inst)) {
+
+            if (!found_exit_cti) {
+                /* We're about to clobber the exit_type and could lose any
+                 * special flags set above, even if the client doesn't change
+                 * the exit target.  We undo such flags after this ilist walk
+                 * to support client removal of syscalls/ints.
+                 * INSTR_IND_JMP_PLT_EXIT is used for indcall2direct, which
+                 * is off by default for CI; it's also used for native_exec,
+                 * but we're not sure if we want to support that with CI.
+                 * xref case 10846
+                 */
+                CLIENT_ASSERT(!TEST(~(LINK_DIRECT | LINK_INDIRECT | LINK_CALL |
+                                      LINK_RETURN | LINK_JMP | LINK_NI_SYSCALL_ALL
+                                      IF_WINDOWS(| LINK_CALLBACK_RETURN)),
+                                    bb->exit_type),
+                              "client unsupported block exit type internal error");
+
+                found_exit_cti = true;
+                bb->instr = inst;
+
+                if (instr_is_ubr(inst) || instr_is_call_direct(inst)) {
+                    CLIENT_ASSERT(instr_is_ubr(inst) ||
+                                  inst == instrlist_last(bb->ilist),
+                                  "an exit call must terminate the block");
+                    /* a ubr need not be the final instr */
+                    if (inst == last_app_instr) {
+                        bb->exit_target = instr_get_branch_target_pc(inst);
+                        bb->exit_type = instr_branch_type(inst);
+                    }
+                }
+                else if (instr_is_cbr(inst)) {
+                    CLIENT_ASSERT(inst == instrlist_last(bb->ilist),
+                                  "an exit cbr must terminate the block");
+                    /* A null exit target specifies a cbr (see below). */
+                    bb->exit_target = NULL;
+                    bb->exit_type = 0;
+                    instr_exit_branch_set_type(bb->instr,
+                                               instr_branch_type(inst));
+                }
+                else {
+                    ASSERT(instr_is_mbr(inst));
+                    CLIENT_ASSERT(inst == instrlist_last(bb->ilist),
+                                  "an exit mbr must terminate the block");
+                    bb->exit_target = get_ibl_routine(dcontext, IBL_LINKED,
+                                                      DEFAULT_IBL_BB(), 
+                                                      get_ibl_branch_type(inst));
+                    bb->exit_type = instr_branch_type(inst);
+                }
+
+                /* since we're walking backward, at the first exit cti
+                 * we can check for post-cti code
+                 */
+                if (inst != instrlist_last(bb->ilist)) {
+                    if (TEST(FRAG_COARSE_GRAIN, bb->flags)) {
+                        /* PR 213005: coarse can't handle code beyond ctis */
+                        bb->flags &= ~FRAG_COARSE_GRAIN;
+                        STATS_INC(coarse_prevent_client);
+                    }
+                    /* decode_fragment can't handle code beyond ctis */
+                    bb->flags |= FRAG_CANNOT_BE_TRACE;
+                }
+
+            }
+
+            /* Case 10784: Clients can confound trace building when they
+             * introduce more than one exit cti; we'll just disable traces
+             * for these fragments.
+             * PR 215179: we're currently later marking them no-trace for pad_jmps
+             * reasons as well.
+             */
+            else {
+                CLIENT_ASSERT(instr_is_ubr(inst),
+                              "a second exit cti must be a ubr");
+                bb->flags |= FRAG_CANNOT_BE_TRACE;
+                /* our cti check above should have already turned off coarse */
+                ASSERT(!TEST(FRAG_COARSE_GRAIN, bb->flags));
+            }
+        }
+    }
+
+    /* To handle the client modifying syscall numbers we cannot inline
+     * syscalls in the middle of a bb.
+     */
+    ASSERT(!DYNAMO_OPTION(inline_ignored_syscalls));
+
+    ASSERT((TEST(FRAG_HAS_SYSCALL, bb->flags) && found_syscall) ||
+           (!TEST(FRAG_HAS_SYSCALL, bb->flags) && !found_syscall));
+    IF_WINDOWS(ASSERT(!TEST(LINK_CALLBACK_RETURN, bb->exit_type) || found_int));
+
+    /* Note that we do NOT remove, or set, FRAG_HAS_DIRECT_CTI based on
+     * client modifications: setting it for a selfmod fragment could
+     * result in an infinite loop, and it is mainly used for elision, which we
+     * are not doing for client ctis.  Clients are not supposed add new
+     * app source regions (PR 215217).
+     */
+
+    /* Client might have truncated: re-set fall-through. */
+    if (last_app_instr != NULL) {
+        /* We do not take instr_length of what the client put in, but rather
+         * the length of the translation target
+         */
+        app_pc last_app_pc = instr_get_translation(last_app_instr);
+        bb->cur_pc = decode_next_pc(dcontext, last_app_pc);
+        LOG(THREAD, LOG_INTERP, 3,
+            "setting cur_pc (for fall-through) to" PFX"\n", bb->cur_pc);
+        /* don't set bb->instr if last instr is still syscall/int.
+         * FIXME: I'm not 100% convinced the logic here covers everything
+         * build_bb_ilist does.
+         * FIXME: what about if last instr was invalid, or if client adds
+         * some invalid instrs: xref bb_process_invalid_instr()
+         */
+        if (bb->instr != NULL || (!found_int && !found_syscall))
+            bb->instr = last_app_instr;
+    } else
+        bb->instr = NULL; /* no app instrs left */
+
+    /* PR 215217: re-scan for accurate eflags.
+     * FIXME: should we not do eflags tracking while decoding, then, and always
+     * do it afterward?
+     */
+    bb->eflags = forward_eflags_analysis(dcontext, bb->ilist, instrlist_first(bb->ilist));
+
+    if (TEST(DR_EMIT_STORE_TRANSLATIONS, emitflags)) {
+        /* PR 214962: let client request storage instead of recreation */
+        bb->flags |= FRAG_HAS_TRANSLATION_INFO;
+        /* if we didn't have record on from start, can't store translation info */
+        ASSERT(bb->record_translation && bb->full_decode);
+    }
+}
+#endif /* CLIENT_INTERFACE */
+
+/* Interprets the application's instructions until the end of a basic
+ * block is found, and prepares the resulting instrlist for creation of
+ * a fragment, but does not create the fragment, just returns the instrlist.
+ * Caller is responsible for freeing the list and its instrs!
+ *
+ * Input parameters in bb control aspects of creation:
+ *   If app_interp is true, this is considered real app code.
+ *   If pass_to_client is true,
+ *     calls instrument routine on bb->ilist before mangling
+ *   If mangle_ilist is true, mangles the ilist, else leaves it in app form
+ *   If for_cache is true, updates the vmareas data structures
+ *     Caller must set and later clear dcontext->bb_build_info.
+ *     For !for_cache, build_bb_ilist() sets and clears it, making the
+ *     assumption that the caller is doing no other reading from the region.
+ *   If record_translation is true, records translation for inserted instrs
+ *   If outf != NULL, does full disassembly with comments to outf
+ *   If overlap_info != NULL, records overlap information for the block in
+ *     the overlap_info (caller must fill in region_start and region_end).
+ *
+ * FIXME: now that we have better control over following direct ctis,
+ * should we have adaptive mechanism to decided whether to follow direct
+ * ctis, since some bmarks are better doing so (gap, vortex, wupwise)
+ * and others are worse (apsi, perlbmk)?
+ */
+static void
+build_bb_ilist(dcontext_t *dcontext, build_bb_t *bb)
+{
+    /* Design decision: we will not try to identify branches that target
+     * instructions in this basic block, when we take those branches we will
+     * just make a new basic block and duplicate part of this one
+     */
+    int total_branches = 0;
+    uint total_instrs = 0;
+    uint total_writes = 0; /* only used for selfmod */
+    instr_t *non_cti;              /* used if !full_decode */
+    byte *non_cti_start_pc; /* used if !full_decode */
+    uint eflags_6 = 0; /* holds arith eflags written so far (in read slots) */
+    /* indirect branch type as an IBL selector */
+    ibl_branch_type_t ibl_branch_type = IBL_GENERIC; /* initialization only */
+#ifdef HOT_PATCHING_INTERFACE
+    bool hotp_should_inject = false, hotp_injected = false;
+#endif
+    app_pc page_start_pc = (app_pc) NULL;
+    bool bb_build_nested = false;
+    /* Caller will free objects allocated here so we must use the passed-in
+     * dcontext for allocation; we need separate var for non-global dcontext.
+     */
+    dcontext_t *my_dcontext = get_thread_private_dcontext();
+    DEBUG_DECLARE(bool regenerated = false;)
+
+    ASSERT(bb->initialized);
+    ASSERT(bb->start_pc != NULL);
+    /* vmlist must start out empty (or N/A) */
+    ASSERT(bb->vmlist == NULL || !bb->for_cache);
+
+#ifdef CUSTOM_TRACES_RET_REMOVAL
+    my_dcontext->num_calls = 0;
+    my_dcontext->num_rets = 0;
+#endif
+
+    /* Support bb abort on decode fault */
+    if (my_dcontext != NULL) {
+        if (bb->for_cache) {
+            /* Caller should have set! */
+            ASSERT(bb == (build_bb_t *) my_dcontext->bb_build_info);
+        } else if (my_dcontext->bb_build_info == NULL) {
+            my_dcontext->bb_build_info = (void *) bb;
+        } else {
+            /* For nested we leave the original, which should be the only vmlist,
+             * and we give up on freeing dangling instr_t and instrlist_t from this decode.
+             * We need the original's for_cache so we know to free the bb_building_lock.
+             * FIXME: use TRY to handle decode exceptions locally?  Shouldn't have
+             * violation remediations on a !for_cache build.
+             */
+            ASSERT(bb->vmlist == NULL && !bb->for_cache &&
+                   ((build_bb_t *)my_dcontext->bb_build_info)->for_cache);
+            /* FIXME: add nested as a field so we can have stat on nested faults */
+            bb_build_nested = true;
+        }
+    } else
+        ASSERT(dynamo_exited);
+    
+    if (bb->record_translation || !bb->for_cache ||
+        IF_CLIENT_INTERFACE_ELSE(INTERNAL_OPTION(full_decode), false))
+        bb->full_decode = true;
+    else {
+#if defined(STEAL_REGISTER) || defined(CHECK_RETURNS_SSE2)
+        bb->full_decode = true;
+#endif
+    }
+
+    LOG(THREAD, LOG_INTERP, 3, "\ninterp: ");
+    BBPRINT(bb, 3, "start_pc = "PFX"\n", bb->start_pc);
+
+    DOSTATS({
+        if (bb->app_interp) {
+            if (fragment_lookup_deleted(dcontext, bb->start_pc)) {
+                /* this will look up private 1st, so yes we will get
+                 * dup stats if multiple threads have regnerated the
+                 * same private tag, or if a shared tag is deleted and
+                 * multiple privates created
+                 */
+                regenerated = true;
+                STATS_INC(num_fragments_deja_vu);
+            }
+        }
+    });
+
+    /* start converting instructions into IR */
+    check_new_page_start(dcontext, bb);
+    bb->cur_pc = bb->start_pc;
+    
+    /* for translation in case we break out of loop before decoding any
+     * instructions, (i.e. check_for_stopping_point()) */
+    bb->instr_start = bb->cur_pc;
+
+    /* create instrlist after check_new_page_start to avoid memory leak
+     * on unreadable memory -- though we now properly clean up and won't leak
+     * on unreadable on any check_thread_vm_area call
+     */
+    bb->ilist = instrlist_create(dcontext);
+    bb->instr = NULL;
+
+    /* avoid discrepancy in finding invalid instructions between fast decode
+     * and the full decode of sandboxing by doing full decode up front
+     */
+    if (TEST(FRAG_SELFMOD_SANDBOXED, bb->flags))
+        bb->full_decode = true;
+    if (TEST(FRAG_HAS_TRANSLATION_INFO, bb->flags)) {
+        bb->full_decode = true;
+        bb->record_translation = true;
+    }
+
+    KSTART(bb_decoding);
+    while (true) {
+        if (check_for_stopping_point(dcontext, bb)) {
+            BBPRINT(bb, 3, "interp: found DynamoRIO stopping point at "PFX"\n",
+                    bb->cur_pc);
+            break;
+        }
+
+        /* fill in a new instr structure and update bb->cur_pc */
+        bb->instr = instr_create(dcontext);
+        /* if !full_decode:
+         * All we need to decode are control-transfer instructions
+         * For efficiency, put all non-cti into a single instr_t structure
+         */
+        non_cti_start_pc = bb->cur_pc;
+        do {
+            /* If the thread's vmareas aren't being added to, indicate the
+             * page that's being decoded. */
+            if (!bb->for_cache
+                && page_start_pc != (app_pc) PAGE_START(bb->cur_pc)) {
+                page_start_pc = (app_pc) PAGE_START(bb->cur_pc);
+                set_thread_decode_page_start(my_dcontext == NULL ?
+                                             dcontext : my_dcontext, page_start_pc);
+            }
+
+            bb->instr_start = bb->cur_pc;
+            if (bb->full_decode) {
+                /* only going through this do loop once! */
+                bb->cur_pc = decode(dcontext, bb->cur_pc, bb->instr);
+                if (bb->record_translation)
+                    instr_set_translation(bb->instr, bb->instr_start);
+            } else {
+                /* must reset, may go through loop multiple times */
+                instr_reset(dcontext, bb->instr);
+                bb->cur_pc = decode_cti(dcontext, bb->cur_pc, bb->instr);
+            }
+
+            ASSERT(!bb->check_vm_area || bb->checked_end != NULL);
+            if (bb->check_vm_area &&
+                bb->cur_pc != NULL && bb->cur_pc-1 > bb->checked_end) {
+                /* We're beyond the vmarea allowed -- so check again.
+                 * Ideally we'd want to check BEFORE we decode from the
+                 * subsequent page, as it could be inaccessible, but not worth
+                 * the time estimating the size from a variable number of bytes
+                 * before the page boundary.  Instead we rely on other
+                 * mechanisms to handle faults while decoding, which we need
+                 * anyway to handle racy unmaps by the app.
+                 */
+                check_new_page_contig(dcontext, bb, bb->cur_pc-1);
+            }
+
+            total_instrs++;
+            DOELOG(3, LOG_INTERP, {
+                disassemble_with_bytes(dcontext, bb->instr_start, THREAD);
+            });
+
+#if defined(INTERNAL) || defined(CLIENT_INTERFACE)
+            if (bb->outf != INVALID_FILE)
+                disassemble_with_bytes(dcontext, bb->instr_start, bb->outf);
+#endif /* INTERNAL || CLIENT_INTERFACE */
+
+            if (!instr_valid(bb->instr))
+                break; /* before eflags analysis! */
+
+            /* Eflags analysis:
+             * We do this even if -unsafe_ignore_eflags_prefix b/c it doesn't cost that
+             * much and we can use the analysis to detect any bb that reads a flag
+             * prior to writing it.
+             */
+            if (bb->eflags != EFLAGS_WRITE_6 && bb->eflags != EFLAGS_READ_OF)
+                bb->eflags = eflags_analysis(bb->instr, bb->eflags, &eflags_6);
+
+            /* stop decoding at an invalid instr (tested above) or a cti
+             *(== opcode valid) or a possible SEH frame push (if
+             * -process_SEH_push). */
+#ifdef WINDOWS
+            if (DYNAMO_OPTION(process_SEH_push) &&
+                instr_get_prefix_flag(bb->instr, PREFIX_SEG_FS)) {
+                STATS_INC(num_bb_build_fs);
+                break;
+            }
+#endif
+
+#ifdef X64
+            if (instr_has_rel_addr_reference(bb->instr)) {
+                /* PR 215397: we need to split these out for re-relativization */
+                break;
+            }
+#endif
+        } while (!instr_opcode_valid(bb->instr) &&
+                 total_instrs <= DYNAMO_OPTION(max_bb_instrs));
+
+        if (bb->cur_pc == NULL) {
+            /* invalid instr: reset bb->cur_pc, will end bb after updating stats */
+            bb->cur_pc = bb->instr_start;
+        }
+
+#ifdef CLIENT_INTERFACE
+        /* For CI builds, we need the translation when mangling direct
+         * calls.  May as well set it for all cti's since there's
+         * really no extra overhead in doing so.  Note that we go
+         * through the above loop only once for cti's, so it's safe
+         * to set the translation here.
+         */
+        if (instr_opcode_valid(bb->instr) && instr_is_cti(bb->instr))
+            instr_set_translation(bb->instr, bb->instr_start);
+#endif
+
+#ifdef HOT_PATCHING_INTERFACE
+        /* If this lookup succeeds then the current bb needs to be patched.
+         * In hotp_inject(), address lookup will be done for each instruction 
+         * pc in this bb and patching will be done if an exact match is found.
+         *
+         * Hot patching should be done only for app interp and recreating
+         * pc, not for reproducing app code.  Hence we use mangle_ilist.
+         * See case 5981.
+         *
+         * FIXME: this lookup can further be reduced by determining whether or
+         *        not the current bb's module needs patching via check_new_page*
+         */
+        if (DYNAMO_OPTION(hot_patching) && bb->mangle_ilist &&
+            !hotp_should_inject) {
+            /* case 8780: we may hold the lock; FIXME: figure out if this can
+             * be avoided - messy to hold hotp_vul_table lock like this for
+             * unnecessary operations. */
+            bool owns_hotp_lock = self_owns_write_lock(hotp_get_lock());
+            if (hotp_does_region_need_patch(non_cti_start_pc, bb->cur_pc,
+                                            owns_hotp_lock)) {
+                BBPRINT(bb, 2, "hotpatch match in "PFX": "PFX"-"PFX"\n",
+                        bb->start_pc, non_cti_start_pc, bb->cur_pc);
+                hotp_should_inject = true;
+                /* Don't elide if we are going to hot patch this bb because
+                 * the patch point can be a direct cti; eliding would result
+                 * in the patch not being applied.  See case 5901.
+                 * FIXME: we could make this more efficient by only turning 
+                 * off follow_direct if the instr is direct cti.
+                 */
+                bb->follow_direct = false;
+                DOSTATS({
+                    if TEST(FRAG_HAS_DIRECT_CTI, bb->flags)
+                        STATS_INC(hotp_num_frag_direct_cti);
+                });
+            }
+        }
+#endif
+
+        if (bb->full_decode) {
+            if (TEST(FRAG_SELFMOD_SANDBOXED, bb->flags) &&
+                instr_valid(bb->instr) && instr_writes_memory(bb->instr)) {
+                /* to allow tailing non-writes, end prior to the write beyond the max */
+                total_writes++;
+                if (total_writes > DYNAMO_OPTION(selfmod_max_writes)) {
+                    BBPRINT(bb, 3, "reached selfmod write limit %d, stopping\n",
+                            DYNAMO_OPTION(selfmod_max_writes));
+                    STATS_INC(num_max_selfmod_writes_enforced);
+                    instr_destroy(dcontext, bb->instr); /* not added to bb->ilist yet */
+                    bb->instr = NULL;
+                    bb->cur_pc = bb->instr_start;
+                    break;
+                }
+            }
+        } else if (bb->instr_start != non_cti_start_pc) {
+            /* instr now holds the cti, so create an instr_t for the non-cti */
+            non_cti = instr_create(dcontext);
+            IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(bb->instr_start - non_cti_start_pc)));
+            instr_set_raw_bits(non_cti, non_cti_start_pc,
+                               (uint)(bb->instr_start - non_cti_start_pc));
+            /* add non-cti instructions to instruction list */
+            instrlist_append(bb->ilist, non_cti);
+        }
+
+        DOSTATS({
+            /* This routine is also called for recreating state, we only want
+             * to count app code when we build new bbs, which is indicated by
+             * the bb->app_interp parameter
+             */
+            if (bb->app_interp && !regenerated) {
+                /* avoid double-counting for adaptive working set */
+                /* FIXME - ubr ellision leads to double couting.  We also
+                 * double count when we have multiple entry points into the
+                 * same block of cti free instructinos. */
+                STATS_ADD(app_code_seen, (bb->cur_pc - non_cti_start_pc));
+                LOG(THREAD, LOG_INTERP, 5, "adding %d bytes to total app code seen\n",
+                    bb->cur_pc - non_cti_start_pc);
+            }
+        });
+
+        if (!instr_valid(bb->instr)) {
+            bb_process_invalid_instr(dcontext, bb);
+            break;
+        }
+
+#ifdef WINDOWS
+        if (DYNAMO_OPTION(process_SEH_push) &&
+            instr_get_prefix_flag(bb->instr, PREFIX_SEG_FS)) {
+            DEBUG_DECLARE(ssize_t dbl_count = bb->cur_pc - bb->instr_start); 
+            if (!bb_process_fs_ref(dcontext, bb)) {
+                DOSTATS({
+                    if (bb->app_interp) {
+                        LOG(THREAD, LOG_INTERP, 3,
+                            "stopping bb at fs-using instr @ "PFX"\n",
+                            bb->instr_start);
+                        STATS_INC(num_process_SEH_bb_early_terminate);
+                        /* don't double count the fs instruction itself
+                         * since we removed it from this bb */
+                        if (!regenerated)
+                            STATS_ADD(app_code_seen, -dbl_count);
+                    }
+                });
+                break;
+            }
+        }
+#else
+        if (instr_get_prefix_flag(bb->instr,
+                                  (SEG_TLS == SEG_GS) ? PREFIX_SEG_GS : PREFIX_SEG_FS)) {
+            /* On linux we use a segment register and do not yet
+             * support the application using the same register!
+             */
+            CLIENT_ASSERT(false, "no support yet for application using non-NPTL segment");
+            ASSERT_BUG_NUM(205276, false);
+        }
+#endif
+
+        if (instr_is_ubr(bb->instr)) {
+            if (bb_process_ubr(dcontext, bb))
+                continue;
+            else
+                break;
+        } else
+            instrlist_append(bb->ilist, bb->instr);
+
+#ifdef RETURN_AFTER_CALL
+        if (bb->app_interp && dynamo_options.ret_after_call) {
+            if (instr_is_call(bb->instr)) {
+                /* add after call instruction to valid return targets */
+                add_return_target(dcontext, bb->instr_start, bb->instr);
+            }
+        }
+#endif /* RETURN_AFTER_CALL */
+
+        if (instr_is_call_direct(bb->instr)) {
+            if (!bb_process_call_direct(dcontext, bb)) {
+                bb->exit_type |= instr_branch_type(bb->instr);
+                break;
+            }
+        }
+        else if (instr_is_mbr(bb->instr)) { /* including indirect calls */
+
+            /* Manage the case where we don't need to perform 'normal'
+             * indirect branch processing.
+             */
+            bool normal_indirect_processing = true;
+            bool elide_and_continue_if_converted = true;
+
+            if (instr_is_return(bb->instr)) {
+                ibl_branch_type = IBL_RETURN;
+                STATS_INC(num_returns);
+            } else if (instr_is_call_indirect(bb->instr)) {
+                STATS_INC(num_all_calls);
+                STATS_INC(num_indirect_calls);
+
+                /* If the indirect call can be converted into a direct one,
+                 * bypass normal indirect call processing.
+                 */
+                if (DYNAMO_OPTION(indcall2direct)
+                    && bb_process_convertible_indcall(dcontext, bb)) {
+                    normal_indirect_processing = false;
+                    elide_and_continue_if_converted = true;
+                } else if (DYNAMO_OPTION(IAT_convert)
+                           && bb_process_IAT_convertible_indcall(dcontext, bb, 
+                                                                 &elide_and_continue_if_converted)) {
+                    normal_indirect_processing = false;
+                }
+                else if (bb_process_indcall_syscall(dcontext, bb,
+                                                    &elide_and_continue_if_converted)) {
+                    normal_indirect_processing = false;
+                }
+                else
+                    ibl_branch_type = IBL_INDCALL;
+            } else {
+                /* indirect jump */
+                /* was prev instr a direct call? if so, this is a PLT-style ind call */
+                instr_t *prev = instr_get_prev(bb->instr);
+                if (prev != NULL && instr_opcode_valid(prev) && instr_is_call_direct(prev)) {
+                    bb->exit_type |= INSTR_IND_JMP_PLT_EXIT;
+                    /* just because we have a CALL to JMP* makes it
+                       only a _likely_ PLT call, we still have to make
+                       sure it goes through IAT - see case 4269
+                    */
+                    STATS_INC(num_indirect_jumps_likely_PLT);
+                }
+
+                elide_and_continue_if_converted = true;
+                        
+                if (DYNAMO_OPTION(IAT_convert) 
+                    && bb_process_IAT_convertible_indjmp(dcontext, bb, 
+                                                         &elide_and_continue_if_converted)) {
+                    /* Clear the IND_JMP_PLT_EXIT flag since we've converted
+                     * the PLT to a direct transition (and possibly elided).
+                     * Xref case 7867 for why leaving this flag in the eliding
+                     * case can cause later failures. */
+                    bb->exit_type &= ~INSTR_IND_JMP_PLT_EXIT;
+                    normal_indirect_processing = false;
+                } else          /* FIXME: this can always be set */
+                    ibl_branch_type = IBL_INDJMP;
+                STATS_INC(num_indirect_jumps);
+            }
+#ifdef CUSTOM_TRACES_RET_REMOVAL
+            if (instr_is_return(bb->instr))
+                my_dcontext->num_rets++;
+            else if (instr_is_call_indirect(bb->instr))
+                my_dcontext->num_calls++;
+#endif
+            /* set exit type since this instruction will get mangled */
+            if (normal_indirect_processing) {
+                bb->exit_type |= instr_branch_type(bb->instr);
+                bb->exit_target = get_ibl_routine(dcontext, IBL_LINKED, DEFAULT_IBL_BB(),
+                                                  ibl_branch_type);
+                break;
+            } else {
+                /* decide whether to stop bb here */
+                if (!elide_and_continue_if_converted)
+                    break;
+                /* fall through for -max_bb_instrs check */
+            }
+        }
+        else if (instr_is_cti(bb->instr) && !instr_is_call(bb->instr)) {
+            total_branches++;
+            if (total_branches >= BRANCH_LIMIT) {
+                /* set type of 1st exit cti for cbr (bb->exit_type is for fall-through) */
+                instr_exit_branch_set_type(bb->instr, instr_branch_type(bb->instr));
+                break;
+            }
+        }
+        else if (instr_is_syscall(bb->instr)) {
+            if (!bb_process_syscall(dcontext, bb))
+                break;
+        } /* end syscall */
+        else if (instr_get_opcode(bb->instr) == OP_int) { /* non-syscall int */
+            if (!bb_process_interrupt(dcontext, bb))
+                break;
+        }
+#ifdef CHECK_RETURNS_SSE2
+        else if (instr_is_sse_or_sse2(bb->instr)) {
+            FATAL_USAGE_ERROR(CHECK_RETURNS_SSE2_XMM_USED, 2, 
+                              get_application_name(), get_application_pid());
+        }
+#endif
+
+        if (total_instrs > DYNAMO_OPTION(max_bb_instrs)) {
+            /* this could be an enormous basic block, or it could
+             * be some degenerate infinite-loop case like a call
+             * to a function that calls exit() and then calls itself,
+             * so just end it here, we'll pick up where we left off
+             * if it's legit
+             */
+            BBPRINT(bb, 3, "reached -max_bb_instrs, stopping\n");
+            STATS_INC(num_max_bb_instrs_enforced);
+            break;
+        }
+
+    } /* end of while (true) */
+    KSTOP(bb_decoding);
+
+#ifdef DEBUG_MEMORY
+    /* make sure anyone who destroyed also set to NULL */
+    ASSERT(bb->instr == NULL ||
+           (bb->instr->bytes != (byte *) HEAP_UNALLOCATED_PTR_UINT &&
+            bb->instr->bytes != (byte *) HEAP_ALLOCATED_PTR_UINT &&
+            bb->instr->bytes != (byte *) HEAP_PAD_PTR_UINT));
+#endif
+
+    check_new_page_contig(dcontext, bb, bb->cur_pc-1);
+    bb->end_pc = bb->cur_pc;
+    BBPRINT(bb, 3, "end_pc = "PFX"\n\n", bb->end_pc);
+
+    /* We could put this in check_new_page_jmp where it already checks
+     * for native_exec overlap, but selfmod ubrs don't even call that routine
+     */
+    if (DYNAMO_OPTION(native_exec) &&
+        DYNAMO_OPTION(native_exec_callcall) &&        
+        !vmvector_empty(native_exec_areas) &&
+        bb->app_interp && bb->instr != NULL && 
+        (instr_is_ubr(bb->instr) || instr_is_call_direct(bb->instr)) &&
+        instrlist_first(bb->ilist) == instrlist_last(bb->ilist)) {
+        /* Case 4564/3558: handle .NET COM method table where a call* targets
+         * a call to a native_exec dll -- we need to put the gateway at the
+         * call* to avoid retaddr mangling of the method table call.
+         * As a side effect we can also handle call*, jmp.
+         * We don't actually verify or care that it was specifically a call*,
+         * whatever at_native_exec_gateway() requires to assure itself that we're
+         * at a return-address-clobberable point.
+         */
+        app_pc tgt = opnd_get_pc(instr_get_target(bb->instr));
+        if (vmvector_overlap(native_exec_areas, tgt, tgt+1) &&
+            at_native_exec_gateway(dcontext, tgt _IF_DEBUG(true/*xfer tgt*/))) {
+            /* replace this ilist w/ a native exec one */
+            LOG(THREAD, LOG_INTERP, 2,
+                "direct xfer @gateway @"PFX" to native_exec module "PFX"\n",
+                bb->start_pc, tgt);
+            bb->native_exec = true;
+            /* add this ubr/call to the native_exec_list, both as an optimization
+             * for future entrances and b/c .NET changes its method table call
+             * from targeting a native_exec image to instead target DGC directly,
+             * thwarting our gateway!
+             * FIXME: if heap region de-allocated, we'll remove, but what if re-used
+             * w/o going through syscalls?  Just written over w/ something else?
+             * We'll keep it on native_exec_list...
+             */
+            ASSERT(bb->end_pc == bb->start_pc + DIRECT_XFER_LENGTH);
+            vmvector_add(native_exec_areas, bb->start_pc, bb->end_pc, NULL);
+            DODEBUG({ report_native_module(dcontext, tgt); });
+            STATS_INC(num_native_module_entrances_callcall);
+            return;
+        }
+    }
+
+    STATS_TRACK_MAX(max_instrs_in_a_bb, total_instrs);
+
+#ifdef LINUX
+    if (bb->invalid_instr_hack) {
+        /* turn off selfmod -- we assume bb will hit exception right away */
+        if (TEST(FRAG_SELFMOD_SANDBOXED, bb->flags))
+            bb->flags &= ~FRAG_SELFMOD_SANDBOXED;
+        /* decode_fragment() can't handle invalid instrs, so store translations */
+        bb->flags |= FRAG_HAS_TRANSLATION_INFO;
+    }
+#endif
+
+    if (TEST(FRAG_SELFMOD_SANDBOXED, bb->flags) &&
+        (TEST(FRAG_HAS_DIRECT_CTI, bb->flags) || !bb->full_decode)) {
+        /* Sandbox requires that bb have no direct cti followings, and
+         * had full decode from the start for -selfmod_max_writes!
+         * check_thread_vm_area should have ensured this for us, except
+         * there is a pathological case where a direct cti occurs, and
+         * then we walk over a page boundary onto a selfmod page -- no
+         * choice but to back out, which for now we do by rebuilding.
+         * FIXME: find better way
+         */
+        /* FIXME: a better assert is needed because this can trigger if 
+         * hot patching turns off follow_direct, the current bb was elided 
+         * earlier and is marked as selfmod.  hotp_num_frag_direct_cti will 
+         * track this for now.
+         */
+        ASSERT(bb->follow_direct == true); /* else, infinite loop possible */
+        BBPRINT(bb, 2,
+                "*** must rebuild bb to avoid following direct cti for selfmod ***\n");
+        STATS_INC(num_bb_end_early);
+        instrlist_clear_and_destroy(dcontext, bb->ilist);
+        if (bb->vmlist != NULL) {
+            vm_area_destroy_list(dcontext, bb->vmlist);
+            bb->vmlist = NULL;
+        }
+        bb->flags = FRAG_SELFMOD_SANDBOXED; /* lose all other flags */
+        bb->full_decode = true; /* full decode -- see comment at top of routine */
+        bb->follow_direct = false; 
+        /* overlap info will be reset by check_new_page_start */
+        build_bb_ilist(dcontext, bb);
+        return;
+    }
+
+#ifdef HOT_PATCHING_INTERFACE
+    /* CAUTION: This can't be moved below client interface as the basic block
+     *          can be changed by the client.  This will mess up hot patching.
+     *          The same is true for mangling.
+     */
+    if (hotp_should_inject) {
+        ASSERT(DYNAMO_OPTION(hot_patching));
+        hotp_injected = hotp_inject(dcontext, bb->ilist);
+
+        /* Fix for 5272.  Hot patch injection uses dr clean call api which 
+         * accesses dcontext fields directly, so the injected bbs can't be
+         * shared until that is changed or the clean call mechanism is replaced
+         * with bb termination to execute hot patchces.
+         * Case 9995 assumes that hotp fragments are fine-grained, which we
+         * achieve today by being private; if we make shared we must explicitly
+         * prevent from being coarse-grained.
+         */
+        if (hotp_injected) {
+            bb->flags &= ~FRAG_SHARED;
+            bb->flags |= FRAG_CANNOT_BE_TRACE;
+        }
+    }
+#endif
+
+#ifdef CLIENT_INTERFACE
+    client_process_bb(dcontext, bb);
+    if (bb->unmangled_ilist != NULL)
+        *bb->unmangled_ilist = instrlist_clone(dcontext, bb->ilist);
+#endif
+
+    /* create a final instruction that will jump to the exit stub
+     * corresponding to the fall-through of the conditional branch or
+     * the target of the final indirect branch (the indirect branch itself
+     * will get mangled into a non-cti)
+     */
+    if (bb->exit_target == NULL) { /* not set by ind branch, etc. */
+        bb->exit_target = (cache_pc) bb->cur_pc; /* fall-through pc */
+        if (bb->instr != NULL && instr_opcode_valid(bb->instr) &&
+            instr_is_cbr(bb->instr) &&
+            (int) (bb->exit_target - bb->start_pc) <= SHRT_MAX &&
+            (int) (bb->exit_target - bb->start_pc) >= SHRT_MIN &&
+            /* rule out jecxz, etc. */
+            !instr_is_cti_loop(bb->instr))
+            bb->flags |= FRAG_CBR_FALLTHROUGH_SHORT;
+    }
+    /* we share all basic blocks except selfmod (since want no-synch quick deletion)
+     * or syscall-containing ones (to bound delay on threads exiting shared cache,
+     * for cache management, both consistency and capacity)
+     * bbs injected with hot patches are also not shared (see case 5272).
+     */
+    if (DYNAMO_OPTION(shared_bbs) && !TEST(FRAG_SELFMOD_SANDBOXED, bb->flags) &&
+        !TEST(FRAG_TEMP_PRIVATE, bb->flags)
+#ifdef HOT_PATCHING_INTERFACE
+        && !hotp_injected
+#endif
+       ) {
+        /* If the fragment doesn't have a syscalls or contains a
+         * non-ignorable one -- meaning that the frag will exit the cache
+         * to execute the syscall -- it can be shared.
+         * We don't support ignorable syscalls in shared fragments, as they
+         * don't set at_syscall and so are incompatible w/ -syscalls_synch_flush.
+         */
+        if (!TEST(FRAG_HAS_SYSCALL, bb->flags) ||
+            TESTANY(LINK_NI_SYSCALL_ALL, bb->exit_type))
+            bb->flags |= FRAG_SHARED;
+#ifdef WINDOWS
+        /* A fragment can be shared if it contains a syscall that will be
+         * executed via the version of shared syscall that can be targetted by
+         * shared frags.
+         */
+        else if (TEST(FRAG_HAS_SYSCALL, bb->flags) &&
+                 DYNAMO_OPTION(shared_fragment_shared_syscalls) &&
+                 bb->exit_target == shared_syscall_routine(dcontext))
+            bb->flags |= FRAG_SHARED;
+        else {
+            ASSERT((TEST(FRAG_HAS_SYSCALL, bb->flags) &&
+                    (DYNAMO_OPTION(ignore_syscalls) ||
+                     (!DYNAMO_OPTION(shared_fragment_shared_syscalls) &&
+                      bb->exit_target == shared_syscall_routine(dcontext)))) &&
+                   "BB not shared for unknown reason");
+        }
+#endif
+    }
+
+    if (TEST(FRAG_COARSE_GRAIN, bb->flags) &&
+        (!TEST(FRAG_SHARED, bb->flags) ||
+         /* Ignorable syscalls on linux are mangled w/ intra-fragment jmps, which
+          * decode_fragment() cannot handle -- and on win32 this overlaps w/
+          * FRAG_MUST_END_TRACE and LINK_NI_SYSCALL
+          */
+         TEST(FRAG_HAS_SYSCALL, bb->flags) ||
+         TEST(FRAG_MUST_END_TRACE, bb->flags) ||
+         TEST(FRAG_CANNOT_BE_TRACE, bb->flags) ||
+         TEST(FRAG_SELFMOD_SANDBOXED, bb->flags) ||
+         /* PR 214142: coarse units does not support storing translations */
+         TEST(FRAG_HAS_TRANSLATION_INFO, bb->flags) ||
+         /* FRAG_HAS_DIRECT_CTI: we never elide (assert is below);
+          * not-inlined call/jmp: we turn off FRAG_COARSE_GRAIN up above
+          */
+#ifdef WINDOWS
+         TEST(LINK_CALLBACK_RETURN, bb->exit_type) ||
+#endif
+         TESTANY(LINK_NI_SYSCALL_ALL, bb->exit_type))){
+        /* Currently not supported in a coarse unit */
+        STATS_INC(num_fine_in_coarse);
+        DOSTATS({
+            if (!TEST(FRAG_SHARED, bb->flags))
+                STATS_INC(coarse_prevent_private); 
+            else if (TEST(FRAG_HAS_SYSCALL, bb->flags))
+                STATS_INC(coarse_prevent_syscall); 
+            else if (TEST(FRAG_MUST_END_TRACE, bb->flags))
+                STATS_INC(coarse_prevent_end_trace); 
+            else if (TEST(FRAG_CANNOT_BE_TRACE, bb->flags))
+                STATS_INC(coarse_prevent_no_trace); 
+            else if (TEST(FRAG_SELFMOD_SANDBOXED, bb->flags))
+                STATS_INC(coarse_prevent_selfmod); 
+            else if (TEST(FRAG_HAS_TRANSLATION_INFO, bb->flags))
+                STATS_INC(coarse_prevent_translation); 
+            else if (IF_WINDOWS_ELSE_0(TEST(LINK_CALLBACK_RETURN, bb->exit_type)))
+                STATS_INC(coarse_prevent_cbret); 
+            else if (TESTANY(LINK_NI_SYSCALL_ALL, bb->exit_type))
+                STATS_INC(coarse_prevent_syscall); 
+            else
+                ASSERT_NOT_REACHED();
+        });
+        bb->flags &= ~FRAG_COARSE_GRAIN;
+    }
+    ASSERT(!TEST(FRAG_COARSE_GRAIN, bb->flags) || !TEST(FRAG_HAS_DIRECT_CTI, bb->flags));
+
+    /* now that we know whether shared, ensure we have the right ibl routine */
+    if (!TEST(FRAG_SHARED, bb->flags) && TEST(LINK_INDIRECT, bb->exit_type)) {
+        ASSERT(bb->exit_target == get_ibl_routine(dcontext, IBL_LINKED, DEFAULT_IBL_BB(),
+                                                  ibl_branch_type));
+        bb->exit_target = get_ibl_routine(dcontext, IBL_LINKED, IBL_BB_PRIVATE,
+                                          ibl_branch_type);
+    }
+
+    if (bb->mangle_ilist &&
+        (bb->instr == NULL || !instr_opcode_valid(bb->instr) ||
+         !instr_is_ubr(bb->instr) || !instr_ok_to_mangle(bb->instr))) {
+        instr_t *exit_instr = INSTR_CREATE_jmp(dcontext, opnd_create_pc(bb->exit_target));
+        if (bb->record_translation) {
+            app_pc translation = NULL;
+            if (bb->instr == NULL) {
+                /* we removed (or mangle will remove) the last instruction
+                 * for special handling (invalid/syscall/int 2b) or there were
+                 * no instructions added (i.e. check_stopping_point in which 
+                 * case instr_start == cur_pc), use last instruction's start 
+                 * address for the translation */
+                translation = bb->instr_start;
+            } else if (instr_is_cti(bb->instr)) {
+                /* last instruction is a cti, consider the exit jmp part of 
+                 * the mangling of the cti (since we might not know the target 
+                 * if, for ex., its indirect) */
+                translation = instr_get_translation(bb->instr);
+            } else {
+                /* target is the instr after the last instr in the list */
+                translation = bb->cur_pc;
+                ASSERT(bb->cur_pc == bb->exit_target);
+            }
+            ASSERT(translation != NULL);
+            instr_set_translation(exit_instr, translation);
+        }
+        /* PR 214962: we need this jmp to be marked as "our mangling" so that
+         * we won't relocate a thread there and re-do a ret pop or call push
+         */
+        instr_set_our_mangling(exit_instr, true);
+        /* here we need to set exit_type */
+        LOG(THREAD, LOG_EMIT, 3, "exit_branch_type=0x%x bb->exit_target="PFX"\n",
+            bb->exit_type, bb->exit_target);
+        instr_exit_branch_set_type(exit_instr, bb->exit_type);
+
+        instrlist_append(bb->ilist, exit_instr);
+    }
+
+    /* set flags */
+#ifdef DGC_DIAGNOSTICS
+    /* no traces in dyngen code, that would mess up our exit tracking */
+    if (TEST(FRAG_DYNGEN, bb->flags))
+        bb->flags |= FRAG_CANNOT_BE_TRACE;
+#endif
+    if (!INTERNAL_OPTION(unsafe_ignore_eflags_prefix)
+        IF_X64(|| !INTERNAL_OPTION(unsafe_ignore_eflags_trace))) {
+        bb->flags |= instr_eflags_to_fragment_eflags(bb->eflags);
+        if (TEST(FRAG_WRITES_EFLAGS_OF, bb->flags)) {
+            LOG(THREAD, LOG_INTERP, 4, "fragment writes OF prior to reading it!\n");
+            STATS_INC(bbs_eflags_writes_of);
+        } else if (TEST(FRAG_WRITES_EFLAGS_6, bb->flags)) {
+            ASSERT(TEST(FRAG_WRITES_EFLAGS_OF, bb->flags));
+            LOG(THREAD, LOG_INTERP, 4,
+                "fragment writes all 6 flags prior to reading any\n");
+            STATS_INC(bbs_eflags_writes_6);
+        } else {
+            DOSTATS({
+                if (bb->eflags == EFLAGS_READ_6) {
+                    /* Reads a flag before writing any.  Won't get here if
+                     * reads one flag and later writes OF, or writes OF and
+                     * later reads one flag before writing that flag.
+                     */
+                    STATS_INC(bbs_eflags_reads);
+                } else {
+                    STATS_INC(bbs_eflags_writes_none);
+                    if (TEST(LINK_INDIRECT, bb->exit_type))
+                        STATS_INC(bbs_eflags_writes_none_ind);
+                }
+            });
+        }
+    }
+#ifdef RETURN_STACK
+    if (bb->exit_target == return_lookup_routine(dcontext))
+        bb->flags |= FRAG_ENDS_WITH_RETURN;
+#endif
+
+    /* can only have proactive translation info if flag was set from the beginning */
+    if (TEST(FRAG_HAS_TRANSLATION_INFO, bb->flags) &&
+        (!bb->record_translation || !bb->full_decode))
+        bb->flags &= ~FRAG_HAS_TRANSLATION_INFO;
+
+    /* if for_cache, caller must clear once done emitting (emitting can deref
+     * app memory so we wait until all done)
+     */
+    if (!bb_build_nested && !bb->for_cache && my_dcontext != NULL) {
+        ASSERT(my_dcontext->bb_build_info == (void *) bb);
+        my_dcontext->bb_build_info = NULL;
+    }
+    bb->instr = NULL;
+
+    /* mangle the instruction list */
+
+    if (!bb->mangle_ilist) {
+        /* do not mangle!
+         * caller must use full_decode to find invalid instrs and avoid
+         * a discrepancy w/ for_cache case that aborts b/c of selfmod sandbox
+         * returning false (in code below)
+         */
+        return;
+    }
+
+    if (!mangle_bb_ilist(dcontext, bb)) {
+        /* have to rebuild bb w/ new bb flags set by mangle_bb_ilist */
+        build_bb_ilist(dcontext, bb);
+        return;
+    }
+}
+
+/* Call when about to throw exception or other drastic action in the
+ * middle of bb building, in order to free resources
+ */
+void
+bb_build_abort(dcontext_t *dcontext, bool clean_vmarea)
+{
+    ASSERT(dcontext->bb_build_info != NULL); /* caller should check */
+    if (dcontext->bb_build_info != NULL) {
+        build_bb_t *bb = (build_bb_t *) dcontext->bb_build_info;
+        /* free instr memory */
+        if (bb->instr != NULL && bb->ilist != NULL &&
+            instrlist_last(bb->ilist) != bb->instr)
+            instr_destroy(dcontext, bb->instr); /* not added to bb->ilist yet */
+        DODEBUG({ bb->instr = NULL; });
+        if (bb->ilist != NULL) {
+            instrlist_clear_and_destroy(dcontext, bb->ilist);
+            DODEBUG({ bb->ilist = NULL; });
+        }
+        if (clean_vmarea) {
+            /* Free the vmlist and any locks held (we could have been in
+             * the middle of check_thread_vm_area and had a decode fault
+             * during code origins checking!)
+             */
+            check_thread_vm_area_abort(dcontext, &bb->vmlist, bb->flags);
+        } /* else we were presumably called from vmarea so caller does cleanup */
+        /* Assumption: bb building lock is held iff bb->for_cache,
+         * and on a nested app bb build where !bb->for_cache we do keep the
+         * original bb info in dcontext (see build_bb_ilist()).
+         */
+        if (bb->has_bb_building_lock) {
+            ASSERT_OWN_MUTEX(USE_BB_BUILDING_LOCK(), &bb_building_lock);
+            SHARED_BB_MUTEX(unlock);
+            KSTOP_REWIND(bb_building);
+        } else
+            ASSERT_DO_NOT_OWN_MUTEX(USE_BB_BUILDING_LOCK(), &bb_building_lock);
+        dcontext->bb_build_info = NULL;
+    }
+}
+
+/* returns false if need to rebuild bb: in that case this routine will
+ * set the bb flags needed to ensure successful mangling 2nd time around
+ */
+static bool
+mangle_bb_ilist(dcontext_t *dcontext, build_bb_t *bb)
+{
+    if (TEST(FRAG_SELFMOD_SANDBOXED, bb->flags)) {
+        /* sandbox requires that bb have no direct cti followings!
+         * check_thread_vm_area should have ensured this for us
+         */
+        ASSERT(!TEST(FRAG_HAS_DIRECT_CTI, bb->flags));
+        LOG(THREAD, LOG_INTERP, 2,
+            "fragment overlaps selfmod area, inserting sandboxing\n");
+        /* only reason can't be trace is don't have mechanism set up
+         * to store app code for each trace bb and update sandbox code
+         * to point there
+         */
+        bb->flags |= FRAG_CANNOT_BE_TRACE;
+        if (!insert_selfmod_sandbox(dcontext, bb->ilist, bb->flags, bb->start_pc,
+                                    bb->cur_pc, bb->record_translation,
+                                    bb->for_cache)) {
+            /* have to rebuild bb using full decode -- it has invalid instrs
+             * in middle, which we don't want to deal w/ for sandboxing!
+             */
+            ASSERT(!bb->full_decode); /* else, how did we get here??? */
+            LOG(THREAD, LOG_INTERP, 2,
+                "*** must rebuild bb to avoid invalid instr in middle ***\n");
+            STATS_INC(num_bb_end_early);
+            instrlist_clear_and_destroy(dcontext, bb->ilist);
+            if (bb->vmlist != NULL) {
+                vm_area_destroy_list(dcontext, bb->vmlist);
+                bb->vmlist = NULL;
+            }
+            bb->flags = FRAG_SELFMOD_SANDBOXED; /* lose all other flags */
+            bb->full_decode = true; /* full decode this time! */
+            bb->follow_direct = false; 
+            /* overlap info will be reset by check_new_page_start */
+            return false;
+        }
+        STATS_INC(num_sandboxed_fragments);
+    }
+
+    DOLOG(4, LOG_INTERP, {
+        LOG(THREAD, LOG_INTERP, 4, "bb ilist before mangling:\n");
+        instrlist_disassemble(dcontext, bb->start_pc, bb->ilist, THREAD);
+    });
+    mangle(dcontext, bb->ilist, bb->flags, true, bb->record_translation);
+    DOLOG(4, LOG_INTERP, {
+        LOG(THREAD, LOG_INTERP, 4, "bb ilist after mangling:\n");
+        instrlist_disassemble(dcontext, bb->start_pc, bb->ilist, THREAD);
+    });
+    return true;
+}
+
+/* Interprets the application's instructions until the end of a basic
+ * block is found, following all the rules that build_bb_ilist follows
+ * with regard to terminating the block.  Does no mangling or anything of
+ * the app code, though -- this routine is intended only for building the
+ * original code!
+ * Caller is responsible for freeing the list and its instrs!
+ * If outf != INVALID_FILE, does full disassembly with comments to outf.
+ */
+instrlist_t *
+build_app_bb_ilist(dcontext_t *dcontext, byte *start_pc, file_t outf)
+{
+    build_bb_t bb;
+    init_build_bb(&bb, start_pc, false/*not interp*/, false/*not for cache*/,
+                  false/*do not mangle*/, false/*no translation*/, outf,
+                  0/*no pre flags*/, NULL/*no overlap*/);
+    build_bb_ilist(dcontext, &bb);
+    return bb.ilist;
+}
+
+#ifdef CLIENT_INTERFACE
+/* Client routine to decode instructions at an arbitrary app address,
+ * following all the rules that DynamoRIO follows internally for
+ * terminating basic blocks.  Note that DynamoRIO does not validate
+ * that start_pc is actually the first instruction of a basic block.
+ * \note Caller is reponsible for freeing the list and its instrs!
+ */
+instrlist_t *
+decode_as_bb(void *drcontext, byte *start_pc)
+{
+    build_bb_t bb;
+
+    /* Case 10009: When we hook ntdll functions, we hide the jump to
+     * the interception buffer from the client BB callback.  If the
+     * client asks to decode that address here, we need to decode the
+     * instructions in the interception buffer instead so that we
+     * again hide our hooking.
+     * We will have the jmp from the buffer back to after the hooked
+     * app code visible to the client (just like it is for the
+     * real bb built there, so at least we're consistent).
+     */
+# ifdef WINDOWS
+    byte *real_pc;
+    if (is_intercepted_app_pc((app_pc)start_pc, &real_pc))
+        start_pc = real_pc;
+# endif
+
+    init_build_bb(&bb, start_pc, false /*not interp*/, 
+                  false /*not for cache*/, false /*do not mangle*/,
+                  true /* translation; xref case 10070 where this
+                        * currently turns on full decode; today we
+                        * provide no way to turn that off, as IR
+                        * expansion routines are not exported (PR 200409). */,
+                  INVALID_FILE, 0 /*no pre flags*/, NULL /*no overlap*/);
+    build_bb_ilist((dcontext_t *)drcontext, &bb);
+    return bb.ilist;
+}
+
+/* Client routine to decode a trace.  We return the instructions in
+ * the original app code, i.e., no client modifications.
+ */
+instrlist_t *
+decode_trace(void *drcontext, void *tag)
+{
+    dcontext_t *dcontext = (dcontext_t *)drcontext;
+    fragment_t *frag = fragment_lookup(dcontext, tag);
+    /* We don't support asking about other threads, for synch purposes
+     * (see recreate_fragment_ilist() synch notes)
+     */
+    if (get_thread_private_dcontext() != dcontext)
+        return NULL;
+
+    if (frag != NULL && TEST(FRAG_IS_TRACE, frag->flags)) {
+        instrlist_t *ilist;
+        bool alloc_res;
+        /* Support being called from bb/trace hook (couldbelinking) or
+         * from cache clean call (nolinking).  We disallow asking about
+         * another thread's private traces.
+         */
+        if (!is_couldbelinking(dcontext))
+            mutex_lock(&thread_initexit_lock);
+        ilist = recreate_fragment_ilist(dcontext, NULL, &frag, &alloc_res,
+                                        false/*no mangling*/
+                                        _IF_CLIENT(false/*do not re-call client*/));
+        ASSERT(!alloc_res);
+        if (!is_couldbelinking(dcontext))
+            mutex_unlock(&thread_initexit_lock);
+        
+        return ilist;
+    }
+
+    return NULL;
+}
+#endif
+
+app_pc
+find_app_bb_end(dcontext_t *dcontext, byte *start_pc, uint flags)
+{
+    build_bb_t bb;
+    init_build_bb(&bb, start_pc, false/*not interp*/, false/*not for cache*/,
+                  false/*do not mangle*/, false/*no translation*/, INVALID_FILE,
+                  flags, NULL/*no overlap*/);
+    build_bb_ilist(dcontext, &bb);
+    instrlist_clear_and_destroy(dcontext, bb.ilist);
+    return bb.end_pc;
+}
+
+bool
+app_bb_overlaps(dcontext_t *dcontext, byte *start_pc, uint flags,
+                byte *region_start, byte *region_end, overlap_info_t *info_res)
+{
+    build_bb_t bb;
+    overlap_info_t info;
+    info.region_start = region_start;
+    info.region_end = region_end;
+    init_build_bb(&bb, start_pc, false/*not interp*/, false/*not for cache*/,
+                  false/*do not mangle*/, false/*no translation*/, INVALID_FILE,
+                  flags, &info);
+    build_bb_ilist(dcontext, &bb);
+    instrlist_clear_and_destroy(dcontext, bb.ilist);
+    info.bb_end = bb.end_pc;
+    if (info_res != NULL)
+        *info_res = info;
+    return info.overlap;
+}
+
+/* pass in name if you already have it, else this routine looks it up from modbase */
+bool
+on_native_exec_list(app_pc modbase, const char *name)
+{
+    const char *modname;
+    bool modlock = false;
+    bool onlist = false;
+
+    if (!DYNAMO_OPTION(native_exec))
+        return false;
+
+    if (name == NULL) {
+        /* acquire name, safely */
+        if (modbase == NULL)
+            return false;
+        os_get_module_info_lock();
+        modlock = true;
+        os_get_module_name(modbase, &modname);
+        if (modname == NULL) {
+            os_get_module_info_unlock();
+            return false;
+        }
+    } else
+        modname = name;
+
+    if (!IS_STRING_OPTION_EMPTY(native_exec_default_list)) {
+        string_option_read_lock();
+        LOG(THREAD_GET, LOG_INTERP|LOG_VMAREAS, 4,
+            "on_native_exec_list: module %s vs default list %s\n",
+            modname==NULL?"null":modname, dynamo_options.native_exec_default_list);
+        onlist = check_filter(dynamo_options.native_exec_default_list, modname);
+        string_option_read_unlock();
+    }
+    if (!onlist &&
+        !IS_STRING_OPTION_EMPTY(native_exec_list)) {
+        string_option_read_lock();
+        LOG(THREAD_GET, LOG_INTERP|LOG_VMAREAS, 4,
+            "on_native_exec_list: module %s vs append list %s\n",
+            modname==NULL?"null":modname, dynamo_options.native_exec_list);
+        onlist = check_filter(dynamo_options.native_exec_list, modname);
+        string_option_read_unlock();
+    }
+    if (modlock)
+        os_get_module_info_unlock();
+    return onlist;
+}
+
+#ifdef DEBUG
+static void
+report_native_module(dcontext_t *dcontext, app_pc modpc)
+{
+    char name[MAX_MODNAME_INTERNAL];
+    const char *modname = name;
+    if (os_get_module_name_buf(modpc, name, BUFFER_SIZE_ELEMENTS(name)) == 0) {
+        /* for native_exec_callcall we do end up putting DGC on native_exec_list */
+        ASSERT(DYNAMO_OPTION(native_exec_callcall));
+        modname = "<DGC>";
+    }
+    LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+        "module %s is on native list, executing natively\n", modname);
+    STATS_INC(num_native_module_entrances);
+    SYSLOG_INTERNAL_WARNING_ONCE("module %s set up for native execution", modname);
+}
+#endif
+
+/* WARNING: breaks all kinds of rules, like ret addr transparency and
+ * assuming app stack and not doing calls out of the cache and not having
+ * control during dll loads, etc...
+ */
+static void
+build_native_exec_bb(dcontext_t *dcontext, build_bb_t *bb)
+{
+    instr_t *in;
+#ifdef X64
+    bool reachable;
+#endif
+    DEBUG_DECLARE(bool ok;)
+    /* if we ever protect from simultaneous thread attacks then this will
+     * be a hole -- for now should work, all protected while native until
+     * another thread goes into DR
+     */
+    /* Create a bb that changes the return address on the app stack such that we
+     * will take control when coming back, and then goes native.  
+     * N.B.: we ASSUME we reached this moduled via a call --
+     * build_basic_block_fragment needs to make sure, since we can't verify here
+     * w/o trying to decode backward from retaddr, and if we're wrong we'll
+     * clobber the stack and never regain control!
+     * We also assume this bb is never reached later through a non-call.
+     */
+    ASSERT(bb->initialized);
+    ASSERT(bb->app_interp);
+    ASSERT(!bb->record_translation);
+    ASSERT(bb->start_pc != NULL);
+    /* vmlist must start out empty (or N/A) */
+    ASSERT(bb->vmlist == NULL);
+    if (TEST(FRAG_HAS_TRANSLATION_INFO, bb->flags))
+        bb->flags &= ~FRAG_HAS_TRANSLATION_INFO;
+    bb->native_exec = true;
+
+    BBPRINT(bb, IF_DGCDIAG_ELSE(1, 2), "build_native_exec_bb @"PFX"\n", bb->start_pc);
+    DOLOG(2, LOG_INTERP, {
+        dump_mcontext(get_mcontext(dcontext), THREAD, DUMP_NOT_XML); });
+    check_new_page_start(dcontext, bb);
+    /* create instrlist after check_new_page_start to avoid memory leak
+     * on unreadable memory
+     * WARNING: do not add any app instructions to this ilist!
+     * If you do you must enable selfmod below.
+     */
+    bb->ilist = instrlist_create(dcontext);
+    /* FIXME PR 303413: we won't properly translate a fault in our app
+     * stack references here.  We mark as our own mangling so we'll at
+     * least return failure from our translate routine.
+     */
+    instrlist_set_our_mangling(bb->ilist, true);
+    /* To regain control we put our interception routine as the retaddr,
+     * assuming of course no transparency problems like longjmp or retaddr examination.
+     * We need to know where to go when we return -- but this can be stdcall,
+     * where it's tough to just push our retaddr after real one as earlier args will be
+     * "cleaned up" and beyond TOS and we don't know how many args there were!
+     * We use two special fields to store the original return address as well as its
+     * stack location.  We assume that we do not re-takeover on exceptions or
+     * APCs (callbacks will be ok) -- if we ever decide to we'll have to either
+     * use the dcontext stack just like for callbacks or stack
+     * native_exec_ret{val,loc} in some other manner.
+     */
+    append_shared_get_dcontext(dcontext, bb->ilist, true/*save xdi*/);
+    instrlist_append(bb->ilist, instr_create_save_to_dc_via_reg
+                     (dcontext, REG_NULL/*default*/, REG_XAX, XAX_OFFSET));
+    instrlist_append(bb->ilist, INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XAX),
+                                                    OPND_CREATE_MEMPTR(REG_XSP, 0)));
+    instrlist_append(bb->ilist, instr_create_save_to_dc_via_reg
+                     (dcontext, REG_NULL/*default*/, REG_XAX,
+                      NATIVE_EXEC_RETVAL_OFFSET));
+    /* we don't count on mcontext being preserved (native syscalls will clobber it),
+     * and when we re-takeover we need the new app state anyway, so we have to use
+     * a special field to store where we clobbered the app retaddr in order to restore
+     * it on a detach
+     */
+    instrlist_append(bb->ilist, instr_create_save_to_dc_via_reg
+                     (dcontext, REG_NULL/*default*/, REG_XSP,
+                      NATIVE_EXEC_RETLOC_OFFSET));
+#ifdef X64
+    /* must go through a register */
+    instrlist_append(bb->ilist, INSTR_CREATE_mov_imm
+                     (dcontext, opnd_create_reg(REG_XAX),
+                      OPND_CREATE_INTPTR((ptr_int_t)back_from_native)));
+    instrlist_append(bb->ilist, INSTR_CREATE_mov_st
+                     (dcontext, OPND_CREATE_MEMPTR(REG_XSP, 0),
+                      opnd_create_reg(REG_XAX)));
+    reachable = rel32_reachable_from_heap(bb->start_pc);
+    if (!reachable) {
+        /* best to store the target at the end of the bb, to keep it readonly,
+         * but that requires a post-pass to patch its value: since native_exec
+         * is already hacky we just go through TLS and ignore multi-thread selfmod.
+         */
+        instrlist_append(bb->ilist, INSTR_CREATE_mov_imm
+                         (dcontext, opnd_create_reg(REG_XAX),
+                          OPND_CREATE_INTPTR((ptr_int_t)bb->start_pc)));
+        instrlist_append(bb->ilist, INSTR_CREATE_mov_st
+                         (dcontext, opnd_create_tls_slot(os_tls_offset
+                                                         (MANGLE_XCX_SPILL_SLOT)),
+                          opnd_create_reg(REG_XAX)));
+    }
+#else
+    instrlist_append(bb->ilist, INSTR_CREATE_mov_st
+                     (dcontext, OPND_CREATE_MEM32(REG_XSP, 0),
+                      OPND_CREATE_INTPTR((ptr_int_t)back_from_native)));
+#endif
+    instrlist_append(bb->ilist, instr_create_restore_from_dc_via_reg
+                     (dcontext, REG_NULL/*default*/, REG_XAX, XAX_OFFSET));
+    append_shared_restore_dcontext_reg(dcontext, bb->ilist);
+    /* need some cleanup prior to native: turn off asynch, clobber trace, etc. */
+    /* FIXME: mark call as do-not-mangle */
+    dr_insert_clean_call(dcontext, bb->ilist, NULL, (void *) entering_native,
+                         false/*!fp*/, 0);
+    /* this is the jump to native code */
+#ifdef X64
+    if (reachable) {
+        instrlist_append(bb->ilist,
+                         INSTR_CREATE_jmp(dcontext, opnd_create_pc(bb->start_pc)));
+    } else {
+        instrlist_append(bb->ilist, INSTR_CREATE_jmp_ind
+                         (dcontext, opnd_create_tls_slot(os_tls_offset
+                                                         (MANGLE_XCX_SPILL_SLOT))));
+    }
+#else
+    instrlist_append(bb->ilist, INSTR_CREATE_jmp(dcontext, opnd_create_pc(bb->start_pc)));
+#endif
+
+    /* mark all as do-not-mangle, so selfmod, etc. will leave alone (in absence
+     * of selfmod only really needed for the jmp to native code)
+     */
+    for (in = instrlist_first(bb->ilist); in != NULL; in = instr_get_next(in))
+        instr_set_ok_to_mangle(in, false);
+
+    /* this is a jump for a dummy exit cti */
+    instrlist_append(bb->ilist, INSTR_CREATE_jmp(dcontext, opnd_create_pc(bb->start_pc)));
+
+    if (DYNAMO_OPTION(shared_bbs))
+        bb->flags |= FRAG_SHARED;
+
+    /* Can't be coarse-grain since has non-exit cti */
+    bb->flags &= ~FRAG_COARSE_GRAIN;
+    STATS_INC(coarse_prevent_native_exec); 
+
+    /* trace barrier */
+    bb->flags |= FRAG_MUST_END_TRACE;
+
+    /* We support mangling here, though currently we don't need it as we don't
+     * include any app code (although we mark this bb as belonging to the start
+     * pc, so we'll get flushed if this region does), and even if target is
+     * selfmod we're running it natively no matter how it modifies itself.  We
+     * only care that transition to target is via a call or call* so we can
+     * clobber the retaddr and regain control, and that no retaddr mangling
+     * happens while native before coming back out.  While the former does not
+     * depend on the target at all, unfortunately we cannot verify the latter.
+     */
+    if (TEST(FRAG_SELFMOD_SANDBOXED, bb->flags))
+        bb->flags &= ~FRAG_SELFMOD_SANDBOXED;
+    DEBUG_DECLARE(ok = ) mangle_bb_ilist(dcontext, bb);
+    ASSERT(ok);
+}
+
+static bool
+at_native_exec_gateway(dcontext_t *dcontext, app_pc start
+                       _IF_DEBUG(bool xfer_target))
+{
+    /* ASSUMPTION: transfer to another module will always be by indirect call
+     * or non-inlined direct call from a fragment that will not be flushed.
+     * For now we will only go native if last_exit was
+     * a call, a true call*, or a PLT-style call,jmp* (and we detect the latter only
+     * if the call is inlined, so if the jmp* table is in a DGC-marked region
+     * or if -no_inline_calls we will miss these: FIXME).
+     * FIXME: what if have PLT-style but no GOT indirection: call,jmp ?!?
+     *
+     * We try to identify funky call* constructions (like
+     * call*,...,jmp* in case 4269) by examining TOS to see whether it's a
+     * retaddr -- we do this if last_exit is a jmp* or is unknown (for the
+     * target_delete ibl path).
+     *
+     * FIXME: we will fail to identify a delay-loaded indirect xfer!
+     * Need to know dynamic link patchup code to look for.
+     *
+     * FIXME: we will fail to take over w/ non-call entrances to a dll, like
+     * NtContinue or direct jmp from DGC.
+     * we could try to take the top-of-stack value and see if it's a retaddr by
+     * decoding the prev instr to see if it's a call.  decode backwards may have
+     * issues, and if really want everything will have to do this on every bb,
+     * not just if lastexit is ind xfer.
+     *
+     * We count up easy-to-identify cases we've missed in the DOSTATS below.
+     */ 
+    bool native_exec_bb = false;
+    if (DYNAMO_OPTION(native_exec) &&
+        !vmvector_empty(native_exec_areas)) {
+        /* do we KNOW that we came from an indirect call? */
+        if (TESTANY(LINK_CALL|LINK_IND_JMP_PLT, dcontext->last_exit->flags) &&
+            /* only check direct calls if native_exec_dircalls is on */
+            (DYNAMO_OPTION(native_exec_dircalls) ||
+             LINKSTUB_INDIRECT(dcontext->last_exit->flags))) {
+            STATS_INC(num_native_entrance_checks);
+            /* we do the overlap check last since it's more costly */
+            if (vmvector_overlap(native_exec_areas, start, start+1)) {
+                native_exec_bb = true;
+                DOSTATS({
+                    if (TEST(LINK_CALL, dcontext->last_exit->flags)) {
+                        if (LINKSTUB_INDIRECT(dcontext->last_exit->flags))
+                            STATS_INC(num_native_module_entrances_indcall);
+                        else
+                            STATS_INC(num_native_module_entrances_call);
+                    } else
+                        STATS_INC(num_native_module_entrances_plt);
+                    
+                });
+            }
+        } 
+        /* can we GUESS that we came from an indirect call? */
+        else if (DYNAMO_OPTION(native_exec_guess_calls) &&
+                 (/* FIXME: require jmp* be in separate module? */
+                  (LINKSTUB_INDIRECT(dcontext->last_exit->flags) &&
+                   TEST(LINK_JMP, dcontext->last_exit->flags)) ||
+                  LINKSTUB_FAKE(dcontext->last_exit))) {
+            /* if unknown last exit, or last exit was jmp*, examine TOS and guess
+             * whether it's a retaddr
+             */
+            app_pc *tos = (app_pc *) get_mcontext(dcontext)->xsp;
+            STATS_INC(num_native_entrance_TOS_checks);
+            /* vector check cheaper than is_readable syscall, etc. so do it before them,
+             * but after last_exit checks above since overlap is more costly
+             */
+            if (vmvector_overlap(native_exec_areas, start, start+1) &&
+                is_readable_without_exception((app_pc)tos, 4)) {
+                enum { MAX_CALL_CONSIDER = 6 /* ignore prefixes */ };
+                app_pc retaddr = *tos;
+                LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+                    "at native_exec target: checking TOS "PFX" => "PFX" for retaddr\n",
+                    tos, retaddr);
+#ifdef RETURN_AFTER_CALL
+                if (DYNAMO_OPTION(ret_after_call)) {
+                    native_exec_bb = is_observed_call_site(dcontext, retaddr);
+                    LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+                        "native_exec: *TOS is %sa call site in ret-after-call table\n",
+                        native_exec_bb ? "" : "NOT ");
+                } else {
+#endif
+                    /* try to decode backward -- make sure readable for decoding */
+                    if (is_readable_without_exception(retaddr - MAX_CALL_CONSIDER,
+                                                      MAX_CALL_CONSIDER +
+                                                      MAX_INSTR_LENGTH)) {
+                        /* ind calls have variable length and form so we decode
+                         * each byte rather than searching for ff and guessing length
+                         */
+                        app_pc pc, next_pc;
+                        instr_t instr;
+                        instr_init(dcontext, &instr);
+                        for (pc = retaddr - MAX_CALL_CONSIDER; pc < retaddr; pc++) {
+                            LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 3,
+                                "native_exec: decoding @"PFX" looking for call\n", pc);
+                            instr_reset(dcontext, &instr);
+                            next_pc = decode_cti(dcontext, pc, &instr);
+                            STATS_INC(num_native_entrance_TOS_decodes);
+                            if (next_pc == retaddr && instr_is_call(&instr)) {
+                                native_exec_bb = true;
+                                LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+                                    "native_exec: found call @ pre-*TOS "PFX"\n", pc);
+                                break;
+                            }
+                        }
+                        instr_free(dcontext, &instr);
+                    }
+#ifdef RETURN_AFTER_CALL
+                }
+#endif
+                DOSTATS({
+                    if (native_exec_bb) {
+                        if (LINKSTUB_FAKE(dcontext->last_exit))
+                            STATS_INC(num_native_module_entrances_TOS_unknown);
+                        else
+                            STATS_INC(num_native_module_entrances_TOS_jmp);
+                    }
+                });
+            }
+        }
+        DOSTATS({
+            /* did we reach a native dll w/o going through an ind call caught above? */
+            if (!xfer_target /* else we'll re-check at the target itself */ &&
+                !native_exec_bb && vmvector_overlap(native_exec_areas, start, start+1)) {
+                LOG(THREAD, LOG_INTERP|LOG_VMAREAS, 2,
+                    "WARNING: pc "PFX" is on native list but reached bypassing gateway!\n",
+                    start);
+                STATS_INC(num_native_entrance_miss);
+                /* do-once since once get into dll past gateway may xfer
+                 * through a bunch of lastexit-null or indjmp to same dll
+                 */
+                ASSERT_CURIOSITY_ONCE(false && "inside native_exec dll");
+            }
+        });
+    }
+
+    return native_exec_bb;
+}
+
+/* Use when calling build_bb_ilist with for_cache = true.
+ * Must hold bb_building_lock.
+ */
+static inline void
+init_interp_build_bb(dcontext_t *dcontext, build_bb_t *bb, app_pc start,
+                     uint initial_flags
+                     _IF_CLIENT(bool for_trace)
+                     _IF_CLIENT(instrlist_t **unmangled_ilist))
+{
+    ASSERT_OWN_MUTEX(USE_BB_BUILDING_LOCK() && !TEST(FRAG_TEMP_PRIVATE, initial_flags),
+                     &bb_building_lock);
+    /* We need to set up for abort prior to native exec and other checks
+     * that can crash */
+    ASSERT(dcontext->bb_build_info == NULL);
+    /* This won't make us be nested b/c for bb.for_cache caller is supposed
+     * to set this up */
+    dcontext->bb_build_info = (void *) bb;
+
+    init_build_bb(bb, start, true/*real interp*/, true/*for cache*/, true/*mangle*/, 
+                  false /* translation: set below for clients */,
+                  INVALID_FILE, initial_flags |
+                  (INTERNAL_OPTION(store_translations) ?
+                   FRAG_HAS_TRANSLATION_INFO : 0), NULL/*no overlap*/);
+    if (!TEST(FRAG_TEMP_PRIVATE, initial_flags))
+        bb->has_bb_building_lock = true;
+#ifdef CLIENT_INTERFACE
+    /* We avoid races where there is no hook when we start building a
+     * bb (and hence we don't record translation or do full decode) yet
+     * a hook when we're ready to call one by storing whether there is a
+     * hook at translation/decode decision time: now.
+     */
+    /* PR 299808: even if no bb hook, for a trace hook we need to
+     * record translation and do full decode.  It's racy to check
+     * dr_trace_hook_exists() here so we rely on trace building having
+     * set unmangled_ilist.
+     */
+    if (dr_bb_hook_exists() || unmangled_ilist != NULL) {
+        bb->pass_to_client = true;
+        /* case 10009/214444: For client interface builds, store the translation.
+         * by default.  This ensures clients can get the correct app address
+         * of any instruction.  We also rely on this for allowing the client
+         * to return DR_EMIT_STORE_TRANSLATIONS and setting the
+         * FRAG_HAS_TRANSLATION_INFO flag after decoding the app code.
+         *
+         * FIXME: xref case 10070/214505.  Currently this means that all
+         * instructions are fully decoded for client interface builds.
+         */
+        bb->record_translation = true;
+        /* PR 200409: If a bb hook exists, we always do a full decode.
+         * Note that we currently do this anyway to get
+         * translation fields, but once we fix case 10070 it
+         * won't be that way.
+         * We do not let the client turn this off (the runtime
+         * option is not dynamic, and off by default anyway), as we
+         * do not export level-handling instr_t routines like *_expand
+         * for walking instrlists and instr_decode().
+         */
+        bb->full_decode = true;
+        /* PR 299808: we give client chance to re-add instrumentation */
+        bb->for_trace = for_trace;
+    }
+    /* we need to clone the ilist pre-mangling */
+    bb->unmangled_ilist = unmangled_ilist;
+#endif
+}
+
+static inline void
+exit_interp_build_bb(dcontext_t *dcontext, build_bb_t *bb)
+{
+    ASSERT(dcontext->bb_build_info == (void *) bb);
+    /* Caller's responsibility to clean up since bb.for_cache */
+    dcontext->bb_build_info = NULL;
+
+    /* free the instrlist_t elements */
+    instrlist_clear_and_destroy(dcontext, bb->ilist);
+}
+
+/* Interprets the application's instructions until the end of a basic
+ * block is found, and then creates a fragment for the basic block.
+ * DOES NOT look in the hashtable to see if such a fragment already exists!
+ */
+fragment_t *
+build_basic_block_fragment(dcontext_t *dcontext, app_pc start, uint initial_flags,
+                           bool link, bool visible _IF_CLIENT(bool for_trace)
+                           _IF_CLIENT(instrlist_t **unmangled_ilist))
+{
+    fragment_t *f;
+    build_bb_t bb;
+    where_am_i_t wherewasi = dcontext->whereami;
+#ifdef WINDOWS
+    bool image_entry;
+#endif
+    KSTART(bb_building);
+    dcontext->whereami = WHERE_INTERP;
+
+    /* Neither thin_client nor hotp_only should be building any bbs. */
+    ASSERT(!RUNNING_WITHOUT_CODE_CACHE());
+
+#ifdef WINDOWS
+    /* ASSUMPTION: image entry is reached via indirect transfer and
+     * so will be the start of a bb
+     * FIXME: ensure compiler inlines this so we don't make the call
+     * if we've already reached the entry point
+     */
+    image_entry = check_for_image_entry(start);
+#endif
+
+    init_interp_build_bb(dcontext, &bb, start, initial_flags
+                         _IF_CLIENT(for_trace) _IF_CLIENT(unmangled_ilist));
+    if (at_native_exec_gateway(dcontext, start _IF_DEBUG(false/*not xfer tgt*/))) {
+        DODEBUG({ report_native_module(dcontext, bb.start_pc); });
+#ifdef CLIENT_INTERFACE
+        /* PR 232617 - build_native_exec_bb doesn't support setting translation
+         * info, but it also doesn't pass the built bb to the client (it
+         * contains no app code) so we don't need it. */
+        bb.record_translation = false;
+#endif
+        build_native_exec_bb(dcontext, &bb);
+    } else {
+        build_bb_ilist(dcontext, &bb);
+        if (bb.native_exec) {
+            /* change bb to be a native_exec gateway */
+            LOG(THREAD, LOG_INTERP, 2, "replacing built bb with native_exec bb\n");
+            instrlist_clear_and_destroy(dcontext, bb.ilist);
+            vm_area_destroy_list(dcontext, bb.vmlist);
+            dcontext->bb_build_info = NULL;
+            init_interp_build_bb(dcontext, &bb, start, initial_flags
+                                 _IF_CLIENT(for_trace) _IF_CLIENT(unmangled_ilist));
+#ifdef CLIENT_INTERFACE
+            /* PR 232617 - build_native_exec_bb doesn't support setting
+             * translation info, but it also doesn't pass the built bb to the
+             * client (it contains no app code) so we don't need it. */
+            bb.record_translation = false;
+#endif
+            build_native_exec_bb(dcontext, &bb);
+        }
+    }
+#ifdef WINDOWS
+    /* case 9652: we do not want to persist the image entry point, so we keep
+     * it fine-grained
+     */
+    if (image_entry)
+        bb.flags &= ~FRAG_COARSE_GRAIN;
+#endif
+
+    /* emit fragment into fcache */
+    KSTART(bb_emit);
+    f = emit_fragment_ex(dcontext, start, bb.ilist, bb.flags, bb.vmlist, link, visible);
+    KSTOP(bb_emit);
+
+#ifdef CUSTOM_TRACES_RET_REMOVAL
+    f->num_calls = dcontext->num_calls;
+    f->num_rets = dcontext->num_rets;
+#endif
+
+#ifdef DGC_DIAGNOSTICS
+    if ((f->flags & FRAG_DYNGEN)) {
+        LOG(THREAD, LOG_INTERP, 1, "new bb is DGC:\n");
+        DOLOG(1, LOG_INTERP, { disassemble_app_bb(dcontext, start, THREAD); });
+        DOLOG(3, LOG_INTERP, { disassemble_fragment(dcontext, f, false); });
+    }
+#endif
+    DOLOG(2, LOG_INTERP, { disassemble_fragment(dcontext, f, stats->loglevel <= 3); });
+    DOLOG(4, LOG_INTERP, {
+        if (TEST(FRAG_SELFMOD_SANDBOXED, f->flags)) {
+            LOG(THREAD, LOG_INTERP, 4,
+                "\nXXXX sandboxed fragment!  original code:\n");
+            disassemble_app_bb(dcontext, f->tag, THREAD);
+            LOG(THREAD, LOG_INTERP, 4, "code cache code:\n");
+            disassemble_fragment(dcontext, f, false);
+        }
+    });
+#ifdef INTERNAL
+    if (INTERNAL_OPTION(bbdump_tags)) {
+        disassemble_fragment_header(dcontext, f, bbdump_file);
+    }
+#endif
+
+#ifdef INTERNAL
+    DODEBUG({
+        if (INTERNAL_OPTION(stress_recreate_pc)) {
+            /* verify recreation */
+            stress_test_recreate(dcontext, f, bb.ilist);
+        }
+    });
+#endif
+
+    exit_interp_build_bb(dcontext, &bb);
+
+    dcontext->whereami = wherewasi;
+    KSTOP(bb_building);
+    return f;
+}
+
+/* Builds an instrlist_t as though building a bb from pc.
+ * Use recreate_fragment_ilist() for building an instrlist_t for a fragment.
+ * If check_vm_area is false, Does NOT call check_thread_vm_area()!
+ *   Make sure you know it will terminate at the right spot.  It does
+ *   check selfmod and native_exec for elision, but otherwise will
+ *   follow ubrs to the limit.  Currently used for
+ *   record_translation_info() (case 3559).
+ */
+instrlist_t *
+recreate_bb_ilist(dcontext_t *dcontext, byte *pc, uint flags, uint *res_flags,
+                  uint *res_exit_type, bool check_vm_area, bool mangle
+                  _IF_CLIENT(bool call_client) _IF_CLIENT(bool for_trace))
+{
+    build_bb_t bb;
+
+    if (!is_readable_without_exception(pc, 4/* don't know full range -- just do simple check now */)) {
+        LOG(THREAD, LOG_INTERP, 3,
+            "recreate_bb_ilist: cannot read memory at "PFX"\n", pc);
+        return NULL;
+    }
+
+    LOG(THREAD, LOG_INTERP, 3, "\nbuilding bb instrlist now *********************\n");
+    init_build_bb(&bb, pc, false/*not interp*/, false/*not for cache*/,
+                  mangle, true/*translation*/, INVALID_FILE,
+                  flags, NULL/*no overlap*/);
+    bb.check_vm_area = check_vm_area;
+#ifdef CLIENT_INTERFACE
+    /* PR 214962: we call bb hook again, unless the client told us
+     * DR_EMIT_STORE_TRANSLATIONS, in which case we shouldn't come here,
+     * except for traces (see below):
+     */
+    bb.pass_to_client = DYNAMO_OPTION(code_api) && call_client;
+    /* PR 299808: we call bb hook again when translating a trace that
+     * didn't have DR_EMIT_STORE_TRANSLATIONS on itself (or on any
+     * for_trace bb if there was no trace hook).
+     */
+    bb.for_trace = for_trace;
+    /* instrument_basic_block, called by build_bb_ilist, verifies that all
+     * non-meta instrs have translation fields */
+#endif
+
+    build_bb_ilist(dcontext, &bb);
+
+    LOG(THREAD, LOG_INTERP, 3, "\ndone building bb instrlist *********************\n\n");
+    if (res_flags != NULL)
+        *res_flags = bb.flags;
+    if (res_exit_type != NULL)
+        *res_exit_type = bb.exit_type;
+    return bb.ilist;
+}
+
+/* Re-creates an ilist of the fragment that currently contains the
+ * passed-in code cache pc, also returns the fragment.
+ *
+ * Exactly one of pc and (f_res or *f_res) must be NULL:
+ * If pc==NULL, assumes that *f_res is the fragment to use;
+ * else, looks up the fragment, allocating it if necessary.
+ *   If f_res!=NULL, the fragment is returned and whether it was allocated
+ *   is returned in the alloc_res param.
+ *   If f_res==NULL, if the fragment was allocated it is freed here.
+ * 
+ * NOTE : does not add prefix instructions to the created ilist, if we change
+ * this to add them be sure to check recreate_app_* for compatibility (for ex.
+ * adding them and setting their translation to pc would break current
+ * implementation, also setting translation to NULL would trigger an assert)
+ *
+ * Returns NULL if unable to recreate the fragment ilist (fragment not found
+ * or fragment is pending deletion and app memory might have changed).
+ * In that case f_res is still pointed at the fragment if it was found, and
+ * alloc is valid.
+ *
+ * For proper synchronization : 
+ * If caller is the dcontext's owner then needs to be couldbelinking, otherwise
+ * the dcontext's owner should be suspended and the callers should own the
+ * thread_initexit_lock
+ */
+instrlist_t *
+recreate_fragment_ilist(dcontext_t *dcontext, byte *pc,
+                        /*IN/OUT*/fragment_t **f_res, /*OUT*/bool *alloc_res,
+                        bool mangle _IF_CLIENT(bool call_client))
+{
+    fragment_t *f;
+    uint flags = 0;
+    instrlist_t *ilist;
+    bool alloc = false;
+    monitor_data_t md = {0,};
+    IF_X64(bool old_mode = true;)
+    /* check synchronization, we need to make sure no one flushes the 
+     * fragment we just looked up while we are recreating it, if it's the
+     * caller's dcontext then just need to be couldbelinking, otherwise need
+     * the thread_initexit_lock since then we are looking up in someone else's 
+     * table (the dcontext's owning thread would also need to be suspended)
+     */
+    ASSERT((dcontext != GLOBAL_DCONTEXT &&
+            get_thread_id() == dcontext->owning_thread && 
+            is_couldbelinking(dcontext)) ||
+           (ASSERT_OWN_MUTEX(true, &thread_initexit_lock), true));
+    STATS_INC(num_recreated_fragments);
+    if (pc == NULL) {
+        ASSERT(f_res != NULL && *f_res != NULL);
+        f = *f_res;
+    } else {
+        /* Ensure callers don't give us both valid f and valid pc */
+        ASSERT(f_res == NULL || *f_res == NULL);
+        LOG(THREAD, LOG_INTERP, 3, "recreate_fragment_ilist: looking up pc "PFX"\n", pc);
+        f = fragment_pclookup_with_linkstubs(dcontext, pc, &alloc);
+        LOG(THREAD, LOG_INTERP, 3, "\tfound F%d\n", f == NULL? -1 : f->id);
+        if (f_res != NULL)
+            *f_res = f;
+        /* ref case 3559, others, we won't be able to reliably recreate if 
+         * target is pending flush, original memory might no longer be there or
+         * the memory might have changed.  caller should use the stored
+         * translation info instead.
+         */
+        if (f == NULL || TEST(FRAG_WAS_DELETED, f->flags)) {
+            ASSERT(f != NULL || !alloc); /* alloc shouldn't be set if no f */
+            ilist = NULL;
+            goto recreate_fragment_done;
+        }
+    }
+
+    /* Recreate in same mode as original fragment */
+    IF_X64(old_mode = set_x86_mode(dcontext, FRAG_IS_32(f->flags)));
+
+    if ((f->flags & FRAG_IS_TRACE) == 0) {
+        /* easy case: just a bb */
+        ilist = recreate_bb_ilist(dcontext, (byte *) f->tag,
+                                  0/*no pre flags*/, &flags, NULL,
+                                  true/*check vm area*/, mangle
+                                  _IF_CLIENT(call_client)
+                                  _IF_CLIENT(false/*not for_trace*/));
+        ASSERT(ilist != NULL);
+        if (ilist == NULL) /* a race */
+            goto recreate_fragment_done;
+        if (PAD_FRAGMENT_JMPS(f->flags))
+            nop_pad_ilist(dcontext, f, ilist, false /* set translation */);
+        goto recreate_fragment_done;
+    } else {
+        /* build trace up one bb at a time */
+        instrlist_t *bb;
+        byte *apc;
+        trace_only_t *t = TRACE_FIELDS(f);
+        uint i;
+        instr_t *last;
+        bool mangle_at_end = can_use_mangle_trace();
+
+        if (mangle_at_end) {
+            /* we need an md for mangle_trace */
+            md.trace_tag = f->tag;
+            /* be sure we ask for translation fields */
+            md.trace_flags = f->flags | FRAG_HAS_TRANSLATION_INFO;
+            md.num_blks = t->num_bbs;
+            md.blk_info = (trace_bb_build_t *)
+                HEAP_ARRAY_ALLOC(dcontext, trace_bb_build_t, md.num_blks, ACCT_TRACE,
+                                 true);
+        }
+
+        ilist = instrlist_create(dcontext);
+        STATS_INC(num_recreated_traces);
+        ASSERT(t->bbs != NULL);
+        for (i=0; i<t->num_bbs; i++) {
+            apc = (byte *) t->bbs[i].tag;
+            bb = recreate_bb_ilist(dcontext, apc, 0/*no pre flags*/,
+                                   &flags, &md.final_exit_flags,
+                                   true/*check vm area*/, !mangle_at_end
+                                   _IF_CLIENT(call_client)
+                                   _IF_CLIENT(true/*for_trace*/));
+            ASSERT(bb != NULL);
+            if (bb == NULL) {
+                instrlist_clear_and_destroy(dcontext, ilist);
+                ilist = NULL;
+                goto recreate_fragment_done;
+            }
+            if (mangle_at_end)
+                md.blk_info[i].info = t->bbs[i];
+            last = instrlist_last(bb);
+            ASSERT(last != NULL);
+#ifdef CLIENT_INTERFACE
+            if (mangle_at_end) {
+                md.blk_info[i].bounds.end_pc =
+                    instr_get_translation(last) + instr_length(dcontext, last);
+            }
+#endif
+
+            /* PR 299808: we need to duplicate what we did when we built the trace.
+             * While if there's no client trace hook we could mangle and fixup as we
+             * go, for simplicity we mangle at the end either way (in either case our
+             * code here is not exactly what we did when we made it anyway)
+             * PR 333597: we can't use mangle_trace if we have elision on.
+             */
+            if (mangle && !mangle_at_end) {
+                /* To duplicate the trace-building logic:
+                 * - call fixup_last_cti()
+                 * - retarget the ibl routine just like extend_trace() does 
+                 */
+                app_pc target = (last != NULL) ? opnd_get_pc(instr_get_target(last))
+                    : NULL; /* FIXME: is it always safe */
+                /* convert a basic block IBL, and retarget it to IBL_TRACE* */
+                if (target != NULL &&
+                    is_indirect_branch_lookup_routine(dcontext, target)) {
+                    target = get_alternate_ibl_routine(dcontext, target, f->flags);
+                    ASSERT(target != NULL);
+                    LOG(THREAD, LOG_MONITOR, 3,
+                        "recreate_fragment_ilist: replacing ibl_routine to target="
+                        PFX"\n", target);
+                    instr_set_target(last, opnd_create_pc(target));
+                }
+                if (DYNAMO_OPTION(pad_jmps) && 
+                    !INTERNAL_OPTION(pad_jmps_shift_bb)) {
+                    /* FIXME - hack, but pad_jmps_shift_bb will be on by 
+                     * default. Synchronize changes here with recreate_fragment_ilist.
+                     * This hack is protected by asserts in nop_pad_ilist() (that
+                     * we never add nops to a bb if -pad_jmps_shift_bb) and in
+                     * extend_trace_pad_bytes (that we only add bbs to traces). */
+                    /* FIXME - on linux the signal fence exit can trigger the
+                     * protective assert in nop_pad_ilist() */
+                    remove_nops_from_ilist(dcontext, bb _IF_DEBUG(true));
+                }
+                if (instrlist_last(ilist) != NULL) {
+                    fixup_last_cti(dcontext, ilist, (app_pc) apc, flags, f->flags, NULL,
+                                   NULL, true/* record translation */, NULL,
+                                   NULL, NULL);
+                }
+            }
+
+            instrlist_append(ilist, instrlist_first(bb));
+            instrlist_init(bb); /* to clear fields to make destroy happy */
+            instrlist_destroy(dcontext, bb);
+        }
+
+#ifdef CLIENT_INTERFACE
+        /* PR 214962: re-apply client changes, this time storing translation
+         * info for modified instrs
+         */
+        if (call_client) /* else it's decode_trace() who is not really recreating */
+            instrument_trace(dcontext, f->tag, ilist, true);
+        /* instrument_trace checks that all non-meta instrs have translation fields */
+#endif
+
+        if (mangle) {
+            if (mangle_at_end) {
+                if (!mangle_trace(dcontext, ilist, &md)) {
+                    instrlist_clear_and_destroy(dcontext, ilist);
+                    ilist = NULL;
+                    goto recreate_fragment_done;
+                }
+            } /* else we mangled one bb at a time up above */
+
+#ifdef INTERNAL
+            /* we only optimize traces */
+            if (dynamo_options.optimize) {
+                /* re-apply all optimizations to ilist
+                 * assumption: all optimizations are deterministic and stateless,
+                 * so we can exactly replicate their results
+                 */
+                LOG(THREAD_GET, LOG_INTERP, 2, 
+                    "\tre-applying optimizations to F%d\n", f->id);
+# ifdef SIDELINE
+                if (dynamo_options.sideline) {
+                    if (!TEST(FRAG_DO_NOT_SIDELINE, f->flags))
+                        optimize_trace(dcontext, f->tag, ilist);
+                    /* else, never optimized */
+                } else
+# endif
+                    optimize_trace(dcontext, f->tag, ilist);
+            }
+#endif
+
+            /* FIXME: case 4718 append_trace_speculate_last_ibl(true)
+             * should be called as well 
+             */
+            if (PAD_FRAGMENT_JMPS(f->flags))
+                nop_pad_ilist(dcontext, f, ilist, false /* set translation */);
+        }
+    }
+
+ recreate_fragment_done:
+    if (md.blk_info != NULL) {
+        HEAP_ARRAY_FREE(dcontext, md.blk_info, trace_bb_build_t, md.num_blks,
+                        ACCT_TRACE, true);
+    }
+    if (alloc_res != NULL)
+        *alloc_res = alloc;
+    if (f_res == NULL && alloc)
+        fragment_free(dcontext, f);
+    IF_X64(set_x86_mode(dcontext, old_mode));
+    return ilist;
+}
+
+
+/*** TRACE BUILDING ROUTINES *****************************************************/
+
+static void
+process_nops_for_trace(dcontext_t *dcontext, instrlist_t *ilist, uint flags
+                       _IF_DEBUG(bool recreating))
+{
+    if (PAD_FRAGMENT_JMPS(flags) && !INTERNAL_OPTION(pad_jmps_shift_bb)) {
+        /* FIXME - hack, but pad_jmps_shift_bb will be on by 
+         * default. Synchronize changes here with recreate_fragment_ilist.
+         * This hack is protected by asserts in nop_pad_ilist() (that
+         * we never add nops to a bb if -pad_jmps_shift_bb) and in
+         * extend_trace_pad_bytes (that we only add bbs to traces). */
+        /* FIXME - on linux the signal fence exit can trigger the
+         * protective assert in nop_pad_ilist() */
+        remove_nops_from_ilist(dcontext, ilist _IF_DEBUG(recreating));
+    }
+}
+
+#ifdef CUSTOM_EXIT_STUBS
+/*
+ * Builds custom exit stub instrlist for exit_cti, whose stub is l
+ * Assumes that intra-fragment cti's in the custom stub only target other
+ * instructions in the same stub, never in the body of the fragment or
+ * in other stubs.  FIXME: is this too restrictive?  If change this,
+ * change the comment in instr_set_exit_stub_code's declaration.
+ */
+static void
+regenerate_custom_exit_stub(dcontext_t *dcontext, instr_t *exit_cti, linkstub_t *l,
+                            fragment_t *f)
+{
+    /* need to decode and restore custom stub instrlist */
+    byte *cspc = EXIT_STUB_PC(dcontext, f, l);
+    byte *stop = EXIT_FIXED_STUB_PC(dcontext, f, l);
+    instr_t *in, *cti;
+    instrlist_t intra_ctis;
+    instrlist_t *cil = instrlist_create(dcontext);
+    cache_pc start_pc = FCACHE_ENTRY_PC(f);
+    ASSERT(DYNAMO_OPTION(indirect_stubs));
+
+    if (l->fixed_stub_offset == 0)
+        return; /* has no custom exit stub */
+
+    LOG(THREAD, LOG_INTERP, 3, "in regenerate_custom_exit_stub\n");
+
+    instrlist_init(&intra_ctis);
+    while (cspc < stop) {
+        in = instr_create(dcontext);
+        cspc = decode(dcontext, cspc, in);
+        ASSERT(cspc != NULL); /* our own code! */
+        if (instr_is_cti(in)) {
+            if (!instr_is_return(in) &&
+                opnd_is_near_pc(instr_get_target(in)) &&
+                (opnd_get_pc(instr_get_target(in)) < start_pc ||
+                 opnd_get_pc(instr_get_target(in)) > start_pc+f->size)) {
+                loginst(dcontext, 3, in, "\tcti has off-fragment target");
+                /* indicate that relative target must be
+                 * re-encoded, and that it is not an exit cti
+                 */
+                instr_set_ok_to_mangle(in, false);
+                instr_set_raw_bits_valid(in, false);
+            } else if (opnd_is_near_pc(instr_get_target(in))) {
+                /* intra-fragment target: we'll change its target operand
+                 * from pc to instr_t in second pass, so remember it here
+                 */
+                instr_t *clone = instr_clone(dcontext, in);
+                /* HACK: use note field! */
+                instr_set_note(clone, (void *) in);
+                instrlist_append(&intra_ctis, clone);
+            }
+        }
+        instrlist_append(cil, in);
+    }
+
+    /* must fix up intra-ilist cti's to have instr_t targets
+     * assumption: they only target other instrs in custom stub
+     * FIXME: allow targeting other instrs?
+     */
+    for (in = instrlist_first(cil); in != NULL; in = instr_get_next(in)) {
+        for (cti = instrlist_first(&intra_ctis); cti != NULL;
+             cti = instr_get_next(cti)) {
+            if (opnd_get_pc(instr_get_target(cti)) == instr_get_raw_bits(in)) {
+                /* cti targets this instr */
+                instr_t *real_cti = (instr_t *) instr_get_note(cti);
+                /* Do not preserve raw bits just in case instrlist changes
+                 * and the instr target moves (xref PR 333691)
+                 */
+                instr_set_target(real_cti, opnd_create_instr(in));
+                loginst(dcontext, 3, real_cti, "\tthis cti: ");
+                loginst(dcontext, 3, in, "\t  targets intra-stub instr");
+                break;
+            }
+        }
+    }
+    instrlist_clear(dcontext,&intra_ctis);
+
+    instr_set_exit_stub_code(exit_cti, cil);
+}
+#endif
+
+/* Combines instrlist_preinsert to ilist and the size calculation of the addition */
+static inline int
+tracelist_add(dcontext_t *dcontext, instrlist_t *ilist, instr_t *where, instr_t *inst)
+{
+    /* when we emit the trace we're going to call instr_length() on all instrs
+     * anyway, and we'll re-use any memory allocated here for an encoding
+     */
+    int size = instr_length(dcontext, inst);
+    instrlist_preinsert(ilist, where, inst);
+    return size;
+}
+
+/* Combines instrlist_postinsert to ilist and the size calculation of the addition */
+static inline int
+tracelist_add_after(dcontext_t *dcontext, instrlist_t *ilist, instr_t *where, instr_t *inst)
+{
+    /* when we emit the trace we're going to call instr_length() on all instrs
+     * anyway, and we'll re-use any memory allocated here for an encoding
+     */
+    int size = instr_length(dcontext, inst);
+    instrlist_postinsert(ilist, where, inst);
+    return size;
+}
+
+#ifdef HASHTABLE_STATISTICS
+/* increments a given counter - assuming XCX is dead */
+int
+insert_increment_stat_counter(dcontext_t *dcontext, instrlist_t *trace, instr_t *next,                 
+                              uint *counter_address)
+{
+    int added_size = 0;
+    /* incrementing a branch-type specific thread private counter */
+    opnd_t private_branchtype_counter = 
+        OPND_CREATE_ABSMEM(counter_address, OPSZ_4);
+
+    /* using LEA to avoid clobbering eflags in a simple load-increment-store */
+    /*>>>    movl    counter, %ecx */
+    /*>>>    lea     1(%ecx), %ecx    */
+    /*>>>    movl    %ecx, counter */
+    /* x64: the counter is still 32 bits */
+    added_size += tracelist_add(dcontext, trace, next,
+                                INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_ECX),
+                                                    private_branchtype_counter)
+                                );
+    added_size += tracelist_add(dcontext, trace, next,
+                                INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ECX),
+                                                 opnd_create_base_disp(REG_ECX, 
+                                                                       REG_NULL, 0, 1, OPSZ_lea)));
+    added_size += tracelist_add(dcontext, trace, next,
+                                INSTR_CREATE_mov_st(dcontext, 
+                                                    private_branchtype_counter,
+                                                    opnd_create_reg(REG_ECX)));
+    return added_size;
+}
+#endif /* HASHTABLE_STATISTICS */
+
+/* inserts proper instruction(s) to restore XCX spilled on indirect branch mangling
+ *    assumes target instrlist is a trace!
+ * returns size to be added to trace 
+ */
+static inline int
+insert_restore_spilled_xcx(dcontext_t *dcontext, instrlist_t *trace, instr_t *next)
+{    
+    int added_size = 0;
+
+    if (DYNAMO_OPTION(private_ib_in_tls)) {
+        added_size += tracelist_add(dcontext, trace, next, 
+            INSTR_CREATE_mov_ld(dcontext,
+                                opnd_create_reg(REG_XCX),
+                                opnd_create_tls_slot(os_tls_offset(MANGLE_XCX_SPILL_SLOT))));
+    } else {
+        /* We need to restore XCX from TLS for shared fragments, but from
+         * mcontext for private fragments, and all traces are private
+         */
+        added_size += tracelist_add(dcontext, trace, next,
+                                    instr_create_restore_from_dcontext
+                                    (dcontext, REG_XCX, XCX_OFFSET));
+    }
+
+    return added_size;
+}
+
+/* 32-bit only: inserts a comparison to speculative_tag with no side effect and
+ * if value is matched continue target is assumed to be immediately
+ * after targeter (which must be < 127 bytes away).
+ * returns size to be added to trace 
+ */
+static int
+insert_transparent_comparison(dcontext_t *dcontext, instrlist_t *trace,
+                              instr_t *targeter, /* exit CTI */
+                              app_pc speculative_tag)
+{
+    int added_size = 0;
+    instr_t *jecxz;
+    instr_t *continue_label = INSTR_CREATE_label(dcontext);
+    /* instead of:
+     *   cmp ecx,const
+     * we use:
+     *   lea -const(ecx) -> ecx
+     *   jecxz continue
+     *   lea const(ecx) -> ecx
+     *   jmp exit   # usual targeter for stay on trace comparison
+     * continue:    # if match, we target post-targeter
+     *
+     * we have to use the landing pad b/c we don't know whether the
+     * stub will be <128 away
+     */
+    /* lea requires OPSZ_lea operand */
+    added_size += tracelist_add
+        (dcontext, trace, targeter,
+         INSTR_CREATE_lea
+         (dcontext, opnd_create_reg(REG_ECX),
+          opnd_create_base_disp(REG_ECX, REG_NULL, 0,
+                                -((int)(ptr_int_t)speculative_tag), OPSZ_lea)));
+    jecxz = INSTR_CREATE_jecxz(dcontext, opnd_create_instr(continue_label));
+    /* do not treat jecxz as exit cti! */
+    instr_set_ok_to_mangle(jecxz, false);
+    added_size += tracelist_add(dcontext, trace, targeter, jecxz);
+    /* need to recover address in ecx */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    added_size += tracelist_add
+        (dcontext, trace, targeter,
+         INSTR_CREATE_lea
+         (dcontext, opnd_create_reg(REG_ECX),
+          opnd_create_base_disp(REG_ECX, REG_NULL, 0,
+                                ((int)(ptr_int_t)speculative_tag), OPSZ_lea)));
+    added_size += tracelist_add_after(dcontext, trace, targeter, continue_label);
+    return added_size;
+}
+
+/* Mangles an indirect branch in a trace where a basic block with tag "tag"
+ * is being added as the next block beyond the indirect branch.
+ * Returns the size of instructions added to trace.
+ */
+static int
+mangle_indirect_branch_in_trace(dcontext_t *dcontext, instrlist_t *trace,
+                                instr_t *targeter, app_pc next_tag, uint next_flags,
+                                instr_t **delete_after/*OUT*/, instr_t *end_instr)
+{
+    int added_size = 0;
+    instr_t *next = instr_get_next(targeter);
+    /* all indirect branches should be ubrs */
+    ASSERT(instr_is_ubr(targeter));
+    /* expecting basic blocks only */
+    ASSERT((end_instr != NULL && targeter == end_instr) ||
+           targeter == instrlist_last(trace));
+
+    ASSERT(delete_after != NULL);
+    *delete_after = (next == NULL || (end_instr != NULL && targeter == end_instr)) ?
+        NULL : instr_get_prev(next);
+
+    STATS_INC(trace_ib_cmp);
+
+    /* Change jump to indirect_branch_lookup to a conditional jump
+     * based on indirect target not equaling next block in trace
+     *
+     * the bb has already done:
+     *   spill xcx to xcx-tls-spill-slot
+     *   mov curtarget, xcx
+     *   <any other side effects of ind branch, like ret xsp adjust>
+     *
+     * and we now want to accomplish:
+     *       cmp ecx,const
+     *
+     * on 32-bit we use:
+     *       lea -const(ecx) -> ecx
+     *       jecxz continue
+     *       lea const(ecx) -> ecx
+     *       jmp exit   # usual targeter for stay on trace comparison
+     *     continue:    # if match, we target post-targeter
+     *       restore ecx
+     * we have to use the landing pad b/c we don't know whether the
+     * stub will be <128 away
+     *
+     * on 64-bit we use (PR 245832):
+     *       mov xax, xax-tls-spill-slot
+     *       mov $staytarget, xax
+     *       if !INTERNAL_OPTION(unsafe_ignore_eflags_{trace,ibl})
+     *         mov xax, xbx-tls-spill-slot
+     *         lahf
+     *         seto al
+     *         cmp xcx, xbx-tls-spill-slot
+     *       else
+     *         cmp xcx, xax
+     *       jne exit
+     *       if xcx live:
+     *         mov xcx-tls-spill-slot, xcx
+     *       if flags live && unsafe options not on:
+     *         add 7f, al
+     *         sahf
+     *       if xax live:
+     *         mov xax-tls-spill-slot, xax
+     */
+
+#ifdef CUSTOM_TRACES_RET_REMOVAL
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    /* try to remove ret
+     * FIXME: also handle ret imm => prev instr is add
+     */
+    inst = instr_get_prev(targeter);
+    if (dcontext->call_depth >= 0 && instr_raw_bits_valid(inst)) {
+        byte *b = inst->bytes + inst->length - 1;
+        /*
+  0x40538115   89 0d ec 68 06 40    mov    %ecx -> 0x400668ec
+  0x4053811b   59                   pop    %esp (%esp) -> %ecx %esp
+  0x4053811c   83 c4 04             add    $0x04 %esp -> %esp
+        */
+        LOG(THREAD, LOG_MONITOR, 4,
+            "ret removal: *b=0x%x, prev="PFX", dcontext="PFX", 0x%x\n",
+            *b, *((int *)(b-4)), dcontext, XCX_OFFSET);
+        if ((*b == 0x59 && *((int *)(b-4)) == ((uint)dcontext)+XCX_OFFSET) ||
+            (*(b-3)==0x59 && *((int *)(b-7)) == ((uint)dcontext)+XCX_OFFSET &&
+             *(b-2)==0x83 && *(b-1)==0xc4)) {
+            uint esp_add;
+            /* already added calls & rets to call depth
+             * if not negative, the call for this ret is earlier in this trace!
+             */
+            LOG(THREAD, LOG_MONITOR, 4, "fixup_last_cti: removing ret!\n");
+            /* delete save ecx and pop */
+            if (*b == 0x59) {
+                instr_set_raw_bits(inst, inst->bytes, inst->length - 7);
+                esp_add = 4;
+            } else { /* delete add too */
+                instr_set_raw_bits(inst, inst->bytes, inst->length - 10);
+                esp_add = 4 + (uint)(*b);
+                LOG(THREAD, LOG_MONITOR, 4, "*b=0x%x, esp_add=%d\n", *b, esp_add);
+            }
+# ifdef DEBUG
+            num_rets_removed++;
+# endif
+            removed_ret = true;
+            added_size += tracelist_add
+                (dcontext, trace, targeter,
+                 INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ESP),
+                                  opnd_create_base_disp(REG_ESP, REG_NULL, 0,
+                                                        esp_add, OPSZ_lea)));
+        }
+    }
+    if (removed_ret) {
+        *delete_after = instr_get_prev(targeter);
+        return added_size;
+    }
+#endif /* CUSTOM_TRACES_RET_REMOVAL */
+
+#ifdef NATIVE_RETURN
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    if (
+# ifdef NATIVE_RETURN_RET_IN_TRACES
+        /* disable inlining ret: keep it as ret, actually gets better
+         * performance, most noticeably on eon, vortex
+         */
+        false && 
+# endif
+        prev_l != NULL && prev_l->ret_pc != 0) {
+        /* we came here through a ret
+         * it was like this:
+         *     ret
+         *     save ecx
+         *     pop ecx
+         *     <add 4,esp>
+         *     jmp ibl
+         * we turn it into:
+         *     save ecx
+         *     pop ecx
+         *     <add 4,esp>
+         *     cmp ecx, code_cache_retaddr
+         *     jne ibl
+         *     restore ecx
+         * with custom stub code for jne ibl:
+         *     push ecx
+         *     restore ecx
+         *     ret <with NO IMMED>
+         * we get code_cache_retaddr by saving it in dispatch each time
+         * that we have to translate a next_tag from cache to app.
+         */
+        instrlist_t *stub_code = instrlist_create(dcontext);
+        /* remove ret (will be added to custom exit stub code) */
+        instr_t *ret = targeter;
+        LOG(THREAD, LOG_MONITOR, 4, "\tprev_l was a ret with last addr "PFX"\n",
+            dcontext->last_retaddr);
+        while (ret != NULL) {
+            ret = instr_get_prev(ret);
+            if (instr_opcode_valid(ret) && instr_is_return(ret))
+                break;
+        }
+        ASSERT(ret != NULL);
+        instrlist_remove(trace, ret);
+        instr_destroy(dcontext, ret);
+        /* can this ret go away, assuming the calling convention? */
+        if (dcontext->num_calls_in_trace > 0) {
+            /* BIG ASSUMPTION: this ret can go away, the corresponding
+             * call is earlier in this trace!
+             * Just need to pop retaddr off stack by adding 4 to esp.
+             */
+            ret = instr_get_prev(targeter);
+            ASSERT(ret != NULL);
+            instrlist_remove(trace, ret);
+            instr_destroy(dcontext, ret);
+            added_size += tracelist_add
+                (dcontext, trace, targeter,
+                 INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ESP),
+                                  opnd_create_base_disp(REG_ESP, REG_NULL, 0,
+                                                        (int)4, OPSZ_lea)));
+            instrlist_remove(trace, targeter);
+            instr_destroy(dcontext, targeter);
+            LOG(THREAD, LOG_MONITOR, 4, "\tjust removed ret (#calls %d)\n",
+                dcontext->num_calls_in_trace);
+            dcontext->num_calls_in_trace--;
+            goto fixup_last_cti_finished;
+        }
+        /* add cmp to cache addr */
+        added_size += tracelist_add
+            (dcontext, trace, targeter,
+             INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_ECX),
+                              OPND_CREATE_INT32((int)dcontext->last_retaddr)));
+        /* change jmp into jne indirect_branch_lookup */
+        instr_set_opcode(targeter, OP_jnz);
+        /* add custom exit stub code */
+        added_size += tracelist_add(dcontext, stub_code, NULL,
+                                    INSTR_CREATE_push(dcontext, opnd_create_reg(REG_ECX)));
+        /* FIXME: When using -unsafe_ignore_eflags, does jmp after call go to
+         * ib prefix of target? need to fix up eflags issues w/ mangle.c */
+        ASSERT_NOT_IMPLEMENTED(!INTERNAL_OPTION(unsafe_ignore_eflags));
+        added_size += tracelist_add
+            (dcontext, stub_code, NULL,
+             instr_create_restore_from_dcontext(dcontext, REG_ECX, XCX_OFFSET));
+        /* can't re-use ret instr, b/c need a ret w/ NO IMMED */
+        added_size += tracelist_add(dcontext, stub_code, NULL,
+                                    INSTR_CREATE_ret(dcontext));
+        instr_set_exit_stub_code(targeter, stub_code);
+            
+        /* now let normal restore-if-on-trace code get added */
+    } else {
+#endif /* NATIVE_RETURN */
+
+#ifdef X64
+        added_size += tracelist_add
+            (dcontext, trace, targeter, INSTR_CREATE_mov_st
+             (dcontext, opnd_create_tls_slot(os_tls_offset(PREFIX_XAX_SPILL_SLOT)),
+              opnd_create_reg(REG_XAX)));
+        added_size += tracelist_add
+            (dcontext, trace, targeter, INSTR_CREATE_mov_imm
+             (dcontext, opnd_create_reg(REG_XAX),
+              OPND_CREATE_INTPTR((ptr_int_t)next_tag)));
+        /* saving in the trace and restoring in ibl means that
+         * -unsafe_ignore_eflags_{trace,ibl} must be equivalent
+         */
+        if (!DYNAMO_OPTION(unsafe_ignore_eflags_trace)) {
+            added_size += tracelist_add
+                (dcontext, trace, targeter, INSTR_CREATE_mov_st
+                 (dcontext, opnd_create_tls_slot(os_tls_offset(INDIRECT_STUB_SPILL_SLOT)),
+                  opnd_create_reg(REG_XAX)));
+            /* FIXME: share w/ insert_save_eflags() */
+            added_size += tracelist_add
+                (dcontext, trace, targeter, INSTR_CREATE_lahf(dcontext));
+            if (!INTERNAL_OPTION(unsafe_ignore_overflow)) { /* OF needs saving */
+                /* Move OF flags into the OF flag spill slot. */
+                added_size += tracelist_add
+                    (dcontext, trace, targeter,
+                     INSTR_CREATE_setcc(dcontext, OP_seto, opnd_create_reg(REG_AL)));
+            }
+            added_size += tracelist_add
+                (dcontext, trace, targeter,
+                 INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_XCX),
+                                  opnd_create_tls_slot(os_tls_offset
+                                                       (INDIRECT_STUB_SPILL_SLOT))));
+        } else {
+            added_size += tracelist_add
+                (dcontext, trace, targeter,
+                 INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_XCX),
+                                  opnd_create_reg(REG_XAX)));
+        }
+        /* change jmp into jne to trace cmp entry of ibl routine (special entry
+         * that is after the eflags save) */
+        instr_set_opcode(targeter, OP_jnz);
+        added_size++; /* jcc is 6 bytes, jmp is 5 bytes */
+        ASSERT(opnd_is_pc(instr_get_target(targeter)));
+        instr_set_target(targeter, opnd_create_pc
+                         (get_trace_cmp_entry(dcontext, opnd_get_pc
+                                              (instr_get_target(targeter)))));
+        /* since the target gets lost we need to OR in this flag */
+        instr_exit_branch_set_type(targeter, instr_exit_branch_type(targeter) |
+                                   INSTR_TRACE_CMP_EXIT);
+#else
+        if (!INTERNAL_OPTION(unsafe_ignore_eflags_trace)) {
+            /* if equal follow to the next instruction after the exit CTI */
+            added_size +=
+                insert_transparent_comparison(dcontext, trace, targeter, next_tag);
+            /* leave jmp as it is, a jmp to exit stub (thence to ind br
+             * lookup) */
+        }
+        else {
+            /* assume eflags don't need to be saved across ind branches,
+             * so go ahead and use cmp, jne
+             */
+            /* FIXME: no way to cmp w/ 64-bit immed */
+            IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+            added_size += tracelist_add
+                (dcontext, trace, targeter,
+                 INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_ECX),
+                                  OPND_CREATE_INT32((int)(ptr_int_t)next_tag)));
+
+            /* Change jmp into jne indirect_branch_lookup */
+            /* CHECK: is that also going to exit stub */
+            instr_set_opcode(targeter, OP_jnz);
+            added_size++; /* jcc is 6 bytes, jmp is 5 bytes */
+        }
+#endif /* X64 */
+        /* PR 214962: our spill restoration needs this whole sequence marked mangle */
+        instr_set_our_mangling(targeter, true);
+
+        LOG(THREAD, LOG_MONITOR, 3,
+            "fixup_last_cti: added cmp vs. "PFX" for ind br\n", next_tag);
+
+#ifdef NATIVE_RETURN
+    }
+#endif
+
+#ifdef HASHTABLE_STATISTICS
+    /* If we do stay on the trace, increment a counter using dead XCX */
+    if (INTERNAL_OPTION(stay_on_trace_stats)) {
+        ibl_type_t ibl_type;
+        /* FIXME: see if can test the instr flags instead */
+        DEBUG_DECLARE(bool ok =)
+            get_ibl_routine_type(dcontext, opnd_get_pc(instr_get_target(targeter)),
+                                 &ibl_type);
+        ASSERT(ok);
+        added_size += insert_increment_stat_counter
+            (dcontext, trace, next, 
+             &get_ibl_per_type_statistics(dcontext, ibl_type.branch_type)->
+             ib_stay_on_trace_stat);
+    }
+#endif /* HASHTABLE_STATISTICS */
+
+    /* If we do stay on the trace, must restore xcx
+     * TODO optimization: check if xcx is live or not in next bb */
+    added_size += insert_restore_spilled_xcx(dcontext, trace, next);
+
+#ifdef X64
+    if (!TEST(FRAG_WRITES_EFLAGS_6, next_flags) &&
+        !DYNAMO_OPTION(unsafe_ignore_eflags_trace)) {
+        if (!TEST(FRAG_WRITES_EFLAGS_OF, next_flags) &&  /* OF was saved */
+            !INTERNAL_OPTION(unsafe_ignore_overflow)) { 
+            /* restore OF using add that overflows if OF was on when we did seto */
+            added_size += tracelist_add
+                (dcontext, trace, next, INSTR_CREATE_add
+                 (dcontext, opnd_create_reg(REG_AL), OPND_CREATE_INT8(0x7f)));
+        }
+        added_size += tracelist_add
+            (dcontext, trace, next, INSTR_CREATE_sahf(dcontext));
+    } else
+        STATS_INC(trace_ib_no_flag_restore);
+    /* TODO optimization: check if xax is live or not in next bb */
+    added_size += tracelist_add
+        (dcontext, trace, next, INSTR_CREATE_mov_ld
+         (dcontext, opnd_create_reg(REG_XAX),
+          opnd_create_tls_slot(os_tls_offset(PREFIX_XAX_SPILL_SLOT))));
+#endif
+
+#ifdef RETURN_STACK
+    if (instr_is_return) {
+        /* if we stay on the trace, still need to clear pair from
+         * return stack
+         */
+        opnd_t top = opnd_create_dcontext_field(dcontext,
+                                                TOP_OF_RSTACK_OFFSET);
+        /* insert prior to restoring flags (add clobbers them)! */
+        added_size += tracelist_add(dcontext, trace, next,
+                                    INSTR_CREATE_add(dcontext, top, OPND_CREATE_INT8(8)));
+        
+        /* FIXME: we need to execute a ret instruction in order to
+         * clear the hardware's return stack!  And if we do that there
+         * is no way to avoid a ret misprediction...
+         */
+        ASSERT_NOT_IMPLEMENTED(false);
+    }
+#endif
+    return added_size;
+}
+
+/* This routine handles the mangling of the cti at the end of the
+ * previous block when adding a new block (f) to the trace fragment.
+ * If prev_l is not NULL, matches the ordinal of prev_l to the nth
+ * exit cti in the trace instrlist_t.
+ *
+ * If prev_l is NULL: WARNING: this routine assumes that the previous
+ * block can only have a single indirect branch -- otherwise there is
+ * no way to determine which indirect exit targeted the new block!  No
+ * assumptions are made about direct exits -- we can walk through them
+ * all to find the one that targeted the new block.
+ *
+ * Returns an upper bound on the size added to the trace with inserted
+ * instructions.
+ * If we change this to add a substantial # of instrs, should update
+ * TRACE_CTI_MANGLE_SIZE_UPPER_BOUND (assert at bottom should notify us)
+ *
+ * If you want to re-add the ability to add the front end of a trace,
+ * revive the now-removed CUSTOM_TRACES_ADD_TRACE define from the attic.
+ */
+static int
+fixup_last_cti(dcontext_t *dcontext, instrlist_t *trace,
+               app_pc next_tag, uint next_flags, uint trace_flags,
+               fragment_t *prev_f, linkstub_t *prev_l,
+               bool record_translation, uint *num_exits_deleted/*OUT*/,
+               /* If non-NULL, only looks inside trace between these two */
+               instr_t *start_instr, instr_t *end_instr)
+{
+    app_pc target_tag;
+    instr_t *inst, *targeter = NULL;
+    /* at end of routine we will delete all instrs after this one: */
+    instr_t *delete_after = NULL;
+    bool is_indirect = false;
+#ifdef RETURN_STACK
+    bool instr_is_return;
+#endif
+    /* Added size for transformations done here.
+     * Use tracelist_add to automate adding inserted instr sizes.
+     */
+    int added_size = 0;
+    uint exits_deleted = 0;
+
+    /* count exit stubs to get the ordinal of the exit that targeted us
+     * start at prev_l, and count up extraneous exits and blks until end
+     */
+    uint nth_exit = 0, cur_exit;
+#ifdef CUSTOM_TRACES_RET_REMOVAL
+    bool removed_ret = false;
+#endif
+    bool have_ordinal = false;
+    if (prev_l != NULL && prev_l == get_deleted_linkstub(dcontext)) {
+        int last_ordinal = get_last_linkstub_ordinal(dcontext);
+        if (last_ordinal != -1) {
+            nth_exit = last_ordinal;
+            have_ordinal = true;
+        }
+    }
+    if (!have_ordinal && prev_l != NULL && !LINKSTUB_FAKE(prev_l)) {
+        linkstub_t *stub = FRAGMENT_EXIT_STUBS(prev_f);
+        while (stub != prev_l)
+            stub = LINKSTUB_NEXT_EXIT(stub);
+        /* if prev_l is cbr followed by ubr, we'll get 1 for ubr,
+         * but we want 0, so we count prev_l itself, then decrement
+         */
+        stub = LINKSTUB_NEXT_EXIT(stub);
+        while (stub != NULL) {
+            nth_exit++;
+            stub = LINKSTUB_NEXT_EXIT(stub);
+        }
+    } /* else, we assume it's the final exit */
+
+    LOG(THREAD, LOG_MONITOR, 4,
+        "fixup_last_cti: looking for %d-th exit cti from bottom\n", nth_exit);
+
+    if (start_instr != NULL) {
+        ASSERT(end_instr != NULL);
+    } else {
+        start_instr = instrlist_first(trace);
+        end_instr = instrlist_last(trace);
+    }
+    start_instr = instr_get_prev(start_instr); /* get open-ended bound */
+
+    cur_exit = nth_exit;
+    /* now match the ordinal to the instrs.
+     * we don't have any way to find boundary with previous-previous block
+     * to make sure we didn't go backwards too far -- does it matter?
+     */
+    for (inst = end_instr; inst != NULL && inst != start_instr;
+         inst = instr_get_prev(inst)) {
+        if (instr_is_exit_cti(inst)) {
+            if (cur_exit == 0) {
+                ibl_type_t ibl_type;
+                /* exit cti is guaranteed to have pc target */
+                target_tag = opnd_get_pc(instr_get_target(inst));
+
+                is_indirect = get_ibl_routine_type(dcontext, target_tag, &ibl_type);
+
+                if (is_indirect) {
+                    /* this should be a trace exit stub therefore it cannot be IBL_BB* */
+                    ASSERT(IS_IBL_TRACE(ibl_type.source_fragment_type));
+#ifdef RETURN_STACK
+                    instr_is_return = ibl_type.branch_type == IBL_RETURN;
+#endif
+                    targeter = inst;
+                    break;
+                } else {
+                    if (prev_l != NULL) {
+                        /* direct jmp, better point to us */
+                        ASSERT(target_tag == next_tag);
+                        targeter = inst;
+                        break;
+                    } else {
+                        /* need to search for targeting jmp */
+                        if (target_tag == next_tag) {
+                            targeter = inst;
+                            break;
+                        }
+                    }
+                }
+            } else if (prev_l != NULL) {
+                LOG(THREAD, LOG_MONITOR, 4,
+                    "counting backwards: %d == target_tag = "PFX"\n",
+                    cur_exit, opnd_get_pc(instr_get_target(inst)));
+                cur_exit--;
+            }
+        } /* is exit cti */
+    }
+    ASSERT(targeter != NULL);
+    if (record_translation)
+        instrlist_set_translation_target(trace, instr_get_translation(targeter));
+    instrlist_set_our_mangling(trace, true); /* PR 267260 */
+    DOLOG(4, LOG_MONITOR, {
+        loginst(dcontext, 4, targeter, "\ttargeter");
+    });
+    if (is_indirect) {
+        added_size += mangle_indirect_branch_in_trace(dcontext, trace, targeter,
+                                                      next_tag, next_flags,
+                                                      &delete_after, end_instr);
+    } else {
+        /* direct jump or conditional branch */
+        instr_t *next = targeter->next;
+        if (instr_is_cbr(targeter)) {
+            LOG(THREAD, LOG_MONITOR, 4, "fixup_last_cti: inverted logic of cbr\n");
+            if (next != NULL && instr_is_ubr(next)) {
+                /* cbr followed by ubr: if cbr got us here, reverse cbr and
+                 * remove ubr
+                 */
+                instr_invert_cbr(targeter);
+                instr_set_target(targeter, instr_get_target(next));
+                ASSERT(next == end_instr);
+                delete_after = targeter;
+                LOG(THREAD, LOG_MONITOR, 4, "\tremoved ubr following cbr\n");
+            } else {
+                ASSERT_NOT_REACHED();
+            }
+        } else if (instr_is_ubr(targeter)) {
+#ifndef CUSTOM_TRACES
+            ASSERT(targeter == end_instr);
+#endif
+            /* remove unnecessary ubr at end of block */
+            delete_after = instr_get_prev(targeter);
+            if (delete_after != NULL) {
+                LOG(THREAD, LOG_MONITOR, 4,     "fixup_last_cti: removed ubr\n");
+            }
+        } else
+            ASSERT_NOT_REACHED();
+    }
+    /* remove all instrs after this cti -- but what if internal
+     * control flow jumps ahead and then comes back?
+     * too expensive to check for such all the time.
+     * FIXME: what to do?
+     *
+     * ifdef CUSTOM_TRACES:
+     * FIXME: rather than adding entire trace on and then chopping off where
+     * we exited, why not add after we know where to stop?
+     */
+    if (delete_after != NULL) {
+        ASSERT(delete_after != end_instr);
+        delete_after = instr_get_next(delete_after);
+        while (delete_after != NULL) {
+            inst = delete_after;
+            if (delete_after == end_instr)
+                delete_after = NULL;
+            else
+                delete_after = instr_get_next(delete_after);
+            if (instr_is_exit_cti(inst)) {
+                /* assumption: passing in cache target to exit_stub_size works
+                 * just as well as linkstub_t target, since only cares whether
+                 * targeting ibl
+                 */
+                app_pc target = opnd_get_pc(instr_get_target(inst));
+                /* we already added all the stub size differences to the trace,
+                 * so we subtract the trace size of the stub here
+                 */
+                added_size -= local_exit_stub_size(dcontext, target, trace_flags);
+                exits_deleted++;
+            } else if (instr_opcode_valid(inst) && instr_is_cti(inst)) {
+                LOG(THREAD, LOG_MONITOR, 3,
+                    "WARNING: deleting non-exit cti in unused tail of frag added to trace\n");
+            }
+            loginst(dcontext, 4, inst, "\tdeleting");
+            instrlist_remove(trace, inst);
+            added_size -= instr_length(dcontext, inst);
+            instr_destroy(dcontext, inst);
+        }
+    }
+
+    if (num_exits_deleted != NULL)
+        *num_exits_deleted = exits_deleted;
+
+#ifdef NATIVE_RETURN
+ fixup_last_cti_finished:
+#endif
+    if (record_translation)
+        instrlist_set_translation_target(trace, NULL);
+    instrlist_set_our_mangling(trace, false); /* PR 267260 */
+    ASSERT(added_size < TRACE_CTI_MANGLE_SIZE_UPPER_BOUND);
+    return added_size;
+}
+
+/* Add a speculative counter on last IBL exit
+ * Returns additional size to add to trace estimate.
+ */
+int
+append_trace_speculate_last_ibl(dcontext_t *dcontext, instrlist_t *trace,
+                                app_pc speculate_next_tag,
+                                bool record_translation)
+{
+    /* unlike fixup_last_cti() here we are about to go directly to the IBL routine */
+    /* spill XCX in a scratch slot - note always using TLS */
+    int added_size = 0;
+    ibl_type_t ibl_type;
+
+    instr_t *inst = instrlist_last(trace); /* currently only relevant to last CTI */
+    instr_t *where = inst;         /* preinsert before last CTI */
+
+    instr_t *next = instr_get_next(inst);
+    DEBUG_DECLARE(bool ok;)
+
+    ASSERT(speculate_next_tag != NULL);
+    ASSERT(inst != NULL);
+    ASSERT(instr_is_exit_cti(inst));
+
+    /* FIXME: see if can test the instr flags instead */
+    DEBUG_DECLARE(ok = )
+        get_ibl_routine_type(dcontext, opnd_get_pc(instr_get_target(inst)), &ibl_type);
+    ASSERT(ok);
+                
+    if (record_translation)
+        instrlist_set_translation_target(trace, instr_get_translation(inst));
+    instrlist_set_our_mangling(trace, true); /* PR 267260 */
+
+    STATS_INC(num_traces_end_at_ibl_speculative_link);
+
+#ifdef HASHTABLE_STATISTICS
+    DOSTATS({
+        if (INTERNAL_OPTION(speculate_last_exit_stats)) {
+            int tls_stat_scratch_slot = os_tls_offset(HTABLE_STATS_SPILL_SLOT);
+
+            added_size += 
+                tracelist_add(dcontext, trace, where, 
+                              INSTR_CREATE_mov_st(dcontext,
+                                                  opnd_create_tls_slot(tls_stat_scratch_slot),
+                                                  opnd_create_reg(REG_XCX)));
+            added_size += 
+                insert_increment_stat_counter(dcontext, trace, where, 
+                                              &get_ibl_per_type_statistics(dcontext, 
+                                                                           ibl_type.branch_type)
+                                              ->ib_trace_last_ibl_exit);
+            added_size += 
+                tracelist_add(dcontext, trace, where, 
+                              INSTR_CREATE_mov_ld(dcontext,
+                                                  opnd_create_reg(REG_XCX),
+                                                  opnd_create_tls_slot(tls_stat_scratch_slot)));
+        }
+    });
+#endif
+    /* preinsert comparison before exit CTI, but increment of success statistics after it */
+
+    /* we need to compare to speculate_next_tag now */
+    /* XCX holds value to match */
+
+    /* should use similar eflags-clobbering scheme to inline cmp */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    /* 
+     *    8d 89 76 9b bf ff    lea    -tag(%ecx) -> %ecx 
+     *    e3 0b                jecxz  continue
+     *    8d 89 8a 64 40 00    lea    tag(%ecx) -> %ecx 
+     *    e9 17 00 00 00       jmp    <exit stub 1: IBL>
+     *
+     * continue:
+     *                        <increment stats>
+     *                        <restore app ecx>  # see FIXME whether to go to prefix or do here
+     *    e9 cc aa dd 00       jmp speculate_next_tag
+     *
+     */
+
+    /* leave jmp as it is, a jmp to exit stub (thence to ind br lookup) */
+    added_size += 
+        insert_transparent_comparison(dcontext, trace, where, speculate_next_tag);
+
+#ifdef HASHTABLE_STATISTICS
+    DOSTATS({
+        if (INTERNAL_OPTION(speculate_last_exit_stats)) {
+            int tls_stat_scratch_slot = os_tls_offset(HTABLE_STATS_SPILL_SLOT);
+            /* XCX already saved */
+
+            added_size += 
+                insert_increment_stat_counter(dcontext, trace, next, 
+                                              &get_ibl_per_type_statistics(dcontext, 
+                                                                           ibl_type.branch_type)
+                                              ->ib_trace_last_ibl_speculate_success);
+            /* restore XCX to app IB target*/
+            added_size += 
+                tracelist_add(dcontext, trace, next, 
+                              INSTR_CREATE_mov_ld(dcontext,
+                                                  opnd_create_reg(REG_XCX),
+                                                  opnd_create_tls_slot(tls_stat_scratch_slot)));
+        }
+    });        
+#endif
+    /* adding a new CTI for speculative target that is a pseudo
+     * direct exit.  Although we could have used the indirect stub
+     * to be the unlinked path, with a new CTI way we can unlink a
+     * speculated fragment without affecting any other targets
+     * reached by the IBL.  Also in general we could decide to add
+     * multiple speculative comparisons and to chain them we'd
+     * need new CTIs for them.
+     */
+
+    /* Ensure all register state is properly preserved on both linked
+     * and unlinked paths - currently only XCX is in use.
+     * 
+     *
+     * Preferrably we should be targeting prefix of target to
+     * save some space for recovering XCX from hot path.  We'd
+     * restore XCX in the exit stub when unlinked.
+     * So it would act like a direct CTI when linked and like indirect
+     * when unlinked.  It could just be an unlinked indirect stub, if
+     * we haven't modified any other registers or flags.
+     *
+     * For simplicity, we currently restore XCX here and use a plain
+     * direct exit stub that goes to target start_pc instead of
+     * prefixes.  
+     *
+     * FIXME: (case 5085) the problem with the current scheme is that
+     * when we exit unlinked the source will be marked as a DIRECT
+     * exit - therefore no security policies will be enforced.
+     *
+     * FIXME: (case 4718) should add speculated target to current list
+     * in case of RCT policy that needs to be invalidated if target is
+     * flushed
+     */
+
+    /* must restore xcx to app value, FIXME: see above for doing this in prefix+stub */
+    added_size += insert_restore_spilled_xcx(dcontext, trace, next);
+
+    /* add a new direct exit stub */
+    added_size += tracelist_add(dcontext, trace, next, 
+                                INSTR_CREATE_jmp(dcontext, opnd_create_pc(speculate_next_tag)));
+    LOG(THREAD, LOG_INTERP, 3,
+        "append_trace_speculate_last_ibl: added cmp vs. "PFX" for ind br\n",
+        speculate_next_tag);
+
+    if (record_translation)
+        instrlist_set_translation_target(trace, NULL);
+    instrlist_set_our_mangling(trace, false); /* PR 267260 */
+
+    return added_size;
+}
+
+#ifdef HASHTABLE_STATISTICS
+/* Add a counter on last IBL exit
+ * if speculate_next_tag is not NULL then check case 4817's possible success
+ */
+/* FIXME: remove this routine once append_trace_speculate_last_ibl()
+ * currently useful only to see statistics without side effects of
+ * adding exit stub
+ */
+int
+append_ib_trace_last_ibl_exit_stat(dcontext_t *dcontext, instrlist_t *trace,
+                                   app_pc speculate_next_tag)
+{
+    /* unlike fixup_last_cti() here we are about to go directly to the IBL routine */
+    /* spill XCX in a scratch slot - note always using TLS */
+    int tls_stat_scratch_slot = os_tls_offset(HTABLE_STATS_SPILL_SLOT);
+    int added_size = 0;
+    ibl_type_t ibl_type;
+
+    instr_t *inst = instrlist_last(trace); /* currently only relevant to last CTI */
+    instr_t *where = inst;         /* preinsert before exit CTI */
+
+    DEBUG_DECLARE(bool ok;)
+
+    /* should use similar eflags-clobbering scheme to inline cmp */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+
+    ASSERT(inst != NULL);
+    ASSERT(instr_is_exit_cti(inst));
+
+    /* FIXME: see if can test the instr flags instead */
+    ok = get_ibl_routine_type(dcontext, opnd_get_pc(instr_get_target(inst)), &ibl_type);
+                
+    ASSERT(ok);
+    added_size += tracelist_add(dcontext, trace, where, 
+                                INSTR_CREATE_mov_st(dcontext,
+                                                    opnd_create_tls_slot(tls_stat_scratch_slot),
+                                                    opnd_create_reg(REG_XCX)));
+    added_size += 
+        insert_increment_stat_counter(dcontext, trace, where, 
+                                      &get_ibl_per_type_statistics(dcontext, ibl_type.branch_type)
+                                      ->ib_trace_last_ibl_exit);
+    added_size += tracelist_add(dcontext, trace, where, 
+                                INSTR_CREATE_mov_ld(dcontext,
+                                                    opnd_create_reg(REG_XCX),
+                                                    opnd_create_tls_slot(tls_stat_scratch_slot)));
+
+    if (speculate_next_tag != NULL) {
+        instr_t *next = instr_get_next(inst);
+        /* preinsert comparison before exit CTI, but increment goes after it */
+
+        /* we need to compare to speculate_next_tag now - just like fixup_last_cti() would do later */
+        /* ECX holds value to match here */
+        
+        /* leave jmp as it is, a jmp to exit stub (thence to ind br lookup) */
+
+        /* continue:
+         *    increment success counter 
+         *    jmp targeter
+         *
+         *   FIXME: now the last instruction is no longer the exit_cti - see if that breaks any assumptions,
+         *   using a short jump to see if anyone erroneously uses this
+         */
+        added_size += 
+            insert_transparent_comparison(dcontext, trace, where, speculate_next_tag);
+
+        /* we'll kill again although ECX restored unnecessarily by comparison routine,   */
+        added_size += 
+            insert_increment_stat_counter(dcontext, trace, next, 
+                                          &get_ibl_per_type_statistics(dcontext, ibl_type.branch_type)
+                                          ->ib_trace_last_ibl_speculate_success);
+        /* restore ECX */
+        added_size += tracelist_add(dcontext, trace, next, 
+                                INSTR_CREATE_mov_ld(dcontext,
+                                                    opnd_create_reg(REG_ECX),
+                                                    opnd_create_tls_slot(tls_stat_scratch_slot)));
+        /* jmp where */
+        added_size += tracelist_add(dcontext, trace, next, 
+                                    INSTR_CREATE_jmp_short(dcontext, opnd_create_instr(where)));
+    }
+
+    return added_size;
+}
+#endif /* HASHTABLE_STATISTICS */
+
+/* Add the fragment f to the end of the trace instrlist_t kept in dcontext
+ *
+ * Note that recreate_fragment_ilist() is making assumptions about its operation
+ * synchronize changes 
+ *
+ * Returns the size change in the trace from mangling the previous block
+ * (assumes the caller has already calculated the size from adding the new block)
+ */
+uint
+extend_trace(dcontext_t *dcontext, fragment_t *f, linkstub_t *prev_l)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    fragment_t *prev_f = NULL;
+    instrlist_t *trace = &(md->trace);
+    instrlist_t *ilist;
+    uint size;
+    uint prev_mangle_size = 0;
+    uint num_exits_deleted = 0;
+    uint new_exits_dir = 0, new_exits_indir = 0;
+
+    STATS_INC(num_traces_extended);
+
+    /* if you want to re-add the ability to add traces, revive
+     * CUSTOM_TRACES_ADD_TRACE from the attic
+     */
+    ASSERT(!TEST(FRAG_IS_TRACE, f->flags)); /* expecting block fragments */
+
+    if (prev_l != NULL) {
+        ASSERT(!LINKSTUB_FAKE(prev_l) ||
+               /* we track the ordinal of the del linkstub so it's ok */
+               prev_l == get_deleted_linkstub(dcontext));
+        prev_f = linkstub_fragment(dcontext, prev_l);
+        LOG(THREAD, LOG_MONITOR, 4, "prev_l = owned by F%d, branch pc "PFX"\n",
+            prev_f->id, EXIT_CTI_PC(prev_f, prev_l));
+    } else {
+        LOG(THREAD, LOG_MONITOR, 4, "prev_l is NULL\n");
+    }
+
+    /* insert code to optimize last branch based on new fragment */
+    if (instrlist_last(trace) != NULL) {
+        prev_mangle_size = fixup_last_cti(dcontext, trace, f->tag, f->flags,
+                                          md->trace_flags, prev_f, prev_l, false,
+                                          &num_exits_deleted, NULL, NULL);
+    }
+    
+#ifdef CUSTOM_TRACES_RET_REMOVAL
+    /* add now, want fixup to operate on depth before adding new blk */
+    dcontext->call_depth += f->num_calls;
+    dcontext->call_depth -= f->num_rets;
+#endif
+
+    LOG(THREAD, LOG_MONITOR, 4,
+        "\tadding block %d == "PFX"\n", md->num_blks, f->tag);
+
+    size = md->trace_buf_size - md->trace_buf_top;
+    LOG(THREAD, LOG_MONITOR, 4, "decoding F%d into trace buf @"PFX" + 0x%x = "PFX"\n",
+        f->id, md->trace_buf, md->trace_buf_top, md->trace_buf + md->trace_buf_top);
+    /* FIXME: PR 307388: if md->pass_to_client, much of this is a waste of time as
+     * we're going to re-mangle and re-fixup after passing our unmangled list to the
+     * client.  We do want to keep the size estimate, which requires having the last
+     * cti at least, so for now we keep all the work.  Of course the size estimate is
+     * less valuable when the client may add a ton of instrumentation.
+     */
+    /* decode_fragment will convert f's ibl routines into those appropriate for
+     * our trace, whether f and the trace are shared or private
+     */
+    ilist = decode_fragment(dcontext, f, md->trace_buf + md->trace_buf_top, &size,
+                            md->trace_flags, &new_exits_dir, &new_exits_indir);
+
+    md->blk_info[md->num_blks].info.tag = f->tag;
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+    if (md->num_blks > 0)
+        md->blk_info[md->num_blks - 1].info.num_exits -= num_exits_deleted;
+    md->blk_info[md->num_blks].info.num_exits = new_exits_dir + new_exits_indir;
+#endif
+    md->num_blks++;
+    
+    /* We need to remove any nops we added for -pad_jmps (we don't expect there
+     * to be any in a bb if -pad_jmps_shift_bb) to avoid screwing up 
+     * fixup_last_cti etc. */
+    process_nops_for_trace(dcontext, ilist, f->flags _IF_DEBUG(false/*!recreating*/));
+
+    DOLOG(5, LOG_MONITOR, {
+        LOG(THREAD, LOG_MONITOR, 5, "post-trace-ibl-fixup, ilist is:\n");
+        instrlist_disassemble(dcontext, f->tag, ilist, THREAD);
+    });
+
+    ASSERT(!instrlist_get_our_mangling(ilist));
+    instrlist_append(trace, instrlist_first(ilist));
+    instrlist_init(ilist); /* clear fields so destroy won't kill instrs now on trace list */
+    instrlist_destroy(dcontext, ilist);
+
+    md->trace_buf_top += size;
+    ASSERT(md->trace_buf_top < md->trace_buf_size);
+    LOG(THREAD, LOG_MONITOR, 4, "post-extend_trace, trace buf + 0x%x => "PFX"\n",
+        md->trace_buf_top, md->trace_buf);
+
+    DOLOG(4, LOG_MONITOR, {
+        LOG(THREAD, LOG_MONITOR, 4, "\nafter extending trace:\n");
+        instrlist_disassemble(dcontext, md->trace_tag, trace, THREAD);
+    });
+    return prev_mangle_size;
+}
+
+/* If branch_type is 0, sets it to the type of a ubr */
+static instr_t *
+create_exit_jmp(dcontext_t *dcontext, app_pc target, app_pc translation,
+                uint branch_type)
+{
+    instr_t *jmp = INSTR_CREATE_jmp(dcontext, opnd_create_pc(target));
+    instr_set_translation(jmp, translation);
+    if (branch_type == 0)
+        instr_exit_branch_set_type(jmp, instr_branch_type(jmp));
+    else
+        instr_exit_branch_set_type(jmp, branch_type);
+    instr_set_our_mangling(jmp, true);
+    return jmp;
+}
+
+static bool
+can_use_mangle_trace(void)
+{
+    /* elision messes up our bb boundary identification */
+    bool ok = (DYNAMO_OPTION(max_elide_jmp) == 0 &&
+               DYNAMO_OPTION(max_elide_call) == 0 &&
+               !DYNAMO_OPTION(indcall2direct));
+#ifdef CLIENT_INTERFACE
+    /* must be able to use it if client has hooks */
+    ASSERT(ok || (!dr_bb_hook_exists() && !dr_trace_hook_exists()));
+#endif
+    return ok;
+}
+
+/* Given an ilist with no mangling or stitching together, this routine does those
+ * things.  This is used both for CLIENT_INTERFACE and for recreating traces
+ * for state translation.
+ * It assumes the ilist abides by client rules: single-mbr bbs, no
+ * changes in source app code, no elision.  Else, it returns false.
+ *
+ * Reads the following fields from md:
+ * - trace_tag
+ * - trace_flags
+ * - num_blks
+ * - blk_info
+ * - final_exit_flags
+ */
+bool
+mangle_trace(dcontext_t *dcontext, instrlist_t *ilist, monitor_data_t *md)
+{
+    instr_t *inst, *next_inst, *start_instr, *jmp;
+    uint blk, num_exits_deleted;
+    app_pc fallthrough = NULL;
+    bool found_syscall = false, found_int = false;
+    int i;
+
+    ASSERT(can_use_mangle_trace());
+
+    LOG(THREAD, LOG_MONITOR, 2, "mangle_trace "PFX"\n", md->trace_tag);
+    DOLOG(4, LOG_INTERP, {
+        LOG(THREAD, LOG_INTERP, 4, "ilist passed to mangle_trace:\n");
+        instrlist_disassemble(dcontext, md->trace_tag, ilist, THREAD);
+    });
+
+    /* We make 3 passes.
+     * 1st walk: find bb boundaries
+     */
+    blk = 0;
+    for (inst = instrlist_first(ilist); inst != NULL; inst = next_inst) {
+        LOG_DECLARE(instr_t *prev = instr_get_prev(inst);)
+        next_inst = instr_get_next(inst);
+
+        if (!instr_ok_to_mangle(inst))
+            continue;
+
+        DOLOG(5, LOG_INTERP, {
+            LOG(THREAD, LOG_MONITOR, 4, "transl "PFX" ", instr_get_translation(inst));
+            loginst(dcontext, 4, inst, "considering non-meta");
+        }); 
+
+        /* Check for bb end with no exit cti.  We don't worry about whether a ubr w/
+         * a translation set to its target is part of this or next bb: all that's
+         * important is blk++ to ensure we find later transitions.  We assume a block
+         * w/ nothing but ubrs must have the final one target the subsequent block
+         * (which will hit our exit cti check below), so we'll always see an instr in
+         * the next block's translation range.
+         */
+        /* PR 366232: we handle empty bbs by looping here */
+        for (i = 1; blk+i < md->num_blks; i++) {
+            /* Since we can have tail-duplication among bbs, make sure to first
+             * check whether we're still in the cur block: b/c this translation
+             * can be both in this block and in next block.  In such a case we
+             * assume a (backward) cti separates them. */
+            if (!(instr_get_translation(inst) >= md->blk_info[blk].info.tag &&
+                  instr_get_translation(inst) < md->blk_info[blk].bounds.end_pc) &&
+                instr_get_translation(inst) >= md->blk_info[blk+i].info.tag &&
+                instr_get_translation(inst) < md->blk_info[blk+i].bounds.end_pc) {
+                /* counting down but adding jmps in forward order */
+                for (; i >= 1; i--) {
+                    DOLOG(4, LOG_INTERP, { /* use prev to avoid added jmp */
+                        loginst(dcontext, 4, prev, "fallthrough end of bb");
+                    }); 
+                    jmp = create_exit_jmp(dcontext, md->blk_info[blk+1].info.tag,
+                                          md->blk_info[blk+1].info.tag, 0);
+                    instrlist_preinsert(ilist, inst, jmp);
+                    md->blk_info[blk].bounds.end_instr = jmp;
+                    blk++;
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+                    /* we'll decrement if jmp gets elided (as it should) */
+                    md->blk_info[blk].info.num_exits++;
+#endif
+                    LOG(THREAD, LOG_MONITOR, 4, "starting next bb "PFX"\n",
+                        md->blk_info[blk].info.tag);
+                }
+                break;
+            }
+        }
+
+#ifdef CLIENT_INTERFACE
+        /* Ensure non-ignorable syscall/int2b terminates trace */
+        if (md->pass_to_client &&
+            !client_check_syscall(ilist, inst, &found_syscall, &found_int))
+            return false;
+
+        /* Clients should not add new source code regions, which would mess us up
+         * here, as well as mess up our cache consistency (both page prot and
+         * selfmod).
+         */
+        if (md->pass_to_client &&
+            !((instr_get_translation(inst) >= md->blk_info[blk].info.tag &&
+               instr_get_translation(inst) < md->blk_info[blk].bounds.end_pc) ||
+              (instr_is_ubr(inst) && opnd_is_pc(instr_get_target(inst)) &&
+               instr_get_translation(inst) == opnd_get_pc(instr_get_target(inst))))) {
+            LOG(THREAD, LOG_MONITOR, 2,
+                "trace error: out-of-bounds transl "PFX" vs start "PFX" end "PFX"\n",
+                instr_get_translation(inst),
+                md->blk_info[blk].info.tag, md->blk_info[blk].bounds.end_pc);
+            CLIENT_ASSERT(false,
+                          "trace's app sources (instr_set_translation() targets) "
+                          "must remain within original bounds");
+            return false;
+        }
+#endif
+
+        /* in case no exit ctis in last block, find last non-meta fall-through */
+        if (blk == md->num_blks - 1)
+            fallthrough = instr_get_translation(inst) + instr_length(dcontext, inst);
+
+        /* PR 299808: identify bb boundaries.  We can't go by translations alone, as
+         * ubrs can point at their targets and theoretically the entire trace could
+         * be ubrs: so we have to go by exits, and limit what the client can do.  We
+         * can assume that each bb should not violate the bb callback rules (PR
+         * 215217): if has cbr, call, or mbr, that must end bb.  We also want to
+         * impose the can't-be-trace rules (PR 215219), which are not documented for
+         * bbs: if more than one exit cti or if code beyond last exit cti then can't
+         * be in a trace.  We can soften a little and allow extra ubrs if they do not
+         * target the subsequent block.  FIXME: we could have stricter translation
+         * reqts for ubrs: make them point at corresponding app ubr (but what if
+         * really correspond to app cbr?): then can handle code past exit ubr.
+         */
+        if (instr_will_be_exit_cti(inst) &&
+            (!instr_is_ubr(inst) ||
+             (inst == instrlist_last(ilist) ||
+              (blk+1 < md->num_blks &&
+               /* client is disallowed from changing bb exits and sequencing in trace
+                * hook; if they change in bb for_trace, will be reflected here.
+                */
+               opnd_get_pc(instr_get_target(inst)) == md->blk_info[blk+1].info.tag)))) {
+
+            DOLOG(4, LOG_INTERP, {
+                loginst(dcontext, 4, inst, "end of bb");
+            }); 
+
+            /* Add jump that fixup_last_cti expects */
+            if (!instr_is_ubr(inst)) {
+                app_pc target;
+                if (instr_is_mbr(inst)) {
+                    target = get_ibl_routine(dcontext, IBL_LINKED, DEFAULT_IBL_TRACE(), 
+                                             get_ibl_branch_type(inst));
+                } else if (instr_is_cbr(inst)) {
+                    target = instr_get_translation(inst) + instr_length(dcontext, inst);
+                } else {
+                    target = opnd_get_pc(instr_get_target(inst));
+                }
+                ASSERT(target != NULL);
+                jmp = create_exit_jmp(dcontext, target, instr_get_translation(inst),
+                                      instr_branch_type(inst));
+                instrlist_postinsert(ilist, inst, jmp);
+                /* we're now done w/ end_pc: replace w/ end instr.
+                 * mangle() shouldn't remove the exit cti.
+                 */
+                md->blk_info[blk].bounds.end_instr = jmp;
+            } else
+                md->blk_info[blk].bounds.end_instr = inst;
+
+            blk++;
+            DOLOG(4, LOG_INTERP, {
+                if (blk < md->num_blks) {
+                    LOG(THREAD, LOG_MONITOR, 4, "starting next bb "PFX"\n",
+                        md->blk_info[blk].info.tag);
+                }
+            });
+            if (blk >= md->num_blks && next_inst != NULL) {
+                CLIENT_ASSERT(false, "unsupported trace modification: too many exits");
+                return false;
+            }
+        }
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+        /* PR 306761: we need to re-calculate md->blk_info[blk].info.num_exits,
+         * and then adjust after fixup_last_cti.
+         */
+        if (instr_will_be_exit_cti(inst))
+            md->blk_info[blk].info.num_exits++;
+#endif
+
+    }
+    if (blk < md->num_blks) {
+        ASSERT(!instr_is_ubr(instrlist_last(ilist)));
+        if (blk + 1 < md->num_blks) {
+            CLIENT_ASSERT(false, "unsupported trace modification: too few exits");
+            return false;
+        }
+        /* must have been no final exit cti: add final fall-through jmp */
+        jmp = create_exit_jmp(dcontext, fallthrough, fallthrough, 0);
+        /* FIXME PR 307284: support client modifying, replacing, or adding
+         * syscalls and ints: need to re-analyze.  Then we wouldn't
+         * need the md->final_exit_flags field anymore.
+         * For now we disallow.
+         */
+        if (found_syscall || found_int) {
+            instr_exit_branch_set_type(jmp, md->final_exit_flags);
+#ifdef WINDOWS
+            /* For INSTR_SHARED_SYSCALL, we set it pre-mangling, and it
+             * survives to here if the instr is not clobbered,
+             * and does not come from md->final_exit_flags
+             */
+            if (TEST(INSTR_SHARED_SYSCALL, instrlist_last(ilist)->flags)) {
+                instr_set_target(jmp, opnd_create_pc(shared_syscall_routine(dcontext)));
+                instr_set_our_mangling(jmp, true); /* undone by target set */
+            }
+            /* FIXME: test for linux too, but allowing ignorable syscalls */
+            if (!TESTANY(LINK_NI_SYSCALL_ALL IF_WINDOWS(| LINK_CALLBACK_RETURN),
+                         md->final_exit_flags) &&
+                !TEST(INSTR_SHARED_SYSCALL, instrlist_last(ilist)->flags)) {
+                CLIENT_ASSERT(false,
+                              "client modified or added a syscall or int: unsupported");
+                return false;
+            }
+#endif
+        }
+        instrlist_append(ilist, jmp);
+        md->blk_info[blk].bounds.end_instr = jmp;
+    } else {
+        CLIENT_ASSERT((!found_syscall && !found_int)
+                      /* On linux we allow ignorable syscalls in middle.
+                       * FIXME PR 307284: see notes above. */
+                      IF_LINUX(|| !TEST(LINK_NI_SYSCALL, md->final_exit_flags)),
+                      "client changed exit target where unsupported\n"
+                      "check if trace ends in a syscall or int");
+    }
+    ASSERT(instr_is_ubr(instrlist_last(ilist)));
+    if (found_syscall)
+        md->trace_flags |= FRAG_HAS_SYSCALL;
+    else
+        md->trace_flags &= ~FRAG_HAS_SYSCALL;
+
+    /* 2nd walk: mangle */
+    DOLOG(4, LOG_INTERP, {
+        LOG(THREAD, LOG_INTERP, 4, "trace ilist before mangling:\n");
+        instrlist_disassemble(dcontext, md->trace_tag, ilist, THREAD);
+    });
+    /* We do not need to remove nops since we never emitted */
+    mangle(dcontext, ilist, md->trace_flags, true/*mangle calls*/,
+           /* we're post-client so we don't need translations unless storing */
+           TEST(FRAG_HAS_TRANSLATION_INFO, md->trace_flags));
+    DOLOG(4, LOG_INTERP, {
+        LOG(THREAD, LOG_INTERP, 4, "trace ilist after mangling:\n");
+        instrlist_disassemble(dcontext, md->trace_tag, ilist, THREAD);
+    });
+
+    /* 3rd walk: stitch together delineated bbs */
+    blk = 0;
+    start_instr = instrlist_first(ilist);
+    for (inst = instrlist_first(ilist); inst != NULL; inst = next_inst) {
+        next_inst = instr_get_next(inst);
+        
+        if (inst == md->blk_info[blk].bounds.end_instr) {
+            /* Chain exit to point to next bb */
+            if (blk + 1 < md->num_blks) {
+                fixup_last_cti(dcontext, ilist, md->blk_info[blk+1].info.tag,
+                               /* FIXME: do eflags analysis for better x64
+                                * inline cmp; for now we pass 0 */
+                               0,
+                               md->trace_flags, NULL, NULL,
+                               TEST(FRAG_HAS_TRANSLATION_INFO, md->trace_flags),
+                               &num_exits_deleted,
+                               /* Only walk ilist between these instrs */
+                               start_instr, inst);
+#if defined(RETURN_AFTER_CALL) || defined(RCT_IND_BRANCH)
+                md->blk_info[blk].info.num_exits -= num_exits_deleted;
+#endif
+            }
+            blk++;
+            if (blk >= md->num_blks && next_inst != NULL) {
+                CLIENT_ASSERT(false, "unsupported trace modification: exits modified");
+                return false;
+            }
+            start_instr = next_inst;
+        }
+    }
+    if (blk < md->num_blks) {
+        CLIENT_ASSERT(false, "unsupported trace modification: cannot find all exits");
+        return false;
+    }
+    return true;
+}
+
+/****************************************************************************
+ * UTILITIES
+ */
+
+/* Converts instr_t EFLAGS_ flags to corresponding fragment_t FRAG_ flags,
+ * assuming that the instr_t flags correspond to the start of the fragment_t
+ */
+uint
+instr_eflags_to_fragment_eflags(uint instr_eflags)
+{
+    uint frag_eflags = 0;
+    if (instr_eflags == EFLAGS_WRITE_OF) {
+        /* this fragment writes OF before reading it 
+         * May still read other flags before writing them.
+         */
+        frag_eflags |= FRAG_WRITES_EFLAGS_OF;
+    } else if (instr_eflags == EFLAGS_WRITE_6) {
+        /* fragment writes all 6 prior to reading */
+        frag_eflags |= FRAG_WRITES_EFLAGS_6;
+        frag_eflags |= FRAG_WRITES_EFLAGS_OF;
+    }
+    return frag_eflags;
+}
+
+/* Returns one of these flags, defined in instr.h:
+ *   EFLAGS_WRITE_6   = writes all 6 flags before reading any
+ *   EFLAGS_WRITE_OF  = writes OF before reading it
+ *   EFLAGS_READ_6    = reads some of 6 before writing
+ *   EFLAGS_READ_OF   = reads OF before writing OF
+ *   0                = no information before 1st cti
+ */
+uint
+forward_eflags_analysis(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr)
+{
+    instr_t *in;
+    uint eflags_6 = 0; /* holds flags written so far (in read slots) */
+    int eflags_result = 0;
+    for (in = instr; in != NULL; in = instr_get_next_expanded(dcontext, ilist, in)) {
+        if (!instr_valid(in) || instr_is_cti(in)) {
+            /* give up */
+            break;
+        }
+        if (eflags_result != EFLAGS_WRITE_6 && eflags_result != EFLAGS_READ_OF)
+            eflags_result = eflags_analysis(in, eflags_result, &eflags_6);
+        DOLOG(4, LOG_INTERP, {
+            loginst(dcontext, 4, in, "forward_eflags_analysis");
+            LOG(THREAD, LOG_INTERP, 4, "\tinstr %x => %x\n",
+                instr_get_eflags(in), eflags_result);
+        });
+    }
+    return eflags_result;
+}
+
+/* This translates f's code into an instrlist_t and returns it.
+ * If buf is NULL:
+ *   The Instrs returned point into f's raw bits, so encode them 
+ *   before you delete f!
+ * Else, f's raw bits are copied into buf, and *bufsz is modified to
+ *   contain the total bytes copied
+ *   FIXME: should have release build checks and not just asserts where
+ *   we rely on caller to have big-enough buffer?
+ * If target_flags differ from f->flags in sharing and/or in trace-ness,
+ *   converts ibl and tls usage in f to match the desired target_flags.  
+ *   FIXME: converting from private to shared tls is not yet
+ *   implemented: we rely on -private_ib_in_tls for adding normal
+ *   private bbs to shared traces, and disallow any extensive mangling
+ *   (native_exec, selfmod) from becoming shared traces.
+ * The caller is responsible for destroying the instrlist and its instrs.
+ * If the fragment ends in an elided jmp, a new jmp instr is created, though
+ *   its bits field is NULL, allowing the caller to set it to do-not-emit if
+ *   trying to exactly duplicate or calculate the size, though most callers
+ *   will want to emit that jmp.  See decode_fragment_exact().
+ */
+
+/* We want to avoid low-loglevel disassembly when we're in the middle of disassembly */
+#define DF_LOGLEVEL(dc) \
+    (((dc) != GLOBAL_DCONTEXT && (dc)->in_opnd_disassemble) ? 6U : 4U)
+
+instrlist_t *
+decode_fragment(dcontext_t *dcontext, fragment_t *f, byte *buf, /*IN/OUT*/uint *bufsz,
+                uint target_flags, /*OUT*/uint *dir_exits, /*OUT*/uint *indir_exits)
+{
+    linkstub_t  *l;
+    cache_pc start_pc, stop_pc, pc, prev_pc = NULL, raw_start_pc;
+    instr_t *instr, *cti = NULL, *raw_instr;
+    instrlist_t *ilist = instrlist_create(dcontext);
+    byte *top_buf = NULL, *cur_buf = NULL;
+    app_pc target_tag;
+    uint num_bytes, offset;
+    uint num_dir = 0, num_indir = 0;
+    bool tls_to_dc;
+    bool shared_to_private =
+        TEST(FRAG_SHARED, f->flags) && !TEST(FRAG_SHARED, target_flags);
+#ifdef WINDOWS
+    /* The fragment could contain an ignorable sysenter instruction if
+     * the following conditions are satisfied. */
+    bool possible_ignorable_sysenter = DYNAMO_OPTION(ignore_syscalls) &&
+        (get_syscall_method() == SYSCALL_METHOD_SYSENTER) &&
+        /* FIXME Traces don't have FRAG_HAS_SYSCALL set so we can't filter on
+         * that flag for all fragments. We should propogate this flag from
+         * a BB to a trace. */
+        (TEST(FRAG_HAS_SYSCALL, f->flags) || TEST(FRAG_IS_TRACE, f->flags));
+#endif
+    instrlist_t intra_ctis;
+    coarse_info_t *info = NULL;
+    bool coarse_elided_ubrs = false;
+#ifdef X64
+    bool old_mode = get_x86_mode(dcontext);
+    /* for decoding and get_ibl routines we need the dcontext mode set */
+    set_x86_mode(dcontext, TEST(FRAG_32_BIT, f->flags));
+#endif
+
+    instrlist_init(&intra_ctis);
+
+    /* Now we need to go through f and make cti's for each of its exit cti's and
+     * non-exit cti's with off-fragment targets that need to be re-pc-relativized.
+     * The rest of the instructions can be lumped into raw instructions.
+     */
+    start_pc = FCACHE_ENTRY_PC(f);
+    pc = start_pc;
+    raw_start_pc = start_pc;
+    if (buf != NULL) {
+        cur_buf = buf;
+        top_buf = cur_buf;
+        ASSERT(bufsz != NULL);
+    }
+    /* Handle code after last exit but before stubs by allowing l to be NULL.
+     * Handle coarse-grain fake fragment_t by discovering exits as we go, with
+     * l being NULL the whole time.
+     */
+    if (TEST(FRAG_FAKE, f->flags)) {
+        ASSERT(TEST(FRAG_COARSE_GRAIN, f->flags));
+        info = get_fragment_coarse_info(f);
+        ASSERT(info != NULL);
+        coarse_elided_ubrs =
+            (info->persisted && TEST(PERSCACHE_ELIDED_UBR, info->flags))
+            || (!info->persisted && DYNAMO_OPTION(coarse_freeze_elide_ubr));
+        /* Assumption: coarse-grain fragments have no ctis w/ off-fragment targets
+         * that are not exit ctis
+         */
+        l = NULL;
+    } else
+        l = FRAGMENT_EXIT_STUBS(f);
+    while (true) {
+        uint l_flags;
+        cti = NULL;
+        if (l != NULL) {
+            stop_pc = EXIT_CTI_PC(f, l);
+        } else if (TEST(FRAG_FAKE, f->flags)) {
+            /* we don't know the size of f */
+            stop_pc = (cache_pc) UNIVERSAL_REGION_END;
+        } else {
+            /* fake fragment_t, or code between last exit but before stubs or padding */
+            stop_pc = fragment_body_end_pc(dcontext, f);
+            if (PAD_FRAGMENT_JMPS(f->flags) && stop_pc != raw_start_pc) {
+                /* We need to adjust stop_pc to account for any padding, only
+                 * way any code could get here is via client interface,
+                 * and there really is no nice way to distinguish it
+                 * from any padding we added.
+                 * PR 213005: we do not support decode_fragment() for bbs
+                 * that have code added beyond the last exit cti (we turn
+                 * off FRAG_COARSE_GRAIN and set FRAG_CANNOT_BE_TRACE).
+                 * Sanity check, make sure it at least looks like there is no
+                 * code here */
+                ASSERT(IS_SET_TO_DEBUG(raw_start_pc, stop_pc - raw_start_pc)); 
+                stop_pc = raw_start_pc;
+            }
+        }
+        IF_X64(ASSERT(TEST(FRAG_FAKE, f->flags) /* no copy made */ ||
+                      CHECK_TRUNCATE_TYPE_uint((stop_pc - raw_start_pc))));
+        num_bytes = (uint) (stop_pc - raw_start_pc);
+        LOG(THREAD, LOG_MONITOR, DF_LOGLEVEL(dcontext),
+            "decoding fragment from "PFX" to "PFX"\n", raw_start_pc, stop_pc);
+        if (num_bytes > 0) {
+            if (buf != NULL) {
+                if (TEST(FRAG_FAKE, f->flags)) {
+                    /* we don't know the size of f, so we copy later, though
+                     * we do point instrs into buf before we copy!
+                     */
+                } else {
+                    /* first copy entire sequence up to exit cti into buf
+                     * so we don't have to copy it in pieces if we find cti's, if we don't
+                     * find any we want one giant piece anyway
+                     */
+                    ASSERT(cur_buf + num_bytes < buf + *bufsz);
+                    memcpy(cur_buf, raw_start_pc, num_bytes);
+                    top_buf = cur_buf + num_bytes;
+                    LOG(THREAD, LOG_MONITOR, DF_LOGLEVEL(dcontext),
+                        "decode_fragment: copied "PFX"-"PFX" to "PFX"-"PFX"\n",
+                        raw_start_pc, raw_start_pc + num_bytes, cur_buf,
+                        cur_buf + num_bytes);
+                    /* cur_buf is incremented later -- it always points to start
+                     * of raw bytes for next-to-add-to-ilist instr, while
+                     * top_buf points to top of copied-to-buf data
+                     */
+                }
+            } else {
+                /* point at bits in code cache */
+                cur_buf = raw_start_pc;
+            }
+            /* now, we can't make a single raw instr for all that, there may
+             * be calls with off-fragment targets in there that need to be
+             * re-pc-relativized (instrumentation, etc. insert calls), or
+             * we may not even know where the exit ctis are (coarse-grain fragments),
+             * so walk through (original bytes!) and decode, looking for cti's
+             */
+            instr = instr_create(dcontext);
+            pc = raw_start_pc;
+            /* do we have to translate the store of xcx from tls to dcontext? 
+             * be careful -- there can be private bbs w/ indirect branches, so
+             * must see if this is a shared fragment we're adding
+             */
+            tls_to_dc = (shared_to_private && !DYNAMO_OPTION(private_ib_in_tls) &&
+                         /* if l==NULL (coarse src) we'll check for xcx every time */
+                         (l == NULL || LINKSTUB_INDIRECT(l->flags)));
+            do {
+#ifdef WINDOWS
+                cache_pc prev_decode_pc = prev_pc; /* store the address of the
+                                                    * previous decode, the instr
+                                                    * before the one 'pc'
+                                                    * currently points to *before*
+                                                    * the call to decode() just
+                                                    * below */
+#endif
+                /* For frozen coarse fragments, ubr eliding forces us to check
+                 * every instr for a potential next fragment start.  This is
+                 * expensive so users are advised to decode from app code if
+                 * possible (case 9325 -- need exact re-mangle + re-instrument),
+                 * though -coarse_pclookup_table helps.
+                 */
+                if (info != NULL && info->frozen &&
+                    coarse_elided_ubrs && pc != start_pc) {
+                    /* case 6532: check for ib stubs as we elide the jmp there too */
+                    bool stop = false;
+                    if (coarse_is_indirect_stub(pc)) {
+                        stop = true;
+                        LOG(THREAD, LOG_MONITOR, DF_LOGLEVEL(dcontext)-1,
+                            "\thit ib stub @"PFX"\n", pc);
+                    } else {
+                        app_pc tag = fragment_coarse_entry_pclookup(dcontext, info, pc);
+                        if (tag != NULL) {
+                            stop = true;
+                            LOG(THREAD, LOG_MONITOR, DF_LOGLEVEL(dcontext)-1,
+                                "\thit frozen tgt: "PFX"."PFX"\n", tag, pc);
+                        }
+                    }
+                    if (stop) {
+                        /* Add the ubr ourselves */
+                        ASSERT(cti == NULL);
+                        cti = INSTR_CREATE_jmp(dcontext, opnd_create_pc(pc));
+                        /* It's up to the caller to decide whether to mark this
+                         * as do-not-emit or not */
+                        /* Process as an exit cti */
+                        stop_pc = pc;
+                        pc = stop_pc;
+                        break;
+                    }
+                }
+                instr_reset(dcontext, instr);
+                prev_pc = pc;
+                pc = decode_cti(dcontext, pc, instr);
+#ifdef WINDOWS
+                /* Perform fixups for ignorable syscalls on XP & 2003. */
+                if (possible_ignorable_sysenter
+                    && instr_opcode_valid(instr)
+                    && instr_is_syscall(instr)) {
+
+                    /* We want to find the instr preceding the sysenter and have
+                     * it point to the post-sysenter instr in the trace, rather than
+                     * remain pointing to the post-sysenter instr in the BB.
+                     */
+                    instr_t *sysenter_prev;
+                    instr_t *sysenter_post;
+
+                    ASSERT(prev_decode_pc != NULL);
+                    LOG(THREAD, LOG_MONITOR, DF_LOGLEVEL(dcontext),
+                        "decode_fragment: sysenter found @"PFX"\n",
+                        instr_get_raw_bits(instr));
+
+                    /* create single raw instr for instructions up to the
+                     * sysenter EXCEPT for the immediately preceding instruction
+                     */
+                    offset = (int)(prev_decode_pc - raw_start_pc);
+                    ASSERT(offset > 0);
+                    raw_instr = instr_create(dcontext);
+                    /* point to buffer bits */
+                    instr_set_raw_bits(raw_instr, cur_buf, offset);
+                    instrlist_append(ilist, raw_instr);
+                    cur_buf += offset;
+
+                    /* Get the "mov" instr just before the sysenter. We know that
+                     * it's there because mangle put it there, so we can safely
+                     * decode at prev_decode_pc.
+                     */
+                    sysenter_prev = instr_create(dcontext);
+                    decode(dcontext, prev_decode_pc, sysenter_prev);
+                    ASSERT(instr_valid(instr) &&
+                           instr_is_mov_imm_to_tos(sysenter_prev));
+                    instrlist_append(ilist, sysenter_prev);
+                    cur_buf += instr_length(dcontext, sysenter_prev);
+
+                    /* Append the sysenter. */
+                    instr_set_raw_bits(instr, cur_buf, (int)(pc - prev_pc));
+                    instrlist_append(ilist, instr);
+                    instr_set_ok_to_mangle(instr, false);
+
+                    /* skip current instr -- the sysenter */
+                    cur_buf += (int)(pc - prev_pc);
+
+                    /* Decode the next instr -- the one after the sysenter. */
+                    sysenter_post = instr_create(dcontext);
+                    prev_decode_pc = pc;
+                    prev_pc = pc;
+                    pc = decode(dcontext, pc, sysenter_post);
+                    if (DYNAMO_OPTION(ignore_syscalls_follow_sysenter))
+                        ASSERT(!instr_is_cti(sysenter_post));
+                    raw_start_pc = pc;
+
+                    /* skip the post-sysenter instr */
+                    cur_buf += (int)(pc - prev_pc);
+
+                    instrlist_append(ilist, sysenter_post);
+                    /* Point the pre-sysenter mov to the post-sysenter instr. */
+                    instr_set_src(sysenter_prev, 0,
+                                  opnd_create_instr(sysenter_post));
+                    instr_set_ok_to_mangle(sysenter_prev, false);
+                    instr_set_ok_to_mangle(sysenter_post, false);
+
+                    DOLOG(DF_LOGLEVEL(dcontext), LOG_INTERP, {
+                        LOG(THREAD, LOG_INTERP, DF_LOGLEVEL(dcontext),
+                            "Post-sysenter -- F%d ("PFX") into:\n", f->id, f->tag);
+                        instrlist_disassemble(dcontext, f->tag, ilist, THREAD);
+                    });
+
+                    /* Set all local state so that we can fall-thru and correctly
+                     * process the post-sysenter instruction. Point instr to the
+                     * already decoded instruction, sysenter_post. At this point,
+                     * pc and raw_start_pc point to just after sysenter_post,
+                     * prev_pc points to sysenter_post, prev_decode_pc points to
+                     * the sysenter itself, and cur_buf points to post_sysenter.
+                     */
+                    instr = sysenter_post;
+                }
+#endif
+                /* look for a cti with an off-fragment target */
+                if (instr_opcode_valid(instr) && instr_is_cti(instr)) {
+                    bool separate_cti = false;
+                    bool re_relativize = false;
+                    DOLOG(DF_LOGLEVEL(dcontext), LOG_MONITOR, {
+                        loginst(dcontext, 4, instr, "decode_fragment: found non-exit cti");
+                    });
+                    if (TEST(FRAG_FAKE, f->flags)) {
+                        /* Case 8711: we don't know the size so we can't even
+                         * distinguish off-fragment from intra-fragment targets.
+                         * Thus we have to assume that any cti is an exit cti, and
+                         * make all fragments for which that is not true into
+                         * fine-grained.
+                         */
+                        /* Process this cti as an exit cti.  FIXME: we will then
+                         * re-copy the raw bytes from this cti to the end of the
+                         * fragment at the top of the next loop iter, but for
+                         * coarse-grain bbs that should be just one instr for cbr bbs
+                         * or none for others, so not worth doing anything about.
+                         */
+                        stop_pc = prev_pc;
+                        pc = stop_pc;
+                        break;
+                    }
+                    if (instr_is_return(instr) ||
+                        !opnd_is_near_pc(instr_get_target(instr))) {
+                        /* just leave it undecoded */
+#ifdef NATIVE_RETURN
+                        /* need to be able to find ret for fixup_last_cti */
+                        if (instr_is_return(instr))
+                            separate_cti = true;
+#endif
+                    } else if (instr_is_cti_short_rewrite(instr, prev_pc)) {
+                        /* Cti-short should only occur as exit ctis, which are
+                         * separated out unless we're decoding a fake fragment.  We
+                         * include this case for future use, as otherwise we'll
+                         * decode just the short cti and think it is an
+                         * intra-fragment cti.
+                         */
+                        ASSERT_NOT_REACHED();
+                        separate_cti = true;
+                        re_relativize = true;
+                    } else if (opnd_get_pc(instr_get_target(instr)) < start_pc ||
+                               opnd_get_pc(instr_get_target(instr)) > start_pc+f->size) {
+                        separate_cti = true;
+                        re_relativize = true;
+                        DOLOG(DF_LOGLEVEL(dcontext), LOG_MONITOR, {
+                            loginst(dcontext, 4, instr, "\tcti has off-fragment target");
+                        });
+                    } else {
+                        /* intra-fragment target: we'll change its target operand
+                         * from pc to instr_t in second pass, so remember it here
+                         */
+                        instr_t *clone = instr_clone(dcontext, instr);
+                        /* HACK: use note field! */
+                        instr_set_note(clone, (void *) instr);
+                        /* we leave the clone pointing at valid original raw bits */
+                        instrlist_append(&intra_ctis, clone);
+                        /* intra-fragment target */
+                        DOLOG(DF_LOGLEVEL(dcontext), LOG_MONITOR, {
+                            loginst(dcontext, 4, instr, "\tcti has intra-fragment target");
+                        });
+#ifdef NATIVE_RETURN
+                        if (instr_is_call(instr)) {
+                            /* assume it's an application call! */
+                            dcontext->num_calls_in_trace++;
+                        }
+#endif
+                        /* since the resulting instrlist could be manipulated,
+                         * we need to change the target operand from pc to instr_t.
+                         * that requires having this instr separated out now so
+                         * our clone-in-note-field hack above works.
+                         */
+                        separate_cti = true;
+                        re_relativize = false;
+                    }
+                    if (separate_cti) {
+                        /* create single raw instr for instructions up to the cti */
+                        offset = (int)(prev_pc - raw_start_pc);
+                        if (offset > 0) {
+                            raw_instr = instr_create(dcontext);
+                            /* point to buffer bits */
+                            instr_set_raw_bits(raw_instr, cur_buf, offset);
+                            instrlist_append(ilist, raw_instr);
+                            cur_buf += offset;
+                            raw_start_pc = prev_pc;
+                        }
+                        /* now append cti, indicating that relative target must be
+                         * re-encoded, and that it is not an exit cti
+                         */
+                        instr_set_ok_to_mangle(instr, false);
+                        if (re_relativize)
+                            instr_set_raw_bits_valid(instr, false);
+                        else
+                            instr_set_raw_bits(instr, cur_buf, (int)(pc - prev_pc));
+                        instrlist_append(ilist, instr);
+                        /* include buf for off-fragment cti, to simplify assert below */
+                        cur_buf += (int)(pc - prev_pc);
+                        raw_start_pc = pc;
+                        /* create new instr for future fast decodes */
+                        instr = instr_create(dcontext);
+                    }
+                } /* is cti */
+                /* instr_is_tls_xcx_spill won't upgrade from level 1 */
+                else if (tls_to_dc && instr_is_tls_xcx_spill(instr)) {
+                    /* shouldn't get here for x64, where everything uses tls */
+                    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+                    LOG(THREAD, LOG_MONITOR, DF_LOGLEVEL(dcontext),
+                        "mangling xcx save from tls to dcontext\n");
+                    /* create single raw instr for instructions up to the xcx save */
+                    offset = (int)(prev_pc - raw_start_pc);
+                    if (offset > 0) {
+                        raw_instr = instr_create(dcontext);
+                        /* point to buffer bits */
+                        instr_set_raw_bits(raw_instr, cur_buf, offset);
+                        instrlist_append(ilist, raw_instr);
+                        cur_buf += offset;
+                        raw_start_pc = prev_pc;
+                    }
+                    /* now append our new xcx save */
+                    instrlist_append(ilist,
+                                     instr_create_save_to_dcontext
+                                     (dcontext, REG_XCX, XCX_OFFSET));
+                    /* make sure skip current instr */
+                    cur_buf += (int)(pc - prev_pc);
+                    raw_start_pc = pc;
+                }
+#ifdef X64
+                else if (instr_has_rel_addr_reference(instr)) {
+                    /* We need to re-relativize, which is done automatically only for
+                     * level 1 instrs (PR 251479), and only when raw bits point to
+                     * their original location.  We assume that all the if statements
+                     * above end up creating a high-level instr, so a cti w/ a
+                     * rip-rel operand is already covered.
+                     */
+                    /* create single raw instr for instructions up to this one */
+                    offset = (int)(prev_pc - raw_start_pc);
+                    if (offset > 0) {
+                        raw_instr = instr_create(dcontext);
+                        /* point to buffer bits */
+                        instr_set_raw_bits(raw_instr, cur_buf, offset);
+                        instrlist_append(ilist, raw_instr);
+                        cur_buf += offset;
+                        raw_start_pc = prev_pc;
+                    }
+                    /* should be valid right now since pointing at original bits */
+                    ASSERT(instr_rip_rel_valid(instr));
+                    if (buf != NULL) {
+                        /* re-relativize into the new buffer */
+                        DEBUG_DECLARE(byte *nxt =)
+                            instr_encode(dcontext, instr, cur_buf);
+                        instr_set_raw_bits(instr, cur_buf, (int)(pc - prev_pc));
+                        instr_set_rip_rel_valid(instr, true);
+                        ASSERT(nxt != NULL);
+                    }
+                    instrlist_append(ilist, instr);
+                    cur_buf += (int)(pc - prev_pc);
+                    raw_start_pc = pc;
+                    /* create new instr for future fast decodes */
+                    instr = instr_create(dcontext);
+                }
+#endif
+            } while (pc < stop_pc);
+            DODEBUG({
+                if (pc != stop_pc) {
+                    LOG(THREAD, LOG_MONITOR, DF_LOGLEVEL(dcontext),
+                        "PC "PFX", stop_pc "PFX"\n", pc, stop_pc);
+                }
+            });
+            ASSERT(pc == stop_pc);
+
+            /* create single raw instr for rest of instructions up to exit cti */
+            if (pc > raw_start_pc) {
+                instr_reset(dcontext, instr);
+                /* point to buffer bits */
+                offset = (int)(pc - raw_start_pc);
+                if (offset > 0) {
+                    instr_set_raw_bits(instr, cur_buf, offset);
+                    instrlist_append(ilist, instr);
+                    cur_buf += offset;
+                }
+                if (buf != NULL && TEST(FRAG_FAKE, f->flags)) {
+                    /* Now that we know the size we can copy into buf.
+                     * We have been incrementing cur_buf all along, though
+                     * we didn't have contents there.
+                     */
+                    ASSERT(top_buf < cur_buf);
+                    IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint((cur_buf - top_buf))));
+                    num_bytes = (uint) (cur_buf - top_buf);
+                    ASSERT(cur_buf + num_bytes < buf + *bufsz);
+                    memcpy(cur_buf, raw_start_pc, num_bytes);
+                    top_buf = cur_buf + num_bytes;
+                    LOG(THREAD, LOG_MONITOR, DF_LOGLEVEL(dcontext),
+                        "decode_fragment: copied "PFX"-"PFX" to "PFX"-"PFX"\n",
+                        raw_start_pc, raw_start_pc + num_bytes, cur_buf,
+                        cur_buf + num_bytes);
+                }
+                ASSERT(buf == NULL || cur_buf == top_buf);
+            } else {
+                /* will reach here if had a processed instr (off-fragment target, etc.)
+                 * immediately prior to exit cti, so now don't need instr -- an
+                 * example (in absence of clients) is trampoline to interception code
+                 */
+                instr_destroy(dcontext, instr);
+            }
+        }
+
+        if (l == NULL && !TEST(FRAG_FAKE, f->flags))
+            break;
+
+        /* decode the exit branch */
+        if (cti != NULL) {
+            /* already created */
+            instr = cti;
+            ASSERT(info != NULL && info->frozen && instr_is_ubr(instr));
+            raw_start_pc = pc;
+        } else {
+            instr = instr_create(dcontext);
+            raw_start_pc = decode(dcontext, stop_pc, instr);
+            ASSERT(raw_start_pc != NULL); /* our own code! */
+            /* pc now points into fragment! */
+        }
+        ASSERT(instr_is_ubr(instr) || instr_is_cbr(instr));
+        /* replace fcache target with target_tag and add to fragment */
+        if (l == NULL) {
+            app_pc instr_tgt;
+            /* Ensure we get proper target for short cti sequence */
+            if (instr_is_cti_short_rewrite(instr, stop_pc))
+                remangle_short_rewrite(dcontext, instr, stop_pc, 0/*same target*/);
+            instr_tgt = opnd_get_pc(instr_get_target(instr));
+            ASSERT(TEST(FRAG_COARSE_GRAIN, f->flags));
+            if (cti == NULL && coarse_is_entrance_stub(instr_tgt)) {
+                target_tag = entrance_stub_target_tag(instr_tgt);
+                l_flags = LINK_DIRECT;
+                /* FIXME; try to get LINK_JMP vs LINK_CALL vs fall-through? */
+                LOG(THREAD, LOG_MONITOR, DF_LOGLEVEL(dcontext)-1,
+                    "\tstub tgt: "PFX" => "PFX"\n", instr_tgt, target_tag);
+            } else if (instr_tgt == raw_start_pc /*target next instr*/
+                       /* could optimize by not checking for stub if
+                        * coarse_elided_ubrs but we need to know whether ALL
+                        * ubrs were elided, which we don't know as normally
+                        * entire-bb-ubrs are not elided (case 9677).
+                        * plus now that we elide jmp-to-ib-stub we must check.
+                        */
+                       && coarse_is_indirect_stub(instr_tgt)) {
+                ibl_type_t ibl_type;
+                DEBUG_DECLARE(bool is_ibl;)
+                target_tag = coarse_indirect_stub_jmp_target(instr_tgt);
+                l_flags = LINK_INDIRECT;
+                DEBUG_DECLARE(is_ibl = )
+                    get_ibl_routine_type_ex(dcontext, target_tag, &ibl_type
+                                            _IF_X64(NULL));
+                ASSERT(is_ibl);
+                l_flags |= ibltype_to_linktype(ibl_type.branch_type);
+                LOG(THREAD, LOG_MONITOR, DF_LOGLEVEL(dcontext)-1,
+                    "\tind stub tgt: "PFX" => "PFX"\n", instr_tgt, target_tag);
+            } else {
+                target_tag = fragment_coarse_entry_pclookup(dcontext, info, instr_tgt);
+                /* Only frozen units don't jump through stubs */
+                ASSERT(info != NULL && info->frozen);
+                ASSERT(target_tag != NULL);
+                l_flags = LINK_DIRECT;
+                LOG(THREAD, LOG_MONITOR, DF_LOGLEVEL(dcontext)-1,
+                    "\tfrozen tgt: "PFX"."PFX"\n", target_tag, instr_tgt);
+            }
+        } else {
+            target_tag = EXIT_TARGET_TAG(dcontext, f, l);
+            l_flags = l->flags;
+        }
+        if (LINKSTUB_DIRECT(l_flags))
+            num_dir++;
+        else
+            num_indir++;
+        ASSERT(target_tag != NULL);
+        if (instr_is_cti_short_rewrite(instr, stop_pc)) {
+            raw_start_pc = remangle_short_rewrite(dcontext, instr, stop_pc, target_tag);
+        } else {
+            app_pc new_target = target_tag;
+            /* don't point to fcache bits */
+            instr_set_raw_bits_valid(instr, false);
+            LOG(THREAD, LOG_MONITOR, DF_LOGLEVEL(dcontext)-1,
+                "decode_fragment exit_cti: pc="PFX" l->target_tag="PFX" l->flags=0x%x\n",
+                stop_pc, target_tag, l_flags);
+
+            /* need to propagate exit branch type flags, 
+             * instr_t flag copied from old fragment linkstub
+             * TODO: when ibl targets are different this won't be necessary 
+             */
+            instr_exit_branch_set_type(instr, linkstub_propagatable_flags(l_flags));
+
+            /* convert to proper ibl */
+            if (is_indirect_branch_lookup_routine(dcontext, target_tag)) {
+                DEBUG_DECLARE(app_pc old_target = new_target;)
+                new_target = get_alternate_ibl_routine(dcontext, target_tag,
+                                                       target_flags);
+                ASSERT(new_target != NULL);
+
+                /* for stats on traces, we assume if target_flags contains
+                 * FRAG_IS_TRACE then we are extending a trace
+                 */
+                DODEBUG({
+                    LOG(THREAD, LOG_MONITOR, DF_LOGLEVEL(dcontext)-1,
+                        "%s: %s ibl_routine "PFX" with %s_target="PFX"\n",
+                        TEST(FRAG_IS_TRACE, target_flags) ? 
+                        "extend_trace" : "decode_fragment",
+                        new_target == old_target ? "maintaining" : "replacing",
+                        old_target, new_target == old_target ? "old" : "new", new_target);
+                    STATS_INC(num_traces_ibl_extended);
+                });
+#ifdef WINDOWS
+                DOSTATS({
+                    if (TEST(FRAG_IS_TRACE, target_flags) &&
+                        old_target == shared_syscall_routine(dcontext))
+                        STATS_INC(num_traces_shared_syscall_extended);
+                });
+#endif
+            }     
+
+            instr_set_target(instr, opnd_create_pc(new_target));
+
+            if (instr_is_cti_short(instr)) {
+                /* make sure non-mangled short ctis, which are generated by
+                 * us and never left there from apps, are not marked as exit ctis
+                 */
+                instr_set_ok_to_mangle(instr, false);
+            }
+        }
+        instrlist_append(ilist, instr);
+#ifdef CUSTOM_EXIT_STUBS
+        if (l != NULL && l->fixed_stub_offset > 0)
+            regenerate_custom_exit_stub(dcontext, instr, l, f);
+#endif
+
+        if (TEST(FRAG_FAKE, f->flags)) {
+            /* Assumption: coarse-grain bbs have 1 ind exit or 2 direct,
+             * and no code beyond the last exit!  Of course frozen bbs
+             * can have their final jmp elided, which we handle above.
+             */
+            if (instr_is_ubr(instr)) {
+                break;
+            }
+        }
+        if (l != NULL) /* if NULL keep going: discovering exits as we go */
+            l = LINKSTUB_NEXT_EXIT(l);
+    } /* end while(true) loop through exit stubs */
+
+    /* now fix up intra-trace cti targets */
+    if (instrlist_first(&intra_ctis) != NULL) {
+        /* We have to undo all of our level 0 blocks by expanding.
+         * Any instrs that need re-relativization should already be
+         * separate, so this should not affect rip-rel instrs.           
+         */
+        int offs = 0;
+        for (instr = instrlist_first_expanded(dcontext, ilist); instr != NULL;
+             instr = instr_get_next_expanded(dcontext, ilist, instr)) {
+            for (cti = instrlist_first(&intra_ctis); cti != NULL;
+                 cti = instr_get_next(cti)) {
+                /* The clone we put in intra_ctis has raw bits equal to the
+                 * original bits, so its target will be in original fragment body.
+                 * We can't rely on the raw bits of the new instrs (since the
+                 * non-level-0 ones may have allocated raw bits) so we
+                 * calculate a running offset as we go.
+                 */
+                if (opnd_get_pc(instr_get_target(cti)) - start_pc == offs) {
+                    /* cti targets this instr */
+                    instr_t *real_cti = (instr_t *) instr_get_note(cti);
+                    /* PR 333691: do not preserve raw bits of real_cti, since
+                     * instrlist may change (e.g., inserted nops).  Must re-encode
+                     * once instrlist is finalized.
+                     */
+                    instr_set_target(real_cti, opnd_create_instr(instr));
+                    DOLOG(DF_LOGLEVEL(dcontext), LOG_MONITOR, {
+                        loginst(dcontext, 4, real_cti, "\tre-set intra-fragment target");
+                    });
+                    break;
+                }
+            }
+            offs += instr_length(dcontext, instr);
+        }
+    }
+
+    instrlist_clear(dcontext,&intra_ctis);
+    DOLOG(DF_LOGLEVEL(dcontext), LOG_INTERP, {
+        LOG(THREAD, LOG_INTERP, DF_LOGLEVEL(dcontext),
+            "Decoded F%d ("PFX"."PFX") into:\n",
+            f->id, f->tag, FCACHE_ENTRY_PC(f));
+        instrlist_disassemble(dcontext, f->tag, ilist, THREAD);
+    });
+
+#ifdef X64
+    set_x86_mode(dcontext, old_mode);
+#endif
+
+    if (dir_exits != NULL)
+        *dir_exits = num_dir;
+    if (indir_exits != NULL)
+        *indir_exits = num_indir;
+    if (buf != NULL) {
+        IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint((top_buf - buf))));
+        *bufsz = (uint) (top_buf - buf);
+    }
+    return ilist;
+}
+#undef DF_LOGLEVEL
+
+/* Just like decode_fragment() but marks any instrs missing in the cache
+ * as do-not-emit
+ */
+instrlist_t *
+decode_fragment_exact(dcontext_t *dcontext, fragment_t *f, byte *buf,
+                      /*IN/OUT*/uint *bufsz, uint target_flags,
+                      /*OUT*/uint *dir_exits, /*OUT*/uint *indir_exits)
+{
+    instrlist_t *ilist =
+        decode_fragment(dcontext, f, buf, bufsz, target_flags, dir_exits, indir_exits);
+    /* If the final jmp was elided we do NOT want to count it in the size! */
+    if (instr_get_raw_bits(instrlist_last(ilist)) == NULL) {
+        instr_set_ok_to_emit(instrlist_last(ilist), false);
+    }
+    return ilist;
+}
+
+/* Makes a new copy of fragment f
+ * If replace is true,
+ *   removes f from the fcache and adds the new copy in its place
+ * Else
+ *   creates f as an invisible fragment (caller is responsible for linking
+ *   the new fragment!)
+ */
+fragment_t *
+copy_fragment(dcontext_t *dcontext, fragment_t *f, bool replace)
+{
+    instrlist_t *trace = instrlist_create(dcontext);
+    instr_t     *instr;
+    uint    *trace_buf;
+    int       trace_buf_top; /* index of next free location in trace_buf */
+    linkstub_t  *l;
+    byte *p;
+    cache_pc  start_pc;
+    int       num_bytes;
+    fragment_t  *new_f;
+    void      *vmlist = NULL;
+    app_pc    target_tag;
+    DEBUG_DECLARE(bool ok;)
+
+    trace_buf = heap_alloc(dcontext, f->size*2 HEAPACCT(ACCT_FRAGMENT));
+
+    start_pc = FCACHE_ENTRY_PC(f);
+    trace_buf_top = 0;
+    p = ((byte *)trace_buf) + trace_buf_top;
+
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false)); /* must re-relativize when copying! */
+
+    for (l = FRAGMENT_EXIT_STUBS(f); l; l = LINKSTUB_NEXT_EXIT(l)) {
+        /* Copy the instruction bytes up to (but not including) the first
+         * control-transfer instruction.  ***WARNING*** This code assumes
+         * that the first link stub corresponds to the first exit branch
+         * in the body. */
+        IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint((EXIT_CTI_PC(f, l) - start_pc))));
+        num_bytes = (uint) (EXIT_CTI_PC(f, l) - start_pc);
+        if (num_bytes > 0) {
+            memcpy(p, (byte *)start_pc, num_bytes);
+            trace_buf_top += num_bytes;
+            start_pc += num_bytes;
+
+            /* build a mongo instruction corresponding to the copied instructions */
+            instr = instr_create(dcontext);
+            instr_set_raw_bits(instr, p, num_bytes);
+            instrlist_append(trace, instr);
+        }
+
+        /* decode the exit branch */
+        instr = instr_create(dcontext);
+        p = decode(dcontext, (byte *)EXIT_CTI_PC(f, l), instr);
+        ASSERT(p != NULL); /* our own code! */
+        /* p now points into fragment! */
+        ASSERT(instr_is_ubr(instr) || instr_is_cbr(instr));
+        /* Replace cache_pc target with target_tag and add to trace.  For
+         * an indirect branch, the target_tag is zero. */
+        target_tag = EXIT_TARGET_TAG(dcontext, f, l);
+        ASSERT(target_tag);
+        if (instr_is_cti_short_rewrite(instr, EXIT_CTI_PC(f, l))) {
+            p = remangle_short_rewrite(dcontext, instr,
+                                       EXIT_CTI_PC(f, l),
+                                       target_tag);
+        } else {
+            /* no short ctis that aren't mangled should be exit ctis */
+            ASSERT(!instr_is_cti_short(instr));
+            instr_set_target(instr, opnd_create_pc(target_tag));
+        }
+        instrlist_append(trace, instr);
+        start_pc += (p - (byte *)EXIT_CTI_PC(f, l));
+    }
+
+    /* emit as invisible fragment */
+    /* We don't support shared fragments, where vm_area_add_to_list can fail */
+    ASSERT_NOT_IMPLEMENTED(!TEST(FRAG_SHARED, f->flags));
+    DEBUG_DECLARE(ok =)
+        vm_area_add_to_list(dcontext, f->tag, &vmlist, f->flags, f, false/*no locks*/);
+    ASSERT(ok); /* should never fail for private fragments */
+    new_f = emit_invisible_fragment(dcontext, f->tag, trace, f->flags, vmlist);
+    if (replace) {
+        /* link and replace old fragment */
+        shift_links_to_new_fragment(dcontext, f, new_f);
+        fragment_replace(dcontext, f, new_f);
+    } else {
+        /* caller is responsible for linking new fragment */
+    }
+
+    ASSERT(new_f->flags == f->flags);
+
+    fragment_copy_data_fields(dcontext, f, new_f);
+
+#ifdef DEBUG
+    if (stats->loglevel > 1) {
+        LOG(THREAD, LOG_ALL, 2, "Copying F%d to F%d\n", f->id, new_f->id);
+        disassemble_fragment(dcontext, f, stats->loglevel < 3);
+        disassemble_fragment(dcontext, new_f, stats->loglevel < 3);
+    }
+#endif /* DEBUG */
+
+    heap_free(dcontext, trace_buf, f->size*2 HEAPACCT(ACCT_FRAGMENT));
+    /* free the instrlist_t elements */
+    instrlist_clear_and_destroy(dcontext, trace);
+    if (replace) {
+        fragment_delete(dcontext, f, FRAGDEL_NO_OUTPUT | FRAGDEL_NO_UNLINK |
+                        FRAGDEL_NO_HTABLE);
+        STATS_INC(num_fragments_deleted_copy_and_replace);
+    }
+    return new_f;
+}
+
+/* Used when the code cache is enlarged by copying to a larger space,
+ * and all of the relative ctis that target outside the cache need
+ * to be shifted. Additionally, sysenter-related patching for ignore-syscalls
+ * on XP/2003 is performed here, as the absolute code cache address pushed
+ * onto the stack must be updated.
+ * Assumption: old code cache has been copied to TOP of new cache, so to
+ * detect for ctis targeting outside of old cache can look at new cache
+ * start plus old cache size.
+ */
+void
+shift_ctis_in_fragment(dcontext_t *dcontext, fragment_t *f, ssize_t shift,
+                       cache_pc fcache_start, cache_pc fcache_end,
+                       size_t old_size)
+{
+    cache_pc pc, prev_pc = NULL;
+    cache_pc start_pc = FCACHE_ENTRY_PC(f);
+    cache_pc stop_pc = fragment_stubs_end_pc(f);
+    /* get what would have been end of cache if just shifted not resized */
+    cache_pc fcache_old_end = fcache_start + old_size;
+#ifdef WINDOWS
+    /* The fragment could contain an ignorable sysenter instruction if
+     * the following conditions are satisfied. */
+    bool possible_ignorable_sysenter = DYNAMO_OPTION(ignore_syscalls) &&
+        (get_syscall_method() == SYSCALL_METHOD_SYSENTER) &&
+        /* FIXME Traces don't have FRAG_HAS_SYSCALL set so we can't filter on
+         * that flag for all fragments. */
+        (TEST(FRAG_HAS_SYSCALL, f->flags) || TEST(FRAG_IS_TRACE, f->flags));
+#endif
+    instr_t instr;
+    instr_init(dcontext, &instr);
+
+    pc = start_pc;
+    while (pc < stop_pc) {
+#ifdef WINDOWS
+        cache_pc prev_decode_pc = prev_pc; /* store the address of the
+                                            * previous decode, the instr
+                                            * before the one 'pc'
+                                            * currently points to *before*
+                                            * the call to decode_cti() just
+                                            * below */
+#endif
+        prev_pc = pc;
+        instr_reset(dcontext, &instr);
+        pc = (cache_pc) decode_cti(dcontext, (byte*)pc, &instr);
+#ifdef WINDOWS
+        /* Perform fixups for sysenter instrs when ignorable syscalls is used on
+         * XP & 2003. These are not cache-external fixups, but it's convenient &
+         * efficient to perform them here since decode_cti() is called on every
+         * instruction, allowing identification of sysenters without additional
+         * decoding.
+         */
+        if (possible_ignorable_sysenter
+            && instr_opcode_valid(&instr)
+            && instr_is_syscall(&instr)) {
+
+            cache_pc next_pc;
+            app_pc target;
+            DEBUG_DECLARE(app_pc old_target;)
+            DEBUG_DECLARE(cache_pc encode_nxt;)
+
+            /* Peek up to find the "mov $post-sysenter -> (%xsp)" */
+            instr_reset(dcontext, &instr);
+            next_pc = decode(dcontext, prev_decode_pc, &instr);
+            ASSERT(next_pc == prev_pc);
+            LOG(THREAD, LOG_MONITOR, 4,
+                "shift_ctis_in_fragment: pre-sysenter mov found @"PFX"\n",
+                instr_get_raw_bits(&instr));
+            ASSERT(instr_is_mov_imm_to_tos(&instr));
+            target = instr_get_raw_bits(&instr) + instr_length(dcontext, &instr)
+                + (pc - prev_pc);
+            DODEBUG(old_target = (app_pc) opnd_get_immed_int(instr_get_src(&instr, 0)););
+            /* PR 253943: we don't support sysenter in x64 */
+            IF_X64(ASSERT_NOT_IMPLEMENTED(false)); /* can't have 8-byte imm-to-mem */
+            instr_set_src(&instr, 0, opnd_create_immed_int((ptr_int_t)target, OPSZ_4));
+            ASSERT(old_target + shift == target);
+            LOG(THREAD, LOG_MONITOR, 4,
+                "shift_ctis_in_fragment: pre-sysenter mov now pts to @"PFX"\n",
+                target);
+            DEBUG_DECLARE(encode_nxt = ) instr_encode(dcontext, &instr, prev_decode_pc);
+            /* must not change size! */
+            ASSERT(encode_nxt != NULL && encode_nxt == next_pc);
+        }
+        /* The following 'if' won't get executed since a sysenter isn't
+         * a CTI instr, so we don't need an else. We do need to take care
+         * that any 'else' clauses are added after the 'if' won't trigger
+         * on a sysenter either.
+         */
+#endif
+        /* look for a pc-relative cti (including exit ctis) w/ out-of-cache
+         * target (anything in-cache is fine, the whole cache was moved)
+         */
+        if (instr_is_cti(&instr) &&
+            /* only ret, ret_far, and iret don't have targets, and
+             * we really shouldn't see them, except possibly if they
+             * are inserted through instrumentation, so go ahead and
+             * check num srcs
+             */
+            instr_num_srcs(&instr) > 0 &&
+            opnd_is_near_pc(instr_get_target(&instr))) {
+            app_pc target = opnd_get_pc(instr_get_target(&instr));
+            if (target < fcache_start || target > fcache_old_end) {
+                DEBUG_DECLARE(byte *nxt_pc;)
+                /* re-encode instr w/ new pc-relative target */
+                instr_set_raw_bits_valid(&instr, false);
+                instr_set_target(&instr, opnd_create_pc(target - shift));
+                DEBUG_DECLARE(nxt_pc = ) instr_encode(dcontext, &instr, prev_pc);
+                /* must not change size! */
+                ASSERT(nxt_pc != NULL && nxt_pc == pc);
+#ifdef DEBUG
+                if ((stats->logmask & LOG_CACHE) != 0) {
+                    loginst(dcontext, 5, &instr,
+                            "shift_ctis_in_fragment: found cti w/ out-of-cache target");
+                }
+#endif
+            }
+        }
+    }
+    instr_free(dcontext, &instr);
+}
+
+#ifdef PROFILE_RDTSC
+/* Add profile call to front of the trace in dc
+ * Must call finalize_profile_call and pass it the fragment_t*
+ * once the trace is turned into a fragment to fix up a few profile
+ * call instructions.
+ */
+void
+add_profile_call(dcontext_t *dcontext)
+{
+    monitor_data_t *md = (monitor_data_t *) dcontext->monitor_field;
+    instrlist_t *trace = &(md->trace);
+    byte *p = ((byte *)md->trace_buf) + md->trace_buf_top;
+    instr_t *instr;
+    uint num_bytes = profile_call_size();
+    ASSERT(num_bytes + md->trace_buf_top < md->trace_buf_size);
+
+    insert_profile_call((cache_pc)p);
+
+    /* use one giant BINARY instruction to hold everything,
+     * to keep dynamo from interpreting the cti instructions as real ones
+     */
+    instr = instr_create(dcontext);
+    instr_set_raw_bits(instr, p, num_bytes);
+    instrlist_prepend(trace, instr);
+
+    md->trace_buf_top += num_bytes;
+}
+#endif
+
+/* emulates the effects of the instruction at pc with the state in mcontext
+ * limited right now to only mov instructions
+ * returns NULL if failed or not yet implemented, else returns the pc of the next instr.
+ */
+app_pc
+emulate(dcontext_t *dcontext, app_pc pc, dr_mcontext_t *mc)
+{
+    instr_t instr;
+    app_pc next_pc = NULL;
+    uint opc;
+    instr_init(dcontext, &instr);
+    next_pc = decode(dcontext, pc, &instr);
+    if (!instr_valid(&instr)) {
+        next_pc = NULL;
+        goto emulate_failure;
+    }
+    DOLOG(2, LOG_INTERP, { loginst(dcontext, 2, &instr, "emulating"); });
+    opc = instr_get_opcode(&instr);
+    if (opc == OP_mov_st) {
+        opnd_t src = instr_get_src(&instr, 0);
+        opnd_t dst = instr_get_dst(&instr, 0);
+        reg_t *target;
+        reg_t val;
+        uint sz = opnd_size_in_bytes(opnd_get_size(dst));
+        ASSERT(opnd_is_memory_reference(dst));
+        if (sz != 4 IF_X64(&& sz != 8)) {
+            next_pc = NULL;
+            goto emulate_failure;
+        }
+        target = (reg_t *) opnd_compute_address(dst, mc);
+        if (opnd_is_reg(src)) {
+            val = reg_get_value(opnd_get_reg(src), mc);
+        } else if (opnd_is_immed_int(src)) {
+            val = (reg_t) opnd_get_immed_int(src);
+        } else {
+            next_pc = NULL;
+            goto emulate_failure;
+        }
+        DODEBUG({
+            uint prot;
+            ASSERT(get_memory_info((app_pc)target, NULL, NULL, &prot));
+            ASSERT(TEST(MEMPROT_WRITE, prot));
+        });
+        LOG(THREAD, LOG_INTERP, 2, "\temulating store by writing "PFX" to "PFX"\n",
+            val, target);
+        if (sz == 4)
+            *((int*)target) = (int) val;
+#ifdef X64
+        else if (sz == 8)
+            *target = val;
+#endif
+    } else if (opc == OP_inc || opc == OP_dec) {
+        opnd_t src = instr_get_src(&instr, 0);
+        reg_t *target;
+        uint sz = opnd_size_in_bytes(opnd_get_size(src));
+        if (sz != 4 IF_X64(&& sz != 8)) {
+            next_pc = NULL;
+            goto emulate_failure;
+        }
+        /* FIXME: handle changing register value */
+        ASSERT(opnd_is_memory_reference(src));
+        /* FIXME: change these to take in dr_mcontext_t* ? */
+        target = (reg_t *) opnd_compute_address(src, mc);
+        DODEBUG({
+            uint prot;
+            ASSERT(get_memory_info((app_pc)target, NULL, NULL, &prot));
+            ASSERT(TEST(MEMPROT_WRITE, prot));
+        });
+        LOG(THREAD, LOG_INTERP, 2, "\temulating %s to "PFX"\n",
+            opc == OP_inc ? "inc" : "dec", target);
+        if (sz == 4) {
+            if (opc == OP_inc)
+                (*((int*)target))++;
+            else
+                (*((int*)target))--;
+        }
+#ifdef X64
+        else if (sz == 8) {
+            if (opc == OP_inc)
+                (*target)++;
+            else
+                (*target)--;
+        }
+#endif
+    }
+ emulate_failure:
+    instr_free(dcontext, &instr);
+    return next_pc;
+}
diff --git a/core/x86/loadtoconst.c b/core/x86/loadtoconst.c
new file mode 100644
index 000000000..418c138cb
--- /dev/null
+++ b/core/x86/loadtoconst.c
@@ -0,0 +1,1990 @@
+/* **********************************************************
+ * Copyright (c) 2002-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+
+#ifdef LOAD_TO_CONST
+
+#include "globals.h"
+#include "instrlist.h"
+#include "arch.h"
+#include "instr.h"
+#include "instr_create.h"
+#include "decode.h"
+#include "decode_fast.h"
+#include "disassemble.h"
+#include "proc.h"
+#include "fragment.h"
+#include "instrument.h"
+#include "emit.h"
+#include "link.h"
+
+
+#include <string.h> /* for memset */
+
+#include "loadtoconst.h"
+
+#ifdef SIDELINE
+fragment_t *frags_waiting_LTC[MAX_TRACES_WAITING_FOR_LTC];
+int num_frags_waiting_LTC;
+DECLARE_CXTSWPROT_VAR(mutex_t waiting_LTC_lock, INIT_LOCK_FREE(waiting_LTC_lock));
+#endif
+
+#ifdef LTC_STATS
+int safe_taken,opt_taken,addrs_analyzed,addrs_made_const,traces_analyzed,addrs_seen;
+#endif
+
+#define NUM_TMP_OPNDS 200
+static opnd_t tmpOpnds[NUM_TMP_OPNDS];
+
+void
+analyze_memrefs(dcontext_t *dcontext, app_pc tag, instrlist_t *trace)
+{
+    instr_t *instr,*writechecker,*accesschecker,*oldtracetop;
+    int a,reg,removalpossibilities=0,instances_of_address;
+    bool regs_modified[8],address_written,regs_written,first_instance_of_address;
+    bool has_back_arc;
+    opnd_t mem_access;
+
+
+    LOG(THREAD, LOG_OPTS,3, "in analyze_memrefs\n");
+#ifdef DEBUG
+    LOG(THREAD, LOG_OPTS,3, "before analyze_memrefs optimization:\n");
+    if (stats->loglevel >= 3)
+        instrlist_disassemble(dcontext, 0, trace, THREAD);
+#endif
+#ifdef LTC_STATS
+    traces_analyzed++;
+#endif
+
+    memset(regs_modified,sizeof(bool)*8,0);
+
+
+    /* first walk through to find which registers are constant */
+    for (instr = instrlist_first(trace); instr!=NULL; instr = instr_get_next(instr)) {
+        for (reg=REG_EAX;reg<=REG_EDI;reg++) {
+            if (instr_writes_to_reg(instr,reg)) {
+                regs_modified[reg-REG_EAX]=true;
+            }
+        }
+    }
+
+    has_back_arc=!!find_next_self_loop(dcontext,tag,instrlist_first(trace));
+    
+    for (instr = instrlist_first(trace); instr!=NULL; instr = instr_get_next(instr)) {
+        switch(instr_get_opcode(instr)) {
+        case OP_int:
+        case OP_int3:
+        case OP_into:
+        case OP_call:
+        case OP_syscall:
+        case OP_sysenter:
+            LOG(THREAD, LOG_OPTS,3,"trace contains a syscall. fuck it");
+            return;
+        default:
+        }
+            
+
+
+        if (instr_reads_memory(instr)) {
+            int basereg,indexreg;
+            address_written=regs_written=false;
+            mem_access=instr_get_src_mem_access(instr);
+            basereg=opnd_get_base(mem_access);
+            indexreg=opnd_get_index(mem_access);
+
+            logopnd(dcontext,3,mem_access, "checking this operand in");
+            loginst(dcontext,3,instr,"\tthis instruction");
+
+            if (mem_access.size==OPSZ_4_short2&&!instr_get_prefix_flag(instr,PREFIX_DATA)) {
+            }
+            else if (mem_access.size==OPSZ_4) {
+            }
+            else
+                {
+                    LOG(THREAD, LOG_OPTS,3,"\tthis operand is not size_v or size_d.  its %s: ",
+                        size_names[mem_access.size]);
+                    loginst(dcontext,3,instr,"");
+                    continue;
+                }
+            
+            if ((basereg!=REG_NULL&&basereg!=REG_EBP)||
+                (indexreg!=REG_NULL&&indexreg!=REG_EBP)) {
+                logopnd(dcontext,3,mem_access,
+                        "\tthis mem access reads a sketch register, so forget it");
+                continue;
+            }
+
+            first_instance_of_address=true;
+            for (accesschecker = instrlist_first(trace); accesschecker!=instr; 
+                 accesschecker = instr_get_next(accesschecker)) {
+                if (instr_reads_memory(accesschecker)&&
+                    opnd_same_address(instr_get_src_mem_access(accesschecker),mem_access)&&
+                    accesschecker!=instr)
+                    first_instance_of_address=false;
+            }
+            if (!first_instance_of_address) {
+                LOG(THREAD, LOG_OPTS, 3,"\tnot the first instance of this address...\n");
+                continue;
+            }
+
+#ifdef LTC_STATS
+            addrs_seen++;
+#endif
+
+            if (basereg!=REG_NULL) {
+                ASSERT(basereg>=REG_EAX&&basereg<=REG_EDI);
+                if (regs_modified[basereg-REG_EAX]==true) {
+                    regs_written=true;
+                    LOG(THREAD, LOG_OPTS, 3, "\tbase register of access written to, so can't optimize\n");
+                }
+            }
+                
+            if (indexreg!=REG_NULL) {
+                ASSERT(indexreg>=REG_EAX&&indexreg<=REG_EDI);
+                if (regs_modified[indexreg-REG_EAX]==true) {
+                    regs_written=true;
+                    LOG(THREAD, LOG_OPTS, 3, "\tindex register of access written to, so can't optimize\n");
+                }
+            }
+
+            instances_of_address=0;
+
+            for (writechecker = instrlist_first(trace); 
+                 writechecker!=NULL; 
+                 writechecker = instr_get_next(writechecker)) {
+                if (instr_writes_memory(writechecker)&&
+                    opnd_same_address(mem_access,instr_get_dst(writechecker,0))) {
+                    address_written=true;
+                    LOG(THREAD, LOG_OPTS, 3,"\tsame address as access written to, so can't optimize\n");
+                }
+
+                if (instr_reads_memory(writechecker)&&
+                    opnd_same_address(mem_access,instr_get_src_mem_access(writechecker)))
+                    instances_of_address++;
+                
+            }
+
+            if (!regs_written&&!address_written&&opnd_is_base_disp(mem_access)&&
+                (instances_of_address>5||has_back_arc)) {
+                //this load could be turned into a constant move
+
+                //save this mem access operand
+                tmpOpnds[removalpossibilities++]=mem_access;
+                ASSERT(removalpossibilities<NUM_TMP_OPNDS);
+
+                LOG(THREAD, LOG_OPTS, 3,"%d instances of addresses which could be constants",
+                    instances_of_address);
+                logopnd(dcontext,3,mem_access,"");
+
+                if (has_back_arc) {
+                    LOG(THREAD, LOG_OPTS, 3,"this trace loops!\n");
+                }
+
+            }
+            else {
+                LOG(THREAD, LOG_OPTS, 3,"this instr not optimized because instances of address: %d. backarc = %d\n",
+                    instances_of_address,has_back_arc);
+            }
+
+        }
+    }
+    
+    LOG(THREAD, LOG_OPTS, 3,"%d removal possibilities in this trace\n",removalpossibilities);
+
+    ASSERT(trace->ltc.mem_refs==NULL);
+
+    if (removalpossibilities>0) {
+        //if there actually might be any vals to make constant
+        //save a list of the memory references for this trace in the fragment struct
+#ifdef LTC_STATS
+        addrs_analyzed+=removalpossibilities;
+#endif
+
+        trace->ltc.mem_refs=heap_alloc(dcontext,sizeof(struct ltc_mem_ref_data)*removalpossibilities HEAPACCT(ACCT_OTHER));
+        trace->ltc.num_mem_addresses=removalpossibilities;
+        trace->ltc.num_mem_samples=0;
+
+        for (a=0;a<removalpossibilities;a++)
+            trace->ltc.mem_refs[a].opnd=tmpOpnds[a];
+
+        oldtracetop=instrlist_first(trace);
+
+        //move the clean call outside of the loop
+        replace_self_loop_with_opnd(dcontext,(app_pc)tag,trace,opnd_create_instr(oldtracetop));
+
+        LOG(THREAD, LOG_OPTS,3,"LTC: inserting clean call in trace tag= "PFX"\n",tag);
+#ifdef SIDELINE
+        if (dynamo_options.sideline)
+            dr_insert_clean_call(dcontext, trace, oldtracetop, (app_pc) check_mem_refs,
+                                 false/*!fp*/, 1, OPND_CREATE_INTPTR(tag));
+        else {
+#endif
+            //if not sideline then insert code to do optimization immediately
+            insert_clean_call_with_arg_jmp_if_ret_true(dcontext, trace, oldtracetop,
+                       (app_pc) check_mem_refs, (int)tag, (app_pc)tag, NULL);
+#ifdef SIDELINE
+        }
+#endif
+
+
+        LOG(THREAD, LOG_OPTS,3,"inserted clean call to check_mem_refs tag= "PFX"\n",tag);
+        LOG(THREAD, LOG_OPTS,3,"newly trace tag="PFX"  addresses=%d samples=%d\n",
+            tag,trace->ltc.num_mem_addresses,trace->ltc.num_mem_samples);
+
+    } 
+    ASSERT(instr_get_opcode(instrlist_last(trace))==OP_jmp);
+
+}
+
+int
+check_mem_refs(app_pc tag, int errno, reg_t eflags,
+                   reg_t reg_edi, reg_t reg_esi, reg_t reg_ebp, reg_t reg_esp,
+                   reg_t reg_ebx, reg_t reg_edx, reg_t reg_ecx, reg_t reg_eax)
+{
+    //regs puts the values into the order of the reg enum from inst.h
+    int regs[8]={reg_eax,reg_ecx,reg_edx,reg_ebx,reg_esp,reg_ebp,reg_esi,reg_edi};
+    int address,val,a;
+    dcontext_t *dcontext;
+    fragment_t *curfrag;
+    trace_only_t *t_curfrag;
+    
+    dcontext=get_thread_private_dcontext();
+
+    ASSERT(dcontext);
+
+    curfrag=fragment_lookup(dcontext,tag);
+    t_curfrag=TRACE_FIELDS(curfrag);
+
+    ASSERT(curfrag&&curfrag->tag);
+
+    //this shouldn't happen very much, just waiting for sideline thread to remove call
+
+    if (t_curfrag->ltc.num_mem_samples==NUM_VALUES_FOR_SPECULATION) {
+#ifdef SIDELINE
+        if (dynamo_options.sideline)
+            return true;
+        else {
+#endif
+            LOG(THREAD, LOG_OPTS,1,"should never get called unnecessarily if not -sideline tag="PFX,tag);
+            ASSERT_NOT_REACHED();
+#ifdef SIDELINE
+        }
+#endif
+
+    }
+
+
+    LOG(THREAD, LOG_OPTS,3,"check_mem_refs fragment "PFX" tag="PFX"  addresses %d samples=%d\n",curfrag,
+        curfrag->tag,t_curfrag->ltc.num_mem_addresses,t_curfrag->ltc.num_mem_samples);
+
+
+    ASSERT(t_curfrag->ltc.num_mem_samples<NUM_VALUES_FOR_SPECULATION);
+    
+    for (a=0; a<t_curfrag->ltc.num_mem_addresses; a++) {
+        opnd_t mem_access=t_curfrag->ltc.mem_refs[a].opnd;
+
+        address=get_mem_address(dcontext,mem_access,regs);
+        t_curfrag->ltc.mem_refs[a].addresses[t_curfrag->ltc.num_mem_samples]=address;
+        
+        val=get_mem_val(dcontext,mem_access,address);
+        t_curfrag->ltc.mem_refs[a].vals[t_curfrag->ltc.num_mem_samples]=val;
+    }
+
+    //after all addresses are read, *then* increment the number of samples
+    //all addresses have the same number of samples
+
+
+    if (++t_curfrag->ltc.num_mem_samples==NUM_VALUES_FOR_SPECULATION) {
+        LOG(THREAD, LOG_OPTS,3,"fragment (tag="PFX") ready for optimization\n",curfrag->tag);
+
+#ifdef DEBUG
+        //this block can be removed, its just printing...
+        if (stats->loglevel>=4) {
+            for (address=0; address<t_curfrag->ltc.num_mem_addresses; address++) {
+                logopnd(dcontext,3,t_curfrag->ltc.mem_refs[address].opnd,"\tgot enough values for");
+
+                for (a=0;a<NUM_VALUES_FOR_SPECULATION;a++)
+
+                    LOG(THREAD, LOG_OPTS,3,"\t\tsample %d: addr="PFX" val="PFX"\n",a,
+                        t_curfrag->ltc.mem_refs[address].addresses[a],
+                        t_curfrag->ltc.mem_refs[address].vals[a]);
+
+
+            }
+        }
+#endif
+    
+#ifdef SIDELINE
+        if (dynamo_options.sideline) {   //only use the frags_waiting list if using sideline
+            ASSERT_NOT_IMPLEMENTED(false && "this lock needs DELETE_LOCK");
+            mutex_lock(&waiting_LTC_lock);
+            ASSERT(num_frags_waiting_LTC<MAX_TRACES_WAITING_FOR_LTC &&
+                   num_frags_waiting_LTC>=0);
+            
+            //check that the fragment isn't already in the list
+            //its possible that checkmemrefs could be called twice (by the trace executing twice
+            //before sideline_optimize was able to replace the trace
+            for (a=0;a<num_frags_waiting_LTC;a++) {
+                if (frags_waiting_LTC[a]==curfrag) { //its already in the list
+                    mutex_unlock(&waiting_LTC_lock);
+                    return false; //return value doesn't matter in sideline case
+                }
+            }
+            
+            //if we get here, then the fragment isn't already in the list
+            frags_waiting_LTC[num_frags_waiting_LTC++]=curfrag;
+            mutex_unlock(&waiting_LTC_lock);
+        }
+        else //if not doing sideline, then do the insertion immediately! (deterministically!)
+#endif 
+            {
+#ifdef DEBUG
+                if (stats->loglevel>=3) {
+                    LOG(THREAD, LOG_OPTS,3,"check mem refs returning true for tag "PFX"\n",tag);
+                    disassemble_fragment(dcontext,curfrag,0);
+                }
+#endif
+            return true; //returning true
+            }
+
+    }
+    return false; //if !sideline, then this value means not enough values
+}
+
+void
+LTC_online_optimize_and_replace(dcontext_t *dcontext,app_pc tag, fragment_t *curfrag)
+{
+    fragment_t *new_f;
+    instrlist_t *ilist;
+    trace_only_t *t_curfrag=TRACE_FIELDS(curfrag);
+    void *vmlist = NULL;
+    DEBUG_DECLARE(bool ok;)
+    ASSERT(curfrag);
+    LOG(THREAD, LOG_OPTS,3,"LTC: doing online optimization of current trace. tag="PFX"\n",tag);
+
+    //all this fragment shit is taken from upgrade_to_trace_head in monitor.c
+    ilist=decode_fragment(dcontext, curfrag, NULL, NULL, curfrag->flags, NULL, NULL);
+    if (instr_get_opcode(instrlist_last(ilist))!=OP_jmp) {
+        LOG(THREAD, LOG_OPTS,3,"This fragment doesn't end in a OP_jmp\n");
+        disassemble_fragment(dcontext,curfrag,0);
+        ASSERT_NOT_REACHED();
+    }
+        
+    remove_mem_ref_check(dcontext,ilist);
+    ltc_trace(dcontext,curfrag,ilist); //actually do the optimization
+
+    ASSERT(t_curfrag->ltc.mem_refs);
+    heap_free(dcontext,t_curfrag->ltc.mem_refs,
+              sizeof(struct ltc_mem_ref_data)*t_curfrag->ltc.num_mem_addresses
+              HEAPACCT(ACCT_OTHER));
+    t_curfrag->ltc.mem_refs=NULL;
+    
+    DEBUG_DECLARE(ok =)
+        vm_area_add_to_list(dcontext, curfrag->tag, &vmlist, curfrag->flags, curfrag,
+                            false/*no locks*/);
+    ASSERT(ok); /* should never fail for private fragments */
+    new_f = emit_invisible_fragment(dcontext, curfrag->tag, ilist,curfrag->flags, vmlist);
+    fragment_copy_data_fields(dcontext, curfrag, new_f);
+    shift_links_to_new_fragment(dcontext, curfrag, new_f);
+    
+    fragment_replace(dcontext,curfrag,new_f);
+    fragment_delete(dcontext, curfrag, FRAGDEL_NO_OUTPUT | FRAGDEL_NO_UNLINK |
+                    FRAGDEL_NO_HTABLE);
+    /* free the instrlist_t elements */
+    instrlist_clear_and_destroy(dcontext, ilist);
+    
+    //returning true means that it should jump back to the top of trace
+    //need to make sure that the exit is not linked so that 
+#ifdef DEBUG
+    if (stats->loglevel>=3) {
+        LOG(THREAD, LOG_OPTS,3,"new fragment after doing ltc\n");
+        disassemble_fragment(dcontext,new_f,0);
+    }
+#endif
+}
+
+
+//the regs param is as defined in check_mem_refs
+int
+get_mem_address(dcontext_t *dcontext,opnd_t mem_access,int regs[8])
+{
+
+    int indexreg,basereg,index,base,scale,disp;
+    int *address;
+    logopnd(dcontext,3,mem_access,"getting this val");
+    ASSERT(opnd_is_near_base_disp(mem_access));
+
+    indexreg=opnd_get_index(mem_access);
+
+    if (indexreg!=REG_NULL) {
+        ASSERT(reg_is_32bit(indexreg));
+        index=regs[indexreg-REG_EAX];
+    }
+    else
+        index=0;
+
+        
+    basereg=opnd_get_base(mem_access);
+    if (basereg!=REG_NULL) {
+        ASSERT(reg_is_32bit(basereg));
+        base=regs[basereg-REG_EAX];
+    }
+    else
+        base=0;
+
+    scale=opnd_get_scale(mem_access);
+    disp=opnd_get_disp(mem_access);
+    
+    address=(int *)(base+index*scale+disp);
+    LOG(THREAD, LOG_OPTS, 3,"base="PFX", index="PFX", scale="PFX", disp="PFX". address = "PFX"\n",
+        base,index,scale,disp,address);
+
+    return (int)address;
+}
+
+int
+get_mem_val(dcontext_t *dcontext,opnd_t mem_access,int address)
+{
+    int val;
+    int *addrp=(int *)address;
+    //its necessary to to switch on the operand size so you don't fetch too much
+    //cuz that could possibly cause a segfault
+    switch (mem_access.size) {
+    case OPSZ_4: //32 bits
+    case OPSZ_4_short2: //32 or 16, but i've only seen 32 so...
+        val=*addrp;
+        break;
+    case OPSZ_1:
+        val= (int)(*(char *)addrp);
+        break;
+    case OPSZ_2:
+        val= (int)(*(short *)addrp);
+        break;
+    default:
+        logopnd(dcontext,3,mem_access,"funky operand");
+        ASSERT_NOT_REACHED();
+    }
+    LOG(THREAD, LOG_OPTS, 3,"in get_mem_val addr= %d ",address);
+    logopnd(dcontext,3,mem_access,"");
+    LOG(THREAD, LOG_OPTS, 3,"value =  "PFX"\n",val);
+    return val;
+}
+
+#ifdef SIDELINE
+void
+LTC_examine_traces()
+{
+    fragment_t *curfrag;
+    trace_only_t *t_curfrag;
+    ASSERT(num_frags_waiting_LTC>=0);
+    ASSERT(num_frags_waiting_LTC<=MAX_TRACES_WAITING_FOR_LTC);
+
+    if (num_frags_waiting_LTC==0) {
+        return;
+    }
+
+    LOG(GLOBAL, LOG_OPTS,3,"in LTC_examine_traces, %d frags need optimizing\n",num_frags_waiting_LTC);
+
+    mutex_lock(&do_not_delete_lock);
+    mutex_lock(&waiting_LTC_lock);      
+
+    curfrag=frags_waiting_LTC[0];
+    t_curfrag=TRACE_FIELDS(curfrag);
+
+    num_frags_waiting_LTC--;
+    memmove(frags_waiting_LTC,&frags_waiting_LTC[1],sizeof(fragment_t *)*num_frags_waiting_LTC);
+
+    mutex_unlock(&waiting_LTC_lock);
+
+    if (t_curfrag->ltc.ltc_already_optimized) {
+        LOG(GLOBAL, LOG_OPTS,3, "LTC_examine_traces: encountered this frag in frags_waiting, but its already optimized\n", curfrag->tag);
+        mutex_unlock(&do_not_delete_lock);
+        return;
+    }
+    
+    LOG(GLOBAL, LOG_OPTS,3, "LTC_examine_traces: about to optimize F%d\n", curfrag->tag);
+    t_curfrag->ltc.ltc_already_optimized=true;
+    sideline_optimize(curfrag,remove_mem_ref_check,ltc_trace);
+    mutex_unlock(&do_not_delete_lock);
+}
+
+// if a fragment is ever deleted, then remove it from the list of fragments 
+// that need to be tested for their 
+
+void
+LTC_fragment_delete(fragment_t *frag)
+{
+    int a;
+    //make sure this list isn't added to while we're in it
+    mutex_lock(&waiting_LTC_lock); 
+
+    for (a=0;a<num_frags_waiting_LTC;a++) {
+        if (frag==frags_waiting_LTC[a]) {
+            //if the frag deleted is in the list of those waiting to be optimized
+            //remove it by shifting the rest of the list down over it.
+            //not efficient but it works...
+
+            memmove(&frags_waiting_LTC[a],
+                    &frags_waiting_LTC[a+1],
+                    sizeof(fragment_t *)*(num_frags_waiting_LTC-a));
+            num_frags_waiting_LTC--;
+        }
+
+    }
+    
+    mutex_unlock(&waiting_LTC_lock); 
+}
+#endif
+
+void
+remove_mem_ref_check(dcontext_t *dcontext,instrlist_t *trace)
+{
+    LOG(THREAD, LOG_OPTS,3,"remove_mem_ref_check\n");
+
+    ASSERT(instr_get_opcode(instrlist_last(trace))==OP_jmp);
+
+
+#ifdef SIDELINE
+    if (dynamo_options.sideline) {
+        pop_instr_off_list(dcontext,trace,OP_mov_st); //first instr in clean_call_arg is a store
+        pop_instr_off_list(dcontext,trace,OP_mov_ld); //then a load
+        pop_instr_off_list(dcontext,trace,OP_pushf);
+        pop_instr_off_list(dcontext,trace,OP_pusha);
+        pop_instr_off_list(dcontext,trace,OP_push_imm);
+        pop_instr_off_list(dcontext,trace,OP_call);
+        pop_instr_off_list(dcontext,trace,OP_pop);
+        pop_instr_off_list(dcontext,trace,OP_popa);
+        pop_instr_off_list(dcontext,trace,OP_popf);
+        pop_instr_off_list(dcontext,trace,OP_mov_ld);
+    }
+    else {
+#endif
+        pop_instr_off_list(dcontext,trace,OP_mov_st); //first instr in clean_call_arg is a store
+        pop_instr_off_list(dcontext,trace,OP_mov_ld); //then a load
+        pop_instr_off_list(dcontext,trace,OP_pushf); // ...
+        pop_instr_off_list(dcontext,trace,OP_pusha);
+        pop_instr_off_list(dcontext,trace,OP_push_imm);
+        pop_instr_off_list(dcontext,trace,OP_call);
+        pop_instr_off_list(dcontext,trace,OP_pop);
+        pop_instr_off_list(dcontext,trace,OP_cmp);
+        pop_instr_off_list(dcontext,trace,OP_je);
+        pop_instr_off_list(dcontext,trace,OP_popa);
+        pop_instr_off_list(dcontext,trace,OP_popf);
+        pop_instr_off_list(dcontext,trace,OP_mov_ld);
+        pop_instr_off_list(dcontext,trace,OP_jmp);
+        pop_instr_off_list(dcontext,trace,OP_popa);
+        pop_instr_off_list(dcontext,trace,OP_popf);
+        pop_instr_off_list(dcontext,trace,OP_mov_ld);
+#ifdef SIDELINE
+    }
+#endif    
+    
+
+    //find the pc that it used to jump to get below the inserted call
+    //and replace that jump with a jump to the new top
+    //replace the jmp to the new top of the trace
+    replace_self_loop_with_opnd(dcontext,instrlist_first(trace)->bytes,
+                                 trace,opnd_create_instr(instrlist_first(trace)));
+
+}
+
+void
+pop_instr_off_list(dcontext_t *dcontext,instrlist_t *trace,int opcode)
+{
+    instr_t *instr=instrlist_first(trace);
+    ASSERT(instr_get_opcode(instr)==opcode);
+    instrlist_remove(trace,instr);
+    instr_destroy(dcontext,instr);
+}
+
+void
+ltc_trace(dcontext_t *dcontext, fragment_t *frag, instrlist_t *trace)
+{
+    instrlist_t *opt_trace,*comparisons;
+    instr_t *top_safe;
+    opnd_t mem_ref,valop;
+    int a,num_replaced;
+    size_t top_opt_bytes;
+    trace_only_t *t_frag=TRACE_FIELDS(frag);
+    num_replaced=0;
+
+    ASSERT(t_frag);
+    ASSERT(trace);
+
+    LOG(THREAD, LOG_OPTS,3,"ltc_trace. should actually do the optimization! tag "PFX", top bytes "PFX"\n",frag->tag,
+        instrlist_first(trace)->bytes);
+
+    LOG(THREAD, LOG_OPTS,3,"trace before ltc_trace\n");
+#ifdef DEBUG
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_OPTS) != 0)
+        instrlist_disassemble(dcontext, 0, trace, THREAD);
+#endif
+
+#ifdef SIDELINE
+    if (!dynamo_options.sideline)
+#endif
+        ASSERT(t_frag->ltc.mem_refs);
+
+    LOG(THREAD, LOG_OPTS,3,"making self loop point to trace's tag");
+    replace_self_loop_with_opnd(dcontext,(app_pc)instrlist_first(trace)->bytes,trace,opnd_create_pc(frag->tag));
+    opt_trace=instrlist_clone(dcontext,trace);
+
+    top_safe=instrlist_first(trace); 
+    LOG(THREAD, LOG_OPTS,3,"calling replace self loop for safe to point at top of safe part\n");
+    replace_self_loop_with_opnd(dcontext,(app_pc)frag->tag,trace,opnd_create_instr(top_safe));
+
+    for (a=0;a<t_frag->ltc.num_mem_addresses;a++)
+        if (should_replace_load(dcontext,t_frag->ltc.mem_refs[a]))
+            num_replaced++;
+
+    if (num_replaced==0) {
+        LOG(THREAD, LOG_OPTS,3,"should_replace returned false for all mem refs. doing nothing\n");
+        return;
+    }
+
+    num_replaced=0;
+
+
+    instrlist_prepend_instrlist(dcontext,trace,restore_eflags_list(dcontext,frag));
+
+#ifdef LTC_STATS
+    instrlist_prepend(trace,INSTR_CREATE_inc(dcontext,OPND_CREATE_MEM32(REG_NULL,(int)&safe_taken)));
+#endif
+
+    top_safe=instrlist_first(trace);
+    top_opt_bytes=(size_t)instrlist_first(opt_trace)->bytes; //save old top
+
+
+    //save eflags at top of comparison
+    comparisons=save_eflags_list(dcontext,frag);
+
+    for (a=0;a<t_frag->ltc.num_mem_addresses;a++) {
+        if (should_replace_load(dcontext,t_frag->ltc.mem_refs[a])) {
+            num_replaced++;
+
+            mem_ref=t_frag->ltc.mem_refs[a].opnd;
+            valop=value_to_replace(t_frag->ltc.mem_refs[a]);
+
+            logopnd(dcontext,3,mem_ref,"\tthis memory ref");
+            logopnd(dcontext,3,valop,"\tgets this value");
+
+            do_single_LTC(dcontext,opt_trace,mem_ref,valop);
+#ifdef LTC_STATS
+            addrs_made_const++;
+#endif
+
+
+            //rather than doing the lea stuff, check the register values directly
+            /*      if (opnd_get_base(mem_ref)!=REG_NULL) {
+                    LOG(THREAD, LOG_OPTS, 3,"base reg isn't null, so inserting a check for it too\n");
+                    instrlist_append(comparisons,INSTR_CREATE_cmp(dcontext,opnd_create_reg(opnd_get_base(mem_ref)),OPND_CREATE_INT32(t_frag->mem_refs[a].base_val)));
+                    
+                    instrlist_append(comparisons,INSTR_CREATE_jcc(dcontext,OP_jne,opnd_create_instr(top_safe)));
+                    }
+                    if (opnd_get_index(mem_ref)!=REG_NULL) {
+                    LOG(THREAD, LOG_OPTS, 3,"index reg isn't null, so inserting a check for it too\n");
+                    instrlist_append(comparisons,INSTR_CREATE_cmp(dcontext,opnd_create_reg(opnd_get_index(mem_ref)),OPND_CREATE_INT32(t_frag->mem_refs[a].index_val)));
+                    instrlist_append(comparisons,INSTR_CREATE_jcc(dcontext,OP_jne,opnd_create_instr(top_safe)));
+                    }
+            */
+            
+            //add a check that the value is what it should be
+            instrlist_append(comparisons,INSTR_CREATE_cmp(dcontext,mem_ref,valop));
+            instrlist_append(comparisons,INSTR_CREATE_jcc(dcontext,OP_jne,opnd_create_instr(top_safe)));
+                                             
+        }                
+        else {
+            logopnd(dcontext,3,t_frag->ltc.mem_refs[a].opnd,"not replacing me because of bad sampled vals");
+        }
+
+    }
+
+    LOG(THREAD, LOG_OPTS,3,"after ltc");
+#ifdef DEBUG
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_OPTS) != 0)
+        instrlist_disassemble(dcontext, 0, trace, THREAD);
+#endif
+
+    if (!dynamo_options.safe_loads_to_const) {
+        constant_propagate(dcontext,opt_trace,frag->tag);
+        instrlist_remove_nops(dcontext,opt_trace);
+    }
+
+    ASSERT(instr_get_prev(instrlist_first(trace))==NULL);
+    ASSERT(instr_get_next(instrlist_last(trace))==NULL);
+
+    if (dynamo_options.remove_dead_code) {
+        LOG(THREAD, LOG_OPTS,3,"doing remove_dead_code on ltc'd trace");
+        remove_dead_code(dcontext,frag->tag,opt_trace);
+    }
+
+    if (dynamo_options.rlr) {
+        LOG(THREAD, LOG_OPTS,3,"doing rlr removal on ltc'd trace");
+        remove_redundant_loads(dcontext,frag->tag,opt_trace);
+    }
+
+    LOG(THREAD, LOG_OPTS,3,"replacing opt self-loop\n");
+    replace_self_loop_with_opnd(dcontext,(app_pc)frag->tag,opt_trace,opnd_create_instr(instrlist_first(opt_trace)));
+
+    instrlist_prepend_instrlist(dcontext,opt_trace,restore_eflags_list(dcontext,frag));
+#ifdef LTC_STATS
+    instrlist_prepend(opt_trace,INSTR_CREATE_inc(dcontext,OPND_CREATE_MEM32(REG_NULL,(int)&opt_taken)));
+#endif
+
+    instrlist_prepend_instrlist(dcontext,trace,opt_trace);
+    instrlist_prepend_instrlist(dcontext,trace,comparisons); //comparisons should be first
+ 
+#ifdef DEBUG
+    LOG(THREAD,LOG_OPTS, 3, "after LTC optimization:\n");
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_OPTS) != 0)
+        instrlist_disassemble(dcontext, 0, trace, THREAD);
+#endif
+
+}
+
+void
+do_single_LTC(dcontext_t *dcontext,instrlist_t *trace, opnd_t mem_access, opnd_t const_value)
+{
+    instr_t *in,*next;                        
+    int val;
+    opnd_t instr_mem_access;
+    val=(int) opnd_get_immed_int(const_value);
+    in=instrlist_first(trace);
+    LOG(THREAD, LOG_OPTS,3,"entering do_single_LTC\n");
+
+    while (in) {
+        next=instr_get_next(in);
+        loginst(dcontext,3,in,"in_single_LTC, examining");
+        ASSERT(instr_is_encoding_possible(in));
+        if (instr_reads_memory(in)) { //could this be if (!op_lea)
+            instr_mem_access=instr_get_src_mem_access(in);
+            if (opnd_same_address(instr_mem_access,mem_access)) {
+                if (instr_get_opcode(in)==OP_mov_ld) {
+                    loginst(dcontext,3,in,"\tdoing LTC on mov");
+                    logopnd(dcontext,3,mem_access,"replacing this operand");
+                    logopnd(dcontext,3,const_value,"with this");
+                    instr_set_opcode(in,OP_mov_imm);
+                    instr_replace_src_opnd(in,mem_access,const_value);
+                }
+                else if (instr_get_opcode(in)==OP_cmp) {
+                    opnd_t orig_op1,orig_op2;
+                    orig_op1=instr_get_src(in,0);
+                    orig_op2=instr_get_src(in,1);
+                    loginst(dcontext,3,in,"\tdoing LTC on cmp");
+                    logopnd(dcontext,3,const_value,"\tnewvalue");
+                    instr_replace_src_opnd(in,mem_access,const_value);
+                    loginst(dcontext,3,in,"\tafter replacing mem access");
+
+                    next=fix_cmp_containing_constant(dcontext,trace,in,orig_op1,orig_op2);
+                }
+                else {
+                    //try to replace and see if its encodable
+                    loginst(dcontext,3,in,"\tdoing LTC on a non mov, cmp");
+                    instr_replace_src_opnd(in,instr_mem_access,const_value);
+                    if (!instr_is_encoding_possible(in)) {
+                        instr_replace_src_opnd(in,const_value,instr_mem_access);
+                        ASSERT(instr_is_encoding_possible(in));
+                    }
+                    loginst(dcontext,3,in,"\tafter replacing mem access");
+                    instr_arithmatic_simplify(dcontext,in);
+                }
+
+            }
+        }
+        in=next;
+        
+    }
+        LOG(THREAD, LOG_OPTS,3,"exiting do_single_LTC\n");
+
+}
+
+/* 
+   takes in a cmp that contains operands that are invalid. does
+   what is necessary to make the instruction encode, either
+   swapping operands or if there are two constants, it eliminates
+   the cmp if possible and walks the rest of the list, fixing
+   jcc's appropriately.  returns the next intruction to examine.
+*/
+instr_t
+*fix_cmp_containing_constant(dcontext_t *dcontext, 
+                                   instrlist_t *trace, instr_t *in,
+                                   opnd_t orig_op1,opnd_t orig_op2) {
+    instr_t *next, *jcc;
+    opnd_t tmp;
+    next=instr_get_next(in);
+    if (instr_is_encoding_possible(in))
+        return next;
+
+    //first, try transposing the operands to make it encode
+    if ((opnd_is_reg(instr_get_src(in,0))||opnd_is_reg(instr_get_src(in,1))) &&
+        safe_to_transpose_cmp(dcontext,in)) {
+
+        tmp=instr_get_src(in,0); //swap operands
+        instr_set_src(in,0,instr_get_src(in,1));
+        instr_set_src(in,1,tmp);
+        if (instr_is_encoding_possible(in)) {
+            //cmp switched, so now switch direction of jcc's that care
+            for (jcc=instr_get_next(in);
+                 (jcc!=NULL)&&((instr_get_arith_flags(jcc)&EFLAGS_WRITE_6)==0)
+                     ; jcc=instr_get_next(jcc)) {
+                if (instr_is_cbr(jcc)) {
+                    loginst(dcontext,3,jcc,"change_cbr_due_to_reversed_cmp called on");
+                    change_cbr_due_to_reversed_cmp(jcc);
+                }
+            }
+        }
+        else { //swapping operands didn't really work. so put them back and try other stuff
+            instr_set_src(in,0,orig_op1);
+            instr_set_src(in,1,orig_op2);
+            loginst(dcontext,3,in,"wasn't swappable, even though one opnd was a reg. whats up with that?");
+            ASSERT_CURIOSITY(false);
+        }
+    }
+
+    else if ((opnd_is_immed(instr_get_src(in,0))&&opnd_is_immed(instr_get_src(in,1))) &&
+             safe_to_delete_cmp(dcontext,in))   {  //cmp gets two constants
+        int op1,op2;
+        instr_t *nextjcc;
+        loginst(dcontext,3,in,"swapping order didn't help, must be 2 constants");
+
+        op1=(int) opnd_get_immed_int(instr_get_src(in,0));
+        op2=(int) opnd_get_immed_int(instr_get_src(in,1));
+        
+        loginst(dcontext,3,in,"got the two constants");
+        
+        ASSERT(instr_get_opcode(in)==OP_cmp);
+
+        for (jcc=instr_get_next(in); 
+             jcc!=NULL&& ((instr_get_arith_flags(jcc)&EFLAGS_WRITE_6)==0); 
+             jcc=nextjcc) {
+            nextjcc=instr_get_next(jcc);
+            loginst(dcontext,3,jcc,"walking to try to remove cmp");
+            if (instr_is_cbr(jcc)) {
+                if (becomes_ubr_from_cmp(jcc,op1,op2)) { 
+                    //the cbr becomes an unconditional jmp
+                    loginst(dcontext,3,jcc,"becomes an unconditional jmp");
+                    instr_set_opcode(jcc,OP_jmp);
+                    ASSERT(instr_is_encoding_possible(jcc));
+                    
+                    jcc=instr_get_next(jcc);
+                    while (jcc) { //removing instrs after jmp
+                        nextjcc=instr_get_next(jcc);
+                        loginst(dcontext,3,jcc,"removed because it follows an unconditional jmp");
+                        instrlist_remove(trace,jcc);
+                        instr_destroy(dcontext,jcc);
+                            jcc=nextjcc;
+                    }
+                    nextjcc=NULL;
+                }
+                else { //otherwise never jumps, so remove
+                    loginst(dcontext,3,jcc,"will never jmp, so removed");
+                    instrlist_remove(trace,jcc);
+                    instr_destroy(dcontext,jcc);
+                }
+            }
+        }
+
+
+        //remove the cmp instruction
+        next=instr_get_next(in);
+        loginst(dcontext,3,in,"this cmp isn't needed, so remove");
+        loginst(dcontext,3,next,"setting next to");
+        instrlist_remove(trace,in);
+        instr_destroy(dcontext,in);
+        in=NULL; 
+    
+    }
+    else { //if none of that shit helped the situation, put back original cmp operands
+        instr_set_src(in,0,orig_op1);
+        instr_set_src(in,1,orig_op2);
+        loginst(dcontext,3,in,"wasn't able to fix this instr by either removing or transposing cmp, original opnds back");
+        ASSERT(instr_is_encoding_possible(in));
+    }
+
+    if (in&&!instr_is_encoding_possible(in)) {
+        loginst(dcontext,0,in,"error encoding me");
+        ASSERT_NOT_REACHED();
+    }
+    return next;
+}
+
+
+
+bool
+safe_to_transpose_cmp(dcontext_t *dcontext,instr_t *testinstr)
+{
+    return safe_to_modify_cmp(dcontext,testinstr,TRANSPOSE);
+}
+bool
+safe_to_delete_cmp(dcontext_t *dcontext,instr_t *testinstr)
+{
+    return safe_to_modify_cmp(dcontext,testinstr,NO_TRANSPOSE);
+}
+
+/* deals only with arithmetic flags */
+bool
+instr_flag_write_necessary(dcontext_t *dcontext, instr_t *in)
+{
+    instr_t *walker;
+    uint eflags;
+
+    //if the instr doesn't write, return false
+    if (!(instr_get_arith_flags(in)&EFLAGS_WRITE_6))
+        return false;
+
+    walker=instr_get_next(in);    
+    //otherwise, see if the written flags are read
+    while (walker) {
+        eflags=instr_get_arith_flags(walker);
+        if (eflags&EFLAGS_READ_6)
+            return true;
+        else if (eflags&EFLAGS_WRITE_6)
+            return false;
+        else if (instr_is_exit_cti(walker) &&
+                 pc_reads_flags_before_writes(dcontext,opnd_get_pc(instr_get_target(walker))))
+            return true;
+        walker=instr_get_next(walker);
+    }
+    return false;
+}    
+
+/*  this func tests if the currently modified cmp instruction could be unsafe
+    a cmp is considered unsafe if it would write flags in a way that they could be
+    seen outside of this trace.
+
+    if transpose==NO_TRANSPOSE, then it is checking whether the cmp instr could be removed as is
+    if TRANSPOSE, then its checking if the instruction could have its operands transposed and it
+    wouldn't affect anything else...
+*/
+
+bool
+safe_to_modify_cmp(dcontext_t *dcontext,instr_t *testinstr, bool transpose)
+{
+    opnd_t cmp_op1,cmp_op2;
+    instr_t *in;
+    bool cmp_both_ops_immed;
+    uint eflags;
+
+
+    //fix: why is this an if? can't I ASSERT(instr_get_opcode(testinstr)==OP_cmp) ?
+
+    if (instr_get_opcode(testinstr)==OP_cmp) {
+        if (!transpose) {
+            cmp_op1=instr_get_src(testinstr,0);
+            cmp_op2=instr_get_src(testinstr,1);
+        }
+        else {
+            cmp_op1=instr_get_src(testinstr,1);
+            cmp_op2=instr_get_src(testinstr,0);
+        }
+        cmp_both_ops_immed=opnd_is_immed(cmp_op1)&&opnd_is_immed(cmp_op2);
+    }
+    else
+        cmp_both_ops_immed=false;
+
+
+    LOG(THREAD, LOG_OPTS,3,"in safe_to_modify_cmp: bothopsimmed=%d\n",cmp_both_ops_immed);
+
+    for (in=instr_get_next(testinstr);in!=NULL;in=instr_get_next(in)) {
+        eflags=instr_get_arith_flags(in);
+
+        loginst(dcontext,3,in,"\texamining");
+
+        //if the jump is always taken
+        if (instr_is_exit_cti(in)) {
+            loginst(dcontext,3,in,"\texit cti");
+            if (cmp_both_ops_immed && instr_is_cbr(in)) {
+                ASSERT(instr_get_opcode(testinstr)==OP_cmp);
+
+                //if its a cbr, then you can find out if its taken
+                if (transpose==TRANSPOSE) {
+                    if (becomes_ubr_from_cmp(in,(int) opnd_get_immed_int(cmp_op2),(int) opnd_get_immed_int(cmp_op1))) {
+                        //if the jump is always taken
+                        if (pc_reads_flags_before_writes(dcontext,opnd_get_pc(instr_get_target(in)))) {
+                            loginst(dcontext,3,in,"jcc always taken, but dest pc depends on flags, true");
+                            return false; //if dest pc depends on the flags
+                        }
+                        else {
+                            loginst(dcontext,3,in,"jcc always taken,  dest pc overwrites flag, ret false");
+                            return true; //if dest pc doesn't depend on flags, then safe to modify cmp
+                        }
+                    }
+                }
+                else {
+                    ASSERT(transpose==NO_TRANSPOSE);
+                    if (becomes_ubr_from_cmp(in,(int) opnd_get_immed_int(cmp_op1),(int) opnd_get_immed_int(cmp_op2))) {
+                        //if the jump is always taken
+                        if (pc_reads_flags_before_writes(dcontext,opnd_get_pc(instr_get_target(in)))) {
+                            loginst(dcontext,3,in,"jcc always taken, but dest pc depends on flags, true");
+                            return false; //if dest pc depends on the flags
+                        }
+                        else {
+                            loginst(dcontext,3,in,"jcc always taken,  dest pc overwrites flag, ret false");
+                            return true; //if dest pc doesn't depend on flags, then safe to modify cmp
+                        }
+                    }
+                }
+            }
+            //if not a cbr, then you have to assume the branch is taken
+            else if (pc_reads_flags_before_writes(dcontext,opnd_get_pc(instr_get_target(in)))) {
+                loginst(dcontext,3,in,"\tin safe_to_modify_cmp, this CBR may be taken and reads flags before writing");
+                return false;
+            }
+            else {
+                loginst(dcontext,3,in,"\tin safe_to_modify_cmp, this CBR may be taken but it writes before reads, so the cmp can be changed");
+            }
+        }
+        else {
+            if ((eflags & EFLAGS_READ_6) != 0) {
+                loginst(dcontext,3,in,"\treads the flags. cmp needed treturning false: ");
+                return false;
+            }
+            if ((eflags & EFLAGS_WRITE_6) != 0) {
+                loginst(dcontext,3,in,"\twritesthe flags. cmp not needed, ret true:");
+                return true;
+            }
+        }
+    }
+
+    //the end of a trace should be a jmp. 
+    //if this poitn is reached, then it means that jmp's dest writs before reading
+    //so its safe to modify the cmp
+    return true;
+
+}
+
+
+
+bool
+pc_reads_flags_before_writes(dcontext_t *dcontext, app_pc target)
+{
+    uint eflags;
+    do {
+        target = decode_eflags_usage(dcontext, target, &eflags);
+        if ((eflags & EFLAGS_READ_6) != 0) {
+            loginst(dcontext, 3, &tinst,
+                    "reads cmpflags before writing");
+            return true;
+        }
+        /* if writes but doesn't read, ok */
+        if ((eflags & EFLAGS_WRITE_6) != 0) {
+            return false;
+        }
+        /* stop at 1st cti, don't try to recurse,
+         * common case should hit a writer of CF
+         * prior to 1st cti
+         */
+    } while (target != NULL && !instr_is_cti(&tinst));
+    
+    return true;
+}
+
+bool
+becomes_ubr_from_cmp(instr_t *in,int op1, int op2)
+{
+    ASSERT(instr_is_cbr(in));
+
+    switch(instr_get_opcode(in)) {
+    case OP_jle:  return (op1<=op2);
+    case OP_jnle: return (op1>op2);
+    case OP_jl:   return (op1<op2);
+    case OP_jnl:  return (op1>=op2);
+    case OP_jz:  return (op1==op2);
+    case OP_jnz: return (op1!=op2);
+        
+    case OP_jb: return ((unsigned)op1<(unsigned)op2);
+    case OP_jnb: return ((unsigned)op1>=(unsigned)op2);
+    case OP_jbe: return ((unsigned)op1<=(unsigned)op2);
+    case OP_jnbe: return ((unsigned)op1>(unsigned)op2);
+    default:
+        ASSERT_NOT_REACHED();
+    }
+    return false;
+}
+
+
+void
+change_cbr_due_to_reversed_cmp(instr_t *in)
+{
+    int opc;
+    ASSERT(instr_is_cbr(in));
+
+    switch(instr_get_opcode(in)) {
+    case OP_jle:  opc = OP_jnl; break;
+    case OP_jnle: opc = OP_jl; break;
+    case OP_jl:   opc = OP_jnle; break;
+    case OP_jnl:  opc = OP_jle; break;
+    case OP_jz: case OP_jnz: return;
+    case OP_jb: opc=OP_jnbe; break;
+    case OP_jnb: opc=OP_jbe; break;
+    case OP_jbe: opc=OP_jnb; break;
+    case OP_jnbe: opc=OP_jb; break;
+    default:
+        ASSERT_NOT_REACHED();
+    }
+    instr_set_opcode(in,opc);
+}
+
+//currently the simple, conservative algorithm is commented out
+#if 0
+/* this function is now just a simple check to see if all the sampled
+   values are the same.  later this could be changed to something
+   more sophisticated 
+*/
+bool
+should_replace_load(struct ltc_mem_ref_data data)
+{
+    int a,firstval,firstaddr;
+    firstval=data.vals[0];
+    firstaddr=data.addresses[0];
+
+    for (a=1;a<NUM_VALUES_FOR_SPECULATION;a++) {
+        if (data.vals[a]!=firstval)
+            return false;
+        if (data.addresses[a]!=firstaddr)
+            return false;
+    }
+    
+    //if they're all the same, return true...
+    return true;
+}
+
+//value_to_replace is intimately involved with the should_replace_load alg
+//if one is changed, so should the other. now its just real simple
+//for no:, this func assumes that they're all the same value
+opnd_t
+value_to_replace(struct ltc_mem_ref_data data)
+{
+    return opnd_create_immed_int(data.vals[0],data.opnd.size); 
+}
+#else
+/* this should_replace_load will replace the load if there are any values
+   that appear more than half the time. much more aggressive
+*/
+bool
+should_replace_load(dcontext_t *dcontext,struct ltc_mem_ref_data data)
+{
+    int vals[NUM_VALUES_FOR_SPECULATION],numvals=0;
+    int freq[NUM_VALUES_FOR_SPECULATION];
+    int a,b,curval;
+
+    logopnd(dcontext,3,data.opnd,"in should_replace_load on this operand");
+
+    for (a=0;a<NUM_VALUES_FOR_SPECULATION;a++) {
+        curval=data.vals[a];
+        for (b=0;b<numvals;b++)
+            if (vals[b]==curval) {
+                freq[b]++;
+                break;
+            }
+        if (b==numvals) { //that means the value wasn't already there
+            freq[numvals]=1;
+            vals[numvals++]=curval; //so add the value to the list!
+        }
+    }
+
+    //now that the freq table is generated, find any vals that are big enough
+    for (a=0;a<numvals;a++) {
+        LOG(THREAD, LOG_OPTS,3,"%d:\t%d samples of value "PFX"\n",a,freq[a],vals[a]);
+        if (freq[a]>(NUM_VALUES_FOR_SPECULATION*SAMPLE_PERCENT)) {
+            LOG(THREAD, LOG_OPTS,3,"yes we should replace, %d of %d samples (%d unique value(s)) had value= "PFX"\n",
+                freq[a],NUM_VALUES_FOR_SPECULATION,numvals,vals[a]);
+            return true;
+        }
+    }
+
+    //if it got this far, then there are no values that happen more than half the time
+    //so we shouldn't replace
+    logopnd(dcontext,3,data.opnd,"should_replace_load returning false on");
+    return false;
+
+}
+
+/* this code is just a copy of should_replace_load, but with a different return value
+ */
+opnd_t
+value_to_replace(struct ltc_mem_ref_data data)
+{
+    int vals[NUM_VALUES_FOR_SPECULATION],numvals=0;
+    int freq[NUM_VALUES_FOR_SPECULATION];
+    int a,b,curval;
+
+    for (a=0;a<NUM_VALUES_FOR_SPECULATION;a++) {
+        curval=data.vals[a];
+        for (b=0;b<numvals;b++)
+            if (vals[b]==curval) {
+                freq[b]++;
+                break;
+            }
+        if (b==numvals) {//that means the value wasn't already there
+            freq[numvals]=1;
+            vals[numvals++]=curval; //so add the value to the list!
+        }
+    }
+
+    //now that the freq table is generated, find any vals that are big enough
+    for (a=0;a<numvals;a++) {
+        if (freq[a]>(NUM_VALUES_FOR_SPECULATION*SAMPLE_PERCENT))
+            return opnd_create_immed_int(vals[a],data.opnd.size); 
+    }
+
+    ASSERT_NOT_REACHED();
+    return opnd_create_immed_int(vals[0],data.opnd.size); 
+}
+
+#endif
+
+/* saves eax to dcontext
+   saves flags in ah and al. don't overwrite them! */
+instrlist_t*
+save_eflags_list(dcontext_t *dcontext,fragment_t *frag)
+{
+    instrlist_t *ilist;
+    ilist=instrlist_create(dcontext);
+    if (dynamo_options.safe_loads_to_const) {
+        instrlist_append(ilist,INSTR_CREATE_nop(dcontext));
+        instrlist_append(ilist,INSTR_CREATE_nop(dcontext));
+        instrlist_append(ilist,INSTR_CREATE_nop(dcontext));
+
+    }
+        //save only some
+    if (((frag->flags & FRAG_WRITES_EFLAGS_6) == 0)||dynamo_options.safe_loads_to_const) {
+        instrlist_append(ilist,instr_create_save_to_dcontext(dcontext, REG_EAX, XAX_OFFSET));
+        instrlist_append(ilist, INSTR_CREATE_lahf(dcontext));
+    }
+
+    if (!TEST(FRAG_WRITES_EFLAGS_OF, frag->flags)||dynamo_options.safe_loads_to_const) {
+        /* must have saved eax */
+        ASSERT(!TEST(FRAG_WRITES_EFLAGS_6, frag->flags)||
+               dynamo_options.safe_loads_to_const);
+        instrlist_append(ilist,INSTR_CREATE_setcc(dcontext, OP_seto,
+                                                  opnd_create_reg(REG_AL)));
+    }
+
+     return ilist;
+}
+
+
+instrlist_t*
+restore_eflags_list(dcontext_t *dcontext,fragment_t *frag)
+{
+    instrlist_t *ilist;
+    
+    ilist=instrlist_create(dcontext);
+
+    /* now do an add such that OF will be set only if seto set al to 1 */
+
+    if (dynamo_options.safe_loads_to_const) {
+        instrlist_append(ilist,INSTR_CREATE_nop(dcontext));
+        instrlist_append(ilist,INSTR_CREATE_nop(dcontext));
+    }
+
+
+    if (((frag->flags & FRAG_WRITES_EFLAGS_OF) == 0)||dynamo_options.safe_loads_to_const) {
+        instrlist_append(ilist,
+                        INSTR_CREATE_add(dcontext, 
+                                         opnd_create_reg(REG_AL),
+                                         OPND_CREATE_INT8(0x7f)));
+
+    
+    }
+    if (((frag->flags & FRAG_WRITES_EFLAGS_6) == 0)||dynamo_options.safe_loads_to_const) {
+        instrlist_append(ilist, INSTR_CREATE_sahf(dcontext));
+        instrlist_append(ilist,instr_create_restore_from_dcontext(dcontext, REG_EAX, XAX_OFFSET));
+    }
+    return ilist;
+}
+
+void
+constant_propagate(dcontext_t *dcontext, instrlist_t *trace, app_pc tag)
+{
+
+    instr_t *instr,*next,*conwalker,*conwalker_next;
+    int cons,reg, cons_size;
+    bool mov_immed_needed,reg_overwritten;
+    opnd_t regop,orig1,orig2;
+    LOG(THREAD, LOG_OPTS,3,"before constant_propagate\n");
+#ifdef DEBUG
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_OPTS) != 0)
+        instrlist_disassemble(dcontext, 0, trace, THREAD);
+#endif
+
+    for (instr = instrlist_first(trace); instr!=NULL; instr = instr_get_next(instr)) {
+        if (instr_get_opcode(instr)==OP_mov_imm) {
+            cons = (int) opnd_get_immed_int(instr_get_src(instr,0)); 
+            cons_size = opnd_get_size(instr_get_src(instr, 0));
+            regop = instr_get_dst(instr,0);
+            reg = opnd_get_reg(regop);
+
+            LOG(THREAD, LOG_OPTS, 3,"reg=%s, val="PFX" ",reg_names[reg],cons);
+            loginst(dcontext,3,instr,"trying to remove me via constant prop");
+            
+            conwalker = instr_get_next(instr);
+            mov_immed_needed = false;
+            reg_overwritten=false;
+            while (conwalker!=NULL ) {
+                conwalker_next=instr_get_next(conwalker);
+
+                if (instr_is_cti(conwalker)&&!mov_immed_needed) {
+                    opnd_t target=instr_get_target(conwalker);
+                    if (opnd_is_near_pc(target)&& (opnd_get_pc(target)==tag)) {
+
+                        loginst(dcontext,3,conwalker,"this CTI is a self-loop");
+                        if (instrlist_depends_on_reg(dcontext,trace,reg)) {
+                            LOG(THREAD, LOG_OPTS,3,"\tsubsequent loop iteration needs the val, so keep mov_immed\n");
+                            mov_immed_needed = true;
+                        }
+                        else {
+                            LOG(THREAD, LOG_OPTS,3,"\tloop doesn't need this mov_immed, so lose it\n");
+                        }
+                    }
+
+                    else {
+                        instr_t *foo;
+                        loginst(dcontext, 3, conwalker, "reached cti without overwriting reg, trying to add instr to the pseudo exitstub");
+                        loginst(dcontext,3,conwalker,"this CTI is NOT a self-loop");
+                        instr_add_to_exitexec_list(dcontext,conwalker,instr_clone(dcontext,instr));
+                        
+#ifdef DEBUG
+                        if (stats->loglevel >= 3)
+                            for (foo=instr;foo!=conwalker->next;foo=instr_get_next(foo))
+                                loginst(dcontext,3,foo,"\twalking\t");
+#endif
+                    }
+                }
+                else if (instr_get_opcode(conwalker)==OP_lahf/*to catch ind branch*/) {
+                    loginst(dcontext, 3, conwalker, "reached cti without overwriting reg, mov_imm needed");
+                    mov_immed_needed = true;
+                }
+
+
+                if (!instr_replace_reg_with_const_in_src(dcontext,conwalker,reg,cons)) {
+                    if (instr_get_opcode(conwalker)==OP_cmp) {
+                        instr_t *oldnext;
+                        loginst(dcontext,3,conwalker,"trying to optimize this cmp");
+                        orig1=instr_get_src(conwalker,0);
+                        orig2=instr_get_src(conwalker,1);
+                        
+                        //replace the register with the constant
+                        logopnd(dcontext,3,regop,"trying to replace this operand");
+                        instr_replace_src_opnd(conwalker,regop,OPND_CREATE_INT32(cons));
+                        loginst(dcontext,3,conwalker,"replaced reg with const int");
+                        //if they're both immeds
+                        oldnext=conwalker_next;
+                        conwalker_next=fix_cmp_containing_constant(dcontext,trace,conwalker,orig1,orig2);
+                        if (oldnext==conwalker_next)
+                            mov_immed_needed=true;
+                    }
+                    else { //if its not a compare
+                        loginst(dcontext, 3, conwalker, "couldn't replace reg in src, mov_immed needed");
+                        mov_immed_needed=true;
+                    }
+                }
+                
+                if (!instr_replace_reg_with_const_in_dst(dcontext,conwalker,reg,cons)) {
+                    loginst(dcontext, 3, conwalker, "couldn't replace reg in dst, mov_immed needed");
+                    mov_immed_needed=true;
+                }
+
+                if (instr_writes_to_reg(conwalker,reg)) {
+                    loginst(dcontext, 3, conwalker,"writes to the reg, so move on");
+                    reg_overwritten=true;
+                    //if it doesn't overwrite the entire register
+                    //then there is still a dependancy on the full 32 bit previous value
+                    //so keep the immediate... man, x86 sucks
+                    if (!instr_writes_to_exact_reg(conwalker,reg)) {
+                        mov_immed_needed=true;
+                        loginst(dcontext,3,conwalker,"\tmov_imm needed because of me");
+                    }
+                    break;
+                }
+                
+                conwalker = conwalker_next;
+            }
+
+            //josh just changed this line
+            if (!reg_overwritten) {
+                mov_immed_needed=true;
+                LOG(THREAD, LOG_OPTS,3,"\tmov immed needed because the reg isn't overwritten");
+            }
+
+            if (!mov_immed_needed) {
+                /* remove it */
+                next = instr_get_next(instr);
+                /* if you're removing the first element in trace
+                 * then fix loop to point to second element (which
+                 * becomes first element)
+                 */
+                
+                if (instr == instrlist_first(trace)) {
+                    loginst(dcontext, 3, instr,"trying to remove the first item in the trace");
+                    replace_self_loop_with_opnd(dcontext, (size_t)0,
+                                                 trace, opnd_create_instr(next));
+                }
+                loginst(dcontext,3,instr,"this mov imm isn't needed, remove");
+                instrlist_remove(trace,instr);
+                instr_destroy(dcontext,instr);
+                instr = next;
+            }
+        }
+    }
+    ASSERT(instr_get_opcode(instrlist_last(trace))==OP_jmp);
+
+    instrlist_setup_pseudo_exitstubs(dcontext,trace);
+
+
+    LOG(THREAD, LOG_OPTS,3,"after constant_propagate\n");
+#ifdef DEBUG
+    if (stats->loglevel >= 3)
+        instrlist_disassemble(dcontext, 0, trace, THREAD);
+#endif
+}
+//returns true is all's good. (either replaced successfully, or no reg. reference)
+//returns false is there's a problem and the mov_immed is needed
+bool
+instr_replace_reg_with_const_in_src(dcontext_t *dcontext,instr_t *in,int reg,int val)
+{
+    int a;
+    opnd_t op,oldop;
+    ASSERT(instr_is_encoding_possible(in));
+
+    for (a=0; a<instr_num_srcs(in); a++) {
+        oldop = op = instr_get_src(in, a);
+
+        if (opnd_is_reg(op)&&!opnd_is_reg_32bit(op)&&(reg_fixer[opnd_get_reg(op)]==reg_fixer[reg])) {
+            LOG(THREAD, LOG_OPTS,3,"problems with sub registers\n");
+            return false; //if its a sub register of reg, then we can't do anything and need the immed
+        }
+
+
+        if (opnd_replace_reg_with_val(&op, reg, val)) {
+            instr_set_src(in,a,op);
+            loginst(dcontext, 3, in, "replaced by this in src");
+            //THIS SHOULD BE FIXED BY FIXING is_encoding_possible so it doesn't
+            //give true when there is a constant in a rep* instruction
+            //when thats fixed, then the next if statement can be 
+
+            //fix me: should handle other things like rep_stos?
+            if ((instr_get_opcode(in)==OP_rep_cmps||instr_get_opcode(in)==OP_rep_movs)
+                 && (reg==REG_ECX)) {
+                loginst(dcontext, 3, in, "quirk involving rep* instructions and ECX");
+                instr_set_src(in, a, oldop);
+                return false;
+            }
+
+            /* this check checks the case where a single register is used as 2 or more sources
+               our code doesn't handle this yet (the constant propagator should be able to do
+               some arithmetic simplification with this case).
+            */
+            if (instr_reg_in_src(in,reg)) { 
+                loginst(dcontext,3,in,"FIXME: doesn't yet handle two src instances of the same register");
+                instr_set_src(in, a, oldop);
+                return false; 
+            }
+
+            instr_arithmatic_simplify(dcontext,in);
+
+        }
+
+        if (!instr_is_encoding_possible(in)) {
+            loginst(dcontext, 3, in,"replace_reg_with_const: encoding not possible, so putting back original");
+            instr_set_src(in, a, oldop);
+            loginst(dcontext, 3, in,"original instr");
+            ASSERT(instr_is_encoding_possible(in));
+            return false;
+        }
+    }
+    return true;
+}
+
+//returns true is all's good. (either replaced successfully, or no reg. reference)
+//returns false is there's a problem and the mov_immed is needed
+bool
+instr_replace_reg_with_const_in_dst(dcontext_t *dcontext,instr_t *in,int reg,int val)
+{
+    int a;
+    opnd_t op,oldop;
+    for (a=0; a<instr_num_dsts(in); a++) {
+        oldop = op = instr_get_dst(in, a);
+        if (opnd_is_memory_reference(oldop)) {
+
+            if (opnd_replace_reg_with_val(&op, reg, val)) {
+                instr_set_dst(in, a, op);
+                loginst(dcontext, 3, in,"replaced by this in dst");
+            }
+            if (!instr_is_encoding_possible(in)) {
+                loginst(dcontext, 3, in,"encoding not possible, so putting back original");
+                instr_set_dst(in, a, oldop);
+                loginst(dcontext, 3, in, "original instr");
+                ASSERT(instr_is_encoding_possible(in));
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
+/* Ignores segments */
+bool
+opnd_replace_reg_with_val(opnd_t *opnd, int old_reg, int val)
+{
+    switch (opnd->kind) {
+    case NULL_kind:
+    case IMMED_INTEGER_kind:
+    case IMMED_FLOAT_kind:
+    case PC_kind:
+    case FAR_PC_kind:
+    case INSTR_kind: 
+    case FAR_INSTR_kind: 
+#ifdef X64
+    case REL_ADDR_kind:
+    case ABS_ADDR_kind:
+#endif
+        return false;
+
+    case REG_kind: 
+        if (old_reg == opnd_get_reg(*opnd)) {
+            *opnd = OPND_CREATE_INT32(val);
+            return true;
+        }
+        return false;
+                
+    case BASE_DISP_kind:
+        {
+            byte size = opnd_get_size(*opnd);
+            if (old_reg == opnd_get_base(*opnd)) {
+                int b = REG_NULL;
+                int i = opnd_get_index(*opnd);
+                int s = opnd_get_scale(*opnd);
+                int d = opnd_get_disp(*opnd)+val;
+                *opnd = opnd_create_base_disp(b, i, s, d, size);
+                return true;
+            }
+            else if (old_reg == opnd_get_index(*opnd)) {
+                int b = opnd_get_base(*opnd);
+                int i = REG_NULL;
+                int s = 0;
+                int d = opnd_get_disp(*opnd)+val*opnd_get_scale(*opnd);
+                *opnd = opnd_create_base_disp(b, i, s, d, size);
+                return true;
+            }
+        }
+        return false;
+
+    case FAR_BASE_DISP_kind:
+        {
+            byte size = opnd_get_size(*opnd);
+            int seg = opnd_get_segment(*opnd);
+            if (old_reg == opnd_get_base(*opnd)) {
+                int b = REG_NULL;
+                int i = opnd_get_index(*opnd);
+                int s = opnd_get_scale(*opnd);
+                int d = opnd_get_disp(*opnd)+val;
+                *opnd = opnd_create_far_base_disp(seg, b, i, s, d, size);
+                return true;
+            }
+            else if (old_reg == opnd_get_index(*opnd)) {
+                int b = opnd_get_base(*opnd);
+                int i = REG_NULL;
+                int s = 0;
+                int d = opnd_get_disp(*opnd)+val*opnd_get_scale(*opnd);
+                *opnd = opnd_create_far_base_disp(seg, b, i, s, d, size);
+
+                return true;
+            }
+        }
+        return false;
+
+    default: 
+        ASSERT_NOT_REACHED(); 
+        return false;
+    }    
+}
+
+void
+replace_self_loop_with_opnd(dcontext_t *dcontext, app_pc tag,
+                                  instrlist_t *trace, opnd_t desiredtarget) {
+    instr_t *top=instrlist_first(trace),*in;
+    opnd_t targetop;
+
+    in=top;
+
+    LOG(THREAD, LOG_OPTS,3,"entering replace_self_loop_with_opnd looking for tag "PFX".\n",tag);
+
+    
+    while (in!=NULL) {
+#ifdef DEBUG
+        loginst(dcontext,3,in,"examining me in replace self loop");
+        LOG(THREAD, LOG_OPTS,3,"my bytes are: "PFX"\n",in->bytes);
+#endif
+        if (instr_is_cbr(in)||instr_is_ubr(in)) {
+            targetop=instr_get_target(in);
+            if (opnd_is_near_pc(targetop)&&opnd_get_pc(targetop)==tag) {
+                loginst(dcontext,3,in,"self_loop (pc target==tag) fixing in");
+                instr_set_target(in,desiredtarget);
+            }
+            else if (opnd_is_near_instr(targetop)&&opnd_get_instr(targetop)==top) {
+                loginst(dcontext,3,in,"self_loop (inter traget==top)fixing in");
+                logopnd(dcontext,3,desiredtarget,"self_loop in now points to");
+                instr_set_target(in,desiredtarget);
+            }
+        }
+        in=instr_get_next(in);
+    }
+    
+}
+
+/*
+  copies replacee into in. frees replacee (with heap_free, not
+  instr_free), so that in uses replacee's operands */
+
+
+
+
+/* this function tries to change the instr into a mov_immed */
+void
+instr_arithmatic_simplify(dcontext_t *dcontext, instr_t *in)
+{
+    opnd_t op1,op2,dst;
+    instr_t *newinstr=NULL;
+    int opcode;
+    int newvalue;
+    loginst(dcontext,3,in,"arithmatic simplify called on");
+
+    opcode=instr_get_opcode(in);
+    if (instr_flag_write_necessary(dcontext,in))
+        return;
+
+    /* first check the single source instructions to make encodable and optimize */
+    if ((instr_num_srcs(in)==1) && 
+        opnd_is_immed_int(op1=instr_get_src(in,0))) {
+        opnd_t dst;
+        int val=(int) opnd_get_immed_int(op1);
+        dst=instr_get_dst(in,0);
+        switch (opcode) {
+        case OP_mov_ld:
+            newinstr=INSTR_CREATE_mov_imm(dcontext,instr_get_dst(in,0),op1);
+            /*      
+                    is this fucking necessary?
+                    if (!instr_is_encoding_possible(newinstr)) {
+                    instr_free(dcontext,newinstr);
+                    newinstr=INSTR_CREATE_mov_st(dcontext,instr_get_dst(in,0),op1);
+                    }
+            */
+            break;
+            /* untested by josh
+        case OP_fild:
+        case OP_fld:
+            if (val==0 && opnd_is_reg(dst)&& (opnd_get_reg(dst)==REG_ST0)) {
+                loginst(dcontext,3,in,"arithmatic simplify: making fload of 0 to store!");
+                newinstr=INSTR_CREATE_fldz(dcontext);
+                break;
+            }
+            */
+        case OP_inc:
+            loginst(dcontext,3,in,"arithmatic simplify: making inc to store!");
+            newinstr=INSTR_CREATE_mov_imm(dcontext,instr_get_dst(in,0),OPND_CREATE_INT32(val+1));
+            break;
+        case OP_dec:
+            loginst(dcontext,3,in,"arithmatic simplify: making dec to store!");
+            newinstr=INSTR_CREATE_mov_imm(dcontext,instr_get_dst(in,0),OPND_CREATE_INT32(val-1));
+            break;
+
+        case OP_push:
+            newinstr=INSTR_CREATE_push_imm(dcontext,op1);
+            break;
+        }
+    }
+    else if (instr_num_srcs(in)==2) {
+        op1=instr_get_src(in,0);
+        op2=instr_get_src(in,1);
+        
+        if (opnd_is_immed_int(op1) && opnd_is_immed_int(op2)) {
+            /* only if both operands are immediates... */
+            int val1,val2;
+            val1=(int) opnd_get_immed_int(op1);
+            val2=(int) opnd_get_immed_int(op2);
+            
+            dst=instr_get_dst(in,0);
+            switch (opcode) {
+            case OP_sar: 
+                newvalue = val2>>val1; //why is this order weird
+                newinstr=INSTR_CREATE_mov_imm(dcontext,dst,OPND_CREATE_INT32(newvalue));
+                break;
+                //fix me: don't optimize away the carry trick
+            case OP_add:
+                newvalue = val2+val1;
+                newinstr=INSTR_CREATE_mov_imm(dcontext,dst,OPND_CREATE_INT32(newvalue));
+                loginst(dcontext,3,in,"arithmatic simplify: OP_ADD!");
+                break;
+            case OP_imul:
+                newvalue = val2*val1;
+                newinstr=INSTR_CREATE_mov_imm(dcontext,dst,OPND_CREATE_INT32(newvalue));
+                loginst(dcontext,3,in,"arithmatic simplify: OP_imul!");
+                break;
+            case OP_mul:
+                newvalue = (unsigned)val2*(unsigned)val1;
+                newinstr=INSTR_CREATE_mov_imm(dcontext,dst,OPND_CREATE_INT32(newvalue));
+                loginst(dcontext,3,in,"arithmatic simplify: OP_mul!");
+                break;
+            case OP_and:
+                newvalue = val2&val1;
+                newinstr=INSTR_CREATE_mov_imm(dcontext,dst,OPND_CREATE_INT32(newvalue));
+                loginst(dcontext,3,in,"arithmatic simplify: OP_and!");
+                break;
+            case OP_or:
+                newvalue = val2|val1;
+                newinstr=INSTR_CREATE_mov_imm(dcontext,dst,OPND_CREATE_INT32(newvalue));
+                loginst(dcontext,3,in,"arithmatic simplify: OP_or!");
+                break;
+            case OP_xor:
+                newvalue = val2^val1;
+                newinstr=INSTR_CREATE_mov_imm(dcontext,dst,OPND_CREATE_INT32(newvalue));
+                loginst(dcontext,3,in,"arithmatic simplify: OP_xor!");
+                break;
+            case OP_sub:
+                newvalue = val2-val1;
+                newinstr=INSTR_CREATE_mov_imm(dcontext,dst,OPND_CREATE_INT32(newvalue));
+                loginst(dcontext,3,in,"arithmatic simplify: OP_sub!");
+                break;
+            }
+        }
+        else if (opnd_is_immed_int(op1) || opnd_is_immed_int(op2)) {
+            /* only gets here if exactly one operand is an immediate */
+            int cons;
+            opnd_t other;
+            if (opnd_is_immed_int(op1)) {
+                cons=(int) opnd_get_immed_int(op1);
+                other=op2;
+            }
+            else {
+                cons=(int) opnd_get_immed_int(op2);
+                other=op1;
+            }
+            dst=instr_get_dst(in,0);
+            switch (opcode) {
+            case OP_and:
+                if (cons==0xffffffff) {
+                    newinstr=INSTR_CREATE_nop(dcontext);
+                    loginst(dcontext,3,in,"arithmatic simplify: turned OP_and to OP_nop");
+                }
+                else if (cons==0) {
+
+                    if (opnd_is_reg(dst)) {
+                        newinstr=INSTR_CREATE_xor(dcontext,dst,dst);
+                        loginst(dcontext,3,in,"arithmatic simplify: turned OP_and to zeroing xor");
+                    }
+                    else {
+                        ASSERT(opnd_is_memory_reference(dst));
+                        newinstr=INSTR_CREATE_mov_imm(dcontext,dst,OPND_CREATE_INT32(0));
+                        loginst(dcontext,3,in,"arithmatic simplify: turned OP_and to zeroing mov_imm");
+                    }
+                }
+                break;
+                
+            case OP_or:
+                if (cons==0xffffffff) {
+                    newinstr=INSTR_CREATE_mov_imm(dcontext,dst,OPND_CREATE_INT32(0xffffffff));
+                    loginst(dcontext,3,in,"arithmatic simplify: turned OP_or into mov 0xffffffff");
+                }
+                else if (cons==0) {
+                    newinstr=INSTR_CREATE_nop(dcontext);
+                    loginst(dcontext,3,in,"arithmatic simplify: turned OP_or into nop");
+                }
+                break;
+            case OP_add:
+                if (cons==0) {
+                    newinstr=INSTR_CREATE_nop(dcontext);
+                    loginst(dcontext,3,in,"arithmatic simplify: turned OP_add into nop");
+                }
+                break;
+            case OP_sub:
+                /* fixme
+                if (cons==0) {
+                    newinstr=INSTR_CREATE_nop(dcontext);
+                    loginst(dcontext,3,in,"arithmatic simplify: turned OP_sub into nop");
+                }
+                */
+                break;
+            case OP_mul:
+            case OP_imul:
+                if (cons==1) {
+                    newinstr=INSTR_CREATE_nop(dcontext);
+                    loginst(dcontext,3,in,"arithmatic simplify: turned OP_add into nop");
+                }
+                else if (cons==0) {
+                    newinstr=INSTR_CREATE_mov_imm(dcontext,dst,OPND_CREATE_INT32(0));
+                    loginst(dcontext,3,in,"arithmatic simplify: turned OP_mul with 0 to mov_immed");
+                }
+                break;
+
+            }
+        }
+    }
+    
+    if (newinstr) {
+        loginst(dcontext,3,newinstr,"with me");
+        
+        ASSERT(instr_is_encoding_possible(newinstr));
+        
+        instr_replace_inplace(dcontext,in,newinstr);
+    }
+    
+    
+}
+
+void
+instr_replace_inplace(dcontext_t *dcontext, instr_t *in, instr_t *replacee)
+{
+    instr_t *next,*prev;
+    next=instr_get_next(in);
+    prev=instr_get_prev(in);
+
+    instr_reset(dcontext,in);
+    memcpy(in,replacee,sizeof(instr_t));
+
+    instr_set_next(in,next);
+    instr_set_prev(in,prev);
+    heap_free(dcontext,replacee,sizeof(instr_t)
+              HEAPACCT(ACCT_INSTR));
+}
+
+void
+instrlist_remove_nops(dcontext_t *dcontext,instrlist_t *trace)
+{
+    instr_t *in, *next,*walker;
+    in=instrlist_first(trace);
+
+
+    while (in) {
+        next=instr_get_next(in);
+
+        if (instr_get_opcode(in)==OP_nop) {
+
+            //fix any instr targets that point to this instr
+            for (walker=instrlist_first(trace);walker;walker=instr_get_next(walker))
+                if (instr_is_cbr(walker)||instr_is_ubr(walker)) {
+                    opnd_t targetop=instr_get_target(walker);
+                    if (opnd_is_near_instr(targetop)&&(in==opnd_get_instr(targetop)))
+                        instr_set_target(walker,opnd_create_instr(instr_get_next(in)));
+                }
+
+            instrlist_remove(trace,in);
+            instr_destroy(dcontext,in);
+        }
+        in=next;
+    }
+
+}
+
+/*
+  checks if the instr list depends on the value of reg
+  but does not check if any instr lists that it calls depend on the value.
+*/
+
+bool
+instrlist_depends_on_reg(dcontext_t *dcontext, instrlist_t *trace, int reg)
+{
+    instr_t *in;
+
+
+    for (in=instrlist_first(trace);in;in=instr_get_next(in)) {
+
+        if (instr_reg_in_src(in,reg))
+            return true;
+
+        if (instr_reg_in_dst(in,reg)) {
+            if (instr_writes_to_reg(in,reg))
+                return false;
+            else
+                return true; //register is read indirect through mem access
+        }
+    }
+
+    
+    return true;
+}
+
+void
+instr_add_to_exitexec_list(dcontext_t *dcontext, instr_t *in, instr_t *exitinstr)
+{
+    ASSERT(instr_is_cti(in));
+
+    if (!in->exitlist)
+        in->exitlist=instrlist_create(dcontext);
+    
+    instrlist_append(in->exitlist,exitinstr);
+    loginst(dcontext,3,exitinstr,"adding this to exit list");
+    LOG(THREAD, LOG_OPTS,3,"exitinstr="PFX"\n",exitinstr);
+}
+void
+instrlist_setup_pseudo_exitstubs(dcontext_t *dcontext,instrlist_t *trace)
+{
+    instrlist_t *exitlist;
+    instr_t *instr,*newtarget;
+
+    exitlist=instrlist_create(dcontext);
+
+    for (instr = instrlist_first(trace); instr!=NULL; instr = instr_get_next(instr)) {
+        if (instr->exitlist) {
+            ASSERT(instr_is_cti(instr));
+            instrlist_append(instr->exitlist,INSTR_CREATE_jmp(dcontext,
+                                                              instr_get_target(instr)));
+            loginst(dcontext,3,instr,"setting up pseudo exit stub for me");
+
+            newtarget=instrlist_first(instr->exitlist);
+            instrlist_append_instrlist(dcontext,exitlist,instr->exitlist);
+
+            instr_set_target(instr,opnd_create_instr(newtarget));
+
+            ASSERT(instr_is_encoding_possible(instr));
+
+            instr->exitlist=NULL;
+#ifdef DEBUG
+            if (stats->loglevel >= 3) {
+                loginst(dcontext,3,instr,"after setting");
+                instrlist_disassemble(dcontext, 0, trace, THREAD);
+                instrlist_disassemble(dcontext, 0, exitlist, THREAD);
+            }
+#endif
+
+        }
+    }
+
+
+    instrlist_append_instrlist(dcontext,trace,exitlist);
+}
+
+
+#endif
diff --git a/core/x86/loadtoconst.h b/core/x86/loadtoconst.h
new file mode 100644
index 000000000..06c5d26c0
--- /dev/null
+++ b/core/x86/loadtoconst.h
@@ -0,0 +1,115 @@
+/* **********************************************************
+ * Copyright (c) 2002-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+
+#ifdef LOAD_TO_CONST
+#ifndef __LOADTOCONST_H_
+#define __LOADTOCONST_H_
+
+#define NUM_VALUES_FOR_SPECULATION 40
+#define SAMPLE_PERCENT (9/10)
+
+
+#define MAX_TRACES_WAITING_FOR_LTC 20
+#include "instr.h"
+
+struct ltc_mem_ref_data{
+    opnd_t opnd;
+    int vals[NUM_VALUES_FOR_SPECULATION];
+    int addresses[NUM_VALUES_FOR_SPECULATION];
+};
+struct ltc_data{
+    struct ltc_mem_ref_data *mem_refs;
+    int num_mem_addresses;
+    int num_mem_samples;
+    bool ltc_already_optimized;
+};
+
+#ifdef LTC_STATS
+extern int safe_taken,opt_taken,addrs_analyzed,addrs_made_const,traces_analyzed,addrs_seen;
+#endif
+
+void analyze_memrefs(dcontext_t *dcontext, app_pc tag, instrlist_t *trace);
+
+
+int check_mem_refs(app_pc tag, int errno, reg_t eflags,
+                   reg_t reg_edi, reg_t reg_esi, reg_t reg_ebp, reg_t reg_esp,
+                   reg_t reg_ebx, reg_t reg_edx, reg_t reg_ecx, reg_t reg_eax);
+int get_mem_address(dcontext_t *dcontext,opnd_t mem_access,int regs[8]);
+int get_mem_val(dcontext_t *dcontext,opnd_t mem_access,int address);
+void ltc_trace(dcontext_t *dcontext, fragment_t *, instrlist_t *trace);
+void remove_mem_ref_check(dcontext_t *dcontext,instrlist_t *trace);
+
+#ifdef SIDELINE
+void LTC_examine_traces(void);
+void LTC_fragment_delete(fragment_t *frag);
+#endif
+
+void LTC_online_optimize_and_replace(dcontext_t *dcontext, app_pc tag,fragment_t *curfrag);
+void do_single_LTC(dcontext_t *dcontext,instrlist_t *trace, opnd_t mem_access, opnd_t value);
+bool should_replace_load(dcontext_t *dcontext,struct ltc_mem_ref_data data);
+opnd_t value_to_replace(struct ltc_mem_ref_data);
+bool instr_replace_reg_with_const_in_src(dcontext_t *dcontext,instr_t *in,int reg,int val);
+bool instr_replace_reg_with_const_in_dst(dcontext_t *dcontext,instr_t *in,int reg,int val);
+instrlist_t *save_eflags_list(dcontext_t *dcontext,fragment_t *frag);
+instrlist_t *restore_eflags_list(dcontext_t *dcontext,fragment_t *frag);
+void change_cbr_due_to_reversed_cmp(instr_t *in);
+bool pc_reads_flags_before_writes(dcontext_t *dcontext,app_pc target);
+
+instr_t *fix_cmp_containing_constant(dcontext_t *dcontext,instrlist_t *trace,instr_t *in,
+                                   opnd_t orig_op1,opnd_t orig_op2);
+
+void replace_self_loop_with_opnd(dcontext_t *dcontext, app_pc tag,
+                                 instrlist_t *ilist, opnd_t desiredtarget);
+#define TRANSPOSE true
+#define NO_TRANSPOSE false
+bool safe_to_transpose_cmp(dcontext_t *dcontext,instr_t *testinstr);
+bool safe_to_delete_cmp(dcontext_t *dcontext,instr_t *testinstr);
+bool safe_to_modify_cmp(dcontext_t *dcontext,instr_t *testinstr, bool transpose);
+bool becomes_ubr_from_cmp(instr_t *in,int op1, int op2);
+bool opnd_replace_reg_with_val(opnd_t *opnd, int old_reg, int val);
+void constant_propagate(dcontext_t *dcontext, instrlist_t *trace,app_pc tag);
+void pop_instr_off_list(dcontext_t *dcontext,instrlist_t *trace,int opcode);
+void instr_optimize_constant(dcontext_t *dcontext, instr_t *in);
+bool instr_flag_write_necessary(dcontext_t *dcontext, instr_t *in);
+
+void instr_replace_inplace(dcontext_t *dcontext, instr_t *in, instr_t *);
+void instr_arithmatic_simplify(dcontext_t *dcontext, instr_t *in);
+void instrlist_remove_nops(dcontext_t *dcontext,instrlist_t *trace);
+bool instrlist_depends_on_reg(dcontext_t *dcontext, instrlist_t *trace, int reg);
+void instr_add_to_exitexec_list(dcontext_t *,instr_t *in,instr_t *exitinstr);
+void instrlist_setup_pseudo_exitstubs(dcontext_t *dcontext,instrlist_t *trace);
+#endif
+#endif
+
diff --git a/core/x86/mangle.c b/core/x86/mangle.c
new file mode 100644
index 000000000..ff4b0adff
--- /dev/null
+++ b/core/x86/mangle.c
@@ -0,0 +1,3814 @@
+/* **********************************************************
+ * Copyright (c) 2000-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/* file "mangle.c" */
+
+#ifdef LINUX
+#include <sys/syscall.h>
+#endif
+
+#include "globals.h"
+#include "arch.h"
+#include "link.h"
+#include "fragment.h"
+#include "instrlist.h"
+#include "arch.h"
+#include "instr.h"
+#include "instr_create.h"
+#include "decode.h"
+#include "decode_fast.h"
+#include "disassemble.h"
+#ifdef STEAL_REGISTER
+#include "steal_reg.h"
+#endif
+#include "instrument.h" /* for dr_insert_call */
+
+#ifdef RCT_IND_BRANCH
+# include "rct.h" /* rct_add_rip_rel_addr */
+#endif
+
+#ifdef WINDOWS
+/* in callback.c */
+extern void callback_start_return(void);
+#endif
+
+/* make code more readable by shortening long lines
+ * we mark everything we add as a meta-instr to avoid hitting
+ * client asserts on setting translation fields
+ */
+#define POST instrlist_meta_postinsert
+#define PRE  instrlist_meta_preinsert
+
+#if defined(NATIVE_RETURN) && !defined(DEBUG)
+extern int num_fragments;
+#endif
+
+/* Convert a short-format CTI into an equivalent one using
+ * near-rel-format.
+ * Remember, the target is kept in the 0th src array position,
+ * and has already been converted from an 8-bit offset to an
+ * absolute PC, so we can just pretend instructions are longer
+ * than they really are.
+ */
+static void
+convert_to_near_rel_common(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr)
+{
+    int opcode = instr_get_opcode(instr);
+    DEBUG_DECLARE(const instr_info_t * info = instr_get_instr_info(instr);)
+    app_pc target = NULL;
+
+    if (opcode == OP_jmp_short) {
+        instr_set_opcode(instr, OP_jmp);
+        return;
+    }
+
+    if (OP_jo_short <= opcode && opcode <= OP_jnle_short) {
+        /* WARNING! following is OP_ enum order specific */
+        instr_set_opcode(instr, opcode - OP_jo_short + OP_jo);
+        return;
+    }
+
+    if (OP_loopne <= opcode && opcode <= OP_jecxz) {
+        /*
+         * from "info as" on GNU/linux system:
+       Note that the `jcxz', `jecxz', `loop', `loopz', `loope', `loopnz'
+       and `loopne' instructions only come in byte displacements, so that if
+       you use these instructions (`gcc' does not use them) you may get an
+       error message (and incorrect code).  The AT&T 80386 assembler tries to
+       get around this problem by expanding `jcxz foo' to
+                     jcxz cx_zero
+                     jmp cx_nonzero
+            cx_zero: jmp foo
+            cx_nonzero:
+        *
+        * We use that same expansion, but we want to treat the entire
+        * three-instruction sequence as a single conditional branch.
+        * Thus we use a special instruction that stores the entire
+        * instruction sequence as mangled bytes, yet w/ a valid target operand
+        * (xref PR 251646).
+        * patch_branch and instr_invert_cbr
+        * know how to find the target pc (final 4 of 9 bytes).
+        * When decoding anything we've written we know the only jcxz or
+        * loop* instructions are part of these rewritten packages, and
+        * we use remangle_short_rewrite to read back in the instr.
+        * (have to do this everywhere call decode() except original
+        * interp, plus in input_trace())
+        *
+        * An alternative is to change 'jcxz foo' to:
+                    <save eflags>
+                    cmpb %cx,$0
+                    je   foo_restore
+                    <restore eflags>
+                    ...
+       foo_restore: <restore eflags>
+               foo:
+        * However the added complications of restoring the eflags on
+        * the taken-branch path made me choose the former solution.
+        */
+
+        /* SUMMARY: 
+         * expand 'shortjump foo' to:
+                          shortjump taken
+                          jmp-short nottaken
+                   taken: jmp foo
+                nottaken:
+        */
+        if (ilist != NULL) {
+            /* PR 266292: for meta instrs, insert separate instrs */
+            /* reverse order */
+            opnd_t tgt = instr_get_target(instr);
+            instr_t *nottaken = INSTR_CREATE_label(dcontext);
+            instr_t *taken = INSTR_CREATE_jmp(dcontext, tgt);
+            ASSERT(!instr_ok_to_mangle(instr));
+            instrlist_meta_postinsert(ilist, instr, nottaken);
+            instrlist_meta_postinsert(ilist, instr, taken);
+            instrlist_meta_postinsert(ilist, instr, INSTR_CREATE_jmp_short
+                                      (dcontext, opnd_create_instr(nottaken)));
+            instr_set_target(instr, opnd_create_instr(taken));
+            return;
+        }
+
+        if (opnd_is_near_pc(instr_get_target(instr)))
+            target = opnd_get_pc(instr_get_target(instr));
+        else if (opnd_is_near_instr(instr_get_target(instr))) {
+            instr_t *tgt = opnd_get_instr(instr_get_target(instr));
+            /* assumption: target's translation or raw bits are set properly */
+            target = instr_get_translation(tgt);
+            if (target == NULL && instr_raw_bits_valid(tgt))
+                target = instr_get_raw_bits(tgt);
+            ASSERT(target != NULL);
+        } else
+            ASSERT_NOT_REACHED();
+
+        /* PR 251646: cti_short_rewrite: target is in src0, so operands are
+         * valid, but raw bits must also be valid, since they hide the multiple
+         * instrs.  For x64, it is marked for re-relativization, but it's
+         * special since the target must be obtained from src0 and not
+         * from the raw bits (since that might not reach).
+         */
+        /* need 9 bytes */
+        instr_allocate_raw_bits(dcontext, instr, CTI_SHORT_REWRITE_LENGTH);
+        /* first 2 bytes: jecxz 8-bit-offset */
+        instr_set_raw_byte(instr, 0, decode_first_opcode_byte(opcode));
+        /* remember pc-relative offsets are from start of next instr */
+        instr_set_raw_byte(instr, 1, (byte)2);
+        /* next 2 bytes: jmp-short 8-bit-offset */
+        instr_set_raw_byte(instr, 2, decode_first_opcode_byte(OP_jmp_short));
+        instr_set_raw_byte(instr, 3, (byte)5);
+        /* next 5 bytes: jmp 32-bit-offset */
+        instr_set_raw_byte(instr, 4, decode_first_opcode_byte(OP_jmp));
+        /* for x64 we may not reach, but we go ahead and try */
+        instr_set_raw_word(instr, 5, (int)
+                           (target - (instr->bytes + CTI_SHORT_REWRITE_LENGTH)));
+        LOG(THREAD, LOG_INTERP, 2, "convert_to_near_rel: jecxz/loop* opcode\n");
+        /* original target operand is still valid */
+        instr_set_operands_valid(instr, true);
+        return;
+    }
+
+    LOG(THREAD, LOG_INTERP, 1, "convert_to_near_rel: unknown opcode: %d %s\n",
+        opcode, info->name);
+    ASSERT_NOT_REACHED();      /* conversion not possible OR not a short-form cti */
+}
+
+void
+convert_to_near_rel_meta(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr)
+{
+    convert_to_near_rel_common(dcontext, ilist, instr);
+}
+
+void
+convert_to_near_rel(dcontext_t *dcontext, instr_t *instr)
+{
+    convert_to_near_rel_common(dcontext, NULL, instr);
+}
+
+/* For jecxz and loop*, we create 3 instructions in a single
+ * instr that we treat like a single conditional branch.
+ * On re-decoding our own output we need to recreate that instr.
+ * This routine assumes that the instructions encoded at pc
+ * are indeed a mangled cti short.
+ * Assumes that the first instr has already been decoded into instr,
+ * that pc points to the start of that instr.
+ * Converts instr into a new 3-raw-byte-instr with a private copy of the
+ * original raw bits.
+ * Optionally modifies the target to "target" if "target" is non-null.
+ * Returns the pc of the instruction after the remangled sequence.
+ */
+byte *
+remangle_short_rewrite(dcontext_t *dcontext,
+                       instr_t *instr, byte *pc, app_pc target)
+{
+    ASSERT(instr_is_cti_short_rewrite(instr, pc));
+
+    /* first set the target in the actual operand src0 */
+    if (target == NULL) {
+        /* acquire existing absolute target */
+        int rel_target = *((int *)(pc+5));
+        target = pc + CTI_SHORT_REWRITE_LENGTH + rel_target;
+    }
+    instr_set_target(instr, opnd_create_pc(target));
+    /* now set up the bundle of raw instructions
+     * we've already read the first 2-byte instruction, jecxz/loop*
+     * they all take up CTI_SHORT_REWRITE_LENGTH bytes
+     */
+    instr_allocate_raw_bits(dcontext, instr, CTI_SHORT_REWRITE_LENGTH);
+    instr_set_raw_bytes(instr, pc, CTI_SHORT_REWRITE_LENGTH);
+    /* for x64 we may not reach, but we go ahead and try */
+    instr_set_raw_word(instr, 5, (int)(target - (pc + CTI_SHORT_REWRITE_LENGTH)));
+    /* now make operands valid */
+    instr_set_operands_valid(instr, true);
+    return (pc+CTI_SHORT_REWRITE_LENGTH);
+}
+
+/* Returns the amount of data pushed.  Does NOT fix up the xsp value pushed
+ * to be the value prior to any pushes for x64 as no caller needs that
+ * currently (they all build a dr_mcontext_t and have to do further xsp
+ * fixups anyway).
+ * Includes xmm0-5 for PR 264138.
+ * If stack_align16 is true, assumes the stack pointer is currently aligned
+ * on a 16-byte boundary.
+ */
+uint
+insert_push_all_registers(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+                          bool stack_align16)
+{
+    PRE(ilist, instr, INSTR_CREATE_lea
+        (dcontext, opnd_create_reg(REG_XSP),
+         OPND_CREATE_MEM_lea(REG_XSP, REG_NULL, 0, - XMM_SLOTS_SIZE)));
+    if (preserve_xmm_caller_saved()) {
+        /* PR 264138: we must preserve xmm0-5 if on a 64-bit kernel */
+        int i;
+        /* PR 266305: see discussion in emit_fcache_enter_shared on
+         * which opcode is better.  Note that the AMD optimization
+         * guide says to use movlps+movhps for unaligned stores, but
+         * for simplicity and smaller code I'm using movups anyway.
+         */
+        uint opcode = (proc_has_feature(FEATURE_SSE2) ?
+                       (stack_align16 ? OP_movdqa : OP_movdqu) :
+                       (stack_align16 ? OP_movaps : OP_movups));
+        ASSERT(proc_has_feature(FEATURE_SSE));
+        for (i=0; i<NUM_XMM_SAVED; i++) {
+            PRE(ilist, instr, instr_create_1dst_1src
+                (dcontext, opcode,
+                 opnd_create_base_disp(REG_XSP, REG_NULL, 0, i*XMM_REG_SIZE, OPSZ_16),
+                 opnd_create_reg(REG_XMM0 + (reg_id_t)i)));
+        }
+        ASSERT(i*XMM_REG_SIZE == XMM_SAVED_SIZE);
+        ASSERT(XMM_SAVED_SIZE <= XMM_SLOTS_SIZE);
+    }
+#ifdef X64
+    /* keep dr_mcontext_t order */
+    PRE(ilist, instr, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_R15)));
+    PRE(ilist, instr, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_R14)));
+    PRE(ilist, instr, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_R13)));
+    PRE(ilist, instr, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_R12)));
+    PRE(ilist, instr, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_R11)));
+    PRE(ilist, instr, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_R10)));
+    PRE(ilist, instr, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_R9)));
+    PRE(ilist, instr, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_R8)));
+    PRE(ilist, instr, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_RAX)));
+    PRE(ilist, instr, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_RCX)));
+    PRE(ilist, instr, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_RDX)));
+    PRE(ilist, instr, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_RBX)));
+    /* we do NOT match pusha xsp value */
+    PRE(ilist, instr, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_RSP)));
+    PRE(ilist, instr, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_RBP)));
+    PRE(ilist, instr, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_RSI)));
+    PRE(ilist, instr, INSTR_CREATE_push(dcontext, opnd_create_reg(REG_RDI)));
+    return 16*XSP_SZ + XMM_SLOTS_SIZE;
+#else
+    PRE(ilist, instr, INSTR_CREATE_pusha(dcontext));
+    return 8*XSP_SZ + XMM_SLOTS_SIZE;
+#endif
+}
+
+/* If stack_align16 is true, assumes the stack pointer is currently aligned
+ * on a 16-byte boundary.
+ */
+void
+insert_pop_all_registers(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+                         bool stack_align16)
+{
+#ifdef X64
+    /* in dr_mcontext_t order */
+    PRE(ilist, instr, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_RDI)));
+    PRE(ilist, instr, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_RSI)));
+    PRE(ilist, instr, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_RBP)));
+    /* skip xsp by popping into dead rbx */
+    PRE(ilist, instr, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_RBX)));
+    PRE(ilist, instr, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_RBX)));
+    PRE(ilist, instr, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_RDX)));
+    PRE(ilist, instr, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_RCX)));
+    PRE(ilist, instr, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_RAX)));
+    PRE(ilist, instr, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_R8)));
+    PRE(ilist, instr, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_R9)));
+    PRE(ilist, instr, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_R10)));
+    PRE(ilist, instr, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_R11)));
+    PRE(ilist, instr, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_R12)));
+    PRE(ilist, instr, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_R13)));
+    PRE(ilist, instr, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_R14)));
+    PRE(ilist, instr, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_R15)));
+#else
+    PRE(ilist, instr, INSTR_CREATE_popa(dcontext));
+#endif
+    if (preserve_xmm_caller_saved()) {
+        /* PR 264138: we must preserve xmm0-5 if on a 64-bit kernel */
+        int i;
+        /* See discussion in emit_fcache_enter_shared on which opcode
+         * is better. */
+        uint opcode = (proc_has_feature(FEATURE_SSE2) ?
+                       (stack_align16 ? OP_movdqa : OP_movdqu) :
+                       (stack_align16 ? OP_movaps : OP_movups));
+        ASSERT(proc_has_feature(FEATURE_SSE));
+        for (i=0; i<NUM_XMM_SAVED; i++) {
+            PRE(ilist, instr, instr_create_1dst_1src
+                (dcontext, opcode, opnd_create_reg(REG_XMM0 + (reg_id_t)i),
+                 opnd_create_base_disp(REG_XSP, REG_NULL, 0, i*XMM_REG_SIZE, OPSZ_16)));
+        }
+        ASSERT(i*XMM_REG_SIZE == XMM_SAVED_SIZE);
+        ASSERT(XMM_SAVED_SIZE <= XMM_SLOTS_SIZE);
+    }
+    PRE(ilist, instr, INSTR_CREATE_lea
+        (dcontext, opnd_create_reg(REG_XSP),
+         OPND_CREATE_MEM_lea(REG_XSP, REG_NULL, 0, XMM_SLOTS_SIZE)));
+}
+
+/* utility routines for inserting clean calls to an instrumentation routine 
+ * strategy is very similar to fcache_enter/return
+ * FIXME: try to share code with fcache_enter/return?
+ *
+ * first swap stacks to DynamoRIO stack:
+ *      SAVE_TO_UPCONTEXT %xsp,xsp_OFFSET
+ *      RESTORE_FROM_DCONTEXT dstack_OFFSET,%xsp
+ * now save app eflags and registers, being sure to lay them out on
+ * the stack in dr_mcontext_t order:
+ *      push $0 # for dr_mcontext_t.pc; wasted, for now
+ *      pushf
+ *      pusha # xsp is dstack-XSP_SZ*2; rest are app values
+ * clear the eflags for our usage
+ * ASSUMPTION (also made in x86.asm): 0 ok, reserved bits are not set by popf,
+ *                                    and clearing, not preserving, is good enough
+ *      push   $0
+ *      popf
+ * save app errno
+ *      .ifdef WINDOWS
+ *      call  _GetLastError@0
+ *      push  %eax  # put errno on top of stack
+ *      .else
+ *      RESTORE_FROM_DCONTEXT errno_OFFSET,%eax
+ *      push  %eax  # for symmetry w/ win32, rather than -> app_errno_OFFSET
+ *      .endif
+ * make the call
+ *      call routine
+ * restore app errno
+ *      .ifdef WINDOWS
+ *      # errno is on top of stack as 1st param
+ *      call  _SetLastError@4
+ *      # win32 API functions use __stdcall = callee clears args!
+ *      .else
+ *      pop    %eax
+ *      SAVE_TO_DCONTEXT %eax,errno_OFFSET
+ *      .endif
+ * restore app regs and eflags
+ *      popa
+ *      popf
+ *      lea XSP_SZ(xsp),xsp # clear dr_mcontext_t.pc slot
+ * restore app stack
+ *      RESTORE_FROM_UPCONTEXT xsp_OFFSET,%xsp
+ */
+
+void
+insert_get_mcontext_base(dcontext_t *dcontext, instrlist_t *ilist, 
+                         instr_t *where, reg_id_t reg)
+{
+    PRE(ilist, where, instr_create_restore_from_tls
+        (dcontext, reg, TLS_DCONTEXT_SLOT));
+
+    /* An extra level of indirection with SELFPROT_DCONTEXT */
+    if (TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask)) {
+        ASSERT_NOT_TESTED();
+        PRE(ilist, where, INSTR_CREATE_mov_ld
+            (dcontext, opnd_create_reg(reg), 
+             OPND_CREATE_MEMPTR(reg, offsetof(dcontext_t, upcontext))));
+    }
+}
+
+/* What prepare_for_clean_call() adds to xsp beyond sizeof(dr_mcontext_t) */
+static inline int
+clean_call_beyond_mcontext(void)
+{
+    return XSP_SZ/*errno*/ IF_X64(+ XSP_SZ/*align*/);
+}
+
+/* prepare_for and cleanup_after assume that the stack looks the same after
+ * the call to the instrumentation routine, since it stores the app state
+ * on the stack.
+ * Returns the size of the data stored on the DR stack.
+ * WARNING: this routine does NOT save the fp/mmx/sse state, to do that the
+ * instrumentation routine should call proc_save_fpstate() and then
+ * proc_restore_fpstate()
+ * (This is because of expense:
+ *   fsave takes 118 cycles!  
+ *   frstor (separated by 6 instrs from fsave) takes 89 cycles
+ *   fxsave and fxrstor are not available on HP machine!
+ *   supposedly they came out in PII
+ *   on balrog: fxsave 91 cycles, fxrstor 173)
+ *
+ * For x64, changes the stack pointer by a multiple of 16.
+ * 
+ * NOTE: The client interface's get/set mcontext functions and the
+ * hotpatching gateway rely on the app's context being available
+ * on the dstack in a particular format.  Do not corrupt this data
+ * unless you update all users of this data!
+ *
+ * NOTE : this routine clobbers TLS_XAX_SLOT and the XSP mcontext slot.
+ * We guarantee to clients that all other slots (except the XAX mcontext slot)
+ * will remain untouched.
+ *
+ * N.B.: insert_parameter_preparation (and our documentation for
+ * dr_prepare_for_call) assumes that this routine only modifies xsp
+ * and xax and no other registers.
+ */
+uint
+prepare_for_clean_call(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr)
+{
+    uint dstack_offs = 0;
+    /* Swap stacks.  For thread-shared, we need to get the dcontext
+     * dynamically rather than use the constant passed in here.  Save
+     * away xax in a TLS slot and then load the dcontext there.
+     */
+    if (SHARED_FRAGMENTS_ENABLED()) {
+        PRE(ilist, instr, instr_create_save_to_tls
+            (dcontext, REG_XAX, TLS_XAX_SLOT));
+
+        insert_get_mcontext_base(dcontext, ilist, instr,
+                                 REG_XAX);
+
+        PRE(ilist, instr, instr_create_save_to_dc_via_reg
+            (dcontext, REG_XAX, REG_XSP, XSP_OFFSET));
+
+        /* DSTACK_OFFSET isn't within the upcontext so if it's separate this won't
+         * work right.  FIXME - the dcontext accessing routines are a mess of shared
+         * vs. no shared support, separate context vs. no separate context support etc. */
+        ASSERT_NOT_IMPLEMENTED(!TEST(SELFPROT_DCONTEXT, dynamo_options.protect_mask));
+
+        PRE(ilist, instr, instr_create_restore_from_dc_via_reg
+            (dcontext, REG_XAX, REG_XSP, DSTACK_OFFSET));
+
+        /* restore xax before pushing the context on the dstack */
+        PRE(ilist, instr, instr_create_restore_from_tls
+            (dcontext, REG_XAX, TLS_XAX_SLOT));  
+    }
+    else {
+        PRE(ilist, instr, instr_create_save_to_dcontext(dcontext, REG_XSP, XSP_OFFSET));
+        PRE(ilist, instr, instr_create_restore_dynamo_stack(dcontext));
+    }
+    
+    /* Save flags and all registers, in dr_mcontext_t order.
+     * Leave a slot for pc, which we do not fill in: it's wasted for now.
+     * FIXME PR 218131: we could have a special dstack+XSP_SZ field that we
+     * start from, and avoid this push; should do that if we start adding more
+     * fields to dr_mcontext_t, like a flags field, that are not set here.
+     */
+    PRE(ilist, instr, INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32(0)));
+    dstack_offs += XSP_SZ;
+    PRE(ilist, instr, INSTR_CREATE_pushf(dcontext));
+    dstack_offs += XSP_SZ;
+    /* Base of dstack is 16-byte aligned, and we've done 2 pushes, so we're
+     * 16-byte aligned for x64
+     */
+    dstack_offs += insert_push_all_registers(dcontext, ilist, instr,
+                                             IF_X64_ELSE(true, false));
+
+    /* Note that we do NOT bother to put the correct pre-push app xsp value on the
+     * stack here, as an optimization for callees who never ask for it: instead we
+     * rely on dr_[gs]et_mcontext() to fix it up if asked for.  We can get away w/
+     * this while hotpatching cannot (hotp_inject_gateway_call() fixes it up every
+     * time) b/c the callee has to ask for the dr_mcontext_t.
+     */
+
+    /* clear eflags for callee's usage */
+    PRE(ilist, instr,
+        INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32(0)));
+    PRE(ilist, instr, INSTR_CREATE_popf(dcontext));
+
+#ifdef WINDOWS
+    /* must preserve the LastErrorCode (if call a Win32 API routine could
+     * overwrite the app's error code) */
+    preinsert_get_last_error(dcontext, ilist, instr,
+                             REG_EAX);
+    /* by pushing errno onto stack, it's then in place to be an argument
+     * to SetLastError for cleanup!
+     */
+    /* FIXME: no longer necessary, update this and cleanup_after_clean_call,
+       All cleanup_call users most importantly, pre_system_call and 
+       post_system_call would need not to reserve room for errno */
+#else
+    /* put shared errno on stack, for symmetry w/ win32, rather than
+     * into app storage slot */
+    if (SHARED_FRAGMENTS_ENABLED()) {
+        /* eax is dead here (already saved to stack) */
+        insert_get_mcontext_base(dcontext, ilist, instr, REG_XAX);
+        PRE(ilist, instr, instr_create_restore_from_dc_via_reg
+            (dcontext, REG_XAX, REG_EAX, ERRNO_OFFSET));
+    } else {
+        PRE(ilist, instr,
+            instr_create_restore_from_dcontext(dcontext, REG_EAX, ERRNO_OFFSET));
+    }
+#endif
+    PRE(ilist, instr,
+        /* top 32 bits were zeroed on x64 */
+        INSTR_CREATE_push(dcontext, opnd_create_reg(REG_XAX)));
+    dstack_offs += XSP_SZ;
+#ifdef X64
+    /* PR 218790: maintain 16-byte rsp alignment.
+     * insert_parameter_preparation() currently assumes we leave rsp aligned.
+     */
+    PRE(ilist, instr, INSTR_CREATE_lea
+        (dcontext, opnd_create_reg(REG_XSP),
+         OPND_CREATE_MEM_lea(REG_XSP, REG_NULL, 0, -(int)XSP_SZ)));
+    dstack_offs += XSP_SZ;
+#endif
+    ASSERT(dstack_offs == sizeof(dr_mcontext_t) + clean_call_beyond_mcontext());
+    return dstack_offs;
+}
+
+void
+cleanup_after_clean_call(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr)
+{
+    /* saved error code is currently on the top of the stack */
+
+#ifdef X64
+    /* PR 218790: remove the padding we added for 16-byte rsp alignment */
+    PRE(ilist, instr, INSTR_CREATE_lea
+        (dcontext, opnd_create_reg(REG_XSP),
+         OPND_CREATE_MEM_lea(REG_XSP, REG_NULL, 0, XSP_SZ)));
+#endif
+    /* restore app's error code */
+    PRE(ilist, instr,
+        /* top 32 bits were zeroed on x64 */
+        INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_XAX)));
+#ifdef WINDOWS
+    /* must preserve the LastErrorCode (if call a Win32 API routine could
+     * overwrite the app's error code) */
+    preinsert_set_last_error(dcontext, ilist, instr, REG_EAX);
+#else
+    if (SHARED_FRAGMENTS_ENABLED()) {
+        /* xbx is dead (haven't restored yet) and eax contains the errno */
+        insert_get_mcontext_base(dcontext, ilist, instr, REG_XBX);
+        PRE(ilist, instr, instr_create_save_to_dc_via_reg
+            (dcontext, REG_XBX, REG_EAX, ERRNO_OFFSET));
+    } else {
+        PRE(ilist, instr,
+            instr_create_save_to_dcontext(dcontext, REG_EAX, ERRNO_OFFSET));
+    }
+#endif
+
+    /* now restore everything */
+    insert_pop_all_registers(dcontext, ilist, instr,
+                             /* see notes in prepare_for_clean_call() */
+                             IF_X64_ELSE(true, false));
+    PRE(ilist, instr, INSTR_CREATE_popf(dcontext));
+
+    /* Swap stacks back.  For thread-shared, we need to get the dcontext
+     * dynamically.  Save xax in TLS so we can use it as scratch.
+     */
+    if (SHARED_FRAGMENTS_ENABLED()) {
+        PRE(ilist, instr, instr_create_save_to_tls
+            (dcontext, REG_XAX, TLS_XAX_SLOT));
+
+        insert_get_mcontext_base(dcontext, ilist, instr,
+                                 REG_XAX);
+
+        PRE(ilist, instr, instr_create_restore_from_dc_via_reg
+            (dcontext, REG_XAX, REG_XSP, XSP_OFFSET));
+
+        PRE(ilist, instr, instr_create_restore_from_tls
+            (dcontext, REG_XAX, TLS_XAX_SLOT));
+    }
+    else {
+        PRE(ilist, instr,
+            instr_create_restore_from_dcontext(dcontext, REG_XSP, XSP_OFFSET));
+    }
+}
+
+bool
+parameters_stack_padded(void)
+{
+    return (REGPARM_MINSTACK > 0 || REGPARM_END_ALIGN > XSP_SZ);
+}
+
+static reg_id_t
+shrink_reg_for_param(reg_id_t regular, opnd_t arg)
+{
+#ifdef X64
+    if (opnd_get_size(arg) == OPSZ_4) { /* we ignore var-sized */
+        /* PR 250976 #2: leave 64-bit only if an immed w/ top bit set (we
+         * assume user wants sign-extension; that is after all what happens
+         * on a push of a 32-bit immed) */
+        if (!opnd_is_immed_int(arg) ||
+            (opnd_get_immed_int(arg) & 0x80000000) == 0)
+            return reg_64_to_32(regular);
+    }
+#endif
+    return regular;
+}
+
+
+/* Returns the change in the stack pointer.
+ * N.B.: due to stack alignment and minimum stack reservation, do
+ * not use parameters involving esp/rsp, as its value can change!
+ *
+ * This routine only supports passing arguments that are integers or
+ * pointers of a size equal or smaller than the register size: i.e., no
+ * floating-point, multimedia, or aggregate data types.
+ *
+ * For 64-bit mode, if a 32-bit immediate integer is specified as an
+ * argument and it has its top bit set, we assume it is intended to be
+ * sign-extended to 64-bits; otherwise we zero-extend it.
+ *
+ * For 64-bit mode, variable-sized argument operands may not work
+ * properly.
+ *
+ * Arguments that reference REG_XSP will work for clean calls, but are not guaranteed
+ * to work for non-clean, especially for 64-bit where we align, etc.  Arguments that
+ * reference sub-register portions of REG_XSP are not supported.
+ *
+ * FIXME PR 307874: w/ a post optimization pass, or perhaps more clever use of
+ * existing passes, we could do much better on calling convention and xsp conflicting
+ * args.  We should also really consider inlining client callees (PR 218907), since
+ * clean calls for 64-bit are enormous (71 instrs/264 bytes for 2-arg x64; 26
+ * instrs/99 bytes for x86) and we could avoid all the xmm saves and replace pushf w/
+ * lahf.
+ */
+static uint
+insert_parameter_preparation(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+                             bool clean_call, uint num_args, va_list ap)
+{
+    uint i;
+    int r;
+    uint preparm_padding = 0;
+    uint param_stack = 0, total_stack = 0;
+    bool push = true;
+    bool restore_xax = false;
+    bool restore_xsp = false;
+    /* we need two passes for PR 250976 optimization */
+    opnd_t *args = NULL;
+    /* Push/mov in reverse order.  We need a label so we can also add
+     * instrs prior to the regular param prep.  So params are POST-mark, while
+     * pre-param-prep is POST-prev or PRE-mark.
+     */
+#ifdef X64
+    uint arg_pre_push = 0, total_pre_push = 0;
+#endif
+    instr_t *prev = (instr == NULL) ? instrlist_last(ilist) : instr_get_prev(instr);
+    instr_t *mark = INSTR_CREATE_label(dcontext);
+    PRE(ilist, instr, mark);
+
+    if (num_args > 0)
+        args = HEAP_ARRAY_ALLOC(dcontext, opnd_t, num_args, ACCT_OTHER, PROTECTED);
+
+    /* For a clean call, xax is dead (clobbered by prepare_for_clean_call()).
+     * Rather than use as scratch and restore prior to each param that uses it,
+     * we restore once up front if any use it, and use regparms[0] as scratch,
+     * which is symmetric with non-clean-calls: regparms[0] is dead since we're
+     * doing args in reverse order.  However, we then can't use regparms[0]
+     * directly if referenced in earlier params, but similarly for xax, so
+     * there's no clear better way. (prepare_for_clean_call also clobbers xsp,
+     * but we just disallow args that use it).
+     */
+
+    /* We can get away w/ one pass, except for PR 250976 we want calling conv
+     * regs to be able to refer to dr_mcontext_t as well as potentially being
+     * pushed: but we need to know the total # pushes ahead of time (since hard
+     * to mark for post-patching)
+     */
+    for (i = 0; i < num_args; i++) {
+        IF_X64(bool is_pre_push = false;)
+        /* There's no way to check num_args vs actual args passed in */
+        args[i] = va_arg(ap, opnd_t);
+        CLIENT_ASSERT(opnd_is_valid(args[i]),
+                      "Call argument: bad operand. Did you create a valid opnd_t?");
+        for (r = 0; r < opnd_num_regs_used(args[i]); r++) {
+            reg_id_t used = opnd_get_reg_used(args[i], r);
+            IF_X64(int parm;)
+            LOG(THREAD, LOG_INTERP, 4,
+                "ipp: considering arg %d reg %d == %s\n", i, r, reg_names[used]);
+            if (clean_call && !restore_xax && reg_overlap(used, REG_XAX))
+                restore_xax = true;
+            if (reg_overlap(used, REG_XSP)) {
+                IF_X64(CLIENT_ASSERT(clean_call,
+                                     "Non-clean-call argument: REG_XSP not supported"));
+                CLIENT_ASSERT(used == REG_XSP,
+                              "Call argument: sub-reg-xsp not supported");
+                if (clean_call && /*x64*/parameters_stack_padded() && !restore_xsp)
+                    restore_xsp = true;
+            }
+#ifdef X64
+            /* PR 250976 #A: count the number of pre-pushes we need */
+            parm = reg_parameter_num(used);
+            /* We can read a register used in an earlier arg since we store that
+             * arg later (we do reverse order), except arg0, which we use as
+             * scratch (we don't always need it, but not worth another pre-pass
+             * through all args to find out), and xsp.  Otherwise, if a plain reg,
+             * we point at mcontext (we restore xsp slot in mcontext if nec.).
+             * If a mem ref, we need to pre-push onto stack.
+             * N.B.: this conditional is duplicated in 2nd loop.
+             */
+            if (!is_pre_push &&
+                ((parm == 0 && num_args > 1) || parm > (int)i ||
+                 reg_overlap(used, REG_XSP)) &&
+                (!clean_call || !opnd_is_reg(args[i]))) {
+                total_pre_push++;
+                is_pre_push = true; /* ignore further regs in same arg */
+            }
+#endif
+        }
+    }
+
+    if (parameters_stack_padded()) {
+        /* For x64, supposed to reserve rsp space in function prologue; we
+         * do next best thing and reserve it prior to setting up the args.
+         */
+        push = false; /* store args to xsp offsets instead of pushing them */
+        total_stack = REGPARM_MINSTACK;
+        if (num_args > NUM_REGPARM)
+            total_stack += XSP_SZ * (num_args - NUM_REGPARM);
+        param_stack = total_stack;
+        IF_X64(total_stack += XSP_SZ * total_pre_push);
+        /* We assume rsp is currently 16-byte aligned.  End of arguments is supposed
+         * to be 16-byte aligned for x64 SysV (note that retaddr will then make
+         * rsp 8-byte-aligned, which is ok: callee has to rectify that).
+         * For clean calls, prepare_for_clean_call leaves rsp aligned for x64.
+         * FIXME PR 218790: we require users of dr_insert_call to ensure
+         * alignment; should we put in support to dynamically align?
+         */
+        preparm_padding =
+            ALIGN_FORWARD_UINT(total_stack, REGPARM_END_ALIGN) - total_stack;
+        total_stack += preparm_padding;
+        /* we have to wait to insert the xsp adjust */
+    } else {
+        ASSERT(NUM_REGPARM == 0);
+        ASSERT(push);
+        IF_X64(ASSERT(total_pre_push == 0));
+        total_stack = XSP_SZ * num_args;
+    }
+    LOG(THREAD, LOG_INTERP, 3,
+        "insert_parameter_preparation: %d args, %d in-reg, %d pre-push, %d/%d stack\n",
+        num_args, NUM_REGPARM, IF_X64_ELSE(total_pre_push, 0), param_stack, total_stack);
+
+    for (i = 0; i < num_args; i++) {
+        /* FIXME PR 302951: we need to handle state restoration if any
+         * of these args references app memory.  We should pull the state from
+         * the dr_mcontext_t on the stack if in a clean call.  FIXME: what if not?
+         */
+        opnd_t arg = args[i];
+        CLIENT_ASSERT(opnd_get_size(arg) == OPSZ_PTR || opnd_is_immed_int(arg)
+                      IF_X64(|| opnd_get_size(arg) == OPSZ_4),
+                      "Clean call arg has unsupported size");
+
+#ifdef X64
+        /* PR 250976 #A: support args that reference param regs */
+        for (r = 0; r < opnd_num_regs_used(arg); r++) {
+            reg_id_t used = opnd_get_reg_used(arg, r);
+            int parm = reg_parameter_num(used);
+            /* See comments in loop above */ 
+            if ((parm == 0 && num_args > 1) || parm > (int)i ||
+                 reg_overlap(used, REG_XSP)) {
+                int disp = 0;
+                if (clean_call && opnd_is_reg(arg)) {
+                    /* We can point at the dr_mcontext_t slot.
+                     * dr_mcontext_t is at the base of dstack: compute offset
+                     * from xsp to the field we want and replace arg.
+                     */
+                    disp += opnd_get_reg_dcontext_offs(opnd_get_reg(arg));
+                    /* skip rest of what prepare_for_clean_call adds */
+                    disp += clean_call_beyond_mcontext();
+                    /* skip what this routine added */
+                    disp += total_stack;
+                } else {
+                    /* Push a temp on the stack and point at it.  We
+                     * could try to optimize by juggling registers, but
+                     * not worth it.
+                     */
+                    /* xsp was adjusted up above; we simply store to xsp offsets */
+                    disp = param_stack + XSP_SZ * arg_pre_push;
+                    if (opnd_is_reg(arg) && opnd_get_size(arg) == OPSZ_PTR) {
+                        POST(ilist, prev, INSTR_CREATE_mov_st
+                             (dcontext, OPND_CREATE_MEMPTR(REG_XSP, disp), arg));
+                    } else {
+                        reg_id_t xsp_scratch = regparms[0];
+                        /* don't want to just change size since will read extra bytes.
+                         * can't do mem-to-mem so go through scratch reg */
+                        if (reg_overlap(used, REG_XSP)) {
+                            /* Get original xsp into scratch[0] and replace in arg */
+                            if (opnd_uses_reg(arg, regparms[0])) {
+                                xsp_scratch = REG_XAX;
+                                ASSERT(!opnd_uses_reg(arg, REG_XAX)); /* can't use 3 */
+                                /* FIXME: rather than putting xsp into mcontext
+                                 * slot, better to just do local get from dcontext
+                                 * like we do for 32-bit below? */
+                                POST(ilist, prev, instr_create_restore_from_tls
+                                     (dcontext, REG_XAX, TLS_XAX_SLOT));
+                            }
+                            opnd_replace_reg(&arg, REG_XSP, xsp_scratch);
+                        }
+                        POST(ilist, prev,
+                             INSTR_CREATE_mov_st(dcontext,
+                                                 OPND_CREATE_MEMPTR(REG_XSP, disp),
+                                                 opnd_create_reg(regparms[0])));
+                        /* If sub-ptr-size, zero-extend is what we want so no movsxd */
+                        POST(ilist, prev, INSTR_CREATE_mov_ld
+                             (dcontext, opnd_create_reg
+                              (shrink_reg_for_param(regparms[0], arg)), arg));
+                        if (reg_overlap(used, REG_XSP)) {
+                            int xsp_disp = opnd_get_reg_dcontext_offs(REG_XSP) +
+                                clean_call_beyond_mcontext() + total_stack;
+                            POST(ilist, prev, INSTR_CREATE_mov_ld
+                                 (dcontext, opnd_create_reg(xsp_scratch),
+                                  OPND_CREATE_MEMPTR(REG_XSP, xsp_disp)));
+                            if (xsp_scratch == REG_XAX) {
+                                POST(ilist, prev, instr_create_save_to_tls
+                                     (dcontext, REG_XAX, TLS_XAX_SLOT));
+                            }
+                        }
+                        if (opnd_uses_reg(arg, regparms[0])) {
+                            /* must restore since earlier arg might have clobbered */
+                            int mc_disp = opnd_get_reg_dcontext_offs(regparms[0]) +
+                                clean_call_beyond_mcontext() + total_stack;
+                            POST(ilist, prev, INSTR_CREATE_mov_ld
+                                 (dcontext, opnd_create_reg(regparms[0]),
+                                  OPND_CREATE_MEMPTR(REG_XSP, mc_disp)));
+                        }
+                    }
+                    arg_pre_push++; /* running counter */
+                }
+                arg = opnd_create_base_disp(REG_XSP, REG_NULL, 0,
+                                            disp, opnd_get_size(arg));
+                break; /* once we've handled arg ignore futher reg refs */
+            }
+        }
+#endif
+
+        if (i < NUM_REGPARM) {
+            reg_id_t regparm = shrink_reg_for_param(regparms[i], arg);
+            if (opnd_is_immed_int(arg)) {
+                POST(ilist, mark,
+                    INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(regparm), arg));
+            } else {
+                POST(ilist, mark,
+                    INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(regparm), arg));
+            }
+        } else {
+            if (push) {
+                IF_X64(ASSERT_NOT_REACHED()); /* no 64-bit push_imm! */
+                if (opnd_is_immed_int(arg))
+                    POST(ilist, mark, INSTR_CREATE_push_imm(dcontext, arg));
+                else {
+                    if (clean_call && opnd_uses_reg(arg, REG_XSP)) {
+                        /* We do a purely local expansion:
+                         * spill eax, mc->eax, esp->eax, arg->eax, push eax, restore eax
+                         */
+                        reg_id_t scratch = REG_XAX;
+                        if (opnd_uses_reg(arg, scratch)) {
+                            scratch = REG_XCX;
+                            ASSERT(!opnd_uses_reg(arg, scratch)); /* can't use 3 regs */
+                        }
+                        opnd_replace_reg(&arg, REG_XSP, scratch);
+                        POST(ilist, mark, instr_create_restore_from_tls
+                             (dcontext, scratch, TLS_XAX_SLOT));
+                        POST(ilist, mark, INSTR_CREATE_push(dcontext, arg));
+                        POST(ilist, mark, instr_create_restore_from_dc_via_reg
+                            (dcontext, scratch, scratch, XSP_OFFSET));
+                        insert_get_mcontext_base
+                            (dcontext, ilist, instr_get_next(mark), scratch);
+                        POST(ilist, mark, instr_create_save_to_tls
+                             (dcontext, scratch, TLS_XAX_SLOT));
+                    } else
+                        POST(ilist, mark, INSTR_CREATE_push(dcontext, arg));
+                }
+            } else {
+                /* xsp was adjusted up above; we simply store to xsp offsets */
+                uint offs = REGPARM_MINSTACK + XSP_SZ * (i - NUM_REGPARM);
+#ifdef X64
+                if (opnd_is_immed_int(arg)) {
+                    /* PR 250976 #3: there is no memory store of 64-bit-immediate,
+                     * so go through scratch reg */
+                    ASSERT(NUM_REGPARM > 0);
+                    POST(ilist, mark,
+                         INSTR_CREATE_mov_st(dcontext,
+                                             OPND_CREATE_MEMPTR(REG_XSP, offs),
+                                             opnd_create_reg(regparms[0])));
+                    POST(ilist, mark,
+                         INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(regparms[0]),
+                                              arg));
+                } else {
+#endif
+                    if (opnd_is_memory_reference(arg)) {
+                        /* can't do mem-to-mem so go through scratch */
+                        ASSERT(NUM_REGPARM > 0);
+                        POST(ilist, mark,
+                             INSTR_CREATE_mov_st(dcontext,
+                                                 OPND_CREATE_MEMPTR(REG_XSP, offs),
+                                                 opnd_create_reg(regparms[0])));
+                        POST(ilist, mark,
+                             INSTR_CREATE_mov_ld(dcontext, opnd_create_reg
+                                                 (shrink_reg_for_param(regparms[0], arg)),
+                                                 arg));
+                    } else {
+                        POST(ilist, mark,
+                             INSTR_CREATE_mov_st(dcontext,
+                                                 OPND_CREATE_MEMPTR(REG_XSP, offs), arg));
+                    }
+#ifdef X64
+                }
+#endif
+            }
+        }
+    }
+    if (!push && total_stack > 0) {
+        POST(ilist, prev, /* before everything else: pre-push and args */
+            /* can we use sub?  may as well preserve eflags */
+            INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XSP),
+                             OPND_CREATE_MEM_lea(REG_XSP, REG_NULL, 0,
+                                                 -(int)total_stack)));
+    }
+    if (restore_xsp) {
+        /* before restore_xax, since we're going to clobber xax */
+        int disp = opnd_get_reg_dcontext_offs(REG_XSP);
+        instr_t *where = instr_get_next(prev);
+        /* skip rest of what prepare_for_clean_call adds */
+        disp += clean_call_beyond_mcontext();
+        insert_get_mcontext_base(dcontext, ilist, where, REG_XAX);
+        PRE(ilist, where, instr_create_restore_from_dc_via_reg
+            (dcontext, REG_XAX, REG_XAX, XSP_OFFSET));
+        PRE(ilist, where,
+            INSTR_CREATE_mov_st(dcontext, OPND_CREATE_MEMPTR(REG_XSP, disp),
+                                opnd_create_reg(REG_XAX)));
+        /* now we need restore_xax to be AFTER this */
+        prev = instr_get_prev(where);
+    }
+    if (restore_xax) {
+        int disp = opnd_get_reg_dcontext_offs(REG_XAX);
+        /* skip rest of what prepare_for_clean_call adds */
+        disp += clean_call_beyond_mcontext();
+        POST(ilist, prev, /* before everything else: pre-push, args, and stack adjust */
+             INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XAX),
+                                 OPND_CREATE_MEMPTR(REG_XSP, disp)));
+    }
+    if (num_args > 0)
+        HEAP_ARRAY_FREE(dcontext, args, opnd_t, num_args, ACCT_OTHER, PROTECTED);
+    return total_stack;
+}
+
+/* Inserts a complete call to callee with the passed-in arguments.
+ * For x64, assumes the stack pointer is currently 16-byte aligned.
+ * Clean calls ensure this by using clean base of dstack and having
+ * dr_prepare_for_call pad to 16 bytes.
+ */
+void
+insert_meta_call_vargs(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+                       bool clean_call, void *callee, uint num_args, va_list ap)
+{
+    instr_t *in = (instr == NULL) ? instrlist_last(ilist) : instr_get_prev(instr);
+    uint stack_for_params = 
+        insert_parameter_preparation(dcontext, ilist, instr, clean_call, num_args, ap);
+    IF_X64(ASSERT(ALIGNED(stack_for_params, 16)));
+    PRE(ilist, instr, INSTR_CREATE_call(dcontext, opnd_create_pc(callee)));
+    if (stack_for_params > 0) {
+        /* FIXME PR 245936: let user decide whether to clean up?
+         * i.e., support calling a stdcall routine?
+         */
+        PRE(ilist, instr,
+            INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XSP),
+                             opnd_create_base_disp(REG_XSP, REG_NULL, 0,
+                                                   stack_for_params, OPSZ_lea)));
+    }
+    /* mark it all meta */
+    if (in == NULL)
+        in = instrlist_first(ilist);
+    else
+        in = instr_get_next(in);
+    while (in != instr) {
+        instr_set_ok_to_mangle(in, false);
+        in = instr_get_next(in);
+    }
+}
+
+/* If jmp_instr == NULL, uses jmp_tag, otherwise uses jmp_instr
+ */
+void
+insert_clean_call_with_arg_jmp_if_ret_true(dcontext_t *dcontext,
+            instrlist_t *ilist, instr_t *instr,  void *callee, int arg,
+                                           app_pc jmp_tag, instr_t *jmp_instr)
+{
+    instr_t *false_popa, *jcc;
+    prepare_for_clean_call(dcontext, ilist, instr);
+
+    dr_insert_call(dcontext, ilist, instr, callee, 1, OPND_CREATE_INT32(arg));
+
+    /* if the return value (xax) is 0, then jmp to internal false path */
+    PRE(ilist,instr, /* can't cmp w/ 64-bit immed so use test (shorter anyway) */
+        INSTR_CREATE_test(dcontext, opnd_create_reg(REG_XAX), opnd_create_reg(REG_XAX)));
+    /* fill in jcc target once have false path */
+    jcc = INSTR_CREATE_jcc(dcontext, OP_jz, opnd_create_pc(NULL));
+    PRE(ilist, instr, jcc);
+    
+    /* if it falls through, then it's true, so restore and jmp to true tag
+     * passed in by caller
+     */
+    cleanup_after_clean_call(dcontext, ilist, instr);
+    if (jmp_instr == NULL) {
+        /* an exit cti, not a meta instr */
+        instrlist_preinsert
+            (ilist, instr, INSTR_CREATE_jmp(dcontext, opnd_create_pc(jmp_tag)));
+    } else {
+        PRE(ilist, instr,
+            INSTR_CREATE_jmp(dcontext, opnd_create_instr(jmp_instr)));
+    }
+
+    /* otherwise (if returned false), just do standard popf and continue */
+    /* get 1st instr of cleanup path */
+    false_popa = instr_get_prev(instr);
+    cleanup_after_clean_call(dcontext, ilist, instr);
+    false_popa = instr_get_next(false_popa);
+    instr_set_target(jcc, opnd_create_instr(false_popa));
+}
+
+/*###########################################################################
+ *###########################################################################
+ *
+ *   M A N G L I N G   R O U T I N E S
+ */
+
+/*###########################################################################*/
+#ifdef NATIVE_RETURN
+
+/* ENORMOUS HACK: written by mangle_direct_call, read by
+ * native_ret_mangle_return
+ */
+static app_pc static_retaddr;
+
+static instr_t *
+native_ret_mangle_direct_call(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+                              instr_t *next_instr, uint retaddr)
+{
+# ifdef NATIVE_RETURN_CALLDEPTH
+    uint flags = 0;
+# endif
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    /* ENORMOUS HACK -- READ BY MANGLE_RETURN */
+    static_retaddr = (app_pc) retaddr;
+
+    /* HACKS to deal with places where ret addr is taken inside a real callee:
+         0x40170199   e8 32 fa ff ff       call   $0x4016fbd0 
+             continuing in callee at 0x4016fbd0
+         0x4016fbd0   8b 1c 24             mov    (%esp) -> %ebx 
+         0x4016fbd3   c3                   ret    %esp (%esp) -> %esp 
+     * also this:
+         0x400a9bab   e8 00 00 00 00       call   $0x400a9bb0 
+             continuing in callee at 0x400a9bb0
+         0x400a9bb0   8d 04 c0             lea    (%eax,%eax,8) -> %eax 
+         0x400a9bb3   03 04 24             add    (%esp) %eax -> %eax 
+         0x400a9bb6   05 0d 00 00 00       add    $0x0000000d %eax -> %eax 
+         0x400a9bbb   83 c4 04             add    $0x04 %esp -> %esp 
+         0x400a9bbe   ff e0                jmp    %eax 
+     */
+    if (instr_raw_bits_valid(next_instr) &&
+        instr_length(dcontext, next_instr) == 3) {
+        byte *b = instr_get_raw_bits(next_instr);
+        if (*b==0x8b && *(b+1)==0x1c && *(b+2)==0x24) {
+            LOG(THREAD, LOG_INTERP, 3, "mangling load of return address!\n");
+            /* we can't delete next_instr (in local var in mangle()) */
+            /* cannot call instr_reset, it will kill prev & next ptrs */
+            instr_free(dcontext, next_instr);
+            instr_set_opcode(next_instr, OP_mov_imm);
+            instr_set_num_opnds(dcontext, next_instr, 1, 1);
+            instr_set_dst(next_instr, 0, opnd_create_reg(REG_EBX));
+            instr_set_src(next_instr, 0, OPND_CREATE_INT32(retaddr));
+        }
+    } else if (instr_raw_bits_valid(next_instr) &&
+               instr_length(dcontext, next_instr) == 14) {
+        byte *b = instr_get_raw_bits(next_instr);
+        if (*b==0x8d && *(b+1)==0x04 && *(b+2)==0xc0 &&
+            *(b+3)==0x03 && *(b+4)==0x04 && *(b+5)==0x24) {
+            LOG(THREAD, LOG_INTERP, 3, "mangling load of return address!\n");
+            instrlist_preinsert(ilist, next_instr,
+                                instr_create_raw_3bytes(dcontext, 0x8d, 0x04, 0xc0));
+            instrlist_preinsert(ilist, next_instr,
+                                INSTR_CREATE_add(dcontext, opnd_create_reg(REG_EAX),
+                                                 OPND_CREATE_INT32(retaddr)));
+            instr_set_raw_bits(next_instr, b+6, 8);
+            next_instr = instr_get_next(instr);
+        }
+    }
+
+    /*********************************************************/
+    /* NEW CALL HANDLING: NATIVE RETURN
+          save flags
+          inc call_depth
+          restore flags
+          call skip
+FIXME: coordinate this w/ ret site:       restore flags
+          jmp app_ret_addr
+        skip:
+          continue in callee
+     */
+# ifdef NATIVE_RETURN_CALLDEPTH
+    if (!TEST(FRAG_WRITES_EFLAGS_6, flags)) {
+        /* save app's eax */
+        instrlist_preinsert(ilist, instr,
+                         instr_create_save_to_dcontext(dcontext, REG_EAX, XAX_OFFSET));
+        /* save flags */
+        instrlist_preinsert(ilist, instr, INSTR_CREATE_lahf(dcontext));
+    }
+    if (!TEST(FRAG_WRITES_EFLAGS_OF, flags)) {
+        /* must have saved eax */
+        ASSERT(!TEST(FRAG_WRITES_EFLAGS_6, flags));
+        /* seto al */
+        instrlist_preinsert(ilist, instr,
+                INSTR_CREATE_setcc(dcontext, OP_seto, opnd_create_reg(REG_AL)));
+    }
+
+    /* inc call_depth */
+    instrlist_preinsert(ilist, instr,
+                        INSTR_CREATE_inc(dcontext, opnd_create_dcontext_field(dcontext,
+                                                                              CALL_DEPTH_OFFSET)));
+
+    if ((flags & FRAG_WRITES_EFLAGS_OF) == 0) {
+        /* now do an add such that OF will be set only if seto set al to 1 */
+        instrlist_preinsert(ilist, instr,
+         INSTR_CREATE_add(dcontext,
+                          opnd_create_reg(REG_AL), OPND_CREATE_INT8(0x7f)));
+    }
+    if ((flags & FRAG_WRITES_EFLAGS_6) == 0) {
+        instrlist_preinsert(ilist, instr, INSTR_CREATE_sahf(dcontext));
+        instrlist_preinsert(ilist, instr,
+                 instr_create_restore_from_dcontext(dcontext, REG_EAX, XAX_OFFSET));
+    }
+# endif
+
+    /* tell call to target next instr */
+    instr_set_target(instr, opnd_create_instr(next_instr));
+
+# ifdef NATIVE_RETURN_CALLDEPTH
+    /* FIXME: these flags should be based on retaddr */
+    if ((flags & FRAG_WRITES_EFLAGS_OF) == 0) {
+        /* now do an add such that OF will be set only if seto set al to 1 */
+        instrlist_preinsert(ilist, instr,
+         INSTR_CREATE_add(dcontext,
+                          opnd_create_reg(REG_AL), OPND_CREATE_INT8(0x7f)));
+    }
+    if ((flags & FRAG_WRITES_EFLAGS_6) == 0) {
+        instrlist_preinsert(ilist, next_instr, INSTR_CREATE_sahf(dcontext));
+        instrlist_preinsert(ilist, next_instr,
+                 instr_create_restore_from_dcontext(dcontext, REG_EAX, XAX_OFFSET));
+    }
+# endif
+
+    instrlist_preinsert(ilist, next_instr, INSTR_CREATE_jmp(dcontext,
+                                                opnd_create_pc((app_pc)retaddr)));
+    return next_instr;
+}
+
+static void
+native_ret_mangle_return(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+                         instr_t *next_instr)
+{
+    bool code_cache_ret = true;
+    /*
+        cmp call_depth, 0
+        je normal_ret
+        dec call_depth
+FIXME: currently restoring flags at call's return site too
+plus need to move the flag saving up above here, now it's after
+save edx
+        restore eflags
+        ret
+     normal_ret:
+        save edx
+        pop edx
+        <add ret_imm, esp>
+        jmp ind_br_lookup (via stub)
+    */
+    uint flags = (FRAG_WRITES_EFLAGS_OF | FRAG_WRITES_EFLAGS_6);
+    instr_t *addinstr = NULL;
+    
+    /* do normal_ret first */
+    instr_t *nxt = instr_get_next(instr); /* next_instr is after flag saving! */
+
+    /* save away ecx so that we can use it */
+    /* (it's restored in x86.s (indirect_branch_lookup) */
+    instr_t *save_ecx = instr_create_save_to_dcontext(dcontext, REG_ECX, XCX_OFFSET);
+    instrlist_preinsert(ilist, nxt, save_ecx);
+
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+
+    /* see if ret has an immed int operand, assumed to be 1st src */
+            
+    if (instr_num_srcs(instr) > 0 && opnd_is_immed_int(instr_get_src(instr, 0))) {
+        /* if has an operand, return removes some stack space,
+         * AFTER the return address is popped
+         */
+        ptr_int_t val = opnd_get_immed_int(instr_get_src(instr, 0));
+        opnd_t add;
+        if (val >= -128 && val <= 127)
+            add = opnd_create_immed_int(val, OPSZ_1);
+        else
+            add = opnd_create_immed_int(val, OPSZ_4);
+        /* addl sizeof_param_area, %esp */
+        /* insert this add AFTER the flags have been saved! */
+         if (!INTERNAL_OPTION(unsafe_ignore_eflags)) {
+             instrlist_preinsert(ilist, next_instr,
+                                 INSTR_CREATE_add(dcontext,
+                                                  opnd_create_reg(REG_ESP), add));
+         }
+         else {
+             addinstr = INSTR_CREATE_add(dcontext, opnd_create_reg(REG_ESP), add);
+         }
+    }
+            
+    if (instr_raw_bits_valid(instr)) {
+        app_pc pc = (app_pc) instr_get_raw_bits(instr);
+        LOG(THREAD, LOG_INTERP, 3, "checking ret at address "PFX"\n", pc);
+        if (is_dynamo_address(pc)) {
+            LOG(THREAD, LOG_INTERP, 3, "found a ret at dynamo address "PFX"\n", pc);
+        }
+        /* 0x4000d090 on cagfarm* and atari, 0x4000d080 on kobold */
+        if (pc == (app_pc) 0x4000d090 || pc == (app_pc) 0x4000d080 ||
+            (is_dynamo_address(pc) &&
+# ifdef DEBUG
+             ((automatic_startup && GLOBAL_STAT(num_fragments) < 32) ||
+              (!automatic_startup && GLOBAL_STAT(num_fragments) < 5))
+# else
+             /* could use GLOBAL_STAT but only if option is on */
+             ((automatic_startup && num_fragments < 32) ||
+              (!automatic_startup && num_fragments < 5))
+# endif
+             )) {
+            code_cache_ret = false;
+        }
+    }
+
+    /* change RET into a POP */
+    instrlist_preinsert(ilist, nxt, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_ECX)));
+
+    if (INTERNAL_OPTION(unsafe_ignore_eflags) && addinstr != NULL)
+        instrlist_preinsert(ilist, nxt, addinstr);
+    /* now do first part, before normal_ret */
+    if (!code_cache_ret) { /*dcontext->call_depth == 0) {*/
+        /* ASSUMPTION: no calls in this basic block preceding the ret,
+         * no other places in program where return below dynamorio start stack
+         * frame
+         */
+        LOG(THREAD, LOG_INTERP, 3, "found a non-code-cache ret\n");
+        /* remove the ret */
+        instrlist_remove(ilist, instr);
+        instr_destroy(dcontext, instr);
+    } else {
+        instr_t *prev;
+
+# ifdef NATIVE_RETURN_CALLDEPTH
+        /* cmp call_depth, 0 */
+        instrlist_preinsert(ilist, instr,
+                            INSTR_CREATE_cmp(dcontext,
+                                             opnd_create_dcontext_field(dcontext, CALL_DEPTH_OFFSET),
+                                             OPND_CREATE_INT32(0)));
+        /* je normal_ret */
+        instrlist_preinsert(ilist, instr, INSTR_CREATE_jcc(dcontext, OP_je,
+                                                           opnd_create_instr(save_ecx)));
+        /* dec call_depth */
+        instrlist_preinsert(ilist, instr,
+                            INSTR_CREATE_dec(dcontext, opnd_create_dcontext_field(dcontext,
+                                                                                  CALL_DEPTH_OFFSET)));
+
+        if ((flags & FRAG_WRITES_EFLAGS_OF) == 0) {
+            /* now do an add such that OF will be set only if seto set al to 1 */
+            instrlist_preinsert(ilist, instr,
+                                INSTR_CREATE_add(dcontext,
+                                                 opnd_create_reg(REG_AL),
+                                                 OPND_CREATE_INT8(0x7f)));
+        }
+        if ((flags & FRAG_WRITES_EFLAGS_6) == 0) {
+            instrlist_preinsert(ilist, instr, INSTR_CREATE_sahf(dcontext));
+            instrlist_preinsert(ilist, instr,
+                                instr_create_restore_from_dcontext(dcontext, REG_EAX, XAX_OFFSET));
+        }
+# endif
+
+        /* leave ret instr where it is */
+
+        /* HACKS to deal with places where ret addr is taken inside a real callee:
+         0x400864b6   8b 4c 24 00          mov    (%esp) -> %ecx 
+         0x400864ba   89 4a 14             mov    %ecx -> 0x14(%edx) 
+         0x400864bd   89 6a 0c             mov    %ebp -> 0xc(%edx) 
+         0x400864c0   89 42 18             mov    %eax -> 0x18(%edx) 
+         0x400864c3   c3                   ret    %esp (%esp) -> %esp 
+         */
+        prev = instr_get_prev(instr);
+        if (prev != NULL && instr_raw_bits_valid(prev) &&
+            instr_length(dcontext, prev) > 4) {
+            uint len = instr_length(dcontext, prev);
+            byte *b = instr_get_raw_bits(prev) + len - 1;
+            if (len > 13 &&
+                       *b==0x18 && *(b-1)==0x42 && *(b-2)==0x89 &&
+                       *(b-3)==0x0c && *(b-4)==0x6a && *(b-5)==0x89 && 
+                       *(b-6)==0x14 && *(b-7)==0x4a && *(b-8)==0x89 && 
+                       *(b-9)==0x00 && *(b-10)==0x24 && *(b-11)==0x4c && *(b-12)==0x8b) {
+                instr_t *raw;
+                LOG(THREAD, LOG_INTERP, 3, "mangling load of return address!\n");
+                instr_set_raw_bits(prev, instr_get_raw_bits(prev), len-13);
+                /* PROBLEM: don't know app retaddr!
+                 * ENORMOUS HACK: assume no calls or threads in between, 
+                 * use statically stored one from last call
+                 * Also assumes only a single caller
+                 */
+                IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+                instrlist_preinsert(ilist, instr,
+                                    INSTR_CREATE_mov_imm(dcontext,
+                                                         opnd_create_reg(REG_ECX),
+                                                         OPND_CREATE_INT32((uint)static_retaddr)));
+                raw = instr_build_bits(dcontext, OP_UNDECODED, 9);
+                instr_set_raw_bytes(raw, b-8, 9);               
+                instrlist_preinsert(ilist, instr, raw);
+            }
+        }
+    }
+}
+#endif /* NATIVE_RETURN */
+
+/*###########################################################################*/
+#ifdef RETURN_STACK
+static instr_t *
+return_stack_mangle_direct_call(dcontext_t *dcontext, instrlist_t *ilist,
+                                instr_t *instr, instr_t *next_instr, uint retaddr)
+{
+    instr_t *cleanup;
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    /*********************************************************/
+    /* NEW CALL HANDLING: RETURN STACK! */
+    /* (optional: build basic block for after call
+     * Then change call to this:
+     *   push app ret addr
+     *   swap to return stack
+     *   push app ret addr
+     *   call cleanup_stack
+     *   jmp after_call_fragment
+     * cleanup_stack:
+     *   swap to app stack
+     */
+    cleanup = instr_create_save_dynamo_return_stack(dcontext);
+    PRE(ilist, instr,
+        INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32(retaddr)));
+    PRE(ilist, instr,
+        instr_create_save_to_dcontext(dcontext, REG_ESP, XSP_OFFSET));
+    PRE(ilist, instr,
+        instr_create_restore_dynamo_return_stack(dcontext));
+    PRE(ilist, instr,
+        INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32(retaddr)));
+    instr_set_target(instr, opnd_create_instr(cleanup));
+    /* an exit cti, not a meta instr */
+    instrlist_preinsert
+        (ilist, next_instr, INSTR_CREATE_jmp(dcontext,
+                                             opnd_create_pc(retaddr)));
+    PRE(ilist, next_instr, cleanup);
+    PRE(ilist, next_instr,
+        instr_create_restore_from_dcontext(dcontext, REG_ESP, XSP_OFFSET));
+    return next_instr;
+}
+#endif /* RETURN_STACK */
+
+void
+insert_push_immed_ptrsz(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+                        ptr_int_t val)
+{
+#ifdef X64
+    /* do push-64-bit-immed in two pieces.  tiny corner-case risk of racy
+     * access to TOS if this thread is suspended in between or another
+     * thread is trying to read its stack, but o/w we have to spill and
+     * restore a register. */
+    PRE(ilist, instr,
+        INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32((int)val)));
+    /* push is sign-extended, so we can skip top half if nothing in top 33 bits */
+    if (val >= 0x80000000) {
+        PRE(ilist, instr,
+            INSTR_CREATE_mov_st(dcontext, OPND_CREATE_MEM32(REG_XSP, 4),
+                                OPND_CREATE_INT32((int)(val >> 32))));
+    }
+#else
+    PRE(ilist, instr,
+        INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32(val)));
+#endif
+}
+
+/* N.B.: keep in synch with instr_check_xsp_mangling() in arch.c */
+static void
+insert_push_retaddr(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+                    ptr_int_t retaddr, opnd_size_t opsize)
+{
+    if (opsize == OPSZ_2) {
+        ptr_int_t val = retaddr & (ptr_int_t) 0x0000ffff;
+        /* can't do a non-default operand size with a push immed so we emulate */
+        PRE(ilist, instr,
+            INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XSP),
+                             opnd_create_base_disp(REG_XSP, REG_NULL, 0, -2,
+                                                   OPSZ_lea)));
+        PRE(ilist, instr,
+            INSTR_CREATE_mov_st(dcontext, OPND_CREATE_MEM16(REG_XSP, 2),
+                                OPND_CREATE_INT16(val)));
+    } else if (opsize == OPSZ_PTR) {
+        insert_push_immed_ptrsz(dcontext, ilist, instr, retaddr);
+    } else {
+#ifdef X64
+        ptr_int_t val = retaddr & (ptr_int_t) 0xffffffff;
+        ASSERT(opsize == OPSZ_4);
+        /* can't do a non-default operand size with a push immed so we emulate */
+        PRE(ilist, instr,
+            INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XSP),
+                             opnd_create_base_disp(REG_XSP, REG_NULL, 0, -4,
+                                                   OPSZ_lea)));
+        PRE(ilist, instr,
+            INSTR_CREATE_mov_st(dcontext, OPND_CREATE_MEM32(REG_XSP, 4),
+                                OPND_CREATE_INT32(val)));
+#else
+        ASSERT_NOT_REACHED();
+#endif
+    }
+}
+
+static void
+insert_push_cs(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+               ptr_int_t retaddr, opnd_size_t opsize)
+{
+#ifdef X64
+    /* "push cs" is invalid; for now we just push 0x33, a common value of cs.
+     * PR 271317 covers doing this properly.
+     */
+    insert_push_retaddr(dcontext, ilist, instr, 0x33, opsize);
+#else
+    opnd_t stackop;
+    /* we go ahead and push cs, but we won't pop into cs */
+    instr_t *push = INSTR_CREATE_push(dcontext, opnd_create_reg(SEG_CS));
+    /* 2nd dest is the stack operand size */
+    stackop = instr_get_dst(push, 1);
+    opnd_set_size(&stackop, opsize);
+    instr_set_dst(push, 1, stackop);
+    PRE(ilist, instr, push);
+#endif
+}
+
+/***************************************************************************
+ * DIRECT CALL
+ * Returns new next_instr
+ */
+static instr_t *
+mangle_direct_call(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+                   instr_t *next_instr, bool mangle_calls)
+{
+    ptr_uint_t retaddr, curaddr;
+    app_pc target = NULL;
+    int len = instr_length(dcontext, instr);
+    opnd_t pushop = instr_get_dst(instr, 1);
+    if (opnd_is_near_pc(instr_get_target(instr)))
+        target = opnd_get_pc(instr_get_target(instr));
+    else if (opnd_is_instr(instr_get_target(instr))) {
+        instr_t *tgt = opnd_get_instr(instr_get_target(instr));
+        /* assumption: target's raw bits are meaningful */
+        target = instr_get_raw_bits(tgt);
+        ASSERT(target != 0);
+        /* FIXME case 6962: for far instr, we ignore the segment and
+         * assume it matches current cs */
+    } else if (opnd_is_far_pc(instr_get_target(instr))) {
+        target = opnd_get_pc(instr_get_target(instr));
+        /* FIXME case 6962: we ignore the segment and assume it matches current cs */
+    } else
+        ASSERT_NOT_REACHED();
+
+    if (!mangle_calls ||
+#ifdef INTERNAL
+        !dynamo_options.inline_calls ||
+#endif
+        must_not_be_inlined(target)) {
+        /* off-trace call that will be executed natively */
+
+        /* relative target must be re-encoded */
+        instr_set_raw_bits_valid(instr, false);
+
+#ifdef STEAL_REGISTER
+        /* FIXME: need to push edi prior to call and pop after.
+         * However, need to push edi prior to any args to this call,
+         * and it may be hard to find pre-arg-pushing spot...
+         * edi is supposed to be callee-saved, we're trusting this
+         * off-trace call to return, we may as well trust it to
+         * not trash edi -- these no-inline calls are dynamo's
+         * own routines, after all.
+         */
+#endif
+        return next_instr;
+    } 
+
+    /* For CI builds, use the translation field so we can handle cases
+     * where the client has changed the target and invalidated the raw
+     * bits.  We'll make sure the translation is always set for direct
+     * calls.
+     */
+    curaddr = (ptr_uint_t) instr_get_translation(instr);
+    if (curaddr == 0 && instr_raw_bits_valid(instr))
+        curaddr = (ptr_uint_t) instr_get_raw_bits(instr);
+    ASSERT(curaddr != 0);
+    retaddr = curaddr + len;
+
+#if defined(RETURN_STACK) || defined(NATIVE_RETURN)
+    /* ASSUMPTION: a call to the next instr is not going to ever have a matching ret!
+     * FIXME: have a flag to turn this off...aggressiveness level?
+     */
+    if (target == (app_pc)retaddr) {
+        LOG(THREAD, LOG_INTERP, 3, "found call to next instruction "PFX"\n", target);
+#endif
+
+#ifdef CHECK_RETURNS_SSE2
+        /* ASSUMPTION: a call to the next instr is not going to ever have a
+         * matching ret! */
+        if (target == (app_pc)retaddr) {
+            LOG(THREAD, LOG_INTERP, 3, "found call to next instruction "PFX"\n", target);
+        } else {
+            check_return_handle_call(dcontext, ilist, next_instr);
+        }
+        /* now do the normal thing for a call */
+#endif
+
+        if (instr_get_opcode(instr) == OP_call_far) {
+            /* N.B.: we do not support other than flat 0-based CS, DS, SS, and ES.
+             * if the app wants to change segments, we won't actually issue
+             * a segment change, and so will only work properly if the new segment
+             * is also 0-based.  To properly issue new segments, we'd need a special
+             * ibl that ends in a far cti, and all prior address manipulations would
+             * need to be relative to the new segment, w/o messing up current segment.
+             * FIXME: can we do better without too much work?
+             */
+            SYSLOG_INTERNAL_WARNING_ONCE("Encountered a far direct call");
+            STATS_INC(num_far_dir_calls);
+            insert_push_cs(dcontext, ilist, instr, 0, opnd_get_size(pushop));
+        }
+
+        /* convert a direct call to a push of the return address */
+        insert_push_retaddr(dcontext, ilist, instr, retaddr, opnd_get_size(pushop));
+        
+        /* remove the call */
+        instrlist_remove(ilist, instr);
+        instr_destroy(dcontext, instr);
+        return next_instr;
+
+#if defined(RETURN_STACK) || defined(NATIVE_RETURN)
+    }
+    /* "real" call (not to next instr) */
+# ifdef NATIVE_RETURN
+    return native_ret_mangle_direct_call(dcontext, ilist, instr,
+                                         next_instr, retaddr);
+# else /* RETURN_STACK */
+    return return_stack_mangle_direct_call(dcontext, ilist, instr,
+                                           next_instr, retaddr);
+# endif
+#endif
+}
+
+
+/* We spill to XCX(private dcontext) slot for private fragments, 
+ * and to TLS MANGLE_XCX_SPILL_SLOT for shared fragments.
+ * (Except for DYNAMO_OPTION(private_ib_in_tls), for which all use tls,
+ *  but that has a performance hit because of the extra data cache line)
+ * We can get away with the split by having the shared ibl routine copy
+ * xcx to the private dcontext, and by having the private ibl never
+ * target shared fragments.
+ * We also have to modify the xcx spill from tls to private dcontext when
+ * adding a shared basic block to a trace.
+ *
+ * FIXME: if we do make non-trace-head basic blocks valid indirect branch
+ * targets, we should have the private ibl have special code to test the
+ * flags and copy xcx to the tls slot if necessary.
+ */
+#define SAVE_TO_DC_OR_TLS(dc, flags, reg, tls_offs, dc_offs)                      \
+    ((DYNAMO_OPTION(private_ib_in_tls) || TEST(FRAG_SHARED, (flags))) ?           \
+     INSTR_CREATE_mov_st(dcontext, opnd_create_tls_slot(os_tls_offset(tls_offs)), \
+                         opnd_create_reg(reg)) :                                  \
+     instr_create_save_to_dcontext((dc), (reg), (dc_offs)))
+
+/***************************************************************************
+ * INDIRECT CALL
+ */
+static void
+mangle_indirect_call(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+                     instr_t *next_instr, bool mangle_calls, uint flags)
+{
+    opnd_t target;
+    ptr_uint_t retaddr, curaddr;
+#ifdef RETURN_STACK
+    instr_t *cleanup;
+#endif
+    opnd_t pushop = instr_get_dst(instr, 1);
+    reg_id_t reg_target = REG_XCX;
+
+    if (!mangle_calls)
+        return;
+
+    /* Convert near, indirect calls.  The jump to the exit_stub that
+     * jumps to indirect_branch_lookup was already inserted into the
+     * instr list by interp EXCEPT for the case in which we're converting
+     * an indirect call to a direct call. In that case, mangle later
+     * inserts a direct exit stub.
+     */
+
+    /* If a client changes an instr, or our own mangle_rel_addr() does,
+     * the raw bits won't be valid but the translation should be */
+    curaddr = (ptr_uint_t) instr_get_translation(instr);
+    if (curaddr == 0 && instr_raw_bits_valid(instr))
+        curaddr = (ptr_uint_t) instr_get_raw_bits(instr);
+    ASSERT(curaddr != 0);
+    if (instr_raw_bits_valid(instr))
+        retaddr = curaddr + instr->length;
+    else /* mangle_rel_addr() may have changed length: use original! */
+        retaddr = (ptr_uint_t) decode_next_pc(dcontext, (byte *)curaddr);
+    ASSERT(retaddr != 0);
+
+    /* If this call is marked for conversion, do minimal processing.
+     * FIXME Just a note that converted calls are not subjected to any of
+     * the specialized builds' processing further down.
+     */
+    if (TEST(INSTR_IND_CALL_DIRECT, instr->flags)) {
+        /* convert the call to a push of the return address */
+        insert_push_retaddr(dcontext, ilist, instr, retaddr, opnd_get_size(pushop));
+        /* remove the call */
+        instrlist_remove(ilist, instr);
+        instr_destroy(dcontext, instr);
+        return;
+    }
+
+#if !defined(RETURN_STACK) && !defined(NATIVE_RETURN)
+    /* put the push AFTER the instruction that calculates
+     * the target, b/c if target depends on xsp we must use
+     * the value of xsp prior to this call instruction!
+     * we insert before next_instr to accomplish this.
+     */
+    if (instr_get_opcode(instr) == OP_call_far_ind) {
+        /* goes right before the push of the ret addr */
+        insert_push_cs(dcontext, ilist, next_instr, 0, opnd_get_size(pushop));
+        /* see notes below -- we don't really support switching segments,
+         * though we do go ahead and push cs, we won't pop into cs
+         */
+    }
+    insert_push_retaddr(dcontext, ilist, next_instr, retaddr, opnd_get_size(pushop));
+#endif
+    
+    /* save away xcx so that we can use it */
+    /* (it's restored in x86.s (indirect_branch_lookup) */
+    PRE(ilist, instr,
+        SAVE_TO_DC_OR_TLS(dcontext, flags, REG_XCX, MANGLE_XCX_SPILL_SLOT, XCX_OFFSET));
+    
+#ifdef STEAL_REGISTER
+    /* Steal edi if call uses it, using original call instruction */
+    steal_reg(dcontext, instr, ilist);
+    if (ilist->flags)
+        restore_state(dcontext, next_instr, ilist);
+    /* It's impossible for our register stealing to use ecx
+     * because no call can simultaneously use 3 registers, right?
+     * Maximum is 2, in something like "call *(edi,ecx,4)"?
+     * If it is possible, need to make sure stealing's use of ecx
+     * doesn't conflict w/ our use
+     */
+#endif    
+    
+    /* change: call /2, Ev -> movl Ev, %xcx */
+    target = instr_get_src(instr, 0);
+    if (instr_get_opcode(instr) == OP_call_far_ind) {
+        opnd_size_t addr_size;
+        /* N.B.: we do not support other than flat 0-based CS, DS, SS, and ES.
+         * if the app wants to change segments, we won't actually issue
+         * a segment change, and so will only work properly if the new segment
+         * is also 0-based.  To properly issue new segments, we'd need a special
+         * ibl that ends in a far cti, and all prior address manipulations would
+         * need to be relative to the new segment, w/o messing up current segment.
+         * FIXME: can we do better without too much work?
+         */
+        SYSLOG_INTERNAL_WARNING_ONCE("Encountered a far indirect call");
+        STATS_INC(num_far_ind_calls);
+        /* opnd type is i_Ep, it's not a far base disp b/c segment is at
+         * memory location, not specified as segment prefix on instr.
+         * we assume register operands are marked as invalid instrs long
+         * before this point.
+         */
+        /* FIXME: if it is a far base disp we assume 0 base */
+        ASSERT(opnd_is_base_disp(target));
+        /* Segment selector is the final 2 bytes.
+         * We ignore it and assume DS base == target cti CS base.
+         */
+        /* if data16 then just 2 bytes for address
+         * if x64 mode and Intel and rex then 8 bytes for address */
+        ASSERT((X64_MODE_DC(dcontext) && opnd_get_size(target) == OPSZ_10 &&
+                proc_get_vendor() != VENDOR_AMD) ||
+               opnd_get_size(target) == OPSZ_6 || opnd_get_size(target) == OPSZ_4);
+        if (opnd_get_size(target) == OPSZ_10) {
+            addr_size = OPSZ_8;
+            reg_target = REG_RCX;
+        } else if (opnd_get_size(target) == OPSZ_6) {
+            addr_size = OPSZ_4;
+            reg_target = REG_ECX;
+        } else /* target has OPSZ_4 */ {
+            addr_size = OPSZ_2;
+            reg_target = REG_XCX; /* we use movzx below so size doesn't have to match */
+        }
+            
+        target = opnd_create_base_disp(opnd_get_base(target), opnd_get_index(target),
+                                       opnd_get_scale(target), opnd_get_disp(target),
+                                       addr_size);
+    }
+    /* cannot call instr_reset, it will kill prev & next ptrs */
+    instr_free(dcontext, instr);
+    instr_set_num_opnds(dcontext, instr, 1, 1);
+    instr_set_opcode(instr, opnd_get_size(target) == OPSZ_2 ? OP_movzx : OP_mov_ld);
+    instr_set_dst(instr, 0, opnd_create_reg(reg_target));
+    instr_set_src(instr, 0, target); /* src stays the same */
+    if (instrlist_get_translation_target(ilist) != NULL) {
+        /* make sure original raw bits are used for translation */
+        instr_set_translation(instr, instr_get_raw_bits(instr));
+    }
+    instr_set_our_mangling(instr, true);
+    
+#ifdef RETURN_STACK
+    /* NEW CALL HANDLING: RETURN STACK! */
+    /* Change call to this:
+     *   push app ret addr
+     *   swap to return stack
+     *   push app ret addr
+     *   call cleanup_stack
+     *   jmp after_call_fragment
+     * cleanup_stack:
+     *   swap to app stack
+     *   jmp exit_stub (already added (==next_instr))
+     */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    cleanup = instr_create_save_dynamo_return_stack(dcontext);
+    PRE(ilist, next_instr,
+        INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32(retaddr)));
+    PRE(ilist, next_instr,
+        instr_create_save_to_dcontext(dcontext, REG_ESP, XSP_OFFSET));
+    PRE(ilist, next_instr,
+        instr_create_restore_dynamo_return_stack(dcontext));
+    PRE(ilist, next_instr,
+        INSTR_CREATE_push_imm(dcontext, OPND_CREATE_INT32(retaddr)));
+    PRE(ilist, next_instr,
+        INSTR_CREATE_call(dcontext, opnd_create_instr(cleanup)));
+    /* an exit cti, not a meta instr */
+    instrlist_preinsert
+        (ilist, next_instr,
+         INSTR_CREATE_jmp(dcontext, opnd_create_pc(retaddr))); 
+    PRE(ilist, next_instr, cleanup);
+    PRE(ilist, next_instr,
+        instr_create_restore_from_dcontext(dcontext, REG_ESP, XSP_OFFSET));
+#endif /* RETURN_STACK */
+
+#ifdef CHECK_RETURNS_SSE2
+    check_return_handle_call(dcontext, ilist, next_instr);
+#endif
+
+#ifdef NATIVE_RETURN
+    /*********************************************************/
+    /* NEW CALL HANDLING: NATIVE RETURN
+          <save flags already here>
+          inc call_depth
+          call skip
+          jmp app_ret_addr
+        skip:
+          jmp exit_stub (already added (==next_instr))
+     */
+    /* inc call_depth */
+# ifdef NATIVE_RETURN_CALLDEPTH
+    instrlist_preinsert(ilist, next_instr,
+                        INSTR_CREATE_inc(dcontext, opnd_create_dcontext_field(dcontext,
+                                                                              CALL_DEPTH_OFFSET)));
+# endif
+    instrlist_preinsert(ilist, next_instr,
+                       INSTR_CREATE_call(dcontext, opnd_create_instr(next_instr)));
+    instrlist_preinsert(ilist, next_instr, INSTR_CREATE_jmp(dcontext,
+                                                opnd_create_pc((app_pc)retaddr)));
+#endif
+}
+
+/***************************************************************************
+ * RETURN
+ */
+static void
+mangle_return(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+              instr_t *next_instr, uint flags)
+{
+    instr_t *pop;
+    opnd_t retaddr;
+
+#ifdef NATIVE_RETURN
+    native_ret_mangle_return(dcontext, ilist, instr, next_instr);
+    return;
+#endif
+
+#ifdef CHECK_RETURNS_SSE2
+    check_return_handle_return(dcontext, ilist, next_instr);
+    /* now do the normal ret mangling */
+#endif
+
+    /* Convert returns.  If aggressive we could take advantage of the
+     * fact that xcx is dead at the return and not bother saving it?
+     * The jump to the exit_stub that jumps to indirect_branch_lookup
+     * was already inserted into the instr list by interp. */
+
+    /* save away xcx so that we can use it */
+    /* (it's restored in x86.s (indirect_branch_lookup) */
+    PRE(ilist, instr,
+        SAVE_TO_DC_OR_TLS(dcontext, flags, REG_XCX, MANGLE_XCX_SPILL_SLOT, XCX_OFFSET));
+
+    /* see if ret has an immed int operand, assumed to be 1st src */
+            
+    if (instr_num_srcs(instr) > 0 && opnd_is_immed_int(instr_get_src(instr, 0))) {
+        /* if has an operand, return removes some stack space,
+         * AFTER the return address is popped
+         */
+        int val = (int) opnd_get_immed_int(instr_get_src(instr, 0));
+        IF_X64(ASSERT_TRUNCATE(val, int, opnd_get_immed_int(instr_get_src(instr, 0))));
+        /* addl sizeof_param_area, %xsp
+         * except that clobbers the flags, so we use lea */
+        PRE(ilist, next_instr,
+            INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XSP),
+                             opnd_create_base_disp(REG_XSP, REG_NULL, 0, val, OPSZ_lea)));
+    }
+            
+    /* don't need to steal edi since return cannot use registers */
+
+    /* the retaddr operand is always the final source for all OP_ret* instrs */
+    retaddr = instr_get_src(instr, instr_num_srcs(instr) - 1);
+
+    if (X64_MODE_DC(dcontext) && 
+        (instr_get_opcode(instr) == OP_iret || instr_get_opcode(instr) == OP_ret_far) &&
+        opnd_get_size(retaddr) == OPSZ_4) {
+        /* N.B.: For some unfathomable reason iret and ret_far default to operand
+         * size 4 in 64-bit mode (making them, along w/ call_far, the only stack
+         * operation instructions to do so). So if we see an iret or far ret with
+         * OPSZ_4 in 64-bit mode we need a 4-byte pop, but since we can't actually
+         * generate a 4-byte pop we have to emulate it here. */
+        SYSLOG_INTERNAL_WARNING_ONCE("Encountered iretd/lretd in 64-bit mode!");
+        /* Note moving into ecx automatically zero extends which is what we want. */
+        PRE(ilist, instr,
+            INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_ECX),
+                                OPND_CREATE_MEM32(REG_RSP, 0)));
+        /* iret could use add since going to pop the eflags, but not lret.
+         * lret could combine w/ segment lea below: but not perf-crit instr, and
+         * anticipating cs preservation PR 271317 I'm leaving separate. */
+        PRE(ilist, instr,
+            INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XSP),
+                             opnd_create_base_disp(REG_XSP, REG_NULL, 0, 4, OPSZ_lea)));
+    } else {
+        /* change RET into a POP, keeping the operand size */
+        pop = INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_XCX));
+        instr_set_src(pop, 1, retaddr);
+        if (opnd_get_size(retaddr) == OPSZ_2)
+            instr_set_dst(pop, 0, opnd_create_reg(REG_CX));
+        /* We can't do a 4-byte pop in 64-bit mode, but excepting iretd and lretd
+         * handled above we should never see one. */
+        ASSERT(!X64_MODE_DC(dcontext) || opnd_get_size(retaddr) != OPSZ_4);
+        PRE(ilist, instr, pop);
+        if (opnd_get_size(retaddr) == OPSZ_2) {
+            /* we need to zero out the top 2 bytes */
+            PRE(ilist, instr, INSTR_CREATE_movzx
+                (dcontext,
+                 opnd_create_reg(REG_ECX), opnd_create_reg(REG_CX)));
+        }
+    }
+
+    if (instr_get_opcode(instr) == OP_ret_far) {
+        /* N.B.: we do not support other than flat 0-based CS, DS, SS, and ES.
+         * if the app wants to change segments, we won't actually issue
+         * a segment change, and so will only work properly if the new segment
+         * is also 0-based.  To properly issue new segments, we'd need a special
+         * ibl that ends in a far cti, and all prior address manipulations would
+         * need to be relative to the new segment, w/o messing up current segment.
+         * FIXME: can we do better without too much work?
+         */
+        SYSLOG_INTERNAL_WARNING_ONCE("Encountered a far ret");
+        STATS_INC(num_far_rets);
+        /* pop selector from stack, but not into cs, just junk it
+         * (the 16-bit selector is expanded to 32 bits on the push, unless data16)
+         */
+        PRE(ilist, instr,
+            INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XSP),
+                             opnd_create_base_disp
+                             (REG_XSP, REG_NULL, 0,
+                              opnd_size_in_bytes(opnd_get_size(retaddr)), OPSZ_lea)));
+    }
+
+    if (instr_get_opcode(instr) == OP_iret) {
+        instr_t *popf;
+
+        /* Xref PR 215553 and PR 191977 - we actually see this on 64-bit Vista */
+#ifndef X64
+        ASSERT_NOT_TESTED();
+#endif
+        LOG(THREAD, LOG_INTERP, 2, "Encountered iret at "PFX" - mangling\n",
+            instr_get_translation(instr));
+        STATS_INC(num_irets);
+
+        /* In 32-bit mode and 64-bit mode with 32-bit operand size this is a
+         * pop->EIP pop->CS pop->eflags.  64-bit mode with 64-bit operand size extends
+         * the above and additionally adds pop->RSP pop->ss.  N.B.: like OP_far_ret we
+         * ignore the CS and SS segment changes (FIXME: see the comments there for why,
+         * can we do better?). */
+        
+        /* Return address is already popped, next up is CS segment which we ignore so
+         * adjust stack pointer. Note we can use an add here since the eflags will
+         * be written below. */
+        PRE(ilist, instr,
+            INSTR_CREATE_add(dcontext, opnd_create_reg(REG_XSP),
+                             OPND_CREATE_INT8
+                             (opnd_size_in_bytes(opnd_get_size(retaddr)))));
+
+        /* Next up is xflags, we use a popf. Popf should be setting the right flags 
+         * (it's difficult to tell because in the docs iret lists the flags it does
+         * set while popf lists the flags it doesn't set). The docs aren't entirely
+         * clear, but any flag that we or a user mode program would care about should
+         * be right. */
+        popf = INSTR_CREATE_popf(dcontext);
+        if (X64_MODE_DC(dcontext) && opnd_get_size(retaddr) == OPSZ_4) {
+            /* We can't actually create a 32-bit popf and there's no easy way to
+             * simulate one.  For now we'll do a 64-bit popf and fixup the stack offset.
+             * If AMD/INTEL ever start using the top half of the rflags register then
+             * we could have problems here. We could also break stack transparency and
+             * do a mov, push, popf to zero extend the value. */
+            ASSERT_NOT_TESTED();
+            PRE(ilist, instr, popf);
+            /* flags are already set, must use lea to fix stack */
+            PRE(ilist, instr,
+                INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XSP),
+                                 opnd_create_base_disp
+                                 (REG_XSP, REG_NULL, 0, -4, OPSZ_lea)));
+        } else {
+            /* get popf size right the same way we do it for the return address */
+            DODEBUG(if (opnd_get_size(retaddr) == OPSZ_2)
+                        ASSERT_NOT_TESTED(););
+            instr_set_src(popf, 1, retaddr);
+            PRE(ilist, instr, popf);
+        }
+
+        /* If the operand size is 64-bits iret additionaly does pop->RSP and pop->ss. */
+        if (opnd_get_size(retaddr) == OPSZ_8) {
+            PRE(ilist, instr, INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_RSP)));
+            /* We're ignoring the set of SS and since we just set RSP we don't need
+             * to do anything to adjust the stack for the pop (since the pop would have
+             * occured with the old RSP). */
+        } else {
+            ASSERT_NOT_TESTED();
+        }
+    }
+
+    /* remove the ret */
+    instrlist_remove(ilist, instr);
+    instr_destroy(dcontext, instr);
+}
+
+/***************************************************************************
+ * INDIRECT JUMP
+ */
+static void
+mangle_indirect_jump(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+                     instr_t *next_instr, uint flags)
+{
+    opnd_t target;
+    reg_id_t reg_target = REG_XCX;
+
+    /* Convert indirect branches (that are not returns).  Again, the
+     * jump to the exit_stub that jumps to indirect_branch_lookup
+     * was already inserted into the instr list by interp. */
+
+    /* save away xcx so that we can use it */
+    /* (it's restored in x86.s (indirect_branch_lookup) */
+    PRE(ilist, instr,
+        SAVE_TO_DC_OR_TLS(dcontext, flags, REG_XCX, MANGLE_XCX_SPILL_SLOT, XCX_OFFSET));
+
+#ifdef STEAL_REGISTER
+    /* Steal edi if branch uses it, using original instruction */
+    steal_reg(dcontext, instr, ilist);
+    if (ilist->flags)
+        restore_state(dcontext, next_instr, ilist);
+#endif
+
+    /* change: jmp /4, i_Ev -> movl i_Ev, %xcx */
+    target = instr_get_target(instr);
+    if (instr_get_opcode(instr) == OP_jmp_far_ind) {
+        opnd_size_t addr_size;
+        /* N.B.: we do not support other than flat 0-based CS, DS, SS, and ES.
+         * if the app wants to change segments, we won't actually issue
+         * a segment change, and so will only work properly if the new segment
+         * is also 0-based.  To properly issue new segments, we'd need a special
+         * ibl that ends in a far cti, and all prior address manipulations would
+         * need to be relative to the new segment, w/o messing up current segment.
+         * FIXME: can we do better without too much work?
+         */
+        SYSLOG_INTERNAL_WARNING_ONCE("Encountered a far indirect jump");
+        STATS_INC(num_far_ind_jmps);
+        /* opnd type is i_Ep, it's not a far base disp b/c segment is at
+         * memory location, not specified as segment prefix on instr
+         */
+        /* FIXME: if it is a far base disp we assume 0 base */
+        ASSERT(opnd_is_base_disp(target));
+        /* Segment selector is the final 2 bytes.
+         * We ignore it and assume DS base == target cti CS base.
+         */
+        /* if data16 then just 2 bytes for address
+         * if x64 mode and Intel and rex then 8 bytes for address */
+        ASSERT((X64_MODE_DC(dcontext) && opnd_get_size(target) == OPSZ_10 &&
+                proc_get_vendor() != VENDOR_AMD) ||
+               opnd_get_size(target) == OPSZ_6 || opnd_get_size(target) == OPSZ_4);
+        if (opnd_get_size(target) == OPSZ_10) {
+            addr_size = OPSZ_8;
+            reg_target = REG_RCX;
+        } else if (opnd_get_size(target) == OPSZ_6) {
+            addr_size = OPSZ_4;
+            reg_target = REG_ECX;
+        } else /* target has OPSZ_4 */ {
+            addr_size = OPSZ_2;
+            reg_target = REG_XCX; /* we use movzx below */
+        }
+
+        target = opnd_create_base_disp(opnd_get_base(target), opnd_get_index(target),
+                                       opnd_get_scale(target), opnd_get_disp(target),
+                                       addr_size);
+    }
+    /* cannot call instr_reset, it will kill prev & next ptrs */
+    instr_free(dcontext, instr);
+    instr_set_num_opnds(dcontext, instr, 1, 1);
+    instr_set_opcode(instr, opnd_get_size(target) == OPSZ_2 ? OP_movzx : OP_mov_ld);
+    instr_set_dst(instr, 0, opnd_create_reg(reg_target));
+    instr_set_src(instr, 0, target); /* src stays the same */
+    if (instrlist_get_translation_target(ilist) != NULL) {
+        /* make sure original raw bits are used for translation */
+        instr_set_translation(instr, instr_get_raw_bits(instr));
+    }
+    instr_set_our_mangling(instr, true);
+
+    /* It's impossible for our register stealing to use ecx
+     * because no branch can simultaneously use 3 registers, right?
+     * Maximum is 2, in something like "jmp *(edi,ebx,4)"?
+     * If it is possible, need to make sure stealing's use of ecx
+     * doesn't conflict w/ our use = FIXME
+     */
+}
+
+
+/***************************************************************************
+ * SYSCALL
+ */
+
+/* Tries to statically find the syscall number for the
+ * syscall instruction instr.
+ * Returns -1 upon failure.
+ */
+int
+find_syscall_num(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr)
+{
+    int syscall = -1;
+    instr_t *prev = instr_get_prev(instr);
+    if (prev != NULL) {
+        prev = instr_get_prev_expanded(dcontext, ilist, instr);
+        /* walk backwards looking for "mov_imm imm->xax"
+         * may be other instrs placing operands into registers
+         * for the syscall in between
+         */
+        while (prev != NULL &&
+               instr_num_dsts(prev) > 0 &&
+               opnd_is_reg(instr_get_dst(prev, 0)) &&
+               opnd_get_reg(instr_get_dst(prev, 0)) != REG_EAX) {
+#ifdef CLIENT_INTERFACE
+            /* if client added cti in between, bail and assume non-ignorable */
+            if (instr_is_cti(prev))
+                return -1;
+#endif
+            prev = instr_get_prev(prev);
+        }
+        if (prev != NULL &&
+            instr_get_opcode(prev) == OP_mov_imm &&
+            opnd_get_reg(instr_get_dst(prev, 0)) == REG_EAX) {
+#ifdef CLIENT_INTERFACE
+            instr_t *walk, *tgt;
+#endif
+            IF_X64(ASSERT_TRUNCATE(int, int,
+                                   opnd_get_immed_int(instr_get_src(prev, 0))));
+            syscall = (int) opnd_get_immed_int(instr_get_src(prev, 0));
+#ifdef CLIENT_INTERFACE
+            /* if client added cti target in between, bail and assume non-ignorable */
+            for (walk = instrlist_first_expanded(dcontext, ilist);
+                 walk != NULL;
+                 walk = instr_get_next_expanded(dcontext, ilist, walk)) {
+                if (instr_is_cti(walk) && opnd_is_instr(instr_get_target(walk))) {
+                    for (tgt = opnd_get_instr(instr_get_target(walk));
+                         tgt != NULL;
+                         tgt = instr_get_next_expanded(dcontext, ilist, tgt)) {
+                        if (tgt == prev)
+                            break;
+                        if (tgt == instr)
+                            return -1;
+                    }
+                }
+            }
+#endif
+        }
+    }
+    return syscall;
+}
+
+#ifdef LINUX
+/* Inserts code to handle clone into ilist.
+ * instr is the syscall instr itself.
+ * Assumes that instructions exist beyond instr in ilist.
+ * pc_to_ecx is an instr that puts the pc after the app's syscall instr
+ * into xcx.
+ * skip decides whether the clone code is skipped by default or not.
+ */
+void
+mangle_insert_clone_code(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+                         instr_t *pc_to_ecx, bool skip _IF_X64(gencode_mode_t mode))
+{
+            /*
+                int 0x80
+            .if don't know sysnum statically:
+                jmp ignore  <-- modifiable jmp
+            .else
+                jmp xchg    # need this so can jmp to ignore if !CLONE_VM
+            .endif
+              xchg:
+                xchg xax,xcx
+                jecxz child
+                jmp parent
+              child:
+                mov <app pc after int>,xcx
+                jmp new_thread_dynamo_start
+              parent:
+                xchg xax,xcx
+              ignore:
+                <post system call, etc.>
+            */
+    instr_t *in = instr_get_next(instr);
+    /* child == pc_to_ecx */
+    instr_t *parent =
+        INSTR_CREATE_xchg(dcontext, opnd_create_reg(REG_XAX),
+                          opnd_create_reg(REG_XCX));
+    instr_t *xchg =
+        INSTR_CREATE_xchg(dcontext, opnd_create_reg(REG_XAX),
+                          opnd_create_reg(REG_XCX));
+    ASSERT(in != NULL);
+    /* we have to dynamically skip or not skip the clone code
+     * see mangle_clone_code below
+     */
+    if (skip) {
+        /* Insert a jmp that normally skips the clone stuff,
+         * pre_system_call will modify it if it really is SYS_clone.
+         */
+        PRE(ilist, in,
+            INSTR_CREATE_jmp(dcontext, opnd_create_instr(in)));
+    } else {
+        /* We have to do this even if we statically know the sysnum
+         * because if CLONE_VM is not set this is a fork, and we then
+         * want to skip our clone code.
+         */
+        PRE(ilist, in,
+            INSTR_CREATE_jmp(dcontext, opnd_create_instr(xchg)));
+    }
+    PRE(ilist, in, xchg);
+    PRE(ilist, in,
+        INSTR_CREATE_jecxz(dcontext, opnd_create_instr(pc_to_ecx)));
+    PRE(ilist, in,
+        INSTR_CREATE_jmp(dcontext, opnd_create_instr(parent)));
+    PRE(ilist, in, pc_to_ecx);
+    /* an exit cti, not a meta instr */
+    instrlist_preinsert
+        (ilist, in,
+         INSTR_CREATE_jmp(dcontext, opnd_create_pc
+                          ((app_pc)get_new_thread_start(dcontext _IF_X64(mode)))));
+    instr_set_ok_to_mangle(instr_get_prev(in), false);
+    PRE(ilist, in, parent);
+}
+#endif /* LINUX */
+
+#ifdef WINDOWS
+/* Note that ignore syscalls processing for XP and 2003 is a two-phase operation.
+ * For this reason, mangle_syscall() might be called with a 'next_instr' that's
+ * not an original app instruction but one inserted by the earlier mangling phase.
+ */
+#endif
+static void
+mangle_syscall(dcontext_t *dcontext, instrlist_t *ilist, uint flags,
+               instr_t *instr, instr_t *next_instr)
+{
+#ifdef LINUX
+    if (get_syscall_method() != SYSCALL_METHOD_INT &&
+        get_syscall_method() != SYSCALL_METHOD_SYSCALL &&
+        get_syscall_method() != SYSCALL_METHOD_SYSENTER) {
+        /* don't know convention on return address from kernel mode! */
+        SYSLOG_INTERNAL_ERROR("unsupported system call method");
+        LOG(THREAD, LOG_INTERP, 1, "don't know convention for this syscall method\n");
+        CLIENT_ASSERT(false, "Unsupported system call method detected. Please "
+                      "reboot with the nosep kernel option if this is a 32-bit "
+                      "2.5 or 2.6 version Linux kernel.");
+    }
+    /* cannot use dynamo stack in code cache, so we cannot insert a
+     * call -- instead we have interp end bbs at interrupts unless
+     * we can identify them as ignorable system calls.  Otherwise,
+     * we just remove the instruction and jump back to dynamo to
+     * handle it.
+     */
+    if (TESTANY(INSTR_NI_SYSCALL_ALL, instr->flags)) {
+        instrlist_remove(ilist, instr);
+        instr_destroy(dcontext, instr);
+        return;
+    }
+
+    /* signal barrier: need to be able to exit fragment immediately
+     * prior to syscall, so we set up an exit cti with a jmp right beforehand
+     * that by default hops over the exit cti.
+     * when we want to exit right before the syscall, we call the
+     * mangle_syscall_code() routine below.
+     */
+    PRE(ilist, instr, INSTR_CREATE_jmp(dcontext, opnd_create_instr(instr)));
+    /* assumption: raw bits of instr == app pc */
+    ASSERT(instr_get_raw_bits(instr) != NULL);
+    /* this should NOT be a meta-instr so we don't use PRE */
+    instrlist_preinsert(ilist, instr, INSTR_CREATE_jmp
+                        (dcontext, opnd_create_pc(instr_get_raw_bits(instr))));
+
+# ifdef STEAL_REGISTER
+    /* in linux, system calls get their parameters via registers.
+     * edi is the last one used, but there are system calls that
+     * use it, so we put the real value into edi.  plus things
+     * like fork() should get the real register values.
+     * it's also a good idea to put the real edi into %edi for
+     * debugger interrupts (int3).
+     */
+
+    /* the only way we can save and then restore our dc
+     * ptr is to use the stack!
+     * this should be fine, all interrupt instructions push
+     * both eflags and return address on stack, so esp must
+     * be valid at this point.  there could be an application
+     * assuming only 2 slots on stack will be used, we use a 3rd
+     * slot, could mess up that app...but what can we do?
+     * also, if kernel examines user stack, we could have problems.
+     *   push edi          # push dcontext ptr
+     *   restore edi       # restore app edi
+     *   <syscall>
+     *   push ebx
+     *   mov edi, ebx
+     *   mov 4(esp), edi   # get dcontext ptr
+     *   save ebx to edi slot
+     *   pop ebx
+     *   add 4,esp         # clean up push of dcontext ptr
+     */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    PRE(ilist, instr,
+        INSTR_CREATE_push(dcontext, opnd_create_reg(REG_EDI)));
+    PRE(ilist, instr,
+        instr_create_restore_from_dcontext(dcontext, REG_EDI, XDI_OFFSET));
+
+    /* insert after in reverse order: */
+    POST(ilist, instr,
+         INSTR_CREATE_add(dcontext, opnd_create_reg(REG_ESP),
+                          OPND_CREATE_INT8(4)));
+    POST(ilist, instr,
+         INSTR_CREATE_pop(dcontext, opnd_create_reg(REG_EBX)));
+    POST(ilist, instr,
+         instr_create_save_to_dcontext(dcontext, REG_EBX, XDI_OFFSET));
+    POST(ilist, instr,
+         INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_EDI),
+                             OPND_CREATE_MEM32(REG_ESP, 4)));
+    POST(ilist, instr,
+         INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_EBX),
+                             opnd_create_reg(REG_EDI)));
+    POST(ilist, instr,
+         INSTR_CREATE_push(dcontext, opnd_create_reg(REG_EBX)));
+# endif /* STEAL_REGISTER */
+
+#else /* WINDOWS */
+    /* special handling of system calls is performed in shared_syscall or
+     * in do_syscall
+     */
+    
+    /* FIXME: for ignorable syscalls,
+     * do we need support for exiting mid-fragment prior to a syscall
+     * like we do on Linux, to bound time in cache?
+     */
+
+    if (does_syscall_ret_to_callsite()) {
+        uint len = instr_length(dcontext, instr);
+        if (TEST(INSTR_SHARED_SYSCALL, instr->flags)) {
+            ASSERT(DYNAMO_OPTION(shared_syscalls));
+            /* this syscall will be performed by the shared_syscall code
+             * we just need to place a return address into the dcontext
+             * xsi slot or the mangle-next-tag tls slot
+             */
+            if (DYNAMO_OPTION(shared_fragment_shared_syscalls)) {
+# ifdef X64
+                ASSERT(instr_raw_bits_valid(instr));
+                /* PR 244741: no 64-bit store-immed-to-mem 
+                 * FIXME: would be nice to move this to the stub and
+                 * use the dead rbx register!
+                 */
+                PRE(ilist, instr,
+                    instr_create_save_to_tls(dcontext, REG_XCX, MANGLE_NEXT_TAG_SLOT));
+                PRE(ilist, instr,
+                    INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XCX),
+                                         OPND_CREATE_INTPTR((instr->bytes + len))));
+                PRE(ilist, instr, INSTR_CREATE_xchg
+                    (dcontext, opnd_create_tls_slot(os_tls_offset(MANGLE_NEXT_TAG_SLOT)),
+                     opnd_create_reg(REG_XCX)));
+# else
+                PRE(ilist, instr, INSTR_CREATE_mov_st
+                    (dcontext, opnd_create_tls_slot(os_tls_offset(MANGLE_NEXT_TAG_SLOT)),
+                     OPND_CREATE_INTPTR((instr->bytes + len))));
+# endif
+            }
+            else {
+                PRE(ilist, instr, instr_create_save_immed_to_dcontext
+                    (dcontext, (uint)(ptr_uint_t)(instr->bytes + len), XSI_OFFSET));
+            }
+        }
+        /* Handle ignorable syscall.  non-ignorable system calls are
+         * destroyed and removed from the list at the end of this func.
+         */
+        else if (!TEST(INSTR_NI_SYSCALL, instr->flags)) {
+            if (get_syscall_method() == SYSCALL_METHOD_INT &&
+                DYNAMO_OPTION(sygate_int)) {
+                /* for Sygate need to mangle into a call to int_syscall_addr
+                 * is anyone going to get screwed up by this change
+                 * (say flags change?) [-ignore_syscalls only]*/
+                ASSERT_NOT_TESTED();
+                instrlist_replace(ilist, instr, create_syscall_instr(dcontext));
+                instr_destroy(dcontext, instr);
+            } else if (get_syscall_method() == SYSCALL_METHOD_SYSCALL)
+                ASSERT_NOT_TESTED();
+            else if (get_syscall_method() == SYSCALL_METHOD_WOW64)
+                ASSERT_NOT_TESTED();
+            return;
+        }
+    } else if (get_syscall_method() == SYSCALL_METHOD_SYSENTER) {
+        /* on XP/2003 we have a choice between inserting a trampoline at the
+         * return pt of the sysenter, which is 0x7ffe0304 (except for
+         * SP2-patched XP), which is bad since it would clobber whatever's after
+         * the ret there (unless we used a 0xcc, like Visual Studio 2005 debugger
+         * does), or replacing the ret addr on the stack -- we choose the
+         * latter as the lesser of two transparency evils. Note that the
+         * page at 0x7ffe0000 can't be made writable anyway, so hooking
+         * isn't possible.
+         */
+        if (TEST(INSTR_SHARED_SYSCALL, instr->flags)) {
+            ASSERT(DYNAMO_OPTION(shared_syscalls));
+        }
+        /* Handle ignorable syscall.  non-ignorable system calls are
+         * destroyed and removed from the list at the end of this func.
+         */
+        else if (!TEST(INSTR_NI_SYSCALL, instr->flags)) {
+
+            instr_t *mov_imm;
+
+            /* even w/ ignorable syscall, need to make sure regain control */
+            ASSERT(next_instr != NULL);
+            ASSERT(DYNAMO_OPTION(indcall2direct));
+            /* for sygate hack need to basically duplicate what is done in
+             * shared_syscall, but here we could be shared so would need to
+             * grab dcontext first etc. */
+            ASSERT_NOT_IMPLEMENTED(!DYNAMO_OPTION(sygate_sysenter));
+            /* PR 253943: we don't support sysenter in x64 */
+            IF_X64(ASSERT_NOT_IMPLEMENTED(false)); /* can't have 8-byte imm-to-mem */
+            /* FIXME PR 303413: we won't properly translate a fault in our
+             * app stack reference here.  It's marked as our own mangling
+             * so we'll at least return failure from our translate routine.
+             */
+            mov_imm = INSTR_CREATE_mov_st(dcontext, OPND_CREATE_MEM32(REG_XSP, 0),
+                                          opnd_create_instr(next_instr));
+            ASSERT(instr_is_mov_imm_to_tos(mov_imm));
+            PRE(ilist, instr, mov_imm);
+            /* do not let any encoding for length be cached!
+             * o/w will lose pc-relative opnd
+             */
+            /* 'next_instr' is executed after the after-syscall vsyscall
+             * 'ret', which is executed natively. */
+            instr_set_ok_to_mangle(instr_get_prev(instr), false);
+            return; /* leave syscall instr alone */
+        }
+    } else {
+        SYSLOG_INTERNAL_ERROR("unsupported system call method");
+        LOG(THREAD, LOG_INTERP, 1, "don't know convention for this syscall method\n");
+        if (!TEST(INSTR_NI_SYSCALL, instr->flags))
+            return;
+        ASSERT_NOT_IMPLEMENTED(false);
+    }
+
+    /* destroy the syscall instruction */
+    instrlist_remove(ilist, instr);
+    instr_destroy(dcontext, instr);
+#endif /* WINDOWS */
+}
+
+#ifdef LINUX
+
+/* Makes sure the jmp immediately after the syscall instruction
+ * either skips or doesn't skip the clone code following it,
+ * as indicated by the parameter skip.
+ * pc must be either the return address of pre_system_call or
+ * the address of do_syscall.
+ */
+void
+mangle_clone_code(dcontext_t *dcontext, byte *pc, bool skip)
+{
+    byte *target, *prev_pc;
+    instr_t instr;
+    instr_init(dcontext, &instr);
+    LOG(THREAD, LOG_SYSCALLS, 3,
+        "mangle_clone_code: pc="PFX", skip=%d\n", pc, skip);
+    do {
+        instr_reset(dcontext, &instr);
+        pc = decode(dcontext, pc, &instr);
+        ASSERT(pc != NULL); /* our own code! */
+    } while (!instr_is_syscall(&instr));
+    /* jmp is right after syscall */
+    instr_reset(dcontext, &instr);
+    prev_pc = pc;
+    pc = decode(dcontext, pc, &instr);
+    ASSERT(pc != NULL); /* our own code! */
+    ASSERT(instr_get_opcode(&instr) == OP_jmp);
+    if (skip) {
+        /* target is after 2nd xchg */
+        instr_t tmp_instr;
+        int num_xchg = 0;
+        target = pc;
+        instr_init(dcontext, &tmp_instr);
+        while (num_xchg <= 1) {
+            instr_reset(dcontext, &tmp_instr);
+            target = decode(dcontext, target, &tmp_instr);
+            ASSERT(target != NULL); /* our own code! */
+            if (instr_get_opcode(&tmp_instr) == OP_xchg)
+                num_xchg++;
+        }
+        /* now target is instr after 2nd xchg */
+    } else {
+        target = pc;
+    }
+    if (opnd_get_pc(instr_get_target(&instr)) != target) {
+        DEBUG_DECLARE(byte *nxt_pc;)
+        LOG(THREAD, LOG_SYSCALLS, 3,
+            "\tmodifying target of after-clone jmp to "PFX"\n", target);
+        instr_set_target(&instr, opnd_create_pc(target));
+        DEBUG_DECLARE(nxt_pc = ) instr_encode(dcontext, &instr, prev_pc);
+        ASSERT(nxt_pc != NULL && nxt_pc == pc);
+    } else {
+        LOG(THREAD, LOG_SYSCALLS, 3,
+            "\ttarget of after-clone jmp is already "PFX"\n", target);
+    }
+    instr_reset(dcontext, &instr);
+}
+
+/* If skip is false:
+ *   changes the jmp right before the next syscall (after pc) to target the
+ *   exit cti immediately following it;
+ * If skip is true:
+ *   changes back to the default, where skip hops over the exit cti,
+ *   which is assumed to be located at pc.
+ */
+void
+mangle_syscall_code(dcontext_t *dcontext, fragment_t *f, byte *pc, bool skip)
+{
+    byte *stop_pc = fragment_body_end_pc(dcontext, f);
+    byte *target, *prev_pc, *cti_pc, *skip_pc;
+    instr_t instr, cti;
+    instr_init(dcontext, &instr);
+    instr_init(dcontext, &cti);
+    LOG(THREAD, LOG_SYSCALLS, 3,
+        "mangle_syscall_code: pc="PFX", skip=%d\n", pc, skip);
+    do {
+        instr_reset(dcontext, &instr);
+        prev_pc = pc;
+        pc = decode(dcontext, pc, &instr);
+        ASSERT(pc != NULL); /* our own code! */
+        if (pc >= stop_pc) {
+            LOG(THREAD, LOG_SYSCALLS, 3, "\tno syscalls found\n");
+            return;
+        }
+    } while (!instr_is_syscall(&instr));
+    /* jmps are right before syscall */
+    cti_pc = prev_pc - 6;
+    skip_pc = cti_pc - 6;
+    instr_reset(dcontext, &instr);
+    pc = decode(dcontext, skip_pc, &instr);
+    ASSERT(pc != NULL); /* our own code! */
+    ASSERT(instr_get_opcode(&instr) == OP_jmp);
+    ASSERT(pc == cti_pc);
+    DODEBUG({
+        pc = decode(dcontext, cti_pc, &cti);
+        ASSERT(pc != NULL); /* our own code! */
+        ASSERT(instr_get_opcode(&cti) == OP_jmp);
+        ASSERT(pc == prev_pc);
+        instr_reset(dcontext, &cti);
+    });
+    if (skip) {
+        /* target is syscall itself */
+        target = prev_pc;
+    } else {
+        /* target is exit cti */
+        target = cti_pc;
+    }
+    /* FIXME : this should work out to just a 1 byte write, but let's make
+     * it more clear that this is atomic! */
+    if (opnd_get_pc(instr_get_target(&instr)) != target) {
+        DEBUG_DECLARE(byte *nxt_pc;)
+        LOG(THREAD, LOG_SYSCALLS, 3,
+            "\tmodifying target of syscall jmp to "PFX"\n", target);
+        instr_set_target(&instr, opnd_create_pc(target));
+        DEBUG_DECLARE(nxt_pc = ) instr_encode(dcontext, &instr, skip_pc);
+        ASSERT(nxt_pc != NULL && nxt_pc == cti_pc);
+    } else {
+        LOG(THREAD, LOG_SYSCALLS, 3,
+            "\ttarget of syscall jmp is already "PFX"\n", target);
+    }
+    instr_reset(dcontext, &instr);
+}
+#endif /* LINUX */
+
+/***************************************************************************
+ * NON-SYSCALL INTERRUPT
+ */
+static void
+mangle_interrupt(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+                 instr_t *next_instr)
+{
+#ifdef WINDOWS
+    int num;
+    if (instr_get_opcode(instr) != OP_int)
+        return;
+    num = instr_get_interrupt_number(instr);
+    if (num == 0x2b) {
+        /* A callback finishes and returns to the interruption
+         * point of the thread with the instruction "int 2b".
+         * The interrupt ends the block; remove the instruction
+         * since we'll come back to dynamo to perform the
+         * interrupt.
+         */
+        instrlist_remove(ilist, instr);
+        instr_destroy(dcontext, instr);
+    }
+#endif /* WINDOWS */
+}
+
+
+/***************************************************************************
+ * CPUID FOOLING
+ */
+#ifdef FOOL_CPUID
+
+/* values returned by cpuid for Mobile Pentium MMX processor (family 5, model 8) 
+ * minus mmx (==0x00800000 in CPUID_1_EDX)
+ * FIXME: change model number to a Pentium w/o MMX!
+ */
+#define CPUID_0_EAX 0x00000001 
+#define CPUID_0_EBX 0x756e6547
+#define CPUID_0_ECX 0x6c65746e
+#define CPUID_0_EDX 0x49656e69
+/* extended family, extended model, type, family, model, stepping id: */
+/* 20:27,           16:19,          12:13, 8:11,  4:7,   0:3    */
+#define CPUID_1_EAX 0x00000581
+#define CPUID_1_EBX 0x00000000
+#define CPUID_1_ECX 0x00000000
+#define CPUID_1_EDX 0x000001bf
+
+static void
+mangle_cpuid(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+             instr_t *next_instr)
+{
+    /* assumption: input value is put in eax on prev instr, or
+     * on instr prior to that and prev is an inc instr.
+     * alternative is to insert conditional branch...and save eflags, etc.
+     */
+    instr_t *prev = instr_get_prev(instr);
+    opnd_t op;
+    int input, out_eax, out_ebx, out_ecx, out_edx;
+
+    LOG(THREAD, LOG_INTERP, 1, "fooling cpuid instruction!\n");
+
+    ASSERT(prev != NULL);
+    prev = instr_get_prev_expanded(dcontext, ilist, instr);
+    instr_decode(dcontext, instr);
+    if (!instr_valid(instr))
+        goto cpuid_give_up;
+    loginst(dcontext, 2, prev, "prior to cpuid");
+
+    /* FIXME: maybe should insert code to dispatch on eax, rather than
+     * this hack, which is based on photoshop, which either does
+     * "xor eax,eax" or "xor eax,eax; inc eax"
+     */
+    if (!instr_is_mov_constant(prev, &input)) {
+        /* we only allow inc here */
+        if (instr_get_opcode(prev) != OP_inc)
+            goto cpuid_give_up;
+        op = instr_get_dst(prev, 0);
+        if (!opnd_is_reg(op) || opnd_get_reg(op) != REG_EAX)
+            goto cpuid_give_up;
+        /* now check instr before inc */
+        prev = instr_get_prev(prev);
+        if (!instr_is_mov_constant(prev, &input) || input != 0)
+            goto cpuid_give_up;
+        input = 1;
+        /* now check that mov 0 is into eax */
+    }
+    if (instr_num_dsts(prev) == 0)
+        goto cpuid_give_up;
+    op = instr_get_dst(prev, 0);
+    if (!opnd_is_reg(op) || opnd_get_reg(op) != REG_EAX)
+        goto cpuid_give_up;
+    
+    if (input == 0) {
+        out_eax = CPUID_0_EAX;
+        out_ebx = CPUID_0_EBX;
+        out_ecx = CPUID_0_ECX;
+        out_edx = CPUID_0_EDX;
+    } else {
+        /* 1 or anything higher all return same info */
+        out_eax = CPUID_1_EAX;
+        out_ebx = CPUID_1_EBX;
+        out_ecx = CPUID_1_ECX;
+        out_edx = CPUID_1_EDX;
+    }
+
+    PRE(ilist, instr,
+        INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_EAX),
+                             OPND_CREATE_INT32(out_eax)));
+    PRE(ilist, instr,
+        INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_EBX),
+                             OPND_CREATE_INT32(out_ebx)));
+    PRE(ilist, instr,
+        INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_ECX),
+                             OPND_CREATE_INT32(out_ecx)));
+    PRE(ilist, instr,
+        INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_EDX),
+                             OPND_CREATE_INT32(out_edx)));
+    
+    /* destroy the cpuid instruction */
+    instrlist_remove(ilist, instr);
+    instr_destroy(dcontext, instr);
+    return;
+ cpuid_give_up:
+    LOG(THREAD, LOG_INTERP, 1, "\tcpuid fool: giving up\n");
+    return;
+}
+#endif /* FOOL_CPUID */
+
+static void
+mangle_exit_cti_prefixes(dcontext_t *dcontext, instr_t *instr)
+{
+    uint prefixes = instr_get_prefixes(instr);
+    if (TESTANY(~(PREFIX_JCC_TAKEN|PREFIX_JCC_NOT_TAKEN), prefixes)) {
+        /* Case 8738: while for transparency it would be best to maintain all
+         * prefixes, our patching and other routines make assumptions about
+         * the length of exit ctis.  Plus our elision removes the whole 
+         * instr in any case.
+         */
+        LOG(THREAD, LOG_INTERP, 4,
+            "\tremoving unknown prefixes "PFX" from "PFX"\n",
+            prefixes, instr_get_raw_bits(instr));
+        prefixes &= (PREFIX_JCC_TAKEN|PREFIX_JCC_NOT_TAKEN);
+        instr_set_prefixes(instr, prefixes);
+    }
+}
+
+#ifdef X64
+/* PR 215397: re-relativize rip-relative data addresses */
+static void
+mangle_rel_addr(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr,
+                instr_t *next_instr)
+{
+    uint opc = instr_get_opcode(instr);
+    app_pc tgt;
+    opnd_t dst, src;
+    ASSERT(instr_has_rel_addr_reference(instr));
+    instr_get_rel_addr_target(instr, &tgt);
+    STATS_INC(rip_rel_instrs);
+# ifdef RCT_IND_BRANCH
+    if (TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_call)) ||
+        TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ind_jump))) {
+        /* PR 215408: record addresses taken via rip-relative instrs */
+        rct_add_rip_rel_addr(dcontext, tgt _IF_DEBUG(instr_get_translation(instr)));
+    }
+# endif
+    if (opc == OP_lea) {
+        /* segment overrides are ignored on lea */
+        opnd_t immed;
+        dst = instr_get_dst(instr, 0);
+        src = instr_get_src(instr, 0);
+        ASSERT(opnd_is_reg(dst));
+        ASSERT(opnd_is_rel_addr(src));
+        ASSERT(opnd_get_addr(src) == tgt);
+        /* Replace w/ an absolute immed of the target app address, following Intel
+         * Table 3-59 "64-bit Mode LEA Operation with Address and Operand Size
+         * Attributes" */
+        /* FIXME PR 253446: optimization: we could leave this as rip-rel if it
+         * still reaches from the code cache. */         
+        if (reg_get_size(opnd_get_reg(dst)) == OPSZ_8) {
+            /* PR 253327: there is no explicit addr32 marker; we assume
+             * that decode or the user already zeroed out the top bits
+             * if there was an addr32 prefix byte or the user wants
+             * that effect */
+            immed = OPND_CREATE_INTPTR((ptr_int_t)tgt);
+        } else if (reg_get_size(opnd_get_reg(dst)) == OPSZ_4)
+            immed = OPND_CREATE_INT32((int)(ptr_int_t)tgt);
+        else {
+            ASSERT(reg_get_size(opnd_get_reg(dst)) == OPSZ_2);
+            immed = OPND_CREATE_INT16((short)(ptr_int_t)tgt);
+        }
+        PRE(ilist, instr,
+            INSTR_CREATE_mov_imm(dcontext, dst, immed));
+        instrlist_remove(ilist, instr);
+        instr_destroy(dcontext, instr);
+        STATS_INC(rip_rel_lea);
+    } else {
+        /* PR 251479 will automatically re-relativize if it reaches,
+         * but if it doesn't we need to handle that here (since that
+         * involves an encoding length change, which complicates many
+         * use cases if done at instr encode time).
+         * We don't yet know exactly where we're going to encode this bb,
+         * so we're conservative and check for all reachability from our
+         * heap (assumed to be a single heap: xref PR 215395, and xref
+         * potential secondary code caches PR 253446.
+         */
+        if (!rel32_reachable_from_heap(tgt)) {
+            int si = -1, di = -1;
+            opnd_t relop, newop;
+            bool spill = true;
+            /* FIXME PR 253446: for mbr, should share the xcx spill */
+            reg_id_t scratch_reg = REG_XAX;
+            si = instr_get_rel_addr_src_idx(instr);
+            di = instr_get_rel_addr_dst_idx(instr);
+            if (si >= 0) {
+                relop = instr_get_src(instr, si);
+                ASSERT(di < 0 || opnd_same(relop, instr_get_dst(instr, di)));
+                /* If it's a load (OP_mov_ld, or OP_movzx, etc.), use dead reg */
+                if (instr_num_srcs(instr) == 1 && /* src is the rip-rel opnd */
+                    instr_num_dsts(instr) == 1 && /* only one dest: a register */
+                    opnd_is_reg(instr_get_dst(instr, 0))) {
+                    opnd_size_t sz = opnd_get_size(instr_get_dst(instr, 0));
+                    /* if target is 16 or 8 bit sub-register the whole reg is not dead
+                     * (for 32-bit, top 32 bits are cleared) */
+                    if (sz == OPSZ_4 || sz == OPSZ_8) {
+                        spill = false;
+                        scratch_reg = opnd_get_reg(instr_get_dst(instr, 0));
+                        if (sz == OPSZ_4)
+                            scratch_reg = reg_32_to_64(scratch_reg);
+                        /* we checked all opnds: should not read reg */
+                        ASSERT(!instr_reads_from_reg(instr, scratch_reg));
+                        STATS_INC(rip_rel_unreachable_nospill);
+                    }
+                }
+            } else {
+                relop = instr_get_dst(instr, di);
+            }
+            /* PR 263369: we can't just look for instr_reads_from_reg here since
+             * our no-spill optimization above may miss some writes.
+             */
+            if (spill && instr_uses_reg(instr, scratch_reg)) {
+                /* mbr (for which we'll use xcx once we optimize) should not
+                 * get here: can't use registers (except xsp) */
+                ASSERT(scratch_reg == REG_XAX);
+                do {
+                    scratch_reg++;
+                    ASSERT(scratch_reg <= REG_STOP_64);
+                } while (instr_uses_reg(instr, scratch_reg));
+            }
+            ASSERT(!instr_reads_from_reg(instr, scratch_reg));
+            ASSERT(!spill || !instr_writes_to_reg(instr, scratch_reg));
+            /* FIXME PR 253446: Optimize by looking ahead for dead registers, and
+             * sharing single spill across whole bb, or possibly building local code
+             * cache to avoid unreachability: all depending on how many rip-rel
+             * instrs we see.  We'll watch the stats.
+             */
+            if (spill) {
+                PRE(ilist, instr,
+                    SAVE_TO_DC_OR_TLS(dcontext, 0, scratch_reg, TLS_XAX_SLOT,
+                                      XAX_OFFSET));
+            }
+            PRE(ilist, instr,
+                INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(scratch_reg),
+                                     OPND_CREATE_INTPTR((ptr_int_t)tgt)));
+
+            newop = opnd_create_far_base_disp(opnd_get_segment(relop), scratch_reg,
+                                              REG_NULL, 0, 0, opnd_get_size(relop));
+            if (si >= 0)
+                instr_set_src(instr, si, newop);
+            if (di >= 0)
+                instr_set_dst(instr, di, newop);
+            /* we need the whole spill...restore region to all be marked mangle */
+            instr_set_our_mangling(instr, true);
+            if (spill) {
+                PRE(ilist, next_instr,
+                    instr_create_restore_from_tls(dcontext, scratch_reg, TLS_XAX_SLOT));
+            }
+            STATS_INC(rip_rel_unreachable);
+        }        
+    }
+}
+#endif
+
+/* TOP-LEVEL MANGLE
+ * This routine is responsible for mangling a fragment into the form
+ * we'd like prior to placing it in the code cache
+ * If mangle_calls is false, ignores calls
+ * If record_translation is true, records translation target for each
+ * inserted instr -- but this slows down encoding in current implementation
+ */
+void
+mangle(dcontext_t *dcontext, instrlist_t *ilist, uint flags,
+       bool mangle_calls, bool record_translation)
+{
+    instr_t *instr, *next_instr;
+#ifdef WINDOWS
+    bool ignorable_sysenter = DYNAMO_OPTION(ignore_syscalls) &&
+        DYNAMO_OPTION(ignore_syscalls_follow_sysenter) &&
+        (get_syscall_method() == SYSCALL_METHOD_SYSENTER) &&
+        TEST(FRAG_HAS_SYSCALL, flags);
+#endif
+
+    /* Walk through instr list:
+     * -- convert exit branches to use near_rel form;
+     * -- convert direct calls into 'push %eip', aka return address;
+     * -- convert returns into 'pop %xcx (; add $imm, %xsp)';
+     * -- convert indirect branches into 'save %xcx; lea EA, %xcx';
+     * -- convert indirect calls as a combination of direct call and
+     *    indirect branch conversion;
+     * -- ifdef STEAL_REGISTER, steal edi for our own use. 
+     */
+
+    KSTART(mangling);
+    instrlist_set_our_mangling(ilist, true); /* PR 267260 */
+    for (instr = instrlist_first(ilist);
+         instr != NULL;
+         instr = next_instr) {
+
+        /* don't mangle anything that mangle inserts! */
+        next_instr = instr_get_next(instr);
+
+        if (!instr_opcode_valid(instr))
+            continue;
+
+        if (record_translation) {
+            /* make sure inserted instrs translate to the original instr */
+            instrlist_set_translation_target(ilist, instr_get_raw_bits(instr));
+        }
+
+#ifdef X64
+        if (instr_has_rel_addr_reference(instr))
+            mangle_rel_addr(dcontext, ilist, instr, next_instr);
+#endif
+
+        if (instr_is_exit_cti(instr)) {
+            mangle_exit_cti_prefixes(dcontext, instr);
+
+            /* to avoid reachability problems we convert all
+             * 8-bit-offset jumps that exit the fragment to 32-bit.
+             * Note that data16 jmps are implicitly converted via the
+             * absolute target and loss of prefix info (xref PR 225937).
+             */
+            if (instr_is_cti_short(instr)) {
+                /* convert short jumps */
+                convert_to_near_rel(dcontext, instr);
+            }
+            
+            if (instr_get_opcode(instr) == OP_jmp_far) {
+                /* FIXME: case 6962: we don't support fully; just convert
+                 * to near jmp.
+                 */
+                SYSLOG_INTERNAL_WARNING_ONCE("Encountered a far direct jump");
+                STATS_INC(num_far_dir_jmps);
+                instr_set_opcode(instr, OP_jmp);
+                instr_set_target(instr,
+                                 opnd_create_pc(opnd_get_pc(instr_get_target(instr))));
+                /* doesn't need to be marked as our_mangling */
+            }
+        }
+
+        /* PR 240258: wow64 call* gateway is considered is_syscall */
+        if (instr_is_syscall(instr)) {
+#ifdef WINDOWS
+            /* For XP & 2003, which use sysenter, we process the syscall after all
+             * mangling is completed, since we need to insert a reference to the
+             * post-sysenter instruction. If that instruction is a 'ret', which
+             * we've seen on both os's at multiple patch levels, we'd have a
+             * dangling reference since it's deleted in mangle_return(). To avoid
+             * that case, we defer syscall processing until mangling is completed.
+             */
+            if (!ignorable_sysenter)
+#endif
+                mangle_syscall(dcontext, ilist, flags, instr, next_instr);
+            continue;
+        }
+        else if (instr_is_interrupt(instr)) { /* non-syscall interrupt */
+            mangle_interrupt(dcontext, ilist, instr, next_instr);
+            continue;
+        }
+#ifdef FOOL_CPUID
+        else if (instr_get_opcode(instr) == OP_cpuid) {
+            mangle_cpuid(dcontext, ilist, instr, next_instr);
+            continue;
+        }
+#endif
+
+        if (!instr_is_cti(instr) || !instr_ok_to_mangle(instr)) {
+#ifdef STEAL_REGISTER
+            steal_reg(dcontext, instr, ilist);
+#endif
+            continue;
+        }
+
+#ifdef STEAL_REGISTER
+        if (ilist->flags) {
+            restore_state(dcontext, instr, ilist); /* end of edi calculation */
+        }
+#endif
+
+        if (instr_is_call_direct(instr)) {
+            /* mangle_direct_call may inline a call and remove next_instr, so
+             * it passes us the updated next instr */
+            next_instr =
+                mangle_direct_call(dcontext, ilist, instr, next_instr, mangle_calls);
+        } else if (instr_is_call_indirect(instr)) {
+            mangle_indirect_call(dcontext, ilist, instr, next_instr, mangle_calls, flags);
+        } else if (instr_is_return(instr)) {
+            mangle_return(dcontext, ilist, instr, next_instr, flags);
+        } else if (instr_is_mbr(instr)) {
+            mangle_indirect_jump(dcontext, ilist, instr, next_instr, flags);
+        } else if (instr_get_opcode(instr) == OP_jmp_far) {
+            /* N.B.: we do not support other than flat 0-based CS, DS, SS, and ES.
+             * if the app wants to change segments, we won't actually issue
+             * a segment change, and so will only work properly if the new segment
+             * is also 0-based.  To properly issue new segments, we'd need a special
+             * ibl that ends in a far cti, and all prior address manipulations would
+             * need to be relative to the new segment, w/o messing up current segment.
+             * FIXME: can we do better without too much work?
+             */
+            SYSLOG_INTERNAL_WARNING_ONCE("Encountered a far direct jmp");
+        }
+        /* else nothing to do, e.g. direct branches */
+    }
+
+#ifdef WINDOWS
+    /* Do XP & 2003 ignore-syscalls processing now. */
+    if (ignorable_sysenter) {
+        /* Check for any syscalls and process them. */
+        for (instr = instrlist_first(ilist);
+             instr != NULL;
+             instr = next_instr) {
+            next_instr = instr_get_next(instr);
+            if (instr_opcode_valid(instr) && instr_is_syscall(instr))
+                mangle_syscall(dcontext, ilist, flags, instr, next_instr);
+        }
+    }
+#endif
+    if (record_translation)
+        instrlist_set_translation_target(ilist, NULL);
+    instrlist_set_our_mangling(ilist, false); /* PR 267260 */
+
+    /* The following assertion should be guaranteed by fact that all
+     * blocks end in some kind of branch, and the code above restores
+     * the register state on a branch. */
+    ASSERT(ilist->flags == 0);
+    KSTOP(mangling);
+}
+
+/* END OF CONTROL-FLOW MANGLING ROUTINES
+ *###########################################################################
+ *###########################################################################
+ */
+
+/* SELF-MODIFYING-CODE SANDBOXING
+ *
+ * When we detect it, we take an exit that targets our own routine
+ * fragment_self_write.  Dispatch checks for that target and if it finds it,
+ * it calls that routine, so don't worry about building a bb for it.
+ * Returns false if the bb has invalid instrs in the middle and it should
+ * be rebuilt from scratch.
+ */
+
+#undef SAVE_TO_DC_OR_TLS
+/* PR 244737: x64 uses tls to avoid reachability issues w/ absolute addresses */
+#ifdef X64
+# define SAVE_TO_DC_OR_TLS(dc, reg, tls_offs, dc_offs) \
+    instr_create_save_to_tls((dc), (reg), (tls_offs))
+# define RESTORE_FROM_DC_OR_TLS(dc, reg, tls_offs, dc_offs) \
+    instr_create_restore_from_tls((dc), (reg), (tls_offs))
+#else
+# define SAVE_TO_DC_OR_TLS(dc, reg, tls_offs, dc_offs) \
+    instr_create_save_to_dcontext((dc), (reg), (dc_offs))
+# define RESTORE_FROM_DC_OR_TLS(dc, reg, tls_offs, dc_offs) \
+    instr_create_restore_from_dcontext((dc), (reg), (dc_offs))
+#endif
+
+static void
+sandbox_rep_instr(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr, instr_t *next,
+                  app_pc start_pc, app_pc end_pc /* end is open */)
+{
+    /* put checks before instr, set some reg as a flag, act on it
+     * after instr (even if overwrite self will execute rep to completion)
+     * want to read DF to find direction (0=inc xsi/xdi, 1=dec),
+     * but only way to read is to do a pushf!
+     * Solution: if cld or std right before rep instr, use that info,
+     * otherwise check for BOTH directions!
+     * xcx is a pre-check, xsi/xdi are inc/dec after memory op, so
+     * xdi+xcx*opndsize == instr of NEXT write, so open-ended there:
+     * if DF==0:
+     *   if (xdi < end_pc && xdi+xcx*opndsize > start_pc) => self-write
+     * if DF==1:
+     *   if (xdi > start_pc && xdi-xcx*opndsize > end_pc) => self-write
+     * both:
+     *   if (xdi-xcx*opndsize < end_pc && xdi+xcx*opndsize > start_pc) => self-write
+     * opndsize is 1,2, or 4 => use lea for mul
+     *   lea (xdi,xcx,opndsize),xcx
+     * 
+     *   save flags and xax
+     *   save xbx
+     *   lea (xdi,xcx,opndsize),xbx
+     * if x64 && (start_pc > 4GB || end_pc > 4GB): save xdx
+     * if x64 && start_pc > 4GB: mov start_pc, xdx
+     *   cmp xbx, IF_X64_>4GB_ELSE(xdx, start_pc)
+     *   mov $0,xbx # for if ok
+     *   jle ok # open b/c address of next rep write
+     *   lea (,xcx,opndsize),xbx
+     *   neg xbx # sub does dst - src
+     *   add xdi,xbx
+     * if x64 && end_pc > 4GB: mov end_pc, xdx
+     *   cmp xbx, IF_X64_>4GB_ELSE(xdx, end_pc)
+     *   mov $0,xbx # for if ok
+     *   jge ok    # end is open
+     *   mov $1,xbx
+     * ok:
+     *   restore flags and xax (xax used by stos)
+     * if x64 && (start_pc > 4GB || end_pc > 4GB): restore xdx
+     *   <rep instr> # doesn't use xbx
+     *     (FIXME PR 267764: restore xbx on cxt xl8 if this instr faults)
+     *   mov xbx,xcx # we can use xcx, it's dead since 0 after rep
+     *   restore xbx
+     *   jecxz ok2  # if xbx was 1 we'll fall through and exit
+     *   mov $0,xcx
+     *   jmp <instr after write, flag as INSTR_BRANCH_SELFMOD_EXIT>
+     * ok2:
+     *   <label> # ok2 can't == next, b/c next may be ind br -> mangled w/ instrs
+     *           # inserted before it, so jecxz would target too far
+     */
+    instr_t *ok = INSTR_CREATE_label(dcontext);
+    instr_t *ok2 = INSTR_CREATE_label(dcontext);
+    instr_t *jmp;
+    app_pc after_write;
+    uint opndsize = opnd_size_in_bytes(opnd_get_size(instr_get_dst(instr,0)));
+    uint flags =
+        instr_eflags_to_fragment_eflags(forward_eflags_analysis(dcontext, ilist, next));
+    bool use_tls = IF_X64_ELSE(true, false);
+    DOLOG(3, LOG_INTERP, { loginst(dcontext, 3, instr, "writes memory"); });
+
+    ASSERT(!instr_is_call_indirect(instr)); /* FIXME: can you have REP on on CALL's */
+
+    if (next != NULL) {
+        if (!instr_raw_bits_valid(next)) {
+            /* next must be the final jmp! */
+            ASSERT(instr_is_ubr(next) && instr_get_next(next) == NULL);
+            after_write = opnd_get_pc(instr_get_target(next));
+        } else
+            after_write = instr_get_raw_bits(next);
+    } else {
+        after_write = end_pc;
+    }
+
+    insert_save_eflags(dcontext, ilist, instr, flags, use_tls, !use_tls);
+    PRE(ilist, instr,
+        SAVE_TO_DC_OR_TLS(dcontext, REG_XBX, TLS_XBX_SLOT, XBX_OFFSET));
+    PRE(ilist, instr,
+        INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XBX),
+                         opnd_create_base_disp(REG_XDI, REG_XCX, opndsize, 0, OPSZ_lea)));
+#ifdef X64
+    if ((ptr_uint_t)start_pc > UINT_MAX || (ptr_uint_t)end_pc > UINT_MAX) {
+        PRE(ilist, instr,
+            SAVE_TO_DC_OR_TLS(dcontext, REG_XDX, TLS_XDX_SLOT, XDX_OFFSET));
+    }
+    if ((ptr_uint_t)start_pc > UINT_MAX) {
+        PRE(ilist, instr,
+            INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XDX),
+                                 OPND_CREATE_INTPTR(start_pc)));
+        PRE(ilist, instr,
+            INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_XBX),
+                             opnd_create_reg(REG_XDX)));
+    } else {
+#endif
+        PRE(ilist, instr,
+            INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_XBX),
+                             OPND_CREATE_INT32((int)(ptr_int_t)start_pc)));
+#ifdef X64
+    }
+#endif
+    PRE(ilist, instr,
+        INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XBX), OPND_CREATE_INT32(0)));
+    PRE(ilist, instr,
+        INSTR_CREATE_jcc(dcontext, OP_jle, opnd_create_instr(ok)));
+    PRE(ilist, instr,
+        INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XBX),
+                         opnd_create_base_disp(REG_NULL, REG_XCX, opndsize, 0, OPSZ_lea)));
+    PRE(ilist, instr,
+        INSTR_CREATE_neg(dcontext, opnd_create_reg(REG_XBX)));
+    PRE(ilist, instr,
+        INSTR_CREATE_add(dcontext, opnd_create_reg(REG_XBX),
+                         opnd_create_reg(REG_XDI)));
+#ifdef X64
+    if ((ptr_uint_t)end_pc > UINT_MAX) {
+        PRE(ilist, instr,
+            INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XDX),
+                                 OPND_CREATE_INTPTR(end_pc)));
+        PRE(ilist, instr,
+            INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_XBX),
+                             opnd_create_reg(REG_XDX)));
+    } else {
+#endif
+        PRE(ilist, instr,
+            INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_XBX),
+                             OPND_CREATE_INT32((int)(ptr_int_t)end_pc)));
+#ifdef X64
+    }
+#endif
+    PRE(ilist, instr,
+        INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XBX),
+                             OPND_CREATE_INT32(0)));
+    PRE(ilist, instr,
+        INSTR_CREATE_jcc(dcontext, OP_jge, opnd_create_instr(ok)));
+    PRE(ilist, instr,
+        INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XBX), OPND_CREATE_INT32(1)));
+    PRE(ilist, instr, ok);
+    insert_restore_eflags(dcontext, ilist, instr, flags, use_tls, !use_tls);
+#ifdef X64
+    if ((ptr_uint_t)start_pc > UINT_MAX || (ptr_uint_t)end_pc > UINT_MAX) {
+        PRE(ilist, instr,
+            RESTORE_FROM_DC_OR_TLS(dcontext, REG_XDX, TLS_XDX_SLOT, XDX_OFFSET));
+    }
+#endif
+    /* instr goes here */
+    PRE(ilist, next,
+        INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_XCX), opnd_create_reg(REG_XBX)));
+    PRE(ilist, next,
+        RESTORE_FROM_DC_OR_TLS(dcontext, REG_XBX, TLS_XBX_SLOT, XBX_OFFSET));
+    PRE(ilist, next,
+        INSTR_CREATE_jecxz(dcontext, opnd_create_instr(ok2)));
+    PRE(ilist, next,
+        INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XCX),
+                             OPND_CREATE_INT32(0))); /* on x64 top 32 bits zeroed */
+    jmp = INSTR_CREATE_jmp(dcontext, opnd_create_pc(after_write));
+    instr_branch_set_selfmod_exit(jmp, true);
+    /* an exit cti, not a meta instr */
+    instrlist_preinsert(ilist, next, jmp);
+    PRE(ilist, next, ok2);
+}
+
+static void
+sandbox_write(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr, instr_t *next,
+              opnd_t op, app_pc start_pc, app_pc end_pc /* end is open */)
+{
+    /* can only test for equality w/o modifying flags, so save them 
+     * if (addr < end_pc && addr+opndsize > start_pc) => self-write
+     *   <write memory>
+     *   save flags and xax
+     *   save xbx
+     *   lea memory,xbx
+     * if x64 && (start_pc > 4GB || end_pc > 4GB): save xcx
+     * if x64 && end_pc > 4GB: mov end_pc, xcx
+     *   cmp xbx, IF_X64_>4GB_ELSE(xcx, end_pc)
+     *   jge ok    # end is open
+     *   lea opndsize(xbx),xbx
+     * if x64 && start_pc > 4GB: mov start_pc, xcx
+     *   cmp xbx, IF_X64_>4GB_ELSE(xcx, start_pc)
+     *   jle ok    # open since added size
+     *   restore flags (using xbx) and xax
+     *   restore xbx
+     * if x64 && (start_pc > 4GB || end_pc > 4GB): restore xcx
+     *   jmp <instr after write, flag as INSTR_BRANCH_SELFMOD_EXIT>
+     * ok:
+     *   restore flags and xax
+     *   restore xbx
+     * if x64 && (start_pc > 4GB || end_pc > 4GB): restore xcx
+     */
+    instr_t *ok = INSTR_CREATE_label(dcontext), *jmp;
+    app_pc after_write = NULL;
+    uint opndsize = opnd_size_in_bytes(opnd_get_size(op));
+    uint flags =
+        instr_eflags_to_fragment_eflags(forward_eflags_analysis(dcontext, ilist, next));
+    bool use_tls = IF_X64_ELSE(true, false);
+    DOLOG(3, LOG_INTERP, { loginst(dcontext, 3, instr, "writes memory"); });
+
+    if (next != NULL) {
+        if (!instr_raw_bits_valid(next)) {
+            /* next must be the final artificially added jmp! */
+            ASSERT(instr_is_ubr(next) && instr_get_next(next) == NULL);
+            /* for sure this is the last jmp out, but it
+             * doesn't have to be a direct jmp but instead
+             * it could be the exit branch we add as an
+             * for an indirect call - which is the only ind branch
+             * that writes to memory
+             * CALL* already means that we're leaving the block and it cannot be a selfmod
+             * instruction even though it writes to memory
+             */
+            DOLOG(4, LOG_INTERP, { 
+                loginst(dcontext, 4, next, "next instr");
+            });
+            after_write = opnd_get_pc(instr_get_target(next));
+            LOG(THREAD, LOG_INTERP, 4, "after_write = "PFX" next should be final jmp\n",
+                after_write);
+        } else
+            after_write = instr_get_raw_bits(next);
+    } else {
+        ASSERT_NOT_TESTED();
+        after_write = end_pc;
+    }
+
+    insert_save_eflags(dcontext, ilist, next, flags, use_tls, !use_tls);
+    PRE(ilist, next,
+        SAVE_TO_DC_OR_TLS(dcontext, REG_XBX, TLS_XBX_SLOT, XBX_OFFSET));
+    /* change to OPSZ_lea for lea */
+    opnd_set_size(&op, OPSZ_lea);
+    PRE(ilist, next,
+        INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XBX), op));
+#ifdef X64
+    if ((ptr_uint_t)start_pc > UINT_MAX || (ptr_uint_t)end_pc > UINT_MAX) {
+        PRE(ilist, next,
+            SAVE_TO_DC_OR_TLS(dcontext, REG_XCX, TLS_XCX_SLOT, XCX_OFFSET));
+    }
+    if ((ptr_uint_t)end_pc > UINT_MAX) {
+        PRE(ilist, next,
+            INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XCX),
+                                 OPND_CREATE_INTPTR(end_pc)));
+        PRE(ilist, next,
+            INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_XBX),
+                             opnd_create_reg(REG_XCX)));
+    } else {
+#endif
+        PRE(ilist, next,
+            INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_XBX),
+                             OPND_CREATE_INT32((int)(ptr_int_t)end_pc)));
+#ifdef X64
+    }
+#endif
+    PRE(ilist, next,
+        INSTR_CREATE_jcc(dcontext, OP_jge, opnd_create_instr(ok)));
+    PRE(ilist, next,
+        INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_XBX),
+                         opnd_create_base_disp(REG_XBX, REG_NULL, 0,
+                                               opndsize, OPSZ_lea)));
+#ifdef X64
+    if ((ptr_uint_t)start_pc > UINT_MAX) {
+        PRE(ilist, next,
+            INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XCX),
+                                 OPND_CREATE_INTPTR(start_pc)));
+        PRE(ilist, next,
+            INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_XBX),
+                             opnd_create_reg(REG_XCX)));
+    } else {
+#endif
+        PRE(ilist, next,
+            INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_XBX),
+                             OPND_CREATE_INT32((int)(ptr_int_t)start_pc)));
+#ifdef X64
+    }
+#endif
+    PRE(ilist, next,
+        INSTR_CREATE_jcc(dcontext, OP_jle, opnd_create_instr(ok)));
+    insert_restore_eflags(dcontext, ilist, next, flags, use_tls, !use_tls);
+    PRE(ilist, next,
+        RESTORE_FROM_DC_OR_TLS(dcontext, REG_XBX, TLS_XBX_SLOT, XBX_OFFSET));
+#ifdef X64
+    if ((ptr_uint_t)start_pc > UINT_MAX || (ptr_uint_t)end_pc > UINT_MAX) {
+        PRE(ilist, next,
+            RESTORE_FROM_DC_OR_TLS(dcontext, REG_XCX, TLS_XCX_SLOT, XCX_OFFSET));
+    }
+#endif
+    jmp = INSTR_CREATE_jmp(dcontext, opnd_create_pc(after_write));
+    instr_branch_set_selfmod_exit(jmp, true);
+    /* an exit cti, not a meta instr */
+    instrlist_preinsert(ilist, next, jmp);
+    PRE(ilist, next, ok);
+    insert_restore_eflags(dcontext, ilist, next, flags, use_tls, !use_tls);
+    PRE(ilist, next,
+        RESTORE_FROM_DC_OR_TLS(dcontext, REG_XBX, TLS_XBX_SLOT, XBX_OFFSET));
+#ifdef X64
+    if ((ptr_uint_t)start_pc > UINT_MAX || (ptr_uint_t)end_pc > UINT_MAX) {
+        PRE(ilist, next,
+            RESTORE_FROM_DC_OR_TLS(dcontext, REG_XCX, TLS_XCX_SLOT, XCX_OFFSET));
+    }
+#endif
+}
+
+static bool
+sandbox_top_of_bb_check_s2ro(dcontext_t *dcontext, app_pc start_pc)
+{
+    return (DYNAMO_OPTION(sandbox2ro_threshold) > 0 &&
+            /* we can't make stack regions ro so don't put in the instrumentation */
+            !is_address_on_stack(dcontext, start_pc) &&
+            /* case 9098 we don't want to ever make RO untrackable driver areas */
+            !is_driver_address(start_pc));
+}
+
+static void
+sandbox_top_of_bb(dcontext_t *dcontext, instrlist_t *ilist,
+                  bool s2ro, uint flags,
+                  app_pc start_pc, app_pc end_pc, /* end is open */
+                  bool for_cache,
+                  /* for obtaining the two patch locations: */
+                  patch_list_t *patchlist,
+                  cache_pc *copy_start_loc, cache_pc *copy_end_loc)
+{
+    /* add check at top of ilist that compares actual app instructions versus
+     * copy we saved, stored in cache right after fragment itself.  leave its
+     * start address blank here, will be touched up after emitting this ilist.
+     *
+     * FIXME case 8165/PR 212600: optimize this: move reg restores to
+     * custom fcache_return, use cmpsd instead of cmpsb, etc.
+     *
+     * if eflags live entering this bb:
+     *   save xax
+     *   lahf
+     *   seto  %al
+     * endif
+     * if (-sandbox2ro_threshold > 0)
+     *  if x64: save xcx
+     *     incl  &vm_area_t->exec_count (for x64, via xcx)
+     *     cmp   sandbox2ro_threshold, vm_area_t->exec_count (for x64, via xcx)
+     *  if eflags live entering this bb, or x64:
+     *     jl    past_threshold
+     *   if x64: restore xcx
+     *   if eflags live entering this bb:
+     *     jmp restore_eflags_and_exit
+     *   else
+     *     jmp   start_pc marked as selfmod exit
+     *   endif
+     *   past_threshold:
+     *  else
+     *     jge   start_pc marked as selfmod exit
+     *  endif
+     * endif
+     * if (-sandbox2ro_threshold == 0) && !x64)
+     *   save xcx
+     * endif
+     *   save xsi
+     *   save xdi
+     * if stats:
+     *   inc num_sandbox_execs stat (for x64, via xsi)
+     * endif
+     *   mov start_pc,xsi
+     *   mov copy_start_pc,xdi  # 1 opcode byte, then offset
+     *       # => patch point 1
+     *   cmpsb
+     * if copy_size > 1 # not an opt: for correctness: if "repe cmpsb" has xcx==0, it
+     *                  # doesn't touch eflags and we treat cmp results as cmpsb results
+     *     jne check_results
+     *   if x64 && start_pc > 4GB
+     *     mov start_pc, xcx
+     *     cmp xsi, xcx
+     *   else
+     *     cmp xsi, start_pc
+     *   endif
+     *     mov copy_size-1, xcx
+     *     jge forward
+     *     mov copy_end_pc, xdi
+     *         # => patch point 2
+     *     mov end_pc, xsi
+     *   forward:
+     *     repe cmpsb
+     *   check_results:
+     *     restore xcx
+     *     restore xsi
+     *     restore xdi
+     * endif # copy_size > 1
+     * if eflags live:
+     *   je start_bb
+     *  restore_eflags_and_exit:
+     *   add   $0x7f,%al
+     *   sahf
+     *   restore xax
+     *   jmp start_pc marked as selfmod exit
+     * else
+     *   jne start_pc marked as selfmod exit
+     * endif
+     * start_bb:
+     * if eflags live:
+     *   add   $0x7f,%al
+     *   sahf
+     *   restore xax
+     * endif
+     */
+    instr_t *instr, *jmp;
+    instr_t *restore_eflags_and_exit = NULL;
+    bool use_tls = IF_X64_ELSE(true, false);
+    bool saved_xcx = false;
+
+    instr = instrlist_first_expanded(dcontext, ilist);
+
+    insert_save_eflags(dcontext, ilist, instr, flags, use_tls, !use_tls);
+
+    if (s2ro) {
+        /* It's difficult to use lea/jecxz here as we want to use a shared
+         * counter but no lock, and thus need a relative comparison, while
+         * lea/jecxz can only do an exact comparison.  We could be exact by
+         * having a separate counter per (private) fragment but by spilling
+         * eflags we can inc memory, making the scheme here not inefficient.
+         */
+        uint thresh = DYNAMO_OPTION(sandbox2ro_threshold);
+        uint *counter;
+        if (for_cache)
+            counter = get_selfmod_exec_counter(start_pc);
+        else {
+            /* Won't find exec area since not a real fragment (probably
+             * a recreation post-flush).  Won't execute, so NULL is fine.
+             */
+            counter = NULL;
+        }
+#ifdef X64
+        PRE(ilist, instr,
+            SAVE_TO_DC_OR_TLS(dcontext, REG_XCX, TLS_XCX_SLOT, XCX_OFFSET));
+        saved_xcx = true;
+        PRE(ilist, instr,
+            INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XCX),
+                                 OPND_CREATE_INTPTR(counter)));
+        PRE(ilist, instr,
+            INSTR_CREATE_inc(dcontext, OPND_CREATE_MEM32(REG_XCX, 0)));
+        PRE(ilist, instr,
+            INSTR_CREATE_cmp(dcontext, OPND_CREATE_MEM32(REG_XCX, 0),
+                             OPND_CREATE_INT_32OR8((int)thresh)));
+#else
+        PRE(ilist, instr,
+            INSTR_CREATE_inc(dcontext, OPND_CREATE_ABSMEM(counter, OPSZ_4)));
+        PRE(ilist, instr,
+            INSTR_CREATE_cmp(dcontext,
+                             OPND_CREATE_ABSMEM(counter, OPSZ_4),
+                             OPND_CREATE_INT_32OR8(thresh)));
+#endif
+        if (TEST(FRAG_WRITES_EFLAGS_6, flags) IF_X64(&& false)) {
+            jmp = INSTR_CREATE_jcc(dcontext, OP_jge, opnd_create_pc(start_pc));
+            instr_branch_set_selfmod_exit(jmp, true);
+            /* an exit cti, not a meta instr */
+            instrlist_preinsert(ilist, instr, jmp);
+        } else {
+            instr_t *past_threshold = INSTR_CREATE_label(dcontext);
+            PRE(ilist, instr,
+                INSTR_CREATE_jcc_short(dcontext, OP_jl_short,
+                                       opnd_create_instr(past_threshold)));
+#ifdef X64
+            PRE(ilist, instr,
+                RESTORE_FROM_DC_OR_TLS(dcontext, REG_XCX, TLS_XCX_SLOT, XCX_OFFSET));
+#endif
+            if (!TEST(FRAG_WRITES_EFLAGS_6, flags)) {
+                ASSERT(restore_eflags_and_exit == NULL);
+                restore_eflags_and_exit = INSTR_CREATE_label(dcontext);
+                PRE(ilist, instr, INSTR_CREATE_jmp
+                    (dcontext, opnd_create_instr(restore_eflags_and_exit)));
+            }
+#ifdef X64
+            else {
+                jmp = INSTR_CREATE_jmp(dcontext, opnd_create_pc(start_pc));
+                instr_branch_set_selfmod_exit(jmp, true);
+                /* an exit cti, not a meta instr */
+                instrlist_preinsert(ilist, instr, jmp);
+            }
+#endif
+            PRE(ilist, instr, past_threshold);
+        }
+    }
+
+    if (!saved_xcx) {
+        PRE(ilist, instr,
+            SAVE_TO_DC_OR_TLS(dcontext, REG_XCX, TLS_XCX_SLOT, XCX_OFFSET));
+    }
+    PRE(ilist, instr,
+        SAVE_TO_DC_OR_TLS(dcontext, REG_XSI, TLS_XBX_SLOT, XSI_OFFSET));
+    PRE(ilist, instr,
+        SAVE_TO_DC_OR_TLS(dcontext, REG_XDI, TLS_XDX_SLOT, XDI_OFFSET));
+    DOSTATS({
+        if (GLOBAL_STATS_ON()) {
+            /* We only do global inc, not bothering w/ thread-private stats.
+             * We don't care about races: ballpark figure is good enough.
+             * We could do a direct inc of memory for 32-bit.
+             */
+            PRE(ilist, instr, INSTR_CREATE_mov_imm
+                (dcontext, opnd_create_reg(REG_XSI),
+                 OPND_CREATE_INTPTR(GLOBAL_STAT_ADDR(num_sandbox_execs))));
+            PRE(ilist, instr, INSTR_CREATE_inc
+                (dcontext, opnd_create_base_disp(REG_XSI, REG_NULL, 0, 0, OPSZ_STATS)));
+         }
+    });
+    PRE(ilist, instr,
+        INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XSI),
+                             OPND_CREATE_INTPTR(start_pc)));
+    PRE(ilist, instr,
+        INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XDI),
+                             /* will become copy start */
+                             OPND_CREATE_INTPTR(start_pc)));
+    if (patchlist != NULL) {
+        ASSERT(copy_start_loc != NULL);
+        add_patch_marker(patchlist, instr_get_prev(instr), PATCH_ASSEMBLE_ABSOLUTE,
+                         -(short)sizeof(cache_pc), (ptr_uint_t*)copy_start_loc);
+    }
+    PRE(ilist, instr, INSTR_CREATE_cmps_1(dcontext));
+    /* For a 1-byte copy size we cannot use "repe cmpsb" as it won't
+     * touch eflags and we'll treat the cmp results as cmpsb results, which
+     * doesn't work (cmp will never be equal)
+     */
+    if (end_pc - start_pc > 1) {
+        instr_t *forward = INSTR_CREATE_label(dcontext);
+        instr_t *check_results = INSTR_CREATE_label(dcontext);
+        PRE(ilist, instr,
+            INSTR_CREATE_jcc(dcontext, OP_jne, opnd_create_instr(check_results)));
+#ifdef X64
+        if ((ptr_uint_t)start_pc > UINT_MAX) {
+            PRE(ilist, instr,
+                INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XCX),
+                                     OPND_CREATE_INTPTR(start_pc)));
+            PRE(ilist, instr,
+                INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_XSI),
+                                 opnd_create_reg(REG_XCX)));
+        } else {
+#endif
+            PRE(ilist, instr,
+                INSTR_CREATE_cmp(dcontext, opnd_create_reg(REG_XSI),
+                                 OPND_CREATE_INT32((int)(ptr_int_t)start_pc)));
+#ifdef X64
+        }
+#endif
+        PRE(ilist, instr,
+            INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XCX),
+                                 OPND_CREATE_INTPTR(end_pc - (start_pc + 1))));
+        PRE(ilist, instr,
+            INSTR_CREATE_jcc(dcontext, OP_jge, opnd_create_instr(forward)));
+        PRE(ilist, instr,
+            INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XDI),
+                                 /* will become copy end */
+                                 OPND_CREATE_INTPTR(end_pc)));
+        if (patchlist != NULL) {
+            ASSERT(copy_end_loc != NULL);
+            add_patch_marker(patchlist, instr_get_prev(instr), PATCH_ASSEMBLE_ABSOLUTE,
+                             -(short)sizeof(cache_pc), (ptr_uint_t*)copy_end_loc);
+        }
+        PRE(ilist, instr,
+            INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_XSI),
+                                 OPND_CREATE_INTPTR(end_pc)));
+        PRE(ilist, instr, forward);
+        PRE(ilist, instr, INSTR_CREATE_rep_cmps_1(dcontext));
+        PRE(ilist, instr, check_results);
+        PRE(ilist, instr,
+            RESTORE_FROM_DC_OR_TLS(dcontext, REG_XCX, TLS_XCX_SLOT, XCX_OFFSET));
+        PRE(ilist, instr,
+            RESTORE_FROM_DC_OR_TLS(dcontext, REG_XSI, TLS_XBX_SLOT, XSI_OFFSET));
+        PRE(ilist, instr,
+            RESTORE_FROM_DC_OR_TLS(dcontext, REG_XDI, TLS_XDX_SLOT, XDI_OFFSET));
+    }
+    if (!TEST(FRAG_WRITES_EFLAGS_6, flags)) {
+        instr_t *start_bb = INSTR_CREATE_label(dcontext);
+        PRE(ilist, instr,
+            INSTR_CREATE_jcc(dcontext, OP_je, opnd_create_instr(start_bb)));
+        if (restore_eflags_and_exit != NULL) /* somebody needs this label */
+            PRE(ilist, instr, restore_eflags_and_exit);
+        insert_restore_eflags(dcontext, ilist, instr, flags, use_tls, !use_tls);
+        jmp = INSTR_CREATE_jmp(dcontext, opnd_create_pc(start_pc));
+        instr_branch_set_selfmod_exit(jmp, true);
+        /* an exit cti, not a meta instr */
+        instrlist_preinsert(ilist, instr, jmp);
+        PRE(ilist, instr, start_bb);
+    } else {
+        jmp = INSTR_CREATE_jcc(dcontext, OP_jne, opnd_create_pc(start_pc));
+        instr_branch_set_selfmod_exit(jmp, true);
+        /* an exit cti, not a meta instr */
+        instrlist_preinsert(ilist, instr, jmp);
+    }
+    insert_restore_eflags(dcontext, ilist, instr, flags, use_tls, !use_tls);
+    /* fall-through to bb start */
+}
+
+/* returns false if failed to add sandboxing b/c of a problematic ilist --
+ * invalid instrs, elided ctis, etc.
+ */
+bool
+insert_selfmod_sandbox(dcontext_t *dcontext, instrlist_t *ilist, uint flags,
+                       app_pc start_pc, app_pc end_pc, /* end is open */
+                       bool record_translation, bool for_cache)
+{
+    instr_t *instr, *next;
+
+    if (!INTERNAL_OPTION(cache_consistency))
+        return true; /* nothing to do */
+
+    /* this code assumes bb covers single, contiguous region */
+    ASSERT((flags & FRAG_HAS_DIRECT_CTI) == 0);
+
+    /* store first instr so loop below will skip top check */
+    instr = instrlist_first_expanded(dcontext, ilist);
+    instrlist_set_our_mangling(ilist, true); /* PR 267260 */
+    if (record_translation) {
+        /* make sure inserted instrs translate to the proper original instr */
+        instrlist_set_translation_target(ilist, instr_get_raw_bits(instr));
+    }
+
+    sandbox_top_of_bb(dcontext, ilist,
+                      sandbox_top_of_bb_check_s2ro(dcontext, start_pc),
+                      flags, start_pc, end_pc, for_cache,
+                      NULL, NULL, NULL);
+
+    if (INTERNAL_OPTION(sandbox_writes)) {
+        for (; instr != NULL; instr = next) {
+            int i, opcode;
+            opnd_t op;
+
+            opcode = instr_get_opcode(instr);
+            if (!instr_valid(instr)) {
+                /* invalid instr -- best to truncate block here, easiest way
+                 * to do that and get all flags right is to re-build it,
+                 * but this time we'll use full decode so we'll avoid the discrepancy
+                 * between fast and full decode on invalid instr detection.
+                 */
+                if (record_translation)
+                    instrlist_set_translation_target(ilist, NULL);
+                instrlist_set_our_mangling(ilist, false); /* PR 267260 */
+                return false;
+            }
+        
+            /* don't mangle anything that mangle inserts! */
+            next = instr_get_next_expanded(dcontext, ilist, instr);
+            if (!instr_ok_to_mangle(instr))
+                continue;
+            if (record_translation) {
+                /* make sure inserted instrs translate to the proper original instr */
+                instrlist_set_translation_target(ilist, instr_get_raw_bits(instr));
+            }
+
+            if (opcode == OP_rep_ins || opcode == OP_rep_movs || opcode == OP_rep_stos) {
+                sandbox_rep_instr(dcontext, ilist, instr, next, start_pc, end_pc);
+                continue;
+            }
+
+            /* FIXME case 8165: optimize for multiple push/pop */
+            for (i=0; i<instr_num_dsts(instr); i++) {
+                op = instr_get_dst(instr, i);
+                if (opnd_is_memory_reference(op)) {
+                    /* ignore CALL* since last anyways */
+                    if (instr_is_call_indirect(instr)) {
+                        ASSERT(next != NULL && !instr_raw_bits_valid(next));
+                        /* FIXME case 8165: why do we ever care about the last
+                         * instruction modifying anything? 
+                         */
+                        /* conversion of IAT calls (but not elision)
+                         * transforms this into a direct CALL,
+                         * in that case 'next' is a direct jmp
+                         * fall through, so has no exit flags
+                         */
+                        ASSERT(TEST(INSTR_CALL_EXIT, instr_exit_branch_type(next)) || 
+                               (DYNAMO_OPTION(IAT_convert) && 
+                                TEST(INSTR_IND_CALL_DIRECT, instr->flags)));
+
+                        
+                        LOG(THREAD, LOG_INTERP, 3, " ignoring CALL* at end of fragment\n");
+                        /* This test could be done outside of this loop on
+                         * destinations, but since it is rare it is faster
+                         * to do it here.  Using continue instead of break in case
+                         * it gets moved out.
+                         */
+                        continue;
+                    }
+                    sandbox_write(dcontext, ilist, instr, next, op, start_pc, end_pc);
+                }
+            }
+        }
+    }
+    if (record_translation)
+        instrlist_set_translation_target(ilist, NULL);
+    instrlist_set_our_mangling(ilist, false); /* PR 267260 */
+    return true;
+}
+
+/* Offsets within selfmod sandbox top-of-bb code that we patch once
+ * the code is emitted, as the values depend on the emitted address.
+ * These vary by whether sandbox_top_of_bb_check_s2ro() and whether
+ * eflags are not written, all written, or just OF is written.
+ * For the copy_size == 1 variation, we simply ignore the 2nd patch point.
+ */
+static bool selfmod_s2ro[] = { false, true };
+static uint selfmod_eflags[] = { FRAG_WRITES_EFLAGS_6, FRAG_WRITES_EFLAGS_OF, 0 };
+#define SELFMOD_NUM_S2RO   (sizeof(selfmod_s2ro)/sizeof(selfmod_s2ro[0]))
+#define SELFMOD_NUM_EFLAGS (sizeof(selfmod_eflags)/sizeof(selfmod_eflags[0]))
+#ifdef X64 /* additional complexity: start_pc > 4GB? */
+static app_pc selfmod_gt4G[] = { NULL, (app_pc)(POINTER_MAX-2)/*so end can be +2*/ };
+# define SELFMOD_NUM_GT4G   (sizeof(selfmod_gt4G)/sizeof(selfmod_gt4G[0]))
+#endif
+uint selfmod_copy_start_offs[SELFMOD_NUM_S2RO][SELFMOD_NUM_EFLAGS]
+    IF_X64([SELFMOD_NUM_GT4G]);
+uint selfmod_copy_end_offs[SELFMOD_NUM_S2RO][SELFMOD_NUM_EFLAGS]
+    IF_X64([SELFMOD_NUM_GT4G]);
+
+void
+set_selfmod_sandbox_offsets(dcontext_t *dcontext)
+{
+    int i, j;
+#ifdef X64
+    int k;
+#endif
+    instrlist_t ilist;
+    patch_list_t patch;
+    static byte buf[256];
+    uint len;
+    /* We assume this is called at init, when .data is +w and we need no
+     * synch on accessing buf */
+    ASSERT(!dynamo_initialized);
+    for (i = 0; i < SELFMOD_NUM_S2RO; i++) {
+        for (j = 0; j < SELFMOD_NUM_EFLAGS; j++) {
+#ifdef X64
+            for (k = 0; k < SELFMOD_NUM_GT4G; k++) {
+#endif
+                cache_pc start_pc, end_pc;
+                app_pc app_start;
+                instrlist_init(&ilist);
+                /* sandbox_top_of_bb assumes there's an instr there */
+                instrlist_append(&ilist, INSTR_CREATE_label(dcontext));
+                init_patch_list(&patch, PATCH_TYPE_ABSOLUTE);
+                app_start = IF_X64_ELSE(selfmod_gt4G[k], NULL);
+                sandbox_top_of_bb(dcontext, &ilist,
+                                  selfmod_s2ro[i], selfmod_eflags[j],
+                                  /* we must have a >1-byte region to get
+                                   * both patch points */
+                                  app_start, app_start + 2, false,
+                                  &patch, &start_pc, &end_pc);
+                len = encode_with_patch_list(dcontext, &patch, &ilist, buf);
+                ASSERT(len < BUFFER_SIZE_BYTES(buf));
+                IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(start_pc - buf)));
+                selfmod_copy_start_offs[i][j]IF_X64([k]) = (uint) (start_pc - buf);
+                IF_X64(ASSERT(CHECK_TRUNCATE_TYPE_uint(end_pc - buf)));
+                selfmod_copy_end_offs[i][j]IF_X64([k]) = (uint) (end_pc - buf);
+                LOG(THREAD, LOG_EMIT, 3, "selfmod offs %d %d"IF_X64(" %d")": %u %u\n",
+                    i, j, IF_X64_(k)
+                    selfmod_copy_start_offs[i][j]IF_X64([k]),
+                    selfmod_copy_end_offs[i][j]IF_X64([k]));
+                /* free the instrlist_t elements */
+                instrlist_clear(dcontext, &ilist);
+#ifdef X64
+            }
+#endif
+        }
+    }
+}
+
+void
+finalize_selfmod_sandbox(dcontext_t *dcontext, fragment_t *f)
+{
+    cache_pc copy_pc = FRAGMENT_SELFMOD_COPY_PC(f);
+    byte *pc;
+    int i, j;
+#ifdef X64
+    int k = ((ptr_uint_t)f->tag) > UINT_MAX ? 1 : 0;
+#endif
+    i = (sandbox_top_of_bb_check_s2ro(dcontext, f->tag)) ? 1 : 0;
+    j = (TEST(FRAG_WRITES_EFLAGS_6, f->flags) ? 0 :
+         (TEST(FRAG_WRITES_EFLAGS_OF, f->flags) ? 1 : 2));
+    pc = FCACHE_ENTRY_PC(f) + selfmod_copy_start_offs[i][j]IF_X64([k]);
+    *((cache_pc*)pc) = copy_pc;
+    if (FRAGMENT_SELFMOD_COPY_SIZE(f) - sizeof(uint) > 1) {
+        pc = FCACHE_ENTRY_PC(f) + selfmod_copy_end_offs[i][j]IF_X64([k]);
+        /* subtract the size itself, stored at the end of the copy */
+        *((cache_pc*)pc) = (copy_pc + FRAGMENT_SELFMOD_COPY_SIZE(f) - sizeof(uint));
+    } /* else, no 2nd patch point */
+}
diff --git a/core/x86/optimize.c b/core/x86/optimize.c
new file mode 100644
index 000000000..2c380c88e
--- /dev/null
+++ b/core/x86/optimize.c
@@ -0,0 +1,4686 @@
+/* **********************************************************
+ * Copyright (c) 2002-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+
+/* optimize.c
+ *
+ * routines to support optimization of traces
+ * (old offline optimization stuff is in mangle.c)
+ */
+
+#include "globals.h"   /* just to disable warning C4206 about an empty file */
+
+#ifdef INTERNAL /* around whole file */
+
+#include "globals.h"
+#include "instrlist.h"
+#include "arch.h"
+#include "instr.h"
+#include "instr_create.h"
+#include "decode.h"
+#include "decode_fast.h"
+#include "fragment.h"
+#include "disassemble.h"
+#include "proc.h"
+#include <string.h> /* for memset */
+
+/* IMPORTANT INSTRUCTIONS FOR WRITING OPTIMIZATIONS:
+ * 
+ * 1) can assume that all instructions are fully decoded -- that is,
+ *    instr_operands_valid(instr) will return true
+ * 2) optimizations MUST BE DETERMINISTIC!  they are re-executed to
+ *    reconstruct the Pc (and in the future the rest of the machine state,
+ *    hopefully) on exceptions/signals
+ */
+
+/****************************************************************************/
+/* forward declarations */
+/* if these are useful elsewhere, un-static them */
+
+/* optimizations */
+static void instr_counts(dcontext_t *dcontext, app_pc tag, instrlist_t *trace, bool pre);
+static void constant_propagation(dcontext_t *dcontext, app_pc tag, instrlist_t *trace);
+static void call_return_matching(dcontext_t *dcontext, app_pc tag, instrlist_t *trace);
+static void remove_unnecessary_zeroing(dcontext_t *dcontext, app_pc tag,
+                                       instrlist_t *trace);
+static void stack_adjust_combiner(dcontext_t *dcontext, app_pc tag, instrlist_t *trace);
+
+static void prefetch_optimize_trace(dcontext_t *dcontext,
+                                    app_pc tag, instrlist_t *trace);
+void remove_redundant_loads(dcontext_t *dcontext, app_pc tag,
+                            instrlist_t *trace);
+
+static void peephole_optimize(dcontext_t *dcontext, app_pc tag, instrlist_t *trace);
+
+static void identify_for_loop(dcontext_t *dcontext,
+                              app_pc tag, instrlist_t *trace);
+static void unroll_loops(dcontext_t *dcontext, app_pc tag, instrlist_t *trace);
+#ifdef IA32_ON_IA64
+static void test_i64(dcontext_t *dcontext, app_pc tag, instrlist_t *trace);
+#endif
+
+/* utility routines */
+bool is_store_to_ecxoff(dcontext_t *dcontext, instr_t *inst);
+bool is_load_from_ecxoff(dcontext_t *dcontext, instr_t *inst);
+bool opnd_is_constant_address(opnd_t address);
+static bool is_zeroing_instr(instr_t *inst);
+static bool replace_inc_with_add(dcontext_t *dcontext, instr_t *inst,
+                                 instrlist_t *trace);
+static bool safe_write(instr_t *mem_writer);
+static bool instruction_affects_mem_access(instr_t *instr,opnd_t mem_access);
+static bool is_dead_register(reg_id_t reg,instr_t *where);
+static reg_id_t find_dead_register_across_instrs(instr_t *start,instr_t *end);
+static bool is_nop(instr_t *inst);
+void remove_inst(dcontext_t *dcontext, instrlist_t *ilist, instr_t *inst);
+
+/* for using a 24 entry bool array to represent some property about 
+ * about normal registers and sub register (eax -> dl) */
+/* propagates the value into all sub registers, doesn't propagate up 
+ * into enclosing registers, index value is checked for bounds */
+static void propagate_down(bool *reg_rep, int index, bool value);
+/* checks the 24 entry array and returns true if it and all sub
+ * registers of the index are true and 0 <= index < 24 */
+static bool check_down(bool *reg_rep, int index);
+
+/* given a cbr, finds the previous instr that writes the flag the cbr reads */
+static instr_t *get_decision_instr(instr_t *jmp);
+
+/* exported utility routines */
+void loginst(dcontext_t *dcontext, uint level, instr_t *instr, char *string);
+void logopnd(dcontext_t *dcontext, uint level, opnd_t opnd, char *string);
+void logtrace(dcontext_t *dcontext, uint level, instrlist_t *trace, char *string);
+
+
+
+
+/****************************************************************************/
+/* master routine */
+
+void 
+optimize_trace(dcontext_t *dcontext, app_pc tag, instrlist_t *trace)
+{
+    /* we have un-truncation-check 32-bit casts for opnd_get_immed_int(), for
+     * one thing, here and in loadtoconst.c */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+
+    /* FIXME: this routine is of course not in its final form
+     * we are still playing with different optimizations
+     */
+
+    /* all opts want to expand all bundles and many want cti info including instr_t
+     * targets, so we go ahead and do that up front
+     */
+    instrlist_decode_cti(dcontext, trace);
+
+#ifdef DEBUG
+    LOG(THREAD, LOG_OPTS, 3, "\noptimize_trace ******************\n");
+    LOG(THREAD, LOG_OPTS, 3, "\nbefore optimization:\n");
+
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_OPTS) != 0)
+        instrlist_disassemble(dcontext, tag, trace, THREAD);
+
+#endif
+
+    if (dynamo_options.instr_counts) {
+        instr_counts(dcontext, tag, trace, true);
+    }
+
+    if (dynamo_options.call_return_matching) {
+        call_return_matching(dcontext, tag, trace);
+    }
+
+    if (dynamo_options.unroll_loops) {
+        unroll_loops(dcontext, tag, trace);
+    }
+
+    if (dynamo_options.vectorize) {
+        identify_for_loop(dcontext, tag, trace);
+    }
+
+    if (dynamo_options.prefetch) {
+        prefetch_optimize_trace(dcontext, tag, trace);
+    }
+
+    if (dynamo_options.rlr) {
+        remove_redundant_loads(dcontext, tag, trace);
+    }
+
+    if (dynamo_options.remove_unnecessary_zeroing) {
+        remove_unnecessary_zeroing(dcontext, tag, trace);
+    }
+
+    if (dynamo_options.constant_prop) {
+        constant_propagation(dcontext, tag, trace);
+    }
+
+    if (dynamo_options.remove_dead_code) {
+        remove_dead_code(dcontext, tag, trace);
+    }
+
+    if (dynamo_options.stack_adjust) {
+        stack_adjust_combiner(dcontext, tag, trace);
+    }
+
+    if (dynamo_options.peephole) {
+        peephole_optimize(dcontext, tag, trace);
+    }
+
+#ifdef IA32_ON_IA64
+    if (dynamo_options.test_i64) {
+        test_i64(dcontext, tag, trace);
+    }
+#endif
+
+    if (dynamo_options.instr_counts) {
+        instr_counts(dcontext, tag, trace, false);
+    }
+    
+#ifdef DEBUG
+    LOG(THREAD, LOG_OPTS, 3, "\nafter optimization:\n");
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_OPTS) != 0)
+        instrlist_disassemble(dcontext, tag, trace, THREAD);
+#endif
+}
+
+#ifdef DEBUG
+static struct {
+    /* rlr */
+    uint loads_removed_from_loads;
+    uint loads_removed_from_stores;
+    uint ctis_in_load_removal;
+    int reg_overwritten;
+    int val_saved_in_dead_reg;
+    uint loads_examined;
+    /* inc->add */
+    int incs_examined;
+    int incs_replaced;
+    /* unrolling */
+    int loops_unrolled;
+    // spill_xmm
+    int vals_spilled_to_xmm;
+    int loads_replaced_by_xmm;
+    int xmm_saves_to_mem;
+    int stores_replaced_by_xmm;
+    int xmm_restored_from_memory;
+    int xmm_mov_to_dead_reg;
+    int loadstore_combos_replaced_by_xmm;
+    int xmm_traces;
+    int mmx_traces;
+    /* constant propagation */
+    int num_instrs_simplified;
+    int num_fail_simplified;
+    int num_opnds_simplified;
+    int num_const_add_const_mem;
+    int num_jmps_simplified;
+    int num_jecxz_instrs_removed;
+    /* remove dead loads */
+    int dead_loads_removed;
+    /* remove unnecssary XOR zeroing */
+    int xors_removed;
+    /* stack adjustment combining */
+    int num_stack_adjust_removed;
+    /* instr_counts */
+    int pre_num_instrs_seen;
+    int pre_num_jmps_seen;
+    int post_num_instrs_seen;
+    int post_num_jmps_seen;
+    /* call return matching */
+    int num_returns_removed;
+    int num_return_instrs_removed;
+#ifdef IA32_ON_IA64
+    bool i64_test;
+    int ia64_num_entries;
+#endif
+} opt_stats_t;
+
+/*this function is called when dynamo exits. prints stats for any optimization
+  that wants to keep them in opt_stats_t and put appropriate code below*/
+
+void
+print_optimization_stats()
+{
+    if (dynamo_options.rlr) {
+        uint top, bottom;
+        LOG(GLOBAL, LOG_OPTS, 1, 
+            "%u loads examined for rlr\n", opt_stats_t.loads_examined);
+        divide_uint64_print(opt_stats_t.loads_removed_from_stores +
+                            opt_stats_t.loads_removed_from_loads, 
+                            opt_stats_t.loads_examined,
+                            true, 2, &top, &bottom); 
+        LOG(GLOBAL, LOG_OPTS,1,"%u.%.2u%% of examined loads removed\n",
+            top, bottom);
+        divide_uint64_print(opt_stats_t.ctis_in_load_removal,
+                            opt_stats_t.loads_removed_from_loads +
+                            opt_stats_t.loads_removed_from_stores,
+                            false, 4, &top, &bottom);
+        LOG(GLOBAL, LOG_OPTS, 1,"%u loads removed from loads\n"
+            "%u loads removed from stores\n"
+            "%u ctis traversed.  %u.%.4u ctis / load\n",
+            opt_stats_t.loads_removed_from_loads, 
+            opt_stats_t.loads_removed_from_stores,
+            opt_stats_t.ctis_in_load_removal, top, bottom);
+        LOG(GLOBAL, LOG_OPTS, 1,"%d rlr's had problems because a reg. was overwritten\n",
+            opt_stats_t.reg_overwritten);
+        LOG(GLOBAL, LOG_OPTS, 1,"%d rlr's were saved by using a dead register to save value\n",
+            opt_stats_t.val_saved_in_dead_reg);
+    }
+    if (dynamo_options.peephole && proc_get_family() == FAMILY_PENTIUM_4) {
+        LOG(GLOBAL, LOG_OPTS, 1, "%d inc/dec examined, %d replaced with add/sub\n",
+            opt_stats_t.incs_examined, opt_stats_t.incs_replaced);
+    }
+    if (dynamo_options.unroll_loops) {
+        LOG(GLOBAL, LOG_OPTS, 1, "%d loops unrolled\n", opt_stats_t.loops_unrolled);
+    }
+
+    if (dynamo_options.call_return_matching) {
+        LOG(GLOBAL, LOG_OPTS, 1, "Call Return Matching - stats\n");
+        LOG(GLOBAL, LOG_OPTS, 1, "   %d returns removed\n", opt_stats_t.num_returns_removed);
+        LOG(GLOBAL, LOG_OPTS, 1, "   %d return instrs removed\n", opt_stats_t.num_return_instrs_removed);
+    }
+
+    if (dynamo_options.constant_prop) {
+        LOG(GLOBAL, LOG_OPTS, 1, "Constant Prop - stats\n");
+        LOG(GLOBAL, LOG_OPTS, 1, "   %d operands simplified\n", opt_stats_t.num_opnds_simplified);
+        LOG(GLOBAL, LOG_OPTS, 1, "   %d constant loads from immutable memory discoverd (included in operands simplified)\n", opt_stats_t.num_const_add_const_mem);
+        LOG(GLOBAL, LOG_OPTS, 1, "   %d instructions simplified\n", opt_stats_t.num_instrs_simplified);
+        LOG(GLOBAL, LOG_OPTS, 1, "   %d instructions failed simplification\n", opt_stats_t.num_fail_simplified);
+        LOG(GLOBAL, LOG_OPTS, 1, "   %d jmps removed\n", opt_stats_t.num_jmps_simplified);
+        LOG(GLOBAL, LOG_OPTS, 1, "   %d jecxz related instrs removed, (6 per jecxz instr)\n", opt_stats_t.num_jecxz_instrs_removed);
+    }
+
+    if (dynamo_options.remove_unnecessary_zeroing) {
+        LOG(GLOBAL, LOG_OPTS, 1, "%d unnecessary zeroing instances removed\n", opt_stats_t.xors_removed);
+
+    }
+
+    if (dynamo_options.stack_adjust) {
+        LOG(GLOBAL, LOG_OPTS, 1, "Stack Adjustment Combiner - stats\n");
+        LOG(GLOBAL, LOG_OPTS, 1, "   %d stack adjustments removed\n", opt_stats_t.num_stack_adjust_removed);
+    }
+    
+    if (dynamo_options.remove_dead_code) {
+        LOG(GLOBAL, LOG_OPTS, 1, "Dead Code Elimination - stats\n");
+        LOG(GLOBAL, LOG_OPTS, 1, "   %d dead instructions removed\n", opt_stats_t.dead_loads_removed);
+    }
+
+    if (dynamo_options.instr_counts) {
+        LOG(GLOBAL, LOG_OPTS, 1, "Prior to optimizations\n");
+        LOG(GLOBAL, LOG_OPTS, 1, "     %d instrs in traces\n", opt_stats_t.pre_num_instrs_seen);
+        LOG(GLOBAL, LOG_OPTS, 1, "     %d jmps (cbr) in traces\n", opt_stats_t.pre_num_jmps_seen);
+        LOG(GLOBAL, LOG_OPTS, 1, "After optimizations\n");
+        LOG(GLOBAL, LOG_OPTS, 1, "     %d instrs in traces\n", opt_stats_t.post_num_instrs_seen);
+        LOG(GLOBAL, LOG_OPTS, 1, "     %d jmps (cbr) in traces\n", opt_stats_t.post_num_jmps_seen);
+    }
+
+#ifdef IA32_ON_IA64
+    if (dynamo_options.test_i64) {
+        if (opt_stats_t.i64_test)
+            LOG(GLOBAL, LOG_OPTS, 1, "IA64 test succeded!\n");
+        else
+            LOG(GLOBAL, LOG_OPTS, 1, "IA64 test failed!\n");
+        LOG(GLOBAL, LOG_OPTS, 1, "%d entries into Itanium code\n", opt_stats_t.ia64_num_entries);
+    }
+#endif
+
+}
+#endif
+
+/****************************************************************************/
+
+/* op1 and op2 are both memory references */
+static bool
+generate_antialias_check(dcontext_t *dcontext, instrlist_t *pre_loop,
+                         opnd_t op1, opnd_t op2)
+{
+    /* basic idea: "lea op1 !overlap lea op2" */
+    opnd_t lea1, lea2;
+    if (opnd_same(op1, op2))
+        return false; /* guaranteed alias */
+    if (!opnd_defines_use(op1, op2))
+        return true; /* guaranteed non-alias */
+    /* FIXME: get pre-loop values of registers */
+    /* FIXME: get unroll factor -- pass to opnd_defines_use too */
+    /* assumption: ebx and ecx are saved at start of pre_loop, restored at end 
+     * FIXME: op1/op2 may use ebx/ecx!
+     */
+    lea1 = op1;
+    opnd_set_size(&lea1, OPSZ_lea);
+    lea2 = op2;
+    opnd_set_size(&lea2, OPSZ_lea);
+    instrlist_append(pre_loop, INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_EBX), lea1));
+    instrlist_append(pre_loop, INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ECX), lea2));
+    instrlist_append(pre_loop, INSTR_CREATE_cmp(dcontext,  opnd_create_reg(REG_EBX),
+                                                opnd_create_reg(REG_ECX)));
+    return true;
+}
+
+#define MAX_INDUCTION_VARS 8
+#define MAX_LCDS 32
+#define MAX_INVARIANTS 32
+
+static void
+identify_for_loop(dcontext_t *dcontext, app_pc tag, instrlist_t *trace)
+{
+    instr_t *inst, *branch, *check;
+    opnd_t opnd;
+    int i, j, k;
+    instr_t *induction_var[MAX_INDUCTION_VARS];
+    int num_induction_vars = 0;
+    opnd_t lcd[MAX_LCDS];
+    int num_lcds = 0;
+    opnd_t invariant[MAX_INVARIANTS];
+    int num_invariants = 0;
+    instrlist_t pre_loop;
+    instrlist_init(&pre_loop);
+
+    /* FIXME: what about loops with cbr at top and ubr at bottom? */
+
+    /* FIXME: for now, we only look for single-basic-block traces */
+
+    LOG(THREAD, LOG_OPTS, 3, "identify_for_loop: examining trace with tag "PFX"\n", tag);
+    /* first, make sure we end with a conditional branch (followed by uncond.
+     * for fall-through)
+     */
+    inst = instrlist_last(trace);
+    if (!instr_is_ubr(inst))
+        return;
+    branch = instr_get_prev(inst);
+    if (!instr_is_cbr(branch))
+        return;
+    /* now look for self-loop */
+    if (opnd_get_pc(instr_get_target(branch)) != tag)
+        return;
+
+#ifdef DEBUG
+    LOG(THREAD, LOG_OPTS, 1, "\nidentify_for_loop: found whole-trace self-loop: tag "PFX"\n", tag);
+    if ((stats->logmask & LOG_OPTS) != 0)
+        instrlist_disassemble(dcontext, tag, trace, THREAD);
+#endif
+
+    /* FIXME: for each pair of read/write and write/write: insert pre-loop check to
+     * ensure no aliases
+     */
+
+    /* make a pass looking for lcds and induction variables
+     *   only look at scalars -- ignore memory references, we deal with them
+     *   separately later
+     * also make sure there's only one exit
+     */
+    for (inst = instrlist_first(trace); inst != branch; inst = instr_get_next(inst)) {
+        /* for now do not allow exits in middle */
+        if (instr_is_exit_cti(inst)) {
+            LOG(THREAD, LOG_OPTS, 1, "internal exit found, giving up\n");
+            return;
+        }
+        /* loop-carried dependence: a read with no writes prior in loop but
+         *   with a write following in loop
+         * loop invariant: a read with no writes anywhere in loop
+         * FIXME: better to store dependence info somehow, or to make passes
+         * through instrlist whenever need info?
+         */
+        loginst(dcontext, 1, inst, "considering");
+        for (i=0; i<instr_num_srcs(inst); i++) {
+            opnd = instr_get_src(inst, i);
+            /* ignore immeds and memory references */
+            if (opnd_is_immed(opnd) || opnd_is_memory_reference(opnd))
+                continue;
+            for (check = instrlist_first(trace); check != inst;
+                 check = instr_get_next(check)) {
+                for (j=0; j<instr_num_dsts(check); j++) {
+                    if (opnd_defines_use(instr_get_dst(check, j), opnd)) {
+                        /* write prior to read: no lcd */
+                        loginst(dcontext, 1, check,
+                                "\twrite prior to read -> no lcd");
+                        goto no_lcd;
+                    }
+                }
+            }
+            for (check = inst; check != branch; check = instr_get_next(check)) {
+                for (j=0; j<instr_num_dsts(check); j++) {
+                    if (opnd_defines_use(instr_get_dst(check, j), opnd)) {
+                        /* write following read: lcd */
+                        goto has_lcd;
+                    }
+                }
+            }
+            /* no writes: loop invariant! */
+            logopnd(dcontext, 1, opnd, "\tloop invariant");
+            invariant[num_invariants] = opnd;
+            num_invariants++;
+            if (num_invariants >= MAX_INVARIANTS) {
+                LOG(THREAD, LOG_OPTS, 1, "too many invariants, giving up\n");
+                return;
+            }
+        }
+        loginst(dcontext, 1, inst, "\tfell off end -> no lcd");
+    no_lcd:
+        continue;
+    has_lcd:
+        loginst(dcontext, 1, inst, "\tfound a loop-carried dependence");
+        /* find basic induction variables: i = i + constant
+         * FIXME: consider loop invariants as well as immeds as constants
+         * FIXME: only consider inc,dec,add,sub?
+         * FIXME: look for dependent induction vars too: j = i + constant
+         */
+        /* assumption: immediate operands are always 1st source */
+        if (instr_get_opcode(inst) == OP_inc || instr_get_opcode(inst) == OP_dec ||
+            (instr_num_srcs(inst) == 2 && instr_num_dsts(inst) == 1 &&
+             opnd_is_immed_int(instr_get_src(inst, 0)) &&
+             opnd_same(instr_get_src(inst, 1), instr_get_dst(inst, 0)))) {
+            loginst(dcontext, 1, inst, "\t\tfound induction variable");
+            induction_var[num_induction_vars] = inst;
+            num_induction_vars++;
+            if (num_induction_vars >= MAX_INDUCTION_VARS) {
+                LOG(THREAD, LOG_OPTS, 1, "too many induction vars, giving up\n");
+                return;
+            }
+        } else {
+            /* not an induction variable, but may be ok if lcd operand
+             * is based on induction var values
+             */
+            lcd[num_lcds] = opnd;
+            num_lcds++;
+            if (num_lcds >= MAX_LCDS) {
+                LOG(THREAD, LOG_OPTS, 1, "too many lcds, giving up\n");
+                return;
+            }
+        }
+    }
+
+    LOG(THREAD, LOG_OPTS, 1, "now looking through lcds for ones we can't handle\n");
+    /* it's ok for an lcd to read a value kept in an induction var
+     * or a loop invariant
+     */
+    for (i=0; i<num_lcds; i++) {
+        if (opnd_is_reg(lcd[i])) {
+            for (j=0; j<num_induction_vars; j++) {
+                if (opnd_same(lcd[i], instr_get_dst(induction_var[j], 0)))
+                    goto next_lcd;
+            }
+            for (j=0; j<num_invariants; j++) {
+                if (opnd_same(lcd[i], invariant[j]))
+                    goto next_lcd;
+            }
+        } else if (opnd_is_memory_reference(lcd[i])) {
+            for (j=0; j<opnd_num_regs_used(lcd[i]); j++) {
+                opnd = opnd_create_reg(opnd_get_reg_used(lcd[i], j));
+                for (k=0; k<num_induction_vars; k++) {
+                    if (opnd_same(opnd, instr_get_dst(induction_var[k], 0)))
+                        break;
+                }
+                if (k==num_induction_vars) {
+                    for (k=0; k<num_invariants; k++) {
+                        if (opnd_same(opnd, invariant[k]))
+                            break;
+                    }
+                    if (k==num_invariants)
+                        goto lcd_bad;
+                }
+            }
+        } else
+            ASSERT_NOT_REACHED();
+    next_lcd:
+        logopnd(dcontext, 1, lcd[i],
+                "\tlcd read is induction var value, so it's ok");
+        continue;
+    lcd_bad:
+        logopnd(dcontext, 1, lcd[i],
+                "\tlcd read is not induction var value, giving up");
+        return;
+    }
+
+    /* now look at loop termination test */
+    inst = get_decision_instr(branch);
+    loginst(dcontext, 1, inst, "looking at decision instr");
+    /* test must involve only induction vars and constants */
+    for (i=0; i<instr_num_srcs(inst); i++) {
+        opnd = instr_get_src(inst, i);
+        if (!opnd_is_immed(opnd)) {
+            for (j=0; j<num_induction_vars; j++) {
+                if (opnd_same(opnd, instr_get_dst(induction_var[j], 0)))
+                    break;
+            }
+            if (j==num_induction_vars) {
+                loginst(dcontext, 1, inst,
+                        "\tloop termination test not just consts & inductions!");
+                return;
+            }
+        }
+    }
+
+    LOG(THREAD, LOG_OPTS, 1, "now looking at memory references\n");
+    for (inst = instrlist_first(trace); inst != branch; inst = instr_get_next(inst)) {
+        /* for each store, generate pre-loop checks to ensure no overlap with
+         * any other store or read
+         */
+        loginst(dcontext, 1, inst, "considering");
+        for (i=0; i<instr_num_dsts(inst); i++) {
+            opnd = instr_get_dst(inst, i);
+            if (!opnd_is_memory_reference(opnd))
+                continue;
+            for (check = instrlist_first(trace); check != branch;
+                 check = instr_get_next(check)) {
+                for (j=0; j<instr_num_dsts(check); j++) {
+                    if (check==inst && j==i)
+                        continue;
+                    if (opnd_is_memory_reference(instr_get_dst(check, j))) {
+                        /* disambiguate these writes */
+                        logopnd(dcontext, 1, instr_get_dst(check, j),
+                                "\tgenerating checks");
+                        if (!generate_antialias_check(dcontext, &pre_loop, opnd,
+                                                      instr_get_dst(check, j))) {
+                            loginst(dcontext, 1, inst,
+                                    "unavoidable alias, giving up");
+                            return;
+                        }
+                    }
+                }
+            }
+        }
+    }
+    if (instrlist_first(&pre_loop) != NULL) {
+        /* if we generated any tests, they assume we have two registers:
+         * save two registers at start, then restore at end, of pre_loop
+         * FIXME: what about eflags?
+         */
+        instrlist_prepend(&pre_loop,
+                          instr_create_save_to_dcontext(dcontext, REG_EBX, XBX_OFFSET));
+        instrlist_prepend(&pre_loop,
+                          instr_create_save_to_dcontext(dcontext, REG_ECX, XCX_OFFSET));
+        instrlist_append(&pre_loop,
+                         instr_create_restore_from_dcontext(dcontext, REG_ECX, XCX_OFFSET));
+        instrlist_append(&pre_loop,
+                         instr_create_restore_from_dcontext(dcontext, REG_EBX, XBX_OFFSET));
+    }
+
+
+    LOG(THREAD, LOG_OPTS, 1, "loop has passed all tests so far!\n");
+#ifdef DEBUG
+    LOG(THREAD, LOG_OPTS, 1, "pre-loop checks are:\n");
+    if (stats->loglevel >= 1  && (stats->logmask & LOG_OPTS) != 0)
+        instrlist_disassemble(dcontext, tag, &pre_loop, THREAD);
+#endif
+
+    /* now look for "load, arithop, store" pattern 
+     * THIS IS A HACK -- just want to identify loop in mmx.c
+     */
+    inst = instrlist_first(trace);
+    if (instr_get_opcode(inst) != OP_mov_ld) {
+        LOG(THREAD, LOG_OPTS, 1, "1st instr not a load, aborting\n");
+        return;
+    }
+    inst = instr_get_next(inst);
+    if (instr_get_opcode(inst) != OP_add) {
+        LOG(THREAD, LOG_OPTS, 1, "2nd instr not an add, aborting\n");
+        return;
+    }
+    inst = instr_get_next(inst);
+    if (instr_get_opcode(inst) != OP_mov_st) {
+        LOG(THREAD, LOG_OPTS, 1, "3rd instr not a store, aborting\n");
+        return;
+    }
+    LOG(THREAD, LOG_OPTS, 1, "found 'load, arithop, store' pattern!\n");
+    for (check = instr_get_next(inst); check != branch;
+         check = instr_get_next(check)) {
+        for (j=0; j<num_induction_vars; j++) {
+            if (induction_var[j] == check)
+                break;
+        }
+        if (j==num_induction_vars) {
+            loginst(dcontext, 1, check, "non-induction var is present");
+            return;
+        }
+    }
+    LOG(THREAD, LOG_OPTS, 1, "vectorizing\n");
+
+    /* prior to unrolling, replace inc with add */
+    for (i=0; i<num_induction_vars; i++) {
+        int opcode = instr_get_opcode(induction_var[i]);
+        if (opcode == OP_inc || opcode == OP_dec) {
+            inst = instr_get_prev(induction_var[i]);
+            if (replace_inc_with_add(dcontext, induction_var[i], trace)) {
+                /* orig induction var instr_t was destroyed, get new copy */
+                if (inst == NULL)
+                    induction_var[i] = instrlist_first(trace);
+                else
+                    induction_var[i] = instr_get_next(inst);
+            } else {
+                loginst(dcontext, 1, induction_var[i],
+                        "couldn't replace inc w/ add b/c of eflags\n");
+                /* FIXME: undo earlier inc->adds */
+                return;
+            }
+        }
+    }
+
+    /********** unroll loop **********/
+#if 0
+    do {
+        body using i;
+        i += inc;
+    } while (i < max);
+
+becomes
+
+/*
+pre-loop to get alignment is tricky when replacing sideline...need to
+measure memory addresses, can't just go on induction var...for
+now do not have a pre-loop and do unaligned simd
+          offs = i % inc;
+          while (i % (inc * unrollfactor) != 0) {
+              body using i;
+              i += inc;
+          }
+*/
+        do {
+            body using i;
+            body using i+inc;
+            i += (inc * unrollfactor);
+        } while (i < max - (inc * unrollfactor) + 1);
+    while (i < max) {
+        body using i;
+        i += inc;
+    }
+#endif
+#if 0
+    /* First do pre-alignment loop
+     * to do mod: idiv divides edx:eax by r/m32, puts quotient
+     *     in eax, remainder in edx
+     *   cdq sign-extends eax into edx:eax
+     *   so x mod y is:
+     *     mov x,%eax
+     *     cdq
+     *     idiv y
+     * answer is now in %edx!  */
+    inst = instrlist_first(trace);
+    instrlist_preinsert(trace, inst,
+                        instr_create_save_to_dcontext(dcontext, REG_EAX, XAX_OFFSET));
+    instrlist_preinsert(trace, inst,
+                        instr_create_save_to_dcontext(dcontext, REG_EDX, XDX_OFFSET));
+    /* FIXME: get termination index var & lower/upper bound 
+     * hardcoding to mmx.c loop for now
+     */
+    instrlist_preinsert(trace, inst,
+                        INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_EAX), opnd_create_reg(REG_ECX)));
+    instrlist_preinsert(trace, inst, INSTR_CREATE_cdq(dcontext));
+    /* can't pass immed to idiv so have to grab register */
+    instrlist_preinsert(trace, inst,
+                        instr_create_save_to_dcontext(dcontext, REG_EBX, XBX_OFFSET));
+    instrlist_preinsert(trace, inst,
+                        INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_EBX),
+                                             OPND_CREATE_INT32(unroll_factor)));
+    instrlist_preinsert(trace, inst,
+                        INSTR_CREATE_idiv_v(dcontext, opnd_create_reg(REG_EBX)));
+    instrlist_preinsert(trace, inst,
+                        instr_create_restore_from_dcontext(dcontext, REG_EBX, XBX_OFFSET));
+    /* mod is now in edx */
+    /* FIXME:
+     *   test edx, edx
+     *   restore eax and edx
+     *   jz unrolled_loop
+     *   <body of loop>
+     *   jmp top of pre loop
+     * unrolled_loop:
+     *   <body of loop> X unroll factor except:
+     *   test: "< bound" => "< bound - unrollfactor + 1"
+     *   ivar: "ivar += k" => "ivar += unrollfactor * k"
+     */
+    instrlist_preinsert(trace, inst,
+                        instr_create_restore_from_dcontext(dcontext, REG_EDX, XDX_OFFSET));
+    instrlist_preinsert(trace, inst,
+                        instr_create_restore_from_dcontext(dcontext, REG_EAX, XAX_OFFSET));
+#endif
+
+    /* HACK: focus on mmx.c sample loop */
+    inst = instrlist_first(trace);
+    opnd = instr_get_src(inst, 0);
+    ASSERT(opnd_is_memory_reference(opnd));
+    opnd_set_size(&opnd, OPSZ_8);
+    check = INSTR_CREATE_movq(dcontext, opnd_create_reg(REG_MM0), opnd);
+    loginst(dcontext, 1, inst, "replacing this");
+    loginst(dcontext, 1, check, "\twith this");
+    replace_inst(dcontext, trace, inst, check);
+
+    inst = instr_get_next(check);
+    opnd = instr_get_src(inst, 0);
+    ASSERT(opnd_is_memory_reference(opnd));
+    opnd_set_size(&opnd, OPSZ_8);
+    check = INSTR_CREATE_paddd(dcontext, opnd_create_reg(REG_MM0), opnd);
+    loginst(dcontext, 1, inst, "replacing this");
+    loginst(dcontext, 1, check, "\twith this");
+    replace_inst(dcontext, trace, inst, check);
+
+    inst = instr_get_next(check);
+    opnd = instr_get_dst(inst, 0);
+    ASSERT(opnd_is_memory_reference(opnd));
+    opnd_set_size(&opnd, OPSZ_8);
+    check = INSTR_CREATE_movq(dcontext, opnd, opnd_create_reg(REG_MM0));
+    loginst(dcontext, 1, inst, "replacing this");
+    loginst(dcontext, 1, check, "\twith this");
+    replace_inst(dcontext, trace, inst, check);
+
+    /* now make induction vars do X unroll duty */
+    for (i=0; i<num_induction_vars; i++) {
+        if (instr_get_opcode(induction_var[i]) == OP_inc ||
+            instr_get_opcode(induction_var[i]) == OP_dec) {
+            /* couldn't convert to add/sub, so duplicate */
+            instrlist_preinsert(trace, induction_var[i],
+                                instr_clone(dcontext, induction_var[i]));
+        } else {
+            opnd = instr_get_src(induction_var[i], 0);
+            ASSERT(opnd_is_immed_int(opnd));
+            opnd = opnd_create_immed_int(opnd_get_immed_int(opnd)*2, opnd_get_size(opnd));
+            instr_set_src(induction_var[i], 0, opnd);
+        }
+    }
+
+#ifdef DEBUG
+    LOG(THREAD, LOG_OPTS, 1, "\nfinal version of trace:\n");
+    if ((stats->logmask & LOG_OPTS) != 0)
+        instrlist_disassemble(dcontext, tag, trace, THREAD);
+#endif
+}
+
+/****************************************************************************/
+
+/* WARNING: this optimization inserts intra-trace loops that are not
+ * considered exit cti's, so they cannot be unlinked/relinked, nor does
+ * linkcount profiling work properly on them.
+ * We need to figure out our official stance on support for this kind
+ * of thing in optimized traces.
+ */
+static void
+unroll_loops(dcontext_t *dcontext, app_pc tag, instrlist_t *trace)
+{
+    instr_t *inst, *temp;
+    instr_t *branch, *decision, *cmp, *recovery_cmp, *final_jmp;
+    int unroll_factor, cmp_vs, i;
+    bool counting_up;
+    opnd_t cmp_const;
+    uint eflags_6 = 0;
+
+    /* FIXME: what about loops with cbr at top and ubr at bottom? */
+
+    LOG(THREAD, LOG_OPTS, 3, "unroll loop: examining trace with tag "PFX"\n", tag);
+    /* first, make sure we end with a conditional branch (followed by uncond.
+     * for fall-through)
+     */
+    final_jmp = instrlist_last(trace);
+    if (!instr_is_ubr(final_jmp))
+        return;
+    branch = instr_get_prev(final_jmp);
+    if (!instr_is_cbr(branch))
+        return;
+    /* now look for self-loop */
+    if (opnd_get_pc(instr_get_target(branch)) != tag)
+        return;
+
+    /* eflags: for simplicity require that eflags be written before read 
+     * only need to look at arith flags
+     */
+#ifdef DEBUG
+    LOG(THREAD, LOG_OPTS, 3, "\nunroll loop -- checking eflags on:\n");
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_OPTS) != 0)
+        instrlist_disassemble(dcontext, tag, trace, THREAD);
+#endif
+    for (inst = instrlist_first(trace); inst != branch; inst = instr_get_next(inst)) {
+        uint eflags = instr_get_arith_flags(inst);
+        if (eflags != 0) {
+            if ((eflags & EFLAGS_READ_6) != 0) {
+                if ((eflags_6 | (eflags & EFLAGS_READ_6)) != eflags_6) {
+                    /* we're reading a flag that has not been written yet */
+                    loginst(dcontext, 3, inst,
+                            "reads flag before writing it, giving up");
+                    return;
+                }
+            } else if ((eflags & EFLAGS_WRITE_6) != 0) {
+                /* store the flags we're writing, but as read bits */
+                eflags_6 |= EFLAGS_WRITE_TO_READ((eflags & EFLAGS_WRITE_6));
+                /* check against read flags (we've shifted them): */
+                if ((eflags_6 & EFLAGS_READ_6) == EFLAGS_READ_6) {
+                    break; /* all written before read */
+                }
+            }
+        }
+    }
+    /* if get here, eflags are written before read -- and we assume that
+     * our cmp checks below will ensure that exiting the trace will not
+     * have different eflags behavior than the unrolled loop
+     * FIXME: I'm not certain of this
+     */
+
+#ifdef DEBUG
+    LOG(THREAD, LOG_OPTS, 3, "\nunroll loop: found whole-trace self-loop: tag "PFX"\n", tag);
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_OPTS) != 0)
+        instrlist_disassemble(dcontext, tag, trace, THREAD);
+#endif
+
+    /********** unroll loop **********/
+    /*
+      do {
+      body using i;
+      i += inc;
+      } while (i < max);
+
+      becomes (assuming no eflags problems):
+
+      while (i < max - (inc * (unrollfactor-1))) {
+      body using i;
+      body using i+inc;
+      body using i+(inc*2);
+      ...
+      body using i+(inc*(unrollfactor-1));
+      i += (inc * unrollfactor);
+      }
+      while (i < max) {
+      body using i;
+      i += inc;
+      }
+    */
+
+    /* FIXME: determine best unroll factor, somehow */
+    unroll_factor = 2;
+
+    /* see if we can get the branch into a state that can
+     * have its bounds changed: "cmp var, immed"
+     */
+    decision = get_decision_instr(branch);
+    if (decision == NULL) {
+        LOG(THREAD, LOG_OPTS, 3, "can't find decision instr\n");
+        return;
+    }
+    loginst(dcontext, 3, decision, "decision instr");
+
+    if (instr_get_opcode(decision) == OP_cmp) {
+        int opcode = instr_get_opcode(branch);
+        switch (opcode) {
+        case OP_jb:   counting_up = true; break;
+        case OP_jnb:  counting_up = false; break;
+        case OP_jbe:  counting_up = true; break;
+        case OP_jnbe: counting_up = false; break;
+        case OP_jl:   counting_up = true; break;
+        case OP_jnl:  counting_up = false; break;
+        case OP_jle:  counting_up = true; break;
+        case OP_jnle: counting_up = false; break;
+        case OP_js:   counting_up = true; break;
+        case OP_jns:  counting_up = false; break;
+        default:
+            loginst(dcontext, 3, branch, "cannot handle decision branch");
+            return;
+        }
+    } else if (instr_get_opcode(decision) == OP_inc ||
+               instr_get_opcode(decision) == OP_add ||
+               instr_get_opcode(decision) == OP_adc)
+        counting_up = true;
+    else if (instr_get_opcode(decision) == OP_dec || instr_get_opcode(decision) == OP_sub)
+        counting_up = false;
+    else {
+        LOG(THREAD, LOG_OPTS, 3, "can't figure out direction of loop index var\n");
+        return;
+    }
+
+    if (instr_get_opcode(decision) == OP_cmp) {
+        cmp = decision;
+    } else {
+        /* common loop type: ends with "dec var, jns" */
+        if (instr_get_opcode(decision) == OP_inc || instr_get_opcode(decision) == OP_dec) {
+            /* FIXME: shadowing cmp_const */
+            opnd_t cmp_var, cmp_const;
+            int opcode;
+            cmp_var = instr_get_dst(decision, 0);
+            if (instr_get_opcode(branch) == OP_jns) {
+                /* jump if non-negative */
+                cmp_const = OPND_CREATE_INT8(0);
+                opcode = OP_jge;
+            } else if (instr_get_opcode(branch) == OP_js) {
+                /* jump if negative */
+                cmp_const = OPND_CREATE_INT8(0);
+                opcode = OP_jl;
+            } else {
+                loginst(dcontext, 3, branch, "can't handle loop branch");
+                return;
+            }
+            cmp = INSTR_CREATE_cmp(dcontext, cmp_var, cmp_const);
+            instrlist_preinsert(trace, branch, cmp);
+            temp = INSTR_CREATE_jcc(dcontext, opcode, instr_get_target(branch));
+            replace_inst(dcontext, trace, branch, temp);
+            branch = temp;
+
+            /* replace with add/sub for easy stride changing
+             * if we fail, give up, not because we can't inc twice, but
+             * because of eflags concerns
+             */
+            if (!replace_inc_with_add(dcontext, decision, trace)) {
+                loginst(dcontext, 3, decision, "couldn't replace with add/sub");
+                return;
+            }
+        } else {
+            /* give up -- if add cases in future, remember to deal w/ eflags */
+            loginst(dcontext, 3, decision, "can't handle loop branch decision");
+            return;
+        }
+    }
+    /* FIXME: detect loop invariants, and allow them as constants 
+     * requires adding extra instructions to compute bounds
+     */
+    if (!opnd_is_immed_int(instr_get_src(cmp, 1))) {
+        loginst(dcontext, 3, cmp, "cmp is not vs. constant");
+        return;
+    }
+
+    /* make recovery loop */
+    recovery_cmp = instr_clone(dcontext, cmp);
+    instrlist_preinsert(trace, final_jmp, recovery_cmp);
+    temp = instr_clone(dcontext, branch);
+    instr_invert_cbr(temp);
+    instr_set_target(temp, instr_get_target(final_jmp));
+    instrlist_preinsert(trace, final_jmp, temp);
+    for (inst = instrlist_first(trace); inst != cmp;
+         inst = instr_get_next(inst)) {
+        instrlist_preinsert(trace, final_jmp, instr_clone(dcontext, inst));
+    }
+    /* now change final jmp to loop to recovery loop check */
+    instr_set_target(final_jmp, opnd_create_instr(recovery_cmp));
+
+    /* unroll:
+     * duplicate every instruction up to cmp at end
+     */
+    temp = instr_get_prev(cmp);
+    for (i=1; i<unroll_factor; i++) {
+        for (inst = instrlist_first(trace); inst != cmp;
+             inst = instr_get_next(inst)) {
+            instrlist_preinsert(trace, cmp, instr_clone(dcontext, inst));
+            if (inst == temp) /* avoid infinite loop */
+                break;
+        }
+    }
+
+    /* now switch unrolled loop from do-while to while */
+
+    /* put jcc up front */
+    temp = instr_clone(dcontext, branch);
+    instr_invert_cbr(temp);
+    instr_set_target(temp, opnd_create_instr(recovery_cmp));
+    instrlist_prepend(trace, temp);
+    
+    /* now stick cmp in front of it */
+    cmp_vs = (int) opnd_get_immed_int(instr_get_src(cmp, 1));
+    if (counting_up)
+        cmp_vs -= (unroll_factor - 1);
+    else
+        cmp_vs += (unroll_factor - 1);
+    if (cmp_vs >= -128 && cmp_vs <= 127)
+        cmp_const = OPND_CREATE_INT8(cmp_vs);
+    else
+        cmp_const = OPND_CREATE_INT32(cmp_vs);
+    instrlist_prepend(trace,
+                      INSTR_CREATE_cmp(dcontext, instr_get_src(cmp, 0), cmp_const));
+
+    /* now change end-of-unrolled-loop jcc to be a jmp to top cmp */
+    instr_set_opcode(branch, OP_jmp);
+    instr_set_target(branch, opnd_create_instr(instrlist_first(trace)));
+    
+    /* remove end-of-unrolled-loop cmp */
+    instrlist_remove(trace, cmp);
+    instr_destroy(dcontext, cmp);
+
+    /* control flow is all set, now combine induction var updates
+     * FIXME: NOT DONE YET
+     */
+
+#ifdef DEBUG
+    opt_stats_t.loops_unrolled++;
+    LOG(THREAD, LOG_OPTS, 3, "\nfinal version of unrolled trace:\n");
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_OPTS) != 0)
+        instrlist_disassemble(dcontext, tag, trace, THREAD);
+#endif
+}
+
+#ifdef IA32_ON_IA64
+/***************************************************************************/
+/* from Tim */
+/* tests the jmpe and jmpe extensions to IA32 for IA64 environments */
+static bool have_done = false;
+static bool checked = false;
+static int test1;
+static int test2;
+static int test3;
+
+static void 
+test_i64(dcontext_t *dcontext, app_pc tag, instrlist_t *trace)
+{       
+    uint * i64_code;
+    uint * i64_code_start;
+    int check = 0;
+    instr_t * inst;
+    if (!have_done) {
+#ifdef DEBUG
+        opt_stats_t.i64_test = false;
+        LOG(THREAD, LOG_OPTS, 1, "\nadding ia64 test\n");
+        if (stats->loglevel >= 1 && (stats->logmask & LOG_OPTS) != 0)
+            instrlist_disassemble(dcontext, tag, trace, THREAD);
+#endif
+        have_done = true;
+        test1 = 0;
+        test2 = 0;
+        test3 = 0;
+        /* get the mem, get extra for alignment considerations */
+        i64_code = (uint *)heap_alloc(dcontext, 16*sizeof(uint) HEAPACCT(ACCT_OTHER));
+        /* ensure proper aligment, 16 byte */
+        while ((((ptr_uint_t)i64_code) & 0xf) != 0) {
+            i64_code++;
+            check++;
+        }
+        ASSERT(check <= 4);
+        i64_code_start = i64_code;
+        /* emit itanium instruction */
+        /* instruction byte order is */ 
+        /* little endian, IA-32 is also little endian so set there */
+        /* use reg 1 since jmpe will leave a return address there */
+        /* branch nop is easy, is just 001000000.. for 41 bits */
+        
+        /* first instruction bundle */
+        /* just move */
+        /*
+         *i64_code = 0x00000002;
+         i64_code++;
+         *i64_code = 0x08100001;
+         i64_code++;
+         *i64_code = 0x00038004;
+         i64_code++;
+         *i64_code = 0x00040000;
+         i64_code++;
+        */
+
+        /* mov and inc reg9 (ecx) bundle */
+        *i64_code = 0x12044802;
+        i64_code++;
+        *i64_code = 0x08102100;
+        i64_code++;
+        *i64_code = 0x00038004;
+        i64_code++;
+        *i64_code = 0x00040000;
+        i64_code++;
+
+        /* br.ia instruction bundle */
+        *i64_code = 0x0000001d;
+        i64_code++;
+        *i64_code = 0x00000001;
+        i64_code++;
+        *i64_code = 0x20000200;
+        i64_code++;
+        *i64_code = 0x00800010;
+        
+
+        /* add testing prefix to trace */
+        inst = instrlist_first(trace);
+
+        /* store state, and pref for tests */
+        instrlist_preinsert(trace, inst,
+                            instr_create_save_to_dcontext(dcontext, REG_ECX, XCX_OFFSET));
+        instrlist_preinsert(trace, inst,
+                            instr_create_save_to_dcontext(dcontext, REG_EBX, XBX_OFFSET));
+#ifdef DEBUG
+        instrlist_preinsert(trace, inst,
+                            instr_create_save_to_dcontext(dcontext, REG_EAX, XAX_OFFSET));
+        instrlist_preinsert(trace, inst,
+                            INSTR_CREATE_lahf(dcontext));
+        instrlist_preinsert(trace, inst,
+                            INSTR_CREATE_inc(dcontext, opnd_create_base_disp(REG_NULL, REG_NULL, 1, (int)&(opt_stats_t.ia64_num_entries), OPSZ_4_short2)));
+#endif
+        instrlist_preinsert(trace, inst,
+                            INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_ECX), OPND_CREATE_INT32(1)));
+        instrlist_preinsert(trace, inst,
+                            INSTR_CREATE_mov_st(dcontext, opnd_create_base_disp(REG_NULL, REG_NULL, 1, (int)&test1, OPSZ_4_short2), opnd_create_reg(REG_ECX)));
+        
+        /* test jmpe */
+        instrlist_preinsert(trace, inst,
+                            INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_EBX), OPND_CREATE_INT32((int)i64_code_start)));
+        instrlist_preinsert(trace, inst,
+                            INSTR_CREATE_jmpe(dcontext, opnd_create_reg(REG_EBX)));
+        instrlist_preinsert(trace, inst,
+                            INSTR_CREATE_mov_st(dcontext, opnd_create_base_disp(REG_NULL, REG_NULL, 1, (int)&test2, OPSZ_4_short2), opnd_create_reg(REG_ECX)));
+
+        /* test jmpe_abs */
+        instrlist_preinsert(trace, inst,
+                            INSTR_CREATE_jmpe_abs(dcontext, opnd_create_pc((app_pc)i64_code_start)));
+        instrlist_preinsert(trace, inst,
+                            INSTR_CREATE_mov_st(dcontext, opnd_create_base_disp(REG_NULL, REG_NULL, 1, (int)&test3, OPSZ_4_short2), opnd_create_reg(REG_ECX)));
+
+        /* cleanup */
+#ifdef DEBUG
+        instrlist_preinsert(trace, inst, 
+                            INSTR_CREATE_sahf(dcontext));
+        instrlist_preinsert(trace, inst,
+                            instr_create_restore_from_dcontext(dcontext, REG_EAX, XAX_OFFSET));
+#endif
+        instrlist_preinsert(trace, inst,
+                            instr_create_restore_from_dcontext(dcontext, REG_ECX, XCX_OFFSET));
+        instrlist_preinsert(trace, inst,
+                            instr_create_restore_from_dcontext(dcontext, REG_EBX, XBX_OFFSET));
+#ifdef DEBUG
+        LOG(THREAD, LOG_OPTS, 1, "\ndone adding ia64 test\n");
+        if (stats->loglevel >= 1 && (stats->logmask & LOG_OPTS) != 0)
+            instrlist_disassemble(dcontext, tag, trace, THREAD);
+#endif
+        LOG(THREAD, LOG_OPTS, 1, "i64_test cur vals test1 : %d    test2 : %d   test3 : %d\n", test1, test2, test3);
+    } else {
+        if (!(test1 == 1 && test2 == 2 && test3 == 3)) {
+            LOG(THREAD, LOG_OPTS, 1, "i64_test cur vals test1 : %d    test2 : %d   test3 : %d\n", test1, test2, test3);
+        } else if (!checked) {
+            LOG(THREAD, LOG_OPTS, 1, "i64_test cur vals test1 : %d    test2 : %d   test3 : %d\n", test1, test2, test3);
+#ifdef DEBUG
+            opt_stats_t.i64_test = true;
+            checked = true;
+        }
+#endif
+    }
+}
+#endif
+
+/***************************************************************************/
+/* from Tim */
+/* a simple non-optimization that counts the number of instructions processed */
+/* maby extend it later to gather more statistics, size distribution          */
+/* op distribution, etc. if ever desired */
+
+static void 
+instr_counts(dcontext_t *dcontext, app_pc tag, instrlist_t *trace, bool pre)
+{
+#ifdef DEBUG
+    instr_t *inst;
+    int jmps = 0;
+    int instrs = 0;
+    for (inst = instrlist_first(trace); inst != NULL; inst = instr_get_next(inst)) {
+        instrs++;
+        if (instr_is_cbr(inst))
+            jmps++;
+    }
+    if (pre) {
+        LOG(THREAD, LOG_OPTS, 2, "Prior to optimization\n     %d instrs in trace\n     %d jmps exiting trace\n", instrs, jmps);
+        opt_stats_t.pre_num_instrs_seen += instrs;
+        opt_stats_t.pre_num_jmps_seen += jmps;
+    } else {
+        LOG(THREAD, LOG_OPTS, 2, "After optimization\n     %d instrs in trace\n     %d jmps exiting trace\n", instrs, jmps);
+        opt_stats_t.post_num_instrs_seen += instrs;
+        opt_stats_t.post_num_jmps_seen += jmps;
+    }
+#endif
+}
+
+
+/***************************************************************************/
+/* from Tim */
+/* Constant Propagation */
+
+/* utility structures */
+#define PS_VALID_VAL       0x01
+#define PS_LVALID_VAL      0x02 /* high and low parts, only used for regs */
+#define PS_HVALID_VAL      0x04
+#define PS_KEEP            0x80
+
+#define NUM_CONSTANT_ADDRESS  24
+#define NUM_STACK_SLOTS   24
+
+
+static int cp_global_aggr;
+static int cp_local_aggr;
+
+/* holds the current state of the propagation */
+typedef struct _prop_state_t {
+    dcontext_t *dcontext;
+    instrlist_t *trace;
+    instr_t *hint;
+    byte reg_state[8];
+    int reg_vals[8];
+    /* constant address */
+    int addresses[NUM_CONSTANT_ADDRESS];
+    int address_vals[NUM_CONSTANT_ADDRESS];
+    byte address_state[NUM_CONSTANT_ADDRESS];
+
+    /* stack */
+    int cur_scope;
+
+    int stack_offsets_ebp[NUM_STACK_SLOTS];
+    int stack_vals[NUM_STACK_SLOTS];
+
+    int stack_scope[NUM_STACK_SLOTS];
+    byte stack_address_state[NUM_STACK_SLOTS];
+    /* add esp offsets in the future ? */
+    bool lost_scope_count;
+} prop_state_t; 
+
+
+static void 
+set_stack_val(prop_state_t *state, int disp, int val, byte flags)
+{
+#ifdef DEBUG
+    dcontext_t *dcontext = state->dcontext;
+#endif
+    bool cont = true;
+    int i;
+    for (i = 0; (i < NUM_STACK_SLOTS) && cont; i++) {
+        if (state->stack_address_state[i] == 0) {
+            state->stack_offsets_ebp[i] = disp;
+            state->stack_vals[i] = val;
+            state->stack_scope[i] = state->cur_scope;
+            state->stack_address_state[i] = flags;
+            cont = false;
+        }
+    }
+    if (cont) {
+        LOG(THREAD, LOG_OPTS, 3, "stack cache overflow\n");
+        i = disp % NUM_STACK_SLOTS;
+        ASSERT(i>0 && i<NUM_STACK_SLOTS);
+        state->stack_offsets_ebp[i] = disp;
+        state->stack_vals[i] = val;
+        state->stack_scope[i] = state->cur_scope;
+        state->stack_address_state[i] = flags;
+    }
+    
+    LOG(THREAD, LOG_OPTS, 3, " stack cache add: "PFX"  val "PFX"  scope depth %d flags 0x%02x\n", disp, val, state->cur_scope, flags);
+} 
+
+static void 
+update_stack_val(prop_state_t *state, int disp, int val)
+{
+#ifdef DEBUG
+    dcontext_t *dcontext = state->dcontext;
+#endif
+    bool cont = true;
+    int i;
+    for (i = 0; i < NUM_STACK_SLOTS && cont; i++) {
+        if (state->stack_offsets_ebp[i] == disp &&
+            state->stack_scope[i] == state->cur_scope) {
+            state->stack_vals[i] = val;
+            state->stack_address_state[i] |= PS_VALID_VAL;
+            cont = false;
+            LOG(THREAD, LOG_OPTS, 3, " stack cache update disp "PFX" to value "PFX" at stack depth %d\n", disp, val, state->cur_scope);
+        }
+    }
+    if (cp_local_aggr > 2) {
+        if (cont)
+            set_stack_val(state, disp, val, PS_VALID_VAL);
+    }
+}
+
+#if 0 /* not used */
+static bool 
+check_stack_val(prop_state_t *state, int disp, int val, bool valid)
+{
+    int i;
+    for (i = 0; i < NUM_STACK_SLOTS; i++) {
+        if (state->stack_offsets_ebp[i] == disp &&
+            state->stack_scope[i] == state->cur_scope) {
+            if ((state->stack_address_state[i] & PS_VALID_VAL) != 0) {
+                if (valid)
+                    ASSERT(state->stack_vals[i] == val);
+                return false;
+            }
+            return valid;
+        }
+    }
+    return true;
+}
+#endif
+
+static void 
+clear_stack_val(prop_state_t *state, int address)
+{
+#ifdef DEBUG
+    dcontext_t *dcontext = state->dcontext;
+#endif
+    int i;
+    bool cont=true;
+    for (i = 0; i < NUM_STACK_SLOTS && cont; i++) {
+        if (state->stack_offsets_ebp[i] == address &&
+            state->stack_scope[i] == state->cur_scope) {
+            state->stack_address_state[i] &= PS_KEEP;
+            cont=false;
+            LOG(THREAD, LOG_OPTS, 3, " load constant cache cleared: disp "PFX" stack depth %d \n", address, state->cur_scope);
+        }
+    }
+}
+
+/* adds an address value pair to the constant address cache */
+static void 
+set_address_val(prop_state_t *state, int address, int val, byte flags)
+{
+#ifdef DEBUG
+    dcontext_t *dcontext = state->dcontext;
+#endif
+    bool cont = true;
+    int i;
+    for (i = 0; (i < NUM_CONSTANT_ADDRESS) && cont; i++) {
+        if (state->address_state[i] == 0) {
+            state->addresses[i] = address;
+            state->address_vals[i] = val;
+            state->address_state[i] = flags;
+            cont = false;
+        }
+    }
+    if (cont) {
+        LOG(THREAD, LOG_OPTS, 3, "constant address cache overflow\n");
+        i = address % NUM_CONSTANT_ADDRESS;
+        ASSERT(i>0 && i<NUM_CONSTANT_ADDRESS);
+        state->addresses[i] = address;
+        state->address_vals[i] = val;
+        state->address_state[i] = flags;
+    }
+    LOG(THREAD, LOG_OPTS, 3, " load const cache add: "PFX"  val "PFX"  flags 0x%02x\n", address, val, flags);
+} 
+
+/* updates and address value pair in the constant address cache if the address is
+ * already there, else adds it */
+static void 
+update_address_val(prop_state_t *state, int address, int val)
+{
+#ifdef DEBUG
+    dcontext_t *dcontext = state->dcontext;
+#endif
+    bool cont = true;
+    int i;
+    for (i = 0; i < NUM_CONSTANT_ADDRESS; i++) {
+        if (state->addresses[i] == address) {
+            state->address_vals[i] = val;
+            state->address_state[i] |= PS_VALID_VAL;
+            cont = false;
+            LOG(THREAD, LOG_OPTS, 3, " load const cache update: "PFX"  val "PFX"\n", address, val);
+        }
+    }
+    if (cp_global_aggr > 2) {
+        if (cont)
+            set_address_val(state, address, val, PS_VALID_VAL);
+    }
+}
+
+/* removes the address from the constant address cache */
+static void 
+clear_address_val(prop_state_t *state, int address)
+{
+#ifdef DEBUG
+    dcontext_t *dcontext = state->dcontext;
+#endif
+    int i;
+    for (i = 0; i < NUM_CONSTANT_ADDRESS; i++) {
+        if (state->addresses[i] == address) {
+            state->address_state[i] &= PS_KEEP;
+            LOG(THREAD, LOG_OPTS, 3, " load constant cache cleared: "PFX"\n", address);
+        }
+    }
+}
+
+/* gets the value of a const address to an immutable region in memory
+ * assumes that const_add_const_mem has already been called on this 
+ * and returned true
+ */
+
+static int 
+get_immutable_value(opnd_t address, prop_state_t *state, int size)
+{
+    int result;
+    
+    switch(size) {
+    case OPSZ_1: 
+        {
+            char *ptr_byte = ((char*)(ptr_int_t)opnd_get_disp(address));
+            IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+            result = *ptr_byte;
+            break;
+        }
+    case OPSZ_2:
+        { 
+            short *ptr_byte = ((short*)(ptr_int_t)opnd_get_disp(address));
+            IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+            result = *ptr_byte;
+            break;
+        }
+    case OPSZ_4:
+        {
+            int *ptr_byte = ((int*)(ptr_int_t)opnd_get_disp(address));
+            IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+            result = *ptr_byte;
+            break;
+        }
+    default :
+        {
+            /* can't handle size, log is usually quadwords for floats */
+#ifdef DEBUG
+            dcontext_t *dcontext = state->dcontext;
+            logopnd(state->dcontext, 3, address, "Couldn't handle size in get_immutable_value"); 
+            LOG(THREAD, LOG_OPTS, 3, "Address size was %d\n", size);
+#endif
+            /* should never get here, since const_address_const_mem should fail */
+            result = 0;
+            ASSERT_NOT_REACHED();
+        }
+    }
+
+    return result;
+}
+
+
+/* returns true if the opnd is a stack  address (ebp) 
+ * i.e. is memory access with ebp as reg base and null as index reg */
+static bool 
+opnd_is_stack_address(opnd_t address)
+{
+    return (opnd_is_near_base_disp(address) && 
+            (opnd_get_base(address) == REG_EBP) && 
+            (opnd_get_index(address) == REG_NULL));
+}
+
+/* true if addresses (which must be a constant address) is an
+ * immutable region of memory */
+static bool 
+const_address_const_mem(opnd_t address, prop_state_t *state, bool prefix_data)
+{
+    bool success = false;
+    int size = opnd_get_size(address);
+    logopnd(state->dcontext, 3, address, " checking const address const mem\n");
+    if (size == OPSZ_4_short2) {
+        if (prefix_data)
+            size = OPSZ_2;
+        else
+            size = OPSZ_4;
+    }
+    if (size != OPSZ_1 && size != OPSZ_2 && size != OPSZ_4) {
+        /* can't handle size, is usually quadwords for floats */
+#ifdef DEBUG
+        dcontext_t *dcontext = state->dcontext;
+        logopnd(state->dcontext, 3, address, "Couldn't handle size in const_address_const_mem"); 
+        LOG(THREAD, LOG_OPTS, 3, "Address size was %d\n", size);
+#endif
+        return false;
+    }
+
+    /* FIXME : is is_execuatable always right here? */
+    /* i.e. is it going to be true, forever, that this location isn't writable */
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false));
+    if (cp_global_aggr > 1 &&
+        is_executable_address((app_pc)(ptr_int_t)opnd_get_disp(address)))
+        success = true;
+
+    return success;
+}
+
+/* takes an opnd and returns a simplified version, simplifies address and
+ * regs based on the information in propState
+ */
+static opnd_t 
+propagate_address(opnd_t old, prop_state_t *state)
+{
+    reg_id_t base_reg, index_reg, seg;
+    int disp, scale;
+    opnd_size_t size;
+    bool modified;
+    
+    if (!opnd_is_memory_reference(old))
+        return old;
+    IF_X64(ASSERT_NOT_IMPLEMENTED(false)); /* rel and abs mem refs NYI */
+    /* tries to simplify the address calculation with propagated values */
+    base_reg = opnd_get_base(old) - REG_START_32;
+    disp = opnd_get_disp(old);
+    index_reg = opnd_get_index(old) - REG_START_32;
+    scale = opnd_get_scale(old);
+    seg = REG_NULL;
+    size = opnd_get_size(old);
+    modified = false;
+
+    if (opnd_is_far_base_disp(old)) {
+        seg = opnd_get_segment(old);
+    }
+
+    if (index_reg < 8 /* rules out underflow */ && 
+        ((state->reg_state[index_reg] & PS_VALID_VAL) != 0)) {
+
+        disp += (state->reg_vals[index_reg] * scale);
+        index_reg = REG_NULL;
+        modified = true;
+    } else {
+        index_reg += REG_START_32;
+    }
+ 
+    if (base_reg < 8 /* rules out underflow */ && 
+        ((state->reg_state[base_reg] & PS_VALID_VAL) != 0)) {
+
+        disp += state->reg_vals[base_reg];
+        /* don't think this is neccesary  *******FIXME*************
+           if ((seg == REG_NULL) && ((base_reg + REG_START_32 == REG_ESP) ||
+           (base_reg + REG_START_32 == REG_EBP))) {
+           seg = SEG_SS;
+           }
+        */
+        base_reg = REG_NULL;
+        modified = true;
+    } else {
+        base_reg += REG_START_32;
+    }
+    
+    if (!modified)
+        return old;
+
+    if (seg == REG_NULL) {
+        /* return base disp */
+        return opnd_create_base_disp(base_reg, index_reg, scale, disp, size);
+    }
+    
+    /* return far base disp */
+    return opnd_create_far_base_disp(seg, base_reg, index_reg, scale, disp, size);
+}
+
+/* attempts to simplify the opnd with propagated information */
+
+static opnd_t 
+propagate_opnd(opnd_t old, prop_state_t *state, bool prefix_data)
+{
+
+    reg_id_t reg;
+    int immed, disp, i;
+    opnd_size_t size=OPSZ_NA;
+#ifdef DEBUG
+    dcontext_t *dcontext = state->dcontext;
+#endif
+
+    if (opnd_is_reg(old)) {
+        reg = opnd_get_reg(old) - REG_START_32;
+        if (reg < 8 /* rules out underflow */) {
+            if ((state->reg_state[reg] & PS_VALID_VAL) != 0) {
+                immed = state->reg_vals[reg];
+                return opnd_create_immed_int(immed, OPSZ_4);
+            } else
+                return old;
+        }
+        reg = opnd_get_reg(old) - REG_START_16;
+        if (reg < 8 /* rules out underflow */) {
+            if ((state->reg_state[reg] & PS_VALID_VAL) != 0 ||
+                ((state->reg_state[reg] & PS_LVALID_VAL) != 0 &&
+                 (state->reg_state[reg] & PS_HVALID_VAL) != 0)) {
+                /* mask out top part of register */
+                immed = state->reg_vals[reg] & 0x0000ffff;
+                /* sign extend if negative */
+                if ((immed & 0x00008000) != 0)
+                    immed |= 0xffff0000; 
+                return opnd_create_immed_int(immed, OPSZ_2);
+            } else
+                return old;
+        }
+        reg = opnd_get_reg(old) - REG_START_8;
+        if (reg < 4 /* rules out underflow */ &&
+            ((state->reg_state[reg] & PS_VALID_VAL) != 0 ||
+             (state->reg_state[reg] & PS_LVALID_VAL) != 0)) {
+            /* is low part */
+            /* mask out top part of register */
+            immed = state->reg_vals[reg] & 0x000000ff;
+            /* sign extend if negative */
+            if ((immed & 0x00000080) != 0)
+                immed |= 0xffffff00;
+            return opnd_create_immed_int(immed, OPSZ_1);
+        }
+        reg -= 4;
+        if (reg < 4 /* rules out underflow */ &&
+            ((state->reg_state[reg] & PS_VALID_VAL) != 0 ||
+             (state->reg_state[reg] & PS_HVALID_VAL) != 0)) {
+            /* is high part */
+            /* mask out part of register */
+            immed = state->reg_vals[reg] & 0x0000ff00;
+            /* shift */
+            immed = immed >> 8;
+            /* sign extend if negative */
+            if ((immed & 0x00000080) != 0)
+                immed |= 0xffffff00;
+            return opnd_create_immed_int(immed, OPSZ_1);
+        }
+        return old;
+    }
+
+    old = propagate_address(old, state);
+
+    /* if address, get size */
+    if (opnd_is_memory_reference(old)) {
+        size = opnd_get_size(old);
+        /* handle variable size */
+        if (size == OPSZ_4_short2) {
+            if (prefix_data)
+                size = OPSZ_2;
+            else
+                size = OPSZ_4;
+        }
+    }
+
+    if (opnd_is_stack_address(old) && cp_local_aggr > 0) {
+        // check stack value
+        disp = opnd_get_disp(old);
+        for (i = 0; i < NUM_STACK_SLOTS; i++) {
+            if (state->stack_offsets_ebp[i] == disp && 
+                state->cur_scope == state->stack_scope[i] &&
+                (state->stack_address_state[i] & PS_VALID_VAL) != 0) {
+                LOG(THREAD, LOG_OPTS, 3, "  at stack depth %d ", state->cur_scope);
+                logopnd(state->dcontext, 3, old, " found cached stack address");
+                immed = state->stack_vals[i];
+                return opnd_create_immed_int(immed, size);
+            }
+        }
+    }
+    
+    if (opnd_is_constant_address(old) && cp_global_aggr > 0) {
+        if (const_address_const_mem(old, state, prefix_data)) {
+#ifdef DEBUG
+            logopnd(state->dcontext, 2, old, " found const address const mem\n");
+            opt_stats_t.num_const_add_const_mem++;
+#endif
+            immed = get_immutable_value(old, state, size);
+            return opnd_create_immed_int(immed, size);      
+        } else {
+            // check for constant address
+            disp = opnd_get_disp(old);
+            for (i = 0; i < NUM_CONSTANT_ADDRESS; i++) {
+                if (state->addresses[i] == disp && (state->address_state[i] & PS_VALID_VAL) != 0) {
+                    logopnd(state->dcontext, 3, old, " found cached constant address\n");
+                    immed = state->address_vals[i];
+                    return opnd_create_immed_int(immed, size);
+                }
+            }
+        }
+    }
+    return old;
+}
+
+/* checks an eflags and eflags_valid to see if a particular flag flag is valid 
+ * and has appropriate value */
+static bool 
+check_flag_val(uint flag, int val, uint eflags, uint eflags_valid)
+{
+    if ((eflags_valid & flag) != 0) {
+        if (((val == 1) && ((flag & eflags) != 0)) ||
+            ((val == 0) && ((flag & eflags) == 0))) {
+            return true;
+        }
+    }
+    return false;
+}
+
+/* checks and eflags and an eflags_valid and checks to see that the two given flags
+ * are both valid and set either same (if same is true) or different (if same is false) */
+static bool 
+compare_flag_vals(uint flag1, uint flag2, bool same, uint eflags, uint eflags_valid)
+{
+    if (((eflags_valid & flag1) != 0) &&
+        ((eflags_valid & flag2) != 0)) {
+        if ((same && (((flag1 & eflags) != 0) == 
+                      ((flag2 & eflags) != 0))) ||
+            ((!same) && (((flag1 & eflags) != 0) !=
+                         ((flag2 & eflags) != 0)))) {
+            return true;
+        }
+    }
+    return false;
+}
+
+/* returns true if, given the passed in flag information the jump
+ * will never be taken */
+static bool 
+removable_jmp(instr_t *inst, uint eflags, uint eflags_valid)
+{
+    int opcode = instr_get_opcode(inst);
+    switch(opcode) {
+    case OP_jno_short: case OP_jno:
+        {
+            return check_flag_val(EFLAGS_READ_OF, 1, eflags, eflags_valid);
+        }
+    case OP_jo_short: case OP_jo:
+        {
+            return check_flag_val(EFLAGS_READ_OF, 0, eflags, eflags_valid);
+        }
+    case OP_jnb_short: case OP_jnb:
+        {
+            return check_flag_val(EFLAGS_READ_CF, 1, eflags, eflags_valid);
+        }
+    case OP_jb_short: case OP_jb:
+        {
+            return check_flag_val(EFLAGS_READ_CF, 0, eflags, eflags_valid);
+        }
+    case OP_jnz_short: case OP_jnz:
+        {
+            return check_flag_val(EFLAGS_READ_ZF, 1, eflags, eflags_valid);
+        }
+    case OP_jz_short: case OP_jz:
+        {
+            return check_flag_val(EFLAGS_READ_ZF, 0, eflags, eflags_valid);
+        }
+    case OP_jnbe_short: case OP_jnbe:
+        {
+            return (check_flag_val(EFLAGS_READ_CF, 1, eflags, eflags_valid) ||
+                    check_flag_val(EFLAGS_READ_ZF, 1, eflags, eflags_valid));
+        }
+    case OP_jbe_short: case OP_jbe:
+        {
+            return (check_flag_val(EFLAGS_READ_CF, 0, eflags, eflags_valid) &&
+                    check_flag_val(EFLAGS_READ_ZF, 0, eflags, eflags_valid));
+        }
+    case OP_jns_short: case OP_jns:
+        {
+            return check_flag_val(EFLAGS_READ_SF, 1, eflags, eflags_valid);
+        }
+    case OP_js_short: case OP_js:
+        {
+            return check_flag_val(EFLAGS_READ_SF, 0, eflags, eflags_valid);
+        }
+    case OP_jnp_short: case OP_jnp:
+        {
+            return check_flag_val(EFLAGS_READ_PF, 1, eflags, eflags_valid);
+        }
+    case OP_jp_short: case OP_jp:
+        {
+            return check_flag_val(EFLAGS_READ_PF, 0, eflags, eflags_valid);
+        }
+    case OP_jnl_short: case OP_jnl:
+        {
+            return compare_flag_vals(EFLAGS_READ_SF, EFLAGS_READ_OF, false, eflags, eflags_valid);
+        }
+    case OP_jl_short: case OP_jl:
+        {
+            return compare_flag_vals(EFLAGS_READ_SF, EFLAGS_READ_OF, true, eflags, eflags_valid);
+        }
+    case OP_jnle_short: case OP_jnle:
+        {
+            return (check_flag_val(EFLAGS_READ_ZF, 1, eflags, eflags_valid) ||
+                    compare_flag_vals(EFLAGS_READ_SF, EFLAGS_READ_OF, false, eflags, eflags_valid));
+        }
+    case OP_jle_short: case OP_jle:
+        {
+            return (check_flag_val(EFLAGS_READ_ZF, 0, eflags, eflags_valid) &&
+                    compare_flag_vals(EFLAGS_READ_SF, EFLAGS_READ_OF, true, eflags, eflags_valid));
+        }
+    }
+    return false;
+}
+
+/* takes in an eflags, eflags_valid and eflags_invalid and propagates the information
+ * forward simplifying instructions and eliminating jumps where possible, returns false
+ * if it reaches a non simplifiable instruction depends on the any of the eflags_valid 
+ * or eflags_invalid before all flags in valid and invalid are overwritten by instructions */
+static bool
+do_forward_check_eflags(instr_t *inst, uint eflags, uint eflags_valid, uint eflags_invalid, prop_state_t *state)
+{
+#ifdef DEBUG
+    dcontext_t *dcontext = state->dcontext;
+#endif
+    instr_t *next_inst, *temp=NULL;
+    uint eflags_check;
+    int opcode;
+    uint temp_ef;
+    bool replace = false;;
+    if ((eflags_valid == 0) && (eflags_invalid == 0))
+        return true;
+    for (inst = instr_get_next(inst); inst != NULL; inst = next_inst) {
+        next_inst = instr_get_next(inst);
+        loginst(state->dcontext, 3, inst, " flag checking ");
+        while ((inst != NULL) && instr_is_cti(inst)) {
+            LOG(THREAD, LOG_OPTS, 3, "attempting to remove, eflags = "PFX" eflags valid = "PFX"\n", eflags, eflags_valid);
+            if (removable_jmp(inst, eflags, eflags_valid)) {
+#ifdef DEBUG
+                opt_stats_t.num_jmps_simplified++;
+                loginst(state->dcontext, 3, inst, " removing this jmp ");
+#endif
+                remove_inst(state->dcontext, state->trace, inst);
+                inst = next_inst;
+                next_inst = instr_get_next(inst);       
+            } else {
+                if (INTERNAL_OPTION(unsafe_ignore_eflags_trace) &&
+                    instr_get_opcode(inst) == OP_jecxz)
+                    return true;
+                LOG(THREAD, LOG_OPTS, 3, "forward eflags check failed (1)\n");
+                return false;
+            }
+        }
+        if ((inst == NULL) || instr_is_interrupt(inst) || instr_is_call(inst)) {
+            LOG(THREAD, LOG_OPTS, 3, "forward eflags check failed (2)\n");
+            return false;
+        }
+
+        // probably move to own method later if expanded to others
+        // FIXME cmov's other setcc's cmc
+        // don't bother to change to xor for zeroing, is not more efficient for 1 byte
+        /* TODO : add set[n]be set[n]l set[n]le, skip p since never used and might not
+         * be setting it right 
+         */
+        opcode = instr_get_opcode(inst);
+        if (((opcode == OP_seto) || (opcode == OP_setno)) && ((eflags_valid & EFLAGS_READ_OF) != 0)) {
+            if (((eflags & EFLAGS_READ_OF) != 0 && opcode == OP_seto) || 
+                ((eflags & EFLAGS_READ_OF) == 0 && opcode == OP_setno)) {
+                temp = INSTR_CREATE_mov_st(state->dcontext, instr_get_dst(inst, 0), OPND_CREATE_INT8(1));
+            } else {
+                temp = INSTR_CREATE_mov_st(state->dcontext, instr_get_dst(inst, 0), OPND_CREATE_INT8(0));
+            }
+            replace = true;
+        }
+        if (((opcode == OP_setz) || (opcode == OP_setnz)) && ((eflags_valid & EFLAGS_READ_ZF) != 0)) {
+            if (((eflags & EFLAGS_READ_ZF) != 0 && opcode == OP_setz) ||
+                ((eflags & EFLAGS_READ_ZF) == 0 && opcode == OP_setnz)) {
+                temp = INSTR_CREATE_mov_st(state->dcontext, instr_get_dst(inst, 0), OPND_CREATE_INT8(1));
+            } else {
+                temp = INSTR_CREATE_mov_st(state->dcontext, instr_get_dst(inst, 0), OPND_CREATE_INT8(0));
+            }
+            replace = true;
+        }
+        if (((opcode == OP_setb) || (opcode == OP_setnb)) && ((eflags_valid & EFLAGS_READ_CF) != 0)) {
+            if (((eflags & EFLAGS_READ_CF) != 0 && opcode == OP_setb) || 
+                ((eflags & EFLAGS_READ_CF) == 0 && opcode == OP_setnb)) {
+                temp = INSTR_CREATE_mov_st(state->dcontext, instr_get_dst(inst, 0), OPND_CREATE_INT8(1));
+            } else {
+                temp = INSTR_CREATE_mov_st(state->dcontext, instr_get_dst(inst, 0), OPND_CREATE_INT8(0));
+            }
+            replace = true;
+        }
+        if (((opcode == OP_sets) || (opcode == OP_setns)) && ((eflags_valid & EFLAGS_READ_SF) != 0)) {
+            if (((eflags & EFLAGS_READ_SF) != 0 && opcode == OP_sets) || 
+                ((eflags & EFLAGS_READ_SF) == 0 && opcode == OP_setns)) {
+                temp = INSTR_CREATE_mov_st(state->dcontext, instr_get_dst(inst, 0), OPND_CREATE_INT8(1));
+            } else {
+                temp = INSTR_CREATE_mov_st(state->dcontext, instr_get_dst(inst, 0), OPND_CREATE_INT8(0));
+            }
+            replace = true;
+        }
+
+        temp_ef = EFLAGS_READ_SF | EFLAGS_READ_ZF | EFLAGS_READ_AF | EFLAGS_READ_PF | EFLAGS_READ_CF;
+        if ((opcode == OP_lahf) && ((eflags_valid & temp_ef) == temp_ef)) {
+            temp_ef = 0x02;
+            if ((eflags & EFLAGS_READ_CF) != 0)
+                temp_ef = temp_ef | 0x01;
+            if ((eflags & EFLAGS_READ_PF) != 0)
+                temp_ef = temp_ef | 0x04;
+            if ((eflags & EFLAGS_READ_AF) != 0)
+                temp_ef = temp_ef | 0x10;
+            if ((eflags & EFLAGS_READ_ZF) != 0)
+                temp_ef = temp_ef | 0x40;
+            // have to sign extend so the create immed int turns out right
+            if ((eflags & EFLAGS_READ_SF) != 0)
+                temp_ef = temp_ef | 0xffffff80;
+            LOG(THREAD, LOG_OPTS, 3, "lahf val  %d  "PFX"\n", temp_ef, temp_ef);
+            temp = INSTR_CREATE_mov_imm(state->dcontext, instr_get_dst(inst, 0), OPND_CREATE_INT8(temp_ef));
+            replace = true;
+        }
+
+        if (replace) {
+#ifdef DEBUG
+            opt_stats_t.num_instrs_simplified++;
+            loginst(state->dcontext, 3, inst, " old instruction");
+            loginst(state->dcontext, 3, temp, " simplified to  ");
+#endif
+            replace_inst(state->dcontext, state->trace, inst, temp);
+            inst = temp;
+            replace = false;
+        }
+
+        eflags_check = instr_get_eflags(inst);
+        if (((eflags_invalid & eflags_check) != 0) || ((eflags_valid & eflags_check) != 0)) {
+            loginst(state->dcontext, 3, inst, " uses eflags!");
+            LOG(THREAD, LOG_OPTS, 3, "forward eflags check failed(3)  "PFX"   "PFX"  "PFX"\n", eflags_valid, eflags_invalid, eflags_check);
+            return false;
+        }
+        eflags_invalid = eflags_invalid & (~ (EFLAGS_WRITE_TO_READ(eflags_check & EFLAGS_WRITE_ALL)));
+        eflags_valid = eflags_valid & (~ (EFLAGS_WRITE_TO_READ(eflags_check & EFLAGS_WRITE_ALL)));
+        if ((eflags_valid == 0) && (eflags_invalid == 0))
+            return true;
+    }
+    LOG(THREAD, LOG_OPTS, 3, "forward eflags check failed(4)\n");
+    return false;
+}
+
+/* looks at the eflags of the instr passed in and checks to see if there
+ * is any dependency on the eflags written, gives up at CTI's
+ * return true if no dependency found
+ */
+static bool 
+forward_check_eflags(instr_t *inst, prop_state_t *state)
+{
+    return do_forward_check_eflags(inst, 0, 0, EFLAGS_WRITE_TO_READ(instr_get_eflags(inst) & EFLAGS_WRITE_ALL), state); 
+}
+
+static instr_t *
+make_imm_store(prop_state_t *state, instr_t *inst, int value)
+{
+    return INSTR_CREATE_mov_st(state->dcontext, instr_get_dst(inst, 0),  OPND_CREATE_INT32(value));
+}
+
+/* replaces inst with a mov imm of value to the same dst */
+static instr_t *
+make_to_imm_store(instr_t *inst, int value, prop_state_t *state)
+{
+    instr_t *replacement;
+    opnd_t dst = instr_get_dst(inst, 0);
+    dcontext_t *dcontext = state->dcontext;
+
+    if (value == 0 && opnd_is_reg(dst)) {
+        replacement = INSTR_CREATE_xor(dcontext, dst, dst);
+        if (instr_get_prefix_flag(inst, PREFIX_DATA)) {
+            instr_set_prefix_flag(replacement, PREFIX_DATA);
+            LOG(THREAD, LOG_OPTS, 3, "carrying data prefix over %d\n", instr_get_prefixes(inst));
+        }
+        if (do_forward_check_eflags(inst, 0, 0, EFLAGS_WRITE_TO_READ(instr_get_eflags(replacement)), state)) {
+            /* check size prefixes, ignore lock and addr prefix */ 
+            replace_inst(dcontext, state->trace, inst, replacement);
+            return replacement;
+        } else {
+            loginst(dcontext, 3, inst, " unable to simplify move zero to xor, e-flags check failed ");
+            instr_destroy(dcontext, replacement);
+        }
+    }
+
+    /* is always creating the right sized imm? */
+    replacement = INSTR_CREATE_mov_st(state->dcontext, dst,  opnd_create_immed_int(value, opnd_get_size(dst)));
+    /* handle prefixes, imm->reg (data) imm->mem (data & addr) */
+    if (instr_get_prefix_flag(inst, PREFIX_DATA)) {
+        instr_set_prefix_flag(replacement, PREFIX_DATA); 
+        LOG(THREAD, LOG_OPTS, 3, "carrying data prefix over %d\n", instr_get_prefixes(inst)); 
+    }
+    if (instr_get_prefix_flag(inst, PREFIX_ADDR) && opnd_is_memory_reference(dst)) {
+        instr_set_prefix_flag(replacement, PREFIX_ADDR);
+        LOG(THREAD, LOG_OPTS, 3, "carrying addr prefix over %d\n", instr_get_prefixes(inst));
+    }  
+    replace_inst(dcontext, state->trace, inst, replacement);
+    return replacement;
+}
+
+static instr_t *
+make_to_nop(prop_state_t *state, instr_t *inst, char *pre, char *post, char *fail)
+{
+    instr_t *backup;
+    if (forward_check_eflags(inst, state)) {
+        loginst(state->dcontext, 3, inst, pre);
+        backup = INSTR_CREATE_nop(state->dcontext);
+        replace_inst(state->dcontext, state->trace, inst, backup);
+        loginst(state->dcontext, 3, backup, post);
+        return backup;
+    } else {
+        loginst(state->dcontext, 3, inst, fail);
+        return inst;
+    }
+}
+
+/* uses < 0 as short for if top bit is set */
+/* calculates zf pf sf flags from some result immed */
+static int 
+calculate_zf_pf_sf(int immed)
+{
+    int result = 0;
+    int i;
+    bool parity = true;
+    if (immed == 0)
+        result = result | EFLAGS_READ_ZF;
+    if (immed < 0)
+        result = result | EFLAGS_READ_SF;
+    for (i = 0; i < 8; i++) {
+        if (((immed >> i) & 0x1) != 0)
+            parity = !parity;
+    }
+    if (parity)
+        result = result | EFLAGS_READ_PF;
+    return result;
+}
+
+/* simplifies an instruction where possible */
+/* NOTE that at this point all subsized arguments have been sign extended */
+/* if op takes subsize note signextend (movzx and shifts for ex.) must */
+/* explicitly check the size of the immed */
+static instr_t *
+prop_simplify(instr_t *inst, prop_state_t *state)
+{
+    instr_t *replacement;
+    int opcode, num_src, num_dst, immed1, immed2, immed3=0, i;
+    opnd_t temp_opnd;
+    uint eflags, eflags_valid, eflags_invalid;
+    dcontext_t *dcontext = state->dcontext;
+    
+    num_src = instr_num_srcs(inst);
+    num_dst = instr_num_dsts(inst);
+    opcode = instr_get_opcode(inst);
+    
+    if (opcode == OP_lea) {
+        temp_opnd = instr_get_src(inst, 0);
+        if (opnd_is_constant_address(temp_opnd)) {
+            inst = make_to_imm_store(inst, opnd_get_disp(temp_opnd), state);
+        }
+        return inst;
+    }
+    
+    if ((num_src == 1) && (num_dst == 1) && opnd_is_immed_int(instr_get_src(inst, 0))) {
+        immed1 = (int) opnd_get_immed_int(instr_get_src(inst, 0));
+        switch(opcode) {
+            // movsx bsf bsr
+        case OP_mov_st:
+        case OP_mov_ld:
+            {
+                inst = make_to_imm_store(inst, immed1, state);
+                break;
+            }
+        case OP_movzx:
+            {
+                if (opnd_get_size(instr_get_src(inst, 0)) == OPSZ_1) {
+                    immed3 = immed1 & 0x000000ff;
+                } else {
+                    immed3 = immed1 & 0x0000ffff;
+                }
+                inst = make_to_imm_store(inst, immed3, state);
+                break;
+            }
+        case OP_movsx:
+            {
+                /* already sign extended */ 
+                immed3 = immed1;
+                inst = make_to_imm_store(inst, immed3, state);
+                break;
+            }
+        case OP_bswap:
+            {
+                immed3 = (((immed1 << 24) & 0xff000000) |
+                          ((immed1 << 8) & 0x00ff0000) |
+                          ((immed1 >> 8) & 0x0000ff00) |
+                          ((immed1 >> 24) & 0x000000ff));
+                inst = make_to_imm_store(inst, immed3, state);
+                break;
+            }
+        case OP_not:
+            {
+                immed3 = ~immed1;
+                inst = make_to_imm_store(inst, immed3, state);
+                break;
+            }
+        case OP_neg:
+            {
+                immed3 = -immed1;
+                inst = make_to_imm_store(inst, immed3, state);
+                break;
+            }
+        case OP_inc:
+            {
+                immed3 = immed1 + 1;
+                eflags = calculate_zf_pf_sf(immed3);
+                eflags_valid = EFLAGS_READ_ZF | EFLAGS_READ_SF | EFLAGS_READ_PF;
+                eflags_invalid = instr_get_eflags(inst) & EFLAGS_READ_ALL & (~eflags_valid);
+                if (do_forward_check_eflags(inst, eflags, eflags_valid, eflags_invalid, state))
+                    inst = make_to_imm_store(inst, immed3, state);
+                else
+                    state->hint = make_imm_store(state, inst, immed3);
+                break;
+            }
+        case OP_dec:
+            {
+                immed3 = immed1 - 1;
+                eflags = calculate_zf_pf_sf(immed3);
+                eflags_valid = EFLAGS_READ_ZF | EFLAGS_READ_SF | EFLAGS_READ_PF;
+                eflags_invalid = instr_get_eflags(inst) & EFLAGS_READ_ALL & (~eflags_valid);
+                if (do_forward_check_eflags(inst, eflags, eflags_valid, eflags_invalid, state))
+                    inst = make_to_imm_store(inst, immed3, state);
+                else
+                    state->hint = make_imm_store(state, inst, immed3);
+                break;
+            }
+        default:
+            {
+                // unable to simplify instruction
+            }
+        }
+        return inst;
+    }
+
+    if ((num_src == 2) && opnd_is_immed_int(instr_get_src(inst, 1)) && !opnd_is_immed_int(instr_get_src(inst, 0))) {
+        immed1 = (int) opnd_get_immed_int(instr_get_src(inst, 1));
+        if (opcode == OP_cmp || opcode == OP_test) {
+            temp_opnd = instr_get_src(inst, 1);
+            if ((immed1 <= 127) && (immed1 >= -128)) {
+                opnd_set_size(&temp_opnd, OPSZ_1);
+                instr_set_src(inst, 1, temp_opnd);
+            }
+        }
+        /* jecxz hack, Should only match our indirect branch handling thing */
+        if (opcode == OP_jecxz) {
+            if (immed1 == 0) {
+                /* NOTE : this hardcodes indirect branch stuff */
+                instr_t *inst2, *inst3;
+                LOG(THREAD, LOG_OPTS, 3, "Found removable jeczx inst noping it and removing 2 prev, and next three instructions\n");
+                replacement = INSTR_CREATE_nop(state->dcontext);    
+                replace_inst(state->dcontext, state->trace, inst, replacement);
+                inst = replacement;
+
+                /* remove control flow after jecxz */
+                inst2 = instr_get_next(inst);
+                loginst(dcontext, 3, inst2, "removing ");
+                ASSERT(instr_get_opcode(inst2) == OP_lea && opnd_get_reg(instr_get_dst(inst2, 0)) == REG_ECX);
+                instrlist_remove(state->trace, inst2);
+                inst2 = instr_get_next(inst);
+                loginst(dcontext, 3, inst2, "removing ");
+                ASSERT(instr_get_opcode(inst2) == OP_jmp);
+                instrlist_remove(state->trace, inst2);
+
+                /* remove prev inst */
+                inst2 = instr_get_prev(inst);
+                inst3 = instr_get_prev(inst2);
+                if (instr_get_opcode(inst2) == OP_nop || 
+                    ((instr_get_opcode(inst2) == OP_mov_imm || 
+                      instr_get_opcode(inst2) == OP_mov_st ||
+                      is_zeroing_instr(inst2)) && 
+                     opnd_get_reg(instr_get_dst(inst2, 0)) == REG_ECX)) {
+                    loginst(dcontext, 3, inst2, "removing ");
+                    instrlist_remove(state->trace, inst2);
+                } else {
+                    loginst(dcontext, 1, inst2, "ERROR : unexpected instruction in constant prop indirect branch removal (1) aborting");
+                    return inst;
+                }
+                inst2 = inst3;
+                /* three possiblities at this point */
+                /* is lea or pop from return = shouldn't happen, well at */
+                /* not till we start propagating stack vals, in which case */
+                /* maby we can ignore since -call_return matching will get */
+                /* is push from indirect call, move imm->reg, save to ecxoff*/
+                /* is mov imm->reg, save to ecxoff */
+                /* !save to ecxoff might be noped, as might any of the other */
+                /* prev. by constant prop, asserts are fragile, remove them? */
+                if (instr_get_opcode(inst2) == OP_pop || instr_get_opcode(inst2) == OP_lea) {
+                    loginst(dcontext, 3, inst2, "ERROR : found what looks like a call return in jecxz removal, but we can't find those yet!! aborting");
+                    return inst;
+                }
+                if (instr_get_opcode(inst2) == OP_push_imm) {
+                    loginst(dcontext, 3, inst2, "skipping ");
+                    inst2 = instr_get_prev(inst2);
+                }
+                inst3 = instr_get_prev(inst2);
+                if (instr_get_opcode(inst2) == OP_nop || 
+                    ((instr_get_opcode(inst2) == OP_mov_imm || 
+                      instr_get_opcode(inst2) == OP_mov_st ||
+                      is_zeroing_instr(inst2)) && 
+                     opnd_get_reg(instr_get_dst(inst2, 0)) == REG_ECX)) {
+                    loginst(dcontext, 3, inst2, "removing ");
+                    instrlist_remove(state->trace, inst2);
+                } else {
+                    loginst(dcontext, 1, inst2, "ERROR : unexpected instruction in constant prop indirect branch removal (2) aborting");
+                    return inst;
+                }
+                inst2 = inst3;
+                if (instr_get_opcode(inst2) == OP_nop || is_store_to_ecxoff(dcontext, inst2)) {
+                    loginst(dcontext, 3, inst2, "removing ");
+                    instrlist_remove(state->trace, inst2);
+                } else {
+                    loginst(dcontext, 1, inst2, "ERROR : unexpected instruction in constant prop indirect branch removal (3) aborting");
+                    return inst;
+                }
+
+                /* remove post inst */
+                /* some op may have removed this already so check to be sure */
+                inst2 = instr_get_next(inst);
+                if (is_load_from_ecxoff(dcontext, inst2)) {
+                    loginst(dcontext, 3, inst2, "removing ");
+                    instrlist_remove(state->trace, inst2);
+                } else {
+                    loginst(dcontext, 1, inst2, "ERROR : unexpected instruction in constant prop indirect branch removal (a), This could be very bad");
+                }
+
+#ifdef DEBUG
+                opt_stats_t.num_jmps_simplified++;
+                opt_stats_t.num_instrs_simplified++;
+                opt_stats_t.num_jecxz_instrs_removed += 6;
+#endif
+            } else {
+                loginst(dcontext, 1, inst, "ERROR : Constant prop predicts indirect branch exit from trace always taken! If this is part of a reconstruct for exception state then the pc calculated is going to be wrong, if it isn't then something is broken regarding constant prop");
+            }
+        }
+        return inst;
+    }
+
+    if ((num_src == 2) && opnd_is_immed_int(instr_get_src(inst, 0))) {
+        immed1 = (int) opnd_get_immed_int(instr_get_src(inst, 0));
+
+        if (!opnd_is_immed_int(instr_get_src(inst, 1))) {
+            switch(opcode) {
+            case OP_sub:
+            case OP_add:
+            case OP_or:
+            case OP_and:
+            case OP_xor:
+                {
+                    temp_opnd = instr_get_src(inst, 0);
+                    if ((immed1 <= 127) && (immed1 >= -128)) {
+                        opnd_set_size(&temp_opnd, OPSZ_1);
+                        instr_set_src(inst, 0, temp_opnd);
+                    }
+                    break;
+                }
+            case OP_test:
+                {
+                    temp_opnd = instr_get_src(inst, 0);
+                    if ((immed1 <= 127) && (immed1 >= -128))
+                        opnd_set_size(&temp_opnd, OPSZ_1);
+                    instr_set_src(inst, 0, instr_get_src(inst, 1));
+                    instr_set_src(inst, 1, temp_opnd);
+                    break;
+                }
+            case OP_push:
+                {
+                    instr_set_opcode(inst, OP_push_imm);
+                    break;
+                }
+            }
+            return inst; 
+        } else {
+            immed2 = (int) opnd_get_immed_int(instr_get_src(inst, 1));
+            switch(num_dst) {
+            case 0:
+                {
+                    switch(opcode) {
+                    case OP_test:
+                        {
+                            immed3 = immed1 & immed2;
+                            eflags_valid = EFLAGS_READ_CF | EFLAGS_READ_PF | EFLAGS_READ_ZF | EFLAGS_READ_SF | EFLAGS_READ_OF | EFLAGS_READ_AF;
+                            // technically AF is undefined, but since no one
+                            // should be relying on it we can set it to zero
+                            eflags = calculate_zf_pf_sf(immed3);
+                            if (do_forward_check_eflags(inst, eflags, eflags_valid, 0, state)) {
+                                replacement = INSTR_CREATE_nop(state->dcontext);    
+                                replace_inst(dcontext, state->trace, inst, replacement);
+                                inst = replacement;
+                            }
+                            break;
+                        }
+                    case OP_cmp:
+                        {
+                            //FIXME of and sf and af correct? FIXME!!
+                            immed3 = immed1 - immed2;
+                            eflags = calculate_zf_pf_sf(immed3);
+                            if (((uint)immed1) < ((uint)immed2)) {
+                                eflags = eflags | EFLAGS_READ_CF;
+                            }
+                            if (((immed1 >= immed2) && ((eflags & EFLAGS_READ_CF) != 0)) ||
+                                ((immed1 < immed2) && ((eflags & EFLAGS_READ_CF) == 0))) {
+                                eflags = eflags | EFLAGS_READ_OF;
+                            }
+                            if ((immed1 & 0x7) < (immed2 & 0x7))
+                                eflags = eflags | EFLAGS_READ_AF;
+                            eflags_valid = EFLAGS_READ_CF | EFLAGS_READ_PF | EFLAGS_READ_ZF | EFLAGS_READ_SF | EFLAGS_READ_OF | EFLAGS_READ_AF;
+                            if (do_forward_check_eflags(inst, eflags, eflags_valid, 0, state)) {
+                                replacement = INSTR_CREATE_nop(state->dcontext);    
+                                replace_inst(state->dcontext, state->trace, inst, replacement);
+                                inst = replacement;
+                            }
+                            break;
+                        }
+                    default:
+                        {
+                            // couldn't handle
+                        }
+                    }
+                    break;
+                }
+            case 1:
+                {
+                    /* TODO: maby explicitly find some flags? */
+                    bool replace = true;
+                    int size;
+                    switch(opcode) {
+                    case OP_add:
+                        immed3 = immed2 + immed1;
+                        break;
+                    case OP_sub:
+                        immed3 = immed2 - immed1;
+                        break;
+                    case OP_or:
+                        immed3 = immed2 | immed1;
+                        break;
+                    case OP_and:
+                        immed3 = immed2 & immed1;
+                        break;
+                    case OP_xor:
+                        immed3 = immed2 ^ immed1;
+                        break;
+                    case OP_shl:
+                        /* same as OP_sal */
+                        size = opnd_get_size(instr_get_src(inst, 1));
+                        immed3 = immed2 << (immed1 & 0x1F);
+                        /* adjust for size */
+                        if (size == OPSZ_1) {
+                            if ((immed3 & 0x00000080) != 0)
+                                immed3 |= 0xffffff00;
+                            else
+                                immed3 &= 0x000000ff;
+                        } else if (size == OPSZ_2) {
+                            if ((immed3 & 0x00008000) != 0)
+                                immed3 |= 0xffff0000;
+                            else
+                                immed3 &= 0x0000ffff;
+                        } else if (size != OPSZ_4)
+                            replace = false;
+                        break;
+                    case OP_sar:
+                        {
+                            bool neg = immed2 < 0;
+                            size = opnd_get_size(instr_get_src(inst, 1));
+                            if (size == OPSZ_1) {
+                                immed3 = immed2 >> (immed1 & 0x1f);
+                                if (neg)
+                                    immed3 |= 0xffffff00;
+                                else
+                                    immed3 &= 0x000000ff;
+                            } else if (size == OPSZ_2) {
+                                immed3 = immed2 >> (immed1 & 0x1f);
+                                if (neg)
+                                    immed3 |= 0xffff0000;
+                                else
+                                    immed3 &= 0x0000ffff;
+                            } else if (size == OPSZ_4) {
+                                immed3 = immed2;
+                                if (neg) {
+                                    for (i = 0; i < (immed1 & 0x1F); i++) 
+                                        immed3 = (immed3 >> 1) | 0x80000000;
+                                } else {
+                                    immed3 = immed3 >> (immed1 & 0x1f);
+                                }
+                            } else
+                                replace = false;
+                            break;
+                        }
+                    case OP_shr:
+                        size = opnd_get_size(instr_get_src(inst, 1));
+                        if (immed1 == 0 || immed2 == 0)
+                            immed3 = immed2;
+                        else {
+                            if (size == OPSZ_1) {
+                                immed3 = (immed2 & 0x000000ff) >> (immed1 & 0x1f);
+                            } else if (size == OPSZ_2) {
+                                immed3 = (immed2 & 0x0000ffff) >> (immed1 & 0x1f);
+                            } else if (size == OPSZ_4) {
+                                if (immed2 > 0) {
+                                    immed3 = immed2 >> (immed1 & 0x1f);
+                                }
+                                else {
+                                    immed3 = immed2;
+                                    for (i = 0; i < (immed1 & 0x1F); i++) 
+                                        immed3 = (immed3 >> 1) & 0x7fffffff;
+                                }
+                            } else
+                                replace = false;
+                        }
+                        break;
+                        /* TODO : rotates, keep size issues in mind */
+                    case OP_ror:
+                    case OP_rol:
+                    default:
+                        replace = false;
+                        /* can't handle this instruction */
+                    }
+                    if (!replace)
+                        break;
+                    if (forward_check_eflags(inst, state))
+                        inst = make_to_imm_store(inst, immed3, state);
+                    else
+                        state->hint = make_imm_store(state, inst, immed3);
+                    break;
+                }
+            case 2:
+                {
+                    // mul divide xchg xadd
+                }
+            default: 
+                {
+                    // unable to simlify this instruction
+                }
+            }
+        }
+        return inst;
+    } 
+
+    // cpuid
+
+    return inst;
+}
+
+/* initializes all the trace constant stuff and add */
+static void 
+get_trace_constant(prop_state_t *state)
+{
+    /* can add all dynamo addresses here, they are never aliased so always */
+    /* safe to optimize, but takes up space in our cache, with new jump code */
+    /* probably only use exc so just put it in, and maby eax to since is fav */
+    /* when need to store flags/pass arg, can always add more location later */
+    /* probably cleaner way of getting addresses but who cares for now */
+    set_address_val(state, opnd_get_disp(opnd_create_dcontext_field(state->dcontext, XCX_OFFSET)), 0, PS_KEEP);
+    set_address_val(state, opnd_get_disp(opnd_create_dcontext_field(state->dcontext, XAX_OFFSET)), 0, PS_KEEP);
+}
+
+/* updates the prop state as appropriate */
+static instr_t *
+update_prop_state(prop_state_t *state, instr_t *inst, bool intrace)
+{
+    int opcode = instr_get_opcode(inst);
+    int num_dst = instr_num_dsts(inst);
+    bool is_zeroing = is_zeroing_instr(inst);
+    dcontext_t *dcontext = state->dcontext;
+    instr_t *backup;
+    opnd_t opnd;
+    int val;
+    reg_id_t reg, i;
+    if (is_zeroing || ((opcode == OP_mov_imm || opcode == OP_mov_st) &&
+                       opnd_is_immed_int(instr_get_src(inst, 0)))) {
+        if (is_zeroing)
+            val = 0;
+        else
+            val = (int) opnd_get_immed_int(instr_get_src(inst, 0));
+        opnd = instr_get_dst(inst, 0);
+        if (opnd_is_reg(opnd)) {
+            reg = opnd_get_reg(opnd) - REG_START_32;
+            if (reg < 8 /* rules out underflow */) {
+                /* if resetting to same value then just nop the instruction */
+                if (intrace && (state->reg_state[reg] & PS_VALID_VAL) != 0 && state->reg_vals[reg] == val) {
+                    inst = make_to_nop(state, inst, " register already set to val, simplify ", " to ", " register already set to val, but unable to simplify due to eflags");
+                } else {
+                    state->reg_state[reg] = PS_VALID_VAL;
+                    state->reg_vals[reg] = val;
+                } 
+            } else {
+                reg = opnd_get_reg(opnd) - REG_START_16;
+                if (reg < 8 /* rules out underflow */) {
+                    /* if resetting to same value then just nop the instruction */
+                    if (intrace && 
+                        ((state->reg_state[reg] & PS_VALID_VAL) != 0 ||
+                         ((state->reg_state[reg] & PS_LVALID_VAL) != 0 &&
+                          (state->reg_state[reg] & PS_HVALID_VAL) != 0)) &&
+                        (state->reg_vals[reg] & 0x0000ffff) == (val & 0x0000ffff)) {
+                        inst = make_to_nop(state, inst, " register already set to val, simplify ", " to ", " register already set to val, but unable to simplify due to eflags");
+                    } else {
+                        state->reg_state[reg] |= PS_LVALID_VAL | PS_HVALID_VAL;
+                        state->reg_vals[reg] = (state->reg_vals[reg] & 0xffff0000) | (val & 0x0000ffff);
+                    }
+                } else {
+                    reg = opnd_get_reg(opnd) - REG_START_8;
+                    if (reg < 4 /* rules out underflow */) {
+                        /* if resetting to same value then just nop the instruction */
+                        if (intrace && 
+                            ((state->reg_state[reg] & PS_VALID_VAL) != 0 ||
+                             (state->reg_state[reg] & PS_LVALID_VAL) != 0) &&
+                            (state->reg_vals[reg] & 0x000000ff) == (val & 0x000000ff)) {
+                            inst = make_to_nop(state, inst, " register already set to val, simplify ", " to ", " register already set to val, but unable to simplify due to eflags");
+                        } else {
+                            state->reg_state[reg] |= PS_LVALID_VAL;
+                            state->reg_vals[reg] = (state->reg_vals[reg] & 0xffffff00) | (val & 0x000000ff);
+                        }
+                    } else {
+                        reg -= 4;
+                        if (reg < 4 /* rules out underflow */) {
+                            /* if resetting to same value then just nop the instruction */
+                            if (intrace && 
+                                ((state->reg_state[reg] & PS_VALID_VAL) != 0 ||
+                                 (state->reg_state[reg] & PS_HVALID_VAL) != 0) &&
+                                (state->reg_vals[reg] & 0x0000ff00) == ((val << 8) & 0x0000ff00)) {
+                                inst = make_to_nop(state, inst, " register already set to val, simplify ", " to ", " register already set to val, but unable to simplify due to eflags");
+                            } else {
+                                state->reg_state[reg] |= PS_HVALID_VAL;
+                                state->reg_vals[reg] = (state->reg_vals[reg] & 0xffff00ff) | ((val << 8) & 0x0000ff00);
+                            }
+                        } else {
+                            /* just in case */
+                            for (i = 0; i < 8; i++) {
+                                if (instr_writes_to_reg(inst, REG_START_32 + i)) {
+                                    state->reg_state[i] = 0;
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        } else {
+            // do constant addresses
+            if (opnd_is_constant_address(opnd) && cp_global_aggr > 0 ) {
+                int disp = opnd_get_disp(opnd);
+                for (i = 0; i < NUM_CONSTANT_ADDRESS; i++) {
+                    if (state->addresses[i] == disp && state->address_vals[i] == val && (state->address_state[i] & PS_VALID_VAL) != 0) {
+                        loginst(dcontext, 3, inst, " mem location already set to val, simplify ");
+                        backup = INSTR_CREATE_nop(dcontext);
+                        replace_inst(dcontext, state->trace, inst, backup);
+                        loginst(dcontext, 3, backup, " to ");
+                        inst = backup;
+                    }
+                }
+                update_address_val(state, disp, val);
+            }
+
+            // do stack vals
+
+            if (opnd_is_stack_address(opnd) && cp_local_aggr > 0) {
+                int disp = opnd_get_disp(opnd);
+                for (i = 0; i < NUM_STACK_SLOTS; i++) {
+                    if (state->stack_offsets_ebp[i] == disp && state->stack_vals[i] == val && (state->stack_address_state[i] & PS_VALID_VAL) != 0 && state->stack_scope[i] == state->cur_scope) {
+                        loginst(dcontext, 3, inst, " mem location already set to val, simplify ");
+                        backup = INSTR_CREATE_nop(dcontext);
+                        replace_inst(dcontext, state->trace, inst, backup);
+                        loginst(dcontext, 3, backup, " to ");
+                        inst = backup;
+                    }
+                }
+
+                update_stack_val(state, disp, val);
+
+            }
+        }
+    } else {
+        // call and int
+        if (instr_is_interrupt(inst) || instr_is_call(inst)) {
+            // can assume mem addresses not touched??
+            // shouldn't be going to app code for call at least
+            for (i = 0; i < 8; i++)
+                state->reg_state[i] = 0;
+        }
+        // update for regs written to, actually if xh then don't need to 
+        // invalidate xl and vice versa, but to much work to check for that probably unlikely occurence
+        for (i = 0; i < 8; i++) {
+            if (instr_writes_to_reg(inst, REG_START_32 + i)) {
+                state->reg_state[i] = 0;
+            }
+        }
+        // update mem caches
+        for (i = 0; i < num_dst; i++) {
+            opnd = instr_get_dst(inst, i);
+            if (opnd_is_constant_address(opnd) && cp_global_aggr >0) {
+                clear_address_val(state, opnd_get_disp(opnd));
+            }
+        }
+        // update stack cahes
+        for (i = 0; i < num_dst; i++) {
+            opnd = instr_get_dst(inst, i);
+            if (opnd_is_stack_address(opnd) && cp_local_aggr > 0) {
+                clear_stack_val(state, opnd_get_disp(opnd));
+            }
+        }
+    }
+    return inst;
+}
+
+typedef struct _two_entry_list_element_t {
+    int disp;
+    int scope;
+    struct _two_entry_list_element_t * next;
+} two_entry_list_element_t;
+
+typedef struct _start_list_element_t {
+    int endscope;
+    two_entry_list_element_t * next;
+} start_list_element_t;
+
+
+/*************************************************************************/
+
+/*This track the scopes. The number indicates the depth of the nested 
+  scopes. Also checks for stack constant instructions */
+instr_t *
+handle_stack(prop_state_t *state, instr_t *inst)
+{
+    dcontext_t *dcontext = state->dcontext;
+    int i;
+    if (instr_get_opcode(inst) == OP_enter ||
+        ((instr_get_opcode(inst) == OP_mov_st || instr_get_opcode(inst) == OP_mov_ld) && 
+         opnd_is_reg(instr_get_src(inst, 0)) &&
+         opnd_get_reg(instr_get_src(inst, 0)) == REG_ESP &&
+         opnd_is_reg(instr_get_dst(inst, 0)) &&
+         opnd_get_reg(instr_get_dst(inst, 0)) == REG_EBP)) {
+        state->cur_scope++;
+        LOG(THREAD, LOG_OPTS, 3, "Adjust scope up to %d\n", state->cur_scope);
+        return inst; 
+    }
+    if (instr_get_opcode(inst) == OP_leave ||
+        (instr_get_opcode(inst) == OP_pop &&
+         opnd_is_reg(instr_get_dst(inst, 0)) &&
+         opnd_get_reg(instr_get_dst(inst, 0)) == REG_EBP)) { 
+        state->cur_scope--;
+
+        for (i = 0; i < NUM_STACK_SLOTS; i++) {
+            if (state->stack_scope[i] > state->cur_scope && 
+                state->stack_address_state[i] != 0) {
+                state->stack_address_state[i] = 0;
+            }
+        }
+        LOG(THREAD, LOG_OPTS, 3, "Adjust scope down to %d\n", state->cur_scope);
+        return inst; 
+    }
+    if (instr_writes_to_reg(inst, REG_EBP)) {
+        loginst(dcontext, 2, inst, "Lost stack scope count"); 
+        state->lost_scope_count = true;
+        for (i = 0; i< NUM_STACK_SLOTS; i++) {
+            state->stack_address_state[i] = 0;
+        }
+    }
+    return inst;
+}  
+
+
+
+/* FIXME : (could affect correctness at higher levels of optimization)
+ * constant address and various operand sizes, at level 1 what if we write/read
+ * a 16bit value, we'll actually do 32bit, hard to detect since ?believe? will
+ * both be OPSZ_4_short2, look at prefix?  Similarly at level 2 we don't do any size
+ * checking at all for constant address, what if is a byte of existing etc.
+ * Ah, just trust the programmer, these are all addresses from him anyways
+ * we can trust that he won't do anything that wierd with them
+ * FIXME :  more robust matching for the dynamorio stack call hints, the pattern
+ * matching at the moment is somewhat birttle, though should fail gracefully
+ * TODO : (doesn't affect correctness only effectiveness)
+ * Easy
+ *   reverse cmp arg. if one constant and can flip any dependat jmps? will have to
+ *      either check target or assume normal eflags
+ *   if write to xh then don't need to invalidate xl, right now invalidate all
+ *      probably not worth to effort to fix, is pretty rare occurrence where it
+ *      matters
+ *   handle more setcc cmovcc instrs in do_forward_eflags_check, probably
+ *      not worth it, they're almost never used
+ *   have more instrs in simplify figure out their eflags, usually not worth it
+ *   handle more instrs in simplify, (already have all the most common)
+ * Hard
+ *   size issues
+ *   any floating point stuff? probably not feasible or worthwhile
+ */ 
+
+/* performs constant prop, loops through all the instruction updating the
+ * prop state for each one, propagating information collected so far into 
+ * opnds and and calling simplify on the results */
+static void
+constant_propagation(dcontext_t *dcontext, app_pc tag, instrlist_t *trace)
+{
+    prop_state_t state;
+    int num_src, num_dst, i;
+    opnd_t opnd, prop_opnd;
+    instr_t *inst, *backup;
+    
+    bool is_zeroing;
+    /* FIXME: this is a data race!
+     * and why set this for every trace?  options are static!
+     * have some kind of optimize_init to set these
+     */
+    cp_global_aggr = dynamo_options.constant_prop % 10;
+    cp_local_aggr = (dynamo_options.constant_prop - cp_global_aggr) / 10;
+
+    LOG(THREAD, LOG_OPTS, 3,
+        "starting constant prop, global aggresiveness %d local aggresiveness %d\n",
+        cp_global_aggr, cp_local_aggr);
+    DOLOG(4, LOG_OPTS, {
+        instrlist_disassemble(dcontext, tag, trace, THREAD);
+    });
+
+    state.dcontext = dcontext;
+    state.trace = trace;
+    state.hint = NULL;
+    state.lost_scope_count = false;
+
+    state.cur_scope = 0;
+    for (i = 0; i < NUM_CONSTANT_ADDRESS; i++)
+        state.address_state[i] = 0;
+    for (i = 0; i < NUM_STACK_SLOTS; i++)
+        state.stack_address_state[i] = 0;
+
+    for (i = 0; i < 8; i++)
+        state.reg_state[i] = 0;
+
+    get_trace_constant(&state);
+
+    for (inst = instrlist_first(trace); inst != NULL; inst = instr_get_next(inst)) {
+        /* backup in case results turns out to be unencodable */
+        backup = NULL;
+        is_zeroing = false;
+
+        inst = handle_stack(&state, inst);
+        ASSERT(inst != NULL);
+
+        /* propagate to sources */
+        num_src = instr_num_srcs(inst);
+
+        loginst(dcontext, 3, inst, " checking");
+
+        for (i = 0; i < num_src; i++) {
+            opnd = instr_get_src(inst, i);
+
+            if (instr_get_opcode(inst) == OP_lea)
+                prop_opnd = propagate_address(opnd, &state);
+            else {
+                prop_opnd = propagate_opnd(opnd, &state, instr_get_prefix_flag(inst, PREFIX_DATA));
+            }
+            if (!opnd_same(opnd, prop_opnd)) {
+#ifdef DEBUG
+                opt_stats_t.num_opnds_simplified++;
+                logopnd(dcontext, 3, opnd, " old operand");
+                logopnd(dcontext, 3, prop_opnd, " simplified to  ");
+#endif
+                if (backup == NULL)
+                    backup = instr_clone(dcontext, inst);
+                instr_set_src(inst, i, prop_opnd);
+            }
+        }
+        // propagate to dsts, just simplify addresses 
+        num_dst = instr_num_dsts(inst);
+        for (i = 0; i < num_dst; i++) {
+            opnd = instr_get_dst(inst, i);
+            prop_opnd = propagate_address(opnd, &state);
+            if (!opnd_same(opnd, prop_opnd)) {
+#ifdef DEBUG
+                opt_stats_t.num_opnds_simplified++;
+                logopnd(dcontext, 3, opnd, " old operand");
+                logopnd(dcontext, 3, prop_opnd, " simplified to  ");
+#endif
+                if (backup == NULL)
+                    backup = instr_clone(dcontext, inst);
+                instr_set_dst(inst, i, prop_opnd);
+            }
+        }
+
+        /* if acutally propgated any info in, attempt to simplify */
+        if (backup != NULL) {
+            inst = prop_simplify(inst, &state);
+            if (instr_is_encoding_possible(inst)) {
+#ifdef DEBUG
+                opt_stats_t.num_instrs_simplified++;
+                loginst(dcontext, 3, backup, " old instruction");
+                loginst(dcontext, 3, inst, " simplified to  ");
+#endif
+                instr_destroy(dcontext, backup);
+                backup = NULL;
+            } else {
+#ifdef DEBUG
+                opt_stats_t.num_fail_simplified++;
+                loginst(dcontext, 3, backup, " was unable to simplify ");
+                loginst(dcontext, 3, inst, " to this ");
+#endif
+                replace_inst(dcontext, trace, inst, backup);
+                inst = backup;
+                backup = NULL;
+            }
+
+
+        }
+
+
+        // update prop state
+        if (state.hint == NULL)
+            inst = update_prop_state(&state, inst, true);
+        else {
+            loginst(dcontext, 3, state.hint, " using hint instruction instead of actual to update prop state ");
+            update_prop_state(&state, state.hint, false);
+            instr_destroy(dcontext, state.hint);
+            state.hint = NULL;
+        }
+    }
+    
+#ifdef DEBUG
+    LOG(THREAD, LOG_OPTS, 3, "done constant prop\n");
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_OPTS) != 0)
+        instrlist_disassemble(dcontext, tag, trace, THREAD);
+#endif
+}
+
+/***************************************************************************/
+/* from Tim */
+/* remove unnecsary zeroing */
+/* removes unnecesary zeroing
+ * This may not always be faster since
+ * the pentium 4 hardware (and maby earlier versions too) recognizes
+ * xor zeroring specially and uses it to break the false dependences
+ *
+ * ignores effects on flags, should perhaps consider them for correctness
+ * but one would not expect a program to use the flags set zeroing a register
+ * for a conditional jmp
+ *
+ * should also catch the adobe case where we for ex. 
+ * xor zero eax, load into ah, use eax, xor zero eax, load into ah ...
+ * 
+ * relies to some degree on the enum in instr.h 
+ */
+static void
+remove_unnecessary_zeroing(dcontext_t *dcontext, app_pc tag, instrlist_t *trace)
+{
+    instr_t *inst, *next_inst;
+    int i, cur_reg, num_dsts;
+    /* keeps track if actually neccesary to mark off dst of non zeroing instructions */
+    bool check_dsts; 
+    bool zeroed[24];
+    opnd_t dst;
+    check_dsts = false;
+    for (i = 0; i < 24; i++) {
+        zeroed[i] = false;
+    }
+    for (inst = instrlist_first(trace); inst != NULL; inst = next_inst) {
+        next_inst = instr_get_next(inst);
+        if (is_zeroing_instr(inst)) {
+            cur_reg = opnd_get_reg(instr_get_dst(inst,0)) - REG_START_32;
+            /* if zeroed (and also of all sub registers) then kill the inst 
+             * otherwise mark reg and sub regs as zeroed  */
+            if (check_down(zeroed, cur_reg)) {
+                /* is ok to remove instruction reg and subregs already zero */
+#ifdef DEBUG
+                loginst(dcontext, 3, inst, "unnecsary xor removed ");
+                opt_stats_t.xors_removed++;
+#endif
+                remove_inst(dcontext, trace, inst);
+            }
+            else {
+                propagate_down(zeroed, cur_reg, true); 
+                check_dsts = true;
+            }
+        }
+        else {
+            /* non-zeroing instruction, check for registers being written 
+             * and mark them non-zero if neccesary*/
+            if (check_dsts) {
+                num_dsts = instr_num_dsts(inst);
+                for (i = 0; i < num_dsts; i++) {
+                    dst = instr_get_dst(inst, i);
+                    if (opnd_is_reg(dst)) {
+                        cur_reg = opnd_get_reg(dst) - REG_START_32;
+                        propagate_down(zeroed, cur_reg, false);
+                    }   
+                }
+                check_dsts = false;
+                for (i = 0; i < 24; i++)
+                    check_dsts |= zeroed[i];
+            }
+        }
+    }
+}
+
+/****************************************************************************/
+/* from Tim */
+/* removes dead code
+ * removes some nops (that use dead registers) but not all
+ * relies to some degree on the ordering of enum reg in instr.h
+ */
+
+#define NUM_ADD_CACHE 16 
+#define ADD_KEEP 0x01
+#define ADD_DEAD 0x02
+
+static int dc_global_aggr;
+static int dc_local_aggr;
+
+static void
+add_address(dcontext_t *dcontext, int address, byte flag, int *adds, byte *flags)
+{
+    bool cont = true;
+    int i;
+    for (i = 0; (i < NUM_ADD_CACHE) && cont; i++) {
+        if (adds[i] == 0) {
+            adds[i] = address;
+            flags[i] = flag;
+            cont = false;
+        }
+    }
+    if (cont) {
+        LOG(THREAD, LOG_OPTS, 3, "constant address cache overflow\n");
+        i = address % NUM_ADD_CACHE;
+        adds[i] = address;
+        flags[i] = flag;
+    }
+    LOG(THREAD, LOG_OPTS, 3, " load const cache add: "PFX"  flags 0x%02x\n", address, flag);
+}
+
+static bool
+address_is_dead(dcontext_t *dcontext, int address, int *adds, byte *flags)
+{
+    int i = 0;
+    for (; i < NUM_ADD_CACHE; i++) 
+        if (adds[i] == address && (flags[i] & ADD_DEAD) != 0) 
+            return true;
+    return false;
+}
+
+static void
+address_set_dead(dcontext_t *dcontext, int address, int *adds, byte *flags, bool dead)
+{
+    int i = 0;
+    for (; i < NUM_ADD_CACHE; i++) {
+        if (adds[i] == address) {
+            if (dead)
+                flags[i] |= ADD_DEAD;
+            else
+                flags[i] &= (~ADD_DEAD);
+            return;
+        }
+    }
+    if (dc_global_aggr > 2) {
+        if (dead)
+            add_address(dcontext, address, ADD_DEAD, adds, flags);
+    }
+}
+
+static void
+add_init(dcontext_t *dcontext, int *addresses, byte *flags)
+{
+    /* can add all dynamo addresses here, they are never aliased so always */
+    /* safe to optimize, but takes up space in our cache, with new jump code */
+    /* probably only use exc so just put it in, and maby eax to since is fav */
+    /* when need to store flags/pass arg, can always add more location later */
+    /* probably cleaner way of getting addresses but who cares for now */
+    add_address(dcontext, opnd_get_disp(opnd_create_dcontext_field(dcontext, XCX_OFFSET)), ADD_KEEP, addresses, flags);
+    add_address(dcontext, opnd_get_disp(opnd_create_dcontext_field(dcontext, XAX_OFFSET)), ADD_KEEP, addresses, flags);
+}
+
+#if 0 /* not used */
+static void
+add_stack_address(dcontext_t *dcontext, int address, byte flag, int scope, int *adds, byte *flags, int *scopes)
+{
+    bool cont = true;
+    int i;
+    for (i = 0; (i < NUM_STACK_SLOTS) && cont; i++) {
+        if (adds[i] == 0) {
+            adds[i] = address;
+            scopes[i] = scope;
+            flags[i] = flag;
+            cont = false;
+        }
+    }
+    if (cont) {
+        LOG(THREAD, LOG_OPTS, 3, "constant address cache overflow\n");
+        i = address % NUM_STACK_SLOTS;
+        adds[i] = address;
+        flags[i] = flag;
+        scopes[i] = scope;
+    }
+    LOG(THREAD, LOG_OPTS, 3, " stack const cache add: "PFX"  flags 0x%02x\n", address, flag);
+}
+#endif
+
+static bool
+stack_address_is_dead(dcontext_t *dcontext, int address, int scope, int *adds, byte *flags, int *scopes)
+{
+    int i;
+    for (i = 0; i < NUM_STACK_SLOTS; i++) {
+        if (adds[i] == address && scopes[i] == scope && (flags[i] & ADD_DEAD) != 0) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static void
+stack_address_set_dead(dcontext_t *dcontext, int address, int scope, int *adds, byte *flags, int *scopes, bool dead)
+{
+    int i;
+    for (i = 0; i < NUM_STACK_SLOTS; i++) {
+        if (adds[i] == address && scopes[i] == scope && flags[i] != 0) {
+            if (dead)
+                flags[i] |= ADD_DEAD;
+            else
+                flags[i] &= (~ADD_DEAD);
+            return;
+        }
+    }
+}
+
+void
+remove_dead_code(dcontext_t *dcontext, app_pc tag, instrlist_t *trace)
+{
+    instr_t *inst, *prev_inst, *ecx_load;
+    opnd_t dst, src;
+    bool free[24];
+    int addresses[NUM_ADD_CACHE];
+    byte address_state[NUM_ADD_CACHE];
+    
+    int stack_scope[NUM_STACK_SLOTS];
+    int stack_offsets_ebp[NUM_STACK_SLOTS];
+    byte stack_state[NUM_STACK_SLOTS];
+    int scope = 0; /* good as any default */
+    
+    int i, j, dst_reg, opcode, num_dsts, num_srcs, src_reg;
+    uint eflags;
+    bool killinst;
+    bool kill_ecx_load = false;
+
+    dc_global_aggr = dynamo_options.remove_dead_code % 10;
+    dc_local_aggr = (dynamo_options.remove_dead_code - dc_global_aggr) / 10;
+#ifdef DEBUG
+    LOG(THREAD, LOG_OPTS, 3, "removing dead loads, global aggressiveness %d local aggressiveness %d\n", dc_global_aggr, dc_local_aggr);
+    if (stats->loglevel >= 4 && (stats->logmask & LOG_OPTS) != 0)
+        instrlist_disassemble(dcontext, tag, trace, THREAD);
+#endif
+    /* initialize */
+
+    eflags = EFLAGS_READ_ALL;
+    for (i = 0; i < 24; i++)
+        free[i] = false;
+    for (i = 0; i < NUM_ADD_CACHE; i++) {
+        addresses[i] = 0;
+        address_state[i] = 0;
+    }
+    for (i = 0; i < NUM_STACK_SLOTS; i++) {
+        stack_offsets_ebp[i] = 0;
+        stack_scope[i] = 0;
+        stack_state[i] = 0;
+    }
+    add_init(dcontext, addresses, address_state);
+    ecx_load = NULL;
+
+    /* main loop runs from bottom of trace to top */
+    for (inst = instrlist_last(trace); inst != NULL; inst = prev_inst) {
+        prev_inst = instr_get_prev(inst);
+        loginst(dcontext,3,inst,"remove_dead_code working on:");
+        if (instr_is_cti(inst) || instr_is_interrupt(inst)) {
+            /* perhaps to a bit of multi-trace search here to see if really 
+             * neccesary to mark all flags and regs as live when hit cti? */
+            eflags = EFLAGS_READ_ALL;
+            for (i = 0; i < 24; i++)
+                free[i] = false;
+            for (i = 0; i < NUM_ADD_CACHE; i++) {
+                if ((address_state[i] & ADD_KEEP) != 0)
+                    address_state[i] &= (~ADD_DEAD);
+                else {
+                    address_state[i] = 0;
+                    addresses[i] = 0;
+                }
+            }
+            for (i = 0; i < NUM_STACK_SLOTS; i++) {
+                if ((stack_state[i] & ADD_KEEP) != 0)
+                    stack_state[i] &= (~ADD_DEAD);
+                else {
+                    stack_state[i] = 0;
+                    stack_offsets_ebp[i] = 0; stack_scope[i]=0;
+                }
+            }
+            ecx_load = NULL;
+            /* skip over the bit of control flow in indirect branch */
+            if (instr_get_opcode(inst) == OP_jmp && prev_inst != NULL &&
+                instr_get_prev(prev_inst) != NULL &&
+                instr_get_opcode(instr_get_prev(inst)) == OP_jecxz)
+                prev_inst = instr_get_prev(instr_get_prev(prev_inst)); 
+        } else {
+            // default to removing instruction, then see if we need it
+            killinst = true;
+            opcode = instr_get_opcode(inst);
+            num_dsts = instr_num_dsts(inst);
+            num_srcs = instr_num_srcs(inst);
+            
+            /* This track the scopes. The number indicates the depth of the 
+             * nested scopes. If it is 0, then we are in the original scope 
+             * and stack optimization can be done.
+             */
+            
+            if (opcode == OP_leave || 
+                (opcode == OP_pop &&
+                 opnd_is_reg(instr_get_dst(inst, 0)) &&
+                 opnd_get_reg(instr_get_dst(inst, 0))== REG_EBP)) { 
+                scope++;
+                LOG(THREAD, LOG_OPTS, 3, "cur scope + to %d\n", scope);
+            } else {
+                if (opcode == OP_enter ||
+                    ((opcode == OP_mov_st || opcode == OP_mov_ld) &&
+                     opnd_is_reg(instr_get_src(inst, 0)) &&
+                     opnd_get_reg(instr_get_src(inst, 0)) == REG_ESP &&
+                     opnd_is_reg(instr_get_dst(inst, 0)) &&
+                     opnd_get_reg(instr_get_dst(inst, 0)) == REG_EBP)) {
+                    scope--;
+                    LOG(THREAD, LOG_OPTS, 3, "cur scope - to %d\n", scope);
+                    for (i = 0; i < NUM_STACK_SLOTS; i++) {
+                        if ((stack_scope[i] > scope) && (stack_state[i] != 0)) {
+                            stack_state[i] = 0;
+                        }
+                    }    
+                } else {
+                    if (instr_writes_to_reg(inst, REG_EBP)) {
+                        LOG(THREAD, LOG_OPTS, 2, "dead code lost count of scope nesting, clearing cache\n");
+                        for (i = 0; i < NUM_STACK_SLOTS; i++)  
+                            stack_state[i] = 0;
+                    }
+                }
+            }
+            
+            /* only eliminate instructions that have a destination or are known to eliminable */
+            /* believe? that any instr with at least 1 dst has no other effects beside that */
+            /* dst and eflags.  Allow test, cmp, sahf to be killed */
+            killinst = killinst && !((num_dsts == 0) && (opcode != OP_sahf) && 
+                                     (opcode != OP_cmp) && (opcode != OP_test));
+            /* check that all destinations are dead, (also not mem etc.) */
+            for (i = 0; (i < num_dsts) && killinst; i++) {
+                dst = instr_get_dst(inst, i);
+                if (opnd_is_reg(dst)) {
+                    dst_reg = opnd_get_reg(dst) - REG_START_32;
+                    killinst = killinst && check_down(free, dst_reg);
+                } else {
+                    if (opnd_is_constant_address(dst)) {
+                        if (ecx_load != NULL && is_store_to_ecxoff(dcontext, inst)) {
+                            killinst = true;
+                            if (kill_ecx_load) {
+#ifdef DEBUG
+                                opt_stats_t.dead_loads_removed++;
+                                loginst(dcontext, 3, ecx_load, "removed dead code ");
+#endif
+                                remove_inst(dcontext, trace, ecx_load);
+                            }
+                        } else {
+                            killinst = killinst && address_is_dead(dcontext, opnd_get_disp(dst), addresses, address_state);
+                        }
+                    } else {
+                        if (opnd_is_stack_address(dst)) {
+                            killinst = killinst && stack_address_is_dead(dcontext, opnd_get_disp(dst), scope, stack_offsets_ebp, stack_state, stack_scope);
+                        } else {
+                            killinst = false;
+                        }
+                    }
+                }
+            }
+            /* check flags if might still be killable */
+            killinst = killinst && 
+                ((EFLAGS_WRITE_TO_READ(instr_get_eflags(inst) & EFLAGS_WRITE_ALL) & 
+                  eflags) == 0);
+            /* always kill if nop */
+            killinst = killinst || is_nop(inst);
+            /* check ecx load */
+            if (is_load_from_ecxoff(dcontext, inst)) {
+                ecx_load = inst;
+                kill_ecx_load = !killinst;
+            }
+            if (killinst) {
+                /* delete the instruction */
+#ifdef DEBUG
+                opt_stats_t.dead_loads_removed++;
+                loginst(dcontext, 3, inst, "removed dead code ");
+#endif
+                remove_inst(dcontext, trace, inst);
+            }
+            else {
+                /* can't be killed so add dependencies */
+                /* add flag constraints */
+                eflags &= ~EFLAGS_WRITE_TO_READ(instr_get_eflags(inst) & 
+                                                EFLAGS_WRITE_ALL);
+                eflags |= instr_get_eflags(inst) & EFLAGS_READ_ALL;
+                // mark destinations as free
+                for (i = 0; i < num_dsts; i++) {
+                    dst = instr_get_dst(inst, i);
+                    if (opnd_is_reg(dst)) {
+                        /* mark dst reg and sub regs as free */
+                        dst_reg = opnd_get_reg(dst) - REG_START_32;
+                        propagate_down(free, dst_reg, true);
+                    }
+                    else {
+                        if (opnd_is_constant_address(dst)) {
+                            address_set_dead(dcontext, opnd_get_disp(dst), 
+                                             addresses, address_state, true);
+                        } else {
+                            if (opnd_is_stack_address(dst)) {
+                                stack_address_set_dead(dcontext, opnd_get_disp(dst), scope, stack_offsets_ebp, stack_state, stack_scope, true); 
+                            } 
+                            /* reg used in address mark as unfree */
+                            for (j=opnd_num_regs_used(dst)-1; j>=0; j--) {
+                                dst_reg = opnd_get_reg_used(dst, j) - REG_START_32;
+                                propagate_down(free, dst_reg, false);
+                            }
+                        }
+                    }
+                }
+                // don't propagate srcs if zeroing instr 
+                // mark sources as needed
+                if (!is_zeroing_instr(inst)) {
+                    for (i = 0; i < num_srcs; i++) {
+                        /* mark src regs and sub regs not free */
+                        src = instr_get_src(inst, i);
+                        if (opnd_is_constant_address(src)) {
+                            address_set_dead(dcontext, opnd_get_disp(src), 
+                                             addresses, address_state, false);
+                        } else {
+                            if (opnd_is_stack_address(src)) {
+                                stack_address_set_dead(dcontext, opnd_get_disp(src), scope, stack_offsets_ebp, stack_state, stack_scope, false); 
+                            } 
+                            for (j=opnd_num_regs_used(src)-1; j>=0; j--) {
+                                src_reg = opnd_get_reg_used(src, j) - REG_START_32;
+                                propagate_down(free, src_reg, false);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+#ifdef DEBUG
+    LOG(THREAD, LOG_OPTS, 3, "done removing dead code\n");
+    if (stats->loglevel >= 4 && (stats->logmask & LOG_OPTS) != 0)
+        instrlist_disassemble(dcontext, tag, trace, THREAD);
+#endif
+}
+
+/***************************************************************************/
+/* from Tim */
+/* attempts to combine multiple adjustments of the esp register into a 
+ * single adjustment, might eventually be usefull for locations, or as a
+ * tool for inlining, but is inspired by ocaml and tinyvm code which have
+ * a lot of manipulation of the stack without much actual use of it once
+ * the other passes have finished
+ *
+ * Most esp manipulation is the result of inlined calls
+ * in general have to worry about amount of space allocated on the stack etc.
+ */
+
+static bool 
+is_stack_adjustment(instr_t *inst)
+{
+    int opcode = instr_get_opcode(inst);
+    return (
+            ((opcode == OP_add || opcode == OP_sub) &&
+             opnd_is_reg(instr_get_dst(inst, 0)) &&
+             opnd_get_reg(instr_get_dst(inst, 0)) == REG_ESP &&
+             opnd_is_immed_int(instr_get_src(inst, 0))) ||
+
+            (opcode == OP_lea && 
+             opnd_get_reg(instr_get_dst(inst, 0)) == REG_ESP && 
+             ((opnd_get_base(instr_get_src(inst, 0)) == REG_ESP && 
+               opnd_get_index(instr_get_src(inst, 0)) == REG_NULL ) ||
+              (opnd_get_base(instr_get_src(inst, 0)) == REG_NULL &&
+               opnd_get_index(instr_get_src(inst, 0)) == REG_ESP &&
+               opnd_get_scale(instr_get_src(inst, 0)) == 1))));
+}
+
+static int 
+get_stack_adjustment(instr_t *inst)
+{
+    int opcode = instr_get_opcode(inst);
+    if (opcode == OP_add)
+        return (int) opnd_get_immed_int(instr_get_src(inst, 0));
+    if (opcode == OP_sub)
+        return -(int) opnd_get_immed_int(instr_get_src(inst, 0));
+    if (opcode == OP_lea)
+        return opnd_get_disp(instr_get_src(inst, 0));
+    else
+        return -1;
+}
+
+static void 
+set_stack_adjustment(instr_t *inst, int adjust)
+{
+    int opcode = instr_get_opcode(inst);
+    opnd_t temp_opnd;
+    if (opcode == OP_lea) {
+        instr_set_src(inst, 0, opnd_create_base_disp(REG_ESP, REG_NULL, 0, adjust, OPSZ_lea));
+        return;
+    }
+    if (opcode == OP_sub)
+        adjust = -adjust;
+    if (adjust < -128 || adjust > 127)
+        temp_opnd = OPND_CREATE_INT32(adjust);
+    else
+        temp_opnd = OPND_CREATE_INT8(adjust);
+    instr_set_src(inst, 0, temp_opnd);
+}
+        
+
+static void 
+stack_adjust_combiner(dcontext_t *dcontext, app_pc tag, instrlist_t *trace)
+{
+    instr_t *inst, *next, *first_adjust, *last_adjust;
+    int opcode, adj, max_off=0, cur_off=0, first_off=0;
+#ifdef DEBUG
+    LOG(THREAD, LOG_OPTS, 3, "combining stack adjustments\n");
+    if (stats->loglevel >= 4 && (stats->logmask & LOG_OPTS) != 0)
+        instrlist_disassemble(dcontext, tag, trace, THREAD);
+#endif
+    last_adjust = NULL;
+    first_adjust = NULL;
+    for (inst = instrlist_first(trace); inst != NULL; inst = next) {
+        next = instr_get_next(inst); /* in case we destroy inst */
+#ifdef DEBUG
+        instr_decode(dcontext, inst);
+        loginst(dcontext,3,inst,"stack adjust considering");
+#endif
+        if (first_adjust == NULL) {
+            if (is_stack_adjustment(inst)) { 
+                first_adjust = inst;
+                first_off = get_stack_adjustment(inst);
+                cur_off = first_off;
+                LOG(THREAD, LOG_OPTS, 3, "  found starting stack adjust, offset %d\n", cur_off);
+                max_off = 1000000; /* something large */
+            } 
+        } else {
+            /* see if we can fold in another adjustment */
+            if (is_stack_adjustment(inst)) {
+                adj = get_stack_adjustment(inst);
+                cur_off += adj;
+                LOG(THREAD, LOG_OPTS, 3, "  found stack adjust adjust by: %d, current offset now: %d\n", adj, cur_off);
+                if (last_adjust != NULL) {
+#ifdef DEBUG
+                    loginst(dcontext, 3, last_adjust, "  removing old last adjustment");
+                    opt_stats_t.num_stack_adjust_removed++;
+#endif
+                    ASSERT(cur_off % 4 == 0);
+                    remove_inst(dcontext, trace, last_adjust);
+                    last_adjust = inst;
+                } else {
+                    last_adjust = inst;
+                }
+                continue;
+            }
+            opcode = instr_get_opcode(inst);
+            /* if instr depends on ESP or leaves trace must restore */
+            /* include interrupt and call, though may not be neccesary */
+            /* could mangle pushes and pops instead of restoring, is */
+            /* helpfull?, check for store to ecx_off, might mangle indirect */
+            /* macro's by inserting a clean up instruction */
+            if (!instr_uses_reg(inst, REG_ESP) && !instr_is_cti(inst) && 
+                !instr_is_interrupt(inst) && !instr_is_call(inst)) {
+                /* skip writes to constant address, presume that they will never be stack */
+                if ((opcode == OP_mov_st || opcode == OP_mov_imm) && 
+                    opnd_is_constant_address(instr_get_dst(inst, 0))) {
+                    LOG(THREAD, LOG_OPTS, 3, "store to constant mem, skipping\n");
+                    continue;
+                }
+                if (instr_writes_memory(inst)) {
+                    /* could be write to the stack, since can't tell in general (aliases) */
+                    /* make sure if the cur_off is negative (allocating space on the stack) */
+                    /* we don't eventually set the offset of the last adjust to reserve less */
+                    /* space then that */
+                    LOG(THREAD, LOG_OPTS, 3, "write to memory");
+                    if (cur_off < max_off) {
+                        LOG(THREAD, LOG_OPTS, 3, "\ncurrent offset %d, less than max offset %d, setting max offset to current offset\n", cur_off, max_off); 
+                        max_off = cur_off;
+                    }
+                }
+                LOG(THREAD, LOG_OPTS, 3, "skipping\n");
+                continue;
+            }
+            /* fixing up */
+            LOG(THREAD, LOG_OPTS, 3, "reached stopping point, clean up\n"); 
+            if (max_off < cur_off) {
+                LOG(THREAD, LOG_OPTS, 3, "  max offset is less than current off set, set and fix\n"); 
+                /* need to be sure to allocate enough space on stack at beginning */
+                loginst(dcontext, 3, first_adjust, "  replacing initial adjustment");
+                set_stack_adjustment(first_adjust, max_off);
+                loginst(dcontext, 3, first_adjust, "  with");
+                ASSERT(max_off % 4 == 0);
+                /* protect eflags use lea */
+                loginst(dcontext, 3, last_adjust, "  replacing last adjustment");
+                set_stack_adjustment(last_adjust, cur_off - max_off);
+                loginst(dcontext, 3, last_adjust, "  with");
+                ASSERT((cur_off - max_off) % 4 == 0);
+            } else {
+                if (cur_off == 0) {
+                    /* remove initial and last adjustment */
+#ifdef DEBUG
+                    opt_stats_t.num_stack_adjust_removed++;
+                    loginst(dcontext, 3, first_adjust, "  curent offset = 0 removing initial adjustment");
+                    if (last_adjust != NULL) {
+                        opt_stats_t.num_stack_adjust_removed++;
+                        loginst(dcontext, 3, last_adjust, "  curent offset = 0 removing last adjustment");
+                    }
+#endif
+                    remove_inst(dcontext, trace, first_adjust);
+                    if (last_adjust != NULL)
+                        remove_inst(dcontext, trace, last_adjust); 
+                } else {
+                    /* change adjustment if neccesary */
+                    if (first_off != cur_off) {
+                        LOG(THREAD, LOG_OPTS, 3, "  current offset %d, initial offset %d\n", cur_off, first_off);
+                        loginst(dcontext, 3, first_adjust, "  replacing intial adjustment");
+                        set_stack_adjustment(first_adjust, cur_off);
+                        loginst(dcontext, 3, first_adjust, "  with");
+                        ASSERT(cur_off % 4 == 0);
+                    } else {
+                        LOG(THREAD, LOG_OPTS, 3, "  current offset = last offset = %d, no change needed\n", cur_off);
+                    }
+                    /* remove last adjust */
+                    if (last_adjust != NULL) {
+#ifdef DEBUG
+                        opt_stats_t.num_stack_adjust_removed++;
+                        loginst(dcontext, 3, last_adjust, "  removing last adjustment");
+#endif
+                        remove_inst(dcontext, trace, last_adjust);
+                    } 
+                }
+            }
+            last_adjust = NULL;
+            first_adjust = NULL;
+        }
+    }
+#ifdef DEBUG
+    LOG(THREAD, LOG_OPTS, 3, "done combining stack adjustments\n");
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_OPTS) != 0)
+        instrlist_disassemble(dcontext, tag, trace, THREAD);
+#endif
+}
+
+
+
+/****************************************************************************/
+/* call return matching, attempts to match calls with their corresponding
+ * returns, may not always be safe
+ */
+
+/* checks to see if the eflags are written before they are read */
+static bool 
+check_eflags_cr(instr_t *inst)
+{
+    uint eflags = EFLAGS_READ_6;
+    uint inst_eflags;
+    for (; inst != NULL; inst = instr_get_next(inst)) {
+        if (instr_is_cti(inst) || instr_is_interrupt(inst))
+            return false;
+        inst_eflags = instr_get_eflags(inst);
+        if ((eflags & inst_eflags) != 0)
+            return false;
+        eflags &= ~(EFLAGS_WRITE_TO_READ(inst_eflags));
+        if (eflags == 0)
+            return true;
+    }
+    return false;
+}
+
+/* removes the return code, pattern matches on our return macro */
+static instr_t *
+remove_return_no_save_eflags(dcontext_t *dcontext, instrlist_t *trace, instr_t *inst)
+{
+    instr_t *inst2;
+    int to_pop = 4;
+    opnd_t replacement;
+#ifdef DEBUG
+    opt_stats_t.num_returns_removed++;
+    opt_stats_t.num_return_instrs_removed += 4;
+#endif
+
+    inst2 = instr_get_next(inst);
+    ASSERT(instr_get_opcode(inst) == OP_mov_st);
+    loginst(dcontext, 3, inst, "removing");
+    remove_inst(dcontext, trace, inst);
+    inst = inst2;
+
+    inst2 = instr_get_next(inst);
+    ASSERT(instr_get_opcode(inst) == OP_pop);
+    loginst(dcontext, 3, inst, "removing");
+    remove_inst(dcontext, trace, inst);
+    inst = inst2;
+
+    inst2 = instr_get_next(inst);
+    if (instr_get_opcode(inst) == OP_lea) {
+        /* this popping of the stack, lea */
+        to_pop += opnd_get_disp(instr_get_src(inst, 0));
+        loginst(dcontext, 3, inst, "removing");
+        remove_inst(dcontext, trace, inst);
+        inst = inst2;
+#ifdef DEBUG
+        opt_stats_t.num_return_instrs_removed++;
+#endif
+    }
+
+    inst2 = instr_get_next(inst);
+    ASSERT(instr_get_opcode(inst) == OP_cmp);
+    loginst(dcontext, 3, inst, "removing");
+    remove_inst(dcontext, trace, inst);
+    inst = inst2;
+
+    inst2 = instr_get_next(inst);
+    ASSERT(instr_get_opcode(inst) == OP_jne);
+    loginst(dcontext, 3, inst, "removing");
+    remove_inst(dcontext, trace, inst);
+    inst = inst2;
+
+    inst2 = instr_get_next(inst);
+    ASSERT(instr_get_opcode(inst) == OP_mov_ld);
+    loginst(dcontext, 3, inst, "removing");
+    remove_inst(dcontext, trace, inst);
+    inst = inst2;
+
+    /* check for add here, is not uncommon to pop off the args after a return, if so */
+    /* can save an instruction */
+    if ((instr_get_opcode(inst) == OP_add) && opnd_is_reg(instr_get_dst(inst, 0)) && (opnd_get_reg(instr_get_dst(inst, 0)) == REG_ESP) && opnd_is_immed_int(instr_get_src(inst, 0))) {
+        to_pop += (int) opnd_get_immed_int(instr_get_src(inst, 0));
+#ifdef DEBUG
+        opt_stats_t.num_return_instrs_removed++;
+#endif
+        if (to_pop == 0) {
+#ifdef DEBUG
+            opt_stats_t.num_return_instrs_removed++;
+#endif
+            return inst;
+        }
+        if ((to_pop <= 127) && (to_pop >= -128)) {
+            replacement = OPND_CREATE_INT8(to_pop);
+        } else {
+            replacement = OPND_CREATE_INT32(to_pop);
+        }
+        loginst(dcontext, 3, inst, " updating stack adjustment :");
+        instr_set_src(inst, 0, replacement);
+        loginst(dcontext, 3, inst, " to :");
+        return inst;
+    }
+    if ((to_pop <= 127) && (to_pop >= -128)) {
+        replacement = OPND_CREATE_INT8(to_pop);
+    } else {
+        replacement = OPND_CREATE_INT32(to_pop);
+    }
+    inst2 = INSTR_CREATE_add(dcontext, opnd_create_reg(REG_ESP), replacement);
+    loginst(dcontext, 3, inst2, "adjusting stack");
+    instrlist_preinsert(trace, inst, inst2);
+    return inst2;
+}
+
+/* removes the return code, pattern matches on our return macro */
+static instr_t *
+remove_return(dcontext_t *dcontext, instrlist_t *trace, instr_t *inst)
+{
+    if (!INTERNAL_OPTION(unsafe_ignore_eflags_trace)) {
+        instr_t *inst2;
+        int to_pop = 4;
+        opnd_t replacement;
+#ifdef DEBUG
+        opt_stats_t.num_returns_removed++;
+        opt_stats_t.num_return_instrs_removed += 6;
+#endif
+
+        inst2 = instr_get_next(inst);
+        ASSERT(instr_get_opcode(inst) == OP_mov_st);
+        loginst(dcontext, 3, inst, "removing");
+        remove_inst(dcontext, trace, inst);
+        inst = inst2;
+
+        inst2 = instr_get_next(inst);
+        ASSERT(instr_get_opcode(inst) == OP_pop);
+        loginst(dcontext, 3, inst, "removing");
+        remove_inst(dcontext, trace, inst);
+        inst = inst2;
+
+        inst2 = instr_get_next(inst);
+        if (instr_get_opcode(inst2) == OP_lea) {
+            /* this popping of the stack, lea */
+            to_pop += opnd_get_disp(instr_get_src(inst, 0));
+            loginst(dcontext, 3, inst, "removing");
+            remove_inst(dcontext, trace, inst);
+            inst = inst2;
+#ifdef DEBUG
+            opt_stats_t.num_return_instrs_removed++;
+#endif
+        }
+
+        inst2 = instr_get_next(inst);
+        ASSERT(instr_get_opcode(inst) == OP_lea);
+        loginst(dcontext, 3, inst, "removing");
+        remove_inst(dcontext, trace, inst);
+        inst = inst2;
+
+        inst2 = instr_get_next(inst);
+        ASSERT(instr_get_opcode(inst) == OP_jecxz);
+        loginst(dcontext, 3, inst, "removing");
+        remove_inst(dcontext, trace, inst);
+        inst = inst2;
+
+        inst2 = instr_get_next(inst);
+        ASSERT(instr_get_opcode(inst) == OP_lea);
+        loginst(dcontext, 3, inst, "removing");
+        remove_inst(dcontext, trace, inst);
+        inst = inst2;
+
+        inst2 = instr_get_next(inst);
+        ASSERT(instr_get_opcode(inst) == OP_jmp);
+        loginst(dcontext, 3, inst, "removing");
+        remove_inst(dcontext, trace, inst);
+        inst = inst2;
+
+        inst2 = instr_get_next(inst);
+        ASSERT(instr_get_opcode(inst) == OP_mov_ld);
+        loginst(dcontext, 3, inst, "removing");
+        remove_inst(dcontext, trace, inst);
+        inst = inst2;
+
+        /* check for add here, is not uncommon to pop off the args after a return,
+         * if so  can save an instruction */
+        if ((instr_get_opcode(inst) == OP_add) &&
+            opnd_is_reg(instr_get_dst(inst, 0)) &&
+            (opnd_get_reg(instr_get_dst(inst, 0)) == REG_ESP) &&
+            opnd_is_immed_int(instr_get_src(inst, 0))) {
+            to_pop += (int) opnd_get_immed_int(instr_get_src(inst, 0));
+#ifdef DEBUG
+            opt_stats_t.num_return_instrs_removed++;
+#endif
+            if (to_pop == 0) {
+#ifdef DEBUG
+                opt_stats_t.num_return_instrs_removed++;
+#endif
+                return inst;
+            }
+            if ((to_pop <= 127) && (to_pop >= -128)) {
+                replacement = OPND_CREATE_INT8(to_pop);
+            } else {
+                replacement = OPND_CREATE_INT32(to_pop);
+            }
+            loginst(dcontext, 3, inst, " updating stack adjustment :");
+            instr_set_src(inst, 0, replacement);
+            loginst(dcontext, 3, inst, " to :");
+            return inst;
+        }
+        if (check_eflags_cr(inst)) {
+            if ((to_pop <= 127) && (to_pop >= -128)) {
+                replacement = OPND_CREATE_INT8(to_pop);
+            } else {
+                replacement = OPND_CREATE_INT32(to_pop);
+            }
+            inst2 =
+                INSTR_CREATE_add(dcontext, opnd_create_reg(REG_ESP), replacement);
+        }
+        else {
+            LOG(THREAD, LOG_OPTS, 3, "Forward eflags check failed using lea to adjust stack instead of add");
+            inst2 =
+                INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ESP),
+                                 opnd_create_base_disp(REG_ESP, REG_NULL, 0,
+                                                       to_pop, OPSZ_lea));
+        }
+        loginst(dcontext, 3, inst2, "adjusting stack");
+        instrlist_preinsert(trace, inst, inst2);
+        return inst2;
+    }
+    else
+        return remove_return_no_save_eflags(dcontext, trace, inst);
+}
+
+/* for some reason can't get instr_same / opnd_same to work for the below so
+ *  just write it out, returns true if the next instruction is likely the pop
+ *  of a return */
+static bool 
+is_return(dcontext_t *dcontext, instr_t *inst)
+{
+    instr_t *pop = instr_get_next(inst);
+    if (!INTERNAL_OPTION(unsafe_ignore_eflags_trace)) {
+        instr_t *lea, *jecxz;
+        if (pop == NULL)
+            return false;
+        lea = instr_get_next(pop);
+        if (lea == NULL)
+            return false;
+        jecxz = instr_get_next(lea);
+        if (jecxz == NULL)
+            return false;
+        if (instr_get_opcode(jecxz) == OP_lea)
+            jecxz = instr_get_next(jecxz); 
+        return (jecxz != NULL && 
+                instr_get_opcode(pop) == OP_pop &&
+                opnd_get_reg(instr_get_dst(pop, 0)) == REG_ECX &&
+                instr_get_opcode(jecxz) == OP_jecxz);
+    }
+    else {
+        instr_t *lea, *cmp, *jne;
+        if (pop == NULL)
+            return false;
+        lea = instr_get_next(pop);
+        if (lea == NULL)
+            return false;
+        if (instr_get_opcode(lea) != OP_lea && instr_get_opcode(lea) != OP_cmp)
+            return false;
+        if (instr_get_opcode(lea) == OP_cmp)
+            cmp = lea;
+        else
+            cmp = instr_get_next(lea);
+        if (cmp == NULL)
+            return false;
+        jne = instr_get_next(cmp); 
+        return (jne != NULL && 
+                instr_get_opcode(pop) == OP_pop &&
+                opnd_get_reg(instr_get_dst(pop, 0)) == REG_ECX &&
+                instr_get_opcode(cmp) == OP_cmp && 
+                instr_get_opcode(jne) == OP_jne);
+    }
+}
+
+/* checks to see if the address pushed by the push instruction matches the
+ * address in the cmp following the pop */
+static bool 
+check_return(dcontext_t *dcontext, instr_t *inst, instr_t *push)
+{
+    if (!INTERNAL_OPTION(unsafe_ignore_eflags_trace)) {
+        instr_t *lea = instr_get_next(inst);
+        opnd_t check;
+        if (instr_get_opcode(lea) != OP_lea)
+            lea = inst;
+        check = instr_get_src(lea, 0);
+        logopnd(dcontext, 3, check, "check opnd");
+        return (opnd_is_near_base_disp(check) &&
+                (opnd_get_disp(check) ==
+                 -(int) opnd_get_immed_int(instr_get_src(push, 0))));
+    }
+    else {
+        instr_t *cmp = instr_get_next(inst);
+        opnd_t check;
+        if (instr_get_opcode(cmp) != OP_cmp)
+            cmp = inst;
+        check = instr_get_src(cmp, 1);
+        logopnd(dcontext, 3, check, "check opnd");
+        return (opnd_is_immed_int(check) &&
+                opnd_get_immed_int(check) ==
+                opnd_get_immed_int(instr_get_src(push, 0)));
+    }
+}
+
+
+#define CALL_RETURN_STACK_SIZE 40
+
+/* attempts to match calls with returns for the purpose of removing the return check
+ * instructions */
+static void 
+call_return_matching(dcontext_t *dcontext, app_pc tag, instrlist_t *trace)
+{
+    instr_t *a[CALL_RETURN_STACK_SIZE];
+    instr_t *inst, *next_inst;
+    int top, i, opcode;
+    top = 0;
+#ifdef DEBUG
+    LOG(THREAD, LOG_OPTS, 3, "starting call return matching\n");
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_OPTS) != 0)
+        instrlist_disassemble(dcontext, tag, trace, THREAD);
+#endif
+    for (inst = instrlist_first(trace); inst != NULL; inst = next_inst) {
+        next_inst = instr_get_next(inst);
+        loginst(dcontext, 3, inst, "checking");
+        // look for push_imm from call and add to state
+        if (next_inst != NULL) {
+            opcode = instr_get_opcode(next_inst);
+            if ((opcode == OP_push_imm) && (opnd_get_size(instr_get_src(next_inst, 0)) == OPSZ_4))  {
+                inst = next_inst;
+                loginst(dcontext, 3, inst, "found call push");
+                if (top < CALL_RETURN_STACK_SIZE) {
+                    a[top] = inst;
+                    top++;
+                } else {
+                    LOG(THREAD, LOG_OPTS, 1, "call return matching stack overflow\n");
+                    for (i = 1; i < CALL_RETURN_STACK_SIZE; i++) {
+                        a[i-1] = a[i];
+                    }
+                    a[top-1] = inst;
+                }
+            }
+        }
+        // look for pop from return and remove instruction is possible
+        if (is_return(dcontext, inst)) {
+            loginst(dcontext, 3, inst, "found start of return");
+            while ((top > 0) && (!check_return(dcontext, instr_get_next(instr_get_next(inst)), a[top-1]))) {
+                top--;
+                loginst(dcontext, 3, a[top], "ignoring probable non call push immed on call return stack");
+            }
+            if (top > 0) {
+                loginst(dcontext, 3, a[top-1], "corresponding push was");
+                LOG(THREAD, LOG_OPTS, 3, "attempting to remove return code\n");
+                next_inst = remove_return(dcontext, trace, inst);
+                top--;
+            } else {
+                LOG(THREAD, LOG_OPTS, 3, "call return stack underflow\n");
+            }
+        }
+    }
+#ifdef DEBUG
+    LOG(THREAD, LOG_OPTS, 3, "done call return matching\n");
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_OPTS) != 0)
+        instrlist_disassemble(dcontext, tag, trace, THREAD);
+#endif
+}
+    
+/****************************************************************************/
+
+/* peephole driver
+ * so we only walk instrlist once
+ * current opts:
+ *   p4 only: inc/dec -> add 1/sub 1
+ *   leave -> mov ebp,esp; pop ebp
+ */
+static void
+peephole_optimize(dcontext_t *dcontext, app_pc tag, instrlist_t *trace)
+{
+    bool p4 = (proc_get_family() == FAMILY_PENTIUM_4);
+    instr_t *inst, *next_inst;
+    int opcode;
+    LOG(THREAD, LOG_OPTS, 3, "peephole_optimize\n");
+    for (inst = instrlist_first(trace); inst != NULL; inst = next_inst) {
+        next_inst = instr_get_next(inst);
+        opcode = instr_get_opcode(inst);
+        if (p4 && (opcode == OP_inc || opcode == OP_dec)) {
+#ifdef DEBUG
+            opt_stats_t.incs_examined++;
+            if (replace_inc_with_add(dcontext, inst, trace))
+                opt_stats_t.incs_replaced++;
+#else
+            replace_inc_with_add(dcontext, inst, trace);
+#endif
+        } else if (opcode == OP_leave) {
+            /* on Pentium II and later, complex instructions like
+             * enter and leave are slower (though smaller) than
+             * their simpler components.
+             *     leave
+             *     =>
+             *     movl %ebp,%esp
+             *     popl %ebp
+             * this makes a difference on microbenchmarks, doesn't
+             * seem to show up on spec though
+             */
+            instrlist_preinsert(trace, inst,
+                                INSTR_CREATE_mov_ld(dcontext,
+                                                    opnd_create_reg(REG_ESP),
+                                                    opnd_create_reg(REG_EBP)));
+            instrlist_preinsert(trace, inst,
+                                INSTR_CREATE_pop(dcontext,
+                                                 opnd_create_reg(REG_EBP)));
+            instrlist_remove(trace, inst);
+            instr_destroy(dcontext, inst);
+        }
+    }
+}
+
+/* replaces inc with add 1, dec with sub 1 
+ * if cannot replace (eflags constraints), leaves original instruction alone
+ * returns true if successful, false if not
+ */
+static bool
+replace_inc_with_add(dcontext_t *dcontext, instr_t *inst, instrlist_t *trace)
+{
+    instr_t *in;
+    uint eflags;
+    int opcode = instr_get_opcode(inst);
+    bool ok_to_replace = false;
+
+    ASSERT(opcode == OP_inc || opcode == OP_dec);
+    LOG(THREAD, LOG_OPTS, 3, "replace_inc_with_add\n");
+
+    /* add/sub writes CF, inc/dec does not, make sure that's ok */
+    for (in = inst; in != NULL; in = instr_get_next(in)) {
+        eflags = instr_get_eflags(in);
+        if ((eflags & EFLAGS_READ_CF) != 0) {
+            loginst(dcontext, 3, in, "reads CF => cannot replace inc with add");
+            return false;
+        }
+        /* if writes but doesn't read, ok */
+        if ((eflags & EFLAGS_WRITE_CF) != 0) {
+            ok_to_replace = true;
+            break;
+        }
+        /* test is down here b/c we want to look at 1st exit 
+         * if direct branch, look at top of target
+         * N.B.: indirect branches: we'll hit lahf first, which reads CF,
+         *   which will stop us from replacing, which is what we want
+         */
+        if (instr_is_exit_cti(in)) {
+            /* FIXME: what if branch is never taken and points to
+             * bogus memory, or we walk beyond interrupt or some
+             * non-cti that we normally stop at?
+             */
+            byte *target;
+            /* FIXME: shadowing eflags */
+            int eflags;
+            instr_t tinst;
+            if (!opnd_is_near_pc(instr_get_target(in)))
+                break;
+            target = (byte *) opnd_get_pc(instr_get_target(in));
+            loginst(dcontext, 3, in, "looking at target");
+            instr_init(dcontext, &tinst);
+            do {
+                instr_reset(dcontext, &tinst);
+                target = decode_cti(dcontext, target, &tinst);
+                ASSERT(instr_valid(&tinst));
+                eflags = instr_get_eflags(&tinst);
+                if ((eflags & EFLAGS_READ_CF) != 0) {
+                    loginst(dcontext, 3, in,
+                            "reads CF => cannot replace inc with add");
+                    return false;
+                }
+                /* if writes but doesn't read, ok */
+                if ((eflags & EFLAGS_WRITE_CF) != 0) {
+                    ok_to_replace = true;
+                    break;
+                }
+                /* stop at 1st cti, don't try to recurse,
+                 * common case should hit a writer of CF
+                 * prior to 1st cti
+                 */
+            } while (!instr_is_cti(&tinst));
+            instr_free(dcontext, &tinst);
+            /* for cbr, exit and fall-through must be ok */
+            if (!instr_is_cbr(in))
+                break;
+            else /* continue for fall-through */
+                ok_to_replace = false;
+        }
+    }
+    if (!ok_to_replace) {
+        LOG(THREAD, LOG_OPTS, 3, "no write to CF => cannot replace inc with add\n");
+        return false;
+    }
+    if (opcode == OP_inc) {
+        LOG(THREAD, LOG_OPTS, 3, "replacing inc with add\n");
+        in = INSTR_CREATE_add(dcontext, instr_get_dst(inst, 0), OPND_CREATE_INT8(1));
+    } else {
+        LOG(THREAD, LOG_OPTS, 3, "replacing dec with sub\n");
+        in = INSTR_CREATE_sub(dcontext, instr_get_dst(inst, 0), OPND_CREATE_INT8(1));
+    }
+    instr_set_prefixes(in, instr_get_prefixes(inst));
+    replace_inst(dcontext, trace, inst, in);
+    return true;
+}
+
+/****************************************************************************/
+/* josh's load removal optimization */
+#define MAX_DIST 40  
+void
+remove_redundant_loads(dcontext_t *dcontext, app_pc tag, instrlist_t *trace)
+{
+    instr_t *instr, *next_inst,*first_mem_access,*reg_write_checker;
+    opnd_t mem_read,orig_reg_opnd ={0}; /* FIXME: check */
+    reg_id_t orig_reg;
+    int dist;
+    uint ctis;
+    bool removed_from_store = false;
+    LOG(THREAD, LOG_OPTS, 3,"entering remove_loads optimization\n");
+
+    for (instr = instrlist_first(trace); instr != NULL; instr = next_inst) {
+        next_inst = instr_get_next(instr);
+
+        //ensures that it is an instruction which reads memory
+        if (instr_reads_memory(instr)) {
+#ifdef DEBUG
+            opt_stats_t.loads_examined++;
+#endif
+            if (instr_get_opcode(instr)==OP_mov_ld) {
+                mem_read=instr_get_src_mem_access(instr);
+            }
+            else if (instr_get_opcode(instr)==OP_add) {
+                mem_read=instr_get_src_mem_access(instr);
+                if (opnd_same(mem_read,instr_get_dst(instr,0))) {
+                    continue;
+                }
+            }
+            else {
+                continue;
+            }
+        }
+        else {
+            continue;
+        }
+        
+        /* to simply things for debugging, just worry about cases where the read
+           is indirect off the base pointer. this should be removed later */
+        if (opnd_get_base(mem_read)!=REG_EBP||opnd_get_index(mem_read)!=REG_NULL)
+            continue;
+        LOG(THREAD, LOG_OPTS, 3,"\n");
+        loginst(dcontext, 3,instr," reads memory, try to eliminate. ");
+
+        //walk backwards to try to find where the memory was written
+        for (dist=0, first_mem_access=instr_get_prev(instr);
+             ((dist<MAX_DIST)&&(first_mem_access!=NULL));
+             first_mem_access=instr_get_prev(first_mem_access), dist++) {
+
+            loginst(dcontext, 3,first_mem_access,"mem_writer walking backwards");
+
+            //if the instr writes to ebp (eventually, any register)
+            if (instruction_affects_mem_access(first_mem_access,mem_read)) {
+                loginst(dcontext,3,first_mem_access,"this instr. probably writes to ebp");
+                first_mem_access=NULL;
+                break;
+            }
+            //if its a move that writes to same memory location
+            else if (instr_writes_memory(first_mem_access)) {
+                opnd_t writeopnd=instr_get_dst(first_mem_access,0);
+
+                //takes care of push/pop. should i make this if (!pushorpop?)
+                if (opnd_is_memory_reference(writeopnd)) { 
+                    loginst(dcontext,3,first_mem_access,"this instr writes to memory");
+                    if (opnd_same_address(writeopnd,mem_read)) {
+
+
+                        //if the writing instruction was a store, then its OK
+
+                        if (instr_get_opcode(first_mem_access)==OP_mov_st) {
+                            loginst(dcontext,3,first_mem_access,"this instr writes to same location");
+                            orig_reg_opnd=instr_get_src(first_mem_access,0);
+                            removed_from_store=true;
+                            if (!opnd_is_reg(orig_reg_opnd)) {
+                                loginst(dcontext,3,first_mem_access,"source isn't a register. can't optimize for now");
+                                first_mem_access=NULL;
+                            }
+                            break;
+                        }
+                        else { //an add or something else wrote to memory
+                            loginst(dcontext,3,first_mem_access,"this non-store accesses memory, stop optimization");
+                            first_mem_access=NULL;
+
+                            break;
+                        }
+                    }
+                    // check alignement of address (for partial or unaligned writes)
+                    // still might be issue with aligned partial writes (data size)
+                    // attempted quick fix for gcc bug for CGO paper 1/10/03 Tim
+                    // if size/alignment issues really are to blame then a more through
+                    // look at the optimization should be made
+                    else if (opnd_is_near_base_disp(writeopnd) && 
+                             opnd_is_near_base_disp(mem_read) &&
+                             (opnd_get_base(mem_read) == opnd_get_base(writeopnd)) &&
+                             (opnd_get_index(mem_read) == opnd_get_index(writeopnd))) {
+                        int scratch = opnd_get_disp(mem_read) - opnd_get_disp(writeopnd);
+                        if ((scratch < 4) && (scratch > -4)) {
+                            first_mem_access=NULL;
+                            break;
+                        }
+                    } // end quick fix
+
+                    if (!safe_write(first_mem_access)) {
+                        loginst(dcontext,3,first_mem_access,"unsafe write, killing optmization");
+                        first_mem_access=NULL;
+
+                        break;
+                    }
+                }
+            }
+            //check if the move reads from that location
+            
+            else if (instr_reads_memory(first_mem_access)) {
+                opnd_t readopnd=instr_get_src(first_mem_access,0);
+
+                //takes care of push/pop. should i make this if (!pushorpop?)
+                if (opnd_is_memory_reference(readopnd)) { 
+                    loginst(dcontext,3,first_mem_access,"this instr reads from memory");
+                    if (opnd_same_address(readopnd,mem_read)) {
+                        //if the writing instruction was a store, then its OK
+                        if (instr_get_opcode(first_mem_access)==OP_mov_ld) {
+                            loginst(dcontext,3,first_mem_access,"this instr reads from the same location");
+                            orig_reg_opnd=instr_get_dst(first_mem_access,0);
+                            removed_from_store=false;
+                            if (!opnd_is_reg(orig_reg_opnd)) {
+                                loginst(dcontext,3,first_mem_access,"dest. isn't a register. can't optimize for now");
+                                ASSERT(0);
+                                first_mem_access=NULL;
+                            }
+
+                            break;
+                        }
+                    }
+                }
+            }
+
+        }    
+        /* if it reached top of trace, or something wrote to ebp */
+        if (!first_mem_access) {
+            LOG(THREAD, LOG_OPTS, 3,"reached top of trace, or an add or other non-move wrote to memory\n");
+            continue;
+        }
+        if (dist>=MAX_DIST) {
+            LOG(THREAD, LOG_OPTS, 3, "passed MAX_DIST threshold of %d\n",MAX_DIST);
+            continue;
+        }
+
+
+        ASSERT(instr_num_dsts(first_mem_access)==1&&instr_num_srcs(first_mem_access)==1);
+
+        
+        ctis=0; //for stats of how many ctis are traversed
+        orig_reg=opnd_get_reg(orig_reg_opnd);
+        LOG(THREAD, LOG_OPTS, 3,"original register=%s\n",reg_names[orig_reg]);
+
+        //check here to see if anything overwrites the register holding the value
+        for (reg_write_checker=instr_get_next(first_mem_access);
+             reg_write_checker!=instr;
+             reg_write_checker=instr_get_next(reg_write_checker)) {
+
+            loginst(dcontext, 3,reg_write_checker,"walking forward");
+            if (instr_is_cti(reg_write_checker)) {
+                loginst(dcontext, 3,reg_write_checker,"holy shit, load-removal across basic blocks!");
+                ctis++;
+            }
+
+            //checks if something overwrites the register
+
+            if (instr_writes_to_reg(reg_write_checker,orig_reg)) {
+#ifdef DEBUG
+                opt_stats_t.reg_overwritten++;
+#endif
+                loginst(dcontext,3,reg_write_checker,"original register was overwritten");
+                break;
+            }
+        }
+
+        if (!opnd_is_reg_32bit(orig_reg_opnd)||!opnd_is_reg_32bit(instr_get_dst(instr,0)))
+            continue;
+
+        if (reg_write_checker==instr) {
+            /* if reg_write_checker reached instr, then nothing overwrites the 
+               register, its OK to do optimization */
+            
+            //replace load with register read
+            bool ok = instr_replace_src_opnd(instr,mem_read,orig_reg_opnd);
+            ASSERT(ok);
+
+            //after optimization, got move w/ same src as dst, so remove the instruction
+            if (((instr_get_opcode(instr)==OP_mov_st)||(instr_get_opcode(instr)==OP_mov_ld))&&
+               (opnd_same(instr_get_src(instr,0),instr_get_dst(instr,0)))) {
+                LOG(THREAD, LOG_OPTS, 3,"replacing mem access with %s made src==dst. removing instr\n",
+                    reg_names[orig_reg]);
+                instrlist_remove(trace,instr);
+                instr_destroy(dcontext,instr);
+            }
+            else if (!instr_is_encoding_possible(instr)) {
+                loginst(dcontext, 3,instr,"encoding not possible ;( reverting to orig. instr\n");
+                ok = instr_replace_src_opnd(instr,opnd_create_reg(orig_reg),mem_read);
+                ASSERT(ok);
+            }
+            else {
+                //update stats
+#ifdef DEBUG
+                opt_stats_t.ctis_in_load_removal+=ctis;
+#endif
+                if (removed_from_store) {
+#ifdef DEBUG
+                    opt_stats_t.loads_removed_from_stores++;
+#endif
+                    loginst(dcontext,3,instr,"replaced original instr with val from store");
+                }
+                else {
+#ifdef DEBUG
+                    opt_stats_t.loads_removed_from_loads++;
+#endif
+                    loginst(dcontext,3,instr,"replaced original instr with val from load");
+                }
+            }
+        }
+        else {
+            //the register was overwritten. try to use some other register to hold the value
+            reg_id_t dead_reg;
+            opnd_t dead_reg_opnd;
+            instr_t *copy_to_dead_instr;
+            dead_reg=find_dead_register_across_instrs(first_mem_access,instr);
+            if (dead_reg!=REG_NULL) {
+
+                dead_reg_opnd=opnd_create_reg(dead_reg);
+                LOG(THREAD, LOG_OPTS, 3,"looks like %s is free to hold the reg though!\n",reg_names[dead_reg]);
+                copy_to_dead_instr=INSTR_CREATE_mov_ld(dcontext,dead_reg_opnd,orig_reg_opnd);
+                instrlist_postinsert(trace,first_mem_access,copy_to_dead_instr);
+                loginst(dcontext,3,copy_to_dead_instr,"inserted this to save val. in dead register");
+                
+                instr_replace_src_opnd(instr,mem_read,dead_reg_opnd);
+                loginst(dcontext,3,instr,"modified this instr to use the new dead register");
+#ifdef DEBUG
+                opt_stats_t.val_saved_in_dead_reg++;
+                opt_stats_t.ctis_in_load_removal+=ctis;
+                if (removed_from_store)
+                    opt_stats_t.loads_removed_from_stores++;
+                else
+                    opt_stats_t.loads_removed_from_loads++;
+#endif          
+
+            }
+
+        }
+
+    }
+    LOG(THREAD, LOG_OPTS, 3,"leaving remove_loads optimization\n");
+}
+
+static reg_id_t
+find_dead_register_across_instrs(instr_t *start,instr_t *end)
+{
+    reg_id_t a;
+    instr_t *instr = NULL;
+
+    for (a=REG_EAX;a<=REG_EDI;a++) {
+        if (is_dead_register(a,start)) {
+            for (instr=start;instr!=end;instr=instr_get_next(instr)) {
+                if (instr_uses_reg(instr,a))
+                    break;
+            }
+        }
+        if (instr==end)
+            return a;
+    }
+    return REG_NULL;
+}
+
+#if 0 /* not used */
+static int 
+find_dead_register_across_instrs_H(instr_t *start,instr_t *end)
+{
+
+    //This is a huristics. Find the longest dead register. If its more than half a trace - it's good for me.
+
+    int a;
+    int reg=REG_NULL;
+    instr_t *instr = NULL;
+    int reg_dead_count[REG_EDI-REG_EAX+1];
+    int count=0;
+    int max=0;
+
+    int num_of_instr=0;
+    
+    for (instr=start;instr!=end;instr=instr_get_next(instr)) {
+        num_of_instr++;
+    }
+
+
+    for (a=REG_EAX;a<=REG_EDI;a++) {
+        count=0;
+        if (is_dead_register(a,start)) {
+            for (instr=start;instr!=end;instr=instr_get_next(instr)) {       
+                count++;
+                if (instr_uses_reg(instr,a)) {
+                    reg_dead_count[a]=count;
+                    if (count>max) {
+                        max=count;
+                        reg=a;
+                    }
+                    break;
+                }
+            }
+        }
+        if (instr==end)   
+            return a;
+    }
+    if (max>=num_of_instr/2)
+        return reg;
+
+    return REG_NULL;
+}
+#endif /* #if 0 */
+
+/****************************************************************************/
+/* prefetching */
+
+#define MIN_PREFETCH_DISTANCE 3
+
+static void
+prefetch_optimize_trace(dcontext_t *dcontext, app_pc tag, instrlist_t *trace)
+{
+    instr_t *instr,*loopinstr,*tracewalker,*prefetchinstr;
+    opnd_t src_mem_access;
+    int distance;
+
+    bool insertprefetch;
+
+    loopinstr=find_next_self_loop(dcontext,tag,instrlist_first(trace));
+    if (loopinstr==NULL) {
+        return;
+    }
+
+    instr=instrlist_first(trace);
+    while (instr!=NULL) {
+        
+        if (instr_reads_memory(instr) &&
+            instr_get_opcode(instr)!=OP_prefetchnta) {
+
+            src_mem_access=instr_get_src_mem_access(instr);
+
+            //only prefetch if the load is register-indirect
+            if (!(opnd_get_base(src_mem_access)||opnd_get_index(src_mem_access)))
+                break;
+
+            prefetchinstr=INSTR_CREATE_prefetchnta(dcontext,
+                                                   opnd_create_base_disp(opnd_get_base(src_mem_access),
+                                                                         opnd_get_index(src_mem_access),
+                                                                         opnd_get_scale(src_mem_access),
+                                                                         opnd_get_disp(src_mem_access), OPSZ_1));
+
+            insertprefetch=true;
+            tracewalker=instr->prev;  //start walk before instruction to prefetch for
+            distance=0;
+            while (1) {
+                distance++;
+                //if beginning of trace is reached, go back to loop spot
+                if (tracewalker==NULL)
+                    tracewalker=loopinstr->prev;
+
+                //reached conflicting instruction
+                if (instruction_affects_mem_access(tracewalker,src_mem_access))
+                    break;
+
+                //came across same prefetch instruction already
+                if (instr_same(prefetchinstr,tracewalker)) {
+                    insertprefetch=false;
+                    break;
+                }
+
+                //looped completely around or load is after the loop instr
+                if ((tracewalker==instr)||(tracewalker==loopinstr)) {
+                    insertprefetch=false;
+                    break;
+                }                   
+                tracewalker=tracewalker->prev;
+            }
+
+            if (insertprefetch&&(distance<MIN_PREFETCH_DISTANCE||
+                                 (tracewalker->next==instr)))
+                insertprefetch=false;
+
+            if (insertprefetch) { 
+#ifdef DEBUG
+                LOG(THREAD, LOG_OPTS, 3,"in trace "PFX" inserting prefetch for:",tag);
+                if (stats->loglevel>=3)
+                    instr_disassemble(dcontext,instr,THREAD);
+                LOG(THREAD, LOG_OPTS, 3," after instruction: ");
+                if (stats->loglevel>=3)
+                    instr_disassemble(dcontext,tracewalker,THREAD);
+                LOG(THREAD, LOG_OPTS, 3,"\n");
+#endif
+                instrlist_postinsert(trace,tracewalker,prefetchinstr);
+            }
+            else
+                instr_destroy(dcontext,prefetchinstr);
+
+
+        }
+        instr=instr_get_next(instr);
+        
+    }
+}
+
+/* Removed josh's attempt at using the SSE2 xmm registers to hold some local vars - 
+   you can find it in the attic optimize.c 1.95 
+
+   The -spill_xmm optimization never speeded anything up - probably because on a
+   P4 xmm<->reg operations were three times more expensive than a
+   cache hit in memory, on a Pentium M the cost ratio may be reversed
+   and the optimization may be worth keeping in mind.  
+   Of course, transparency and memory aliasing problems don't make it very appealing.
+*/
+
+
+/****************************************************************************/
+/* utility routines */
+
+bool
+is_store_to_ecxoff(dcontext_t *dcontext, instr_t *inst)
+{
+    int opcode = instr_get_opcode(inst);
+    return ((opcode == OP_mov_imm || opcode == OP_mov_st) &&
+            opnd_is_near_base_disp(instr_get_dst(inst, 0)) &&
+            opnd_get_disp(instr_get_dst(inst, 0)) == opnd_get_disp(opnd_create_dcontext_field(dcontext, XCX_OFFSET)));
+}
+
+bool
+is_load_from_ecxoff(dcontext_t *dcontext, instr_t *inst)
+{
+    return (instr_get_opcode(inst) == OP_mov_ld &&
+            opnd_is_near_base_disp(instr_get_src(inst, 0)) &&
+            opnd_get_disp(instr_get_src(inst, 0)) == opnd_get_disp(opnd_create_dcontext_field(dcontext, XCX_OFFSET)));
+}
+
+/* returns true if the opnd is a constant address 
+ * i.e. is memory access with null base and index registers */
+bool
+opnd_is_constant_address(opnd_t address)
+{
+    return (opnd_is_abs_addr(address) IF_X64(|| opnd_is_rel_addr(address)));
+}
+
+/* checks to see if the instr zeros a reg (and does nothing else) */
+static bool 
+is_zeroing_instr(instr_t *inst)
+{
+    int opcode = instr_get_opcode(inst);
+    if (((opcode == OP_xor) || (opcode == OP_pxor) || (opcode == OP_sub)) &&
+        opnd_same(instr_get_src(inst,0), instr_get_src(inst,1)))
+        return true;
+    return false;
+}
+
+static bool 
+is_dead_register(reg_id_t reg,instr_t *where)
+{
+    //something tells me its a bad call to mess with these...
+    if (reg==REG_EBP||reg==REG_ESP)  
+        return false;
+
+    while (!instr_is_cti(where)) {
+        if (instr_reg_in_src(where,reg))
+            return false;
+        else if (instr_writes_to_reg(where,reg))
+            return true;
+        //!instr_writes_to_reg(...).  probably writing to mem indirectly through reg
+        else if (instr_reg_in_dst(where,reg)) 
+            return false;
+
+        where=instr_get_next(where);
+    }
+    return false;
+}
+
+#if 0 /* not used */
+static int 
+find_dead_register(instr_t *where)
+{
+    int a;
+    for (a=REG_EAX;a<=REG_EDI;a++) {
+        if (is_dead_register(a,where))
+            return a;
+    }
+    return REG_NULL;
+}
+
+/* checks to see if, starting at inst, the value in reg 
+ * is dead (is written to before it is next read), 
+ * could later recurse into other traces/fragments but right now
+ * assumes everything is live at a CTI */
+static bool 
+is_dead_reg(dcontext_t *dcontext, instr_t *start_inst, int reg)
+{
+    instr_t *inst, *next_inst;
+    if (reg == REG_NULL)
+        return true;
+    for (inst = start_inst; inst != NULL; inst=next_inst) {
+        next_inst = instr_get_next(inst);
+        if (instr_is_cti(inst))
+            return false;
+        if (instr_reg_in_src(inst, reg))
+            return false;
+        if (instr_writes_to_reg(inst, reg))
+            return true;
+        else if (instr_reg_in_dst(inst, reg))
+            return false;
+    }
+    /* in case something goes wrong */
+    return false;
+}
+#endif
+
+/* replaces old with new and destroys old inst */
+void
+replace_inst(dcontext_t *dcontext, instrlist_t *ilist, instr_t *old, instr_t *new)
+{
+    instrlist_preinsert(ilist, old, new);
+    instrlist_remove(ilist, old);
+    instr_destroy(dcontext, old);
+}
+
+/* removes and deystroys inst */
+void
+remove_inst(dcontext_t *dcontext, instrlist_t *ilist, instr_t *inst)
+{
+    instrlist_remove(ilist, inst);
+    instr_destroy(dcontext, inst);
+}
+
+/* checks if instr writes to any registers that mem_access depends on */
+static bool
+instruction_affects_mem_access(instr_t *instr,opnd_t mem_access)
+{
+    reg_id_t base,index;
+    ASSERT(instr);
+    base=opnd_get_base(mem_access);
+    if (base!=REG_NULL&&instr_writes_to_reg(instr,base))
+        return true;
+    index=opnd_get_index(mem_access);
+    if (index!=REG_NULL&&instr_writes_to_reg(instr,index))
+        return true;
+
+    return false;
+}
+
+/* a simplistic check to see if a write could overwrite some arbitrary place*/
+static bool 
+safe_write(instr_t *mem_writer)
+{
+    opnd_t mem_write=instr_get_dst(mem_writer,0);
+
+    if (!opnd_is_base_disp(mem_write))
+        return true;
+
+    else if (opnd_get_base(mem_write)==REG_NULL) //if there's no base, its prob. a constant mem addr
+        return true;
+
+    else if (opnd_get_base(mem_write)!=REG_EBP || opnd_get_index(mem_write)!=REG_NULL)
+        return false;
+
+    return true;
+}
+
+opnd_t
+instr_get_src_mem_access(instr_t *instr)
+{
+    int a;
+    opnd_t curop;
+
+    for (a=0;a<instr_num_srcs(instr);a++) {
+        curop=instr_get_src(instr,a);
+        if (opnd_is_memory_reference(curop)) {
+            return curop;
+        }
+    }
+    ASSERT_NOT_REACHED();
+    curop.kind = NULL_kind;
+    return curop;
+}
+
+instr_t *
+find_next_self_loop(dcontext_t *dcontext, app_pc tag, instr_t *instr)
+{
+    app_pc target;
+
+    while (instr != NULL) {
+        if (instr_is_cbr(instr)||instr_is_ubr(instr)) {
+            target=opnd_get_pc(instr_get_target(instr));
+            if (target==tag) {
+                return instr;
+            }
+
+        }
+
+        instr=instr_get_next(instr);
+    }
+    return NULL;
+}
+
+void
+replace_self_loop_with_instr(dcontext_t *dcontext, app_pc tag,
+                             instrlist_t *trace, instr_t *desiredtargetinstr)
+{
+    instr_t *top=instrlist_first(trace),*in;
+    opnd_t targetop;
+
+    in=top;
+
+    LOG(THREAD, LOG_OPTS, 3,
+        "entering replace_self_loop_with_instr looking for tag "PFX".\n",
+        tag);
+    while (in!=NULL) {
+        if (instr_is_cbr(in)||instr_is_ubr(in)) {
+            targetop=instr_get_target(in);
+            if (opnd_is_near_pc(targetop)&&opnd_get_pc(targetop)==tag) {
+                loginst(dcontext,3,in,"self_loop (pc target==tag) fixing in");
+                instr_set_target(in,opnd_create_instr(desiredtargetinstr));
+            }
+            else if (opnd_is_near_instr(targetop)&&opnd_get_instr(targetop)==top) {
+                loginst(dcontext,3,in,"self_loop (inter traget==top)fixing in");
+                logopnd(dcontext,3,opnd_create_instr(desiredtargetinstr),"self_loop in now points to");
+                instr_set_target(in,opnd_create_instr(desiredtargetinstr));
+            }
+        }
+        in=instr_get_next(in);
+    }
+    
+}
+
+
+/* given a cbr, finds the previous instr that writes the flag the cbr reads */
+static instr_t *
+get_decision_instr(instr_t *jmp)
+{
+    instr_t *inst;
+    uint flag_tested = EFLAGS_READ_TO_WRITE(instr_get_eflags(jmp));
+    uint eflags;
+
+    ASSERT(instr_is_cbr(jmp));
+    inst = instr_get_prev(jmp);
+    while (inst != NULL) {
+        eflags = instr_get_eflags(inst);
+        if ((eflags & flag_tested) != 0) {
+            return inst;
+        }
+        inst = instr_get_prev(inst);
+    }
+    return NULL;
+}
+
+static void 
+propagate_down(bool *reg_rep, int index, bool value)
+{
+    if ((index >= 0) && (index < 24)) {
+        reg_rep[index] = value;
+        if (index < 12) {
+            reg_rep[index+8] = value;
+            if (index < 4) {
+                reg_rep[index+16] = value;
+                reg_rep[index+20] = value;
+            }
+            else if (index >= 8)
+                reg_rep[index+12] = value;
+        }
+    }
+}
+
+static bool 
+check_down(bool *reg_rep, int index)
+{
+    return ((index >= 0) && (index < 24) && reg_rep[index] &&
+            ((index >= 12) || (reg_rep[index+8] &&
+                               ((index < 8) || reg_rep[index+12]) && 
+                               ((index >= 4) || (reg_rep[index+16] &&
+                                                 reg_rep[index+20])))));
+}
+
+/* return true, if this instr is a nop, of one of a class of nops */
+/* does not check for all types of nops, since there are many */
+/* these seem to be the most common */
+static bool 
+is_nop(instr_t *inst)
+{
+    int opcode = instr_get_opcode(inst);
+    if (opcode == OP_nop)
+        return true;
+    if ((opcode == OP_mov_ld || opcode == OP_mov_st || opcode == OP_xchg) && 
+        opnd_same(instr_get_src(inst, 0), instr_get_dst(inst, 0))) 
+        return true;
+    if (opcode == OP_lea && 
+        opnd_get_disp(instr_get_src(inst, 0)) == 0 && 
+        ((opnd_get_base(instr_get_src(inst, 0)) == opnd_get_reg(instr_get_dst(inst, 0)) &&
+          opnd_get_index(instr_get_src(inst, 0)) == REG_NULL) ||
+         (opnd_get_index(instr_get_src(inst, 0)) == opnd_get_reg(instr_get_dst(inst, 0)) && 
+          opnd_get_base(instr_get_src(inst, 0)) == REG_NULL && 
+          opnd_get_scale(instr_get_src(inst, 0)) == 1)))
+        return true;
+    // other cases
+    return false;
+}
+
+#endif /* INTERNAL around whole file */
diff --git a/core/x86/pre_inject_asm.asm b/core/x86/pre_inject_asm.asm
new file mode 100644
index 000000000..ca163bf72
--- /dev/null
+++ b/core/x86/pre_inject_asm.asm
@@ -0,0 +1,215 @@
+/* **********************************************************
+ * Copyright (c) 2007-2009 VMware, Inc.  All rights reserved.
+ * ********************************************************** */
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "asm_defines.asm"
+START_FILE
+
+#ifdef X64
+# define FREE_LIBRARY_NAME FreeLibrary
+#else
+# define FREE_LIBRARY_NAME FreeLibrary@4
+#endif
+DECL_EXTERN(FREE_LIBRARY_NAME)
+DECL_EXTERN(process_attach)
+
+/* DllMain for win32/pre_inject.c 
+ * if we start including in multiple dlls will need custom entry names or ifdefs
+ */
+    /* We want to unload ourselves on process attach, regardless of
+     * whether we inject DR or not.
+     * One strategy is to return false:
+     *   see msdn DllMain docs, return value ignored for all cases other than 
+     *   DLL_PROCESS_ATTACH, in which case returning false means initialization
+     *   failed and the dll is unloaded (exactly what we want!!) if it was loaded
+     *   by load library (it terminates process if was loaded as part of process 
+     *   initialization, but that's not our case)
+     * However, that doesn't work when shimeng gets in the way and initializes us,
+     * as it ignores the return value.  Furthermore, it causes a "dll init
+     * failure" popup on NT in every process.  So instead we construct a call
+     * to FreeLibrary that will return to our caller, skipping our freed code.
+     * FreeLibrary returns non-zero on success, zero on failure -- so if
+     * it fails we'll have the popup on NT and the non-removal w/ shimeng,
+     * but on success we'll be returning the proper value and preventing the popup.
+     * WARNING: we're freeing our library in the middle of loader code
+     * initializing us -- in practice the loader seems to handle it gracefully
+     * and never blindly uses a handle to us (at least we see no exceptions),
+     * but future loader code could be more fragile.
+     */
+#define DLL_PROCESS_ATTACH 1 /* from winnt.h */
+#ifdef X64
+# define DLL_MAIN_STDCALL_NAME DllMain
+#else
+# define DLL_MAIN_STDCALL_NAME DllMain@12
+#endif
+        DECLARE_FUNC(DLL_MAIN_STDCALL_NAME)
+GLOBAL_LABEL(DLL_MAIN_STDCALL_NAME:)
+        cmp      ARG2, DLL_PROCESS_ATTACH
+        jne      DllMain_not_pattach
+#ifdef X64
+        /* push gets us back to 16 byte alignment, ready for next call */
+        push     rcx /* preserve arg1, which is caller-saved */
+#endif
+        CALLC0(process_attach)
+#ifdef X64
+        /* The 32 bytes of shadow space (4 args' worth) are there regardless
+         * of the number of args: we have 3, FreeLibrary has 1, so no
+         * change to stack.  We only need to preserve arg1.
+         */
+        pop      rcx
+#else
+        /* we and FreeLibrary are stdcall, so we have to change the
+         * current 3 args + retaddr for our call to 1 arg + retaddr
+         * to pretend our caller called FreeLibrary
+         */
+        /* copy ret addr to 2 slots down */
+        mov      eax, [esp]
+        mov      [8 + esp], eax
+        /* copy our hModule arg 2 slots down to become arg for FreeLibrary */
+        mov      eax, [4 + esp]
+        mov      [12 + esp], eax
+        /* adjust esp and we're off */
+        add      esp, 8
+#endif
+        jmp      FREE_LIBRARY_NAME
+DllMain_not_pattach:
+        mov      eax, 1
+#ifdef X64
+        ret
+#else
+        ret      HEX(0c)
+#endif
+        END_FUNC(DLL_MAIN_STDCALL_NAME)
+
+
+/***************************************************************************/
+#ifndef X64
+
+/* Routines to switch to 64-bit mode from 32-bit WOW64, make a 64-bit
+ * call, and then return to 32-bit mode.
+ */
+
+/* FIXME: check these selector values on all platforms: these are for XPSP2.
+ * Keep in synch w/ defines in arch.h.
+ */
+# define CS32_SELECTOR HEX(23)
+# define CS64_SELECTOR HEX(33)
+
+/*
+ * int switch_modes_and_load(void *ntdll64_LdrLoadDll, 
+ *                           UNICODE_STRING_64 *lib,
+ *                           HANDLE *result)
+ */
+# define FUNCNAME switch_modes_and_load
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        /* get args before we change esp */
+        mov      eax, ARG1
+        mov      ecx, ARG2
+        mov      edx, ARG3
+        /* save callee-saved registers */
+        push     ebx
+        /* far jmp to next instr w/ 64-bit switch: jmp 0033:<sml_transfer_to_64> */
+        RAW(ea)
+        DD offset sml_transfer_to_64
+        DB CS64_SELECTOR
+        RAW(00)
+sml_transfer_to_64:
+    /* Below here is executed in 64-bit mode, but with guarantees that
+     * no address is above 4GB, as this is a WOW64 process.
+     */
+       /* Call LdrLoadDll to load 64-bit lib:
+        *   LdrLoadDll(IN PWSTR DllPath OPTIONAL,
+        *              IN PULONG DllCharacteristics OPTIONAL,
+        *              IN PUNICODE_STRING DllName,
+        *              OUT PVOID *DllHandle));
+        */
+        RAW(4c) RAW(8b) RAW(ca)  /* mov r9, rdx : 4th arg: result */
+        RAW(4c) RAW(8b) RAW(c1)  /* mov r8, rcx : 3rd arg: lib */
+        push     0               /* slot for &DllCharacteristics */
+        lea      edx, dword ptr [esp] /* 2nd arg: &DllCharacteristics */
+        xor      ecx, ecx        /* 1st arg: DllPath = NULL */
+        /* align the stack pointer */
+        mov      ebx, esp        /* save esp in callee-preserved reg */
+        sub      esp, 32         /* call conv */
+        and      esp, HEX(fffffff0) /* align to 16-byte boundary */
+        call     eax
+        mov      esp, ebx        /* restore esp */
+        /* far jmp to next instr w/ 32-bit switch: jmp 0023:<sml_return_to_32> */
+        push     offset sml_return_to_32  /* 8-byte push */
+        mov      dword ptr [esp + 4], CS32_SELECTOR /* top 4 bytes of prev push */
+        jmp      fword ptr [esp]
+sml_return_to_32:
+        add      esp, 16         /* clean up far jmp target and &DllCharacteristics */
+        pop      ebx             /* restore callee-saved reg */
+        ret                      /* return value already in eax */
+        END_FUNC(FUNCNAME)
+
+/*
+ * int switch_modes_and_call(void_func_t func, void *arg)
+ */
+# undef FUNCNAME
+# define FUNCNAME switch_modes_and_call
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        mov      eax, ARG1
+        mov      ecx, ARG2
+        /* save callee-saved registers */
+        push     ebx
+        /* far jmp to next instr w/ 64-bit switch: jmp 0033:<smc_transfer_to_64> */
+        RAW(ea)
+        DD offset smc_transfer_to_64
+        DB CS64_SELECTOR
+        RAW(00)
+smc_transfer_to_64:
+    /* Below here is executed in 64-bit mode, but with guarantees that
+     * no address is above 4GB, as this is a WOW64 process.
+     */
+        /* align the stack pointer */
+        mov      ebx, esp        /* save esp in callee-preserved reg */
+        sub      esp, 32         /* call conv */
+        and      esp, HEX(fffffff0) /* align to 16-byte boundary */
+        call     eax             /* arg is already in rcx */
+        mov      esp, ebx        /* restore esp */
+        /* far jmp to next instr w/ 32-bit switch: jmp 0023:<smc_return_to_32> */
+        push     offset smc_return_to_32  /* 8-byte push */
+        mov      dword ptr [esp + 4], CS32_SELECTOR /* top 4 bytes of prev push */
+        jmp      fword ptr [esp]
+smc_return_to_32:
+        add      esp, 8          /* clean up far jmp target */
+        pop      ebx             /* restore callee-saved reg */
+        ret                      /* return value already in eax */
+        END_FUNC(FUNCNAME)
+
+#endif /* !X64 */
+/***************************************************************************/
+
+END_FILE
diff --git a/core/x86/proc.c b/core/x86/proc.c
new file mode 100644
index 000000000..ecfd08a9d
--- /dev/null
+++ b/core/x86/proc.c
@@ -0,0 +1,620 @@
+/* **********************************************************
+ * Copyright (c) 2000-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * proc.c - processor-specific routines
+ */
+
+#include "globals.h"
+#include "proc.h"
+#include "instr.h" /* for dr_insert_{save,restore}_fpstate */
+#include "instrument.h" /* for dr_insert_{save,restore}_fpstate */
+#include "instr_create.h" /* for dr_insert_{save,restore}_fpstate */
+#include "decode.h" /* for dr_insert_{save,restore}_fpstate */
+
+#ifdef DEBUG
+/* case 10450: give messages to clients */
+# undef ASSERT /* N.B.: if have issues w/ DYNAMO_OPTION, re-instate */
+# undef ASSERT_TRUNCATE
+# undef ASSERT_BITFIELD_TRUNCATE
+# undef ASSERT_NOT_REACHED
+# define ASSERT DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_TRUNCATE DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_BITFIELD_TRUNCATE DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+# define ASSERT_NOT_REACHED DO_NOT_USE_ASSERT_USE_CLIENT_ASSERT_INSTEAD
+#endif
+
+/* Intel processors:  ebx:edx:ecx spell GenuineIntel */
+#define INTEL_EBX /* Genu */ 0x756e6547
+#define INTEL_EDX /* ineI */ 0x49656e69
+#define INTEL_ECX /* ntel */ 0x6c65746e
+
+/* AMD processors:  ebx:edx:ecx spell AuthenticAMD */
+#define AMD_EBX /* Auth */ 0x68747541
+#define AMD_EDX /* enti */ 0x69746e65
+#define AMD_ECX /* cAMD */ 0x444d4163
+
+/* cache_line_size is exported for efficient access.
+ * FIXME: In case the processor doesn't support the
+ * cpuid instruction, use a default value of 32.
+ * (see case 463 for discussion)
+ */
+size_t cache_line_size = 32;
+static ptr_uint_t mask; /* bits that should be 0 to be cache-line-aligned */
+
+static uint L1_icache_size = CACHE_SIZE_UNKNOWN;
+static uint L1_dcache_size = CACHE_SIZE_UNKNOWN;
+static uint L2_cache_size  = CACHE_SIZE_UNKNOWN;
+
+static uint vendor   = VENDOR_UNKNOWN;
+static uint family   = 0;
+static uint type     = 0;
+static uint model    = 0;
+static uint stepping = 0;
+
+/* Feature bits in 4 32-bit values: features in edx,
+ * features in ecx, extended features in edx, and
+ * extended features in ecx
+ */
+static features_t features = {0, 0, 0, 0};
+
+/* The brand string is a 48-character, null terminated string.
+ * Declare as a 12-element uint so the compiler won't complain 
+ * when we store GPRs to it.  Initialization is "unknown" .
+ */
+static uint brand_string[12] = {0x6e6b6e75, 0x006e776f};
+
+/*
+ * On Pentium through Pentium III, I-cache lines are 32 bytes.
+ * On Pentium IV they are 64 bytes.
+ */
+static void
+get_processor_specific_info(void)
+{
+    /* use cpuid instruction to get processor info.  For details, see
+     * http://download.intel.com/design/Xeon/applnots/24161830.pdf
+     * "AP-485: Intel Processor Identification and the CPUID
+     * instruction", 96 pages, January 2006
+     */
+    volatile /* don't put in register since we can't communicate constraints
+              * due to PIC error when we clobber ebx, even if safely */
+        uint res_eax, res_ebx = 0, res_ecx = 0, res_edx = 0;
+    uint max_val, max_ext_val;
+    int cpuid_res_local[4]; /* eax, ebx, ecx, and edx registers (in that order) */
+    /* declare as uint so compiler won't complain when we write GP regs to the array */
+    uint cache_codes[4];
+    int i;
+
+    /* First check for existence of the cpuid instruction
+     * by attempting to modify bit 21 of eflags
+     */
+    /* FIXME: Perhaps we should abort when the cpuid instruction
+     * doesn't exist since the cache_line_size may be incorrect.
+     * (see case 463 for discussion)
+    */
+    if (!cpuid_supported()) {
+        ASSERT_CURIOSITY(false && "cpuid instruction unsupported");
+        SYSLOG_INTERNAL_WARNING("cpuid instruction unsupported -- cache_line_size "
+                                "may be incorrect");
+        return;
+    }
+
+    /* first verify on Intel processor */
+#ifdef LINUX
+    our_cpuid(cpuid_res_local, 0);
+#else
+    __cpuid(cpuid_res_local, 0);
+#endif
+    res_eax = cpuid_res_local[0];
+    res_ebx = cpuid_res_local[1];
+    res_ecx = cpuid_res_local[2];
+    res_edx = cpuid_res_local[3];
+    max_val = res_eax;
+
+    if (res_ebx == INTEL_EBX) {
+        vendor = VENDOR_INTEL;
+        CLIENT_ASSERT(res_edx == INTEL_EDX && res_ecx == INTEL_ECX,
+                      "unknown Intel processor type");
+    } else if (res_ebx == AMD_EBX) {
+        vendor = VENDOR_AMD;
+        CLIENT_ASSERT(res_edx == AMD_EDX && res_ecx == AMD_ECX,
+                      "unknown AMD processor type");
+    } else {
+        vendor = VENDOR_UNKNOWN;
+        SYSLOG_INTERNAL_ERROR("Running on unknown processor type");
+        LOG(GLOBAL, LOG_TOP, 1, "cpuid returned "PFX" "PFX" "PFX" "PFX"\n",
+            res_eax, res_ebx, res_ecx, res_edx);
+    }
+
+    /* now get processor info */
+#ifdef LINUX
+    our_cpuid(cpuid_res_local, 1);
+#else
+    __cpuid(cpuid_res_local, 1);
+#endif
+    res_eax = cpuid_res_local[0];
+    res_ebx = cpuid_res_local[1];
+    res_ecx = cpuid_res_local[2];
+    res_edx = cpuid_res_local[3];
+    /* eax contains basic info:
+     *   extended family, extended model, type, family, model, stepping id
+     *   20:27,           16:19,          12:13, 8:11,  4:7,   0:3
+     */
+    type   = (res_eax >> 12) & 0x3;
+    family = (res_eax >>  8) & 0xf;
+    model  = (res_eax >>  4) & 0xf;
+    stepping = res_eax & 0xf;
+
+    /* Pages 3-164 and 3-165 of the IA-32 instruction set 
+     * reference instruct us to adjust the family and model
+     * numbers as follows.
+     */
+    if (family == 0x6 || family == 0xf) {
+        uint ext_model = (res_eax >> 16) & 0xf;
+        model += (ext_model << 4);
+
+        if (family == 0xf) {
+            uint ext_family = (res_eax >> 20) & 0xff;
+            family += ext_family;
+        }
+    }
+
+    features.flags_edx = res_edx;
+    features.flags_ecx = res_ecx;
+
+    if (family == FAMILY_PENTIUM_4) {
+        /* Pentium IV */
+        cache_line_size = (res_ebx & 0x0000ff00) >> 5; /* (x >> 8) * 8 == x >> 5 */
+    } else if (vendor == VENDOR_INTEL &&
+               (family == FAMILY_PENTIUM_3 || family == FAMILY_PENTIUM_2)) {
+        /* Pentium III, Pentium II */
+        cache_line_size = 32;
+    } else if (vendor == VENDOR_AMD && family == FAMILY_ATHLON) {
+        /* Athlon */
+        cache_line_size = 64;
+#ifdef IA32_ON_IA64
+    } else if (vendor == VENDOR_INTEL && family == FAMILY_IA64) {
+        /* Itanium */
+        cache_line_size = 32;
+#endif
+    } else {
+        LOG(GLOBAL, LOG_TOP, 1, "Warning: running on unsupported processor family %d\n",
+            family);
+        cache_line_size = 32;
+    }
+    /* people who use this in ALIGN* macros are assuming it's a power of 2 */
+    CLIENT_ASSERT((cache_line_size & (cache_line_size - 1)) == 0,
+                  "invalid cache line size");
+
+    /* get L1 and L2 cache sizes */
+    if (max_val >= 2) {
+#ifdef LINUX
+        our_cpuid((int*)cache_codes, 2);
+#else
+        __cpuid(cache_codes, 2);
+#endif
+        /* The lower 8 bits of eax specify the number of times cpuid
+         * must be executed to obtain a complete picture of the cache
+         * characteristics.
+         */
+        CLIENT_ASSERT((cache_codes[0] & 0xff) == 1, "cpuid error");
+        cache_codes[0] &= ~0xff;
+
+        /* Cache codes are stored in consecutive bytes in the 
+         * GP registers.  For each register, a 1 in bit 31 
+         * indicates that the codes should be ignored... zero
+         * all four bytes when that happens
+         */
+        for (i=0; i<4; i++) {
+            if (cache_codes[i] & 0x80000000)
+                cache_codes[i] = 0;
+        }
+
+        /* Table 3-17, pg 3-171 of IA-32 instruction set reference lists
+         * all codes.  Omitting L3 cache characteristics for now...
+         */
+        for (i=0; i<16; i++) {
+            switch (((uchar*)cache_codes)[i]) {
+            case 0x06: L1_icache_size = CACHE_SIZE_8_KB; break;
+            case 0x08: L1_icache_size = CACHE_SIZE_16_KB; break;
+            case 0x0a: L1_dcache_size = CACHE_SIZE_8_KB; break;
+            case 0x0c: L1_dcache_size = CACHE_SIZE_16_KB; break;
+            case 0x2c: L1_dcache_size = CACHE_SIZE_32_KB; break;
+            case 0x30: L1_icache_size = CACHE_SIZE_32_KB; break;
+            case 0x41: L2_cache_size = CACHE_SIZE_128_KB; break;
+            case 0x42: L2_cache_size = CACHE_SIZE_256_KB; break;
+            case 0x43: L2_cache_size = CACHE_SIZE_512_KB; break;
+            case 0x44: L2_cache_size = CACHE_SIZE_1_MB; break;
+            case 0x45: L2_cache_size = CACHE_SIZE_2_MB; break;
+            case 0x60: L1_dcache_size = CACHE_SIZE_16_KB; break;
+            case 0x66: L1_dcache_size = CACHE_SIZE_8_KB; break;
+            case 0x67: L1_dcache_size = CACHE_SIZE_16_KB; break;
+            case 0x68: L1_dcache_size = CACHE_SIZE_32_KB; break;
+            case 0x78: L2_cache_size = CACHE_SIZE_1_MB; break;
+            case 0x79: L2_cache_size = CACHE_SIZE_128_KB; break;
+            case 0x7a: L2_cache_size = CACHE_SIZE_256_KB; break;
+            case 0x7b: L2_cache_size = CACHE_SIZE_512_KB; break;
+            case 0x7c: L2_cache_size = CACHE_SIZE_1_MB; break;
+            case 0x7d: L2_cache_size = CACHE_SIZE_2_MB; break;
+            case 0x7f: L2_cache_size = CACHE_SIZE_512_KB; break;
+            case 0x82: L2_cache_size = CACHE_SIZE_256_KB; break;
+            case 0x83: L2_cache_size = CACHE_SIZE_512_KB; break;
+            case 0x84: L2_cache_size = CACHE_SIZE_1_MB; break;
+            case 0x85: L2_cache_size = CACHE_SIZE_2_MB; break;
+            case 0x86: L2_cache_size = CACHE_SIZE_512_KB; break;
+            case 0x87: L2_cache_size = CACHE_SIZE_1_MB; break;
+            default: break;
+            }
+        }
+    }
+
+    /* Try to get extended cpuid information */
+#ifdef LINUX
+    our_cpuid(cpuid_res_local, 0x80000000);
+#else
+    __cpuid(cpuid_res_local, 0x80000000);
+#endif
+    res_eax = cpuid_res_local[0];
+    max_ext_val = res_eax;
+
+    /* Processor brand string */
+    if (max_ext_val >= 0x80000004) {
+#ifdef LINUX
+        our_cpuid((int*)&brand_string[0], 0x80000002);
+    our_cpuid((int*)&brand_string[4], 0x80000003);
+    our_cpuid((int*)&brand_string[8], 0x80000004);
+#else
+    __cpuid(&brand_string[0], 0x80000002);
+    __cpuid(&brand_string[4], 0x80000003);
+    __cpuid(&brand_string[8], 0x80000004);
+#endif
+    }
+
+    /* Extended feature flags */
+    if (max_ext_val >= 0x80000001) {
+#ifdef LINUX
+        our_cpuid(cpuid_res_local, 0x80000001);
+#else
+        __cpuid(cpuid_res_local, 0x80000001);
+#endif
+        res_ecx = cpuid_res_local[2];
+        res_edx = cpuid_res_local[3];
+        features.ext_flags_edx = res_edx;
+        features.ext_flags_ecx = res_ecx;
+    }
+}
+
+void
+proc_init(void)
+{
+    LOG(GLOBAL, LOG_TOP, 1, "Running on a %d CPU machine\n", get_num_processors());
+
+    get_processor_specific_info();
+    CLIENT_ASSERT(cache_line_size > 0, "invalid cache line size");
+    mask = (cache_line_size - 1);
+
+    LOG(GLOBAL, LOG_TOP, 1, "Cache line size is %d bytes\n", cache_line_size);
+    LOG(GLOBAL, LOG_TOP, 1, "Processor brand string = %s\n", brand_string);
+    LOG(GLOBAL, LOG_TOP, 1, "Type=0x%x, Family=0x%x, Model=0x%x, Stepping=0x%x\n",
+        type, family, model, stepping);
+
+#ifdef X64
+    CLIENT_ASSERT(proc_has_feature(FEATURE_LAHF),
+                  "Unsupported processor type - processor must support LAHF/SAHF in "
+                  "64bit mode.");
+    if (!proc_has_feature(FEATURE_LAHF)) {
+        FATAL_USAGE_ERROR(UNSUPPORTED_PROCESSOR_LAHF, 2,
+                          get_application_name(), get_application_pid());
+    }
+#endif
+
+#ifdef DEBUG
+    /* FIXME: This is a small subset of processor features.  If we 
+     * care enough to add more, it would probably be best to loop
+     * through a const array of feature names.
+    */
+    if (stats->loglevel > 0 && (stats->logmask & LOG_TOP) != 0) {
+        if (proc_has_feature(FEATURE_XD_Bit))
+            LOG(GLOBAL, LOG_TOP, 1, "\tProcessor has XD Bit\n");
+        if (proc_has_feature(FEATURE_MMX))
+            LOG(GLOBAL, LOG_TOP, 1, "\tProcessor has MMX\n");
+        if (proc_has_feature(FEATURE_FXSR))
+            LOG(GLOBAL, LOG_TOP, 1, "\tProcessor has fxsave/fxrstor\n");
+        if (proc_has_feature(FEATURE_SSE))
+            LOG(GLOBAL, LOG_TOP, 1, "\tProcessor has SSE\n");
+        if (proc_has_feature(FEATURE_SSE2))
+            LOG(GLOBAL, LOG_TOP, 1, "\tProcessor has SSE2\n");
+        if (proc_has_feature(FEATURE_SSE3))
+            LOG(GLOBAL, LOG_TOP, 1, "\tProcessor has SSE3\n");
+    }
+#endif
+    /* PR 264138: for 32-bit CONTEXT we assume fxsave layout */
+    CLIENT_ASSERT((proc_has_feature(FEATURE_FXSR) && proc_has_feature(FEATURE_SSE)) ||
+                  (!proc_has_feature(FEATURE_FXSR) && !proc_has_feature(FEATURE_SSE)),
+                  "Unsupported processor type: SSE and FXSR must match");
+}
+
+uint
+proc_get_vendor(void)
+{
+    return vendor;
+}
+
+uint
+proc_get_family(void)
+{
+    return family;
+}
+
+uint proc_get_type(void)
+{
+    return type;
+}
+
+/* FIXME: Add MODEL_ constants to proc.h?? */
+uint proc_get_model(void)
+{
+    return model;
+}
+
+uint proc_get_stepping(void)
+{
+    return stepping;
+}
+
+bool
+proc_has_feature(feature_bit_t f)
+{
+    uint bit, val = 0;
+
+    if (f >= 0 && f <= 31) {
+        val = features.flags_edx;
+    } else if (f >= 32 && f <= 63) {
+        val = features.flags_ecx;
+    } else if (f >= 64 && f <= 95) {
+        val = features.ext_flags_edx;
+    } else if (f >= 96 && f <= 127) {
+        val = features.ext_flags_ecx;
+    } else {
+        CLIENT_ASSERT(false, "proc_has_feature: invalid parameter");
+    }
+
+    bit = f % 32;
+    return TEST((1 << bit), val);
+}
+
+features_t *
+proc_get_all_feature_bits(void)
+{
+    return &features;
+}
+
+char *
+proc_get_brand_string(void)
+{
+    return (char *)brand_string;
+}
+
+cache_size_t
+proc_get_L1_icache_size(void)
+{
+    return L1_icache_size;
+}
+
+cache_size_t
+proc_get_L1_dcache_size(void)
+{
+    return L1_dcache_size;
+}
+
+cache_size_t
+proc_get_L2_cache_size(void)
+{
+    return L2_cache_size;
+}
+
+const char *
+proc_get_cache_size_str(cache_size_t size)
+{
+    static const char *strings[] = {
+        "8 KB",
+        "16 KB",
+        "32 KB",
+        "128 KB",
+        "256 KB",
+        "512 KB",
+        "1 MB",
+        "2 MB",
+        "unknown"
+    };
+    CLIENT_ASSERT(size <= CACHE_SIZE_UNKNOWN, "proc_get_cache_size_str: invalid size");
+    return strings[size];
+}
+
+size_t
+proc_get_cache_line_size(void)
+{
+    return cache_line_size;
+}
+
+/* check to see if addr is cache aligned */
+bool
+proc_is_cache_aligned(void *addr)
+{
+    return (((ptr_uint_t)addr & mask) == 0x0);
+}
+
+/* Given an address or number of bytes sz, return a number >= sz that is divisible
+   by the cache line size. */
+ptr_uint_t
+proc_bump_to_end_of_cache_line(ptr_uint_t sz)
+{
+    if ((sz & mask) == 0x0)
+        return sz;      /* sz already a multiple of the line size */
+
+    return ((sz + cache_line_size) & ~mask);
+}
+
+/* yes same result as PAGE_START...FIXME: get rid of one of them? */
+void *
+proc_get_containing_page(void *addr)
+{
+    return (void *) (((ptr_uint_t)addr) & ~(PAGE_SIZE-1));
+}
+
+/* No synchronization routines necessary.  The Pentium hardware
+ * guarantees that the i and d caches are consistent. */
+void
+machine_cache_sync(void *pc_start, void *pc_end, bool flush_icache)
+{
+    /* empty */
+}
+
+DR_API
+/**
+ * Returns the size in bytes needed for a buffer for saving the floating point state.
+ */
+size_t
+proc_fpstate_save_size(void)
+{
+    CLIENT_ASSERT(opnd_size_in_bytes(OPSZ_512) == 512 &&
+                  opnd_size_in_bytes(OPSZ_108) == 108,
+                  "internal sizing discrepancy");
+    return (proc_has_feature(FEATURE_FXSR) ?  512 : 108);
+}
+
+DR_API
+/* Saves the floating point state into the 16-byte-aligned buffer buf,
+ * which must be 512 bytes for processors with the FXSR feature, and
+ * 108 bytes for those without (where this routine does not support
+ * 16-bit operand sizing).
+ * 
+ * DynamoRIO does NOT save the application's floating-point, MMX, or SSE state
+ * on context switches!  Thus if a client performs any floating-point operations
+ * in its main routines called by DynamoRIO, the client must save and restore
+ * the floating-point/MMX/SSE state.
+ * If the client needs to do so inside the code cache the client should implement
+ * that itself.
+ * return number of bytes written 
+ */
+size_t
+proc_save_fpstate(byte *buf)
+{
+    /* MUST be 16-byte aligned */
+    CLIENT_ASSERT((((ptr_uint_t)buf) & 0x0000000f) == 0,
+                  "proc_save_fpstate: buf must be 16-byte aligned");
+    if (proc_has_feature(FEATURE_FXSR)) {
+#ifdef WINDOWS
+        dr_fxsave(buf);
+#else
+        asm volatile("fxsave %0 ; fnclex ; finit"  : "=m" ((*buf)));
+#endif
+    } else {
+#ifdef WINDOWS
+        dr_fnsave(buf);
+#else
+        asm volatile("fnsave %0 ; fwait" : "=m" ((*buf)));
+#endif
+    }
+    return proc_fpstate_save_size();
+}
+
+DR_API
+/* Restores the floating point state from the 16-byte-aligned buffer
+ * buf, which must be 512 bytes for processors with the FXSR feature,
+ * and 108 bytes for those without (where this routine does not
+ * support 16-bit operand sizing).
+ */
+void 
+proc_restore_fpstate(byte *buf)
+{
+    /* MUST be 16-byte aligned */
+    CLIENT_ASSERT((((ptr_uint_t)buf) & 0x0000000f) == 0,
+                  "proc_restore_fpstate: buf must be 16-byte aligned");
+    if (proc_has_feature(FEATURE_FXSR)) {
+#ifdef WINDOWS
+        dr_fxrstor(buf);
+#else
+        asm volatile("fxrstor %0" : : "m" ((*buf)));
+#endif
+    } else {
+#ifdef WINDOWS
+        dr_frstor(buf);
+#else
+        asm volatile("frstor %0" : : "m" ((*buf)));
+#endif
+    }
+}
+
+void
+dr_insert_save_fpstate(void *drcontext, instrlist_t *ilist, instr_t *where,
+                       opnd_t buf)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    if (proc_has_feature(FEATURE_FXSR)) {
+        /* we want "fxsave, fnclex, finit" */
+        CLIENT_ASSERT(opnd_get_size(buf) == OPSZ_512,
+                      "dr_insert_save_fpstate: opnd size must be OPSZ_512");
+        instrlist_meta_preinsert(ilist, where, INSTR_CREATE_fxsave(dcontext, buf));
+        instrlist_meta_preinsert(ilist, where, INSTR_CREATE_fnclex(dcontext));
+        instrlist_meta_preinsert(ilist, where, INSTR_CREATE_fwait(dcontext));
+        instrlist_meta_preinsert(ilist, where, INSTR_CREATE_fninit(dcontext));
+    } else {
+        /* auto-adjust opnd size so it will encode */
+        if (opnd_get_size(buf) == OPSZ_512)
+            opnd_set_size(&buf, OPSZ_108);
+        /* FIXME: why is this appending fwait, vs "fsave" which prepends? */
+        instrlist_meta_preinsert(ilist, where, INSTR_CREATE_fnsave(dcontext, buf));
+        instrlist_meta_preinsert(ilist, where, INSTR_CREATE_fwait(dcontext));
+    }
+}
+
+void
+dr_insert_restore_fpstate(void *drcontext, instrlist_t *ilist, instr_t *where,
+                          opnd_t buf)
+{
+    dcontext_t *dcontext = (dcontext_t *) drcontext;
+    if (proc_has_feature(FEATURE_FXSR)) {
+        CLIENT_ASSERT(opnd_get_size(buf) == OPSZ_512,
+                      "dr_insert_save_fpstate: opnd size must be OPSZ_512");
+        instrlist_meta_preinsert(ilist, where, INSTR_CREATE_fxrstor(dcontext, buf));
+    } else {
+        /* auto-adjust opnd size so it will encode */
+        if (opnd_get_size(buf) == OPSZ_512)
+            opnd_set_size(&buf, OPSZ_108);
+        instrlist_meta_preinsert(ilist, where, INSTR_CREATE_frstor(dcontext, buf));
+    }
+}
diff --git a/core/x86/proc.h b/core/x86/proc.h
new file mode 100644
index 000000000..478d2af69
--- /dev/null
+++ b/core/x86/proc.h
@@ -0,0 +1,341 @@
+/* **********************************************************
+ * Copyright (c) 2000-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/*
+ * proc.h - processor implementation specific interfaces
+ */
+
+#ifndef _PROC_H_
+#define _PROC_H_ 1
+
+/* DR_API EXPORT TOFILE dr_proc.h */
+/* DR_API EXPORT BEGIN */
+/****************************************************************************
+ * PROCESSOR-SPECIFIC UTILITY ROUTINES AND CONSTANTS
+ */
+/**
+ * @file dr_proc.h
+ * @brief Utility routines for identifying features of the processor.
+ */
+
+/* page size is 4K on all DR-supported platforms */
+#define PAGE_SIZE (4*1024) /**< Size of a page of memory. */
+
+/**< Convenience macro to align to the start of a page of memory. */
+#define PAGE_START(x) (((ptr_uint_t)(x)) & ~((PAGE_SIZE)-1))
+
+/** Constants returned by proc_get_vendor(). */
+enum {
+    VENDOR_INTEL,   /**< proc_get_vendor() processor identification: Intel */
+    VENDOR_AMD,     /**< proc_get_vendor() processor identification: AMD */
+    VENDOR_UNKNOWN, /**< proc_get_vendor() processor identification: unknown */
+};
+
+/* Family and Model
+ *   Intel 486                 Family 4
+ *   Intel Pentium             Family 5
+ *   Intel Pentium Pro         Family 6, Model 0 and 1 
+ *   Intel Pentium 2           Family 6, Model 3, 5, and 6
+ *   Intel Celeron             Family 6, Model 5 and 6
+ *   Intel Pentium 3           Family 6, Model 7, 8, 10, 11
+ *   Intel Pentium 4           Family 15, Extended 0
+ *   Intel Itanium             Family 7
+ *   Intel Itanium 2           Family 15, Extended 1 and 2
+ *   Intel Pentium M           Family 6, Model 9 and 13
+ *   Intel Core                Family 6, Model 14
+ *   Intel Core 2              Family 6, Model 15
+ */
+/* DR_API EXPORT END */
+#ifdef IA32_ON_IA64 /* don't export IA64 stuff! */
+/* IA-64 */
+#define FAMILY_IA64         7
+#endif
+/* DR_API EXPORT BEGIN */
+/* Remember that we add extended family to family as Intel suggests */
+#define FAMILY_ITANIUM_2_DC 17 /**< proc_get_family() processor family: Itanium 2 DC */
+#define FAMILY_ITANIUM_2    16 /**< proc_get_family() processor family: Itanium 2 */
+#define FAMILY_PENTIUM_4    15 /**< proc_get_family() processor family: Pentium 4 */
+#define FAMILY_ITANIUM       7 /**< proc_get_family() processor family: Itanium */
+/* Pentium Pro, Pentium II, Pentium III, Athlon, Pentium M, Core, Core 2 */
+#define FAMILY_CORE_2        6 /**< proc_get_family() processor family: Core 2 */
+#define FAMILY_CORE          6 /**< proc_get_family() processor family: Core */
+#define FAMILY_PENTIUM_M     6 /**< proc_get_family() processor family: Pentium M */
+#define FAMILY_PENTIUM_3     6 /**< proc_get_family() processor family: Pentium 3 */
+#define FAMILY_PENTIUM_2     6 /**< proc_get_family() processor family: Pentium 2 */
+#define FAMILY_PENTIUM_PRO   6 /**< proc_get_family() processor family: Pentium Pro */
+#define FAMILY_ATHLON        6 /**< proc_get_family() processor family: Athlon */
+/* Pentium (586) */
+#define FAMILY_PENTIUM       5 /**< proc_get_family() processor family: Pentium */
+#define FAMILY_K6            5 /**< proc_get_family() processor family: K6 */
+#define FAMILY_K5            5 /**< proc_get_family() processor family: K5 */
+/* 486 */
+#define FAMILY_486           4 /**< proc_get_family() processor family: 486 */
+
+/* We do not enumerate all models; just relevant ones needed to distinguish
+ * major processors in the same family.
+ */
+#define MODEL_PENTIUM_M_1MB  9 /**< proc_get_model() processor model: Pentium M 1MB L2 */
+#define MODEL_PENTIUM_M     13 /**< proc_get_model() processor model: Pentium M 2MB L2 */
+#define MODEL_CORE          14 /**< proc_get_model() processor model: Core */
+#define MODEL_CORE_2        15 /**< proc_get_model() processor model: Core 2 */
+
+/**
+ * Struct to hold all 4 32-bit feature values returned by cpuid.
+ * Used by proc_get_all_feature_bits().
+ */
+typedef struct {
+    uint flags_edx;             /**< feature flags stored in edx */
+    uint flags_ecx;             /**< feature flags stored in ecx */
+    uint ext_flags_edx;         /**< extended feature flags stored in edx */
+    uint ext_flags_ecx;         /**< extended feature flags stored in ecx */
+} features_t;
+
+/**
+ * Feature bits returned by cpuid.  Pass one of these values to proc_has_feature() to
+ * determine whether the underlying processor has the feature.
+ */
+typedef enum {
+    /* features returned in edx */
+    FEATURE_FPU =       0,              /**< Floating-point unit on chip */
+    FEATURE_VME =       1,              /**< Virtual Mode Extension */
+    FEATURE_DE =        2,              /**< Debugging Extension */
+    FEATURE_PSE =       3,              /**< Page Size Extension */
+    FEATURE_TSC =       4,              /**< Time-Stamp Counter */
+    FEATURE_MSR =       5,              /**< Model Specific Registers */
+    FEATURE_PAE =       6,              /**< Physical Address Extension */
+    FEATURE_MCE =       7,              /**< Machine Check Exception */
+    FEATURE_CX8 =       8,              /**< CMPXCHG8 Instruction Supported */
+    FEATURE_APIC =      9,              /**< On-chip APIC Hardware Supported */
+    FEATURE_SEP =       11,             /**< Fast System Call */
+    FEATURE_MTRR =      12,             /**< Memory Type Range Registers */
+    FEATURE_PGE =       13,             /**< Page Global Enable */
+    FEATURE_MCA =       14,             /**< Machine Check Architecture */
+    FEATURE_CMOV =      15,             /**< Conditional Move Instruction */
+    FEATURE_PAT =       16,             /**< Page Attribute Table */
+    FEATURE_PSE_36 =    17,             /**< 36-bit Page Size Extension */
+    FEATURE_PSN =       18,             /**< Processor serial # present & enabled */
+    FEATURE_CLFSH =     19,             /**< CLFLUSH Instruction supported */
+    FEATURE_DS =        21,             /**< Debug Store */
+    FEATURE_ACPI =      22,             /**< Thermal monitor & SCC supported */
+    FEATURE_MMX =       23,             /**< MMX technology supported */
+    FEATURE_FXSR =      24,             /**< Fast FP save and restore */
+    FEATURE_SSE =       25,             /**< SSE Extensions supported */
+    FEATURE_SSE2 =      26,             /**< SSE2 Extensions supported */
+    FEATURE_SS =        27,             /**< Self-snoop */
+    FEATURE_HTT =       28,             /**< Hyper-threading Technology */
+    FEATURE_TM =        29,             /**< Thermal Monitor supported */
+    FEATURE_IA64 =      30,             /**< IA64 Capabilities */
+    FEATURE_PBE =       31,             /**< Pending Break Enable */
+    /* features returned in ecx */
+    FEATURE_SSE3 =      0 + 32,         /**< SSE3 Extensions supported */
+    FEATURE_MONITOR =   3 + 32,         /**< MONITOR/MWAIT instructions supported */
+    FEATURE_DS_CPL =    4 + 32,         /**< CPL Qualified Debug Store */
+    FEATURE_VMX =       5 + 32,         /**< Virtual Machine Extensions */
+    FEATURE_EST =       7 + 32,         /**< Enhanced Speedstep Technology */
+    FEATURE_TM2 =       8 + 32,         /**< Thermal Monitor 2 */
+    FEATURE_SSSE3 =     9 + 32,         /**< SSSE3 Extensions supported */
+    FEATURE_CID =       10 + 32,        /**< Context ID */ 
+    FEATURE_CX16 =      13 + 32,        /**< CMPXCHG16B instruction supported */
+    FEATURE_xPTR =      14 + 32,        /**< Send Task Priority Messages */
+    /* extended features returned in edx */
+    FEATURE_SYSCALL =   11 + 64,        /**< SYSCALL/SYSRET instructions supported */
+    FEATURE_XD_Bit =    20 + 64,        /**< Execution Disable bit */
+    FEATURE_EM64T =     29 + 64,        /**< Extended Memory 64 Technology */
+    /* extended features returned in ecx */
+    FEATURE_LAHF =      0 + 96          /**< LAHF/SAHF available in 64-bit mode */
+} feature_bit_t;
+
+/**
+ * L1 and L2 cache sizes, used by proc_get_L1_icache_size(),
+ * proc_get_L1_dcache_size(), proc_get_L2_cache_size(), and
+ * proc_get_cache_size_str().
+ */
+#ifdef AVOID_API_EXPORT
+/* Make sure to keep this in sync with proc_get_cache_size_str() in proc.c */
+#endif
+typedef enum {
+    CACHE_SIZE_8_KB,    /**< L1 or L2 cache size of 8 KB. */
+    CACHE_SIZE_16_KB,   /**< L1 or L2 cache size of 16 KB. */
+    CACHE_SIZE_32_KB,   /**< L1 or L2 cache size of 32 KB. */
+    CACHE_SIZE_128_KB,  /**< L1 or L2 cache size of 128 KB. */
+    CACHE_SIZE_256_KB,  /**< L1 or L2 cache size of 256 KB. */
+    CACHE_SIZE_512_KB,  /**< L1 or L2 cache size of 512 KB. */
+    CACHE_SIZE_1_MB,    /**< L1 or L2 cache size of 1 MB. */
+    CACHE_SIZE_2_MB,    /**< L1 or L2 cache size of 2 MB. */
+    CACHE_SIZE_UNKNOWN  /**< Unknown L1 or L2 cache size. */
+} cache_size_t;
+
+/* DR_API EXPORT END */
+
+/* exported for efficient access */
+extern size_t cache_line_size;
+
+#define CACHE_LINE_SIZE() cache_line_size
+
+void proc_init(void);
+
+DR_API
+/** Returns the cache line size in bytes of the processor. */
+size_t
+proc_get_cache_line_size(void);
+
+DR_API
+/** Returns true only if \p addr is cache-line-aligned. */
+bool 
+proc_is_cache_aligned(void *addr);
+
+void
+machine_cache_sync(void *pc_start, void *pc_end, bool flush_icache);
+
+DR_API
+/** Returns n >= \p sz such that n is a multiple of the cache line size. */
+ptr_uint_t
+proc_bump_to_end_of_cache_line(ptr_uint_t sz);
+
+DR_API
+/** Returns n <= \p addr such that n is a multiple of the page size. */
+void *
+proc_get_containing_page(void *addr);
+
+DR_API
+/** Returns one of the VENDOR_ constants. */
+uint
+proc_get_vendor(void);
+
+DR_API
+/** 
+ * Returns the processor family as given by the cpuid instruction,
+ * adjusted by the extended family as described in the Intel documentation.
+ * The FAMILY_ constants identify important family values.
+ */
+uint
+proc_get_family(void);
+
+DR_API
+/** Returns the processor type as given by the cpuid instruction. */
+uint
+proc_get_type(void);
+
+DR_API
+/** 
+ * Returns the processor model as given by the cpuid instruction,
+ * adjusted by the extended model as described in the Intel documentation.
+ * The MODEL_ constants identify important model values.
+ */
+uint
+proc_get_model(void);
+
+DR_API
+/** Returns the processor stepping ID. */
+uint
+proc_get_stepping(void);
+
+DR_API
+/** Tests if processor has selected feature. */
+bool
+proc_has_feature(feature_bit_t feature);
+
+DR_API
+/**
+ * Returns all 4 32-bit feature values.  Use proc_has_feature to test
+ * for specific features.
+ */
+features_t *
+proc_get_all_feature_bits(void);
+
+DR_API
+/** Returns the processor brand string as given by the cpuid instruction. */
+char*
+proc_get_brand_string(void);
+
+DR_API
+/** Returns the size of the L1 instruction cache. */
+cache_size_t
+proc_get_L1_icache_size(void);
+
+DR_API
+/** Returns the size of the L1 data cache. */
+cache_size_t
+proc_get_L1_dcache_size(void);
+
+DR_API
+/** Returns the size of the L2 cache. */
+cache_size_t
+proc_get_L2_cache_size(void);
+
+DR_API
+/** Converts a cache_size_t type to a string. */
+const char*
+proc_get_cache_size_str(cache_size_t size);
+
+DR_API
+/**
+ * Returns the size in bytes needed for a buffer for saving the floating point state.
+ */
+size_t
+proc_fpstate_save_size(void);
+
+DR_API
+/**
+ * Saves the floating point state into the 16-byte-aligned buffer \p buf,
+ * which must be 512 bytes for processors with the FXSR feature, and
+ * 108 bytes for those without (where this routine does not support
+ * 16-bit operand sizing).  \note proc_fpstate_save_size() can be used
+ * to determine the particular size needed.
+ * 
+ * DR does NOT save the application's floating-point, MMX, or SSE state
+ * on context switches!  Thus if a client performs any floating-point 
+ * operations in its main routines called by DR, the client must save 
+ * and restore the floating-point/MMX/SSE state.
+ * If the client needs to do so inside the code cache the client should implement
+ * that itself.
+ * Returns number of bytes written.
+ */
+size_t
+proc_save_fpstate(byte *buf);
+
+DR_API
+/**
+ * Restores the floating point state from the 16-byte-aligned buffer
+ * \p buf, which must be 512 bytes for processors with the FXSR feature,
+ * and 108 bytes for those without (where this routine does not
+ * support 16-bit operand sizing).  \note proc_fpstate_save_size() can
+ * be used to determine the particular size needed.
+ */
+void 
+proc_restore_fpstate(byte *buf);
+
+#endif /* _PROC_H_ */
diff --git a/core/x86/retcheck.c b/core/x86/retcheck.c
new file mode 100644
index 000000000..aeaff60ce
--- /dev/null
+++ b/core/x86/retcheck.c
@@ -0,0 +1,1960 @@
+/* **********************************************************
+ * Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2003 Massachusetts Institute of Technology */
+
+/*
+ * retcheck.c
+ * Routines for the RETURN_AFTER_CALL and CHECK_RETURNS_SSE2 security features.
+ * FIXME: Experimental.
+ */
+
+#include "globals.h"
+#include "instrlist.h"
+#include "arch.h"
+#include "instr.h"
+#include "instr_create.h"
+#include "decode.h"
+
+#include "link.h"               /* for frag tag */
+#include "fragment.h"
+#include "rct.h"
+#include "instrument.h" /* for dr_insert_clean_call */
+
+#ifdef CHECK_RETURNS_SSE2
+/*
+ * retcheck.c
+ * Routines for the CHECK_RETURNS_SSE2 security feature.
+ *
+ * UNFINISHED:
+ * There are two methods, one use a stack pointer the other a
+ * constant top of stack.  Both can be optimized by using a
+ * shared routine to reduce code bloat.  Need to evaluate an
+ * optimized form of both and decide which is better!
+ * Without shared code, the shift version is faster on gcc,
+ * crafty, vortex, but table version is actually faster on the others!
+ *
+ * Crashes on release build on some programs
+ * Stack ptr dies on eon & swim
+ *
+ * Need to provide asm code for win32 (currently #error)
+ */
+
+#include <string.h> /* for memcpy */
+
+/* we have two ways of keeping our stack in the xmm registers:
+ * use one of them as a stack pointer, or have a constant top of
+ * stack and always shift the registers.
+ */
+#define SSE2_USE_STACK_POINTER 0
+
+/* keep mprotected stack in local or global heap? */
+#define USE_LOCAL_MPROT_STACK 0
+
+#if SSE2_USE_STACK_POINTER /* stack pointer and jump table method */
+# include "fragment.h"
+# include "link.h"
+#endif
+
+/* make code more readable by shortening long lines */
+#define POST instrlist_postinsert
+#define PRE  instrlist_preinsert
+
+
+/* UNFINISHED:
+ * start of code to have a shared routine for big table of sse2 instrs,
+ * to reduce code bloat.
+ * there is also code in arch.c and arch.h, under the same define
+ * (CHECK_RETURNS_SSE2_EMIT)
+ */
+#ifdef CHECK_RETURNS_SSE2_EMIT
+/* in arch.c */
+cache_pc get_pextrw_entry(dcontext_t *dcontext);
+cache_pc get_pinsrw_entry(dcontext_t *dcontext);
+
+byte *
+emit_pextrw(dcontext_t *dcontext, byte *pc)
+{
+    instrlist_t ilist;
+
+    /* initialize the ilist */
+    instrlist_init(&ilist);
+
+    for (i=0; i<62; i++) {
+        instrlist_append(&ilist,
+            INSTR_CREATE_pinsrw(dcontext, opnd_create_reg(REG_START_XMM + (i / 8)),
+                                OPND_CREATE_MEM32(REG_ESP, 0),
+                                OPND_CREATE_INT8(i % 8)));
+        instrlist_append(&ilist, INSTR_CREATE_jmp(dcontext, opnd_create_instr(end)));
+        instrlist_append(&ilist, INSTR_CREATE_nop(dcontext));
+    }
+    /* entry 62 */
+    instrlist_append(&ilist,
+        INSTR_CREATE_pinsrw(dcontext, opnd_create_reg(REG_START_XMM + (62 / 8)),
+                            OPND_CREATE_MEM32(REG_ESP, 0),
+                            OPND_CREATE_INT8(62 % 8)));
+    dr_insert_clean_call(dcontext, ilist, instr, (app_pc)check_return_too_deep,
+                         false/*!fp*/, 1, OPND_CREATE_INTPTR(dcontext));
+
+
+    /* now encode the instructions */
+    pc = instrlist_encode(dcontext, &ilist, pc, false /* no instr targets */);
+    ASSERT(pc != NULL);
+
+    /* free the instrlist_t elements */
+    instrlist_clear(dcontext, &ilist);
+
+    return pc;
+}
+#endif /* CHECK_RETURNS_SSE2_EMIT */
+
+
+#if SSE2_USE_STACK_POINTER /* stack pointer and jump table method */
+/* ################################################################################# */
+
+/* instr should be the instr AFTER the call instr */
+void
+check_return_handle_call(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr)
+{
+    /* ON CALL, STORE RETURN ADDRESS:
+          push ra  // normal push of ra
+          save ecx
+          pextrw xmm7,7 -> ecx
+          lea (,ecx,4) -> ecx      // ecx = ecx * 4, then ecx + ecx*2 = 12
+          lea next_addr(ecx,ecx,2) -> ecx
+          jmp ecx                  // pinsrw,jmp = 6+5 = 11 bytes, pad to 12
+          0:  pinsrw (esp),0 -> xmm0; jmp end; nop
+          1:  pinsrw (esp),1 -> xmm0; jmp end; nop
+          8:  pinsrw (esp),0 -> xmm1; jmp end; nop
+          62: pinsrw (esp),6 -> xmm7
+          <clean call to check_return_too_deep>
+                  // move 0..31 -> memory, mprotect the memory
+                  // then slide 32..63 down
+                  // set xmm7:7 to 30, let next instr inc it to get 31
+        end:  
+          pextrw xmm7,7 -> ecx
+          lea 1(ecx) -> ecx   // inc ecx
+          pinsrw ecx,7 -> xmm7
+          restore ecx
+     */
+    int i;
+    instr_t *end =
+        INSTR_CREATE_pextrw(dcontext, opnd_create_reg(REG_ECX),
+                            opnd_create_reg(REG_XMM7),
+                            OPND_CREATE_INT8(7));
+    PRE(ilist, instr,
+        instr_create_save_to_dcontext(dcontext, REG_ECX, XCX_OFFSET));
+    PRE(ilist, instr,
+        INSTR_CREATE_pextrw(dcontext, opnd_create_reg(REG_ECX),
+                            opnd_create_reg(REG_XMM7),
+                            OPND_CREATE_INT8(7)));
+    /* to get base+ecx*12, we do "ecx=ecx*4, ecx=base + ecx + ecx*2" */
+    PRE(ilist, instr,
+        INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ECX),
+                         opnd_create_base_disp(REG_NULL, REG_ECX, 4, 0, OPSZ_lea)));
+    PRE(ilist, instr,
+        INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ECX),
+                         opnd_create_base_disp(REG_ECX, REG_ECX, 2, 0xaaaaaaaa, OPSZ_lea)));
+#if DISABLE_FOR_ANALYSIS
+    PRE(ilist, instr,
+        INSTR_CREATE_jmp(dcontext, opnd_create_instr(end)));
+#else
+    PRE(ilist, instr,
+        INSTR_CREATE_jmp_ind(dcontext, opnd_create_reg(REG_ECX)));
+#endif
+    for (i=0; i<62; i++) {
+        PRE(ilist, instr,
+            INSTR_CREATE_pinsrw(dcontext, opnd_create_reg(REG_START_XMM + (i / 8)),
+                                OPND_CREATE_MEM32(REG_ESP, 0),
+                                OPND_CREATE_INT8(i % 8)));
+        PRE(ilist, instr, INSTR_CREATE_jmp(dcontext, opnd_create_instr(end)));
+        PRE(ilist, instr, INSTR_CREATE_nop(dcontext));
+    }
+    /* entry 62 */
+    PRE(ilist, instr,
+        INSTR_CREATE_pinsrw(dcontext, opnd_create_reg(REG_START_XMM + (62 / 8)),
+                            OPND_CREATE_MEM32(REG_ESP, 0),
+                            OPND_CREATE_INT8(62 % 8)));
+    dr_insert_clean_call(dcontext, ilist, instr, (app_pc)check_return_too_deep,
+                         false/*!fp*/, 1, OPND_CREATE_INTPTR(dcontext));
+    PRE(ilist, instr, end);
+    PRE(ilist, instr,
+        INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ECX),
+                         opnd_create_base_disp(REG_ECX, REG_NULL, 0, 1, OPSZ_lea)));
+#if !DISABLE_FOR_ANALYSIS
+    PRE(ilist, instr,
+        INSTR_CREATE_pinsrw(dcontext, opnd_create_reg(REG_XMM7),
+                            opnd_create_reg(REG_ECX),
+                            OPND_CREATE_INT8(7)));
+#endif
+    PRE(ilist, instr,
+        instr_create_restore_from_dcontext(dcontext, REG_ECX, XCX_OFFSET));
+}
+
+void
+check_return_handle_return(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr)
+{
+    /* ON RETURN, CHECK RETURN ADDRESS:
+          pop ra -> ecx  // normal pop
+          save edx
+          mov ecx, edx
+          pextrw xmm7,7 -> ecx
+          jecxz at_zero
+          lea -1(ecx) -> ecx  // dec ecx
+          jmp non_zero
+        at_zero:
+          mov 31, ecx
+          <clean call to check_return_too_shallow> // restore 0..31 from memory
+        non_zero:
+          pinsrw ecx,7 -> xmm7     // store it back
+          lea (,ecx,2) -> ecx      // ecx = ecx * 2
+          lea next_addr(ecx,ecx,4) -> ecx   // ecx = ecx+ecx*4 = old_ecx*10
+          jmp ecx                  // pextrw,jmp = 5+5 = 10 bytes
+          0:  pextrw xmm0,0 -> ecx; jmp end
+          1:  pextrw xmm0,1 -> ecx; jmp end
+          8:  pextrw xmm1,0 -> ecx; jmp end
+          62: pextrw xmm7,6 -> ecx;
+        end:
+          movzx dx,edx  // clear top 16 bits, for cmp w/ stored bottom 16 bits
+          not %ecx
+          lea 1(%ecx,%edx,1),%ecx  // "not ecx + 1" => -ecx, to cmp w/ edx
+          jecxz ra_not_mangled
+          call ra_mangled
+        ra_not_mangled:
+          restore edx
+// FIXME: can't count on below esp not being clobbered!  (could get signal->handler!)
+          mov -4(esp),ecx // restore return address
+    */
+    int i;
+    instr_t *ra_not_mangled = 
+        instr_create_restore_from_dcontext(dcontext, REG_EDX, XDX_OFFSET);
+    instr_t *end =
+        INSTR_CREATE_movzx(dcontext, opnd_create_reg(REG_EDX), opnd_create_reg(REG_DX));
+    instr_t *at_zero =
+        INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_ECX), OPND_CREATE_INT32(31));
+    instr_t *non_zero =
+        INSTR_CREATE_pinsrw(dcontext, opnd_create_reg(REG_XMM7),
+                            opnd_create_reg(REG_ECX),
+                            OPND_CREATE_INT8(7));
+    PRE(ilist, instr,
+        instr_create_save_to_dcontext(dcontext, REG_EDX, XDX_OFFSET));
+    PRE(ilist, instr,
+        INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_EDX), opnd_create_reg(REG_ECX)));
+    PRE(ilist, instr,
+        INSTR_CREATE_pextrw(dcontext, opnd_create_reg(REG_ECX),
+                            opnd_create_reg(REG_XMM7),
+                            OPND_CREATE_INT8(7)));
+#if DISABLE_FOR_ANALYSIS
+    PRE(ilist, instr,
+        INSTR_CREATE_jmp(dcontext, opnd_create_instr(ra_not_mangled)));
+#endif
+    PRE(ilist, instr,
+        INSTR_CREATE_jecxz(dcontext, opnd_create_instr(at_zero)));
+    PRE(ilist, instr,
+        INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ECX),
+                         opnd_create_base_disp(REG_ECX, REG_NULL, 0, -1, OPSZ_lea)));
+    PRE(ilist, instr,
+        INSTR_CREATE_jmp(dcontext, opnd_create_instr(non_zero)));
+    PRE(ilist, instr, at_zero);
+    dr_insert_clean_call(dcontext, ilist, instr, (app_pc)check_return_too_shallow,
+                         false/*!fp*/, 1, OPND_CREATE_INTPTR(dcontext));
+    PRE(ilist, instr, non_zero);
+    /* to get base+ecx*10, we do "ecx=ecx*2, ecx=base + ecx + ecx*4" */
+    PRE(ilist, instr,
+        INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ECX),
+                         opnd_create_base_disp(REG_NULL, REG_ECX, 2, 0, OPSZ_lea)));
+    PRE(ilist, instr,
+        INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ECX),
+                         opnd_create_base_disp(REG_ECX, REG_ECX, 4, 0xaaaaaaaa, OPSZ_lea)));
+    PRE(ilist, instr,
+        INSTR_CREATE_jmp_ind(dcontext, opnd_create_reg(REG_ECX)));
+    for (i=0; i<63; i++) {
+        PRE(ilist, instr,
+            INSTR_CREATE_pextrw(dcontext, opnd_create_reg(REG_ECX),
+                                opnd_create_reg(REG_START_XMM + (i / 8)),
+                                OPND_CREATE_INT8(i % 8)));
+        PRE(ilist, instr,
+            INSTR_CREATE_jmp(dcontext, opnd_create_instr(end)));
+    }
+    PRE(ilist, instr, end);
+    PRE(ilist, instr, INSTR_CREATE_not(dcontext, opnd_create_reg(REG_ECX)));
+    PRE(ilist, instr,
+        INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ECX),
+                         opnd_create_base_disp(REG_ECX, REG_EDX, 1, 1, OPSZ_lea)));
+    PRE(ilist, instr,
+        INSTR_CREATE_jecxz(dcontext, opnd_create_instr(ra_not_mangled)));
+    dr_insert_clean_call(dcontext, ilist, instr, (app_pc)check_return_ra_mangled,
+                         false/*!fp*/, 1, OPND_CREATE_INTPTR(dcontext));
+    PRE(ilist, instr, ra_not_mangled);
+    PRE(ilist, instr,
+        INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_ECX), OPND_CREATE_MEM32(REG_ESP, -4)));
+}
+
+/* touches up jmp* for table (needs address of start of table) */
+void
+finalize_return_check(dcontext_t *dcontext, fragment_t *f)
+{
+    byte *start_pc = (byte *) FCACHE_ENTRY_PC(f);
+    byte *end_pc = fragment_body_end_pc(dcontext, f);
+    byte *pc, *prev_pc;
+    int leas_next = 0;
+    instr_t instr;
+    instr_init(dcontext, &instr);
+    LOG(THREAD, LOG_ALL, 3, "finalize_return_check\n");
+
+    SELF_PROTECT_CACHE(dcontext, f, WRITABLE);
+
+    /* must fix up indirect jmp */
+    pc = start_pc;
+    do {
+        prev_pc = pc;
+        instr_reset(dcontext, &instr);
+        pc = decode(dcontext, pc, &instr);
+        ASSERT(instr_valid(&instr)); /* our own code! */
+        if (leas_next == 2) {
+            loginst(dcontext, 3, &instr, "\tlea 2");
+            if (instr_get_opcode(&instr) == OP_lea) {
+                opnd_t op = instr_get_src(&instr, 0);
+                int scale = opnd_get_scale(op);
+                DEBUG_DECLARE(byte *nxt_pc;)
+                /* put in pc of instr after jmp: jmp is 2 bytes long */
+                instr_set_src(&instr, 0,
+                              opnd_create_base_disp(REG_ECX, REG_ECX, scale,
+                                                    (int)(pc+2), OPSZ_lea));
+                DEBUG_DECLARE(nxt_pc = ) instr_encode(dcontext, &instr, prev_pc);
+                ASSERT(nxt_pc != NULL);
+            }
+            leas_next = 0;
+        }
+        if (leas_next == 1) {
+            loginst(dcontext, 3, &instr, "\tlea 1");
+            if (instr_get_opcode(&instr) == OP_lea)
+                leas_next = 2;
+            else
+                leas_next = 0;
+        }
+        /* we don't allow program to use sse, so pextrw/pinsrw are all ours */
+        if (leas_next == 0 &&
+            instr_get_opcode(&instr) == OP_pextrw &&
+            opnd_is_reg(instr_get_src(&instr, 0)) &&
+            opnd_get_reg(instr_get_src(&instr, 0)) == REG_XMM7 &&
+            opnd_is_immed_int(instr_get_src(&instr, 1)) &&
+            opnd_get_immed_int(instr_get_src(&instr, 1)) == 7) {
+            loginst(dcontext, 3, &instr, "\tfound pextrw");
+            leas_next = 1;
+        }
+        else if (leas_next == 0 &&
+            instr_get_opcode(&instr) == OP_pinsrw &&
+            opnd_is_reg(instr_get_dst(&instr, 0)) &&
+            opnd_get_reg(instr_get_dst(&instr, 0)) == REG_XMM7 &&
+            opnd_is_immed_int(instr_get_src(&instr, 1)) &&
+            opnd_get_immed_int(instr_get_src(&instr, 1)) == 7) {
+            loginst(dcontext, 3, &instr, "\tfound pinsrw");
+            leas_next = 1;
+        }
+    } while (pc < end_pc);
+    instr_free(dcontext, &instr);
+
+    SELF_PROTECT_CACHE(dcontext, f, READONLY);
+}
+
+typedef struct _call_stack_32 {
+    byte retaddr[32][2];
+    struct _call_stack_32 *next;
+} call_stack_32_t;
+
+/* move 0..31 -> memory, mprotect the memory
+ * then slide 32..63 down
+ * set xmm7:7 to 30, let next instr inc it to get 31
+ */
+void
+check_return_too_deep(dcontext_t *dcontext,
+                      volatile int errno, volatile reg_t eflags,
+                      volatile reg_t reg_edi, volatile reg_t reg_esi,
+                      volatile reg_t reg_ebp, volatile reg_t reg_esp,
+                      volatile reg_t reg_ebx, volatile reg_t reg_edx,
+                      volatile reg_t reg_ecx, volatile reg_t reg_eax)
+{
+    byte xmm[8][16]; /* each sse2 is 128 bits = 16 bytes */
+    call_stack_32_t *stack;
+
+    ENTERING_DR();
+#if USE_LOCAL_MPROT_STACK
+    SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+#endif
+
+#if USE_LOCAL_MPROT_STACK
+    stack = heap_alloc(dcontext, sizeof(call_stack_32_t));
+#else
+    stack = global_heap_alloc(sizeof(call_stack_32_t) HEAPACCT(ACCT_OTHER));
+#endif
+    stack->next = dcontext->call_stack;
+    dcontext->call_stack = stack;
+
+    LOG(THREAD, LOG_ALL, 3, "check_return_too_deep\n");
+
+    /* move from registers into memory where we can work with it */
+    /* FIXME: align xmm so can use movdqa! */
+#ifdef LINUX
+    asm("movdqu %%xmm0, %0" : "=m"(xmm[0]));
+    asm("movdqu %%xmm1, %0" : "=m"(xmm[1]));
+    asm("movdqu %%xmm2, %0" : "=m"(xmm[2]));
+    asm("movdqu %%xmm3, %0" : "=m"(xmm[3]));
+    asm("movdqu %%xmm4, %0" : "=m"(xmm[4]));
+    asm("movdqu %%xmm5, %0" : "=m"(xmm[5]));
+    asm("movdqu %%xmm6, %0" : "=m"(xmm[6]));
+    asm("movdqu %%xmm7, %0" : "=m"(xmm[7]));
+#else
+#error NYI
+#endif
+
+    LOG(THREAD, LOG_ALL, 3, "\tjust copied registers\n");
+
+    /* we want 0..31 into our stack, that's the first 64 bytes */
+    memcpy(stack->retaddr, xmm[0], 64);
+
+#ifdef DEBUG
+    if (stats->loglevel >= 3) {
+        int i,j;
+        LOG(THREAD, LOG_ALL, 3, "Copied into stored stack:\n");
+        for (i=0; i<4; i++) {
+            for (j=0; j<8; j++) {
+                LOG(THREAD, LOG_ALL, 3, "\t%d %d 0x%02x%02x", i, j,
+                    stack->retaddr[i*8+j][0],  stack->retaddr[i*8+j][1]);
+                if (j % 4 == 3)
+                    LOG(THREAD, LOG_ALL, 3, "\n");
+            }
+        }
+        LOG(THREAD, LOG_ALL, 3, "Before shifting:\n");
+        for (i=0; i<8; i++) {
+            for (j=0; j<8; j++) {
+                LOG(THREAD, LOG_ALL, 3, "\t%d %d 0x%02x%02x", i, j, xmm[i][j*2], xmm[i][j*2+1]);
+                if (j % 4 == 3)
+                    LOG(THREAD, LOG_ALL, 3, "\n");
+            }
+        }
+    }
+#endif
+
+    /* now slide 32..63 down */
+    memcpy(xmm[0], xmm[4], 64);
+
+    /* move back into registers */
+#ifdef LINUX
+    asm("movdqu %0, %%xmm0" : : "m"(xmm[0][0]));
+    asm("movdqu %0, %%xmm1" : : "m"(xmm[1][0]));
+    asm("movdqu %0, %%xmm2" : : "m"(xmm[2][0]));
+    asm("movdqu %0, %%xmm3" : : "m"(xmm[3][0]));
+    asm("movl $30, %eax");
+    asm("pinsrw $7,%eax,%xmm7");
+#else
+#error NYI
+#endif
+
+    dcontext->call_depth++;
+
+    LOG(THREAD, LOG_ALL, 3, "\tdone, call depth is now %d\n", dcontext->call_depth);
+
+#if USE_LOCAL_MPROT_STACK
+    SELF_PROTECT_LOCAL(dcontext, READONLY);
+#endif
+    EXITING_DR();
+}
+
+void
+check_return_too_shallow(dcontext_t *dcontext,
+                         volatile int errno, volatile reg_t eflags,
+                         volatile reg_t reg_edi, volatile reg_t reg_esi,
+                         volatile reg_t reg_ebp, volatile reg_t reg_esp,
+                         volatile reg_t reg_ebx, volatile reg_t reg_edx,
+                         volatile reg_t reg_ecx, volatile reg_t reg_eax)
+{
+    ENTERING_DR();
+#if USE_LOCAL_MPROT_STACK
+    SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+#endif
+
+    LOG(THREAD, LOG_ALL, 3, "check_return_too_shallow\n");
+    if (dcontext->call_depth == 0) {
+        LOG(THREAD, LOG_ALL, 3, "\tbottomed out of dynamo, ignoring\n");
+        reg_ecx = 0; /* undo the set to 31 prior to this call */
+        /* FIXME: would like to avoid rest of checks...but then have to put
+         * clean-call-cleanup at bottom...instead we have a hack where we put
+         * in a ret addr that will match, namely the real ret addr, sitting in edx
+         */
+#ifdef LINUX
+        asm("movl %0, %%eax" : : "m"(reg_edx));
+        asm("pinsrw $0,%eax,%xmm0");
+#else
+#error NYI
+#endif
+        LOG(THREAD, LOG_ALL, 3, "\tset xmm0:0 to "PFX"\n", reg_edx);
+    } else {
+        /* restore 0..31 from memory */
+        call_stack_32_t *stack = dcontext->call_stack;
+        ASSERT(stack != NULL);
+        /* move back into registers */
+#ifdef LINUX
+        asm("movl %0, %%eax" : : "m"(stack->retaddr));
+        asm("movdqu (%eax),     %xmm0");
+        asm("movdqu 0x10(%eax), %xmm1");
+        asm("movdqu 0x20(%eax), %xmm2");
+        asm("movdqu 0x30(%eax), %xmm3");
+#else
+#error NYI
+#endif
+#ifdef DEBUG
+        if (stats->loglevel >= 3) {
+            int i,j;
+            LOG(THREAD, LOG_ALL, 3, "Restored:\n");
+            for (i=0; i<4; i++) {
+                for (j=0; j<8; j++) {
+                    LOG(THREAD, LOG_ALL, 3, "\t%d %d 0x%02x%02x", i, j,
+                        stack->retaddr[i*8+j][0],  stack->retaddr[i*8+j][1]);
+                    if (j % 4 == 3)
+                        LOG(THREAD, LOG_ALL, 3, "\n");
+                }
+            }
+        }
+#endif
+        stack = stack->next;
+#if USE_LOCAL_MPROT_STACK
+        heap_free(dcontext, dcontext->call_stack, sizeof(call_stack_32_t));
+#else
+        global_heap_free(dcontext->call_stack, sizeof(call_stack_32_t) HEAPACCT(ACCT_OTHER));
+#endif
+        dcontext->call_stack = stack;
+        dcontext->call_depth--;
+        LOG(THREAD, LOG_ALL, 3, "\tdone, call depth is now %d\n", dcontext->call_depth);
+    }
+
+#if USE_LOCAL_MPROT_STACK
+    SELF_PROTECT_LOCAL(dcontext, READONLY);
+#endif
+    EXITING_DR();
+}
+
+void
+check_return_ra_mangled(dcontext_t *dcontext,
+                        volatile int errno, volatile reg_t eflags,
+                        volatile reg_t reg_edi, volatile reg_t reg_esi,
+                        volatile reg_t reg_ebp, volatile reg_t reg_esp,
+                        volatile reg_t reg_ebx, volatile reg_t reg_edx,
+                        volatile reg_t reg_ecx, volatile reg_t reg_eax)
+{
+    /* ecx had addr, then we did ecx' = edx-ecx, so old ecx = edx - ecx' */
+    int stored_addr = reg_edx - reg_ecx;
+
+    ENTERING_DR();
+    SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+
+#ifdef DEBUG
+    if (stats->loglevel >= 3 && (stats->logmask & LOG_ALL) != 0) {
+        int idx;
+# ifdef LINUX
+        asm("pextrw $7,%xmm7,%eax");
+        asm("movl %%eax, %0" : "=m"(idx));
+# else
+# error NYI
+# endif
+        LOG(THREAD, LOG_ALL, 3,
+            "check_return_ra_mangled: stored="PFX" vs real="PFX", idx=%d\n",
+            stored_addr, reg_edx, idx);
+    }
+#endif
+
+    SYSLOG_INTERNAL_ERROR("ERROR: return address was mangled (bottom 16 bits: "
+                          "0x%04x => 0x%04x)", (reg_edx & 0x0000ffff), stored_addr);
+    ASSERT_NOT_REACHED();
+
+    SELF_PROTECT_LOCAL(dcontext, READONLY);
+    EXITING_DR();
+}
+
+#else /* !SSE2_USE_STACK_POINTER */
+/* ################################################################################# */
+/* instr should be the instr AFTER the call instr */
+void
+check_return_handle_call(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr)
+{
+    /* ON CALL, STORE RETURN ADDRESS:
+          push ra  // normal push of ra
+          save ecx
+          pextrw xmm7,7 -> ecx
+          lea -63(ecx) -> ecx
+          jecxz overflow
+          jmp non_overflow
+        overflow:
+          <clean call to check_return_too_deep>
+                  // move 31..62 -> memory, mprotect the memory
+                  // set xmm7:7 to 32 by setting ecx = 32-64
+        non_overflow:
+          pslldq xmm7,2        # shift left one word
+          lea 64(ecx) -> ecx   # restore, plus increment, the index
+          pinsrw ecx,7 -> xmm7 # put index in its slot
+          pextrw xmm6,7 -> ecx # move top of 6 to bottom of 7
+          pinsrw ecx,0 -> xmm7 #
+          pslldq xmm6,2        # now shift 6 left one word
+          pextrw xmm5,7 -> ecx # move top of 5 to bottom of 6
+          pinsrw ecx,0 -> xmm6 #
+          pslldq xmm5,2        # now shift 5 left one word
+          pextrw xmm4,7 -> ecx # move top of 4 to bottom of 5
+          pinsrw ecx,0 -> xmm5 #
+          pslldq xmm4,2        # now shift 4 left one word
+          pextrw xmm3,7 -> ecx # move top of 3 to bottom of 4
+          pinsrw ecx,0 -> xmm4 #
+          pslldq xmm3,2        # now shift 3 left one word
+          pextrw xmm2,7 -> ecx # move top of 2 to bottom of 3
+          pinsrw ecx,0 -> xmm3 #
+          pslldq xmm2,2        # now shift 2 left one word
+          pextrw xmm1,7 -> ecx # move top of 1 to bottom of 2
+          pinsrw ecx,0 -> xmm2 #
+          pslldq xmm1,2        # now shift 1 left one word
+          pextrw xmm0,7 -> ecx # move top of 0 to bottom of 1
+          pinsrw ecx,0 -> xmm1 #
+          pslldq xmm0,2        # now shift 0 left one word
+          pinsrw (esp),0 -> xmm0   # now store new return address
+        end:  
+          restore ecx
+     */
+    int i;
+    instr_t *end =
+        instr_create_restore_from_dcontext(dcontext, REG_ECX, XCX_OFFSET);
+    instr_t *overflow = INSTR_CREATE_nop(dcontext);
+    instr_t *non_overflow =
+        INSTR_CREATE_pslldq(dcontext, opnd_create_reg(REG_XMM7), OPND_CREATE_INT8(2));
+    PRE(ilist, instr,
+        instr_create_save_to_dcontext(dcontext, REG_ECX, XCX_OFFSET));
+    PRE(ilist, instr,
+        INSTR_CREATE_pextrw(dcontext, opnd_create_reg(REG_ECX),
+                            opnd_create_reg(REG_XMM7), OPND_CREATE_INT8(7)));
+    PRE(ilist, instr,
+        INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ECX),
+                         opnd_create_base_disp(REG_ECX, REG_NULL, 0, -63, OPSZ_lea)));
+    PRE(ilist, instr,
+        INSTR_CREATE_jecxz(dcontext, opnd_create_instr(overflow)));
+    PRE(ilist, instr,
+        INSTR_CREATE_jmp(dcontext, opnd_create_instr(non_overflow)));
+    PRE(ilist, instr, overflow);
+    dr_insert_clean_call(dcontext, ilist, instr, (app_pc)check_return_too_deep,
+                         false/*!fp*/, 1, OPND_CREATE_INTPTR(dcontext));
+    PRE(ilist, instr, non_overflow);
+    PRE(ilist, instr,
+        INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ECX),
+                         opnd_create_base_disp(REG_ECX, REG_NULL, 0, 64, OPSZ_lea)));
+    PRE(ilist, instr,
+        INSTR_CREATE_pinsrw(dcontext, opnd_create_reg(REG_XMM7),
+                            opnd_create_reg(REG_ECX), OPND_CREATE_INT8(7)));
+    for (i=6; i>=0; i--) {
+        PRE(ilist, instr,
+            INSTR_CREATE_pextrw(dcontext, opnd_create_reg(REG_ECX),
+                                opnd_create_reg(REG_START_XMM + i), OPND_CREATE_INT8(7)));
+        PRE(ilist, instr,
+            INSTR_CREATE_pinsrw(dcontext, opnd_create_reg(REG_START_XMM + i + 1),
+                                opnd_create_reg(REG_ECX), OPND_CREATE_INT8(0)));
+        PRE(ilist, instr,
+            INSTR_CREATE_pslldq(dcontext, opnd_create_reg(REG_START_XMM + i),
+                                OPND_CREATE_INT8(2)));
+    }
+    PRE(ilist, instr,
+        INSTR_CREATE_pinsrw(dcontext, opnd_create_reg(REG_XMM0),
+                            OPND_CREATE_MEM32(REG_ESP, 0), OPND_CREATE_INT8(0)));
+    PRE(ilist, instr, end);
+}
+
+#ifdef DEBUG
+# if 0 /* not used */
+static void
+check_debug_regs(dcontext_t *dcontext,
+             volatile int errno, volatile reg_t eflags,
+             volatile reg_t reg_edi, volatile reg_t reg_esi,
+             volatile reg_t reg_ebp, volatile reg_t reg_esp,
+             volatile reg_t reg_ebx, volatile reg_t reg_edx,
+             volatile reg_t reg_ecx, volatile reg_t reg_eax)
+{
+    LOG(THREAD, LOG_ALL, 3, "check_debug2: eax="PFX" ecx="PFX" edx="PFX" ebx="PFX"\n"
+        "esp="PFX" ebp="PFX" esi="PFX" edi="PFX"\n",
+        reg_eax, reg_ecx, reg_edx, reg_ebx, reg_esp, reg_ebp, reg_esi, reg_edi);
+}
+# endif
+
+static void
+check_debug(dcontext_t *dcontext,
+            volatile int errno, volatile reg_t eflags,
+            volatile reg_t reg_edi, volatile reg_t reg_esi,
+            volatile reg_t reg_ebp, volatile reg_t reg_esp,
+            volatile reg_t reg_ebx, volatile reg_t reg_edx,
+            volatile reg_t reg_ecx, volatile reg_t reg_eax)
+{
+    ENTERING_DR();
+    SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+    if (stats->loglevel >= 3) {
+        int i, j;
+        byte xmm[8][16]; /* each sse2 is 128 bits = 16 bytes */
+        /* move from registers into memory where we can work with it */
+#ifdef LINUX
+        asm("movdqu %%xmm0, %0" : "=m"(xmm[0]));
+        asm("movdqu %%xmm1, %0" : "=m"(xmm[1]));
+        asm("movdqu %%xmm2, %0" : "=m"(xmm[2]));
+        asm("movdqu %%xmm3, %0" : "=m"(xmm[3]));
+        asm("movdqu %%xmm4, %0" : "=m"(xmm[4]));
+        asm("movdqu %%xmm5, %0" : "=m"(xmm[5]));
+        asm("movdqu %%xmm6, %0" : "=m"(xmm[6]));
+        asm("movdqu %%xmm7, %0" : "=m"(xmm[7]));
+#else
+#error NYI
+#endif
+        LOG(THREAD, LOG_ALL, 3, "on our stack (in edx is "PFX"):\n", reg_edx);
+        for (i=0; i<8; i++) {
+            for (j=0; j<8; j++) {
+                LOG(THREAD, LOG_ALL, 3, "\t%d %d 0x%02x%02x",
+                    i, j, xmm[i][j*2+1], xmm[i][j*2]);
+                if (j % 4 == 3)
+                    LOG(THREAD, LOG_ALL, 3, "\n");
+            }
+        }
+    }
+    SELF_PROTECT_LOCAL(dcontext, READONLY);
+    EXITING_DR();
+}
+#endif /* DEBUG */
+
+void
+check_return_handle_return(dcontext_t *dcontext, instrlist_t *ilist, instr_t *instr)
+{
+    /* ON RETURN, CHECK RETURN ADDRESS:
+          pop ra -> ecx  // normal pop
+          save edx
+          mov ecx, edx
+          save ebx
+          pextrw xmm7,7 -> ecx
+          jecxz at_zero
+          lea -1(ecx) -> ecx    # dec ecx
+          pinsrw ecx,7 -> xmm7  # store index
+          jmp non_zero
+        at_zero:
+          <clean call to check_return_too_shallow>
+                  // restore from memory to 0..31
+                  // copy xmm0:0 into ebx
+                  // shift 1..31 down into 0..30
+                  // set xmm7:7 to 31
+          jmp end
+        non_zero:
+          pextrw xmm0,0 -> ebx
+          psrldq xmm0,2         # shift 0 right one word
+          pextrw xmm1,0 -> ecx  # move bottom of 1 to top of 0
+          pinsrw ecx,7 -> xmm0
+          psrldq xmm1,2         
+          pextrw xmm2,0 -> ecx  # move bottom of 2 to top of 1
+          pinsrw ecx,7 -> xmm1
+          psrldq xmm2,2         
+          pextrw xmm3,0 -> ecx  # move bottom of 3 to top of 2
+          pinsrw ecx,7 -> xmm2
+          psrldq xmm3,2         
+          pextrw xmm4,0 -> ecx  # move bottom of 4 to top of 3
+          pinsrw ecx,7 -> xmm3
+          psrldq xmm4,2         
+          pextrw xmm5,0 -> ecx  # move bottom of 5 to top of 4
+          pinsrw ecx,7 -> xmm4
+          psrldq xmm5,2         
+          pextrw xmm6,0 -> ecx  # move bottom of 6 to top of 5
+          pinsrw ecx,7 -> xmm5
+          psrldq xmm6,2         
+          pextrw xmm7,0 -> ecx  # move bottom of 7 to top of 6
+          pinsrw ecx,7 -> xmm6
+          psrldq xmm7,2         
+          pextrw xmm7,6 -> ecx  # shift index back to top slot
+          pinsrw ecx,7 -> xmm7
+        end:
+          mov edx,ecx
+          movzx cx,ecx  // clear top 16 bits, for cmp w/ stored bottom 16 bits
+          not %ebx
+          lea 1(%ebx,%ecx,1),%ecx  // "not ebx + 1" => -ecx, to cmp w/ ecx
+          jecxz ra_not_mangled
+          call ra_mangled
+        ra_not_mangled:
+          restore ebx
+          mov edx, ecx // restore ret addr
+          restore edx
+    */
+    int i;
+    instr_t *ra_not_mangled = 
+        instr_create_restore_from_dcontext(dcontext, REG_EBX, XBX_OFFSET);
+    instr_t *end =
+        INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_ECX), opnd_create_reg(REG_EDX));
+    instr_t *at_zero = INSTR_CREATE_nop(dcontext);
+    instr_t *non_zero =
+        INSTR_CREATE_pextrw(dcontext, opnd_create_reg(REG_EBX),
+                            opnd_create_reg(REG_XMM0), OPND_CREATE_INT8(0));
+    PRE(ilist, instr,
+        instr_create_save_to_dcontext(dcontext, REG_EDX, XDX_OFFSET));
+    PRE(ilist, instr,
+        INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_EDX), opnd_create_reg(REG_ECX)));
+    PRE(ilist, instr,
+        instr_create_save_to_dcontext(dcontext, REG_EBX, XBX_OFFSET));
+
+#ifdef DEBUG
+    if (stats->loglevel >= 4) {
+        dr_insert_clean_call(dcontext, ilist, instr, (app_pc)check_debug,
+                             false/*!fp*/, 1, OPND_CREATE_INTPTR(dcontext));
+    }
+#endif
+
+    PRE(ilist, instr,
+        INSTR_CREATE_pextrw(dcontext, opnd_create_reg(REG_ECX),
+                            opnd_create_reg(REG_XMM7), OPND_CREATE_INT8(7)));
+    PRE(ilist, instr,
+        INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_EBX), opnd_create_reg(REG_ECX)));
+    PRE(ilist, instr,
+        INSTR_CREATE_jecxz(dcontext, opnd_create_instr(at_zero)));
+    PRE(ilist, instr,
+        INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ECX),
+                         opnd_create_base_disp(REG_ECX, REG_NULL, 0, -1, OPSZ_lea)));
+    PRE(ilist, instr,
+        INSTR_CREATE_pinsrw(dcontext, opnd_create_reg(REG_XMM7),
+                            opnd_create_reg(REG_ECX), OPND_CREATE_INT8(7)));
+    PRE(ilist, instr,
+        INSTR_CREATE_jmp(dcontext, opnd_create_instr(non_zero)));
+    PRE(ilist, instr, at_zero);
+    dr_insert_clean_call(dcontext, ilist, instr, (app_pc)check_return_too_shallow,
+                         false/*!fp*/, 1, OPND_CREATE_INTPTR(dcontext));
+    PRE(ilist, instr,
+        INSTR_CREATE_jmp(dcontext, opnd_create_instr(end)));
+    PRE(ilist, instr, non_zero);
+    PRE(ilist, instr,
+        INSTR_CREATE_psrldq(dcontext, opnd_create_reg(REG_XMM0), OPND_CREATE_INT8(2)));
+    for (i=1; i<=7; i++) {
+        PRE(ilist, instr,
+            INSTR_CREATE_pextrw(dcontext, opnd_create_reg(REG_ECX),
+                                opnd_create_reg(REG_START_XMM + i), OPND_CREATE_INT8(0)));
+        PRE(ilist, instr,
+            INSTR_CREATE_pinsrw(dcontext, opnd_create_reg(REG_START_XMM + i - 1),
+                                opnd_create_reg(REG_ECX), OPND_CREATE_INT8(7)));
+        PRE(ilist, instr,
+            INSTR_CREATE_psrldq(dcontext, opnd_create_reg(REG_START_XMM + i),
+                                OPND_CREATE_INT8(2)));
+    }
+    PRE(ilist, instr,
+        INSTR_CREATE_pextrw(dcontext, opnd_create_reg(REG_ECX),
+                            opnd_create_reg(REG_XMM7), OPND_CREATE_INT8(6)));
+    PRE(ilist, instr,
+        INSTR_CREATE_pinsrw(dcontext, opnd_create_reg(REG_XMM7),
+                            opnd_create_reg(REG_ECX), OPND_CREATE_INT8(7)));
+    PRE(ilist, instr, end);
+    PRE(ilist, instr,
+        INSTR_CREATE_movzx(dcontext, opnd_create_reg(REG_ECX), opnd_create_reg(REG_CX)));
+    PRE(ilist, instr, INSTR_CREATE_not(dcontext, opnd_create_reg(REG_EBX)));
+    PRE(ilist, instr,
+        INSTR_CREATE_lea(dcontext, opnd_create_reg(REG_ECX),
+                         opnd_create_base_disp(REG_EBX, REG_ECX, 1, 1, OPSZ_lea)));
+    PRE(ilist, instr,
+        INSTR_CREATE_jecxz(dcontext, opnd_create_instr(ra_not_mangled)));
+    dr_insert_clean_call(dcontext, ilist, instr, (app_pc)check_return_ra_mangled,
+                         false/*!fp*/, 1, OPND_CREATE_INTPTR(dcontext));
+    PRE(ilist, instr, ra_not_mangled);
+    PRE(ilist, instr,
+        INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_ECX), opnd_create_reg(REG_EDX)));
+    PRE(ilist, instr,
+        instr_create_restore_from_dcontext(dcontext, REG_EDX, XDX_OFFSET));
+}
+
+/* touches up jmp* for table (needs address of start of table) */
+void
+finalize_return_check(dcontext_t *dcontext, fragment_t *f)
+{
+}
+
+typedef struct _call_stack_32 {
+    byte retaddr[32][2];
+    struct _call_stack_32 *next;
+} call_stack_32_t;
+
+/* move 0..31 -> memory, mprotect the memory
+ * then slide 32..63 down
+ * set xmm7:7 to 30, let next instr inc it to get 31
+ */
+void
+check_return_too_deep(dcontext_t *dcontext,
+                      volatile int errno, volatile reg_t eflags,
+                      volatile reg_t reg_edi, volatile reg_t reg_esi,
+                      volatile reg_t reg_ebp, volatile reg_t reg_esp,
+                      volatile reg_t reg_ebx, volatile reg_t reg_edx,
+                      volatile reg_t reg_ecx, volatile reg_t reg_eax)
+{
+    byte xmm[8][16]; /* each sse2 is 128 bits = 16 bytes */
+    call_stack_32_t *stack;
+
+    ENTERING_DR();
+#if USE_LOCAL_MPROT_STACK
+    SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+#endif
+
+#if USE_LOCAL_MPROT_STACK
+    stack = heap_alloc(dcontext, sizeof(call_stack_32_t));
+#else
+    stack = global_heap_alloc(sizeof(call_stack_32_t) HEAPACCT(ACCT_OTHER));
+#endif
+    stack->next = dcontext->call_stack;
+    dcontext->call_stack = stack;
+
+    LOG(THREAD, LOG_ALL, 3, "check_return_too_deep\n");
+
+    /* move from registers into memory where we can work with it */
+    /* FIXME: align xmm so can use movdqa! */
+#ifdef LINUX
+    asm("movdqu %%xmm0, %0" : "=m"(xmm[0]));
+    asm("movdqu %%xmm1, %0" : "=m"(xmm[1]));
+    asm("movdqu %%xmm2, %0" : "=m"(xmm[2]));
+    asm("movdqu %%xmm3, %0" : "=m"(xmm[3]));
+    asm("movdqu %%xmm4, %0" : "=m"(xmm[4]));
+    asm("movdqu %%xmm5, %0" : "=m"(xmm[5]));
+    asm("movdqu %%xmm6, %0" : "=m"(xmm[6]));
+    asm("movdqu %%xmm7, %0" : "=m"(xmm[7]));
+#else
+#error NYI
+#endif
+
+    LOG(THREAD, LOG_ALL, 3, "\tjust copied registers\n");
+
+    /* we want 31..62 into our stack, that's the last 64 bytes before index */
+    memcpy(stack->retaddr, &xmm[3][14], 64);
+
+#ifdef DEBUG
+    if (stats->loglevel >= 3) {
+        int i,j;
+        LOG(THREAD, LOG_ALL, 3, "Copied into stored stack:\n");
+        for (i=0; i<4; i++) {
+            for (j=0; j<8; j++) {
+                LOG(THREAD, LOG_ALL, 3, "\t%d %d 0x%02x%02x", i, j,
+                    stack->retaddr[i*8+j][1],  stack->retaddr[i*8+j][0]);
+                if (j % 4 == 3)
+                    LOG(THREAD, LOG_ALL, 3, "\n");
+            }
+        }
+        LOG(THREAD, LOG_ALL, 3, "Before shifting:\n");
+        for (i=0; i<8; i++) {
+            for (j=0; j<8; j++) {
+                LOG(THREAD, LOG_ALL, 3, "\t%d %d 0x%02x%02x", i, j, xmm[i][j*2+1], xmm[i][j*2]);
+                if (j % 4 == 3)
+                    LOG(THREAD, LOG_ALL, 3, "\n");
+            }
+        }
+    }
+#endif
+
+#if !DISABLE_FOR_ANALYSIS
+    /* move back into registers */
+#ifdef LINUX
+    asm("movdqu %0, %%xmm0" : : "m"(xmm[0][0]));
+    asm("movdqu %0, %%xmm1" : : "m"(xmm[1][0]));
+    asm("movdqu %0, %%xmm2" : : "m"(xmm[2][0]));
+    asm("movdqu %0, %%xmm3" : : "m"(xmm[3][0]));
+    asm("movl $30, %eax");
+    asm("pinsrw $7,%eax,%xmm7");
+#else
+#error NYI
+#endif
+#endif
+
+    /* set to 32...but will have 64 added to it, so sub that now */
+    reg_ecx = 32 - 64;
+
+    dcontext->call_depth++;
+
+    LOG(THREAD, LOG_ALL, 3, "\tdone, call depth is now %d\n", dcontext->call_depth);
+
+#if USE_LOCAL_MPROT_STACK
+    SELF_PROTECT_LOCAL(dcontext, READONLY);
+#endif
+    EXITING_DR();
+}
+
+void
+check_return_too_shallow(dcontext_t *dcontext,
+                         volatile int errno, volatile reg_t eflags,
+                         volatile reg_t reg_edi, volatile reg_t reg_esi,
+                         volatile reg_t reg_ebp, volatile reg_t reg_esp,
+                         volatile reg_t reg_ebx, volatile reg_t reg_edx,
+                         volatile reg_t reg_ecx, volatile reg_t reg_eax)
+{
+    ENTERING_DR();
+#if USE_LOCAL_MPROT_STACK
+    SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+#endif
+
+    LOG(THREAD, LOG_ALL, 3, "check_return_too_shallow\n");
+    if (dcontext->call_depth == 0) {
+        LOG(THREAD, LOG_ALL, 3, "\tbottomed out of dynamo, ignoring\n");
+#ifdef LINUX
+        asm("movl $0, %eax");
+        asm("pinsrw $7,%eax,%xmm7");
+#else
+#error NYI
+#endif
+        /* we set ebx so that check will succeed */
+        reg_ebx = (reg_edx & 0x0000ffff);
+    } else {
+        /* restore 0..31 from memory */
+        call_stack_32_t *stack = dcontext->call_stack;
+        ASSERT(stack != NULL);
+
+        reg_ebx = (stack->retaddr[0][1] << 8) | (stack->retaddr[0][0]);
+        LOG(THREAD, LOG_ALL, 3, "\tsetting reg_ebx to stored retaddr "PFX"\n", reg_ebx);
+
+        /* move back into registers */
+#ifdef LINUX
+        /* gcc 4.0 doesn't like:  "m"(stack->retaddr) */
+        void *retaddr = stack->retaddr;
+        asm("movl %0, %%eax" : : "m"(retaddr));
+        /* off by one to get 1..31 into slots 0..30 */
+        asm("movdqu 0x02(%eax), %xmm0");
+        asm("movdqu 0x12(%eax), %xmm1");
+        asm("movdqu 0x22(%eax), %xmm2");
+        asm("movdqu 0x32(%eax), %xmm3");
+        asm("movl $31, %eax");
+        asm("pinsrw $7,%eax,%xmm7");
+#else
+#error NYI
+#endif
+#ifdef DEBUG
+        if (stats->loglevel >= 3) {
+            int i,j;
+            LOG(THREAD, LOG_ALL, 3, "Restored:\n");
+            for (i=0; i<4; i++) {
+                for (j=0; j<8; j++) {
+                    LOG(THREAD, LOG_ALL, 3, "\t%d %d 0x%02x%02x", i, j,
+                        stack->retaddr[i*8+j][1],  stack->retaddr[i*8+j][0]);
+                    if (j % 4 == 3)
+                        LOG(THREAD, LOG_ALL, 3, "\n");
+                }
+            }
+        }
+#endif
+        stack = stack->next;
+#if USE_LOCAL_MPROT_STACK
+        heap_free(dcontext, dcontext->call_stack, sizeof(call_stack_32_t));
+#else
+        global_heap_free(dcontext->call_stack, sizeof(call_stack_32_t) HEAPACCT(ACCT_OTHER));
+#endif
+        dcontext->call_stack = stack;
+        dcontext->call_depth--;
+        LOG(THREAD, LOG_ALL, 3, "\tdone, call depth is now %d\n", dcontext->call_depth);
+    }
+
+#if USE_LOCAL_MPROT_STACK
+    SELF_PROTECT_LOCAL(dcontext, READONLY);
+#endif
+    EXITING_DR();
+}
+
+void
+check_return_ra_mangled(dcontext_t *dcontext,
+                        volatile int errno, volatile reg_t eflags,
+                        volatile reg_t reg_edi, volatile reg_t reg_esi,
+                        volatile reg_t reg_ebp, volatile reg_t reg_esp,
+                        volatile reg_t reg_ebx, volatile reg_t reg_edx,
+                        volatile reg_t reg_ecx, volatile reg_t reg_eax)
+{
+    /* ebx had addr, then we did ebx = ~ebx */
+    int stored_addr = ~reg_ebx;
+
+    ENTERING_DR();
+    SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+
+#ifdef DEBUG
+    if (stats->loglevel >= 3) {
+        int idx, i, j;
+        byte xmm[8][16]; /* each sse2 is 128 bits = 16 bytes */
+        /* move from registers into memory where we can work with it */
+#ifdef LINUX
+        asm("movdqu %%xmm0, %0" : "=m"(xmm[0]));
+        asm("movdqu %%xmm1, %0" : "=m"(xmm[1]));
+        asm("movdqu %%xmm2, %0" : "=m"(xmm[2]));
+        asm("movdqu %%xmm3, %0" : "=m"(xmm[3]));
+        asm("movdqu %%xmm4, %0" : "=m"(xmm[4]));
+        asm("movdqu %%xmm5, %0" : "=m"(xmm[5]));
+        asm("movdqu %%xmm6, %0" : "=m"(xmm[6]));
+        asm("movdqu %%xmm7, %0" : "=m"(xmm[7]));
+#else
+#error NYI
+#endif
+        LOG(THREAD, LOG_ALL, 3, "on our stack:\n");
+        for (i=0; i<8; i++) {
+            for (j=0; j<8; j++) {
+                LOG(THREAD, LOG_ALL, 3, "\t%d %d 0x%02x%02x",
+                    i, j, xmm[i][j*2+1], xmm[i][j*2]);
+                if (j % 4 == 3)
+                    LOG(THREAD, LOG_ALL, 3, "\n");
+            }
+        }
+
+# ifdef LINUX
+        asm("pextrw $7,%xmm7,%eax");
+        asm("movl %%eax, %0" : "=m"(idx));
+# else
+# error NYI
+# endif
+        LOG(THREAD, LOG_ALL, 3, "check_return_ra_mangled: stored="PFX" vs real="PFX", idx=%d\n",
+            stored_addr, reg_edx, idx);
+    }
+#endif
+    SYSLOG_INTERNAL_ERROR("ERROR: return address was mangled (bottom 16 bits: 0x%04x => 0x%04x)",
+                          (reg_edx & 0x0000ffff), stored_addr);
+    ASSERT_NOT_REACHED();
+
+    SELF_PROTECT_LOCAL(dcontext, READONLY);
+    EXITING_DR();
+}
+
+#endif /* !SSE2_USE_STACK_POINTER */
+/*################################################################################*/
+
+
+#endif /* CHECK_RETURNS_SSE2 */
+
+#ifdef RETURN_AFTER_CALL
+/* Return instructions are allowed to target only instructions immediately following  */
+/* a call instruction that has already been executed. */
+
+static void
+add_call_site(dcontext_t *dcontext, app_pc target_pc, bool direct)
+{
+    /* TODO: should be part of vm_area_t to allow flushing  */
+    fragment_add_after_call(dcontext, target_pc);
+}
+
+/* return 0 if not found */
+static int
+find_call_site(dcontext_t *dcontext, app_pc target_pc)
+{
+    if (fragment_after_call_lookup(dcontext, target_pc) != NULL)
+        return 1;
+    else
+        return 0;                       /* not found */
+}
+
+/* check only the table */        
+bool
+is_observed_call_site(dcontext_t *dcontext, app_pc retaddr)
+{
+    return (find_call_site(dcontext, retaddr) != 0);
+}
+
+static int INLINE_ONCE
+start_enforcing(dcontext_t *dcontext, app_pc target_pc)
+{
+    static int start_enforcing = 0; /* FIXME: should be thread local. this will handle vfork */
+    int at_bottom;
+
+    LOG(THREAD, LOG_INTERP, 3, "RCT: start_enforcing = %d\n", start_enforcing);
+
+    if (start_enforcing)
+        return 1;
+    
+    at_bottom = at_initial_stack_bottom(dcontext, target_pc);
+    if (!at_bottom) {
+        LOG(THREAD, LOG_INTERP, 1, "RCT: no bottom - start enforcing now\n");
+        SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+        start_enforcing = 1;
+        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+        return 1;
+    }
+
+    /* FIXME: we reach the stack bottom on Windows quite late at 
+       fragment_t 2768, tag 0x77f9fb67 <ntdll.dll~KiUserApcDispatcher+0x7>
+       can we do better?
+       All other threads running at that time will ignore attacks. 
+       FIXME: therefore start_enforcing should be thread local 
+    */
+
+    if (at_bottom == 1) {
+        LOG(THREAD, LOG_INTERP, 1, "RCT: seen bottom - start enforcing after this \n");
+        SELF_UNPROTECT_DATASEC(DATASEC_RARELY_PROT);
+        start_enforcing = 1;
+        SELF_PROTECT_DATASEC(DATASEC_RARELY_PROT);
+        return 0;               /* let this last one through */
+    }
+
+    return 0; /* do not enforce yet */
+}
+
+void
+add_return_target(dcontext_t *dcontext, app_pc instr_pc, instr_t *instr)
+{
+    bool direct = instr_is_call_direct(instr);
+    app_pc after_call_pc = instr_pc + instr_length(dcontext, instr);
+    /* CHECK: is this always faster than decode_next_pc(dcontext, instr_pc) */
+    add_call_site(dcontext, after_call_pc, direct);
+    STATS_INC(ret_after_call_added);
+
+    DOLOG(1, LOG_INTERP, {
+        if (direct) {
+            LOG(THREAD, LOG_INTERP, 3,
+                "RCT: call at "PFX"\tafter_call="PFX"\ttarget="PFX"\n",
+                instr_pc, after_call_pc, opnd_get_pc(instr_get_target(instr)));
+        } else {
+            /* of course, while building a basic block we can't tell the indirect call target */
+            LOG(THREAD, LOG_INTERP, 3, "RCT: ind call at "PFX"\tafter_call="PFX"\n",
+                instr_pc, after_call_pc);
+        }
+    });
+}
+
+#ifdef DIRECT_CALL_CHECK
+#warning not yet implemented
+/* Further restrict return to existing code, to only target indirect after call sites,
+   since direct calls have known return targets.  Usually compilers generate only a single
+   RET instruction, but if we cannot count on that (i.e. assembly hacks), 
+   then this check will also have false positives 
+*/
+/* This reverse check of (call 1->1 return) can be implemented relatively efficiently:
+   we have to have _all_ return lookups actually check if the stored tag is of a 
+   direct call (which should be the common case so check can be made on miss path).
+   If target is indeed a direct call then they compare themselves with the stored value,
+   [unless first call in which case the valid value is yet unknown]
+
+   Note that we have a many-to-one relationship of (calls *->1 return) 
+   and also a 1-to-many for (ind call 1->* returns).  
+*/ 
+  
+unsigned first_ret_from[MAX_CALL_CNT];  /* the first registered return */
+
+enum {
+    RETURN_FROM_EXPECTED_CALLEE = 1, /* all good */
+    RETURN_FOR_FIRST_TIME = 2, /* probably good, 
+                                  as long as no one corrupted it before first use.
+                                  unfortunately, for attacks on uncommon paths 
+                                  this protection doesn't add much
+                               */
+    RETURN_UNKNOWN_CALLEE = -1
+};
+
+/* return >0 if ok */
+int 
+reverse_check_ret_source(app_pc target_pc, app_pc source_pc)
+{
+    uint call_site_ndx = find_call_site(dcontext, target_pc);
+    ASSERT_NOT_TESTED();
+    ASSERT(call_site_ndx < MAX_CALL_CNT);
+    if (first_ret_from[call_site_ndx] == source_pc) 
+        return RETURN_FROM_EXPECTED_CALLEE;    /* all good */
+    if (!first_ret_from[call_site_ndx]) { /* never returned to */
+        /* assigning first callee */
+        first_ret_from[call_site_ndx] = source_pc;
+        return RETURN_FOR_FIRST_TIME;
+    } else {
+        /* direct call returned to from a different address than last time */
+        return 0;               /*  mismatch - possible RA corruption */
+    }
+}
+#endif /* DIRECT_CALL_CHECK */
+
+static bool
+at_iret_exception(dcontext_t *dcontext, app_pc target_pc, app_pc source_pc)
+{
+#if defined(X64) && defined(WINDOWS)
+    /* Check if this ntdll64!RtlRestoreContext's iret.  While my instance
+     * of ntdll64 has RtlRestoreContext as straight-line code, it could
+     * easily be split up in the future, so we only check for an iret
+     * being in ntdll itself.
+     */
+    bool res = false;
+    instrlist_t *ilist = build_app_bb_ilist(dcontext, source_pc, INVALID_FILE);
+    instr_t *iret = instrlist_last(ilist);
+    instr_t *ipush = (iret != NULL) ? instr_get_prev(iret) : NULL;
+
+    if (get_module_base(source_pc) == get_ntdll_base()) {
+        /* We could check that this bb starts w/ fxrstor but rather than be
+         * too fragile I'm allowing any iret inside ntdll */
+        if (instr_get_opcode(instrlist_last(ilist)) == OP_iret) {
+            SYSLOG_INTERNAL_WARNING_ONCE("RCT: iret matched @"PFX, source_pc);
+            res = true;
+        }
+    }
+    instrlist_clear_and_destroy(dcontext, ilist);
+    /* case 9398: build_app_bb_ilist modified last decode page, so restore here */
+    set_thread_decode_page_start(dcontext, (app_pc) PAGE_START(target_pc));
+    return res;
+#else
+    return false;
+#endif
+}
+
+/* similar to vbjmp, though here we have a push of a register */
+static bool
+at_pushregret_exception(dcontext_t *dcontext, app_pc target_pc, app_pc source_pc)
+{
+    /* Check if this a "push reg; ret" seen in mscoree (case 7317):
+     *   push reg
+     *   ret
+     */
+    bool res = false;
+
+    instrlist_t *ilist = build_app_bb_ilist(dcontext, source_pc, INVALID_FILE);
+    instr_t *iret = instrlist_last(ilist);
+    instr_t *ipush = (iret != NULL) ? instr_get_prev(iret) : NULL;
+
+    if (ipush != NULL && instr_get_opcode(ipush) == OP_push
+        && opnd_is_reg(instr_get_src(ipush, 0))
+        && instr_is_return(iret) && instr_num_srcs(iret) == 2 /* no ret immed */) {
+        /* sanity check: is reg value the ret target? */
+        reg_id_t reg = opnd_get_reg(instr_get_src(ipush, 0));
+        reg_t val = reg_get_value(reg, get_mcontext(dcontext));
+        LOG(GLOBAL, LOG_INTERP, 3,
+            "RCT: at_pushregret_exception: push %d reg == "PFX"; ret\n",
+            reg, val);
+        if ((app_pc)val == target_pc) {
+            SYSLOG_INTERNAL_WARNING_ONCE("RCT: push reg/ret matched @"PFX, target_pc);
+            res = true;
+        }
+    }
+    instrlist_clear_and_destroy(dcontext, ilist);
+    /* case 9398: build_app_bb_ilist modified last decode page, so restore here */
+    set_thread_decode_page_start(dcontext, (app_pc) PAGE_START(target_pc));
+    return res;
+}
+
+static bool
+at_vbjmp_exception(dcontext_t *dcontext, app_pc target_pc, app_pc source_pc)
+{
+    /* Verify if this a VB generated push/ret, where the immediate in the push has
+       the target_pc as an immediate
+     * (this is seen in winword (case 670)):
+     *   push target-address
+     *   ret
+     */
+    bool res = false;
+
+    instrlist_t *ilist = build_app_bb_ilist(dcontext, source_pc, INVALID_FILE);
+    instr_t *iret = instrlist_last(ilist);
+    instr_t *ipush = iret ? instr_get_prev(iret) : NULL;
+
+    /* FIXME: need to restrict to only two instructions */
+    if (ipush && instr_get_opcode(ipush) == OP_push_imm && instr_is_return(iret) &&
+        opnd_get_size(instr_get_src(ipush, 0)) == OPSZ_4) {
+        ptr_uint_t immed = (ptr_uint_t) opnd_get_immed_int(instr_get_src(ipush, 0));
+        IF_X64(ASSERT_TRUNCATE(immed, uint,
+                               opnd_get_immed_int(instr_get_src(ipush, 0))));
+        LOG(GLOBAL, LOG_INTERP, 3,
+            "RCT: at_vbjmp_exception: testing target "PFX" for push $"PFX
+            "; ret pattern\n", target_pc, immed);
+        if ((app_pc)immed == target_pc) {
+            SYSLOG_INTERNAL_WARNING_ONCE("RCT: push/ret matched @"PFX, target_pc);
+            res = true;
+        }
+    }
+    instrlist_clear_and_destroy(dcontext, ilist);
+    /* case 9398: build_app_bb_ilist modified last decode page, so restore here */
+    set_thread_decode_page_start(dcontext, (app_pc) PAGE_START(target_pc));
+    return res;
+}
+
+static bool
+at_vbpop_exception(dcontext_t *dcontext, app_pc target_pc, app_pc source_pc)
+{
+    /* Verify if this a VB generated sequence where RETurn just goes to the next instruction.
+     * (this is seen in FMStocks_Bus.dll and FMStocks_Bus.dll (case 1718))
+     * The functions called seem generic enough, to allow for another pattern on this.
+     * All we're checking for now is (source_pc + 1) == target_pc.
+     110045E0                 call    ebx ; __vbaStrMove
+     110045E2                 push    offset loc_1100462A
+     110045E7                 jmp     short loc_11004620
+
+     11004620 loc_11004620:  ; CODE XREF: sub_11004510+D7j
+     11004620                 lea     ecx, [ebp+var_20]
+     11004623                 call    ds:__vbaFreeStr
+     11004629                 retn
+     1100462A loc_1100462A:  ; DATA XREF: sub_11004510+D2o
+     1100462A                 mov     ecx, [ebp-14h]
+     */
+    /* FIXME: make this part of at_vbjmp_exception() */
+    /* FIXME: also see security-common/vbjmp-rac-test.c and why 
+       we may end up having to treat specially a "push $code; jmp "
+       for a slightly more general handling of this.
+    */
+    
+    /* We assume that the RET instruction is a single one and is in
+       its own basic block, so we expect it to be at source_pc.  
+       FIXME: If it doesn't works this way, we'll have
+       to build a basic block like at_vbjmp_exception() does. */
+    /* FIXME: What if the source_pc is a trace, then we'd need to find
+       the exiting branch and make sure it matches? */
+    if ( (source_pc + 1) == target_pc) {
+        LOG(THREAD, LOG_INTERP, 2, "RCT: at_vbpop_exception; matched ret "PFX" to next "PFX" pattern\n", 
+            source_pc, target_pc);
+        SYSLOG_INTERNAL_WARNING_ONCE("RCT: ret/next matched @"PFX,
+                                     source_pc, target_pc);
+        return true;
+    }
+    return false;
+}
+
+static bool
+at_mso_rct_exception(dcontext_t *dcontext, app_pc target_pc)
+{
+    /* winlogon.exe (case 1214) and mso.dll (case 1158) in Office 10
+       (from Winstone 2002) appear to have a very weird code that for
+       many function calls modifies the return address on the stack so
+       that it skips several bytes to reach the real instruction. 
+
+       The purpose of that code is not yet grokked.  I have not
+       identified if that is supposed to be an exception handling
+       mechanism or another language construct, in any case, it breaks the ABI.
+
+       There are ~500 places in winlogon.exe that have this pattern:
+       jmp *(fptr)
+       pushfd
+       pushad
+       push args dwords
+       call <some func>    ; there are only 10 responsible for the 500 call sites in winlogon
+    ac:                    ; after call instruction yet return targets realac 
+       sub esp, 0x400+ function of args
+       popad
+       popfd
+       <a few instrs usually pop's or push'es, or add esp>
+    realac: ; actual return target
+       add esp, 4*args
+       lea ebx, &after
+       mov [fptr], ebx
+       popad
+       popfd
+    after:
+
+       37 places in mso.dll dll also match this pattern: 
+
+       30bf58f9 e87f80f1ff       call    MSO+0xd97d (30b0d97d)  ; two locations here
+       -> this is the pushed real after call address
+I am not sure if this is supposed to be data or instructions for something
+       30bf58fe 81ec14040000     sub     esp,?0x414? ; immediate varies
+      [30bf5904 83c408           add     esp,0x8 ]; this "instruction" not always here...
+       30bf5907 61               popad
+       30bf5908 9d               popfd
+       30bf5909 5d               pop     ebp
+       -> this is the address where the above call returns to
+       30bf590a 83c428           add     esp,?0x28?  ; varies
+
+       The way the callees are passed the offset to the realac is
+       unclear - after several unexplainable hoops of adding known
+       constants to the return address on the stack they end up there.
+
+       What we do when a RAC fails:
+
+        I prefer not building a basic block for suspect attacker
+        controlled data - keep in mind we do this check before we check
+        code origins, therefore we'll match raw bytes.
+
+        is_readable_without_exception(target_pc, 17)
+
+        For now I'll go with this pattern match on the target_pc:
+        ; this cleans up the arguments to the call so it shouldn't be huge (I've seen 52)
+        83 c4   at the target_pc,   add esp, 0xbyte 
+        ; the address they are loading in here is at the end of the code block!
+        8d 1d [target_pc+1]  at target_pc+3      lea ebx, target_pc+17
+        ; In case we start doing something about indirect jumps - we can keep
+        ; the information about the indirect jump targeting target_pc+17
+        ; We don't match the next line's pattern
+        89 1d   at target_pc+9      mov [?addr32], ebx 
+        61      at target_pc+15     popad
+        9d      at target_pc+16     popfd
+        target_pc+17:
+
+        We should then check that there is a valid after call site in
+        the 32 bytes preceding the target_pc (I've seen 16 but giving some margin).
+        This will make it a little stricter in order to keep it
+        independent of code origins - so that attackers can't point us
+        to random code with the above prefix.
+
+    */
+    enum {MSO_PATTERN_SIZE               = 17,
+          MSO_PATTERN_ADD_ESP            = 0xc483,
+          MSO_PATTERN_LEA_EBX_OFFSET     = 3,
+          MSO_PATTERN_LEA_EBX            = 0x1d8d,
+          MSO_PATTERN_LEA_EBX_DISP_OFFSET= 2 + MSO_PATTERN_LEA_EBX_OFFSET,
+          MSO_PATTERN_POPAD_POPFD_OFFSET = 15,
+          MSO_PATTERN_POPAD_POPFD        = 0x9d61,
+          MSO_PATTERN_MAX_AC_OFFSET      = 32};
+
+    if (!is_readable_without_exception(target_pc, MSO_PATTERN_SIZE)) 
+        return false;
+
+    LOG(GLOBAL, LOG_INTERP, 3, "RCT: at_mso_rct_exception("PFX")\n", target_pc);
+
+#ifdef X64
+    /* let's wait until we hit this so we know what the new pattern
+     * looks like */
+    return false;
+#endif
+
+    if ((*(uint*)(target_pc + MSO_PATTERN_LEA_EBX_DISP_OFFSET) 
+         == (uint)(ptr_uint_t)(target_pc + MSO_PATTERN_SIZE)) &&
+        (*(ushort*)(target_pc + MSO_PATTERN_LEA_EBX_OFFSET) 
+         == MSO_PATTERN_LEA_EBX) &&
+        (*(ushort*)(target_pc + MSO_PATTERN_POPAD_POPFD_OFFSET) 
+         == MSO_PATTERN_POPAD_POPFD) &&
+        *(ushort*)(target_pc) == MSO_PATTERN_ADD_ESP) {
+        uint fromac;
+    
+        LOG(GLOBAL, LOG_INTERP, 2, "RCT: at_mso_rct_exception("PFX"): pattern matched, "
+            "testing if after call\n", target_pc);
+
+        for (fromac = 0; fromac < MSO_PATTERN_MAX_AC_OFFSET; fromac++) {
+            if (find_call_site(dcontext, target_pc - fromac)) {
+                SYSLOG_INTERNAL_WARNING_ONCE("RCT: mso rct matched @"PFX, target_pc);
+   
+                LOG(GLOBAL, LOG_INTERP, 2, "RCT: at_mso_rct_exception("PFX"): "
+                    "pattern matched %d after real after call site", target_pc, fromac);
+
+                /* CHECK: in case we see many of these exceptions at the same target
+                   then we should add this target_pc as a valid after_call_site 
+                   so we don't have to match it in the future
+                */
+                return true;
+            }
+        }
+    }
+        
+    return false;
+}
+
+
+/* licdll.dll (case 1690) Licensing agent 
+   that is used by Automatic updates has several RCT violations.
+
+   I have no idea why are they breaking the API again - only
+   possible explanation is for some sort of obfuscation.  I
+   wouldn't be surprised if debugging this changes its behaviour.
+
+
+   licdll.dll from XP SP1
+   55a6877d 8d1de9813b60     lea     ebx,[603b81e9]         ; EBX=603b81e9
+   55a68783 83ec1c           sub     esp,0x1c
+   55a68786 891c24           mov     [esp],ebx              ; [ESP] = 0x603b81e9
+   ...
+   55a687a5 812c2428fa940a   sub     dword ptr [esp],0xa94fa28  
+   ; [ESP] = 0x603b81e9 - 0xa94fa28 = 0x55a687c1!
+   ...
+   55a687bc e90a230000       jmp     licdll!Ordinal221+0xaacb (55a6aacb)
+
+   [1] BAD TARGET       
+   55a687c1 8b542424         mov     edx,[esp+0x24]
+   55a687c5 8b0c24           mov     ecx,[esp]
+   55a687c8 891424           mov     [esp],edx
+   55a687cb 894c2424         mov     [esp+0x24],ecx
+   55a687cf 9d               popfd
+   55a687d0 61               popad
+   55a687d1 c3               ret
+   [2] BAD SOURCE - the ret of this same fragment is then targeting a piece of DGC 
+   on two freshly created pages.
+
+   Exactly the same code appears at 0x55a65446 in the same dll on xp.  
+   in the win2003 version of it 0x62FB5478 and 0x62FB87AE have the same fragment.
+
+   FIXME: Of the three different SUB [esp] offsets 0A94FA28h, 0C98F744h, and 0EEF3E64h,
+   the latter two exhibit different potential target patterns.  Need to test those.
+       
+   What we do when a RAC fails, see comments in
+   at_mso_rct_exception() on why we match raw bytes:
+
+   Note that this same fragment is both a source and a target, so
+   it may be worthwhile matching it as close as possible.
+
+   1) FIXME: check if source fragment is in module licdll.dll
+
+   2) In case the target is a future executable then we don't look at
+   the target but instead look at the source in the next step.
+
+   3) is_readable_without_exception(target_pc, 17)
+
+   4) Pattern match:
+   8b 54 24 24 pc        mov edx,[esp+0x24]
+   24          pc+13     mov [esp+0x24], ecx ; 89 4c 24 24]
+   9d          pc+14     popfd
+   61          pc+15     popad
+   c3          pc+16     ret
+   pc+17:
+*/
+
+static bool
+licdll_pattern_match(dcontext_t *dcontext, app_pc pattern_pc)
+{
+    enum {LICDLL_PATTERN_SIZE               = 17,
+          LICDLL_PATTERN_MOV_EDX_ESP_24     = 0x2424548b,
+          LICDLL_PATTERN_24_POPFD_OFFSET    = 13,
+          LICDLL_PATTERN_24_POPFD_POPAD_RET = 0xc3619d24
+    };
+
+    if (!is_readable_without_exception(pattern_pc, LICDLL_PATTERN_SIZE)) 
+        return false;
+
+    LOG(THREAD, LOG_INTERP, 2, "RCT: at_licdll_rct_exception("PFX")\n", pattern_pc);
+
+#ifdef X64
+    /* let's wait until we hit this so we know what the new pattern
+     * looks like */
+    return false;
+#endif
+
+    if ((*(uint*)(pattern_pc + LICDLL_PATTERN_24_POPFD_OFFSET) 
+         == LICDLL_PATTERN_24_POPFD_POPAD_RET) &&
+        (*(uint*)(pattern_pc)
+         == LICDLL_PATTERN_MOV_EDX_ESP_24)) {
+    
+        LOG(THREAD, LOG_INTERP, 1,
+            "RCT: at_licdll_rct_exception("PFX"): pattern matched\n", pattern_pc);
+
+        return true;
+    }
+    return false;
+}
+
+static bool
+at_licdll_rct_exception(dcontext_t *dcontext, app_pc target_pc, app_pc source_pc)
+{
+
+    /* 1) FIXME: check if source fragment is in module licdll.dll
+       we could do that with get_module_short_name(source_pc), 
+       but it looks like both licdll and dpcdll need this */
+
+    /* 2) FIXME: In case the target is a future executable then we
+       don't look at the target but instead look at the source in the next step. */
+
+    /* CHECK: in case we see many of these exceptions at the same target
+       then we should add this target_pc as a valid after_call_site 
+       so we don't have to match it in the future
+    */
+
+    if (licdll_pattern_match(dcontext, target_pc)) { 
+        SYSLOG_INTERNAL_WARNING_ONCE("RCT: licdll rct matched target @"PFX,
+                                     target_pc);
+        return true;
+    }
+    /* case 9398: set to source for our check
+     * FIXME: we could read off the end of the page!  Should use TRY or safe_read.
+     */
+    ASSERT(check_in_last_thread_vm_area(dcontext, (app_pc) PAGE_START(target_pc)));
+    set_thread_decode_page_start(dcontext, (app_pc) PAGE_START(source_pc));
+    /* the same piece of code then is then RETurning into some DGC */
+    if (licdll_pattern_match(dcontext, source_pc)) { 
+        SYSLOG_INTERNAL_WARNING_ONCE("RCT: licdll rct matched source @"PFX,
+                                     source_pc);
+        /* We assume any match will abort future app derefs so we don't need
+         * to restore the last decode page */
+        return true;
+    }
+    /* case 9398: now restore (if return true we assume no more derefs) */
+    set_thread_decode_page_start(dcontext, (app_pc) PAGE_START(target_pc));
+        
+    return false;
+}
+
+/* return after call check 
+   called by dispatch after inlined return lookup routine has failed */
+/* FIXME: return value is ignored */
+int
+ret_after_call_check(dcontext_t *dcontext, app_pc target_addr, app_pc src_addr)
+{
+/* FIXME If we change shared_syscalls to use the ret table (instead of the jmp
+ * table), we need to fix up the use of instr_addr further down, since it could
+ * store a nonsensical value and cause reverse_check_ret_source() to return a
+ * failure code.
+ */
+#if defined(DEBUG) || defined(DIRECT_CALL_CHECK)
+    cache_pc instr_addr = EXIT_CTI_PC(dcontext->last_fragment, dcontext->last_exit);
+#endif
+
+    LOG(THREAD, LOG_INTERP, 3, "RCT: return \taddr = "PFX"\ttarget = "PFX"\n",
+        instr_addr, target_addr);
+
+    STATS_INC(ret_after_call_validations);
+
+    /* FIXME: currently this is only a partial check,
+       a trace lookup will not exit the fcache for a check like this 
+       to fully provide the return-after-call guarantee.
+
+       [Note that there is an ibl even in basic blocks and currently
+       those simply look for any trace, the next step is to restrict
+       the return hashtable only to valid "after call" targets]
+
+       Yet false positives with this simpler check
+       would be something to get worried about already.
+    */
+
+    /* TODO: write a unit test that forms a trace and then modifies 
+       the return address to show this needs to be done from within */
+
+    /* Case 9398: handle unreadable races from derefs in checks below.
+     * Any checks that read src must set back to target.
+     * FIXME: better to use TRY, or safe_read for each?  if use TRY
+     * then have to make sure to call bb_build_abort() if necessary,
+     * since TRY fault takes precedence over decode fault.
+     * FIXME: we could read off the end of the page!  This is just a quick fix,
+     * not foolproof.
+     */
+    set_thread_decode_page_start(dcontext, (app_pc) PAGE_START(target_addr));
+
+    if (!find_call_site(dcontext, target_addr)) {
+        LOG(THREAD, LOG_INTERP, 1, "RCT: bad return target: "PFX"\n",
+            target_addr);
+        if (!start_enforcing(dcontext, target_addr)) {
+            /* FIXME to be fixed whenever we figure out how to start first */
+            LOG(THREAD, LOG_INTERP, 1, "RCT: haven't started yet --ok\n");
+            STATS_INC(ret_after_call_before_start);
+            /* do not add exemption */
+            return 2;
+        }
+
+        /* Now come the known cases of unjustified ugliness from Microsoft apps.
+           For regression testing purposes we test for them on all platforms */
+
+        /* FIXME: see case 285 for a better method of obtaining source_pc,
+           which for all uses here is assumed to be a bb tag,
+           and will likely break if a trace containing these bb's is build.
+           Also see case 1858 about storing into RAC table validated targets.
+        */
+        if (DYNAMO_OPTION(vbpop_rct) && 
+            at_vbpop_exception(dcontext, target_addr, src_addr)) {
+            LOG(THREAD, LOG_INTERP, 1, "RCT: known exception on VB pop --ok\n");
+            STATS_INC(ret_after_call_known_exceptions);
+            goto exempted;
+        }
+
+        if (DYNAMO_OPTION(vbjmp_allowed) &&
+            at_vbjmp_exception(dcontext, target_addr, src_addr)) {
+            LOG(THREAD, LOG_INTERP, 1, "RCT: known exception on VB jmp --ok\n");
+            STATS_INC(ret_after_call_known_exceptions);
+            goto exempted;
+        }
+
+        if (DYNAMO_OPTION(mso_rct) && 
+            at_mso_rct_exception(dcontext, target_addr)) {
+            LOG(THREAD, LOG_INTERP, 1, "RCT: known exception on mso ret --ok\n");
+            STATS_INC(ret_after_call_known_exceptions);
+            goto exempted;
+        }
+
+        if (DYNAMO_OPTION(licdll_rct) && 
+            at_licdll_rct_exception(dcontext, target_addr,src_addr)) {
+            LOG(THREAD, LOG_INTERP, 1, "RCT: known exception on licdll ret --ok\n");
+            STATS_INC(ret_after_call_known_exceptions);
+            goto exempted;
+        }
+
+        if (DYNAMO_OPTION(pushregret_rct) &&
+            at_pushregret_exception(dcontext, target_addr, src_addr)) {
+            LOG(THREAD, LOG_INTERP, 1, "RCT: known exception on push reg; ret --ok\n");
+            STATS_INC(ret_after_call_known_exceptions);
+            STATS_INC(ret_after_call_pushregret);
+            /* FIXME: we don't want to cache the target of this pattern
+             * as the usage we've seen is once-only.  But, it has also been
+             * to DGC-only, which is currently not cached anyway.
+             */
+            goto exempted;
+        }
+
+        if (DYNAMO_OPTION(iret_rct) &&
+            at_iret_exception(dcontext, target_addr, src_addr)) {
+            LOG(THREAD, LOG_INTERP, 1, "RCT: known exception on iret --ok\n");
+            goto exempted;
+        }
+
+        /* additional handling for known OS specific exceptions is in 
+           linux/signal.c (for ld) and 
+           win32/callback.c (for exempt modules, Win2003 fibers, and SEH)
+        */
+        if (at_known_exception(dcontext, target_addr, src_addr)) {
+            LOG(THREAD, LOG_INTERP, 1, "RCT: known exception --ok\n");
+            STATS_INC(ret_after_call_known_exceptions);
+            goto exempted;
+        }
+
+        LOG(THREAD, LOG_INTERP, 1,
+            "RCT: BAD[%d] real problem target="PFX" src fragment="PFX"\n",
+            GLOBAL_STAT(ret_after_call_violations), target_addr, src_addr);
+        STATS_INC(ret_after_call_violations);
+
+        if (DYNAMO_OPTION(unloaded_target_exception) && 
+            is_unreadable_or_currently_unloaded_region(target_addr)) {
+            /* we know we either had unload in progress, or we're
+             * beyond unload, but unlike other violations we want to
+             * know the difference between unreadable due to unload,
+             * vs other unreadable ranges
+             */
+            /* if it is currently tracked as unloaded we'll just ignore */
+            /* case 9364 - we may prefer to kill a thread when
+             * unreadable memory that hasn't been unloaded
+             * Alternatively, if throwing an exception is always OK,
+             * we could exempt in all cases when we reach this.
+             */
+            /* We assume that we'll throw an unreadable exception for
+             * both unloaded and unreadable memory later.  (Note that
+             * we flush the fragments after we flush the RAC during
+             * process_mmap(), so there is a small chance that we'll
+             * in fact completely allow execution - which is OK since
+             * still a possible APP race.)  FIXME: it may be
+             * preferable to throw our own exception here, if DLLs
+             * are in inconsistent state a lot longer while unloaded
+             * under us compared to native, then any execution during
+             * unload would be bad.
+             */
+            /* if we are unreadable, we could be _after_ unload */
+            if (is_in_last_unloaded_region(target_addr)) {
+                DODEBUG({
+                    if (!is_readable_without_exception(target_addr, 4)) {
+                        /* if currently unreadable and in last unloaded module
+                         * we'd let this through and assume that we'll throw
+                         * an exception to the app
+                         */
+                        LOG(THREAD, LOG_RCT, 1, 
+                            "RCT: DLL unload in progress, "PFX" --ok\n", target_addr);
+                        STATS_INC(num_unloaded_race_during);
+                    } else {
+                        LOG(THREAD, LOG_RCT, 1, 
+                            "RCT: target in already unloaded DLL, "PFX" --ok\n", 
+                            target_addr);
+                        STATS_INC(num_unloaded_race_after);
+                    }
+                });
+                /* case 6008 should apply this exemption to unreadable all
+                 * unloaded DLLs not only the last one memory execution 
+                 */
+
+                /* do not add exemption */ 
+                return 3; /* allow, don't throw .C */
+            } else {
+                /* we probably were just unreadable, bad app or possibly attack,
+                 * leaving to rct_ret_unreadable further down
+                 */
+                /* FIXME: case 6008 there is also a possibility of a
+                 * race (that we were during unload at the time we
+                 * checked, but since we only keep the last unmap,
+                 * another one could have taken place, so we would get
+                 * here even if we wanted to exempt.
+                 */
+                /* fall through */
+                ASSERT_NOT_TESTED();
+            }
+        }
+        
+        /* ASLR: check if is in wouldbe region, if so report as failure */
+        if (aslr_is_possible_attack(target_addr)) {
+            LOG(THREAD, LOG_RCT, 1, 
+                "RCT: ASLR: wouldbe a preferred DLL, "PFX" --BAD\n", target_addr);
+            /* fall through and report */
+            ASSERT_NOT_TESTED();
+            /* FIXME: case 7017 ASLR_NORMALIZE_ID handling */
+            STATS_INC(aslr_rct_ret_wouldbe);
+        }
+
+        /* special handling of unreadable memory targets - most likely
+         * corrupted app, but could also be an unsuccesful attack
+         */
+        if (TEST(OPTION_ENABLED, DYNAMO_OPTION(rct_ret_unreadable))) {
+            if (!is_readable_without_exception(target_addr, 4)) {
+                SYSLOG_INTERNAL_WARNING_ONCE("return target "PFX" unreadable",
+                                             target_addr);
+
+                /* We will eventually throw an exception unless
+                 * security violation handles this differently. 
+                 * e.g. if OPTION_NO_REPORT|OPTION_BLOCK we may kill a thread
+                 */
+                /* the current defaults will let all of this through */
+                /* FIXME: for now only OPTION_NO_REPORT is supported
+                 * by security_violation() and that's all we currently need */
+                if (security_violation(dcontext, target_addr, RETURN_TARGET_VIOLATION, 
+                                       DYNAMO_OPTION(rct_ret_unreadable)) ==
+                    RETURN_TARGET_VIOLATION) {
+                    /* do not cache unreadable memory target */
+                    return -1;
+                } else {
+                    /* do not cache unreadable memory target */
+                    return 1;
+                }
+            }
+        }
+
+        SYSLOG_INTERNAL_WARNING_ONCE("return target "PFX" with no known caller",
+                                     target_addr);
+        /* does not return in protect mode */
+        if (security_violation(dcontext, target_addr, RETURN_TARGET_VIOLATION, OPTION_BLOCK|OPTION_REPORT) ==
+            RETURN_TARGET_VIOLATION) {
+            /* running in detect mode */
+            ASSERT(DYNAMO_OPTION(detect_mode)
+                   /* case 9712: client security callback can modify the action.
+                    * FIXME: if a client changes the action to ACTION_CONTINUE,
+                    * this address will be exempted and we won't complain again. 
+                    * In the future we may need to add another action type. */
+                   IF_CLIENT_INTERFACE(||!IS_INTERNAL_STRING_OPTION_EMPTY(client_lib)));
+            /* we'll cache violation target */            
+            goto exempted;
+        } else { /* decided not to throw the violation */ 
+            /* exempted Threat ID */
+            /* we'll cache violation target */
+            goto exempted;
+        }
+    exempted:
+        /* add target if in a module (code or data section), but not in DGC */
+        if (DYNAMO_OPTION(rct_cache_exempt) == RCT_CACHE_EXEMPT_ALL ||
+            (DYNAMO_OPTION(rct_cache_exempt) == RCT_CACHE_EXEMPT_MODULES && 
+             (get_module_base(target_addr) != NULL))) {
+            /* FIXME: extra system calls may be become more expensive
+             * than extra exits for simple pattern matches, should
+             * have a cheap way of determining whether an address is
+             * in a module code section  */
+
+            fragment_add_after_call(dcontext, target_addr);
+            ASSERT_CURIOSITY(is_executable_address(target_addr));
+            STATS_INC(ret_after_call_exempt_added);
+        } else {
+        }
+        return 1;
+    }
+#ifdef DIRECT_CALL_CHECK
+    else {
+        /* extra check on direct calls */
+        /* TODO: verify if target is direct call */
+        /* FIXME: make sure that instr_addr gets shifted properly on unit resize 
+           i.e. considered as a normal fragment address, then this check is ok to use a cache_pc */
+        if (reverse_check_ret_source(target_addr, instr_addr) < 0) {
+            LOG(1, "RCT: bad return source:"PFX" for after call target: "PFX"\n",
+                instr_addr, target_addr);
+            return -1;
+        }
+    }
+#endif /* DIRECT_CALL_CHECK */
+    LOG(THREAD, LOG_INTERP, 3, "RCT: good return to "PFX"\n", target_addr);
+    STATS_INC(ret_after_call_good);
+
+    return 1;
+}
+
+#endif /* RETURN_AFTER_CALL */
diff --git a/core/x86/sideline.c b/core/x86/sideline.c
new file mode 100644
index 000000000..5f5ea7ec6
--- /dev/null
+++ b/core/x86/sideline.c
@@ -0,0 +1,1009 @@
+/* **********************************************************
+ * Copyright (c) 2002-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+/*
+ * sideline.c - simultaneous optimization support
+ */
+
+#include "globals.h"
+
+#ifdef SIDELINE /* around whole file */
+
+/****************************************************************************/
+/* include files and typedefs */
+
+#ifdef WINDOWS
+#    include <windows.h>
+#    include <ntdll.h>
+#    define RUN_SIG WINAPI
+#else /* LINUX */
+#    include <sys/types.h> /* for wait */
+#    include <sys/wait.h>  /* for wait */
+#    include <sched.h>     /* for clone */
+#    include <signal.h>    /* for SIGCHLD */
+#    include <string.h>    /* for memset */
+#    include <unistd.h>    /* for nice */
+#    define RUN_SIG
+     typedef pid_t thread_t;
+#    define THREAD_STACK_SIZE   (32*1024) /* FIXME: why different than DYNAMORIO_STACK_SIZE? */
+#endif
+
+#include "sideline.h"
+#include "instrlist.h"
+#include "arch.h"
+#include "instr.h"
+#include "decode.h"
+#include "fragment.h"
+#include "emit.h"
+#include "link.h"
+#include "fcache.h"
+
+#ifdef SIDELINE_COUNT_STUDY
+# ifndef PROFILE_LINKCOUNT
+# error SIDELINE_COUNT_STUDY requires PROFILE_LINKCOUNT
+# endif
+#endif
+
+#define OPTVERB_3 4
+//#define VERB_3 3
+#define VERB_3 4
+#define VERB_2 2
+
+/* minimum number of samples before a trace is optimized */
+#define SAMPLE_COUNT_THRESHOLD 3
+/* frequency, in number of samples, at which we optimize hottest trace */
+#define SAMPLE_TO_OPTIMIZE_RATIO 5000
+
+/****************************************************************************/
+/* global vars */
+
+/* sampled by sideline thread to find hot traces */
+volatile fragment_t * sideline_trace;
+
+/* number of processors we're running on */
+int num_processors;
+
+/* used to signal which thread we need to pause in dispatch */ 
+thread_id_t pause_for_sideline;
+event_t paused_for_sideline_event;
+event_t resume_from_sideline_event;
+
+DECLARE_CXTSWPROT_VAR(mutex_t sideline_lock, INIT_LOCK_FREE(sideline_lock));
+
+/* for sideline logging */
+file_t logfile;
+
+/****************************************************************************/
+/* sampling data structures */
+
+/*
+  data structure for samples: either counter field in fragment_t ->
+  waste of space (have separate trace_t?), or have own hashtable here --
+  not worth it to have data structure that maintains sorted order,
+  just have fast lookup to incr counter, every so often can do linear
+  search for top => have own hashtable, then don't have to synch w/
+  thread's ftable.  Instead of linear search, can simply optimize frag
+  when incrementing its count and hitting some threshold.
+*/
+
+typedef struct _sample_entry_t {
+    ptr_uint_t    tag;
+    int     counter;
+    struct  _sample_entry_t * next;    /* for chaining entries */
+} sample_entry_t;
+
+typedef struct _sample_table_t {
+    uint  hash_bits;
+    uint  hash_mask;
+    hash_function_t  hash_func;
+    uint           hash_mask_offset; /* ignores given number of LSB bits */
+    uint  capacity;
+    uint  entries;
+    mutex_t         lock;   /* lock to sequence update of this table */
+    sample_entry_t **table;  /* hash_bits-bit addressed hash table */
+} sample_table_t;
+
+#define INITIAL_HASH_BITS 12
+
+static sample_table_t table;
+DECLARE_CXTSWPROT_VAR(mutex_t do_not_delete_lock, INIT_LOCK_FREE(do_not_delete_lock));
+static fragment_t * fragment_now_optimizing;
+
+/* List to remember fragments we've replaced but can't delete until
+ * owning thread reaches a safe point.
+ * List of lists of fragment_t*'s, one sublist per dcontext.
+ */
+
+typedef struct _remember_entry_t {
+    fragment_t  *f;
+#ifdef SIDELINE_COUNT_STUDY
+    fragment_t  *replacement;
+#endif
+    struct _remember_entry_t * next;
+} remember_entry_t;
+
+typedef struct _remember_list_t {
+    dcontext_t *dcontext;
+    remember_entry_t *list;
+    struct _remember_list_t * next;
+} remember_list_t;
+
+static remember_list_t *remember;
+DECLARE_CXTSWPROT_VAR(static mutex_t remember_lock, INIT_LOCK_FREE(remember_lock));
+
+static int num_samples;
+
+#ifdef DEBUG
+static int num_optimized;
+static int num_opt_with_no_synch;
+#endif
+
+/****************************************************************************/
+/* thread synchronization */
+
+static thread_id_t child_tid;
+#ifdef WINDOWS
+static HANDLE child_handle;
+#else /* LINUX */
+static thread_t child;
+static void *stack;
+#endif
+
+static event_t wake_event;
+static event_t asleep_event;
+static event_t exited_event;
+
+/* both of these are read by child and written by parent */
+static volatile bool child_sleep;
+static volatile bool child_exit;
+
+/****************************************************************************/
+/* forward declarations */
+
+static int RUN_SIG sideline_run(void *arg);
+static void sideline_sample(void);
+static void sideline_examine_traces(void);
+static sample_entry_t *update_sample_entry(uint tag);
+static sample_entry_t *find_hottest_entry(void);
+static void remove_sample_entry(uint tag);
+
+static void add_remember_entry(dcontext_t *dcontext, fragment_t *f
+#ifdef SIDELINE_COUNT_STUDY
+                               , fragment_t *new_f
+#endif
+                               );
+#ifdef LINUX
+static thread_t create_thread(int (*fcn)(void *), void *arg, void **stack);
+static void delete_thread(thread_t thread, void *stack);
+#endif
+
+/****************************************************************************/
+
+/* initialization */
+void 
+sideline_init()
+{
+#ifdef WINDOWS
+    int res;
+#endif
+    pause_for_sideline = (thread_id_t) 0;
+    paused_for_sideline_event = create_event();
+    resume_from_sideline_event = create_event();
+
+    num_processors = get_num_processors();
+    LOG(GLOBAL, LOG_TOP|LOG_SIDELINE, 1,
+        "Number of processors: %d\n", num_processors);
+
+    wake_event = create_event();
+    asleep_event = create_event();
+    exited_event = create_event();
+    child_exit = false;
+    child_sleep = true;
+
+    table.hash_bits = INITIAL_HASH_BITS;
+    table.hash_mask = HASH_MASK(table.hash_bits);
+    table.hash_func = HASH_FUNCTION_NONE;
+    table.capacity = HASHTABLE_SIZE(table.hash_bits);
+    table.entries = 0;
+    ASSIGN_INIT_LOCK_FREE(table.lock, sideline_table_lock);
+    table.table = (sample_entry_t**)
+        global_heap_alloc(table.capacity*sizeof(sample_entry_t*) HEAPACCT(ACCT_SIDELINE));
+    memset(table.table, 0, table.capacity*sizeof(sample_entry_t*));
+    fragment_now_optimizing = NULL;
+
+    remember = NULL;
+
+    num_samples = 0;
+#ifdef DEBUG
+    num_optimized = 0;
+    num_opt_with_no_synch = 0;
+#endif
+
+#ifdef WINDOWS
+    /* start thread suspended so can add_thread before it runs */
+    child_handle = create_thread(NT_CURRENT_PROCESS, IF_X64_ELSE(true, false),
+                                 (void *)sideline_run, NULL, NULL, 0,
+                                 15*PAGE_SIZE, 12*PAGE_SIZE, true, &child_tid);
+
+# if 0 /* need non-kernel32 implementation, use NtSetInformationThread */
+    /* give sideline thread a lower priority */
+    SetThreadPriority(child_handle, THREAD_PRIORITY_BELOW_NORMAL);
+    /* if want to glue thread to a single processor, use
+     * SetThreadAffinity function -- I don't think we want to do that though
+     */
+# else
+    ASSERT_NOT_IMPLEMENTED(false);
+# endif
+
+#else /* LINUX */
+    child = create_thread(sideline_run, NULL, &stack);
+    child_tid = child;
+
+    /* priority of child can only be set by child itself, in sideline_run() */
+#endif /* LINUX */
+
+    /* tell dynamo core about new thread so it won't be treated as app thread */
+    add_thread(IF_WINDOWS_(child_handle) child_tid, false, NULL);
+
+    LOG(GLOBAL, LOG_SIDELINE, 1, "Sideline thread (id %d) created\n", child_tid);
+
+#ifdef WINDOWS
+    /* now let child thread run */
+    res = nt_thread_resume(child_handle, NULL);
+    ASSERT(res);
+#endif
+
+    sideline_start();
+}
+
+
+/* atexit cleanup */
+void 
+sideline_exit()
+{
+    uint i;
+    sample_entry_t *sample;
+    remember_list_t *l, *nextl;
+    remember_entry_t *e, *nexte;
+
+    if (child_exit) {
+        /* we were invoked in recursive loop through assert in delete_thread */
+        return;
+    }
+
+    LOG(logfile, LOG_SIDELINE, VERB_3, "sideline_exit\n");
+
+    child_exit = true;
+
+    if (pause_for_sideline != 0) {
+        /* sideline thread could be waiting for another thread */
+        signal_event(paused_for_sideline_event);
+    }
+
+    if (child_sleep) {
+        child_sleep = false;
+        signal_event(wake_event);
+    }
+    wait_for_event(exited_event);
+
+#ifdef WINDOWS
+    /* wait for child to die */
+    nt_wait_event_with_timeout(child_handle, INFINITE_WAIT);
+#else /* LINUX */
+    delete_thread(child, stack);
+#endif
+    LOG(logfile, LOG_SIDELINE, 1, "Sideline thread destroyed\n");
+
+    LOG(logfile, LOG_SIDELINE|LOG_STATS, 1,
+        "Sideline samples taken: %d\n", num_samples);
+#ifdef DEBUG
+    LOG(logfile, LOG_SIDELINE|LOG_STATS, 1,
+        "Sideline optimizations performed: %d\n", num_optimized);
+    LOG(logfile, LOG_SIDELINE|LOG_STATS, 1,
+        "Sideline optimizations performed w/o synch: %d\n",
+        num_opt_with_no_synch);
+#endif
+
+    for (i=0; i<table.capacity; i++) {
+        while (table.table[i]) {
+            sample = table.table[i];
+            table.table[i] = sample->next;
+            global_heap_free(sample, sizeof(sample_entry_t) HEAPACCT(ACCT_SIDELINE));
+        }
+    }
+    global_heap_free(table.table, table.capacity*sizeof(sample_entry_t*) HEAPACCT(ACCT_SIDELINE));
+    
+    l = remember;
+    while (l != NULL) {
+        nextl = l->next;
+        e = l->list;
+        while (e != NULL) {
+            nexte = e->next;
+            global_heap_free(e, sizeof(remember_entry_t) HEAPACCT(ACCT_SIDELINE));
+            e = nexte;
+        }
+        global_heap_free(l, sizeof(remember_list_t) HEAPACCT(ACCT_SIDELINE));
+        l = nextl;
+    }
+
+    destroy_event(wake_event);
+    destroy_event(asleep_event);
+    destroy_event(exited_event);
+
+    destroy_event(paused_for_sideline_event);
+    destroy_event(resume_from_sideline_event);
+
+    os_close(logfile);
+
+    DELETE_LOCK(sideline_lock);
+    DELETE_LOCK(do_not_delete_lock);
+    DELETE_LOCK(remember_lock);
+    DELETE_LOCK(table.lock);
+}
+
+/* add profiling to top of trace 
+ * at top of trace store fragment_t * in global slot
+ * (clear it at top of shared_syscall & fcache_return)
+ * sideline thread samples that slot to find hot traces
+ */
+void
+add_sideline_prefix(dcontext_t *dcontext, instrlist_t *trace)
+{
+    instr_t *inst = instr_build(dcontext, OP_mov_st, 1, 1);
+    instr_set_src(inst, 0, opnd_create_immed_int(0x12345678, OPSZ_4));
+    instr_set_dst(inst, 0, opnd_create_base_disp(REG_NULL, REG_NULL, 0,
+                                    (int)&sideline_trace, OPSZ_4));
+    instrlist_prepend(trace, inst);
+}
+
+void 
+finalize_sideline_prefix(dcontext_t *dcontext, fragment_t *trace_f)
+{
+    byte *start_pc = (byte *) FCACHE_ENTRY_PC(trace_f);
+    byte *pc;
+
+    /* ASSUMPTION: sideline prefix is at top of fragment! */
+
+    /* fill in address of owning fragment now that that fragment exists */
+    pc = start_pc + 6; /* 2 bytes of opcode, 4 bytes of store address */
+    *((int *)pc) = (int)trace_f;
+}
+
+static void
+remove_sideline_profiling(dcontext_t *dcontext, instrlist_t *trace)
+{
+    /* remove sideline profiling instruction */
+    instr_t *instr;
+    instr = instrlist_first(trace);
+    ASSERT(instr_get_opcode(instr) == OP_mov_st &&
+           opnd_is_near_base_disp(instr_get_dst(instr, 0)) &&
+           opnd_get_disp(instr_get_dst(instr, 0)) == (int)&sideline_trace);
+    instrlist_remove(trace, instr);
+    instr_destroy(dcontext, instr);
+}
+
+void 
+sideline_start()
+{
+    if (child_sleep) {
+        LOG(logfile, LOG_SIDELINE, VERB_3, "sideline: in sideline_start()\n");
+        child_sleep = false;
+        signal_event(wake_event);
+    }
+}
+
+void 
+sideline_stop()
+{
+    if (!child_sleep) {
+        LOG(logfile, LOG_SIDELINE, VERB_3, "SIDELINE: in sideline_stop()\n");
+        child_sleep = true;
+        /* signal paused_for_sideline_event to prevent wait-forever */
+        signal_event(paused_for_sideline_event);
+        wait_for_event(asleep_event);
+    }
+}
+
+/* Procedure executed by sideline thread
+ */
+static int RUN_SIG
+sideline_run(void *arg)
+{
+    logfile = open_log_file("sideline", NULL, 0);
+    LOG(logfile, LOG_SIDELINE, VERB_3, "SIDELINE: in sideline_run()\n");
+
+#ifdef LINUX
+    /* priority can only be set by thread itself, so do it here */
+    nice(10);
+#endif
+
+    while (!child_exit) {
+        if (child_sleep) {
+            LOG(logfile, LOG_SIDELINE, VERB_3, "SIDELINE: sideline thread going to sleep\n");
+            signal_event(asleep_event);
+            wait_for_event(wake_event);
+            continue;
+        }
+
+        /* take a sample, find a hot trace to optimize */
+        sideline_sample();
+
+        /* let other threads run */
+        thread_yield();
+    }
+    signal_event(exited_event);
+
+    LOG(logfile, LOG_SIDELINE, VERB_3, "SIDELINE: sideline thread exiting sideline_run()\n");
+#ifdef WINDOWS
+    /* with current create_thread implementation can't return from the run
+     * function */
+    os_terminate(NULL, TERMINATE_THREAD);
+    ASSERT_NOT_REACHED();
+#endif
+    return 0;
+}
+
+/* optimize_trace takes a tag, this routine pulls tag from frag */
+static void
+optimize_trace_wrapper(dcontext_t *dcontext, fragment_t *frag, instrlist_t *trace)
+{
+    optimize_trace(dcontext, frag->tag, trace);
+}
+
+static void 
+sideline_sample()
+{
+    sample_entry_t *e;
+    fragment_t *sample = (fragment_t *) sideline_trace; /* the sample! */
+
+    /* WARNING: we're using fragment_t* to make it easier to find target thread
+     * and its trace at once, but the trace could have been deleted, so be
+     * careful what you do with the fragment_t* -- here we never dereference it
+     */
+
+    /* first increment counter on current sample */
+    if (sample == NULL) {
+        LOG(logfile, LOG_SIDELINE, VERB_3, "\tSIDELINE: sample slot empty\n");
+    } else {
+        e = update_sample_entry((ptr_uint_t)sample);
+        LOG(logfile, LOG_SIDELINE, VERB_3,
+            "\tSIDELINE: sample now is "PFX" == F%d with count %d\n", 
+            sample, ((fragment_t*)sample)->id, e->counter);
+    }
+
+    /* we would clear the entry -- but a write to the shared memory is a big
+     * performance hit on SMP (not on SMT though...FIXME: distinguish?)
+     * we'll just accept an inability to distinguish a loop from
+     * a blocked thread.
+     */
+   
+    num_samples++;
+    if (num_samples % SAMPLE_TO_OPTIMIZE_RATIO == 0) {
+        /* clear entry to prevent stale samples after optimizing */
+        sideline_trace = NULL;
+        sideline_examine_traces();
+    }
+}
+
+static void 
+sideline_examine_traces()
+{
+    sample_entry_t *e;
+    fragment_t *f;
+
+    /* WARNING: we're using fragment_t* to make it easier to find target thread
+     * and its trace at once, but the trace could have been deleted, so be
+     * careful what you do with the fragment_t* -- we use this do_not_delete_lock
+     * to make sure no fragments are deleted until our last dereference of
+     * the hot fragment
+     */
+    mutex_lock(&do_not_delete_lock);
+
+    /* now find hottest trace */
+    e = find_hottest_entry();
+    if (e == NULL || e->counter <= SAMPLE_COUNT_THRESHOLD) {
+        LOG(logfile, LOG_SIDELINE, VERB_3,
+            "\tSIDELINE: cannot find a hot entry w/ count > %d\n",
+            SAMPLE_COUNT_THRESHOLD);
+        mutex_unlock(&do_not_delete_lock);
+        return;
+    }
+
+    f = (fragment_t *) e->tag;
+    LOG(logfile, LOG_SIDELINE, VERB_3,
+        "\tSIDELINE: hottest entry is "PFX" == F%d with count %d\n",
+        f, f->id, e->counter);
+    /* don't need entry anymore, no matter what happens */
+    remove_sample_entry(e->tag);
+
+    /* a trace we optimized could still run old code that posts samples */
+    if ((f->flags & FRAG_DO_NOT_SIDELINE) != 0) {
+        LOG(logfile, LOG_SIDELINE, VERB_3,
+            "\tSIDELINE: hottest entry F%d already sidelined\n",
+            f->id);
+        /* don't loop here looking for another entry -- let run() loop */
+        mutex_unlock(&do_not_delete_lock);
+        return;
+    } else {
+        LOG(logfile, LOG_SIDELINE, VERB_2,
+            "\tSIDELINE: optimizing hottest entry == F%d with count %d\n",
+            f->id, e->counter);
+        f = sideline_optimize(f, remove_sideline_profiling, optimize_trace_wrapper);
+#ifdef DEBUG
+        if (f != NULL)
+            LOG(logfile, LOG_SIDELINE, VERB_2,
+                "\t  SIDELINE: optimized fragment is F%d\n", f->id);
+#endif
+    }
+
+    mutex_unlock(&do_not_delete_lock);
+}
+
+fragment_t *
+sideline_optimize(fragment_t *f,
+                  void (*remove_profiling_func)(dcontext_t *,instrlist_t *),
+                  void (*optimize_function)(dcontext_t *,fragment_t *,instrlist_t *))
+{
+    fragment_t *new_f = NULL;
+#if defined (PROFILE_LINKCOUNT) && defined(SIDELINE_COUNT_STUDY)
+    trace_only_t *new_t;
+#endif
+    dcontext_t *dcontext;
+    instrlist_t *ilist;
+    uint flags;
+    void *vmlist = NULL;
+    DEBUG_DECLARE(bool ok;)
+
+#ifdef PROFILE_LINKCOUNT
+#ifdef SIDELINE_COUNT_STUDY
+    linkcount_type_t pre_opt_count = get_total_linkcount(f);
+#endif
+    if (stats->loglevel >= VERB_2 && (stats->logmask & LOG_SIDELINE) != 0) {
+        /* print out total count of exit counters */
+        LOG(logfile, LOG_SIDELINE, VERB_2, "\ttotal times F%d executed prior to sideline: "
+            LINKCOUNT_FORMAT_STRING "\n", f->id, get_total_linkcount(f));
+    }
+#endif
+
+    LOG(logfile, LOG_SIDELINE, VERB_3, "\nsideline_optimize: F%d\n", f->id);
+    ASSERT((f->flags & FRAG_IS_TRACE) != 0);
+
+    LOG(logfile, LOG_SIDELINE,1, "\nsideline_optimize:  tag= "PFX, f->tag);
+
+    dcontext = get_dcontext_for_fragment(f);
+    /* HACK to work with routines like unlink_branch that don't
+     * get a dcontext passed in and instead call get_thread_private_dcontext()
+     */
+    set_thread_private_dcontext(dcontext);
+
+    /* To avoid synch problems we do not let target thread (the one that owns
+     * the trace f) execute dynamo code while we are in this routine
+     */
+    pause_for_sideline = dcontext->owning_thread;
+    ASSERT(is_thread_known(pause_for_sideline));
+
+    if (dcontext->whereami != WHERE_FCACHE) {
+        /* wait for thread to reach waiting point in dispatch */
+        LOG(logfile, LOG_SIDELINE, VERB_3,
+            "\nsideline_optimize: waiting for target thread %d\n",
+            pause_for_sideline);
+        if (!dynamo_exited && !child_sleep && !child_exit) {
+            /* must give up do_not_delete_lock in case app thread is going to
+             * call fragment_delete, but must know if f ends up being deleted,
+             * so store it in fragment_now_optimizing
+             * ASSUMPTION: will not examine sample hashtable again, so ok to
+             * delete other fragments than f
+             */
+            fragment_now_optimizing = f;
+            mutex_unlock(&do_not_delete_lock);
+            wait_for_event(paused_for_sideline_event);
+            mutex_lock(&do_not_delete_lock);
+            /* if f was deleted, exit now */
+            if (fragment_now_optimizing == NULL)
+                goto exit_sideline_optimize;
+        }
+    }
+    if (dynamo_exited || child_sleep || child_exit) {
+        /* a different thread than the one we asked to pause could be exiting
+         * (hence the child_sleep command), so we better wake up the paused
+         * thread
+         */
+        goto exit_sideline_optimize;
+    }
+
+    /* build IR */
+    ilist = decode_fragment(dcontext, f, NULL, NULL, f->flags, NULL, NULL);
+
+#ifdef DEBUG
+    ASSERT(instr_get_opcode(instrlist_last(ilist))==OP_jmp);
+    LOG(logfile, LOG_SIDELINE, VERB_3, "\nbefore removing profiling:\n");
+    if (stats->loglevel >= VERB_3 && (stats->logmask & LOG_SIDELINE) != 0)
+        instrlist_disassemble(dcontext, f->tag, ilist, THREAD);
+#endif
+    remove_profiling_func(dcontext, ilist);
+
+#ifdef DEBUG
+    LOG(logfile, LOG_SIDELINE, VERB_3, "\nafter removing profiling:\n");
+    if (stats->loglevel >= VERB_3 && (stats->logmask & LOG_SIDELINE) != 0)
+        instrlist_disassemble(dcontext, f->tag, ilist, THREAD);
+#endif
+
+    /* FIXME: separate always-do-online optimizations from
+     * sideline optimizations
+     * for now, we do all online or all sideline
+     * FIXME: our ilist is already fully decoded, so we could avoid the
+     * full decode pass in optimize_trace
+     */
+#ifdef DEBUG
+    LOG(logfile, LOG_SIDELINE, OPTVERB_3, "\nbefore optimization:\n");
+    if (stats->loglevel >= OPTVERB_3 && (stats->logmask & LOG_SIDELINE) != 0)
+        instrlist_disassemble(dcontext, f->tag, ilist, THREAD);
+#endif
+    optimize_function(dcontext, f, ilist);
+#ifdef DEBUG
+    LOG(logfile, LOG_SIDELINE, OPTVERB_3, "\nafter optimization:\n");
+    if (stats->loglevel >= OPTVERB_3 && (stats->logmask & LOG_SIDELINE) != 0)
+        instrlist_disassemble(dcontext, f->tag, ilist, THREAD);
+#endif
+    /* Note that the offline optimization interface cannot be used
+     * here because it requires inserting the optimized code prior
+     * to creating a fragment.  Here we must replace the entire
+     * old fragment.
+     */
+
+    /*
+   replacement can be done w/o synch: opt thread moves all links from
+   old trace to new (single writes -> atomic) -- don't unlink and relink,
+   instead change_link to ensure even prof_counts is atomic.
+   moves link in hashtable to point to new trace, but leaves old trace's
+   next hashtable pointer intact.  only when app thread is in dynamo
+   code does it fully remove old trace from hashtable.
+    */
+
+    /* Emit fragment but don't make visible yet */
+    /* mark both old and new as DO_NOT_SIDELINE */
+    /* FIXME: if f is shared we must hold change_linking_lock here */
+    ASSERT(!TEST(FRAG_SHARED, f->flags));
+    f->flags |= FRAG_DO_NOT_SIDELINE;
+    flags = f->flags;
+    DEBUG_DECLARE(ok =)
+        vm_area_add_to_list(dcontext, f->tag, &vmlist, flags, f, false/*no locks*/);
+    ASSERT(ok); /* should never fail for private fragments */
+    new_f = emit_invisible_fragment(dcontext, f->tag, ilist, flags, vmlist);
+    fragment_copy_data_fields(dcontext, f, new_f);
+
+#if defined (PROFILE_LINKCOUNT) && defined(SIDELINE_COUNT_STUDY)
+    new_t = TRACE_FIELDS(new_f);
+    new_t->count_old_pre = pre_opt_count;
+#endif
+
+    LOG(logfile, LOG_SIDELINE, VERB_3, "emitted invisible fragment F%d\n", new_f->id);
+
+    shift_links_to_new_fragment(dcontext, f, new_f);
+
+    /* use fragment_replace to insert new_f into table while leaving f's
+     * next field intact in case hashtable lookup routine is examining ftable
+     */
+    fragment_replace(dcontext, f, new_f);
+
+    /* remember old fragment so can delete it later */
+    add_remember_entry(dcontext, f
+#ifdef SIDELINE_COUNT_STUDY
+                       , new_f
+#endif
+                       );
+
+    /* clean up */
+    instrlist_clear_and_destroy(dcontext, ilist);
+
+#ifdef DEBUG
+    num_optimized++;
+    if (stats->loglevel >= 2 && (stats->logmask & LOG_SIDELINE) != 0) {
+        disassemble_fragment(dcontext, new_f, stats->loglevel < 3);
+        LOG(logfile, LOG_SIDELINE, 2, "\tSIDELINE: emitted optimized F%d to replace F%d\n",
+            new_f->id, f->id);
+    }
+#endif /* DEBUG */
+
+ exit_sideline_optimize:
+    pause_for_sideline = (thread_id_t) 0;
+    if (!mutex_trylock(&sideline_lock)) {
+        /* thread is waiting in dispatch */
+        signal_event(resume_from_sideline_event);
+        mutex_lock(&sideline_lock);
+        /* at this point we know thread has read our resume event 
+         * clear all state
+         */
+        reset_event(paused_for_sideline_event);
+        reset_event(resume_from_sideline_event);
+    }
+#ifdef DEBUG
+    else
+        num_opt_with_no_synch++;
+#endif
+    mutex_unlock(&sideline_lock);
+    /* see HACK above */
+    set_thread_private_dcontext(NULL);
+
+    return new_f;
+}
+
+/* called by target thread when it is at safe point so a replaced trace can
+ * be completely removed
+ */
+void
+sideline_cleanup_replacement(dcontext_t *dcontext)
+{
+    remember_list_t *l, *prev_l;
+    remember_entry_t *e, *next_e;
+#ifdef PROFILE_LINKCOUNT
+# ifdef SIDELINE_COUNT_STUDY
+    trace_only_t *t;
+# endif
+#endif
+
+    /* clear sample entry, it could still contain a pointer to a fragment
+     * we're about to delete
+     */
+    sideline_trace = NULL;
+        
+    mutex_lock(&remember_lock);
+    for (l = remember, prev_l = NULL; l != NULL; prev_l = l, l = l->next) {
+        if (l->dcontext == dcontext) {
+            e = l->list;
+            while (e != NULL) {
+                next_e = e->next;
+                
+                /* clean up fragment 
+                 * deliberately do not call incoming_remove_fragment
+                 */
+                LOG(logfile, LOG_SIDELINE, VERB_3, "sideline_cleanup: cleaning up F%d\n",
+                    e->f->id);
+
+#ifdef PROFILE_LINKCOUNT
+# ifdef SIDELINE_COUNT_STUDY
+                t = TRACE_FIELDS(e->replacement);
+                t->count_old_post = get_total_linkcount(e->f);
+# endif
+                if (stats->loglevel >= VERB_2 && (stats->logmask & LOG_SIDELINE) != 0) {
+                    /* print out total count of exit counters */
+                    LOG(logfile, LOG_SIDELINE, VERB_2,
+                        "\ttotal times F%d executed prior to cleanup: "
+                        LINKCOUNT_FORMAT_STRING "\n", e->f->id,
+                        get_total_linkcount(e->f));
+                }
+#endif
+
+                fragment_delete(dcontext, e->f, FRAGDEL_NO_OUTPUT |
+                                FRAGDEL_NO_UNLINK | FRAGDEL_NO_HTABLE);
+                STATS_INC(num_fragments_deleted_sideline);
+
+                global_heap_free(e, sizeof(remember_entry_t) HEAPACCT(ACCT_SIDELINE));
+                e = next_e;
+            }
+            if (prev_l != NULL)
+                prev_l->next = l->next;
+            else
+                remember = l->next;
+            global_heap_free(l, sizeof(remember_list_t) HEAPACCT(ACCT_SIDELINE));
+            mutex_unlock(&remember_lock);
+            return;
+        }
+    }
+    mutex_unlock(&remember_lock);
+}
+
+
+static sample_entry_t *
+find_hottest_entry()
+{
+    uint i;
+    sample_entry_t *e, *max = NULL;
+    int max_counter = 0;
+    
+    mutex_lock(&table.lock);
+    for (i=0; i<table.capacity; i++) {
+        for (e = table.table[i]; e; e = e->next) {
+            if (e->counter > max_counter) {
+                max_counter = e->counter;
+                max = e;
+            }
+        }
+    }
+    mutex_unlock(&table.lock);
+    return max;
+}
+
+static sample_entry_t *
+update_sample_entry(ptr_uint_t tag)
+{
+    uint hindex;
+    sample_entry_t *e;
+    bool found = false;
+
+    /* look up entry for id */
+    mutex_lock(&table.lock);
+    hindex = HASH_FUNC((ptr_uint_t)tag, &table);
+    for (e = table.table[hindex]; e; e = e->next) {
+        if (e->tag == tag) {
+            found = true;
+            break;
+        }
+    }
+
+    if (found) {
+        e->counter++;
+    } else {
+        /* make new entry */
+        e = (sample_entry_t*) global_heap_alloc(sizeof(sample_entry_t) HEAPACCT(ACCT_SIDELINE));
+        e->tag = tag;
+        e->counter = 1;
+        e->next = table.table[hindex];
+        table.table[hindex] = e;
+    }
+
+    mutex_unlock(&table.lock);
+    return e;
+}
+
+/* executed by an application thread when it deletes fragment f */
+void
+sideline_fragment_delete(fragment_t *f)
+{
+    if ((f->flags & FRAG_IS_TRACE) != 0) {
+        /* see notes above on reasons for this extra lock */
+        mutex_lock(&do_not_delete_lock);
+        /* clear sample entry, it could still contain a pointer to f */
+        sideline_trace = NULL;
+        remove_sample_entry((ptr_uint_t)f);
+        /* let sideline_optimize know if we delete its fragment while it's waiting */
+        if (fragment_now_optimizing == f)
+            fragment_now_optimizing = NULL;
+
+        mutex_unlock(&do_not_delete_lock);
+    }
+}
+
+static void
+remove_sample_entry(ptr_uint_t tag)
+{
+    uint hindex;
+    sample_entry_t *e, *preve;
+
+    mutex_lock(&table.lock);
+    hindex = HASH_FUNC((ptr_uint_t)tag, &table);
+    for (e = table.table[hindex], preve = NULL; e != NULL; preve = e, e = e->next) {
+        if (e->tag == tag) {
+            if (preve)
+                preve->next = e->next;
+            else
+                table.table[hindex] = e->next;
+            global_heap_free(e, sizeof(sample_entry_t) HEAPACCT(ACCT_SIDELINE));
+            break;
+        }
+    }
+    mutex_unlock(&table.lock);
+}
+
+static void
+add_remember_entry(dcontext_t *dcontext, fragment_t *f
+#ifdef SIDELINE_COUNT_STUDY
+                   , fragment_t *new_f
+#endif
+                   )
+{
+    remember_list_t *l;
+    remember_entry_t *e;
+    mutex_lock(&remember_lock);
+    l = remember;
+    while (l != NULL) {
+        if (l->dcontext == dcontext) {
+            /* make new entry */
+            e = (remember_entry_t*) global_heap_alloc(sizeof(remember_entry_t) HEAPACCT(ACCT_SIDELINE));
+            e->f = f;
+#ifdef SIDELINE_COUNT_STUDY
+            e->replacement = new_f;
+#endif
+            e->next = l->list;
+            l->list = e;
+            mutex_unlock(&remember_lock);
+            return;
+        }
+        l = l->next;
+    }
+    /* make new list */
+    l = (remember_list_t*) global_heap_alloc(sizeof(remember_list_t) HEAPACCT(ACCT_SIDELINE));
+    l->dcontext = dcontext;
+    l->list = (remember_entry_t*) global_heap_alloc(sizeof(remember_entry_t) HEAPACCT(ACCT_SIDELINE));
+    l->list->f = f;
+#ifdef SIDELINE_COUNT_STUDY
+    l->list->replacement = new_f;
+#endif
+    l->list->next = NULL;
+    l->next = remember;
+    remember = l;
+    mutex_unlock(&remember_lock);
+}
+
+#ifdef LINUX /***************************************************************/
+/* Create a new thread. It should be passed "fcn", a function which
+ * takes two arguments, (the second one is a dummy, always 4). The
+ * first argument is passed in "arg". Returns the PID of the new
+ * thread 
+ * QUESTION: why does bbthreads use asm syscall instead of libc wrapper?
+ * WARNING: clone is not present in libc5 or earlier
+ */
+static thread_t
+create_thread(int (*fcn)(void *), void *arg, void **stack)
+{
+    thread_t thread;
+    
+    int flags;
+    void *my_stack;
+    my_stack = stack_alloc(THREAD_STACK_SIZE);
+    /* need SIGCHLD so parent will get that signal when child dies,
+     * else have "no children" errors doing a wait
+     */
+    flags = SIGCHLD | CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND;
+    thread = clone(fcn, my_stack, flags, arg); /* old: __clone */
+  
+    if (thread == -1) {
+        SYSLOG_INTERNAL_ERROR("SIDELINE: Error calling __clone");
+        stack_free(my_stack, THREAD_STACK_SIZE);
+        ASSERT_NOT_REACHED();
+    }
+
+    *stack = my_stack;
+
+    return thread;
+}
+
+static void 
+delete_thread(thread_t thread, void *stack)
+{
+    pid_t result;
+    result = waitpid(thread, NULL, 0);
+    stack_free(stack, THREAD_STACK_SIZE);
+    if (result == -1) {
+        perror("delete_thread");
+        SYSLOG_INTERNAL_ERROR("SIDELINE: Error deleting thread");
+        ASSERT_NOT_REACHED();
+    }
+}
+#endif /* LINUX ***************************************************************/
+
+
+#endif /* SIDELINE -- around whole file */
diff --git a/core/x86/sideline.h b/core/x86/sideline.h
new file mode 100644
index 000000000..af1d2f1a6
--- /dev/null
+++ b/core/x86/sideline.h
@@ -0,0 +1,82 @@
+/* **********************************************************
+ * Copyright (c) 2002-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+
+/*
+ * sideline.h - multiprocessor support
+ */
+
+#ifndef _SIDELINE_H_
+#define _SIDELINE_H_ 1
+
+/* sampled by sideline thread to find hot traces */
+extern volatile fragment_t * sideline_trace;
+
+extern int num_processors;
+
+/* used for synchronization with other threads */
+extern mutex_t sideline_lock;
+extern thread_id_t pause_for_sideline;
+extern event_t paused_for_sideline_event;
+extern event_t resume_from_sideline_event;
+extern mutex_t do_not_delete_lock;
+/* initialization */
+void sideline_init(void);
+
+/* atexit cleanup */
+void sideline_exit(void);
+
+/* tells sideline threads when dynamo is active */
+void sideline_start(void);
+void sideline_stop(void);
+
+/* adds profiling for identification of hot traces  */
+void add_sideline_prefix(dcontext_t *dcontext, instrlist_t *trace);
+void finalize_sideline_prefix(dcontext_t *dcontext, fragment_t *trace_f);
+
+/* called when target thread is at safe point so a replaced trace can
+ * be completely removed
+ */
+void sideline_cleanup_replacement(dcontext_t *dcontext);
+
+/* called by app thread to remove f from sideline data structures */
+void sideline_fragment_delete(fragment_t *f);
+
+/* calls optimize_function on the trace of interest, safely handles replacement */
+fragment_t *
+sideline_optimize(fragment_t *f,
+                  void (*remove_sideline_profiling)(dcontext_t *,instrlist_t *),
+                  void (*optimize_function)(dcontext_t *,fragment_t *,instrlist_t *));
+
+#endif /* _SIDELINE_H_ */
diff --git a/core/x86/steal_reg.c b/core/x86/steal_reg.c
new file mode 100644
index 000000000..16785d1df
--- /dev/null
+++ b/core/x86/steal_reg.c
@@ -0,0 +1,624 @@
+/* **********************************************************
+ * Copyright (c) 2000-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/* x86/steal_reg.c */
+
+#include "globals.h"
+#include "instrlist.h"
+
+#include "arch.h"
+#include "instr.h"
+#include "steal_reg.h"
+#include "fragment.h"
+
+/* N.B.:
+ * For debugging reg stealing, it's nice to be able to only do reg stealing
+ * on selected fragments.  To support this we allow STEAL_REGISTER to be defined
+ * even when DCONTEXT_IN_EDI is not.
+ */
+
+/* around whole file: */
+#ifdef STEAL_REGISTER 
+
+
+/* In the following functions, edi is assumed to be a pointer to the
+ * dynamo context object.  The application's version of edi is kept
+ * in memory, and we use ebx as a scratch register.  We use esi as
+ * an alternative scratch register.  */
+
+/* x86 notes:
+ * cti instruction reg usage:
+ *   jmp,jcc direct: 'J' = immed only = no regs used
+ *   jmp indirect:   'E' = either single reg or base or base + index 
+ *                       = max 2 regs
+ *   ret/lret: 'I' = immed only = no regs used
+ *   jcxz and loop*: 'J' + ecx = uses ecx only
+ *   calls are just like jmps
+ * other instructions:
+ *   string instrs use esi and edi
+ *   rep uses ecx
+ *   xlat uses al, ebx
+ *   wrmsr uses eax, ecx, and edx -- but requires kernel mode
+ * eax and edx are used for call return values
+ *   (edx for 64-bit values)
+ *
+ * Must treat 8-bit registers separately!
+ * 16-bit versions (e.g., w/ data size prefix) can be treated just
+ * like 32-bit since in terms of modrm bits DI==EDI and we'll change
+ * it to use BX/SI/DX just like in 32-bit mode we'd use EBX/ESI/EDX.
+ */
+
+
+
+/* you must pass in the NEXT instruction!
+ * restore_state restores state prior to the instruction passed in!
+ * you can pass in null, in which case restore_state will append
+ * to ilist.
+ */
+void
+restore_state(dcontext_t *dcontext, instr_t *instr, instrlist_t *ilist)
+{
+    if (ilist->flags == EDI_VAL_IN_MEM)
+        return;
+
+    if (instr != NULL) {
+        /* insert store if edi's value in ebx doesn't match dcontext->xdi */
+        if (ilist->flags != EDI_VAL_IN_EBX_AND_MEM) {
+            instrlist_preinsert(ilist, instr,
+                                instr_create_save_to_dcontext(dcontext, REG_EBX, XDI_OFFSET));
+        }
+
+        /* restore application's value for ebx in ebx */
+        instrlist_preinsert(ilist, instr,
+                            instr_create_restore_from_dcontext(dcontext, REG_EBX, XBX_OFFSET));
+
+#ifndef DCONTEXT_IN_EDI
+        /* restore app's edi */
+        instrlist_preinsert(ilist, instr,
+                            instr_create_restore_from_dcontext(dcontext, REG_EDI, XDI_OFFSET));
+#endif
+
+    } else {
+        if (ilist->flags != EDI_VAL_IN_EBX_AND_MEM) {
+            instrlist_append(ilist,
+                             instr_create_save_to_dcontext(dcontext, REG_EBX, XDI_OFFSET));
+        }
+
+        instrlist_append(ilist,
+                         instr_create_restore_from_dcontext(dcontext, REG_EBX, XBX_OFFSET));
+
+#ifndef DCONTEXT_IN_EDI
+        /* restore app's edi */
+        instrlist_append(ilist,
+                         instr_create_restore_from_dcontext(dcontext, REG_EDI, XDI_OFFSET));
+#endif
+    }
+
+    ilist->flags = EDI_VAL_IN_MEM;
+}
+
+
+static void
+expand_pusha(dcontext_t *dcontext, instr_t *instr, instrlist_t *ilist)
+{
+    /* Convert into the following sequence:
+         pusha
+         addl  $4, %esp         # %esp = %esp + 4
+         push  edi_offset(%edi) # push real edi value onto stack
+    */
+
+    if (ilist->flags != EDI_VAL_IN_MEM)
+        restore_state(dcontext, instr, ilist);
+
+#ifndef DCONTEXT_IN_EDI
+    return;
+#endif
+
+    /* reverse order! */
+
+    /* build and insert push instruction as binary */
+    instrlist_postinsert(ilist, instr,
+                      instr_create_raw_3bytes(dcontext,
+                                              0xff,
+                                              0x77,/* %edi + 8-bit offset + /6 */
+                                              XDI_OFFSET));
+
+    /* build and insert addl instruction as binary */
+    instrlist_postinsert(ilist, instr,
+                      instr_create_raw_3bytes(dcontext,
+                                              0x83,
+                                              0xc4, /* %esp + 8-bit immed + /0 */
+                                              4));
+}
+
+static void
+expand_popa(dcontext_t *dcontext, instr_t *instr, instrlist_t *ilist)
+{
+#ifndef DCONTEXT_IN_EDI
+    return;
+#endif
+
+    /* Convert into the following sequence:
+         movl  (%esp), %ebx     # get edi value from stack
+         movl  %ebx, edi_offset(%edi)   # save it in context
+         movl  %edi, 12(%esp)   # squirrel away context ptr in esp location
+         popa                   # NOTE: doesn't restore esp from stack
+         movl  -20(%esp), %edi  # restore context ptr
+    */
+    instrlist_preinsert(ilist, instr,
+                        INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_EBX),
+                                            OPND_CREATE_MEM32(REG_ESP, 0)));
+    instrlist_preinsert(ilist, instr,
+                        instr_create_save_to_dcontext(dcontext, REG_EBX, XDI_OFFSET));
+    instrlist_preinsert(ilist, instr,
+                        INSTR_CREATE_mov_st(dcontext,
+                                            OPND_CREATE_MEM32(REG_ESP, 12),
+                                            opnd_create_reg(REG_EDI)));
+    instrlist_postinsert(ilist, instr,
+                        INSTR_CREATE_mov_ld(dcontext, opnd_create_reg(REG_EDI),
+                                            OPND_CREATE_MEM32(REG_ESP, -20)));
+    ilist->flags = EDI_VAL_IN_MEM;
+}
+
+
+/* Handle implicit references to edi.  This routine assumes that
+ * edi is both read and written, that no explicit operands exist
+ * in the instruction, and that ebx is not used. */
+/* 
+example:
+  0x00405178   f3 a5                repz movs  %ds:%esi,%es:%edi
+becomes
+  0x0142cb94   89 5f 04             mov   %ebx,0x4(%edi)
+  0x0142cb97   89 fb                mov   %edi,%ebx
+  0x0142cb99   8b 7b 14             mov   0x14(%ebx),%edi
+  0x0142cb9c   f3 a5                repz movs  %ds:%esi,%es:%edi
+  0x0142cb9e   87 fb                xchg  %edi,%ebx
+  0x0142cba0   89 5f 14             mov   %ebx,0x14(%edi)
+  0x0142cba3   8b 5f 04             mov   0x4(%edi),%ebx
+*/
+static void
+use_edi(dcontext_t *dcontext,
+        instr_t *instr, instrlist_t *ilist)
+{
+#ifndef DCONTEXT_IN_EDI
+    /* save cur edi, then bring dcontext into edi */
+    instrlist_preinsert(ilist, instr,
+                        instr_create_save_to_dcontext(dcontext, REG_EDI, XDI_OFFSET));
+    instrlist_preinsert(ilist, instr,
+                        INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_EDI),
+                                             opnd_create_immed_int((int)dcontext,
+                                                                   OPSZ_PTR)));
+#endif
+
+    if (ilist->flags == EDI_VAL_IN_MEM) {
+        instrlist_preinsert(ilist, instr,
+                           instr_create_save_to_dcontext(dcontext, REG_EBX, XBX_OFFSET));
+        instrlist_preinsert(ilist, instr,
+                            INSTR_CREATE_mov(dcontext, opnd_create_reg(REG_EBX),
+                                             opnd_create_reg(REG_EDI));
+        instrlist_preinsert(ilist, instr,
+                            load_instr(dcontext, REG_EDI, REG_EBX, XDI_OFFSET));
+    } else {
+        /* create 'xchg %edi, %ebx' */
+        instrlist_preinsert(ilist, instr,
+                           INSTR_CREATE_xchg(dcontext, opnd_create_reg(REG_EDI),
+                                             opnd_create_reg(REG_EBX)));
+    }
+
+    /* instruction can now use edi */
+
+    /* create 'xchg %edi, %ebx' */
+    instrlist_postinsert(ilist, instr,
+                      INSTR_CREATE_xchg(dcontext, opnd_create_reg(REG_EDI),
+                                        opnd_create_reg(REG_EBX)));
+
+    ilist->flags = EDI_VAL_IN_EBX;
+}
+
+
+/* An alternative rewriter: Rewrite the modrm byte using esi or edx, since
+ *  ebx is unavailable (i.e., simultaneously used).
+ *
+example, with following instr that uses edi too (but not ebx):
+  0x0040a492   8a 1f                mov   (%edi),%bl
+  0x0040a494   47                   inc   %edi
+becomes:
+  0x0142faba   89 77 10             mov   %esi,0x10(%edi)
+  0x0142fabd   8b 77 14             mov   0x14(%edi),%esi
+  0x0142fac0   8a 1e                mov   (%esi),%bl
+  0x0142fac2   8b 77 10             mov   0x10(%edi),%esi
+  0x0142fac5   89 5f 04             mov   %ebx,0x4(%edi)
+  0x0142fac8   8b 5f 14             mov   0x14(%edi),%ebx
+  0x0142facb   43                   inc   %ebx
+  0x0142facc   89 5f 14             mov   %ebx,0x14(%edi)
+  0x0142facf   8b 5f 04             mov   0x4(%edi),%ebx
+*/
+static void
+use_different_reg(dcontext_t *dcontext,
+                  instr_t *instr, instrlist_t *ilist, int reg, int offs,
+                  bool read, bool write)
+{
+    if (ilist->flags != EDI_VAL_IN_MEM)
+        restore_state(dcontext, instr, ilist);  /* cannot use ebx */
+
+#ifndef DCONTEXT_IN_EDI
+    /* save cur edi, then bring shared_dcontext into edi */
+    instrlist_preinsert(ilist, instr,
+                        instr_create_save_to_dcontext(dcontext, REG_EDI, XDI_OFFSET));
+    instrlist_preinsert(ilist, instr,
+                        INSTR_CREATE_mov_imm(dcontext, opnd_create_reg(REG_EDI),
+                                             opnd_create_immed_int((int)dcontext,
+                                                                   OPSZ_PTR)));
+#endif
+
+    /* save current reg */
+    instrlist_preinsert(ilist, instr,
+                       instr_create_save_to_dcontext(dcontext, reg, offs));
+    if (read) {
+        /* bring in current value of edi */
+        instrlist_preinsert(ilist, instr,
+                           instr_create_restore_from_dcontext(dcontext, reg, XDI_OFFSET));
+    }
+
+    /* instr goes here */
+
+    /* add to ilist in reverse order */
+#ifndef DCONTEXT_IN_EDI
+    /* restore app's edi */
+    instrlist_postinsert(ilist, instr,
+                      load_abs_instr(dcontext, REG_EDI,
+                                     ((int)(&shared_dcontext)) + XDI_OFFSET));
+#endif
+    /* restore previous value of reg */
+    instrlist_postinsert(ilist, instr,
+                         instr_create_restore_from_dcontext(dcontext, reg, offs));
+    if (write) {
+        /* copy value of edi back into memory location */
+        instrlist_postinsert(ilist, instr,
+                             instr_create_save_to_dcontext(dcontext, reg, XDI_OFFSET));
+    }
+}
+
+
+/*###########################################################################
+ *
+ * Here are a bunch of (old) examples
+ * Note that the current code will put in a direct memory reference when
+ * it can, so push %edi => push 0x14(%edi)
+ */
+/* Instruction directly reads and writes edi, e.g. INC %edi.
+   Register specifier is encoded directly in opcode byte. */
+/*
+example:
+  0x00405d2a   47                   inc   %edi
+becomes:
+  0x0142b8c9   89 5f 04             mov   %ebx,0x4(%edi)
+  0x0142b8cc   8b 5f 14             mov   0x14(%edi),%ebx
+  0x0142b8cf   43                   inc   %ebx
+  0x0142b8d0   89 5f 14             mov   %ebx,0x14(%edi)
+  0x0142b8d3   8b 5f 04             mov   0x4(%edi),%ebx
+example, with prev instr that uses edi:
+  0x00409e49   0f b6 37             movzx (%edi),%esi
+  0x00409e4c   47                   inc   %edi
+becomes:
+  0x0142fe80   89 5f 04             mov   %ebx,0x4(%edi)
+  0x0142fe83   8b 5f 14             mov   0x14(%edi),%ebx
+  0x0142fe86   0f b6 33             movzx (%ebx),%esi
+  0x0142fe89   43                   inc   %ebx
+  0x0142fe8a   89 5f 14             mov   %ebx,0x14(%edi)
+  0x0142fe8d   8b 5f 04             mov   0x4(%edi),%ebx
+*/
+/* Instruction directly reads edi, e.g. PUSH %edi.
+   Register specifier is encoded directly in opcode byte. */
+/*
+example:
+  0x00409d60   57                   push  %edi
+becomes:
+  0x0142fbd1   89 5f 04             mov   %ebx,0x4(%edi)
+  0x0142fbd4   8b 5f 14             mov   0x14(%edi),%ebx
+  0x0142fbd7   53                   push  %ebx
+  0x0142fbd8   8b 5f 04             mov   0x4(%edi),%ebx
+example, with following instr that uses edi as well:
+  0x00409e16   57                   push  %edi
+  0x00409e17   8b 7c 24 14          mov   0x14(%esp),%edi
+becomes:
+  0x0142ff15   89 5f 04             mov   %ebx,0x4(%edi)
+  0x0142ff18   8b 5f 14             mov   0x14(%edi),%ebx
+  0x0142ff1b   53                   push  %ebx
+  0x0142ff1c   8b 5c 24 14          mov   0x14(%esp),%ebx
+  0x0142ff20   89 5f 14             mov   %ebx,0x14(%edi)
+  0x0142ff23   8b 5f 04             mov   0x4(%edi),%ebx
+*/
+/* Instruction directly writes edi, e.g. POP %edi.
+   Register specifier is encoded directly in opcode byte. */
+/*
+example:
+  0x00409e99   5f                   pop   %edi
+becomes:
+  0x0142fce0   89 5f 04             mov   %ebx,0x4(%edi)
+  0x0142fce3   5b                   pop   %ebx
+  0x0142fce4   89 5f 14             mov   %ebx,0x14(%edi)
+  0x0142fce7   8b 5f 04             mov   0x4(%edi),%ebx
+*/
+/*
+example:
+  0x00409e33   0f b6 07             movzx (%edi),%eax
+becomes:
+  0x0142fec0   89 5f 04             mov   %ebx,0x4(%edi)
+  0x0142fec3   8b 5f 14             mov   0x14(%edi),%ebx
+  0x0142fec6   0f b6 03             movzx (%ebx),%eax
+  0x0142fec9   8b 5f 04             mov   0x4(%edi),%ebx
+new example:
+  0x00403de3   8b 3e                mov   (%esi) -> %edi 
+  0x00403de5   2b f8                sub   %eax %edi -> %edi 
+becomes:
+  0x0536e663   89 5f 04             mov   %ebx -> 0x4(%edi) 
+  0x0536e666   8b 1e                mov   (%esi) -> %ebx 
+  0x0536e668   2b d8                sub   %eax %ebx -> %ebx 
+  0x0536e66a   89 5f 14             mov   %ebx -> 0x14(%edi) 
+  0x0536e66d   8b 5f 04             mov   0x4(%edi) -> %ebx 
+here's a good example of reg steal + indirect jmp mangling:
+  0x77f831ff   ff 24 bd c0 32 f8 77 jmp   0x77f832c0(,%edi,4) 
+becomes:
+  0x052de580   89 57 0c             mov   %edx -> 0xc(%edi) 
+  0x052de583   89 5f 04             mov   %ebx -> 0x4(%edi) 
+  0x052de586   8b 5f 14             mov   0x14(%edi) -> %ebx 
+  0x052de589   8b 14 bd c0 32 f8 77 mov   0x77f832c0(,%edi,4) -> %edx 
+  0x052de590   8b 5f 04             mov   0x4(%edi) -> %ebx 
+  0x052de593   e9 00 00 00 00       jmp   0x52de598 <exit stub 0> 
+*/
+/*
+ * end of examples
+ *
+ *###########################################################################*/
+
+static void
+use_ebx(dcontext_t *dcontext,
+        instr_t *instr, instrlist_t *ilist, bool read, bool write)
+{
+    if (ilist->flags == EDI_VAL_IN_MEM) {
+
+#ifndef DCONTEXT_IN_EDI
+        /* save cur edi, then bring shared_dcontext into edi */
+        instrlist_preinsert(ilist, instr,
+                       store_abs_instr(dcontext, REG_EDI,
+                                       ((int)(&shared_dcontext)) + XDI_OFFSET));
+        instrlist_preinsert(ilist, instr,
+                           move_immed_instr(dcontext, (int)&shared_dcontext, REG_EDI));
+#endif
+
+        /* save current ebx */
+        instrlist_preinsert(ilist, instr,
+                           instr_create_save_to_dcontext(dcontext, REG_EBX, XBX_OFFSET));
+        if (read) {
+            /* bring value of edi into ebx */
+            instrlist_preinsert(ilist, instr,
+                               instr_create_restore_from_dcontext(dcontext, REG_EBX, XDI_OFFSET));
+            ilist->flags = EDI_VAL_IN_EBX_AND_MEM;
+        }
+    }
+
+    /* instr goes here */
+
+    if (write) {
+        /* record that memory location of edi is stale */
+        ilist->flags = EDI_VAL_IN_EBX;
+    }
+}
+
+
+void
+steal_reg(dcontext_t *dcontext, instr_t *instr, instrlist_t *ilist)
+{
+    opnd_t * uses[5];
+#ifdef DCONTEXT_IN_EDI
+    opnd_t save[5];
+#endif
+    int i, num_uses;
+    bool writes, reads;
+
+#ifndef DCONTEXT_IN_EDI
+#if 0
+    i = GLOBAL_STAT(num_fragments);
+    /* use this to narrow location of reg steal bug */
+    if (i < 0 || i > 1000000)
+        return;
+#endif
+#endif
+        
+    /* special cases */
+    switch (instr_get_opcode(instr)) {
+    case OP_pusha:
+        expand_pusha(dcontext, instr, ilist);
+        return;
+    case OP_popa:
+        expand_popa(dcontext, instr, ilist);
+        return;
+    case OP_ins:     case OP_rep_ins:
+    case OP_outs:    case OP_rep_outs:
+    case OP_movs:    case OP_rep_movs:
+    case OP_stos:    case OP_rep_stos:
+    case OP_lods:    case OP_rep_lods:
+    case OP_cmps:    case OP_rep_cmps:    case OP_repne_cmps:
+    case OP_scas:    case OP_rep_scas:    case OP_repne_scas:
+        /* these all use edi implicitly, there's no way around it */
+        use_edi(dcontext, instr, ilist);
+        return;
+   }
+
+    if (!instr_uses_reg(instr, REG_EDI) && !uses_reg(instr, REG_DI)) {
+        if (ilist->flags != EDI_VAL_IN_MEM)
+            restore_state(dcontext, instr, ilist);
+        return;
+    }
+    
+    /* if we get to here we know we're going to change the operands,
+     * possibly by directly writing to operand bytes, so we have to explicitly
+     * set original bits invalid:
+     */
+    instr_set_raw_bits_valid(instr, false);
+
+    /* gather edi usage info */
+    num_uses = 0;
+    writes = reads = false;
+    for (i=0; i<instr_num_dsts(instr); i++) {
+        if (opnd_uses_reg(instr_get_dst(instr, i), REG_EDI) ||
+            opnd_uses_reg(instr_get_dst(instr, i), REG_DI)) {
+            /* FIXME: this code was written back when accessed instr fields
+             * directly...need to fix this up
+             */
+#error FIXME -- OLD CODE
+            uses[num_uses++] = &instr->dsts[i];
+            if (opnd_is_memory_reference(instr_get_dst(instr, i)))
+                reads = true;
+            else if (opnd_uses_reg(instr_get_dst(instr, i), REG_DI)) {
+                reads = true; /* need to copy in upper 16 bits of edi */
+                writes = true;
+            } else
+                writes = true;
+        }
+    }
+    for (i=0; i<instr_num_src(instr); i++) {
+        if (opnd_uses_reg(instr_get_src(instr, i), REG_EDI) ||
+            opnd_uses_reg(instr_get_src(instr, i), REG_DI)) {
+            /* FIXME: this code was written back when accessed instr fields
+             * directly...need to fix this up
+             */
+#error FIXME -- OLD CODE
+            if (i == 0)
+                uses[num_uses++] = &instr->src0;
+            else
+                uses[num_uses++] = &instr->srcs[i];
+            reads = true;
+        }
+    }
+    ASSERT(num_uses > 0);
+
+    /* Try replacing register edi with direct memory access
+     * FIXME: try to use memory access to replace di?
+     * need to select 16-bit data size, + 16 bit offset from 0x14(%edi)?
+     * no 16-bit offset b/c of little-endian-ness!
+     */
+#ifdef DCONTEXT_IN_EDI
+    if (ilist->flags == EDI_VAL_IN_MEM) {
+        for (i=0; i<num_uses; i++) {
+            save[i] = *uses[i];
+            if (opnd_is_reg(save[i]) && opnd_get_reg(save[i]) == REG_EDI) {
+                *uses[i] = opnd_create_base_disp(REG_EDI, REG_NULL, 0, XDI_OFFSET,
+                                                 reg_get_size(REG_EDI));
+            } else {
+                /* give up */
+                int j;
+                for (j=0; j<i; j++)
+                    *uses[j] = save[j];
+                i = 0;
+                break;
+            }
+        }
+        if (i > 0) { /* we didn't give up */
+            /* switch to store if currently reg-reg "load" */
+            if (instr_get_opcode(instr) == OP_mov_ld && num_uses == 1 && writes) {
+                instr_set_opcode(instr, OP_mov_st);
+                i = 0;
+            }
+            if (instr_is_encoding_possible(instr)) {
+                /* It worked, we're done */
+                LOG(THREAD, LOG_INTERP, 3,
+                    "*** Successfully used memory to replace edi!\n");
+                return;
+            }
+            /* Else, restore */
+            if (i == 0) /* flag saying "changed OP_mov_ld to OP_mov_st" */
+                instr_set_opcode(instr, OP_mov_ld);
+            for (i=0; i<num_uses; i++) {
+                *uses[i] = save[i];
+            }
+        }
+    }
+#endif
+
+    if (!instr_uses_reg(instr, REG_EBX) && !instr_uses_reg(instr, REG_BX) &&
+        !instr_uses_reg(instr, REG_BL) && !instr_uses_reg(instr, REG_BH)) {
+        /* use ebx */
+        use_ebx(dcontext, instr, ilist, reads, writes);
+        for (i=0; i<num_uses; i++) {
+            opnd_replace_reg(uses[i], REG_EDI, REG_EBX);
+            opnd_replace_reg(uses[i], REG_DI, REG_BX);
+        }
+    }
+    else if (!instr_uses_reg(instr, REG_ESI) && !instr_uses_reg(instr, REG_SI)) {
+        /* use esi */
+        use_different_reg(dcontext, instr, ilist, REG_ESI, XSI_OFFSET, reads, writes);
+        for (i=0; i<num_uses; i++) {
+            opnd_replace_reg(uses[i], REG_EDI, REG_ESI);
+            opnd_replace_reg(uses[i], REG_DI, REG_SI);
+        }
+    }
+    else {
+        ASSERT(!instr_uses_reg(instr, REG_EDX) && !instr_uses_reg(instr, REG_DX) &&
+               !instr_uses_reg(instr, REG_DL) && !instr_uses_reg(instr, REG_DH));
+        /* use edx */
+        use_different_reg(dcontext, instr, ilist, REG_EDX, XDX_OFFSET, reads, writes);
+        for (i=0; i<num_uses; i++) {
+            opnd_replace_reg(uses[i], REG_EDI, REG_EDX);
+            opnd_replace_reg(uses[i], REG_DI, REG_DX);
+        }
+    }
+}
+
+/*
+sequence to study for better stealing:
+  0x77fca2da   0f b7 38             movzx  (%eax) -> %edi 
+  0x77fca2dd   2b fb                sub    %ebx %edi -> %edi 
+  0x77fca2df   89 7d a8             mov    %edi -> 0xffffffa8(%ebp) 
+becomes:
+  0x0265e0cd   89 5f 04             mov    %ebx -> 0x4(%edi) 
+  0x0265e0d0   0f b7 18             movzx  (%eax) -> %ebx 
+  0x0265e0d3   89 5f 14             mov    %ebx -> 0x14(%edi) 
+  0x0265e0d6   8b 5f 04             mov    0x4(%edi) -> %ebx 
+  0x0265e0d9   89 77 10             mov    %esi -> 0x10(%edi) 
+  0x0265e0dc   8b 77 14             mov    0x14(%edi) -> %esi 
+  0x0265e0df   2b f3                sub    %ebx %esi -> %esi 
+  0x0265e0e1   89 77 14             mov    %esi -> 0x14(%edi) 
+  0x0265e0e4   8b 77 10             mov    0x10(%edi) -> %esi 
+  0x0265e0e7   89 5f 04             mov    %ebx -> 0x4(%edi) 
+  0x0265e0ea   8b 5f 14             mov    0x14(%edi) -> %ebx 
+  0x0265e0ed   89 5d a8             mov    %ebx -> 0xffffffa8(%ebp) 
+  0x0265e0f0   8b 5f 04             mov    0x4(%edi) -> %ebx 
+*/
+
+
+#endif /* STEAL_REGISTER (around whole file) */
diff --git a/core/x86/steal_reg.h b/core/x86/steal_reg.h
new file mode 100644
index 000000000..cf3e76104
--- /dev/null
+++ b/core/x86/steal_reg.h
@@ -0,0 +1,62 @@
+/* **********************************************************
+ * Copyright (c) 2000-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2000-2001 Hewlett-Packard Company */
+
+/* file "x86/steal_reg.h" */
+
+#ifndef X86_STEAL_REG_H
+#define X86_STEAL_REG_H
+
+/* This header file declares the tables and functions that help decode
+   x86 instructions so that we can steal edi for our own uses. */
+
+void steal_reg(dcontext_t *dcontext, instr_t *instr, instrlist_t *ilist);
+void restore_state(dcontext_t *dcontext, instr_t *instr, instrlist_t *ilist);
+
+/* We use the instrlist_t flags field to signify when the application's
+ * value in ebx is in dcontext and ebx actually contains the
+ * application's edi value (true when bit 0 of flags is asserted).  As
+ * you can see in the .c file, we're pretty conservative with this
+ * optimization.  We're really only concerned with the stretches where
+ * edi is involved in a calculation.  As such, whenever an instruction
+ * doesn't use edi, we place dcontext->xbx back in ebx.  We also keep
+ * track of when the edi value in ebx matches dcontext->xdi (true when
+ * bit 1 of flags is asserted) so that we don't have to dump ebx into
+ * dcontext->xdi unnecessarily. */
+#define EDI_VAL_IN_MEM 0
+#define EDI_VAL_IN_EBX 1
+#define EDI_VAL_IN_EBX_AND_MEM 3
+
+#endif /* X86_STEAL_REG_H */
diff --git a/core/x86/x86.asm b/core/x86/x86.asm
new file mode 100644
index 000000000..46b0b5c73
--- /dev/null
+++ b/core/x86/x86.asm
@@ -0,0 +1,1724 @@
+/* **********************************************************
+ * Copyright (c) 2001-2009 VMware, Inc.  All rights reserved.
+ * ********************************************************** */
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2001 Hewlett-Packard Company */
+
+/*
+ * x86.asm - x86 specific assembly and trampoline code
+ *
+ * This file is used for both linux and windows.
+ * We used to use the gnu assembler on both platforms, but
+ * gas does not support 64-bit windows.
+ * Thus we now use masm on windows and gas with the new intel-syntax-specifying
+ * options so that our code here only needs a minimum of macros to
+ * work on both.
+ *
+ * Note that for gas on cygwin we used to need to prepend _ to global
+ * symbols: we don't need that for linux gas or masm so we don't do it anymore.
+ */
+
+/* We handle different registers and calling conventions with a CPP pass.
+ * It can be difficult to choose registers that work across all ABI's we're
+ * trying to support: we need to move each ARG into a register in case
+ * it is passed in memory, but we have to pick registers that don't already
+ * hold other arguments.  Typically, use this order:
+ *   REG_XAX, REG_XBX, REG_XDI, REG_XSI, REG_XDX, REG_XCX
+ * Note that REG_XBX is by convention used on linux for PIC base: if we want 
+ * to try and avoid relocations (case 7852) we should avoid using it 
+ * to avoid confusion (though we can always pick a different register, 
+ * even varying by function).
+ * FIXME: should we use virtual registers instead?
+ * FIXME: should we have ARG1_IN_REG macro that is either nop or load from stack?
+ * For now not bothering, but if we add more routines we'll want more support.
+ * Naturally the ARG* macros are only valid at function entry.
+ */
+
+#include "asm_defines.asm"
+START_FILE
+
+#define RESTORE_FROM_DCONTEXT_VIA_REG(reg,offs,dest) mov dest, PTRSZ [offs + reg]
+#define SAVE_TO_DCONTEXT_VIA_REG(reg,offs,src) mov PTRSZ [offs + reg], src
+        
+/* For the few remaining dcontext_t offsets we need here: */
+#ifdef WINDOWS
+# ifdef X64
+#  define UPCXT_EXTRA 8 /* at_syscall + 4 bytes of padding */
+# else
+#  define UPCXT_EXTRA 4 /* at_syscall */
+# endif
+#else
+# define UPCXT_EXTRA 8 /* errno + at_syscall */
+#endif
+
+/* We should give asm_defines.asm all unique names and then include globals.h
+ * and avoid all this duplication!
+ */
+#ifdef X64
+# ifdef WINDOWS
+#  define NUM_XMM_SLOTS 6 /* xmm0-5 */
+# else
+#  define NUM_XMM_SLOTS 16 /* xmm0-15 */
+# endif
+#else
+# define NUM_XMM_SLOTS 8 /* xmm0-7 */
+#endif
+#define XMM_SAVED_SIZE ((NUM_XMM_SLOTS)*16) /* xmm0-5/15 for PR 264138/PR 302107 */
+
+/* Should we generate all of our asm code instead of having it static?
+ * As it is we're duplicating insert_push_all_registers(), dr_insert_call(), etc.,
+ * but it's not that much code here in these macros, and this is simpler
+ * than emit_utils.c-style code.
+ */
+#ifdef X64
+/* push all registers in dr_mcontext_t order.  does NOT make xsp have a
+ * pre-push value as no callers need that (they all use PUSH_DR_MCONTEXT).
+ * Leaves space for, but does NOT fill in, the xmm0-5 slots (PR 264138),
+ * since it's hard to dynamically figure out during bootstrapping whether
+ * movdqu or movups are legal instructions.  The caller is expected
+ * to fill in the xmm values prior to any calls that may clobber them.
+ */
+# define PUSHALL \
+        lea      rsp, [rsp - XMM_SAVED_SIZE] @N@\
+        push     r15 @N@\
+        push     r14 @N@\
+        push     r13 @N@\
+        push     r12 @N@\
+        push     r11 @N@\
+        push     r10 @N@\
+        push     r9  @N@\
+        push     r8  @N@\
+        push     rax @N@\
+        push     rcx @N@\
+        push     rdx @N@\
+        push     rbx @N@\
+        /* not the pusha pre-push rsp value but see above */ @N@\
+        push     rsp @N@\
+        push     rbp @N@\
+        push     rsi @N@\
+        push     rdi
+# define POPALL        \
+        pop      rdi @N@\
+        pop      rsi @N@\
+        pop      rbp @N@\
+        pop      rbx /* rsp into dead rbx */ @N@\
+        pop      rbx @N@\
+        pop      rdx @N@\
+        pop      rcx @N@\
+        pop      rax @N@\
+        pop      r8  @N@\
+        pop      r9  @N@\
+        pop      r10 @N@\
+        pop      r11 @N@\
+        pop      r12 @N@\
+        pop      r13 @N@\
+        pop      r14 @N@\
+        pop      r15 @N@\
+        lea      rsp, [rsp + XMM_SAVED_SIZE]
+# define DR_MCONTEXT_SIZE (18*ARG_SZ + XMM_SAVED_SIZE)
+# define dstack_OFFSET     (DR_MCONTEXT_SIZE+UPCXT_EXTRA+3*ARG_SZ)
+#else
+# define PUSHALL \
+        lea      esp, [esp - XMM_SAVED_SIZE] @N@\
+        pusha
+# define POPALL  \
+        popa @N@\
+        lea      esp, [esp + XMM_SAVED_SIZE]
+# define DR_MCONTEXT_SIZE (10*ARG_SZ + XMM_SAVED_SIZE)
+# define dstack_OFFSET     (DR_MCONTEXT_SIZE+UPCXT_EXTRA+3*ARG_SZ)
+#endif
+#define is_exiting_OFFSET (dstack_OFFSET+3*ARG_SZ+4)
+#define PUSHALL_XSP_OFFS  (3*ARG_SZ)
+#define MCONTEXT_XSP_OFFS (PUSHALL_XSP_OFFS)
+#define MCONTEXT_PC_OFFS  (DR_MCONTEXT_SIZE-ARG_SZ)
+
+/* Pushes a dr_mcontext_t on the stack, with an xsp value equal to the
+ * xsp before the pushing.  Clobbers xax!
+ * Does fill in xmm0-5, if necessary, for PR 264138.
+ * Assumes that DR has been initialized (get_xmm_vals() checks proc feature bits).
+ * Caller should ensure 16-byte stack alignment prior to the push (PR 306421).
+ */
+#define PUSH_DR_MCONTEXT(pc)                              \
+        push     pc                                    @N@\
+        PUSHF                                          @N@\
+        PUSHALL                                        @N@\
+        lea      REG_XAX, [REG_XSP]                    @N@\
+        CALLC1(get_xmm_vals, REG_XAX)                  @N@\
+        lea      REG_XAX, [DR_MCONTEXT_SIZE + REG_XSP] @N@\
+        mov      [PUSHALL_XSP_OFFS + REG_XSP], REG_XAX
+
+
+/****************************************************************************/
+/****************************************************************************/
+#ifndef NOT_DYNAMORIO_CORE_PROPER
+
+DECL_EXTERN(get_own_context_integer_control)
+DECL_EXTERN(get_xmm_vals)
+DECL_EXTERN(auto_setup)
+DECL_EXTERN(back_from_native_C)
+DECL_EXTERN(dispatch)
+#ifdef DR_APP_EXPORTS
+DECL_EXTERN(dr_app_start_helper)
+#endif
+DECL_EXTERN(dynamo_process_exit)
+DECL_EXTERN(dynamo_thread_exit)
+DECL_EXTERN(dynamo_thread_stack_free_and_exit)
+DECL_EXTERN(dynamorio_app_take_over_helper)
+DECL_EXTERN(found_modified_code)
+DECL_EXTERN(get_cleanup_and_terminate_global_do_syscall_entry)
+#ifdef INTERNAL
+DECL_EXTERN(internal_error)
+#endif
+DECL_EXTERN(internal_exception_info)
+DECL_EXTERN(is_currently_on_dstack)
+DECL_EXTERN(nt_continue_setup)
+#if defined(LINUX) && defined(X64)
+DECL_EXTERN(master_signal_handler_C)
+#endif
+
+/* non-functions: these make us non-PIC! (PR 212290) */
+DECL_EXTERN(exiting_thread_count)
+DECL_EXTERN(initstack)
+DECL_EXTERN(initstack_mutex)
+DECL_EXTERN(int_syscall_address)
+DECL_EXTERN(syscalls)
+DECL_EXTERN(sysenter_ret_address)
+DECL_EXTERN(sysenter_tls_offset)
+#ifdef WINDOWS
+DECL_EXTERN(wow64_index)
+# ifdef X64
+DECL_EXTERN(syscall_argsz)
+# endif
+#endif
+        
+
+#ifdef WINDOWS
+/* dynamo_auto_start: used for non-early follow children.
+ * Assumptions: The saved dr_mcontext_t for the start of the app is on
+ * the stack, followed by a pointer to a region of memory to free
+ * (which can be NULL) and its size.  This routine is reached by a jmp
+ * so be aware of that for address calculation. This routine does
+ * not return.
+ *
+ * On win32, note that in order to export this from the dynamo dll, which is
+ * required for non early follow children, we have to explicitly tell the
+ * linker to do so.  This is done in the Makefile.
+ * Note that if it weren't for wanting local go-native code we would have
+ * auto_setup in x86_code.c be dynamo_auto_start.
+ */
+        DECLARE_FUNC(dynamo_auto_start)
+GLOBAL_LABEL(dynamo_auto_start:)
+        /* we pass a pointer to TOS as a parameter.
+         * a param in xsp won't work w/ win64 padding so put in xax */
+        mov      REG_XAX, REG_XSP
+        CALLC1(auto_setup, REG_XAX)
+        /* if auto_setup returns, we need to go native */
+        jmp      load_dynamo_failure
+        END_FUNC(dynamo_auto_start)
+#endif
+
+#ifdef LINUX
+/* We avoid performance problems with messing up the RSB by using
+ * a separate routine.  The caller needs to use a plain call
+ * with _GLOBAL_OFFSET_TABLE_ on the exact return address instruction.
+ */
+        DECLARE_FUNC(get_pic_xax)
+GLOBAL_LABEL(get_pic_xax:)
+        mov      REG_XAX, PTRSZ [REG_XSP]
+        ret
+        END_FUNC(get_pic_xax)
+
+        DECLARE_FUNC(get_pic_xdi)
+GLOBAL_LABEL(get_pic_xdi:)
+        mov      REG_XDI, PTRSZ [REG_XSP]
+        ret
+        END_FUNC(get_pic_xdi)
+#endif
+
+/*
+ * Calls the specified function 'func' after switching to the stack 'stack'.  If we're
+ * currently on the initstack 'free_initstack' should be set so we release the
+ * initstack lock.  The supplied 'dcontext' will be passed as an argument to 'func'.
+ * If 'func' returns then 'return_on_return' is checked. If set we swap back stacks and
+ * return to the caller.  If not set then it's assumed that func wasn't supposed to
+ * return and we go to an error routine unexpected_return() below.
+ *
+ * void call_switch_stack(dcontext_t *dcontext,       // 1*ARG_SZ+XAX
+ *                        byte *stack,                // 2*ARG_SZ+XAX
+ *                        void (*func)(dcontext_t *), // 3*ARG_SZ+XAX
+ *                        bool free_initstack,        // 4*ARG_SZ+XAX
+ *                        bool return_on_return)      // 5*ARG_SZ+XAX
+ */
+        DECLARE_FUNC(call_switch_stack)
+GLOBAL_LABEL(call_switch_stack:)
+        /* get all args with same offset(xax) regardless of plaform */
+#ifdef X64
+# ifdef WINDOWS
+        mov      REG_XAX, REG_XSP
+        lea      REG_XSP, [-ARG_SZ + REG_XSP] /* maintain align-16: offset retaddr */
+# else
+        /* no padding so we make our own space. odd #slots keeps align-16 w/ retaddr */
+        lea      REG_XSP, [-5*ARG_SZ + REG_XSP]
+        /* xax points one beyond TOS to get same offset as having retaddr there */
+        lea      REG_XAX, [-ARG_SZ + REG_XSP]
+        mov      [5*ARG_SZ + REG_XAX], ARG5
+# endif
+        mov      [1*ARG_SZ + REG_XAX], ARG1
+        mov      [2*ARG_SZ + REG_XAX], ARG2
+        mov      [3*ARG_SZ + REG_XAX], ARG3
+        mov      [4*ARG_SZ + REG_XAX], ARG4
+#else
+        mov      REG_XAX, REG_XSP
+#endif
+        /* we need a callee-saved reg across our call so save it onto stack */
+        push     REG_XBX
+        push     REG_XDI /* still 16-aligned */
+        mov      REG_XBX, REG_XAX
+        mov      REG_XDI, REG_XSP
+        /* set up for call */
+        mov      REG_XDX, [3*ARG_SZ + REG_XAX] /* func */
+        mov      REG_XCX, [1*ARG_SZ + REG_XAX] /* dcontext */
+        mov      REG_XSP, [2*ARG_SZ + REG_XAX] /* stack */
+        cmp      DWORD [4*ARG_SZ + REG_XAX], 0 /* free_initstack */
+        je       call_dispatch_alt_stack_no_free
+#if !defined(X64) && defined(LINUX)
+        /* PR 212290: avoid text relocations: get PIC base into xax
+         * Can't use CALLC0 since it inserts a nop: we need the exact retaddr.
+         */
+        call     get_pic_xax
+	lea      REG_XAX, [_GLOBAL_OFFSET_TABLE_ + REG_XAX]
+	lea      REG_XAX, VAR_VIA_GOT(REG_XAX, initstack_mutex)
+        mov      DWORD [REG_XAX], 0
+#else
+        mov      DWORD SYMREF(initstack_mutex), 0 /* rip-relative on x64 */
+#endif
+call_dispatch_alt_stack_no_free:
+        CALLC1(REG_XDX, REG_XCX)
+        mov      REG_XSP, REG_XDI
+        mov      REG_XAX, REG_XBX
+        cmp      DWORD [5*ARG_SZ + REG_XAX], 0 /* return_on_return */
+        je       unexpected_return
+        pop      REG_XDI
+        pop      REG_XBX
+#ifdef X64
+# ifdef WINDOWS
+        mov      REG_XSP, REG_XAX
+# else
+        lea      REG_XSP, [5*ARG_SZ + REG_XSP]
+# endif
+#else
+        mov      REG_XSP, REG_XAX
+#endif
+        ret
+        END_FUNC(call_switch_stack)
+
+/*
+ * For debugging: report an error if the function called by call_switch_stack()
+ * unexpectedly returns.
+ */
+        DECLARE_FUNC(unexpected_return)
+GLOBAL_LABEL(unexpected_return:)
+#ifdef INTERNAL
+        CALLC3(internal_error, 0, -99 /* line # */, 0)
+        /* internal_error never returns */
+#endif
+        /* infinite loop is intentional: FIXME: do better in release build!
+         * FIXME - why not an int3? */
+        jmp      unexpected_return
+        END_FUNC(unexpected_return)
+
+/*
+ * dr_app_start - Causes application to run under Dynamo control 
+ */
+#ifdef DR_APP_EXPORTS
+        DECLARE_EXPORTED_FUNC(dr_app_start)
+GLOBAL_LABEL(dr_app_start:)
+
+        /* grab exec state and pass as param in a dr_mcontext_t struct */
+        PUSH_DR_MCONTEXT([REG_XSP])  /* push return address as pc */
+
+        /* do the rest in C */
+        lea      REG_XAX, [REG_XSP] /* stack grew down, so dr_mcontext_t at tos */
+        CALLC1(dr_app_start_helper, REG_XAX)
+
+        /* if we come back, then DR is not taking control so 
+         * clean up stack and return */
+        add      REG_XSP, DR_MCONTEXT_SIZE
+        ret
+        END_FUNC(dr_app_start)
+
+/*
+ * dr_app_take_over - For the client interface, we'll export 'dr_app_take_over'
+ * for consistency with the dr_ naming convention of all exported functions.  
+ * We'll keep 'dynamorio_app_take_over' for compatibility with the preinjector.
+ */
+        DECLARE_EXPORTED_FUNC(dr_app_take_over)
+GLOBAL_LABEL(dr_app_take_over:  )
+        jmp      dynamorio_app_take_over 
+        END_FUNC(dr_app_take_over)
+#endif
+                
+/*
+ * dynamorio_app_take_over - Causes application to run under Dynamo
+ * control.  Dynamo never releases control.
+ */
+        DECLARE_EXPORTED_FUNC(dynamorio_app_take_over)
+GLOBAL_LABEL(dynamorio_app_take_over:)
+
+        /* grab exec state and pass as param in a dr_mcontext_t struct */
+        PUSH_DR_MCONTEXT([REG_XSP])  /* push return address as pc */
+
+        /* do the rest in C */
+        lea      REG_XAX, [REG_XSP] /* stack grew down, so dr_mcontext_t at tos */
+        CALLC1(dynamorio_app_take_over_helper, REG_XAX)
+
+        /* if we come back, then DR is not taking control so 
+         * clean up stack and return */
+        add      REG_XSP, DR_MCONTEXT_SIZE
+        ret
+        END_FUNC(dynamorio_app_take_over)
+        
+/*
+ * cleanup_and_terminate(dcontext_t *dcontext,     // 1*ARG_SZ+XBP
+ *                       int sysnum,               // 2*ARG_SZ+XBP = syscall #
+ *                       int sys_arg1/param_base,  // 3*ARG_SZ+XBP = arg1 for syscall
+ *                       int sys_arg2,             // 4*ARG_SZ+XBP = arg2 for syscall
+ *                       bool exitproc)            // 5*ARG_SZ+XBP
+ *
+ * Calls dynamo_exit_process if exitproc is true, else calls dynamo_exit_thread.
+ * Uses the current dstack, but instructs the cleanup routines not to
+ * de-allocate it, does a custom de-allocate after swapping to initstack (don't
+ * want to use initstack the whole time, that's too long to hold the mutex).
+ * Then calls system call sysnum with parameter base param_base, which is presumed
+ * to be either NtTerminateThread or NtTerminateProcess or exit.
+ * For x64 Windows, args are in ecx and edx (terminate syscalls have only 2 args).
+ * For x64 Linux, 1st 2 args are in rdi and rsi.
+ *
+ * Note that the caller is responsible for placing the actual syscall arguments
+ * at the correct offset from edx (or ebx).  See SYSCALL_PARAM_OFFSET in
+ * win32 os.c for more info.
+ *
+ * Note that this routine does not return and thus clobbers callee-saved regs.
+ */
+        DECLARE_FUNC(cleanup_and_terminate)
+GLOBAL_LABEL(cleanup_and_terminate:)
+        /* get all args with same offset(xbp) regardless of plaform, to save
+         * across our calls.
+         */
+#ifdef X64
+# ifdef WINDOWS
+        mov      REG_XBP, REG_XSP
+        lea      REG_XSP, [-ARG_SZ + REG_XSP] /* maintain align-16: offset retaddr */
+# else
+        /* no padding so we make our own space. odd #slots keeps align-16 w/ retaddr */
+        lea      REG_XSP, [-5*ARG_SZ + REG_XSP]
+        /* xbp points one beyond TOS to get same offset as having retaddr there */
+        lea      REG_XBP, [-ARG_SZ + REG_XSP]
+        mov      [5*ARG_SZ + REG_XBP], ARG5
+# endif
+        mov      [1*ARG_SZ + REG_XBP], ARG1
+        mov      [2*ARG_SZ + REG_XBP], ARG2
+        mov      [3*ARG_SZ + REG_XBP], ARG3
+        mov      [4*ARG_SZ + REG_XBP], ARG4
+#else
+        mov      REG_XBP, REG_XSP
+#endif
+        /* increment exiting_thread_count so that we don't get killed after 
+         * thread_exit removes us from the all_threads list */
+#if !defined(X64) && defined(LINUX)
+        /* PR 212290: avoid text relocations: get PIC base into callee-saved xdi.
+         * Can't use CALLC0 since it inserts a nop: we need the exact retaddr.
+         */
+        call     get_pic_xdi
+	lea      REG_XDI, [_GLOBAL_OFFSET_TABLE_ + REG_XDI]
+	lea      REG_XAX, VAR_VIA_GOT(REG_XDI, exiting_thread_count)
+        lock inc DWORD [REG_XAX]
+#else
+        lock inc DWORD SYMREF(exiting_thread_count) /* rip-rel for x64 */
+#endif
+        /* save dcontext->dstack for freeing later and set dcontext->is_exiting */
+        mov      REG_XBX, ARG1 /* xbx is callee-saved and not an x64 param */
+        SAVE_TO_DCONTEXT_VIA_REG(REG_XBX,is_exiting_OFFSET,1)
+        CALLC1(is_currently_on_dstack, REG_XBX) /* xbx is callee-saved */
+        cmp      REG_XAX, 0
+        jnz      cat_save_dstack
+        mov      REG_XBX, 0 /* save 0 for dstack to avoid double-free */
+        jmp      cat_done_saving_dstack
+cat_save_dstack:
+        RESTORE_FROM_DCONTEXT_VIA_REG(REG_XBX,dstack_OFFSET,REG_XBX)
+cat_done_saving_dstack:
+        /* PR 306421: xbx is callee-saved for all platforms, so don't push yet,
+         * to maintain 16-byte stack alignment
+         */
+        /* avoid sygate sysenter version as our stack may be static const at 
+         * that point, caller will take care of sygate hack */
+        CALLC0(get_cleanup_and_terminate_global_do_syscall_entry)
+        push     REG_XBX /* 16-byte aligned again */
+        push     REG_XAX
+        mov      REG_XSI, [5*ARG_SZ + REG_XBP] /* exitproc */
+        cmp      REG_XSI, 0
+        jz       cat_thread_only
+        CALLC0(dynamo_process_exit)
+        jmp      cat_no_thread
+cat_thread_only:
+        CALLC0(dynamo_thread_exit)
+cat_no_thread:
+        /* now switch to initstack for cleanup of dstack 
+         * could use initstack for whole thing but that's too long 
+         * of a time to hold global initstack_mutex */
+        mov      ecx, 1
+#if !defined(X64) && defined(LINUX)
+        /* PIC base is still in xdi */
+	lea      REG_XAX, VAR_VIA_GOT(REG_XDI, initstack_mutex)
+#endif
+cat_spin:       
+#if !defined(X64) && defined(LINUX)
+        xchg     DWORD [REG_XAX], ecx
+#else
+        xchg     DWORD SYMREF(initstack_mutex), ecx /* rip-relative on x64 */
+#endif
+        jecxz    cat_have_lock
+        /* try again -- too few free regs to call sleep() */
+        pause    /* good thing gas now knows about pause */
+        jmp      cat_spin
+cat_have_lock:
+        /* need to grab everything off dstack first */
+        mov      REG_XSI, [2*ARG_SZ + REG_XBP]  /* sysnum */
+        pop      REG_XAX             /* syscall */
+        pop      REG_XCX             /* dstack */
+        mov      REG_XBX, [3*ARG_SZ + REG_XBP] /* sys_arg1 */
+        mov      REG_XDX, [4*ARG_SZ + REG_XBP] /* sys_arg2 */
+        /* swap stacks */
+#if !defined(X64) && defined(LINUX)
+        /* PIC base is still in xdi */
+	lea      REG_XBP, VAR_VIA_GOT(REG_XDI, initstack)
+        mov      REG_XSP, PTRSZ [REG_XBP]
+#else
+        mov      REG_XSP, PTRSZ SYMREF(initstack) /* rip-relative on x64 */
+#endif
+        /* now save registers */
+        push     REG_XDX   /* sys_arg2 */
+        push     REG_XBX   /* sys_arg1 */
+        push     REG_XAX   /* syscall */
+        push     REG_XSI   /* sysnum => xsp 16-byte aligned */
+        /* free dstack and call the EXIT_DR_HOOK */
+        CALLC1(dynamo_thread_stack_free_and_exit, REG_XCX) /* pass dstack */
+        /* finally, execute the termination syscall */
+        pop      REG_XAX   /* sysnum */
+#ifdef X64
+        /* We assume we're doing "syscall" on Windows & Linux, where r10 is dead */
+        pop      r10       /* syscall, in reg dead at syscall */
+# ifdef LINUX
+        pop      REG_XDI   /* sys_arg1 */
+        pop      REG_XSI   /* sys_arg2 */
+# else
+        pop      REG_XCX   /* sys_arg1 */
+        pop      REG_XDX   /* sys_arg2 */
+# endif
+#else
+        pop      REG_XSI   /* syscall */
+# ifdef LINUX
+        pop      REG_XBX   /* sys_arg1 */
+        pop      REG_XCX   /* sys_arg2 */
+# else
+        pop      REG_XDX   /* sys_arg1 == param_base */
+        /* sys_arg2 unused */
+# endif
+#endif
+        /* give up initstack mutex -- potential problem here with a thread getting 
+         *   an asynch event that then uses initstack, but syscall should only care 
+         *   about ebx and edx */
+#if !defined(X64) && defined(LINUX)
+        /* PIC base is still in xdi */
+	lea      REG_XBP, VAR_VIA_GOT(REG_XDI, initstack_mutex)
+        mov      DWORD [REG_XBP], 0
+#else
+        mov      DWORD SYMREF(initstack_mutex), 0 /* rip-relative on x64 */
+#endif
+        /* we are finished with all shared resources, decrement the  
+         * exiting_thread_count (allows another thread to kill us) */
+#if !defined(X64) && defined(LINUX)
+        /* PIC base is still in xdi */
+	lea      REG_XBP, VAR_VIA_GOT(REG_XDI, exiting_thread_count)
+        lock dec DWORD [REG_XBP]
+#else
+        lock dec DWORD SYMREF(exiting_thread_count) /* rip-rel on x64 */
+#endif
+#ifdef X64
+        jmp      r10      /* go do the syscall! */
+#else
+        jmp      REG_XSI  /* go do the syscall! */
+#endif
+        END_FUNC(cleanup_and_terminate)
+
+/*
+ * cleanup_and_terminate_client_thread(
+ *                       int sysnum,               // 1*ARG_SZ+XBP = syscall #
+ *                       byte *stack_base,         // 2*ARG_SZ+XBP =base of stack to free
+ *                       int sys_arg1/param_base,  // 3*ARG_SZ+XBP = arg1 for syscall
+ *                       int sys_arg2)              // 4*ARG_SZ+XBP = arg2 for syscall
+ *
+ * Swaps to initstack, calls nt_remote_free_virtual_memory on
+ * [stack_base, stack_base+stack_size), and then calls system call sysnum
+ * with parameter base param_base.
+ *
+ * Note that the caller is responsible for placing the actual syscall arguments
+ * at the correct offset from edx (or ebx).  See SYSCALL_PARAM_OFFSET in
+ * win32 os.c for more info.
+ * Assumes the caller already incremented exiting_thread_count and
+ * removed the thread from all_threads_list.
+ */
+#ifdef CLIENT_SIDELINE /* PR 222812 */
+#ifdef WINDOWS /* we hardcoded nt_free_virtual_memory for now */
+        DECLARE_FUNC(cleanup_and_terminate_client_thread)
+GLOBAL_LABEL(cleanup_and_terminate_client_thread:)
+        mov      REG_XSP, REG_XBP
+#ifdef X64
+        lea      REG_XSP, [-ARG_SZ + REG_XSP] /* maintain align-16: undo retaddr */
+        /* param regs are caller-saved so we must save to stack here */
+# ifdef LINUX
+        /* no padding so we make our own space */
+        lea      REG_XSP, [-4*ARG_SZ + REG_XSP]
+# endif
+        mov      [1*ARG_SZ + REG_XBP], ARG1
+        mov      [2*ARG_SZ + REG_XBP], ARG2
+        mov      [3*ARG_SZ + REG_XBP], ARG3
+        mov      [4*ARG_SZ + REG_XBP], ARG4
+#endif
+        /* switch to initstack for cleanup of current stack */
+        mov      ecx, 1
+catct_spin:       
+        xchg     ecx, DWORD SYMREF(initstack_mutex) /* rip-relative on x64 */
+        jecxz    catct_have_lock
+        /* try again -- too few free regs to call sleep() */
+        pause
+        jmp      catct_spin
+catct_have_lock:
+        /* swap stacks */
+        mov      REG_XSP, DWORD SYMREF(initstack) /* rip-relative on x64 */
+        /* save what we need for termination syscall */
+        push     [4*ARG_SZ + REG_XBP] /* sys_arg2 */
+        push     [3*ARG_SZ + REG_XBP] /* sys_arg1 => 16-byte aligned */
+        /* avoid sygate sysenter version as our stack may be static const at 
+         * that point, caller will take care of sygate hack */
+        CALLC0(get_cleanup_and_terminate_global_do_syscall_entry)
+        push     REG_XAX
+        push     [1*ARG_SZ + REG_XBP]  /* sysnum => 16-byte aligned */
+        /* make call to nt_remote_free_virtual_memory */
+        mov      REG_XAX, [2*ARG_SZ + REG_XBP]  /* stack_base */
+        CALLC1(nt_free_virtual_memory, REG_XAX)
+        /* finally, execute the termination syscall */
+        pop      REG_XAX   /* sysnum */
+#ifdef X64
+        /* We assume we're doing "syscall" on Windows & Linux, where r10 is dead */
+        pop      r10       /* syscall, in reg dead at syscall */
+# ifdef LINUX
+        pop      REG_XDI   /* sys_arg1 */
+        pop      REG_XSI   /* sys_arg2 */
+# else
+        pop      REG_XCX   /* sys_arg1 */
+        pop      REG_XDX   /* sys_arg2 */
+# endif
+#else
+        pop      REG_XSI   /* syscall */
+# ifdef LINUX
+        pop      REG_XBX   /* sys_arg1 */
+        pop      REG_XCX   /* sys_arg2 */
+# else
+        pop      REG_XDX   /* sys_arg1 == param_base */
+        /* sys_arg2 unused */
+# endif
+#endif
+        /* give up initstack mutex -- potential problem here with a thread getting 
+         *   an asynch event that then uses initstack, but syscall should only care 
+         *   about ebx and edx */
+        mov      DWORD SYMREF(initstack_mutex), 0 /* rip-relative on x64 */
+        /* we are finished with all shared resources, decrement the  
+         * exiting_thread_count (allows another thread to kill us) */
+        lock dec DWORD SYMREF(exiting_thread_count) /* rip-relative on x64 */
+#ifdef X64
+        jmp      r10      /* go do the syscall! */
+#else
+        jmp      REG_XSI  /* go do the syscall! */
+#endif
+        END_FUNC(cleanup_and_terminate_client_thread)
+#endif /* WINDOWS */
+#endif /* CLIENT_SIDELINE */
+
+/* global_do_syscall_int
+ * Caller is responsible for all set up.  For windows this means putting the
+ * syscall num in eax and putting the args at edx.  For linux this means putting
+ * the syscall num in eax, and the args in ebx, ecx, edx, esi, edi and ebp (in
+ * that order, as needed).  global_do_syscall is only used with system calls
+ * that we don't expect to return, so for debug builds we go into an infinite
+ * loop if syscall returns.
+ */
+        DECLARE_FUNC(global_do_syscall_int)
+GLOBAL_LABEL(global_do_syscall_int:)
+#ifdef WINDOWS
+        int      HEX(2e)
+#else
+        int      HEX(80)
+#endif
+#ifdef DEBUG
+        jmp      debug_infinite_loop
+#endif
+        END_FUNC(global_do_syscall_int)
+
+/* For sygate hack need to indirect the system call through ntdll. */
+#ifdef WINDOWS
+        DECLARE_FUNC(global_do_syscall_sygate_int)
+GLOBAL_LABEL(global_do_syscall_sygate_int:)
+        /* would be nicer to call so we could return to debug_infinite_loop on 
+         * failure, but on some paths (cleanup_and_terminate) we can no longer 
+         * safetly use the stack */
+        jmp      PTRSZ SYMREF(int_syscall_address)
+        END_FUNC(global_do_syscall_sygate_int)
+#endif
+
+/* global_do_syscall_sysenter
+ * Caller is responsible for all set up, this means putting the syscall num
+ * in eax and putting the args at edx+8 (windows specific, we don't yet support
+ * linux sysenter).  global_do_syscall is only used with system calls that we
+ * don't expect to return.  As edx becomes esp, if the syscall does return it
+ * will go to the address in [edx] (again windows specific) (if any debugging
+ * code is desired should be pointed to there, do note that edx will become esp
+ * so be aware of stack limitations/protections).
+ */
+        DECLARE_FUNC(global_do_syscall_sysenter)
+GLOBAL_LABEL(global_do_syscall_sysenter:)
+#if defined(X64) && defined(WINDOWS)
+        syscall  /* FIXME ml64 won't take "sysenter" so half-fixing now */
+#else
+        sysenter
+#endif
+#ifdef DEBUG
+        /* We'll never ever reach here, sysenter won't/can't return to this 
+         * address since it doesn't know it, but we'll put in a jmp to 
+         * debug_infinite_loop just in case */
+        jmp      debug_infinite_loop
+#endif
+        END_FUNC(global_do_syscall_sysenter)
+
+/* Sygate case 5441 hack - the return address (edx) needs to point to
+ * ntdll to pass their verification.  Global_do_syscall is really only
+ * used with system calls that aren't expected to return so we don't
+ * have to be too careful.  Just shuffle the stack using the sysret addr.
+ * If there is already a return address we'll keep that (just move down
+ * a slot).
+ */
+#ifdef WINDOWS
+        DECLARE_FUNC(global_do_syscall_sygate_sysenter)
+GLOBAL_LABEL(global_do_syscall_sygate_sysenter:)
+        mov      REG_XSP, REG_XDX
+        /* move existing ret down a slot (note target address is 
+         * computed with already inc'ed esp [see intel docs]) */
+        pop      PTRSZ [REG_XSP]
+        push     PTRSZ SYMREF(sysenter_ret_address)
+#if defined(X64) && defined(WINDOWS)
+        syscall  /* FIXME ml64 won't take "sysenter" so half-fixing now */
+#else
+        sysenter
+#endif
+#ifdef DEBUG
+        /* We'll never ever reach here, sysenter won't/can't return to this 
+         * address since it doesn't know it, but we'll put in a jmp to 
+         * debug_infinite_loop just in case */
+        jmp      debug_infinite_loop
+#endif
+        END_FUNC(global_do_syscall_sygate_sysenter)
+#endif
+
+/* Both Windows and Linux put rcx into r10 since rcx is used as the return addr */
+#ifdef X64
+/* global_do_syscall_syscall
+ * Caller is responsible for all set up: putting the syscall num in eax
+ * and the args in registers/memory.  Only used with system calls
+ * that we don't expect to return, so for debug builds we go into an infinite
+ * loop if syscall returns.
+  */
+       DECLARE_FUNC(global_do_syscall_syscall)
+GLOBAL_LABEL(global_do_syscall_syscall:)
+        mov      r10, REG_XCX
+        syscall
+#   ifdef DEBUG
+        jmp      debug_infinite_loop
+#   endif
+        END_FUNC(global_do_syscall_syscall)
+#endif
+
+#ifdef WINDOWS
+
+/* global_do_syscall_wow64
+ * Xref case 3922
+ * Caller is responsible for all set up: putting the syscall num in eax,
+ * the wow64 index into ecx, and the args in edx.  Only used with system calls
+ * that we don't expect to return, so for debug builds we go into an infinite
+ * loop if syscall returns.
+  */
+       DECLARE_FUNC(global_do_syscall_wow64)
+GLOBAL_LABEL(global_do_syscall_wow64:)
+        call     PTRSZ SEGMEM(fs,HEX(0c0))
+#ifdef DEBUG
+        jmp      debug_infinite_loop
+#endif
+        END_FUNC(global_do_syscall_wow64)
+
+/* global_do_syscall_wow64_index0
+ * Sames as global_do_syscall_wow64, except zeros out ecx.
+ */
+        DECLARE_FUNC(global_do_syscall_wow64_index0)
+GLOBAL_LABEL(global_do_syscall_wow64_index0:)
+        xor      ecx, ecx
+        call     PTRSZ SEGMEM(fs,HEX(0c0))
+#ifdef DEBUG
+        jmp      debug_infinite_loop
+#endif
+        END_FUNC(global_do_syscall_wow64_index0)
+
+#endif /* WINDOWS */
+
+#ifdef DEBUG
+/* Just an infinite CPU eating loop used to mark certain failures.
+ */
+        DECLARE_FUNC(debug_infinite_loop)
+GLOBAL_LABEL(debug_infinite_loop:)
+        jmp      debug_infinite_loop
+        END_FUNC(debug_infinite_loop)
+#endif
+        
+#ifdef WINDOWS
+/* We use our own syscall wrapper for key win32 system calls.
+ *
+ * We would use a dynamically generated routine created by decoding
+ * a real ntdll wrapper and tweaking it, but we need to use
+ * this for our own syscalls and have a bootstrapping problem -- so 
+ * rather than hacking to get the power to decode w/o a heap, we hardcode
+ * the types we support here.
+ *
+ * We assume that all syscall wrappers are identical, and they have
+ * specific instruction sequences -- thus this routine needs to be updated
+ * with any syscall sequence change in a future version of ntdll.dll!
+ *
+ * We construct our own minimalist versions that use C calling convention
+ * and take as a first argument the system call number:
+ *
+ * ref case 5217, for Sygate compatibility the int needs to come from
+ * ntdll.dll, we use a call to NtYieldExecution+9 (int 2e; ret;)
+ *
+ * 1)   mov immed, eax        mov 4(esp), eax  
+ *      lea 4(esp), edx  ==>  lea 8(esp), edx  
+ *      int 2e                  int 2e            
+ *      ret 4*numargs          ret               
+ *
+ * 2)   mov immed, eax               mov 4(esp), eax          
+ *      mov 0x7ffe0300, edx          mov esp, edx
+ *      call {edx,(edx)}             < juggle stack, see below >
+ *         NOTE - to support the sygate case 5441 hack the actual instructions
+ *              - we use are different, but the end up doing the same thing
+ *    callee:                   ==>    sysenter                   
+ *        mov esp, edx             our_ret:
+ *        sysenter                     ret                    
+ *        ret                                         
+ *      ret 4*numargs
+ *
+ * => signature: dynamorio_syscall_{int2e,sysenter}(sysnum, arg1, arg2, ...)
+ */
+        DECLARE_FUNC(dynamorio_syscall_int2e)
+GLOBAL_LABEL(dynamorio_syscall_int2e:)
+        mov      eax, [4 + esp]
+        lea      edx, [8 + esp]
+        int      HEX(2e)
+        ret
+        END_FUNC(dynamorio_syscall_int2e)
+
+        DECLARE_FUNC(dynamorio_syscall_sygate_int2e)
+GLOBAL_LABEL(dynamorio_syscall_sygate_int2e:)
+        mov      eax, [4 + esp]
+        lea      edx, [8 + esp]
+        call     PTRSZ SYMREF(int_syscall_address)
+        ret
+        END_FUNC(dynamorio_syscall_sygate_int2e)
+        
+        DECLARE_FUNC(dynamorio_syscall_sysenter)
+GLOBAL_LABEL(dynamorio_syscall_sysenter:)
+        /* esp + 0    return address 
+         *       4    syscall num 
+         *       8+   syscall args 
+         * Ref case 5461 edx serves as both the argument pointer (edx+8) and the 
+         * top of stack for the kernel sysexit. */
+        mov      eax, [4 + esp]
+        mov      REG_XDX, REG_XSP
+#if defined(X64) && defined(WINDOWS)
+        syscall  /* FIXME ml64 won't take "sysenter" so half-fixing now */
+#else
+        sysenter
+#endif
+        /* Kernel sends control to hardcoded location, which does ret, 
+         * which will return directly back to the caller.  Thus the following 
+         * ret will never execute. */
+        ret
+        END_FUNC(dynamorio_syscall_sysenter)
+
+        DECLARE_GLOBAL(dynamorio_sysenter_fixup)
+        DECLARE_FUNC(dynamorio_syscall_sygate_sysenter)
+GLOBAL_LABEL(dynamorio_syscall_sygate_sysenter:)
+        /* stack looks like: 
+         * esp + 0    return address 
+         *       4    syscall num 
+         *       8+   syscall args 
+         * Ref case 5461 edx serves as both the argument pointer (edx+8) and the 
+         * top  of stack for the kernel sysexit. While we could do nothing and 
+         * just have the sysenter return straight back to the caller, we use 
+         * sysenter_ret_address indirection to support the Sygate compatibility 
+         * fix for case 5441 where steal a ret from ntdll.dll so need to mangle 
+         * our stack to look like 
+         * esp + 0    sysenter_ret_address 
+         *       4    dynamorio_sysenter_fixup 
+         *       8+   syscall args 
+         * sysenter_tls_slot    return address 
+         * before we do the edx <- esp 
+         * 
+         * NOTE - we can NOT just have 
+         * esp + 0    sysenter_ret_address 
+         *       4    return address 
+         *       8    args 
+         * as even though this will go the right place, the stack will be one 
+         * off on the return (debug builds with frame ptr are ok, but not 
+         * release).  We could roll our own custom calling convention for this 
+         * but would be a pain given how this function is called.  So we use a 
+         * tls slot to store the return address around the system call since 
+         * there isn't room on the stack, thus is not re-entrant, but neither is 
+         * dr and we don't make alertable system calls.  An alternate scheme 
+         * kept the return address off the top of the stack which works fine 
+         * (nothing alertable), but just seemed too risky. 
+         * FIXME - any perf impact from breaking hardware return predictor */
+        pop      REG_XDX
+        mov      eax, DWORD SYMREF(sysenter_tls_offset)
+        mov      SEGMEM(fs,eax), edx
+        pop      REG_XAX
+#ifdef X64
+        /* Can't push a 64-bit immed */
+        mov      REG_XCX, dynamorio_sysenter_fixup
+        push     REG_XCX
+#else
+        push     dynamorio_sysenter_fixup
+#endif
+        push     PTRSZ SYMREF(sysenter_ret_address)
+        mov      REG_XDX, REG_XSP
+#if defined(X64) && defined(WINDOWS)
+        syscall  /* FIXME ml64 won't take "sysenter" so half-fixing now */
+#else
+        sysenter
+#endif
+ADDRTAKEN_LABEL(dynamorio_sysenter_fixup:)
+        /* push whatever (was the slot for the eax arg) */
+        push     REG_XAX
+        /* ecx/edx should be dead here, just borrow one */
+        mov      edx, DWORD SYMREF(sysenter_tls_offset)
+        push     PTRSZ SEGMEM(fs,edx)
+        ret
+        END_FUNC(dynamorio_syscall_sygate_sysenter)
+
+# ifdef X64
+/* With the 1st 4 args in registers, we don't want the sysnum to shift them
+ * all as it's not easy to un-shift.  So, we put the 1st arg last, and 
+ * the SYS enum value first.  We use the syscall_argsz array to restore 
+ * the 1st arg.  Since the return value is never larger than 64 bits, we
+ * never have to worry about a hidden 1st arg that shifts the rest.
+ */
+        DECLARE_FUNC(dynamorio_syscall_syscall)
+GLOBAL_LABEL(dynamorio_syscall_syscall:)
+        mov      rax, QWORD SYMREF(syscalls)
+        /* the upper 32 bits are automatically zeroed */
+        mov      eax, DWORD [rax + ARG1*4] /* sysnum in rax */
+        mov      r10, syscall_argsz
+        /* the upper 32 bits are automatically zeroed */
+        mov      r10d, DWORD [r10 + ARG1*4] /* # args in r10 */
+        cmp      r10, 0
+        je       dynamorio_syscall_syscall_ready
+        cmp      r10, 1
+        je       dynamorio_syscall_syscall_1arg
+        cmp      r10, 2
+        je       dynamorio_syscall_syscall_2arg
+        cmp      r10, 3
+        je       dynamorio_syscall_syscall_3arg
+        /* else, >= 4 args, so pull from arg slot of (#args + 1) */
+        mov      ARG1, QWORD [rsp + r10*8 + 8]
+        jmp      dynamorio_syscall_syscall_ready
+dynamorio_syscall_syscall_1arg:
+        mov      ARG1, ARG2
+        jmp      dynamorio_syscall_syscall_ready
+dynamorio_syscall_syscall_2arg:
+        mov      ARG1, ARG3
+        jmp      dynamorio_syscall_syscall_ready
+dynamorio_syscall_syscall_3arg:
+        mov      ARG1, ARG4
+        /* fall-through */
+dynamorio_syscall_syscall_ready:
+        mov      r10, rcx /* put rcx in r10 just like Nt wrappers (syscall writes rcx) */
+        syscall
+        ret
+        END_FUNC(dynamorio_syscall_syscall)
+# endif
+
+/* For WOW64 (case 3922) the syscall wrappers call *teb->WOW32Reserved (==
+ * wow64cpu!X86SwitchTo64BitMode), which is a far jmp that switches to the
+ * 64-bit cs segment (0x33 selector).  They pass in ecx an index into
+ * a function table of argument conversion routines.
+ *        
+ * 3)   mov sysnum, eax
+ *      mov tableidx, ecx
+ *      call *fs:0xc0
+ *    callee:
+ *        jmp  0x33:wow64cpu!CpupReturnFromSimulatedCode
+ *      ret 4*numargs
+ *
+ * rather than taking in sysnum and tableidx, we take in sys_enum and
+ * look up the sysnum and tableidx to keep the same args as the other
+ * dynamorio_syscall_* routines
+ * => signature: dynamorio_syscall_wow64(sys_enum, arg1, arg2, ...)
+ */
+        DECLARE_FUNC(dynamorio_syscall_wow64)
+GLOBAL_LABEL(dynamorio_syscall_wow64:)
+        mov      eax, [4 + esp]
+        mov      edx, DWORD SYMREF(wow64_index)
+        mov      ecx, [edx + eax*4]
+        mov      edx, DWORD SYMREF(syscalls)
+        mov      eax, [edx + eax*4]
+        lea      edx, [8 + esp]
+        call     PTRSZ SEGMEM(fs,HEX(0c0))
+        ret
+        END_FUNC(dynamorio_syscall_wow64)
+      
+#else /* LINUX  */
+/* to avoid libc wrappers we roll our own syscall here
+ * hardcoded to use int 0x80 for 32-bit -- FIXME: use something like do_syscall
+ * and syscall for 64-bit.
+ * signature: dynamorio_syscall(sysnum, num_args, arg1, arg2, ...)
+ */
+        DECLARE_FUNC(dynamorio_syscall)
+GLOBAL_LABEL(dynamorio_syscall:)
+        /* x64 kernel doesn't clobber all the callee-saved registers */
+        push     REG_XBX
+# ifdef X64
+        /* reverse order so we don't clobber earlier args */
+        mov      REG_XBX, ARG2 /* put num_args where we can reference it longer */
+        mov      rax, ARG1 /* sysnum: only need eax, but need rax to use ARG1 (or movzx) */
+        cmp      REG_XBX, 0
+        je       syscall_ready
+        mov      ARG1, ARG3
+        cmp      REG_XBX, 1
+        je       syscall_ready
+        mov      ARG2, ARG4
+        cmp      REG_XBX, 2
+        je       syscall_ready
+        mov      ARG3, ARG5
+        cmp      REG_XBX, 3
+        je       syscall_ready
+        mov      ARG4, ARG6
+        cmp      REG_XBX, 4
+        je       syscall_ready
+        mov      ARG5, [2*ARG_SZ + REG_XSP] /* arg7: above xbx and retaddr */
+        cmp      REG_XBX, 5
+        je       syscall_ready
+        mov      ARG6, [3*ARG_SZ + REG_XSP] /* arg8: above arg7, xbx, retaddr */
+syscall_ready:
+        mov      r10, rcx
+        syscall
+# else
+        push     REG_XBP
+        push     REG_XSI
+        push     REG_XDI
+        /* add 16 to skip the 4 pushes 
+         * FIXME: rather than this dispatch, could have separate routines 
+         * for each #args, or could just blindly read upward on the stack. 
+         * for dispatch, if assume size of mov instr can do single ind jmp */
+        mov      ecx, [16+ 8 + esp] /* num_args */
+        cmp      ecx, 0
+        je       syscall_0args
+        cmp      ecx, 1
+        je       syscall_1args
+        cmp      ecx, 2
+        je       syscall_2args
+        cmp      ecx, 3
+        je       syscall_3args
+        cmp      ecx, 4
+        je       syscall_4args
+        cmp      ecx, 5
+        je       syscall_5args
+        mov      ebp, [16+32 + esp] /* arg6 */
+syscall_5args:
+        mov      edi, [16+28 + esp] /* arg5 */
+syscall_4args:
+        mov      esi, [16+24 + esp] /* arg4 */
+syscall_3args:
+        mov      edx, [16+20 + esp] /* arg3 */
+syscall_2args:
+        mov      ecx, [16+16 + esp] /* arg2 */
+syscall_1args:
+        mov      ebx, [16+12 + esp] /* arg1 */
+syscall_0args:
+        mov      eax, [16+ 4 + esp] /* sysnum */
+        /* PR 254280: we assume int$80 is ok even for LOL64 */
+        int      HEX(80)
+        pop      REG_XDI
+        pop      REG_XSI
+        pop      REG_XBP
+#endif
+        pop      REG_XBX
+        /* return val is in eax for us */
+        ret
+        END_FUNC(dynamorio_syscall)
+
+/* while with pre-2.6.9 kernels we were able to rely on the kernel's
+ * default sigreturn code sequence and be more platform independent,
+ * case 6700 necessitates having our own code, which for now, like
+ * dynamorio_syscall, hardcodes int 0x80
+ */
+        DECLARE_FUNC(dynamorio_sigreturn)
+GLOBAL_LABEL(dynamorio_sigreturn:)
+#ifdef X64
+        mov      eax, HEX(f)
+        mov      r10, rcx
+        syscall
+#else
+        mov      eax, HEX(ad)
+        /* PR 254280: we assume int$80 is ok even for LOL64 */
+        int      HEX(80)
+#endif
+        /* should not return.  if we somehow do,infinite loop is intentional.
+         * FIXME: do better in release build! FIXME - why not an int3? */
+        jmp      unexpected_return
+        END_FUNC(dynamorio_sigreturn)
+        
+#ifndef X64
+/* since our handler is rt, we have no source for the kernel's/libc's
+ * default non-rt sigreturn, so we set up our own.
+ */
+        DECLARE_FUNC(dynamorio_nonrt_sigreturn)
+GLOBAL_LABEL(dynamorio_nonrt_sigreturn:)
+        pop      eax /* I don't understand why */
+        mov      eax, HEX(77)
+        /* PR 254280: we assume int$80 is ok even for LOL64 */
+        int      HEX(80)
+        /* should not return.  if we somehow do,infinite loop is intentional.
+         * FIXME: do better in release build! FIXME - why not an int3? */
+        jmp      unexpected_return
+        END_FUNC(dynamorio_sigreturn)
+#endif
+        
+#ifdef X64
+/* PR 305020: for x64 we can't use args to get the original stack pointer,
+ * so we use a stub routine here that adds a 4th arg to our C routine:
+ *   master_signal_handler(int sig, siginfo_t *siginfo, kernel_ucontext_t *ucxt)
+ */
+        DECLARE_FUNC(master_signal_handler)
+GLOBAL_LABEL(master_signal_handler:)
+        mov      rcx, rsp /* pass as 4th arg */
+        jmp      master_signal_handler_C
+        /* master_signal_handler_C will do the ret */
+        END_FUNC(master_signal_handler)
+#endif
+
+#endif /* LINUX */
+
+
+#ifdef WINDOWS    
+/*
+ * nt_continue_dynamo_start -- invoked to give dynamo control over
+ * exception handler continuation (after a call to NtContinue).
+ * identical to internal_dynamo_start except it calls nt_continue_start_setup
+ * to get the real next pc, and has an esp adjustment at the start.
+ */
+        DECLARE_FUNC(nt_continue_dynamo_start)
+GLOBAL_LABEL(nt_continue_dynamo_start:)
+        /* assume valid esp  
+         * FIXME: this routine should really not assume esp */
+
+        /* grab exec state and pass as param in a dr_mcontext_t struct */
+        PUSH_DR_MCONTEXT(0 /* for dr_mcontext_t.pc */)
+        lea      REG_XAX, [REG_XSP] /* stack grew down, so dr_mcontext_t at tos */
+
+        /* Call nt_continue_setup passing the dr_mcontext_t.  It will 
+         * obtain and initialize this thread's dcontext pointer and
+         * begin execution with the passed-in state.
+         */
+        CALLC1(nt_continue_setup, REG_XAX)
+        /* should not return */
+        jmp      unexpected_return
+        END_FUNC(nt_continue_dynamo_start)
+#endif /* WINDOWS */
+
+
+/*
+ * back_from_native -- for taking control back after letting a module
+ * execute natively
+ * assumptions: app stack is valid
+ */
+        DECLARE_FUNC(back_from_native)
+GLOBAL_LABEL(back_from_native:)
+        /* assume valid esp  
+         * FIXME: more robust if don't use app's esp -- should use initstack
+         */
+        /* grab exec state and pass as param in a dr_mcontext_t struct */
+        PUSH_DR_MCONTEXT(0 /* for dr_mcontext_t.pc */)
+        lea      REG_XAX, [REG_XSP] /* stack grew down, so dr_mcontext_t at tos */
+
+        /* Call back_from_native_C passing the dr_mcontext_t.  It will 
+         * obtain this thread's dcontext pointer and
+         * begin execution with the passed-in state.
+         */
+        CALLC1(back_from_native_C, REG_XAX)
+        /* should not return */
+        jmp      unexpected_return
+        END_FUNC(back_from_native)
+
+        
+#ifdef RETURN_STACK
+/*#############################################################################
+ *#############################################################################
+ *
+ * FIXME: this is the final routine that uses the old method of
+ * having a statically specified templace of assembly code that
+ * is copied and patched to produce thread-private versions.
+ * All the rest use exclusively generated code using our
+ * IR.  This needs to be changed as well, or else the macros
+ * need to handle dynamically changing offsets.
+ * UPDATE: this code has been removed.  We can pull it out of the
+ * attic and fix it up if necessary.
+ * We used to have unlinked_return, end_return_lookup, and return_lookup
+ * entry points defined here.
+ */
+.error RETURN_STACK needs generated routine not asm template
+/*#############################################################################
+ *#############################################################################
+ */
+#endif /* RETURN_STACK */
+
+/* Our version of setjmp & long jmp.  We don't want to modify app state like
+ * SEH or do unwinding which is done by standard versions.
+ *
+ * int cdecl dr_setjmp(dr_jmp_buf *buf);
+ */
+        DECLARE_FUNC(dr_setjmp)
+GLOBAL_LABEL(dr_setjmp:)
+        mov      REG_XDX, ARG1
+        mov      [       0 + REG_XDX], REG_XBX
+        mov      [  ARG_SZ + REG_XDX], REG_XCX
+        mov      [2*ARG_SZ + REG_XDX], REG_XDI
+        mov      [3*ARG_SZ + REG_XDX], REG_XSI
+        mov      [4*ARG_SZ + REG_XDX], REG_XBP
+        mov      [5*ARG_SZ + REG_XDX], REG_XSP
+        mov      REG_XAX, [REG_XSP]
+        mov      [6*ARG_SZ + REG_XDX], REG_XAX
+#ifdef X64
+        mov      [ 7*ARG_SZ + REG_XDX], r8
+        mov      [ 8*ARG_SZ + REG_XDX], r9
+        mov      [ 9*ARG_SZ + REG_XDX], r10
+        mov      [10*ARG_SZ + REG_XDX], r11
+        mov      [11*ARG_SZ + REG_XDX], r12
+        mov      [12*ARG_SZ + REG_XDX], r13
+        mov      [13*ARG_SZ + REG_XDX], r14
+        mov      [14*ARG_SZ + REG_XDX], r15
+#endif
+        xor      eax, eax
+        ret
+        END_FUNC(dr_setjmp)
+
+/* int cdecl dr_longjmp(dr_jmp_buf *buf, int retval);
+ */
+        DECLARE_FUNC(dr_longjmp)
+GLOBAL_LABEL(dr_longjmp:)
+        mov      REG_XDX, ARG1
+        mov      REG_XAX, ARG2
+
+        mov      REG_XBX, [       0 + REG_XDX]
+        mov      REG_XDI, [2*ARG_SZ + REG_XDX]
+        mov      REG_XSI, [3*ARG_SZ + REG_XDX]
+        mov      REG_XBP, [4*ARG_SZ + REG_XDX]
+        mov      REG_XSP, [5*ARG_SZ + REG_XDX] /* now we've switched to the old stack */
+        mov      REG_XCX, [6*ARG_SZ + REG_XDX]
+        mov      [REG_XSP], REG_XCX    /* restore the return address on to the stack */
+        mov      REG_XCX, [  ARG_SZ + REG_XDX]
+#ifdef X64
+        mov      r8,  [ 7*ARG_SZ + REG_XDX]
+        mov      r9,  [ 8*ARG_SZ + REG_XDX]
+        mov      r10, [ 9*ARG_SZ + REG_XDX]
+        mov      r11, [10*ARG_SZ + REG_XDX]
+        mov      r12, [11*ARG_SZ + REG_XDX]
+        mov      r13, [12*ARG_SZ + REG_XDX]
+        mov      r14, [13*ARG_SZ + REG_XDX]
+        mov      r15, [14*ARG_SZ + REG_XDX]
+#endif
+        ret
+        END_FUNC(dr_longjmp)
+
+
+/*#############################################################################
+ *#############################################################################
+ * Utility routines moved here due to the lack of inline asm support
+ * in VC8.
+ */
+
+/* uint atomic_swap(uint *addr, uint value)
+ * return current contents of addr and replace contents with value.
+ * on win32 could use InterlockedExchange intrinsic instead.
+ */
+        DECLARE_FUNC(atomic_swap)
+GLOBAL_LABEL(atomic_swap:)
+        mov      REG_XAX, ARG2
+        mov      REG_XCX, ARG1 /* nop on win64 (ditto for linux64 if used rdi) */
+        xchg     [REG_XCX], eax
+        ret
+        END_FUNC(atomic_swap)
+
+/* bool cpuid_supported(void)
+ * Checks for existence of the cpuid instr by attempting to modify bit 21 of eflags
+ */
+        DECLARE_FUNC(cpuid_supported)
+GLOBAL_LABEL(cpuid_supported:)
+        PUSHF
+        pop      REG_XAX
+        mov      ecx, eax      /* original eflags in ecx */
+        xor      eax, HEX(200000) /* try to modify bit 21 of eflags */
+        push     REG_XAX
+        POPF
+        PUSHF
+        pop      REG_XAX
+        cmp      ecx, eax
+        mov      eax, 0        /* zero out top bytes */
+        setne    al
+        push     REG_XCX         /* now restore original eflags */
+        POPF
+        ret
+        END_FUNC(cpuid_supported)
+
+/* void our_cpuid(int res[4], int eax)
+ * Executes cpuid instr, which is hard for x64 inline asm b/c clobbers rbx and can't
+ * push in middle of func.
+ */
+        DECLARE_FUNC(our_cpuid)
+GLOBAL_LABEL(our_cpuid:)
+        mov      REG_XDX, ARG1
+        mov      REG_XAX, ARG2
+        push     REG_XBX /* callee-saved */
+        push     REG_XDI /* callee-saved */
+        /* not making a call so don't bother w/ 16-byte stack alignment */
+        mov      REG_XDI, REG_XDX
+        cpuid
+        mov      [ 0 + REG_XDI], eax
+        mov      [ 4 + REG_XDI], ebx
+        mov      [ 8 + REG_XDI], ecx
+        mov      [12 + REG_XDI], edx
+        pop      REG_XDI /* callee-saved */
+        pop      REG_XBX /* callee-saved */
+        ret
+        END_FUNC(our_cpuid)
+
+#ifdef WINDOWS /* on linux we use inline asm versions */
+
+/* byte *get_frame_ptr(void)
+ * returns the value of ebp
+ */
+        DECLARE_FUNC(get_frame_ptr)
+GLOBAL_LABEL(get_frame_ptr:)
+        mov      REG_XAX, REG_XBP
+        ret
+        END_FUNC(get_frame_ptr)
+
+/* void dr_fxsave(byte *buf_aligned) */
+        DECLARE_FUNC(dr_fxsave)
+GLOBAL_LABEL(dr_fxsave:)
+        mov      REG_XAX, ARG1
+        fxsave   [REG_XAX]
+        fnclex
+        finit
+        ret
+        END_FUNC(dr_fxsave)
+
+/* void fnsave(byte *buf_aligned) */
+        DECLARE_FUNC(dr_fnsave)
+GLOBAL_LABEL(dr_fnsave:)
+        mov      REG_XAX, ARG1
+        /* FIXME: do we need an fwait prior to the fnsave? */
+        fnsave   [REG_XAX]
+        fwait
+        ret
+        END_FUNC(dr_fnsave)
+
+/* void fxrstor(byte *buf_aligned) */
+        DECLARE_FUNC(dr_fxrstor)
+GLOBAL_LABEL(dr_fxrstor:)
+        mov      REG_XAX, ARG1
+        fxrstor  [REG_XAX]
+        ret
+        END_FUNC(dr_fxrstor)
+
+/* void frstor(byte *buf_aligned) */
+        DECLARE_FUNC(dr_frstor)
+GLOBAL_LABEL(dr_frstor:)
+        mov      REG_XAX, ARG1
+        frstor   [REG_XAX]
+        ret
+        END_FUNC(dr_frstor)
+
+/*
+ * void call_modcode_alt_stack(dcontext_t *dcontext,
+ *                             EXCEPTION_RECORD *pExcptRec,
+ *                             CONTEXT *cxt, app_pc target, uint flags,
+ *                             bool using_initstack)
+ * custom routine used to transfer control from check_for_modified_code()
+ * to found_modified_code() win32/callback.c.
+ */
+#define dcontext        ARG1
+#define pExcptRec       ARG2
+#define cxt             ARG3
+#define target          ARG4
+#define flags           ARG5
+#define using_initstack ARG6
+        DECLARE_FUNC(call_modcode_alt_stack)
+GLOBAL_LABEL(call_modcode_alt_stack:)
+        mov      REG_XAX, dcontext /* be careful not to clobber other in-reg params */
+        mov      REG_XBX, pExcptRec
+        mov      REG_XDI, cxt
+        mov      REG_XSI, target
+        mov      REG_XDX, flags
+        cmp      using_initstack, 0
+        je       call_modcode_alt_stack_no_free
+        mov      DWORD SYMREF(initstack_mutex), 0 /* rip-relative on x64 */
+call_modcode_alt_stack_no_free:
+        RESTORE_FROM_DCONTEXT_VIA_REG(REG_XAX,dstack_OFFSET,REG_XSP)
+        CALLC5(found_modified_code, REG_XAX, REG_XBX, REG_XDI, REG_XSI, REG_XDX)
+        /* should never return */
+        jmp      unexpected_return
+        ret
+        END_FUNC(call_modcode_alt_stack)
+#undef dcontext
+#undef pExcptRec
+#undef cxt
+#undef target
+#undef flags
+#undef using_initstack
+
+#ifdef STACK_GUARD_PAGE
+/*
+ * void call_intr_excpt_alt_stack(dcontext_t *dcontext, EXCEPTION_RECORD *pExcptRec,
+ *                                CONTEXT *cxt, byte *stack)
+ *       
+ * Routine to switch to a separate exception stack before calling
+ * internal_exception_info().  This switch is useful if the dstack
+ * is exhausted and we want to ensure we have enough space for
+ * error reporting.
+ */
+#define dcontext        ARG1
+#define pExcptRec       ARG2
+#define cxt             ARG3
+#define stack           ARG4
+        DECLARE_FUNC(call_intr_excpt_alt_stack)
+GLOBAL_LABEL(call_intr_excpt_alt_stack:)
+        mov      REG_XAX, dcontext
+        mov      REG_XBX, pExcptRec
+        mov      REG_XDI, cxt
+        mov      REG_XSI, REG_XSP
+        mov      REG_XSP, stack
+# ifdef X64
+        /* retaddr + this push => 16-byte alignment prior to call */
+# endif
+        push     REG_XSI       /* save xsp */
+        CALLC4(internal_exception_info, \
+               REG_XAX /* dcontext */,  \
+               REG_XBX /* pExcptRec */, \
+               REG_XDI /* cxt */,       \
+               1       /* dstack overflow == true */)
+        pop      REG_XSP
+        ret
+        END_FUNC(call_intr_excpt_alt_stack)
+#undef dcontext
+#undef pExcptRec
+#undef cxt
+#undef stack
+#endif
+
+
+/* void get_segments_defg(cxt_seg_t *ds, cxt_seg_t *es, cxt_seg_t *fs, cxt_seg_t *gs) */
+        DECLARE_FUNC(get_segments_defg)
+/* CONTEXT.Seg* is WORD for x64 but DWORD for x86 */
+#ifdef X64
+# define REG_XAX_SEGWIDTH ax
+#else
+# define REG_XAX_SEGWIDTH eax
+#endif
+GLOBAL_LABEL(get_segments_defg:)
+        xor      eax, eax
+        mov      REG_XBX, ARG1
+        mov      ax, ds
+        mov      [REG_XBX], REG_XAX_SEGWIDTH
+        mov      REG_XBX, ARG2
+        mov      ax, es
+        mov      [REG_XBX], REG_XAX_SEGWIDTH
+        mov      REG_XBX, ARG3
+        mov      ax, fs
+        mov      [REG_XBX], REG_XAX_SEGWIDTH
+        mov      REG_XBX, ARG4
+        mov      ax, gs
+        mov      [REG_XBX], REG_XAX_SEGWIDTH
+        ret
+        END_FUNC(get_segments_defg)
+#undef REG_XAX_SEGWIDTH
+
+/* void get_own_context_helper(CONTEXT *cxt)
+ * does not fix up xsp to match the call site
+ * does not preserve callee-saved registers
+ */
+        DECLARE_FUNC(get_own_context_helper)
+GLOBAL_LABEL(get_own_context_helper:)
+        /* push callee-saved registers that we use */
+        push     REG_XBX
+        push     REG_XSI
+        push     REG_XDI
+#ifdef  X64
+        /* w/ retaddr, we're now at 16-byte alignment */
+        /* save argument register (PUSH_DR_MCONTEXT calls out to c code) */
+        mov REG_XDI, ARG1
+#endif
+        
+        /* grab exec state and pass as param in a dr_mcontext_t struct */
+        PUSH_DR_MCONTEXT([(3 * ARG_SZ) + REG_XSP] /* use retaddr for pc */)
+        /* we don't have enough registers to avoid parameter regs so we carefully
+         * use the suggested register order
+         */
+        lea      REG_XSI, [REG_XSP] /* stack grew down, so dr_mcontext_t at tos */
+#ifdef X64
+        mov      REG_XAX, REG_XDI
+#else
+        /* 4 * arg_sz = 3 callee saved registers pushed to stack plus return addr */
+        mov      REG_XAX, [DR_MCONTEXT_SIZE + (4 * ARG_SZ) + REG_XSP]
+#endif
+        xor      edi, edi
+        mov      di, ss
+        xor      ebx, ebx
+        mov      bx, cs
+        CALLC4(get_own_context_integer_control, REG_XAX, REG_XBX, REG_XDI, REG_XSI)
+        add      REG_XSP, DR_MCONTEXT_SIZE
+        pop      REG_XDI
+        pop      REG_XSI
+        pop      REG_XBX
+        ret
+        END_FUNC(get_own_context_helper)
+
+#endif /* WINDOWS */
+
+/* void get_xmm_caller_saved(byte *xmm_caller_saved_buf)
+ *   stores the values of xmm0 through xmm5 consecutively into xmm_caller_saved_buf.
+ *   xmm_caller_saved_buf need not be 16-byte aligned.
+ *   for linux, also saves xmm6-15 (PR 302107).
+ *   caller must ensure that the underlying processor supports SSE!
+ * FIXME PR 266305: AMD optimization guide says to use movlps+movhps for unaligned
+ * stores, instead of movups (movups is best for loads): but for
+ * simplicity I'm sticking with movups (assumed not perf-critical here).
+ */
+        DECLARE_FUNC(get_xmm_caller_saved)
+GLOBAL_LABEL(get_xmm_caller_saved:)
+        mov      REG_XAX, ARG1
+        movups   [REG_XAX + 0*16], xmm0
+        movups   [REG_XAX + 1*16], xmm1
+        movups   [REG_XAX + 2*16], xmm2
+        movups   [REG_XAX + 3*16], xmm3
+        movups   [REG_XAX + 4*16], xmm4
+        movups   [REG_XAX + 5*16], xmm5
+#if defined(LINUX) || !defined(X64)
+        movups   [REG_XAX + 6*16], xmm6
+        movups   [REG_XAX + 7*16], xmm7
+#endif
+#if defined(LINUX) && defined(X64)
+        movups   [REG_XAX + 8*16], xmm8
+        movups   [REG_XAX + 9*16], xmm9
+        movups   [REG_XAX + 10*16], xmm10
+        movups   [REG_XAX + 11*16], xmm11
+        movups   [REG_XAX + 12*16], xmm12
+        movups   [REG_XAX + 13*16], xmm13
+        movups   [REG_XAX + 14*16], xmm14
+        movups   [REG_XAX + 15*16], xmm15
+#endif
+        ret
+        END_FUNC(get_xmm_caller_saved)
+
+/*#############################################################################
+ *#############################################################################
+ */
+
+/****************************************************************************/
+/****************************************************************************/
+#endif /* !NOT_DYNAMORIO_CORE_PROPER */
+
+/****************************************************************************
+ * routines shared with NOT_DYNAMORIO_CORE_PROPER
+ */
+#ifdef WINDOWS
+
+/* byte *get_stack_ptr(void)
+ * returns the value of xsp before the call
+ */
+        DECLARE_FUNC(get_stack_ptr)
+GLOBAL_LABEL(get_stack_ptr:)
+        mov      REG_XAX, REG_XSP
+        add      REG_XAX, ARG_SZ /* remove return address space */
+        ret
+        END_FUNC(get_stack_ptr)
+
+/* void load_dynamo(void)
+ *
+ * used for injection into a child process
+ * N.B.:  if the code here grows, SIZE_OF_LOAD_DYNAMO in win32/inject.c
+ * must be updated.
+ */
+        DECLARE_FUNC(load_dynamo)
+GLOBAL_LABEL(load_dynamo:)
+    /* the code for this routine is copied into an allocation in the app
+       and invoked upon return from the injector. When it is invoked, 
+       it expects the app's stack to look like this:
+
+                xsp-->| &LoadLibrary  |
+                      | &dynamo_path  |
+                      | &GetProcAddr  |
+                      | &dynamo_entry |___
+                      |               |   |
+                      |(saved context)| dr_mcontext_t struct
+                      | &code_alloc   |   | pointer to the code allocation
+                      | sizeof(code_alloc)| size of the code allocation
+                      |_______________|___|
+       &dynamo_path-->|               |   |
+                      | (dynamo path) | TEXT(DYNAMORIO_DLL_PATH)
+                      |_______________|___|
+      &dynamo_entry-->|               |   |
+                      | (dynamo entry)| "dynamo_auto_start"
+                      |               |___|
+                      
+
+        in separate allocation         ___
+                      |               |   |
+                      |     CODE      |  load_dynamo() code
+                      |               |___|
+
+        The load_dynamo routine will load the dynamo DLL into memory, then jump
+        to its dynamo_auto_start entry point, passing it the saved app context as
+        parameters.
+     */
+        /* two byte NOP to satisfy third party braindead-ness documented in case 3821 */
+        mov      edi, edi
+#ifdef LOAD_DYNAMO_DEBUGBREAK
+        /* having this code in front may hide the problem addressed with the 
+         * above padding */
+        /* giant loop so can attach debugger, then change ebx to 1 
+         * to step through rest of code */
+        mov      ebx, HEX(7fffffff)
+load_dynamo_repeat_outer:       
+        mov      eax, HEX(7fffffff)
+load_dynamo_repeatme:   
+        dec      eax
+        cmp      eax, 0
+        jg       load_dynamo_repeatme
+        dec      ebx
+        cmp      ebx, 0
+        jg       load_dynamo_repeat_outer
+
+        /* TOS has &DebugBreak */
+        pop      REG_XBX /* pop   REG_XBX = &DebugBreak */
+        CALLWIN0(REG_XBX) /* call DebugBreak  (in kernel32.lib) */
+#endif
+        /* TOS has &LoadLibraryA */
+        pop      REG_XBX /* pop   REG_XBX = &LoadLibraryA */
+        /* TOS has &dynamo_path */
+        pop      REG_XAX /* for 32-bit we're doing "pop eax, push eax" */
+        CALLWIN1(REG_XBX, REG_XAX) /* call LoadLibraryA  (in kernel32.lib) */
+
+        /* check result */
+        cmp      REG_XAX, 0
+        jne      load_dynamo_success
+        pop      REG_XBX /* pop off &GetProcAddress */
+        pop      REG_XBX /* pop off &dynamo_entry */
+        jmp      load_dynamo_failure
+load_dynamo_success:
+        /* TOS has &GetProcAddress */
+        pop      REG_XBX /* pop   REG_XBX = &GetProcAddress */
+        /* dynamo_handle is now in REG_XAX (returned by call LoadLibrary) */
+        /* TOS has &dynamo_entry */
+        pop      REG_XDI /* for 32-bit we're doing "pop edi, push edi" */
+        CALLWIN2(REG_XBX, REG_XAX, REG_XDI) /* call GetProcAddress */
+        cmp      REG_XAX, 0
+        je       load_dynamo_failure
+
+        /* jump to dynamo_auto_start (returned by GetProcAddress) */
+        jmp      REG_XAX
+        /* dynamo_auto_start will take over or continue natively at the saved 
+         * context via load_dynamo_failure.
+        */
+        END_FUNC(load_dynamo)
+/* N.B.: load_dynamo_failure MUST follow load_dynamo, as both are
+ * copied in one fell swoop by inject_into_thread()!
+ */
+/* not really a function but having issues getting both masm and gas to
+ * let other asm routines jump here.
+ * targeted by load_dynamo and dynamo_auto_start by a jump, not a call,
+ * when we should not take over and should go native instead.
+ * Xref case 7654: we come here to the child's copy from dynamo_auto_start 
+ * instead of returning to the parent's copy post-load_dynamo to avoid
+ * incompatibilites with stack layout accross dr versions.
+ */
+        DECLARE_FUNC(load_dynamo_failure)
+GLOBAL_LABEL(load_dynamo_failure:)
+        /* Would be nice if we could free our allocation here as well, but 
+         * that's too much of a pain (esp. here). 
+         * Note TOS has the saved context at this point, xref layout in 
+         * auto_setup. Note this code is duplicated in dynamo_auto_start. */
+        mov      REG_XAX, [MCONTEXT_XSP_OFFS + REG_XSP] /* load app xsp */
+        mov      REG_XBX, [MCONTEXT_PC_OFFS + REG_XSP] /* load app start_pc */
+        /* write app start_pc off top of app stack */
+        mov      [-ARG_SZ + REG_XAX], REG_XBX
+        /* it's ok to write past app TOS since we're just overwriting part of 
+         * the dynamo_entry string which is dead at this point, won't affect 
+         * the popping of the saved context */
+        POPALL /* we assume xmm0-5 do not need to be restored */
+        POPF
+        /* we assume reading beyond TOS is ok here (no signals on windows) */
+         /* restore app xsp (POPALL doesn't): didn't pop pc, so +ARG_SZ */
+        mov      REG_XSP, [-DR_MCONTEXT_SIZE + MCONTEXT_XSP_OFFS + ARG_SZ + REG_XSP]
+        jmp      PTRSZ [-ARG_SZ + REG_XSP]      /* jmp to app start_pc */
+
+        ret
+        END_FUNC(load_dynamo_failure)
+        
+#endif /* WINDOWS */
+
+END_FILE
diff --git a/core/x86/x86_code.c b/core/x86/x86_code.c
new file mode 100644
index 000000000..ef8b6dc15
--- /dev/null
+++ b/core/x86/x86_code.c
@@ -0,0 +1,378 @@
+/* **********************************************************
+ * Copyright (c) 2001-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2001-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2001 Hewlett-Packard Company */
+
+/*
+ * x86_code.c - auxiliary C routines to assembly routines in x86.asm
+ */
+#include "globals.h"
+#include "fragment.h"
+#include "dispatch.h"
+#include "monitor.h"
+#include "arch.h"
+#include <string.h> /* for memcpy */
+
+#ifdef LINUX
+/* in signal.c */
+extern void signal_thread_inherit(dcontext_t *dcontext);
+#endif
+
+/* Helper routine for the x86.asm PUSH_DR_MCONTEXT, to fill in the xmm0-5 values
+ * only if necessary.
+ */
+void
+get_xmm_vals(dr_mcontext_t *mc)
+{
+    if (preserve_xmm_caller_saved()) {
+        ASSERT(proc_has_feature(FEATURE_SSE));
+        get_xmm_caller_saved(&mc->xmm[0]);
+    }
+}
+
+/* initializes dcontext and performs other initialization
+ * intended to be done each time a thread comes under dynamo control
+ */
+static void
+thread_starting(dcontext_t *dcontext)
+{
+    initialize_dynamo_context(dcontext);
+    dynamo_thread_under_dynamo(dcontext);
+#ifdef WINDOWS
+    LOG(THREAD, LOG_INTERP, 2, "thread_starting: interpreting thread %d\n",
+        get_thread_id());
+#endif
+}
+
+/* Initializes a dcontext with the supplied state and calls dispatch */
+void
+dynamo_start(dr_mcontext_t *mc)
+{
+    dr_mcontext_t *mcontext;
+    dcontext_t *dcontext = get_thread_private_dcontext();
+    ASSERT(dcontext != NULL);
+    thread_starting(dcontext);
+
+    /* Set return address */
+    dcontext->next_tag = mc->pc;
+    ASSERT(dcontext->next_tag != NULL);
+
+    /* transfer exec state to mcontext */
+    mcontext = get_mcontext(dcontext);
+    *mcontext = *mc;
+
+    /* clear pc */
+    mcontext->pc = 0;
+
+    /* Swap stacks so dispatch is invoked outside the application.
+     * We begin interpretation at the application return point,
+     * and thus we need to look like DR returned -- adjust the app
+     * stack to account for the return address.
+     */
+    mcontext->xsp += XSP_SZ;
+    call_switch_stack(dcontext, dcontext->dstack, dispatch,
+                      false/*not on initstack*/, true/*return on error*/);
+    /* In release builds, this will simply return and continue native
+     * execution.  That's better than calling unexpected_return() which
+     * goes into an infinite loop.
+     */
+    ASSERT_NOT_REACHED();
+}
+
+/* auto_setup: called by dynamo_auto_start for non-early follow children.
+ * This routine itself would be dynamo_auto_start except that we want
+ * our own go-native path separate from load_dynamo (we could still have
+ * this by dynamo_auto_start and jump to an asm routine for go-native,
+ * but keeping the entry in asm is more flexible).
+ * Assumptions: The saved dr_mcontext_t for the start of the app is on
+ * the stack, followed by a pointer to a region of memory to free
+ * (which can be NULL) and its size.  If we decide not to take over
+ * this process, this routine returns; otherwise it does not return.
+ */
+void
+auto_setup(ptr_uint_t appstack)
+{
+    dcontext_t *dcontext;
+    dr_mcontext_t *mcontext;
+    byte *pappstack;
+    byte        *addr;
+
+    pappstack = (byte *)appstack;
+
+    /* Our parameter points at a dr_mcontext_t struct, beyond which are
+     * two other fields:
+       pappstack --> +0  dr_mcontext_t struct
+                     +x  addr of memory to free (can be NULL)
+                     +y  sizeof memory to free
+    */
+
+    automatic_startup = true;
+    /* we should control all threads */
+    control_all_threads = true;
+    dynamorio_app_init();
+    if (INTERNAL_OPTION(nullcalls)) {
+        dynamorio_app_exit();
+        return;
+    }
+
+    /* For apps injected using follow_children, this is where control should be 
+     * allowed to go native for hotp_only & thin_client.
+     */
+    if (RUNNING_WITHOUT_CODE_CACHE())
+        return;
+
+    /* useful to debug fork-following */
+    DOLOG(4, LOG_TOP, { SYSLOG_INTERNAL_INFO("dynamo auto start"); });
+
+    dcontext = get_thread_private_dcontext();
+    ASSERT(dcontext);
+    thread_starting(dcontext);
+
+    /* copy over the app state into mcontext */
+    mcontext = get_mcontext(dcontext);
+    *mcontext = *((dr_mcontext_t *)pappstack);
+    pappstack += sizeof(dr_mcontext_t);
+    dcontext->next_tag = mcontext->pc;
+    ASSERT(dcontext->next_tag != NULL);
+
+    /* free memory */
+    addr = (byte *) *((byte **)pappstack);
+    pappstack += sizeof(byte*);
+    if (addr != NULL) {
+        size_t size = *((size_t*)pappstack);
+        heap_error_code_t error_code;
+        /* since this is rx it was added to our exec list, remove now 
+         * ASSUMPTION: no fragments in the region so no need to flush
+         */
+        /* flushing would align for us but we have to do it ourselves here */
+        size_t alloc_size = ALIGN_FORWARD(size, PAGE_SIZE);
+        DODEBUG({
+            if (SHARED_FRAGMENTS_ENABLED())
+                ASSERT(!thread_vm_area_overlap(GLOBAL_DCONTEXT, addr, addr+alloc_size));
+        });
+        ASSERT(!thread_vm_area_overlap(dcontext, addr, addr+alloc_size));
+        remove_executable_region(addr, alloc_size, false/*do not have lock*/);
+        os_heap_free(addr, size, &error_code);
+    }
+
+    /* FIXME : for transparency should we zero out the appstack where we
+     * stored injection information? would be safe to do so here */
+
+    LOG(THREAD, LOG_INTERP, 1, "DynamoRIO auto start at 0x%08x\n",
+        dcontext->next_tag);
+    DOLOG(LOG_INTERP, 2, {
+        dump_mcontext(mcontext, THREAD, DUMP_NOT_XML);
+    });
+
+    call_switch_stack(dcontext, dcontext->dstack, dispatch,
+                      false/*not on initstack*/, false/*shouldn't return*/);
+    ASSERT_NOT_REACHED();
+}
+
+/****************************************************************************/
+#ifdef LINUX
+
+/* Called by new_thread_dynamo_start to initialize the dcontext
+ * structure for the current thread and start executing at mc->pc.
+ * Assumes that it is called on the initstack.
+ */
+void
+new_thread_setup(dr_mcontext_t *mc)
+{
+    dcontext_t *dcontext;
+    int rc;
+    /* this is where a new thread first touches other than the initstack,
+     * so we "enter" DR here
+     */
+    ENTERING_DR();
+    LOG(GLOBAL, LOG_INTERP, 1,
+        "new_thread_setup: thread %d, pc "PFX"\n", get_thread_id(), mc->pc);
+    rc = dynamo_thread_init();
+    ASSERT(rc != -1); /* this better be a new thread */
+    dcontext = get_thread_private_dcontext();
+    ASSERT(dcontext != NULL);
+    signal_thread_inherit(dcontext);
+    thread_starting(dcontext);
+    dcontext->next_tag = (app_pc) mc->pc;
+    ASSERT(dcontext->next_tag != NULL);
+
+    *get_mcontext(dcontext) = *mc;
+    /* get app xsp from initstack_app_xsp */
+    get_mcontext(dcontext)->xsp = (reg_t) initstack_app_xsp;
+    /* clear xax (was used to hold next pc) */
+    ASSERT(get_mcontext(dcontext)->xax == (reg_t) dcontext->next_tag);
+    get_mcontext(dcontext)->xax = 0;
+    /* clear pc */
+    get_mcontext(dcontext)->pc = 0;
+
+    call_switch_stack(dcontext, dcontext->dstack, dispatch,
+                      true/*on initstack*/, false/*shouldn't return*/);
+    ASSERT_NOT_REACHED();
+}
+
+#endif /* LINUX */
+
+#ifdef WINDOWS
+
+/* Called by nt_continue_dynamo_start when we're about to execute
+ * the continuation of an exception or APC: after NtContinue.
+ * next_pc is bogus, the real next pc has been stored in dcontext->next_tag.
+ * This routine is also used by NtSetContextThread.
+ */
+void
+nt_continue_setup(dr_mcontext_t *mc)
+{
+    app_pc next_pc;
+    dcontext_t *dcontext;
+    ENTERING_DR();
+    dcontext = get_thread_private_dcontext();
+    ASSERT(dcontext != NULL);
+    SELF_PROTECT_LOCAL(dcontext, WRITABLE);
+    /* save target in temp var during init of dcontext */
+    /* we have to use a different slot since next_tag ends up holding the do_syscall
+     * entry when entered from dispatch
+     */
+    if (dcontext->asynch_target != NULL)
+        next_pc = dcontext->asynch_target;
+    else {
+        ASSERT(DYNAMO_OPTION(shared_syscalls));
+        next_pc = dcontext->next_tag;
+    }
+    LOG(THREAD, LOG_ASYNCH, 2, "nt_continue_setup: target is "PFX"\n", next_pc);
+    initialize_dynamo_context(dcontext);
+    dcontext->next_tag = next_pc;
+    ASSERT(dcontext->next_tag != NULL);
+    set_last_exit(dcontext, (linkstub_t *) get_asynch_linkstub());
+    dcontext->whereami = WHERE_TRAMPOLINE;
+
+    *get_mcontext(dcontext) = *mc;
+    /* clear pc */
+    get_mcontext(dcontext)->pc = 0;
+
+    call_switch_stack(dcontext, dcontext->dstack, dispatch,
+                      false/*not on initstack*/, false/*shouldn't return*/);
+    ASSERT_NOT_REACHED();
+}
+
+#endif /* WINDOWS */
+/****************************************************************************/
+
+/***************************************************************************
+ * NATIVE EXECUTION EXPERIMENTAL FEATURE
+ */
+
+/* WARNING: this feature breaks all kinds of rules, like ret addr transparency and
+ * assuming app stack and not doing calls out of the cache and no self-protection
+ * and not catching hand-rolled syscalls, etc. -- use at own risk!
+ */
+
+void
+entering_native()
+{
+    dcontext_t *dcontext;
+    ENTERING_DR();
+    dcontext = get_thread_private_dcontext();
+    ASSERT(dcontext != NULL);
+#ifdef WINDOWS
+    /* turn off asynch interception for this thread while native
+     * FIXME: what if callbacks and apcs are destined for other modules?
+     * should instead run dispatcher under DR every time, if going to native dll
+     * will go native then?  have issues w/ missing the cb ret, though...
+     * N.B.: if allow some asynch, have to find another place to store the real
+     * return addr (currently in next_tag)
+     *
+     * We can't revert memory prots, since other threads are under DR
+     * control, but we do handle our-fault write faults in native threads.
+     */
+    set_asynch_interception(dcontext->owning_thread, false);
+#endif
+    /* FIXME: setting same var that set_asynch_interception is! */
+    dcontext->thread_record->under_dynamo_control = false;
+
+    /* if we were building a trace, kill it */
+    if (is_building_trace(dcontext)) {
+        LOG(THREAD, LOG_ASYNCH, 2, "entering_native: squashing old trace\n");
+        trace_abort(dcontext);
+    }
+    set_last_exit(dcontext, (linkstub_t *) get_native_exec_linkstub());
+    /* now we're in app! */
+    dcontext->whereami = WHERE_APP;
+    SYSLOG_INTERNAL_WARNING_ONCE("entered at least one module natively");
+    LOG(THREAD, LOG_ASYNCH, 1, "!!!! Entering module NATIVELY, retaddr="PFX"\n\n",
+        dcontext->native_exec_retval);
+    STATS_INC(num_native_module_enter);
+    EXITING_DR();
+}
+
+/* work that's easier to do in C code than in the asm routine back_from_native()
+ */
+void
+back_from_native_C(dr_mcontext_t *mc)
+{
+    dcontext_t *dcontext;
+    ENTERING_DR();
+    dcontext = get_thread_private_dcontext();
+    ASSERT(dcontext != NULL);
+    LOG(THREAD, LOG_ASYNCH, 1, "\n!!!! Returned from NATIVE module to "PFX"\n",
+        dcontext->native_exec_retval);
+    SYSLOG_INTERNAL_WARNING_ONCE("returned from at least one native module");
+    STATS_INC(num_native_module_exit);
+    /* ASSUMPTION: was native entire time, don't need to initialize dcontext
+     * or anything, and next_tag is still there!
+     */
+    ASSERT(dcontext->whereami == WHERE_APP);
+    ASSERT(dcontext->native_exec_retval != NULL);
+    ASSERT(dcontext->last_exit == get_native_exec_linkstub());
+    dcontext->next_tag = dcontext->native_exec_retval;
+    dcontext->native_exec_retval = NULL;
+    dcontext->native_exec_retloc = NULL;
+    /* tell dispatch() why we're coming there */
+    dcontext->whereami = WHERE_FCACHE;
+#ifdef WINDOWS
+    /* asynch back on */
+    set_asynch_interception(dcontext->owning_thread, true);
+#endif
+    /* FIXME: setting same var that set_asynch_interception is! */
+    dcontext->thread_record->under_dynamo_control = true;
+
+    *get_mcontext(dcontext) = *mc;
+    /* clear pc */
+    get_mcontext(dcontext)->pc = 0;
+
+    call_switch_stack(dcontext, dcontext->dstack, dispatch,
+                      false/*not on initstack*/, false/*shouldn't return*/);
+    ASSERT_NOT_REACHED();
+}
+
+/****************************************************************************/
diff --git a/libutil/Makefile b/libutil/Makefile
new file mode 100644
index 000000000..ac5bfa653
--- /dev/null
+++ b/libutil/Makefile
@@ -0,0 +1,305 @@
+# **********************************************************
+# Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+# **********************************************************/
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+
+ifndef DYNAMORIO_MAKE
+  DYNAMORIO_MAKE := ../make
+endif
+
+include $(DYNAMORIO_MAKE)/compiler.mk
+
+ifndef DYNAMORIO_HOME
+  DYNAMORIO_HOME := ..
+endif
+WIN_DYNAMORIO_HOME := $(shell $(CYGPATH) -ma ${DYNAMORIO_HOME})
+
+ifndef DYNAMORIO_HOTPATCH
+  DYNAMORIO_HOTPATCH := ../liveshield
+endif
+WIN_DYNAMORIO_HOTPATCH := $(shell $(CYGPATH) -ma $(DYNAMORIO_HOTPATCH))
+
+# FIXME: fix the underlying cause of these warnings and
+# remove these exceptions
+CFLAGS += /wd4996 /wd4047 /wd4024 /wd4133 /wd4127 /wd4100
+CFLAGS += /wd4292 /wd4127 /wd4100
+# avoid security_check_cookie issues when linking our library w/ others
+CFLAGS += /GS-
+
+ifndef VMAP
+# We default to VMSAFE configuration
+CFLAGS += /DPROGRAM_SHEPHERDING /DMF_API /DHOT_PATCHING_INTERFACE /DPROBE_API
+endif
+
+ifndef DIFFNUM
+  DIFFNUM := 1
+endif
+
+# xref PR 192750 - runregression overrides this
+DBG = /Zi
+LINK_DBG = /debug
+
+CORE_DIRNAME := core
+include $(DYNAMORIO_HOME)/$(CORE_DIRNAME)/Makefile.custom_build
+
+TARGET_LIB := policy
+CONFIG_LIB := drconfig
+MFAPI_LIB  := mfapi
+
+BNFLAGS  := $(D)BUILD_NUMBER=$(BUILD_NUMBER)
+TYPE_EXE := VFT_APP
+TYPE_DLL := VFT_DLL
+
+RCDEFS  := $(RESOURCE_DEFINES) $(BNFLAGS)
+RCINCL  := /I"$(WIN_DYNAMORIO_HOME)/$(CORE_DIRNAME)/lib" $(MSINCLUDE)
+RCFLAGS := -r $(RCINCL) $(RCDEFS)
+
+RCFILE  := $(DYNAMORIO_HOME)/$(CORE_DIRNAME)/win32/resources.rc
+WIN_RCFILE := $(shell $(CYGPATH) -ma $(RCFILE))
+
+DEFINES := /DSHARED_CACHE /DRETURN_AFTER_CALL /DPROGRAM_SHEPHERDING \
+	   /DCLIENT_INTERFACE /DPROFILE_LINKCOUNT /DSIMULATE_ATTACK /DGBOP \
+	   /DWINDOWS_PC_SAMPLE /DHOT_PATCHING_INTERFACE /DNOT_DYNAMORIO_CORE \
+	   /DX86 /DWINDOWS /DUNICODE /D_UNICODE /DDEBUG /DHOT_PATCHING_INTERFACE \
+	   /D_CRT_SECURE_NO_WARNINGS
+
+INCLUDES := /I"$(WIN_DYNAMORIO_HOME)/$(CORE_DIRNAME)" \
+	    /I"$(WIN_DYNAMORIO_HOME)/$(CORE_DIRNAME)/lib" \
+	    /I"$(WIN_DYNAMORIO_HOME)/$(CORE_DIRNAME)/win32" \
+	    $(MSINCLUDE)
+
+CFLAGS += /W4 /WX $(DEFINES) $(INCLUDES)
+DEBUG_CFLAGS := $(CFLAGS) $(DBG)
+STATIC_DEBUG_CFLAGS := $(DEBUG_CFLAGS) /MTd
+CFLAGS += /O2
+STATIC_CFLAGS := $(CFLAGS) /MT
+
+LDFLAGS := $(MSLIB)
+LDFLAGS += /nodefaultlib:libc.lib
+DEBUG_LDFLAGS := $(MSLIB) $(LINK_DBG)
+
+GENERICLIBS := advapi32.lib
+
+HEADERS := config.h elm.h share.h processes.h services.h utils.h events.h
+
+CORE_SRCS := options.c inject_shared.c drmarker.c module_shared.c 
+CORE_OBJS := $(subst .c,.obj,$(CORE_SRCS))
+CORE_OBJS_D := $(subst .c,_d.obj,$(CORE_SRCS))
+CORE_OBJS_S := $(subst .c,_static.obj,$(CORE_SRCS))
+CORE_OBJS_SD := $(subst .c,_staticd.obj,$(CORE_SRCS))
+
+BASE_LIST := services config elm processes utils policy parser
+ROOT_LIST := $(BASE_LIST) detach mfapi $(subst .c,,$(CORE_SRCS))
+
+LIB_OBJS   := $(patsubst %,%_static.obj,$(ROOT_LIST))
+LIB_OBJS_D := $(patsubst %,%_staticd.obj,$(ROOT_LIST))
+
+CONFIG_LIST   := $(ROOT_LIST) dr_config
+CONFIG_OBJS   := $(patsubst %,%.obj,$(CONFIG_LIST))
+CONFIG_OBJS_D := $(patsubst %,%_d.obj,$(CONFIG_LIST))
+
+MFAPI_LIST := $(ROOT_LIST) mfapi
+MFAPI_OBJS := $(patsubst %,%.obj,$(MFAPI_LIST))
+
+# FIXME: unit tests for detach?
+TEST_LIST  := $(BASE_LIST) tests
+TEST_EXES  := $(patsubst %,%_test.exe,$(TEST_LIST))
+UNIT_TESTS := $(patsubst %,%_test,$(TEST_LIST))
+
+# see tests.h
+TESTER_EXES  := tester_1.exe tester_2.exe
+TEST_LDFLAGS := $(LINK_DBG) /pdb:test.pdb
+
+ifdef SINGLE_TEST_NAME
+  SINGLETESTDEFS := /DSINGLE_TEST=$(SINGLE_TEST_NAME)
+else
+  SINGLETESTDEFS := 
+endif
+
+ifdef FAIL_ON_TEST_ASSERT
+  FAILASSERTDEFS := /DFAIL_ON_TEST_ASSERT
+else
+  FAILASSERTDEFS :=
+endif
+
+ENGINE := 30000
+DR_LIB_DIR := $(DYNAMORIO_HOME)/lib32
+HOTP_LIB_DIR := $(DYNAMORIO_HOME)/lib32/hotp/$(ENGINE)
+HOTP_DEFS_FILE := ls-defs.cfg
+HOTP_MODES_FILE := test-modes.cfg
+CONFIG_DIR := $(DYNAMORIO_HOME)/conf
+
+
+.PHONY: all debug clean tester
+
+.PRECIOUS: $(TEST_EXES) $(TESTER_EXES)
+
+
+all: $(TARGET_LIB)_static.lib $(CONFIG_LIB).lib dumpevts.exe
+
+debug: $(TARGET_LIB)_staticd.lib $(CONFIG_LIB)_d.lib
+
+events.h: $(DYNAMORIO_HOME)/$(CORE_DIRNAME)/win32/events.mc
+	$(MC) $(WIN_DYNAMORIO_HOME)/$(CORE_DIRNAME)/win32/events.mc
+
+%.obj: %.c $(HEADERS)
+	$(CC) $(CFLAGS) /c $< /Fo$@ /Fd$(subst .obj,.pdb,$@)
+
+%_d.obj: %.c $(HEADERS)
+	$(CC) $(DEBUG_CFLAGS) /c $< /Fo$@ /Fd$(subst .obj,.pdb,$@)
+
+%_static.obj: %.c $(HEADERS)
+	$(CC) $(STATIC_CFLAGS) /c $< /Fo$@ /Fd$(subst .obj,.pdb,$@)
+
+%_staticd.obj: %.c $(HEADERS)
+	$(CC) $(STATIC_DEBUG_CFLAGS) /c $< /Fo$@ /Fd$(subst .obj,.pdb,$@)
+
+%_test.obj: %.c $(HEADERS) tests.h
+	$(CC) $(DEBUG_CFLAGS) $(SINGLETESTDEFS) $(FAILASSERTDEFS) /DUNIT_TEST /c $< /Fo$@ /Fd$(subst .obj,.pdb,$@)
+
+# FIXME: if we use WIN_DYNAMORIO_HOME then some older makes parse the drive
+# specification colon as part of the rule and complain; any cleaner way
+# than invoking all these cygpaths?
+%.obj: $(DYNAMORIO_HOME)/$(CORE_DIRNAME)/%.c
+	$(CC) $(CFLAGS) /c $(shell cygpath -ma $<) /Fo$@ /Fd$*.pdb
+
+%_d.obj: $(DYNAMORIO_HOME)/$(CORE_DIRNAME)/%.c
+	$(CC) $(DEBUG_CFLAGS) /c $(shell cygpath -ma $<) /Fo$@ /Fd$*.pdb
+
+%_static.obj: $(DYNAMORIO_HOME)/$(CORE_DIRNAME)/%.c
+	$(CC) $(STATIC_CFLAGS) /c $(shell cygpath -ma $<) /Fo$@ /Fd$*.pdb
+
+%_staticd.obj: $(DYNAMORIO_HOME)/$(CORE_DIRNAME)/%.c
+	$(CC) $(STATIC_DEBUG_CFLAGS) $(shell cygpath -ma $<) /Fo$@ /Fd$*.pdb
+
+%.obj: $(DYNAMORIO_HOME)/$(CORE_DIRNAME)/win32/%.c
+	$(CC) $(CFLAGS) /c $(shell cygpath -ma $<) /Fo$@ /Fd$*.pdb
+
+%_d.obj: $(DYNAMORIO_HOME)/$(CORE_DIRNAME)/win32/%.c
+	$(CC) $(DEBUG_CFLAGS) /c $(shell cygpath -ma $<) /Fo$@ /Fd$*.pdb
+
+%_static.obj: $(DYNAMORIO_HOME)/$(CORE_DIRNAME)/win32/%.c
+	$(CC) $(STATIC_CFLAGS) /c $(shell cygpath -ma $<) /Fo$@ /Fd$*.pdb
+
+%_staticd.obj: $(DYNAMORIO_HOME)/$(CORE_DIRNAME)/win32/%.c
+	$(CC) $(STATIC_DEBUG_CFLAGS) $(shell cygpath -ma $<) /Fo$@ /Fd$*.pdb
+
+dumpevts.exe: dumpevts.obj elm.obj dumpevts.res
+	$(LD) $(LDFLAGS) $^ $(GENERICLIBS) /out:$@
+
+dumpevts.res: $(RCFILE)
+	$(RC) /Fo$@ $(RCFLAGS) $(D)FILE_TYPE=$(TYPE_EXE) $(D)FILE_NAME=\"dumpevts.exe\" $(D)FILE_DESCRIPTION="PRODUCT_NAME \" event dumping tool\"" $(WIN_RCFILE)
+
+$(TARGET_LIB)_static.lib: $(LIB_OBJS)
+	$(AR) $(LDFLAGS) $^ $(GENERICLIBS) /out:$@
+
+$(TARGET_LIB)_staticd.lib: $(LIB_OBJS_D)
+	$(AR) $(DEBUG_LDFLAGS) /machine:I386 $^ $(GENERICLIBS) /out:$@
+
+$(CONFIG_LIB).lib: $(CONFIG_LIB).dll
+
+$(CONFIG_LIB).dll: $(CONFIG_OBJS)
+	$(LD) $(LDFLAGS) /dll $(LINK_DBG) /opt:ref /opt:icf /def:$(CONFIG_LIB).def $^ $(GENERICLIBS) /out:$@ /pdb:$(CONFIG_LIB).pdb /mapinfo:exports
+
+$(CONFIG_LIB)_d.lib: $(CONFIG_LIB)_d.dll
+
+$(CONFIG_LIB)_d.dll: $(CONFIG_OBJS_D)
+	$(LD) $(DEBUG_LDFLAGS) /dll /opt:ref /opt:icf /def:$(CONFIG_LIB).def $^ $(GENERICLIBS) /out:$@ /pdb:$(CONFIG_LIB)_d.pdb /mapinfo:exports
+
+$(MFAPI_LIB).lib: $(MFAPI_LIB).dll
+
+$(MFAPI_LIB).dll: $(MFAPI_OBJS) $(MFAPI_LIB).res
+	$(LD) $(LDFLAGS) $(LINK_DBG) /opt:ref /opt:icf /pdb:$(MFAPI_LIB).pdb /DLL /def:$(MFAPI_LIB).def $^ $(GENERICLIBS) /out:$@
+
+$(MFAPI_LIB).res: $(RCFILE)
+	$(RC) /Fo$@ $(RCFLAGS) $(D)FILE_TYPE=$(TYPE_DLL) $(D)FILE_NAME=\"$(MFAPI_LIB).dll\" $(D)FILE_DESCRIPTION="PRODUCT_NAME \" api interface dll\"" $(WIN_RCFILE)
+
+
+#
+# FIXME: the tests target almost certainly hasn't been tried for a while
+#
+tests: $(TARGET_LIB)_static.lib $(UNIT_TESTS)
+	@echo "  ** All Unit Tests Passed **  "
+
+tester: tester.exe $(HOTP_LIB_DIR) $(DR_LIB_DIR) $(CONFIG_DIR)/$(ENGINE)/$(HOTP_DEFS_FILE)
+	@echo run tester.exe w/ appropriate options
+	./$< -h
+
+# hmm phony does not seem to work for this?? so use FORCE workaround.
+%_test: %_test.exe $(TESTER_EXES) $(HOTP_LIB_DIR) $(DR_LIB_DIR) $(CONFIG_DIR)/$(ENGINE)/$(HOTP_DEFS_FILE) FORCE
+	@echo Executing $@:
+	WIN_DYNAMORIO_HOME=$(WIN_DYNAMORIO_HOME) ./$<
+
+FORCE:
+
+check_hotp_dir:
+	@(if [ ! -d $(DYNAMORIO_HOTPATCH)/build/pkg ]; then \
+	echo "DYNAMORIO_HOTPATCH must be defined (or at ..) and built!"; \
+	false; fi)
+
+check_dr_dir:
+	@(if [ ! -f $(DYNAMORIO_HOME)/exports/x86_win32_rel/dynamorio.dll ]; then \
+	echo "core must be built!"; \
+	false; fi)
+
+$(DYNAMORIO_HOTPATCH)/test/hotp-test.lib: check_hotp_dir
+
+$(DR_LIB_DIR): check_dr_dir
+	mkdir -p $(DR_LIB_DIR)
+	cp -f $(DYNAMORIO_HOME)/exports/x86_win32_rel/*.dll $(DR_LIB_DIR)
+
+$(HOTP_LIB_DIR): $(DR_LIB_DIR) check_hotp_dir
+	cp -f $(DYNAMORIO_HOTPATCH)/test/hotp-test.dll .
+	mkdir -p $(HOTP_LIB_DIR)
+	cp -f $(DYNAMORIO_HOTPATCH)/build/pkg/liveshield-?????/lib/$(ENGINE)/* $(HOTP_LIB_DIR)
+
+$(CONFIG_DIR):
+	mkdir -p $(CONFIG_DIR)
+
+$(CONFIG_DIR)/$(ENGINE): $(CONFIG_DIR)
+	mkdir -p $@
+
+$(CONFIG_DIR)/$(ENGINE)/$(HOTP_DEFS_FILE): check_hotp_dir $(HOTP_LIB_DIR) $(CONFIG_DIR)/$(ENGINE)
+	cp $(DYNAMORIO_HOTPATCH)/build/pkg/liveshield-?????/*-defs-*.cfg $@
+
+%_test.exe: %_test.obj $(LIB_OBJS_D)
+	$(LD) $(LDFLAGS) $^ $(TEST_LDFLAGS) $(GENERICLIBS) /out:$@
+
+tester.obj: tester.c check_hotp_dir
+	$(CC) $(DEBUG_CFLAGS) /c $< /Fo$@
+
+tester.exe: tester.obj policy_staticd.lib $(DYNAMORIO_HOTPATCH)/test/hotp-test.lib
+	$(LD) /OPT:NOREF user32.lib tester.obj policy_staticd.lib $(WIN_DYNAMORIO_HOTPATCH)/test/hotp-test.lib /out:$@
+
+tester_%.exe: tester.obj policy_staticd.lib $(DYNAMORIO_HOTPATCH)/test/hotp-test.lib
+	$(LD) /OPT:NOREF user32.lib tester.obj policy_staticd.lib $(WIN_DYNAMORIO_HOTPATCH)/test/hotp-test.lib /out:$@
+
+
+clean:
+	$(RM) -r *.exe *.obj *.lib *.dll *~ *.exp *.pch *.idb *.pdb *.ilk test.mfp test2.mfp testcr.txt utils.tst $(DR_LIB_DIR) $(HOTP_MODES_FILE) $(CONFIG_DIR) tester.out* test-modes-2.cfg events.h
diff --git a/libutil/config.c b/libutil/config.c
new file mode 100644
index 000000000..846bc0f99
--- /dev/null
+++ b/libutil/config.c
@@ -0,0 +1,2175 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ *
+ * config.c: general configuration interface
+ *
+ */
+
+#include "share.h"
+#include "config.h"
+#include "processes.h"
+#include "string.h"
+#include "parser.h"
+#include <stdio.h>
+
+
+#ifndef UNIT_TEST
+
+
+void
+configpath_to_registry_path(WCHAR *path)
+{
+    UINT i=0;
+
+    if (NULL == path)
+        return;
+
+    for (i = 0; i < wcslen(path); i++)
+        if(CONFIG_PATH_SEPARATOR == path[i])
+            path[i] = L'\\';
+}
+
+
+DWORD
+get_key_handle(HKEY *key, HKEY parent, const WCHAR *path, 
+               BOOL absolute, DWORD flags)
+{
+    WCHAR keyname[MAX_PATH];
+
+    DO_ASSERT(key != NULL);
+
+    if (path == NULL) {
+        wcsncpy(keyname, CONFIGURATION_ROOT_REGISTRY_KEY, MAX_PATH);
+    }
+    else if (parent == NULL && !absolute) {
+        _snwprintf(keyname, MAX_PATH, L"%s\\%s",
+                   CONFIGURATION_ROOT_REGISTRY_KEY, path);
+    }
+    else {
+        wcsncpy(keyname, path, MAX_PATH);
+    }
+
+    DO_DEBUG(DL_VERB,
+             printf("get_key_handle using %S as keyname\n", keyname);
+             );
+
+    configpath_to_registry_path(keyname);
+
+    return RegCreateKeyEx(parent == NULL ? DYNAMORIO_REGISTRY_HIVE : parent,
+                          keyname, 0, NULL, REG_OPTION_NON_VOLATILE,
+                          platform_key_flags() | flags,
+                          NULL, key, NULL);
+}
+
+ConfigGroup *
+get_child(const WCHAR *name, ConfigGroup *c)
+{
+    ConfigGroup *cur = NULL;
+
+    if (c == NULL)
+        return NULL;
+
+    for(cur = c->children; NULL != cur; cur = cur->next) {
+        if (0 == wcsicmp(name, cur->name))
+            return cur;
+    }
+
+    return NULL;
+}
+
+void
+remove_child(const WCHAR *child, ConfigGroup *config)
+{
+    ConfigGroup *cur = NULL, *prev = NULL;
+
+    if (NULL == config)
+        return;
+
+    for(cur = config->children; NULL != cur; cur = cur->next) {
+        if (0 == wcsicmp(child, cur->name)) {
+            if (NULL == prev)
+                config->children = cur->next;
+            else
+                prev->next = cur->next;
+
+            if (config->lastchild == cur)
+                config->lastchild = prev;
+
+            free_config_group(cur);
+
+            return;
+        }
+        prev = cur;
+    }
+}
+
+BOOL
+count_params(ConfigGroup *c)
+{
+    int i = 0;
+    NameValuePairNode *cur;
+    for(cur = c->params; NULL != cur; cur = cur->next)
+        i++;
+    return i;
+}
+
+BOOL
+is_param(const WCHAR *name, ConfigGroup *c)
+{
+    NameValuePairNode *cur;
+
+    if (c == NULL)
+        return FALSE;
+
+    for(cur = c->params; NULL != cur; cur = cur->next) {
+        if (0 == wcsicmp(name, cur->name))
+            return TRUE;
+    }
+
+    return FALSE;
+}
+
+ConfigGroup *
+new_config_group(const WCHAR *name)
+{
+    ConfigGroup *c = (ConfigGroup *)malloc(sizeof(ConfigGroup));
+    c->next = NULL;
+    c->children = NULL;
+    c->lastchild = NULL;
+    c->params = NULL;
+    c->should_clear = FALSE;
+    c->name = (NULL == name) ? NULL : wcsdup(name);
+    
+    return c;
+}
+
+ConfigGroup *
+copy_config_group(ConfigGroup *config, BOOL deep)
+{
+    NameValuePairNode *tmpnvp;
+    ConfigGroup *tmpcfg;
+
+    ConfigGroup *c = new_config_group(config->name);
+    c->should_clear = config->should_clear;
+    tmpnvp = config->params;
+    while (tmpnvp != NULL) {
+        set_config_group_parameter(c, tmpnvp->name, tmpnvp->value);
+        tmpnvp = tmpnvp->next;
+    }
+
+    if (deep) {
+        tmpcfg = config->children;
+        while (tmpcfg != NULL) {
+            add_config_group(c, copy_config_group(tmpcfg, deep));
+            tmpcfg = tmpcfg->next;
+        }
+    }
+
+    return c;
+}
+
+void
+remove_children(ConfigGroup *config)
+{
+    ConfigGroup *cur = config->children, *prev;
+    
+    while(NULL != cur) {
+        prev = cur;
+        cur = cur->next;
+        free_config_group(prev);
+    }
+
+    config->children = NULL;
+    config->lastchild = NULL;
+}
+
+NameValuePairNode *
+new_nvp_node(const WCHAR *name)
+{
+    NameValuePairNode *nvp = 
+        (NameValuePairNode *)malloc(sizeof(NameValuePairNode));
+    nvp->next = NULL;
+    nvp->name = wcsdup(name);
+    nvp->value = NULL;
+
+    return nvp;
+}
+
+void
+free_nvp(NameValuePairNode *nvpn)
+{
+    if (nvpn->name != NULL)
+        free(nvpn->name);
+
+    if (nvpn->value != NULL)
+        free(nvpn->value);
+
+    free(nvpn);
+}
+
+void
+free_config_group(ConfigGroup *config)
+{
+    NameValuePairNode *nvpn = config->params, *tmp;
+
+    remove_children(config);
+
+    while(nvpn != NULL) {
+        tmp = nvpn->next;
+        free_nvp(nvpn);
+        nvpn = tmp;
+    }
+
+    if (config->name != NULL)
+        free(config->name);
+
+    free(config);
+}
+
+NameValuePairNode *
+get_nvp_node(ConfigGroup *config, const WCHAR *name)
+{
+    NameValuePairNode *nvpn = config->params;
+    while (NULL != nvpn && 0 != wcsicmp(nvpn->name, name))
+        nvpn = nvpn->next;
+    return nvpn;
+}
+
+WCHAR *
+get_config_group_parameter(ConfigGroup *config, const WCHAR *name)
+{
+    NameValuePairNode *nvpn = get_nvp_node(config, name);
+    return (NULL == nvpn) ? NULL : nvpn->value;
+}
+
+NameValuePairNode *
+add_nvp_node(ConfigGroup *config, const WCHAR *name)
+{
+    NameValuePairNode *nvpn = new_nvp_node(name);
+    nvpn->next = config->params;
+    config->params = nvpn;
+    return nvpn;
+}
+
+void
+set_config_group_parameter(ConfigGroup *config, 
+                           const WCHAR *name, const WCHAR *value)
+{
+    NameValuePairNode *nvpn = get_nvp_node(config, name);
+
+    if (NULL == nvpn) {
+        nvpn = add_nvp_node(config, name);
+    }
+
+    if (NULL != nvpn->value) {
+        free(nvpn->value);
+        nvpn->value = NULL;
+    }
+
+    nvpn->value = (NULL == value) ? NULL : wcsdup(value);
+}
+
+void
+remove_config_group_parameter(ConfigGroup *config, const WCHAR *name)
+{
+    NameValuePairNode *cur = NULL, *prev = NULL;
+    
+    if (NULL == config || NULL == name)
+        return;
+
+    for(cur = config->params; NULL != cur; cur = cur->next) {
+        if (0 == wcsicmp(cur->name, name)) {
+            if (NULL == prev)
+                config->params = cur->next;
+            else
+                prev->next = cur->next;
+
+            free_nvp(cur);
+
+            return;
+        }
+        prev = cur;
+    }
+}
+
+BOOL
+get_config_group_parameter_bool(ConfigGroup *config, const WCHAR *name)
+{
+    WCHAR *val = get_config_group_parameter(config, name);
+    return (val != NULL && 0 == wcscmp(val, L"TRUE"));
+}
+
+int
+get_config_group_parameter_int(ConfigGroup *config, const WCHAR *name)
+{
+    WCHAR *val = get_config_group_parameter(config, name);
+    return val == NULL ? 0 : _wtoi(val);
+}
+
+void
+set_config_group_parameter_bool(ConfigGroup *config, const WCHAR *name,
+                                BOOL value)
+{
+    set_config_group_parameter(config, name, value ? L"TRUE" : L"FALSE");
+}
+
+void
+set_config_group_parameter_int(ConfigGroup *config, const WCHAR *name,
+                               int value)
+{
+    WCHAR buf[MAX_PATH];
+    _snwprintf(buf, MAX_PATH, L"%d", value);
+    set_config_group_parameter(config, name, buf);
+}
+
+void
+set_config_group_parameter_ascii(ConfigGroup *config, const WCHAR *name,
+                                 char *value)
+{
+    WCHAR buf[MAX_PATH];
+    _snwprintf(buf, MAX_PATH, L"%S", value);
+    buf[MAX_PATH-1] = L'\0';
+    set_config_group_parameter(config, name, buf);
+}
+
+void
+set_config_group_parameter_scrambled(ConfigGroup *config, const WCHAR *name,
+                                     const WCHAR *value)
+{
+    /* this swaps high and low bytes of WCHAR. obviously not strong,
+     *  just meant to be something other than plaintext. */    
+    WCHAR buf[MAX_PATH];
+    UINT i = 0;
+    while (value[i] != 0 && i < MAX_PATH - 1) {
+        buf[i] = (WCHAR)
+            (((value[i] & 0x00ff) << 8) + ((value[i] & 0xff00) >> 8));
+        i++;
+    }
+    buf[i] = L'\0';
+    set_config_group_parameter(config, name, buf);
+}
+
+void
+get_config_group_parameter_scrambled(ConfigGroup *config, const WCHAR *name,
+                                     WCHAR *buffer, UINT maxchars)
+{
+    WCHAR *value = get_config_group_parameter(config, name);
+    UINT i = 0;
+
+    if (value == NULL) {
+        buffer [0] = L'\0';
+        return;
+    }
+
+    while (value[i] != 0 && i < maxchars - 1) {
+        buffer[i] = (WCHAR)
+            (((value[i] & 0x00ff) << 8) + ((value[i] & 0xff00) >> 8));
+        i++;
+    }
+    buffer[i] = L'\0';
+}
+
+
+void
+add_config_group(ConfigGroup *parent, ConfigGroup *new_child)
+{
+    if (NULL != get_child(new_child->name, parent)) {
+        DO_DEBUG(DL_WARN,
+                 printf("adding multiple child: %S\n", new_child->name);
+                 );
+    }
+
+    if (parent->children == NULL)
+        parent->children = new_child;
+
+    if (parent->lastchild != NULL)
+        parent->lastchild->next = new_child;
+
+    parent->lastchild = new_child;
+    new_child->next = NULL;
+}
+
+typedef DWORD (*custom_config_read_handler)(ConfigGroup *config,
+                                            const WCHAR *name,
+                                            const WCHAR *value);
+
+#define CURRENT_MODES_VERSION 42000
+#define MAX_MODES_FILE_SIZE (64*1024)
+
+DWORD
+custom_hotp_modes_read_handler(ConfigGroup *config,
+                               const WCHAR *name,
+                               const WCHAR *value)
+{
+    char modes_file[MAX_MODES_FILE_SIZE];
+    WCHAR modes_path[MAX_PATH];
+    WCHAR parambuf[MAX_PATH];
+    WCHAR valbuf[MAX_PATH];
+    DWORD res;
+    unsigned int needed = 0;
+    BOOL done = FALSE;
+    char *modes_line;
+    ConfigGroup *hotp_config;
+
+    if (name == NULL || value == NULL)
+        return ERROR_SUCCESS;
+
+    /* read in modes file */
+    _snwprintf(modes_path, MAX_PATH, L"%s\\%d\\%S",
+               value, CURRENT_MODES_VERSION, HOTP_MODES_FILENAME);
+    modes_path[MAX_PATH - 1] = L'\0';
+
+    res = read_file_contents(modes_path, modes_file, 
+                             MAX_MODES_FILE_SIZE, &needed);
+
+    /* if the modes file isn't there, no worries; assume no modes */
+    if (ERROR_FILE_NOT_FOUND == res)
+        return ERROR_SUCCESS;
+
+    if (ERROR_SUCCESS != res)
+        return res;
+
+    if (needed > MAX_MODES_FILE_SIZE - 2)
+        return ERROR_INSUFFICIENT_BUFFER;
+    
+    /* create child config group */
+    hotp_config = new_config_group(L_DYNAMORIO_VAR_HOT_PATCH_MODES);
+    
+    /* modes_file has contents, now sscanf and set all nvp. */
+    modes_line = parse_line_sep(modes_file, ':', &done, 
+                                parambuf, valbuf, MAX_PATH);
+    DO_DEBUG(DL_VERB,
+             printf("hotp modes first line %S:%S\n", parambuf, valbuf);
+             );
+
+    /* expect num lines first */
+    if (valbuf[0] != '\0')
+        return ERROR_PARSE_ERROR;
+
+    while (!done) {
+        modes_line = parse_line_sep(modes_line, ':', &done, 
+                                    parambuf, valbuf, MAX_PATH);
+        if (done)
+            break;
+        set_config_group_parameter(hotp_config, parambuf, valbuf);
+    }
+        
+    add_config_group(config, hotp_config);
+
+    return ERROR_SUCCESS;
+}
+
+custom_config_read_handler
+get_custom_config_read_handler(const WCHAR *name)
+{
+    if (name == NULL)
+        return NULL;
+    
+    if (0 == wcscmp(name, L_DYNAMORIO_VAR_HOT_PATCH_MODES)) {
+        return (&custom_hotp_modes_read_handler);
+    }
+
+    return NULL;
+}
+
+static DWORD
+read_config_group_from_registry(HKEY parent, ConfigGroup **configptr,
+                                const WCHAR *name, BOOL recursive)
+{
+    HKEY config_key = NULL;
+    ConfigGroup *config = NULL;
+    WCHAR keyname[MAX_PATH];
+    WCHAR keyvalue[MAX_PARAM_LEN];
+    DWORD idx, keySz, valSz, res = ERROR_SUCCESS;
+    FILETIME writetime;
+    custom_config_read_handler handler;
+
+    if (name == NULL) {
+        config_key = parent;
+    }
+    else {
+        WCHAR translated_name[MAX_PATH];
+        wcsncpy(translated_name, name, MAX_PATH);
+        configpath_to_registry_path(translated_name);
+        res = RegOpenKeyEx(parent, translated_name, 0,
+                           platform_key_flags() | KEY_READ,
+                           &config_key);
+        if (res != ERROR_SUCCESS)
+            goto read_config_out;
+    }
+
+    config = new_config_group(name);
+
+    /* first read in all values */
+    for (idx = 0; /* TRUE */; idx++) {
+        keySz = MAX_PATH;
+        valSz = sizeof(keyvalue);
+        res = RegEnumValue(config_key, idx, keyname, &keySz,
+                           0, NULL, (LPBYTE) keyvalue, &valSz);
+        if (res == ERROR_NO_MORE_ITEMS) {
+            res = ERROR_SUCCESS;
+            break;
+        }
+        else if (res != ERROR_SUCCESS) {
+            goto read_config_out;
+        }
+
+        if (NULL != (handler = get_custom_config_read_handler(keyname))) {
+            res = handler(config, keyname, keyvalue);
+            if (res != ERROR_SUCCESS)
+                goto read_config_out;
+        }
+        else {
+            set_config_group_parameter(config, keyname, keyvalue);
+        }
+    }
+    
+    /* and read in all children (if desired) */
+    for (idx = 0; recursive; idx++) {
+        ConfigGroup *child_config;
+
+        keySz = MAX_PATH;
+        res = RegEnumKeyEx(config_key, idx, keyname, &keySz,
+                           0, NULL, NULL, &writetime);
+        if (res == ERROR_NO_MORE_ITEMS) {
+            res = ERROR_SUCCESS;
+            break;
+        }
+        else if (res != ERROR_SUCCESS) {
+            goto read_config_out;
+        }
+        
+        res = read_config_group_from_registry(config_key, &child_config,
+                                              keyname, recursive);
+        if (res != ERROR_SUCCESS)
+            return res;
+
+        add_config_group(config, child_config);
+    }
+
+    *configptr = config;
+
+ read_config_out:
+    
+    if (NULL != config_key && NULL != name)
+        RegCloseKey(config_key);
+
+    return res;
+}
+
+DWORD
+read_config_group(ConfigGroup **configptr, const WCHAR *name, 
+                  BOOL read_children_recursively)
+{
+    HKEY rootkey;
+    DWORD res = ERROR_SUCCESS;
+
+    DO_ASSERT(configptr != NULL);
+    DO_ASSERT(name != NULL);
+    *configptr = NULL;
+
+    res = get_key_handle(&rootkey, NULL, NULL, FALSE, KEY_READ);
+    if (res != ERROR_SUCCESS)
+        return res;
+    
+    res = read_config_group_from_registry(rootkey, configptr, name, 
+                                          read_children_recursively);
+
+    RegCloseKey(rootkey);
+
+    return res;
+    
+}
+
+typedef DWORD (*custom_config_write_handler)(HKEY parent,
+                                             ConfigGroup *config,
+                                             ConfigGroup *config_parent);
+
+
+DWORD
+custom_hotp_modes_write_handler(HKEY parent, ConfigGroup *config, 
+                                ConfigGroup *config_parent)
+{
+    /* need to write modes file path to key, and the modes file
+     * contents. */
+    DWORD res;
+    WCHAR modes_file[MAX_PATH];
+    WCHAR modes_key[MAX_PATH];
+    char modes_file_contents[MAX_MODES_FILE_SIZE];
+    char modes_file_line[MAX_PATH];
+    BOOL changed = FALSE;
+    NameValuePairNode *nvpn;
+
+    /* the modes file name is based on the parent ConfigGroup; if the
+     *  parent config is the root, then the modes file goes in the
+     *  top-level config dir, otherwise it goes in an app-specific
+     *  config dir. */
+    if (0 == wcscmp(L_PRODUCT_NAME, config_parent->name)) {
+        _snwprintf(modes_key, MAX_PATH, L"%s\\config", get_dynamorio_home());
+    }
+    else {
+        _snwprintf(modes_key, MAX_PATH, L"%s\\config\\%s", 
+                   get_dynamorio_home(), config_parent->name);
+    }
+    modes_key[MAX_PATH - 1] = L'\0';    
+
+    /* now write modes file content; name=patch ID, value=mode */
+    _snprintf(modes_file_contents, MAX_MODES_FILE_SIZE,
+              "%d\n", count_params(config));
+    modes_file_contents[MAX_MODES_FILE_SIZE - 1] = '\0';
+    for (nvpn = config->params; NULL != nvpn; nvpn = nvpn->next) {
+        _snprintf(modes_file_line, MAX_PATH, "%S:%S\n", 
+                  nvpn->name, nvpn->value);
+        modes_file_line[MAX_PATH - 1] = '\0';
+        strncat(modes_file_contents, modes_file_line, 
+                MAX_MODES_FILE_SIZE - strlen(modes_file_contents));
+        modes_file_contents[MAX_MODES_FILE_SIZE - 1] = '\0';
+    }
+
+    if (strlen(modes_file_contents) > MAX_MODES_FILE_SIZE - 2) {
+        return ERROR_INSUFFICIENT_BUFFER;
+    }
+
+    /* first, mkdir -p */
+    _snwprintf(modes_file, MAX_PATH, L"%s\\%d\\%S",
+               modes_key, CURRENT_MODES_VERSION, HOTP_MODES_FILENAME);
+    modes_file[MAX_PATH - 1] = L'\0';
+    ensure_directory_exists_for_file(modes_file);
+
+    /* and then write file */
+    res = write_file_contents_if_different(modes_file,
+                                           modes_file_contents,
+                                           &changed);
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    /* finally, write the filename to the modes key */
+    return write_reg_string(parent, 
+                            L_DYNAMORIO_VAR_HOT_PATCH_MODES,
+                            modes_key);
+}
+
+custom_config_write_handler
+get_custom_config_write_handler(ConfigGroup *config)
+{
+    if (config == NULL || config->name == NULL)
+        return NULL;
+
+    if (0 == wcscmp(config->name, L_DYNAMORIO_VAR_HOT_PATCH_MODES)) {
+        return (&custom_hotp_modes_write_handler);
+    }
+
+    return NULL;
+}
+
+DWORD
+write_config_group_to_registry(HKEY parent, ConfigGroup *config, 
+                               ConfigGroup *parent_config)
+{
+    HKEY config_key = NULL;
+    NameValuePairNode *nvpn;
+    DWORD res = ERROR_SUCCESS;
+    custom_config_write_handler handler = NULL;
+
+    if (NULL == config->name) {
+        config_key = parent;
+    }
+    else if (NULL != (handler = get_custom_config_write_handler(config))) {
+        res = handler(parent, config, parent_config);
+        if (res != ERROR_SUCCESS)
+            goto write_config_out;
+        /* only continue down the chain if custom-handled */
+        if (NULL != config->next) {
+            res = write_config_group_to_registry(parent,
+                                                 config->next,
+                                                 parent_config);
+        }
+        goto write_config_out;
+    }
+    else {
+        /* open or create */
+        res = get_key_handle(&config_key, parent, config->name, FALSE,
+                             KEY_WRITE|KEY_ENUMERATE_SUB_KEYS);
+        if (res != ERROR_SUCCESS)
+            goto write_config_out;
+    }
+    
+    /* write the NVP's */
+    for (nvpn = config->params; NULL != nvpn; nvpn = nvpn->next) {
+        /* hook for forced deletion of a key */
+        if (0 == wcscmp(nvpn->value, L_DELETE_PARAMETER_KEY)) {
+            RegDeleteValue(config_key, nvpn->name);
+            res = ERROR_SUCCESS;
+        }
+        else {
+            res = write_reg_string(config_key, nvpn->name, nvpn->value);
+        }
+        if (res != ERROR_SUCCESS)
+            goto write_config_out;
+    }
+    
+    /* write the children */
+    if (NULL != config->children) {
+        res = write_config_group_to_registry(config_key, 
+                                             config->children, 
+                                             config);
+        if (res != ERROR_SUCCESS)
+            goto write_config_out;
+    }
+    
+    /* continue down the chain */
+    if (NULL != config->next) {
+        res = write_config_group_to_registry(parent,
+                                             config->next,
+                                             parent_config);
+        if (res != ERROR_SUCCESS)
+            goto write_config_out;
+    }
+
+ write_config_out:
+
+    if (NULL != config_key && NULL != config->name)
+        RegCloseKey(config_key);
+
+    return res;
+}
+
+/* 
+ * this enforces that the key tree matches the config group exactly,
+ * e.g., all keys/values are deleted that do not appear in 'filter' 
+ * 
+ * this complexity is necessary in order to allow 
+ * write_config_group_to_registry to be atomic, in the sense that 
+ * config information is updated on a key-by-key atomic basis.
+ */
+DWORD
+recursive_delete_key(HKEY parent, const WCHAR *keyname, ConfigGroup *filter)
+{
+    HKEY subkey = NULL;
+    WCHAR subkeyname[MAX_PATH];
+    FILETIME writetime;
+    DWORD res, keySz, idx;
+    ConfigGroup *child;
+
+    res = get_key_handle(&subkey, parent, keyname, TRUE, KEY_WRITE|KEY_READ);
+    if (res != ERROR_SUCCESS)
+        goto recursive_delete_out;
+
+    /* and read in all children (if desired) */
+    for (idx = 0;;) {
+        /* note that since deleting, we always pass index 0 */
+        keySz = MAX_PATH;
+        res = RegEnumKeyEx(subkey, idx, subkeyname, &keySz,
+                           0, NULL, NULL, &writetime);
+        if (res == ERROR_NO_MORE_ITEMS) {
+            res = ERROR_SUCCESS;
+            break;
+        }
+        else if (res != ERROR_SUCCESS) {
+            goto recursive_delete_out;
+        }
+
+        /* note that recursive_delete_key will only delete the entire
+         *  subkey if it's not a child (eg, if the 'filter' parameter
+         *  is NULL) */
+        child = get_child(subkeyname, filter);
+        res = recursive_delete_key(subkey, subkeyname, child);
+        if (res != ERROR_SUCCESS)
+            goto recursive_delete_out;
+        
+        /* increment the index if we're not deleting the key */
+        if (NULL != child)
+            idx++;
+    }
+
+    if (NULL != filter) {
+        /* prune values too */
+        for (idx = 0;;) {
+            keySz = MAX_PATH;
+            res = RegEnumValue(subkey, idx, subkeyname, &keySz,
+                               0, NULL, NULL, NULL);
+            if (res == ERROR_NO_MORE_ITEMS) {
+                res = ERROR_SUCCESS;
+                break;
+            }
+            else if (res != ERROR_SUCCESS) {
+                goto recursive_delete_out;
+            }
+
+            /* don't delete if the param exists, OR if this is a custom
+             * handled name and the child exists. */
+            if (is_param(subkeyname, filter) ||
+                (get_child(subkeyname, filter) && 
+                 get_custom_config_read_handler(subkeyname))) {
+                /* if we're not deleting, increment the index */
+                idx++;
+            }
+            else {
+                res = RegDeleteValue(subkey, subkeyname);
+                if (res != ERROR_SUCCESS)
+                    goto recursive_delete_out;
+            }
+        }
+    }
+    
+    RegCloseKey(subkey);
+    subkey = NULL;
+
+    /* only delete the key itself if it's not being filtered out. */
+    if (filter == NULL) {
+        /* PR 244206: we assume we're only going to recursively delete keys in
+         * HKLM\Software\Company
+         */
+        res = delete_product_key(parent, keyname);
+    }
+
+ recursive_delete_out:
+
+    if (subkey != NULL)
+        RegCloseKey(subkey);
+
+    return res;
+}
+
+DWORD
+write_config_group(ConfigGroup *config)
+{
+    HKEY rootkey;
+    DWORD res;
+
+    res = get_key_handle(&rootkey, NULL, NULL, FALSE, 
+                         KEY_WRITE|KEY_ENUMERATE_SUB_KEYS);
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    res = write_config_group_to_registry(rootkey, config, NULL);
+
+    /* prune if necessary */
+    if (ERROR_SUCCESS == res && config->should_clear) {
+        res = recursive_delete_key(rootkey, config->name, config);
+    }
+
+    RegCloseKey(rootkey);
+
+    return res;
+}
+
+
+/* single parameter config functions */
+
+DWORD
+set_config_parameter(const WCHAR *path, BOOL absolute, 
+                     const WCHAR *name, const WCHAR *value)
+{
+    HKEY rootkey;
+    DWORD res;
+
+    res = get_key_handle(&rootkey, NULL, path, absolute, KEY_WRITE);
+    if (res != ERROR_SUCCESS)
+        return res;
+    
+    res = write_reg_string(rootkey, name, value);
+    
+    RegCloseKey(rootkey);
+
+    return res;   
+}
+
+DWORD
+get_config_parameter(const WCHAR *path, BOOL absolute, 
+                     const WCHAR *name, WCHAR *value, int maxlen)
+{
+    HKEY rootkey;
+    DWORD res;
+
+    res = get_key_handle(&rootkey, NULL, path, absolute, KEY_READ);
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    res = read_reg_string(rootkey, name, value, maxlen);
+
+    RegCloseKey(rootkey);
+
+    return res;   
+}
+
+DWORD
+read_reg_string(HKEY subkey, const WCHAR *keyname, WCHAR *value, int valchars)
+{
+    DWORD len = valchars*sizeof(WCHAR);
+    return RegQueryValueEx(subkey, keyname, 0, NULL,
+                           (LPBYTE) value, &len);
+}
+
+/* if value is NULL, the key will be deleted */
+DWORD 
+write_reg_string(HKEY subkey, const WCHAR *keyname, const WCHAR *value)
+{
+    if (value)
+        return RegSetValueEx(subkey, keyname, 0, REG_SZ, (LPBYTE) value,
+                             (wcslen(value)+1)*sizeof(WCHAR));
+    else 
+        return RegDeleteValue(subkey, keyname);
+}
+
+
+/* process identification routines */
+
+
+/* tries both with and without no_strip */
+ConfigGroup *
+get_qualified_config_group(ConfigGroup *config, 
+                           const WCHAR *exename, const WCHAR *cmdline)
+{
+    WCHAR qname[MAX_PATH];
+    ConfigGroup *c;
+
+    /* need to try both with and without NO_STRIP! */
+    _snwprintf(qname, MAX_PATH, L"%s-", exename);
+    if (get_commandline_qualifier(cmdline, 
+                                  qname + wcslen(qname), 
+                                  MAX_PATH - wcslen(qname), FALSE)) {
+        c = get_child(qname, config);
+        if (NULL != c)
+            return c;
+    }
+
+    _snwprintf(qname, MAX_PATH, L"%s-", exename);
+    if (get_commandline_qualifier(cmdline,
+                                  qname + wcslen(qname), 
+                                  MAX_PATH - wcslen(qname), TRUE)) {
+        c = get_child(qname, config);
+        if (NULL != c)
+            return c;
+    }
+
+    return NULL;
+}
+
+BOOL
+is_parent_of_qualified_config_group(ConfigGroup *config)
+{
+    WCHAR *run_under = NULL;
+
+    if (config == NULL)
+        return FALSE;
+    run_under = get_config_group_parameter(config, L_DYNAMORIO_VAR_RUNUNDER);
+    if (run_under == NULL)
+        return FALSE;
+    
+    return TEST(RUNUNDER_COMMANDLINE_DISPATCH, _wtoi(run_under));
+}
+
+ConfigGroup *
+get_process_config_group(ConfigGroup *config, int pid)
+{
+    WCHAR buf[MAX_PATH];
+    DWORD res;
+    ConfigGroup *c;
+
+    res = get_process_name(pid, buf, MAX_PATH);
+
+    if (res != ERROR_SUCCESS)
+        return NULL;
+    
+    wcstolower(buf);
+    c = get_child(buf, config);
+
+    if (c == NULL || is_parent_of_qualified_config_group(c)) {
+        WCHAR cmdlbuf[MAX_PATH];
+        ConfigGroup *qualified;
+        get_process_cmdline(pid, cmdlbuf, MAX_PATH);
+        qualified = get_qualified_config_group(config, buf, cmdlbuf);
+        if (NULL != qualified)
+            return qualified;
+    }
+    
+    return c;
+}
+
+
+/******************
+ * list functions *
+ ******************/
+
+
+/* 
+ * given a ;-separated list and a filename, return a pointer to 
+ *  the filename in the list, if it appears. comparisons are 
+ *  case insensitive and independent of path; eg, 
+ *     get_entry_location("c:\\foo\\bar.dll;blah;...", "D:\\Bar.DLL")
+ *  would return a pointer to the beginning of the list.
+ */
+WCHAR *
+get_entry_location(const WCHAR *list, const WCHAR *filename, WCHAR separator)
+{
+    WCHAR *lowerlist, *lowername, *entry;
+
+    wcstolower(lowerlist = wcsdup(list));
+    wcstolower(lowername = wcsdup(get_exename_from_path(filename)));
+
+    entry = wcsstr(lowerlist, lowername);
+
+    while (NULL != entry) {
+        WCHAR last = *(entry + wcslen(lowername));
+
+        /* make sure it's not just a substring */
+        if ((last == separator || last == L'\0') &&
+            (entry == lowerlist || *(entry - 1) == separator ||
+             *(entry - 1) == L'\\' || *(entry - 1) == L'/')) {
+            /* everything's cool; now put the path back in and translate */
+            WCHAR *cur = lowerlist;
+            while (NULL != wcschr(cur, separator) &&
+                   wcschr(cur, separator) < entry)
+                cur = wcschr(cur, separator) + 1;
+            entry = ((WCHAR *) list) + (cur - lowerlist);
+            break;
+        }
+        else {
+            entry = wcsstr(entry + 1, lowername);
+        }
+    }
+
+    free(lowername);
+    free(lowerlist);
+
+    return entry;
+}
+
+BOOL
+is_in_file_list(const WCHAR *list, const WCHAR *filename, WCHAR separator)
+{
+    return NULL != get_entry_location(list, filename, separator);
+}
+
+/* returns a new list which needs to be freed, and frees old list */
+WCHAR *
+add_to_file_list(WCHAR *list, const WCHAR *filename, 
+                 BOOL check_for_duplicates, BOOL add_to_front,
+                 BOOL overwrite_existing, WCHAR separator)
+{
+    WCHAR *new_list;
+    UINT new_list_size;
+
+    if (NULL == list || 0 == wcslen(list))
+        return wcsdup(filename);
+
+    if (overwrite_existing)
+        remove_from_file_list(list, filename, separator);
+
+    if (check_for_duplicates && is_in_file_list(list, filename, separator))
+        return list;
+
+    new_list_size = wcslen(list) + wcslen(filename) + 2;
+
+    new_list = (WCHAR *)malloc(sizeof(WCHAR) * new_list_size);
+
+    if (wcslen(list) == 0) {
+        wcsncpy(new_list, filename, new_list_size);
+    }
+    else {
+        _snwprintf(new_list, new_list_size, L"%s%c%s",
+                   add_to_front ? filename : list, 
+                   separator,
+                   add_to_front ? list : filename);
+    }
+
+    free(list);
+
+    return new_list;
+}
+
+/* provide these helpers to avoid multiple libc problems */
+WCHAR *
+new_file_list(UINT initial_chars)
+{
+    WCHAR *list = (WCHAR *)malloc(sizeof(WCHAR) * (1+initial_chars));
+    *list = L'\0';
+    return list;
+}
+
+void
+free_file_list(WCHAR *list)
+{
+    free(list);
+}
+
+/* removes all occurrences */
+void
+remove_from_file_list(WCHAR *list, const WCHAR *filename, WCHAR separator)
+{
+    WCHAR *entry, *end_of_entry;
+
+    entry = get_entry_location(list, filename, separator);
+    if (NULL == entry)
+        return;
+
+    end_of_entry = wcschr(entry, separator);
+
+    if (NULL == end_of_entry) {
+        if (entry == list)
+            *entry = L'\0';
+        else
+            *(entry - 1) = L'\0';
+    }
+    else {
+        /* note the whole wcslen(end_of_entry) to get the null
+         *  terminator (since we're doing end_of_entry + 1) 
+         *
+         * note that memmove handles appropriately the case
+         *  where the src and dest regions overlap. */
+        memmove(entry, end_of_entry + 1, 
+                (wcslen(end_of_entry)) * sizeof(WCHAR));
+    }
+
+    /* recurse for multiple occurrences */
+    remove_from_file_list(list, filename, separator);
+}
+
+#define BLACK 0
+#define WHITE 1
+
+BOOL
+filter(WCHAR *list, const WCHAR *filter, DWORD black_or_white, 
+       BOOL check_only, WCHAR separator)
+{
+    WCHAR *working_list, *cur, *next;
+    BOOL satisfied = TRUE, remove_entry;
+
+    if (list == NULL || wcslen(list) == 0)
+        return TRUE;
+
+    next = cur = working_list = wcsdup(list);
+
+    do {
+
+        if (NULL != next)
+            next = wcschr(cur, separator);
+
+        if (NULL != next)
+            *next = L'\0';
+        
+        cur = get_exename_from_path(cur);
+        remove_entry = 
+            !(is_in_file_list(filter, cur, separator) ^ (BLACK == black_or_white));
+
+        if (remove_entry) {
+            if (check_only)
+                return FALSE;
+            else 
+                remove_from_file_list(list, cur, separator);
+
+            satisfied = FALSE;
+        }
+
+        cur = next + 1;
+
+    } while (NULL != next);
+
+    free(working_list);
+
+    return satisfied;
+}
+
+BOOL
+blacklist_filter(WCHAR *list, const WCHAR *blacklist, 
+                 BOOL check_only, WCHAR separator)
+{
+    return filter(list, blacklist, BLACK, check_only, separator);
+}
+
+BOOL
+whitelist_filter(WCHAR *list, const WCHAR *whitelist, 
+                 BOOL check_only, WCHAR separator)
+{
+    return filter(list, whitelist, WHITE, check_only, separator);
+}
+
+
+
+/* appinit key */
+
+DWORD
+set_autoinjection_ex(BOOL inject, DWORD flags, 
+                     const WCHAR *blacklist, const WCHAR *whitelist,
+                     /* OUT */ DWORD *list_error, 
+                     const WCHAR *custom_preinject_name,
+                     /* OUT */ WCHAR *current_list, UINT maxchars)
+{
+    WCHAR curlist[MAX_PARAM_LEN];
+    WCHAR preinject_name[MAX_PATH];
+    WCHAR *list;
+    DWORD res;
+    BOOL list_ok_ = TRUE;
+    BOOL using_system32 = using_system32_for_preinject(custom_preinject_name);
+
+    res = get_config_parameter(INJECT_ALL_KEY_L, TRUE, INJECT_ALL_SUBKEY_L, 
+                               curlist, MAX_PARAM_LEN);
+    if (res != ERROR_SUCCESS) {
+        /* if it's not there, then we create it */
+        curlist[0] = L'\0';
+    }
+
+    list = new_file_list(wcslen(curlist) + 1);
+    wcsncpy(list, curlist, wcslen(curlist) + 1);
+
+    if (NULL != current_list)
+        wcsncpy(current_list, curlist, maxchars);
+
+    if (using_system32 && 
+        TEST(APPINIT_SYS32_CLEAR_OTHERS, flags)){
+        /* if we're using system32, and the clear flag is set, clear it */
+        curlist[0] = L'\0';
+    }
+
+    if (NULL == custom_preinject_name) {
+        res = get_preinject_name(preinject_name, MAX_PATH);
+        if (res != ERROR_SUCCESS)
+            return res;
+    }
+    else {
+        wcsncpy(preinject_name, custom_preinject_name, MAX_PATH);
+    }
+
+    /* if using system32, make sure to copy the DLL there from
+     *  the standard place (or remove it if we're turning off) */
+    if (using_system32) {
+        WCHAR src_path[MAX_PATH], dst_path[MAX_PATH];
+        
+        if (NULL == custom_preinject_name) {
+            res = get_preinject_path(src_path, MAX_PATH, TRUE, TRUE);
+            if (res != ERROR_SUCCESS)
+                return res;
+            wcsncat(src_path, L"\\" L_EXPAND_LEVEL(INJECT_DLL_NAME), 
+                    MAX_PATH - wcslen(src_path));
+        } else {
+            wcsncpy(src_path, custom_preinject_name, MAX_PATH);
+        }
+        
+        get_preinject_path(dst_path, MAX_PATH, FALSE, TRUE);
+        if (res != ERROR_SUCCESS)
+            return res;
+        wcsncat(dst_path, L"\\" L_EXPAND_LEVEL(INJECT_DLL_NAME), 
+                MAX_PATH - wcslen(dst_path));
+        
+        if (inject) {
+            /* We used to only copy if (!file_exists(dst_path))
+             * but it seems that we should clobber to avoid
+             * upgrade issues, at risk of messing up another product
+             * w/ a dll of the same name.
+             */
+            CopyFile(src_path, dst_path, FALSE/*don't fail if exists*/);
+        }
+        else {
+            /* FIXME: do we want to use delete_file_rename_in_use?
+             * this should only be called at uninstall or by
+             *  tools users, so there shouldn't be any problem 
+             *  leaving one .tmp file around...
+             * alternatively, we could try renaming to a path in
+             *  our installation directory, which would be 
+             *  wiped out on installation. however, it gets
+             *  complicated if our installation folder is on
+             *  a different volume.
+             * another option is to rename to %SYSTEM32%\..\TEMP,
+             *  which should always exist i think...
+             */
+            DeleteFile(dst_path);
+        }
+
+        /* now we want just the name, since in system32 */
+        if (get_exename_from_path(preinject_name) != NULL) {
+            wcsncpy(preinject_name, get_exename_from_path(preinject_name),
+                    BUFFER_SIZE_ELEMENTS(preinject_name));
+        }
+    }
+        
+    if (inject) {
+        BOOL force_overwrite;
+        WCHAR *old = get_entry_location(list, preinject_name, 
+                                        APPINIT_SEPARATOR_CHAR);
+
+        /* first, if there's something there, make sure it exists. if
+         *  not, remove it before proceeding (to ensure overwrite)
+         * cf case 4053 */
+        if (NULL != old) {
+            if (using_system32) {
+                /* PR 232765: we want to replace any full path w/ filename */
+                force_overwrite = true;
+            } else {
+                WCHAR *firstsep, tmpbuf[MAX_PATH];
+                wcsncpy(tmpbuf, old, MAX_PATH);
+                firstsep = wcschr(tmpbuf, APPINIT_SEPARATOR_CHAR);
+                if (NULL != firstsep)
+                    *firstsep = L'\0';
+                force_overwrite = !file_exists(tmpbuf);
+            }
+        }
+        else {
+            /* force overwrite if someone cared enough to set one of these */
+            force_overwrite = 
+                TEST((APPINIT_FORCE_TO_FRONT | APPINIT_FORCE_TO_BACK), flags);
+        }
+        /* always overwrite if asked to */
+        force_overwrite = force_overwrite | TEST(APPINIT_OVERWRITE, flags);
+        
+        /* add !TEST(APPINIT_FORCE_TO_BACK, flags) so that we favor adding
+         *  to the front if neither are set. */
+        list = add_to_file_list(list, preinject_name, TRUE,
+                                TEST(APPINIT_FORCE_TO_FRONT, flags) ||
+                                ! TEST(APPINIT_FORCE_TO_BACK, flags),
+                                force_overwrite, APPINIT_SEPARATOR_CHAR);
+    }
+    else {
+        remove_from_file_list(list, preinject_name, APPINIT_SEPARATOR_CHAR);
+    }
+
+
+    if (TEST(APPINIT_USE_WHITELIST, flags)) {
+        if (NULL == whitelist)
+            res = ERROR_INVALID_PARAMETER;
+        else
+            list_ok_ = whitelist_filter(list, whitelist, 
+                                        TEST(APPINIT_CHECK_LISTS_ONLY, flags),
+                                        APPINIT_SEPARATOR_CHAR);
+    }
+    else if (TEST(APPINIT_USE_BLACKLIST, flags)) {
+        /* else if since whitelist subsumes blacklist */
+        if (NULL == blacklist)
+            res = ERROR_INVALID_PARAMETER;
+        else
+            list_ok_ = blacklist_filter(list, blacklist, 
+                                        TEST(APPINIT_CHECK_LISTS_ONLY, flags),
+                                        APPINIT_SEPARATOR_CHAR);
+    }
+
+    if (list_error != NULL && !list_ok_)
+        *list_error = ERROR_LIST_VIOLATION;
+
+    if (res == ERROR_INVALID_PARAMETER)
+        return res;
+
+    if (!list_ok_ && TEST(APPINIT_BAIL_ON_LIST_VIOLATION, flags))
+        remove_from_file_list(list, preinject_name, APPINIT_SEPARATOR_CHAR);
+
+
+    /* now, system32 flag checks */
+    if (using_system32) {
+
+        /* not yet supported */
+        if (TEST(APPINIT_SYS32_USE_LENGTH_WORKAROUND, flags))
+            return ERROR_INVALID_PARAMETER;
+        
+        if (wcslen(list) > APPINIT_SYSTEM32_LENGTH_LIMIT) {
+
+            if (list_error != NULL)
+                *list_error = ERROR_LENGTH_VIOLATION;
+
+            if (TEST(APPINIT_SYS32_FAIL_ON_LENGTH_ERROR, flags)) {
+                remove_from_file_list(list, preinject_name, 
+                                      APPINIT_SEPARATOR_CHAR);
+            }
+            else {
+                /* truncate, if the flags specify it. */
+                if (TEST(APPINIT_SYS32_TRUNCATE, flags)) {
+                    list[APPINIT_SYSTEM32_LENGTH_LIMIT] = L'\0';
+                }
+            }
+        }
+    }
+
+    /* only write if it's changed */
+    if (0 != wcscmp(list, curlist)) {
+        res = set_config_parameter(INJECT_ALL_KEY_L, TRUE, 
+                                   INJECT_ALL_SUBKEY_L, list);
+        if (res != ERROR_SUCCESS)
+            return res;
+    }
+
+    free_file_list(list);
+
+    return ERROR_SUCCESS;
+}
+
+DWORD
+set_custom_autoinjection(const WCHAR *preinject, DWORD flags)
+{
+    return set_autoinjection_ex(TRUE, flags, NULL, NULL, NULL, preinject, NULL, 0);
+}
+
+DWORD
+set_autoinjection()
+{
+    return set_autoinjection_ex(TRUE, 0, NULL, NULL, NULL, NULL, NULL, 0);
+}
+
+DWORD
+unset_autoinjection()
+{
+    return set_autoinjection_ex(FALSE, 0, NULL, NULL, NULL, NULL, NULL, 0);
+}
+
+/* NOTE: that this returns the current status -- which on NT
+ *  is not necessarily the actual status, which is cached by the 
+ *  os at boot time. 
+ * FIXME: add a helper method for determining the status of 
+ *  appinit that is being used for the current boot session. */
+BOOL
+is_autoinjection_set()
+{
+    WCHAR list[MAX_PARAM_LEN], preinject[MAX_PATH];
+    DWORD res;
+    
+    res = get_config_parameter(INJECT_ALL_KEY_L, TRUE, INJECT_ALL_SUBKEY_L, 
+                               list, MAX_PARAM_LEN);
+    if (res != ERROR_SUCCESS)
+        return FALSE;
+
+    res = get_preinject_name(preinject, MAX_PATH);
+    if (res != ERROR_SUCCESS)
+        return FALSE;
+
+    return is_in_file_list(list, preinject, APPINIT_SEPARATOR_CHAR);
+}
+
+/* FIXME - once we fully support Vista should use get_platform */
+BOOL
+is_vista()
+{
+    OSVERSIONINFO info;
+    info.dwOSVersionInfoSize = sizeof(info);
+
+    if (GetVersionEx(&info)) {
+        return (info.dwMajorVersion == 6);
+    } else {
+        DO_ASSERT(false);
+    }
+    return FALSE;
+}
+
+DWORD
+set_loadappinit_value(DWORD value)
+{
+    HKEY rootkey;
+    DWORD res;
+
+    if (!is_vista())
+        return ERROR_UNSUPPORTED_OS;
+
+    res = get_key_handle(&rootkey, INJECT_ALL_HIVE, INJECT_ALL_KEY_L,
+                         TRUE, KEY_WRITE);
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    return RegSetValueEx(rootkey, INJECT_ALL_LOAD_SUBKEY_L, 0, REG_DWORD,
+                         (LPBYTE) &value, sizeof(value));
+}
+
+DWORD
+set_loadappinit()
+{
+    return set_loadappinit_value(1);
+}
+    
+
+DWORD
+unset_loadappinit()
+{
+    return set_loadappinit_value(0);
+}
+
+BOOL
+is_loadappinit_set()
+{
+    HKEY rootkey;
+    DWORD value;
+    DWORD size = sizeof(value);
+    DWORD res;
+
+    res = get_key_handle(&rootkey, INJECT_ALL_HIVE, INJECT_ALL_KEY_L,
+                         TRUE, KEY_READ);
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    res = RegQueryValueEx(rootkey, INJECT_ALL_LOAD_SUBKEY_L, NULL, NULL,
+                          (LPBYTE) &value, &size);
+    
+    return (res == ERROR_SUCCESS && value != 0);
+}
+
+/* Deletes the product eventlog.  If no one else (nodemgr/pe) is using the eventlog also
+ * deletes the base key and the eventlog file. */
+DWORD
+destroy_eventlog()
+{
+    DWORD res, res2;
+    /* PR 244206: we don't need the wow64 flag since not HKLM\Software
+     * (if we do want it, need to add on create/open as well, including
+     * elm.c, which requires linking w/ utils.c)
+     */
+    res = RegDeleteKey(EVENTLOG_HIVE, L_EVENT_SOURCE_SUBKEY);
+    res2 = RegDeleteKey(EVENTLOG_HIVE, L_EVENT_LOG_SUBKEY);
+    if (res2 == ERROR_SUCCESS) {
+        /* we deleted the top level key (which means it had no subkeys left) which means
+         * no one else is using our eventlog, we go ahead and free the file. */
+        WCHAR *file = is_vista() ? L_EVENT_FILE_NAME_VISTA : L_EVENT_FILE_NAME_PRE_VISTA;
+        WCHAR *truncate_start;
+        WCHAR file_exp[MAX_PATH];
+        ExpandEnvironmentStrings(file, file_exp, BUFFER_SIZE_ELEMENTS(file_exp));
+        NULL_TERMINATE_BUFFER(file_exp);
+        if (file_exists(file_exp)) {
+            /* If in use by the eventlog can't delete or re-name */
+            if (!DeleteFile(file_exp))
+                delete_file_on_boot(file_exp);
+        }
+
+        /* It appears the generated file is usually truncated to 8.3 by the eventlog
+         * program (though apparently not always since some of my machines also have
+         * the full name).  Will try to delete the truncated to 8.3 versions. */        
+        truncate_start = wcsrchr(file_exp, L'\\');
+        if (truncate_start != NULL) {
+            WCHAR *type = wcsrchr(file_exp, L'.');
+            truncate_start += 8;
+            if (type != NULL && truncate_start < type) {
+                WCHAR tmp_buf[MAX_PATH];
+                wcsncpy(tmp_buf, type, BUFFER_SIZE_ELEMENTS(tmp_buf));
+                NULL_TERMINATE_BUFFER(tmp_buf);
+                wcsncpy(truncate_start, tmp_buf,
+                        BUFFER_SIZE_ELEMENTS(file_exp)-(truncate_start-file_exp));
+                NULL_TERMINATE_BUFFER(file_exp);
+                /* If in use by the eventlog can't delete or re-name */
+                if (!DeleteFile(file_exp))
+                    delete_file_on_boot(file_exp);
+            }
+        }
+    }
+    return res;
+}
+
+DWORD
+create_eventlog(const WCHAR *dll_path)
+{
+    HKEY eventlog_key;
+    HKEY eventsrc_key;
+    WCHAR *wvalue;
+    DWORD dvalue;
+    DWORD res;
+
+    res = get_key_handle(&eventlog_key, EVENTLOG_HIVE, L_EVENT_LOG_SUBKEY,
+                         TRUE, KEY_WRITE);
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    if (is_vista()) {
+        wvalue = L_EVENT_FILE_NAME_VISTA;
+    } else {
+        wvalue = L_EVENT_FILE_NAME_PRE_VISTA;
+    }
+    /* REG_EXPAND_SZ since we use %systemroot% in the path which needs
+     * to be expanded */
+    res = RegSetValueEx(eventlog_key, L_EVENT_FILE_VALUE_NAME, 0,
+                        REG_EXPAND_SZ, (LPBYTE) wvalue, 
+                        (wcslen(wvalue)+1)*sizeof(WCHAR));
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    dvalue = EVENT_MAX_SIZE;
+    res = RegSetValueEx(eventlog_key, L_EVENT_MAX_SIZE_NAME, 0, REG_DWORD,
+                        (LPBYTE) &dvalue, sizeof(dvalue));
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    dvalue = EVENT_RETENTION;
+    res = RegSetValueEx(eventlog_key, L_EVENT_RETENTION_NAME, 0, REG_DWORD,
+                        (LPBYTE) &dvalue, sizeof(dvalue));
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    res = get_key_handle(&eventsrc_key, EVENTLOG_HIVE, L_EVENT_SOURCE_SUBKEY,
+                         TRUE, KEY_WRITE);
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    dvalue = EVENT_TYPES_SUPPORTED;
+    res = RegSetValueEx(eventsrc_key, L_EVENT_TYPES_SUPPORTED_NAME, 0,
+                        REG_DWORD, (LPBYTE) &dvalue, sizeof(dvalue));
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    dvalue = EVENT_CATEGORY_COUNT;
+    res = RegSetValueEx(eventsrc_key, L_EVENT_CATEGORY_COUNT_NAME, 0,
+                        REG_DWORD, (LPBYTE) &dvalue, sizeof(dvalue));
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    res = RegSetValueEx(eventsrc_key, L_EVENT_CATEGORY_FILE_NAME, 0, REG_SZ,
+                        (LPBYTE) dll_path, (wcslen(dll_path)+1)*sizeof(WCHAR));
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    res = RegSetValueEx(eventsrc_key, L_EVENT_MESSAGE_FILE, 0, REG_SZ,
+                        (LPBYTE) dll_path, (wcslen(dll_path)+1)*sizeof(WCHAR));
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    return ERROR_SUCCESS;
+}
+
+/* Copies the two drearlyhelper?.dlls located in "dir" to system32.
+ * They are only needed on win2k, but it's up to the caller to check
+ * for that.
+ */
+DWORD
+copy_earlyhelper_dlls(const WCHAR *dir)
+{
+    WCHAR src_path[MAX_PATH], dst_path[MAX_PATH];
+    DWORD len;
+
+    /* We copy drearlyhelp2.dll first, to be on the safe side.
+     * drearlyhelp1.dll has a dependency on drearlyhelp2.dll, so if #1 exists
+     * and number 2 doesn't the loader will raise an error (and we're pre-image
+     * entry point so it becomes a "process failed to initialize" error).
+     */
+    _snwprintf(src_path, BUFFER_SIZE_ELEMENTS(src_path),
+               L"%s\\%s", dir, L_EXPAND_LEVEL(INJECT_HELPER_DLL2_NAME));
+    NULL_TERMINATE_BUFFER(src_path);
+
+    /* Get system32 path */
+    len = GetSystemDirectory(dst_path, MAX_PATH);
+    if (len == 0)
+        return GetLastError();
+    wcsncat(dst_path, L"\\" L_EXPAND_LEVEL(INJECT_HELPER_DLL2_NAME), 
+            BUFFER_SIZE_ELEMENTS(dst_path) - wcslen(dst_path));
+        
+    /* We could check file_exists(dst_path) but better to just
+     * clobber so we can upgrade nicely (at risk of clobbering
+     * some other product's same-name dll)
+     */
+    if (!CopyFile(src_path, dst_path, FALSE/*don't fail if exists*/))
+        return GetLastError();
+
+    _snwprintf(src_path, BUFFER_SIZE_ELEMENTS(src_path),
+               L"%s\\%s", dir, L_EXPAND_LEVEL(INJECT_HELPER_DLL1_NAME));
+    NULL_TERMINATE_BUFFER(src_path);
+
+    /* Get system32 path, again: we could cache it */
+    len = GetSystemDirectory(dst_path, MAX_PATH);
+    if (len == 0)
+        return GetLastError();
+    wcsncat(dst_path, L"\\" L_EXPAND_LEVEL(INJECT_HELPER_DLL1_NAME), 
+            BUFFER_SIZE_ELEMENTS(dst_path) - wcslen(dst_path));
+
+    if (!CopyFile(src_path, dst_path, FALSE/*don't fail if exists*/))
+        return GetLastError();
+
+    /* FIXME PR 232738: add a param for removing the files */
+
+    return ERROR_SUCCESS;
+}
+
+void
+dump_nvp(NameValuePairNode *nvpn)
+{
+    printf("%S=%S", nvpn->name,
+           nvpn->value == NULL ? L"<null>" : nvpn->value);
+}
+
+void
+dump_config_group(char *prefix, char *incr, ConfigGroup *c, BOOL traverse)
+{
+    NameValuePairNode *nvpn = NULL;
+    
+    printf("%sConfig Group: %S\n", prefix, 
+           c->name == NULL ? L"<null>" : c->name);
+    printf("%sshould_clear: %d\n", prefix, c->should_clear);
+    printf("%sparams:\n", prefix);
+    for(nvpn = c->params; NULL != nvpn; nvpn = nvpn->next) {
+        printf("%s%s%s", prefix, incr, incr);
+        dump_nvp(nvpn);
+        printf("\n");
+    }
+    if (c->children != NULL) {
+        char p2[MAX_PATH];
+        _snprintf(p2, MAX_PATH, "%s%s", prefix, incr);
+        dump_config_group(p2, incr, c->children, TRUE);
+    }
+    if (traverse && c->next != NULL) {
+        dump_config_group(prefix, incr, c->next, TRUE);
+    }
+}
+
+
+#else // ifdef UNIT_TEST
+
+
+
+int
+main()
+{
+    ConfigGroup *c, *c2, *c3, *c4, *cp;
+    DWORD res;
+    WCHAR sep = LIST_SEPARATOR_CHAR;
+
+    WCHAR *list1, *list2, *tmplist;
+
+    WCHAR buf[MAX_PARAM_LEN];
+
+    set_debuglevel(DL_INFO);
+    set_abortlevel(DL_WARN);
+
+    list1 = new_file_list(16);
+
+    /* basic list operations */
+    {
+        DO_ASSERT_WSTR_EQ(L"", list1);
+
+        list1 = add_to_file_list(list1, L"c:\\foo.dll", 
+                                 TRUE, TRUE, FALSE, sep);
+        DO_ASSERT_WSTR_EQ(L"c:\\foo.dll", list1);
+        
+        list1 = add_to_file_list(list1, L"C:\\bar.dll", 
+                                 TRUE, TRUE, FALSE, sep);
+        DO_ASSERT_WSTR_EQ(L"C:\\bar.dll;c:\\foo.dll", list1);
+        
+        list1 = add_to_file_list(list1, L"foo.dll", 
+                                 TRUE, TRUE, FALSE, sep);
+        DO_ASSERT_WSTR_EQ(L"C:\\bar.dll;c:\\foo.dll", list1);
+
+        list1 = add_to_file_list(list1, L"C:\\shr\\Foo.dll", 
+                                 TRUE, TRUE, FALSE, sep);
+        DO_ASSERT_WSTR_EQ(L"C:\\bar.dll;c:\\foo.dll", list1);
+
+        list1 = add_to_file_list(list1, L"C:\\shr\\Foo.dll", 
+                                 FALSE, TRUE, FALSE, sep);
+        DO_ASSERT_WSTR_EQ(L"C:\\shr\\Foo.dll;C:\\bar.dll;c:\\foo.dll", list1);
+
+        list1 = add_to_file_list(list1, L"d:\\gee.dll", 
+                                 TRUE, FALSE, FALSE, sep);
+        DO_ASSERT_WSTR_EQ(L"C:\\shr\\Foo.dll;C:\\bar.dll;c:\\foo.dll;d:\\gee.dll", list1);
+
+        list1 = add_to_file_list(list1, L"ar.dll", 
+                                 TRUE, FALSE, FALSE, sep);
+        DO_ASSERT_WSTR_EQ(L"C:\\shr\\Foo.dll;C:\\bar.dll;c:\\foo.dll;d:\\gee.dll;ar.dll", list1);
+    }
+    
+    /* used in tests below */
+    DO_ASSERT_WSTR_EQ(L"C:\\shr\\Foo.dll;C:\\bar.dll;c:\\foo.dll;d:\\gee.dll;ar.dll", list1);
+
+    /* remove */
+    {
+        tmplist = wcsdup(list1);
+        remove_from_file_list(tmplist, L"ar.dll", sep);
+        DO_ASSERT_WSTR_EQ(L"C:\\shr\\Foo.dll;C:\\bar.dll;c:\\foo.dll;d:\\gee.dll", tmplist);
+        
+        remove_from_file_list(tmplist, L"foo.dll", sep);
+        DO_ASSERT_WSTR_EQ(L"C:\\bar.dll;d:\\gee.dll", tmplist);
+        
+        free_file_list(tmplist);
+
+        tmplist = wcsdup(list1);
+        remove_from_file_list(tmplist, L"Ar.dll", sep);
+        DO_ASSERT_WSTR_EQ(L"C:\\shr\\Foo.dll;C:\\bar.dll;c:\\foo.dll;d:\\gee.dll", tmplist);
+
+        remove_from_file_list(tmplist, L"Foo.DLL", sep);
+        DO_ASSERT_WSTR_EQ(L"C:\\bar.dll;d:\\gee.dll", tmplist);
+    
+        free_file_list(tmplist);
+    }
+
+    /* path checking */
+    {
+        tmplist = wcsdup(list1);
+        tmplist = add_to_file_list(tmplist, 
+                                   L"C:\\Program Files\\blah\\blah\\foo.dll", 
+                                   TRUE, TRUE, TRUE, sep);
+        DO_ASSERT_WSTR_EQ(L"C:\\Program Files\\blah\\blah\\foo.dll;C:\\bar.dll;d:\\gee.dll;ar.dll", tmplist);
+        
+        free_file_list(tmplist);
+    }
+
+    /* list filters */
+    {
+        list2 = new_file_list(1);
+        list2 = add_to_file_list(list2, L"foo.dll", TRUE, TRUE, FALSE, sep);
+        list2 = add_to_file_list(list2, L"gee.dll", TRUE, TRUE, FALSE, sep);
+        
+        tmplist = wcsdup(list1);
+        DO_ASSERT(!blacklist_filter(tmplist, list2, TRUE, sep));
+        DO_ASSERT_WSTR_EQ(L"C:\\shr\\Foo.dll;C:\\bar.dll;c:\\foo.dll;d:\\gee.dll;ar.dll", tmplist);
+        
+        DO_ASSERT(!blacklist_filter(tmplist, list2, FALSE, sep));
+        DO_ASSERT_WSTR_EQ(L"C:\\bar.dll;ar.dll", tmplist);
+        
+        DO_ASSERT(blacklist_filter(tmplist, list2, TRUE, sep));
+        free_file_list(tmplist);
+        
+        tmplist = wcsdup(list1);
+        DO_ASSERT(!whitelist_filter(tmplist, list2, FALSE, sep));
+        DO_ASSERT_WSTR_EQ(L"C:\\shr\\Foo.dll;c:\\foo.dll;d:\\gee.dll", tmplist);
+        
+        DO_ASSERT(whitelist_filter(tmplist, list2, TRUE, sep));
+        
+        free_file_list(tmplist);
+
+        free_file_list(list2);
+    }
+
+    free_file_list(list1);
+
+    /* force to front */
+    {
+        list1 = L"home.dll;C:\\PROGRA~1\\DETERM~1\\SECURE~1\\lib\\DRPREI~1.DLL;foo.dll;bar.dll";
+        list2 = new_file_list(wcslen(list1) + 1);
+        wcsncpy(list2, list1, wcslen(list1) + 1);
+        list2 = add_to_file_list(list2, L"C:\\PROGRA~1\\DETERM~1\\SECURE~1\\lib\\DRPREI~1.DLL", TRUE, TRUE, TRUE, sep);
+        DO_ASSERT_WSTR_EQ(L"C:\\PROGRA~1\\DETERM~1\\SECURE~1\\lib\\DRPREI~1.DLL;home.dll;foo.dll;bar.dll", list2);
+        DO_ASSERT(wcslen(list1) == wcslen(list2));
+    }
+
+
+    /* autoinject (FIXME: needs more..) */
+    {
+        res = get_config_parameter(INJECT_ALL_KEY_L, TRUE, INJECT_ALL_SUBKEY_L, 
+                                   buf, MAX_PARAM_LEN);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        
+        res = set_config_parameter(INJECT_ALL_KEY_L, TRUE, 
+                                   INJECT_ALL_SUBKEY_L, 
+                                   L"foo.dll;bar.dll;home.dll");
+        DO_ASSERT(res == ERROR_SUCCESS);
+        
+
+        /* FIXME: should all full automated tests of all
+         *  APPINIT_FLAGS etc. */
+        //res = set_autoinjection_ex(TRUE, );
+        //DO_ASSERT(res == ERROR_SUCCESS);
+        
+
+        res = set_config_parameter(INJECT_ALL_KEY_L, TRUE, 
+                                   INJECT_ALL_SUBKEY_L, buf);
+        DO_ASSERT(res == ERROR_SUCCESS);
+    }
+
+    /* config group */
+    {
+        c = new_config_group(L"foo");
+        DO_ASSERT_WSTR_EQ(c->name, L"foo");
+
+        set_config_group_parameter(c, L"bar", L"wise");
+        
+        c2 = new_config_group(L"foo2");
+        set_config_group_parameter(c2, L"bar2", L"dumb");
+        set_config_group_parameter(c2, L"bar3", L"dumber");
+        add_config_group(c, c2);
+
+        c3 = new_config_group(L"foo4");
+        set_config_group_parameter(c3, L"bar4", L"eloquent");
+        add_config_group(c, c3);
+
+        c4 = new_config_group(L"foo5");
+        set_config_group_parameter(c4, L"bar5", L"dull");
+        add_config_group(c3,c4);
+        c->should_clear = TRUE;
+
+        DO_DEBUG(DL_VERB,
+                 dump_config_group("","  ",c,FALSE);
+                 );
+
+        res = write_config_group(c);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        DO_ASSERT_WSTR_EQ(c->name, L"foo");
+    }
+
+    /* copy config group */
+    {
+        ConfigGroup *cc, *copy = copy_config_group(c, TRUE);
+        DO_ASSERT_WSTR_EQ(copy->name, L"foo");
+
+        cc = get_child(L"foo4", copy);
+        DO_ASSERT(NULL != cc);
+        DO_ASSERT_WSTR_EQ(L"eloquent", get_config_group_parameter(cc, L"bar4"));
+
+        cc = get_child(L"foo5", cc);
+        DO_ASSERT(NULL != cc);
+        DO_ASSERT_WSTR_EQ(L"dull", get_config_group_parameter(cc, L"bar5"));
+
+        free_config_group(copy);
+
+        copy = copy_config_group(c, FALSE);
+        DO_ASSERT_WSTR_EQ(copy->name, L"foo");
+        DO_ASSERT_WSTR_EQ(L"wise", get_config_group_parameter(copy, L"bar"));
+
+        cc = get_child(L"foo4", copy);
+        DO_ASSERT(NULL == cc);
+
+        free_config_group(copy);
+    }
+
+    /* remove child */
+    {
+        DO_ASSERT(c2 == get_child(L"foo2", c));
+        remove_child(L"foo2", c);
+        DO_ASSERT(NULL == get_child(L"foo2", c));
+        
+        DO_ASSERT(c3 == get_child(L"foo4", c));
+        DO_ASSERT(c4 == get_child(L"foo5", c3));
+        
+        DO_ASSERT(c3 == get_child(L"Foo4", c));
+        DO_ASSERT(c3 == get_child(L"FOO4", c));
+        
+        /* make sure we don't die */
+        remove_child(L"foo2", c);
+    }
+
+    /* remove parameter */
+    {
+        set_config_group_parameter(c, L"testremove", L"sucks");
+        DO_ASSERT_WSTR_EQ(get_config_group_parameter(c, L"testremove"), 
+                          L"sucks");
+        DO_ASSERT_WSTR_EQ(get_config_group_parameter(c, L"bar"), L"wise");
+        remove_config_group_parameter(c, L"testremove");
+        DO_ASSERT(NULL == get_config_group_parameter(c, L"testremove"));
+        DO_ASSERT_WSTR_EQ(get_config_group_parameter(c, L"bar"), L"wise");
+
+        free_config_group(c);
+    }
+
+    /* read config */
+    {
+        res = read_config_group(&c, L"foo", TRUE);
+        DO_ASSERT_HANDLE(res == ERROR_SUCCESS, 
+                         printf("res=%d\n", res); 
+                         return -1;
+                         );
+        DO_ASSERT_WSTR_EQ(c->name, L"foo");        
+        DO_ASSERT(NULL != get_child(L"foo2", c));       
+        DO_ASSERT(NULL != get_child(L"foo4", c));
+        DO_ASSERT(NULL != get_child(L"foo5", get_child(L"foo4", c)));
+        
+        free_config_group(c);
+    }
+
+    /* read deep config */
+    {
+        res = read_config_group(&c, L"foo:foo2", TRUE);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        DO_DEBUG(DL_VERB,
+                 printf("c->name: >%S<\n", c->name);
+                 );
+        DO_ASSERT_WSTR_EQ(c->name, L"foo:foo2");
+        free_config_group(c);
+    }
+
+    /* read deeper config */
+    {
+        res = read_config_group(&c, L"foo:foo4:foo5", TRUE);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        DO_ASSERT_WSTR_EQ(c->name, L"foo:foo4:foo5");
+        DO_ASSERT_WSTR_EQ(get_config_group_parameter(c, L"bar5"), L"dull");
+        free_config_group(c);
+    }
+
+
+    /* test write */
+    {
+        res = read_config_group(&c, L"foo:foo4:foo5", TRUE);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        set_config_group_parameter(c, L"testwrite", L"rocks");
+        DO_ASSERT(get_config_group_parameter(c, L"testwrite") != NULL);
+        DO_ASSERT_WSTR_EQ(get_config_group_parameter(c, L"testwrite"), 
+                          L"rocks");
+        res = write_config_group(c);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        free_config_group(c);
+
+        res = read_config_group(&c, L"foo:foo4:foo5", TRUE);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        DO_ASSERT_WSTR_EQ(get_config_group_parameter(c, L"testwrite"), 
+                          L"rocks");
+        DO_ASSERT_WSTR_EQ(get_config_group_parameter(c, L"bar5"), L"dull");
+        free_config_group(c);
+    }
+
+    /* test write patches */
+    {
+        ConfigGroup *hp = new_config_group(L_DYNAMORIO_VAR_HOT_PATCH_MODES);
+
+        c = new_config_group(L"testapp.exe");
+        DO_ASSERT(c != NULL);
+        set_config_group_parameter(hp, L"TEST.0001", L"0");
+        set_config_group_parameter(hp, L"TEST.0002", L"1");
+        set_config_group_parameter(hp, L"MS06.019A", L"2");
+        add_config_group(c, hp);
+        hp = get_child(L_DYNAMORIO_VAR_HOT_PATCH_MODES, c);
+        DO_ASSERT(hp != NULL);
+        DO_ASSERT(get_config_group_parameter(hp, L"TEST.0001") != NULL);
+        DO_ASSERT_WSTR_EQ(get_config_group_parameter(hp, L"TEST.0001"), L"0");
+        res = write_config_group(c);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        free_config_group(c);
+
+        res = read_config_group(&c, L"testapp.exe", TRUE);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        hp = get_child(L_DYNAMORIO_VAR_HOT_PATCH_MODES, c);
+        DO_ASSERT(hp != NULL);
+        DO_ASSERT(get_config_group_parameter(hp, L"TEST.0001") != NULL);
+        DO_ASSERT_WSTR_EQ(get_config_group_parameter(hp, L"TEST.0001"), L"0");
+        DO_DEBUG(DL_VERB,
+                 dump_config_group("","  ",c,FALSE);
+                 );
+        free_config_group(c);
+    }
+
+    /* case insensitive read */
+    {
+        res = read_config_group(&c, L"FOO:FOO4:FOO5", TRUE);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        DO_ASSERT_WSTR_EQ(get_config_group_parameter(c, L"bar5"), L"dull");
+        
+        free_config_group(c);
+    }
+
+    /* write back with foo2 removed */
+    {
+        res = read_config_group(&c, L"foo", TRUE);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        DO_ASSERT_WSTR_EQ(c->name, L"foo");
+        c->should_clear = TRUE;
+
+        DO_ASSERT(NULL != get_child(L"foo2", c));
+        remove_child(L"foo2", c);
+        DO_ASSERT(NULL == get_child(L"foo2", c));
+
+        res = write_config_group(c);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        DO_ASSERT_WSTR_EQ(c->name, L"foo");
+
+        free_config_group(c);
+
+        /* read back and make sure foo2 is still gone */
+        res = read_config_group(&c, L"foo", TRUE);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        DO_ASSERT_WSTR_EQ(c->name, L"foo");
+        
+        DO_ASSERT(NULL == get_child(L"foo2", c));
+        DO_ASSERT(NULL != get_child(L"foo4", c));
+        DO_ASSERT(NULL != get_child(L"foo5", get_child(L"foo4", c)));
+        
+        free_config_group(c);
+    }
+
+    /* absolute config */
+    {
+        WCHAR buf[MAX_PATH];
+        res = set_config_parameter(L"Software\\Microsoft", 
+                                   TRUE, L"foo", L"bar");
+        DO_ASSERT(res == ERROR_SUCCESS);
+        
+        res = get_config_parameter(L"Software\\Microsoft", TRUE, L"foo",
+                                   buf, MAX_PATH);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        DO_ASSERT_WSTR_EQ(buf, L"bar");
+
+        res = set_config_parameter(L"Software\\Microsoft", 
+                                   TRUE, L"foo", L"rebar");
+        DO_ASSERT(res == ERROR_SUCCESS);
+        
+        res = get_config_parameter(L"Software\\Microsoft", TRUE, L"foo",
+                                   buf, MAX_PATH);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        DO_ASSERT_WSTR_EQ(buf, L"rebar");
+
+        res = set_config_parameter(L"Software\\Microsoft", TRUE, L"foo", NULL);
+        DO_ASSERT(res == ERROR_SUCCESS);
+
+        res = get_config_parameter(L"Software\\Microsoft", TRUE, L"foo",
+                                   buf, MAX_PATH);
+        DO_ASSERT(res != ERROR_SUCCESS);
+    }
+
+    /* qnames */
+    {
+        c = new_config_group(L"apps");
+        c2 = new_config_group(L"bar.exe");
+        c3 = new_config_group(L"bar.exe-bar");
+        c4 = new_config_group(L"foo.exe");
+
+        add_config_group(c, c2);
+        add_config_group(c, c3);
+        add_config_group(c, c4);
+    
+        set_config_group_parameter(c2, L"DYNAMORIO_RUNUNDER", L"48");
+        set_config_group_parameter(c3, L"DYNAMORIO_RUNUNDER", L"49");
+
+        cp = get_qualified_config_group(c, L"bar.exe", L"bar.exe /bar");
+        DO_ASSERT(cp != NULL);
+
+        cp = get_qualified_config_group(c, L"bar.exe", 
+                                        L"bar.exe /b /a /r");    
+        DO_ASSERT(cp != NULL);
+        
+        cp = get_qualified_config_group(c, L"bar.exe", L"bar.exe /c bar");    
+        DO_ASSERT(cp != NULL);
+        
+        cp = get_qualified_config_group(c, L"bar.exe", L"bar.exe /c Bar");    
+        DO_ASSERT(cp != NULL);
+        
+        cp = get_qualified_config_group(c, L"bar.exe", L"Bar.exe /c bar");    
+        DO_ASSERT(cp != NULL);
+    
+        DO_ASSERT(is_parent_of_qualified_config_group(c2));
+        DO_ASSERT(is_parent_of_qualified_config_group(c3));
+        DO_ASSERT(!is_parent_of_qualified_config_group(c4));
+
+        set_config_group_parameter(c4, L"DYNAMORIO_RUNUNDER", L"5");
+        DO_ASSERT(!is_parent_of_qualified_config_group(c4));
+    }
+
+    /* autoinject tests */
+    {
+        res = unset_autoinjection();
+        DO_ASSERT(res == ERROR_SUCCESS);
+
+        res = set_autoinjection();
+        DO_ASSERT(res == ERROR_SUCCESS);
+
+        res = unset_autoinjection();
+        DO_ASSERT(res == ERROR_SUCCESS);
+    }
+
+    printf("All Test Passed\n");
+
+    return 0;
+}
+
+#endif
diff --git a/libutil/config.h b/libutil/config.h
new file mode 100644
index 000000000..1f1be49df
--- /dev/null
+++ b/libutil/config.h
@@ -0,0 +1,299 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+
+/*
+ * 
+ * configuartion interface definitions
+ *
+ *
+ * this is intended to serve as a replacement for the cluttered
+ *  policy.c code for reading and writing core parameters. OOP
+ *  style has been kept in mind to allow easy porting to c++ 
+ *  if/when we decide to make that switch.
+ *
+ * usage:
+ *  a ConfigGroup is more or less equivalent to a registry key: it can
+ *  hold name-value pairs and 'children' ConfigGroups. ConfigGroup 
+ *  paths are ":" separated, and for registry purposes are assumed to
+ *  be rooted at HKLM/Software/Determina. the idea is that if we move
+ *  away from the registry (eg, due to moving to another platform or
+ *  to config files for core params), this interface should still be
+ *  usable, and the only change would be in {read,write}_config_group.
+ *
+ *  there are also direct-access single-parameter config
+ *  functions; these allow for arbitray registry read/write. however, 
+ *  unless otherwise specified with the 'absolute' parameter, these
+ *  are still based at the determina key. 
+ *
+ *
+ */
+
+#ifndef _DETERMINA_CONFIG_H_
+#define _DETERMINA_CONFIG_H_
+
+#ifdef __cplusplus
+extern "C"{
+#endif 
+
+#define MAX_PARAM_LEN 1024
+#define CONFIG_PATH_SEPARATOR L':'
+#define LIST_SEPARATOR_CHAR L';'
+#define APPINIT_SEPARATOR_CHAR L','
+#define CONFIGURATION_ROOT_REGISTRY_KEY L"Software\\"L_EXPAND_LEVEL(COMPANY_NAME)
+
+/* this provides a hook for forced parameter deletion, even if
+ * "should_clear" is FALSE. */
+#define L_DELETE_PARAMETER_KEY L"__DELETE_PARAMETER_KEY"
+
+
+/* note this does NOT include the null terminator: the limit
+ *  is 31 chars. */
+#define APPINIT_SYSTEM32_LENGTH_LIMIT  31
+
+typedef struct NameValuePairNode_ {
+    struct NameValuePairNode_ *next;
+    WCHAR *name;
+    WCHAR *value;
+} NameValuePairNode;
+
+typedef struct ConfigGroup_ {
+    struct ConfigGroup_ *next;
+    struct ConfigGroup_ *children;
+    /* used internally only for efficiency */
+    struct ConfigGroup_ *lastchild;
+    NameValuePairNode *params;
+    BOOL should_clear;
+    WCHAR *name;
+} ConfigGroup;
+
+
+/* group config management */
+
+DWORD
+get_key_handle(HKEY *key, HKEY parent, const WCHAR *path, 
+               BOOL absolute, DWORD flags);
+
+DWORD
+recursive_delete_key(HKEY parent, const WCHAR *keyname, ConfigGroup *filter);
+
+ConfigGroup *
+new_config_group(const WCHAR *name);
+
+ConfigGroup *
+copy_config_group(ConfigGroup *config, BOOL deep);
+
+/* name is the subkey name from the system registry root (e.g., 
+ *  HKLM/Software/Determina). */
+DWORD
+read_config_group(ConfigGroup **configptr, const WCHAR *name, 
+                  BOOL read_children_recursively);
+
+void
+set_should_clear(ConfigGroup *config, BOOL should_clear);
+
+void
+remove_children(ConfigGroup *config);
+
+void
+remove_child(const WCHAR *child, ConfigGroup *config);
+
+WCHAR *
+get_config_group_parameter(ConfigGroup *config, const WCHAR *name);
+
+void
+set_config_group_parameter(ConfigGroup *config, 
+                           const WCHAR *name, const WCHAR *value);
+
+void
+remove_config_group_parameter(ConfigGroup *config, const WCHAR *name);
+
+void
+add_config_group(ConfigGroup *parent, ConfigGroup *new_child);
+
+ConfigGroup *
+get_child(const WCHAR *name, ConfigGroup *c);
+
+DWORD
+write_config_group(ConfigGroup *config);
+
+void
+free_config_group(ConfigGroup *configptr);
+
+void
+dump_nvp(NameValuePairNode *nvpn);
+
+void
+dump_config_group(char *prefix, char *incr, ConfigGroup *c, BOOL traverse);
+
+BOOL
+get_config_group_parameter_bool(ConfigGroup *config, const WCHAR *name);
+
+int
+get_config_group_parameter_int(ConfigGroup *config, const WCHAR *name);
+
+void
+get_config_group_parameter_scrambled(ConfigGroup *config, const WCHAR *name,
+                                     WCHAR *buffer, UINT maxchars);
+
+void
+set_config_group_parameter_bool(ConfigGroup *config, const WCHAR *name,
+                                BOOL value);
+
+void
+set_config_group_parameter_int(ConfigGroup *config, const WCHAR *name,
+                               int value);
+
+void
+set_config_group_parameter_ascii(ConfigGroup *config, const WCHAR *name,
+                                 char *value);
+
+void
+set_config_group_parameter_scrambled(ConfigGroup *config, const WCHAR *name,
+                                     const WCHAR *value);
+
+
+/* single parameter config functions */
+
+DWORD
+set_config_parameter(const WCHAR *path, BOOL absolute, 
+                     const WCHAR *name, const WCHAR *value);
+
+DWORD
+get_config_parameter(const WCHAR *path, BOOL absolute, 
+                     const WCHAR *name, WCHAR *value, int maxlen);
+
+DWORD
+read_reg_string(HKEY subkey, const WCHAR *keyname, WCHAR *value, int valchars);
+
+/* if value is NULL, the value will be deleted */
+DWORD 
+write_reg_string(HKEY subkey, const WCHAR *keyname, const WCHAR *value);
+
+/* identifies processes relative to a ConfigGroup */
+ConfigGroup *
+get_process_config_group(ConfigGroup *config, int pid);
+
+BOOL
+is_parent_of_qualified_config_group(ConfigGroup *config);
+
+/* tries both with and without no_strip */
+ConfigGroup *
+get_qualified_config_group(ConfigGroup *config, 
+                           const WCHAR *exename, const WCHAR *cmdline);
+
+/* some list management and utility routines */
+/* all lists are ;-separated */
+/* comparisons are case insensitive */
+/* filename comparisons are independent of path */
+
+WCHAR *
+new_file_list(UINT initial_chars);
+
+void
+free_file_list(WCHAR *list);
+
+/* 
+ * given a ;-separated list and a filename, return a pointer to 
+ *  the filename in the list, if it appears. comparisons are 
+ *  case insensitive and independent of path; eg, 
+ *     get_entry_location("c:\\foo\\bar.dll;blah;...", "D:\\Bar.DLL")
+ *  would return a pointer to the beginning of the list.
+ */
+WCHAR *
+get_entry_location(const WCHAR *list, const WCHAR *filename, WCHAR separator);
+
+BOOL
+is_in_file_list(const WCHAR *list, const WCHAR *filename, WCHAR separator);
+
+/* frees the old list and returns a newly alloc'ed one */
+WCHAR *
+add_to_file_list(WCHAR *list, const WCHAR *filename, 
+                 BOOL check_for_duplicates, BOOL add_to_front, 
+                 BOOL overwrite_existing, WCHAR separator);
+
+void
+remove_from_file_list(WCHAR *list, const WCHAR *filename, WCHAR separator);
+
+BOOL
+blacklist_filter(WCHAR *list, const WCHAR *blacklist, 
+                 BOOL check_only, WCHAR separator);
+
+BOOL
+whitelist_filter(WCHAR *list, const WCHAR *whitelist, 
+                 BOOL check_only, WCHAR separator);
+
+DWORD
+set_autoinjection_ex(BOOL inject, DWORD flags, 
+                     const WCHAR *blacklist, 
+                     const WCHAR *whitelist, DWORD *list_error, 
+                     const WCHAR *custom_preinject_name, 
+                     WCHAR *current_list, UINT maxchars);
+
+DWORD
+set_custom_autoinjection(const WCHAR *preinject, DWORD flags);
+
+DWORD
+set_autoinjection();
+
+DWORD
+unset_autoinjection();
+
+BOOL
+is_autoinjection_set();
+
+DWORD
+set_loadappinit();
+
+DWORD
+unset_loadappinit();
+
+BOOL
+is_loadappinit_set();
+
+DWORD
+create_eventlog(const WCHAR *dll_path);
+
+DWORD
+destroy_eventlog();
+
+BOOL
+is_vista();
+
+DWORD
+copy_earlyhelper_dlls(const WCHAR *dir);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif _DETERMINA_CONFIG_H_
diff --git a/libutil/detach.c b/libutil/detach.c
new file mode 100644
index 000000000..cf34a3caa
--- /dev/null
+++ b/libutil/detach.c
@@ -0,0 +1,673 @@
+/* **********************************************************
+ * Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* note - include order matters */
+#include <tchar.h>
+#include "share.h"
+#include "win32/ntdll.h"  /* just for typedefs and defines */
+#include "processes.h"
+#include "win32/drmarker.h"
+#include <AccCtrl.h>
+#include <Aclapi.h>
+#include <stdio.h>
+
+/* detach.c(74) : warning C4055: 'type cast' : from data pointer 
+ *  'void *' to function pointer 'unsigned long (__stdcall *)(void *)'
+ * (cf. drmarker.h)
+ */
+#pragma warning(disable : 4055)
+
+#define PAGE_SIZE 0x1000
+#define MIN_ALLOCATION_SIZE 0x10000
+
+typedef NTSTATUS (NTAPI *NtCreateThreadType) (OUT PHANDLE ThreadHandle,
+                                              IN ACCESS_MASK DesiredAccess,
+                                              IN POBJECT_ATTRIBUTES ObjectAttributes,
+                                              IN HANDLE ProcessHandle,
+                                              OUT PCLIENT_ID ClientId,
+                                              IN PCONTEXT ThreadContext,
+                                              IN PUSER_STACK UserStack,
+                                              IN BOOLEAN CreateSuspended);
+NtCreateThreadType NtCreateThread = NULL;
+
+DWORD WINAPI dummy_func(LPVOID dummy_arg)
+{
+    return (DWORD)dummy_arg;
+}
+
+/* Gets kernel32!BaseThreadStartThunk which, unfortunately, isn't exported. */
+/* Returns NULL on error. NOTE - on Vista there is no kernel32!BaseThreadStartThunk,
+ * rather there is the moral equivalent ntdll!RtlUserThreadStart which is exported
+ * (so doesn't require this convoluted lookup). */
+static void*
+get_kernel_thread_start_thunk()
+{
+    static void* start_address = NULL;
+    DWORD platform = 0;
+
+    get_platform(&platform);
+    DO_ASSERT(platform != 0 && platform != PLATFORM_VISTA);
+    if (platform == PLATFORM_VISTA)
+        return NULL;
+
+    if (start_address == NULL) {
+        HANDLE hThread = CreateThread(NULL, 0, dummy_func, NULL,
+                                      CREATE_SUSPENDED, NULL);
+        if (hThread != NULL) {
+            CONTEXT cxt;
+            cxt.ContextFlags = CONTEXT_FULL;
+            if (GetThreadContext(hThread, &cxt)) {
+                /* FIXME - would be a little more elegant to have a lock
+                 * protecting start_address, but is not neccesary for
+                 * correctness as we'll always be writing the same value and
+                 * the actual write itself should be atomic.  There are places
+                 * in utils.c that really do need synchronization so if we
+                 * ever add some synchronization mechanisms to share module
+                 * might as well use here as well. */
+                start_address = (void *)cxt.Eip;
+#ifdef DEBUG
+                {
+                    TCHAR buf[MAX_PATH];
+                    MEMORY_BASIC_INFORMATION mbi;
+                    int i = 0;
+
+                    memset(buf, 0, sizeof(buf));
+                    VirtualQuery(start_address, &mbi, sizeof(mbi));
+                    /* a module handle is just the allocation base of the
+                     * module */
+                    GetModuleFileName((HMODULE)mbi.AllocationBase,
+                                      buf, BUFFER_SIZE_ELEMENTS(buf));
+                    NULL_TERMINATE_BUFFER(buf);
+                    /* can't believe there is no stristr */
+                    while(buf[i] != _T('\0')) {
+                        buf[i] = _totupper(buf[i]);
+                        i++;
+                    }
+                    DO_ASSERT(_tcsstr(buf, _T("KERNEL32.DLL")) != NULL);
+                }
+#endif
+            }
+            /* don't terminate, otherwise won't be cleaned up on 2k */
+            /* thread routine is empty so just resume */
+            ResumeThread(hThread);
+            CloseHandle(hThread);
+        }
+    }
+    DO_ASSERT(start_address != NULL);
+    return start_address;
+}
+
+/* returns NULL on error */
+/* FIXME - is similar to core create_thread, but uses API routines where
+ *     possible, could try to share. */
+/* NOTE - stack_reserve and stack commit must be multiples of PAGE_SIZE and reserve
+ *     should be at least 5 pages larger then commit */
+/* NOTE - if !target_api, target thread routine can't exit by by returning, instead it
+ *     must terminate itself.
+ * NOTE - caller or target thread routine is responsible for informing csrss (if
+ *    neccessary) and freeing the the thread stack (caller is informed of the stack
+ *    via OUT remote_stack arg).
+ * If arg_buf is non-null then the arg_buf_size bytes of arg_buf are copied onto the
+ * new thread's stack and pointer to them is passed as the argument to the start
+ * routine instead of arg.
+ * FIXME - on Vista we could be using NtCreateThreadEx (like all the api routines do),
+ *    but this still works fine and that would complicate argument handling.
+ */
+static HANDLE
+nt_create_thread(HANDLE hProcess, PTHREAD_START_ROUTINE start_addr,
+                 void *arg, const void *arg_buf, SIZE_T arg_buf_size,
+                 uint stack_reserve, uint stack_commit, bool suspended,
+                 uint *tid, bool target_api, void **remote_stack /* OUT */)
+{
+    HANDLE hThread = NULL;
+    USER_STACK stack = {0};
+    OBJECT_ATTRIBUTES oa;
+    CLIENT_ID cid;
+    CONTEXT context = {0};
+    uint num_commit_bytes, code;
+    unsigned long old_prot;
+    void *p;
+    SECURITY_DESCRIPTOR *sd = NULL;
+    DWORD platform = 0;
+    void *thread_arg = arg;
+    BOOL wow64 = is_wow64(hProcess);
+
+    if (NtCreateThread == NULL) {
+        /* FIXME - on Vista we could instead use NtCreateThreadEx which is what all
+         * the api routines use and requires less setup. But it also uses structs that
+         * we haven't fully reversed or found good documentation for. */
+        /* Don't need a lock, writing the pointer (4 byte write) is atomic. */
+        NtCreateThread = (NtCreateThreadType)
+            GetProcAddress(GetModuleHandle(L"ntdll.dll"), "NtCreateThread");
+        if (NtCreateThread == NULL)
+            goto error;
+    }
+
+    /* both stack size and stack reserve must be multiples of PAGE_SIZE */
+    DO_ASSERT((stack_reserve & (PAGE_SIZE - 1)) == 0);
+    DO_ASSERT((stack_commit & (PAGE_SIZE - 1)) == 0);
+    /* We stick a non-committed page on each end just to be safe and windows
+     * needs three pages at the end to properly handle end of expandable stack
+     * case (wants to pass exception back to the app on overflow, so needs some
+     * stack for that).  Plus we need an extra page for wow64 (PR 252008). */
+    DO_ASSERT(stack_reserve >= stack_commit + (5 + (wow64 ? 1 : 0) * PAGE_SIZE));
+
+    /* Use the security descriptor from the target process for creating the
+     * thread so that once created the thread will be able to open a full
+     * access handle to itself (xref case 2096). */
+    /* NOTES - tried many ways to impersonate based on target process token
+     * so we could just use the default and was unable to get anywhere with
+     * that.  Easiest thing to do here is just create a new security descriptor
+     * with a NULL (not empty) DACL [just InitializeSecurityDescriptor();
+     * SetSecurityDescriptorDacl()], but that's a privilege escalation
+     * problem (allows anybody full access to the thread)].  If we instead get
+     * the full security descriptor from the target process and try to use that
+     * the kernel complains that its a bad choice of owner.  What we do instead
+     * is get just the DACL and leave the rest empty (will be filled in with
+     * defaults during create thread).  Thus the security descriptor for the
+     * thread will end up having the owner, group, and SACL from this
+     * process and the DACL from the target.  Upshot is the the thread pseudo
+     * handle will have full permissions (from the DACL), but the owner will be
+     * us and, even though the handle we get back from CreateThread will be
+     * fully permissioned as we request, any subsequent attempts by us to
+     * OpenThread will fail since we aren't on the DACL.  We could always add
+     * ourselves to the DACL later or we can use the SE_DEBUG_PRIVILEGE to
+     * allow us to open it anyways.  Note if for some reason we want to view the
+     * SACL we need to enable the ACCESS_SYSTEM_SECURITY privilege when opening
+     * the handle.
+     * FIXME - we could instead build our own DACL combining the two, we could
+     * also try setting the owner/group after the thread is created if we
+     * really wanted to look like the target process thread, and could also
+     * start with a NULL sd and set the DACL later if want to match
+     * CreateThread as closely as possible.  If we do anything post system
+     * call should be sure to always create the thread suspended.
+     */
+    code = GetSecurityInfo(hProcess, SE_KERNEL_OBJECT,
+                           DACL_SECURITY_INFORMATION,
+                           NULL, NULL, NULL, NULL, &sd);
+    DO_ASSERT(code == ERROR_SUCCESS);
+ 
+    InitializeObjectAttributes(&oa, NULL, OBJ_CASE_INSENSITIVE, NULL, sd);
+  
+    stack.ExpandableStackBottom =
+        VirtualAllocEx(hProcess, NULL,
+                       /* we leave the top page MEM_FREE: we could reserve it instead
+                        * and then adjust our get_stack_bounds() core assert */
+                       stack_reserve - PAGE_SIZE,
+                       MEM_RESERVE, PAGE_READWRITE);
+    if (remote_stack != NULL)
+        *remote_stack = stack.ExpandableStackBottom;
+    if (stack.ExpandableStackBottom == NULL)
+        goto error;
+
+    /* We provide non-committed boundary page on each side of the stack just to
+     * be safe (note we will get a stack overflow exception if stack grows to
+     * 3rd to last page of this region (xpsp2)). */
+    stack.ExpandableStackBottom =
+        ((byte *)stack.ExpandableStackBottom) + PAGE_SIZE;
+    stack.ExpandableStackBase = 
+        ((byte *)stack.ExpandableStackBottom) + stack_reserve - (2*PAGE_SIZE);
+    /* PR 252008: WOW64's initial APC uses the stack base, ignoring CONTEXT.Esp,
+     * so we put an extra page in place for the nudge arg.  It should be
+     * freed w/ no problems since the Bottom's region is freed.
+     * An alternative is a separate allocation and setting NUDGE_FREE_ARG,
+     * but the caller is the one who knows the structure of the arg.            
+     */
+    if (wow64)
+        stack.ExpandableStackBase = ((byte*)stack.ExpandableStackBase) - PAGE_SIZE;
+
+    stack.ExpandableStackLimit = 
+        ((byte *)stack.ExpandableStackBase) - stack_commit;
+    num_commit_bytes = stack_commit + PAGE_SIZE;
+    p = ((byte *)stack.ExpandableStackBase) - num_commit_bytes;
+    if (wow64)
+        num_commit_bytes += PAGE_SIZE;
+    p = VirtualAllocEx(hProcess, p, num_commit_bytes, MEM_COMMIT, PAGE_READWRITE);
+    if (p == NULL)
+        goto error;
+    if (!VirtualProtectEx(hProcess, p, PAGE_SIZE, PAGE_READWRITE|PAGE_GUARD, &old_prot))
+        goto error;
+
+    get_platform(&platform);
+    DO_ASSERT(platform != 0);
+
+    /* set the context: initialize with our own */
+    context.ContextFlags = CONTEXT_FULL;
+    GetThreadContext(GetCurrentThread(), &context);
+
+    /* set esp */
+    context.Esp = (uint)stack.ExpandableStackBase;
+    /* On vista, the kernel sets esp a random number of bytes in from the base of
+     * the stack as part of the stack ASLR.  RtlUserThreadStart assumes that
+     * esp will be set at least 8 bytes into the region by writing to esp+4 and
+     * esp+8. So we need to move up by at least 8 bytes.  The smallest offset
+     * I've seen is 56 bytes so we'll go with that as there could be another
+     * place assuming there's room above esp. We do it irregardless of whether we
+     * target api or not in case it's relied on elsewhere. */
+#define VISTA_THREAD_STACK_PAD 56
+    if (platform == PLATFORM_VISTA) {
+        context.Esp -= VISTA_THREAD_STACK_PAD;
+    }
+    /* Anticipating x64 we align to 16 bytes everywhere */
+#define STACK_ALIGNMENT 16
+    /* write the argument buffer to the stack */
+    if (arg_buf != NULL) {
+        SIZE_T written;
+        if (wow64) { /* PR 252008: see above */
+            thread_arg = (void *)
+                (((byte *)stack.ExpandableStackBase) + PAGE_SIZE - arg_buf_size);
+        } else {
+            context.Esp -= ALIGN_FORWARD(arg_buf_size, STACK_ALIGNMENT);
+            thread_arg = (void *)context.Esp;
+        }
+        if (!WriteProcessMemory(hProcess, thread_arg, arg_buf, arg_buf_size, &written) ||
+            written != arg_buf_size) {
+            goto error;
+        }
+        if (platform == PLATFORM_VISTA) {
+            /* pad after our argument so RtlUserThreadStart won't clobber it */
+            context.Esp -= VISTA_THREAD_STACK_PAD;
+        }
+    }
+
+    /* set eip and argument */
+    if (target_api) {
+        if (platform == PLATFORM_VISTA) {
+            context.Eip = (uint)GetProcAddress(GetModuleHandle(L"ntdll.dll"),
+                                               "RtlUserThreadStart");
+        } else {
+            context.Eip = (uint)get_kernel_thread_start_thunk();
+        }
+        /* For kernel32!BaseThreadStartThunk and ntdll!RltUserThreadStartThunk
+         * Eax contains the address of the thread routine and Ebx the arg */
+        context.Eax = (uint)start_addr;
+        context.Ebx = (uint)thread_arg;
+    } else {
+        void *buf[2];
+        bool res;
+        SIZE_T written;
+        /* directly targeting the start_address */
+        context.Eip = (uint)start_addr;
+        context.Esp -= sizeof(buf);
+        /* set up arg on stack, give NULL return address */
+        buf[1] = thread_arg;
+        buf[0] = NULL;
+        res = WriteProcessMemory(hProcess, (void *)context.Esp, &buf,
+                                 sizeof(buf), &written);
+        if (!res || written != sizeof(buf)) {
+            goto error;
+        }
+    }                           
+    if (context.Eip == 0) {
+        DO_ASSERT(false);
+        goto error;
+    }
+
+    /* NOTE - CreateThread passes NULL for object attributes so despite Nebbet
+     * must be optional (checked NTsp6a, XPsp2). We don't pass NULL so we can
+     * specify the security descriptor. */
+    if (!NT_SUCCESS(NtCreateThread(&hThread, THREAD_ALL_ACCESS, &oa, 
+                                   hProcess, &cid, &context, &stack, 
+                                   (byte)(suspended ? TRUE : FALSE)))) {
+        goto error;
+    }
+
+    if (tid != NULL)
+        *tid = (uint)cid.UniqueThread;
+
+ exit:
+    if (sd != NULL) {
+        /* Free the security descriptor. */
+        LocalFree(sd);
+    }
+
+    return hThread;
+
+ error:
+    if (stack.ExpandableStackBottom != NULL) {
+        /* Free remote stack on error. */
+        VirtualFreeEx(hProcess, stack.ExpandableStackBottom, 0, MEM_RELEASE);
+        if (remote_stack != NULL)
+            *remote_stack = NULL;
+    }
+    DO_ASSERT(hThread == NULL);
+    hThread = NULL; /* just to be safe */
+    goto exit;
+}
+
+/* flag defines for create_remote_thread */
+/* Use ntdll!NtCreateThread instead of kernel32!CreateThread to allow creating
+ * threads in a different session (case 872). */
+#define CREATE_REMOTE_THREAD_USE_NT               0x01
+
+/* If ...USE_NT then target the natively created thread to the same entry point as
+ * the api routines would (kernel32!BaseThreadStartThunk for pre-Vista and
+ * ntdll!RtlUserThreadStart for Vista). This allows the target thread to exit by
+ * returning from its thread function.  NOTE - be very careful about who frees the
+ * new thread's stack. On NT and 2k returning from the thread function will free the
+ * stack while on >= XP it won't. */
+#define CREATE_REMOTE_THREAD_TARGET_API           0x02
+
+/* 64kb, same as allocation granularity so is as small as we can get */
+#define STACK_RESERVE 0x10000
+/* 12kb, matches current core stack size, note can expand to 
+ * STACK_RESERVE - (5 * PAGE_SIZE), i.e. 44kb */
+#define STACK_COMMIT 0x3000
+
+/* returns NULL on failure */
+static HANDLE
+create_remote_thread(HANDLE hProc, PTHREAD_START_ROUTINE pfnThreadRtn,
+                     void *arg, const void *arg_buf, SIZE_T arg_buf_size,
+                     uint flags, void **remote_stack /* OUT */)
+{
+    HANDLE hThread = NULL;
+    if (remote_stack != NULL)
+        *remote_stack = NULL;
+    if (TEST(CREATE_REMOTE_THREAD_USE_NT, flags)) {
+        /* FIXME - should we allow the caller to specify the stack sizes?
+         * we already just use defaults if we're using CreateRemoteThread */
+        hThread = nt_create_thread(hProc, pfnThreadRtn, arg, arg_buf, arg_buf_size,
+                                   STACK_RESERVE, STACK_COMMIT, false, NULL,
+                                   TEST(CREATE_REMOTE_THREAD_TARGET_API, flags),
+                                   remote_stack);
+    } else {
+        DO_ASSERT(false && "not tested (not currently used)");
+        DO_ASSERT(false && "if used for nudge need update flags for stack/arg freeing");
+        DO_ASSERT(arg_buf == NULL && "no support for buffer arguments to "
+                  "create_remote_thread when not using NT");
+
+        hThread = CreateRemoteThread(hProc, NULL, 0, pfnThreadRtn, arg, 0, NULL);
+    }
+    return hThread;
+}
+
+/* define this here so that clients don't need to know about dr_marker_t */
+DWORD
+get_dr_marker(int ProcessID, dr_marker_t *marker, 
+              hotp_policy_status_table_t **hotp_status, int *found);
+
+DWORD
+nudge_dr(int pid, BOOL allow_upgraded_perms, DWORD timeout_ms, nudge_arg_t *nudge_arg)
+{
+    DWORD res = ERROR_SUCCESS;
+    int found;
+    dr_marker_t marker;
+    PTHREAD_START_ROUTINE pfnThreadRtn;
+    /* FIXME: case 7038 */
+    /* It is not safe to convert our routines to a PTHREAD_START_ROUTINE?
+     * That one is expected to be a
+     * DWORD (WINAPI *PTHREAD_START_ROUTINE)(LPVOID lpThreadParameter);
+     * and if we are using the wrong calling convention the application may fail
+     * MSDN notes that this is common but crashes only on 64-bit
+     */
+
+    HANDLE hThread = NULL, hProcess = NULL;
+    void *remote_stack = NULL;
+
+    DWORD platform = 0;
+    get_platform(&platform);
+    DO_ASSERT(platform != 0);
+
+    /* Open the process handle before getting the drmarker, is slight perf.
+     * hit if process turns out not to be running under DR, but we avoid the
+     * race of the process exiting and it's id getting recycled, the os won't
+     * recycle the id till we free our handle */
+    hProcess = OpenProcess(PROCESS_QUERY_INFORMATION + 
+                           PROCESS_VM_WRITE + 
+                           PROCESS_VM_READ + 
+                           PROCESS_VM_OPERATION + 
+                           PROCESS_CREATE_THREAD + 
+                           READ_CONTROL +
+                           SYNCHRONIZE, 
+                           FALSE, pid);
+    res = GetLastError();
+    if (hProcess == NULL) {
+        if (allow_upgraded_perms) {
+            acquire_privileges();
+            hProcess = OpenProcess(PROCESS_QUERY_INFORMATION + 
+                                   PROCESS_VM_WRITE + 
+                                   PROCESS_VM_READ + 
+                                   PROCESS_VM_OPERATION + 
+                                   PROCESS_CREATE_THREAD +
+                                   READ_CONTROL +
+                                   SYNCHRONIZE, 
+                                   FALSE, pid);
+            res = GetLastError();
+            release_privileges();
+            if (hProcess == NULL) {
+                if (res != ERROR_SUCCESS)
+                    res = ERROR_DETACH_ERROR;
+                goto exit;
+            }
+        } else {
+            if (res != ERROR_SUCCESS)
+                res = ERROR_DETACH_ERROR;
+            goto exit;
+        }
+    }
+
+    res = get_dr_marker(pid, &marker, NULL, &found);
+    /* first make sure the PID is running under dr */
+    if (res != ERROR_SUCCESS || found != DR_MARKER_FOUND) {
+        res = ERROR_MOD_NOT_FOUND;
+        goto exit;
+    }
+
+    pfnThreadRtn = (PTHREAD_START_ROUTINE) marker.dr_generic_nudge_target;
+
+    if (pfnThreadRtn != NULL) {    /* Fix for case 5464. */
+        /* We use the native api to create the thread (CREATE_REMOTE_THREAD_USE_NT)
+         * to avoid session id issues.  FIXME - we don't really need to TARGET_API
+         * anymore since the nudge routine never returns now (which could simplify
+         * the core nudge thread detection a little).  Note - if we stop using USE_NT
+         * we need to update the stack freeing code below and change the nudge flags. */
+        hThread = create_remote_thread(hProcess, pfnThreadRtn,
+                                       NULL, (void *)nudge_arg, sizeof(nudge_arg_t),
+                                       CREATE_REMOTE_THREAD_USE_NT |
+                                       CREATE_REMOTE_THREAD_TARGET_API,
+                                       &remote_stack);
+        if (hThread == NULL) {
+            res = GetLastError();
+            goto exit;
+        }
+    } else {
+        hThread = NULL;
+        res = ERROR_INVALID_FUNCTION;
+        goto exit;
+    }
+
+    res = WaitForSingleObject(hThread, timeout_ms);
+
+    if (res == WAIT_FAILED) {
+        res = GetLastError();
+    } else if (res == WAIT_TIMEOUT) {
+        res = ERROR_TIMEOUT;
+    } else {
+        res = ERROR_SUCCESS;
+        /* We have a general problem on how to free the application stack for nudges.
+         * Currently the app/os will never free a nudge thread's app stack:
+         *  On NT and 2k ExitThread would normally free the app stack, but we always
+         *   terminate nudge threads instead of allowing them to return and exit normally.
+         *  On XP and 2k3 none of our nudge creation routines inform csrss of the new
+         *   thread(which is who typically frees the stacks) assuming we don't use switch
+         *   to using CreateRemoteThread() here.
+         *  On Vista we don't use NtCreateThreadEx to create the nudge threads so the
+         *   kernel doesn't free the stack.
+         * As such we are left with two options: free the app stack here (nudgee free) or
+         * have the nudge thread creator free the app stack (nudger free). We use
+         * nudge flags to specify which we are using. */ 
+        if (TEST(NUDGE_NUDGER_FREE_STACK, nudge_arg->flags) && remote_stack != NULL) {
+            VirtualFreeEx(hProcess, remote_stack, 0, MEM_RELEASE);
+        }
+    }
+
+ exit:
+    if (hThread != NULL)
+        CloseHandle(hThread);
+    if (hProcess != NULL)
+        CloseHandle(hProcess);
+    return res;
+}
+
+/* Detaches from the specified process by creating a thread in the target 
+ * process that directly targets the detach routine.  
+ */
+DWORD
+detach(int pid, BOOL allow_upgraded_perms, DWORD timeout_ms)
+{
+    return generic_nudge(pid, allow_upgraded_perms, NUDGE_GENERIC(detach), 0,
+                         NULL, timeout_ms);
+}
+
+/* generic nudge DR, action mask determines which actions will be executed */
+DWORD
+generic_nudge(int pid, BOOL allow_upgraded_perms, DWORD action_mask, 
+              client_id_t client_id /* optional */, uint64 client_arg /* optional */,
+              DWORD timeout_ms)
+{
+    nudge_arg_t arg = {0};
+    arg.version = NUDGE_ARG_CURRENT_VERSION;
+    arg.nudge_action_mask = action_mask;
+    /* default flags */
+    arg.client_id = client_id;
+    arg.client_arg = client_arg;
+    return nudge_dr(pid, allow_upgraded_perms, timeout_ms, &arg);
+}
+
+/* Loads the specified dll in process pid and waits on the loading thread, does
+ * not free the dll once the loading thread exits. Usual usage is for the 
+ * loaded dll to do something in its DllMain. If you do not want the dll to 
+ * stay loaded its DllMain should return false. To unload a dll from a process 
+ * later, inject another dll whose dll main unloads that dll and then returns 
+ * false. If loading_thread != NULL returns a handle to the loading thread (dll
+ * could call FreeLibraryAndExitThread on itself in its dll main to return a 
+ * value out via the exit code). inject_dll provides no way to pass arguments 
+ * in to the dll. */
+DWORD
+inject_dll(int pid, const WCHAR *dll_name, BOOL allow_upgraded_perms, 
+           DWORD timeout_ms, PHANDLE loading_thread)
+{
+    DWORD res;
+    PTHREAD_START_ROUTINE pfnThreadRtn= NULL;
+    HANDLE hThread = NULL, hProcess = NULL, file_tmp = NULL;
+    void *remote_stack = NULL;
+    DWORD platform = 0;
+    get_platform(&platform);
+    DO_ASSERT(platform != 0);
+
+    if (loading_thread != NULL)
+        *loading_thread = NULL;
+
+    file_tmp = CreateFile(dll_name, 0, 0, NULL, OPEN_EXISTING, 
+                          FILE_ATTRIBUTE_NORMAL, NULL);
+    if (file_tmp == INVALID_HANDLE_VALUE)
+        return ERROR_FILE_NOT_FOUND;
+    CloseHandle(file_tmp);
+        
+    hProcess = OpenProcess(PROCESS_QUERY_INFORMATION + 
+                           PROCESS_VM_WRITE + 
+                           PROCESS_VM_READ + 
+                           PROCESS_VM_OPERATION + 
+                           PROCESS_CREATE_THREAD +
+                           READ_CONTROL +
+                           SYNCHRONIZE, 
+                           FALSE, pid);
+    if (hProcess == NULL) {
+        if (allow_upgraded_perms) {
+            acquire_privileges();
+            hProcess = OpenProcess(PROCESS_QUERY_INFORMATION + 
+                                   PROCESS_VM_WRITE + 
+                                   PROCESS_VM_READ + 
+                                   PROCESS_VM_OPERATION + 
+                                   PROCESS_CREATE_THREAD +
+                                   READ_CONTROL +
+                                   SYNCHRONIZE, 
+                                   FALSE, pid);
+            release_privileges();
+            if (hProcess == NULL) {
+                return GetLastError();
+            }
+        } else { 
+            return GetLastError();
+        }
+    }
+
+    pfnThreadRtn = (PTHREAD_START_ROUTINE) GetProcAddress(GetModuleHandle(TEXT("Kernel32")), "LoadLibraryW");
+    if (pfnThreadRtn == NULL) {
+        res = GetLastError();
+        goto exit;
+    }
+    /* USE_NT to avoid session id issues, TARGET_API so LoadLibrary can return, if
+     * stop using NT need to update stack freeing case below and allocate space to
+     * hold the remote library name ourselves like we used to. */
+    hThread = create_remote_thread(hProcess, pfnThreadRtn,
+                                   NULL, dll_name, sizeof(WCHAR)*(1 + wcslen(dll_name)),
+                                   CREATE_REMOTE_THREAD_USE_NT |
+                                   CREATE_REMOTE_THREAD_TARGET_API , &remote_stack);
+    if (hThread == NULL) {
+        res = GetLastError();
+        goto exit;
+    }
+    
+    res = WaitForSingleObject(hThread, timeout_ms);
+    if (res == WAIT_FAILED) {
+        res = GetLastError();
+        goto exit;
+    } else if (res == WAIT_TIMEOUT) {
+        res = ERROR_TIMEOUT;
+        goto exit;
+    }
+
+    /* thread has finished */
+    /* On NT and 2k the remote stack will be freed by the remote thread exiting
+     * (returning from LoadLibrary or calling ExitThread), though not if it terminates
+     * itself (which it shouldn't do, lock issues etc.).  On XP and higher the stack
+     * won't be freed since we didn't inform csrss of this thread so we free here. */
+    if (remote_stack != NULL &&
+        platform != PLATFORM_WIN_NT_4 && platform != PLATFORM_WIN_2000) {
+        VirtualFreeEx(hProcess, remote_stack, 0, MEM_RELEASE);
+    }
+    res = ERROR_SUCCESS;
+
+ exit:
+    if (hThread != NULL) {
+        if (loading_thread != NULL) {
+            *loading_thread = hThread;
+        } else {
+            CloseHandle(hThread);
+        }
+    }
+    if (hProcess != NULL)
+        CloseHandle(hProcess);
+
+    return res;
+}
diff --git a/libutil/dr_config.c b/libutil/dr_config.c
new file mode 100644
index 000000000..95f45f5e7
--- /dev/null
+++ b/libutil/dr_config.c
@@ -0,0 +1,1152 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include <stdio.h>
+#include "utils.h"
+#include "processes.h"
+#include "globals_shared.h"
+#include "mfapi.h" /* for PLATFORM_WIN_2000 */
+#include "lib/dr_config.h"
+
+#define RELEASE32_DLL   L"\\lib32\\release\\dynamorio.dll"
+#define DEBUG32_DLL     L"\\lib32\\debug\\dynamorio.dll"
+#define RELEASE64_DLL   L"\\lib64\\release\\dynamorio.dll"
+#define DEBUG64_DLL     L"\\lib64\\debug\\dynamorio.dll"
+#define LOG_SUBDIR      L"\\logs"
+#define LIB32_SUBDIR    L"\\lib32"
+#define PREINJECT32_DLL L"\\lib32\\drpreinject.dll"
+#define PREINJECT64_DLL L"\\lib64\\drpreinject.dll"
+
+/* The minimum option size is 3, e.g., "-x ".  Note that we need the
+ * NULL term too so "-x -y" needs 6 characters.
+*/
+#define MAX_NUM_OPTIONS (DR_MAX_OPTIONS_LENGTH / 3)
+
+/* Data structs to hold info about the DYNAMORIO_OPTION registry entry */
+typedef struct _client_opt_t {
+    WCHAR *path;
+    client_id_t id;
+    WCHAR *opts;
+} client_opt_t;
+
+typedef struct _opt_info_t {
+    dr_operation_mode_t mode;
+    WCHAR *extra_opts[MAX_NUM_OPTIONS];
+    size_t num_extra_opts;
+    /* note that clients are parsed and stored in priority order */
+    client_opt_t *client_opts[MAX_CLIENT_LIBS];
+    size_t num_clients;
+} opt_info_t;
+
+
+/* Function to iterate over the options in a DYNAMORIO_OPTIONS string.
+ * For the purposes of this function, we're not differentiating
+ * between an option and an option argument.  We're simply looking for
+ * space-separated strings while taking into account that some strings
+ * can be quoted.  'ptr' should point to the current location in the
+ * options string; the option is copied to 'token'
+ */
+static WCHAR *
+get_next_token(WCHAR* ptr, WCHAR *token)
+{
+    /* advance to next non-space character */
+    while (*ptr == L' ') {
+        ptr++;
+    }
+
+    /* check for end-of-string */
+    if (*ptr == L'\0') {
+        token[0] = L'\0';
+        return NULL;
+    }
+
+    /* for quoted options, copy until the closing quote */
+    if (*ptr == L'\"') {
+        *token++ = *ptr++;
+        while (*ptr != L'\0') {
+            *token++ = *ptr++;
+            if (ptr[-1] == L'\"' && ptr[-2] != L'\\') {
+                break;
+            }
+        }
+    }
+    
+    /* otherwise copy until the next space character */
+    else {
+        while (*ptr != L' ' && *ptr != L'\0') {
+            *token++ = *ptr++;
+        }
+    }
+
+    *token = L'\0';
+    return ptr;
+}
+
+
+/* Allocate a new client_opt_t */
+static client_opt_t *
+new_client_opt(const WCHAR *path, client_id_t id, const WCHAR *opts)
+{
+    size_t len;
+    client_opt_t *opt = (client_opt_t *)malloc(sizeof(client_opt_t));
+    if (opt == NULL) {
+        return NULL;
+    }
+
+    opt->id = id;
+
+    len = MIN(MAX_PATH-1, wcslen(path));
+    opt->path = malloc((len+1) * sizeof(opt->path[0]));
+    wcsncpy(opt->path, path, len);
+    opt->path[len] = L'\0';
+
+    len = MIN(DR_MAX_OPTIONS_LENGTH-1, wcslen(opts));
+    opt->opts = malloc((len+1) * sizeof(opt->opts[0]));
+    wcsncpy(opt->opts, opts, len);
+    opt->opts[len] = L'\0';
+
+    return opt;
+}
+
+/* Free a client_opt_t */
+static void
+free_client_opt(client_opt_t *opt)
+{
+    if (opt == NULL) {
+        return;
+    }
+    if (opt->path != NULL) {
+        free(opt->path);
+    }
+    if (opt->opts != NULL) {
+        free(opt->opts);
+    }
+}
+
+
+/* Add another client to an opt_info_t struct */
+static dr_config_status_t
+add_client_lib(opt_info_t *opt_info, client_id_t id, size_t pri,
+               const WCHAR *path, const WCHAR *opts)
+{
+    size_t i;
+    
+    if (opt_info->num_clients >= MAX_CLIENT_LIBS) {
+        return DR_FAILURE;
+    }
+
+    if (pri > opt_info->num_clients) {
+        return DR_ID_INVALID;
+    }
+
+    /* shift existing entries to make space for the new client info */
+    for (i=opt_info->num_clients; i>pri; i--) {
+        opt_info->client_opts[i] = opt_info->client_opts[i-1];
+    }
+
+    opt_info->client_opts[pri] = new_client_opt(path, id, opts);
+    opt_info->num_clients++;
+
+    return DR_SUCCESS;
+}
+
+
+static dr_config_status_t
+remove_client_lib(opt_info_t *opt_info, client_id_t id)
+{
+    size_t i, j;
+    for (i=0; i<opt_info->num_clients; i++) {
+        if (opt_info->client_opts[i]->id == id) {
+            free_client_opt(opt_info->client_opts[i]);
+
+            /* shift remaining entries down */
+            for (j=i; j<opt_info->num_clients-1; j++) {
+                opt_info->client_opts[j] = opt_info->client_opts[j+1];
+            }
+
+            opt_info->num_clients--;
+            return DR_SUCCESS;
+        }
+    }
+
+    return DR_ID_INVALID;
+}
+
+
+/* Add an 'extra' option (non-client related option) to an opt_info_t struct */
+static dr_config_status_t
+add_extra_option(opt_info_t *opt_info, const WCHAR *opt)
+{
+    if (opt != NULL && opt[0] != L'\0') {
+        size_t idx, len;
+        idx = opt_info->num_extra_opts;
+        if (idx >= MAX_NUM_OPTIONS) {
+            return DR_FAILURE;
+        }
+        
+        len = MIN(DR_MAX_OPTIONS_LENGTH-1, wcslen(opt));
+        opt_info->extra_opts[idx] = malloc
+            ((len+1) * sizeof(opt_info->extra_opts[idx][0]));
+        wcsncpy(opt_info->extra_opts[idx], opt, len);
+        opt_info->extra_opts[idx][len] = L'\0';
+
+        opt_info->num_extra_opts++;
+    }
+
+    return DR_SUCCESS;
+}
+
+static dr_config_status_t
+add_extra_option_char(opt_info_t *opt_info, const char *opt)
+{
+    if (opt != NULL && opt[0] != '\0') {
+        WCHAR wbuf[DR_MAX_OPTIONS_LENGTH];
+        _snwprintf(wbuf, DR_MAX_OPTIONS_LENGTH, L"%S", opt);
+        NULL_TERMINATE_BUFFER(wbuf);
+        return add_extra_option(opt_info, wbuf);
+    }
+
+    return DR_SUCCESS;
+}
+
+
+/* Free allocated memory in an opt_info_t */
+static void
+free_opt_info(opt_info_t *opt_info)
+{
+    size_t i;
+    for (i=0; i<opt_info->num_clients; i++) {
+        free_client_opt(opt_info->client_opts[i]);
+    }
+    for (i=0; i<opt_info->num_extra_opts; i++) {
+        free(opt_info->extra_opts[i]);
+    }
+}
+
+
+/* Read a DYNAMORIO_OPTIONS string from 'wbuf' and populate an opt_info_t struct */
+static dr_config_status_t
+read_options(opt_info_t *opt_info, ConfigGroup *proc_policy)
+{
+    WCHAR *ptr, token[DR_MAX_OPTIONS_LENGTH], tmp[DR_MAX_OPTIONS_LENGTH];
+    opt_info_t null_opt_info = {0,};
+    size_t len;
+    
+    *opt_info = null_opt_info;
+
+    ptr = get_config_group_parameter(proc_policy, L_DYNAMORIO_VAR_OPTIONS);
+    if (ptr == NULL) {
+        return DR_FAILURE;
+    }
+
+    /* We'll be safe and not trust that get_config_group_parameter
+     * returns a nice NULL-terminated string with no more than
+     * DR_MAX_OPTIONS_LENGTH characters.  Making sure we have such a
+     * string makes get_next_token() simpler.  A more efficient
+     * approach would be to keep track of a string length and pass
+     * that to get_next_token().
+     */
+    len = MIN(DR_MAX_OPTIONS_LENGTH-1, wcslen(ptr));
+    wcsncpy(tmp, ptr, len);
+    tmp[len] = L'\0';
+
+    opt_info->mode = DR_MODE_NONE;
+
+    ptr = tmp;
+    while (ptr != NULL) {
+        ptr = get_next_token(ptr, token);
+        
+        /*
+         * look for the mode 
+         */
+        if (wcscmp(token, L"-code_api") == 0) {
+            if (opt_info->mode != DR_MODE_NONE) {
+                goto error;
+            }
+            opt_info->mode = DR_MODE_CODE_MANIPULATION;
+        }
+#ifdef MF_API
+        else if (wcscmp(token, L"-security_api") == 0) {
+            if (opt_info->mode != DR_MODE_NONE) {
+                goto error;
+            }
+            opt_info->mode = DR_MODE_MEMORY_FIREWALL;
+        }
+#endif
+        else if (wcscmp(token, L"-probe_api") == 0) {
+#ifdef PROBE_API
+            /* nothing; we assign the mode when we see -code_api */
+#else
+            /* we shouldn't see -probe_api */
+            goto error;
+#endif
+        }
+#ifdef PROBE_API
+        else if (wcscmp(token, L"-hotp_only") == 0) {
+            if (opt_info->mode != DR_MODE_NONE) {
+                goto error;
+            }
+            opt_info->mode = DR_MODE_PROBE;
+        }
+#endif
+        
+        /* 
+         * look for client options
+         */
+        else if (wcscmp(token, L"-client_lib") == 0) {
+            WCHAR *path_str, *id_str, *opt_str;
+            client_id_t id;
+
+            ptr = get_next_token(ptr, token);
+            if (ptr == NULL) {
+                goto error;
+            }
+
+            /* handle enclosing quotes */
+            path_str = token;
+            if (path_str[0] == L'\"') {
+                size_t last;
+
+                path_str++;
+                last = wcslen(path_str)-1;
+                if (path_str[last] != L'\"') {
+                    goto error;
+                }
+                path_str[last] = L'\0';
+            }
+
+            /* -client_lib options should have the form path;ID;options.
+             * Client priority is left-to-right.
+             */
+            id_str = wcsstr(path_str, L";");
+            if (id_str == NULL) {
+                goto error;
+            }
+
+            *id_str = L'\0';
+            id_str++;
+
+            opt_str = wcsstr(id_str, L";");
+            if (opt_str == NULL) {
+                goto error;
+            }
+
+            *opt_str = L'\0';
+            opt_str++;
+
+            /* client IDs are in hex */
+            id = wcstoul(id_str, NULL, 16);
+
+            /* add the client info to our opt_info structure */
+            if (add_client_lib(opt_info, id, opt_info->num_clients, 
+                               path_str, opt_str) != DR_SUCCESS) {
+                goto error;
+            }
+        }
+
+        /*
+         * Any remaining options are not related to clients.  Put all
+         * these options (and their arguments) in one array.
+         */
+        else {
+            if (add_extra_option(opt_info, token) != DR_SUCCESS) {
+                goto error;
+            }
+        }
+    }
+
+    return DR_SUCCESS;
+
+ error:
+    free_opt_info(opt_info);
+    *opt_info = null_opt_info;
+    return DR_FAILURE;
+}
+
+
+/* Write the options stored in an opt_info_t to 'wbuf' in the form expected
+ * by the DYNAMORIO_OPTIONS registry entry.
+ */
+static void
+write_options(opt_info_t *opt_info, WCHAR *wbuf)
+{
+    size_t i;
+    char *mode_str = "";
+
+    /* NOTE - mode_str must come first since we want to give
+     * client-supplied options the chance to override (for
+     * ex. -stack_size which -code_api sets, see PR 247436
+     */
+    switch (opt_info->mode) {
+#ifdef MF_API
+        case DR_MODE_MEMORY_FIREWALL:
+            mode_str = "-security_api";
+            break;
+#endif
+        case DR_MODE_CODE_MANIPULATION:
+#ifdef PROBE_API
+            mode_str = "-code_api -probe_api";
+#else
+            mode_str = "-code_api";
+#endif
+            break;
+#ifdef PROBE_API
+        case DR_MODE_PROBE:
+            mode_str = "-probe_api -hotp_only";
+            break;
+#endif
+        default:
+            DO_ASSERT(false);
+    }
+
+    _snwprintf(wbuf, DR_MAX_OPTIONS_LENGTH, L"%S", mode_str);
+
+    /* extra options */
+    for (i=0; i<opt_info->num_extra_opts; i++) {
+        /* FIXME: Note that we're blindly allowing any options
+         * provided so we can allow users to specify "undocumented"
+         * options.  Maybe we should be checking that the options are
+         * actually valid?
+         */
+        _snwprintf(wbuf, DR_MAX_OPTIONS_LENGTH, L"%s %s", wbuf, opt_info->extra_opts[i]);
+    }
+
+    /* client lib options */
+    for (i=0; i<opt_info->num_clients; i++) {
+        client_opt_t *client_opts = opt_info->client_opts[i];
+        _snwprintf(wbuf, DR_MAX_OPTIONS_LENGTH, L"%s -client_lib \"%s;%x;%s\"",
+                   wbuf, client_opts->path, client_opts->id, client_opts->opts);
+    }
+
+    wbuf[DR_MAX_OPTIONS_LENGTH-1] = L'\0';
+}
+
+/* caller must call free_config_group() on the returned ConfigGroup */
+static ConfigGroup *
+get_policy(dr_platform_t dr_platform)
+{
+    ConfigGroup *policy;
+
+    /* PR 244206: set the registry view before any registry access.
+     * If we are ever part of a persistent agent maybe we should
+     * restore to the default platform at the end of this routine?
+     */
+    set_dr_platform(dr_platform);
+
+    if (read_config_group(&policy, L_PRODUCT_NAME, TRUE) != ERROR_SUCCESS) {
+        return NULL;
+    }
+
+    return policy;
+}
+
+/* As a sub policy only the parent policy (from get_policy()) need be freed */
+static ConfigGroup * 
+get_proc_policy(ConfigGroup *policy, const char *process_name)
+{
+    ConfigGroup *res = NULL;
+    if (policy != NULL) {
+        WCHAR wbuf[MAX_PATH];
+        _snwprintf(wbuf, MAX_PATH, L"%S", process_name);
+        NULL_TERMINATE_BUFFER(wbuf);
+        res = get_child(wbuf, policy);
+    }
+    return res;
+}                
+
+
+dr_config_status_t
+dr_register_process(const char *process_name,
+                    const char *dr_root_dir,
+                    dr_operation_mode_t dr_mode,
+                    bool debug,
+                    dr_platform_t dr_platform,
+                    const char *dr_options)
+{
+    ConfigGroup *policy, *proc_policy;
+    WCHAR wbuf[MAX(MAX_PATH,DR_MAX_OPTIONS_LENGTH)];
+    DWORD platform;
+    opt_info_t opt_info = {0,};
+
+    /* PR 244206: set the registry view before any registry access.
+     * If we are ever part of a persistent agent maybe we should
+     * restore to the default platform at the end of this routine?
+     */
+    set_dr_platform(dr_platform);
+
+    /* create top-level Determina/SecureCore key */
+    if (create_root_key() != ERROR_SUCCESS) {
+        return DR_FAILURE;
+    }
+    if (read_config_group(&policy, L_PRODUCT_NAME, TRUE) != ERROR_SUCCESS) {
+        return DR_FAILURE;
+    }
+    
+    /* create process key */
+    _snwprintf(wbuf, MAX_PATH, L"%S", process_name);    
+    NULL_TERMINATE_BUFFER(wbuf);
+    proc_policy = get_child(wbuf, policy);
+    if (proc_policy == NULL) {
+        proc_policy = new_config_group(wbuf);
+        add_config_group(policy, proc_policy);
+    }
+    else {
+        return DR_PROC_REG_EXISTS;
+    }
+
+    /* set the options string */
+    opt_info.mode = dr_mode;
+    add_extra_option_char(&opt_info, dr_options);
+    write_options(&opt_info, wbuf);
+    set_config_group_parameter(proc_policy, L_DYNAMORIO_VAR_OPTIONS, wbuf);
+    free_opt_info(&opt_info);
+
+    /* set the rununder string */
+    _snwprintf(wbuf, MAX_PATH, L"1");
+    NULL_TERMINATE_BUFFER(wbuf);
+    set_config_group_parameter(proc_policy, L_DYNAMORIO_VAR_RUNUNDER, wbuf);
+
+    /* set the autoinject string (i.e., path to dynamorio.dll */
+    if (debug) {
+        if (get_dr_platform() != DR_PLATFORM_64BIT)
+            _snwprintf(wbuf, MAX_PATH, L"%S"DEBUG32_DLL, dr_root_dir);
+        else
+            _snwprintf(wbuf, MAX_PATH, L"%S"DEBUG64_DLL, dr_root_dir);
+    } else {
+        if (get_dr_platform() != DR_PLATFORM_64BIT)
+            _snwprintf(wbuf, MAX_PATH, L"%S"RELEASE32_DLL, dr_root_dir);
+        else
+            _snwprintf(wbuf, MAX_PATH, L"%S"RELEASE64_DLL, dr_root_dir);
+    }
+    NULL_TERMINATE_BUFFER(wbuf);
+    set_config_group_parameter(proc_policy, L_DYNAMORIO_VAR_AUTOINJECT, wbuf);
+
+    /* set the logdir string */
+    _snwprintf(wbuf, MAX_PATH, L"%S"LOG_SUBDIR, dr_root_dir);
+    NULL_TERMINATE_BUFFER(wbuf);
+    set_config_group_parameter(proc_policy, L_DYNAMORIO_VAR_LOGDIR, wbuf);
+
+    /* write the registry */
+    if (write_config_group(policy) != ERROR_SUCCESS) {
+        return DR_FAILURE;
+    }
+
+    /* Set the appinit key */
+    if (get_dr_platform() != DR_PLATFORM_64BIT)
+        _snwprintf(wbuf, MAX_PATH, L"%S"PREINJECT32_DLL, dr_root_dir);
+    else
+        _snwprintf(wbuf, MAX_PATH, L"%S"PREINJECT64_DLL, dr_root_dir);
+    NULL_TERMINATE_BUFFER(wbuf);
+    /* Always overwrite, in case we have an older drpreinject version in there */
+    if (set_custom_autoinjection(wbuf, APPINIT_OVERWRITE) != ERROR_SUCCESS ||
+        (is_vista() && set_loadappinit() != ERROR_SUCCESS)) {
+        return DR_FAILURE;
+    }
+
+    /* If on win2k, copy drearlyhelper?.dll to system32 */
+    if (get_platform(&platform) == ERROR_SUCCESS && platform == PLATFORM_WIN_2000) {
+        _snwprintf(wbuf, MAX_PATH, L"%S"LIB32_SUBDIR, dr_root_dir);
+        NULL_TERMINATE_BUFFER(wbuf);
+        copy_earlyhelper_dlls(wbuf);
+    }
+
+    return DR_SUCCESS;
+}
+
+
+dr_config_status_t
+dr_unregister_process(const char *process_name,
+                      dr_platform_t dr_platform)
+{
+    ConfigGroup *policy = get_policy(dr_platform);
+    ConfigGroup *proc_policy = get_proc_policy(policy, process_name);
+    WCHAR wbuf[MAX_PATH];
+    dr_config_status_t status = DR_SUCCESS;
+
+    if (proc_policy == NULL) {
+        status = DR_PROC_REG_INVALID;
+        goto exit;
+    }
+
+    /* remove it */
+    _snwprintf(wbuf, MAX_PATH, L"%S", process_name);
+    NULL_TERMINATE_BUFFER(wbuf);
+    remove_child(wbuf, policy);
+    policy->should_clear = TRUE;
+    
+    /* write the registry */
+    if (write_config_group(policy) != ERROR_SUCCESS) {
+        status = DR_FAILURE;
+        goto exit;
+    }
+
+    /* FIXME PR 232738: we should remove the drdearlyhelp?.dlls and preinject
+     * from system32, and remove the base reg keys, if dr_unregister_process()
+     * removes the last registered process.
+     */
+
+ exit:
+    if (policy != NULL)
+        free_config_group(policy);
+    return status;
+}
+
+static void
+read_process_policy(ConfigGroup *proc_policy,
+                    char *process_name /* OUT */,
+                    char *dr_root_dir /* OUT */,
+                    dr_operation_mode_t *dr_mode /* OUT */,
+                    bool *debug /* OUT */,
+                    char *dr_options /* OUT */)
+{
+    WCHAR *autoinject;
+    opt_info_t opt_info;
+    
+    if (dr_mode != NULL)
+        *dr_mode = DR_MODE_NONE;
+    if (process_name != NULL)
+        *process_name = '\0';
+    if (dr_root_dir != NULL)
+        *dr_root_dir = '\0';
+    if (dr_options != NULL)
+        *dr_options = '\0';
+
+    if (process_name != NULL && proc_policy->name != NULL) {
+        int len = MIN(wcslen(proc_policy->name), MAX_PATH-1);
+        _snprintf(process_name, len, "%S", proc_policy->name);
+        process_name[len] = '\0';
+    }
+
+    autoinject = get_config_group_parameter(proc_policy, L_DYNAMORIO_VAR_AUTOINJECT);    
+    if (dr_root_dir != NULL && autoinject != NULL) {
+        WCHAR *vers = wcsstr(autoinject, RELEASE32_DLL);
+        if (vers == NULL) {
+            vers = wcsstr(autoinject, DEBUG32_DLL);
+        }
+        if (vers == NULL) {
+            vers = wcsstr(autoinject, RELEASE64_DLL);
+        }
+        if (vers == NULL) {
+            vers = wcsstr(autoinject, DEBUG64_DLL);
+        }
+        if (vers != NULL) {
+            size_t len = MIN(MAX_PATH-1, vers - autoinject);
+            _snprintf(dr_root_dir, len, "%S", autoinject);
+            dr_root_dir[len] = '\0';
+        }
+        else {
+            dr_root_dir[0] = '\0';
+        }
+    }
+    
+    if (read_options(&opt_info, proc_policy) != DR_SUCCESS) {
+        /* note: read_options() frees any memory it allocates if it fails */
+        return;
+    }
+
+    if (dr_mode != NULL) {
+        *dr_mode = opt_info.mode;
+    }
+
+    if (debug != NULL) {
+        if (wcsstr(autoinject, DEBUG32_DLL) != NULL ||
+            wcsstr(autoinject, DEBUG64_DLL) != NULL) {
+            *debug = true;
+        }
+        else {
+            *debug = false;
+        }
+    }
+         
+    if (dr_options != NULL) {
+        uint i;
+        size_t len_remain = DR_MAX_OPTIONS_LENGTH-1, cur_off = 0;
+        dr_options[0] = '\0';
+        for (i = 0; i < opt_info.num_extra_opts; i++) {
+            size_t len;
+            if (i > 0 && len_remain > 0) {
+                len_remain--;
+                dr_options[cur_off++] = ' ';
+            }
+            len = MIN(len_remain, wcslen(opt_info.extra_opts[i]));
+            _snprintf(dr_options+cur_off, len, "%S", opt_info.extra_opts[i]);
+            cur_off += len;
+            len_remain -= len;
+            dr_options[cur_off] = '\0';
+        }
+    }
+
+    free_opt_info(&opt_info);
+}
+
+struct _dr_registered_process_iterator_t {
+    ConfigGroup *policy;
+    ConfigGroup *cur;
+};
+
+dr_registered_process_iterator_t *
+dr_registered_process_iterator_start(dr_platform_t dr_platform)
+{
+    dr_registered_process_iterator_t *iter = (dr_registered_process_iterator_t *)
+        malloc(sizeof(dr_registered_process_iterator_t));
+    iter->policy = get_policy(dr_platform);
+    if (iter->policy != NULL)
+        iter->cur = iter->policy->children;
+    else
+        iter->cur = NULL;
+    return iter;
+}
+
+bool
+dr_registered_process_iterator_hasnext(dr_registered_process_iterator_t *iter)
+{
+    return iter->policy != NULL && iter->cur != NULL;
+}
+
+void
+dr_registered_process_iterator_next(dr_registered_process_iterator_t *iter,
+                                    char *process_name /* OUT */,
+                                    char *dr_root_dir /* OUT */,
+                                    dr_operation_mode_t *dr_mode /* OUT */,
+                                    bool *debug /* OUT */,
+                                    char *dr_options /* OUT */)
+{
+    read_process_policy(iter->cur, process_name, dr_root_dir, dr_mode, debug, dr_options);
+    iter->cur = iter->cur->next;
+}
+
+void
+dr_registered_process_iterator_stop(dr_registered_process_iterator_t *iter)
+{
+    if (iter->policy != NULL)
+        free_config_group(iter->policy);
+    free(iter);
+}
+
+bool
+dr_process_is_registered(const char *process_name,
+                         dr_platform_t dr_platform /* OUT */,
+                         char *dr_root_dir /* OUT */,
+                         dr_operation_mode_t *dr_mode /* OUT */,
+                         bool *debug /* OUT */,
+                         char *dr_options /* OUT */)
+{
+    ConfigGroup *policy = get_policy(dr_platform);
+    ConfigGroup *proc_policy = get_proc_policy(policy, process_name);
+    bool result = false;
+
+    if (proc_policy == NULL)
+        goto exit;
+    result = true;
+    
+    read_process_policy(proc_policy, NULL, dr_root_dir, dr_mode, debug, dr_options);
+
+ exit:
+    if (policy != NULL)
+        free_config_group(policy);
+    return result;
+}
+
+struct _dr_client_iterator_t {
+    opt_info_t opt_info;
+    uint cur;
+    bool valid;
+};
+
+dr_client_iterator_t *
+dr_client_iterator_start(const char *process_name,
+                         dr_platform_t dr_platform)
+{
+    ConfigGroup *policy = get_policy(dr_platform);
+    ConfigGroup *proc_policy = get_proc_policy(policy, process_name);
+    dr_client_iterator_t *iter = (dr_client_iterator_t *)
+        malloc(sizeof(dr_client_iterator_t));
+    
+    iter->valid = false;
+    iter->cur = 0;
+    if (proc_policy == NULL)
+        return iter;
+    if (read_options(&iter->opt_info, proc_policy) != DR_SUCCESS)
+        return iter;
+    iter->valid = true;
+
+    return iter;
+}
+
+bool
+dr_client_iterator_hasnext(dr_client_iterator_t *iter)
+{
+    return iter->valid && iter->cur < iter->opt_info.num_clients;
+}
+
+void
+dr_client_iterator_next(dr_client_iterator_t *iter,
+                        client_id_t *client_id, /* OUT */
+                        size_t *client_pri,     /* OUT */
+                        char *client_path,      /* OUT */
+                        char *client_options    /* OUT */)
+{
+    client_opt_t *client_opt = iter->opt_info.client_opts[iter->cur];
+
+    if (client_pri != NULL)
+        *client_pri = iter->cur;
+
+    if (client_path != NULL) {
+        size_t len = MIN(MAX_PATH-1, wcslen(client_opt->path));
+        _snprintf(client_path, len, "%S", client_opt->path);
+        client_path[len] = '\0';
+    }
+
+    if (client_id != NULL)
+        *client_id = client_opt->id;
+
+    if (client_options != NULL) {
+        size_t len = MIN(DR_MAX_OPTIONS_LENGTH-1, wcslen(client_opt->opts));
+        _snprintf(client_options, len, "%S", client_opt->opts);
+        client_options[len] = '\0';
+    }
+    
+    iter->cur++;
+}           
+
+void
+dr_client_iterator_stop(dr_client_iterator_t *iter)
+{
+    if (iter->valid)
+        free_opt_info(&iter->opt_info);
+    free(iter);
+}
+
+size_t
+dr_num_registered_clients(const char *process_name,
+                          dr_platform_t dr_platform)
+{
+    opt_info_t opt_info;
+    size_t num = 0;
+    ConfigGroup *policy = get_policy(dr_platform);
+    ConfigGroup *proc_policy = get_proc_policy(policy, process_name);
+
+    if (proc_policy == NULL)
+        goto exit;
+
+    if (read_options(&opt_info, proc_policy) != DR_SUCCESS)
+        goto exit;
+
+    num = opt_info.num_clients;
+    free_opt_info(&opt_info);
+    
+ exit:
+    if (policy != NULL)
+        free_config_group(policy);
+    return num;
+}
+
+
+dr_config_status_t
+dr_get_client_info(const char *process_name,
+                   dr_platform_t dr_platform,
+                   client_id_t client_id,
+                   size_t *client_pri,
+                   char *client_path,
+                   char *client_options)
+{
+    opt_info_t opt_info;
+    dr_config_status_t status;
+    size_t i;
+    ConfigGroup *policy = get_policy(dr_platform);
+    ConfigGroup *proc_policy = get_proc_policy(policy, process_name);
+
+    if (proc_policy == NULL) {
+        status = DR_PROC_REG_INVALID;
+        goto exit;
+    }
+
+    status = read_options(&opt_info, proc_policy);
+    if (status != DR_SUCCESS)
+        goto exit;
+
+    for (i=0; i<opt_info.num_clients; i++) {
+        if (opt_info.client_opts[i]->id == client_id) {
+            client_opt_t *client_opt = opt_info.client_opts[i];
+
+            if (client_pri != NULL) {
+                *client_pri = i;
+            }
+
+            if (client_path != NULL) {
+                size_t len = MIN(MAX_PATH-1, wcslen(client_opt->path));
+                _snprintf(client_path, len, "%S", client_opt->path);
+                client_path[len] = '\0';
+            }
+
+            if (client_options != NULL) {
+                size_t len = MIN(DR_MAX_OPTIONS_LENGTH-1, wcslen(client_opt->opts));
+                _snprintf(client_options, len, "%S", client_opt->opts);
+                client_options[len] = '\0';
+            }
+
+            status = DR_SUCCESS;
+            goto exit;
+        }
+    }
+
+    status = DR_ID_INVALID;
+
+ exit:
+    if (policy != NULL)
+        free_config_group(policy);
+    return status;
+}
+
+
+dr_config_status_t
+dr_register_client(const char *process_name,
+                   dr_platform_t dr_platform,
+                   client_id_t client_id,
+                   size_t client_pri,
+                   const char *client_path,
+                   const char *client_options)
+{
+    WCHAR new_opts[DR_MAX_OPTIONS_LENGTH];
+    WCHAR wpath[MAX_PATH], woptions[DR_MAX_OPTIONS_LENGTH];
+    ConfigGroup *policy = get_policy(dr_platform);
+    ConfigGroup *proc_policy = get_proc_policy(policy, process_name);
+    dr_config_status_t status;
+    opt_info_t opt_info;
+    bool opt_info_alloc = false;
+    size_t i;
+
+    if (proc_policy == NULL) {
+        status = DR_PROC_REG_INVALID;
+        goto exit;
+    }
+
+    status = read_options(&opt_info, proc_policy);
+    if (status != DR_SUCCESS) {
+        goto exit;
+    }
+    opt_info_alloc = true;
+
+    for (i=0; i<opt_info.num_clients; i++) {
+        if (opt_info.client_opts[i]->id == client_id) {
+            status = DR_ID_CONFLICTING;
+            goto exit;
+        }
+    }
+
+    if (client_pri > opt_info.num_clients) {
+        status = DR_PRIORITY_INVALID;
+        goto exit;
+    }
+
+    _snwprintf(wpath, MAX_PATH, L"%S", client_path);
+    NULL_TERMINATE_BUFFER(wpath);
+    _snwprintf(woptions, DR_MAX_OPTIONS_LENGTH, L"%S", client_options);
+    NULL_TERMINATE_BUFFER(woptions);
+
+    status = add_client_lib(&opt_info, client_id, client_pri, wpath, woptions);
+    if (status != DR_SUCCESS) {
+        goto exit;
+    }
+
+    /* write the registry */
+    write_options(&opt_info, new_opts);
+    set_config_group_parameter(proc_policy, L_DYNAMORIO_VAR_OPTIONS, new_opts);
+
+    if (write_config_group(policy) != ERROR_SUCCESS) {
+        status = DR_FAILURE;
+        goto exit;
+    }
+
+    status = DR_SUCCESS;
+
+ exit:
+    if (policy != NULL)
+        free_config_group(policy);
+    if (opt_info_alloc)
+        free_opt_info(&opt_info);
+    return status;
+}
+
+
+dr_config_status_t
+dr_unregister_client(const char *process_name,
+                     dr_platform_t dr_platform,
+                     client_id_t client_id)
+{
+    WCHAR new_opts[DR_MAX_OPTIONS_LENGTH];
+    ConfigGroup *policy = get_policy(dr_platform);
+    ConfigGroup *proc_policy = get_proc_policy(policy, process_name);
+    dr_config_status_t status;
+    opt_info_t opt_info;
+    bool opt_info_alloc = false;
+
+    policy = get_policy(dr_platform);
+    proc_policy = get_proc_policy(policy, process_name);
+    if (proc_policy == NULL) {
+        status = DR_PROC_REG_INVALID;
+        goto exit;
+    }
+
+    status = read_options(&opt_info, proc_policy);
+    if (status != DR_SUCCESS) {
+        goto exit;
+    }
+    opt_info_alloc = true;
+
+    status = remove_client_lib(&opt_info, client_id);
+    if (status != DR_SUCCESS) {
+        goto exit;
+    }
+
+    /* write the registry */
+    write_options(&opt_info, new_opts);
+    set_config_group_parameter(proc_policy, L_DYNAMORIO_VAR_OPTIONS, new_opts);
+
+    if (write_config_group(policy) != ERROR_SUCCESS) {
+        status = DR_FAILURE;
+        goto exit;
+    }
+
+    status = DR_SUCCESS;
+
+ exit:
+    if (policy != NULL)
+        free_config_group(policy);
+    if (opt_info_alloc)
+        free_opt_info(&opt_info);
+    return status;
+}
+
+
+typedef struct {
+    const char *process_name; /* if non-null nudges processes with matching name */
+    bool all; /* if set attempts to nudge all processes */
+    client_id_t client_id;
+    uint64 argument;
+    int count; /* number of nudges succesfully delivered */
+    DWORD res; /* last failing error code */
+    DWORD timeout; /* amount of time to wait for nudge to finish */
+} pw_nudge_callback_data_t;
+
+static
+BOOL pw_nudge_callback(process_info_t *pi, void **param)
+{
+    char buf[MAX_PATH];
+    pw_nudge_callback_data_t *data = (pw_nudge_callback_data_t *)param;
+
+    if (pi->ProcessID == 0)
+        return true; /* skip system process */
+
+    buf[0] = '\0';
+    if (pi->ProcessName != NULL)
+        _snprintf(buf, MAX_PATH, "%S", pi->ProcessName);
+    NULL_TERMINATE_BUFFER(buf);
+    if (data->all || (data->process_name != NULL &&
+                      strnicmp(data->process_name, buf, MAX_PATH) == 0)) {
+        DWORD res = generic_nudge(pi->ProcessID, true, NUDGE_GENERIC(client),
+                                  data->client_id, data->argument, data->timeout);
+        if (res == ERROR_SUCCESS || res == ERROR_TIMEOUT) {
+            data->count++;
+            if (res == ERROR_TIMEOUT && data->timeout != 0)
+                data->res = ERROR_TIMEOUT;
+        }
+        else if (res != ERROR_MOD_NOT_FOUND /* so failed for a good reason */)
+            data->res = res;
+    }
+
+    return true;
+}
+
+/* TODO: must be careful in invoking the correct VIPA's nudge handler,
+ * particularly a problem with multiple agents, but can be a problem even in a
+ * single agent if some other dll exports dr_nudge_handler() (remote
+ * contingency).
+ */
+
+dr_config_status_t
+dr_nudge_process(const char *process_name, client_id_t client_id, uint64 arg,
+                 uint timeout_ms, int *nudge_count /*OUT */)
+{
+    pw_nudge_callback_data_t data = {0};
+    data.process_name = process_name;
+    data.client_id = client_id;
+    data.argument = arg;
+    data.timeout = timeout_ms;
+    data.res = ERROR_SUCCESS;
+    process_walk(&pw_nudge_callback, (void **)&data);
+    if (nudge_count != NULL)
+        *nudge_count = data.count;
+
+    if (data.res == ERROR_SUCCESS)
+        return DR_SUCCESS;
+    else if (data.res == ERROR_TIMEOUT)
+        return DR_NUDGE_TIMEOUT;
+    else
+        return DR_FAILURE;
+}
+
+dr_config_status_t
+dr_nudge_pid(process_id_t process_id, client_id_t client_id, uint64 arg, uint timeout_ms)
+{
+    DWORD res = generic_nudge(process_id, true, NUDGE_GENERIC(client),
+                              client_id, arg, timeout_ms);
+
+    if (res == ERROR_SUCCESS)
+        return DR_SUCCESS;
+    if (res == ERROR_MOD_NOT_FOUND)
+        return DR_NUDGE_PID_NOT_INJECTED;
+    if (res == ERROR_TIMEOUT && timeout_ms != 0)
+        return DR_NUDGE_TIMEOUT;
+    return DR_FAILURE;
+}
+
+dr_config_status_t
+dr_nudge_all(client_id_t client_id, uint64 arg, uint timeout_ms, int *nudge_count /*OUT*/)
+{
+    pw_nudge_callback_data_t data = {0};
+    data.all = true;
+    data.client_id = client_id;
+    data.argument = arg;
+    data.timeout = timeout_ms;
+    data.res = ERROR_SUCCESS;
+    process_walk(&pw_nudge_callback, (void **)&data);
+    if (nudge_count != NULL)
+        *nudge_count = data.count;
+
+    if (data.res == ERROR_SUCCESS)
+        return DR_SUCCESS;
+    if (data.res == ERROR_TIMEOUT)
+        return DR_NUDGE_TIMEOUT;
+    return DR_FAILURE;
+}
diff --git a/libutil/drconfig.def b/libutil/drconfig.def
new file mode 100644
index 000000000..e5d995428
--- /dev/null
+++ b/libutil/drconfig.def
@@ -0,0 +1,49 @@
+;; **********************************************************
+;; Copyright (c) 2008 VMware, Inc.  All rights reserved.
+;; **********************************************************
+
+;; Redistribution and use in source and binary forms, with or without
+;; modification, are permitted provided that the following conditions are met:
+;; 
+;; * Redistributions of source code must retain the above copyright notice,
+;;   this list of conditions and the following disclaimer.
+;; 
+;; * Redistributions in binary form must reproduce the above copyright notice,
+;;   this list of conditions and the following disclaimer in the documentation
+;;   and/or other materials provided with the distribution.
+;; 
+;; * Neither the name of VMware, Inc. nor the names of its contributors may be
+;;   used to endorse or promote products derived from this software without
+;;   specific prior written permission.
+;; 
+;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+;; ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+;; LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+;; OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+;; DAMAGE.
+
+EXPORTS
+   dr_register_process
+   dr_unregister_process
+   dr_process_is_registered
+   dr_registered_process_iterator_start
+   dr_registered_process_iterator_hasnext
+   dr_registered_process_iterator_next
+   dr_registered_process_iterator_stop
+   dr_register_client
+   dr_unregister_client
+   dr_num_registered_clients
+   dr_get_client_info
+   dr_client_iterator_start
+   dr_client_iterator_hasnext
+   dr_client_iterator_next
+   dr_client_iterator_stop
+   dr_nudge_process
+   dr_nudge_pid
+   dr_nudge_all
diff --git a/libutil/dumpevts.c b/libutil/dumpevts.c
new file mode 100644
index 000000000..ef729a47a
--- /dev/null
+++ b/libutil/dumpevts.c
@@ -0,0 +1,127 @@
+/* **********************************************************
+ * Copyright (c) 2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+
+#include "share.h"
+#include "elm.h"
+
+#include <stdio.h>
+#include <time.h>
+
+int count = 0;
+int last = -1;
+
+void my_elm_formatted_cb(unsigned int mID, 
+                         unsigned int type,
+                         WCHAR *message,
+                         DWORD timestamp)
+{
+    char *type_s;
+    count++;
+
+    switch (type) {
+    case 0x1 : type_s = "ERROR"; break;
+    case 0x2 : type_s = "WARNING"; break;
+    case 0x4 : type_s = "INFO"; break;
+    default : type_s = "<unknown>"; break;
+    }
+    printf("Record %d, type=%s, %s%S\n", 
+           mID, type_s, ctime((long *)&timestamp), message); 
+}
+
+void
+my_elm_err_cb(unsigned int errcode, WCHAR *msg)
+{
+    if (errcode == ELM_ERR_FATAL) {
+        fprintf(stderr, "elm FATAL error: %S\n", msg);
+        exit(1);
+    }
+    else if (errcode == ELM_ERR_WARN) {
+        fprintf(stderr, "elm warning: %S\n", msg);
+        fflush(stderr);
+    }
+}
+
+extern BOOL do_once;
+
+void
+usage()
+{
+    fprintf(stderr, "Usage: dumpevts [-clear | -start N]\n");
+    fprintf(stderr, "   With no args, dumps full eventlog; use -start N to\n");
+    fprintf(stderr, "     dump starting with record N.\n");
+    exit(-1);
+}
+
+int
+main(int argc, char **argv) 
+{
+    DWORD res;
+
+    if (argc > 1) {
+        if (0 == strcmp(argv[1], "-clear")) {
+            DWORD res = clear_eventlog();
+            if (res != ERROR_SUCCESS)
+                fprintf(stderr, "Error %d clearing Event Log!\n", res);
+            else
+                printf("Eventlog cleared.\n");
+            return 0;
+        }
+        else if (0 == strcmp(argv[1], "-start")) {
+            if (argc > 2) {
+                last = atoi(argv[2]);
+            }
+            else {
+                usage();
+            }
+        }
+        else {
+            usage();
+        }
+    }
+
+    do_once = TRUE;
+    res = start_eventlog_monitor(TRUE, &my_elm_formatted_cb, 
+                                 NULL, &my_elm_err_cb,
+                                 last);
+    
+    if (res != ERROR_SUCCESS) {
+        printf("error %d starting monitor\n");
+    }
+
+    WaitForSingleObject(get_elm_thread_handle(), INFINITE);
+
+    if (count == 0)
+        printf("No Events found.\n");
+
+    return 0;
+}
diff --git a/libutil/elm.c b/libutil/elm.c
new file mode 100644
index 000000000..69f38969c
--- /dev/null
+++ b/libutil/elm.c
@@ -0,0 +1,506 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+
+//
+// -wrapping event log ids
+// -what happens when log is cleared
+//
+
+#include "share.h"
+#include "elm.h"
+#include "events.h"
+
+#ifndef UNIT_TEST
+
+#define BUFFER_SIZE 8192
+#define MAX_MSG_STRINGS 16
+
+int control;
+HANDLE elm_thread = NULL;
+
+/* if true, the formatted callback is used; otherwise the raw callback */
+BOOL format_messages;
+
+DWORD WINAPI 
+eventLogMonitorThreadProc(LPVOID elm_info_param);
+
+typedef 
+struct eventlogmon_info_ {
+    eventlog_formatted_callback cb_format;
+    eventlog_raw_callback cb_raw;
+    eventlog_error_callback cb_err;
+
+    /* this is passed to start_eventlog_monitor; nodemanager keeps
+     *  track of it in the registry. */
+    DWORD next_record;
+} eventlogmon_info;
+
+
+void
+stop_eventlog_monitor()
+{
+    BOOL wait = TRUE;
+    control = 1;
+    if (wait) {
+        WaitForSingleObject(elm_thread, 5000);
+    }
+    CloseHandle(elm_thread);
+    elm_thread = NULL;
+}
+
+/* returns a handle to the thread, which can be waited on */
+HANDLE
+get_elm_thread_handle()
+{
+    return elm_thread;
+}
+
+DWORD
+start_eventlog_monitor(BOOL use_formatted_callback,
+                       eventlog_formatted_callback cb_format, 
+                       eventlog_raw_callback cb_raw, 
+                       eventlog_error_callback cb_err,
+                       DWORD next_eventlog_record)
+{
+    DWORD tid;
+    eventlogmon_info *elmi;
+
+    if ((use_formatted_callback && cb_format == NULL) ||
+        (!use_formatted_callback && cb_raw == NULL))
+        return ERROR_INVALID_PARAMETER;
+
+    /* make sure there isn't already an elm running */
+    if (elm_thread != NULL)
+        return ERROR_ALREADY_INITIALIZED;
+
+    elmi = (eventlogmon_info *)malloc(sizeof(eventlogmon_info));
+    elmi->cb_format = cb_format;
+    elmi->cb_raw = cb_raw;
+    elmi->cb_err = cb_err;
+    elmi->next_record = next_eventlog_record;
+    format_messages = use_formatted_callback;
+    
+    elm_thread = CreateThread(NULL,
+                              0,
+                              &eventLogMonitorThreadProc,
+                              elmi,
+                              0,
+                              &tid);
+    if (elm_thread == NULL)
+        return GetLastError();
+
+    return ERROR_SUCCESS;
+}
+
+
+#define MAX_EVENTLOG_SOURCES 8
+
+
+BOOL eventsources_loaded = FALSE;
+
+DWORD
+get_formatted_message(EVENTLOGRECORD *pevlr, WCHAR *buf, DWORD maxchars)
+{
+    static int num_eventlog_sources = 0;
+    static HMODULE hm[MAX_EVENTLOG_SOURCES];
+    HKEY hes = NULL, hel = NULL;
+    DWORD len;
+    WCHAR message_file_buf[MAX_PATH];
+    LPBYTE msgarray[MAX_MSG_STRINGS];
+    int i = 0;
+    DWORD res;
+
+    /* load sources upon first request */
+    if (num_eventlog_sources == 0) {
+
+        /* We don't use product_key_flags() (PR 244206) b/c it's a pain to link
+         * with utils.obj, and we don't need the flags for HKLM\System
+         */
+        res = RegOpenKey(HKEY_LOCAL_MACHINE, EVENT_LOG_KEY, &hel);
+
+        if (res != ERROR_SUCCESS) {
+            return res;
+            goto formatted_msg_error;
+        }
+
+        for (;;) {
+
+            if (num_eventlog_sources >= MAX_EVENTLOG_SOURCES) {
+                res = ERROR_INSUFFICIENT_BUFFER;
+                goto formatted_msg_error;
+            }
+
+            len = MAX_PATH;
+            res = RegEnumKeyEx(hel, i++, message_file_buf, &len,
+                               0, NULL, NULL, NULL);
+            if (res == ERROR_NO_MORE_ITEMS) {
+                res = ERROR_SUCCESS;
+                break;
+            }
+            else if (res != ERROR_SUCCESS) {
+                goto formatted_msg_error;
+            }
+
+            /* We don't use product_key_flags() (PR 244206) b/c it's a pain to link
+             * with utils.obj, and we don't need the flags for HKLM\System
+             */
+            res = RegOpenKey(hel, message_file_buf, &hes);
+            if (res == ERROR_SUCCESS) {
+                len = MAX_PATH;
+                res = RegQueryValueEx(hes, L"EventMessageFile", NULL, 
+                                      NULL, (LPBYTE) message_file_buf, &len);
+                if (res == ERROR_SUCCESS) {
+                    /* load the module for the message resources */
+                    hm[num_eventlog_sources++] = 
+                        LoadLibraryEx(message_file_buf, NULL, 
+                                      LOAD_LIBRARY_AS_DATAFILE);
+                }
+            }
+            if (hes != NULL) {
+                RegCloseKey(hes);
+                hes = NULL;
+            }
+        }
+    }
+
+    /* now, format */
+    if(pevlr->NumStrings > MAX_MSG_STRINGS) {
+        res = ERROR_INSUFFICIENT_BUFFER;
+        goto formatted_msg_error;
+    }
+
+    msgarray[0]=(LPBYTE) pevlr + pevlr->StringOffset;
+
+    for(i = 1; i < pevlr->NumStrings; ++i) 
+        msgarray[i] = 
+            msgarray[i-1] + 
+            sizeof(WCHAR)*(1+wcslen((LPTSTR) msgarray[i-1]));
+
+    /* try each event source, returning only when FormatMessage succeeds.
+     * this is a little slutty as we could probably look up the event
+     *  source based from the event itself, but it's not too bad. */
+    len = 0;
+    for (i = 0; i < num_eventlog_sources; i++) {
+        if (hm[i] != NULL) {
+            len = FormatMessage(FORMAT_MESSAGE_FROM_HMODULE | 
+                                FORMAT_MESSAGE_ARGUMENT_ARRAY,
+                                hm[i],
+                                pevlr->EventID,
+                                0,
+                                buf,
+                                maxchars,
+                                (char **) msgarray);
+            if (len > 0)
+                break;
+        }
+    }
+
+    if(len == 0) {
+        res = ERROR_PARSE_ERROR;
+        goto formatted_msg_error;
+    }
+    else {
+        buf[maxchars - 1] = L'\0';
+        res = ERROR_SUCCESS;
+    }
+
+ formatted_msg_error:
+
+    if (hes != NULL)
+        RegCloseKey(hes);
+    if (hel != NULL)
+        RegCloseKey(hel);
+
+    return res;
+}
+
+
+
+BOOL do_once = FALSE;
+
+DWORD WINAPI
+eventLogMonitorThreadProc(LPVOID elm_info_param)
+{
+    EVENTLOGRECORD *pevlr; 
+    BYTE bBuffer[BUFFER_SIZE] = { 0 }; 
+    DWORD dwRead, dwNeeded, res;
+    DWORD reported_next_record, num_records;
+    BOOL skip_first = FALSE;
+    HANDLE log = NULL, event = NULL;
+    WCHAR msgbuf[BUFFER_SIZE];
+
+    eventlogmon_info *elm_info = (eventlogmon_info *) elm_info_param;
+  
+    control = 0;
+
+    log = OpenEventLog(NULL, L_COMPANY_NAME);
+    if (log == NULL) {
+        (*elm_info->cb_err)(ELM_ERR_FATAL, 
+                            L"Could not open the " L_COMPANY_NAME L" event log."); 
+        goto exit_elm_thread_error;
+    }
+    
+    event = CreateEvent(NULL, FALSE, FALSE, NULL);
+    NotifyChangeEventLog(log, event);   
+    
+    pevlr = (EVENTLOGRECORD *) &bBuffer; 
+
+    if (!GetNumberOfEventLogRecords(log, &num_records) ||
+        !GetOldestEventLogRecord(log, &reported_next_record)) {
+        _snwprintf(msgbuf, BUFFER_SIZE_ELEMENTS(msgbuf),
+                   L"error %d getting eventlog info", GetLastError());
+        (*elm_info->cb_err)(ELM_ERR_FATAL, msgbuf);
+        goto exit_elm_thread_error;
+    }
+
+    /* FIXME: we don't handle the situation when the eventlog was cleared,
+     *  but our pointer is less than the number of new records. for this
+     *  we'll probably have to store a timestamp, and compare against the
+     *  event record at next_record. */
+    if (((int)elm_info->next_record) < 0) {
+        elm_info->next_record = reported_next_record;
+    } 
+    else if (elm_info->next_record > (reported_next_record + num_records + 1)) {
+        /* looks like the eventlog was cleared since we last checked
+         *  it. issue a warning and reset */
+        elm_info->next_record = reported_next_record;
+        (*elm_info->cb_err)(ELM_ERR_CLEARED, L"Eventlog was cleared!\n");
+    }
+    else {
+        /* we need to ensure we SEEK to a valid record; but since
+         * it's already been reported, don't report it again. */
+        elm_info->next_record--;
+        skip_first = TRUE;
+    }
+
+    /* first seek to the last record 
+     * EVENTLOG_FORWARDS_READ indicates we will get messages in
+     *  chronological order.
+     * FIXME: test to make sure that this works properly on
+     *  overwrite-wrapped logs */
+    if (!ReadEventLog(log,                
+                      EVENTLOG_FORWARDS_READ | EVENTLOG_SEEK_READ,
+                      elm_info->next_record,
+                      pevlr,
+                      BUFFER_SIZE,
+                      &dwRead,
+                      &dwNeeded)) {
+        dwRead = 0;
+        dwNeeded = 0;
+    }
+
+    for(;;) {        
+        do {
+            /* case 5813: if pevlr->Length is 0, we'll have an infinite
+             * loop. this could possibly happen if drive is full?
+             * just abort if we detect it. */
+            while (dwRead > 0 && pevlr->Length > 0) {
+                
+                if (format_messages && !skip_first) {
+
+                    res = get_formatted_message(pevlr, msgbuf, BUFFER_SIZE);
+                    if (res != ERROR_SUCCESS) {
+                        _snwprintf(msgbuf, BUFFER_SIZE_ELEMENTS(msgbuf),
+                                   L"FormatMessage error %d\n", res);
+                        (*elm_info->cb_err)(ELM_ERR_WARN, msgbuf);
+                    }
+                    else {
+                        /* invoke the callback */
+                        (*elm_info->cb_format)(pevlr->RecordNumber, 
+                                               pevlr->EventType, 
+                                               msgbuf, pevlr->TimeGenerated);
+                    }
+                } else if(!skip_first) {
+                    /* xref case 3065: insurance */
+                    if (pevlr->RecordNumber != 0 ||
+                        pevlr->TimeGenerated != 0) {
+                        /* raw callback */
+                        (*elm_info->cb_raw)(pevlr);
+                    }
+                } else {
+                    skip_first = FALSE;
+                }
+
+                dwRead -= pevlr->Length; 
+                pevlr = (EVENTLOGRECORD *) ((LPBYTE) pevlr + pevlr->Length); 
+            }
+            
+            pevlr = (EVENTLOGRECORD *) &bBuffer; 
+        } while (ReadEventLog(log,                
+                              EVENTLOG_FORWARDS_READ |  
+                              EVENTLOG_SEQUENTIAL_READ, 
+                              0,            
+                              pevlr,        
+                              BUFFER_SIZE,  
+                              &dwRead,      
+                              &dwNeeded));
+        
+        if((res = GetLastError()) != ERROR_HANDLE_EOF) {
+            //FIXME: assert GetLastError() is appropriate
+            _snwprintf(msgbuf, BUFFER_SIZE_ELEMENTS(msgbuf),
+                       L"Unexpected error %d reading event log\n", res);
+            (*elm_info->cb_err)(ELM_ERR_WARN, msgbuf);
+        }
+
+        if (do_once)
+            break;
+
+        /* the event is auto-reset.
+           timeout because NotifyChangeEventLog is not reliable. */
+        WaitForSingleObject(event, MINIPULSE);
+
+        if(control)
+            break;
+    }
+    
+exit_elm_thread_error:
+
+    if (log != NULL)
+        CloseEventLog(log); 
+
+    if (event != NULL)
+        CloseHandle(event);
+
+    free(elm_info);
+
+    /* FIXME: need ExitThread()? */
+    return 0;
+}
+
+/* insertion strings are null separated. for pevlr of type
+ *  EVENTLOGRECORD, the first string is at 
+ * ((LPBYTE) pevlr) + pevlr->StringOffset
+ *  undefined results if next_message_string is called more than
+ *  NumStrings times. */
+const WCHAR *
+next_message_string(const WCHAR *prev_string)
+{
+    return prev_string + wcslen(prev_string) + 1;
+}
+
+const WCHAR *
+get_message_strings(EVENTLOGRECORD *pevlr)
+{
+    return (WCHAR *)(((LPBYTE)pevlr) + pevlr->StringOffset);
+}
+
+const WCHAR *
+get_event_exename(EVENTLOGRECORD *pevlr)
+{
+    /* exename is always the first msg string */
+    WCHAR *exename;
+    const WCHAR *exepath = get_message_strings(pevlr);
+    exename = wcsrchr(exepath, L'\\');
+    if (exename == NULL)
+        return exepath;
+    else
+        return exename + 1;
+}
+
+UINT
+get_event_pid(EVENTLOGRECORD *pevlr)
+{
+    DO_ASSERT(pevlr != NULL);
+
+    /* PID is always the second msg string */
+    return (UINT) _wtoi(next_message_string(get_message_strings(pevlr)));
+    
+}
+
+/* For a MSG_SEC_FORENSICS eventlog record, returns the filename of the forensics file
+ * generated. */
+const WCHAR *
+get_forensics_filename(EVENTLOGRECORD *pevlr)
+{
+    DO_ASSERT(pevlr != NULL && pevlr->EventID == MSG_SEC_FORENSICS);
+    /* forensics file pathname will be third string */
+    return next_message_string(next_message_string(get_message_strings(pevlr)));
+}
+
+BOOL
+is_violation_event(DWORD eventType)
+{
+    switch(eventType) {
+    case MSG_HOT_PATCH_VIOLATION:
+    case MSG_SEC_VIOLATION_TERMINATED:
+    case MSG_SEC_VIOLATION_CONTINUE:
+    case MSG_SEC_VIOLATION_THREAD:
+    case MSG_SEC_VIOLATION_EXCEPTION:
+        return TRUE;
+    default:
+        return FALSE;
+    }
+}
+
+const WCHAR *
+get_event_threatid(EVENTLOGRECORD *pevlr)
+{
+    DO_ASSERT(pevlr != NULL);
+
+    /* The Threat ID, if available, is always the 3rd parameter. */
+    if (is_violation_event(pevlr->EventID)) {
+        return next_message_string(next_message_string(get_message_strings(pevlr)));
+    }
+    else {
+        return NULL;
+    }
+}
+
+DWORD
+clear_eventlog()
+{
+    DWORD res = ERROR_SUCCESS;
+    HANDLE log = OpenEventLog(NULL, L_COMPANY_NAME);
+    if (log == NULL)
+        return GetLastError();
+    if (!ClearEventLog(log, NULL))
+        res = GetLastError();
+    CloseHandle(log);
+    return res;
+}
+
+#else //ifdef UNIT_TEST
+
+
+int
+main()
+{
+    //DWORD res;
+
+    printf("All Test Passed\n");
+
+    return 0;
+
+}
+
+#endif
diff --git a/libutil/elm.h b/libutil/elm.h
new file mode 100644
index 000000000..03bdd7bc0
--- /dev/null
+++ b/libutil/elm.h
@@ -0,0 +1,78 @@
+/* **********************************************************
+ * Copyright (c) 2003-2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ *
+ * elm.h
+ *
+ * TODO:
+ *  -allow reporting of only "recent" events (eg after a given time)
+ */
+
+#ifndef _DETERMINA_ELM_H_
+#define _DETERMINA_ELM_H_
+
+/*
+ *
+ * BEWARE! from the msdn docs:
+
+" When an event is written to the logfile specified by hEventLog, the
+system uses the PulseEvent function to set the event specified by the
+hEvent parameter to the signaled state. If the thread is not waiting
+on the event when the system calls PulseEvent, the thread will not
+receive the notification. Therefore, you should create a separate
+thread to wait for notifications.
+
+Note that the system calls PulseEvent no more than once every five
+seconds. Therefore, even if more than one event log change occurs
+within a five-second interval, you will only receive one
+notification."
+
+ *
+ * to this end, the 'MINIPULSE' define below sets the polling 
+ *  interval, in ms
+ */
+
+#define MINIPULSE 100
+
+HANDLE
+get_elm_thread_handle();
+
+const WCHAR *
+get_message_strings(EVENTLOGRECORD *pevlr);
+
+/* For a MSG_SEC_FORENSICS eventlog record, returns the filename of the forensics file
+ * generated. */
+const WCHAR *
+get_forensics_filename(EVENTLOGRECORD *pevlr);
+
+#endif
diff --git a/libutil/mfapi.c b/libutil/mfapi.c
new file mode 100644
index 000000000..8383afbf7
--- /dev/null
+++ b/libutil/mfapi.c
@@ -0,0 +1,214 @@
+/* **********************************************************
+ * Copyright (c) 2005-2006 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ *
+ * implementations of API wrappers
+ *
+ */
+
+#include "share.h"
+#include "config.h"
+#include "processes.h"
+#include "elm.h"
+
+
+#ifndef UNIT_TEST
+
+
+DWORD
+disable_protection()
+{
+    return unset_autoinjection();
+}
+
+DWORD
+enable_protection()
+{
+    return set_autoinjection();
+}
+
+BOOL
+is_protection_enabled()
+{
+    return is_autoinjection_set();
+}
+
+DWORD
+enable_protection_ex(BOOL inject, DWORD flags, 
+                     const WCHAR *blacklist, 
+                     const WCHAR *whitelist, DWORD *list_error, 
+                     const WCHAR *custom_preinject_name, 
+                     WCHAR *current_list, UINT maxchars)
+{
+    return set_autoinjection_ex(inject, flags, blacklist, whitelist,
+                                list_error, custom_preinject_name,
+                                current_list, maxchars);
+}
+
+DWORD
+inject_status(int pid, DWORD *status, DWORD *build)
+{
+    DWORD dll_stat, res = ERROR_SUCCESS;
+    
+    dll_stat = under_dynamorio_ex(pid, build);
+
+    switch (dll_stat) {
+    case DLL_NONE:
+        *status = INJECT_STATUS_NATIVE;
+        break;
+    case DLL_UNKNOWN:
+        *status = INJECT_STATUS_UNKNOWN;
+        break;
+    case DLL_RELEASE:
+    case DLL_DEBUG:
+    case DLL_PROFILE:
+    case DLL_CUSTOM:
+        *status = INJECT_STATUS_PROTECTED;
+        break;
+    case DLL_PATHHAS:
+    default:
+        *status = INJECT_STATUS_UNKNOWN;
+        res = ERROR_INVALID_PARAMETER;
+        break;
+    }
+
+    return res;
+}
+
+DWORD
+consistency_detach(DWORD timeout)
+{
+    DWORD res;
+    ConfigGroup *policy;
+
+    if (is_protection_enabled()) {
+        res = read_config_group(&policy, L_PRODUCT_NAME, TRUE);
+        if (res != ERROR_SUCCESS)
+            return res;
+
+        res = detach_all_not_in_config_group(policy, timeout);
+
+        free_config_group(policy);
+    }
+    else {
+        res = detach_all(timeout);
+    }
+
+    return res;
+}
+
+BOOL
+is_process_pending_restart(int pid)
+{
+    DWORD res;
+    ConfigGroup *policy;
+    BOOL pending_restart;
+
+    res = read_config_group(&policy, L_PRODUCT_NAME, TRUE);
+    if (res != ERROR_SUCCESS)
+        return FALSE;
+
+    res = check_status_and_pending_restart(policy, pid, 
+                                           &pending_restart, 
+                                           NULL, NULL);
+    if (res != ERROR_SUCCESS)
+        return FALSE;
+
+    free_config_group(policy);
+
+    return pending_restart;
+}
+
+BOOL
+is_any_process_pending_restart()
+{
+    DWORD res;
+    ConfigGroup *policy;
+    BOOL pending_restart;
+
+    res = read_config_group(&policy, L_PRODUCT_NAME, TRUE);
+    if (res != ERROR_SUCCESS)
+        return FALSE;
+
+    res = is_anything_pending_restart(policy, &pending_restart);
+    if (res != ERROR_SUCCESS)
+        return FALSE;
+
+    free_config_group(policy);
+
+    return pending_restart;
+}
+
+HANDLE
+get_eventlog_monitor_thread_handle()
+{
+    return get_elm_thread_handle();
+}
+
+
+const WCHAR *
+get_installation_path()
+{
+    return get_dynamorio_home();
+}
+
+const WCHAR *
+get_product_name()
+{
+    return L_PRODUCT_NAME;
+}
+
+
+
+#else // ifdef UNIT_TEST
+
+
+int
+main()
+{
+    set_debuglevel(DL_INFO);
+    set_abortlevel(DL_WARN);
+
+    /* FIXME: wrapper testing (but may not be necessary...) */
+
+    DO_ASSERT(get_eventlog_monitor_thread_handle() == NULL);
+    CHECKED_OPERATION(clear_policy());
+    DO_ASSERT(!is_any_process_pending_restart());
+    
+    printf("All Test Passed\n");
+
+    return 0;
+
+}
+
+#endif
diff --git a/libutil/mfapi.def b/libutil/mfapi.def
new file mode 100644
index 000000000..e89c2b521
--- /dev/null
+++ b/libutil/mfapi.def
@@ -0,0 +1,65 @@
+;; **********************************************************
+;; Copyright (c) 2005-2006 VMware, Inc.  All rights reserved.
+;; **********************************************************
+
+;; Redistribution and use in source and binary forms, with or without
+;; modification, are permitted provided that the following conditions are met:
+;;
+;; * Redistributions of source code must retain the above copyright notice,
+;;   this list of conditions and the following disclaimer.
+;;
+;; * Redistributions in binary form must reproduce the above copyright notice,
+;;   this list of conditions and the following disclaimer in the documentation
+;;   and/or other materials provided with the distribution.
+;;
+;; * Neither the name of VMware, Inc. nor the names of its contributors may be
+;;   used to endorse or promote products derived from this software without
+;;   specific prior written permission.
+;;
+;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+;; ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+;; LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+;; OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+;; DAMAGE.
+
+LIBRARY MFAPI
+EXPORTS
+   policy_import
+   policy_export
+   load_policy
+   save_policy
+   validate_policy
+   clear_policy
+   enable_protection
+   is_protection_enabled
+   disable_protection
+   enable_protection_ex
+   enumerate_processes
+   inject_status
+   is_process_pending_restart
+   is_any_process_pending_restart
+   detach
+   detach_all
+   detach_exe
+   consistency_detach
+   hotp_notify_modes_update
+   hotp_notify_defs_update
+   hotp_notify_all_modes_update
+   hotp_notify_exe_modes_update
+   hotp_notify_all_defs_update
+   start_eventlog_monitor
+   stop_eventlog_monitor
+   get_eventlog_monitor_thread_handle
+   next_message_string
+   get_event_exename
+   get_event_pid
+   get_platform
+   get_installation_path
+   get_product_name
+ 
diff --git a/libutil/mfapi.h b/libutil/mfapi.h
new file mode 100644
index 000000000..fe72ae675
--- /dev/null
+++ b/libutil/mfapi.h
@@ -0,0 +1,580 @@
+/* **********************************************************
+ * Copyright (c) 2005-2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * mfapi.h
+ *
+ * Interface for controlling Determina protection on a local machine
+ *
+ * (assumes a proper installation exists)
+ *
+ */
+
+
+#ifndef _DETERMINA_MFAPI_H_
+#define _DETERMINA_MFAPI_H_
+
+#ifndef _WIN32_WINNT
+#define _WIN32_WINNT 0x0500
+#endif
+
+#include <windows.h>
+#include <wchar.h>
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+/* custom Win32-style error codes; using "high" constants so that
+ *   it's obvious it's not win32. */
+#define ERROR_OPTION_NOT_FOUND     0xffffffff
+#define ERROR_NOT_INITIALIZED      0xfffffffe
+#define ERROR_UNKNOWN_ENTRY        0xfffffffd
+#define ERROR_DETACH_NOT_ALLOWED   0xfffffffc
+#define ERROR_UNSUPPORTED_OS       0xfffffffb
+#define ERROR_LIST_VIOLATION       0xfffffffa
+#define ERROR_LENGTH_VIOLATION     0xfffffff9
+#define ERROR_DETACH_ERROR         0xfffffff8
+#define ERROR_PARSE_ERROR          0xfffffff7
+#define ERROR_DRMARKER_ERROR       0xfffffff6
+
+/************************
+ * policy import/export *
+ ************************/
+
+/*
+ *
+ * policy definition format specification
+ * --------------------------------------
+ *
+ *
+ * the syntax of the policy definition message is as follows:
+
+ * <policy_message> ::==
+ *  POLICY_VERSION=30000
+ *  APPINITFLAGS=<string>
+ *  APPINITWHITELIST=<string>
+ *  APPINITBLACKLIST=<string>
+ *  GLOBAL_PROTECT=<boolean>
+ *  <application_block>*
+ * 
+ * <application_block> ::==
+ *  BEGIN_BLOCK
+ *  <app_specifier>
+ *  [ <hot_patch_block> ]
+ *  <dynamorio_option_line>*
+ *  END_BLOCK
+ * 
+ * <app_specifier> ::== [ GLOBAL | APP_NAME=<string> ]
+ * 
+ * <dynamorio_option_line> ::== <dynamorio_option>=<string>
+ * 
+ * <dynamorio_option> ::== 
+ *  [ DYNAMORIO_OPTIONS |
+ *    DYNAMORIO_AUTOINJECT |
+ *    DYNAMORIO_RUNUNDER |
+ *    DYNAMORIO_HOT_PATCH_MODES |
+ *    DYNAMORIO_HOT_PATCH_POLICIES ]
+ * 
+ * <hot_patch_block> ::==
+ *  BEGIN_MP_MODES
+ *  <num_hot_patch_lines>
+ *  <hot_patch_line>*
+ *  END_MP_MODES
+ * 
+ * <num_hot_patch_lines> ::== (int)
+ *
+ * <hot_patch_line> ::==
+ *  <patch_id>:<patch_mode>
+ *
+ * <patch_mode> ::== [ 0 | 1 | 2 ]
+ *
+ * <patch_id>: provided by Determina
+ *
+ * <boolean> ::== [ 0 | 1 ]
+ * 
+ * 
+ * 
+ * details:
+ * 
+ * (1) the APPINITFLAGS, together with the APPINITBLACKLIST and
+ * APPINITWHITELIST, controls how our bootstrap dll is added to the
+ * AppInit_DLLs registry key. the value of the flags should be a sum
+ * of the APPINIT_* flags as defined in share/config.h
+ * 
+ * The APPINITBLACKLIST and APPINITWHITELIST values are only used if
+ * specified by the flags. we'll provide the whitelist/blacklist.
+ * 
+ * 
+ * (2) GLOBAL_PROTECT: OPTIONAL: if this is 0, then protection is
+ * disabled (and all application blocks are optional). in normal
+ * situations this will be 1. if this is not specified, then the
+ * current setting is kept (see also enable_protection)
+ * 
+ * 
+ * (3) there must be a GLOBAL block, which should come first, and it
+ * must have DYNAMORIO_RUNUNDER=1 set as an option. for consistency it
+ * should set DYNAMORIO_OPTIONS to blank (DYNAMORIO_OPTIONS=).
+ * 
+ * 
+ * (4) the APP_NAME must be a valid "application id". this is usually
+ * the .exe filename, but can also be "qualified". we will provide a
+ * complete list of supported applications.
+ * 
+ * 
+ * (5) some applications will also require special values for
+ * DYNAMORIO_RUNUNDER; we will provide this information as well.
+ * 
+ * 
+ * (6) DYNAMORIO_AUTOINJECT must be specified for every non-global
+ * application block, and it should be of the format
+ * "\lib\NNNNN\dynamorio.dll", where NNNNN is the version number of
+ * the core dll being used. in general, any value that starts with a
+ * "\" will have the local installation path prepended when using the
+ * actual value (eg, it indicates a relative installation path).
+ * 
+ * 
+ * (7) DYNAMORIO_OPTIONS is a string consisting of protection options
+ * to our core. the variability in this will depend on the level of
+ * configurability desired in your UI; but the majority of the options
+ * will not need to change. probably the only configuration necessary
+ * will be whether an app should be on or off; and this should be
+ * controlled by including or excluding the corresponding app block.
+ * 
+ * 
+ * (8) to enable hot patching, there are three additional 
+ *     requirements:
+ *    (a) the global DYNAMORIO_HOT_PATCH_DEFS option should be 
+ *        declared, and point to the relative subpath of the hot patch 
+ *        definitions file (provided with the update package). Typically 
+ *        this should just be "\config" -- the engine version and
+ *        file name is standard. Of course this file must exist in the 
+ *        proper place: "\config\30000\hotp-defs.cfg".
+ *    (b) similary, each app must provide the DYNAMORIO_HOT_PATCH_MODES
+ *        subpath for the hotp modes file. Typically this will be the
+ *        config directory followed by the app name, e.g., 
+ *        "\config\detertest.exe". The modes files are created by the 
+ *        import process and should NOT be created manually.
+ *    (c) Each app that requires patches must specify a list of patch 
+ *        id's and their "modes".  The mode codes are 0 = Off, 1 =
+ *        Detect Only, 2 = Protect.  Typically only modes 0 and 2 will
+ *        be used. As indicated above, the first line must be the number
+ *        of patches specified. The list of patch id's and their 
+ *        descriptions is provided by Determina in the patch update 
+ *        packages.
+ * 
+ * See sample.mfp for an example policy string.
+ * 
+ */
+
+
+/* import the specified configuration, thus enabling Determina
+ *  protection for all newly launched processes. note that, according
+ *  to the policy definition (GLOBAL_PROTECT), this will either call
+ *  enable_protection or disable_protection.
+ *
+ * if synchronize_system, then any processes were under Determina
+ *  but are now configured to be off will be detached. otherwise,
+ *  no changes will be made to any running processes. 
+ *
+ * if inject_flag is not NULL, it will return whether or not injection
+ *  should be turned on, e.g., with the enable_protection()
+ *  method. this is mainly to allow clients to call
+ *  enable_protection_ex() with various custom flags. if inject_flag
+ *  is NULL, the import method will take care of enabling or disabling
+ *  protection based on the policy.
+ *
+ * if warning is not NULL, it will be set to an error
+ *   code indicating if a non-critical error (such as detach) was 
+ *   encountered.
+ *
+ * note that unlike other methods, the policy definition is a char
+ *   (not WCHAR) -- this is because (presumably) the policy 
+ *   definition is being read from an ASCII file or a network 
+ *   connection. */
+DWORD
+policy_import(char *policy_definition, BOOL synchronize_system, 
+              BOOL *inject_flag, DWORD *warning);
+
+/* returns a policy definition string based on the current registry
+ *  configuration. 
+ * may return ERROR_MORE_DATA, in which case the call should
+ *  be re-tried with a larger buffer. 
+ * again note char buffer. */
+DWORD
+policy_export(char *policy_buffer, UINT maxchars, UINT *needed);
+
+/* convenience method for loading from a file */
+DWORD
+load_policy(WCHAR *filename, BOOL synchronize_system, DWORD *warning);
+
+/* convenience method for storing to a file */
+DWORD
+save_policy(WCHAR *filename);
+
+/* returns ERROR_SUCCESS unless the policy_buffer is invalid. */
+DWORD
+validate_policy(char *policy_buffer);
+
+/* writes an empty policy to the registry (so that no apps will be
+ *  protected). distinct from disable_protection, which just clears
+ *  the AppInit_DLLs key. */
+DWORD
+clear_policy();
+
+
+/************************
+ * AppInit_DLLs setting *
+ ************************/
+
+/* enable Determina protection on the system. note that this is not
+ *  required if implicitly invoked in policy_import. */
+DWORD 
+enable_protection();
+
+/* checks the current protection status. */
+BOOL
+is_protection_enabled();
+
+/* disable Determina protection on the system. */
+DWORD
+disable_protection();
+
+
+/*
+ * For more detailed control over the AppInit_DLLs key, use
+ *  the following function.
+ *
+ * AppInit_DLLs flags:
+ *   (for enable_protection_ex())
+ *
+ * APPINIT_FORCE_TO_FRONT
+ *   Forces preinject dll to the front of the AppInit_DLLs list.
+ *
+ * APPINIT_FORCE_TO_BACK
+ *   Forces preinject dll to the back of the AppInit_DLLs list.
+ *
+ * APPINIT_USE_BLACKLIST
+ *   Will read blacklist from blacklist config parameter and 
+ *     validate against it.
+ *
+ * APPINIT_USE_WHITELIST
+ *   Will read whitelist from whitelist config parameter and 
+ *     validate against it.
+ *
+ * APPINIT_CHECK_LISTS_ONLY
+ *   If set, the lists are only checked, not enforced; intended
+ *     for use with at least one of the two flags below.
+ *
+ * APPINIT_WARN_ON_LIST_VIOLATION
+ *   Whether to generate a warning if a whitelist/blacklist
+ *    violation was detected.
+ *
+ * APPINIT_BAIL_ON_LIST_VIOLATION
+ *   If set, the preinjector will not be added to the
+ *     list, thus effectively disabling Determina.
+ * 
+ *
+ * The following flags apply only for platforms which 
+ *   require preinject to be in SYSTEM32 (ie, NT4 for now).
+ *
+ * APPINIT_SYS32_USE_LENGTH_WORKAROUND
+ *   Not yet supported -- must be zero. Placeholder
+ *     for using some alternate technique to handle
+ *     other dlls which may be in the list.
+ *
+ * APPINIT_SYS32_FAIL_ON_LENGTH_ERROR
+ *   If existing entries + preinject name 
+ *    exceeds the 31-character limit, leave key
+ *    untouched and report an error.
+ *
+ * APPINIT_SYS32_CLEAR_OTHERS
+ *   If set, any other appinit entries are
+ *    cleared, regardless of length. otherwise,
+ *    all entries are left intact, and an 
+ *    operation that results in too long of 
+ *    a string will be truncated but succeed
+ *    unless APPINIT_SYS32_FAIL_ON_LENGTH_ERROR
+ *    is set.
+ *
+ * APPINIT_SYS32_TRUNCATE
+ *   If set, appinit is truncated to 31 chars
+ *    before writing; otherwise, the entire
+ *    string is written, even if over the limit.
+ *
+ */
+
+#define APPINIT_FORCE_TO_FRONT               0x1
+#define APPINIT_FORCE_TO_BACK                0x2
+#define APPINIT_USE_BLACKLIST                0x4
+#define APPINIT_USE_WHITELIST                0x8
+#define APPINIT_CHECK_LISTS_ONLY             0x10
+#define APPINIT_WARN_ON_LIST_VIOLATION       0x20
+#define APPINIT_BAIL_ON_LIST_VIOLATION       0x40
+#define APPINIT_SYS32_USE_LENGTH_WORKAROUND  0x100
+#define APPINIT_SYS32_FAIL_ON_LENGTH_ERROR   0x200
+#define APPINIT_SYS32_CLEAR_OTHERS           0x400
+#define APPINIT_SYS32_TRUNCATE               0x800
+#define APPINIT_OVERWRITE                    0x1000
+
+/* The flags, blacklist, whitelist, list_error parameters are as
+ *  described above.
+ * The inject parameter controls whether protection is enabled or
+ *  disabled. If disabled all other parameters are ignored.
+ * The custom_preinject_name allows a non-standard DLL to be used;
+ *  generally this should be NULL.
+ * The current_list buffer (of size maxchars) is optional; if provided
+ *  it will contain the contents of the AppInit_DLLs key before any
+ *  changes are made; this is useful in the case that list_error
+ *  violation is reported.
+ */
+DWORD
+enable_protection_ex(BOOL inject, DWORD flags, 
+                     const WCHAR *blacklist, 
+                     const WCHAR *whitelist, DWORD *list_error, 
+                     const WCHAR *custom_preinject_name, 
+                     WCHAR *current_list, UINT maxchars);
+
+
+/*************************
+ *    process status     *
+ *************************/
+
+/* generic process walk callback routine; the callback should 
+ *  return FALSE to abort the walk. optionally takes a parameter
+ *  which can be passed to the enumerate_processes method. */
+typedef BOOL (*process_callback)(ULONG pid, WCHAR *process_name, 
+                                 void **param);
+
+/* helper method: for each running processes on the system, 
+ *  executes the callback with the specified info. (everything is
+ *  synchronous here.) 
+ * note that this is much cleaner than the Win32 api's tlhelp32 
+ *  interface, which should NEVER be used! */ 
+DWORD
+enumerate_processes(process_callback pcb, void **param);
+
+
+/* process status codes */
+#define INJECT_STATUS_PROTECTED 1
+#define INJECT_STATUS_NATIVE    2
+#define INJECT_STATUS_UNKNOWN   3
+
+/* returns one of above status definitions in the status pointer.
+ * if the build pointer is non-NULL, returns the build number of the
+ *   core (if the process is protected).*/
+DWORD
+inject_status(int pid, DWORD *status, DWORD *build);
+
+/* returns TRUE if the indicated process is configured for 
+ *  protection, yet is not running under Determina (this means it 
+ *  must be restarted before protection will take effect).
+ * returns FALSE if the process is under Determina or not configured 
+ *  for protection. (or in the case of any error.)
+ * note this is fairly expensive because it needs to load the current
+ *  policy from the registry and check against it. a more efficient
+ *  API may be exposed in the future.
+ */
+BOOL
+is_process_pending_restart(int pid);
+
+/* returns TRUE if any process is_pending_restart */
+BOOL
+is_any_process_pending_restart();
+
+
+
+
+/********************
+ *  detach / nudge  *
+ ********************/
+
+
+/* in ms */
+#define DETACH_RECOMMENDED_TIMEOUT 60000
+#define NUDGE_NO_DELAY 0
+
+/* this is recommended for all hotp notification methods. */
+#define NUDGE_RECOMMENDED_PAUSE 100 
+
+
+/* detach from indicated process -- if allow_upgraded_perms, then 
+ *  attempt to acquire necessary privileges (usually necessary) */
+DWORD
+detach(int pid, BOOL allow_upgraded_perms, DWORD timeout_ms);
+
+/* detach from all running processes (with timeout).
+ * timeout is per process */
+DWORD
+detach_all(DWORD timeout);
+
+/* detach from all processes with the matching exe name */
+DWORD
+detach_exe(WCHAR *exename, DWORD timeout_ms);
+
+/* make sure that only configured processes are running under
+ *   Determina */
+DWORD
+consistency_detach(DWORD timeout);
+
+/* use this to notify a process to use new configuration information
+ *  in the memory patch "modes" file. */
+DWORD
+hotp_notify_modes_update(int pid, BOOL allow_upgraded_perms, DWORD timeout_ms);
+
+/* use this to notify a process to use new configuration information
+ *  in the memory patch "definitions" file -- note that since new
+ *  policy definitions will be re-loaded, this will also cause the
+ *  modes information to be re-read as well. */
+DWORD
+hotp_notify_defs_update(int pid, BOOL allow_upgraded_perms, DWORD timeout_ms);
+
+/* get modes update for all running processes. timeout is per process */
+DWORD
+hotp_notify_all_modes_update(DWORD timeout_ms);
+
+/* get defs update for all running processes. timeout is per process */
+DWORD
+hotp_notify_all_defs_update(DWORD timeout_ms);
+
+/* get modes update for all running processes matching exe name. 
+ *  timeout is per process */
+DWORD
+hotp_notify_exe_modes_update(WCHAR *exename, DWORD timeout_ms);
+
+
+/************************
+ *      event log       *
+ ************************/
+
+/*
+ * start_eventlog_monitor:
+ * spawns a new thread which monitors the Araksha event log for
+ *  new messages. the eventlog_callback is called for each new
+ *  event with parameters as described above. any warnings, errors,
+ *  failurse, etc. trigger the cb_err callback.
+ * returns a handle to the listener thread.
+ *
+ * use stop_eventlog_monitor to cleanly destroy the listener thread
+ */
+
+/* monitor error codes */
+#define ELM_ERR_FATAL 1
+#define ELM_ERR_WARN  2
+#define ELM_ERR_CLEARED 3
+
+typedef void (*eventlog_formatted_callback)(unsigned int mID, 
+                                            unsigned int type,
+                                            WCHAR *message,
+                                            DWORD timestamp);
+
+typedef void (*eventlog_raw_callback)(EVENTLOGRECORD *record);
+
+typedef void (*eventlog_error_callback)(unsigned int errcode,
+                                        WCHAR *message);
+
+/* last_record must be a valid eventlog record number!!
+ * this can be obtained from the "raw" callback by the RecordNumber
+ *   value of the EVENTLOGRECORD struct.
+ * pass -1 to retrieve all event records. */
+DWORD
+start_eventlog_monitor(BOOL use_formatted_callback,
+                       eventlog_formatted_callback cb_format, 
+                       eventlog_raw_callback cb_raw, 
+                       eventlog_error_callback cb_err,
+                       DWORD next_record);
+
+void
+stop_eventlog_monitor();
+
+/* removes all events from the Determina event log */
+DWORD
+clear_eventlog();
+
+HANDLE
+get_eventlog_monitor_thread_handle();
+
+BOOL
+is_violation_event(DWORD eventType);
+
+/* helper method for parsing message strings in an EVENTLOGRECORD */
+const WCHAR *
+next_message_string(const WCHAR *prev_string);
+
+/* helper method for getting the (pathless) executable name out 
+ *  of an EVENTLOGRECORD */
+const WCHAR *
+get_event_exename(EVENTLOGRECORD *pevlr);
+
+/* helper method for getting the PID out of an EVENTLOGRECORD */
+UINT
+get_event_pid(EVENTLOGRECORD *pevlr);
+
+/* helper method for getting the Threat ID out of an EVENTLOGRECORD.
+ * If the event type does not have a Threat ID parameter (e.g., for 
+ * information events), then this will return NULL. */
+const WCHAR *
+get_event_threatid(EVENTLOGRECORD *pevlr);
+
+/* helper method for getting the formatted text from an EVENTLOGRECORD
+ * structure. */
+DWORD
+get_formatted_message(EVENTLOGRECORD *pevlr, WCHAR *buf, DWORD maxchars);
+
+/***************************************
+ *  system information / installation  *
+ ***************************************/
+
+/* supported platform identifiers */
+#define PLATFORM_WIN_2000      100
+#define PLATFORM_WIN_XP        110
+#define PLATFORM_WIN_2003      120
+#define PLATFORM_WIN_NT_4      130
+#define PLATFORM_VISTA         140
+
+DWORD
+get_platform(DWORD *platform);
+
+
+const WCHAR *
+get_installation_path();
+
+const WCHAR *
+get_product_name();
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/libutil/parser.c b/libutil/parser.c
new file mode 100644
index 000000000..e31fb74b3
--- /dev/null
+++ b/libutil/parser.c
@@ -0,0 +1,452 @@
+/* **********************************************************
+ * Copyright (c) 2005-2006 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+
+
+#include "share.h"
+#include "parser.h"
+#include <stdio.h>
+
+
+#ifndef UNIT_TEST
+
+char *
+next_token_sep(char *start, UINT *len, char sep)
+{
+    char *curtok = start;
+    char seps[16];
+    _snprintf(seps, 16, "%c\r\n", sep);
+
+    DO_ASSERT(start != NULL);
+    DO_ASSERT(len != NULL);
+
+    *len = 0;
+
+    if (start == NULL)
+        return NULL;
+
+    /* special case: '=' followed by a newline */
+    if (start[0] == sep && (start[1] == '\r' || start[1] == '\n')) {
+        curtok += 1;
+        return curtok;
+    }
+        
+    /* determine the next token */
+    while (curtok[0] != '\0' && *len == 0) {
+        *len = strcspn(curtok, seps);
+        if (*len == 0)
+            curtok = curtok + strspn(curtok, seps);
+    }
+
+    return curtok;
+}
+
+char *
+next_token(char *start, UINT *len)
+{
+    return next_token_sep(start, len, '=');
+}
+
+/* assumes that start points directly after the start_delimeter
+ * returns the size of the message block, or -1 on error */
+char *
+get_message_block_size(char *start, WCHAR *end_delimiter_w, UINT *size)
+{
+    char *endptr;
+    char end_delimiter[MAX_PATH];
+
+    DO_ASSERT(start != NULL);
+    DO_ASSERT(end_delimiter_w != NULL);
+    DO_ASSERT(size != NULL);
+
+    _snprintf(end_delimiter, MAX_PATH, "%S", end_delimiter_w);
+    endptr = strstr(start, end_delimiter);
+
+    if (endptr == NULL) {
+        DO_DEBUG(DL_WARN, 
+                 printf("No %s end delimiter!\n", end_delimiter);
+                 );
+        *size = 0xffffffff;
+        return NULL;
+    }
+    else {
+        DO_DEBUG(DL_VERB, 
+                 printf("Block size %d for %s:\n", 
+                        endptr - start, end_delimiter);
+                 );
+        *size = endptr - start;
+        return endptr + strlen(end_delimiter);
+    }
+}
+
+
+
+/* takes an optional separator, by default '=' */
+char *
+parse_line_sep(char *start, char sep, BOOL *done,
+               WCHAR *param, WCHAR *value, UINT maxchars)
+{
+    char *curtok;
+    UINT toklen = 0;
+    const WCHAR *prefix;
+
+    DO_ASSERT(start != NULL);
+    DO_ASSERT(done != NULL);
+    DO_ASSERT(param != NULL);
+    DO_ASSERT(maxchars > 0);
+
+    *done = FALSE;
+    param[0] = L'\0';
+    value[0] = L'\0';
+
+    curtok = start;
+
+    curtok = next_token_sep(start, &toklen, sep);
+
+    DO_DEBUG(DL_FINEST, 
+             printf("curtok (tl=%d):\n%s",
+                    toklen, curtok);
+             );
+
+    if (toklen == 0 && curtok[0] == '\0') {
+        *done = TRUE;
+        goto parse_line_out;
+    }
+
+    _snwprintf(param, MIN(toklen, maxchars-1), L"%S", curtok);
+    param[MIN(toklen, maxchars-1)] = L'\0';
+
+    DO_DEBUG(DL_FINEST, 
+             printf("curtok: %s, nc=%d (%d or %d?)\n", 
+                    curtok, curtok[toklen + 1], '\r', '\n');
+             );
+
+    /* handle the empty option properly */
+    if (curtok[toklen] == '\r' || curtok[toklen] == '\n') {
+        value[0] = L'\0';
+    }
+    else {
+        curtok = next_token_sep(curtok + toklen, &toklen, sep);
+
+        if (toklen == 0 && curtok[0] == '\0') {
+            *done = TRUE;
+            goto parse_line_out;
+        }
+
+        DO_DEBUG(DL_FINEST,
+                 printf("vt(%d): %s\n", toklen, curtok);
+                 );
+
+        /* all values that start with \\ have DR_HOME prepended */
+        if ('\\' == curtok[0])
+            prefix = get_dynamorio_home();
+        else
+            prefix = L"";
+        
+        _snwprintf(value, MIN(toklen + wcslen(prefix), maxchars - 1),
+                   L"%s%S", prefix, curtok);
+        value[MIN(toklen + wcslen(prefix), maxchars - 1)] = L'\0';
+    }
+
+ parse_line_out:
+
+    DO_DEBUG(DL_FINEST, 
+             printf("parsed line: %S(%d)=%S\n", 
+                    param, toklen, value);
+             );
+
+    return curtok + toklen;
+}
+
+
+/* first argument is where to start searching from.
+ * returns a value which is the right place to start searching
+ *  from for the next token. 
+ * a line is either a single token, or an NVP (separated by =). */
+char *
+parse_line(char *start, BOOL *done,
+           WCHAR *param, WCHAR *value, UINT maxchars)
+{
+    return parse_line_sep(start, '=', done, param, value, maxchars);
+}
+
+
+void
+msg_append(char *msg_buffer, UINT maxchars, WCHAR *data, UINT *accumlen)
+{
+    DO_ASSERT(accumlen != NULL);
+
+    if (data == NULL)
+        return;
+
+    if (msg_buffer != NULL && 
+        strlen(msg_buffer) + wcslen(data) < maxchars) {
+        UINT oldsz = strlen(msg_buffer);
+        DO_DEBUG(DL_FINEST,
+                 printf("msg_buffer: %s, os=%d, data=%S\n", 
+                        msg_buffer, oldsz, data);
+                 );
+        _snprintf(msg_buffer + oldsz, maxchars - oldsz, "%S", data);
+        msg_buffer[oldsz + wcslen(data)] = '\0';
+        DO_DEBUG(DL_FINEST,
+                 printf("msg_buffer: %s\n", msg_buffer);
+                 );
+        DO_ASSERT(strlen(msg_buffer) == oldsz + wcslen(data));
+    }
+
+    *accumlen += wcslen(data);
+    return;
+}
+
+/* ok this is annoying.
+ * 
+ * apparently there's a bug in MS's CRT that causes \r\n to get
+ *  converted to \r\r\n.
+ * 
+ * see the unit tests below for an example of this bug.
+ *
+ * i guess we're ok with \r\r\n since notepad handles it fine. that's
+ *  preferable to just \n which notepad can't handle. 
+ */
+
+void
+msg_append_nvp(char *msg_buffer, UINT maxchars, UINT *accumlen,
+               WCHAR *name, WCHAR *value)
+{
+    /* exclude installation-specific parameters */
+    if (0 == wcscmp(name, L_DYNAMORIO_VAR_HOME) ||
+        0 == wcscmp(name, L_DYNAMORIO_VAR_LOGDIR))
+        return;
+
+    msg_append(msg_buffer, maxchars, name, accumlen);
+    msg_append(msg_buffer, maxchars, L_EQUALS, accumlen);
+
+    /* relativize any paths to DRHOME */
+    if (0 == wcsncmp(value, get_dynamorio_home(), 
+                     wcslen(get_dynamorio_home())))
+        msg_append(msg_buffer, maxchars, 
+                   value + wcslen(get_dynamorio_home()), accumlen);
+    else
+        msg_append(msg_buffer, maxchars, value, accumlen);
+
+    msg_append(msg_buffer, maxchars, L_NEWLINE, accumlen);
+}
+
+
+
+#else // ifdef UNIT_TEST
+
+
+
+int
+main()
+{
+    char *testline = "GLOBAL_PROTECT=1\r\nBEGIN_BLOCK\r\nAPP_NAME=inetinfo.exe\r\nDYNAMORIO_OPTIONS=\r\nFOO=\\bar.dll\r\n";
+    WCHAR *sample = L"sample.mfp";
+    UINT len;
+    DWORD res;
+    char *policy;
+
+    set_debuglevel(DL_INFO);
+    set_abortlevel(DL_WARN);
+
+
+    /* load the sample policy file for testing */
+    res = read_file_contents(sample, NULL, 0, &len);
+    DO_ASSERT(res == ERROR_MORE_DATA);
+    DO_ASSERT(len > 1000);
+    
+    policy = (char *) malloc(len);
+    res = read_file_contents(sample, policy, len, NULL);
+    DO_ASSERT(res == ERROR_SUCCESS);    
+
+
+    /* next_token tests */
+    {
+        char *tok, *line, *ptr;
+        UINT len;
+
+        line = testline;
+        tok = next_token(line, &len);
+    
+        DO_ASSERT(line == tok);
+        DO_ASSERT(len == strlen("GLOBAL_PROTECT"));
+
+        ptr = line + len;
+        tok = next_token(ptr, &len);
+        DO_ASSERT(ptr + 1 == tok);
+        DO_ASSERT(len == 1);
+
+        ptr = tok + len;
+        tok = next_token(ptr, &len);
+        DO_ASSERT(0 == strncmp(tok, "BEGIN_BLOCK", strlen("BEGIN_BLOCK")));
+        DO_ASSERT(len == strlen("BEGIN_BLOCK"));
+
+        ptr = tok + len;
+        tok = next_token(ptr, &len);
+        DO_ASSERT(0 == strncmp(tok, "APP_NAME", strlen("APP_NAME")));
+        DO_ASSERT(len == strlen("APP_NAME"));
+
+        ptr = tok + len;
+        tok = next_token(ptr, &len);
+        DO_ASSERT(0 == strncmp(tok, "inetinfo.exe", strlen("inetinfo.exe")));
+        DO_ASSERT(len == strlen("inetinfo.exe"));
+
+        ptr = tok + len;
+        tok = next_token(ptr, &len);
+        DO_ASSERT(0 == strncmp(tok, "DYNAMORIO_OPTIONS", strlen("DYNAMORIO_OPTIONS")));
+        DO_ASSERT(len == strlen("DYNAMORIO_OPTIONS"));
+
+        ptr = tok + len;
+        tok = next_token(ptr, &len);
+        DO_ASSERT(len == 0);
+
+        ptr = tok + len;
+        tok = next_token(ptr, &len);
+        DO_ASSERT(0 == strncmp(tok, "FOO", strlen("FOO")));
+        DO_ASSERT(len == strlen("FOO"));
+
+        ptr = tok + len;
+        tok = next_token(ptr, &len);
+        DO_ASSERT(len == strlen("\\bar.dll"));
+
+        /* FIXME: long string / overrun tets.. */
+    }
+
+    /* get_message_block_size tests */
+    {
+        char *line, *ptr;
+        UINT len;
+
+        line = "a b\r\nfoo bar=blah\r\nEND_TEST_BLOCK\r\n";
+        ptr = get_message_block_size(line, L"END_TEST_BLOCK", &len);
+        DO_ASSERT(len == 19);
+        DO_ASSERT(ptr == line + strlen(line) - 2);
+    }
+
+    /* parse_line tests */
+    {
+        WCHAR param[MAX_PATH], value[MAX_PATH];
+        BOOL done;
+        char *ptr, *line = testline;
+
+        ptr =
+            parse_line(line, &done, param, value, MAX_PATH);
+        DO_ASSERT(!done);
+        DO_ASSERT_WSTR_EQ(param, L"GLOBAL_PROTECT");
+        DO_ASSERT_WSTR_EQ(value, L"1");
+
+        ptr =
+            parse_line(ptr, &done, param, value, MAX_PATH);
+        DO_ASSERT(!done);
+        DO_ASSERT_WSTR_EQ(param, L"BEGIN_BLOCK");
+        DO_ASSERT_WSTR_EQ(value, L"");
+
+        ptr =
+            parse_line(ptr, &done, param, value, MAX_PATH);
+        DO_ASSERT(!done);
+        DO_ASSERT_WSTR_EQ(param, L"APP_NAME");
+        DO_ASSERT_WSTR_EQ(value, L"inetinfo.exe");
+
+        ptr =
+            parse_line(ptr, &done, param, value, MAX_PATH);
+        DO_ASSERT(!done);
+        DO_ASSERT_WSTR_EQ(param, L"DYNAMORIO_OPTIONS");
+        DO_ASSERT_WSTR_EQ(value, L"");
+
+        ptr =
+            parse_line(ptr, &done, param, value, MAX_PATH);
+        DO_ASSERT(!done);
+        DO_ASSERT_WSTR_EQ(param, L"FOO");
+        DO_ASSERT(NULL != wcsstr(value, L"bar"));
+        DO_ASSERT(NULL != wcsstr(value, get_dynamorio_home()));
+
+    }
+
+
+    /* msg_append tests */
+    {
+        char buf[MAX_PATH];
+        UINT loc;
+        buf[0] = '\0';
+
+        loc = 0;
+        msg_append(buf, 0, L"GLOBAL", &loc);
+        DO_ASSERT(0 == strlen(buf));
+        DO_ASSERT(6 == loc);
+
+        loc = 0;
+        msg_append(buf, MAX_PATH, L"GLOBAL", &loc);
+        DO_ASSERT(0 == strcmp(buf, "GLOBAL"));
+        DO_ASSERT(6 == loc);
+
+        loc = 6;
+        msg_append(buf, 0, L"_PROTECT", &loc);
+        DO_ASSERT(0 == strcmp(buf, "GLOBAL"));
+        DO_ASSERT(14 == loc);
+
+        loc = 6;
+        msg_append(buf, 6, L"_PROTECT", &loc);
+        DO_ASSERT(0 == strcmp(buf, "GLOBAL"));
+        DO_ASSERT(14 == loc);
+
+        loc = 6;
+        msg_append(buf, MAX_PATH, L"_PROTECT", &loc);
+        DO_ASSERT(0 == strcmp(buf, "GLOBAL_PROTECT"));
+        DO_ASSERT(14 == loc);
+    }
+
+    /* MSVCRT \r\r\n bug */
+    {
+        WCHAR *outfn = L"testcr.txt";
+        char *str="line\r\n";
+        char buf[MAX_PATH];
+
+        res = write_file_contents(outfn, str, TRUE);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        
+        res = read_file_contents(outfn, buf, MAX_PATH, NULL);
+        DO_ASSERT(res == ERROR_SUCCESS);
+
+        DO_ASSERT(0 == strcmp(buf, str));
+
+        /* this passes: but check out the file in emacs! 
+         * apparently the bug works both ways. */
+    }
+
+    printf("All Test Passed\n");
+
+    return 0;
+
+}
+
+#endif
diff --git a/libutil/parser.h b/libutil/parser.h
new file mode 100644
index 000000000..17f38c423
--- /dev/null
+++ b/libutil/parser.h
@@ -0,0 +1,62 @@
+/* **********************************************************
+ * Copyright (c) 2005-2006 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+
+
+#ifndef _DETERMINA_PARSER_H_
+#define _DETERMINA_PARSER_H_
+
+
+#define L_NEWLINE L"\r\n"
+#define L_EQUALS L"="
+
+char *
+next_token(char *start, UINT *len);
+
+char *
+parse_line(char *start, BOOL *done, WCHAR *param, WCHAR *value, UINT maxchars);
+
+char *
+parse_line_sep(char *start, char sep, BOOL *done,
+               WCHAR *param, WCHAR *value, UINT maxchars);
+
+char *
+get_message_block_size(char *start, WCHAR *end_delimiter_w, UINT *size);
+
+void
+msg_append(char *msg_buffer, UINT maxchars, WCHAR *data, UINT *accumlen);
+
+void
+msg_append_nvp(char *msg_buffer, UINT maxchars, UINT *accumlen,
+               WCHAR *name, WCHAR *value);
+
+#endif
diff --git a/libutil/policy.c b/libutil/policy.c
new file mode 100644
index 000000000..6edb1fdea
--- /dev/null
+++ b/libutil/policy.c
@@ -0,0 +1,777 @@
+/* **********************************************************
+ * Copyright (c) 2004-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+
+
+#include "share.h"
+#include "policy.h"
+#include "parser.h"
+#include "config.h"
+#include "processes.h"
+
+
+#ifndef UNIT_TEST
+
+WCHAR *msg_id_keys[] = {
+#define MSG_FIELD(x) L#x,
+    POLICY_DEF_KEYS
+#undef MSG_FIELD
+    L"<invalid message field>"
+};
+
+msg_id
+get_msgkey_id(WCHAR *msgk) 
+{
+    msg_id id = -1;
+
+    DO_DEBUG(DL_FINEST, 
+             printf("trying to ID %S\n", msgk);
+             );
+    
+    while (++id != MSGKEY_BAD_FIELD)
+        if(0 == wcscmp(msg_id_keys[id], msgk))
+            break;
+    
+    return id;
+}
+
+
+char *
+parse_policy_line(char *start, BOOL *done, msg_id *mfield,
+                  WCHAR *param, WCHAR *value, UINT maxchars)
+{
+    char *next = parse_line(start, done, param, value, maxchars);
+
+    if (mfield != NULL)
+        *mfield = get_msgkey_id(param);
+
+    return next;
+}
+
+
+#define MAX_SUPPORTED_ENGINES 16
+
+void add_to_engines(int *engines, const WCHAR *neweng)
+{
+    int i;
+    for (i = 0; engines[i] != 0; i++);
+    engines[i] = _wtoi(neweng);
+}
+
+DWORD
+parse_policy(char *policy_definition,
+             /* OUT */ ConfigGroup **config,
+             /* OUT */ ConfigGroup **options,
+             BOOL validating)
+{
+    WCHAR namebuf[MAX_PARAM_LEN], valuebuf[MAX_PARAM_LEN];
+    WCHAR app_name[MAX_PATH];
+    msg_id mfield;
+    DWORD res;
+    char *polstr = policy_definition;
+    ConfigGroup *policy, *app;
+    char *modes_file;
+    UINT modes_file_size;
+    BOOL parsing_done;
+    int engines[MAX_SUPPORTED_ENGINES] = { 0 };
+
+    DO_ASSERT(config != NULL);
+    DO_ASSERT(options != NULL);
+
+    *config = NULL;
+    *options = NULL;
+
+    DO_DEBUG(DL_VERB,
+             printf("policy string received: %s\n ", polstr);
+             );
+
+    res = read_config_group(config, L_PRODUCT_NAME, FALSE);
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    *options = new_config_group(L"options");
+
+    /* alias the pointer for clarity */
+    policy = *config;
+
+    policy->should_clear = TRUE;
+
+    /* starting parsing the polstr */
+    polstr = parse_policy_line(polstr, &parsing_done, &mfield,
+                               namebuf, valuebuf, MAX_PARAM_LEN);
+
+    /* first, load the options at the beginning
+     *  (GLOBAL_PROTECT, VERSION, etc) */
+    while (!parsing_done && mfield != MSGKEY_BEGIN_BLOCK) {
+        if (0 == wcscmp(namebuf, L"ENGINE")) {
+            add_to_engines(engines, valuebuf);
+        }
+        else {
+            set_config_group_parameter(*options, namebuf, valuebuf);
+        }
+        polstr = parse_policy_line(polstr, &parsing_done, &mfield,
+                                   namebuf, valuebuf, MAX_PARAM_LEN);
+    }
+
+    /* now do all of the blocks */
+    while(!parsing_done) {
+
+        if (mfield != MSGKEY_BEGIN_BLOCK) {
+            DO_DEBUG(DL_ERROR, 
+                     printf("BEGIN_BLOCK not found, instead %S\n", namebuf);
+                     );
+            return ERROR_PARSE_ERROR;
+        }
+
+        polstr = parse_policy_line(polstr, &parsing_done, &mfield,
+                                   namebuf, valuebuf, MAX_PARAM_LEN);
+
+        if (mfield == MSGKEY_GLOBAL) {
+            app = policy;
+        } else if (mfield == MSGKEY_APP_NAME) {
+            wcsncpy(app_name, valuebuf, MAX_PATH);
+            app_name[MAX_PATH] = L'\0';
+            app = new_config_group(app_name);
+            add_config_group(policy, app);
+        }
+        else {
+            DO_DEBUG(DL_ERROR,
+                     printf("bad appname token: %S\n", namebuf);
+                     );
+            return ERROR_PARSE_ERROR;
+        }
+
+        DO_DEBUG(DL_FINEST, 
+                 printf("'%S' is the app being parsed\n", app->name);
+                 );
+
+        modes_file = NULL;
+        modes_file_size = 0xffffffff;
+
+        for(;;) {
+
+            polstr = parse_policy_line(polstr, &parsing_done, &mfield,
+                                       namebuf, valuebuf, MAX_PARAM_LEN);
+
+            if (parsing_done || mfield == MSGKEY_END_BLOCK)
+                break;
+
+            if (mfield == MSGKEY_BEGIN_MP_MODES) {
+                polstr = next_token(polstr, &modes_file_size);
+                modes_file = polstr;
+                DO_DEBUG(DL_VERB,
+                         printf("mf=%s\n", modes_file);
+                         );
+                polstr = 
+                    get_message_block_size(polstr,
+                                           msg_id_keys[MSGKEY_END_MP_MODES],
+                                           &modes_file_size);
+                if (modes_file_size == 0xffffffff)
+                    return ERROR_PARSE_ERROR;
+                
+                continue;
+            }
+
+            DO_DEBUG(DL_VERB, 
+                     printf("option setting: %S, %S=%S\n", 
+                            app_name, namebuf, valuebuf);
+                     );
+
+            set_config_group_parameter(app, namebuf, valuebuf);
+            
+        } //for(;;)
+        
+        /* FIXME:
+         * strictly speaking, this isn't parsing, and so it's
+         *  kind of messy to be doing this here. (eg: notice that
+         *  validate_policy at this point will write all of the
+         *  modes files!) but it's ok for now..sort of... */
+        if (!validating && modes_file != NULL) {
+            char backup;
+            WCHAR *modes_path;
+            BOOL changed;
+            WCHAR modes_filename[MAX_PATH];
+            int j;
+
+            modes_path = 
+                get_config_group_parameter(app, 
+                                           L_DYNAMORIO_VAR_HOT_PATCH_MODES);
+            if (modes_path == NULL) {
+                DO_DEBUG(DL_ERROR,
+                         printf("missing modes file name!\n");
+                         );
+                return ERROR_PARSE_ERROR;
+            }
+
+            backup = modes_file[modes_file_size];
+            modes_file[modes_file_size] = L'\0';
+
+            /* need to write modes file for all supported engines */
+            for (j = 0; engines[j] != 0; j++) {
+                _snwprintf(modes_filename, MAX_PATH, L"%s\\%d\\%S",
+                           modes_path, engines[j], HOTP_MODES_FILENAME);
+                res = write_file_contents_if_different(modes_filename, 
+                                                       modes_file, &changed);
+            }
+
+            modes_file[modes_file_size] = backup;
+
+            if (res != ERROR_SUCCESS)
+                return res;
+
+            /* FIXME: we have the 'changed' info, so we may want to nudge
+             *  selectively based on that...but it gets kind of dangerous
+             *  so just nudge_all for now. */
+        }
+        else {
+            /* FIXME: should we delete old modes files? */
+        }
+
+        /* and prep the next line */
+        polstr = parse_policy_line(polstr, &parsing_done, &mfield,
+                                   namebuf, valuebuf, MAX_PARAM_LEN);
+
+    } // while(!parsing_done)
+
+    return ERROR_SUCCESS;
+}
+
+/* the amount of time to wait after startup before doing
+ * a nudge reset on all processes. */
+#define DEFAULT_RESET_INTERVAL_MS 2*60*1000
+
+/* timeout for nudge reset operation */
+#define DEFAULT_RESET_TIMEOUT_MS 30*1000
+
+/* to mitigate the possibility of bringing the system to a halt, wait 
+ * 2 seconds between process resets. */
+#define DEFAULT_RESET_DELAY_MS 2*1000
+
+DWORD
+policy_import(char *policy_definition, BOOL synchronize_system, 
+              BOOL *inject_flag, DWORD *warning)
+{
+    ConfigGroup *policy;
+    ConfigGroup *options;
+    DWORD res;
+    WCHAR *global_protect;
+
+    DO_ASSERT(policy_definition != NULL);
+
+    if (warning != NULL)
+        *warning = ERROR_SUCCESS;
+
+    res = parse_policy(policy_definition, &policy, &options, FALSE);
+
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    res = write_config_group(policy);
+
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    /* note global protect is optional */
+    global_protect = 
+        get_config_group_parameter(options, 
+                                   msg_id_keys[MSGKEY_GLOBAL_PROTECT]);
+
+
+    if (NULL != global_protect) {
+
+        if (_wtoi(global_protect)) {
+            if (inject_flag != NULL)
+                *inject_flag = TRUE;
+            else 
+                set_autoinjection();
+        }
+        else {
+            if (inject_flag != NULL) {
+                *inject_flag = FALSE;
+            }
+            else {
+                if (!using_system32_for_preinject(NULL)) {
+                    /* NOTE: on NT the appinit value is cached per-boot;
+                     *  so instead of clearing it, we just leave our
+                     *  preinject in permanently, and only remove it at
+                     *  uninstall.  this is ok for global protect = OFF
+                     *  since that message will come with an empty list of
+                     *  apps, so nothing will be run under dr. 
+                     * but for slightly saner platforms we prefer to take
+                     *  the safe route of clearing out the appinit value. */
+                    unset_autoinjection();
+                }
+            }
+            res = detach_all(DETACH_RECOMMENDED_TIMEOUT);
+            if (res != ERROR_SUCCESS && warning != NULL)
+                *warning = res;
+        }
+    }
+
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    if (synchronize_system) {
+        /* FIXME: This is a ugly hack to fix case 9436; reasonable because 4.2
+         * release is close and this is the minimal change.  The better fix
+         * would be to make the core flag thin_client via drmarker and use it in
+         * system_info_cb.  Though that is better, it still isn't great because
+         * node manager will still be issuing detach to all that aren't in the
+         * config set.  The best would be either to make EV send an incremental
+         * policy update rather than a full one or have node manager identify
+         * which ones to nudge by maintaining the prior policy update. */
+        wchar_t *global_options;
+        global_options = get_config_group_parameter(policy,
+                                                    L_DYNAMORIO_VAR_OPTIONS);
+        if (global_options == NULL ||
+            wcsstr(global_options, L"-thin_client") == NULL) {
+            res = detach_all_not_in_config_group(policy, 
+                                                 DETACH_RECOMMENDED_TIMEOUT);
+        }
+        if (res != ERROR_SUCCESS) {
+            DO_DEBUG(DL_WARN,
+                     printf("error %d doing consistency detach!\n", res);
+                     );
+            
+            if (warning && *warning != ERROR_SUCCESS)
+                *warning = res;
+        }
+
+        /* FIXME: very inefficient to issue a process control nudge each time
+         * there is an generic update; ev should split update messages into
+         * regular, process control, hotpatch mode, hotpatch defs, etc and use
+         * them only as needed - will prevent needless performance bottlenecks,
+         * esp. with hotpatch nudges.  02-Nov-06: Bharath. */
+        /* FIXME: reset_threadproc() just seems to do a generic nudge with
+         * sleep - parameterize the sleep and check its uses to see if it can
+         * be made generic.  02-Nov-06: Bharath. */
+        /* FIXME: Also, there are two layers of delaying/sleeping one at
+         * reset_threadproc() level and one at the process_walk level for 
+         * nudges.  Why wasn't one enough?  02-Nov-06e: Bharath */
+        /* FIXME: return value from generic_nudge_all and
+         * hotp_notify_all_modes_update (and hotp_notify_all_defs_update else
+         * where) haven't been checked.  Don't even know what that means for a
+         * nudge all.  Is it supposed to indicate that the whole thing failed
+         * or just one nudge?  If so what is the intended action.  02-Nov-06:
+         * Bharath. */
+        /* FIXME: make both nodemanager and core use the generic nudge
+         * interface to do hotpatch and detach nudges and do away with the
+         * nudge-specific code.  02-Nov-06: Bharath. */
+        generic_nudge_all(NUDGE_GENERIC(process_control), NULL,
+                                DEFAULT_RESET_TIMEOUT_MS, 0);
+
+
+        /* FIXME: for now we do this at every policy update, but 
+         *  maybe should be more efficient? */
+        res = hotp_notify_all_modes_update(DETACH_RECOMMENDED_TIMEOUT);
+
+        if (res != ERROR_SUCCESS) {
+
+            DO_DEBUG(DL_WARN, 
+                     printf("Hotpatch nudge failed! %d\n", res);
+                     );
+
+            if (warning && *warning != ERROR_SUCCESS)
+                *warning = res;
+        }
+
+    }
+
+    free_config_group(options);
+    free_config_group(policy);
+
+    DO_DEBUG(DL_INFO, 
+             printf("Processed policy update.\n");
+             );
+
+    return ERROR_SUCCESS;
+}
+
+DWORD
+clear_policy()
+{
+    ConfigGroup *config;
+    DWORD res;
+
+    /* wipe out the registry by writing an empty policy. */
+    res = read_config_group(&config, L_PRODUCT_NAME, FALSE);
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    config->should_clear = TRUE;
+    res = write_config_group(config);
+    free_config_group(config);
+
+    return res;
+}
+
+
+/* returns ERROR_SUCCESS unless the policy_buffer is invalid. */
+DWORD
+validate_policy(char *policy_definition)
+{
+    ConfigGroup *policy;
+    ConfigGroup *options;
+    DWORD res;
+
+    DO_ASSERT(policy_definition != NULL);
+
+    res = parse_policy(policy_definition, &policy, &options, TRUE);
+
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    free_config_group(policy);
+    free_config_group(options);
+
+    return ERROR_SUCCESS;
+}
+
+
+
+void
+append_policy_block(char *policy_buffer, UINT maxchars, UINT *accumlen,
+                    ConfigGroup *cfg)
+{
+    NameValuePairNode *nvpn = NULL;
+
+    msg_append(policy_buffer, maxchars, 
+               msg_id_keys[MSGKEY_BEGIN_BLOCK], accumlen);
+    msg_append(policy_buffer, maxchars, L_NEWLINE, accumlen);
+
+    if (0 == wcscmp(cfg->name, L_PRODUCT_NAME)) {
+        msg_append(policy_buffer, maxchars, 
+                   msg_id_keys[MSGKEY_GLOBAL], accumlen);
+        msg_append(policy_buffer, maxchars, L_NEWLINE, accumlen);
+    }
+    else {
+        msg_append_nvp(policy_buffer, maxchars, accumlen,
+                          msg_id_keys[MSGKEY_APP_NAME],
+                          cfg->name);
+    }
+
+    for(nvpn = cfg->params; NULL != nvpn; nvpn = nvpn->next) {
+        msg_append_nvp(policy_buffer, maxchars, accumlen,
+                       nvpn->name, nvpn->value);
+    }
+
+    msg_append(policy_buffer, maxchars, 
+               msg_id_keys[MSGKEY_END_BLOCK], accumlen);    
+    msg_append(policy_buffer, maxchars, L_NEWLINE, accumlen);
+}
+
+/* FIXME: does not export modes! */
+DWORD
+policy_export(char *policy_buffer, UINT maxchars, UINT *needed)
+{
+    ConfigGroup *config, *chld;
+    DWORD res;
+    UINT accumlen = 0;
+
+    res = read_config_group(&config, L_PRODUCT_NAME, TRUE);
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    /* NOTE: we don't specify global protect when exporting */
+
+    /* FIXME: hardcoded ID and version */
+    msg_append_nvp(policy_buffer, maxchars, &accumlen,
+                   L"POLICY_VERSION", L"30000");
+
+    append_policy_block(policy_buffer, maxchars, &accumlen, config);
+
+    for (chld = config->children; chld != NULL; chld = chld->next) {
+        append_policy_block(policy_buffer, maxchars, &accumlen, chld);
+    }
+
+    /* for the null terminator */
+    accumlen++;
+
+    if (needed != NULL)
+        *needed = accumlen;
+
+    if (policy_buffer == NULL || maxchars < accumlen)
+        return ERROR_MORE_DATA;
+    else
+        return ERROR_SUCCESS;
+}
+
+DWORD
+load_policy(WCHAR *filename, BOOL synchronize_system, DWORD *warning)
+{
+    DWORD res;
+    UINT len;
+    char *policy;
+
+    DO_ASSERT(filename != NULL);
+
+    res = read_file_contents(filename, NULL, 0, &len);
+    DO_ASSERT(res == ERROR_MORE_DATA);
+    
+    policy = (char *) malloc(len);
+    res = read_file_contents(filename, policy, len, NULL);
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    res = policy_import(policy, synchronize_system, NULL, warning);
+
+    free(policy);
+
+    return res;
+}
+
+DWORD
+save_policy(WCHAR *filename)
+{
+    DWORD res;
+    UINT len;
+    char *policy;
+
+    res = policy_export(NULL, 0, &len);
+    if (res != ERROR_MORE_DATA && res != ERROR_SUCCESS)
+        return res;
+    
+    policy = (char *) malloc(len);
+    policy[0] = '\0';
+
+    res = policy_export(policy, len, NULL);
+    
+    if (res == ERROR_SUCCESS) {
+        res = write_file_contents(filename, policy, TRUE);
+    }
+
+    free(policy);
+
+    return res;
+}
+
+
+
+#else // ifdef UNIT_TEST
+
+
+
+void
+test_sample_mfp(ConfigGroup *globals, ConfigGroup *config)
+{
+    ConfigGroup *chld;
+
+    if (globals != NULL) {
+        DO_ASSERT_WSTR_EQ(L"77777",
+                          get_config_group_parameter(globals, 
+                                                     L"POLICY_VERSION"));
+    }
+
+    DO_ASSERT_WSTR_EQ(L"1", 
+                      get_config_group_parameter(config, 
+                                                 L"DYNAMORIO_RUNUNDER"));
+    DO_ASSERT_WSTR_EQ(L"", 
+                      get_config_group_parameter(config, 
+                                                 L"DYNAMORIO_OPTIONS"));
+    DO_ASSERT(NULL != 
+              wcsstr(get_config_group_parameter(config, 
+                                                L"DYNAMORIO_AUTOINJECT"),
+                     L"\\lib\\77777\\dynamorio.dll"));
+    
+    chld = get_child(L"svchost.exe-bitsgroup", config);
+    DO_ASSERT(chld != NULL);
+
+    DO_ASSERT_WSTR_EQ(L"17",
+                      get_config_group_parameter(chld, L"DYNAMORIO_RUNUNDER"));
+    DO_ASSERT_WSTR_EQ(L"-report_max 0 -kill_thread -kill_thread_max 1000", 
+                      get_config_group_parameter(chld, L"DYNAMORIO_OPTIONS"));
+    DO_ASSERT(NULL !=
+              wcsstr(get_config_group_parameter(chld, 
+                                                L"DYNAMORIO_AUTOINJECT"),
+                     L"\\lib\\77777\\dynamorio.dll"));
+}
+
+int
+main()
+{
+    char *testline = "GLOBAL_PROTECT=1\r\nBEGIN_BLOCK\r\nAPP_NAME=inetinfo.exe\r\nDYNAMORIO_OPTIONS=\r\nFOO=\\bar.dll\r\n";
+    WCHAR *sample = L"sample.mfp";
+    UINT len;
+    DWORD res;
+    char *policy;
+
+    set_debuglevel(DL_INFO);
+    set_abortlevel(DL_WARN);
+
+
+    /* load the sample policy file for testing */
+    res = read_file_contents(sample, NULL, 0, &len);
+    DO_ASSERT(res == ERROR_MORE_DATA);
+    DO_ASSERT(len > 1000);
+    
+    policy = (char *) malloc(len);
+    res = read_file_contents(sample, policy, len, NULL);
+    DO_ASSERT(res == ERROR_SUCCESS);    
+
+
+    /* parse_policy_line tests */
+    {
+        WCHAR param[MAX_PATH], value[MAX_PATH];
+        BOOL done;
+        msg_id mfield;
+        char *ptr, *line = testline;
+
+        ptr =
+            parse_policy_line(line, &done, &mfield, param, value, MAX_PATH);
+        DO_ASSERT(!done);
+        DO_ASSERT(mfield == MSGKEY_GLOBAL_PROTECT);
+        DO_ASSERT_WSTR_EQ(param, L"GLOBAL_PROTECT");
+        DO_ASSERT_WSTR_EQ(value, L"1");
+
+        ptr =
+            parse_policy_line(ptr, &done, &mfield, param, value, MAX_PATH);
+        DO_ASSERT(!done);
+        DO_ASSERT(mfield == MSGKEY_BEGIN_BLOCK);
+        DO_ASSERT_WSTR_EQ(param, L"BEGIN_BLOCK");
+        DO_ASSERT_WSTR_EQ(value, L"");
+
+        ptr =
+            parse_policy_line(ptr, &done, &mfield, param, value, MAX_PATH);
+        DO_ASSERT(!done);
+        DO_ASSERT(mfield == MSGKEY_APP_NAME);
+        DO_ASSERT_WSTR_EQ(param, L"APP_NAME");
+        DO_ASSERT_WSTR_EQ(value, L"inetinfo.exe");
+
+        ptr =
+            parse_policy_line(ptr, &done, &mfield, param, value, MAX_PATH);
+        DO_ASSERT(!done);
+        DO_ASSERT(mfield == MSGKEY_BAD_FIELD);
+        DO_ASSERT_WSTR_EQ(param, L"DYNAMORIO_OPTIONS");
+        DO_ASSERT_WSTR_EQ(value, L"");
+
+        ptr =
+            parse_policy_line(ptr, &done, &mfield, param, value, MAX_PATH);
+        DO_ASSERT(!done);
+        DO_ASSERT(mfield == MSGKEY_BAD_FIELD);
+        DO_ASSERT_WSTR_EQ(param, L"FOO");
+        DO_ASSERT(NULL != wcsstr(value, L"bar"));
+        DO_ASSERT(NULL != wcsstr(value, get_dynamorio_home()));
+
+    }
+
+    /* parse policy tests */
+    { 
+        ConfigGroup *config, *globals;
+
+        res = parse_policy(policy, &config, &globals, FALSE);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        DO_ASSERT(globals != NULL);
+        DO_ASSERT(config != NULL);
+
+        test_sample_mfp(globals, config);
+
+        free_config_group(globals);
+        free_config_group(config);
+    }
+
+
+    /* validate policy tests */
+    {
+    }
+
+    /* import policy tests */
+    { 
+        DWORD warning;
+        ConfigGroup *config;
+
+        warning = ERROR_SUCCESS;
+        res = policy_import(policy, FALSE, NULL, &warning);
+        DO_ASSERT(res == ERROR_SUCCESS);
+
+        res = read_config_group(&config, L_PRODUCT_NAME, TRUE);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        test_sample_mfp(NULL, config);
+
+        free_config_group(config);
+    }
+
+    /* export policy tests */
+    {
+        WCHAR *outfn = L"test.mfp";
+        char *outpol;
+        DWORD warning;
+
+        warning = ERROR_SUCCESS;
+        res = policy_import(policy, FALSE, NULL, &warning);
+        DO_ASSERT(res == ERROR_SUCCESS);
+
+        /* load the sample policy file for testing */
+        res = policy_export(NULL, 0, &len);
+        DO_ASSERT(res == ERROR_MORE_DATA);
+        DO_ASSERT(len > 1000);
+    
+        outpol = (char *) malloc(len);
+        outpol[0] = '\0';
+        res = policy_export(outpol, len, NULL);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        DO_ASSERT(len == 1 + strlen(outpol));
+
+        res = write_file_contents(outfn, outpol, TRUE);
+        DO_ASSERT(res == ERROR_SUCCESS);
+        //DO_ASSERT(0 == strcmp(policy, outpol));
+    }
+
+    /* load/save */
+    {
+        DWORD warning = ERROR_SUCCESS;
+        ConfigGroup *c;
+
+        /* TODO: better */
+
+        CHECKED_OPERATION(clear_policy());
+        CHECKED_OPERATION(load_policy(sample, FALSE, &warning));
+        DO_ASSERT(warning == ERROR_SUCCESS);
+
+        CHECKED_OPERATION(read_config_group(&c, L_PRODUCT_NAME, TRUE));
+        test_sample_mfp(NULL, c);
+        free_config_group(c);
+        
+        delete_file_rename_in_use(L"test2.mfp");
+        CHECKED_OPERATION(save_policy(L"test2.mfp"));
+    }
+
+    /* cleanup */
+    CHECKED_OPERATION(clear_policy());
+
+    printf("All Test Passed\n");
+
+    return 0;
+
+}
+
+#endif
+
diff --git a/libutil/policy.h b/libutil/policy.h
new file mode 100644
index 000000000..9c33e6866
--- /dev/null
+++ b/libutil/policy.h
@@ -0,0 +1,162 @@
+/* **********************************************************
+ * Copyright (c) 2003-2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+
+ policy definition format specification
+-----------------------------------------
+
+
+the syntax of the policy definition message is as follows:
+
+<policy_message> ::==
+ POLICY_VERSION=<string>
+ APPINITFLAGS=<string>
+ APPINITWHITELIST=<string>
+ APPINITBLACKLIST=<string>
+ GLOBAL_PROTECT=<boolean>
+ <application_block>*
+
+<application_block> ::==
+ BEGIN_BLOCK
+ <app_specifier>
+ <dynamorio_option_line>*
+ END_BLOCK
+
+<app_specifier> ::== [ GLOBAL | APP_NAME=<string> ]
+
+<dynamorio_option_line> ::== <dynamorio_option>=<string>
+
+<dynamorio_option> ::== 
+ [ DYNAMORIO_OPTIONS |
+   DYNAMORIO_AUTOINJECT |
+   DYNAMORIO_RUNUNDER ]
+
+<boolean> ::== [ 0 | 1 ]
+
+
+
+details:
+
+(1) the APPINITFLAGS, together with the APPINITBLACKLIST and
+    APPINITWHITELIST, controls how our bootstrap dll is added to the
+    AppInit_DLLs registry key. the value of the flags should be a sum
+    of the APPINIT_* flags as defined in share/config.h
+
+    the APPINITBLACKLIST and APPINITWHITELIST values are only used if
+    specified by the flags. we'll provide the whitelist/blacklist.
+
+
+(2) GLOBAL_PROTECT: OPTIONAL: if this is 0, then protection is 
+    disabled (and all application blocks are optional). in normal 
+    situations this will be 1. if this is not specified, then the
+    current setting is kept (see also enable_protection)
+
+
+(3) there must be a GLOBAL block, which should come first, and it must
+have DYNAMORIO_RUNUNDER=1 set as an option. for consistency it should
+set DYNAMORIO_OPTIONS to blank (DYNAMORIO_OPTIONS=).
+
+
+(4) the APP_NAME must be a valid "application id". this is usually the
+.exe filename, but can also be "qualified". we will provide a complete
+list of supported applications.
+
+
+(5) some applications will also require special values for
+DYNAMORIO_RUNUNDER; we will provide this information as well.
+
+
+(6) DYNAMORIO_AUTOINJECT must be specified for every non-global
+application block, and it should be of the format
+"\lib\NNNNN\dynamorio.dll", where NNNNN is the version number of the
+core dll being used. in general, any value that starts with a "\" will
+have the local installation path prepended when using the actual
+value (eg, it indicates a relative installation path).
+
+
+(7) DYNAMORIO_OPTIONS is a string consisting of protection
+options to our core. the variability in this will depend on the level
+of configurability desired in your UI; but the majority of the options
+will not need to change. probably the only configuration necessary
+will be whether an app should be on or off; and this should be
+controlled by including or excluding the corresponding app block.
+
+
+see sample.mfp for an example policy string:
+
+
+*/
+
+#ifndef _DETERMINA_POLICY_H_
+#define _DETERMINA_POLICY_H_
+
+
+#include "config.h"
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#define POLICY_DEF_KEYS  \
+  MSG_FIELD(APP_NAME)  \
+  MSG_FIELD(GLOBAL)  \
+  MSG_FIELD(BEGIN_BLOCK)  \
+  MSG_FIELD(END_BLOCK)  \
+  MSG_FIELD(GLOBAL_PROTECT)  \
+  MSG_FIELD(BEGIN_MP_MODES)  \
+  MSG_FIELD(END_MP_MODES)
+
+typedef enum {
+#define MSG_FIELD(x) MSGKEY_##x,
+    POLICY_DEF_KEYS
+#undef MSG_FIELD
+    MSGKEY_BAD_FIELD
+} msg_id;
+
+
+char *
+parse_policy_line(char *start, BOOL *done, msg_id *mfield,
+                  WCHAR *param, WCHAR *value, UINT maxchars);
+
+DWORD
+parse_policy(char *policy_definition,
+             /* OUT */ ConfigGroup **config,
+             /* OUT */ ConfigGroup **options,
+             BOOL validating);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/libutil/processes.c b/libutil/processes.c
new file mode 100644
index 000000000..fab5363ae
--- /dev/null
+++ b/libutil/processes.c
@@ -0,0 +1,1119 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+
+#include "share.h"
+
+#include "win32/drmarker.h" // from src tree
+#include "win32/ntdll.h" // from src tree
+#include "win32/inject_shared.h" // only for w_get_short_name
+
+#include "processes.h"
+
+#include <stddef.h> // for offsetof
+#include <limits.h> // for USHRT_MAX
+
+#ifndef UNIT_TEST
+
+
+/* the GET_NTDLL macro below violates this warning */
+#pragma warning(disable : 4127)
+
+/* A wrapper to define kernel entry point in a static function */
+/* In C use only at the end of a block prologue */
+#undef GET_NTDLL
+#define GET_NTDLL(NtFunction, type)                             \
+  typedef NTSTATUS (WINAPI *NtFunction##Type) type;             \
+  static NtFunction##Type NtFunction;                           \
+  do {                                                          \
+      if (ntdll_handle == NULL)                                 \
+          ntdll_handle = GetModuleHandle(L"ntdll.dll");         \
+      if (NtFunction == NULL && ntdll_handle != NULL) {         \
+          NtFunction = (NtFunction##Type) GetProcAddress(ntdll_handle, (LPCSTR)#NtFunction);\
+      }                                                         \
+  } while (0);
+static HANDLE ntdll_handle = NULL;
+
+
+
+int
+under_dynamorio(int ProcessID)
+{
+    return under_dynamorio_ex(ProcessID, NULL);
+}
+
+DWORD
+read_hotp_status(const HANDLE hproc, const void *table_ptr, 
+                 hotp_policy_status_table_t **hotp_status)
+{
+    UINT crc_and_size[2];
+    UINT crc, size;
+    DWORD len;
+
+    DO_ASSERT(table_ptr != NULL);
+
+    /* first, read the size and crc */
+    if (!ReadProcessMemory(hproc, table_ptr, &crc_and_size, 
+                           sizeof(crc_and_size), &len) ||
+        len != sizeof(crc_and_size)) {
+        return GetLastError();
+    }
+
+    /* see drmarker.h: crc is 1st uint, size is 2nd */
+    DO_ASSERT(0 == offsetof(hotp_policy_status_table_t, crc));
+    DO_ASSERT(sizeof(UINT) == offsetof(hotp_policy_status_table_t, size));
+    crc = crc_and_size[0];
+    size = crc_and_size[1];
+
+    /* allocate *hotp_status */
+    *hotp_status = (hotp_policy_status_table_t *) malloc(size);
+
+    /* read size bytes into *hotp_status */
+    if (!ReadProcessMemory(hproc, table_ptr, *hotp_status, size, &len) ||
+        len != size) {
+        free(*hotp_status);
+        *hotp_status = NULL;
+        return GetLastError();
+    }
+
+#if 0
+    /* FIXME: crc32 is not defined where share/ can get at it,
+     *  and we may be changing it anyway (case 5346) -- so just
+     *  don't do a check for now. */
+    /* verify crc: crc starts at size elt, see globals_shared.h */
+    if ((*hotp_status)->crc != 
+        crc32((char *)*hotp_status + sizeof(UINT), size - sizeof(UINT))) {
+        free(*hotp_status);
+        *hotp_status = NULL;
+        return ERROR_DRMARKER_ERROR;
+    }
+#endif
+
+    /* need to fixup internal pointer to our address space! */
+    (*hotp_status)->policy_status_array = (hotp_policy_status_t *)
+        ((char *) &((*hotp_status)->policy_status_array) + sizeof(void *));
+
+    DO_DEBUG(DL_VERB,
+             printf("np = %d\n", (*hotp_status)->num_policies);
+             {
+                 UINT i;
+                 for (i = 0; i < (*hotp_status)->num_policies; i++)
+                     printf(" patch %s, status=%d\n",
+                            (*hotp_status)->policy_status_array[i].policy_id,
+                            (*hotp_status)->policy_status_array[i].inject_status);
+             }
+             );
+
+    return ERROR_SUCCESS;
+}
+
+void
+free_hotp_status_table(hotp_policy_status_table_t *hotp_status)
+{
+    free(hotp_status);
+}
+
+DWORD
+get_dr_marker_helper(int ProcessID, dr_marker_t *marker, 
+                     hotp_policy_status_table_t **hotp_status, int *found)
+{
+    DWORD res = ERROR_SUCCESS;
+    HANDLE hproc;
+
+    DO_ASSERT(marker != NULL);
+    DO_ASSERT(found != NULL);
+
+    DO_DEBUG(DL_VERB,
+             printf("getting dr marker, hps=0x%x\n", hotp_status);
+             );
+
+    acquire_privileges();
+    hproc = OpenProcess(PROCESS_VM_READ, FALSE, ProcessID);
+    if (hproc != NULL) {
+        *found = read_and_verify_dr_marker(hproc, marker);
+        if (*found == DR_MARKER_FOUND && hotp_status != NULL &&
+            marker->dr_hotp_policy_status_table != NULL) {
+            res = read_hotp_status(hproc, 
+                                   marker->dr_hotp_policy_status_table,
+                                   hotp_status);
+        }
+        else if (hotp_status != NULL) {
+            /* return an error if we couldn't get the hotp status */
+            res = ERROR_DRMARKER_ERROR;
+        }
+        CloseHandle(hproc);
+    }
+    else {
+        res = GetLastError();
+        if (res != ERROR_SUCCESS)
+            res = ERROR_DRMARKER_ERROR;
+    }
+    release_privileges();
+
+    DO_DEBUG(DL_VERB,
+             printf("getting dr marker, err=%d\n", res);
+             );
+
+    return res;
+}
+
+
+#define NUM_DR_MARKER_RETRIES 2
+
+/* we retry in case synchronization issues (crc failure, in the 
+ *  middle of a detach, etc) cause this to fail. 
+ * NOTE: this is redefined in detach.c to avoid include issues! */
+DWORD
+get_dr_marker(int ProcessID, dr_marker_t *marker, 
+              hotp_policy_status_table_t **hotp_status, int *found)
+{
+    DWORD res = ERROR_SUCCESS;
+    int i;
+
+    DO_ASSERT(marker != NULL);
+    DO_ASSERT(found != NULL);
+
+    for (i = 0; i < NUM_DR_MARKER_RETRIES; i++) {
+        res = get_dr_marker_helper(ProcessID, marker, hotp_status, found);
+        if (res == ERROR_SUCCESS && *found != DR_MARKER_ERROR)
+            return ERROR_SUCCESS;
+    }
+
+    return res;
+}
+
+DWORD
+get_hotp_status(int pid, hotp_policy_status_table_t **hotp_status)
+{
+    dr_marker_t marker;
+    int found;
+    DWORD res = get_dr_marker(pid, &marker, hotp_status, &found);
+    if (res == ERROR_SUCCESS && found == DR_MARKER_ERROR)
+        return ERROR_DRMARKER_ERROR;
+    else
+        return res;
+}
+
+void
+free_dynamorio_stats(dr_statistics_t *stats)
+{
+    free(stats);
+}
+
+/* Caller must call free_dynamorio_stats on return value, if it's non-NULL */
+struct _dr_statistics_t *
+get_dynamorio_stats(int pid)
+{
+    dr_marker_t marker;
+    dr_statistics_t *stats = NULL;
+    dr_statistics_t stats_tmp;
+    int found;
+    HANDLE hproc;
+
+    acquire_privileges();
+    hproc = OpenProcess(PROCESS_VM_READ, FALSE, pid);
+    if (hproc != NULL) {
+        found = read_and_verify_dr_marker(hproc, &marker);
+        if (found == DR_MARKER_FOUND && marker.stats != NULL) {
+            uint alloc_size;
+            DWORD read;
+            if (ReadProcessMemory(hproc, marker.stats, &stats_tmp,
+                                  sizeof(stats_tmp), &read) &&
+                read == sizeof(stats_tmp) &&
+                /* if unreasonably large, probably some error.
+                 * could return error code */
+                stats_tmp.num_stats < USHRT_MAX) {
+                /* The USHRT_MAX check above avoids integer overflow */
+                alloc_size = offsetof(dr_statistics_t, stats) +
+                    sizeof(single_stat_t) * stats_tmp.num_stats;
+                stats = (dr_statistics_t *) malloc(alloc_size);
+                if (stats != NULL) {
+                    if (!ReadProcessMemory(hproc, marker.stats, stats, alloc_size,
+                                           &read) ||
+                        read != alloc_size) {
+                        free(stats);
+                        stats = NULL;
+                    } else if (stats->num_stats != stats_tmp.num_stats) {
+                        /* malicious modification, or some error.
+                         * could return error code */
+                        free(stats);
+                        stats = NULL;
+                    }
+                }
+            }
+        }
+        CloseHandle(hproc);
+    }
+    release_privileges();
+    return stats;
+}
+
+
+/* NOTE in v 1.17 this had a kernel32 method of getting the dll file version
+ * that might be useful in other situations */
+int
+under_dynamorio_ex(int ProcessID, DWORD *build_num)
+{
+    dr_marker_t marker;
+    DWORD err;
+    int res;
+    
+    err = get_dr_marker(ProcessID, &marker, NULL, &res);
+
+    if (err != ERROR_SUCCESS || res == DR_MARKER_ERROR)
+        return DLL_UNKNOWN;
+    else if (res == DR_MARKER_NOT_FOUND)
+        return DLL_NONE;
+    else if (res == DR_MARKER_FOUND) {
+        if (build_num != NULL) {
+            *build_num = marker.build_num;
+        }
+        /* NOTE - profile build can be combined with DEBUG or RELEASE so we check it
+         * first.  Perhaps we should just get rid of the notion of profile builds. */
+        if (TEST(DR_MARKER_PROFILE_BUILD, marker.flags))
+            return DLL_PROFILE;
+        if (TEST(DR_MARKER_RELEASE_BUILD, marker.flags)) 
+            return DLL_RELEASE;
+        if (TEST(DR_MARKER_DEBUG_BUILD, marker.flags))
+            return DLL_DEBUG;
+    }
+    /* should never get here */
+    return DLL_UNKNOWN;
+
+}
+
+DWORD
+check_status_and_pending_restart(ConfigGroup *config, int pid, 
+                                 BOOL *pending_restart, 
+                                 int *status, ConfigGroup **process_cfg)
+{
+    WCHAR *rununder_param = NULL;
+    int stat, rununder;
+    ConfigGroup *process_config = NULL;
+
+    DO_ASSERT(pending_restart != NULL);
+
+    /* may pass NULL config ==> pending_restart is FALSE */
+    if (NULL == config)
+        *pending_restart = FALSE;
+    else
+        process_config = get_process_config_group(config, pid);
+
+    if (NULL != process_cfg)
+        *process_cfg = process_config;
+
+    if (NULL == process_config) {
+        rununder = 0;
+    }
+    else {
+        rununder_param = get_config_group_parameter(process_config, 
+                                                    L_DYNAMORIO_VAR_RUNUNDER);
+        if (rununder_param == NULL)
+            rununder_param = 
+                get_config_group_parameter(config,
+                                           L_DYNAMORIO_VAR_RUNUNDER);
+        
+        /* bad news; just abort */
+        if (rununder_param == NULL)
+            return ERROR_OPTION_NOT_FOUND;
+        
+        rununder = _wtoi(rununder_param) & RUNUNDER_ON;
+    }
+
+    stat = under_dynamorio(pid);
+    if (status != NULL)
+        *status = stat;
+
+    /* FIXME: for now assume unknown == off  (?) */
+    if (stat == DLL_UNKNOWN)
+        return ERROR_DETACH_ERROR;
+
+    *pending_restart = 
+        (rununder && stat == DLL_NONE) ||
+        (!rununder && stat != DLL_NONE);
+
+    DO_DEBUG(DL_VERB,
+             printf("  -> ru=%d, stat=%d, pr=%d, c=0x%x, pc=0x%x, pxru=%S\n", 
+                    rununder, stat, *pending_restart, config, 
+                    process_config, rununder_param);
+             );
+
+    return ERROR_SUCCESS;
+}
+
+DWORD
+hotp_notify_modes_update(int pid, BOOL allow_upgraded_perms, DWORD timeout_ms)
+{
+    return generic_nudge(pid, allow_upgraded_perms, NUDGE_GENERIC(mode), 0,
+                         NULL, timeout_ms);
+}
+
+DWORD
+hotp_notify_defs_update(int pid, BOOL allow_upgraded_perms, DWORD timeout_ms)
+{
+    return generic_nudge(pid, allow_upgraded_perms, NUDGE_GENERIC(policy), 0,
+                         NULL, timeout_ms);
+}
+
+/*
+ * helper info for process walk callback functions
+ */
+
+typedef enum {
+    CB_INVALID = 0,
+    CB_CHECK_PENDING,
+    CB_DETACH,
+    CB_DETACH_EXE,
+    CB_DETACH_NOT_IN_POLICY,
+    CB_NUDGE_MODES,
+    CB_NUDGE_DEFS,
+    CB_NUDGE_EXE,
+    CB_NUDGE_GENERIC,
+} CB_TYPE;
+
+/* process iterator shared state */
+typedef struct process_status_info_s {
+    ConfigGroup *policy;
+    int num_running;
+    int num_detached;
+    BOOL is_pending;
+    DWORD res;
+    DWORD process_nonfatal_res;
+    DWORD timeout_ms;
+    WCHAR *exename;
+    CB_TYPE callback_type;
+    DWORD nudge_action_mask;
+    void *nudge_client_arg;
+    DWORD delay_ms;
+} process_status_info_t;
+
+
+/* the param is a process_status_info_t* */
+BOOL 
+system_info_cb(process_info_t *pi, void **param)
+{
+    int status;
+    process_status_info_t *sinfo = (process_status_info_t *)param;
+    ConfigGroup *process_config;
+    BOOL is_pending;
+    DWORD res = ERROR_SUCCESS;
+
+    /* always skip the idle process */
+    if (pi->ProcessID == 0)
+        return TRUE;
+
+    res = check_status_and_pending_restart(sinfo->policy, pi->ProcessID,
+                                           &is_pending, 
+                                           &status, &process_config);
+
+    /* for the walk methods, only record process-specific errors, 
+     *  but ignore and keep trying the rest. */
+    if (res != ERROR_SUCCESS) {
+        sinfo->process_nonfatal_res = res;
+        return TRUE;
+    }
+
+    DO_DEBUG(DL_VERB,
+             printf("  pid=%d, type=%d, name=%S, ip=%d, sta=%d:\n", 
+                    pi->ProcessID, sinfo->callback_type, pi->ProcessName,
+                    is_pending, status);
+             );
+
+    switch (sinfo->callback_type) {
+    case CB_CHECK_PENDING:
+        if (is_pending) {
+            sinfo->is_pending = TRUE;
+            return FALSE;
+        }
+        break;
+    case CB_DETACH_EXE:
+        /* fall through to detach only if the process name
+         *  matches */
+        if (sinfo->exename == NULL ||
+            0 != wcsicmp(sinfo->exename, pi->ProcessName)) {
+            break;
+        }
+    case CB_DETACH:
+        /* for detach, set is_pending = TRUE and fall through, to
+         *  force DETACH_NOT_IN_POLICY to detach if it's running. */
+        is_pending = TRUE;
+    case CB_DETACH_NOT_IN_POLICY:
+        if (status != DLL_NONE && is_pending) {
+            res = detach(pi->ProcessID, TRUE, sinfo->timeout_ms);
+            if (sinfo->res == ERROR_SUCCESS)
+                sinfo->res = res;
+        }
+        break;
+    case CB_NUDGE_EXE:
+        /* fall through to nudge only if the process name
+         *  matches; and .exe nudge is by defn a modes nudge. */
+        if (sinfo->exename == NULL ||
+            0 != wcsicmp(sinfo->exename, pi->ProcessName)) {
+            break;
+        }
+    case CB_NUDGE_DEFS:
+        /* fall through and use the same code. */
+    case CB_NUDGE_MODES:
+        /* FIXME: we used to only nudge apps that have the 
+         *  DYNAMORIO_HOTPATCH_MODES key set; but this is dangerous
+         *  if, e.g., hotpatching was on and then turned off. so
+         *  we just nudge anything under DR, at least for now. */
+        if (status != DLL_NONE) {
+            if (sinfo->callback_type == CB_NUDGE_DEFS)
+                res = hotp_notify_defs_update(pi->ProcessID, TRUE, sinfo->timeout_ms);
+            
+            if (sinfo->callback_type == CB_NUDGE_MODES)
+                res = hotp_notify_modes_update(pi->ProcessID, TRUE, sinfo->timeout_ms);
+
+            if (sinfo->res == ERROR_SUCCESS)
+                sinfo->res = res;
+        }
+        break;
+    case CB_NUDGE_GENERIC:
+        /* generic nudge available under HOT_PATCHING_INTERFACE */
+        if (status != DLL_NONE) {
+            res = generic_nudge(pi->ProcessID, TRUE, 
+                                sinfo->nudge_action_mask, 
+                                0, /* client ID arbitrary */
+                                sinfo->nudge_client_arg,
+                                sinfo->timeout_ms);
+            if (sinfo->res == ERROR_SUCCESS)
+                sinfo->res = res;
+            /* pause after nudging */
+            if (sinfo->delay_ms != 0)
+                SleepEx(sinfo->delay_ms, FALSE);
+        }
+        break;
+
+    default:
+        sinfo->res = ERROR_INVALID_PARAMETER;
+        return FALSE;
+    }
+    
+    return TRUE;
+}
+
+DWORD
+execute_sysinfo_walk(process_status_info_t *sinfo)
+{
+    DWORD res;
+
+    DO_DEBUG(DL_VERB,
+             printf("starting walk...\n");
+             );
+
+    res = process_walk(&system_info_cb, (void **)sinfo);
+
+    DO_DEBUG(DL_VERB,
+             printf("walk done, res=%d, sr=%d\n", res, sinfo->res);
+             );
+
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    if (sinfo->res != ERROR_SUCCESS)
+        return sinfo->res;
+
+    /* FIXME: report status_info.process_nonfatal_res? */
+ 
+    return ERROR_SUCCESS;
+}
+    
+/* pending_restart is required OUT parameter */
+DWORD
+is_anything_pending_restart(ConfigGroup *c, BOOL *pending_restart)
+{
+    process_status_info_t status_info;
+    DWORD res;
+
+    DO_ASSERT(pending_restart != NULL);
+
+    memset(&status_info, 0, sizeof(process_status_info_t));
+    status_info.policy = c;
+    status_info.callback_type = CB_CHECK_PENDING;
+
+
+    res = execute_sysinfo_walk(&status_info);
+
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    *pending_restart = status_info.is_pending;
+
+    return ERROR_SUCCESS;
+}
+
+/* pending_restart is required OUT parameter */
+DWORD
+detach_all_not_in_config_group(ConfigGroup *c, DWORD timeout_ms)
+{
+    process_status_info_t status_info;
+
+    memset(&status_info, 0, sizeof(process_status_info_t));
+    status_info.policy = c;
+    status_info.callback_type = CB_DETACH_NOT_IN_POLICY;
+    status_info.timeout_ms = timeout_ms;
+
+    return execute_sysinfo_walk(&status_info);
+}
+
+DWORD
+detach_exe(WCHAR *exename, DWORD timeout_ms)
+{
+    process_status_info_t status_info;
+
+    memset(&status_info, 0, sizeof(process_status_info_t));
+    status_info.callback_type = CB_DETACH_EXE;
+    status_info.timeout_ms = timeout_ms;
+    status_info.exename = exename;
+
+    return execute_sysinfo_walk(&status_info);
+}
+
+DWORD
+detach_all(DWORD timeout_ms)
+{
+    process_status_info_t status_info;
+
+    memset(&status_info, 0, sizeof(process_status_info_t));
+    status_info.callback_type = CB_DETACH;
+    status_info.timeout_ms = timeout_ms;
+
+    return execute_sysinfo_walk(&status_info);
+}
+
+DWORD
+hotp_notify_all_modes_update(DWORD timeout_ms)
+{
+    process_status_info_t status_info;
+
+    memset(&status_info, 0, sizeof(process_status_info_t));
+    status_info.callback_type = CB_NUDGE_MODES;
+    status_info.timeout_ms = timeout_ms;
+
+    return execute_sysinfo_walk(&status_info);
+}
+
+DWORD
+hotp_notify_all_defs_update(DWORD timeout_ms)
+{
+    process_status_info_t status_info;
+
+    memset(&status_info, 0, sizeof(process_status_info_t));
+    status_info.callback_type = CB_NUDGE_DEFS;
+    status_info.timeout_ms = timeout_ms;
+
+    return execute_sysinfo_walk(&status_info);
+}
+
+DWORD
+hotp_notify_exe_modes_update(WCHAR *exename, DWORD timeout_ms)
+{
+    process_status_info_t status_info;
+
+    memset(&status_info, 0, sizeof(process_status_info_t));
+    status_info.callback_type = CB_NUDGE_EXE;
+    status_info.timeout_ms = timeout_ms;
+    status_info.exename = exename;
+
+    return execute_sysinfo_walk(&status_info);
+}
+
+/* generic nudge DR, action mask determines which actions will be executed,
+ *   timeout_ms is the maximum time for a single process nudge
+ *   delay_ms is pause between processing each process, 0 no pause
+ */
+DWORD
+generic_nudge_all(DWORD action_mask, uint64 client_arg /* optional */,
+                  DWORD timeout_ms, DWORD delay_ms)
+{
+    process_status_info_t status_info;
+
+    memset(&status_info, 0, sizeof(process_status_info_t));
+    status_info.callback_type = CB_NUDGE_GENERIC;
+    status_info.timeout_ms = timeout_ms;
+    status_info.delay_ms = delay_ms;
+    status_info.nudge_action_mask = action_mask;
+    status_info.nudge_client_arg = client_arg;
+
+    return execute_sysinfo_walk(&status_info);
+}
+
+
+/* read process_handle's PEB into peb */
+DWORD
+get_process_peb(HANDLE process_handle, PEB *peb)
+{
+    PROCESS_BASIC_INFORMATION info;
+    unsigned long got;
+    DWORD res;
+    GET_NTDLL(NtQueryInformationProcess, (IN HANDLE ProcessHandle,
+                                          IN PROCESSINFOCLASS ProcessInformationClass, 
+                                          IN PVOID ProcessInformation, 
+                                          IN ULONG ProcessInformationLength, 
+                                          OUT PULONG ReturnLength OPTIONAL));
+
+    if (NtQueryInformationProcess == NULL) {
+        return GetLastError();
+    }
+        
+    res = NtQueryInformationProcess(process_handle, ProcessBasicInformation,
+                                    &info, sizeof(PROCESS_BASIC_INFORMATION),
+                                    &got);
+
+    if (!NT_SUCCESS(res)) {
+        return res;
+    } else if (got != sizeof(PROCESS_BASIC_INFORMATION)) {
+        return ERROR_BAD_LENGTH;
+    }
+
+    /* Read process PEB */
+    res = ReadProcessMemory(process_handle, (LPVOID)info.PebBaseAddress,
+                            peb, sizeof(PEB), &got);
+
+    if (!res || got != sizeof(PEB)) {
+        return GetLastError();
+    }
+
+    return  ERROR_SUCCESS;
+}
+
+/* this is somewhat duplicated from get_process_imgname_cmdline in the src
+ * module, share? FIXME */
+/* name returns just the name of the executable (strips off the path if
+ * it's there) to be compatible with previous implementations */
+/* NOTE len's are in bytes */
+DWORD
+get_process_name_and_cmdline(int pid, WCHAR *name_buf, int name_len, WCHAR *cmdline_buf, int cmdline_len)
+{
+    HANDLE process_handle = NULL;
+    unsigned long nbytes;
+    DWORD res;
+    PEB peb;
+    RTL_USER_PROCESS_PARAMETERS process_parameters;
+    DWORD error = ERROR_SUCCESS;
+
+    acquire_privileges();
+    process_handle = OpenProcess(PROCESS_ALL_ACCESS,
+                                 FALSE,
+                                 pid);
+    error = GetLastError();
+    release_privileges();
+
+    if (process_handle == NULL) {
+        goto exit;
+    }
+
+    res = get_process_peb(process_handle, &peb);
+
+    if (res != ERROR_SUCCESS) {
+        error = res;
+        goto exit;
+    }
+
+    /* Follow on to process parameters */
+    res = ReadProcessMemory(process_handle, (LPVOID)peb.ProcessParameters,
+                            &process_parameters, sizeof(process_parameters), 
+                            &nbytes);
+    
+    if (!res) {
+        error = GetLastError();
+        goto exit;
+    }
+
+    if (cmdline_buf != NULL) {
+        /* note that Buffer is a pointer here, compare win32/os.c
+         * it seems that something during process initialization converts
+         *  Buffer from an offset to a pointer...
+         */
+        void *cmdline_location = (void*)((uint)process_parameters.CommandLine.Buffer);
+        
+        int cmdlen = process_parameters.CommandLine.Length;
+        cmdlen = (cmdlen > cmdline_len - 1) ? cmdline_len - 1 : cmdlen;
+        
+        res = ReadProcessMemory(process_handle, (LPVOID)cmdline_location,
+                                cmdline_buf, cmdlen, &nbytes);
+        
+        if (!res) {
+            error = GetLastError();
+            goto exit;
+        }
+        
+        cmdline_buf[cmdlen/sizeof(WCHAR)] = 0;
+        
+    }
+    
+    if (name_buf != NULL) {
+        /* note that Buffer is a pointer here, compare win32/os.c
+         * it seems that something during process initialization converts
+         *  Buffer from an offset to a pointer...
+         */
+        WCHAR buf[MAX_PATH];
+        void *name_location = (void*)((uint)process_parameters.ImagePathName.Buffer);
+        
+        int namelen = process_parameters.ImagePathName.Length;
+        namelen = (namelen > name_len - 1) ? name_len - 1 : namelen;
+        
+        res = ReadProcessMemory(process_handle, (LPVOID)name_location,
+                                buf, sizeof(buf), &nbytes);
+        
+        if (!res) {
+            error = GetLastError();
+            goto exit;
+        }
+        
+        buf[MAX_PATH-1] = L'\0';
+
+        // return just the executable name
+        wcsncpy(name_buf, w_get_short_name(buf), namelen/sizeof(WCHAR));
+        
+        name_buf[namelen/sizeof(WCHAR)] = 0;
+    }
+
+ exit:
+    if (process_handle != NULL)
+        CloseHandle(process_handle);
+    return error;
+}
+
+DWORD
+get_process_cmdline(int pid, WCHAR *buf, int len)
+{
+    return get_process_name_and_cmdline(pid, NULL, 0, buf, len);
+}
+
+DWORD
+get_process_name(int pid, WCHAR *buf, int len)
+{
+    return get_process_name_and_cmdline(pid, buf, len, NULL, 0);   
+}
+
+#define MAX_PROCESS_WALK_BUFFER_LENGTH 0x1000000
+
+DWORD
+process_walk(processwalk_callback pwcb, void **param)
+{
+    void *proc_base = NULL;
+    SYSTEM_PROCESSES *proc_snap;
+    unsigned long got, proc_bytes = 4096 /* is doubled till large enough */;
+    DWORD res;
+    GET_NTDLL(NtQuerySystemInformation, (IN SYSTEM_INFORMATION_CLASS SystemInformationClass,
+                                         IN OUT PVOID SystemInformation, 
+                                         IN ULONG SystemInformationLength, 
+                                         OUT PULONG ReturnLength OPTIONAL));
+
+    if (NtQuerySystemInformation == NULL) {
+        return GetLastError();
+    }
+
+    do {
+        if (proc_base != NULL)
+            free(proc_base);
+        proc_bytes *= 2;
+        if (proc_bytes > MAX_PROCESS_WALK_BUFFER_LENGTH)
+            return ERROR_NOT_ENOUGH_MEMORY;
+        proc_base = malloc(proc_bytes);
+        if (proc_base == NULL)
+            return ERROR_NOT_ENOUGH_MEMORY; 
+        proc_snap = (SYSTEM_PROCESSES *)proc_base;
+        res = NtQuerySystemInformation(SystemProcessesAndThreadsInformation,
+                                       proc_snap, proc_bytes, &got);
+    } while (res == STATUS_INFO_LENGTH_MISMATCH);
+
+    if (NT_SUCCESS(res)) {
+        while (proc_snap != NULL) {
+            process_info_t info;
+
+            info.ThreadCount = proc_snap->ThreadCount;
+            info.CreateTime = proc_snap->CreateTime;
+            info.UserTime = proc_snap->UserTime;
+            info.KernelTime = proc_snap->KernelTime;
+            // NOTE - the system idle process has a NULL name
+            if (proc_snap->ProcessName.Buffer == NULL)
+                info.ProcessName = L"";
+            else
+                info.ProcessName = proc_snap->ProcessName.Buffer;
+            info.BasePriority = proc_snap->BasePriority;
+            info.ProcessID = proc_snap->ProcessId;
+            info.InheritedFromProcessID = proc_snap->InheritedFromProcessId;
+            info.HandleCount = proc_snap->HandleCount;
+            info.VmCounters = proc_snap->VmCounters;
+
+            //callback with the info
+            if( ! (*pwcb)(&info, param) )
+                break;
+            
+            if (proc_snap->NextEntryDelta == 0) {
+                proc_snap = NULL;
+            } else { 
+                proc_snap = (SYSTEM_PROCESSES *)
+                    (((uint)proc_snap) + proc_snap->NextEntryDelta);
+            }
+        }
+        res = ERROR_SUCCESS;
+    }
+
+    free(proc_base);
+
+    return res;
+}
+
+typedef struct cb_helper_ {
+    process_callback pcb;
+    void ** param;
+} cb_helper_t;
+
+BOOL 
+translate_cb(process_info_t *pi, void **param)
+{
+    cb_helper_t * cbht = (cb_helper_t *) param;
+    return (*cbht->pcb)(pi->ProcessID, pi->ProcessName, cbht->param);
+}
+
+DWORD
+enumerate_processes(process_callback pcb, void **param)
+{
+    cb_helper_t cbht;
+    cbht.pcb = pcb;
+    cbht.param = param;
+    return process_walk(&translate_cb, (void **)&cbht);
+}
+
+#define MAX_MODULE_LIST_INFINITE_LOOP_THRESHOLD 2048
+
+DWORD
+dll_walk_proc(int ProcessID, dllwalk_callback dwcb, void **param)
+{
+    PEB peb;
+    PEB_LDR_DATA ldr;
+    LIST_ENTRY *first;
+    LDR_MODULE mod;
+    HANDLE hproc;
+    DWORD res;
+    int i=0;
+    unsigned long nbytes;
+
+    // use a read process memory implementation (like psapi) since toolhelp
+    // is too invasive and not available on all platforms
+    
+    acquire_privileges();
+    hproc = OpenProcess(PROCESS_VM_READ|PROCESS_QUERY_INFORMATION, FALSE, 
+                        ProcessID);
+    res = GetLastError();
+    release_privileges();
+    if (hproc == NULL)
+        return res;
+
+    res = get_process_peb(hproc, &peb);
+    if (res != ERROR_SUCCESS)
+        goto exit;
+
+    res = ReadProcessMemory(hproc, peb.LoaderData, &ldr, sizeof(ldr), &nbytes);
+    if (!res || nbytes != sizeof(ldr)) {
+        res = GetLastError();
+        goto exit;
+    }
+
+    // arbitrary - use the InLoadOrderList since it has simplest offsets
+    first = (LIST_ENTRY *)(((uint)peb.LoaderData) + 
+                           offsetof(PEB_LDR_DATA, InLoadOrderModuleList));
+    // prime the loop
+    mod.InLoadOrderModuleList.Flink = ldr.InLoadOrderModuleList.Flink;
+
+    do {
+        module_info_t info;
+        BOOL callback_ret_val;
+
+        res = ReadProcessMemory(hproc, mod.InLoadOrderModuleList.Flink, &mod, 
+                                sizeof(mod), &nbytes);
+        if (!res || nbytes != sizeof(mod)) {
+            res = GetLastError();
+            goto exit;
+        }
+
+        // Fill info struct for the callback
+        info.BaseAddress = mod.BaseAddress;
+        info.EntryPoint = mod.EntryPoint;
+        info.SizeOfImage = mod.SizeOfImage;
+        info.LoadCount = mod.LoadCount;
+        info.TlsIndex = mod.TlsIndex;
+        info.TimeDateStamp = mod.TimeDateStamp;
+        info.ProcessID = ProcessID;
+        
+        // now copy over the strings
+
+        // allocate length + sizeof(wchar) [for null termination]
+        info.FullDllName = (WCHAR *)malloc(mod.FullDllName.Length + sizeof(WCHAR));
+        res = ReadProcessMemory(hproc, mod.FullDllName.Buffer, 
+                                info.FullDllName, mod.FullDllName.Length, 
+                                &nbytes);
+        if (!res || nbytes != mod.FullDllName.Length) {
+            res = GetLastError();
+            goto exit;
+        }
+        // Be sure to NULL terminate
+        info.FullDllName[mod.FullDllName.Length/sizeof(WCHAR)] = 0;
+
+        // allocate length + sizeof(wchar) [for null termination]
+        info.BaseDllName = (WCHAR *)malloc(mod.BaseDllName.Length + sizeof(WCHAR));
+        res = ReadProcessMemory(hproc, mod.BaseDllName.Buffer, 
+                                info.BaseDllName, mod.BaseDllName.Length, 
+                                &nbytes);
+        if (!res || nbytes != mod.BaseDllName.Length) {
+            res = GetLastError();
+            goto exit;
+        }
+        // Be sure to NULL terminate
+        info.BaseDllName[mod.BaseDllName.Length/sizeof(WCHAR)] = 0;
+
+        // do callback
+        callback_ret_val = (*dwcb)(&info, param);
+
+        // free strings
+        free(info.FullDllName);
+        free(info.BaseDllName);
+        
+        // now check the callback return value for early loop exit
+        if (!callback_ret_val)
+            break;
+
+    } while (mod.InLoadOrderModuleList.Flink != first && 
+             i++ < MAX_MODULE_LIST_INFINITE_LOOP_THRESHOLD);
+
+    if (i >= MAX_MODULE_LIST_INFINITE_LOOP_THRESHOLD)
+        res = ERROR_TOO_MANY_MODULES;
+    else
+        res = ERROR_SUCCESS;
+
+ exit:
+    CloseHandle(hproc);
+    return res;
+}
+
+typedef struct dll_walk_all_cb_info_s {
+    dllwalk_callback dwcb;
+    void **param;
+} dll_walk_all_cb_info_t;
+
+BOOL
+dll_walk_all_cb(process_info_t *pi, void **param)
+{
+    dll_walk_all_cb_info_t *info = (dll_walk_all_cb_info_t *)param;
+    // will instantly silently crash if we try to peer into 
+    // system-idle process (pid 0) with upgraded permissions 
+    if (pi->ProcessID != 0)
+        dll_walk_proc(pi->ProcessID, info->dwcb, info->param);
+    return true;
+}
+
+DWORD
+dll_walk_all(dllwalk_callback dwcb, void **param)
+{
+    dll_walk_all_cb_info_t info;
+    info.dwcb = dwcb;
+    info.param = param;
+    return process_walk((processwalk_callback)dll_walk_all_cb, (void **)&info);
+}
+
+DWORD
+terminate_process(int pid)
+{
+    HANDLE hproc;
+    DWORD res = 0;
+
+    acquire_privileges();
+    hproc = OpenProcess(PROCESS_TERMINATE, FALSE, pid);
+    release_privileges();
+
+    if (!hproc) 
+        return GetLastError();
+
+    if(!TerminateProcess(hproc,(UINT) -1))
+        res = GetLastError();
+
+    CloseHandle(hproc);
+
+    return res;
+}
+
+typedef struct kill_helper_s {
+    WCHAR *exename;
+    DWORD res;
+} kill_helper_t;
+
+BOOL
+kill_callback(process_info_t *pi, void **param)
+{
+    kill_helper_t *khs = (kill_helper_t *)param;
+
+    if (khs->exename == NULL)
+        return FALSE;
+
+    if (0 == wcsicmp(khs->exename, pi->ProcessName)) {
+        khs->res = terminate_process(pi->ProcessID);
+        if (khs->res != ERROR_SUCCESS)
+            return FALSE;
+    }
+    
+    return TRUE;
+}
+
+DWORD
+terminate_process_by_exe(WCHAR *exename)
+{
+    kill_helper_t khs;
+    khs.exename = exename;
+    khs.res = ERROR_SUCCESS;
+
+    process_walk(&kill_callback, (void **)&khs);
+
+    return khs.res;
+}
+
+
+#else //ifdef UNIT_TEST
+
+int
+main()
+{
+    set_debuglevel(DL_INFO);
+    set_abortlevel(DL_WARN);
+
+    /* most of these are hard to do without the test framework.. */
+
+    printf("All Test Passed\n");
+
+    return 0;
+
+}
+
+#endif
diff --git a/libutil/processes.h b/libutil/processes.h
new file mode 100644
index 000000000..7fa5de148
--- /dev/null
+++ b/libutil/processes.h
@@ -0,0 +1,228 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* 
+ *
+ * processes.h
+ *
+ */
+
+#ifndef _DETERMINA_PROCESSES_H_
+#define _DETERMINA_PROCESSES_H_
+
+#include "config.h"
+
+#ifdef __cplusplus
+extern "C"{
+#endif 
+
+#include "lib/dr_stats.h" // from src tree
+
+/* dup KPRIORITY and VM_COUNTERS from src/win32/ntdll.h, in order
+ *  to have process_info_s self-contained.
+ * (so that, eg, src/ module is not necessary to build nodemgr.) */
+#ifndef _NTDLL_H_
+
+typedef LONG KPRIORITY;
+
+/* format of data returned by QueryInformationProcess ProcessVmCounters */
+typedef struct _VM_COUNTERS {
+    SIZE_T PeakVirtualSize;
+    SIZE_T VirtualSize;
+    ULONG PageFaultCount;
+    SIZE_T PeakWorkingSetSize;
+    SIZE_T WorkingSetSize;
+    SIZE_T QuotaPeakPagedPoolUsage;
+    SIZE_T QuotaPagedPoolUsage;
+    SIZE_T QuotaPeakNonPagedPoolUsage;
+    SIZE_T QuotaNonPagedPoolUsage;
+    SIZE_T PagefileUsage;
+    SIZE_T PeakPagefileUsage;
+} VM_COUNTERS;
+
+#endif
+
+typedef struct process_info_s {
+    ULONG ThreadCount;
+    LARGE_INTEGER CreateTime;
+    LARGE_INTEGER UserTime;
+    LARGE_INTEGER KernelTime;
+    WCHAR *ProcessName;
+    KPRIORITY BasePriority;
+    ULONG ProcessID;
+    ULONG InheritedFromProcessID;
+    ULONG HandleCount;
+    VM_COUNTERS VmCounters;
+} process_info_t;
+
+
+typedef struct module_info_s {
+    PVOID BaseAddress;
+    PVOID EntryPoint;
+    ULONG SizeOfImage;
+    // strings are NULL terminated, non persistant and in this address space
+    WCHAR *FullDllName; // dll name with path (i.e. "c:\win\sys32\foo.dll")
+    WCHAR *BaseDllName; // just dll name (i.e. "foo.dll")
+    SHORT LoadCount;
+    SHORT TlsIndex;
+    ULONG TimeDateStamp;
+    ULONG ProcessID; // ID of Process this module is loaded into
+} module_info_t;
+
+
+/* for process and dll callback routines; the callback should
+ *  return FALSE to abort the walk */
+typedef BOOL (*processwalk_callback)(process_info_t *pi, void **param);
+
+typedef BOOL (*dllwalk_callback)(module_info_t *mi, void **param);
+
+
+
+/*
+ * generic process information methods
+ */ 
+
+
+DWORD
+dll_walk_proc(int ProcessID, dllwalk_callback dwcb, void **param);
+
+DWORD
+dll_walk_all(dllwalk_callback dwcb, void **param);
+
+DWORD
+process_walk(processwalk_callback pwcb, void **param);
+
+DWORD
+get_process_name_and_cmdline(int pid, WCHAR *name_buf, int name_len, 
+                             WCHAR *cmdline_buf, int cmdline_len);
+
+DWORD
+get_process_name(int pid, WCHAR *buf, int len);
+
+DWORD
+get_process_cmdline(int pid, WCHAR *buf, int len);
+
+
+
+/*
+ * heavy-duty kill
+ */ 
+
+DWORD
+terminate_process(int pid);
+
+DWORD
+terminate_process_by_exe(WCHAR *exename);
+
+
+
+/*
+ * DR process status / drmarker functions
+ */ 
+
+int
+under_dynamorio(int ProcessID);
+
+int
+under_dynamorio_ex(int ProcessID, DWORD *build_num);
+
+/* pending_restart is required OUT parameter.
+ * status is optional OUT parameter -- if not NULL, will have the status
+ *  of the process after successful return.
+ * process_cfg is optional OUT parameter -- if not NULL, will have a pointer
+ *  to the config group in the config for this process, if available.
+ * for internal convenience, may pass NULL for config, in which case
+ *  status is the only well-defined output parameter. */
+DWORD
+check_status_and_pending_restart(ConfigGroup *config, int pid, 
+                                 BOOL *pending_restart, 
+                                 int *status, ConfigGroup **process_cfg);
+
+/* pending_restart is required OUT parameter */
+DWORD
+is_anything_pending_restart(ConfigGroup *c, BOOL *pending_restart);
+
+/* hotpatch status: the returned pointer must be freed with 
+ *  free_hotp_status_table below. */
+DWORD
+get_hotp_status(int pid, hotp_policy_status_table_t **hotp_status);
+
+/* if a hotp_status table was requested, it must be freed. */
+void
+free_hotp_status_table(hotp_policy_status_table_t *hotp_status);
+
+dr_statistics_t *
+get_dynamorio_stats(int pid);
+
+void
+free_dynamorio_stats(dr_statistics_t *stats);
+
+/*
+ * Detach and nudge functions
+ */
+
+/* Loads the specified dll in process pid and waits on the loading thread, does
+ * not free the dll once the loading thread exits. Usual usage is for the 
+ * loaded dll to do something in its DllMain. If you do not want the dll to 
+ * stay loaded its DllMain should return false. To unload a dll from a process 
+ * later, inject another dll whose dll main unloads that dll and then returns 
+ * false. If loading_thread != NULL returns a handle to the loading thread (dll
+ * could call FreeLibraryAndExitThread on itself in its dll main to return a 
+ * value out via the exit code). inject_dll provides no way to pass arguments 
+ * in to the dll. */
+DWORD
+inject_dll(int pid, const WCHAR *dll_name, BOOL allow_upgraded_perms, 
+           DWORD timeout_ms, PHANDLE loading_thread);
+/* pending_restart is required OUT parameter */
+DWORD
+detach_all_not_in_config_group(ConfigGroup *c, DWORD timeout_ms);
+
+/* generic nudge DR, action mask determines which actions will be executed */
+DWORD
+generic_nudge(int pid, BOOL allow_upgraded_perms, DWORD action_mask, 
+              client_id_t id /* optional */, uint64 client_arg /* optional */,
+              DWORD timeout_ms);
+
+/* generic nudge DR, action mask determines which actions will be executed,
+ *   timeout_ms is the maximum time for a single process nudge
+ *   delay_ms is pause between processing each process, 0 no pause
+ */
+DWORD
+generic_nudge_all(DWORD action_mask, uint64 client_arg /* optional */,
+                  DWORD timeout_ms, DWORD delay_ms);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif //_NM_PROCESSES_H_
diff --git a/libutil/sample.mfp b/libutil/sample.mfp
new file mode 100644
index 000000000..77a5aff24
--- /dev/null
+++ b/libutil/sample.mfp
@@ -0,0 +1,3488 @@
+POLICY_VERSION=30000
+GLOBAL_PROTECT=1
+BEGIN_BLOCK
+GLOBAL
+DYNAMORIO_HOT_PATCH_POLICIES=\config
+DYNAMORIO_OPTIONS=
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+DYNAMORIO_RUNUNDER=1
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=detertest.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\detertest.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=nodemgr.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\nodemgr.exe
+DYNAMORIO_OPTIONS=-report_max 0
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=mysqld-nt.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\mysqld-nt.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=openssl.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\openssl.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=nmupgrade.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\nmupgrade.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=msitwdl.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\msitwdl.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=tomcat.exe-RSTomcatSC
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\tomcat.exe-RSTomcatSC
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+DYNAMORIO_RUNUNDER=49
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=tomcat.exe
+DYNAMORIO_RUNUNDER=48
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=detertray.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\detertray.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=wld.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\wld.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=svchost.exe-bitsgroup
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\svchost.exe-bitsgroup
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+DYNAMORIO_RUNUNDER=17
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=svchost.exe
+DYNAMORIO_RUNUNDER=16
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=certsrv.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\certsrv.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=cisvc.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\cisvc.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=clipsrv.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\clipsrv.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=clussvc.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\clussvc.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=dfssvc.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\dfssvc.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=dmadmin.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\dmadmin.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=dns.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\dns.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=faxsvc.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\faxsvc.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=grovel.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\grovel.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=ismserv.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\ismserv.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+DYNAMORIO_RUNUNDER=5
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=llssrv.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\llssrv.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=locator.exe
+BEGIN_MP_MODES
+36
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.001A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\locator.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=lsass.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\lsass.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=lserver.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\lserver.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=mnmsrvc.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\mnmsrvc.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=mqsvc.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\mqsvc.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=msdtc.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\msdtc.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=msiexec.exe-V
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\msiexec.exe-V
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+DYNAMORIO_RUNUNDER=49
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=msiexec.exe
+DYNAMORIO_RUNUNDER=48
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=mstask.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\mstask.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=netdde.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\netdde.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=svchost.exe-netsvcs
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\svchost.exe-netsvcs
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+DYNAMORIO_RUNUNDER=17
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=svchost.exe
+DYNAMORIO_RUNUNDER=16
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=nscm.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\nscm.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=nslservice.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\nslservice.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=nspm.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\nspm.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=nspmon.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\nspmon.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=nsum.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\nsum.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=ntfrs.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\ntfrs.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=regsvc.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\regsvc.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+DYNAMORIO_RUNUNDER=5
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=svchost.exe-rpcss
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\svchost.exe-rpcss
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+DYNAMORIO_RUNUNDER=17
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=svchost.exe
+DYNAMORIO_RUNUNDER=16
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=rseng.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\rseng.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=rsfsa.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\rsfsa.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=rssub.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\rssub.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=rsvp.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\rsvp.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=scardsvr.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\scardsvr.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+DYNAMORIO_RUNUNDER=5
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=services.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\services.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=sfmprint.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\sfmprint.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=sfmsvc.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\sfmsvc.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=smlogsvc.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\smlogsvc.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=snmp.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\snmp.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+DYNAMORIO_RUNUNDER=5
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=snmptrap.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\snmptrap.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+DYNAMORIO_RUNUNDER=5
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=spoolsv.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\spoolsv.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+DYNAMORIO_RUNUNDER=5
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=svchost.exe-tapisrv
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\svchost.exe-tapisrv
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+DYNAMORIO_RUNUNDER=17
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=svchost.exe
+DYNAMORIO_RUNUNDER=16
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=tcpsvcs.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\tcpsvcs.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+DYNAMORIO_RUNUNDER=5
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=termsrv.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\termsrv.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=tftpd.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\tftpd.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=tlntsvr.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\tlntsvr.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=ups.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\ups.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=utilman.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\utilman.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=winlogon.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\winlogon.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=winmgmt.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\winmgmt.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=wins.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\wins.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=svchost.exe-wugroup
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\svchost.exe-wugroup
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+DYNAMORIO_RUNUNDER=17
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=svchost.exe
+DYNAMORIO_RUNUNDER=16
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=wuauclt.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\wuauclt.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=dllhost.exe-Processid02D4B3F1FD8811D1960D00805FC79235
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\dllhost.exe-Processid02D4B3F1FD8811D1960D00805FC79235
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+DYNAMORIO_RUNUNDER=49
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=dllhost.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\dllhost.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 100
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+DYNAMORIO_RUNUNDER=48
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=inetinfo.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\inetinfo.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 100
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=dllhost.exe-Processid3D14228DFBE111D0995D00C04FD919C1
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\dllhost.exe-Processid3D14228DFBE111D0995D00C04FD919C1
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+DYNAMORIO_RUNUNDER=49
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=sqlservr.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\sqlservr.exe
+DYNAMORIO_OPTIONS=-report_max 0 -throw_exception -throw_exception_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=sqladhlp.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\sqladhlp.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=msmdsrv.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\msmdsrv.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+DYNAMORIO_RUNUNDER=49
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=sqlagent.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\sqlagent.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=msde.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\msde.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=store.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\store.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=events.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\events.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=exmgmt.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\exmgmt.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=emsmta.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\emsmta.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=srsmain.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\srsmain.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
+BEGIN_BLOCK
+APP_NAME=mad.exe
+BEGIN_MP_MODES
+35
+0DAY.0001:2
+0DAY.0002:2
+0DAY.0004:2
+BUGF.0001:2
+BUGF.0002:2
+MS00.078A:2
+MS01.026A:2
+MS02.012A:2
+MS02.018A:2
+MS03.007A:2
+MS03.020A:2
+MS03.022A:2
+MS03.026A:2
+MS03.039A:2
+MS03.039B:2
+MS03.043A:2
+MS04.007A:2
+MS04.011A:2
+MS04.011B:2
+MS04.011C:2
+MS04.011D:2
+MS04.036A:2
+MS05.002A:2
+MS05.038A:2
+MS05.039A:2
+MS05.047A:2
+MS05.051A:2
+MS05.053A:2
+MS05.054B:2
+MS06.001A:2
+MS06.005A:2
+MS06.008A:2
+MS06.013A:2
+MS06.013B:2
+TEST.000A:2
+END_MP_MODES
+DYNAMORIO_HOT_PATCH_MODES=\config\mad.exe
+DYNAMORIO_OPTIONS=-report_max 0 -kill_thread -kill_thread_max 1000
+DYNAMORIO_AUTOINJECT=\lib\40000\dynamorio.dll
+END_BLOCK
diff --git a/libutil/services.c b/libutil/services.c
new file mode 100644
index 000000000..5ac7ef168
--- /dev/null
+++ b/libutil/services.c
@@ -0,0 +1,620 @@
+/* **********************************************************
+ * Copyright (c) 2003-2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* 
+ * services.c
+ *
+ * helper methods dealing with services
+ *  
+ */
+
+#include "share.h"
+#include "services.h"
+
+#ifndef UNIT_TEST
+
+UINT NUM_SERVICES = 0;
+
+typedef struct ServiceInfo_ {
+    ServiceHandle svch;
+    DWORD start_type;
+    WCHAR *binary_pathname;
+    WCHAR *service_name;
+    WCHAR *service_display_name;
+} ServiceInfo;
+
+ServiceInfo *services = NULL;
+
+/* the service manager database handle */
+SC_HANDLE scmdb;
+
+BOOL services_initialized = FALSE;
+
+#define SVC_BUFSZ 4096
+
+DWORD
+get_service_strings()
+{
+    DWORD needed, nsvcs, next = 0, i;
+    BOOL bRes;
+    ENUM_SERVICE_STATUS *infobuf;
+    SC_HANDLE sch;
+    QUERY_SERVICE_CONFIG *scfg;
+    BYTE buffer[SVC_BUFSZ];
+    
+    /* now read in the service information */
+    scfg = (QUERY_SERVICE_CONFIG *)buffer;
+
+    EnumServicesStatus(scmdb, SERVICE_WIN32, SERVICE_STATE_ALL, NULL, 0,
+                       &needed, &nsvcs, &next);
+    infobuf = (ENUM_SERVICE_STATUS *)malloc(needed);
+
+    bRes = EnumServicesStatus(scmdb, 
+                              SERVICE_WIN32, 
+                              SERVICE_STATE_ALL, 
+                              infobuf, 
+                              needed, 
+                              &needed, 
+                              &nsvcs, 
+                              &next);
+    
+    if(!bRes)
+        return GetLastError();
+
+    NUM_SERVICES = nsvcs;
+    services = (ServiceInfo *)malloc(nsvcs * sizeof(ServiceInfo));
+    memset(services, 0, nsvcs * sizeof(ServiceInfo));
+
+    for(i = 0; i < nsvcs; i++) {
+        services[i].svch = i;
+        services[i].service_name = wcsdup(infobuf[i].lpServiceName);
+        
+        sch = OpenService(scmdb, services[i].service_name,
+                          SERVICE_QUERY_CONFIG);
+        if (QueryServiceConfig(sch, scfg, SVC_BUFSZ, &needed)) {
+            services[i].service_display_name =
+                wcsdup(scfg->lpDisplayName);
+
+            services[i].binary_pathname =
+                wcsdup(scfg->lpBinaryPathName);
+
+            services[i].start_type = scfg->dwStartType;
+        }
+
+        CloseServiceHandle(sch);
+    }
+
+    free(infobuf);
+
+    return ERROR_SUCCESS;
+}
+
+void
+free_service_strings()
+{
+    UINT i;
+
+    if (services == NULL)
+        return;
+
+    for(i = 0; i < NUM_SERVICES; i++) {
+        free(services[i].service_name);
+        free(services[i].service_display_name);
+        free(services[i].binary_pathname);
+    }
+
+    free(services);
+
+    services = NULL;
+}
+
+DWORD
+reload_service_info()
+{
+    free_service_strings();
+    return get_service_strings();
+}
+
+
+/*
+ *
+ * interface functions
+ *
+ */
+
+DWORD
+enumerate_services(services_callback cb, void **param)
+{
+    ServiceHandle svc;
+
+    for (svc = 0; svc < NUM_SERVICES; svc++)
+        if ( ! (*cb)(svc, param) )
+            break;
+
+    return ERROR_SUCCESS;
+}
+
+ServiceHandle
+get_service_by_name(const WCHAR *name)
+{
+    ServiceHandle svc;
+    for (svc = 0; svc < NUM_SERVICES; ++svc)
+        if (!wcsicmp(services[svc].service_name, name))
+            return svc;
+
+    return INVALID_SERVICE_HANDLE;
+}
+
+const WCHAR *
+get_service_name(ServiceHandle service)
+{
+    if(0 <= service && service < NUM_SERVICES)
+        return services[service].service_name;
+    else
+        return NULL;
+}
+
+const WCHAR *
+get_service_display_name(ServiceHandle service)
+{
+    if (0 <= service && service < NUM_SERVICES)
+        return services[service].service_display_name;
+    else
+        return NULL;
+}
+
+DWORD
+services_init()
+{
+    DWORD res;
+    if (services_initialized)
+        return ERROR_ALREADY_INITIALIZED;
+
+    /* FIXME: probably only need read access, except
+     *  for change config...but that should just open its own
+     *  handle. */
+    scmdb = OpenSCManager(NULL, NULL, SC_MANAGER_ALL_ACCESS);
+    if (scmdb == NULL) {
+        /* probably because of non-admin privileges. open read only. */
+        scmdb = OpenSCManager(NULL, NULL, GENERIC_READ);
+        if (scmdb == NULL) {
+            res = GetLastError();
+            return res == ERROR_SUCCESS ? -1 : res;
+        }
+    }
+
+    res = get_service_strings();
+    if (res != ERROR_SUCCESS)
+        return res;
+    services_initialized = TRUE;
+
+    return ERROR_SUCCESS;
+}
+
+DWORD
+services_cleanup()
+{
+    if (!services_initialized)
+        return ERROR_SUCCESS;
+
+    services_initialized = FALSE;
+
+    free_service_strings();
+
+    CloseServiceHandle(scmdb);
+    NUM_SERVICES = 0;
+
+    return ERROR_SUCCESS;
+}
+
+
+int
+service_status(ServiceHandle service)
+{
+    SERVICE_STATUS ss;
+
+    SC_HANDLE hsvc = OpenService(scmdb, 
+                                 services[service].service_name,
+                                 SERVICE_QUERY_STATUS);
+
+    QueryServiceStatus(hsvc, &ss);
+    CloseServiceHandle(hsvc);
+    
+    return ss.dwCurrentState;
+}
+
+DWORD
+add_dependent_service(ServiceHandle service, ServiceHandle requiredService)
+{
+    BOOL cscres;
+    WCHAR dep[MAX_PATH];
+    WCHAR *nextdep = NULL;
+    QUERY_SERVICE_CONFIG *scfg;
+    BYTE buffer[SVC_BUFSZ];
+    DWORD size;
+    SC_HANDLE hsvc = OpenService(scmdb,
+                                 services[service].service_name,
+                                 SERVICE_QUERY_CONFIG|SERVICE_CHANGE_CONFIG);
+
+    if (hsvc == NULL)
+        return GetLastError();
+
+    scfg = (QUERY_SERVICE_CONFIG *)buffer;
+    if (!QueryServiceConfig(hsvc, scfg, SVC_BUFSZ, &size)) {
+        return GetLastError();
+    }
+
+    nextdep = scfg->lpDependencies;
+    size = 0;
+
+    while (nextdep != NULL) {
+        if (nextdep[0] == L'\0')
+            break;
+        size += wcslen(nextdep) + 1;
+        nextdep = scfg->lpDependencies + size;
+    }
+
+    if (size > MAX_PATH - 2)
+        return ERROR_INSUFFICIENT_BUFFER;
+
+    if (size != 0) {
+        memcpy(dep, scfg->lpDependencies, size * sizeof(WCHAR));
+    }
+
+    wcsncpy(&dep[size], services[requiredService].service_name, 
+            MAX_PATH - size); 
+
+    size += wcslen(services[requiredService].service_name) + 1;
+
+    if (size > MAX_PATH - 2) {
+        CloseServiceHandle(hsvc);
+        return ERROR_INSUFFICIENT_BUFFER;
+    }
+        
+    dep[size + 1] = L'\0';
+
+    cscres = ChangeServiceConfig(hsvc,
+                                 SERVICE_NO_CHANGE,
+                                 SERVICE_NO_CHANGE,
+                                 SERVICE_NO_CHANGE,
+                                 NULL,
+                                 NULL,
+                                 NULL,
+                                 dep,
+                                 NULL,
+                                 NULL,
+                                 NULL);
+    
+    CloseServiceHandle(hsvc);
+
+    if (!cscres)
+        return GetLastError();
+
+    return reload_service_info();
+}
+
+DWORD
+reset_dependent_services(ServiceHandle service)
+{
+    BOOL cscres;
+    SC_HANDLE hsvc = OpenService(scmdb,
+                                 services[service].service_name,
+                                 SERVICE_CHANGE_CONFIG);
+
+    if (hsvc == NULL)
+        return GetLastError();
+
+    cscres = ChangeServiceConfig(hsvc,
+                                 SERVICE_NO_CHANGE,
+                                 SERVICE_NO_CHANGE,
+                                 SERVICE_NO_CHANGE,
+                                 NULL,
+                                 NULL,
+                                 NULL,
+                                 L"",
+                                 NULL,
+                                 NULL,
+                                 NULL);
+    
+    CloseServiceHandle(hsvc);
+
+    if (!cscres)
+        return GetLastError();
+
+    return reload_service_info();
+}
+
+DWORD
+set_service_start_type(ServiceHandle service, DWORD dwStartType)
+{
+    BOOL cscres;
+    SC_HANDLE hsvc = OpenService(scmdb,
+                                 services[service].service_name,
+                                 SERVICE_CHANGE_CONFIG);
+
+    if (hsvc == NULL)
+        return GetLastError();
+
+    cscres = ChangeServiceConfig(hsvc,
+                                 SERVICE_NO_CHANGE,
+                                 dwStartType,
+                                 SERVICE_NO_CHANGE,
+                                 NULL,
+                                 NULL,
+                                 NULL,
+                                 NULL,
+                                 NULL,
+                                 NULL,
+                                 NULL);
+    
+    CloseServiceHandle(hsvc);
+
+    if (!cscres)
+        return GetLastError();
+
+    return reload_service_info();
+}
+
+DWORD
+get_service_start_type(ServiceHandle service)
+{
+    return services[service].start_type;
+}
+
+/* delay to wait before restarting the service after a failure. */
+#define SERVICE_RESTART_DELAY_MS 60000
+
+/* QueryServiceConfig2 and ChangeServiceConfig2 are not supported
+ *  on NT, so we load these dynamically.
+ */
+typedef BOOL (WINAPI *QueryServiceConfig2Func)(SC_HANDLE hService, 
+                                               DWORD dwInfoLevel,
+                                               LPBYTE lpBuffer,
+                                               DWORD cbBufSize,
+                                               LPDWORD pcbBytesNeeded);
+
+typedef BOOL (WINAPI *ChangeServiceConfig2Func)(SC_HANDLE hService, 
+                                                DWORD dwInfoLevel,
+                                                LPVOID lpInfo);
+
+/* sets the windows service config to autostart. we do this here
+ *  because InstallShield doesn't provide a nice interface for it, it
+ *  would require adding a custom DLL to the nodemanager
+ *  install. plus, we get the additional benefit that we always make
+ *  sure this is set, even if it gets turned off somehow. in the
+ *  future we can have a controller-configurable parameter that
+ *  controls whether we always enforce auto-restart of the service. 
+ *
+ * if fatal_error_encountered, turn this shit off so we don't spin
+ *  wheels and write one eventlog message every minute. */
+DWORD
+set_service_restart_type(WCHAR *svcname, BOOL disable)
+{
+    SC_HANDLE scmdb = NULL, service = NULL;
+    SC_ACTION restart_action = 
+        { SC_ACTION_RESTART, SERVICE_RESTART_DELAY_MS };
+    SERVICE_FAILURE_ACTIONS failure_actions = 
+        { 0,   /* dwResetPeriod: we only have one failure action, so
+                  just keep at zero*/
+          L"", /* we don't force server reboot on failure */
+          L"", /* nor do we execute any commands */
+          1,   /* we have one action in our array */
+          NULL /* which will be set to &restart_action below */
+        };
+    /* if this buffer size is exceeded then the QueryServiceConfig
+     *  will fail with ERROR_INSUFFICIENT_BUFFER, in which case we can
+     *  be sure that someone mucked our settings. */ 
+    BYTE current_actions_buffer[sizeof(SERVICE_FAILURE_ACTIONS) +
+                                5*sizeof(SC_ACTION)] = { 0 };
+    SERVICE_FAILURE_ACTIONS * current_actions = 
+        (SERVICE_FAILURE_ACTIONS *)&current_actions_buffer;
+    DWORD res = ERROR_SUCCESS, needed;
+    QueryServiceConfig2Func qscf;
+    ChangeServiceConfig2Func cscf;
+    HMODULE advapi = NULL;
+
+    /* finish setting up the failure_actions data structure */
+    failure_actions.lpsaActions = &restart_action;
+
+    /* first: check if we've got the necessary API available. if not,
+     *  this means we're on NT -- abort with ERROR_NOT_SUPPORTED (NT
+     *  does not support service failure actions). */
+    advapi = GetModuleHandle(L"advapi32.dll");
+    if (advapi == NULL) {
+        return ERROR_NOT_SUPPORTED;
+    }
+
+    qscf = (QueryServiceConfig2Func) 
+        GetProcAddress(advapi, "QueryServiceConfig2W");
+    cscf = (ChangeServiceConfig2Func) 
+        GetProcAddress(advapi, "ChangeServiceConfig2W");
+    
+    if (qscf == NULL || cscf == NULL) {
+        return ERROR_NOT_SUPPORTED;
+    }
+
+    /* okay, we've got the API, now do the work */
+
+    if (disable)
+        restart_action.Type = SC_ACTION_NONE;
+
+    scmdb = OpenSCManager(NULL, NULL, SC_MANAGER_ALL_ACCESS);
+    if (scmdb == NULL) {
+        res = GetLastError();
+        goto autorestart_out;
+    }
+    
+    service = OpenService(scmdb,
+                          svcname,
+                          SERVICE_ALL_ACCESS);
+
+    if (service == NULL) {
+        res = GetLastError();
+        goto autorestart_out;
+    }
+
+    if ((*qscf)(service, 
+                SERVICE_CONFIG_FAILURE_ACTIONS,
+                (LPBYTE) current_actions,
+                sizeof(current_actions_buffer),
+                &needed) ||
+        (res = GetLastError()) == ERROR_INSUFFICIENT_BUFFER) {
+        /* check if it's already set up */
+        SC_ACTION *current_action = current_actions->lpsaActions;
+        if (!disable &&
+            current_actions->cActions == 1 &&
+            current_action->Type == SC_ACTION_RESTART)
+            goto autorestart_out;
+
+        /* if not, change the configuration */
+        if (!(*cscf)(service, 
+                     SERVICE_CONFIG_FAILURE_ACTIONS,
+                     (LPVOID) &failure_actions)) {
+            res = GetLastError();
+            goto autorestart_out;
+        }
+        
+    }
+    else {
+        res = GetLastError();
+        goto autorestart_out;
+    }
+
+ autorestart_out:
+    if (service != NULL)
+        CloseServiceHandle(service);
+
+    if (scmdb != NULL)
+        CloseServiceHandle(scmdb);
+
+    return res;
+}
+
+
+
+#else //ifdef UNIT_TEST
+
+
+
+BOOL found;
+ServiceHandle target;
+int num;
+
+BOOL
+svc_cb(ServiceHandle service, void **param)
+{
+    if (service == target)
+        found = TRUE;
+    num++;
+    DO_ASSERT(wcslen(get_service_name(service)) > 0);
+    return TRUE;
+}
+
+
+int
+main()
+{
+    DWORD res, type;
+    WCHAR *s1, *sn1;
+    const WCHAR *c1;
+    ServiceHandle sh1, sh2 = 0;
+
+    set_debuglevel(DL_INFO);
+    set_abortlevel(DL_WARN);
+
+    res = services_init();
+    DO_ASSERT(res == ERROR_SUCCESS);
+
+    sn1 = L"Eventlog";
+    s1 = L"Event Log";
+    sh1 = get_service_by_name(sn1);
+    DO_ASSERT(sh1 != INVALID_SERVICE_HANDLE);
+
+    c1 = get_service_name(sh1);
+    DO_ASSERT_WSTR_EQ(sn1, (WCHAR *)c1);
+    
+    c1 = get_service_display_name(sh1);
+    DO_ASSERT_WSTR_EQ(s1, (WCHAR *)c1);
+
+    found = FALSE;
+    target = sh1;
+    res = enumerate_services(&svc_cb, NULL);
+    DO_ASSERT(res == ERROR_SUCCESS);
+    DO_ASSERT(num > 10);
+    DO_ASSERT(found);
+    printf("%d services found on local machine.\n", num);
+
+
+    sh1 = get_service_by_name(L"Alerter");
+    DO_ASSERT(sh1 != INVALID_SERVICE_HANDLE);
+    type = get_service_start_type(sh1);
+    DO_ASSERT(res == ERROR_SUCCESS);
+
+    res = set_service_start_type(sh1, SERVICE_DEMAND_START);
+    DO_ASSERT(res == ERROR_SUCCESS);
+    
+    res = reload_service_info();
+    DO_ASSERT(res == ERROR_SUCCESS);
+    DO_ASSERT(SERVICE_DEMAND_START == get_service_start_type(sh1));
+
+    res = set_service_start_type(sh1, SERVICE_AUTO_START);
+    DO_ASSERT(res == ERROR_SUCCESS);
+    
+    DO_ASSERT(SERVICE_AUTO_START == get_service_start_type(sh1));
+
+    res = set_service_start_type(sh1, type);
+    DO_ASSERT(type == get_service_start_type(sh1));
+
+#if 0
+    /* FIXME: should have a usable test here */
+    sh1 = get_service_by_name(L"TomcatSC");
+    DO_ASSERT(sh1 != INVALID_SERVICE_HANDLE);
+    sh2 = get_service_by_name(L"MySQL");
+    DO_ASSERT(sh2 != INVALID_SERVICE_HANDLE);
+
+    res = add_dependent_service(sh1, sh2);
+    DO_ASSERT(res == ERROR_SUCCESS);
+#endif
+    
+    res = services_cleanup();
+    DO_ASSERT(res == ERROR_SUCCESS);
+
+
+
+    printf("All Test Passed\n");
+
+    return 0;
+
+}
+
+
+#endif
diff --git a/libutil/services.h b/libutil/services.h
new file mode 100644
index 000000000..07bc1b8d4
--- /dev/null
+++ b/libutil/services.h
@@ -0,0 +1,105 @@
+/* **********************************************************
+ * Copyright (c) 2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+
+/*
+ * services.h
+ *
+ * helper methods dealing with services
+ *
+ */
+
+#ifndef _DETERMINA_SERVICES_H_
+#define _DETERMINA_SERVICES_H_
+
+
+typedef DWORD ServiceHandle;
+
+#define INVALID_SERVICE_HANDLE 0xffffffff
+
+
+
+/* Loads services information from the registry and sets up
+ *  ArakshaService handles; must be called before using any service
+ *  methods below. */
+DWORD
+services_init();
+
+/* call when finished using the interface */
+DWORD
+services_cleanup();
+
+DWORD
+reload_service_info();
+
+ServiceHandle
+get_service_by_name(const WCHAR *name);
+
+const WCHAR *
+get_service_name(ServiceHandle service);
+
+const WCHAR *
+get_service_display_name(ServiceHandle service);
+
+
+/* the callback should return FALSE to abort the walk */
+typedef BOOL (*services_callback)(ServiceHandle service,
+                                  void **param);
+
+/* enumeration callback for all of the services on the system */
+DWORD
+enumerate_services(services_callback cb, void **param);
+
+/* 
+ * returns SERVICE_STOPPED, SERVICE_RUNNING, SERVICE_PENDING_*...
+ */
+int
+service_status(ServiceHandle service);
+
+DWORD
+set_service_start_type(ServiceHandle service, DWORD dwStartType);
+
+DWORD
+get_service_start_type(ServiceHandle service);
+
+DWORD
+set_service_restart_type(WCHAR *svcname, BOOL disable);
+
+DWORD
+add_dependent_service(ServiceHandle service, ServiceHandle requiredService);
+
+DWORD
+reset_dependent_services(ServiceHandle service);
+
+
+
+#endif
diff --git a/libutil/share.h b/libutil/share.h
new file mode 100644
index 000000000..0d8dc5624
--- /dev/null
+++ b/libutil/share.h
@@ -0,0 +1,72 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ *
+ * internal common header file for share interface
+ *
+ */
+
+#ifndef _DETERMINA_SHARE_H_
+#define _DETERMINA_SHARE_H_
+
+/* 
+C:\PROGRAM FILES\MICROSOFT VISUAL STUDIO\VC98\INCLUDE\rpcasync.h(45) : 
+warning C4115: '_RPC_ASYNC_STATE' : named type definition in parentheses
+*/
+#pragma warning(disable : 4115)
+
+#include "mfapi.h"
+
+#ifdef __cplusplus
+extern "C"{
+#endif 
+
+/* make sure; for example, mmcgui shouldn't have to know about this */
+#ifndef NOT_DYNAMORIO_CORE
+#  define NOT_DYNAMORIO_CORE
+#endif
+
+/* FIXME: this should be stripped out of the core at some point...*/
+#ifndef HOT_PATCHING_INTERFACE
+#  define HOT_PATCHING_INTERFACE
+#endif
+
+#include "globals_shared.h"
+
+#ifdef __cplusplus
+}
+#endif
+
+#include "utils.h"
+
+#endif
diff --git a/libutil/tester.c b/libutil/tester.c
new file mode 100644
index 000000000..f00b99fb8
--- /dev/null
+++ b/libutil/tester.c
@@ -0,0 +1,316 @@
+/* **********************************************************
+ * Copyright (c) 2005-2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+
+#include <stdio.h>
+
+#include "share.h"
+#include "utils.h"
+#include "hotp-test.h"
+
+#define OUTFILE L"tester.out"
+
+#define ATTACK_NONE   0
+#define ATTACK_STACK  1
+#define ATTACK_HEAP   2
+
+int attack_status = ATTACK_NONE;
+
+DWORD 
+WINAPI
+heap_attack(LPVOID param);
+
+DWORD 
+WINAPI
+stack_attack(LPVOID param);
+
+DWORD 
+WINAPI
+ls_attack(LPVOID param);
+
+enum { stack=1, heap, liveshield };
+
+/* disable type case warning for attacks below. */
+#pragma warning(disable : 4055)
+
+void usage() {
+    fprintf(stderr,
+        " usage: tester [initial_sleep_ms] [attack] [num_attacks]\n"
+        "\n"
+        " tester will then do:\n"
+        "  (1) [initial_sleep_ms] sleep before doing anything\n"
+        "  (2) if [attack]/[num_attacks] params are not set, exits\n"
+        "  (3) [attack=1,2 or 3] (executes 1=stack, 2=heap or 3=liveshield attack \n"
+        "      (1) liveshield attacks execute test_reg and test_control_flow hotpatches\n"
+        "          (equivalent of araktest liveshield buttons 1 and 2)\n"
+        "      (2) write to \"tester.out.xx\" two characters:\n"
+        "          -- 0 or 1 according to whether test_reg was patched\n"
+        "          -- 0 or 1 according to whether test_control_flow was patched\n"
+        "  (4) repeat step (3) [num_attacks] times\n"
+        "\n"
+        " To run under DR setup appropriate registry settings for tester.exe\n");
+}
+
+int
+main(int argc, char **argv)
+{
+    int timeout = 5000;
+    /*char output[MAX_PATH];*/
+    if (argc == 2
+        && (_stricmp(argv[1], "-h") == 0 || _stricmp(argv[1], "-help") == 0)) {
+        usage();
+        exit(0);
+    }
+
+    if (argc >= 2)
+        timeout = atoi(argv[1]);
+
+    Sleep(timeout);
+    
+    if (argc >= 3) {
+        int i, count = 1;
+        int attack = atoi(argv[2]);
+        if (attack == 0) {
+            fprintf(stderr, "attack=%d not expected\n", attack);
+            usage();
+            exit (-1);
+        }
+        
+        if (argc >= 4)
+            count = atoi(argv[3]);
+
+        for (i = 0; i < count; i++) {
+            HANDLE athr;
+            fprintf(stderr, "loop %d\n",i);
+            if (attack == 1)
+                athr= CreateThread(NULL, 0, stack_attack, NULL, 0, NULL);
+            else if (attack == 2)
+                athr= CreateThread(NULL, 0, heap_attack, NULL, 0, NULL);
+            else if (attack == 3)
+                athr= CreateThread(NULL, 0, ls_attack, (void*)i, 0, NULL);
+            else {
+                fprintf(stderr, "attack=%d can take 1,2 or 3\n", attack);
+                usage();
+                exit(-1);
+            }
+
+            if (athr == NULL) {
+                fprintf(stderr, "tc %d\n", GetLastError());
+                exit (-1);
+            }
+            WaitForSingleObject(athr, INFINITE);
+            CloseHandle(athr);
+        }
+    }
+
+    return 0;
+}
+
+
+/*
+123:
+124:  void test(void(*f)(int),  int i)
+125:
+126:  {
+004012A0 55                   push        ebp
+004012A1 8B EC                mov         ebp,esp
+004012A3 83 EC 40             sub         esp,40h
+004012A6 53                   push        ebx
+004012A7 56                   push        esi
+004012A8 57                   push        edi
+004012A9 8D 7D C0             lea         edi,[ebp-40h]
+004012AC B9 10 00 00 00       mov         ecx,10h
+004012B1 B8 CC CC CC CC       mov         eax,0CCCCCCCCh
+004012B6 F3 AB                rep stos    dword ptr [edi] ****GET RID OF THIS CRAP ****
+127:      (f)(i);
+004012B8 8B F4                mov         esi,esp
+004012BA 8B 45 0C             mov         eax,dword ptr [ebp+0Ch]
+004012BD 50                   push        eax
+004012BE FF 55 08             call        dword ptr [ebp+8]
+004012C1 83 C4 04             add         esp,4
+004012C4 3B F4                cmp         esi,esp
+004012C6 E8 55 10 00 00       call        _chkesp (00402320) ***GET RID OF THIS B'COZ ABS ADDR ****
+128:      return;
+129:  }
+004012CB 5F                   pop         edi
+004012CC 5E                   pop         esi
+004012CD 5B                   pop         ebx
+004012CE 83 C4 40             add         esp,40h
+004012D1 3B EC                cmp         ebp,esp
+004012D3 E8 48 10 00 00       call        _chkesp (00402320) ***GET RID OF THIS B'COZ ABS ADDR ****
+004012D8 8B E5                mov         esp,ebp
+004012DA 5D                   pop         ebp
+004012DB C3                   ret
+
+
+*/
+unsigned char test_compiled_with_debug[] = {
+    0x55,
+    0x8B, 0xEC,
+    0x83, 0xEC, 0x40,
+    0x53,
+    0x56,
+    0x57,
+    0x8D, 0x7D, 0xC0,
+    0xB9, 0x10, 0x00, 0x00, 0x00,
+    0xB8, 0xCC, 0xCC, 0xCC, 0xCC,
+    /* 0xF3, 0xAB, */
+    0x8B, 0xF4,
+    0x8B, 0x45, 0x0C,
+    0x50,
+    0xFF, 0x55, 0x08,
+    0x83, 0xC4, 0x04,
+    0x3B, 0xF4,
+    /* 0xE8, 0xC5, 0x0E, 0x00, 0x00, */
+    0x5F,
+    0x5E,
+    0x5B,
+    0x83, 0xC4, 0x40,
+    0x3B, 0xEC,
+    /* 0xE8, 0xB8, 0x0E, 0x00, 0x00, */
+    0x8B, 0xE5,
+    0x5D,
+    0xC3,
+    0x00, 0x00, 0x00, 0x00 };
+
+        
+void test(void(*f)(int),  int i)
+{
+    (f)(i);
+    return;
+}
+
+/*
+123:
+124:  void test(void(*f)(int),  int i)
+125:
+126:  {
+004012A0 55                   push        ebp
+004012A1 8B EC                mov         ebp,esp
+127:      (f)(i);
+004012BA 8B 45 0C             mov         eax,dword ptr [ebp+0Ch]
+004012BD 50                   push        eax
+004012BE FF 55 08             call        dword ptr [ebp+8]
+128:      return;
+129:  }
+004012D8 8B E5                mov         esp,ebp
+004012DA 5D                   pop         ebp
+004012DB C3                   ret
+
+
+*/
+unsigned char sendfunc[] = {
+    0x55,
+    0x8B, 0xEC,
+    0x8B, 0x45, 0x0C,
+    0x50,
+    0xFF, 0x55, 0x08,
+    0x8B, 0xE5,
+    0x5D,
+    0xC3,
+    0x00, 0x00, 0x00, 0x00 };    
+
+
+void set_attack(int i) 
+{
+    attack_status = i;
+}
+
+
+void fool_opt_compiler(unsigned char * foo) 
+{
+    foo[0] =1;
+}
+
+DWORD 
+WINAPI
+stack_attack(LPVOID param)
+{
+    unsigned char myfunc[1024];
+    int i;
+
+    /* use the param to make compiler happy */
+    if (param == NULL)
+        i = 0;
+
+    if(sizeof(sendfunc)>=sizeof(myfunc))
+        MessageBox(NULL, L"ERROR", L"Memory allocation problem", MB_OK);
+
+    for(i=0; i<sizeof(sendfunc); i++)
+        myfunc[i] = sendfunc[i];
+    
+    ((void(*)(void(*)(int),int))((void *)myfunc))(set_attack, ATTACK_STACK);
+    fool_opt_compiler(myfunc);
+    return ERROR_SUCCESS;
+}
+
+
+DWORD 
+WINAPI
+heap_attack(LPVOID param)
+{
+    unsigned char  * myfunc;
+    int i;
+
+    /* use the param to make compiler happy */
+    if (param == NULL)
+        i = 0;
+
+    myfunc = (unsigned char *) malloc(sizeof(sendfunc));
+    
+    if(myfunc == NULL)
+        MessageBox(NULL, L"ERROR", L"Memory allocation problem", MB_OK);
+
+    for(i=0; i<sizeof(sendfunc); i++)
+        myfunc[i] = sendfunc[i];
+    
+    ((void(*)(void(*)(int),int))((void *)myfunc))(set_attack, ATTACK_HEAP);
+    free(myfunc);
+    return ERROR_SUCCESS;
+}
+
+DWORD 
+WINAPI
+ls_attack(LPVOID param)
+{
+    char output[MAX_PATH];
+    WCHAR filename[MAX_PATH];
+
+    _snwprintf(filename, MAX_PATH, L"%s.%d", OUTFILE, (int)param);
+    delete_file_rename_in_use(filename);
+
+    _snprintf(output, MAX_PATH,
+              "%d%d\n", hotp_test_reg(), hotp_test_control_flow());
+    write_file_contents(filename, output, TRUE);
+
+    return ERROR_SUCCESS;
+}
diff --git a/libutil/tests.c b/libutil/tests.c
new file mode 100644
index 000000000..779604604
--- /dev/null
+++ b/libutil/tests.c
@@ -0,0 +1,160 @@
+/* **********************************************************
+ * Copyright (c) 2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+
+/* ok this is more than a little shady...we'll just run through
+ *  assertions, so that if a test fails, we'll keep on trying
+ *  other ones.
+ * probably the only way to handle this better is to come up with
+ *  a way for running these tests in separate processes.
+ *  (or: make this driver c++ so we can use try/catch?) */
+#ifndef FAIL_ON_TEST_ASSERT
+#  define ASSERTION_EXPRESSION(msg) report_assertion(msg)
+#  define EXIT_ON_ASSERT FALSE
+#endif
+
+#include "share.h"
+#include "processes.h"
+
+#ifdef SINGLE_TEST
+#  define TESTNAME STRINGIFY(SINGLE_TEST)
+#  define RESTRICTED_BOOL TRUE
+#else
+#  define TESTNAME ""
+#  define RESTRICTED_BOOL FALSE
+#endif
+
+
+BOOL test_asserted;
+char *assert_message;
+
+void
+report_assertion(char *msg)
+{
+    if (!test_asserted)
+        assert_message = msg;
+    test_asserted = TRUE;
+}
+
+void
+display_failure(char *testname)
+{
+    WCHAR evt_filename[MAX_PATH], w_testname[MAX_PATH];
+    FILE *evtfile;
+
+    fflush(stdout);
+    fflush(stderr);
+    printf("  Test FAILURE: %s\n", assert_message);
+    _snwprintf(w_testname, MAX_PATH, L"%S", testname);
+    DO_ASSERT(get_unique_filename(L".", w_testname,
+                                  L".evtlog", evt_filename, MAX_PATH));
+    evtfile = _wfopen(evt_filename, L"w");
+    show_all_events(evtfile);
+    fclose(evtfile);
+    printf("  Events written to: %S\n", evt_filename);
+    fflush(stdout);
+}
+
+int
+main()
+{
+    WCHAR coredir[MAX_PATH];
+    WCHAR old_drhome[MAX_PATH];
+    WCHAR preinject[MAX_PATH];
+    WCHAR mp_cfg_file[MAX_PATH];
+    HANDLE *dummy = NULL; /* see LAUNCH_APP_WAIT_HANDLE macro */
+    BOOL restricted = RESTRICTED_BOOL;
+    int numtests = 0, passed = 0;
+    
+    set_debuglevel(DL_WARN);
+    set_abortlevel(DL_WARN);
+
+    _snwprintf(old_drhome, MAX_PATH, L"%s", get_dynamorio_home());
+
+    get_testdir(coredir, MAX_PATH);
+    CHECKED_OPERATION(setup_installation(coredir, TRUE));
+
+    _snwprintf(mp_cfg_file, MAX_PATH, L"%s\\conf\\mp-defs.cfg", 
+               get_dynamorio_home());
+    DO_ASSERT(file_exists(mp_cfg_file));
+
+    CHECKED_OPERATION(get_preinject_name(preinject, MAX_PATH));
+    DO_ASSERT(file_exists(preinject));
+
+    /* cleanup */
+    terminate_process_by_exe(L"tester_1.exe");
+    terminate_process_by_exe(L"tester_2.exe");
+
+    /* reset appinit to make sure it's set to the custom value */
+    CHECKED_OPERATION(disable_protection());
+    CHECKED_OPERATION(clear_policy());
+    CHECKED_OPERATION(enable_protection());
+
+    DO_ASSERT_WSTR_EQ(get_dynamorio_home(), coredir);
+
+#  define DO_TEST_HP(name, appstr, use_hotp, block)             \
+ {                                                              \
+     if (!restricted || 0 == strcmp(#name, TESTNAME)) {         \
+         test_asserted = FALSE;                                 \
+         numtests++;                                            \
+         printf("Executing " #name " test...\n");               \
+         clear_eventlog();                                      \
+         reset_last_event();                                    \
+         CHECKED_OPERATION(load_test_config(appstr, use_hotp)); \
+         block                                                  \
+         if (test_asserted) {                                   \
+             display_failure(#name);                            \
+         } else {                                               \
+             passed++;                                          \
+             printf("  Passed.\n");                             \
+         }                                                      \
+     }                                                          \
+ }
+
+#    include "tests.h"
+
+#  undef DO_TEST
+
+    CHECKED_OPERATION(disable_protection());
+    CHECKED_OPERATION(clear_policy());
+
+    /* restore original drhome */
+    CHECKED_OPERATION(setup_installation(old_drhome, TRUE));
+
+    printf("\nTest results: %s [%d/%d tests passed]\n", 
+           passed == numtests ? "PASS" : "FAIL", passed, numtests);
+
+    if (passed == numtests)
+        return 0;
+    else 
+        return -1;
+}
diff --git a/libutil/tests.h b/libutil/tests.h
new file mode 100644
index 000000000..fcde7e10b
--- /dev/null
+++ b/libutil/tests.h
@@ -0,0 +1,581 @@
+/* **********************************************************
+ * Copyright (c) 2005-2006 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+
+/* DR core / shared library interface unit tests */
+/* executed as part of the unit tests for processes.c */
+
+
+
+/* need to have a core build anyway for these tests to work */
+#  include "win32/events.h"
+
+
+/* nudge tends to hang/timeout if you nudge right after the 
+ *  process starts. so in order to let the basic tests pass, 
+ *  they all sleep for at least this long before trying to 
+ *  nudge.
+ * of course this should be fixed and then we should create
+ *  stress tests to address this specifically. */
+#define NUDGE_LET_PROCESS_START_WAIT 500
+
+#define TEST_TIMEOUT 2000
+#define LAUNCH_TIMEOUT 1000
+
+#define WAIT_FOR_APP(hProc) DO_ASSERT(WAIT_OBJECT_0 == WaitForSingleObject(hProc, TEST_TIMEOUT*2))
+ 
+#define LONG_WAIT_FOR_APP(hProc) DO_ASSERT(WAIT_OBJECT_0 == WaitForSingleObject(hProc, TEST_TIMEOUT*20))
+
+#define DO_TEST(name, appstr, block) DO_TEST_HP(name, appstr, TRUE, block)
+
+#define TESTER_1_BLOCK  "BEGIN_BLOCK\n" \
+                        "APP_NAME=tester_1.exe\n" \
+                        "DYNAMORIO_OPTIONS=-kill_thread -kill_thread_max 1000 -report_max 0 -dumpcore_mask 0xff\n" \
+                        "END_BLOCK\n"
+
+#define TESTER_2_BLOCK  "BEGIN_BLOCK\n" \
+                        "APP_NAME=tester_2.exe\n" \
+                        "DYNAMORIO_OPTIONS=-dumpcore_mask 0xff\n" \
+                        "END_BLOCK\n"
+
+#define TESTER_1_HOT_PATCH_BLOCK \
+        "BEGIN_BLOCK\n" \
+        "APP_NAME=tester_1.exe\n" \
+        "DYNAMORIO_OPTIONS=-kill_thread -dumpcore_mask 0xff\n" \
+        "DYNAMORIO_HOT_PATCH_MODES=\\conf\\test-modes.cfg\n" \
+        "BEGIN_MP_MODES\n" \
+        "1\n" \
+        "TEST.000A:2\n" \
+        "END_MP_MODES\n" \
+        "END_BLOCK\n"
+
+#define TESTER_1_HOT_PATCH_DETECT_BLOCK \
+        "BEGIN_BLOCK\n" \
+        "APP_NAME=tester_1.exe\n" \
+        "DYNAMORIO_OPTIONS=-kill_thread -dumpcore_mask 0xff\n" \
+        "DYNAMORIO_HOT_PATCH_MODES=\\conf\\test-modes.cfg\n" \
+        "BEGIN_MP_MODES\n" \
+        "1\n" \
+        "TEST.000A:1\n" \
+        "END_MP_MODES\n" \
+        "END_BLOCK\n"
+
+/* simple detach */
+DO_TEST(detach,
+        TESTER_1_BLOCK,
+        {
+            UINT pid;
+            LAUNCH_APP(L"tester_1.exe 5000", &pid);
+            /* FIXME: if this isn't here, the first run (right after
+             *  building the tests) always fails. */
+            Sleep(LAUNCH_TIMEOUT*2);
+            VERIFY_UNDER_DR(pid);
+            CHECKED_OPERATION(detach(pid, TRUE, DETACH_RECOMMENDED_TIMEOUT));
+            VERIFY_NOT_UNDER_DR(pid);
+            TERMINATE_PROCESS(pid);
+        }
+        );
+
+/* pending restart */
+DO_TEST(pending_restart,
+        TESTER_1_BLOCK,
+        {
+            UINT pid;
+            LAUNCH_APP(L"tester_1.exe 5000", &pid);
+            Sleep(LAUNCH_TIMEOUT);
+            VERIFY_UNDER_DR(pid);
+            CHECKED_OPERATION(detach(pid, TRUE, DETACH_RECOMMENDED_TIMEOUT));
+            VERIFY_NOT_UNDER_DR(pid);
+            DO_ASSERT(is_process_pending_restart(pid));
+            DO_ASSERT(is_any_process_pending_restart());
+            TERMINATE_PROCESS(pid);
+        }
+        );
+
+
+/* detach_exe */
+DO_TEST(detach_exe,
+        TESTER_1_BLOCK TESTER_2_BLOCK,
+        {
+            UINT pid1;
+            UINT pid2;
+            UINT pid3;
+            LAUNCH_APP(L"tester_1.exe", &pid1);
+            LAUNCH_APP(L"tester_2.exe", &pid2);
+            LAUNCH_APP(L"tester_1.exe", &pid3);
+            Sleep(LAUNCH_TIMEOUT);
+            VERIFY_UNDER_DR(pid1);
+            VERIFY_UNDER_DR(pid2);
+            VERIFY_UNDER_DR(pid3);
+            CHECKED_OPERATION(detach_exe(L"tester_1.exe", 
+                                         DETACH_RECOMMENDED_TIMEOUT));
+            VERIFY_NOT_UNDER_DR(pid1);
+            VERIFY_UNDER_DR(pid2);
+            VERIFY_NOT_UNDER_DR(pid3);
+            DO_ASSERT(is_process_pending_restart(pid1));
+            DO_ASSERT(!is_process_pending_restart(pid2));
+            DO_ASSERT(is_any_process_pending_restart());
+            TERMINATE_PROCESS(pid1);
+            TERMINATE_PROCESS(pid2);
+            TERMINATE_PROCESS(pid3);
+        }
+        );
+
+/* detach_all */
+DO_TEST(detach_all,
+        TESTER_1_BLOCK 
+        TESTER_2_BLOCK,
+        {
+            UINT pid1;
+            UINT pid2;
+            UINT pid3;
+            LAUNCH_APP(L"tester_1.exe", &pid1);
+            LAUNCH_APP(L"tester_2.exe", &pid2);
+            LAUNCH_APP(L"tester_1.exe", &pid3);
+            Sleep(LAUNCH_TIMEOUT);
+            VERIFY_UNDER_DR(pid1);
+            VERIFY_UNDER_DR(pid2);
+            VERIFY_UNDER_DR(pid3);
+            CHECKED_OPERATION(detach_all(DETACH_RECOMMENDED_TIMEOUT));
+            VERIFY_NOT_UNDER_DR(pid1);
+            VERIFY_NOT_UNDER_DR(pid2);
+            VERIFY_NOT_UNDER_DR(pid3);
+            DO_ASSERT(is_process_pending_restart(pid1));
+            DO_ASSERT(is_any_process_pending_restart());
+            TERMINATE_PROCESS(pid1);
+            TERMINATE_PROCESS(pid2);
+            TERMINATE_PROCESS(pid3);
+        }
+        );
+
+/* consistency_detach */
+DO_TEST(consistency_detach,
+        TESTER_1_BLOCK 
+        TESTER_2_BLOCK,
+        {
+            UINT pid1;
+            UINT pid2;
+            UINT pid3;
+            LAUNCH_APP(L"tester_1.exe", &pid1);
+            LAUNCH_APP(L"tester_2.exe", &pid2);
+            LAUNCH_APP(L"tester_1.exe", &pid3);
+            Sleep(LAUNCH_TIMEOUT);
+            VERIFY_UNDER_DR(pid1);
+            VERIFY_UNDER_DR(pid2);
+            VERIFY_UNDER_DR(pid3);
+            CHECKED_OPERATION(load_test_config(TESTER_2_BLOCK, FALSE));
+            CHECKED_OPERATION(consistency_detach(DETACH_RECOMMENDED_TIMEOUT));
+            VERIFY_NOT_UNDER_DR(pid1);
+            VERIFY_UNDER_DR(pid2);
+            VERIFY_NOT_UNDER_DR(pid3);
+            TERMINATE_PROCESS(pid1);
+            TERMINATE_PROCESS(pid2);
+            TERMINATE_PROCESS(pid3);
+        }
+        );
+
+/* check start/stop events */
+DO_TEST(start_stop_event,
+        TESTER_1_BLOCK,
+        {
+            UINT pid;
+            HANDLE hProc;
+            LAUNCH_APP_HANDLE(L"tester_1.exe 10", &pid, &hProc);
+            WAIT_FOR_APP(hProc);
+            DO_ASSERT(check_for_event(MSG_INFO_PROCESS_START,
+                                      L"tester_1.exe",
+                                      pid, NULL, NULL, 0));
+            DO_ASSERT(check_for_event(MSG_INFO_PROCESS_STOP,
+                                      L"tester_1.exe",
+                                      pid, NULL, NULL, 0));
+        }
+        );
+
+/* check detach event */
+DO_TEST(detach_event,
+        TESTER_1_BLOCK,
+        {
+            UINT pid;
+            LAUNCH_APP(L"tester_1.exe 2000", &pid);
+            Sleep(LAUNCH_TIMEOUT);
+            VERIFY_UNDER_DR(pid);
+            CHECKED_OPERATION(detach(pid, TRUE, DETACH_RECOMMENDED_TIMEOUT));
+            VERIFY_NOT_UNDER_DR(pid);
+            Sleep(100);
+            DO_ASSERT(check_for_event(MSG_INFO_PROCESS_START,
+                                      L"tester_1.exe",
+                                      pid, NULL, NULL, 0));
+            DO_ASSERT(check_for_event(MSG_INFO_DETACHING,
+                                      L"tester_1.exe",
+                                      pid, NULL, NULL, 0));
+            TERMINATE_PROCESS(pid);
+        }
+        );
+
+/* check attack event */
+DO_TEST(attack_event,
+        TESTER_1_BLOCK,
+        {
+            UINT pid;
+            HANDLE hProc;
+            WCHAR s3[MAX_PATH];
+            WCHAR s4[MAX_PATH];
+            LAUNCH_APP_HANDLE(L"tester_1.exe 10 10 1", &pid, &hProc);
+            WAIT_FOR_APP(hProc);
+            DO_ASSERT(check_for_event(MSG_SEC_VIOLATION_THREAD,
+                                      L"tester_1.exe",
+                                      pid, s3, s4, MAX_PATH));
+            /* make sure threat id looks ok */
+            DO_ASSERT(11 == wcslen(s3));
+        }
+        );
+
+/* check forensics event/file */
+DO_TEST(forensics_file,
+        TESTER_1_BLOCK,
+        {
+            UINT pid;
+            HANDLE hProc;
+            WCHAR s3[MAX_PATH];
+            WCHAR s4[MAX_PATH];
+            LAUNCH_APP_HANDLE(L"tester_1.exe 10 10 1", &pid, &hProc);
+            WAIT_FOR_APP(hProc);
+            DO_ASSERT(check_for_event(MSG_SEC_VIOLATION_THREAD,
+                                      L"tester_1.exe",
+                                      pid, NULL, NULL, 0));
+            DO_ASSERT(check_for_event(MSG_SEC_FORENSICS,
+                                      L"tester_1.exe",
+                                      pid, s3, s4, MAX_PATH));
+            DO_ASSERT(file_exists(s3));
+        }
+        );
+
+/* stress forensics event/file */
+DO_TEST(forensics_stress,
+        TESTER_1_BLOCK,
+        {
+            UINT pid;
+            HANDLE hProc;
+            WCHAR s3[MAX_PATH];
+            WCHAR s4[MAX_PATH];
+            int i;
+            LAUNCH_APP_HANDLE(L"tester_1.exe 10 10 0 100", &pid, &hProc);
+            LONG_WAIT_FOR_APP(hProc);
+            for (i=0; i<100; i++) {
+                DO_ASSERT(check_for_event(MSG_SEC_VIOLATION_THREAD,
+                                          L"tester_1.exe",
+                                          pid, NULL, NULL, 0));
+                DO_ASSERT(check_for_event(MSG_SEC_FORENSICS,
+                                          L"tester_1.exe",
+                                          pid, s3, s4, MAX_PATH));
+                DO_ASSERT(file_exists(s3));
+            }
+            DO_ASSERT(check_for_event(MSG_INFO_PROCESS_STOP,
+                                      L"tester_1.exe",
+                                      pid, NULL, NULL, 0));
+        }
+        );
+
+/* check to make sure nudge doesn't leave the code lying around */
+DO_TEST(check_nudge,
+        TESTER_1_BLOCK,
+        {
+            UINT pid;
+            WCHAR nudge_code_buf[MAX_PATH];
+
+            LAUNCH_APP(L"tester_1.exe", &pid);
+            Sleep(LAUNCH_TIMEOUT);
+            VERIFY_UNDER_DR(pid);
+            Sleep(NUDGE_LET_PROCESS_START_WAIT);
+            CHECKED_OPERATION(hotp_notify_modes_update(pid, TRUE, TEST_TIMEOUT));
+            DO_ASSERT(ERROR_SUCCESS != 
+                      get_config_parameter(L_PRODUCT_NAME, 
+                                           FALSE,
+                                           L_DYNAMORIO_VAR_NUDGE,
+                                           nudge_code_buf, 
+                                           MAX_PATH));
+            TERMINATE_PROCESS(pid);
+        }
+        );
+
+/* simple test for hotpatching */
+DO_TEST(hotp_protect,
+        TESTER_1_HOT_PATCH_BLOCK,
+        {
+            UINT pid;
+            char fc[MAX_PATH];
+            LAUNCH_APP_AND_WAIT(L"tester_1.exe 100", &pid);
+            CHECKED_OPERATION(read_file_contents(L"tester.out", 
+                                                 fc, MAX_PATH, NULL));
+            DO_ASSERT(0 == strncmp(fc, "10", 2));
+        }
+        );
+
+/* detect should report 00 */
+DO_TEST(hotp_detect,
+        TESTER_1_HOT_PATCH_DETECT_BLOCK,
+        {
+            UINT pid;
+            char fc[MAX_PATH];
+            LAUNCH_APP_AND_WAIT(L"tester_1.exe 100", &pid);
+            CHECKED_OPERATION(read_file_contents(L"tester.out", 
+                                                 fc, MAX_PATH, NULL));
+            DO_ASSERT(0 == strncmp(fc, "00", 2));
+        }
+        );
+
+/* basic nudge test */
+DO_TEST_HP(hotp_defs_nudge,
+        TESTER_1_BLOCK,
+        FALSE, /* don't load the modes file! */
+        {
+            UINT pid;
+            char fc[MAX_PATH];
+            HANDLE hProc;
+            hotp_policy_status_table_t *status_table = NULL;
+            
+            /* first launch app w/o hotpatch */
+            LAUNCH_APP_AND_WAIT(L"tester_1.exe 10", &pid);
+            CHECKED_OPERATION(read_file_contents(L"tester.out", 
+                                                 fc, MAX_PATH, NULL));
+            DO_ASSERT(0 == strncmp(fc, "00", 2));
+
+            /* now, same thing with longer wait */
+            LAUNCH_APP_HANDLE(L"tester_1.exe 2000", &pid, &hProc);
+            Sleep(LAUNCH_TIMEOUT);
+            VERIFY_UNDER_DR(pid);
+            
+            /* make sure nothing's there */
+            DO_ASSERT(ERROR_DRMARKER_ERROR == 
+                      get_hotp_status(pid, &status_table));
+
+            /* load the new config -- this time with hot patching*/
+            CHECKED_OPERATION(load_test_config(TESTER_1_HOT_PATCH_BLOCK, TRUE));
+
+            /* and nudge */
+            Sleep(NUDGE_LET_PROCESS_START_WAIT);
+            CHECKED_OPERATION(hotp_notify_defs_update(pid, TRUE, TEST_TIMEOUT));
+            VERIFY_UNDER_DR(pid);
+            WAIT_FOR_APP(hProc);
+
+            CHECKED_OPERATION(read_file_contents(L"tester.out", 
+                                                 fc, MAX_PATH, NULL));
+            DO_ASSERT(0 == strncmp(fc, "10", 2));
+        }
+        );
+
+DO_TEST(hotp_modes_nudge,
+        TESTER_1_HOT_PATCH_BLOCK,
+        {
+            UINT pid;
+            char fc[MAX_PATH];
+            HANDLE hProc;
+
+            /* first launch app w/hotpatch protect */
+            LAUNCH_APP_AND_WAIT(L"tester_1.exe 10", &pid);
+            CHECKED_OPERATION(read_file_contents(L"tester.out", 
+                                                 fc, MAX_PATH, NULL));
+            DO_ASSERT(0 == strncmp(fc, "10", 2));
+
+            /* now, same thing with longer wait */
+            LAUNCH_APP_HANDLE(L"tester_1.exe 2000", &pid, &hProc);
+            Sleep(LAUNCH_TIMEOUT);
+            VERIFY_UNDER_DR(pid);
+
+            /* load the new config */
+            CHECKED_OPERATION(load_test_config(TESTER_1_HOT_PATCH_DETECT_BLOCK, TRUE));
+
+            /* and do a modes nudge */
+            Sleep(NUDGE_LET_PROCESS_START_WAIT);
+            CHECKED_OPERATION(hotp_notify_modes_update(pid, TRUE, TEST_TIMEOUT));
+            VERIFY_UNDER_DR(pid);
+            WAIT_FOR_APP(hProc);
+
+            CHECKED_OPERATION(read_file_contents(L"tester.out", 
+                                                 fc, MAX_PATH, NULL));
+            DO_ASSERT(0 == strncmp(fc, "00", 2));
+        }
+        );
+
+/* nudge twice to make sure we don't die */
+DO_TEST_HP(hotp_nudge_twice,
+        TESTER_1_BLOCK,
+        FALSE, 
+        {
+            UINT pid;
+            char fc[MAX_PATH];
+            HANDLE hProc;
+
+            /* first launch app w/o hotpatch */
+            LAUNCH_APP_AND_WAIT(L"tester_1.exe 10", &pid);
+            CHECKED_OPERATION(read_file_contents(L"tester.out", 
+                                                 fc, MAX_PATH, NULL));
+            DO_ASSERT(0 == strncmp(fc, "00", 2));
+
+            LAUNCH_APP_HANDLE(L"tester_1.exe 2000", &pid, &hProc);
+            Sleep(LAUNCH_TIMEOUT);
+            VERIFY_UNDER_DR(pid);
+
+            /* load the new config */
+            CHECKED_OPERATION(load_test_config(TESTER_1_HOT_PATCH_BLOCK, TRUE));
+            Sleep(NUDGE_LET_PROCESS_START_WAIT);
+            CHECKED_OPERATION(hotp_notify_defs_update(pid, TRUE, TEST_TIMEOUT));
+            VERIFY_UNDER_DR(pid);
+
+            /* load the old config back */
+            CHECKED_OPERATION(load_test_config(TESTER_1_HOT_PATCH_DETECT_BLOCK, TRUE));
+            CHECKED_OPERATION(hotp_notify_defs_update(pid, TRUE, TEST_TIMEOUT));
+            VERIFY_UNDER_DR(pid);
+            WAIT_FOR_APP(hProc);
+
+            CHECKED_OPERATION(read_file_contents(L"tester.out", 
+                                                 fc, MAX_PATH, NULL));
+            DO_ASSERT(0 == strncmp(fc, "00", 2));
+        }
+        );
+
+/* tests state of the patches */
+DO_TEST(hotp_protect_status,
+        TESTER_1_HOT_PATCH_BLOCK,
+        {
+            UINT pid;
+            HANDLE hProc;
+            hotp_policy_status_table_t *status_table = NULL;
+            
+            LAUNCH_APP_HANDLE(L"tester_1.exe 10 2500", &pid, &hProc);
+            Sleep(LAUNCH_TIMEOUT);
+            VERIFY_UNDER_DR(pid);
+            Sleep(500);
+            CHECKED_OPERATION(get_hotp_status(pid, &status_table));
+            DO_ASSERT(status_table != NULL && status_table->num_policies > 0);
+            if (status_table != NULL && status_table->num_policies > 0) {
+                UINT j;
+                for (j = 0; j < status_table->num_policies; j++) {
+                    if (0 == strcmp(status_table->policy_status_array[j].policy_id, "TEST.000A") ||
+                        0 == strcmp(status_table->policy_status_array[j].policy_id, "TEST.000B")) {
+                        DO_ASSERT(status_table->policy_status_array[j].inject_status == HOTP_INJECT_PROTECT);
+                    }
+                }
+                free_hotp_status_table(status_table);
+            }
+            WAIT_FOR_APP(hProc);
+        }
+        );
+
+/* tests state of the patches */
+DO_TEST(hotp_pending_status,
+        TESTER_1_HOT_PATCH_BLOCK,
+        {
+            UINT pid;
+            HANDLE hProc;
+            hotp_policy_status_table_t *status_table = NULL;
+            
+            LAUNCH_APP_HANDLE(L"tester_1.exe 2500", &pid, &hProc);
+            Sleep(LAUNCH_TIMEOUT);
+            VERIFY_UNDER_DR(pid);
+            Sleep(200);
+            CHECKED_OPERATION(get_hotp_status(pid, &status_table));
+            DO_ASSERT(status_table != NULL && status_table->num_policies > 0);
+            if (status_table != NULL && status_table->num_policies > 0) {
+                UINT j;
+                for (j = 0; j < status_table->num_policies; j++) {
+                    if (0 == strcmp(status_table->policy_status_array[j].policy_id, "TEST.000A") ||
+                        0 == strcmp(status_table->policy_status_array[j].policy_id, "TEST.000B")) {
+                        DO_ASSERT(status_table->policy_status_array[j].inject_status == HOTP_INJECT_PENDING);
+                    }
+                }
+                free_hotp_status_table(status_table);
+            }
+            WAIT_FOR_APP(hProc);
+        }
+        );
+
+/* sanity check to make sure we have different state */
+DO_TEST(hotp_detect_status,
+        TESTER_1_HOT_PATCH_DETECT_BLOCK,
+        {
+            UINT pid;
+            HANDLE hProc;
+            hotp_policy_status_table_t *status_table = NULL;
+            
+            LAUNCH_APP_HANDLE(L"tester_1.exe 10 2500", &pid, &hProc);
+            Sleep(LAUNCH_TIMEOUT);
+            VERIFY_UNDER_DR(pid);
+            Sleep(500);
+            CHECKED_OPERATION(get_hotp_status(pid, &status_table));
+            DO_ASSERT(status_table != NULL && status_table->num_policies > 0);
+            if (status_table != NULL && status_table->num_policies > 0) {
+                UINT j;
+                for (j = 0; j < status_table->num_policies; j++) {
+                    if (0 == strcmp(status_table->policy_status_array[j].policy_id, "TEST.000A") ||
+                        0 == strcmp(status_table->policy_status_array[j].policy_id, "TEST.000B")) {
+                        DO_ASSERT(status_table->policy_status_array[j].inject_status == HOTP_INJECT_DETECT);
+                    }
+                }
+                free_hotp_status_table(status_table);
+            }
+            WAIT_FOR_APP(hProc);
+        }
+        );
+
+DO_TEST(hotp_modes_nudge_all,
+        TESTER_1_HOT_PATCH_BLOCK,
+        {
+            UINT pid;
+            char fc[MAX_PATH];
+            HANDLE hProc;
+
+            /* first launch app w/hotpatch protect */
+            LAUNCH_APP_AND_WAIT(L"tester_1.exe 10", &pid);
+            CHECKED_OPERATION(read_file_contents(L"tester.out", 
+                                                 fc, MAX_PATH, NULL));
+            DO_ASSERT(0 == strncmp(fc, "10", 2));
+
+            /* now, same thing with longer wait */
+            LAUNCH_APP_HANDLE(L"tester_1.exe 2000", &pid, &hProc);
+            Sleep(LAUNCH_TIMEOUT);
+            VERIFY_UNDER_DR(pid);
+
+            /* load the new config */
+            CHECKED_OPERATION(load_test_config(TESTER_1_HOT_PATCH_DETECT_BLOCK, TRUE));
+
+            /* and do a modes nudge */
+            Sleep(NUDGE_LET_PROCESS_START_WAIT);
+            CHECKED_OPERATION(hotp_notify_all_modes_update(TEST_TIMEOUT));
+            VERIFY_UNDER_DR(pid);
+            WAIT_FOR_APP(hProc);
+            
+            CHECKED_OPERATION(read_file_contents(L"tester.out", 
+                                                 fc, MAX_PATH, NULL));
+            DO_ASSERT(0 == strncmp(fc, "00", 2));
+        }
+        );
diff --git a/libutil/utils.c b/libutil/utils.c
new file mode 100644
index 000000000..fa58dadd4
--- /dev/null
+++ b/libutil/utils.c
@@ -0,0 +1,2263 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+
+#include "share.h"
+#include "config.h"
+#include "elm.h"
+#include "events.h" /* for canary */
+#include "processes.h" /* for canary */
+
+#include <stdio.h>
+#include <io.h> /* for canary */
+#include <Fcntl.h> /* for canary */
+
+#include <aclapi.h>
+
+#ifndef UNIT_TEST
+
+
+# ifdef _DEBUG
+
+int debuglevel = DL_FATAL;
+int abortlevel = DL_FATAL;
+
+void
+set_debuglevel(int level)
+{
+    debuglevel = level;
+}
+
+void
+set_abortlevel(int level)
+{
+    abortlevel = level;
+}
+
+#define CONFIG_MAX 8192
+
+#define HEADER_SNIPPET(defsfile) \
+                       "POLICY_VERSION=30000\n"                             \
+                       "BEGIN_BLOCK\n"                                      \
+                       "GLOBAL\n"                                           \
+                       "DYNAMORIO_OPTIONS=\n"                               \
+                       "DYNAMORIO_RUNUNDER=1\n"                             \
+                       "DYNAMORIO_AUTOINJECT=\\lib\\dynamorio.dll\n"        \
+                       "DYNAMORIO_HOT_PATCH_POLICIES=" defsfile "\n"        \
+                       "DYNAMORIO_UNSUPPORTED=\n"                           \
+                       "END_BLOCK\n"
+
+DWORD
+load_test_config(const char *snippet, BOOL use_hotpatch_defs)
+{
+    char buf[CONFIG_MAX];
+    _snprintf(buf, CONFIG_MAX, "%s%s", 
+              use_hotpatch_defs ? 
+              HEADER_SNIPPET("\\conf") : HEADER_SNIPPET(""), 
+              snippet);
+    NULL_TERMINATE_BUFFER(buf);
+    DO_ASSERT(strlen(buf) < CONFIG_MAX - 2);
+    DO_DEBUG(DL_VERB,
+             printf("importing %s\n", buf);
+             );
+    CHECKED_OPERATION(policy_import(buf, FALSE, NULL, NULL));
+    return ERROR_SUCCESS;
+}
+
+void
+get_testdir(WCHAR *buf, UINT maxchars)
+{
+    WCHAR *filePart;
+    WCHAR tmp[MAX_PATH];
+    DWORD len;
+
+    len = GetEnvironmentVariable(L"DYNAMORIO_WINDIR", 
+                                 tmp, maxchars);
+    if (len == 0) {
+        len = GetEnvironmentVariable(L_DYNAMORIO_VAR_HOME, 
+                                     tmp, maxchars);
+        /* check for cygwin paths on windows */
+        if (!file_exists(buf))
+            len = 0;
+
+        DO_DEBUG(DL_INFO,
+                 printf("ignoring invalid-looking DYNAMORIO_HOME=%S\n", tmp);
+                 );
+    }
+    
+    if (len == 0) {
+        wcsncpy(tmp, L"..", MAX_PATH);
+    }
+
+    len = GetFullPathName(tmp, maxchars, buf, &filePart);
+
+    DO_DEBUG(DL_INFO,
+             printf("using drhome: %S\n", buf);
+             );
+
+    DO_ASSERT(len != 0);
+
+    return;
+}
+
+void 
+error_cb(unsigned int errcode, WCHAR *message)
+{
+    if (errcode || !errcode || message) {
+        DO_ASSERT(0);
+    }
+}
+
+extern BOOL do_once;
+
+typedef struct evthelp__ {
+    DWORD type;
+    WCHAR *exename;
+    ULONG pid;
+    WCHAR *s3;
+    WCHAR *s4;
+    UINT maxchars;
+    BOOL found;
+} evthelp;
+
+evthelp *cb_eh = NULL;
+int last_record = -1;
+
+void 
+check_event_cb(EVENTLOGRECORD *record)
+{
+    const WCHAR *strings;
+
+    if (cb_eh->found)
+        return;
+
+    last_record = record->RecordNumber;
+
+    if (record->EventID == cb_eh->type) {
+
+        if (cb_eh->exename != NULL && 
+            0 != wcscmp(get_event_exename(record), cb_eh->exename))
+            return;
+
+        if (cb_eh->pid != 0 && cb_eh->pid != get_event_pid(record))
+            return;
+
+        strings = get_message_strings(record);
+        strings = next_message_string(strings);
+        strings = next_message_string(strings);
+        if (cb_eh->s3 != NULL) {
+            cb_eh->s3[0] = L'\0';
+            if (strings != NULL) {
+                wcsncpy(cb_eh->s3, strings, cb_eh->maxchars);
+                cb_eh->s3[cb_eh->maxchars-1] = L'\0';
+            }
+        }
+        if (cb_eh->s4 != NULL) {
+            cb_eh->s4[0] = L'\0';
+            strings = next_message_string(strings);
+            if (strings != NULL) {
+                wcsncpy(cb_eh->s4, strings, cb_eh->maxchars);
+                cb_eh->s4[cb_eh->maxchars-1] = L'\0';
+            }
+        }
+        cb_eh->found = TRUE;
+    }
+}
+
+void
+reset_last_event()
+{
+    last_record = -1;
+}
+
+/* checks for events matching type, exename (if not null), and pid (if
+ *  not 0). fills in s3 and s4 with 3rd and 4th message strings of
+ *  match, if not null. 
+ * next search will start with event after matched event. */
+BOOL
+check_for_event(DWORD type, WCHAR *exename, ULONG pid, 
+                WCHAR *s3, WCHAR *s4, UINT maxchars)
+{
+    evthelp eh;
+
+    eh.type = type;
+    eh.exename = exename;
+    eh.pid = pid;
+    eh.s3 = s3;
+    eh.s4 = s4;
+    eh.maxchars = maxchars;
+    eh.found = FALSE;
+
+    cb_eh = &eh;
+
+
+    /* backdoor */
+    do_once = TRUE;
+    
+    CHECKED_OPERATION(start_eventlog_monitor(FALSE, NULL, check_event_cb, 
+                                             error_cb, last_record));
+    DO_ASSERT(WAIT_OBJECT_0 == 
+              WaitForSingleObject(get_eventlog_monitor_thread_handle(), 
+                                  10000));
+    stop_eventlog_monitor();
+
+    return eh.found;    
+}
+
+FILE *event_list_fp;
+
+void 
+show_event_cb(unsigned int mID, 
+              unsigned int type,
+              WCHAR *message,
+              DWORD timestamp)
+{
+    /* fool the compiler */
+    DO_ASSERT(type == 0 || type != 0 || timestamp == 0);
+    fprintf(event_list_fp, " Event %d: %S\n", mID, message);
+}
+
+void
+show_all_events(FILE *fp)
+{
+    DO_ASSERT (fp != NULL);
+
+    event_list_fp = fp;
+    /* backdoor */
+    do_once = TRUE;
+    CHECKED_OPERATION(start_eventlog_monitor(TRUE, show_event_cb, NULL,
+                                             error_cb, (DWORD) -1));
+    DO_ASSERT(WAIT_OBJECT_0 == 
+              WaitForSingleObject(get_eventlog_monitor_thread_handle(), 
+                                  10000));
+    stop_eventlog_monitor();
+
+    return;
+}
+
+# endif
+
+
+# include "options.h" //from src module
+
+
+void
+wcstolower(WCHAR *str)
+{
+    UINT i;
+    for (i=0; i < wcslen(str); i++)
+        str[i] = towlower(str[i]);
+}
+
+WCHAR *
+get_exename_from_path(const WCHAR *path)
+{
+    WCHAR *name = wcsrchr(path, L'\\');
+    if (name == NULL)
+        name = (WCHAR *) path;
+    else
+        name += 1;
+    return name;
+}
+
+/* returns the exe without the path info. the buf must be tolower()ed. 
+ *  in len the length of just the exe is stored (eg, w/o cmdline) */
+WCHAR *
+get_exeptr(const WCHAR *buf, int *exenamelen)
+{
+    WCHAR *head, *target = wcsstr(buf, L_EXE_SUFFIX);
+
+    if (target == NULL)
+        return NULL;
+    
+    head = (WCHAR *)buf;
+    while (wcschr(head, L'\\') && wcschr(head, L'\\') < target)
+        head = wcschr(head, L'\\') + 1;
+    *exenamelen = target - head + 4;
+    return head;
+}
+
+DWORD
+acquire_shutdown_privilege()
+{
+    HANDLE hToken = NULL;
+    TOKEN_PRIVILEGES Priv;
+
+    // get current thread token 
+    if  (!OpenThreadToken(GetCurrentThread(), 
+                          TOKEN_QUERY|TOKEN_ADJUST_PRIVILEGES, 
+                          FALSE, 
+                          &hToken)) {
+        // can't get thread token, try process token instead
+        if(!OpenProcessToken(GetCurrentProcess(), 
+                             TOKEN_QUERY|TOKEN_ADJUST_PRIVILEGES, 
+                             &hToken)) {
+            return GetLastError();
+        }
+    }
+
+    Priv.PrivilegeCount = 1;
+    Priv.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
+    LookupPrivilegeValue(NULL, SE_SHUTDOWN_NAME, &Priv.Privileges[0].Luid);
+    // try to enable the privilege
+    if(!AdjustTokenPrivileges(hToken, FALSE, &Priv, sizeof(Priv), NULL, 0))
+        return GetLastError();
+    
+    return ERROR_SUCCESS;
+}
+
+/*
+ * FIXME: shutdown reason. we should probably use this, BUT 
+ *  InitiateSystemShutdownEx is not included in VS6.0, so we'll have
+ *  to dynamically link it in. 
+ *
+ * from msdn:
+ *  SHTDN_REASON_FLAG_PLANNED: The shutdown was planned. On Windows .NET
+ *   Server, the system generates a state snapshot. For more information,
+ *   see the help for Shutdown Event Tracker.
+ *
+ *  various combinations of major/minor flags are "recognized by the 
+ *   system"; the APPLICATION|RECONFIG is NOT one of these. However,
+ *   "You can also define your own shutdown reasons and add them to the
+ *    the registry." we should probably do this at installation, once
+ *    we're happy we've got a good reason code.
+ */
+DWORD
+reboot_system()
+{
+    DWORD res;
+
+    res = acquire_shutdown_privilege();
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    /* do we need to harden this at all?
+     * "If the the system is not ready to handle the request, the last 
+     *  error code is ERROR_NOT_READY. The application should wait a 
+     *  short while and retry the call." 
+     * also ERROR_MACHINE_LOCKED, ERROR_SHUTDOWN_IN_PROGRESS, etc. */
+
+    res = InitiateSystemShutdown(NULL, L"A System Restart was requested.", 
+                                 30, TRUE, TRUE);
+    //                                SHTDN_REASON_MAJOR_APPLICATION | 
+    //                                SHTDN_REASON_MINOR_RECONFIG);
+
+    return res;
+}
+
+
+#define LAST_WCHAR(wstr) wstr[wcslen(wstr) - 1]
+
+/* this sucks.
+ * i can't believe this is best way to implement this in Win32...
+ *  but i can't seem to find a better way. 
+ * msdn suggests using CreateFile() with CREATE_NEW or OPEN_EXISTING,
+ *   and then checking error codes; but the problem there is that C:\\
+ *   returns PATH_NOT_FOUND regardless. */
+BOOL
+file_exists(const WCHAR *fn)
+{
+    WIN32_FIND_DATA fd;
+    HANDLE search;
+
+    DO_ASSERT(fn != NULL);
+
+    search = FindFirstFile(fn, &fd);
+    
+    if (search == INVALID_HANDLE_VALUE) {
+
+        /* special handling for e.g. C:\\ */
+        if (LAST_WCHAR(fn) == L'\\' || LAST_WCHAR(fn) == L':') {
+            WCHAR buf[MAX_PATH];
+            _snwprintf(buf, MAX_PATH, L"%S%S*", 
+                       fn, LAST_WCHAR(fn) == L'\\' ? L"" : L"\\");
+            NULL_TERMINATE_BUFFER(buf);
+            search = FindFirstFile(buf, &fd);
+            if (search != INVALID_HANDLE_VALUE) {
+                FindClose(search);
+                return TRUE;
+            } else {
+                DO_DEBUG(DL_VERB,
+                         printf("%S: even though we tried hard, %d\n", 
+                                buf, GetLastError());
+                         );
+            }
+        }
+
+        DO_DEBUG(DL_VERB,
+                 printf("%S doesn't exist because of: %d\n", 
+                        fn, GetLastError());
+                 );
+        return FALSE;
+    }
+    else {
+        FindClose(search);
+        return TRUE;
+    }
+} 
+
+#define MAX_COUNTER 999999
+
+/* grokked from the core. 
+ * FIXME: shareme!
+ * if NULL is passed for directory, then it is ignored and no directory
+ *  check is done, and filename_base is assumed to be absolute.
+ * TODO: make this a proactive check: make sure the file can be
+ *  opened, eg, do a create/delete on the filename to be returned.
+ */
+BOOL
+get_unique_filename(const WCHAR* directory,
+                    const WCHAR* filename_base,
+                    const WCHAR* file_type,
+                    WCHAR* filename_buffer, 
+                    UINT maxlen)
+{
+    UINT counter = 0;
+    
+    if (directory != NULL && !file_exists(directory))
+        return FALSE;
+
+    do {
+        if (directory == NULL)
+            _snwprintf(filename_buffer, maxlen, L"%s.%.8d%s", 
+                       filename_base, counter, file_type);
+        else 
+            _snwprintf(filename_buffer, maxlen, L"%s\\%s.%.8d%s", 
+                       directory, filename_base, counter, file_type);
+        filename_buffer[maxlen-1] = L'\0';
+    }
+    while (file_exists(filename_buffer) && (++counter < MAX_COUNTER));
+
+    return (counter < MAX_COUNTER);
+}
+
+DWORD
+delete_file_on_boot(WCHAR *filename)
+{
+    DWORD res;
+    bool success = 
+        MoveFileEx(filename, NULL, MOVEFILE_DELAY_UNTIL_REBOOT);
+
+    /* reboot removal adds an entry to
+     * HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Session Manager\PendingFileRenameOperations
+     * and smss.exe will delete the file on next boot
+     */
+    if (success)
+        res = ERROR_SUCCESS;
+    else
+        res = GetLastError();
+    return res;
+}
+
+DWORD
+delete_file_rename_in_use(WCHAR *filename)
+{
+    DWORD res;
+    BOOL success = DeleteFile(filename);
+
+    if (success)
+        return ERROR_SUCCESS;
+
+    /* xref case 4512: if we leave a dll in a process after we're done
+     *  using it, we won't be able to delete it; however, hopefully 
+     *  we can rename it so there won't be issues replacing it later. */
+    res = GetLastError();
+    if (res != ERROR_SUCCESS) {
+        WCHAR tempname[MAX_PATH];
+        if (get_unique_filename(NULL, filename, L".tmp", tempname, MAX_PATH)) {
+            success = MoveFile(filename, tempname);
+            if (success) {
+                res = ERROR_SUCCESS;
+
+                /* as best effort, we also schedule cleanup of the
+                 * temporary file on next boot */
+                delete_file_on_boot(tempname);
+            } else
+                res = GetLastError();
+        }
+    }
+
+    return res;
+}
+
+#ifndef PROTECTED_DACL_SECURITY_INFORMATION
+# define PROTECTED_DACL_SECURITY_INFORMATION (0x80000000L)
+#endif
+
+/*
+ * quick permissions xfer workaround for updating permissions 
+ *  on upgrade.
+ */
+DWORD
+copy_file_permissions(WCHAR *filedst, WCHAR *filesrc)
+{
+    DWORD res = ERROR_SUCCESS;
+    SECURITY_DESCRIPTOR *sd  = NULL;
+    ACL *dacl = NULL;
+    res = GetNamedSecurityInfo(filesrc, SE_FILE_OBJECT, 
+                               DACL_SECURITY_INFORMATION,
+                               NULL, NULL, &dacl, NULL, &sd);
+
+    if (res != ERROR_SUCCESS)
+        return res;
+
+    res = SetNamedSecurityInfo(filedst, SE_FILE_OBJECT,
+                               DACL_SECURITY_INFORMATION |
+                               PROTECTED_DACL_SECURITY_INFORMATION,
+                               NULL, NULL, dacl, NULL);
+
+    LocalFree(sd);
+
+    return res;
+}
+
+
+/* NOTE: for now we only consider the major/minor versions and 
+ *  platform id. 
+ * 
+ * the osinfo.szCSDVersion string contains service pack information,
+ *  which could be used to distinguish e.g. XPSP2, 2K3SP1, if 
+ *  necessary.
+ */
+
+DWORD
+get_platform(DWORD *platform)
+{
+    /* determine the OS version information */
+    OSVERSIONINFO osinfo = { sizeof(OSVERSIONINFO) };
+    if (GetVersionEx(&osinfo)) {
+
+        DO_DEBUG(DL_VERB,
+                 WCHAR verbuf[MAX_PATH];
+                 _snwprintf(verbuf, MAX_PATH, 
+                            L"Major=%d, Minor=%d, Build=%d, SPinfo=%s",
+                            osinfo.dwMajorVersion, osinfo.dwMinorVersion,
+                            osinfo.dwBuildNumber, osinfo.szCSDVersion);
+                 NULL_TERMINATE_BUFFER(verbuf);
+                 printf("%S\n", verbuf);
+                 );
+
+        if (osinfo.dwPlatformId != VER_PLATFORM_WIN32_NT)
+            return ERROR_UNSUPPORTED_OS;
+
+        if (osinfo.dwMajorVersion == 4) {
+            if (osinfo.dwMinorVersion == 0) {
+                *platform = PLATFORM_WIN_NT_4;
+                return ERROR_SUCCESS;
+            }
+        }
+        else if (osinfo.dwMajorVersion == 5) {
+            if (osinfo.dwMinorVersion == 0) {
+                *platform = PLATFORM_WIN_2000;
+                return ERROR_SUCCESS;
+            } 
+            else if (osinfo.dwMinorVersion == 1) {
+                *platform = PLATFORM_WIN_XP;
+                return ERROR_SUCCESS;
+            } 
+            else if (osinfo.dwMinorVersion == 2) {
+                *platform = PLATFORM_WIN_2003;
+                return ERROR_SUCCESS;
+            }
+        }
+        else if (osinfo.dwMajorVersion == 6) {
+            if (osinfo.dwMinorVersion == 0) {
+                *platform = PLATFORM_VISTA;
+                return ERROR_SUCCESS;
+            }
+        }
+        return ERROR_UNSUPPORTED_OS;
+    } else {
+        return GetLastError();
+    }
+}
+
+BOOL
+is_wow64(HANDLE hProcess)
+{
+    /* IsWow64Pocess is only available on XP+ */
+    typedef DWORD (WINAPI *IsWow64Process_Type)(HANDLE hProcess,
+                                               PBOOL isWow64Process);
+    static HANDLE kernel32_handle;
+    static IsWow64Process_Type IsWow64Process;
+    if (kernel32_handle == NULL)
+        kernel32_handle = GetModuleHandle(L"kernel32.dll");
+    if (IsWow64Process == NULL && kernel32_handle != NULL) {
+        IsWow64Process = (IsWow64Process_Type)
+            GetProcAddress(kernel32_handle, "IsWow64Process");
+    }
+    if (IsWow64Process == NULL) {
+        /* should be NT or 2K */
+        DO_DEBUG(DL_INFO, {
+            DWORD platform = 0;
+            get_platform(&platform);
+            DO_ASSERT(platform == PLATFORM_WIN_NT_4 || platform == PLATFORM_WIN_2000);
+        });
+        return FALSE;
+    } else {
+        BOOL res;
+        if (!IsWow64Process(hProcess, &res))
+            return FALSE;
+        return res;
+    }
+}
+
+static const WCHAR *
+get_dynamorio_home_helper(BOOL reset)
+{
+    static WCHAR dynamorio_home[MAX_PATH] = { 0 };
+    DWORD res;
+
+    if (reset)
+        dynamorio_home[0] = L'\0';
+
+    if (dynamorio_home[0] != L'\0')
+        return dynamorio_home;
+
+    res = get_config_parameter(L_PRODUCT_NAME, FALSE,
+                               L_DYNAMORIO_VAR_HOME, dynamorio_home, MAX_PATH);
+
+    if (res == ERROR_SUCCESS && dynamorio_home[0] != L'\0')
+        return dynamorio_home;
+    else
+        return NULL;
+}
+
+const WCHAR *
+get_dynamorio_home() 
+{
+    return get_dynamorio_home_helper(FALSE);
+}
+
+static const WCHAR *
+get_dynamorio_logdir_helper(BOOL reset)
+{
+    static WCHAR dynamorio_logdir[MAX_PATH] = { 0 };
+    DWORD res;
+
+    if (reset)
+        dynamorio_logdir[0] = L'\0';
+
+    if (dynamorio_logdir[0] != L'\0')
+        return dynamorio_logdir;
+
+    res = get_config_parameter(L_PRODUCT_NAME, FALSE,
+                               L_DYNAMORIO_VAR_LOGDIR, dynamorio_logdir, MAX_PATH);
+
+    if (res == ERROR_SUCCESS && dynamorio_logdir[0] != L'\0')
+        return dynamorio_logdir;
+    else
+        return NULL;
+}
+
+const WCHAR *
+get_dynamorio_logdir() 
+{
+    return get_dynamorio_logdir_helper(FALSE);
+}
+
+/* If a path is passed in, it is checked for 8.3 compatibility; else,
+ * the default path is checked.  This routine does not check the
+ * actual 8.3 reg key.
+ */
+BOOL
+using_system32_for_preinject(const WCHAR *preinject)
+{
+    DWORD platform = 0;
+    get_platform(&platform);
+    if (platform == PLATFORM_WIN_NT_4) {
+        return TRUE;
+    }
+    else {
+        /* case 7586: we need to check if the system has disabled
+         *  8.3 names; if so, we need to use the system32 for
+         *  preinject (since spaces are not allowed in AppInitDLLs) 
+         */ 
+        WCHAR short_path[MAX_PATH];
+        WCHAR long_path[MAX_PATH];
+        if (preinject == NULL) {
+            /* note: with force_local_path == TRUE, we don't have
+             *  to worry about get_preinject_path() calling this
+             *  method back, and it will always return success. 
+             */  
+            get_preinject_path(short_path, MAX_PATH, TRUE, TRUE);
+            wcsncat(short_path, L"\\" L_EXPAND_LEVEL(INJECT_DLL_8_3_NAME), 
+                    MAX_PATH - wcslen(short_path));
+            NULL_TERMINATE_BUFFER(short_path);
+            get_preinject_path(long_path, MAX_PATH, TRUE, FALSE);
+            wcsncat(long_path, L"\\" L_EXPAND_LEVEL(INJECT_DLL_8_3_NAME), 
+                    MAX_PATH - wcslen(long_path));
+            NULL_TERMINATE_BUFFER(long_path);
+        } else {
+            /* Check the passed-in file */
+            GetShortPathName(preinject, short_path, BUFFER_SIZE_ELEMENTS(short_path));
+            NULL_TERMINATE_BUFFER(short_path);
+            wcsncpy(long_path, preinject, BUFFER_SIZE_ELEMENTS(long_path));
+        }
+
+        /* if 8.3 names are disabled, file_exists will return FALSE on
+         * the GetShortPathName()'ed path.
+         */
+        return (file_exists(long_path) && !file_exists(short_path));
+    }
+}
+
+/* if force_local_path, then this returns the in-installation
+ *  path regardless of using_system32_for_preinject().
+ * otherwise, this returns the path to the actuall DLL that 
+ *  will be injected, which depends on 
+ *  using_system32_for_preinject() 
+ * if short_path, calls GetShortPathName() on the path before returning it.
+ *  for a canonical preinject path, this parameter should be TRUE.
+ */
+DWORD
+get_preinject_path(WCHAR *buf, int nchars, BOOL force_local_path, BOOL short_path)
+{
+    if (!force_local_path && using_system32_for_preinject(NULL)) {
+        UINT len;
+        len = GetSystemDirectory(buf, MAX_PATH);
+
+        if (len == 0)
+            return GetLastError();
+    }
+    else {
+        const WCHAR *home = get_dynamorio_home();
+        /* using_system32_for_preinject() assumes we always succeed */
+        _snwprintf(buf, nchars, L"%s\\lib", home == NULL ? L"" : home);
+    }
+
+    buf[nchars - 1] = L'\0';
+    if (short_path)
+        GetShortPathName(buf, buf, nchars);
+
+    return ERROR_SUCCESS;
+}
+
+DWORD
+get_preinject_name(WCHAR *buf, int nchars)
+{
+    DWORD res;
+
+    if (using_system32_for_preinject(NULL)) {
+        wcsncpy(buf, L_EXPAND_LEVEL(INJECT_DLL_NAME), nchars);
+    }
+    else {
+        res = get_preinject_path(buf, nchars, FALSE, TRUE);
+        if (res != ERROR_SUCCESS)
+            return res;
+        wcsncat(buf, L"\\" L_EXPAND_LEVEL(INJECT_DLL_8_3_NAME), 
+                nchars - wcslen(buf));
+    }
+
+    buf[nchars - 1] = L'\0';
+
+    return ERROR_SUCCESS;
+}
+
+static dr_platform_t registry_view = DR_PLATFORM_DEFAULT;
+
+void
+set_dr_platform(dr_platform_t platform)
+{
+    registry_view = platform;
+}
+
+dr_platform_t 
+get_dr_platform()
+{
+    if (registry_view == DR_PLATFORM_64BIT)
+        return DR_PLATFORM_64BIT;
+    return DR_PLATFORM_32BIT;
+}
+
+DWORD
+platform_key_flags()
+{
+    /* PR 244206: have control over whether using WOW64 redirection or
+     * raw 64-bit registry view.
+     * These flags should be used for all Reg{Create,Open,Delete}KeyEx calls,
+     * on XP+ (invalid on earlier platforms) on redirected keys
+     * (most of HKLM\Software).
+     * The flags don't matter on non-redirected trees like HKLM\System.
+     * Since too many functions in libutil/ end up calling something
+     * that reads/writes the registry, we don't pass the dr_platform_t
+     * around and instead use a global variable.
+     */
+    DWORD platform = 0;
+    get_platform(&platform);
+    if (platform == PLATFORM_WIN_NT_4 || platform == PLATFORM_WIN_2000)
+        return 0;
+    else {
+        switch (registry_view) {
+        case DR_PLATFORM_DEFAULT:  return 0;
+        case DR_PLATFORM_32BIT:    return KEY_WOW64_32KEY;
+        case DR_PLATFORM_64BIT:    return KEY_WOW64_64KEY;
+        default: DO_ASSERT(false); return 0;
+        }
+    }
+}
+
+/* PR 244206: use this instead of RegDeleteKey for deleting redirected keys 
+ * (most of HKLM\Software)
+ */
+DWORD
+delete_product_key(HKEY hkey, LPCWSTR subkey)
+{
+    /* RegDeleteKeyEx is only available on XP+.  We cannot delete
+     * from 64-bit registry if we're WOW64 using RegDeleteKey, so we
+     * dynamically look up RegDeleteKeyEx.
+     * We could instead use NtDeleteKey and first open the subkey HKEY:
+     * we could link with the core's ntdll.c, and also use is_wow64_process().
+     */
+    typedef DWORD (WINAPI *RegDeleteKeyExW_Type)(HKEY hKey,
+                                                      LPCWSTR lpSubKey,
+                                                      REGSAM samDesired,
+                                                      DWORD Reserved);
+    static HANDLE advapi32_handle;
+    static RegDeleteKeyExW_Type RegDeleteKeyExW;
+    if (advapi32_handle == NULL)
+        advapi32_handle = GetModuleHandle(L"advapi32.dll");
+    if (RegDeleteKeyExW == NULL && advapi32_handle != NULL) {
+        RegDeleteKeyExW = (RegDeleteKeyExW_Type)
+            GetProcAddress(advapi32_handle, "RegDeleteKeyExW");
+    }
+    if (RegDeleteKeyExW == NULL) {
+        /* should be NT or 2K */
+        DO_DEBUG(DL_INFO, {
+            DWORD platform = 0;
+            get_platform(&platform);
+            DO_ASSERT(platform == PLATFORM_WIN_NT_4 || platform == PLATFORM_WIN_2000);
+        });
+        return RegDeleteKey(hkey, subkey);
+    } else
+        return RegDeleteKeyExW(hkey, subkey, platform_key_flags(), 0);
+}
+
+DWORD
+create_root_key()
+{
+    int res;
+    HKEY hkroot;
+
+    res = RegCreateKeyEx(DYNAMORIO_REGISTRY_HIVE,
+                         L_DYNAMORIO_REGISTRY_KEY,
+                         0,
+                         NULL,
+                         REG_OPTION_NON_VOLATILE,
+                         platform_key_flags()|KEY_WRITE|KEY_ENUMERATE_SUB_KEYS,
+                         NULL,
+                         &hkroot, 
+                         NULL);
+    RegCloseKey(hkroot);
+
+    return res;
+}
+
+/* Deletes the reg key created by create_root_key/setup_installation and the parent
+ * company key if it's empty afterwards (might not be if PE or nodemgr has config subkeys
+ * there. */
+DWORD
+destroy_root_key()
+{
+    DWORD res;
+    /* This deletes just the product key. */
+    res = recursive_delete_key(DYNAMORIO_REGISTRY_HIVE, L_DYNAMORIO_REGISTRY_KEY, NULL);
+    /* Delete the company key (this will only work if it is empty, so no need to worry
+     * about clobbering any config settings or doing too much damage if we screw up. */
+    if (res == ERROR_SUCCESS) {
+        WCHAR company_key[MAX_PATH];
+        WCHAR *pop;
+        wcsncpy(company_key, L_DYNAMORIO_REGISTRY_KEY, MAX_PATH);
+        NULL_TERMINATE_BUFFER(company_key);
+        pop = wcsstr(company_key, L_COMPANY_NAME);
+        if (pop != NULL) {
+            pop += wcslen(L_COMPANY_NAME);
+            /* sanity check */
+            if (pop == wcsrchr(company_key, L'\\')) {
+                *pop = L'\0';
+                delete_product_key(DYNAMORIO_REGISTRY_HIVE, company_key);
+            } else
+                res = ERROR_BAD_FORMAT;
+        } else
+            res = ERROR_BAD_FORMAT;
+    }
+    return res;
+}
+
+DWORD
+setup_installation(const WCHAR *path, BOOL overwrite)
+{
+    WCHAR buf[MAX_PATH];
+
+    /* if there's something there, leave it */
+    if (!overwrite && get_dynamorio_home() != NULL)
+        return ERROR_SUCCESS;
+
+    DO_DEBUG(DL_INFO,
+             printf("setting up installation at: %S\n", path);
+             );
+
+    mkdir_with_parents(path);
+
+    if (!file_exists(path))
+        return ERROR_PATH_NOT_FOUND;
+
+    _snwprintf(buf, MAX_PATH, L"%s\\%s", path, L"conf");
+    NULL_TERMINATE_BUFFER(buf);
+
+    DO_DEBUG(DL_INFO,
+             printf("making config dir: %S\n", buf);
+             );
+
+    mkdir_with_parents(buf);
+
+    if (!file_exists(buf))
+        return ERROR_PATH_NOT_FOUND;
+
+    _snwprintf(buf, MAX_PATH, L"%s\\%s", path, L"logs");
+    NULL_TERMINATE_BUFFER(buf);
+
+    DO_DEBUG(DL_INFO,
+             printf("making logdir: %S\n", buf);
+             );
+
+    mkdir_with_parents(buf);
+
+    if (!file_exists(buf))
+        return ERROR_PATH_NOT_FOUND;
+
+    CHECKED_OPERATION(create_root_key());
+
+    CHECKED_OPERATION(set_config_parameter(L_PRODUCT_NAME, FALSE, 
+                                           L_DYNAMORIO_VAR_HOME, path));
+
+    CHECKED_OPERATION(set_config_parameter(L_PRODUCT_NAME, FALSE, 
+                                           L_DYNAMORIO_VAR_LOGDIR, buf));
+
+    /* reset the DR_HOME cache */
+    get_dynamorio_home_helper(TRUE);
+
+    return ERROR_SUCCESS;
+}
+
+/* modifies permissions for 4.3 cache/User-SID directories to be
+ * created by users themselves
+ */
+DWORD
+setup_cache_permissions(WCHAR *cacheRootDirectory)
+{
+    DWORD result = ERROR_UNSUPPORTED_OS;
+
+#define NUM_ACES 2
+    /* in C const int isn't good enough */
+    EXPLICIT_ACCESS ea[NUM_ACES];
+
+    PSID pSIDEveryone = NULL;
+    PSID pSIDCreatorOwner = NULL;
+    PACL pACL = NULL;
+    PACL pOldDACL = NULL;
+
+    SID_IDENTIFIER_AUTHORITY SIDAuthWorld =
+            SECURITY_WORLD_SID_AUTHORITY;
+    SID_IDENTIFIER_AUTHORITY SIDAuthCreator = 
+        SECURITY_CREATOR_SID_AUTHORITY;
+    DWORD dwRes;
+
+    SECURITY_DESCRIPTOR *pSD  = NULL;
+
+    DWORD platform = 0;         /* accomodating NT permissions */
+    get_platform(&platform);
+
+    /* Note that we prefer to not create ACLs from scratch, so that we
+     * can accomodate Administrator groups unknown to us that would
+     * have been inherited from \Program Files\.  We should always
+     * start with a known ACL and just edit the new ACEs
+     */
+
+    dwRes = GetNamedSecurityInfo(cacheRootDirectory, SE_FILE_OBJECT, 
+                                 DACL_SECURITY_INFORMATION,
+                                 NULL, NULL, &pOldDACL, NULL, &pSD);
+
+    if (dwRes != ERROR_SUCCESS)
+        return dwRes;
+
+    /* Note: Although we are ADDING possibly existing ACE, it seems
+     * like this is handled well and we don't grow the ACL.  For now
+     * this doesn't matter to us, since we expect to have just copied
+     * the flags from the lib\ directory so can't really accumulate.
+     */
+
+    // Create a SID for the Everyone group.
+    if (!AllocateAndInitializeSid(&SIDAuthWorld, 1,
+                                  SECURITY_WORLD_RID,
+                                  0,
+                                  0, 0, 0, 0, 0, 0,
+                                  &pSIDEveryone)) {
+        DO_DEBUG(DL_VERB,
+                 printf("AllocateAndInitializeSid (Everyone).\n");
+                 );
+
+        goto cleanup;
+    }
+
+    // Create a SID for the CREATOR OWNER group
+    if (!AllocateAndInitializeSid(&SIDAuthCreator, 1,
+                                  SECURITY_CREATOR_OWNER_RID,
+                                  0,
+                                  0, 0, 0, 0, 0, 0,
+                                  &pSIDCreatorOwner)) {
+        DO_DEBUG(DL_VERB,
+                 printf("AllocateAndInitializeSid (CreatorOwner).\n");
+                 );
+
+        goto cleanup;
+    }
+
+    ZeroMemory(&ea, NUM_ACES * sizeof(EXPLICIT_ACCESS));
+
+    /* Grant create directory access to Everyone, which will be in
+     * addition to existing Read/Execute permissions we are starting
+     * with.
+     */
+    ea[0].grfAccessPermissions = FILE_ADD_SUBDIRECTORY;
+    ea[0].grfAccessMode = GRANT_ACCESS; /* not SET_ACCESS */
+    ea[0].grfInheritance = NO_INHERITANCE; /* ONLY in cache\ folder! */
+    ea[0].Trustee.TrusteeForm = TRUSTEE_IS_SID;
+    ea[0].Trustee.TrusteeType = TRUSTEE_IS_WELL_KNOWN_GROUP;
+    ea[0].Trustee.ptstrName = (LPTSTR) pSIDEveryone;
+
+    /* Set full control for CREATOR OWNER on any subfolders */
+    ea[1].grfAccessPermissions = GENERIC_ALL;
+    ea[1].grfAccessMode = SET_ACCESS; /* we SET ALL */
+    if (platform == PLATFORM_WIN_NT_4) {
+        /* case 10502 INHERIT_ONLY_ACE seems to not work */
+        /* we are mostly interested in any subdirectory, and cache/ is
+         * already created (and also trusted), so adding it there
+         * doesn't affect anything.
+         */
+        ea[1].grfInheritance = 
+            OBJECT_INHERIT_ACE | CONTAINER_INHERIT_ACE;
+    } else {
+        /* not using the same as NT, since Creator Owner may already
+         * have this ACE (and normally does) so we'll clutter with a
+         * new incomplete one */
+        ea[1].grfInheritance = INHERIT_ONLY_ACE | 
+            OBJECT_INHERIT_ACE | CONTAINER_INHERIT_ACE;
+    }
+    ea[1].Trustee.TrusteeForm = TRUSTEE_IS_SID;
+    ea[1].Trustee.TrusteeType = TRUSTEE_IS_WELL_KNOWN_GROUP;
+    ea[1].Trustee.ptstrName = (LPTSTR) pSIDCreatorOwner;
+
+    /* FIXME: we may want to disable the default group maybe should
+     * set CREATOR GROUP to no access otherwise we get the default
+     * Domain Users group (which usually is the Primary group) added,
+     * e.g. KRAMMER\None:R(ead)
+     */
+
+    /* MSDN gave a false alarm that this doesn't exist on NT - It is
+     * present at least on sp6.  FIXME: may want to use GetProcAddress
+     * if we support earlier versions, but we'll know early enough.
+     * We don't really need to support anything other than User SYSTEM
+     * on NT for which we don't need this to work and can return
+     * ERROR_UNSUPPORTED_OS
+     */
+    if (ERROR_SUCCESS != SetEntriesInAcl(NUM_ACES,
+                                         ea,
+                                         pOldDACL, /* original DACL */
+                                         &pACL))
+    {
+        DO_DEBUG(DL_VERB,
+                 printf("SetEntriesInAcl 0x%x\n", GetLastError());
+                 );
+        goto cleanup;
+    }
+
+    // Try to modify the object's DACL.
+    result = 
+        SetNamedSecurityInfo(cacheRootDirectory,          // name of the object
+                             SE_FILE_OBJECT,              // type of object
+                             DACL_SECURITY_INFORMATION |
+                             PROTECTED_DACL_SECURITY_INFORMATION
+                             ,   // change only the object's DACL
+                             NULL, NULL,                  // do not change owner or group
+                             pACL,                        // new DACL specified
+                             NULL);                       // do not change SACL
+
+    if (ERROR_SUCCESS == result) 
+    {
+        DO_DEBUG(DL_VERB,
+                 printf("Successfully changed DACL\n");
+                 );
+    }
+
+ cleanup:
+    if (pSIDEveryone)
+        FreeSid(pSIDEveryone); 
+    if (pSIDCreatorOwner)
+        FreeSid(pSIDCreatorOwner); 
+    if (pACL) 
+        LocalFree(pACL);
+    if (pSD)
+        LocalFree(pSD);
+
+    return result;
+#undef NUM_ACES
+}
+
+/* cache_root should normally be get_dynamorio_home() */
+DWORD
+setup_cache_shared_directories(const WCHAR *cache_root)
+{
+    DWORD res;
+
+    /* support for new-in-4.2 directories, update the permissions 
+     * on the cache/ to be the same as those on lib/, and the
+     * cache/shared/ folder to be the same as those on logs/
+     *
+     * note that the relative paths of the cache and shared cache
+     * directories here should match the values set in
+     * setup_cache_shared_registry()
+     */
+    WCHAR libpath[MAX_PATH];
+    WCHAR cachepath[MAX_PATH];
+    WCHAR logspath[MAX_PATH];
+    WCHAR sharedcachepath[MAX_PATH];
+
+    _snwprintf(libpath, MAX_PATH, L"%s\\lib", get_dynamorio_home());
+    NULL_TERMINATE_BUFFER(libpath);
+    _snwprintf(cachepath, MAX_PATH, L"%s\\cache", cache_root);
+    NULL_TERMINATE_BUFFER(cachepath);
+    _snwprintf(logspath, MAX_PATH, L"%s\\logs", get_dynamorio_home());
+    NULL_TERMINATE_BUFFER(logspath);
+    _snwprintf(sharedcachepath, MAX_PATH, L"%s\\shared", cachepath);
+    NULL_TERMINATE_BUFFER(sharedcachepath);
+
+    mkdir_with_parents(sharedcachepath);
+    /* FIXME: no error checking */
+
+    res = copy_file_permissions(cachepath, libpath);
+    if (res != ERROR_SUCCESS) {
+        return res;
+    }
+
+    res = copy_file_permissions(sharedcachepath, logspath);
+    if (res != ERROR_SUCCESS) {
+        return res;
+    }
+
+    /* For in 4.3 ONLY if all users (most importantly services)
+     * validate their per-user directory (or files) for ownership
+     */
+    res = setup_cache_permissions(cachepath);
+    if (res != ERROR_SUCCESS) {
+        return res;
+    }
+
+    return ERROR_SUCCESS;
+}
+
+/* cache_root should normally be get_dynamorio_home() */
+DWORD
+setup_cache_shared_registry(const WCHAR *cache_root, 
+                            ConfigGroup *policy)
+{
+    /* note that nodemgr doesn't need to do call this routine,
+     * since the registry keys are added to the node policies in
+     * controller/servlets/PolicyUpdateResponseHandler.java in the controller. but
+     * anyway we expect these to be forever the same, and in any
+     * case not configurable from the controller. 
+     */
+    WCHAR wpathbuf [MAX_PATH];
+
+    /* set up cache\ shared\ registry keys */
+    _snwprintf(wpathbuf, MAX_PATH, L"%s\\cache", cache_root);
+    NULL_TERMINATE_BUFFER(wpathbuf);
+    set_config_group_parameter(policy, 
+                               L_IF_WIN(DYNAMORIO_VAR_CACHE_ROOT),
+                               wpathbuf);
+
+    /* set up cache\ shared\ registry keys */
+    _snwprintf(wpathbuf, MAX_PATH, L"%s\\cache\\shared", cache_root);
+    NULL_TERMINATE_BUFFER(wpathbuf);
+    set_config_group_parameter(policy, 
+                               L_IF_WIN(DYNAMORIO_VAR_CACHE_SHARED),
+                               wpathbuf);
+    return ERROR_SUCCESS;
+}
+
+/* note that this checks the opstring against the
+ *  version of core that matches this build, NOT the version
+ *  of the core that's actually installed! */
+BOOL
+check_opstring(const WCHAR *opstring)
+{
+    char *cbuf;
+    options_t ops;
+    int res;
+    size_t cbuf_size =  wcslen(opstring) + 1;
+    cbuf = (char *)malloc(cbuf_size);
+    /* FIXME: if malloc fails, do something */
+    _snprintf(cbuf, cbuf_size, "%S", opstring);
+    cbuf[cbuf_size-1] = '\0';
+    res = set_dynamo_options(&ops, cbuf);
+    free(cbuf);
+    return !res;
+}
+
+
+HANDLE hToken = NULL;
+TOKEN_PRIVILEGES Priv, OldPriv;
+DWORD PrivSize = sizeof(OldPriv);
+
+DWORD 
+acquire_privileges() 
+{
+    /* if the privileges are already acquired, don't bother.
+       this almost certainly will cause failures if multiple
+        threads are trying to acquire privileges.
+     */
+    // FIXME - this should have real synchronization!!!
+    if (hToken != NULL)
+        return ERROR_ALREADY_INITIALIZED;
+
+    // get current thread token 
+    if  (!OpenThreadToken(GetCurrentThread(), 
+                          TOKEN_QUERY|TOKEN_ADJUST_PRIVILEGES, 
+                          FALSE, 
+                          &hToken)) {
+        // can't get thread token, try process token instead
+        if(!OpenProcessToken(GetCurrentProcess(), 
+                             TOKEN_QUERY|TOKEN_ADJUST_PRIVILEGES, 
+                             &hToken)) {
+            return GetLastError();
+        }
+    }
+
+    Priv.PrivilegeCount = 1;
+    Priv.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
+    LookupPrivilegeValue(NULL, SE_DEBUG_NAME, &Priv.Privileges[0].Luid);
+    // try to enable the privilege
+    if(!AdjustTokenPrivileges(hToken, FALSE, &Priv, sizeof(Priv), 
+                              &OldPriv, &PrivSize)) {
+        return GetLastError();
+    }
+
+    return ERROR_SUCCESS;
+}
+
+DWORD
+release_privileges() 
+{
+    if(hToken == NULL)
+        return ERROR_NO_SUCH_PRIVILEGE;
+
+    AdjustTokenPrivileges(hToken, 
+                          FALSE, 
+                          &OldPriv, 
+                          sizeof(OldPriv), 
+                          NULL, 
+                          NULL);
+    CloseHandle(hToken);
+    hToken = NULL;
+
+    return ERROR_SUCCESS;
+}
+
+void
+wstr_replace(WCHAR *str, WCHAR orig, WCHAR new)
+{
+    UINT i;
+    for (i = 0; i < wcslen(str); i++)
+        if (str[i] == orig)
+            str[i] = new;
+    return;
+}
+
+/* FIXME: should return error code if the directory wasn't created and
+ * doesn't exist already 
+ */
+void
+mkdir_with_parents(const WCHAR *dirname)
+{
+    WCHAR buf[MAX_PATH], *temp_subdir;
+
+    wcsncpy(buf, dirname, MAX_PATH);
+    NULL_TERMINATE_BUFFER(buf);
+
+    /* ensure proper slashes */
+    wstr_replace(buf, L'/', L'\\');
+
+    temp_subdir = buf;
+
+    while (temp_subdir != NULL) {
+        temp_subdir = wcschr(temp_subdir, L'\\');
+        if (temp_subdir != NULL)
+            *temp_subdir = L'\0';
+
+        DO_DEBUG(DL_VERB,
+                 printf("trying to make: %S\n", buf);
+                 );
+        
+        /* ok if this fails, eg the first time it will be C: */
+        CreateDirectory(buf, NULL);
+        
+        if (temp_subdir != NULL) {
+            *temp_subdir = L'\\';
+            temp_subdir = temp_subdir + 1;
+        }
+    }
+    return;
+}
+
+void
+ensure_directory_exists_for_file(WCHAR *filename)
+{
+    WCHAR *slashptr, buf[MAX_PATH];
+    wcsncpy(buf, filename, MAX_PATH);
+    NULL_TERMINATE_BUFFER(buf);
+
+    slashptr = wcsrchr(buf, L'\\');
+    if (slashptr == NULL)
+        return;
+
+    *slashptr = L'\0';
+
+    mkdir_with_parents(buf);
+}
+
+/* FIXME: apparently there's a bug in MSVCRT that converts
+ *  \r\n to \r\r\n ? anyway that's what google and the evidence
+ *  seem to indicate. (see policy.c for more)
+ *
+ * so we may want to convert this to using Win32 API instead of
+ *  CRT. but then again we may not, just on principle. */
+DWORD
+write_file_contents(WCHAR *path, char *contents, BOOL overwrite)
+{
+    FILE *fp = NULL;
+    DWORD res = ERROR_SUCCESS;
+
+    ensure_directory_exists_for_file(path);
+
+    fp = _wfopen(path, L"r");
+    if (fp != NULL) {
+        if (!overwrite)
+            return ERROR_ALREADY_EXISTS;
+        fclose(fp);
+    }
+
+    fp = _wfopen(path, L"w");
+    if(fp == NULL) {
+        res = delete_file_rename_in_use(path);
+        if (res != ERROR_SUCCESS || (fp = _wfopen(path, L"w")) == NULL) {
+            DO_DEBUG(DL_ERROR,
+                     printf("Unable to open file: %S (%d)\n", path, errno);
+                     );
+            return res;
+        }
+    }
+    
+    if(strlen(contents) != 
+       fwrite(contents, 1, strlen(contents), fp)) {
+        DO_DEBUG(DL_ERROR, 
+                 printf("Write failed to file: %S (errno=%d)\n", 
+                        path, errno);
+                 );
+        res = ERROR_WRITE_FAULT;
+    }
+
+    DO_DEBUG(DL_INFO, 
+             printf("wrote file %S\n", path);
+             );
+
+    fclose(fp);
+    return res;
+}
+
+DWORD
+write_file_contents_if_different(WCHAR *path, char *contents, BOOL *changed)
+{
+    char *existing;
+    DWORD res;
+
+    DO_ASSERT(path != NULL);
+    DO_ASSERT(contents != NULL);
+    DO_ASSERT(changed != NULL);
+
+    existing = (char *)malloc(strlen(contents) + 1);
+    res = read_file_contents(path, existing, strlen(contents) + 1, NULL);
+
+    if (res == ERROR_SUCCESS && 0 == strcmp(contents, existing)) {
+        *changed = FALSE;
+        res = ERROR_SUCCESS;
+    }
+    else {
+        *changed = TRUE;
+        res = write_file_contents(path, contents, TRUE);
+    }
+
+    free(existing);
+
+    return res;
+}
+
+
+#define READ_BUF_SZ 1024
+
+DWORD
+read_file_contents(WCHAR *path, char *contents, 
+                   UINT maxchars, UINT *needed)
+{
+    FILE *fp = NULL;
+    DWORD res = ERROR_SUCCESS;
+    UINT n_read = 0;
+    UINT n_needed = 0;
+    char buf[READ_BUF_SZ];
+
+    DO_ASSERT(path != NULL);
+    DO_ASSERT(contents != NULL || needed != NULL);
+    DO_ASSERT(contents == NULL || maxchars > 0);
+
+    fp = _wfopen(path, L"r");
+    if (fp == NULL) {
+        DO_DEBUG(DL_INFO,
+                 printf("Not found: %S\n", path);
+                 );
+        return ERROR_FILE_NOT_FOUND;
+    }
+
+    if (contents != NULL) {
+        n_read = fread(contents, 1, maxchars, fp);
+
+        /* NULL terminate string. */
+        contents[n_read == maxchars ? n_read - 1 : n_read] = '\0';
+
+        DO_DEBUG(DL_FINEST,
+                 printf("*Read %d bytes from %S (max=%d)\n", 
+                        n_read, path, maxchars);
+                 );
+    }
+    
+    n_needed = n_read;
+
+    while (!feof(fp)) {
+        res = ERROR_MORE_DATA;
+        
+        n_read = fread(buf, 1, READ_BUF_SZ, fp);
+
+        DO_DEBUG(DL_FINEST,
+                 printf("  Read an additional %d bytes\n", n_read);
+                 );
+
+        if (n_read == 0 && !feof(fp)) {
+            res = ERROR_READ_FAULT;
+            break;
+        }
+
+        n_needed += n_read;
+    }
+
+    /* + 1 for the NULL terminator */
+    n_needed += 1;
+
+    if (needed != NULL)
+        *needed = n_needed;
+
+    fclose(fp);
+
+    if (res == ERROR_SUCCESS || res == ERROR_MORE_DATA) {
+        DO_DEBUG(DL_VERB,
+                 printf("file %S contents: (%d needed)\n\n%s\n", 
+                        path, n_needed, contents);
+                 );
+    }
+    else {
+        DO_DEBUG(DL_ERROR,
+                 printf("read failed, error %d\n", res);
+                 );
+    }
+
+    return res;
+}
+
+
+DWORD
+delete_tree(const WCHAR *path)
+{
+    WIN32_FIND_DATA data;
+    HANDLE hFind;
+    WCHAR pathbuf[MAX_PATH], subdirbuf[MAX_PATH];
+
+    if (path == NULL)
+        return ERROR_INVALID_PARAMETER;
+
+    _snwprintf(pathbuf, MAX_PATH, L"%s\\*.*", path);
+    NULL_TERMINATE_BUFFER(pathbuf);
+
+    hFind = FindFirstFile(pathbuf, &data);
+    if (hFind == INVALID_HANDLE_VALUE)
+        return GetLastError();
+
+    DO_DEBUG(DL_VERB,
+             printf("dt working on %S\n", path);
+             );
+    do {
+        if (wcscmp(data.cFileName, L".") == 0 ||
+            wcscmp(data.cFileName, L"..") == 0)
+            continue;
+
+        /* case 7407: FindNextFile on a FAT32 filesystem returns files in
+         * the order they were written to disk, which could be different
+         * from NTFS where the order is alphabetical (from MSDN).
+         * Also on FAT32, FindNextFile sometimes puts us back in the loop
+         * for the file we just renamed and we try to
+         * delete_file_rename_in_use the file we just renamed for a very
+         * long time (>3 hrs).
+         *
+         * FIXME: temporary hack:  if filename has .tmp in its name
+         * (first ocurrance), assume we just renamed it and skip.
+         * 
+         * note we may want to doublecheck that the file is indeed not
+         * deletable although we now add it to the
+         * PendingFileRenameOperations so such unused files can't stay
+         * around for too long.
+         */
+        if (wcsstr(data.cFileName, L".tmp") != NULL)
+            continue;
+
+        DO_DEBUG(DL_VERB,
+                 printf("dt still working on %S, %d\n", data.cFileName,
+                        data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY);
+                 );
+
+        _snwprintf(subdirbuf, MAX_PATH, L"%s\\%s", path, data.cFileName);
+        NULL_TERMINATE_BUFFER(subdirbuf);
+
+        /* case 4512: use rename trick if file is in use, so that
+         *  the uninstall/reinstall case will work */
+        if (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+            delete_tree(subdirbuf);
+        else
+            delete_file_rename_in_use(subdirbuf);
+    }
+    while (FindNextFile(hFind, &data));
+
+    if (!FindClose(hFind))
+        return GetLastError();
+
+    if (!RemoveDirectory(path))
+        return GetLastError();
+
+    return ERROR_SUCCESS;
+}
+
+
+/* 
+ * helper function for registry permissions workaround. stopgap until
+ *  we can make a decent permissions api.
+ *
+ * based on example code obtained from:
+ *  http://www.codeproject.com/system/secntobj.asp
+ */
+
+PSID
+getSID(WCHAR *user)
+{
+    DWORD dwSidLen = 0, dwDomainLen = 0;
+    SID_NAME_USE SidNameUse;
+    PSID pRet = NULL;
+    PSID pSid = NULL; 
+    WCHAR *lpDomainName = NULL;
+    
+    /* The function on the first call retives the length that we need
+     * to initialize the SID & domain name pointers */
+    if(!LookupAccountName(NULL, user, NULL, &dwSidLen, 
+                          NULL, &dwDomainLen, &SidNameUse)) {
+        if(ERROR_INSUFFICIENT_BUFFER == GetLastError()) {
+
+            pSid = LocalAlloc(LMEM_ZEROINIT, dwSidLen);
+            lpDomainName = LocalAlloc(LMEM_ZEROINIT,
+                                      dwDomainLen*sizeof(WCHAR));
+            
+            if(pSid && lpDomainName &&
+               LookupAccountName(NULL, user, pSid, &dwSidLen,
+                                 lpDomainName, &dwDomainLen, &SidNameUse)) {
+                pRet = pSid;
+                pSid = NULL;
+            }
+        }
+    }
+
+    /* if successful, was set to NULL and left in pRet */
+    LocalFree(pSid);
+    LocalFree(lpDomainName);
+
+    return pRet;
+}
+
+BOOL
+make_acl(DWORD count,
+         WCHAR **userArray,
+         DWORD *maskArray,
+         ACL **acl)
+{
+    DWORD    dwLoop = 0;
+    DWORD    dwAclLen = 0;
+    PACL     pRetAcl = NULL;
+    PSID     *ppStoreSid = NULL;
+    BOOL     bRes = FALSE;
+
+    if(acl == NULL)
+        goto cleanup;
+
+    ppStoreSid = LocalAlloc(LMEM_ZEROINIT, count * sizeof(void*));
+
+    if (ppStoreSid == NULL)
+        goto cleanup;
+
+    for (dwLoop = 0; dwLoop < count; dwLoop++) {
+
+        ppStoreSid[dwLoop] = getSID(userArray[dwLoop]);
+
+        if (ppStoreSid[dwLoop] == NULL)
+            goto cleanup;
+        
+        dwAclLen += GetLengthSid(ppStoreSid[dwLoop]) + 
+            sizeof(ACCESS_ALLOWED_ACE) - sizeof(DWORD);
+
+    }
+
+    dwAclLen += sizeof(ACL);
+    pRetAcl = LocalAlloc(LMEM_ZEROINIT, dwAclLen);
+
+    if(pRetAcl == NULL || !InitializeAcl(pRetAcl, dwAclLen, ACL_REVISION))
+        goto cleanup;
+    
+    for (dwLoop = 0; dwLoop < count; dwLoop++) {
+        /* only adding access allowed ACE's */
+        if(!AddAccessAllowedAce(pRetAcl, ACL_REVISION, 
+                                maskArray[dwLoop], ppStoreSid[dwLoop]))
+            goto cleanup;
+    }
+    
+    *acl = pRetAcl;
+    pRetAcl = NULL;
+    bRes = TRUE;
+
+ cleanup:
+
+    if (ppStoreSid != NULL) {
+        for (dwLoop = 0; dwLoop < count; dwLoop++)
+            LocalFree(ppStoreSid[dwLoop]);
+        LocalFree(ppStoreSid);
+    }
+
+    /* if successful, was set to NULL and left in *acl */
+    LocalFree(pRetAcl);
+    
+    return bRes;
+}
+
+#define NUM_ACL_ENTRIES 4
+
+DWORD
+set_registry_permissions_for_user(WCHAR *hklm_keyname, WCHAR *user)
+{
+    SECURITY_DESCRIPTOR sd;
+    SID *owner = NULL;
+    ACL *acl1 = NULL;
+    DWORD res;
+    HKEY hkey = NULL;
+
+    WCHAR *users[NUM_ACL_ENTRIES] = {
+        L"Administrators",
+        L"Everyone",
+        L"SYSTEM",
+        NULL,
+    };
+
+    DWORD masks[NUM_ACL_ENTRIES] = {
+        KEY_ALL_ACCESS | DELETE | READ_CONTROL | WRITE_DAC | WRITE_OWNER,
+        KEY_READ,
+        KEY_ALL_ACCESS | DELETE | READ_CONTROL | WRITE_DAC | WRITE_OWNER,
+        KEY_ALL_ACCESS,
+    };
+
+    users[NUM_ACL_ENTRIES - 1] = user;
+
+    DO_DEBUG(DL_VERB,
+             printf("Starting acl..\n");
+             );
+
+    res = RegOpenKeyEx(HKEY_LOCAL_MACHINE, hklm_keyname,
+                       0, platform_key_flags()|KEY_ALL_ACCESS, &hkey);
+
+    if (res != ERROR_SUCCESS)
+        goto error_out;
+
+    DO_DEBUG(DL_VERB,
+             printf("Got key handle.\n");
+             );
+
+    if (!InitializeSecurityDescriptor(&sd, SECURITY_DESCRIPTOR_REVISION)) {
+        res = GetLastError();
+        goto error_out;
+    }
+
+    owner = getSID(users[0]);
+
+    if (NULL == owner) {
+        res = ERROR_INVALID_DATA;
+        goto error_out;
+    }
+
+    if (!SetSecurityDescriptorOwner(&sd, owner, FALSE)) {
+        res = GetLastError();
+        goto error_out;
+    }
+
+    DO_DEBUG(DL_VERB,
+             printf("Set owner.\n");
+             );
+
+    if (!make_acl(NUM_ACL_ENTRIES, users, masks, &acl1)) {
+        res = ERROR_ACCESS_DENIED;
+        goto error_out;
+    }
+
+    DO_DEBUG(DL_VERB,
+             printf("Made ACL.\n");
+             );
+
+    if(!SetSecurityDescriptorDacl(&sd, TRUE, acl1, FALSE)) {
+        res = GetLastError();
+        goto error_out;
+    }
+
+    if (!IsValidSecurityDescriptor(&sd)) {
+        res = GetLastError();
+        goto error_out;
+    }
+
+    res = RegSetKeySecurity(hkey, 
+                            DACL_SECURITY_INFORMATION | 
+                            OWNER_SECURITY_INFORMATION,
+                            &sd);
+
+    DO_DEBUG(DL_VERB, 
+             printf("Set sacl.\n");
+             );
+
+    goto cleanup;
+
+ error_out:
+
+    /* make sure to return an error */
+    if (res == ERROR_SUCCESS)
+        res = ERROR_ACCESS_DENIED;
+
+ cleanup:
+
+    if (hkey != NULL)
+        RegCloseKey(hkey);
+
+    LocalFree(owner);
+    LocalFree(acl1);
+
+    return res;
+}
+
+/* will limit to 1 MB */
+#define MAX_INSERT_SIZE (1024 * 1024)
+
+static void
+insert_file(FILE *file, wchar_t *file_src_name, BOOL delete)
+{
+    int fd_src = _wopen(file_src_name, _O_RDONLY|_O_BINARY);
+    long length;
+    int error;
+    
+    if (fd_src == -1) {
+        fprintf(file, "Unable to open file \"%S\" for inserting\n",
+               file_src_name);
+        return;
+    }
+    length = _filelength(fd_src);
+    if (length == -1L) {
+        fprintf(file, "Unable to get file length for file \"%S\"\n",
+               file_src_name);
+        return;
+    }
+    if (length > MAX_INSERT_SIZE) {
+        fprintf(file, "File size exceeds max insert length, truncating from %d to %d\n",
+                length, MAX_INSERT_SIZE);
+        length = MAX_INSERT_SIZE;
+    }
+
+    fprintf(file, "Inserting file: name=\"%S\" length=%d\n", file_src_name, length);
+    /* hmm, there's prob. a better way to do this ... */
+#define COPY_BUF_SIZE 4096
+    {
+        char buf[COPY_BUF_SIZE] = {0};
+        long i = 0;
+        while (i + COPY_BUF_SIZE <= length) {
+            _read(fd_src, buf, COPY_BUF_SIZE);
+            fwrite(buf, 1, COPY_BUF_SIZE, file);
+            i += COPY_BUF_SIZE;
+        }
+        if (i < length) {
+            _read(fd_src, buf, length - i);
+            fwrite(buf, 1, length - i, file);
+        }
+    }
+
+    fprintf(file, "Finished inserting file\n");
+    if (_read(fd_src, &error, sizeof(error)) != 0)
+        fprintf(file, "ERROR : file continues beyond length\n");
+    _close(fd_src);
+    if (delete) {
+        DeleteFile(file_src_name);
+    }
+    return;
+}
+
+/* see utils.h for description */
+DWORD
+get_violation_info(EVENTLOGRECORD *pevlr, /* INOUT */ VIOLATION_INFO *info)
+{
+    DO_ASSERT(pevlr != NULL && info != NULL && pevlr->EventID == MSG_SEC_FORENSICS);
+    info->report = NULL;
+    if (pevlr->EventID != MSG_SEC_FORENSICS)
+        return ERROR_INVALID_PARAMETER;
+    info->report = get_forensics_filename(pevlr);
+    if (file_exists(info->report))
+        return ERROR_SUCCESS;
+    else 
+        return ERROR_FILE_NOT_FOUND;
+}
+
+wchar_t *canary_process_names[] = {L"canary.exe", L"services.exe", L"iexplore.exe"};
+#define num_canary_processes BUFFER_SIZE_ELEMENTS(canary_process_names)
+/* how long to wait for an apparently hung canary process */
+#define CANARY_HANG_WAIT 20000
+/* interval to wait for the canary process to do something */
+#define CANARY_SLEEP_WAIT 100
+
+#define OPTIONS_CANARY_NATIVE L" -list_modules -check_for_hooked_mods_list ntdll.dll"
+#define OPTIONS_CANARY_THIN_CLIENT L""
+#define OPTIONS_CANARY_CLIENT L""
+#define OPTIONS_CANARY_MF L""
+#define OPTIONS_CANARY_INJECT L"-wait"
+
+/* FIXME - could even get ldmps ... */
+/* FIXME - xref case 10322 on -syslog_mask 0, eventually should remove and verify
+ * expected eventlog output (and get PE to ignore them). */
+#define OPTIONS_THIN_CLIENT L"-thin_client -syslog_mask 0"
+#define OPTIONS_CLIENT L"-client -syslog_mask 0"
+/* FIXME - temporary hack so virus scan correctly identified by canary. Weird case
+ * since this is considered a survivable violation by default (and so ignores kill proc).
+ */
+#define OPTIONS_MF L"-apc_policy 0 -syslog_mask 0"
+
+/* returns the appropriate canary fail code */
+static int
+run_individual_canary_test(FILE *file, WCHAR *logbase, WCHAR *dr_options, int exe_index,
+                           ConfigGroup *policy, WCHAR *exe, WCHAR *exe_args,
+                           BOOL inject_test, char *type, BOOL early_test)
+{
+    STARTUPINFO sinfo = {sizeof(sinfo), NULL, L"", 0, 0, 0, 0, 0, 0, 0, 0,
+                         0, 0, 0, NULL, NULL, NULL, NULL};
+    PROCESS_INFORMATION pinfo;
+    int canary_code = CANARY_SUCCESS;
+    WCHAR logbuf[MAX_PATH] = {0};
+    WCHAR outfile[MAX_PATH];
+    WCHAR cmd_buf[5*MAX_PATH];
+
+    /* set up registry */
+    get_unique_filename(logbase, L"canary_logs", L"",
+                        logbuf, BUFFER_SIZE_ELEMENTS(logbuf));
+    CreateDirectory(logbuf, NULL);
+    set_config_group_parameter(get_child(canary_process_names[exe_index], policy),
+                               L_DYNAMORIO_VAR_LOGDIR, logbuf);
+    set_config_group_parameter(get_child(canary_process_names[exe_index], policy),
+                               L_DYNAMORIO_VAR_OPTIONS, dr_options);
+    write_config_group(policy);
+
+    /* set up cmd_buf */
+    _snwprintf(outfile, BUFFER_SIZE_ELEMENTS(outfile), L"%s\\out.rep", logbuf);
+    NULL_TERMINATE_BUFFER(outfile);
+    if (early_test) {
+        /* we get the canary_process to re-launch itself to run test with early inject */
+        _snwprintf(cmd_buf, BUFFER_SIZE_ELEMENTS(cmd_buf),
+                   L"\"%s\" \"%s\" -launch_child %s%d \"\\\"%s\\\" %s\"",
+                   exe, outfile, inject_test ? L"-verify_inject " : L"",
+                   CANARY_HANG_WAIT / 2, outfile, exe_args);
+    } else {
+        _snwprintf(cmd_buf, BUFFER_SIZE_ELEMENTS(cmd_buf), L"\"%s\" \"%s\" %s",
+                   exe, outfile, exe_args);
+    }
+    NULL_TERMINATE_BUFFER(cmd_buf);
+
+    fprintf(file, "Starting Canary Process \"%S\" core_ops=\"%S\" type=%s%s\n",
+            cmd_buf, dr_options, type, inject_test ? " inject" : "");
+    if (CreateProcess(NULL, cmd_buf, NULL, NULL, TRUE, 0, NULL, NULL,
+                      &sinfo, &pinfo)) {
+        if (inject_test && !early_test) {
+            DWORD sleep_count = 0, under_dr_code, ws, build = 0;
+            do {
+                ws = WaitForSingleObject(pinfo.hProcess, CANARY_SLEEP_WAIT);
+                sleep_count += CANARY_SLEEP_WAIT;
+                under_dr_code = under_dynamorio_ex(pinfo.dwProcessId, &build);
+            } while (ws == WAIT_TIMEOUT && sleep_count < CANARY_HANG_WAIT &&
+                     (under_dr_code == DLL_UNKNOWN || under_dr_code == DLL_NONE));
+            if (under_dr_code == DLL_UNKNOWN || under_dr_code == DLL_NONE) {
+                canary_code = CANARY_FAIL_APP_INIT_INJECTION;
+                fprintf(file, "Injection Failed - verify registry settings\n");
+            } else {
+                fprintf(file, "Verified Injection, build %d\n", build);
+            }
+            if (ws == WAIT_TIMEOUT)
+                terminate_process(pinfo.dwProcessId);
+        } else {
+            DWORD ws = WaitForSingleObject(pinfo.hProcess, CANARY_HANG_WAIT);
+            if (ws == WAIT_TIMEOUT) {
+                if (early_test && inject_test) {
+                    canary_code = CANARY_FAIL_EARLY_INJECTION;
+                    fprintf(file, "Early Injection Failed\n");
+                } else {
+                    canary_code = CANARY_FAIL_HUNG;
+                    fprintf(file, "Canary Hung\n");
+                }
+                terminate_process(pinfo.dwProcessId);
+            } else {
+                DWORD exit_code = 0;
+                GetExitCodeProcess(pinfo.hProcess, &exit_code);
+                /* FIXME - check return value, shouldn't ever fail though */
+                if (exit_code != CANARY_PROCESS_EXP_EXIT_CODE) {
+                    /* FIXME - the -1 is based on the core value for kill
+                     * proc, should export that and use it, or really just check for 
+                     * violations since we'll want the forensics anyways. Doesn't
+                     * disambiguate between dr error and violation. */
+                    if (exit_code == (DWORD)-1) {
+                        canary_code = CANARY_FAIL_VIOLATION;
+                        fprintf(file, "Canary Violation or DR error\n");
+                    } else {
+                        canary_code = CANARY_FAIL_CRASH;
+                        fprintf(file, "Canary Crashed 0x%08x\n", exit_code);
+                    }
+                } else if (early_test && inject_test) {
+                    fprintf(file, "Verified Early Injection\n");   
+                }
+            }
+        }
+        CloseHandle(pinfo.hProcess);
+        CloseHandle(pinfo.hThread);
+        {
+            HANDLE hFind;
+            WIN32_FIND_DATA data;
+            WCHAR file_name[MAX_PATH], pattern[MAX_PATH];
+
+            _snwprintf(pattern, BUFFER_SIZE_ELEMENTS(pattern), L"%s\\*.*", logbuf);
+            NULL_TERMINATE_BUFFER(pattern);
+            hFind = FindFirstFile(pattern, &data);
+            if (hFind != INVALID_HANDLE_VALUE) {
+                do {
+                    if (wcscmp(data.cFileName, L".") == 0 ||
+                        wcscmp(data.cFileName, L"..") == 0)
+                        continue;
+                    _snwprintf(file_name, BUFFER_SIZE_ELEMENTS(file_name), L"%s\\%s",
+                               logbuf, data.cFileName);
+                    NULL_TERMINATE_BUFFER(file_name);
+                    insert_file(file, file_name, FALSE);
+                } while (FindNextFile(hFind, &data));
+                FindClose(hFind);
+            }
+        }
+        fprintf(file, "Canary Finished\n");
+    } else {
+        fprintf(file, "Canary \"%S\" Failed to Launch\n", cmd_buf);
+    }
+    return canary_code;
+}
+
+#pragma warning( disable : 4127) //conditional expression is constant i.e while (FALSE)
+
+/* see utils.h for description */
+BOOL
+run_canary_test_ex(FILE *file, /* INOUT */ CANARY_INFO *info,
+                   const WCHAR *scratch_folder, const WCHAR *canary_process)
+{
+    ConfigGroup *policy, *save_policy;
+    WCHAR exe_buf[num_canary_processes][MAX_PATH];
+    WCHAR log_folder[MAX_PATH];
+    DWORD i;
+    BOOL autoinject_set = is_autoinjection_set();
+    
+    info->canary_code = ERROR_SUCCESS;
+    info->url = L"CFail";
+    info->msg = L"Canary Failed";
+
+    read_config_group(&save_policy, L_PRODUCT_NAME, TRUE);
+    save_policy->should_clear = TRUE;
+    read_config_group(&policy, L_PRODUCT_NAME, TRUE);
+    policy->should_clear = TRUE;
+    remove_children(policy);
+    
+    _snwprintf(log_folder, BUFFER_SIZE_ELEMENTS(log_folder),
+               L"%s\\canary_logs", scratch_folder);
+    NULL_TERMINATE_BUFFER(log_folder);
+    CreateDirectory(log_folder, NULL);
+
+    for (i = 0; i < num_canary_processes; i++) {
+        _snwprintf(exe_buf[i], BUFFER_SIZE_ELEMENTS(exe_buf[i]), L"%s\\%s",
+                   scratch_folder, canary_process_names[i]);
+        NULL_TERMINATE_BUFFER(exe_buf[i]);
+        if (CopyFile(canary_process, exe_buf[i], FALSE) == 0) {
+            fprintf(file, "Failed to copy canary file %S to %S\n",
+                    canary_process, exe_buf[i]);
+            /* FIXME- continue if file exists from a previous run that didn't clean up */
+            info->canary_code = CANARY_UNABLE_TO_TEST;
+            goto canary_exit;
+        }
+        add_config_group(policy, new_config_group(canary_process_names[i]));
+        set_config_group_parameter(get_child(canary_process_names[i], policy),
+                                   L_DYNAMORIO_VAR_RUNUNDER, L"1");
+    }
+    write_config_group(policy);
+    
+    /* FIXME - monitor eventlog though we should still detect via forensics and/or
+     * exit code (crash/violation).  Xref 10322, for now we suppress eventlogs. */
+    /* FIXME - the verify injection tests need work, should just talk to canary proc. */
+    /* FIXME - verify canary output - neccesary? not clear what action would be */
+    /* Files are copied, begin runs */
+
+#define DO_RUN(run_flag, core_ops, canary_options, inject, run_name, test_type) do {  \
+    if (TEST(run_flag, info->run_flags)) {                                            \
+        WCHAR *canary_ops = TEST(run_flag, info->fault_run) ?                         \
+                            info->canary_fault_args : canary_options;                 \
+        for (i = 0; i < num_canary_processes; i++) {                                  \
+            int code =                                                                \
+                run_individual_canary_test(file, log_folder, core_ops, i, policy,     \
+                                           exe_buf[i], canary_ops, inject,            \
+                                           run_name, FALSE /* not early */);          \
+            if (code >= 0 && test_type != CANARY_TEST_TYPE_NATIVE) {                  \
+                code =                                                                \
+                    run_individual_canary_test(file, log_folder, core_ops, i, policy, \
+                                               exe_buf[i], canary_ops, inject,        \
+                                               run_name, TRUE /* early inject*/);     \
+            }                                                                         \
+            if (code < 0) {                                                           \
+                if (CANARY_RUN_REQUIRES_PASS(run_flag, info->run_flags)) {            \
+                    info->canary_code = GET_CANARY_CODE(test_type, code);             \
+                    goto canary_exit;                                                 \
+                }                                                                     \
+                break; /* skip remaining tests in run once first failure found */     \
+            }                                                                         \
+        }                                                                             \
+    }                                                                                 \
+} while (FALSE)
+
+    /* First the native runs. */
+    unset_autoinjection();
+
+    /* native info gathering run. */
+    DO_RUN(CANARY_RUN_NATIVE, L"", OPTIONS_CANARY_NATIVE,
+           FALSE, "native", CANARY_TEST_TYPE_NATIVE);
+
+    set_autoinjection(); /* Going to do the non-native runs now */
+    
+    /* Now the -thin_client inject run */
+    DO_RUN(CANARY_RUN_THIN_CLIENT_INJECT, OPTIONS_THIN_CLIENT, OPTIONS_CANARY_INJECT,
+           TRUE, "-thin_client", CANARY_TEST_TYPE_THIN_CLIENT);
+
+    /* now the full -thin_client run */
+    DO_RUN(CANARY_RUN_THIN_CLIENT, OPTIONS_THIN_CLIENT, OPTIONS_CANARY_THIN_CLIENT,
+           FALSE, "-thin_client", CANARY_TEST_TYPE_THIN_CLIENT);
+
+    /* Now the -client run */
+    DO_RUN(CANARY_RUN_CLIENT, OPTIONS_CLIENT, OPTIONS_CANARY_CLIENT,
+           FALSE, "-client", CANARY_TEST_TYPE_CLIENT);
+    
+    /* Now the MF run */
+    DO_RUN(CANARY_RUN_MF, OPTIONS_MF, OPTIONS_CANARY_MF,
+           FALSE, "MF", CANARY_TEST_TYPE_MF);
+
+#undef DO_RUN
+
+ canary_exit:
+    if (autoinject_set)
+        set_autoinjection();
+    else
+        unset_autoinjection();
+    free_config_group(policy);
+    write_config_group(save_policy);
+    free_config_group(save_policy);
+    fprintf(file, "Canary code 0x%08x\n", info->canary_code);
+    if (info->canary_code >= 0) {
+        info->url = L"ctest";
+        info->msg = L"Canary success";
+    }
+    return (info->canary_code >= 0);
+}
+
+/* see utils.h for description */
+BOOL
+run_canary_test(/* INOUT */ CANARY_INFO *info, WCHAR *version_msg)
+{
+    BOOL result;
+    DWORD res;
+    FILE *report_file;
+    WCHAR scratch_folder[MAX_PATH], canary_process[MAX_PATH];
+    const WCHAR *dynamorio_home = get_dynamorio_home();
+    const WCHAR *dynamorio_logdir = get_dynamorio_logdir();
+    _snwprintf(canary_process, BUFFER_SIZE_ELEMENTS(canary_process),
+               L"%s\\bin\\canary.exe", dynamorio_home);
+    NULL_TERMINATE_BUFFER(canary_process);
+    _snwprintf(scratch_folder, BUFFER_SIZE_ELEMENTS(scratch_folder),
+               L"%s\\canary_test", dynamorio_logdir);
+    NULL_TERMINATE_BUFFER(scratch_folder);
+    /* xref case 10157, let's try to make sure this stays clean */
+    delete_tree(scratch_folder);
+    CreateDirectory(scratch_folder, NULL);
+    /* FIXME - verify directory created */
+    /* Using get unique file name since we plan to run this more then once,
+     * though only an issue if the caller doesn't cleanup the report file and
+     * leaves it locked. */
+    get_unique_filename(dynamorio_logdir, L"canary_report", L".crep",
+                        info->buf_report, BUFFER_SIZE_ELEMENTS(info->buf_report));
+    info->report = info->buf_report;
+    report_file = _wfopen(info->report, L"wb");
+    /* FIXME - verify file creation */
+    fprintf(report_file, "%S\n", version_msg == NULL ? L"unkown version" : version_msg);
+    result = run_canary_test_ex(report_file, info, scratch_folder, canary_process);
+    res = delete_tree(scratch_folder);
+    fprintf(report_file, "Deleted scratch folder \"%S\", code %d\n",
+            scratch_folder, res);
+    fclose(report_file);
+    return result;
+}
+
+#else //ifdef UNIT_TEST
+
+int
+main()
+{
+    set_debuglevel(DL_INFO);
+    set_abortlevel(DL_WARN);
+
+
+    /* read/write file */
+    {
+        char *test1, *test2;
+        char buffy[1024];
+        WCHAR *fn = L"utils.tst";
+        UINT needed;
+        BOOL changed;
+
+        test1 = "This is a stupid file.\r\n\r\nDon't you think?\r\n";
+        test2 = "foo\r\n";
+
+        CHECKED_OPERATION(write_file_contents(fn, test1, TRUE));
+        
+        DO_ASSERT(ERROR_MORE_DATA == read_file_contents(fn, NULL, 0, &needed));
+        DO_ASSERT(strlen(test1) + 1 == needed);
+
+        CHECKED_OPERATION(read_file_contents(fn, buffy, needed, NULL));
+        DO_ASSERT(0 == strcmp(test1, buffy));
+
+        CHECKED_OPERATION(write_file_contents_if_different(fn, test1, 
+                                                           &changed));
+        DO_ASSERT(!changed);
+        
+        CHECKED_OPERATION(write_file_contents_if_different(fn, test2, 
+                                                           &changed));
+        DO_ASSERT(changed);
+
+        CHECKED_OPERATION(read_file_contents(fn, buffy, 1024, NULL));
+        DO_ASSERT(0 == strcmp(test2, buffy));
+    }
+
+    /* file existence */
+    {
+        WCHAR *fn = L"tester-file";
+
+        DeleteFile(fn);
+        DO_ASSERT(!file_exists(fn));
+        DO_ASSERT(!file_exists(fn));
+
+        CHECKED_OPERATION(write_file_contents(fn, "testing", TRUE));
+        DO_ASSERT(file_exists(fn));
+        DeleteFile(fn);
+
+        DO_ASSERT(file_exists(L"C:\\"));
+        DO_ASSERT(!file_exists(L"%%RY:\\\\zZsduf"));
+    }
+
+    /* mkdir_with_parents / delete_tree */
+    {
+        delete_tree(L"__foo_test");
+        mkdir_with_parents(L"__foo_test");
+        DO_ASSERT(file_exists(L"__foo_test"));
+        mkdir_with_parents(L"__foo_test\\foo\\bar\\goo");
+        DO_ASSERT(file_exists(L"__foo_test\\foo\\bar\\goo"));
+        mkdir_with_parents(L"__foo_test/lib/bar/goo/dood");
+        DO_ASSERT(file_exists(L"__foo_test\\lib\\bar\\goo\\dood"));
+        CHECKED_OPERATION(delete_tree(L"__foo_test"));
+        DO_ASSERT(!file_exists(L"__foo_test"));
+        DO_ASSERT(!file_exists(L"__foo_test\\foo\\bar\\goo"));
+        DO_ASSERT(!file_exists(L"__foo_test\\lib\\bar\\goo\\dood"));
+    }
+
+    /* setup_installation */
+    {
+        CHECKED_OPERATION(setup_installation(L"C:\\", TRUE));
+        CHECKED_OPERATION(setup_installation(L"C:\\foobarra", FALSE));
+        DO_ASSERT_WSTR_EQ(L"C:\\", get_dynamorio_home());
+        CHECKED_OPERATION(setup_installation(L"C:\\foobarra", TRUE));
+        DO_ASSERT_WSTR_EQ(L"C:\\foobarra", get_dynamorio_home());
+    }
+
+    {
+        WCHAR piname[MAX_PATH];
+        BOOL bres = using_system32_for_preinject(NULL);
+        printf("Using SYSTEM32 for preinject: %s\n", bres ? "TRUE" : "FALSE");
+        CHECKED_OPERATION(get_preinject_name(piname, MAX_PATH));
+        printf("Preinject name: %S\n", piname);
+    }
+
+    printf("All Test Passed\n");
+
+    return 0;
+
+}
+
+#endif
diff --git a/libutil/utils.h b/libutil/utils.h
new file mode 100644
index 000000000..9ab3de6a3
--- /dev/null
+++ b/libutil/utils.h
@@ -0,0 +1,457 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+
+#ifndef _DETERMINA_SHAREUTILS_H_
+#define _DETERMINA_SHAREUTILS_H_
+
+#include "config.h"
+
+#ifndef __cplusplus
+# ifndef bool
+typedef int bool;
+# endif /* !bool */
+#endif /* __cplusplus */
+
+#ifndef WINDOWS
+/* must define WINDOWS for dr_config.h */
+# define WINDOWS
+# define _HAD_TO_DEFINE_WINDOWS
+#endif
+#include "lib/globals_shared.h"
+#include "lib/dr_config.h"
+#ifndef _HAD_TO_DEFINE_WINDOWS
+# undef WINDOWS
+# undef _HAD_TO_DEFINE_WINDOWS
+#endif
+
+# ifdef _DEBUG
+#  include <stdio.h>
+#  include <crtdbg.h>
+# endif
+
+#ifdef __cplusplus
+extern "C"{
+#endif 
+
+void
+wcstolower(WCHAR *str);
+
+WCHAR *
+get_exename_from_path(const WCHAR *path);
+
+BOOL
+get_commandline_qualifier(const WCHAR* command_line,
+                          WCHAR* derived_name, 
+                          UINT max_derived_length /* in elements */,
+                          BOOL no_strip);
+
+DWORD
+acquire_shutdown_privilege();
+
+DWORD
+reboot_system();
+
+BOOL
+file_exists(const WCHAR *fn);
+
+/* grokked from the core. 
+ * FIXME: shareme!
+ * if NULL is passed for directory, then it is ignored and no directory
+ *  check is done, and filename_base is assumed to be absolute.
+ * TODO: make this a proactive check: make sure the file can be
+ *  opened, eg, do a create/delete on the filename to be returned.
+ */
+BOOL
+get_unique_filename(const WCHAR* directory,
+                    const WCHAR* filename_base,
+                    const WCHAR* file_type,
+                    WCHAR* filename_buffer, 
+                    UINT maxlen);
+
+DWORD
+delete_file_rename_in_use(WCHAR *filename);
+
+DWORD
+delete_file_on_boot(WCHAR *filename);
+
+DWORD
+delete_tree(const WCHAR *path);
+
+DWORD
+copy_file_permissions(WCHAR *filedst, WCHAR *filesrc);
+
+DWORD
+get_platform(DWORD *platform);
+
+BOOL
+is_wow64(HANDLE hProcess);
+
+BOOL
+using_system32_for_preinject(const WCHAR *preinject);
+
+DWORD
+get_preinject_path(WCHAR *buf, int nchars, BOOL force_local_path, BOOL short_path);
+
+DWORD
+get_preinject_name(WCHAR *buf, int nchars);
+
+void
+set_dr_platform(dr_platform_t platform);
+
+/* return DR_PLATFORM_32BIT or DR_PLATFORM_64BIT */
+dr_platform_t 
+get_dr_platform();
+
+/* PR 244206: these flags should be used for all Reg{Create,Open,Delete}KeyEx calls
+ * for HKLM\Software keys
+ */
+DWORD
+platform_key_flags();
+
+/* PR 244206: use this instead of RegDeleteKey for HKLM\Software keys */
+DWORD
+delete_product_key(HKEY hkey, LPCWSTR subkey);
+
+DWORD
+create_root_key();
+
+DWORD
+destroy_root_key();
+
+/* note that this checks the opstring against the
+ *  version of core that matches this build, NOT the version
+ *  of the core that's actually installed! */
+BOOL
+check_opstring(const WCHAR *opstring);
+
+/* acquires the privileges necessary to perform tasks like detach, 
+ *  nudge, etc. */
+DWORD
+acquire_privileges();
+
+/* said privileges should always be released after usage */
+DWORD
+release_privileges();
+
+void
+ensure_directory_exists_for_file(WCHAR *filename);
+
+void
+mkdir_with_parents(const WCHAR *dirname);
+
+DWORD
+write_file_contents(WCHAR *path, char *contents, BOOL overwrite);
+
+DWORD
+write_file_contents_if_different(WCHAR *path, char *contents, BOOL *changed);
+
+DWORD
+read_file_contents(WCHAR *path, char *contents, 
+                   UINT maxchars, UINT *needed);
+
+const WCHAR *
+get_dynamorio_home();
+
+const WCHAR *
+get_dynamorio_logdir();
+
+DWORD
+setup_installation(const WCHAR *path, BOOL overwrite);
+
+DWORD
+setup_cache_shared_directories(const WCHAR *cache_root);
+DWORD
+setup_cache_shared_registry(const WCHAR *cache_root, 
+                            ConfigGroup *policy);
+
+DWORD
+set_registry_permissions_for_user(WCHAR *hklm_keyname, WCHAR *user);
+
+/* used by get_violation_info() */
+typedef struct _VIOLATION_INFO {
+    DWORD         flags; /* IN, NYI */
+    const WCHAR  *report; /* OUT, Filename of generated report file. NULL if error.*/
+    WCHAR         buf[MAX_PATH]; /* space for filename */
+} VIOLATION_INFO;
+  
+/* Takes in a MSG_SEC_FORENSICS event log record pevlr and generates a report file
+ * (info->report) as specifed by the flags field in info.  On success returns
+ * ERROR_SUCCESS, else returns a failure code. */
+DWORD
+get_violation_info(EVENTLOGRECORD *pevlr, /* INOUT */ VIOLATION_INFO *info);
+
+/* The final canary code is (FAIL_CODE << 16) | (TEST_TYPE && 0xffff) */
+#define CANARY_UNABLE_TO_TEST            1
+#define CANARY_SUCCESS                   0
+#define CANARY_FAIL_HUNG                -1
+#define CANARY_FAIL_CRASH               -2
+#define CANARY_FAIL_VIOLATION           -3
+#define CANARY_FAIL_DR_ERROR            -4
+#define CANARY_FAIL_APP_INIT_INJECTION  -5
+#define CANARY_FAIL_EARLY_INJECTION     -6
+/* future error codes */
+#define CANARY_TEST_TYPE_NATIVE      1
+#define CANARY_TEST_TYPE_THIN_CLIENT 2
+#define CANARY_TEST_TYPE_CLIENT      3
+#define CANARY_TEST_TYPE_MF          4
+/* future types */
+#define GET_CANARY_CODE(test_type, fail_code) ((fail_code << 16) | (test_type && 0xffff))
+
+#define CANARY_RUN_NO_REQUIRE_PASS(run) (run|(run << 16))
+#define CANARY_RUN_REQUIRES_PASS(run, flags) (((flags >> 16) & run) == 0)
+#define CANARY_RUN_NATIVE              0x0001
+#define CANARY_RUN_THIN_CLIENT_INJECT  0x0002
+#define CANARY_RUN_THIN_CLIENT         0x0004
+#define CANARY_RUN_CLIENT              0x0008
+#define CANARY_RUN_MF                  0x0010
+
+#define CANARY_RUN_FLAGS_DEFAULT \
+    (CANARY_RUN_NATIVE | CANARY_RUN_THIN_CLIENT_INJECT | CANARY_RUN_THIN_CLIENT | \
+     CANARY_RUN_CLIENT | CANARY_RUN_NO_REQUIRE_PASS(CANARY_RUN_MF))
+/* NYI */
+#define CANARY_INFO_FLAGS_DEFAULT 0
+
+#define CANARY_URL_SIZE 20 /* NYI so arbitrary */
+#define CANARY_MESSAGE_SIZE 1024 /* NYI so arbitrary */
+typedef struct _CANARY_INFO {
+    DWORD         run_flags; /* tests to run */
+    DWORD         info_flags; /* info to gather, NYI */
+    int           canary_code; /* canary return code, like an NTSTATUS */ 
+    const WCHAR  *report; /* OUT, filename of generated report file */
+    const WCHAR  *url; /* OUT, url string to use for querying determina */
+    const WCHAR  *msg; /* OUT, msg to display to user */
+    WCHAR         buf_report[MAX_PATH]; /* space for report filename */
+    WCHAR         buf_url[CANARY_URL_SIZE]; /* space for url */
+    WCHAR         buf_message[CANARY_MESSAGE_SIZE]; /* space for use message */
+    /* Used by DRcontrol to inject faults, FIXME get rid of these and the flags and
+     * go to a more data driven model. Other users should set fault_run to 0. */
+    DWORD         fault_run;
+    WCHAR        *canary_fault_args;
+} CANARY_INFO;
+/* NOTE - arbitrary value, but shouldn't be -1 (core kill_proc value) or overlapping
+ * an NTSTATUS (such as ACCESS_DENIED etc.). */
+#define CANARY_PROCESS_EXP_EXIT_CODE 0
+
+/* Returns TRUE if the canary tests succeeded and protection should be enabled. Returns
+ * FALSE if one of the canary tests failed and therefore protection should not be
+ * enabled. Flags fields of info are used to specify which tests to run and what 
+ * information to gather, other fields are used to return additional information
+ * to the caller. */
+BOOL
+run_canary_test(/* INOUT */ CANARY_INFO *info, WCHAR *version_msg);
+
+/* Like run_canary_test(), except takes in as args what run_canary_test() implicitly
+ * finds via the registry for greater customization. */
+BOOL
+run_canary_test_ex(FILE *file, /* INOUT */ CANARY_INFO *info,
+                   const WCHAR *scratch_folder, const WCHAR *canary_process);
+
+# ifdef _DEBUG
+
+extern int debuglevel;
+extern int abortlevel;
+
+void
+set_debuglevel(int level);
+
+void
+set_debugfile(FILE *fp);
+
+void
+set_abortlevel(int level);
+
+#  define DL_FATAL    0
+#  define DL_ERROR    2
+#  define DL_WARN     4
+#  define DL_INFO     6
+#  define DL_VERB     8
+#  define DL_FINEST   10
+
+#  pragma warning(disable : 4127)
+
+#  ifndef EXIT_ON_ASSERT
+#    define EXIT_ON_ASSERT TRUE
+#  endif
+
+#  ifndef ASSERTION_EXPRESSION
+#    define  ASSERTION_EXPRESSION(msg) _ASSERTE(0)
+#  endif
+ 
+#  define DO_ASSERT_EXPR(msg, expr, handle, handler)  {         \
+    if ( ! (expr) ) {                                           \
+        WCHAR ___buf[MAX_PATH];                                 \
+        _snwprintf(___buf, MAX_PATH, L"%S:%d [%S]",             \
+                   __FILE__, __LINE__, msg);                    \
+        if (handle) {                                           \
+            handler                                             \
+        }                                                       \
+        else if (EXIT_ON_ASSERT) {                              \
+            fprintf(stderr, "ASSERT: %S\n", ___buf);            \
+            exit(-1);                                           \
+        }                                                       \
+        else {                                                  \
+            ASSERTION_EXPRESSION(msg);                          \
+        }                                                       \
+    }                                                           \
+}
+
+#  define NULL_HANDLER ;
+
+#  define DO_ASSERT(expr) DO_ASSERT_EXPR(#expr, expr, 0, NULL_HANDLER)
+
+#  define DO_ASSERT_HANDLE(expr, handler) DO_ASSERT_EXPR(#expr, expr, 1, handler)
+
+#  define DO_DEBUG(l,expr)  {                                   \
+    if ( (l) <= debuglevel ) {                                  \
+        expr                                                    \
+    }                                                           \
+    if ( (l) <= abortlevel ) {                                  \
+        DO_ASSERT_EXPR("DEBUG failure", 0, 0, NULL_HANDLER);    \
+    }                                                           \
+    fflush(stdout);                                             \
+    fflush(stderr);                                             \
+}
+
+#  define DO_ASSERT_WSTR_EQ(s1, s2)             \
+    DO_ASSERT(s1 != NULL);                      \
+    DO_ASSERT(s2 != NULL);                      \
+    DO_ASSERT(0 == wcscmp(s1, s2))
+
+#  define DO_ASSERT_STR_EQ(s1, s2)              \
+    DO_ASSERT(s1 != NULL);                      \
+    DO_ASSERT(s2 != NULL);                      \
+    if (s1 != NULL && s2 != NULL) { DO_ASSERT(0 == strcmp(s1, s2)); }
+
+#  define CHECKED_OPERATION(expr) {             \
+    DWORD res = expr;                           \
+    if (res != ERROR_SUCCESS)                   \
+        printf("res=%d\n", res);                \
+    DO_ASSERT(res == ERROR_SUCCESS);            \
+}
+
+#define LAUNCH_APP_WAIT_HANDLE(relpath, pidvar, wait, handle) { \
+    STARTUPINFO si = { 0 };                                     \
+    PROCESS_INFORMATION pi = { 0 };                             \
+    WCHAR cmdl[MAX_PATH];                                       \
+    _snwprintf(cmdl, MAX_PATH, L"%s", relpath);                 \
+    DO_ASSERT(CreateProcess(NULL,                               \
+                            cmdl,                               \
+                            NULL, NULL, FALSE,                  \
+                            0,                                  \
+                            NULL,                               \
+                            NULL,                               \
+                            &si, &pi));                         \
+    *pidvar = pi.dwProcessId;                                   \
+    DO_DEBUG(DL_VERB,                                           \
+             printf("Launched %d=%S\n", *pidvar, cmdl);         \
+             );                                                 \
+    if (wait) {                                                 \
+        DWORD waitres = WaitForSingleObject(pi.hProcess, 5000); \
+        DO_ASSERT(waitres == WAIT_OBJECT_0);                    \
+    }                                                           \
+    else {                                                      \
+        Sleep(100);                                             \
+    }                                                           \
+    if (handle == NULL) {                                       \
+        CloseHandle(pi.hThread);                                \
+        CloseHandle(pi.hProcess);                               \
+    }                                                           \
+    else {                                                      \
+        *handle = pi.hProcess;                                  \
+    }                                                           \
+}
+
+#define LAUNCH_APP_HANDLE(relpath, pidvar, handle) LAUNCH_APP_WAIT_HANDLE(relpath, pidvar, FALSE, handle)
+#define LAUNCH_APP_WAIT(relpath, pidvar, wait) LAUNCH_APP_WAIT_HANDLE(relpath, pidvar, wait, dummy)
+#define LAUNCH_APP(relpath, pidvar) LAUNCH_APP_WAIT(relpath, pidvar, FALSE)
+#define LAUNCH_APP_AND_WAIT(relpath, pidvar) LAUNCH_APP_WAIT(relpath, pidvar, TRUE)
+
+#  define TERMINATE_PROCESS(pid)  {             \
+    DO_DEBUG(DL_VERB,                           \
+             printf("terminating %d\n", pid);   \
+             );                                 \
+    terminate_process(pid);                     \
+    Sleep(100);                                 \
+}
+
+#  define VERIFY_UNDER_DR(pid)  {                       \
+    int stat = under_dynamorio(pid);                    \
+    DO_ASSERT(stat != DLL_NONE && stat != DLL_UNKNOWN); \
+}
+
+#  define VERIFY_NOT_UNDER_DR(pid)  {           \
+    int stat = under_dynamorio(pid);            \
+    DO_ASSERT(stat == DLL_NONE);                \
+}
+
+
+DWORD
+load_test_config(const char *snippet, BOOL use_hotpatch_defs);
+
+void
+get_testdir(WCHAR *buf, UINT maxchars);
+
+/* quick helpers for unit tests */
+BOOL
+check_for_event(DWORD type, WCHAR *exename, ULONG pid, 
+                WCHAR *s3, WCHAR *s4, UINT maxchars);
+void
+reset_last_event();
+
+void
+show_all_events();
+
+# else //#ifdef DEBUG
+
+#  define DO_DEBUG(l,x)
+#  define DO_ASSERT(x)
+
+#  define CHECKED_OPERATION(expr) {             \
+    DWORD res = expr;                           \
+    if (res != ERROR_SUCCESS)                   \
+        return res;                             \
+}
+ 
+# endif /* DEBUG */
+
+/* alignment helpers, alignment must be power of 2 */
+#define ALIGNED(x, alignment) ((((INT_PTR)x) & ((alignment)-1)) == 0)
+#define ALIGN_BACKWARD(x, alignment) (((INT_PTR)x) & (~((alignment)-1)))
+#define ALIGN_FORWARD(x, alignment) \
+    ((((INT_PTR)x) + ((alignment)-1)) & (~((alignment)-1)))
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/make/compiler.mk b/make/compiler.mk
new file mode 100644
index 000000000..8cac44874
--- /dev/null
+++ b/make/compiler.mk
@@ -0,0 +1,379 @@
+# **********************************************************
+# Copyright (c) 2007-2009 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# use toolchain or buildtools if available
+# see http://wiki.eng.vmware.com/Build/Toolchain/Access for toolchain setup
+#   view = //toolchain/main/win32/winsdk-6.1.6000/...
+#   server = perforce-toolchain.eng.vmware.com:1666
+# FIXME: use toolchain on linux -- PR 234917
+
+# For windows builds, our makefiles assume cygwin is available
+# and all necessary utilities are on the path.  For building
+# on a non-cygwin windows machine, use the .bat scripts in the
+# 'batch' directory.  Those take care of setting up the cygwin-
+# related registry settings (i.e., mount points), specifying
+# the path, and launching a cygwin bash shell with the right
+# build command.  Note that we've been unable to get all the
+# cygwin tools we need added to toolchain (xref PR 233837).
+# Therefore, we have our own copy of cygwin utils in our local
+# 'toolchain' module.  You can specify the location of that 
+# module with the DYNAMORIO_TOOLCHAIN var.  If unspecified,
+# we assume it's at ../toolchain.
+
+ifndef DYNAMORIO_MAKE
+  $(error DYNAMORIO_MAKE undefined. How did you include compiler.mk?)
+endif
+
+# Only Windows should have the OS env var set
+# We can't use uname since we need to avoid hardcoded shell cmds
+ifeq ($(OS),)
+  MACHINE := linux
+else
+  MACHINE := win32
+endif
+
+#
+# Variables for shell utilities
+#
+SHELL := /bin/bash
+
+# These all come from PATH set up for either local cygwin or
+# toolchain cygwin.  For Linux we swap in toolchain versions,
+# if available, below.
+AWK      := awk
+CAT      := cat
+CHMOD    := chmod
+CP       := cp -f
+CUT      := cut
+CYGPATH  := cygpath
+DATE     := date
+DIFF     := diff
+ECHO     := echo
+GREP     := grep
+HEAD     := head
+LS       := ls
+MKDIR    := mkdir
+MV       := mv -f
+PERL     := perl
+RM       := rm -f
+SED      := sed
+STRINGS  := strings
+TAIL     := tail
+TEE      := tee
+WC       := wc
+XARGS    := xargs
+# the following are for building our docs
+# PR 340793: toolchain cygwin doxygen is version 1.5.1, which is
+# missing some features vs 1.5.3+: no defines in Globals list, e.g.
+DOXYGEN  := doxygen
+MOGRIFY  := mogrify
+FIG2DEV  := fig2dev
+# for tools/DRinstall/
+OD       := od
+
+#
+# Windows
+#
+ifeq ($(MACHINE), win32)
+  ifdef ENABLE_TOOLCHAIN
+    ifneq ($(ENABLE_TOOLCHAIN),1)
+      # we don't support the old bora-winroot
+      $(error !ENABLE_TOOLCHAIN is not supported)
+    endif
+  endif
+
+  #
+  # Build variables
+  # 
+  I := /I
+  D := /D
+  U := /U
+  L := /LIBPATH:
+  INCR    := /c
+  OUT     := /Fo
+  EXEOUT  := /Fe
+  # there's no way to specify a non-default name with /P
+  PREPROC := /E
+  CPPOUT  := >
+  CPP_SFX :=i
+  CPP_KEEP_COMMENTS := /C
+  CPP_NO_LINENUM := /EP
+
+  ifdef TCROOT
+    TCROOT := $(shell $(CYGPATH) -ma '$(TCROOT)')
+
+    ifeq ($(ARCH), x64)
+      BINSUBDIR := /x86_x64
+      LIBSUBDIR := /x64
+      # should rename this one: toolchain bug
+      MFCLIBSUBDIR := /amd64
+    else
+      BINSUBDIR :=
+      LIBSUBDIR :=
+      MFCLIBSUBDIR :=
+    endif
+    # xref bora/make/mk/defs-compilersWin32.mk
+    ifeq ($(USE_VC71),1)
+      WROOT_BASE := $(TCROOT)/win32/visualstudio-2003
+      WROOT_VC := $(WROOT_BASE)/VC7
+      WROOT_PSDK := $(WROOT_VC)/PlatformSDK
+      WROOT_SDKTOOLS := $(WROOT_BASE)/Common7/Tools/Bin
+      MFCDIR := atlmfc
+    else
+     ifeq ($(USE_VC60),1)
+       WROOT_BASE := $(TCROOT)/win32/visual-studio-98
+       WROOT_VC := $(WROOT_BASE)/vc98
+       # no SDK, so missing Common/MSDev98/Bin for rc.exe, ml.exe!
+       # Below we use the vc7 versions
+       WROOT_PSDK := $(WROOT_VC)
+       WROOT_SDKTOOLS := $(WROOT_VC)/Bin
+       MFCDIR := MFC
+     # windows vista sdk compiler is the default
+     else
+       WROOT_BASE := $(TCROOT)/win32/winsdk-6.1.6000
+       WROOT_VC := $(WROOT_BASE)/VC
+       WROOT_PSDK := $(WROOT_BASE)
+       WROOT_SDKTOOLS := $(WROOT_PSDK)/Bin
+       MFCDIR := atlmfc
+     endif
+    endif
+    # we are not setting INCLUDE or LIB env vars, so we set all paths here
+    MSINCLUDE := $(I)'$(shell $(CYGPATH) -wa "$(WROOT_PSDK)/Include")' \
+                 $(I)'$(shell $(CYGPATH) -wa "$(WROOT_VC)/INCLUDE")' \
+                 $(I)'$(shell $(CYGPATH) -wa "$(WROOT_VC)/$(MFCDIR)/include")'
+    MSLIB     := $(L)'$(shell $(CYGPATH) -wa "$(WROOT_PSDK)/Lib$(LIBSUBDIR)")' \
+                 $(L)'$(shell $(CYGPATH) -wa "$(WROOT_VC)/LIB$(LIBSUBDIR)")' \
+                 $(L)'$(shell $(CYGPATH) -wa "$(WROOT_VC)/$(MFCDIR)/lib$(MFCLIBSUBDIR)")'
+    # for open-source release we can't distribute ntdll.lib so point at it
+    # note that we want the oldest ntdll.lib for maximum compatibility
+    DDK_BASE := $(TCROOT)/win32/winddk-6000.16386
+    NTDLL_LIBPATH_X86 := $(DDK_BASE)/lib/w2k/i386
+    NTDLL_LIBPATH_X64 := $(DDK_BASE)/lib/wnet/amd64
+    # we do not set PATH and rely on mspdb80.dll and mspdbsrv.exe being
+    # in same dir as cl.exe.
+    CC  := $(WROOT_VC)/Bin$(BINSUBDIR)/cl.exe /nologo
+    CPP := $(CC)
+    LD  := $(WROOT_VC)/Bin$(BINSUBDIR)/link.exe /nologo
+    ifeq ($(ARCH), x64)
+      AS := $(WROOT_VC)/Bin$(BINSUBDIR)/ml64.exe /nologo
+    else
+      AS := $(WROOT_VC)/Bin$(BINSUBDIR)/ml.exe /nologo
+    endif
+    MC := $(WROOT_SDKTOOLS)/mc.exe
+    MT := $(WROOT_SDKTOOLS)/mt.exe
+    ifeq ($(USE_VC60),1)
+      # FIXME: ask toolchain people to add these; for now we use vc7 versions
+      RC := $(TCROOT)/win32/visualstudio-2003/Common7/Tools/Bin/rc.exe
+      ML := $(TCROOT)/win32/visualstudio-2003/VC/Bin/ml.exe
+    else
+      RC := $(WROOT_SDKTOOLS)/rc.exe
+      ML := $(WROOT_VC)/Bin/ml.exe
+    endif
+    AR := $(WROOT_VC)/Bin$(BINSUBDIR)/lib.exe /nologo
+    BIND := $(WROOT_SDKTOOLS)/bind.exe
+    DUMPBIN := $(WROOT_VC)/Bin$(BINSUBDIR)/dumpbin.exe
+  else
+    ifndef BUILDTOOLS
+      $(error compiler.mk requires TCROOT or BUILDTOOLS)
+    endif
+    ifeq ($(USE_VC60),1)
+      # note that to build with vc6.0sp5 you'll need to re-instate the separate
+      # use of cpp to handle vararg macros
+      COMPILER := VC/6.0sp5
+    else
+      ifeq ($(USE_VC71),1)
+        COMPILER := VC/7.1
+      else
+        # default to 8.0
+        COMPILER := VC/8.0
+      endif
+    endif
+    include $(BUILDTOOLS)/$(COMPILER)/Setup.mk
+    MSINCLUDE := 
+    MSLIB := 
+    LD += /nologo
+    AR := lib.exe /nologo
+    CPP := $(CC)
+  endif # TCROOT
+
+#
+# Linux
+#
+else
+  # 
+  # Build variables
+  #
+  # -Iquote adds a dir to search path for "" only and not for <>
+  # which is needed when we have conflicting names like link.h!
+  I := -iquote
+  D := -D
+  U := -U
+  L := -L
+  INCR    := -c
+  OUT     := -o 
+  EXEOUT  := -o 
+  PREPROC := -E
+  CPPOUT  := -o 
+  CPP_SFX :=i
+  CPP_KEEP_COMMENTS := -C
+  CPP_NO_LINENUM := -P
+
+  # We hardcode to use a gcc in /build/toolchain. Users need 
+  # to mount /build/toolchain or provide a local copy of the 
+  # neccessary toolchain modules in /build/toolchain. Note that
+  # gcc is configured to look for its headers and libraries in
+  # /build/toolchain, so we do not need to specify any of those
+  # paths on the gcc command line.
+  ifeq ($(USE_GCC_3.4.6),1)
+    # Some components (spec tests) need older compiler version.
+    # For this we need :
+    #   /build/toolchain/lin32/gcc-3.4.6
+    #   /build/toolchain/lin32/binutils-2.16.1-vt
+    #   /build/toolchain/lin32/glibc-2.2.5-44  (note - same as for 4.1.2-2)
+    GCCDIR := /build/toolchain/lin32/gcc-3.4.6
+    BINUTILS := /build/toolchain/lin32/binutils-2.16.1-vt
+  else
+    # For Linux, we'll default to the same compiler as the monitor.
+    # (https://wiki.eng.vmware.com/Build/Toolchain/Compilers)
+    # We therefore need the following:
+    #   /build/toolchain/lin32/gcc-4.1.2-2
+    #   /build/toolchain/lin32/binutils-2.17.50.0.15
+    #   /build/toolchain/lin32/glibc-2.2.5-44
+    #   /build/toolchain/lin64/glibc-2.3.2-95.44
+    GCCDIR := /build/toolchain/lin32/gcc-4.1.2-2
+    BINUTILS := /build/toolchain/lin32/binutils-2.17.50.0.15
+  endif
+
+  ifeq ($(ARCH), x64)
+    BINPREFIX := x86_64
+  else
+    BINPREFIX := i686
+  endif
+
+  CC  := $(GCCDIR)/bin/$(BINPREFIX)-linux-gcc
+  CPP := $(GCCDIR)/bin/$(BINPREFIX)-linux-cpp
+  LD  := $(GCCDIR)/$(BINPREFIX)-linux/bin/ld
+  AR  := $(GCCDIR)/$(BINPREFIX)-linux/bin/ar
+  SIZE := $(BINUTILS)/bin/$(BINPREFIX)-linux-size
+  STRINGS := $(BINUTILS)/bin/$(BINPREFIX)-linux-strings
+  STRIP := $(BINUTILS)/bin/$(BINPREFIX)-linux-strip
+  READELF := $(BINUTILS)/bin/$(BINPREFIX)-linux-readelf
+
+  # We require gas >= 2.18.50 for --32, --64, and the new -msyntax=intel, etc.
+  BINUTILS_2_18 := /build/toolchain/lin32/binutils-2.18.50.0.9
+  AS := $(BINUTILS_2_18)/bin/$(BINPREFIX)-linux-as
+  ASMFLAGS := -mmnemonic=intel -msyntax=intel -mnaked-reg --noexecstack
+
+  ###################################
+  #
+  # For utility apps we allow local versions for easier building by
+  # remote developers.  But, we have toolchain versions of everything for
+  # consistent, reproducible official builds.
+
+  # Since our utils are all present in lin32 and we're not linking w/
+  # any libraries from toolchain we don't bother to use
+  # the lin64 versions on an x64 machine.
+
+  COREUTILS := /build/toolchain/lin32/coreutils-5.97/bin
+  ifneq ($(wildcard $(COREUTILS)),)
+    CAT      := $(COREUTILS)/cat
+    CHMOD    := $(COREUTILS)/chmod
+    CP       := $(COREUTILS)/cp -f
+    CUT      := $(COREUTILS)/cut
+    DATE     := $(COREUTILS)/date
+    ECHO     := $(COREUTILS)/echo
+    HEAD     := $(COREUTILS)/head
+    LS       := $(COREUTILS)/ls
+    MKDIR    := $(COREUTILS)/mkdir
+    MV       := $(COREUTILS)/mv -f
+    RM       := $(COREUTILS)/rm -f
+    TAIL     := $(COREUTILS)/tail
+    TEE      := $(COREUTILS)/tee
+    WC       := $(COREUTILS)/wc
+    OD       := $(COREUTILS)/od
+  endif
+
+  TC_AWK     := /build/toolchain/lin32/gawk-3.1.5/bin
+  ifneq ($(wildcard $(TC_AWK)),)
+    AWK      := $(TC_AWK)/awk
+  endif
+  TC_DIFF    := /build/toolchain/lin32/diffutils-2.8/bin
+  ifneq ($(wildcard $(TC_DIFF)),)
+    DIFF     := $(TC_DIFF)/diff
+  endif
+  TC_GREP    := /build/toolchain/lin32/grep-2.5.1a/bin
+  ifneq ($(wildcard $(TC_GREP)),)
+    GREP     := $(TC_GREP)/grep
+  endif
+  TC_PERL    := /build/toolchain/lin32/perl-5.8.8/bin
+  ifneq ($(wildcard $(TC_PERL)),)
+    PERL     := $(TC_PERL)/perl
+  endif
+  TC_SED     := /build/toolchain/lin32/sed-4.1.5/bin
+  ifneq ($(wildcard $(TC_SED)),)
+    SED      := $(TC_SED)/sed
+  endif
+  TC_XARGS   := /build/toolchain/lin32/findutils-4.2.27/bin
+  ifneq ($(wildcard $(TC_XARGS)),)
+    XARGS    := $(TC_XARGS)/xargs
+  endif
+  # PR 340793: this older doxygen doesn't quite match 1.5.6:
+  # seems to omit globals starting w/ 't', maybe other harder-to-spot
+  # errors, but so far minor enough that we ignore them.
+  TC_DOXYGEN := /build/toolchain/lin32/doxygen-1.5.3/bin
+  ifneq ($(wildcard $(TC_DOXYGEN)),)
+    DOXYGEN  := $(TC_DOXYGEN)/doxygen
+  endif
+  TC_MOGRIFY := /build/toolchain/lin32/ImageMagick-6.4.1-5/bin
+  ifneq ($(wildcard $(TC_MOGRIFY)),)
+    MOGRIFY  := $(TC_MOGRIFY)/mogrify
+  endif
+  TC_FIG2DEV := /build/toolchain/lin32/transfig-3.2.5/bin
+  ifneq ($(wildcard $(TC_FIG2DEV)),)
+    FIG2DEV  := $(TC_FIG2DEV)/fig2dev
+  endif
+
+  #################
+  # Utilities we access by setting PATH:
+
+  # gs is used by fig2dev
+  TC_GS      := /build/toolchain/lin32/ghostscript-8.62/bin
+  # pdflatex and makeindex used by auto-generated api/docs/latex/Makefile
+  TC_TETEX   := /build/toolchain/lin32/tetex-3.0/bin
+
+  # For makezip.sh util usage
+  TC_TAR     := /build/toolchain/lin32/tar-1.15.1/bin
+  # gzip is invoked by tar -z
+  TC_GZIP    := /build/toolchain/lin32/gzip-1.2.4a/bin
+  TC_ZIP     := /build/toolchain/lin32/zip-2.32/bin
+  TC_MAKE    := /build/toolchain/lin32/make-3.81/bin
+  TC_FIND    := /build/toolchain/lin32/findutils-4.2.27/bin
+
+endif # win32
diff --git a/suite/nightly/cygwin_nightly.bat b/suite/nightly/cygwin_nightly.bat
new file mode 100644
index 000000000..f879dbe41
--- /dev/null
+++ b/suite/nightly/cygwin_nightly.bat
@@ -0,0 +1,35 @@
+REM **********************************************************
+REM Copyright (c) 2005-2006 VMware, Inc.  All rights reserved.
+REM **********************************************************
+
+REM Redistribution and use in source and binary forms, with or without
+REM modification, are permitted provided that the following conditions are met:
+REM 
+REM * Redistributions of source code must retain the above copyright notice,
+REM   this list of conditions and the following disclaimer.
+REM 
+REM * Redistributions in binary form must reproduce the above copyright notice,
+REM   this list of conditions and the following disclaimer in the documentation
+REM   and/or other materials provided with the distribution.
+REM 
+REM * Neither the name of VMware, Inc. nor the names of its contributors may be
+REM   used to endorse or promote products derived from this software without
+REM   specific prior written permission.
+REM 
+REM THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+REM AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+REM IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+REM ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+REM FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+REM DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+REM SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+REM CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+REM LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+REM OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+REM DAMAGE.
+
+@echo off
+
+c:
+chdir C:\cygwin\bin
+bash --login bin\do_nightly.sh
diff --git a/suite/nightly/do_nightly.sh b/suite/nightly/do_nightly.sh
new file mode 100644
index 000000000..f73b8838e
--- /dev/null
+++ b/suite/nightly/do_nightly.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+# **********************************************************
+# Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+log=$PWD/do_nightly.log
+rm $log
+cd $(cygpath -u $TCROOT)
+p4 -d $(cygpath -w $PWD) sync 2>&1 | tee $log
+cd $NIGHTLY_HOME
+p4 -d $(cygpath -w $PWD) sync 2>&1 | tee -a $log
+rm -rf core/Makefile  2>&1 | tee -a $log
+p4 -d $(cygpath -w $PWD) sync -f core/Makefile  2>&1 | tee -a $log
+
+# A descriptive name for the host OS should be passed as an
+# arg to the nightly_windows script. For example, '2000' for
+# Windows 2000, '2003' for Windows 2003, 'XP' for Windows XP
+# and 'Vista' for Vista. 
+# NOTE since some regression tests behave differently on 
+# different os versions, this string is used when verifying
+# output, so expect failures if you don't use the exact string
+# (as shown above) for your os as used by runregression
+cd $NIGHTLY_NIGHTLY
+./other_platforms/nightly_windows.sh $NIGHTLY_PLATFORM 2>&1 | tee -a $log
diff --git a/suite/nightly/east_coast_daily_rsync.sh b/suite/nightly/east_coast_daily_rsync.sh
new file mode 100644
index 000000000..485673b35
--- /dev/null
+++ b/suite/nightly/east_coast_daily_rsync.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# **********************************************************
+# Copyright (c) 2005 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# update some folders, taked on to here since it runs every night
+cd /c/dynamo/base
+cvs update -P -A -d -C
+cd /c/dynamo/bench
+cvs update -P -A -d -C
+cd /c/dynamo/full
+cvs update -P -A -d -C
+cd
+
+# Building the file lists takes a long time over the slow connection so we do
+# a subset of the full rsync every weekday night with a longer run on the
+# weekends that rsyncs different folders.  This the short run.
+
+echo ""
+date
+date >> do_rsync.log
+
+# sync builds 
+echo "rsync builds"
+rsync -rltDuv --include=/*/ --include=ver_*/ --include=lib/ --include=lib**/* --include=config/ --include=config**/* --include=msi/ --include=msi/Data1.cab --include=msi/SecureCore.msi --include=tools/ --exclude=tools/external**/* --include=tools**/* --exclude=* /mnt/deter-build_nightly/ /e/all_shares/builds/ >> do_rsync.log
+echo "done"
+
+# sync the weekly builds
+date
+echo "rsync weekly builds"
+rsync -rltDuv /mnt/deter-build_weekly/ /e/all_shares/builds/weekly/ >> do_rsync.log
+
+# get latest pdb's, just dynamorio.dll's to speed things up
+date
+echo "rsync product symbols (just dynamorio.dll pdbs) <- west coast"
+rsync -rltDuv /mnt/deter-build_symbols/product/dynamorio.pdb/ /e/all_shares/symbols/symbols/product/dynamorio.pdb/ >> do_rsync.log
+echo "done"
+
+# sync bugs folder
+date
+echo "rsync bugs <- west coast"
+rsync -rltDuv /mnt/bugs/ /e/all_shares/bugs/ >> do_rsync.log
+echo "done"
+
+date
+# fixup permission, fixme why do we keep having to do this? also modify issues
+chmod -R aug+rxw /e/all_shares/builds
+chmod -R aug+rxw /e/all_shares/symbols/symbols/Product
+chmod -R aug+rxw /e/all_shares/bugs
+date
+echo "All Finished"
+date >> do_rsync.log
+echo "All Finished" >> do_rsync.log
\ No newline at end of file
diff --git a/suite/nightly/east_coast_weekend_rsync.sh b/suite/nightly/east_coast_weekend_rsync.sh
new file mode 100644
index 000000000..a02e8d87c
--- /dev/null
+++ b/suite/nightly/east_coast_weekend_rsync.sh
@@ -0,0 +1,98 @@
+#!/bin/bash
+
+# **********************************************************
+# Copyright (c) 2005 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# update some folders, taked on to here since it runs every night
+cd /c/dynamo/base
+cvs update -P -A -d -C
+cd /c/dynamo/bench
+cvs update -P -A -d -C
+cd /c/dynamo/full
+cvs update -P -A -d -C
+cd
+
+# Building the file lists takes a long time over the slow connection so we do
+# a subset of the full rsync every weekday night with a longer run on the
+# weekends that rsyncs different folders.  This the long run.
+
+echo ""
+date
+date >> do_rsync.log
+
+# sync bugs folder both ways
+date
+echo "rsync bugs -> west coast"
+rsync -rltDuv /e/all_shares/bugs/ /mnt/bugs/ >> do_rsync.log
+echo "done"
+date
+echo "rsync bugs <- west coast"
+rsync -rltDuv /mnt/bugs/ /e/all_shares/bugs/ >> do_rsync.log
+echo "done"
+
+# sync builds, long run so get them from the build machine (official source)
+date
+echo "rsync builds"
+rsync -rltDuv --include=/*/ --include=ver_*/ --include=lib/ --include=lib**/* --include=config/ --include=config**/* --include=msi/ --include=msi/Data1.cab --include=msi/SecureCore.msi --include=tools/ --exclude=tools/external**/* --include=tools**/* --exclude=* /mnt/deter-build_nightly/ /e/all_shares/builds/ >> do_rsync.log
+echo "done"
+
+# sync the weekly builds
+date
+echo "rsync weekly builds"
+rsync -rltDuv /mnt/deter-build_weekly/ /e/all_shares/builds/weekly/ >> do_rsync.log
+
+# sync product symbols
+date
+echo "rsync product symbols <- west coast"
+rsync -rltDuv /mnt/deter-build_symbols/product/ /e/all_shares/symbols/symbols/product/ >> do_rsync.log
+date
+echo "done"
+echo "rsync product symbols -> west coast"
+rsync -rltDuv /e/all_shares/symbols/symbols/product/ /mnt/deter-build_symbols/product/ >> do_rsync.log
+
+# if there is time synch the os symbols
+date
+echo "rsync os symbols -> west coast"
+rsync -rltDuv /e/all_shares/symbols/symbols/os/ /mnt/deter-build_symbols/os/ >> do_rsync.log
+echo "done"
+date
+echo "rsync os symbols <- west coast"
+rsync -rltDuv /mnt/deter-build_symbols/os/ /e/all_shares/symbols/symbols/os/ >> do_rsync.log
+echo "done"
+
+date
+# fixup permission, fixme why do we keep having to do this? also modify issues
+chmod -R aug+rxw /e/all_shares/builds
+chmod -R aug+rxw /e/all_shares/symbols/symbols/Product
+chmod -R aug+rxw /e/all_shares/bugs
+date
+echo "All Finished"
+date >> do_rsync.log
+echo "All Finished" >> do_rsync.log
\ No newline at end of file
diff --git a/suite/nightly/linux-memory.sh b/suite/nightly/linux-memory.sh
new file mode 100644
index 000000000..6a775e9df
--- /dev/null
+++ b/suite/nightly/linux-memory.sh
@@ -0,0 +1,95 @@
+#!/bin/bash
+
+# **********************************************************
+# Copyright (c) 2005-2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# won't src .bashrc so set env var here
+export CVS_RSH=ssh
+remotehost="amdxp"
+remotepath="/home/sshrsa/nightly"
+localtree="/home/sshrsa/dr/trees/nightly"
+remotetree="/home/sshrsa/dr/trees/nightly"
+remotetools="/home/sshrsa/dr/tools"
+rsyncout="rsync -e ssh -auvz --delete"
+rsynclinkref="rsync -e ssh -Lptgouvz"
+rsyncin="rsync -e ssh -auvz"
+shellrc="~/.bashrc"
+remotescript="./memory_xp.sh"
+export CVSROOT=/mnt/data/cvseast
+
+mailto="eng-nightly-results@determina.com"
+
+dstamp=`date | awk '{print $6 "-" $2$3}'`
+prefix="memory-"$dstamp
+log=$prefix".log"
+email=$prefix".email"
+
+if [ -e $log ]; then echo "Error: $log already exists"; exit -1; fi
+
+ssh ${remotehost} "rm -rf ${remotetree}/src" > $log 2>&1 
+${rsynclinkref} ${remotescript} ${remotehost}:${remotepath} >> $log 2>&1
+curdir=`pwd`;
+pushd ${localtree} >> $log 2>&1
+rm -rf src tools >> $curdir/$log 2>&1
+cvs -d $CVSROOT co src tools >> $curdir/$log 2>&1
+popd >> $log 2>&1
+${rsyncout} ${localtree}/src ${remotehost}:${remotetree} >> $log 2>&1
+${rsyncout} ${localtree}/tools ${remotehost}:${remotetools} >> $log 2>&1
+ssh ${remotehost} "${remotepath}/${remotescript}" >> $log 2>&1 
+
+# create summary for mailing
+echo "From: $mailto" > $email
+echo "To: $mailto" >> $email
+echo "Errors-To: derek@determina.com" >> $email
+echo "Subject: desktop memory breakdown "$dstamp >> $email
+echo "desktop memory and performance breakdown for "`date` >> $email
+echo "details at dereksha:"`pwd`/$log >> $email
+echo "" >> $email
+# calculate % change since last result
+prev=`ls -1t *.email | head -2 | tail -1`
+echo "Change in total committed memory from last run $prev:" >> $email
+tmpfile=$email-tmp
+grep '100\.00' $prev | awk '{print $(NF-1)}' > $tmpfile
+grep '100\.00' $log | awk '{print $(NF-1)}' | paste - $tmpfile | awk '{printf "Previous: %10d => %5.1f%% change\n", $2, 100*($1-$2)/$1}' >> $email
+rm $tmpfile
+echo "" >> $email
+# pull out title, /usr/bin/time results, and heapacct breakdown
+grep -E '^@|elapsed|RSS,|%.* KB' $log >> $email
+echo "" >> $email
+
+# mail summary out
+#   sendmail on RH8 puts in the X-Authentication-Warning, despite the manual's
+#   claims that turning off authwarnings or editing the trusted user's file will
+#   prevent that -- in any case not using -f and instead just using a From:
+#   seems to work and avoids the warning
+#   (but, I remember in the past it caused problems w/ picky mailservers?)
+# update: using -f to avoid alink mail provider's stupid rejection of unknown
+# senders
+/usr/sbin/sendmail -f$mailto $mailto < $email
diff --git a/suite/nightly/linux-nightly.sh b/suite/nightly/linux-nightly.sh
new file mode 100755
index 000000000..6bc21e944
--- /dev/null
+++ b/suite/nightly/linux-nightly.sh
@@ -0,0 +1,123 @@
+#!/bin/bash
+
+# **********************************************************
+# Copyright (c) 2004-2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+###########################################################################
+##
+## nightly regression suite script
+##
+
+# assumptions: that you've already done this:
+#   cvs co benchmarks spec2kdata tools
+#   ln -s `pwd`/spec2kdata benchmarks/spec2000/data
+#   ln -s `pwd`/benchmarks/spec2000/Makefile.dynamo benchmarks/spec2000/Makefile.defines
+#   ln -s `pwd`/tools/runstats.c benchmarks/spec2000/tools
+# actually those symlinks need to be made relative to work on the remote machine!
+
+# To avoid impacting the cvs server we run the tests on a separate machine.
+# For security reasons we don't want that machine being able to ssh to dereksha
+# for cvs updates, so we push from dereksha instead.
+# We run this as a lower-privileged user with blank-rsa access to the other machine.
+
+# FIXME: correlate w/ windows build numbers?
+# RESOLUTION: no, windows builds are separate, these linux runs will
+# only be labeled by date.
+
+# won't src .bashrc so set env var here
+export CVS_RSH=ssh
+remotehost="rhel4-as"
+remotepath="/home/sshrsa/regression"
+# note we do NOT use -u, to support changing to older branches
+rsyncout="rsync -e ssh -avz --delete"
+rsynclinkref="rsync -e ssh -Lptgovz"
+rsyncin="rsync -e ssh -auvz"
+shellrc="~/.bashrc"
+remotescript="./linux-slave.sh"
+remotehelper="./utreport.pl"
+
+mailto="eng-nightly-results@determina.com"
+
+dstamp=`date | awk '{print $6 "-" $2$3}'`
+prefix="regression-"$dstamp
+masterlog=$prefix"/"$prefix".masterlog"
+email=$prefix"/"$prefix".email"
+
+if [ -e $prefix ]; then echo "Error: $prefix already exists"; exit -1; fi
+mkdir $prefix
+
+# get fresh copies of these
+if [ -e suite ]; then rm -rf suite; fi
+if [ -e src ]; then rm -rf src; fi
+if [ -e tools ]; then rm -rf tools; fi
+
+cvs -d menlo:/mnt/data/cvseast co suite tools > $masterlog 2>&1
+cvs -d menlo:/mnt/data/cvseast co src >> $masterlog 2>&1
+cd benchmarks; cvs update >> ../$masterlog 2>&1; cd ..
+
+# now mirror on remote host as this user, making sure to delete as well as update
+$rsynclinkref $remotescript $remotehost:$remotepath/ >> $masterlog 2>&1
+$rsynclinkref $remotehelper $remotehost:$remotepath/ >> $masterlog 2>&1
+$rsyncout suite/ $remotehost:$remotepath/suite >> $masterlog 2>&1
+$rsyncout src/ $remotehost:$remotepath/src >> $masterlog 2>&1
+$rsyncout tools/ $remotehost:$remotepath/tools >> $masterlog 2>&1
+$rsyncout benchmarks/ $remotehost:$remotepath/benchmarks >> $masterlog 2>&1
+
+echo "running regression now..." >> $masterlog
+
+# We must source .bashrc to get the compiler path, etc. set up
+# (cygwin bash non-interactive does not source it even if started by sshd).
+(ssh $remotehost "source $shellrc; cd $remotepath; $remotescript $prefix $mailto") >> $masterlog 2>&1
+
+echo "finished running regression" >> $masterlog
+
+# get results
+$rsyncin $remotehost:$remotepath/$prefix/ $prefix >> $masterlog 2>&1
+if [ ! -e $email ]; then echo "Error: $email not found"; exit -1; fi
+
+# mail summary out
+#   sendmail on RH8 puts in the X-Authentication-Warning, despite the manual's
+#   claims that turning off authwarnings or editing the trusted user's file will
+#   prevent that -- in any case not using -f and instead just using a From:
+#   seems to work and avoids the warning
+#   (but, I remember in the past it caused problems w/ picky mailservers?)
+#/usr/sbin/sendmail -f$mailto $mailto < $email
+/usr/sbin/sendmail -f $mailto $mailto < $email
+
+echo "checking up on windows now" >> $masterlog
+
+# check on the win32 suite
+./linux-win32checkup.sh
+
+# mirror on west coast -- do this last in case it hangs
+# send results to log, get errors on all symlinks
+# FIXME: tar all non-symlinks and just send those over?
+cp -r $prefix /mnt/fileserver/nightly/linux/ >> $masterlog 2>&1
+
diff --git a/suite/nightly/linux-slave.sh b/suite/nightly/linux-slave.sh
new file mode 100755
index 000000000..981e81bf6
--- /dev/null
+++ b/suite/nightly/linux-slave.sh
@@ -0,0 +1,97 @@
+#!/bin/bash
+
+# **********************************************************
+# Copyright (c) 2005-2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+###########################################################################
+##
+## Nightly regression suite script for linux
+##
+## This is the slave script that assumes someone has set up the latest
+## suite, src, and tools modules.  This script then runs the regression
+## tests and creates a file for emailing but does not email it.
+## That allows the master to have the option of running this script remotely
+## and doing the cvs update and emailing itself instead of the remote host,
+## or simply running on the local machine.
+##
+## ASSUMPTION: you've already checked out spec2kdata, and that the symlinks
+## from the master will work here (make them local!)
+
+summaryscript="./utreport.pl"
+
+usage="<prefix> <mailto>"
+# mandatory 2 arguments
+if [ $# -lt 2 ]; then
+    echo "Usage: $0 $usage"
+    exit 127
+fi
+prefix=$1
+mailto=$2
+
+log=$prefix"/"$prefix".log"
+email=$prefix"/"$prefix".email"
+details=$prefix"/"$prefix".details"
+logdir=$prefix"/"$prefix".001"
+
+if [ -e $prefix ]; then echo "Error: $prefix already exists"; exit -1; fi
+mkdir $prefix
+
+# we assume the modules are placed in the same dir as this script.
+if ! test -e suite ; then
+    echo "Error: cannot find suite"
+    exit 127
+fi
+if ! test -e src ; then
+    echo "Error: cannot find src"
+    exit 127
+fi
+if ! test -e tools ; then
+    echo "Error: cannot find tools"
+    exit 127
+fi
+
+export DYNAMORIO_HOME=`pwd`
+export DYNAMORIO_TOOLS=`pwd`/tools
+export DYNAMORIO_LOGDIR=.
+
+cd suite
+./runregression `pwd`/.. -long >> ../$log 2>&1
+mv regression-2* ../$prefix
+cd ..
+
+# create summary for mailing
+echo "From: $mailto" > $email
+echo "To: $mailto" >> $email
+echo "Errors-To: derek@determina.com" >> $email
+echo "Subject: linux regression results for "`date` >> $email
+echo "linux regression results for "`date` >> $email
+echo "details at dereksha:"`pwd`/$prefix >> $email
+echo "mirrored at //fileserver/nightly/linux/"$prefix >> $email
+$summaryscript $log $logdir $details >> $email
diff --git a/suite/nightly/linux-win32checkup.sh b/suite/nightly/linux-win32checkup.sh
new file mode 100755
index 000000000..64da3e143
--- /dev/null
+++ b/suite/nightly/linux-win32checkup.sh
@@ -0,0 +1,123 @@
+#!/bin/bash
+
+# **********************************************************
+# Copyright (c) 2004-2005 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+###########################################################################
+##
+## nightly windows regression run and attack suite checkup
+##
+
+mailto="eng-nightly-results@determina.com"
+
+# arg1: local email file, arg2: subject string, arg3: body file
+function send_email()
+{
+    # create email
+    echo "From: $mailto" > $1
+    echo "To: $mailto" >> $1
+    echo "Errors-To: derek@determina.com" >> $1
+    echo "Subject: $2" >> $1
+    echo "" >> $1
+    echo "This is an automatically generated email." >> $1
+    echo "" >> $1
+    cat $3 >> $1
+    echo "" >> $1
+    
+    # mail summary out
+    #   sendmail on RH8 puts in the X-Authentication-Warning, despite the manual's
+    #   claims that turning off authwarnings or editing the trusted user's file will
+    #   prevent that -- in any case not using -f and instead just using a From:
+    #   seems to work and avoids the warning
+    #   (but, I remember in the past it caused problems w/ picky mailservers?)
+    #/usr/sbin/sendmail -f$mailto $mailto < $1
+
+    /usr/sbin/sendmail -f$mailto $mailto < $1
+}
+
+# builds are all merged now, so look for latest build
+# FIXME: temporarily look only at 25xxx builds
+# in the future, check multiple builds: 22xxx and 25xxx
+latest_num=`ls -1t /mnt/build | grep '^25' | head -1`;
+latest=/mnt/build/$latest_num
+latest_log=$latest/runregression.log
+local_log=win32/$latest_num-failures.log
+local_email=win32/$latest_num-email
+
+if [ -z $latest_num ]; then
+    echo "Error: cannot find latest build: perhaps a /mnt/build mount problem?"
+    exit -1
+fi
+
+send=0
+missing=0
+if [ -e $latest_log ]; then 
+    (grep -A 5000 'Summary of results' $latest_log | grep -B 2 -E '[1-9][0-9]* failed|s in build') > $local_log 2>&1
+    if [ ! -s $local_log ]; then
+        # make sure regression log not mostly empty
+        lines=`wc $latest_log | awk '{print $1}'`;
+        if [ $lines -lt 30 ]; then
+            missing=1
+            echo "Regression log $latest_log looks incomplete" > $local_log 2>&1
+            echo "Did the core regression suite run properly?" >> $local_log 2>&1
+        else
+            echo "No tests failed" > $local_log 2>&1
+            # do not send email
+            send=0
+        fi
+    else
+        # new policy: win32 includes failure summary, so only send email if cannot
+        # find regression results!
+        send=0
+    fi
+else
+    missing=1
+    echo "Cannot find $latest_log" > $local_log 2>&1
+    echo "Did the core regression suite run at all?" >> $local_log 2>&1
+fi
+
+if [ $missing == 0 ]; then
+    echo "The following tests FAILED:" >> $local_log
+    echo "" >> $local_log
+fi
+
+if [ $send == 1 ]; then
+    send_email $local_email "PROBLEM with build $latest_num windows regression run" $local_log
+fi
+
+
+# check attack suite as well
+attack_run_dir="/mnt/qa/QA-Results/Autoattack/$latest_num-1";
+attack_log=win32/$latest_num-attack.log
+attack_email=win32/$latest_num-attack-email
+if [ ! -e $attack_run_dir ]; then
+    echo "Cannot find attack suite results $attack_run_dir" > $attack_log
+    send_email $attack_email "PROBLEM with build $latest_num attack suite" $attack_log
+fi
diff --git a/suite/nightly/memory_xp.sh b/suite/nightly/memory_xp.sh
new file mode 100644
index 000000000..9d2d53daa
--- /dev/null
+++ b/suite/nightly/memory_xp.sh
@@ -0,0 +1,166 @@
+#!/bin/bash
+
+# **********************************************************
+# Copyright (c) 2005-2007 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+###########################################################################
+## VC++ Environment Variables
+## From C:\Program Files\Microsoft Visual Studio\VC98\Bin\VCVARS32.BAT
+##
+VSBaseDirTemp=`cygpath -ms /c/Program\ Files/Microsoft\ Visual\ Studio`
+VSBaseDirBASH=`cygpath -u $VSBaseDirTemp`
+
+# Root of Visual Developer Studio Common files.
+VSCommonDirBASH="${VSBaseDirBASH}/Common"
+VSCommonDir=`cygpath -d ${VSCommonDirBASH}`
+
+# Root of Visual Developer Studio installed files.
+MSDevDirBASH="${VSCommonDirBASH}/MSDev98"
+MSDevDir=`cygpath -d ${MSDevDirBASH}`
+
+# Root of Visual C++ installed files.
+MSVCDirBASH="${VSBaseDirBASH}/VC98"
+MSVCDir=`cygpath -d ${MSVCDirBASH}`
+
+# VcOsDir is used to help create either a Windows 95 or Windows NT specific path.
+if [ "$OS" = "Windows_NT" ]; then export VcOsDir=WINNT; else export VcOsDir=WIN95; fi
+
+# Set environment for using Microsoft Visual C++ tools.
+if [ "$OS" = "Windows_NT" ]; then PATH="$MSDevDirBASH/BIN:$MSVCDirBASH/BIN:$VSCommonDirBASH/TOOLS/$VcOsDir:$VSCommonDirBASH/TOOLS:$PATH"; fi
+if [ "$OS" = "" ]; then PATH="$PATH:$MSDevDirBASH/BIN:$MSVCDirBASH/BIN:$VSCommonDirBASH/TOOLS/$VcOsDir:$VSCommonDirBASH/TOOLS:$windir/SYSTEM"; fi
+export INCLUDE="$MSVCDir\\ATL\\INCLUDE;$MSVCDir\\INCLUDE;$MSVCDir\\MFC\\INCLUDE;$INCLUDE"
+export LIB="$MSVCDir\\LIB;$MSVCDir\\MFC\\LIB;$LIB"
+
+unset VSBaseDirTemp
+unset VSBaseDirBASH
+unset VSCommonDirBASH
+unset VSCommonDir
+unset MSDevDirBASH
+unset MSDevDir
+unset MSVCDirBASH
+unset MSVCDir
+#
+###########################################################################
+
+# aliases not expanded in non-interactive shell by default
+shopt -s expand_aliases
+
+myip=`netsh interface ip show address | grep Address | awk '{print $NF}'`
+export COLONHOME=`cygpath -m ${HOME}`
+export DYNAMORIO_TREEROOT=${COLONHOME}/dr/trees
+export DYNAMORIO_HOME=${DYNAMORIO_TREEROOT}/nightly
+export DYNAMORIO_LOGDIR=${COLONHOME}/dr/logs-nightly
+export DYNAMORIO_TOOLS=${COLONHOME}/dr/tools
+export DYNAMORIO_DESKTOP=${COLONHOME}/dr/benchmarks/desktop
+
+alias meminjector='echo "DYNAMORIO_HOME=$DYNAMORIO_HOME"; time $DYNAMORIO_HOME/exports/x86_win32_dbg/drinject.exe -mem -stats ${DYNAMORIO_HOME//\//\\\\}\\exports\\x86_win32_dbg\\dynamorio.dll'
+
+cd ${DYNAMORIO_HOME}/src
+make clear
+make
+make DEBUG=0
+
+rm -rf ${DYNAMORIO_LOGDIR}
+mkdir ${DYNAMORIO_LOGDIR}
+
+function getlogdata {
+    pushd ${DYNAMORIO_LOGDIR}
+    logdir=`ls -1td *[0-9] | head -1`
+    if test -e $logdir.xml ; then
+        echo "ERROR: $logdir.xml exists"
+    fi
+    cd $logdir
+    log=`ls *exe* *EXE*`
+    echo "log is ${DYNAMORIO_LOGDIR}/$logdir/$log"
+    ${DYNAMORIO_TOOLS}/memsummary.sh $log
+    ${DYNAMORIO_TOOLS}/heapacct.pl $log
+    cd ..
+    dstamp=`date | awk '{print $6 "-" $2$3}'`
+    tar czf $dstamp-$logdir.tgz $logdir
+    rm -rf $logdir
+    popd
+}
+
+###########################################################################
+# do the runs
+
+cd ${DYNAMORIO_DESKTOP}/winword
+echo "@@@@@@@@ WINWORD NATIVE @@@@@@@@"
+make NATIVE=1 run
+echo "@@@@@@@@ WINWORD RELEASE @@@@@@@@"
+export DYNAMORIO_OPTIONS=''
+make run
+echo "@@@@@@@@ WINWORD DEBUG @@@@@@@@"
+export DYNAMORIO_OPTIONS='-loglevel 1'
+make DEBUG=1 run
+getlogdata
+echo "@@@@@@@@ WINWORD DEBUG -coarse_units @@@@@@@@"
+export DYNAMORIO_OPTIONS='-coarse_units -loglevel 1'
+make DEBUG=1 run
+getlogdata
+
+cd ${DYNAMORIO_DESKTOP}/excel
+echo "@@@@@@@@ EXCEL NATIVE @@@@@@@@"
+make NATIVE=1 run
+echo "@@@@@@@@ EXCEL RELEASE  (running w/ -C b/c of case 6626) @@@@@@@@"
+export DYNAMORIO_OPTIONS='-C'
+make run
+echo "@@@@@@@@ EXCEL DEBUG (running w/ -C b/c of case 6626) @@@@@@@@"
+export DYNAMORIO_OPTIONS='-C -loglevel 1'
+make DEBUG=1 run
+getlogdata
+echo "@@@@@@@@ EXCEL DEBUG -coarse_units (running w/ -C b/c of case 6626) @@@@@@@@"
+export DYNAMORIO_OPTIONS='-coarse_units -C -loglevel 1'
+make DEBUG=1 run
+getlogdata
+
+echo "@@@@@@@@ IEXPLORE DEBUG @@@@@@@@"
+# if IIS is running this is easiest way to get javascript to run:
+#meminjector C:\\Program\ Files\\Internet\ Explorer\\IEXPLORE.EXE http://localhost/close.html
+# on XP though we point to local share (hard to get file: to relax security)
+# FIXME: w/ sshrsa user as "limited" type I can run calc over ssh, but not IE,
+# so I made sshrsa a sysadmin, which works.  not sure what the problem is -- can
+# see share either way.
+export DYNAMORIO_OPTIONS='-loglevel 1'
+meminjector C:\\Program\ Files\\Internet\ Explorer\\IEXPLORE.EXE \\\\${myip}\\share\\close.html
+getlogdata
+
+echo "@@@@@@@@ IEXPLORE -hotp_only @@@@@@@@"
+# see notes above on methods of running IIS
+export DYNAMORIO_OPTIONS='-hotp_only -loglevel 1'
+meminjector C:\\Program\ Files\\Internet\ Explorer\\IEXPLORE.EXE \\\\${myip}\\share\\close.html
+getlogdata
+
+echo "@@@@@@@@ IEXPLORE -coarse_units @@@@@@@@"
+# see notes above on methods of running IIS
+export DYNAMORIO_OPTIONS='-coarse_units -loglevel 1'
+meminjector C:\\Program\ Files\\Internet\ Explorer\\IEXPLORE.EXE \\\\${myip}\\share\\close.html
+getlogdata
+
diff --git a/suite/nightly/nightly_bashrc b/suite/nightly/nightly_bashrc
new file mode 100644
index 000000000..96fd4ccf5
--- /dev/null
+++ b/suite/nightly/nightly_bashrc
@@ -0,0 +1,75 @@
+# **********************************************************
+# Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+echo "sourcing .bashrc"
+
+export MAKE_MODE=unix
+
+export PATH="$PATH:/c/emacs-22.1-barebin-i386/emacs-22.1/bin"
+
+alias more=less
+alias determina-cvs="ssh tgarnett@determina-cvs"
+
+#perforce setup
+export P4CONFIG=.p4config
+export P4EDITOR=emacs.exe
+export P4PORT=perforce-tiger:1947
+export P4USER=tgarnett
+export P4DIFF=diff
+export P4PATH_OVERRIDE='-d $(shell cygpath -m $(PWD))'
+alias p4='p4 -d $(cygpath -w $PWD)'
+
+#build setup
+export TCROOT=c:/nightly/toolchain
+
+#dynamorio setup
+export DYNAMORIO_TOOLS=/c/nightly/tools
+export DYNAMORIO_HOME=/c/nightly
+export DYNAMORIO_LIBUTIL=/c/nightly/libutil
+export DYNAMORIO_OPTIONS="-no_hide -msgbox_mask 0xc"
+export DYNAMORIO_LOGDIR=d:/logs
+export DYNAMORIO_SUITE=/c/nightly/suite
+export DYNAMORIO_BENCHMARKS=/c/nightly/benchmarks
+export DYNAMORIO_HOTPATCH=/c/nightly/liveshield
+export DYNAMORIO_MAKE=/c/nightly/make
+
+export NIGHTLY_NIGHTLY=/c/nightly/nightly
+export NIGHTLY_TOOLS=/c/nightly/tools
+export NIGHTLY_HOME=/c/nightly
+export NIGHTLY_LIBUTIL=/c/nightly/libutil
+export NIGHTLY_OPTIONS="-no_hide -msgbox_mask 0xc"
+export NIGHTLY_LOGDIR=d:/logs
+export NIGHTLY_LOGDIR_BASE=d:
+export NIGHTLY_SUITE=/c/nightly/suite
+export NIGHTLY_BENCHMARKS=/c/nightly/benchmarks
+export NIGHTLY_HOTPATCH=/c/nightly/liveshield
+export NIGHTLY_MAKE=/c/nightly/make
+export NIGHTLY_MAIL=tgarnett@determina-cvs
+export NIGHTLY_PLATFORM=Vista
diff --git a/suite/nightly/nightly_linux.sh b/suite/nightly/nightly_linux.sh
new file mode 100644
index 000000000..bfc2ba8b3
--- /dev/null
+++ b/suite/nightly/nightly_linux.sh
@@ -0,0 +1,148 @@
+#!/bin/bash
+
+# **********************************************************
+# Copyright (c) 2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+export OS_VER="$1"
+export RUNREGRESSION_OS_VER="RUNREGRESSION_$1"
+export HOSTNAME=`hostname`
+
+# Let's mark the start time with a file and then check the file's
+# existence before starting a new run.  If something's gone wrong and
+# the file is still around from the previous run, send mail and don't
+# start a new regression.
+cd $DYNAMORIO_NIGHTLY
+start_dir=`pwd`
+
+if [ -e ./test_started ]; then
+    echo "To:dynamorio-devnull@vmware.com" > mail_results
+    echo "Subject: Linux $OS_VER ($HOSTNAME) $NIGHTLY_TAG regression ERROR: previous run not finished!" >> mail_results
+    echo "" >> mail_results
+    read prev_start < ./test_started
+    echo "Previous run started on $prev_start did not finish" >> mail_results
+    echo "(or was aborted and 'test_started' was not removed)." >> mail_results
+    echo "Cancelling current job." >> mail_results
+
+    ssh $NIGHTLY_MAIL "rm -f mail_results.$OS_VER.$HOSTNAME"
+    scp mail_results $NIGHTLY_MAIL:mail_results.$OS_VER.$HOSTNAME
+    ssh $NIGHTLY_MAIL "/usr/sbin/sendmail -t -fdynamorio-devnull@vmware.com < mail_results.$OS_VER.$HOSTNAME"
+
+    rm -f mail_results.last
+    mv -f mail_results mail_results.last
+    exit
+else
+    date > ./test_started
+fi
+
+export NIGHTLY_START_TIME=`date`
+echo $DYNAMORIO_LOGDIR
+cd $DYNAMORIO_HOME
+p4 sync
+#in case of network trouble, make sure we get an updated src module
+#rm -rf src
+#FIXME how to do this
+
+cd $DYNAMORIO_TOOLS
+make clean
+
+cd $DYNAMORIO_LOGDIR_BASE
+rm -rf logs
+mkdir logs
+
+cd $DYNAMORIO_SUITE/tests
+make clean
+cd $DYNAMORIO_SUITE
+rm -rf regression-*.0*
+
+echo "running regression" 
+./runregression -long 2>&1 | tee regression.log
+
+echo "sending summary"
+#get output directory name
+ls -td regression-*.0* > tmp
+read dir_name < tmp
+rm -f tmp
+
+#get failure count
+grep -c " \*\*\* .* failed:" regression.log > tmp
+read fail_count < tmp
+rm -f tmp
+
+#start preparing results mail 
+echo "To:dynamorio-devnull@vmware.com" > mail_results
+echo "Subject: Linux $OS_VER ($HOSTNAME) $NIGHTLY_TAG regression results : started $NIGHTLY_START_TIME : $fail_count failing runs" >> mail_results
+echo "" >> mail_results
+echo "this automatically generated email contains the following sections: " >> mail_results
+echo " 1) summary of win32 $OS_VER ($HOSTNAME) $NIGHTLY_TAG regression results" >> mail_results
+echo " 2) details of win32 $OS_VER ($HOSTNAME) $NIGHTLY_TAG regression (unit test) results" >> mail_results
+echo "" >> mail_results
+echo "====================================================" >> mail_results
+
+#get runregression summary
+dstamp=`date | awk '{print $6 "-" $2$3}'`
+details=w32-reg-$dstamp-$OS_VER.$HOSTNAME
+../nightly/utreport.pl regression.log $dir_name $details > w32-reg-results 2>&1
+
+echo "regression suite results:" >> mail_results
+echo "" >> mail_results
+lines=`wc -l w32-reg-results | awk '{print $1}'`;
+if [ $lines -lt 50 ]; then
+    echo "RESULTS LOOK INCOMPLETE!  CHECK DISK USAGE:" >> mail_results;
+    df -h -l >> mail_results;
+    echo "" >> mail_results;
+    grep 'No space' regression.log | tail >> mail_results;
+    echo "" >> mail_results;
+fi
+cat w32-reg-results >> mail_results
+
+# mail mail_results
+ssh $NIGHTLY_MAIL "rm -f mail_results.$OS_VER.$HOSTNAME"
+scp mail_results $NIGHTLY_MAIL:mail_results.$OS_VER.$HOSTNAME
+#FIXME - where to store this?
+#scp $details $NIGHTLY_MAIL:/mnt/determina/bugs/nightly/
+ssh $NIGHTLY_MAIL "/usr/sbin/sendmail -t -fdynamorio-devnull@vmware.com < mail_results.$OS_VER.$HOSTNAME"
+
+#cleanup
+tar_name="${dir_name%.*}"
+tar_ext=".tgz"
+# FIXME - disabling till case 5168 and 5000 are fixed, till then will only keep
+#  the last night's results
+#tar -cf $tar_name$tar_ext regression.log $dir_name w32-reg-results mail_results --gzip
+rm -rf regression-dir-last
+rm -f regression.log.last
+rm -f mail_results.last
+rm -f w32-reg-results.last
+mv -f regression.log regression.log.last
+mv -f $dir_name regression-dir-last
+mv -f mail_results  mail_results.last
+mv -f w32-reg-results w32-reg-results.last
+
+rm -f $start_dir/test_started
+exit
diff --git a/suite/nightly/nightly_windows.sh b/suite/nightly/nightly_windows.sh
new file mode 100755
index 000000000..339a3e80c
--- /dev/null
+++ b/suite/nightly/nightly_windows.sh
@@ -0,0 +1,161 @@
+#!/bin/bash
+
+# **********************************************************
+# Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+export OS_VER="$1"
+export RUNREGRESSION_OS_VER="RUNREGRESSION_$1"
+export HOSTNAME=`hostname`
+
+# Let's mark the start time with a file and then check the file's
+# existence before starting a new run.  If something's gone wrong and
+# the file is still around from the previous run, send mail and don't
+# start a new regression.
+start_dir=`pwd`
+
+if [ -e ./test_started ]; then
+    echo "To:dynamorio-devnull@vmware.com" > mail_results
+    echo "Subject: windows $OS_VER ($HOSTNAME) $NIGHTLY_TAG regression ERROR: previous run not finished!" >> mail_results
+    echo "" >> mail_results
+    read prev_start < ./test_started
+    echo "Previous run started on $prev_start did not finish" >> mail_results
+    echo "(or was aborted and 'test_started' was not removed)." >> mail_results
+    echo "Cancelling current job." >> mail_results
+
+    ssh $NIGHTLY_MAIL "rm -f mail_results.$OS_VER.$HOSTNAME"
+    scp mail_results $NIGHTLY_MAIL:mail_results.$OS_VER.$HOSTNAME
+    ssh $NIGHTLY_MAIL "/usr/sbin/sendmail -t -fdynamorio-devnull@vmware.com < mail_results.$OS_VER.$HOSTNAME"
+
+    rm -f mail_results.last
+    mv -f mail_results mail_results.last
+    exit
+else
+    date > ./test_started
+fi
+
+export DYNAMORIO_LOGDIR=$NIGHTLY_LOGDIR_BASE/logs
+export DYNAMORIO_BENCHMARKS=$NIGHTLY_BENCHMARKS
+export DYNAMORIO_HOME=$NIGHTLY_HOME
+export DYNAMORIO_TOOLS=$NIGHTLY_TOOLS
+export DYNAMORIO_LIBUTIL=$NIGHTLY_LIBUTIL
+export NIGHTLY_START_TIME=`date`
+echo $DYNAMORIO_LOGDIR
+cd $DYNAMORIO_HOME
+#in case of network trouble, make sure we get an updated src module
+#rm -rf src
+#FIXME how to do this
+
+cd $DYNAMORIO_LIBUTIL
+make clean
+
+cd $DYNAMORIO_TOOLS
+make clean
+
+cd $NIGHTLY_LOGDIR_BASE
+rm -rf logs
+mkdir logs
+
+cd $NIGHTLY_SUITE
+cd tests
+make clean
+cd ..
+rm -rf regression-*.0*
+
+echo "stopping nodemgr"
+net stop scnodemgr
+
+echo "running regression"
+./runregression -long 2>&1 | tee regression.log
+
+echo "done running regression, starting nodemgr"
+net start scnodemgr
+
+echo "sending summary"
+#get output directory name
+ls -td regression-*.0* > tmp
+read dir_name < tmp
+rm -f tmp
+
+#get failure count
+grep -c " \*\*\* .* failed:" regression.log > tmp
+read fail_count < tmp
+rm -f tmp
+
+#start preparing results mail 
+echo "To:dynamorio-devnull@vmware.com" > mail_results
+echo "Subject: windows $OS_VER ($HOSTNAME) $NIGHTLY_TAG regression results : started $NIGHTLY_START_TIME : $fail_count failing runs" >> mail_results
+echo "" >> mail_results
+echo "this automatically generated email contains the following sections: " >> mail_results
+echo " 1) summary of win32 $OS_VER ($HOSTNAME) $NIGHTLY_TAG regression results" >> mail_results
+echo " 2) details of win32 $OS_VER ($HOSTNAME) $NIGHTLY_TAG regression (unit test) results" >> mail_results
+echo "" >> mail_results
+echo "====================================================" >> mail_results
+
+#get runregression summary
+dstamp=`date | awk '{print $6 "-" $2$3}'`
+details=w32-reg-$dstamp-$OS_VER.$HOSTNAME
+../nightly/utreport.pl regression.log $dir_name $details > w32-reg-results 2>&1
+
+echo "regression suite results:" >> mail_results
+echo "" >> mail_results
+lines=`wc -l w32-reg-results | awk '{print $1}'`;
+if [ $lines -lt 50 ]; then
+    echo "RESULTS LOOK INCOMPLETE!  CHECK DISK USAGE:" >> mail_results;
+    df -h -l >> mail_results;
+    echo "" >> mail_results;
+    grep 'No space' regression.log | tail >> mail_results;
+    echo "" >> mail_results;
+fi
+cat w32-reg-results >> mail_results
+
+# mail mail_results
+ssh $NIGHTLY_MAIL "rm -f mail_results.$OS_VER.$HOSTNAME"
+scp mail_results $NIGHTLY_MAIL:mail_results.$OS_VER.$HOSTNAME
+#FIXME - where to store this?
+#scp $details $NIGHTLY_MAIL:/mnt/determina/bugs/nightly/
+ssh $NIGHTLY_MAIL "/usr/sbin/sendmail -t -fdynamorio-devnull@vmware.com < mail_results.$OS_VER.$HOSTNAME"
+
+#cleanup
+tar_name="${dir_name%.*}"
+tar_ext=".tgz"
+# FIXME - disabling till case 5168 and 5000 are fixed, till then will only keep
+#  the last night's results
+#tar -cf $tar_name$tar_ext regression.log $dir_name w32-reg-results mail_results --gzip
+rm -rf regression-dir-last
+rm -f regression.log.last
+rm -f mail_results.last
+rm -f w32-reg-results.last
+mv -f regression.log regression.log.last
+mv -f $dir_name regression-dir-last
+mv -f mail_results  mail_results.last
+mv -f w32-reg-results w32-reg-results.last
+
+rm -f $start_dir/test_started
+exit
diff --git a/suite/nightly/utreport.pl b/suite/nightly/utreport.pl
new file mode 100755
index 000000000..dedea92d9
--- /dev/null
+++ b/suite/nightly/utreport.pl
@@ -0,0 +1,287 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# creates a unit test summary report from runregression log and runlog.*
+
+if ( ($#ARGV+1) != 3) {
+    print ("Usage: utreport <winrunreg.log> <winrunlog dir> <details-file-name>\n");
+    exit;
+}
+
+$winrunlogdir=$ARGV[1];
+open(OUT,">$ARGV[2]") or die ("Cannot open output file :$!");
+
+sub parserunlog;
+
+#
+# windows details
+#
+print OUT "\n====================== Regression Suite Details ======================\n\n";
+
+open(IN,"<$ARGV[0]") or die ("Cannot open input file :$!");
+
+while(($val=<IN>) && $val !~ /Summary of results:/) {
+    #nothing
+}
+
+$val=<IN>;
+while (1) {
+
+    if($val =~ /^==========/) {
+        last;
+    }
+
+    @tmp=split(' ', $val);
+    @tmp2=split(':', $tmp[1]);
+    $nmbr=$tmp2[0];
+
+    if ($val =~ /^Build/) {
+        $next=<IN>;
+        $next_next=<IN>;
+        if ($next =~ /<no errors>/ && $next_next =~ /no warnings/) {
+            print OUT "\nBuild ", $nmbr, " successful.\n";
+        } else {
+            print OUT "\nBuild ", $nmbr, " failure!\n";
+        }
+        $val=<IN>;
+        if ($val =~ /SHARETESTS/) {
+            print OUT $val;
+        } else {
+            # next iter will handle
+            next;
+        }
+    }
+    elsif ($val =~ /Run/) {
+
+        parserunlog($nmbr, "long");
+        
+        # read in the summary lines
+        $val=<IN>;
+        if ($val =~ /SPEC:/) {
+            $val=<IN>;
+        }
+        if ($val !~ /TESTS:/) {
+            # next iter will handle
+            next;
+        }
+    }
+    elsif ($val =~ /^(SHARE|TOOLS)/) {
+        print OUT $val;
+    }
+    elsif (length($val)>1) {
+        print OUT "Warning! unexpected result line>", $val;
+    }
+    if (!($val=<IN>)) {
+        last;
+    }
+}
+close(OUT);
+
+# summaries go to stdout
+
+# 
+# windows summary
+#
+print "\n====================== Regression Suite Core Errors ======================\n\n";
+
+# get summary of DR errors
+system("grep -E 'Unrecoverable|Internal DynamoRIO Error|CURIOSITY' $ARGV[2] | sort | uniq -c | awk '{s+=\$1} END {printf \"Total DR errors: %d\\n\\n\", s}'");
+system("grep -E -o 'Unrecoverable.*|Internal DynamoRIO Error.*|CURIOSITY.*' $ARGV[2] | sort | uniq -c | sort -nr");
+
+print "\n====================== Regression Suite Test Failures ======================\n\n";
+
+# get summary of DR errors
+system("grep -E '^------------- ' $ARGV[2] | sed 's/---*//g' | sort | uniq -c | awk '{s+=\$1} END {printf \"Total failing tests: %d\\n\\n\", s}'");
+system("grep -E '^------------- ' $ARGV[2] | sed 's/---*//g' | sort | uniq -c | sort -nr");
+
+print "\n====================== Regression Suite Failures ======================\n\n";
+
+print "Total failing runs: ";
+system("grep -c \" \\*\\*\\* .* failed:\" $ARGV[0]");
+print "\n";
+
+# first, get summary of failures
+# could move this grep to caller but nice to all be from this report script
+system("grep -A 5000 'Summary of results' $ARGV[0] | grep -E 'Run |[1-9][0-9]* failed|s in build|FAILED' | grep -B 1 -E '[1-9][0-9]* failed|s in build|FAILED'");
+
+print "\n====================== Regression Suite Summary ======================\n\n";
+
+open(IN,"<$ARGV[0]") or die ("Cannot open input file :$!");
+
+while(($val=<IN>) && $val !~ /Summary of results:/) {
+    #nothing
+}
+
+# print entire summary as is, regression now prints which tests failed
+while(($val=<IN>) && $val !~ /=====/) {
+    print $val;
+}
+
+close(IN);
+
+exit;
+
+sub parserunlog
+{
+    $nmbr = shift;
+    $type = shift;
+    $specpass = 0;
+    $specfail = 0;
+    $runlogdir = $winrunlogdir;
+
+    if ($type !~ /short/ && $type !~ /long/) {
+        print OUT "parameter error!";
+        exit;
+    }
+
+    if($type =~ "short") {
+        $runlogdir = shift;
+    }   
+    
+    my $file;
+    my $rlval;
+    $file = sprintf("%s/runlog.%03d", $runlogdir, $nmbr);
+    
+    open(RLIN,"<$file") or die ("Cannot open input file \"$file\" : $!");
+
+    if($type =~ "long") {
+        print OUT "\n======================= Run ", $nmbr, " =========================\n";
+    }
+
+    #first scan for spec, if present
+    while ($rlval=<RLIN>) {
+        if ($rlval =~ /Benchmark/ || $rlval =~ /Using build/) {
+            last;
+        }
+    }
+
+    if ($rlval =~ /Benchmark/) {
+        if($type =~ "short") {
+            print OUT "    ";
+        }
+        print OUT "SPEC: ";
+        do {
+            $rlval=<RLIN>;
+            @tmps=split(' ', $rlval);
+            if ($tmps[2] =~ "ok") {
+                $specpass++;
+            }
+            elsif ($tmps[2] =~ "FAIL") {
+                $specfail++;
+                print OUT $tmps[0], " ";
+            }
+            else {
+                $rlval="done";
+            }
+        } while ($rlval !~ "done");
+        
+        if ($specfail > 0) {
+            print OUT "failed, ";
+        }
+        print OUT $specpass, " tests passed.\n";
+
+        while ($rlval=<RLIN>) {
+            if ($rlval =~ /Using build/) {
+                last;
+            }
+        }
+    }
+
+    # now the unit tests, if present
+    if ($rlval =~ /Using build/) {
+        if ($type =~ "long") {
+            print OUT "\nunit tests:\n\n";
+        }
+        else {
+            print OUT "    TESTS: ";
+        }
+        
+        $utpass=0;
+        $utfail=0;
+        my $pass;
+        
+        while ($val=<RLIN>) {
+            
+            if ($val =~ /^----- /) {
+                $next = <RLIN>;
+                @tmp=split(' ', $val);
+                if ($next =~ /^PASS/) {
+                    $pass[$utpass]=$tmp[1];
+                    $utpass++;
+                }
+                elsif ($next =~ /^SKIP/) {
+                    # do nothing
+                }
+                else {
+                    if ($type =~ "long") {
+                        print OUT "------------- ", $tmp[1], " -----------------\n";
+                        $badln=1;
+                        do {
+                            print OUT $next;
+                            $next = <RLIN>;
+                            $badln++;
+                        } while ($next !~ /^FAIL/ && $next !~ /^----- / && $badln < 20);
+                        if ($badln == 20) {
+                            print OUT "   --<diff truncated>--\n";
+                        }
+                        print OUT "\n";
+                        $utfail++;
+                    }
+                    else { #short
+                        print OUT $tmp[1], " ";
+                    }
+                }
+            }
+        }
+        
+        if ($type =~ "long") {
+            print OUT $utfail, " unit tests ";
+        }
+        
+        if ($type =~ "long" || $utfail > 0) {
+            print OUT  "failed. ";
+        }
+        
+        if($type =~ "long") {
+            print OUT "\n--------------------------------------\npassed tests:  ";
+            
+            for ($j=0; $j<$utpass; $j++) {
+                print OUT $pass[$j], ", ";
+            }
+            print OUT "\n";
+        }
+        
+        print OUT $utpass, " tests passed.\n";
+    }
+
+}
+
diff --git a/suite/old-runregression-runs-2-6-08 b/suite/old-runregression-runs-2-6-08
new file mode 100644
index 000000000..30bb95299
--- /dev/null
+++ b/suite/old-runregression-runs-2-6-08
@@ -0,0 +1,423 @@
+# **********************************************************
+# Copyright (c) 2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+
+### os-specific cases that are appended to the long run
+@windows_cases = (
+# PR 234665: 64-bit debug build
+    "BUILD::ARCH=x64",
+### Following two runs don't build on windows (though we continue to build them on linux)
+### FIXME : if we get these working again move them from the os_specific cases to the
+### common cases
+  ##"BUILD::ADD_DEFINES=\"${D}CHECK_RETURNS_SSE2\"",
+### strace, no run
+    "BUILD::DEBUG=1 INTERNAL=1 STRACE_CLIENT=1",
+### no longer actively supported
+### test building linked to libc
+##    "BUILD::DEBUG=1 INTERNAL=1 NOLIBC=0",
+##        "ONLY::spec::DYNAMORIO_OPTIONS=\\\"-loglevel 1 -protect_mask 0\\\" RUNALL_SLEEP=5s",
+                  );
+
+@linux_cases = (
+### the following run doesn't build on windows, move back to common once they are fixed up
+    "BUILD::ADD_DEFINES=\"${D}CHECK_RETURNS_SSE2\"",
+### FIXME: Move this to common_cases below once the client interface tests
+### are working with Linux.  xref case 5412.
+    "BUILD::CLIENT_INTERFACE=1",
+        "DYNAMORIO_OPTIONS=\\\"\\\"",
+                  );
+
+### These will be run with both release and debug builds
+### and on both Linux and Windows
+@common_product_runs = (
+        "DYNAMORIO_OPTIONS=\\\"-code_api\\\"",
+        "ONLY::runnointerface::DYNAMORIO_OPTIONS=\\\"\\\"",
+        "ONLY::runnointerface::DYNAMORIO_OPTIONS=\\\"-security -liveshields\\\"",
+        # options exposed to users
+        "DYNAMORIO_OPTIONS=\\\"-thread_private\\\"",
+        "DYNAMORIO_OPTIONS=\\\"-disable_traces\\\"",
+        # put disabled-from-short test runs here to keep early in email
+        "ONLY::runall/earlythread.runrunall security-win32/selfmod-threads.runinjector::DYNAMORIO_OPTIONS=\\\"\\\"",
+        # security option tests:
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -no_executable_if_alloc\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -no_executable_if_trampoline\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -no_executable_if_hook\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -no_executable_if_rx_text -no_coarse_merge_iat\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -executable_if_text\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -executable_if_x\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -X\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -no_ret_after_call\\\"",
+        # or its synonym
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -C\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -no_seh_rct\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -pushregret_rct\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -xdata_rct\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -executable_heap\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -executable_stack\\\"",
+        # RCT policies
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -rct_ind_call 11 -rct_ind_jump 11\\\"",
+        # don't block, just OPTION_ENABLED|OPTION_REPORT
+        # or don't report, just block, just OPTION_ENABLED|OPTION_BLOCK
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -rct_ind_call 9 -rct_ind_jump 3\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -rct_ind_call 3 -rct_ind_jump 9\\\"",
+        # disabled using the alias to -rct_ind_call 0 -rct_ind_jump 0
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -E -F\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -rct_cache_exempt 0\\\"", # default 1
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -rct_section_type 0xe0\\\"", # default 0x20
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -detect_mode -rct_cache_exempt 2\\\"",
+        # coarse units
+        "DYNAMORIO_OPTIONS=\\\"-no_shared_traces\\\"",
+        "DYNAMORIO_OPTIONS=\\\"-no_coarse_units\\\"",
+        # make sure we continue to support both shared and non-shared cache
+        "DYNAMORIO_OPTIONS=\\\"-no_coarse_units -no_shared_traces\\\"",
+        "DYNAMORIO_OPTIONS=\\\"-no_coarse_units -no_shared_bbs\\\"",
+        "DYNAMORIO_OPTIONS=\\\"-no_coarse_units -no_shared_traces -no_shared_bbs\\\"",
+        "DYNAMORIO_OPTIONS=\\\"-no_coarse_units -shared_bbs_only\\\"",
+        "DYNAMORIO_OPTIONS=\\\"-inline_trace_ibl\\\"",
+        # attack handling tests
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -kill_thread\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -throw_exception\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -throw_exception -kill_thread\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -throw_exception -throw_exception_max 1\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -kill_thread -kill_thread_max 1\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -kill_thread -kill_thread_max 1 -throw_exception  -throw_exception_max 1\\\"",
+        # adaptive working set is part of default build now
+        # We specify "runsecurity" to get our weirder tests, some w/ multiple threads
+        "ONLY::spec runsecurity::DYNAMORIO_OPTIONS=\\\"-cache_bb_regen 1\\\"",
+        "ONLY::spec runsecurity::DYNAMORIO_OPTIONS=\\\"-cache_trace_regen 1\\\"",
+        "ONLY::spec runsecurity::DYNAMORIO_OPTIONS=\\\"-finite_shared_bb_cache -cache_shared_bb_regen 80\\\"",
+        "ONLY::spec runsecurity::DYNAMORIO_OPTIONS=\\\"-finite_shared_trace_cache -cache_shared_trace_regen 80\\\"",
+        # reverse of the default options
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-no_guard_pages\\\"",
+        # see case 7813 for why nativeterminate test is problematic with
+        # -native_exec (default) and -no_native_exec_syscalls
+        # FIXME - we should have an ADD_DISABLED or something like that so
+        # that we can have default disabled tests as well
+        # on win2k -native_exec_syscalls is required for on by default
+        # -early_inject when the process itself wasn't early_injected into
+        "ONLY::DISABLED_WIN32=\"| grep -v nativeterminate\" all::DYNAMORIO_OPTIONS=\\\"-no_native_exec_syscalls -no_early_inject\\\"",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-native_exec\\\"",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-hot_patching\\\"",
+        # shared IBT tables
+        "DYNAMORIO_OPTIONS=\\\"-shared_trace_ibt_tables\\\"",
+        "DYNAMORIO_OPTIONS=\\\"-disable_traces -shared_bb_ibt_tables\\\"",
+        "DYNAMORIO_OPTIONS=\\\"-shared_trace_ibt_tables -no_ref_count_shared_ibt_tables\\\"",
+        # IAT conversion
+        "DYNAMORIO_OPTIONS=\\\"-IAT_convert\\\"",
+        "DYNAMORIO_OPTIONS=\\\"-no_coarse_units -IAT_convert -IAT_elide\\\"",
+        "DYNAMORIO_OPTIONS=\\\"-no_coarse_units -IAT_convert -IAT_elide -max_elide_jmp 1 -max_elide_call 1\\\"",
+        "DYNAMORIO_OPTIONS=\\\"-max_elide_jmp 1 -max_elide_call 1\\\"",
+        # patch proof - prevent hooking of targeted victim modules
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-patch_proof_default_list secur32.dll -patch_proof_list 'kernel32.dll'\\\"",
+        # patch proof - blanket protect policy
+        # FIXME: case 6635 0) 'kernel32.dll;advapi32.dll' 
+        # FIXME: case 6635 2) gen_expect should use subst to match * here
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-patch_proof_list '*=ALL'\\\"",
+        # case 6640
+        "DYNAMORIO_OPTIONS=\\\"-remove_trace_components\\\"",
+        # case 147 - BB2BB IBL w/trace building
+        "DYNAMORIO_OPTIONS=\\\"-no_coarse_units -bb_ibl_targets\\\"",
+        "DYNAMORIO_OPTIONS=\\\"-no_coarse_units -bb_ibl_targets -shared_trace_ibt_tables\\\"",
+        "DYNAMORIO_OPTIONS=\\\"-no_coarse_units -bb_ibl_targets -bb_single_restore_prefix\\\"",
+        "DYNAMORIO_OPTIONS=\\\"-no_coarse_units -bb_ibl_targets -bb_ibt_table_includes_traces\\\"",
+        "DYNAMORIO_OPTIONS=\\\"-no_coarse_units -bb_ibl_targets -bb_ibt_table_includes_traces -bb_single_restore_prefix -trace_single_restore_prefix\\\"",
+        # Test smaller prefixes on traces
+        "DYNAMORIO_OPTIONS=\\\"-trace_single_restore_prefix\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-ro2sandbox_threshold 1\\\"",
+        # -sandbox2ro_threshold 1 isn't allowed, 2 is lowest
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-sandbox2ro_threshold 2\\\"",
+        # stress the pad_jmps code
+        "DYNAMORIO_OPTIONS=\\\"-pad_jmps_set_alignment 8 -enable_traces -no_coarse_units\\\"",
+                        );
+
+@release_runs = (
+        "SHORT::ONLY::runinterface::DYNAMORIO_OPTIONS=\\\"-code_api -probe_api\\\"",
+        "SHORT::ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -detect_mode -report_max 0 -no_diagnostics\\\"",
+        "SHORT::ONLY::runrunall::DYNAMORIO_OPTIONS=\\\"-probe_api -hotp_only\\\"",
+        # test running out of our memory reservation
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-vm_size 2M -reset_at_vmm_percent_free_limit 0\\\"",
+                 );
+
+### These will be run with debug builds on both Linux and Windows
+@common_debug_runs = (
+        # runallbuthotp is to handle PR 234043 
+        "SHORT::ONLY::spec runallbuthotp::DYNAMORIO_OPTIONS=\\\"-code_api -probe_api\\\"",
+        "SHORT::ONLY::runos runsecurity runrunall::DYNAMORIO_OPTIONS=\\\"-security\\\"",
+        "SHORT::ONLY::runrunall::DYNAMORIO_OPTIONS=\\\"-desktop\\\"",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-loglevel 1\\\" RUNALL_SLEEP=5s",
+        # test running out of our memory reservation
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-vm_size 2M -reset_at_vmm_percent_free_limit 0 -skip_out_of_vm_reserve_curiosity\\\"",
+        #IBLs
+        # no traces, or lots of traces
+        # FIXME Re-activate the test below using -no_link_ibl instead
+        # of -no_ibl_shared_bb after the former option has been
+        # implemented (case 5182).
+        # "DYNAMORIO_OPTIONS=\\\"-shared_bbs -disable_traces -no_ibl_table_per_type -no_ibl_shared_bb\\\" TEST_MINS=15",
+        "DYNAMORIO_OPTIONS=\\\"-no_coarse_units -no_shared_bbs -disable_traces\\\" TEST_MINS=15",
+        "ONLY::runcommon::DYNAMORIO_OPTIONS=\\\"-disable_traces -no_bb_ibl_targets\\\"",
+        "DYNAMORIO_OPTIONS=\\\"-trace_threshold 1\\\"",
+        # no inlining, or inline both bbs and traces
+        "ONLY::spec runsecurity::DYNAMORIO_OPTIONS=\\\"-atomic_inlined_linking -no_shared_traces\\\"",
+        "ONLY::spec runsecurity::DYNAMORIO_OPTIONS=\\\"-no_inline_trace_ibl -no_shared_traces\\\"",
+        "ONLY::spec runsecurity::DYNAMORIO_OPTIONS=\\\"-no_coarse_units -inline_bb_ibl -atomic_inlined_linking\\\"",
+        "ONLY::spec runsecurity::DYNAMORIO_OPTIONS=\\\"-no_coarse_units -inline_bb_ibl -atomic_inlined_linking -no_shared_bbs\\\"",
+        "ONLY::spec runsecurity::DYNAMORIO_OPTIONS=\\\"-no_coarse_units -inline_trace_ibl -inline_bb_ibl -atomic_inlined_linking\\\"",
+        # for max cache sizes, here are max usage for test runs of spec:
+        #              bb (KB)    trace (KB)
+        #              --------   ----------
+        #    crafty     521        356
+        #    eon        642-649    200-216
+        #    gcc       2967       2149
+        #    perlbmk    290-635     19-146
+        #    mcf        115         43
+        # 64KB bb max ends up taking 10 minutes for single gcc run!
+        # Min cache max of 4 takes A LONG TIME -> disabled for default run
+        # Instead of min of 4 we do something intermediate.
+        # We specify "runsecurity" to get our weirder tests, some w/ multiple threads
+        "ONLY::spec runsecurity::DYNAMORIO_OPTIONS=\\\"-cache_bb_max 128\\\" TEST_MINS=30",
+        "ONLY::spec runsecurity::DYNAMORIO_OPTIONS=\\\"-cache_trace_max 128\\\" TEST_MINS=30",
+        "ONLY::spec runsecurity::DYNAMORIO_OPTIONS=\\\"-no_elide_back_jmps\\\"",
+        "ONLY::spec runsecurity::DYNAMORIO_OPTIONS=\\\"-no_elide_back_calls\\\"",
+        "ONLY::spec runsecurity::DYNAMORIO_OPTIONS=\\\"-no_free_private_stubs\\\"",
+        "ONLY::spec runsecurity::DYNAMORIO_OPTIONS=\\\"-no_separate_private_stubs -no_free_private_stubs\\\"",
+        "ONLY::spec runsecurity::DYNAMORIO_OPTIONS=\\\"-no_separate_shared_stubs\\\"",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-no_hashtable_ibl_stats\\\"",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-unsafe_ignore_overflow\\\"",
+        # common/{ops,conflict,eflags} are all expected to fail
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-unsafe_ignore_eflags\\\"",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-unsafe_ignore_eflags_trace\\\"",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-unsafe_ignore_eflags_prefix\\\"",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-unsafe_ignore_eflags_ibl -unsafe_ignore_eflags_prefix\\\"",
+        # debugging aids
+        "DYNAMORIO_OPTIONS=\\\"-detect_dangling_fcache\\\"",
+         # cache alignment: debug-only since -no_pad_jmps_shift_trace is an internal op
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-cache_trace_align 1 -no_pad_jmps_shift_trace\\\"",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-cache_trace_align 16 -no_pad_jmps_shift_trace\\\"",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-cache_bb_align 16\\\"",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-no_pad_jmps_shift_trace\\\"",
+        # internal-only options (for now)
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-safe_translate_flushed\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-no_coarse_units -store_translations\\\"",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-no_coarse_units -no_indirect_stubs -C -E -F\\\"",
+        # we're now using tracedump for perf analysis independently of -prof_counts
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-prof_pcs -gendump -tracedump_binary\\\" TEST_MINS=20 RUNALL_SLEEP=15s",
+                      );
+
+if ($on_linux) {
+    @product_runs = ( @common_product_runs,
+        #no linux specific product runs right now
+                      );
+    @debug_runs = ( @common_debug_runs,
+        # prof_pcs is an internal option on linux right now
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-prof_pcs\\\"",
+                    );
+} else {
+    # WIN32 specific runs
+    @product_runs = ( @common_product_runs,
+        # all product configurations that may prevent a succesful boot should be run in SHORT
+        "ONLY::runnointerface::DYNAMORIO_OPTIONS=\\\"-probe_api -hotp_only\\\"",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-desktop\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -detect_mode -report_max 0 -no_diagnostics\\\"",
+        "ONLY::runnointerface::DYNAMORIO_OPTIONS=\\\"-client\\\"",# explorer.exe and desktop apps
+        "ONLY::runnointerface::DYNAMORIO_OPTIONS=\\\"-client -aslr_cache 0\\\"", # winlogon, spoolsv run in this
+        "ONLY::runnointerface::DYNAMORIO_OPTIONS=\\\"-thin_client\\\"",
+        # pcache tests: per-user so each app will merge w/ previous, plus merge
+        # w/ at-unload persists from earlier -desktop run 
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-desktop -coarse_freeze_at_exit\\\"",
+        # run again so each must use the merged pcaches
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-desktop -coarse_freeze_at_exit\\\"",
+        # Test with -use_moduledb relaxations, (is on by default, but tests Makefile
+        # adds -staged which turns it off, we re-enable by turning the option back on).
+        # The tests Makefile adds different version resource information to different
+        # tests in runall/ and security-{common,win32}/ to fully exercise this.
+        "DYNAMORIO_OPTIONS=\\\"-security -use_moduledb\\\"",
+        "ONLY::runnointerface::DYNAMORIO_OPTIONS=\\\"-security -client\\\"",
+        "ONLY::runnointerface::DYNAMORIO_OPTIONS=\\\"-security -client -medium\\\"",
+        "ONLY::runnointerface::DYNAMORIO_OPTIONS=\\\"-security -low\\\"", # not a default config
+
+        # FIXME - next 2 runs temporary for new 4.2 features
+        "ONLY::runos::DYNAMORIO_OPTIONS=\\\"-suspend_on_synch_failure_for_app_suspend -synch_thread_sleep_MP -synch_thread_max_loops 1500 -synch_all_threads_max_loops 3000\\\"",
+        "ONLY::runos::DYNAMORIO_OPTIONS=\\\"-synch_thread_max_loops 0xffffffff\\\"",
+        # self-protection: FIXME: once case 8023 re-enables on linux, move to
+        #   common_product_runs list
+        "DYNAMORIO_OPTIONS=\\\"-security -protect_mask 0\\\"", # no options
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-security -protect_mask 2\\\" TEST_MINS=8", # .fspdata only
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-security -protect_mask 0x103\\\" TEST_MINS=8", # .data, .fspdata, gencode
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-security -protect_mask 0x107\\\" TEST_MINS=8", # .data, .fspdata, .cspdata, gencode
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-security -protect_mask 8\\\" TEST_MINS=8", # global heap
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-security -protect_mask 0x10f\\\" TEST_MINS=8", # global heap, all data sections, gencode
+        "DYNAMORIO_OPTIONS=\\\"-no_shared_syscalls\\\"",
+        "DYNAMORIO_OPTIONS=\\\"-security -no_pad_jmps\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -no_executable_if_flush\\\"",
+        # there is an overlap, so combination should be tested
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -executable_after_load -no_executable_if_rx_text -no_coarse_merge_iat\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -no_executable_if_hook -executable_if_text\\\"",
+        # reverse of the default options
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-ignore_syscalls\\\"",
+        # test internal detach, mainly looking for asserts or
+        # exceptions/hangs as we don't verify that we really 
+        # detach, FIXME verify somehow
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -internal_detach_mask 0x1\\\"", 
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-no_hide\\\"", 
+        # test sygate compatibility
+        "DYNAMORIO_OPTIONS=\\\"-security -sygate\\\"",
+        # reset
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-no_enable_reset\\\"",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-reset_at_nth_thread 2\\\"",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-reset_every_nth_pending 2\\\"",
+        # we have to target LCD here -- gcc gets pretty slow if go lower
+        # than these, but most apps won't reset at all, so we only run spec
+        "ONLY::spec::DYNAMORIO_OPTIONS=\\\"-reset_at_nth_bb_unit 22\\\"",
+        "ONLY::spec::DYNAMORIO_OPTIONS=\\\"-reset_at_nth_trace_unit 15\\\"",
+        # profiling
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-prof_pcs\\\"",
+        # ASLR - DLLs
+        "ONLY::runnointerface::DYNAMORIO_OPTIONS=\\\"-security -aslr 0x7 -hotp_only\\\"", # non-shared
+        "DYNAMORIO_OPTIONS=\\\"-security -aslr 0x7\\\"", # DLL + heap + stack
+        "DYNAMORIO_OPTIONS=\\\"-security -aslr 0x1\\\"", # only DLL
+        "DYNAMORIO_OPTIONS=\\\"-security -aslr 0x4\\\"", # only heap pad
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -aslr 0x3\\\"", # DLL + parent stack
+        "DYNAMORIO_OPTIONS=\\\"-security -aslr 0x47\\\"",# DLL + heap + stack + heap fill
+        "DYNAMORIO_OPTIONS=\\\"-security -aslr 0x1 -aslr_action 0x1\\\"",
+        "DYNAMORIO_OPTIONS=\\\"-security -aslr 0x1 -aslr_dll_base 0x50000000 -aslr_dll_top 0x70000000 -aslr_dll_offset 0x01000000 -aslr_dll_pad 0x01000000\\\"",
+        # ASLR - shared cache
+        "ONLY::runnointerface::DYNAMORIO_OPTIONS=\\\"-security -client -aslr_cache 0x3170\\\"", # hotp+ASLR+SHARED demo
+        "ONLY::runos::DYNAMORIO_OPTIONS=\\\"-security -client -aslr_cache_list 0\\\"",
+        "ONLY::runos::DYNAMORIO_OPTIONS=\\\"-security -client -aslr_cache_list 2\\\"",
+        "ONLY::runos::DYNAMORIO_OPTIONS=\\\"-security -aslr 0x7 -aslr_cache 0x3170\\\"", # full DR+ASLR+SHARED demo
+
+        # stress testing exemptions
+        "DYNAMORIO_OPTIONS=\\\"-security -aslr 0x1 -exempt_aslr_list '*'\\\"",
+        # GBOP exclusion test
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -low -gbop_exclude_list 'kernel32.dll!LoadLibraryA'\\\"",
+        # Don't use TLS slots for IBL table addresses & masks
+        "DYNAMORIO_OPTIONS=\\\"-no_ibl_table_in_tls\\\"",
+                      );
+    # note all product runs are shared between release and debug runs
+    @debug_runs = ( @common_debug_runs,
+        "SHORT::ONLY::runos runsecurity runrunall::DYNAMORIO_OPTIONS=\\\"-probe_api -hotp_only\\\"",
+        "SHORT::ONLY::runrunall::DYNAMORIO_OPTIONS=\\\"-thin_client\\\"",
+        # to save time in short we only do debug version of -low
+        "ONLY::runnointerface::DYNAMORIO_OPTIONS=\\\"-security -low\\\"",
+        "DYNAMORIO_OPTIONS=\\\"-reset_at_fragment_count 1000\\\"",
+        # ASLR internal stress testing only
+        "DYNAMORIO_OPTIONS=\\\"-security -aslr 1 -aslr_internal 0x1000\\\"",
+                    );
+}
+
+#FIXME: options that need product runs 
+#               dynamic_options
+
+#FIXME: internal options to be added to internal runs
+#               nolink
+#               tracedump_threshold
+#               thread_stats global_stats 
+#               thread_stats_interval 1000
+#               global_stats_interval 1000
+#               hashtable_ibl_stats
+#               deadlock_avoidance
+#               heap_accounting_assert
+#               prof_pcs_* with non-default shifts should have a run
+#               ...
+
+# this is used in common_cases and to determine whether to run the
+#  share tests in the short run.
+$release_build = "BUILD::DEBUG=0 INTERNAL=0";
+$external_debug_build = "BUILD::DEBUG=1 INTERNAL=0";
+
+#### The long run contains pretty much all configurations we would like to maintain
+#### lines marked #l were taken out to make it shorter
+#### Add here runs that are not to be run in release builds, otherwise use common_product_runs
+@common_cases = (
+### we do NOT prepend short cases since we want more runs of each
+### debug internal version
+    "BUILD::DEBUG=1 INTERNAL=1",
+        # shared runs with release build
+        @product_runs,
+        # debug build or internal only options
+        @debug_runs,
+### default product build
+    $release_build,
+        # shared runs with debug build
+        @product_runs,
+        # release only runs
+        @release_runs,
+### debug external version, maybe part of product
+    $external_debug_build,
+        "DYNAMORIO_OPTIONS=\\\"\\\"",
+### non-debug internal version
+    "BUILD::DEBUG=0 INTERNAL=1",
+        "DYNAMORIO_OPTIONS=\\\"\\\"",
+### non-product configurations that we want to not only build but also run
+# turn off everything
+    "BUILD::CLIENT_INTERFACE=0 PROGRAM_SHEPHERDING=0 GBOP=0 HOT_PATCHING_INTERFACE=0 PROCESS_CONTROL=0",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"\\\"",
+### profiling
+### FIXME: add tests of client interface routines, including app start/stop interface?
+### (i.e., re-activate api/startstop test and add new tests)
+# 
+### would only want to run for this build
+    "BUILD::ADD_DEFINES=\"${D}PROFILE_LINKCOUNT ${D}LINKCOUNT_64_BITS\"",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-no_inline_trace_ibl\\\"",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-no_shared_traces -prof_counts -tracedump_binary -no_inline_trace_ibl\\\" TEST_MINS=20 RUNALL_SLEEP=15s",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-prof_counts -tracedump_binary -no_inline_trace_ibl\\\" TEST_MINS=20 RUNALL_SLEEP=15s",
+        "ONLY::runcommon::DYNAMORIO_OPTIONS=\\\"-prof_counts -tracedump_text -no_inline_trace_ibl\\\" TEST_MINS=20 RUNALL_SLEEP=15s",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"-tracedump_binary -tracedump_origins -no_inline_trace_ibl\\\" TEST_MINS=20",
+        "ONLY::runcommon::DYNAMORIO_OPTIONS=\\\"-tracedump_text -tracedump_origins -no_inline_trace_ibl\\\" TEST_MINS=20 RUNALL_SLEEP=15s",
+### client interface
+# We have some client interface tests working for Windows only, so the following 
+# build is currently placed in the os-specific sections above.  Once these 
+# tests are verified to work with Linux, we should move all client interface
+# entries to here.
+#    "BUILD::CLIENT_INTERFACE=1",
+#        "DYNAMORIO_OPTIONS=\\\"\\\"",
+## we won't bother to keep explicitly supporting this combination
+##    "BUILD::CLIENT_INTERFACE=1 ADD_DEFINES=\"${D}PROFILE_LINKCOUNT ${D}LINKCOUNT_64_BITS\"",
+#        "DYNAMORIO_OPTIONS=\\\"-no_inline_trace_ibl\\\"",
+### sideline no longer supported
+##    "BUILD::CLIENT_INTERFACE=1 ADD_DEFINES=\"${D}CLIENT_SIDELINE\"",
+### release-build kstats and callprof
+    "BUILD::DEBUG=0 INTERNAL=0 KSTATS=1 CALLPROF=1",
+        "ONLY::all::DYNAMORIO_OPTIONS=\\\"\\\"",
+### defines we don't want to break -- no runs though
+### no longer supported
+##    "BUILD::STATIC=1",
+    "BUILD::ADD_DEFINES=\"${D}DYNAMORIO_APP_EXPORTS ${D}DGC_DIAGNOSTICS\"",
+### no longer supported
+##    "BUILD::ADD_DEFINES=\"${D}SIMULATE_ATTACK\"",
+### no longer supported
+##    "BUILD::ADD_DEFINES=\"${D}SIDELINE\"",
+### not worth supporting by itself
+##    "BUILD::ADD_DEFINES=\"${D}RCT_IND_BRANCH\"",
+### may want to add in future -- don't build right now though
+##  "BUILD::ADD_DEFINES=\"${D}NATIVE_RETURN ${D}ASSUME_NORMAL_EFLAGS ${D}CUSTOM_EXIT_STUBS\"",
+##  "BUILD::ADD_DEFINES=\"${D}RETURN_STACK\"",
+### os-specific cases will be inserted here
+          );
diff --git a/suite/regression-expect b/suite/regression-expect
new file mode 100644
index 000000000..340e674a4
--- /dev/null
+++ b/suite/regression-expect
@@ -0,0 +1,101 @@
+# **********************************************************
+# Copyright (c) 2003-2004 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+============================================================
+Summary of results:
+
+Build 1: "DEBUG=0 INTERNAL=0"
+       <no errors>
+       <no warnings>
+  Run 1: DYNAMORIO_OPTIONS=\"\"
+       SPEC: 4 passed, 0 failed
+       TESTS: 33 passed, 0 failed
+  Run 2: DYNAMORIO_OPTIONS=\"-detect_mode -report_max 0 -no_diagnostics\"
+       SPEC: 4 passed, 0 failed
+       TESTS: 33 passed, 0 failed
+Build 2: "DEBUG=1 INTERNAL=0"
+       <no errors>
+       <no warnings>
+Build 3: "DEBUG=0 INTERNAL=1"
+       <no errors>
+       <no warnings>
+Build 4: "DEBUG=1 INTERNAL=1"
+       <no errors>
+       <no warnings>
+  Run 3: DYNAMORIO_OPTIONS=\"\"
+       SPEC: 4 passed, 0 failed
+       TESTS: 33 passed, 0 failed
+  Run 4: DYNAMORIO_OPTIONS=\"-detect_mode -report_max 0 -no_diagnostics\"
+       SPEC: 4 passed, 0 failed
+       TESTS: 33 passed, 0 failed
+Build 5: "DEFAULT_DEFINES="""
+       <no errors>
+       <no warnings>
+Build 6: "DEFAULT_DEFINES="-DTRACE_HEAD_CACHE_INCR""
+       <no errors>
+       <no warnings>
+Build 7: "CLIENT_INTERFACE=1 INTERNAL=1 DEFAULT_DEFINES="-DTRACE_HEAD_CACHE_INCR""
+       <no errors>
+       <no warnings>
+Build 8: "CLIENT_INTERFACE=1 INTERNAL=1 DEFAULT_DEFINES="-DTRACE_HEAD_CACHE_INCR -DPROFILE_LINKCOUNT -DLINKCOUNT_64_BITS""
+       <no errors>
+       <no warnings>
+Build 9: "DEBUG=0 DEFAULT_DEFINES="-DTRACE_HEAD_CACHE_INCR -DHASHTABLE_NOCHASING""
+       <no errors>
+       <no warnings>
+Build 10: "STATIC=1"
+       <no errors>
+       <no warnings>
+Build 11: "ADD_DEFINES="-DDYNAMORIO_APP_EXPORTS""
+       <no errors>
+       <no warnings>
+Build 12: "PROTECT_FROM_APP=0"
+       <no errors>
+       <no warnings>
+Build 13: "ADD_DEFINES="-DSIMULATE_ATTACK""
+       <no errors>
+       <no warnings>
+Build 14: "ADD_DEFINES="-DDGC_DIAGNOSTICS""
+       <no errors>
+       <no warnings>
+Build 15: "ADD_DEFINES="-DSHARING_STUDY""
+       <no errors>
+       <no warnings>
+Build 16: "ADD_DEFINES="-DSIDELINE""
+       <no errors>
+       <no warnings>
+Build 17: "CLIENT_INTERFACE=1 INTERNAL=1 DEFAULT_DEFINES="-DTRACE_HEAD_CACHE_INCR -DCLIENT_SIDELINE""
+       <no errors>
+       <no warnings>
+Build 18: "DEBUG=0 INTERNAL=0"
+       <no errors>
+       <no warnings>
+
+============================================================
diff --git a/suite/runregression b/suite/runregression
new file mode 100755
index 000000000..7f9d8916e
--- /dev/null
+++ b/suite/runregression
@@ -0,0 +1,955 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2002-2009 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+###########################################################################
+##
+## REGRESSION TESTER
+##
+# The default run is the short run.
+# The short run contains a smattering of configurations we do not want to mess up.
+# It does all builds that don't specify NOSHORT, and all runs prepended with SHORT::
+# The long run does all builds and all runs (including those prepended with SHORT::).
+#
+# See case syntax below for specifying runs.
+#
+# FIXME: for stress tests we may want to mix spec (for larger apps) with
+# multi-threaded and corner-case apps.  "spec runsecurity" is a good mix,
+# but perhaps we want a "stress" target in suite/tests/Makefile that
+# includes just codemod-threads and a handful of other good all-around
+# tests from win32/, etc.
+
+# "rm -r" hangs on open files on cygwin so we use perl rmtree
+use File::Path;
+
+if (`uname -s` =~ Linux) {
+    $on_linux = 1;
+    $on_windows = 0;
+    $D = '-D';
+} else {
+    $on_linux = 0;
+    $on_windows = 1;
+    $D = '/D';
+}
+
+# run at low priority to avoid hosing machines!
+$nice = "nice -19";
+
+# spawn as many jobs as possible -- I would limit to $NUMBER_OF_PROCESSORS
+# on win32 and `grep processor /proc/cpuinfo | xargs | awk '{print NR}'`
+# on linux except on cygwin we get errors:
+#   make[1]: warning: jobserver unavailable: using -j1.  Add `+' to parent make rule.
+# b/c of improper fork/submake support.  -j does unlimited children, seems
+# to run just as quickly though with slightly higher loads.
+$make_parallel_flag = "-j";
+# case 5424: we don't yet build in parallel as our Makefile doesn't support it
+$parallel_build = "";
+# case 5424: we don't yet run in parallel due to output interleaving problems
+$parallel_run = "";
+
+# Shortcuts for common build options
+$rel_ext = "DEBUG=0 INTERNAL=0";
+$rel_int = "DEBUG=0 INTERNAL=1";
+$dbg_ext = "DEBUG=1 INTERNAL=0";
+$dbg_int = "DEBUG=1 INTERNAL=1";
+
+$probe_mode = "-probe_api -hotp_only";
+$code_mode = "-code_api -probe_api";
+$mf_mode = "-security_api";
+
+# cases syntax :
+# A case is either a build or a run.
+#
+# A build takes the form of :
+#     BUILD::[WIN::|LIN::][NOSHORT::]<build options>
+# where WIN or LIN specifies that build is only to be performed on that platform
+# and NOSHORT specifies not to do that build for the short regression.
+#
+# A run is perfromed using the last build.  It can either be a sample build
+#     [SHORT::][WIN::|LIN::]BUILDSAMPLES::<api/samples makefile targets>
+# or a docs build
+#     [SHORT::][WIN::|LIN::]BUILDDOCS::<api/docs makefile targets>
+# or a regression test run
+#     [SHORT::][WIN::|LIN::][ONLY::<targerts>::]DYNAMORIO_OPTIONS=\\\"...\\\"
+#         [suite/tests Makefile var overrides]
+# where WIN and LIN work as before and SHORT says do the regression test run for a
+# short run of runregression. The targets for ONLY are "spec" to run the spec tests
+# and any valid suite/tests Makefile targets (if no ONLY then does "spec all").
+# We treat doc and sample builds as runs because they depend on the last build of core.
+#
+# See old-runregression-runs-2-6-08 for a very long list of runs and targets mostly
+# from Determina.  We've got a new smaller set to match our VMware interface focus.
+
+@cases = (
+# product builds and runs
+## VMAP builds and runs
+    "BUILD::VMAP=1 $dbg_int",
+        # our supported configurations
+        "SHORT::ONLY::spec runallbuthotp::DYNAMORIO_OPTIONS=\\\"-code_api\\\"",
+        # Assume is similar enough to VMSAFE that it's tests cover -thread_private
+        # and -disable_traces well enough.
+        # Test supported other options (-stack_size, logging, -no_hide, NOTE -no_hide is
+        # windows only!)
+        "WIN::ONLY::runcommon runinterface::DYNAMORIO_OPTIONS=\\\"-code_api -stack_size 120K -loglevel 1 -no_hide\\\"",
+        "LIN::ONLY::runcommon runinterface::DYNAMORIO_OPTIONS=\\\"-code_api -stack_size 120K -loglevel 1\\\"",
+        # cover -tracedump_* options, just a couple combinations
+        "ONLY::spec::DYNAMORIO_OPTIONS=\\\"-code_api -tracedump_text -tracedump_origins\\\" TEST_MINS=5",
+        "ONLY::spec::DYNAMORIO_OPTIONS=\\\"-code_api -thread_private -tracedump_binary \\\" TEST_MINS=5",
+        # make sure we at least sometimes exercise -checklevel > 0
+        "DYNAMORIO_OPTIONS=\\\"-checklevel 2\\\" TEST_MINS=5",
+
+    "BUILD::VMAP=1 $rel_ext",
+        # xref PR 306319, make sure samples build and run against release build too
+        "BUILDSAMPLES::EXCLUDE=MF_moduledb.c",
+        # just a few tests since we expect internal debug to catch most problems
+        "ONLY::runinterface::DYNAMORIO_OPTIONS=\\\"-code_api\\\"",
+
+    "BUILD::VMAP=1 $dbg_ext",
+        # build docs and samples
+        "SHORT::BUILDDOCS::pdf",
+        "SHORT::BUILDSAMPLES::EXCLUDE=MF_moduledb.c",
+        # just a few tests since we expect internal debug to catch most problems
+        "ONLY::runinterface::DYNAMORIO_OPTIONS=\\\"-code_api\\\"",
+
+## VMSAFE builds and runs
+    "BUILD::VMSAFE=1 $dbg_int",
+        # our supported configurations
+        "SHORT::ONLY::spec runallbuthotp::DYNAMORIO_OPTIONS=\\\"$code_mode\\\"",
+        "SHORT::ONLY::runsecurity runrunall runinterface::DYNAMORIO_OPTIONS=\\\"$mf_mode\\\"",
+        "ONLY::spec runallbuthotp::DYNAMORIO_OPTIONS=\\\"$mf_mode\\\"",
+        "SHORT::WIN::ONLY::runos runrunall::DYNAMORIO_OPTIONS=\\\"$probe_mode\\\"",
+        "WIN::ONLY::spec runcommon runsecurity::DYNAMORIO_OPTIONS=\\\"$probe_mode\\\"",
+        "ONLY::spec runallbuthotp::DYNAMORIO_OPTIONS=\\\"$code_mode -thread_private\\\"",
+        "ONLY::spec runallbuthotp::DYNAMORIO_OPTIONS=\\\"$code_mode -disable_traces\\\"",
+        "ONLY::spec runallbuthotp::DYNAMORIO_OPTIONS=\\\"$code_mode -thread_private -disable_traces\\\"",
+        "ONLY::spec::DYNAMORIO_OPTIONS=\\\"$code_mode -loglevel 1\\\"",
+        "ONLY::spec::DYNAMORIO_OPTIONS=\\\"$code_mode -stack_size 120k\\\"",
+        "SHORT::WIN::ONLY::runrunall runinterface::DYNAMORIO_OPTIONS=\\\"-enable_full_api\\\"",
+        # other configurations we may eventually support
+        "ONLY::runallbuthotp::DYNAMORIO_OPTIONS=\\\"-security_api -code_api\\\"",
+        "ONLY::runallbuthotp runprobeapi::DYNAMORIO_OPTIONS=\\\"-security_api -probe_api\\\"",
+        "ONLY::runallbuthotp runprobeapi::DYNAMORIO_OPTIONS=\\\"-security_api -code_api -probe_api\\\"",
+        # pcache tests: per-user so each app will merge w/ previous, plus merge
+        # w/ at-unload persists from earlier -desktop run 
+        "SHORT::WIN::ONLY::runrunall::DYNAMORIO_OPTIONS=\\\"-desktop -coarse_freeze_at_exit\\\"",
+        # run again so each must use the merged pcaches
+        "WIN::ONLY::runrunall::DYNAMORIO_OPTIONS=\\\"-desktop -coarse_freeze_at_exit\\\"",
+        "SHORT::WIN::ONLY::runrunall::DYNAMORIO_OPTIONS=\\\"-thin_client\\\"",
+
+    "BUILD::VMSAFE=1 $rel_ext",
+        # just a few tests since we expect internal debug to catch most problems
+        "SHORT::ONLY::runinterface::DYNAMORIO_OPTIONS=\\\"$code_mode\\\"",
+        "SHORT::ONLY::runsecurity runinterface::DYNAMORIO_OPTIONS=\\\"$mf_mode\\\"",
+        "SHORT::WIN::ONLY::runrunall::DYNAMORIO_OPTIONS=\\\"$probe_mode\\\"",
+
+    "BUILD::VMSAFE=1 $dbg_ext",
+        # build docs and samples
+        "SHORT::BUILDDOCS::VMSAFE=1 pdf",
+        "SHORT::BUILDSAMPLES::EXCLUDE=stats.c",
+        # just a few tests since we expect internal debug to catch most problems
+        "ONLY::runinterface::DYNAMORIO_OPTIONS=\\\"$code_mode\\\"",
+        "ONLY::runsecurity runinterface::DYNAMORIO_OPTIONS=\\\"$mf_mode\\\"",
+        "SHORT::WIN::ONLY::runrunall::DYNAMORIO_OPTIONS=\\\"$probe_mode\\\"",
+
+## Determina VPS builds and runs
+    "BUILD::VPS=1 $dbg_int",
+        "ONLY::spec runnointerface::DYNAMORIO_OPTIONS=\\\"\\\"",
+        "ONLY::spec runnointerface::DYNAMORIO_OPTIONS=\\\"-security\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -detect_mode -report_max 0 -no_diagnostics\\\"",
+        "WIN::ONLY::runnointerface::DYNAMORIO_OPTIONS=\\\"-security -liveshields\\\"",
+        "WIN::ONLY::runrunall runsecurity::DYNAMORIO_OPTIONS=\\\"-security -client\\\"",
+        "WIN::ONLY::runrunall runsecurity::DYNAMORIO_OPTIONS=\\\"-security -low\\\"", # not a default config
+        # Test with -use_moduledb relaxations, (is on by default, but tests Makefile
+        # adds -staged which turns it off, we re-enable by turning the option back on).
+        # The tests Makefile adds different version resource information to different
+        # tests in runall/ and security-{common,win32}/ to fully exercise this.
+        "WIN::ONLY::runrunall runsecurity::DYNAMORIO_OPTIONS=\\\"-security -use_moduledb\\\"",
+        
+    "BUILD::VPS=1 $rel_ext",
+        # just a few tests since we expect internal debug to catch most problems
+        "ONLY::runnointerface::DYNAMORIO_OPTIONS=\\\"\\\"",
+        "ONLY::runsecurity::DYNAMORIO_OPTIONS=\\\"-security -detect_mode -report_max 0 -no_diagnostics\\\"",
+        "WIN::ONLY::runnointerface::DYNAMORIO_OPTIONS=\\\"-security -liveshields\\\"",
+        "WIN::ONLY::runrunall runsecurity::DYNAMORIO_OPTIONS=\\\"-security -client\\\"",
+
+# non-product builds and runs
+
+## we don't really use internal release builds for anything, but keep it working
+    "BUILD::$rel_int",
+        "ONLY::runallbuthotp::DYNAMORIO_OPTIONS=\\\"\\\"",
+
+## build with nothing enabled
+    "BUILD::NOSHORT::VMSAFE=0 VMAP=0 VPS=0 $dbg_int",
+        "ONLY::runbase runsecurity runrunall::DYNAMORIO_OPTIONS=\\\"\\\"",
+
+## edge profiling
+    "BUILD::ADD_DEFINES=\"${D}PROFILE_LINKCOUNT ${D}LINKCOUNT_64_BITS\"",
+        "ONLY::spec::DYNAMORIO_OPTIONS=\\\"-prof_counts -tracedump_binary -no_inline_trace_ibl\\\" TEST_MINS=5",
+
+## release-build kstats and callprof
+    "BUILD::NOSHORT::DEBUG=0 INTERNAL=0 KSTATS=1 CALLPROF=1 PROFILE=1",
+        "ONLY::spec runallbuthotp::DYNAMORIO_OPTIONS=\\\"\\\"",
+
+## FIXME PR 215424: we should have runs of these but no x64 regression machines set up
+    "BUILD::ARCH=x64",
+    "BUILD::ARCH=x64 DEBUG=0",
+
+## defines we don't want to break -- no runs though since we don't currently use these
+    "BUILD::NOSHORT::ADD_DEFINES=\"${D}DGC_DIAGNOSTICS\"",
+    "BUILD::WIN::NOSHORT::DEBUG=1 INTERNAL=1 STRACE_CLIENT=1",
+    "BUILD::LIN::NOSHORT::ADD_DEFINES=\"${D}CHECK_RETURNS_SSE2\"",
+);
+
+$DYNAMORIO_HOME = $ENV{'DYNAMORIO_HOME'};
+$DYNAMORIO_LIBUTIL = $ENV{'DYNAMORIO_LIBUTIL'}; # win32-only
+$DYNAMORIO_LIBUTIL = "../libutil" if ($DYNAMORIO_LIBUTIL eq '');
+$DYNAMORIO_TOOLS = $ENV{'DYNAMORIO_TOOLS'};
+$DYNAMORIO_TOOLS = "../tools" if ($DYNAMORIO_TOOLS eq '');
+$DYNAMORIO_BENCHMARKS = $ENV{'DYNAMORIO_BENCHMARKS'};
+$DYNAMORIO_BENCHMARKS = "../benchmarks" if ($DYNAMORIO_BENCHMARKS eq '');
+$DYNAMORIO_API = $ENV{'DYNAMORIO_API'};
+$DYNAMORIO_API = "../api" if ($DYNAMORIO_API eq '');
+$DYNAMORIO_LOGDIR = $ENV{'DYNAMORIO_LOGDIR'};
+$DYNAMORIO_LOGDIR =~ s|\\|/|g;
+
+###########################################################################
+# arguments
+
+$usage = "Usage: $0 [<bmarkdir>] [-justbuild] [-justspec] [-justtests]
+  [-justapidocs] [-noapidocs]
+  [-justsharetests] [-nosharetests] [-test <test target>] [-run <run number>]
+  [-long] [-n] [-parallel_build] [-parallel_run] [-ignore_only]\n";
+$justbuild = 0;
+$justspec = 0;
+$justtests = 0;
+$justshow = 0;
+$short = 1;
+$sharetests = 0;
+$shareshort = 0; # disabling share tests as they've failed for years now
+$apidocs = 1;
+$singletest = "";
+$onlyrun = 0;
+$ignoreonly = 0;
+
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq '-justbuild') {
+        $justbuild = 1;
+    } elsif ($ARGV[0] eq '-justspec') {
+        $justspec = 1;
+    } elsif ($ARGV[0] eq '-justtests') {
+        $justtests = 1;
+    } elsif ($ARGV[0] eq '-parallel_build') {
+        $parallel_build = $make_parallel_flag;
+    } elsif ($ARGV[0] eq '-parallel_run') {
+        # don't do any makes in parallel
+        $parallel_run = $make_parallel_flag;
+    } elsif ($ARGV[0] eq '-ignore_only') {
+        # run all tests for all runs
+        $ignoreonly = 1;
+    } elsif ($ARGV[0] eq '-test') {
+        die $usage if ($#ARGV == 0);
+        shift;
+        $singletest = $ARGV[0];
+        print "tests target is $singletest\n";
+    } elsif ($ARGV[0] eq '-run') {
+        die $usage if ($#ARGV == 0);
+        shift;
+        # FIXME: may allow a list of runs here
+        # Run number obtained by runregression -n
+        $onlyrun = $ARGV[0];
+        print "run only Run $onlyrun\n";
+    } elsif ($ARGV[0] eq '-long') {
+        $short = 0;
+        $sharetests = 1;
+    } elsif ($ARGV[0] eq '-n') {
+        $justshow = 1;
+        print "just showing\n";
+    } elsif ($ARGV[0] eq '-justsharetests') {
+        $justbuild = 1;
+        $sharetests = 1;
+    } elsif ($ARGV[0] eq '-nosharetests') {
+        $sharetests = 0;
+        $shareshort = 0;
+    } elsif ($ARGV[0] eq '-justapidocs') {
+        $justbuild = 1;
+        $apidocs = 1;
+    } elsif ($ARGV[0] eq '-noapidocs') {
+        $apidocs = 0;
+    } elsif ($ARGV[0] !~ /^-/ && !defined($bmarkbase)) {
+        $bmarkbase = $ARGV[0];
+    } else {
+        print "$usage";
+        exit 0;
+    }
+    shift;
+}
+
+if (!defined($bmarkbase)) {
+    $bmarkbase = $DYNAMORIO_BENCHMARKS;
+}
+$bmarkbase =~ s|benchmarks$||;
+if (!$justtests && !$justbuild && ! -d "$bmarkbase/benchmarks") {
+    print "$bmarkbase is not a benchmark directory\n";
+    print "$usage";
+    exit 0;
+}
+$topdir = `pwd`;
+chop $topdir;
+
+$dstamp = `date | awk '{print \$6 "-" \$2\$3}'`;
+chop $dstamp;
+$nstamp = 1;
+$stamp = sprintf("regression-%s.%03d", $dstamp, $nstamp);
+while (-d $stamp) {
+    $nstamp++;
+    $stamp = sprintf("regression-%s.%03d", $dstamp, $nstamp);
+}
+$curdate = `date`;
+print "beginning run @ $curdate";
+print "log directory is $stamp\n";
+if (!$justshow) {
+    mkdir($stamp);
+    chdir($stamp);
+}
+
+$coredir = "core";
+
+if (! -d "$DYNAMORIO_HOME/$coredir") {
+    print "ERROR: Cannot find $DYNAMORIO_HOME/$coredir, exiting\n";
+    exit 0;
+}
+if (!$justshow) {
+    print "Copying sources from $DYNAMORIO_HOME/$coredir\n";
+    system("cp -r $DYNAMORIO_HOME/$coredir .");
+    # On Vista regressions mc complains access denied opening win32/events.mc 
+    system("chmod -R aug+rxw $coredir");
+    # eliminate any custom settings
+    if (-f "$coredir/Makefile.mydefines") {
+        system("rm $coredir/Makefile.mydefines");
+    }
+}
+
+$build_ok = 1;
+
+if (!$justshow) {
+    system("cd $coredir; make -s ugly >> $topdir/$stamp/make_ugly_s.log 2>&1");
+    print "\n============================================================\n\n";
+    $make_ugly_result = "make ugly: succeeded\n";
+    open(IN, "< $topdir/$stamp/make_ugly_s.log");
+    while (<IN>) {
+        # Handle CR in case non-unix default cygwin
+        s/\r//;
+        if (/^Make ugly rule:/) {
+            $make_ugly_rule = $_;
+        } else {
+            $make_ugly_result = "make ugly: FAILED\n";
+            print $make_ugly_rule;
+            $make_ugly_rule = "";
+            print $_;
+        }
+    }
+    print $make_ugly_result;
+    $summary .= $make_ugly_result
+}
+
+# Make sure the libutil module builds (needed for tools below).
+if ($on_windows) {
+    print "\n============================================================\n";
+    print "\nbuilding libutil module\n";
+    if (!$justshow && system("cd $DYNAMORIO_LIBUTIL && make DBG= LINK_DBG= clean all >> $topdir/$stamp/libutil_build.log 2>&1") != 0) {
+        print "libutil module build ERROR\n";
+        $summary .= "LIBUTIL: FAILED\n";
+        $build_ok = 0;
+    } else {
+        print "libutil module build succeeded\n";
+        $summary .= "LIBUTIL: succeeded\n";
+    }
+}
+
+# Make sure the tools module builds (some tools are needed for the regressions
+# including runstats on Linux).
+print "\nbuilding tools module\n";
+# case 192750: cannot create pdb files over public key ssh, so we unset {LINK_,}DBG
+if (!$justshow && system("cd $DYNAMORIO_TOOLS && make DBG= LINK_DBG= clean all >> $topdir/$stamp/tools_build.log 2>&1") != 0) {
+    print "tools module build ERROR\n";
+    $summary .= "TOOLS: FAILED\n";
+    $build_ok = 0;
+} else {
+    print "tools module build succeeded\n";
+    $summary .= "TOOLS: succeeded\n";
+}
+
+if ($on_windows) {
+    # stop nodemgr from interfering with the runall tests
+    print "\nStopping nodemgr\n";
+    if (!$justshow) {
+        $nodemgr_running = (system("net stop scnodemgr") == 0);
+    } else {
+        $nodemgr_running = 0; # no need to restart
+    }
+}
+
+print "\ncleaning tests \n";
+# FIXME: should make runnative if we are to extract addresses from it
+if (!$justshow && system("cd $topdir/tests && make clean >> $topdir/$stamp/tests_clean.log 2>&1") != 0) {
+    $summary .= "TESTS: clean failed\n";
+}
+
+# FIXME: all of this should be replaced with functionality redirecting syslog to stderr
+# NOTE - should be kept in synch with the default OPS value in tests/Makefile
+$MESSAGES_TO_STDERR_LINUX = "-stderr_mask 0xC -dumpcore_mask 0";
+$ADDITIONAL_TEST_OPTIONS_LINUX = $MESSAGES_TO_STDERR_LINUX;
+$MESSAGES_TO_STDERR_WIN32 = "-stderr_mask 0xC -msgbox_mask 0 -dumpcore_mask 0xfd";
+$ADDITIONAL_TEST_OPTIONS_WIN32 = $MESSAGES_TO_STDERR_WIN32 . " -staged";
+
+# Save the current registry settings, and then make sure the registry
+# is in a usable state.
+if (!$justshow && $on_windows) {
+    $wintopdir = `cygpath -da $topdir`;
+    chomp($wintopdir);
+
+    # drcontrol -save fails when there is no top-level Determina registry
+    # entry, so do -create first.  (only creates if non-existent)
+    system("$DYNAMORIO_TOOLS/DRcontrol -create \"$wintopdir\"");
+
+    $regression_backup = "$wintopdir\\regression-backup.mfp";
+    system("rm -f $regression_backup") if (-e $regresion_backup);
+    system("$DYNAMORIO_TOOLS/drcontrol -save \"$regression_backup\"");
+
+    # Some tests rely on having valid CACHE_ROOT and CACHE_SHARED
+    # registry settings and directories.
+    # DRcontrol copies permissions from $DYNAMORIO_HOME/logs and lib,
+    # so make sure those directories exist.
+    $dr_home = `$DYNAMORIO_TOOLS/DRcontrol -fulldump|grep DYNAMORIO_HOME=|head -1`;
+    $dr_home =~ s/.*DYNAMORIO_HOME=([\w:~\\\/]+).*\n/$1/;
+    system("mkdir -p \"$dr_home/logs\"") if (! -e "$dr_home/logs");
+    system("mkdir -p \"$dr_home/lib\"") if (! -e "$dr_home/lib");
+    # We use a new cache directory to isolate the suite.  Rather than
+    # have a per-run dir (in $stamp), we re-use one in suite/ to save
+    # disk space.  Make sure the directory is fresh when we start the
+    # regression.  We don't need drcontrol -sharedcache here, however,
+    # because runalltest.sh will do that where needed.
+    system("rm -rf $topdir/cache/*") if (-e "$topdir/cache");
+    $ENV{'DYNAMORIO_CACHE_ROOT'} = "$wintopdir\\cache";
+    $ENV{'DYNAMORIO_CACHE_SHARED'} = "$wintopdir\\cache\\shared";
+}
+
+chdir("$coredir");
+$srcdir = `pwd`;
+chop $srcdir;
+$b = 1; # build number
+$n = 1; # to match runsuite's .001, .002, etc.
+$i = 0;
+$runtarget = "";
+while ($i <= $#cases) {
+    if (! ($cases[$i] =~ /^BUILD::/) ) {
+        print "Error: not a build option: $cases[$i]\n";
+        last; #exit;
+    } else {
+        $case = $cases[$i];
+        $case =~ s/^BUILD:://;
+    }
+
+    if (($case =~ /^WIN::/ && $on_linux) ||
+        ($case =~ /^LIN::/ && $on_windows)) {
+        $i++;
+        while ($i <= $#cases && !($cases[$i] =~ /^BUILD::/)) {
+            $i++;
+        }
+        next;
+    }
+    $case =~ s/^LIN:://;
+    $case =~ s/^WIN:://;
+
+    if ($case =~ /^NOSHORT::/ && $short) {
+        $i++;
+        while ($i <= $#cases && !($cases[$i] =~ /^BUILD::/)) {
+            $i++;
+        }
+        next;
+    }
+    $case =~ s/^NOSHORT:://;
+
+    $logfile = sprintf("buildlog.%03d", $b);
+    print "\n============================================================\n";
+    print "Build $b: $case\n\n";
+    print "Building......\n";
+    
+    system("echo \"$n: $case\n\n\" >> ../$logfile 2>&1");
+    # use clear instead of clean so we don't have to worry about defines
+    # xref case 9783, need to override BUILD for newer cygwin versions for clear,
+    # but must unset USEDEP so we don't try to include "/buildmark.dep"
+    # FIXME case 192750: cannot create pdb files over public key ssh
+    if (!$justshow) {
+        system("($nice make USEDEP=0 BUILD= clear; $nice make $parallel_build $case DBG= LINK_DBG= install) >> ../$logfile 2>&1");
+    }
+    system("grep -i [^W]error ../$logfile");
+    $errors = `grep -i [^W]error ../$logfile`;
+    if ($errors eq '') {
+        $errors = "<no errors>\n";
+    } else {
+        $errors = "*** errors in build ***\n";
+        $build_ok = 0;
+    }
+    $summary .= "Build $b: \"$case\"\n       $errors";
+    system("grep -i warning ../$logfile");
+    $warn = `grep -i warning ../$logfile`;
+    if ($warn eq '') {
+        $warn = "<no warnings>\n";
+    } else {
+        $warn = "*** warnings in build ***\n";
+        $build_ok = 0;
+    }
+    $summary .= "       $warn";
+    $b++;
+
+    # by default on the short run we run the share tests for the
+    #  release build only.
+    if ($on_windows && 
+        ($sharetests || ($shareshort && $cases[$i] eq $release_build)) &&
+        $onlyrun == 0) {
+        print "Running share tests......\n";
+        $logfile = sprintf("share_tests_%03d.log", $b-1);
+        if (!$justshow) {
+            $shareres = system("cd $DYNAMORIO_LIBUTIL && DYNAMORIO_HOME=$topdir/$stamp make -s tests_test >> $topdir/$stamp/$logfile 2>&1");
+        }
+        # FIXME: keep getting problems where these testers are left around
+        # when I have AppInit already set or other conditions, should fix up
+        # share tests to handle that but for now killing
+        system("$DYNAMORIO_TOOLS/drkill -exe tester_1.exe");
+
+        $shareresults = `grep -i 'Test Results' $topdir/$stamp/$logfile`;
+        print "libutil module $shareresults\n";
+
+        if (!$justshow && $shareres != 0) {
+            $summary .= "  SHARETESTS: FAILURE! $shareresults\n";
+            $build_ok = 0;
+        }
+        else {
+            $summary .= "  SHARETESTS: $shareresults\n";
+        }
+    }
+
+    for ($i++; $i <= $#cases; $i++) {
+        if ($cases[$i] =~ /^BUILD::/) {
+            last;
+        }
+
+        if ($cases[$i] =~ /^SHORT::/) {
+            $cases[$i] =~ s/^SHORT:://;
+        } else {
+            if ($short) {
+                next;
+            }
+        }
+
+        if (($cases[$i] =~ /^WIN::/ && $on_linux) ||
+            ($cases[$i] =~ /^LIN::/ && $on_windows)) {
+            next;
+        }
+        $cases[$i] =~ s/^LIN:://;
+        $cases[$i] =~ s/^WIN:://;
+
+        # make sure our docs build
+        if ($cases[$i] =~ /^BUILDDOCS::/) {
+            $cases[$i] =~ s/^BUILDDOCS:://;
+            if (! $apidocs) {
+                next;
+            }
+            print "\n------------------------------\n";
+            print "API DOCS \"$cases[$i]\"\n";
+            $logfile = sprintf("apilog_docs.%03d.log", $b);
+            my $homedir = "$topdir/$stamp";
+            if ($on_windows) {
+                $homedir = `cygpath -ma $homedir`;
+                chomp $homedir;
+            }
+            # when we rely on the "make clean pdf" to remove genimages and
+            # remake it, I get a "permission denied" on the "mkdir genimages":
+            # can't repro manually, but it seems to be that the rm is not
+            # yet done.  so, we split that out in a separate command.
+            my $docres = system
+            ("export DYNAMORIO_HOME=$homedir; ".
+             "(rm -rf $DYNAMORIO_API/docs/genimages && ".
+             "make -C $DYNAMORIO_API/docs clean $cases[$i]) >> $topdir/$stamp/$logfile 2>&1");
+            if ($docres != 0) {
+                $summary .= "  DOCS: BUILD FAILURE!\n";
+                $build_ok = 0;
+            } else {
+                $summary .= "  DOCS: success\n";
+            }
+            next;
+        }
+
+        # make sure our api samples build and run
+        if ($cases[$i] =~ /^BUILDSAMPLES::/) {
+            $cases[$i] =~ s/^BUILDSAMPLES:://;
+            if (! $apidocs) {
+                next;
+            }
+            print "\n------------------------------\n";
+            print "API Samples \"$cases[$i]\"\n";
+            my $homedir = "$topdir/$stamp";
+            if ($on_windows) {
+                $homedir = `cygpath -ma $homedir`;
+                chomp $homedir;
+            }
+            $logfile = sprintf("apilog_samples.%03d.log", $b);
+            my $SETINC = '';
+            my $SETLIB = '';
+            my $SETPATH = '';
+            if ($on_windows) {
+                # set up LIB, PATH, and INCLUDE env vars (samples/Makefile can't assume TCROOT)
+                $SETINC = 'INCLUDE=`cygpath -wa "${TCROOT}/win32/winsdk-6.1.6000/VC/INCLUDE"`\;`cygpath -wa "${TCROOT}/win32/winsdk-6.1.6000/Include"`';
+                $SETLIB = 'LIB=`cygpath -wa "${TCROOT}/win32/winsdk-6.1.6000/VC/LIB"`\;`cygpath -wa "${TCROOT}/win32/winsdk-6.1.6000/Lib"`';
+                $SETPATH = 'export PATH=\"`cygpath -wa "${TCROOT}/win32/winsdk-6.1.6000/VC/Bin"`:$PATH\";';
+            }
+            my $sampleres = system("$SETPATH DYNAMORIO_HOME=$homedir $SETINC $SETLIB ".
+                                   "make -C $DYNAMORIO_API/samples DEBUG=1 SHOW_RESULTS=0 DBG= ".
+                                   "clean $cases[$i] all >> $topdir/$stamp/$logfile 2>&1");
+            if ($sampleres != 0) {
+                $summary .= "  SAMPLES: BUILD FAILURE!\n";
+                $build_ok = 0;
+            } else {
+                $summary .= "  SAMPLES: success\n";
+            }
+            if ($on_windows) {
+                # Try running the clients.  This is pretty hacky: due to lack of
+                # symlinks on windows, we copy to a temp suite/tests/ dir and
+                # invoke "make %.runinterface" w/ generated control files.
+                $logfile = sprintf("apilog_runs.%03d.log", $b);
+                my $samples = `make -s -C $DYNAMORIO_API/samples $cases[$i] listsrcs`;
+                print "\n";
+                foreach $s (split(' ', $samples)) {
+                    print "Testing sample client $s\n";
+                    $s =~ s|\.c||; # get base
+                    my $tmpdir = "temp_tests";
+                    my $tgt = "$topdir/tests/$tmpdir";
+                    my $runall = "notepad.exe\\\\nUntitled - Notepad\\\\n<>\\\\n";
+                    my $expect = "Process notepad.exe, running SC debug\\\\n".
+                        "Close message sent.\\\\nNo such process found.";
+                    if ($s eq "stats") {
+                        # we leave this printout as a sanity check of shmem
+                        # this will fail on NT: need to remove the Global\ there
+                        $expect = "Shared memory key is: \\\"Global\\\\DynamoRIO_Client_Statistics".
+                            ".000\\\"\\\\n" . $expect;
+                    }
+                    my $runres = system
+                        ("export DYNAMORIO_HOME=$homedir; (mkdir -p $tgt; ".
+                         "cp $DYNAMORIO_API/samples/$s.dll $tgt/$s.dll.dll; ".
+                         "echo -e $runall > $tgt/$s.runall; ".
+                         "echo -e $expect > $tgt/$s.expect; ".
+                         "cd $topdir/tests; make DEBUG=1 $tmpdir/$s.runinterface)".
+                         " >> $topdir/$stamp/$logfile 2>&1");
+                    my $grepres = system("grep -q FAIL $topdir/$stamp/$logfile");
+                    if ($runres != 0 || $grepres == 0) {
+                        $summary .= "  SAMPLE RUN $s: FAILURE!\n";
+                        $build_ok = 0;
+                    } else {
+                        $summary .= "  SAMPLE RUN $s: success\n";
+                    }
+                    system("rm -rf $tgt");
+                }
+            }
+            next;
+        }
+
+        if ($justbuild) {
+            $n++;
+            next;
+        }
+
+        if ($onlyrun != 0 && $onlyrun != $n) {
+            $n++;
+            next; 
+        }
+
+        if (!$ignoreonly && $cases[$i] =~ /^ONLY::/) {
+            $cases[$i] =~ s/^ONLY::([^:]+):://;
+            $runtarget = $1;
+        } else {
+            # default is to run spec and all tests
+            $runtarget = "spec all";
+        }
+
+        $logfile = sprintf("runlog.%03d", $n);
+        print "\n------------------------------\n";
+        print "Run $n: $cases[$i]\n";
+        $curdate = `date`;
+        print "beginning run @ $curdate\n";
+        chdir("..");
+        $curdir = `pwd`;
+        chop $curdir;
+        $summary .= "  Run $n: $runtarget : $cases[$i]\n";
+        print "target for this run is \"$runtarget\"\n";
+
+        # Note we could let the tests Makefile add these for us (by setting
+        # DYNAMORIO_OPTIONS instead of OPS on the cmd line) but since we need it for the
+        # spec tests too we just go ahead and do it here. The added options go first so
+        # individual runs can override them at need.
+        if (`uname -s` =~ Linux) {
+            $cases[$i] =~ s/\"(.*)\\/\"$ADDITIONAL_TEST_OPTIONS_LINUX \1\\/;
+        } else {
+            $cases[$i] =~ s/\"(.*)\\/\"$ADDITIONAL_TEST_OPTIONS_WIN32 \1\\/;
+        }
+
+        if (!$justtests && $runtarget =~ /spec/) {
+            $runtarget =~ s/ *spec *//;
+            ###################################################
+            # SPEC
+            $cmd = "export DYNAMORIO_HOME=$curdir; $nice $topdir/runsuite $bmarkbase regression regression $cases[$i]";
+            print "$cmd\n";
+            system("(echo; echo \"$n: $cases[$i]\n\"; echo) >> $logfile 2>&1");
+            if (!$justshow) {
+                system("$cmd >> $topdir/$stamp/$logfile 2>&1");
+            }
+            print("Results of SPEC run:\n");
+            system("grep -i fail $logfile");
+            print("\n");
+            # keep track of errors for summary at end
+            $errors = `grep -i fail $logfile | grep -v '^No '`;
+            $passes = `grep -c " ok " $logfile`;
+            chop $passes;
+            if ($errors eq "") {
+                $sum_failed = "0 failed";
+                $sum_mark = "   ";
+            } else {
+                $sum_mark = "***";
+                if ($errors =~ /failed: ([0-9]+)/) {
+                    $num_fails = $1;
+                } else {
+                    $num_fails = 0;
+                }
+                $sum_failed = "$num_fails failed";
+                if ($errors =~ /post-verify/) {
+                    # this number includes all sub-tests
+                    $errors =~ /post-verify.*([0-9]+)/;
+                    $sum_failed .= ", $1 sub-tests failed";
+                    system("grep -E '[Ee]rror:' $logfile");
+                    print "\n";
+                }
+            }
+            $summary .= "   $sum_mark SPEC: $passes passed, $sum_failed $sum_mark\n";
+            if (!$justshow) {
+                # set up links to SPEC dirs and files
+                $rundir = `grep "Running in" $logfile`;
+                chop $rundir;
+                $rundir =~ s/.*$stamp\///;
+                system("ln -s $rundir $link");
+                print "run directory is $rundir <= $link\n";
+                $detailedlogfile = sprintf("runlog-details.%03d", $n);
+                system("ln -s $rundir/log.regression $detailedlogfile");
+                $resultsfile = sprintf("results.%03d", $n);
+                system("ln -s $rundir/results.regression $resultsfile");
+            }
+            if (!$justspec) {
+                # add separators before tests
+                system("(echo \"===========================================================================\") >> $logfile 2>&1");
+                print "\n";
+            }
+        } else {
+            $runtarget =~ s/ *spec *//;
+        }
+
+        if (!$justspec && $runtarget ne "") {
+            ###################################################
+            # our own tests
+            system("(echo; echo \"suite/tests\"; echo) >> $logfile 2>&1");
+            $link = sprintf("rundir.%03d", $n);
+            $ops = $cases[$i];
+            $ops =~ s/DYNAMORIO_OPTIONS=\\(.*)\\/\1/;
+            $defs_dir = $curdir;
+            $defs_dir .="/build/x86_";
+            if (`uname -s` =~ Linux) {
+                $defs_dir .="linux_";
+            } else {
+                $defs_dir .="win32_";
+            }       
+            if ($case =~ /DEBUG=0/) {
+                if ($case =~ /INTERNAL=1/) {
+                    $defs_dir .="intr";
+                } else {
+                    $defs_dir .="rel";
+                }
+            } else {
+                $defs_dir .="dbg";
+            }
+            if (!$justshow) {
+                $defs = `grep "DX86" $defs_dir/defines.h`;
+                $defs =~ s/\#define DYNAMORIO_DEFINES (.*)/\1/;
+                chop $defs;
+            }
+            # running the tests make in parallel simply means doing runbase,
+            # runsecurity, and runrunall in parallel -- there is no danger of
+            # individual runall tests running in parallel!
+            # FIXME: we could do better and parallelize all the non-runall tests
+            if ($singletest ne "") {
+                $runtarget = " $singletest";
+            }
+            # case 192750: cannot create pdb files over public key ssh,
+            # so we unset {LINK_,}DBG
+            $cmd = "export DYNAMORIO_HOME=$curdir; cd $topdir/tests; $nice make -k $parallel_run RESDIR=$topdir/$stamp/$link-tests OPS=$ops DEFS=$defs DBG= LINK_DBG= $runtarget";
+            print "$cmd\n";
+            if (!$justshow) {
+                system("$cmd >> $topdir/$stamp/$logfile 2>&1");
+            }
+            print("Results of tests run:\n");
+            $fails = 0;
+            $passes = 0;
+            $fail_names = "";
+            open(RES, "< $logfile") || die "Error opening $logfile";
+            while (<RES>) {
+                if (/^----- ([^ ]+) --/) {
+                    $name = $1;
+                } elsif (/^FAIL/ || /^make.*Error/) {
+                    $fails++;
+                    if (/^make.*\[([^\]]+)\].*Error/) {
+                        $name = $1;
+                    }
+                    $fail_names .= "$name ";
+                } elsif (/^PASS/) {
+                    $passes++;
+                }
+            }
+            close(RES);
+            if ($fails == 0) {
+                print "No failures -- $passes tests passed\n";
+                $summary .= "       TESTS: $passes passed, 0 failed\n";
+            } else {
+                print "****** $passes passed, $fails failed: $fail_names ******\n";
+                $summary .= "   *** TESTS: $passes passed, $fails failed: $fail_names ***\n";
+            }
+            # don't leave these guys running, sometimes they don't die nicely even w/ runstats
+            if (`uname -s` =~ Linux) {
+                system("killall -9 pthreads ptsig > /dev/null 2>&1");
+            }
+            if ($on_windows) {
+                # TEMPORARY workaround for case 10798
+                system("$DYNAMORIO_TOOLS/drkill -exe aslr-dll.exe");
+                # threadinjection is also often left around
+                system("$DYNAMORIO_TOOLS/drkill -exe threadinjection.exe");
+            }
+            # compress any ldmps produced
+            # TODO: gzips tend to hang and block the rest of the regression 
+            # tests on some ldmps, so run them in the background and kill them 
+            # after 10 seconds; xref case 9756
+            system("cd $DYNAMORIO_LOGDIR; gzip *.ldmp > /dev/null 2>&1 &");
+            sleep 10;
+            system("$DYNAMORIO_TOOLS/drkill -exe gzip.exe");
+        }
+
+        if (!$justbuild) {
+            if ($on_linux) {
+                # don't leave these guys running, sometimes they don't die nicely even w/ runstats
+                # (since they create children of children and runstats only kills grandparent?)
+                # kill after reach run rather than at end since they can eat up too many cycles
+                system("killall -9 pthreads ptsig clone vfork-fib > /dev/null 2>&1");
+                
+                # clean up any stale shared memory (fills up fast)
+                # key will be 0x0*
+                system("for i in `ipcs -m | grep 0x0 | awk '{print $2}'`; do ipcrm shm $i > /dev/null; done");
+            }
+        }
+
+        chdir($srcdir);
+        $n++;
+    }
+}
+chdir("..");
+    
+# restore registry
+if (!$justshow && $on_windows) {
+    system("$DYNAMORIO_TOOLS/DRcontrol -load \"$regression_backup\"")
+        if (-e $regresion_backup);
+}
+
+if ($justshow) {
+    system("rm buildlog* runlog*");
+} else {
+    # save space
+    # don't need duplicate copies of core and exports for each run
+    # yes the builds do differ but exports add up to >10MB
+    system("find . -name $coredir.tar.bz2 -exec rm {} \\;");
+    system("find . -name exports.tar.bz2 -exec rm {} \\;");
+    system("tar cjf $coredir.tar.bz2 $coredir");
+    rmtree("$coredir", 0, 0);
+    system("tar cjf exports.tar.bz2 exports");
+    rmtree(["exports", "build", "inject", "lib"], 0, 0);
+    if (!$justspec) {
+        # tests run dirs take up a lot of space, compress them
+        system("tar czf rundir-alltests.tgz rundir.*tests");
+        rmtree([glob("rundir.*tests")], 0, 0);
+    }
+}
+
+print "\n============================================================\n";
+print "Summary of results:\n\n";
+print $summary;
+print "\n============================================================\n";
+$curdate = `date`;
+print "ending run @ $curdate\n";
+
+print "\n====================== Regression Suite Core Errors ======================\n\n";
+
+# get summary of DR errors
+system("grep -h -E 'Unrecoverable|Internal DynamoRIO Error|CURIOSITY' runlog.* | sort | uniq -c | awk '{s+=\$1} END {printf \"Total DR errors: %d\\n\\n\", s}'");
+system("grep -h -o -E 'Unrecoverable.*|Internal DynamoRIO Error.*|CURIOSITY.*' runlog.* | sort | uniq -c | sort -nr");
+
+print "\n====================== Regression Suite Test Failures ======================\n\n";
+
+# get summary of DR errors
+system("grep -h -E '^----|FAIL' runlog.* | grep -B 1 FAIL | grep '^----' | sed 's/---*//g' | sort | uniq -c | awk '{s+=\$1} END {printf \"Total failing tests: %d\\n\\n\", s}'");
+system("grep -h -E '^----|FAIL' runlog.* | grep -B 1 FAIL | grep '^----' | sed 's/---*//g' | sort | uniq -c | sort -nr");
+print "\n";
+system("grep -h '^make' runlog.* | sort | uniq -c | awk '{s+=\$1} END {printf \"Total make errors: %d\\n\\n\", s}'");
+system("grep -h '^make' runlog.* | sort | uniq -c | sort -nr");
+
+print "\n";
+
+if ($on_windows) {
+    $hung = `$DYNAMORIO_TOOLS/DRview -listall -cmdline | grep -B 1 suite | grep '^PID' | wc`;
+    # we expect to see 2 processes: sh.exe and grep.exe
+    if ($hung !~ /^\s*2\s/) {
+        print "Hang check FAILED: Processes may be hung, > the 2 expected in the list:\n";
+        system("$DYNAMORIO_TOOLS/DRview -listall -cmdline | grep -B 1 suite | grep '^PID'");
+    }
+
+    if ($nodemgr_running) {
+        # re-start nodemgr
+        system("net start scnodemgr");
+    }
+}
+
+if ($build_ok) {
+    print "all builds succesful\n";
+} else {
+    print "BUILD ERROR\n";
+}
diff --git a/suite/runsuite b/suite/runsuite
new file mode 100755
index 000000000..3cf3babd0
--- /dev/null
+++ b/suite/runsuite
@@ -0,0 +1,400 @@
+#!/bin/bash
+
+# **********************************************************
+# Copyright (c) 2002-2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+###########################################################################
+##
+## BENCHMARK SUITE LAUNCHER FOR SPEC2000
+##
+## Purpose: to keep all of the data from our runs in one place.
+## All data is kept under /home/rio/suite/<key>/<machine>/<dataset-date.run>
+## Cagfarm machines are divided into cagfarm-{low,med,high} based on:
+##   cagfarm 1-10:  846 MHz Pentium 3
+##   cagfarm 11-26: 993 MHz Pentium 3
+##   cagfarm 27-49: 2.2 GHz Pentium 4
+## After a run all of the generated dynamorio log files are tarred up and stored
+## in the data directory so that the benchmarks directory can be cleaned
+## and spec data file space reclaimed without losing dynamo log files.
+## Also, unless the remotedynamo flag is used, the dynamo source and binary files
+## are copied from $DYNAMORIO_HOME into the data directory so we can recreate
+## runs in the future.
+##
+## You must give the base of your benchmarks directory as the first argument.
+## Then you tell it test or ref.
+## Then you give it a "key", which will be used for the script files as
+## well as the run directory names.  If you use a preset key "native", "base",
+## or "sidebase", your initial options will be set for you.  You can list
+## additional options as arguments after the key.
+## Right now there is no explicit support for optimization, but you can list
+## the OPTIMIZE=no option.  You can also override the default COPYDATA=yes
+## option by adding COPYDATA=no to the end of the options.
+##
+## There will be forthcoming support on automatically generating
+## data on slowdown vs. established native run times.
+##
+## examples of usage (note how quotes must be escaped):
+##   runsuite /home/bits6/iye ref native
+##   runsuite /home/bits6/iye test base
+##   runsuite /home/bits6/iye test myop DYNAMORIO_OPTIONS=\"-loglevel 1 -superop\" COPYDATA=no
+## using the intel compiler:
+##   runsuite /scratch/iye rebuild test native CCOMP=\"icc -O3\" FCOMP=\"ifc -O3\"
+
+###########################################################################
+# arguments
+
+usage="<bmarkdir> <dataset> [remotedynamo] [rebuild] [cleanall] [noclean] native|release|base|sidebase|<key> [<runops>]"
+
+# mandatory 1st 2 arguments
+if [ $# -lt 2 ]; then
+    echo "Usage: $0 $usage"
+    exit 127
+fi
+bmarkbase=$1
+if ! test -e "$bmarkbase/benchmarks" ; then
+    echo "Error: cannot find directory $bmarkbase/benchmarks"
+    echo "Usage: $0 $usage"
+    exit 127
+fi
+shift
+dataset=$1
+shift
+
+# rest of arguments
+key=""
+noclean=0
+rebuild=0
+cleanall=0
+remotedynamo=0
+while [ $# -ge 1 ]; do
+    case $1 in
+    # separate options
+    help)     echo "Usage: $0 $usage"; exit 127 ;;
+    rebuild)  rebuild=1 ;;
+    noclean)  noclean=1 ;;
+    cleanall) cleanall=1 ;;
+    remotedynamo) remotedynamo=1 ;;
+    # pre-set keys
+    native)   key="native"; shift; break ;;
+    release)  key="release"; shift; break ;;
+    base)     key="base"; shift; break ;;
+    sidebase) key="sidebase"; shift; break ;;
+    *)        key=$1; shift; break ;;
+    esac
+    shift
+done
+
+# all remaining arguments are part of options
+# quoted arguments must have their quotes escaped (\") to remain
+# quoted when passed through to RunSuite
+ops=""
+while [ $# -ge 1 ]; do
+    ops="$ops $1"
+    shift
+done
+
+# check some requirements
+if [ -z $key ] ; then
+    echo "Usage: $0 $usage"
+    exit 127
+fi
+
+# get location of runsuite
+# FIXME: find way to avoid hardcoding script name here
+scriptdir=${0/runsuite/}
+if [ -z $scriptdir ] || [ ${scriptdir:0:1} != "/" ]; then
+    scriptdir=`pwd`"/$scriptdir";
+fi
+
+function rmrfcmd {
+    if [ `uname -s` == "Linux" ]; then
+        rm -rf $*;
+    else
+        # get around rm -r bug where it hangs on open files
+        # we could use "cmd /c rmdir /s/q" but then we'd have to change
+        # paths to win32 so we invoke perl rmtree instead
+        perl -e 'use File::Path; rmtree([glob("$*")], 1, 0);';
+    fi
+}
+
+# can't use rmrf function w/ find's -exec so we have to duplicate here
+# also can't have var point to perl string as have problems w/ eval of var
+function findrmrfcmd {
+    if [ `uname -s` == "Linux" ]; then
+        find $* -exec rm -rf {} \;
+    else
+        # get around rm -r bug where it hangs on open files
+        # we could use "cmd /c rmdir /s/q" but then we'd have to change
+        # paths to win32 so we invoke perl rmtree instead
+        find $* -exec perl -e 'use File::Path; rmtree([glob("{}")], 1, 0);' \;
+    fi
+}
+
+###########################################################################
+# build directory for this run
+
+# make dir for this run: key/host/date.run
+if ! test -e $key ; then 
+    mkdir $key
+fi
+cd $key
+hostnm=`hostname`
+host=$hostnm
+if [ ${host:0:8} == "cagfarm-" ]; then
+    # split cagfarms up into three groups, low, med, and high:
+    #   cagfarm 1-10:  846 MHz Pentium 3
+    #   cagfarm 11-26: 993 MHz Pentium 3
+    #   cagfarm 27-49: 2.2 GHz Pentium 4
+    if [ ${host:8:1} == "0" ] || [ $host == "cagfarm-10" ]; then
+        host="cagfarm-low"
+    elif [ ${host:8:1} == "3" ] || [ ${host:8:1} == "4" ] ||
+         [ $host == "cagfarm-27" ] || [ $host == "cagfarm-28" ] ||
+         [ $host == "cagfarm-29" ]; then
+        host="cagfarm-high"
+    else
+        host="cagfarm-med"
+    fi
+fi
+if ! test -e $host ; then 
+    mkdir $host
+fi
+cd $host
+dstamp=`date | awk '{print $6 "-" $2$3}'`.
+nstamp=1
+stamp=$dataset"-"$dstamp"00"$nstamp
+while test -e $stamp ; do
+    nstamp=$((++nstamp))
+    if [ $nstamp -lt 10 ]; then
+        stamp=$dataset"-"$dstamp"00"$nstamp
+    elif [ $nstamp -lt 100 ]; then
+        stamp=$dataset"-"$dstamp"0"$nstamp
+    else
+        stamp=$dataset"-"$dstamp$nstamp
+    fi
+done
+mkdir $stamp
+cd $stamp
+
+# print useful info
+echo "Running $key suite on $hostnm == $host" | tee -a suitelog.$key
+rundir=`pwd`;
+echo "Running in $rundir" | tee -a suitelog.$key
+echo "Script dir is $scriptdir" | tee -a suitelog.$key
+
+if [ $remotedynamo == 0 ] && [ $key != "native" ]; then
+    echo "Copying dynamo sources and binaries from $DYNAMORIO_HOME" | tee -a suitelog.$key
+    if test -e $DYNAMORIO_HOME; then
+        cp -r $DYNAMORIO_HOME/core .
+        cp -r $DYNAMORIO_HOME/exports .
+        # workaround for case 7852:
+        # The text relocation policy only seems to exist in v19 and later (==FC4+)
+        # and is only enforced by default in v20 and later (==FC5+).
+        # FIXME: determine precisely whether that particular policy is in effect.
+	if [ -e /selinux/policyvers ] && (( `cat /selinux/policyvers` >= 19 )) &&
+           [ -e /usr/sbin/getenforce ] && [ `/usr/sbin/getenforce` == "Enforcing" ]; then
+            chcon -t texrel_shlib_t ./exports/*/*.so
+        fi
+    else
+        echo "*** Error: DR home $DYNAMORIO_HOME does not exist"
+        exit 127
+    fi
+    if ! test -e $DYNAMORIO_TOOLS; then
+        echo "*** Error: DR tools dir $DYNAMORIO_TOOLS does not exist"
+        exit 127
+    fi
+    export DYNAMORIO_HOME=$rundir
+fi
+if [ $key != "native" ]; then
+    echo "DYNAMORIO_HOME is $DYNAMORIO_HOME" | tee -a suitelog.$key
+fi
+echo "DYNAMORIO_TOOLS is $DYNAMORIO_TOOLS" | tee -a suitelog.$key
+
+###########################################################################
+# preparation
+
+if [ $rebuild == 1 ]; then
+    echo | tee -a suitelog.$key
+    echo "clearing for rebuild" | tee -a suitelog.$key
+    date
+    sed "s/BMARKBASE/${bmarkbase//\//\\/}/" < $scriptdir/suite-all > clear-sed1
+    sed 's/REPLACEHERE/clear/' < clear-sed1 > clear
+    $bmarkbase/benchmarks/RunSuite clear > log-clear
+    date
+    echo "done with clearing for rebuild" | tee -a suitelog.$key
+fi
+
+if [ $cleanall == 1 ]; then
+    echo | tee -a suitelog.$key
+    echo "cleaning up old run dirs" | tee -a suitelog.$key
+    date
+    pushd $bmarkbase/benchmarks/spec2000/ > /dev/null
+
+    find . -name \*pt-ref\* -type d -exec chmod -R +w {} \;
+    find . -name \*pt-train\* -type d -exec chmod -R +w {} \;
+    findrmrfcmd . -name \*pt-test\* -type d
+    popd > /dev/null
+    echo "done with cleaning up old run dirs" | tee -a suitelog.$key
+fi
+
+if [ $noclean == 0 ]; then
+    echo | tee -a suitelog.$key
+    echo "cleaning up previous $key run dirs" | tee -a suitelog.$key
+    date
+    pushd $bmarkbase/benchmarks/spec2000/ > /dev/null
+
+    find . -name $key-$hostnm-\* -type d -print
+    find . -name $key-$hostnm-\* -type d -exec chmod -R +w {} \; > /dev/null 2>&1
+    findrmrfcmd . -name $key-$hostnm-\* -type d > /dev/null 2>&1
+    popd > /dev/null
+    date
+    echo "done with cleaning up previous $key run dirs" | tee -a suitelog.$key
+else
+    echo "not cleaning previous $key run dirs, times may show old runs" | tee -a suitelog.$key
+fi
+
+###########################################################################
+# run the suite
+
+# must escape all forward slashes, for sed
+sedops=${ops//\//\\\/}
+if [ $key == "native" ]; then
+    replaceme="NATIVE=yes $sedops"
+elif [ $key == "release" ]; then
+    replaceme="DYNAMORIO_OPTIONS=\"\" $sedops"
+elif [ $key == "base" ]; then
+    replaceme="DYNAMORIO_OPTIONS=\"-loglevel 1\" $sedops"
+elif [ $key == "sidebase" ]; then
+    replaceme="DYNAMORIO_OPTIONS=\"-loglevel 1 -sideline\" $sedops"
+else
+    replaceme=$sedops
+fi
+
+template="suite-$dataset"
+if ! test -e "$scriptdir$template" ; then
+    echo "Error: cannot find template file $scriptdir$template"
+    exit 127
+fi
+sed "s/BMARKBASE/${bmarkbase//\//\\/}/" < $scriptdir$template > $key-sed1
+sed "s/REPLACEHERE/RUNDIR=$key-$hostnm $replaceme/" < $key-sed1 > $key
+rm $key-sed1
+
+echo | tee -a suitelog.$key
+echo "$key" | tee -a suitelog.$key
+date | tee -a suitelog.$key
+$bmarkbase/benchmarks/RunSuite $key | tee -a suitelog.$key
+# make better names
+mv $key.log log.$key
+mv $key cmds.$key
+grep -E 'make:|Verify|Elapsed:|^real|Error:|%%%% /|elapsed|RSS' log.$key > results.$key
+date | tee -a suitelog.$key
+echo "done with $key" | tee -a suitelog.$key
+
+###########################################################################
+# gather data
+
+# N.B.: I used to use `dirs | awk '{print $2}'` to get the run dir after a pushd,
+# but something changed b/c now there's a ~ in there and it's not being evaluated,
+# so I just use the stored $rundir
+
+# get $key times
+if [ $key == "native" ]; then
+    pushd $bmarkbase/benchmarks/spec2000/ > /dev/null
+    $DYNAMORIO_TOOLS/Summary -times -find "find . -name $key-$hostnm-\* | grep -v '\.[0-9]'" > $rundir/stimes.$key
+    popd > /dev/null
+else
+    pushd $bmarkbase/benchmarks/spec2000/ > /dev/null
+    $DYNAMORIO_TOOLS/Summary -times -find "find . -name $key-$hostnm-\* | grep -v '\.[0-9]'" > $rundir/stimes.$key
+    # gather up dynamorio files
+    logfiles=`find . -name \*.exe.\* -type f | grep "$key-$hostnm-[^0-9]*/" | wc  | awk '{print $1}'`
+    if [ $logfiles == 0 ]; then
+        popd > /dev/null
+        echo "****** Warning: no *log.* files found from run!" | tee -a suitelog.$key
+    else
+        $DYNAMORIO_TOOLS/Summary -find "find . -type f -name mainlog.\* | grep '$key-$hostnm-[^0-9]*/'" > $rundir/summary.$key
+        # support both old and new logdir names
+        find . -name dynamorio.\* -o -name \*.\[0-9\]\*.0000000\[0-9\] -type d | grep "$key-$hostnm-[^0-9]*/" | xargs tar cjf $key-logs.tar.bz2
+         if [ -e $key-logs.tar.bz2 ]; then
+             mv $key-logs.tar.bz2 $rundir
+         fi
+        popd > /dev/null
+    fi
+fi
+
+# use bmtable.pl for a nice summary, including memory info
+$DYNAMORIO_TOOLS/bmtable.pl log.$key > t-$key
+$DYNAMORIO_TOOLS/bmtable.pl -k log.$key > tk-$key
+$DYNAMORIO_TOOLS/bmtable.pl -v log.$key > tv-$key
+
+echo "Verification results:"
+failures=`find . -name results.\* | xargs grep -E 'FAIL|make:' | wc | awk '{print $1}'`
+if [ $failures == 0 ]; then
+    echo "No failures -- all benchmarks succeeded" | tee -a suitelog.$key
+else
+    echo "****** Number of benchmarks that failed: $failures ******" | tee -a suitelog.$key
+fi
+
+echo "Post-verification results:"
+# bmtable.pl considers [Ee]rror: a failure, we report it separately here
+errors=`find . -name results.\* | xargs grep -E '[Ee]rror:' | wc | awk '{print $1}'`
+if [ $errors == 0 ]; then
+    echo "No post-verify failures" | tee -a suitelog.$key
+else
+    echo "****** Number of post-verify failures: $errors ******" | tee -a suitelog.$key
+    echo "Error messages:"
+    find . -name results.\* | xargs grep -E '[Ee]rror:'
+    echo ""
+fi
+
+if [ $key != "native" ]; then
+    # save space (shrinks size by over 75%)
+    echo "Tarring up core and exports dirs to save space" | tee -a suitelog.$key
+    tar cjf core.tar.bz2 core
+    rmrfcmd core
+    tar cjf exports.tar.bz2 exports
+    rmrfcmd exports
+    # auto-compute slowdown vs. native
+    lines=`wc t-$key | awk '{print $1}'`
+    native_times=../../native/$host/t-native
+    if [ $lines == 22 ] && [ test -e "$native_times" ]; then
+        # full run of non-native: calculate slowdown vs. native
+        echo "calculating slowdown of $key vs. $native_times" | tee -a suitelog.$key
+        cd ..
+        $DYNAMORIO_TOOLS/bmcmp.pl $stamp/t-$key $native_times > vs-$key-native
+    fi
+    # FIXME: run graph.pl to build slowdown graph...but the proper perl packages
+    # aren't installed on cagfarm machines!
+fi
+
+# finish with useful info
+echo "Done running $key suite on $host" | tee -a suitelog.$key
+echo "Results are in $key/$host/$stamp" | tee -a suitelog.$key
+echo ""
+cat tv-$key
+echo ""
diff --git a/suite/suite-all b/suite/suite-all
new file mode 100644
index 000000000..385c86245
--- /dev/null
+++ b/suite/suite-all
@@ -0,0 +1,59 @@
+# **********************************************************
+# Copyright (c) 2002 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+## just a list of all (including FORTRAN 90) benchmarks
+
+BMARKBASE/benchmarks/spec2000/int/vpr	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/int/mcf	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/int/bzip2	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/int/twolf	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/int/crafty	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/int/gap	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/int/vortex	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/int/perlbmk	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/int/parser	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/int/eon	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/int/gzip	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/int/gcc	(make REPLACEHERE)
+
+BMARKBASE/benchmarks/spec2000/fp/mesa	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/fp/art	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/fp/equake	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/fp/apsi	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/fp/swim	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/fp/ammp	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/fp/sixtrack	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/fp/mgrid	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/fp/wupwise	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/fp/applu	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/fp/facerec	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/fp/fma3d	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/fp/galgel	(make REPLACEHERE)
+BMARKBASE/benchmarks/spec2000/fp/lucas	(make REPLACEHERE)
diff --git a/suite/suite-ref b/suite/suite-ref
new file mode 100644
index 000000000..f76ef85c4
--- /dev/null
+++ b/suite/suite-ref
@@ -0,0 +1,59 @@
+# **********************************************************
+# Copyright (c) 2002-2003 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+## Suite for SPEC2000 on ref dataset
+BMARKBASE/benchmarks/spec2000/int/crafty	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/gap	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/vortex	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/perlbmk	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/gcc	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/vpr	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/mcf	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/mesa	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/art	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/twolf	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/equake	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/parser	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/eon	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/bzip2	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/gzip	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/apsi	(make REF_MINS=120 COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/swim	(make REF_MINS=150 COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/ammp	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/sixtrack	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/wupwise	(make REF_MINS=300 COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/mgrid	(make REF_MINS=200 COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/applu	(make REF_MINS=200 COPYDATA=yes REPLACEHERE run_ref)
+
+# FORTRAN 90
+BMARKBASE/benchmarks/spec2000/fp/facerec	(make REF_MINS=300 COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/fma3d	(make REF_MINS=300 COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/galgel	(make REF_MINS=300 COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/lucas	(make REF_MINS=300 COPYDATA=yes REPLACEHERE run_ref)
diff --git a/suite/suite-reffp b/suite/suite-reffp
new file mode 100644
index 000000000..cd92ba6eb
--- /dev/null
+++ b/suite/suite-reffp
@@ -0,0 +1,50 @@
+# **********************************************************
+# Copyright (c) 2002 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+## Suite for SPEC2000 on ref dataset -- just fp benchmarks!
+BMARKBASE/benchmarks/spec2000/fp/mesa	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/art	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/equake	(make COPYDATA=yes REPLACEHERE run_ref)
+#-SPLIT
+BMARKBASE/benchmarks/spec2000/fp/apsi	(make REF_MINS=120 COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/swim	(make REF_MINS=150 COPYDATA=yes REPLACEHERE run_ref)
+#-SPLIT
+BMARKBASE/benchmarks/spec2000/fp/ammp	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/sixtrack	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/mgrid	(make REF_MINS=200 COPYDATA=yes REPLACEHERE run_ref)
+#-SPLIT
+BMARKBASE/benchmarks/spec2000/fp/wupwise	(make REF_MINS=300 COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/applu	(make REF_MINS=200 COPYDATA=yes REPLACEHERE run_ref)
+
+# FORTRAN 90
+BMARKBASE/benchmarks/spec2000/fp/facerec	(make REF_MINS=300 COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/fma3d	(make REF_MINS=300 COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/galgel	(make REF_MINS=300 COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/fp/lucas	(make REF_MINS=300 COPYDATA=yes REPLACEHERE run_ref)
diff --git a/suite/suite-refint b/suite/suite-refint
new file mode 100644
index 000000000..66a933f07
--- /dev/null
+++ b/suite/suite-refint
@@ -0,0 +1,43 @@
+# **********************************************************
+# Copyright (c) 2002-2003 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+## Suite for SPEC2000 on ref dataset -- just integer benchmarks!
+BMARKBASE/benchmarks/spec2000/int/crafty	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/gap	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/vortex	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/perlbmk	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/gcc	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/vpr	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/mcf	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/bzip2	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/twolf	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/parser	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/eon	(make COPYDATA=yes REPLACEHERE run_ref)
+BMARKBASE/benchmarks/spec2000/int/gzip	(make COPYDATA=yes REPLACEHERE run_ref)
diff --git a/suite/suite-regression b/suite/suite-regression
new file mode 100644
index 000000000..460ff81bb
--- /dev/null
+++ b/suite/suite-regression
@@ -0,0 +1,38 @@
+# **********************************************************
+# Copyright (c) 2003-2007 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# regression suite
+BMARKBASE/benchmarks/spec2000/int/mcf	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/crafty	(make COPYDATA=yes REPLACEHERE run)
+# PR 225156: eon doesn't build w/ our standard toolchain: no iostream.h
+#BMARKBASE/benchmarks/spec2000/int/eon	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/gcc	(make COPYDATA=yes REPLACEHERE run)
+#removed jbrink cuz perlbmk.exe crashes and causes whole suite to stop
+#BMARKBASE/benchmarks/spec2000/int/perlbmk	(make COPYDATA=yes REPLACEHERE run)
diff --git a/suite/suite-test b/suite/suite-test
new file mode 100644
index 000000000..7a6752034
--- /dev/null
+++ b/suite/suite-test
@@ -0,0 +1,62 @@
+# **********************************************************
+# Copyright (c) 2002-2003 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+## Suite for SPEC2000 on test dataset
+BMARKBASE/benchmarks/spec2000/int/vpr	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/mcf	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/fp/mesa	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/bzip2	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/fp/art	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/twolf	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/fp/equake	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/crafty	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/gap	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/vortex	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/perlbmk	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/parser	(make COPYDATA=yes REPLACEHERE run)
+#-SPLIT
+BMARKBASE/benchmarks/spec2000/int/eon	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/fp/apsi	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/fp/swim	(make COPYDATA=yes REPLACEHERE run)
+#-SPLIT
+BMARKBASE/benchmarks/spec2000/int/gzip	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/fp/ammp	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/fp/sixtrack	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/fp/mgrid	(make TEST_MINS=10 COPYDATA=yes REPLACEHERE run)
+#-SPLIT
+BMARKBASE/benchmarks/spec2000/int/gcc	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/fp/wupwise	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/fp/applu	(make COPYDATA=yes REPLACEHERE run)
+
+# FORTRAN 90
+BMARKBASE/benchmarks/spec2000/fp/facerec	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/fp/fma3d	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/fp/galgel	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/fp/lucas	(make COPYDATA=yes REPLACEHERE run)
diff --git a/suite/suite-testint b/suite/suite-testint
new file mode 100644
index 000000000..35850ffe4
--- /dev/null
+++ b/suite/suite-testint
@@ -0,0 +1,43 @@
+# **********************************************************
+# Copyright (c) 2002-2003 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+## Suite for SPEC2000 on test dataset: integer bmarks only
+BMARKBASE/benchmarks/spec2000/int/vpr	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/mcf	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/bzip2	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/twolf	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/crafty	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/gap	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/vortex	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/perlbmk	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/parser	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/eon	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/gzip	(make COPYDATA=yes REPLACEHERE run)
+BMARKBASE/benchmarks/spec2000/int/gcc	(make COPYDATA=yes REPLACEHERE run)
diff --git a/suite/tests/Makefile b/suite/tests/Makefile
new file mode 100644
index 000000000..c136c6714
--- /dev/null
+++ b/suite/tests/Makefile
@@ -0,0 +1,1128 @@
+# **********************************************************
+# Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+###########################################################################
+##
+## USAGE:
+##
+## Just say "make" and it will run all tests and diff results with
+## expected results.  
+##
+## The output of the programs are placed in a directory named "results"
+## by default, it can be set like this "make RESDIR='myresdir`".
+##
+## To change DynamoRIO options: say "make OPS='whatever'"
+##
+
+.PHONY: all run clean runbase runsecurity runrunall runhotp runnogui \
+        runrunall-nogui runapi runinterface runcommon runos runbasesecurity \
+	runossecurity runnointerface runheader reportbuild
+
+MAKEFLAGS := -s
+
+ifndef DYNAMORIO_MAKE
+  DYNAMORIO_MAKE := ../../make
+endif
+include $(DYNAMORIO_MAKE)/compiler.mk
+
+ifndef ARCH
+# default is x86 instead of x64
+ARCH = x86
+endif
+
+# Set default values for the DYNAMORIO_ env vars
+ifeq ($(MACHINE), win32)
+ifndef DYNAMORIO_HOME
+  DYNAMORIO_HOME := $(shell $(CYGPATH) -ma $(CURDIR)/../..)
+endif
+ifndef DYNAMORIO_TOOLS
+  # export for runall/runalltest.sh
+  export DYNAMORIO_TOOLS := $(shell $(CYGPATH) -ma $(CURDIR)/../../tools)
+endif
+ifndef DYNAMORIO_LOGDIR
+  export DYNAMORIO_LOGDIR := $(shell $(CYGPATH) -ma $(CURDIR)/logs)
+endif
+ifndef DYNAMORIO_CACHE_ROOT
+  export DYNAMORIO_CACHE_ROOT := $(shell $(CYGPATH) -ma $(CURDIR)/cache)
+endif
+else
+ifndef DYNAMORIO_HOME
+  DYNAMORIO_HOME := $(CURDIR)/../..
+endif
+ifndef DYNAMORIO_TOOLS
+  # export for runall/runalltest.sh
+  DYNAMORIO_TOOLS := $(CURDIR)/../../tools
+endif
+ifndef DYNAMORIO_LOGDIR
+  export DYNAMORIO_LOGDIR := $(CURDIR)/logs
+endif
+ifndef DYNAMORIO_CACHE_ROOT
+  export DYNAMORIO_CACHE_ROOT := $(CURDIR)/cache
+endif
+endif
+
+ifeq ($(MACHINE), linux)
+# EXTRA TESTS
+# Until we need file-based .runall-type functionality, for now I'm
+# adding a few extra runs of other test programs solely in the Makefile:
+EXTRA_OS_TESTS := linux/clone-reset linux/thread-reset
+endif
+
+EXTRA_COMMON_TESTS += common/decode-stress
+
+EXTRA_TESTS := $(EXTRA_COMMON_TESTS) $(EXTRA_OS_TESTS)
+EXTRA_TESTS_EXE := $(patsubst %,%.$(EXE_SFX),$(EXTRA_TESTS))
+
+# **** DISABLED TESTS ****
+DISABLED_LINUX := | $(GREP) -vE '/vfork'
+# PR 215238 covers re-enabling threadinjection
+DISABLED_WIN32 := | $(GREP) -vE '/debugger|/threadinjection'
+# case 4757 covers re-enabling selfmod-threads
+DISABLED_SECURITY_WIN32 := | $(GREP) -vE 'selfmod-threads'
+# case 9423 covers re-enabling
+DISABLED_RUNALL := | $(GREP) -vE 'earlythread'
+# pthreads tests are also _completely_ disabled down below
+DISABLED_HOTP := | $(GREP) -vE 'gui'
+# FIXME: PR 199115 to re-enable fragdel, get some more of the LINUX tests working
+ifeq ($(MACHINE), win32)
+DISABLED_INTERFACE := | $(GREP) -vE '/fragdel|/syscall-mod|/signal'
+else
+DISABLED_INTERFACE := | $(GREP) -vE '/fragdel|/modules|/pc-check|/syscall\.|/cbr3|/cbr4|/nudge_test'
+endif
+# PR 217255: removed to shorten the regression suite
+DISABLED_COMMON := | $(GREP) -vE '/hello|/conflict|/ops|/recurse'
+
+ifeq ($(MACHINE), linux)
+  DISABLED_OS := $(DISABLED_LINUX)
+  DISABLED_SECURITY_OS := 
+else
+  DISABLED_OS := $(DISABLED_WIN32)
+  DISABLED_SECURITY_OS := $(DISABLED_SECURITY_WIN32)
+endif
+
+DISABLED_ALL := $(DISABLED_OS) $(DISABLED_SECURITY_OS) $(DISABLED_RUNALL) \
+		$(DISABLED_HOTP) $(DISABLED_INTERFACE) $(DISABLED_COMMON)
+
+# Some things that are nice to be able to turn off from the cmd line
+WARN_DEFS := yes
+
+# default directory to put results in
+RESDIR := results
+
+# windows vs. linux flags, etc.
+ifeq ($(MACHINE), linux)
+WARN    =-Wall -Werror
+OPT     =-O3
+NOPT    =-O0
+DBG     =-g3
+OBJ_SFX =o
+EXE_SFX =
+LIB_SFX =.so
+LIB_NAME =libdynamorio.so
+else # win32
+# avoid problems w/ line endings on #! files that cygwin's sh has
+DLLOUT  =/dll /out:
+LIBOUT  =/implib:
+WARN    =/W3 /WX
+OPT     =/O2
+NOPT    =/Od
+DBG     =/Zi
+LINK_DBG=/debug
+OBJ_SFX =obj
+EXE_SFX =.exe
+LIB_SFX =.dll
+LIB_NAME =dynamorio.dll
+endif
+
+# use release build if it exists, unless DEBUG=yes
+ifeq ($(MACHINE), linux)
+DYNAMORIO_DIR_BASE = $(DYNAMORIO_HOME)/exports/$(ARCH)_linux_
+DYNAMORIO_BUILD_DIR_BASE = $(DYNAMORIO_HOME)/build/$(ARCH)_linux_
+else
+DYNAMORIO_DIR_BASE = $(DYNAMORIO_HOME)/exports/$(ARCH)_win32_
+DYNAMORIO_BUILD_DIR_BASE = $(DYNAMORIO_HOME)/build/$(ARCH)_win32_
+endif
+
+# allow yes or 1 for DEBUG or INTERNAL
+USE_DEBUG=$(DEBUG)
+ifeq ($(DEBUG),yes)
+USE_DEBUG=1
+endif
+USE_INTERNAL=$(INTERNAL)
+ifeq ($(INTERNAL),yes)
+USE_INTERNAL=1
+endif
+USE_PROFILE=$(PROFILE)
+ifeq ($(PROFILE),yes)
+USE_PROFILE=1
+endif
+
+# We check for a build based on the following order
+# Relase, Internal, Profile, Debug
+# FIXME - is that the order we want?
+PROFILE_EXISTS=$(wildcard $(DYNAMORIO_DIR_BASE)prof)
+INTERNAL_EXISTS=$(wildcard $(DYNAMORIO_DIR_BASE)intr)
+RELEASE_EXISTS=$(wildcard $(DYNAMORIO_DIR_BASE)rel)
+
+ifeq ($(USE_DEBUG),1)
+RELEASE_EXISTS=""
+INTERNAL_EXISTS=""
+PROFILE_EXISTS=""
+else
+ifeq ($(USE_PROFILE),1)
+RELEASE_EXISTS=""
+INTERNAL_EXIST=""
+else
+ifeq ($(USE_INTERNAL),1)
+RELEASE_EXISTS=""
+endif # intr
+endif # profile
+endif # debug
+
+ifeq ($(RELEASE_EXISTS), $(DYNAMORIO_DIR_BASE)rel)
+BUILD_SUFFIX=rel
+else
+ifeq ($(INTERNAL_EXISTS), $(DYNAMORIO_DIR_BASE)intr)
+BUILD_SUFFIX=intr
+else
+ifeq ($(PROFILE_EXISTS), $(DYNAMORIO_DIR_BASE)prof)
+BUILD_SUFFIX=prof
+else
+BUILD_SUFFIX=dbg
+endif # profile
+endif # intr
+endif # rel
+
+DYNAMORIO_DIR = $(DYNAMORIO_DIR_BASE)$(BUILD_SUFFIX)
+DYNAMORIO_BUILD_DIR = $(DYNAMORIO_BUILD_DIR_BASE)$(BUILD_SUFFIX)
+
+CORE_DIRNAME=core
+
+ifeq ($(wildcard $(DYNAMORIO_BUILD_DIR)/defines.h), $(DYNAMORIO_BUILD_DIR)/defines.h)
+DEFS := $(subst ",,$(subst \#define DYNAMORIO_DEFINES ,,$(shell $(GREP) "DX86" $(DYNAMORIO_BUILD_DIR)/defines.h | $(TAIL) -n 1)))
+else 
+ifeq ($(NATIVE), 1)
+DEFS := NATIVE
+else
+DEFS := $(shell $(STRINGS) $(DYNAMORIO_DIR)/$(LIB_NAME) | $(GREP) "DX86" | $(TAIL) -n 1)
+ifeq ( , $(DEFS))
+DEFS_T := $(D)INLINE_IND_BR_LOOKUP $(D)TRACE_HEAD_CACHE_INCR $(D)X86 $(D)RELEASE
+ifeq (detect_mode, $(shell $(STRINGS) $(DYNAMORIO_DIR)/$(LIB_NAME) | $(GREP) "detect_mode" | $(TAIL) -n 1))
+DEFS_T += $(D)PROGRAM_SHEPHERDING
+endif
+ifeq ($(WARN_DEFS), yes)
+DEFS += $(warning Can't determine defines for release build, guessing $(DEFS_T))
+endif
+DEFS += $(DEFS_T)
+endif
+endif # NATIVE
+endif # defines.h
+
+# runall max wait value. It's okay to use a large number here; closewnd
+# closes the app as soon as it is found. We only reach the maximum
+# when something goes wrong.
+RUNALL_MAX_WAIT=10
+
+# Max wait value for client interface tests using the runall script. 
+# Make this larger than RUNALL_MAX_WAIT since client libraries tend
+# to slow things down.
+INTERFACE_MAX_WAIT=30
+
+# turn on no debugbox, stderr mask, and staged for tests
+# NOTE - should be kept in synch with the MESSAGES_TO_STDERR_* values in runregression
+ifeq ($(MACHINE), linux)
+OPS = -stderr_mask 0xC -dumpcore_mask 0 $(DYNAMORIO_OPTIONS)
+else
+OPS = -stderr_mask 0xC -msgbox_mask 0 -dumpcore_mask 0xfd -staged $(DYNAMORIO_OPTIONS)
+endif
+
+# use separate value than that passed in on cmdline so tests can modify it
+DR_OPS = $(OPS)
+
+ifeq ($(MACHINE), linux)
+DYNAMORIO_INC := $(I)$(DYNAMORIO_DIR) 
+else
+DYNAMORIO_INC := $(I)$(subst /,\\,$(shell $(CYGPATH) -m ${DYNAMORIO_DIR}))
+DYNAMORIO_LIBDIR := $(subst /,\\,$(shell $(CYGPATH) -m ${DYNAMORIO_DIR}))
+endif
+
+CC_BASE = $(CC) $(DBG) $I. $(MSINCLUDE) tools.c
+
+# TODO: use common flags between the liveshield creation system and this
+HOTP_DLL_INC_DIR = $(shell $(CYGPATH) -m $(DYNAMORIO_HOME))/$(CORE_DIRNAME)/lib
+
+# DBG and LINK_DBG flags are set to empty by runregression so that regressions
+# over ssh can work (PR 192750).  However, this breaks hotp tests (PR 234043)
+# as they need debug info to compute .cfg files.  As ssh builds are used by some
+# for short regressions, the solution was to move hotp tests to long regressions
+# and hard code the debug flags for hotp tests.  Thus short regressions will
+# work over ssh (hotp tests will fail if run in short regressions) and hotp
+# tests will pass in long regressions (which aren't run over ssh).
+HOTP_DLL_CFLAGS = $(INCR) $(OPT) /Zi /Zl /X
+HOTP_DLL_LDFLAGS = /NOENTRY /INCREMENTAL:NO /debug /OPT:ICF /OPT:NOWIN98 /RELEASE /MAP
+
+# our default is to not optimize, because we have some programs whose purpose
+# is destroyed by optimizing (in particular by inlining, for example common/getretaddr.c)
+# (it's hard to add hacks to prevent inlining)
+ifeq ($(OPTIMIZE),yes)
+CC_BASE += $(OPT)
+else
+CC_BASE += $(NOPT)
+endif
+
+ifeq ($(ARCH), x64)
+DEFINES += $(D)X86_64 $(D)X64
+else
+DEFINES += $(D)X86_32
+endif
+
+# some apps need LINUX defined even if not using API
+ifeq ($(MACHINE), win32)
+  CC_BASE += /MTd
+  DEFINES += $(D)WINDOWS
+else
+  DEFINES += $(D)LINUX
+  ifeq ($(ARCH), x64)
+    TGT_CFLAGS += -m64
+  else
+    TGT_CFLAGS += -m32
+  endif
+endif
+
+NIGHTLY = 1 # short enough to be tolerable for short runs
+PERF = 0 # longer iterations and should measure performance
+STRESS = 0 # even longer iterations (or infinite unless bound by TIMER_TIME)
+
+ifeq ($(STRESS), 1)
+DEFINES += $(D)STRESS
+NIGHTLY = 0
+endif
+
+ifeq ($(PERF), 1)
+DEFINES += $(D)PERF
+NIGHTLY = 0
+endif
+
+ifneq ($(NIGHTLY), 0)
+DEFINES += $(D)NIGHTLY_REGRESSION
+endif
+
+CC_BASE += $(TGT_CFLAGS) $(DEFINES)
+
+CC_BASE_DYNAMORIO = $(CC_BASE) $(DYNAMORIO_INC) $(D)DYNAMO $(D)X86
+
+# We need a different command line for building client library dlls.
+# tools.c uses some external libraries, which clients should avoid.
+ifeq ($(MACHINE), linux)
+CC_BASE_INTERFACE = $(CC) $(TGT_CFLAGS) $(DBG) $(OPT) $(WARN) $(DEFINES)
+else
+CC_BASE_INTERFACE = $(CC) $(TGT_CFLAGS) $(DBG) /MTd $(OPT) $(WARN) $(MSINCLUDE) $(DEFINES)
+# several client-interface tests have started hitting security_cookie issues
+CC_BASE_INTERFACE += /GS-
+endif
+
+ifeq ($(MACHINE), linux)
+C++_BASE = $(CC_BASE:gcc=g++)
+else # win32
+C++_BASE = $(CC_BASE) /TP
+endif
+
+ifeq ($(MACHINE), linux)
+FCOMP = g77 -g
+else
+FCOMP = $(error "Fortran not yet supported on windows")
+endif
+
+ifeq ($(MACHINE), linux)
+LINK = -lm -ldl
+LINK_DYNAMORIO = $(L)$(DYNAMORIO_DIR) -ldynamorio -lm -Xlinker -rpath -Xlinker $(DYNAMORIO_DIR)
+else # win32
+  # /MTd must be included when compiling, not on link line!
+  # else get weird link errors
+  WINLIBS =  kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib \
+             advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib \
+             odbc32.lib odbccp32.lib
+  LINK = /link $(MSLIB) $(WINLIBS) /subsystem:console
+  ifeq ($(ARCH), x64)
+    LINK += /machine:X64
+  else
+    LINK += /machine:I386
+  endif
+  LINK_DYNAMORIO = $(L)$(DYNAMORIO_LIBDIR) dynamorio.lib $(MSLIB) $(WINLIBS) /subsystem:console
+endif
+# no harm in putting everywhere
+LINK += $(LINK_ASM)
+LINK += tools_asm.$(OBJ_SFX)
+
+###########################################################################
+# PR 262902: 64-bit tests
+# Since we can't have inline asm for cl, and we don't have 64-bit gcc on cygwin,
+# we put asm routines outside of the main c files in local _asm.asm files.
+# We share the core's platform-neutral format with preprocessing to
+# enable the assembly to work on linux and windows.
+
+# asm flags
+ASM_CORE_INC := $(DYNAMORIO_HOME)/$(CORE_DIRNAME)/x86
+ifeq ($(MACHINE), linux)
+  ASMPREPROC := $(I)$(ASM_CORE_INC)
+  DEFINES += $(D)ASSEMBLE_WITH_GAS
+  ASM_DEFINES := $(patsubst $(D)%, --defsym %=1, $(DEFINES))
+  ASM_DEFINES := $(patsubst $(U)%, --defsym %=0, $(ASM_DEFINES))
+  ifeq ($(ARCH), x64)
+    ASMFLAGS += --64
+  else
+    ASMFLAGS += --32
+  endif
+else
+  ASMPREPROC := $(I)$(subst /,\\,$(shell $(CYGPATH) -m ${ASM_CORE_INC}))
+  DEFINES += $(D)ASSEMBLE_WITH_MASM
+  ASM_DEFINES := $(DEFINES)
+  ASMFLAGS += $(INCR)
+  ifeq ($(DEBUG),1)
+    ASMFLAGS += $(DBG)
+  endif
+  # ml can't handle the line number markers
+  ASMPREPROC += $(CPP_NO_LINENUM)
+endif
+ASMFLAGS += $(ASM_DEFINES)
+
+# Some files use asm code, which must be separate for x64.
+# Rather than have separate rules, we use an expansion on the link line
+# that evaluates to empty if there is no asm code.
+# However, wildcard and shell expansions do not work on the prereq
+# line; they're supposed to if using .SECONDEXPANSION, but I could not
+# get that to work with a rule of the format "many-files: %.exe: %.c",
+# so I'm adding the prereqs a simpler way (the hack way is to have an
+# "if test -e then make" command, but that's less efficient).
+ASM_SRCS := $(shell grep -l asm_defines.asm */*.c) # FIXME: use SRCS_ALL
+ASM_EXES := $(subst .c,$(EXE_SFX),$(ASM_SRCS))
+$(ASM_EXES): %$(EXE_SFX): %_asm.$(OBJ_SFX)
+
+# This must be = and not := since we need $@ expanded at the use site
+# Note the space after $@, to avoid substring matching
+ifeq ($(MACHINE), linux)
+  LINK_ASM = $(addsuffix _asm.$(OBJ_SFX), $(findstring $@ ,$(ASM_EXES)))
+else
+  LINK_ASM = $(subst $(EXE_SFX),_asm.$(OBJ_SFX),$(findstring $@ ,$(ASM_EXES)))
+endif
+
+# Like the core, we use platform-independent assembly code, but we
+# embed it inside our .c files and pull it out with the preprocessor
+# (since we'd be pre-processing an .asm file anyway).
+
+# asm code inside .c file -> .s
+# we want ".ifdef FOO" to not turn into ".ifdef 1" so we say "-DFOO=FOO":
+ASM_CPP_DEFINES := $(join $(DEFINES),$(patsubst $(D)%,=%,$(DEFINES)))
+%_asm.s: %.c
+	$(CPP) $(D)ASM_CODE_ONLY $(PREPROC) $(ASMPREPROC) $(CPPFLAGS) $(ASM_CPP_DEFINES) $< $(CPPOUT)$@
+# since we don't want to duplicate gas and ml macros we use cpp macros w/
+# the added feature of insertable newlines, needed for asm code, via "@N@":
+	$(SED) -i 's/\@N\@/\n/g' $@
+ifeq ($(MACHINE), linux)
+# must remove _ from global symbols, rest is done via macros in .asm file
+	$(SED) -i 's/\([^a-zA-Z0-9_]\)_\([a-zA-Z][a-zA-Z0-9_]*[a-zA-Z0-9]\)/\1\2/' $@
+endif
+
+# .s -> .o
+%.$(OBJ_SFX): %.s
+# ml and ml64 ignore /Fo unless it is BEFORE the source file
+	$(AS) $(ASMFLAGS) $(OUT)$@ $<
+
+###########################################################################
+
+SRCS_COMMON := $(shell $(LS) common/*.c ${DISABLED_COMMON})
+
+SRCS_API := $(shell $(LS) api/*.c)
+# This is for probe-api tests and test clients; they have a 1-to-1 mapping.
+SRCS_PROBE_API := $(shell $(LS) probe-api/*.c | $(GREP) -E -v '\.(client|dll.*|probe.*)\.c$$')
+# These may not exist for a probe-api test.
+SRCS_DLL_PROBE_API := $(wildcard probe-api/*.dll.c)       # probe-api test dlls
+SRCS_PROBES_PROBE_API := $(wildcard probe-api/*.probe*.c) # probe-api probe dlls
+
+# skip the security test if we are !PROGRAM_SHEPHERDING
+ifeq (,$(findstring PROGRAM_SHEPHERDING, $(DEFS)))
+EXTRA_INTERFACE_DISABLE := | $(GREP) -vE '/security'
+endif
+SRCS_DLL_INTERFACE := $(shell $(LS) client-interface/*.dll.c ${DISABLED_INTERFACE} ${EXTRA_INTERFACE_DISABLE})
+SRCS_EXE_INTERFACE := $(shell $(LS) client-interface/*.c | $(GREP) -v '\.dll\.c$$' ${DISABLED_INTERFACE} ${EXTRA_INTERFACE_DISABLE})
+
+SRCS_SECURITY_COMMON := $(shell $(LS) security-common/*.c)
+
+ifeq ($(MACHINE), linux)
+# FIXME: re-enable signal tests when have time to fix core signal bugs
+# FIXME: re-enable vfork* tests when have time to fix core bugs --
+# they hang, causing ssh regression process to not return
+SRCS_OS := $(shell $(LS) linux/*.c ${DISABLED_OS})
+SRCS_PTHREADS := $(shell $(LS) pthreads/*.c)
+SRCS_SECURITY_OS := $(shell $(LS) security-linux/*.c)
+else
+# .dll.c signifies a dependent dll source for the main .c test
+SRCS_OS := $(shell $(LS) win32/*.c | $(GREP) -v '\.dll\.c$$' ${DISABLED_OS})
+SRCS_OS_NODLL := $(foreach file,$(SRCS_OS),$(if $(wildcard $(subst .c,.dll.c,$(file))),,$(file)))
+SRCS_OS_DLL := $(foreach file,$(SRCS_OS),$(if $(wildcard $(subst .c,.dll.c,$(file))),$(file),))
+
+SRCS_PTHREADS := 
+
+# .dll.c signifies a dependent dll source for the main .c test
+SRCS_SECURITY_OS := $(shell $(LS) security-win32/*.c | $(GREP) -v '\.dll\.c$$' ${DISABLED_SECURITY_OS})
+SRCS_SECURITY_OS_NODLL := $(foreach file,$(SRCS_SECURITY_OS),$(if $(wildcard $(subst .c,.dll.c,$(file))),,$(file)))
+SRCS_SECURITY_OS_DLL := $(foreach file,$(SRCS_SECURITY_OS),$(if $(wildcard $(subst .c,.dll.c,$(file))),$(file),))
+
+# .dll.c signifies a dependent dll source for the main .c test
+SRCS_RUNALL := $(shell $(LS) runall/*.c | $(GREP) -v '\.dll\.c$$' ${DISABLED_RUNALL})
+SRCS_RUNALL_NODLL := $(foreach file,$(SRCS_RUNALL),$(if $(wildcard $(subst .c,.dll.c,$(file))),,$(file)))
+SRCS_RUNALL_DLL := $(foreach file,$(SRCS_RUNALL),$(if $(wildcard $(subst .c,.dll.c,$(file))),$(file),))
+endif
+
+# get name of hotp tests, exclude dlls used by tests and hotp dlls; they all
+# have the same name but different suffixes.
+# TODO: currently only one dll supported for the test and for the patch dll.
+SRCS_HOTP := $(shell $(LS) hotp/*/*.c | $(GREP) -v '.hotp\.c$$' | $(GREP) -v '.dll\.c$$' ${DISABLED_HOTP})
+
+# we have to have roots separate for win32 .exe suffix
+PGMS_ROOT_COMMON := $(patsubst %.c,%,$(SRCS_COMMON))
+PGMS_ROOT_COMMON += $(EXTRA_COMMON_TESTS)
+PGMS_ROOT_API := $(patsubst %.c,%,$(SRCS_API))
+PGMS_ROOT_PROBE_API := $(patsubst %.c,%,$(SRCS_PROBE_API))
+PGMS_ROOT_INTERFACE := $(patsubst %.dll.c,%,$(SRCS_DLL_INTERFACE))
+PGMS_ROOT_OS := $(patsubst %.c,%,$(SRCS_OS))
+PGMS_ROOT_OS += $(EXTRA_OS_TESTS)
+PGMS_ROOT_PTHREADS := $(patsubst %.c,%,$(SRCS_PTHREADS))
+PGMS_ROOT_SECURITY_COMMON := $(patsubst %.c,%,$(SRCS_SECURITY_COMMON))
+PGMS_ROOT_SECURITY_OS := $(patsubst %.c,%,$(SRCS_SECURITY_OS))
+PGMS_ROOT_HOTP := $(patsubst %.c,%,$(SRCS_HOTP))
+PGMS_ROOT := $(PGMS_ROOT_COMMON) $(PGMS_ROOT_API) $(PGMS_ROOT_PROBE_API) $(PGMS_ROOT_INTERFACE) $(PGMS_ROOT_OS) $(PGMS_ROOT_PTHREADS) $(PGMS_ROOT_SECURITY_COMMON) $(PGMS_ROOT_SECURITY_COMMON) $(PGMS_ROOT_SECURITY_OS) $(PGMS_ROOT_HOTP)
+
+PGMS_EXE_COMMON := $(patsubst %.c,%$(EXE_SFX),$(SRCS_COMMON))
+PGMS_EXE_API := $(patsubst %.c,%$(EXE_SFX),$(SRCS_API))
+PGMS_EXE_PROBE_API := $(patsubst %.c,%$(EXE_SFX),$(SRCS_PROBE_API))
+PGMS_CLIENT_PROBE_API := $(patsubst %.c,%.client.dll,$(SRCS_PROBE_API))
+PGMS_DLL_PROBE_API := $(patsubst %.c,%.dll,$(SRCS_DLL_PROBE_API))
+PGMS_PROBES_PROBE_API := $(patsubst %.c,%.dll,$(SRCS_PROBES_PROBE_API))
+PGMS_EXE_INTERFACE := $(patsubst %.c,%$(EXE_SFX),$(SRCS_EXE_INTERFACE))
+PGMS_DLL_INTERFACE := $(patsubst %.dll.c,%.dll$(LIB_SFX),$(SRCS_DLL_INTERFACE))
+ifeq ($(MACHINE), linux)
+PGMS_EXE_OS := $(patsubst %.c,%$(EXE_SFX),$(SRCS_OS))
+PGMS_EXE_SECURITY_OS := $(patsubst %.c,%$(EXE_SFX),$(SRCS_SECURITY_OS))
+else
+PGMS_EXE_OS_NODLL := $(patsubst %.c,%$(EXE_SFX),$(SRCS_OS_NODLL))
+PGMS_EXE_OS_DLL := $(patsubst %.c,%$(EXE_SFX),$(SRCS_OS_DLL))
+PGMS_EXE_SECURITY_OS_NODLL := $(patsubst %.c,%$(EXE_SFX),$(SRCS_SECURITY_OS_NODLL))
+PGMS_EXE_SECURITY_OS_DLL := $(patsubst %.c,%$(EXE_SFX),$(SRCS_SECURITY_OS_DLL))
+endif
+PGMS_EXE_PTHREADS := $(patsubst %.c,%$(EXE_SFX),$(SRCS_PTHREADS))
+PGMS_EXE_SECURITY_COMMON := $(patsubst %.c,%$(EXE_SFX),$(SRCS_SECURITY_COMMON))
+PGMS_EXE_RUNALL_NODLL := $(patsubst %.c,%$(EXE_SFX),$(SRCS_RUNALL_NODLL))
+PGMS_EXE_RUNALL_DLL := $(patsubst %.c,%$(EXE_SFX),$(SRCS_RUNALL_DLL))
+PGMS_EXE_HOTP := $(patsubst %.c,%$(EXE_SFX),$(SRCS_HOTP))
+PGMS_DLL_HOTP := $(patsubst %.c,%.dll.dll,$(SRCS_HOTP))
+PGMS_DLL_HOTP_PATCH := $(patsubst %.c,%.hotp.dll,$(SRCS_HOTP))
+PGMS_EXE := $(PGMS_EXE_COMMON) $(PGMS_EXE_API) $(PGMS_EXE_PROBE_API) $(PGMS_CLIENT_PROBE_API) $(PGMS_DLL_PROBE_API) $(PGMS_PROBES_PROBE_API) $(PGMS_EXE_INTERFACE) $(PGMS_DLL_INTERFACE) $(PGMS_EXE_OS_NODLL) $(PGMS_EXE_OS_DLL) $(PGMS_EXE_OS) $(PGMS_EXE_PTHREADS) $(PGMS_EXE_SECURITY_COMMON) $(PGMS_EXE_SECURITY_OS_NODLL) $(PGMS_EXE_SECURITY_OS_DLL) $(PGMS_EXE_RUNALL_NODLL) $(PGMS_EXE_RUNALL_DLL) $(PGMS_EXE_HOTP) $(PGMS_DLL_HOTP) $(PGMS_DLL_HOTP_PATCH)
+
+TEMPLATE_COMMON := $(shell $(LS) common/*.template security-common/*.template)
+PGMS_EXP_COMMON := $(patsubst %.template, %.expect, $(TEMPLATE_COMMON))
+
+ifeq  ($(MACHINE), linux)
+TEMPLATE_OS := $(shell $(LS) security-linux/*.template)
+else
+TEMPLATE_OS := $(shell $(LS) security-win32/*.template win32/*.template runall/*.template)
+endif
+PGMS_EXP_OS := $(patsubst %.template, %.expect, $(TEMPLATE_OS))
+
+TEMPLATE_INTERFACE := $(shell $(LS) client-interface/*.template)
+PGMS_EXP_INTERFACE := $(patsubst %.template, %.expect, $(TEMPLATE_INTERFACE))
+
+TEMPLATE_API := $(shell $(LS) api/*.template)
+PGMS_EXP_API := $(patsubst %.template, %.expect, $(TEMPLATE_API))
+
+TEMPLATE_PROBE_API := $(shell $(LS) probe-api/*.template)
+PGMS_EXP_PROBE_API := $(patsubst %.template, %.expect, $(TEMPLATE_PROBE_API))
+
+TEMPLATE_HOTP := $(shell $(LS) hotp/*/*.template)
+PGMS_EXP_HOTP := $(patsubst %.template, %.expect, $(TEMPLATE_HOTP))
+
+#for testing run all, done through a shell script
+RUNALL_SCRIPTS := $(shell $(LS) runall/*.runall ${DISABLED_RUNALL})
+RUNALL_ROOTS := $(patsubst %.runall,%,$(RUNALL_SCRIPTS))
+# runall tests that do NOT open a window
+RUNALL_NOGUI_SCRIPTS := $(shell $(GREP) -l '<nowindow>' runall/*.runall ${DISABLED_RUNALL})
+RUNALL_NOGUI_ROOTS := $(patsubst %.runall,%,$(RUNALL_NOGUI_SCRIPTS))
+
+# need absolute dir ref.
+CURWD := $(CURDIR)
+ifeq ($(MACHINE), win32)
+  CURWD := $(shell $(CYGPATH) -ma $(CURWD))
+endif
+
+ifeq ($(MACHINE), win32) 
+  DYNAMORIO_WINPREINJECT := $(CURWD)/drpreinject.dll
+  DYNAMORIO_PREINJECT := $(shell $(CYGPATH) -ua $(DYNAMORIO_WINPREINJECT))
+endif
+
+ifeq ($(MACHINE), linux)
+all: run
+else
+all: $(DYNAMORIO_PREINJECT) run
+endif
+
+
+# this is much like runregression
+runlog: 
+	$(MAKE) --no-print-directory run | $(TEE) testlog
+	$(MAKE) --no-print-directory proclog
+
+proclog:
+	@$(ECHO) `$(GREP) -c PASS testlog` passed `$(GREP) -c FAIL testlog` failed
+	@$(GREP) -c FAIL testlog > /dev/null && $(ECHO) Failed || $(ECHO) Success
+
+savelog:
+	$(MV) testlog testlog.`$(DATE) +"%Y-%b%d-%H:%M"`
+
+
+$(PGMS_EXP_COMMON): %.expect: %.template
+	$(PERL) gen_expect.pl '$(DEFS)' '$(DR_OPS)' $*.template
+
+TOOLS := tools.h tools.c tools_asm.$(OBJ_SFX)
+
+##################################################
+ifeq ($(MACHINE), win32)
+TYPE_EXE := VFT_APP
+TYPE_DLL := VFT_DLL
+
+RESOURCE_FILE := $(shell $(CYGPATH) -ua $(DYNAMORIO_HOME)/${CORE_DIRNAME}/win32/resources.rc)
+WINRESOURCE_FILE := $(shell $(CYGPATH) -ma ${RESOURCE_FILE})
+DYNAMORIO_WINDIR := $(shell $(CYGPATH) -ma ${DYNAMORIO_HOME})
+RCFLAGS := $(MSINCLUDE) -r
+
+dresources.res: $(RESOURCE_FILE)
+	$(RC) $(RCFLAGS) -fo $(subst /,\\,$@) /I "$(DYNAMORIO_WINDIR)\\${CORE_DIRNAME}\\lib" $(D)FILE_NAME=\"\" $(D)FILE_TYPE=$(TYPE_EXE) $(D)FILE_DESCRIPTION=\"regression_test\" $(WINRESOURCE_FILE)
+
+mresources.res: mresources.rc
+	$(RC) $(RCFLAGS) -fo $(subst /,\\,$@) mresources.rc
+
+oresources.res: oresources.rc
+	$(RC) $(RCFLAGS) -fo $(subst /,\\,$@) oresources.rc
+
+# using %$(EXE_SFX) doesn't seem to work well on win32, so we duplicate
+# we send output to a file since cl prints out crap like file name
+$(PGMS_EXE_PROBE_API): LINK += /MAP     # all probe exes/dlls need map files
+$(PGMS_EXE_COMMON)    \
+$(PGMS_EXE_PROBE_API) \
+$(PGMS_EXE_INTERFACE): %.exe: %.c $(TOOLS)
+	$(CC_BASE) $(EXEOUT)$@ $< $(LINK) > $*.clout 2>&1
+
+# we add dr resource version information to only some of our security tests to test both
+# sides of the -use_moduledb functionality, and ensure the matching works
+$(PGMS_EXE_SECURITY_OS_NODLL): %.exe: %.c $(TOOLS) dresources.res
+	$(CC_BASE) $(EXEOUT)$@ $< $(LINK) dresources.res > $*.clout 2>&1
+
+# we support a single .dll statically linked w/ a runall/ test
+# we add microsoft like version information to some tests to better test -use_moduledb
+$(PGMS_EXE_RUNALL_DLL): %.exe: %.c $(TOOLS) %.dll.dll mresources.res
+	$(CC_BASE) $(EXEOUT)$@ $< $(LINK) mresources.res $*.dll.lib > $*.clout 2>&1
+
+# copy existing test
+%-stress.exe: %.exe
+	$(CP) -f $< $@
+%-stress.expect: %.template
+	$(CP) -f $< $*-stress.template
+	$(PERL) gen_expect.pl '$(DEFS)' '$(DR_OPS)' $*-stress.template
+
+$(PGMS_EXE_OS_NODLL) $(PGMS_EXE_RUNALL_NODLL): %.exe: %.c $(TOOLS)
+	$(CC_BASE) $(EXEOUT)$@ $< $(LINK) > $*.clout 2>&1
+
+# we support a single .dll statically linked w/ a win32/ test
+$(PGMS_EXE_OS_DLL) $(PGMS_EXE_SECURITY_OS_DLL): %.exe: %.c $(TOOLS) %.dll.dll
+	$(CC_BASE) $(EXEOUT)$@ $< $(LINK) $*.dll.lib > $*.clout 2>&1
+
+%.dll: %.c $(TOOLS)
+	$(CC_BASE) $< $(LINK) $(DLLOUT)$@ $(LIBOUT)$*.lib > $*.clout 2>&1
+
+%.bind: %
+	$(BIND) -u $<
+
+# PR 230274: w/ new compiler we have to explicitly link to libc before dynamorio.lib 
+# or else we get floating point callout failures from DR exporting sin, cos,
+# and _fltused (from PR 219380).  However, if we link to msvcrt.lib we hit
+# CRT SxS issues (PR 236914), so make sure to use libcmt.lib (static libc).
+$(PGMS_EXE_API): %.exe: %.c $(TOOLS) $(DYNAMORIO_DIR)
+	$(CC_BASE_DYNAMORIO) $(D)USE_DYNAMO $(EXEOUT)$@ $< /link /nodefaultlib libcmt.lib $(LINK_DYNAMORIO) tools_asm.$(OBJ_SFX) > $*.clout 2>&1
+
+ifeq ($(ARCH), x64)
+CLIENT_LINK := /base:"0x72000000" /FIXED
+else
+CLIENT_LINK := 
+endif
+$(PGMS_DLL_INTERFACE): %.dll.dll: %.dll.c
+	$(CC_BASE_INTERFACE) $< $(DYNAMORIO_INC) /link /NODEFAULTLIB /NOENTRY /LIBPATH:$(DYNAMORIO_LIBDIR) dynamorio.lib /DLL /OUT:$@ $(CLIENT_LINK)
+
+# Just like client interface tests, probe clients can't depend upon $(TOOLS).
+# Note: client dlls need /MAP too because probe callback functions can be in 
+# in client dlls.
+$(PGMS_CLIENT_PROBE_API): %.dll: %.c
+	$(CC_BASE_INTERFACE) $< $(DYNAMORIO_INC) /link /NODEFAULTLIB /NOENTRY /LIBPATH:$(DYNAMORIO_LIBDIR) dynamorio.lib /MAP /DLL /OUT:$@
+
+$(PGMS_EXP_INTERFACE): %.expect: %.template
+	$(PERL) gen_expect.pl '$(DEFS) $(RUNREGRESSION_OS_VER)' '$(DR_OPS)' $*.template
+
+$(PGMS_EXP_API): %.expect: %.template
+	$(PERL) gen_expect.pl '$(DEFS) $(RUNREGRESSION_OS_VER)' '$(DR_OPS)' $*.template
+
+$(PGMS_EXE_PTHREADS): %.exe: %.c $(TOOLS)
+	$(CC_BASE) $(EXEOUT)$@ $< $(LINK) -lpthread > $*.clout 2>&1
+
+# we add non-ms non-dr version information to some tests to better test -use_moduledb
+$(PGMS_EXE_SECURITY_COMMON): %.exe: %.c $(TOOLS) oresources.res
+	$(CC_BASE) $(EXEOUT)$@ $< $(LINK) oresources.res > $*.clout 2>&1
+
+$(PGMS_EXP_OS) $(PGMS_EXP_HOTP) $(PGMS_EXP_PROBE_API): %.expect: %.template
+	$(PERL) gen_expect.pl '$(DEFS)' '$(DR_OPS)' $*.template
+
+# Note: only one test dll and one hot patch dll can be built today; the former
+#       being conditional, i.e., if the test.dll.c exists, it will be built.
+#       TODO: Multiple & variable number of test dlls.
+# Note: also, the test dll won't be linked with app; test has to load it - 
+#       while restrictive, it allows to test more common scenarios with dll 
+#       loading that the former doesn't allow
+# Note: /Fm needed to generate .map files which are used to generate hotp 
+#       config files
+$(PGMS_EXE_HOTP): %.exe: %.c $(TOOLS)
+	$(CC_BASE) $(OPT) /Fm $(EXEOUT)$@ $< $(LINK) > $*.clout 2>&1
+	@(if test -e $*.dll.c; then $(MAKE) -s $*.dll.dll; fi)
+
+# dlls used by both hotp & probe tests (not probe or client dlls) need map files
+$(PGMS_DLL_PROBE_API) $(PGMS_DLL_HOTP): %.dll: %.c $(TOOLS)
+	$(CC_BASE) $(OPT) /Fm $< $(LINK) $(DLLOUT)$@ > $*.clout 2>&1
+
+# Probes need DR API header files, and those in turn need stdio.h.
+$(PGMS_PROBES_PROBE_API): HOTP_DLL_CFLAGS += $(DYNAMORIO_INC) $(MSINCLUDE)
+# dlls containing probes or hot patches are built the same way. Also these dlls
+# shouldn't depend upon $(TOOLS) as $(TOOLS) links agains external dlls.
+$(PGMS_PROBES_PROBE_API) \
+$(PGMS_DLL_HOTP_PATCH): %.dll: %.c $(HOTP_DLL_INC_DIR)/hotpatch_interface.h
+	$(CC) $(HOTP_DLL_CFLAGS) $(I)$(HOTP_DLL_INC_DIR) $(OUT)$*.obj $< > $*.clout 2>&1
+	$(LD) $(HOTP_DLL_LDFLAGS) $(DLLOUT)$@ $*.obj >> $*.clout 2>&1
+
+# For some weird reason make decides all .cfg files are intermediate and deletes
+# them after running that hotp test; not good for debugging failures, so make it
+# "my precious".
+.PRECIOUS : %-defs.cfg
+
+# TODO: generates hotp defs only for .exe for now; and only one patch point
+# TODO: for now all modes are set to protect; there is no way to specify 
+#       different modes for different patch points; well, we can specify 
+#       multiple patch points for now anyway :)
+%-defs.cfg: %.exe
+	@cd $(CURWD)/$(@D); $(CURWD)/hotp/bin/build-cfg.pl -target $(notdir $*).exe -ppoint `$(CURWD)/hotp/bin/get_ppoints.pl $(notdir $*).exe`
+
+##################################################
+else # linux
+
+$(PGMS_EXE_COMMON): %: %.c $(TOOLS)
+	$(CC_BASE) $(EXEOUT)$@ $< $(LINK)
+$(PGMS_EXE_OS): %: %.c $(TOOLS)
+	$(CC_BASE) $(EXEOUT)$@ $< $(LINK)
+# extra runs that use existing apps: we copy the app and expect file
+%-reset: %
+	$(CP) -f $< $@
+%-reset.expect: %.expect
+	$(CP) -f $< $@
+# copy existing test
+%-stress: %
+	$(CP) -f $< $@
+%-stress.expect: %.template
+	$(CP) -f $< $*-stress.template
+	$(PERL) gen_expect.pl '$(DEFS)' '$(DR_OPS)' $*-stress.template
+$(PGMS_EXE_API): %: %.c $(TOOLS) $(DYNAMORIO_DIR)
+	$(CC_BASE_DYNAMORIO) $(D)USE_DYNAMO $(EXEOUT)$@ $< $(LINK_DYNAMORIO) tools_asm.$(OBJ_SFX) 
+$(PGMS_EXE_INTERFACE): %: %.c $(TOOLS)
+	$(CC_BASE) $(EXEOUT)$@ $< $(LINK)
+
+  ifeq ($(ARCH), x64)
+    # Currently required on 64bit that a client library have a
+    # preferred base address in the lower 2GB
+    PREFERRED_BASE := 0x72000000
+    LD_SCRIPT := ldscript
+    LD_USE_SCRIPT := -Xlinker -dT -Xlinker $(LD_SCRIPT)
+$(LD_SCRIPT):
+    # To set a preferred base address we tweak the default linker script
+	ld --verbose | grep -A 1000 SECTIONS | grep -B 1000 '^}' | \
+	  sed 's/0x400000/$(PREFERRED_BASE)/g; /^{/a . = $(PREFERRED_BASE);' > $@
+  else
+    CFLAGS := -m32 $(DBG) $(D)LINUX
+    LD_USE_SCRIPT :=
+  endif
+$(PGMS_DLL_INTERFACE): %.dll.so: %.dll.c $(LD_SCRIPT)
+	$(CC_BASE_INTERFACE) $(DYNAMORIO_INC) -fPIC -shared -nostartfiles $< -o $@ \
+        $(LD_USE_SCRIPT)
+
+$(PGMS_EXP_INTERFACE): %.expect: %.template
+	$(PERL) gen_expect.pl '$(DEFS)' '$(DR_OPS)' $*.template
+$(PGMS_EXP_API): %.expect: %.template
+	$(PERL) gen_expect.pl '$(DEFS)' '$(DR_OPS)' $*.template
+$(PGMS_EXE_PTHREADS): %: %.c $(TOOLS)
+	$(CC_BASE) $(EXEOUT)$@ $< $(LINK) -lpthread
+$(PGMS_EXE_SECURITY_COMMON): %: %.c $(TOOLS)
+	$(CC_BASE) $(EXEOUT)$@ $< $(LINK)
+$(PGMS_EXE_SECURITY_OS): %: %.c $(TOOLS)
+	$(CC_BASE) $(EXEOUT)$@ $< $(LINK)
+$(PGMS_EXP_OS): %.expect: %.template
+	$(PERL) gen_expect.pl '$(DEFS)' '$(DR_OPS)' $*.template
+
+endif # linux
+
+# for clean (xref PR 245246)
+ALL_TEMPLATES := $(shell $(LS) */*.template)
+ALL_GENERATED_EXPECT := $(patsubst %.template, %.expect, $(ALL_TEMPLATES))
+
+##################################################
+
+###########################################################################
+# test-specific rules
+
+win32/nativeexec.runinjector: DR_OPS += -native_exec_list nativeexec.dll.dll
+win32/nativeterminate.runinjector: DR_OPS += -native_exec_list nativeterminate.dll.dll
+
+linux/clone-reset.runinjector: DR_OPS += -reset_at_fragment_count 100
+linux/thread-reset.runinjector: DR_OPS += -reset_at_fragment_count 100
+
+common/decode-stress.runinjector: DR_OPS += -stress_recreate_state
+
+runall/calc-freeze.runrunall: DR_OPS += -coarse_enable_freeze
+# derek: I'm putting -no_probe_api here since the check below is not
+#   triggering over remote regression runs and I don't want to take the
+#   time to figure it out
+runall/calc-persist.runrunall: DR_OPS += -coarse_enable_freeze -use_persisted -no_probe_api
+ifneq (,$(filter /DCLIENT_INERFACE, $(DEFS)))
+# xref PR 226578 which tracks incompatibility between probe_api & pcaches
+runall/calc-persist.runrunall: DR_OPS += -no_probe_api
+endif
+
+win32/delaybind.runnative win32/delaybind.runinjector: LINK += /delayload:delaybind.dll.dll /delay:unload
+win32/delaybind.runnative win32/delaybind.runinjector: win32/delaybind.exe.bind win32/delaybind.dll.dll.bind
+security-win32/sec-fixed.dll.dll: LINK += /FIXED
+security-win32/secalign-fixed.dll.dll: LINK += /FIXED
+# PR 229292: we want over-ssh and local builds to match the template so
+# we always build w/o pdbs
+security-win32/except-execution.exe: DBG=
+win32/membar.exe: CC_BASE += $(OPT) # need optimizations
+
+ifeq ($(MACHINE), linux)
+# just happens to not make as many traces
+client-interface/events.runinterface: DR_OPS += -trace_threshold 5
+endif
+
+client-interface/security.runinterface: DR_OPS += -security_api
+client-interface/thread.runinterface: CLIENT_OPS := -paramx -paramy
+client-interface/file_io.runinterface: CLIENT_OPS := $(CURWD)/client-interface/file_io_data.txt
+client-interface/cbr4.runinterface: DR_OPS += -thread_private
+# See PR 244090 - currently there is no way for a client to see what options DR was 
+# run with.  The flush tests needs to know whether it can use the unlink
+# flush routines or not which depends on having either -thread_private or -enable_full_api
+# We check here and pass a client option if the unlink flush is safe to use.
+ifneq (,$(filter -thread_private -enable_full_api, $(DR_OPS)))
+client-interface/flush.runinterface: CLIENT_OPS := use_unlink
+endif
+
+api/dis.runapi: DR_OPS += -syntax_intel
+api/dis.runapi: EXE_ARGS = $(CURWD)/api/dis-udis86-randtest.raw
+
+###########################################################################
+### for test suite
+
+# API needs to be built with dynamo and then run natively
+# All others are built as native programs and run using injector
+
+TIMER_TIME = -s 60
+
+ifeq ($(MACHINE), win32)
+TIMER_NATIVE     = $(DYNAMORIO_TOOLS)/winstats.exe $(TIMER_TIME) -silent
+DYNAMORIO_DLL := $(subst /,\\,$(shell $(CYGPATH) -m $(DYNAMORIO_DIR)/dynamorio.dll))
+TIMER_INJECTOR = export DYNAMORIO_OPTIONS='$(DR_OPS)'; \
+    $(DYNAMORIO_DIR)/drinject.exe $(TIMER_TIME) $(DYNAMORIO_DLL)
+TIMER_API = export DYNAMORIO_OPTIONS='$(DR_OPS)'; \
+    $(DYNAMORIO_DIR)/drinject.exe $(TIMER_TIME) -noinject
+else
+TIMER_NATIVE     = $(DYNAMORIO_TOOLS)/runstats $(TIMER_TIME) -silent
+TIMER_INJECTOR   = $(DYNAMORIO_TOOLS)/runstats $(TIMER_TIME) -silent \
+   -env LD_LIBRARY_PATH '$(DYNAMORIO_DIR):$(LD_LIBRARY_PATH)' \
+   -env LD_PRELOAD 'libdrpreload.so libdynamorio.so' \
+   -env DYNAMORIO_OPTIONS '$(DR_OPS)'
+TIMER_API   = $(DYNAMORIO_TOOLS)/runstats $(TIMER_TIME) -silent \
+   -env DYNAMORIO_OPTIONS '$(DR_OPS)'
+endif
+
+# to ignore win32/unix file differences we do -b
+DIFF += -b
+
+ifeq ($(MACHINE), win32) 
+#only for win32
+PRELOAD_BUILD_DIR = TMPPREINJDIR
+$(DYNAMORIO_PREINJECT):
+	@(if $(LS) $(DYNAMORIO_HOME)/lib32/drpreinject.dll > .trash 2>&1; then \
+		$(CP) $(DYNAMORIO_HOME)/lib32/drpreinject.dll .; \
+	else \
+		cd $(CURWD); $(MKDIR) -p $(PRELOAD_BUILD_DIR); \
+		$(CP) -r $(DYNAMORIO_HOME)/${CORE_DIRNAME} $(PRELOAD_BUILD_DIR)/; \
+		$(CP) -r $(DYNAMORIO_MAKE) $(PRELOAD_BUILD_DIR)/; \
+		$(RM) $(PRELOAD_BUILD_DIR)/${CORE_DIRNAME}/Makefile.mydefines; \
+		cd $(PRELOAD_BUILD_DIR)/${CORE_DIRNAME}; \
+		$(MAKE) -s DEBUG=0 clean install; \
+		$(CP) $(CURWD)/$(PRELOAD_BUILD_DIR)/lib32/drpreinject.dll $(CURWD); \
+		cd $(CURWD); \
+	fi)
+endif
+
+# we run API tests first as those are more easily broken
+run: runheader runinterface runapi runnointerface
+
+# runallbuthotp is used only as solution for PR 234043
+runallbuthotp: runheader runinterface runapi runbase runsecurity runrunall
+
+runnointerface: runbase runsecurity runrunall runhotp
+
+runnogui: runbase runsecurity runrunall-nogui
+
+runnorunall: runbase runsecurity
+
+# we split these up for more parallelism w/ make -j
+runbase: runcommon runos
+
+runapi: reportbuild $(patsubst %,%.runapi,$(PGMS_ROOT_API))
+
+runinterface: reportbuild $(patsubst %,%.runinterface,$(PGMS_ROOT_INTERFACE))
+
+runcommon: reportbuild $(patsubst %,%.runinjector,$(PGMS_ROOT_COMMON))
+
+runprobeapi: reportbuild $(patsubst %,%.runprobeapi,$(PGMS_ROOT_PROBE_API))
+
+runos: reportbuild $(patsubst %,%.runinjector,$(PGMS_ROOT_OS))
+
+# FIXME: watch these: xref PR 205306 (bug 828)
+# they used to not die nicely and time out
+ifeq ($(MACHINE), linux)
+runos: $(patsubst %,%.runinjector,$(PGMS_ROOT_PTHREADS))
+endif
+
+# we split these up for more parallelism w/ make -j
+runsecurity: runbasesecurity runossecurity
+runbasesecurity: reportbuild $(patsubst %,%.runinjector,$(PGMS_ROOT_SECURITY_COMMON))
+
+runossecurity: reportbuild $(patsubst %,%.runinjector,$(PGMS_ROOT_SECURITY_OS))
+
+ifeq ($(MACHINE), win32)
+runrunall-nogui: reportbuild $(patsubst %,%.runrunall,$(RUNALL_NOGUI_ROOTS))
+else
+runrunall-nogui:
+endif
+
+ifeq ($(MACHINE), win32)
+runrunall: reportbuild $(patsubst %,%.runrunall,$(RUNALL_ROOTS))
+else
+runrunall:
+endif
+
+ifeq ($(MACHINE), win32)
+runhotp: reportbuild $(patsubst %,%.runhotp,$(PGMS_ROOT_HOTP))
+else
+runhotp:
+endif
+
+runheader:
+	@$(ECHO) "Disabled tests are -- ${DISABLED_ALL} --"
+
+reportbuild:
+	@$(ECHO) "Using build in $(DYNAMORIO_DIR)"
+
+# FIXME: share the parallelized runinjector cmd split
+runnative: $(patsubst %,%.runnative,$(PGMS_ROOT_API)) \
+           $(patsubst %,%.runnative,$(PGMS_ROOT_COMMON)) \
+           $(patsubst %,%.runnative,$(PGMS_ROOT_OS)) \
+           $(patsubst %,%.runnative,$(PGMS_ROOT_PTHREADS)) \
+           $(patsubst %,%.runnative,$(PGMS_ROOT_SECURITY_COMMON)) \
+           $(patsubst %,%.runnative,$(PGMS_ROOT_SECURITY_OS)) 
+
+# FIXME: share the parallelized runinjector cmd split
+expect: $(patsubst %,%.runexpect,$(PGMS_ROOT_API)) \
+        $(patsubst %,%.runexpect,$(PGMS_ROOT_COMMON)) \
+        $(patsubst %,%.runexpect,$(PGMS_ROOT_OS)) \
+        $(patsubst %,%.runexpect,$(PGMS_ROOT_PTHREADS)) \
+        $(patsubst %,%.runexpect,$(PGMS_ROOT_SECURITY_COMMON)) \
+        $(patsubst %,%.runexpect,$(PGMS_ROOT_SECURITY_OS))
+
+%.announce:
+	@(if ! test -e $(RESDIR); then $(MKDIR) $(RESDIR); fi)
+	@(if ! test -e $(RESDIR)/$(dir $*); then $(MKDIR) -p $(RESDIR)/$(dir $*); fi)
+	@(if ! test -e $$DYNAMORIO_LOGDIR; then $(MKDIR) -p $$DYNAMORIO_LOGDIR; fi)
+	@(if ! test -e $$DYNAMORIO_CACHE_ROOT; then $(MKDIR) -p $$DYNAMORIO_CACHE_ROOT; fi)
+	@$(ECHO)
+	@$(ECHO) ----- $* ----------------------
+
+.PHONY : %.runrunall $.runhotp
+
+%.runrunall: $(DYNAMORIO_PREINJECT) %.announce %.expect
+ifeq ($(MACHINE), win32)
+	@if test -e $*.c; then $(MAKE) -s $*.exe; fi
+	@cd $(RESDIR); export DYNAMORIO_OPTIONS='$(DR_OPS)'; $(SHELL) $(CURWD)/runall/runalltest.sh $(CURWD)/$*.runall $(DYNAMORIO_LIBDIR)\\dynamorio.dll $(DYNAMORIO_WINPREINJECT) $(RUNALL_MAX_WAIT) > $*.out 2>&1; cd $(CURWD)
+	@if $(DIFF) -i $(CURWD)/$*.expect $(RESDIR)/$*.out; then $(ECHO) "PASS"; else $(ECHO) "FAIL"; fi
+	@$(ECHO)
+	@(if $(LS) $(CURWD)/$*.template > .trash 2>&1; then $(MV) $(CURWD)/$*.expect $(CURWD)/$*.expect.last; fi)
+else
+	@$(ECHO) "run all not supported on linux"
+endif
+
+# prepend to avoid -thin_client assert
+%.runhotp: DR_OPS := -hot_patching $(DR_OPS)
+%.runhotp: $(DYNAMORIO_PREINJECT) %.exe %.hotp.dll %-defs.cfg %.announce %.expect
+ifeq ($(MACHINE), win32)
+	@cd $(RESDIR); $(PERL) $(CURWD)/hotp/bin/run_test.pl $(DYNAMORIO_DIR)/dynamorio.dll "$(DR_OPS)" $(DYNAMORIO_DIR)/drinject.exe $(CURWD)/$*.exe > $*.out 2>&1; cd $(CURWD)
+	@if $(DIFF) -i $(CURWD)/$*.expect $(RESDIR)/$*.out; then $(ECHO) "PASS"; else $(ECHO) "FAIL"; fi
+	@$(ECHO)
+	@(if $(LS) $(CURWD)/$*.template > .trash 2>&1; then $(MV) $(CURWD)/$*.expect $(CURWD)/$*.expect.last; fi)
+else
+	@$(ECHO) "hot patching not supported on linux; well, not yet"
+endif
+
+ifeq ($(MACHINE), win32)
+# again, we have to duplicate, b/c %$(EXE_SFX) doesn't work
+%.runnative: %.announce %.exe 
+else
+%.runnative: %.announce % 
+endif
+ifeq ($(MACHINE), win32)
+	@cd $(RESDIR); $(TIMER_NATIVE) $(CURWD)/$*$(EXE_SFX) > $*.out 2>&1; cd $(CURWD)
+else
+	@cd $(RESDIR); $(TIMER_NATIVE) $(CURWD)/$*$(EXE_SFX) > $*.out 2>&1; cd $(CURWD)
+endif
+	@(if $(LS) $(CURWD)/$*.template > .trash 2>&1; then $(PERL) gen_expect.pl "DNATIVE" "" $(CURWD)/$*.template; fi)	
+	@(if $(DIFF) $(CURWD)/$*.expect $(RESDIR)/$*.out; then $(ECHO) "PASS"; else $(ECHO) "FAIL"; fi)
+	@$(ECHO)	
+	@(if $(LS) $(CURWD)/$*.template > .trash 2>&1; then $(MV) $(CURWD)/$*.expect $(CURWD)/$*.expect.last; fi)	
+
+ifeq ($(MACHINE), win32)
+# again, we have to duplicate, b/c %$(EXE_SFX) doesn't work
+%.runapi: %.announce %.exe %.expect
+else
+%.runapi: %.announce % %.expect
+endif
+# set ops for API runs
+# must copy dll to . so on path for win32 (must be syswide path)
+# for PGMS_EXE_API
+ifeq ($(MACHINE), win32)
+	@cd $(RESDIR); $(CP) $(DYNAMORIO_DIR)/dynamorio.dll .; $(TIMER_API) $(CURWD)/$*$(EXE_SFX) $(EXE_ARGS) > $*.out 2>&1; cd $(CURWD)
+else
+	@cd $(RESDIR); $(TIMER_API) $(CURWD)/$*$(EXE_SFX) $(EXE_ARGS) > $*.out 2>&1; cd $(CURWD)
+endif
+	@(if $(DIFF) $(CURWD)/$*.expect $(RESDIR)/$*.out; then $(ECHO) "PASS"; else $(ECHO) "FAIL"; fi)
+	@$(ECHO)	
+	@(if $(LS) $(CURWD)/$*.template > .trash 2>&1; then $(MV) $(CURWD)/$*.expect $(CURWD)/$*.expect.last; fi)	
+
+# We have two types of client interface tests: 1) a client library + a custom
+# C test program we compile, and 2) a client library + a standard win32 
+# executable we launch with the runall script.  The presence of a .runall
+# input file vs. a .exe distinguishes the two rules below
+ifeq ($(MACHINE), win32)
+%.runinterface: DR_OPS += -code_api -client_lib "$(shell $(CYGPATH) -da $*.dll.dll);0;$(CLIENT_OPS)"
+%.runinterface: $(DYNAMORIO_PREINJECT) %.exe %.dll.dll %.announce %.expect
+else
+%.runinterface: DR_OPS += -code_api -client_lib "$(CURWD)/$*.dll.so;0;$(CLIENT_OPS)"
+%.runinterface: % %.dll.so %.announce %.expect
+endif
+	@cd $(RESDIR); $(TIMER_INJECTOR) $(CURWD)/$*$(EXE_SFX) > $*.out 2>&1; cd $(CURWD)
+	@(if $(DIFF) $(CURWD)/$*.expect $(RESDIR)/$*.out; then $(ECHO) "PASS"; else $(ECHO) "FAIL"; fi)
+	@$(ECHO)
+	@(if $(LS) $(CURWD)/$*.template > .trash 2>&1; then $(MV) $(CURWD)/$*.expect $(CURWD)/$*.expect.last; fi)
+
+ifeq ($(MACHINE), win32)
+%.runinterface: $(DYNAMORIO_PREINJECT) %.dll.dll %.runall %.announce %.expect
+	@cd $(RESDIR); export DYNAMORIO_OPTIONS='$(DR_OPS)'; $(SHELL) $(CURWD)/runall/runalltest.sh $(CURWD)/$*.runall $(DYNAMORIO_LIBDIR)\\dynamorio.dll $(DYNAMORIO_WINPREINJECT) $(INTERFACE_MAX_WAIT) > $*.out 2>&1; cd $(CURWD)
+	@if $(DIFF) -i $(CURWD)/$*.expect $(RESDIR)/$*.out; then $(ECHO) "PASS"; else $(ECHO) "FAIL"; fi
+	@$(ECHO)
+	@(if $(LS) $(CURWD)/$*.template > .trash 2>&1; then $(MV) $(CURWD)/$*.expect $(CURWD)/$*.expect.last; fi)
+else
+%.runinterface: %.dll.so %.runall %.announce %.expect
+	@$(ECHO) "SKIP - client interface runall not implemented on linux"
+endif
+
+%.runprobeapi: DR_OPS += -code_api -probe_api -client_lib "$(shell cygpath -da $*.client.dll);0;$(CLIENT_OPS)"
+# A probe-api test consists of an exe (test.exe) and a client
+# (test.client.dll).  Test dlls (test.dll.dll - for now allowing just one) and
+# probe dlls (test.probe*.dll) are optional.  By using SECONDEXPANSION
+# the dependency on a varying number of dlls can be enforced
+.SECONDEXPANSION:
+%.runprobeapi: %.exe %.client.dll \
+    $$(subst .c,.dll,$$(wildcard $$(@D)/$$*.probe*.c $$(@D)/$$*.dll.c)) \
+    %.announce %.expect
+ifeq ($(MACHINE), win32)
+	@cd $(RESDIR); $(TIMER_INJECTOR) $(CURWD)/$*$(EXE_SFX) > $*.out 2>&1; cd $(PWD)
+	@if $(DIFF) -i $(PWD)/$*.expect $(RESDIR)/$*.out; then echo "PASS"; else echo "FAIL"; fi
+	@echo
+	@(if ls $(PWD)/$*.template > .trash 2>&1; then mv -f $(PWD)/$*.expect $(PWD)/$*.expect.last; fi)
+else
+	@echo "SKIP - probe api NYI on linux"
+endif
+
+ifeq ($(MACHINE), win32)
+# again, we have to duplicate, b/c %$(EXE_SFX) doesn't work
+%.runinjector: %.announce %.exe %.expect
+# injector includes timer args
+	@cd $(RESDIR); $(TIMER_INJECTOR) $(CURWD)/$*$(EXE_SFX) > $*.out 2>&1; cd $(CURWD)
+else
+%.runinjector: %.announce % %.expect
+# timer includes injector args
+	@cd $(RESDIR); $(TIMER_INJECTOR) $(CURWD)/$* > $*.out 2>&1; cd $(CURWD)
+endif
+	@(if $(DIFF) $(CURWD)/$*.expect $(RESDIR)/$*.out; then $(ECHO) "PASS"; else $(ECHO) "FAIL"; fi)
+	@$(ECHO)
+	@(if $(LS) $(CURWD)/$*.template > .trash 2>&1; then $(MV) $(CURWD)/$*.expect $(CURWD)/$*.expect.last; fi)
+
+# put .out in same dir as .expect
+ifeq ($(MACHINE), win32)
+# again, we have to duplicate, b/c %$(EXE_SFX) doesn't work
+%.runexpect: %.announce %.exe %.expect
+else
+%.runexpect: %.announce % %.expect
+endif
+	$(TIMER_NATIVE) $*$(EXE_SFX) > $*.out 2>&1; cd .
+	$(DIFF) $*.expect $*.out; cd .
+	@$(ECHO)
+	@(if $(LS) $(CURWD)/$*.template > .trash 2>&1; then $(MV) $(CURWD)/$*.expect $(CURWD)/$*.expect.last; fi)
+
+# can't get brace expansion to work on cygwin!
+clean:
+	$(RM) tools_asm.$(OBJ_SFX)
+	$(RM) $(PGMS_EXE) $(EXTRA_TESTS_EXE) .trash
+	$(RM) */*.out */*.clout */*.$(OBJ_SFX) */*.pdb */*.ilk */*.opt */*.c~ */*.expect.last */*.dll */*.dll.so */*.lib */*.exp */*.manifest */*.s
+	$(RM) *.obj *.pdb *.clout *.trash drpreinject.dll *.res
+	$(RM) $(ALL_GENERATED_EXPECT)
+	$(RM) -r $(RESDIR)
+	$(RM) -r $(PRELOAD_BUILD_DIR)
+	$(RM) $(patsubst %,%.expect,$(EXTRA_TESTS))
+	$(RM) $(patsubst %,%.template,$(EXTRA_TESTS))
diff --git a/suite/tests/api/dis-udis86-randtest.raw b/suite/tests/api/dis-udis86-randtest.raw
new file mode 100644
index 000000000..c3a6e9c17
Binary files /dev/null and b/suite/tests/api/dis-udis86-randtest.raw differ
diff --git a/suite/tests/api/dis.c b/suite/tests/api/dis.c
new file mode 100644
index 000000000..5385d2225
--- /dev/null
+++ b/suite/tests/api/dis.c
@@ -0,0 +1,108 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Code Manipulation API Sample:
+ * dis.c
+ *
+ * Disassembles a binary file containing nothing but code.
+ */
+
+#include "dr_api.h"
+#include <assert.h>
+
+#define VERBOSE 1
+
+#define BUF_SIZE 4096
+/* arbitrary pc for pc-relative operands for consistent output */
+#define ORIG_PC ((app_pc)0x10000000)
+
+static void
+read_data(file_t f, void *drcontext)
+{
+    byte sbuf[BUF_SIZE];
+    byte *pc, *prev_pc;
+    int len, prev_buf_len = 0;
+
+    /* FIXME: re-run 64-bit asking for 32-bit mode */
+
+    do {
+        len = dr_read_file(f, sbuf, sizeof(sbuf));
+        pc = sbuf;
+        while (pc < sbuf+len) {
+            /* FIXME: want to cut it off instead of reading beyond for
+             * end of file!  If weren't printing it out as go along could
+             * mark invalid after seeing whether instr overflows.
+             */
+            prev_pc = pc;
+#if VERBOSE
+            dr_printf("+0x%04x  ", prev_pc - sbuf + prev_buf_len);
+#endif
+            pc = disassemble_from_copy(drcontext, pc, ORIG_PC, STDOUT,
+                                       false/*don't show pc*/,
+#if VERBOSE
+                                       true/*show bytes*/
+#else
+                                       false/*do not show bytes*/
+#endif
+                                       );
+            /* If invalid, try next byte */
+            /* FIXME: udis86 is going to byte after the one that makes it
+             * invalid: so if 1st byte is invalid opcode, go to 2nd;
+             * if modrm makes it invalid (0xc5 0xc5), go to 3rd.
+             * not clear that's nec. better but we need to reconcile that w/
+             * their diff for automated testing.
+             */
+            if (pc == NULL)
+                pc = prev_pc + 1;
+        }
+        prev_buf_len += sizeof(sbuf);
+    } while (len == sizeof(sbuf));
+}
+    
+int
+main(int argc, char *argv[])
+{
+    file_t f;
+    void *drcontext = dr_standalone_init();
+    if (argc != 2) {
+        dr_fprintf(STDERR, "Usage: %s <objfile>\n", argv[0]);
+        return 1;
+    }
+    f = dr_open_file(argv[1], DR_FILE_READ | DR_FILE_ALLOW_LARGE);
+    if (f == INVALID_FILE) {
+        dr_fprintf(STDERR, "Error opening %s\n", argv[1]);
+        return 1;
+    }
+    read_data(f, drcontext);
+    dr_close_file(f);
+    return 0;
+}
diff --git a/suite/tests/api/dis.template b/suite/tests/api/dis.template
new file mode 100644
index 000000000..8c336c254
--- /dev/null
+++ b/suite/tests/api/dis.template
@@ -0,0 +1,6244 @@
+#if +X64 :
++0x0000   bf ae 4f 80 95       mov    edi, 0x95804fae
++0x0005   f1                   int1   
++0x0006   fa                   cli    
++0x0007   43 8d 00             lea    eax, [r8]
++0x000a   0c eb                or     al, 0xeb
++0x000c   37...??              <INVALID>
++0x000d   05 9d 3a ca cf       add    eax, 0xcfca3a9d
++0x0012   35 35 32 a1 fc       xor    eax, 0xfca13235
++0x0017   65 83 00 f8          add    dword ptr [gs:rax], 0xf8
++0x001b   ef                   out    eax, dx
++0x001c   ea 11 6e 63 78 c2 60...??  <INVALID>
++0x001d   11 6e 63             adc    dword ptr [rsi+0x63], ebp
++0x0020   78 c2                js     0x000000000fffffc4
++0x0022   60...??              <INVALID>
++0x0023   6e                   outsb  
++0x0024   8e 79 92...??        <INVALID>
++0x0025   79 92                jns    0x000000000fffff94
++0x0027   7b 83                jnp    0x000000000fffff85
++0x0029   96                   xchg   esi, eax
++0x002a   06...??              <INVALID>
++0x002b   9f                   lahf   
++0x002c   02 dd                add    bl, ch
++0x002e   e6 9f                out    0x9f, al
++0x0030   d1 79 9f             sar    dword ptr [rcx-0x61], 0x01
++0x0033   ea 60 3f e3 85 60 f0...??  <INVALID>
++0x0034   60...??              <INVALID>
++0x0035   3f...??              <INVALID>
++0x0036   e3 85                jrcxz  0x000000000fffff87
++0x0038   60...??              <INVALID>
++0x0039   f0 7c 45...??        <INVALID>
++0x003a   7c 45                jl     0x0000000010000047
++0x003c   b9 22 25 d3 c0       mov    ecx, 0xc0d32522
++0x0041   e7 70                out    0x70, eax
++0x0043   56                   push   rsi
++0x0044   17...??              <INVALID>
++0x0045   37...??              <INVALID>
++0x0046   ec                   in     al
++0x0047   91                   xchg   ecx, eax
++0x0048   23 ee                and    ebp, esi
++0x004a   51                   push   rcx
++0x004b   b1 c8                mov    cl, 0xc8
++0x004d   1c 78                sbb    al, 0x78
++0x004f   ce...??              <INVALID>
++0x0050   d1 ed                shr    ebp, 0x01
++0x0052   12 7c 21 7b          adc    bh, byte ptr [rcx+0x7b]
++0x0056   8f 40 ff             pop    qword ptr [rax-0x01]
++0x0059   72 e3                jb     0x000000000fffffe5
++0x005b   d7                   xlatb  
++0x005c   62 83 84 d1 f7 99...??  <INVALID>
++0x005d   83 84 d1 f7 99 f4 6d add    dword ptr [rcx+rdx*8+0x6df499f7], 0x97
+     97
++0x0065   8e 2e                mov    gs, dword ptr [rsi]
++0x0067   e1 08                loope  0x000000001000000a
++0x0069   b0 96                mov    al, 0x96
++0x006b   10 a3 5e 58 6c 92    adc    byte ptr [rbx-0x6d93a7a2], ah
++0x0071   c2 85 7e             ret    0x7e85
++0x0074   02 ed                add    ch, ch
++0x0076   17...??              <INVALID>
++0x0077   27...??              <INVALID>
++0x0078   96                   xchg   esi, eax
++0x0079   b3 e4                mov    bl, 0xe4
++0x007b   8e b1 07 18 af fd...??  <INVALID>
++0x007c   b1 07                mov    cl, 0x07
++0x007e   18 af fd fe 04 ab    sbb    byte ptr [rdi-0x54fb0103], ch
++0x0084   81 1c 4e 44 6f 88 9c sbb    dword ptr [rsi+rcx*2], 0x9c886f44
++0x008b   2e 57                push   rdi
++0x008d   b9 48 a4 09 d4       mov    ecx, 0xd409a448
++0x0092   14 e9                adc    al, 0xe9
++0x0094   6d                   insd   
++0x0095   28 d9                sub    cl, bl
++0x0097   dd 2e...??           <INVALID>
++0x0098   2e f5                cmc    
++0x009a   62 00...??           <INVALID>
++0x009b   00 1f                add    byte ptr [rdi], bl
++0x009d   59                   pop    rcx
++0x009e   88 2f                mov    byte ptr [rdi], ch
++0x00a0   92                   xchg   edx, eax
++0x00a1   d6...??              <INVALID>
++0x00a2   21 21                and    dword ptr [rcx], esp
++0x00a4   53                   push   rbx
++0x00a5   76 37                jbe    0x0000000010000039
++0x00a7   89 39                mov    dword ptr [rcx], edi
++0x00a9   d2 f9                sar    cl, cl
++0x00ab   00 4b 5e             add    byte ptr [rbx+0x5e], cl
++0x00ae   45 cf                iretd  
++0x00b0   27...??              <INVALID>
++0x00b1   8e 2a                mov    gs, dword ptr [rdx]
++0x00b3   15 e1 e2 07 11       adc    eax, 0x1107e2e1
++0x00b8   50                   push   rax
++0x00b9   4b f2 65 65 a4       movsb  gs
++0x00be   ae                   scasb  
++0x00bf   67 44 73 6d          jnb    0x0000000010000071
++0x00c3   c3                   ret    
++0x00c4   3d 75 14 e1 5f       cmp    eax, 0x5fe11475
++0x00c9   ab                   stosd  
++0x00ca   6d                   insd   
++0x00cb   31 b4 a4 98 00 f8 69 xor    dword ptr [rsp+0x69f80098], esi
++0x00d2   33 c5                xor    eax, ebp
++0x00d4   cc                   int3   
++0x00d5   6a c8                push   0xc8
++0x00d7   3b b7 58 29 c3 9b    cmp    esi, dword ptr [rdi-0x643cd6a8]
++0x00dd   27...??              <INVALID>
++0x00de   32 80 bf c8 51 a1    xor    al, byte ptr [rax-0x5eae3741]
++0x00e4   a3 e8 a0 5f 70 ed 01 mov    dword ptr [0x93de01ed705fa0e8], eax
+     de 93
++0x00ed   80 71 4f 37          xor    byte ptr [rcx+0x4f], 0x37
++0x00f1   77 b3                jnbe   0x000000000fffffb5
++0x00f3   d7                   xlatb  
++0x00f4   66 d2 20             shl    byte ptr [rax], cl
++0x00f7   b7 91                mov    bh, 0x91
++0x00f9   55                   push   rbp
++0x00fa   56                   push   rsi
++0x00fb   90                   nop    
++0x00fc   54                   push   rsp
++0x00fd   b3 83                mov    bl, 0x83
++0x00ff   65 8f f2...??        <INVALID>
++0x0100   8f f2...??           <INVALID>
++0x0101   f2 59                pop    rcx
++0x0103   bb e3 c4 58 cf       mov    ebx, 0xcf58c4e3
++0x0108   8b 48 20             mov    ecx, dword ptr [rax+0x20]
++0x010b   76 38                jbe    0x000000001000003a
++0x010d   66 4d 78 83          js     0x000000000fffff87
++0x0111   32 68 9b             xor    ch, byte ptr [rax-0x65]
++0x0114   9e                   sahf   
++0x0115   49 0a be 9a dd c2 db or     dil, byte ptr [r14-0x243d2266]
++0x011c   25 da fc 34 81       and    eax, 0x8134fcda
++0x0121   86 3a                xchg   byte ptr [rdx], bh
++0x0123   b0 d0                mov    al, 0xd0
++0x0125   94                   xchg   esp, eax
++0x0126   16...??              <INVALID>
++0x0127   2d a4 27 fb e7       sub    eax, 0xe7fb27a4
++0x012c   f5                   cmc    
++0x012d   25 f6 47 5f 7f       and    eax, 0x7f5f47f6
++0x0132   71 64                jno    0x0000000010000066
++0x0134   81 66 87 80 c9 d0 84 and    dword ptr [rsi-0x79], 0x84d0c980
++0x013b   99                   cdq    
++0x013c   66 f7 77 bc          div    dx, ax, word ptr [rdi-0x44]
++0x0140   2a dd                sub    bl, ch
++0x0142   54                   push   rsp
++0x0143   08 91 4a 9a d3 ad    or     byte ptr [rcx-0x522c65b6], dl
++0x0149   7b 9f                jnp    0x000000000fffffa1
++0x014b   a5                   movsd  
++0x014c   2c e0                sub    al, 0xe0
++0x014e   9b                   fwait  
++0x014f   16...??              <INVALID>
++0x0150   c5 c5...??           <INVALID>
++0x0151   c5 a8 7b cc a7 e4...??  <INVALID>
++0x0152   a8 7b                test   al, 0x7b
++0x0154   cc                   int3   
++0x0155   a7                   cmpsd  
++0x0156   e4 ac                in     al, 0xac
++0x0158   b2 d8                mov    dl, 0xd8
++0x015a   54                   push   rsp
++0x015b   01 cf                add    edi, ecx
++0x015d   d6...??              <INVALID>
++0x015e   16...??              <INVALID>
++0x015f   0f cd                bswap  ebp
++0x0161   1d c4 63 24 13       sbb    eax, 0x132463c4
++0x0166   28 81 f5 a3 b4 98    sub    byte ptr [rcx-0x674b5c0b], al
++0x016c   df d1                fstp   st1, st0
++0x016e   0c 40                or     al, 0x40
++0x0170   ef                   out    eax, dx
++0x0171   09 61 b9             or     dword ptr [rcx-0x47], esp
++0x0174   73 ec                jnb    0x000000000fffffee
++0x0176   e8 b8 81 85 95       call   0xffffffffa58581bd
++0x017b   fe a6 da 11 a4 ea...??  <INVALID>
++0x017c   a6                   cmpsb  
++0x017d   da 11                ficom  dword ptr [rcx]
++0x017f   a4                   movsb  
++0x0180   ea b0 78 35 32 68 52...??  <INVALID>
++0x0181   b0 78                mov    al, 0x78
++0x0183   35 32 68 52 62       xor    eax, 0x62526832
++0x0188   f1                   int1   
++0x0189   08 88 ef d1 8f e6    or     byte ptr [rax-0x19702e11], cl
++0x018f   91                   xchg   ecx, eax
++0x0190   79 80                jns    0x000000000fffff82
++0x0192   9c                   pushfq 
++0x0193   72 38                jb     0x000000001000003a
++0x0195   de 16                ficom  word ptr [rsi]
++0x0197   0e...??              <INVALID>
++0x0198   2f...??              <INVALID>
++0x0199   e0 b4                loopne 0x000000000fffffb6
++0x019b   50                   push   rax
++0x019c   31 60 2f             xor    dword ptr [rax+0x2f], esp
++0x019f   fa                   cli    
++0x01a0   6d                   insd   
++0x01a1   c4 74 2d 69...??     <INVALID>
++0x01a2   74 2d                jz     0x000000001000002f
++0x01a4   69 6e 03 37 74 c6 4c imul   ebp, dword ptr [rsi+0x03], 0x4cc67437
++0x01ab   df 30                fbstp  tbyte ptr [rax]
++0x01ad   c2 91 7e             ret    0x7e91
++0x01b0   35 9e 18 d1 09       xor    eax, 0x09d1189e
++0x01b5   97                   xchg   edi, eax
++0x01b6   2d 63 b5 4f e3       sub    eax, 0xe34fb563
++0x01bb   83 52 50 be          adc    dword ptr [rdx+0x50], 0xbe
++0x01bf   8e 26                mov    fs, dword ptr [rsi]
++0x01c1   1d ba df 07 85       sbb    eax, 0x8507dfba
++0x01c6   5e                   pop    rsi
++0x01c7   64 87 b7 6d 0d 2f 77 xchg   dword ptr [fs:rdi+0x772f0d6d], esi
++0x01ce   a4                   movsb  
++0x01cf   0e...??              <INVALID>
++0x01d0   5b                   pop    rbx
++0x01d1   01 93 af 4f d4 c3    add    dword ptr [rbx-0x3c2bb051], edx
++0x01d7   03 2e                add    ebp, dword ptr [rsi]
++0x01d9   33 74 d3 8d          xor    esi, dword ptr [rbx+rdx*8-0x73]
++0x01dd   d7                   xlatb  
++0x01de   bc 5b 41 4e 08       mov    esp, 0x084e415b
++0x01e3   0e...??              <INVALID>
++0x01e4   cf                   iretd  
++0x01e5   23 73 42             and    esi, dword ptr [rbx+0x42]
++0x01e8   cc                   int3   
++0x01e9   56                   push   rsi
++0x01ea   1b 27                sbb    esp, dword ptr [rdi]
++0x01ec   75 34                jnz    0x0000000010000036
++0x01ee   ba 95 66 65 27       mov    edx, 0x27656695
++0x01f3   30 d2                xor    dl, dl
++0x01f5   6e                   outsb  
++0x01f6   d0 e2                shl    dl, 0x01
++0x01f8   5d                   pop    rbp
++0x01f9   d5 47...??           <INVALID>
++0x01fa   47 29 46 f7          sub    dword ptr [r14-0x09], r8d
++0x01fe   e8 bb 48 04 2d       call   0x000000003d0448c0
++0x0203   c2 d9 d5             ret    0xd5d9
++0x0206   e0 8b                loopne 0x000000000fffff8d
++0x0208   93                   xchg   ebx, eax
++0x0209   0b 12                or     edx, dword ptr [rdx]
++0x020b   fc                   cld    
++0x020c   7a 19                jp     0x000000001000001b
++0x020e   47 22 eb             and    r13l, r11l
++0x0211   17...??              <INVALID>
++0x0212   cb                   retf   
++0x0213   65 95                xchg   ebp, eax
++0x0215   e5 f1                in     eax, 0xf1
++0x0217   f3 60...??           <INVALID>
++0x0218   60...??              <INVALID>
++0x0219   ae                   scasb  
++0x021a   41 16...??           <INVALID>
++0x021b   16...??              <INVALID>
++0x021c   60...??              <INVALID>
++0x021d   a1 9e 4d 57 4f be 1e mov    eax, dword ptr [0xddd71ebe4f574d9e]
+     d7 dd
++0x0226   ac                   lodsb  
++0x0227   34 a3                xor    al, 0xa3
++0x0229   94                   xchg   esp, eax
++0x022a   c1 a5 ce ce 9c b3 48 shl    dword ptr [rbp-0x4c633132], 0x48
++0x0231   60...??              <INVALID>
++0x0232   a8 dd                test   al, 0xdd
++0x0234   bf d3 c7 bb 4e       mov    edi, 0x4ebbc7d3
++0x0239   41 a4                movsb  
++0x023b   0c 7c                or     al, 0x7c
++0x023d   ab                   stosd  
++0x023e   51                   push   rcx
++0x023f   7b ec                jnp    0x000000000fffffee
++0x0241   32 99 b3 55 93 87    xor    bl, byte ptr [rcx-0x786caa4d]
++0x0247   c0 4a b1 f2          ror    byte ptr [rdx-0x4f], 0xf2
++0x024b   95                   xchg   ebp, eax
++0x024c   f0 53...??           <INVALID>
++0x024d   53                   push   rbx
++0x024e   91                   xchg   ecx, eax
++0x024f   55                   push   rbp
++0x0250   c4 0d cb e5 f5 bc...??  <INVALID>
++0x0251   0d cb e5 f5 bc       or     eax, 0xbcf5e5cb
++0x0256   d2 ca                ror    dl, cl
++0x0258   a0 ba 04 13 c8 db e9 mov    al, byte ptr [0xd37de9dbc81304ba]
+     7d d3
++0x0261   11 e7                adc    edi, esp
++0x0263   fd                   std    
++0x0264   31 3e                xor    dword ptr [rsi], edi
++0x0266   e6 9b                out    0x9b, al
++0x0268   62 c0...??           <INVALID>
++0x0269   c0 44 35 33 47       rol    byte ptr [rbp+rsi+0x33], 0x47
++0x026e   0c 73                or     al, 0x73
++0x0270   7e 87                jle    0x000000000fffff89
++0x0272   50                   push   rax
++0x0273   a2 e4 87 a6 5f fb 68 mov    byte ptr [0xb14468fb5fa687e4], al
+     44 b1
++0x027c   34 f2                xor    al, 0xf2
++0x027e   a3 c3 b4 8a 3d 40 b5 mov    dword ptr [0x0287b5403d8ab4c3], eax
+     87 02
++0x0287   e2 de                loop   0x000000000fffffe0
++0x0289   dc f8                fdiv   st0, st0
++0x028b   ad                   lodsd  
++0x028c   9e                   sahf   
++0x028d   e0 57                loopne 0x0000000010000059
++0x028f   5d                   pop    rbp
++0x0290   a3 18 61 3a 4a c3 2a mov    dword ptr [0xbf562ac34a3a6118], eax
+     56 bf
++0x0299   92                   xchg   edx, eax
++0x029a   a1 8f 2e 66 d9 f7 b3 mov    eax, dword ptr [0xb5f0b3f7d9662e8f]
+     f0 b5
++0x02a3   f4                   hlt    
++0x02a4   d0 54 f8 c1          rcl    byte ptr [rax+rdi*8-0x3f], 0x01
++0x02a8   64 30 6f d5          xor    byte ptr [fs:rdi-0x2b], ch
++0x02ac   9f                   lahf   
++0x02ad   53                   push   rbx
++0x02ae   c6 87 b8 7c d2 8e ac mov    byte ptr [rdi-0x712d8348], 0xac
++0x02b5   ca 11 e4             retf   0xe411
++0x02b8   18 9c d5 aa 1c ed eb sbb    byte ptr [rbp+rdx*8-0x1412e356], bl
++0x02bf   a3 fc 6b 34 bc 08 8a mov    dword ptr [0x8f318a08bc346bfc], eax
+     31 8f
++0x02c8   5d                   pop    rbp
++0x02c9   1e...??              <INVALID>
++0x02ca   01 15 e4 b4 ef 22    add    <rel> dword ptr [0x0000000032efb4ea], edx
++0x02d0   0a 0e                or     cl, byte ptr [rsi]
++0x02d2   54                   push   rsp
++0x02d3   63 15 16 c2 b9 d4    movsxd edx, <rel> dword ptr [0xffffffffe4b9c21c]
++0x02d9   d2 79 7d             sar    byte ptr [rcx+0x7d], cl
++0x02dc   cb                   retf   
++0x02dd   c7 7c bb e4 8e 81 93...??  <INVALID>
+     44
++0x02de   7c bb                jl     0x000000000fffffbd
++0x02e0   e4 8e                in     al, 0x8e
++0x02e2   81 93 44 7f e2 e1 66 adc    dword ptr [rbx-0x1e1d80bc], 0x87eac066
+     c0 ea 87
++0x02ec   db e0...??           <INVALID>
++0x02ed   e0 91                loopne 0x000000000fffff93
++0x02ef   10 57 b6             adc    byte ptr [rdi-0x4a], dl
++0x02f2   af                   scasd  
++0x02f3   ff b7 6f 94 3c 9d    push   qword ptr [rdi-0x62c36b91]
++0x02f9   af                   scasd  
++0x02fa   e4 1b                in     al, 0x1b
++0x02fc   57                   push   rdi
++0x02fd   dc 71 5f             fdiv   qword ptr [rcx+0x5f]
++0x0300   73 b4                jnb    0x000000000fffffb6
++0x0302   7f 93                jnle   0x000000000fffff95
++0x0304   67 f2 82 a9 25 00 7c...??  <INVALID>
+     9b 81
++0x0305   f2 82 a9 25 00 7c 9b...??  <INVALID>
+     81
++0x0306   82 a9 25 00 7c 9b 81...??  <INVALID>
++0x0307   a9 25 00 7c 9b       test   eax, 0x9b7c0025
++0x030c   81 2b 3d 3c 1c fe    sub    dword ptr [rbx], 0xfe1c3c3d
++0x0312   b5 ec                mov    ch, 0xec
++0x0314   a7                   cmpsd  
++0x0315   7a 82                jp     0x000000000fffff84
++0x0317   42 ff 39...??        <INVALID>
++0x0318   ff 39...??           <INVALID>
++0x0319   39 57 31             cmp    dword ptr [rdi+0x31], edx
++0x031c   bc 4a 1f 84 98       mov    esp, 0x98841f4a
++0x0321   04 bd                add    al, 0xbd
++0x0323   ed                   in     eax, dx
++0x0324   a7                   cmpsd  
++0x0325   b2 f0                mov    dl, 0xf0
++0x0327   7b 31                jnp    0x0000000010000033
++0x0329   19 4f bc             sbb    dword ptr [rdi-0x44], ecx
++0x032c   e1 64                loope  0x0000000010000066
++0x032e   01 8d d9 81 15 f0    add    dword ptr [rbp-0x0fea7e27], ecx
++0x0334   a4                   movsb  
++0x0335   f4                   hlt    
++0x0336   a4                   movsb  
++0x0337   ce...??              <INVALID>
++0x0338   1c 2d                sbb    al, 0x2d
++0x033a   a6                   cmpsb  
++0x033b   98                   cwde   
++0x033c   4a 6d                insd   
++0x033e   c6 e5 a3...??        <INVALID>
++0x033f   e5 a3                in     eax, 0xa3
++0x0341   6f                   outsd  
++0x0342   e0 c7                loopne 0x000000000fffffc9
++0x0344   54                   push   rsp
++0x0345   19 f8                sbb    eax, edi
++0x0347   3d 31 55 54 bf       cmp    eax, 0xbf545531
++0x034c   64 97                xchg   edi, eax
++0x034e   57                   push   rdi
++0x034f   78 30                js     0x0000000010000032
++0x0351   7d bb                jnl    0x000000000fffffbd
++0x0353   48 dd 79 81          fnstsw word ptr [rcx-0x7f]
++0x0357   36 1f...??           <INVALID>
++0x0358   1f...??              <INVALID>
++0x0359   c1 21 e0             shl    dword ptr [rcx], 0xe0
++0x035c   0b a7 26 29 d5 d4    or     esp, dword ptr [rdi-0x2b2ad6da]
++0x0362   57                   push   rdi
++0x0363   63 9f 91 82 ed 38    movsxd ebx, dword ptr [rdi+0x38ed8291]
++0x0369   ed                   in     eax, dx
++0x036a   4f f7 1d 67 c3 58 1d neg    <rel> qword ptr [0x000000002d58c36e]
++0x0371   f9                   stc    
++0x0372   6d                   insd   
++0x0373   b1 34                mov    cl, 0x34
++0x0375   b2 1b                mov    dl, 0x1b
++0x0377   a1 f8 a1 33 0c bb 7c mov    eax, dword ptr [0x35867cbb0c33a1f8]
+     86 35
++0x0380   d6...??              <INVALID>
++0x0381   88 03                mov    byte ptr [rbx], al
++0x0383   44 06...??           <INVALID>
++0x0384   06...??              <INVALID>
++0x0385   ee                   out    al, dx
++0x0386   ad                   lodsd  
++0x0387   17...??              <INVALID>
++0x0388   75 2b                jnz    0x000000001000002d
++0x038a   63 c7                movsxd eax, edi
++0x038c   a7                   cmpsd  
++0x038d   f4                   hlt    
++0x038e   bf 57 23 ac 95       mov    edi, 0x95ac2357
++0x0393   55                   push   rbp
++0x0394   43 5b                pop    r11
++0x0396   15 c4 8b a8 8b       adc    eax, 0x8ba88bc4
++0x039b   d3 e8                shr    eax, cl
++0x039d   4b f6 31             div    ah, al, byte ptr [r9]
++0x03a0   62 59 df...??        <INVALID>
++0x03a1   59                   pop    rcx
++0x03a2   df d8                fstp   st0, st0
++0x03a4   bd 13 1a 4b a4       mov    ebp, 0xa44b1a13
++0x03a9   33 76 d9             xor    esi, dword ptr [rsi-0x27]
++0x03ac   19 16                sbb    dword ptr [rsi], edx
++0x03ae   d2 89 22 bc 3e bb    ror    byte ptr [rcx-0x44c143de], cl
++0x03b4   6f                   outsd  
++0x03b5   23 be 35 f4 35 d5    and    edi, dword ptr [rsi-0x2aca0bcb]
++0x03bb   af                   scasd  
++0x03bc   00 30                add    byte ptr [rax], dh
++0x03be   78 1c                js     0x000000001000001e
++0x03c0   ea 22 85 df bf c5 23...??  <INVALID>
++0x03c1   22 85 df bf c5 23    and    al, byte ptr [rbp+0x23c5bfdf]
++0x03c7   7d 10                jnl    0x0000000010000012
++0x03c9   97                   xchg   edi, eax
++0x03ca   84 59 36             test   byte ptr [rcx+0x36], bl
++0x03cd   35 51 dd f2 2e       xor    eax, 0x2ef2dd51
++0x03d2   be 1f 5b dd 6c       mov    esi, 0x6cdd5b1f
++0x03d7   50                   push   rax
++0x03d8   dd 98 77 4f 56 7a    fstp   qword ptr [rax+0x7a564f77]
++0x03de   a4                   movsb  
++0x03df   bc dc 8f dc 6e       mov    esp, 0x6edc8fdc
++0x03e4   35 d1 a5 e4 57       xor    eax, 0x57e4a5d1
++0x03e9   bf bc a0 90 48       mov    edi, 0x4890a0bc
++0x03ee   d8 fa                fdivr  st0, st2
++0x03f0   9d                   popfq  
++0x03f1   70 ac                jo     0x000000000fffffae
++0x03f3   27...??              <INVALID>
++0x03f4   90                   nop    
++0x03f5   2e 64 e5 0e          in     eax, 0x0e
++0x03f9   e5 c1                in     eax, 0xc1
++0x03fb   38 a4 5d 9a e6 18 1e cmp    byte ptr [rbp+rbx*2+0x1e18e69a], ah
++0x0402   87 8b bd 44 4c 51    xchg   dword ptr [rbx+0x514c44bd], ecx
++0x0408   60...??              <INVALID>
++0x0409   a7                   cmpsd  
++0x040a   6f                   outsd  
++0x040b   43 ab                stosd  
++0x040d   9b                   fwait  
++0x040e   c6 ff b8...??        <INVALID>
++0x040f   ff b8 9a f4 26 e4...??  <INVALID>
++0x0410   b8 9a f4 26 e4       mov    eax, 0xe426f49a
++0x0415   be 5c b0 7d 3d       mov    esi, 0x3d7db05c
++0x041a   21 73 34             and    dword ptr [rbx+0x34], esi
++0x041d   da 69 09             fisubr dword ptr [rcx+0x09]
++0x0420   aa                   stosb  
++0x0421   38 8f 9c b8 9c a0    cmp    byte ptr [rdi-0x5f634764], cl
++0x0427   7f 21                jnle   0x0000000010000023
++0x0429   38 b6 87 b4 f9 55    cmp    byte ptr [rsi+0x55f9b487], dh
++0x042f   df 6f 31             fild   qword ptr [rdi+0x31]
++0x0432   58                   pop    rax
++0x0433   d9 4d b1...??        <INVALID>
++0x0434   4d b1 1f             mov    r9l, 0x1f
++0x0437   f8                   clc    
++0x0438   79 08                jns    0x000000001000000a
++0x043a   8b 2a                mov    ebp, dword ptr [rdx]
++0x043c   e5 06                in     eax, 0x06
++0x043e   b4 85                mov    ah, 0x85
++0x0440   13 4b e3             adc    ecx, dword ptr [rbx-0x1d]
++0x0443   47 7b a7             jnp    0x000000000fffffaa
++0x0446   94                   xchg   esp, eax
++0x0447   3e c7 53 7f 45 62 14...??  <INVALID>
+     11
++0x0448   c7 53 7f 45 62 14 11...??  <INVALID>
++0x0449   53                   push   rbx
++0x044a   7f 45                jnle   0x0000000010000047
++0x044c   62 14 11...??        <INVALID>
++0x044d   14 11                adc    al, 0x11
++0x044f   de b5 c3 54 41 4b    fidiv  word ptr [rbp+0x4b4154c3]
++0x0455   7e 84                jle    0x000000000fffff86
++0x0457   85 85 22 51 68 72    test   dword ptr [rbp+0x72685122], eax
++0x045d   f6 52 50             not    byte ptr [rdx+0x50]
++0x0460   a2 be d9 3a ab 70 f2 mov    byte ptr [0xe4e9f270ab3ad9be], al
+     e9 e4
++0x0469   05 46 36 04 6f       add    eax, 0x6f043646
++0x046e   a3 e8 9c c9 b9 ad eb mov    dword ptr [0x1d76ebadb9c99ce8], eax
+     76 1d
++0x0477   d1 b1 9b ac 0d 8e    shl    dword ptr [rcx-0x71f25365], 0x01
++0x047d   15 f1 2a df ad       adc    eax, 0xaddf2af1
++0x0482   24 52                and    al, 0x52
++0x0484   88 c1                mov    cl, al
++0x0486   52                   push   rdx
++0x0487   25 62 06 05 f6       and    eax, 0xf6050662
++0x048c   ec                   in     al
++0x048d   56                   push   rsi
++0x048e   eb 63                jmp    0x0000000010000065
++0x0490   53                   push   rbx
++0x0491   fc                   cld    
++0x0492   0c e4                or     al, 0xe4
++0x0494   75 27                jnz    0x0000000010000029
++0x0496   d0 30                shl    byte ptr [rax], 0x01
++0x0498   b9 3c 58 a2 10       mov    ecx, 0x10a2583c
++0x049d   bc da 32 14 8c       mov    esp, 0x8c1432da
++0x04a2   7a 0b                jp     0x000000001000000d
++0x04a4   61...??              <INVALID>
++0x04a5   46 ff ab 15 27 eb f6 jmp    
++0x04ac   cd bb                int    0xbb
++0x04ae   4f 6f                outsd  
++0x04b0   b2 55                mov    dl, 0x55
++0x04b2   1c bb                sbb    al, 0xbb
++0x04b4   94                   xchg   esp, eax
++0x04b5   1d 7e a7 52 67       sbb    eax, 0x6752a77e
++0x04ba   25 01 d4 a1 d7       and    eax, 0xd7a1d401
++0x04bf   f7 c8 92 50 cf ca    test   eax, 0xcacf5092
++0x04c5   c1 be f0 ce 7a 88 ce sar    dword ptr [rsi-0x77853110], 0xce
++0x04cc   b7 f7                mov    bh, 0xf7
++0x04ce   24 80                and    al, 0x80
++0x04d0   38 34 90             cmp    byte ptr [rax+rdx*4], dh
++0x04d3   e5 cf                in     eax, 0xcf
++0x04d5   30 2e                xor    byte ptr [rsi], ch
++0x04d7   af                   scasd  
++0x04d8   3d 89 3c 82 2b       cmp    eax, 0x2b823c89
++0x04dd   88 30                mov    byte ptr [rax], dh
++0x04df   be b9 f5 a0 59       mov    esi, 0x59a0f5b9
++0x04e4   7e 99                jle    0x000000000fffff9b
++0x04e6   f4                   hlt    
++0x04e7   37...??              <INVALID>
++0x04e8   a4                   movsb  
++0x04e9   8d b3 81 82 42 33    lea    esi, [rbx+0x33428281]
++0x04ef   1f...??              <INVALID>
++0x04f0   bc 8f cb 77 07       mov    esp, 0x0777cb8f
++0x04f5   9b                   fwait  
++0x04f6   f2 56                push   rsi
++0x04f8   14 b7                adc    al, 0xb7
++0x04fa   41 43 4e 9b          fwait  
++0x04fe   3d 8a a7 68 84       cmp    eax, 0x8468a78a
++0x0503   dd 9a 40 30 e9 97    fstp   qword ptr [rdx-0x6816cfc0]
++0x0509   d4 07...??           <INVALID>
++0x050a   07...??              <INVALID>
++0x050b   a3 a1 23 70 2d cb 1b mov    dword ptr [0xd3b81bcb2d7023a1], eax
+     b8 d3
++0x0514   49 bc ff 9f b7 3a 36 mov    r12, 0xb4a79c363ab79fff
+     9c a7 b4
++0x051e   ad                   lodsd  
++0x051f   0a 32                or     dh, byte ptr [rdx]
++0x0521   36 26 16...??        <INVALID>
++0x0522   26 16...??           <INVALID>
++0x0523   16...??              <INVALID>
++0x0524   42 31 09             xor    dword ptr [rcx], ecx
++0x0527   bf 47 01 44 a1       mov    edi, 0xa1440147
++0x052c   39 43 e3             cmp    dword ptr [rbx-0x1d], eax
++0x052f   c7 44 b4 90 20 a4 8a mov    dword ptr [rsp+rsi*4-0x70], 0xb08aa420
+     b0
++0x0537   95                   xchg   ebp, eax
++0x0538   19 dd                sbb    ebp, ebx
++0x053a   b3 42                mov    bl, 0x42
++0x053c   87 7d e1             xchg   dword ptr [rbp-0x1f], edi
++0x053f   35 90 2d 06 23       xor    eax, 0x23062d90
++0x0544   f0 02 5b 02...??     <INVALID>
++0x0545   02 5b 02             add    bl, byte ptr [rbx+0x02]
++0x0548   65 9d                popfq  
++0x054a   c9                   leave  
++0x054b   c1 79 9f 7d          sar    dword ptr [rcx-0x61], 0x7d
++0x054f   7a a1                jp     0x000000000fffffa3
++0x0551   93                   xchg   ebx, eax
++0x0552   b9 5d 0e 70 76       mov    ecx, 0x76700e5d
++0x0557   d8 a0 eb 11 de 38    fsub   dword ptr [rax+0x38de11eb]
++0x055d   af                   scasd  
++0x055e   f9                   stc    
++0x055f   96                   xchg   esi, eax
++0x0560   a0 a2 36 fd 4e fb dd mov    al, byte ptr [0x3429ddfb4efd36a2]
+     29 34
++0x0569   e7 f6                out    0xf6, eax
++0x056b   05 cd 6c f7 28       add    eax, 0x28f76ccd
++0x0570   4c 16...??           <INVALID>
++0x0571   16...??              <INVALID>
++0x0572   f3 46 a7             rep cmpsd 
++0x0575   c9                   leave  
++0x0576   5f                   pop    rdi
++0x0577   e3 b4                jrcxz  0x000000000fffffb6
++0x0579   ca 50 34             retf   0x3450
++0x057c   8a ec                mov    ch, ah
++0x057e   a5                   movsd  
++0x057f   aa                   stosb  
++0x0580   fb                   sti    
++0x0581   0a 9b 99 41 a5 1e    or     bl, byte ptr [rbx+0x1ea54199]
++0x0587   a0 d6 c0 3c 38 93 4d mov    al, byte ptr [0x4a4f4d93383cc0d6]
+     4f 4a
++0x0590   4a 01 08             add    qword ptr [rax], rcx
++0x0593   14 6e                adc    al, 0x6e
++0x0595   40 c9                leave  
++0x0597   14 26                adc    al, 0x26
++0x0599   eb bb                jmp    0x000000000fffffbd
++0x059b   c7 9c 6d 60 62 24 c7...??  <INVALID>
+     2c 77 0c ae
++0x059c   9c                   pushfq 
++0x059d   6d                   insd   
++0x059e   60...??              <INVALID>
++0x059f   62 24 c7...??        <INVALID>
++0x05a0   24 c7                and    al, 0xc7
++0x05a2   2c 77                sub    al, 0x77
++0x05a4   0c ae                or     al, 0xae
++0x05a6   94                   xchg   esp, eax
++0x05a7   62 cb...??           <INVALID>
++0x05a8   cb                   retf   
++0x05a9   f2 47 4d 3d d0 bc 7c cmp    rax, 0x787cbcd0
+     78
++0x05b1   3a f7                cmp    dh, bh
++0x05b3   09 f2                or     edx, esi
++0x05b5   70 a4                jo     0x000000000fffffa6
++0x05b7   34 9b                xor    al, 0x9b
++0x05b9   f6 85 64 01 74 bd e9 test   byte ptr [rbp-0x428bfe9c], 0xe9
++0x05c0   a3 7a e7 e9 d2 52 f3 mov    dword ptr [0xb052f352d2e9e77a], eax
+     52 b0
++0x05c9   77 b1                jnbe   0x000000000fffffb3
++0x05cb   da b2 ec 6d ae 91    fidiv  dword ptr [rdx-0x6e519214]
++0x05d1   d0 c7                rol    bh, 0x01
++0x05d3   25 88 ad c6 47       and    eax, 0x47c6ad88
++0x05d8   fa                   cli    
++0x05d9   3e e7 66             out    0x66, eax
++0x05dc   77 12                jnbe   0x0000000010000014
++0x05de   a2 1c a3 4d d2 7e 8a mov    byte ptr [0x73068a7ed24da31c], al
+     06 73
++0x05e7   0d 77 39 13 d9       or     eax, 0xd9133977
++0x05ec   93                   xchg   ebx, eax
++0x05ed   d7                   xlatb  
++0x05ee   99                   cdq    
++0x05ef   1f...??              <INVALID>
++0x05f0   9e                   sahf   
++0x05f1   bb 10 59 34 ff       mov    ebx, 0xff345910
++0x05f6   a0 c1 f4 c8 5e 1f a6 mov    al, byte ptr [0x83e8a61f5ec8f4c1]
+     e8 83
++0x05ff   27...??              <INVALID>
++0x0600   3f...??              <INVALID>
++0x0601   79 b6                jns    0x000000000fffffb8
++0x0603   b3 7a                mov    bl, 0x7a
++0x0605   13 f0                adc    esi, eax
++0x0607   ea f4 24 3b 10 14 ce...??  <INVALID>
++0x0608   f4                   hlt    
++0x0609   24 3b                and    al, 0x3b
++0x060b   10 14 ce             adc    byte ptr [rsi+rcx*8], dl
++0x060e   41 f3 dd 58 23       fstp   qword ptr [rax+0x23]
++0x0613   66 ef                out    ax, dx
++0x0615   71 84                jno    0x000000000fffff86
++0x0617   6b e5 fd             imul   esp, ebp, 0xfd
++0x061a   08 a5 a2 5b 44 ae    or     byte ptr [rbp-0x51bba45e], ah
++0x0620   e3 b2                jrcxz  0x000000000fffffb4
++0x0622   3a fc                cmp    bh, ah
++0x0624   ed                   in     eax, dx
++0x0625   3f...??              <INVALID>
++0x0626   90                   nop    
++0x0627   1c d8                sbb    al, 0xd8
++0x0629   9a 71 c2 d8 79 b6 43...??  <INVALID>
++0x062a   71 c2                jno    0x000000000fffffc4
++0x062c   d8 79 b6             fdivr  dword ptr [rcx-0x4a]
++0x062f   43 65 47 44 9d       popfq  
++0x0634   2f...??              <INVALID>
++0x0635   5f                   pop    rdi
++0x0636   e4 a8                in     al, 0xa8
++0x0638   93                   xchg   ebx, eax
++0x0639   b6 9f                mov    dh, 0x9f
++0x063b   d9 d5...??           <INVALID>
++0x063c   d5 ef...??           <INVALID>
++0x063d   ef                   out    eax, dx
++0x063e   90                   nop    
++0x063f   a9 1d 74 d9 6e       test   eax, 0x6ed9741d
++0x0644   a1 10 49 34 53 f7 c6 mov    eax, dword ptr [0x2761c6f753344910]
+     61 27
++0x064d   61...??              <INVALID>
++0x064e   ef                   out    eax, dx
++0x064f   13 9a a9 48 d9 f2    adc    ebx, dword ptr [rdx-0x0d26b757]
++0x0655   2c 46                sub    al, 0x46
++0x0657   b8 41 c4 fe 83       mov    eax, 0x83fec441
++0x065c   b1 9d                mov    cl, 0x9d
++0x065e   af                   scasd  
++0x065f   39 c6                cmp    esi, eax
++0x0661   29 47 9a             sub    dword ptr [rdi-0x66], eax
++0x0664   4d 82 fb 61...??     <INVALID>
++0x0665   82 fb 61...??        <INVALID>
++0x0666   fb                   sti    
++0x0667   61...??              <INVALID>
++0x0668   a0 f3 eb e6 25 41 60 mov    al, byte ptr [0xcca5604125e6ebf3]
+     a5 cc
++0x0671   a8 c0                test   al, 0xc0
++0x0673   b1 39                mov    cl, 0x39
++0x0675   6d                   insd   
++0x0676   eb af                jmp    0x000000000fffffb1
++0x0678   05 5d 29 66 da       add    eax, 0xda66295d
++0x067d   39 7f 22             cmp    dword ptr [rdi+0x22], edi
++0x0680   50                   push   rax
++0x0681   2c 5e                sub    al, 0x5e
++0x0683   1b 08                sbb    ecx, dword ptr [rax]
++0x0685   91                   xchg   ecx, eax
++0x0686   4d e7 e2             out    0xe2, eax
++0x0689   5b                   pop    rbx
++0x068a   7a c7                jp     0x000000000fffffc9
++0x068c   2e 26 ea 3b aa 79 fa...??  <INVALID>
+     79 2c
++0x068d   26 ea 3b aa 79 fa 79...??  <INVALID>
+     2c
++0x068e   ea 3b aa 79 fa 79 2c...??  <INVALID>
++0x068f   3b aa 79 fa 79 2c    cmp    ebp, dword ptr [rdx+0x2c79fa79]
++0x0695   d0 a2 25 e3 66 b9    shl    byte ptr [rdx-0x46991cdb], 0x01
++0x069b   ec                   in     al
++0x069c   ad                   lodsd  
++0x069d   6c                   insb   
++0x069e   05 0b b9 2e 4e       add    eax, 0x4e2eb90b
++0x06a3   3c fb                cmp    al, 0xfb
++0x06a5   2d d8 a8 f2 52       sub    eax, 0x52f2a8d8
++0x06aa   8b 4e 08             mov    ecx, dword ptr [rsi+0x08]
++0x06ad   23 88 52 f4 f8 b9    and    ecx, dword ptr [rax-0x46070bae]
++0x06b3   d8 d7                fcom   st0, st7
++0x06b5   f1                   int1   
++0x06b6   be 08 57 0d 70       mov    esi, 0x700d5708
++0x06bb   bd a1 58 8e 67       mov    ebp, 0x678e58a1
++0x06c0   13 21                adc    esp, dword ptr [rcx]
++0x06c2   83 10 bf             adc    dword ptr [rax], 0xbf
++0x06c5   83 24 94 c6          and    dword ptr [rsp+rdx*4], 0xc6
++0x06c9   53                   push   rbx
++0x06ca   2d af a2 1f 9e       sub    eax, 0x9e1fa2af
++0x06cf   b9 ae 29 df dc       mov    ecx, 0xdcdf29ae
++0x06d4   f8                   clc    
++0x06d5   3a c9                cmp    cl, cl
++0x06d7   f5                   cmc    
++0x06d8   c8 3f 4f f8          enter  0x4f3f, 0xf8
++0x06dc   82 92 71 8e 4b eb b6...??  <INVALID>
++0x06dd   92                   xchg   edx, eax
++0x06de   71 8e                jno    0x000000000fffff90
++0x06e0   4b eb b6             jmp    0x000000000fffffb9
++0x06e3   e2 c8                loop   0x000000000fffffca
++0x06e5   74 87                jz     0x000000000fffff89
++0x06e7   d4 1b...??           <INVALID>
++0x06e8   1b 53 e8             sbb    edx, dword ptr [rbx-0x18]
++0x06eb   17...??              <INVALID>
++0x06ec   2b 84 6e 62 a8 70 c8 sub    eax, dword ptr [rsi+rbp*2-0x378f579e]
++0x06f3   bc ab 1a e7 d4       mov    esp, 0xd4e71aab
++0x06f8   0a 0d 6c 3f 1a 1b    or     cl, <rel> byte ptr [0x000000002b1a3f72]
++0x06fe   e5 7c                in     eax, 0x7c
++0x0700   1d 07 4d b9 b9       sbb    eax, 0xb9b94d07
++0x0705   6d                   insd   
++0x0706   b3 8a                mov    bl, 0x8a
++0x0708   6a f9                push   0xf9
++0x070a   1b 88 9e 66 31 1b    sbb    ecx, dword ptr [rax+0x1b31669e]
++0x0710   be 3d 5b de d1       mov    esi, 0xd1de5b3d
++0x0715   78 e5                js     0x000000000fffffe7
++0x0717   1d d5 e0 ff 06       sbb    eax, 0x06ffe0d5
++0x071c   7d a8                jnl    0x000000000fffffaa
++0x071e   6c                   insb   
++0x071f   45 c2 a6 0d          ret    0x0da6
++0x0723   bc 3b ba e9 39       mov    esp, 0x39e9ba3b
++0x0728   91                   xchg   ecx, eax
++0x0729   7e 68                jle    0x000000001000006a
++0x072b   ba e3 d8 0b 84       mov    edx, 0x840bd8e3
++0x0730   b6 d1                mov    dh, 0xd1
++0x0732   c4 32...??           <INVALID>
++0x0733   32 84 c8 f1 dd 05 fa xor    al, byte ptr [rax+rcx*8-0x05fa220f]
++0x073a   11 c3                adc    ebx, eax
++0x073c   7e 66                jle    0x0000000010000068
++0x073e   73 18                jnb    0x000000001000001a
++0x0740   5e                   pop    rsi
++0x0741   2f...??              <INVALID>
++0x0742   f4                   hlt    
++0x0743   76 ec                jbe    0x000000000fffffee
++0x0745   7d 47                jnl    0x0000000010000049
++0x0747   c3                   ret    
++0x0748   3e ec                in     al
++0x074a   ad                   lodsd  
++0x074b   77 01                jnbe   0x0000000010000003
++0x074d   5b                   pop    rbx
++0x074e   af                   scasd  
++0x074f   fc                   cld    
++0x0750   b7 a0                mov    bh, 0xa0
++0x0752   12 20                adc    ah, byte ptr [rax]
++0x0754   1c 53                sbb    al, 0x53
++0x0756   69 81 8a 93 0e 9b dc imul   eax, dword ptr [rcx-0x64f16c76], 0x53744fdc
+     4f 74 53
++0x0760   a6                   cmpsb  
++0x0761   87 aa 82 50 91 64    xchg   dword ptr [rdx+0x64915082], ebp
++0x0767   43 f6 6a a3          imul   ax, byte ptr [r10-0x5d]
++0x076b   9f                   lahf   
++0x076c   05 de ad 86 60       add    eax, 0x6086adde
++0x0771   0c fb                or     al, 0xfb
++0x0773   2e 0d 7b 6d 92 a0    or     eax, 0xa0926d7b
++0x0779   ed                   in     eax, dx
++0x077a   e4 36                in     al, 0x36
++0x077c   76 bf                jbe    0x000000000fffffc1
++0x077e   f1                   int1   
++0x077f   d7                   xlatb  
++0x0780   da 36                fidiv  dword ptr [rsi]
++0x0782   88 17                mov    byte ptr [rdi], dl
++0x0784   fe c0                inc    al
++0x0786   89 a1 53 62 47 98    mov    dword ptr [rcx-0x67b89dad], esp
++0x078c   d7                   xlatb  
++0x078d   ba 50 ea ab 54       mov    edx, 0x54abea50
++0x0792   70 1a                jo     0x000000001000001c
++0x0794   02 37                add    dh, byte ptr [rdi]
++0x0796   dc 21                fsub   qword ptr [rcx]
++0x0798   31 6c 0f 48          xor    dword ptr [rdi+rcx+0x48], ebp
++0x079c   9f                   lahf   
++0x079d   63 6a b5             movsxd ebp, dword ptr [rdx-0x4b]
++0x07a0   c6 30 f8...??        <INVALID>
++0x07a1   30 f8                xor    al, bh
++0x07a3   bc 34 a8 bb 9c       mov    esp, 0x9cbba834
++0x07a8   fd                   std    
++0x07a9   e3 f5                jrcxz  0x000000000ffffff7
++0x07ab   4f 03 6f c0          add    r13, qword ptr [r15-0x40]
++0x07af   ba 22 a5 17 97       mov    edx, 0x9717a522
++0x07b4   28 2c bc             sub    byte ptr [rsp+rdi*4], ch
++0x07b7   9f                   lahf   
++0x07b8   bc 18 4f e9 47       mov    esp, 0x47e94f18
++0x07bd   3e 87 36             xchg   dword ptr [ds:rsi], esi
++0x07c0   9c                   pushfq 
++0x07c1   55                   push   rbp
++0x07c2   3b 17                cmp    edx, dword ptr [rdi]
++0x07c4   74 6e                jz     0x0000000010000070
++0x07c6   94                   xchg   esp, eax
++0x07c7   49 d6...??           <INVALID>
++0x07c8   d6...??              <INVALID>
++0x07c9   51                   push   rcx
++0x07ca   d5 f9...??           <INVALID>
++0x07cb   f9                   stc    
++0x07cc   fe 82 f5 8b fc 53    inc    byte ptr [rdx+0x53fc8bf5]
++0x07d2   1a a6 65 9e 06 0e    sbb    ah, byte ptr [rsi+0x0e069e65]
++0x07d8   d3 ea                shr    edx, cl
++0x07da   ec                   in     al
++0x07db   17...??              <INVALID>
++0x07dc   94                   xchg   esp, eax
++0x07dd   21 0a                and    dword ptr [rdx], ecx
++0x07df   49 9b                fwait  
++0x07e1   27...??              <INVALID>
++0x07e2   7e 95                jle    0x000000000fffff97
++0x07e4   1e...??              <INVALID>
++0x07e5   88 61 9b             mov    byte ptr [rcx-0x65], ah
++0x07e8   40 07...??           <INVALID>
++0x07e9   07...??              <INVALID>
++0x07ea   7b 08                jnp    0x000000001000000a
++0x07ec   e9 36 d4 1e 73       jmp    0x00000000831ed43b
++0x07f1   79 a3                jns    0x000000000fffffa5
++0x07f3   b1 ba                mov    cl, 0xba
++0x07f5   35 76 d3 13 57       xor    eax, 0x5713d376
++0x07fa   7a 7f                jp     0x0000000010000081
++0x07fc   3b 40 d6             cmp    eax, dword ptr [rax-0x2a]
++0x07ff   81 37 cd b7 56 be    xor    dword ptr [rdi], 0xbe56b7cd
++0x0805   79 08                jns    0x000000001000000a
++0x0807   97                   xchg   edi, eax
++0x0808   ec                   in     al
++0x0809   c2 88 fc             ret    0xfc88
++0x080c   94                   xchg   esp, eax
++0x080d   6f                   outsd  
++0x080e   b9 e0 9f 62 cb       mov    ecx, 0xcb629fe0
++0x0813   6f                   outsd  
++0x0814   9d                   popfq  
++0x0815   42 54                push   rsp
++0x0817   91                   xchg   ecx, eax
++0x0818   b9 40 b7 f2 3f       mov    ecx, 0x3ff2b740
++0x081d   15 6a 7f cf b4       adc    eax, 0xb4cf7f6a
++0x0822   63 9e 7b 50 bd bb    movsxd ebx, dword ptr [rsi-0x4442af85]
++0x0828   ff 1c 3d e9 61 a4 63 call   
++0x082f   8b 45 4d             mov    eax, dword ptr [rbp+0x4d]
++0x0832   56                   push   rsi
++0x0833   60...??              <INVALID>
++0x0834   0b 45 ec             or     eax, dword ptr [rbp-0x14]
++0x0837   56                   push   rsi
++0x0838   1d ae c0 8f 3e       sbb    eax, 0x3e8fc0ae
++0x083d   48 22 47 c5          and    al, byte ptr [rdi-0x3b]
++0x0841   51                   push   rcx
++0x0842   37...??              <INVALID>
++0x0843   87 61 6b             xchg   dword ptr [rcx+0x6b], esp
++0x0846   da cc                fcmove st0, st4
++0x0848   ea 95 93 ff 34 23 28...??  <INVALID>
++0x0849   95                   xchg   ebp, eax
++0x084a   93                   xchg   ebx, eax
++0x084b   ff 34 23             push   qword ptr [rbx]
++0x084e   28 10                sub    byte ptr [rax], dl
++0x0850   f8                   clc    
++0x0851   35 e2 2e dc 00       xor    eax, 0x00dc2ee2
++0x0856   01 83 86 d7 fc 55    add    dword ptr [rbx+0x55fcd786], eax
++0x085c   7f 30                jnle   0x0000000010000032
++0x085e   e9 96 c4 dd 88       jmp    0xffffffff98ddc49b
++0x0863   5d                   pop    rbp
++0x0864   fc                   cld    
++0x0865   ec                   in     al
++0x0866   23 8e fd f6 14 f3    and    ecx, dword ptr [rsi-0x0ceb0903]
++0x086c   30 46 08             xor    byte ptr [rsi+0x08], al
++0x086f   dd 58 92             fstp   qword ptr [rax-0x6e]
++0x0872   77 8f                jnbe   0x000000000fffff91
++0x0874   82 80 1f 8a ef 6b 97...??  <INVALID>
++0x0875   80 1f 8a             sbb    byte ptr [rdi], 0x8a
++0x0878   ef                   out    eax, dx
++0x0879   6b 97 6e 10 6d b5 59 imul   edx, dword ptr [rdi-0x4a92ef92], 0x59
++0x0880   2b 35 e5 f4 7f 2d    sub    esi, <rel> dword ptr [0x000000003d7ff4eb]
++0x0886   f9                   stc    
++0x0887   8c 1b                mov    dword ptr [rbx], ds
++0x0889   81 0c dc 42 64 ee 63 or     dword ptr [rsp+rbx*8], 0x63ee6442
++0x0890   06...??              <INVALID>
++0x0891   50                   push   rax
++0x0892   2f...??              <INVALID>
++0x0893   de e5                fsubrp st5, st0
++0x0895   7c 72                jl     0x0000000010000074
++0x0897   e2 99                loop   0x000000000fffff9b
++0x0899   bc a6 b4 be 34       mov    esp, 0x34beb4a6
++0x089e   15 38 58 74 c6       adc    eax, 0xc6745838
++0x08a3   e6 71                out    0x71, al
++0x08a5   3a ea                cmp    ch, dl
++0x08a7   2b 8a 27 91 51 5c    sub    ecx, dword ptr [rdx+0x5c519127]
++0x08ad   19 8f c1 5c ca 33    sbb    dword ptr [rdi+0x33ca5cc1], ecx
++0x08b3   d6...??              <INVALID>
++0x08b4   0b 42 00             or     eax, dword ptr [rdx+0x00]
++0x08b7   2f...??              <INVALID>
++0x08b8   a5                   movsd  
++0x08b9   f5                   cmc    
++0x08ba   a6                   cmpsb  
++0x08bb   00 6c 83 11          add    byte ptr [rbx+rax*4+0x11], ch
++0x08bf   74 87                jz     0x000000000fffff89
++0x08c1   21 6d d3             and    dword ptr [rbp-0x2d], ebp
++0x08c4   48 2e cb             retf   
++0x08c7   18 11                sbb    byte ptr [rcx], dl
++0x08c9   b6 35                mov    dh, 0x35
++0x08cb   d3 57 e1             rcl    dword ptr [rdi-0x1f], cl
++0x08ce   08 74 60 c8          or     byte ptr [rax-0x38], dh
++0x08d2   1f...??              <INVALID>
++0x08d3   21 78 d1             and    dword ptr [rax-0x2f], edi
++0x08d6   b1 c8                mov    cl, 0xc8
++0x08d8   e3 77                jrcxz  0x0000000010000079
++0x08da   25 e6 82 66 e2       and    eax, 0xe26682e6
++0x08df   2a c6                sub    al, dh
++0x08e1   2a 38                sub    bh, byte ptr [rax]
++0x08e3   16...??              <INVALID>
++0x08e4   1a 92 d3 50 55 f5    sbb    dl, byte ptr [rdx-0x0aaaaf2d]
++0x08ea   76 4e                jbe    0x0000000010000050
++0x08ec   c9                   leave  
++0x08ed   13 13                adc    edx, dword ptr [rbx]
++0x08ef   b1 bb                mov    cl, 0xbb
++0x08f1   53                   push   rbx
++0x08f2   23 44 e0 0a          and    eax, dword ptr [rax+0x0a]
++0x08f6   cf                   iretd  
++0x08f7   6f                   outsd  
++0x08f8   31 96 e8 d0 f4 10    xor    dword ptr [rsi+0x10f4d0e8], edx
++0x08fe   3d 99 d7 f5 1c       cmp    eax, 0x1cf5d799
++0x0903   c5 40 e2...??        <INVALID>
++0x0904   40 e2 f2             loop   0x000000000ffffff5
++0x0907   76 2e                jbe    0x0000000010000030
++0x0909   4f 23 eb             and    r13, r11
++0x090c   1a 80 12 0b 88 2f    sbb    al, byte ptr [rax+0x2f880b12]
++0x0912   5f                   pop    rdi
++0x0913   90                   nop    
++0x0914   94                   xchg   esp, eax
++0x0915   fb                   sti    
++0x0916   0f...??              <INVALID>
++0x0917   3b 63 a4             cmp    esp, dword ptr [rbx-0x5c]
++0x091a   e1 77                loope  0x0000000010000079
++0x091c   1f...??              <INVALID>
++0x091d   da 31                fidiv  dword ptr [rcx]
++0x091f   a0 1b 07 b8 bd 9a 7b mov    al, byte ptr [0x58d87b9abdb8071b]
+     d8 58
++0x0928   0e...??              <INVALID>
++0x0929   5a                   pop    rdx
++0x092a   2d 5c 3f 17 da       sub    eax, 0xda173f5c
++0x092f   cb                   retf   
++0x0930   25 a9 4a 67 71       and    eax, 0x71674aa9
++0x0935   a1 12 ed 0b 31 32 5c mov    eax, dword ptr [0x9bfd5c32310bed12]
+     fd 9b
++0x093e   37...??              <INVALID>
++0x093f   d7                   xlatb  
++0x0940   40 28 38             sub    byte ptr [rax], dil
++0x0943   75 8e                jnz    0x000000000fffff90
++0x0945   ea 48 23 be f1 fe a9...??  <INVALID>
++0x0946   48 23 be f1 fe a9 a4 and    rdi, qword ptr [rsi-0x5b56010f]
++0x094d   36 0b 58 2d          or     ebx, dword ptr [ss:rax+0x2d]
++0x0951   13 eb                adc    ebp, ebx
++0x0953   11 8e 35 d2 1f 95    adc    dword ptr [rsi-0x6ae02dcb], ecx
++0x0959   89 46 3f             mov    dword ptr [rsi+0x3f], eax
++0x095c   77 f9                jnbe   0x000000000ffffffb
++0x095e   2b e2                sub    esp, edx
++0x0960   83 37 a7             xor    dword ptr [rdi], 0xa7
++0x0963   e2 d1                loop   0x000000000fffffd3
++0x0965   fd                   std    
++0x0966   8a b9 52 91 b4 46    mov    bh, byte ptr [rcx+0x46b49152]
++0x096c   ec                   in     al
++0x096d   b2 45                mov    dl, 0x45
++0x096f   92                   xchg   edx, eax
++0x0970   6a c5                push   0xc5
++0x0972   9b                   fwait  
++0x0973   9c                   pushfq 
++0x0974   fb                   sti    
++0x0975   ec                   in     al
++0x0976   7d 89                jnl    0x000000000fffff8b
++0x0978   76 44                jbe    0x0000000010000046
++0x097a   92                   xchg   edx, eax
++0x097b   47 92                xchg   r10d, eax
++0x097d   38 cd                cmp    ch, cl
++0x097f   e2 52                loop   0x0000000010000054
++0x0981   49 ad                lodsq  
++0x0983   5b                   pop    rbx
++0x0984   86 a7 60 93 48 ab    xchg   byte ptr [rdi-0x54b76ca0], ah
++0x098a   78 e2                js     0x000000000fffffe4
++0x098c   b9 04 a6 dc af       mov    ecx, 0xafdca604
++0x0991   ab                   stosd  
++0x0992   ce...??              <INVALID>
++0x0993   13 b1 28 d8 17 ae    adc    esi, dword ptr [rcx-0x51e827d8]
++0x0999   d0 5d 3a             rcr    byte ptr [rbp+0x3a], 0x01
++0x099c   29 01                sub    dword ptr [rcx], eax
++0x099e   bd cb 42 d3 ff       mov    ebp, 0xffd342cb
++0x09a3   22 e5                and    ah, ch
++0x09a5   54                   push   rsp
++0x09a6   26 e1 0b             loope  0x000000001000000e
++0x09a9   4d 32 d3             xor    r10l, r11l
++0x09ac   ee                   out    al, dx
++0x09ad   89 37                mov    dword ptr [rdi], esi
++0x09af   7c d1                jl     0x000000000fffffd3
++0x09b1   c7 eb dd 45 84 6d...??  <INVALID>
++0x09b2   eb dd                jmp    0x000000000fffffdf
++0x09b4   45 84 6d 9b          test   byte ptr [r13-0x65], r13l
++0x09b8   4f 57                push   r15
++0x09ba   13 5a d9             adc    ebx, dword ptr [rdx-0x27]
++0x09bd   d6...??              <INVALID>
++0x09be   83 31 5c             xor    dword ptr [rcx], 0x5c
++0x09c1   fe 4e c1             dec    byte ptr [rsi-0x3f]
++0x09c4   09 02                or     dword ptr [rdx], eax
++0x09c6   92                   xchg   edx, eax
++0x09c7   ce...??              <INVALID>
++0x09c8   6f                   outsd  
++0x09c9   31 97 77 4b 31 84    xor    dword ptr [rdi-0x7bceb489], edx
++0x09cf   8d 54 69 f3          lea    edx, [rcx+rbp*2-0x0d]
++0x09d3   ba df bc 94 be       mov    edx, 0xbe94bcdf
++0x09d8   b9 7e 36 14 13       mov    ecx, 0x1314367e
++0x09dd   16...??              <INVALID>
++0x09de   f0 c0 33 56...??     <INVALID>
++0x09df   c0 33 56             shl    byte ptr [rbx], 0x56
++0x09e2   8c 96 6c 4c 4b 19    mov    dword ptr [rsi+0x194b4c6c], ss
++0x09e8   e6 bb                out    0xbb, al
++0x09ea   29 c1                sub    ecx, eax
++0x09ec   0c af                or     al, 0xaf
++0x09ee   71 29                jno    0x000000001000002b
++0x09f0   2f...??              <INVALID>
++0x09f1   4e 9d                popfq  
++0x09f3   31 a1 f6 88 09 2d    xor    dword ptr [rcx+0x2d0988f6], esp
++0x09f9   9b                   fwait  
++0x09fa   a1 91 98 bf bf 5c 61 mov    eax, dword ptr [0x9002615cbfbf9891]
+     02 90
++0x0a03   02 5d 7c             add    bl, byte ptr [rbp+0x7c]
++0x0a06   c1 d7 0e             rcl    edi, 0x0e
++0x0a09   cf                   iretd  
++0x0a0a   fd                   std    
++0x0a0b   cc                   int3   
++0x0a0c   ab                   stosd  
++0x0a0d   d1 72 cc             shl    dword ptr [rdx-0x34], 0x01
++0x0a10   38 f2                cmp    dl, dh
++0x0a12   71 ed                jno    0x000000000fffffef
++0x0a14   12 92 71 ac 71 a2    adc    dl, byte ptr [rdx-0x5d8e538f]
++0x0a1a   d6...??              <INVALID>
++0x0a1b   51                   push   rcx
++0x0a1c   cc                   int3   
++0x0a1d   ea 2e 9c 39 dd 7e be...??  <INVALID>
++0x0a1e   2e 9c                pushfq 
++0x0a20   39 dd                cmp    ebp, ebx
++0x0a22   7e be                jle    0x000000000fffffc0
++0x0a24   82 38 67...??        <INVALID>
++0x0a25   38 67 5f             cmp    byte ptr [rdi+0x5f], ah
++0x0a28   7e 6e                jle    0x0000000010000070
++0x0a2a   e4 2f                in     al, 0x2f
++0x0a2c   11 03                adc    dword ptr [rbx], eax
++0x0a2e   f7 fd                idiv   edx, eax, ebp
++0x0a30   06...??              <INVALID>
++0x0a31   8d 6a 92             lea    ebp, [rdx-0x6e]
++0x0a34   45 5e                pop    r14
++0x0a36   31 25 e4 8c ee 1a    xor    <rel> dword ptr [0x000000002aee8cea], esp
++0x0a3c   74 67                jz     0x0000000010000069
++0x0a3e   ce...??              <INVALID>
++0x0a3f   9e                   sahf   
++0x0a40   c1 66 47 f0          shl    dword ptr [rsi+0x47], 0xf0
++0x0a44   a5                   movsd  
++0x0a45   45 d1 81 6a 2a 14 1b rol    dword ptr [r9+0x1b142a6a], 0x01
++0x0a4c   cc                   int3   
++0x0a4d   b1 8a                mov    cl, 0x8a
++0x0a4f   f0 f2 3b c5...??     <INVALID>
++0x0a50   f2 3b c5             cmp    eax, ebp
++0x0a53   88 04 de             mov    byte ptr [rsi+rbx*8], al
++0x0a56   04 91                add    al, 0x91
++0x0a58   63 17                movsxd edx, dword ptr [rdi]
++0x0a5a   75 10                jnz    0x0000000010000012
++0x0a5c   0e...??              <INVALID>
++0x0a5d   13 4f 7a             adc    ecx, dword ptr [rdi+0x7a]
++0x0a60   7a 26                jp     0x0000000010000028
++0x0a62   37...??              <INVALID>
++0x0a63   53                   push   rbx
++0x0a64   88 7d ce             mov    byte ptr [rbp-0x32], bh
++0x0a67   0c 23                or     al, 0x23
++0x0a69   cf                   iretd  
++0x0a6a   42 0f d8 fc          psubusb mm7, mm4
++0x0a6e   e3 3f                jrcxz  0x0000000010000041
++0x0a70   2f...??              <INVALID>
++0x0a71   2f...??              <INVALID>
++0x0a72   93                   xchg   ebx, eax
++0x0a73   b2 8e                mov    dl, 0x8e
++0x0a75   a8 cd                test   al, 0xcd
++0x0a77   c6 9b 14 5b 1d 0b 66...??  <INVALID>
++0x0a78   9b                   fwait  
++0x0a79   14 5b                adc    al, 0x5b
++0x0a7b   1d 0b 66 08 07       sbb    eax, 0x0708660b
++0x0a80   f2 c0 e0 f5          shl    al, 0xf5
++0x0a84   07...??              <INVALID>
++0x0a85   d6...??              <INVALID>
++0x0a86   93                   xchg   ebx, eax
++0x0a87   69 03 a2 9e d4 59    imul   eax, dword ptr [rbx], 0x59d49ea2
++0x0a8d   91                   xchg   ecx, eax
++0x0a8e   49 62 c7...??        <INVALID>
++0x0a8f   62 c7...??           <INVALID>
++0x0a90   c7 10 e0 ea 72 4f...??  <INVALID>
++0x0a91   10 e0                adc    al, ah
++0x0a93   ea 72 4f 23 e4 e8 e3...??  <INVALID>
++0x0a94   72 4f                jb     0x0000000010000051
++0x0a96   23 e4                and    esp, esp
++0x0a98   e8 e3 85 28 dd       call   0xffffffffed2885e8
++0x0a9d   79 a1                jns    0x000000000fffffa3
++0x0a9f   82 b0 fc 0e 75 ef b9...??  <INVALID>
++0x0aa0   b0 fc                mov    al, 0xfc
++0x0aa2   0e...??              <INVALID>
++0x0aa3   75 ef                jnz    0x000000000ffffff1
++0x0aa5   b9 a2 b0 f1 ba       mov    ecx, 0xbaf1b0a2
++0x0aaa   39 94 fd 61 df 34 a9 cmp    dword ptr [rbp+rdi*8-0x56cb209f], edx
++0x0ab1   4b 2e a4             movsb  cs
++0x0ab4   ec                   in     al
++0x0ab5   b9 92 69 e4 8e       mov    ecx, 0x8ee46992
++0x0aba   44 ba 0b 61 3d f7    mov    edx, 0xf73d610b
++0x0ac0   85 1c 1b             test   dword ptr [rbx+rbx], ebx
++0x0ac3   6d                   insd   
++0x0ac4   b5 66                mov    ch, 0x66
++0x0ac6   1b 82 07 31 39 9a    sbb    eax, dword ptr [rdx-0x65c6cef9]
++0x0acc   e8 c8 66 74 f9       call   0x00000000097466cd
++0x0ad1   0f bb 39             btc    dword ptr [rcx], edi
++0x0ad4   17...??              <INVALID>
++0x0ad5   7c 49                jl     0x000000001000004b
++0x0ad7   e3 56                jrcxz  0x0000000010000058
++0x0ad9   af                   scasd  
++0x0ada   a6                   cmpsb  
++0x0adb   9f                   lahf   
++0x0adc   1e...??              <INVALID>
++0x0add   04 97                add    al, 0x97
++0x0adf   9c                   pushfq 
++0x0ae0   d6...??              <INVALID>
++0x0ae1   d5 06...??           <INVALID>
++0x0ae2   06...??              <INVALID>
++0x0ae3   61...??              <INVALID>
++0x0ae4   d4 72...??           <INVALID>
++0x0ae5   72 c8                jb     0x000000000fffffca
++0x0ae7   a7                   cmpsd  
++0x0ae8   fb                   sti    
++0x0ae9   60...??              <INVALID>
++0x0aea   ef                   out    eax, dx
++0x0aeb   89 42 43             mov    dword ptr [rdx+0x43], eax
++0x0aee   43 35 48 4a 60 00    xor    eax, 0x00604a48
++0x0af4   dc 75 ed             fdiv   qword ptr [rbp-0x13]
++0x0af7   99                   cdq    
++0x0af8   25 e6 52 04 e7       and    eax, 0xe70452e6
++0x0afd   9f                   lahf   
++0x0afe   1e...??              <INVALID>
++0x0aff   74 00                jz     0x0000000010000002
++0x0b01   53                   push   rbx
++0x0b02   0b 63 ea             or     esp, dword ptr [rbx-0x16]
++0x0b05   df 37                fbstp  tbyte ptr [rdi]
++0x0b07   81 b4 36 7f 0d 92 5a xor    dword ptr [rsi+rsi+0x5a920d7f], 0xc38b1a72
+     72 1a 8b c3
++0x0b12   c5 46 d1...??        <INVALID>
++0x0b13   46 d1 40 f8          rol    dword ptr [rax-0x08], 0x01
++0x0b17   68 ab 87 f7 b5       push   0xb5f787ab
++0x0b1c   39 2d fa 95 7e bb    cmp    <rel> dword ptr [0xffffffffcb7e9600], ebp
++0x0b22   4d 85 a7 3c b1 8a 74 test   qword ptr [r15+0x748ab13c], r12
++0x0b29   1e...??              <INVALID>
++0x0b2a   22 16                and    dl, byte ptr [rsi]
++0x0b2c   86 bc c0 ed 52 68 36 xchg   byte ptr [rax+rax*8+0x366852ed], bh
++0x0b33   58                   pop    rax
++0x0b34   ed                   in     eax, dx
++0x0b35   aa                   stosb  
++0x0b36   9d                   popfq  
++0x0b37   fa                   cli    
++0x0b38   43 ea 28 a1 49 15 12...??  <INVALID>
+     d9
++0x0b39   ea 28 a1 49 15 12 d9...??  <INVALID>
++0x0b3a   28 a1 49 15 12 d9    sub    byte ptr [rcx-0x26edeab7], ah
++0x0b40   da 22                fisub  dword ptr [rdx]
++0x0b42   5f                   pop    rdi
++0x0b43   9e                   sahf   
++0x0b44   fd                   std    
++0x0b45   7b b6                jnp    0x000000000fffffb8
++0x0b47   68 b7 0d 7c 65       push   0x657c0db7
++0x0b4c   57                   push   rdi
++0x0b4d   27...??              <INVALID>
++0x0b4e   cc                   int3   
++0x0b4f   b5 b4                mov    ch, 0xb4
++0x0b51   15 47 47 74 fc       adc    eax, 0xfc744747
++0x0b56   b6 9b                mov    dh, 0x9b
++0x0b58   4c 79 72             jns    0x0000000010000075
++0x0b5b   fd                   std    
++0x0b5c   30 c6                xor    dh, al
++0x0b5e   52                   push   rdx
++0x0b5f   bd f4 4f 13 95       mov    ebp, 0x95134ff4
++0x0b64   32 24 48             xor    ah, byte ptr [rax+rcx*2]
++0x0b67   7f 42                jnle   0x0000000010000044
++0x0b69   16...??              <INVALID>
++0x0b6a   85 8a 94 36 90 ea    test   dword ptr [rdx-0x156fc96c], ecx
++0x0b70   f3 ac                rep lodsb 
++0x0b72   f2 b9 9e 9c 3b 79    mov    ecx, 0x793b9c9e
++0x0b78   f9                   stc    
++0x0b79   ab                   stosd  
++0x0b7a   82 d6 3e...??        <INVALID>
++0x0b7b   d6...??              <INVALID>
++0x0b7c   3e 1c 04             sbb    al, 0x04
++0x0b7f   a2 94 f9 c2 cb 72 a6 mov    byte ptr [0x171fa672cbc2f994], al
+     1f 17
++0x0b88   c1 3d 47 6b 38 44 be sar    <rel> dword ptr [0x0000000054386b4e], 0xbe
++0x0b8f   22 99 6b c6 6f f8    and    bl, byte ptr [rcx-0x07903995]
++0x0b95   b5 93                mov    ch, 0x93
++0x0b97   5a                   pop    rdx
++0x0b98   cd 82                int    0x82
++0x0b9a   97                   xchg   edi, eax
++0x0b9b   8a 5b aa             mov    bl, byte ptr [rbx-0x56]
++0x0b9e   aa                   stosb  
++0x0b9f   9a 46 b4 4d 0b 82 4f...??  <INVALID>
++0x0ba0   46 b4 4d             mov    spl, 0x4d
++0x0ba3   0b 82 4f 0a 6d 8a    or     eax, dword ptr [rdx-0x7592f5b1]
++0x0ba9   78 9e                js     0x000000000fffffa0
++0x0bab   09 8c 30 4d ae 1a 3f or     dword ptr [rax+rsi+0x3f1aae4d], ecx
++0x0bb2   7a a7                jp     0x000000000fffffa9
++0x0bb4   a5                   movsd  
++0x0bb5   6e                   outsb  
++0x0bb6   90                   nop    
++0x0bb7   2b 0a                sub    ecx, dword ptr [rdx]
++0x0bb9   45 ee                out    al, dx
++0x0bbb   2b 02                sub    eax, dword ptr [rdx]
++0x0bbd   f9                   stc    
++0x0bbe   5f                   pop    rdi
++0x0bbf   14 bf                adc    al, 0xbf
++0x0bc1   f2 a1 35 0f 5f d4 75 mov    eax, dword ptr [0xd8ee6375d45f0f35]
+     63 ee d8
++0x0bcb   6b 1d bc 72 6e 69 e0 imul   ebx, <rel> dword ptr [0x00000000796e72c3], 0xe0
++0x0bd2   78 c9                js     0x000000000fffffcb
++0x0bd4   09 28                or     dword ptr [rax], ebp
++0x0bd6   07...??              <INVALID>
++0x0bd7   f0 20 6b 55          lock and    byte ptr [rbx+0x55], ch
++0x0bdb   a8 ca                test   al, 0xca
++0x0bdd   a7                   cmpsd  
++0x0bde   34 db                xor    al, 0xdb
++0x0be0   4e f0 8a cb...??     <INVALID>
++0x0be1   f0 8a cb...??        <INVALID>
++0x0be2   8a cb                mov    cl, bl
++0x0be4   5d                   pop    rbp
++0x0be5   9a cd bc 05 e7 26 70...??  <INVALID>
++0x0be6   cd bc                int    0xbc
++0x0be8   05 e7 26 70 4e       add    eax, 0x4e7026e7
++0x0bed   4c b5 50             mov    bpl, 0x50
++0x0bf0   d7                   xlatb  
++0x0bf1   85 80 7a 48 e4 66    test   dword ptr [rax+0x66e4487a], eax
++0x0bf7   0d f4 61 10 d3       or     eax, 0xd31061f4
++0x0bfc   e8 34 95 45 b9       call   0xffffffffc9459539
++0x0c01   f2 9a 7e d9 5b 81 0c...??  <INVALID>
+     50
++0x0c02   9a 7e d9 5b 81 0c 50...??  <INVALID>
++0x0c03   7e d9                jle    0x000000000fffffdb
++0x0c05   5b                   pop    rbx
++0x0c06   81 0c 50 54 5a e3 d2 or     dword ptr [rax+rdx*2], 0xd2e35a54
++0x0c0d   de 2a                fisubr word ptr [rdx]
++0x0c0f   9d                   popfq  
++0x0c10   ef                   out    eax, dx
++0x0c11   20 3e                and    byte ptr [rsi], bh
++0x0c13   bb 23 96 08 08       mov    ebx, 0x08089623
++0x0c18   52                   push   rdx
++0x0c19   70 67                jo     0x0000000010000069
++0x0c1b   32 6a 04             xor    ch, byte ptr [rdx+0x04]
++0x0c1e   72 e4                jb     0x000000000fffffe6
++0x0c20   43 f5                cmc    
++0x0c22   f6 5e ad             neg    byte ptr [rsi-0x53]
++0x0c25   5a                   pop    rdx
++0x0c26   1c 3f                sbb    al, 0x3f
++0x0c28   c5 b9 56 43 a8 af...??  <INVALID>
++0x0c29   b9 56 43 a8 af       mov    ecx, 0xafa84356
++0x0c2e   90                   nop    
++0x0c2f   b0 b5                mov    al, 0xb5
++0x0c31   80 48 71 43          or     byte ptr [rax+0x71], 0x43
++0x0c35   92                   xchg   edx, eax
++0x0c36   d4 63...??           <INVALID>
++0x0c37   63 9a 6a c1 2e ed    movsxd ebx, dword ptr [rdx-0x12d13e96]
++0x0c3d   37...??              <INVALID>
++0x0c3e   ee                   out    al, dx
++0x0c3f   15 b9 a4 97 a8       adc    eax, 0xa897a4b9
++0x0c44   67 74 ff             jz     0x0000000010000002
++0x0c47   a4                   movsb  
++0x0c48   6b b6 a1 c5 fb 53 5a imul   esi, dword ptr [rsi+0x53fbc5a1], 0x5a
++0x0c4f   7f 89                jnle   0x000000000fffff8b
++0x0c51   82 8f 49 76 36 9f 7d...??  <INVALID>
++0x0c52   8f 49 76...??        <INVALID>
++0x0c53   49 76 36             jbe    0x0000000010000039
++0x0c56   9f                   lahf   
++0x0c57   7d 50                jnl    0x0000000010000052
++0x0c59   69 38 9b a5 4a 2c    imul   edi, dword ptr [rax], 0x2c4aa59b
++0x0c5f   28 5d 3e             sub    byte ptr [rbp+0x3e], bl
++0x0c62   23 99 08 0d 88 ce    and    ebx, dword ptr [rcx-0x3177f2f8]
++0x0c68   44 29 4d 11          sub    dword ptr [rbp+0x11], r9d
++0x0c6c   5c                   pop    rsp
++0x0c6d   49 55                push   r13
++0x0c6f   d6...??              <INVALID>
++0x0c70   b4 36                mov    ah, 0x36
++0x0c72   7d c1                jnl    0x000000000fffffc3
++0x0c74   7b 3c                jnp    0x000000001000003e
++0x0c76   5a                   pop    rdx
++0x0c77   41 43 0a e8          or     bpl, r8l
++0x0c7b   28 2b                sub    byte ptr [rbx], ch
++0x0c7d   66 72 79             jb     0x000000001000007c
++0x0c80   2f...??              <INVALID>
++0x0c81   2d 27 29 c3 c6       sub    eax, 0xc6c32927
++0x0c86   a3 32 d5 62 e2 45 ee mov    dword ptr [0xcd93ee45e262d532], eax
+     93 cd
++0x0c8f   3c 63                cmp    al, 0x63
++0x0c91   b9 e3 df 9b d8       mov    ecx, 0xd89bdfe3
++0x0c96   c5 f0...??           <INVALID>
++0x0c97   f0 f0 8f f1...??     <INVALID>
++0x0c98   f0 8f f1...??        <INVALID>
++0x0c99   8f f1...??           <INVALID>
++0x0c9a   f1                   int1   
++0x0c9b   86 e8                xchg   al, ch
++0x0c9d   05 71 d3 9e 35       add    eax, 0x359ed371
++0x0ca2   2a 87 c5 c6 32 69    sub    al, byte ptr [rdi+0x6932c6c5]
++0x0ca8   eb 8f                jmp    0x000000000fffff91
++0x0caa   d7                   xlatb  
++0x0cab   0e...??              <INVALID>
++0x0cac   32 37                xor    dh, byte ptr [rdi]
++0x0cae   63 0c 7c             movsxd ecx, dword ptr [rsp+rdi*2]
++0x0cb1   a8 4e                test   al, 0x4e
++0x0cb3   df 64 49 f3          fbld   tbyte ptr [rcx+rcx*2-0x0d]
++0x0cb7   58                   pop    rax
++0x0cb8   5f                   pop    rdi
++0x0cb9   83 87 2a 6c ca 56 15 add    dword ptr [rdi+0x56ca6c2a], 0x15
++0x0cc0   ae                   scasb  
++0x0cc1   c6 fa 1d...??        <INVALID>
++0x0cc2   fa                   cli    
++0x0cc3   1d 4f 40 44 a7       sbb    eax, 0xa744404f
++0x0cc8   cd 66                int    0x66
++0x0cca   19 36                sbb    dword ptr [rsi], esi
++0x0ccc   21 36                and    dword ptr [rsi], esi
++0x0cce   c6 1e 28...??        <INVALID>
++0x0ccf   1e...??              <INVALID>
++0x0cd0   28 cf                sub    bh, cl
++0x0cd2   e8 6a 7f bd c8       call   0xffffffffd8bd7f6f
++0x0cd7   12 1e                adc    bl, byte ptr [rsi]
++0x0cd9   ab                   stosd  
++0x0cda   74 17                jz     0x0000000010000019
++0x0cdc   d0 c4                rol    ah, 0x01
++0x0cde   f1                   int1   
++0x0cdf   b3 4b                mov    bl, 0x4b
++0x0ce1   06...??              <INVALID>
++0x0ce2   0c d5                or     al, 0xd5
++0x0ce4   69 8a d3 7b 18 1c e8 imul   ecx, dword ptr [rdx+0x1c187bd3], 0x6f7bffe8
+     ff 7b 6f
++0x0cee   1b fd                sbb    edi, ebp
++0x0cf0   53                   push   rbx
++0x0cf1   a8 4a                test   al, 0x4a
++0x0cf3   de bb e7 72 d4 db    fidivr word ptr [rbx-0x242b8d19]
++0x0cf9   a3 c4 3a 1e fa 97 c3 mov    dword ptr [0x5752c397fa1e3ac4], eax
+     52 57
++0x0d02   16...??              <INVALID>
++0x0d03   4e 12 c5             adc    r8l, bpl
++0x0d06   9b                   fwait  
++0x0d07   99                   cdq    
++0x0d08   ee                   out    al, dx
++0x0d09   85 8b ac 8c 89 d8    test   dword ptr [rbx-0x27767354], ecx
++0x0d0f   d8 50 db             fcom   dword ptr [rax-0x25]
++0x0d12   aa                   stosb  
++0x0d13   9f                   lahf   
++0x0d14   67 bb 22 1d eb ec    mov    ebx, 0xeceb1d22
++0x0d1a   d0 91 86 c6 a9 cc    rcl    byte ptr [rcx-0x3356397a], 0x01
++0x0d20   c9                   leave  
++0x0d21   22 b7 67 e8 9f 10    and    dh, byte ptr [rdi+0x109fe867]
++0x0d27   cb                   retf   
++0x0d28   26 bf 58 aa 84 7c    mov    edi, 0x7c84aa58
++0x0d2e   fc                   cld    
++0x0d2f   2e 39 dc             cmp    esp, ebx
++0x0d32   87 53 96             xchg   dword ptr [rbx-0x6a], edx
++0x0d35   1e...??              <INVALID>
++0x0d36   8e 7d 1b...??        <INVALID>
++0x0d37   7d 1b                jnl    0x000000001000001d
++0x0d39   16...??              <INVALID>
++0x0d3a   fa                   cli    
++0x0d3b   33 88 8e ca 5c dc    xor    ecx, dword ptr [rax-0x23a33572]
++0x0d41   72 e2                jb     0x000000000fffffe4
++0x0d43   69 db 44 68 97 30    imul   ebx, ebx, 0x30976844
++0x0d49   1d cb c2 37 0d       sbb    eax, 0x0d37c2cb
++0x0d4e   2f...??              <INVALID>
++0x0d4f   40 81 36 00 cb b6 1d xor    dword ptr [rsi], 0x1db6cb00
++0x0d56   b1 2c                mov    cl, 0x2c
++0x0d58   89 7e 91             mov    dword ptr [rsi-0x6f], edi
++0x0d5b   c9                   leave  
++0x0d5c   88 3b                mov    byte ptr [rbx], bh
++0x0d5e   69 06 0a 63 4b 89    imul   eax, dword ptr [rsi], 0x894b630a
++0x0d64   03 46 65             add    eax, dword ptr [rsi+0x65]
++0x0d67   97                   xchg   edi, eax
++0x0d68   2b fa                sub    edi, edx
++0x0d6a   46 f2 0b 68 a8       or     ebp, dword ptr [rax-0x58]
++0x0d6f   e1 8a                loope  0x000000000fffff8c
++0x0d71   2d 3e 57 9f 4b       sub    eax, 0x4b9f573e
++0x0d76   8d 17                lea    edx, [rdi]
++0x0d78   e4 80                in     al, 0x80
++0x0d7a   73 01                jnb    0x0000000010000003
++0x0d7c   97                   xchg   edi, eax
++0x0d7d   1c 22                sbb    al, 0x22
++0x0d7f   82 9f 37 16 c6 96 52...??  <INVALID>
++0x0d80   9f                   lahf   
++0x0d81   37...??              <INVALID>
++0x0d82   16...??              <INVALID>
++0x0d83   c6 96 52 71 1e 07 4c...??  <INVALID>
++0x0d84   96                   xchg   esi, eax
++0x0d85   52                   push   rdx
++0x0d86   71 1e                jno    0x0000000010000020
++0x0d88   07...??              <INVALID>
++0x0d89   4c c2 e4 c7          ret    0xc7e4
++0x0d8d   16...??              <INVALID>
++0x0d8e   df 23                fbld   tbyte ptr [rbx]
++0x0d90   bf 9b 47 ad 10       mov    edi, 0x10ad479b
++0x0d95   f2 9e                sahf   
++0x0d97   1a a8 c7 e4 b6 39    sbb    ch, byte ptr [rax+0x39b6e4c7]
++0x0d9d   dc 89 c2 c8 14 7b    fmul   qword ptr [rcx+0x7b14c8c2]
++0x0da3   97                   xchg   edi, eax
++0x0da4   f1                   int1   
++0x0da5   8b 1c 31             mov    ebx, dword ptr [rcx+rsi]
++0x0da8   df c8                fxch   st0, st0, st0
++0x0daa   41 5f                pop    r15
++0x0dac   9d                   popfq  
++0x0dad   35 70 d8 b5 78       xor    eax, 0x78b5d870
++0x0db2   bc 3b 7c aa 72       mov    esp, 0x72aa7c3b
++0x0db7   3d af 82 a6 dc       cmp    eax, 0xdca682af
++0x0dbc   2f...??              <INVALID>
++0x0dbd   d3 74 2c f4          shl    dword ptr [rsp+rbp-0x0c], cl
++0x0dc1   fd                   std    
++0x0dc2   34 1d                xor    al, 0x1d
++0x0dc4   46 0d 06 57 13 84    or     eax, 0x84135706
++0x0dca   f1                   int1   
++0x0dcb   60...??              <INVALID>
++0x0dcc   e6 fe                out    0xfe, al
++0x0dce   f6 40 94 2f          test   byte ptr [rax-0x6c], 0x2f
++0x0dd2   63 6e 37             movsxd ebp, dword ptr [rsi+0x37]
++0x0dd5   0c a8                or     al, 0xa8
++0x0dd7   30 22                xor    byte ptr [rdx], ah
++0x0dd9   96                   xchg   esi, eax
++0x0dda   91                   xchg   ecx, eax
++0x0ddb   00 42 08             add    byte ptr [rdx+0x08], al
++0x0dde   7c 16                jl     0x0000000010000018
++0x0de0   da 16                ficom  dword ptr [rsi]
++0x0de2   80 21 be             and    byte ptr [rcx], 0xbe
++0x0de5   24 fd                and    al, 0xfd
++0x0de7   85 f6                test   esi, esi
++0x0de9   2c 33                sub    al, 0x33
++0x0deb   f1                   int1   
++0x0dec   76 4e                jbe    0x0000000010000050
++0x0dee   0e...??              <INVALID>
++0x0def   34 89                xor    al, 0x89
++0x0df1   bd 42 99 ea 24       mov    ebp, 0x24ea9942
++0x0df6   51                   push   rcx
++0x0df7   75 e3                jnz    0x000000000fffffe5
++0x0df9   c7 2e cb 42 ae 82...??  <INVALID>
++0x0dfa   2e cb                retf   
++0x0dfc   42 ae                scasb  
++0x0dfe   82 49 a2 10...??     <INVALID>
++0x0dff   49 a2 10 da 4f 76 69 mov    byte ptr [0xa334ad69764fda10], al
+     ad 34 a3
++0x0e09   32 85 0f 31 65 13    xor    al, byte ptr [rbp+0x1365310f]
++0x0e0f   05 6a 75 36 92       add    eax, 0x9236756a
++0x0e14   b7 9f                mov    bh, 0x9f
++0x0e16   b8 0d 54 73 c9       mov    eax, 0xc973540d
++0x0e1b   35 52 4f 14 0d       xor    eax, 0x0d144f52
++0x0e20   c0 85 6b 55 95 c5 1f rol    byte ptr [rbp-0x3a6aaa95], 0x1f
++0x0e27   73 7e                jnb    0x0000000010000080
++0x0e29   31 5e 58             xor    dword ptr [rsi+0x58], ebx
++0x0e2c   0a 19                or     bl, byte ptr [rcx]
++0x0e2e   23 5a fa             and    ebx, dword ptr [rdx-0x06]
++0x0e31   84 b7 c5 b0 2a 4b    test   byte ptr [rdi+0x4b2ab0c5], dh
++0x0e37   23 bc 61 95 0c e4 c6 and    edi, dword ptr [rcx-0x391bf36b]
++0x0e3e   f9                   stc    
++0x0e3f   a0 be 47 a5 4d 23 7d mov    al, byte ptr [0xd7717d234da547be]
+     71 d7
++0x0e48   f0 76 7e...??        <INVALID>
++0x0e49   76 7e                jbe    0x0000000010000080
++0x0e4b   b5 a9                mov    ch, 0xa9
++0x0e4d   39 62 52             cmp    dword ptr [rdx+0x52], esp
++0x0e50   34 22                xor    al, 0x22
++0x0e52   07...??              <INVALID>
++0x0e53   80 ee 45             sub    dh, 0x45
++0x0e56   2d 9d d7 34 b2       sub    eax, 0xb234d79d
++0x0e5b   b7 55                mov    bh, 0x55
++0x0e5d   b9 c1 c2 32 2e       mov    ecx, 0x2e32c2c1
++0x0e62   58                   pop    rax
++0x0e63   0b 83 d1 8e ac 00    or     eax, dword ptr [rbx+0x00ac8ed1]
++0x0e69   2f...??              <INVALID>
++0x0e6a   2d de 5f 4f 89       sub    eax, 0x894f5fde
++0x0e6f   68 b1 1a 4e e0       push   0xe04e1ab1
++0x0e74   ab                   stosd  
++0x0e75   e7 bc                out    0xbc, eax
++0x0e77   5d                   pop    rbp
++0x0e78   25 00 8d 82 4f       and    eax, 0x4f828d00
++0x0e7d   90                   nop    
++0x0e7e   53                   push   rbx
++0x0e7f   34 20                xor    al, 0x20
++0x0e81   71 09                jno    0x000000001000000b
++0x0e83   81 9b 24 e9 3c 02 7e sbb    dword ptr [rbx+0x023ce924], 0xf8a9277e
+     27 a9 f8
++0x0e8d   b2 0b                mov    dl, 0x0b
++0x0e8f   dc de                fcomp  st0, st6
++0x0e91   43 ee                out    al, dx
++0x0e93   29 93 ca 86 18 2a    sub    dword ptr [rbx+0x2a1886ca], edx
++0x0e99   da 99 e8 59 3e 52    ficomp dword ptr [rcx+0x523e59e8]
++0x0e9f   39 2c 83             cmp    dword ptr [rbx+rax*4], ebp
++0x0ea2   e4 dc                in     al, 0xdc
++0x0ea4   5e                   pop    rsi
++0x0ea5   f7 52 49             not    dword ptr [rdx+0x49]
++0x0ea8   ac                   lodsb  
++0x0ea9   e4 43                in     al, 0x43
++0x0eab   1e...??              <INVALID>
++0x0eac   a3 6e 27 88 fc 5a 9a mov    dword ptr [0xd5e59a5afc88276e], eax
+     e5 d5
++0x0eb5   b5 ef                mov    ch, 0xef
++0x0eb7   f2 a3 9a 7a 21 02 fa mov    dword ptr [0xa3ec8cfa02217a9a], eax
+     8c ec a3
++0x0ec1   c2 76 e6             ret    0xe676
++0x0ec4   86 90 3a 35 65 a2    xchg   byte ptr [rax-0x5d9acac6], dl
++0x0eca   11 27                adc    dword ptr [rdi], esp
++0x0ecc   02 f2                add    dh, dl
++0x0ece   a7                   cmpsd  
++0x0ecf   da 99 df bc 93 63    ficomp dword ptr [rcx+0x6393bcdf]
++0x0ed5   b1 51                mov    cl, 0x51
++0x0ed7   fa                   cli    
++0x0ed8   9d                   popfq  
++0x0ed9   16...??              <INVALID>
++0x0eda   55                   push   rbp
++0x0edb   10 a5 71 85 43 90    adc    byte ptr [rbp-0x6fbc7a8f], ah
++0x0ee1   01 dd                add    ebp, ebx
++0x0ee3   0f f7 50 e9...??     <INVALID>
++0x0ee4   f7 50 e9             not    dword ptr [rax-0x17]
++0x0ee7   78 fc                js     0x000000000ffffffe
++0x0ee9   8a d7                mov    dl, bh
++0x0eeb   57                   push   rdi
++0x0eec   57                   push   rdi
++0x0eed   d3 e8                shr    eax, cl
++0x0eef   36 0d 2c 31 e8 25    or     eax, 0x25e8312c
++0x0ef5   93                   xchg   ebx, eax
++0x0ef6   4f 1c 80             sbb    al, 0x80
++0x0ef9   23 31                and    esi, dword ptr [rcx]
++0x0efb   a8 14                test   al, 0x14
++0x0efd   60...??              <INVALID>
++0x0efe   c3                   ret    
++0x0eff   41 11 66 3b          adc    dword ptr [r14+0x3b], esp
++0x0f03   8f 34 3b...??        <INVALID>
++0x0f04   34 3b                xor    al, 0x3b
++0x0f06   59                   pop    rcx
++0x0f07   cc                   int3   
++0x0f08   d2 86 74 9d 7b 47    rol    byte ptr [rsi+0x477b9d74], cl
++0x0f0e   1b c3                sbb    eax, ebx
++0x0f10   99                   cdq    
++0x0f11   53                   push   rbx
++0x0f12   de dc...??           <INVALID>
++0x0f13   dc 77 72             fdiv   qword ptr [rdi+0x72]
++0x0f16   4e 8f 74 15 c6...??  <INVALID>
++0x0f17   8f 74 15 c6...??     <INVALID>
++0x0f18   74 15                jz     0x0000000010000017
++0x0f1a   c6 79 fd 35...??     <INVALID>
++0x0f1b   79 fd                jns    0x000000000fffffff
++0x0f1d   35 9b 87 ad 5b       xor    eax, 0x5bad879b
++0x0f22   f9                   stc    
++0x0f23   1a 94 df a0 96 97 46 sbb    dl, byte ptr [rdi+rbx*8+0x469796a0]
++0x0f2a   e3 aa                jrcxz  0x000000000fffffac
++0x0f2c   8c 72 22...??        <INVALID>
++0x0f2d   72 22                jb     0x0000000010000024
++0x0f2f   97                   xchg   edi, eax
++0x0f30   69 f7 5f 3f b5 a2    imul   esi, edi, 0xa2b53f5f
++0x0f36   50                   push   rax
++0x0f37   39 24 a9             cmp    dword ptr [rcx+rbp*4], esp
++0x0f3a   62 1e...??           <INVALID>
++0x0f3b   1e...??              <INVALID>
++0x0f3c   0b f7                or     esi, edi
++0x0f3e   64 e1 42             loope  0x0000000010000045
++0x0f41   6f                   outsd  
++0x0f42   e8 f8 91 f1 96       call   0xffffffffa6f191fd
++0x0f47   c7 3a 64 fb b9 0f...??  <INVALID>
++0x0f48   3a 64 fb b9          cmp    ah, byte ptr [rbx+rdi*8-0x47]
++0x0f4c   0f 61 b9 2b 9f 22 bb punpcklwd mm7, qword ptr [rcx-0x44dd60d5]
++0x0f53   08 9c f4 4b f7 6b e7 or     byte ptr [rsp+rsi*8-0x189408b5], bl
++0x0f5a   a8 67                test   al, 0x67
++0x0f5c   c1 22 17             shl    dword ptr [rdx], 0x17
++0x0f5f   e5 fd                in     eax, 0xfd
++0x0f61   9b                   fwait  
++0x0f62   3d 7f f3 16 08       cmp    eax, 0x0816f37f
++0x0f67   11 28                adc    dword ptr [rax], ebp
++0x0f69   68 cf d8 4d 84       push   0x844dd8cf
++0x0f6e   2a 01                sub    al, byte ptr [rcx]
++0x0f70   aa                   stosb  
++0x0f71   c1 a8 e7 4d 99 5a 0d shr    dword ptr [rax+0x5a994de7], 0x0d
++0x0f78   41 bf 4c 3e e6 35    mov    r15d, 0x35e63e4c
++0x0f7e   98                   cwde   
++0x0f7f   00 6f f6             add    byte ptr [rdi-0x0a], ch
++0x0f82   2c 99                sub    al, 0x99
++0x0f84   7e f3                jle    0x000000000ffffff5
++0x0f86   56                   push   rsi
++0x0f87   a6                   cmpsb  
++0x0f88   41 64 81 6c 60 68 89 sub    dword ptr [fs:rax+0x68], 0x7ef16c89
+     6c f1 7e
++0x0f92   30 76 b2             xor    byte ptr [rsi-0x4e], dh
++0x0f95   3c a4                cmp    al, 0xa4
++0x0f97   c0 8e 02 55 00 4d 08 ror    byte ptr [rsi+0x4d005502], 0x08
++0x0f9e   ae                   scasb  
++0x0f9f   f7 e7                mul    edx, eax, edi
++0x0fa1   37...??              <INVALID>
++0x0fa2   e6 11                out    0x11, al
++0x0fa4   0a a7 7a 92 e5 7e    or     ah, byte ptr [rdi+0x7ee5927a]
++0x0faa   dd c9                fxch   st0, st1, st0
++0x0fac   59                   pop    rcx
++0x0fad   04 60                add    al, 0x60
++0x0faf   45 60...??           <INVALID>
++0x0fb0   60...??              <INVALID>
++0x0fb1   32 9b ce 87 f3 35    xor    bl, byte ptr [rbx+0x35f387ce]
++0x0fb7   02 87 e3 87 80 1c    add    al, byte ptr [rdi+0x1c8087e3]
++0x0fbd   9e                   sahf   
++0x0fbe   2b f1                sub    esi, ecx
++0x0fc0   c4 e8...??           <INVALID>
++0x0fc1   e8 4a 6e 24 50       call   0x0000000060246e4f
++0x0fc6   9d                   popfq  
++0x0fc7   5d                   pop    rbp
++0x0fc8   cd 92                int    0x92
++0x0fca   d1 95 30 c4 f7 19    rcl    dword ptr [rbp+0x19f7c430], 0x01
++0x0fd0   cf                   iretd  
++0x0fd1   a2 61 d9 14 98 16 ab mov    byte ptr [0x13a0ab169814d961], al
+     a0 13
++0x0fda   4d 44 e6 29          out    0x29, al
++0x0fde   93                   xchg   ebx, eax
++0x0fdf   2a 18                sub    bl, byte ptr [rax]
++0x0fe1   f7 fe                idiv   edx, eax, esi
++0x0fe3   2d 1c 97 d0 8f       sub    eax, 0x8fd0971c
++0x0fe8   52                   push   rdx
++0x0fe9   36 1b cc             sbb    ecx, esp
++0x0fec   3e ca b0 3a          retf   0x3ab0
++0x0ff0   34 97                xor    al, 0x97
++0x0ff2   11 38                adc    dword ptr [rax], edi
++0x0ff4   7e 97                jle    0x000000000fffff99
++0x0ff6   11 bb 1d 2e ac 12    adc    dword ptr [rbx+0x12ac2e1d], edi
++0x0ffc   d1 d3                rcl    ebx, 0x01
++0x0ffe   7b a9                jnp    0x000000000fffffab
++0x1000   db 54 f7 61          fist   dword ptr [rdi+rsi*8+0x61]
++0x1004   77 aa                jnbe   0x000000000fffffac
++0x1006   af                   scasd  
++0x1007   50                   push   rax
++0x1008   bc 27 58 0f 4e       mov    esp, 0x4e0f5827
++0x100d   4f f9                stc    
++0x100f   8f 94 7b 14 94 53 57...??  <INVALID>
++0x1010   94                   xchg   esp, eax
++0x1011   7b 14                jnp    0x0000000010000016
++0x1013   94                   xchg   esp, eax
++0x1014   53                   push   rbx
++0x1015   57                   push   rdi
++0x1016   75 47                jnz    0x0000000010000049
++0x1018   a3 68 44 9b 92 2c 57 mov    dword ptr [0xa481572c929b4468], eax
+     81 a4
++0x1021   b6 56                mov    dh, 0x56
++0x1023   96                   xchg   esi, eax
++0x1024   02 0e                add    cl, byte ptr [rsi]
++0x1026   d2 6a 8b             shr    byte ptr [rdx-0x75], cl
++0x1029   71 96                jno    0x000000000fffff98
++0x102b   f8                   clc    
++0x102c   3f...??              <INVALID>
++0x102d   91                   xchg   ecx, eax
++0x102e   e9 2e d2 fe 41       jmp    0x0000000051fed233
++0x1033   c7 d8 80 3c 0c d0...??  <INVALID>
++0x1034   d8 80 3c 0c d0 58    fadd   dword ptr [rax+0x58d00c3c]
++0x103a   4d 7a 14             jp     0x0000000010000017
++0x103d   6f                   outsd  
++0x103e   a8 78                test   al, 0x78
++0x1040   b7 d3                mov    bh, 0xd3
++0x1042   e7 69                out    0x69, eax
++0x1044   a2 22 11 66 3a d5 f7 mov    byte ptr [0x315cf7d53a661122], al
+     5c 31
++0x104d   61...??              <INVALID>
++0x104e   b1 cf                mov    cl, 0xcf
++0x1050   4f 8c 18             mov    qword ptr [r8], ds
++0x1053   a2 77 ce 89 66 13 80 mov    byte ptr [0xb3b880136689ce77], al
+     b8 b3
++0x105c   08 9b 19 31 d3 ec    or     byte ptr [rbx-0x132ccee7], bl
++0x1062   78 d1                js     0x000000000fffffd3
++0x1064   e9 c8 fb 20 20       jmp    0x000000003020fbcd
++0x1069   3f...??              <INVALID>
++0x106a   a7                   cmpsd  
++0x106b   7d 23                jnl    0x0000000010000025
++0x106d   8c bf c3 aa 45 64...??  <INVALID>
++0x106e   bf c3 aa 45 64       mov    edi, 0x6445aac3
++0x1073   85 68 57             test   dword ptr [rax+0x57], ebp
++0x1076   50                   push   rax
++0x1077   e4 10                in     al, 0x10
++0x1079   cc                   int3   
++0x107a   e1 d5                loope  0x000000000fffffd7
++0x107c   79 d6                jns    0x000000000fffffd8
++0x107e   e8 87 74 f4 8d       call   0xffffffff9df4748c
++0x1083   b4 7c                mov    ah, 0x7c
++0x1085   d1 3c 02             sar    dword ptr [rdx+rax], 0x01
++0x1088   be e8 db de 2e       mov    esi, 0x2ededbe8
++0x108d   c1 83 9b e5 65 1d 60 rol    dword ptr [rbx+0x1d65e59b], 0x60
++0x1094   af                   scasd  
++0x1095   b1 e0                mov    cl, 0xe0
++0x1097   4a 09 79 93          or     qword ptr [rcx-0x6d], rdi
++0x109b   27...??              <INVALID>
++0x109c   04 7c                add    al, 0x7c
++0x109e   ce...??              <INVALID>
++0x109f   81 d4 15 17 10 05    adc    esp, 0x05101715
++0x10a5   75 e2                jnz    0x000000000fffffe4
++0x10a7   1c 4d                sbb    al, 0x4d
++0x10a9   4f 0a 22             or     r12l, byte ptr [r10]
++0x10ac   68 02 30 07 4c       push   0x4c073002
++0x10b1   3e 4a 66 fc          cld    
++0x10b5   64 18 1e             sbb    byte ptr [fs:rsi], bl
++0x10b8   77 82                jnbe   0x000000000fffff84
++0x10ba   5d                   pop    rbp
++0x10bb   55                   push   rbp
++0x10bc   ca 95 b2             retf   0xb295
++0x10bf   56                   push   rsi
++0x10c0   0f b4 95 59 1c 08 dd lfs    edx, [rbp-0x22f7e3a7]
++0x10c7   88 88 d4 11 85 0a    mov    byte ptr [rax+0x0a8511d4], cl
++0x10cd   c5 cb...??           <INVALID>
++0x10ce   cb                   retf   
++0x10cf   cf                   iretd  
++0x10d0   9a eb af a3 05 40 81...??  <INVALID>
++0x10d1   eb af                jmp    0x000000000fffffb1
++0x10d3   a3 05 40 81 e3 4a 97 mov    dword ptr [0x319a974ae3814005], eax
+     9a 31
++0x10dc   f4                   hlt    
++0x10dd   56                   push   rsi
++0x10de   98                   cwde   
++0x10df   aa                   stosb  
++0x10e0   f0 ab...??           <INVALID>
++0x10e1   ab                   stosd  
++0x10e2   64 56                push   rsi
++0x10e4   81 c8 68 a8 69 57    or     eax, 0x5769a868
++0x10ea   d9 74 43 3a          fnstenv [rbx+rax*2+0x3a]
++0x10ee   dc 4a 8a             fmul   qword ptr [rdx-0x76]
++0x10f1   36 17...??           <INVALID>
++0x10f2   17...??              <INVALID>
++0x10f3   5d                   pop    rbp
++0x10f4   7e ad                jle    0x000000000fffffaf
++0x10f6   5b                   pop    rbx
++0x10f7   15 1f b1 e1 90       adc    eax, 0x90e1b11f
++0x10fc   ef                   out    eax, dx
++0x10fd   94                   xchg   esp, eax
++0x10fe   49 a7                cmpsq  
++0x1100   96                   xchg   esi, eax
++0x1101   2b fd                sub    edi, ebp
++0x1103   f6 af bb 85 0a 09    imul   ax, byte ptr [rdi+0x090a85bb]
++0x1109   d1 d3                rcl    ebx, 0x01
++0x110b   be 26 ab 7e 2e       mov    esi, 0x2e7eab26
++0x1110   e0 d9                loopne 0x000000000fffffdb
++0x1112   73 6c                jnb    0x000000001000006e
++0x1114   f8                   clc    
++0x1115   e6 76                out    0x76, al
++0x1117   58                   pop    rax
++0x1118   e2 1d                loop   0x000000001000001f
++0x111a   a2 df 0f 4d 04 13 4e mov    byte ptr [0xb52e4e13044d0fdf], al
+     2e b5
++0x1123   de ca                fmulp  st2, st0
++0x1125   df 02                fild   word ptr [rdx]
++0x1127   ed                   in     eax, dx
++0x1128   e0 83                loopne 0x000000000fffff85
++0x112a   22 55 28             and    dl, byte ptr [rbp+0x28]
++0x112d   fe a7 38 14 85 8b...??  <INVALID>
++0x112e   a7                   cmpsd  
++0x112f   38 14 85 8b 2c 61 f6 cmp    byte ptr [rax*4-0x099ed375], dl
++0x1136   a5                   movsd  
++0x1137   ad                   lodsd  
++0x1138   3b 82 12 be 4b 95    cmp    eax, dword ptr [rdx-0x6ab441ee]
++0x113e   50                   push   rax
++0x113f   f1                   int1   
++0x1140   ad                   lodsd  
++0x1141   70 c8                jo     0x000000000fffffca
++0x1143   5e                   pop    rsi
++0x1144   52                   push   rdx
++0x1145   88 e0                mov    al, ah
++0x1147   cf                   iretd  
++0x1148   55                   push   rbp
++0x1149   81 c0 2a be 14 30    add    eax, 0x3014be2a
++0x114f   91                   xchg   ecx, eax
++0x1150   3b 71 08             cmp    esi, dword ptr [rcx+0x08]
++0x1153   c2 83 6b             ret    0x6b83
++0x1156   b6 be                mov    dh, 0xbe
++0x1158   6c                   insb   
++0x1159   63 0a                movsxd ecx, dword ptr [rdx]
++0x115b   21 17                and    dword ptr [rdi], edx
++0x115d   f8                   clc    
++0x115e   e1 83                loope  0x000000000fffff85
++0x1160   f3 99                cdq    
++0x1162   1f...??              <INVALID>
++0x1163   7b aa                jnp    0x000000000fffffac
++0x1165   1d f1 c8 64 9a       sbb    eax, 0x9a64c8f1
++0x116a   b7 98                mov    bh, 0x98
++0x116c   29 97 b1 1a e8 d9    sub    dword ptr [rdi-0x2617e54f], edx
++0x1172   ce...??              <INVALID>
++0x1173   51                   push   rcx
++0x1174   a3 80 f2 84 b8 fb 90 mov    dword ptr [0x579790fbb884f280], eax
+     97 57
++0x117d   78 ff                js     0x0000000010000001
++0x117f   f2 6c                insb   
++0x1181   0f 53 e7             rcpps  xmm4, xmm7
++0x1184   dd 1d 47 c7 ac 67    fstp   <rel> qword ptr [0x0000000077acc74d]
++0x118a   03 91 f2 04 92 8a    add    edx, dword ptr [rcx-0x756dfb0e]
++0x1190   2f...??              <INVALID>
++0x1191   c5 55 18...??        <INVALID>
++0x1192   55                   push   rbp
++0x1193   18 3c ce             sbb    byte ptr [rsi+rcx*8], bh
++0x1196   9d                   popfq  
++0x1197   d0 5d dc             rcr    byte ptr [rbp-0x24], 0x01
++0x119a   79 a3                jns    0x000000000fffffa5
++0x119c   13 25 e8 06 9f 44    adc    esp, <rel> dword ptr [0x00000000549f06ee]
++0x11a2   bc 52 76 82 81       mov    esp, 0x81827652
++0x11a7   fd                   std    
++0x11a8   c9                   leave  
++0x11a9   ce...??              <INVALID>
++0x11aa   c6 f3 0e...??        <INVALID>
++0x11ab   f3 0e...??           <INVALID>
++0x11ac   0e...??              <INVALID>
++0x11ad   b9 00 4d ba 83       mov    ecx, 0x83ba4d00
++0x11b2   f5                   cmc    
++0x11b3   1d 76 b3 33 42       sbb    eax, 0x4233b376
++0x11b8   5d                   pop    rbp
++0x11b9   0e...??              <INVALID>
++0x11ba   17...??              <INVALID>
++0x11bb   f2 c1 10 25          rcl    dword ptr [rax], 0x25
++0x11bf   fe 6f 72...??        <INVALID>
++0x11c0   6f                   outsd  
++0x11c1   72 18                jb     0x000000001000001a
++0x11c3   a8 b6                test   al, 0xb6
++0x11c5   ad                   lodsd  
++0x11c6   e1 2e                loope  0x0000000010000030
++0x11c8   ce...??              <INVALID>
++0x11c9   e3 30                jrcxz  0x0000000010000032
++0x11cb   d7                   xlatb  
++0x11cc   54                   push   rsp
++0x11cd   c1 07 91             rol    dword ptr [rdi], 0x91
++0x11d0   8a 8f 17 7c 22 7c    mov    cl, byte ptr [rdi+0x7c227c17]
++0x11d6   a3 35 de b6 4a bf f1 mov    dword ptr [0xfc12f1bf4ab6de35], eax
+     12 fc
++0x11df   41 cb                retf   
++0x11e1   0c e5                or     al, 0xe5
++0x11e3   24 e3                and    al, 0xe3
++0x11e5   fd                   std    
++0x11e6   8c b4 77 b6 9b ac e9...??  <INVALID>
++0x11e7   b4 77                mov    ah, 0x77
++0x11e9   b6 9b                mov    dh, 0x9b
++0x11eb   ac                   lodsb  
++0x11ec   e9 a1 cb 96 a8       jmp    0xffffffffb896cba6
++0x11f1   f6 24 fe             mul    ax, byte ptr [rsi+rdi*8]
++0x11f4   ef                   out    eax, dx
++0x11f5   4e 9c                pushfq 
++0x11f7   95                   xchg   ebp, eax
++0x11f8   96                   xchg   esi, eax
++0x11f9   32 0b                xor    cl, byte ptr [rbx]
++0x11fb   83 12 1d             adc    dword ptr [rdx], 0x1d
++0x11fe   6a 1b                push   0x1b
++0x1200   ef                   out    eax, dx
++0x1201   35 e7 7a d7 ea       xor    eax, 0xead77ae7
++0x1206   c3                   ret    
++0x1207   08 2a                or     byte ptr [rdx], ch
++0x1209   66 d6...??           <INVALID>
++0x120a   d6...??              <INVALID>
++0x120b   5a                   pop    rdx
++0x120c   72 cc                jb     0x000000000fffffce
++0x120e   4b 0a df             or     bl, r15l
++0x1211   9f                   lahf   
++0x1212   e8 50 ec 74 73       call   0x000000008374ec55
++0x1217   09 50 1e             or     dword ptr [rax+0x1e], edx
++0x121a   ed                   in     eax, dx
++0x121b   eb 35                jmp    0x0000000010000037
++0x121d   68 c4 27 fc 44       push   0x44fc27c4
++0x1222   9e                   sahf   
++0x1223   60...??              <INVALID>
++0x1224   cd 97                int    0x97
++0x1226   d5 64...??           <INVALID>
++0x1227   64 f1                int1   
++0x1229   05 7a f2 b9 2d       add    eax, 0x2db9f27a
++0x122e   48 19 fc             sbb    rsp, rdi
++0x1231   1e...??              <INVALID>
++0x1232   3b c5                cmp    eax, ebp
++0x1234   eb b2                jmp    0x000000000fffffb4
++0x1236   a0 d5 d4 9b 89 1d c0 mov    al, byte ptr [0x15e1c01d899bd4d5]
+     e1 15
++0x123f   b6 2a                mov    dh, 0x2a
++0x1241   b1 25                mov    cl, 0x25
++0x1243   e2 3a                loop   0x000000001000003c
++0x1245   0e...??              <INVALID>
++0x1246   81 61 8a 36 86 9f b9 and    dword ptr [rcx-0x76], 0xb99f8636
++0x124d   84 89 43 a9 bb c7    test   byte ptr [rcx-0x384456bd], cl
++0x1253   30 57 c3             xor    byte ptr [rdi-0x3d], dl
++0x1256   1b 00                sbb    eax, dword ptr [rax]
++0x1258   86 1a                xchg   byte ptr [rdx], bl
++0x125a   a2 2e 39 0c ab f4 6c mov    byte ptr [0x08436cf4ab0c392e], al
+     43 08
++0x1263   5c                   pop    rsp
++0x1264   58                   pop    rax
++0x1265   9e                   sahf   
++0x1266   ee                   out    al, dx
++0x1267   a3 4f 32 2d e5 4f 5a mov    dword ptr [0x6ea05a4fe52d324f], eax
+     a0 6e
++0x1270   31 f8                xor    eax, edi
++0x1272   ab                   stosd  
++0x1273   84 3a                test   byte ptr [rdx], bh
++0x1275   d1 2f                shr    dword ptr [rdi], 0x01
++0x1277   dc c1                fadd   st1, st0
++0x1279   47 de 7d d3          fidivr word ptr [r13-0x2d]
++0x127d   5c                   pop    rsp
++0x127e   8c d1                mov    ecx, ss
++0x1280   ef                   out    eax, dx
++0x1281   8f 7b 04...??        <INVALID>
++0x1282   7b 04                jnp    0x0000000010000006
++0x1284   b6 70                mov    dh, 0x70
++0x1286   ef                   out    eax, dx
++0x1287   66 d1 29             shr    word ptr [rcx], 0x01
++0x128a   0f 79 55 ca          vmwrite edx, dword ptr [rbp-0x36]
++0x128e   07...??              <INVALID>
++0x128f   8a f6                mov    dh, dh
++0x1291   9e                   sahf   
++0x1292   78 16                js     0x0000000010000018
++0x1294   31 2e                xor    dword ptr [rsi], ebp
++0x1296   2e d6...??           <INVALID>
++0x1297   d6...??              <INVALID>
++0x1298   d1 8f ae 12 17 7d    ror    dword ptr [rdi+0x7d1712ae], 0x01
++0x129e   5d                   pop    rbp
++0x129f   3c d6                cmp    al, 0xd6
++0x12a1   c0 69 8a ab          shr    byte ptr [rcx-0x76], 0xab
++0x12a5   36 73 6f             jnb    0x0000000010000072
++0x12a8   11 f0                adc    eax, esi
++0x12aa   b3 2a                mov    bl, 0x2a
++0x12ac   a0 cf e9 ef 01 74 6c mov    al, byte ptr [0x671a6c7401efe9cf]
+     1a 67
++0x12b5   8e a6 b8 09 92 dd    mov    fs, dword ptr [rsi-0x226df648]
++0x12bb   03 84 69 04 d2 b3 74 add    eax, dword ptr [rcx+rbp*2+0x74b3d204]
++0x12c2   11 b1 e7 d0 74 58    adc    dword ptr [rcx+0x5874d0e7], esi
++0x12c8   dd a9 b6 99 56 ad...??  <INVALID>
++0x12c9   a9 b6 99 56 ad       test   eax, 0xad5699b6
++0x12ce   6c                   insb   
++0x12cf   f3 cd c5             int    0xc5
++0x12d2   69 b6 f4 ba 18 d4 a9 imul   esi, dword ptr [rsi-0x2be7450c], 0xdc3e6aa9
+     6a 3e dc
++0x12dc   a0 67 a3 f5 14 da c9 mov    al, byte ptr [0x6219c9da14f5a367]
+     19 62
++0x12e5   9c                   pushfq 
++0x12e6   3d d1 46 7b 3f       cmp    eax, 0x3f7b46d1
++0x12eb   17...??              <INVALID>
++0x12ec   93                   xchg   ebx, eax
++0x12ed   fd                   std    
++0x12ee   ee                   out    al, dx
++0x12ef   e4 2d                in     al, 0x2d
++0x12f1   29 8c a9 1b ed 57 32 sub    dword ptr [rcx+rbp*4+0x3257ed1b], ecx
++0x12f8   f4                   hlt    
++0x12f9   ca a7 52             retf   0x52a7
++0x12fc   f2 27...??           <INVALID>
++0x12fd   27...??              <INVALID>
++0x12fe   f8                   clc    
++0x12ff   7a 71                jp     0x0000000010000073
++0x1301   59                   pop    rcx
++0x1302   b5 16                mov    ch, 0x16
++0x1304   5f                   pop    rdi
++0x1305   a5                   movsd  
++0x1306   44 a0 88 6f 67 0f a5 mov    al, byte ptr [0xd61f8da50f676f88]
+     8d 1f d6
++0x1310   1f...??              <INVALID>
++0x1311   00 f9                add    cl, bh
++0x1313   16...??              <INVALID>
++0x1314   48 c3                ret    
++0x1316   c5 bf 93 bc a0 9a...??  <INVALID>
++0x1317   bf 93 bc a0 9a       mov    edi, 0x9aa0bc93
++0x131c   dd 55 a0             fst    qword ptr [rbp-0x60]
++0x131f   32 69 f4             xor    ch, byte ptr [rcx-0x0c]
++0x1322   66 7c f6             jl     0x000000000ffffff9
++0x1325   f1                   int1   
++0x1326   4a 7f 24             jnle   0x0000000010000027
++0x1329   5b                   pop    rbx
++0x132a   17...??              <INVALID>
++0x132b   3c 16                cmp    al, 0x16
++0x132d   21 c9                and    ecx, ecx
++0x132f   54                   push   rsp
++0x1330   ce...??              <INVALID>
++0x1331   fb                   sti    
++0x1332   a1 6d 2e 5f 2e 3a 1b mov    eax, dword ptr [0xf5d91b3a2e5f2e6d]
+     d9 f5
++0x133b   45 67 6f             outsd  
++0x133e   60...??              <INVALID>
++0x133f   41 de bf fd fb a7 19 fidivr word ptr [r15+0x19a7fbfd]
++0x1346   c4 07...??           <INVALID>
++0x1347   07...??              <INVALID>
++0x1348   e5 3a                in     eax, 0x3a
++0x134a   1e...??              <INVALID>
++0x134b   69 a5 9a af 2d b2 78 imul   esp, dword ptr [rbp-0x4dd25066], 0xf9b49a78
+     9a b4 f9
++0x1355   65 82 9d a0 f2 52 94...??  <INVALID>
+     eb
++0x1356   82 9d a0 f2 52 94 eb...??  <INVALID>
++0x1357   9d                   popfq  
++0x1358   a0 f2 52 94 eb b6 5a mov    al, byte ptr [0xa6e05ab6eb9452f2]
+     e0 a6
++0x1361   08 2d f1 31 1f c9    or     <rel> byte ptr [0xffffffffd91f31f7], ch
++0x1367   1b c5                sbb    eax, ebp
++0x1369   6e                   outsb  
++0x136a   16...??              <INVALID>
++0x136b   de d8...??           <INVALID>
++0x136c   d8 bb db 3f 5a d1    fdivr  dword ptr [rbx-0x2ea5c025]
++0x1372   23 eb                and    ebp, ebx
++0x1374   a4                   movsb  
++0x1375   a7                   cmpsd  
++0x1376   36 ea f7 94 d0 f3 f8...??  <INVALID>
+     69
++0x1377   ea f7 94 d0 f3 f8 69...??  <INVALID>
++0x1378   f7 94 d0 f3 f8 69 04 not    dword ptr [rax+rdx*8+0x0469f8f3]
++0x137f   4e 87 26             xchg   qword ptr [rsi], r12
++0x1382   19 71 6f             sbb    dword ptr [rcx+0x6f], esi
++0x1385   58                   pop    rax
++0x1386   1e...??              <INVALID>
++0x1387   8e a0 5c d5 d6 cf    mov    fs, dword ptr [rax-0x30292aa4]
++0x138d   63 62 4d             movsxd esp, dword ptr [rdx+0x4d]
++0x1390   88 f2                mov    dl, dh
++0x1392   af                   scasd  
++0x1393   d7                   xlatb  
++0x1394   2a 98 85 29 d5 cd    sub    bl, byte ptr [rax-0x322ad67b]
++0x139a   a1 94 51 13 9a 65 3e mov    eax, dword ptr [0x35653e659a135194]
+     65 35
++0x13a3   5f                   pop    rdi
++0x13a4   ef                   out    eax, dx
++0x13a5   07...??              <INVALID>
++0x13a6   ce...??              <INVALID>
++0x13a7   50                   push   rax
++0x13a8   8e d5                mov    ss, ebp
++0x13aa   6c                   insb   
++0x13ab   80 64 85 ac 9f       and    byte ptr [rbp+rax*4-0x54], 0x9f
++0x13b0   54                   push   rsp
++0x13b1   bc 68 0f 95 e5       mov    esp, 0xe5950f68
++0x13b6   a0 12 fe fe 9d ad f2 mov    al, byte ptr [0xdd71f2ad9dfefe12]
+     71 dd
++0x13bf   72 09                jb     0x000000001000000b
++0x13c1   d7                   xlatb  
++0x13c2   09 23                or     dword ptr [rbx], esp
++0x13c4   61...??              <INVALID>
++0x13c5   7a a9                jp     0x000000000fffffab
++0x13c7   ff 9c ca 96 7f 41 9a call   
++0x13ce   be 0e 14 ba e6       mov    esi, 0xe6ba140e
++0x13d3   e1 8a                loope  0x000000000fffff8c
++0x13d5   f1                   int1   
++0x13d6   00 da                add    dl, bl
++0x13d8   d3 32                shl    dword ptr [rdx], cl
++0x13da   38 fc                cmp    ah, bh
++0x13dc   15 35 b3 e8 6c       adc    eax, 0x6ce8b335
++0x13e1   d0 e9                shr    cl, 0x01
++0x13e3   2e 36 d9 95 37 eb 48 fst    dword ptr [ss:rbp+0x1548eb37]
+     15
++0x13eb   14 16                adc    al, 0x16
++0x13ed   c7 06 6e e9 74 9b    mov    dword ptr [rsi], 0x9b74e96e
++0x13f3   a0 53 10 d8 d7 bd e0 mov    al, byte ptr [0xecffe0bdd7d81053]
+     ff ec
++0x13fc   a9 23 c9 72 f0       test   eax, 0xf072c923
++0x1401   d9 4f 99...??        <INVALID>
++0x1402   4f 99                cdq    
++0x1404   af                   scasd  
++0x1405   6a a3                push   0xa3
++0x1407   a5                   movsd  
++0x1408   71 f0                jno    0x000000000ffffff2
++0x140a   f1                   int1   
++0x140b   9d                   popfq  
++0x140c   45 c2 d8 07          ret    0x07d8
++0x1410   42 3e 66 b8 54 3b    mov    ax, 0x3b54
++0x1416   69 21 c3 5b ee 49    imul   esp, dword ptr [rcx], 0x49ee5bc3
++0x141c   5c                   pop    rsp
++0x141d   25 8a f2 0e 91       and    eax, 0x910ef28a
++0x1422   4c 96                xchg   rsi, rax
++0x1424   f9                   stc    
++0x1425   7e 13                jle    0x0000000010000015
++0x1427   eb 99                jmp    0x000000000fffff9b
++0x1429   85 2d d4 4c 2f cd    test   <rel> dword ptr [0xffffffffdd2f4cda], ebp
++0x142f   be 0a 61 06 a8       mov    esi, 0xa806610a
++0x1434   d9 b8 95 41 22 e1    fnstcw word ptr [rax-0x1eddbe6b]
++0x143a   92                   xchg   edx, eax
++0x143b   a7                   cmpsd  
++0x143c   44 56                push   rsi
++0x143e   19 2c b8             sbb    dword ptr [rax+rdi*4], ebp
++0x1441   c9                   leave  
++0x1442   d9 82 9d 43 57 a5    fld    dword ptr [rdx-0x5aa8bc63]
++0x1448   8b 8f 89 63 2c 58    mov    ecx, dword ptr [rdi+0x582c6389]
++0x144e   a9 bd ae d7 22       test   eax, 0x22d7aebd
++0x1453   78 d4                js     0x000000000fffffd6
++0x1455   c1 ba ec 2c 33 4f df sar    dword ptr [rdx+0x4f332cec], 0xdf
++0x145c   30 e1                xor    cl, ah
++0x145e   a2 45 a8 31 9c 4c 42 mov    byte ptr [0xa401424c9c31a845], al
+     01 a4
++0x1467   33 f4                xor    esi, esp
++0x1469   64 ba ff fd 76 8d    mov    edx, 0x8d76fdff
++0x146f   3b b9 17 37 84 55    cmp    edi, dword ptr [rcx+0x55843717]
++0x1475   93                   xchg   ebx, eax
++0x1476   c8 ab d9 fc          enter  0xd9ab, 0xfc
++0x147a   d2 55 3f             rcl    byte ptr [rbp+0x3f], cl
++0x147d   11 82 b3 dd 4f 83    adc    dword ptr [rdx-0x7cb0224d], eax
++0x1483   2c 3a                sub    al, 0x3a
++0x1485   a0 d2 70 bb c6 21 42 mov    al, byte ptr [0x75ce4221c6bb70d2]
+     ce 75
++0x148e   41 5c                pop    r12
++0x1490   08 69 9f             or     byte ptr [rcx-0x61], ch
++0x1493   80 ef e4             sub    bh, 0xe4
++0x1496   e6 e3                out    0xe3, al
++0x1498   88 e4                mov    ah, ah
++0x149a   3a cb                cmp    cl, bl
++0x149c   95                   xchg   ebp, eax
++0x149d   78 43                js     0x0000000010000045
++0x149f   05 38 be 7d 64       add    eax, 0x647dbe38
++0x14a4   9e                   sahf   
++0x14a5   93                   xchg   ebx, eax
++0x14a6   e1 35                loope  0x0000000010000037
++0x14a8   9d                   popfq  
++0x14a9   a0 05 6e 29 7f 3d 3b mov    al, byte ptr [0x3b3c3b3d7f296e05]
+     3c 3b
++0x14b2   28 2c b8             sub    byte ptr [rax+rdi*4], ch
++0x14b5   44 9b                fwait  
++0x14b7   93                   xchg   ebx, eax
++0x14b8   21 31                and    dword ptr [rcx], esi
++0x14ba   b9 a3 bf b9 26       mov    ecx, 0x26b9bfa3
++0x14bf   b4 0e                mov    ah, 0x0e
++0x14c1   c4 42 e1...??        <INVALID>
++0x14c2   42 e1 7e             loope  0x0000000010000081
++0x14c5   55                   push   rbp
++0x14c6   a4                   movsb  
++0x14c7   f0 1c 1a...??        <INVALID>
++0x14c8   1c 1a                sbb    al, 0x1a
++0x14ca   7c e1                jl     0x000000000fffffe3
++0x14cc   2c 21                sub    al, 0x21
++0x14ce   84 13                test   byte ptr [rbx], dl
++0x14d0   03 c6                add    eax, esi
++0x14d2   6d                   insd   
++0x14d3   45 f5                cmc    
++0x14d5   35 66 af f5 58       xor    eax, 0x58f5af66
++0x14da   c8 a7 00 a9          enter  0x00a7, 0xa9
++0x14de   83 0a ba             or     dword ptr [rdx], 0xba
++0x14e1   46 bb ca f0 7f e8    mov    ebx, 0xe87ff0ca
++0x14e7   ea 5e 4d 69 3c 2f d2...??  <INVALID>
++0x14e8   5e                   pop    rsi
++0x14e9   4d 69 3c 2f d2 47 f1 imul   r15, qword ptr [r15+rbp], 0xe5f147d2
+     e5
++0x14f1   77 ee                jnbe   0x000000000ffffff0
++0x14f3   24 f7                and    al, 0xf7
++0x14f5   ab                   stosd  
++0x14f6   20 a0 ac ba 0f a0    and    byte ptr [rax-0x5ff04554], ah
++0x14fc   56                   push   rsi
++0x14fd   75 bd                jnz    0x000000000fffffbf
++0x14ff   74 3f                jz     0x0000000010000041
++0x1501   ce...??              <INVALID>
++0x1502   64 46 a0 76 84 97 f9 mov    al, byte ptr [fs:0x77cac247f9978476]
+     47 c2 ca 77
++0x150d   15 9d 32 7e f1       adc    eax, 0xf17e329d
++0x1512   b5 ad                mov    ch, 0xad
++0x1514   f2 4e de 21          fisub  word ptr [rcx]
++0x1518   b7 95                mov    bh, 0x95
++0x151a   af                   scasd  
++0x151b   bd fa 74 29 e4       mov    ebp, 0xe42974fa
++0x1520   2e c1 70 7d b5       shl    dword ptr [cs:rax+0x7d], 0xb5
++0x1525   62 12...??           <INVALID>
++0x1526   12 6f d8             adc    ch, byte ptr [rdi-0x28]
++0x1529   ed                   in     eax, dx
++0x152a   dc 51 a9             fcom   qword ptr [rcx-0x57]
++0x152d   3e 98                cwde   
++0x152f   62 52 7f...??        <INVALID>
++0x1530   52                   push   rdx
++0x1531   7f 50                jnle   0x0000000010000052
++0x1533   4f 99                cdq    
++0x1535   36 07...??           <INVALID>
++0x1536   07...??              <INVALID>
++0x1537   fe 04 a4             inc    byte ptr [rsp]
++0x153a   08 ec                or     ah, ch
++0x153c   1f...??              <INVALID>
++0x153d   8d 22                lea    esp, [rdx]
++0x153f   be 7f 71 fa 22       mov    esi, 0x22fa717f
++0x1544   f0 b6 1d...??        <INVALID>
++0x1545   b6 1d                mov    dh, 0x1d
++0x1547   72 81                jb     0x000000000fffff83
++0x1549   2d 1d 1f 72 e4       sub    eax, 0xe4721f1d
++0x154e   3e ee                out    al, dx
++0x1550   9b                   fwait  
++0x1551   12 81 2e 9d 15 7e    adc    al, byte ptr [rcx+0x7e159d2e]
++0x1557   86 48 59             xchg   byte ptr [rax+0x59], cl
++0x155a   4d ca f8 d5          retf   0xd5f8
++0x155e   f0 88 c4...??        <INVALID>
++0x155f   88 c4                mov    ah, al
++0x1561   9f                   lahf   
++0x1562   a2 ea f5 4b 39 a8 45 mov    byte ptr [0x11e645a8394bf5ea], al
+     e6 11
++0x156b   da 3b                fidivr dword ptr [rbx]
++0x156d   39 e4                cmp    esp, esp
++0x156f   34 01                xor    al, 0x01
++0x1571   9f                   lahf   
++0x1572   58                   pop    rax
++0x1573   1b 29                sbb    ebp, dword ptr [rcx]
++0x1575   5c                   pop    rsp
++0x1576   eb 29                jmp    0x000000001000002b
++0x1578   11 b8 a9 cc 82 ae    adc    dword ptr [rax-0x517d3357], edi
++0x157e   bb 5a 77 81 40       mov    ebx, 0x4081775a
++0x1583   36 51                push   rcx
++0x1585   22 b7 1c 0a 7a 16    and    dh, byte ptr [rdi+0x167a0a1c]
++0x158b   d5 60...??           <INVALID>
++0x158c   60...??              <INVALID>
++0x158d   0c 84                or     al, 0x84
++0x158f   3d ad 82 a7 9c       cmp    eax, 0x9ca782ad
++0x1594   87 4a 02             xchg   dword ptr [rdx+0x02], ecx
++0x1597   00 a9 d7 6e a0 0d    add    byte ptr [rcx+0x0da06ed7], ch
++0x159d   08 91 cb 52 5f 15    or     byte ptr [rcx+0x155f52cb], dl
++0x15a3   27...??              <INVALID>
++0x15a4   2d e3 79 5c 5c       sub    eax, 0x5c5c79e3
++0x15a9   53                   push   rbx
++0x15aa   63 2c 8f             movsxd ebp, dword ptr [rdi+rcx*4]
++0x15ad   79 a9                jns    0x000000000fffffab
++0x15af   49 b8 a2 84 a9 a9 a8 mov    r8, 0xea11d2a8a9a984a2
+     d2 11 ea
++0x15b9   8a 3e                mov    bh, byte ptr [rsi]
++0x15bb   30 c2                xor    dl, al
++0x15bd   86 32                xchg   byte ptr [rdx], dh
++0x15bf   66 b4 c3             mov    ah, 0xc3
++0x15c2   9b                   fwait  
++0x15c3   00 4b fc             add    byte ptr [rbx-0x04], cl
++0x15c6   43 f2 cf             iretd  
++0x15c9   32 30                xor    dh, byte ptr [rax]
++0x15cb   6e                   outsb  
++0x15cc   8c 2b                mov    dword ptr [rbx], gs
++0x15ce   c9                   leave  
++0x15cf   01 da                add    edx, ebx
++0x15d1   41 52                push   r10
++0x15d3   c3                   ret    
++0x15d4   2c 73                sub    al, 0x73
++0x15d6   74 ec                jz     0x000000000fffffee
++0x15d8   74 07                jz     0x0000000010000009
++0x15da   f7 a9 07 36 42 c6    imul   edx, eax, dword ptr [rcx-0x39bdc9f9]
++0x15e0   a5                   movsd  
++0x15e1   5f                   pop    rdi
++0x15e2   fd                   std    
++0x15e3   15 cb 70 a3 a9       adc    eax, 0xa9a370cb
++0x15e8   dc 3f                fdivr  qword ptr [rdi]
++0x15ea   80 e4 e2             and    ah, 0xe2
++0x15ed   ab                   stosd  
++0x15ee   4f d4 42...??        <INVALID>
++0x15ef   d4 42...??           <INVALID>
++0x15f0   42 7f e9             jnle   0x000000000fffffec
++0x15f3   a4                   movsb  
++0x15f4   85 e7                test   edi, esp
++0x15f6   44 10 df             adc    dil, r11l
++0x15f9   67 e9 7c 12 9e 3f    jmp    0x000000004f9e1282
++0x15ff   cc                   int3   
++0x1600   7a 17                jp     0x0000000010000019
++0x1602   e5 ad                in     eax, 0xad
++0x1604   d1 9f 0a d6 c0 7c    rcr    dword ptr [rdi+0x7cc0d60a], 0x01
++0x160a   c7 dc f5 13 d5 88...??  <INVALID>
++0x160b   dc f5                fdivr  st5, st0
++0x160d   13 d5                adc    edx, ebp
++0x160f   88 d0                mov    al, dl
++0x1611   2a bd 04 cb 86 93    sub    bh, byte ptr [rbp-0x6c7934fc]
++0x1617   19 79 06             sbb    dword ptr [rcx+0x06], edi
++0x161a   14 e4                adc    al, 0xe4
++0x161c   07...??              <INVALID>
++0x161d   b3 1e                mov    bl, 0x1e
++0x161f   c1 58 a5 13          rcr    dword ptr [rax-0x5b], 0x13
++0x1623   06...??              <INVALID>
++0x1624   25 65 fe 7f f3       and    eax, 0xf37ffe65
++0x1629   57                   push   rdi
++0x162a   92                   xchg   edx, eax
++0x162b   25 38 9b 99 23       and    eax, 0x23999b38
++0x1630   c0 15 ca 1e 18 01 2f rcl    <rel> byte ptr [0x0000000011181ed1], 0x2f
++0x1637   a6                   cmpsb  
++0x1638   9a fd 22 70 ae 08 be...??  <INVALID>
++0x1639   fd                   std    
++0x163a   22 70 ae             and    dh, byte ptr [rax-0x52]
++0x163d   08 be d6 77 6a da    or     byte ptr [rsi-0x2595882a], bh
++0x1643   33 75 cd             xor    esi, dword ptr [rbp-0x33]
++0x1646   71 ca                jno    0x000000000fffffcc
++0x1648   f4                   hlt    
++0x1649   f8                   clc    
++0x164a   16...??              <INVALID>
++0x164b   80 d4 af             adc    ah, 0xaf
++0x164e   61...??              <INVALID>
++0x164f   4f a2 af dc d1 6e a0 mov    byte ptr [0xeabb59a06ed1dcaf], al
+     59 bb ea
++0x1659   56                   push   rsi
++0x165a   40 3b dd             cmp    ebx, ebp
++0x165d   d9 b2 44 85 71 91    fnstenv [rdx-0x6e8e7abc]
++0x1663   1d 29 9a 7a c4       sbb    eax, 0xc47a9a29
++0x1668   8d 2a                lea    ebp, [rdx]
++0x166a   9d                   popfq  
++0x166b   71 b9                jno    0x000000000fffffbb
++0x166d   e4 9e                in     al, 0x9e
++0x166f   56                   push   rsi
++0x1670   67 f4                hlt    
++0x1672   27...??              <INVALID>
++0x1673   a2 e7 ff 5e 90 93 c8 mov    byte ptr [0x1efcc893905effe7], al
+     fc 1e
++0x167c   ff d4                call   
++0x167e   ba e0 9e e9 a5       mov    edx, 0xa5e99ee0
++0x1683   d6...??              <INVALID>
++0x1684   68 48 85 ec 67       push   0x67ec8548
++0x1689   5d                   pop    rbp
++0x168a   3a 67 30             cmp    ah, byte ptr [rdi+0x30]
++0x168d   e6 e6                out    0xe6, al
++0x168f   2b 06                sub    eax, dword ptr [rsi]
++0x1691   d6...??              <INVALID>
++0x1692   7d 1f                jnl    0x0000000010000021
++0x1694   49 f8                clc    
++0x1696   3e 80 54 e2 e2 3e    adc    byte ptr [ds:rdx-0x1e], 0x3e
++0x169c   f3 dd 1f             fstp   qword ptr [rdi]
++0x169f   51                   push   rcx
++0x16a0   96                   xchg   esi, eax
++0x16a1   a8 96                test   al, 0x96
++0x16a3   f9                   stc    
++0x16a4   74 bb                jz     0x000000000fffffbd
++0x16a6   be 04 b6 cf 49       mov    esi, 0x49cfb604
++0x16ab   0d 5e 44 a9 6e       or     eax, 0x6ea9445e
++0x16b0   6a 77                push   0x77
++0x16b2   e2 ae                loop   0x000000000fffffb0
++0x16b4   3f...??              <INVALID>
++0x16b5   97                   xchg   edi, eax
++0x16b6   c6 74 fe 1c f0...??  <INVALID>
++0x16b7   74 fe                jz     0x0000000010000000
++0x16b9   1c f0                sbb    al, 0xf0
++0x16bb   2f...??              <INVALID>
++0x16bc   3d a8 32 56 ce       cmp    eax, 0xce5632a8
++0x16c1   30 12                xor    byte ptr [rdx], dl
++0x16c3   2e 95                xchg   ebp, eax
++0x16c5   34 70                xor    al, 0x70
++0x16c7   f6 23                mul    ax, byte ptr [rbx]
++0x16c9   c1 75 13 62          shl    dword ptr [rbp+0x13], 0x62
++0x16cd   ae                   scasb  
++0x16ce   c5 77 b8...??        <INVALID>
++0x16cf   77 b8                jnbe   0x000000000fffffba
++0x16d1   6f                   outsd  
++0x16d2   92                   xchg   edx, eax
++0x16d3   53                   push   rbx
++0x16d4   e6 d9                out    0xd9, al
++0x16d6   66 32 1b             xor    bl, byte ptr [rbx]
++0x16d9   36 f1                int1   
++0x16db   69 ae 18 81 41 06 83 imul   ebp, dword ptr [rsi+0x06418118], 0xbb5f1483
+     14 5f bb
++0x16e5   2b 15 c7 70 c2 3e    sub    edx, <rel> dword ptr [0x000000004ec270cd]
++0x16eb   34 0c                xor    al, 0x0c
++0x16ed   cf                   iretd  
++0x16ee   9f                   lahf   
++0x16ef   61...??              <INVALID>
++0x16f0   ca 74 04             retf   0x0474
++0x16f3   70 95                jo     0x000000000fffff97
++0x16f5   3a 95 b5 5e 9b 43    cmp    dl, byte ptr [rbp+0x439b5eb5]
++0x16fb   f1                   int1   
++0x16fc   c7 5f b6 82 8a b1 18...??  <INVALID>
++0x16fd   5f                   pop    rdi
++0x16fe   b6 82                mov    dh, 0x82
++0x1700   8a b1 18 10 38 bd    mov    dh, byte ptr [rcx-0x42c7efe8]
++0x1706   90                   nop    
++0x1707   24 da                and    al, 0xda
++0x1709   77 05                jnbe   0x0000000010000007
++0x170b   a0 a2 37 4e 07 4c c8 mov    al, byte ptr [0x8c8dc84c074e37a2]
+     8d 8c
++0x1714   2d 87 ad 2c 9e       sub    eax, 0x9e2cad87
++0x1719   f7 4e 5e d7 92 9f a7 test   dword ptr [rsi+0x5e], 0xa79f92d7
++0x1720   33 7c 9f 35          xor    edi, dword ptr [rdi+rbx*4+0x35]
++0x1724   b4 ef                mov    ah, 0xef
++0x1726   86 67 8c             xchg   byte ptr [rdi-0x74], ah
++0x1729   5c                   pop    rsp
++0x172a   da a0 a0 69 7e 5f    fisub  dword ptr [rax+0x5f7e69a0]
++0x1730   10 6d 56             adc    byte ptr [rbp+0x56], ch
++0x1733   71 47                jno    0x0000000010000049
++0x1735   bc 95 d0 48 e1       mov    esp, 0xe148d095
++0x173a   c2 bf 5a             ret    0x5abf
++0x173d   80 88 e2 b9 44 83 a1 or     byte ptr [rax-0x7cbb461e], 0xa1
++0x1744   ba a1 24 30 79       mov    edx, 0x793024a1
++0x1749   ee                   out    al, dx
++0x174a   05 64 04 76 38       add    eax, 0x38760464
++0x174f   c2 50 f7             ret    0xf750
++0x1752   8b 85 f8 4e 33 22    mov    eax, dword ptr [rbp+0x22334ef8]
++0x1758   37...??              <INVALID>
++0x1759   1b 92 64 61 a3 62    sbb    edx, dword ptr [rdx+0x62a36164]
++0x175f   5e                   pop    rsi
++0x1760   ca ab 78             retf   0x78ab
++0x1763   56                   push   rsi
++0x1764   aa                   stosb  
++0x1765   09 2f                or     dword ptr [rdi], ebp
++0x1767   0a 6c f6 67          or     ch, byte ptr [rsi+rsi*8+0x67]
++0x176b   fd                   std    
++0x176c   73 c0                jnb    0x000000000fffffc2
++0x176e   f7 47 8a 86 86 14 4d test   dword ptr [rdi-0x76], 0x4d148686
++0x1775   fb                   sti    
++0x1776   5a                   pop    rdx
++0x1777   64 ac                lodsb  fs
++0x1779   f5                   cmc    
++0x177a   01 0d 4f 6f ad 09    add    <rel> dword ptr [0x0000000019ad6f55], ecx
++0x1780   f4                   hlt    
++0x1781   13 c8                adc    ecx, eax
++0x1783   8a dd                mov    bl, ch
++0x1785   ea 6b 5c f3 3b 27 e2...??  <INVALID>
++0x1786   6b 5c f3 3b 27       imul   ebx, dword ptr [rbx+rsi*8+0x3b], 0x27
++0x178b   e2 d8                loop   0x000000000fffffda
++0x178d   f1                   int1   
++0x178e   3e 0d 7b 6a 3b 72    or     eax, 0x723b6a7b
++0x1794   5d                   pop    rbp
++0x1795   41 e8 fd 51 e5 61    call   0x0000000071e55203
++0x179b   cc                   int3   
++0x179c   1b 16                sbb    edx, dword ptr [rsi]
++0x179e   9d                   popfq  
++0x179f   a4                   movsb  
++0x17a0   d8 12                fcom   dword ptr [rdx]
++0x17a2   d3 9d ab 55 54 e4    rcr    dword ptr [rbp-0x1babaa55], cl
++0x17a8   f7 cd 09 5a e6 9b    test   ebp, 0x9be65a09
++0x17ae   11 02                adc    dword ptr [rdx], eax
++0x17b0   ee                   out    al, dx
++0x17b1   de 55 36             ficom  word ptr [rbp+0x36]
++0x17b4   f7 ed                imul   edx, eax, ebp
++0x17b6   61...??              <INVALID>
++0x17b7   be 12 91 2b 42       mov    esi, 0x422b9112
++0x17bc   09 c2                or     edx, eax
++0x17be   86 9e 0b 09 71 55    xchg   byte ptr [rsi+0x5571090b], bl
++0x17c4   06...??              <INVALID>
++0x17c5   6e                   outsb  
++0x17c6   7f 2a                jnle   0x000000001000002c
++0x17c8   a7                   cmpsd  
++0x17c9   bc 93 0b b4 3f       mov    esp, 0x3fb40b93
++0x17ce   a7                   cmpsd  
++0x17cf   94                   xchg   esp, eax
++0x17d0   a4                   movsb  
++0x17d1   bc 64 04 c3 da       mov    esp, 0xdac30464
++0x17d6   f3 43 9c             pushfq 
++0x17d9   0c 20                or     al, 0x20
++0x17db   be b2 97 8c 2a       mov    esi, 0x2a8c97b2
++0x17e0   69 d4 6e a7 0c bb    imul   edx, esp, 0xbb0ca76e
++0x17e6   78 ce                js     0x000000000fffffd0
++0x17e8   74 01                jz     0x0000000010000003
++0x17ea   4c ff 69 f6          jmp    
++0x17ee   a1 ef 7e 39 a1 76 76 mov    eax, dword ptr [0x9d9f7676a1397eef]
+     9f 9d
++0x17f7   a3 c8 f5 35 5b a8 e9 mov    dword ptr [0xa095e9a85b35f5c8], eax
+     95 a0
++0x1800   a9 3e 7b bd c8       test   eax, 0xc8bd7b3e
++0x1805   a3 77 1f 84 8d 79 25 mov    dword ptr [0x85ab25798d841f77], eax
+     ab 85
++0x180e   0b 03                or     eax, dword ptr [rbx]
++0x1810   ad                   lodsd  
++0x1811   29 91 eb 80 99 90    sub    dword ptr [rcx-0x6f667f15], edx
++0x1817   6b 25 8c fa 5c 93 c4 imul   esp, <rel> dword ptr [0xffffffffa35cfa93], 0xc4
++0x181e   c3                   ret    
++0x181f   6c                   insb   
++0x1820   32 2f                xor    ch, byte ptr [rdi]
++0x1822   ca d4 d0             retf   0xd0d4
++0x1825   90                   nop    
++0x1826   be 8d e2 6d 85       mov    esi, 0x856de28d
++0x182b   9e                   sahf   
++0x182c   63 a6 ab f2 0f a1    movsxd esp, dword ptr [rsi-0x5ef00d55]
++0x1832   17...??              <INVALID>
++0x1833   b1 00                mov    cl, 0x00
++0x1835   14 6c                adc    al, 0x6c
++0x1837   cb                   retf   
++0x1838   3f...??              <INVALID>
++0x1839   0e...??              <INVALID>
++0x183a   40 43 6a e6          push   0xe6
++0x183e   4d 72 fc             jb     0x000000000fffffff
++0x1841   eb cb                jmp    0x000000000fffffcd
++0x1843   c0 f8 ef             sar    al, 0xef
++0x1846   e0 9e                loopne 0x000000000fffffa0
++0x1848   0b c5                or     eax, ebp
++0x184a   cc                   int3   
++0x184b   fe 4b d3             dec    byte ptr [rbx-0x2d]
++0x184e   aa                   stosb  
++0x184f   9b                   fwait  
++0x1850   28 97 22 09 ae f0    sub    byte ptr [rdi-0x0f51f6de], dl
++0x1856   35 e6 65 a7 04       xor    eax, 0x04a765e6
++0x185b   6d                   insd   
++0x185c   8f c0                pop    rax
++0x185e   95                   xchg   ebp, eax
++0x185f   9c                   pushfq 
++0x1860   5d                   pop    rbp
++0x1861   25 f2 81 c3 0d       and    eax, 0x0dc381f2
++0x1866   c1 f4 76             shl    esp, 0x76
++0x1869   50                   push   rax
++0x186a   42 08 0d 4e 4f d9 a8 or     <rel> byte ptr [0xffffffffb8d94f55], cl
++0x1871   ae                   scasb  
++0x1872   3c 68                cmp    al, 0x68
++0x1874   5c                   pop    rsp
++0x1875   af                   scasd  
++0x1876   00 35 eb 76 76 73    add    <rel> byte ptr [0x00000000837676f1], dh
++0x187c   10 9d 02 74 4a 0f    adc    byte ptr [rbp+0x0f4a7402], bl
++0x1882   b0 de                mov    al, 0xde
++0x1884   e7 4d                out    0x4d, eax
++0x1886   8b 9a fe 12 5c 0a    mov    ebx, dword ptr [rdx+0x0a5c12fe]
++0x188c   73 6b                jnb    0x000000001000006d
++0x188e   9e                   sahf   
++0x188f   d7                   xlatb  
++0x1890   ae                   scasb  
++0x1891   f4                   hlt    
++0x1892   b9 74 4b 7d 45       mov    ecx, 0x457d4b74
++0x1897   ee                   out    al, dx
++0x1898   79 51                jns    0x0000000010000053
++0x189a   f5                   cmc    
++0x189b   63 3f                movsxd edi, dword ptr [rdi]
++0x189d   1d c9 0e 25 20       sbb    eax, 0x20250ec9
++0x18a2   91                   xchg   ecx, eax
++0x18a3   bf c7 76 b1 43       mov    edi, 0x43b176c7
++0x18a8   cf                   iretd  
++0x18a9   60...??              <INVALID>
++0x18aa   d5 e6...??           <INVALID>
++0x18ab   e6 9c                out    0x9c, al
++0x18ad   fd                   std    
++0x18ae   5c                   pop    rsp
++0x18af   34 91                xor    al, 0x91
++0x18b1   53                   push   rbx
++0x18b2   d2 28                shr    byte ptr [rax], cl
++0x18b4   a3 75 bb 5b f3 75 ec mov    dword ptr [0x470aec75f35bbb75], eax
+     0a 47
++0x18bd   33 e7                xor    esp, edi
++0x18bf   27...??              <INVALID>
++0x18c0   d3 c0                rol    eax, cl
++0x18c2   84 53 3f             test   byte ptr [rbx+0x3f], dl
++0x18c5   57                   push   rdi
++0x18c6   25 bb ba b2 32       and    eax, 0x32b2babb
++0x18cb   56                   push   rsi
++0x18cc   e7 5e                out    0x5e, eax
++0x18ce   70 62                jo     0x0000000010000064
++0x18d0   6f                   outsd  
++0x18d1   f9                   stc    
++0x18d2   56                   push   rsi
++0x18d3   e2 0d                loop   0x000000001000000f
++0x18d5   29 82 02 3d d7 ed    sub    dword ptr [rdx-0x1228c2fe], eax
++0x18db   21 b8 b9 1e 82 4f    and    dword ptr [rax+0x4f821eb9], edi
++0x18e1   92                   xchg   edx, eax
++0x18e2   2f...??              <INVALID>
++0x18e3   7a 0d                jp     0x000000001000000f
++0x18e5   62 75 28...??        <INVALID>
++0x18e6   75 28                jnz    0x000000001000002a
++0x18e8   c0 82 36 e5 53 94 4f rol    byte ptr [rdx-0x6bac1aca], 0x4f
++0x18ef   73 f2                jnb    0x000000000ffffff4
++0x18f1   93                   xchg   ebx, eax
++0x18f2   5f                   pop    rdi
++0x18f3   e7 95                out    0x95, eax
++0x18f5   d4 90...??           <INVALID>
++0x18f6   90                   nop    
++0x18f7   c2 6e 47             ret    0x476e
++0x18fa   cf                   iretd  
++0x18fb   d3 ba 23 9f 48 87    sar    dword ptr [rdx-0x78b760dd], cl
++0x1901   82 84 b8 c2 a2 c8 5f...??  <INVALID>
+     5f
++0x1902   84 b8 c2 a2 c8 5f    test   byte ptr [rax+0x5fc8a2c2], bh
++0x1908   5f                   pop    rdi
++0x1909   ee                   out    al, dx
++0x190a   71 a8                jno    0x000000000fffffaa
++0x190c   24 a1                and    al, 0xa1
++0x190e   92                   xchg   edx, eax
++0x190f   d5 93...??           <INVALID>
++0x1910   93                   xchg   ebx, eax
++0x1911   9b                   fwait  
++0x1912   74 66                jz     0x0000000010000068
++0x1914   db 81 71 93 00 0c    fild   dword ptr [rcx+0x0c009371]
++0x191a   2b 88 70 f2 79 79    sub    ecx, dword ptr [rax+0x7979f270]
++0x1920   89 cf                mov    edi, ecx
++0x1922   db b4 8c 88 75 09 cc...??  <INVALID>
++0x1923   b4 8c                mov    ah, 0x8c
++0x1925   88 75 09             mov    byte ptr [rbp+0x09], dh
++0x1928   cc                   int3   
++0x1929   05 fb b9 7a 87       add    eax, 0x877ab9fb
++0x192e   cf                   iretd  
++0x192f   1d df 45 93 4d       sbb    eax, 0x4d9345df
++0x1934   e4 b7                in     al, 0xb7
++0x1936   f8                   clc    
++0x1937   50                   push   rax
++0x1938   33 0c 25 3a 07 10 04 xor    ecx, dword ptr [0x0410073a]
++0x193f   d3 92 a6 ab ce ac    rcl    dword ptr [rdx-0x5331545a], cl
++0x1945   5e                   pop    rsi
++0x1946   7b 5c                jnp    0x000000001000005e
++0x1948   68 b8 2a 47 80       push   0x80472ab8
++0x194d   10 b1 3e b0 43 c1    adc    byte ptr [rcx-0x3ebc4fc2], dh
++0x1953   58                   pop    rax
++0x1954   54                   push   rsp
++0x1955   68 23 03 34 6d       push   0x6d340323
++0x195a   ee                   out    al, dx
++0x195b   bc 32 6e c3 01       mov    esp, 0x01c36e32
++0x1960   33 6f 0e             xor    ebp, dword ptr [rdi+0x0e]
++0x1963   be c7 f9 42 4c       mov    esi, 0x4c42f9c7
++0x1968   35 60 09 34 d6       xor    eax, 0xd6340960
++0x196d   0e...??              <INVALID>
++0x196e   5c                   pop    rsp
++0x196f   01 0c cb             add    dword ptr [rbx+rcx*8], ecx
++0x1972   3d 1a 19 b1 32       cmp    eax, 0x32b1191a
++0x1977   4e 1a 90 2c a1 37 45 sbb    r10l, byte ptr [rax+0x4537a12c]
++0x197e   98                   cwde   
++0x197f   b9 7a 67 38 0c       mov    ecx, 0x0c38677a
++0x1984   38 b0 a5 b9 fc 61    cmp    byte ptr [rax+0x61fcb9a5], dh
++0x198a   19 9e 68 74 23 a3    sbb    dword ptr [rsi-0x5cdc8b98], ebx
++0x1990   3b 42 4e             cmp    eax, dword ptr [rdx+0x4e]
++0x1993   74 5b                jz     0x000000001000005d
++0x1995   e2 c5                loop   0x000000000fffffc7
++0x1997   3a 3d 21 5d 0f 48    cmp    bh, <rel> byte ptr [0x00000000580f5d27]
++0x199d   02 4c a9 e3          add    cl, byte ptr [rcx+rbp*4-0x1d]
++0x19a1   17...??              <INVALID>
++0x19a2   88 c9                mov    cl, cl
++0x19a4   89 8c 1c 35 ae 79 d6 mov    dword ptr [rsp+rbx-0x298651cb], ecx
++0x19ab   ed                   in     eax, dx
++0x19ac   30 6b 2f             xor    byte ptr [rbx+0x2f], ch
++0x19af   c5 fe...??           <INVALID>
++0x19b0   fe 52 84...??        <INVALID>
++0x19b1   52                   push   rdx
++0x19b2   84 be d2 80 4e a2    test   byte ptr [rsi-0x5db17f2e], bh
++0x19b8   6f                   outsd  
++0x19b9   aa                   stosb  
++0x19ba   64 00 14 20          add    byte ptr [fs:rax], dl
++0x19be   aa                   stosb  
++0x19bf   71 e0                jno    0x000000000fffffe2
++0x19c1   63 4e 57             movsxd ecx, dword ptr [rsi+0x57]
++0x19c4   3f...??              <INVALID>
++0x19c5   b0 d4                mov    al, 0xd4
++0x19c7   2b f8                sub    edi, eax
++0x19c9   ca 58 3e             retf   0x3e58
++0x19cc   39 56 e2             cmp    dword ptr [rsi-0x1e], edx
++0x19cf   fb                   sti    
++0x19d0   6e                   outsb  
++0x19d1   94                   xchg   esp, eax
++0x19d2   e1 79                loope  0x000000001000007b
++0x19d4   7a 65                jp     0x0000000010000067
++0x19d6   c4 1c 0f...??        <INVALID>
++0x19d7   1c 0f                sbb    al, 0x0f
++0x19d9   27...??              <INVALID>
++0x19da   8f 48 ca...??        <INVALID>
++0x19db   48 ca 6c bf          retf   0xbf6c
++0x19df   88 c6                mov    dh, al
++0x19e1   8b 9d 6f 63 fd 30    mov    ebx, dword ptr [rbp+0x30fd636f]
++0x19e7   5d                   pop    rbp
++0x19e8   32 f9                xor    bh, cl
++0x19ea   88 6a 93             mov    byte ptr [rdx-0x6d], ch
++0x19ed   91                   xchg   ecx, eax
++0x19ee   fb                   sti    
++0x19ef   d3 79 40             sar    dword ptr [rcx+0x40], cl
++0x19f2   bb 97 9f 7e 1a       mov    ebx, 0x1a7e9f97
++0x19f7   67 fb                sti    
++0x19f9   00 06                add    byte ptr [rsi], al
++0x19fb   5d                   pop    rbp
++0x19fc   da c2                fcmovb st0, st2
++0x19fe   d1 72 cc             shl    dword ptr [rdx-0x34], 0x01
++0x1a01   52                   push   rdx
++0x1a02   64 57                push   rdi
++0x1a04   56                   push   rsi
++0x1a05   56                   push   rsi
++0x1a06   ff a4 32 68 0a 11 78 jmp    
++0x1a0d   19 c3                sbb    ebx, eax
++0x1a0f   d0 4f c8             ror    byte ptr [rdi-0x38], 0x01
++0x1a12   d3 23                shl    dword ptr [rbx], cl
++0x1a14   ff 9c 51 f7 08 0c be call   
++0x1a1b   80 3c 9d c8 f9 79 92 cmp    byte ptr [rbx*4-0x6d860638], 0x49
+     49
++0x1a23   4e 70 29             jo     0x000000001000002c
++0x1a26   ac                   lodsb  
++0x1a27   0c ca                or     al, 0xca
++0x1a29   ea aa 7f 7d 9c 62 84...??  <INVALID>
++0x1a2a   aa                   stosb  
++0x1a2b   7f 7d                jnle   0x000000001000007f
++0x1a2d   9c                   pushfq 
++0x1a2e   62 84 4e 43 41 f7 5c...??  <INVALID>
++0x1a2f   84 4e 43             test   byte ptr [rsi+0x43], cl
++0x1a32   41 f7 5c eb 04       neg    dword ptr [r11+rbp*8+0x04]
++0x1a37   ab                   stosd  
++0x1a38   3e bc 82 5c 47 40    mov    esp, 0x40475c82
++0x1a3e   6d                   insd   
++0x1a3f   08 84 d0 5c 26 9a d4 or     byte ptr [rax+rdx*8-0x2b65d9a4], al
++0x1a46   62 07...??           <INVALID>
++0x1a47   07...??              <INVALID>
++0x1a48   91                   xchg   ecx, eax
++0x1a49   5e                   pop    rsi
++0x1a4a   90                   nop    
++0x1a4b   5c                   pop    rsp
++0x1a4c   38 a6 95 60 ea c7    cmp    byte ptr [rsi-0x38159f6b], ah
++0x1a52   da c1                fcmovb st0, st1
++0x1a54   91                   xchg   ecx, eax
++0x1a55   59                   pop    rcx
++0x1a56   9c                   pushfq 
++0x1a57   66 f5                cmc    
++0x1a59   e8 ef 54 e6 8c       call   0xffffffff9ce654f4
++0x1a5e   46 49 b3 c9          mov    r11l, 0xc9
++0x1a62   27...??              <INVALID>
++0x1a63   70 64                jo     0x0000000010000066
++0x1a65   4b db f5             fcomi  st0, st5
++0x1a68   f2 6e                outsb  
++0x1a6a   da ee...??           <INVALID>
++0x1a6b   ee                   out    al, dx
++0x1a6c   0a 78 39             or     bh, byte ptr [rax+0x39]
++0x1a6f   2a 20                sub    ah, byte ptr [rax]
++0x1a71   b2 b3                mov    dl, 0xb3
++0x1a73   2c 98                sub    al, 0x98
++0x1a75   52                   push   rdx
++0x1a76   88 60 0e             mov    byte ptr [rax+0x0e], ah
++0x1a79   d2 a7 6c 86 b5 6e    shl    byte ptr [rdi+0x6eb5866c], cl
++0x1a7f   45 78 29             js     0x000000001000002c
++0x1a82   a4                   movsb  
++0x1a83   94                   xchg   esp, eax
++0x1a84   3c 46                cmp    al, 0x46
++0x1a86   a1 a4 d8 68 85 05 de mov    eax, dword ptr [0xb8fbde058568d8a4]
+     fb b8
++0x1a8f   4f 6d                insd   
++0x1a91   b0 0a                mov    al, 0x0a
++0x1a93   70 70                jo     0x0000000010000072
++0x1a95   31 e3                xor    ebx, esp
++0x1a97   1f...??              <INVALID>
++0x1a98   78 ec                js     0x000000000fffffee
++0x1a9a   90                   nop    
++0x1a9b   9c                   pushfq 
++0x1a9c   75 4b                jnz    0x000000001000004d
++0x1a9e   56                   push   rsi
++0x1a9f   ca 4b 69             retf   0x694b
++0x1aa2   d2 52 aa             rcl    byte ptr [rdx-0x56], cl
++0x1aa5   ee                   out    al, dx
++0x1aa6   af                   scasd  
++0x1aa7   c5 7f 0e...??        <INVALID>
++0x1aa8   7f 0e                jnle   0x0000000010000010
++0x1aaa   9e                   sahf   
++0x1aab   69 90 d1 e1 de 6d ee imul   edx, dword ptr [rax+0x6ddee1d1], 0xbec11bee
+     1b c1 be
++0x1ab5   18 7f 39             sbb    byte ptr [rdi+0x39], bh
++0x1ab8   43 a0 12 8f 4c 98 4e mov    al, byte ptr [0x40299b4e984c8f12]
+     9b 29 40
++0x1ac2   b8 26 a8 3a 01       mov    eax, 0x013aa826
++0x1ac7   b3 79                mov    bl, 0x79
++0x1ac9   77 d2                jnbe   0x000000000fffffd4
++0x1acb   98                   cwde   
++0x1acc   e9 06 f2 e3 f3       jmp    0x0000000003e3f20b
++0x1ad1   ba 22 3c d6 31       mov    edx, 0x31d63c22
++0x1ad6   4a 9a 64 14 17 74 46...??  <INVALID>
+     bb
++0x1ad7   9a 64 14 17 74 46 bb...??  <INVALID>
++0x1ad8   64 14 17             adc    al, 0x17
++0x1adb   74 46                jz     0x0000000010000048
++0x1add   bb 9d 98 79 c3       mov    ebx, 0xc379989d
++0x1ae2   d8 0f                fmul   dword ptr [rdi]
++0x1ae4   89 53 cb             mov    dword ptr [rbx-0x35], edx
++0x1ae7   7e d8                jle    0x000000000fffffda
++0x1ae9   ad                   lodsd  
++0x1aea   17...??              <INVALID>
++0x1aeb   cf                   iretd  
++0x1aec   88 aa cd 82 15 78    mov    byte ptr [rdx+0x781582cd], ch
++0x1af2   0b 3a                or     edi, dword ptr [rdx]
++0x1af4   3e 83 84 0a e1 66 45 add    dword ptr [ds:rdx+rcx-0x4dba991f], 0x4e
+     b2 4e
++0x1afd   02 90 65 e5 fc cf    add    dl, byte ptr [rax-0x30031a9b]
++0x1b03   8f ff...??           <INVALID>
++0x1b04   ff 4f 73             dec    dword ptr [rdi+0x73]
++0x1b07   cc                   int3   
++0x1b08   d4 6b...??           <INVALID>
++0x1b09   6b 1b b2             imul   ebx, dword ptr [rbx], 0xb2
++0x1b0c   e6 5d                out    0x5d, al
++0x1b0e   86 bb 10 14 80 de    xchg   byte ptr [rbx-0x217febf0], bh
++0x1b14   f1                   int1   
++0x1b15   3c 91                cmp    al, 0x91
++0x1b17   8b 51 46             mov    edx, dword ptr [rcx+0x46]
++0x1b1a   1c 68                sbb    al, 0x68
++0x1b1c   6e                   outsb  
++0x1b1d   14 91                adc    al, 0x91
++0x1b1f   b0 09                mov    al, 0x09
++0x1b21   e9 06 1c b2 f0       jmp    0x0000000000b21c0b
++0x1b26   ff 12                call   
++0x1b28   05 bb 8c 44 1b       add    eax, 0x1b448cbb
++0x1b2d   1f...??              <INVALID>
++0x1b2e   2e b5 79             mov    ch, 0x79
++0x1b31   e7 ca                out    0xca, eax
++0x1b33   c7 c2 67 7a b0 8a    mov    edx, 0x8ab07a67
++0x1b39   d5 51...??           <INVALID>
++0x1b3a   51                   push   rcx
++0x1b3b   c1 ef b7             shr    edi, 0xb7
++0x1b3e   31 13                xor    dword ptr [rbx], edx
++0x1b40   5c                   pop    rsp
++0x1b41   8e f8...??           <INVALID>
++0x1b42   f8                   clc    
++0x1b43   d3 b1 04 e3 4f 86    shl    dword ptr [rcx-0x79b01cfc], cl
++0x1b49   21 1d 79 05 54 5e    and    <rel> dword ptr [0x000000006e54057f], ebx
++0x1b4f   e7 e2                out    0xe2, eax
++0x1b51   32 76 7a             xor    dh, byte ptr [rsi+0x7a]
++0x1b54   04 a3                add    al, 0xa3
++0x1b56   64 9a 44 e6 50 fd 9a...??  <INVALID>
+     46
++0x1b57   9a 44 e6 50 fd 9a 46...??  <INVALID>
++0x1b58   44 e6 50             out    0x50, al
++0x1b5b   fd                   std    
++0x1b5c   9a 46 2e 86 b9 fd bf...??  <INVALID>
++0x1b5d   46 2e 86 b9 fd bf 23 xchg   byte ptr [cs:rcx-0x1fdc4003], bh
+     e0
++0x1b65   00 ad db 8c c8 13    add    byte ptr [rbp+0x13c88cdb], ch
++0x1b6b   4e d9 b1 89 41 b5 96 fnstenv [rcx-0x694abe77]
++0x1b72   1a e1                sbb    ah, cl
++0x1b74   97                   xchg   edi, eax
++0x1b75   10 e9                adc    cl, ch
++0x1b77   e4 61                in     al, 0x61
++0x1b79   e3 c4                jrcxz  0x000000000fffffc6
++0x1b7b   a5                   movsd  
++0x1b7c   bf 96 7b ba 0f       mov    edi, 0x0fba7b96
++0x1b81   95                   xchg   ebp, eax
++0x1b82   4c 25 bd aa 8b f9    and    rax, 0xf98baabd
++0x1b88   de 7b 9b             fidivr word ptr [rbx-0x65]
++0x1b8b   10 8b 2a d5 62 af    adc    byte ptr [rbx-0x509d2ad6], cl
++0x1b91   ff b9 78 d3 a8 15...??  <INVALID>
++0x1b92   b9 78 d3 a8 15       mov    ecx, 0x15a8d378
++0x1b97   48 fb                sti    
++0x1b99   4b aa                stosb  
++0x1b9b   0a f5                or     dh, ch
++0x1b9d   d9 25 b9 8a 44 85    fldenv <rel> [0xffffffff95448abf]
++0x1ba3   42 ec                in     al
++0x1ba5   02 bb 89 47 5b 66    add    bh, byte ptr [rbx+0x665b4789]
++0x1bab   88 37                mov    byte ptr [rdi], dh
++0x1bad   be 3b 77 1f a2       mov    esi, 0xa21f773b
++0x1bb2   42 1b 43 f5          sbb    eax, dword ptr [rbx-0x0b]
++0x1bb6   37...??              <INVALID>
++0x1bb7   ce...??              <INVALID>
++0x1bb8   56                   push   rsi
++0x1bb9   3d b6 84 b4 68       cmp    eax, 0x68b484b6
++0x1bbe   5e                   pop    rsi
++0x1bbf   3c 52                cmp    al, 0x52
++0x1bc1   1a 5b 3a             sbb    bl, byte ptr [rbx+0x3a]
++0x1bc4   bc 63 53 d6 ae       mov    esp, 0xaed65363
++0x1bc9   b8 39 d1 63 5e       mov    eax, 0x5e63d139
++0x1bce   b6 e9                mov    dh, 0xe9
++0x1bd0   6e                   outsb  
++0x1bd1   f2 66 19 08          sbb    word ptr [rax], cx
++0x1bd5   af                   scasd  
++0x1bd6   62 81 95 10 47 57...??  <INVALID>
++0x1bd7   81 95 10 47 57 84 c1 adc    dword ptr [rbp-0x7ba8b8f0], 0x48c539c1
+     39 c5 48
++0x1be1   f4                   hlt    
++0x1be2   f2 86 5c 46 ac       xchg   byte ptr [rsi+rax*2-0x54], bl
++0x1be7   0d 5e 0c 65 d8       or     eax, 0xd8650c5e
++0x1bec   75 e9                jnz    0x000000000fffffeb
++0x1bee   38 fb                cmp    bl, bh
++0x1bf0   09 49 d8             or     dword ptr [rcx-0x28], ecx
++0x1bf3   c2 2c d2             ret    0xd22c
++0x1bf6   8f a8 5f ae 87 4d...??  <INVALID>
++0x1bf7   a8 5f                test   al, 0x5f
++0x1bf9   ae                   scasb  
++0x1bfa   87 4d 07             xchg   dword ptr [rbp+0x07], ecx
++0x1bfd   1f...??              <INVALID>
++0x1bfe   48 a4                movsb  
++0x1c00   c3                   ret    
++0x1c01   7c db                jl     0x000000000fffffdd
++0x1c03   5a                   pop    rdx
++0x1c04   fd                   std    
++0x1c05   bd f4 7b 4c ab       mov    ebp, 0xab4c7bf4
++0x1c0a   4f 60...??           <INVALID>
++0x1c0b   60...??              <INVALID>
++0x1c0c   34 7a                xor    al, 0x7a
++0x1c0e   c2 9c f2             ret    0xf29c
++0x1c11   8d ce...??           <INVALID>
++0x1c12   ce...??              <INVALID>
++0x1c13   29 a6 93 ba 87 66    sub    dword ptr [rsi+0x6687ba93], esp
++0x1c19   54                   push   rsp
++0x1c1a   b2 bb                mov    dl, 0xbb
++0x1c1c   d9 26                fldenv [rsi]
++0x1c1e   1c a1                sbb    al, 0xa1
++0x1c20   db bd 8d d8 97 f5    fstp   tbyte ptr [rbp-0x0a682773]
++0x1c26   a7                   cmpsd  
++0x1c27   b0 3f                mov    al, 0x3f
++0x1c29   ec                   in     al
++0x1c2a   fa                   cli    
++0x1c2b   94                   xchg   esp, eax
++0x1c2c   bf cb 43 05 b3       mov    edi, 0xb30543cb
++0x1c31   5b                   pop    rbx
++0x1c32   c1 5b 30 67          rcr    dword ptr [rbx+0x30], 0x67
++0x1c36   2a 75 1d             sub    dh, byte ptr [rbp+0x1d]
++0x1c39   5f                   pop    rdi
++0x1c3a   84 6b 6e             test   byte ptr [rbx+0x6e], ch
++0x1c3d   44 87 a3 b5 a8 7a 65 xchg   dword ptr [rbx+0x657aa8b5], r12d
++0x1c44   d6...??              <INVALID>
++0x1c45   80 a1 eb 56 31 70 b5 and    byte ptr [rcx+0x703156eb], 0xb5
++0x1c4c   1a 2d 65 0c 86 16    sbb    ch, <rel> byte ptr [0x0000000026860c6b]
++0x1c52   39 50 e3             cmp    dword ptr [rax-0x1d], edx
++0x1c55   8e 82 db cc cb 25    mov    es, dword ptr [rdx+0x25cbccdb]
++0x1c5b   36 31 a0 d0 cf 97 2c xor    dword ptr [ss:rax+0x2c97cfd0], esp
++0x1c62   f5                   cmc    
++0x1c63   49 e2 21             loop   0x0000000010000024
++0x1c66   e0 ef                loopne 0x000000000ffffff1
++0x1c68   7b d6                jnp    0x000000000fffffd8
++0x1c6a   8f b4 f7 9c a3 4b 41...??  <INVALID>
++0x1c6b   b4 f7                mov    ah, 0xf7
++0x1c6d   9c                   pushfq 
++0x1c6e   a3 4b 41 6d 15 90 d6 mov    dword ptr [0x85e0d690156d414b], eax
+     e0 85
++0x1c77   a4                   movsb  
++0x1c78   2f...??              <INVALID>
++0x1c79   5d                   pop    rbp
++0x1c7a   e7 5c                out    0x5c, eax
++0x1c7c   0e...??              <INVALID>
++0x1c7d   8f fa...??           <INVALID>
++0x1c7e   fa                   cli    
++0x1c7f   57                   push   rdi
++0x1c80   13 fa                adc    edi, edx
++0x1c82   1b 85 04 37 33 8d    sbb    eax, dword ptr [rbp-0x72ccc8fc]
++0x1c88   2e 8e c2             mov    es, edx
++0x1c8b   cd 30                int    0x30
++0x1c8d   6f                   outsd  
++0x1c8e   a1 de 89 e4 6d e1 2b mov    eax, dword ptr [0x750e2be16de489de]
+     0e 75
++0x1c97   cc                   int3   
++0x1c98   1f...??              <INVALID>
++0x1c99   7d f9                jnl    0x000000000ffffffb
++0x1c9b   05 e6 a2 db 03       add    eax, 0x03dba2e6
++0x1ca0   3a f5                cmp    dh, ch
++0x1ca2   48 bf 79 0f c9 63 6d mov    rdi, 0x77705d6d63c90f79
+     5d 70 77
++0x1cac   0f 1c 4f 65          nop    dword ptr [rdi+0x65]
++0x1cb0   74 2a                jz     0x000000001000002c
++0x1cb2   12 cc                adc    cl, ah
++0x1cb4   37...??              <INVALID>
++0x1cb5   1f...??              <INVALID>
++0x1cb6   a2 26 ab b4 d7 93 cf mov    byte ptr [0x19d6cf93d7b4ab26], al
+     d6 19
++0x1cbf   3f...??              <INVALID>
++0x1cc0   47 e4 2c             in     al, 0x2c
++0x1cc3   59                   pop    rcx
++0x1cc4   5b                   pop    rbx
++0x1cc5   f3 65 6d             rep insd 
++0x1cc8   af                   scasd  
++0x1cc9   82 06 bd...??        <INVALID>
++0x1cca   06...??              <INVALID>
++0x1ccb   bd 07 b7 c9 c6       mov    ebp, 0xc6c9b707
++0x1cd0   7a 99                jp     0x000000000fffff9b
++0x1cd2   d7                   xlatb  
++0x1cd3   ef                   out    eax, dx
++0x1cd4   cb                   retf   
++0x1cd5   24 70                and    al, 0x70
++0x1cd7   55                   push   rbp
++0x1cd8   0e...??              <INVALID>
++0x1cd9   5f                   pop    rdi
++0x1cda   3d 82 7e 91 67       cmp    eax, 0x67917e82
++0x1cdf   0b e3                or     esp, ebx
++0x1ce1   cb                   retf   
++0x1ce2   e4 3a                in     al, 0x3a
++0x1ce4   71 33                jno    0x0000000010000035
++0x1ce6   38 79 b9             cmp    byte ptr [rcx-0x47], bh
++0x1ce9   e4 35                in     al, 0x35
++0x1ceb   b8 bd b9 64 52       mov    eax, 0x5264b9bd
++0x1cf0   54                   push   rsp
++0x1cf1   70 2b                jo     0x000000001000002d
++0x1cf3   48 b4 89             mov    spl, 0x89
++0x1cf6   d3 40 9b             rol    dword ptr [rax-0x65], cl
++0x1cf9   e5 2d                in     eax, 0x2d
++0x1cfb   b4 36                mov    ah, 0x36
++0x1cfd   c6 8e fb 52 c4 c0 ac...??  <INVALID>
++0x1cfe   8e fb...??           <INVALID>
++0x1cff   fb                   sti    
++0x1d00   52                   push   rdx
++0x1d01   c4 c0...??           <INVALID>
++0x1d02   c0 ac 0c 9d b8 4d fc shr    byte ptr [rsp+rcx-0x03b24763], 0x0b
+     0b
++0x1d0a   17...??              <INVALID>
++0x1d0b   1a 97 37 a6 f1 b3    sbb    dl, byte ptr [rdi-0x4c0e59c9]
++0x1d11   34 89                xor    al, 0x89
++0x1d13   cd 35                int    0x35
++0x1d15   9e                   sahf   
++0x1d16   e7 81                out    0x81, eax
++0x1d18   62 f0...??           <INVALID>
++0x1d19   f0 fb...??           <INVALID>
++0x1d1a   fb                   sti    
++0x1d1b   91                   xchg   ecx, eax
++0x1d1c   cb                   retf   
++0x1d1d   c4 ec...??           <INVALID>
++0x1d1e   ec                   in     al
++0x1d1f   e9 8c f3 e4 db       jmp    0xffffffffebe4f391
++0x1d24   51                   push   rcx
++0x1d25   4b 71 ab             jno    0x000000000fffffae
++0x1d28   d9 ae bf c7 d2 12    fldcw  word ptr [rsi+0x12d2c7bf]
++0x1d2e   0f c8                bswap  eax
++0x1d30   bf 99 48 35 d7       mov    edi, 0xd7354899
++0x1d35   d4 bf...??           <INVALID>
++0x1d36   bf 98 1f 64 b0       mov    edi, 0xb0641f98
++0x1d3b   5b                   pop    rbx
++0x1d3c   1f...??              <INVALID>
++0x1d3d   c0 0e ed             ror    byte ptr [rsi], 0xed
++0x1d40   09 f3                or     ebx, esi
++0x1d42   d8 7f 25             fdivr  dword ptr [rdi+0x25]
++0x1d45   86 44 c1 b9          xchg   byte ptr [rcx+rax*8-0x47], al
++0x1d49   29 0d 7f 70 de fc    sub    <rel> dword ptr [0x000000000cde7085], ecx
++0x1d4f   73 16                jnb    0x0000000010000018
++0x1d51   73 09                jnb    0x000000001000000b
++0x1d53   9b                   fwait  
++0x1d54   a7                   cmpsd  
++0x1d55   27...??              <INVALID>
++0x1d56   ba 96 e5 bc 8d       mov    edx, 0x8dbce596
++0x1d5b   7e 2d                jle    0x000000001000002f
++0x1d5d   64 cf                iretd  
++0x1d5f   fd                   std    
++0x1d60   ac                   lodsb  
++0x1d61   19 aa 2b c6 6c 20    sbb    dword ptr [rdx+0x206cc62b], ebp
++0x1d67   c5 2e...??           <INVALID>
++0x1d68   2e 9c                pushfq 
++0x1d6a   eb d2                jmp    0x000000000fffffd4
++0x1d6c   06...??              <INVALID>
++0x1d6d   07...??              <INVALID>
++0x1d6e   bf 71 38 cd 82       mov    edi, 0x82cd3871
++0x1d73   ba a6 9f 4f b2       mov    edx, 0xb24f9fa6
++0x1d78   81 9f 75 83 7a 3d 4b sbb    dword ptr [rdi+0x3d7a8375], 0x7f3dbb4b
+     bb 3d 7f
++0x1d82   8d 73 0b             lea    esi, [rbx+0x0b]
++0x1d85   2c 7b                sub    al, 0x7b
++0x1d87   66 77 39             jnbe   0x000000001000003c
++0x1d8a   c1 ff 0f             sar    edi, 0x0f
++0x1d8d   99                   cdq    
++0x1d8e   98                   cwde   
++0x1d8f   a1 6c 0b 6c 77 b6 a7 mov    eax, dword ptr [0x4902a7b6776c0b6c]
+     02 49
++0x1d98   7e 54                jle    0x0000000010000056
++0x1d9a   8a 28                mov    ch, byte ptr [rax]
++0x1d9c   21 f2                and    edx, esi
++0x1d9e   ae                   scasb  
++0x1d9f   dc d7                fcom   st0, st7
++0x1da1   dd 1c d0             fstp   qword ptr [rax+rdx*8]
++0x1da4   ac                   lodsb  
++0x1da5   54                   push   rsp
++0x1da6   75 8d                jnz    0x000000000fffff8f
++0x1da8   ee                   out    al, dx
++0x1da9   9b                   fwait  
++0x1daa   12 ab 7d 93 d5 56    adc    ch, byte ptr [rbx+0x56d5937d]
++0x1db0   a9 e4 3f 71 54       test   eax, 0x54713fe4
++0x1db5   e0 b4                loopne 0x000000000fffffb6
++0x1db7   33 85 5e 7a fd 72    xor    eax, dword ptr [rbp+0x72fd7a5e]
++0x1dbd   eb 05                jmp    0x0000000010000007
++0x1dbf   e4 fd                in     al, 0xfd
++0x1dc1   ce...??              <INVALID>
++0x1dc2   67 b8 ca 07 9c 3c    mov    eax, 0x3c9c07ca
++0x1dc8   c4 dd...??           <INVALID>
++0x1dc9   dd 49 c3             fisttp qword ptr [rcx-0x3d]
++0x1dcc   22 d8                and    bl, al
++0x1dce   3e 8c eb             mov    ebx, gs
++0x1dd1   31 37                xor    dword ptr [rdi], esi
++0x1dd3   16...??              <INVALID>
++0x1dd4   51                   push   rcx
++0x1dd5   99                   cdq    
++0x1dd6   b2 83                mov    dl, 0x83
++0x1dd8   f1                   int1   
++0x1dd9   c2 54 0b             ret    0x0b54
++0x1ddc   ee                   out    al, dx
++0x1ddd   3c 92                cmp    al, 0x92
++0x1ddf   96                   xchg   esi, eax
++0x1de0   18 ce                sbb    dh, cl
++0x1de2   8b 5a b7             mov    ebx, dword ptr [rdx-0x49]
++0x1de5   07...??              <INVALID>
++0x1de6   d7                   xlatb  
++0x1de7   ac                   lodsb  
++0x1de8   6b a8 2b 03 94 a5 18 imul   ebp, dword ptr [rax-0x5a6bfcd5], 0x18
++0x1def   90                   nop    
++0x1df0   f1                   int1   
++0x1df1   e0 06                loopne 0x0000000010000008
++0x1df3   4f b1 1a             mov    r9l, 0x1a
++0x1df6   8a e8                mov    ch, al
++0x1df8   73 fc                jnb    0x000000000ffffffe
++0x1dfa   44 57                push   rdi
++0x1dfc   2c e4                sub    al, 0xe4
++0x1dfe   0a e3                or     ah, bl
++0x1e00   93                   xchg   ebx, eax
++0x1e01   1f...??              <INVALID>
++0x1e02   dc 62 ba             fsub   qword ptr [rdx-0x46]
++0x1e05   e6 d4                out    0xd4, al
++0x1e07   30 e0                xor    al, ah
++0x1e09   cc                   int3   
++0x1e0a   7a 42                jp     0x0000000010000044
++0x1e0c   21 f1                and    ecx, esi
++0x1e0e   36 e8 e5 fc 12 31    call   0x000000004112fceb
++0x1e14   5a                   pop    rdx
++0x1e15   6f                   outsd  
++0x1e16   07...??              <INVALID>
++0x1e17   09 90 86 5f b4 04    or     dword ptr [rax+0x04b45f86], edx
++0x1e1d   38 43 ed             cmp    byte ptr [rbx-0x13], al
++0x1e20   77 e3                jnbe   0x000000000fffffe5
++0x1e22   5c                   pop    rsp
++0x1e23   76 0c                jbe    0x000000001000000e
++0x1e25   7d f3                jnl    0x000000000ffffff5
++0x1e27   c8 45 14 e4          enter  0x1445, 0xe4
++0x1e2b   c4 7c a6 5e...??     <INVALID>
++0x1e2c   7c a6                jl     0x000000000fffffa8
++0x1e2e   5e                   pop    rsi
++0x1e2f   90                   nop    
++0x1e30   7e 44                jle    0x0000000010000046
++0x1e32   37...??              <INVALID>
++0x1e33   77 70                jnbe   0x0000000010000072
++0x1e35   77 f2                jnbe   0x000000000ffffff4
++0x1e37   09 0b                or     dword ptr [rbx], ecx
++0x1e39   17...??              <INVALID>
++0x1e3a   a5                   movsd  
++0x1e3b   30 c2                xor    dl, al
++0x1e3d   80 07 83             add    byte ptr [rdi], 0x83
++0x1e40   f2 eb 86             jmp    0x000000000fffff89
++0x1e43   e9 01 02 5d e2       jmp    0xfffffffff25d0206
++0x1e48   8b 52 25             mov    edx, dword ptr [rdx+0x25]
++0x1e4b   b2 fe                mov    dl, 0xfe
++0x1e4d   5e                   pop    rsi
++0x1e4e   f1                   int1   
++0x1e4f   85 fd                test   ebp, edi
++0x1e51   eb f4                jmp    0x000000000ffffff6
++0x1e53   a5                   movsd  
++0x1e54   bf 0b 21 cb 3a       mov    edi, 0x3acb210b
++0x1e59   70 ea                jo     0x000000000fffffec
++0x1e5b   b4 1d                mov    ah, 0x1d
++0x1e5d   20 7c 10 37          and    byte ptr [rax+rdx+0x37], bh
++0x1e61   ee                   out    al, dx
++0x1e62   0d 33 83 d1 8e       or     eax, 0x8ed18333
+#else
++0x0000   bf ae 4f 80 95       mov    edi, 0x95804fae
++0x0005   f1                   int1   
++0x0006   fa                   cli    
++0x0007   43                   inc    ebx
++0x0008   8d 00                lea    eax, [eax]
++0x000a   0c eb                or     al, 0xeb
++0x000c   37                   aaa    
++0x000d   05 9d 3a ca cf       add    eax, 0xcfca3a9d
++0x0012   35 35 32 a1 fc       xor    eax, 0xfca13235
++0x0017   65 83 00 f8          add    dword ptr [gs:eax], 0xf8
++0x001b   ef                   out    eax, dx
++0x001c   ea 11 6e 63 78 c2 60 jmp    0x60c2:0x78636e11
++0x0023   6e                   outsb   
++0x0024   8e 79 92...??        <INVALID>
++0x0025   79 92                jns    0x0fffff94
++0x0027   7b 83                jnp    0x0fffff85
++0x0029   96                   xchg   esi, eax
++0x002a   06                   push   es
++0x002b   9f                   lahf   
++0x002c   02 dd                add    bl, ch
++0x002e   e6 9f                out    0x9f, al
++0x0030   d1 79 9f             sar    dword ptr [ecx-0x61], 0x01
++0x0033   ea 60 3f e3 85 60 f0 jmp    0xf060:0x85e33f60
++0x003a   7c 45                jl     0x10000047
++0x003c   b9 22 25 d3 c0       mov    ecx, 0xc0d32522
++0x0041   e7 70                out    0x70, eax
++0x0043   56                   push   esi
++0x0044   17                   pop    ss
++0x0045   37                   aaa    
++0x0046   ec                   in     al
++0x0047   91                   xchg   ecx, eax
++0x0048   23 ee                and    ebp, esi
++0x004a   51                   push   ecx
++0x004b   b1 c8                mov    cl, 0xc8
++0x004d   1c 78                sbb    al, 0x78
++0x004f   ce                   into   
++0x0050   d1 ed                shr    ebp, 0x01
++0x0052   12 7c 21 7b          adc    bh, byte ptr [ecx+0x7b]
++0x0056   8f 40 ff             pop    dword ptr [eax-0x01]
++0x0059   72 e3                jb     0x0fffffe5
++0x005b   d7                   xlatb   
++0x005c   62 83 84 d1 f7 99    bound  eax, qword ptr [ebx+0x99f7d184]
++0x0062   f4                   hlt    
++0x0063   6d                   insd    
++0x0064   97                   xchg   edi, eax
++0x0065   8e 2e                mov    gs, dword ptr [esi]
++0x0067   e1 08                loope  0x1000000a
++0x0069   b0 96                mov    al, 0x96
++0x006b   10 a3 5e 58 6c 92    adc    byte ptr [ebx+0x926c585e], ah
++0x0071   c2 85 7e             ret    0x7e85
++0x0074   02 ed                add    ch, ch
++0x0076   17                   pop    ss
++0x0077   27                   daa    
++0x0078   96                   xchg   esi, eax
++0x0079   b3 e4                mov    bl, 0xe4
++0x007b   8e b1 07 18 af fd...??  <INVALID>
++0x007c   b1 07                mov    cl, 0x07
++0x007e   18 af fd fe 04 ab    sbb    byte ptr [edi+0xab04fefd], ch
++0x0084   81 1c 4e 44 6f 88 9c sbb    dword ptr [esi+ecx*2], 0x9c886f44
++0x008b   2e 57                push   edi
++0x008d   b9 48 a4 09 d4       mov    ecx, 0xd409a448
++0x0092   14 e9                adc    al, 0xe9
++0x0094   6d                   insd    
++0x0095   28 d9                sub    cl, bl
++0x0097   dd 2e...??           <INVALID>
++0x0098   2e f5                cmc    
++0x009a   62 00                bound  eax, qword ptr [eax]
++0x009c   1f                   pop    ds
++0x009d   59                   pop    ecx
++0x009e   88 2f                mov    byte ptr [edi], ch
++0x00a0   92                   xchg   edx, eax
++0x00a1   d6                   salc   
++0x00a2   21 21                and    dword ptr [ecx], esp
++0x00a4   53                   push   ebx
++0x00a5   76 37                jbe    0x10000039
++0x00a7   89 39                mov    dword ptr [ecx], edi
++0x00a9   d2 f9                sar    cl, cl
++0x00ab   00 4b 5e             add    byte ptr [ebx+0x5e], cl
++0x00ae   45                   inc    ebp
++0x00af   cf                   iretd   
++0x00b0   27                   daa    
++0x00b1   8e 2a                mov    gs, dword ptr [edx]
++0x00b3   15 e1 e2 07 11       adc    eax, 0x1107e2e1
++0x00b8   50                   push   eax
++0x00b9   4b                   dec    ebx
++0x00ba   f2 65 65 a4          movsb   gs
++0x00be   ae                   scasb   
++0x00bf   67 44                inc    esp
++0x00c1   73 6d                jnb    0x1000006f
++0x00c3   c3                   ret    
++0x00c4   3d 75 14 e1 5f       cmp    eax, 0x5fe11475
++0x00c9   ab                   stosd   
++0x00ca   6d                   insd    
++0x00cb   31 b4 a4 98 00 f8 69 xor    dword ptr [esp+0x69f80098], esi
++0x00d2   33 c5                xor    eax, ebp
++0x00d4   cc                   int3   
++0x00d5   6a c8                push   0xc8
++0x00d7   3b b7 58 29 c3 9b    cmp    esi, dword ptr [edi+0x9bc32958]
++0x00dd   27                   daa    
++0x00de   32 80 bf c8 51 a1    xor    al, byte ptr [eax+0xa151c8bf]
++0x00e4   a3 e8 a0 5f 70       mov    dword ptr [0x705fa0e8], eax
++0x00e9   ed                   in     eax, dx
++0x00ea   01 de                add    esi, ebx
++0x00ec   93                   xchg   ebx, eax
++0x00ed   80 71 4f 37          xor    byte ptr [ecx+0x4f], 0x37
++0x00f1   77 b3                jnbe   0x0fffffb5
++0x00f3   d7                   xlatb   
++0x00f4   66 d2 20             shl    byte ptr [eax], cl
++0x00f7   b7 91                mov    bh, 0x91
++0x00f9   55                   push   ebp
++0x00fa   56                   push   esi
++0x00fb   90                   nop    
++0x00fc   54                   push   esp
++0x00fd   b3 83                mov    bl, 0x83
++0x00ff   65 8f f2...??        <INVALID>
++0x0100   8f f2...??           <INVALID>
++0x0101   f2 59                pop    ecx
++0x0103   bb e3 c4 58 cf       mov    ebx, 0xcf58c4e3
++0x0108   8b 48 20             mov    ecx, dword ptr [eax+0x20]
++0x010b   76 38                jbe    0x1000003a
++0x010d   66 4d                dec    bp
++0x010f   78 83                js     0x0fffff85
++0x0111   32 68 9b             xor    ch, byte ptr [eax-0x65]
++0x0114   9e                   sahf   
++0x0115   49                   dec    ecx
++0x0116   0a be 9a dd c2 db    or     bh, byte ptr [esi+0xdbc2dd9a]
++0x011c   25 da fc 34 81       and    eax, 0x8134fcda
++0x0121   86 3a                xchg   byte ptr [edx], bh
++0x0123   b0 d0                mov    al, 0xd0
++0x0125   94                   xchg   esp, eax
++0x0126   16                   push   ss
++0x0127   2d a4 27 fb e7       sub    eax, 0xe7fb27a4
++0x012c   f5                   cmc    
++0x012d   25 f6 47 5f 7f       and    eax, 0x7f5f47f6
++0x0132   71 64                jno    0x10000066
++0x0134   81 66 87 80 c9 d0 84 and    dword ptr [esi-0x79], 0x84d0c980
++0x013b   99                   cdq    
++0x013c   66 f7 77 bc          div    dx, ax, word ptr [edi-0x44]
++0x0140   2a dd                sub    bl, ch
++0x0142   54                   push   esp
++0x0143   08 91 4a 9a d3 ad    or     byte ptr [ecx+0xadd39a4a], dl
++0x0149   7b 9f                jnp    0x0fffffa1
++0x014b   a5                   movsd   
++0x014c   2c e0                sub    al, 0xe0
++0x014e   9b                   fwait  
++0x014f   16                   push   ss
++0x0150   c5 c5...??           <INVALID>
++0x0151   c5 a8 7b cc a7 e4    lds    ebp, [eax+0xe4a7cc7b]
++0x0157   ac                   lodsb   
++0x0158   b2 d8                mov    dl, 0xd8
++0x015a   54                   push   esp
++0x015b   01 cf                add    edi, ecx
++0x015d   d6                   salc   
++0x015e   16                   push   ss
++0x015f   0f cd                bswap  ebp
++0x0161   1d c4 63 24 13       sbb    eax, 0x132463c4
++0x0166   28 81 f5 a3 b4 98    sub    byte ptr [ecx+0x98b4a3f5], al
++0x016c   df d1                fstp   st1, st0
++0x016e   0c 40                or     al, 0x40
++0x0170   ef                   out    eax, dx
++0x0171   09 61 b9             or     dword ptr [ecx-0x47], esp
++0x0174   73 ec                jnb    0x0fffffee
++0x0176   e8 b8 81 85 95       call   0xa58581bd
++0x017b   fe a6 da 11 a4 ea...??  <INVALID>
++0x017c   a6                   cmpsb   
++0x017d   da 11                ficom  dword ptr [ecx]
++0x017f   a4                   movsb   
++0x0180   ea b0 78 35 32 68 52 jmp    0x5268:0x323578b0
++0x0187   62 f1...??           <INVALID>
++0x0188   f1                   int1   
++0x0189   08 88 ef d1 8f e6    or     byte ptr [eax+0xe68fd1ef], cl
++0x018f   91                   xchg   ecx, eax
++0x0190   79 80                jns    0x0fffff82
++0x0192   9c                   pushfd  
++0x0193   72 38                jb     0x1000003a
++0x0195   de 16                ficom  word ptr [esi]
++0x0197   0e                   push   cs
++0x0198   2f                   das    
++0x0199   e0 b4                loopne 0x0fffffb6
++0x019b   50                   push   eax
++0x019c   31 60 2f             xor    dword ptr [eax+0x2f], esp
++0x019f   fa                   cli    
++0x01a0   6d                   insd    
++0x01a1   c4 74 2d 69          les    esi, [ebp+ebp+0x69]
++0x01a5   6e                   outsb   
++0x01a6   03 37                add    esi, dword ptr [edi]
++0x01a8   74 c6                jz     0x0fffffc8
++0x01aa   4c                   dec    esp
++0x01ab   df 30                fbstp  tbyte ptr [eax]
++0x01ad   c2 91 7e             ret    0x7e91
++0x01b0   35 9e 18 d1 09       xor    eax, 0x09d1189e
++0x01b5   97                   xchg   edi, eax
++0x01b6   2d 63 b5 4f e3       sub    eax, 0xe34fb563
++0x01bb   83 52 50 be          adc    dword ptr [edx+0x50], 0xbe
++0x01bf   8e 26                mov    fs, dword ptr [esi]
++0x01c1   1d ba df 07 85       sbb    eax, 0x8507dfba
++0x01c6   5e                   pop    esi
++0x01c7   64 87 b7 6d 0d 2f 77 xchg   dword ptr [fs:edi+0x772f0d6d], esi
++0x01ce   a4                   movsb   
++0x01cf   0e                   push   cs
++0x01d0   5b                   pop    ebx
++0x01d1   01 93 af 4f d4 c3    add    dword ptr [ebx+0xc3d44faf], edx
++0x01d7   03 2e                add    ebp, dword ptr [esi]
++0x01d9   33 74 d3 8d          xor    esi, dword ptr [ebx+edx*8-0x73]
++0x01dd   d7                   xlatb   
++0x01de   bc 5b 41 4e 08       mov    esp, 0x084e415b
++0x01e3   0e                   push   cs
++0x01e4   cf                   iretd   
++0x01e5   23 73 42             and    esi, dword ptr [ebx+0x42]
++0x01e8   cc                   int3   
++0x01e9   56                   push   esi
++0x01ea   1b 27                sbb    esp, dword ptr [edi]
++0x01ec   75 34                jnz    0x10000036
++0x01ee   ba 95 66 65 27       mov    edx, 0x27656695
++0x01f3   30 d2                xor    dl, dl
++0x01f5   6e                   outsb   
++0x01f6   d0 e2                shl    dl, 0x01
++0x01f8   5d                   pop    ebp
++0x01f9   d5 47                aad    0x47
++0x01fb   29 46 f7             sub    dword ptr [esi-0x09], eax
++0x01fe   e8 bb 48 04 2d       call   0x3d0448c0
++0x0203   c2 d9 d5             ret    0xd5d9
++0x0206   e0 8b                loopne 0x0fffff8d
++0x0208   93                   xchg   ebx, eax
++0x0209   0b 12                or     edx, dword ptr [edx]
++0x020b   fc                   cld    
++0x020c   7a 19                jp     0x1000001b
++0x020e   47                   inc    edi
++0x020f   22 eb                and    ch, bl
++0x0211   17                   pop    ss
++0x0212   cb                   retf   
++0x0213   65 95                xchg   ebp, eax
++0x0215   e5 f1                in     eax, 0xf1
++0x0217   f3 60                pushad  
++0x0219   ae                   scasb   
++0x021a   41                   inc    ecx
++0x021b   16                   push   ss
++0x021c   60                   pushad  
++0x021d   a1 9e 4d 57 4f       mov    eax, dword ptr [0x4f574d9e]
++0x0222   be 1e d7 dd ac       mov    esi, 0xacddd71e
++0x0227   34 a3                xor    al, 0xa3
++0x0229   94                   xchg   esp, eax
++0x022a   c1 a5 ce ce 9c b3 48 shl    dword ptr [ebp+0xb39ccece], 0x48
++0x0231   60                   pushad  
++0x0232   a8 dd                test   al, 0xdd
++0x0234   bf d3 c7 bb 4e       mov    edi, 0x4ebbc7d3
++0x0239   41                   inc    ecx
++0x023a   a4                   movsb   
++0x023b   0c 7c                or     al, 0x7c
++0x023d   ab                   stosd   
++0x023e   51                   push   ecx
++0x023f   7b ec                jnp    0x0fffffee
++0x0241   32 99 b3 55 93 87    xor    bl, byte ptr [ecx+0x879355b3]
++0x0247   c0 4a b1 f2          ror    byte ptr [edx-0x4f], 0xf2
++0x024b   95                   xchg   ebp, eax
++0x024c   f0 53...??           <INVALID>
++0x024d   53                   push   ebx
++0x024e   91                   xchg   ecx, eax
++0x024f   55                   push   ebp
++0x0250   c4 0d cb e5 f5 bc    les    ecx, [0xbcf5e5cb]
++0x0256   d2 ca                ror    dl, cl
++0x0258   a0 ba 04 13 c8       mov    al, byte ptr [0xc81304ba]
++0x025d   db e9                fucomi st0, st1
++0x025f   7d d3                jnl    0x0fffffd5
++0x0261   11 e7                adc    edi, esp
++0x0263   fd                   std    
++0x0264   31 3e                xor    dword ptr [esi], edi
++0x0266   e6 9b                out    0x9b, al
++0x0268   62 c0...??           <INVALID>
++0x0269   c0 44 35 33 47       rol    byte ptr [ebp+esi+0x33], 0x47
++0x026e   0c 73                or     al, 0x73
++0x0270   7e 87                jle    0x0fffff89
++0x0272   50                   push   eax
++0x0273   a2 e4 87 a6 5f       mov    byte ptr [0x5fa687e4], al
++0x0278   fb                   sti    
++0x0279   68 44 b1 34 f2       push   0xf234b144
++0x027e   a3 c3 b4 8a 3d       mov    dword ptr [0x3d8ab4c3], eax
++0x0283   40                   inc    eax
++0x0284   b5 87                mov    ch, 0x87
++0x0286   02 e2                add    ah, dl
++0x0288   de dc...??           <INVALID>
++0x0289   dc f8                fdiv   st0, st0
++0x028b   ad                   lodsd   
++0x028c   9e                   sahf   
++0x028d   e0 57                loopne 0x10000059
++0x028f   5d                   pop    ebp
++0x0290   a3 18 61 3a 4a       mov    dword ptr [0x4a3a6118], eax
++0x0295   c3                   ret    
++0x0296   2a 56 bf             sub    dl, byte ptr [esi-0x41]
++0x0299   92                   xchg   edx, eax
++0x029a   a1 8f 2e 66 d9       mov    eax, dword ptr [0xd9662e8f]
++0x029f   f7 b3 f0 b5 f4 d0    div    edx, eax, dword ptr [ebx+0xd0f4b5f0]
++0x02a5   54                   push   esp
++0x02a6   f8                   clc    
++0x02a7   c1 64 30 6f d5       shl    dword ptr [eax+esi+0x6f], 0xd5
++0x02ac   9f                   lahf   
++0x02ad   53                   push   ebx
++0x02ae   c6 87 b8 7c d2 8e ac mov    byte ptr [edi+0x8ed27cb8], 0xac
++0x02b5   ca 11 e4             retf   0xe411
++0x02b8   18 9c d5 aa 1c ed eb sbb    byte ptr [ebp+edx*8+0xebed1caa], bl
++0x02bf   a3 fc 6b 34 bc       mov    dword ptr [0xbc346bfc], eax
++0x02c4   08 8a 31 8f 5d 1e    or     byte ptr [edx+0x1e5d8f31], cl
++0x02ca   01 15 e4 b4 ef 22    add    dword ptr [0x22efb4e4], edx
++0x02d0   0a 0e                or     cl, byte ptr [esi]
++0x02d2   54                   push   esp
++0x02d3   63 15 16 c2 b9 d4    arpl   word ptr [0xd4b9c216], dx
++0x02d9   d2 79 7d             sar    byte ptr [ecx+0x7d], cl
++0x02dc   cb                   retf   
++0x02dd   c7 7c bb e4 8e 81 93...??  <INVALID>
+     44
++0x02de   7c bb                jl     0x0fffffbd
++0x02e0   e4 8e                in     al, 0x8e
++0x02e2   81 93 44 7f e2 e1 66 adc    dword ptr [ebx+0xe1e27f44], 0x87eac066
+     c0 ea 87
++0x02ec   db e0...??           <INVALID>
++0x02ed   e0 91                loopne 0x0fffff93
++0x02ef   10 57 b6             adc    byte ptr [edi-0x4a], dl
++0x02f2   af                   scasd   
++0x02f3   ff b7 6f 94 3c 9d    push   dword ptr [edi+0x9d3c946f]
++0x02f9   af                   scasd   
++0x02fa   e4 1b                in     al, 0x1b
++0x02fc   57                   push   edi
++0x02fd   dc 71 5f             fdiv   qword ptr [ecx+0x5f]
++0x0300   73 b4                jnb    0x0fffffb6
++0x0302   7f 93                jnle   0x0fffff95
++0x0304   67 f2 82 a9 25 00 7c sub    byte ptr [bx+di+0x0025], 0x7c
++0x030b   9b                   fwait  
++0x030c   81 2b 3d 3c 1c fe    sub    dword ptr [ebx], 0xfe1c3c3d
++0x0312   b5 ec                mov    ch, 0xec
++0x0314   a7                   cmpsd   
++0x0315   7a 82                jp     0x0fffff84
++0x0317   42                   inc    edx
++0x0318   ff 39...??           <INVALID>
++0x0319   39 57 31             cmp    dword ptr [edi+0x31], edx
++0x031c   bc 4a 1f 84 98       mov    esp, 0x98841f4a
++0x0321   04 bd                add    al, 0xbd
++0x0323   ed                   in     eax, dx
++0x0324   a7                   cmpsd   
++0x0325   b2 f0                mov    dl, 0xf0
++0x0327   7b 31                jnp    0x10000033
++0x0329   19 4f bc             sbb    dword ptr [edi-0x44], ecx
++0x032c   e1 64                loope  0x10000066
++0x032e   01 8d d9 81 15 f0    add    dword ptr [ebp+0xf01581d9], ecx
++0x0334   a4                   movsb   
++0x0335   f4                   hlt    
++0x0336   a4                   movsb   
++0x0337   ce                   into   
++0x0338   1c 2d                sbb    al, 0x2d
++0x033a   a6                   cmpsb   
++0x033b   98                   cwde   
++0x033c   4a                   dec    edx
++0x033d   6d                   insd    
++0x033e   c6 e5 a3...??        <INVALID>
++0x033f   e5 a3                in     eax, 0xa3
++0x0341   6f                   outsd   
++0x0342   e0 c7                loopne 0x0fffffc9
++0x0344   54                   push   esp
++0x0345   19 f8                sbb    eax, edi
++0x0347   3d 31 55 54 bf       cmp    eax, 0xbf545531
++0x034c   64 97                xchg   edi, eax
++0x034e   57                   push   edi
++0x034f   78 30                js     0x10000032
++0x0351   7d bb                jnl    0x0fffffbd
++0x0353   48                   dec    eax
++0x0354   dd 79 81             fnstsw word ptr [ecx-0x7f]
++0x0357   36 1f                pop    ds
++0x0359   c1 21 e0             shl    dword ptr [ecx], 0xe0
++0x035c   0b a7 26 29 d5 d4    or     esp, dword ptr [edi+0xd4d52926]
++0x0362   57                   push   edi
++0x0363   63 9f 91 82 ed 38    arpl   word ptr [edi+0x38ed8291], bx
++0x0369   ed                   in     eax, dx
++0x036a   4f                   dec    edi
++0x036b   f7 1d 67 c3 58 1d    neg    dword ptr [0x1d58c367]
++0x0371   f9                   stc    
++0x0372   6d                   insd    
++0x0373   b1 34                mov    cl, 0x34
++0x0375   b2 1b                mov    dl, 0x1b
++0x0377   a1 f8 a1 33 0c       mov    eax, dword ptr [0x0c33a1f8]
++0x037c   bb 7c 86 35 d6       mov    ebx, 0xd635867c
++0x0381   88 03                mov    byte ptr [ebx], al
++0x0383   44                   inc    esp
++0x0384   06                   push   es
++0x0385   ee                   out    al, dx
++0x0386   ad                   lodsd   
++0x0387   17                   pop    ss
++0x0388   75 2b                jnz    0x1000002d
++0x038a   63 c7                arpl   di, ax
++0x038c   a7                   cmpsd   
++0x038d   f4                   hlt    
++0x038e   bf 57 23 ac 95       mov    edi, 0x95ac2357
++0x0393   55                   push   ebp
++0x0394   43                   inc    ebx
++0x0395   5b                   pop    ebx
++0x0396   15 c4 8b a8 8b       adc    eax, 0x8ba88bc4
++0x039b   d3 e8                shr    eax, cl
++0x039d   4b                   dec    ebx
++0x039e   f6 31                div    ah, al, byte ptr [ecx]
++0x03a0   62 59 df             bound  ebx, qword ptr [ecx-0x21]
++0x03a3   d8 bd 13 1a 4b a4    fdivr  dword ptr [ebp+0xa44b1a13]
++0x03a9   33 76 d9             xor    esi, dword ptr [esi-0x27]
++0x03ac   19 16                sbb    dword ptr [esi], edx
++0x03ae   d2 89 22 bc 3e bb    ror    byte ptr [ecx+0xbb3ebc22], cl
++0x03b4   6f                   outsd   
++0x03b5   23 be 35 f4 35 d5    and    edi, dword ptr [esi+0xd535f435]
++0x03bb   af                   scasd   
++0x03bc   00 30                add    byte ptr [eax], dh
++0x03be   78 1c                js     0x1000001e
++0x03c0   ea 22 85 df bf c5 23 jmp    0x23c5:0xbfdf8522
++0x03c7   7d 10                jnl    0x10000012
++0x03c9   97                   xchg   edi, eax
++0x03ca   84 59 36             test   byte ptr [ecx+0x36], bl
++0x03cd   35 51 dd f2 2e       xor    eax, 0x2ef2dd51
++0x03d2   be 1f 5b dd 6c       mov    esi, 0x6cdd5b1f
++0x03d7   50                   push   eax
++0x03d8   dd 98 77 4f 56 7a    fstp   qword ptr [eax+0x7a564f77]
++0x03de   a4                   movsb   
++0x03df   bc dc 8f dc 6e       mov    esp, 0x6edc8fdc
++0x03e4   35 d1 a5 e4 57       xor    eax, 0x57e4a5d1
++0x03e9   bf bc a0 90 48       mov    edi, 0x4890a0bc
++0x03ee   d8 fa                fdivr  st0, st2
++0x03f0   9d                   popfd   
++0x03f1   70 ac                jo     0x0fffffae
++0x03f3   27                   daa    
++0x03f4   90                   nop    
++0x03f5   2e 64 e5 0e          in     eax, 0x0e
++0x03f9   e5 c1                in     eax, 0xc1
++0x03fb   38 a4 5d 9a e6 18 1e cmp    byte ptr [ebp+ebx*2+0x1e18e69a], ah
++0x0402   87 8b bd 44 4c 51    xchg   dword ptr [ebx+0x514c44bd], ecx
++0x0408   60                   pushad  
++0x0409   a7                   cmpsd   
++0x040a   6f                   outsd   
++0x040b   43                   inc    ebx
++0x040c   ab                   stosd   
++0x040d   9b                   fwait  
++0x040e   c6 ff b8...??        <INVALID>
++0x040f   ff b8 9a f4 26 e4...??  <INVALID>
++0x0410   b8 9a f4 26 e4       mov    eax, 0xe426f49a
++0x0415   be 5c b0 7d 3d       mov    esi, 0x3d7db05c
++0x041a   21 73 34             and    dword ptr [ebx+0x34], esi
++0x041d   da 69 09             fisubr dword ptr [ecx+0x09]
++0x0420   aa                   stosb   
++0x0421   38 8f 9c b8 9c a0    cmp    byte ptr [edi+0xa09cb89c], cl
++0x0427   7f 21                jnle   0x10000023
++0x0429   38 b6 87 b4 f9 55    cmp    byte ptr [esi+0x55f9b487], dh
++0x042f   df 6f 31             fild   qword ptr [edi+0x31]
++0x0432   58                   pop    eax
++0x0433   d9 4d b1...??        <INVALID>
++0x0434   4d                   dec    ebp
++0x0435   b1 1f                mov    cl, 0x1f
++0x0437   f8                   clc    
++0x0438   79 08                jns    0x1000000a
++0x043a   8b 2a                mov    ebp, dword ptr [edx]
++0x043c   e5 06                in     eax, 0x06
++0x043e   b4 85                mov    ah, 0x85
++0x0440   13 4b e3             adc    ecx, dword ptr [ebx-0x1d]
++0x0443   47                   inc    edi
++0x0444   7b a7                jnp    0x0fffffa9
++0x0446   94                   xchg   esp, eax
++0x0447   3e c7 53 7f 45 62 14...??  <INVALID>
+     11
++0x0448   c7 53 7f 45 62 14 11...??  <INVALID>
++0x0449   53                   push   ebx
++0x044a   7f 45                jnle   0x10000047
++0x044c   62 14 11             bound  edx, qword ptr [ecx+edx]
++0x044f   de b5 c3 54 41 4b    fidiv  word ptr [ebp+0x4b4154c3]
++0x0455   7e 84                jle    0x0fffff86
++0x0457   85 85 22 51 68 72    test   dword ptr [ebp+0x72685122], eax
++0x045d   f6 52 50             not    byte ptr [edx+0x50]
++0x0460   a2 be d9 3a ab       mov    byte ptr [0xab3ad9be], al
++0x0465   70 f2                jo     0x0ffffff4
++0x0467   e9 e4 05 46 36       jmp    0x464605e9
++0x046c   04 6f                add    al, 0x6f
++0x046e   a3 e8 9c c9 b9       mov    dword ptr [0xb9c99ce8], eax
++0x0473   ad                   lodsd   
++0x0474   eb 76                jmp    0x10000078
++0x0476   1d d1 b1 9b ac       sbb    eax, 0xac9bb1d1
++0x047b   0d 8e 15 f1 2a       or     eax, 0x2af1158e
++0x0480   df ad 24 52 88 c1    fild   qword ptr [ebp+0xc1885224]
++0x0486   52                   push   edx
++0x0487   25 62 06 05 f6       and    eax, 0xf6050662
++0x048c   ec                   in     al
++0x048d   56                   push   esi
++0x048e   eb 63                jmp    0x10000065
++0x0490   53                   push   ebx
++0x0491   fc                   cld    
++0x0492   0c e4                or     al, 0xe4
++0x0494   75 27                jnz    0x10000029
++0x0496   d0 30                shl    byte ptr [eax], 0x01
++0x0498   b9 3c 58 a2 10       mov    ecx, 0x10a2583c
++0x049d   bc da 32 14 8c       mov    esp, 0x8c1432da
++0x04a2   7a 0b                jp     0x1000000d
++0x04a4   61                   popad   
++0x04a5   46                   inc    esi
++0x04a6   ff ab 15 27 eb f6    jmp    
++0x04ac   cd bb                int    0xbb
++0x04ae   4f                   dec    edi
++0x04af   6f                   outsd   
++0x04b0   b2 55                mov    dl, 0x55
++0x04b2   1c bb                sbb    al, 0xbb
++0x04b4   94                   xchg   esp, eax
++0x04b5   1d 7e a7 52 67       sbb    eax, 0x6752a77e
++0x04ba   25 01 d4 a1 d7       and    eax, 0xd7a1d401
++0x04bf   f7 c8 92 50 cf ca    test   eax, 0xcacf5092
++0x04c5   c1 be f0 ce 7a 88 ce sar    dword ptr [esi+0x887acef0], 0xce
++0x04cc   b7 f7                mov    bh, 0xf7
++0x04ce   24 80                and    al, 0x80
++0x04d0   38 34 90             cmp    byte ptr [eax+edx*4], dh
++0x04d3   e5 cf                in     eax, 0xcf
++0x04d5   30 2e                xor    byte ptr [esi], ch
++0x04d7   af                   scasd   
++0x04d8   3d 89 3c 82 2b       cmp    eax, 0x2b823c89
++0x04dd   88 30                mov    byte ptr [eax], dh
++0x04df   be b9 f5 a0 59       mov    esi, 0x59a0f5b9
++0x04e4   7e 99                jle    0x0fffff9b
++0x04e6   f4                   hlt    
++0x04e7   37                   aaa    
++0x04e8   a4                   movsb   
++0x04e9   8d b3 81 82 42 33    lea    esi, [ebx+0x33428281]
++0x04ef   1f                   pop    ds
++0x04f0   bc 8f cb 77 07       mov    esp, 0x0777cb8f
++0x04f5   9b                   fwait  
++0x04f6   f2 56                push   esi
++0x04f8   14 b7                adc    al, 0xb7
++0x04fa   41                   inc    ecx
++0x04fb   43                   inc    ebx
++0x04fc   4e                   dec    esi
++0x04fd   9b                   fwait  
++0x04fe   3d 8a a7 68 84       cmp    eax, 0x8468a78a
++0x0503   dd 9a 40 30 e9 97    fstp   qword ptr [edx+0x97e93040]
++0x0509   d4 07                aam    0x07
++0x050b   a3 a1 23 70 2d       mov    dword ptr [0x2d7023a1], eax
++0x0510   cb                   retf   
++0x0511   1b b8 d3 49 bc ff    sbb    edi, dword ptr [eax-0x0043b62d]
++0x0517   9f                   lahf   
++0x0518   b7 3a                mov    bh, 0x3a
++0x051a   36 9c                pushfd  
++0x051c   a7                   cmpsd   
++0x051d   b4 ad                mov    ah, 0xad
++0x051f   0a 32                or     dh, byte ptr [edx]
++0x0521   36 26 16             push   ss
++0x0524   42                   inc    edx
++0x0525   31 09                xor    dword ptr [ecx], ecx
++0x0527   bf 47 01 44 a1       mov    edi, 0xa1440147
++0x052c   39 43 e3             cmp    dword ptr [ebx-0x1d], eax
++0x052f   c7 44 b4 90 20 a4 8a mov    dword ptr [esp+esi*4-0x70], 0xb08aa420
+     b0
++0x0537   95                   xchg   ebp, eax
++0x0538   19 dd                sbb    ebp, ebx
++0x053a   b3 42                mov    bl, 0x42
++0x053c   87 7d e1             xchg   dword ptr [ebp-0x1f], edi
++0x053f   35 90 2d 06 23       xor    eax, 0x23062d90
++0x0544   f0 02 5b 02...??     <INVALID>
++0x0545   02 5b 02             add    bl, byte ptr [ebx+0x02]
++0x0548   65 9d                popfd   
++0x054a   c9                   leave  
++0x054b   c1 79 9f 7d          sar    dword ptr [ecx-0x61], 0x7d
++0x054f   7a a1                jp     0x0fffffa3
++0x0551   93                   xchg   ebx, eax
++0x0552   b9 5d 0e 70 76       mov    ecx, 0x76700e5d
++0x0557   d8 a0 eb 11 de 38    fsub   dword ptr [eax+0x38de11eb]
++0x055d   af                   scasd   
++0x055e   f9                   stc    
++0x055f   96                   xchg   esi, eax
++0x0560   a0 a2 36 fd 4e       mov    al, byte ptr [0x4efd36a2]
++0x0565   fb                   sti    
++0x0566   dd 29...??           <INVALID>
++0x0567   29 34 e7             sub    dword ptr [edi], esi
++0x056a   f6 05 cd 6c f7 28 4c test   byte ptr [0x28f76ccd], 0x4c
++0x0571   16                   push   ss
++0x0572   f3 46                inc    esi
++0x0574   a7                   cmpsd   
++0x0575   c9                   leave  
++0x0576   5f                   pop    edi
++0x0577   e3 b4                jecxz  0x0fffffb6
++0x0579   ca 50 34             retf   0x3450
++0x057c   8a ec                mov    ch, ah
++0x057e   a5                   movsd   
++0x057f   aa                   stosb   
++0x0580   fb                   sti    
++0x0581   0a 9b 99 41 a5 1e    or     bl, byte ptr [ebx+0x1ea54199]
++0x0587   a0 d6 c0 3c 38       mov    al, byte ptr [0x383cc0d6]
++0x058c   93                   xchg   ebx, eax
++0x058d   4d                   dec    ebp
++0x058e   4f                   dec    edi
++0x058f   4a                   dec    edx
++0x0590   4a                   dec    edx
++0x0591   01 08                add    dword ptr [eax], ecx
++0x0593   14 6e                adc    al, 0x6e
++0x0595   40                   inc    eax
++0x0596   c9                   leave  
++0x0597   14 26                adc    al, 0x26
++0x0599   eb bb                jmp    0x0fffffbd
++0x059b   c7 9c 6d 60 62 24 c7...??  <INVALID>
+     2c 77 0c ae
++0x059c   9c                   pushfd  
++0x059d   6d                   insd    
++0x059e   60                   pushad  
++0x059f   62 24 c7             bound  esp, qword ptr [edi+eax*8]
++0x05a2   2c 77                sub    al, 0x77
++0x05a4   0c ae                or     al, 0xae
++0x05a6   94                   xchg   esp, eax
++0x05a7   62 cb...??           <INVALID>
++0x05a8   cb                   retf   
++0x05a9   f2 47                inc    edi
++0x05ab   4d                   dec    ebp
++0x05ac   3d d0 bc 7c 78       cmp    eax, 0x787cbcd0
++0x05b1   3a f7                cmp    dh, bh
++0x05b3   09 f2                or     edx, esi
++0x05b5   70 a4                jo     0x0fffffa6
++0x05b7   34 9b                xor    al, 0x9b
++0x05b9   f6 85 64 01 74 bd e9 test   byte ptr [ebp+0xbd740164], 0xe9
++0x05c0   a3 7a e7 e9 d2       mov    dword ptr [0xd2e9e77a], eax
++0x05c5   52                   push   edx
++0x05c6   f3 52                push   edx
++0x05c8   b0 77                mov    al, 0x77
++0x05ca   b1 da                mov    cl, 0xda
++0x05cc   b2 ec                mov    dl, 0xec
++0x05ce   6d                   insd    
++0x05cf   ae                   scasb   
++0x05d0   91                   xchg   ecx, eax
++0x05d1   d0 c7                rol    bh, 0x01
++0x05d3   25 88 ad c6 47       and    eax, 0x47c6ad88
++0x05d8   fa                   cli    
++0x05d9   3e e7 66             out    0x66, eax
++0x05dc   77 12                jnbe   0x10000014
++0x05de   a2 1c a3 4d d2       mov    byte ptr [0xd24da31c], al
++0x05e3   7e 8a                jle    0x0fffff8c
++0x05e5   06                   push   es
++0x05e6   73 0d                jnb    0x1000000f
++0x05e8   77 39                jnbe   0x1000003b
++0x05ea   13 d9                adc    ebx, ecx
++0x05ec   93                   xchg   ebx, eax
++0x05ed   d7                   xlatb   
++0x05ee   99                   cdq    
++0x05ef   1f                   pop    ds
++0x05f0   9e                   sahf   
++0x05f1   bb 10 59 34 ff       mov    ebx, 0xff345910
++0x05f6   a0 c1 f4 c8 5e       mov    al, byte ptr [0x5ec8f4c1]
++0x05fb   1f                   pop    ds
++0x05fc   a6                   cmpsb   
++0x05fd   e8 83 27 3f 79       call   0x893f2788
++0x0602   b6 b3                mov    dh, 0xb3
++0x0604   7a 13                jp     0x10000015
++0x0606   f0 ea f4 24 3b 10 14...??  <INVALID>
+     ce
++0x0607   ea f4 24 3b 10 14 ce jmp    0xce14:0x103b24f4
++0x060e   41                   inc    ecx
++0x060f   f3 dd 58 23          fstp   qword ptr [eax+0x23]
++0x0613   66 ef                out    ax, dx
++0x0615   71 84                jno    0x0fffff86
++0x0617   6b e5 fd             imul   esp, ebp, 0xfd
++0x061a   08 a5 a2 5b 44 ae    or     byte ptr [ebp+0xae445ba2], ah
++0x0620   e3 b2                jecxz  0x0fffffb4
++0x0622   3a fc                cmp    bh, ah
++0x0624   ed                   in     eax, dx
++0x0625   3f                   aas    
++0x0626   90                   nop    
++0x0627   1c d8                sbb    al, 0xd8
++0x0629   9a 71 c2 d8 79 b6 43 call   0x43b6:0x79d8c271
++0x0630   65 47                inc    edi
++0x0632   44                   inc    esp
++0x0633   9d                   popfd   
++0x0634   2f                   das    
++0x0635   5f                   pop    edi
++0x0636   e4 a8                in     al, 0xa8
++0x0638   93                   xchg   ebx, eax
++0x0639   b6 9f                mov    dh, 0x9f
++0x063b   d9 d5...??           <INVALID>
++0x063c   d5 ef                aad    0xef
++0x063e   90                   nop    
++0x063f   a9 1d 74 d9 6e       test   eax, 0x6ed9741d
++0x0644   a1 10 49 34 53       mov    eax, dword ptr [0x53344910]
++0x0649   f7 c6 61 27 61 ef    test   esi, 0xef612761
++0x064f   13 9a a9 48 d9 f2    adc    ebx, dword ptr [edx+0xf2d948a9]
++0x0655   2c 46                sub    al, 0x46
++0x0657   b8 41 c4 fe 83       mov    eax, 0x83fec441
++0x065c   b1 9d                mov    cl, 0x9d
++0x065e   af                   scasd   
++0x065f   39 c6                cmp    esi, eax
++0x0661   29 47 9a             sub    dword ptr [edi-0x66], eax
++0x0664   4d                   dec    ebp
++0x0665   82 fb 61             cmp    bl, 0x61
++0x0668   a0 f3 eb e6 25       mov    al, byte ptr [0x25e6ebf3]
++0x066d   41                   inc    ecx
++0x066e   60                   pushad  
++0x066f   a5                   movsd   
++0x0670   cc                   int3   
++0x0671   a8 c0                test   al, 0xc0
++0x0673   b1 39                mov    cl, 0x39
++0x0675   6d                   insd    
++0x0676   eb af                jmp    0x0fffffb1
++0x0678   05 5d 29 66 da       add    eax, 0xda66295d
++0x067d   39 7f 22             cmp    dword ptr [edi+0x22], edi
++0x0680   50                   push   eax
++0x0681   2c 5e                sub    al, 0x5e
++0x0683   1b 08                sbb    ecx, dword ptr [eax]
++0x0685   91                   xchg   ecx, eax
++0x0686   4d                   dec    ebp
++0x0687   e7 e2                out    0xe2, eax
++0x0689   5b                   pop    ebx
++0x068a   7a c7                jp     0x0fffffc9
++0x068c   2e 26 ea 3b aa 79 fa jmp    0x2c79:0xfa79aa3b
+     79 2c
++0x0695   d0 a2 25 e3 66 b9    shl    byte ptr [edx+0xb966e325], 0x01
++0x069b   ec                   in     al
++0x069c   ad                   lodsd   
++0x069d   6c                   insb    
++0x069e   05 0b b9 2e 4e       add    eax, 0x4e2eb90b
++0x06a3   3c fb                cmp    al, 0xfb
++0x06a5   2d d8 a8 f2 52       sub    eax, 0x52f2a8d8
++0x06aa   8b 4e 08             mov    ecx, dword ptr [esi+0x08]
++0x06ad   23 88 52 f4 f8 b9    and    ecx, dword ptr [eax+0xb9f8f452]
++0x06b3   d8 d7                fcom   st0, st7
++0x06b5   f1                   int1   
++0x06b6   be 08 57 0d 70       mov    esi, 0x700d5708
++0x06bb   bd a1 58 8e 67       mov    ebp, 0x678e58a1
++0x06c0   13 21                adc    esp, dword ptr [ecx]
++0x06c2   83 10 bf             adc    dword ptr [eax], 0xbf
++0x06c5   83 24 94 c6          and    dword ptr [esp+edx*4], 0xc6
++0x06c9   53                   push   ebx
++0x06ca   2d af a2 1f 9e       sub    eax, 0x9e1fa2af
++0x06cf   b9 ae 29 df dc       mov    ecx, 0xdcdf29ae
++0x06d4   f8                   clc    
++0x06d5   3a c9                cmp    cl, cl
++0x06d7   f5                   cmc    
++0x06d8   c8 3f 4f f8          enter  0x4f3f, 0xf8
++0x06dc   82 92 71 8e 4b eb b6 adc    byte ptr [edx+0xeb4b8e71], 0xb6
++0x06e3   e2 c8                loop   0x0fffffca
++0x06e5   74 87                jz     0x0fffff89
++0x06e7   d4 1b                aam    0x1b
++0x06e9   53                   push   ebx
++0x06ea   e8 17 2b 84 6e       call   0x7e842b1c
++0x06ef   62 a8 70 c8 bc ab    bound  ebp, qword ptr [eax+0xabbcc870]
++0x06f5   1a e7                sbb    ah, bh
++0x06f7   d4 0a                aam    0x0a
++0x06f9   0d 6c 3f 1a 1b       or     eax, 0x1b1a3f6c
++0x06fe   e5 7c                in     eax, 0x7c
++0x0700   1d 07 4d b9 b9       sbb    eax, 0xb9b94d07
++0x0705   6d                   insd    
++0x0706   b3 8a                mov    bl, 0x8a
++0x0708   6a f9                push   0xf9
++0x070a   1b 88 9e 66 31 1b    sbb    ecx, dword ptr [eax+0x1b31669e]
++0x0710   be 3d 5b de d1       mov    esi, 0xd1de5b3d
++0x0715   78 e5                js     0x0fffffe7
++0x0717   1d d5 e0 ff 06       sbb    eax, 0x06ffe0d5
++0x071c   7d a8                jnl    0x0fffffaa
++0x071e   6c                   insb    
++0x071f   45                   inc    ebp
++0x0720   c2 a6 0d             ret    0x0da6
++0x0723   bc 3b ba e9 39       mov    esp, 0x39e9ba3b
++0x0728   91                   xchg   ecx, eax
++0x0729   7e 68                jle    0x1000006a
++0x072b   ba e3 d8 0b 84       mov    edx, 0x840bd8e3
++0x0730   b6 d1                mov    dh, 0xd1
++0x0732   c4 32                les    esi, [edx]
++0x0734   84 c8                test   al, cl
++0x0736   f1                   int1   
++0x0737   dd 05 fa 11 c3 7e    fld    qword ptr [0x7ec311fa]
++0x073d   66 73 18             jnb    0x0000001b
++0x0740   5e                   pop    esi
++0x0741   2f                   das    
++0x0742   f4                   hlt    
++0x0743   76 ec                jbe    0x0fffffee
++0x0745   7d 47                jnl    0x10000049
++0x0747   c3                   ret    
++0x0748   3e ec                in     al
++0x074a   ad                   lodsd   
++0x074b   77 01                jnbe   0x10000003
++0x074d   5b                   pop    ebx
++0x074e   af                   scasd   
++0x074f   fc                   cld    
++0x0750   b7 a0                mov    bh, 0xa0
++0x0752   12 20                adc    ah, byte ptr [eax]
++0x0754   1c 53                sbb    al, 0x53
++0x0756   69 81 8a 93 0e 9b dc imul   eax, dword ptr [ecx+0x9b0e938a], 0x53744fdc
+     4f 74 53
++0x0760   a6                   cmpsb   
++0x0761   87 aa 82 50 91 64    xchg   dword ptr [edx+0x64915082], ebp
++0x0767   43                   inc    ebx
++0x0768   f6 6a a3             imul   ax, byte ptr [edx-0x5d]
++0x076b   9f                   lahf   
++0x076c   05 de ad 86 60       add    eax, 0x6086adde
++0x0771   0c fb                or     al, 0xfb
++0x0773   2e 0d 7b 6d 92 a0    or     eax, 0xa0926d7b
++0x0779   ed                   in     eax, dx
++0x077a   e4 36                in     al, 0x36
++0x077c   76 bf                jbe    0x0fffffc1
++0x077e   f1                   int1   
++0x077f   d7                   xlatb   
++0x0780   da 36                fidiv  dword ptr [esi]
++0x0782   88 17                mov    byte ptr [edi], dl
++0x0784   fe c0                inc    al
++0x0786   89 a1 53 62 47 98    mov    dword ptr [ecx+0x98476253], esp
++0x078c   d7                   xlatb   
++0x078d   ba 50 ea ab 54       mov    edx, 0x54abea50
++0x0792   70 1a                jo     0x1000001c
++0x0794   02 37                add    dh, byte ptr [edi]
++0x0796   dc 21                fsub   qword ptr [ecx]
++0x0798   31 6c 0f 48          xor    dword ptr [edi+ecx+0x48], ebp
++0x079c   9f                   lahf   
++0x079d   63 6a b5             arpl   word ptr [edx-0x4b], bp
++0x07a0   c6 30 f8...??        <INVALID>
++0x07a1   30 f8                xor    al, bh
++0x07a3   bc 34 a8 bb 9c       mov    esp, 0x9cbba834
++0x07a8   fd                   std    
++0x07a9   e3 f5                jecxz  0x0ffffff7
++0x07ab   4f                   dec    edi
++0x07ac   03 6f c0             add    ebp, dword ptr [edi-0x40]
++0x07af   ba 22 a5 17 97       mov    edx, 0x9717a522
++0x07b4   28 2c bc             sub    byte ptr [esp+edi*4], ch
++0x07b7   9f                   lahf   
++0x07b8   bc 18 4f e9 47       mov    esp, 0x47e94f18
++0x07bd   3e 87 36             xchg   dword ptr [ds:esi], esi
++0x07c0   9c                   pushfd  
++0x07c1   55                   push   ebp
++0x07c2   3b 17                cmp    edx, dword ptr [edi]
++0x07c4   74 6e                jz     0x10000070
++0x07c6   94                   xchg   esp, eax
++0x07c7   49                   dec    ecx
++0x07c8   d6                   salc   
++0x07c9   51                   push   ecx
++0x07ca   d5 f9                aad    0xf9
++0x07cc   fe 82 f5 8b fc 53    inc    byte ptr [edx+0x53fc8bf5]
++0x07d2   1a a6 65 9e 06 0e    sbb    ah, byte ptr [esi+0x0e069e65]
++0x07d8   d3 ea                shr    edx, cl
++0x07da   ec                   in     al
++0x07db   17                   pop    ss
++0x07dc   94                   xchg   esp, eax
++0x07dd   21 0a                and    dword ptr [edx], ecx
++0x07df   49                   dec    ecx
++0x07e0   9b                   fwait  
++0x07e1   27                   daa    
++0x07e2   7e 95                jle    0x0fffff97
++0x07e4   1e                   push   ds
++0x07e5   88 61 9b             mov    byte ptr [ecx-0x65], ah
++0x07e8   40                   inc    eax
++0x07e9   07                   pop    es
++0x07ea   7b 08                jnp    0x1000000a
++0x07ec   e9 36 d4 1e 73       jmp    0x831ed43b
++0x07f1   79 a3                jns    0x0fffffa5
++0x07f3   b1 ba                mov    cl, 0xba
++0x07f5   35 76 d3 13 57       xor    eax, 0x5713d376
++0x07fa   7a 7f                jp     0x10000081
++0x07fc   3b 40 d6             cmp    eax, dword ptr [eax-0x2a]
++0x07ff   81 37 cd b7 56 be    xor    dword ptr [edi], 0xbe56b7cd
++0x0805   79 08                jns    0x1000000a
++0x0807   97                   xchg   edi, eax
++0x0808   ec                   in     al
++0x0809   c2 88 fc             ret    0xfc88
++0x080c   94                   xchg   esp, eax
++0x080d   6f                   outsd   
++0x080e   b9 e0 9f 62 cb       mov    ecx, 0xcb629fe0
++0x0813   6f                   outsd   
++0x0814   9d                   popfd   
++0x0815   42                   inc    edx
++0x0816   54                   push   esp
++0x0817   91                   xchg   ecx, eax
++0x0818   b9 40 b7 f2 3f       mov    ecx, 0x3ff2b740
++0x081d   15 6a 7f cf b4       adc    eax, 0xb4cf7f6a
++0x0822   63 9e 7b 50 bd bb    arpl   word ptr [esi+0xbbbd507b], bx
++0x0828   ff 1c 3d e9 61 a4 63 call   
++0x082f   8b 45 4d             mov    eax, dword ptr [ebp+0x4d]
++0x0832   56                   push   esi
++0x0833   60                   pushad  
++0x0834   0b 45 ec             or     eax, dword ptr [ebp-0x14]
++0x0837   56                   push   esi
++0x0838   1d ae c0 8f 3e       sbb    eax, 0x3e8fc0ae
++0x083d   48                   dec    eax
++0x083e   22 47 c5             and    al, byte ptr [edi-0x3b]
++0x0841   51                   push   ecx
++0x0842   37                   aaa    
++0x0843   87 61 6b             xchg   dword ptr [ecx+0x6b], esp
++0x0846   da cc                fcmove st0, st4
++0x0848   ea 95 93 ff 34 23 28 jmp    0x2823:0x34ff9395
++0x084f   10 f8                adc    al, bh
++0x0851   35 e2 2e dc 00       xor    eax, 0x00dc2ee2
++0x0856   01 83 86 d7 fc 55    add    dword ptr [ebx+0x55fcd786], eax
++0x085c   7f 30                jnle   0x10000032
++0x085e   e9 96 c4 dd 88       jmp    0x98ddc49b
++0x0863   5d                   pop    ebp
++0x0864   fc                   cld    
++0x0865   ec                   in     al
++0x0866   23 8e fd f6 14 f3    and    ecx, dword ptr [esi+0xf314f6fd]
++0x086c   30 46 08             xor    byte ptr [esi+0x08], al
++0x086f   dd 58 92             fstp   qword ptr [eax-0x6e]
++0x0872   77 8f                jnbe   0x0fffff91
++0x0874   82 80 1f 8a ef 6b 97 add    byte ptr [eax+0x6bef8a1f], 0x97
++0x087b   6e                   outsb   
++0x087c   10 6d b5             adc    byte ptr [ebp-0x4b], ch
++0x087f   59                   pop    ecx
++0x0880   2b 35 e5 f4 7f 2d    sub    esi, dword ptr [0x2d7ff4e5]
++0x0886   f9                   stc    
++0x0887   8c 1b                mov    dword ptr [ebx], ds
++0x0889   81 0c dc 42 64 ee 63 or     dword ptr [esp+ebx*8], 0x63ee6442
++0x0890   06                   push   es
++0x0891   50                   push   eax
++0x0892   2f                   das    
++0x0893   de e5                fsubrp st5, st0
++0x0895   7c 72                jl     0x10000074
++0x0897   e2 99                loop   0x0fffff9b
++0x0899   bc a6 b4 be 34       mov    esp, 0x34beb4a6
++0x089e   15 38 58 74 c6       adc    eax, 0xc6745838
++0x08a3   e6 71                out    0x71, al
++0x08a5   3a ea                cmp    ch, dl
++0x08a7   2b 8a 27 91 51 5c    sub    ecx, dword ptr [edx+0x5c519127]
++0x08ad   19 8f c1 5c ca 33    sbb    dword ptr [edi+0x33ca5cc1], ecx
++0x08b3   d6                   salc   
++0x08b4   0b 42 00             or     eax, dword ptr [edx+0x00]
++0x08b7   2f                   das    
++0x08b8   a5                   movsd   
++0x08b9   f5                   cmc    
++0x08ba   a6                   cmpsb   
++0x08bb   00 6c 83 11          add    byte ptr [ebx+eax*4+0x11], ch
++0x08bf   74 87                jz     0x0fffff89
++0x08c1   21 6d d3             and    dword ptr [ebp-0x2d], ebp
++0x08c4   48                   dec    eax
++0x08c5   2e cb                retf   
++0x08c7   18 11                sbb    byte ptr [ecx], dl
++0x08c9   b6 35                mov    dh, 0x35
++0x08cb   d3 57 e1             rcl    dword ptr [edi-0x1f], cl
++0x08ce   08 74 60 c8          or     byte ptr [eax-0x38], dh
++0x08d2   1f                   pop    ds
++0x08d3   21 78 d1             and    dword ptr [eax-0x2f], edi
++0x08d6   b1 c8                mov    cl, 0xc8
++0x08d8   e3 77                jecxz  0x10000079
++0x08da   25 e6 82 66 e2       and    eax, 0xe26682e6
++0x08df   2a c6                sub    al, dh
++0x08e1   2a 38                sub    bh, byte ptr [eax]
++0x08e3   16                   push   ss
++0x08e4   1a 92 d3 50 55 f5    sbb    dl, byte ptr [edx+0xf55550d3]
++0x08ea   76 4e                jbe    0x10000050
++0x08ec   c9                   leave  
++0x08ed   13 13                adc    edx, dword ptr [ebx]
++0x08ef   b1 bb                mov    cl, 0xbb
++0x08f1   53                   push   ebx
++0x08f2   23 44 e0 0a          and    eax, dword ptr [eax+0x0a]
++0x08f6   cf                   iretd   
++0x08f7   6f                   outsd   
++0x08f8   31 96 e8 d0 f4 10    xor    dword ptr [esi+0x10f4d0e8], edx
++0x08fe   3d 99 d7 f5 1c       cmp    eax, 0x1cf5d799
++0x0903   c5 40 e2             lds    eax, [eax-0x1e]
++0x0906   f2 76 2e             jbe    0x10000031
++0x0909   4f                   dec    edi
++0x090a   23 eb                and    ebp, ebx
++0x090c   1a 80 12 0b 88 2f    sbb    al, byte ptr [eax+0x2f880b12]
++0x0912   5f                   pop    edi
++0x0913   90                   nop    
++0x0914   94                   xchg   esp, eax
++0x0915   fb                   sti    
++0x0916   0f...??              <INVALID>
++0x0917   3b 63 a4             cmp    esp, dword ptr [ebx-0x5c]
++0x091a   e1 77                loope  0x10000079
++0x091c   1f                   pop    ds
++0x091d   da 31                fidiv  dword ptr [ecx]
++0x091f   a0 1b 07 b8 bd       mov    al, byte ptr [0xbdb8071b]
++0x0924   9a 7b d8 58 0e 5a 2d call   0x2d5a:0x0e58d87b
++0x092b   5c                   pop    esp
++0x092c   3f                   aas    
++0x092d   17                   pop    ss
++0x092e   da cb                fcmove st0, st3
++0x0930   25 a9 4a 67 71       and    eax, 0x71674aa9
++0x0935   a1 12 ed 0b 31       mov    eax, dword ptr [0x310bed12]
++0x093a   32 5c fd 9b          xor    bl, byte ptr [ebp+edi*8-0x65]
++0x093e   37                   aaa    
++0x093f   d7                   xlatb   
++0x0940   40                   inc    eax
++0x0941   28 38                sub    byte ptr [eax], bh
++0x0943   75 8e                jnz    0x0fffff90
++0x0945   ea 48 23 be f1 fe a9 jmp    0xa9fe:0xf1be2348
++0x094c   a4                   movsb   
++0x094d   36 0b 58 2d          or     ebx, dword ptr [ss:eax+0x2d]
++0x0951   13 eb                adc    ebp, ebx
++0x0953   11 8e 35 d2 1f 95    adc    dword ptr [esi+0x951fd235], ecx
++0x0959   89 46 3f             mov    dword ptr [esi+0x3f], eax
++0x095c   77 f9                jnbe   0x0ffffffb
++0x095e   2b e2                sub    esp, edx
++0x0960   83 37 a7             xor    dword ptr [edi], 0xa7
++0x0963   e2 d1                loop   0x0fffffd3
++0x0965   fd                   std    
++0x0966   8a b9 52 91 b4 46    mov    bh, byte ptr [ecx+0x46b49152]
++0x096c   ec                   in     al
++0x096d   b2 45                mov    dl, 0x45
++0x096f   92                   xchg   edx, eax
++0x0970   6a c5                push   0xc5
++0x0972   9b                   fwait  
++0x0973   9c                   pushfd  
++0x0974   fb                   sti    
++0x0975   ec                   in     al
++0x0976   7d 89                jnl    0x0fffff8b
++0x0978   76 44                jbe    0x10000046
++0x097a   92                   xchg   edx, eax
++0x097b   47                   inc    edi
++0x097c   92                   xchg   edx, eax
++0x097d   38 cd                cmp    ch, cl
++0x097f   e2 52                loop   0x10000054
++0x0981   49                   dec    ecx
++0x0982   ad                   lodsd   
++0x0983   5b                   pop    ebx
++0x0984   86 a7 60 93 48 ab    xchg   byte ptr [edi+0xab489360], ah
++0x098a   78 e2                js     0x0fffffe4
++0x098c   b9 04 a6 dc af       mov    ecx, 0xafdca604
++0x0991   ab                   stosd   
++0x0992   ce                   into   
++0x0993   13 b1 28 d8 17 ae    adc    esi, dword ptr [ecx+0xae17d828]
++0x0999   d0 5d 3a             rcr    byte ptr [ebp+0x3a], 0x01
++0x099c   29 01                sub    dword ptr [ecx], eax
++0x099e   bd cb 42 d3 ff       mov    ebp, 0xffd342cb
++0x09a3   22 e5                and    ah, ch
++0x09a5   54                   push   esp
++0x09a6   26 e1 0b             loope  0x1000000e
++0x09a9   4d                   dec    ebp
++0x09aa   32 d3                xor    dl, bl
++0x09ac   ee                   out    al, dx
++0x09ad   89 37                mov    dword ptr [edi], esi
++0x09af   7c d1                jl     0x0fffffd3
++0x09b1   c7 eb dd 45 84 6d...??  <INVALID>
++0x09b2   eb dd                jmp    0x0fffffdf
++0x09b4   45                   inc    ebp
++0x09b5   84 6d 9b             test   byte ptr [ebp-0x65], ch
++0x09b8   4f                   dec    edi
++0x09b9   57                   push   edi
++0x09ba   13 5a d9             adc    ebx, dword ptr [edx-0x27]
++0x09bd   d6                   salc   
++0x09be   83 31 5c             xor    dword ptr [ecx], 0x5c
++0x09c1   fe 4e c1             dec    byte ptr [esi-0x3f]
++0x09c4   09 02                or     dword ptr [edx], eax
++0x09c6   92                   xchg   edx, eax
++0x09c7   ce                   into   
++0x09c8   6f                   outsd   
++0x09c9   31 97 77 4b 31 84    xor    dword ptr [edi+0x84314b77], edx
++0x09cf   8d 54 69 f3          lea    edx, [ecx+ebp*2-0x0d]
++0x09d3   ba df bc 94 be       mov    edx, 0xbe94bcdf
++0x09d8   b9 7e 36 14 13       mov    ecx, 0x1314367e
++0x09dd   16                   push   ss
++0x09de   f0 c0 33 56...??     <INVALID>
++0x09df   c0 33 56             shl    byte ptr [ebx], 0x56
++0x09e2   8c 96 6c 4c 4b 19    mov    dword ptr [esi+0x194b4c6c], ss
++0x09e8   e6 bb                out    0xbb, al
++0x09ea   29 c1                sub    ecx, eax
++0x09ec   0c af                or     al, 0xaf
++0x09ee   71 29                jno    0x1000002b
++0x09f0   2f                   das    
++0x09f1   4e                   dec    esi
++0x09f2   9d                   popfd   
++0x09f3   31 a1 f6 88 09 2d    xor    dword ptr [ecx+0x2d0988f6], esp
++0x09f9   9b                   fwait  
++0x09fa   a1 91 98 bf bf       mov    eax, dword ptr [0xbfbf9891]
++0x09ff   5c                   pop    esp
++0x0a00   61                   popad   
++0x0a01   02 90 02 5d 7c c1    add    dl, byte ptr [eax+0xc17c5d02]
++0x0a07   d7                   xlatb   
++0x0a08   0e                   push   cs
++0x0a09   cf                   iretd   
++0x0a0a   fd                   std    
++0x0a0b   cc                   int3   
++0x0a0c   ab                   stosd   
++0x0a0d   d1 72 cc             shl    dword ptr [edx-0x34], 0x01
++0x0a10   38 f2                cmp    dl, dh
++0x0a12   71 ed                jno    0x0fffffef
++0x0a14   12 92 71 ac 71 a2    adc    dl, byte ptr [edx+0xa271ac71]
++0x0a1a   d6                   salc   
++0x0a1b   51                   push   ecx
++0x0a1c   cc                   int3   
++0x0a1d   ea 2e 9c 39 dd 7e be jmp    0xbe7e:0xdd399c2e
++0x0a24   82 38 67             cmp    byte ptr [eax], 0x67
++0x0a27   5f                   pop    edi
++0x0a28   7e 6e                jle    0x10000070
++0x0a2a   e4 2f                in     al, 0x2f
++0x0a2c   11 03                adc    dword ptr [ebx], eax
++0x0a2e   f7 fd                idiv   edx, eax, ebp
++0x0a30   06                   push   es
++0x0a31   8d 6a 92             lea    ebp, [edx-0x6e]
++0x0a34   45                   inc    ebp
++0x0a35   5e                   pop    esi
++0x0a36   31 25 e4 8c ee 1a    xor    dword ptr [0x1aee8ce4], esp
++0x0a3c   74 67                jz     0x10000069
++0x0a3e   ce                   into   
++0x0a3f   9e                   sahf   
++0x0a40   c1 66 47 f0          shl    dword ptr [esi+0x47], 0xf0
++0x0a44   a5                   movsd   
++0x0a45   45                   inc    ebp
++0x0a46   d1 81 6a 2a 14 1b    rol    dword ptr [ecx+0x1b142a6a], 0x01
++0x0a4c   cc                   int3   
++0x0a4d   b1 8a                mov    cl, 0x8a
++0x0a4f   f0 f2 3b c5...??     <INVALID>
++0x0a50   f2 3b c5             cmp    eax, ebp
++0x0a53   88 04 de             mov    byte ptr [esi+ebx*8], al
++0x0a56   04 91                add    al, 0x91
++0x0a58   63 17                arpl   word ptr [edi], dx
++0x0a5a   75 10                jnz    0x10000012
++0x0a5c   0e                   push   cs
++0x0a5d   13 4f 7a             adc    ecx, dword ptr [edi+0x7a]
++0x0a60   7a 26                jp     0x10000028
++0x0a62   37                   aaa    
++0x0a63   53                   push   ebx
++0x0a64   88 7d ce             mov    byte ptr [ebp-0x32], bh
++0x0a67   0c 23                or     al, 0x23
++0x0a69   cf                   iretd   
++0x0a6a   42                   inc    edx
++0x0a6b   0f d8 fc             psubusb mm7, mm4
++0x0a6e   e3 3f                jecxz  0x10000041
++0x0a70   2f                   das    
++0x0a71   2f                   das    
++0x0a72   93                   xchg   ebx, eax
++0x0a73   b2 8e                mov    dl, 0x8e
++0x0a75   a8 cd                test   al, 0xcd
++0x0a77   c6 9b 14 5b 1d 0b 66...??  <INVALID>
++0x0a78   9b                   fwait  
++0x0a79   14 5b                adc    al, 0x5b
++0x0a7b   1d 0b 66 08 07       sbb    eax, 0x0708660b
++0x0a80   f2 c0 e0 f5          shl    al, 0xf5
++0x0a84   07                   pop    es
++0x0a85   d6                   salc   
++0x0a86   93                   xchg   ebx, eax
++0x0a87   69 03 a2 9e d4 59    imul   eax, dword ptr [ebx], 0x59d49ea2
++0x0a8d   91                   xchg   ecx, eax
++0x0a8e   49                   dec    ecx
++0x0a8f   62 c7...??           <INVALID>
++0x0a90   c7 10 e0 ea 72 4f...??  <INVALID>
++0x0a91   10 e0                adc    al, ah
++0x0a93   ea 72 4f 23 e4 e8 e3 jmp    0xe3e8:0xe4234f72
++0x0a9a   85 28                test   dword ptr [eax], ebp
++0x0a9c   dd 79 a1             fnstsw word ptr [ecx-0x5f]
++0x0a9f   82 b0 fc 0e 75 ef b9 xor    byte ptr [eax+0xef750efc], 0xb9
++0x0aa6   a2 b0 f1 ba 39       mov    byte ptr [0x39baf1b0], al
++0x0aab   94                   xchg   esp, eax
++0x0aac   fd                   std    
++0x0aad   61                   popad   
++0x0aae   df 34 a9             fbstp  tbyte ptr [ecx+ebp*4]
++0x0ab1   4b                   dec    ebx
++0x0ab2   2e a4                movsb   cs
++0x0ab4   ec                   in     al
++0x0ab5   b9 92 69 e4 8e       mov    ecx, 0x8ee46992
++0x0aba   44                   inc    esp
++0x0abb   ba 0b 61 3d f7       mov    edx, 0xf73d610b
++0x0ac0   85 1c 1b             test   dword ptr [ebx+ebx], ebx
++0x0ac3   6d                   insd    
++0x0ac4   b5 66                mov    ch, 0x66
++0x0ac6   1b 82 07 31 39 9a    sbb    eax, dword ptr [edx+0x9a393107]
++0x0acc   e8 c8 66 74 f9       call   0x097466cd
++0x0ad1   0f bb 39             btc    dword ptr [ecx], edi
++0x0ad4   17                   pop    ss
++0x0ad5   7c 49                jl     0x1000004b
++0x0ad7   e3 56                jecxz  0x10000058
++0x0ad9   af                   scasd   
++0x0ada   a6                   cmpsb   
++0x0adb   9f                   lahf   
++0x0adc   1e                   push   ds
++0x0add   04 97                add    al, 0x97
++0x0adf   9c                   pushfd  
++0x0ae0   d6                   salc   
++0x0ae1   d5 06                aad    0x06
++0x0ae3   61                   popad   
++0x0ae4   d4 72                aam    0x72
++0x0ae6   c8 a7 fb 60          enter  0xfba7, 0x60
++0x0aea   ef                   out    eax, dx
++0x0aeb   89 42 43             mov    dword ptr [edx+0x43], eax
++0x0aee   43                   inc    ebx
++0x0aef   35 48 4a 60 00       xor    eax, 0x00604a48
++0x0af4   dc 75 ed             fdiv   qword ptr [ebp-0x13]
++0x0af7   99                   cdq    
++0x0af8   25 e6 52 04 e7       and    eax, 0xe70452e6
++0x0afd   9f                   lahf   
++0x0afe   1e                   push   ds
++0x0aff   74 00                jz     0x10000002
++0x0b01   53                   push   ebx
++0x0b02   0b 63 ea             or     esp, dword ptr [ebx-0x16]
++0x0b05   df 37                fbstp  tbyte ptr [edi]
++0x0b07   81 b4 36 7f 0d 92 5a xor    dword ptr [esi+esi+0x5a920d7f], 0xc38b1a72
+     72 1a 8b c3
++0x0b12   c5 46 d1             lds    eax, [esi-0x2f]
++0x0b15   40                   inc    eax
++0x0b16   f8                   clc    
++0x0b17   68 ab 87 f7 b5       push   0xb5f787ab
++0x0b1c   39 2d fa 95 7e bb    cmp    dword ptr [0xbb7e95fa], ebp
++0x0b22   4d                   dec    ebp
++0x0b23   85 a7 3c b1 8a 74    test   dword ptr [edi+0x748ab13c], esp
++0x0b29   1e                   push   ds
++0x0b2a   22 16                and    dl, byte ptr [esi]
++0x0b2c   86 bc c0 ed 52 68 36 xchg   byte ptr [eax+eax*8+0x366852ed], bh
++0x0b33   58                   pop    eax
++0x0b34   ed                   in     eax, dx
++0x0b35   aa                   stosb   
++0x0b36   9d                   popfd   
++0x0b37   fa                   cli    
++0x0b38   43                   inc    ebx
++0x0b39   ea 28 a1 49 15 12 d9 jmp    0xd912:0x1549a128
++0x0b40   da 22                fisub  dword ptr [edx]
++0x0b42   5f                   pop    edi
++0x0b43   9e                   sahf   
++0x0b44   fd                   std    
++0x0b45   7b b6                jnp    0x0fffffb8
++0x0b47   68 b7 0d 7c 65       push   0x657c0db7
++0x0b4c   57                   push   edi
++0x0b4d   27                   daa    
++0x0b4e   cc                   int3   
++0x0b4f   b5 b4                mov    ch, 0xb4
++0x0b51   15 47 47 74 fc       adc    eax, 0xfc744747
++0x0b56   b6 9b                mov    dh, 0x9b
++0x0b58   4c                   dec    esp
++0x0b59   79 72                jns    0x10000074
++0x0b5b   fd                   std    
++0x0b5c   30 c6                xor    dh, al
++0x0b5e   52                   push   edx
++0x0b5f   bd f4 4f 13 95       mov    ebp, 0x95134ff4
++0x0b64   32 24 48             xor    ah, byte ptr [eax+ecx*2]
++0x0b67   7f 42                jnle   0x10000044
++0x0b69   16                   push   ss
++0x0b6a   85 8a 94 36 90 ea    test   dword ptr [edx+0xea903694], ecx
++0x0b70   f3 ac                rep lodsb 
++0x0b72   f2 b9 9e 9c 3b 79    mov    ecx, 0x793b9c9e
++0x0b78   f9                   stc    
++0x0b79   ab                   stosd   
++0x0b7a   82 d6 3e             adc    dh, 0x3e
++0x0b7d   1c 04                sbb    al, 0x04
++0x0b7f   a2 94 f9 c2 cb       mov    byte ptr [0xcbc2f994], al
++0x0b84   72 a6                jb     0x0fffffa8
++0x0b86   1f                   pop    ds
++0x0b87   17                   pop    ss
++0x0b88   c1 3d 47 6b 38 44 be sar    dword ptr [0x44386b47], 0xbe
++0x0b8f   22 99 6b c6 6f f8    and    bl, byte ptr [ecx+0xf86fc66b]
++0x0b95   b5 93                mov    ch, 0x93
++0x0b97   5a                   pop    edx
++0x0b98   cd 82                int    0x82
++0x0b9a   97                   xchg   edi, eax
++0x0b9b   8a 5b aa             mov    bl, byte ptr [ebx-0x56]
++0x0b9e   aa                   stosb   
++0x0b9f   9a 46 b4 4d 0b 82 4f call   0x4f82:0x0b4db446
++0x0ba6   0a 6d 8a             or     ch, byte ptr [ebp-0x76]
++0x0ba9   78 9e                js     0x0fffffa0
++0x0bab   09 8c 30 4d ae 1a 3f or     dword ptr [eax+esi+0x3f1aae4d], ecx
++0x0bb2   7a a7                jp     0x0fffffa9
++0x0bb4   a5                   movsd   
++0x0bb5   6e                   outsb   
++0x0bb6   90                   nop    
++0x0bb7   2b 0a                sub    ecx, dword ptr [edx]
++0x0bb9   45                   inc    ebp
++0x0bba   ee                   out    al, dx
++0x0bbb   2b 02                sub    eax, dword ptr [edx]
++0x0bbd   f9                   stc    
++0x0bbe   5f                   pop    edi
++0x0bbf   14 bf                adc    al, 0xbf
++0x0bc1   f2 a1 35 0f 5f d4    mov    eax, dword ptr [0xd45f0f35]
++0x0bc7   75 63                jnz    0x10000065
++0x0bc9   ee                   out    al, dx
++0x0bca   d8 6b 1d             fsubr  dword ptr [ebx+0x1d]
++0x0bcd   bc 72 6e 69 e0       mov    esp, 0xe0696e72
++0x0bd2   78 c9                js     0x0fffffcb
++0x0bd4   09 28                or     dword ptr [eax], ebp
++0x0bd6   07                   pop    es
++0x0bd7   f0 20 6b 55          lock and    byte ptr [ebx+0x55], ch
++0x0bdb   a8 ca                test   al, 0xca
++0x0bdd   a7                   cmpsd   
++0x0bde   34 db                xor    al, 0xdb
++0x0be0   4e                   dec    esi
++0x0be1   f0 8a cb...??        <INVALID>
++0x0be2   8a cb                mov    cl, bl
++0x0be4   5d                   pop    ebp
++0x0be5   9a cd bc 05 e7 26 70 call   0x7026:0xe705bccd
++0x0bec   4e                   dec    esi
++0x0bed   4c                   dec    esp
++0x0bee   b5 50                mov    ch, 0x50
++0x0bf0   d7                   xlatb   
++0x0bf1   85 80 7a 48 e4 66    test   dword ptr [eax+0x66e4487a], eax
++0x0bf7   0d f4 61 10 d3       or     eax, 0xd31061f4
++0x0bfc   e8 34 95 45 b9       call   0xc9459539
++0x0c01   f2 9a 7e d9 5b 81 0c call   0x500c:0x815bd97e
+     50
++0x0c09   54                   push   esp
++0x0c0a   5a                   pop    edx
++0x0c0b   e3 d2                jecxz  0x0fffffd4
++0x0c0d   de 2a                fisubr word ptr [edx]
++0x0c0f   9d                   popfd   
++0x0c10   ef                   out    eax, dx
++0x0c11   20 3e                and    byte ptr [esi], bh
++0x0c13   bb 23 96 08 08       mov    ebx, 0x08089623
++0x0c18   52                   push   edx
++0x0c19   70 67                jo     0x10000069
++0x0c1b   32 6a 04             xor    ch, byte ptr [edx+0x04]
++0x0c1e   72 e4                jb     0x0fffffe6
++0x0c20   43                   inc    ebx
++0x0c21   f5                   cmc    
++0x0c22   f6 5e ad             neg    byte ptr [esi-0x53]
++0x0c25   5a                   pop    edx
++0x0c26   1c 3f                sbb    al, 0x3f
++0x0c28   c5 b9 56 43 a8 af    lds    edi, [ecx+0xafa84356]
++0x0c2e   90                   nop    
++0x0c2f   b0 b5                mov    al, 0xb5
++0x0c31   80 48 71 43          or     byte ptr [eax+0x71], 0x43
++0x0c35   92                   xchg   edx, eax
++0x0c36   d4 63                aam    0x63
++0x0c38   9a 6a c1 2e ed 37 ee call   0xee37:0xed2ec16a
++0x0c3f   15 b9 a4 97 a8       adc    eax, 0xa897a4b9
++0x0c44   67 74 ff             jz     0x10000002
++0x0c47   a4                   movsb   
++0x0c48   6b b6 a1 c5 fb 53 5a imul   esi, dword ptr [esi+0x53fbc5a1], 0x5a
++0x0c4f   7f 89                jnle   0x0fffff8b
++0x0c51   82 8f 49 76 36 9f 7d or     byte ptr [edi+0x9f367649], 0x7d
++0x0c58   50                   push   eax
++0x0c59   69 38 9b a5 4a 2c    imul   edi, dword ptr [eax], 0x2c4aa59b
++0x0c5f   28 5d 3e             sub    byte ptr [ebp+0x3e], bl
++0x0c62   23 99 08 0d 88 ce    and    ebx, dword ptr [ecx+0xce880d08]
++0x0c68   44                   inc    esp
++0x0c69   29 4d 11             sub    dword ptr [ebp+0x11], ecx
++0x0c6c   5c                   pop    esp
++0x0c6d   49                   dec    ecx
++0x0c6e   55                   push   ebp
++0x0c6f   d6                   salc   
++0x0c70   b4 36                mov    ah, 0x36
++0x0c72   7d c1                jnl    0x0fffffc3
++0x0c74   7b 3c                jnp    0x1000003e
++0x0c76   5a                   pop    edx
++0x0c77   41                   inc    ecx
++0x0c78   43                   inc    ebx
++0x0c79   0a e8                or     ch, al
++0x0c7b   28 2b                sub    byte ptr [ebx], ch
++0x0c7d   66 72 79             jb     0x0000007c
++0x0c80   2f                   das    
++0x0c81   2d 27 29 c3 c6       sub    eax, 0xc6c32927
++0x0c86   a3 32 d5 62 e2       mov    dword ptr [0xe262d532], eax
++0x0c8b   45                   inc    ebp
++0x0c8c   ee                   out    al, dx
++0x0c8d   93                   xchg   ebx, eax
++0x0c8e   cd 3c                int    0x3c
++0x0c90   63 b9 e3 df 9b d8    arpl   word ptr [ecx+0xd89bdfe3], di
++0x0c96   c5 f0...??           <INVALID>
++0x0c97   f0 f0 8f f1...??     <INVALID>
++0x0c98   f0 8f f1...??        <INVALID>
++0x0c99   8f f1...??           <INVALID>
++0x0c9a   f1                   int1   
++0x0c9b   86 e8                xchg   al, ch
++0x0c9d   05 71 d3 9e 35       add    eax, 0x359ed371
++0x0ca2   2a 87 c5 c6 32 69    sub    al, byte ptr [edi+0x6932c6c5]
++0x0ca8   eb 8f                jmp    0x0fffff91
++0x0caa   d7                   xlatb   
++0x0cab   0e                   push   cs
++0x0cac   32 37                xor    dh, byte ptr [edi]
++0x0cae   63 0c 7c             arpl   word ptr [esp+edi*2], cx
++0x0cb1   a8 4e                test   al, 0x4e
++0x0cb3   df 64 49 f3          fbld   tbyte ptr [ecx+ecx*2-0x0d]
++0x0cb7   58                   pop    eax
++0x0cb8   5f                   pop    edi
++0x0cb9   83 87 2a 6c ca 56 15 add    dword ptr [edi+0x56ca6c2a], 0x15
++0x0cc0   ae                   scasb   
++0x0cc1   c6 fa 1d...??        <INVALID>
++0x0cc2   fa                   cli    
++0x0cc3   1d 4f 40 44 a7       sbb    eax, 0xa744404f
++0x0cc8   cd 66                int    0x66
++0x0cca   19 36                sbb    dword ptr [esi], esi
++0x0ccc   21 36                and    dword ptr [esi], esi
++0x0cce   c6 1e 28...??        <INVALID>
++0x0ccf   1e                   push   ds
++0x0cd0   28 cf                sub    bh, cl
++0x0cd2   e8 6a 7f bd c8       call   0xd8bd7f6f
++0x0cd7   12 1e                adc    bl, byte ptr [esi]
++0x0cd9   ab                   stosd   
++0x0cda   74 17                jz     0x10000019
++0x0cdc   d0 c4                rol    ah, 0x01
++0x0cde   f1                   int1   
++0x0cdf   b3 4b                mov    bl, 0x4b
++0x0ce1   06                   push   es
++0x0ce2   0c d5                or     al, 0xd5
++0x0ce4   69 8a d3 7b 18 1c e8 imul   ecx, dword ptr [edx+0x1c187bd3], 0x6f7bffe8
+     ff 7b 6f
++0x0cee   1b fd                sbb    edi, ebp
++0x0cf0   53                   push   ebx
++0x0cf1   a8 4a                test   al, 0x4a
++0x0cf3   de bb e7 72 d4 db    fidivr word ptr [ebx+0xdbd472e7]
++0x0cf9   a3 c4 3a 1e fa       mov    dword ptr [0xfa1e3ac4], eax
++0x0cfe   97                   xchg   edi, eax
++0x0cff   c3                   ret    
++0x0d00   52                   push   edx
++0x0d01   57                   push   edi
++0x0d02   16                   push   ss
++0x0d03   4e                   dec    esi
++0x0d04   12 c5                adc    al, ch
++0x0d06   9b                   fwait  
++0x0d07   99                   cdq    
++0x0d08   ee                   out    al, dx
++0x0d09   85 8b ac 8c 89 d8    test   dword ptr [ebx+0xd8898cac], ecx
++0x0d0f   d8 50 db             fcom   dword ptr [eax-0x25]
++0x0d12   aa                   stosb   
++0x0d13   9f                   lahf   
++0x0d14   67 bb 22 1d eb ec    mov    ebx, 0xeceb1d22
++0x0d1a   d0 91 86 c6 a9 cc    rcl    byte ptr [ecx+0xcca9c686], 0x01
++0x0d20   c9                   leave  
++0x0d21   22 b7 67 e8 9f 10    and    dh, byte ptr [edi+0x109fe867]
++0x0d27   cb                   retf   
++0x0d28   26 bf 58 aa 84 7c    mov    edi, 0x7c84aa58
++0x0d2e   fc                   cld    
++0x0d2f   2e 39 dc             cmp    esp, ebx
++0x0d32   87 53 96             xchg   dword ptr [ebx-0x6a], edx
++0x0d35   1e                   push   ds
++0x0d36   8e 7d 1b...??        <INVALID>
++0x0d37   7d 1b                jnl    0x1000001d
++0x0d39   16                   push   ss
++0x0d3a   fa                   cli    
++0x0d3b   33 88 8e ca 5c dc    xor    ecx, dword ptr [eax+0xdc5cca8e]
++0x0d41   72 e2                jb     0x0fffffe4
++0x0d43   69 db 44 68 97 30    imul   ebx, ebx, 0x30976844
++0x0d49   1d cb c2 37 0d       sbb    eax, 0x0d37c2cb
++0x0d4e   2f                   das    
++0x0d4f   40                   inc    eax
++0x0d50   81 36 00 cb b6 1d    xor    dword ptr [esi], 0x1db6cb00
++0x0d56   b1 2c                mov    cl, 0x2c
++0x0d58   89 7e 91             mov    dword ptr [esi-0x6f], edi
++0x0d5b   c9                   leave  
++0x0d5c   88 3b                mov    byte ptr [ebx], bh
++0x0d5e   69 06 0a 63 4b 89    imul   eax, dword ptr [esi], 0x894b630a
++0x0d64   03 46 65             add    eax, dword ptr [esi+0x65]
++0x0d67   97                   xchg   edi, eax
++0x0d68   2b fa                sub    edi, edx
++0x0d6a   46                   inc    esi
++0x0d6b   f2 0b 68 a8          or     ebp, dword ptr [eax-0x58]
++0x0d6f   e1 8a                loope  0x0fffff8c
++0x0d71   2d 3e 57 9f 4b       sub    eax, 0x4b9f573e
++0x0d76   8d 17                lea    edx, [edi]
++0x0d78   e4 80                in     al, 0x80
++0x0d7a   73 01                jnb    0x10000003
++0x0d7c   97                   xchg   edi, eax
++0x0d7d   1c 22                sbb    al, 0x22
++0x0d7f   82 9f 37 16 c6 96 52 sbb    byte ptr [edi+0x96c61637], 0x52
++0x0d86   71 1e                jno    0x10000020
++0x0d88   07                   pop    es
++0x0d89   4c                   dec    esp
++0x0d8a   c2 e4 c7             ret    0xc7e4
++0x0d8d   16                   push   ss
++0x0d8e   df 23                fbld   tbyte ptr [ebx]
++0x0d90   bf 9b 47 ad 10       mov    edi, 0x10ad479b
++0x0d95   f2 9e                sahf   
++0x0d97   1a a8 c7 e4 b6 39    sbb    ch, byte ptr [eax+0x39b6e4c7]
++0x0d9d   dc 89 c2 c8 14 7b    fmul   qword ptr [ecx+0x7b14c8c2]
++0x0da3   97                   xchg   edi, eax
++0x0da4   f1                   int1   
++0x0da5   8b 1c 31             mov    ebx, dword ptr [ecx+esi]
++0x0da8   df c8                fxch   st0, st0, st0
++0x0daa   41                   inc    ecx
++0x0dab   5f                   pop    edi
++0x0dac   9d                   popfd   
++0x0dad   35 70 d8 b5 78       xor    eax, 0x78b5d870
++0x0db2   bc 3b 7c aa 72       mov    esp, 0x72aa7c3b
++0x0db7   3d af 82 a6 dc       cmp    eax, 0xdca682af
++0x0dbc   2f                   das    
++0x0dbd   d3 74 2c f4          shl    dword ptr [esp+ebp-0x0c], cl
++0x0dc1   fd                   std    
++0x0dc2   34 1d                xor    al, 0x1d
++0x0dc4   46                   inc    esi
++0x0dc5   0d 06 57 13 84       or     eax, 0x84135706
++0x0dca   f1                   int1   
++0x0dcb   60                   pushad  
++0x0dcc   e6 fe                out    0xfe, al
++0x0dce   f6 40 94 2f          test   byte ptr [eax-0x6c], 0x2f
++0x0dd2   63 6e 37             arpl   word ptr [esi+0x37], bp
++0x0dd5   0c a8                or     al, 0xa8
++0x0dd7   30 22                xor    byte ptr [edx], ah
++0x0dd9   96                   xchg   esi, eax
++0x0dda   91                   xchg   ecx, eax
++0x0ddb   00 42 08             add    byte ptr [edx+0x08], al
++0x0dde   7c 16                jl     0x10000018
++0x0de0   da 16                ficom  dword ptr [esi]
++0x0de2   80 21 be             and    byte ptr [ecx], 0xbe
++0x0de5   24 fd                and    al, 0xfd
++0x0de7   85 f6                test   esi, esi
++0x0de9   2c 33                sub    al, 0x33
++0x0deb   f1                   int1   
++0x0dec   76 4e                jbe    0x10000050
++0x0dee   0e                   push   cs
++0x0def   34 89                xor    al, 0x89
++0x0df1   bd 42 99 ea 24       mov    ebp, 0x24ea9942
++0x0df6   51                   push   ecx
++0x0df7   75 e3                jnz    0x0fffffe5
++0x0df9   c7 2e cb 42 ae 82...??  <INVALID>
++0x0dfa   2e cb                retf   
++0x0dfc   42                   inc    edx
++0x0dfd   ae                   scasb   
++0x0dfe   82 49 a2 10          or     byte ptr [ecx-0x5e], 0x10
++0x0e02   da 4f 76             fimul  dword ptr [edi+0x76]
++0x0e05   69 ad 34 a3 32 85 0f imul   ebp, dword ptr [ebp+0x8532a334], 0x1365310f
+     31 65 13
++0x0e0f   05 6a 75 36 92       add    eax, 0x9236756a
++0x0e14   b7 9f                mov    bh, 0x9f
++0x0e16   b8 0d 54 73 c9       mov    eax, 0xc973540d
++0x0e1b   35 52 4f 14 0d       xor    eax, 0x0d144f52
++0x0e20   c0 85 6b 55 95 c5 1f rol    byte ptr [ebp+0xc595556b], 0x1f
++0x0e27   73 7e                jnb    0x10000080
++0x0e29   31 5e 58             xor    dword ptr [esi+0x58], ebx
++0x0e2c   0a 19                or     bl, byte ptr [ecx]
++0x0e2e   23 5a fa             and    ebx, dword ptr [edx-0x06]
++0x0e31   84 b7 c5 b0 2a 4b    test   byte ptr [edi+0x4b2ab0c5], dh
++0x0e37   23 bc 61 95 0c e4 c6 and    edi, dword ptr [ecx+0xc6e40c95]
++0x0e3e   f9                   stc    
++0x0e3f   a0 be 47 a5 4d       mov    al, byte ptr [0x4da547be]
++0x0e44   23 7d 71             and    edi, dword ptr [ebp+0x71]
++0x0e47   d7                   xlatb   
++0x0e48   f0 76 7e...??        <INVALID>
++0x0e49   76 7e                jbe    0x10000080
++0x0e4b   b5 a9                mov    ch, 0xa9
++0x0e4d   39 62 52             cmp    dword ptr [edx+0x52], esp
++0x0e50   34 22                xor    al, 0x22
++0x0e52   07                   pop    es
++0x0e53   80 ee 45             sub    dh, 0x45
++0x0e56   2d 9d d7 34 b2       sub    eax, 0xb234d79d
++0x0e5b   b7 55                mov    bh, 0x55
++0x0e5d   b9 c1 c2 32 2e       mov    ecx, 0x2e32c2c1
++0x0e62   58                   pop    eax
++0x0e63   0b 83 d1 8e ac 00    or     eax, dword ptr [ebx+0x00ac8ed1]
++0x0e69   2f                   das    
++0x0e6a   2d de 5f 4f 89       sub    eax, 0x894f5fde
++0x0e6f   68 b1 1a 4e e0       push   0xe04e1ab1
++0x0e74   ab                   stosd   
++0x0e75   e7 bc                out    0xbc, eax
++0x0e77   5d                   pop    ebp
++0x0e78   25 00 8d 82 4f       and    eax, 0x4f828d00
++0x0e7d   90                   nop    
++0x0e7e   53                   push   ebx
++0x0e7f   34 20                xor    al, 0x20
++0x0e81   71 09                jno    0x1000000b
++0x0e83   81 9b 24 e9 3c 02 7e sbb    dword ptr [ebx+0x023ce924], 0xf8a9277e
+     27 a9 f8
++0x0e8d   b2 0b                mov    dl, 0x0b
++0x0e8f   dc de                fcomp  st0, st6
++0x0e91   43                   inc    ebx
++0x0e92   ee                   out    al, dx
++0x0e93   29 93 ca 86 18 2a    sub    dword ptr [ebx+0x2a1886ca], edx
++0x0e99   da 99 e8 59 3e 52    ficomp dword ptr [ecx+0x523e59e8]
++0x0e9f   39 2c 83             cmp    dword ptr [ebx+eax*4], ebp
++0x0ea2   e4 dc                in     al, 0xdc
++0x0ea4   5e                   pop    esi
++0x0ea5   f7 52 49             not    dword ptr [edx+0x49]
++0x0ea8   ac                   lodsb   
++0x0ea9   e4 43                in     al, 0x43
++0x0eab   1e                   push   ds
++0x0eac   a3 6e 27 88 fc       mov    dword ptr [0xfc88276e], eax
++0x0eb1   5a                   pop    edx
++0x0eb2   9a e5 d5 b5 ef f2 a3 call   0xa3f2:0xefb5d5e5
++0x0eb9   9a 7a 21 02 fa 8c ec call   0xec8c:0xfa02217a
++0x0ec0   a3 c2 76 e6 86       mov    dword ptr [0x86e676c2], eax
++0x0ec5   90                   nop    
++0x0ec6   3a 35 65 a2 11 27    cmp    dh, byte ptr [0x2711a265]
++0x0ecc   02 f2                add    dh, dl
++0x0ece   a7                   cmpsd   
++0x0ecf   da 99 df bc 93 63    ficomp dword ptr [ecx+0x6393bcdf]
++0x0ed5   b1 51                mov    cl, 0x51
++0x0ed7   fa                   cli    
++0x0ed8   9d                   popfd   
++0x0ed9   16                   push   ss
++0x0eda   55                   push   ebp
++0x0edb   10 a5 71 85 43 90    adc    byte ptr [ebp+0x90438571], ah
++0x0ee1   01 dd                add    ebp, ebx
++0x0ee3   0f f7 50 e9...??     <INVALID>
++0x0ee4   f7 50 e9             not    dword ptr [eax-0x17]
++0x0ee7   78 fc                js     0x0ffffffe
++0x0ee9   8a d7                mov    dl, bh
++0x0eeb   57                   push   edi
++0x0eec   57                   push   edi
++0x0eed   d3 e8                shr    eax, cl
++0x0eef   36 0d 2c 31 e8 25    or     eax, 0x25e8312c
++0x0ef5   93                   xchg   ebx, eax
++0x0ef6   4f                   dec    edi
++0x0ef7   1c 80                sbb    al, 0x80
++0x0ef9   23 31                and    esi, dword ptr [ecx]
++0x0efb   a8 14                test   al, 0x14
++0x0efd   60                   pushad  
++0x0efe   c3                   ret    
++0x0eff   41                   inc    ecx
++0x0f00   11 66 3b             adc    dword ptr [esi+0x3b], esp
++0x0f03   8f 34 3b...??        <INVALID>
++0x0f04   34 3b                xor    al, 0x3b
++0x0f06   59                   pop    ecx
++0x0f07   cc                   int3   
++0x0f08   d2 86 74 9d 7b 47    rol    byte ptr [esi+0x477b9d74], cl
++0x0f0e   1b c3                sbb    eax, ebx
++0x0f10   99                   cdq    
++0x0f11   53                   push   ebx
++0x0f12   de dc...??           <INVALID>
++0x0f13   dc 77 72             fdiv   qword ptr [edi+0x72]
++0x0f16   4e                   dec    esi
++0x0f17   8f 74 15 c6...??     <INVALID>
++0x0f18   74 15                jz     0x10000017
++0x0f1a   c6 79 fd 35...??     <INVALID>
++0x0f1b   79 fd                jns    0x0fffffff
++0x0f1d   35 9b 87 ad 5b       xor    eax, 0x5bad879b
++0x0f22   f9                   stc    
++0x0f23   1a 94 df a0 96 97 46 sbb    dl, byte ptr [edi+ebx*8+0x469796a0]
++0x0f2a   e3 aa                jecxz  0x0fffffac
++0x0f2c   8c 72 22...??        <INVALID>
++0x0f2d   72 22                jb     0x10000024
++0x0f2f   97                   xchg   edi, eax
++0x0f30   69 f7 5f 3f b5 a2    imul   esi, edi, 0xa2b53f5f
++0x0f36   50                   push   eax
++0x0f37   39 24 a9             cmp    dword ptr [ecx+ebp*4], esp
++0x0f3a   62 1e                bound  ebx, qword ptr [esi]
++0x0f3c   0b f7                or     esi, edi
++0x0f3e   64 e1 42             loope  0x10000045
++0x0f41   6f                   outsd   
++0x0f42   e8 f8 91 f1 96       call   0xa6f191fd
++0x0f47   c7 3a 64 fb b9 0f...??  <INVALID>
++0x0f48   3a 64 fb b9          cmp    ah, byte ptr [ebx+edi*8-0x47]
++0x0f4c   0f 61 b9 2b 9f 22 bb punpcklwd mm7, qword ptr [ecx+0xbb229f2b]
++0x0f53   08 9c f4 4b f7 6b e7 or     byte ptr [esp+esi*8+0xe76bf74b], bl
++0x0f5a   a8 67                test   al, 0x67
++0x0f5c   c1 22 17             shl    dword ptr [edx], 0x17
++0x0f5f   e5 fd                in     eax, 0xfd
++0x0f61   9b                   fwait  
++0x0f62   3d 7f f3 16 08       cmp    eax, 0x0816f37f
++0x0f67   11 28                adc    dword ptr [eax], ebp
++0x0f69   68 cf d8 4d 84       push   0x844dd8cf
++0x0f6e   2a 01                sub    al, byte ptr [ecx]
++0x0f70   aa                   stosb   
++0x0f71   c1 a8 e7 4d 99 5a 0d shr    dword ptr [eax+0x5a994de7], 0x0d
++0x0f78   41                   inc    ecx
++0x0f79   bf 4c 3e e6 35       mov    edi, 0x35e63e4c
++0x0f7e   98                   cwde   
++0x0f7f   00 6f f6             add    byte ptr [edi-0x0a], ch
++0x0f82   2c 99                sub    al, 0x99
++0x0f84   7e f3                jle    0x0ffffff5
++0x0f86   56                   push   esi
++0x0f87   a6                   cmpsb   
++0x0f88   41                   inc    ecx
++0x0f89   64 81 6c 60 68 89 6c sub    dword ptr [fs:eax+0x68], 0x7ef16c89
+     f1 7e
++0x0f92   30 76 b2             xor    byte ptr [esi-0x4e], dh
++0x0f95   3c a4                cmp    al, 0xa4
++0x0f97   c0 8e 02 55 00 4d 08 ror    byte ptr [esi+0x4d005502], 0x08
++0x0f9e   ae                   scasb   
++0x0f9f   f7 e7                mul    edx, eax, edi
++0x0fa1   37                   aaa    
++0x0fa2   e6 11                out    0x11, al
++0x0fa4   0a a7 7a 92 e5 7e    or     ah, byte ptr [edi+0x7ee5927a]
++0x0faa   dd c9                fxch   st0, st1, st0
++0x0fac   59                   pop    ecx
++0x0fad   04 60                add    al, 0x60
++0x0faf   45                   inc    ebp
++0x0fb0   60                   pushad  
++0x0fb1   32 9b ce 87 f3 35    xor    bl, byte ptr [ebx+0x35f387ce]
++0x0fb7   02 87 e3 87 80 1c    add    al, byte ptr [edi+0x1c8087e3]
++0x0fbd   9e                   sahf   
++0x0fbe   2b f1                sub    esi, ecx
++0x0fc0   c4 e8...??           <INVALID>
++0x0fc1   e8 4a 6e 24 50       call   0x60246e4f
++0x0fc6   9d                   popfd   
++0x0fc7   5d                   pop    ebp
++0x0fc8   cd 92                int    0x92
++0x0fca   d1 95 30 c4 f7 19    rcl    dword ptr [ebp+0x19f7c430], 0x01
++0x0fd0   cf                   iretd   
++0x0fd1   a2 61 d9 14 98       mov    byte ptr [0x9814d961], al
++0x0fd6   16                   push   ss
++0x0fd7   ab                   stosd   
++0x0fd8   a0 13 4d 44 e6       mov    al, byte ptr [0xe6444d13]
++0x0fdd   29 93 2a 18 f7 fe    sub    dword ptr [ebx+0xfef7182a], edx
++0x0fe3   2d 1c 97 d0 8f       sub    eax, 0x8fd0971c
++0x0fe8   52                   push   edx
++0x0fe9   36 1b cc             sbb    ecx, esp
++0x0fec   3e ca b0 3a          retf   0x3ab0
++0x0ff0   34 97                xor    al, 0x97
++0x0ff2   11 38                adc    dword ptr [eax], edi
++0x0ff4   7e 97                jle    0x0fffff99
++0x0ff6   11 bb 1d 2e ac 12    adc    dword ptr [ebx+0x12ac2e1d], edi
++0x0ffc   d1 d3                rcl    ebx, 0x01
++0x0ffe   7b a9                jnp    0x0fffffab
++0x1000   db 54 f7 61          fist   dword ptr [edi+esi*8+0x61]
++0x1004   77 aa                jnbe   0x0fffffac
++0x1006   af                   scasd   
++0x1007   50                   push   eax
++0x1008   bc 27 58 0f 4e       mov    esp, 0x4e0f5827
++0x100d   4f                   dec    edi
++0x100e   f9                   stc    
++0x100f   8f 94 7b 14 94 53 57...??  <INVALID>
++0x1010   94                   xchg   esp, eax
++0x1011   7b 14                jnp    0x10000016
++0x1013   94                   xchg   esp, eax
++0x1014   53                   push   ebx
++0x1015   57                   push   edi
++0x1016   75 47                jnz    0x10000049
++0x1018   a3 68 44 9b 92       mov    dword ptr [0x929b4468], eax
++0x101d   2c 57                sub    al, 0x57
++0x101f   81 a4 b6 56 96 02 0e and    dword ptr [esi+esi*4+0x0e029656], 0x718b6ad2
+     d2 6a 8b 71
++0x102a   96                   xchg   esi, eax
++0x102b   f8                   clc    
++0x102c   3f                   aas    
++0x102d   91                   xchg   ecx, eax
++0x102e   e9 2e d2 fe 41       jmp    0x51fed233
++0x1033   c7 d8 80 3c 0c d0...??  <INVALID>
++0x1034   d8 80 3c 0c d0 58    fadd   dword ptr [eax+0x58d00c3c]
++0x103a   4d                   dec    ebp
++0x103b   7a 14                jp     0x10000016
++0x103d   6f                   outsd   
++0x103e   a8 78                test   al, 0x78
++0x1040   b7 d3                mov    bh, 0xd3
++0x1042   e7 69                out    0x69, eax
++0x1044   a2 22 11 66 3a       mov    byte ptr [0x3a661122], al
++0x1049   d5 f7                aad    0xf7
++0x104b   5c                   pop    esp
++0x104c   31 61 b1             xor    dword ptr [ecx-0x4f], esp
++0x104f   cf                   iretd   
++0x1050   4f                   dec    edi
++0x1051   8c 18                mov    dword ptr [eax], ds
++0x1053   a2 77 ce 89 66       mov    byte ptr [0x6689ce77], al
++0x1058   13 80 b8 b3 08 9b    adc    eax, dword ptr [eax+0x9b08b3b8]
++0x105e   19 31                sbb    dword ptr [ecx], esi
++0x1060   d3 ec                shr    esp, cl
++0x1062   78 d1                js     0x0fffffd3
++0x1064   e9 c8 fb 20 20       jmp    0x3020fbcd
++0x1069   3f                   aas    
++0x106a   a7                   cmpsd   
++0x106b   7d 23                jnl    0x10000025
++0x106d   8c bf c3 aa 45 64...??  <INVALID>
++0x106e   bf c3 aa 45 64       mov    edi, 0x6445aac3
++0x1073   85 68 57             test   dword ptr [eax+0x57], ebp
++0x1076   50                   push   eax
++0x1077   e4 10                in     al, 0x10
++0x1079   cc                   int3   
++0x107a   e1 d5                loope  0x0fffffd7
++0x107c   79 d6                jns    0x0fffffd8
++0x107e   e8 87 74 f4 8d       call   0x9df4748c
++0x1083   b4 7c                mov    ah, 0x7c
++0x1085   d1 3c 02             sar    dword ptr [edx+eax], 0x01
++0x1088   be e8 db de 2e       mov    esi, 0x2ededbe8
++0x108d   c1 83 9b e5 65 1d 60 rol    dword ptr [ebx+0x1d65e59b], 0x60
++0x1094   af                   scasd   
++0x1095   b1 e0                mov    cl, 0xe0
++0x1097   4a                   dec    edx
++0x1098   09 79 93             or     dword ptr [ecx-0x6d], edi
++0x109b   27                   daa    
++0x109c   04 7c                add    al, 0x7c
++0x109e   ce                   into   
++0x109f   81 d4 15 17 10 05    adc    esp, 0x05101715
++0x10a5   75 e2                jnz    0x0fffffe4
++0x10a7   1c 4d                sbb    al, 0x4d
++0x10a9   4f                   dec    edi
++0x10aa   0a 22                or     ah, byte ptr [edx]
++0x10ac   68 02 30 07 4c       push   0x4c073002
++0x10b1   3e 4a                dec    edx
++0x10b3   66 fc                cld    
++0x10b5   64 18 1e             sbb    byte ptr [fs:esi], bl
++0x10b8   77 82                jnbe   0x0fffff84
++0x10ba   5d                   pop    ebp
++0x10bb   55                   push   ebp
++0x10bc   ca 95 b2             retf   0xb295
++0x10bf   56                   push   esi
++0x10c0   0f b4 95 59 1c 08 dd lfs    edx, [ebp+0xdd081c59]
++0x10c7   88 88 d4 11 85 0a    mov    byte ptr [eax+0x0a8511d4], cl
++0x10cd   c5 cb...??           <INVALID>
++0x10ce   cb                   retf   
++0x10cf   cf                   iretd   
++0x10d0   9a eb af a3 05 40 81 call   0x8140:0x05a3afeb
++0x10d7   e3 4a                jecxz  0x1000004c
++0x10d9   97                   xchg   edi, eax
++0x10da   9a 31 f4 56 98 aa f0 call   0xf0aa:0x9856f431
++0x10e1   ab                   stosd   
++0x10e2   64 56                push   esi
++0x10e4   81 c8 68 a8 69 57    or     eax, 0x5769a868
++0x10ea   d9 74 43 3a          fnstenv [ebx+eax*2+0x3a]
++0x10ee   dc 4a 8a             fmul   qword ptr [edx-0x76]
++0x10f1   36 17                pop    ss
++0x10f3   5d                   pop    ebp
++0x10f4   7e ad                jle    0x0fffffaf
++0x10f6   5b                   pop    ebx
++0x10f7   15 1f b1 e1 90       adc    eax, 0x90e1b11f
++0x10fc   ef                   out    eax, dx
++0x10fd   94                   xchg   esp, eax
++0x10fe   49                   dec    ecx
++0x10ff   a7                   cmpsd   
++0x1100   96                   xchg   esi, eax
++0x1101   2b fd                sub    edi, ebp
++0x1103   f6 af bb 85 0a 09    imul   ax, byte ptr [edi+0x090a85bb]
++0x1109   d1 d3                rcl    ebx, 0x01
++0x110b   be 26 ab 7e 2e       mov    esi, 0x2e7eab26
++0x1110   e0 d9                loopne 0x0fffffdb
++0x1112   73 6c                jnb    0x1000006e
++0x1114   f8                   clc    
++0x1115   e6 76                out    0x76, al
++0x1117   58                   pop    eax
++0x1118   e2 1d                loop   0x1000001f
++0x111a   a2 df 0f 4d 04       mov    byte ptr [0x044d0fdf], al
++0x111f   13 4e 2e             adc    ecx, dword ptr [esi+0x2e]
++0x1122   b5 de                mov    ch, 0xde
++0x1124   ca df 02             retf   0x02df
++0x1127   ed                   in     eax, dx
++0x1128   e0 83                loopne 0x0fffff85
++0x112a   22 55 28             and    dl, byte ptr [ebp+0x28]
++0x112d   fe a7 38 14 85 8b...??  <INVALID>
++0x112e   a7                   cmpsd   
++0x112f   38 14 85 8b 2c 61 f6 cmp    byte ptr [eax*4+0xf6612c8b], dl
++0x1136   a5                   movsd   
++0x1137   ad                   lodsd   
++0x1138   3b 82 12 be 4b 95    cmp    eax, dword ptr [edx+0x954bbe12]
++0x113e   50                   push   eax
++0x113f   f1                   int1   
++0x1140   ad                   lodsd   
++0x1141   70 c8                jo     0x0fffffca
++0x1143   5e                   pop    esi
++0x1144   52                   push   edx
++0x1145   88 e0                mov    al, ah
++0x1147   cf                   iretd   
++0x1148   55                   push   ebp
++0x1149   81 c0 2a be 14 30    add    eax, 0x3014be2a
++0x114f   91                   xchg   ecx, eax
++0x1150   3b 71 08             cmp    esi, dword ptr [ecx+0x08]
++0x1153   c2 83 6b             ret    0x6b83
++0x1156   b6 be                mov    dh, 0xbe
++0x1158   6c                   insb    
++0x1159   63 0a                arpl   word ptr [edx], cx
++0x115b   21 17                and    dword ptr [edi], edx
++0x115d   f8                   clc    
++0x115e   e1 83                loope  0x0fffff85
++0x1160   f3 99                cdq    
++0x1162   1f                   pop    ds
++0x1163   7b aa                jnp    0x0fffffac
++0x1165   1d f1 c8 64 9a       sbb    eax, 0x9a64c8f1
++0x116a   b7 98                mov    bh, 0x98
++0x116c   29 97 b1 1a e8 d9    sub    dword ptr [edi+0xd9e81ab1], edx
++0x1172   ce                   into   
++0x1173   51                   push   ecx
++0x1174   a3 80 f2 84 b8       mov    dword ptr [0xb884f280], eax
++0x1179   fb                   sti    
++0x117a   90                   nop    
++0x117b   97                   xchg   edi, eax
++0x117c   57                   push   edi
++0x117d   78 ff                js     0x10000001
++0x117f   f2 6c                insb    
++0x1181   0f 53 e7             rcpps  xmm4, xmm7
++0x1184   dd 1d 47 c7 ac 67    fstp   qword ptr [0x67acc747]
++0x118a   03 91 f2 04 92 8a    add    edx, dword ptr [ecx+0x8a9204f2]
++0x1190   2f                   das    
++0x1191   c5 55 18             lds    edx, [ebp+0x18]
++0x1194   3c ce                cmp    al, 0xce
++0x1196   9d                   popfd   
++0x1197   d0 5d dc             rcr    byte ptr [ebp-0x24], 0x01
++0x119a   79 a3                jns    0x0fffffa5
++0x119c   13 25 e8 06 9f 44    adc    esp, dword ptr [0x449f06e8]
++0x11a2   bc 52 76 82 81       mov    esp, 0x81827652
++0x11a7   fd                   std    
++0x11a8   c9                   leave  
++0x11a9   ce                   into   
++0x11aa   c6 f3 0e...??        <INVALID>
++0x11ab   f3 0e                push   cs
++0x11ad   b9 00 4d ba 83       mov    ecx, 0x83ba4d00
++0x11b2   f5                   cmc    
++0x11b3   1d 76 b3 33 42       sbb    eax, 0x4233b376
++0x11b8   5d                   pop    ebp
++0x11b9   0e                   push   cs
++0x11ba   17                   pop    ss
++0x11bb   f2 c1 10 25          rcl    dword ptr [eax], 0x25
++0x11bf   fe 6f 72...??        <INVALID>
++0x11c0   6f                   outsd   
++0x11c1   72 18                jb     0x1000001a
++0x11c3   a8 b6                test   al, 0xb6
++0x11c5   ad                   lodsd   
++0x11c6   e1 2e                loope  0x10000030
++0x11c8   ce                   into   
++0x11c9   e3 30                jecxz  0x10000032
++0x11cb   d7                   xlatb   
++0x11cc   54                   push   esp
++0x11cd   c1 07 91             rol    dword ptr [edi], 0x91
++0x11d0   8a 8f 17 7c 22 7c    mov    cl, byte ptr [edi+0x7c227c17]
++0x11d6   a3 35 de b6 4a       mov    dword ptr [0x4ab6de35], eax
++0x11db   bf f1 12 fc 41       mov    edi, 0x41fc12f1
++0x11e0   cb                   retf   
++0x11e1   0c e5                or     al, 0xe5
++0x11e3   24 e3                and    al, 0xe3
++0x11e5   fd                   std    
++0x11e6   8c b4 77 b6 9b ac e9...??  <INVALID>
++0x11e7   b4 77                mov    ah, 0x77
++0x11e9   b6 9b                mov    dh, 0x9b
++0x11eb   ac                   lodsb   
++0x11ec   e9 a1 cb 96 a8       jmp    0xb896cba6
++0x11f1   f6 24 fe             mul    ax, byte ptr [esi+edi*8]
++0x11f4   ef                   out    eax, dx
++0x11f5   4e                   dec    esi
++0x11f6   9c                   pushfd  
++0x11f7   95                   xchg   ebp, eax
++0x11f8   96                   xchg   esi, eax
++0x11f9   32 0b                xor    cl, byte ptr [ebx]
++0x11fb   83 12 1d             adc    dword ptr [edx], 0x1d
++0x11fe   6a 1b                push   0x1b
++0x1200   ef                   out    eax, dx
++0x1201   35 e7 7a d7 ea       xor    eax, 0xead77ae7
++0x1206   c3                   ret    
++0x1207   08 2a                or     byte ptr [edx], ch
++0x1209   66 d6                salc   
++0x120b   5a                   pop    edx
++0x120c   72 cc                jb     0x0fffffce
++0x120e   4b                   dec    ebx
++0x120f   0a df                or     bl, bh
++0x1211   9f                   lahf   
++0x1212   e8 50 ec 74 73       call   0x8374ec55
++0x1217   09 50 1e             or     dword ptr [eax+0x1e], edx
++0x121a   ed                   in     eax, dx
++0x121b   eb 35                jmp    0x10000037
++0x121d   68 c4 27 fc 44       push   0x44fc27c4
++0x1222   9e                   sahf   
++0x1223   60                   pushad  
++0x1224   cd 97                int    0x97
++0x1226   d5 64                aad    0x64
++0x1228   f1                   int1   
++0x1229   05 7a f2 b9 2d       add    eax, 0x2db9f27a
++0x122e   48                   dec    eax
++0x122f   19 fc                sbb    esp, edi
++0x1231   1e                   push   ds
++0x1232   3b c5                cmp    eax, ebp
++0x1234   eb b2                jmp    0x0fffffb4
++0x1236   a0 d5 d4 9b 89       mov    al, byte ptr [0x899bd4d5]
++0x123b   1d c0 e1 15 b6       sbb    eax, 0xb615e1c0
++0x1240   2a b1 25 e2 3a 0e    sub    dh, byte ptr [ecx+0x0e3ae225]
++0x1246   81 61 8a 36 86 9f b9 and    dword ptr [ecx-0x76], 0xb99f8636
++0x124d   84 89 43 a9 bb c7    test   byte ptr [ecx+0xc7bba943], cl
++0x1253   30 57 c3             xor    byte ptr [edi-0x3d], dl
++0x1256   1b 00                sbb    eax, dword ptr [eax]
++0x1258   86 1a                xchg   byte ptr [edx], bl
++0x125a   a2 2e 39 0c ab       mov    byte ptr [0xab0c392e], al
++0x125f   f4                   hlt    
++0x1260   6c                   insb    
++0x1261   43                   inc    ebx
++0x1262   08 5c 58 9e          or     byte ptr [eax+ebx*2-0x62], bl
++0x1266   ee                   out    al, dx
++0x1267   a3 4f 32 2d e5       mov    dword ptr [0xe52d324f], eax
++0x126c   4f                   dec    edi
++0x126d   5a                   pop    edx
++0x126e   a0 6e 31 f8 ab       mov    al, byte ptr [0xabf8316e]
++0x1273   84 3a                test   byte ptr [edx], bh
++0x1275   d1 2f                shr    dword ptr [edi], 0x01
++0x1277   dc c1                fadd   st1, st0
++0x1279   47                   inc    edi
++0x127a   de 7d d3             fidivr word ptr [ebp-0x2d]
++0x127d   5c                   pop    esp
++0x127e   8c d1                mov    ecx, ss
++0x1280   ef                   out    eax, dx
++0x1281   8f 7b 04...??        <INVALID>
++0x1282   7b 04                jnp    0x10000006
++0x1284   b6 70                mov    dh, 0x70
++0x1286   ef                   out    eax, dx
++0x1287   66 d1 29             shr    word ptr [ecx], 0x01
++0x128a   0f 79 55 ca...??     <INVALID>
++0x128b   79 55                jns    0x10000057
++0x128d   ca 07 8a             retf   0x8a07
++0x1290   f6 9e 78 16 31 2e    neg    byte ptr [esi+0x2e311678]
++0x1296   2e d6                salc   
++0x1298   d1 8f ae 12 17 7d    ror    dword ptr [edi+0x7d1712ae], 0x01
++0x129e   5d                   pop    ebp
++0x129f   3c d6                cmp    al, 0xd6
++0x12a1   c0 69 8a ab          shr    byte ptr [ecx-0x76], 0xab
++0x12a5   36 73 6f             jnb    0x10000072
++0x12a8   11 f0                adc    eax, esi
++0x12aa   b3 2a                mov    bl, 0x2a
++0x12ac   a0 cf e9 ef 01       mov    al, byte ptr [0x01efe9cf]
++0x12b1   74 6c                jz     0x1000006e
++0x12b3   1a 67 8e             sbb    ah, byte ptr [edi-0x72]
++0x12b6   a6                   cmpsb   
++0x12b7   b8 09 92 dd 03       mov    eax, 0x03dd9209
++0x12bc   84 69 04             test   byte ptr [ecx+0x04], ch
++0x12bf   d2 b3 74 11 b1 e7    shl    byte ptr [ebx+0xe7b11174], cl
++0x12c5   d0 74 58 dd          shl    byte ptr [eax+ebx*2-0x23], 0x01
++0x12c9   a9 b6 99 56 ad       test   eax, 0xad5699b6
++0x12ce   6c                   insb    
++0x12cf   f3 cd c5             int    0xc5
++0x12d2   69 b6 f4 ba 18 d4 a9 imul   esi, dword ptr [esi+0xd418baf4], 0xdc3e6aa9
+     6a 3e dc
++0x12dc   a0 67 a3 f5 14       mov    al, byte ptr [0x14f5a367]
++0x12e1   da c9                fcmove st0, st1
++0x12e3   19 62 9c             sbb    dword ptr [edx-0x64], esp
++0x12e6   3d d1 46 7b 3f       cmp    eax, 0x3f7b46d1
++0x12eb   17                   pop    ss
++0x12ec   93                   xchg   ebx, eax
++0x12ed   fd                   std    
++0x12ee   ee                   out    al, dx
++0x12ef   e4 2d                in     al, 0x2d
++0x12f1   29 8c a9 1b ed 57 32 sub    dword ptr [ecx+ebp*4+0x3257ed1b], ecx
++0x12f8   f4                   hlt    
++0x12f9   ca a7 52             retf   0x52a7
++0x12fc   f2 27                daa    
++0x12fe   f8                   clc    
++0x12ff   7a 71                jp     0x10000073
++0x1301   59                   pop    ecx
++0x1302   b5 16                mov    ch, 0x16
++0x1304   5f                   pop    edi
++0x1305   a5                   movsd   
++0x1306   44                   inc    esp
++0x1307   a0 88 6f 67 0f       mov    al, byte ptr [0x0f676f88]
++0x130c   a5                   movsd   
++0x130d   8d 1f                lea    ebx, [edi]
++0x130f   d6                   salc   
++0x1310   1f                   pop    ds
++0x1311   00 f9                add    cl, bh
++0x1313   16                   push   ss
++0x1314   48                   dec    eax
++0x1315   c3                   ret    
++0x1316   c5 bf 93 bc a0 9a    lds    edi, [edi+0x9aa0bc93]
++0x131c   dd 55 a0             fst    qword ptr [ebp-0x60]
++0x131f   32 69 f4             xor    ch, byte ptr [ecx-0x0c]
++0x1322   66 7c f6             jl     0x0000fff9
++0x1325   f1                   int1   
++0x1326   4a                   dec    edx
++0x1327   7f 24                jnle   0x10000026
++0x1329   5b                   pop    ebx
++0x132a   17                   pop    ss
++0x132b   3c 16                cmp    al, 0x16
++0x132d   21 c9                and    ecx, ecx
++0x132f   54                   push   esp
++0x1330   ce                   into   
++0x1331   fb                   sti    
++0x1332   a1 6d 2e 5f 2e       mov    eax, dword ptr [0x2e5f2e6d]
++0x1337   3a 1b                cmp    bl, byte ptr [ebx]
++0x1339   d9 f5                fprem1 
++0x133b   45                   inc    ebp
++0x133c   67 6f                outsd   
++0x133e   60                   pushad  
++0x133f   41                   inc    ecx
++0x1340   de bf fd fb a7 19    fidivr word ptr [edi+0x19a7fbfd]
++0x1346   c4 07                les    eax, [edi]
++0x1348   e5 3a                in     eax, 0x3a
++0x134a   1e                   push   ds
++0x134b   69 a5 9a af 2d b2 78 imul   esp, dword ptr [ebp+0xb22daf9a], 0xf9b49a78
+     9a b4 f9
++0x1355   65 82 9d a0 f2 52 94 sbb    byte ptr [gs:ebp+0x9452f2a0], 0xeb
+     eb
++0x135d   b6 5a                mov    dh, 0x5a
++0x135f   e0 a6                loopne 0x0fffffa8
++0x1361   08 2d f1 31 1f c9    or     byte ptr [0xc91f31f1], ch
++0x1367   1b c5                sbb    eax, ebp
++0x1369   6e                   outsb   
++0x136a   16                   push   ss
++0x136b   de d8...??           <INVALID>
++0x136c   d8 bb db 3f 5a d1    fdivr  dword ptr [ebx+0xd15a3fdb]
++0x1372   23 eb                and    ebp, ebx
++0x1374   a4                   movsb   
++0x1375   a7                   cmpsd   
++0x1376   36 ea f7 94 d0 f3 f8 jmp    0x69f8:0xf3d094f7
+     69
++0x137e   04 4e                add    al, 0x4e
++0x1380   87 26                xchg   dword ptr [esi], esp
++0x1382   19 71 6f             sbb    dword ptr [ecx+0x6f], esi
++0x1385   58                   pop    eax
++0x1386   1e                   push   ds
++0x1387   8e a0 5c d5 d6 cf    mov    fs, dword ptr [eax+0xcfd6d55c]
++0x138d   63 62 4d             arpl   word ptr [edx+0x4d], sp
++0x1390   88 f2                mov    dl, dh
++0x1392   af                   scasd   
++0x1393   d7                   xlatb   
++0x1394   2a 98 85 29 d5 cd    sub    bl, byte ptr [eax+0xcdd52985]
++0x139a   a1 94 51 13 9a       mov    eax, dword ptr [0x9a135194]
++0x139f   65 3e 65 35 5f ef 07 xor    eax, 0xce07ef5f
+     ce
++0x13a7   50                   push   eax
++0x13a8   8e d5                mov    ss, ebp
++0x13aa   6c                   insb    
++0x13ab   80 64 85 ac 9f       and    byte ptr [ebp+eax*4-0x54], 0x9f
++0x13b0   54                   push   esp
++0x13b1   bc 68 0f 95 e5       mov    esp, 0xe5950f68
++0x13b6   a0 12 fe fe 9d       mov    al, byte ptr [0x9dfefe12]
++0x13bb   ad                   lodsd   
++0x13bc   f2 71 dd             jno    0x0fffffe0
++0x13bf   72 09                jb     0x1000000b
++0x13c1   d7                   xlatb   
++0x13c2   09 23                or     dword ptr [ebx], esp
++0x13c4   61                   popad   
++0x13c5   7a a9                jp     0x0fffffab
++0x13c7   ff 9c ca 96 7f 41 9a call   
++0x13ce   be 0e 14 ba e6       mov    esi, 0xe6ba140e
++0x13d3   e1 8a                loope  0x0fffff8c
++0x13d5   f1                   int1   
++0x13d6   00 da                add    dl, bl
++0x13d8   d3 32                shl    dword ptr [edx], cl
++0x13da   38 fc                cmp    ah, bh
++0x13dc   15 35 b3 e8 6c       adc    eax, 0x6ce8b335
++0x13e1   d0 e9                shr    cl, 0x01
++0x13e3   2e 36 d9 95 37 eb 48 fst    dword ptr [ss:ebp+0x1548eb37]
+     15
++0x13eb   14 16                adc    al, 0x16
++0x13ed   c7 06 6e e9 74 9b    mov    dword ptr [esi], 0x9b74e96e
++0x13f3   a0 53 10 d8 d7       mov    al, byte ptr [0xd7d81053]
++0x13f8   bd e0 ff ec a9       mov    ebp, 0xa9ecffe0
++0x13fd   23 c9                and    ecx, ecx
++0x13ff   72 f0                jb     0x0ffffff2
++0x1401   d9 4f 99...??        <INVALID>
++0x1402   4f                   dec    edi
++0x1403   99                   cdq    
++0x1404   af                   scasd   
++0x1405   6a a3                push   0xa3
++0x1407   a5                   movsd   
++0x1408   71 f0                jno    0x0ffffff2
++0x140a   f1                   int1   
++0x140b   9d                   popfd   
++0x140c   45                   inc    ebp
++0x140d   c2 d8 07             ret    0x07d8
++0x1410   42                   inc    edx
++0x1411   3e 66 b8 54 3b       mov    ax, 0x3b54
++0x1416   69 21 c3 5b ee 49    imul   esp, dword ptr [ecx], 0x49ee5bc3
++0x141c   5c                   pop    esp
++0x141d   25 8a f2 0e 91       and    eax, 0x910ef28a
++0x1422   4c                   dec    esp
++0x1423   96                   xchg   esi, eax
++0x1424   f9                   stc    
++0x1425   7e 13                jle    0x10000015
++0x1427   eb 99                jmp    0x0fffff9b
++0x1429   85 2d d4 4c 2f cd    test   dword ptr [0xcd2f4cd4], ebp
++0x142f   be 0a 61 06 a8       mov    esi, 0xa806610a
++0x1434   d9 b8 95 41 22 e1    fnstcw word ptr [eax+0xe1224195]
++0x143a   92                   xchg   edx, eax
++0x143b   a7                   cmpsd   
++0x143c   44                   inc    esp
++0x143d   56                   push   esi
++0x143e   19 2c b8             sbb    dword ptr [eax+edi*4], ebp
++0x1441   c9                   leave  
++0x1442   d9 82 9d 43 57 a5    fld    dword ptr [edx+0xa557439d]
++0x1448   8b 8f 89 63 2c 58    mov    ecx, dword ptr [edi+0x582c6389]
++0x144e   a9 bd ae d7 22       test   eax, 0x22d7aebd
++0x1453   78 d4                js     0x0fffffd6
++0x1455   c1 ba ec 2c 33 4f df sar    dword ptr [edx+0x4f332cec], 0xdf
++0x145c   30 e1                xor    cl, ah
++0x145e   a2 45 a8 31 9c       mov    byte ptr [0x9c31a845], al
++0x1463   4c                   dec    esp
++0x1464   42                   inc    edx
++0x1465   01 a4 33 f4 64 ba ff add    dword ptr [ebx+esi-0x00459b0c], esp
++0x146c   fd                   std    
++0x146d   76 8d                jbe    0x0fffff8f
++0x146f   3b b9 17 37 84 55    cmp    edi, dword ptr [ecx+0x55843717]
++0x1475   93                   xchg   ebx, eax
++0x1476   c8 ab d9 fc          enter  0xd9ab, 0xfc
++0x147a   d2 55 3f             rcl    byte ptr [ebp+0x3f], cl
++0x147d   11 82 b3 dd 4f 83    adc    dword ptr [edx+0x834fddb3], eax
++0x1483   2c 3a                sub    al, 0x3a
++0x1485   a0 d2 70 bb c6       mov    al, byte ptr [0xc6bb70d2]
++0x148a   21 42 ce             and    dword ptr [edx-0x32], eax
++0x148d   75 41                jnz    0x10000043
++0x148f   5c                   pop    esp
++0x1490   08 69 9f             or     byte ptr [ecx-0x61], ch
++0x1493   80 ef e4             sub    bh, 0xe4
++0x1496   e6 e3                out    0xe3, al
++0x1498   88 e4                mov    ah, ah
++0x149a   3a cb                cmp    cl, bl
++0x149c   95                   xchg   ebp, eax
++0x149d   78 43                js     0x10000045
++0x149f   05 38 be 7d 64       add    eax, 0x647dbe38
++0x14a4   9e                   sahf   
++0x14a5   93                   xchg   ebx, eax
++0x14a6   e1 35                loope  0x10000037
++0x14a8   9d                   popfd   
++0x14a9   a0 05 6e 29 7f       mov    al, byte ptr [0x7f296e05]
++0x14ae   3d 3b 3c 3b 28       cmp    eax, 0x283b3c3b
++0x14b3   2c b8                sub    al, 0xb8
++0x14b5   44                   inc    esp
++0x14b6   9b                   fwait  
++0x14b7   93                   xchg   ebx, eax
++0x14b8   21 31                and    dword ptr [ecx], esi
++0x14ba   b9 a3 bf b9 26       mov    ecx, 0x26b9bfa3
++0x14bf   b4 0e                mov    ah, 0x0e
++0x14c1   c4 42 e1             les    eax, [edx-0x1f]
++0x14c4   7e 55                jle    0x10000057
++0x14c6   a4                   movsb   
++0x14c7   f0 1c 1a...??        <INVALID>
++0x14c8   1c 1a                sbb    al, 0x1a
++0x14ca   7c e1                jl     0x0fffffe3
++0x14cc   2c 21                sub    al, 0x21
++0x14ce   84 13                test   byte ptr [ebx], dl
++0x14d0   03 c6                add    eax, esi
++0x14d2   6d                   insd    
++0x14d3   45                   inc    ebp
++0x14d4   f5                   cmc    
++0x14d5   35 66 af f5 58       xor    eax, 0x58f5af66
++0x14da   c8 a7 00 a9          enter  0x00a7, 0xa9
++0x14de   83 0a ba             or     dword ptr [edx], 0xba
++0x14e1   46                   inc    esi
++0x14e2   bb ca f0 7f e8       mov    ebx, 0xe87ff0ca
++0x14e7   ea 5e 4d 69 3c 2f d2 jmp    0xd22f:0x3c694d5e
++0x14ee   47                   inc    edi
++0x14ef   f1                   int1   
++0x14f0   e5 77                in     eax, 0x77
++0x14f2   ee                   out    al, dx
++0x14f3   24 f7                and    al, 0xf7
++0x14f5   ab                   stosd   
++0x14f6   20 a0 ac ba 0f a0    and    byte ptr [eax+0xa00fbaac], ah
++0x14fc   56                   push   esi
++0x14fd   75 bd                jnz    0x0fffffbf
++0x14ff   74 3f                jz     0x10000041
++0x1501   ce                   into   
++0x1502   64 46                inc    esi
++0x1504   a0 76 84 97 f9       mov    al, byte ptr [0xf9978476]
++0x1509   47                   inc    edi
++0x150a   c2 ca 77             ret    0x77ca
++0x150d   15 9d 32 7e f1       adc    eax, 0xf17e329d
++0x1512   b5 ad                mov    ch, 0xad
++0x1514   f2 4e                dec    esi
++0x1516   de 21                fisub  word ptr [ecx]
++0x1518   b7 95                mov    bh, 0x95
++0x151a   af                   scasd   
++0x151b   bd fa 74 29 e4       mov    ebp, 0xe42974fa
++0x1520   2e c1 70 7d b5       shl    dword ptr [cs:eax+0x7d], 0xb5
++0x1525   62 12                bound  edx, qword ptr [edx]
++0x1527   6f                   outsd   
++0x1528   d8 ed                fsubr  st0, st5
++0x152a   dc 51 a9             fcom   qword ptr [ecx-0x57]
++0x152d   3e 98                cwde   
++0x152f   62 52 7f             bound  edx, qword ptr [edx+0x7f]
++0x1532   50                   push   eax
++0x1533   4f                   dec    edi
++0x1534   99                   cdq    
++0x1535   36 07                pop    es
++0x1537   fe 04 a4             inc    byte ptr [esp]
++0x153a   08 ec                or     ah, ch
++0x153c   1f                   pop    ds
++0x153d   8d 22                lea    esp, [edx]
++0x153f   be 7f 71 fa 22       mov    esi, 0x22fa717f
++0x1544   f0 b6 1d...??        <INVALID>
++0x1545   b6 1d                mov    dh, 0x1d
++0x1547   72 81                jb     0x0fffff83
++0x1549   2d 1d 1f 72 e4       sub    eax, 0xe4721f1d
++0x154e   3e ee                out    al, dx
++0x1550   9b                   fwait  
++0x1551   12 81 2e 9d 15 7e    adc    al, byte ptr [ecx+0x7e159d2e]
++0x1557   86 48 59             xchg   byte ptr [eax+0x59], cl
++0x155a   4d                   dec    ebp
++0x155b   ca f8 d5             retf   0xd5f8
++0x155e   f0 88 c4...??        <INVALID>
++0x155f   88 c4                mov    ah, al
++0x1561   9f                   lahf   
++0x1562   a2 ea f5 4b 39       mov    byte ptr [0x394bf5ea], al
++0x1567   a8 45                test   al, 0x45
++0x1569   e6 11                out    0x11, al
++0x156b   da 3b                fidivr dword ptr [ebx]
++0x156d   39 e4                cmp    esp, esp
++0x156f   34 01                xor    al, 0x01
++0x1571   9f                   lahf   
++0x1572   58                   pop    eax
++0x1573   1b 29                sbb    ebp, dword ptr [ecx]
++0x1575   5c                   pop    esp
++0x1576   eb 29                jmp    0x1000002b
++0x1578   11 b8 a9 cc 82 ae    adc    dword ptr [eax+0xae82cca9], edi
++0x157e   bb 5a 77 81 40       mov    ebx, 0x4081775a
++0x1583   36 51                push   ecx
++0x1585   22 b7 1c 0a 7a 16    and    dh, byte ptr [edi+0x167a0a1c]
++0x158b   d5 60                aad    0x60
++0x158d   0c 84                or     al, 0x84
++0x158f   3d ad 82 a7 9c       cmp    eax, 0x9ca782ad
++0x1594   87 4a 02             xchg   dword ptr [edx+0x02], ecx
++0x1597   00 a9 d7 6e a0 0d    add    byte ptr [ecx+0x0da06ed7], ch
++0x159d   08 91 cb 52 5f 15    or     byte ptr [ecx+0x155f52cb], dl
++0x15a3   27                   daa    
++0x15a4   2d e3 79 5c 5c       sub    eax, 0x5c5c79e3
++0x15a9   53                   push   ebx
++0x15aa   63 2c 8f             arpl   word ptr [edi+ecx*4], bp
++0x15ad   79 a9                jns    0x0fffffab
++0x15af   49                   dec    ecx
++0x15b0   b8 a2 84 a9 a9       mov    eax, 0xa9a984a2
++0x15b5   a8 d2                test   al, 0xd2
++0x15b7   11 ea                adc    edx, ebp
++0x15b9   8a 3e                mov    bh, byte ptr [esi]
++0x15bb   30 c2                xor    dl, al
++0x15bd   86 32                xchg   byte ptr [edx], dh
++0x15bf   66 b4 c3             mov    ah, 0xc3
++0x15c2   9b                   fwait  
++0x15c3   00 4b fc             add    byte ptr [ebx-0x04], cl
++0x15c6   43                   inc    ebx
++0x15c7   f2 cf                iretd   
++0x15c9   32 30                xor    dh, byte ptr [eax]
++0x15cb   6e                   outsb   
++0x15cc   8c 2b                mov    dword ptr [ebx], gs
++0x15ce   c9                   leave  
++0x15cf   01 da                add    edx, ebx
++0x15d1   41                   inc    ecx
++0x15d2   52                   push   edx
++0x15d3   c3                   ret    
++0x15d4   2c 73                sub    al, 0x73
++0x15d6   74 ec                jz     0x0fffffee
++0x15d8   74 07                jz     0x10000009
++0x15da   f7 a9 07 36 42 c6    imul   edx, eax, dword ptr [ecx+0xc6423607]
++0x15e0   a5                   movsd   
++0x15e1   5f                   pop    edi
++0x15e2   fd                   std    
++0x15e3   15 cb 70 a3 a9       adc    eax, 0xa9a370cb
++0x15e8   dc 3f                fdivr  qword ptr [edi]
++0x15ea   80 e4 e2             and    ah, 0xe2
++0x15ed   ab                   stosd   
++0x15ee   4f                   dec    edi
++0x15ef   d4 42                aam    0x42
++0x15f1   7f e9                jnle   0x0fffffeb
++0x15f3   a4                   movsb   
++0x15f4   85 e7                test   edi, esp
++0x15f6   44                   inc    esp
++0x15f7   10 df                adc    bh, bl
++0x15f9   67 e9 7c 12 9e 3f    jmp    0x4f9e1282
++0x15ff   cc                   int3   
++0x1600   7a 17                jp     0x10000019
++0x1602   e5 ad                in     eax, 0xad
++0x1604   d1 9f 0a d6 c0 7c    rcr    dword ptr [edi+0x7cc0d60a], 0x01
++0x160a   c7 dc f5 13 d5 88...??  <INVALID>
++0x160b   dc f5                fdivr  st5, st0
++0x160d   13 d5                adc    edx, ebp
++0x160f   88 d0                mov    al, dl
++0x1611   2a bd 04 cb 86 93    sub    bh, byte ptr [ebp+0x9386cb04]
++0x1617   19 79 06             sbb    dword ptr [ecx+0x06], edi
++0x161a   14 e4                adc    al, 0xe4
++0x161c   07                   pop    es
++0x161d   b3 1e                mov    bl, 0x1e
++0x161f   c1 58 a5 13          rcr    dword ptr [eax-0x5b], 0x13
++0x1623   06                   push   es
++0x1624   25 65 fe 7f f3       and    eax, 0xf37ffe65
++0x1629   57                   push   edi
++0x162a   92                   xchg   edx, eax
++0x162b   25 38 9b 99 23       and    eax, 0x23999b38
++0x1630   c0 15 ca 1e 18 01 2f rcl    byte ptr [0x01181eca], 0x2f
++0x1637   a6                   cmpsb   
++0x1638   9a fd 22 70 ae 08 be call   0xbe08:0xae7022fd
++0x163f   d6                   salc   
++0x1640   77 6a                jnbe   0x1000006c
++0x1642   da 33                fidiv  dword ptr [ebx]
++0x1644   75 cd                jnz    0x0fffffcf
++0x1646   71 ca                jno    0x0fffffcc
++0x1648   f4                   hlt    
++0x1649   f8                   clc    
++0x164a   16                   push   ss
++0x164b   80 d4 af             adc    ah, 0xaf
++0x164e   61                   popad   
++0x164f   4f                   dec    edi
++0x1650   a2 af dc d1 6e       mov    byte ptr [0x6ed1dcaf], al
++0x1655   a0 59 bb ea 56       mov    al, byte ptr [0x56eabb59]
++0x165a   40                   inc    eax
++0x165b   3b dd                cmp    ebx, ebp
++0x165d   d9 b2 44 85 71 91    fnstenv [edx+0x91718544]
++0x1663   1d 29 9a 7a c4       sbb    eax, 0xc47a9a29
++0x1668   8d 2a                lea    ebp, [edx]
++0x166a   9d                   popfd   
++0x166b   71 b9                jno    0x0fffffbb
++0x166d   e4 9e                in     al, 0x9e
++0x166f   56                   push   esi
++0x1670   67 f4                hlt    
++0x1672   27                   daa    
++0x1673   a2 e7 ff 5e 90       mov    byte ptr [0x905effe7], al
++0x1678   93                   xchg   ebx, eax
++0x1679   c8 fc 1e ff          enter  0x1efc, 0xff
++0x167d   d4 ba                aam    0xba
++0x167f   e0 9e                loopne 0x0fffffa0
++0x1681   e9 a5 d6 68 48       jmp    0x5868d6aa
++0x1686   85 ec                test   esp, ebp
++0x1688   67 5d                pop    ebp
++0x168a   3a 67 30             cmp    ah, byte ptr [edi+0x30]
++0x168d   e6 e6                out    0xe6, al
++0x168f   2b 06                sub    eax, dword ptr [esi]
++0x1691   d6                   salc   
++0x1692   7d 1f                jnl    0x10000021
++0x1694   49                   dec    ecx
++0x1695   f8                   clc    
++0x1696   3e 80 54 e2 e2 3e    adc    byte ptr [ds:edx-0x1e], 0x3e
++0x169c   f3 dd 1f             fstp   qword ptr [edi]
++0x169f   51                   push   ecx
++0x16a0   96                   xchg   esi, eax
++0x16a1   a8 96                test   al, 0x96
++0x16a3   f9                   stc    
++0x16a4   74 bb                jz     0x0fffffbd
++0x16a6   be 04 b6 cf 49       mov    esi, 0x49cfb604
++0x16ab   0d 5e 44 a9 6e       or     eax, 0x6ea9445e
++0x16b0   6a 77                push   0x77
++0x16b2   e2 ae                loop   0x0fffffb0
++0x16b4   3f                   aas    
++0x16b5   97                   xchg   edi, eax
++0x16b6   c6 74 fe 1c f0...??  <INVALID>
++0x16b7   74 fe                jz     0x10000000
++0x16b9   1c f0                sbb    al, 0xf0
++0x16bb   2f                   das    
++0x16bc   3d a8 32 56 ce       cmp    eax, 0xce5632a8
++0x16c1   30 12                xor    byte ptr [edx], dl
++0x16c3   2e 95                xchg   ebp, eax
++0x16c5   34 70                xor    al, 0x70
++0x16c7   f6 23                mul    ax, byte ptr [ebx]
++0x16c9   c1 75 13 62          shl    dword ptr [ebp+0x13], 0x62
++0x16cd   ae                   scasb   
++0x16ce   c5 77 b8             lds    esi, [edi-0x48]
++0x16d1   6f                   outsd   
++0x16d2   92                   xchg   edx, eax
++0x16d3   53                   push   ebx
++0x16d4   e6 d9                out    0xd9, al
++0x16d6   66 32 1b             xor    bl, byte ptr [ebx]
++0x16d9   36 f1                int1   
++0x16db   69 ae 18 81 41 06 83 imul   ebp, dword ptr [esi+0x06418118], 0xbb5f1483
+     14 5f bb
++0x16e5   2b 15 c7 70 c2 3e    sub    edx, dword ptr [0x3ec270c7]
++0x16eb   34 0c                xor    al, 0x0c
++0x16ed   cf                   iretd   
++0x16ee   9f                   lahf   
++0x16ef   61                   popad   
++0x16f0   ca 74 04             retf   0x0474
++0x16f3   70 95                jo     0x0fffff97
++0x16f5   3a 95 b5 5e 9b 43    cmp    dl, byte ptr [ebp+0x439b5eb5]
++0x16fb   f1                   int1   
++0x16fc   c7 5f b6 82 8a b1 18...??  <INVALID>
++0x16fd   5f                   pop    edi
++0x16fe   b6 82                mov    dh, 0x82
++0x1700   8a b1 18 10 38 bd    mov    dh, byte ptr [ecx+0xbd381018]
++0x1706   90                   nop    
++0x1707   24 da                and    al, 0xda
++0x1709   77 05                jnbe   0x10000007
++0x170b   a0 a2 37 4e 07       mov    al, byte ptr [0x074e37a2]
++0x1710   4c                   dec    esp
++0x1711   c8 8d 8c 2d          enter  0x8c8d, 0x2d
++0x1715   87 ad 2c 9e f7 4e    xchg   dword ptr [ebp+0x4ef79e2c], ebp
++0x171b   5e                   pop    esi
++0x171c   d7                   xlatb   
++0x171d   92                   xchg   edx, eax
++0x171e   9f                   lahf   
++0x171f   a7                   cmpsd   
++0x1720   33 7c 9f 35          xor    edi, dword ptr [edi+ebx*4+0x35]
++0x1724   b4 ef                mov    ah, 0xef
++0x1726   86 67 8c             xchg   byte ptr [edi-0x74], ah
++0x1729   5c                   pop    esp
++0x172a   da a0 a0 69 7e 5f    fisub  dword ptr [eax+0x5f7e69a0]
++0x1730   10 6d 56             adc    byte ptr [ebp+0x56], ch
++0x1733   71 47                jno    0x10000049
++0x1735   bc 95 d0 48 e1       mov    esp, 0xe148d095
++0x173a   c2 bf 5a             ret    0x5abf
++0x173d   80 88 e2 b9 44 83 a1 or     byte ptr [eax+0x8344b9e2], 0xa1
++0x1744   ba a1 24 30 79       mov    edx, 0x793024a1
++0x1749   ee                   out    al, dx
++0x174a   05 64 04 76 38       add    eax, 0x38760464
++0x174f   c2 50 f7             ret    0xf750
++0x1752   8b 85 f8 4e 33 22    mov    eax, dword ptr [ebp+0x22334ef8]
++0x1758   37                   aaa    
++0x1759   1b 92 64 61 a3 62    sbb    edx, dword ptr [edx+0x62a36164]
++0x175f   5e                   pop    esi
++0x1760   ca ab 78             retf   0x78ab
++0x1763   56                   push   esi
++0x1764   aa                   stosb   
++0x1765   09 2f                or     dword ptr [edi], ebp
++0x1767   0a 6c f6 67          or     ch, byte ptr [esi+esi*8+0x67]
++0x176b   fd                   std    
++0x176c   73 c0                jnb    0x0fffffc2
++0x176e   f7 47 8a 86 86 14 4d test   dword ptr [edi-0x76], 0x4d148686
++0x1775   fb                   sti    
++0x1776   5a                   pop    edx
++0x1777   64 ac                lodsb   fs
++0x1779   f5                   cmc    
++0x177a   01 0d 4f 6f ad 09    add    dword ptr [0x09ad6f4f], ecx
++0x1780   f4                   hlt    
++0x1781   13 c8                adc    ecx, eax
++0x1783   8a dd                mov    bl, ch
++0x1785   ea 6b 5c f3 3b 27 e2 jmp    0xe227:0x3bf35c6b
++0x178c   d8 f1                fdiv   st0, st1
++0x178e   3e 0d 7b 6a 3b 72    or     eax, 0x723b6a7b
++0x1794   5d                   pop    ebp
++0x1795   41                   inc    ecx
++0x1796   e8 fd 51 e5 61       call   0x71e55202
++0x179b   cc                   int3   
++0x179c   1b 16                sbb    edx, dword ptr [esi]
++0x179e   9d                   popfd   
++0x179f   a4                   movsb   
++0x17a0   d8 12                fcom   dword ptr [edx]
++0x17a2   d3 9d ab 55 54 e4    rcr    dword ptr [ebp+0xe45455ab], cl
++0x17a8   f7 cd 09 5a e6 9b    test   ebp, 0x9be65a09
++0x17ae   11 02                adc    dword ptr [edx], eax
++0x17b0   ee                   out    al, dx
++0x17b1   de 55 36             ficom  word ptr [ebp+0x36]
++0x17b4   f7 ed                imul   edx, eax, ebp
++0x17b6   61                   popad   
++0x17b7   be 12 91 2b 42       mov    esi, 0x422b9112
++0x17bc   09 c2                or     edx, eax
++0x17be   86 9e 0b 09 71 55    xchg   byte ptr [esi+0x5571090b], bl
++0x17c4   06                   push   es
++0x17c5   6e                   outsb   
++0x17c6   7f 2a                jnle   0x1000002c
++0x17c8   a7                   cmpsd   
++0x17c9   bc 93 0b b4 3f       mov    esp, 0x3fb40b93
++0x17ce   a7                   cmpsd   
++0x17cf   94                   xchg   esp, eax
++0x17d0   a4                   movsb   
++0x17d1   bc 64 04 c3 da       mov    esp, 0xdac30464
++0x17d6   f3 43                inc    ebx
++0x17d8   9c                   pushfd  
++0x17d9   0c 20                or     al, 0x20
++0x17db   be b2 97 8c 2a       mov    esi, 0x2a8c97b2
++0x17e0   69 d4 6e a7 0c bb    imul   edx, esp, 0xbb0ca76e
++0x17e6   78 ce                js     0x0fffffd0
++0x17e8   74 01                jz     0x10000003
++0x17ea   4c                   dec    esp
++0x17eb   ff 69 f6             jmp    
++0x17ee   a1 ef 7e 39 a1       mov    eax, dword ptr [0xa1397eef]
++0x17f3   76 76                jbe    0x10000078
++0x17f5   9f                   lahf   
++0x17f6   9d                   popfd   
++0x17f7   a3 c8 f5 35 5b       mov    dword ptr [0x5b35f5c8], eax
++0x17fc   a8 e9                test   al, 0xe9
++0x17fe   95                   xchg   ebp, eax
++0x17ff   a0 a9 3e 7b bd       mov    al, byte ptr [0xbd7b3ea9]
++0x1804   c8 a3 77 1f          enter  0x77a3, 0x1f
++0x1808   84 8d 79 25 ab 85    test   byte ptr [ebp+0x85ab2579], cl
++0x180e   0b 03                or     eax, dword ptr [ebx]
++0x1810   ad                   lodsd   
++0x1811   29 91 eb 80 99 90    sub    dword ptr [ecx+0x909980eb], edx
++0x1817   6b 25 8c fa 5c 93 c4 imul   esp, dword ptr [0x935cfa8c], 0xc4
++0x181e   c3                   ret    
++0x181f   6c                   insb    
++0x1820   32 2f                xor    ch, byte ptr [edi]
++0x1822   ca d4 d0             retf   0xd0d4
++0x1825   90                   nop    
++0x1826   be 8d e2 6d 85       mov    esi, 0x856de28d
++0x182b   9e                   sahf   
++0x182c   63 a6 ab f2 0f a1    arpl   word ptr [esi+0xa10ff2ab], sp
++0x1832   17                   pop    ss
++0x1833   b1 00                mov    cl, 0x00
++0x1835   14 6c                adc    al, 0x6c
++0x1837   cb                   retf   
++0x1838   3f                   aas    
++0x1839   0e                   push   cs
++0x183a   40                   inc    eax
++0x183b   43                   inc    ebx
++0x183c   6a e6                push   0xe6
++0x183e   4d                   dec    ebp
++0x183f   72 fc                jb     0x0ffffffe
++0x1841   eb cb                jmp    0x0fffffcd
++0x1843   c0 f8 ef             sar    al, 0xef
++0x1846   e0 9e                loopne 0x0fffffa0
++0x1848   0b c5                or     eax, ebp
++0x184a   cc                   int3   
++0x184b   fe 4b d3             dec    byte ptr [ebx-0x2d]
++0x184e   aa                   stosb   
++0x184f   9b                   fwait  
++0x1850   28 97 22 09 ae f0    sub    byte ptr [edi+0xf0ae0922], dl
++0x1856   35 e6 65 a7 04       xor    eax, 0x04a765e6
++0x185b   6d                   insd    
++0x185c   8f c0                pop    eax
++0x185e   95                   xchg   ebp, eax
++0x185f   9c                   pushfd  
++0x1860   5d                   pop    ebp
++0x1861   25 f2 81 c3 0d       and    eax, 0x0dc381f2
++0x1866   c1 f4 76             shl    esp, 0x76
++0x1869   50                   push   eax
++0x186a   42                   inc    edx
++0x186b   08 0d 4e 4f d9 a8    or     byte ptr [0xa8d94f4e], cl
++0x1871   ae                   scasb   
++0x1872   3c 68                cmp    al, 0x68
++0x1874   5c                   pop    esp
++0x1875   af                   scasd   
++0x1876   00 35 eb 76 76 73    add    byte ptr [0x737676eb], dh
++0x187c   10 9d 02 74 4a 0f    adc    byte ptr [ebp+0x0f4a7402], bl
++0x1882   b0 de                mov    al, 0xde
++0x1884   e7 4d                out    0x4d, eax
++0x1886   8b 9a fe 12 5c 0a    mov    ebx, dword ptr [edx+0x0a5c12fe]
++0x188c   73 6b                jnb    0x1000006d
++0x188e   9e                   sahf   
++0x188f   d7                   xlatb   
++0x1890   ae                   scasb   
++0x1891   f4                   hlt    
++0x1892   b9 74 4b 7d 45       mov    ecx, 0x457d4b74
++0x1897   ee                   out    al, dx
++0x1898   79 51                jns    0x10000053
++0x189a   f5                   cmc    
++0x189b   63 3f                arpl   word ptr [edi], di
++0x189d   1d c9 0e 25 20       sbb    eax, 0x20250ec9
++0x18a2   91                   xchg   ecx, eax
++0x18a3   bf c7 76 b1 43       mov    edi, 0x43b176c7
++0x18a8   cf                   iretd   
++0x18a9   60                   pushad  
++0x18aa   d5 e6                aad    0xe6
++0x18ac   9c                   pushfd  
++0x18ad   fd                   std    
++0x18ae   5c                   pop    esp
++0x18af   34 91                xor    al, 0x91
++0x18b1   53                   push   ebx
++0x18b2   d2 28                shr    byte ptr [eax], cl
++0x18b4   a3 75 bb 5b f3       mov    dword ptr [0xf35bbb75], eax
++0x18b9   75 ec                jnz    0x0fffffee
++0x18bb   0a 47 33             or     al, byte ptr [edi+0x33]
++0x18be   e7 27                out    0x27, eax
++0x18c0   d3 c0                rol    eax, cl
++0x18c2   84 53 3f             test   byte ptr [ebx+0x3f], dl
++0x18c5   57                   push   edi
++0x18c6   25 bb ba b2 32       and    eax, 0x32b2babb
++0x18cb   56                   push   esi
++0x18cc   e7 5e                out    0x5e, eax
++0x18ce   70 62                jo     0x10000064
++0x18d0   6f                   outsd   
++0x18d1   f9                   stc    
++0x18d2   56                   push   esi
++0x18d3   e2 0d                loop   0x1000000f
++0x18d5   29 82 02 3d d7 ed    sub    dword ptr [edx+0xedd73d02], eax
++0x18db   21 b8 b9 1e 82 4f    and    dword ptr [eax+0x4f821eb9], edi
++0x18e1   92                   xchg   edx, eax
++0x18e2   2f                   das    
++0x18e3   7a 0d                jp     0x1000000f
++0x18e5   62 75 28             bound  esi, qword ptr [ebp+0x28]
++0x18e8   c0 82 36 e5 53 94 4f rol    byte ptr [edx+0x9453e536], 0x4f
++0x18ef   73 f2                jnb    0x0ffffff4
++0x18f1   93                   xchg   ebx, eax
++0x18f2   5f                   pop    edi
++0x18f3   e7 95                out    0x95, eax
++0x18f5   d4 90                aam    0x90
++0x18f7   c2 6e 47             ret    0x476e
++0x18fa   cf                   iretd   
++0x18fb   d3 ba 23 9f 48 87    sar    dword ptr [edx+0x87489f23], cl
++0x1901   82 84 b8 c2 a2 c8 5f add    byte ptr [eax+edi*4+0x5fc8a2c2], 0x5f
+     5f
++0x1909   ee                   out    al, dx
++0x190a   71 a8                jno    0x0fffffaa
++0x190c   24 a1                and    al, 0xa1
++0x190e   92                   xchg   edx, eax
++0x190f   d5 93                aad    0x93
++0x1911   9b                   fwait  
++0x1912   74 66                jz     0x10000068
++0x1914   db 81 71 93 00 0c    fild   dword ptr [ecx+0x0c009371]
++0x191a   2b 88 70 f2 79 79    sub    ecx, dword ptr [eax+0x7979f270]
++0x1920   89 cf                mov    edi, ecx
++0x1922   db b4 8c 88 75 09 cc...??  <INVALID>
++0x1923   b4 8c                mov    ah, 0x8c
++0x1925   88 75 09             mov    byte ptr [ebp+0x09], dh
++0x1928   cc                   int3   
++0x1929   05 fb b9 7a 87       add    eax, 0x877ab9fb
++0x192e   cf                   iretd   
++0x192f   1d df 45 93 4d       sbb    eax, 0x4d9345df
++0x1934   e4 b7                in     al, 0xb7
++0x1936   f8                   clc    
++0x1937   50                   push   eax
++0x1938   33 0c 25 3a 07 10 04 xor    ecx, dword ptr [0x0410073a]
++0x193f   d3 92 a6 ab ce ac    rcl    dword ptr [edx+0xacceaba6], cl
++0x1945   5e                   pop    esi
++0x1946   7b 5c                jnp    0x1000005e
++0x1948   68 b8 2a 47 80       push   0x80472ab8
++0x194d   10 b1 3e b0 43 c1    adc    byte ptr [ecx+0xc143b03e], dh
++0x1953   58                   pop    eax
++0x1954   54                   push   esp
++0x1955   68 23 03 34 6d       push   0x6d340323
++0x195a   ee                   out    al, dx
++0x195b   bc 32 6e c3 01       mov    esp, 0x01c36e32
++0x1960   33 6f 0e             xor    ebp, dword ptr [edi+0x0e]
++0x1963   be c7 f9 42 4c       mov    esi, 0x4c42f9c7
++0x1968   35 60 09 34 d6       xor    eax, 0xd6340960
++0x196d   0e                   push   cs
++0x196e   5c                   pop    esp
++0x196f   01 0c cb             add    dword ptr [ebx+ecx*8], ecx
++0x1972   3d 1a 19 b1 32       cmp    eax, 0x32b1191a
++0x1977   4e                   dec    esi
++0x1978   1a 90 2c a1 37 45    sbb    dl, byte ptr [eax+0x4537a12c]
++0x197e   98                   cwde   
++0x197f   b9 7a 67 38 0c       mov    ecx, 0x0c38677a
++0x1984   38 b0 a5 b9 fc 61    cmp    byte ptr [eax+0x61fcb9a5], dh
++0x198a   19 9e 68 74 23 a3    sbb    dword ptr [esi+0xa3237468], ebx
++0x1990   3b 42 4e             cmp    eax, dword ptr [edx+0x4e]
++0x1993   74 5b                jz     0x1000005d
++0x1995   e2 c5                loop   0x0fffffc7
++0x1997   3a 3d 21 5d 0f 48    cmp    bh, byte ptr [0x480f5d21]
++0x199d   02 4c a9 e3          add    cl, byte ptr [ecx+ebp*4-0x1d]
++0x19a1   17                   pop    ss
++0x19a2   88 c9                mov    cl, cl
++0x19a4   89 8c 1c 35 ae 79 d6 mov    dword ptr [esp+ebx+0xd679ae35], ecx
++0x19ab   ed                   in     eax, dx
++0x19ac   30 6b 2f             xor    byte ptr [ebx+0x2f], ch
++0x19af   c5 fe...??           <INVALID>
++0x19b0   fe 52 84...??        <INVALID>
++0x19b1   52                   push   edx
++0x19b2   84 be d2 80 4e a2    test   byte ptr [esi+0xa24e80d2], bh
++0x19b8   6f                   outsd   
++0x19b9   aa                   stosb   
++0x19ba   64 00 14 20          add    byte ptr [fs:eax], dl
++0x19be   aa                   stosb   
++0x19bf   71 e0                jno    0x0fffffe2
++0x19c1   63 4e 57             arpl   word ptr [esi+0x57], cx
++0x19c4   3f                   aas    
++0x19c5   b0 d4                mov    al, 0xd4
++0x19c7   2b f8                sub    edi, eax
++0x19c9   ca 58 3e             retf   0x3e58
++0x19cc   39 56 e2             cmp    dword ptr [esi-0x1e], edx
++0x19cf   fb                   sti    
++0x19d0   6e                   outsb   
++0x19d1   94                   xchg   esp, eax
++0x19d2   e1 79                loope  0x1000007b
++0x19d4   7a 65                jp     0x10000067
++0x19d6   c4 1c 0f             les    ebx, [edi+ecx]
++0x19d9   27                   daa    
++0x19da   8f 48 ca...??        <INVALID>
++0x19db   48                   dec    eax
++0x19dc   ca 6c bf             retf   0xbf6c
++0x19df   88 c6                mov    dh, al
++0x19e1   8b 9d 6f 63 fd 30    mov    ebx, dword ptr [ebp+0x30fd636f]
++0x19e7   5d                   pop    ebp
++0x19e8   32 f9                xor    bh, cl
++0x19ea   88 6a 93             mov    byte ptr [edx-0x6d], ch
++0x19ed   91                   xchg   ecx, eax
++0x19ee   fb                   sti    
++0x19ef   d3 79 40             sar    dword ptr [ecx+0x40], cl
++0x19f2   bb 97 9f 7e 1a       mov    ebx, 0x1a7e9f97
++0x19f7   67 fb                sti    
++0x19f9   00 06                add    byte ptr [esi], al
++0x19fb   5d                   pop    ebp
++0x19fc   da c2                fcmovb st0, st2
++0x19fe   d1 72 cc             shl    dword ptr [edx-0x34], 0x01
++0x1a01   52                   push   edx
++0x1a02   64 57                push   edi
++0x1a04   56                   push   esi
++0x1a05   56                   push   esi
++0x1a06   ff a4 32 68 0a 11 78 jmp    
++0x1a0d   19 c3                sbb    ebx, eax
++0x1a0f   d0 4f c8             ror    byte ptr [edi-0x38], 0x01
++0x1a12   d3 23                shl    dword ptr [ebx], cl
++0x1a14   ff 9c 51 f7 08 0c be call   
++0x1a1b   80 3c 9d c8 f9 79 92 cmp    byte ptr [ebx*4+0x9279f9c8], 0x49
+     49
++0x1a23   4e                   dec    esi
++0x1a24   70 29                jo     0x1000002b
++0x1a26   ac                   lodsb   
++0x1a27   0c ca                or     al, 0xca
++0x1a29   ea aa 7f 7d 9c 62 84 jmp    0x8462:0x9c7d7faa
++0x1a30   4e                   dec    esi
++0x1a31   43                   inc    ebx
++0x1a32   41                   inc    ecx
++0x1a33   f7 5c eb 04          neg    dword ptr [ebx+ebp*8+0x04]
++0x1a37   ab                   stosd   
++0x1a38   3e bc 82 5c 47 40    mov    esp, 0x40475c82
++0x1a3e   6d                   insd    
++0x1a3f   08 84 d0 5c 26 9a d4 or     byte ptr [eax+edx*8+0xd49a265c], al
++0x1a46   62 07                bound  eax, qword ptr [edi]
++0x1a48   91                   xchg   ecx, eax
++0x1a49   5e                   pop    esi
++0x1a4a   90                   nop    
++0x1a4b   5c                   pop    esp
++0x1a4c   38 a6 95 60 ea c7    cmp    byte ptr [esi+0xc7ea6095], ah
++0x1a52   da c1                fcmovb st0, st1
++0x1a54   91                   xchg   ecx, eax
++0x1a55   59                   pop    ecx
++0x1a56   9c                   pushfd  
++0x1a57   66 f5                cmc    
++0x1a59   e8 ef 54 e6 8c       call   0x9ce654f4
++0x1a5e   46                   inc    esi
++0x1a5f   49                   dec    ecx
++0x1a60   b3 c9                mov    bl, 0xc9
++0x1a62   27                   daa    
++0x1a63   70 64                jo     0x10000066
++0x1a65   4b                   dec    ebx
++0x1a66   db f5                fcomi  st0, st5
++0x1a68   f2 6e                outsb   
++0x1a6a   da ee...??           <INVALID>
++0x1a6b   ee                   out    al, dx
++0x1a6c   0a 78 39             or     bh, byte ptr [eax+0x39]
++0x1a6f   2a 20                sub    ah, byte ptr [eax]
++0x1a71   b2 b3                mov    dl, 0xb3
++0x1a73   2c 98                sub    al, 0x98
++0x1a75   52                   push   edx
++0x1a76   88 60 0e             mov    byte ptr [eax+0x0e], ah
++0x1a79   d2 a7 6c 86 b5 6e    shl    byte ptr [edi+0x6eb5866c], cl
++0x1a7f   45                   inc    ebp
++0x1a80   78 29                js     0x1000002b
++0x1a82   a4                   movsb   
++0x1a83   94                   xchg   esp, eax
++0x1a84   3c 46                cmp    al, 0x46
++0x1a86   a1 a4 d8 68 85       mov    eax, dword ptr [0x8568d8a4]
++0x1a8b   05 de fb b8 4f       add    eax, 0x4fb8fbde
++0x1a90   6d                   insd    
++0x1a91   b0 0a                mov    al, 0x0a
++0x1a93   70 70                jo     0x10000072
++0x1a95   31 e3                xor    ebx, esp
++0x1a97   1f                   pop    ds
++0x1a98   78 ec                js     0x0fffffee
++0x1a9a   90                   nop    
++0x1a9b   9c                   pushfd  
++0x1a9c   75 4b                jnz    0x1000004d
++0x1a9e   56                   push   esi
++0x1a9f   ca 4b 69             retf   0x694b
++0x1aa2   d2 52 aa             rcl    byte ptr [edx-0x56], cl
++0x1aa5   ee                   out    al, dx
++0x1aa6   af                   scasd   
++0x1aa7   c5 7f 0e             lds    edi, [edi+0x0e]
++0x1aaa   9e                   sahf   
++0x1aab   69 90 d1 e1 de 6d ee imul   edx, dword ptr [eax+0x6ddee1d1], 0xbec11bee
+     1b c1 be
++0x1ab5   18 7f 39             sbb    byte ptr [edi+0x39], bh
++0x1ab8   43                   inc    ebx
++0x1ab9   a0 12 8f 4c 98       mov    al, byte ptr [0x984c8f12]
++0x1abe   4e                   dec    esi
++0x1abf   9b                   fwait  
++0x1ac0   29 40 b8             sub    dword ptr [eax-0x48], eax
++0x1ac3   26 a8 3a             test   al, 0x3a
++0x1ac6   01 b3 79 77 d2 98    add    dword ptr [ebx+0x98d27779], esi
++0x1acc   e9 06 f2 e3 f3       jmp    0x03e3f20b
++0x1ad1   ba 22 3c d6 31       mov    edx, 0x31d63c22
++0x1ad6   4a                   dec    edx
++0x1ad7   9a 64 14 17 74 46 bb call   0xbb46:0x74171464
++0x1ade   9d                   popfd   
++0x1adf   98                   cwde   
++0x1ae0   79 c3                jns    0x0fffffc5
++0x1ae2   d8 0f                fmul   dword ptr [edi]
++0x1ae4   89 53 cb             mov    dword ptr [ebx-0x35], edx
++0x1ae7   7e d8                jle    0x0fffffda
++0x1ae9   ad                   lodsd   
++0x1aea   17                   pop    ss
++0x1aeb   cf                   iretd   
++0x1aec   88 aa cd 82 15 78    mov    byte ptr [edx+0x781582cd], ch
++0x1af2   0b 3a                or     edi, dword ptr [edx]
++0x1af4   3e 83 84 0a e1 66 45 add    dword ptr [ds:edx+ecx+0xb24566e1], 0x4e
+     b2 4e
++0x1afd   02 90 65 e5 fc cf    add    dl, byte ptr [eax+0xcffce565]
++0x1b03   8f ff...??           <INVALID>
++0x1b04   ff 4f 73             dec    dword ptr [edi+0x73]
++0x1b07   cc                   int3   
++0x1b08   d4 6b                aam    0x6b
++0x1b0a   1b b2 e6 5d 86 bb    sbb    esi, dword ptr [edx+0xbb865de6]
++0x1b10   10 14 80             adc    byte ptr [eax+eax*4], dl
++0x1b13   de f1                fdivrp st1, st0
++0x1b15   3c 91                cmp    al, 0x91
++0x1b17   8b 51 46             mov    edx, dword ptr [ecx+0x46]
++0x1b1a   1c 68                sbb    al, 0x68
++0x1b1c   6e                   outsb   
++0x1b1d   14 91                adc    al, 0x91
++0x1b1f   b0 09                mov    al, 0x09
++0x1b21   e9 06 1c b2 f0       jmp    0x00b21c0b
++0x1b26   ff 12                call   
++0x1b28   05 bb 8c 44 1b       add    eax, 0x1b448cbb
++0x1b2d   1f                   pop    ds
++0x1b2e   2e b5 79             mov    ch, 0x79
++0x1b31   e7 ca                out    0xca, eax
++0x1b33   c7 c2 67 7a b0 8a    mov    edx, 0x8ab07a67
++0x1b39   d5 51                aad    0x51
++0x1b3b   c1 ef b7             shr    edi, 0xb7
++0x1b3e   31 13                xor    dword ptr [ebx], edx
++0x1b40   5c                   pop    esp
++0x1b41   8e f8...??           <INVALID>
++0x1b42   f8                   clc    
++0x1b43   d3 b1 04 e3 4f 86    shl    dword ptr [ecx+0x864fe304], cl
++0x1b49   21 1d 79 05 54 5e    and    dword ptr [0x5e540579], ebx
++0x1b4f   e7 e2                out    0xe2, eax
++0x1b51   32 76 7a             xor    dh, byte ptr [esi+0x7a]
++0x1b54   04 a3                add    al, 0xa3
++0x1b56   64 9a 44 e6 50 fd 9a call   0x469a:0xfd50e644
+     46
++0x1b5e   2e 86 b9 fd bf 23 e0 xchg   byte ptr [cs:ecx+0xe023bffd], bh
++0x1b65   00 ad db 8c c8 13    add    byte ptr [ebp+0x13c88cdb], ch
++0x1b6b   4e                   dec    esi
++0x1b6c   d9 b1 89 41 b5 96    fnstenv [ecx+0x96b54189]
++0x1b72   1a e1                sbb    ah, cl
++0x1b74   97                   xchg   edi, eax
++0x1b75   10 e9                adc    cl, ch
++0x1b77   e4 61                in     al, 0x61
++0x1b79   e3 c4                jecxz  0x0fffffc6
++0x1b7b   a5                   movsd   
++0x1b7c   bf 96 7b ba 0f       mov    edi, 0x0fba7b96
++0x1b81   95                   xchg   ebp, eax
++0x1b82   4c                   dec    esp
++0x1b83   25 bd aa 8b f9       and    eax, 0xf98baabd
++0x1b88   de 7b 9b             fidivr word ptr [ebx-0x65]
++0x1b8b   10 8b 2a d5 62 af    adc    byte ptr [ebx+0xaf62d52a], cl
++0x1b91   ff b9 78 d3 a8 15...??  <INVALID>
++0x1b92   b9 78 d3 a8 15       mov    ecx, 0x15a8d378
++0x1b97   48                   dec    eax
++0x1b98   fb                   sti    
++0x1b99   4b                   dec    ebx
++0x1b9a   aa                   stosb   
++0x1b9b   0a f5                or     dh, ch
++0x1b9d   d9 25 b9 8a 44 85    fldenv [0x85448ab9]
++0x1ba3   42                   inc    edx
++0x1ba4   ec                   in     al
++0x1ba5   02 bb 89 47 5b 66    add    bh, byte ptr [ebx+0x665b4789]
++0x1bab   88 37                mov    byte ptr [edi], dh
++0x1bad   be 3b 77 1f a2       mov    esi, 0xa21f773b
++0x1bb2   42                   inc    edx
++0x1bb3   1b 43 f5             sbb    eax, dword ptr [ebx-0x0b]
++0x1bb6   37                   aaa    
++0x1bb7   ce                   into   
++0x1bb8   56                   push   esi
++0x1bb9   3d b6 84 b4 68       cmp    eax, 0x68b484b6
++0x1bbe   5e                   pop    esi
++0x1bbf   3c 52                cmp    al, 0x52
++0x1bc1   1a 5b 3a             sbb    bl, byte ptr [ebx+0x3a]
++0x1bc4   bc 63 53 d6 ae       mov    esp, 0xaed65363
++0x1bc9   b8 39 d1 63 5e       mov    eax, 0x5e63d139
++0x1bce   b6 e9                mov    dh, 0xe9
++0x1bd0   6e                   outsb   
++0x1bd1   f2 66 19 08          sbb    word ptr [eax], cx
++0x1bd5   af                   scasd   
++0x1bd6   62 81 95 10 47 57    bound  eax, qword ptr [ecx+0x57471095]
++0x1bdc   84 c1                test   cl, al
++0x1bde   39 c5                cmp    ebp, eax
++0x1be0   48                   dec    eax
++0x1be1   f4                   hlt    
++0x1be2   f2 86 5c 46 ac       xchg   byte ptr [esi+eax*2-0x54], bl
++0x1be7   0d 5e 0c 65 d8       or     eax, 0xd8650c5e
++0x1bec   75 e9                jnz    0x0fffffeb
++0x1bee   38 fb                cmp    bl, bh
++0x1bf0   09 49 d8             or     dword ptr [ecx-0x28], ecx
++0x1bf3   c2 2c d2             ret    0xd22c
++0x1bf6   8f a8 5f ae 87 4d...??  <INVALID>
++0x1bf7   a8 5f                test   al, 0x5f
++0x1bf9   ae                   scasb   
++0x1bfa   87 4d 07             xchg   dword ptr [ebp+0x07], ecx
++0x1bfd   1f                   pop    ds
++0x1bfe   48                   dec    eax
++0x1bff   a4                   movsb   
++0x1c00   c3                   ret    
++0x1c01   7c db                jl     0x0fffffdd
++0x1c03   5a                   pop    edx
++0x1c04   fd                   std    
++0x1c05   bd f4 7b 4c ab       mov    ebp, 0xab4c7bf4
++0x1c0a   4f                   dec    edi
++0x1c0b   60                   pushad  
++0x1c0c   34 7a                xor    al, 0x7a
++0x1c0e   c2 9c f2             ret    0xf29c
++0x1c11   8d ce...??           <INVALID>
++0x1c12   ce                   into   
++0x1c13   29 a6 93 ba 87 66    sub    dword ptr [esi+0x6687ba93], esp
++0x1c19   54                   push   esp
++0x1c1a   b2 bb                mov    dl, 0xbb
++0x1c1c   d9 26                fldenv [esi]
++0x1c1e   1c a1                sbb    al, 0xa1
++0x1c20   db bd 8d d8 97 f5    fstp   tbyte ptr [ebp+0xf597d88d]
++0x1c26   a7                   cmpsd   
++0x1c27   b0 3f                mov    al, 0x3f
++0x1c29   ec                   in     al
++0x1c2a   fa                   cli    
++0x1c2b   94                   xchg   esp, eax
++0x1c2c   bf cb 43 05 b3       mov    edi, 0xb30543cb
++0x1c31   5b                   pop    ebx
++0x1c32   c1 5b 30 67          rcr    dword ptr [ebx+0x30], 0x67
++0x1c36   2a 75 1d             sub    dh, byte ptr [ebp+0x1d]
++0x1c39   5f                   pop    edi
++0x1c3a   84 6b 6e             test   byte ptr [ebx+0x6e], ch
++0x1c3d   44                   inc    esp
++0x1c3e   87 a3 b5 a8 7a 65    xchg   dword ptr [ebx+0x657aa8b5], esp
++0x1c44   d6                   salc   
++0x1c45   80 a1 eb 56 31 70 b5 and    byte ptr [ecx+0x703156eb], 0xb5
++0x1c4c   1a 2d 65 0c 86 16    sbb    ch, byte ptr [0x16860c65]
++0x1c52   39 50 e3             cmp    dword ptr [eax-0x1d], edx
++0x1c55   8e 82 db cc cb 25    mov    es, dword ptr [edx+0x25cbccdb]
++0x1c5b   36 31 a0 d0 cf 97 2c xor    dword ptr [ss:eax+0x2c97cfd0], esp
++0x1c62   f5                   cmc    
++0x1c63   49                   dec    ecx
++0x1c64   e2 21                loop   0x10000023
++0x1c66   e0 ef                loopne 0x0ffffff1
++0x1c68   7b d6                jnp    0x0fffffd8
++0x1c6a   8f b4 f7 9c a3 4b 41...??  <INVALID>
++0x1c6b   b4 f7                mov    ah, 0xf7
++0x1c6d   9c                   pushfd  
++0x1c6e   a3 4b 41 6d 15       mov    dword ptr [0x156d414b], eax
++0x1c73   90                   nop    
++0x1c74   d6                   salc   
++0x1c75   e0 85                loopne 0x0fffff87
++0x1c77   a4                   movsb   
++0x1c78   2f                   das    
++0x1c79   5d                   pop    ebp
++0x1c7a   e7 5c                out    0x5c, eax
++0x1c7c   0e                   push   cs
++0x1c7d   8f fa...??           <INVALID>
++0x1c7e   fa                   cli    
++0x1c7f   57                   push   edi
++0x1c80   13 fa                adc    edi, edx
++0x1c82   1b 85 04 37 33 8d    sbb    eax, dword ptr [ebp+0x8d333704]
++0x1c88   2e 8e c2             mov    es, edx
++0x1c8b   cd 30                int    0x30
++0x1c8d   6f                   outsd   
++0x1c8e   a1 de 89 e4 6d       mov    eax, dword ptr [0x6de489de]
++0x1c93   e1 2b                loope  0x1000002d
++0x1c95   0e                   push   cs
++0x1c96   75 cc                jnz    0x0fffffce
++0x1c98   1f                   pop    ds
++0x1c99   7d f9                jnl    0x0ffffffb
++0x1c9b   05 e6 a2 db 03       add    eax, 0x03dba2e6
++0x1ca0   3a f5                cmp    dh, ch
++0x1ca2   48                   dec    eax
++0x1ca3   bf 79 0f c9 63       mov    edi, 0x63c90f79
++0x1ca8   6d                   insd    
++0x1ca9   5d                   pop    ebp
++0x1caa   70 77                jo     0x10000079
++0x1cac   0f 1c 4f 65          nop    dword ptr [edi+0x65]
++0x1cb0   74 2a                jz     0x1000002c
++0x1cb2   12 cc                adc    cl, ah
++0x1cb4   37                   aaa    
++0x1cb5   1f                   pop    ds
++0x1cb6   a2 26 ab b4 d7       mov    byte ptr [0xd7b4ab26], al
++0x1cbb   93                   xchg   ebx, eax
++0x1cbc   cf                   iretd   
++0x1cbd   d6                   salc   
++0x1cbe   19 3f                sbb    dword ptr [edi], edi
++0x1cc0   47                   inc    edi
++0x1cc1   e4 2c                in     al, 0x2c
++0x1cc3   59                   pop    ecx
++0x1cc4   5b                   pop    ebx
++0x1cc5   f3 65 6d             rep insd 
++0x1cc8   af                   scasd   
++0x1cc9   82 06 bd             add    byte ptr [esi], 0xbd
++0x1ccc   07                   pop    es
++0x1ccd   b7 c9                mov    bh, 0xc9
++0x1ccf   c6 7a 99 d7...??     <INVALID>
++0x1cd0   7a 99                jp     0x0fffff9b
++0x1cd2   d7                   xlatb   
++0x1cd3   ef                   out    eax, dx
++0x1cd4   cb                   retf   
++0x1cd5   24 70                and    al, 0x70
++0x1cd7   55                   push   ebp
++0x1cd8   0e                   push   cs
++0x1cd9   5f                   pop    edi
++0x1cda   3d 82 7e 91 67       cmp    eax, 0x67917e82
++0x1cdf   0b e3                or     esp, ebx
++0x1ce1   cb                   retf   
++0x1ce2   e4 3a                in     al, 0x3a
++0x1ce4   71 33                jno    0x10000035
++0x1ce6   38 79 b9             cmp    byte ptr [ecx-0x47], bh
++0x1ce9   e4 35                in     al, 0x35
++0x1ceb   b8 bd b9 64 52       mov    eax, 0x5264b9bd
++0x1cf0   54                   push   esp
++0x1cf1   70 2b                jo     0x1000002d
++0x1cf3   48                   dec    eax
++0x1cf4   b4 89                mov    ah, 0x89
++0x1cf6   d3 40 9b             rol    dword ptr [eax-0x65], cl
++0x1cf9   e5 2d                in     eax, 0x2d
++0x1cfb   b4 36                mov    ah, 0x36
++0x1cfd   c6 8e fb 52 c4 c0 ac...??  <INVALID>
++0x1cfe   8e fb...??           <INVALID>
++0x1cff   fb                   sti    
++0x1d00   52                   push   edx
++0x1d01   c4 c0...??           <INVALID>
++0x1d02   c0 ac 0c 9d b8 4d fc shr    byte ptr [esp+ecx+0xfc4db89d], 0x0b
+     0b
++0x1d0a   17                   pop    ss
++0x1d0b   1a 97 37 a6 f1 b3    sbb    dl, byte ptr [edi+0xb3f1a637]
++0x1d11   34 89                xor    al, 0x89
++0x1d13   cd 35                int    0x35
++0x1d15   9e                   sahf   
++0x1d16   e7 81                out    0x81, eax
++0x1d18   62 f0...??           <INVALID>
++0x1d19   f0 fb...??           <INVALID>
++0x1d1a   fb                   sti    
++0x1d1b   91                   xchg   ecx, eax
++0x1d1c   cb                   retf   
++0x1d1d   c4 ec...??           <INVALID>
++0x1d1e   ec                   in     al
++0x1d1f   e9 8c f3 e4 db       jmp    0xebe4f391
++0x1d24   51                   push   ecx
++0x1d25   4b                   dec    ebx
++0x1d26   71 ab                jno    0x0fffffad
++0x1d28   d9 ae bf c7 d2 12    fldcw  word ptr [esi+0x12d2c7bf]
++0x1d2e   0f c8                bswap  eax
++0x1d30   bf 99 48 35 d7       mov    edi, 0xd7354899
++0x1d35   d4 bf                aam    0xbf
++0x1d37   98                   cwde   
++0x1d38   1f                   pop    ds
++0x1d39   64 b0 5b             mov    al, 0x5b
++0x1d3c   1f                   pop    ds
++0x1d3d   c0 0e ed             ror    byte ptr [esi], 0xed
++0x1d40   09 f3                or     ebx, esi
++0x1d42   d8 7f 25             fdivr  dword ptr [edi+0x25]
++0x1d45   86 44 c1 b9          xchg   byte ptr [ecx+eax*8-0x47], al
++0x1d49   29 0d 7f 70 de fc    sub    dword ptr [0xfcde707f], ecx
++0x1d4f   73 16                jnb    0x10000018
++0x1d51   73 09                jnb    0x1000000b
++0x1d53   9b                   fwait  
++0x1d54   a7                   cmpsd   
++0x1d55   27                   daa    
++0x1d56   ba 96 e5 bc 8d       mov    edx, 0x8dbce596
++0x1d5b   7e 2d                jle    0x1000002f
++0x1d5d   64 cf                iretd   
++0x1d5f   fd                   std    
++0x1d60   ac                   lodsb   
++0x1d61   19 aa 2b c6 6c 20    sbb    dword ptr [edx+0x206cc62b], ebp
++0x1d67   c5 2e                lds    ebp, [esi]
++0x1d69   9c                   pushfd  
++0x1d6a   eb d2                jmp    0x0fffffd4
++0x1d6c   06                   push   es
++0x1d6d   07                   pop    es
++0x1d6e   bf 71 38 cd 82       mov    edi, 0x82cd3871
++0x1d73   ba a6 9f 4f b2       mov    edx, 0xb24f9fa6
++0x1d78   81 9f 75 83 7a 3d 4b sbb    dword ptr [edi+0x3d7a8375], 0x7f3dbb4b
+     bb 3d 7f
++0x1d82   8d 73 0b             lea    esi, [ebx+0x0b]
++0x1d85   2c 7b                sub    al, 0x7b
++0x1d87   66 77 39             jnbe   0x0000003c
++0x1d8a   c1 ff 0f             sar    edi, 0x0f
++0x1d8d   99                   cdq    
++0x1d8e   98                   cwde   
++0x1d8f   a1 6c 0b 6c 77       mov    eax, dword ptr [0x776c0b6c]
++0x1d94   b6 a7                mov    dh, 0xa7
++0x1d96   02 49 7e             add    cl, byte ptr [ecx+0x7e]
++0x1d99   54                   push   esp
++0x1d9a   8a 28                mov    ch, byte ptr [eax]
++0x1d9c   21 f2                and    edx, esi
++0x1d9e   ae                   scasb   
++0x1d9f   dc d7                fcom   st0, st7
++0x1da1   dd 1c d0             fstp   qword ptr [eax+edx*8]
++0x1da4   ac                   lodsb   
++0x1da5   54                   push   esp
++0x1da6   75 8d                jnz    0x0fffff8f
++0x1da8   ee                   out    al, dx
++0x1da9   9b                   fwait  
++0x1daa   12 ab 7d 93 d5 56    adc    ch, byte ptr [ebx+0x56d5937d]
++0x1db0   a9 e4 3f 71 54       test   eax, 0x54713fe4
++0x1db5   e0 b4                loopne 0x0fffffb6
++0x1db7   33 85 5e 7a fd 72    xor    eax, dword ptr [ebp+0x72fd7a5e]
++0x1dbd   eb 05                jmp    0x10000007
++0x1dbf   e4 fd                in     al, 0xfd
++0x1dc1   ce                   into   
++0x1dc2   67 b8 ca 07 9c 3c    mov    eax, 0x3c9c07ca
++0x1dc8   c4 dd...??           <INVALID>
++0x1dc9   dd 49 c3             fisttp qword ptr [ecx-0x3d]
++0x1dcc   22 d8                and    bl, al
++0x1dce   3e 8c eb             mov    ebx, gs
++0x1dd1   31 37                xor    dword ptr [edi], esi
++0x1dd3   16                   push   ss
++0x1dd4   51                   push   ecx
++0x1dd5   99                   cdq    
++0x1dd6   b2 83                mov    dl, 0x83
++0x1dd8   f1                   int1   
++0x1dd9   c2 54 0b             ret    0x0b54
++0x1ddc   ee                   out    al, dx
++0x1ddd   3c 92                cmp    al, 0x92
++0x1ddf   96                   xchg   esi, eax
++0x1de0   18 ce                sbb    dh, cl
++0x1de2   8b 5a b7             mov    ebx, dword ptr [edx-0x49]
++0x1de5   07                   pop    es
++0x1de6   d7                   xlatb   
++0x1de7   ac                   lodsb   
++0x1de8   6b a8 2b 03 94 a5 18 imul   ebp, dword ptr [eax+0xa594032b], 0x18
++0x1def   90                   nop    
++0x1df0   f1                   int1   
++0x1df1   e0 06                loopne 0x10000008
++0x1df3   4f                   dec    edi
++0x1df4   b1 1a                mov    cl, 0x1a
++0x1df6   8a e8                mov    ch, al
++0x1df8   73 fc                jnb    0x0ffffffe
++0x1dfa   44                   inc    esp
++0x1dfb   57                   push   edi
++0x1dfc   2c e4                sub    al, 0xe4
++0x1dfe   0a e3                or     ah, bl
++0x1e00   93                   xchg   ebx, eax
++0x1e01   1f                   pop    ds
++0x1e02   dc 62 ba             fsub   qword ptr [edx-0x46]
++0x1e05   e6 d4                out    0xd4, al
++0x1e07   30 e0                xor    al, ah
++0x1e09   cc                   int3   
++0x1e0a   7a 42                jp     0x10000044
++0x1e0c   21 f1                and    ecx, esi
++0x1e0e   36 e8 e5 fc 12 31    call   0x4112fceb
++0x1e14   5a                   pop    edx
++0x1e15   6f                   outsd   
++0x1e16   07                   pop    es
++0x1e17   09 90 86 5f b4 04    or     dword ptr [eax+0x04b45f86], edx
++0x1e1d   38 43 ed             cmp    byte ptr [ebx-0x13], al
++0x1e20   77 e3                jnbe   0x0fffffe5
++0x1e22   5c                   pop    esp
++0x1e23   76 0c                jbe    0x1000000e
++0x1e25   7d f3                jnl    0x0ffffff5
++0x1e27   c8 45 14 e4          enter  0x1445, 0xe4
++0x1e2b   c4 7c a6 5e          les    edi, [esi+0x5e]
++0x1e2f   90                   nop    
++0x1e30   7e 44                jle    0x10000046
++0x1e32   37                   aaa    
++0x1e33   77 70                jnbe   0x10000072
++0x1e35   77 f2                jnbe   0x0ffffff4
++0x1e37   09 0b                or     dword ptr [ebx], ecx
++0x1e39   17                   pop    ss
++0x1e3a   a5                   movsd   
++0x1e3b   30 c2                xor    dl, al
++0x1e3d   80 07 83             add    byte ptr [edi], 0x83
++0x1e40   f2 eb 86             jmp    0x0fffff89
++0x1e43   e9 01 02 5d e2       jmp    0xf25d0206
++0x1e48   8b 52 25             mov    edx, dword ptr [edx+0x25]
++0x1e4b   b2 fe                mov    dl, 0xfe
++0x1e4d   5e                   pop    esi
++0x1e4e   f1                   int1   
++0x1e4f   85 fd                test   ebp, edi
++0x1e51   eb f4                jmp    0x0ffffff6
++0x1e53   a5                   movsd   
++0x1e54   bf 0b 21 cb 3a       mov    edi, 0x3acb210b
++0x1e59   70 ea                jo     0x0fffffec
++0x1e5b   b4 1d                mov    ah, 0x1d
++0x1e5d   20 7c 10 37          and    byte ptr [eax+edx+0x37], bh
++0x1e61   ee                   out    al, dx
++0x1e62   0d 33 83 d1 8e       or     eax, 0x8ed18333
+#endif
diff --git a/suite/tests/api/ir.c b/suite/tests/api/ir.c
new file mode 100644
index 000000000..5001db71e
--- /dev/null
+++ b/suite/tests/api/ir.c
@@ -0,0 +1,654 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Uses the DR CLIENT_INTERFACE API, using DR as a standalone library, rather than
+ * being a client library working with DR on a target program.
+ *
+ * To run, you need to put dynamorio.dll into either the current directory
+ * or system32.
+ */
+
+#ifndef USE_DYNAMO
+#error NEED USE_DYNAMO
+#endif
+
+#include "dr_api.h"
+#include "tools.h"
+
+#ifdef WINDOWS
+# define _USE_MATH_DEFINES 1
+# include <math.h> /* for M_PI, M_LN2, and M_LN10 for OP_fldpi, etc. */
+#endif
+
+#define VERBOSE 0
+
+#define ASSERT(x) \
+    ((void)((!(x)) ? \
+        (dr_fprintf(STDERR, "ASSERT FAILURE: %s:%d: %s\n", __FILE__,  __LINE__, #x),\
+         dr_abort(), 0) : 0))
+
+#define BOOLS_MATCH(b1, b2) (((b1) && (b2)) || (!(b1) && !(b2)))
+
+static char buf[4096];
+
+/* make sure the following are consistent (though they could still all be wrong :))
+ * with respect to instr length and opcode:
+ * - decode_fast
+ * - decode
+ * - INSTR_CREATE_
+ * - encode
+ */
+static void
+test_all_opcodes(void *dc)
+{
+    byte *pc, *next_pc;
+    byte *end;
+    instrlist_t *ilist = instrlist_create(dc);
+    instr_t *instr;
+
+    /* we cannot pass on variadic args as separate args to another
+     * macro, so we must split ours by # args (xref PR 208603)
+     */
+
+#   define MEMARG(sz) (opnd_create_base_disp(REG_XCX, REG_NULL, 0, 0x37, sz))
+#   define IMMARG(sz)  opnd_create_immed_int(37, sz)
+#   define TGTARG      opnd_create_instr(instrlist_last(ilist))
+#   define REGARG(reg) opnd_create_reg(REG_##reg)
+#   define X86_ONLY    1
+#   define X64_ONLY    2
+
+#   define OPCODE(opc, icnm, ...) \
+    int len_##icnm;
+#   include "ir_0args.h"
+#   include "ir_1args.h"
+#   include "ir_2args.h"
+#   include "ir_3args.h"
+#   include "ir_4args.h"
+#   undef OPCODE
+
+    /* we can encode+fast-decode some instrs cross-platform but we
+     * leave that testing to the regression run on that platform */ 
+
+#   define OPCODE(opc, icnm, flags) do { \
+    if ((flags & IF_X64_ELSE(X86_ONLY, X64_ONLY)) == 0) { \
+        instrlist_append(ilist, INSTR_CREATE_##icnm(dc)); \
+        len_##icnm = instr_length(dc, instrlist_last(ilist)); \
+    } } while (0);
+#   include "ir_0args.h"
+#   undef OPCODE
+
+#   define OPCODE(opc, icnm, flags, arg1) do { \
+    if ((flags & IF_X64_ELSE(X86_ONLY, X64_ONLY)) == 0) { \
+        instrlist_append(ilist, INSTR_CREATE_##icnm(dc, arg1)); \
+        len_##icnm = instr_length(dc, instrlist_last(ilist)); \
+    } } while (0);
+#   include "ir_1args.h"
+#   undef OPCODE
+
+#   define OPCODE(opc, icnm, flags, arg1, arg2) do { \
+    if ((flags & IF_X64_ELSE(X86_ONLY, X64_ONLY)) == 0) { \
+        instrlist_append(ilist, INSTR_CREATE_##icnm(dc, arg1, arg2)); \
+        len_##icnm = instr_length(dc, instrlist_last(ilist)); \
+    } } while (0);
+#   include "ir_2args.h"
+#   undef OPCODE
+
+#   define OPCODE(opc, icnm, flags, arg1, arg2, arg3) do { \
+    if ((flags & IF_X64_ELSE(X86_ONLY, X64_ONLY)) == 0) { \
+        instrlist_append(ilist, INSTR_CREATE_##icnm(dc, arg1, arg2, arg3)); \
+        len_##icnm = instr_length(dc, instrlist_last(ilist)); \
+    } } while (0);
+#   include "ir_3args.h"
+#   undef OPCODE
+
+#   define OPCODE(opc, icnm, flags, arg1, arg2, arg3, arg4) do { \
+    if ((flags & IF_X64_ELSE(X86_ONLY, X64_ONLY)) == 0) { \
+        instrlist_append(ilist, INSTR_CREATE_##icnm(dc, arg1, arg2, arg3, arg4)); \
+        len_##icnm = instr_length(dc, instrlist_last(ilist)); \
+    } } while (0);
+#   include "ir_4args.h"
+#   undef OPCODE
+
+    end = instrlist_encode(dc, ilist, buf, false);
+
+    instr = instr_create(dc);
+    pc = buf;
+
+#   define OPCODE(opc, icnm, flags, ...) do { \
+    if ((flags & IF_X64_ELSE(X86_ONLY, X64_ONLY)) == 0 && len_##icnm != 0) { \
+        instr_reset(dc, instr); \
+        next_pc = decode(dc, pc, instr); \
+        ASSERT((next_pc - pc) == decode_sizeof(dc, pc, NULL _IF_X64(NULL))); \
+        ASSERT((next_pc - pc) == len_##icnm); \
+        ASSERT(instr_get_opcode(instr) == OP_##opc); \
+        pc = next_pc; \
+    } } while (0);
+#   include "ir_0args.h"
+#   include "ir_1args.h"
+#   include "ir_2args.h"
+#   include "ir_3args.h"
+#   include "ir_4args.h"
+#   undef OPCODE
+
+#if VERBOSE
+    for (pc = buf; pc < end; )
+        pc = disassemble_with_info(dc, pc, STDOUT, true, true);
+#endif
+
+    instr_destroy(dc, instr);
+    instrlist_clear_and_destroy(dc, ilist);
+}
+
+static void
+test_disp_control_helper(void *dc, int disp,
+                         bool encode_zero_disp, bool force_full_disp, bool disp16,
+                         uint len_expect)
+{
+    byte *pc;
+    uint len;
+    instr_t *instr = INSTR_CREATE_mov_ld
+        (dc, opnd_create_reg(REG_ECX),
+         opnd_create_base_disp_ex(disp16 ? REG_BX : REG_EBX, REG_NULL, 0,
+                                  disp, OPSZ_4,
+                                  encode_zero_disp, force_full_disp, disp16));
+    pc = instr_encode(dc, instr, buf);
+    len = (int) (pc - (byte *)buf);
+#if VERBOSE
+    pc = disassemble_with_info(dc, buf, STDOUT, true, true);
+#endif
+    ASSERT(len == len_expect);
+    instr_reset(dc, instr);
+    decode(dc, buf, instr);
+    ASSERT(instr_num_srcs(instr) == 1 &&
+           opnd_is_base_disp(instr_get_src(instr, 0)) &&
+           BOOLS_MATCH(encode_zero_disp, 
+                       opnd_is_disp_encode_zero(instr_get_src(instr, 0))) &&
+           BOOLS_MATCH(force_full_disp,
+                       opnd_is_disp_force_full(instr_get_src(instr, 0))) &&
+           BOOLS_MATCH(disp16,
+                       opnd_is_disp_short_addr(instr_get_src(instr, 0))));
+    instr_destroy(dc, instr);
+}
+
+/* Test encode_zero_disp and force_full_disp control from case 4457 */
+static void
+test_disp_control(void *dc)
+{
+    /*
+    0x004275b4   8b 0b                mov    (%ebx) -> %ecx 
+    0x004275b4   8b 4b 00             mov    $0x00(%ebx) -> %ecx 
+    0x004275b4   8b 8b 00 00 00 00    mov    $0x00000000 (%ebx) -> %ecx 
+    0x004275b4   8b 4b 7f             mov    $0x7f(%ebx) -> %ecx 
+    0x004275b4   8b 8b 7f 00 00 00    mov    $0x0000007f (%ebx) -> %ecx 
+    0x00430258   67 8b 4f 7f          addr16 mov    0x7f(%bx) -> %ecx 
+    0x00430258   67 8b 8f 7f 00       addr16 mov    0x007f(%bx) -> %ecx 
+    */
+    test_disp_control_helper(dc, 0, false, false, false, 2);
+    test_disp_control_helper(dc, 0, true,  false, false, 3);
+    test_disp_control_helper(dc, 0, true,  true,  false,  6);
+    test_disp_control_helper(dc, 0x7f, false, false, false, 3);
+    test_disp_control_helper(dc, 0x7f, false, true,  false,  6);
+    test_disp_control_helper(dc, 0x7f, false, false, true,  4);
+    test_disp_control_helper(dc, 0x7f, false, true,  true,  5);
+}
+
+/* emits the instruction to buf (for tests that wish to do additional checks on
+ * the output) */
+static void
+test_instr_encode(void *dc, instr_t *instr, uint len_expect)
+{
+    instr_t *decin;
+    uint len;
+    byte *pc = instr_encode(dc, instr, buf);
+    len = (int) (pc - (byte *)buf);
+#if VERBOSE
+    disassemble_with_info(dc, buf, STDOUT, true, true);
+#endif
+    ASSERT(len == len_expect);
+    decin = instr_create(dc);
+    decode(dc, buf, decin);
+    ASSERT(instr_same(instr, decin));
+    instr_destroy(dc, instr);
+    instr_destroy(dc, decin);
+}
+
+/* emits the instruction to buf (for tests that wish to do additional checks on
+ * the output) */
+static void
+test_instr_encode_and_decode(void *dc, instr_t *instr, uint len_expect,
+                             /* also checks one operand's size */
+                             bool src, uint opnum, opnd_size_t sz, uint bytes)
+{
+    opnd_t op;
+    opnd_size_t opsz;
+    instr_t *decin;
+    uint len;
+    byte *pc = instr_encode(dc, instr, buf);
+    len = (int) (pc - (byte *)buf);
+#if VERBOSE
+    disassemble_with_info(dc, buf, STDOUT, true, true);
+#endif
+    ASSERT(len == len_expect);
+    decin = instr_create(dc);
+    decode(dc, buf, decin);
+    ASSERT(instr_same(instr, decin));
+
+    /* PR 245805: variable sizes should be resolved on decode */
+    if (src)
+        op = instr_get_src(decin, opnum);
+    else
+        op = instr_get_dst(decin, opnum);
+    opsz = opnd_get_size(op);
+    ASSERT(opsz == sz && opnd_size_in_bytes(opsz) == bytes);
+
+    instr_destroy(dc, instr);
+    instr_destroy(dc, decin);
+}
+
+static void
+test_indirect_cti(void *dc)
+{
+    /*
+    0x004275f4   ff d1                call   %ecx %esp -> %esp (%esp) 
+    0x004275f4   66 ff d1             data16 call   %cx %esp -> %esp (%esp) 
+    0x004275f4   67 ff d1             addr16 call   %ecx %sp -> %sp (%sp) 
+    0x00427794   ff 19                lcall  (%ecx) %esp -> %esp (%esp) 
+    0x00427794   66 ff 19             data16 lcall  (%ecx) %esp -> %esp (%esp) 
+    0x00427794   67 ff 1f             addr16 lcall  (%bx) %sp -> %sp (%sp) 
+    */
+    instr_t *instr;
+    instr = INSTR_CREATE_call_ind(dc, opnd_create_reg(REG_ECX));
+    test_instr_encode(dc, instr, 2);
+    instr = instr_create_2dst_2src(dc, OP_call_ind, opnd_create_reg(REG_ESP),
+                                   opnd_create_base_disp(REG_ESP, REG_NULL, 0, 0, OPSZ_2),
+                                   opnd_create_reg(REG_CX), opnd_create_reg(REG_ESP));
+    test_instr_encode(dc, instr, 3);
+    instr = instr_create_2dst_2src(dc, OP_call_ind, opnd_create_reg(REG_SP),
+                                   opnd_create_base_disp(REG_SP, REG_NULL, 0, 0, OPSZ_4_short2),
+                                   opnd_create_reg(REG_ECX), opnd_create_reg(REG_SP));
+    test_instr_encode(dc, instr, 3);
+
+    /* invalid to have far call go through reg since needs 6 bytes */
+    instr = INSTR_CREATE_call_far_ind(dc, opnd_create_base_disp(REG_ECX, REG_NULL, 0, 0,
+                                                                OPSZ_6));
+    test_instr_encode(dc, instr, 2);
+    instr = instr_create_2dst_2src(dc, OP_call_far_ind, opnd_create_reg(REG_ESP),
+                                   opnd_create_base_disp(REG_ESP, REG_NULL, 0, 0, OPSZ_2),
+                                   opnd_create_base_disp(REG_ECX, REG_NULL, 0, 0,
+                                                         OPSZ_4),
+                                   opnd_create_reg(REG_ESP));
+    test_instr_encode(dc, instr, 3);
+    instr = instr_create_2dst_2src(dc, OP_call_far_ind, opnd_create_reg(REG_SP),
+                                   opnd_create_base_disp(REG_SP, REG_NULL, 0, 0, OPSZ_4_short2),
+                                   opnd_create_base_disp(REG_BX, REG_NULL, 0, 0,
+                                                         OPSZ_6),
+                                   opnd_create_reg(REG_SP));
+    test_instr_encode(dc, instr, 3);
+
+    /* case 10710: make sure we can encode these guys
+         0x00428844   0e                   push   %cs %esp -> %esp (%esp) 
+         0x00428844   1e                   push   %ds %esp -> %esp (%esp) 
+         0x00428844   16                   push   %ss %esp -> %esp (%esp) 
+         0x00428844   06                   push   %es %esp -> %esp (%esp) 
+         0x00428844   0f a0                push   %fs %esp -> %esp (%esp) 
+         0x00428844   0f a8                push   %gs %esp -> %esp (%esp) 
+         0x00428844   1f                   pop    %esp (%esp) -> %ds %esp 
+         0x00428844   17                   pop    %esp (%esp) -> %ss %esp 
+         0x00428844   07                   pop    %esp (%esp) -> %es %esp 
+         0x00428844   0f a1                pop    %esp (%esp) -> %fs %esp 
+         0x00428844   0f a9                pop    %esp (%esp) -> %gs %esp 
+     */
+    test_instr_encode(dc, INSTR_CREATE_push(dc, opnd_create_reg(SEG_CS)), 1);
+    test_instr_encode(dc, INSTR_CREATE_push(dc, opnd_create_reg(SEG_DS)), 1);
+    test_instr_encode(dc, INSTR_CREATE_push(dc, opnd_create_reg(SEG_SS)), 1);
+    test_instr_encode(dc, INSTR_CREATE_push(dc, opnd_create_reg(SEG_ES)), 1);
+    test_instr_encode(dc, INSTR_CREATE_push(dc, opnd_create_reg(SEG_FS)), 2);
+    test_instr_encode(dc, INSTR_CREATE_push(dc, opnd_create_reg(SEG_GS)), 2);
+    test_instr_encode(dc, INSTR_CREATE_pop(dc, opnd_create_reg(SEG_DS)), 1);
+    test_instr_encode(dc, INSTR_CREATE_pop(dc, opnd_create_reg(SEG_SS)), 1);
+    test_instr_encode(dc, INSTR_CREATE_pop(dc, opnd_create_reg(SEG_ES)), 1);
+    test_instr_encode(dc, INSTR_CREATE_pop(dc, opnd_create_reg(SEG_FS)), 2);
+    test_instr_encode(dc, INSTR_CREATE_pop(dc, opnd_create_reg(SEG_GS)), 2);
+}
+
+static void
+test_cti_prefixes(void *dc)
+{
+    /* case 10689: test decoding jmp/call w/ 16-bit prefixes
+     *   0x00428844   66 e9 ab cd          data16 jmp    $0x55f3 
+     *   0x00428844   67 e9 ab cd ef 12    addr16 jmp    $0x133255f5 
+     */
+    buf[0] = 0x66;
+    buf[1] = 0xe9;
+    buf[2] = 0xab;
+    buf[3] = 0xcd;
+    buf[4] = 0xef;
+    buf[5] = 0x12;
+    /* data16 (0x66) == 4 bytes, while addr16 (0x67) == 6 bytes */
+    ASSERT(decode_next_pc(dc, buf) == (byte *) &buf[4]);
+#if VERBOSE
+    disassemble_with_info(dc, buf, STDOUT, true, true);
+#endif
+    buf[0] = 0x67;
+    ASSERT(decode_next_pc(dc, buf) == (byte *) &buf[6]);
+#if VERBOSE
+    disassemble_with_info(dc, buf, STDOUT, true, true);
+#endif
+}
+
+static void
+test_modrm16_helper(void *dc, reg_id_t base, reg_id_t scale, uint disp, uint len)
+{
+    instr_t *instr;
+    /* Avoid REG_EAX b/c of the special 0xa0-0xa3 opcodes */
+    instr = INSTR_CREATE_mov_ld(dc, opnd_create_reg(REG_EBX),
+                                opnd_create_base_disp(base, scale,
+                                                      (scale == REG_NULL ? 0 : 1),
+                                                      /* we need OPSZ_4_short2 to match
+                                                       * instr_same on decode! */
+                                                      disp, OPSZ_4_short2));
+    if (base == REG_NULL && scale == REG_NULL) {
+        /* Don't need _ex unless abs addr, in which case should get 32-bit
+         * disp!  Test both sides. */
+        test_instr_encode(dc, instr, len + 1/*32-bit disp but no prefix*/);
+        instr = INSTR_CREATE_mov_ld(dc, opnd_create_reg(REG_EBX),
+                                    opnd_create_base_disp_ex(base, scale,
+                                                             (scale == REG_NULL ? 0 : 1),
+                                                             /* we need OPSZ_4_short2 to match
+                                                              * instr_same on decode! */
+                                                             disp, OPSZ_4_short2,
+                                                             false, false, true));
+        test_instr_encode(dc, instr, len);
+    } else {
+        test_instr_encode(dc, instr, len);
+    }
+}
+
+static void
+test_modrm16(void *dc)
+{
+    /*
+     *   0x00428964   67 8b 18             addr16 mov    (%bx,%si,1) -> %ebx 
+     *   0x00428964   67 8b 19             addr16 mov    (%bx,%di,1) -> %ebx 
+     *   0x00428964   67 8b 1a             addr16 mov    (%bp,%si,1) -> %ebx 
+     *   0x00428964   67 8b 1b             addr16 mov    (%bp,%di,1) -> %ebx 
+     *   0x00428964   67 8b 1c             addr16 mov    (%si) -> %ebx 
+     *   0x00428964   67 8b 1d             addr16 mov    (%di) -> %ebx 
+     *   0x004289c4   8b 1d 7f 00 00 00    mov    0x7f -> %ebx 
+     *   0x004289c4   67 8b 1e 7f 00       addr16 mov    0x7f -> %ebx 
+     *   0x004289c4   67 8b 5e 00          addr16 mov    (%bp) -> %ebx 
+     *   0x004289c4   67 8b 1f             addr16 mov    (%bx) -> %ebx 
+     *   0x004289c4   67 8b 58 7f          addr16 mov    0x7f(%bx,%si,1) -> %ebx 
+     *   0x004289c4   67 8b 59 7f          addr16 mov    0x7f(%bx,%di,1) -> %ebx 
+     *   0x004289c4   67 8b 5a 7f          addr16 mov    0x7f(%bp,%si,1) -> %ebx 
+     *   0x004289c4   67 8b 5b 7f          addr16 mov    0x7f(%bp,%di,1) -> %ebx 
+     *   0x004289c4   67 8b 5c 7f          addr16 mov    0x7f(%si) -> %ebx 
+     *   0x004289c4   67 8b 5d 7f          addr16 mov    0x7f(%di) -> %ebx 
+     *   0x004289c4   67 8b 5e 7f          addr16 mov    0x7f(%bp) -> %ebx 
+     *   0x004289c4   67 8b 5f 7f          addr16 mov    0x7f(%bx) -> %ebx 
+     *   0x004289c4   67 8b 98 80 00       addr16 mov    0x0080(%bx,%si,1) -> %ebx 
+     *   0x004289c4   67 8b 99 80 00       addr16 mov    0x0080(%bx,%di,1) -> %ebx 
+     *   0x004289c4   67 8b 9a 80 00       addr16 mov    0x0080(%bp,%si,1) -> %ebx 
+     *   0x004289c4   67 8b 9b 80 00       addr16 mov    0x0080(%bp,%di,1) -> %ebx 
+     *   0x004289c4   67 8b 9c 80 00       addr16 mov    0x0080(%si) -> %ebx 
+     *   0x004289c4   67 8b 9d 80 00       addr16 mov    0x0080(%di) -> %ebx 
+     *   0x004289c4   67 8b 9e 80 00       addr16 mov    0x0080(%bp) -> %ebx 
+     *   0x004289c4   67 8b 9f 80 00       addr16 mov    0x0080(%bx) -> %ebx 
+     */
+    test_modrm16_helper(dc, REG_BX, REG_SI,      0, 3);
+    test_modrm16_helper(dc, REG_BX, REG_DI,      0, 3);
+    test_modrm16_helper(dc, REG_BP, REG_SI,      0, 3);
+    test_modrm16_helper(dc, REG_BP, REG_DI,      0, 3);
+    test_modrm16_helper(dc, REG_SI, REG_NULL,    0, 3);
+    test_modrm16_helper(dc, REG_DI, REG_NULL,    0, 3);
+    test_modrm16_helper(dc, REG_NULL, REG_NULL, 0x7f, 5); /* must do disp16 */
+    test_modrm16_helper(dc, REG_BP, REG_NULL,    0, 4); /* must do disp8 */
+    test_modrm16_helper(dc, REG_BX, REG_NULL,    0, 3);
+
+    test_modrm16_helper(dc, REG_BX, REG_SI,   0x7f, 4);
+    test_modrm16_helper(dc, REG_BX, REG_DI,   0x7f, 4);
+    test_modrm16_helper(dc, REG_BP, REG_SI,   0x7f, 4);
+    test_modrm16_helper(dc, REG_BP, REG_DI,   0x7f, 4);
+    test_modrm16_helper(dc, REG_SI, REG_NULL, 0x7f, 4);
+    test_modrm16_helper(dc, REG_DI, REG_NULL, 0x7f, 4);
+    test_modrm16_helper(dc, REG_BP, REG_NULL, 0x7f, 4);
+    test_modrm16_helper(dc, REG_BX, REG_NULL, 0x7f, 4);
+
+    test_modrm16_helper(dc, REG_BX, REG_SI,   0x80, 5);
+    test_modrm16_helper(dc, REG_BX, REG_DI,   0x80, 5);
+    test_modrm16_helper(dc, REG_BP, REG_SI,   0x80, 5);
+    test_modrm16_helper(dc, REG_BP, REG_DI,   0x80, 5);
+    test_modrm16_helper(dc, REG_SI, REG_NULL, 0x80, 5);
+    test_modrm16_helper(dc, REG_DI, REG_NULL, 0x80, 5);
+    test_modrm16_helper(dc, REG_BP, REG_NULL, 0x80, 5);
+    test_modrm16_helper(dc, REG_BX, REG_NULL, 0x80, 5);
+}
+
+/* PR 215143: auto-magically add size prefixes */
+static void
+test_size_changes(void *dc)
+{
+    /*
+     *   0x004299d4   67 51                addr16 push   %ecx %sp -> %sp (%sp) 
+     *   0x004299d4   66 51                data16 push   %cx %esp -> %esp (%esp) 
+     *   0x004299d4   66 67 51             data16 addr16 push   %cx %sp -> %sp (%sp) 
+     *   0x004298a4   e3 fe                jecxz  $0x004298a4 %ecx 
+     *   0x004298a4   67 e3 fd             addr16 jecxz  $0x004298a4 %cx 
+     *   0x080a5260   67 e2 fd             addr16 loop   $0x080a5260 %cx -> %cx
+     *   0x080a5260   67 e1 fd             addr16 loope  $0x080a5260 %cx -> %cx
+     *   0x080a5260   67 e0 fd             addr16 loopne $0x080a5260 %cx -> %cx
+     */
+    instr_t *instr;
+    /* push addr16 */
+    instr = instr_create_2dst_2src(dc, OP_push, opnd_create_reg(REG_SP),
+                                   opnd_create_base_disp(REG_SP, REG_NULL, 0, 0, OPSZ_4_short2),
+                                   opnd_create_reg(REG_ECX), opnd_create_reg(REG_SP));
+    test_instr_encode(dc, instr, 2);
+    /* push data16 */
+    instr = instr_create_2dst_2src(dc, OP_push, opnd_create_reg(REG_ESP),
+                                   opnd_create_base_disp(REG_ESP, REG_NULL, 0, 0, OPSZ_2),
+                                   opnd_create_reg(REG_CX), opnd_create_reg(REG_ESP));
+    test_instr_encode(dc, instr, 2);
+    /* push addr16 and data16 */
+    instr = instr_create_2dst_2src(dc, OP_push, opnd_create_reg(REG_SP),
+                                   opnd_create_base_disp(REG_SP, REG_NULL, 0, 0, OPSZ_2),
+                                   opnd_create_reg(REG_CX), opnd_create_reg(REG_SP));
+    test_instr_encode(dc, instr, 3);
+    /* jecxz and jcxz */
+    test_instr_encode(dc, INSTR_CREATE_jecxz(dc, opnd_create_pc(buf)), 2);
+    instr = instr_create_0dst_2src
+        (dc, OP_jecxz, opnd_create_pc(buf), opnd_create_reg(REG_CX));
+    test_instr_encode(dc, instr, 3);
+    instr = instr_create_1dst_2src
+        (dc, OP_loop, opnd_create_reg(REG_CX), opnd_create_pc(buf),
+         opnd_create_reg(REG_CX));
+    test_instr_encode(dc, instr, 3);
+    instr = instr_create_1dst_2src
+        (dc, OP_loope, opnd_create_reg(REG_CX), opnd_create_pc(buf),
+         opnd_create_reg(REG_CX));
+    test_instr_encode(dc, instr, 3);
+    instr = instr_create_1dst_2src
+        (dc, OP_loopne, opnd_create_reg(REG_CX), opnd_create_pc(buf),
+         opnd_create_reg(REG_CX));
+    test_instr_encode(dc, instr, 3);
+    
+
+    /*
+     *   0x004ee0b8   a6                   cmps   %ds:(%esi) %es:(%edi) %esi %edi -> %esi %edi 
+     *   0x004ee0b8   67 a6                addr16 cmps   %ds:(%si) %es:(%di) %si %di -> %si %di 
+     *   0x004ee0b8   66 a7                data16 cmps   %ds:(%esi) %es:(%edi) %esi %edi -> %esi %edi 
+     *   0x004ee0b8   d7                   xlat   %ds:(%ebx,%al,1) -> %al 
+     *   0x004ee0b8   67 d7                addr16 xlat   %ds:(%bx,%al,1) -> %al 
+     *   0x004ee0b8   0f f7 c1             maskmovq %mm0 %mm1 -> %ds:(%edi) 
+     *   0x004ee0b8   67 0f f7 c1          addr16 maskmovq %mm0 %mm1 -> %ds:(%di) 
+     *   0x004ee0b8   66 0f f7 c1          maskmovdqu %xmm0 %xmm1 -> %ds:(%edi) 
+     *   0x004ee0b8   67 66 0f f7 c1       addr16 maskmovdqu %xmm0 %xmm1 -> %ds:(%di)
+     */
+    test_instr_encode(dc, INSTR_CREATE_cmps_1(dc), 1);
+    instr = instr_create_2dst_4src
+        (dc, OP_cmps, opnd_create_reg(REG_SI), opnd_create_reg(REG_DI),
+         opnd_create_far_base_disp(SEG_DS, REG_SI, REG_NULL, 0, 0, OPSZ_1),
+         opnd_create_far_base_disp(SEG_ES, REG_DI, REG_NULL, 0, 0, OPSZ_1),
+         opnd_create_reg(REG_SI), opnd_create_reg(REG_DI));
+    test_instr_encode(dc, instr, 2);
+
+    instr = instr_create_2dst_4src
+        (dc, OP_cmps, opnd_create_reg(REG_ESI), opnd_create_reg(REG_EDI),
+         opnd_create_far_base_disp(SEG_DS, REG_ESI, REG_NULL, 0, 0, OPSZ_2),
+         opnd_create_far_base_disp(SEG_ES, REG_EDI, REG_NULL, 0, 0, OPSZ_2),
+         opnd_create_reg(REG_ESI), opnd_create_reg(REG_EDI));
+    test_instr_encode_and_decode(dc, instr, 2, true/*src*/, 0, OPSZ_2, 2);
+
+    test_instr_encode(dc, INSTR_CREATE_xlat(dc), 1);
+    instr = instr_create_1dst_1src
+        (dc, OP_xlat, opnd_create_reg(REG_AL),
+         opnd_create_far_base_disp(SEG_DS, REG_BX, REG_AL, 1, 0, OPSZ_1));
+    test_instr_encode(dc, instr, 2);
+
+    instr = INSTR_CREATE_maskmovq(dc, opnd_create_reg(REG_MM0),
+                                  opnd_create_reg(REG_MM1));
+    test_instr_encode(dc, instr, 3);
+    instr = instr_create_1dst_2src
+        (dc, OP_maskmovq, 
+         opnd_create_far_base_disp(SEG_DS, REG_DI, REG_NULL, 0, 0, OPSZ_8),
+         opnd_create_reg(REG_MM0), opnd_create_reg(REG_MM1));
+    test_instr_encode(dc, instr, 4);
+
+    instr = INSTR_CREATE_maskmovdqu(dc, opnd_create_reg(REG_XMM0),
+                                    opnd_create_reg(REG_XMM1));
+    test_instr_encode(dc, instr, 4);
+    instr = instr_create_1dst_2src
+        (dc, OP_maskmovdqu, 
+         opnd_create_far_base_disp(SEG_DS, REG_DI, REG_NULL, 0, 0, OPSZ_16),
+         opnd_create_reg(REG_XMM0), opnd_create_reg(REG_XMM1));
+    test_instr_encode(dc, instr, 5);
+
+    /* Test iretw, iretd, iretq (unlike most stack operation iretd (and lretd on AMD)
+     * exist and are the default in 64-bit mode. As such, it has a different size/type
+     * then most other stack operations).  Our instr_create routine should match stack
+     * (iretq on 64-bit, iretd on 32-bit). See PR 191977. */
+    instr = INSTR_CREATE_iret(dc);
+#ifdef X64
+    test_instr_encode_and_decode(dc, instr, 2, true /*src*/, 1, OPSZ_8, 8);
+    ASSERT(buf[0] == 0x48); /* check for rex.w prefix */
+#else
+    test_instr_encode_and_decode(dc, instr, 1, true /*src*/, 1, OPSZ_4, 4);
+#endif
+    instr = instr_create_1dst_2src
+        (dc, OP_iret, opnd_create_reg(REG_XSP), opnd_create_reg(REG_XSP),
+         opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_4));
+    test_instr_encode_and_decode(dc, instr, 1, true /*src*/, 1, OPSZ_4, 4);
+    instr = instr_create_1dst_2src
+        (dc, OP_iret, opnd_create_reg(REG_XSP), opnd_create_reg(REG_XSP),
+         opnd_create_base_disp(REG_XSP, REG_NULL, 0, 0, OPSZ_2));
+    test_instr_encode_and_decode(dc, instr, 2, true /*src*/, 1, OPSZ_2, 2);
+    ASSERT(buf[0] == 0x66); /* check for data prefix */
+}
+
+/* PR 332254: test xchg vs nop */
+static void
+test_nop_xchg(void *dc)
+{
+    /*   0x0000000000671460  87 c0                xchg   %eax %eax -> %eax %eax
+     *   0x0000000000671460  48 87 c0             xchg   %rax %rax -> %rax %rax
+     *   0x0000000000671460  41 87 c0             xchg   %r8d %eax -> %r8d %eax
+     *   0x0000000000671460  46 90                nop
+     *   0x0000000000671460  4e 90                nop
+     *   0x0000000000671460  41 90                xchg   %r8d %eax -> %r8d %eax
+     */
+    instr_t *instr;
+    instr = INSTR_CREATE_xchg(dc, opnd_create_reg(REG_EAX), opnd_create_reg(REG_EAX));
+    test_instr_encode(dc, instr, 2);
+#ifdef X64
+    /* we don't do the optimal "48 90" instead of "48 87 c0" */
+    instr = INSTR_CREATE_xchg(dc, opnd_create_reg(REG_RAX), opnd_create_reg(REG_RAX));
+    test_instr_encode(dc, instr, 3);
+    /* we don't do the optimal "41 90" instead of "41 87 c0" */
+    instr = INSTR_CREATE_xchg(dc, opnd_create_reg(REG_R8D), opnd_create_reg(REG_EAX));
+    test_instr_encode(dc, instr, 3);
+    /* ensure we treat as nop and NOT xchg if doesn't have rex.b */
+    buf[0] = 0x46;
+    buf[1] = 0x90;
+    instr = instr_create(dc);
+# if VERBOSE
+    disassemble_with_info(dc, buf, STDOUT, true, true);
+# endif
+    decode(dc, buf, instr);
+    ASSERT(instr_get_opcode(instr) == OP_nop);
+    instr_destroy(dc, instr);
+    buf[0] = 0x4e;
+    buf[1] = 0x90;
+    instr = instr_create(dc);
+# if VERBOSE
+    disassemble_with_info(dc, buf, STDOUT, true, true);
+# endif
+    decode(dc, buf, instr);
+    ASSERT(instr_get_opcode(instr) == OP_nop);
+    instr_destroy(dc, instr);
+    buf[0] = 0x41;
+    buf[1] = 0x90;
+    instr = instr_create(dc);
+# if VERBOSE
+    disassemble_with_info(dc, buf, STDOUT, true, true);
+# endif
+    decode(dc, buf, instr);
+    ASSERT(instr_get_opcode(instr) == OP_xchg);
+    instr_destroy(dc, instr);
+#endif
+}
+
+int
+main(int argc, char *argv[])
+{
+    void *dcontext = dr_standalone_init();
+
+    test_all_opcodes(dcontext);
+
+#ifdef X64
+    /* FIXME: NYI */
+#else
+    test_disp_control(dcontext);
+
+    test_indirect_cti(dcontext);
+
+    test_cti_prefixes(dcontext);
+
+    test_modrm16(dcontext);
+
+    test_size_changes(dcontext);
+#endif
+
+    test_nop_xchg(dcontext);
+
+    print("all done\n");
+    return 0;
+}
diff --git a/suite/tests/api/ir.expect b/suite/tests/api/ir.expect
new file mode 100755
index 000000000..de6391430
--- /dev/null
+++ b/suite/tests/api/ir.expect
@@ -0,0 +1 @@
+all done
diff --git a/suite/tests/api/ir_0args.h b/suite/tests/api/ir_0args.h
new file mode 100644
index 000000000..8a2b5f1bd
--- /dev/null
+++ b/suite/tests/api/ir_0args.h
@@ -0,0 +1,164 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/****************************************************************************/
+/* no arguments */
+
+OPCODE(fwait, fwait, 0)
+OPCODE(hlt, hlt, 0)
+OPCODE(cmc, cmc, 0)
+OPCODE(clc, clc, 0)
+OPCODE(stc, stc, 0)
+OPCODE(cli, cli, 0)
+OPCODE(sti, sti, 0)
+OPCODE(cld, cld, 0)
+OPCODE(std, std, 0)
+OPCODE(clts, clts, 0)
+OPCODE(invd, invd, 0)
+OPCODE(wbinvd, wbinvd, 0)
+OPCODE(ud2a, ud2a, 0)
+OPCODE(emms, emms, 0)
+OPCODE(rsm, rsm, 0)
+OPCODE(ud2b, ud2b, 0)
+OPCODE(lfence, lfence, 0)
+OPCODE(mfence, mfence, 0)
+OPCODE(sfence, sfence, 0)
+OPCODE(nop, nop, 0)
+OPCODE(pause, pause, 0)
+OPCODE(fnop, fnop, 0)
+OPCODE(fdecstp, fdecstp, 0)
+OPCODE(fincstp, fincstp, 0)
+OPCODE(fnclex, fnclex, 0)
+OPCODE(fninit, fninit, 0)
+OPCODE(sysret, sysret, 0)
+OPCODE(femms, femms, 0)
+OPCODE(swapgs, swapgs, X64_ONLY)
+OPCODE(vmcall, vmcall, X64_ONLY)
+OPCODE(vmlaunch, vmlaunch, X64_ONLY)
+OPCODE(vmresume, vmresume, X64_ONLY)
+OPCODE(vmxoff, vmxoff, X64_ONLY)
+OPCODE(fxam, fxam, 0)
+OPCODE(sahf, sahf, 0)
+OPCODE(mwait, mwait, 0)
+OPCODE(monitor, monitor, 0)
+OPCODE(fucompp, fucompp, 0)
+OPCODE(fcompp, fcompp, 0)
+OPCODE(lahf, lahf, 0)
+OPCODE(sysenter, sysenter, 0)
+OPCODE(sysexit, sysexit, 0)
+OPCODE(syscall, syscall, 0)
+OPCODE(daa, daa, X86_ONLY)
+OPCODE(das, das, X86_ONLY)
+OPCODE(aaa, aaa, X86_ONLY)
+OPCODE(aas, aas, X86_ONLY)
+OPCODE(cwde, cwde, 0)
+OPCODE(xlat, xlat, 0)
+OPCODE(fchs, fchs, 0)
+OPCODE(fabs, fabs, 0)
+OPCODE(ftst, ftst, 0)
+OPCODE(fld1, fld1, 0)
+OPCODE(fldl2t, fldl2t, 0)
+OPCODE(fldl2e, fldl2e, 0)
+OPCODE(fldpi, fldpi, 0)
+OPCODE(fldlg2, fldlg2, 0)
+OPCODE(fldln2, fldln2, 0)
+OPCODE(fldz, fldz, 0)
+OPCODE(f2xm1, f2xm1, 0)
+OPCODE(fptan, fptan, 0)
+OPCODE(fxtract, fxtract, 0)
+OPCODE(fsqrt, fsqrt, 0)
+OPCODE(fsincos, fsincos, 0)
+OPCODE(frndint, frndint, 0)
+OPCODE(fsin, fsin, 0)
+OPCODE(fcos, fcos, 0)
+OPCODE(fscale, fscale, 0)
+OPCODE(fyl2x, fyl2x, 0)
+OPCODE(fyl2xp1, fyl2xp1, 0)
+OPCODE(fpatan, fpatan, 0)
+OPCODE(fprem, fprem, 0)
+OPCODE(fprem1, fprem1, 0)
+OPCODE(popf, popf, 0)
+OPCODE(ret, ret, 0)
+OPCODE(ret_far, ret_far, 0)
+OPCODE(iret, iret, 0)
+OPCODE(rdtsc, rdtsc, 0)
+OPCODE(cdq, cdq, 0)
+OPCODE(pushf, pushf, 0)
+OPCODE(int3, int3, 0)
+OPCODE(into, into, X86_ONLY)
+OPCODE(rdmsr, rdmsr, 0)
+OPCODE(rdpmc, rdpmc, 0)
+OPCODE(leave, leave, 0)
+OPCODE(pusha, pusha, X86_ONLY)
+OPCODE(wrmsr, wrmsr, 0)
+OPCODE(cpuid, cpuid, 0)
+OPCODE(popa, popa, X86_ONLY)
+OPCODE(salc, salc, X86_ONLY)
+OPCODE(int1, int1, 0)
+
+OPCODE(LABEL, label, 0)
+OPCODE(out, out_1, 0)
+OPCODE(out, out_4, 0)
+OPCODE(in, in_1, 0)
+OPCODE(in, in_4, 0)
+OPCODE(ins, ins_1, 0)
+OPCODE(ins, ins_4, 0)
+OPCODE(stos, stos_1, 0)
+OPCODE(stos, stos_4, 0)
+OPCODE(lods, lods_1, 0)
+OPCODE(lods, lods_4, 0)
+OPCODE(movs, movs_1, 0)
+OPCODE(movs, movs_4, 0)
+OPCODE(rep_ins, rep_ins_1, 0)
+OPCODE(rep_ins, rep_ins_4, 0)
+OPCODE(rep_stos, rep_stos_1, 0)
+OPCODE(rep_stos, rep_stos_4, 0)
+OPCODE(rep_lods, rep_lods_1, 0)
+OPCODE(rep_lods, rep_lods_4, 0)
+OPCODE(rep_movs, rep_movs_1, 0)
+OPCODE(rep_movs, rep_movs_4, 0)
+OPCODE(outs, outs_1, 0)
+OPCODE(outs, outs_4, 0)
+OPCODE(cmps, cmps_1, 0)
+OPCODE(cmps, cmps_4, 0)
+OPCODE(scas, scas_1, 0)
+OPCODE(scas, scas_4, 0)
+OPCODE(rep_outs, rep_outs_1, 0)
+OPCODE(rep_outs, rep_outs_4, 0)
+OPCODE(rep_cmps, rep_cmps_1, 0)
+OPCODE(rep_cmps, rep_cmps_4, 0)
+OPCODE(repne_cmps, repne_cmps_1, 0)
+OPCODE(repne_cmps, repne_cmps_4, 0)
+OPCODE(rep_scas, rep_scas_1, 0)
+OPCODE(rep_scas, rep_scas_4, 0)
+OPCODE(repne_scas, repne_scas_1, 0)
+OPCODE(repne_scas, repne_scas_4, 0)
diff --git a/suite/tests/api/ir_1args.h b/suite/tests/api/ir_1args.h
new file mode 100644
index 000000000..005cf55ba
--- /dev/null
+++ b/suite/tests/api/ir_1args.h
@@ -0,0 +1,157 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/****************************************************************************/
+/* single memory argument */
+
+OPCODE(lldt, lldt, 0, MEMARG(OPSZ_2))
+OPCODE(ltr, ltr, 0, MEMARG(OPSZ_2))
+OPCODE(verr, verr, 0, MEMARG(OPSZ_2))
+OPCODE(verw, verw, 0, MEMARG(OPSZ_2))
+OPCODE(lgdt, lgdt, 0, MEMARG(OPSZ_lgdt))
+OPCODE(lidt, lidt, 0, MEMARG(OPSZ_lidt))
+OPCODE(lmsw, lmsw, 0, MEMARG(OPSZ_2))
+OPCODE(invlpg, invlpg, 0, MEMARG(OPSZ_lea))
+OPCODE(fxrstor, fxrstor, 0, MEMARG(OPSZ_fxrstor))
+OPCODE(ldmxcsr, ldmxcsr, 0, MEMARG(OPSZ_4))
+OPCODE(nop_modrm, nop_modrm, 0, MEMARG(OPSZ_4))
+OPCODE(prefetchnta, prefetchnta, 0, MEMARG(OPSZ_prefetch))
+OPCODE(prefetcht0, prefetcht0, 0, MEMARG(OPSZ_prefetch))
+OPCODE(prefetcht1, prefetcht1, 0, MEMARG(OPSZ_prefetch))
+OPCODE(prefetcht2, prefetcht2, 0, MEMARG(OPSZ_prefetch))
+OPCODE(prefetch, prefetch, 0, MEMARG(OPSZ_prefetch))
+OPCODE(prefetchw, prefetchw, 0, MEMARG(OPSZ_prefetch))
+OPCODE(clflush, clflush, 0, MEMARG(OPSZ_clflush))
+OPCODE(fldenv, fldenv, 0, MEMARG(OPSZ_fldenv))
+OPCODE(fldcw, fldcw, 0, MEMARG(OPSZ_2))
+OPCODE(frstor, frstor, 0, MEMARG(OPSZ_frstor))
+OPCODE(fcom, fcom, 0, MEMARG(OPSZ_8))
+OPCODE(fcomp, fcomp, 0, MEMARG(OPSZ_8))
+OPCODE(sldt, sldt, 0, MEMARG(OPSZ_2))
+OPCODE(str, str, 0, MEMARG(OPSZ_2))
+OPCODE(sgdt, sgdt, 0, MEMARG(OPSZ_sgdt))
+OPCODE(sidt, sidt, 0, MEMARG(OPSZ_sidt))
+OPCODE(smsw, smsw, 0, MEMARG(OPSZ_2))
+OPCODE(fxsave, fxsave, 0, MEMARG(OPSZ_fxsave))
+OPCODE(stmxcsr, stmxcsr, 0, MEMARG(OPSZ_4))
+OPCODE(fnstenv, fnstenv, 0, MEMARG(OPSZ_fnstenv))
+OPCODE(fnstcw, fnstcw, 0, MEMARG(OPSZ_2))
+OPCODE(fnsave, fnsave, 0, MEMARG(OPSZ_fnsave))
+OPCODE(fnstsw, fnstsw, 0, MEMARG(OPSZ_2))
+OPCODE(inc, inc, 0, MEMARG(OPSZ_4))
+OPCODE(dec, dec, 0, MEMARG(OPSZ_4))
+OPCODE(not, not, 0, MEMARG(OPSZ_4))
+OPCODE(neg, neg, 0, MEMARG(OPSZ_4))
+OPCODE(fst, fst, 0, MEMARG(OPSZ_4))
+OPCODE(fstp, fstp, 0, MEMARG(OPSZ_10))
+OPCODE(fld, fld, 0, MEMARG(OPSZ_10))
+OPCODE(fist, fist, 0, MEMARG(OPSZ_4))
+OPCODE(fistp, fistp, 0, MEMARG(OPSZ_4))
+OPCODE(fisttp, fisttp, 0, MEMARG(OPSZ_4))
+OPCODE(fbstp, fbstp, 0, MEMARG(OPSZ_10))
+OPCODE(fild, fild, 0, MEMARG(OPSZ_4))
+OPCODE(fbld, fbld, 0, MEMARG(OPSZ_10))
+OPCODE(fiadd, fiadd, 0, MEMARG(OPSZ_4))
+OPCODE(fimul, fimul, 0, MEMARG(OPSZ_4))
+OPCODE(fidiv, fidiv, 0, MEMARG(OPSZ_4))
+OPCODE(fidivr, fidivr, 0, MEMARG(OPSZ_4))
+OPCODE(fisub, fisub, 0, MEMARG(OPSZ_4))
+OPCODE(fisubr, fisubr, 0, MEMARG(OPSZ_4))
+OPCODE(ficom, ficom, 0, MEMARG(OPSZ_4))
+OPCODE(ficomp, ficomp, 0, MEMARG(OPSZ_4))
+OPCODE(imul, imul_1, 0, MEMARG(OPSZ_1))
+OPCODE(imul, imul_4, 0, MEMARG(OPSZ_4))
+OPCODE(mul, mul_1, 0, MEMARG(OPSZ_1))
+OPCODE(mul, mul_4, 0, MEMARG(OPSZ_4))
+OPCODE(div, div_1, 0, MEMARG(OPSZ_1))
+OPCODE(div, div_4, 0, MEMARG(OPSZ_4))
+OPCODE(idiv, idiv_1, 0, MEMARG(OPSZ_1))
+OPCODE(idiv, idiv_4, 0, MEMARG(OPSZ_4))
+OPCODE(pop, pop, 0, MEMARG(OPSZ_4x8))
+OPCODE(push, push, 0, MEMARG(OPSZ_4x8))
+OPCODE(cmpxchg8b, cmpxchg8b, 0, MEMARG(OPSZ_8))
+OPCODE(jmp_ind, jmp_ind, 0, MEMARG(OPSZ_4x8))
+OPCODE(call_ind, call_ind, 0, MEMARG(OPSZ_4x8))
+OPCODE(vmptrst, vmptrst, X64_ONLY, MEMARG(OPSZ_8))
+OPCODE(vmptrld, vmptrld, X64_ONLY, MEMARG(OPSZ_8))
+OPCODE(vmxon, vmxon, X64_ONLY, MEMARG(OPSZ_8))
+OPCODE(vmclear, vmclear, X64_ONLY, MEMARG(OPSZ_8))
+
+/****************************************************************************/
+/* single immed argument */
+
+OPCODE(out, out_1_imm, 0, IMMARG(OPSZ_1))
+OPCODE(out, out_4_imm, 0, IMMARG(OPSZ_1))
+OPCODE(in, in_1_imm, 0, IMMARG(OPSZ_1))
+OPCODE(in, in_4_imm, 0, IMMARG(OPSZ_1))
+OPCODE(aam, aam, X86_ONLY, IMMARG(OPSZ_1))
+OPCODE(aad, aad, X86_ONLY, IMMARG(OPSZ_1))
+OPCODE(ret, ret_imm, 0, IMMARG(OPSZ_2))
+OPCODE(ret_far, ret_far_imm, 0, IMMARG(OPSZ_2))
+OPCODE(push_imm, push_imm, 0, IMMARG(OPSZ_4))
+OPCODE(int, int, 0, IMMARG(OPSZ_1))
+
+/****************************************************************************/
+/* single register argument */
+
+OPCODE(bswap, bswap, 0, REGARG(EAX))
+OPCODE(fcomi, fcomi, 0, REGARG(ST0))
+OPCODE(fcomip, fcomip, 0, REGARG(ST0))
+OPCODE(fucomi, fucomi, 0, REGARG(ST0))
+OPCODE(fucomip, fucomip, 0, REGARG(ST0))
+OPCODE(fucom, fucom, 0, REGARG(ST0))
+OPCODE(fucomp, fucomp, 0, REGARG(ST0))
+OPCODE(ffree, ffree, 0, REGARG(ST0))
+OPCODE(faddp, faddp, 0, REGARG(ST0))
+OPCODE(fmulp, fmulp, 0, REGARG(ST0))
+OPCODE(fdivp, fdivp, 0, REGARG(ST0))
+OPCODE(fdivrp, fdivrp, 0, REGARG(ST0))
+OPCODE(fsubp, fsubp, 0, REGARG(ST0))
+OPCODE(fsubrp, fsubrp, 0, REGARG(ST0))
+OPCODE(fxch, fxch, 0, REGARG(ST0))
+OPCODE(ffreep, ffreep, 0, REGARG(ST0))
+
+/****************************************************************************/
+/* single pc/instr argument */
+
+OPCODE(jmp, jmp, 0, TGTARG)
+OPCODE(jmp_short, jmp_short, 0, TGTARG)
+OPCODE(jecxz, jecxz, 0, TGTARG)
+OPCODE(jcxz, jcxz, X86_ONLY, TGTARG)
+OPCODE(loopne, loopne, 0, TGTARG)
+OPCODE(loope, loope, 0, TGTARG)
+OPCODE(loop, loop, 0, TGTARG)
+OPCODE(jmp_far, jmp_far, X86_ONLY, opnd_create_far_pc(0x1234, 0))
+OPCODE(jmp_far_ind, jmp_far_ind, 0, MEMARG(OPSZ_6))
+OPCODE(call, call, 0, TGTARG)
+OPCODE(call_far, call_far, X86_ONLY, opnd_create_far_pc(0x1234, 0))
+OPCODE(call_far_ind, call_far_ind, 0, MEMARG(OPSZ_6))
diff --git a/suite/tests/api/ir_2args.h b/suite/tests/api/ir_2args.h
new file mode 100644
index 000000000..638664163
--- /dev/null
+++ b/suite/tests/api/ir_2args.h
@@ -0,0 +1,356 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+OPCODE(add, add, 0, REGARG(EAX), MEMARG(OPSZ_4))
+OPCODE(or, or, 0, REGARG(EAX), MEMARG(OPSZ_4))
+OPCODE(adc, adc, 0, REGARG(EAX), MEMARG(OPSZ_4))
+OPCODE(sbb, sbb, 0, REGARG(EAX), MEMARG(OPSZ_4))
+OPCODE(and, and, 0, REGARG(EAX), MEMARG(OPSZ_4))
+OPCODE(sub, sub, 0, REGARG(EAX), MEMARG(OPSZ_4))
+OPCODE(xor, xor, 0, REGARG(EAX), MEMARG(OPSZ_4))
+OPCODE(arpl, arpl, X86_ONLY, MEMARG(OPSZ_2), REGARG(AX))
+OPCODE(lea, lea, 0, REGARG(EAX), MEMARG(OPSZ_lea))
+OPCODE(mov_ld, mov_ld, 0, REGARG(EAX), MEMARG(OPSZ_4))
+OPCODE(mov_st, mov_st, 0, MEMARG(OPSZ_4), REGARG(EAX))
+OPCODE(mov_imm, mov_imm, 0, REGARG(EAX), IMMARG(OPSZ_4))
+OPCODE(mov_seg, mov_seg, 0, REGARG(EAX), opnd_create_reg(SEG_FS))
+OPCODE(mov_priv, mov_priv, 0, REGARG(XAX), REGARG(CR0))
+OPCODE(lar, lar, 0, REGARG(EAX), MEMARG(OPSZ_2))
+OPCODE(lsl, lsl, 0, REGARG(EAX), MEMARG(OPSZ_2))
+OPCODE(movd, movd, 0, REGARG(MM0), MEMARG(OPSZ_4))
+OPCODE(movq, movq, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(movdqu, movdqu, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(movdqa, movdqa, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(movzx, movzx, 0, REGARG(EAX), MEMARG(OPSZ_2))
+OPCODE(movsx, movsx, 0, REGARG(EAX), MEMARG(OPSZ_2))
+OPCODE(bsf, bsf, 0, REGARG(EAX), MEMARG(OPSZ_4))
+OPCODE(bsr, bsr, 0, REGARG(EAX), MEMARG(OPSZ_4))
+
+OPCODE(imul, imul, 0, REGARG(EAX), MEMARG(OPSZ_4))
+OPCODE(lds, lds, X86_ONLY, REGARG(EAX), MEMARG(OPSZ_6_irex10_short4))
+OPCODE(lss, lss, 0, REGARG(EAX), MEMARG(OPSZ_6_irex10_short4))
+OPCODE(les, les, X86_ONLY, REGARG(EAX), MEMARG(OPSZ_6_irex10_short4))
+OPCODE(lfs, lfs, 0, REGARG(EAX), MEMARG(OPSZ_6_irex10_short4))
+OPCODE(lgs, lgs, 0, REGARG(EAX), MEMARG(OPSZ_6_irex10_short4))
+OPCODE(xchg, xchg, 0, MEMARG(OPSZ_4), REGARG(EAX))
+OPCODE(xadd, xadd, 0, MEMARG(OPSZ_4), REGARG(EAX))
+OPCODE(cmpxchg, cmpxchg_1, 0, MEMARG(OPSZ_1), REGARG(AL))
+OPCODE(cmpxchg, cmpxchg_4, 0, MEMARG(OPSZ_4), REGARG(EAX))
+OPCODE(cmp, cmp, 0, REGARG(EAX), MEMARG(OPSZ_4))
+OPCODE(test, test, 0, MEMARG(OPSZ_4), REGARG(EAX))
+OPCODE(bound, bound, X86_ONLY, REGARG(EAX), MEMARG(OPSZ_bound))
+
+OPCODE(setnz, setcc, 0, OP_setnz, REGARG(AL))
+
+OPCODE(bts, bts, 0, MEMARG(OPSZ_4), REGARG(EAX))
+OPCODE(btr, btr, 0, MEMARG(OPSZ_4), REGARG(EAX))
+OPCODE(btc, btc, 0, MEMARG(OPSZ_4), REGARG(EAX))
+
+OPCODE(rol, rol, 0, MEMARG(OPSZ_4), IMMARG(OPSZ_1))
+OPCODE(ror, ror, 0, MEMARG(OPSZ_4), IMMARG(OPSZ_1))
+OPCODE(rcl, rcl, 0, MEMARG(OPSZ_4), IMMARG(OPSZ_1))
+OPCODE(rcr, rcr, 0, MEMARG(OPSZ_4), IMMARG(OPSZ_1))
+OPCODE(shl, shl, 0, MEMARG(OPSZ_4), IMMARG(OPSZ_1))
+OPCODE(shr, shr, 0, MEMARG(OPSZ_4), IMMARG(OPSZ_1))
+OPCODE(sar, sar, 0, MEMARG(OPSZ_4), IMMARG(OPSZ_1))
+
+OPCODE(pmovmskb, pmovmskb, 0, REGARG(EAX), REGARG(MM0))
+OPCODE(ptest, ptest, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(movups, movups, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(movss, movss, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(movupd, movupd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(movsd, movsd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+
+OPCODE(movlps, movlps, 0, REGARG(XMM0), MEMARG(OPSZ_8))
+OPCODE(movlpd, movlpd, 0, REGARG(XMM0), MEMARG(OPSZ_8))
+OPCODE(movhps, movhps, 0, REGARG(XMM0), MEMARG(OPSZ_8))
+OPCODE(movhpd, movhpd, 0, REGARG(XMM0), MEMARG(OPSZ_8))
+
+OPCODE(movaps, movaps, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(movapd, movapd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(cvtpi2ps, cvtpi2ps, 0, REGARG(XMM0), MEMARG(OPSZ_8))
+OPCODE(cvtsi2ss, cvtsi2ss, 0, REGARG(XMM0), MEMARG(OPSZ_4))
+OPCODE(cvtpi2pd, cvtpi2pd, 0, REGARG(XMM0), MEMARG(OPSZ_8))
+OPCODE(cvtsi2sd, cvtsi2sd, 0, REGARG(XMM0), MEMARG(OPSZ_4))
+OPCODE(cvttps2pi, cvttps2pi, 0, REGARG(MM0), MEMARG(OPSZ_16))
+OPCODE(cvttss2si, cvttss2si, 0, REGARG(EAX), MEMARG(OPSZ_16))
+OPCODE(cvttpd2pi, cvttpd2pi, 0, REGARG(MM0), MEMARG(OPSZ_16))
+OPCODE(cvttsd2si, cvttsd2si, 0, REGARG(EAX), MEMARG(OPSZ_16))
+OPCODE(cvtps2pi, cvtps2pi, 0, REGARG(MM0), MEMARG(OPSZ_16))
+OPCODE(cvtss2si, cvtss2si, 0, REGARG(EAX), MEMARG(OPSZ_16))
+OPCODE(cvtpd2pi, cvtpd2pi, 0, REGARG(MM0), MEMARG(OPSZ_16))
+OPCODE(cvtsd2si, cvtsd2si, 0, REGARG(EAX), MEMARG(OPSZ_16))
+OPCODE(cvtps2pd, cvtps2pd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(cvtss2sd, cvtss2sd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(cvtpd2ps, cvtpd2ps, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(cvtsd2ss, cvtsd2ss, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(cvtdq2ps, cvtdq2ps, 0, REGARG(XMM0), MEMARG(OPSZ_8))
+OPCODE(cvttps2dq, cvttps2dq, 0, REGARG(MM0), MEMARG(OPSZ_16))
+OPCODE(cvtps2dq, cvtps2dq, 0, REGARG(MM0), MEMARG(OPSZ_16))
+OPCODE(cvtdq2pd, cvtdq2pd, 0, REGARG(XMM0), MEMARG(OPSZ_8))
+OPCODE(cvttpd2dq, cvttpd2dq, 0, REGARG(MM0), MEMARG(OPSZ_16))
+OPCODE(cvtpd2dq, cvtpd2dq, 0, REGARG(MM0), MEMARG(OPSZ_16))
+
+OPCODE(movmskps, movmskps, 0, REGARG(EAX), REGARG(XMM0))
+OPCODE(movmskpd, movmskpd, 0, REGARG(EAX), REGARG(XMM0))
+OPCODE(movddup, movddup, 0, REGARG(XMM0), MEMARG(OPSZ_8))
+OPCODE(popcnt, popcnt, 0, REGARG(EAX), MEMARG(OPSZ_4))
+OPCODE(lzcnt, lzcnt, 0, REGARG(EAX), MEMARG(OPSZ_4))
+
+OPCODE(movntss, movntss, 0, MEMARG(OPSZ_16), REGARG(XMM0))
+OPCODE(movntsd, movntsd, 0, MEMARG(OPSZ_16), REGARG(XMM0))
+OPCODE(movntps, movntps, 0, MEMARG(OPSZ_16), REGARG(XMM0))
+OPCODE(movntpd, movntpd, 0, MEMARG(OPSZ_16), REGARG(XMM0))
+OPCODE(movntdqa, movntdqa, 0, MEMARG(OPSZ_16), REGARG(XMM0))
+OPCODE(movntq, movntq, 0, MEMARG(OPSZ_8), REGARG(MM0))
+OPCODE(movntdq, movntdq, 0, MEMARG(OPSZ_16), REGARG(XMM0))
+OPCODE(movnti, movnti, 0, MEMARG(OPSZ_4), REGARG(EAX))
+
+OPCODE(sqrtps, sqrtps, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(sqrtss, sqrtss, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(sqrtpd, sqrtpd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(sqrtsd, sqrtsd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(rsqrtps, rsqrtps, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(rsqrtss, rsqrtss, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(rcpps, rcpps, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(rcpss, rcpss, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(lddqu, lddqu, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(movsldup, movsldup, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(movshdup, movshdup, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pshufb, pshufb, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmovsxbw, pmovsxbw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmovsxbd, pmovsxbd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmovsxbq, pmovsxbq, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmovsxdw, pmovsxdw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmovsxwq, pmovsxwq, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmovsxdq, pmovsxdq, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmovzxbw, pmovzxbw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmovzxbd, pmovzxbd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmovzxbq, pmovzxbq, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmovzxdw, pmovzxdw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmovzxwq, pmovzxwq, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmovzxdq, pmovzxdq, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(phminposuw, phminposuw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(punpcklbw, punpcklbw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(punpcklwd, punpcklwd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(punpckldq, punpckldq, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(packsswb, packsswb, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pcmpgtb, pcmpgtb, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pcmpgtw, pcmpgtw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pcmpgtd, pcmpgtd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(packuswb, packuswb, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(punpckhbw, punpckhbw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(punpckhwd, punpckhwd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(punpckhdq, punpckhdq, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(packssdw, packssdw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(punpcklqdq, punpcklqdq, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(punpckhqdq, punpckhqdq, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pcmpeqb, pcmpeqb, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pcmpeqw, pcmpeqw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pcmpeqd, pcmpeqd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+
+OPCODE(psrlw, psrlw, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(psrld, psrld, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(psrlq, psrlq, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(paddq, paddq, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pmullw, pmullw, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(psubusb, psubusb, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(psubusw, psubusw, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pminub, pminub, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pand, pand, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(paddusb, paddusb, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(paddusw, paddusw, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pmaxub, pmaxub, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pandn, pandn, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pavgb, pavgb, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(psraw, psraw, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(psrad, psrad, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pavgw, pavgw, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pmulhuw, pmulhuw, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pmulhw, pmulhw, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(psubsb, psubsb, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(psubsw, psubsw, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pminsw, pminsw, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(por, por, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(paddsb, paddsb, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(paddsw, paddsw, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pmaxsw, pmaxsw, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pxor, pxor, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(psllw, psllw, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pslld, pslld, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(psllq, psllq, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pmuludq, pmuludq, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pmaddwd, pmaddwd, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(psadbw, psadbw, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(psubb, psubb, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(psubw, psubw, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(psubd, psubd, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(psubq, psubq, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(paddb, paddb, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(paddw, paddw, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(paddd, paddd, 0, REGARG(MM0), MEMARG(OPSZ_8))
+
+OPCODE(psrldq, psrldq, 0, REGARG(XMM0), IMMARG(OPSZ_1))
+OPCODE(pslldq, pslldq, 0, REGARG(XMM0), IMMARG(OPSZ_1))
+
+OPCODE(unpcklps, unpcklps, 0, REGARG(XMM0), MEMARG(OPSZ_8))
+OPCODE(unpcklpd, unpcklpd, 0, REGARG(XMM0), MEMARG(OPSZ_8))
+OPCODE(unpckhps, unpckhps, 0, REGARG(XMM0), MEMARG(OPSZ_8))
+OPCODE(unpckhpd, unpckhpd, 0, REGARG(XMM0), MEMARG(OPSZ_8))
+
+OPCODE(andps, andps, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(andpd, andpd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(andnps, andnps, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(andnpd, andnpd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(orps, orps, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(orpd, orpd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(xorps, xorps, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(xorpd, xorpd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(addps, addps, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(addss, addss, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(addpd, addpd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(addsd, addsd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(mulps, mulps, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(mulss, mulss, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(mulpd, mulpd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(mulsd, mulsd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(subps, subps, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(subss, subss, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(subpd, subpd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(subsd, subsd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(minps, minps, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(minss, minss, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(minpd, minpd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(minsd, minsd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(divps, divps, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(divss, divss, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(divpd, divpd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(divsd, divsd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(maxps, maxps, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(maxss, maxss, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(maxpd, maxpd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(maxsd, maxsd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(haddpd, haddpd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(haddps, haddps, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(hsubpd, hsubpd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(hsubps, hsubps, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(addsubpd, addsubpd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(addsubps, addsubps, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+
+OPCODE(phaddw, phaddw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(phaddd, phaddd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(phaddsw, phaddsw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmaddubsw, pmaddubsw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(phsubw, phsubw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(phsubd, phsubd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(phsubsw, phsubsw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(psignb, psignb, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(psignw, psignw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(psignd, psignd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmulhrsw, pmulhrsw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pabsb, pabsb, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pabsw, pabsw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pabsd, pabsd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pblendvb, pblendvb, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(blendvps, blendvps, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(blendvpd, blendvpd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(packusdw, packusdw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pcmpeqq, pcmpeqq, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pcmpgtq, pcmpgtq, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pminsb, pminsb, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pminsd, pminsd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pminuw, pminuw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pminud, pminud, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmaxsb, pmaxsb, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmaxsd, pmaxsd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmaxuw, pmaxuw, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmaxud, pmaxud, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmuldq, pmuldq, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(pmulld, pmulld, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+
+OPCODE(ucomiss, ucomiss, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(ucomisd, ucomisd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(comiss, comiss, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+OPCODE(comisd, comisd, 0, REGARG(XMM0), MEMARG(OPSZ_16))
+
+OPCODE(crc32, crc32, 0, REGARG(EAX), MEMARG(OPSZ_4))
+
+OPCODE(vmread, vmread, X64_ONLY, MEMARG(OPSZ_4), REGARG(EAX))
+OPCODE(vmwrite, vmwrite, X64_ONLY, REGARG(EAX), MEMARG(OPSZ_4))
+OPCODE(movsxd, movsxd, X64_ONLY, REGARG(RAX), MEMARG(OPSZ_4))
+
+OPCODE(fadd, fadd, 0, REGARG(ST0), MEMARG(OPSZ_4))
+OPCODE(fmul, fmul, 0, REGARG(ST0), MEMARG(OPSZ_4))
+OPCODE(fdiv, fdiv, 0, REGARG(ST0), MEMARG(OPSZ_4))
+OPCODE(fdivr, fdivr, 0, REGARG(ST0), MEMARG(OPSZ_4))
+OPCODE(fsub, fsub, 0, REGARG(ST0), MEMARG(OPSZ_4))
+OPCODE(fsubr, fsubr, 0, REGARG(ST0), MEMARG(OPSZ_4))
+OPCODE(fcmovbe, fcmovcc, 0, OP_fcmovbe, REGARG(ST0))
+
+OPCODE(extrq, extrq, 0, REGARG(XMM0), REGARG(XMM1))
+OPCODE(insertq, insertq, 0, REGARG(XMM0), REGARG(XMM1))
+OPCODE(bt, bt, 0, MEMARG(OPSZ_4), REGARG(EAX))
+
+OPCODE(maskmovq, maskmovq, 0, REGARG(MM0), REGARG(MM0))
+OPCODE(maskmovdqu, maskmovdqu, 0, REGARG(XMM0), REGARG(XMM0))
+
+OPCODE(enter, enter, 0, IMMARG(OPSZ_2), IMMARG(OPSZ_1))
+
+OPCODE(jnz, jcc, 0, OP_jnz, TGTARG)
+OPCODE(jno_short, jcc_short, 0, OP_jno_short, TGTARG)
+
+/****************************************************************************/
+/* 3D Now */
+OPCODE(pavgusb, pavgusb, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pfadd, pfadd, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pfacc, pfacc, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pfcmpge, pfcmpge, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pfcmpgt, pfcmpgt, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pfcmpeq, pfcmpeq, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pfmin, pfmin, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pfmax, pfmax, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pfmul, pfmul, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pfrcp, pfrcp, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pfrcpit1, pfrcpit1, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pfrcpit2, pfrcpit2, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pfrsqrt, pfrsqrt, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pfrsqit1, pfrsqit1, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pmulhrw, pmulhrw, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pfsub, pfsub, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pfsubr, pfsubr, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pi2fd, pi2fd, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pf2id, pf2id, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pi2fw, pi2fw, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pf2iw, pf2iw, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pfnacc, pfnacc, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pfpnacc, pfpnacc, 0, REGARG(MM0), MEMARG(OPSZ_8))
+OPCODE(pswapd, pswapd, 0, REGARG(MM0), MEMARG(OPSZ_8))
diff --git a/suite/tests/api/ir_3args.h b/suite/tests/api/ir_3args.h
new file mode 100644
index 000000000..ab2612251
--- /dev/null
+++ b/suite/tests/api/ir_3args.h
@@ -0,0 +1,75 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+OPCODE(cmovae, cmovcc, 0, OP_cmovae, REGARG(EAX), MEMARG(OPSZ_4))
+OPCODE(imul, imul_imm, 0, REGARG(EAX), MEMARG(OPSZ_4), IMMARG(OPSZ_4))
+
+OPCODE(extrq, extrq_imm, 0, REGARG(XMM0), IMMARG(OPSZ_1), IMMARG(OPSZ_1))
+
+OPCODE(shld, shld, 0, MEMARG(OPSZ_4), REGARG(EAX), IMMARG(OPSZ_1))
+OPCODE(shrd, shrd, 0, MEMARG(OPSZ_4), REGARG(EAX), IMMARG(OPSZ_1))
+
+OPCODE(pshufw, pshufw, 0, REGARG(MM0), MEMARG(OPSZ_8), IMMARG(OPSZ_1))
+OPCODE(pshufd, pshufd, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+OPCODE(pshufhw, pshufhw, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+OPCODE(pshuflw, pshuflw, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+OPCODE(pinsrw, pinsrw, 0, REGARG(XMM0), MEMARG(OPSZ_4), IMMARG(OPSZ_1))
+OPCODE(pextrw, pextrw, 0, REGARG(EAX), REGARG(XMM0), IMMARG(OPSZ_1))
+OPCODE(pextrb, pextrb, 0, REGARG(EAX), REGARG(XMM0), IMMARG(OPSZ_1))
+OPCODE(pextrd, pextrd, 0, REGARG(EAX), REGARG(XMM0), IMMARG(OPSZ_1))
+OPCODE(extractps, extractps, 0, REGARG(EAX), REGARG(XMM0), IMMARG(OPSZ_1))
+OPCODE(roundps, roundps, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+OPCODE(roundpd, roundpd, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+OPCODE(roundss, roundss, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+OPCODE(roundsd, roundsd, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+OPCODE(blendps, blendps, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+OPCODE(blendpd, blendpd, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+OPCODE(pblendw, pblendw, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+OPCODE(pinsrb, pinsrb, 0, REGARG(XMM0), MEMARG(OPSZ_1), IMMARG(OPSZ_1))
+OPCODE(insertps, insertps, 0, REGARG(XMM0), REGARG(XMM1), IMMARG(OPSZ_1))
+OPCODE(pinsrd, pinsrd, 0, REGARG(XMM0), MEMARG(OPSZ_4), IMMARG(OPSZ_1))
+OPCODE(shufps, shufps, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+OPCODE(shufpd, shufpd, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+OPCODE(cmpps, cmpps, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+OPCODE(cmpss, cmpss, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+OPCODE(cmppd, cmppd, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+OPCODE(cmpsd, cmpsd, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+OPCODE(palignr, palignr, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+
+OPCODE(dpps, dpps, 0, REGARG(XMM0), MEMARG(OPSZ_1), IMMARG(OPSZ_1))
+OPCODE(dppd, dppd, 0, REGARG(XMM0), MEMARG(OPSZ_1), IMMARG(OPSZ_1))
+OPCODE(mpsadbw, mpsadbw, 0, REGARG(XMM0), MEMARG(OPSZ_1), IMMARG(OPSZ_1))
+
+OPCODE(pcmpistrm, pcmpistrm, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+OPCODE(pcmpistri, pcmpistri, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+OPCODE(pcmpestrm, pcmpestrm, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
+OPCODE(pcmpestri, pcmpestri, 0, REGARG(XMM0), MEMARG(OPSZ_16), IMMARG(OPSZ_1))
diff --git a/suite/tests/api/ir_4args.h b/suite/tests/api/ir_4args.h
new file mode 100644
index 000000000..2fd44eb7e
--- /dev/null
+++ b/suite/tests/api/ir_4args.h
@@ -0,0 +1,34 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+OPCODE(insertq, insertq_imm, 0, REGARG(XMM0), REGARG(XMM1), IMMARG(OPSZ_1), 
+       IMMARG(OPSZ_1))
diff --git a/suite/tests/api/startstop.c b/suite/tests/api/startstop.c
new file mode 100644
index 000000000..33e1f729e
--- /dev/null
+++ b/suite/tests/api/startstop.c
@@ -0,0 +1,77 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <math.h>
+#ifdef USE_DYNAMO
+#include "dr_api.h"
+#endif
+
+#define ITERS 150000
+
+void foo() 
+{
+}
+
+main()
+{
+    double res = 0.;
+    int i,j;
+#ifdef USE_DYNAMO
+    dr_app_setup();
+#endif
+    
+    for (j=0; j<10; j++) {
+#ifdef USE_DYNAMO
+        dr_app_start();
+#endif
+        for (i=0; i<ITERS; i++) {
+            if (i % 2 == 0) {
+                res += cos(1./(double)(i+1));
+            } else {
+                res += sin(1./(double)(i+1));
+	    }
+	}
+	foo();
+#ifdef USE_DYNAMO
+	dr_app_stop();
+#endif
+    }
+    /* PR : we get different floating point results on different platforms,
+     * so we no longer print out res */
+    printf("all done: %d iters\n", i);
+    
+#ifdef USE_DYNAMO
+    dr_app_cleanup();
+#endif
+}
diff --git a/suite/tests/api/startstop.expect b/suite/tests/api/startstop.expect
new file mode 100644
index 000000000..9025bc851
--- /dev/null
+++ b/suite/tests/api/startstop.expect
@@ -0,0 +1 @@
+all done: 150000 iters
diff --git a/suite/tests/client-interface/abort.c b/suite/tests/client-interface/abort.c
new file mode 100644
index 000000000..77d3c9f51
--- /dev/null
+++ b/suite/tests/client-interface/abort.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <stdio.h>
+
+int main()
+{
+    printf("ERROR: this shouldn't print\n");
+}
diff --git a/suite/tests/client-interface/abort.dll.c b/suite/tests/client-interface/abort.dll.c
new file mode 100644
index 000000000..5f1f976ae
--- /dev/null
+++ b/suite/tests/client-interface/abort.dll.c
@@ -0,0 +1,64 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+
+static
+void my_abort(app_pc tag)
+{
+    dr_printf("aborting now\n");
+    dr_abort();
+}
+
+#define MINSERT instrlist_meta_preinsert
+
+static
+dr_emit_flags_t bb_event(void* drcontext, void *tag, instrlist_t* bb, bool for_trace, bool translating)
+{
+    instr_t* instr = instrlist_first(bb);
+
+    dr_prepare_for_call(drcontext, bb, instr);
+
+    MINSERT(bb, instr, INSTR_CREATE_call
+            (drcontext, opnd_create_pc((void*)my_abort)));
+        
+    dr_cleanup_after_call(drcontext, bb, instr, 0);
+
+    return DR_EMIT_DEFAULT;
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    dr_register_bb_event(bb_event);
+}
+
diff --git a/suite/tests/client-interface/abort.expect b/suite/tests/client-interface/abort.expect
new file mode 100644
index 000000000..d3776c8e7
--- /dev/null
+++ b/suite/tests/client-interface/abort.expect
@@ -0,0 +1 @@
+aborting now
diff --git a/suite/tests/client-interface/alloc.c b/suite/tests/client-interface/alloc.c
new file mode 100644
index 000000000..ff57e685c
--- /dev/null
+++ b/suite/tests/client-interface/alloc.c
@@ -0,0 +1,36 @@
+/* **********************************************************
+ * Copyright (c) 2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+int main()
+{
+    return 0;
+}
diff --git a/suite/tests/client-interface/alloc.dll.c b/suite/tests/client-interface/alloc.dll.c
new file mode 100644
index 000000000..6c256c103
--- /dev/null
+++ b/suite/tests/client-interface/alloc.dll.c
@@ -0,0 +1,155 @@
+/* **********************************************************
+ * Copyright (c) 2007-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+
+char *global;
+#define SIZE 10
+#define VAL 17
+
+static
+void write_array(char *array)
+{
+    int i;
+    for (i=0; i<SIZE; i++)
+        array[i] = VAL;
+}
+
+static
+void global_test(void)
+{
+    char *array;
+    uint prot;
+    dr_fprintf(STDERR, "  testing global memory alloc...");
+    array = dr_global_alloc(SIZE);
+    write_array(array);
+    dr_query_memory((const byte *)array, NULL, NULL, &prot);
+    if (prot != (DR_MEMPROT_READ|DR_MEMPROT_WRITE))
+        dr_fprintf(STDERR, "[error: prot %d doesn't match rw] ", prot);
+    dr_global_free(array, SIZE);
+    dr_fprintf(STDERR, "success\n");
+}
+
+/* FIXME: 32-bit apps on my 64-bit linux machine end up with +x for
+ * all of these regardless of what's allocated up front or what's
+ * mprotected: DR is doing the right thing, the OS is adding it.
+ */
+static
+void nonheap_test(void)
+{
+    uint prot;
+    char *array =
+        dr_nonheap_alloc(SIZE, DR_MEMPROT_READ|DR_MEMPROT_WRITE|DR_MEMPROT_EXEC);
+    dr_fprintf(STDERR, "  testing nonheap memory alloc...");
+    write_array(array);
+    dr_query_memory((const byte *)array, NULL, NULL, &prot);
+    if (prot != (DR_MEMPROT_READ|DR_MEMPROT_WRITE|DR_MEMPROT_EXEC))
+        dr_fprintf(STDERR, "[error: prot %d doesn't match rwx] ", prot);
+    dr_memory_protect(array, SIZE, DR_MEMPROT_NONE);
+    dr_query_memory((const byte *)array, NULL, NULL, &prot);
+    if (prot != DR_MEMPROT_NONE)
+        dr_fprintf(STDERR, "[error: prot %d doesn't match r] ", prot);
+    dr_memory_protect(array, SIZE, DR_MEMPROT_READ);
+    dr_query_memory((const byte *)array, NULL, NULL, &prot);
+    if (prot != DR_MEMPROT_READ)
+        dr_fprintf(STDERR, "[error: prot %d doesn't match r] ", prot);
+    if (dr_safe_write(array, 1, (const void *) &prot, NULL))
+        dr_fprintf(STDERR, "[error: should not be writable] ");
+    dr_nonheap_free(array, SIZE);
+    dr_fprintf(STDERR, "success\n");
+}
+
+static
+void local_test(void *drcontext)
+{
+    char *array;
+    dr_fprintf(STDERR, "  testing local memory alloc....");
+    array = dr_thread_alloc(drcontext, SIZE);
+    write_array(array);
+    dr_thread_free(drcontext, array, SIZE);
+    dr_fprintf(STDERR, "success\n");
+}
+
+static
+void thread_init_event(void *drcontext)
+{
+    static bool tested = false;
+    if (!tested) {
+        dr_fprintf(STDERR, "thread initialization:\n");
+        local_test(drcontext);
+        global_test();
+        tested = true;
+    }
+}
+
+static
+void inline_alloc_test(void)
+{
+    dr_fprintf(STDERR, "code cache:\n");
+    local_test(dr_get_current_drcontext());
+    global_test();
+    nonheap_test();
+}
+
+#define MINSERT instrlist_meta_preinsert
+
+static
+dr_emit_flags_t bb_event(void* drcontext, void *tag, instrlist_t* bb, bool for_trace, bool translating)
+{
+    static bool inserted = false;
+    if (!inserted) {
+        instr_t* instr = instrlist_first(bb);
+
+        dr_prepare_for_call(drcontext, bb, instr);
+
+        MINSERT(bb, instr, INSTR_CREATE_call
+                (drcontext, opnd_create_pc((void*)inline_alloc_test)));
+        
+        dr_cleanup_after_call(drcontext, bb, instr, 0);
+
+        inserted = true;
+    }
+
+    /* store, since we're not deterministic */
+    return DR_EMIT_STORE_TRANSLATIONS;
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    dr_fprintf(STDERR, "thank you for testing the client interface\n");
+    global_test();
+    nonheap_test();
+
+    dr_register_bb_event(bb_event);
+    dr_register_thread_init_event(thread_init_event);
+}
diff --git a/suite/tests/client-interface/alloc.expect b/suite/tests/client-interface/alloc.expect
new file mode 100644
index 000000000..2567a8a2a
--- /dev/null
+++ b/suite/tests/client-interface/alloc.expect
@@ -0,0 +1,10 @@
+thank you for testing the client interface
+  testing global memory alloc...success
+  testing nonheap memory alloc...success
+thread initialization:
+  testing local memory alloc....success
+  testing global memory alloc...success
+code cache:
+  testing local memory alloc....success
+  testing global memory alloc...success
+  testing nonheap memory alloc...success
diff --git a/suite/tests/client-interface/call-retarget.c b/suite/tests/client-interface/call-retarget.c
new file mode 100644
index 000000000..2c4fb8b70
--- /dev/null
+++ b/suite/tests/client-interface/call-retarget.c
@@ -0,0 +1,64 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <stdio.h>
+
+#ifdef WINDOWS
+# define NOP __nop()
+#else /* LINUX */
+# define NOP asm("nop")
+#endif
+
+static void foo()
+{
+    printf("called foo()\n");
+}
+
+static void bar()
+{
+    printf("called bar()\n");
+}
+
+int main()
+{
+    /* Kind of a hack, but seems to work: Use a nop to mark a call
+     * instruction whose target address we can steal, and another nop
+     * to mark the instruction we want to retarget.  We recognize 2 NOPS in
+     * row followed by a direct call (on Linux some libc code has 
+     * nop; call direct; already).
+     */
+    NOP; NOP;
+    foo();
+    NOP; NOP;
+    bar();
+    return 0;
+}
diff --git a/suite/tests/client-interface/call-retarget.dll.c b/suite/tests/client-interface/call-retarget.dll.c
new file mode 100644
index 000000000..062a67a9c
--- /dev/null
+++ b/suite/tests/client-interface/call-retarget.dll.c
@@ -0,0 +1,78 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+
+static
+dr_emit_flags_t bb_event(void *drcontext, void* tag, instrlist_t *bb, bool for_trace, bool translating)
+{
+    instr_t *instr, *next_instr, *next_next_instr;
+    static app_pc target = NULL;
+    
+    /* Look for pattern: nop; nop; call direct; */
+    for (instr = instrlist_first(bb);
+         instr != NULL; instr = next_instr) {
+        next_instr = instr_get_next(instr);
+        if (next_instr != NULL)
+            next_next_instr = instr_get_next(next_instr);
+        else
+            next_next_instr = NULL;
+
+        if (instr_is_nop(instr) && 
+            next_instr != NULL && instr_is_nop(next_instr) && 
+            instr_is_call_direct(next_next_instr)) {
+
+            /* The first nop followed by a direct call is a marker
+             * for the function we want to call.
+             */
+            if (target == NULL) {
+                target = instr_get_branch_target_pc(next_next_instr);
+            }
+            /* The second nop followd by a direct call is a marker
+             * for the call we want to modify.
+             */
+            else {
+                instr_set_branch_target_pc(next_next_instr, target);
+            }
+
+            break;
+        }
+    }
+
+    return DR_EMIT_DEFAULT;
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    dr_register_bb_event(bb_event);
+}
diff --git a/suite/tests/client-interface/call-retarget.expect b/suite/tests/client-interface/call-retarget.expect
new file mode 100644
index 000000000..3da52859a
--- /dev/null
+++ b/suite/tests/client-interface/call-retarget.expect
@@ -0,0 +1,2 @@
+called foo()
+called foo()
diff --git a/suite/tests/client-interface/cbr.dll.c b/suite/tests/client-interface/cbr.dll.c
new file mode 100644
index 000000000..7515ea08a
--- /dev/null
+++ b/suite/tests/client-interface/cbr.dll.c
@@ -0,0 +1,109 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+
+static
+void thread_init_event(void *drcontext)
+{
+    dr_set_tls_field(drcontext, NULL);
+}
+
+static
+void at_cbr(app_pc inst_addr, app_pc targ_addr, int taken) 
+{
+    void *drcontext = dr_get_current_drcontext();
+    
+    if (taken == 1) {
+        dr_set_tls_field(drcontext, targ_addr);
+    }
+    else if (taken == 0) {
+        app_pc fall_addr = decode_next_pc(drcontext, inst_addr);
+        dr_set_tls_field(drcontext, fall_addr);
+    }
+    else {
+        dr_printf("ERROR: expecting 'taken' to be 0 or 1\n");
+    }
+}
+
+static
+void at_bb(void *drcontext, app_pc bb_addr)
+{
+    app_pc cbr_addr = dr_get_tls_field(drcontext);
+
+    if (cbr_addr != NULL && cbr_addr != bb_addr) {
+        dr_printf("ERROR: expected jmp to "PFX", but entered BB at "PFX"\n",
+                  cbr_addr, bb_addr);
+    }
+
+    dr_set_tls_field(drcontext, NULL);
+}
+
+#define MINSERT instrlist_meta_preinsert
+
+static
+dr_emit_flags_t bb_event(void *drcontext, void *tag, instrlist_t *bb, bool for_trace, bool translating)
+{
+    instr_t *instr, *next_instr;
+
+    app_pc bb_addr = dr_fragment_app_pc(tag);
+
+    instr = instrlist_first(bb);
+
+    dr_prepare_for_call(drcontext, bb, instr);
+
+    MINSERT(bb, instr, INSTR_CREATE_push_imm
+            (drcontext, OPND_CREATE_INT32((ptr_uint_t)bb_addr)));
+    MINSERT(bb, instr, INSTR_CREATE_push_imm
+            (drcontext, OPND_CREATE_INT32((ptr_uint_t)drcontext)));
+    MINSERT(bb, instr, INSTR_CREATE_call
+            (drcontext, opnd_create_pc((void*)at_bb)));
+
+    dr_cleanup_after_call(drcontext, bb, instr, 8);
+
+    for (; instr != NULL; instr = next_instr) {
+        next_instr = instr_get_next(instr);
+
+        if (instr_is_cbr(instr))
+            dr_insert_cbr_instrumentation(drcontext, bb, instr, (app_pc)at_cbr); 
+    }
+
+    return DR_EMIT_DEFAULT;
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    dr_printf("thank you for testing the client interface\n");
+    dr_register_bb_event(bb_event);
+    dr_register_thread_init_event(thread_init_event);
+}
diff --git a/suite/tests/client-interface/cbr.runall b/suite/tests/client-interface/cbr.runall
new file mode 100644
index 000000000..787417df8
--- /dev/null
+++ b/suite/tests/client-interface/cbr.runall
@@ -0,0 +1,3 @@
+notepad.exe
+Untitled - Notepad
+<>
diff --git a/suite/tests/client-interface/cbr.template b/suite/tests/client-interface/cbr.template
new file mode 100644
index 000000000..a203ad3b1
--- /dev/null
+++ b/suite/tests/client-interface/cbr.template
@@ -0,0 +1,18 @@
+thank you for testing the client interface
+#if +RUNREGRESSION_XP +PROGRAM_SHEPHERDING : +-security +-no_executable_if_rx_text --executable_after_load
+# SEC_VIO_STOP
+No such process found.
+Window not found: Untitled - Notepad
+#else
+# if +DEBUG :
+Process notepad.exe, running SC debug
+# else
+#  if +PROFILE :
+Process notepad.exe, running SC profile
+#  else
+Process notepad.exe, running SC release
+#  endif
+# endif
+Close message sent.
+#endif
+No such process found.
diff --git a/suite/tests/client-interface/cbr2.dll.c b/suite/tests/client-interface/cbr2.dll.c
new file mode 100644
index 000000000..231724555
--- /dev/null
+++ b/suite/tests/client-interface/cbr2.dll.c
@@ -0,0 +1,181 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+
+#define MINSERT instrlist_meta_preinsert
+
+#ifdef WINDOWS
+# define ASSERT(x)                                        \
+  if (!(x)) {                                             \
+    dr_messagebox("ASSERT failed on line %d", __LINE__);  \
+    dr_abort();                                           \
+  }
+#else /* LINUX */
+# define ASSERT(x)                                            \
+  if (!(x)) {                                                 \
+    dr_fprintf(STDERR, "ASSERT failed on line %d", __LINE__); \
+    dr_abort();                                               \
+  }
+#endif
+
+static
+void thread_init_event(void *drcontext)
+{
+    dr_set_tls_field(drcontext, NULL);
+}
+
+static
+void at_taken(app_pc targ_addr)
+{
+    void *drcontext = dr_get_current_drcontext();
+    dr_set_tls_field(drcontext, targ_addr);
+}
+
+static
+void at_not_taken(app_pc fall_addr)
+{
+    void *drcontext = dr_get_current_drcontext();
+    dr_set_tls_field(drcontext, fall_addr);
+}
+
+static
+void at_bb(app_pc bb_addr)
+{
+    void *drcontext = dr_get_current_drcontext();
+    app_pc cbr_addr = dr_get_tls_field(drcontext);
+
+    if (cbr_addr != NULL && cbr_addr != bb_addr) {
+        dr_printf("ERROR: expected branch to "PFX", but entered BB at "PFX"\n",
+                  cbr_addr, bb_addr);
+    }
+
+    dr_set_tls_field(drcontext, NULL);
+}
+
+static
+dr_emit_flags_t bb_event(void *drcontext, void *tag, instrlist_t *bb, bool for_trace, bool translating)
+{
+    instr_t *instr, *next_instr;
+
+    /* 
+     * Callback on entry to each basic block.  Dynamically verify that
+     * when the last executed BB ends in a cbr, the target (or
+     * fallthrough) matches next addr of the next executed BB.
+     */
+    app_pc bb_addr = dr_fragment_app_pc(tag);
+    instr = instrlist_first(bb);
+
+    dr_prepare_for_call(drcontext, bb, instr);
+
+    MINSERT(bb, instr, INSTR_CREATE_push_imm
+            (drcontext, OPND_CREATE_INT32((ptr_uint_t)bb_addr)));
+    MINSERT(bb, instr, INSTR_CREATE_call
+            (drcontext, opnd_create_pc((void*)at_bb)));
+
+    dr_cleanup_after_call(drcontext, bb, instr, 4);
+
+    for (; instr != NULL; instr = next_instr) {
+        next_instr = instr_get_next(instr);
+
+        if (instr_is_cbr(instr)) {
+            instr_t* jmp_ft;
+            app_pc targ, fall, instr_addr;
+            opnd_t opnd;
+                
+            /*
+             * Build the callback sequence for the not-taken case
+             */
+            instr_addr = instr_get_app_pc(instr);
+            fall = (app_pc)decode_next_pc(drcontext, (byte *)instr_addr);
+
+            dr_prepare_for_call(drcontext, bb, NULL);
+                    
+            MINSERT(bb, NULL, INSTR_CREATE_push_imm
+                    (drcontext, OPND_CREATE_INT32((ptr_uint_t)fall)));
+            MINSERT(bb, NULL, INSTR_CREATE_call
+                    (drcontext, opnd_create_pc((void*)at_not_taken)));
+                    
+            dr_cleanup_after_call(drcontext, bb, NULL, 4);
+
+            /* jump to the original fallthrough block (this should not
+             * be a meta-instruction). */
+            jmp_ft = INSTR_XL8(INSTR_CREATE_jmp(drcontext, opnd_create_pc(fall)), fall);
+            instrlist_preinsert(bb, NULL, jmp_ft);
+                    
+            /* 
+             * Build the callback sequence for the taken case 
+             */
+            opnd = instr_get_target(instr);
+            ASSERT(opnd_is_pc(opnd));                
+            targ = opnd_get_pc(opnd);
+                    
+            dr_prepare_for_call(drcontext, bb, NULL);
+                    
+            MINSERT(bb, NULL, INSTR_CREATE_push_imm
+                    (drcontext, OPND_CREATE_INT32((ptr_uint_t)targ)));
+            MINSERT(bb, NULL, INSTR_CREATE_call
+                    (drcontext, opnd_create_pc((void*)at_taken)));
+                    
+            dr_cleanup_after_call(drcontext, bb, NULL, 4);
+                    
+            /* jump to the original target block (this should
+             * not be a meta-instruction). */
+            instrlist_preinsert(bb, NULL,
+                                INSTR_XL8(INSTR_CREATE_jmp
+                                          (drcontext, opnd_create_pc(targ)), targ));
+                    
+            /* 
+             * Redirect the cbr to jump to our 'taken' callback.
+             * The fallthrough will hit the 'not-taken' callback.
+             */
+            instr_set_ok_to_mangle(instr, false);
+            instr_set_translation(instr, NULL);
+            instr_set_target(instr, opnd_create_instr(instr_get_next(jmp_ft)));
+            /* If this is a short cti, make sure it can reach its new target */
+            if (instr_is_cti_short(instr))
+                instr_convert_short_meta_jmp_to_long(drcontext, bb, instr);
+        }
+    }
+    /* since our added instrumentation is not constant, we ask to store
+     * translations now
+     */
+    return DR_EMIT_STORE_TRANSLATIONS;
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    dr_printf("thank you for testing the client interface\n");
+    dr_register_bb_event(bb_event);
+    dr_register_thread_init_event(thread_init_event);
+}
diff --git a/suite/tests/client-interface/cbr2.runall b/suite/tests/client-interface/cbr2.runall
new file mode 100644
index 000000000..0c6d953b3
--- /dev/null
+++ b/suite/tests/client-interface/cbr2.runall
@@ -0,0 +1,3 @@
+calc.exe
+Calculator
+<>
diff --git a/suite/tests/client-interface/cbr2.template b/suite/tests/client-interface/cbr2.template
new file mode 100644
index 000000000..3d9d40f8a
--- /dev/null
+++ b/suite/tests/client-interface/cbr2.template
@@ -0,0 +1,18 @@
+thank you for testing the client interface
+#if +RUNREGRESSION_XP +PROGRAM_SHEPHERDING : +-security +-no_executable_if_rx_text --executable_after_load
+# SEC_VIO_STOP
+No such process found.
+Window not found: Calculator
+#else
+# if +DEBUG :
+Process calc.exe, running SC debug
+# else
+#  if +PROFILE :
+Process calc.exe, running SC profile
+#  else
+Process calc.exe, running SC release
+#  endif
+# endif
+Close message sent.
+#endif
+No such process found.
diff --git a/suite/tests/client-interface/cbr3.c b/suite/tests/client-interface/cbr3.c
new file mode 100644
index 000000000..f87b353a0
--- /dev/null
+++ b/suite/tests/client-interface/cbr3.c
@@ -0,0 +1,71 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+__declspec(dllexport,noinline)
+void start_instrument()
+{}
+
+__declspec(dllexport,noinline)
+void stop_instrument()
+{}
+
+int main(int argc)
+{
+    int i, x, y;
+
+    /* dummy marker to inform client lib to start instrumenting */
+    start_instrument();
+
+    x = argc;
+    y = 0;
+
+    /* We should have one branch that alternates between taken and not
+     * taken, and a back edge that is taken 9 times and not taken
+     * once.  Without optimization, however, we seem to have one extra
+     * cbr that's always taken.  If we turn on optimization, we'll
+     * have to adjust the expected output.
+     */
+    i = 0;
+    do {
+        if (x % 2) {
+            y++;
+        }
+        i++;
+        x++;
+    } while (i < 10);
+
+    /* dummy marker to inform client lib to stop instrumenting */
+    stop_instrument();
+    printf("y=%d\n", y);
+
+    return 0;
+}
diff --git a/suite/tests/client-interface/cbr3.dll.c b/suite/tests/client-interface/cbr3.dll.c
new file mode 100644
index 000000000..a19df9bc4
--- /dev/null
+++ b/suite/tests/client-interface/cbr3.dll.c
@@ -0,0 +1,423 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * Tests re-instrumentation using the following scheme:
+ * 1) Insert instrumentation normally in the BB event.
+ * 2) Flush the fragment the first time that code executes to
+ *    remove the instrumentation and force a new BB event.
+ * 3) Insert new instrumentation the next time we see the BB event.
+ *
+ * For this test we focus on cbr ops, inserting instrumentation to
+ * capture the fallthrough and taken addresses.  After the first
+ * branch, we re-instrument the BB to remove the instrumentation for
+ * the direction taken.  If and when we see the other direction, we
+ * remove all instrumentation for that branch.
+ *
+ * This test is adapted from a dynamic CFG builder, where we want to
+ * record each control-flow edge, but we don't want to pay the
+ * execution overhead of the instrumentation after we've recorded
+ * the edge.
+ */
+
+#include "dr_api.h"
+
+#define MINSERT instrlist_meta_preinsert
+
+/* We need a table to store the state of each cbr (i.e., "seen taken
+ * edge", "seen fallthrough edge", or "seen both").  The test program
+ * itself is small, so the following hash table is overkill, but I
+ * guess it makes this client easily extendable.
+ */
+#define HASH_TABLE_SIZE 7919
+
+#define ASSERT(x) \
+    do {                                                        \
+        if (!(x)) {                                             \
+            dr_printf("ASSERT failed on line %d", __LINE__);    \
+            dr_flush_file(dr_get_stdout_file());                \
+            dr_abort();                                         \
+        }                                                       \
+    } while (0)                             
+
+typedef enum {
+    CBR_NONE      = 0x00,
+    CBR_TAKEN     = 0x01,
+    CBR_NOT_TAKEN = 0x10
+} cbr_state_t;
+
+typedef struct _elem_t {
+    struct _elem_t *next;
+    cbr_state_t state;
+    app_pc addr;
+} elem_t;
+
+typedef struct _list_t {
+    elem_t *head;
+    elem_t *tail;
+} list_t;
+
+
+/* Global hash table */
+typedef list_t **hash_table_t;
+hash_table_t table = NULL;
+
+static inline
+elem_t *new_elem(app_pc addr, cbr_state_t state)
+{
+    elem_t *elem = (elem_t *)dr_global_alloc(sizeof(elem_t));
+    ASSERT(elem != NULL);
+
+    elem->next  = NULL;
+    elem->addr  = addr;
+    elem->state = state;
+
+    return elem;
+}
+
+static inline
+void delete_elem(elem_t *elem)
+{
+    dr_global_free(elem, sizeof(elem_t));
+}
+
+static inline
+void append(list_t *list, elem_t *elem)
+{
+    if (list->head == NULL) {
+        ASSERT(list->tail == NULL);
+        list->head = elem;
+        list->tail = elem;
+    }
+    else {
+        list->tail->next = elem;
+        list->tail = elem;
+    }
+}
+
+static inline
+elem_t *find(list_t *list, app_pc addr)
+{
+    elem_t *elem = list->head;
+    while (elem != NULL) {
+        if (elem->addr == addr);
+            return elem;
+        elem = elem->next;
+    }
+
+    return NULL;
+}
+
+static inline
+list_t *new_list()
+{
+    list_t *list = (list_t *)dr_global_alloc(sizeof(list_t));
+    list->head = NULL;
+    list->tail = NULL;
+    return list;
+}
+
+static inline
+void delete_list(list_t *list)
+{
+    elem_t *iter = list->head;
+    while (iter != NULL) {
+        elem_t *next = iter->next;
+        delete_elem(iter);
+        iter = next;
+    }
+
+    dr_global_free(list, sizeof(list_t));
+}
+
+hash_table_t new_table()
+{
+    int i;
+    hash_table_t table = (hash_table_t)dr_global_alloc
+        (sizeof(list_t *) * HASH_TABLE_SIZE);
+
+    for (i=0; i<HASH_TABLE_SIZE; i++) {
+        table[i] = NULL;
+    }
+
+    return table;
+}
+
+void delete_table(hash_table_t table)
+{
+    int i;
+    for (i=0; i<HASH_TABLE_SIZE; i++) {
+        if (table[i] != NULL) {
+            delete_list(table[i]);
+        }
+    }
+
+    dr_global_free(table, sizeof(list_t *) * HASH_TABLE_SIZE);
+}
+
+static inline
+uint hash_func(app_pc addr)
+{
+    return ((ptr_uint_t)addr % HASH_TABLE_SIZE);
+}
+
+elem_t *lookup(hash_table_t table, app_pc addr)
+{
+    list_t *list = table[hash_func(addr)];
+    if (list != NULL)
+        return find(list, addr);
+
+    return NULL;
+}
+
+void insert(hash_table_t table, app_pc addr, cbr_state_t state)
+{
+    elem_t *elem = new_elem(addr, state);
+
+    uint index = hash_func(addr);
+    list_t *list = table[index];
+    if (list == NULL) {
+        list = new_list();
+        table[index] = list;
+    }
+
+    append(list, elem);
+}
+
+/*
+ * End hash table implementation 
+ */
+
+static void at_taken(app_pc src, app_pc targ)
+{
+    int app_errno;
+    dr_mcontext_t mcontext;
+    void *drcontext = dr_get_current_drcontext();
+
+    /* 
+     * Record the fact that we've seen the taken case.
+     */
+    elem_t *elem = lookup(table, src);
+    ASSERT(elem != NULL);
+    elem->state |= CBR_TAKEN;
+
+    dr_printf("cbr taken\n");
+
+    /* 
+     * Remove the bb from the cache so it will be re-built the next
+     * time it is executed.  Since the flush will remove the fragment
+     * we're already in, redirect execution to the target.
+     */
+    dr_flush_region(src, 1);
+    dr_get_mcontext(drcontext, &mcontext, &app_errno);
+    mcontext.eip = targ;
+    dr_redirect_execution(&mcontext, app_errno);
+}
+
+
+static void at_not_taken(app_pc src, app_pc fall)
+{
+    int app_errno;
+    dr_mcontext_t mcontext;
+    void *drcontext = dr_get_current_drcontext();
+
+    /* 
+     * Record the fact that we've seen the fallthrough case.
+     */
+    elem_t *elem = lookup(table, src);
+    ASSERT(elem != NULL);
+    elem->state |= CBR_NOT_TAKEN;
+
+    dr_printf("cbr not taken\n");
+
+    /* 
+     * Remove the bb from the cache so it will be re-built the next
+     * time it is executed.  Since the flush will remove the fragment
+     * we're already in, redirect execution to the target.
+     */
+    dr_flush_region(src, 1);
+    dr_get_mcontext(drcontext, &mcontext, &app_errno);
+    mcontext.eip = fall;
+    dr_redirect_execution(&mcontext, app_errno);
+}
+
+
+app_pc start_pc = NULL;         /* pc to start instrumenting */
+app_pc stop_pc = NULL;          /* pc to stop instrumenting */
+bool instrument = false;        /* insert instrumentation? */
+
+dr_emit_flags_t bb_event(void *drcontext, void *tag, instrlist_t *bb, bool for_trace, bool translating)
+{
+    instr_t *instr, *next_instr;
+    
+    app_pc bb_addr = dr_fragment_app_pc(tag);
+    if (bb_addr == start_pc) {
+        instrument = true;
+    }
+    else if (bb_addr == stop_pc) {
+        instrument = false;
+    }
+
+    if (!instrument) {
+        return DR_EMIT_DEFAULT;
+    }
+
+    for (instr = instrlist_first(bb); instr != NULL; instr = next_instr) {
+        next_instr = instr_get_next(instr);
+
+        /*
+         * Conditional branch. We can determine the target and
+         * fallthrough addresses here, but we need to instrument if we
+         * want to record the edge only if it actually executes at
+         * runtime. Instead of using dr_insert_cbr_instrumentation,
+         * we'll insert separate instrumentation for the taken and not
+         * taken cases and remove it separately after we see each
+         * case.
+         */
+        if (instr_is_cbr(instr)) {
+            app_pc src = instr_get_app_pc(instr);
+
+            cbr_state_t state;
+            bool insert_taken, insert_not_taken;
+            
+            /* First look up the state of this branch so we
+             * know what instrumentation to insert, if any.
+             */
+            elem_t *elem = lookup(table, src);
+            
+            if (elem == NULL) {
+                state = CBR_NONE;
+                insert(table, src, CBR_NONE);
+            }
+            else {
+                state = elem->state;
+            }
+            
+            insert_taken = (state & CBR_TAKEN) == 0;
+            insert_not_taken = (state & CBR_NOT_TAKEN) == 0;
+            
+            if (insert_taken || insert_not_taken) {
+                app_pc fall = (app_pc)decode_next_pc(drcontext, (byte *)src);
+                app_pc targ = instr_get_branch_target_pc(instr);
+                
+                /* 
+                 * Redirect the cbr to jump to the 'taken' callout.
+                 * We'll insert a 'not-taken' callout at fallthrough
+                 * address.
+                 */
+                instr_t *label = INSTR_CREATE_label(drcontext);
+                instr_set_ok_to_mangle(instr, false);
+                instr_set_translation(instr, NULL);
+                instr_set_target(instr, opnd_create_instr(label));
+                /* If this is a short cti, make sure it can reach its new target */
+                if (instr_is_cti_short(instr))
+                    instr_convert_short_meta_jmp_to_long(drcontext, bb, instr);
+                
+                if (insert_not_taken) {
+                    /*
+                     * Callout for the not-taken case
+                     */
+                    dr_insert_clean_call(drcontext, bb, NULL, 
+                                         (void*)at_not_taken,
+                                         false /* don't save fp state */, 
+                                         2 /* 2 args for at_not_taken */,
+                                         OPND_CREATE_INT32((ptr_uint_t)src),
+                                         OPND_CREATE_INT32((ptr_uint_t)fall));
+                }
+                
+                /*
+                 * Jump to the original fall-through address.
+                 * (This should not be a meta-instruction).
+                 */
+                instrlist_preinsert
+                    (bb, NULL, INSTR_XL8
+                     (INSTR_CREATE_jmp(drcontext, opnd_create_pc(fall)), fall));
+                
+                /* label goes before the 'taken' callout */
+                MINSERT(bb, NULL, label);
+                
+                if (insert_taken) {
+                    /* 
+                     * Callout for the taken case 
+                     */
+                    dr_insert_clean_call(drcontext, bb, NULL,
+                                         (void*)at_taken,
+                                         false /* don't save fp state */,
+                                         2 /* 2 args for at_taken */,
+                                         OPND_CREATE_INT32((ptr_uint_t)src),
+                                         OPND_CREATE_INT32((ptr_uint_t)targ));
+                }
+                
+                /* 
+                 * Jump to the original target block (this should
+                 * not be a meta-instruction).
+                 */
+                instrlist_preinsert
+                    (bb, NULL, INSTR_XL8
+                     (INSTR_CREATE_jmp(drcontext, opnd_create_pc(targ)), targ));
+            }
+        }
+    }
+    /* since our added instrumentation is not constant, we ask to store
+     * translations now
+     */
+    return DR_EMIT_STORE_TRANSLATIONS;
+}
+
+
+void dr_exit(void)
+{
+    delete_table(table);
+}
+
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    /* Look up start_instrument() and stop_instrument() in the app.
+     * These functions are markers that tell us when to start and stop
+     * instrumenting.
+     */
+    module_data_t *prog = dr_lookup_module_by_name("cbr3.exe");
+    ASSERT(prog != NULL);
+
+    start_pc = (app_pc)dr_get_proc_address(prog->start, "start_instrument");
+    stop_pc = (app_pc)dr_get_proc_address(prog->start, "stop_instrument");
+    
+    ASSERT(start_pc != NULL && stop_pc != NULL);
+    dr_free_module_data(prog);
+
+    table = new_table();
+
+    dr_register_bb_event(bb_event);
+    dr_register_exit_event(dr_exit);
+}
diff --git a/suite/tests/client-interface/cbr3.expect b/suite/tests/client-interface/cbr3.expect
new file mode 100644
index 000000000..9907e3221
--- /dev/null
+++ b/suite/tests/client-interface/cbr3.expect
@@ -0,0 +1,6 @@
+cbr taken
+cbr not taken
+cbr taken
+cbr taken
+cbr not taken
+y=5
diff --git a/suite/tests/client-interface/cbr4.c b/suite/tests/client-interface/cbr4.c
new file mode 100644
index 000000000..f87b353a0
--- /dev/null
+++ b/suite/tests/client-interface/cbr4.c
@@ -0,0 +1,71 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+__declspec(dllexport,noinline)
+void start_instrument()
+{}
+
+__declspec(dllexport,noinline)
+void stop_instrument()
+{}
+
+int main(int argc)
+{
+    int i, x, y;
+
+    /* dummy marker to inform client lib to start instrumenting */
+    start_instrument();
+
+    x = argc;
+    y = 0;
+
+    /* We should have one branch that alternates between taken and not
+     * taken, and a back edge that is taken 9 times and not taken
+     * once.  Without optimization, however, we seem to have one extra
+     * cbr that's always taken.  If we turn on optimization, we'll
+     * have to adjust the expected output.
+     */
+    i = 0;
+    do {
+        if (x % 2) {
+            y++;
+        }
+        i++;
+        x++;
+    } while (i < 10);
+
+    /* dummy marker to inform client lib to stop instrumenting */
+    stop_instrument();
+    printf("y=%d\n", y);
+
+    return 0;
+}
diff --git a/suite/tests/client-interface/cbr4.dll.c b/suite/tests/client-interface/cbr4.dll.c
new file mode 100644
index 000000000..ecc5a45d0
--- /dev/null
+++ b/suite/tests/client-interface/cbr4.dll.c
@@ -0,0 +1,434 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * Tests re-instrumentation using the following scheme:
+ * 1) Insert instrumentation normally in the BB event.
+ * 2) In the callout in the instrumented code, directly replace
+ *    the instrumentation.
+ *
+ * In contrast to cbr3, which uses a flushing scheme, we replace the
+ * instrumented code directly with dr_replace_fragment().  As with
+ * cbr3, we focus on cbr ops, inserting instrumentation to capture the
+ * fallthrough and taken addresses.  After the first branch, we
+ * re-instrument the BB to remove the instrumentation for the
+ * direction taken.  If and when we see the other direction, we remove
+ * all instrumentation for that branch.
+ *
+ * This test is adapted from a dynamic CFG builder, where we want to
+ * record each control-flow edge, but we don't want to pay the
+ * execution overhead of the instrumentation after we've recorded
+ * the edge.
+ */
+
+#include "dr_api.h"
+
+#define MINSERT instrlist_meta_preinsert
+#define inline __inline
+
+/* We need a table to store the state of each cbr (i.e., "seen taken
+ * edge", "seen fallthrough edge", or "seen both").  The test program
+ * itself is small, so the following hash table is overkill, but I
+ * guess it makes this client easily extendable.
+ */
+#define HASH_TABLE_SIZE 7919
+
+#define ASSERT(x) \
+    do {                                                        \
+        if (!(x)) {                                             \
+            dr_printf("ASSERT failed on line %d", __LINE__);    \
+            dr_flush_file(dr_get_stdout_file());                \
+            dr_abort();                                         \
+        }                                                       \
+    } while (0)                             
+
+typedef enum {
+    CBR_NONE      = 0x00,
+    CBR_TAKEN     = 0x01,
+    CBR_NOT_TAKEN = 0x10
+} cbr_state_t;
+
+typedef struct _elem_t {
+    struct _elem_t *next;
+    cbr_state_t state;
+    app_pc addr;
+} elem_t;
+
+typedef struct _list_t {
+    elem_t *head;
+    elem_t *tail;
+} list_t;
+
+
+/* Global hash table */
+typedef list_t **hash_table_t;
+hash_table_t table = NULL;
+
+static inline
+elem_t *new_elem(app_pc addr, cbr_state_t state)
+{
+    elem_t *elem = (elem_t *)dr_global_alloc(sizeof(elem_t));
+    ASSERT(elem != NULL);
+
+    elem->next  = NULL;
+    elem->addr  = addr;
+    elem->state = state;
+
+    return elem;
+}
+
+static inline
+void delete_elem(elem_t *elem)
+{
+    dr_global_free(elem, sizeof(elem_t));
+}
+
+static inline
+void append(list_t *list, elem_t *elem)
+{
+    if (list->head == NULL) {
+        ASSERT(list->tail == NULL);
+        list->head = elem;
+        list->tail = elem;
+    }
+    else {
+        list->tail->next = elem;
+        list->tail = elem;
+    }
+}
+
+static inline
+elem_t *find(list_t *list, app_pc addr)
+{
+    elem_t *elem = list->head;
+    while (elem != NULL) {
+        if (elem->addr == addr);
+            return elem;
+        elem = elem->next;
+    }
+
+    return NULL;
+}
+
+static inline
+list_t *new_list()
+{
+    list_t *list = (list_t *)dr_global_alloc(sizeof(list_t));
+    list->head = NULL;
+    list->tail = NULL;
+    return list;
+}
+
+static inline
+void delete_list(list_t *list)
+{
+    elem_t *iter = list->head;
+    while (iter != NULL) {
+        elem_t *next = iter->next;
+        delete_elem(iter);
+        iter = next;
+    }
+
+    dr_global_free(list, sizeof(list_t));
+}
+
+hash_table_t new_table()
+{
+    int i;
+    hash_table_t table = (hash_table_t)dr_global_alloc
+        (sizeof(list_t *) * HASH_TABLE_SIZE);
+
+    for (i=0; i<HASH_TABLE_SIZE; i++) {
+        table[i] = NULL;
+    }
+
+    return table;
+}
+
+void delete_table(hash_table_t table)
+{
+    int i;
+    for (i=0; i<HASH_TABLE_SIZE; i++) {
+        if (table[i] != NULL) {
+            delete_list(table[i]);
+        }
+    }
+
+    dr_global_free(table, sizeof(list_t *) * HASH_TABLE_SIZE);
+}
+
+static inline
+uint hash_func(app_pc addr)
+{
+    return ((ptr_uint_t)addr % HASH_TABLE_SIZE);
+}
+
+elem_t *lookup(hash_table_t table, app_pc addr)
+{
+    list_t *list = table[hash_func(addr)];
+    if (list != NULL)
+        return find(list, addr);
+
+    return NULL;
+}
+
+void insert(hash_table_t table, app_pc addr, cbr_state_t state)
+{
+    elem_t *elem = new_elem(addr, state);
+
+    uint index = hash_func(addr);
+    list_t *list = table[index];
+    if (list == NULL) {
+        list = new_list();
+        table[index] = list;
+    }
+
+    append(list, elem);
+}
+
+/*
+ * End hash table implementation 
+ */
+
+static dr_emit_flags_t
+instrument_bb(void *drcontext, void *tag, instrlist_t *bb,
+              bool for_trace, bool translating);
+
+static void at_taken(app_pc src, app_pc targ, void *tag)
+{
+    instrlist_t *bb;
+    void *drcontext = dr_get_current_drcontext();
+
+    /* 
+     * Record the fact that we've seen the taken case.
+     */
+    elem_t *elem = lookup(table, src);
+    ASSERT(elem != NULL);
+    elem->state |= CBR_TAKEN;
+
+    dr_printf("cbr taken\n");
+
+    /* 
+     * Re-instrument and replace the fragment.
+     */
+    ASSERT(dr_bb_exists_at(drcontext, tag));
+    bb = decode_as_bb(drcontext, tag);
+    instrument_bb(drcontext, tag, bb, false, false);
+    dr_replace_fragment(drcontext, tag, bb);
+}
+
+
+static void at_not_taken(app_pc src, app_pc fall, void *tag)
+{
+    instrlist_t *bb;
+    void *drcontext = dr_get_current_drcontext();
+
+    /* 
+     * Record the fact that we've seen the fallthrough case.
+     */
+    elem_t *elem = lookup(table, src);
+    ASSERT(elem != NULL);
+    elem->state |= CBR_NOT_TAKEN;
+
+    dr_printf("cbr not taken\n");
+
+    /* 
+     * Re-instrument and replace the fragment.
+     */
+    ASSERT(dr_bb_exists_at(drcontext, tag));
+    bb = decode_as_bb(drcontext, tag);
+    instrument_bb(drcontext, tag, bb, false, false);
+    dr_replace_fragment(drcontext, tag, bb);
+}
+
+
+static dr_emit_flags_t
+instrument_bb(void *drcontext, void *tag, instrlist_t *bb, bool for_trace,
+              bool translating)
+{
+    instr_t *instr, *next_instr;
+    
+    for (instr = instrlist_first(bb); instr != NULL; instr = next_instr) {
+        next_instr = instr_get_next(instr);
+        
+        /*
+         * Conditional branch. We can determine the target and
+         * fallthrough addresses here, but we need to instrument if we
+         * want to record the edge only if it actually executes at
+         * runtime. Instead of using dr_insert_cbr_instrumentation,
+         * we'll insert separate instrumentation for the taken and not
+         * taken cases and remove it separately after we see each
+         * case.
+         */
+        if (instr_is_cbr(instr)) {
+            app_pc src = instr_get_app_pc(instr);
+            
+            cbr_state_t state;
+            bool insert_taken, insert_not_taken;
+            
+            /* First look up the state of this branch so we
+             * know what instrumentation to insert, if any.
+             */
+            elem_t *elem = lookup(table, src);
+            
+            if (elem == NULL) {
+                state = CBR_NONE;
+                insert(table, src, CBR_NONE);
+            }
+            else {
+                state = elem->state;
+            }
+            
+            insert_taken = (state & CBR_TAKEN) == 0;
+            insert_not_taken = (state & CBR_NOT_TAKEN) == 0;
+            
+            if (insert_taken || insert_not_taken) {
+                app_pc fall = (app_pc)decode_next_pc(drcontext, (byte *)src);
+                app_pc targ = instr_get_branch_target_pc(instr);
+                
+                /* 
+                 * Redirect the cbr to jump to the 'taken' callout.
+                 * We'll insert a 'not-taken' callout at fallthrough
+                 * address.
+                 */
+                instr_t *label = INSTR_CREATE_label(drcontext);
+                instr_set_ok_to_mangle(instr, false);
+                instr_set_translation(instr, NULL);
+                instr_set_target(instr, opnd_create_instr(label));
+                /* If this is a short cti, make sure it can reach its new target */
+                if (instr_is_cti_short(instr))
+                    instr_convert_short_meta_jmp_to_long(drcontext, bb, instr);
+                
+                if (insert_not_taken) {
+                    /*
+                     * Callout for the not-taken case
+                     */
+                    dr_insert_clean_call(drcontext, bb, NULL, 
+                                         (void*)at_not_taken,
+                                         false /* don't save fp state */, 
+                                         3 /* 3 args for at_not_taken */,
+                                         OPND_CREATE_INT32((ptr_uint_t)src),
+                                         OPND_CREATE_INT32((ptr_uint_t)fall),
+                                         OPND_CREATE_INT32((ptr_uint_t)tag));
+                }
+                
+                /*
+                 * Jump to the original fall-through address.
+                 * (This should not be a meta-instruction).
+                 */
+                instrlist_preinsert
+                    (bb, NULL, INSTR_XL8
+                     (INSTR_CREATE_jmp(drcontext, opnd_create_pc(fall)), fall));
+                
+                /* label goes before the 'taken' callout */
+                MINSERT(bb, NULL, label);
+                
+                if (insert_taken) {
+                    /* 
+                     * Callout for the taken case 
+                     */
+                    dr_insert_clean_call(drcontext, bb, NULL,
+                                         (void*)at_taken,
+                                         false /* don't save fp state */,
+                                         3 /* 3 args for at_taken */,
+                                         OPND_CREATE_INT32((ptr_uint_t)src),
+                                         OPND_CREATE_INT32((ptr_uint_t)targ),
+                                         OPND_CREATE_INT32((ptr_uint_t)tag));
+                }
+                
+                /* 
+                 * Jump to the original target block (this should
+                 * not be a meta-instruction).
+                 */
+                instrlist_preinsert
+                    (bb, NULL, INSTR_XL8
+                     (INSTR_CREATE_jmp(drcontext, opnd_create_pc(targ)), targ));
+            }
+        }
+    }
+    /* since our added instrumentation is not constant, we ask to store
+     * translations now
+     */
+    return DR_EMIT_STORE_TRANSLATIONS;
+}
+
+
+app_pc start_pc = NULL;         /* pc to start instrumenting */
+app_pc stop_pc = NULL;          /* pc to stop instrumenting */
+bool instrument = false;        /* insert instrumentation? */
+
+dr_emit_flags_t bb_event(void *drcontext, void *tag, instrlist_t *bb,
+                         bool for_trace, bool translating)
+{
+    app_pc bb_addr = dr_fragment_app_pc(tag);
+    dr_emit_flags_t flags = DR_EMIT_DEFAULT;
+    if (bb_addr == start_pc) {
+        instrument = true;
+    }
+    else if (bb_addr == stop_pc) {
+        instrument = false;
+    }
+
+    if (instrument) {
+        flags = instrument_bb(drcontext, tag, bb, for_trace, translating);
+    }
+    return flags;
+}
+
+
+void dr_exit(void)
+{
+    delete_table(table);
+}
+
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    /* Look up start_instrument() and stop_instrument() in the app.
+     * These functions are markers that tell us when to start and stop
+     * instrumenting.
+     */
+    module_data_t *prog = dr_lookup_module_by_name("cbr4.exe");
+    ASSERT(prog != NULL);
+
+    start_pc = (app_pc)dr_get_proc_address(prog->start, "start_instrument");
+    stop_pc = (app_pc)dr_get_proc_address(prog->start, "stop_instrument");
+    
+    ASSERT(start_pc != NULL && stop_pc != NULL);
+    dr_free_module_data(prog);
+
+    table = new_table();
+
+    dr_register_bb_event(bb_event);
+    dr_register_exit_event(dr_exit);
+}
diff --git a/suite/tests/client-interface/cbr4.expect b/suite/tests/client-interface/cbr4.expect
new file mode 100644
index 000000000..9907e3221
--- /dev/null
+++ b/suite/tests/client-interface/cbr4.expect
@@ -0,0 +1,6 @@
+cbr taken
+cbr not taken
+cbr taken
+cbr taken
+cbr not taken
+y=5
diff --git a/suite/tests/client-interface/cleancall.c b/suite/tests/client-interface/cleancall.c
new file mode 100644
index 000000000..79814e78c
--- /dev/null
+++ b/suite/tests/client-interface/cleancall.c
@@ -0,0 +1,160 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* We would have the client handle clean call crashes, but today they
+ * go to the app (PR 213600); plus there is no signal/exception event
+ * on linux (PR 304708)
+ */
+
+#ifdef LINUX
+# include <unistd.h>
+# include <signal.h>
+# include <ucontext.h>
+# include <errno.h>
+# include <stdlib.h>
+#endif
+
+#include <setjmp.h>
+#include "tools.h" /* for print() */
+
+#ifdef WINDOWS
+# define NOP __nop()
+#else /* LINUX */
+# define NOP asm("nop")
+#endif
+
+static void foo(void)
+{ /* nothing: just a marker */ }
+
+jmp_buf mark;
+
+/* just use single-arg handlers */
+typedef void (*handler_t)(int);
+typedef void (*handler_3_t)(int, struct siginfo *, void *);
+
+static int count;
+
+#ifdef LINUX
+# define ALT_STACK_SIZE  (SIGSTKSZ*3)
+
+static void
+signal_handler(int sig)
+{
+    if (sig == SIGSEGV) {
+        count++;
+        print("Access violation, instance %d\n", count);
+        longjmp(mark, count);
+    }
+    exit(-1);
+}
+
+#define ASSERT_NOERR(rc) do {                                   \
+  if (rc) {                                                     \
+     fprintf(stderr, "%s:%d rc=%d errno=%d %s\n",               \
+             __FILE__, __LINE__,                                \
+             rc, errno, strerror(errno));                       \
+  }                                                             \
+} while (0);
+
+/* set up signal_handler as the handler for signal "sig" */
+static void
+intercept_signal(int sig, handler_t handler)
+{
+    int rc;
+    struct sigaction act;
+
+    act.sa_sigaction = (handler_3_t) handler;
+    /* FIXME: due to DR bug 840 we cannot block ourself in the handler
+     * since the handler does not end in a sigreturn, so we have an empty mask
+     * and we use SA_NOMASK
+     */
+    rc = sigemptyset(&act.sa_mask); /* block no signals within handler */
+    ASSERT_NOERR(rc);
+    /* FIXME: due to DR bug #654 we use SA_SIGINFO -- change it once DR works */
+    act.sa_flags = SA_NOMASK | SA_SIGINFO | SA_ONSTACK;
+    
+    /* arm the signal */
+    rc = sigaction(sig, &act, NULL);
+    ASSERT_NOERR(rc);
+}
+
+#else
+
+# include <windows.h>
+/* top-level exception handler */
+static LONG
+our_top_handler(struct _EXCEPTION_POINTERS * pExceptionInfo)
+{
+    if (pExceptionInfo->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
+#if VERBOSE
+        /* DR gets the target addr wrong for far call/jmp so we don't print it */
+        print("\tPC "PFX" tried to %s address "PFX"\n",
+            pExceptionInfo->ExceptionRecord->ExceptionAddress, 
+            (pExceptionInfo->ExceptionRecord->ExceptionInformation[0]==0)?"read":"write",
+            pExceptionInfo->ExceptionRecord->ExceptionInformation[1]);
+#endif
+        count++;
+        print("Access violation, instance %d\n", count);
+        longjmp(mark, count);
+    }
+    print("Exception "PFX" occurred, process about to die silently\n",
+          pExceptionInfo->ExceptionRecord->ExceptionCode);
+    return EXCEPTION_EXECUTE_HANDLER; /* => global unwind and silent death */
+}
+#endif
+
+int main(int argc, char *argv[])
+{
+    int i, j;
+#ifdef LINUX
+    intercept_signal(SIGSEGV, signal_handler);
+#else
+    SetUnhandledExceptionFilter((LPTOP_LEVEL_EXCEPTION_FILTER) our_top_handler);
+#endif
+
+    /* Each test in cleancall.dll.c crashes at the end */
+    for (j = 0; j < 5; j++) {
+        i = setjmp(mark);
+        if (i == 0) {
+            print("testing clean calls\n");
+            /* use 2 NOPs + call to indicate it's ok to do the tests
+             * now that the handlers are all set up
+             */
+            NOP; NOP;
+            foo();
+            print("did not crash\n");
+        }
+    }
+    print("done testing clean calls\n");
+
+    return 0;
+}
diff --git a/suite/tests/client-interface/cleancall.dll.c b/suite/tests/client-interface/cleancall.dll.c
new file mode 100644
index 000000000..60c34fb6b
--- /dev/null
+++ b/suite/tests/client-interface/cleancall.dll.c
@@ -0,0 +1,345 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+
+#define MINSERT instrlist_meta_preinsert
+
+#ifdef WINDOWS
+# define ASSERT(x)                                        \
+  if (!(x)) {                                             \
+    dr_messagebox("ASSERT failed on line %d", __LINE__);  \
+    dr_abort();                                           \
+  }
+#else /* LINUX */
+# define ASSERT(x)                                            \
+  if (!(x)) {                                                 \
+    dr_fprintf(STDERR, "ASSERT failed on line %d", __LINE__); \
+    dr_abort();                                               \
+  }
+#endif
+
+#ifdef X64
+/* we use this for clean call base-disp refs */
+static reg_t buf[] = { 0xcafebabe, 0xfeedadad, 0xeeeeeeee, 0xbadcabee };
+#endif
+
+static void
+foo(reg_t a1, reg_t a2, reg_t a3, reg_t a4, reg_t a5, reg_t a6, reg_t a7, reg_t a8)
+{
+    dr_fprintf(STDERR, "foo "PFX" "PFX" "PFX" "PFX"\n    "PFX" "PFX" "PFX" "PFX" \n",
+               a1, 
+               /* printing addr of buf would be non-deterministic */
+               IF_X64_ELSE((a2 == (reg_t) buf) ? 1 : -1, a2),
+               a3, a4, a5, a6, a7, a8);
+}
+
+static void
+bar(reg_t a1, reg_t a2)
+{
+    dr_fprintf(STDERR, "bar "PFX" "PFX"\n", a1, a2);
+}
+
+static void
+save_test()
+{
+    int i;
+    void *drcontext = dr_get_current_drcontext();
+    dr_fprintf(STDERR, "verifying values\n");
+    if (*(((reg_t *)dr_get_tls_field(drcontext))+2) != 1) {
+        dr_fprintf(STDERR, "Write to client tls from cache failed, got %d, expected %d\n",
+                   *(((reg_t *)dr_get_tls_field(drcontext))+2), 1);
+    }
+    for (i = SPILL_SLOT_1; i <= SPILL_SLOT_MAX; i++) {
+        reg_t value = dr_read_saved_reg(drcontext, i);
+        if (value != i + 1 - SPILL_SLOT_1) {
+            dr_fprintf(STDERR, "slot %d value %d doesn't match expected value %d\n",
+                       i, value, i + 1 - SPILL_SLOT_1);
+        }
+        if (i % 2 == 0) {
+            /* set every other reg */
+            value = 100 - i;
+            dr_write_saved_reg(drcontext, i, value);
+        }
+    }
+}
+
+static int post_crash = 0;
+static void *tag_of_interest;
+
+static void
+restore_state_event(void *drcontext, void *tag, dr_mcontext_t *mcontext,
+                    bool restore_memory, bool app_code_consistent)
+{
+    if (tag == tag_of_interest) {
+        dr_fprintf(STDERR, "in restore_state for our clean call crash %d\n", post_crash);
+        /* flush, so we can use different instrumentation next time */
+        dr_delay_flush_region(dr_fragment_app_pc(tag), 1, 0, NULL);
+    }
+}
+
+static dr_emit_flags_t
+bb_event(void *drcontext, void* tag, instrlist_t *bb, bool for_trace, bool translating)
+{
+    instr_t *instr, *next_instr, *next_next_instr;
+    bool modified = false;
+    
+# define PRE(bb, i) \
+    instrlist_preinsert(bb, instr, INSTR_XL8(i, dr_fragment_app_pc(tag)))
+# define PREM(bb, i) \
+    instrlist_meta_preinsert(bb, instr, INSTR_XL8(i, dr_fragment_app_pc(tag)))
+
+    /* Look for 3 nops to indicate handler is set up */
+    for (instr = instrlist_first(bb); instr != NULL; instr = next_instr) {
+        next_instr = instr_get_next(instr);
+
+        if (next_instr != NULL)
+            next_next_instr = instr_get_next(next_instr);
+        else
+            next_next_instr = NULL;
+
+        if (instr_is_nop(instr) && 
+            next_instr != NULL && instr_is_nop(next_instr) && 
+            next_next_instr != NULL && instr_is_call_direct(next_next_instr)) {
+
+            ASSERT(tag_of_interest == NULL || tag_of_interest == tag);
+            tag_of_interest = tag;
+            modified = true;
+
+            /* # of crashes is tied to # of setjmps in cleancall.c */
+
+            if (post_crash == 0) {
+                /* Test crash in 1st clean call arg */
+                dr_fprintf(STDERR, "inserting clean call crash code 1\n");
+                dr_insert_clean_call(drcontext, bb, instrlist_first(bb), (void*) foo,
+                                     false /* don't save fp state */, 1,
+                                     OPND_CREATE_ABSMEM(NULL, OPSZ_4));
+                post_crash++;
+            } else if (post_crash == 1) {
+                /* Test crash in 2nd clean call arg */
+                dr_fprintf(STDERR, "inserting clean call crash code 2\n");
+                dr_insert_clean_call(drcontext, bb, instrlist_first(bb), (void*) foo,
+                                     false /* don't save fp state */, 2,
+                                     OPND_CREATE_INT32(0),
+                                     OPND_CREATE_ABSMEM(NULL, OPSZ_4));
+                post_crash++;
+            } else if (post_crash == 2) {
+                /* PR 307242: test xsp args */
+                reg_id_t scratch;
+#ifdef X64
+# ifdef WINDOWS
+                scratch = REG_XCX;
+# else
+                scratch = REG_XDI;
+# endif
+#else
+                scratch = REG_XAX;
+#endif
+
+                dr_fprintf(STDERR, "inserting xsp arg testing\n");
+                /* See notes below: we crash after, so can clobber regs */
+                PRE(bb, INSTR_CREATE_mov_imm(drcontext, opnd_create_reg(scratch),
+                                             OPND_CREATE_INT32(sizeof(reg_t))));
+                PRE(bb, INSTR_CREATE_push_imm(drcontext,
+                                              OPND_CREATE_INT32((int)0xbcbcaba0)));
+                PRE(bb, INSTR_CREATE_push_imm(drcontext,
+                                              OPND_CREATE_INT32((int)0xbcbcaba1)));
+                dr_insert_clean_call(drcontext, bb, instr, (void*) bar,
+                                     false /* don't save fp state */, 2,
+                                     OPND_CREATE_MEM32(REG_XSP, 0),
+                                     /* test conflicting w/ scratch reg */
+                                     opnd_create_base_disp(REG_XSP, scratch, 1,
+                                                           0, OPSZ_PTR));
+                PRE(bb, INSTR_CREATE_mov_ld(drcontext, opnd_create_reg(REG_XAX),
+                                            OPND_CREATE_ABSMEM(NULL, OPSZ_PTR)));
+                post_crash++;
+            } else if (post_crash == 3) {
+#ifdef X86_64
+                /* For x64, test using calling convention regs as params.
+                 * We do different things depending on order, whether a
+                 * memory reference, etc.
+                 * To test our values, we clobber app registers.  The app
+                 * has a setjmp set up, so we crash after for a deterministic
+                 * result.
+                 */
+                dr_fprintf(STDERR, "inserting clean call arg testing\n");
+                /* We do not translate the regs back */
+
+                /* We arrange to have our base-disps all be small offsets off buf */
+                PRE(bb, INSTR_CREATE_mov_imm(drcontext, opnd_create_reg(REG_RDX),
+                                             OPND_CREATE_INT32(sizeof(reg_t))));
+                PRE(bb, INSTR_CREATE_mov_imm(drcontext, opnd_create_reg(REG_RCX),
+                                             OPND_CREATE_INTPTR(buf)));
+                PRE(bb, INSTR_CREATE_mov_imm(drcontext, opnd_create_reg(REG_R8),
+                                             OPND_CREATE_INT32(-42)));
+                PRE(bb, INSTR_CREATE_mov_imm(drcontext, opnd_create_reg(REG_R9),
+                                             OPND_CREATE_INT32((int)0xdeadbeef)));
+                PRE(bb, INSTR_CREATE_mov_imm(drcontext, opnd_create_reg(REG_RAX),
+                                             OPND_CREATE_INT32(2*sizeof(reg_t))));
+                PRE(bb, INSTR_CREATE_mov_imm(drcontext, opnd_create_reg(REG_RBP),
+                                             OPND_CREATE_INT32(3*sizeof(reg_t))));
+                dr_insert_clean_call(drcontext, bb, instr, (void*) foo,
+                                     false /* don't save fp state */, 8,
+                                     /* Pick registers used by both Windows and Linux */
+                                     opnd_create_reg(REG_RDX),
+                                     opnd_create_reg(REG_RCX),
+                                     opnd_create_reg(REG_R9),
+                                     opnd_create_reg(REG_R8),
+                                     OPND_CREATE_MEM32(REG_RCX, 0),
+                                     /* test having only index register conflict */
+                                     opnd_create_base_disp(REG_RBP, REG_RCX, 1,
+                                                           0, OPSZ_PTR),
+                                     /* test OPSZ_4, and using register modified
+                                      * by clean call setup (rax) */
+                                     opnd_create_base_disp(REG_RAX, REG_RCX, 1,
+                                                           0, OPSZ_4),
+                                     /* test having both base and index conflict */
+                                     opnd_create_base_disp(REG_RDX, REG_RCX, 1,
+                                                           0, OPSZ_PTR));
+#endif
+                PRE(bb, INSTR_CREATE_mov_ld(drcontext, opnd_create_reg(REG_XAX),
+                                            OPND_CREATE_ABSMEM(NULL, OPSZ_PTR)));
+                post_crash++;
+            } else {
+                /* Test register saving and restoring and access to saved registers
+                 * from outside the cache. */
+                int i;
+                instr_t *fault = INSTR_CREATE_mov_ld(drcontext, opnd_create_reg(REG_XAX),
+                                                     OPND_CREATE_ABSMEM(NULL, OPSZ_PTR));
+                instr_t *post_fault = INSTR_CREATE_label(drcontext);
+                dr_fprintf(STDERR, "inserting saved reg access testing\n");
+
+                /* we want to test all the slots so juggle around to save xax and flags
+                 * to client's tls. */
+                dr_save_reg(drcontext, bb, instr, REG_XBX, SPILL_SLOT_1);
+                dr_insert_read_tls_field(drcontext, bb, instr, REG_XBX);
+                PREM(bb, INSTR_CREATE_mov_st
+                     (drcontext, opnd_create_base_disp(REG_XBX, REG_NULL, 0,
+                                                       0 * sizeof(reg_t), OPSZ_PTR),
+                      opnd_create_reg(REG_XAX)));
+                dr_save_arith_flags(drcontext, bb, instr, SPILL_SLOT_2);
+                PREM(bb, INSTR_CREATE_mov_st
+                     (drcontext, opnd_create_base_disp(REG_XBX, REG_NULL, 0,
+                                                       1 * sizeof(reg_t), OPSZ_PTR),
+                      opnd_create_reg(REG_XAX)));
+                PREM(bb, INSTR_CREATE_mov_imm(drcontext, opnd_create_reg(REG_XAX),
+                                             OPND_CREATE_INT32(1)));
+                /* test tls writing */
+                PREM(bb, INSTR_CREATE_mov_st
+                     (drcontext, opnd_create_base_disp(REG_XBX, REG_NULL, 0,
+                                                       2 * sizeof(reg_t), OPSZ_PTR),
+                      opnd_create_reg(REG_XAX)));
+                dr_restore_reg(drcontext, bb, instr, REG_XBX, SPILL_SLOT_1);
+                
+                /* now test the slots */
+                /* xax is our tls + 0, flags is our tls + sizeof(reg_t) */
+                for (i = SPILL_SLOT_1; i <= SPILL_SLOT_MAX; i++) {
+                    dr_save_reg(drcontext, bb, instr, REG_XAX, i);
+                    PREM(bb, INSTR_CREATE_inc(drcontext, opnd_create_reg(REG_XAX)));
+                }
+                dr_insert_clean_call(drcontext, bb, instr, (void *)save_test,
+                                     true /* try saving the fp state */, 0);
+                for (i = SPILL_SLOT_1; i <= SPILL_SLOT_MAX; i++) {
+                    /* test using opnd */
+                    if (i < dr_max_opnd_accessible_spill_slot()) {
+                        PREM(bb, INSTR_CREATE_cmp
+                             (drcontext, dr_reg_spill_slot_opnd(drcontext, i),
+                              (i % 2 == 0) ?
+                              OPND_CREATE_INT8(100 - i) :
+                              OPND_CREATE_INT8(i + 1 - SPILL_SLOT_1)));
+                        PREM(bb, INSTR_CREATE_jcc
+                             (drcontext, OP_jne, opnd_create_instr(fault)));
+                    }
+                    /* test using restore routine */
+                    dr_restore_reg(drcontext, bb, instr, REG_XAX, i);
+                    PREM(bb, INSTR_CREATE_cmp
+                         (drcontext, opnd_create_reg(REG_XAX),
+                          (i % 2 == 0) ?
+                          OPND_CREATE_INT8(100 - i) :
+                          OPND_CREATE_INT8(i + 1 - SPILL_SLOT_1)));
+                    PREM(bb, INSTR_CREATE_jcc
+                         (drcontext, OP_jne, opnd_create_instr(fault)));
+                }
+                PREM(bb, INSTR_CREATE_jmp_short
+                     (drcontext, opnd_create_instr(post_fault)));
+                PRE(bb, fault); /* pre not prem since we want this to be an app fault */
+                PREM(bb, post_fault);
+
+                /* Now juggle xax and flags back from client tls */
+                dr_save_reg(drcontext, bb, instr, REG_XBX, SPILL_SLOT_1);
+                dr_insert_read_tls_field(drcontext, bb, instr, REG_XBX);
+                PREM(bb, INSTR_CREATE_mov_ld
+                     (drcontext, opnd_create_reg(REG_XAX),
+                      opnd_create_base_disp(REG_XBX, REG_NULL, 0,
+                                            1 * sizeof(reg_t), OPSZ_PTR)));
+                dr_restore_arith_flags(drcontext, bb, instr, SPILL_SLOT_MAX);
+                PREM(bb, INSTR_CREATE_mov_ld
+                     (drcontext, opnd_create_reg(REG_XAX),
+                      opnd_create_base_disp(REG_XBX, REG_NULL, 0,
+                                            0 * sizeof(reg_t), OPSZ_PTR)));
+                dr_restore_reg(drcontext, bb, instr, REG_XBX, SPILL_SLOT_1);
+
+#if VERBOSE /* for debugging */
+                instrlist_disassemble(drcontext, tag, bb, dr_get_stdout_file());
+#endif
+                post_crash++; /* note we don't actually crash so this must be the
+                               * last test */
+            }
+        }
+    }
+
+    if (modified) /* store since not constant instrumentation */
+        return DR_EMIT_STORE_TRANSLATIONS;
+    else
+        return DR_EMIT_DEFAULT;
+}
+
+static void
+thread_exit(void *drcontext)
+{
+    dr_thread_free(drcontext, dr_get_tls_field(drcontext), 3 * sizeof(reg_t));
+}
+
+static void
+thread_start(void *drcontext)
+{
+    dr_set_tls_field(drcontext, dr_thread_alloc(drcontext, 3 * sizeof(reg_t)));
+}
+
+DR_EXPORT void
+dr_init(client_id_t id)
+{
+    dr_register_bb_event(bb_event);
+    dr_register_thread_init_event(thread_start);
+    dr_register_thread_exit_event(thread_exit);
+    dr_register_restore_state_event(restore_state_event);
+}
diff --git a/suite/tests/client-interface/cleancall.template b/suite/tests/client-interface/cleancall.template
new file mode 100644
index 000000000..56bdb21af
--- /dev/null
+++ b/suite/tests/client-interface/cleancall.template
@@ -0,0 +1,30 @@
+testing clean calls
+inserting clean call crash code 1
+in restore_state for our clean call crash 1
+Access violation, instance 1
+testing clean calls
+inserting clean call crash code 2
+in restore_state for our clean call crash 2
+Access violation, instance 2
+testing clean calls
+inserting xsp arg testing
+#if +X64 : 
+bar 0x00000000bcbcaba1 0xffffffffbcbcaba0
+#else
+bar 0xbcbcaba1 0xbcbcaba0
+#endif
+in restore_state for our clean call crash 3
+Access violation, instance 3
+testing clean calls
+#if +X64 : 
+inserting clean call arg testing
+foo 0x0000000000000008 0x0000000000000001 0xffffffffdeadbeef 0xffffffffffffffd6
+    0x00000000cafebabe 0x00000000badcabee 0x00000000eeeeeeee 0x00000000feedadad
+#endif
+in restore_state for our clean call crash 4
+Access violation, instance 4
+testing clean calls
+inserting saved reg access testing
+verifying values
+did not crash
+done testing clean calls
diff --git a/suite/tests/client-interface/count-ctis.c b/suite/tests/client-interface/count-ctis.c
new file mode 100644
index 000000000..c1fadf26e
--- /dev/null
+++ b/suite/tests/client-interface/count-ctis.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <stdio.h>
+
+int main()
+{
+    printf("thank you for testing the client interface\n");
+}
diff --git a/suite/tests/client-interface/count-ctis.dll.c b/suite/tests/client-interface/count-ctis.dll.c
new file mode 100644
index 000000000..e0c48c166
--- /dev/null
+++ b/suite/tests/client-interface/count-ctis.dll.c
@@ -0,0 +1,161 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+
+#define MINSERT instrlist_meta_preinsert
+
+#ifdef WINDOWS
+# define ASSERT(x)                                        \
+  if (!(x)) {                                             \
+    dr_messagebox("ASSERT failed on line %d", __LINE__);  \
+    dr_abort();                                           \
+  }
+#else /* LINUX */
+# define ASSERT(x)                                            \
+  if (!(x)) {                                                 \
+    dr_fprintf(STDERR, "ASSERT failed on line %d", __LINE__); \
+    dr_abort();                                               \
+  }
+#endif
+
+uint num_call_dir = 0;
+uint num_call_ind = 0;
+uint num_jump_dir = 0;
+uint num_jump_ind = 0;
+uint num_br_cond  = 0;
+uint num_ret      = 0;
+
+static void at_call_dir(app_pc src, app_pc dst)
+{
+    num_call_dir++;
+}
+
+static void at_call_ind(app_pc src, app_pc dst)
+{
+    num_call_ind++;
+}
+
+static void at_jump_dir(app_pc src, app_pc dst)
+{
+    num_jump_dir++;
+}
+
+static void at_jump_ind(app_pc src, app_pc dst)
+{
+    num_jump_ind++;
+}
+
+static void at_br_cond(app_pc src, app_pc dst, int taken)
+{
+    num_br_cond++;
+}
+
+static void at_ret(app_pc src, app_pc dst)
+{
+    num_ret++;
+}
+
+static
+dr_emit_flags_t bb_event(void *drcontext, void *tag, instrlist_t *bb, bool for_trace, bool translating)
+{
+    instr_t *instr, *next_instr;
+    
+    for (instr = instrlist_first(bb);
+         instr != NULL; instr = next_instr) {
+        next_instr = instr_get_next(instr);
+
+        if (instr_is_cti(instr)) {
+ 
+            if (instr_is_ubr(instr)) {
+                dr_insert_ubr_instrumentation(drcontext, bb, instr, at_jump_dir);
+            }
+
+            else if (instr_is_call_direct(instr)) {
+                dr_insert_call_instrumentation(drcontext, bb, instr, at_call_dir);
+            }
+            
+            else if (instr_is_call_indirect(instr)) {
+                dr_insert_mbr_instrumentation(drcontext, bb, instr, at_call_ind,
+                                              SPILL_SLOT_1);
+            }
+
+            else if (instr_is_return(instr)) {
+                dr_insert_mbr_instrumentation(drcontext, bb, instr, at_ret,
+                                              SPILL_SLOT_1);
+            }
+
+            else if (instr_is_mbr(instr)) {
+                dr_insert_mbr_instrumentation(drcontext, bb, instr, at_jump_ind,
+                                              SPILL_SLOT_1);
+            }
+
+            else if (instr_is_cbr(instr)) {
+                dr_insert_cbr_instrumentation(drcontext, bb, instr, at_br_cond);
+            }
+
+            else {
+                ASSERT(false);
+            }
+        }
+    }
+    return DR_EMIT_DEFAULT;
+}
+
+static
+void check(uint count, char *str)
+{
+    dr_printf("%s... ", str);
+    if (count > 30) {
+        dr_printf("yes\n");
+    }
+    else {
+        dr_printf("no\n");
+    }
+}
+
+void exit_event(void)
+{
+    check(num_call_dir, "direct calls");
+    check(num_call_ind, "indirect calls");
+    check(num_jump_dir, "direct jumps");
+    check(num_jump_ind, "indirect jumps");
+    check(num_br_cond, "conditional branches");
+    check(num_ret, "returns");
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    dr_register_bb_event(bb_event);
+    dr_register_exit_event(exit_event);
+}
diff --git a/suite/tests/client-interface/count-ctis.template b/suite/tests/client-interface/count-ctis.template
new file mode 100644
index 000000000..3df3ebf9c
--- /dev/null
+++ b/suite/tests/client-interface/count-ctis.template
@@ -0,0 +1,11 @@
+thank you for testing the client interface
+direct calls... yes
+indirect calls... yes
+direct jumps... yes
+#if +WINDOWS :
+indirect jumps... yes
+#else
+indirect jumps... no
+#endif
+conditional branches... yes
+returns... yes
diff --git a/suite/tests/client-interface/custom_traces.dll.c b/suite/tests/client-interface/custom_traces.dll.c
new file mode 100644
index 000000000..dd8195f95
--- /dev/null
+++ b/suite/tests/client-interface/custom_traces.dll.c
@@ -0,0 +1,335 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Code Manipulation API Sample:
+ * trace-inline.c
+ *
+ * Uses the custom trace API to inline entire callees into traces.
+ */
+
+#include "dr_api.h"
+#include <string.h> /* memset */
+
+/****************************************************************************/
+/* global */
+
+static int num_complete_inlines;
+
+static void *htable_mutex; /* for multithread support */
+
+static void event_exit(void);
+static dr_emit_flags_t event_basic_block(void *drcontext, void *tag, instrlist_t *bb,
+                                         bool for_trace, bool translating);
+static void event_fragment_deleted(void *drcontext, void *tag);
+static dr_custom_trace_action_t
+query_end_trace(void *drcontext, void * trace_tag, void *next_tag);
+
+/****************************************************************************/
+/* hashtable so we know if a particular tag is for a call trace or a
+ * normal back branch trace
+ */
+
+typedef struct _trace_head_entry_t {
+    void *tag;
+    bool is_trace_head;
+    bool has_ret;
+    /* have to end at next block after see return */
+    int end_next;
+    /* some callees are too large to inline, so have size limit */
+    uint size;
+    struct _trace_head_entry_t *next;
+} trace_head_entry_t;
+
+/* global table */
+static trace_head_entry_t **htable;
+
+/* max call-trace size */
+#define INLINE_SIZE_LIMIT (4*1024)
+
+/* no instruction alignment -> use the lsb! */
+#define HASH_MASK(num_bits) ((~0U)>>(32-(num_bits)))
+#define HASH_FUNC_BITS(val, num_bits) ((val) & (HASH_MASK(num_bits)))
+#define HASH_FUNC(val, mask) ((val) & (mask))
+#define HASHTABLE_SIZE(num_bits) (1U << (num_bits))
+
+#define HASH_BITS 13
+#define TABLE_SIZE HASHTABLE_SIZE(HASH_BITS) * sizeof(trace_head_entry_t*)
+
+/* if drcontext == NULL uses global memory */
+static trace_head_entry_t **
+htable_create(void *drcontext)
+{
+    trace_head_entry_t **table = (trace_head_entry_t**) dr_global_alloc(TABLE_SIZE);
+    /* assume during process init so no lock needed */
+    memset(table, 0, TABLE_SIZE);
+    return table;
+}
+
+/* if drcontext == NULL uses global memory */
+static void
+htable_free(void *drcontext, trace_head_entry_t **table)
+{
+    /* assume during process exit so no lock needed */
+    int i;
+    /* clean up memory */
+    for (i = 0; i < HASHTABLE_SIZE(HASH_BITS); i++) {
+	trace_head_entry_t *e = table[i];
+	while (e) {
+	    trace_head_entry_t *nexte = e->next;
+	    dr_global_free(e, sizeof(trace_head_entry_t));
+	    e = nexte;
+	}
+	table[i] = NULL;
+    }
+    dr_global_free(table, TABLE_SIZE);
+}
+
+/* Caller must hold htable_mutex if drcontext == NULL. */
+static trace_head_entry_t *
+add_trace_head_entry(void *drcontext, void *tag)
+{
+    trace_head_entry_t **table = htable;
+    trace_head_entry_t *e;
+    uint hindex;
+    e = (trace_head_entry_t *) dr_global_alloc(sizeof(trace_head_entry_t));
+    e->tag = tag;
+    e->end_next = 0;
+    e->size = 0;
+    e->has_ret = false;
+    e->is_trace_head = false;
+    hindex = HASH_FUNC_BITS((ptr_uint_t)tag, HASH_BITS);
+    e->next = table[hindex];
+    table[hindex] = e;
+    return e;
+}
+
+/* Lookup an entry by pc and return a pointer to the corresponding entry .
+ * Returns NULL if no such entry exists.
+ * Caller must hold htable_mutex if drcontext == NULL.
+ */
+static trace_head_entry_t *
+lookup_trace_head_entry(void *drcontext, void *tag)
+{
+    trace_head_entry_t **table = htable;
+    trace_head_entry_t *e;
+    uint hindex;
+    hindex = HASH_FUNC_BITS((ptr_uint_t)tag, HASH_BITS);
+    for (e = table[hindex]; e; e = e->next) {
+	if (e->tag == tag)
+	    return e;
+    }
+    return NULL;
+}
+
+/* Lookup an entry by tag and index and delete it.
+ * Returns false if no such entry exists.
+ * Caller must hold htable_mutex if drcontext == NULL.
+ */
+static bool
+remove_trace_head_entry(void *drcontext, void *tag)
+{
+    trace_head_entry_t **table = htable;
+    trace_head_entry_t *e, *prev;
+    uint hindex;
+    hindex = HASH_FUNC_BITS((ptr_uint_t)tag, HASH_BITS);
+    for (prev = NULL, e = table[hindex]; e; prev = e, e = e->next) {
+	if (e->tag == tag) {
+	    if (prev)
+		prev->next = e->next;
+	    else
+		table[hindex] = e->next;
+	    dr_global_free(e, sizeof(trace_head_entry_t));
+	    return true;
+	}
+    }
+    return false;
+}
+
+/****************************************************************************/
+
+DR_EXPORT void 
+dr_init(client_id_t id)
+{
+    htable_mutex = dr_mutex_create();
+
+    /* global HASH_BITS-bit addressed hash table */
+    htable = htable_create(NULL/*global*/);
+
+    dr_register_exit_event(event_exit);
+    dr_register_bb_event(event_basic_block);
+    dr_register_delete_event(event_fragment_deleted);
+    dr_register_end_trace_event(query_end_trace);
+
+    /* make it easy to tell, by looking at log file, which client executed */
+    dr_log(NULL, LOG_ALL, 1, "Client 'inline' initializing\n");
+    num_complete_inlines = 0;
+}
+
+static void 
+event_exit(void)
+{
+    /* On WOW64xpsp2 I see 440+, but only 230+ on 2k3 */
+    if (num_complete_inlines > 100)
+        dr_printf("Inlined callees in >100 traces\n");
+    else
+        dr_printf("Inlined callees in %d traces: < 100!!!\n", num_complete_inlines);
+    htable_free(NULL/*global*/, htable);
+    dr_mutex_destroy(htable_mutex);
+}
+
+/****************************************************************************/
+/* the work itself */
+
+static dr_emit_flags_t 
+event_basic_block(void *drcontext, void *tag, instrlist_t *bb, bool for_trace, bool translating)
+{
+    instr_t *instr;
+    trace_head_entry_t *e = NULL;
+    if (translating)
+        return DR_EMIT_DEFAULT;
+    for (instr = instrlist_first(bb); instr != NULL; instr = instr_get_next(instr)) {
+	/* blocks containing calls are trace heads */
+	if (instr_is_call(instr)) {
+	    dr_mark_trace_head(drcontext, tag);
+            dr_mutex_lock(htable_mutex);
+	    e = add_trace_head_entry(NULL, tag);
+	    e->is_trace_head = true;
+            dr_mutex_unlock(htable_mutex);
+#ifdef VERBOSE
+	    dr_log(drcontext, LOG_ALL, 3,
+		   "inline: marking bb "PFX" as trace head\n", tag);
+#endif
+	    /* doesn't matter what's in rest of bb */
+	    return DR_EMIT_DEFAULT;
+	} else if (instr_is_return(instr)) {
+            dr_mutex_lock(htable_mutex);
+            e = add_trace_head_entry(NULL, tag);
+	    e->has_ret = true;
+            dr_mutex_unlock(htable_mutex);
+	}
+    }
+    return DR_EMIT_DEFAULT;
+}
+
+/* to keep the size of our hashtable down */
+static void
+event_fragment_deleted(void *drcontext, void *tag)
+{
+    dr_mutex_lock(htable_mutex);
+    remove_trace_head_entry(NULL, tag);
+    dr_mutex_unlock(htable_mutex);
+}
+
+/* Ask whether to end trace prior to adding next_tag fragment.
+ * Return values:
+ *   CUSTOM_TRACE_DR_DECIDES = use standard termination criteria
+ *   CUSTOM_TRACE_END_NOW   = end trace
+ *   CUSTOM_TRACE_CONTINUE  = do not end trace
+ */
+static dr_custom_trace_action_t
+query_end_trace(void *drcontext, void *trace_tag, void *next_tag)
+{
+    /* if this is a call trace, only end on the block AFTER a return
+     *   (need to get the return inlined!)
+     * if this is a standard back branch trace, end it if we see a
+     *   block with a call (so that we'll go into the call trace).
+     *   otherwise return 0 and let DynamoRIO determine whether to
+     *   terminate the trace now.
+     */
+    trace_head_entry_t *e;
+    dr_mutex_lock(htable_mutex);
+    e = lookup_trace_head_entry(NULL, trace_tag);
+    if (e == NULL || !e->is_trace_head) {
+	e = lookup_trace_head_entry(NULL, next_tag);
+	if (e == NULL || !e->is_trace_head) {
+            dr_mutex_unlock(htable_mutex);
+	    return CUSTOM_TRACE_DR_DECIDES;
+        } else {
+	    /* we've found a call, end this trace now so it won't keep going and
+	     * end up never entering the call trace
+	     */
+#ifdef VERBOSE
+	    dr_log(drcontext, LOG_ALL, 3,
+		   "inline: ending trace "PFX" before block "PFX" containing call\n",
+		   trace_tag, next_tag);
+#endif
+            dr_mutex_unlock(htable_mutex);
+	    return CUSTOM_TRACE_END_NOW;
+	}
+    } else if (e->end_next > 0) {
+	e->end_next--;
+	if (e->end_next == 0) {
+#ifdef VERBOSE
+	    dr_log(drcontext, LOG_ALL, 3,
+		   "inline: ending trace "PFX" before "PFX"\n",
+		   trace_tag, next_tag);
+#endif
+            num_complete_inlines++;
+            dr_mutex_unlock(htable_mutex);
+	    return CUSTOM_TRACE_END_NOW;
+	}
+    } else {
+	trace_head_entry_t *nxte = lookup_trace_head_entry(NULL, next_tag);
+	uint size = dr_fragment_size(drcontext, next_tag);
+	e->size += size;
+	if (e->size > INLINE_SIZE_LIMIT) {
+#ifdef VERBOSE
+	    dr_log(drcontext, LOG_ALL, 3,
+		   "inline: ending trace "PFX" before "PFX" because reached size limit\n",
+		   trace_tag, next_tag);
+#endif
+            dr_mutex_unlock(htable_mutex);
+	    return CUSTOM_TRACE_END_NOW;
+	}
+	if (nxte != NULL && nxte->has_ret && !nxte->is_trace_head) {
+	    /* end trace after NEXT block */
+	    e->end_next = 2;
+#ifdef VERBOSE
+	    dr_log(drcontext, LOG_ALL, 3,
+		   "inline: going to be ending trace "PFX" after "PFX"\n",
+		   trace_tag, next_tag);
+#endif
+            dr_mutex_unlock(htable_mutex);	
+            return CUSTOM_TRACE_CONTINUE;
+	}
+    }
+    /* do not end trace */
+#ifdef VERBOSE
+    dr_log(drcontext, LOG_ALL, 3,
+	   "inline: NOT ending trace "PFX" after "PFX"\n", trace_tag, next_tag);
+#endif
+    dr_mutex_unlock(htable_mutex);
+    return CUSTOM_TRACE_CONTINUE;
+}
+
+
+
diff --git a/suite/tests/client-interface/custom_traces.runall b/suite/tests/client-interface/custom_traces.runall
new file mode 100644
index 000000000..0c6d953b3
--- /dev/null
+++ b/suite/tests/client-interface/custom_traces.runall
@@ -0,0 +1,3 @@
+calc.exe
+Calculator
+<>
diff --git a/suite/tests/client-interface/custom_traces.template b/suite/tests/client-interface/custom_traces.template
new file mode 100644
index 000000000..4b8a01317
--- /dev/null
+++ b/suite/tests/client-interface/custom_traces.template
@@ -0,0 +1,18 @@
+#if +RUNREGRESSION_XP +PROGRAM_SHEPHERDING : +-security +-no_executable_if_rx_text --executable_after_load
+# SEC_VIO_STOP
+No such process found.
+Window not found: Calculator
+#else
+# if +DEBUG :
+Process calc.exe, running SC debug
+# else
+#  if +PROFILE :
+Process calc.exe, running SC profile
+#  else
+Process calc.exe, running SC release
+#  endif
+# endif
+Close message sent.
+#endif
+Inlined callees in >100 traces
+No such process found.
diff --git a/suite/tests/client-interface/decode-bb.dll.c b/suite/tests/client-interface/decode-bb.dll.c
new file mode 100644
index 000000000..3ba68f9f3
--- /dev/null
+++ b/suite/tests/client-interface/decode-bb.dll.c
@@ -0,0 +1,147 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+
+static void *last_trace_mutex;
+static void *last_trace_tag;
+static instrlist_t *last_trace_ilist;
+static void *last_trace_drcontext;
+
+static void
+verify_identical(instrlist_t *il1, instrlist_t *il2, app_pc label)
+{
+    instr_t *i = instrlist_first(il1);
+    instr_t *j = instrlist_first(il2);
+
+    while (i != NULL && j != NULL) {
+        app_pc i_addr = instr_get_app_pc(i);
+        app_pc j_addr = instr_get_app_pc(j);
+
+        if (i_addr != j_addr || !instr_same(i, j)) {
+            break;
+        }
+
+        i = instr_get_next(i);
+        j = instr_get_next(j);
+    }
+
+    if (i != NULL || j != NULL) {
+        dr_printf("ERROR: mismatch in block at "PFX"\n", label);
+    }
+}
+
+static
+dr_emit_flags_t bb_event(void *drcontext, void *tag, instrlist_t *bb,
+                         bool for_trace, bool translating)
+{
+    if (!translating) {
+        app_pc pc = dr_fragment_app_pc(tag);
+        
+        instrlist_t *copy = decode_as_bb(drcontext, pc);
+        verify_identical(bb, copy, pc);
+        instrlist_clear_and_destroy(drcontext, copy);
+        
+        dr_mutex_lock(last_trace_mutex);
+        if (last_trace_tag != NULL) {
+            pc = dr_fragment_app_pc(last_trace_tag);
+            if (pc != NULL) {
+                copy = decode_trace(drcontext, pc);
+                if (copy != NULL) {
+                    verify_identical(last_trace_ilist, copy, pc);
+                    instrlist_clear_and_destroy(drcontext, copy);
+                }
+            }
+            instrlist_clear_and_destroy(last_trace_drcontext, last_trace_ilist);
+            last_trace_tag = NULL;
+        }
+        dr_mutex_unlock(last_trace_mutex);
+    }
+
+    return DR_EMIT_DEFAULT;
+}
+
+static
+dr_emit_flags_t trace_event(void *drcontext, void *tag, instrlist_t *trace,
+                            bool translating)
+{
+    instr_t *i;
+    for (i = instrlist_first(trace); i != NULL; i = instr_get_next(i)) {
+        if (instr_get_app_pc(i) == NULL) {
+            dr_printf("ERROR: app pc not available for all trace instrs "PFX"\n",
+                      dr_fragment_app_pc(tag));
+        }
+    }
+
+    if (!translating) {
+        /* we can't call decode_trace() until after it's emitted, so remember the tag
+         * and call it in the next bb hook
+         */
+        dr_mutex_lock(last_trace_mutex);
+        if (last_trace_tag != NULL)
+            instrlist_clear_and_destroy(last_trace_drcontext, last_trace_ilist);
+        last_trace_tag = tag;
+        last_trace_ilist = instrlist_clone(drcontext, trace);
+        last_trace_drcontext = drcontext;
+        dr_mutex_unlock(last_trace_mutex);
+    }
+
+    return DR_EMIT_DEFAULT;
+}
+
+static void
+dr_thread_exit(void *drcontext)
+{
+    dr_mutex_lock(last_trace_mutex);
+    if (last_trace_tag != NULL && last_trace_drcontext == drcontext) {
+        instrlist_clear_and_destroy(last_trace_drcontext, last_trace_ilist);
+        last_trace_tag = NULL;
+    }
+    dr_mutex_unlock(last_trace_mutex);
+}
+
+static void
+dr_exit(void)
+{
+    dr_mutex_destroy(last_trace_mutex);
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    last_trace_mutex = dr_mutex_create();
+    dr_printf("thank you for testing the client interface\n");
+    dr_register_bb_event(bb_event);
+    dr_register_trace_event(trace_event);
+    dr_register_thread_exit_event(dr_thread_exit);
+    dr_register_exit_event(dr_exit);
+}
diff --git a/suite/tests/client-interface/decode-bb.runall b/suite/tests/client-interface/decode-bb.runall
new file mode 100644
index 000000000..787417df8
--- /dev/null
+++ b/suite/tests/client-interface/decode-bb.runall
@@ -0,0 +1,3 @@
+notepad.exe
+Untitled - Notepad
+<>
diff --git a/suite/tests/client-interface/decode-bb.template b/suite/tests/client-interface/decode-bb.template
new file mode 100644
index 000000000..a203ad3b1
--- /dev/null
+++ b/suite/tests/client-interface/decode-bb.template
@@ -0,0 +1,18 @@
+thank you for testing the client interface
+#if +RUNREGRESSION_XP +PROGRAM_SHEPHERDING : +-security +-no_executable_if_rx_text --executable_after_load
+# SEC_VIO_STOP
+No such process found.
+Window not found: Untitled - Notepad
+#else
+# if +DEBUG :
+Process notepad.exe, running SC debug
+# else
+#  if +PROFILE :
+Process notepad.exe, running SC profile
+#  else
+Process notepad.exe, running SC release
+#  endif
+# endif
+Close message sent.
+#endif
+No such process found.
diff --git a/suite/tests/client-interface/events.c b/suite/tests/client-interface/events.c
new file mode 100644
index 000000000..4b02e979a
--- /dev/null
+++ b/suite/tests/client-interface/events.c
@@ -0,0 +1,210 @@
+/* **********************************************************
+ * Copyright (c) 2008-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * API regression test that registers for all supported event callbacks
+ * (except the nudge and security violation callback)
+ */
+
+#ifdef WINDOWS
+# include <windows.h>
+#else
+# include <stdlib.h>
+# include <stdio.h>
+# include <string.h>
+# include <unistd.h>
+# include <dlfcn.h>
+# include <signal.h>
+# include <ucontext.h>
+# include <assert.h>
+#endif
+
+#ifdef LINUX
+/* handler with SA_SIGINFO flag set gets three arguments: */
+typedef void (*handler_t)(int, struct siginfo *, void *);
+
+static void
+signal_handler(int sig, siginfo_t *siginfo, ucontext_t *ucxt)
+{
+    if (sig == SIGUSR1)
+	fprintf(stderr, "Got SIGUSR1\n");
+    else if (sig == SIGUSR2)
+	fprintf(stderr, "Got SIGUSR2\n");
+    else if (sig == SIGURG)
+	fprintf(stderr, "Got SIGURG\n");
+}
+
+/* set up signal_handler as the handler for signal "sig" */
+static void
+intercept_signal(int sig, handler_t handler)
+{
+    int rc;
+    struct sigaction act;
+    act.sa_sigaction = handler;
+    rc = sigfillset(&act.sa_mask); /* block all signals within handler */
+    assert(rc == 0);
+    act.sa_flags = SA_SIGINFO | SA_ONSTACK; /* send 3 args to handler */
+    /* arm the signal */
+    rc = sigaction(sig, &act, NULL);
+    assert(rc == 0);
+}
+#endif /* LINUX */
+
+int
+main(int argc, char** argv)
+{
+#ifdef WINDOWS
+    HMODULE hmod;
+
+    /* 
+     * Cause an exception event
+     */
+    __try {
+        HANDLE heap = GetProcessHeap();
+        char *buf = (char *)HeapAlloc(((char *)heap)+1, HEAP_GENERATE_EXCEPTIONS, 10);
+    }
+    __except (GetExceptionCode() == STATUS_ACCESS_VIOLATION ? 
+              EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) {
+    }
+
+    /* 
+     * Load and unload a module to cause a module unload event
+     */
+    hmod = LoadLibrary("shell32.dll");
+    FreeLibrary(hmod);
+#endif
+
+#ifdef LINUX
+    void *hmod;
+    char buf[1000];
+    size_t len = 0;
+    char *end_path = NULL;
+    /* 
+     * Load and unload a module to cause a module unload event
+     */
+#if 1
+    strncpy(buf, argv[0], sizeof(buf));
+    buf[sizeof(buf)-1] = '\0';
+    end_path = strrchr(buf, '/');
+    if (end_path == NULL)
+        len = 0;
+    else
+        len = (end_path - buf) + 1;
+#else
+    getcwd(buf, sizeof(buf));
+    buf[sizeof(buf)-1] = '\0';
+    len = strlen(buf);
+    strncpy(buf+len, "/../client-interface/", sizeof(buf)-len);
+    buf[sizeof(buf)-1] = '\0';
+    len = strlen(buf);
+#endif
+
+      /* small .bss */
+#ifdef X64
+    strncpy(buf+len, "events64_dummy1.so", sizeof(buf)-len); 
+#else
+    strncpy(buf+len, "events_dummy1.so", sizeof(buf)-len);
+#endif
+    buf[sizeof(buf)-1] = '\0';
+    hmod = dlopen(buf, RTLD_LAZY|RTLD_LOCAL);
+    if (hmod != NULL)
+	dlclose(hmod);
+    else
+	printf("module load of %s failed\n", buf);
+    
+    /* large .bss */
+#ifdef X64
+    strncpy(buf+len, "events64_dummy2.so", sizeof(buf)-len); 
+#else
+    strncpy(buf+len, "events_dummy2.so", sizeof(buf)-len);
+#endif
+    buf[sizeof(buf)-1] = '\0';
+    hmod = dlopen(buf, RTLD_LAZY|RTLD_LOCAL);
+    if (hmod != NULL)
+	dlclose(hmod);
+    else
+	printf("module load of %s failed\n", buf);
+
+    /* test load of non-existent file */
+    hmod = dlopen("foo_bar_no_exist.so", RTLD_LAZY|RTLD_LOCAL);
+    if (hmod != NULL) {
+	printf("ERROR - module load of %s succeeded\n", buf);
+	dlclose(hmod);
+    }
+
+    intercept_signal(SIGUSR1, (handler_t) signal_handler);
+    intercept_signal(SIGUSR2, (handler_t) signal_handler);
+    intercept_signal(SIGURG, (handler_t) signal_handler);
+    fprintf(stderr, "Sending SIGUSR1\n");
+    kill(getpid(), SIGUSR1);
+    fprintf(stderr, "Sending SIGUSR2\n");
+    kill(getpid(), SIGUSR2);
+    fprintf(stderr, "Sending SIGURG\n");
+    kill(getpid(), SIGURG);
+    fprintf(stderr, "Done\n");
+
+    /*
+     * Cause a fork event
+     */
+    if (fork() == 0) {
+        wait(NULL);
+    }
+    else {
+        abort();
+    }
+#endif /* LINUX */
+
+#ifdef WINDOWS
+    /* 
+     * Cause an exception event, we test redirecting the application to redirect()
+     */
+    __try {
+        HANDLE heap = GetProcessHeap();
+        char *buf = (char *)HeapAlloc(((char *)heap)+1, HEAP_GENERATE_EXCEPTIONS, 10);
+    }
+    __except (GetExceptionCode() == STATUS_ACCESS_VIOLATION ? 
+              EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) {
+    }
+    /* Never Reached */
+    printf("Shouldn't be reached\n");
+#endif
+}
+
+#ifdef WINDOWS
+__declspec(dllexport) 
+void
+redirect()
+{
+    printf("Redirect success!\n");
+    exit(0);
+}
+#endif
diff --git a/suite/tests/client-interface/events.dll.c b/suite/tests/client-interface/events.dll.c
new file mode 100644
index 000000000..71f7c9384
--- /dev/null
+++ b/suite/tests/client-interface/events.dll.c
@@ -0,0 +1,403 @@
+/* **********************************************************
+ * Copyright (c) 2008-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * API regression test that registers for all supported event callbacks
+ * (except the nudge and security violation callback)
+ */
+
+#include "dr_api.h"
+
+static
+void exit_event1(void)
+{
+    dr_printf("exit event 1\n");
+    if (!dr_unregister_exit_event(exit_event1))
+        dr_printf("unregister failed!\n");
+}
+
+static
+void exit_event2(void)
+{
+    dr_printf("exit event 2\n");
+    if (!dr_unregister_exit_event(exit_event2))
+        dr_printf("unregister failed!\n");
+}
+
+static
+void thread_init_event1(void *drcontext)
+{
+    dr_printf("thread init event 1\n");
+    if (!dr_unregister_thread_init_event(thread_init_event1))
+        dr_printf("unregister failed!\n");
+}
+static
+void thread_init_event2(void *drcontext)
+{
+    dr_printf("thread init event 2\n");
+    if (!dr_unregister_thread_init_event(thread_init_event2))
+        dr_printf("unregister failed!\n");
+}
+
+static
+void thread_exit_event1(void *drcontext)
+{
+    dr_printf("thread exit event 1\n");
+    if (!dr_unregister_thread_exit_event(thread_exit_event1))
+        dr_printf("unregister failed!\n");
+}
+
+static
+void thread_exit_event2(void *drcontext)
+{
+    dr_printf("thread exit event 2\n");
+    if (!dr_unregister_thread_exit_event(thread_exit_event2))
+        dr_printf("unregister failed!\n");
+}
+
+#ifdef LINUX
+static
+void fork_init_event1(void *drcontext)
+{
+    dr_printf("fork init event 1\n");
+    if (!dr_unregister_fork_init_event(fork_init_event1))
+        dr_printf("unregister failed!\n");
+}
+static
+void fork_init_event2(void *drcontext)
+{
+    dr_printf("fork init event 2\n");
+    if (!dr_unregister_fork_init_event(fork_init_event2))
+        dr_printf("unregister failed!\n");
+}
+
+#endif
+static
+dr_emit_flags_t bb_event1(void *dcontext, void *tag, instrlist_t *bb,
+                          bool for_trace, bool translating)
+{
+    dr_printf("bb event 1\n");
+    if (!dr_unregister_bb_event(bb_event1))
+        dr_printf("unregister failed!\n");
+    return DR_EMIT_DEFAULT;
+}
+
+static
+dr_emit_flags_t bb_event2(void *dcontext, void *tag, instrlist_t *bb,
+                          bool for_trace, bool translating)
+{
+    dr_printf("bb event 2\n");
+    if (!dr_unregister_bb_event(bb_event2))
+        dr_printf("unregister failed!\n");
+    return DR_EMIT_DEFAULT;
+}
+
+static
+dr_emit_flags_t trace_event1(void *dcontext, void *tag, instrlist_t *trace,
+                             bool translating)
+{
+    dr_printf("trace event 1\n");
+    if (!dr_unregister_trace_event(trace_event1))
+        dr_printf("unregister failed!\n");
+    return DR_EMIT_DEFAULT;
+}
+
+static
+dr_emit_flags_t trace_event2(void *dcontext, void *tag, instrlist_t *trace,
+                             bool translating)
+{
+    dr_printf("trace event 2\n");
+    if (!dr_unregister_trace_event(trace_event2))
+        dr_printf("unregister failed!\n");
+    return DR_EMIT_DEFAULT;
+}
+
+static
+dr_custom_trace_action_t end_trace_event1(void *dcontext, void *trace_tag, void *next_tag)
+{
+    dr_printf("end trace event 1\n");
+    if (!dr_unregister_end_trace_event(end_trace_event1))
+        dr_printf("unregister failed!\n");
+    return 0;
+}
+
+static
+dr_custom_trace_action_t end_trace_event2(void *dcontext, void *trace_tag, void *next_tag)
+{
+    dr_printf("end trace event 2\n");
+    if (!dr_unregister_end_trace_event(end_trace_event2))
+        dr_printf("unregister failed!\n");
+    return 0;
+}
+
+static
+void delete_event1(void *dcontext, void *tag)
+{
+    dr_printf("delete event 1\n");
+    if (!dr_unregister_delete_event(delete_event1))
+        dr_printf("unregister failed!\n");
+}
+static
+void delete_event2(void *dcontext, void *tag)
+{
+    dr_printf("delete event 2\n");
+    if (!dr_unregister_delete_event(delete_event2))
+        dr_printf("unregister failed!\n");
+}
+
+static
+void module_load_event1(void *drcontext, const module_data_t *info, bool loaded)
+{
+    dr_printf("module load event 1\n");
+    if (!dr_unregister_module_load_event(module_load_event1))
+        dr_printf("unregister failed!\n");
+}
+
+static
+void module_load_event2(void *drcontext, const module_data_t *info, bool loaded)
+{
+    dr_printf("module load event 2\n");
+    if (!dr_unregister_module_load_event(module_load_event2))
+        dr_printf("unregister failed!\n");
+}
+
+static
+void module_unload_event1(void *drcontext, const module_data_t *info)
+{
+    dr_printf("module unload event 1\n");
+    if (!dr_unregister_module_unload_event(module_unload_event1))
+        dr_printf("unregister failed!\n");
+}
+
+static
+void module_unload_event2(void *drcontext, const module_data_t *info)
+{
+    dr_printf("module unload event 2\n");
+    if (!dr_unregister_module_unload_event(module_unload_event2))
+        dr_printf("unregister failed!\n");
+}
+
+static
+bool pre_syscall_event1(void *drcontext, int sysnum)
+{
+    dr_printf("pre syscall event 1\n");
+    if (!dr_unregister_pre_syscall_event(pre_syscall_event1))
+        dr_printf("unregister failed!\n");
+    return true;
+}
+
+static
+bool pre_syscall_event2(void *drcontext, int sysnum)
+{
+    dr_printf("pre syscall event 2\n");
+    if (!dr_unregister_pre_syscall_event(pre_syscall_event2))
+        dr_printf("unregister failed!\n");
+    return true;
+}
+
+static
+void post_syscall_event1(void *drcontext, int sysnum)
+{
+    dr_printf("post syscall event 1\n");
+    if (!dr_unregister_post_syscall_event(post_syscall_event1))
+        dr_printf("unregister failed!\n");
+}
+
+static
+void post_syscall_event2(void *drcontext, int sysnum)
+{
+    dr_printf("post syscall event 2\n");
+    if (!dr_unregister_post_syscall_event(post_syscall_event2))
+        dr_printf("unregister failed!\n");
+}
+
+static
+bool filter_syscall_event1(void *drcontext, int sysnum)
+{
+    dr_printf("filter syscall event 1\n");
+    if (!dr_unregister_filter_syscall_event(filter_syscall_event1))
+        dr_printf("unregister failed!\n");
+    return true;
+}
+
+static
+bool filter_syscall_event2(void *drcontext, int sysnum)
+{
+    dr_printf("filter syscall event 2\n");
+    if (!dr_unregister_filter_syscall_event(filter_syscall_event2))
+        dr_printf("unregister failed!\n");
+    return true;
+}
+
+
+#ifdef WINDOWS
+static
+void exception_event_redirect(void *dcontext, dr_exception_t *excpt)
+{
+    app_pc addr;
+    dr_mcontext_t mcontext;
+    module_data_t *data = dr_lookup_module_by_name("events.exe");
+    dr_printf("exception event redirect\n");
+    if (data == NULL) {
+        dr_printf("couldn't find events.exe module\n");
+        return;
+    }
+    addr = (app_pc)dr_get_proc_address(data->handle, "redirect");
+    dr_free_module_data(data);
+    mcontext = excpt->mcontext;
+    mcontext.pc = addr;
+    if (addr == NULL) {
+        dr_printf("Couldn't find function redirect in events.exe\n");
+        return;
+    }
+    dr_redirect_execution(&mcontext, 0);
+    dr_printf("should not be reached, dr_redirect_execution() should not return\n");
+}
+
+static
+void exception_event1(void *dcontext, dr_exception_t *excpt)
+{
+    if (excpt->record->ExceptionCode == STATUS_ACCESS_VIOLATION)
+        dr_printf("exception event 1\n");
+
+    if (!dr_unregister_exception_event(exception_event1))
+        dr_printf("unregister failed!\n");
+
+    /* ensure we get our deletion events */
+    dr_flush_region(excpt->record->ExceptionAddress, 1);
+}
+
+static
+void exception_event2(void *dcontext, dr_exception_t *excpt)
+{    
+    if (excpt->record->ExceptionCode == STATUS_ACCESS_VIOLATION)
+        dr_printf("exception event 2\n");
+    
+    dr_register_exception_event(exception_event_redirect);
+
+    if (!dr_unregister_exception_event(exception_event2))
+        dr_printf("unregister failed!\n");
+}
+#else
+static
+dr_signal_action_t signal_event1(void *dcontext, dr_siginfo_t *info)
+{
+    dr_printf("signal event 1 sig=%d\n", info->sig);
+
+    if (info->sig == SIGUSR2)
+        return DR_SIGNAL_SUPPRESS;
+    else if (info->sig == SIGURG) {
+        if (!dr_unregister_signal_event(signal_event1))
+            dr_printf("unregister failed!\n");
+        return DR_SIGNAL_BYPASS;
+    }
+    return DR_SIGNAL_DELIVER;
+}
+
+static
+dr_signal_action_t signal_event2(void *dcontext, dr_siginfo_t *info)
+{    
+    dr_printf("signal event 2 sig=%d\n", info->sig);
+    
+    if (!dr_unregister_signal_event(signal_event2))
+        dr_printf("unregister failed!\n");
+
+    return DR_SIGNAL_DELIVER;
+}
+#endif /* WINDOWS */
+
+static void
+restore_state_event1(void *drcontext, void *tag, dr_mcontext_t *mcontext,
+                     bool restore_memory, bool app_code_consistent)
+{
+    dr_printf("in restore_state event 1\n");
+
+    if (!dr_unregister_restore_state_event(restore_state_event1))
+        dr_printf("unregister failed!\n");
+}
+
+static void
+restore_state_event2(void *drcontext, void *tag, dr_mcontext_t *mcontext,
+                     bool restore_memory, bool app_code_consistent)
+{
+    dr_printf("in restore_state event 2\n");
+
+    if (!dr_unregister_restore_state_event(restore_state_event2))
+        dr_printf("unregister failed!\n");
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    /* FIXME: we should test the nudge events as well, but that
+     * would require some extra stuff our testing infrastructure
+     * doesn't currently support.
+     */
+
+    dr_register_exit_event(exit_event1);
+    dr_register_exit_event(exit_event2);
+    dr_register_thread_init_event(thread_init_event1);
+    dr_register_thread_init_event(thread_init_event2);
+    dr_register_thread_exit_event(thread_exit_event1);
+    dr_register_thread_exit_event(thread_exit_event2);
+#ifdef LINUX
+    dr_register_fork_init_event(fork_init_event1);
+    dr_register_fork_init_event(fork_init_event2);
+#endif
+    dr_register_bb_event(bb_event1);
+    dr_register_bb_event(bb_event2);
+    dr_register_trace_event(trace_event1);
+    dr_register_trace_event(trace_event2);
+    dr_register_end_trace_event(end_trace_event1);
+    dr_register_end_trace_event(end_trace_event2);
+    dr_register_delete_event(delete_event1);
+    dr_register_delete_event(delete_event2);
+    dr_register_restore_state_event(restore_state_event1);
+    dr_register_restore_state_event(restore_state_event2);
+    dr_register_module_load_event(module_load_event1);
+    dr_register_module_load_event(module_load_event2);
+    dr_register_module_unload_event(module_unload_event1);
+    dr_register_module_unload_event(module_unload_event2);
+    dr_register_pre_syscall_event(pre_syscall_event1);
+    dr_register_pre_syscall_event(pre_syscall_event2);
+    dr_register_post_syscall_event(post_syscall_event1);
+    dr_register_post_syscall_event(post_syscall_event2);
+    dr_register_filter_syscall_event(filter_syscall_event1);
+    dr_register_filter_syscall_event(filter_syscall_event2);
+#ifdef WINDOWS
+    dr_register_exception_event(exception_event1);
+    dr_register_exception_event(exception_event2);
+#else
+    dr_register_signal_event(signal_event1);
+    dr_register_signal_event(signal_event2);
+#endif
+}
diff --git a/suite/tests/client-interface/events.template b/suite/tests/client-interface/events.template
new file mode 100644
index 000000000..4c4597aeb
--- /dev/null
+++ b/suite/tests/client-interface/events.template
@@ -0,0 +1,68 @@
+#if +WINDOWS :
+module load event 2
+module load event 1
+thread init event 2
+thread init event 1
+bb event 2
+bb event 1
+end trace event 2
+end trace event 1
+delete event 2
+delete event 1
+trace event 2
+trace event 1
+filter syscall event 2
+filter syscall event 1
+pre syscall event 2
+pre syscall event 1
+post syscall event 2
+post syscall event 1
+in restore_state event 2
+in restore_state event 1
+exception event 2
+exception event 1
+module unload event 2
+module unload event 1
+exception event redirect
+Redirect success!
+thread exit event 2
+thread exit event 1
+exit event 2
+exit event 1
+#else
+module load event 2
+module load event 1
+thread init event 2
+thread init event 1
+bb event 2
+bb event 1
+end trace event 2
+end trace event 1
+delete event 2
+delete event 1
+trace event 2
+trace event 1
+filter syscall event 2
+filter syscall event 1
+pre syscall event 2
+pre syscall event 1
+post syscall event 2
+post syscall event 1
+module unload event 2
+module unload event 1
+Sending SIGUSR1
+signal event 2 sig=10
+signal event 1 sig=10
+Got SIGUSR1
+Sending SIGUSR2
+signal event 1 sig=12
+Sending SIGURG
+signal event 1 sig=23
+Done
+fork init event 2
+fork init event 1
+thread exit event 2
+thread exit event 1
+exit event 2
+exit event 1
+#endif
diff --git a/suite/tests/client-interface/events64_dummy1.so b/suite/tests/client-interface/events64_dummy1.so
new file mode 100755
index 000000000..85da0a231
Binary files /dev/null and b/suite/tests/client-interface/events64_dummy1.so differ
diff --git a/suite/tests/client-interface/events64_dummy2.so b/suite/tests/client-interface/events64_dummy2.so
new file mode 100755
index 000000000..871a32e50
Binary files /dev/null and b/suite/tests/client-interface/events64_dummy2.so differ
diff --git a/suite/tests/client-interface/events_dummy1.so b/suite/tests/client-interface/events_dummy1.so
new file mode 100755
index 000000000..37aebe1b9
Binary files /dev/null and b/suite/tests/client-interface/events_dummy1.so differ
diff --git a/suite/tests/client-interface/events_dummy2.so b/suite/tests/client-interface/events_dummy2.so
new file mode 100755
index 000000000..acd0c6507
Binary files /dev/null and b/suite/tests/client-interface/events_dummy2.so differ
diff --git a/suite/tests/client-interface/file_io.c b/suite/tests/client-interface/file_io.c
new file mode 100644
index 000000000..ff57e685c
--- /dev/null
+++ b/suite/tests/client-interface/file_io.c
@@ -0,0 +1,36 @@
+/* **********************************************************
+ * Copyright (c) 2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+int main()
+{
+    return 0;
+}
diff --git a/suite/tests/client-interface/file_io.dll.c b/suite/tests/client-interface/file_io.dll.c
new file mode 100644
index 000000000..74116e253
--- /dev/null
+++ b/suite/tests/client-interface/file_io.dll.c
@@ -0,0 +1,212 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+
+/* check if all bits in mask are set in var */
+#define TESTALL(mask, var) (((mask) & (var)) == (mask))
+/* check if any bit in mask is set in var */
+#define TESTANY(mask, var) (((mask) & (var)) != 0)
+/* check if a single bit is set in var */
+#define TEST TESTANY
+
+#if LINUX
+/* Need to declare memset.  It's intrinsic so should be ok. */
+void *memset(void *s, int c, size_t n);
+#endif
+
+byte * find_prot_edge(const byte *start, uint prot_flag)
+{
+    uint prot;
+    byte *base = (byte *)start;
+    byte *last;
+    size_t size = 0;
+
+    do {
+        last = base + size;
+    } while (dr_query_memory(last, &base, &size, &prot) && TESTALL(prot_flag, prot));
+    
+    if (last == start)
+        dr_printf("error in find_prot_edge");
+
+    return last;
+}
+
+bool
+memchk(byte *start, byte value, size_t size)
+{
+    size_t i;
+    for (i = 0; i < size && *(start+i) == value; i++) ;
+    return i == size;
+}
+
+const byte read_only_buf[2*PAGE_SIZE] = {0};
+
+/* NOTE - we need to initialize the writable buffers to some non-zero value so that they
+ * will all be placed in the same memory region (on Linux only the first page is part
+ * of the map and the remaining pages are just allocated instead of mapped if these are
+ * 0). */
+byte safe_buf[2*PAGE_SIZE+100] = {1};
+
+byte writable_buf[2*PAGE_SIZE] = {1};
+
+bool dummy_func()
+{
+    return true;
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    file_t file;
+    char buf[100];
+    int64 pos;
+    int i;
+    uint prot;
+    byte *base_pc;
+    size_t size;
+    size_t bytes_read, bytes_written;
+    byte *edge;
+
+    /* The Makefile will pass a full absolute path (for Windows and Linux) as the client
+     * option to a dummy file in the which we use to exercise the file api routines.
+     * TODO - these tests should be a lot more thorough, but the basic functionality
+     * is there (should add write tests, file_exists, directory etc. tests). */
+    file = dr_open_file(dr_get_options(id), DR_FILE_READ);
+    if (file == INVALID_FILE)
+        dr_printf("Error opening file\n");
+    memset(buf, 0, sizeof(buf));
+    dr_read_file(file, buf, 10);
+    pos = dr_file_tell(file);
+    if (pos < 0)
+        dr_printf("tell error\n");
+    dr_printf("%s\n", buf);
+    if (!dr_file_seek(file, 0, DR_SEEK_SET))
+        dr_printf("seek error\n");
+    memset(buf, 0, sizeof(buf));
+    dr_read_file(file, buf, 5);
+    dr_printf("%s\n", buf);
+    for (i = 0; i < 100; i++) buf[i] = 0;
+    if (!dr_file_seek(file, pos - 5, DR_SEEK_CUR))
+        dr_printf("seek error\n");
+    memset(buf, 0, sizeof(buf));
+    dr_read_file(file, buf, 7);
+    dr_printf("%s\n", buf);
+    if (!dr_file_seek(file, -6, DR_SEEK_END))
+        dr_printf("seek error\n");
+    memset(buf, 0, sizeof(buf));
+    dr_read_file(file, buf, 6);
+    dr_printf("%s\n", buf);
+    dr_close_file(file);
+
+    /* Test the memory query routines */
+    dummy_func();
+    if (!dr_memory_is_readable((byte *)dummy_func, 1) ||
+        !dr_memory_is_readable(read_only_buf+1000, 4000) ||
+        !dr_memory_is_readable(writable_buf+1000, 4000)) {
+        dr_printf("ERROR : dr_memory_is_readable() incorrect results\n");
+    }
+
+    if (!dr_query_memory((byte *)dummy_func, &base_pc, &size, &prot))
+        dr_printf("ERROR : can't find dummy_func mem region\n");
+    dr_printf("dummy_func is %s%s%s\n", TEST(DR_MEMPROT_READ, prot) ? "r" : "",
+              TEST(DR_MEMPROT_WRITE, prot) ? "w" : "",
+              TEST(DR_MEMPROT_EXEC, prot) ? "x" : "");
+    if (base_pc > (byte *)dummy_func || base_pc + size < (byte *)dummy_func)
+        dr_printf("dummy_func region mismatch");
+
+    memset(writable_buf, 0, sizeof(writable_buf)); /* strip off write copy */
+    if (!dr_query_memory(writable_buf+100, &base_pc, &size, &prot))
+        dr_printf("ERROR : can't find dummy_func mem region\n");
+    dr_printf("writable_buf is %s%s%s\n", TEST(DR_MEMPROT_READ, prot) ? "r" : "",
+              TEST(DR_MEMPROT_WRITE, prot) ? "w" : "",
+#if LINUX
+	      /* Linux sometimes (probably depends on version and hardware NX
+	       * support) lists all readable regions as also exectuable in the
+	       * maps file.  We just skip checking here for Linux to make
+	       * matching the template file easier. */
+	      ""
+#else
+              TEST(DR_MEMPROT_EXEC, prot) ? "x" : ""
+#endif
+	      );
+    if (base_pc > writable_buf || base_pc + size < writable_buf)
+        dr_printf("writable_buf region mismatch\n");
+    if (base_pc + size < writable_buf + sizeof(writable_buf))
+        dr_printf("writable_buf size mismatch "PFX" "PFX" "PFX" "PFX"\n",
+                  base_pc, size, writable_buf, sizeof(writable_buf));
+
+    if (!dr_query_memory(read_only_buf+100, &base_pc, &size, &prot))
+        dr_printf("ERROR : can't find dummy_func mem region\n");
+    dr_printf("read_only_buf is %s%s\n", TEST(DR_MEMPROT_READ, prot) ? "r" : "",
+              TEST(DR_MEMPROT_WRITE, prot) ? "w" : "");
+    if (base_pc > read_only_buf || base_pc + size < read_only_buf)
+        dr_printf("read_only_buf region mismatch");
+    if (base_pc + size < read_only_buf + sizeof(read_only_buf))
+        dr_printf("read_only_buf size mismatch");
+    
+    /* test the safe_read functions */
+    /* TODO - extend test to cover racy writes and reads (won't work on Linux yet). */
+    memset(safe_buf, 0xcd, sizeof(safe_buf));
+    if (!dr_safe_read(read_only_buf + 4000, 1000, safe_buf, &bytes_read) ||
+        bytes_read != 1000 || !memchk(safe_buf, 0, 1000) || *(safe_buf+1000) != 0xcd) {
+        dr_printf("ERROR in plain dr_safe_read()\n");
+    }
+    memset(safe_buf, 0xcd, sizeof(safe_buf));
+    edge = find_prot_edge(read_only_buf, DR_MEMPROT_READ);
+    bytes_read = 0xcdcdcdcd;
+    if (dr_safe_read(edge - (PAGE_SIZE + 10), PAGE_SIZE+20, safe_buf, &bytes_read) ||
+        bytes_read == 0xcdcdcdcd || bytes_read > PAGE_SIZE+10 || 
+        !memchk(safe_buf, 0, bytes_read)) {
+        dr_printf("ERROR in overlap dr_safe_read()\n");
+    }
+    dr_printf("dr_safe_read() check\n");
+
+    memset(safe_buf, 0xcd, sizeof(safe_buf));
+    if (!dr_safe_write(writable_buf, 1000, safe_buf, &bytes_written) ||
+        bytes_written != 1000 || !memchk(writable_buf, 0xcd, 1000) ||
+        !memchk(writable_buf+1000, 0, 1000)) {
+        dr_printf("ERROR in plain dr_safe_write()\n");
+    }
+    memset(writable_buf, 0, sizeof(writable_buf));
+    edge = find_prot_edge(writable_buf, DR_MEMPROT_WRITE);
+    bytes_written = 0xcdcdcdcd;
+    if (dr_safe_write(edge - (PAGE_SIZE + 10), PAGE_SIZE+20, safe_buf, &bytes_written) ||
+        bytes_written == 0xcdcdcdcd || bytes_written > PAGE_SIZE+10 ||
+        !memchk(edge - (PAGE_SIZE + 10), 0xcd, bytes_written)) {
+        dr_printf("ERROR in overlap dr_safe_write() "PFX" "PFX" %d\n", writable_buf, edge, bytes_written);
+    }
+    dr_printf("dr_safe_write() check\n");
+
+    dr_printf("done\n");
+}
+
diff --git a/suite/tests/client-interface/file_io.expect b/suite/tests/client-interface/file_io.expect
new file mode 100644
index 000000000..f874eba04
--- /dev/null
+++ b/suite/tests/client-interface/file_io.expect
@@ -0,0 +1,11 @@
+0123456789
+01234
+abcdefg
+vwxyz
+
+dummy_func is rx
+writable_buf is rw
+read_only_buf is r
+dr_safe_read() check
+dr_safe_write() check
+done
diff --git a/suite/tests/client-interface/file_io_data.txt b/suite/tests/client-interface/file_io_data.txt
new file mode 100644
index 000000000..db0ba5e46
--- /dev/null
+++ b/suite/tests/client-interface/file_io_data.txt
@@ -0,0 +1 @@
+0123456789abcdefghijklmnopqrstuvwxyz
diff --git a/suite/tests/client-interface/flush.c b/suite/tests/client-interface/flush.c
new file mode 100644
index 000000000..f245f9fe0
--- /dev/null
+++ b/suite/tests/client-interface/flush.c
@@ -0,0 +1,78 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifndef ASM_CODE_ONLY /* C code */
+#include <stdio.h>
+
+int test = 1;
+
+int main()
+{
+    int i = 0;
+    int count = 0;
+    do {
+        if (test) {
+            /* I can't think of a good way to do this -- I want to
+             * target a specific BB in the client, so I'll use a
+             * couple of nops as markers.  
+             *
+             * Given the value of 'test', this side of the conditional
+             * should become part of a trace.
+             */
+            /* FIXME - pick unusual nops that instr_is_nop() recognizes.
+             * 2 regular nops in a row is hit on Linux frequently so going with
+             * xchg ebp, ebp since I haven't seen that one used before (mov edi, edi
+             * or xchg eax, eax is more typical for 2 bytes). */
+            marker();
+            count++;
+        }
+        else {
+            count--;
+        }
+
+        i++;
+    } while (i < 402);
+
+    printf("count = %d\n", count);
+}
+
+#else /* asm code *************************************************************/
+#include "asm_defines.asm"
+START_FILE
+        DECLARE_FUNC(marker)
+GLOBAL_LABEL(marker:)
+        nop
+        xchg REG_XBP, REG_XBP
+        ret
+        END_FUNC(marker)
+END_FILE
+#endif
diff --git a/suite/tests/client-interface/flush.dll.c b/suite/tests/client-interface/flush.dll.c
new file mode 100644
index 000000000..a23e7e96f
--- /dev/null
+++ b/suite/tests/client-interface/flush.dll.c
@@ -0,0 +1,294 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+#include <string.h>
+
+#define MINSERT instrlist_meta_preinsert
+
+/* FIXME - module events are not supported on Linux */
+
+#ifdef WINDOWS
+static app_pc start = NULL;
+static app_pc end = NULL;
+#endif
+
+static bool use_unlink = false;
+static bool delay_flush_at_next_build = false;
+
+static int bb_build_count = 0;
+static uint callback_count = 0;
+
+/* Keep a list that tracks which tags have been created and deleted.
+ * We need to make sure we're informed of all flushed fragments.
+ * The list is global -- we should use a mutex when accessing, but
+ * the test is single threaded.
+ */
+typedef struct _elem_t {
+    void *tag;
+    int count;
+    struct _elem_t *next;
+    struct _elem_t *prev;
+} elem_t;
+
+elem_t *head = NULL;
+elem_t *tail = NULL;
+
+static
+elem_t *find(void *tag)
+{
+    elem_t *iter = head;
+    while (iter) {
+        if (iter->tag == tag)
+            return iter;
+        iter = iter->next;
+    }
+
+    return NULL;
+}
+
+static
+void increment(void *tag)
+{
+    elem_t *elem;
+
+    elem = find(tag);
+    if (elem != NULL) {
+        elem->count++;
+    }
+    else {
+        elem = dr_global_alloc(sizeof(elem_t));
+
+        elem->tag = tag;
+        elem->count = 1;
+        elem->next = NULL;
+        elem->prev = NULL;
+
+        if (head == NULL) {
+            head = elem;
+            tail = elem;
+        }
+        else {
+            tail->next = elem;
+            elem->prev = tail;
+            tail = elem;
+        }
+    }
+}
+
+static
+void decrement(void *tag)
+{
+    elem_t *elem;
+
+    elem = find(tag);
+    if (elem == NULL) {
+        dr_printf("ERROR removing "PFX"\n", tag);
+    }
+    else {
+        elem->count--;
+
+        if (elem->count == 0) {
+            if (head == elem) {
+                head = elem->next;
+            }
+            if (tail == elem) {
+                tail = elem->prev;
+            }
+            if (elem->next) {
+                elem->next->prev = elem->prev;
+            }
+            if (elem->prev) {
+                elem->prev->next = elem->next;
+            }
+
+            dr_global_free(elem, sizeof(elem_t));
+        }
+    }
+}
+
+static
+void exit_event(void)
+{
+    int count = 0;
+
+    elem_t *iter = head;
+    while (iter) {
+        elem_t *next = iter->next;
+        count += iter->count;
+        dr_printf("ERROR: "PFX" undeleted\n", iter->tag);
+        dr_global_free(iter, sizeof(elem_t));
+        iter = next;
+    }
+
+    dr_printf("%d undeleted fragments\n", count);
+    dr_printf("constructed BB %d times\n", bb_build_count);
+}
+
+static
+dr_emit_flags_t trace_event(void *drcontext, void *tag, instrlist_t *trace,
+                            bool translating)
+{
+    if (!translating)
+        increment(tag);
+    return DR_EMIT_DEFAULT;
+}
+
+static
+void deleted_event(void *dcontext, void *tag)
+{
+    decrement(tag);
+}
+
+void flush_event(int flush_id)
+{
+    dr_printf("Flush completion id=%d\n", flush_id);
+}
+
+static 
+void callback(void *tag, app_pc next_pc)
+{
+    callback_count++;
+
+    /* Flush all fragments containing this tag twice every hundred calls alternating
+     * between a sync_all and delay flush (if available) and an unlink and delay flush
+     * (if available). */
+    if (callback_count % 100 == 0) {
+        if (callback_count % 200 == 0) {
+            /* For windows test dr_flush_region() half the time */
+            dr_mcontext_t mcontext;
+            int errno;
+            
+            dr_delay_flush_region((app_pc)tag - 20, 30, callback_count, flush_event);
+            dr_get_mcontext(dr_get_current_drcontext(), &mcontext, &errno);
+            mcontext.pc = next_pc;
+            dr_flush_region(tag, 1);
+            dr_redirect_execution(&mcontext, errno);
+            *(uint *)NULL = 0; /* ASSERT_NOT_REACHED() */
+        } else if (use_unlink) {
+            /* Test dr_unlink_flush_region() half the time (if available).
+             * FIXME - extend once we add unlink callback. */
+            delay_flush_at_next_build = true;
+            dr_unlink_flush_region(tag, 1);
+        }
+    }
+}
+
+#ifdef WINDOWS
+static
+bool string_match(const char *str1, const char *str2)
+{
+    if (str1 == NULL || str2 == NULL)
+        return false;
+
+    while (*str1 == *str2) {
+        if (*str1 == '\0')
+            return true;
+        str1++;
+        str2++;
+    }
+
+    return false;
+}
+
+static
+void module_load_event(void *dcontext, const module_data_t *data, bool loaded)
+{
+    if (string_match(dr_module_preferred_name(data), "flush.exe")) {
+        start = data->start;
+        end = data->end;
+    }
+}
+#endif
+
+static
+dr_emit_flags_t bb_event(void* drcontext, void *tag, instrlist_t *bb,
+                         bool for_trace, bool translating)
+{
+    instr_t *instr;
+    if (!translating)
+        increment(tag);
+
+    /* I'm looking for a specific BB in the test .exe.  I've marked
+     * it with a couple nops.
+     */
+#ifdef WINDOWS
+    if ((app_pc)tag >= start && (app_pc)tag < end) {
+#endif
+        instr = instrlist_first(bb);
+
+        if (instr_is_nop(instr)) {
+            instr_t *next = instr_get_next(instr);
+
+            /* The test app uses two nops as a marker to identify a specific bb.  Since
+             * 2 nop instructions in a row aren't that uncommon on Linux (where we can't
+             * restrict our search to just the test.exe module) we use an unusual nop
+             * for the second one: xchg xbp, xbp */
+            if (next != NULL && instr_is_nop(next) &&
+                instr_get_opcode(next) == OP_xchg &&
+                instr_writes_to_exact_reg(next, REG_XBP)) {
+
+                bb_build_count++;
+
+                if (delay_flush_at_next_build) {
+                    delay_flush_at_next_build = false;
+                    dr_delay_flush_region((app_pc)tag - 20, 30, callback_count, flush_event);
+                }
+
+                dr_insert_clean_call(drcontext, bb, instr, (void *)callback,
+                                     false, 2, OPND_CREATE_INTPTR(tag),
+                                     OPND_CREATE_INTPTR(instr_get_app_pc(instr)));
+            }
+        }
+#ifdef WINDOWS
+    }
+#endif
+    return DR_EMIT_DEFAULT;
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    const char *options = dr_get_options(id);
+    dr_printf("options = %s\n", options);
+    if (options != NULL && strstr(options, "use_unlink") != NULL) {
+        use_unlink = true;
+    }
+#ifdef WINDOWS
+    dr_register_module_load_event(module_load_event);
+#endif
+    dr_register_exit_event(exit_event);
+    dr_register_trace_event(trace_event);
+    dr_register_delete_event(deleted_event);
+    dr_register_bb_event(bb_event);
+    
+}
diff --git a/suite/tests/client-interface/flush.template b/suite/tests/client-interface/flush.template
new file mode 100644
index 000000000..5cea6a8e9
--- /dev/null
+++ b/suite/tests/client-interface/flush.template
@@ -0,0 +1,18 @@
+#if : +-thread_private || : +-enable_full_api
+options = use_unlink
+Flush completion id=100
+Flush completion id=200
+Flush completion id=300
+Flush completion id=400
+#else
+options = 
+Flush completion id=200
+Flush completion id=400
+#endif
+count = 402
+0 undeleted fragments
+#if : +-thread_private || : +-enable_full_api
+constructed BB 9 times
+#else
+constructed BB 5 times
+#endif
diff --git a/suite/tests/client-interface/fragdel.c b/suite/tests/client-interface/fragdel.c
new file mode 100644
index 000000000..121f2f6b8
--- /dev/null
+++ b/suite/tests/client-interface/fragdel.c
@@ -0,0 +1,43 @@
+/* **********************************************************
+ * Copyright (c) 2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <stdio.h>
+
+int main()
+{
+    int i;
+    printf("counting to 10:\n");
+    for (i=1; i<=10; i++)
+        printf("  %d\n", i);
+
+    return 0;
+}
diff --git a/suite/tests/client-interface/fragdel.dll.c b/suite/tests/client-interface/fragdel.dll.c
new file mode 100644
index 000000000..a40a5b26d
--- /dev/null
+++ b/suite/tests/client-interface/fragdel.dll.c
@@ -0,0 +1,119 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+
+static void *mutex = NULL;
+static int deletions = 0;
+
+app_pc start = NULL;
+app_pc end   = NULL;
+
+
+static bool
+my_strcmp(const char* in1, const char* in2)
+{
+    if (in1 == NULL || in2 == NULL)
+        return false;
+
+    while (*in1 == *in2) {
+        if (*in1 == '\0')
+            return true;
+        in1++;
+        in2++;
+    }
+    return false;
+}
+
+static void
+delete_fragment(void *drcontext, app_pc tag)
+{
+    dr_mutex_lock(mutex);
+    deletions++;
+    dr_mutex_unlock(mutex);
+
+    dr_delete_fragment(drcontext, tag);
+}
+
+#define MINSERT instrlist_meta_preinsert
+
+static
+dr_emit_flags_t bb_event(void* drcontext, app_pc tag, instrlist_t* bb, bool for_trace, bool translating)
+{
+    if (tag >= start && tag < end) {
+        instr_t* instr = instrlist_first(bb);
+
+        dr_prepare_for_call(drcontext, bb, instr);
+    
+        MINSERT(bb, instr, INSTR_CREATE_push_imm
+                (drcontext, OPND_CREATE_INT32((ptr_uint_t)tag)));
+        MINSERT(bb, instr, INSTR_CREATE_push_imm
+                (drcontext, OPND_CREATE_INT32((ptr_uint_t)drcontext)));
+        MINSERT(bb, instr, INSTR_CREATE_call
+                (drcontext, opnd_create_pc((void*)delete_fragment)));
+        
+        dr_cleanup_after_call(drcontext, bb, instr, 8);
+    }
+    return DR_EMIT_DEFAULT;
+}
+
+static
+void exit_event(void)
+{
+    dr_mutex_exit(mutex);
+    if (deletions > 10000)
+        dr_printf("deleted > 10k fragments\n");
+    else
+        dr_printf("deleted %d fragments\n", deletions);
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    module_iterator_t *iter;
+
+    dr_register_bb_event(bb_event);
+    dr_register_exit_event(exit_event);
+
+    mutex = dr_mutex_init();
+
+    iter = module_iterator_start();
+    while (module_iterator_hasnext(iter)) {
+        module_data_t *data = module_iterator_next(iter);
+        if (my_strcmp(module_name(data), "fragdel.exe")) {
+            start = data->start;
+            end = data->end;
+            break;
+        }
+    }
+    module_iterator_stop(iter);
+}
diff --git a/suite/tests/client-interface/fragdel.expect b/suite/tests/client-interface/fragdel.expect
new file mode 100644
index 000000000..8154f8e19
--- /dev/null
+++ b/suite/tests/client-interface/fragdel.expect
@@ -0,0 +1,12 @@
+counting to 10:
+  1
+  2
+  3
+  4
+  5
+  6
+  7
+  8
+  9
+  10
+deleted > 10k fragments
diff --git a/suite/tests/client-interface/modules.c b/suite/tests/client-interface/modules.c
new file mode 100644
index 000000000..022bc9591
--- /dev/null
+++ b/suite/tests/client-interface/modules.c
@@ -0,0 +1,40 @@
+/* **********************************************************
+ * Copyright (c) 2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+
+int main()
+{
+    HMODULE handle;
+    handle = LoadLibrary("advapi32.dll");
+    FreeLibrary(handle);
+}
diff --git a/suite/tests/client-interface/modules.dll.c b/suite/tests/client-interface/modules.dll.c
new file mode 100644
index 000000000..e8c47de53
--- /dev/null
+++ b/suite/tests/client-interface/modules.dll.c
@@ -0,0 +1,78 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+
+bool string_match(const char *str1, const char *str2)
+{
+    if (str1 == NULL || str2 == NULL)
+        return false;
+
+    while (*str1 == *str2) {
+        if (*str1 == '\0')
+            return true;
+        str1++;
+        str2++;
+    }
+
+    return false;
+}
+
+static
+void module_load_event(void *dcontext, const module_data_t *data, bool loaded)
+{
+    /* It's easier to simply print all module loads and unloads, but
+     * it appears that loading a module like advapi32.dll causes
+     * different modules to load on different windows versions.  Even
+     * worse, they seem to be loaded in a different order for
+     * different runs.  For the sake of making this test robust, I'll
+     * just look for the module in question.
+     */
+    if (string_match(data->names.module_name, "ADVAPI32.dll"))
+        dr_printf("LOADED MODULE: %s\n", data->names.module_name);
+}
+
+static
+void module_unload_event(void *dcontext, const module_data_t *data)
+{
+    if (string_match(data->names.module_name, "ADVAPI32.dll"))
+        dr_printf("UNLOADED MODULE: %s\n", data->names.module_name);
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    dr_register_module_load_event(module_load_event);
+    dr_register_module_unload_event(module_unload_event);    
+}
+
+/* TODO - add more module interface tests. */
diff --git a/suite/tests/client-interface/modules.expect b/suite/tests/client-interface/modules.expect
new file mode 100644
index 000000000..56fbcb7cc
--- /dev/null
+++ b/suite/tests/client-interface/modules.expect
@@ -0,0 +1,2 @@
+LOADED MODULE: ADVAPI32.dll
+UNLOADED MODULE: ADVAPI32.dll
diff --git a/suite/tests/client-interface/nudge_test.dll.c b/suite/tests/client-interface/nudge_test.dll.c
new file mode 100644
index 000000000..44176685e
--- /dev/null
+++ b/suite/tests/client-interface/nudge_test.dll.c
@@ -0,0 +1,65 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+
+static client_id_t client_id = 0;
+
+static
+void nudge_event(void *drcontext, uint64 arg)
+{
+    /* first nudge is external */
+    static bool first = true;
+
+    dr_printf("nudge delivered %x\n", (uint)arg);
+
+    if (first) {
+        first = false;
+        /* test internal nudge */
+        dr_nudge_client(client_id, arg+1);
+    }
+}
+
+static
+void dr_exit(void)
+{
+    dr_printf("done\n");
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    client_id = id;
+    dr_printf("thank you for testing the client interface\n");
+    dr_register_nudge_event(nudge_event, id);
+    dr_register_exit_event(dr_exit);
+}
diff --git a/suite/tests/client-interface/nudge_test.runall b/suite/tests/client-interface/nudge_test.runall
new file mode 100644
index 000000000..9cb014fa9
--- /dev/null
+++ b/suite/tests/client-interface/nudge_test.runall
@@ -0,0 +1,3 @@
+notepad.exe
+Untitled - Notepad
+<client_nudge>
diff --git a/suite/tests/client-interface/nudge_test.template b/suite/tests/client-interface/nudge_test.template
new file mode 100644
index 000000000..5420699be
--- /dev/null
+++ b/suite/tests/client-interface/nudge_test.template
@@ -0,0 +1,30 @@
+thank you for testing the client interface
+#if +RUNREGRESSION_XP +PROGRAM_SHEPHERDING : +-security +-no_executable_if_rx_text --executable_after_load
+# SEC_VIO_STOP
+No such process found.
+Window not found: Untitled - Notepad
+#else
+# if +DEBUG :
+Process notepad.exe, running SC debug
+# else
+#  if +PROFILE :
+Process notepad.exe, running SC profile
+#  else
+Process notepad.exe, running SC release
+#  endif
+# endif
+nudge delivered 10
+nudge delivered 11
+# if +DEBUG :
+Process notepad.exe, running SC debug
+# else
+#  if +PROFILE :
+Process notepad.exe, running SC profile
+#  else
+Process notepad.exe, running SC release
+#  endif
+# endif
+Close message sent.
+done
+#endif
+No such process found.
diff --git a/suite/tests/client-interface/pc-check.dll.c b/suite/tests/client-interface/pc-check.dll.c
new file mode 100644
index 000000000..c58029e75
--- /dev/null
+++ b/suite/tests/client-interface/pc-check.dll.c
@@ -0,0 +1,122 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+
+/* Tests dr_fragment_app_pc() and some of the module routines (dr_lookup_module(),
+ * dr_lookup_module_by_name(), and dr_get_proc_address()). Also checks that the client
+ * isn't seeing code from weird locations. */
+
+static
+bool is_in_known_module(app_pc pc, bool *found_section, IMAGE_SECTION_HEADER *section)
+{
+    bool found = false;
+    module_data_t *data = dr_lookup_module(pc);
+    if (data != NULL) {
+        found = true;
+        *found_section = dr_lookup_module_section(data->handle, pc, section);
+    }
+    dr_free_module_data(data);
+    return found;
+}
+
+/* FIXME - is global because of compiler switching issues, can't find security_cookie
+ * with new compiler which is triggered by having this as a local to bb_event.
+ * FIXME - test target notepad.exe is single threaded so access is ok for now. */
+IMAGE_SECTION_HEADER section;
+
+app_pc exit_proc_addr;
+
+static
+dr_emit_flags_t bb_event(void *drcontext, app_pc tag, instrlist_t *bb, bool for_trace, bool translating)
+{
+    instr_t *instr, *next_instr;
+    app_pc bb_addr, instr_addr;
+    bool found_section;
+    
+    bb_addr = dr_fragment_app_pc(tag);
+
+    /* vsyscall: some versions of windows jump to 0x7ffe0300 to
+     * execute a syscall; this address is not contained in any module.
+     */
+    if ((ptr_uint_t)bb_addr == 0x7ffe0300 ||
+        (ptr_uint_t)bb_addr == 0x7ffe0302)
+        return DR_EMIT_DEFAULT;
+
+    if (!is_in_known_module(bb_addr, &found_section, &section)) {
+        dr_printf("ERROR: BB addr "PFX" in unknown module\n", bb_addr);
+    }
+    if (!found_section) {
+        dr_printf("ERROR: BB addr "PFX" isn't within a module section\n", bb_addr);
+    }
+    if ((section.Characteristics & IMAGE_SCN_CNT_CODE) == 0) {
+        dr_printf("ERROR: BB addr "PFX" isn't within a code section\n", bb_addr);   
+    }
+
+    for (instr = instrlist_first(bb);
+         instr != NULL; instr = next_instr) {
+        next_instr = instr_get_next(instr);
+
+        instr_addr = instr_get_app_pc(instr);
+        if (!is_in_known_module(instr_addr, &found_section, &section)) {
+            dr_printf("ERROR: instr addr "PFX" in unknown module\n", instr_addr);
+        }
+        if (!found_section) {
+            dr_printf("ERROR: instr addr "PFX" isn't within a module section\n", instr_addr);
+        }
+        if ((section.Characteristics & IMAGE_SCN_CNT_CODE) == 0) {
+            dr_printf("ERROR: instr addr "PFX" isn't within a code section\n", instr_addr);   
+        }
+        if (instr_addr == exit_proc_addr) {
+            dr_printf("Hit kernel32!ExitProcess\n");
+            exit_proc_addr = NULL;
+        }
+    }
+    return DR_EMIT_DEFAULT;
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    module_data_t *data;
+    dr_printf("thank you for testing the client interface\n");
+    dr_register_bb_event(bb_event);
+    data = dr_lookup_module_by_name("kernel32.dll");
+    if (data != NULL) {
+        exit_proc_addr = (void *)dr_get_proc_address(data->handle, "ExitProcess");
+        if (exit_proc_addr == NULL)
+            dr_printf("ERROR: unable to find kernel32!ExitProcess\n");
+        dr_free_module_data(data);
+    } else {
+        dr_printf("ERROR: unable to find ntdll.dll\n");
+    }
+}
diff --git a/suite/tests/client-interface/pc-check.runall b/suite/tests/client-interface/pc-check.runall
new file mode 100644
index 000000000..787417df8
--- /dev/null
+++ b/suite/tests/client-interface/pc-check.runall
@@ -0,0 +1,3 @@
+notepad.exe
+Untitled - Notepad
+<>
diff --git a/suite/tests/client-interface/pc-check.template b/suite/tests/client-interface/pc-check.template
new file mode 100644
index 000000000..62b1bb61f
--- /dev/null
+++ b/suite/tests/client-interface/pc-check.template
@@ -0,0 +1,19 @@
+thank you for testing the client interface
+#if +RUNREGRESSION_XP +PROGRAM_SHEPHERDING : +-security +-no_executable_if_rx_text --executable_after_load
+# SEC_VIO_STOP
+No such process found.
+Window not found: Untitled - Notepad
+#else
+# if +DEBUG :
+Process notepad.exe, running SC debug
+# else
+#  if +PROFILE :
+Process notepad.exe, running SC profile
+#  else
+Process notepad.exe, running SC release
+#  endif
+# endif
+Close message sent.
+Hit kernel32!ExitProcess
+#endif
+No such process found.
diff --git a/suite/tests/client-interface/security.c b/suite/tests/client-interface/security.c
new file mode 100644
index 000000000..910931b4d
--- /dev/null
+++ b/suite/tests/client-interface/security.c
@@ -0,0 +1,61 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+void baz()
+{
+    printf("** Return address successfully overwritten **\n");
+    fflush(stdout);
+    exit(1);
+}
+
+void bar()
+{
+    unsigned int a[2];
+    printf("** Return address successfully overwritten **\n");
+    fflush(stdout);
+    a[3] = (unsigned)baz;
+}
+
+void foo()
+{
+    unsigned int a[2];
+    a[3] = (unsigned)bar;
+}
+
+int main()
+{
+    foo();
+    return 0;
+}
diff --git a/suite/tests/client-interface/security.dll.c b/suite/tests/client-interface/security.dll.c
new file mode 100644
index 000000000..6d65c2e3c
--- /dev/null
+++ b/suite/tests/client-interface/security.dll.c
@@ -0,0 +1,92 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+
+static
+void security_event(void *drcontext, void *source_tag,
+                    app_pc source_pc, app_pc target_pc,
+                    dr_security_violation_type_t violation,
+                    dr_mcontext_t *mcontext,
+                    dr_security_violation_action_t *action)
+{
+    static int violations = 0;
+
+    char *violation_str = NULL;
+    switch (violation) {
+    case DR_RCO_STACK_VIOLATION:
+        violation_str = "stack execution violation";
+        break;
+    case DR_RCO_HEAP_VIOLATION:
+        violation_str = "heap execution violation";
+        break;
+    case DR_RCT_RETURN_VIOLATION:
+        violation_str = "return target violation";
+        break;
+    case DR_RCT_INDIRECT_CALL_VIOLATION:
+        violation_str = "call rct violation";
+        break;
+    case DR_RCT_INDIRECT_JUMP_VIOLATION:
+        violation_str = "jump rct violation";
+        break;
+    default:
+        violation_str = "unknown";
+        break;
+    }
+
+    dr_printf("security violation: \"%s\"\n", violation_str);
+#if 0
+    dr_printf("Source tag="PFX" pc="PFX" Target pc="PFX"\n",
+              source_tag, source_pc, target_pc);
+#endif
+
+    violations++;
+
+    if (violations == 1) {
+        dr_printf("continuing...\n");
+        *action = DR_VIOLATION_ACTION_CONTINUE;
+    }
+    else {
+        dr_printf("terminating...\n");
+        *action = DR_VIOLATION_ACTION_KILL_PROCESS;
+#if 0
+        dr_write_forensics_report(drcontext, dr_get_stdout_file(), violation,
+                                  *action, "Fatal Violation");
+#endif
+    }
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    dr_register_security_event(security_event);
+}
diff --git a/suite/tests/client-interface/security.expect b/suite/tests/client-interface/security.expect
new file mode 100644
index 000000000..7be18c8cf
--- /dev/null
+++ b/suite/tests/client-interface/security.expect
@@ -0,0 +1,11 @@
+security violation: "return target violation"
+continuing...
+<Execution security violation was intercepted!
+Contact your vendor for a security vulnerability fix.
+Program continuing!>
+** Return address successfully overwritten **
+security violation: "return target violation"
+terminating...
+<Execution security violation was intercepted!
+Contact your vendor for a security vulnerability fix.
+Program terminated.>
\ No newline at end of file
diff --git a/suite/tests/client-interface/signal.c b/suite/tests/client-interface/signal.c
new file mode 100644
index 000000000..030361591
--- /dev/null
+++ b/suite/tests/client-interface/signal.c
@@ -0,0 +1,176 @@
+/* **********************************************************
+ * Copyright (c) 2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * API regression test for signal event
+ */
+
+#ifdef LINUX /* whole file */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <dlfcn.h>
+#include <signal.h>
+#include <ucontext.h>
+#include <assert.h>
+#include <setjmp.h>
+
+#ifdef WINDOWS
+# define NOP __nop()
+#else /* LINUX */
+# define NOP asm("nop")
+#endif
+
+static jmp_buf mark;
+static int bar;
+
+static void foo(void)
+{ /* nothing: just a marker */ }
+
+static void
+redirect_target(void)
+{
+    /* use 2 NOPs + call so client can locate this spot (since we don't
+     * have dr_get_proc_address() on linux yet)
+     */
+    NOP; NOP;
+    foo();
+
+    fprintf(stderr, "Redirected\n");
+    longjmp(mark, 1);
+}
+
+/* handler with SA_SIGINFO flag set gets three arguments: */
+typedef void (*handler_t)(int, struct siginfo *, void *);
+
+static void
+signal_handler(int sig, siginfo_t *siginfo, ucontext_t *ucxt)
+{
+    if (sig == SIGUSR1)
+	fprintf(stderr, "Got SIGUSR1\n");
+    else if (sig == SIGUSR2)
+	fprintf(stderr, "Got SIGUSR2\n");
+    else if (sig == SIGURG)
+	fprintf(stderr, "Got SIGURG\n");
+    else if (sig == SIGSEGV)
+	fprintf(stderr, "Got SIGSEGV\n");
+}
+
+/* set up signal_handler as the handler for signal "sig" */
+static void
+intercept_signal(int sig, handler_t handler)
+{
+    int rc;
+    struct sigaction act;
+    act.sa_sigaction = handler;
+    rc = sigfillset(&act.sa_mask); /* block all signals within handler */
+    assert(rc == 0);
+    act.sa_flags = SA_SIGINFO | SA_ONSTACK; /* send 3 args to handler */
+    /* arm the signal */
+    rc = sigaction(sig, &act, NULL);
+    assert(rc == 0);
+}
+
+static void
+unintercept_signal(int sig)
+{
+    int rc;
+    struct sigaction act;
+    act.sa_sigaction = (handler_t) SIG_DFL;
+    /* disarm the signal */
+    rc = sigaction(sig, &act, NULL);
+    assert(rc == 0);
+}
+
+int
+main(int argc, char** argv)
+{
+    intercept_signal(SIGUSR1, (handler_t) signal_handler);
+    intercept_signal(SIGUSR2, (handler_t) signal_handler);
+    intercept_signal(SIGURG, (handler_t) signal_handler);
+    intercept_signal(SIGSEGV, (handler_t) signal_handler);
+
+    fprintf(stderr, "Sending SIGURG\n");
+    kill(getpid(), SIGURG);
+    fprintf(stderr, "Sending SIGURG\n");
+    kill(getpid(), SIGURG);
+    fprintf(stderr, "Sending SIGURG\n");
+    kill(getpid(), SIGURG);
+
+    unintercept_signal(SIGURG);
+
+    fprintf(stderr, "Sending SIGURG\n");
+    kill(getpid(), SIGURG);
+    fprintf(stderr, "Sending SIGURG\n");
+    kill(getpid(), SIGURG);
+    fprintf(stderr, "Sending SIGURG\n");
+    kill(getpid(), SIGURG);
+
+    fprintf(stderr, "Sending SIGTERM\n");
+    kill(getpid(), SIGTERM);
+
+    if (setjmp(mark) == 0) {
+        /* execute so that client sees the spot */
+        redirect_target();
+    }
+    if (setjmp(mark) == 0) {
+        fprintf(stderr, "Sending SIGUSR2\n");
+        kill(getpid(), SIGUSR2);
+    }
+
+    /* generate SIGSEGV; we'll re-crash post-handler unless client
+     * modifies mcontext
+     */
+#ifdef X86_64
+# define EAX "%rax"
+# define ECX "%rcx"
+#else
+# define EAX "%eax"
+# define ECX "%ecx"
+#endif
+    asm("push "EAX);
+    asm("push "ECX);
+    asm("mov %0, %"ECX"" : : "g"(&bar)); /* ok place to read from */
+    asm("mov $0, "EAX);
+    asm("mov ("EAX"), "EAX);
+    asm("pop "ECX);
+    asm("pop "EAX);
+
+    fprintf(stderr, "Sending SIGUSR1\n");
+    kill(getpid(), SIGUSR1);
+
+    fprintf(stderr, "Done\n");
+}
+
+#endif /* LINUX */
diff --git a/suite/tests/client-interface/signal.dll.c b/suite/tests/client-interface/signal.dll.c
new file mode 100644
index 000000000..565d642c2
--- /dev/null
+++ b/suite/tests/client-interface/signal.dll.c
@@ -0,0 +1,127 @@
+/* **********************************************************
+ * Copyright (c) 2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * API regression test for signal event
+ */
+
+#include "dr_api.h"
+#include <signal.h>
+
+#ifdef WINDOWS
+# define ASSERT(x)                                        \
+  if (!(x)) {                                             \
+    dr_messagebox("ASSERT failed on line %d", __LINE__);  \
+    dr_abort();                                           \
+  }
+#else /* LINUX */
+# define ASSERT(x)                                             \
+  if (!(x)) {                                                  \
+    dr_fprintf(STDERR, "ASSERT failed on line %d\n", __LINE__);\
+    dr_abort();                                                \
+  }
+#endif
+
+static app_pc redirect_tag;
+
+static
+dr_signal_action_t signal_event(void *dcontext, dr_siginfo_t *info)
+{
+    static int count = -1;
+    count++;
+    dr_printf("signal event %d sig=%d\n", count, info->sig);
+
+    if (info->sig == SIGURG) {
+        static int count_urg = -1;
+        count_urg++;
+        switch (count_urg) {
+            /* test delayable signal with each return value */
+            case 0: return DR_SIGNAL_DELIVER;
+            case 1: return DR_SIGNAL_SUPPRESS;
+            case 2: return DR_SIGNAL_BYPASS;
+            case 3: return DR_SIGNAL_DELIVER;
+            case 4: return DR_SIGNAL_SUPPRESS;
+            case 5: return DR_SIGNAL_BYPASS;
+            default: dr_printf("too many SIGURG\n");
+        }
+    } else if (info->sig == SIGTERM) {
+        return DR_SIGNAL_SUPPRESS;
+    } else if (info->sig == SIGUSR2) {
+        ASSERT(redirect_tag != NULL);
+        info->mcontext.xip = redirect_tag;
+        return DR_SIGNAL_REDIRECT;
+    } else if (info->sig == SIGSEGV) {
+        /* test non-delayable signal */
+        static int count_segv = -1;
+        count_segv++;
+        if (count_segv == 0)
+            return DR_SIGNAL_SUPPRESS;
+        else {
+            /* test mcontext changes on delivery.
+             * fix up to avoid crash on re-execution.
+             */
+            info->mcontext.xax = info->mcontext.xcx;
+            return DR_SIGNAL_DELIVER;
+        }
+    }
+
+    return DR_SIGNAL_DELIVER;
+}
+
+static dr_emit_flags_t
+bb_event(void *drcontext, void* tag, instrlist_t *bb, bool for_trace, bool translating)
+{
+    instr_t *instr, *next_instr, *next_next_instr;
+    /* Look for 3 nops to locate redirection target */
+    for (instr = instrlist_first(bb); instr != NULL; instr = next_instr) {
+        next_instr = instr_get_next(instr);
+        if (next_instr != NULL)
+            next_next_instr = instr_get_next(next_instr);
+        else
+            next_next_instr = NULL;
+        if (instr_is_nop(instr) && 
+            next_instr != NULL && instr_is_nop(next_instr) && 
+            next_next_instr != NULL && instr_is_call_direct(next_next_instr)) {
+
+            redirect_tag = tag;
+            break;
+        }
+    }
+    return DR_EMIT_DEFAULT;
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    dr_register_bb_event(bb_event);
+    dr_register_signal_event(signal_event);
+}
diff --git a/suite/tests/client-interface/signal.expect b/suite/tests/client-interface/signal.expect
new file mode 100644
index 000000000..75d4b24dc
--- /dev/null
+++ b/suite/tests/client-interface/signal.expect
@@ -0,0 +1,26 @@
+Sending SIGURG
+signal event 0 sig=23
+Got SIGURG
+Sending SIGURG
+signal event 1 sig=23
+Sending SIGURG
+signal event 2 sig=23
+Sending SIGURG
+signal event 3 sig=23
+Sending SIGURG
+signal event 4 sig=23
+Sending SIGURG
+signal event 5 sig=23
+Sending SIGTERM
+signal event 6 sig=15
+Redirected
+Sending SIGUSR2
+signal event 7 sig=12
+Redirected
+signal event 8 sig=11
+signal event 9 sig=11
+Got SIGSEGV
+Sending SIGUSR1
+signal event 10 sig=10
+Got SIGUSR1
+Done
diff --git a/suite/tests/client-interface/syscall-mod.c b/suite/tests/client-interface/syscall-mod.c
new file mode 100644
index 000000000..aae5cdc70
--- /dev/null
+++ b/suite/tests/client-interface/syscall-mod.c
@@ -0,0 +1,56 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <stdio.h>
+#include <syscall.h>
+
+#define EXPANDSTR(x) #x
+#define STRINGIFY(x) EXPANDSTR(x)
+
+int main()
+{
+    int pid;
+    printf("starting\n");
+    /* we don't want vsyscall since we rely on mov immed, eax being in same bb.
+     * plus, libc getpid might cache the pid value.
+     */
+    asm("mov $" STRINGIFY(SYS_getpid) ", %eax");
+#ifdef X64
+    asm("syscall");
+#else
+    asm("int $0x80");
+#endif
+    asm("mov %%eax, %0" : "=m"(pid));
+    printf("pid = %d\n", pid);
+
+    return 0;
+}
diff --git a/suite/tests/client-interface/syscall-mod.dll.c b/suite/tests/client-interface/syscall-mod.dll.c
new file mode 100644
index 000000000..751bffa73
--- /dev/null
+++ b/suite/tests/client-interface/syscall-mod.dll.c
@@ -0,0 +1,70 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+#include <syscall.h>
+
+/* Tests removing a system call. */
+
+#define MINSERT instrlist_meta_preinsert
+
+static
+dr_emit_flags_t bb_event(void* drcontext, void *tag, instrlist_t* bb,
+                         bool for_trace, bool translating)
+{
+    instr_t *instr;
+    instr_t *next_instr;
+    reg_t in_eax = -1;
+
+    for (instr = instrlist_first(bb); instr != NULL; instr = next_instr) {
+        next_instr = instr_get_next(instr);
+        if (instr_get_opcode(instr) == OP_mov_imm &&
+            opnd_get_reg(instr_get_dst(instr, 0)) == REG_EAX)
+            in_eax = opnd_get_immed_int(instr_get_src(instr, 0));
+        if (instr_is_syscall(instr) &&
+            in_eax == SYS_getpid) {
+            instr_t *myval = INSTR_CREATE_mov_imm
+                (drcontext, opnd_create_reg(REG_EAX), OPND_CREATE_INT32(-7));
+            instr_set_translation(myval, instr_get_app_pc(instr));
+            instrlist_preinsert(bb, instr, myval);
+            instrlist_remove(bb, instr);
+            instr_destroy(drcontext, instr);
+        }
+    }
+    return DR_EMIT_DEFAULT;
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    dr_register_bb_event(bb_event);
+}
diff --git a/suite/tests/client-interface/syscall-mod.expect b/suite/tests/client-interface/syscall-mod.expect
new file mode 100644
index 000000000..6a7e77e94
--- /dev/null
+++ b/suite/tests/client-interface/syscall-mod.expect
@@ -0,0 +1,2 @@
+starting
+pid = -7
diff --git a/suite/tests/client-interface/syscall.c b/suite/tests/client-interface/syscall.c
new file mode 100644
index 000000000..a33956d2c
--- /dev/null
+++ b/suite/tests/client-interface/syscall.c
@@ -0,0 +1,107 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include <stdio.h>
+
+char *get_systemroot(char** env)
+{
+    const char sroot[] = "SYSTEMROOT=";
+    int len = sizeof(sroot)-1;
+    int i;
+
+    for (i=0; env[i] != NULL; i++) {
+        if (!strncmp(env[i], sroot, len)) {
+            return env[i] + len;
+        }
+    }
+
+    return "C:\\WINDOWS";
+}
+
+__declspec(dllexport)
+void start_monitor()
+{}
+
+__declspec(dllexport)
+void stop_monitor()
+{}
+
+void create_proc(char *cmd, char *cmdline, STARTUPINFO *sinfo)
+{
+    PROCESS_INFORMATION pinfo;
+    if (!CreateProcess(cmd,     /* application name */
+                       cmdline, /* command line */
+                       NULL,    /* new proc cannot be inherited */
+                       NULL,    /* new thread cannot be inherited */
+                       TRUE,    /* inherit handles from this proc */
+                       0,       /* no creation flags */
+                       NULL,    /* use environment of this proc */
+                       NULL,    /* same directory of this proc */
+                       sinfo,   /* start up info */
+                       &pinfo   /* out: process information */
+                       )) {
+
+        fprintf(stderr, "ERROR creating new process: %s %s\n", cmd, cmdline);
+        exit(1);
+    }
+}
+
+void main(int argc, char** argv, char** env)
+{
+    STARTUPINFO sinfo;
+
+    /* This test prints out all system calls.  Creating a new process
+     * seems to be a good way to cause a bunch of them to execute.
+     * Launch cmd.exe /c exit.
+     */
+    char cmd[MAX_PATH];
+    char *cmdline = "/c exit";
+    sprintf(cmd, "%s\\system32\\cmd.exe", get_systemroot(env));
+
+    /* It seems that CreateProcess follows a slightly different
+     * control-flow when executing the first time under DR vs.
+     * native.  This phenomenon is probably due to DR calling certain
+     * ntdll routines itself.  We'll just call CreateProcess twice and
+     * trace the system calls for the 2nd invocation only.
+     */
+    GetStartupInfo(&sinfo);
+    create_proc(cmd, cmdline, &sinfo);
+
+    /* dummy marker to inform client lib to start monitoring */
+    start_monitor();
+
+    create_proc(cmd, cmdline, &sinfo);
+
+    /* dummy marker to inform client lib to stop monitoring */
+    stop_monitor();
+}
diff --git a/suite/tests/client-interface/syscall.dll.c b/suite/tests/client-interface/syscall.dll.c
new file mode 100644
index 000000000..ebbb423ba
--- /dev/null
+++ b/suite/tests/client-interface/syscall.dll.c
@@ -0,0 +1,116 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+
+/* Tests instrumenting system calls.  Also tests module_iterator interface and 
+ * dr_get_proc_address(). */
+
+#define MINSERT instrlist_meta_preinsert
+
+static app_pc start_pc = NULL;
+static app_pc stop_pc = NULL;
+static bool monitoring = false;
+
+static
+void at_syscall()
+{
+    if (monitoring) {
+        dr_mcontext_t mcontext;
+        void *drcontext = dr_get_current_drcontext();
+        dr_get_mcontext(drcontext, &mcontext, NULL);
+        dr_printf("%x\n", mcontext.eax);
+    }
+}
+
+static
+dr_emit_flags_t bb_event(void* drcontext, void *tag, instrlist_t* bb, bool for_trace, bool translating)
+{
+    app_pc pc = dr_fragment_app_pc(tag);
+
+    if (pc == start_pc) {
+        dr_printf("starting syscall monitoring\n");
+        monitoring = true;
+    }
+    else if (pc == stop_pc) {
+        dr_printf("stopping syscall monitoring\n");
+        monitoring = false;
+    }
+    else {
+        instr_t* instr;
+        instr_t* next_instr;
+
+        for (instr = instrlist_first(bb);
+             instr != NULL;
+             instr = next_instr) {
+
+            next_instr = instr_get_next(instr);
+
+            /* Insert a callback to at_syscall before every system call */
+            if (instr_is_syscall(instr)) {
+                dr_insert_clean_call(drcontext, bb, instr, at_syscall, false, 0);
+            }
+        }
+    }
+    return DR_EMIT_DEFAULT;
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    /* Look up start_monitor() and stop_monitor() in the target app.
+     * These functions are dummy markers that tell us when to start
+     * and stop printing syscalls.
+     */
+    /* NOTE - we could use dr_module_lookup_by_name, but we use the iterator instead
+     * to test it out. */
+    dr_module_iterator_t *iter = dr_module_iterator_start();
+    while (dr_module_iterator_hasnext(iter)) {
+        module_data_t *data = dr_module_iterator_next(iter);
+        if (strncmp(dr_module_preferred_name(data), "syscall.exe", 11) == 0) {
+            module_handle_t lib = data->handle;
+            start_pc = (app_pc)dr_get_proc_address(lib, "start_monitor");
+            stop_pc = (app_pc)dr_get_proc_address(lib, "stop_monitor");
+            dr_free_module_data(data);
+            break;
+        }
+        dr_free_module_data(data);
+    }
+    dr_module_iterator_stop(iter);
+
+    if (start_pc == NULL || stop_pc == NULL) {
+        dr_printf("ERROR: did not find start/stop markers\n");
+    }
+
+    /* Register the BB hook */
+    dr_register_bb_event(bb_event);
+}
diff --git a/suite/tests/client-interface/syscall.template b/suite/tests/client-interface/syscall.template
new file mode 100755
index 000000000..fcee4a69e
--- /dev/null
+++ b/suite/tests/client-interface/syscall.template
@@ -0,0 +1,226 @@
+#if +RUNREGRESSION_Vista :
+#endif
+#if +RUNREGRESSION_2003 :
+starting syscall monitoring
+62
+7a
+34
+af
+bb
+13
+88
+82
+a3
+1b
+7d
+7d
+7d
+32
+a1
+c2
+c2
+c2
+7a
+12
+11f
+12
+11f
+11f
+11f
+12
+12
+8f
+37
+d0
+d6
+1b
+1b
+stopping syscall monitoring
+#endif
+#if +RUNREGRESSION_XP :
+starting syscall monitoring
+98
+74
+32
+b3
+10f
+bc
+25
+97
+32
+6c
+97
+25
+77
+b1
+19
+77
+b1
+19
+25
+74
+91
+19
+9a
+e4
+8b
+74
+91
+19
+74
+91
+19
+74
+91
+19
+9a
+e4
+9a
+77
+82
+7c
+9c
+19
+77
+77
+8b
+9a
+e4
+8b
+74
+32
+19
+6c
+19
+10b
+8b
+25
+32
+19
+6c
+19
+9a
+e4
+b2
+b2
+10b
+9a
+e4
+8b
+74
+32
+19
+6c
+19
+10b
+8b
+25
+32
+19
+6c
+19
+9a
+e4
+b2
+10b
+77
+82
+7c
+9c
+19
+77
+9a
+e4
+8b
+74
+91
+19
+74
+91
+19
+74
+91
+19
+9a
+e4
+10f
+bc
+10b
+19
+19
+81
+7b
+77
+a0
+19
+82
+7c
+9c
+19
+77
+e3
+77
+7f
+aa
+19
+b3
+97
+9a
+e4
+8b
+74
+91
+19
+74
+91
+19
+74
+91
+19
+9a
+e4
+97
+32
+6c
+19
+10b
+77
+b1
+19
+9c
+9c
+19
+a7
+77
+ad
+30
+9a
+ba
+74
+ba
+ba
+8e
+9a
+11
+11
+115
+11
+115
+115
+11
+115
+115
+53
+11
+11
+35
+c8
+ce
+19
+19
+stopping syscall monitoring
+#endif
+#if +RUNREGRESSION_2000 :
+#endif
+#if +RUNREGRESSION_NT :
+#endif
diff --git a/suite/tests/client-interface/thread.c b/suite/tests/client-interface/thread.c
new file mode 100644
index 000000000..c1fadf26e
--- /dev/null
+++ b/suite/tests/client-interface/thread.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <stdio.h>
+
+int main()
+{
+    printf("thank you for testing the client interface\n");
+}
diff --git a/suite/tests/client-interface/thread.dll.c b/suite/tests/client-interface/thread.dll.c
new file mode 100644
index 000000000..bfb0ef07e
--- /dev/null
+++ b/suite/tests/client-interface/thread.dll.c
@@ -0,0 +1,176 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+
+/* Eventually this routine will test i/o by waiting on a file */
+
+#define MINSERT instrlist_meta_preinsert
+
+static uint num_lea = 0;
+
+/* we don't want msgboxes for regressions */
+#define ASSERT(x) \
+    ((void)((!(x)) ? \
+        (dr_fprintf(STDERR, "ASSERT FAILURE: %s:%d: %s\n", \
+                    __FILE__,  __LINE__, #x), \
+         dr_abort(), 0) : 0))
+
+static void 
+at_lea(uint opc, app_pc tag)
+{
+    /* PR 223285: test (one side of) DR_ASSERT for something we know will succeed
+     * (we don't want msgboxes in regressions)
+     */
+    DR_ASSERT(opc == OP_lea);
+    ASSERT((process_id_t) dr_get_tls_field(dr_get_current_drcontext()) ==
+           dr_get_process_id() + 1 /*we added 1 inline*/);
+    dr_set_tls_field(dr_get_current_drcontext(), (void *) dr_get_process_id());
+    num_lea++;
+    /* FIXME: should do some fp ops and really test the fp state preservation */
+}
+              
+static
+dr_emit_flags_t bb_event(void *drcontext, void *tag, instrlist_t *bb, bool for_trace, bool translating)
+{
+    instr_t *instr, *next_instr;
+    
+    for (instr = instrlist_first(bb);
+         instr != NULL; instr = next_instr) {
+        next_instr = instr_get_next(instr);
+        if (instr_get_opcode(instr) == OP_lea) {
+            /* PR 200411: test inline tls access by adding 1 */
+            dr_save_reg(drcontext, bb, instr, REG_EAX, SPILL_SLOT_1);
+            dr_insert_read_tls_field(drcontext, bb, instr, REG_EAX);
+            instrlist_meta_preinsert(bb, instr, INSTR_CREATE_lea
+                                     (drcontext, opnd_create_reg(REG_EAX),
+                                      opnd_create_base_disp(REG_EAX, REG_NULL, 0, 1,
+                                                            OPSZ_lea)));
+            dr_insert_write_tls_field(drcontext, bb, instr, REG_EAX);
+            dr_restore_reg(drcontext, bb, instr, REG_EAX, SPILL_SLOT_1);
+            dr_insert_clean_call(drcontext, bb, instr, at_lea, true/*save fp*/,
+                                 2, OPND_CREATE_INT32(instr_get_opcode(instr)),
+                                 OPND_CREATE_INTPTR(tag));
+        }
+    }
+    return DR_EMIT_DEFAULT;
+}
+
+void exit_event(void)
+{
+    ASSERT(num_lea > 0);
+}
+
+static bool
+str_eq(const char *s1, const char *s2)
+{
+    if (s1 == NULL || s2 == NULL)
+        return false;
+    while (*s1 == *s2) {
+        if (*s1 == '\0')
+            return true;
+        s1++;
+        s2++;
+    }
+    return false;
+}
+
+/* FIXME PR 222812: not supported yet */
+#ifdef CLIENT_SIDELINE
+static bool child_alive;
+static bool child_continue;
+
+static void
+thread_func(void *arg)
+{
+    /* FIXME: should really test corner cases: do raw system calls, etc. to
+     * ensure we're treating it as a true native thread
+     */
+    ASSERT((process_id_t) arg == dr_get_process_id());
+    child_alive = true;
+    dr_printf("client thread is alive\n");
+    /* FIXME: do we now have to provide condition vars, etc.?!? */
+    while (!child_continue)
+        dr_thread_yield();
+    dr_printf("client thread is dying\n");
+    dr_terminate_client_thread();
+    DR_ASSERT_MSG(false, "should not get here");
+}
+#endif
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+#ifdef CLIENT_SIDELINE
+    bool success;
+#endif
+    /* PR 216931: client options */
+    const char * ops = dr_get_options(id);
+    ASSERT(str_eq(ops, "-paramx -paramy"));
+    dr_printf("PR 216931: client options are %s\n", ops);
+
+    dr_register_bb_event(bb_event);
+    dr_register_exit_event(exit_event);
+
+    /* PR 219381: dr_get_application_name() and dr_get_process_id() */
+#ifdef WINDOWS
+    dr_printf("inside app %s\n", dr_get_application_name());
+#else /* LINUX - append .exe so can use same expect file. */
+    dr_printf("inside app %s.exe\n", dr_get_application_name());
+#endif
+    dr_set_tls_field(dr_get_current_drcontext(), (void *) dr_get_process_id());
+
+    {
+        /* test PR 198871: client locks are all at same rank */
+        void *lock1 = dr_mutex_create();
+        void *lock2 = dr_mutex_create();
+        dr_mutex_lock(lock1);
+        dr_mutex_lock(lock2);
+        dr_printf("PR 198871 locking test...");
+        dr_mutex_unlock(lock2);
+        dr_mutex_unlock(lock1);
+        dr_mutex_destroy(lock1);
+        dr_mutex_destroy(lock2);
+        dr_printf("...passed\n");
+    }
+
+#ifdef CLIENT_SIDELINE
+    /* PR 222812: client thread */
+    success = dr_create_client_thread(thread_func, (void *) dr_get_process_id(),
+                                      12*PAGE_SIZE, 10*PAGE_SIZE);
+    ASSERT(success);
+    while (!child_alive)
+        dr_thread_yield();
+    child_continue = true;
+    dr_printf("PR 222812: client thread test passed\n");
+#endif
+}
diff --git a/suite/tests/client-interface/thread.template b/suite/tests/client-interface/thread.template
new file mode 100644
index 000000000..024b0b359
--- /dev/null
+++ b/suite/tests/client-interface/thread.template
@@ -0,0 +1,9 @@
+PR 216931: client options are -paramx -paramy
+inside app thread.exe
+PR 198871 locking test......passed
+#if +CLIENT_SIDELINE :
+client thread is alive
+client thread is dying
+PR 222812: client thread test passed
+#endif
+thank you for testing the client interface
diff --git a/suite/tests/client-interface/truncate.c b/suite/tests/client-interface/truncate.c
new file mode 100644
index 000000000..67d9833c2
--- /dev/null
+++ b/suite/tests/client-interface/truncate.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <stdio.h>
+
+int main()
+{
+    printf("hello world!\n");
+}
diff --git a/suite/tests/client-interface/truncate.dll.c b/suite/tests/client-interface/truncate.dll.c
new file mode 100644
index 000000000..fba9ae34a
--- /dev/null
+++ b/suite/tests/client-interface/truncate.dll.c
@@ -0,0 +1,54 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+
+/* PR 306971: test bb truncation */
+static
+dr_emit_flags_t bb_event(void *drcontext, void *tag, instrlist_t *bb,
+                         bool for_trace, bool translating)
+{
+    /* remove last instr from every multi-instr bb */
+    instr_t *last = instrlist_last(bb);
+    if (last != instrlist_first(bb)) {
+        instrlist_remove(bb, last);
+        instr_destroy(drcontext, last);
+    }
+    return DR_EMIT_DEFAULT;
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    dr_printf("thank you for testing the client interface\n");
+    dr_register_bb_event(bb_event);
+}
diff --git a/suite/tests/client-interface/truncate.expect b/suite/tests/client-interface/truncate.expect
new file mode 100644
index 000000000..59e4f35ca
--- /dev/null
+++ b/suite/tests/client-interface/truncate.expect
@@ -0,0 +1,2 @@
+thank you for testing the client interface
+hello world!
diff --git a/suite/tests/client-interface/unregister.c b/suite/tests/client-interface/unregister.c
new file mode 100644
index 000000000..6b9bb39a7
--- /dev/null
+++ b/suite/tests/client-interface/unregister.c
@@ -0,0 +1,39 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <stdio.h>
+
+int main()
+{
+    printf("thank you for testing the client interface");
+    return 0;
+}
diff --git a/suite/tests/client-interface/unregister.dll.c b/suite/tests/client-interface/unregister.dll.c
new file mode 100644
index 000000000..e09908262
--- /dev/null
+++ b/suite/tests/client-interface/unregister.dll.c
@@ -0,0 +1,57 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "dr_api.h"
+
+static void unregister();
+
+static
+dr_emit_flags_t bb_event(void* drcontext, void* tag, instrlist_t* bb, bool for_trace, bool translating)
+{
+    dr_printf("got BB event\n");
+    dr_insert_clean_call(drcontext, bb, instrlist_first(bb),
+                         (void *)unregister, false, 0);
+    return DR_EMIT_DEFAULT;
+}
+
+static
+void unregister()
+{
+    dr_unregister_bb_event(bb_event);
+}
+
+DR_EXPORT
+void dr_init(client_id_t id)
+{
+    dr_register_bb_event(bb_event);
+}
+
diff --git a/suite/tests/client-interface/unregister.expect b/suite/tests/client-interface/unregister.expect
new file mode 100644
index 000000000..0c0fd5bf8
--- /dev/null
+++ b/suite/tests/client-interface/unregister.expect
@@ -0,0 +1,2 @@
+got BB event
+thank you for testing the client interface
diff --git a/suite/tests/common/broadfun.c b/suite/tests/common/broadfun.c
new file mode 100644
index 000000000..ae3e73fa8
--- /dev/null
+++ b/suite/tests/common/broadfun.c
@@ -0,0 +1,142 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* single RET - many targets */
+/* the target function however should have a lot of independent paths to evaluate
+ * fib() is NOT a good example 
+ */
+
+/* got 66s native, vs 79s DR  = 1m6s 1m19s ~ 20%*/
+/* with another compare in the body went to 30% */
+/* (/ 21 16.0) */
+/* (/ 29 22.0) 1.380952380952381 for THREE compare's in a single body  */
+/* (/ 59 43.0) 1.37 for THREE compare's in a single body  */
+
+/* undefine this for a performance test */
+#ifndef NIGHTLY_REGRESSION
+# define NIGHTLY_REGRESSION
+#endif
+
+#include "tools.h"
+
+#include <stdio.h>
+
+#ifdef WINDOWS
+/* and compile with user32.lib */
+#include <windows.h>
+#endif
+
+#ifdef NIGHTLY_REGRESSION
+#  define ITER 400*1000
+#else
+#  define ITER 400*100000 // *100
+#endif
+
+//#define MORE  more fanout 
+//#define TOO_MUCH or more depth and more work
+
+int compare( const void *arg1, const void *arg2 )
+{
+   /* Compare all of both strings: */
+   return 
+#ifdef WINDOWS
+       _stricmp
+#else
+       strcasecmp
+#endif
+       ( * ( char** ) arg1, * ( char** ) arg2 );
+}
+
+int
+sort()
+{
+    int argc = 5;
+    char *argv[] = {"one", "two", "three","five", "six", "unsorted"};
+    int i;
+
+#ifdef TOO_MUCH
+   /* Sort remaining args using Quicksort algorithm: */
+   qsort( (void *)argv, (size_t)argc, sizeof( char * ), compare );
+#endif
+   
+   compare(&argv[4], &argv[2]);
+   compare(&argv[3], &argv[2]);
+   return compare(&argv[1], &argv[2]);
+}
+
+int
+main(int argc, char** argv) 
+{
+    int i;
+    
+    INIT();
+    USE_USER32();
+
+    /* now a little more realistic depths that fit in the RSB */
+    for (i=0; i<=ITER; i++) {
+	sort();
+	sort();
+	sort();
+        /* more pronounced with a few more - though three is sufficiently visible */
+#ifdef MORE
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+	sort();
+#endif
+    }
+
+    i = sort();
+    print("sort() = %s\n", i > 0 ? ">" : (i == 0 ? "=" : "<"));
+
+    print("done\n");
+}
diff --git a/suite/tests/common/broadfun.expect b/suite/tests/common/broadfun.expect
new file mode 100644
index 000000000..021ada9b8
--- /dev/null
+++ b/suite/tests/common/broadfun.expect
@@ -0,0 +1,2 @@
+sort() = >
+done
diff --git a/suite/tests/common/conflict.c b/suite/tests/common/conflict.c
new file mode 100644
index 000000000..7cc9cd664
--- /dev/null
+++ b/suite/tests/common/conflict.c
@@ -0,0 +1,77 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <math.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#define ITERS 1500000
+
+static int a[ITERS];
+
+void arch_init()
+{
+    printf("in conflict's arch init\n");
+}
+
+int main()
+{
+    double res = 0.;
+    int i,j;
+#ifdef USE_DYNAMO
+    int rc = dynamorio_app_init();
+    assert(rc == 0);
+    dynamorio_app_start();
+#endif
+    arch_init();
+  
+    for (i=0; i<ITERS; i++) {
+	if (i % 2 == 0) {
+	    res += cos(1./(double)(i+1));
+	} else {
+	    res += sin(1./(double)(i+1));
+	}
+	j = (i << 4) / (i | 0x38);
+	a[i] += j;
+    }
+    printf("%f\n", res);
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
diff --git a/suite/tests/common/conflict.expect b/suite/tests/common/conflict.expect
new file mode 100644
index 000000000..ba036be07
--- /dev/null
+++ b/suite/tests/common/conflict.expect
@@ -0,0 +1,2 @@
+in conflict's arch init
+750006.451810
diff --git a/suite/tests/common/decode-bad.c b/suite/tests/common/decode-bad.c
new file mode 100644
index 000000000..80c3ffd37
--- /dev/null
+++ b/suite/tests/common/decode-bad.c
@@ -0,0 +1,386 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifndef ASM_CODE_ONLY /* C code */
+#include <assert.h>
+#include <stdio.h>
+#include <math.h>
+
+#ifdef LINUX
+# include <unistd.h>
+# include <signal.h>
+# include <ucontext.h>
+# include <errno.h>
+# include <stdlib.h>
+#endif
+
+#include <setjmp.h>
+#include "tools.h" /* for print() */
+
+jmp_buf mark;
+static int count = 0;
+static bool invalid_lock;
+
+/* just use single-arg handlers */
+typedef void (*handler_t)(int);
+typedef void (*handler_3_t)(int, struct siginfo *, void *);
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#ifdef LINUX
+static void
+signal_handler(int sig)
+{
+    if (sig == SIGILL) {
+        count++;
+        if (invalid_lock) {
+            print("Invalid lock sequence, instance %d\n", count);
+            /* add this so output matches test on windows, FIXME : would like to test
+             * this on linux too, but won't work now (bug 651, 832) */
+            print("eax=1 ebx=2 ecx=3 edx=4 edi=5 esi=6 ebp=7\n");
+        } else 
+            print("Bad instruction, instance %d\n", count);
+        longjmp(mark, count);
+    }
+    if (sig == SIGSEGV) {
+        count++;
+        /* We can't distinguish but this is the only segv we expect */
+        print("Privileged instruction, instance %d\n", count);
+        longjmp(mark, count);
+    }
+    exit(-1);
+}
+
+#define ASSERT_NOERR(rc) do {                                   \
+  if (rc) {                                                     \
+     fprintf(stderr, "%s:%d rc=%d errno=%d %s\n",               \
+             __FILE__, __LINE__,                                \
+             rc, errno, strerror(errno));                       \
+  }                                                             \
+} while (0);
+
+/* set up signal_handler as the handler for signal "sig" */
+static void
+intercept_signal(int sig, handler_t handler)
+{
+    int rc;
+    struct sigaction act;
+
+    act.sa_sigaction = (handler_3_t) handler;
+    /* FIXME: due to DR bug 840 we cannot block ourself in the handler
+     * since the handler does not end in a sigreturn, so we have an empty mask
+     * and we use SA_NOMASK
+     */
+    rc = sigemptyset(&act.sa_mask); /* block no signals within handler */
+    ASSERT_NOERR(rc);
+    /* FIXME: due to DR bug #654 we use SA_SIGINFO -- change it once DR works */
+    act.sa_flags = SA_NOMASK | SA_SIGINFO | SA_ONSTACK;
+    
+    /* arm the signal */
+    rc = sigaction(sig, &act, NULL);
+    ASSERT_NOERR(rc);
+}
+
+#else
+/* sort of a hack to avoid the MessageBox of the unhandled exception spoiling
+ * our batch runs
+ */
+# include <windows.h>
+/* top-level exception handler */
+static LONG
+our_top_handler(struct _EXCEPTION_POINTERS * pExceptionInfo)
+{
+    if (pExceptionInfo->ExceptionRecord->ExceptionCode ==
+        /* Windows doesn't have an EXCEPTION_ constant to equal the invalid lock code */
+        0xc000001e/*STATUS_INVALID_LOCK_SEQUENCE*/
+        /* FIXME: DR doesn't generate the invalid lock exception */
+        || (invalid_lock && pExceptionInfo->ExceptionRecord->ExceptionCode ==
+            STATUS_ILLEGAL_INSTRUCTION)) {
+        CONTEXT *cxt = pExceptionInfo->ContextRecord;
+        count++;
+        print("Invalid lock sequence, instance %d\n", count);
+        /* FIXME : add CXT_XFLAGS (currently comes back incorrect), eip, esp? */
+        print("eax="SZFMT" ebx="SZFMT" ecx="SZFMT" edx="SZFMT" "
+              "edi="SZFMT" esi="SZFMT" ebp="SZFMT"\n", 
+              cxt->CXT_XAX, cxt->CXT_XBX, cxt->CXT_XCX, cxt->CXT_XDX,
+              cxt->CXT_XDI, cxt->CXT_XSI, cxt->CXT_XBP);
+        longjmp(mark, count);
+    }
+    if (pExceptionInfo->ExceptionRecord->ExceptionCode == STATUS_ILLEGAL_INSTRUCTION) {
+        count++;
+        print("Bad instruction, instance %d\n", count);
+        longjmp(mark, count);
+    }
+    if (pExceptionInfo->ExceptionRecord->ExceptionCode ==
+        STATUS_PRIVILEGED_INSTRUCTION) {
+        count++;
+        print("Privileged instruction, instance %d\n", count);
+        longjmp(mark, count);
+    }
+    /* Shouldn't get here in normal operation so this isn't #if VERBOSE */
+    print("Exception 0x"PFMT" occurred, process about to die silently\n",
+          pExceptionInfo->ExceptionRecord->ExceptionCode);
+    if (pExceptionInfo->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
+        print("\tPC "PFX" tried to %s address "PFX"\n",
+            pExceptionInfo->ExceptionRecord->ExceptionAddress, 
+            (pExceptionInfo->ExceptionRecord->ExceptionInformation[0]==0)?"read":"write",
+            pExceptionInfo->ExceptionRecord->ExceptionInformation[1]);
+    }
+    return EXCEPTION_EXECUTE_HANDLER; /* => global unwind and silent death */
+}
+#endif
+
+int main(int argc, char *argv[])
+{
+    double res = 0.;
+    int i,j;
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+  
+#ifdef LINUX
+    intercept_signal(SIGILL, signal_handler);
+    intercept_signal(SIGSEGV, signal_handler);
+#else
+# ifdef X64_DEBUGGER
+    /* FIXME: the vectored handler works fine in the debugger, but natively
+     * the app crashes here: yet the SetUnhandled hits infinite fault loops
+     * in the debugger, and works fine natively!
+     */
+    AddVectoredExceptionHandler(1/*first*/,
+                                (PVECTORED_EXCEPTION_HANDLER) our_top_handler);
+# else
+    SetUnhandledExceptionFilter((LPTOP_LEVEL_EXCEPTION_FILTER) our_top_handler);
+# endif
+#endif
+
+    /* privileged instructions */
+    print("Privileged instructions about to happen\n");
+    count = 0;
+    i = setjmp(mark);
+    switch (i) {
+    case 0: test_priv_0();
+    case 1: test_priv_1();
+    case 2: test_priv_2();
+    case 3: test_priv_3();
+    }
+
+    /* prefix tests */
+    print("OK instr about to happen\n");
+    /* multiple prefixes */
+    /* FIXME: actually these prefixes on a jmp are "reserved" but this seems to work */
+    test_prefix_0();
+    print("Bad instr about to happen\n");
+    /* lock prefix, which is illegal instruction if placed on jmp */
+    count = 0;
+    invalid_lock = true;
+    if (setjmp(mark) == 0) {
+        test_prefix_1();
+    }
+    invalid_lock = false;
+
+    print("Invalid instructions about to happen\n");
+    count = 0;
+    i = setjmp(mark);
+    switch (i) {
+        /* note that we decode until a CTI, so for every case the suffix is decoded
+         * and changes in later cases may fail even the earlier ones.
+         */
+    case 0: test_inval_0();
+    case 1: test_inval_1();
+    case 2: test_inval_2();
+    case 3: test_inval_3();
+    case 4: test_inval_4();
+    case 5: test_inval_5();
+    case 6: test_inval_6();
+    case 7: test_inval_7();
+    default: ;
+    }
+
+    print("All done\n");
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
+
+#else /* asm code *************************************************************/
+#include "asm_defines.asm"
+START_FILE
+
+#define FUNCNAME test_priv_0
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        mov  REG_XAX, dr0
+        ret
+        END_FUNC(FUNCNAME)
+#undef FUNCNAME
+#define FUNCNAME test_priv_1
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        mov  dr7, REG_XAX
+        ret
+        END_FUNC(FUNCNAME)
+#undef FUNCNAME
+#define FUNCNAME test_priv_2
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        mov  REG_XAX, cr0
+        ret
+        END_FUNC(FUNCNAME)
+#undef FUNCNAME
+#define FUNCNAME test_priv_3
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        mov  cr3, REG_XAX
+        ret
+        END_FUNC(FUNCNAME)
+
+#undef FUNCNAME
+#define FUNCNAME test_prefix_0
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        RAW(65) RAW(f2) RAW(f3) RAW(eb) RAW(00)
+        ret
+        END_FUNC(FUNCNAME)
+
+#undef FUNCNAME
+#define FUNCNAME test_prefix_1
+       /* If we make this a leaf function (no SEH directives), our top-level handler
+        * does not get called!  Annoying.
+        */
+        DECLARE_FUNC_SEH(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        /* push callee-saved registers */
+        PUSH_SEH(REG_XBX)
+        PUSH_SEH(REG_XBP)
+        PUSH_SEH(REG_XSI)
+        PUSH_SEH(REG_XDI)
+        END_PROLOG
+        mov  eax, 1
+        mov  ebx, 2
+        mov  ecx, 3
+        mov  edx, 4
+        mov  edi, 5
+        mov  esi, 6
+        mov  ebp, 7
+        RAW(f0) RAW(eb) RAW(00)
+        add      REG_XSP, 0 /* make a legal SEH64 epilog */
+        pop      REG_XDI
+        pop      REG_XSI
+        pop      REG_XBP
+        pop      REG_XBX
+        ret
+        END_FUNC(FUNCNAME)
+
+#undef FUNCNAME
+#define FUNCNAME test_inval_0
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        RAW(df) RAW(fa)
+        ret
+        END_FUNC(FUNCNAME)
+#undef FUNCNAME
+#define FUNCNAME test_inval_1
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        RAW(0f) RAW(04)
+        ret
+        END_FUNC(FUNCNAME)
+#undef FUNCNAME
+#define FUNCNAME test_inval_2
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        RAW(fe) RAW(30)
+        ret
+        END_FUNC(FUNCNAME)
+#undef FUNCNAME
+#define FUNCNAME test_inval_3
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        RAW(ff) RAW(38)
+        ret
+        END_FUNC(FUNCNAME)
+#undef FUNCNAME
+#define FUNCNAME test_inval_4
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        RAW(f3) RAW(0f) RAW(13)
+        ret
+        END_FUNC(FUNCNAME)
+#undef FUNCNAME
+#define FUNCNAME test_inval_5
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        /* case 8840 we crash after going through this bb */
+        /* ud2 */
+        RAW(0f) RAW(0b)
+        RAW(20) RAW(0f) 
+#if 0
+        RAW(8c) RAW(c6) 
+#endif
+        RAW(ff) RAW(ff) 
+        RAW(ff) RAW(d9)
+        ret
+        END_FUNC(FUNCNAME)
+#undef FUNCNAME
+#define FUNCNAME test_inval_6
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        /* case 6962 - invalid Mod byte for a call far,
+         * so should get #UD as well
+         * we should make sure we either mark invalid during decode
+         * or properly detect it is invalid during execution
+         */
+        /* we just have to force ecx = 0x13 to make sure the same result */
+        RAW(ff) RAW(d9)
+        ret
+        END_FUNC(FUNCNAME)
+#undef FUNCNAME
+#define FUNCNAME test_inval_7
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+         /* Although data16 means it is 4 bytes and fits in
+         * a register, this is invalid.
+         */
+        RAW(66) RAW(ff) RAW(d9)
+        ret
+        END_FUNC(FUNCNAME)
+
+END_FILE
+#endif
diff --git a/suite/tests/common/decode-bad.expect b/suite/tests/common/decode-bad.expect
new file mode 100644
index 000000000..18f4ff785
--- /dev/null
+++ b/suite/tests/common/decode-bad.expect
@@ -0,0 +1,19 @@
+Privileged instructions about to happen
+Privileged instruction, instance 1
+Privileged instruction, instance 2
+Privileged instruction, instance 3
+Privileged instruction, instance 4
+OK instr about to happen
+Bad instr about to happen
+Invalid lock sequence, instance 1
+eax=1 ebx=2 ecx=3 edx=4 edi=5 esi=6 ebp=7
+Invalid instructions about to happen
+Bad instruction, instance 1
+Bad instruction, instance 2
+Bad instruction, instance 3
+Bad instruction, instance 4
+Bad instruction, instance 5
+Bad instruction, instance 6
+Bad instruction, instance 7
+Bad instruction, instance 8
+All done
diff --git a/suite/tests/common/decode.c b/suite/tests/common/decode.c
new file mode 100644
index 000000000..e6a17e9d4
--- /dev/null
+++ b/suite/tests/common/decode.c
@@ -0,0 +1,608 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifndef ASM_CODE_ONLY /* C code */
+#include <assert.h>
+#include <stdio.h>
+#include <math.h>
+
+#ifdef LINUX
+# include <unistd.h>
+# include <signal.h>
+# include <ucontext.h>
+# include <errno.h>
+# include <stdlib.h>
+#endif
+
+#include <setjmp.h>
+#include "tools.h" /* for print() */
+
+jmp_buf mark;
+static int count = 0;
+static bool print_access_vio = true;
+
+/* just use single-arg handlers */
+typedef void (*handler_t)(int);
+typedef void (*handler_3_t)(int, struct siginfo *, void *);
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#define VERBOSE 0
+
+#define ITERS 1500000
+
+static int a[ITERS];
+
+#ifdef LINUX
+# define ALT_STACK_SIZE  (SIGSTKSZ*3)
+
+static void
+signal_handler(int sig)
+{
+    if (sig == SIGILL) {
+        count++;
+        print("Bad instruction, instance %d\n", count);
+        longjmp(mark, count);
+    } else if (sig == SIGSEGV) {
+        count++;
+        if (print_access_vio)
+            print("Access violation, instance %d\n", count);
+        longjmp(mark, count);
+    }
+    exit(-1);
+}
+
+#define ASSERT_NOERR(rc) do {                                   \
+  if (rc) {                                                     \
+     fprintf(stderr, "%s:%d rc=%d errno=%d %s\n",               \
+             __FILE__, __LINE__,                                \
+             rc, errno, strerror(errno));                       \
+  }                                                             \
+} while (0);
+
+/* set up signal_handler as the handler for signal "sig" */
+static void
+intercept_signal(int sig, handler_t handler)
+{
+    int rc;
+    struct sigaction act;
+
+    act.sa_sigaction = (handler_3_t) handler;
+    /* FIXME: due to DR bug 840 we cannot block ourself in the handler
+     * since the handler does not end in a sigreturn, so we have an empty mask
+     * and we use SA_NOMASK
+     */
+    rc = sigemptyset(&act.sa_mask); /* block no signals within handler */
+    ASSERT_NOERR(rc);
+    /* FIXME: due to DR bug #654 we use SA_SIGINFO -- change it once DR works */
+    act.sa_flags = SA_NOMASK | SA_SIGINFO | SA_ONSTACK;
+    
+    /* arm the signal */
+    rc = sigaction(sig, &act, NULL);
+    ASSERT_NOERR(rc);
+}
+
+#else
+/* sort of a hack to avoid the MessageBox of the unhandled exception spoiling
+ * our batch runs
+ */
+# include <windows.h>
+/* top-level exception handler */
+static LONG
+our_top_handler(struct _EXCEPTION_POINTERS * pExceptionInfo)
+{
+    if (pExceptionInfo->ExceptionRecord->ExceptionCode == STATUS_ILLEGAL_INSTRUCTION) {
+        count++;
+        print("Bad instruction, instance %d\n", count);
+        longjmp(mark, count);
+    }
+    /* Shouldn't get here in normal operation so this isn't #if VERBOSE */
+    if (pExceptionInfo->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
+#if VERBOSE
+        /* DR gets the target addr wrong for far call/jmp so we don't print it */
+        print("\tPC "PFX" tried to %s address "PFX"\n",
+            pExceptionInfo->ExceptionRecord->ExceptionAddress, 
+            (pExceptionInfo->ExceptionRecord->ExceptionInformation[0]==0)?"read":"write",
+            pExceptionInfo->ExceptionRecord->ExceptionInformation[1]);
+#endif
+        count++;
+        if (print_access_vio)
+            print("Access violation, instance %d\n", count);
+        longjmp(mark, count);
+    }
+    print("Exception "PFX" occurred, process about to die silently\n",
+          pExceptionInfo->ExceptionRecord->ExceptionCode);
+    return EXCEPTION_EXECUTE_HANDLER; /* => global unwind and silent death */
+}
+#endif
+
+int main(int argc, char *argv[])
+{
+    double res = 0.;
+    int i,j;
+    char *buf;
+#ifdef LINUX
+    stack_t sigstack;
+#endif
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+  
+#ifdef LINUX
+    /* our modrm16 tests clobber esp so we need an alternate stack */
+    sigstack.ss_sp = (char *) malloc(ALT_STACK_SIZE);
+    sigstack.ss_size = ALT_STACK_SIZE;
+    sigstack.ss_flags = SS_ONSTACK;
+    i = sigaltstack(&sigstack, NULL);
+    assert(i == 0);
+    intercept_signal(SIGILL, signal_handler);
+    intercept_signal(SIGSEGV, signal_handler);
+#else
+    SetUnhandledExceptionFilter((LPTOP_LEVEL_EXCEPTION_FILTER) our_top_handler);
+#endif
+
+    buf = allocate_mem(7*256+1, ALLOW_READ|ALLOW_WRITE|ALLOW_EXEC);
+    assert(buf != NULL);
+
+#ifndef X64
+    print("Jumping to a sequence of every addr16 modrm byte\n");
+    for (j=0; j<256; j++) {
+        int mod = ((j >> 6) & 0x3); /* top 2 bits */
+        int reg = ((j >> 3) & 0x7); /* middle 3 bits */
+        int rm  = (j & 0x7);        /* bottom 3 bits */
+# if defined(LINUX) || defined(X64)
+        buf[j*7 + 0] = 0x65; /* gs: */
+# else
+        buf[j*7 + 0] = 0x64; /* fs: */
+# endif
+        buf[j*7 + 1] = 0x67; /* addr16 */
+        buf[j*7 + 2] = 0x8b; /* load */
+# ifdef WINDOWS
+        /* Windows can't handle stack pointer being off */
+        if (reg == 4) { /* xsp */
+            buf[j*7 + 3] = j | 0x8;
+        } else
+            buf[j*7 + 3] = j; /* nearly every single modrm byte */
+# else
+        buf[j*7 + 3] = j; /* every single modrm byte */
+# endif
+        if (mod == 1) {
+            buf[j*7 + 4] = 0x03; /* disp */
+            buf[j*7 + 5] = 0xc3;
+        } else if (mod == 2 || (mod == 0 && rm == 6)) {
+            buf[j*7 + 4] = 0x03; /* disp */
+            buf[j*7 + 5] = 0x00; /* disp */
+        } else {
+            buf[j*7 + 4] = 0xc3; /* ret */
+            buf[j*7 + 5] = 0xc3;
+        }
+        buf[j*7 + 6] = 0xc3;
+    }
+    buf[256*7] = 0xcc;
+    print_access_vio = false;
+    for (j=0; j<256; j++) {
+        i = setjmp(mark);
+        if (i == 0)
+            test_modrm16(&buf[j*7]);
+        else
+            continue;
+    }
+    print("Done with modrm test: tested %d\n", j);
+    count = 0;
+    print_access_vio = true;
+#endif /* !X64 */
+
+    /* multi-byte nop tests (case 9862) */
+    i = setjmp(mark);
+    if (i == 0) {
+        print("Testing nops\n");
+        test_nops();
+        print("Done with nops\n");
+    }
+
+    /* SSE3 and 3DNow instrs will not run on all processors so we can't have this
+     * regression test fully test everything: for now its main use is running
+     * manually on the proper machines, or manually verifying decoding of these,
+     * but we'll leave as a suite/ regression test.
+     */
+
+    /* SSE3 tests: mostly w/ modrm of (%edx) */
+    i = setjmp(mark);
+    if (i == 0) {
+        print("Testing SSE3\n");
+        test_sse3(buf);
+        print("Should not get here\n");
+    }
+
+    /* 3D-Now tests: mostly w/ modrm of (%ebx) */
+    i = setjmp(mark);
+    if (i == 0) {
+        print("Testing 3D-Now\n");
+        test_3dnow(buf);
+        print("Should not get here\n");
+    }
+
+    /* case 6962: far call/jmp tests 
+     * Note that DR currently gets the target address wrong for all of these
+     * since we skip the segment and only care about the address:
+     * not going to fix that anytime soon.
+     */
+    print("Testing far call/jmp\n");
+    test_far_cti();
+
+    /* PR 242815: data16 mbr */
+    print("Testing data16 mbr\n");
+    test_data16_mbr();
+
+#ifdef LINUX
+    free(sigstack.ss_sp);
+#endif
+
+    print("All done\n");
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
+
+
+#else /* asm code *************************************************************/
+#include "asm_defines.asm"
+START_FILE
+
+#define FUNCNAME test_modrm16
+        DECLARE_FUNC_SEH(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        /* can't push ARG1 here b/c of SEH64 epilog restrictions */
+        mov      REG_XAX, ARG1
+        /* push callee-saved registers */
+        PUSH_SEH(REG_XBX)
+        PUSH_SEH(REG_XBP)
+        PUSH_SEH(REG_XSI)
+        PUSH_SEH(REG_XDI)
+        PUSH_NONCALLEE_SEH(REG_XAX)
+        END_PROLOG
+        mov      ax, 4
+        mov      bx, 8
+        mov      cx, 4
+        mov      dx, 8
+        mov      si, 4
+        mov      di, 8
+        mov      bp, 8
+        CALLC0(PTRSZ [REG_XSP] /* ARG1 */)
+        pop      REG_XAX /* arg1 */
+        add      REG_XSP, 0 /* make a legal SEH64 epilog */
+        pop      REG_XDI
+        pop      REG_XSI
+        pop      REG_XBP
+        pop      REG_XBX
+        ret
+        END_FUNC(FUNCNAME)
+
+#undef FUNCNAME
+#define FUNCNAME test_nops
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        RAW(66) RAW(90)
+        RAW(67) RAW(90)
+        RAW(f2) RAW(90)
+        RAW(f3) RAW(90)
+        RAW(66) RAW(66) RAW(66) RAW(66) RAW(66) RAW(90)
+        RAW(0f) RAW(1f) RAW(00)
+        RAW(0f) RAW(1f) RAW(40) RAW(00)
+        RAW(0f) RAW(1f) RAW(44) RAW(00) RAW(00)
+        RAW(66) RAW(0f) RAW(1f) RAW(44) RAW(00) RAW(00)
+        RAW(0f) RAW(1f) RAW(80) RAW(00) RAW(00) RAW(00) RAW(00)
+        RAW(0f) RAW(1f) RAW(84) RAW(00) RAW(00) RAW(00) RAW(00) RAW(00)
+        RAW(66) RAW(0f) RAW(1f) RAW(84) RAW(00) RAW(00) RAW(00) RAW(00) RAW(00)
+        ret
+        END_FUNC(FUNCNAME)
+
+#undef FUNCNAME
+#define FUNCNAME test_sse3
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        mov    REG_XAX, ARG1
+        RAW(f2) RAW(0f) RAW(7c) RAW(20) /* haddps (%rax), xmm4 */
+        RAW(f2) RAW(0f) RAW(7d) RAW(20) /* hsubps (%rax), xmm4 */
+        RAW(f2) RAW(0f) RAW(d0) RAW(20) /* addsubps (%rax), xmm4 */
+        RAW(f2) RAW(0f) RAW(f0) RAW(20) /* lddqu (%rax), xmm4 */
+        RAW(f3) RAW(0f) RAW(12) RAW(20) /* movsldup (%rax), xmm4 */
+        RAW(f2) RAW(0f) RAW(12) RAW(20) /* movddup (%rax), xmm4 */
+        RAW(f3) RAW(0f) RAW(16) RAW(20) /* movshdup (%rax), xmm4 */
+        mov  eax, 0
+        mov  ecx, 0
+        mov  edx, 0
+        RAW(0f) RAW(01) RAW(c8) /* monitor */
+        RAW(0f) RAW(01) RAW(c9) /* mwait */
+        /* we want failure on sse3 machine, to have constant output
+         * (this will produce "Invalid opcode encountered" warning) */
+        RAW(f3) RAW(0f) RAW(7c) RAW(20) /* bad */
+        ret
+        END_FUNC(FUNCNAME)
+
+#undef FUNCNAME
+#define FUNCNAME test_3dnow
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        mov    REG_XAX, ARG1 /* these instrs use (%xax) */
+        RAW(0f) RAW(0e)                   /* femms */
+        RAW(0f) RAW(0f) RAW(08) RAW(bf) /* pavgusb (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(9e) /* pfadd (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(ae) /* pfacc (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(90) /* pfcmpge (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(a0) /* pfcmpgt (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(b0) /* pfcmpeq (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(94) /* pfmin (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(a4) /* pfmax (%xax),%mm1 */
+        /* try 10(%eax) */
+        RAW(0f) RAW(0f) RAW(48) RAW(0a) RAW(b4) /* pfmul 10(%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(96) /* pfrcp (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(a6) /* pfrcpit1 (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(b6) /* pfrcpit2 (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(97) /* pfrsqrt (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(a7) /* pfrsqit1 (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(b7) /* pmulhrw (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(9a) /* pfsub (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(aa) /* pfsubr (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(0d) /* pi2fd (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(1d) /* pf2id (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(0c) /* pi2fw (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(1c) /* pf2iw (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(8a) /* pfnacc (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(8e) /* pfpnacc (%xax),%mm1 */
+        RAW(0f) RAW(0f) RAW(08) RAW(bb) /* pswapd (%xax),%mm1 */
+        /* try unknown opcode: we want failure on amd machine
+         * anyway, to have constant output */
+        RAW(0f) RAW(0f) RAW(08) RAW(00) /* unknown */
+        ret
+        END_FUNC(FUNCNAME)
+
+DECL_EXTERN(mark)
+
+#ifdef WINDOWS
+# ifdef X64
+DECL_EXTERN(setjmp)
+#  define CALL_SETJMP \
+        lea   REG_XAX, mark @N@\
+        CALLC2(setjmp, REG_XAX, REG_XSP)
+# else
+DECL_EXTERN(_setjmp3)
+#  define CALL_SETJMP \
+        lea   REG_XAX, mark @N@\
+        CALLC2(_setjmp3, REG_XAX, 0)
+# endif
+#else
+DECL_EXTERN(setjmp)
+# define CALL_SETJMP \
+        lea   REG_XAX, mark @N@\
+        CALLC1(setjmp, REG_XAX)
+#endif
+
+/* FIXME PR 271834: we need some far ctis that actually succeed, to fully test
+ * DR's handling of them.  We should test the resulting target address
+ * and stack pointer.  We can do that initially by targeting the current
+ * cs, and perhaps adding other segments once we have PR 271317.
+ *
+ * I did some manual testing to confirm AMD vs Intel 64-bit behavior
+ * on far cti operand sizes, and should incorporate those tests here.
+ * Note that for data16 I don't yet know how to get SEH64 to recover
+ * (from a call or ret).  Here are my tests in my notes:
+ *
+ * ret_far:
+ *  on Intel:  push 0x00331234; RAW(66) RAW(cb) => rip=0x1234
+ *    mov rax, HEX(000000339abcdef0); push rax; RAW(cb) => rip=0x9abcdef0
+ *    push 0x33; mov rax, HEX(000000339abcdef0); push rax; RAW(48) RAW(cb)
+ *      => rip=0x339abcdef0
+ *  AMD: same behavior
+ *  => ret_far is like iret: default is still 32-bits, and expands and shrinks
+ *
+ * call_far:
+ *   push HEX(00331234); RAW(66) RAW(ff) RAW(1c) RAW(24) == lcallw (%rsp)
+ *     => rip=0x1234, pushed 2-byte segment and 2-byte retaddr (truncated)
+ *   mov rax, HEX(000000339abcdef0); push rax; lcallq (%rsp)
+ *     => rip=0x9abcdef0, pushed 4-byte segment and 4-byte retaddr
+ *   push 0x33; mov rax, HEX(000000339abcdef0); push rax; rex.w lcallq (%rsp)
+ *     => rip=0x9abcdef0, pushed 8-byte segment and 8-byte retaddr
+ *     for AMD, same, except rip=0x9abcdef0 (this is PR 270274)
+ * so AMD honors the rex.w to give you a 64-bit retaddr, but still truncates your
+ *   target address.  crazy.
+ *
+ * Should add rex.w tests (xref PR 241573).
+ * Should also add iret tests (xref PR 191977).
+ */
+#undef FUNCNAME
+#define FUNCNAME test_far_cti
+        DECLARE_FUNC_SEH(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        END_PROLOG
+        CALL_SETJMP
+        cmp   REG_XAX, 0
+        jne   test_far_cti_1
+        /* ljmp to kernel address space: ljmp %0xbc9a:0xf8563412 */
+        RAW(ea) RAW(12) RAW(34) RAW(56) RAW(f8) RAW(9a) RAW(bc)
+    test_far_cti_1:
+        CALL_SETJMP
+        cmp   REG_XAX, 0
+        jne   test_far_cti_2
+        /* ljmp to user address space: ljmp %0xbc9a:0x78563412 */
+        RAW(ea) RAW(12) RAW(34) RAW(56) RAW(78) RAW(9a) RAW(bc)
+    test_far_cti_2:
+        CALL_SETJMP
+        cmp   REG_XAX, 0
+        jne   test_far_cti_3
+        /* lcall to kernel address space */
+        RAW(9a) RAW(12) RAW(34) RAW(56) RAW(f8) RAW(9a) RAW(bc)
+    test_far_cti_3:
+        CALL_SETJMP
+        cmp   REG_XAX, 0
+        jne   test_far_cti_4
+        /* lcall to user address space */
+        RAW(9a) RAW(12) RAW(34) RAW(56) RAW(78) RAW(9a) RAW(bc)
+    test_far_cti_4:
+        CALL_SETJMP
+        cmp   REG_XAX, 0
+        jne   test_far_cti_5
+        /* targeting what's stored there, but a crash is a crash */
+        mov   eax, HEX(deadbeef)
+        RAW(ff) RAW(28) /* ljmp (%eax) */
+    test_far_cti_5:
+        CALL_SETJMP
+        cmp   REG_XAX, 0
+        jne   test_far_cti_6
+        /* targeting what's stored there, but a crash is a crash */
+        mov   eax, HEX(deadbeef)
+        RAW(ff) RAW(18) /* lcall (%eax) */
+    test_far_cti_6:
+        ret
+        END_FUNC(FUNCNAME)
+
+#undef FUNCNAME
+#define FUNCNAME test_data16_mbr
+        DECLARE_FUNC_SEH(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        /* if we don't push something we'll clobber the real retaddr
+         * when we do our data16 ret and data16 lret below,
+         * and not even longjmp will save us (seems to happen to work
+         * natively though but not for DR: didn't look further: seems like
+         * no guarantee natively) */
+        /* I can't get push imm to work w/ gas using Intel syntax:
+         * it doesn't like hex constants, or $, or decimal w/ over 8 digits!
+         * Definitely a bug there.  So instead we go through a register.
+         * We don't care about zero vs sign extend.
+         * Ugh, SEH64 fails when going through a register: I don't see why.
+         * I don't see any documentation saying I can't do a mov imm into
+         * a volatile register.  Not worth my time: just splitting.
+         */ 
+#ifdef WINDOWS
+        PUSH_NONCALLEE_SEH(HEX(deadbeef))
+#else
+        mov   REG_XAX, HEX(deadbeef)
+        PUSH_NONCALLEE_SEH(REG_XAX)
+#endif
+        END_PROLOG
+        CALL_SETJMP
+        cmp   REG_XAX, 0
+        jne   test_data16_mbr_1
+        mov   ecx, HEX(deadbeef) /* needed for 64-bit where we target rcx
+                                  * (may still get NX fault here, but won't
+                                  * under DR: PR 210383) */
+        /* FIXME PR 271863: this dies on 64-bit AMD: not sure how to get
+         * SEH64 to recover from the stack misalignment.
+         */
+        RAW(66) RAW(ff) RAW(d1) /* call %cx */
+    test_data16_mbr_1:
+        CALL_SETJMP
+        cmp   REG_XAX, 0
+        jne   test_data16_mbr_2
+        mov   ecx, HEX(deadbeef) /* needed for 64-bit where we target rcx */
+        RAW(66) RAW(ff) RAW(e1) /* jmp %cx */
+    test_data16_mbr_2:
+        CALL_SETJMP
+        cmp   REG_XAX, 0
+        jne   test_data16_mbr_3
+        /* the push above means we won't clobber retaddr (can't push here: SEH64!) */
+        /* 64-bit:
+         * Note that on Intel this is a 64-bit ret, and since SEH64 looks only
+         * from RIP forward it thinks this is an epilog, which is why the
+         * unwind succeeds (else it would pop rax and mess up the stack).
+         * FIXME PR 271863: this will die on AMD, natively and in DR: can't have
+         * handler for this routine (matches epilog), so can we recover w/ an
+         * outer handler, or will we die since stack is not aligned?
+         */
+        RAW(66) RAW(c3) /* data16 ret */
+    test_data16_mbr_3:
+    /* repeat all the far tests w/ data16 */
+        CALL_SETJMP
+        cmp   REG_XAX, 0
+        jne   test_data16_mbr_4
+        /* data16 ljmp %0xbc9a:0x7856 */
+        RAW(66) RAW(ea) RAW(56) RAW(78) RAW(9a) RAW(bc)
+    test_data16_mbr_4:
+        CALL_SETJMP
+        cmp   REG_XAX, 0
+        jne   test_data16_mbr_5
+        /* data16 lcall %0xbc9a:0x7856 */
+        RAW(66) RAW(9a) RAW(56) RAW(f8) RAW(9a) RAW(bc)
+    test_data16_mbr_5:
+        CALL_SETJMP
+        cmp   REG_XAX, 0
+        jne   test_data16_mbr_6
+        mov   eax, HEX(deadbeef)
+        RAW(66) RAW(ff) RAW(28) /* data16 ljmp (%eax) */
+    test_data16_mbr_6:
+        CALL_SETJMP
+        cmp   REG_XAX, 0
+        jne   test_data16_mbr_7
+        mov   eax, HEX(deadbeef)
+        RAW(66) RAW(ff) RAW(18) /* data16 lcall (%eax) */
+    test_data16_mbr_7:
+        CALL_SETJMP
+        cmp   REG_XAX, 0
+        jne   test_data16_mbr_8
+        /* the push above means we won't clobber retaddr (can't push here: SEH64!) */
+#ifdef X64
+        /* Since we don't preserve the cs change we don't get the same fault
+         * that we get natively, and our current SEH64 setup won't catch
+         * the fault from a misaligned ret mid-routine (see above as well).
+         * PR 271317 covers handing the cs change. */
+        RAW(cb) /* lret */
+#else
+        RAW(66) RAW(cb) /* data16 lret */
+#endif
+    test_data16_mbr_8:
+        CALL_SETJMP
+        cmp   REG_XAX, 0
+        jne   test_data16_mbr_9
+        RAW(66) RAW(e9) RAW(00) RAW(00) /* data16 jmp next instr */
+        /* Intel x64: 1st 2 are part of disp (ignores data16), then crash; AMD: crash */
+        RAW(00) RAW(00) RAW(00) RAW(00)
+    test_data16_mbr_9:
+        CALL_SETJMP
+        cmp   REG_XAX, 0
+        jne   test_data16_mbr_10
+        RAW(66) RAW(e8) RAW(00) RAW(00) /* data16 call next instr */
+        /* Intel x64: 1st 2 are part of disp (ignores data16), then crash; AMD: crash */
+        RAW(00) RAW(00) RAW(00) RAW(00)
+    test_data16_mbr_10:
+        add      REG_XSP, ARG_SZ /* make a legal SEH64 epilog, and clean up push */
+        ret
+        END_FUNC(FUNCNAME)
+
+END_FILE
+#endif
diff --git a/suite/tests/common/decode.template b/suite/tests/common/decode.template
new file mode 100644
index 000000000..49488b4db
--- /dev/null
+++ b/suite/tests/common/decode.template
@@ -0,0 +1,51 @@
+#if -X64 :
+Jumping to a sequence of every addr16 modrm byte
+# if +PROGRAM_SHEPHERDING : +-security +-no_executable_if_alloc --low --client --thin_client
+#  if : --detect_mode
+#   SEC_VIO_AUTO_STOP
+#   if : --internal_detach_mask
+#    STOP
+#   endif
+#  else
+#   SEC_VIO_CONT
+#  endif
+# endif
+Done with modrm test: tested 256
+#endif
+Testing nops
+Done with nops
+Testing SSE3
+Bad instruction, instance 1
+Testing 3D-Now
+Bad instruction, instance 2
+Testing far call/jmp
+#if +X64 :
+Bad instruction, instance 3
+Bad instruction, instance 4
+Bad instruction, instance 5
+Bad instruction, instance 6
+#else
+Access violation, instance 3
+Access violation, instance 4
+Access violation, instance 5
+Access violation, instance 6
+#endif
+Access violation, instance 7
+Access violation, instance 8
+Testing data16 mbr
+Access violation, instance 9
+Access violation, instance 10
+Access violation, instance 11
+#if +X64 :
+Bad instruction, instance 12
+Bad instruction, instance 13
+#else
+Access violation, instance 12
+Access violation, instance 13
+#endif
+Access violation, instance 14
+Access violation, instance 15
+Access violation, instance 16
+Access violation, instance 17
+Access violation, instance 18
+All done
diff --git a/suite/tests/common/eflags.c b/suite/tests/common/eflags.c
new file mode 100644
index 000000000..f63f7c042
--- /dev/null
+++ b/suite/tests/common/eflags.c
@@ -0,0 +1,165 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifndef ASM_CODE_ONLY /* C code */
+/* Tests preservation of eflags across indirect branches
+ */
+
+#include "tools.h"
+
+#ifdef USE_DYNAMO
+# include "dynamorio.h"
+#endif
+
+#define VERBOSE 0
+
+/*
+ * eflags we care about:
+ *  11 10  9  8  7  6  5  4  3  2  1  0
+ *  OF DF       SF ZF    AF    PF    CF 
+ */
+const char *flags[] = {
+    "CF", "", "PF", "", "AF", "", "ZF", "SF", "",  "", "DF", "OF"
+};
+
+const uint eflag_pos[] = {
+    0, 2, 4, 6, 7, 10, 11
+};
+#define NUM_FLAGS (sizeof(eflag_pos)/sizeof(eflag_pos[0]))
+
+void
+test_flag(uint eflags, uint pos, bool set)
+{
+#if VERBOSE
+    print("eflags where %d should be %d: "PFX"\n", pos, set, eflags);
+#endif
+    if ((set && ((eflags & (1 << pos)) == 0)) ||
+        (!set && ((eflags & (1 << pos)) != 0)))
+        print("ERROR %d %s\n", set, flags[pos]);
+    else
+        print("OK %d %s\n", set, flags[pos]);
+}
+
+int
+main()
+{
+    uint i;
+    INIT();
+    
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    for (i=0; i<NUM_FLAGS; i++) {
+        test_eflags_pos(eflag_pos[i]);
+    }
+    
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
+
+
+#else /* asm code *************************************************************/
+#include "asm_defines.asm"
+START_FILE
+
+DECL_EXTERN(test_flag)
+
+#define FUNCNAME test_eflags_pos
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        /* We don't bother w/ SEH64 directives, though we're an illegal leaf routine! */
+        mov      REG_XCX, ARG1
+        push     REG_XCX /* save */
+        CALLC1(set_flag, REG_XCX)
+        mov      REG_XCX, PTRSZ [REG_XSP]
+        PUSHF
+        pop      REG_XAX
+        /* having DF set messes up printing for x64 */
+        push     0
+        POPF
+        CALLC3(test_flag, REG_XAX, REG_XCX, 1)
+
+        mov      REG_XCX, PTRSZ [REG_XSP]
+        CALLC1(clear_flag, REG_XCX)
+        mov      REG_XCX, PTRSZ [REG_XSP]
+        PUSHF
+        pop      REG_XAX
+        /* having DF set messes up printing for x64 */
+        push     0
+        POPF
+        CALLC3(test_flag, REG_XAX, REG_XCX, 0)
+
+        pop      REG_XCX /* clean up */
+        ret
+        END_FUNC(FUNCNAME)
+
+    /* void set_flag(uint pos) */
+#undef FUNCNAME
+#define FUNCNAME set_flag
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        /* We don't bother w/ SEH64 directives, though we're an illegal leaf routine! */
+        PUSHF
+        pop      REG_XAX
+        mov      REG_XCX, ARG1
+        mov      REG_XDX, 1
+        shl      REG_XDX, cl
+        or       REG_XAX, REG_XDX
+        push     REG_XAX
+        POPF
+        ret
+        END_FUNC(FUNCNAME)
+
+    /* void clear_flag(uint pos) */
+#undef FUNCNAME
+#define FUNCNAME clear_flag
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        /* We don't bother w/ SEH64 directives, though we're an illegal leaf routine! */
+        PUSHF
+        pop      REG_XAX
+        mov      REG_XCX, ARG1
+        mov      REG_XDX, 1
+        shl      REG_XDX, cl
+        not      REG_XDX
+        and      REG_XAX, REG_XDX
+        push     REG_XAX
+        POPF
+        ret
+        END_FUNC(FUNCNAME)
+
+END_FILE
+#endif
diff --git a/suite/tests/common/eflags.expect b/suite/tests/common/eflags.expect
new file mode 100644
index 000000000..032c8dbad
--- /dev/null
+++ b/suite/tests/common/eflags.expect
@@ -0,0 +1,14 @@
+OK 1 CF
+OK 0 CF
+OK 1 PF
+OK 0 PF
+OK 1 AF
+OK 0 AF
+OK 1 ZF
+OK 0 ZF
+OK 1 SF
+OK 0 SF
+OK 1 DF
+OK 0 DF
+OK 1 OF
+OK 0 OF
diff --git a/suite/tests/common/fib.c b/suite/tests/common/fib.c
new file mode 100644
index 000000000..cf641ceed
--- /dev/null
+++ b/suite/tests/common/fib.c
@@ -0,0 +1,99 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* undefine this for a performance test */
+#ifndef NIGHTLY_REGRESSION
+# define NIGHTLY_REGRESSION
+#endif
+
+#include "tools.h"
+
+#include <stdio.h>
+
+#ifdef WINDOWS
+/* and compile with user32.lib */
+#include <windows.h>
+#endif
+
+#ifdef NIGHTLY_REGRESSION
+#  define ITER 10*1000
+#else
+#  define ITER 10*200*1000 // *100
+#endif
+
+#define GOAL 32 /* recursive fib of course is exponential here */
+
+/* now stay a little more realistic depths that fit in the RSB */
+#define DEPTH 12
+
+/* ignore overflow errors */
+int 
+fib(int n) {
+    if (n <= 1) return 1;
+    return fib(n-1) + fib(n-2);
+}
+
+int
+main(int argc, char** argv) 
+{
+    int i, t;
+    
+    INIT();
+    USE_USER32();
+
+    print("fib(%d)=%d\n", 5, fib(5));
+    print("fib(%d)=%d\n", 15, fib(15));
+    /* deep recursion */
+    print("fib(%d)=%d\n", 25, fib(25));
+
+    /* show recursion */
+    for (i=0; i<=GOAL; i++) {
+	t = fib(i);
+	print("fib(%d)=%d\n", i, t);
+    }
+
+    for (i=0; i<=ITER; i++) {
+	t = fib(DEPTH);
+    }
+
+    print("fib(%d)=%d\n", DEPTH, t);
+}
+
+/* 
+It is amazing that with default options native=13s dr=12s
+
+only when optimized differences show up in the other direction
+$ cl /O2 -I.. fib.c
+native=8s dr=11s   ITER=? DEPTH=?
+ 
+$ cl /O2  /Zi fib.c -I.. /link /incremental:no user32.lib
+*/
diff --git a/suite/tests/common/fib.expect b/suite/tests/common/fib.expect
new file mode 100644
index 000000000..5f930141e
--- /dev/null
+++ b/suite/tests/common/fib.expect
@@ -0,0 +1,37 @@
+fib(5)=8
+fib(15)=987
+fib(25)=121393
+fib(0)=1
+fib(1)=1
+fib(2)=2
+fib(3)=3
+fib(4)=5
+fib(5)=8
+fib(6)=13
+fib(7)=21
+fib(8)=34
+fib(9)=55
+fib(10)=89
+fib(11)=144
+fib(12)=233
+fib(13)=377
+fib(14)=610
+fib(15)=987
+fib(16)=1597
+fib(17)=2584
+fib(18)=4181
+fib(19)=6765
+fib(20)=10946
+fib(21)=17711
+fib(22)=28657
+fib(23)=46368
+fib(24)=75025
+fib(25)=121393
+fib(26)=196418
+fib(27)=317811
+fib(28)=514229
+fib(29)=832040
+fib(30)=1346269
+fib(31)=2178309
+fib(32)=3524578
+fib(12)=233
diff --git a/suite/tests/common/getretaddr.c b/suite/tests/common/getretaddr.c
new file mode 100644
index 000000000..9ac899c5d
--- /dev/null
+++ b/suite/tests/common/getretaddr.c
@@ -0,0 +1,131 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifndef ASM_CODE_ONLY /* C code */
+#include "tools.h"
+#include <stdio.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#define VERBOSE 0
+
+ptr_int_t get_retaddr(void);
+ptr_int_t get_retaddr_from_frameptr(void);
+
+#ifdef X64
+# ifdef WINDOWS  /* 5th param is on the stack */
+void foo(int x1,int x2,int x3,int x4,int x)
+# else  /* 7th param is on the stack */
+void foo(int x1,int x2,int x3,int x4,int x5,int x6,int x)
+# endif
+#else
+void foo(int x)
+#endif
+{
+    ptr_int_t myaddr1, myaddr2;
+#if defined(X64) && defined(WINDOWS)
+    /* no frame pointer available w/ the compiler we're using */
+#else
+    myaddr1 = get_retaddr_from_frameptr();
+# if VERBOSE
+    print("my own return address is "PFX"\n", myaddr1);
+# endif
+#endif
+    /* alternative method: */
+    myaddr2 = *(((ptr_int_t*)&x) - IF_X64_ELSE(IF_WINDOWS_ELSE(5, 1), 1));
+#if defined(X64) && defined(WINDOWS)
+    myaddr1 = myaddr2;
+#endif
+    if (myaddr1 == myaddr2)
+        print("return addresses match\n");
+    else
+        print("ERROR -- return addresses do not match\n");
+#if VERBOSE
+    print("my own return address is "PFX"\n", myaddr2);
+#endif
+}
+
+int main()
+{
+    ptr_int_t myaddr;
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+    /* make sure dynamorio can handle this non-call */
+    myaddr = get_retaddr();
+#if VERBOSE
+    print("my address is something like "PFX"\n", myaddr);
+#endif
+#ifdef X64
+# ifdef WINDOWS  /* 5th param is on the stack */
+    foo(3,3,3,3,3);
+# else  /* 7th param is on the stack */
+    foo(3,3,3,3,3,3,3);
+# endif
+#else
+    foo(3);
+#endif
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
+
+
+#else /* asm code *************************************************************/
+#include "asm_defines.asm"
+START_FILE
+
+#define FUNCNAME get_retaddr
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        /* We don't bother w/ SEH64 directives, though we're an illegal leaf routine! */
+        call     next_instr
+    next_instr:
+        pop      REG_XAX
+        ret
+        END_FUNC(FUNCNAME)
+
+#undef FUNCNAME
+#define FUNCNAME get_retaddr_from_frameptr
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        mov      REG_XAX, PTRSZ [REG_XBP + ARG_SZ]
+        ret
+        END_FUNC(FUNCNAME)
+
+END_FILE
+#endif
diff --git a/suite/tests/common/getretaddr.expect b/suite/tests/common/getretaddr.expect
new file mode 100644
index 000000000..bf416914b
--- /dev/null
+++ b/suite/tests/common/getretaddr.expect
@@ -0,0 +1 @@
+return addresses match
diff --git a/suite/tests/common/hello.c b/suite/tests/common/hello.c
new file mode 100644
index 000000000..87ba2101b
--- /dev/null
+++ b/suite/tests/common/hello.c
@@ -0,0 +1,50 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+main()
+{
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+    printf("Hello world!\n");
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
diff --git a/suite/tests/common/hello.expect b/suite/tests/common/hello.expect
new file mode 100644
index 000000000..cd0875583
--- /dev/null
+++ b/suite/tests/common/hello.expect
@@ -0,0 +1 @@
+Hello world!
diff --git a/suite/tests/common/memoryhog.template b/suite/tests/common/memoryhog.template
new file mode 100644
index 000000000..a76cc9e8f
--- /dev/null
+++ b/suite/tests/common/memoryhog.template
@@ -0,0 +1,7 @@
+starting up
+#OUT_OF_MEMORY
+#if +WINDOWS
+// on linux we currently can't kill all threads so
+// the parent thread hangs on a lock held by dead child
+about to exit
+#endif
diff --git a/suite/tests/common/ops.c b/suite/tests/common/ops.c
new file mode 100644
index 000000000..4093201eb
--- /dev/null
+++ b/suite/tests/common/ops.c
@@ -0,0 +1,73 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <math.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#define ITERS 1500000
+
+static int a[ITERS];
+
+int main(int argc, char *argv[])
+{
+    double res = 0.;
+    int i,j;
+
+#ifdef USE_DYNAMO
+    int rc = dynamorio_app_init();
+    assert(rc == 0);
+    dynamorio_app_start();
+#endif
+  
+    for (i=0; i<ITERS; i++) {
+	if (i % 2 == 0) {
+	    res += cos(1./(double)(i+1));
+	} else {
+	    res += sin(1./(double)(i+1));
+	}
+	j = (i << 4) / (i | 0x38);
+	a[i] += j;
+    }
+    printf("%f\n", res);
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
+
diff --git a/suite/tests/common/ops.expect b/suite/tests/common/ops.expect
new file mode 100644
index 000000000..76dd8063a
--- /dev/null
+++ b/suite/tests/common/ops.expect
@@ -0,0 +1 @@
+750006.451810
diff --git a/suite/tests/common/protect-dstack.c b/suite/tests/common/protect-dstack.c
new file mode 100644
index 000000000..8dcd8b73c
--- /dev/null
+++ b/suite/tests/common/protect-dstack.c
@@ -0,0 +1,353 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifndef ASM_CODE_ONLY /* C code */
+/* Tests protection of DynamoRIO's stack
+ * This test when run natively will fail with an error message
+ */
+
+#include "tools.h"
+
+#ifdef LINUX
+# include <unistd.h>
+# include <signal.h>
+# include <ucontext.h>
+# include <errno.h>
+#endif
+
+#include <setjmp.h>
+
+#ifdef USE_DYNAMO
+# include "dynamorio.h"
+#endif
+
+#define VERBOSE 0
+
+#define EXPANDSTR(x) #x
+#define STRINGIFY(x) EXPANDSTR(x)
+#define ALIGNED(x, alignment) ((((ptr_uint_t)x) & ((alignment)-1)) == 0)
+
+/* bottom page is a guard page, so ignore it -- consider only top 8KB */
+#define DSTACK_SIZE (8*1024)
+
+/* N.B.: dependent on exact DR offsets here! */
+#ifdef LINUX
+/* this used to be 44 prior to 1/24/06 commit, and 20 prior to Mar 11 2006 */
+# define DCONTEXT_TLS_OFFSET IF_X64_ELSE(32, 16)
+/* PR 264138 added 0x60 to dr_mcontext_t Apr 17 2008 */
+/* this used to be 0x3c prior to Jun 3 2005, then 0x40 prior to Mar 11 2006 */
+# define DSTACK_OFFSET_IN_DCONTEXT IF_X64_ELSE(0x1b0,0xbc)
+# ifdef X64
+# define GET_DCONTEXT(var)                                  \
+    asm("push %rax");                                       \
+    asm("mov  %fs:"STRINGIFY(DCONTEXT_TLS_OFFSET)", %rax"); \
+    asm("mov  %%rax, %0" : "=m"((var)));                    \
+    asm("pop  %rax");
+# else
+# define GET_DCONTEXT(var)                                  \
+    asm("push %eax");                                       \
+    asm("movl %fs:"STRINGIFY(DCONTEXT_TLS_OFFSET)", %eax"); \
+    asm("movl %%eax, %0" : "=m"((var)));                    \
+    asm("popl %eax");
+# endif
+#else
+unsigned int dcontext_tls_offset;
+
+/* PR 264138 added 0x60 to dr_mcontext_t Apr 17 2008 */
+/* this used to be 0x38 prior to Jun 3 2005, then 0x3c prior to Mar 11 2006 */
+# define DSTACK_OFFSET_IN_DCONTEXT IF_X64_ELSE(0x110,0xb8)
+# define GET_DCONTEXT(var)                   \
+    var = (void *) IF_X64_ELSE(__readgsqword,__readfsdword)(dcontext_tls_offset);
+
+/*     0:001> dt getdc owning_thread    
+ *        +0x05c owning_thread
+ * PR 264138 added 0x60 to dr_mcontext_t Apr 17 2008
+ * this is now 0x58 as of Mar 11 2006 (case 4837 commit)
+ * this is now 0x5c as of Aug 15 2006 (case 5758 commit)
+ */
+#define OWNING_THREAD_OFFSET_IN_DCONTEXT IF_X64_ELSE(0x150,0x0dc)
+/* offset varies based on release/debug build (# of slots we need)
+ * and cache line size (must be aligned) and the -ibl_table_in_tls
+ * option being set to true
+ */
+
+#endif
+
+jmp_buf mark;
+int where; /* 0 = normal, 1 = segfault longjmp, 2 = evil takeover */
+
+#ifdef LINUX
+/* just use single-arg handlers */
+typedef void (*handler_t)(int);
+typedef void (*handler_3_t)(int, struct siginfo *, void *);
+
+static void
+signal_handler(int sig)
+{
+    if (sig == SIGSEGV) {
+#if VERBOSE
+        print("Got seg fault\n");
+#endif
+        longjmp(mark, 1);
+    }
+    exit(-1);
+}
+
+#define ASSERT_NOERR(rc) do {                                   \
+  if (rc) {                                                     \
+     fprintf(stderr, "%s:%d rc=%d errno=%d %s\n",               \
+             __FILE__, __LINE__,                                \
+             rc, errno, strerror(errno));                       \
+  }                                                             \
+} while (0);
+
+/* set up signal_handler as the handler for signal "sig" */
+static void
+intercept_signal(int sig, handler_t handler)
+{
+    int rc;
+    struct sigaction act;
+
+    act.sa_sigaction = (handler_3_t) handler;
+    /* FIXME: due to DR bug 840 we cannot block ourself in the handler
+     * since the handler does not end in a sigreturn, so we have an empty mask
+     * and we use SA_NOMASK
+     */
+    rc = sigemptyset(&act.sa_mask); /* block no signals within handler */
+    ASSERT_NOERR(rc);
+    /* FIXME: due to DR bug #654 we use SA_SIGINFO -- change it once DR works */
+    act.sa_flags = SA_NOMASK | SA_SIGINFO | SA_ONSTACK;
+    
+    /* arm the signal */
+    rc = sigaction(sig, &act, NULL);
+    ASSERT_NOERR(rc);
+}
+
+#else
+/* sort of a hack to avoid the MessageBox of the unhandled exception spoiling
+ * our batch runs
+ */
+# include <windows.h>
+/* top-level exception handler */
+static LONG
+our_top_handler(struct _EXCEPTION_POINTERS * pExceptionInfo)
+{
+    if (pExceptionInfo->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
+# if VERBOSE
+        /* DR gets the target addr wrong for far call/jmp so we don't print it */
+        print("\tPC "PFX" tried to %s address "PFX"\n",
+            pExceptionInfo->ExceptionRecord->ExceptionAddress, 
+            (pExceptionInfo->ExceptionRecord->ExceptionInformation[0]==0)?"read":"write",
+            pExceptionInfo->ExceptionRecord->ExceptionInformation[1]);
+# endif
+        longjmp(mark, 1);
+    }
+# if VERBOSE
+    print("Exception occurred, process about to die silently\n");
+# endif
+    return EXCEPTION_EXECUTE_HANDLER; /* => global unwind and silent death */
+}
+#endif
+
+/* global so that evil can use it */
+int *dstack_base;
+
+/* the goal is to get DR to jmp to here by clobbering the fcache_return ret addr
+ * on the dstack
+ */
+static void
+evil()
+{
+    /* popf of DR's eflags (in old DR design) w/ our clobbered value could
+     * set some funny flags -- clear them all here
+     */
+    clear_eflags();
+    /* don't trusk stack -- hopefully enough there to call longjmp 
+     * certainly can't return from this function since not called
+     */
+    longjmp(mark, 2);
+}
+
+int
+main()
+{
+    int *pc;
+    int release_build = 0; /* 1 == release, 0 == debug */
+    void *dcontext;
+    int *dstack;
+    int tls_offs;
+    ptr_int_t owning_thread;
+    INIT();
+    
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+#ifdef LINUX
+    intercept_signal(SIGSEGV, signal_handler);
+#else
+    SetUnhandledExceptionFilter((LPTOP_LEVEL_EXCEPTION_FILTER) our_top_handler);
+#endif
+
+#ifdef WINDOWS
+    /* brute force loop over all TLS entries,
+     * and see whether owning_thread is GetCurrentThreadId()
+     *     0:001> dt getdc owning_thread    
+     *        +0x05c owning_thread : 0xed8
+     *
+     *      0:001> dt _TEB TLS64
+     *        +0xe10 TLS64 : [64] Ptr32 Void
+     */
+    for (tls_offs = 63; tls_offs >=0; tls_offs--) {
+        enum {offsetof_TLS64_in_TEB = IF_X64_ELSE(0x1480, 0xe10)};
+        dcontext_tls_offset = offsetof_TLS64_in_TEB + 
+            tls_offs*sizeof(void*);
+        GET_DCONTEXT(dcontext);
+#if VERBOSE
+        print("%d idx, %x offs\n", tls_offs, dcontext_tls_offset);
+#endif
+        where = setjmp(mark);
+        if (where == 0) {
+            owning_thread = *(ptr_int_t *)(((char *)dcontext) + 
+                                           OWNING_THREAD_OFFSET_IN_DCONTEXT);
+            /* we didn't crash reading, is it really thread ID? */
+#if VERBOSE
+            print("     %d thread %d vs %d\n", tls_offs, owning_thread, GetCurrentThreadId());
+#endif
+            if (owning_thread == GetCurrentThreadId()) {
+#if VERBOSE
+                print("     %d is dcontext!\n", tls_offs);
+#endif
+                break;
+            }
+        } else {
+#if VERBOSE
+            print("     %d crashed\n", tls_offs);
+#endif
+            /* we crashed reading, try next offset */
+        }
+    }
+    if (tls_offs < 0) {
+        print("error obtaining dcontext: are you running natively?!?\n");
+        exit(1);
+    }
+#endif
+    where = setjmp(mark);
+    if (where != 0) {
+        print("error obtaining dcontext: are you running natively?!?\n");
+        exit(1);
+    }
+    GET_DCONTEXT(dcontext)
+#if VERBOSE
+    print("dcontext is "PFX"\n", dcontext);
+#endif
+    dstack = *(int **)(((char *)dcontext) + DSTACK_OFFSET_IN_DCONTEXT);
+    if (!ALIGNED(dstack, PAGE_SIZE)) {
+        print("can't find dstack: old build, or new where dstack offset changed?\n");
+        exit(-1);
+    }
+    dstack_base = (int *) (((char *)dstack) - DSTACK_SIZE);
+#if VERBOSE
+    print("dstack is "PFX"-"PFX"\n", dstack_base, dstack);
+#endif
+    print("dcontext->dstack successfully obtained\n");
+    where = setjmp(mark);
+#if VERBOSE
+    print("setjmp returned %d\n", where);
+#endif
+    if (where == 0) {
+        /* if we do the copy in a C loop, trace heads cause us to exit before
+         * we've hit the cxt switch return address, so we crash rather than taking
+         * control -- so we hand-code a copy that in C looks like this:
+         *          for (pc = dstack_base; pc++; pc < dstack)
+         *              *pc = (int) evil;
+         * we assume df is cleared
+         * FIXME: popf in old fcache_return can trigger a trap crash before
+         * get to ret that goes to evil!
+         * FIXME: I had this getting to evil w/o crashing first, but it's
+         * a little fragile, and on win32 I get issues later b/c we have
+         * trampolines, etc. and so don't completely lose control.
+         * But, in all cases we fail, so whether it's a nice shell code
+         * execution or a crash doesn't matter -- the test does what it's supposed
+         * to do!
+         */
+        evil_copy(dstack_base, DSTACK_SIZE / sizeof(int), evil);
+        print("wrote to entire dstack without incident!\n");
+    } else if (where == 1) {
+        print("error writing to "PFX" in expected dstack "PFX"-"PFX"\n",
+              pc, dstack_base, dstack);
+    } else if (where == 2) {
+        print("DR has been cracked!  Malicious code is now runnning...\n");
+    }
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
+
+
+#else /* asm code *************************************************************/
+#include "asm_defines.asm"
+START_FILE
+
+#define FUNCNAME clear_eflags
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        /* We don't bother w/ SEH64 directives, though we're an illegal leaf routine! */
+        push     0
+        popf
+        ret
+        END_FUNC(FUNCNAME)
+
+/* void evil_copy(void *start, int count, ptr_int_t value); */
+#undef FUNCNAME
+#define FUNCNAME evil_copy
+        DECLARE_FUNC_SEH(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+       /* Must use ARG macros before any pushes; but must preserve xdi */
+        mov      REG_XAX, ARG3
+        mov      REG_XCX, ARG1
+        mov      REG_XDX, ARG2
+        PUSH_SEH(REG_XDI)
+        END_PROLOG
+        xchg     REG_XDX, REG_XCX /* contortions b/c of x64 param regs */
+        mov      REG_XDI, REG_XDX
+        rep stosd
+        add      REG_XSP, 0 /* make a legal SEH64 epilog */
+        pop      REG_XDI
+        ret
+        END_FUNC(FUNCNAME)
+
+END_FILE
+#endif
diff --git a/suite/tests/common/protect-dstack.expect b/suite/tests/common/protect-dstack.expect
new file mode 100644
index 000000000..bff71448a
--- /dev/null
+++ b/suite/tests/common/protect-dstack.expect
@@ -0,0 +1,2 @@
+dcontext->dstack successfully obtained
+wrote to entire dstack without incident!
diff --git a/suite/tests/common/recurse.c b/suite/tests/common/recurse.c
new file mode 100644
index 000000000..c4a5ce0ed
--- /dev/null
+++ b/suite/tests/common/recurse.c
@@ -0,0 +1,89 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <stdio.h>
+#include <assert.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#define GOAL 256
+
+int bar(int n)
+{
+    printf("bar %d\n", n);
+    if (n==0) return 1;
+    if (n % 2 == 0)
+	return n + foo(n-1);
+    if (n % 2 == 1)
+	return n + bar(n-1);
+    printf("\tdone with bar %d\n", n);
+    return 0;
+}
+
+int foo(int n)
+{
+    printf("foo %d\n", n);
+    if (n==0) return 1;
+    if (n % 2 == 0)
+	return n + foo(n-1);
+    if (n % 2 == 1)
+	return n + bar(n-1);
+    printf("\tdone with foo %d\n", n);
+    return 0;
+}
+
+
+main()
+{
+    int i, t = 0;
+
+#ifdef USE_DYNAMO
+    int rc = dynamorio_app_init();
+    assert(rc == 0);
+    dynamorio_app_start();
+#endif
+
+    for (i=GOAL; i<=GOAL; i++) {
+	t = foo(i);
+	printf("%d %d\n", i, t);
+    }
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+
+    return 0;
+}
+	
diff --git a/suite/tests/common/recurse.expect b/suite/tests/common/recurse.expect
new file mode 100644
index 000000000..8998c0c7a
--- /dev/null
+++ b/suite/tests/common/recurse.expect
@@ -0,0 +1,258 @@
+foo 256
+foo 255
+bar 254
+foo 253
+bar 252
+foo 251
+bar 250
+foo 249
+bar 248
+foo 247
+bar 246
+foo 245
+bar 244
+foo 243
+bar 242
+foo 241
+bar 240
+foo 239
+bar 238
+foo 237
+bar 236
+foo 235
+bar 234
+foo 233
+bar 232
+foo 231
+bar 230
+foo 229
+bar 228
+foo 227
+bar 226
+foo 225
+bar 224
+foo 223
+bar 222
+foo 221
+bar 220
+foo 219
+bar 218
+foo 217
+bar 216
+foo 215
+bar 214
+foo 213
+bar 212
+foo 211
+bar 210
+foo 209
+bar 208
+foo 207
+bar 206
+foo 205
+bar 204
+foo 203
+bar 202
+foo 201
+bar 200
+foo 199
+bar 198
+foo 197
+bar 196
+foo 195
+bar 194
+foo 193
+bar 192
+foo 191
+bar 190
+foo 189
+bar 188
+foo 187
+bar 186
+foo 185
+bar 184
+foo 183
+bar 182
+foo 181
+bar 180
+foo 179
+bar 178
+foo 177
+bar 176
+foo 175
+bar 174
+foo 173
+bar 172
+foo 171
+bar 170
+foo 169
+bar 168
+foo 167
+bar 166
+foo 165
+bar 164
+foo 163
+bar 162
+foo 161
+bar 160
+foo 159
+bar 158
+foo 157
+bar 156
+foo 155
+bar 154
+foo 153
+bar 152
+foo 151
+bar 150
+foo 149
+bar 148
+foo 147
+bar 146
+foo 145
+bar 144
+foo 143
+bar 142
+foo 141
+bar 140
+foo 139
+bar 138
+foo 137
+bar 136
+foo 135
+bar 134
+foo 133
+bar 132
+foo 131
+bar 130
+foo 129
+bar 128
+foo 127
+bar 126
+foo 125
+bar 124
+foo 123
+bar 122
+foo 121
+bar 120
+foo 119
+bar 118
+foo 117
+bar 116
+foo 115
+bar 114
+foo 113
+bar 112
+foo 111
+bar 110
+foo 109
+bar 108
+foo 107
+bar 106
+foo 105
+bar 104
+foo 103
+bar 102
+foo 101
+bar 100
+foo 99
+bar 98
+foo 97
+bar 96
+foo 95
+bar 94
+foo 93
+bar 92
+foo 91
+bar 90
+foo 89
+bar 88
+foo 87
+bar 86
+foo 85
+bar 84
+foo 83
+bar 82
+foo 81
+bar 80
+foo 79
+bar 78
+foo 77
+bar 76
+foo 75
+bar 74
+foo 73
+bar 72
+foo 71
+bar 70
+foo 69
+bar 68
+foo 67
+bar 66
+foo 65
+bar 64
+foo 63
+bar 62
+foo 61
+bar 60
+foo 59
+bar 58
+foo 57
+bar 56
+foo 55
+bar 54
+foo 53
+bar 52
+foo 51
+bar 50
+foo 49
+bar 48
+foo 47
+bar 46
+foo 45
+bar 44
+foo 43
+bar 42
+foo 41
+bar 40
+foo 39
+bar 38
+foo 37
+bar 36
+foo 35
+bar 34
+foo 33
+bar 32
+foo 31
+bar 30
+foo 29
+bar 28
+foo 27
+bar 26
+foo 25
+bar 24
+foo 23
+bar 22
+foo 21
+bar 20
+foo 19
+bar 18
+foo 17
+bar 16
+foo 15
+bar 14
+foo 13
+bar 12
+foo 11
+bar 10
+foo 9
+bar 8
+foo 7
+bar 6
+foo 5
+bar 4
+foo 3
+bar 2
+foo 1
+bar 0
+256 32897
diff --git a/suite/tests/common/segfault.c b/suite/tests/common/segfault.c
new file mode 100644
index 000000000..eafc86a6f
--- /dev/null
+++ b/suite/tests/common/segfault.c
@@ -0,0 +1,137 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <math.h>
+
+#ifdef LINUX
+# include <unistd.h>
+# include <signal.h>
+# include <ucontext.h>
+# include <errno.h>
+#endif
+
+#include "tools.h" /* for print() */
+
+/* just use single-arg handlers */
+typedef void (*handler_t)(int);
+typedef void (*handler_3_t)(int, struct siginfo *, void *);
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#define ITERS 1500000
+
+static int a[ITERS];
+
+#ifdef LINUX
+static void
+signal_handler(int sig)
+{
+    if (sig == SIGSEGV)
+        print("Got a seg fault\n");
+    abort();
+}
+
+#define ASSERT_NOERR(rc) do {					\
+  if (rc) {							\
+     fprintf(stderr, "%s:%d rc=%d errno=%d %s\n", 		\
+	     __FILE__, __LINE__,				\
+	     rc, errno, strerror(errno));  		        \
+  }								\
+} while (0);
+
+/* set up signal_handler as the handler for signal "sig" */
+static void
+intercept_signal(int sig, handler_t handler)
+{
+    int rc;
+    struct sigaction act;
+
+    act.sa_sigaction = (handler_3_t) handler;
+    rc = sigfillset(&act.sa_mask); /* block all signals within handler */
+    ASSERT_NOERR(rc);
+    act.sa_flags = SA_ONSTACK;
+
+    /* arm the signal */
+    rc = sigaction(sig, &act, NULL);
+    ASSERT_NOERR(rc);
+}
+
+#else
+/* sort of a hack to avoid the MessageBox of the unhandled exception spoiling
+ * our batch runs
+ */
+# include <windows.h>
+/* top-level exception handler */
+static LONG
+our_top_handler(struct _EXCEPTION_POINTERS * pExceptionInfo)
+{
+    if (pExceptionInfo->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION)
+        print("Got a seg fault\n");
+# if VERBOSE
+    print("Exception occurred, process about to die silently\n");
+# endif
+    return EXCEPTION_EXECUTE_HANDLER; /* => global unwind and silent death */
+}
+#endif
+
+int main(int argc, char *argv[])
+{
+    double res = 0.;
+    int i,j;
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+  
+#ifdef LINUX
+    intercept_signal(SIGSEGV, signal_handler);
+#else
+    SetUnhandledExceptionFilter((LPTOP_LEVEL_EXCEPTION_FILTER) our_top_handler);
+#endif
+
+    print("Segfault about to happen\n");
+
+    *((int *)0) = 4;
+
+    print("SHOULD NEVER GET HERE\n");
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
diff --git a/suite/tests/common/segfault.expect b/suite/tests/common/segfault.expect
new file mode 100644
index 000000000..e29852337
--- /dev/null
+++ b/suite/tests/common/segfault.expect
@@ -0,0 +1,2 @@
+Segfault about to happen
+Got a seg fault
diff --git a/suite/tests/gen_expect.pl b/suite/tests/gen_expect.pl
new file mode 100755
index 000000000..d0c9a2457
--- /dev/null
+++ b/suite/tests/gen_expect.pl
@@ -0,0 +1,247 @@
+#!/usr/local/bin/perl
+
+# **********************************************************
+# Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### gen_expect.pl 
+### this file reads in an expect file template for a unit test and 
+### generates and expect file based on the Compile defines and options
+### passed in 
+
+### template file format
+### name = <test_name>.template
+### #if : #else #endif #stop #sec_vio_cont #sec_vio_stop
+
+$sec_violation_con = "";
+$sec_violation_stop = "";
+
+$usage = "Usage: gen_expect <build_defines> <run_options> <template_file>\n";
+
+$verbose = 0;
+    
+print stderr "$#ARGV\n" if ($verbose);
+print stderr "$ARGV[0]\n" if ($verbose);
+print stderr "$ARGV[1]\n" if ($verbose);
+print stderr "$ARGV[2]\n" if ($verbose);
+
+if ($#ARGV != 2) {
+    print "$usage";
+    exit 0;
+}
+
+$build_defs = $ARGV[0];
+$run_ops = $ARGV[1];
+$file = $ARGV[2];
+
+open(IN, "<$file") || print stderr "Error : couldn't open template file $file \n";
+$outfile = $file;
+$outfile =~ s/\.template$/\.expect/;
+open(OUT, ">$outfile") || print stderr "Error : couldn't open expect file $file for generation \n";
+
+$yes = "y";
+$no = "n";
+$ignore = "i";
+
+$copy_mode = $yes;
+$nesting_depth = 0;
+
+$win32 = 0;
+if ($build_defs =~ /WINDOWS/) {
+    $win32 = 1;
+
+    # Figure out the windows version and add the corresponding
+    # RUNREGRESSION_ define to build_defs.  We'll use 'cmd /c' to get
+    # the cmd shell version, which should match the OS version.
+    # Launching a shell command from perl is slightly problematic if
+    # we want this script to work with both the windows cmd shell and
+    # cygwin.  Cygwin wants /'s in the path and cmd shell wants \'s.
+    # Let's assume cmd is always in the path and omit the full path.
+    $verout = `cmd /c ver`;
+    if ($verout =~ /Windows NT Version 4\.0/) {
+        $winver = "NT";
+    } elsif ($verout =~ /Microsoft Windows 2000/) {
+        $winver = "2000";
+    } elsif ($verout =~ /Microsoft Windows XP/) {
+        $winver = "XP";
+    } elsif ($verout =~ /Microsoft Windows \[Version 5\.2\..+\]/) {
+        $winver = "2003";
+    } elsif ($verout =~ /Microsoft Windows \[Version 6\.0\..+\]/) {
+        $winver = "Vista";
+    } else {
+        die "Unknown Windows version\n";
+    }
+
+    $build_defs .= "RUNREGRESSION_$winver ";
+}
+
+$throw_exception = 0;
+if ($run_ops =~ /-throw_exception/) {
+    $throw_exception = 1;
+}
+$kill_thread = 0;
+if ($run_ops =~ /-kill_thread/) {
+    $kill_thread = 1;
+}
+
+while (<IN>) {
+    chop;
+    # handle dos end of line
+    if ($_ =~ /\r$/) { chop; };
+    $line = $_;
+
+    if ($line =~ /^#\s*end/) {
+        $nesting_depth -= 1;
+        chop $copy_mode;
+        print stderr "new nesting depth " . $nesting_depth . "\n" if ($verbose);
+        print stderr "copy mode " . $copy_mode . "\n" if ($verbose);
+    } elsif ($line =~ /^#\s*else/) {
+        $copy_mode =~ /(.*)(.)$/;
+        if ($2 eq $no) {
+            $copy_mode = $1;
+            $copy_mode .= $yes;
+        } elsif ($2 eq $yes) {
+            $copy_mode = $1;
+            $copy_mode .= $no;
+        }
+        print stderr "hit else copy mode " . $copy_mode . "\n" if ($verbose);
+    } elsif ($line =~ /^#\s*if(.*)/) {
+        @expr = split(/ /, $1);
+        $check_in = $build_defs;
+        $nesting_depth += 1;
+        print stderr "new nesting depth " . $nesting_depth . "\n" if ($verbose);
+        if ($copy_mode =~ /.*$yes$/) {
+            $success = $yes;
+            $index = 1;
+            while ($index < @expr) {
+                print "token is $expr[$index]\n" if ($verbose);
+                if ($check_in ne $run_ops && $expr[$index] =~ /\:/) {
+                    $check_in = $run_ops;
+                } elsif ($expr[$index] =~ /\|\|/) {
+                    print stderr "found an or, success now is $success\n" if ($verbose);
+                    # primitive "or" feature: no composability, only single top-level or
+                    if ($success eq $yes) {
+                        # short-circuit: ignore other side of the or
+                        last;
+                    } else {
+                        # start over
+                        $success = $yes;
+                        $check_in = $build_defs;
+                    }
+                } elsif ($expr[$index] =~ /\+(.*)/) {
+                    $tok = $1;
+                    # allow for parentheses to avoid spaces around option arguments
+                    # replace ( with a space, and remove final )
+                    if ($tok =~ /(.*)\((.*)\)/) {
+                        $tok = "$1 $2";
+                    }
+                    if (not($check_in =~ /$tok/)) {
+                        print "BAD: required token $tok missing\n" if ($verbose);
+                        $success = $no;
+                    }
+                } elsif ($expr[$index] =~ /\-(.*)/) {
+                    $tok = $1;
+                    # allow for parentheses to avoid spaces around option arguments
+                    # replace ( with a space, and remove final )
+                    if ($tok =~ /(.*)\((.*)\)/) {
+                        $tok = "$1 $2";
+                    }
+                    if ($check_in =~ /$tok/) {
+                        print "BAD: unwanted token $tok present\n" if ($verbose);
+                        $success = $no;
+                    }
+                } elsif ($expr[$index] !~ /^\s*$/) {
+                    # don't leave partial file
+                    # FIXME: any cleaner way?  make doesn't auto-remove since
+                    # we didn't succeed, and make aborts before any other make
+                    # cmds can come into play.
+                    close(OUT);
+                    unlink $outfile;
+                    die "Bad format of template file: \"$expr[$index]\" on line \"$line\"\n";
+                }
+                $index += 1;
+            }
+            print "success is now $success\n" if ($verbose);
+            if ($success eq $yes) {
+                $copy_mode .= $yes;
+            } else {
+                $copy_mode .= $no;
+            }
+        } else {
+            $copy_mode .= $ignore;
+        }
+        print stderr "copy mode " . $copy_mode . "\n" if ($verbose);
+    } else {
+        if ($copy_mode =~ /.*$yes$/) {
+            $VIOLATION = "<Execution security violation was intercepted!\nContact your vendor for a security vulnerability fix.\n";
+            $CONTINUE_MSG = "Program continuing!";
+            $TERMINATE_MSG = "Program terminated.";
+            $THREAD_MSG = "Program continuing after terminating thread.";
+            $THROW_MSG = "Program continuing after throwing an exception.";
+            $MEMORY_MSG = "Out of memory.  Program aborted.";
+            if ($line =~ /^#\s*SEC_VIO_CONT/) {
+                print OUT $VIOLATION,$CONTINUE_MSG,">\n";
+            } elsif ($line =~ /^#\s*SEC_VIO_STOP/) {
+                print OUT $VIOLATION,$TERMINATE_MSG,">\n";
+            } elsif ($line =~ /^#\s*SEC_VIO_EXCEPTION/) {
+                print OUT $VIOLATION,$THROW_MSG,">\n";
+            } elsif ($line =~ /^#\s*SEC_VIO_THREAD/) {
+                print OUT $VIOLATION,$THREAD_MSG,">\n";
+            } elsif ($line =~ /^#\s*OUT_OF_MEMORY/) {
+                print OUT "<$MEMORY_MSG>\n";
+            } elsif ($line =~ /^#\s*UNHANDLED_EXCEPTION/) {
+                if ($win32) {
+                   print OUT $VIOLATION,"Unhandled exception caught.\n";
+                }
+            } elsif ($line =~ /^#\s*SEC_VIO_AUTO_STOP/) {
+                if ($throw_exception) {
+                    print OUT $VIOLATION,$THROW_MSG,">\n";
+                    if ($win32) {
+                        print OUT "Unhandled exception caught.\n";
+                    }
+                } elsif ($kill_thread) {
+                    print OUT $VIOLATION,$THREAD_MSG,">\n";
+                } else {
+                    print OUT $VIOLATION,$TERMINATE_MSG,">\n";
+                }
+            } elsif ($line =~ /^#\s*STOP/) {
+                close(IN);
+                close(OUT);
+                exit();
+            } elsif ($line =~ /^\/\//) {
+                # comment!
+            } else {
+                print OUT "$line\n";
+            }
+        }
+    }
+}
+
+close(IN);
+close(OUT); 
diff --git a/suite/tests/linux/clone.c b/suite/tests/linux/clone.c
new file mode 100644
index 000000000..893eaf719
--- /dev/null
+++ b/suite/tests/linux/clone.c
@@ -0,0 +1,209 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * test of clone call
+ */
+
+#include <sys/types.h> /* for wait and mmap */
+#include <sys/wait.h>  /* for wait */
+#include <sched.h>     /* for __clone and CLONE_ flags */
+#include <time.h>      /* for nanosleep */
+#include <sys/mman.h>  /* for mmap */
+#include <assert.h>
+#include <signal.h>
+#include <stdio.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#define false (0)
+#define true (1)
+typedef int bool;
+#define THREAD_STACK_SIZE   (32*1024)
+
+/* forward declarations */
+static pid_t create_thread(int (*fcn)(void *), void *arg, void **stack);
+static void delete_thread(pid_t pid, void *stack);
+int run(void *arg);
+static void* stack_alloc(int size);
+static void stack_free(void *p, int size);
+
+/* vars for child thread */
+static pid_t child;
+static void *stack;
+
+/* these are used solely to provide deterministic output */
+/* this is read by child, written by parent, tells child whether to exit */
+static volatile bool child_exit;
+/* this is read by parent, written by child, tells parent whether child done */
+static volatile bool child_done;
+
+static struct timespec sleeptime;
+
+int main()
+{
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    sleeptime.tv_sec = 0;
+    sleeptime.tv_nsec = 10*1000*1000; /* 10ms */
+
+    child_exit = false;
+    child_done = false;
+    child = create_thread(run, NULL, &stack);
+    assert(child > -1);
+
+    /* waste some time */
+    nanosleep(&sleeptime, NULL);
+    
+    child_exit = true;
+    /* we want deterministic printf ordering */
+    while (!child_done)
+        nanosleep(&sleeptime, NULL);
+    delete_thread(child, stack);
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
+
+/* Procedure executed by sideline threads
+ */
+int run(void *arg)
+{
+    int i = 0;
+    printf("Sideline thread started\n");
+    while (true) {
+	/* do nothing for now */
+	i++;
+	if (i % 2500000 == 0)
+	    printf("i = %d\n", i);
+	if (i % 25000000 == 0)
+	    break;
+    }
+    while (!child_exit)
+        nanosleep(&sleeptime, NULL);
+    printf("Sideline thread finished\n");
+    child_done = true;
+    return 0;
+}
+
+/* Create a new thread. It should be passed "fcn", a function which
+ * takes two arguments, (the second one is a dummy, always 4). The
+ * first argument is passed in "arg". Returns the PID of the new
+ * thread */
+static pid_t
+create_thread(int (*fcn)(void *), void *arg, void **stack)
+{
+    pid_t newpid; 
+    int flags;
+    void *my_stack;
+    
+    my_stack = stack_alloc(THREAD_STACK_SIZE);
+    /* need SIGCHLD so parent will get that signal when child dies,
+     * else have errors doing a wait */
+    flags = SIGCHLD | CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND;
+    newpid = __clone(fcn, my_stack, flags, arg);
+  
+    if (newpid == -1) {
+	printf("smp.c: Error calling __clone\n");
+	stack_free(my_stack, THREAD_STACK_SIZE);
+	return -1;
+    }
+
+    *stack = my_stack;
+    return newpid;
+}
+
+static void 
+delete_thread(pid_t pid, void *stack)
+{
+    pid_t result;
+    /* do not print out pids to make diff easy */
+    printf("Waiting for child to exit\n");
+    result = waitpid(pid, NULL, 0);
+    printf("Child has exited\n");
+    if (result == -1 || result != pid)
+	perror("delete_thread waitpid");
+    stack_free(stack, THREAD_STACK_SIZE);
+}
+
+/* allocate stack storage on the app's heap */
+void*
+stack_alloc(int size)
+{
+    size_t sp;
+    void *q = NULL;
+    void *p;
+
+#if STACK_OVERFLOW_PROTECT
+    /* allocate an extra page and mark it non-accessible to trap stack overflow */
+    q = mmap(0, PAGE_SIZE, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0);
+    assert(q);
+    stack_redzone_start = (size_t) q;
+#endif 
+
+    p = mmap(q, size, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0);
+    assert(p);
+#ifdef DEBUG
+    memset(p, 0xab, size);
+#endif 
+
+    /* stack grows from high to low addresses, so return a ptr to the top of the
+       allocated region */
+    sp = (size_t)p + size;
+
+    return (void*) sp;
+}
+
+/* free memory-mapped stack storage */
+void
+stack_free(void *p, int size)
+{
+    size_t sp = (size_t)p - size;
+
+#ifdef DEBUG
+    memset((void *)sp, 0xcd, size);
+#endif 
+    munmap((void *)sp, size);
+
+#if STACK_OVERFLOW_PROTECT
+    sp = sp - PAGE_SIZE;
+    munmap((void*) sp, PAGE_SIZE);
+#endif 
+}
+
diff --git a/suite/tests/linux/clone.expect b/suite/tests/linux/clone.expect
new file mode 100644
index 000000000..242de3e92
--- /dev/null
+++ b/suite/tests/linux/clone.expect
@@ -0,0 +1,14 @@
+Sideline thread started
+i = 2500000
+i = 5000000
+i = 7500000
+i = 10000000
+i = 12500000
+i = 15000000
+i = 17500000
+i = 20000000
+i = 22500000
+i = 25000000
+Sideline thread finished
+Waiting for child to exit
+Child has exited
diff --git a/suite/tests/linux/common.h b/suite/tests/linux/common.h
new file mode 100644
index 000000000..08d963e19
--- /dev/null
+++ b/suite/tests/linux/common.h
@@ -0,0 +1,78 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <string.h>
+
+/* these are defined in /usr/src/linux/fs/proc/array.c */
+#define MAPS_LINE_LENGTH	4096
+/* for systems with sizeof(void*) == 4: */
+#define MAPS_LINE_FORMAT4	  "%08lx-%08lx %s %*x %*s %*u %4096s"
+#define MAPS_LINE_MAX4	49 /* sum of 8  1  8  1 4 1 8 1 5 1 10 1 */
+/* for systems with sizeof(void*) == 8: */
+#define MAPS_LINE_FORMAT8	  "%016lx-%016lx %s %*x %*s %*u %4096s"
+#define MAPS_LINE_MAX8	73 /* sum of 16  1  16  1 4 1 16 1 5 1 10 1 */
+
+#define MAPS_LINE_MAX	MAPS_LINE_MAX8
+
+int
+find_dynamo_library()
+{
+    FILE *maps;
+    char 	line[MAPS_LINE_LENGTH];
+    int 	count = 0;
+
+    maps=fopen("/proc/self/maps","r");
+
+    while(!feof(maps)){
+	void * vm_start, * vm_end;
+	char perm[16];
+	char comment_buffer[MAPS_LINE_LENGTH];
+	int len;
+    
+	if (NULL==fgets(line, sizeof(line), maps))
+	    break;
+	len = sscanf(line,
+		     sizeof(void*) == 4 ? MAPS_LINE_FORMAT4 : MAPS_LINE_FORMAT8,
+		     (unsigned long*)&vm_start, (unsigned long*)&vm_end, perm,
+		     comment_buffer);
+	if (len<4)
+	    comment_buffer[0]='\0';
+	if (strstr(comment_buffer, "dynamorio") != 0) {
+	    fclose(maps);
+	    return 1;
+	}
+    }
+  
+    fclose(maps);
+    return 0;
+}
+
diff --git a/suite/tests/linux/execve-null.c b/suite/tests/linux/execve-null.c
new file mode 100644
index 000000000..29930c5c6
--- /dev/null
+++ b/suite/tests/linux/execve-null.c
@@ -0,0 +1,159 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * test of execve
+ */
+
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/types.h> /* for wait and mmap */
+#include <sys/wait.h>  /* for wait */
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#define NULL_ENV 1
+
+/***************************************************************************/
+/* a hopefuly portable /proc/@self/maps reader */
+
+/* these are defined in /usr/src/linux/fs/proc/array.c */
+#define MAPS_LINE_LENGTH	4096
+/* for systems with sizeof(void*) == 4: */
+#define MAPS_LINE_FORMAT4	  "%08lx-%08lx %s %*x %*s %*u %4096s"
+#define MAPS_LINE_MAX4	49 /* sum of 8  1  8  1 4 1 8 1 5 1 10 1 */
+/* for systems with sizeof(void*) == 8: */
+#define MAPS_LINE_FORMAT8	  "%016lx-%016lx %s %*x %*s %*u %4096s"
+#define MAPS_LINE_MAX8	73 /* sum of 16  1  16  1 4 1 16 1 5 1 10 1 */
+
+#define MAPS_LINE_MAX	MAPS_LINE_MAX8
+
+int
+find_dynamo_library()
+{
+    pid_t pid = getpid();
+    char 	proc_pid_maps[64];	/* file name */
+
+    FILE *maps;
+    char 	line[MAPS_LINE_LENGTH];
+    int 	count = 0;
+
+    // open file's /proc/id/maps virtual map description
+    int n = snprintf(proc_pid_maps, sizeof(proc_pid_maps),
+		     "/proc/%d/maps", pid);
+    if (n<0 || n==sizeof(proc_pid_maps))
+	assert(0); /* paranoia */
+  
+    maps=fopen(proc_pid_maps,"r");
+
+    while(!feof(maps)){
+	void * vm_start, * vm_end;
+	char perm[16];
+	char comment_buffer[MAPS_LINE_LENGTH];
+	int len;
+    
+	if (NULL==fgets(line, sizeof(line), maps))
+	    break;
+	len = sscanf(line,
+		     sizeof(void*) == 4 ? MAPS_LINE_FORMAT4 : MAPS_LINE_FORMAT8,
+		     (unsigned long*)&vm_start, (unsigned long*)&vm_end, perm,
+		     comment_buffer);
+	if (len<4)
+	    comment_buffer[0]='\0';
+	if (strstr(comment_buffer, "dynamorio") != 0) {
+	    fclose(maps);
+	    return 1;
+	}
+    }
+  
+    fclose(maps);
+    return 0;
+}
+
+/***************************************************************************/
+
+int main(int argc, char *argv[])
+{
+    pid_t child;
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    if (find_dynamo_library())
+	printf("parent is running under DynamoRIO\n");
+    else
+	printf("parent is running natively\n");
+    child = fork();
+    if (child < 0) {
+	perror("ERROR on fork");
+    } else if (child > 0) {
+	pid_t result;
+	printf("parent waiting for child\n");
+	result = waitpid(child, NULL, 0);
+	assert(result == child);
+	printf("child has exited\n");
+    } else {
+	int result;
+	char *arg[3];
+#if NULL_ENV
+	char **env = NULL;
+#else
+	char *env0 = "LD_LIBRARY_PATH=/bin:.";
+	char *env[2];
+	env[0] = env0;
+	env[1] = NULL;
+#endif
+	arg[0] = "/bin/basename";
+	arg[1] = "/fake/path/it_worked";
+	arg[2] = NULL;
+	if (find_dynamo_library())
+	    printf("child is running under DynamoRIO\n");
+	else
+	    printf("child is running natively\n");
+	result = execve("/bin/bogus_will_fail", arg, env);
+	result = execve("/bin/basename", arg, env);
+	if (result < 0)
+	    perror("ERROR in execve");
+    }	
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
diff --git a/suite/tests/linux/execve-null.expect b/suite/tests/linux/execve-null.expect
new file mode 100644
index 000000000..f69788861
--- /dev/null
+++ b/suite/tests/linux/execve-null.expect
@@ -0,0 +1,4 @@
+it_worked
+parent is running under DynamoRIO
+parent waiting for child
+child has exited
diff --git a/suite/tests/linux/execve-rec.c b/suite/tests/linux/execve-rec.c
new file mode 100644
index 000000000..da94fda80
--- /dev/null
+++ b/suite/tests/linux/execve-rec.c
@@ -0,0 +1,100 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * test of execve recursively
+ */
+
+#include <sys/types.h>
+#include <unistd.h>
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "common.h"
+
+#define NULL_ENV 0
+
+const int N = 10;
+
+int main(int argc, char **argv)
+{
+    int n, sum = 0;
+    int result;
+    char *arg[5];
+    char **env = NULL;
+    char carg[10], csum[10];
+
+    if (find_dynamo_library())
+	printf("under DynamoRIO\n");
+    else
+	printf("natively\n");
+
+    //printf("%d %s %s %s\n", argc, argv[0], argv[1], argv[2]);
+    if (1 == argc) {  // no args
+	n = N;
+	sum = 0;
+	printf("Sum(%d)\n", n);
+    } else if (2 == argc) {  // execve-rec 10
+	n = atoi(argv[1]);
+	sum = 0;
+	printf("Sum(%d)\n", n);
+    } else {
+	assert(argc == 3);  // execve-rec 9 10
+	n = atoi(argv[1]);
+	sum = atoi(argv[2]);
+    }
+    if (0 == n) {
+	printf("  = %d\n", sum);
+	exit(0);
+    }
+
+    sum+=n;
+    n--;
+    snprintf(carg, 10, "%d", n);
+    snprintf(csum, 10, "%d", sum);
+    arg[0] = argv[0];
+    arg[1] = carg;
+    arg[2] = csum;
+    arg[3] = NULL;
+
+#if  0
+    printf("%d,%d\n", n, sum);
+    printf("execing %d %s %s=%s %s\n", 
+	   3, arg[0], carg, arg[1], arg[2]);
+#endif
+
+    fflush(stdout);
+    result = execve(arg[0], arg, env);
+    if (result < 0)
+	perror("ERROR in execve");
+}
diff --git a/suite/tests/linux/execve-rec.expect b/suite/tests/linux/execve-rec.expect
new file mode 100644
index 000000000..160fa026b
--- /dev/null
+++ b/suite/tests/linux/execve-rec.expect
@@ -0,0 +1,13 @@
+under DynamoRIO
+Sum(10)
+under DynamoRIO
+under DynamoRIO
+under DynamoRIO
+under DynamoRIO
+under DynamoRIO
+under DynamoRIO
+under DynamoRIO
+under DynamoRIO
+under DynamoRIO
+under DynamoRIO
+  = 55
diff --git a/suite/tests/linux/execve.c b/suite/tests/linux/execve.c
new file mode 100644
index 000000000..4a5a85b83
--- /dev/null
+++ b/suite/tests/linux/execve.c
@@ -0,0 +1,159 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * test of execve
+ */
+
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/types.h> /* for wait and mmap */
+#include <sys/wait.h>  /* for wait */
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#define NULL_ENV 0
+
+/***************************************************************************/
+/* a hopefuly portable /proc/@self/maps reader */
+
+/* these are defined in /usr/src/linux/fs/proc/array.c */
+#define MAPS_LINE_LENGTH	4096
+/* for systems with sizeof(void*) == 4: */
+#define MAPS_LINE_FORMAT4	  "%08lx-%08lx %s %*x %*s %*u %4096s"
+#define MAPS_LINE_MAX4	49 /* sum of 8  1  8  1 4 1 8 1 5 1 10 1 */
+/* for systems with sizeof(void*) == 8: */
+#define MAPS_LINE_FORMAT8	  "%016lx-%016lx %s %*x %*s %*u %4096s"
+#define MAPS_LINE_MAX8	73 /* sum of 16  1  16  1 4 1 16 1 5 1 10 1 */
+
+#define MAPS_LINE_MAX	MAPS_LINE_MAX8
+
+int
+find_dynamo_library()
+{
+    pid_t pid = getpid();
+    char 	proc_pid_maps[64];	/* file name */
+
+    FILE *maps;
+    char 	line[MAPS_LINE_LENGTH];
+    int 	count = 0;
+
+    // open file's /proc/id/maps virtual map description
+    int n = snprintf(proc_pid_maps, sizeof(proc_pid_maps),
+		     "/proc/%d/maps", pid);
+    if (n<0 || n==sizeof(proc_pid_maps))
+	assert(0); /* paranoia */
+  
+    maps=fopen(proc_pid_maps,"r");
+
+    while(!feof(maps)){
+	void * vm_start, * vm_end;
+	char perm[16];
+	char comment_buffer[MAPS_LINE_LENGTH];
+	int len;
+    
+	if (NULL==fgets(line, sizeof(line), maps))
+	    break;
+	len = sscanf(line,
+		     sizeof(void*) == 4 ? MAPS_LINE_FORMAT4 : MAPS_LINE_FORMAT8,
+		     (unsigned long*)&vm_start, (unsigned long*)&vm_end, perm,
+		     comment_buffer);
+	if (len<4)
+	    comment_buffer[0]='\0';
+	if (strstr(comment_buffer, "dynamorio") != 0) {
+	    fclose(maps);
+	    return 1;
+	}
+    }
+  
+    fclose(maps);
+    return 0;
+}
+
+/***************************************************************************/
+
+int main(int argc, char *argv[])
+{
+    pid_t child;
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    if (find_dynamo_library())
+	printf("parent is running under DynamoRIO\n");
+    else
+	printf("parent is running natively\n");
+    child = fork();
+    if (child < 0) {
+	perror("ERROR on fork");
+    } else if (child > 0) {
+	pid_t result;
+	printf("parent waiting for child\n");
+	result = waitpid(child, NULL, 0);
+	assert(result == child);
+	printf("child has exited\n");
+    } else {
+	int result;
+	char *arg[3];
+#if NULL_ENV
+	char **env = NULL;
+#else
+	char *env0 = "LD_LIBRARY_PATH=/bin:.";
+	char *env[2];
+	env[0] = env0;
+	env[1] = NULL;
+#endif
+	arg[0] = "/bin/basename";
+	arg[1] = "/fake/path/it_worked";
+	arg[2] = NULL;
+	if (find_dynamo_library())
+	    printf("child is running under DynamoRIO\n");
+	else
+	    printf("child is running natively\n");
+	result = execve("/bin/bogus_will_fail", arg, env);
+	result = execve("/bin/basename", arg, env);
+	if (result < 0)
+	    perror("ERROR in execve");
+    }	
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
diff --git a/suite/tests/linux/execve.expect b/suite/tests/linux/execve.expect
new file mode 100644
index 000000000..f69788861
--- /dev/null
+++ b/suite/tests/linux/execve.expect
@@ -0,0 +1,4 @@
+it_worked
+parent is running under DynamoRIO
+parent waiting for child
+child has exited
diff --git a/suite/tests/linux/exit.c b/suite/tests/linux/exit.c
new file mode 100644
index 000000000..798e90b66
--- /dev/null
+++ b/suite/tests/linux/exit.c
@@ -0,0 +1,60 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+int main(int argc, char *argv[])
+{
+    int i;
+#ifdef USE_DYNAMO
+    int rc = dynamorio_app_init();
+    assert(rc == 0);
+    dynamorio_app_start();
+#endif
+
+    printf("Inside main\n");
+    fflush(stdout);
+
+    exit(0);
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
diff --git a/suite/tests/linux/exit.expect b/suite/tests/linux/exit.expect
new file mode 100644
index 000000000..726efcb92
--- /dev/null
+++ b/suite/tests/linux/exit.expect
@@ -0,0 +1 @@
+Inside main
diff --git a/suite/tests/linux/fork.c b/suite/tests/linux/fork.c
new file mode 100644
index 000000000..755b54491
--- /dev/null
+++ b/suite/tests/linux/fork.c
@@ -0,0 +1,140 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * test of fork
+ */
+
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/types.h> /* for wait and mmap */
+#include <sys/wait.h>  /* for wait */
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+/***************************************************************************/
+/* a hopefuly portable /proc/@self/maps reader */
+
+/* these are defined in /usr/src/linux/fs/proc/array.c */
+#define MAPS_LINE_LENGTH	4096
+/* for systems with sizeof(void*) == 4: */
+#define MAPS_LINE_FORMAT4	  "%08lx-%08lx %s %*x %*s %*u %4096s"
+#define MAPS_LINE_MAX4	49 /* sum of 8  1  8  1 4 1 8 1 5 1 10 1 */
+/* for systems with sizeof(void*) == 8: */
+#define MAPS_LINE_FORMAT8	  "%016lx-%016lx %s %*x %*s %*u %4096s"
+#define MAPS_LINE_MAX8	73 /* sum of 16  1  16  1 4 1 16 1 5 1 10 1 */
+
+#define MAPS_LINE_MAX	MAPS_LINE_MAX8
+
+int
+find_dynamo_library()
+{
+    pid_t pid = getpid();
+    char 	proc_pid_maps[64];	/* file name */
+
+    FILE *maps;
+    char 	line[MAPS_LINE_LENGTH];
+    int 	count = 0;
+
+    // open file's /proc/id/maps virtual map description
+    int n = snprintf(proc_pid_maps, sizeof(proc_pid_maps),
+		     "/proc/%d/maps", pid);
+    if (n<0 || n==sizeof(proc_pid_maps))
+	assert(0); /* paranoia */
+  
+    maps=fopen(proc_pid_maps,"r");
+
+    while(!feof(maps)){
+	void * vm_start, * vm_end;
+	char perm[16];
+	char comment_buffer[MAPS_LINE_LENGTH];
+	int len;
+    
+	if (NULL==fgets(line, sizeof(line), maps))
+	    break;
+	len = sscanf(line,
+		     sizeof(void*) == 4 ? MAPS_LINE_FORMAT4 : MAPS_LINE_FORMAT8,
+		     (unsigned long*)&vm_start, (unsigned long*)&vm_end, perm,
+		     comment_buffer);
+	if (len<4)
+	    comment_buffer[0]='\0';
+	if (strstr(comment_buffer, "dynamorio") != 0) {
+	    fclose(maps);
+	    return 1;
+	}
+    }
+  
+    fclose(maps);
+    return 0;
+}
+
+/***************************************************************************/
+
+int main(int argc, char** argv)
+{
+    pid_t child;
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    if (find_dynamo_library())
+	printf("parent is running under DynamoRIO\n");
+    else
+	printf("parent is running natively\n");
+    child = fork();
+    if (child < 0) {
+	perror("ERROR on fork");
+    } else if (child > 0) {
+	pid_t result;
+	printf("parent waiting for child\n");
+	result = waitpid(child, NULL, 0);
+	assert(result == child);
+	printf("child has exited\n");
+    } else {
+	if (find_dynamo_library())
+	    printf("child is running under DynamoRIO\n");
+	else
+	    printf("child is running natively\n");
+    }	
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
diff --git a/suite/tests/linux/fork.expect b/suite/tests/linux/fork.expect
new file mode 100644
index 000000000..6140610fd
--- /dev/null
+++ b/suite/tests/linux/fork.expect
@@ -0,0 +1,5 @@
+parent is running under DynamoRIO
+child is running under DynamoRIO
+parent is running under DynamoRIO
+parent waiting for child
+child has exited
diff --git a/suite/tests/linux/infinite.c b/suite/tests/linux/infinite.c
new file mode 100644
index 000000000..6b55d9825
--- /dev/null
+++ b/suite/tests/linux/infinite.c
@@ -0,0 +1,55 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* basic block builder may build infinite loop here */
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+int main()
+{
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+    __asm__("jmp    aroundexit");
+    __asm__("doexit:          ");
+    __asm__("       movl   $1,%eax       # exit");
+    __asm__("       movl   $0,%ebx       # exit code");
+    __asm__("       int    $0x80         # kernel");
+    __asm__("aroundexit: call doexit");
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
diff --git a/suite/tests/linux/infinite.expect b/suite/tests/linux/infinite.expect
new file mode 100644
index 000000000..e69de29bb
diff --git a/suite/tests/linux/longjmp.c b/suite/tests/linux/longjmp.c
new file mode 100644
index 000000000..a645d3a5f
--- /dev/null
+++ b/suite/tests/linux/longjmp.c
@@ -0,0 +1,74 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <stdio.h>
+#include <setjmp.h>
+#include <stdlib.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+jmp_buf mark;
+
+void foo()
+{
+   printf("about to do longjmp\n");
+   fflush(stdout);
+   longjmp(mark, -1);
+}
+
+int main()
+{
+   int jmpret;
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+   /* Save stack environment for return in case of error. First 
+    * time through, jmpret is 0, so true conditional is executed. 
+    * If an error occurs, jmpret will be set to -1 and false 
+    * conditional will be executed.
+    */
+   jmpret = setjmp(mark);
+   if (jmpret == 0) {
+       printf("doing stuff\n");
+       foo();
+   } else {
+       printf("after longjmp\n");
+   }
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+   return 0;
+}
diff --git a/suite/tests/linux/longjmp.expect b/suite/tests/linux/longjmp.expect
new file mode 100644
index 000000000..c3432b032
--- /dev/null
+++ b/suite/tests/linux/longjmp.expect
@@ -0,0 +1,3 @@
+doing stuff
+about to do longjmp
+after longjmp
diff --git a/suite/tests/linux/mmap.c b/suite/tests/linux/mmap.c
new file mode 100644
index 000000000..e69d81ed5
--- /dev/null
+++ b/suite/tests/linux/mmap.c
@@ -0,0 +1,81 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <unistd.h>
+#include <sys/mman.h>
+#include <stdio.h>
+#include "tools.h"
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+int
+main()
+{
+    const int size = 0x00008765;
+    int newsize = size + 0x100; /* these values of size & newsize work for
+                                 * the mremap() on p3rh9 w/the 2.4.21-4
+                                 * kernel
+                                 */
+    void *p;
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+    print("Calling mmap(0, "PFX", "PFX", "PFX", "PFX", 0)\n",
+	   size, PROT_EXEC|PROT_READ|PROT_WRITE,
+	   MAP_ANON|MAP_PRIVATE, -1);
+    p = mmap(0, size, PROT_EXEC|PROT_READ|PROT_WRITE,
+		   MAP_ANON|MAP_PRIVATE, -1, 0);
+    if (p == MAP_FAILED) {
+	print("mmap ERROR "PFX"\n", p);
+	return 1;
+    }
+#if VERBOSE
+    print("mmap returned "PFX"\n", p);
+#endif
+    p = (void *)mremap(p, size, newsize, 0);
+    if ((ptr_int_t) p == -1) {
+	print("mremap ERROR "PFX"\n", p);
+	return 1;
+    }
+#if VERBOSE
+    print("mremap returned "PFX"\n", p);
+#endif
+    munmap(p, newsize);
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
diff --git a/suite/tests/linux/mmap.expect b/suite/tests/linux/mmap.expect
new file mode 100644
index 000000000..56842ca1e
--- /dev/null
+++ b/suite/tests/linux/mmap.expect
@@ -0,0 +1 @@
+Calling mmap(0, 0x00008765, 0x00000007, 0x00000022, 0xffffffff, 0)
diff --git a/suite/tests/linux/signal-base.h b/suite/tests/linux/signal-base.h
new file mode 100644
index 000000000..c49e2c5fd
--- /dev/null
+++ b/suite/tests/linux/signal-base.h
@@ -0,0 +1,292 @@
+/* **********************************************************
+ * Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* not really a header file, but we don't want to run this standalone 
+ * to use, defines these, then include this file:
+#define USE_LONGJMP 0
+#define BLOCK_IN_HANDLER 0
+#define USE_SIGSTACK 0
+#define USE_TIMER 0
+
+#include "signal-base.h"
+ *
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <math.h>
+#include <unistd.h>
+#include <signal.h>
+#include <ucontext.h>
+#include <sys/time.h> /* itimer */
+#include <string.h> /*  memset */
+
+/* handler with SA_SIGINFO flag set gets three arguments: */
+typedef void (*handler_t)(int, struct siginfo *, void *);
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#ifdef X64
+# define SC_XIP rip
+#else
+# define SC_XIP eip
+#endif
+
+#if USE_LONGJMP
+#include <setjmp.h>
+static jmp_buf env;
+#endif
+
+#if USE_SIGSTACK
+# include <stdlib.h> /* malloc */
+/* need more space if might get nested signals */
+# if USE_TIMER
+#  define ALT_STACK_SIZE  (SIGSTKSZ*4)
+# else
+#  define ALT_STACK_SIZE  (SIGSTKSZ*2)
+# endif
+#endif
+
+#if USE_TIMER
+/* need to run long enough to get itimer hit */
+# define ITERS 3500000
+#else
+# define ITERS 500000
+#endif
+
+static int a[ITERS];
+
+/* strategy: anything that won't be the same across multiple runs,
+ * hide inside #if VERBOSE.
+ * timer hits won't be the same, just make sure we get at least one.
+ */
+#if USE_TIMER
+static int timer_hits = 0;
+#endif
+
+#include <errno.h>
+
+#define ASSERT_NOERR(rc) do {					\
+  if (rc) {							\
+     fprintf(stderr, "%s:%d rc=%d errno=%d %s\n",		\
+	     __FILE__, __LINE__,				\
+	     rc, errno, strerror(errno));  		        \
+     _exit(1);							\
+  }								\
+} while (0);
+
+static void
+signal_handler(int sig, siginfo_t *siginfo, ucontext_t *ucxt)
+{
+#if VERBOSE
+    fprintf(stderr, "signal_handler: sig=%d, retaddr=0x%08x, fpregs=0x%08x\n",
+	    sig, *(&sig - 1), ucxt->uc_mcontext.fpregs);
+#else
+# if USE_TIMER
+    if (sig != SIGVTALRM)
+# endif
+	fprintf(stderr, "signal_handler: sig=%d\n", sig);
+#endif
+
+    switch (sig) {
+
+    case SIGSEGV: {
+	struct sigcontext *sc = (struct sigcontext *) &(ucxt->uc_mcontext);
+	void *pc = (void *) sc->SC_XIP;
+#if USE_LONGJMP && BLOCK_IN_HANDLER
+	sigset_t set;
+	int rc;
+#endif
+#if VERBOSE
+	fprintf(stderr, "Got SIGSEGV @ 0x%08x\n", pc);
+#else
+	fprintf(stderr, "Got SIGSEGV\n");
+#endif
+#if USE_LONGJMP
+# if BLOCK_IN_HANDLER
+	/* longjmp will bypass sigreturn, and sigreturn is what resets
+	 * the set of blocked signals, so we have to unblock them here
+	 */
+	rc = sigemptyset(&set); /* reset blocked signals */
+	ASSERT_NOERR(rc);
+	sigprocmask(SIG_SETMASK, &set, NULL);
+# endif
+	longjmp(env, 1);
+#endif
+        break;
+    }
+
+    case SIGUSR1: {
+	struct sigcontext *sc = (struct sigcontext *) &(ucxt->uc_mcontext);
+	void *pc = (void *) sc->SC_XIP;
+#if VERBOSE
+	fprintf(stderr, "Got SIGUSR1 @ 0x%08x\n", pc);
+#else
+	fprintf(stderr, "Got SIGUSR1\n");
+#endif
+        break;
+    }
+
+#if USE_TIMER
+    case SIGVTALRM: {
+	struct sigcontext *sc = (struct sigcontext *) &(ucxt->uc_mcontext);
+	void *pc = (void *) sc->SC_XIP;
+#if VERBOSE
+	fprintf(stderr, "Got SIGVTALRM @ 0x%08x\n", pc);
+#endif
+	timer_hits++;
+	break;
+    }
+#endif
+
+    default:
+	assert(0);
+    }
+}
+
+/* set up signal_handler as the handler for signal "sig" */
+static void
+intercept_signal(int sig, handler_t handler)
+{
+    int rc;
+    struct sigaction act;
+
+    act.sa_sigaction = handler;
+#if BLOCK_IN_HANDLER
+    rc = sigfillset(&act.sa_mask); /* block all signals within handler */
+#else
+    rc = sigemptyset(&act.sa_mask); /* no signals are blocked within handler */
+#endif
+    ASSERT_NOERR(rc);
+    act.sa_flags = SA_SIGINFO | SA_ONSTACK; /* send 3 args to handler */
+    
+    /* arm the signal */
+    rc = sigaction(sig, &act, NULL);
+    ASSERT_NOERR(rc);
+}
+
+
+int main(int argc, char *argv[])
+{
+    double res = 0.;
+    int i, j, rc;
+#if USE_SIGSTACK
+    stack_t sigstack;
+#endif
+#if USE_TIMER
+    struct itimerval t;
+#endif
+  
+#ifdef USE_DYNAMO
+    rc = dynamorio_app_init();
+    ASSERT_NOERR(rc);
+    dynamorio_app_start();
+#endif
+
+#if USE_TIMER
+    intercept_signal(SIGVTALRM, (handler_t) signal_handler);
+    t.it_interval.tv_sec = 0;
+    t.it_interval.tv_usec = 10000;
+    t.it_value.tv_sec = 0;
+    t.it_value.tv_usec = 10000;
+    rc = setitimer(ITIMER_VIRTUAL, &t, NULL);
+    ASSERT_NOERR(rc);
+#endif
+
+#if USE_SIGSTACK
+    sigstack.ss_sp = (char *) malloc(ALT_STACK_SIZE);
+    sigstack.ss_size = ALT_STACK_SIZE;
+    sigstack.ss_flags = SS_ONSTACK;
+    rc = sigaltstack(&sigstack, NULL);
+    ASSERT_NOERR(rc);
+# if VERBOSE
+    fprintf(stderr, "Set up sigstack: 0x%08x - 0x%08x\n",
+	    sigstack.ss_sp, sigstack.ss_sp + sigstack.ss_size);
+# endif
+#endif
+
+    intercept_signal(SIGSEGV, (handler_t) signal_handler);
+    intercept_signal(SIGUSR1, (handler_t) signal_handler);
+    intercept_signal(SIGUSR2, (handler_t) SIG_IGN);
+
+    res = cos(0.56);
+
+    printf("Sending SIGUSR2\n");
+    kill(getpid(), SIGUSR2);
+
+    printf("Sending SIGUSR1\n");
+    kill(getpid(), SIGUSR1);
+
+    printf("Generating SIGSEGV\n");
+#if USE_LONGJMP
+    res = setjmp(env);
+    if (res == 0) {
+	*((int *)0) = 4;
+    }
+#else
+    kill(getpid(), SIGSEGV);
+#endif
+
+    for (i=0; i<ITERS; i++) {
+	if (i % 2 == 0) {
+	    res += cos(1./(double)(i+1));
+	} else {
+	    res += sin(1./(double)(i+1));
+	}
+	j = (i << 4) / (i | 0x38);
+	a[i] += j;
+    }
+    printf("%f\n", res);
+
+#if USE_TIMER
+    memset(&t, 0, sizeof(t));
+    rc = setitimer(ITIMER_VIRTUAL, &t, NULL);
+    ASSERT_NOERR(rc);
+
+    if (timer_hits == 0)
+	printf("Got 0 timer hits!\n");
+    else
+	printf("Got some timer hits!\n");
+#endif
+
+#if USE_SIGSTACK
+    free(sigstack.ss_sp);
+#endif
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
diff --git a/suite/tests/linux/signal0000.c b/suite/tests/linux/signal0000.c
new file mode 100644
index 000000000..e0133b465
--- /dev/null
+++ b/suite/tests/linux/signal0000.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define USE_LONGJMP 0
+#define BLOCK_IN_HANDLER 0
+#define USE_SIGSTACK 0
+#define USE_TIMER 0
+
+#include "signal-base.h"
diff --git a/suite/tests/linux/signal0000.expect b/suite/tests/linux/signal0000.expect
new file mode 100644
index 000000000..adf9a08af
--- /dev/null
+++ b/suite/tests/linux/signal0000.expect
@@ -0,0 +1,8 @@
+signal_handler: sig=10
+Got SIGUSR1
+signal_handler: sig=11
+Got SIGSEGV
+Sending SIGUSR2
+Sending SIGUSR1
+Generating SIGSEGV
+250006.749760
diff --git a/suite/tests/linux/signal0001.c b/suite/tests/linux/signal0001.c
new file mode 100644
index 000000000..b098a1282
--- /dev/null
+++ b/suite/tests/linux/signal0001.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define USE_LONGJMP 0
+#define BLOCK_IN_HANDLER 0
+#define USE_SIGSTACK 0
+#define USE_TIMER 1
+
+#include "signal-base.h"
diff --git a/suite/tests/linux/signal0001.expect b/suite/tests/linux/signal0001.expect
new file mode 100644
index 000000000..5e7204bb7
--- /dev/null
+++ b/suite/tests/linux/signal0001.expect
@@ -0,0 +1,9 @@
+signal_handler: sig=10
+Got SIGUSR1
+signal_handler: sig=11
+Got SIGSEGV
+Sending SIGUSR2
+Sending SIGUSR1
+Generating SIGSEGV
+1750007.722714
+Got some timer hits!
diff --git a/suite/tests/linux/signal0010.c b/suite/tests/linux/signal0010.c
new file mode 100644
index 000000000..ad393214c
--- /dev/null
+++ b/suite/tests/linux/signal0010.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define USE_LONGJMP 0
+#define BLOCK_IN_HANDLER 0
+#define USE_SIGSTACK 1
+#define USE_TIMER 0
+
+#include "signal-base.h"
diff --git a/suite/tests/linux/signal0010.expect b/suite/tests/linux/signal0010.expect
new file mode 100644
index 000000000..adf9a08af
--- /dev/null
+++ b/suite/tests/linux/signal0010.expect
@@ -0,0 +1,8 @@
+signal_handler: sig=10
+Got SIGUSR1
+signal_handler: sig=11
+Got SIGSEGV
+Sending SIGUSR2
+Sending SIGUSR1
+Generating SIGSEGV
+250006.749760
diff --git a/suite/tests/linux/signal0011.c b/suite/tests/linux/signal0011.c
new file mode 100644
index 000000000..4b10771e3
--- /dev/null
+++ b/suite/tests/linux/signal0011.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define USE_LONGJMP 0
+#define BLOCK_IN_HANDLER 0
+#define USE_SIGSTACK 1
+#define USE_TIMER 1
+
+#include "signal-base.h"
diff --git a/suite/tests/linux/signal0011.expect b/suite/tests/linux/signal0011.expect
new file mode 100644
index 000000000..5e7204bb7
--- /dev/null
+++ b/suite/tests/linux/signal0011.expect
@@ -0,0 +1,9 @@
+signal_handler: sig=10
+Got SIGUSR1
+signal_handler: sig=11
+Got SIGSEGV
+Sending SIGUSR2
+Sending SIGUSR1
+Generating SIGSEGV
+1750007.722714
+Got some timer hits!
diff --git a/suite/tests/linux/signal0100.c b/suite/tests/linux/signal0100.c
new file mode 100644
index 000000000..d39f974c0
--- /dev/null
+++ b/suite/tests/linux/signal0100.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define USE_LONGJMP 0
+#define BLOCK_IN_HANDLER 1
+#define USE_SIGSTACK 0
+#define USE_TIMER 0
+
+#include "signal-base.h"
diff --git a/suite/tests/linux/signal0100.expect b/suite/tests/linux/signal0100.expect
new file mode 100644
index 000000000..adf9a08af
--- /dev/null
+++ b/suite/tests/linux/signal0100.expect
@@ -0,0 +1,8 @@
+signal_handler: sig=10
+Got SIGUSR1
+signal_handler: sig=11
+Got SIGSEGV
+Sending SIGUSR2
+Sending SIGUSR1
+Generating SIGSEGV
+250006.749760
diff --git a/suite/tests/linux/signal0101.c b/suite/tests/linux/signal0101.c
new file mode 100644
index 000000000..407943a75
--- /dev/null
+++ b/suite/tests/linux/signal0101.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define USE_LONGJMP 0
+#define BLOCK_IN_HANDLER 1
+#define USE_SIGSTACK 0
+#define USE_TIMER 1
+
+#include "signal-base.h"
diff --git a/suite/tests/linux/signal0101.expect b/suite/tests/linux/signal0101.expect
new file mode 100644
index 000000000..5e7204bb7
--- /dev/null
+++ b/suite/tests/linux/signal0101.expect
@@ -0,0 +1,9 @@
+signal_handler: sig=10
+Got SIGUSR1
+signal_handler: sig=11
+Got SIGSEGV
+Sending SIGUSR2
+Sending SIGUSR1
+Generating SIGSEGV
+1750007.722714
+Got some timer hits!
diff --git a/suite/tests/linux/signal0110.c b/suite/tests/linux/signal0110.c
new file mode 100644
index 000000000..199399c13
--- /dev/null
+++ b/suite/tests/linux/signal0110.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define USE_LONGJMP 0
+#define BLOCK_IN_HANDLER 1
+#define USE_SIGSTACK 1
+#define USE_TIMER 0
+
+#include "signal-base.h"
diff --git a/suite/tests/linux/signal0110.expect b/suite/tests/linux/signal0110.expect
new file mode 100644
index 000000000..adf9a08af
--- /dev/null
+++ b/suite/tests/linux/signal0110.expect
@@ -0,0 +1,8 @@
+signal_handler: sig=10
+Got SIGUSR1
+signal_handler: sig=11
+Got SIGSEGV
+Sending SIGUSR2
+Sending SIGUSR1
+Generating SIGSEGV
+250006.749760
diff --git a/suite/tests/linux/signal0111.c b/suite/tests/linux/signal0111.c
new file mode 100644
index 000000000..9e032b29a
--- /dev/null
+++ b/suite/tests/linux/signal0111.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define USE_LONGJMP 0
+#define BLOCK_IN_HANDLER 1
+#define USE_SIGSTACK 1
+#define USE_TIMER 1
+
+#include "signal-base.h"
diff --git a/suite/tests/linux/signal0111.expect b/suite/tests/linux/signal0111.expect
new file mode 100644
index 000000000..5e7204bb7
--- /dev/null
+++ b/suite/tests/linux/signal0111.expect
@@ -0,0 +1,9 @@
+signal_handler: sig=10
+Got SIGUSR1
+signal_handler: sig=11
+Got SIGSEGV
+Sending SIGUSR2
+Sending SIGUSR1
+Generating SIGSEGV
+1750007.722714
+Got some timer hits!
diff --git a/suite/tests/linux/signal1000.c b/suite/tests/linux/signal1000.c
new file mode 100644
index 000000000..c95e77679
--- /dev/null
+++ b/suite/tests/linux/signal1000.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define USE_LONGJMP 1
+#define BLOCK_IN_HANDLER 0
+#define USE_SIGSTACK 0
+#define USE_TIMER 0
+
+#include "signal-base.h"
diff --git a/suite/tests/linux/signal1000.expect b/suite/tests/linux/signal1000.expect
new file mode 100644
index 000000000..a26d68ec1
--- /dev/null
+++ b/suite/tests/linux/signal1000.expect
@@ -0,0 +1,8 @@
+signal_handler: sig=10
+Got SIGUSR1
+signal_handler: sig=11
+Got SIGSEGV
+Sending SIGUSR2
+Sending SIGUSR1
+Generating SIGSEGV
+250006.902505
diff --git a/suite/tests/linux/signal1001.c b/suite/tests/linux/signal1001.c
new file mode 100644
index 000000000..48c849f10
--- /dev/null
+++ b/suite/tests/linux/signal1001.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define USE_LONGJMP 1
+#define BLOCK_IN_HANDLER 0
+#define USE_SIGSTACK 0
+#define USE_TIMER 1
+
+#include "signal-base.h"
diff --git a/suite/tests/linux/signal1001.expect b/suite/tests/linux/signal1001.expect
new file mode 100644
index 000000000..35b15d7c1
--- /dev/null
+++ b/suite/tests/linux/signal1001.expect
@@ -0,0 +1,9 @@
+signal_handler: sig=10
+Got SIGUSR1
+signal_handler: sig=11
+Got SIGSEGV
+Sending SIGUSR2
+Sending SIGUSR1
+Generating SIGSEGV
+1750007.875459
+Got some timer hits!
diff --git a/suite/tests/linux/signal1010.c b/suite/tests/linux/signal1010.c
new file mode 100644
index 000000000..0aeaf50b2
--- /dev/null
+++ b/suite/tests/linux/signal1010.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define USE_LONGJMP 1
+#define BLOCK_IN_HANDLER 0
+#define USE_SIGSTACK 1
+#define USE_TIMER 0
+
+#include "signal-base.h"
diff --git a/suite/tests/linux/signal1010.expect b/suite/tests/linux/signal1010.expect
new file mode 100644
index 000000000..a26d68ec1
--- /dev/null
+++ b/suite/tests/linux/signal1010.expect
@@ -0,0 +1,8 @@
+signal_handler: sig=10
+Got SIGUSR1
+signal_handler: sig=11
+Got SIGSEGV
+Sending SIGUSR2
+Sending SIGUSR1
+Generating SIGSEGV
+250006.902505
diff --git a/suite/tests/linux/signal1011.c b/suite/tests/linux/signal1011.c
new file mode 100644
index 000000000..c15c37ad7
--- /dev/null
+++ b/suite/tests/linux/signal1011.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define USE_LONGJMP 1
+#define BLOCK_IN_HANDLER 0
+#define USE_SIGSTACK 1
+#define USE_TIMER 1
+
+#include "signal-base.h"
diff --git a/suite/tests/linux/signal1011.expect b/suite/tests/linux/signal1011.expect
new file mode 100644
index 000000000..35b15d7c1
--- /dev/null
+++ b/suite/tests/linux/signal1011.expect
@@ -0,0 +1,9 @@
+signal_handler: sig=10
+Got SIGUSR1
+signal_handler: sig=11
+Got SIGSEGV
+Sending SIGUSR2
+Sending SIGUSR1
+Generating SIGSEGV
+1750007.875459
+Got some timer hits!
diff --git a/suite/tests/linux/signal1100.c b/suite/tests/linux/signal1100.c
new file mode 100644
index 000000000..fb436068a
--- /dev/null
+++ b/suite/tests/linux/signal1100.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define USE_LONGJMP 1
+#define BLOCK_IN_HANDLER 1
+#define USE_SIGSTACK 0
+#define USE_TIMER 0
+
+#include "signal-base.h"
diff --git a/suite/tests/linux/signal1100.expect b/suite/tests/linux/signal1100.expect
new file mode 100644
index 000000000..a26d68ec1
--- /dev/null
+++ b/suite/tests/linux/signal1100.expect
@@ -0,0 +1,8 @@
+signal_handler: sig=10
+Got SIGUSR1
+signal_handler: sig=11
+Got SIGSEGV
+Sending SIGUSR2
+Sending SIGUSR1
+Generating SIGSEGV
+250006.902505
diff --git a/suite/tests/linux/signal1101.c b/suite/tests/linux/signal1101.c
new file mode 100644
index 000000000..2a436b8d0
--- /dev/null
+++ b/suite/tests/linux/signal1101.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define USE_LONGJMP 1
+#define BLOCK_IN_HANDLER 1
+#define USE_SIGSTACK 0
+#define USE_TIMER 1
+
+#include "signal-base.h"
diff --git a/suite/tests/linux/signal1101.expect b/suite/tests/linux/signal1101.expect
new file mode 100644
index 000000000..35b15d7c1
--- /dev/null
+++ b/suite/tests/linux/signal1101.expect
@@ -0,0 +1,9 @@
+signal_handler: sig=10
+Got SIGUSR1
+signal_handler: sig=11
+Got SIGSEGV
+Sending SIGUSR2
+Sending SIGUSR1
+Generating SIGSEGV
+1750007.875459
+Got some timer hits!
diff --git a/suite/tests/linux/signal1110.c b/suite/tests/linux/signal1110.c
new file mode 100644
index 000000000..04fd6be72
--- /dev/null
+++ b/suite/tests/linux/signal1110.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define USE_LONGJMP 1
+#define BLOCK_IN_HANDLER 1
+#define USE_SIGSTACK 1
+#define USE_TIMER 0
+
+#include "signal-base.h"
diff --git a/suite/tests/linux/signal1110.expect b/suite/tests/linux/signal1110.expect
new file mode 100644
index 000000000..a26d68ec1
--- /dev/null
+++ b/suite/tests/linux/signal1110.expect
@@ -0,0 +1,8 @@
+signal_handler: sig=10
+Got SIGUSR1
+signal_handler: sig=11
+Got SIGSEGV
+Sending SIGUSR2
+Sending SIGUSR1
+Generating SIGSEGV
+250006.902505
diff --git a/suite/tests/linux/signal1111.c b/suite/tests/linux/signal1111.c
new file mode 100644
index 000000000..cb4619355
--- /dev/null
+++ b/suite/tests/linux/signal1111.c
@@ -0,0 +1,38 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define USE_LONGJMP 1
+#define BLOCK_IN_HANDLER 1
+#define USE_SIGSTACK 1
+#define USE_TIMER 1
+
+#include "signal-base.h"
diff --git a/suite/tests/linux/signal1111.expect b/suite/tests/linux/signal1111.expect
new file mode 100644
index 000000000..35b15d7c1
--- /dev/null
+++ b/suite/tests/linux/signal1111.expect
@@ -0,0 +1,9 @@
+signal_handler: sig=10
+Got SIGUSR1
+signal_handler: sig=11
+Got SIGSEGV
+Sending SIGUSR2
+Sending SIGUSR1
+Generating SIGSEGV
+1750007.875459
+Got some timer hits!
diff --git a/suite/tests/linux/sigplain-base.h b/suite/tests/linux/sigplain-base.h
new file mode 100644
index 000000000..1ed0868b8
--- /dev/null
+++ b/suite/tests/linux/sigplain-base.h
@@ -0,0 +1,214 @@
+/* **********************************************************
+ * Copyright (c) 2003-2009 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* not really a header file, but we don't want to run this standalone 
+ * to use, defines these, then include this file:
+
+#define BLOCK_IN_HANDLER 0
+#define USE_SIGSTACK 0
+#define USE_TIMER 0
+
+#include "sigplain-base.h"
+ *
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <math.h>
+#include <unistd.h>
+#include <signal.h>
+#include <ucontext.h>
+#include <sys/time.h> /* itimer */
+#include <string.h> /*  memset */
+
+/* just use single-arg handlers */
+typedef void (*handler_t)(int);
+typedef void (*handler_3_t)(int, struct siginfo *, void *);
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#if USE_SIGSTACK
+# include <stdlib.h> /* malloc */
+/* need more space if might get nested signals */
+# if USE_TIMER
+#  define ALT_STACK_SIZE  (SIGSTKSZ*4)
+# else
+#  define ALT_STACK_SIZE  (SIGSTKSZ*2)
+# endif
+#endif
+
+#if USE_TIMER
+/* need to run long enough to get itimer hit */
+# define ITERS 3500000
+#else
+# define ITERS 500000
+#endif
+
+static int a[ITERS];
+
+
+/* strategy: anything that won't be the same across multiple runs,
+ * hide inside #if VERBOSE.
+ * timer hits won't be the same, just make sure we get at least one.
+ */
+#if USE_TIMER
+static int timer_hits = 0;
+#endif
+
+#include <errno.h>
+
+#define ASSERT_NOERR(rc) do {					\
+  if (rc) {							\
+     fprintf(stderr, "%s:%d rc=%d errno=%d %s\n", 		\
+	     __FILE__, __LINE__,				\
+	     rc, errno, strerror(errno));  		        \
+  }								\
+} while (0);
+
+static void
+signal_handler(int sig)
+{
+#if USE_TIMER
+    if (sig == SIGVTALRM)
+	timer_hits++;
+    else
+#endif
+	fprintf(stderr, "signal_handler: sig=%d\n", sig);
+}
+
+/* set up signal_handler as the handler for signal "sig" */
+static void
+intercept_signal(int sig, handler_t handler)
+{
+    int rc;
+    struct sigaction act;
+
+    act.sa_sigaction = (handler_3_t) handler;
+#if BLOCK_IN_HANDLER
+    rc = sigfillset(&act.sa_mask); /* block all signals within handler */
+#else
+    rc = sigemptyset(&act.sa_mask); /* no signals are blocked within handler */
+#endif
+    ASSERT_NOERR(rc);
+    act.sa_flags = SA_ONSTACK;
+    
+    /* arm the signal */
+    rc = sigaction(sig, &act, NULL);
+    ASSERT_NOERR(rc);
+}
+
+
+int main(int argc, char *argv[])
+{
+    double res = 0.;
+    int i, j, rc;
+#if USE_SIGSTACK
+    stack_t sigstack;
+#endif
+#if USE_TIMER
+    struct itimerval t;
+#endif
+  
+#ifdef USE_DYNAMO
+    rc = dynamorio_app_init();
+    ASSERT_NOERR(rc);
+    dynamorio_app_start();
+#endif
+
+#if USE_TIMER
+    intercept_signal(SIGVTALRM, (handler_t) signal_handler);
+    t.it_interval.tv_sec = 0;
+    t.it_interval.tv_usec = 20000;
+    t.it_value.tv_sec = 0;
+    t.it_value.tv_usec = 20000;
+    rc = setitimer(ITIMER_VIRTUAL, &t, NULL);
+    ASSERT_NOERR(rc);
+#endif
+
+#if USE_SIGSTACK
+    sigstack.ss_sp = (char *) malloc(ALT_STACK_SIZE);
+    sigstack.ss_size = ALT_STACK_SIZE;
+    sigstack.ss_flags = SS_ONSTACK;
+    rc = sigaltstack(&sigstack, NULL);
+    ASSERT_NOERR(rc);
+# if VERBOSE
+    fprintf(stderr, "Set up sigstack: 0x%08x - 0x%08x\n",
+	    sigstack.ss_sp, sigstack.ss_sp + sigstack.ss_size);
+# endif
+#endif
+
+    intercept_signal(SIGSEGV, (handler_t) signal_handler);
+    intercept_signal(SIGUSR1, (handler_t) signal_handler);
+    intercept_signal(SIGUSR2, (handler_t) SIG_IGN);
+
+    res = cos(0.56);
+
+    printf("Sending SIGUSR2\n");
+    kill(getpid(), SIGUSR2);
+
+    printf("Sending SIGUSR1\n");
+    kill(getpid(), SIGUSR1);
+
+    for (i=0; i<ITERS; i++) {
+	if (i % 2 == 0) {
+	    res += cos(1./(double)(i+1));
+	} else {
+	    res += sin(1./(double)(i+1));
+	}
+	j = (i << 4) / (i | 0x38);
+	a[i] += j;
+    }
+    printf("%f\n", res);
+
+#if USE_TIMER
+    memset(&t, 0, sizeof(t));
+    rc = setitimer(ITIMER_VIRTUAL, &t, NULL);
+    ASSERT_NOERR(rc);
+
+    if (timer_hits == 0)
+	printf("Got 0 timer hits!\n");
+    else
+	printf("Got some timer hits!\n");
+#endif
+
+#if USE_SIGSTACK
+    free(sigstack.ss_sp);
+#endif
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
diff --git a/suite/tests/linux/sigplain000.c b/suite/tests/linux/sigplain000.c
new file mode 100644
index 000000000..606a43fc0
--- /dev/null
+++ b/suite/tests/linux/sigplain000.c
@@ -0,0 +1,37 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define BLOCK_IN_HANDLER 0
+#define USE_SIGSTACK 0
+#define USE_TIMER 0
+
+#include "sigplain-base.h"
diff --git a/suite/tests/linux/sigplain000.expect b/suite/tests/linux/sigplain000.expect
new file mode 100644
index 000000000..57013e088
--- /dev/null
+++ b/suite/tests/linux/sigplain000.expect
@@ -0,0 +1,4 @@
+signal_handler: sig=10
+Sending SIGUSR2
+Sending SIGUSR1
+250006.749760
diff --git a/suite/tests/linux/sigplain001.c b/suite/tests/linux/sigplain001.c
new file mode 100644
index 000000000..818a6b778
--- /dev/null
+++ b/suite/tests/linux/sigplain001.c
@@ -0,0 +1,37 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define BLOCK_IN_HANDLER 0
+#define USE_SIGSTACK 0
+#define USE_TIMER 1
+
+#include "sigplain-base.h"
diff --git a/suite/tests/linux/sigplain001.expect b/suite/tests/linux/sigplain001.expect
new file mode 100644
index 000000000..a6529f76c
--- /dev/null
+++ b/suite/tests/linux/sigplain001.expect
@@ -0,0 +1,5 @@
+signal_handler: sig=10
+Sending SIGUSR2
+Sending SIGUSR1
+1750007.722714
+Got some timer hits!
diff --git a/suite/tests/linux/sigplain010.c b/suite/tests/linux/sigplain010.c
new file mode 100644
index 000000000..5240fd921
--- /dev/null
+++ b/suite/tests/linux/sigplain010.c
@@ -0,0 +1,37 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define BLOCK_IN_HANDLER 0
+#define USE_SIGSTACK 1
+#define USE_TIMER 0
+
+#include "sigplain-base.h"
diff --git a/suite/tests/linux/sigplain010.expect b/suite/tests/linux/sigplain010.expect
new file mode 100644
index 000000000..57013e088
--- /dev/null
+++ b/suite/tests/linux/sigplain010.expect
@@ -0,0 +1,4 @@
+signal_handler: sig=10
+Sending SIGUSR2
+Sending SIGUSR1
+250006.749760
diff --git a/suite/tests/linux/sigplain011.c b/suite/tests/linux/sigplain011.c
new file mode 100644
index 000000000..fcca7ae35
--- /dev/null
+++ b/suite/tests/linux/sigplain011.c
@@ -0,0 +1,37 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define BLOCK_IN_HANDLER 0
+#define USE_SIGSTACK 1
+#define USE_TIMER 1
+
+#include "sigplain-base.h"
diff --git a/suite/tests/linux/sigplain011.expect b/suite/tests/linux/sigplain011.expect
new file mode 100644
index 000000000..a6529f76c
--- /dev/null
+++ b/suite/tests/linux/sigplain011.expect
@@ -0,0 +1,5 @@
+signal_handler: sig=10
+Sending SIGUSR2
+Sending SIGUSR1
+1750007.722714
+Got some timer hits!
diff --git a/suite/tests/linux/sigplain100.c b/suite/tests/linux/sigplain100.c
new file mode 100644
index 000000000..daff5e8a3
--- /dev/null
+++ b/suite/tests/linux/sigplain100.c
@@ -0,0 +1,37 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define BLOCK_IN_HANDLER 1
+#define USE_SIGSTACK 0
+#define USE_TIMER 0
+
+#include "sigplain-base.h"
diff --git a/suite/tests/linux/sigplain100.expect b/suite/tests/linux/sigplain100.expect
new file mode 100644
index 000000000..57013e088
--- /dev/null
+++ b/suite/tests/linux/sigplain100.expect
@@ -0,0 +1,4 @@
+signal_handler: sig=10
+Sending SIGUSR2
+Sending SIGUSR1
+250006.749760
diff --git a/suite/tests/linux/sigplain101.c b/suite/tests/linux/sigplain101.c
new file mode 100644
index 000000000..a80dcac9c
--- /dev/null
+++ b/suite/tests/linux/sigplain101.c
@@ -0,0 +1,37 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define BLOCK_IN_HANDLER 1
+#define USE_SIGSTACK 0
+#define USE_TIMER 1
+
+#include "sigplain-base.h"
diff --git a/suite/tests/linux/sigplain101.expect b/suite/tests/linux/sigplain101.expect
new file mode 100644
index 000000000..a6529f76c
--- /dev/null
+++ b/suite/tests/linux/sigplain101.expect
@@ -0,0 +1,5 @@
+signal_handler: sig=10
+Sending SIGUSR2
+Sending SIGUSR1
+1750007.722714
+Got some timer hits!
diff --git a/suite/tests/linux/sigplain110.c b/suite/tests/linux/sigplain110.c
new file mode 100644
index 000000000..bf2dc5a6a
--- /dev/null
+++ b/suite/tests/linux/sigplain110.c
@@ -0,0 +1,37 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define BLOCK_IN_HANDLER 1
+#define USE_SIGSTACK 1
+#define USE_TIMER 0
+
+#include "sigplain-base.h"
diff --git a/suite/tests/linux/sigplain110.expect b/suite/tests/linux/sigplain110.expect
new file mode 100644
index 000000000..57013e088
--- /dev/null
+++ b/suite/tests/linux/sigplain110.expect
@@ -0,0 +1,4 @@
+signal_handler: sig=10
+Sending SIGUSR2
+Sending SIGUSR1
+250006.749760
diff --git a/suite/tests/linux/sigplain111.c b/suite/tests/linux/sigplain111.c
new file mode 100644
index 000000000..fbaf6a3da
--- /dev/null
+++ b/suite/tests/linux/sigplain111.c
@@ -0,0 +1,37 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define BLOCK_IN_HANDLER 1
+#define USE_SIGSTACK 1
+#define USE_TIMER 1
+
+#include "sigplain-base.h"
diff --git a/suite/tests/linux/sigplain111.expect b/suite/tests/linux/sigplain111.expect
new file mode 100644
index 000000000..a6529f76c
--- /dev/null
+++ b/suite/tests/linux/sigplain111.expect
@@ -0,0 +1,5 @@
+signal_handler: sig=10
+Sending SIGUSR2
+Sending SIGUSR1
+1750007.722714
+Got some timer hits!
diff --git a/suite/tests/linux/thread.c b/suite/tests/linux/thread.c
new file mode 100644
index 000000000..27a6ebd60
--- /dev/null
+++ b/suite/tests/linux/thread.c
@@ -0,0 +1,240 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * test of clone call
+ */
+
+#include <sys/types.h> /* for wait and mmap */
+#include <sys/wait.h>  /* for wait */
+#include <sched.h>     /* for __clone and CLONE_ flags */
+#include <time.h>      /* for nanosleep */
+#include <sys/mman.h>  /* for mmap */
+#include <assert.h>
+#include <signal.h>
+#include <stdio.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+/* FIXME: our toolchain headers are too old for these so hardcoding: */
+/* linux/sched.h is hard to include properly */
+#define CLONE_THREAD	0x00010000	/* Same thread group? */
+#define CLONE_CHILD_CLEARTID 0x00200000      /* clear the TID in the child */
+
+#ifdef __i386__
+# define __NR_set_tid_address 258
+# define __NR_gettid 224
+#else
+# define __NR_set_tid_address 218
+# define __NR_gettid 186
+#endif
+#define SYS_set_tid_address __NR_set_tid_address
+#define SYS_gettid __NR_gettid
+
+#define false (0)
+#define true (1)
+typedef int bool;
+#define THREAD_STACK_SIZE   (32*1024)
+
+/* forward declarations */
+static pid_t create_thread(int (*fcn)(void *), void *arg, void **stack);
+static void delete_thread(pid_t pid, void *stack);
+int run(void *arg);
+static void* stack_alloc(int size);
+static void stack_free(void *p, int size);
+
+/* vars for child thread */
+static pid_t child;
+static void *stack;
+
+/* these are used solely to provide deterministic output */
+/* this is read by child, written by parent, tells child whether to exit */
+static volatile bool child_exit;
+/* this is read by parent, written by child, tells parent whether child done */
+static volatile bool child_done;
+
+static struct timespec sleeptime;
+
+int main()
+{
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    sleeptime.tv_sec = 0;
+    sleeptime.tv_nsec = 10*1000*1000; /* 10ms */
+
+    child_exit = false;
+    child_done = false;
+    child = create_thread(run, NULL, &stack);
+    assert(child > -1);
+
+    /* waste some time */
+    nanosleep(&sleeptime, NULL);
+    
+    child_exit = true;
+    /* we want deterministic printf ordering */
+    while (!child_done)
+        nanosleep(&sleeptime, NULL);
+    delete_thread(child, stack);
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
+
+/* Procedure executed by sideline threads
+ */
+int run(void *arg)
+{
+    int i = 0;
+    /* for CLONE_CHILD_CLEARTID for signaling parent.  if we used raw
+     * clone system call we could get kernel to do this for us. 
+     */
+    child = syscall(SYS_gettid);
+    syscall(SYS_set_tid_address, &child);
+    printf("Sideline thread started\n");
+    while (true) {
+	/* do nothing for now */
+	i++;
+	if (i % 2500000 == 0)
+	    printf("i = %d\n", i);
+	if (i % 25000000 == 0)
+	    break;
+    }
+    while (!child_exit)
+        nanosleep(&sleeptime, NULL);
+    printf("Sideline thread finished\n");
+    child_done = true;
+    return 0;
+}
+
+/* Create a new thread. It should be passed "fcn", a function which
+ * takes two arguments, (the second one is a dummy, always 4). The
+ * first argument is passed in "arg". Returns the TID of the new
+ * thread */
+static pid_t
+create_thread(int (*fcn)(void *), void *arg, void **stack)
+{
+    pid_t newpid; 
+    int flags;
+    void *my_stack;
+    
+    my_stack = stack_alloc(THREAD_STACK_SIZE);
+    /* need SIGCHLD so parent will get that signal when child dies,
+     * else have errors doing a wait */
+    flags = SIGCHLD | CLONE_THREAD | CLONE_VM |
+        /* CLONE_THREAD => no signal to parent on termination; have to use
+         * CLONE_CHILD_CLEARTID to get that.  Since we're using library call
+         * instead of raw system call we don't have child_tidptr argument,
+         * so we set the location in the child itself via set_tid_address(). */
+        CLONE_CHILD_CLEARTID | 
+        CLONE_FS | CLONE_FILES | CLONE_SIGHAND;
+    newpid = __clone(fcn, my_stack, flags, arg);
+    /* this is really a tid since we passed CLONE_THREAD: child has same pid as us */
+  
+    if (newpid == -1) {
+	printf("smp.c: Error calling __clone\n");
+	stack_free(my_stack, THREAD_STACK_SIZE);
+	return -1;
+    }
+
+    *stack = my_stack;
+    return newpid;
+}
+
+static void 
+delete_thread(pid_t pid, void *stack)
+{
+    pid_t result;
+    /* do not print out pids to make diff easy */
+    printf("Waiting for child to exit\n");
+    /* pid is really a tid, and since we used CLONE_THREAD, we cannot use
+     * any wait() routine since our parent has the child not us.
+     * so we rely on CLONE_CHILD_CLEARTID.  FIXME: use futex here.
+     * for now being really simple.
+     */
+    while (child != 0)
+        nanosleep(&sleeptime, NULL);
+    printf("Child has exited\n");
+    stack_free(stack, THREAD_STACK_SIZE);
+}
+
+/* allocate stack storage on the app's heap */
+void*
+stack_alloc(int size)
+{
+    size_t sp;
+    void *q = NULL;
+    void *p;
+
+#if STACK_OVERFLOW_PROTECT
+    /* allocate an extra page and mark it non-accessible to trap stack overflow */
+    q = mmap(0, PAGE_SIZE, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0);
+    assert(q);
+    stack_redzone_start = (size_t) q;
+#endif 
+
+    p = mmap(q, size, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0);
+    assert(p);
+#ifdef DEBUG
+    memset(p, 0xab, size);
+#endif 
+
+    /* stack grows from high to low addresses, so return a ptr to the top of the
+       allocated region */
+    sp = (size_t)p + size;
+
+    return (void*) sp;
+}
+
+/* free memory-mapped stack storage */
+void
+stack_free(void *p, int size)
+{
+    size_t sp = (size_t)p - size;
+
+#ifdef DEBUG
+    memset((void *)sp, 0xcd, size);
+#endif 
+    munmap((void *)sp, size);
+
+#if STACK_OVERFLOW_PROTECT
+    sp = sp - PAGE_SIZE;
+    munmap((void*) sp, PAGE_SIZE);
+#endif 
+}
+
diff --git a/suite/tests/linux/thread.expect b/suite/tests/linux/thread.expect
new file mode 100644
index 000000000..242de3e92
--- /dev/null
+++ b/suite/tests/linux/thread.expect
@@ -0,0 +1,14 @@
+Sideline thread started
+i = 2500000
+i = 5000000
+i = 7500000
+i = 10000000
+i = 12500000
+i = 15000000
+i = 17500000
+i = 20000000
+i = 22500000
+i = 25000000
+Sideline thread finished
+Waiting for child to exit
+Child has exited
diff --git a/suite/tests/linux/vfork-fib.c b/suite/tests/linux/vfork-fib.c
new file mode 100644
index 000000000..25de0d5bc
--- /dev/null
+++ b/suite/tests/linux/vfork-fib.c
@@ -0,0 +1,192 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * test of fork
+ */
+
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/types.h> /* for wait and mmap */
+#include <sys/wait.h>  /* for wait */
+#include <assert.h>
+#include <stdio.h>
+
+#include "common.h"
+
+const int N = 8;
+
+/***************************************************************************/
+
+int fib(int n) {
+    if (n <= 1) return 1;
+    return fib(n-1) + fib(n-2);
+}
+
+#ifdef DEBUG
+#  define pf printf
+#else
+#  define pf if(0)printf
+#endif
+
+
+int main(int argc, char** argv)
+{
+    pid_t child1, child2;
+    int do_vfork = 1; //argc == 1;
+
+    int n, sum = 0;
+    int result;
+    char *arg[5];
+    char **env = NULL;
+    char carg[10], csum[10];
+
+    if (find_dynamo_library())
+	printf("rio\n");
+    else
+	printf("native\n");
+
+    //printf("%d %s %s %s\n", argc, argv[0], argv[1], argv[2]);
+    if (argc < 3) {  // start calculation
+	if (2 == argc)  // vfork-fib 10
+	    n = atoi(argv[1]);
+	else 
+	    n = N;
+
+	printf("parent fib(%d)=%d\n", n, fib(n));
+	sum = 0;
+    } else {
+	assert(argc == 3);  // vfork-fib fib 10
+	n = atoi(argv[2]);
+	sum = 0;
+    }
+
+    pf("\tfib %d\n", n);
+
+    if (n <= 1) { // base case
+	pf("base case\n");
+	exit(1);
+    }
+
+    // now spawn two children
+    arg[0] = argv[0];
+    arg[1] = "fib";
+    arg[3] = NULL;
+
+
+    if (do_vfork) {		/* default */
+	pf("using vfork()\n");
+	child1 = vfork(); 
+    } else {
+	pf("using fork()\n");
+	child1 = fork();
+    }
+
+    if (child1 < 0) {
+	perror("ERROR on fork");
+    } else if (child1 == 0) {
+	snprintf(carg, 10, "%d", n-2);
+	arg[2] = carg;
+
+#if  0
+	pf("execing %d %s %s=%s %s\n", 
+	       3, arg[0], carg, arg[1], arg[2]);
+#endif
+	result = execve(arg[0], arg, env);
+	if (result < 0)
+	    perror("ERROR in execve");
+    } else {
+	pid_t result;
+	int status;
+	int children = 2;
+
+	if (do_vfork) {		/* default */
+	    pf("second child using vfork()\n");
+	    child2 = vfork(); 
+	} else {
+	    pf("second child using fork()\n");
+	    child2 = fork();
+	}
+	if (child2 < 0) {
+	    perror("ERROR on fork");
+	} else if (child2 == 0) {
+	    snprintf(carg, 10, "%d", n-1);
+	    arg[2] = carg;
+	    result = execve(arg[0], arg, env);
+	    if (result < 0) perror("ERROR in execve");
+	} 
+
+	while(children > 0) {
+	    pf("parent waiting for %d children\n", children);
+	    result = wait(&status);
+
+	    assert(result == child2 || result == child1);
+	    assert(WIFEXITED(status));
+	    //printf("child %d has exited with status=%d %d\n", result, status, WEXITSTATUS(status));
+	    sum+=WEXITSTATUS(status);
+
+	    if (children == 2 && result == child1) 
+		pf("first child before second\n");
+	    else
+		pf("second child before first\n");
+
+	    children--;
+	}
+
+#if 0
+	result = waitpid(child2, &status, 0);
+	assert(result == child2);
+	assert(WIFEXITED(status));
+	pf("child2 has exited with status=%d %d\n", status, WEXITSTATUS(status));
+	sum+=WEXITSTATUS(status);
+
+	pf("parent waiting for child1 %d\n", child1);
+	result = waitpid(child1, &status, 0);
+	assert(result == child1);
+	assert(WIFEXITED(status));
+	pf("child1 has exited with status=%d %d\n", status, WEXITSTATUS(status));
+	sum+=WEXITSTATUS(status);
+#endif
+    } 
+
+#ifdef DEBUG
+    printf("\tfib(%d)=%d [%d] %s\n", n, sum, fib(n), sum == fib(n) ? "OK" : "BAD");
+#else
+    if (argc == 1) 
+	printf("\tfib(%d)=%d [%d] %s\n", n, sum, fib(n), sum == fib(n) ? "OK" : "BAD");
+#endif
+    exit(sum);
+}
+
+
+// TODO it would be nice to have in the tests a measure for nondeterminism
+// to also a guarantee that we don't introduce extra synchronization to stifle any parallelism
diff --git a/suite/tests/linux/vfork-fib.expect b/suite/tests/linux/vfork-fib.expect
new file mode 100644
index 000000000..38ffcee20
--- /dev/null
+++ b/suite/tests/linux/vfork-fib.expect
@@ -0,0 +1,69 @@
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+rio
+parent fib(8)=34
+	fib(8)=34 [34] OK
diff --git a/suite/tests/linux/vfork.c b/suite/tests/linux/vfork.c
new file mode 100644
index 000000000..5de377630
--- /dev/null
+++ b/suite/tests/linux/vfork.c
@@ -0,0 +1,150 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * test of fork
+ */
+
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/types.h> /* for wait and mmap */
+#include <sys/wait.h>  /* for wait */
+#include <assert.h>
+#include <stdio.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+/***************************************************************************/
+/* a hopefuly portable /proc/@self/maps reader */
+
+/* these are defined in /usr/src/linux/fs/proc/array.c */
+#define MAPS_LINE_LENGTH	4096
+/* for systems with sizeof(void*) == 4: */
+#define MAPS_LINE_FORMAT4	  "%08lx-%08lx %s %*x %*s %*u %4096s"
+#define MAPS_LINE_MAX4	49 /* sum of 8  1  8  1 4 1 8 1 5 1 10 1 */
+/* for systems with sizeof(void*) == 8: */
+#define MAPS_LINE_FORMAT8	  "%016lx-%016lx %s %*x %*s %*u %4096s"
+#define MAPS_LINE_MAX8	73 /* sum of 16  1  16  1 4 1 16 1 5 1 10 1 */
+
+#define MAPS_LINE_MAX	MAPS_LINE_MAX8
+
+int
+find_dynamo_library()
+{
+    pid_t pid = getpid();
+    char 	proc_pid_maps[64];	/* file name */
+
+    FILE *maps;
+    char 	line[MAPS_LINE_LENGTH];
+    int 	count = 0;
+
+    // open file's /proc/id/maps virtual map description
+    int n = snprintf(proc_pid_maps, sizeof(proc_pid_maps),
+		     "/proc/%d/maps", pid);
+    if (n<0 || n==sizeof(proc_pid_maps))
+	assert(0); /* paranoia */
+  
+    maps=fopen(proc_pid_maps,"r");
+
+    while(!feof(maps)){
+	void * vm_start, * vm_end;
+	char perm[16];
+	char comment_buffer[MAPS_LINE_LENGTH];
+	int len;
+    
+	if (NULL==fgets(line, sizeof(line), maps))
+	    break;
+	len = sscanf(line,
+		     sizeof(void*) == 4 ? MAPS_LINE_FORMAT4 : MAPS_LINE_FORMAT8,
+		     (unsigned long*)&vm_start, (unsigned long*)&vm_end, perm,
+		     comment_buffer);
+	if (len<4)
+	    comment_buffer[0]='\0';
+	if (strstr(comment_buffer, "dynamorio") != 0) {
+	    fclose(maps);
+	    return 1;
+	}
+    }
+  
+    fclose(maps);
+    return 0;
+}
+
+/***************************************************************************/
+
+int main(int argc, char** argv)
+{
+    pid_t child;
+    int do_vfork = argc == 1;
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    if (find_dynamo_library())
+	printf("parent is running under DynamoRIO\n");
+    else
+	printf("parent is running natively\n");
+
+    if (do_vfork) {		/* default */
+	printf("trying vfork()\n");
+	child = vfork(); 
+    } else {
+	printf("trying fork()\n");
+	child = fork();
+    }
+
+    if (child < 0) {
+	perror("ERROR on fork");
+    } else if (child > 0) {
+	pid_t result;
+	printf("parent waiting for child\n");
+	result = waitpid(child, NULL, 0);
+	assert(result == child);
+	printf("child has exited\n");
+    } else {
+	if (find_dynamo_library())
+	    printf("child is running under DynamoRIO\n");
+	else
+	    printf("child is running natively\n");
+	if (argc == 1)
+	    _exit(0);
+    }	
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
diff --git a/suite/tests/linux/vfork.expect b/suite/tests/linux/vfork.expect
new file mode 100644
index 000000000..17f1c57d8
--- /dev/null
+++ b/suite/tests/linux/vfork.expect
@@ -0,0 +1,5 @@
+parent is running under DynamoRIO
+trying vfork()
+child is running under DynamoRIO
+parent waiting for child
+child has exited
diff --git a/suite/tests/mresources.rc b/suite/tests/mresources.rc
new file mode 100755
index 000000000..0fd5dda4f
--- /dev/null
+++ b/suite/tests/mresources.rc
@@ -0,0 +1,83 @@
+// **********************************************************
+// Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+// **********************************************************/
+
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// 
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+// 
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the following disclaimer in the documentation
+//   and/or other materials provided with the distribution.
+// 
+// * Neither the name of VMware, Inc. nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+// DAMAGE.
+
+// Copyright (c) 2006-2007 Determina Corp.
+
+// resources.rc
+
+#include "winres.h"
+
+#ifndef BUILD_NUMBER
+# define BUILD_NUMBER 0
+#endif
+
+#ifndef VERSION_COMMA_DELIMITED
+# define VERSION_COMMA_DELIMITED 4,2,0
+#endif
+
+#define EXPANDSTR(x) #x
+#define STRINGIFY(x) EXPANDSTR(x)
+
+#define VERSION_DOT_STRING STRINGIFY(VERSION_COMMA_DELIMITED)
+#define BUILD_NUMBER_STR STRINGIFY(BUILD_NUMBER)
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Version
+//
+
+LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
+
+VS_VERSION_INFO VERSIONINFO
+ FILEVERSION VERSION_COMMA_DELIMITED,BUILD_NUMBER
+ PRODUCTVERSION VERSION_COMMA_DELIMITED,BUILD_NUMBER
+ FILEFLAGSMASK 0x3fL
+ FILEFLAGS 0x0L
+ FILEOS 0x4L
+ FILETYPE VFT_DLL
+ FILESUBTYPE 0x0L
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+        BLOCK "040904b0"
+        BEGIN
+// CHECK: make sure it is OK not to put the terminating null here
+#define ASCIIZ // original contents was in the form "string \0"
+            VALUE "CompanyName", "Microsoft Corporation" ASCIIZ
+            VALUE "FileDescription", "Regression Test (not actually Microsoft)" ASCIIZ
+            VALUE "LegalCopyright", "Copyright (C) Determina, Inc. 2003-2006" ASCIIZ
+            VALUE "FileVersion", VERSION_DOT_STRING "," BUILD_NUMBER_STR ASCIIZ
+        END
+    END
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Translation", 0x409, 1200
+    END
+END
diff --git a/suite/tests/oresources.rc b/suite/tests/oresources.rc
new file mode 100755
index 000000000..73e947d53
--- /dev/null
+++ b/suite/tests/oresources.rc
@@ -0,0 +1,83 @@
+// **********************************************************
+// Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+// **********************************************************/
+
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// 
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+// 
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the following disclaimer in the documentation
+//   and/or other materials provided with the distribution.
+// 
+// * Neither the name of VMware, Inc. nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+// DAMAGE.
+
+// Copyright (c) 2006-2007 Determina Corp.
+
+// resources.rc
+
+#include "winres.h"
+
+#ifndef BUILD_NUMBER
+# define BUILD_NUMBER 0
+#endif
+
+#ifndef VERSION_COMMA_DELIMITED
+# define VERSION_COMMA_DELIMITED 4,2,0
+#endif
+
+#define EXPANDSTR(x) #x
+#define STRINGIFY(x) EXPANDSTR(x)
+
+#define VERSION_DOT_STRING STRINGIFY(VERSION_COMMA_DELIMITED)
+#define BUILD_NUMBER_STR STRINGIFY(BUILD_NUMBER)
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Version
+//
+
+LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
+
+VS_VERSION_INFO VERSIONINFO
+ FILEVERSION VERSION_COMMA_DELIMITED,BUILD_NUMBER
+ PRODUCTVERSION VERSION_COMMA_DELIMITED,BUILD_NUMBER
+ FILEFLAGSMASK 0x3fL
+ FILEFLAGS 0x0L
+ FILEOS 0x4L
+ FILETYPE VFT_DLL
+ FILESUBTYPE 0x0L
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+        BLOCK "040904b0"
+        BEGIN
+// CHECK: make sure it is OK not to put the terminating null here
+#define ASCIIZ // original contents was in the form "string \0"
+            VALUE "CompanyName", "Microblob Spam Corp" ASCIIZ
+            VALUE "FileDescription", "Regression Test (not actually Microblob)" ASCIIZ
+            VALUE "LegalCopyright", "Copyright (C) Determina, Inc. 2003-2006" ASCIIZ
+            VALUE "FileVersion", VERSION_DOT_STRING "," BUILD_NUMBER_STR ASCIIZ
+        END
+    END
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Translation", 0x409, 1200
+    END
+END
diff --git a/suite/tests/probe-api/insert_liboffs.c b/suite/tests/probe-api/insert_liboffs.c
new file mode 100644
index 000000000..da9f67fbf
--- /dev/null
+++ b/suite/tests/probe-api/insert_liboffs.c
@@ -0,0 +1,46 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+int doubler(int num)
+{
+    return num*2;
+}
+
+int main (void)
+{
+    int i = 4;
+    int j = doubler(i);
+    printf("The whole is greater than the sum of parts: ");
+    printf("%d + %d = %d\n", i, i, j);
+    if (j == i*2)
+        printf(" - apparently not!\n");
+}
diff --git a/suite/tests/probe-api/insert_liboffs.client.c b/suite/tests/probe-api/insert_liboffs.client.c
new file mode 100644
index 000000000..d59f7faea
--- /dev/null
+++ b/suite/tests/probe-api/insert_liboffs.client.c
@@ -0,0 +1,165 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* This client registers a probe with library offset. */
+
+/* To prevent cl from complaining about calling standard c library routine. */
+#define _CRT_SECURE_NO_WARNINGS
+#define _CRT_SECURE_NO_DEPRECATE
+
+#include "dr_api.h"
+#include "dr_probe.h"
+#include "dr_tools.h"
+
+#define NUM_PROBES 1
+dr_probe_desc_t probes[NUM_PROBES];
+
+/*----------------------------------------------------------------------------*/
+#include "dr_defines.h"
+
+/* This probe increments the argument to insert_liboffs.c:doubler() */
+__declspec(dllexport)
+void doubler_probe(dr_mcontext_t *cxt)
+{
+    volatile reg_t *arg_p = (reg_t *)(cxt->xsp + sizeof(reg_t));
+    *arg_p = (*arg_p) + 1;
+}
+/*----------------------------------------------------------------------------*/
+/* TODO: Figure out a way to share this code across all probe-api tests.  Can't
+ * put it inside tools.c as it links against external libraries.  Ok for now as
+ * this is the only one test.
+ */
+extern unsigned long strtoul(const char *nptr, char **endptr, int base);
+/* Library offset has to be computed before the probe library is loaded
+ * into memory.  Reading it from the map file is one of the easiest ways to
+ * do it.
+ */
+unsigned int get_symbol_offset_from_map(const char *map_file, const char *symbol)
+{
+    const char *pref_addr_str = "Preferred load address is ";
+    unsigned int pref_base, sym_addr, offset = 0xdeadc0de;
+    ssize_t file_sz;
+    file_t fd = INVALID_FILE;
+    char *buf, *temp;
+
+    fd = dr_open_file(map_file, DR_FILE_READ);
+    if (fd == INVALID_FILE)
+        goto _get_module_offset_exit; 
+
+    /* This seems to be the easiest way to get the size of the file. */
+    if (!dr_file_seek(fd, 0, DR_SEEK_END))
+        goto _get_module_offset_exit; 
+    file_sz = (ssize_t) dr_file_tell(fd);
+    if (file_sz <= 0)
+        goto _get_module_offset_exit; 
+    if (!dr_file_seek(fd, 0, DR_SEEK_SET))
+        goto _get_module_offset_exit; 
+
+    /* Read the whole file. */
+    buf = dr_global_alloc(file_sz + 1);
+    if (buf == NULL)
+        goto _get_module_offset_exit; 
+    dr_read_file(fd, buf, file_sz);
+    buf[file_sz] = '\0';
+
+    /* Locate preferred base & symbol address. */
+    temp = strstr(buf, pref_addr_str);
+    if (temp != NULL)
+    {
+        pref_base = strtoul(temp + strlen(pref_addr_str), NULL, 16);
+        temp = strstr(buf, symbol);
+        if (temp != NULL)
+            sym_addr = strtoul(temp + strlen(symbol), NULL, 16);
+        offset = sym_addr - pref_base;
+    }
+
+    dr_global_free(buf, file_sz + 1);
+
+ _get_module_offset_exit:
+    if (fd != INVALID_FILE)
+        dr_close_file(fd);
+    return offset;
+}
+
+/* This routine builds the full map file path from the name of the client.
+ * Assumes that map files for the test_{dll,exe}, probe and client dlls are all
+ * in the same directory as the client, which is true for tests.
+ */
+unsigned int get_symbol_offset(const char *map_file, const char *symbol)
+{
+    unsigned long offset = 0xdeadc0de, client_path_len;
+    char *client_path = NULL, *map_path, *ptr;
+
+    client_path = (char *) dr_get_client_path();
+    if (client_path != NULL) {
+        client_path_len = strlen(client_path);
+        map_path = dr_global_alloc(client_path_len * 2);        /* be safe */
+        strcpy(map_path, client_path);
+        ptr = map_path + client_path_len;
+        while (*--ptr != '\\');             /* get the dirname */
+        strcpy(++ptr, map_file);
+    }
+
+    offset = get_symbol_offset_from_map(map_path, symbol);
+
+    if (map_path != NULL)
+        dr_global_free(map_path, client_path_len * 2);
+
+    return offset;
+}
+/*----------------------------------------------------------------------------*/
+static void probe_def_init(void)
+{
+    probes[0].name = "insert_liboffs.exe probe";
+
+    probes[0].insert_loc.type = DR_PROBE_ADDR_LIB_OFFS;
+    probes[0].insert_loc.lib_offs.library = "insert_liboffs.exe";
+    probes[0].insert_loc.lib_offs.offset =
+        get_symbol_offset("insert_liboffs.map", "_doubler");
+
+    probes[0].callback_func.type = DR_PROBE_ADDR_LIB_OFFS;
+    probes[0].callback_func.lib_offs.library = "../probe-api/insert_liboffs.client.dll";
+    /* probes[0].callback_func.lib_offs.library = "insert_liboffs.probe.dll"; */
+
+    probes[0].callback_func.lib_offs.offset =
+        get_symbol_offset("insert_liboffs.client.map", "_doubler_probe");
+}
+
+DR_EXPORT
+void dr_init()
+{
+    dr_probe_status_t stat;
+
+    probe_def_init();
+    dr_register_probes(&probes[0], NUM_PROBES);
+    dr_get_probe_status(probes[0].id, &stat);
+}
diff --git a/suite/tests/probe-api/insert_liboffs.template b/suite/tests/probe-api/insert_liboffs.template
new file mode 100644
index 000000000..3230123b3
--- /dev/null
+++ b/suite/tests/probe-api/insert_liboffs.template
@@ -0,0 +1,6 @@
+#if +CLIENT_INTERFACE +HOT_PATCHING_INTERFACE : +-probe_api --thin_client
+The whole is greater than the sum of parts: 4 + 4 = 10
+# else
+The whole is greater than the sum of parts: 4 + 4 = 8
+ - apparently not!
+#endif
diff --git a/suite/tests/pthreads/pthreads.c b/suite/tests/pthreads/pthreads.c
new file mode 100644
index 000000000..5266287c0
--- /dev/null
+++ b/suite/tests/pthreads/pthreads.c
@@ -0,0 +1,130 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Build with:
+ * gcc -o pthreads pthreads.c -lpthread -D_REENTRANT -I../lib -L../lib -ldynamo -ldl -lbfd -liberty
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <pthread.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+volatile double pi = 0.0;  /* Approximation to pi (shared) */
+pthread_mutex_t pi_lock;   /* Lock for above */
+volatile double intervals; /* How many intervals? */
+
+void *
+process(void *arg)
+{
+    char *id = (char *) arg;
+    register double width, localsum;
+    register int i;
+    register int iproc;
+
+#if VERBOSE
+    fprintf(stderr, "\tthread %s starting\n", id);
+#endif
+    iproc = (*((char *) arg) - '0');
+
+    /* Set width */
+    width = 1.0 / intervals;
+
+    /* Do the local computations */
+    localsum = 0;
+    for (i=iproc; i<intervals; i+=2) {
+	register double x = (i + 0.5) * width;
+	localsum += 4.0 / (1.0 + x * x);
+    }
+    localsum *= width;
+
+    /* Lock pi for update, update it, and unlock */
+    pthread_mutex_lock(&pi_lock);
+    pi += localsum;
+    pthread_mutex_unlock(&pi_lock);
+
+#if VERBOSE
+    fprintf(stderr, "\tthread %s exiting\n", id);
+#endif
+    return(NULL);
+}
+
+int
+main(int argc, char **argv)
+{
+    pthread_t thread0, thread1;
+    void * retval;
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+#if 0
+    /* Get the number of intervals */
+    if (argc != 2) {
+	fprintf(stderr, "Usage: %s <intervals>\n", argv[0]);
+	exit(0);
+    }
+    intervals = atoi(argv[1]);
+#else /* for batch mode */
+    intervals = 10;
+#endif
+
+    /* Initialize the lock on pi */
+    pthread_mutex_init(&pi_lock, NULL);
+
+    /* Make the two threads */
+    if (pthread_create(&thread0, NULL, process, "0") ||
+	pthread_create(&thread1, NULL, process, "1")) {
+	fprintf(stderr, "%s: cannot make thread\n", argv[0]);
+	exit(1);
+    }
+    
+    /* Join (collapse) the two threads */
+    if (pthread_join(thread0, &retval) ||
+	pthread_join(thread1, &retval)) {
+	fprintf(stderr, "%s: thread join failed\n", argv[0]);
+	exit(1);
+    }
+
+    /* Print the result */
+    printf("Estimation of pi is %16.15f\n", pi);
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
diff --git a/suite/tests/pthreads/pthreads.expect b/suite/tests/pthreads/pthreads.expect
new file mode 100644
index 000000000..aa6787193
--- /dev/null
+++ b/suite/tests/pthreads/pthreads.expect
@@ -0,0 +1 @@
+Estimation of pi is 3.142425985001098
diff --git a/suite/tests/pthreads/ptsig.c b/suite/tests/pthreads/ptsig.c
new file mode 100644
index 000000000..fada4aa53
--- /dev/null
+++ b/suite/tests/pthreads/ptsig.c
@@ -0,0 +1,197 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Build with:
+ * gcc -o pthreads pthreads.c -lpthread -D_REENTRANT -I../lib -L../lib -ldynamo -ldl -lbfd -liberty
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <pthread.h>
+#include <signal.h>
+#include <ucontext.h>
+#include <assert.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+/* handler with SA_SIGINFO flag set gets three arguments: */
+typedef void (*handler_t)(int, struct siginfo *, void *);
+
+volatile double pi = 0.0;  /* Approximation to pi (shared) */
+pthread_mutex_t pi_lock;   /* Lock for above */
+volatile double intervals; /* How many intervals? */
+
+static void
+signal_handler(int sig, siginfo_t *siginfo, ucontext_t *ucxt)
+{
+#if VERBOSE
+    fprintf(stderr, "thread %d signal_handler: sig=%d, retaddr="PFX", fpregs="PFX"\n",
+	    getpid(), sig, *(&sig - 1), ucxt->uc_mcontext.fpregs);
+#endif
+
+    switch (sig) {
+    case SIGUSR1: {
+	struct sigcontext *sc = (struct sigcontext *) &(ucxt->uc_mcontext);
+	void *pc = (void *) sc->eip;
+#if VERBOSE
+	fprintf(stderr, "thread %d got SIGUSR1 @ "PFX"\n", getpid(), pc);
+#endif
+        break;
+    }
+    default:
+	assert(0);
+    }
+}
+
+/* set up signal_handler as the handler for signal "sig" */
+static void
+intercept_signal(int sig, handler_t handler)
+{
+    int rc;
+    struct sigaction act;
+
+    act.sa_sigaction = handler;
+#if BLOCK_IN_HANDLER
+    rc = sigfillset(&act.sa_mask); /* block all signals within handler */
+#else
+    rc = sigemptyset(&act.sa_mask); /* no signals are blocked within handler */
+#endif
+    assert(rc == 0);
+    act.sa_flags = SA_SIGINFO | SA_ONSTACK; /* send 3 args to handler */
+    
+    /* arm the signal */
+    rc = sigaction(sig, &act, NULL);
+    assert(rc == 0);
+}
+
+void *
+process(void *arg)
+{
+    char *id = (char *) arg;
+    register double width, localsum;
+    register int i;
+    register int iproc;
+
+#if VERBOSE
+    fprintf(stderr, "thread %s starting\n", id);
+#endif
+    if (((char *)arg)[0] == '1') {
+	intercept_signal(SIGUSR1, (handler_t) SIG_IGN);
+#if VERBOSE
+	fprintf(stderr, "thread %d ignoring SIGUSR1\n", getpid());
+#endif
+    }
+#if VERBOSE
+    fprintf(stderr, "thread %d sending SIGUSR1\n", getpid());
+#endif
+    kill(getpid(), SIGUSR1);
+
+    iproc = (*((char *) arg) - '0');
+
+    /* Set width */
+    width = 1.0 / intervals;
+
+    /* Do the local computations */
+    localsum = 0;
+    for (i=iproc; i<intervals; i+=2) {
+	register double x = (i + 0.5) * width;
+	localsum += 4.0 / (1.0 + x * x);
+    }
+    localsum *= width;
+
+    /* Lock pi for update, update it, and unlock */
+    pthread_mutex_lock(&pi_lock);
+    pi += localsum;
+    pthread_mutex_unlock(&pi_lock);
+
+#if VERBOSE
+    fprintf(stderr, "thread %s exiting\n", id);
+#endif
+    return(NULL);
+}
+
+int
+main(int argc, char **argv)
+{
+    pthread_t thread0, thread1;
+    void * retval;
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+#if 0
+    /* Get the number of intervals */
+    if (argc != 2) {
+	fprintf(stderr, "Usage: %s <intervals>\n", argv[0]);
+	exit(0);
+    }
+    intervals = atoi(argv[1]);
+#else /* for batch mode */
+    intervals = 10;
+#endif
+
+    /* Initialize the lock on pi */
+    pthread_mutex_init(&pi_lock, NULL);
+
+    intercept_signal(SIGUSR1, (handler_t) signal_handler);
+
+    /* Make the two threads */
+    if (pthread_create(&thread0, NULL, process, "0") ||
+	pthread_create(&thread1, NULL, process, "1")) {
+	fprintf(stderr, "%s: cannot make thread\n", argv[0]);
+	exit(1);
+    }
+    
+    /* Join (collapse) the two threads */
+    if (pthread_join(thread0, &retval) ||
+	pthread_join(thread1, &retval)) {
+	fprintf(stderr, "%s: thread join failed\n", argv[0]);
+	exit(1);
+    }
+
+#if VERBOSE
+    fprintf(stderr, "thread %d sending SIGUSR1\n", getpid());
+#endif
+    kill(getpid(), SIGUSR1);
+
+    /* Print the result */
+    printf("Estimation of pi is %16.15f\n", pi);
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
diff --git a/suite/tests/pthreads/ptsig.expect b/suite/tests/pthreads/ptsig.expect
new file mode 100644
index 000000000..aa6787193
--- /dev/null
+++ b/suite/tests/pthreads/ptsig.expect
@@ -0,0 +1 @@
+Estimation of pi is 3.142425985001098
diff --git a/suite/tests/runall/calc-detach.runall b/suite/tests/runall/calc-detach.runall
new file mode 100755
index 000000000..9bd990bf7
--- /dev/null
+++ b/suite/tests/runall/calc-detach.runall
@@ -0,0 +1,3 @@
+calc.exe
+Calculator
+<detach>
diff --git a/suite/tests/runall/calc-detach.template b/suite/tests/runall/calc-detach.template
new file mode 100755
index 000000000..e3e925707
--- /dev/null
+++ b/suite/tests/runall/calc-detach.template
@@ -0,0 +1,19 @@
+#if +RUNREGRESSION_XP +PROGRAM_SHEPHERDING : +-security +-no_executable_if_rx_text --executable_after_load
+# SEC_VIO_STOP
+No such process found.
+No such process found.
+Window not found: Calculator
+#else
+# if +DEBUG :
+Process calc.exe, running SC debug
+# else
+#  if +PROFILE :
+Process calc.exe, running SC profile
+#  else
+Process calc.exe, running SC release
+#  endif
+# endif
+Process calc.exe, running native
+Close message sent.
+#endif
+No such process found.
diff --git a/suite/tests/runall/calc-freeze.runall b/suite/tests/runall/calc-freeze.runall
new file mode 100755
index 000000000..6b07b163c
--- /dev/null
+++ b/suite/tests/runall/calc-freeze.runall
@@ -0,0 +1,3 @@
+calc.exe
+Calculator
+<freeze>
diff --git a/suite/tests/runall/calc-freeze.template b/suite/tests/runall/calc-freeze.template
new file mode 100755
index 000000000..c6b86832c
--- /dev/null
+++ b/suite/tests/runall/calc-freeze.template
@@ -0,0 +1,27 @@
+#if +RUNREGRESSION_XP +PROGRAM_SHEPHERDING : +-security +-no_executable_if_rx_text --executable_after_load
+# SEC_VIO_STOP
+No such process found.
+No such process found.
+Window not found: Calculator
+#else
+# if +DEBUG :
+Process calc.exe, running SC debug
+# else
+#  if +PROFILE :
+Process calc.exe, running SC profile
+#  else
+Process calc.exe, running SC release
+#  endif
+# endif
+#if +DEBUG :
+Process calc.exe, running SC debug
+#else
+# if +PROFILE :
+Process calc.exe, running SC profile
+# else
+Process calc.exe, running SC release
+# endif
+#endif
+Close message sent.
+#endif
+No such process found.
diff --git a/suite/tests/runall/calc-hotp.runall b/suite/tests/runall/calc-hotp.runall
new file mode 100755
index 000000000..9f5ba83dc
--- /dev/null
+++ b/suite/tests/runall/calc-hotp.runall
@@ -0,0 +1,3 @@
+calc.exe
+Calculator
+<hotp>
diff --git a/suite/tests/runall/calc-hotp.template b/suite/tests/runall/calc-hotp.template
new file mode 100755
index 000000000..c6b86832c
--- /dev/null
+++ b/suite/tests/runall/calc-hotp.template
@@ -0,0 +1,27 @@
+#if +RUNREGRESSION_XP +PROGRAM_SHEPHERDING : +-security +-no_executable_if_rx_text --executable_after_load
+# SEC_VIO_STOP
+No such process found.
+No such process found.
+Window not found: Calculator
+#else
+# if +DEBUG :
+Process calc.exe, running SC debug
+# else
+#  if +PROFILE :
+Process calc.exe, running SC profile
+#  else
+Process calc.exe, running SC release
+#  endif
+# endif
+#if +DEBUG :
+Process calc.exe, running SC debug
+#else
+# if +PROFILE :
+Process calc.exe, running SC profile
+# else
+Process calc.exe, running SC release
+# endif
+#endif
+Close message sent.
+#endif
+No such process found.
diff --git a/suite/tests/runall/calc-persist.runall b/suite/tests/runall/calc-persist.runall
new file mode 100755
index 000000000..e18ebcd6a
--- /dev/null
+++ b/suite/tests/runall/calc-persist.runall
@@ -0,0 +1,3 @@
+calc.exe
+Calculator
+<persist>
diff --git a/suite/tests/runall/calc-persist.template b/suite/tests/runall/calc-persist.template
new file mode 100755
index 000000000..e6ebc372d
--- /dev/null
+++ b/suite/tests/runall/calc-persist.template
@@ -0,0 +1,34 @@
+#if +RUNREGRESSION_XP +PROGRAM_SHEPHERDING : +-security +-no_executable_if_rx_text --executable_after_load
+# SEC_VIO_STOP
+No such process found.
+No such process found.
+Window not found: Calculator
+#else
+# if +DEBUG :
+Process calc.exe, running SC debug
+# else
+#  if +PROFILE :
+Process calc.exe, running SC profile
+#  else
+Process calc.exe, running SC release
+#  endif
+# endif
+#if : +-no_coarse_units || : +-client || : +-low || : +-hotp_only || : +-thin_client 
+No persisted caches found
+#endif
+#if +DEBUG :
+Process calc.exe, running SC debug
+Process calc.exe, running SC debug
+#else
+# if +PROFILE :
+Process calc.exe, running SC profile
+Process calc.exe, running SC profile
+# else
+Process calc.exe, running SC release
+Process calc.exe, running SC release
+# endif
+#endif
+Close message sent.
+Close message sent.
+#endif
+No such process found.
diff --git a/suite/tests/runall/calc-reset.runall b/suite/tests/runall/calc-reset.runall
new file mode 100755
index 000000000..ce43f0b16
--- /dev/null
+++ b/suite/tests/runall/calc-reset.runall
@@ -0,0 +1,3 @@
+calc.exe
+Calculator
+<reset>
diff --git a/suite/tests/runall/calc-reset.template b/suite/tests/runall/calc-reset.template
new file mode 100755
index 000000000..c6b86832c
--- /dev/null
+++ b/suite/tests/runall/calc-reset.template
@@ -0,0 +1,27 @@
+#if +RUNREGRESSION_XP +PROGRAM_SHEPHERDING : +-security +-no_executable_if_rx_text --executable_after_load
+# SEC_VIO_STOP
+No such process found.
+No such process found.
+Window not found: Calculator
+#else
+# if +DEBUG :
+Process calc.exe, running SC debug
+# else
+#  if +PROFILE :
+Process calc.exe, running SC profile
+#  else
+Process calc.exe, running SC release
+#  endif
+# endif
+#if +DEBUG :
+Process calc.exe, running SC debug
+#else
+# if +PROFILE :
+Process calc.exe, running SC profile
+# else
+Process calc.exe, running SC release
+# endif
+#endif
+Close message sent.
+#endif
+No such process found.
diff --git a/suite/tests/runall/calc.runall b/suite/tests/runall/calc.runall
new file mode 100644
index 000000000..0c6d953b3
--- /dev/null
+++ b/suite/tests/runall/calc.runall
@@ -0,0 +1,3 @@
+calc.exe
+Calculator
+<>
diff --git a/suite/tests/runall/calc.template b/suite/tests/runall/calc.template
new file mode 100755
index 000000000..8d529c5df
--- /dev/null
+++ b/suite/tests/runall/calc.template
@@ -0,0 +1,17 @@
+#if +RUNREGRESSION_XP +PROGRAM_SHEPHERDING : +-security +-no_executable_if_rx_text --executable_after_load
+# SEC_VIO_STOP
+No such process found.
+Window not found: Calculator
+#else
+# if +DEBUG :
+Process calc.exe, running SC debug
+# else
+#  if +PROFILE :
+Process calc.exe, running SC profile
+#  else
+Process calc.exe, running SC release
+#  endif
+# endif
+Close message sent.
+#endif
+No such process found.
diff --git a/suite/tests/runall/detach_test.c b/suite/tests/runall/detach_test.c
new file mode 100644
index 000000000..7900260f6
--- /dev/null
+++ b/suite/tests/runall/detach_test.c
@@ -0,0 +1,327 @@
+/* **********************************************************
+ * Copyright (c) 2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Tests detaching from threads in a variety of states. */
+
+#include <windows.h>
+#include "tools.h"
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#define PROC_NAME "detach_test.exe"
+#define MAX_COUNT 10
+
+static HWND hwnd = NULL;
+static BOOL thread_ready = FALSE;
+
+void CALLBACK SendAsyncProc(HWND hwnd, UINT uMsg, ULONG *dwData, LRESULT lResult);
+void do_test(int count);
+
+int do_busy_work(int c)
+{
+    int i, t;
+    for (i = 0, t = 0; i < c; i++)
+        t = (i * c) - t;
+    if (t != c*((c+1)/2))
+        print("Failure %d != %d\n", t, c*((c+1)/2));
+    return t;
+}
+
+/* detach from in cache thread */
+static BOOL in_busy_work = FALSE;
+static BOOL exit_busy_work = FALSE;
+DWORD WINAPI
+ThreadProcBusyWork(LPVOID param)
+{
+    print("Starting busy work\n");
+    in_busy_work = TRUE;
+    while (!exit_busy_work)
+        do_busy_work(20);
+    in_busy_work = FALSE;
+    exit_busy_work = FALSE;
+    print("Done busy working\n");
+    return 0;
+}
+
+/* detach from actively building thread */
+static BOOL in_busy_build = FALSE;
+static BOOL exit_busy_build = FALSE;
+#define BUF_LEN 0x1000
+DWORD WINAPI
+ThreadProcBusyBuild(LPVOID param)
+{
+    char *buf = VirtualAlloc(NULL, BUF_LEN, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
+    /* use selfmod to keep building */
+    copy_to_buf(buf, BUF_LEN, NULL, CODE_SELF_MOD, COPY_NORMAL);
+    print("Starting busy build\n");
+    in_busy_build = TRUE;
+    while (!exit_busy_build) {
+        /* don't do more than sandbox2ro_threshold iters (20) to avoid case 9908
+         * triggered resets (xref 10036 too) leading to hangs from detach at same time
+         * (case 8492). We're only trying to test code creation here anyways.
+         * Using 2 different values so there's a real change with the write. */
+        test(buf, 4);
+        test(buf, 5);
+    }
+    VirtualFree(buf, 0, MEM_RELEASE);
+    in_busy_build = FALSE;
+    exit_busy_build = FALSE;
+    print("Done busy building\n");
+    return 0;
+}
+
+/* see win32/tls.c test for alt. method of starting a detach, this way is preferable
+ * since we may at some point disallow the process from detaching itself */
+void detach()
+{
+    char buf[2*MAX_PATH] = "\"";
+    char *tools = getenv("DYNAMORIO_WINTOOLS");
+    int size;
+    STARTUPINFO sinfo = {sizeof(sinfo), NULL, "", 0, 0, 0, 0, 0, 0, 0, 0,
+                         STARTF_USESTDHANDLES, 0, 0, NULL,
+                         GetStdHandle(STD_INPUT_HANDLE),
+                         GetStdHandle(STD_OUTPUT_HANDLE),
+                         GetStdHandle(STD_ERROR_HANDLE)};
+    PROCESS_INFORMATION pinfo;
+    strncat(buf, tools == NULL ? "" : tools, BUFFER_ROOM_LEFT(buf));
+    NULL_TERMINATE_BUFFER(buf);
+    strncat(buf, "\\DRcontrol.exe\" -detachexe "PROC_NAME, BUFFER_ROOM_LEFT(buf));
+    NULL_TERMINATE_BUFFER(buf);
+    print("Detaching\n");
+    if (CreateProcess(NULL, buf, NULL, NULL, TRUE, 0, NULL, NULL, &sinfo, &pinfo)) {
+        /* This thread will be detached at non-dr intercepted syscall (the wait below)
+         * with multiple stacked callbacks (from SendAsyncProc). */
+        WaitForSingleObject(pinfo.hProcess, INFINITE);
+        CloseHandle(pinfo.hProcess);
+        CloseHandle(pinfo.hThread);
+    } else {
+        print("Detach Failed!\n");
+    }
+}
+
+static BOOL did_send_callback[MAX_COUNT];
+static BOOL action_detach = FALSE;
+static BOOL action_exit = FALSE;
+void CALLBACK SendAsyncProc(HWND hwnd, UINT uMsg, ULONG *dwData, LRESULT lResult) {
+    int count = (int)dwData;
+    did_send_callback[count] = TRUE;
+    if (count > 0) {
+        do_test(count - 1);
+    } else {
+        if (action_detach) {
+            HANDLE ht_busy_build, ht_busy_work;
+            DWORD tid;
+            ht_busy_work = CreateThread(NULL, 0, &ThreadProcBusyWork, NULL, 0, &tid);
+            while (!in_busy_work)
+                Sleep(10);
+            ht_busy_build = CreateThread(NULL, 0, &ThreadProcBusyBuild, NULL, 0, &tid);
+            while (!in_busy_build)
+                Sleep(10);
+            detach();
+            print("Detach finished\n");
+            exit_busy_build = TRUE;
+            WaitForSingleObject(ht_busy_build, INFINITE);
+            exit_busy_work = TRUE;
+            WaitForSingleObject(ht_busy_work, INFINITE);
+            CloseHandle(ht_busy_build);
+            CloseHandle(ht_busy_work);
+        } else if (action_exit) {
+            print("Exiting with stacked callbacks\n");
+            ExitThread(0);
+        }
+    }
+}
+
+/* Some website claimed that EnumWindows was done by a kernel callback, but it's not
+ * (on XP at least).  SendMessageCallback does, however, use a kernel callback for its
+ * callback (interestingly not just any alertable system call will do, has to be
+ * a message system call for the callback to be delivered). So we nest those to build
+ * up a callback stack. This routine will end up being recursively called count times
+ * building up count stacked callbacks. */
+#define MAX_SLEEP 30000
+void do_test(int count)
+{
+    MSG msg;
+    int total_slept = 0;
+
+    did_send_callback[count] = FALSE;
+    if (!SendMessageCallback(hwnd, WM_NULL, 0, 0, &SendAsyncProc, count)) {
+        print("SendMsg failed.\n");
+        return;
+    }
+
+    while (!did_send_callback[count] && total_slept < MAX_SLEEP) {
+        Sleep(100);
+        total_slept += 100;
+        /* FIXME - all callbacks will have the same after address, need to mix in a
+         * GetMessage or some such so can verify order of callback stack.  */
+        PeekMessage(&msg, NULL, 0, 0, PM_NOREMOVE);
+    }
+    if (total_slept >= MAX_SLEEP) {
+        print("Callback never delivered.\n");
+    }
+}
+
+DWORD WINAPI
+ThreadProcDoTest(LPVOID param)
+{
+    do_test((int)param);
+    print("Finished test (Not Reached)\n");
+}
+
+/* detach from self suspended thread (also covers thread at dr intercepted syscall) */
+DWORD WINAPI
+ThreadProcSelfSuspend(LPVOID param)
+{
+    SuspendThread(GetCurrentThread());
+    print("SuspendSelf resumed, exiting\n");
+    return 0;
+}
+
+LRESULT CALLBACK
+wnd_callback(HWND hwnd, UINT message, WPARAM wParam, LPARAM lParam)
+{
+    return DefWindowProc(hwnd, message, wParam, lParam);
+}
+
+int WINAPI
+window_func(void * arg)
+{
+    MSG msg;
+    char *winName = "foobar";
+    WNDCLASS wndclass = {0, wnd_callback, 0, 0, NULL/* WinMain hwnd would be here */,
+                         NULL, NULL, NULL, NULL, winName};
+    
+    if (!RegisterClass(&wndclass)) {
+        print("Unable to create window class\n");
+        return 0;
+    }
+    hwnd = CreateWindow(winName, winName, 0, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT,
+                        CW_USEDEFAULT, NULL, NULL, NULL/* WinMain hwnd would be here */,
+                        NULL);
+    if (hwnd == NULL) {
+        print("Error creating window\n");
+        return 0;
+    }
+    /* deliberately not calling ShowWindow */
+
+    thread_ready = true;
+    while (true) {
+        if (GetMessage(&msg, NULL, 0, 0)) {
+            /* Messages not auto-sent to callbacks are processed here */
+            TranslateMessage(&msg); /* convert virtual-key msgs to character msgs */
+            DispatchMessage(&msg);
+        }
+    }
+    return msg.wParam;
+}
+
+int
+main()
+{
+    HANDLE ht_selfsuspend, ht_exit, ht_window;
+    DWORD tid, res;
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    INIT();
+
+    print("creating window\n");
+    ht_window = CreateThread(NULL, 0, window_func, NULL, 0, &tid);
+    if (ht_window == NULL) { 
+        print("Error creating window thread\n");
+        return -1;
+    }
+    while (!thread_ready)
+        Sleep(20);
+
+    print("detach_callback start\n");
+
+    ht_selfsuspend = CreateThread(NULL, 0, &ThreadProcSelfSuspend, NULL, 0, &tid);
+    if (ht_selfsuspend == NULL) { 
+        print("Error creating self-suspend thread\n");
+        return -1;
+    }
+    /* wait for thread to suspend itself */
+    res = 0;
+    while (res == 0) {
+        res = SuspendThread(ht_selfsuspend);
+        if (res == 0) {
+            /* Thread might not yet have gotten around to suspending itself */
+            ResumeThread(ht_selfsuspend);
+            /* short sleep to wait */
+            Sleep(20);
+        }
+    }
+
+    do_test(2);
+    print("finished first callback test\n");
+
+    action_exit = TRUE;
+    ht_exit = CreateThread(NULL, 0, &ThreadProcDoTest, (void *)2, 0, &tid);
+    if (ht_exit == NULL) { 
+        print("Error creating exit thread\n");
+        return -1;
+    }
+    WaitForSingleObject(ht_exit, INFINITE);
+    CloseHandle(ht_exit);
+    print("finished exit test\n");
+    action_exit = FALSE;
+
+    action_detach = TRUE;
+    do_test(2);
+    print("finished detach test\n");
+    action_detach = FALSE;
+    /* we are now detached */
+
+    /* just a little extra work to make sure everything looks ok natively */
+    do_test(1);
+    print("finished second callback test\n");
+
+    /* verify selfsuspended thread detached okay */
+    ResumeThread(ht_selfsuspend);
+    ResumeThread(ht_selfsuspend);
+    WaitForSingleObject(ht_selfsuspend, INFINITE);
+    CloseHandle(ht_selfsuspend);
+
+    print("detach_callback done\n");
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
diff --git a/suite/tests/runall/detach_test.expect b/suite/tests/runall/detach_test.expect
new file mode 100755
index 000000000..23e44d24f
--- /dev/null
+++ b/suite/tests/runall/detach_test.expect
@@ -0,0 +1,15 @@
+creating window
+detach_callback start
+finished first callback test
+Exiting with stacked callbacks
+finished exit test
+Starting busy work
+Starting busy build
+Detaching
+Detach finished
+Done busy building
+Done busy working
+finished detach test
+finished second callback test
+SuspendSelf resumed, exiting
+detach_callback done
diff --git a/suite/tests/runall/detach_test.runall b/suite/tests/runall/detach_test.runall
new file mode 100755
index 000000000..b1fc7339b
--- /dev/null
+++ b/suite/tests/runall/detach_test.runall
@@ -0,0 +1,3 @@
+<path>/detach_test.exe
+<nowindow>
+<>
diff --git a/suite/tests/runall/earlythread.c b/suite/tests/runall/earlythread.c
new file mode 100644
index 000000000..e009409be
--- /dev/null
+++ b/suite/tests/runall/earlythread.c
@@ -0,0 +1,255 @@
+/* **********************************************************
+ * Copyright (c) 2006-2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* FIXME: nothing here tests that a child is under DR */
+/* case 9385 test */
+/* based on runall/processchain.c */
+/* should compare with win32/threadinjection.c */
+
+/* TODO: need to test this both with -early_inject and without so that
+ * we can exercise the way we are running winlogon.exe and maybe other
+ * interrupted chain services
+ */
+
+/* TWEAKS:
+ * depth (controlled by STRESS=1 or PERF=1)
+ * roundrobin 
+ * extra_threads
+ * sleep_under_ldrlock in earlythread.dll.c
+ */
+
+#include "tools.h"
+#include "Windows.h"
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#define VERBOSE 0
+#define VERY_VERBOSE 0
+
+/* undefine this for a performance test */
+
+#ifdef NIGHTLY_REGRESSION
+#  define DEPTH 5
+#else
+# ifdef PERF
+#  define DEPTH 10              /* PERF */
+# else 
+#  define DEPTH 100             /* STRESS */
+# endif
+#endif
+
+static int depth = DEPTH;
+
+/* whether we'll forcefully change which thread goes first */
+/* note that as seen with case 9467 Win2003 does allow at least
+ * CtrlRoutine threads to start earlier than the primary thread, so it
+ * may be a possible natural order as well.
+ */
+int roundrobin = 1;
+
+int extra_threads = 5;
+
+static int 
+main(int argc, char **argv) 
+{
+    STARTUPINFO si = { sizeof(STARTUPINFO) };
+    PROCESS_INFORMATION pi;
+    HANDLE hThread_child;
+    char cmdline[128];
+    DWORD result = 0;
+
+    INIT();
+    USE_USER32();               /* can't be in runall otherwise! */
+    
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    if (argc == 1) {
+        /* normal execution */
+
+        _snprintf(cmdline, sizeof(cmdline), "%s %d", argv[0], depth);
+
+        print("starting chain %d...\n", depth);
+    }
+
+    if (argc == 2) {
+        depth = atoi(argv[1]);
+#if VERY_VERBOSE
+        /* this is non-deterministic with respect to second thread execution */
+        print("subprocess %d running.\n", depth);
+#endif
+
+        _snprintf(cmdline, sizeof(cmdline), "%s %d", argv[0], depth-1);
+        /* this thread could do some work or just sleep a little */
+        Sleep(10);
+#if VERBOSE
+        print("done some sleeping\n");
+#endif
+    }
+
+    if (depth != 0) {
+
+        /* FIXME: may want to add CREATE_NEW_PROCESS_GROUP | CREATE_SUSPENDED
+         * so we can send GenerateConsoleCtrlEvent(CTRL_BREAK_EVENT, child_group)
+         * and see if that ever gets delivered before the first process thread in any 
+         * grandchild
+         */
+
+        if (!CreateProcess(argv[0],
+                           cmdline,
+                           NULL,
+                           NULL,
+                           FALSE,
+                           CREATE_SUSPENDED,
+                           NULL,
+                           NULL,
+                           &si,
+                           &pi)) 
+            print("CreateProcess failure\n"); 
+        else {
+            DWORD exit_code = 0;
+
+            DWORD proc_affinity;
+            DWORD sys_affinity;
+            
+            static const char* mylib = "earlythread.dll.dll";
+            /* note we're too lazy to allocate this string in the
+             * child, but if exe is not ASLRed in child we should be fine
+             */
+
+            /* creating a thread to LoadLibrary */
+           
+            hThread_child = 
+                CreateRemoteThread(pi.hProcess, 0, 0, &LoadLibrary, (void*)mylib, 
+                                   CREATE_SUSPENDED, NULL);
+
+            if (hThread_child == NULL)
+            {
+                print("Error in CreateRemoteThread(Code %d)\n",GetLastError());
+            }
+
+#if PROCAFFINITY /* case 4181 - can't be set to 0 */
+            if (!GetProcessAffinityMask(pi.hProcess, &proc_affinity, &sys_affinity)) {
+                print("GetProcessAffinityMask() failure %d\n", GetLastError());
+            }
+#if VERBOSE
+            print("GetProcessAffinityMask = %x %x\n", proc_affinity, sys_affinity);
+#endif
+
+            if (!SetProcessAffinityMask(pi.hProcess, 0)) {
+                print("SetProcessAffinityMask(0) failure %d\n", GetLastError()); 
+            }
+
+            /* restore original */
+            if (!SetProcessAffinityMask(pi.hProcess, proc_affinity)) {
+                print("SetProcessAffinityMask(original) failure %d\n", GetLastError()); 
+                /* 
+                 * 1:001> !error c000000d 
+                 * Error code: (NTSTATUS) 0xc000000d (3221225485) - An invalid parameter was passed to a service or function.
+                 */
+            }
+#endif /* PROCAFFINITY */
+
+            /* FIXME: should play around with the order of these */
+            /* resume initial thread */
+            if (!roundrobin || (depth % 2) == 0) { /* even rounds primary goes first */
+                result = ResumeThread(pi.hThread);
+                if (result == -1) {
+                    print("ResumeThread primary thread failure %d\n", GetLastError()); 
+                }
+
+                /* FIXME: should play around with extra sleep if too
+                 * deterministic otherwise */
+                Sleep(depth*10);
+            } else {
+                /* we'll wake up after injected thread */
+            }
+
+            result = ResumeThread(hThread_child);
+            if (result == -1) {
+                print("ResumeThread second thread failure %d\n", GetLastError()); 
+            }
+
+            if (roundrobin && 
+                ((depth % 2) == 1)) { /* odd rounds injected thread goes first */
+                /* FIXME: may want to be able to disable this */
+                Sleep(depth*10);
+
+                result = ResumeThread(pi.hThread);
+                if (result == -1) {
+                    print("ResumeThread primary thread failure %d\n", GetLastError()); 
+                }
+            }
+
+            while (extra_threads-- < 0) {
+                /* inject a few more threads for kicks  */
+                HANDLE extra_child = 
+                    CreateRemoteThread(pi.hProcess, 0, 0, &LoadLibrary, (void*)mylib, 
+                                       0 /* ready to run */, NULL);
+                /* do we really have to wait on them? no.. */
+#if WAITEXTRA
+                WaitForSingleObject(extra_child, INFINITE);
+#endif
+            } 
+
+            if (hThread_child != NULL){
+                WaitForSingleObject(hThread_child, INFINITE);
+                CloseHandle(hThread_child);
+#if VERBOSE
+                print("done with second thread\n"); 
+#endif
+            }
+
+#if VERBOSE
+            print("%d: waiting for child process\n", depth); 
+#endif
+            WaitForSingleObject(pi.hProcess, INFINITE);
+            if (!GetExitCodeProcess(pi.hProcess, &exit_code)) {
+                print("GetExitCodeProcess failure %d\n", GetLastError()); 
+            } else {
+                print("process returned %d\n", exit_code);
+            }
+        }
+    }
+    
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+
+    return depth * 10;
+    /* make sure not 259 STILL_ACTIVE which may be returned by GetExitCodeProcess() */
+}
diff --git a/suite/tests/runall/earlythread.dll.c b/suite/tests/runall/earlythread.dll.c
new file mode 100644
index 000000000..da075c3cf
--- /dev/null
+++ b/suite/tests/runall/earlythread.dll.c
@@ -0,0 +1,62 @@
+/* **********************************************************
+ * Copyright (c) 2006 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "tools.h"
+
+int sleep_under_ldrlock = 100;
+/* FIXME: it would be nice to get some work done outside the loader
+ * lock, but for that we may need to create a new thread (fishy!) that
+ * targets import_me (but we can't synchronize with such a new thread
+ * of course it will run only once we let go of the LdrLock)
+ */
+
+int  __declspec(dllexport)
+import_me(int x)
+{
+    print("in import\n");
+    return 2*x;
+}
+
+BOOL APIENTRY 
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    switch (reason_for_call) {
+    case DLL_PROCESS_ATTACH:
+        print("earlythreaddll.dll.dll process attach\n");
+        Sleep(sleep_under_ldrlock);
+        break;
+    case DLL_PROCESS_DETACH:
+        print("earlythreaddll.dll.dll process detach\n");
+        break;        
+    }
+    return TRUE;
+}
diff --git a/suite/tests/runall/earlythread.runall b/suite/tests/runall/earlythread.runall
new file mode 100644
index 000000000..e709f41c9
--- /dev/null
+++ b/suite/tests/runall/earlythread.runall
@@ -0,0 +1,3 @@
+<path>/earlythread.exe
+<nowindow>
+<>
diff --git a/suite/tests/runall/earlythread.template b/suite/tests/runall/earlythread.template
new file mode 100644
index 000000000..32f91ad6c
--- /dev/null
+++ b/suite/tests/runall/earlythread.template
@@ -0,0 +1,19 @@
+starting chain 5...
+earlythreaddll.dll.dll process attach
+earlythreaddll.dll.dll process attach
+earlythreaddll.dll.dll process attach
+earlythreaddll.dll.dll process attach
+earlythreaddll.dll.dll process attach
+earlythreaddll.dll.dll process attach
+earlythreaddll.dll.dll process detach
+process returned 0
+earlythreaddll.dll.dll process detach
+process returned 10
+earlythreaddll.dll.dll process detach
+process returned 20
+earlythreaddll.dll.dll process detach
+process returned 30
+earlythreaddll.dll.dll process detach
+process returned 40
+earlythreaddll.dll.dll process detach
+process returned 50
diff --git a/suite/tests/runall/initapc.c b/suite/tests/runall/initapc.c
new file mode 100644
index 000000000..f9802bc1c
--- /dev/null
+++ b/suite/tests/runall/initapc.c
@@ -0,0 +1,94 @@
+/* **********************************************************
+ * Copyright (c) 2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* initapc: tests having DllMain of a statically-linked dll send an APC
+ * (thus the APC is prior to the image entry point)
+ *
+ * must be run w/ AppInit injection, else the APC will be prior to DR taking over
+ *
+ * expect:
+ * w/ -native_exec_syscalls:
+ *   *** re-taking-over @INTERCEPT_SYSCALL after losing control ***
+ * w/ -no_native_exec_syscalls:
+ *   *** re-taking-over @INTERCEPT_EARLY_ASYNCH after losing control ***
+ * and, of course, to have no .C violations from bottoming out
+ *
+ * FIXME: add mechanism to get info from core w/o getting start/stop and
+ * w/o sending to eventlog since don't want customers to get even if
+ * ask for info events!
+ * Right now, since can't do DRview on test that completes right away,
+ * and aren't getting info from core, this test will pass even if it runs
+ * natively.
+ */
+#include <windows.h>
+#include "tools.h"
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+/* from initapc-dll.dll */
+__declspec(dllimport)
+import_me(int x);
+
+int
+main()
+{
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    INIT();
+
+    print("initapc main()\n");
+    import_me(37);
+
+    /* ensure still checking ret-after-call after stack bottom issues w/ APC 
+     * use a nop between push and ret to avoid VB push/ret pattern match
+     */
+    __asm {
+        push offset next_instr
+        nop
+        ret
+    next_instr:
+        nop
+    }
+
+    print("*** invalid ret allowed!\n");
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
diff --git a/suite/tests/runall/initapc.dll.c b/suite/tests/runall/initapc.dll.c
new file mode 100644
index 000000000..efee008db
--- /dev/null
+++ b/suite/tests/runall/initapc.dll.c
@@ -0,0 +1,85 @@
+/* **********************************************************
+ * Copyright (c) 2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* initapc.dll: tests having DllMain of a statically-linked dll send an APC
+ * (thus the APC is prior to the image entry point)
+ * (the APC code is borrowed from the older test winapc.c)
+ */
+#define _WIN32_WINNT 0x0400
+#include <windows.h>
+#include <winbase.h> /* for QueueUserAPC */
+#include "tools.h"
+
+void __declspec(dllexport)
+import_me(int x)
+{
+    print("initapc.dll:import_me(%d)\n", x);
+}
+
+static int result = 0;
+static int apc_arg = 0;
+
+static void WINAPI
+apc_func(unsigned long arg)
+{
+    result += 100;
+    apc_arg = arg;
+}
+
+static void
+send_apc()
+{
+    int res = QueueUserAPC(apc_func, GetCurrentThread(), 37);
+    print("QueueUserAPC returned %d\n", res);
+    /* an alertable system call so we receive the APC */
+    res = SleepEx(100, 1);
+    /* is going to return 192 since recieved apc during sleep call 
+     * well technically 192 is io completion interruption, but seems to 
+     * report that for any interrupting APC */
+    print("SleepEx returned %d\n", res);
+    print("Apc arg = %d\n", apc_arg);
+    print("Result = %d\n", result);
+}
+
+BOOL APIENTRY 
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    switch (reason_for_call) {
+    case DLL_PROCESS_ATTACH:
+        /* make sure we import user32.dll, for AppInit injection */
+        GetParent(NULL);
+        print("initapc.dll:DllMain()\n");
+        send_apc();
+        break;
+    }
+    return TRUE;
+}
diff --git a/suite/tests/runall/initapc.runall b/suite/tests/runall/initapc.runall
new file mode 100644
index 000000000..38c3f30f5
--- /dev/null
+++ b/suite/tests/runall/initapc.runall
@@ -0,0 +1,3 @@
+<path>/initapc.exe
+<nowindow>
+<>
diff --git a/suite/tests/runall/initapc.template b/suite/tests/runall/initapc.template
new file mode 100755
index 000000000..3b8b95b28
--- /dev/null
+++ b/suite/tests/runall/initapc.template
@@ -0,0 +1,18 @@
+initapc.dll:DllMain()
+QueueUserAPC returned 1
+SleepEx returned 192
+Apc arg = 37
+Result = 100
+initapc main()
+initapc.dll:import_me(37)
+#if +PROGRAM_SHEPHERDING : +-security --no_ret_after_call --C --X --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+*** invalid ret allowed!
diff --git a/suite/tests/runall/notepad.runall b/suite/tests/runall/notepad.runall
new file mode 100644
index 000000000..787417df8
--- /dev/null
+++ b/suite/tests/runall/notepad.runall
@@ -0,0 +1,3 @@
+notepad.exe
+Untitled - Notepad
+<>
diff --git a/suite/tests/runall/notepad.template b/suite/tests/runall/notepad.template
new file mode 100755
index 000000000..fcac15856
--- /dev/null
+++ b/suite/tests/runall/notepad.template
@@ -0,0 +1,17 @@
+#if +RUNREGRESSION_XP +PROGRAM_SHEPHERDING : +-security +-no_executable_if_rx_text --executable_after_load
+# SEC_VIO_STOP
+No such process found.
+Window not found: Untitled - Notepad
+#else
+# if +DEBUG :
+Process notepad.exe, running SC debug
+# else
+#  if +PROFILE :
+Process notepad.exe, running SC profile
+#  else
+Process notepad.exe, running SC release
+#  endif
+# endif
+Close message sent.
+#endif
+No such process found.
diff --git a/suite/tests/runall/preunload.c b/suite/tests/runall/preunload.c
new file mode 100644
index 000000000..4ae708609
--- /dev/null
+++ b/suite/tests/runall/preunload.c
@@ -0,0 +1,137 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* preunload: ensures that drpreinject.dll is unloaded
+ */
+#include <windows.h>
+#include "tools.h"
+
+#define PREINJECT_NAME "drpreinject.dll"
+#define PREINJECT_BASE 0x14000000
+
+#define INJECT_ALL_HIVE    HKEY_LOCAL_MACHINE
+#define INJECT_ALL_KEY     "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows"
+#define INJECT_ALL_SUBKEY  "AppInit_DLLs"
+
+/* two different checks: GetModuleFileNameA and VirtualQuery */
+BOOL
+ensure_no_preinject()
+{
+    int len;
+    char name[MAX_PATH];
+    MEMORY_BASIC_INFORMATION mbi;
+
+    len = GetModuleFileNameA((HINSTANCE) PREINJECT_BASE, name,
+                             (sizeof(name)/sizeof(name[0])));
+    if (len > 0) {
+        print("ERROR: found module %s at "PFX"\n", name, PREINJECT_BASE);
+        return FALSE;
+    }
+    
+    len = VirtualQuery((void *)PREINJECT_BASE, &mbi, sizeof(mbi));
+    if (len != sizeof(mbi)) {
+        print("ERROR: error querying "PFX"\n", PREINJECT_BASE);
+        return FALSE;
+    } else {
+        if (mbi.State != MEM_FREE) {
+            print("ERROR: "PFX" is not MEM_FREE!\n", PREINJECT_BASE);
+            return FALSE;
+        }
+    }
+    return TRUE;
+}
+
+BOOL
+load_preinject()
+{
+    HKEY hk;
+    HMODULE hlib;
+    char val[MAX_PATH];
+    int res, len = MAX_PATH * sizeof(val[0]);
+
+    res = RegOpenKeyEx(INJECT_ALL_HIVE, INJECT_ALL_KEY, 0, 
+                       KEY_READ, &hk);
+    if (res != ERROR_SUCCESS)
+        return FALSE;
+
+    res = RegQueryValueEx(hk, INJECT_ALL_SUBKEY, 0, 
+                          0, (LPBYTE) val, &len);
+    RegCloseKey(hk);
+    if (res != ERROR_SUCCESS)
+        return FALSE;
+
+    if (val[0] == '\0') {
+        print("ERROR: no preinject library set\n");
+        return FALSE;
+    }
+    print("loading in preinject library\n", val);
+    hlib = LoadLibrary(val);
+    if (hlib == NULL) {
+        /* with the old preinject, this returns NULL b/c its DllMain
+         * returns FALSE -- but the new preinject will return success
+         */
+        if (GetLastError() == ERROR_DLL_INIT_FAILED) {
+            print("DLL init routine failed -- are you using an old %s?\n",
+                  PREINJECT_NAME);
+        }
+    }
+    return (hlib != NULL);
+}
+
+int
+main()
+{
+    HANDLE hlib;
+    INIT();
+
+    print("preunload main()\n");
+
+    /* use something from user32 so that AppInit injection works */
+    print("using user32: %d\n", IsCharAlpha('4'));
+
+    /* check #1: preinject should be gone */
+    if (ensure_no_preinject())
+        print("%s not found\n", PREINJECT_NAME);
+
+    /* check #2: re-loading preinject shouldn't cause problems, and it
+     * should get unloaded again -- except with a debug build preinject
+     * we'll get a popup warning about double injection!  so we only
+     * run with release build preinjects, which is the case for any
+     * clean-slate runs, like our nightly regressions.
+     */
+    if (!load_preinject())
+        print("failed to load %s\n", PREINJECT_NAME);
+    if (ensure_no_preinject())
+        print("%s not found\n", PREINJECT_NAME);
+
+    return 0;
+}
diff --git a/suite/tests/runall/preunload.runall b/suite/tests/runall/preunload.runall
new file mode 100755
index 000000000..7ac5d7309
--- /dev/null
+++ b/suite/tests/runall/preunload.runall
@@ -0,0 +1,3 @@
+<path>/preunload.exe
+<nowindow>
+<>
diff --git a/suite/tests/runall/preunload.template b/suite/tests/runall/preunload.template
new file mode 100755
index 000000000..8232e9318
--- /dev/null
+++ b/suite/tests/runall/preunload.template
@@ -0,0 +1,5 @@
+preunload main()
+using user32: 0
+drpreinject.dll not found
+loading in preinject library
+drpreinject.dll not found
diff --git a/suite/tests/runall/processchain.c b/suite/tests/runall/processchain.c
new file mode 100644
index 000000000..5c66da245
--- /dev/null
+++ b/suite/tests/runall/processchain.c
@@ -0,0 +1,115 @@
+/* **********************************************************
+ * Copyright (c) 2006 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* FIXME: nothing here tests that a child is under DR */
+#include "tools.h"
+#include "Windows.h"
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+/* undefine this for a performance test */
+#define NIGHTLY_REGRESSION
+
+#ifdef NIGHTLY_REGRESSION
+#  define DEPTH 5
+#else
+#  define DEPTH 10
+#endif
+
+static int depth = DEPTH;
+
+static int 
+main(int argc, char **argv) 
+{
+    STARTUPINFO si = { sizeof(STARTUPINFO) };
+    PROCESS_INFORMATION pi;
+    char cmdline[128];
+    DWORD result = 0;
+
+    INIT();
+    USE_USER32();               /* can't be in runall otherwise! */
+    
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    if (argc == 1) {
+        /* normal execution */
+
+        _snprintf(cmdline, sizeof(cmdline), "%s %d", argv[0], depth);
+
+        print("starting chain %d...\n", depth);
+
+    }
+
+    if (argc == 2) {
+        depth = atoi(argv[1]);
+        print("subprocess %d running.\n", depth);
+
+        _snprintf(cmdline, sizeof(cmdline), "%s %d", argv[0], depth-1);
+    }
+
+    if (depth != 0) {
+        if (!CreateProcess(argv[0],
+                           cmdline,
+                           NULL,
+                           NULL,
+                           FALSE,
+                           0,
+                           NULL,
+                           NULL,
+                           &si,
+                           &pi)) 
+            print("CreateProcess failure\n"); 
+        else {
+            DWORD exit_code = 0;
+            print("waiting for child\n"); 
+            WaitForSingleObject(pi.hProcess, INFINITE);
+            if (!GetExitCodeProcess(pi.hProcess, &exit_code)) {
+                print("GetExitCodeProcess failure %d\n", GetLastError()); 
+            } else {
+                print("process returned %d\n", exit_code);
+            }
+        }
+    }
+    
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+
+    return depth * 10;
+    /* make sure not 259 STILL_ACTIVE which may be returned by GetExitCodeProcess() */
+}
diff --git a/suite/tests/runall/processchain.runall b/suite/tests/runall/processchain.runall
new file mode 100644
index 000000000..2ac8efed5
--- /dev/null
+++ b/suite/tests/runall/processchain.runall
@@ -0,0 +1,3 @@
+<path>/processchain.exe
+<nowindow>
+<>
diff --git a/suite/tests/runall/processchain.template b/suite/tests/runall/processchain.template
new file mode 100755
index 000000000..696416084
--- /dev/null
+++ b/suite/tests/runall/processchain.template
@@ -0,0 +1,19 @@
+starting chain 5...
+waiting for child
+subprocess 5 running.
+waiting for child
+subprocess 4 running.
+waiting for child
+subprocess 3 running.
+waiting for child
+subprocess 2 running.
+waiting for child
+subprocess 1 running.
+waiting for child
+subprocess 0 running.
+process returned 0
+process returned 10
+process returned 20
+process returned 30
+process returned 40
+process returned 50
diff --git a/suite/tests/runall/runalltest.sh b/suite/tests/runall/runalltest.sh
new file mode 100755
index 000000000..2fba2847f
--- /dev/null
+++ b/suite/tests/runall/runalltest.sh
@@ -0,0 +1,231 @@
+#!/bin/bash
+
+# **********************************************************
+# Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+#notes: 
+#current DYNAMORIO_OPTIONS env var will be used to determine options
+#DYNAMORIO_TOOLS must be set
+
+usage="runalltest <runallfile> <drdll> <win32_drpreinject_path> <runall_max_wait (optional)>"
+
+# 3 seconds seems to be the magic number here. I've seen problems
+# with <freeze> tests for anything less.
+RUNALL_SLEEP=3
+
+RUNALL_MAX_WAIT=10
+if [ $# -ne 3 ] ; then
+if [ $# -ne 4 ] ; then
+    echo "Usage: $0 $usage"
+    exit 127
+else
+    RUNALL_MAX_WAIT=$4
+fi
+fi
+
+exepath=`head -1 $1`
+caption=`head -2 $1 | tail -1`
+mode=`head -3 $1 | tail -1`
+drdll=$2
+drpreinject=$3
+exename=${exepath##*/}
+if [ ${exepath:0:6} == "<path>" ]; then
+    # support for running executables not on system path, but in runall/ dir
+    runalldir=${1%/*} # assumption: .runall file is in same place as exe
+    exepath=${exepath/<path>/$runalldir}
+fi
+
+function nudge_exe {
+    pid=`$DYNAMORIO_TOOLS/DRview -exe $1 | gawk '{print $2}' | sed 's/,//'`
+    # case 10382: look for "No such process found", since if we
+    # pass a non-number to DRcontrol it will nudge all processes!
+    if [ "$pid" = "such" ]; then
+        echo "Error: $1 process not found for nudge"
+    else
+        $DYNAMORIO_TOOLS/DRcontrol $2 $pid
+    fi
+}
+
+
+#some tests need to be able to find the tools folder
+export DYNAMORIO_WINTOOLS=`cygpath -w $DYNAMORIO_TOOLS`
+
+# make sure the registry set is there (only creates if nonexistent).
+dr_home=`cygpath -da $PWD`
+$DYNAMORIO_TOOLS/DRcontrol -create "$dr_home"
+
+# save the old registry settings so we can leave the registry in the
+# same state we found it.
+prev_preinject=`$DYNAMORIO_TOOLS/DRcontrol -preinject REPORT`
+prev_settings=_prev_settings
+$DYNAMORIO_TOOLS/DRcontrol -save $prev_settings
+
+$DYNAMORIO_TOOLS/DRcontrol -preinject OFF
+$DYNAMORIO_TOOLS/DRcontrol -app "$exename" -run 1 -options "$DYNAMORIO_OPTIONS"  -drlib $drdll
+$DYNAMORIO_TOOLS/DRcontrol -preinject $drpreinject
+$DYNAMORIO_TOOLS/DRcontrol -app "$exename" -logdir "$DYNAMORIO_LOGDIR"
+
+# clear pcache dir before starting app and then create the registry
+# entry and associated dir
+if [ -e cache ]; then
+    # remove existing perscache dir(s)
+    rm -rf cache;
+fi
+
+# we have a discrepancy between DYNAMORIO_CACHE_ROOT, which drinject
+# uses, and DRcontrol -sharedcache.  The former specifies the full
+# path to the cache dir, but the latter specifies the path to DR_HOME
+# (i.e., without a trailing /cache).  We do this so that DRcontrol can
+# find the lib and logs directory and copy their permissions.  
+# FIXME: What a hack.  Can't we just set the permissions directly
+# and make DRcontrol -sharedcache more intuitive?
+cache_root=`dirname "$DYNAMORIO_CACHE_ROOT"`
+
+# drcontrol -sharedcache gives an error message if it can't copy
+# permissions from the lib and logs directories (e.g., if they don't
+# exist).  Swallow the error message so it doesn't mess up the output.
+junk=`$DYNAMORIO_TOOLS/DRcontrol -app "$exename" -sharedcache "$cache_root" 2>&1`
+
+# for 3rd party apps w/ windows, we use DRview results to ensure we're in control
+if [ "$caption" != "<nowindow>" ]; then
+    $exepath &
+
+    sleep $RUNALL_SLEEP
+        
+    $DYNAMORIO_TOOLS/DRcontrol -preinject OFF
+
+    $DYNAMORIO_TOOLS/DRview -exe $exename -nopid -nobuildnum
+
+    if [ "$mode" == "<detach>" ]; then
+        $DYNAMORIO_TOOLS/DRcontrol -detachexe $exename
+
+        # Currently DRcontrol -detachexe waits for the injected detach thread
+        # to exit at which point should be finished detaching, so we don't need
+        # to sleep here. DRcontrol does time out if wait is too long so we
+        # won't get stuck here.
+
+        $DYNAMORIO_TOOLS/DRview -exe $exename -nopid -nobuildnum
+
+    elif [ "$mode" == "<reset>" ]; then
+        # test two in a row
+        nudge_exe $exename "-nudge reset -pid";
+        nudge_exe $exename "-nudge reset -pid";
+
+        # Currently DRcontrol -nudge waits for the injected nudge thread to
+        # exit, so we don't need to sleep here. DRcontrol does time out if
+        # wait is too long so we won't get stuck here.
+        $DYNAMORIO_TOOLS/DRview -exe $exename -nopid -nobuildnum
+
+    elif [ "$mode" == "<hotp>" ]; then
+        # test both kinds of hotp nudge (even if no patches match)
+        nudge_exe $exename -hot_patch_nudge;
+        nudge_exe $exename -hot_patch_modes_nudge;
+
+        # Currently DRcontrol -nudge waits for the injected nudge thread to
+        # exit, so we don't need to sleep here. DRcontrol does time out if
+        # wait is too long so we won't get stuck here.
+        $DYNAMORIO_TOOLS/DRview -exe $exename -nopid -nobuildnum
+
+    elif [ "$mode" == "<freeze>" ]; then
+        # test two in a row
+        nudge_exe $exename "-nudge freeze -pid";
+        nudge_exe $exename "-nudge freeze -pid";
+
+        # Currently DRcontrol -nudge waits for the injected nudge thread to
+        # exit, so we don't need to sleep here. DRcontrol does time out if
+        # wait is too long so we won't get stuck here.
+        $DYNAMORIO_TOOLS/DRview -exe $exename -nopid -nobuildnum
+
+    elif [ "$mode" == "<persist>" ]; then
+        # we now have per-user and not per-app dirs
+        # we'll use the 'whoami' tool in tools/external/ to get the SID
+        cursid=`$DYNAMORIO_TOOLS/external/whoami.exe /user | tail -1 | gawk '{print $2}'`
+        cachedir="$cache_root/cache/$cursid"
+
+        # test persisting and re-using
+        nudge_exe $exename "-nudge persist -pid";
+
+        if [ ! -d $cachedir ]; then
+            echo "No persisted caches found";
+        fi
+
+        # run a second copy
+        $DYNAMORIO_TOOLS/DRcontrol -preinject $drpreinject
+        $exepath &
+        sleep $RUNALL_SLEEP
+        $DYNAMORIO_TOOLS/DRcontrol -preinject OFF
+        # we assume that it's using the persisted caches
+        # FIXME: add way to find out
+
+        # Currently DRcontrol -nudge waits for the injected nudge thread to
+        # exit, so we don't need to sleep here. DRcontrol does time out if
+        # wait is too long so we won't get stuck here.
+        $DYNAMORIO_TOOLS/DRview -exe $exename -nopid -nobuildnum
+
+        $DYNAMORIO_TOOLS/closewnd "$caption" $RUNALL_MAX_WAIT
+    elif [ "$mode" == "<client_nudge>" ]; then
+        # send a client nudge, we pick arbitrary value 10 for argument
+        nudge_exe $exename "-client_nudge 10 -pid";
+
+        sleep $RUNALL_SLEEP
+
+        # Currently DRcontrol -nudge waits for the injected nudge thread to
+        # exit, so we don't need to sleep here. DRcontrol does time out if
+        # wait is too long so we won't get stuck here.
+        $DYNAMORIO_TOOLS/DRview -exe $exename -nopid -nobuildnum
+    fi
+
+    $DYNAMORIO_TOOLS/closewnd "$caption" $RUNALL_MAX_WAIT
+
+    sleep $RUNALL_SLEEP
+
+    $DYNAMORIO_TOOLS/DRview -exe $exename -nobuildnum
+else
+    $DYNAMORIO_TOOLS/winstats -m 2 -silent $exepath
+
+    $DYNAMORIO_TOOLS/DRcontrol -preinject OFF
+fi
+
+#just in case
+$DYNAMORIO_TOOLS/DRkill -exe $exename -quiet
+
+# restore registry settings
+$DYNAMORIO_TOOLS/DRcontrol -remove "$exename"
+
+if [[ ${#prev_preinject} > 0 ]]; then
+    $DYNAMORIO_TOOLS/DRcontrol -preinject "$prev_preinject"
+else
+    $DYNAMORIO_TOOLS/DRcontrol -preinject OFF
+fi
+
+if [ -e $prev_settings ]; then
+    $DYNAMORIO_TOOLS/DRcontrol -load $prev_settings
+    rm -f $prev_settings
+fi
diff --git a/suite/tests/security-common/TestAllocWE.c b/suite/tests/security-common/TestAllocWE.c
new file mode 100644
index 000000000..6661ddb5a
--- /dev/null
+++ b/suite/tests/security-common/TestAllocWE.c
@@ -0,0 +1,71 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "tools.h"
+
+#define BUF_LEN 160
+
+int
+main()
+{
+    /* allocate buffer */
+    char *buf = allocate_mem(BUF_LEN, ALLOW_WRITE|ALLOW_EXEC);
+    INIT();
+
+#if USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    print("starting up\n");
+
+    copy_to_buf(buf, BUF_LEN, NULL, CODE_INC, COPY_NORMAL);
+    
+    test_print(buf, 1);
+
+    copy_to_buf(buf, BUF_LEN, NULL, CODE_DEC, COPY_NORMAL);
+
+    test_print(buf, 1);
+    test_print(buf, 2);
+
+    copy_to_buf(buf, BUF_LEN, NULL, CODE_SELF_MOD, COPY_NORMAL);
+
+    test_print(buf, 4660);
+    test_print(buf, 43981);
+
+    print("about to exit\n");
+
+#if USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
diff --git a/suite/tests/security-common/TestAllocWE.template b/suite/tests/security-common/TestAllocWE.template
new file mode 100755
index 000000000..859579f62
--- /dev/null
+++ b/suite/tests/security-common/TestAllocWE.template
@@ -0,0 +1,39 @@
+starting up
+#if -PROGRAM_SHEPHERDING : 
+2
+0
+1
+4660
+43981
+#else 
+# if : +-exectutable_heap
+2
+0
+1
+4660
+43981
+# else 
+#  if : +-no_executable_if_alloc
+#   if : --detect_mode
+#    SEC_VIO_AUTO_STOP
+#    STOP
+#   else
+#    SEC_VIO_CONT
+2
+#    SEC_VIO_CONT
+0  
+1
+#    SEC_VIO_CONT
+4660
+43981
+#   endif
+#  else
+2
+0  
+1
+4660
+43981
+#  endif
+# endif
+#endif
+about to exit
diff --git a/suite/tests/security-common/TestMemProtChg.c b/suite/tests/security-common/TestMemProtChg.c
new file mode 100644
index 000000000..c694f1e93
--- /dev/null
+++ b/suite/tests/security-common/TestMemProtChg.c
@@ -0,0 +1,102 @@
+/* **********************************************************
+ * Copyright (c) 2003-2004 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "tools.h"
+
+#define BUFFER_SIZE 3*PAGE_SIZE
+
+static char buffer[BUFFER_SIZE];
+
+static int prot_codes[7] = {ALLOW_READ, ALLOW_READ|ALLOW_EXEC, ALLOW_WRITE, ALLOW_READ|ALLOW_WRITE, ALLOW_READ|ALLOW_WRITE|ALLOW_EXEC, ALLOW_WRITE|ALLOW_EXEC, ALLOW_EXEC};
+
+static void 
+do_test(char *buf, int len)
+{
+    int i, j, copied_len, cur;
+    char *code;
+    protect_mem(buf, len, ALLOW_READ|ALLOW_WRITE);
+    for (i = 0; i < 7; i++) {
+        for (j = 0; j < 7; j++) {
+            code = copy_to_buf(buf, len, NULL, CODE_INC, COPY_NORMAL); 
+            protect_mem_check(buf, len, prot_codes[i], ALLOW_READ|ALLOW_WRITE);
+            protect_mem_check(buf, len, prot_codes[j], prot_codes[i]);
+            test_print(code, 5);
+            test_print(code, 2);
+            if (j > 1 && j < 6) {
+                code = copy_to_buf(buf, len, NULL, CODE_DEC, COPY_NORMAL);
+                test_print(code, 3);
+                test_print(code, 1);
+                code = copy_to_buf(buf, len, NULL, CODE_SELF_MOD, COPY_NORMAL);
+                test_print(code, 43981);
+                test_print(code, 4660);
+            }
+            buf++;
+            len--;
+            if (j > 1 && j < 6) {
+                protect_mem_check(buf, len, ALLOW_READ|ALLOW_WRITE, prot_codes[j]);
+                code = copy_to_buf(buf, len, NULL, CODE_SELF_MOD, COPY_NORMAL);
+                protect_mem_check(buf, len, prot_codes[i], ALLOW_READ|ALLOW_WRITE);
+                protect_mem_check(buf, len, prot_codes[j], prot_codes[i]);
+                test_print(code, 4660);
+                test_print(code, 43981);
+            }
+            protect_mem_check(buf, len, ALLOW_READ|ALLOW_WRITE, prot_codes[j]);
+        }
+    }
+}
+
+int
+main()
+{
+    /* get aligned test buffers */
+    char buffer_stack[BUFFER_SIZE];
+
+    char *buf = page_align(buffer);
+    char *buf_stack = page_align(buffer_stack);
+    INIT();
+    
+#if USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    print("starting up\n");
+    do_test(buf, 2*PAGE_SIZE);
+    print("starting stack tests\n");
+    do_test(buf_stack, 2*PAGE_SIZE);
+    print("about to exit\n");
+
+#if USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
diff --git a/suite/tests/security-common/TestMemProtChg.template b/suite/tests/security-common/TestMemProtChg.template
new file mode 100755
index 000000000..ed24955ff
--- /dev/null
+++ b/suite/tests/security-common/TestMemProtChg.template
@@ -0,0 +1,1462 @@
+starting up
+#if -PROGRAM_SHEPHERDING : || +PROGRAM_SHEPHERDING : --security || +PROGRAM_SHEPHERDING : +-security +-internal_detach_mask +0x1 || +PROGRAM_SHEPHERDING : +-executable_heap || +PROGRAM_SHEPHERDING : +-thin_client || +PROGRAM_SHEPHERDING : +-client || +PROGRAM_SHEPHERDING : +-low || +PROGRAM_SHEPHERDING : +-use_moduledb
+# if +PROGRAM_SHEPHERDING : +-security +-internal_detach_mask +0x1 --low --client --thin_client --use_moduledb
+#  SEC_VIO_STOP
+# endif
+6
+3
+6
+3
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+6
+3
+6
+3
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+6
+3
+6
+3
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+6
+3
+6
+3
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+6
+3
+6
+3
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+6
+3
+6
+3
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+6
+3
+6
+3
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+#else 
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  STOP
+# endif
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+43981
+4660
+# SEC_VIO_CONT
+4660
+43981
+# SEC_VIO_CONT
+6
+3
+#endif
+starting stack tests
+#if -PROGRAM_SHEPHERDING : || +PROGRAM_SHEPHERDING : --security || +PROGRAM_SHEPHERDING : +-security +-internal_detach_mask +0x1 || +PROGRAM_SHEPHERDING : +-executable_stack || +PROGRAM_SHEPHERDING : +-thin_client || +PROGRAM_SHEPHERDING : +-client || +PROGRAM_SHEPHERDING : +-low
+6
+3
+6
+3
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+6
+3
+6
+3
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+6
+3
+6
+3
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+6
+3
+6
+3
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+6
+3
+6
+3
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+6
+3
+6
+3
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+6
+3
+6
+3
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+2
+0
+43981
+4660
+4660
+43981
+6
+3
+#else 
+# if  : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  STOP
+# endif
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+# SEC_VIO_CONT
+2
+0
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+# SEC_VIO_CONT
+4660
+# SEC_VIO_CONT
+43981
+# SEC_VIO_CONT
+6
+3
+#endif
+about to exit
diff --git a/suite/tests/security-common/codemod.c b/suite/tests/security-common/codemod.c
new file mode 100644
index 000000000..1cb16df03
--- /dev/null
+++ b/suite/tests/security-common/codemod.c
@@ -0,0 +1,73 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "tools.h"
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#define ON_STACK 1
+
+#if !ON_STACK
+char buf[32];
+#endif
+
+int
+main()
+{
+#if ON_STACK
+    char buf[32];
+#endif
+    char *foo = buf;
+
+    INIT();
+#if USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    print("starting up\n");
+
+    buf[0] = 0xc3; /* ret */
+
+    ((void (*)(void)) foo)();
+
+    print("about to exit\n");
+
+    buf[1] = 0xc3; /* ret */
+
+#if USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
diff --git a/suite/tests/security-common/codemod.template b/suite/tests/security-common/codemod.template
new file mode 100755
index 000000000..2705a70f8
--- /dev/null
+++ b/suite/tests/security-common/codemod.template
@@ -0,0 +1,12 @@
+starting up
+#if +PROGRAM_SHEPHERDING : +-security --executable_stack --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+about to exit
diff --git a/suite/tests/security-common/decode-bad-stack.c b/suite/tests/security-common/decode-bad-stack.c
new file mode 100644
index 000000000..36c73c0e8
--- /dev/null
+++ b/suite/tests/security-common/decode-bad-stack.c
@@ -0,0 +1,159 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifndef ASM_CODE_ONLY /* C code */
+#include <assert.h>
+#include <stdio.h>
+#include <math.h>
+
+#ifdef LINUX
+# include <unistd.h>
+# include <signal.h>
+# include <ucontext.h>
+# include <errno.h>
+#endif
+
+#include "tools.h" /* for print() */
+
+/* just use single-arg handlers */
+typedef void (*handler_t)(int);
+typedef void (*handler_3_t)(int, struct siginfo *, void *);
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#define ITERS 1500000
+
+static int a[ITERS];
+
+#ifdef LINUX
+static void
+signal_handler(int sig)
+{
+    if (sig == SIGILL)
+        print("Got an illegal instruction\n");
+    abort();
+}
+
+#define ASSERT_NOERR(rc) do {					\
+  if (rc) {							\
+     fprintf(stderr, "%s:%d rc=%d errno=%d %s\n", 		\
+	     __FILE__, __LINE__,				\
+	     rc, errno, strerror(errno));  		        \
+  }								\
+} while (0);
+
+/* set up signal_handler as the handler for signal "sig" */
+static void
+intercept_signal(int sig, handler_t handler)
+{
+    int rc;
+    struct sigaction act;
+
+    act.sa_sigaction = (handler_3_t) handler;
+    rc = sigfillset(&act.sa_mask); /* block all signals within handler */
+    ASSERT_NOERR(rc);
+    /* FIXME: due to DR bug #654 we use SA_SIGINFO -- change it once DR works */
+    act.sa_flags = SA_SIGINFO | SA_ONSTACK;
+    
+    /* arm the signal */
+    rc = sigaction(sig, &act, NULL);
+    ASSERT_NOERR(rc);
+}
+
+#else
+/* sort of a hack to avoid the MessageBox of the unhandled exception spoiling
+ * our batch runs
+ */
+# include <windows.h>
+/* top-level exception handler */
+static LONG
+our_top_handler(struct _EXCEPTION_POINTERS * pExceptionInfo)
+{
+    if (pExceptionInfo->ExceptionRecord->ExceptionCode == EXCEPTION_ILLEGAL_INSTRUCTION)
+        print("Got an illegal instruction\n");
+# if VERBOSE
+    print("Exception occurred, process about to die silently\n");
+# endif
+    return EXCEPTION_EXECUTE_HANDLER; /* => global unwind and silent death */
+}
+#endif
+
+int main(int argc, char *argv[])
+{
+    double res = 0.;
+    int i,j;
+    unsigned char buf[8];
+    buf[0] = 0xdf;
+    buf[1] = 0xfa;
+
+    /* FIXME we don't call INIT here - template can't use SEC_VIO_AUTO_STOP */
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+  
+#ifdef LINUX
+    intercept_signal(SIGILL, signal_handler);
+#else
+    SetUnhandledExceptionFilter((LPTOP_LEVEL_EXCEPTION_FILTER) our_top_handler);
+#endif
+
+    print("Bad instr about to happen\n");
+
+    /* jmp to bad instr on stack */
+    jumpto(buf);
+
+    print("SHOULD NEVER GET HERE\n");
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
+
+#else /* asm code *************************************************************/
+#include "asm_defines.asm"
+START_FILE
+
+#define FUNCNAME jumpto
+        DECLARE_FUNC_SEH(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        jmp      ARG1
+        /* won't come back */
+        ret
+        END_FUNC(FUNCNAME)
+
+END_FILE
+#endif
diff --git a/suite/tests/security-common/decode-bad-stack.template b/suite/tests/security-common/decode-bad-stack.template
new file mode 100644
index 000000000..afb8f707d
--- /dev/null
+++ b/suite/tests/security-common/decode-bad-stack.template
@@ -0,0 +1,22 @@
+Bad instr about to happen
+#if +PROGRAM_SHEPHERDING : +-security --low --client --thin_client
+# if : --executable_stack
+#  if : --detect_mode
+#   if : -+throw_exception
+#    SEC_VIO_EXCEPTION
+#   else
+#     if : -+kill_thread
+#      SEC_VIO_THREAD
+#     else  
+#      SEC_VIO_STOP
+#     endif
+#   endif
+#   if : --internal_detach_mask
+#    STOP
+#   endif
+#  else
+#   SEC_VIO_CONT
+#  endif
+# endif
+#endif
+Got an illegal instruction
diff --git a/suite/tests/security-common/jmp_from_trace.c b/suite/tests/security-common/jmp_from_trace.c
new file mode 100644
index 000000000..11550a483
--- /dev/null
+++ b/suite/tests/security-common/jmp_from_trace.c
@@ -0,0 +1,88 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Generates a .B from a trace.
+ * FIXME: need to put in a template
+ * FIXME: need to make native work (exception handling stuff)
+ * FIXME: need to make it work on linux
+ * FIXME: need to make it work with -detect_mode
+ */
+
+#include "tools.h"
+ 
+#define NUM_TIMES 100
+#define SWITCH_AFTER 50
+
+/* totally random dummy function */
+int dummycall() {
+    int i = 1, j = 2;
+
+    if ((i += 10) < (j -= 50))
+        return i + j - 32;
+    else
+        return i - j + 32;
+}
+
+int badtarget = 10;  /* .B */
+int bad_trace4()
+{
+    int i, j;
+    int a = 1, b = 100, c = 1000;
+    int (*fnptr)();
+    fnptr = dummycall;
+    for (i = 0; i < NUM_TIMES; i++) {
+        for (j = 0; j < NUM_TIMES; j++) {
+            if (a < b) {
+                a++;
+                c--;
+                fnptr();
+            } else {
+                a -= 5;
+                c--;
+                fnptr();
+            }
+            if (c < SWITCH_AFTER) {
+                fnptr = (int (*)())&badtarget;
+                print("Next time around jump to data section\n");
+            }
+        }
+    }
+    return 0;
+}
+
+int main ()
+{
+  INIT();
+  print ("Start\n");
+  bad_trace4();
+  print("SHOULD NEVER GET HERE\n");
+}
diff --git a/suite/tests/security-common/jmp_from_trace.template b/suite/tests/security-common/jmp_from_trace.template
new file mode 100755
index 000000000..276505806
--- /dev/null
+++ b/suite/tests/security-common/jmp_from_trace.template
@@ -0,0 +1,13 @@
+Start
+Next time around jump to data section
+#if +PROGRAM_SHEPHERDING : +-security --ibl_table_per_type --no_ret_after_call --X --C --low --client --thin_client --use_moduledb
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask 
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+Unhandled exception caught.
diff --git a/suite/tests/security-common/ret_noncall_trace.c b/suite/tests/security-common/ret_noncall_trace.c
new file mode 100644
index 000000000..c68a37f0e
--- /dev/null
+++ b/suite/tests/security-common/ret_noncall_trace.c
@@ -0,0 +1,97 @@
+/* **********************************************************
+ * Copyright (c) 2004 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Description:	This test is intended to check if the -ibl_table_per_type 
+ *   option works, which is done by generating a bogus-return-address-security-
+ *   violation.  The nature of the violation is  such that it won't be caught
+ *   by DR unless the -ibl_table_per_type option is turned on.  The program
+ *   tries to return to a fragment that is classified as a tracehead and isn't
+ *   the return site of a call.
+ *   
+ * Assumption: This test program will always be compiled /Od on Windows and 
+ *   with default optimization on Linux.
+ *
+ * Notes: If the compilers change, their default optimization levels are 
+ *   changed or if this code is optimized, the offset 6, used in next_num() for 
+ *   saved_eip will change.  The change may be different on Windows and Linux.  
+ *   The offset value in the code has to be modified.
+ */
+
+#include "tools.h"
+ 
+#define NUM_TIMES 100
+#define INNER_LOOP_COUNT 4
+#define MAX_SUM (NUM_TIMES * (NUM_TIMES + 1) / 2 * INNER_LOOP_COUNT)
+
+static unsigned int saved_eip;
+
+int next_num (int max_val)
+{
+  static int counter;
+
+  counter++;
+  saved_eip = *(&max_val - 1);
+  saved_eip += 6;		/* Set rp to main()'s do-while loop. */
+  return counter;
+}
+
+int check_sum (int sum)
+{
+  *(&sum - 1) = saved_eip;
+
+  return 1;			/* Make the bogus transition here. */
+}
+
+int main ()
+{
+  int i, val, inner_loop, sum = 0;
+
+  INIT();
+
+  print ("I think, therefore I am\n");
+
+  for (i = 0; i < NUM_TIMES; i++)
+  {
+    inner_loop = INNER_LOOP_COUNT;
+    val = next_num(NUM_TIMES);		/* Get next_num() into a trace. */
+    do {
+      sum += val;
+      if (sum > MAX_SUM)
+      {
+	print (" ... in serious trouble!\n");
+	exit (-1);
+      }
+    } while (--inner_loop);
+  }
+
+  check_sum(sum);
+}
diff --git a/suite/tests/security-common/ret_noncall_trace.template b/suite/tests/security-common/ret_noncall_trace.template
new file mode 100644
index 000000000..d449d23fa
--- /dev/null
+++ b/suite/tests/security-common/ret_noncall_trace.template
@@ -0,0 +1,12 @@
+I think, therefore I am
+#if +PROGRAM_SHEPHERDING : +-security --ibl_table_per_type --no_ret_after_call --X --C --low --client --thin_client --use_moduledb
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask 
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+ ... in serious trouble!
diff --git a/suite/tests/security-common/retexisting.c b/suite/tests/security-common/retexisting.c
new file mode 100644
index 000000000..04f344b5a
--- /dev/null
+++ b/suite/tests/security-common/retexisting.c
@@ -0,0 +1,88 @@
+/* **********************************************************
+ * Copyright (c) 2003-2004 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "tools.h"
+
+int
+precious()
+{
+#ifdef USER32    /* map user32.dll for a RunAll test */
+    MessageBeep(0);
+#endif
+    print("M-m-my PRECIOUS is stolen! ATTACK SUCCESSFUL!\n");
+    exit(1);
+}
+
+int
+ring(int num)
+{
+    print("looking at ring\n");
+    *(int*) (&num - 1) = (int)&precious;
+    return num;
+}
+
+int
+twofoo()
+{
+    int a = foo();
+    print("first foo a=%d\n", a);
+
+    a += foo();
+    print("second foo a=%d\n", a);
+    return a;
+}
+
+int
+foo()
+{
+    print("in foo\n");
+    return 1;
+}
+
+int
+bar()
+{
+    print("in bar\n");
+    return 3;
+}
+
+int
+main()
+{
+    INIT();
+
+    print("starting good function\n");
+    twofoo();
+    print("starting bad function\n");
+    ring(1);
+    print("all done [not seen]\n");
+}
diff --git a/suite/tests/security-common/retexisting.template b/suite/tests/security-common/retexisting.template
new file mode 100644
index 000000000..b3a6ab0c8
--- /dev/null
+++ b/suite/tests/security-common/retexisting.template
@@ -0,0 +1,18 @@
+starting good function
+in foo
+first foo a=1
+in foo
+second foo a=2
+starting bad function
+looking at ring
+#if +PROGRAM_SHEPHERDING : +-security --no_ret_after_call --C --X --low --client --thin_client --use_moduledb
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+M-m-my PRECIOUS is stolen! ATTACK SUCCESSFUL!
diff --git a/suite/tests/security-common/retnonexisting.c b/suite/tests/security-common/retnonexisting.c
new file mode 100644
index 000000000..5c37673b3
--- /dev/null
+++ b/suite/tests/security-common/retnonexisting.c
@@ -0,0 +1,188 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "tools.h"
+#include <setjmp.h>
+
+#ifdef LINUX
+# include <unistd.h>
+# include <signal.h>
+# include <ucontext.h>
+# include <errno.h>
+#endif
+
+jmp_buf mark;
+int where; /* 0 = normal, 1 = segfault longjmp */
+
+int
+ring(int num)
+{
+    int addr;
+    print("looking at ring "PFX"\n", num);
+    *(int*) (&num - 1) = num;
+    return num;
+}
+
+int
+twofoo()
+{
+    int a = foo();
+    print("first foo a=%d\n", a);
+
+    a += foo();
+    print("second foo a=%d\n", a);
+    return a;
+}
+
+int
+foo()
+{
+    print("in foo\n");
+    return 1;
+}
+
+int
+bar()
+{
+    print("in bar\n");
+    return 3;
+}
+
+
+#ifdef LINUX
+/* just use single-arg handlers */
+typedef void (*handler_t)(int);
+typedef void (*handler_3_t)(int, struct siginfo *, void *);
+
+static void
+signal_handler(int sig)
+{
+    if (sig == SIGSEGV) {
+#if VERY_VERBOSE
+        print("Got seg fault\n");
+#endif
+        longjmp(mark, 1);
+    }
+    exit(-1);
+}
+
+#define ASSERT_NOERR(rc) do {                                   \
+  if (rc) {                                                     \
+     fprintf(stderr, "%s:%d rc=%d errno=%d %s\n",               \
+             __FILE__, __LINE__,                                \
+             rc, errno, strerror(errno));                       \
+  }                                                             \
+} while (0);
+
+/* set up signal_handler as the handler for signal "sig" */
+static void
+intercept_signal(int sig, handler_t handler)
+{
+    int rc;
+    struct sigaction act;
+
+    act.sa_sigaction = (handler_3_t) handler;
+    /* FIXME: due to DR bug 840 we cannot block ourself in the handler
+     * since the handler does not end in a sigreturn, so we have an empty mask
+     * and we use SA_NOMASK
+     */
+    rc = sigemptyset(&act.sa_mask); /* block no signals within handler */
+    ASSERT_NOERR(rc);
+    /* FIXME: due to DR bug #654 we use SA_SIGINFO -- change it once DR works */
+    act.sa_flags = SA_NOMASK | SA_SIGINFO | SA_ONSTACK;
+    
+    /* arm the signal */
+    rc = sigaction(sig, &act, NULL);
+    ASSERT_NOERR(rc);
+}
+
+#else
+/* sort of a hack to avoid the MessageBox of the unhandled exception spoiling
+ * our batch runs
+ */
+# include <windows.h>
+/* top-level exception handler */
+static LONG
+our_top_handler(struct _EXCEPTION_POINTERS * pExceptionInfo)
+{
+    if (pExceptionInfo->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
+#if VERY_VERBOSE
+        print("Got segfault\n");
+#endif
+        longjmp(mark, 1);
+    }
+# if VERBOSE
+    print("Exception occurred, process about to die silently\n");
+# endif
+    return EXCEPTION_EXECUTE_HANDLER; /* => global unwind and silent death */
+}
+#endif
+
+int
+invalid_ret(int num) 
+{
+    where = setjmp(mark);
+    if (where == 0) {
+        ring(num);
+        print("unexpectedly we came back!");
+    } else {
+        print("fault caught on "PFX"\n", num);
+    }
+}
+
+int
+main()
+{
+    INIT();
+
+#ifdef LINUX
+    intercept_signal(SIGSEGV, signal_handler);
+#else
+    SetUnhandledExceptionFilter((LPTOP_LEVEL_EXCEPTION_FILTER) our_top_handler);
+#endif
+
+    print("starting good function\n");
+    twofoo();
+    print("starting bad function\n");
+
+    invalid_ret(1);                    /* zero page */
+    /* FIXME: should wrap all of these in setjmp() blocks */
+    invalid_ret(0);                    /* NULL */
+    invalid_ret(0x00badbad);           /* user mode */
+    invalid_ret(0x7fffffff);           /* user mode */
+    invalid_ret(0x80000000);           /* kernel addr */
+    invalid_ret(0xbadbad00);           /* kernel addr */
+    invalid_ret(0xfffffffe);           /* just bad */
+    invalid_ret(0xffffffff);           /* just bad */
+
+    print("all done [not seen]\n");
+}
diff --git a/suite/tests/security-common/retnonexisting.template b/suite/tests/security-common/retnonexisting.template
new file mode 100644
index 000000000..21ec6fb58
--- /dev/null
+++ b/suite/tests/security-common/retnonexisting.template
@@ -0,0 +1,34 @@
+starting good function
+in foo
+first foo a=1
+in foo
+second foo a=2
+starting bad function
+looking at ring 0x00000001
+#if +PROGRAM_SHEPHERDING : +-security --no_ret_after_call --C --X --low --client --thin_client --detect_mode --throw_exception --internal_detach_mask --hotp_only
+// case 9364 - we silently kill (thread or process)
+#   STOP
+# else
+//match native behaviour
+#endif
+fault caught on 0x00000001
+looking at ring 0x00000000
+fault caught on 0x00000000
+looking at ring 0x00badbad
+#if +PROGRAM_SHEPHERDING : +-security +-throw_exception_max(1)
+// note we don't count the 0x00000000 as a .C exception
+// case 8267
+# STOP
+#endif
+fault caught on 0x00badbad
+looking at ring 0x7fffffff
+fault caught on 0x7fffffff
+looking at ring 0x80000000
+fault caught on 0x80000000
+looking at ring 0xbadbad00
+fault caught on 0xbadbad00
+looking at ring 0xfffffffe
+fault caught on 0xfffffffe
+looking at ring 0xffffffff
+fault caught on 0xffffffff
+all done [not seen]
diff --git a/suite/tests/security-common/selfmod-big.c b/suite/tests/security-common/selfmod-big.c
new file mode 100644
index 000000000..fe8e15974
--- /dev/null
+++ b/suite/tests/security-common/selfmod-big.c
@@ -0,0 +1,1957 @@
+/* **********************************************************
+ * Copyright (c) 2006 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Test having a selfmod bb with a ton of write instrs that creates a too-big
+ * selfmod fragment.  Xref case 7893.
+ */
+
+#include "tools.h"
+
+#ifdef USE_DYNAMO
+# include "dynamorio.h"
+#endif
+
+static uint big[1024];
+
+void
+foo(int iters)
+{
+    /* executes iters iters, by modifing the iters bound using
+     * self-modifying code
+     */
+    int total;
+    /* first we make the foo code writable.  we can't trust "foo" as a func
+     * ptr b/c it may point to the ILT.  we go ahead and re-enable protection
+     * every time, just guessing at the # pages needed.
+     */
+#ifdef LINUX
+    asm("  call next_inst_prot");
+    asm("next_inst_prot:");
+    asm("  popl  %%edx" : : : "edx");
+    asm("  pushl %0" : : "i" (ALLOW_READ|ALLOW_WRITE|ALLOW_EXEC));
+    /* protect_mem will expand size to page.  If go too far (even
+     * PAGE_SIZE here) will evict later code from executable_areas
+     * that won't come back and will get double violations (case
+     * 7940).
+     */
+    asm("  pushl %0" : : "i" (1));
+    asm("  pushl %edx");
+    asm("  call protect_mem");
+    asm("  addl  $12, %%esp" : : : "esp");
+
+    asm("  movl %0, %%ecx" : : "r" (iters) : "ecx");
+    asm("  call next_inst");
+    asm("next_inst:");
+    asm("  pop %edx");
+    /* add to retaddr: 1 == pop
+     *                 3 == mov ecx into target
+     *                 1 == opcode of target movl
+     */
+    asm("  movl %ecx, 0x5(%edx)"); /* the modifying store */
+    asm("  movl $0x12345678,%eax"); /* this instr's immed gets overwritten */
+
+        /* now we have as many write instrs as necessary to cause a too-big
+         * selfmod fragment.  xref case 7893.
+         */
+    asm("  movl %0, %%ecx" : : "r" (iters) : "ecx");
+    asm("  movl %0, %%edx" : : "g" (big));
+    asm("  movl %ecx, 0(%edx)");
+    asm("  movl %ecx, 1(%edx)");
+    asm("  movl %ecx, 2(%edx)");
+    asm("  movl %ecx, 3(%edx)");
+    asm("  movl %ecx, 4(%edx)");
+    asm("  movl %ecx, 5(%edx)");
+    asm("  movl %ecx, 6(%edx)");
+    asm("  movl %ecx, 7(%edx)");
+    asm("  movl %ecx, 8(%edx)");
+    asm("  movl %ecx, 9(%edx)");
+    asm("  movl %ecx, 10(%edx)");
+    asm("  movl %ecx, 11(%edx)");
+    asm("  movl %ecx, 12(%edx)");
+    asm("  movl %ecx, 13(%edx)");
+    asm("  movl %ecx, 14(%edx)");
+    asm("  movl %ecx, 15(%edx)");
+    asm("  movl %ecx, 16(%edx)");
+    asm("  movl %ecx, 17(%edx)");
+    asm("  movl %ecx, 18(%edx)");
+    asm("  movl %ecx, 19(%edx)");
+    asm("  movl %ecx, 20(%edx)");
+    asm("  movl %ecx, 21(%edx)");
+    asm("  movl %ecx, 22(%edx)");
+    asm("  movl %ecx, 23(%edx)");
+    asm("  movl %ecx, 24(%edx)");
+    asm("  movl %ecx, 25(%edx)");
+    asm("  movl %ecx, 26(%edx)");
+    asm("  movl %ecx, 27(%edx)");
+    asm("  movl %ecx, 28(%edx)");
+    asm("  movl %ecx, 29(%edx)");
+    asm("  movl %ecx, 30(%edx)");
+    asm("  movl %ecx, 31(%edx)");
+    asm("  movl %ecx, 32(%edx)");
+    asm("  movl %ecx, 33(%edx)");
+    asm("  movl %ecx, 34(%edx)");
+    asm("  movl %ecx, 35(%edx)");
+    asm("  movl %ecx, 36(%edx)");
+    asm("  movl %ecx, 37(%edx)");
+    asm("  movl %ecx, 38(%edx)");
+    asm("  movl %ecx, 39(%edx)");
+    asm("  movl %ecx, 40(%edx)");
+    asm("  movl %ecx, 41(%edx)");
+    asm("  movl %ecx, 42(%edx)");
+    asm("  movl %ecx, 43(%edx)");
+    asm("  movl %ecx, 44(%edx)");
+    asm("  movl %ecx, 45(%edx)");
+    asm("  movl %ecx, 46(%edx)");
+    asm("  movl %ecx, 47(%edx)");
+    asm("  movl %ecx, 48(%edx)");
+    asm("  movl %ecx, 49(%edx)");
+    asm("  movl %ecx, 50(%edx)");
+    asm("  movl %ecx, 51(%edx)");
+    asm("  movl %ecx, 52(%edx)");
+    asm("  movl %ecx, 53(%edx)");
+    asm("  movl %ecx, 54(%edx)");
+    asm("  movl %ecx, 55(%edx)");
+    asm("  movl %ecx, 56(%edx)");
+    asm("  movl %ecx, 57(%edx)");
+    asm("  movl %ecx, 58(%edx)");
+    asm("  movl %ecx, 59(%edx)");
+    asm("  movl %ecx, 60(%edx)");
+    asm("  movl %ecx, 61(%edx)");
+    asm("  movl %ecx, 62(%edx)");
+    asm("  movl %ecx, 63(%edx)");
+    asm("  movl %ecx, 64(%edx)");
+    asm("  movl %ecx, 65(%edx)");
+    asm("  movl %ecx, 66(%edx)");
+    asm("  movl %ecx, 67(%edx)");
+    asm("  movl %ecx, 68(%edx)");
+    asm("  movl %ecx, 69(%edx)");
+    asm("  movl %ecx, 70(%edx)");
+    asm("  movl %ecx, 71(%edx)");
+    asm("  movl %ecx, 72(%edx)");
+    asm("  movl %ecx, 73(%edx)");
+    asm("  movl %ecx, 74(%edx)");
+    asm("  movl %ecx, 75(%edx)");
+    asm("  movl %ecx, 76(%edx)");
+    asm("  movl %ecx, 77(%edx)");
+    asm("  movl %ecx, 78(%edx)");
+    asm("  movl %ecx, 79(%edx)");
+    asm("  movl %ecx, 80(%edx)");
+    asm("  movl %ecx, 81(%edx)");
+    asm("  movl %ecx, 82(%edx)");
+    asm("  movl %ecx, 83(%edx)");
+    asm("  movl %ecx, 84(%edx)");
+    asm("  movl %ecx, 85(%edx)");
+    asm("  movl %ecx, 86(%edx)");
+    asm("  movl %ecx, 87(%edx)");
+    asm("  movl %ecx, 88(%edx)");
+    asm("  movl %ecx, 89(%edx)");
+    asm("  movl %ecx, 90(%edx)");
+    asm("  movl %ecx, 91(%edx)");
+    asm("  movl %ecx, 92(%edx)");
+    asm("  movl %ecx, 93(%edx)");
+    asm("  movl %ecx, 94(%edx)");
+    asm("  movl %ecx, 95(%edx)");
+    asm("  movl %ecx, 96(%edx)");
+    asm("  movl %ecx, 97(%edx)");
+    asm("  movl %ecx, 98(%edx)");
+    asm("  movl %ecx, 99(%edx)");
+    asm("  movl %ecx, 100(%edx)");
+    asm("  movl %ecx, 101(%edx)");
+    asm("  movl %ecx, 102(%edx)");
+    asm("  movl %ecx, 103(%edx)");
+    asm("  movl %ecx, 104(%edx)");
+    asm("  movl %ecx, 105(%edx)");
+    asm("  movl %ecx, 106(%edx)");
+    asm("  movl %ecx, 107(%edx)");
+    asm("  movl %ecx, 108(%edx)");
+    asm("  movl %ecx, 109(%edx)");
+    asm("  movl %ecx, 110(%edx)");
+    asm("  movl %ecx, 111(%edx)");
+    asm("  movl %ecx, 112(%edx)");
+    asm("  movl %ecx, 113(%edx)");
+    asm("  movl %ecx, 114(%edx)");
+    asm("  movl %ecx, 115(%edx)");
+    asm("  movl %ecx, 116(%edx)");
+    asm("  movl %ecx, 117(%edx)");
+    asm("  movl %ecx, 118(%edx)");
+    asm("  movl %ecx, 119(%edx)");
+    asm("  movl %ecx, 120(%edx)");
+    asm("  movl %ecx, 121(%edx)");
+    asm("  movl %ecx, 122(%edx)");
+    asm("  movl %ecx, 123(%edx)");
+    asm("  movl %ecx, 124(%edx)");
+    asm("  movl %ecx, 125(%edx)");
+    asm("  movl %ecx, 126(%edx)");
+    asm("  movl %ecx, 127(%edx)");
+    asm("  movl %ecx, 128(%edx)");
+    asm("  movl %ecx, 129(%edx)");
+    asm("  movl %ecx, 130(%edx)");
+    asm("  movl %ecx, 131(%edx)");
+    asm("  movl %ecx, 132(%edx)");
+    asm("  movl %ecx, 133(%edx)");
+    asm("  movl %ecx, 134(%edx)");
+    asm("  movl %ecx, 135(%edx)");
+    asm("  movl %ecx, 136(%edx)");
+    asm("  movl %ecx, 137(%edx)");
+    asm("  movl %ecx, 138(%edx)");
+    asm("  movl %ecx, 139(%edx)");
+    asm("  movl %ecx, 140(%edx)");
+    asm("  movl %ecx, 141(%edx)");
+    asm("  movl %ecx, 142(%edx)");
+    asm("  movl %ecx, 143(%edx)");
+    asm("  movl %ecx, 144(%edx)");
+    asm("  movl %ecx, 145(%edx)");
+    asm("  movl %ecx, 146(%edx)");
+    asm("  movl %ecx, 147(%edx)");
+    asm("  movl %ecx, 148(%edx)");
+    asm("  movl %ecx, 149(%edx)");
+    asm("  movl %ecx, 150(%edx)");
+    asm("  movl %ecx, 151(%edx)");
+    asm("  movl %ecx, 152(%edx)");
+    asm("  movl %ecx, 153(%edx)");
+    asm("  movl %ecx, 154(%edx)");
+    asm("  movl %ecx, 155(%edx)");
+    asm("  movl %ecx, 156(%edx)");
+    asm("  movl %ecx, 157(%edx)");
+    asm("  movl %ecx, 158(%edx)");
+    asm("  movl %ecx, 159(%edx)");
+    asm("  movl %ecx, 160(%edx)");
+    asm("  movl %ecx, 161(%edx)");
+    asm("  movl %ecx, 162(%edx)");
+    asm("  movl %ecx, 163(%edx)");
+    asm("  movl %ecx, 164(%edx)");
+    asm("  movl %ecx, 165(%edx)");
+    asm("  movl %ecx, 166(%edx)");
+    asm("  movl %ecx, 167(%edx)");
+    asm("  movl %ecx, 168(%edx)");
+    asm("  movl %ecx, 169(%edx)");
+    asm("  movl %ecx, 170(%edx)");
+    asm("  movl %ecx, 171(%edx)");
+    asm("  movl %ecx, 172(%edx)");
+    asm("  movl %ecx, 173(%edx)");
+    asm("  movl %ecx, 174(%edx)");
+    asm("  movl %ecx, 175(%edx)");
+    asm("  movl %ecx, 176(%edx)");
+    asm("  movl %ecx, 177(%edx)");
+    asm("  movl %ecx, 178(%edx)");
+    asm("  movl %ecx, 179(%edx)");
+    asm("  movl %ecx, 180(%edx)");
+    asm("  movl %ecx, 181(%edx)");
+    asm("  movl %ecx, 182(%edx)");
+    asm("  movl %ecx, 183(%edx)");
+    asm("  movl %ecx, 184(%edx)");
+    asm("  movl %ecx, 185(%edx)");
+    asm("  movl %ecx, 186(%edx)");
+    asm("  movl %ecx, 187(%edx)");
+    asm("  movl %ecx, 188(%edx)");
+    asm("  movl %ecx, 189(%edx)");
+    asm("  movl %ecx, 190(%edx)");
+    asm("  movl %ecx, 191(%edx)");
+    asm("  movl %ecx, 192(%edx)");
+    asm("  movl %ecx, 193(%edx)");
+    asm("  movl %ecx, 194(%edx)");
+    asm("  movl %ecx, 195(%edx)");
+    asm("  movl %ecx, 196(%edx)");
+    asm("  movl %ecx, 197(%edx)");
+    asm("  movl %ecx, 198(%edx)");
+    asm("  movl %ecx, 199(%edx)");
+    asm("  movl %ecx, 200(%edx)");
+    asm("  movl %ecx, 201(%edx)");
+    asm("  movl %ecx, 202(%edx)");
+    asm("  movl %ecx, 203(%edx)");
+    asm("  movl %ecx, 204(%edx)");
+    asm("  movl %ecx, 205(%edx)");
+    asm("  movl %ecx, 206(%edx)");
+    asm("  movl %ecx, 207(%edx)");
+    asm("  movl %ecx, 208(%edx)");
+    asm("  movl %ecx, 209(%edx)");
+    asm("  movl %ecx, 210(%edx)");
+    asm("  movl %ecx, 211(%edx)");
+    asm("  movl %ecx, 212(%edx)");
+    asm("  movl %ecx, 213(%edx)");
+    asm("  movl %ecx, 214(%edx)");
+    asm("  movl %ecx, 215(%edx)");
+    asm("  movl %ecx, 216(%edx)");
+    asm("  movl %ecx, 217(%edx)");
+    asm("  movl %ecx, 218(%edx)");
+    asm("  movl %ecx, 219(%edx)");
+    asm("  movl %ecx, 220(%edx)");
+    asm("  movl %ecx, 221(%edx)");
+    asm("  movl %ecx, 222(%edx)");
+    asm("  movl %ecx, 223(%edx)");
+    asm("  movl %ecx, 224(%edx)");
+    asm("  movl %ecx, 225(%edx)");
+    asm("  movl %ecx, 226(%edx)");
+    asm("  movl %ecx, 227(%edx)");
+    asm("  movl %ecx, 228(%edx)");
+    asm("  movl %ecx, 229(%edx)");
+    asm("  movl %ecx, 230(%edx)");
+    asm("  movl %ecx, 231(%edx)");
+    asm("  movl %ecx, 232(%edx)");
+    asm("  movl %ecx, 233(%edx)");
+    asm("  movl %ecx, 234(%edx)");
+    asm("  movl %ecx, 235(%edx)");
+    asm("  movl %ecx, 236(%edx)");
+    asm("  movl %ecx, 237(%edx)");
+    asm("  movl %ecx, 238(%edx)");
+    asm("  movl %ecx, 239(%edx)");
+    asm("  movl %ecx, 240(%edx)");
+    asm("  movl %ecx, 241(%edx)");
+    asm("  movl %ecx, 242(%edx)");
+    asm("  movl %ecx, 243(%edx)");
+    asm("  movl %ecx, 244(%edx)");
+    asm("  movl %ecx, 245(%edx)");
+    asm("  movl %ecx, 246(%edx)");
+    asm("  movl %ecx, 247(%edx)");
+    asm("  movl %ecx, 248(%edx)");
+    asm("  movl %ecx, 249(%edx)");
+    asm("  movl %ecx, 250(%edx)");
+    asm("  movl %ecx, 251(%edx)");
+    asm("  movl %ecx, 252(%edx)");
+    asm("  movl %ecx, 253(%edx)");
+    asm("  movl %ecx, 254(%edx)");
+    asm("  movl %ecx, 255(%edx)");
+    asm("  movl %ecx, 256(%edx)");
+    asm("  movl %ecx, 257(%edx)");
+    asm("  movl %ecx, 258(%edx)");
+    asm("  movl %ecx, 259(%edx)");
+    asm("  movl %ecx, 260(%edx)");
+    asm("  movl %ecx, 261(%edx)");
+    asm("  movl %ecx, 262(%edx)");
+    asm("  movl %ecx, 263(%edx)");
+    asm("  movl %ecx, 264(%edx)");
+    asm("  movl %ecx, 265(%edx)");
+    asm("  movl %ecx, 266(%edx)");
+    asm("  movl %ecx, 267(%edx)");
+    asm("  movl %ecx, 268(%edx)");
+    asm("  movl %ecx, 269(%edx)");
+    asm("  movl %ecx, 270(%edx)");
+    asm("  movl %ecx, 271(%edx)");
+    asm("  movl %ecx, 272(%edx)");
+    asm("  movl %ecx, 273(%edx)");
+    asm("  movl %ecx, 274(%edx)");
+    asm("  movl %ecx, 275(%edx)");
+    asm("  movl %ecx, 276(%edx)");
+    asm("  movl %ecx, 277(%edx)");
+    asm("  movl %ecx, 278(%edx)");
+    asm("  movl %ecx, 279(%edx)");
+    asm("  movl %ecx, 280(%edx)");
+    asm("  movl %ecx, 281(%edx)");
+    asm("  movl %ecx, 282(%edx)");
+    asm("  movl %ecx, 283(%edx)");
+    asm("  movl %ecx, 284(%edx)");
+    asm("  movl %ecx, 285(%edx)");
+    asm("  movl %ecx, 286(%edx)");
+    asm("  movl %ecx, 287(%edx)");
+    asm("  movl %ecx, 288(%edx)");
+    asm("  movl %ecx, 289(%edx)");
+    asm("  movl %ecx, 290(%edx)");
+    asm("  movl %ecx, 291(%edx)");
+    asm("  movl %ecx, 292(%edx)");
+    asm("  movl %ecx, 293(%edx)");
+    asm("  movl %ecx, 294(%edx)");
+    asm("  movl %ecx, 295(%edx)");
+    asm("  movl %ecx, 296(%edx)");
+    asm("  movl %ecx, 297(%edx)");
+    asm("  movl %ecx, 298(%edx)");
+    asm("  movl %ecx, 299(%edx)");
+    asm("  movl %ecx, 300(%edx)");
+    asm("  movl %ecx, 301(%edx)");
+    asm("  movl %ecx, 302(%edx)");
+    asm("  movl %ecx, 303(%edx)");
+    asm("  movl %ecx, 304(%edx)");
+    asm("  movl %ecx, 305(%edx)");
+    asm("  movl %ecx, 306(%edx)");
+    asm("  movl %ecx, 307(%edx)");
+    asm("  movl %ecx, 308(%edx)");
+    asm("  movl %ecx, 309(%edx)");
+    asm("  movl %ecx, 310(%edx)");
+    asm("  movl %ecx, 311(%edx)");
+    asm("  movl %ecx, 312(%edx)");
+    asm("  movl %ecx, 313(%edx)");
+    asm("  movl %ecx, 314(%edx)");
+    asm("  movl %ecx, 315(%edx)");
+    asm("  movl %ecx, 316(%edx)");
+    asm("  movl %ecx, 317(%edx)");
+    asm("  movl %ecx, 318(%edx)");
+    asm("  movl %ecx, 319(%edx)");
+    asm("  movl %ecx, 320(%edx)");
+    asm("  movl %ecx, 321(%edx)");
+    asm("  movl %ecx, 322(%edx)");
+    asm("  movl %ecx, 323(%edx)");
+    asm("  movl %ecx, 324(%edx)");
+    asm("  movl %ecx, 325(%edx)");
+    asm("  movl %ecx, 326(%edx)");
+    asm("  movl %ecx, 327(%edx)");
+    asm("  movl %ecx, 328(%edx)");
+    asm("  movl %ecx, 329(%edx)");
+    asm("  movl %ecx, 330(%edx)");
+    asm("  movl %ecx, 331(%edx)");
+    asm("  movl %ecx, 332(%edx)");
+    asm("  movl %ecx, 333(%edx)");
+    asm("  movl %ecx, 334(%edx)");
+    asm("  movl %ecx, 335(%edx)");
+    asm("  movl %ecx, 336(%edx)");
+    asm("  movl %ecx, 337(%edx)");
+    asm("  movl %ecx, 338(%edx)");
+    asm("  movl %ecx, 339(%edx)");
+    asm("  movl %ecx, 340(%edx)");
+    asm("  movl %ecx, 341(%edx)");
+    asm("  movl %ecx, 342(%edx)");
+    asm("  movl %ecx, 343(%edx)");
+    asm("  movl %ecx, 344(%edx)");
+    asm("  movl %ecx, 345(%edx)");
+    asm("  movl %ecx, 346(%edx)");
+    asm("  movl %ecx, 347(%edx)");
+    asm("  movl %ecx, 348(%edx)");
+    asm("  movl %ecx, 349(%edx)");
+    asm("  movl %ecx, 350(%edx)");
+    asm("  movl %ecx, 351(%edx)");
+    asm("  movl %ecx, 352(%edx)");
+    asm("  movl %ecx, 353(%edx)");
+    asm("  movl %ecx, 354(%edx)");
+    asm("  movl %ecx, 355(%edx)");
+    asm("  movl %ecx, 356(%edx)");
+    asm("  movl %ecx, 357(%edx)");
+    asm("  movl %ecx, 358(%edx)");
+    asm("  movl %ecx, 359(%edx)");
+    asm("  movl %ecx, 360(%edx)");
+    asm("  movl %ecx, 361(%edx)");
+    asm("  movl %ecx, 362(%edx)");
+    asm("  movl %ecx, 363(%edx)");
+    asm("  movl %ecx, 364(%edx)");
+    asm("  movl %ecx, 365(%edx)");
+    asm("  movl %ecx, 366(%edx)");
+    asm("  movl %ecx, 367(%edx)");
+    asm("  movl %ecx, 368(%edx)");
+    asm("  movl %ecx, 369(%edx)");
+    asm("  movl %ecx, 370(%edx)");
+    asm("  movl %ecx, 371(%edx)");
+    asm("  movl %ecx, 372(%edx)");
+    asm("  movl %ecx, 373(%edx)");
+    asm("  movl %ecx, 374(%edx)");
+    asm("  movl %ecx, 375(%edx)");
+    asm("  movl %ecx, 376(%edx)");
+    asm("  movl %ecx, 377(%edx)");
+    asm("  movl %ecx, 378(%edx)");
+    asm("  movl %ecx, 379(%edx)");
+    asm("  movl %ecx, 380(%edx)");
+    asm("  movl %ecx, 381(%edx)");
+    asm("  movl %ecx, 382(%edx)");
+    asm("  movl %ecx, 383(%edx)");
+    asm("  movl %ecx, 384(%edx)");
+    asm("  movl %ecx, 385(%edx)");
+    asm("  movl %ecx, 386(%edx)");
+    asm("  movl %ecx, 387(%edx)");
+    asm("  movl %ecx, 388(%edx)");
+    asm("  movl %ecx, 389(%edx)");
+    asm("  movl %ecx, 390(%edx)");
+    asm("  movl %ecx, 391(%edx)");
+    asm("  movl %ecx, 392(%edx)");
+    asm("  movl %ecx, 393(%edx)");
+    asm("  movl %ecx, 394(%edx)");
+    asm("  movl %ecx, 395(%edx)");
+    asm("  movl %ecx, 396(%edx)");
+    asm("  movl %ecx, 397(%edx)");
+    asm("  movl %ecx, 398(%edx)");
+    asm("  movl %ecx, 399(%edx)");
+    asm("  movl %ecx, 400(%edx)");
+    asm("  movl %ecx, 401(%edx)");
+    asm("  movl %ecx, 402(%edx)");
+    asm("  movl %ecx, 403(%edx)");
+    asm("  movl %ecx, 404(%edx)");
+    asm("  movl %ecx, 405(%edx)");
+    asm("  movl %ecx, 406(%edx)");
+    asm("  movl %ecx, 407(%edx)");
+    asm("  movl %ecx, 408(%edx)");
+    asm("  movl %ecx, 409(%edx)");
+    asm("  movl %ecx, 410(%edx)");
+    asm("  movl %ecx, 411(%edx)");
+    asm("  movl %ecx, 412(%edx)");
+    asm("  movl %ecx, 413(%edx)");
+    asm("  movl %ecx, 414(%edx)");
+    asm("  movl %ecx, 415(%edx)");
+    asm("  movl %ecx, 416(%edx)");
+    asm("  movl %ecx, 417(%edx)");
+    asm("  movl %ecx, 418(%edx)");
+    asm("  movl %ecx, 419(%edx)");
+    asm("  movl %ecx, 420(%edx)");
+    asm("  movl %ecx, 421(%edx)");
+    asm("  movl %ecx, 422(%edx)");
+    asm("  movl %ecx, 423(%edx)");
+    asm("  movl %ecx, 424(%edx)");
+    asm("  movl %ecx, 425(%edx)");
+    asm("  movl %ecx, 426(%edx)");
+    asm("  movl %ecx, 427(%edx)");
+    asm("  movl %ecx, 428(%edx)");
+    asm("  movl %ecx, 429(%edx)");
+    asm("  movl %ecx, 430(%edx)");
+    asm("  movl %ecx, 431(%edx)");
+    asm("  movl %ecx, 432(%edx)");
+    asm("  movl %ecx, 433(%edx)");
+    asm("  movl %ecx, 434(%edx)");
+    asm("  movl %ecx, 435(%edx)");
+    asm("  movl %ecx, 436(%edx)");
+    asm("  movl %ecx, 437(%edx)");
+    asm("  movl %ecx, 438(%edx)");
+    asm("  movl %ecx, 439(%edx)");
+    asm("  movl %ecx, 440(%edx)");
+    asm("  movl %ecx, 441(%edx)");
+    asm("  movl %ecx, 442(%edx)");
+    asm("  movl %ecx, 443(%edx)");
+    asm("  movl %ecx, 444(%edx)");
+    asm("  movl %ecx, 445(%edx)");
+    asm("  movl %ecx, 446(%edx)");
+    asm("  movl %ecx, 447(%edx)");
+    asm("  movl %ecx, 448(%edx)");
+    asm("  movl %ecx, 449(%edx)");
+    asm("  movl %ecx, 450(%edx)");
+    asm("  movl %ecx, 451(%edx)");
+    asm("  movl %ecx, 452(%edx)");
+    asm("  movl %ecx, 453(%edx)");
+    asm("  movl %ecx, 454(%edx)");
+    asm("  movl %ecx, 455(%edx)");
+    asm("  movl %ecx, 456(%edx)");
+    asm("  movl %ecx, 457(%edx)");
+    asm("  movl %ecx, 458(%edx)");
+    asm("  movl %ecx, 459(%edx)");
+    asm("  movl %ecx, 460(%edx)");
+    asm("  movl %ecx, 461(%edx)");
+    asm("  movl %ecx, 462(%edx)");
+    asm("  movl %ecx, 463(%edx)");
+    asm("  movl %ecx, 464(%edx)");
+    asm("  movl %ecx, 465(%edx)");
+    asm("  movl %ecx, 466(%edx)");
+    asm("  movl %ecx, 467(%edx)");
+    asm("  movl %ecx, 468(%edx)");
+    asm("  movl %ecx, 469(%edx)");
+    asm("  movl %ecx, 470(%edx)");
+    asm("  movl %ecx, 471(%edx)");
+    asm("  movl %ecx, 472(%edx)");
+    asm("  movl %ecx, 473(%edx)");
+    asm("  movl %ecx, 474(%edx)");
+    asm("  movl %ecx, 475(%edx)");
+    asm("  movl %ecx, 476(%edx)");
+    asm("  movl %ecx, 477(%edx)");
+    asm("  movl %ecx, 478(%edx)");
+    asm("  movl %ecx, 479(%edx)");
+    asm("  movl %ecx, 480(%edx)");
+    asm("  movl %ecx, 481(%edx)");
+    asm("  movl %ecx, 482(%edx)");
+    asm("  movl %ecx, 483(%edx)");
+    asm("  movl %ecx, 484(%edx)");
+    asm("  movl %ecx, 485(%edx)");
+    asm("  movl %ecx, 486(%edx)");
+    asm("  movl %ecx, 487(%edx)");
+    asm("  movl %ecx, 488(%edx)");
+    asm("  movl %ecx, 489(%edx)");
+    asm("  movl %ecx, 490(%edx)");
+    asm("  movl %ecx, 491(%edx)");
+    asm("  movl %ecx, 492(%edx)");
+    asm("  movl %ecx, 493(%edx)");
+    asm("  movl %ecx, 494(%edx)");
+    asm("  movl %ecx, 495(%edx)");
+    asm("  movl %ecx, 496(%edx)");
+    asm("  movl %ecx, 497(%edx)");
+    asm("  movl %ecx, 498(%edx)");
+    asm("  movl %ecx, 499(%edx)");
+    asm("  movl %ecx, 500(%edx)");
+    asm("  movl %ecx, 501(%edx)");
+    asm("  movl %ecx, 502(%edx)");
+    asm("  movl %ecx, 503(%edx)");
+    asm("  movl %ecx, 504(%edx)");
+    asm("  movl %ecx, 505(%edx)");
+    asm("  movl %ecx, 506(%edx)");
+    asm("  movl %ecx, 507(%edx)");
+    asm("  movl %ecx, 508(%edx)");
+    asm("  movl %ecx, 509(%edx)");
+    asm("  movl %ecx, 510(%edx)");
+    asm("  movl %ecx, 511(%edx)");
+    asm("  movl %ecx, 512(%edx)");
+    asm("  movl %ecx, 513(%edx)");
+    asm("  movl %ecx, 514(%edx)");
+    asm("  movl %ecx, 515(%edx)");
+    asm("  movl %ecx, 516(%edx)");
+    asm("  movl %ecx, 517(%edx)");
+    asm("  movl %ecx, 518(%edx)");
+    asm("  movl %ecx, 519(%edx)");
+    asm("  movl %ecx, 520(%edx)");
+    asm("  movl %ecx, 521(%edx)");
+    asm("  movl %ecx, 522(%edx)");
+    asm("  movl %ecx, 523(%edx)");
+    asm("  movl %ecx, 524(%edx)");
+    asm("  movl %ecx, 525(%edx)");
+    asm("  movl %ecx, 526(%edx)");
+    asm("  movl %ecx, 527(%edx)");
+    asm("  movl %ecx, 528(%edx)");
+    asm("  movl %ecx, 529(%edx)");
+    asm("  movl %ecx, 530(%edx)");
+    asm("  movl %ecx, 531(%edx)");
+    asm("  movl %ecx, 532(%edx)");
+    asm("  movl %ecx, 533(%edx)");
+    asm("  movl %ecx, 534(%edx)");
+    asm("  movl %ecx, 535(%edx)");
+    asm("  movl %ecx, 536(%edx)");
+    asm("  movl %ecx, 537(%edx)");
+    asm("  movl %ecx, 538(%edx)");
+    asm("  movl %ecx, 539(%edx)");
+    asm("  movl %ecx, 540(%edx)");
+    asm("  movl %ecx, 541(%edx)");
+    asm("  movl %ecx, 542(%edx)");
+    asm("  movl %ecx, 543(%edx)");
+    asm("  movl %ecx, 544(%edx)");
+    asm("  movl %ecx, 545(%edx)");
+    asm("  movl %ecx, 546(%edx)");
+    asm("  movl %ecx, 547(%edx)");
+    asm("  movl %ecx, 548(%edx)");
+    asm("  movl %ecx, 549(%edx)");
+    asm("  movl %ecx, 550(%edx)");
+    asm("  movl %ecx, 551(%edx)");
+    asm("  movl %ecx, 552(%edx)");
+    asm("  movl %ecx, 553(%edx)");
+    asm("  movl %ecx, 554(%edx)");
+    asm("  movl %ecx, 555(%edx)");
+    asm("  movl %ecx, 556(%edx)");
+    asm("  movl %ecx, 557(%edx)");
+    asm("  movl %ecx, 558(%edx)");
+    asm("  movl %ecx, 559(%edx)");
+    asm("  movl %ecx, 560(%edx)");
+    asm("  movl %ecx, 561(%edx)");
+    asm("  movl %ecx, 562(%edx)");
+    asm("  movl %ecx, 563(%edx)");
+    asm("  movl %ecx, 564(%edx)");
+    asm("  movl %ecx, 565(%edx)");
+    asm("  movl %ecx, 566(%edx)");
+    asm("  movl %ecx, 567(%edx)");
+    asm("  movl %ecx, 568(%edx)");
+    asm("  movl %ecx, 569(%edx)");
+    asm("  movl %ecx, 570(%edx)");
+    asm("  movl %ecx, 571(%edx)");
+    asm("  movl %ecx, 572(%edx)");
+    asm("  movl %ecx, 573(%edx)");
+    asm("  movl %ecx, 574(%edx)");
+    asm("  movl %ecx, 575(%edx)");
+    asm("  movl %ecx, 576(%edx)");
+    asm("  movl %ecx, 577(%edx)");
+    asm("  movl %ecx, 578(%edx)");
+    asm("  movl %ecx, 579(%edx)");
+    asm("  movl %ecx, 580(%edx)");
+    asm("  movl %ecx, 581(%edx)");
+    asm("  movl %ecx, 582(%edx)");
+    asm("  movl %ecx, 583(%edx)");
+    asm("  movl %ecx, 584(%edx)");
+    asm("  movl %ecx, 585(%edx)");
+    asm("  movl %ecx, 586(%edx)");
+    asm("  movl %ecx, 587(%edx)");
+    asm("  movl %ecx, 588(%edx)");
+    asm("  movl %ecx, 589(%edx)");
+    asm("  movl %ecx, 590(%edx)");
+    asm("  movl %ecx, 591(%edx)");
+    asm("  movl %ecx, 592(%edx)");
+    asm("  movl %ecx, 593(%edx)");
+    asm("  movl %ecx, 594(%edx)");
+    asm("  movl %ecx, 595(%edx)");
+    asm("  movl %ecx, 596(%edx)");
+    asm("  movl %ecx, 597(%edx)");
+    asm("  movl %ecx, 598(%edx)");
+    asm("  movl %ecx, 599(%edx)");
+    asm("  movl %ecx, 600(%edx)");
+    asm("  movl %ecx, 601(%edx)");
+    asm("  movl %ecx, 602(%edx)");
+    asm("  movl %ecx, 603(%edx)");
+    asm("  movl %ecx, 604(%edx)");
+    asm("  movl %ecx, 605(%edx)");
+    asm("  movl %ecx, 606(%edx)");
+    asm("  movl %ecx, 607(%edx)");
+    asm("  movl %ecx, 608(%edx)");
+    asm("  movl %ecx, 609(%edx)");
+    asm("  movl %ecx, 610(%edx)");
+    asm("  movl %ecx, 611(%edx)");
+    asm("  movl %ecx, 612(%edx)");
+    asm("  movl %ecx, 613(%edx)");
+    asm("  movl %ecx, 614(%edx)");
+    asm("  movl %ecx, 615(%edx)");
+    asm("  movl %ecx, 616(%edx)");
+    asm("  movl %ecx, 617(%edx)");
+    asm("  movl %ecx, 618(%edx)");
+    asm("  movl %ecx, 619(%edx)");
+    asm("  movl %ecx, 620(%edx)");
+    asm("  movl %ecx, 621(%edx)");
+    asm("  movl %ecx, 622(%edx)");
+    asm("  movl %ecx, 623(%edx)");
+    asm("  movl %ecx, 624(%edx)");
+    asm("  movl %ecx, 625(%edx)");
+    asm("  movl %ecx, 626(%edx)");
+    asm("  movl %ecx, 627(%edx)");
+    asm("  movl %ecx, 628(%edx)");
+    asm("  movl %ecx, 629(%edx)");
+    asm("  movl %ecx, 630(%edx)");
+    asm("  movl %ecx, 631(%edx)");
+    asm("  movl %ecx, 632(%edx)");
+    asm("  movl %ecx, 633(%edx)");
+    asm("  movl %ecx, 634(%edx)");
+    asm("  movl %ecx, 635(%edx)");
+    asm("  movl %ecx, 636(%edx)");
+    asm("  movl %ecx, 637(%edx)");
+    asm("  movl %ecx, 638(%edx)");
+    asm("  movl %ecx, 639(%edx)");
+    asm("  movl %ecx, 640(%edx)");
+    asm("  movl %ecx, 641(%edx)");
+    asm("  movl %ecx, 642(%edx)");
+    asm("  movl %ecx, 643(%edx)");
+    asm("  movl %ecx, 644(%edx)");
+    asm("  movl %ecx, 645(%edx)");
+    asm("  movl %ecx, 646(%edx)");
+    asm("  movl %ecx, 647(%edx)");
+    asm("  movl %ecx, 648(%edx)");
+    asm("  movl %ecx, 649(%edx)");
+    asm("  movl %ecx, 650(%edx)");
+    asm("  movl %ecx, 651(%edx)");
+    asm("  movl %ecx, 652(%edx)");
+    asm("  movl %ecx, 653(%edx)");
+    asm("  movl %ecx, 654(%edx)");
+    asm("  movl %ecx, 655(%edx)");
+    asm("  movl %ecx, 656(%edx)");
+    asm("  movl %ecx, 657(%edx)");
+    asm("  movl %ecx, 658(%edx)");
+    asm("  movl %ecx, 659(%edx)");
+    asm("  movl %ecx, 660(%edx)");
+    asm("  movl %ecx, 661(%edx)");
+    asm("  movl %ecx, 662(%edx)");
+    asm("  movl %ecx, 663(%edx)");
+    asm("  movl %ecx, 664(%edx)");
+    asm("  movl %ecx, 665(%edx)");
+    asm("  movl %ecx, 666(%edx)");
+    asm("  movl %ecx, 667(%edx)");
+    asm("  movl %ecx, 668(%edx)");
+    asm("  movl %ecx, 669(%edx)");
+    asm("  movl %ecx, 670(%edx)");
+    asm("  movl %ecx, 671(%edx)");
+    asm("  movl %ecx, 672(%edx)");
+    asm("  movl %ecx, 673(%edx)");
+    asm("  movl %ecx, 674(%edx)");
+    asm("  movl %ecx, 675(%edx)");
+    asm("  movl %ecx, 676(%edx)");
+    asm("  movl %ecx, 677(%edx)");
+    asm("  movl %ecx, 678(%edx)");
+    asm("  movl %ecx, 679(%edx)");
+    asm("  movl %ecx, 680(%edx)");
+    asm("  movl %ecx, 681(%edx)");
+    asm("  movl %ecx, 682(%edx)");
+    asm("  movl %ecx, 683(%edx)");
+    asm("  movl %ecx, 684(%edx)");
+    asm("  movl %ecx, 685(%edx)");
+    asm("  movl %ecx, 686(%edx)");
+    asm("  movl %ecx, 687(%edx)");
+    asm("  movl %ecx, 688(%edx)");
+    asm("  movl %ecx, 689(%edx)");
+    asm("  movl %ecx, 690(%edx)");
+    asm("  movl %ecx, 691(%edx)");
+    asm("  movl %ecx, 692(%edx)");
+    asm("  movl %ecx, 693(%edx)");
+    asm("  movl %ecx, 694(%edx)");
+    asm("  movl %ecx, 695(%edx)");
+    asm("  movl %ecx, 696(%edx)");
+    asm("  movl %ecx, 697(%edx)");
+    asm("  movl %ecx, 698(%edx)");
+    asm("  movl %ecx, 699(%edx)");
+    asm("  movl %ecx, 700(%edx)");
+    asm("  movl %ecx, 701(%edx)");
+    asm("  movl %ecx, 702(%edx)");
+    asm("  movl %ecx, 703(%edx)");
+    asm("  movl %ecx, 704(%edx)");
+    asm("  movl %ecx, 705(%edx)");
+    asm("  movl %ecx, 706(%edx)");
+    asm("  movl %ecx, 707(%edx)");
+    asm("  movl %ecx, 708(%edx)");
+    asm("  movl %ecx, 709(%edx)");
+    asm("  movl %ecx, 710(%edx)");
+    asm("  movl %ecx, 711(%edx)");
+    asm("  movl %ecx, 712(%edx)");
+    asm("  movl %ecx, 713(%edx)");
+    asm("  movl %ecx, 714(%edx)");
+    asm("  movl %ecx, 715(%edx)");
+    asm("  movl %ecx, 716(%edx)");
+    asm("  movl %ecx, 717(%edx)");
+    asm("  movl %ecx, 718(%edx)");
+    asm("  movl %ecx, 719(%edx)");
+    asm("  movl %ecx, 720(%edx)");
+    asm("  movl %ecx, 721(%edx)");
+    asm("  movl %ecx, 722(%edx)");
+    asm("  movl %ecx, 723(%edx)");
+    asm("  movl %ecx, 724(%edx)");
+    asm("  movl %ecx, 725(%edx)");
+    asm("  movl %ecx, 726(%edx)");
+    asm("  movl %ecx, 727(%edx)");
+    asm("  movl %ecx, 728(%edx)");
+    asm("  movl %ecx, 729(%edx)");
+    asm("  movl %ecx, 730(%edx)");
+    asm("  movl %ecx, 731(%edx)");
+    asm("  movl %ecx, 732(%edx)");
+    asm("  movl %ecx, 733(%edx)");
+    asm("  movl %ecx, 734(%edx)");
+    asm("  movl %ecx, 735(%edx)");
+    asm("  movl %ecx, 736(%edx)");
+    asm("  movl %ecx, 737(%edx)");
+    asm("  movl %ecx, 738(%edx)");
+    asm("  movl %ecx, 739(%edx)");
+    asm("  movl %ecx, 740(%edx)");
+    asm("  movl %ecx, 741(%edx)");
+    asm("  movl %ecx, 742(%edx)");
+    asm("  movl %ecx, 743(%edx)");
+    asm("  movl %ecx, 744(%edx)");
+    asm("  movl %ecx, 745(%edx)");
+    asm("  movl %ecx, 746(%edx)");
+    asm("  movl %ecx, 747(%edx)");
+    asm("  movl %ecx, 748(%edx)");
+    asm("  movl %ecx, 749(%edx)");
+    asm("  movl %ecx, 750(%edx)");
+    asm("  movl %ecx, 751(%edx)");
+    asm("  movl %ecx, 752(%edx)");
+    asm("  movl %ecx, 753(%edx)");
+    asm("  movl %ecx, 754(%edx)");
+    asm("  movl %ecx, 755(%edx)");
+    asm("  movl %ecx, 756(%edx)");
+    asm("  movl %ecx, 757(%edx)");
+    asm("  movl %ecx, 758(%edx)");
+    asm("  movl %ecx, 759(%edx)");
+    asm("  movl %ecx, 760(%edx)");
+    asm("  movl %ecx, 761(%edx)");
+    asm("  movl %ecx, 762(%edx)");
+    asm("  movl %ecx, 763(%edx)");
+    asm("  movl %ecx, 764(%edx)");
+    asm("  movl %ecx, 765(%edx)");
+    asm("  movl %ecx, 766(%edx)");
+    asm("  movl %ecx, 767(%edx)");
+    asm("  movl %ecx, 768(%edx)");
+    asm("  movl %ecx, 769(%edx)");
+    asm("  movl %ecx, 770(%edx)");
+    asm("  movl %ecx, 771(%edx)");
+    asm("  movl %ecx, 772(%edx)");
+    asm("  movl %ecx, 773(%edx)");
+    asm("  movl %ecx, 774(%edx)");
+    asm("  movl %ecx, 775(%edx)");
+    asm("  movl %ecx, 776(%edx)");
+    asm("  movl %ecx, 777(%edx)");
+    asm("  movl %ecx, 778(%edx)");
+    asm("  movl %ecx, 779(%edx)");
+    asm("  movl %ecx, 780(%edx)");
+    asm("  movl %ecx, 781(%edx)");
+    asm("  movl %ecx, 782(%edx)");
+    asm("  movl %ecx, 783(%edx)");
+    asm("  movl %ecx, 784(%edx)");
+    asm("  movl %ecx, 785(%edx)");
+    asm("  movl %ecx, 786(%edx)");
+    asm("  movl %ecx, 787(%edx)");
+    asm("  movl %ecx, 788(%edx)");
+    asm("  movl %ecx, 789(%edx)");
+    asm("  movl %ecx, 790(%edx)");
+    asm("  movl %ecx, 791(%edx)");
+    asm("  movl %ecx, 792(%edx)");
+    asm("  movl %ecx, 793(%edx)");
+    asm("  movl %ecx, 794(%edx)");
+    asm("  movl %ecx, 795(%edx)");
+    asm("  movl %ecx, 796(%edx)");
+    asm("  movl %ecx, 797(%edx)");
+    asm("  movl %ecx, 798(%edx)");
+    asm("  movl %ecx, 799(%edx)");
+    asm("  movl %ecx, 800(%edx)");
+    asm("  movl %ecx, 801(%edx)");
+    asm("  movl %ecx, 802(%edx)");
+    asm("  movl %ecx, 803(%edx)");
+    asm("  movl %ecx, 804(%edx)");
+    asm("  movl %ecx, 805(%edx)");
+    asm("  movl %ecx, 806(%edx)");
+    asm("  movl %ecx, 807(%edx)");
+    asm("  movl %ecx, 808(%edx)");
+    asm("  movl %ecx, 809(%edx)");
+    asm("  movl %ecx, 810(%edx)");
+    asm("  movl %ecx, 811(%edx)");
+    asm("  movl %ecx, 812(%edx)");
+    asm("  movl %ecx, 813(%edx)");
+    asm("  movl %ecx, 814(%edx)");
+    asm("  movl %ecx, 815(%edx)");
+    asm("  movl %ecx, 816(%edx)");
+    asm("  movl %ecx, 817(%edx)");
+    asm("  movl %ecx, 818(%edx)");
+    asm("  movl %ecx, 819(%edx)");
+    asm("  movl %ecx, 820(%edx)");
+    asm("  movl %ecx, 821(%edx)");
+    asm("  movl %ecx, 822(%edx)");
+    asm("  movl %ecx, 823(%edx)");
+    asm("  movl %ecx, 824(%edx)");
+    asm("  movl %ecx, 825(%edx)");
+    asm("  movl %ecx, 826(%edx)");
+    asm("  movl %ecx, 827(%edx)");
+    asm("  movl %ecx, 828(%edx)");
+    asm("  movl %ecx, 829(%edx)");
+    asm("  movl %ecx, 830(%edx)");
+    asm("  movl %ecx, 831(%edx)");
+    asm("  movl %ecx, 832(%edx)");
+    asm("  movl %ecx, 833(%edx)");
+    asm("  movl %ecx, 834(%edx)");
+    asm("  movl %ecx, 835(%edx)");
+    asm("  movl %ecx, 836(%edx)");
+    asm("  movl %ecx, 837(%edx)");
+    asm("  movl %ecx, 838(%edx)");
+    asm("  movl %ecx, 839(%edx)");
+    asm("  movl %ecx, 840(%edx)");
+    asm("  movl %ecx, 841(%edx)");
+    asm("  movl %ecx, 842(%edx)");
+    asm("  movl %ecx, 843(%edx)");
+    asm("  movl %ecx, 844(%edx)");
+    asm("  movl %ecx, 845(%edx)");
+    asm("  movl %ecx, 846(%edx)");
+    asm("  movl %ecx, 847(%edx)");
+    asm("  movl %ecx, 848(%edx)");
+    asm("  movl %ecx, 849(%edx)");
+    asm("  movl %ecx, 850(%edx)");
+    asm("  movl %ecx, 851(%edx)");
+    asm("  movl %ecx, 852(%edx)");
+    asm("  movl %ecx, 853(%edx)");
+    asm("  movl %ecx, 854(%edx)");
+    asm("  movl %ecx, 855(%edx)");
+    asm("  movl %ecx, 856(%edx)");
+    asm("  movl %ecx, 857(%edx)");
+    asm("  movl %ecx, 858(%edx)");
+    asm("  movl %ecx, 859(%edx)");
+    asm("  movl %ecx, 860(%edx)");
+    asm("  movl %ecx, 861(%edx)");
+    asm("  movl %ecx, 862(%edx)");
+    asm("  movl %ecx, 863(%edx)");
+    asm("  movl %ecx, 864(%edx)");
+    asm("  movl %ecx, 865(%edx)");
+    asm("  movl %ecx, 866(%edx)");
+    asm("  movl %ecx, 867(%edx)");
+    asm("  movl %ecx, 868(%edx)");
+    asm("  movl %ecx, 869(%edx)");
+    asm("  movl %ecx, 870(%edx)");
+    asm("  movl %ecx, 871(%edx)");
+    asm("  movl %ecx, 872(%edx)");
+    asm("  movl %ecx, 873(%edx)");
+    asm("  movl %ecx, 874(%edx)");
+    asm("  movl %ecx, 875(%edx)");
+    asm("  movl %ecx, 876(%edx)");
+    asm("  movl %ecx, 877(%edx)");
+    asm("  movl %ecx, 878(%edx)");
+    asm("  movl %ecx, 879(%edx)");
+    asm("  movl %ecx, 880(%edx)");
+    asm("  movl %ecx, 881(%edx)");
+    asm("  movl %ecx, 882(%edx)");
+    asm("  movl %ecx, 883(%edx)");
+    asm("  movl %ecx, 884(%edx)");
+    asm("  movl %ecx, 885(%edx)");
+    asm("  movl %ecx, 886(%edx)");
+    asm("  movl %ecx, 887(%edx)");
+    asm("  movl %ecx, 888(%edx)");
+    asm("  movl %ecx, 889(%edx)");
+    asm("  movl %ecx, 890(%edx)");
+    asm("  movl %ecx, 891(%edx)");
+    asm("  movl %ecx, 892(%edx)");
+    asm("  movl %ecx, 893(%edx)");
+    asm("  movl %ecx, 894(%edx)");
+    asm("  movl %ecx, 895(%edx)");
+    asm("  movl %ecx, 896(%edx)");
+    asm("  movl %ecx, 897(%edx)");
+    asm("  movl %ecx, 898(%edx)");
+    asm("  movl %ecx, 899(%edx)");
+
+    asm("  movl $0x0,%ecx"); /* counter for diagnostics */
+    asm("repeata:");
+    asm("  decl %eax");
+    asm("  inc  %ecx");
+    asm("  cmpl $0x0,%eax");
+    asm("  jnz repeata");
+    asm("  movl %%ecx, %0" : "=r" (total));
+#else
+    __asm {
+        call next_inst_prot
+      next_inst_prot:
+        pop  edx
+        push ALLOW_READ|ALLOW_WRITE|ALLOW_EXEC
+        push PAGE_SIZE*3
+        push edx
+        call protect_mem
+        add  esp, 12
+
+        mov  ecx, iters
+        call next_inst
+      next_inst:
+        pop  edx
+    /* add to retaddr: 1 == pop
+     *                 3 == mov ecx into target
+     *                 1 == opcode of target movl
+     */
+        mov  dword ptr [edx + 0x5], ecx /* the modifying store */
+        mov  eax,0x12345678 /* this instr's immed gets overwritten */
+
+        /* now we have as many write instrs as necessary to cause a too-big
+         * selfmod fragment.  xref case 7893.
+         */
+        mov  ecx, iters
+        mov  edx, offset big
+        mov  dword ptr [edx + 0], ecx
+        mov  dword ptr [edx + 1], ecx
+        mov  dword ptr [edx + 2], ecx
+        mov  dword ptr [edx + 3], ecx
+        mov  dword ptr [edx + 4], ecx
+        mov  dword ptr [edx + 5], ecx
+        mov  dword ptr [edx + 6], ecx
+        mov  dword ptr [edx + 7], ecx
+        mov  dword ptr [edx + 8], ecx
+        mov  dword ptr [edx + 9], ecx
+        mov  dword ptr [edx + 10], ecx
+        mov  dword ptr [edx + 11], ecx
+        mov  dword ptr [edx + 12], ecx
+        mov  dword ptr [edx + 13], ecx
+        mov  dword ptr [edx + 14], ecx
+        mov  dword ptr [edx + 15], ecx
+        mov  dword ptr [edx + 16], ecx
+        mov  dword ptr [edx + 17], ecx
+        mov  dword ptr [edx + 18], ecx
+        mov  dword ptr [edx + 19], ecx
+        mov  dword ptr [edx + 20], ecx
+        mov  dword ptr [edx + 21], ecx
+        mov  dword ptr [edx + 22], ecx
+        mov  dword ptr [edx + 23], ecx
+        mov  dword ptr [edx + 24], ecx
+        mov  dword ptr [edx + 25], ecx
+        mov  dword ptr [edx + 26], ecx
+        mov  dword ptr [edx + 27], ecx
+        mov  dword ptr [edx + 28], ecx
+        mov  dword ptr [edx + 29], ecx
+        mov  dword ptr [edx + 30], ecx
+        mov  dword ptr [edx + 31], ecx
+        mov  dword ptr [edx + 32], ecx
+        mov  dword ptr [edx + 33], ecx
+        mov  dword ptr [edx + 34], ecx
+        mov  dword ptr [edx + 35], ecx
+        mov  dword ptr [edx + 36], ecx
+        mov  dword ptr [edx + 37], ecx
+        mov  dword ptr [edx + 38], ecx
+        mov  dword ptr [edx + 39], ecx
+        mov  dword ptr [edx + 40], ecx
+        mov  dword ptr [edx + 41], ecx
+        mov  dword ptr [edx + 42], ecx
+        mov  dword ptr [edx + 43], ecx
+        mov  dword ptr [edx + 44], ecx
+        mov  dword ptr [edx + 45], ecx
+        mov  dword ptr [edx + 46], ecx
+        mov  dword ptr [edx + 47], ecx
+        mov  dword ptr [edx + 48], ecx
+        mov  dword ptr [edx + 49], ecx
+        mov  dword ptr [edx + 50], ecx
+        mov  dword ptr [edx + 51], ecx
+        mov  dword ptr [edx + 52], ecx
+        mov  dword ptr [edx + 53], ecx
+        mov  dword ptr [edx + 54], ecx
+        mov  dword ptr [edx + 55], ecx
+        mov  dword ptr [edx + 56], ecx
+        mov  dword ptr [edx + 57], ecx
+        mov  dword ptr [edx + 58], ecx
+        mov  dword ptr [edx + 59], ecx
+        mov  dword ptr [edx + 60], ecx
+        mov  dword ptr [edx + 61], ecx
+        mov  dword ptr [edx + 62], ecx
+        mov  dword ptr [edx + 63], ecx
+        mov  dword ptr [edx + 64], ecx
+        mov  dword ptr [edx + 65], ecx
+        mov  dword ptr [edx + 66], ecx
+        mov  dword ptr [edx + 67], ecx
+        mov  dword ptr [edx + 68], ecx
+        mov  dword ptr [edx + 69], ecx
+        mov  dword ptr [edx + 70], ecx
+        mov  dword ptr [edx + 71], ecx
+        mov  dword ptr [edx + 72], ecx
+        mov  dword ptr [edx + 73], ecx
+        mov  dword ptr [edx + 74], ecx
+        mov  dword ptr [edx + 75], ecx
+        mov  dword ptr [edx + 76], ecx
+        mov  dword ptr [edx + 77], ecx
+        mov  dword ptr [edx + 78], ecx
+        mov  dword ptr [edx + 79], ecx
+        mov  dword ptr [edx + 80], ecx
+        mov  dword ptr [edx + 81], ecx
+        mov  dword ptr [edx + 82], ecx
+        mov  dword ptr [edx + 83], ecx
+        mov  dword ptr [edx + 84], ecx
+        mov  dword ptr [edx + 85], ecx
+        mov  dword ptr [edx + 86], ecx
+        mov  dword ptr [edx + 87], ecx
+        mov  dword ptr [edx + 88], ecx
+        mov  dword ptr [edx + 89], ecx
+        mov  dword ptr [edx + 90], ecx
+        mov  dword ptr [edx + 91], ecx
+        mov  dword ptr [edx + 92], ecx
+        mov  dword ptr [edx + 93], ecx
+        mov  dword ptr [edx + 94], ecx
+        mov  dword ptr [edx + 95], ecx
+        mov  dword ptr [edx + 96], ecx
+        mov  dword ptr [edx + 97], ecx
+        mov  dword ptr [edx + 98], ecx
+        mov  dword ptr [edx + 99], ecx
+        mov  dword ptr [edx + 100], ecx
+        mov  dword ptr [edx + 101], ecx
+        mov  dword ptr [edx + 102], ecx
+        mov  dword ptr [edx + 103], ecx
+        mov  dword ptr [edx + 104], ecx
+        mov  dword ptr [edx + 105], ecx
+        mov  dword ptr [edx + 106], ecx
+        mov  dword ptr [edx + 107], ecx
+        mov  dword ptr [edx + 108], ecx
+        mov  dword ptr [edx + 109], ecx
+        mov  dword ptr [edx + 110], ecx
+        mov  dword ptr [edx + 111], ecx
+        mov  dword ptr [edx + 112], ecx
+        mov  dword ptr [edx + 113], ecx
+        mov  dword ptr [edx + 114], ecx
+        mov  dword ptr [edx + 115], ecx
+        mov  dword ptr [edx + 116], ecx
+        mov  dword ptr [edx + 117], ecx
+        mov  dword ptr [edx + 118], ecx
+        mov  dword ptr [edx + 119], ecx
+        mov  dword ptr [edx + 120], ecx
+        mov  dword ptr [edx + 121], ecx
+        mov  dword ptr [edx + 122], ecx
+        mov  dword ptr [edx + 123], ecx
+        mov  dword ptr [edx + 124], ecx
+        mov  dword ptr [edx + 125], ecx
+        mov  dword ptr [edx + 126], ecx
+        mov  dword ptr [edx + 127], ecx
+        mov  dword ptr [edx + 128], ecx
+        mov  dword ptr [edx + 129], ecx
+        mov  dword ptr [edx + 130], ecx
+        mov  dword ptr [edx + 131], ecx
+        mov  dword ptr [edx + 132], ecx
+        mov  dword ptr [edx + 133], ecx
+        mov  dword ptr [edx + 134], ecx
+        mov  dword ptr [edx + 135], ecx
+        mov  dword ptr [edx + 136], ecx
+        mov  dword ptr [edx + 137], ecx
+        mov  dword ptr [edx + 138], ecx
+        mov  dword ptr [edx + 139], ecx
+        mov  dword ptr [edx + 140], ecx
+        mov  dword ptr [edx + 141], ecx
+        mov  dword ptr [edx + 142], ecx
+        mov  dword ptr [edx + 143], ecx
+        mov  dword ptr [edx + 144], ecx
+        mov  dword ptr [edx + 145], ecx
+        mov  dword ptr [edx + 146], ecx
+        mov  dword ptr [edx + 147], ecx
+        mov  dword ptr [edx + 148], ecx
+        mov  dword ptr [edx + 149], ecx
+        mov  dword ptr [edx + 150], ecx
+        mov  dword ptr [edx + 151], ecx
+        mov  dword ptr [edx + 152], ecx
+        mov  dword ptr [edx + 153], ecx
+        mov  dword ptr [edx + 154], ecx
+        mov  dword ptr [edx + 155], ecx
+        mov  dword ptr [edx + 156], ecx
+        mov  dword ptr [edx + 157], ecx
+        mov  dword ptr [edx + 158], ecx
+        mov  dword ptr [edx + 159], ecx
+        mov  dword ptr [edx + 160], ecx
+        mov  dword ptr [edx + 161], ecx
+        mov  dword ptr [edx + 162], ecx
+        mov  dword ptr [edx + 163], ecx
+        mov  dword ptr [edx + 164], ecx
+        mov  dword ptr [edx + 165], ecx
+        mov  dword ptr [edx + 166], ecx
+        mov  dword ptr [edx + 167], ecx
+        mov  dword ptr [edx + 168], ecx
+        mov  dword ptr [edx + 169], ecx
+        mov  dword ptr [edx + 170], ecx
+        mov  dword ptr [edx + 171], ecx
+        mov  dword ptr [edx + 172], ecx
+        mov  dword ptr [edx + 173], ecx
+        mov  dword ptr [edx + 174], ecx
+        mov  dword ptr [edx + 175], ecx
+        mov  dword ptr [edx + 176], ecx
+        mov  dword ptr [edx + 177], ecx
+        mov  dword ptr [edx + 178], ecx
+        mov  dword ptr [edx + 179], ecx
+        mov  dword ptr [edx + 180], ecx
+        mov  dword ptr [edx + 181], ecx
+        mov  dword ptr [edx + 182], ecx
+        mov  dword ptr [edx + 183], ecx
+        mov  dword ptr [edx + 184], ecx
+        mov  dword ptr [edx + 185], ecx
+        mov  dword ptr [edx + 186], ecx
+        mov  dword ptr [edx + 187], ecx
+        mov  dword ptr [edx + 188], ecx
+        mov  dword ptr [edx + 189], ecx
+        mov  dword ptr [edx + 190], ecx
+        mov  dword ptr [edx + 191], ecx
+        mov  dword ptr [edx + 192], ecx
+        mov  dword ptr [edx + 193], ecx
+        mov  dword ptr [edx + 194], ecx
+        mov  dword ptr [edx + 195], ecx
+        mov  dword ptr [edx + 196], ecx
+        mov  dword ptr [edx + 197], ecx
+        mov  dword ptr [edx + 198], ecx
+        mov  dword ptr [edx + 199], ecx
+        mov  dword ptr [edx + 200], ecx
+        mov  dword ptr [edx + 201], ecx
+        mov  dword ptr [edx + 202], ecx
+        mov  dword ptr [edx + 203], ecx
+        mov  dword ptr [edx + 204], ecx
+        mov  dword ptr [edx + 205], ecx
+        mov  dword ptr [edx + 206], ecx
+        mov  dword ptr [edx + 207], ecx
+        mov  dword ptr [edx + 208], ecx
+        mov  dword ptr [edx + 209], ecx
+        mov  dword ptr [edx + 210], ecx
+        mov  dword ptr [edx + 211], ecx
+        mov  dword ptr [edx + 212], ecx
+        mov  dword ptr [edx + 213], ecx
+        mov  dword ptr [edx + 214], ecx
+        mov  dword ptr [edx + 215], ecx
+        mov  dword ptr [edx + 216], ecx
+        mov  dword ptr [edx + 217], ecx
+        mov  dword ptr [edx + 218], ecx
+        mov  dword ptr [edx + 219], ecx
+        mov  dword ptr [edx + 220], ecx
+        mov  dword ptr [edx + 221], ecx
+        mov  dword ptr [edx + 222], ecx
+        mov  dword ptr [edx + 223], ecx
+        mov  dword ptr [edx + 224], ecx
+        mov  dword ptr [edx + 225], ecx
+        mov  dword ptr [edx + 226], ecx
+        mov  dword ptr [edx + 227], ecx
+        mov  dword ptr [edx + 228], ecx
+        mov  dword ptr [edx + 229], ecx
+        mov  dword ptr [edx + 230], ecx
+        mov  dword ptr [edx + 231], ecx
+        mov  dword ptr [edx + 232], ecx
+        mov  dword ptr [edx + 233], ecx
+        mov  dword ptr [edx + 234], ecx
+        mov  dword ptr [edx + 235], ecx
+        mov  dword ptr [edx + 236], ecx
+        mov  dword ptr [edx + 237], ecx
+        mov  dword ptr [edx + 238], ecx
+        mov  dword ptr [edx + 239], ecx
+        mov  dword ptr [edx + 240], ecx
+        mov  dword ptr [edx + 241], ecx
+        mov  dword ptr [edx + 242], ecx
+        mov  dword ptr [edx + 243], ecx
+        mov  dword ptr [edx + 244], ecx
+        mov  dword ptr [edx + 245], ecx
+        mov  dword ptr [edx + 246], ecx
+        mov  dword ptr [edx + 247], ecx
+        mov  dword ptr [edx + 248], ecx
+        mov  dword ptr [edx + 249], ecx
+        mov  dword ptr [edx + 250], ecx
+        mov  dword ptr [edx + 251], ecx
+        mov  dword ptr [edx + 252], ecx
+        mov  dword ptr [edx + 253], ecx
+        mov  dword ptr [edx + 254], ecx
+        mov  dword ptr [edx + 255], ecx
+        mov  dword ptr [edx + 256], ecx
+        mov  dword ptr [edx + 257], ecx
+        mov  dword ptr [edx + 258], ecx
+        mov  dword ptr [edx + 259], ecx
+        mov  dword ptr [edx + 260], ecx
+        mov  dword ptr [edx + 261], ecx
+        mov  dword ptr [edx + 262], ecx
+        mov  dword ptr [edx + 263], ecx
+        mov  dword ptr [edx + 264], ecx
+        mov  dword ptr [edx + 265], ecx
+        mov  dword ptr [edx + 266], ecx
+        mov  dword ptr [edx + 267], ecx
+        mov  dword ptr [edx + 268], ecx
+        mov  dword ptr [edx + 269], ecx
+        mov  dword ptr [edx + 270], ecx
+        mov  dword ptr [edx + 271], ecx
+        mov  dword ptr [edx + 272], ecx
+        mov  dword ptr [edx + 273], ecx
+        mov  dword ptr [edx + 274], ecx
+        mov  dword ptr [edx + 275], ecx
+        mov  dword ptr [edx + 276], ecx
+        mov  dword ptr [edx + 277], ecx
+        mov  dword ptr [edx + 278], ecx
+        mov  dword ptr [edx + 279], ecx
+        mov  dword ptr [edx + 280], ecx
+        mov  dword ptr [edx + 281], ecx
+        mov  dword ptr [edx + 282], ecx
+        mov  dword ptr [edx + 283], ecx
+        mov  dword ptr [edx + 284], ecx
+        mov  dword ptr [edx + 285], ecx
+        mov  dword ptr [edx + 286], ecx
+        mov  dword ptr [edx + 287], ecx
+        mov  dword ptr [edx + 288], ecx
+        mov  dword ptr [edx + 289], ecx
+        mov  dword ptr [edx + 290], ecx
+        mov  dword ptr [edx + 291], ecx
+        mov  dword ptr [edx + 292], ecx
+        mov  dword ptr [edx + 293], ecx
+        mov  dword ptr [edx + 294], ecx
+        mov  dword ptr [edx + 295], ecx
+        mov  dword ptr [edx + 296], ecx
+        mov  dword ptr [edx + 297], ecx
+        mov  dword ptr [edx + 298], ecx
+        mov  dword ptr [edx + 299], ecx
+        mov  dword ptr [edx + 300], ecx
+        mov  dword ptr [edx + 301], ecx
+        mov  dword ptr [edx + 302], ecx
+        mov  dword ptr [edx + 303], ecx
+        mov  dword ptr [edx + 304], ecx
+        mov  dword ptr [edx + 305], ecx
+        mov  dword ptr [edx + 306], ecx
+        mov  dword ptr [edx + 307], ecx
+        mov  dword ptr [edx + 308], ecx
+        mov  dword ptr [edx + 309], ecx
+        mov  dword ptr [edx + 310], ecx
+        mov  dword ptr [edx + 311], ecx
+        mov  dword ptr [edx + 312], ecx
+        mov  dword ptr [edx + 313], ecx
+        mov  dword ptr [edx + 314], ecx
+        mov  dword ptr [edx + 315], ecx
+        mov  dword ptr [edx + 316], ecx
+        mov  dword ptr [edx + 317], ecx
+        mov  dword ptr [edx + 318], ecx
+        mov  dword ptr [edx + 319], ecx
+        mov  dword ptr [edx + 320], ecx
+        mov  dword ptr [edx + 321], ecx
+        mov  dword ptr [edx + 322], ecx
+        mov  dword ptr [edx + 323], ecx
+        mov  dword ptr [edx + 324], ecx
+        mov  dword ptr [edx + 325], ecx
+        mov  dword ptr [edx + 326], ecx
+        mov  dword ptr [edx + 327], ecx
+        mov  dword ptr [edx + 328], ecx
+        mov  dword ptr [edx + 329], ecx
+        mov  dword ptr [edx + 330], ecx
+        mov  dword ptr [edx + 331], ecx
+        mov  dword ptr [edx + 332], ecx
+        mov  dword ptr [edx + 333], ecx
+        mov  dword ptr [edx + 334], ecx
+        mov  dword ptr [edx + 335], ecx
+        mov  dword ptr [edx + 336], ecx
+        mov  dword ptr [edx + 337], ecx
+        mov  dword ptr [edx + 338], ecx
+        mov  dword ptr [edx + 339], ecx
+        mov  dword ptr [edx + 340], ecx
+        mov  dword ptr [edx + 341], ecx
+        mov  dword ptr [edx + 342], ecx
+        mov  dword ptr [edx + 343], ecx
+        mov  dword ptr [edx + 344], ecx
+        mov  dword ptr [edx + 345], ecx
+        mov  dword ptr [edx + 346], ecx
+        mov  dword ptr [edx + 347], ecx
+        mov  dword ptr [edx + 348], ecx
+        mov  dword ptr [edx + 349], ecx
+        mov  dword ptr [edx + 350], ecx
+        mov  dword ptr [edx + 351], ecx
+        mov  dword ptr [edx + 352], ecx
+        mov  dword ptr [edx + 353], ecx
+        mov  dword ptr [edx + 354], ecx
+        mov  dword ptr [edx + 355], ecx
+        mov  dword ptr [edx + 356], ecx
+        mov  dword ptr [edx + 357], ecx
+        mov  dword ptr [edx + 358], ecx
+        mov  dword ptr [edx + 359], ecx
+        mov  dword ptr [edx + 360], ecx
+        mov  dword ptr [edx + 361], ecx
+        mov  dword ptr [edx + 362], ecx
+        mov  dword ptr [edx + 363], ecx
+        mov  dword ptr [edx + 364], ecx
+        mov  dword ptr [edx + 365], ecx
+        mov  dword ptr [edx + 366], ecx
+        mov  dword ptr [edx + 367], ecx
+        mov  dword ptr [edx + 368], ecx
+        mov  dword ptr [edx + 369], ecx
+        mov  dword ptr [edx + 370], ecx
+        mov  dword ptr [edx + 371], ecx
+        mov  dword ptr [edx + 372], ecx
+        mov  dword ptr [edx + 373], ecx
+        mov  dword ptr [edx + 374], ecx
+        mov  dword ptr [edx + 375], ecx
+        mov  dword ptr [edx + 376], ecx
+        mov  dword ptr [edx + 377], ecx
+        mov  dword ptr [edx + 378], ecx
+        mov  dword ptr [edx + 379], ecx
+        mov  dword ptr [edx + 380], ecx
+        mov  dword ptr [edx + 381], ecx
+        mov  dword ptr [edx + 382], ecx
+        mov  dword ptr [edx + 383], ecx
+        mov  dword ptr [edx + 384], ecx
+        mov  dword ptr [edx + 385], ecx
+        mov  dword ptr [edx + 386], ecx
+        mov  dword ptr [edx + 387], ecx
+        mov  dword ptr [edx + 388], ecx
+        mov  dword ptr [edx + 389], ecx
+        mov  dword ptr [edx + 390], ecx
+        mov  dword ptr [edx + 391], ecx
+        mov  dword ptr [edx + 392], ecx
+        mov  dword ptr [edx + 393], ecx
+        mov  dword ptr [edx + 394], ecx
+        mov  dword ptr [edx + 395], ecx
+        mov  dword ptr [edx + 396], ecx
+        mov  dword ptr [edx + 397], ecx
+        mov  dword ptr [edx + 398], ecx
+        mov  dword ptr [edx + 399], ecx
+        mov  dword ptr [edx + 400], ecx
+        mov  dword ptr [edx + 401], ecx
+        mov  dword ptr [edx + 402], ecx
+        mov  dword ptr [edx + 403], ecx
+        mov  dword ptr [edx + 404], ecx
+        mov  dword ptr [edx + 405], ecx
+        mov  dword ptr [edx + 406], ecx
+        mov  dword ptr [edx + 407], ecx
+        mov  dword ptr [edx + 408], ecx
+        mov  dword ptr [edx + 409], ecx
+        mov  dword ptr [edx + 410], ecx
+        mov  dword ptr [edx + 411], ecx
+        mov  dword ptr [edx + 412], ecx
+        mov  dword ptr [edx + 413], ecx
+        mov  dword ptr [edx + 414], ecx
+        mov  dword ptr [edx + 415], ecx
+        mov  dword ptr [edx + 416], ecx
+        mov  dword ptr [edx + 417], ecx
+        mov  dword ptr [edx + 418], ecx
+        mov  dword ptr [edx + 419], ecx
+        mov  dword ptr [edx + 420], ecx
+        mov  dword ptr [edx + 421], ecx
+        mov  dword ptr [edx + 422], ecx
+        mov  dword ptr [edx + 423], ecx
+        mov  dword ptr [edx + 424], ecx
+        mov  dword ptr [edx + 425], ecx
+        mov  dword ptr [edx + 426], ecx
+        mov  dword ptr [edx + 427], ecx
+        mov  dword ptr [edx + 428], ecx
+        mov  dword ptr [edx + 429], ecx
+        mov  dword ptr [edx + 430], ecx
+        mov  dword ptr [edx + 431], ecx
+        mov  dword ptr [edx + 432], ecx
+        mov  dword ptr [edx + 433], ecx
+        mov  dword ptr [edx + 434], ecx
+        mov  dword ptr [edx + 435], ecx
+        mov  dword ptr [edx + 436], ecx
+        mov  dword ptr [edx + 437], ecx
+        mov  dword ptr [edx + 438], ecx
+        mov  dword ptr [edx + 439], ecx
+        mov  dword ptr [edx + 440], ecx
+        mov  dword ptr [edx + 441], ecx
+        mov  dword ptr [edx + 442], ecx
+        mov  dword ptr [edx + 443], ecx
+        mov  dword ptr [edx + 444], ecx
+        mov  dword ptr [edx + 445], ecx
+        mov  dword ptr [edx + 446], ecx
+        mov  dword ptr [edx + 447], ecx
+        mov  dword ptr [edx + 448], ecx
+        mov  dword ptr [edx + 449], ecx
+        mov  dword ptr [edx + 450], ecx
+        mov  dword ptr [edx + 451], ecx
+        mov  dword ptr [edx + 452], ecx
+        mov  dword ptr [edx + 453], ecx
+        mov  dword ptr [edx + 454], ecx
+        mov  dword ptr [edx + 455], ecx
+        mov  dword ptr [edx + 456], ecx
+        mov  dword ptr [edx + 457], ecx
+        mov  dword ptr [edx + 458], ecx
+        mov  dword ptr [edx + 459], ecx
+        mov  dword ptr [edx + 460], ecx
+        mov  dword ptr [edx + 461], ecx
+        mov  dword ptr [edx + 462], ecx
+        mov  dword ptr [edx + 463], ecx
+        mov  dword ptr [edx + 464], ecx
+        mov  dword ptr [edx + 465], ecx
+        mov  dword ptr [edx + 466], ecx
+        mov  dword ptr [edx + 467], ecx
+        mov  dword ptr [edx + 468], ecx
+        mov  dword ptr [edx + 469], ecx
+        mov  dword ptr [edx + 470], ecx
+        mov  dword ptr [edx + 471], ecx
+        mov  dword ptr [edx + 472], ecx
+        mov  dword ptr [edx + 473], ecx
+        mov  dword ptr [edx + 474], ecx
+        mov  dword ptr [edx + 475], ecx
+        mov  dword ptr [edx + 476], ecx
+        mov  dword ptr [edx + 477], ecx
+        mov  dword ptr [edx + 478], ecx
+        mov  dword ptr [edx + 479], ecx
+        mov  dword ptr [edx + 480], ecx
+        mov  dword ptr [edx + 481], ecx
+        mov  dword ptr [edx + 482], ecx
+        mov  dword ptr [edx + 483], ecx
+        mov  dword ptr [edx + 484], ecx
+        mov  dword ptr [edx + 485], ecx
+        mov  dword ptr [edx + 486], ecx
+        mov  dword ptr [edx + 487], ecx
+        mov  dword ptr [edx + 488], ecx
+        mov  dword ptr [edx + 489], ecx
+        mov  dword ptr [edx + 490], ecx
+        mov  dword ptr [edx + 491], ecx
+        mov  dword ptr [edx + 492], ecx
+        mov  dword ptr [edx + 493], ecx
+        mov  dword ptr [edx + 494], ecx
+        mov  dword ptr [edx + 495], ecx
+        mov  dword ptr [edx + 496], ecx
+        mov  dword ptr [edx + 497], ecx
+        mov  dword ptr [edx + 498], ecx
+        mov  dword ptr [edx + 499], ecx
+        mov  dword ptr [edx + 500], ecx
+        mov  dword ptr [edx + 501], ecx
+        mov  dword ptr [edx + 502], ecx
+        mov  dword ptr [edx + 503], ecx
+        mov  dword ptr [edx + 504], ecx
+        mov  dword ptr [edx + 505], ecx
+        mov  dword ptr [edx + 506], ecx
+        mov  dword ptr [edx + 507], ecx
+        mov  dword ptr [edx + 508], ecx
+        mov  dword ptr [edx + 509], ecx
+        mov  dword ptr [edx + 510], ecx
+        mov  dword ptr [edx + 511], ecx
+        mov  dword ptr [edx + 512], ecx
+        mov  dword ptr [edx + 513], ecx
+        mov  dword ptr [edx + 514], ecx
+        mov  dword ptr [edx + 515], ecx
+        mov  dword ptr [edx + 516], ecx
+        mov  dword ptr [edx + 517], ecx
+        mov  dword ptr [edx + 518], ecx
+        mov  dword ptr [edx + 519], ecx
+        mov  dword ptr [edx + 520], ecx
+        mov  dword ptr [edx + 521], ecx
+        mov  dword ptr [edx + 522], ecx
+        mov  dword ptr [edx + 523], ecx
+        mov  dword ptr [edx + 524], ecx
+        mov  dword ptr [edx + 525], ecx
+        mov  dword ptr [edx + 526], ecx
+        mov  dword ptr [edx + 527], ecx
+        mov  dword ptr [edx + 528], ecx
+        mov  dword ptr [edx + 529], ecx
+        mov  dword ptr [edx + 530], ecx
+        mov  dword ptr [edx + 531], ecx
+        mov  dword ptr [edx + 532], ecx
+        mov  dword ptr [edx + 533], ecx
+        mov  dword ptr [edx + 534], ecx
+        mov  dword ptr [edx + 535], ecx
+        mov  dword ptr [edx + 536], ecx
+        mov  dword ptr [edx + 537], ecx
+        mov  dword ptr [edx + 538], ecx
+        mov  dword ptr [edx + 539], ecx
+        mov  dword ptr [edx + 540], ecx
+        mov  dword ptr [edx + 541], ecx
+        mov  dword ptr [edx + 542], ecx
+        mov  dword ptr [edx + 543], ecx
+        mov  dword ptr [edx + 544], ecx
+        mov  dword ptr [edx + 545], ecx
+        mov  dword ptr [edx + 546], ecx
+        mov  dword ptr [edx + 547], ecx
+        mov  dword ptr [edx + 548], ecx
+        mov  dword ptr [edx + 549], ecx
+        mov  dword ptr [edx + 550], ecx
+        mov  dword ptr [edx + 551], ecx
+        mov  dword ptr [edx + 552], ecx
+        mov  dword ptr [edx + 553], ecx
+        mov  dword ptr [edx + 554], ecx
+        mov  dword ptr [edx + 555], ecx
+        mov  dword ptr [edx + 556], ecx
+        mov  dword ptr [edx + 557], ecx
+        mov  dword ptr [edx + 558], ecx
+        mov  dword ptr [edx + 559], ecx
+        mov  dword ptr [edx + 560], ecx
+        mov  dword ptr [edx + 561], ecx
+        mov  dword ptr [edx + 562], ecx
+        mov  dword ptr [edx + 563], ecx
+        mov  dword ptr [edx + 564], ecx
+        mov  dword ptr [edx + 565], ecx
+        mov  dword ptr [edx + 566], ecx
+        mov  dword ptr [edx + 567], ecx
+        mov  dword ptr [edx + 568], ecx
+        mov  dword ptr [edx + 569], ecx
+        mov  dword ptr [edx + 570], ecx
+        mov  dword ptr [edx + 571], ecx
+        mov  dword ptr [edx + 572], ecx
+        mov  dword ptr [edx + 573], ecx
+        mov  dword ptr [edx + 574], ecx
+        mov  dword ptr [edx + 575], ecx
+        mov  dword ptr [edx + 576], ecx
+        mov  dword ptr [edx + 577], ecx
+        mov  dword ptr [edx + 578], ecx
+        mov  dword ptr [edx + 579], ecx
+        mov  dword ptr [edx + 580], ecx
+        mov  dword ptr [edx + 581], ecx
+        mov  dword ptr [edx + 582], ecx
+        mov  dword ptr [edx + 583], ecx
+        mov  dword ptr [edx + 584], ecx
+        mov  dword ptr [edx + 585], ecx
+        mov  dword ptr [edx + 586], ecx
+        mov  dword ptr [edx + 587], ecx
+        mov  dword ptr [edx + 588], ecx
+        mov  dword ptr [edx + 589], ecx
+        mov  dword ptr [edx + 590], ecx
+        mov  dword ptr [edx + 591], ecx
+        mov  dword ptr [edx + 592], ecx
+        mov  dword ptr [edx + 593], ecx
+        mov  dword ptr [edx + 594], ecx
+        mov  dword ptr [edx + 595], ecx
+        mov  dword ptr [edx + 596], ecx
+        mov  dword ptr [edx + 597], ecx
+        mov  dword ptr [edx + 598], ecx
+        mov  dword ptr [edx + 599], ecx
+        mov  dword ptr [edx + 600], ecx
+        mov  dword ptr [edx + 601], ecx
+        mov  dword ptr [edx + 602], ecx
+        mov  dword ptr [edx + 603], ecx
+        mov  dword ptr [edx + 604], ecx
+        mov  dword ptr [edx + 605], ecx
+        mov  dword ptr [edx + 606], ecx
+        mov  dword ptr [edx + 607], ecx
+        mov  dword ptr [edx + 608], ecx
+        mov  dword ptr [edx + 609], ecx
+        mov  dword ptr [edx + 610], ecx
+        mov  dword ptr [edx + 611], ecx
+        mov  dword ptr [edx + 612], ecx
+        mov  dword ptr [edx + 613], ecx
+        mov  dword ptr [edx + 614], ecx
+        mov  dword ptr [edx + 615], ecx
+        mov  dword ptr [edx + 616], ecx
+        mov  dword ptr [edx + 617], ecx
+        mov  dword ptr [edx + 618], ecx
+        mov  dword ptr [edx + 619], ecx
+        mov  dword ptr [edx + 620], ecx
+        mov  dword ptr [edx + 621], ecx
+        mov  dword ptr [edx + 622], ecx
+        mov  dword ptr [edx + 623], ecx
+        mov  dword ptr [edx + 624], ecx
+        mov  dword ptr [edx + 625], ecx
+        mov  dword ptr [edx + 626], ecx
+        mov  dword ptr [edx + 627], ecx
+        mov  dword ptr [edx + 628], ecx
+        mov  dword ptr [edx + 629], ecx
+        mov  dword ptr [edx + 630], ecx
+        mov  dword ptr [edx + 631], ecx
+        mov  dword ptr [edx + 632], ecx
+        mov  dword ptr [edx + 633], ecx
+        mov  dword ptr [edx + 634], ecx
+        mov  dword ptr [edx + 635], ecx
+        mov  dword ptr [edx + 636], ecx
+        mov  dword ptr [edx + 637], ecx
+        mov  dword ptr [edx + 638], ecx
+        mov  dword ptr [edx + 639], ecx
+        mov  dword ptr [edx + 640], ecx
+        mov  dword ptr [edx + 641], ecx
+        mov  dword ptr [edx + 642], ecx
+        mov  dword ptr [edx + 643], ecx
+        mov  dword ptr [edx + 644], ecx
+        mov  dword ptr [edx + 645], ecx
+        mov  dword ptr [edx + 646], ecx
+        mov  dword ptr [edx + 647], ecx
+        mov  dword ptr [edx + 648], ecx
+        mov  dword ptr [edx + 649], ecx
+        mov  dword ptr [edx + 650], ecx
+        mov  dword ptr [edx + 651], ecx
+        mov  dword ptr [edx + 652], ecx
+        mov  dword ptr [edx + 653], ecx
+        mov  dword ptr [edx + 654], ecx
+        mov  dword ptr [edx + 655], ecx
+        mov  dword ptr [edx + 656], ecx
+        mov  dword ptr [edx + 657], ecx
+        mov  dword ptr [edx + 658], ecx
+        mov  dword ptr [edx + 659], ecx
+        mov  dword ptr [edx + 660], ecx
+        mov  dword ptr [edx + 661], ecx
+        mov  dword ptr [edx + 662], ecx
+        mov  dword ptr [edx + 663], ecx
+        mov  dword ptr [edx + 664], ecx
+        mov  dword ptr [edx + 665], ecx
+        mov  dword ptr [edx + 666], ecx
+        mov  dword ptr [edx + 667], ecx
+        mov  dword ptr [edx + 668], ecx
+        mov  dword ptr [edx + 669], ecx
+        mov  dword ptr [edx + 670], ecx
+        mov  dword ptr [edx + 671], ecx
+        mov  dword ptr [edx + 672], ecx
+        mov  dword ptr [edx + 673], ecx
+        mov  dword ptr [edx + 674], ecx
+        mov  dword ptr [edx + 675], ecx
+        mov  dword ptr [edx + 676], ecx
+        mov  dword ptr [edx + 677], ecx
+        mov  dword ptr [edx + 678], ecx
+        mov  dword ptr [edx + 679], ecx
+        mov  dword ptr [edx + 680], ecx
+        mov  dword ptr [edx + 681], ecx
+        mov  dword ptr [edx + 682], ecx
+        mov  dword ptr [edx + 683], ecx
+        mov  dword ptr [edx + 684], ecx
+        mov  dword ptr [edx + 685], ecx
+        mov  dword ptr [edx + 686], ecx
+        mov  dword ptr [edx + 687], ecx
+        mov  dword ptr [edx + 688], ecx
+        mov  dword ptr [edx + 689], ecx
+        mov  dword ptr [edx + 690], ecx
+        mov  dword ptr [edx + 691], ecx
+        mov  dword ptr [edx + 692], ecx
+        mov  dword ptr [edx + 693], ecx
+        mov  dword ptr [edx + 694], ecx
+        mov  dword ptr [edx + 695], ecx
+        mov  dword ptr [edx + 696], ecx
+        mov  dword ptr [edx + 697], ecx
+        mov  dword ptr [edx + 698], ecx
+        mov  dword ptr [edx + 699], ecx
+        mov  dword ptr [edx + 700], ecx
+        mov  dword ptr [edx + 701], ecx
+        mov  dword ptr [edx + 702], ecx
+        mov  dword ptr [edx + 703], ecx
+        mov  dword ptr [edx + 704], ecx
+        mov  dword ptr [edx + 705], ecx
+        mov  dword ptr [edx + 706], ecx
+        mov  dword ptr [edx + 707], ecx
+        mov  dword ptr [edx + 708], ecx
+        mov  dword ptr [edx + 709], ecx
+        mov  dword ptr [edx + 710], ecx
+        mov  dword ptr [edx + 711], ecx
+        mov  dword ptr [edx + 712], ecx
+        mov  dword ptr [edx + 713], ecx
+        mov  dword ptr [edx + 714], ecx
+        mov  dword ptr [edx + 715], ecx
+        mov  dword ptr [edx + 716], ecx
+        mov  dword ptr [edx + 717], ecx
+        mov  dword ptr [edx + 718], ecx
+        mov  dword ptr [edx + 719], ecx
+        mov  dword ptr [edx + 720], ecx
+        mov  dword ptr [edx + 721], ecx
+        mov  dword ptr [edx + 722], ecx
+        mov  dword ptr [edx + 723], ecx
+        mov  dword ptr [edx + 724], ecx
+        mov  dword ptr [edx + 725], ecx
+        mov  dword ptr [edx + 726], ecx
+        mov  dword ptr [edx + 727], ecx
+        mov  dword ptr [edx + 728], ecx
+        mov  dword ptr [edx + 729], ecx
+        mov  dword ptr [edx + 730], ecx
+        mov  dword ptr [edx + 731], ecx
+        mov  dword ptr [edx + 732], ecx
+        mov  dword ptr [edx + 733], ecx
+        mov  dword ptr [edx + 734], ecx
+        mov  dword ptr [edx + 735], ecx
+        mov  dword ptr [edx + 736], ecx
+        mov  dword ptr [edx + 737], ecx
+        mov  dword ptr [edx + 738], ecx
+        mov  dword ptr [edx + 739], ecx
+        mov  dword ptr [edx + 740], ecx
+        mov  dword ptr [edx + 741], ecx
+        mov  dword ptr [edx + 742], ecx
+        mov  dword ptr [edx + 743], ecx
+        mov  dword ptr [edx + 744], ecx
+        mov  dword ptr [edx + 745], ecx
+        mov  dword ptr [edx + 746], ecx
+        mov  dword ptr [edx + 747], ecx
+        mov  dword ptr [edx + 748], ecx
+        mov  dword ptr [edx + 749], ecx
+        mov  dword ptr [edx + 750], ecx
+        mov  dword ptr [edx + 751], ecx
+        mov  dword ptr [edx + 752], ecx
+        mov  dword ptr [edx + 753], ecx
+        mov  dword ptr [edx + 754], ecx
+        mov  dword ptr [edx + 755], ecx
+        mov  dword ptr [edx + 756], ecx
+        mov  dword ptr [edx + 757], ecx
+        mov  dword ptr [edx + 758], ecx
+        mov  dword ptr [edx + 759], ecx
+        mov  dword ptr [edx + 760], ecx
+        mov  dword ptr [edx + 761], ecx
+        mov  dword ptr [edx + 762], ecx
+        mov  dword ptr [edx + 763], ecx
+        mov  dword ptr [edx + 764], ecx
+        mov  dword ptr [edx + 765], ecx
+        mov  dword ptr [edx + 766], ecx
+        mov  dword ptr [edx + 767], ecx
+        mov  dword ptr [edx + 768], ecx
+        mov  dword ptr [edx + 769], ecx
+        mov  dword ptr [edx + 770], ecx
+        mov  dword ptr [edx + 771], ecx
+        mov  dword ptr [edx + 772], ecx
+        mov  dword ptr [edx + 773], ecx
+        mov  dword ptr [edx + 774], ecx
+        mov  dword ptr [edx + 775], ecx
+        mov  dword ptr [edx + 776], ecx
+        mov  dword ptr [edx + 777], ecx
+        mov  dword ptr [edx + 778], ecx
+        mov  dword ptr [edx + 779], ecx
+        mov  dword ptr [edx + 780], ecx
+        mov  dword ptr [edx + 781], ecx
+        mov  dword ptr [edx + 782], ecx
+        mov  dword ptr [edx + 783], ecx
+        mov  dword ptr [edx + 784], ecx
+        mov  dword ptr [edx + 785], ecx
+        mov  dword ptr [edx + 786], ecx
+        mov  dword ptr [edx + 787], ecx
+        mov  dword ptr [edx + 788], ecx
+        mov  dword ptr [edx + 789], ecx
+        mov  dword ptr [edx + 790], ecx
+        mov  dword ptr [edx + 791], ecx
+        mov  dword ptr [edx + 792], ecx
+        mov  dword ptr [edx + 793], ecx
+        mov  dword ptr [edx + 794], ecx
+        mov  dword ptr [edx + 795], ecx
+        mov  dword ptr [edx + 796], ecx
+        mov  dword ptr [edx + 797], ecx
+        mov  dword ptr [edx + 798], ecx
+        mov  dword ptr [edx + 799], ecx
+        mov  dword ptr [edx + 800], ecx
+        mov  dword ptr [edx + 801], ecx
+        mov  dword ptr [edx + 802], ecx
+        mov  dword ptr [edx + 803], ecx
+        mov  dword ptr [edx + 804], ecx
+        mov  dword ptr [edx + 805], ecx
+        mov  dword ptr [edx + 806], ecx
+        mov  dword ptr [edx + 807], ecx
+        mov  dword ptr [edx + 808], ecx
+        mov  dword ptr [edx + 809], ecx
+        mov  dword ptr [edx + 810], ecx
+        mov  dword ptr [edx + 811], ecx
+        mov  dword ptr [edx + 812], ecx
+        mov  dword ptr [edx + 813], ecx
+        mov  dword ptr [edx + 814], ecx
+        mov  dword ptr [edx + 815], ecx
+        mov  dword ptr [edx + 816], ecx
+        mov  dword ptr [edx + 817], ecx
+        mov  dword ptr [edx + 818], ecx
+        mov  dword ptr [edx + 819], ecx
+        mov  dword ptr [edx + 820], ecx
+        mov  dword ptr [edx + 821], ecx
+        mov  dword ptr [edx + 822], ecx
+        mov  dword ptr [edx + 823], ecx
+        mov  dword ptr [edx + 824], ecx
+        mov  dword ptr [edx + 825], ecx
+        mov  dword ptr [edx + 826], ecx
+        mov  dword ptr [edx + 827], ecx
+        mov  dword ptr [edx + 828], ecx
+        mov  dword ptr [edx + 829], ecx
+        mov  dword ptr [edx + 830], ecx
+        mov  dword ptr [edx + 831], ecx
+        mov  dword ptr [edx + 832], ecx
+        mov  dword ptr [edx + 833], ecx
+        mov  dword ptr [edx + 834], ecx
+        mov  dword ptr [edx + 835], ecx
+        mov  dword ptr [edx + 836], ecx
+        mov  dword ptr [edx + 837], ecx
+        mov  dword ptr [edx + 838], ecx
+        mov  dword ptr [edx + 839], ecx
+        mov  dword ptr [edx + 840], ecx
+        mov  dword ptr [edx + 841], ecx
+        mov  dword ptr [edx + 842], ecx
+        mov  dword ptr [edx + 843], ecx
+        mov  dword ptr [edx + 844], ecx
+        mov  dword ptr [edx + 845], ecx
+        mov  dword ptr [edx + 846], ecx
+        mov  dword ptr [edx + 847], ecx
+        mov  dword ptr [edx + 848], ecx
+        mov  dword ptr [edx + 849], ecx
+        mov  dword ptr [edx + 850], ecx
+        mov  dword ptr [edx + 851], ecx
+        mov  dword ptr [edx + 852], ecx
+        mov  dword ptr [edx + 853], ecx
+        mov  dword ptr [edx + 854], ecx
+        mov  dword ptr [edx + 855], ecx
+        mov  dword ptr [edx + 856], ecx
+        mov  dword ptr [edx + 857], ecx
+        mov  dword ptr [edx + 858], ecx
+        mov  dword ptr [edx + 859], ecx
+        mov  dword ptr [edx + 860], ecx
+        mov  dword ptr [edx + 861], ecx
+        mov  dword ptr [edx + 862], ecx
+        mov  dword ptr [edx + 863], ecx
+        mov  dword ptr [edx + 864], ecx
+        mov  dword ptr [edx + 865], ecx
+        mov  dword ptr [edx + 866], ecx
+        mov  dword ptr [edx + 867], ecx
+        mov  dword ptr [edx + 868], ecx
+        mov  dword ptr [edx + 869], ecx
+        mov  dword ptr [edx + 870], ecx
+        mov  dword ptr [edx + 871], ecx
+        mov  dword ptr [edx + 872], ecx
+        mov  dword ptr [edx + 873], ecx
+        mov  dword ptr [edx + 874], ecx
+        mov  dword ptr [edx + 875], ecx
+        mov  dword ptr [edx + 876], ecx
+        mov  dword ptr [edx + 877], ecx
+        mov  dword ptr [edx + 878], ecx
+        mov  dword ptr [edx + 879], ecx
+        mov  dword ptr [edx + 880], ecx
+        mov  dword ptr [edx + 881], ecx
+        mov  dword ptr [edx + 882], ecx
+        mov  dword ptr [edx + 883], ecx
+        mov  dword ptr [edx + 884], ecx
+        mov  dword ptr [edx + 885], ecx
+        mov  dword ptr [edx + 886], ecx
+        mov  dword ptr [edx + 887], ecx
+        mov  dword ptr [edx + 888], ecx
+        mov  dword ptr [edx + 889], ecx
+        mov  dword ptr [edx + 890], ecx
+        mov  dword ptr [edx + 891], ecx
+        mov  dword ptr [edx + 892], ecx
+        mov  dword ptr [edx + 893], ecx
+        mov  dword ptr [edx + 894], ecx
+        mov  dword ptr [edx + 895], ecx
+        mov  dword ptr [edx + 896], ecx
+        mov  dword ptr [edx + 897], ecx
+        mov  dword ptr [edx + 898], ecx
+        mov  dword ptr [edx + 899], ecx
+        mov  ecx,0x0 /* counter for diagnostics */
+      repeata:
+        dec  eax
+        inc  ecx
+        cmp  eax,0x0
+        jnz  repeata
+        mov  total,ecx
+    }
+#endif
+    print("Executed 0x%x iters\n", total);
+}
+
+int
+main()
+{
+#ifndef LINUX
+    int old;
+#endif
+    INIT();
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    foo(0xabcd);
+    foo(0x1234);
+    foo(0xef01);
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+
+    return 0;
+}
diff --git a/suite/tests/security-common/selfmod-big.template b/suite/tests/security-common/selfmod-big.template
new file mode 100644
index 000000000..d2663ff4a
--- /dev/null
+++ b/suite/tests/security-common/selfmod-big.template
@@ -0,0 +1,38 @@
+#if +PROGRAM_SHEPHERDING +WINDOWS : +-security --executable_heap +-no_executable_if_hook --executable_if_text --executable_if_x --X || +PROGRAM_SHEPHERDING +LINUX : +-security --executable_heap --executable_if_x --X
+# if : --detect_mode
+#  if : +-throw_exception
+// The exception handler is bad, so we loop to -throw_exception_max_per_thread
+#   SEC_VIO_EXCEPTION
+#   if : --throw_exception_max 1
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#   endif
+#  endif
+#  if : +-kill_thread
+#    SEC_VIO_THREAD
+#  else  
+#    SEC_VIO_STOP
+#  endif
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+Executed 0xabcd iters
+#if +PROGRAM_SHEPHERDING +LINUX : +-security --executable_heap --executable_if_x --X +-detect_mode
+# SEC_VIO_CONT
+#endif
+Executed 0x1234 iters
+#if +PROGRAM_SHEPHERDING +LINUX : +-security --executable_heap --executable_if_x --X +-detect_mode
+# SEC_VIO_CONT
+#endif
+Executed 0xef01 iters
diff --git a/suite/tests/security-common/selfmod.c b/suite/tests/security-common/selfmod.c
new file mode 100644
index 000000000..ebb7324fe
--- /dev/null
+++ b/suite/tests/security-common/selfmod.c
@@ -0,0 +1,73 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "tools.h"
+
+#ifdef USE_DYNAMO
+# include "dynamorio.h"
+#endif
+
+void
+foo(int iters)
+{
+    int total = code_self_mod(iters);
+    print("Executed 0x%x iters\n", total);
+}
+
+int
+main()
+{
+#ifndef LINUX
+    int old;
+#endif
+    INIT();
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    /* make foo code writable */
+    protect_mem(code_self_mod, PAGE_SIZE, ALLOW_READ|ALLOW_WRITE|ALLOW_EXEC);
+    // Note that main and the exception handler __except_handler3 are on this page too
+
+    foo(0xabcd);
+    foo(0x1234);
+    foo(0xef01);
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+
+    return 0;
+}
diff --git a/suite/tests/security-common/selfmod.template b/suite/tests/security-common/selfmod.template
new file mode 100755
index 000000000..c4346bb85
--- /dev/null
+++ b/suite/tests/security-common/selfmod.template
@@ -0,0 +1,32 @@
+#if +PROGRAM_SHEPHERDING +WINDOWS : +-security --executable_heap +-no_executable_if_hook --executable_if_text --executable_if_x --X || +PROGRAM_SHEPHERDING +LINUX : +-security --executable_heap --executable_if_x --X
+# if : --detect_mode
+#  if : +-throw_exception
+// The exception handler is bad, so we loop to -throw_exception_max_per_thread
+#   SEC_VIO_EXCEPTION
+#   if : --throw_exception_max 1
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#   endif
+#  endif
+#  if : +-kill_thread
+#    SEC_VIO_THREAD
+#  else  
+#    SEC_VIO_STOP
+#  endif
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+Executed 0xabcd iters
+Executed 0x1234 iters
+Executed 0xef01 iters
diff --git a/suite/tests/security-common/selfmod2.c b/suite/tests/security-common/selfmod2.c
new file mode 100644
index 000000000..cd37b7dd6
--- /dev/null
+++ b/suite/tests/security-common/selfmod2.c
@@ -0,0 +1,121 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifndef ASM_CODE_ONLY /* C code */
+#include "tools.h"
+
+#ifdef USE_DYNAMO
+# include "dynamorio.h"
+#endif
+
+/* Can't just copy from (ptr_int_t)bar, since that's often a separate entry point
+ * and not the start of bar's code, so we just have a hunk of code
+ * sitting here:
+ *
+ *  int
+ *  bar(int value)
+ *  {
+ *      return value*2;
+ *  }
+ * =>
+ *  00401030: 55                 push        ebp
+ *  00401031: 8B EC              mov         ebp,esp
+ *  00401033: 8B 45 08           mov         eax,dword ptr [ebp+8]
+ *  00401036: D1 E0              shl         eax,1
+ *  00401038: 5D                 pop         ebp
+ *  00401039: C3                 ret
+ *
+ */
+unsigned char bar_code[] = {
+    0x55, 0x8b, 0xec, 0x8b, 0x45, 0x08, 0xd1, 0xe0, 0x5d, 0xc3
+};
+unsigned int bar_code_size = sizeof(bar_code);
+int foo(int value);
+
+int
+main()
+{
+#ifndef LINUX
+    int old;
+#endif
+    INIT();
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    protect_mem(foo, PAGE_SIZE, ALLOW_EXEC|ALLOW_WRITE|ALLOW_READ);
+    
+    print("foo returned %d\n", foo(10));
+    print("foo returned %d\n", foo(10));
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+
+    return 0;
+}
+
+#else /* asm code *************************************************************/
+#include "asm_defines.asm"
+START_FILE
+
+DECL_EXTERN(bar_code)
+DECL_EXTERN(bar_code_size)
+
+/* int foo(int value)
+ *   copies bar over the front of itself, so future invocations will
+ *   run bar's code
+ */
+#define FUNCNAME foo
+        DECLARE_FUNC_SEH(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        mov  REG_XAX, ARG1
+        /* save callee-saved regs */
+        push REG_XSI
+        push REG_XDI
+        /* set up for copy */
+        mov  REG_XSI, offset bar_code
+        mov  REG_XDI, offset foo
+        mov  ecx, DWORD SYMREF(bar_code_size)
+        cld
+        rep movsb
+        /* restore callee-saved regs */
+        pop REG_XDI
+        pop REG_XSI
+        ret
+        END_FUNC(FUNCNAME)
+
+END_FILE
+#endif
diff --git a/suite/tests/security-common/selfmod2.template b/suite/tests/security-common/selfmod2.template
new file mode 100755
index 000000000..439872dff
--- /dev/null
+++ b/suite/tests/security-common/selfmod2.template
@@ -0,0 +1,31 @@
+#if +PROGRAM_SHEPHERDING +WINDOWS : +-security --executable_heap +-no_executable_if_hook --executable_if_text --executable_if_x --X || +PROGRAM_SHEPHERDING +LINUX : +-security --executable_heap --executable_if_x --X
+# if : --detect_mode
+#  if : +-throw_exception
+// The exception handler is bad, so we loop to -throw_exception_max_per_thread
+#   SEC_VIO_EXCEPTION
+#   if : --throw_exception_max 1
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#   endif
+#  endif
+#  if : +-kill_thread
+#    SEC_VIO_THREAD
+#  else  
+#    SEC_VIO_STOP
+#  endif
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else 
+#  SEC_VIO_CONT
+# endif
+#endif
+foo returned 10
+foo returned 20
diff --git a/suite/tests/security-common/vbjmp-rac-test.c b/suite/tests/security-common/vbjmp-rac-test.c
new file mode 100644
index 000000000..db588cbf2
--- /dev/null
+++ b/suite/tests/security-common/vbjmp-rac-test.c
@@ -0,0 +1,135 @@
+/* **********************************************************
+ * Copyright (c) 2003-2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <math.h>
+
+#ifdef LINUX
+# include <unistd.h>
+# include <signal.h>
+# include <ucontext.h>
+# include <errno.h>
+#endif
+
+#include "tools.h" /* for print() */
+
+int
+foo()
+{
+    print("in foo\n");
+    exit(1);
+}
+
+int
+bar()
+{
+    print("in bar\n");
+}
+
+int
+vbpop()
+{
+    print("in vbpop\n");
+#ifdef LINUX
+    __asm__ volatile ( 
+            "push    $bad_target\n"
+            "jmp     some_func\n"
+       "some_func:\n"
+            "call bar\n"
+            // this call needed to get the RET in its own fragment 
+            // otherwise we actually match the loose pattern in at_vbjmp_exception()
+            // which is very similar to what's happening here in at_vbpop_exception()
+            "ret\n"
+       "bad_target:\n"
+            "xor %eax,%eax\n"
+            );
+#else
+    __asm {
+            push    offset bad_target
+            jmp     some_func
+       some_func:
+            call bar
+            ret
+       bad_target:
+            xor eax,eax
+    }
+#endif
+    print("vbpop success\n");
+}
+
+int
+vbjmp()
+{
+#ifdef LINUX
+    __asm__ volatile ( 
+            "mov     $0x38,%eax\n"
+            "cmp     $0xc033,%ax\n"
+            "mov     $0x1d6600,%edx\n"
+            "push    $foo\n"
+            "ret"
+            );
+#else
+    __asm {
+            mov     eax,0x38
+            cmp     ax,0xc033
+            mov     edx,0x1d6600
+            push    offset foo
+            ret
+    }
+#endif
+}
+
+int main(int argc, char *argv[])
+{
+    int i;
+    unsigned char original670[] = 
+"\xb8\x38\x00\x00\x00"       /* mov     eax,0x38 */
+"\x66\x3d\x33\xc0"           /* cmp     ax,0xc033 */
+"\xba\x00\x66\x1d\x00"       /* mov     edx,0x1d6600 */
+"\x68\xfc\xe4\x00\x65"       /* push    0x6500e4fc */
+"\xc3"                       /* ret */
+        ;
+    INIT();
+
+    print("VB ret $+1 about to happen\n");
+    for (i = 0; i < 10; i++) {
+        vbpop();
+    }
+
+    print("VB push;ret about to happen\n");
+    vbjmp();
+
+    print("SHOULD NEVER GET HERE\n");
+
+    return 0;
+}
diff --git a/suite/tests/security-common/vbjmp-rac-test.template b/suite/tests/security-common/vbjmp-rac-test.template
new file mode 100644
index 000000000..06b15e59a
--- /dev/null
+++ b/suite/tests/security-common/vbjmp-rac-test.template
@@ -0,0 +1,50 @@
+# if : --no_vbjmp_allowed
+VB ret $+1 about to happen
+in vbpop
+in bar
+vbpop success
+in vbpop
+in bar
+vbpop success
+in vbpop
+in bar
+vbpop success
+in vbpop
+in bar
+vbpop success
+in vbpop
+in bar
+vbpop success
+in vbpop
+in bar
+vbpop success
+in vbpop
+in bar
+vbpop success
+in vbpop
+in bar
+vbpop success
+in vbpop
+in bar
+vbpop success
+in vbpop
+in bar
+vbpop success
+VB push;ret about to happen
+in foo
+#else
+VB ret $+1 about to happen
+in vbpop
+in bar
+vbpop success
+VB push;ret about to happen
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+in foo
+#endif
diff --git a/suite/tests/security-linux/stacktest.c b/suite/tests/security-linux/stacktest.c
new file mode 100644
index 000000000..3a6550d08
--- /dev/null
+++ b/suite/tests/security-linux/stacktest.c
@@ -0,0 +1,119 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * Written by Solar Designer and placed in the public domain.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+void caller(void (*trampoline)(void))
+{
+    fprintf(stderr, "Attempting to call a trampoline...\n");
+
+    trampoline();
+}
+
+void do_trampoline(void)
+{
+    void nested(void) {
+        fprintf(stderr, "Succeeded.\n");
+    }
+
+    caller(nested);
+}
+
+void do_exploit(void)
+{
+    fprintf(stderr, "Attempting to simulate a buffer overflow exploit...\n");
+
+#ifdef __i386__
+    __asm__ __volatile__(
+                         "movl $1f,%%eax\n\t"
+                         ".byte 0x68; popl %%ecx; jmp *%%eax; nop\n\t"
+                         "pushl %%esp\n\t"
+                         "ret\n\t"
+                         "1:"
+                         : : : "ax", "cx");
+#else
+#error Wrong architecture
+#endif
+
+    fprintf(stderr, "Succeeded.\n");
+}
+
+#define USAGE \
+"Usage: %s OPTION\n" \
+"Non-executable user stack area tests\n\n" \
+"  -t\tcall a GCC trampoline\n" \
+"  -e\tsimulate a buffer overflow exploit\n" \
+"  -b\tsimulate an exploit after a trampoline call\n"
+
+void usage(char *name)
+{
+    printf(USAGE, name ? name : "stacktest");
+    exit(1);
+}
+
+int main(int argc, char **argv)
+{
+    if (argc != 2) {
+        /* default behavior for unit test */
+        do_trampoline();
+        do_exploit();
+        return 0;
+    }
+
+    if (argv[1][0] != '-' || strlen(argv[1]) != 2) usage(argv[0]);
+
+    switch (argv[1][1]) {
+    case 't':
+        do_trampoline();
+        break;
+
+    case 'b':
+        do_trampoline();
+
+    case 'e':
+        do_exploit();
+        break;
+
+    default:
+        usage(argv[0]);
+        break;
+    }
+
+    return 0;
+}
diff --git a/suite/tests/security-linux/stacktest.template b/suite/tests/security-linux/stacktest.template
new file mode 100644
index 000000000..74a0152b4
--- /dev/null
+++ b/suite/tests/security-linux/stacktest.template
@@ -0,0 +1,32 @@
+Attempting to call a trampoline...
+#if +PROGRAM_SHEPHERDING : +-security +-no_executable_if_trampoline || +PROGRAM_SHEPHERDING : +-security +-X
+# if : --executable_stack
+#  if : --detect_mode
+#   SEC_VIO_AUTO_STOP
+#   STOP
+#  else
+#   SEC_VIO_CONT
+#  endif
+# endif
+#endif
+Succeeded.
+Attempting to simulate a buffer overflow exploit...
+#if +PROGRAM_SHEPHERDING : +-security 
+# if : --no_ret_after_call --C --X
+#  if : --detect_mode
+#   SEC_VIO_AUTO_STOP
+#   STOP
+#  else
+#   SEC_VIO_CONT
+#  endif
+# endif
+# if : --executable_stack
+#  if : --detect_mode
+#   SEC_VIO_AUTO_STOP
+#   STOP
+#  else
+#   SEC_VIO_CONT
+#  endif
+# endif
+#endif
+Succeeded.
diff --git a/suite/tests/security-linux/trampoline.c b/suite/tests/security-linux/trampoline.c
new file mode 100644
index 000000000..a979db43d
--- /dev/null
+++ b/suite/tests/security-linux/trampoline.c
@@ -0,0 +1,63 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * Test of gcc nested function trampoline
+ * gcc uses a piece of code on the stack as a closure
+ * to allow exporting a pointer to a nested function.
+ */
+#include <stdio.h>
+
+int ind(int (*func)(int), int arg)
+{
+    fprintf(stderr, "inside ind\n");
+    (*func)(arg);
+}
+
+int outer(int x)
+{
+    int y = 37;
+
+    int nested(int x) {
+        int z = y + x;
+        fprintf(stderr, "inside nested %d\n", z);
+    }
+
+    fprintf(stderr, "inside outer\n");
+    ind(nested, 4);
+}
+
+int main()
+{
+    outer(4);
+    return 0;
+}
diff --git a/suite/tests/security-linux/trampoline.template b/suite/tests/security-linux/trampoline.template
new file mode 100644
index 000000000..9b11cc741
--- /dev/null
+++ b/suite/tests/security-linux/trampoline.template
@@ -0,0 +1,13 @@
+inside outer
+inside ind
+#if +PROGRAM_SHEPHERDING : +-security +-no_executable_if_trampoline || +PROGRAM_SHEPHERDING : +-security +-X
+# if : --executable_stack
+#  if : --detect_mode
+#   SEC_VIO_AUTO_STOP
+#   STOP
+#  else
+#   SEC_VIO_CONT
+#  endif
+# endif
+#endif
+inside nested 41
diff --git a/suite/tests/security-win32/TestNTFlush.c b/suite/tests/security-win32/TestNTFlush.c
new file mode 100644
index 000000000..6cc6b449a
--- /dev/null
+++ b/suite/tests/security-win32/TestNTFlush.c
@@ -0,0 +1,104 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "tools.h"
+
+#define BUF_LEN 160 /* nice somewhat arbitrary length */
+#define BUF2_LEN 3*PAGE_SIZE
+
+static char buf[BUF_LEN];
+static char buf2[BUF2_LEN];
+
+#define VERBOSE2(msg, arg1, arg2)
+#define VERBOSE3(msg, arg1, arg2, arg3)
+
+static void 
+do_test(char *buf, int len, Copy_Mode mode) 
+{
+    char *code;
+    int copied_len;
+    code = copy_to_buf(buf, len, &copied_len, CODE_INC, mode);
+    VERBOSE2("wrote code "PFX"-"PFX"\n", code, code+copied_len);
+    NTFlush(code, copied_len);
+    VERBOSE3("flushed "PFX"-"PFX" (0x%x bytes)\n", code, code+copied_len, copied_len);
+    VERBOSE2("executing code "PFX"-"PFX"\n", code, code+copied_len);
+    test_print(code, 1); //2
+    VERBOSE2("executing code "PFX"-"PFX"\n", code, code+copied_len);
+    test_print(code, 2); //3
+    code = copy_to_buf(buf, len, &copied_len, CODE_DEC, mode);
+    VERBOSE2("wrote code "PFX"-"PFX"\n", code, code+copied_len);
+    VERBOSE2("executing code "PFX"-"PFX"\n", code, code+copied_len);
+    test_print(code, 1); //0
+    code = copy_to_buf(buf, len, &copied_len, CODE_INC, mode);
+    VERBOSE2("wrote code "PFX"-"PFX"\n", code, code+copied_len);
+    /* do whole buf here so catch region of self mod below */
+    NTFlush(buf, len);
+    VERBOSE3("flushed "PFX"-"PFX" (0x%x bytes)\n", buf, buf+len, len);
+    VERBOSE2("executing code "PFX"-"PFX"\n", code, code+copied_len);
+    test_print(code, 2); //3
+    code = copy_to_buf(buf, len, &copied_len, CODE_SELF_MOD, mode);
+    VERBOSE2("wrote code "PFX"-"PFX"\n", code, code+copied_len);
+    VERBOSE2("executing self-mod code "PFX"-"PFX"\n", code, code+copied_len);
+    test_print(code, 43981);
+    NTFlush(code, copied_len);
+    VERBOSE3("flushed "PFX"-"PFX" (0x%x bytes)\n", code, code+copied_len, copied_len);
+    VERBOSE2("executing self-mod code "PFX"-"PFX"\n", code, code+copied_len);
+    test_print(code, 4660);
+}
+
+int
+main()
+{
+    char buf_stack[BUF_LEN];
+    char buf2_stack[BUF2_LEN];
+    INIT();
+
+#if USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    print("starting tests\n");
+
+    do_test(buf, BUF_LEN, COPY_NORMAL);
+    do_test(buf2, BUF2_LEN, COPY_CROSS_PAGE);
+
+    do_test(buf_stack, BUF_LEN, COPY_NORMAL);
+    do_test(buf2_stack, BUF2_LEN, COPY_CROSS_PAGE);
+
+    print("about to exit\n");
+
+#if USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
diff --git a/suite/tests/security-win32/TestNTFlush.template b/suite/tests/security-win32/TestNTFlush.template
new file mode 100755
index 000000000..3c73bc590
--- /dev/null
+++ b/suite/tests/security-win32/TestNTFlush.template
@@ -0,0 +1,359 @@
+starting tests
+#if -PROGRAM_SHEPHERDING : || +PROGRAM_SHEPHERDING : --security || +PROGRAM_SHEPHERDING : +-security +-internal_detach_mask +0x1 || +PROGRAM_SHEPHERDING : +-thin_client || +PROGRAM_SHEPHERDING : +-client || +PROGRAM_SHEPHERDING : +-low
+2
+3
+0
+3
+43981
+4660
+2
+3
+0
+3
+43981
+4660
+2
+3
+# if +PROGRAM_SHEPHERDING : +-security +-internal_detach_mask +0x1 --low --client --thin_client
+#  SEC_VIO_STOP
+# endif
+0
+3
+43981
+4660
+2
+3
+0
+3
+43981
+4660
+#else
+# if : +-executable_heap
+2
+3
+0
+3
+43981
+4660
+// FIXME: case 3947
+2
+3
+0
+3
+43981
+4660
+# else
+#  if : +-no_executable_if_flush || : +-X
+#   if : --detect_mode
+#    SEC_VIO_AUTO_STOP
+#    STOP
+#   else
+#    SEC_VIO_CONT
+#   endif
+#  endif
+#  if : +-no_executable_if_flush || : +-X
+2
+3
+#   SEC_VIO_CONT
+0
+#   SEC_VIO_CONT
+3
+#   SEC_VIO_CONT
+43981
+4660
+#   SEC_VIO_CONT
+2
+3
+#   SEC_VIO_CONT
+0
+#   SEC_VIO_CONT
+3
+#   SEC_VIO_CONT
+43981
+4660
+#  else
+// default options ''
+2
+3
+0
+3
+43981
+4660
+// case 3947 only if looking at data sections
+# if +PROGRAM_SHEPHERDING : +-security --E +-rct_section_type(0xe0)
+# if : --detect_mode
+// OPTION_REPORT=8 is on by default unless overriden 
+#  if : --rct_ind_call(3)
+#    SEC_VIO_AUTO_STOP
+#  endif
+// FIXME: case 2144 - we should be testing for rct_ind_call(9) to ignore OPTION_BLOCK=2
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+# endif
+2
+// FIXME: case 3947 
+# if +PROGRAM_SHEPHERDING : +-security --E +-rct_section_type(0xe0) +-detect_mode
+#  SEC_VIO_CONT
+# endif
+3
+# if +PROGRAM_SHEPHERDING : +-security --E +-rct_section_type(0xe0) +-detect_mode
+#  SEC_VIO_CONT
+# endif
+0
+# if +PROGRAM_SHEPHERDING : +-security --E +-rct_section_type(0xe0) +-detect_mode
+#  SEC_VIO_CONT
+# endif
+3
+# if +PROGRAM_SHEPHERDING : +-security --E +-rct_section_type(0xe0) +-detect_mode
+#  SEC_VIO_CONT
+# endif
+43981
+# if +PROGRAM_SHEPHERDING : +-security --E +-rct_section_type(0xe0) +-detect_mode
+#  SEC_VIO_CONT
+# endif
+4660
+#  endif
+# endif
+# if : +-executable_stack
+2
+3
+0
+3
+43981
+4660
+2
+3
+0
+3
+43981
+4660
+# else
+#  if : +-no_executable_if_flush || : +-X
+#   if : --detect_mode
+#    SEC_VIO_AUTO_STOP
+#    STOP
+#   else
+#    SEC_VIO_CONT
+#   endif
+#  endif
+#  if : +-no_executable_if_flush || : +-X
+2
+// no violation here since same code as the 2 output and still on exec list
+3
+#   SEC_VIO_CONT
+0
+#   SEC_VIO_CONT
+3
+#   SEC_VIO_CONT
+#   SEC_VIO_CONT
+43981
+#   SEC_VIO_CONT
+4660
+#   SEC_VIO_CONT
+2
+// no violation here since same code as the 2 output and still on exec list
+3
+#   SEC_VIO_CONT
+0
+#   SEC_VIO_CONT
+3
+// no violation on entry into selfmod since it's on previous page than the
+// inc/dec code, just one violation for after selfmod itself
+#   SEC_VIO_CONT
+43981
+// first violation is unknown code b/c the continue above only added
+// the second page of the selfmod code, now we're executing first page
+// (doesn't happen w/ first stack buf b/c it's all on same page)
+#   SEC_VIO_CONT
+#   SEC_VIO_CONT
+4660
+#  else
+2
+3
+#   if : --detect_mode
+#    SEC_VIO_AUTO_STOP
+#    STOP
+#   else
+#    SEC_VIO_CONT
+#   endif
+0
+3
+#   SEC_VIO_CONT
+#   SEC_VIO_CONT
+43981
+// no violation here b/c selfmod entry is on exec list from the prev continue
+// after the selfmod write, the exec area is removed since on stack, but
+// it's put back on for rest of selfmod code post-write, since it's on
+// future list (was not removed since future not triggered earlier, since
+// selfmod entry was on exec list, as state above)
+4660
+2
+3
+#   SEC_VIO_CONT
+0
+3
+// no violation on entry into selfmod since it's on previous page than the
+// inc/dec code, just one violation for after selfmod itself
+#   SEC_VIO_CONT
+43981
+#   SEC_VIO_CONT
+4660
+#  endif
+# endif
+#endif
+about to exit
+//
+//
+//  ===========================================================================
+//  To better understand the behavior of the two stack tests, here is a
+//  collection of output from the test w/ verbose printing enabled and
+//  from the logfiles:
+//  
+//  ===========================================================================
+//  buf_stack (aligned):
+//  
+//  starting tests
+//  wrote code 0x0012fee4-0x0012feef
+//  flushed 0x0012fee4-0x0012feef (0xb bytes)
+//  >> syscall: NtFlushInstructionCache 0x0012fee4 size=0xb
+//  || new FUTURE executable vm area: 0x0012fee4-0x0012feef WNtFlushInstructionCache
+//  executing code 0x0012fee4-0x0012feef
+//  >> WARNING: pc = 0x0012fee4 is future executable, allowing
+//  >> future exec region is on stack, removing from future list now
+//  || removing FUTURE executable vm area: 0x0012fee4-0x0012feef
+//  || new executable vm area: 0x0012fee4-0x0012feef WFSD unexpected vm area
+//  2
+//  executing code 0x0012fee4-0x0012feef
+//  3
+//  wrote code 0x0012fee4-0x0012ff78
+//  executing code 0x0012fee4-0x0012ff78
+//  >> WARNING: code on stack 0x0012fee4-0x0012feef @tag 0x0012fee4 written to
+//  || removing executable vm area: 0x0012fee4-0x0012feef
+//  <Execution policy violation occurred!
+//  Contact your vendor for a security vulnerability fix.
+//  Program continuing!>
+//  >> WARNING: violating basic block target @0x0012fee4: 55 8b ec 8b 45 8 83 e8 1 5d c3 55
+//  || new executable vm area: 0x0012f000-0x00130000 W-SD unexpected vm area
+//  0
+//  wrote code 0x0012fee4-0x0012feef
+//  flushed 0x0012fee4-0x0012ff80 (0x9c bytes)
+//  >> syscall: NtFlushInstructionCache 0x0012fee4 size=0x9c
+//  || new FUTURE executable vm area: 0x0012fee4-0x0012ff80 WNtFlushInstructionCache
+//  executing code 0x0012fee4-0x0012feef
+//  >> WARNING: code on stack 0x0012f000-0x00130000 @tag 0x0012fee4 written to
+//  || removing executable vm area: 0x0012f000-0x00130000
+//  >> WARNING: pc = 0x0012fee4 is future executable, allowing
+//  >> future exec region is on stack, removing from future list now
+//  || removing FUTURE executable vm area: 0x0012fee4-0x0012ff80
+//  || new executable vm area: 0x0012fee4-0x0012ff80 WFSD unexpected vm area
+//  3
+//  wrote code 0x0012fee4-0x0012ff15
+//  executing self-mod code 0x0012fee4-0x0012ff15
+//  >> WARNING: code on stack 0x0012fee4-0x0012ff80 @tag 0x0012fee4 written to
+//  || removing executable vm area: 0x0012fee4-0x0012ff80
+//  <Execution policy violation occurred!
+//  Contact your vendor for a security vulnerability fix.
+//  Program continuing!>
+//  >> WARNING: violating basic block target @0x0012fee4: 55 8b ec 51 53 56 57 8b 4d 8 e8 0
+//  || new executable vm area: 0x0012f000-0x00130000 W-SD unexpected vm area
+//  >> WARNING: code on stack 0x0012f000-0x00130000 @tag 0x0012fef3 written to
+//  || removing executable vm area: 0x0012f000-0x00130000
+//  <Execution policy violation occurred!
+//  Contact your vendor for a security vulnerability fix.
+//  Program continuing!>
+//  >> WARNING: violating basic block target @0x0012fef7: b8 cd ab 0 0 b9 0 0 0 0 48 41
+//  || new executable vm area: 0x0012f000-0x00130000 W-SD unexpected vm area
+//  43981
+//  flushed 0x0012fee4-0x0012ff15 (0x31 bytes)
+//  >> syscall: NtFlushInstructionCache 0x0012fee4 size=0x31
+//  || new FUTURE executable vm area: 0x0012fee4-0x0012ff15 WNtFlushInstructionCache
+//  executing self-mod code 0x0012fee4-0x0012ff15
+//  >> WARNING: code on stack 0x0012f000-0x00130000 @tag 0x0012fef3 written to
+//  || removing executable vm area: 0x0012f000-0x00130000
+//  >> WARNING: pc = 0x0012fef7 is future executable, allowing
+//  >> future exec region is on stack, removing from future list now
+//  || removing FUTURE executable vm area: 0x0012fee4-0x0012ff15
+//  || new executable vm area: 0x0012fee4-0x0012ff15 WFSD unexpected vm area
+//  4660
+//  about to exit
+//  
+//  ===========================================================================
+//  buf_stack2 (cross page):
+//  
+//  starting tests
+//  wrote code 0x0012dffe-0x0012e009
+//  flushed 0x0012dffe-0x0012e009 (0xb bytes)
+//  >> syscall: NtFlushInstructionCache 0x0012dffe size=0xb
+//  || new FUTURE executable vm area: 0x0012dffe-0x0012e009 WNtFlushInstructionCache
+//  executing code 0x0012dffe-0x0012e009
+//  >> WARNING: pc = 0x0012dffe is future executable, allowing
+//  >> future exec region is on stack, removing from future list now
+//  || removing FUTURE executable vm area: 0x0012dffe-0x0012e009
+//  || new executable vm area: 0x0012dffe-0x0012e009 WFSD unexpected vm area
+//  2
+//  executing code 0x0012dffe-0x0012e009
+//  3
+//  wrote code 0x0012dffe-0x0012e092
+//  executing code 0x0012dffe-0x0012e092
+//  >> WARNING: code on stack 0x0012dffe-0x0012e009 @tag 0x0012dffe written to
+//  || removing executable vm area: 0x0012dffe-0x0012e009
+//  <Execution policy violation occurred!
+//  Contact your vendor for a security vulnerability fix.
+//  Program continuing!>
+//  >> WARNING: violating basic block target @0x0012dffe: 55 8b ec 8b 45 8 83 e8 1 5d c3 55
+//  || new executable vm area: 0x0012d000-0x00130000 W-SD unexpected vm area
+//  0
+//  wrote code 0x0012dffe-0x0012e009
+//  flushed 0x0012cee4-0x0012fee4 (0x3000 bytes)
+//  >> syscall: NtFlushInstructionCache 0x0012cee4 size=0x3000
+//  || new FUTURE executable vm area: 0x0012cee4-0x0012fee4 WNtFlushInstructionCache
+//  executing code 0x0012dffe-0x0012e009
+//  >> WARNING: code on stack 0x0012d000-0x00130000 @tag 0x0012dffe written to
+//  || removing executable vm area: 0x0012d000-0x00130000
+//  >> WARNING: pc = 0x0012dffe is future executable, allowing
+//  >> future exec region is on stack, removing from future list now
+//  || removing FUTURE executable vm area: 0x0012cee4-0x0012fee4
+//  || new executable vm area: 0x0012cee4-0x0012fee4 WFSD unexpected vm area
+//  3
+//  wrote code 0x0012dff0-0x0012e021
+//  executing self-mod code 0x0012dff0-0x0012e021
+//  >>   exec of selfmod has no msgs since it's not same bb, so don't notice code has
+//  >>   changed like do if it has same start addr, and it's still within same future
+//  >>   region that was added to exec list 
+//  >> WARNING: code on stack 0x0012cee4-0x0012fee4 @tag 0x0012dfff written to
+//  >>   this is the actual selfmod write!
+//  || removing executable vm area: 0x0012cee4-0x0012fee4
+//  <Execution policy violation occurred!
+//  Contact your vendor for a security vulnerability fix.
+//  Program continuing!>
+//  >> WARNING: violating basic block target @0x0012e003: b8 cd ab 0 0 b9 0 0 0 0 48 41
+//  >>   error executing rest of selfmod code
+//  || new executable vm area: 0x0012e000-0x00130000 W-SD unexpected vm area
+//  43981
+//  flushed 0x0012dff0-0x0012e021 (0x31 bytes)
+//  >> syscall: NtFlushInstructionCache 0x0012dff0 size=0x31
+//  || new FUTURE executable vm area: 0x0012dff0-0x0012e021 WNtFlushInstructionCache
+//  executing self-mod code 0x0012dff0-0x0012e021
+//  >>    we have unknown code here b/c the after-violation continue only added
+//  >>    the second page of the selfmod code, since the selfmod part itself is
+//  >>    on second page, and now we're back executing from first page
+//  >> WARNING: pc = 0x0012dff0 is future executable, allowing
+//  >> future exec region is on stack, removing from future list now
+//  || removing FUTURE executable vm area: 0x0012dff0-0x0012e021
+//  || new executable vm area: 0x0012dff0-0x00130000 W-SD unexpected vm area
+//  >> WARNING: code on stack 0x0012dff0-0x00130000 @tag 0x0012dfff written to
+//  >>   this is the actual selfmod write!
+//  || removing executable vm area: 0x0012dff0-0x00130000
+//  <Execution policy violation occurred!
+//  Contact your vendor for a security vulnerability fix.
+//  Program continuing!>
+//  >> WARNING: violating basic block target @0x0012e003: b8 34 12 0 0 b9 0 0 0 0 48 41
+//  || new executable vm area: 0x0012e000-0x00130000 W-SD unexpected vm area
+//  4660
+//  about to exit
+//  
+//  ===========================================================================
diff --git a/suite/tests/security-win32/apc-shellcode.c b/suite/tests/security-win32/apc-shellcode.c
new file mode 100644
index 000000000..2ffa867c1
--- /dev/null
+++ b/suite/tests/security-win32/apc-shellcode.c
@@ -0,0 +1,398 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* case 9016 apc-shellcode.c: 
+ * (the APC code is borrowed from initapc.dll.c <- winapc.c)
+ *
+ * fun code with 7 nested and 2 queued up APCs using user-mode QueueUserAPC
+ * two shellcodes sent using native NtQueueApcThread as well as with QueueUserAPC
+ * FIXME: sent to current thread only
+ */
+
+/* FIXME: for case 8451 we'd want to add a library that get's loaded */
+#define _WIN32_WINNT 0x0400
+#include <windows.h>
+#include <winbase.h> /* for QueueUserAPC */
+#include "tools.h"
+
+#include <windows.h>
+#include "tools.h"
+
+static int result = 0;
+static int apc_arg = 0;
+
+/* forwards */
+static void
+send_apc(PAPCFUNC func, unsigned long depth);
+static void WINAPI
+apc_func(unsigned long arg);
+
+typedef VOID (NTAPI *  PKNORMAL_ROUTINE )
+     (IN PVOID NormalContext, IN PVOID SystemArgument1, IN PVOID SystemArgument2); 
+
+/* we need native APCs */
+int
+native_queue_apc(HANDLE thread, PKNORMAL_ROUTINE apc_dispatch, 
+                 PAPCFUNC func, unsigned long arg)
+{
+    NTSTATUS status;
+    GET_NTDLL(NtQueueApcThread, 
+              (
+               IN HANDLE ThreadHandle,
+               IN PKNORMAL_ROUTINE ApcRoutine,
+               IN PVOID ApcContext OPTIONAL,
+               IN PVOID Argument1 OPTIONAL,
+               IN PVOID Argument2 OPTIONAL
+               ));
+
+    /* 
+     *   This is what I see in QueueUserAPC just before calling NtQueueApcThread
+     *   so ApcContext in that case is the argument to the function
+     *   dds esp 
+     *   0012fb58  fffffffe
+     *   0012fb5c  7c82c0e6 kernel32!BaseDispatchAPC
+     *   0012fb60  0041624b apc_shellcode!apc_func [security-win32/apc-shellcode.c @ 79]
+     *   0012fb64  00000006
+     *   0012fb68  00000000
+     */
+    /* FIXME: should really be (thread, apc_dispatch, func, arg, NULL)
+     * but for more devious testing using this broken version for now
+     */
+    status = NtQueueApcThread(thread, apc_dispatch, NULL /* no context */, 
+                              func, (void*)arg);
+    if (!NT_SUCCESS(status)) {
+        print("Error using NtQueueApcThread %x\n", status);
+        return 0;
+    } else {
+        return 1;
+    }
+}
+
+/* our replacement of kernel32!BaseDispatchAPC */
+int
+NTAPI
+our_dispatch_apc(PVOID context, PAPCFUNC func, unsigned long arg)
+{
+    /* no SEH stuff for us */
+    func(arg);
+    return 0;
+}
+
+/* our replacement of kernel32!QueueUserAPC() */
+int
+queue_apc(bool native, PAPCFUNC func, 
+          HANDLE thread, unsigned long arg)
+{
+    if (native) {
+        return native_queue_apc(thread, (PKNORMAL_ROUTINE)&our_dispatch_apc,
+                                func, arg);
+    } else {
+        return QueueUserAPC(func, thread, arg);
+    }
+}
+
+static void WINAPI
+apc_func(unsigned long arg)
+{
+    result += 100;
+    apc_arg = arg;
+
+    print("apc_func %d\n", arg);
+    /* nested APC */
+    if (arg > 0) {
+        send_apc(arg % 3 == 0 ? apc_func : apc_func, /* FIXME: get fancy */
+                  arg - 1);
+    }
+}
+
+/* PR 229292: we need to isolate these vars to avoid flushes to them via
+ * writes to other vars
+ */
+#pragma data_seg(".isolate")
+
+/* match PIC shellcode header, for example
+ * 0013004c 53               push    ebx
+ * 0013004d e800000000       call    00130052
+ */
+unsigned char vse_datacode[] = 
+"\x53"                          /* push    ebx */
+"\xe8" "\x00\x00\x00\x00" 
+/* simplified version */
+"\x5b"                          /* pop ebx */
+"\x5b"                          /* pop ebx */
+"\xc2\x04\x00"                  /* ret 4 */
+;
+
+unsigned char vse_native_datacode[] = 
+"\x53"                          /* push    ebx */
+"\xe8" "\x00\x00\x00\x00" 
+/* simplified version */
+"\x5b"                          /* pop ebx */
+"\x5b"                          /* pop ebx */
+"\xc2\x0c\x00"                  /* ret c */
+;
+
+/* FIXME: should be copied onto a bad VirtualAllocate page */
+PAPCFUNC vse_apc_func = (PAPCFUNC)vse_datacode;
+
+unsigned char other_datacode[] = 
+"\xc2\x04\x00"                  /* ret 4 */
+"\xc2\x04\x00"                  /* ret 4 */
+;
+
+unsigned char other_native_datacode[] = 
+"\xc2\x0c\x00"                  /* ret c */
+"\xc2\x0c\x00"                  /* ret c */
+;
+
+#pragma data_seg()
+
+PAPCFUNC other_apc_func = (PAPCFUNC)other_datacode;
+
+static void WINAPI
+other_apc_func_helper(unsigned long arg)
+{
+    print("webcam or crash and burn in interop issues\n");
+}
+
+static void
+send_apc(PAPCFUNC func, unsigned long depth)
+{
+    int res = queue_apc(false, func, GetCurrentThread(), depth);
+    print("QueueUserAPC returned %d\n", res);
+    if (depth > 0) {
+        /* we queue up two APCs at a time (producing a Fibonaccian growth) */
+        /* and for change we use our own wrapper */
+        res = queue_apc(true /* native */, func, GetCurrentThread(), depth - 1);
+        print("second QueueUserAPC returned %d\n", res);
+    }
+    /* an alertable system call so we receive the APCs (FIFO order) */
+    res = SleepEx(100, 1);
+    /* is going to return 192 since recieved apc during sleep call 
+     * well technically 192 is io completion interruption, but seems to 
+     * report that for any interrupting APC */
+    print("SleepEx returned %d\n", res);
+    print("Apc arg = %d\n", apc_arg);
+    print("Result = %d\n", result);
+}
+
+static void
+native_send_apc(PKNORMAL_ROUTINE native_func1,
+                PKNORMAL_ROUTINE native_func2)
+{
+    int res = native_queue_apc(GetCurrentThread(),
+                               native_func1,
+                               NULL, NULL);
+    print("native_queue_apc returned %d\n", res);
+
+    /* we queue up two APCs at a time maybe of different type */
+    /* note that these just queue, they WILL NOT stack up unless the
+     * APC functions themselves get in Alertable state 
+     * FIXME: should put a TestAlert in the loaded DLL, 
+     * I don't think there is one already in LoadLibrary
+     * (if there is it may be bad for hotpatch DLLs)
+     */
+    res = native_queue_apc(GetCurrentThread(),
+                           native_func2,
+                           NULL, NULL);
+    print("second native_queue_apc returned %d\n", res);
+
+    /* an alertable system call so we receive the APCs (FIFO order) */
+    res = SleepEx(100, 1);
+    /* is going to return 192 since recieved apc during sleep call 
+     * well technically 192 is io completion interruption, but seems to 
+     * report that for any interrupting APC */
+    print("SleepEx returned %d\n", res);
+    /* FIXME: don't have a good sign that the shellcodes did execute */
+}
+
+
+
+int
+main()
+{
+    INIT();
+
+    print("apc-shellcode\n");
+
+    print("normal (nested) apc\n");
+    send_apc(apc_func, 7);
+    
+    __try {
+        print("VSE-like native mode\n");
+        native_send_apc((PKNORMAL_ROUTINE*)vse_native_datacode,
+                        (PKNORMAL_ROUTINE*)vse_native_datacode);
+        print("VSE native shellcode returned\n");
+    } 
+    __except (EXCEPTION_EXECUTE_HANDLER) {
+        print("VSE native shellcode exception!\n");
+    }
+
+    __try {
+        print("other APC native mode\n");
+        native_send_apc((PKNORMAL_ROUTINE*)other_native_datacode,
+                        (PKNORMAL_ROUTINE*)other_native_datacode);
+        print("*** other APC native shellcode returned\n");
+    } 
+    __except (EXCEPTION_EXECUTE_HANDLER) {
+        print("APC native shellcode exception!\n");
+    }
+
+    print("VSE-like user mode\n");
+    send_apc(vse_apc_func, 7);
+    print("*** VSE user shellcode allowed!\n");
+
+    print("other APC user shellcode\n");
+    send_apc(other_apc_func, 7);
+    print("*** other APC user shellcode allowed!\n");
+
+    return 0;
+}
+
+/*
+0:000> uf kernel32!QueueUserAPC
+kernel32!QueueUserAPC:
+7c82c082 8bff             mov     edi,edi
+7c82c084 55               push    ebp
+7c82c085 8bec             mov     ebp,esp
+7c82c087 51               push    ecx
+7c82c088 51               push    ecx
+7c82c089 8365f800         and     dword ptr [ebp-0x8],0x0
+7c82c08d 57               push    edi
+7c82c08e 33c0             xor     eax,eax
+7c82c090 50               push    eax
+7c82c091 6a08             push    0x8
+7c82c093 8d7dfc           lea     edi,[ebp-0x4]
+7c82c096 ab               stosd
+7c82c097 8d45f8           lea     eax,[ebp-0x8]
+7c82c09a 50               push    eax
+7c82c09b 6a01             push    0x1
+7c82c09d 6a00             push    0x0
+7c82c09f 6a00             push    0x0
+7c82c0a1 6a01             push    0x1
+7c82c0a3 ff15ec14807c call dword ptr [kernel32!_imp__RtlQueryInformationActivationContext (7c8014ec)]
+7c82c0a9 85c0             test    eax,eax
+7c82c0ab 5f               pop     edi
+7c82c0ac 0f8c51890100     jl      kernel32!QueueUserAPC+0x2c (7c844a03)
+
+kernel32!QueueUserAPC+0x2c:
+7c844a03 50               push    eax
+7c844a04 68804a847c       push    0x7c844a80
+7c844a09 68284a847c       push    0x7c844a28
+        7c844a80  "QueueUserAPC"
+        0:000> da 0x7c844a28
+        7c844a28  "SXS: %s failing because RtlQuery"
+        7c844a48  "InformationActivationContext() r"
+        7c844a68  "eturned status %08lx."
+7c844a0e e856ae0300       call    kernel32!DbgPrint (7c87f869)
+7c844a13 83c40c           add     esp,0xc
+7c844a16 33c0             xor     eax,eax
+7c844a18 e9c076feff       jmp     kernel32!QueueUserAPC+0x6d (7c82c0dd)
+
+kernel32!QueueUserAPC+0x43:
+7c82c0b2 f645fc01         test    byte ptr [ebp-0x4],0x1
+7c82c0b6 8b45f8           mov     eax,[ebp-0x8]
+7c82c0b9 0f855e890100     jne     kernel32!QueueUserAPC+0x4c (7c844a1d)
+
+kernel32!QueueUserAPC+0x4c:
+7c844a1d 83c8ff           or      eax,0xffffffff
+7c844a20 e99a76feff       jmp     kernel32!QueueUserAPC+0x4f (7c82c0bf)
+
+kernel32!QueueUserAPC+0x4f:
+7c82c0bf 50               push    eax
+7c82c0c0 ff7510           push    dword ptr [ebp+0x10]
+7c82c0c3 ff7508           push    dword ptr [ebp+0x8]
+7c82c0c6 68e6c0827c       push    0x7c82c0e6  ; kernel32!BaseDispatchAPC
+7c82c0cb ff750c           push    dword ptr [ebp+0xc] ; thread
+7c82c0ce ff152015807c call dword ptr [kernel32!_imp__NtQueueApcThread (7c801520)]
+7c82c0d4 33c9             xor     ecx,ecx
+7c82c0d6 85c0             test    eax,eax
+7c82c0d8 0f9dc1           setge   cl
+7c82c0db 8bc1             mov     eax,ecx
+
+kernel32!QueueUserAPC+0x6d:
+7c82c0dd c9               leave
+7c82c0de c20c00           ret     0xc
+
+0:000> uf 0x7c82c0e6
+kernel32!BaseDispatchAPC:
+seems to be mostly concerned about SXS
+kernel32!BaseDispatchAPC+0x33:
+7c82c13a c20c00           ret     0xc
+
+0:000> uf kernel32!BaseDispatchApc
+kernel32!BaseDispatchAPC:
+7c82c0e6 6a20             push    0x20
+7c82c0e8 6840c1827c       push    0x7c82c140
+7c82c0ed e8d463fdff       call    kernel32!_SEH_prolog (7c8024c6)
+7c82c0f2 c745d014000000   mov     dword ptr [ebp-0x30],0x14
+7c82c0f9 c745d401000000   mov     dword ptr [ebp-0x2c],0x1
+7c82c100 33c0             xor     eax,eax
+7c82c102 8d7dd8           lea     edi,[ebp-0x28]
+7c82c105 ab               stosd
+7c82c106 ab               stosd
+7c82c107 ab               stosd
+7c82c108 8b4510           mov     eax,[ebp+0x10]
+7c82c10b 8945e4           mov     [ebp-0x1c],eax
+7c82c10e 83f8ff           cmp     eax,0xffffffff
+7c82c111 0f84e1880100     je      kernel32!BaseDispatchAPC+0x2d (7c8449f8)
+
+kernel32!BaseDispatchAPC+0x3b:
+7c82c117 50               push    eax
+7c82c118 8d45d0           lea     eax,[ebp-0x30]
+7c82c11b 50               push    eax
+7c82c11c ff153c12807c call dword ptr [kernel32!_imp__RtlActivateActivationContextUnsafeFast (7c80123c)]
+7c82c122 8365fc00         and     dword ptr [ebp-0x4],0x0
+7c82c126 ff750c           push    dword ptr [ebp+0xc]
+7c82c129 ff5508           call    dword ptr [ebp+0x8]
+7c82c12c 834dfcff         or      dword ptr [ebp-0x4],0xffffffff
+7c82c130 e81c000000       call    kernel32!BaseDispatchAPC+0x5b (7c82c151)
+
+kernel32!BaseDispatchAPC+0x33:
+7c82c135 e8c763fdff       call    kernel32!_SEH_epilog (7c802501)
+7c82c13a c20c00           ret     0xc
+
+kernel32!BaseDispatchAPC+0x2d:
+7c8449f8 ff750c           push    dword ptr [ebp+0xc]
+7c8449fb ff5508           call    dword ptr [ebp+0x8]
+7c8449fe e93277feff       jmp     kernel32!BaseDispatchAPC+0x33 (7c82c135)
+
+
+kernel32!BaseDispatchAPC+0x5b: helper
+7c82c151 8d45d0           lea     eax,[ebp-0x30]
+7c82c154 50               push    eax
+7c82c155 ff153812807c call dword ptr [kernel32!_imp__RtlDeactivateActivationContextUnsafeFast (7c801238)]
+7c82c15b ff75e4           push    dword ptr [ebp-0x1c]
+7c82c15e ff15e414807c call dword ptr [kernel32!_imp__RtlReleaseActivationContext (7c8014e4)]
+7c82c164 c3               ret
+
+*/
diff --git a/suite/tests/security-win32/apc-shellcode.template b/suite/tests/security-win32/apc-shellcode.template
new file mode 100644
index 000000000..a220327b7
--- /dev/null
+++ b/suite/tests/security-win32/apc-shellcode.template
@@ -0,0 +1,388 @@
+apc-shellcode
+normal (nested) apc
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 7
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 6
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 6
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 5
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 5
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 4
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 5
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 4
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 4
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 3
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 4
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 3
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 3
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 2
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 4
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 3
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 3
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 2
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 3
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 2
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 2
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 1
+QueueUserAPC returned 1
+apc_func 3
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 2
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 2
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 1
+QueueUserAPC returned 1
+apc_func 2
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 1
+QueueUserAPC returned 1
+apc_func 1
+QueueUserAPC returned 1
+apc_func 0
+apc_func 3
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 2
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 2
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 1
+QueueUserAPC returned 1
+apc_func 2
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 1
+QueueUserAPC returned 1
+apc_func 1
+QueueUserAPC returned 1
+apc_func 0
+apc_func 2
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 1
+QueueUserAPC returned 1
+apc_func 1
+QueueUserAPC returned 1
+apc_func 0
+apc_func 1
+QueueUserAPC returned 1
+apc_func 0
+apc_func 0
+apc_func 2
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 1
+QueueUserAPC returned 1
+apc_func 1
+QueueUserAPC returned 1
+apc_func 0
+apc_func 1
+QueueUserAPC returned 1
+apc_func 0
+apc_func 0
+apc_func 1
+QueueUserAPC returned 1
+apc_func 0
+apc_func 0
+apc_func 0
+apc_func 2
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+apc_func 1
+QueueUserAPC returned 1
+apc_func 1
+QueueUserAPC returned 1
+apc_func 0
+apc_func 1
+QueueUserAPC returned 1
+apc_func 0
+apc_func 0
+apc_func 1
+QueueUserAPC returned 1
+apc_func 0
+apc_func 0
+apc_func 0
+apc_func 1
+QueueUserAPC returned 1
+apc_func 0
+apc_func 0
+apc_func 0
+apc_func 0
+apc_func 1
+QueueUserAPC returned 1
+apc_func 0
+apc_func 0
+apc_func 0
+apc_func 0
+apc_func 0
+apc_func 1
+QueueUserAPC returned 1
+apc_func 0
+apc_func 0
+apc_func 0
+apc_func 0
+apc_func 0
+apc_func 0
+apc_func 0
+apc_func 0
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+VSE-like native mode
+native_queue_apc returned 1
+second native_queue_apc returned 1
+#if +PROGRAM_SHEPHERDING : +-security --low --client --thin_client
+#if : --apc_policy
+# if : --exempt_threat_list
+#    SEC_VIO_THREAD
+#    SEC_VIO_THREAD
+# else
+# endif
+#endif
+#endif
+SleepEx returned 192
+VSE native shellcode returned
+other APC native mode
+native_queue_apc returned 1
+second native_queue_apc returned 1
+#if +PROGRAM_SHEPHERDING : +-security --low --client --thin_client
+#if : --apc_policy
+# if : --silent_block_threat_list
+#    SEC_VIO_THREAD
+#    SEC_VIO_THREAD
+# else
+# endif
+#endif
+#endif
+SleepEx returned 192
+*** other APC native shellcode returned
+VSE-like user mode
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+#if +PROGRAM_SHEPHERDING : +-security --low --client --thin_client
+#if : --detect_mode
+# SEC_VIO_AUTO_STOP
+# STOP
+#else
+# SEC_VIO_CONT
+#endif
+#endif
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+*** VSE user shellcode allowed!
+other APC user shellcode
+QueueUserAPC returned 1
+second QueueUserAPC returned 1
+SleepEx returned 192
+Apc arg = 0
+Result = 8700
+*** other APC user shellcode allowed!
diff --git a/suite/tests/security-win32/aslr-ind.c b/suite/tests/security-win32/aslr-ind.c
new file mode 100644
index 000000000..fad707c74
--- /dev/null
+++ b/suite/tests/security-win32/aslr-ind.c
@@ -0,0 +1,137 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* aslr-ind.c */
+/* FIXME: should make this process start itself so that can test with early injection enabled */
+
+#include <windows.h>
+#include "tools.h"
+
+// #define VERBOSE
+typedef int (*fiptr)(void);
+
+fiptr
+__declspec(dllexport)
+giveme_target(int arg);
+
+fiptr go_where;
+
+void*
+get_module_preferred_base(void* module_base)
+{
+    IMAGE_DOS_HEADER *dos;
+    IMAGE_NT_HEADERS *nt;
+    dos = (IMAGE_DOS_HEADER *) module_base;
+    nt = (IMAGE_NT_HEADERS *) (((ptr_uint_t)dos) + dos->e_lfanew);
+    return (void*)nt->OptionalHeader.ImageBase;
+}
+
+void
+__declspec(dllimport)
+precious(void);
+
+/* from retexisting.c */
+int
+ring(int num)
+{
+    print("looking at ring\n");
+    *(int*) (&num - 1) = (int)&precious;
+    return num;
+}
+
+int
+main(int argc)
+{
+    char* base;
+    void* hmod = GetModuleHandle("aslr-ind.dll.dll");
+
+    INIT();
+    USE_USER32();
+
+    print("aslr-ind main()\n");
+
+#ifdef VERBOSE
+    print("0x%x\n", hmod);
+#endif
+
+    go_where = giveme_target(332);
+    print("%d\n", (*go_where)());
+
+    base = get_module_preferred_base(hmod);
+    if (base == hmod) 
+        print("at base, no ASLR\n");
+    else {
+        print("targeting original base\n");
+        go_where = (fiptr) ((char*)go_where + (base - hmod));
+    }
+
+    /* in wrong address space, but a good entry!  
+     * FIXME: we may want
+     * to flag that explicitly - could be an FP!
+     */
+    __try {
+        print("%d\n", go_where());
+        print("*** invalid indirect call at preferred base!\n");
+    } __except (EXCEPTION_EXECUTE_HANDLER) {
+        print("invalid indirect call 1 caught\n");
+    }
+
+    /* now this should definitely not be good, no matter aslr */
+    /*     0012fb20 b801000000       mov     eax,0x1 */
+    /*     0012fb25 */
+    /* FIXME: should go over jmp as well */
+    go_where = (fiptr)((char *)go_where + 5);
+    
+    __try {
+        print("%d\n", go_where());
+        print("*** invalid indirect call allowed!\n");
+    } __except (EXCEPTION_EXECUTE_HANDLER) {
+        print("invalid indirect call 2 caught\n");
+    }
+
+    print("indirect call done\n");
+
+    __try {
+        print("starting bad return function\n");
+        ring(1);
+        print("*** invalid RET - can't really get here\n");
+    } __except (EXCEPTION_EXECUTE_HANDLER) {
+        print("invalid return caught\n");
+    }
+
+    /* FIXME: should be able to allocate memory with VirtualAlloc() at
+     * the would be location and verify our handling of execution
+     * there in that case
+     */
+
+    return 0;
+}
diff --git a/suite/tests/security-win32/aslr-ind.dll.c b/suite/tests/security-win32/aslr-ind.dll.c
new file mode 100644
index 000000000..008b5459b
--- /dev/null
+++ b/suite/tests/security-win32/aslr-ind.dll.c
@@ -0,0 +1,89 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* case 7017 - showing off ASLR prevention */
+
+#include "tools.h"
+#include <windows.h>
+
+typedef int (*fiptr)();
+
+#define NAKED __declspec( naked ) 
+
+/* could be an export but instead is address taken callback */
+NAKED
+/* PR 229292: must be static to avoid having an ILT entry in debug build */
+static int
+funny_target()
+{
+    __asm {
+        mov eax,1
+        jmp over
+badtarget:
+        mov eax,2               /* bad target */
+over:
+        add eax,eax
+        add eax,eax
+        add eax,eax
+        add eax,eax
+        ret
+    }
+}
+
+fiptr
+__declspec(dllexport)
+giveme_target(int arg)
+{
+    print("ready to go %d\n", arg);
+    return funny_target;
+}
+
+void
+__declspec(dllexport)
+precious(void)
+{
+    print("PRECIOUS in a DLL, ATTACK SUCCESSFUL!\n");
+    /* stack not clean have to stop here */
+    exit(1);
+}
+
+
+BOOL APIENTRY 
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    switch (reason_for_call) {
+    case DLL_PROCESS_ATTACH:
+        /* nothing */
+        break;
+    }
+    return TRUE;
+}
diff --git a/suite/tests/security-win32/aslr-ind.template b/suite/tests/security-win32/aslr-ind.template
new file mode 100644
index 000000000..007348f7b
--- /dev/null
+++ b/suite/tests/security-win32/aslr-ind.template
@@ -0,0 +1,40 @@
+aslr-ind main()
+ready to go 332
+16
+at base, no ASLR
+16
+*** invalid indirect call at preferred base!
+// .E expected
+#if +PROGRAM_SHEPHERDING : +-security --X --E --low --client --thin_client --use_moduledb
+# if : --detect_mode --throw_exception
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  if : +-detect_mode
+#   SEC_VIO_CONT
+#  else
+#   SEC_VIO_EXCEPTION
+#  endif
+# endif
+invalid indirect call 2 caught
+#else
+0
+*** invalid indirect call allowed!
+#endif
+indirect call done
+starting bad return function
+looking at ring
+// .C expected
+#if +PROGRAM_SHEPHERDING : +-security --X --C --low --client --thin_client --use_moduledb
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+PRECIOUS in a DLL, ATTACK SUCCESSFUL!
diff --git a/suite/tests/security-win32/codemod-threads.c b/suite/tests/security-win32/codemod-threads.c
new file mode 100644
index 000000000..5817dd3ff
--- /dev/null
+++ b/suite/tests/security-win32/codemod-threads.c
@@ -0,0 +1,102 @@
+/* **********************************************************
+ * Copyright (c) 2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* threaded version of codemod.c to test
+ * cache consistency concurrency cases (say that 3 times fast)
+ */
+
+#include <windows.h>
+#include <process.h> /* for _beginthreadex */
+#include <stdio.h>
+#include "tools.h"
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#define ITERS 150
+#define NUM_THREADS 6
+char buf[32];
+
+int WINAPI
+run_func(void * arg)
+{
+    char *foo = buf;
+    int i;
+    for (i=0; i<ITERS; i++) {
+#ifdef LINUX
+        asm("mov  %0,%%eax" : : "r"(foo));
+        asm("call *%eax");
+#else
+        __asm {
+            mov   eax, foo
+            call  eax
+        }
+#endif
+        buf[1] = 0xc3; /* ret */
+        /* we're not testing security here, just consistency, so make it kosher */
+        NTFlush(buf, 2);
+    }
+    return i;
+}
+
+int
+main()
+{
+    int tid, i;
+    unsigned long hThread[NUM_THREADS];
+
+    INIT();
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    print("starting up\n");
+
+    /* set up initial value of code */
+    buf[0] = 0xc3; /* ret */
+    /* we're not testing security here, just consistency, so make it kosher */
+    NTFlush(buf, 1);
+
+    for (i=0; i<NUM_THREADS; i++)
+        hThread[i] = _beginthreadex(NULL, 0, run_func, NULL, 0, &tid);
+    for (i=0; i<NUM_THREADS; i++)
+        WaitForSingleObject((HANDLE)hThread[i], INFINITE);
+
+    print("all done\n");
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
diff --git a/suite/tests/security-win32/codemod-threads.template b/suite/tests/security-win32/codemod-threads.template
new file mode 100755
index 000000000..88fc85445
--- /dev/null
+++ b/suite/tests/security-win32/codemod-threads.template
@@ -0,0 +1,37 @@
+starting up
+#if -PROGRAM_SHEPHERDING : || +PROGRAM_SHEPHERDING : +-security +-internal_detach_mask +0x1
+// nothing
+#else
+#  if : +-no_executable_if_flush || : +-X
+#    if : --executable_heap --executable_if_dot_data --executable_if_image
+#      if : --detect_mode
+#        SEC_VIO_AUTO_STOP
+// FIXME: can get multiple termination messages here if multiple threads
+// generate them before an os_terminate goes through!
+#        STOP
+#      else
+#        SEC_VIO_CONT
+#        SEC_VIO_CONT
+#        SEC_VIO_CONT
+#        SEC_VIO_CONT
+#        SEC_VIO_CONT
+#        SEC_VIO_CONT
+#        SEC_VIO_CONT
+#        SEC_VIO_CONT
+#        SEC_VIO_CONT
+#        SEC_VIO_CONT
+#        SEC_VIO_CONT
+#        SEC_VIO_CONT
+#        SEC_VIO_CONT
+#        SEC_VIO_CONT
+#        SEC_VIO_CONT
+#        SEC_VIO_CONT
+#        SEC_VIO_CONT
+#        SEC_VIO_CONT
+#        SEC_VIO_CONT
+#        SEC_VIO_CONT
+#      endif
+#    endif
+#  endif
+#endif
+all done
diff --git a/suite/tests/security-win32/except-execution.c b/suite/tests/security-win32/except-execution.c
new file mode 100644
index 000000000..c2540ec84
--- /dev/null
+++ b/suite/tests/security-win32/except-execution.c
@@ -0,0 +1,470 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#include "except.h"
+
+/* for repeatability we need a consistent stack */
+#define STACK_BASE ((void *)0x14000000)
+
+/***************************************************************************/
+/* This is all copied from ntdll.h and from share/detach.c */
+
+#include <AccCtrl.h> /* for SE_KERNEL_OBJECT */
+#include <Aclapi.h>
+
+typedef struct _UNICODE_STRING {
+    /* Length field is size in bytes not counting final 0 */
+    USHORT Length;
+    USHORT MaximumLength;
+    PWSTR  Buffer;
+} UNICODE_STRING;
+typedef UNICODE_STRING *PUNICODE_STRING;
+
+typedef struct _OBJECT_ATTRIBUTES {
+    ULONG Length;
+    HANDLE RootDirectory;
+    PUNICODE_STRING ObjectName;
+    ULONG Attributes;
+    PVOID SecurityDescriptor;        // Points to type SECURITY_DESCRIPTOR
+    PVOID SecurityQualityOfService;  // Points to type SECURITY_QUALITY_OF_SERVICE
+} OBJECT_ATTRIBUTES;
+typedef OBJECT_ATTRIBUTES *POBJECT_ATTRIBUTES;
+
+#define InitializeObjectAttributes( p, n, a, r, s ) { \
+    (p)->Length = sizeof( OBJECT_ATTRIBUTES );          \
+    (p)->RootDirectory = r;                             \
+    (p)->Attributes = a;                                \
+    (p)->ObjectName = n;                                \
+    (p)->SecurityDescriptor = s;                        \
+    (p)->SecurityQualityOfService = NULL;               \
+    }
+
+#define OBJ_CASE_INSENSITIVE    0x00000040L
+/* N.B.: this is an invalid parameter on NT4! */
+#define OBJ_KERNEL_HANDLE       0x00000200L
+typedef ULONG ACCESS_MASK;
+
+typedef struct _CLIENT_ID {
+    HANDLE UniqueProcess;
+    HANDLE UniqueThread;
+} CLIENT_ID;
+typedef CLIENT_ID *PCLIENT_ID;
+
+typedef struct _USER_STACK {
+    PVOID FixedStackBase;
+    PVOID FixedStackLimit;
+    PVOID ExpandableStackBase;
+    PVOID ExpandableStackLimit;
+    PVOID ExpandableStackBottom;
+} USER_STACK, *PUSER_STACK;
+
+/* 64kb, same as allocation granularity so is as small as we can get */
+#define STACK_RESERVE 0x10000
+/* 12kb, matches current core stack size, note can expand to 
+ * STACK_RESERVE - (5 * PAGE_SIZE), i.e. 44kb */
+#define STACK_COMMIT 0x3000
+
+typedef unsigned int (__stdcall * threadfunc_t)(void *);
+
+/* returns NULL on error */
+/* FIXME - is similar to core create_thread, but uses API routines where
+ * possible, could try to share. */
+/* stack_reserve and stack commit must be multiples of PAGE_SIZE and reserve
+ * should be at least 5 pages larger then commit */
+/* NOTE - For !target_kernel32 : 
+ *  target thread routine can't exit by by returning, instead it must call
+ *    ExitThread or the like
+ *  caller or target thread routine is responsible for informing csrss (if
+ *    neccessary) and freeing the the thread stack 
+ */
+static HANDLE
+nt_create_thread(HANDLE hProcess, PTHREAD_START_ROUTINE start_addr, void *arg,
+                 uint stack_reserve, uint stack_commit, bool suspended,
+                 uint *tid, bool target_kernel32)
+{
+    HANDLE hThread = NULL;
+    USER_STACK stack = {0};
+    OBJECT_ATTRIBUTES oa;
+    CLIENT_ID cid;
+    CONTEXT context = {0};
+    uint num_commit_bytes, code;
+    unsigned long old_prot;
+    void *p;
+    SECURITY_DESCRIPTOR *sd = NULL;
+
+    GET_NTDLL(NtCreateThread, (OUT PHANDLE ThreadHandle,
+                               IN ACCESS_MASK DesiredAccess,
+                               IN POBJECT_ATTRIBUTES ObjectAttributes,
+                               IN HANDLE ProcessHandle,
+                               OUT PCLIENT_ID ClientId,
+                               IN PCONTEXT ThreadContext,
+                               IN PUSER_STACK UserStack,
+                               IN BOOLEAN CreateSuspended));
+
+    /* both stack size and stack reserve must be multiples of PAGE_SIZE */
+    assert((stack_reserve & (PAGE_SIZE - 1)) == 0);
+    assert((stack_commit & (PAGE_SIZE - 1)) == 0);
+    /* We stick a non-committed page on each end just to be safe and windows
+     * needs three pages at the end to properly handle end of expandable stack
+     * case (wants to pass exception back to the app on overflow, so needs some
+     * stack for that). */
+    assert(stack_reserve >= stack_commit + (5 * PAGE_SIZE));
+
+    /* Use the security descriptor from the target process for creating the
+     * thread so that once created the thread will be able to open a full
+     * access handle to itself (xref case 2096). */
+    /* NOTES - tried many ways to impersonate based on target process token
+     * so we could just use the default and was unable to get anywhere with
+     * that.  Easiest thing to do here is just create a new security descriptor
+     * with a NULL (not empty) DACL [just InitializeSecurityDescriptor();
+     * SetSecurityDescriptorDacl()], but that's a privilege escalation
+     * problem (allows anybody full access to the thread)].  If we instead get
+     * the full security descriptor from the target process and try to use that
+     * the kernel complains that its a bad choice of owner.  What we do instead
+     * is get just the DACL and leave the rest empty (will be filled in with
+     * defaults during create thread).  Thus the security descriptor for the
+     * thread will end up having the owner, group, and SACL from this
+     * process and the DACL from the target.  Upshot is the the thread pseudo
+     * handle will have full permissions (from the DACL), but the owner will be
+     * us and, even though the handle we get back from CreateThread will be
+     * fully permissioned as we request, any subsequent attempts by us to
+     * OpenThread will fail since we aren't on the DACL.  We could always add
+     * ourselves to the DACL later or we can use the SE_DEBUG_PRIVILEGE to
+     * allow us to open it anyways.  Note if for some reason we want to view the
+     * SACL we need to enable the ACCESS_SYSTEM_SECURITY privilege when opening
+     * the handle.
+     * FIXME - we could instead build our own DACL combining the two, we could
+     * also try setting the owner/group after the thread is created if we
+     * really wanted to look like the target process thread, and could also
+     * start with a NULL sd and set the DACL later if want to match
+     * CreateThread as closely as possible.  If we do anything post system
+     * call should be sure to always create the thread suspended.
+     */
+    code = GetSecurityInfo(hProcess, SE_KERNEL_OBJECT,
+                           DACL_SECURITY_INFORMATION,
+                           NULL, NULL, NULL, NULL, &sd);
+    assert(code == ERROR_SUCCESS);
+ 
+    InitializeObjectAttributes(&oa, NULL, OBJ_CASE_INSENSITIVE, NULL, sd);
+  
+    stack.ExpandableStackBottom = VirtualAllocEx(hProcess, STACK_BASE,
+                                                 stack_reserve - PAGE_SIZE,
+                                                 MEM_RESERVE, PAGE_READWRITE);
+    if (stack.ExpandableStackBottom == NULL)
+        goto error;
+
+    /* We provide non-committed boundary page on each side of the stack just to
+     * be safe (note we will get a stack overflow exception if stack grows to
+     * 3rd to last page of this region (xpsp2)). */
+    stack.ExpandableStackBottom =
+        ((byte *)stack.ExpandableStackBottom) + PAGE_SIZE;
+    stack.ExpandableStackBase = 
+        ((byte *)stack.ExpandableStackBottom) + stack_reserve - (2*PAGE_SIZE);
+
+    stack.ExpandableStackLimit = 
+        ((byte *)stack.ExpandableStackBase) - stack_commit;
+    num_commit_bytes = stack_commit + PAGE_SIZE;
+    p = ((byte *)stack.ExpandableStackBase) - num_commit_bytes;
+    p = VirtualAllocEx(hProcess, p, num_commit_bytes, MEM_COMMIT, PAGE_READWRITE);
+    if (p == NULL)
+        goto error;
+    if (!VirtualProtectEx(hProcess, p, PAGE_SIZE, PAGE_READWRITE|PAGE_GUARD, &old_prot))
+        goto error;
+
+    /* set the context: initialize with our own */
+    context.ContextFlags = CONTEXT_FULL;
+    GetThreadContext(GetCurrentThread(), &context);
+    if (target_kernel32) {
+        assert(false);
+#if 0 /* not implemented here */
+        /* For kernel32!BaseThreadStartThunk CXT_XAX contains the address of the
+         * thread routine and CXT_XBX the arg */
+        context.CXT_XSP = (ptr_uint_t)stack.ExpandableStackBase;
+        context.CXT_XIP = (ptr_uint_t)get_kernel_thread_start_thunk();
+        context.CXT_XAX = (ptr_uint_t)start_addr;
+        context.CXT_XBX = (ptr_uint_t)arg;
+#endif
+    } else {
+        ptr_uint_t buf[2];
+        bool res;
+        SIZE_T written;
+        /* directly targeting the start_address */
+        context.CXT_XSP = ((ptr_uint_t)stack.ExpandableStackBase) - sizeof(buf);
+        context.CXT_XIP = (ptr_uint_t)start_addr;
+        /* set up arg on stack, give NULL return address */
+        buf[0] = (ptr_uint_t)arg;
+        buf[1] = 0;
+        res = WriteProcessMemory(hProcess, (void *)context.CXT_XSP, &buf,
+                                 sizeof(buf), &written);
+        if (!res || written != sizeof(buf)) {
+            goto error;
+        }
+    }                           
+    if (context.CXT_XIP == 0) {
+        goto error;
+    }
+
+    /* NOTE - CreateThread passes NULL for object attributes so despite Nebbet
+     * must be optional (checked NTsp6a, XPsp2). We don't pass NULL so we can
+     * specify the security descriptor. */
+    if (!NT_SUCCESS(NtCreateThread(&hThread, THREAD_ALL_ACCESS, &oa, 
+                                   hProcess, &cid, &context, &stack, 
+                                   (byte)(suspended ? TRUE : FALSE)))) {
+        goto error;
+    }
+
+    if (tid != NULL)
+        *tid = (ptr_uint_t)cid.UniqueThread;
+
+ exit:
+    if (sd != NULL) {
+        /* Free the security descriptor. */
+        LocalFree(sd);
+    }
+
+    return hThread;
+
+ error:
+    if (stack.ExpandableStackBottom != NULL) {
+        /* Free remote stack on error. */
+        VirtualFreeEx(hProcess, stack.ExpandableStackBottom, 0, MEM_RELEASE);
+    }
+    assert(hThread == NULL);
+    hThread = NULL; /* just to be safe */
+    goto exit;
+}
+
+/***************************************************************************/
+
+typedef void (*fptr)();
+
+char* badfunc;
+
+void
+run_test()
+{
+    EXCEPTION_RECORD exception;
+    CONTEXT *context;
+    /* allocate on the stack for more reliable addresses for template
+     * than if in data segment
+     */
+    char badfuncbuf[1000];
+
+    badfunc = (char *) ALIGN_FORWARD(badfuncbuf, 256);
+
+    /* FIXME: make this fancier */
+    badfunc[0] = 0xc3; /* ret */
+
+    /* FIXME: move this to a general win32/ test */
+    /* Verifying RaiseException, FIXME: maybe move to a separate unit test */
+    __try {
+        ULONG arguments[] = {0, 0xabcd};
+        initialize_registry_context();
+        RaiseException(EXCEPTION_ACCESS_VIOLATION, 0, 2, arguments);
+        print("Never after RaiseException\n");
+    }
+    __except (
+              print("In RaiseException filter\n"),
+#ifdef RAISE_EXCEPTION_EIP_NOT_CONSTANT
+              dump_exception_info((GetExceptionInformation())->ExceptionRecord,
+                                  (GetExceptionInformation())->ContextRecord),
+#endif
+              EXCEPTION_EXECUTE_HANDLER
+              ) {
+        print("In RaiseException handler\n");
+    }
+
+    /* FIXME: move this to a general win32/ test except-unreadable.c 
+          see case 197 and should also cover a direct CTI to bad memory, 
+          the latter not very likely in real apps though */
+
+    /* These invalid address exceptions are to unreadable memory
+       and we should try to transparently return normal exceptions
+       [most likely these will remain unhandled]
+       
+       We want to make sure people get the seriousness of the situation
+       and every case in which this happens should be taken as a real attack vector.
+       Usually they would see this as a bug, 
+       but it may also be an attacker probing with AAAA.
+
+       Therefore, we should let execution continue only in -detect_mode
+       (fake) exceptions should be generated as if generated normally
+    */
+    __try {
+        __try {
+            __try {
+                initialize_registry_context();
+                __asm {
+                    mov   eax, 0xbadcdef0
+                    call  dword ptr [eax]
+                }
+                print("At statement after exception\n");
+            }
+            __except (
+                      context = (GetExceptionInformation())->ContextRecord,
+                      print("Inside first filter eax=%x\n", context->CXT_XAX),
+                      dump_exception_info((GetExceptionInformation())->ExceptionRecord,
+                                          context),
+                      context->CXT_XAX = 0xcafebabe,
+                      /* FIXME: should get a CONTINUE to work  */
+                      EXCEPTION_CONTINUE_EXECUTION, 
+                      EXCEPTION_EXECUTE_HANDLER) {
+                print("Inside first handler\n");
+            }
+            print("At statement after 1st try-except\n");
+            __try {
+                initialize_registry_context();
+                __asm {
+                    mov   edx, 0xdeadbeef
+                    call  edx
+                }
+                print("NEVER Inside 2nd try\n");
+            } __except (print("Inside 2nd filter\n"), 
+                        EXCEPTION_CONTINUE_SEARCH) {
+                print("This should NOT be printed\n");
+            }
+        }
+        __finally {
+            print("Finally!\n");
+        }
+        print("NEVER At statement after 2nd try-finally\n");
+    }
+    __except (print("Inside 3rd filter\n"), 
+              exception = *(GetExceptionInformation())->ExceptionRecord,
+              context = (GetExceptionInformation())->ContextRecord,
+              dump_exception_info(&exception, context),
+              (GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION) ?
+              EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) {
+        /* copy of exception can be used */
+        print("Expected memory access violation, ignoring it!\n");
+    }
+    print("After exception handler\n");
+
+    /* Security violation handled by exceptions here */
+    /* Now we should start getting different results with code origins,
+       because the addresses are normally readable, but now we impose our own policy
+
+       -detect_mode should behave normally 
+          no exceptions are triggered
+
+       -throw_exception -no_detect_mode 
+          should behave similarly to the handling of unreadable memory above
+          (fake) exceptions should be generated claiming that badfunc is not executable
+     */
+
+    print("Attempting execution of badfunc\n");
+    __try {
+        __try {
+            __try {
+                initialize_registry_context();
+                ((fptr)badfunc)();
+                print("DATA: At statement after exception\n");
+            }
+            __except (
+                      context = (GetExceptionInformation())->ContextRecord,
+                      print("DATA VIOLATION: Inside first filter eax=%x\n", 
+                            context->CXT_XAX),
+                      dump_exception_info((GetExceptionInformation())->ExceptionRecord,
+                                          context),
+                      EXCEPTION_EXECUTE_HANDLER) {
+                print("DATA VIOLATION: Inside first handler\n");
+            }
+            print("DATA: At statement after 1st try-except\n");
+            __try {
+                initialize_registry_context();
+                ((fptr)badfunc)(); 
+                print("DATA: Inside 2nd try\n");
+            } __except (EXCEPTION_CONTINUE_SEARCH) {
+                print("DATA: This should NOT be printed\n");
+            }
+        }
+        __finally {
+            print("DATA: Finally!\n");
+        }
+        print("DATA: At statement after 2nd try-finally\n");
+    }
+    __except (
+              exception = *(GetExceptionInformation())->ExceptionRecord,
+              context = (GetExceptionInformation())->ContextRecord,
+              (GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION) ?
+              EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) {
+        print("DATA: Expected execution violation!\n");
+        dump_exception_info(&exception, context);
+    }
+    print("DATA: After exception handler\n");
+}
+
+int
+thread_func()
+{
+    __try {
+        run_test();
+    } __except(EXCEPTION_EXECUTE_HANDLER) {
+        print("Should never have exception bubble up to thread function\n");
+    }
+    /* thread is hanging so just exit here */
+    ExitThread(0);
+    return 0;
+}
+
+int
+main()
+{
+    HANDLE hThread;
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    /* I tried just making a new stack and swapping to it but had trouble
+     * w/ exception unwinding walking off the stack even though I put 0
+     * in fs:0.  In any case, making a raw thread does the trick.
+     */
+    hThread = (HANDLE)
+        nt_create_thread(GetCurrentProcess(), (threadfunc_t) thread_func,
+                         NULL, STACK_RESERVE, STACK_COMMIT, false, NULL, false);
+    WaitForSingleObject(hThread, INFINITE);
+
+        
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
diff --git a/suite/tests/security-win32/except-execution.template b/suite/tests/security-win32/except-execution.template
new file mode 100644
index 000000000..cae659bb7
--- /dev/null
+++ b/suite/tests/security-win32/except-execution.template
@@ -0,0 +1,117 @@
+In RaiseException filter
+In RaiseException handler
+Inside first filter eax=badcdef0
+	exception code = 0xc0000005, ExceptionFlags=0x00000000
+	record=00000000, params=2
+	PC 0x00401815 tried to read address 0xbadcdef0
+	pc=0x00401815 eax=0xbadcdef0
+#if +RUNREGRESSION_NT 
+  ContextFlags=0x0001001f 
+#else
+  ContextFlags=0x0001003f 
+#endif
+  Edi=0xeecdcdcd Esi=0xffcdcdcd Ebx=0xbbcdcdcd 
+  Edx=0xddcdcdcd Ecx=0xcccdcdcd 
+
+  Eax=0xbadcdef0 
+  Ebp=0x1400efc4 Eip=0x00401815 SegCs=0x0000001b 
+  EFlags & 0xFFFF=0x00000202 Esp=0x1400eb48 SegSs=0x00000023 
+  <floating point area>
+  SegGs=0x00000000 SegFs=0x0000003b SegEs=0x00000023 SegDs=0x00000023 
+Inside first handler
+At statement after 1st try-except
+Inside 2nd filter
+Inside 3rd filter
+	exception code = 0xc0000005, ExceptionFlags=0x00000000
+	record=00000000, params=2
+	PC 0xdeadbeef tried to read address 0xdeadbeef
+	pc=0xdeadbeef eax=0x00000000
+#if +RUNREGRESSION_NT 
+  ContextFlags=0x0001001f 
+#else
+  ContextFlags=0x0001003f 
+#endif
+  Edi=0xeecdcdcd Esi=0xffcdcdcd Ebx=0xbbcdcdcd 
+  Edx=0xdeadbeef Ecx=0xcccdcdcd 
+
+  Eax=0x00000000
+  Ebp=0x1400efc4 Eip=0xdeadbeef SegCs=0x0000001b 
+  EFlags & 0xFFFF=0x00000202 Esp=0x1400eb44 SegSs=0x00000023 
+  <floating point area>
+  SegGs=0x00000000 SegFs=0x0000003b SegEs=0x00000023 SegDs=0x00000023 
+Finally!
+Expected memory access violation, ignoring it!
+After exception handler
+Attempting execution of badfunc
+#if +PROGRAM_SHEPHERDING : +-security --detect_mode --kill_thread --throw_exception --executable_stack --low --client --thin_client || +PROGRAM_SHEPHERDING : +-security +-internal_detach_mask +-0x1
+#  SEC_VIO_STOP
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+#endif
+#if +PROGRAM_SHEPHERDING : +-security +-detect_mode --low --client --thin_client
+#  SEC_VIO_CONT
+#endif
+#if +PROGRAM_SHEPHERDING : +-security --detect_mode +-kill_thread --throw_exception --low --client --thin_client
+#  SEC_VIO_THREAD
+#  STOP
+#endif
+#if +PROGRAM_SHEPHERDING : +-security --detect_mode +-throw_exception --low --client --thin_client
+#  SEC_VIO_EXCEPTION
+DATA VIOLATION: Inside first filter eax=0
+	exception code = 0xc0000005, ExceptionFlags=0x00000000
+	record=00000000, params=2
+	PC 0x1400ec00 tried to read address 0x1400ec00
+	pc=0x1400ec00 eax=0x00000000
+#if +RUNREGRESSION_NT 
+  ContextFlags=0x0001001f 
+#else
+  ContextFlags=0x0001003f 
+#endif
+  Edi=0xeecdcdcd Esi=0xffcdcdcd Ebx=0xbbcdcdcd 
+  Edx=0xddcdcdcd Ecx=0xcccdcdcd 
+  
+  Eax=0x00000000 
+  Ebp=0x1400efc4 Eip=0x1400ec00 SegCs=0x0000001b 
+  EFlags & 0xFFFF=0x00000202 Esp=0x1400eb54 SegSs=0x00000023 
+  <floating point area>
+  SegGs=0x00000000 SegFs=0x0000003b SegEs=0x00000023 SegDs=0x00000023 
+DATA VIOLATION: Inside first handler
+DATA: At statement after 1st try-except
+#  if +PROGRAM_SHEPHERDING : +-security +-throw_exception +-throw_exception_max +1
+#    if  : +-kill_thread
+#      SEC_VIO_THREAD
+#    else
+#      SEC_VIO_STOP
+#    endif
+#  else
+#    SEC_VIO_EXCEPTION
+DATA: Finally!
+DATA: Expected execution violation!
+	exception code = 0xc0000005, ExceptionFlags=0x00000000
+	record=00000000, params=2
+	PC 0x1400ec00 tried to read address 0x1400ec00
+	pc=0x1400ec00 eax=0x00000000
+#if +RUNREGRESSION_NT 
+  ContextFlags=0x0001001f 
+#else
+  ContextFlags=0x0001003f 
+#endif
+  Edi=0xeecdcdcd Esi=0xffcdcdcd Ebx=0xbbcdcdcd 
+  Edx=0xddcdcdcd Ecx=0xcccdcdcd 
+  
+  Eax=0x00000000 
+  Ebp=0x1400efc4 Eip=0x1400ec00 SegCs=0x0000001b 
+  EFlags & 0xFFFF=0x00000202 Esp=0x1400eb54 SegSs=0x00000023 
+  <floating point area>
+  SegGs=0x00000000 SegFs=0x0000003b SegEs=0x00000023 SegDs=0x00000023 
+DATA: After exception handler
+#  endif
+#  STOP
+#endif
+DATA: At statement after exception
+DATA: At statement after 1st try-except
+DATA: Inside 2nd try
+DATA: Finally!
+DATA: At statement after 2nd try-finally
+DATA: After exception handler
diff --git a/suite/tests/security-win32/except-thread.c b/suite/tests/security-win32/except-thread.c
new file mode 100644
index 000000000..ccee81473
--- /dev/null
+++ b/suite/tests/security-win32/except-thread.c
@@ -0,0 +1,149 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#include "except.h"
+
+typedef void (*fptr)();
+
+/* PR 229292: we need to isolate this var to avoid flushes to it via
+ * writes to other vars
+ */
+#pragma data_seg(".isolate")
+char badfuncbuf[1000] = {0,};
+#pragma data_seg()
+char* badfunc;
+
+DWORD WINAPI
+call_bad_code(LPVOID parm)
+{
+    EXCEPTION_RECORD exception;
+    CONTEXT *context;
+
+    /* Now we should start getting different results with code origins,
+       because the addresses are normally readable, but now we impose our own policy
+
+       -detect_mode should behave normally 
+          no exceptions are triggered
+
+       -throw_exception -no_detect_mode 
+          should behave similarly to the handling of unreadable memory above
+          (fake) exceptions should be generated claiming that badfunc is not executable
+     */
+
+    print("Attempting execution of badfunc\n");
+    __try {
+	__try {
+	    __try {
+                initialize_registry_context();
+                ((fptr)badfunc)();
+		print("DATA: At statement after exception\n");
+	    }
+	    __except (
+                      exception = *((GetExceptionInformation())->ExceptionRecord),
+		      context = (GetExceptionInformation())->ContextRecord,
+                      print("DATA VIOLATION: Inside first filter eax=%x\n", 
+                             context->CXT_XAX),
+#ifdef DUMP_REGISTER_STATE
+                      dump_exception_info(&exception, context),
+#else 
+                      print("Address match : %s\n", (exception.ExceptionAddress == badfunc && context->CXT_XIP == badfunc) ? "yes" : "no"),
+                      print("Exception match : %s\n", (exception.ExceptionCode == 0xc0000005 && exception.ExceptionInformation[0] == 0 && exception.ExceptionInformation[1] == badfunc) ? "yes" : "no"),
+#endif
+		      EXCEPTION_EXECUTE_HANDLER) {
+		print("DATA VIOLATION: Inside first handler\n");
+	    }
+	    print("DATA: At statement after 1st try-except\n");
+	    __try {
+                initialize_registry_context();
+                ((fptr)badfunc)(); 
+		print("DATA: Inside 2nd try\n");
+	    } __except (EXCEPTION_CONTINUE_SEARCH) {
+		print("DATA: This should NOT be printed\n");
+	    }
+	}
+	__finally {
+	    print("DATA: Finally!\n");
+	}
+	print("DATA: At statement after 2nd try-finally\n");
+    }
+    __except (
+	      exception = *((GetExceptionInformation())->ExceptionRecord),
+	      context = (GetExceptionInformation())->ContextRecord,
+	      (GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION) ?
+	      EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) {
+	print("DATA: Expected execution violation!\n");
+#ifdef DUMP_REGISTER_STATE
+        dump_exception_info(&exception, context);
+#else 
+        print("Address match : %s\n", (exception.ExceptionAddress == badfunc && context->CXT_XIP == badfunc) ? "yes" : "no");
+        print("Exception match : %s\n", (exception.ExceptionCode == 0xc0000005 && exception.ExceptionInformation[0] == 0 && exception.ExceptionInformation[1] == badfunc) ? "yes" : "no");
+#endif
+    }
+    print("DATA: After exception handler\n");
+    return 0;
+}
+
+int
+main()
+{
+    DWORD tid;
+    HANDLE ht;
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    badfunc = ALIGN_FORWARD(badfuncbuf, 512);
+    
+    /* FIXME: make this fancier */
+    badfunc[0] = 0xc3; /* ret */
+
+    print("THREAD0: Creating thread 1\n");
+    ht = CreateThread(NULL, 0, &call_bad_code, NULL, 0, &tid);
+    WaitForSingleObject(ht, INFINITE);
+    print("THREAD0: After running other thread\n");
+
+    call_bad_code(0);
+    print("THREAD0: After calling more bad code here\n");
+	
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
diff --git a/suite/tests/security-win32/except-thread.template b/suite/tests/security-win32/except-thread.template
new file mode 100644
index 000000000..68d496a3f
--- /dev/null
+++ b/suite/tests/security-win32/except-thread.template
@@ -0,0 +1,152 @@
+THREAD0: Creating thread 1
+Attempting execution of badfunc
+#if +PROGRAM_SHEPHERDING : +-security --detect_mode --kill_thread --throw_exception --executable_heap --low --client --thin_client
+# SEC_VIO_STOP
+# if : --internal_detach_mask
+#  STOP
+# endif
+#endif
+#if +PROGRAM_SHEPHERDING : +-security --detect_mode +-kill_thread --throw_exception
+# SEC_VIO_THREAD
+THREAD0: After running other thread
+Attempting execution of badfunc
+#  if +PROGRAM_SHEPHERDING : +-security +-kill_thread_max +1
+#   SEC_VIO_STOP
+#  else
+#   SEC_VIO_THREAD
+#  endif
+# STOP
+#endif
+#if +PROGRAM_SHEPHERDING : +-security --detect_mode +-throw_exception
+# SEC_VIO_EXCEPTION
+DATA VIOLATION: Inside first filter eax=0
+# if +DUMP_REGISTER_STATE :
+ /* Fixme : register state non consisten across different machines */
+	exception code = 0xc0000005, ExceptionFlags=0x00000000
+	record=00000000, params=2
+	PC 0x00423400 tried to read address 0x00423400
+	pc=0x00423400 eax=0x00000000
+  ContextFlags=0x0001003f 
+  Edi=0x00420220 Esi=0x7ffdf000 Ebx=0x00000000 
+  
+  Eax=0x00000000 
+  Ebp=0x0052ffb4 Eip=0x00423400 SegCs=0x0000001b 
+  EFlags & 0xFFFF=0x00000202 Esp=0x0052ff34 SegSs=0x00000023 
+  <floating point area>
+  SegGs=0x00000000 SegFs=0x00000038 SegEs=0x00000023 SegDs=0x00000023
+# else
+Address match : yes
+Exception match : yes
+# endif
+DATA VIOLATION: Inside first handler
+DATA: At statement after 1st try-except
+#  if +PROGRAM_SHEPHERDING : +-security +-throw_exception_max +1 +-kill_thread
+#    SEC_VIO_THREAD
+#  endif
+#  if : --throw_exception_max_per_thread +1 --throw_exception_max +1
+#    SEC_VIO_EXCEPTION
+DATA: Finally!
+DATA: Expected execution violation!
+#   if +DUMP_REGISTER_STATE :
+	exception code = 0xc0000005, ExceptionFlags=0x00000000
+	record=00000000, params=2
+	PC 0x00423400 tried to read address 0x00423400
+	pc=0x00423400 eax=0x00000000
+  ContextFlags=0x0001003f 
+  Edi=0x0041c020 Esi=0x00000002 Ebx=0x0052ffa4 
+  
+  Eax=0x00000000 
+  Ebp=0x0052ffb4 Eip=0x00423400 SegCs=0x0000001b 
+  EFlags & 0xFFFF=0x00000202 Esp=0x0052ff34 SegSs=0x00000023 
+  <floating point area>
+  SegGs=0x00000000 SegFs=0x0000003b SegEs=0x00000023 SegDs=0x00000023
+#   else
+Address match : yes
+Exception match : yes
+#   endif 
+DATA: After exception handler
+#  endif
+#  if +PROGRAM_SHEPHERDING : +-security +-throw_exception_max +1 --kill_thread
+#    SEC_VIO_STOP
+#    STOP
+#  endif
+THREAD0: After running other thread
+Attempting execution of badfunc
+#  if +PROGRAM_SHEPHERDING : +-security +-throw_exception_max +1 +-kill_thread_max +1
+#    SEC_VIO_STOP
+#    STOP
+#  endif
+#  if +PROGRAM_SHEPHERDING : +-security +-throw_exception_max +1 +-kill_thread
+#    SEC_VIO_THREAD
+#    STOP
+#  endif
+#  SEC_VIO_EXCEPTION
+DATA VIOLATION: Inside first filter eax=0
+#  ifdef +DUMP_REGISTER_STATE :
+	exception code = 0xc0000005, ExceptionFlags=0x00000000
+	record=00000000, params=2
+	PC 0x00423400 tried to read address 0x00423400
+	pc=0x00423400 eax=0x00000000
+  ContextFlags=0x0001003f 
+  Edi=0x0012eb0c Esi=0x00420220 Ebx=0x7ffdf000 
+  
+  Eax=0x00000000 
+  Ebp=0x0012ff60 Eip=0x00423400 SegCs=0x0000001b 
+  EFlags & 0xFFFF=0x00000216 Esp=0x0012fee0 SegSs=0x00000023 
+  <floating point area>
+  SegGs=0x00000000 SegFs=0x00000038 SegEs=0x00000023 SegDs=0x00000023
+#  else
+Address match : yes
+Exception match : yes
+#  endif 
+DATA VIOLATION: Inside first handler
+DATA: At statement after 1st try-except
+#  SEC_VIO_EXCEPTION
+DATA: Finally!
+DATA: Expected execution violation!
+#  if +DUMP_REGISTER_STATE :
+	exception code = 0xc0000005, ExceptionFlags=0x00000000
+	record=00000000, params=2
+	PC 0x00423400 tried to read address 0x00423400
+	pc=0x00423400 eax=0x00000000
+  ContextFlags=0x0001003f 
+  Edi=0x0041c020 Esi=0x00000002 Ebx=0x0012ff50 
+  
+  Eax=0x00000000 
+  Ebp=0x0012ff60 Eip=0x00423400 SegCs=0x0000001b 
+  EFlags & 0xFFFF=0x00000206 Esp=0x0012fee0 SegSs=0x00000023 
+  <floating point area>
+  SegGs=0x00000000 SegFs=0x0000003b SegEs=0x00000023 SegDs=0x00000023
+#  else
+Address match : yes
+Exception match : yes
+#  endif 
+DATA: After exception handler
+THREAD0: After calling more bad code here
+# STOP
+#endif
+#if +PROGRAM_SHEPHERDING : +-security --detect_mode +-throw_exception +-throw_exception_max +1 +-kill_thread +-kill_thread_max +1 --low --client --thin_client
+# SEC_VIO_EXCEPTION
+# SEC_VIO_THREAD
+# SEC_VIO_EXCEPTION
+# SEC_VIO_STOP
+# STOP
+#endif
+#if +PROGRAM_SHEPHERDING : +-security +-detect_mode
+# SEC_VIO_CONT
+#endif
+DATA: At statement after exception
+DATA: At statement after 1st try-except
+DATA: Inside 2nd try
+DATA: Finally!
+DATA: At statement after 2nd try-finally
+DATA: After exception handler
+THREAD0: After running other thread
+Attempting execution of badfunc
+DATA: At statement after exception
+DATA: At statement after 1st try-except
+DATA: Inside 2nd try
+DATA: Finally!
+DATA: At statement after 2nd try-finally
+DATA: After exception handler
+THREAD0: After calling more bad code here
diff --git a/suite/tests/security-win32/except.h b/suite/tests/security-win32/except.h
new file mode 100644
index 000000000..92d822752
--- /dev/null
+++ b/suite/tests/security-win32/except.h
@@ -0,0 +1,134 @@
+/* **********************************************************
+ * Copyright (c) 2003-2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* except.h  shared exception info routines */
+/* NOTE this is not really a header! */
+
+#ifndef EXCEPT_H
+#define EXCEPT_H
+
+#include "tools.h"
+
+#define initialize_registry_context() /* stick to a known value */      \
+     __asm { push 0 }                                                   \
+     __asm { popfd }                                                    \
+     __asm { mov  ebx, 0xbbcdcdcd }                                     \
+     __asm { mov  ecx, 0xcccdcdcd }                                     \
+     __asm { mov  edx, 0xddcdcdcd }                                     \
+     __asm { mov  edi, 0xeecdcdcd }                                     \
+     __asm { mov  esi, 0xffcdcdcd }                                     
+
+#define NO_DEBUG_REGISTERS      /* We shouldn't be messing with these registers, 
+                                   so diffs here maybe not our fault (VMware's?) */
+
+void
+dump_context_info(CONTEXT *context, int all)
+{
+#define DUMP(r) print(#r"=0x%08x ", context->r);
+    print("  ");
+    DUMP(ContextFlags);
+    print("\n  ");
+
+    if (all || context->ContextFlags & CONTEXT_INTEGER) {
+        DUMP(Edi);
+        DUMP(Esi);
+        DUMP(Ebx);
+        print("\n  ");
+        DUMP(Edx);
+        DUMP(Ecx);
+        print("\n  ");
+        print("\n  ");
+        DUMP(Eax);
+        print("\n  ");
+    }
+
+    if (all || context->ContextFlags & CONTEXT_CONTROL) {
+        DUMP(Ebp);
+        DUMP(Eip);
+        DUMP(SegCs);              // MUST BE SANITIZED
+        print("\n  ");
+        /* only printing low word - RF is different between SP0 & SP4 */
+        DUMP(EFlags & 0xFFFF);
+        DUMP(Esp);
+        DUMP(SegSs);
+        print("\n  ");
+    }
+
+#ifndef NO_DEBUG_REGISTERS
+    if (all || context->ContextFlags & CONTEXT_DEBUG_REGISTERS) {
+        DUMP(Dr0);
+        DUMP(Dr1);
+        DUMP(Dr2);
+        DUMP(Dr3);
+        print("\n  ");
+        DUMP(Dr6);
+        DUMP(Dr7);
+        print("\n  ");
+    }
+#endif
+
+    if (all || context->ContextFlags & CONTEXT_FLOATING_POINT) {
+        print("<floating point area>\n  ");
+    }
+
+    if (all || context->ContextFlags & CONTEXT_SEGMENTS) {
+        DUMP(SegGs);
+        DUMP(SegFs);
+        DUMP(SegEs);
+        DUMP(SegDs);
+    }
+
+#undef DUMP
+    print("\n");
+}
+
+void
+dump_exception_info(EXCEPTION_RECORD* exception, CONTEXT *context)
+{
+    print("\texception code = 0x%08x, ExceptionFlags=0x%08x\n\trecord=%p, params=%d\n",
+           exception->ExceptionCode,
+           exception->ExceptionFlags,
+           exception->ExceptionRecord, /* follow if non NULL */
+           exception->NumberParameters);
+    if(exception->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
+        print("\tPC 0x%08x tried to %s address 0x%08x\n",
+               exception->ExceptionAddress, 
+               (exception->ExceptionInformation[0]==0)?"read":"write",
+               exception->ExceptionInformation[1]);
+    }
+    print("\tpc=0x%08x eax=0x%08x\n", 
+           context->Eip, context->Eax);
+    dump_context_info(context, 0); /* existing context */
+}
+
+
+#endif
diff --git a/suite/tests/security-win32/gbop-test.c b/suite/tests/security-win32/gbop-test.c
new file mode 100644
index 000000000..281b44f87
--- /dev/null
+++ b/suite/tests/security-win32/gbop-test.c
@@ -0,0 +1,156 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* gbop-test.c */
+
+#include <windows.h>
+#include "tools.h"
+
+#ifdef VERBOSE 
+#  define pf print
+#else
+#  define pf if(0)print
+#endif
+
+#define REPORT(res)                                             \
+    pf("res=%x\n", res);                                        \
+    print("res=%s\n",                                           \
+          (res == kernel32_base) ? "kernel32_base" : "BAD");    \
+    res = 0;
+
+int
+main()
+{
+    char* arg0 = "kernel32.dll";
+    int res;
+    int stack = 0;
+    int kernel32_base;
+
+    INIT();
+
+    print("plain C call\n");
+    res = LoadLibrary(arg0);
+
+    kernel32_base = res;
+    REPORT(res);
+
+    print("plain asm call\n");
+
+    __asm {
+        push arg0
+            /* note using the IAT directly */
+        call dword ptr LoadLibraryA
+        mov res, eax
+    }
+    REPORT(res);
+
+    print("pretend on stack\n");
+    /* note that this is a pseudo attack, in fact we get a .C
+     * violation on the return back, while GBOP will react to it
+     * immediately before executing the target.
+     */
+    __try {
+        __asm {
+            push arg0
+                push esp
+                jmp dword ptr LoadLibraryA
+                mov res, eax
+              }
+    } __except (
+                (res = (GetExceptionInformation())->ContextRecord->CXT_XAX),
+                EXCEPTION_EXECUTE_HANDLER) {
+        print("exception since not cleaning up stack\n");
+    }
+    REPORT(res);
+
+    __asm {
+        mov stack, esp
+    }
+    /* now make some portion of the stack look good - this will be
+     * forever in this mode
+     */
+    NTFlush(stack - 0x1000, 0x2000);
+    
+    print("pretend on flushed stack\n");
+    /* note that this is a pseudo attack, in fact we get a .C
+     * violation on the return back, while GBOP will react to it
+     * immediately before executing the target.
+     */
+    __try {
+        __asm {
+            push arg0
+                push esp
+                jmp dword ptr LoadLibraryA
+                mov res, eax
+              }
+    } __except (
+                (res = (GetExceptionInformation())->ContextRecord->CXT_XAX),
+                EXCEPTION_EXECUTE_HANDLER) {
+        print("exception since not cleaning up stack\n");
+    }
+    REPORT(res);
+
+    print("pretend in image but not after call\n");
+    __try {
+        __asm {
+                push arg0
+                /* note this is a true replacement of a CALL instruction 
+                 * but we don't currently allow this - may change with GBOP_IS_JMP
+                 */
+                push offset after_jmp
+                jmp dword ptr LoadLibraryA
+                mov res, eax
+              }
+    } __except (
+                (res = (GetExceptionInformation())->ContextRecord->CXT_XAX),
+                EXCEPTION_EXECUTE_HANDLER) {
+        /* stack should be properly cleaned up */
+        print("native exception unexpected, unless detected as violation\n");
+    }
+    REPORT(res);
+
+    goto done;
+
+    __asm {
+        jmp after_jmp
+    after_jmp:
+        mov res, eax
+
+    }
+    REPORT(res);
+
+    print("JMP allowed!\n");
+
+ done:
+    print("done\n");
+    return 0;
+}
diff --git a/suite/tests/security-win32/gbop-test.template b/suite/tests/security-win32/gbop-test.template
new file mode 100644
index 000000000..f7677522c
--- /dev/null
+++ b/suite/tests/security-win32/gbop-test.template
@@ -0,0 +1,45 @@
+plain C call
+res=kernel32_base
+plain asm call
+res=kernel32_base
+pretend on stack
+#if +PROGRAM_SHEPHERDING : +-security --C --gbop_exclude_list(kernel32.dll!LoadLibraryA) --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+#if +PROGRAM_SHEPHERDING : +-security --A --gbop_exclude_list(kernel32.dll!LoadLibraryA) --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+exception since not cleaning up stack
+res=kernel32_base
+pretend on flushed stack
+#if +PROGRAM_SHEPHERDING : +-security --A --gbop_exclude_list(kernel32.dll!LoadLibraryA) --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+exception since not cleaning up stack
+res=kernel32_base
+pretend in image but not after call
+#if +PROGRAM_SHEPHERDING : +-security --A --gbop_exclude_list(kernel32.dll!LoadLibraryA) --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+res=kernel32_base
+JMP allowed!
+done
diff --git a/suite/tests/security-win32/hooker-ntdll.c b/suite/tests/security-win32/hooker-ntdll.c
new file mode 100644
index 000000000..7f40a943c
--- /dev/null
+++ b/suite/tests/security-win32/hooker-ntdll.c
@@ -0,0 +1,225 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* case 2525 - hooker for LdrLoadDll or NtProtectVirtualMemory */
+
+/* one should use a CALL and the other should use a JMP 
+*   just to be sure
+* 
+*  FIXME: need to get this test also to be done like initapc.dll.c so that 
+*  this all happens BEFORE we take control
+**/
+
+#include "tools.h"
+#include <windows.h>
+
+typedef void (*fptr)();
+
+#define NAKED __declspec( naked ) 
+
+/* check for some unexpected behaviours with size = 5 and size = 0x1000, or even 0x2000 */
+enum {HOOK_SIZE = 0x1000};
+
+/*
+ * NTSTATUS LdrLoadDll(LPCWSTR path_name, DWORD flags, 
+ *                     PUNICODE_STRING libname, HMODULE* hModule)
+*/
+
+NAKED
+hooker4()
+{
+    __asm {
+        pusha
+        pushf
+    }
+    /* here we can do whatever we need to */
+
+    __asm {
+        popf
+        popa
+    }
+
+    /* know the number of args */
+    __asm {
+        ret 4*4;
+    }
+}
+
+NAKED
+hooker5()
+{
+    __asm {
+        pusha
+        pushf
+    }
+    /* here we can do whatever we need to */
+
+    __asm {
+        popf
+        popa
+    }
+
+    /* know the number of args */
+    __asm {
+        ret 5*4;
+    }
+}
+
+void
+do_hook(const char* hookfn, int args, int use_call)
+{
+    HANDLE ntdll = GetModuleHandle(TEXT("ntdll.DLL"));
+    unsigned char* hooktarget = (char*)GetProcAddress(ntdll, hookfn);
+
+    DWORD old_code1 = *(DWORD*)hooktarget;
+    DWORD old_code2 = *(DWORD*)(hooktarget+1);
+
+    DWORD size = HOOK_SIZE; /* although 5 bytes would have been just fine */
+    DWORD prev = 0xbadcde;
+    int res;
+
+    DWORD pc_rel_target;
+
+    enum {op_JMP  = 0xe9};
+    enum {op_CALL = 0xe8};
+
+    void* trampoline_target;
+
+    if (use_call)
+        args += 1;                /* clean up addr of CALL */
+
+    switch (args) {
+    case 4: 
+        trampoline_target = hooker4; break;
+    case 5: 
+        trampoline_target = hooker5; break;
+    default:
+        print("BAD args\n");
+    }
+
+    __try {
+        *hooktarget = 0xbaad;
+        print("bad: why is this writable?\n");
+    }
+    __except (EXCEPTION_EXECUTE_HANDLER) {
+        print("ok: can't write\n");
+    }
+    
+    res = VirtualProtect(hooktarget, 
+                         size,
+                         PAGE_EXECUTE_READWRITE,
+                         &prev);
+    print("VirtualProtect(%s["PFX"],%d,PAGE_EXECUTE_READWRITE,prev) = %d GLE="PFMT" prev="PFMT"\n", 
+          hookfn, (hooktarget /* disabled */,0), size, (res /* eax noisy */, 0) , GetLastError(), prev);
+
+#ifdef VERBOSE
+    print("before hooking %s = %02x %02x %02x %02x %02x\n", 
+          hookfn, 
+          hooktarget[0], hooktarget[1], hooktarget[2], hooktarget[3], hooktarget[4]);
+#endif
+
+    pc_rel_target =  (DWORD)trampoline_target - ((DWORD)hooktarget + 5); /* target = offset + cur pc + size*/ 
+
+    __try {
+        *hooktarget = use_call ? op_CALL : op_JMP;
+        *(DWORD*)(hooktarget+1) = pc_rel_target;
+
+        /* now let's get smart here and see if hook worked */
+#ifdef VERBOSE
+        print("after  hooking %s = %02x %02x %02x %02x %02x\n", 
+              hookfn, 
+              hooktarget[0], hooktarget[1], hooktarget[2], hooktarget[3], hooktarget[4]);
+#endif
+        if (*(DWORD*)(hooktarget+1) != pc_rel_target)
+            print("there be witches! what happened to my write?\n");
+        else
+            print("hooked %s\n", hookfn);
+        /* FIXME: try it out and see what happens */
+
+        /* restore */
+        *(DWORD*)hooktarget     = old_code1;
+        *(DWORD*)(hooktarget+1) = old_code2;
+
+        print("restored old code\n");
+    }
+    __except (EXCEPTION_EXECUTE_HANDLER) {
+        print("bad: can't write, though made writable\n");
+    }
+
+
+    /* restore page permissoins now, could be optional */
+    res = VirtualProtect(hooktarget, 
+                         size,
+                         PAGE_EXECUTE_READ,
+                         &prev);
+    print("VirtualProtect(%s["PFX"],%d,PAGE_EXECUTE_READ,...) = %d GLE="PFMT" \n", 
+          hookfn, (hooktarget /* disabled */,0), size, (res /* eax noisy */, 0) , GetLastError());
+    print("old permissions ...prev="PFMT")\n", prev);
+
+    __try {
+        *hooktarget = 0xbaad;
+        print("bad: why is this writable?\n");
+    }
+    __except (EXCEPTION_EXECUTE_HANDLER) {
+        print("ok: can't write\n");
+    }
+
+    print("all should be good\n");
+}
+
+int
+main()
+{
+    INIT();
+    
+    print("ready to hook\n");
+
+    do_hook("LdrLoadDll", 4, 1);
+    do_hook("LdrLoadDll", 4, 0);
+
+    /* hack: we'll pass 4 args instead of 3 */
+    do_hook("NtFlushInstructionCache", 4, 1);
+    do_hook("NtFlushInstructionCache", 4, 0);
+
+    /* hack: we'll pass 4 args instead of 0 */
+    /* hooking a function we really don't care much about */
+    /* FIXME: should we let this through or not? */
+    do_hook("NtFlushWriteBuffer", 4, 1);
+    do_hook("NtFlushWriteBuffer", 4, 0);
+
+    /* we have 4 writes to ntdll memory 
+     * on each of 6 calls to do_hook
+     * should get app_modify_ntdll_writes = 24
+     * FIXME: how to scrape a log for this?
+     */
+    print("hooking done with\n");
+}
diff --git a/suite/tests/security-win32/hooker-ntdll.template b/suite/tests/security-win32/hooker-ntdll.template
new file mode 100644
index 000000000..f152dcb47
--- /dev/null
+++ b/suite/tests/security-win32/hooker-ntdll.template
@@ -0,0 +1,98 @@
+ready to hook
+ok: can't write
+VirtualProtect(LdrLoadDll[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+#if +PROGRAM_SHEPHERDING : --thin_client
+there be witches! what happened to my write?
+#else
+hooked LdrLoadDll
+#endif
+restored old code
+VirtualProtect(LdrLoadDll[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+#if +PROGRAM_SHEPHERDING : --thin_client
+old permissions ...prev=00000020)
+#else
+old permissions ...prev=00000040)
+#endif
+ok: can't write
+all should be good
+ok: can't write
+VirtualProtect(LdrLoadDll[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+#if +PROGRAM_SHEPHERDING : --thin_client
+there be witches! what happened to my write?
+#else
+hooked LdrLoadDll
+#endif
+restored old code
+VirtualProtect(LdrLoadDll[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+#if +PROGRAM_SHEPHERDING : --thin_client
+old permissions ...prev=00000020)
+#else 
+old permissions ...prev=00000040)
+#endif 
+ok: can't write
+all should be good
+ok: can't write
+VirtualProtect(NtFlushInstructionCache[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+#if +PROGRAM_SHEPHERDING : --thin_client
+there be witches! what happened to my write?
+#else 
+hooked NtFlushInstructionCache
+#endif 
+restored old code
+VirtualProtect(NtFlushInstructionCache[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+#if +PROGRAM_SHEPHERDING : --thin_client
+old permissions ...prev=00000020)
+#else 
+old permissions ...prev=00000040)
+#endif 
+ok: can't write
+all should be good
+ok: can't write
+VirtualProtect(NtFlushInstructionCache[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+#if +PROGRAM_SHEPHERDING : --thin_client
+there be witches! what happened to my write?
+#else 
+hooked NtFlushInstructionCache
+#endif 
+restored old code
+VirtualProtect(NtFlushInstructionCache[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+#if +PROGRAM_SHEPHERDING : --thin_client
+old permissions ...prev=00000020)
+#else 
+old permissions ...prev=00000040)
+#endif 
+ok: can't write
+all should be good
+ok: can't write
+VirtualProtect(NtFlushWriteBuffer[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+#if +PROGRAM_SHEPHERDING : --thin_client
+there be witches! what happened to my write?
+#else 
+hooked NtFlushWriteBuffer
+#endif 
+restored old code
+VirtualProtect(NtFlushWriteBuffer[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+#if +PROGRAM_SHEPHERDING : --thin_client
+old permissions ...prev=00000020)
+#else 
+old permissions ...prev=00000040)
+#endif 
+ok: can't write
+all should be good
+ok: can't write
+VirtualProtect(NtFlushWriteBuffer[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+#if +PROGRAM_SHEPHERDING : --thin_client
+there be witches! what happened to my write?
+#else 
+hooked NtFlushWriteBuffer
+#endif 
+restored old code
+VirtualProtect(NtFlushWriteBuffer[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+#if +PROGRAM_SHEPHERDING : --thin_client
+old permissions ...prev=00000020)
+#else 
+old permissions ...prev=00000040)
+#endif 
+ok: can't write
+all should be good
+hooking done with
diff --git a/suite/tests/security-win32/hooker.c b/suite/tests/security-win32/hooker.c
new file mode 100644
index 000000000..47e4a7e2a
--- /dev/null
+++ b/suite/tests/security-win32/hooker.c
@@ -0,0 +1,257 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* case 3102 - multiple VirtualProtect calls on the same page */
+
+/* tasks
+Hook three times the same guy with different hooks
+VirtualProtect()
+VirtualProtect()
+VirtualProtect()
+
+and make sure that they get properly executed
+since we'd have a cache consistency problem when leaving RWX
+
+Then we have a real hook that doesn't let the syscall through,
+therefore TWO writes are considered bad and we have a security violation.
+
+*/
+
+#include "tools.h"
+#include <windows.h>
+
+typedef void (*fptr)();
+
+const char* hookfn = "VirtualProtect"; /* and then next page */
+
+#define NDEP /* for testing case 10387 */
+#define FARAWAY_DLL "ADVAPI32.DLL"
+const char* faraway_hook = "RegOpenKeyA";
+/* FIXME: a lot better will be in one another module */
+
+#define NAKED __declspec( naked ) 
+
+/* FIXME: not fully fleshed out to add in regression suite need to
+ * find a function with the same prologue or actually do some real
+ * work to get it copied around.
+ */
+NAKED
+hooker1()
+{
+#if 0
+    /* a real hooker will take more than what's necessary for this test */
+    asm {
+        pusha
+        pushf
+    }
+    /* here we can do whatever we need to */
+
+    asm {
+        popf
+        popa
+    }
+
+    /* now it's time to get back to the original code */
+
+    /* HACK: assuming beginning of function always looks like */
+    asm {
+     push    ebp
+     mov     ebp,esp
+     push    dword ptr [ebp+0x14]
+    }
+    /* and now we should know where we're going */
+    /* JMP through memory to the address following our original instruction */
+#endif
+}
+
+/* check for some unexpected behaviours with size = 5 and size = 0x1000, or even 0x2000 */
+enum {HOOK_SIZE = 0x1000};
+
+
+/* FIXME: we have to test this on something other than kernel32!VirtualProtect for DEP 
+ * machines, for now keeping here
+ */
+void
+unset_x(DWORD* hooktarget, DWORD prot)
+{
+    DWORD prev;
+    DWORD size = HOOK_SIZE; /* although 5 bytes would have been just fine */
+    int res = VirtualProtect(hooktarget, 
+                             size,
+                             prot,
+                             &prev);
+    DWORD gle = GetLastError();
+    print("VirtualProtect(%s["PFX"],%d,"PFX",prev) = %d GLE="PFMT" prev="PFMT"\n", 
+          hookfn, (hooktarget /* disabled */,0), size, prot, res, gle, prev);
+
+    /* we don't touch anything here */
+
+    /* note we don't even bother to make again PAGE_EXECUTE_READ */
+}
+
+void
+hook(DWORD* hooktarget)
+{
+    DWORD prev;
+    DWORD size = HOOK_SIZE; /* although 5 bytes would have been just fine */
+    DWORD prot = PAGE_EXECUTE_READWRITE;
+    int res = VirtualProtect(hooktarget, 
+                             size,
+                             prot,
+                             &prev);
+    DWORD gle = GetLastError();
+    print("VirtualProtect(%s["PFX"],%d,"PFX",prev) = %d GLE="PFMT" prev="PFMT"\n", 
+          hookfn, (hooktarget /* disabled */,0), size, prot, res, gle, prev);
+
+    /* pretend hooking - actually a NOP */
+    *hooktarget = *hooktarget;
+
+    /* note we don't even bother to make again PAGE_EXECUTE_READ */
+}
+
+void
+ret_hook(DWORD* hooktarget)
+{
+    DWORD old_code = *hooktarget;
+
+    DWORD prev = 0xbadcde;
+    DWORD size = HOOK_SIZE; /* although 5 bytes would have been just fine */
+    int res;
+
+    /* Simple hook that simply does a ret 0x10 - to clear up the 4 arguments */
+    *hooktarget = 0x900010c2;     /* ret 0x10; nop  */
+
+    /* prev should remain unchanged */
+    res = VirtualProtect(hooktarget, 
+                         size,
+                         PAGE_EXECUTE_READWRITE,
+                         &prev);
+    /* FIXME: eax may be garbage - but seems to be the same as GLE */
+    print("VirtualProtect(%s["PFX"],%d,PAGE_EXECUTE_READWRITE,prev) = %d GLE="PFMT" prev="PFMT"\n", 
+          hookfn, (hooktarget /* disabled */,0), size, (res /* eax noisy */, 0) , GetLastError(), prev);
+
+    *hooktarget = old_code;
+}
+
+char *
+prot_string(uint prot)
+{
+    uint ignore_extras = prot & ~(PAGE_GUARD | PAGE_NOCACHE | PAGE_WRITECOMBINE);
+    switch (ignore_extras) {
+    case PAGE_NOACCESS:          return "----";
+    case PAGE_READONLY:          return "r---";
+    case PAGE_READWRITE:         return "rw--";
+    case PAGE_WRITECOPY:         return "rw-c";
+    case PAGE_EXECUTE:           return "--x-";
+    case PAGE_EXECUTE_READ:      return "r-x-";
+    case PAGE_EXECUTE_READWRITE: return "rwx-";
+    case PAGE_EXECUTE_WRITECOPY: return "rwxc";
+    }
+    return "(error)";
+}
+
+
+bool
+prot_is_executable(uint prot) 
+{
+    prot &= ~(PAGE_GUARD | PAGE_NOCACHE | PAGE_WRITECOMBINE);
+    return (prot == PAGE_EXECUTE || prot == PAGE_EXECUTE_READ ||
+            prot == PAGE_EXECUTE_READWRITE || prot == PAGE_EXECUTE_WRITECOPY);
+}
+
+void
+query(DWORD* hooktarget)
+{
+    MEMORY_BASIC_INFORMATION mbi;
+    DWORD res = VirtualQuery(hooktarget, &mbi, sizeof(mbi));
+    if ( res == sizeof(mbi)) {
+        print("VirtualQuery("PFX") = %d GLE="PFMT" prev="PFMT" %s\n",       
+              (hooktarget /* disabled */,0), res, GetLastError(), mbi.Protect, prot_string(mbi.Protect));
+        /* at least confirm EXECUTABLE */
+        print(" DEP => %s\n", prot_is_executable(mbi.Protect) ? 
+              "ok" : "NOT EXECUTABLE");
+    } else {
+        print("VirtualQuery("PFX") = %d GLE="PFMT"\n", 
+              (hooktarget /* disabled */,0), res , GetLastError());
+
+    }
+}
+
+int
+main()
+{
+    HANDLE kern32 = GetModuleHandle(TEXT("KERNEL32.DLL"));
+    DWORD* addr_hook = (DWORD)GetProcAddress(kern32, hookfn);
+
+    HANDLE far_dll = LoadLibrary(TEXT(FARAWAY_DLL));
+    DWORD* unset_hook = (DWORD)GetProcAddress(far_dll, faraway_hook);
+
+    INIT();
+    
+#ifdef NDEP
+    assert(far_dll != NULL);
+    assert(unset_hook != NULL);
+    assert(((ptr_int_t)unset_hook >> 12) != ((ptr_int_t)addr_hook >> 12));
+    /* we don't really execute from this */
+    print("unset X bit\n");
+    unset_x(unset_hook, PAGE_WRITECOPY);
+    unset_x(unset_hook, PAGE_READWRITE);
+    unset_x(unset_hook, PAGE_READWRITE|PAGE_GUARD);
+    unset_x(unset_hook, PAGE_READWRITE|PAGE_GUARD);
+    print("ready to hook far\n");
+    hook(unset_hook);
+    print("doublecheck flags\n");
+    query(unset_hook);
+#endif
+
+    print("ready to hook\n");
+    hook(addr_hook);
+    print("one more\n");
+    /* pretend we're hooking something else */
+    hook(addr_hook);
+    /* case 3102 triggers here */
+
+    print("now third ...\n");
+    /* and once more to see what happens */
+    hook(addr_hook);
+    print("doublecheck flags\n");
+    query(addr_hook);
+
+    /* now see if we didn't have proper consistency what would happen */
+    print("check consistency ...\n");
+    ret_hook(addr_hook);
+
+    /* and once more to see what happens */
+    hook(addr_hook);
+
+    print("hooking done with\n");
+}
diff --git a/suite/tests/security-win32/hooker.template b/suite/tests/security-win32/hooker.template
new file mode 100644
index 000000000..d127916b8
--- /dev/null
+++ b/suite/tests/security-win32/hooker.template
@@ -0,0 +1,43 @@
+unset X bit
+VirtualProtect(VirtualProtect[0x00000000],4096,0x00000008,prev) = 1 GLE=00000000 prev=00000020
+VirtualProtect(VirtualProtect[0x00000000],4096,0x00000004,prev) = 1 GLE=00000000 prev=00000008
+VirtualProtect(VirtualProtect[0x00000000],4096,0x00000104,prev) = 1 GLE=00000000 prev=00000008
+VirtualProtect(VirtualProtect[0x00000000],4096,0x00000104,prev) = 1 GLE=00000000 prev=00000108
+ready to hook far
+VirtualProtect(VirtualProtect[0x00000000],4096,0x00000040,prev) = 1 GLE=00000000 prev=00000108
+// case 10438 - we are losing the PAGE_WRITECOPY and had 004 or 0x08
+// and also PAGE_GUARD                Protect: PAGE_WRITECOPY + PAGE_GUARD            00000108
+doublecheck flags
+VirtualQuery(0x00000000) = 28 GLE=00000000 prev=00000040 rwx-
+ DEP => ok
+ready to hook
+#if +PROGRAM_SHEPHERDING : +-security +-no_executable_if_hook --executable_if_text --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+VirtualProtect(VirtualProtect[0x00000000],4096,0x00000040,prev) = 1 GLE=00000000 prev=00000020
+one more
+VirtualProtect(VirtualProtect[0x00000000],4096,0x00000040,prev) = 1 GLE=00000000 prev=00000040
+now third ...
+VirtualProtect(VirtualProtect[0x00000000],4096,0x00000040,prev) = 1 GLE=00000000 prev=00000040
+doublecheck flags
+VirtualQuery(0x00000000) = 28 GLE=00000000 prev=00000040 rwx-
+ DEP => ok
+check consistency ...
+VirtualProtect(VirtualProtect[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00badcde
+#if +PROGRAM_SHEPHERDING : +-security --executable_heap --executable_if_x --executable_if_text --X --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  STOP
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+VirtualProtect(VirtualProtect[0x00000000],4096,0x00000040,prev) = 1 GLE=00000000 prev=00000040
+hooking done with
diff --git a/suite/tests/security-win32/indexisting.c b/suite/tests/security-win32/indexisting.c
new file mode 100644
index 000000000..1d23ea098
--- /dev/null
+++ b/suite/tests/security-win32/indexisting.c
@@ -0,0 +1,213 @@
+/* **********************************************************
+ * Copyright (c) 2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* tests for indirect call and indirect jump RCT policies */
+
+/* FIXME: this should really be in security-common yet
+ *    rct_ind_{call,jump} is not yet supported on Linux, and all *
+ *    assembly needs to be recoded for gcc anyways */
+#include "tools.h"
+
+#include <ctype.h>
+
+typedef int (*fconvert)(int c);
+typedef int (*fmult)(int);
+
+int foo(int a, int lower)
+{
+    fconvert f = toupper;
+    int res;
+    if (lower)
+        f = tolower;
+    res = f(a);
+    print("%c\n", res);
+    return res;
+}
+
+int f2(int a)
+{
+    return 2*a;
+}
+
+int f3(int a)
+{
+    return 3*a;
+}
+
+int f7(int a)
+{
+    return 7*a;
+}
+
+int 
+bar(int a, fmult f)
+{
+    int x = f2(a);
+    int y = f3(a);
+    int z = f(a);
+    
+    print("%d\n", z);
+    return z;
+}
+
+/*  Writable yet initialized data indeed needs to be processed.*/
+fmult farr[2] = {f2, f7};
+
+static void 
+test_good_indcalls()
+{
+    foo('a', true); //a
+    foo('a', false);// 'A'
+    foo('Z', true); // 'z'
+    bar(5, f2);//10
+    bar(7, f3);// 21
+    bar(7, f3);// 21
+}
+
+int some_global = 123456;
+
+int
+precious(int arg)
+{
+    /* this is a very important function - also not address taken by itself */
+#ifdef USER32    /* map user32.dll for a RunAll test */
+    MessageBeep(0);
+#endif
+    print("PRECIOUS function shouldn't be reachable! ATTACK SUCCESSFUL!\n");
+    /* can continue */
+    return -666;
+}
+
+int
+good(int* arg)
+{
+    print("this is a normal function %d\n", *arg);
+    return 1;
+}
+
+int
+good2(int* arg)
+{
+    print("this is another normal function %d, but shouldn't be called!\n", *arg);
+    return 2;
+}
+
+static void
+test_PLT_with_indcalls(int table_index)
+{
+    table_index *= 5;  /* general instructions */
+    print("calling via PLT-style call\n");
+    __asm {
+        push offset some_global   // note that data references will also be absolute!
+
+        push offset plt_ind_call_table  // note that this will also be absolute!
+        pop eax
+        add eax, table_index
+
+        call eax   // THIS indirect call should succeed at going only to the first instruction
+                   // in this pseudo jump-table
+        jmp after_plt_call
+    plt_ind_call_table: 
+        jmp good     // label address taken 
+        jmp precious //  this address is not address taken
+        jmp good2    //  this address is not address taken
+
+    after_plt_call:
+        add esp,4 // arg
+    }
+}
+
+static void
+test_PLT_with_indjumps(int table_index)
+{
+    table_index *= 5;  /* general instructions */
+    print("calling via PLT-style indirect jmp\n");
+    __asm {
+        push offset some_global   // note that data references will also be absolute!
+
+        push offset plt_ind_call_table  // note that this will also be absolute!
+        pop eax
+        add eax, table_index
+
+        push offset after_plt_call // in fact desired location
+/* FIXME: note however that we're not going to like this with a .C violation! 
+ * unless we start allowing all addresses added with 'push offset'
+ */
+        jmp eax   // THIS indirect call should succeed at going only to the first instruction
+                  // in this pseudo jump-table
+    plt_ind_call_table:
+        jmp good   
+        jmp precious
+        jmp good2
+
+    after_plt_call:
+        add esp,4 // arg
+    }
+}
+
+/* TODO: need to add a test that a jump instead of a return is OK */
+
+__declspec( naked ) 
+int
+jmp_instead_of_ret()
+{
+    __asm {
+        mov eax, 42
+        add esp, 4  // is it safe to use the stack in-between these two ops?
+        jmp dword ptr [esp-4]
+    }
+}
+
+static void
+test_jmp_instead_of_ret()
+{
+    print("the answer is %d\n", jmp_instead_of_ret());
+}
+
+
+int
+main()
+{
+    INIT();
+
+    test_good_indcalls();
+    test_jmp_instead_of_ret();
+
+    test_PLT_with_indcalls(0);
+    test_PLT_with_indcalls(1);  /* VIOLATION expected */
+    test_PLT_with_indcalls(2);  /* VIOLATION expected */
+
+    test_PLT_with_indjumps(0);
+    test_PLT_with_indjumps(1); /* VIOLATION expected */
+
+    print("done\n");
+}
diff --git a/suite/tests/security-win32/indexisting.template b/suite/tests/security-win32/indexisting.template
new file mode 100644
index 000000000..210e1ca38
--- /dev/null
+++ b/suite/tests/security-win32/indexisting.template
@@ -0,0 +1,76 @@
+a
+A
+z
+10
+21
+21
+the answer is 42
+calling via PLT-style call
+this is a normal function 123456
+calling via PLT-style call
+// the default value is currently -rct_ind_call 0 
+// FIXME: we need support for option parameters when the default changes to 11
+// .E expected
+#if +PROGRAM_SHEPHERDING : +-security --X --E --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+PRECIOUS function shouldn't be reachable! ATTACK SUCCESSFUL!
+calling via PLT-style call
+// .E expected
+#if +PROGRAM_SHEPHERDING : +-security --X --E --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+this is another normal function 123456, but shouldn't be called!
+calling via PLT-style indirect jmp
+this is a normal function 123456
+// .C expected
+#if +PROGRAM_SHEPHERDING : +-security --no_ret_after_call --X --C --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+calling via PLT-style indirect jmp
+// .F expected
+#if +PROGRAM_SHEPHERDING : +-security --X --F --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+PRECIOUS function shouldn't be reachable! ATTACK SUCCESSFUL!
+// .C expected
+// unless target already cached
+#if +PROGRAM_SHEPHERDING : +-security --no_ret_after_call --X --C +-rct_cache_exempt(0) --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+done
diff --git a/suite/tests/security-win32/patterns.c b/suite/tests/security-win32/patterns.c
new file mode 100644
index 000000000..b0c857f45
--- /dev/null
+++ b/suite/tests/security-win32/patterns.c
@@ -0,0 +1,252 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <math.h>
+#include <process.h> /* for _beginthreadex */
+
+#include "tools.h" /* for print() */
+
+/* patterns.c
+ * test for case 7357: ensure code origins patterns disallow other code on 
+ * page and self-modifying and cross-modified patterns 
+ * - case 4020 scenarios:
+       * other code on the page
+       * same thread changes pattern
+         this was incorrectly allowed before 4020 fix
+       * another thread changes pattern, also allowed before 4020 fix
+       * pattern code itself changes pattern, also allowed before 4020 and
+         only fixed by 4020 due to luck
+ * - -trampoline_displaced_code 
+ * - -trampoline_dirjmp
+ */
+
+/* some code in the data section */
+
+/* case 6615: -trampoline_displaced_code:
+00364f8d 8bff             mov     edi,edi
+00364f8f 55               push    ebp
+00364f90 8bec             mov     ebp,esp
+00364f92 e9c5835b7c jmp ntdll!LdrQueryImageFileExecutionOptions+0x5 (7c91d35c)
+*/
+unsigned char datacode[] = 
+"\x8b\xff"                   /* mov     edi,edi */
+"\x55"                       /* push    ebp */
+"\x8b\xec"                   /* mov     ebp,esp */
+"\xe9\x00\x00\x00\x00"       /* jmp     image_target+5 */
+/* other code on the page */
+"\x90"                       /* nop (avoid -trampoline_dirjmp) */
+"\xe9\x00\x00\x00\x00"       /* jmp     maliciousness */
+;
+
+/* pc after the 1st jmp */
+#define DATACODE_POST_JMP ((ptr_int_t)datacode + sizeof(datacode) - 1 - 6)
+/* array index of 1st jmp's operand */
+#define DATACODE_JMP_OPND_IDX (sizeof(datacode)-1 - 5 - 1 - 4)
+/* pc after the 2nd jmp */
+#define DATACODE_POST_2ND_JMP ((ptr_int_t)datacode + sizeof(datacode) - 1)
+/* array index of 2nd jmp's operand */
+#define DATACODE_2ND_JMP_OPND_IDX (sizeof(datacode)-1 - 4)
+
+/* another match for -trampoline_displaced_code but that is capable
+ * of modifying itself.
+ * example of mov immed to abs addr:
+      0x24722535   c7 05 78 41 6d 24 b4 mov    $0x151b88b4 -> 0x246d4178 
+                   88 1b 15
+*/
+unsigned char datacode2[] = 
+"\xc7\x05\x00\x00\x00\x00\x00\x00\x00\x00" /* mov $immed, abs-addr */
+"\xe9\x00\x00\x00\x00"       /* jmp     image_target2+10 */
+;
+
+static __declspec( naked ) void
+image_target()
+{
+    __asm {
+            jmp  offset datacode
+            pop  ebp
+            ret
+    }
+}
+
+static __declspec( naked ) void
+image_target2()
+{
+    __asm {
+            jmp  offset datacode2
+            mov edi,edi
+            mov edi,edi
+            nop
+            ret
+    }
+}
+
+void
+maliciousness()
+{
+    static int instances = 0;
+    print("malicious code executing #%d!\n", ++instances);
+}
+
+int WINAPI
+run_func(void * arg)
+{
+    int offs;
+    /* have 2nd instr make direct jmp to maliciousness */
+    offs = (ptr_int_t)&maliciousness - ((ptr_int_t)datacode + 2/*first instr*/ + 5/*this new jmp*/);
+    datacode[2] = 0xe9;
+    *((int *)(&datacode[3])) = offs;
+    __asm {
+            pusha
+            call offset datacode
+            popa
+    }
+}
+
+int main(int argc, char *argv[])
+{
+    int offs, tid;
+    unsigned long hThread;
+
+    INIT();
+
+    print("testing hook pattern\n");
+    /* make it executable so that natively it works on NX */
+    protect_mem(datacode, sizeof(datacode), ALLOW_READ|ALLOW_WRITE|ALLOW_EXEC);
+    protect_mem(datacode2, sizeof(datacode), ALLOW_READ|ALLOW_WRITE|ALLOW_EXEC);
+    /* ensure our same-page test is relevant */
+    assert((((ptr_int_t)datacode) & ~(PAGE_SIZE -1)) ==
+           (((ptr_int_t)&datacode[sizeof(datacode)-1]) & ~(PAGE_SIZE -1)));
+
+    /****************************************************************************/
+    /* datacode */
+
+    /* we need to set the 1st jmp so we'll match the pattern */
+    offs = ((ptr_int_t)&image_target + 5/*skip jmp*/) - DATACODE_POST_JMP;
+    /* make direct jmp go to image_target */
+    *((int *)(&datacode[DATACODE_JMP_OPND_IDX])) = offs;
+    __asm {
+            pusha
+            call offset datacode
+            popa
+    }
+
+    print("testing non-pattern-match on same page\n");
+    offs = (ptr_int_t)&maliciousness - DATACODE_POST_2ND_JMP;
+    /* make 2nd direct jmp go to maliciousness */
+    *((int *)(&datacode[DATACODE_2ND_JMP_OPND_IDX])) = offs;
+    offs = DATACODE_POST_JMP;
+    __asm {
+            pusha
+            call dword ptr offs
+            popa
+    }
+
+    print("testing non-pattern-match in same region\n");
+     /* have 2nd instr make direct jmp to maliciousness */
+    offs = (ptr_int_t)&maliciousness - ((ptr_int_t)datacode + 2/*first instr*/ + 5/*this new jmp*/);
+    datacode[2] = 0xe9;
+    *((int *)(&datacode[3])) = offs;
+    __asm {
+            pusha
+            call offset datacode
+            popa
+    }
+
+    /* put the code back */
+    datacode[2] = 0x55;
+    datacode[3] = 0x8b;
+    datacode[4] = 0xec;
+    datacode[5] = 0xe9;
+    offs = ((ptr_int_t)&image_target + 5/*skip jmp*/) - DATACODE_POST_JMP;
+    print("testing hook pattern again\n");
+    /* make direct jmp go to image_target */
+    *((int *)(&datacode[DATACODE_JMP_OPND_IDX])) = offs;
+    __asm {
+            pusha
+            call offset datacode
+            popa
+    }
+
+    print("testing non-pattern-match in same region by another thread\n");
+    /* now have another thread do the same thing */
+    hThread = _beginthreadex(NULL, 0, run_func, NULL, 0, &tid);
+    WaitForSingleObject((HANDLE)hThread, INFINITE);
+
+    print("testing different pattern match in same region\n");
+    /* now change to have 1st instr make direct jmp to maliciousness */
+    offs = (ptr_int_t)&maliciousness - ((ptr_int_t)datacode + 5);
+    datacode[0] = 0xe9;
+    *((int *)(&datacode[1])) = offs;
+    __asm {
+            pusha
+            call offset datacode
+            popa
+    }
+
+    /****************************************************************************/
+    /* datacode2 */
+
+    /* for -detect_mode we may have added datacode2 -- so force removal of it */
+    protect_mem(datacode2, sizeof(datacode), ALLOW_READ|ALLOW_WRITE);
+    /* but we have to make sure it works on nx */
+    protect_mem(datacode2, sizeof(datacode), ALLOW_READ|ALLOW_WRITE|ALLOW_EXEC);
+
+    print("testing pattern match that modifies itself to be a non-match\n");
+    /* would be allowed w/ last_area 4020 impl but shared->private check deletes
+     * shared area and we end up getting lucky.
+     */
+    /* MUST be just after change 1st instr of datacode to jmp to maliciousness */
+    offs = ((ptr_int_t)&image_target2 + 10/*skip length of pre-jmp instrs*/) -
+        ((ptr_int_t)datacode2 + sizeof(datacode2) - 1);
+    /* make direct jmp go to image_target */
+    *((int *)(&datacode2[sizeof(datacode2)-5])) = offs;
+    /* make the mov modify the jmp to go to the jmp at the start of
+     * datacode (I would put another jmp at end of datacode2 but we'll
+     * just elide and allow!) which will go to maliciousness
+     */
+    offs = (ptr_int_t)(&datacode) -
+        ((ptr_int_t)datacode2 + sizeof(datacode2) - 1);
+    /* immed comes last */
+    *((int *)(&datacode2[6])) = offs;
+    /* target of mov comes before immed */
+    *((int *)(&datacode2[2])) = (ptr_int_t) &datacode2[11];
+    __asm {
+            pusha
+            call offset datacode2
+            popa
+    }
+
+    print("finished\n");
+    return 0;
+}
diff --git a/suite/tests/security-win32/patterns.template b/suite/tests/security-win32/patterns.template
new file mode 100755
index 000000000..318ba234b
--- /dev/null
+++ b/suite/tests/security-win32/patterns.template
@@ -0,0 +1,83 @@
+testing hook pattern
+#if +PROGRAM_SHEPHERDING : +-security +-no_trampoline_displaced_code --X --executable_if_x --B --executable_heap --low --client --thin_client || +PROGRAM_SHEPHERDING : +-security +-no_executable_if_trampoline --X --executable_if_x --B --executable_heap
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask 
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+testing non-pattern-match on same page
+#if +PROGRAM_SHEPHERDING : +-security --X --executable_if_x --B --executable_heap --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask 
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+malicious code executing #1!
+testing non-pattern-match in same region
+#if +PROGRAM_SHEPHERDING : +-security --X --executable_if_x --B --executable_heap --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask 
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+malicious code executing #2!
+testing hook pattern again
+#if +PROGRAM_SHEPHERDING : +-security +-no_trampoline_displaced_code --X --executable_if_x --B --executable_heap --low --client --thin_client || +PROGRAM_SHEPHERDING : +-security +-no_executable_if_trampoline --X --executable_if_x --B --executable_heap
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask 
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+testing non-pattern-match in same region by another thread
+#if +PROGRAM_SHEPHERDING : +-security --X --executable_if_x --B --executable_heap --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask 
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+malicious code executing #3!
+testing different pattern match in same region
+#if +PROGRAM_SHEPHERDING : +-security +-no_trampoline_dirjmp --X --executable_if_x --B --executable_heap --low --client --thin_client || +PROGRAM_SHEPHERDING : +-security +-no_executable_if_trampoline --X --executable_if_x --B --executable_heap
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask 
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+malicious code executing #4!
+testing pattern match that modifies itself to be a non-match
+#if +PROGRAM_SHEPHERDING : +-security --X --executable_if_x --B --executable_heap --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask 
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+malicious code executing #5!
+finished
diff --git a/suite/tests/security-win32/ret-SEH.c b/suite/tests/security-win32/ret-SEH.c
new file mode 100644
index 000000000..75389427a
--- /dev/null
+++ b/suite/tests/security-win32/ret-SEH.c
@@ -0,0 +1,148 @@
+/* **********************************************************
+ * Copyright (c) 2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include "tools.h"
+
+#ifdef USE_DYNAMO
+# include "dynamo.h"
+#endif
+
+/* the VC exception handling data structures that the core expects */
+typedef struct scopetable_entry {
+    DWORD previousTryLevel;
+    PVOID lpfnFilter;
+    PVOID lpfnHandler;
+} scopetable_entry;
+ 
+/* regular exception record, referred to in WINNT.H as
+ * EXCEPTION_REGISTRATION_RECORD 
+ */
+typedef struct _EXCEPTION_REGISTRATION {
+     struct _EXCEPTION_REGISTRATION* prev;
+     PVOID                   handler;
+} EXCEPTION_REGISTRATION, *PEXCEPTION_REGISTRATION_RECORD;
+
+/* The extended exception frame used by Visual C++ */
+typedef struct _VC_EXCEPTION_REGISTRATION 
+{
+    EXCEPTION_REGISTRATION   exception_base;
+    struct scopetable_entry *scopetable;
+    int                      trylevel;
+    int                      _ebp;
+} VC_EXCEPTION_REGISTRATION;
+
+static scopetable_entry scopes[] = {
+    { -1, NULL, NULL },
+    {  0, NULL, NULL },
+    {  1, NULL, NULL },
+};
+#define NUM_SCOPE_ENTRIES (sizeof(scopes)/sizeof(scopetable_entry))
+
+static VC_EXCEPTION_REGISTRATION vcex = {
+    {NULL, NULL}, &scopes, 0, 0
+};
+
+void
+foo(int level)
+{
+    print("in foo level %d\n", level);
+}
+
+void
+ret_SEH(int level)
+{
+    int i;
+    PVOID storeme = &scopes[level+1].lpfnHandler;
+    print("ret-SEH test: trylevel %d\n", level);
+    vcex.trylevel = level;
+    /* ensure DR checks the proper trylevel by clearing the rest */
+    for (i=0; i<NUM_SCOPE_ENTRIES; i++) {
+        scopes[level+1].lpfnHandler = NULL;
+    }
+    __asm {
+        /* cannot refer to asm symbol from C except in a goto, so
+         * we do scopes[level+1].lpfnHandler = myhandler from asm
+         */
+        mov eax, storeme
+        mov dword ptr [eax], offset myhandler
+        /* now that SEH scopetable is set up, we do our funny call
+         * with a push immed of the retaddr immediately prior to
+         * our SEH handler, to match the pattern seen on NT4
+         */
+        /* arg to foo */
+        push level
+        /* retaddr */
+        push offset myretpt
+    myhandler:
+        /* some amount of code in here */
+        nop
+        jmp foo
+        /* make ret point not be the next instr */
+        nop
+    myretpt:
+        /* clean up arg to foo */
+        pop eax
+    }
+}
+
+int
+main(int argc, char *argv[])
+{
+    int i;
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+    INIT();
+
+    /* set up SEH */
+    __asm {
+        push eax
+        mov eax, offset vcex
+        mov dword ptr fs:[0], eax
+        pop eax
+    }
+
+    print("ret-SEH test starting\n");
+    for (i=0; i<NUM_SCOPE_ENTRIES; i++) {
+        /* levels start at -1 */
+        ret_SEH(i-1);
+    }
+    print("ret-SEH test stopping\n");
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
diff --git a/suite/tests/security-win32/ret-SEH.template b/suite/tests/security-win32/ret-SEH.template
new file mode 100644
index 000000000..e5a56717e
--- /dev/null
+++ b/suite/tests/security-win32/ret-SEH.template
@@ -0,0 +1,39 @@
+ret-SEH test starting
+ret-SEH test: trylevel -1
+in foo level -1
+#if +PROGRAM_SHEPHERDING : +-security +-no_seh_rct --C
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask 
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+ret-SEH test: trylevel 0
+in foo level 0
+// if internal_detach_mask we won't do any more DR stuff
+#if +PROGRAM_SHEPHERDING : +-security +-no_seh_rct --C --internal_detach_mask
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask 
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+ret-SEH test: trylevel 1
+in foo level 1
+#if +PROGRAM_SHEPHERDING : +-security +-no_seh_rct --C --internal_detach_mask
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask 
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+ret-SEH test stopping
diff --git a/suite/tests/security-win32/sd_tester.c b/suite/tests/security-win32/sd_tester.c
new file mode 100644
index 000000000..b8834c995
--- /dev/null
+++ b/suite/tests/security-win32/sd_tester.c
@@ -0,0 +1,133 @@
+/* **********************************************************
+ * Copyright (c) 2004 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ *
+ *  Copyright (c) 2004 Determina, Inc
+ *  All rights reserved
+ *
+ *  based on sd_tester.c for security evaluation
+ *
+ *  Copyright (c) 2002-2003 By Next Generation Security S.L.
+ *  All rights  reserved
+ *
+ */
+
+
+#include "tools.h"
+
+typedef unsigned char * (*func_TYPE)();
+
+#define NOP 0x90
+#define NUM_NOPS 1000
+
+char data_section[4096]; // For .data tests
+
+unsigned char shellcode[] = "\xc3";
+
+void usage(char* prog) {
+    print("%s [stack | heap | newheap | crtheap | virtual | virtual_x | .data]\n", prog);
+}
+
+int buffer_test(char* location) {
+    char stack_buffer[4096];
+    char *ptr=NULL;
+    func_TYPE func;
+
+    if (strcmp(location,"stack")==0) {
+        ptr=stack_buffer;
+    }
+
+    if (strcmp(location,"heap")==0) {
+        ptr=HeapAlloc(GetProcessHeap(),HEAP_ZERO_MEMORY,4000);
+    }
+
+    if (strcmp(location,"newheap")==0) {
+        ptr=HeapAlloc(HeapCreate(0,8000,16000),HEAP_ZERO_MEMORY,4000);
+    }
+
+    if (strcmp(location,"virtual")==0) {
+        ptr=VirtualAlloc(NULL,4096,MEM_COMMIT|MEM_RESERVE,PAGE_READWRITE);
+    }
+
+    if (strcmp(location,"virtual_x")==0) {
+        ptr=VirtualAlloc(NULL,4096,MEM_COMMIT|MEM_RESERVE,PAGE_EXECUTE_READWRITE);
+    }
+
+    if (strcmp(location,"crtheap")==0) {
+        ptr=malloc(4096);
+    }
+
+    if (strcmp(location,".data")==0) {
+        ptr=data_section;
+    }
+
+    if (ptr==NULL) {
+        usage("use");
+        return(-1);
+    }
+
+    memset(ptr,NOP,NUM_NOPS);
+    memcpy(ptr+NUM_NOPS,shellcode,sizeof(shellcode));
+
+    func=(func_TYPE)ptr;
+
+    VERBOSE_PRINT("ptr: %#x\n",ptr);
+    print("Executing %s shellcode...\n", location);
+    (*func)();
+    print("success!\n");
+
+    return(0);
+}
+
+int main(int argc, char **argv) 
+{
+    INIT();
+
+    if (argc==2 && strcmp(argv[1],"help")==0) {
+        usage(argv[0]);
+        return(-1);
+    }
+
+    if (argc==1) {
+        print("full suite run\n");
+        
+        buffer_test("stack");
+        buffer_test("heap");
+        buffer_test("newheap");
+        buffer_test("crtheap");
+        buffer_test("virtual");
+        buffer_test("virtual_x");
+        buffer_test(".data");
+        /* TODO:  .text, TEB */
+    }
+}
diff --git a/suite/tests/security-win32/sd_tester.template b/suite/tests/security-win32/sd_tester.template
new file mode 100644
index 000000000..72c5fc43f
--- /dev/null
+++ b/suite/tests/security-win32/sd_tester.template
@@ -0,0 +1,65 @@
+full suite run
+Executing stack shellcode...
+#if +PROGRAM_SHEPHERDING : +-security --executable_stack --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+success!
+Executing heap shellcode...
+#if +PROGRAM_SHEPHERDING : +-security --executable_heap --internal_detach_mask --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  STOP
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+success!
+Executing newheap shellcode...
+#if +PROGRAM_SHEPHERDING : +-security --executable_heap --internal_detach_mask --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  STOP
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+success!
+Executing crtheap shellcode...
+#if +PROGRAM_SHEPHERDING : +-security --executable_heap --internal_detach_mask --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  STOP
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+success!
+Executing virtual shellcode...
+#if +PROGRAM_SHEPHERDING : +-security --executable_heap --internal_detach_mask --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  STOP
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+success!
+Executing virtual_x shellcode...
+success!
+Executing .data shellcode...
+#if +PROGRAM_SHEPHERDING : +-security --executable_heap --internal_detach_mask --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  STOP
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+success!
diff --git a/suite/tests/security-win32/sec-adata.c b/suite/tests/security-win32/sec-adata.c
new file mode 100644
index 000000000..0a20f3ab2
--- /dev/null
+++ b/suite/tests/security-win32/sec-adata.c
@@ -0,0 +1,95 @@
+/* **********************************************************
+ * Copyright (c) 2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include "tools.h"
+
+#define VERBOSE 0
+
+/* an rwx code section that will have rct violations from emitted code */
+#pragma comment(linker, "/SECTION:.adata,ERW")
+#pragma code_seg(".adata") 
+int
+expendable_function()
+{
+    return 1;
+}
+#pragma code_seg() 
+
+/* a normal code section that will have rct violations for control */
+#pragma comment(linker, "/SECTION:.acode,ER")
+#pragma code_seg(".acode") 
+int
+expendable_function2()
+{
+    return 1;
+}
+#pragma code_seg() 
+
+static char *dll_name = "sec-xdata.dll.dll";
+
+byte *
+get_buf_target(byte *func)
+{
+    byte *buf = func;
+    /* handle a jmp link */
+    if (*func == 0xe9)
+        buf = func+(*(uint *)(func+1))+5;
+    return buf+0x10; /* arbitrary, we need to get away from the reloc */
+}
+
+int main()
+{
+    HMODULE lib;
+    byte *code;
+    byte *adata_buf = (byte *)expendable_function;
+    byte *acode_buf = (byte *)expendable_function2;
+    int old_prot;
+    INIT();
+    adata_buf = get_buf_target(adata_buf);
+    acode_buf = get_buf_target(acode_buf);
+
+    print("starting good adata test\n");
+    code = copy_to_buf(adata_buf, PAGE_SIZE /* section will be at least page size */,
+                       NULL, CODE_INC, COPY_NORMAL); 
+    test_print(code, 0);
+
+    print("starting bad acode test\n");
+    VirtualProtect(acode_buf, 1024, PAGE_EXECUTE_READWRITE, &old_prot);
+    code = copy_to_buf(acode_buf, 1024 /* section will be at least page size */,
+                       NULL, CODE_INC, COPY_NORMAL);
+    VirtualProtect(acode_buf, 1024, old_prot, &old_prot);
+    test_print(code, 0);
+
+    print("done\n");
+    return 0;
+}
diff --git a/suite/tests/security-win32/sec-adata.template b/suite/tests/security-win32/sec-adata.template
new file mode 100755
index 000000000..4c9baa755
--- /dev/null
+++ b/suite/tests/security-win32/sec-adata.template
@@ -0,0 +1,15 @@
+starting good adata test
+1
+starting bad acode test
+# if +PROGRAM_SHEPHERDING : +-security --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask 
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+1
+done
diff --git a/suite/tests/security-win32/sec-fixed.c b/suite/tests/security-win32/sec-fixed.c
new file mode 100644
index 000000000..e405180a4
--- /dev/null
+++ b/suite/tests/security-win32/sec-fixed.c
@@ -0,0 +1,92 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include <stdio.h>
+#include "tools.h"
+
+#define VERBOSE 0
+
+HMODULE
+myload(char* lib) 
+{
+    HMODULE hm = LoadLibrary(lib);
+    if (hm == NULL) {
+        print("error loading library %s\n", lib);
+    } else {
+        BOOL res;
+        BOOL (WINAPI *proc)(DWORD);
+        print("loaded %s\n", lib);
+#if VERBOSE
+        print("library is at "PFX"\n", hm);
+#endif
+    }
+    return hm;
+}
+
+int main()
+{
+    HMODULE lib;
+    HMODULE cmd;
+    int res;
+    lib = myload("sec-fixed.dll.dll");
+    FreeLibrary(lib);
+
+    /* unclear what this code is supposed to do on other platforms
+     * FIXME: move to its own load-exe
+     */
+    /* real use seen of PCHealth\HelpCtr\Binaries\HelpCtr.exe */
+
+    cmd = LoadLibraryExW(L"cmd.exe", NULL, LOAD_LIBRARY_AS_DATAFILE);
+    assert(cmd != NULL);
+    res = FreeLibrary(cmd);
+    assert(res);
+    print("cmd.exe as data\n");
+
+    /* FIXME: for some reason the loader reuses the exe - if we ask for cmd.exe again here  */
+    cmd = LoadLibraryExW(L"calc.exe", NULL, DONT_RESOLVE_DLL_REFERENCES | LOAD_LIBRARY_AS_DATAFILE);
+    assert(cmd != NULL);
+    res = FreeLibrary(cmd);
+    assert(res);
+    print("calc.exe as data and no resolve\n");
+
+    /* note that windbg will show only this one as a module */
+    cmd = LoadLibraryExW(L"rundll32.exe", NULL, DONT_RESOLVE_DLL_REFERENCES);
+    assert(cmd != NULL);
+    res = FreeLibrary(cmd);
+    assert(res);
+    print("rundl32.exe as no resolve\n");
+
+    print("done\n");
+
+    return 0;
+}
diff --git a/suite/tests/security-win32/sec-fixed.dll.c b/suite/tests/security-win32/sec-fixed.dll.c
new file mode 100644
index 000000000..6c9749d1a
--- /dev/null
+++ b/suite/tests/security-win32/sec-fixed.dll.c
@@ -0,0 +1,61 @@
+/* **********************************************************
+ * Copyright (c) 2006 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* case 2298 this DLL is fixed and shouldn't be ASLRed
+ * cf. secalign-fixed.dll.c which is generated in a different way
+ */
+
+#include "tools.h"
+#include <windows.h>
+
+/* we don't need an explicit jmp* or call* since CRT ends up using a call* */
+
+/* unlike secalign-fixed.dll.c we don't use /fixed as a pragma comment */
+
+/* our Makefile expects a .lib */
+int
+__declspec(dllexport)
+make_a_lib(int arg)
+{
+    return 1;
+}
+
+BOOL APIENTRY 
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    switch (reason_for_call) {
+    case DLL_PROCESS_ATTACH:
+        print("in fixed dll\n");
+        break;
+    }
+    return TRUE;
+}
diff --git a/suite/tests/security-win32/sec-fixed.expect b/suite/tests/security-win32/sec-fixed.expect
new file mode 100644
index 000000000..d24dff29f
--- /dev/null
+++ b/suite/tests/security-win32/sec-fixed.expect
@@ -0,0 +1,6 @@
+in fixed dll
+loaded sec-fixed.dll.dll
+cmd.exe as data
+calc.exe as data and no resolve
+rundl32.exe as no resolve
+done
diff --git a/suite/tests/security-win32/sec-xdata.c b/suite/tests/security-win32/sec-xdata.c
new file mode 100644
index 000000000..a09a136bd
--- /dev/null
+++ b/suite/tests/security-win32/sec-xdata.c
@@ -0,0 +1,67 @@
+/* **********************************************************
+ * Copyright (c) 2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include "tools.h"
+
+#define VERBOSE 0
+
+/* a bad xdata section (rw instead of rwx) */
+#pragma comment(linker, "/SECTION:.xdata,RW")
+#pragma data_seg(".xdata") 
+/* NOTE - compilers are stupid, without the "= {0}" this will be put in the regular
+ * .data section and now .xdata section will be made. wtf??? */ 
+char bad_xdata_buf[1024] = {0};
+#pragma data_seg() 
+
+static char *dll_name = "sec-xdata.dll.dll";
+
+int main()
+{
+    HMODULE lib;
+    char *code;
+    INIT();
+
+    print("starting good xdata test\n");
+    lib = LoadLibrary(dll_name);
+    if (lib == NULL) {
+        print("error loading library %s\n", dll_name);
+    }
+    FreeLibrary(lib);
+
+    print("starting bad xdata test\n");
+    code = copy_to_buf(bad_xdata_buf, sizeof(bad_xdata_buf),
+                       NULL, CODE_INC, COPY_NORMAL); 
+    test_print(code, 0);
+    print("done\n");
+    return 0;
+}
diff --git a/suite/tests/security-win32/sec-xdata.dll.c b/suite/tests/security-win32/sec-xdata.dll.c
new file mode 100644
index 000000000..69958d8f5
--- /dev/null
+++ b/suite/tests/security-win32/sec-xdata.dll.c
@@ -0,0 +1,75 @@
+/* **********************************************************
+ * Copyright (c) 2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include "tools.h"
+
+#define VERBOSE 0
+
+#if 0
+typedef struct _code_buf {
+    char code_buffer[1024];
+} code_buf;
+code_buf good_code_buffer = {0};
+#endif
+
+/* a good xdata section */
+#pragma comment(linker, "/SECTION:.xdata,ERW")
+#pragma data_seg(".xdata")
+/* NOTE - compilers are stupid, without the "= {0}" this will be put in the regular
+ * .data section and now .xdata section will be made. wtf??? */ 
+char good_xdata_buf[1024] = {0};
+#pragma data_seg() 
+
+
+/* our Makefile expects a .lib */
+int
+__declspec(dllexport)
+make_a_lib(int arg)
+{
+    return 1;
+}
+
+
+BOOL APIENTRY 
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    char *code;
+    switch (reason_for_call) {
+    case DLL_PROCESS_ATTACH:
+        code = copy_to_buf(good_xdata_buf, sizeof(good_xdata_buf),
+                           NULL, CODE_INC, COPY_NORMAL); 
+        test_print(code, 0);
+        break;
+    }
+    return TRUE;
+}
diff --git a/suite/tests/security-win32/sec-xdata.template b/suite/tests/security-win32/sec-xdata.template
new file mode 100755
index 000000000..d5c03b90d
--- /dev/null
+++ b/suite/tests/security-win32/sec-xdata.template
@@ -0,0 +1,15 @@
+starting good xdata test
+1
+starting bad xdata test
+# if +PROGRAM_SHEPHERDING : +-security --low --client --thin_client
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask 
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+1
+done
diff --git a/suite/tests/security-win32/secalign-fixed.c b/suite/tests/security-win32/secalign-fixed.c
new file mode 100644
index 000000000..ae975803a
--- /dev/null
+++ b/suite/tests/security-win32/secalign-fixed.c
@@ -0,0 +1,63 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include <stdio.h>
+#include "tools.h"
+
+#define VERBOSE 0
+
+HMODULE
+myload(char* lib) 
+{
+    HMODULE hm = LoadLibrary(lib);
+    if (hm == NULL) {
+        print("error loading library %s\n", lib);
+    } else {
+        BOOL res;
+        BOOL (WINAPI *proc)(DWORD);
+        print("loaded %s\n", lib);
+#if VERBOSE
+        print("library is at "PFX"\n", hm);
+#endif
+    }
+    return hm;
+}
+
+int main()
+{
+    HMODULE lib;
+    lib = myload("secalign-fixed.dll.dll");
+    FreeLibrary(lib);
+
+    return 0;
+}
diff --git a/suite/tests/security-win32/secalign-fixed.dll.c b/suite/tests/security-win32/secalign-fixed.dll.c
new file mode 100644
index 000000000..a47bd15f8
--- /dev/null
+++ b/suite/tests/security-win32/secalign-fixed.dll.c
@@ -0,0 +1,74 @@
+/* **********************************************************
+ * Copyright (c) 2006 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* case 7572 testing non-committed pages in a dll
+ * this dll is /fixed, so even -rct_reloc will not find the references
+ */
+
+/*
+To build need to align to 0x2000 to get non-committed pages as padding
+
+cl /nologo /Zi /I. tools.c /Od /MTd win32/secalign.dll.c kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /link /subsystem:console /dll /out:win32/secalign.dll.dll /implib:win32/secalign.dll.lib /align:0x2000 /driver
+
+new link.exe has /section, accomplishes same thing, still need /driver:
+"/c/Program Files/Microsoft Visual Studio 8/VC/bin/link.exe" win32/secalign.dll.obj /subsystem:console /dll /out:win32/secalign.dll.dll /implib:win32/secalign.dll.lib /section:.text,,align=0x2000 /driver 
+*/
+
+#include "tools.h"
+#include <windows.h>
+
+/* we don't need an explicit jmp* or call* since CRT ends up using a call* */
+
+/* Linker requires /driver if specifying /align.
+ * Documentation says that "The linker will perform some special
+ * optimizations if this option is selected." -- not sure if any other changes.
+ */
+#pragma comment(linker, "/fixed /align:0x2000 /driver")
+
+/* our Makefile expects a .lib */
+int
+__declspec(dllexport)
+make_a_lib(int arg)
+{
+    return 1;
+}
+
+BOOL APIENTRY 
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    switch (reason_for_call) {
+    case DLL_PROCESS_ATTACH:
+        print("in fixed dll\n");
+        break;
+    }
+    return TRUE;
+}
diff --git a/suite/tests/security-win32/secalign-fixed.expect b/suite/tests/security-win32/secalign-fixed.expect
new file mode 100644
index 000000000..a05407c3b
--- /dev/null
+++ b/suite/tests/security-win32/secalign-fixed.expect
@@ -0,0 +1,2 @@
+in fixed dll
+loaded secalign-fixed.dll.dll
diff --git a/suite/tests/security-win32/selfmod-threads.c b/suite/tests/security-win32/selfmod-threads.c
new file mode 100644
index 000000000..f63996930
--- /dev/null
+++ b/suite/tests/security-win32/selfmod-threads.c
@@ -0,0 +1,144 @@
+/* **********************************************************
+ * Copyright (c) 2006 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* threaded version of selfmod.c to test
+ * cache consistency concurrency cases (say that 3 times fast)
+ */
+
+#include <windows.h>
+#include <process.h> /* for _beginthreadex */
+#include <stdio.h>
+#include "tools.h"
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#define ITERS 10
+#define NUM_THREADS 12
+int go_threads = 0;
+
+void
+foo(int iters)
+{
+    /* executes iters iters, by modifing the iters bound using
+     * self-modifying code
+     */
+    int total;
+    while (!go_threads)
+        Sleep(1);
+#ifdef LINUX
+    asm("  movl %0, %%ecx" : : "r" (iters));
+    asm("  call next_inst");
+    asm("next_inst:");
+    asm("  pop %edx");
+    /* add to retaddr: 1 == pop
+     *                 3 == mov ecx into target
+     *                 1 == opcode of target movl
+     */
+    asm("  movl %ecx, 0x5(%edx)"); /* the modifying store */
+    asm("  movl $0x12345678,%eax"); /* this instr's immed gets overwritten */
+    asm("  movl $0x0,%ecx"); /* counter for diagnostics */
+    asm("repeata:");
+    asm("  decl %eax");
+    asm("  inc  %ecx");
+    asm("  cmpl $0x0,%eax");
+    asm("  jnz repeata");
+    asm("  movl %%ecx, %0" : "=r" (total));
+#else
+    __asm {
+	mov  ecx, iters
+        call next_inst
+      next_inst:
+        pop  edx
+    /* add to retaddr: 1 == pop
+     *                 3 == mov ecx into target
+     *                 1 == opcode of target movl
+     */
+        mov  dword ptr [edx + 0x5], ecx /* the modifying store */
+        mov  eax,0x12345678 /* this instr's immed gets overwritten */
+        mov  ecx,0x0 /* counter for diagnostics */
+      repeata:
+        dec  eax
+        inc  ecx
+        cmp  eax,0x0
+        jnz  repeata
+        mov  total,ecx
+    }
+#endif
+#if VERBOSE
+    print("Executed 0x%x iters\n", total);
+#endif
+}
+
+int WINAPI
+run_func(void * arg)
+{
+    int i;
+    for (i=0; i<ITERS; i++) {
+        foo(0xabcd);
+        foo(0x1234);
+        foo(0xef01);
+    }
+}
+
+int
+main()
+{
+    int tid, i;
+    unsigned long hThread[NUM_THREADS];
+
+    INIT();
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    print("starting up\n");
+
+    /* make foo code writable */
+    protect_mem(foo, PAGE_SIZE, ALLOW_READ|ALLOW_WRITE|ALLOW_EXEC);
+    // Note that main and the exception handler __except_handler3 are on this page too
+
+    for (i=0; i<NUM_THREADS; i++)
+        hThread[i] = _beginthreadex(NULL, 0, run_func, NULL, 0, &tid);
+    go_threads = 1;
+    for (i=0; i<NUM_THREADS; i++)
+        WaitForSingleObject((HANDLE)hThread[i], INFINITE);
+
+    print("all done\n");
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
diff --git a/suite/tests/security-win32/selfmod-threads.template b/suite/tests/security-win32/selfmod-threads.template
new file mode 100644
index 000000000..519529ce1
--- /dev/null
+++ b/suite/tests/security-win32/selfmod-threads.template
@@ -0,0 +1,31 @@
+starting up
+#if +PROGRAM_SHEPHERDING +WINDOWS : +-security --executable_heap +-no_executable_if_hook --executable_if_text --executable_if_x --X || +PROGRAM_SHEPHERDING +LINUX : +-security --executable_heap --executable_if_x --X
+# if : --detect_mode
+#  if : +-throw_exception
+// The exception handler is bad, so we loop to -throw_exception_max_per_thread
+#   SEC_VIO_EXCEPTION
+#   if : --throw_exception_max 1
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#   endif
+#  endif
+#  if : +-kill_thread
+#    SEC_VIO_THREAD
+#  else  
+#    SEC_VIO_STOP
+#  endif
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+all done
diff --git a/suite/tests/threads.h b/suite/tests/threads.h
new file mode 100644
index 000000000..214d158cd
--- /dev/null
+++ b/suite/tests/threads.h
@@ -0,0 +1,173 @@
+/* **********************************************************
+ * Copyright (c) 2004-2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifndef THREADS_H
+#define THREADS_H
+
+/***************************************************************************/
+#ifdef LINUX
+
+#define WINAPI
+
+#include <sys/types.h> /* for wait and mmap */
+#include <sys/wait.h>  /* for wait */
+#include <sched.h>     /* for __clone */
+#include <sys/mman.h>  /* for mmap */
+#include <unistd.h>    /* for sleep */
+
+typedef pid_t thread_t;
+
+#define thread_sleep sleep
+
+#define THREAD_STACK_SIZE   (32*1024)
+
+/* allocate stack storage on the app's heap */
+void *
+stack_alloc(int size)
+{
+    void *q, *p;
+
+#if STACK_OVERFLOW_PROTECT
+    /* allocate an extra page and mark it non-accessible to trap stack overflow */
+    q = mmap(0, PAGE_SIZE, PROT_NONE, MAP_ANON|MAP_PRIVATE, -1, 0);
+    assert(q);
+    stack_redzone_start = (size_t) q;
+#endif 
+
+    p = mmap(q, size, PROT_READ|PROT_WRITE, MAP_ANON|MAP_PRIVATE, -1, 0);
+    assert(p);
+    /* stack grows from high to low addresses, so return a ptr to the top of the
+       allocated region */
+    return (void *) ((size_t)p + size);
+}
+
+/* free memory-mapped stack storage */
+void
+stack_free(void *p, int size)
+{
+    void *sp = (void *) ((size_t)p - size);
+    munmap(sp, size);
+#if STACK_OVERFLOW_PROTECT
+    sp = sp - PAGE_SIZE;
+    munmap(sp, PAGE_SIZE);
+#endif 
+}
+
+/* Create a new thread. It should be passed "run_func", a function which
+ * takes one argument ("arg"), for the thread to execute.
+ * If *stack == NULL a new stack is allocated.
+ * Returns the tid of the new thread.
+ */
+thread_t
+create_thread(int (*run_func)(void *), void *arg, void **stack)
+{
+    thread_t newpid; 
+    int flags;
+    void *my_stack;
+    
+    if (*stack == NULL)
+        my_stack = stack_alloc(THREAD_STACK_SIZE);
+    else
+        my_stack = *stack;
+    /* need SIGCHLD so parent will get that signal when child dies,
+     * else have errors doing a wait */
+    flags = SIGCHLD | CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND;
+    newpid = __clone(run_func, my_stack, flags, arg);
+  
+    if (newpid == -1) {
+	print("smp.c: Error calling __clone\n");
+	stack_free(my_stack, THREAD_STACK_SIZE);
+	return -1;
+    }
+
+    if (*stack == NULL)
+        *stack = my_stack;
+    assert(newpid > -1);
+    return newpid;
+}
+
+void 
+delete_thread(thread_t pid, void *stack)
+{
+    thread_t result;
+    /* do not print out pids to make diff easy */
+    VERBOSE_PRINT("Waiting for child to exit\n");
+    result = waitpid(pid, NULL, 0);
+    VERBOSE_PRINT("Child has exited\n");
+    if (result == -1 || result != pid)
+	perror("delete_thread waitpid");
+    stack_free(stack, THREAD_STACK_SIZE);
+}
+
+/***************************************************************************/
+#else /* WINDOWS */
+
+#include <windows.h>
+#include <process.h> /* for _beginthreadex */
+
+typedef HANDLE thread_t;
+
+/* make it easier to write portable code --
+ * we'll just ignore the stack stuff
+ */
+#define stack_alloc(s) NULL
+#define stack_free(p, s) NULL
+#define THREAD_STACK_SIZE   (32*1024)
+
+/* Create a new thread. It should be passed "run_func", a function which
+ * takes one argument ("arg"), for the thread to execute.
+ * Ignores the stack argument.
+ * Returns the tid of the new thread.
+ */
+thread_t
+create_thread(int (WINAPI *run_func)(void *), void *arg, void **stack)
+{
+    int tid;
+    return (thread_t) _beginthreadex(NULL, 0, run_func, NULL, 0, &tid);
+}
+
+void 
+delete_thread(thread_t thread, void *stack)
+{
+    VERBOSE_PRINT("Waiting for child to exit\n");
+    WaitForSingleObject(thread, INFINITE);
+    VERBOSE_PRINT("Child has exited\n");
+}
+
+void
+thread_sleep(int s)
+{
+    Sleep(1000*s);
+}
+
+#endif /* WINDOWS */
+#endif /* THREADS_H */
diff --git a/suite/tests/tools.c b/suite/tests/tools.c
new file mode 100644
index 000000000..eb56c3f52
--- /dev/null
+++ b/suite/tests/tools.c
@@ -0,0 +1,318 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifndef ASM_CODE_ONLY /* C code */
+#include "tools.h"
+
+#define ASSERT_NOT_IMPLEMENTED() do { print("NYI\n"); abort();} while (0)
+
+#ifdef WINDOWS
+/* returns 0 on failure */
+/* FIXME - share with core get_os_version() */
+int
+get_windows_version()
+{
+    OSVERSIONINFO version;
+    int res;
+    version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
+    res = GetVersionEx(&version);
+    assert(res != 0);
+    if (version.dwPlatformId == VER_PLATFORM_WIN32_NT) {
+        /* WinNT or descendents */
+        if (version.dwMajorVersion == 6 && version.dwMinorVersion == 0) {
+            return WINDOWS_VERSION_VISTA;
+        } else if (version.dwMajorVersion == 5 && version.dwMinorVersion == 2) {
+            return WINDOWS_VERSION_2003;
+        } else if (version.dwMajorVersion == 5 && version.dwMinorVersion == 1) {
+            return WINDOWS_VERSION_XP;
+        } else if (version.dwMajorVersion == 5 && version.dwMinorVersion == 0) {
+            return WINDOWS_VERSION_2000;
+        } else if (version.dwMajorVersion == 4) {
+            return WINDOWS_VERSION_NT;
+        }
+    }
+    return 0;
+}
+#endif
+
+/* Thread related functions */
+thread_handle
+create_thread(fptr f)
+{
+    thread_handle th;
+
+#ifdef LINUX
+    /* FIXME: use the one from linux/clone.c */
+    ASSERT_NOT_IMPLEMENTED();
+#else
+    thread_handle tid;
+    th = (thread_handle)_beginthreadex(NULL, 0, f, NULL, 0, &tid);
+#endif
+    return th;
+}
+
+void
+suspend_thread(thread_handle th)
+{
+#ifdef LINUX
+    ASSERT_NOT_IMPLEMENTED();
+#else
+    SuspendThread(th);
+#endif
+}
+
+void
+resume_thread(thread_handle th)
+{
+#ifdef LINUX
+    ASSERT_NOT_IMPLEMENTED();
+#else
+    ResumeThread(th);
+#endif
+}
+
+void
+join_thread(thread_handle th)
+{
+#ifdef LINUX
+    ASSERT_NOT_IMPLEMENTED();
+#else
+    WaitForSingleObject(th, INFINITE);
+#endif
+}
+
+void
+thread_yield()
+{
+#ifdef LINUX
+    ASSERT_NOT_IMPLEMENTED();
+#else
+    Sleep(0); /* stay ready */
+#endif
+}
+
+void *
+reserve_memory(int size)
+{
+#ifdef LINUX
+    void *p = mmap(0, size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0);
+    if (p == MAP_FAILED)
+        return NULL;
+    return p;
+#else
+    return VirtualAlloc(0, size, MEM_RESERVE, PAGE_EXECUTE_READWRITE);
+#endif
+}
+
+/* Intel processors:  ebx:edx:ecx spell GenuineIntel */
+#define INTEL_EBX /* Genu */ 0x756e6547
+#define INTEL_EDX /* ineI */ 0x49656e69
+#define INTEL_ECX /* ntel */ 0x6c65746e
+
+/* AMD processors:  ebx:edx:ecx spell AuthenticAMD */
+#define AMD_EBX /* Auth */ 0x68747541
+#define AMD_EDX /* enti */ 0x69746e65
+#define AMD_ECX /* cAMD */ 0x444d4163
+
+#define VENDOR_INTEL    0
+#define VENDOR_AMD      1
+#define VENDOR_UNKNOWN  2
+
+/* Pentium IV -- FIXME: need to distinguish extended family bits? */
+#define FAMILY_PENTIUM_IV  15
+/* Pentium Pro, Pentium II, Pentium III, Athlon */
+#define FAMILY_PENTIUM_III  6
+#define FAMILY_PENTIUM_II   6
+#define FAMILY_PENTIUM_PRO  6
+#define FAMILY_ATHLON       6
+/* Pentium (586) */
+#define FAMILY_PENTIUM      5
+
+/*
+ * On Pentium through Pentium III and Pentium 4, I-cache lines are 32 bytes.
+ * On Pentium IV they are 64 bytes.
+ */
+unsigned int
+get_cache_line_size()
+{
+    /* use cpuid to get cache line size */
+    unsigned int cache_line_size, vendor, family;
+    unsigned int res_eax, res_ebx = 0, res_ecx = 0, res_edx = 0;
+#ifdef WINDOWS
+    int cpuid_res_local[4]; /* eax, ebx, ecx, and edx registers (in that order) */
+#endif
+    /* first verify on Intel processor */
+#ifdef LINUX
+# ifdef X64
+    assert(false); /* no pusha! */
+# else
+    /* GOT pointer is kept in ebx, go ahead and save all the registers */
+    asm("pusha");
+    asm("movl $0, %eax");
+    asm("cpuid");
+    asm("movl %%eax, %0" : "=m"(res_eax));
+    asm("movl %%ebx, %0" : "=m"(res_ebx));
+    asm("movl %%ecx, %0" : "=m"(res_ecx));
+    asm("movl %%edx, %0" : "=m"(res_edx));
+    asm("popa");
+# endif
+#else
+    __cpuid(cpuid_res_local, 0);
+    res_eax = cpuid_res_local[0];
+    res_ebx = cpuid_res_local[1];
+    res_ecx = cpuid_res_local[2];
+    res_edx = cpuid_res_local[3];
+#endif
+
+    if (res_ebx == INTEL_EBX) {
+        vendor = VENDOR_INTEL;
+        assert(res_edx == INTEL_EDX && res_ecx == INTEL_ECX);
+    } else if (res_ebx == AMD_EBX) {
+        vendor = VENDOR_AMD;
+        assert(res_edx == AMD_EDX && res_ecx == AMD_ECX);
+    } else {
+        vendor = VENDOR_UNKNOWN;
+        print("get_cache_line_size: unknown processor type");
+    }
+
+    /* now get processor info */
+#ifdef LINUX
+    /* GOT pointer is kept in ebx, go ahead and save all the registers */
+# ifdef X64
+    assert(false); /* no pusha! */
+# else
+    asm("pusha");
+    asm("movl $1, %eax");
+    asm("cpuid");
+    asm("movl %%eax, %0" : "=m"(res_eax));
+    asm("movl %%ebx, %0" : "=m"(res_ebx));
+    asm("movl %%ecx, %0" : "=m"(res_ecx));
+    asm("movl %%edx, %0" : "=m"(res_edx));
+    asm("popa");
+# endif
+#else
+    __cpuid(cpuid_res_local, 1);
+    res_eax = cpuid_res_local[0];
+    res_ebx = cpuid_res_local[1];
+    res_ecx = cpuid_res_local[2];
+    res_edx = cpuid_res_local[3];
+#endif
+    /* eax contains basic info:
+     *   extended family, extended model, type, family, model, stepping id
+     *   20:27,           16:19,          12:13, 8:11,  4:7,   0:3
+     */
+    /* family bits: 8 through 11 */
+    family = (res_eax & 0x00000f00) >> 8;
+    if (family == FAMILY_PENTIUM_IV) {
+        /* Pentium IV */
+        /* FIXME: need to read extended family bits? */
+        cache_line_size = (res_ebx & 0x0000ff00) >> 5; /* (x >> 8) * 8 == x >> 5 */
+    } else if (vendor == VENDOR_INTEL &&
+               (family == FAMILY_PENTIUM_III || family == FAMILY_PENTIUM_II)) {
+        /* Pentium III, Pentium II */
+        cache_line_size = 32;
+    } else if (vendor == VENDOR_AMD && family == FAMILY_ATHLON) {
+        /* Athlon */
+        cache_line_size = 64;
+    } else {
+        print("get_cache_line_size: unsupported processor family %d\n",
+              family);
+        cache_line_size = 32;
+    }
+    /* people who use this in ALIGN* macros are assuming it's a power of 2 */
+    assert((cache_line_size & (cache_line_size - 1)) == 0);
+    return cache_line_size;
+}
+
+
+#else /* asm code *************************************************************/
+/*
+ * Assembly routines shared among multiple tests
+ */
+
+#include "asm_defines.asm"
+
+START_FILE
+
+/* int code_self_mod(int iters)
+ * executes iters iters, by modifing the iters bound using self-modifying code.
+ */
+        DECLARE_FUNC(code_self_mod)
+GLOBAL_LABEL(code_self_mod:)
+        mov  REG_XCX, ARG1
+        call next_instr
+      next_instr:
+        pop  REG_XDX
+    /* add to retaddr: 1 == pop
+     *                 3 == mov ecx into target
+     *                 1 == opcode of target movl
+     */
+        mov  DWORD [REG_XDX + 5], ecx /* the modifying store */
+        mov  eax,HEX(12345678) /* this instr's immed gets overwritten */
+        mov  ecx,0 /* counter for diagnostics */
+      repeat1:
+        dec  eax
+        inc  ecx
+        cmp  eax,0
+        jnz  repeat1
+        mov  eax,ecx
+        ret
+        END_FUNC(code_self_mod)
+
+#undef FUNCNAME
+#define FUNCNAME code_inc
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        mov  REG_XAX, ARG1
+        inc  REG_XAX
+        ret
+        END_FUNC(FUNCNAME)
+
+#undef FUNCNAME
+#define FUNCNAME code_dec
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        mov  REG_XAX, ARG1
+        dec  REG_XAX
+        ret
+        END_FUNC(FUNCNAME)
+
+#undef FUNCNAME
+#define FUNCNAME dummy
+        DECLARE_FUNC(FUNCNAME)
+GLOBAL_LABEL(FUNCNAME:)
+        mov  REG_XAX, 1
+        ret
+        END_FUNC(FUNCNAME)
+
+END_FILE
+#endif
diff --git a/suite/tests/tools.h b/suite/tests/tools.h
new file mode 100644
index 000000000..ead3287b8
--- /dev/null
+++ b/suite/tests/tools.h
@@ -0,0 +1,620 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#ifndef TOOLS_H
+#define TOOLS_H
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <string.h> /* memcpy */
+#include <assert.h>
+
+#ifdef LINUX
+# include <sys/mman.h>
+# include <stdlib.h> /* abort */
+# include <errno.h>
+#else
+# include <windows.h>
+# define NTSTATUS DWORD
+# define NT_SUCCESS(status) (status >= 0)
+#endif
+
+#ifdef USE_DYNAMO
+# include "dr_api.h"
+#else
+typedef unsigned int bool;
+# ifdef LINUX
+typedef unsigned int uint;
+typedef unsigned long long int uint64;
+typedef long long int int64;
+# else
+typedef unsigned __int64 uint64;
+typedef __int64 int64;
+#  define uint DWORD
+# endif
+# define PAGE_SIZE 0x00001000
+#endif
+
+#ifdef WINDOWS
+# define IF_WINDOWS(x) x
+# define IF_WINDOWS_ELSE(x,y) (x)
+# define INT64_FORMAT "I64"
+#else
+# define IF_WINDOWS(x)
+# define IF_WINDOWS_ELSE(x,y) (y)
+# define INT64_FORMAT "ll"
+#endif
+
+/* some tests include dr_api.h and tools.h, so avoid duplicating */
+#ifndef IF_X64
+#  ifdef X64
+   typedef uint64 ptr_uint_t;
+   typedef int64 ptr_int_t;
+#   define PFMT "%016"INT64_FORMAT"x"
+#   define SZFMT "%"INT64_FORMAT"d"
+#   define IF_X64(x) x
+#   define IF_X64_ELSE(x, y) x
+#  else
+   typedef uint ptr_uint_t;
+   typedef int ptr_int_t;
+#   define PFMT "%08x"
+#   define SZFMT "%d"
+#   define IF_X64(x)
+#   define IF_X64_ELSE(x, y) y
+#  endif
+#  define PFX "0x"PFMT
+#endif
+
+/* convenience macros for secure string buffer operations */
+#define BUFFER_SIZE_BYTES(buf)      sizeof(buf)
+#define BUFFER_SIZE_ELEMENTS(buf)   (BUFFER_SIZE_BYTES(buf) / sizeof(buf[0]))
+#define BUFFER_LAST_ELEMENT(buf)    buf[BUFFER_SIZE_ELEMENTS(buf) - 1]
+#define NULL_TERMINATE_BUFFER(buf)  BUFFER_LAST_ELEMENT(buf) = 0
+#define BUFFER_ROOM_LEFT_W(wbuf)    (BUFFER_SIZE_ELEMENTS(wbuf) - wcslen(wbuf) - 1)
+#define BUFFER_ROOM_LEFT(abuf)      (BUFFER_SIZE_ELEMENTS(abuf) - strlen(abuf) - 1)
+
+#define PUSHF_MASK 0x00fcffff
+
+#define ALLOW_READ   0x01
+#define ALLOW_WRITE  0x02
+#define ALLOW_EXEC   0x04
+
+typedef enum {
+    CODE_INC,
+    CODE_DEC,
+    CODE_SELF_MOD,
+} Code_Snippet;
+
+typedef enum {
+    COPY_NORMAL,
+    COPY_CROSS_PAGE,
+} Copy_Mode;
+
+#define ALIGN_FORWARD(x, alignment) ((((uint)x) + ((alignment)-1)) & (~((alignment)-1)))
+#define ALIGN_BACKWARD(x, alignment) (((uint)x) & (~((alignment)-1)))
+
+#ifndef true
+# define true  (1)
+# define false (0)
+#endif
+
+#if VERBOSE
+# define VERBOSE_PRINT print
+#else
+/* FIXME: varargs for windows...for now since we don't care about efficiency we do this: */
+static void VERBOSE_PRINT(char *fmt, ...) {}
+#endif
+
+#ifdef WINDOWS
+/* FIXME: share w/ core/win32/os_exports.h */
+# ifdef X64
+#  define CXT_XIP Rip
+#  define CXT_XAX Rax
+#  define CXT_XCX Rcx
+#  define CXT_XDX Rdx
+#  define CXT_XBX Rbx
+#  define CXT_XSP Rsp
+#  define CXT_XBP Rbp
+#  define CXT_XSI Rsi
+#  define CXT_XDI Rdi
+#  define CXT_XFLAGS EFlags
+# else
+#  define CXT_XIP Eip
+#  define CXT_XAX Eax
+#  define CXT_XCX Ecx
+#  define CXT_XDX Edx
+#  define CXT_XBX Ebx
+#  define CXT_XSP Esp
+#  define CXT_XBP Ebp
+#  define CXT_XSI Esi
+#  define CXT_XDI Edi
+#  define CXT_XFLAGS EFlags
+# endif
+#endif
+
+/* DynamoRIO prints to stderr, so we need to too to get right output, also need to flush since DynamoRIO writes directly bypassing the buffering */
+static void
+print(char *fmt, ...)
+{
+    va_list ap;
+    va_start(ap, fmt);
+    vfprintf(stderr, fmt, ap);
+    fflush(stderr);
+}
+
+/* just to be sure */
+#define printf do_not_use_printf__use_print
+
+/* in tools_asm.asm */
+int code_self_mod(int iters);
+/* these don't need asm, but must be adjacent to code_self_mod and in order */
+int code_inc(int foo);
+int code_dec(int foo);
+int dummy(void);
+
+static int
+size(Code_Snippet func)
+{
+    int val1 = 0, val2 = 0, ret_val;
+    switch(func) {
+    case CODE_INC:
+        val1 = (int)code_inc;
+        val2 = (int)code_dec;
+        break;
+    case CODE_DEC:
+        val1 = (int)code_dec;
+        val2 = (int)dummy;
+        break;
+    case CODE_SELF_MOD:
+        val1 = (int)code_self_mod;
+        val2 = (int)code_inc;
+        break;
+    default:
+        return 0;
+    }
+    ret_val = val2 - val1;
+    if (ret_val < 0) {
+        print("Code layout assumption violation");
+        return 0;
+    }
+    return ret_val;
+}
+
+static int
+test(void *foo, int val)
+{
+    return (*(int (*) (int))foo)(val);
+}
+
+static int
+call(Code_Snippet func, int val) 
+{
+    switch(func){
+    CODE_INC:
+        return code_inc(val);
+    CODE_DEC:
+        return code_dec(val);
+    CODE_SELF_MOD:
+        return code_self_mod(val);
+    default:
+        print("Failed to find func to run\n");
+    }
+    return -1;
+}
+
+static char*
+page_align(char *buf)
+{
+    return (char *)(((ptr_int_t)buf + PAGE_SIZE - 1) & ~(PAGE_SIZE-1));
+}
+
+static char*
+copy_to_buf_normal(char *buf, int buf_len, int* copied_len, Code_Snippet func) 
+{
+    void *start;
+    size_t len = size(func);
+    switch(func) {
+    case CODE_INC:
+        start = (void *)code_inc;
+        break;
+    case CODE_DEC:
+        start = (void *)code_dec;
+        break;
+    case CODE_SELF_MOD:
+        start = (void *)code_self_mod;
+        break;
+    default:
+        print("Failed to copy func\n");
+    }
+    if (len > buf_len) {
+        print("Insufficient buffer for copy, have %d need %d\n", buf_len, len);
+        len = buf_len;
+    }
+    if (len < 0) {
+        print("Negative buffer available for copying\n");
+        len = 0;
+    }
+    memcpy(buf, start, len);
+    if (copied_len != NULL) *copied_len = len;
+    return buf;
+}
+
+static char*
+copy_to_buf_cross_page(char *buf, int buf_len, int *copied_len, Code_Snippet func) 
+{
+    char* buf_back = buf;
+    switch(func) {
+    case CODE_INC:
+    case CODE_DEC:
+        buf = (char *) ((ptr_int_t)page_align(buf) + PAGE_SIZE - 0x02);
+        buf_len = buf_len - ((ptr_int_t)buf - (ptr_int_t)buf_back) - PAGE_SIZE + 0x02;
+        break;
+    case CODE_SELF_MOD:
+        buf = (char *) ((ptr_int_t)page_align(buf) + PAGE_SIZE - 0x10);
+        buf_len = buf_len - ((ptr_int_t)buf - (ptr_int_t)buf_back) - PAGE_SIZE + 0x10;
+        break;
+    default:
+        ;
+    }
+    return copy_to_buf_normal(buf, buf_len, copied_len, func);
+}
+
+static char*
+copy_to_buf(char *buf, int buf_len, int* copied_len, Code_Snippet func, Copy_Mode mode) 
+{
+    switch(mode) {
+    case COPY_NORMAL:
+        return copy_to_buf_normal(buf, buf_len, copied_len, func);
+    case COPY_CROSS_PAGE:
+        return copy_to_buf_cross_page(buf, buf_len, copied_len, func);
+    default:
+        print("Improper copy mode\n");
+    }
+    *copied_len = 0;
+    return buf;
+}
+
+#ifndef LINUX 
+/****************************************************************************
+ * ntdll.dll interface 
+ * cleaner to use ntdll.lib but too lazy to add to build process
+ *
+ * WARNING: the Native API is an undocumented API and
+ * could change without warning with a new version of Windows.
+ */
+static HANDLE ntdll_handle = NULL;
+
+#define GET_PROC_ADDR(func, type, name) do { \
+    if (ntdll_handle == NULL) \
+        ntdll_handle = GetModuleHandle((LPCTSTR)"ntdll.dll"); \
+    if (func == NULL) {                      \
+        assert(ntdll_handle != NULL);        \
+        func = (type) GetProcAddress(ntdll_handle, (LPCSTR)name); \
+        assert(func != NULL);                \
+    }                                        \
+} while (0)
+
+/* A wrapper to define kernel entry point in a static function */
+/* In C use only at the end of a block prologue! */
+#define GET_NTDLL(NtFunction, type)                             \
+  typedef int (WINAPI *NtFunction##Type) type;                  \
+  static NtFunction##Type NtFunction;                           \
+  GET_PROC_ADDR(NtFunction, NtFunction##Type, #NtFunction);
+
+/* returns 0 if flush performed ok, not-zero otherwise */
+static int
+NTFlush(char *buf, int len)
+{
+    NTSTATUS status;
+    GET_NTDLL(NtFlushInstructionCache,
+              (IN HANDLE ProcessHandle,
+               IN PVOID BaseAddress OPTIONAL,
+               IN ULONG FlushSize));
+    status = NtFlushInstructionCache(GetCurrentProcess(), buf, len);
+    if (!NT_SUCCESS(status)) {
+        print("Error using NTFlush method\n");
+    } else {
+        return 0;
+    }
+    return -1;
+}
+
+typedef enum _PROCESSINFOCLASS {
+    ProcessBasicInformation,
+    ProcessQuotaLimits,
+    ProcessIoCounters,
+    ProcessVmCounters,
+    ProcessTimes,
+    ProcessBasePriority,
+    ProcessRaisePriority,
+    ProcessDebugPort,
+    ProcessExceptionPort,
+    ProcessAccessToken,
+    ProcessLdtInformation,
+    ProcessLdtSize,
+    ProcessDefaultHardErrorMode,
+    ProcessIoPortHandlers,          // Note: this is kernel mode only
+    ProcessPooledUsageAndLimits,
+    ProcessWorkingSetWatch,
+    ProcessUserModeIOPL,
+    ProcessEnableAlignmentFaultFixup,
+    ProcessPriorityClass,
+    ProcessWx86Information,
+    ProcessHandleCount,
+    ProcessAffinityMask,
+    ProcessPriorityBoost,
+    ProcessDeviceMap,
+    ProcessSessionInformation,
+    ProcessForegroundInformation,
+    ProcessWow64Information,
+    MaxProcessInfoClass
+} PROCESSINFOCLASS;
+
+/* format of data returned by QueryInformationProcess ProcessVmCounters */
+typedef struct _VM_COUNTERS {
+    SIZE_T PeakVirtualSize;
+    SIZE_T VirtualSize;
+    ULONG PageFaultCount;
+    SIZE_T PeakWorkingSetSize;
+    SIZE_T WorkingSetSize;
+    SIZE_T QuotaPeakPagedPoolUsage;
+    SIZE_T QuotaPagedPoolUsage;
+    SIZE_T QuotaPeakNonPagedPoolUsage;
+    SIZE_T QuotaNonPagedPoolUsage;
+    SIZE_T PagefileUsage;
+    SIZE_T PeakPagefileUsage;
+} VM_COUNTERS;
+
+static int
+get_process_mem_stats(HANDLE h, VM_COUNTERS *info)
+{
+    int i, len = 0;
+    /* could share w/ other process info routines... */
+    GET_NTDLL(NtQueryInformationProcess,
+              (IN HANDLE ProcessHandle,
+               IN PROCESSINFOCLASS ProcessInformationClass,
+               OUT PVOID ProcessInformation,
+               IN ULONG ProcessInformationLength,
+               OUT PULONG ReturnLength OPTIONAL));
+    i = NtQueryInformationProcess((HANDLE) h, ProcessVmCounters,
+                                  info, sizeof(VM_COUNTERS), &len);
+    if (i != 0) {
+        /* function failed */
+        memset(info, 0, sizeof(VM_COUNTERS));
+        return 0;
+    } else
+        assert(len == sizeof(VM_COUNTERS));
+    return 1;
+}
+#endif
+
+static int
+get_os_prot_word(int prot) 
+{
+#ifdef LINUX
+    return (((prot & ALLOW_READ) != 0) ? PROT_READ : 0) |
+        (((prot & ALLOW_WRITE) != 0) ? PROT_WRITE : 0) |
+        (((prot & ALLOW_EXEC) != 0) ? PROT_EXEC : 0);
+#else
+    if ((prot & ALLOW_WRITE) != 0) {
+        if ((prot & ALLOW_EXEC) != 0) {
+            return PAGE_EXECUTE_READWRITE;
+        } else {
+            return PAGE_READWRITE;
+        }
+    } else {
+        if ((prot & ALLOW_READ) != 0) {
+            if ((prot & ALLOW_EXEC) != 0) {
+                return PAGE_EXECUTE_READ;
+            } else {
+                return PAGE_READONLY;
+            }
+        } else {
+            if ((prot & ALLOW_EXEC) != 0) {
+                return PAGE_EXECUTE;
+            } else {
+                return PAGE_NOACCESS;
+            }
+        }
+    }
+#endif
+}
+
+static char *
+allocate_mem(int size, int prot) 
+{
+#ifdef LINUX
+    return (char *) mmap((void *)0, size, get_os_prot_word(prot), MAP_PRIVATE|MAP_ANON, 0, 0);
+#else
+    return (char *) VirtualAlloc(NULL, size, MEM_COMMIT, get_os_prot_word(prot));
+#endif
+}
+
+static void
+protect_mem(void *start, int len, int prot) 
+{
+#ifdef LINUX
+    void *page_start = (void *)(((ptr_int_t)start) & ~(PAGE_SIZE -1));
+    int page_len = (len + ((ptr_int_t)start - (ptr_int_t)page_start) + PAGE_SIZE - 1)
+        & ~(PAGE_SIZE - 1);
+    if (mprotect(page_start, page_len, get_os_prot_word(prot)) != 0) {
+        print("Error on mprotect: %d\n", errno);
+    }
+#else
+    int old;
+    if (VirtualProtect(start, len, get_os_prot_word(prot), &old) == 0) {
+        print("Error on VirtualProtect\n");
+    }
+#endif
+}
+
+static void
+protect_mem_check(void *start, int len, int prot, int expected)
+{
+#ifdef LINUX 
+    /* FIXME : add check */
+    protect_mem(start, len, prot);
+#else 
+    int old;
+    if (VirtualProtect(start, len, get_os_prot_word(prot), &old) == 0) {
+        print("Error on VirtualProtect\n");
+    }
+    if (old != get_os_prot_word(expected)) {
+        print("Unexpected previous permissions\n");
+    }
+#endif
+}
+
+void *
+reserve_memory(int size);
+
+static void
+test_print(void *buf, int n) 
+{
+    print("%d\n", test(buf, n));
+}
+
+#ifdef LINUX
+#  define INIT()
+#  define USE_USER32()
+#else
+#  define USE_USER32() do { if (argc > 5) MessageBeep(0); } while (0)
+
+#  define INIT() set_global_filter()
+
+# define WINDOWS_VERSION_VISTA  60
+# define WINDOWS_VERSION_2003   52
+# define WINDOWS_VERSION_XP     51
+# define WINDOWS_VERSION_2000   50
+# define WINDOWS_VERSION_NT     40
+
+#ifdef WINDOWS
+/* returns 0 on failure */
+int
+get_windows_version();
+#endif
+
+static LONG WINAPI 
+our_exception_filter(struct _EXCEPTION_POINTERS * pExceptionInfo)
+{
+    /* use EXCEPTION_CONTINUE_SEARCH to let it go all the way  */
+    if (pExceptionInfo->ExceptionRecord->ExceptionCode != EXCEPTION_ACCESS_VIOLATION) {
+        print("ERROR: Unexpected exception 0x%x caught\n", pExceptionInfo->ExceptionRecord->ExceptionCode);
+    }
+    print("Unhandled exception caught.\n");
+    return EXCEPTION_EXECUTE_HANDLER;
+}
+
+static
+set_global_filter()
+{
+    // Set the global unhandled exception filter to the exception filter
+    SetUnhandledExceptionFilter(our_exception_filter);
+}
+
+#endif /* LINUX */
+
+typedef void (*fptr)();
+
+#ifdef LINUX
+typedef int thread_handle;
+#else
+typedef HANDLE thread_handle;
+#endif
+
+/* Thread related functions */
+thread_handle
+create_thread(fptr f);
+void
+suspend_thread(thread_handle th);
+
+void
+resume_thread(thread_handle th);
+
+void
+join_thread(thread_handle th);
+
+void
+thread_yield();
+
+#ifdef WINDOWS
+static byte *
+get_drmarker_field(uint offset)
+{
+    /* read DR marker
+     * just hardcode the offsets for now
+     */
+    byte *field;
+    HANDLE ntdll_handle = GetModuleHandle("ntdll.dll");
+    byte *cbd = (byte *) GetProcAddress(ntdll_handle, "KiUserCallbackDispatcher");
+    byte *drmarker, *landing_pad;
+    if (*cbd != 0xe9) /* no jmp there */
+        return NULL;
+
+    /* see win32/callback.c:emit_landing_pad_code() for details */
+    landing_pad = *((int *)(cbd+1)) /* skip jmp opcode */
+        + cbd + 5; /* relative */
+#ifdef X64
+    drmarker = *((byte **)(landing_pad-8)); /* skip jmp opcode */
+#else
+    drmarker = *((byte **)(landing_pad+1)) /* skip jmp opcode */
+        + (uint)landing_pad + 5; /* relative */
+#endif
+    drmarker = (byte *) ALIGN_BACKWARD((uint)drmarker, PAGE_SIZE);
+    /* FIXME: check magic fields */
+    field = *((byte **)(drmarker + offset));
+    return field;
+}
+
+/* For hot patch testing only.  Only defined for Windows - hot patching hasn't
+ * been implemented for Linux yet.
+ * TODO: need to parameterize it with number or string to annotate a patch
+ *      point - needed to handle multiple patch points automatically for 
+ *      defs.cfg generation; another option would be to maintain a counter 
+ *      that increments with each use of the macro.
+ */
+#define INSERT_HOTP_PATCH_POINT()   \
+__asm {             \
+    __asm jmp foo   \
+    __asm _emit '$' \
+    __asm _emit 'p' \
+    __asm _emit 'p' \
+    __asm _emit 'o' \
+    __asm _emit 'i' \
+    __asm _emit 'n' \
+    __asm _emit 't' \
+    __asm _emit '$' \
+    __asm foo:      \
+}
+#endif
+
+#endif /* TOOLS_H */
diff --git a/suite/tests/win32/aslr-dll.c b/suite/tests/win32/aslr-dll.c
new file mode 100644
index 000000000..854520505
--- /dev/null
+++ b/suite/tests/win32/aslr-dll.c
@@ -0,0 +1,227 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* case 5325 ASLR on DLLs
+ * - FIXME: kernel32.dll user32.dll and ntdll.dll should eventually be rebased
+ *   and if done statically we wouldn't know that they are fine
+ */
+
+/* code inherited from hooker-ntdll.c */
+
+#include "tools.h"
+#include <windows.h>
+
+#define VERBOSE 0
+#define KNOWN_DLLS 1
+
+void*
+get_module_preferred_base(void* module_base)
+{
+    IMAGE_DOS_HEADER *dos;
+    IMAGE_NT_HEADERS *nt;
+    dos = (IMAGE_DOS_HEADER *) module_base;
+    nt = (IMAGE_NT_HEADERS *) (((ptr_uint_t)dos) + dos->e_lfanew);
+    return nt->OptionalHeader.ImageBase;
+}
+
+int num_checks = 0;
+int num_at_base = 0;
+int num_no_module = 0;
+
+void
+do_check(const char* hook_dll, const char* hookfn)
+{
+    HANDLE target_mod = GetModuleHandle(hook_dll);
+    void* preferred = NULL;
+    unsigned char* hooktarget;
+
+    if (target_mod == NULL) {
+        HMODULE hmod = LoadLibrary(hook_dll);
+        target_mod = GetModuleHandle(hook_dll);
+        if (hmod != target_mod || target_mod == NULL) {
+
+            /* NOTE: very funny LoadLibrary("apphelp.dll ") works, but
+             * GetModuleHandle("apphelp.dll ") doesn't find the same
+             * target!  can't think of a way to exploit ;)
+             */
+            print("GLE: %d\n", GetLastError());
+            print("error: hmod "PFX", target_mod "PFX"\n", hmod, target_mod);
+        }
+        if (target_mod == NULL) {
+            num_no_module++;
+            return;
+        }
+    }
+
+    hooktarget = (char*)GetProcAddress(target_mod, hookfn);
+
+    if (hooktarget == NULL) {
+        print("error: couldn't find %s!%s\n", hook_dll, hookfn);
+    } else 
+        print("%s!%s ok\n", hook_dll, hookfn);
+
+    if (target_mod != NULL)
+        preferred = get_module_preferred_base(target_mod);
+#if VERBOSE
+    print("%s at "PFX", preferred "PFX"\n", hook_dll, target_mod, 
+          preferred);
+    print("%s!%s "PFX"\n", hook_dll, hookfn, hooktarget);
+#endif
+    print("%s at %s base\n", hook_dll, 
+          (preferred == target_mod) ? "preferred" : "randomized");
+
+    num_checks++;
+    if (preferred == target_mod) 
+        num_at_base++;
+
+    print("all should be good\n");
+}
+
+int
+main()
+{
+    INIT();
+    
+    do_check("kernel32.dll", "GetProcessHeaps");
+    do_check("kernel32.dll", "Sleep");
+
+    do_check("user32.dll", "MessageBeep");
+    do_check("user32.dll", "MessageBoxW");
+
+    do_check("aslr-dll.exe", "executable");
+
+    /* Not in KnownDlls! */
+    do_check("ntdll.dll", "NtCreateSection");
+
+#if KNOWN_DLLS
+    /* all KnownDlls */
+    do_check("advapi32.dll", "GetAclInformation");
+    do_check("comdlg32.dll", "GetOpenFileNameW");
+    do_check("gdi32.dll", "GdiPlayEMF");
+    do_check("imagehlp.dll", "ImageRvaToVa");
+    // imagehlp!ImageDirectoryEntryToDataEx 
+    do_check("kernel32.dll", "findexport");
+    do_check("lz32.dll", "findexport");
+    do_check("ole32.dll", "findexport");
+    do_check("oleaut32.dll", "findexport");
+    do_check("olecli32.dll", "findexport");
+    do_check("olecnv32.dll", "findexport");
+    do_check("olesvr32.dll", "findexport");
+    do_check("olethk32.dll", "findexport");
+    do_check("rpcrt4.dll", "findexport");
+    do_check("shell32.dll", "findexport");
+    do_check("url.dll", "findexport");
+    do_check("urlmon.dll", "findexport");
+    do_check("user32.dll", "findexport");
+    do_check("version.dll", "findexport");
+    do_check("wininet.dll", "findexport");
+    do_check("wldap32.dll", "findexport");
+    /* transitive closure */
+    do_check("apphelp.dll", "findexport");
+    do_check("comctl32.dll", "findexport");
+    do_check("crypt32.dll", "findexport");
+    do_check("cryptui.dll", "findexport");
+    do_check("mpr.dll", "findexport");
+    do_check("msasn1.dll", "findexport");
+    do_check("msvcrt.dll", "findexport");
+    do_check("netapi32.dll", "findexport");
+    do_check("shdocvw.dll", "findexport");
+    do_check("shlwapi.dll", "findexport");
+    do_check("userenv.dll", "findexport");
+    do_check("wintrust.dll", "findexport");
+    do_check("wow32.dll", "findexport");
+#endif
+
+    do_check("secur32.dll", "LsaLogonUser");
+    do_check("secur32.dll", "MakeSignature");
+
+    /* leave error checking */
+    do_check("unknown.dll", "LsaLogonUser");
+    do_check("secur32.dll", "MissingExport");
+
+    /* case 8705 sfc.dll, though not present on all platforms */
+    do_check("sfc.dll", "SfpVerifyFile");
+
+    print("%d checked\n", num_checks);
+    print("%d at base\n", num_at_base);
+    print("%d DLL not found\n", num_no_module);
+    print("done\n");
+}
+
+
+
+/* KnownDlls 
+
+
+in Registry
+"advapi32"="advapi32.dll"
+"comdlg32"="comdlg32.dll"
+"gdi32"="gdi32.dll"
+"imagehlp"="imagehlp.dll"
+"kernel32"="kernel32.dll"
+"lz32"="lz32.dll"
+"ole32"="ole32.dll"
+"oleaut32"="oleaut32.dll"
+"olecli32"="olecli32.dll"
+"olecnv32"="olecnv32.dll"
+"olesvr32"="olesvr32.dll"
+"olethk32"="olethk32.dll"
+"rpcrt4"="rpcrt4.dll"
+"shell32"="shell32.dll"
+"url"="url.dll"
+"urlmon"="urlmon.dll"
+"user32"="user32.dll"
+"version"="version.dll"
+"wininet"="wininet.dll"
+"wldap32"="wldap32.dll"
+
+in WinObj - this is the transitive closure of statically linked for the above DLLs
+
+apphelp.dll   <- wow32.dll      <- olethk32.dll
+comctl32.dll  <- comdlg32.dll
+crypt32.dll
+cryptui.dll
+mpr.dll        <- olecli32.dll
+msasn1.dll
+msvcrt.dll     <- imagehlp.dll
+netapi32.dll
+shdocvw.dll
+shlwapi.dll    <- comdlg32.dll
+userenv.dll    <- wow32.dll      <- olethk32.dll
+wintrust.dll
+wow32.dll      <- olethk32.dll
+
+TOTEST: why is ntvdm.exe not in the list?
+
+TOTEST: lz32.dll has no IMPORTS -> confuses dumpbin and depends
+
+*/
diff --git a/suite/tests/win32/aslr-dll.template b/suite/tests/win32/aslr-dll.template
new file mode 100644
index 000000000..cbd3fc764
--- /dev/null
+++ b/suite/tests/win32/aslr-dll.template
@@ -0,0 +1,304 @@
+kernel32.dll!GetProcessHeaps ok
+kernel32.dll at preferred base
+all should be good
+kernel32.dll!Sleep ok
+kernel32.dll at preferred base
+all should be good
+user32.dll!MessageBeep ok
+user32.dll at preferred base
+all should be good
+user32.dll!MessageBoxW ok
+user32.dll at preferred base
+all should be good
+error: couldn't find aslr-dll.exe!executable
+aslr-dll.exe at preferred base
+all should be good
+ntdll.dll!NtCreateSection ok
+ntdll.dll at preferred base
+all should be good
+advapi32.dll!GetAclInformation ok
+#if : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache
+advapi32.dll at randomized base
+#else
+advapi32.dll at preferred base
+#endif
+all should be good
+comdlg32.dll!GetOpenFileNameW ok
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+comdlg32.dll at randomized base
+#else
+comdlg32.dll at preferred base
+#endif
+all should be good
+gdi32.dll!GdiPlayEMF ok
+#if : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache
+gdi32.dll at randomized base
+#else
+gdi32.dll at preferred base
+#endif
+all should be good
+imagehlp.dll!ImageRvaToVa ok
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+imagehlp.dll at randomized base
+#else
+imagehlp.dll at preferred base
+#endif
+all should be good
+error: couldn't find kernel32.dll!findexport
+kernel32.dll at preferred base
+all should be good
+error: couldn't find lz32.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+lz32.dll at randomized base
+#else
+lz32.dll at preferred base
+#endif
+all should be good
+error: couldn't find ole32.dll!findexport
+ole32.dll at preferred base
+all should be good
+error: couldn't find oleaut32.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+oleaut32.dll at randomized base
+#else
+oleaut32.dll at preferred base
+#endif
+all should be good
+error: couldn't find olecli32.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+olecli32.dll at randomized base
+#else
+olecli32.dll at preferred base
+#endif
+all should be good
+#if +RUNREGRESSION_Vista : 
+GLE: 126
+error: hmod 0x00000000, target_mod 0x00000000
+#else
+error: couldn't find olecnv32.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+olecnv32.dll at randomized base
+#else
+olecnv32.dll at preferred base
+#endif
+all should be good
+#endif
+error: couldn't find olesvr32.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+olesvr32.dll at randomized base
+#else
+olesvr32.dll at preferred base
+#endif
+all should be good
+error: couldn't find olethk32.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+olethk32.dll at randomized base
+#else
+olethk32.dll at preferred base
+#endif
+all should be good
+error: couldn't find rpcrt4.dll!findexport
+#if : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache
+rpcrt4.dll at randomized base
+#else
+rpcrt4.dll at preferred base
+#endif
+all should be good
+error: couldn't find shell32.dll!findexport
+#if : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache
+shell32.dll at randomized base
+#else
+shell32.dll at preferred base
+#endif
+all should be good
+error: couldn't find url.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+url.dll at randomized base
+#else
+url.dll at preferred base
+#endif
+all should be good
+error: couldn't find urlmon.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+urlmon.dll at randomized base
+#else
+urlmon.dll at preferred base
+#endif
+all should be good
+error: couldn't find user32.dll!findexport
+user32.dll at preferred base
+all should be good
+error: couldn't find version.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+version.dll at randomized base
+#else
+version.dll at preferred base
+#endif
+all should be good
+error: couldn't find wininet.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+wininet.dll at randomized base
+#else
+wininet.dll at preferred base
+#endif
+all should be good
+error: couldn't find wldap32.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+wldap32.dll at randomized base
+#else
+wldap32.dll at preferred base
+#endif
+all should be good
+#if +RUNREGRESSION_2000 : 
+GLE: 126
+error: hmod 0x00000000, target_mod 0x00000000
+#else
+error: couldn't find apphelp.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+apphelp.dll at randomized base
+#else
+apphelp.dll at preferred base
+#endif
+all should be good
+#endif
+error: couldn't find comctl32.dll!findexport
+#if : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache
+comctl32.dll at randomized base
+#else
+comctl32.dll at preferred base
+#endif
+all should be good
+error: couldn't find crypt32.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+crypt32.dll at randomized base
+#else
+crypt32.dll at preferred base
+#endif
+all should be good
+error: couldn't find cryptui.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+cryptui.dll at randomized base
+#else
+cryptui.dll at preferred base
+#endif
+all should be good
+error: couldn't find mpr.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+mpr.dll at randomized base
+#else
+mpr.dll at preferred base
+#endif
+all should be good
+error: couldn't find msasn1.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+msasn1.dll at randomized base
+#else
+msasn1.dll at preferred base
+#endif
+all should be good
+error: couldn't find msvcrt.dll!findexport
+#if : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache
+msvcrt.dll at randomized base
+#else
+msvcrt.dll at preferred base
+#endif
+all should be good
+error: couldn't find netapi32.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+netapi32.dll at randomized base
+#else
+netapi32.dll at preferred base
+#endif
+all should be good
+error: couldn't find shdocvw.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+shdocvw.dll at randomized base
+#else
+shdocvw.dll at preferred base
+#endif
+all should be good
+error: couldn't find shlwapi.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+shlwapi.dll at randomized base
+#else
+shlwapi.dll at preferred base
+#endif
+all should be good
+error: couldn't find userenv.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+userenv.dll at randomized base
+#else
+userenv.dll at preferred base
+#endif
+all should be good
+error: couldn't find wintrust.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+wintrust.dll at randomized base
+#else
+wintrust.dll at preferred base
+#endif
+all should be good
+error: couldn't find wow32.dll!findexport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+wow32.dll at randomized base
+#else
+wow32.dll at preferred base
+#endif
+all should be good
+secur32.dll!LsaLogonUser ok
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+secur32.dll at randomized base
+#else
+secur32.dll at preferred base
+#endif
+all should be good
+secur32.dll!MakeSignature ok
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+secur32.dll at randomized base
+#else
+secur32.dll at preferred base
+#endif
+all should be good
+GLE: 126
+error: hmod 0x00000000, target_mod 0x00000000
+error: couldn't find secur32.dll!MissingExport
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+secur32.dll at randomized base
+#else
+secur32.dll at preferred base
+#endif
+all should be good
+#if -RUNREGRESSION_NT
+sfc.dll!SfpVerifyFile ok
+sfc.dll at preferred base
+all should be good
+#else
+GLE: 126
+error: hmod 0x00000000, target_mod 0x00000000
+#endif
+#if +RUNREGRESSION_2000 : 
+41 checked
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+#if : +-client || : +-aslr -0x4 --exempt_aslr_list +-aslr_cache
+14 at base
+#else
+8 at base
+#endif
+#else
+41 at base
+#endif
+2 DLL not found
+#else
+43 checked
+#if : +-client --aslr_cache_list || : +-aslr +0x47 || : +-aslr -0x4 --exempt_aslr_list --aslr_cache_list
+#if : +-client || : +-aslr -0x4 --exempt_aslr_list +-aslr_cache
+16 at base
+#else
+10 at base
+#endif
+#else
+43 at base
+#endif
+1 DLL not found
+#endif
+done
diff --git a/suite/tests/win32/callback.c b/suite/tests/win32/callback.c
new file mode 100644
index 000000000..f37e39aa0
--- /dev/null
+++ b/suite/tests/win32/callback.c
@@ -0,0 +1,205 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include "tools.h"
+#include <stdio.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+static bool thread_ready = false;
+static bool past_crash = false;
+static uint last_received = 0;
+static HWND hwnd;
+static uint msgdata;
+
+static const UINT MSG_CUSTOM = WM_APP + 1;
+static const LRESULT MSG_SUCCESS = 1;
+
+static const WPARAM WP_NOP = 0;
+static const WPARAM WP_EXIT = 1;
+static const WPARAM WP_CRASH = 3;
+
+static const uint BAD_WRITE = 0x40;
+
+/* This is where all our callbacks come.  We get 4 default messages:
+ *   WM_GETMINMAXINFO                0x0024
+ *   WM_NCCREATE                     0x0081
+ *   WM_NCCALCSIZE                   0x0083
+ *   WM_CREATE                       0x0001
+ * and then our 2 custom messages that we send.
+ */
+LRESULT CALLBACK
+wnd_callback(HWND hwnd, UINT message, WPARAM wParam, LPARAM lParam)
+{
+    if (message == MSG_CUSTOM) {
+        print("in wnd_callback "PFX" %d %d\n", message, wParam, lParam);
+        if (wParam == WP_CRASH) {
+            /* ensure SendMessage returns prior to our crash */
+            ReplyMessage(TRUE);
+            print("About to crash\n");
+            *((int *)BAD_WRITE) = 4;
+            print("Should not get here\n");
+        }
+        return MSG_SUCCESS;
+    } else {
+        /* lParam varies so don't make template nondet */
+        print("in wnd_callback "PFX" %d\n", message, wParam);
+        return DefWindowProc(hwnd, message, wParam, lParam);
+    }
+}
+
+int WINAPI
+run_func(void * arg)
+{
+    EXCEPTION_RECORD exception;
+    MSG msg;
+    char *winName = "foobar";
+    WNDCLASS wndclass = {0, wnd_callback, 0, 0, NULL/* WinMain hwnd would be here */,
+                         NULL, NULL,
+                         NULL, NULL, winName};
+    
+    if (!RegisterClass(&wndclass)) {
+        print("Unable to create window class\n");
+        return 0;
+    }
+    hwnd = CreateWindow(winName, winName, 0, CW_USEDEFAULT, CW_USEDEFAULT, CW_USEDEFAULT,
+                        CW_USEDEFAULT, NULL, NULL, NULL/* WinMain hwnd would be here */,
+                        NULL);
+    /* deliberately not calling ShowWindow */
+
+    /* For case 10579 we want a handled system call in this thread prior
+     * to our crash inside a callback */
+    VirtualAlloc(0, 1024, MEM_RESERVE, PAGE_EXECUTE_READWRITE);
+
+    thread_ready = true;
+    while (true) {
+        __try {
+            if (GetMessage(&msg, NULL, 0, 0)) {
+                /* Messages not auto-sent to callbacks are processed here */
+                if (msg.message != MSG_CUSTOM || msg.wParam != WP_NOP) {
+                    print("Got message "PFX" %d %d\n",
+                          msg.message, msg.wParam, msg.lParam);
+                }
+                last_received = msg.message;
+                if (msg.message == MSG_CUSTOM && msg.wParam == WP_EXIT)
+                    break; /* Done */
+                TranslateMessage(&msg); /* convert virtual-key msgs to character msgs */
+                DispatchMessage(&msg);
+            }
+        }
+        __except (/* Only catch the bad write, to not mask DR errors (like
+                   * case 10579) */
+                  (GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION &&
+                   (GetExceptionInformation())->ExceptionRecord->ExceptionInformation[0]
+                   == 1 /* write */ &&
+                   (GetExceptionInformation())->ExceptionRecord->ExceptionInformation[1]
+                   == BAD_WRITE) ?
+                  EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) {
+            /* This should have crossed the callback boundary
+             * On xpsp2 and earlier we never see a callback return for
+             * the crashing callback, while on 2k3sp1 we do see one.
+             */
+            print("Inside handler\n");
+            past_crash = true;
+            continue;
+        }
+    }
+    return msg.wParam;
+}
+
+int
+main()
+{
+    int tid;
+    HANDLE hThread;
+    MSG msg;
+    uint msgnum = 0;
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    print("About to create thread\n");
+    hThread = CreateThread(NULL, 0, run_func, NULL, 0, &tid);
+    if (hThread == NULL) { 
+        print("Error creating thread\n");
+        return -1;
+    }
+    while (!thread_ready)
+        Sleep(0);
+
+    /* We have to send a message to a window to get a callback.
+     * We go ahead and use the blocking SendMessage for simplicity;
+     * could use SendMessageCallback and get a callback back, but have to
+     * ask for messages to receive it and then have no clear exit path.
+     */
+    if (SendMessage(hwnd, MSG_CUSTOM, WP_CRASH, msgnum++) != MSG_SUCCESS) {
+        print("Error %d posting window message\n", GetLastError());
+        return -1;
+    }
+    /* On bucephalus (win2k3sp1) I need to send a message to get the
+     * thread to go into the except block: it sits waiting in the
+     * kernel at the NtCallbackReturn from
+     * KiUserCallbackExceptionHandler, and that is where it receives
+     * the callback for this message: seems problematic natively?
+     */
+    PostThreadMessage(tid, MSG_CUSTOM, WP_NOP, msgnum++);
+    while (!past_crash)
+        Sleep(0);
+    if (SendMessage(hwnd, MSG_CUSTOM, WP_NOP, msgnum++) != MSG_SUCCESS) {
+        print("Error %d posting window message\n", GetLastError());
+        return -1;
+    }
+
+    /* Message not sent to a window is processed inside the GetMessage loop,
+     * with no callback involved.  So this bit here is mainly to get the thread
+     * to exit. */
+    if (!PostThreadMessage(tid, MSG_CUSTOM, WP_EXIT, msgnum++)) {
+        print("Error %d posting thread message\n", GetLastError());
+        return -1;
+    }
+    while (last_received != MSG_CUSTOM)
+        Sleep(0);
+
+    WaitForSingleObject(hThread, INFINITE);
+    print("All done\n");
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
diff --git a/suite/tests/win32/callback.expect b/suite/tests/win32/callback.expect
new file mode 100755
index 000000000..2089a5782
--- /dev/null
+++ b/suite/tests/win32/callback.expect
@@ -0,0 +1,11 @@
+About to create thread
+in wnd_callback 0x00000024 0
+in wnd_callback 0x00000081 0
+in wnd_callback 0x00000083 0
+in wnd_callback 0x00000001 0
+in wnd_callback 0x00008001 3 0
+About to crash
+Inside handler
+in wnd_callback 0x00008001 0 2
+Got message 0x00008001 1 3
+All done
diff --git a/suite/tests/win32/crtprcs.c b/suite/tests/win32/crtprcs.c
new file mode 100644
index 000000000..20a111b98
--- /dev/null
+++ b/suite/tests/win32/crtprcs.c
@@ -0,0 +1,84 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+
+#include "Windows.h"
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+static int 
+main(int argc, char **argv) 
+{
+    STARTUPINFO si = { sizeof(STARTUPINFO) };
+    PROCESS_INFORMATION pi;
+    char cmdline[128];
+
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    if (argc == 1) {
+        /* normal execution */
+
+        sprintf(cmdline,"crtprcs.exe 1");
+
+        printf("creating subprocess...\n");
+
+        if (!CreateProcess(argv[0],
+                           cmdline,
+                           NULL,
+                           NULL,
+                           FALSE,
+                           0,
+                           NULL,
+                           NULL,
+                           &si,
+                           &pi)) 
+            printf("CreateProcess failure\n"); 
+        else
+            WaitForSingleObject(pi.hProcess, INFINITE);
+    }
+    else { /* child process */
+        printf("subprocess running.\n");
+    }
+    
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+
+    return 0;
+}
diff --git a/suite/tests/win32/crtprcs.template b/suite/tests/win32/crtprcs.template
new file mode 100755
index 000000000..aebc313d4
--- /dev/null
+++ b/suite/tests/win32/crtprcs.template
@@ -0,0 +1,2 @@
+subprocess running.
+creating subprocess...
diff --git a/suite/tests/win32/debugger.c b/suite/tests/win32/debugger.c
new file mode 100644
index 000000000..29e7ce167
--- /dev/null
+++ b/suite/tests/win32/debugger.c
@@ -0,0 +1,188 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* TOTEST: windbg -d  to ignore all exceptions should have them all handled */
+
+#include "../security-win32/except.h"
+
+#include <setjmp.h>
+#include "tools.h" /* for print() */
+
+jmp_buf mark;
+int count = 0;
+
+/* INT3 is a trap-type exception
+ * the saved EIP should point to the byte after the instruction
+ * which should be the beginning of the next instruction.
+ * (Note debuggers normally overwrite existing instructions
+ * and then would have to decrement EIP by 1 (since they know they used 0XCC)
+ * to execute to original interrupted instruction.)
+ */
+
+
+#define TRAP_OPT()                              \
+__pragma optimize("", off)                      \
+__debugbreak()                                  \
+__pragma optimize("", on)
+
+/* don't use at CPL0 INT3 (or INT1, or BOUNDS) - will disable interrupts */
+#define TRAP() __asm { int 3 }  /* 0xcc */
+
+#define TRAP_TWO_BYTE() __asm { __emit 0xcd __emit 0x03 }
+/* see Intel manual - this version doesn't have all special characteristics of the 0xcc,
+ * but in our mode should be handled ok
+ */
+
+#define SINGLE_STEP() __asm { int 1 }
+
+#define DEBUGGER_INTERFACE() __asm { int 2d }
+
+/* FIXME: test single stepping with TRAP bit set in EFLAGS - can one control this at CPL3? */
+
+/* FIXME: case 11058 I don't understand this - INT3 is supposed to save the EIP _AFTER_ the instruction
+ * why am I getting the original instruction natively?
+ */
+/* as a workaround we increment EIP again */
+#define SKIP_INT3 ((GetExceptionInformation())->ContextRecord->CXT_XIP++)
+
+
+
+int
+TrapIfDebugger() 
+{
+    int trapped = 0;
+    __try {
+        TRAP();
+        trapped = 1;
+    } __except (SKIP_INT3, EXCEPTION_CONTINUE_EXECUTION) { }
+    return !trapped;
+}
+
+int
+TrapIfDebuggerVerbose() 
+{
+    EXCEPTION_RECORD exception;
+    CONTEXT *context;
+
+    int dbg = 0;
+    int trapped = 0;
+    print("about to trap\n");
+    __try {
+        initialize_registry_context();
+        TRAP();
+        print("continued after trap\n");
+        dbg = 1;
+    } __except (context = (GetExceptionInformation())->ContextRecord,
+                dump_exception_info((GetExceptionInformation())->ExceptionRecord,
+                                    context),
+                trapped = 1,
+                print("in filter\n"), 
+                (GetExceptionInformation())->ExceptionRecord->ExceptionCode == STATUS_BREAKPOINT ? 
+                /* breakpoint - continue next */ SKIP_INT3, EXCEPTION_CONTINUE_EXECUTION : 
+                /* not ours */ EXCEPTION_CONTINUE_SEARCH) {
+        print("handler NOT REACHED\n");
+    }
+    if (!trapped) print("didn't trap, continued in debugger?!\n");
+    return !trapped;
+}
+
+int
+invalid_handle()
+{
+    int dbg;
+    /* case 11051 - why is there an STATUS_INVALID_HANDLE exception on Vista kernel32!CloseHandle */
+    print("Invalid handle about to happen\n");
+    print("about to close\n");
+    __try {
+        HANDLE h = (HANDLE)9999;
+        dbg = 1;
+        CloseHandle(h);
+        dbg = 0;
+    } __except(print("in close filter\n", EXCEPTION_CONTINUE_EXECUTION)) {
+    }
+    print("continued succesfully\n");
+    return dbg;
+}
+
+#define name_status(x, s) if ((x) == (s)) print(#s); 
+
+# include <windows.h>
+/* top-level exception handler */
+static LONG
+our_top_handler(struct _EXCEPTION_POINTERS * pExceptionInfo)
+{
+    print("caught exception "PFX": ", pExceptionInfo->ExceptionRecord->ExceptionCode);
+    /* 0x8.. */
+    name_status(pExceptionInfo->ExceptionRecord->ExceptionCode, STATUS_GUARD_PAGE_VIOLATION);
+    /* STATUS_DATATYPE_MISALIGNMENT maybe on some syscalls? */
+    name_status(pExceptionInfo->ExceptionRecord->ExceptionCode, STATUS_BREAKPOINT);
+    name_status(pExceptionInfo->ExceptionRecord->ExceptionCode, STATUS_SINGLE_STEP);
+
+    /* 0xC.. */
+    name_status(pExceptionInfo->ExceptionRecord->ExceptionCode, STATUS_INVALID_HANDLE);
+
+    count++;
+    print("exception, instance %d\n", count);
+    longjmp(mark, count);
+}
+
+int 
+main(int argc, char *argv[])
+{
+    int i,j;
+    int dbg;
+    USE_USER32();
+    INIT();
+
+    SetUnhandledExceptionFilter((LPTOP_LEVEL_EXCEPTION_FILTER) our_top_handler);
+
+    i = setjmp(mark);
+    print("Test %d\n", i);
+
+    switch (i) {
+
+    case 0: dbg = TrapIfDebuggerVerbose();
+        print("%s\n", dbg ? "debugger handled on first chance" : "not handled");
+        /* continue */
+    case 1: dbg = TrapIfDebugger();
+        print("%s\n", dbg ? "debugger handled on first chance" : "not handled");
+        /* continue */
+    case 2: dbg = invalid_handle();
+        print("%s\n", dbg ? "debugger handled on first chance" : "not handled");
+        /* continue */
+    default:;
+    }
+
+    print("All done\n");
+
+    return 0;
+}
diff --git a/suite/tests/win32/debugger.template b/suite/tests/win32/debugger.template
new file mode 100644
index 000000000..68db85bd1
--- /dev/null
+++ b/suite/tests/win32/debugger.template
@@ -0,0 +1,23 @@
+Test 0
+about to trap
+	exception code = 0x80000003, ExceptionFlags=0x00000000
+	record=00000000, params=3
+	pc=0x00401858 eax=0x00000000
+  ContextFlags=0x0001003f 
+  Edi=0xeecdcdcd Esi=0xffcdcdcd Ebx=0xbbcdcdcd 
+  Edx=0xddcdcdcd Ecx=0xcccdcdcd 
+  
+  Eax=0x00000000 
+  Ebp=0x0012ff5c Eip=0x00401858 SegCs=0x0000001b 
+  EFlags & 0xFFFF=0x00000202 Esp=0x0012fed8 SegSs=0x00000023 
+  <floating point area>
+  SegGs=0x00000000 SegFs=0x0000003b SegEs=0x00000023 SegDs=0x00000023 
+in filter
+continued after trap
+not handled
+not handled
+Invalid handle about to happen
+about to close
+continued succesfully
+not handled
+All done
diff --git a/suite/tests/win32/delaybind.c b/suite/tests/win32/delaybind.c
new file mode 100644
index 000000000..8404b996d
--- /dev/null
+++ b/suite/tests/win32/delaybind.c
@@ -0,0 +1,122 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include <stdio.h>
+#include "tools.h"
+
+#include <delayimp.h>
+
+#define VERBOSE 0
+
+/* WATCH OUT to not use the bash internal bind */
+/* `which bind` -v -u delaybind.exe delaybind.dll.dll */
+
+/* FIXME: I can't get a delay loaded DLL to get bound with bind.exe */
+
+/* support unloading with, otherwise we won't be able to unload the DLL */
+
+/* The following pragma does not seem to be correctly applying these
+ * link options, so we'll put them on the command line in the Makefile.
+ *
+ * #pragma comment(linker, "/delayload:delaybind.dll.dll /DELAY:UNLOAD")
+ */
+
+/* do not use /DELAY:NOBIND since that was the purpose of this test */
+
+#pragma comment(lib, "delayimp")
+/* using the standard implementation of delay loading in delayimp.lib */
+
+#pragma comment(lib, "delaybind.dll.lib")
+
+/* note that in later Visual Studio __FUnloadDelayLoadedDLL has
+ * been renamed to __FUnloadDelayLoadedDLL2 since they had to
+ * change the ImgDelayDescr structure to use RVAs instead of
+ * pointers.
+ */
+#define UNLOAD __FUnloadDelayLoadedDLL2
+
+int
+myloader(void)
+{
+    int TestReturn;
+
+    // dll will get loaded at this point
+    make_a_lib(3);
+
+    // dll will unload at this point
+    TestReturn = UNLOAD("delaybind.dll.dll");
+   
+    if (TestReturn)
+        print("\nDLL was unloaded\n");
+    else
+        print("\nDLL was not unloaded\n");
+    return TestReturn;
+}
+
+int main()
+{
+    HMODULE lib;
+    print("starting delaybind\n");
+
+    myloader();
+    
+    /* try again */
+    myloader();
+
+    print("done with delaybind\n");
+
+    return 0;
+}
+
+
+/* Note that dumpbin from VC98 is crashing of showing imports of say
+ * user32.dll on XP SP2, so the format of some fields may have changed!
+ */
+
+#ifdef DELAYHLP
+/* from Microsoft Visual Studio/VC98/Include/DELAYHLP.CPP */
+
+ if (pinh->Signature == IMAGE_NT_SIGNATURE &&
+                TimeStampOfImage(pinh) == pidd->dwTimeStamp &&
+                FLoadedAtPreferredAddress(pinh, hmod)) {
+
+                OverlayIAT(pidd->pIAT, pidd->pBoundIAT);
+
+TimeStampOfImage(PIMAGE_NT_HEADERS pinh) {
+    return pinh->FileHeader.TimeDateStamp;
+ }
+
+FLoadedAtPreferredAddress(PIMAGE_NT_HEADERS pinh, HMODULE hmod) {
+    return DWORD(hmod) == pinh->OptionalHeader.ImageBase;
+ }
+#endif
diff --git a/suite/tests/win32/delaybind.dll.c b/suite/tests/win32/delaybind.dll.c
new file mode 100644
index 000000000..0827b0431
--- /dev/null
+++ b/suite/tests/win32/delaybind.dll.c
@@ -0,0 +1,183 @@
+/* **********************************************************
+ * Copyright (c) 2006 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* case 8507 delay load testing */
+
+#include "tools.h"
+#include <windows.h>
+
+/* note that we want _this_ DLL bound normally to advapi32.dll  */
+
+/* too bad that most of the useful functions in advapi32
+ * or aren't present on NT [ConvertSidToStringSid()]  or 2000 [CreateWellKnownSid()]
+ */
+
+/* Windows NT 4.0 and earlier: ConvertSidToStringSid and
+ *  ConvertStringSidToSid are not supported. Use the following code to
+ *   convert a SID to string format.
+ */
+
+BOOL GetTextualSid(
+    PSID pSid,            // binary SID
+    LPTSTR TextualSid,    // buffer for Textual representation of SID
+    LPDWORD lpdwBufferLen // required/provided TextualSid buffersize
+    )
+{
+    PSID_IDENTIFIER_AUTHORITY psia;
+    DWORD dwSubAuthorities;
+    DWORD dwSidRev=SID_REVISION;
+    DWORD dwCounter;
+    DWORD dwSidSize;
+
+    // Validate the binary SID.
+
+    if(!IsValidSid(pSid)) return FALSE;
+
+    // Get the identifier authority value from the SID.
+
+    psia = GetSidIdentifierAuthority(pSid);
+
+    // Get the number of subauthorities in the SID.
+
+    dwSubAuthorities = *GetSidSubAuthorityCount(pSid);
+
+    // Compute the buffer length.
+    // S-SID_REVISION- + IdentifierAuthority- + subauthorities- + NULL
+
+    dwSidSize=(15 + 12 + (12 * dwSubAuthorities) + 1) * sizeof(TCHAR);
+
+    // Check input buffer length.
+    // If too small, indicate the proper size and set the last error.
+
+    if (*lpdwBufferLen < dwSidSize)
+    {
+        *lpdwBufferLen = dwSidSize;
+        SetLastError(ERROR_INSUFFICIENT_BUFFER);
+        return FALSE;
+    }
+
+    // Add 'S' prefix and revision number to the string.
+
+    dwSidSize=wsprintf(TextualSid, TEXT("S-%lu-"), dwSidRev );
+
+    // Add a SID identifier authority to the string.
+
+    if ( (psia->Value[0] != 0) || (psia->Value[1] != 0) )
+    {
+        dwSidSize+=wsprintf(TextualSid + lstrlen(TextualSid),
+                    TEXT("0x%02hx%02hx%02hx%02hx%02hx%02hx"),
+                    (USHORT)psia->Value[0],
+                    (USHORT)psia->Value[1],
+                    (USHORT)psia->Value[2],
+                    (USHORT)psia->Value[3],
+                    (USHORT)psia->Value[4],
+                    (USHORT)psia->Value[5]);
+    }
+    else
+    {
+        dwSidSize+=wsprintf(TextualSid + lstrlen(TextualSid),
+                    TEXT("%lu"),
+                    (ULONG)(psia->Value[5]      )   +
+                    (ULONG)(psia->Value[4] <<  8)   +
+                    (ULONG)(psia->Value[3] << 16)   +
+                    (ULONG)(psia->Value[2] << 24)   );
+    }
+
+    // Add SID subauthorities to the string.
+    //
+    for (dwCounter=0 ; dwCounter < dwSubAuthorities ; dwCounter++)
+    {
+        dwSidSize+=wsprintf(TextualSid + dwSidSize, TEXT("-%lu"),
+                    *GetSidSubAuthority(pSid, dwCounter) );
+    }
+
+    return TRUE;
+}
+
+void
+test_sid(void)
+{
+    TCHAR buf[100];
+    DWORD size = sizeof(buf);
+
+    PSID psid;
+
+    /*  from 
+     * http://msdn.microsoft.com/library/default.asp?url=/library/en-us/secauthz/security/searching_for_a_sid_in_an_access_token_in_c__.asp
+     */
+
+    SID_IDENTIFIER_AUTHORITY SIDAuth = SECURITY_NT_AUTHORITY;
+    BOOL ok;
+
+    // Create a SID for the BUILTIN\Administrators group.
+
+    if(! AllocateAndInitializeSid( &SIDAuth, 2,
+                                   SECURITY_BUILTIN_DOMAIN_RID,
+                                   DOMAIN_ALIAS_RID_ADMINS,
+                                   0, 0, 0, 0, 0, 0,
+                                   &psid) ) {
+        print( "AllocateAndInitializeSid Error %u\n", GetLastError() );
+        return FALSE;
+    }
+
+    ok = GetTextualSid(psid, buf, &size);
+
+    if (ok) {
+        print("BUILTIN\\Administrators: %s\n", buf);
+    } else {
+        print("FAILED!\n", buf);
+
+        if(!IsValidSid(psid)) 
+            print("invalid SID!\n", buf);
+    }
+}
+
+
+
+int
+__declspec(dllexport)
+make_a_lib(int arg)
+{
+    test_sid();
+    return 0;
+}
+
+BOOL APIENTRY 
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    switch (reason_for_call) {
+    case DLL_PROCESS_ATTACH:
+        print("in delay bind dll\n");
+        break;
+    }
+    return TRUE;
+}
diff --git a/suite/tests/win32/delaybind.template b/suite/tests/win32/delaybind.template
new file mode 100644
index 000000000..6d743037c
--- /dev/null
+++ b/suite/tests/win32/delaybind.template
@@ -0,0 +1,10 @@
+starting delaybind
+in delay bind dll
+BUILTIN\Administrators: S-1-5-32-544
+
+DLL was unloaded
+in delay bind dll
+BUILTIN\Administrators: S-1-5-32-544
+
+DLL was unloaded
+done with delaybind
diff --git a/suite/tests/win32/dll.c b/suite/tests/win32/dll.c
new file mode 100644
index 000000000..db1af671f
--- /dev/null
+++ b/suite/tests/win32/dll.c
@@ -0,0 +1,181 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include <stdio.h>
+#include "tools.h"
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#define VERBOSE 0
+
+static char*
+get_short_name(char* exename)
+{
+    char *exe;
+    exe = strrchr(exename, '\\');
+    if (exe == NULL) {
+        exe = strrchr(exename, '/');
+        if (exe == NULL)
+            exe = exename;
+    } else
+        exe++; /* skip backslash */
+    return exe;
+}
+
+static char * modules[] = {
+    "dynamorio.dll",
+    "dll.exe",
+    "dll.dll.dll",
+    "kernel32.dll",
+    "ntdll.dll",
+};
+
+static int num_modules = sizeof(modules) / sizeof(char *);
+
+static BOOL module_found[sizeof(modules) / sizeof(char *)];
+
+int
+check_mbi(MEMORY_BASIC_INFORMATION *mbi)
+{
+    int nLen, i, num_found = 0;
+    char szModName[MAX_PATH];
+    if (mbi->Type != MEM_IMAGE ||
+        mbi->AllocationBase != mbi->BaseAddress ||
+        mbi->AllocationBase == NULL) {
+        nLen = 0;
+    } else {
+        nLen = GetModuleFileNameA((HINSTANCE) mbi->AllocationBase, szModName,
+                                  (sizeof(szModName)/sizeof(szModName[0])));
+    }
+    if (nLen > 0) {
+        char *nm = get_short_name(szModName);
+        for (i = 0; i < num_modules; i++) {
+            if (_stricmp(nm, modules[i]) == 0) {
+                num_found++;
+                module_found[i] = TRUE;
+#if VERBOSE
+                print("Found %s\n", modules[i]);
+#endif
+            }
+        }
+#if VERBOSE
+        print("\n%p-%s", mbi->AllocationBase, szModName);
+#endif
+    }
+    return num_found;
+}
+
+void
+print_modules()
+{
+    /* Try to print out addresses of all loaded dlls */
+    PBYTE pb = NULL;
+    MEMORY_BASIC_INFORMATION mbi;
+    int i, num_found = 0;
+    memset(module_found, 0, sizeof(module_found));
+#if VERBOSE
+    print("\nLoaded Modules:");
+#endif
+    while (VirtualQuery(pb, &mbi, sizeof(mbi)) == sizeof(mbi)) {
+        num_found += check_mbi(&mbi);
+        pb += mbi.RegionSize;
+    }
+#if VERBOSE
+    print("\n\n");
+#endif
+    print("Found %d of %d expected modules\n", num_found, num_modules);
+    for (i = 0; i < num_modules; i++) {
+        if (!module_found[i]) {
+            print("Didn't find module %s\n", modules[i]);
+        }
+    }
+
+    /* dr intentionally screws up QueryVirtualMemory calls on the dr dll to
+     * hide from walks like the above, check our dll address directly to see
+     * if we are still on the module list */
+    mbi.Type = MEM_IMAGE;
+    mbi.BaseAddress = 0x71000000; /* release */
+    mbi.AllocationBase = 0x71000000;
+    num_found += check_mbi(&mbi);
+    mbi.BaseAddress = 0x15000000; /* debug */
+    mbi.AllocationBase = 0x15000000;
+    num_found += check_mbi(&mbi);
+
+    print("Found %d of %d expected modules\n", num_found, num_modules);
+    for (i = 0; i < num_modules; i++) {
+        if (!module_found[i]) {
+            print("Didn't find module %s\n", modules[i]);
+        }
+    }
+
+    fflush(stdout);
+}
+
+int main()
+{
+    HANDLE lib;
+    int i;
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+    print_modules();
+    lib = LoadLibrary("dll.dll.dll");
+    if (lib == NULL) {
+        print("error loading library\n");
+    } else {
+        BOOL res;
+        BOOL (WINAPI *proc)(DWORD);
+        print("loaded dll.dll.dll\n");
+#if VERBOSE
+        print("library is at "PFX"\n", lib);
+        fflush(stdout);
+#endif
+        print_modules();
+        proc = (BOOL (WINAPI *)(DWORD))
+            GetProcAddress(lib, (LPCSTR)"import_me");
+        res = (*proc)(5);
+        print("Called import_me with 5, result is %d\n", res);
+        FreeLibrary(lib);
+        print("freed library\n");
+        print_modules();
+        fflush(stdout);
+    }
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
diff --git a/suite/tests/win32/dll.dll.c b/suite/tests/win32/dll.dll.c
new file mode 100644
index 000000000..1fd9bc4c2
--- /dev/null
+++ b/suite/tests/win32/dll.dll.c
@@ -0,0 +1,54 @@
+/* **********************************************************
+ * Copyright (c) 2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "tools.h"
+
+int  __declspec(dllexport)
+import_me(int x)
+{
+    print("in import\n");
+    return 2*x;
+}
+
+BOOL APIENTRY 
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    switch (reason_for_call) {
+    case DLL_PROCESS_ATTACH:
+        print("dll.dll.dll process attach\n");
+        break;
+    case DLL_PROCESS_DETACH:
+        print("dll.dll.dll process detach\n");
+        break;        
+    }
+    return TRUE;
+}
diff --git a/suite/tests/win32/dll.template b/suite/tests/win32/dll.template
new file mode 100755
index 000000000..ea5024cfa
--- /dev/null
+++ b/suite/tests/win32/dll.template
@@ -0,0 +1,36 @@
+Found 3 of 5 expected modules
+Didn't find module dynamorio.dll
+Didn't find module dll.dll.dll
+#if : +-no_hide
+Found 4 of 5 expected modules
+Didn't find module dll.dll.dll
+#else
+Found 3 of 5 expected modules
+Didn't find module dynamorio.dll
+Didn't find module dll.dll.dll
+#endif
+dll.dll.dll process attach
+loaded dll.dll.dll
+Found 4 of 5 expected modules
+Didn't find module dynamorio.dll
+#if : +-no_hide
+Found 5 of 5 expected modules
+#else
+Found 4 of 5 expected modules
+Didn't find module dynamorio.dll
+#endif
+in import
+Called import_me with 5, result is 10
+dll.dll.dll process detach
+freed library
+Found 3 of 5 expected modules
+Didn't find module dynamorio.dll
+Didn't find module dll.dll.dll
+#if : +-no_hide
+Found 4 of 5 expected modules
+Didn't find module dll.dll.dll
+#else
+Found 3 of 5 expected modules
+Didn't find module dynamorio.dll
+Didn't find module dll.dll.dll
+#endif
\ No newline at end of file
diff --git a/suite/tests/win32/except.c b/suite/tests/win32/except.c
new file mode 100644
index 000000000..662f4e5f3
--- /dev/null
+++ b/suite/tests/win32/except.c
@@ -0,0 +1,190 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include "tools.h"
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#define DEFAULT_TARGET_ADDR 0x4
+
+static int exception_location;
+static int target_addr;
+static int reg_eflags;
+static int reg_eax;
+static int reg_ebx;
+static int reg_ecx;
+static int reg_edx;
+static int reg_edi;
+static int reg_esi;
+static int reg_ebp;
+static int reg_esp;
+
+/* FIXME : would it be better to set some of these register to known values? */
+static void
+fault()
+{
+    __asm {
+        lea   eax, [exception_location_tag]
+        mov   exception_location, eax
+        pushfd
+        pop   eax
+        mov   reg_eflags, eax
+        mov   eax, DEFAULT_TARGET_ADDR
+        mov   reg_eax, eax
+        mov   reg_ebx, ebx
+        mov   reg_ecx, ecx
+        mov   reg_edx, edx
+        mov   reg_edi, edi
+        mov   reg_esi, esi
+        mov   reg_esp, esp
+        mov   reg_ebp, ebp
+      exception_location_tag:
+        mov   dword ptr [eax], 0x2
+    }
+}
+
+static void
+fault_selfmod()
+{
+    __asm {
+        pushfd
+        pop   eax
+        mov   reg_eflags, eax
+        mov   reg_ecx, ecx
+        mov   reg_edx, edx
+        mov   reg_edi, edi
+        mov   reg_esi, esi
+        mov   reg_esp, esp
+        mov   reg_ebp, ebp
+        lea   eax, [selfmod_location_tag]
+        mov   exception_location, eax
+        lea   eax, [exception_location_tag]
+        mov   ebx, [eax]
+        mov   reg_ebx, ebx
+        mov   reg_eax, eax
+        mov   target_addr, eax
+      selfmod_location_tag:
+        mov   [eax], ebx
+        mov   exception_location, eax
+        mov   eax, DEFAULT_TARGET_ADDR
+        mov   reg_eax, eax
+      exception_location_tag:
+        mov   dword ptr [eax], 0x2
+    }
+}
+
+void
+do_run(void (*func) (), uint *target_addr_location)
+{
+    EXCEPTION_RECORD exception;
+    CONTEXT *context;
+    int slot;
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    __try {
+	__try {
+	    __try {
+                func();
+		print("At statement after exception\n");
+	    }
+	    __except (
+		      context = (GetExceptionInformation())->ContextRecord,
+		      context->CXT_XAX = (unsigned long) &slot,
+		      EXCEPTION_CONTINUE_EXECUTION) {
+		print("Inside first handler (should NOT be printed)\n");
+	    }
+	    print("At statement after 1st try-except\n");
+	    __try {
+                func();
+                print("This should NOT be printed1\n");
+	    } __except (EXCEPTION_CONTINUE_SEARCH) {
+		print("This should NOT be printed2\n");
+	    }
+	}
+	__finally {
+	    print("Finally!\n");
+	}
+	print("At statement after 2nd try-finally (should NOT be printed)\n");
+    }
+    __except (
+	      exception = *(GetExceptionInformation())->ExceptionRecord,
+	      context = (GetExceptionInformation())->ContextRecord,
+	      (GetExceptionCode() == EXCEPTION_ACCESS_VIOLATION) ?
+	      EXCEPTION_EXECUTE_HANDLER : EXCEPTION_CONTINUE_SEARCH) {
+	print("Caught my own memory access violation, ignoring it!\n");
+        if (exception.ExceptionAddress == (void *)exception_location &&
+            exception.ExceptionInformation[1] == *target_addr_location &&
+            exception.ExceptionInformation[0] == 1) {
+            print("Exception address and target match!\n");
+            if (reg_eax == context->CXT_XAX && reg_ebx == context->CXT_XBX &&
+                reg_ecx == context->CXT_XCX && reg_edx == context->CXT_XDX &&
+                reg_esi == context->CXT_XSI && reg_edi == context->CXT_XDI &&
+                reg_esp == context->CXT_XSP && reg_ebp == context->CXT_XBP &&
+                reg_eflags == (context->EFlags & PUSHF_MASK)) {
+                print("Register match!\n");
+            } else {
+                print("Register mismatch!");
+            }
+        } else {
+            print("PC "PFX" (expected "PFX") tried to %s address "PFX" (expected "PFX")\n",
+                   exception.ExceptionAddress, exception_location,
+                   (exception.ExceptionInformation[0]==0)?"read":"write",
+                   exception.ExceptionInformation[1], *target_addr_location);
+        }
+    }
+    print("After exception handler\n");
+	
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
+
+int
+main()
+{
+    uint default_target = DEFAULT_TARGET_ADDR;
+    do_run(fault, &default_target);
+    protect_mem(fault_selfmod, PAGE_SIZE, ALLOW_READ|ALLOW_WRITE|ALLOW_EXEC);
+    do_run(fault_selfmod, &default_target);
+    /* reprotect page, faulting address/target is now the selfmod write */
+    protect_mem(fault_selfmod, PAGE_SIZE, ALLOW_READ|ALLOW_EXEC);
+    do_run(fault_selfmod, &target_addr);
+    return 0;
+}
diff --git a/suite/tests/win32/except.template b/suite/tests/win32/except.template
new file mode 100755
index 000000000..62f07d46c
--- /dev/null
+++ b/suite/tests/win32/except.template
@@ -0,0 +1,50 @@
+At statement after exception
+At statement after 1st try-except
+Finally!
+Caught my own memory access violation, ignoring it!
+Exception address and target match!
+Register match!
+After exception handler
+#if +PROGRAM_SHEPHERDING : +-security --executable_heap +-no_executable_if_hook --executable_if_text --executable_if_x --X
+# if : --detect_mode
+#  if : +-throw_exception
+#   SEC_VIO_EXCEPTION
+//  the exception handler is also bad (on same page as faulting function)
+#   if : --throw_exception_max 1
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#    SEC_VIO_EXCEPTION
+#   endif
+#  endif
+#  if : +-kill_thread
+#    SEC_VIO_THREAD
+#  else  
+#    SEC_VIO_STOP
+#  endif
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+At statement after exception
+At statement after 1st try-except
+Finally!
+Caught my own memory access violation, ignoring it!
+Exception address and target match!
+Register match!
+After exception handler
+At statement after exception
+At statement after 1st try-except
+Finally!
+Caught my own memory access violation, ignoring it!
+Exception address and target match!
+Register match!
+After exception handler
diff --git a/suite/tests/win32/fiber-rac.c b/suite/tests/win32/fiber-rac.c
new file mode 100644
index 000000000..5050f8d4e
--- /dev/null
+++ b/suite/tests/win32/fiber-rac.c
@@ -0,0 +1,101 @@
+/* **********************************************************
+ * Copyright (c) 2004 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define _WIN32_WINNT 0x0400
+
+#include <windows.h>
+#include "tools.h"
+
+/* case 1543 - Fibers on Win2003 RAC false positive */
+
+typedef struct {
+   LPVOID pFiberUI;
+} FIBERINFO, *PFIBERINFO;
+
+FIBERINFO g_FiberInfo;
+
+void WINAPI FiberFunc(PVOID pvParam) {
+
+   PFIBERINFO pFiberInfo = (PFIBERINFO) pvParam;
+
+   print("in worker fiber\n");
+   GetCurrentFiber();
+   if (GetFiberData() != pvParam) {
+       print("GetFiberData() mismatch!\n");
+       abort();
+   }
+
+   print("back to main\n");
+   SwitchToFiber(pFiberInfo->pFiberUI);
+
+   print("in worker fiber again\n");
+
+   // Reschedule the UI thread. When the UI thread is running
+   // and has no events to process, the thread is put to sleep.
+   // NOTE: If we just allow the fiber function to return,
+   // the thread and the UI fiber die -- we don't want this!
+   SwitchToFiber(pFiberInfo->pFiberUI);
+   print("SHOULD NOT GET HERE!\n");
+
+   /* map user32.dll for RunAll testing */
+   MessageBeep(0);
+}
+
+int
+main()
+{
+    int i;
+    PVOID pFiberCounter = NULL;   
+    INIT();
+
+    print("in main thread\n");
+
+    g_FiberInfo.pFiberUI = ConvertThreadToFiber(NULL);
+
+    print("main thread converted to fiber\n");
+
+    for(i = 0; i < 2; i++) {
+        print("creating worker fiber %d\n", i);
+        pFiberCounter = CreateFiber(0, FiberFunc, &g_FiberInfo);
+
+        print("switching to worker fiber first time\n");
+        SwitchToFiber(pFiberCounter);
+
+        print("switching to worker fiber second time\n");
+        SwitchToFiber(pFiberCounter);
+
+        print("deleting worker fiber %d\n", i);
+        DeleteFiber(pFiberCounter);
+    }
+
+    print("all done\n");
+}
diff --git a/suite/tests/win32/fiber-rac.expect b/suite/tests/win32/fiber-rac.expect
new file mode 100644
index 000000000..0f2904cc1
--- /dev/null
+++ b/suite/tests/win32/fiber-rac.expect
@@ -0,0 +1,17 @@
+in main thread
+main thread converted to fiber
+creating worker fiber 0
+switching to worker fiber first time
+in worker fiber
+back to main
+switching to worker fiber second time
+in worker fiber again
+deleting worker fiber 0
+creating worker fiber 1
+switching to worker fiber first time
+in worker fiber
+back to main
+switching to worker fiber second time
+in worker fiber again
+deleting worker fiber 1
+all done
diff --git a/suite/tests/win32/finally.c b/suite/tests/win32/finally.c
new file mode 100644
index 000000000..b8fb894a0
--- /dev/null
+++ b/suite/tests/win32/finally.c
@@ -0,0 +1,89 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include <setjmp.h>
+jmp_buf mark;              /* Address for long jump to jump to */
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+void
+finally_proc() 
+{
+    __try {
+	__try {
+	    printf("This should be printed\n");
+	    __leave;
+	    printf("This should NOT be printed\n");
+	}
+	__finally {
+	    printf("Inside first finally\n");
+	}
+	printf("At statement after 1st try-finally\n");
+    }
+    __finally {
+	printf("Inside second finally\n");
+	longjmp(mark, 1);
+	printf("This should NOT be printed\n");
+    }
+}
+
+int
+main()
+{
+    EXCEPTION_RECORD exception;
+    CONTEXT context;
+    int jmpret;
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    __try {
+	jmpret = setjmp(mark);
+	if (jmpret == 0)
+	    finally_proc();
+	else
+	    printf("done with longjmp\n");
+    }
+    __finally {
+	printf("In final finally\n");
+    }
+	
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
diff --git a/suite/tests/win32/finally.expect b/suite/tests/win32/finally.expect
new file mode 100644
index 000000000..60b1dc0d3
--- /dev/null
+++ b/suite/tests/win32/finally.expect
@@ -0,0 +1,6 @@
+This should be printed
+Inside first finally
+At statement after 1st try-finally
+Inside second finally
+done with longjmp
+In final finally
diff --git a/suite/tests/win32/fpe.c b/suite/tests/win32/fpe.c
new file mode 100644
index 000000000..54238c7c3
--- /dev/null
+++ b/suite/tests/win32/fpe.c
@@ -0,0 +1,139 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* This is adapted from MSDN help on 'longjmp' */
+
+#include <stdio.h>
+#include <signal.h>
+#include <setjmp.h>
+#include <stdlib.h>
+#include <float.h>
+#include <math.h>
+#include <string.h>
+
+jmp_buf mark;              /* Address for long jump to jump to */
+int     fperr;             /* Global error number */
+
+void __cdecl fphandler(int sig, int num);   /* Prototypes */
+void fpcheck(void);
+
+void test(double n1, double n2)
+{
+    double r;
+    int jmpret;
+    /* Save stack environment for return in case of error. First 
+     * time through, jmpret is 0, so true conditional is executed. 
+     * If an error occurs, jmpret will be set to -1 and false 
+     * conditional will be executed.
+     */
+    jmpret = setjmp(mark);
+    if (jmpret == 0) {
+	r = n1 / n2;
+	/* This won't be reached if error occurs. */
+	printf("\n\n%4.3g / %4.3g = %4.3g\n", n1, n2, r);
+	r = n1 * n2;
+	/* This won't be reached if error occurs. */
+	printf("\n\n%4.3g * %4.3g = %4.3g\n", n1, n2, r);
+    }
+    else
+	fpcheck();
+}
+
+int main()
+{
+    /* Unmask all floating-point exceptions. */
+    _control87(0, _MCW_EM);
+    /* Set up floating-point error handler. The compiler
+     * will generate a warning because it expects
+     * signal-handling functions to take only one argument.
+     */
+    if (signal(SIGFPE, fphandler) == SIG_ERR) {
+	fprintf(stderr, "Couldn't set SIGFPE\n");
+	abort();   
+    }
+    test(4., 0.);
+    test(0., DBL_MAX);
+    test(DBL_MAX, DBL_MAX);
+    return 0;
+}
+
+/* fphandler handles SIGFPE (floating-point error) interrupt. Note
+ * that this prototype accepts two arguments and that the 
+ * prototype for signal in the run-time library expects a signal 
+ * handler to have only one argument.
+ *
+ * The second argument in this signal handler allows processing of
+ * _FPE_INVALID, _FPE_OVERFLOW, _FPE_UNDERFLOW, and 
+ * _FPE_ZERODIVIDE, all of which are Microsoft-specific symbols 
+ * that augment the information provided by SIGFPE. The compiler 
+ * will generate a warning, which is harmless and expected.
+
+ */
+void fphandler(int sig, int num)
+{
+    /* Set global for outside check since we don't want
+     * to do I/O in the handler.
+    */
+    fperr = num;
+    /* Initialize floating-point package. */
+    _fpreset();
+    /* Restore calling environment and jump back to setjmp. Return 
+    * -1 so that setjmp will return false for conditional test.
+    */
+    printf("about to do longjmp\n");
+    fflush(stdout);
+    longjmp(mark, -1);
+}
+
+void fpcheck(void)
+{
+    char fpstr[30];
+    switch(fperr) {
+    case _FPE_INVALID:
+	strcpy(fpstr, "Invalid number");
+	break;
+    case _FPE_OVERFLOW:
+	strcpy(fpstr, "Overflow");
+	break;
+    case _FPE_UNDERFLOW:
+	strcpy(fpstr, "Underflow");
+	break;
+    case _FPE_ZERODIVIDE:
+	strcpy(fpstr, "Divide by zero");
+	break;
+    default:
+	strcpy(fpstr, "Other floating point error");
+	break;
+    }
+    printf("Error %d: %s\n", fperr, fpstr);
+}
+
diff --git a/suite/tests/win32/fpe.expect b/suite/tests/win32/fpe.expect
new file mode 100644
index 000000000..6eeaa5088
--- /dev/null
+++ b/suite/tests/win32/fpe.expect
@@ -0,0 +1,14 @@
+about to do longjmp
+Error 131: Divide by zero
+
+
+   0 / 1.8e+308 =    0
+
+
+   0 * 1.8e+308 =    0
+
+
+1.8e+308 / 1.8e+308 =    1
+
+
+1.8e+308 * 1.8e+308 = 1.#J
diff --git a/suite/tests/win32/getthreadcontext.c b/suite/tests/win32/getthreadcontext.c
new file mode 100644
index 000000000..8c18def05
--- /dev/null
+++ b/suite/tests/win32/getthreadcontext.c
@@ -0,0 +1,126 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "tools.h"
+#include <windows.h>
+#include <assert.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+DWORD control;
+DWORD start_addr;
+
+static DWORD WINAPI
+ThreadProc1(LPVOID parm) 
+{
+    print("starting thread...\n");
+    /* asm so we have compiler-independent control over pc range here */
+    __asm {
+        mov  eax, 0xdeadbeef
+        call next_instr
+      next_instr:
+        pop  edx
+        mov  start_addr, edx
+      wait4control:
+        cmp  control, 0
+        je   wait4control
+    }
+    print("exiting thread\n");
+}
+
+static int 
+main(void) 
+{
+    HANDLE ht;
+    DWORD tid;
+    CONTEXT tc;
+    
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    control = 0;
+    start_addr = 0;
+
+    ht = CreateThread(NULL, 0, &ThreadProc1, NULL, 0, &tid);
+
+    while (start_addr == 0)
+        ; /* wait for thread to set start_addr */
+ 
+    SuspendThread(ht);
+
+    memset(&tc, 0, sizeof(tc));
+    tc.ContextFlags = CONTEXT_CONTROL;
+    GetThreadContext(ht, &tc);
+    assert(tc.ContextFlags == CONTEXT_CONTROL);
+    /* 16 bytes is through je wait4control */
+    if (tc.CXT_XIP >= start_addr && tc.CXT_XIP <= start_addr + 0x10)
+        print("eip is valid\n");
+    else
+        print("invalid eip: %p\n", (void *)tc.CXT_XIP);
+
+    /* test non-CONTROL flags */
+    memset(&tc, 0, sizeof(tc));
+    tc.ContextFlags = CONTEXT_INTEGER;
+    GetThreadContext(ht, &tc);
+    assert(tc.ContextFlags == CONTEXT_INTEGER);
+    if (tc.CXT_XAX == 0xdeadbeef)
+        print("eax is valid\n");
+    else
+        print("invalid eax: %p\n", (void *)tc.CXT_XAX);
+    
+    control = 1; /* stop thread */
+    ResumeThread(ht);
+    WaitForSingleObject(ht, INFINITE);
+
+    /* try getting own context (technically you're not supposed to do this, msdn says
+     * result is undefined, but we should handle it reasonably */
+    tc.ContextFlags = CONTEXT_FULL;
+    GetThreadContext(GetCurrentThread(), &tc);
+    /* FIXME - ok we got some result, what can we do to check it? we shouldn't screw up
+     * the reg state, so ignore that, pc should be the post syscall pc (based on
+     * observations of native behavior) so either in ntdll.dll or vsyscall page */
+    print("get context self eip > 0x74000000? %s\n",
+          tc.CXT_XIP > 0x74000000 ? "yes" : "no");
+    
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+
+    print("done");
+
+    return 0;
+}
diff --git a/suite/tests/win32/getthreadcontext.expect b/suite/tests/win32/getthreadcontext.expect
new file mode 100644
index 000000000..546a484f4
--- /dev/null
+++ b/suite/tests/win32/getthreadcontext.expect
@@ -0,0 +1,6 @@
+starting thread...
+eip is valid
+eax is valid
+exiting thread
+get context self eip > 0x74000000? yes
+done
\ No newline at end of file
diff --git a/suite/tests/win32/hooker-secur32.c b/suite/tests/win32/hooker-secur32.c
new file mode 100644
index 000000000..03b69c888
--- /dev/null
+++ b/suite/tests/win32/hooker-secur32.c
@@ -0,0 +1,242 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* case 6423 - citrix function patching hooker for secur32.dll
+ * - we'll test here by hooking kernel32.dll and secur32.dll (LoadLibrary)
+ */
+
+/* mostly COPIED from hooker-ntdll.c */
+
+/* one should use a CALL and the other should use a JMP 
+*   just to be sure
+* 
+*  FIXME: need to get this test also to be done like initapc.dll.c so that 
+*  this all happens BEFORE we take control
+**/
+
+#include "tools.h"
+#include <windows.h>
+
+typedef void (*fptr)();
+
+#define NAKED __declspec( naked ) 
+
+/* check for some unexpected behaviours with size = 5 and size = 0x1000, or even 0x2000 */
+enum {HOOK_SIZE = 0x1000};
+
+/*
+ * NTSTATUS LdrLoadDll(LPCWSTR path_name, DWORD flags, 
+ *                     PUNICODE_STRING libname, HMODULE* hModule)
+*/
+
+NAKED
+hooker4()
+{
+    __asm {
+        pusha
+        pushf
+    }
+    /* here we can do whatever we need to */
+
+    __asm {
+        popf
+        popa
+    }
+
+    /* know the number of args */
+    __asm {
+        ret 4*4;
+    }
+}
+
+NAKED
+hooker5()
+{
+    __asm {
+        pusha
+        pushf
+    }
+    /* here we can do whatever we need to */
+
+    __asm {
+        popf
+        popa
+    }
+
+    /* know the number of args */
+    __asm {
+        ret 5*4;
+    }
+}
+
+void
+do_hook(const char* hook_dll, const char* hookfn, int args, int use_call)
+{
+    HANDLE target_dll = GetModuleHandle(hook_dll);
+    unsigned char* hooktarget = (char*)GetProcAddress(target_dll, hookfn);
+
+    DWORD old_code1;
+    DWORD old_code2;
+
+    DWORD size = HOOK_SIZE; /* although 5 bytes would have been just fine */
+    DWORD prev = 0xbadcde;
+    int res;
+
+    DWORD pc_rel_target;
+
+    enum {op_JMP  = 0xe9};
+    enum {op_CALL = 0xe8};
+
+    void* trampoline_target;
+
+    if (use_call)
+        args += 1;                /* clean up addr of CALL */
+
+    if (hooktarget == NULL) {
+        print("BAD: couldn't find %s!%s\n", hook_dll, hookfn);
+    }
+
+    old_code1 = *(DWORD*)hooktarget;
+    old_code2 = *(DWORD*)(hooktarget+1);
+
+    switch (args) {
+    case 4: 
+        trampoline_target = hooker4; break;
+    case 5: 
+        trampoline_target = hooker5; break;
+    default:
+        print("BAD args\n");
+    }
+
+    __try {
+        *hooktarget = 0xbaad;
+        print("bad: why is this writable?\n");
+    }
+    __except (EXCEPTION_EXECUTE_HANDLER) {
+        print("ok: can't write\n");
+    }
+    
+    res = VirtualProtect(hooktarget, 
+                         size,
+                         PAGE_EXECUTE_READWRITE,
+                         &prev);
+    print("VirtualProtect(%s!%s["PFX"],%d,PAGE_EXECUTE_READWRITE,prev) = %d GLE="PFMT" prev="PFMT"\n", 
+          hook_dll, 
+          hookfn, (hooktarget /* disabled */,0), size, (res /* eax noisy */, 0) , GetLastError(), prev);
+
+#ifdef VERBOSE
+    print("before hooking %s = %02x %02x %02x %02x %02x\n", 
+          hookfn, 
+          hooktarget[0], hooktarget[1], hooktarget[2], hooktarget[3], hooktarget[4]);
+#endif
+
+    pc_rel_target =  (DWORD)trampoline_target - ((DWORD)hooktarget + 5); /* target = offset + cur pc + size*/ 
+
+    __try {
+        *hooktarget = use_call ? op_CALL : op_JMP;
+        *(DWORD*)(hooktarget+1) = pc_rel_target;
+
+        /* now let's get smart here and see if hook worked */
+#ifdef VERBOSE
+        print("after  hooking %s = %02x %02x %02x %02x %02x\n", 
+              hookfn, 
+              hooktarget[0], hooktarget[1], hooktarget[2], hooktarget[3], hooktarget[4]);
+#endif
+        if (*(DWORD*)(hooktarget+1) != pc_rel_target)
+            print("there be witches! what happened to my write?\n");
+        else
+            print("hooked %s\n", hookfn);
+        /* FIXME: try it out and see what happens */
+
+        /* restore */
+        *(DWORD*)hooktarget     = old_code1;
+        *(DWORD*)(hooktarget+1) = old_code2;
+
+        print("restored old code\n");
+    }
+    __except (EXCEPTION_EXECUTE_HANDLER) {
+        print("bad: can't write, though made writable\n");
+    }
+
+
+    /* restore page permissoins now, could be optional */
+    res = VirtualProtect(hooktarget, 
+                         size,
+                         PAGE_EXECUTE_READ,
+                         &prev);
+    print("VirtualProtect(%s["PFX"],%d,PAGE_EXECUTE_READ,...) = %d GLE="PFMT" \n", 
+          hookfn, (hooktarget /* disabled */,0), size, (res /* eax noisy */, 0) , GetLastError());
+    print("old permissions ...prev="PFMT")\n", prev);
+
+    __try {
+        *hooktarget = 0xbaad;
+        print("bad: why is this writable?\n");
+    }
+    __except (EXCEPTION_EXECUTE_HANDLER) {
+        print("ok: can't write\n");
+    }
+
+    print("all should be good\n");
+}
+
+int
+main()
+{
+    INIT();
+    
+    print("ready to hook\n");
+
+    /* FIXME: not intended to be called */
+    do_hook("kernel32.dll", "GetProcessHeaps", 4, 1);
+    /* FIXME case 10012: re-enable once we can handle it being on
+     * same page as _SEH_prolog on xpsp2 machines.
+     * Don't forget to update the proc name in .template as well.
+     */
+#if 0
+    do_hook("kernel32.dll", "Sleep", 4, 0);
+#else
+    do_hook("kernel32.dll", "VirtualQueryEx", 4, 0);
+#endif
+
+    print("loading secur32\n");
+    LoadLibrary("secur32.dll");
+
+    do_hook("secur32.dll", "LsaLogonUser", 4, 0);
+    do_hook("secur32.dll", "MakeSignature", 4, 1);
+
+    /* we have 4 writes to module memory 
+     * on each of 4 calls to do_hook
+     * should get app_modify_pretend_writes = 24
+     * FIXME: how to scrape a log for this?
+     */
+    print("hooking done with\n");
+}
diff --git a/suite/tests/win32/hooker-secur32.template b/suite/tests/win32/hooker-secur32.template
new file mode 100644
index 000000000..5e5a7a74b
--- /dev/null
+++ b/suite/tests/win32/hooker-secur32.template
@@ -0,0 +1,100 @@
+ready to hook
+ok: can't write
+VirtualProtect(kernel32.dll!GetProcessHeaps[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+//FIXME * is expanded to regexp in gen_expect.pl
+#if +PROGRAM_SHEPHERDING : --patch_proof_list(*=ALL) --patch_proof_list(kernel32) --patch_proof_default_list(kernel32)
+hooked GetProcessHeaps
+#else
+#if -PROGRAM_SHEPHERDING
+hooked GetProcessHeaps
+#else
+there be witches! what happened to my write?
+#endif
+#endif
+restored old code
+VirtualProtect(GetProcessHeaps[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+#if +PROGRAM_SHEPHERDING : --patch_proof_list(*=ALL) --patch_proof_list(kernel32) --patch_proof_default_list(kernel32)
+old permissions ...prev=00000040)
+#else
+#if -PROGRAM_SHEPHERDING
+old permissions ...prev=00000040)
+#else
+old permissions ...prev=00000020)
+#endif
+#endif
+ok: can't write
+all should be good
+ok: can't write
+VirtualProtect(kernel32.dll!VirtualQueryEx[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+#if +PROGRAM_SHEPHERDING : --patch_proof_list(*=ALL) --patch_proof_list(kernel32) --patch_proof_default_list(kernel32)
+hooked VirtualQueryEx
+#else
+#if -PROGRAM_SHEPHERDING
+hooked VirtualQueryEx
+#else
+there be witches! what happened to my write?
+#endif
+#endif
+restored old code
+VirtualProtect(VirtualQueryEx[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+#if +PROGRAM_SHEPHERDING : --patch_proof_list(*=ALL) --patch_proof_list(kernel32) --patch_proof_default_list(kernel32)
+old permissions ...prev=00000040)
+#else
+#if -PROGRAM_SHEPHERDING
+old permissions ...prev=00000040)
+#else
+old permissions ...prev=00000020)
+#endif
+#endif
+ok: can't write
+all should be good
+loading secur32
+ok: can't write
+VirtualProtect(secur32.dll!LsaLogonUser[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+#if +PROGRAM_SHEPHERDING : --patch_proof_list(*=ALL) --patch_proof_list(secur32) --patch_proof_default_list(secur32)
+hooked LsaLogonUser
+#else
+#if -PROGRAM_SHEPHERDING
+hooked LsaLogonUser
+#else
+there be witches! what happened to my write?
+#endif
+#endif
+restored old code
+VirtualProtect(LsaLogonUser[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+#if +PROGRAM_SHEPHERDING : --patch_proof_list(*=ALL) --patch_proof_list(secur32) --patch_proof_default_list(secur32)
+old permissions ...prev=00000040)
+#else
+#if -PROGRAM_SHEPHERDING
+old permissions ...prev=00000040)
+#else
+old permissions ...prev=00000020)
+#endif
+#endif
+ok: can't write
+all should be good
+ok: can't write
+VirtualProtect(secur32.dll!MakeSignature[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+#if +PROGRAM_SHEPHERDING : --patch_proof_list(*=ALL) --patch_proof_list(secur32) --patch_proof_default_list(secur32)
+hooked MakeSignature
+#else
+#if -PROGRAM_SHEPHERDING
+hooked MakeSignature
+#else
+there be witches! what happened to my write?
+#endif
+#endif
+restored old code
+VirtualProtect(MakeSignature[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+#if +PROGRAM_SHEPHERDING : --patch_proof_list(*=ALL) --patch_proof_list(secur32) --patch_proof_default_list(secur32)
+old permissions ...prev=00000040)
+#else
+#if -PROGRAM_SHEPHERDING
+old permissions ...prev=00000040)
+#else
+old permissions ...prev=00000020)
+#endif
+#endif
+ok: can't write
+all should be good
+hooking done with
diff --git a/suite/tests/win32/hookerfirst.c b/suite/tests/win32/hookerfirst.c
new file mode 100644
index 000000000..fdd25472a
--- /dev/null
+++ b/suite/tests/win32/hookerfirst.c
@@ -0,0 +1,107 @@
+/* **********************************************************
+ * Copyright (c) 2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* hookerfirst: tests having DllMain of a statically-linked dll 
+ * (thus prior to the image entry point) hook ntdll.dll
+ * and then unhook in main() of the executable
+ *
+ * must be run w/ drinject injection, else we'd have to make sure
+ * user32 is loaded after the DLL.
+ *
+ * case 2525
+ *3) Ent hooks first
+ *  We hook -- need to chain -- mangle their call
+ *  Ent unhooks (dynamic off) -- need to unchain by emulating their write and then restoring
+ *  they could come back -- then follow 1)
+ *
+ *4) Ent hooks first
+ *  We hook -- need to chain
+ *  we unhook (detach) -- need to unchain - unmangle their call
+ *  ent unhooks -- should be fine
+ *
+ */
+#include <windows.h>
+#include "tools.h"
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+/* from hookerfirst.dll */
+__declspec(dllimport)
+hookit(int x);
+
+__declspec(dllimport)
+unhookit(int x);
+
+int
+badfunc(void)
+{
+    __asm {
+        push offset next_instr
+        nop
+        ret
+    next_instr:
+        nop
+    }
+}
+
+int
+main()
+{
+    INIT();
+
+    print("hookerfirst main()\n");
+    /* TODO: at this point we may want to detach in some cases - so
+     * should figure out how to trigger detach on error here.
+     */
+
+    /* this is for testing with -internal_detach 0x2 -- we'd need to detach ourselves cleanly */
+
+    /* ensure still checking ret-after-call
+     * use a nop between push and ret to avoid VB push/ret pattern match
+     */
+    __try {
+        badfunc();
+        print("*** invalid ret allowed!\n");
+    } __except (EXCEPTION_EXECUTE_HANDLER) {
+        print("invalid ret caught\n");
+    }
+
+    /* this is tested with -no_ret_after_call */
+    unhookit(37);
+
+    hookit(37);
+    unhookit(37);
+
+    return 0;
+}
diff --git a/suite/tests/win32/hookerfirst.dll.c b/suite/tests/win32/hookerfirst.dll.c
new file mode 100644
index 000000000..63168cce2
--- /dev/null
+++ b/suite/tests/win32/hookerfirst.dll.c
@@ -0,0 +1,330 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* case 2525 - hooker that takes over first - with a static library doing the work */
+
+/* one should use a CALL and the other should use a JMP 
+*   just to be sure
+* 
+*  FIXME: need to get this test also to be done like initapc.dll.c so that 
+*  this all happens BEFORE we take control
+**/
+
+/* FIXME: we can't use a LdrLoadDll unless we chain properly - otherwise we don't get in! */
+
+#include "tools.h"
+#include <windows.h>
+
+typedef void (*fptr)();
+
+#define NAKED __declspec( naked ) 
+
+/* FIXME: check for some unexpected behaviours with size = 5 and size = 0x1000, or even 0x2000 */
+enum {HOOK_SIZE = 0x1000};
+
+
+NAKED
+hooker4()
+{
+    __asm {
+        pusha
+        pushf
+    }
+    /* here we can do whatever we need to */
+
+    __asm {
+        popf
+        popa
+    }
+
+    /* know the number of args */
+    __asm {
+        ret 4*4;
+    }
+}
+
+NAKED
+hooker5()
+{
+    __asm {
+        pusha
+        pushf
+    }
+    /* here we can do whatever we need to */
+
+    __asm {
+        popf
+        popa
+    }
+
+    /* know the number of args */
+    __asm {
+        ret 5*4;
+    }
+}
+
+void
+do_hook(const char* hookfn, int args, int use_call,
+        DWORD *old_code)
+{
+    HANDLE ntdll = GetModuleHandle(TEXT("ntdll.DLL"));
+    unsigned char* hooktarget = (char*)GetProcAddress(ntdll, hookfn);
+
+    DWORD size = HOOK_SIZE; /* although 5 bytes would have been just fine */
+    DWORD prev = 0xbadcde;
+    int res;
+
+    enum {op_JMP  = 0xe9};
+    enum {op_CALL = 0xe8};
+
+    DWORD pc_rel_target;
+
+    void* trampoline_target;
+
+    *old_code = *(DWORD*)hooktarget;
+    *(old_code+1) = *(DWORD*)(hooktarget+1);
+
+    if (use_call)
+        args += 1;                /* clean up addr of CALL */
+
+    switch (args) {
+    case 4: 
+        trampoline_target = hooker4; break;
+    case 5: 
+        trampoline_target = hooker5; break;
+    default:
+        print("BAD args\n");
+    }
+
+    __try {
+        *hooktarget = 0xbaad;
+        print("bad: why is this writable?\n");
+    }
+    __except (EXCEPTION_EXECUTE_HANDLER) {
+        print("ok: can't write\n");
+    }
+    
+    res = VirtualProtect(hooktarget, 
+                         size,
+                         PAGE_EXECUTE_READWRITE,
+                         &prev);
+    print("VirtualProtect(%s["PFX"],%d,PAGE_EXECUTE_READWRITE,prev) = %d GLE="PFMT" prev="PFMT"\n", 
+          hookfn, (hooktarget /* disabled */,0), size, (res /* eax noisy */, 0) , GetLastError(), prev);
+
+#ifdef VERBOSE
+    print("before hooking %s = %02x %02x %02x %02x %02x\n", 
+          hookfn, 
+          hooktarget[0], hooktarget[1], hooktarget[2], hooktarget[3], hooktarget[4]);
+#endif
+
+    pc_rel_target =  (DWORD)trampoline_target - ((DWORD)hooktarget + 5); /* target = offset + cur pc + size*/ 
+
+    __try {
+        *hooktarget = use_call ? op_CALL : op_JMP;
+        *(DWORD*)(hooktarget+1) = pc_rel_target;
+
+        /* now let's get smart here and see if hook worked */
+#ifdef VERBOSE
+        print("after  hooking %s = %02x %02x %02x %02x %02x\n", 
+              hookfn, 
+              hooktarget[0], hooktarget[1], hooktarget[2], hooktarget[3], hooktarget[4]);
+#endif
+        if (*(DWORD*)(hooktarget+1) != pc_rel_target)
+            print("there be witches! what happened to my write?\n");
+        else
+            print("hooked %s\n", hookfn);
+        /* FIXME: try it out and see what happens */
+    }
+    __except (EXCEPTION_EXECUTE_HANDLER) {
+        print("bad: can't write, though made writable\n");
+    }
+
+    /* restore page permissoins now, could be optional */
+    res = VirtualProtect(hooktarget, 
+                         size,
+                         PAGE_EXECUTE_READ,
+                         &prev);
+    print("VirtualProtect(%s["PFX"],%d,PAGE_EXECUTE_READ,...) = %d GLE="PFMT" \n", 
+          hookfn, (hooktarget /* disabled */,0), size, (res /* eax noisy */, 0) , GetLastError());
+    print("old permissions ...prev="PFMT")\n", prev);
+
+    __try {
+        *hooktarget = 0xbaad;
+        print("bad: why is this writable?\n");
+    }
+    __except (EXCEPTION_EXECUTE_HANDLER) {
+        print("ok: can't write\n");
+    }
+
+    /* save hook code */
+    *(old_code+2) = *(DWORD*)hooktarget;
+    *(old_code+3) = *(DWORD*)(hooktarget+1);
+
+    print("do_hook() done\n");
+}
+
+
+void
+do_unhook(const char* hookfn, int args, int use_call,
+          DWORD *old_code)
+{
+    HANDLE ntdll = GetModuleHandle(TEXT("ntdll.DLL"));
+    unsigned char* hooktarget = (char*)GetProcAddress(ntdll, hookfn);
+
+    DWORD size = HOOK_SIZE; /* although 5 bytes would have been just fine */
+    DWORD prev = 0xbadcde;
+    int res;
+
+
+    res = VirtualProtect(hooktarget, 
+                         size,
+                         PAGE_EXECUTE_READWRITE,
+                         &prev);
+    print("VirtualProtect(%s["PFX"],%d,PAGE_EXECUTE_READWRITE,prev) = %d GLE="PFMT" prev="PFMT"\n", 
+          hookfn, (hooktarget /* disabled */,0), size, (res /* eax noisy */, 0) , GetLastError(), prev);
+
+    __try {
+        /* verify */
+        if (*(DWORD*)hooktarget != *(old_code+2) /* in fact new code */
+            || *(DWORD*)(hooktarget+1) != *(old_code+3))
+            {
+                print("there be witches! what happened to my previous hook?\n");
+            } else 
+                print("my hook is still there, will remove now\n");
+
+        /* restore */
+        *(DWORD*)hooktarget     = *old_code;
+        *(DWORD*)(hooktarget+1) = *(old_code+1);
+
+        if (*(DWORD*)(hooktarget+1) != *(old_code+1))
+            print("there be witches! my good unhooking intentions were squashed on %s\n", hookfn);
+        else
+            print("unhooked %s\n", hookfn);
+
+        print("restored old code\n");
+    }
+    __except (EXCEPTION_EXECUTE_HANDLER) {
+        print("bad: can't write, though made writable\n");
+    }
+
+
+    /* restore page permissoins now, could be optional */
+    res = VirtualProtect(hooktarget, 
+                         size,
+                         PAGE_EXECUTE_READ,
+                         &prev);
+    print("VirtualProtect(%s["PFX"],%d,PAGE_EXECUTE_READ,...) = %d GLE="PFMT" \n", 
+          hookfn, (hooktarget /* disabled */,0), size, (res /* eax noisy */, 0) , GetLastError());
+    print("old permissions ...prev="PFMT")\n", prev);
+
+    __try {
+        *hooktarget = 0xbaad;
+        print("bad: why is this writable?\n");
+    }
+    __except (EXCEPTION_EXECUTE_HANDLER) {
+        print("ok: can't write\n");
+    }
+
+    print("do_unhook() done\n");
+}
+
+/* having a lot of trouble with LdrUnloadDll or NtFlushInstructionCache for now trying these */
+#define FUNC1 "NtTerminateProcess"
+#define FUNC2 "NtTerminateThread"
+
+#define BUF_SIZE 2+2            /* old + new code */
+DWORD func1_buf[BUF_SIZE];
+DWORD func2_buf[BUF_SIZE];
+DWORD ntflushwritebuffer_buf[BUF_SIZE];
+
+int
+__declspec(dllexport)
+hookit(int arg)
+{
+    print("ready to hook %d\n", arg);
+
+    do_hook(FUNC1, 4, 1, func1_buf);
+
+    /* hack: we'll pass 4 args instead of 3 */
+    do_hook(FUNC2, 4, 0, func2_buf);
+
+    /* hack: we'll pass 4 args instead of 0 */
+    /* hooking a function we really don't care much about */
+    /* FIXME: should we let this through or not? */
+    do_hook("NtFlushWriteBuffer", 4, 1, ntflushwritebuffer_buf);
+
+    /* we have 4 writes to ntdll memory 
+     * on each of 6 calls to do_hook
+     * should get app_modify_ntdll_writes = 24
+     * FIXME: how to scrape a log for this?
+     */
+    print("hooking done with\n");
+    return 0;
+}
+
+int
+__declspec(dllexport)
+unhookit(int arg)
+{
+    print("ready to unhook %d\n", arg);
+
+    do_unhook(FUNC1, 4, 1, func1_buf);
+
+    /* hack: we'll pass 4 args instead of 3 */
+    do_unhook(FUNC2, 4, 0, func2_buf);
+
+    /* hack: we'll pass 4 args instead of 0 */
+    /* hooking a function we really don't care much about */
+    /* FIXME: should we let this through or not? */
+    do_unhook("NtFlushWriteBuffer", 4, 1, ntflushwritebuffer_buf);
+
+    /* we have 4 writes to ntdll memory 
+     * on each of 6 calls to do_hook
+     * should get app_modify_ntdll_writes = 24
+     * FIXME: how to scrape a log for this?
+     */
+    print("unhooking done with\n");
+    return 0;
+}
+
+
+BOOL APIENTRY 
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    switch (reason_for_call) {
+    case DLL_PROCESS_ATTACH:
+        hookit(1);
+
+        break;
+    }
+    return TRUE;
+}
diff --git a/suite/tests/win32/hookerfirst.template b/suite/tests/win32/hookerfirst.template
new file mode 100644
index 000000000..cd2107362
--- /dev/null
+++ b/suite/tests/win32/hookerfirst.template
@@ -0,0 +1,174 @@
+ready to hook 1
+ok: can't write
+VirtualProtect(NtTerminateProcess[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+hooked NtTerminateProcess
+VirtualProtect(NtTerminateProcess[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+old permissions ...prev=00000040)
+ok: can't write
+do_hook() done
+ok: can't write
+VirtualProtect(NtTerminateThread[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+hooked NtTerminateThread
+VirtualProtect(NtTerminateThread[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+old permissions ...prev=00000040)
+ok: can't write
+do_hook() done
+ok: can't write
+VirtualProtect(NtFlushWriteBuffer[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+hooked NtFlushWriteBuffer
+VirtualProtect(NtFlushWriteBuffer[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+old permissions ...prev=00000040)
+ok: can't write
+do_hook() done
+hooking done with
+hookerfirst main()
+#if +PROGRAM_SHEPHERDING : +-security --no_ret_after_call --C --X --low --client --thin_client --use_moduledb
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+*** invalid ret allowed!
+ready to unhook 37
+VirtualProtect(NtTerminateProcess[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+#if +PROGRAM_SHEPHERDING : --hook_conflict --thin_client
+there be witches! what happened to my previous hook?
+#else
+my hook is still there, will remove now
+#endif
+#if +PROGRAM_SHEPHERDING : --hook_conflict --thin_client
+there be witches! my good unhooking intentions were squashed on NtTerminateProcess
+#else
+unhooked NtTerminateProcess
+#endif
+restored old code
+VirtualProtect(NtTerminateProcess[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+#if +PROGRAM_SHEPHERDING : --hook_conflict --thin_client
+old permissions ...prev=00000020)
+#else
+old permissions ...prev=00000040)
+#endif
+ok: can't write
+do_unhook() done
+VirtualProtect(NtTerminateThread[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+#if +PROGRAM_SHEPHERDING : --hook_conflict
+there be witches! what happened to my previous hook?
+#else
+my hook is still there, will remove now
+#endif
+#if +PROGRAM_SHEPHERDING : --hook_conflict --thin_client
+there be witches! my good unhooking intentions were squashed on NtTerminateThread
+#else
+unhooked NtTerminateThread
+#endif
+restored old code
+VirtualProtect(NtTerminateThread[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+#if +PROGRAM_SHEPHERDING : --hook_conflict --thin_client
+old permissions ...prev=00000020)
+#else
+old permissions ...prev=00000040)
+#endif
+ok: can't write
+do_unhook() done
+VirtualProtect(NtFlushWriteBuffer[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+my hook is still there, will remove now
+#if +PROGRAM_SHEPHERDING : --hook_conflict --thin_client
+there be witches! my good unhooking intentions were squashed on NtFlushWriteBuffer
+#else
+unhooked NtFlushWriteBuffer
+#endif
+restored old code
+VirtualProtect(NtFlushWriteBuffer[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+#if +PROGRAM_SHEPHERDING : --hook_conflict --thin_client
+old permissions ...prev=00000020)
+#else
+old permissions ...prev=00000040)
+#endif
+ok: can't write
+do_unhook() done
+unhooking done with
+ready to hook 37
+ok: can't write
+VirtualProtect(NtTerminateProcess[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+#if +PROGRAM_SHEPHERDING : --hook_conflict --thin_client
+there be witches! what happened to my write?
+#else
+hooked NtTerminateProcess
+#endif
+VirtualProtect(NtTerminateProcess[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+#if +PROGRAM_SHEPHERDING : --hook_conflict --thin_client
+old permissions ...prev=00000020)
+#else
+old permissions ...prev=00000040)
+#endif
+ok: can't write
+do_hook() done
+ok: can't write
+VirtualProtect(NtTerminateThread[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+#if +PROGRAM_SHEPHERDING : --hook_conflict --thin_client
+there be witches! what happened to my write?
+#else
+hooked NtTerminateThread
+#endif
+VirtualProtect(NtTerminateThread[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+#if +PROGRAM_SHEPHERDING : --hook_conflict --thin_client
+old permissions ...prev=00000020)
+#else
+old permissions ...prev=00000040)
+#endif
+ok: can't write
+do_hook() done
+ok: can't write
+VirtualProtect(NtFlushWriteBuffer[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+hooked NtFlushWriteBuffer
+VirtualProtect(NtFlushWriteBuffer[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+#if +PROGRAM_SHEPHERDING : --hook_conflict --thin_client
+old permissions ...prev=00000020)
+#else
+old permissions ...prev=00000040)
+#endif
+ok: can't write
+do_hook() done
+hooking done with
+ready to unhook 37
+VirtualProtect(NtTerminateProcess[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+my hook is still there, will remove now
+unhooked NtTerminateProcess
+restored old code
+VirtualProtect(NtTerminateProcess[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+#if +PROGRAM_SHEPHERDING : --hook_conflict --thin_client
+old permissions ...prev=00000020)
+#else
+old permissions ...prev=00000040)
+#endif
+ok: can't write
+do_unhook() done
+VirtualProtect(NtTerminateThread[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+my hook is still there, will remove now
+unhooked NtTerminateThread
+restored old code
+VirtualProtect(NtTerminateThread[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+#if +PROGRAM_SHEPHERDING : --hook_conflict --thin_client
+old permissions ...prev=00000020)
+#else
+old permissions ...prev=00000040)
+#endif
+ok: can't write
+do_unhook() done
+VirtualProtect(NtFlushWriteBuffer[0x00000000],4096,PAGE_EXECUTE_READWRITE,prev) = 0 GLE=00000000 prev=00000020
+my hook is still there, will remove now
+unhooked NtFlushWriteBuffer
+restored old code
+VirtualProtect(NtFlushWriteBuffer[0x00000000],4096,PAGE_EXECUTE_READ,...) = 0 GLE=00000000 
+#if +PROGRAM_SHEPHERDING : --hook_conflict --thin_client
+old permissions ...prev=00000020)
+#else
+old permissions ...prev=00000040)
+#endif
+ok: can't write
+do_unhook() done
+unhooking done with
diff --git a/suite/tests/win32/multisec.c b/suite/tests/win32/multisec.c
new file mode 100644
index 000000000..1b4ece49d
--- /dev/null
+++ b/suite/tests/win32/multisec.c
@@ -0,0 +1,102 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include <stdio.h>
+#include "tools.h"
+
+#define VERBOSE 0
+
+#pragma code_seg(".mycode1")
+void func2() { 
+    print("func2\n");
+}
+
+fptr f2 = &func2;
+
+#pragma code_seg(".my_code2")   /* 2 will be truncated - up to 8 char limit */
+void func3() { 
+    print("exe calling f2\n");
+    (*f2)();
+    print("exe func3\n");
+}
+
+#pragma code_seg(".my_code3") 
+/* interesting - while the PE file will have an 8 byte section limit ".my_code"
+ * this section is still going to be created as different from the .my_code2
+*/
+void func4() {
+    print("exe func4\n");
+}
+#pragma code_seg()                /* back in .text */
+
+const fptr cf = &func3;
+fptr f = &func2;
+
+HMODULE
+myload(char* lib) 
+{
+    HMODULE hm = LoadLibrary(lib);
+    if (hm == NULL) {
+        print("error loading library %s\n", lib);
+    } else {
+        BOOL res;
+        BOOL (WINAPI *proc)(DWORD);
+        print("loaded %s\n", lib);
+#if VERBOSE
+        print("library is at "PFX"\n", hm);
+#endif
+    }
+    return hm;
+}
+
+int main()
+{
+    HMODULE lib1;
+    HMODULE lib2;
+
+    /* same as rebased test */
+    lib1 = myload("multisec.dll.dll");
+    lib2 = myload("multis~1.dll");
+    if (lib1 == lib2) {
+        print("there is a problem - should have collided, maybe missing\n");
+    }
+
+    f();
+    func3();
+    func4();
+
+    FreeLibrary(lib1);
+    FreeLibrary(lib2);
+
+    return 0;
+}
diff --git a/suite/tests/win32/multisec.dll.c b/suite/tests/win32/multisec.dll.c
new file mode 100644
index 000000000..7e8013835
--- /dev/null
+++ b/suite/tests/win32/multisec.dll.c
@@ -0,0 +1,134 @@
+/* **********************************************************
+ * Copyright (c) 2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* case 286 testing multiple code sections:
+ *
+ * Add functions to multiple code segments and have references to them
+ * from other, including data segments just like rebased make sure DLL
+ * and image are conflicting so that DLL goes somewhere else
+ */
+
+#include "tools.h"
+#include <windows.h>
+
+typedef void (*fptr)();
+
+void
+foo() 
+{
+    print("foo\n");
+}
+
+void
+bar() 
+{
+    print("bar\n");
+}
+
+void
+func1() 
+{
+    print("bar\n");
+}
+
+#pragma code_seg(".mycode1")
+void func2() { 
+    print("func2\n");
+}
+
+#pragma data_seg(".data2")
+fptr f2 = &func2;
+
+#pragma data_seg()
+
+/* VC7 has support for push and pop that we can't use in VC6
+ * #pragma code_seg(push, r1, ".mycode2")
+ */
+#pragma code_seg(".mycode2")
+void func3() { 
+    print("calling f2\n");
+    (*f2)();
+    print("func3\n");
+}
+
+#pragma code_seg(".mycode1")      /* back to my_code1 */
+void func4() {
+    print("func4\n");
+}
+#pragma code_seg()                /* back in .text */
+
+const fptr cf = &foo;
+fptr f = &foo;
+
+#pragma data_seg(".data1")
+fptr f4 = &func4;
+
+#pragma data_seg()
+
+void
+dlltest(void) 
+{
+    print("dlltest\n");
+    f();
+    cf();
+    f = &bar;
+    f();
+    cf();
+    func3();
+    f2 = &func4;
+    func3();
+    f4();
+}
+
+
+/* our Makefile expects a .lib */
+int
+__declspec(dllexport)
+data_attack(int arg)
+{
+    print("data_attack\n");
+    /* FIXME: will do this some other time */
+    dlltest();
+    return 1;
+}
+
+BOOL APIENTRY 
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    switch (reason_for_call) {
+    case DLL_PROCESS_ATTACH:
+        dlltest();
+
+        break;
+    }
+    return TRUE;
+}
diff --git a/suite/tests/win32/multisec.template b/suite/tests/win32/multisec.template
new file mode 100644
index 000000000..776d125cc
--- /dev/null
+++ b/suite/tests/win32/multisec.template
@@ -0,0 +1,31 @@
+dlltest
+foo
+foo
+bar
+foo
+calling f2
+func2
+func3
+calling f2
+func4
+func3
+func4
+loaded multisec.dll.dll
+dlltest
+foo
+foo
+bar
+foo
+calling f2
+func2
+func3
+calling f2
+func4
+func3
+func4
+loaded multis~1.dll
+func2
+exe calling f2
+func2
+exe func3
+exe func4
diff --git a/suite/tests/win32/nativeexec.c b/suite/tests/win32/nativeexec.c
new file mode 100644
index 000000000..8bde2e555
--- /dev/null
+++ b/suite/tests/win32/nativeexec.c
@@ -0,0 +1,109 @@
+/* **********************************************************
+ * Copyright (c) 2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* nativeexec.exe that calls routines in nativeexec.dll.dll via 
+ * different call* constructions
+ */
+#include <windows.h>
+#include "tools.h"
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+/* from nativeexec.dll.dll */
+__declspec(dllimport) import_me1(int x);
+__declspec(dllimport) import_me2(int x);
+__declspec(dllimport) import_me3(int x);
+
+int
+main()
+{
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    INIT();
+
+    print("calling via IAT-style call\n");
+    import_me1(57);
+
+    print("calling via PLT-style call\n");
+    __asm {
+        push 37
+        call plt_ind_call
+        jmp after_plt_call
+    plt_ind_call:
+        jmp dword ptr [import_me2]
+    after_plt_call:
+        add esp,4
+    }
+
+    /* funky ind call is only caught by us w/ -native_exec_guess_calls
+     * FIXME: add a -no_native_exec_guess_calls runregression run
+     * for that run:
+     *    FIXME: assert curiosity in debug run, would like to add to template!
+     *    FIXME: have way for nativeexec.dll.c to know whether native or not?
+     *      call DR routine?
+     *      then can have release build die too
+     *
+     *    06:13 PM ~/dr/suite/tests
+     *    % useops -no_native_exec_guess_calls
+     *    % make win32/nativeexec.runinjector
+     *    PASS
+     *    % make DEBUG=yes win32/nativeexec.runinjector
+     *    > <CURIOSITY : false && "inside native_exec dll" in file x86/interp.c line 1967
+     */
+
+    print("calling via funky ind call\n");
+    __asm {
+        push 17
+        call funky_ind_call
+        jmp after_funky_call
+    funky_ind_call:
+        xor eax,eax
+        push eax
+        pop eax
+        jmp dword ptr [import_me3]
+    after_funky_call:
+        add esp,4
+    }
+
+    print("all done\n");
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
diff --git a/suite/tests/win32/nativeexec.dll.c b/suite/tests/win32/nativeexec.dll.c
new file mode 100644
index 000000000..bc6898765
--- /dev/null
+++ b/suite/tests/win32/nativeexec.dll.c
@@ -0,0 +1,60 @@
+/* **********************************************************
+ * Copyright (c) 2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* nativexec.dll.dll
+ * nativeexec.exe calls routines here w/ different call* constructions
+ */
+#include "tools.h"
+
+void __declspec(dllexport)
+import_me1(int x)
+{
+    print("nativeexec.dll:import_me1(%d)\n", x);
+}
+
+void __declspec(dllexport)
+import_me2(int x)
+{
+    print("nativeexec.dll:import_me2(%d)\n", x);
+}
+
+void __declspec(dllexport)
+import_me3(int x)
+{
+    print("nativeexec.dll:import_me3(%d)\n", x);
+}
+
+BOOL APIENTRY 
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    return TRUE;
+}
diff --git a/suite/tests/win32/nativeexec.expect b/suite/tests/win32/nativeexec.expect
new file mode 100755
index 000000000..caf4580d3
--- /dev/null
+++ b/suite/tests/win32/nativeexec.expect
@@ -0,0 +1,7 @@
+calling via IAT-style call
+nativeexec.dll:import_me1(57)
+calling via PLT-style call
+nativeexec.dll:import_me2(37)
+calling via funky ind call
+nativeexec.dll:import_me3(17)
+all done
diff --git a/suite/tests/win32/nativeterminate.c b/suite/tests/win32/nativeterminate.c
new file mode 100644
index 000000000..76d7d7134
--- /dev/null
+++ b/suite/tests/win32/nativeterminate.c
@@ -0,0 +1,80 @@
+/* **********************************************************
+ * Copyright (c) 2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* nativeterminate.exe that calls routines in nativeterminate.dll.dll 
+ * running in separate threads
+ * reproduces case 5455 on Windows2000
+ */
+#include <windows.h>
+#include "tools.h"
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+/* from nativeterminate.dll.dll */
+__declspec(dllimport) import_me1(int x);
+__declspec(dllimport) import_me_die(int x);
+
+int
+main()
+{
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    INIT();
+
+    print("calling via IAT-style call\n");
+    import_me1(57);
+    print("calling in a thread\n");
+    join_thread(create_thread(import_me1));
+
+    print("calling in a thread that dies\n");
+    join_thread(create_thread(import_me_die));
+    print("case 5455 regression passed\n");
+
+    print("all done\n");
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
+
+
+/* 
+$ useops -loglevel 1 -dumpcore_mask 253 -stderr_mask 21 -native_exec_list nativeterminate.dll.dll; make win32/nativeterminate.runinjector
+ * make sure correctly executed
+ */
diff --git a/suite/tests/win32/nativeterminate.dll.c b/suite/tests/win32/nativeterminate.dll.c
new file mode 100644
index 000000000..face6a2d9
--- /dev/null
+++ b/suite/tests/win32/nativeterminate.dll.c
@@ -0,0 +1,57 @@
+/* **********************************************************
+ * Copyright (c) 2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* nativeterminate.dll.dll
+ * nativeterminate calls these to see syscall handling
+ * and whether we are correctly using given stacks
+ */
+#include "tools.h"
+
+void __declspec(dllexport)
+import_me1(int x)
+{
+    print("nativeexec.dll:import_me1(%d)\n", x);
+}
+
+void __declspec(dllexport)
+import_me_die(int x)
+{
+    print("nativeexec.dll:import_me_die(%d)\n", x);
+    ExitThread(0);
+    print("NOT REACHED\n");
+}
+
+BOOL APIENTRY 
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    return TRUE;
+}
diff --git a/suite/tests/win32/nativeterminate.template b/suite/tests/win32/nativeterminate.template
new file mode 100644
index 000000000..d78180aa7
--- /dev/null
+++ b/suite/tests/win32/nativeterminate.template
@@ -0,0 +1,8 @@
+calling via IAT-style call
+nativeexec.dll:import_me1(57)
+calling in a thread
+nativeexec.dll:import_me1(0)
+calling in a thread that dies
+nativeexec.dll:import_me_die(0)
+case 5455 regression passed
+all done
diff --git a/suite/tests/win32/oomtest.c b/suite/tests/win32/oomtest.c
new file mode 100644
index 000000000..438b0a69a
--- /dev/null
+++ b/suite/tests/win32/oomtest.c
@@ -0,0 +1,145 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "tools.h"
+
+/* FIXME: case 10711
+ * For now, use 50 threads so this test will pass for thin_client. We
+ * should put this back to 100 once we address memory usage issues.
+ * xref cases 8960, 9366, 10376
+ */
+enum {THREADS = 50,
+      THREAD_STACK = 8192};
+
+enum {LOOP_WORK = 100};
+
+#define UNIPROC
+
+#ifdef UNIPROC
+#  define YIELD() thread_yield()    /* or do nothing on a multi processor */
+#else
+#  define YIELD() /* or do nothing on a multi processor */
+#endif
+
+
+/* --------- */
+ptr_int_t thread[THREADS];
+
+long global_started = 0;                   /* unsynchronized */
+long global_finished = 0;                   /* unsynchronized */
+
+int compare( const void *arg1, const void *arg2 )
+{
+   /* Compare all of both strings: */
+   return _stricmp( * ( char** ) arg1, * ( char** ) arg2 );
+}
+
+void
+sort()
+{
+    int argc = 5;
+    char *argv[] = {"one", "two", "three","five", "six", "unsorted"};
+    int i;
+
+   /* Sort remaining args using Quicksort algorithm: */
+   qsort( (void *)argv, (size_t)argc, sizeof( char * ), compare );
+
+#ifdef VERY_VERBOSE
+   /* Output sorted list: */
+   for( i = 0; i < argc; ++i )
+       print( " %s", argv[i] );
+   print( "\n" );
+#endif
+}
+
+DWORD WINAPI
+executor(LPVOID parm) 
+{
+    int w;
+    sort();                 /* do some work */
+    InterlockedIncrement(&global_started);
+
+    for (w = 0; w < LOOP_WORK; w++) {
+        sort();                 /* do more work */
+        if (w % 10 == 0)
+            YIELD();
+    }
+    InterlockedIncrement(&global_finished);
+    return 0;
+}
+
+int
+main()
+{
+    void *p;
+    BOOL res;
+    int reserved = 0;
+    int commit = 0;
+    int t;
+
+    INIT();
+
+    /*  create all threads suspended - so DR doesn't allocate private structures yet */
+    for (t = 0; t < THREADS; t++) {
+        /* could use STACK_SIZE_PARAM_IS_A_RESERVATION */
+        thread[t] = (ptr_int_t)CreateThread(NULL, THREAD_STACK, executor, NULL, CREATE_SUSPENDED, NULL);
+        if (thread[t] == 0) 
+            print("GLE: %d\n", GetLastError());
+        assert(thread[t] != 0);
+    }
+    VERBOSE_PRINT("created %d\n", THREADS);
+
+    /* then reserve all remaining memory */
+    do {
+        int size = 8*PAGE_SIZE; /* only 32K - the other 32K are still not usable */
+        p = reserve_memory(size);
+        if (p) reserved += size;
+    } while (p != NULL);
+
+    VERBOSE_PRINT("reserved %d\n", reserved);
+
+
+    for (t = 0; t < THREADS; t++) {
+        resume_thread(thread[t]);
+    }
+    VERBOSE_PRINT("resumed %d\n", THREADS);
+
+    while (global_started < THREADS / 2)
+        YIELD();
+
+    VERBOSE_PRINT("started %d, finished %d\n", global_started, global_finished);
+
+    print("Successful\n");
+#ifdef WAIT
+    getchar();
+#endif
+}
diff --git a/suite/tests/win32/oomtest.template b/suite/tests/win32/oomtest.template
new file mode 100644
index 000000000..59f5915d4
--- /dev/null
+++ b/suite/tests/win32/oomtest.template
@@ -0,0 +1,7 @@
+// we'd like to have an inequality but no support for that so we
+// hardcode our regression option of 2M
+#if : +-vm_size +2M
+# OUT_OF_MEMORY
+#else
+Successful
+#endif
diff --git a/suite/tests/win32/partial_map.c b/suite/tests/win32/partial_map.c
new file mode 100644
index 000000000..6446be06f
--- /dev/null
+++ b/suite/tests/win32/partial_map.c
@@ -0,0 +1,162 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include <stdio.h>
+#include "tools.h"
+
+#define VERBOSE 0
+
+char sysroot[MAX_PATH];
+
+static void
+myload(char* lib, bool append_to_sysroot) 
+{
+    HANDLE file, mapping;
+    DWORD size = 0, size_to_map;
+    char *map_addr;
+    MEMORY_BASIC_INFORMATION mbi;
+    char file_name_buf[MAX_PATH];
+    char *file_name = lib;
+
+    if (append_to_sysroot) {
+        _snprintf(file_name_buf, BUFFER_SIZE_ELEMENTS(file_name_buf),
+                  "%s%s", sysroot, lib);
+        file_name = file_name_buf;
+    }
+
+    file = CreateFileA(file_name, GENERIC_READ, FILE_SHARE_READ, NULL,
+                       OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+    if (file == INVALID_HANDLE_VALUE) {
+        print("error opening file \"%s\", code=%d\n", file_name, GetLastError());
+        return;
+    }
+
+    mapping = CreateFileMapping(file, NULL, PAGE_WRITECOPY|SEC_IMAGE, 0, 0, NULL);
+    if (mapping == NULL) {
+        print("error creating mapping for file \"%s\", code=%d\n",
+              file_name, GetLastError());
+        return;
+    }
+
+    map_addr = MapViewOfFile(mapping, FILE_MAP_COPY, 0, 0, 0);
+    if (map_addr == NULL) {
+        print("error mapping file \"%s\", code=%d\n", file_name, GetLastError());
+        return;
+    } else {
+        print("test map of %s succeeded\n", lib);
+    }
+    /* FIXME - can't find a good way to get the size of the section, can't use file size
+     * since as an image will be larger, msdn says to use VirtualQuery which of course
+     * doesn't work on a image.  We'll just walk instead (what we really want is
+     * NtQuerySection:SectionBasicInformation but the API routines don't appear to
+     * expose that). */
+    while (VirtualQuery(map_addr+size, &mbi, sizeof(mbi)) == sizeof(mbi) &&
+           mbi.State != MEM_FREE && mbi.AllocationBase == map_addr) {
+        size += mbi.RegionSize;
+    }
+#if VERBOSE
+    print("mapping size = "PFX"\n", size);
+#endif
+    UnmapViewOfFile(map_addr);
+    
+    /* FIXME - for additional tests we could call into the section, we could also map
+     * at offset and call into as well.  FIXME - check what happes when we ask for
+     * non-page multiple (esp if file and/or section alignment is < page. */
+    for (size_to_map = PAGE_SIZE; size_to_map <= size; size_to_map += PAGE_SIZE) {
+        map_addr = MapViewOfFile(mapping, FILE_MAP_COPY, 0, 0, size_to_map);
+        UnmapViewOfFile(map_addr);
+    }
+}
+
+int main()
+{
+    /* Note - if not part of session 0 requires SeCreateGlobalPrivilege for XPsp2
+     * 2ksp4 and 2k3 (and presumably Vista). */
+    HANDLE hToken = NULL;
+    TOKEN_PRIVILEGES Priv;
+    DWORD res;
+    if  (!OpenThreadToken(GetCurrentThread(), 
+                          TOKEN_QUERY|TOKEN_ADJUST_PRIVILEGES, 
+                          FALSE, 
+                          &hToken)) {
+        /* can't get thread token, try process token instead */
+        if(!OpenProcessToken(GetCurrentProcess(), 
+                             TOKEN_QUERY|TOKEN_ADJUST_PRIVILEGES, 
+                             &hToken)) {
+            print("error opening token, code=%d\n", GetLastError());
+        }
+    }
+    if (hToken != NULL) {
+        Priv.PrivilegeCount = 1;
+        Priv.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
+        LookupPrivilegeValue(NULL, "SeCreateGlobalPrivilege", &Priv.Privileges[0].Luid);
+        if (!AdjustTokenPrivileges(hToken, FALSE, &Priv, sizeof(Priv), NULL, 0)) {
+            print("error adjusting privileges, code=%d\n", GetLastError());
+        }
+    }
+
+    res = GetEnvironmentVariable("SYSTEMROOT", sysroot, BUFFER_SIZE_ELEMENTS(sysroot));
+    NULL_TERMINATE_BUFFER(sysroot);
+    if (res == 0 || res >= BUFFER_SIZE_ELEMENTS(sysroot))
+        print("Unable to get system root\n");
+
+    /* FIXME - add specially crafted .exe/.dlls that have page boundaries at interesting
+     * locations. */
+
+    /* We don't yet safetly handle exports, so we limit the test to .exe's (which usually
+     * don't have exports) to excessive test failures.  The case 9717, the driving case
+     * for this test, partial maps are limited to only. exe's so we should be ok for
+     * now. FIXME */
+    myload("\\system32\\user32.dll", true);
+#if 0
+    /* still need to quiet some aslr asserts */
+    myload("\\system32\\shell32.dll", true);
+    /* TODO : add more dlls, maybe custom made to have strange boundaries. */
+#endif
+
+    /* Test some .exe images, none of these have exports and except explorer.exe none
+     * of these have reloc sections either, so we generally expect these to succeed
+     * (after the image entry not in module fix).  But, they are still good for showing
+     * asserts. */
+    myload("\\system32\\calc.exe", true);
+    myload("\\system32\\notepad.exe", true);
+    myload("\\system32\\svchost.exe", true);
+    myload("\\system32\\rundll32.exe", true);
+    /* This is the actual troublesome .exe partially mapped in case 9717, it is unusual
+     * in that explorer.exe is one of the only .exe's I've seen that have a reloc section
+     * (no exports though, so we don't have to worry about that at least). It's the reloc
+     * section walk that led to the crash in 9717. */
+    myload("\\explorer.exe", true);
+    print("done\n");
+    return 0;
+}
diff --git a/suite/tests/win32/partial_map.expect b/suite/tests/win32/partial_map.expect
new file mode 100755
index 000000000..9bfc63480
--- /dev/null
+++ b/suite/tests/win32/partial_map.expect
@@ -0,0 +1,7 @@
+test map of \system32\user32.dll succeeded
+test map of \system32\calc.exe succeeded
+test map of \system32\notepad.exe succeeded
+test map of \system32\svchost.exe succeeded
+test map of \system32\rundll32.exe succeeded 
+test map of \explorer.exe succeeded
+done
diff --git a/suite/tests/win32/protect-datasec.c b/suite/tests/win32/protect-datasec.c
new file mode 100644
index 000000000..38cb9febb
--- /dev/null
+++ b/suite/tests/win32/protect-datasec.c
@@ -0,0 +1,256 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Tests protection of DynamoRIO's stack
+ * This test when run natively will fail with an error message
+ */
+
+#include "tools.h"
+
+#ifdef LINUX
+# include <unistd.h>
+# include <signal.h>
+# include <ucontext.h>
+# include <errno.h>
+#endif
+
+#include <setjmp.h>
+
+#ifdef USE_DYNAMO
+# include "dynamorio.h"
+#endif
+
+#define VERBOSE 0
+
+#define EXPANDSTR(x) #x
+#define STRINGIFY(x) EXPANDSTR(x)
+#define ALIGNED(x, alignment) ((((unsigned int)x) & ((alignment)-1)) == 0)
+
+/****************************************************************************/
+
+/* the data sections to try to write to */
+static const char * const datasec_names[] = {
+    ".data",
+    ".fspdata",
+    ".cspdata",
+    ".nspdata"
+};
+#define DATASEC_NUM (sizeof(datasec_names)/sizeof(datasec_names[0]))
+
+typedef unsigned char byte;
+
+#ifdef LINUX
+# error NYI
+#else
+bool
+get_named_section_bounds(byte *module_base, const char *name,
+                         byte **start/*OUT*/, byte **end/*OUT*/)
+{
+    IMAGE_DOS_HEADER *dos;
+    IMAGE_NT_HEADERS *nt;
+    IMAGE_SECTION_HEADER *sec;
+    uint i;
+    bool prev_sec_same_chars = false;
+    assert(module_base != NULL);
+    dos = (IMAGE_DOS_HEADER *) module_base;
+    if (dos->e_magic != IMAGE_DOS_SIGNATURE)
+        return false;
+    nt = (IMAGE_NT_HEADERS *) (((ptr_uint_t)dos) + dos->e_lfanew);
+    if (nt == NULL || nt->Signature != IMAGE_NT_SIGNATURE)
+        return false;
+    assert(start != NULL || end != NULL);
+    sec = IMAGE_FIRST_SECTION(nt);
+    for (i = 0; i < nt->FileHeader.NumberOfSections; i++) {
+        if (sec->Name != NULL && strncmp(sec->Name, name, strlen(name)) == 0) {
+            if (start != NULL)
+                *start = module_base + sec->VirtualAddress;
+            if (end != NULL)
+                *end = module_base + sec->VirtualAddress +
+                    ALIGN_FORWARD(sec->Misc.VirtualSize, PAGE_SIZE);
+            return true;
+        }
+        sec++;
+    }
+    return false;
+}
+
+byte *
+get_DR_base()
+{
+    /* read DR marker
+     * just hardcode the offsets for now
+     */
+    static const uint DR_BASE_OFFSET = 0x1c;
+    return get_drmarker_field(DR_BASE_OFFSET);
+}
+#endif /* WINDOWS */
+
+/****************************************************************************/
+
+
+jmp_buf mark;
+int where; /* 0 = normal, 1 = segfault longjmp */
+
+#ifdef LINUX
+/* just use single-arg handlers */
+typedef void (*handler_t)(int);
+typedef void (*handler_3_t)(int, struct siginfo *, void *);
+
+static void
+signal_handler(int sig)
+{
+    if (sig == SIGSEGV) {
+#if VERBOSE
+        print("Got seg fault\n");
+#endif
+        longjmp(mark, 1);
+    }
+    exit(-1);
+}
+
+#define ASSERT_NOERR(rc) do {                                   \
+  if (rc) {                                                     \
+     fprintf(stderr, "%s:%d rc=%d errno=%d %s\n",               \
+             __FILE__, __LINE__,                                \
+             rc, errno, strerror(errno));                       \
+  }                                                             \
+} while (0);
+
+/* set up signal_handler as the handler for signal "sig" */
+static void
+intercept_signal(int sig, handler_t handler)
+{
+    int rc;
+    struct sigaction act;
+
+    act.sa_sigaction = (handler_3_t) handler;
+    /* FIXME: due to DR bug 840 we cannot block ourself in the handler
+     * since the handler does not end in a sigreturn, so we have an empty mask
+     * and we use SA_NOMASK
+     */
+    rc = sigemptyset(&act.sa_mask); /* block no signals within handler */
+    ASSERT_NOERR(rc);
+    /* FIXME: due to DR bug #654 we use SA_SIGINFO -- change it once DR works */
+    act.sa_flags = SA_NOMASK | SA_SIGINFO | SA_ONSTACK;
+    
+    /* arm the signal */
+    rc = sigaction(sig, &act, NULL);
+    ASSERT_NOERR(rc);
+}
+
+#else
+/* sort of a hack to avoid the MessageBox of the unhandled exception spoiling
+ * our batch runs
+ */
+# include <windows.h>
+/* top-level exception handler */
+static LONG
+our_top_handler(struct _EXCEPTION_POINTERS * pExceptionInfo)
+{
+    if (pExceptionInfo->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
+#if VERBOSE
+        print("Got segfault\n");
+#endif
+        longjmp(mark, 1);
+    }
+# if VERBOSE
+    print("Exception occurred, process about to die silently\n");
+# endif
+    return EXCEPTION_EXECUTE_HANDLER; /* => global unwind and silent death */
+}
+#endif
+
+int
+main()
+{
+    uint i;
+    uint writes = 0;
+    byte *DR_base;
+    INIT();
+    
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+#ifdef LINUX
+    intercept_signal(SIGSEGV, signal_handler);
+#else
+    SetUnhandledExceptionFilter((LPTOP_LEVEL_EXCEPTION_FILTER) our_top_handler);
+#endif
+
+    where = setjmp(mark);
+    if (where == 0) {
+        DR_base = get_DR_base();
+        for (i=0; i<DATASEC_NUM; i++) {
+            byte *start, *end, *pc;
+            bool found = get_named_section_bounds(DR_base,
+                                                  /*FIXME use drmarker to find*/
+                                                  datasec_names[i], &start, &end);
+            assert(found);
+#if VERBOSE
+            print("data section %s: "PFX"-"PFX"\n", datasec_names[i], start, end);
+#endif
+            print("about to write to every page in %s\n", datasec_names[i]);
+            for (pc = start; pc < end; pc += PAGE_SIZE) {
+                /* try to write to every single page */
+                where = setjmp(mark);
+                if (where == 0) {
+                    uint old = *((uint *)pc);
+                    *((uint *)pc) = 0x0badbad0;
+                    /* restore in same bb so we don't crash DR */
+                    *((uint *)pc) = old;
+                    /* if the section is protected we shouldn't get here */
+                    writes++;
+#if VERBOSE
+                    print("successfully wrote to "PFX" in %s!\n",
+                          pc, datasec_names[i]);
+#endif
+                }
+            }
+#if VERBOSE
+            print("successfully wrote to %d pages!\n", writes);
+#endif
+            if (writes > 0)
+                print("successfully wrote to %s\n", datasec_names[i]);
+        }
+    } else {
+        print("no DR or no data sections found: are you running natively?\n");
+    }
+
+    print("all done\n");
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
diff --git a/suite/tests/win32/protect-datasec.template b/suite/tests/win32/protect-datasec.template
new file mode 100755
index 000000000..363d8eab0
--- /dev/null
+++ b/suite/tests/win32/protect-datasec.template
@@ -0,0 +1,17 @@
+about to write to every page in .data
+// FIXME: nicer way to specify bitmasks? :)
+// Note that 0x* matches plain 0 so we have to explicitly exclude 0x
+#if : +-protect_mask(0) --protect_mask(0x) || : +-protect_mask(2) || : +-protect_mask(4) || : +-protect_mask(6) || : +-protect_mask(8)
+successfully wrote to .data
+#endif
+about to write to every page in .fspdata
+#if : --protect_mask(3) --protect_mask(7) --protect_mask(0x103) --protect_mask(0x107) --protect_mask(0x10f)
+successfully wrote to .fspdata
+#endif
+about to write to every page in .cspdata
+#if : --protect_mask(7) --protect_mask(0x107) --protect_mask(0x10f)
+successfully wrote to .cspdata
+#endif
+about to write to every page in .nspdata
+successfully wrote to .nspdata
+all done
diff --git a/suite/tests/win32/rebased.c b/suite/tests/win32/rebased.c
new file mode 100644
index 000000000..e376b97a7
--- /dev/null
+++ b/suite/tests/win32/rebased.c
@@ -0,0 +1,71 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include <stdio.h>
+#include "tools.h"
+
+#define VERBOSE 0
+
+HMODULE
+myload(char* lib) 
+{
+    HMODULE hm = LoadLibrary(lib);
+    if (hm == NULL) {
+        print("error loading library %s\n", lib);
+    } else {
+        BOOL res;
+        BOOL (WINAPI *proc)(DWORD);
+        print("loaded %s\n", lib);
+#if VERBOSE
+        print("library is at "PFX"\n", hm);
+#endif
+    }
+    return hm;
+}
+
+int main()
+{
+    HMODULE lib1;
+    HMODULE lib2;
+
+    lib1 = myload("rebased.dll.dll");
+    lib2 = myload("rebase~1.dll");
+    if (lib1 == lib2) {
+        print("there is a problem - should have collided, maybe missing\n");
+    }
+
+    FreeLibrary(lib1);
+    FreeLibrary(lib2);
+
+    return 0;
+}
diff --git a/suite/tests/win32/rebased.dll.c b/suite/tests/win32/rebased.dll.c
new file mode 100644
index 000000000..aa7903461
--- /dev/null
+++ b/suite/tests/win32/rebased.dll.c
@@ -0,0 +1,120 @@
+/* **********************************************************
+ * Copyright (c) 2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* case 4347 - testing rebasing 
+ * make sure DLL and image are conflicting so that DLL goes somewhere else
+ *
+ *
+ * Nothing to worry about: the two NtFlushInstructionCache calls are
+ * of (0,0) and of (IAT,IAT_size) there is no explicit flush on a
+ * .data section for which the original inquiry was filed.
+ * 
+ * intercept_load_dll: rebase~1.dll
+ * syscall: NtMapViewOfSection 0x00350000 size=0x29000 prot=rw-- => 1073741827
+ * syscall: NtProtectVirtualMemory process=0xffffffff base=0x00351000 size=0x1b000 prot=rw-- 0x4
+ * WARNING: executable region being made writable and non-executable
+ * syscall: NtProtectVirtualMemory process=0xffffffff base=0x0036c000 size=0x3000 prot=rw-- 0x4
+ * syscall: NtProtectVirtualMemory process=0xffffffff base=0x00377000 size=0x2000 prot=rw-- 0x4
+ * syscall: NtProtectVirtualMemory process=0xffffffff base=0x00351000 size=0x1b000 prot=--x- 0x10
+ * WARNING: data region 0x00351000 is being made executable
+ * syscall: NtProtectVirtualMemory process=0xffffffff base=0x0036c000 size=0x3000 prot=r--- 0x2
+ * syscall: NtProtectVirtualMemory process=0xffffffff base=0x00377000 size=0x2000 prot=r--- 0x2
+ * syscall: NtFlushInstructionCache 0x00000000 size=0x0
+ * syscall: NtProtectVirtualMemory process=0xffffffff base=0x003761b4 size=0x18c prot=rw-- 0x4
+ * syscall: NtProtectVirtualMemory process=0xffffffff base=0x00376000 size=0x1000 prot=rw-c 0x8
+ * syscall: NtFlushInstructionCache 0x00376000 size=0x18c
+ * module is being loaded, ignoring flush
+ * intercept_unload_dll: i:\vlk\trees\t4347-rebase\suite\tests\win32\rebased.dll.dll @0x10000000 size 0x29000
+ * 
+ * FIXME: add these to the Makefile
+ * # force rebasing conflicts
+ * # LDFLAGS_B = /base:"0x00400000"
+ * # FIXME: even more evil would be to set to conflict with dynamorio.dll
+ *
+ * LINK = $(LDFLAGS_B) $(WINLIBS) /link /subsystem:console
+ * # FIXME: should also reorder when user32.dll is mapped in 
+ */
+
+#include "tools.h"
+#include <windows.h>
+
+typedef void (*fptr)();
+
+void
+foo() 
+{
+    print("foo\n");
+}
+
+void
+bar() 
+{
+    print("bar\n");
+}
+
+const fptr cf = &foo;
+fptr f = &foo;
+
+void
+dlltest(void) 
+{
+    print("dlltest\n");
+    f();
+    cf();
+    f = &bar;
+    f();
+    cf();
+}
+
+
+/* our Makefile expects a .lib */
+int
+__declspec(dllexport)
+data_attack(int arg)
+{
+    print("data_attack\n");
+    /* FIXME: will do this some other time */
+    dlltest();
+    return 1;
+}
+
+BOOL APIENTRY 
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    switch (reason_for_call) {
+    case DLL_PROCESS_ATTACH:
+        dlltest();
+
+        break;
+    }
+    return TRUE;
+}
diff --git a/suite/tests/win32/rebased.template b/suite/tests/win32/rebased.template
new file mode 100644
index 000000000..1ce5342ba
--- /dev/null
+++ b/suite/tests/win32/rebased.template
@@ -0,0 +1,17 @@
+dlltest
+foo
+foo
+bar
+foo
+loaded rebased.dll.dll
+#if +PROGRAM_SHEPHERDING : +-security +-no_executable_if_rx_text --executable_after_load
+# SEC_VIO_STOP
+# STOP
+#else
+dlltest
+foo
+foo
+bar
+foo
+loaded rebase~1.dll
+#endif
diff --git a/suite/tests/win32/reload-newaddr.c b/suite/tests/win32/reload-newaddr.c
new file mode 100644
index 000000000..d87870ce0
--- /dev/null
+++ b/suite/tests/win32/reload-newaddr.c
@@ -0,0 +1,253 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* reload.c: tests repeatedly loads and unloads a dll, and executes from it
+ * in between, to stress our cache management
+ */
+
+#include <windows.h>
+#include <process.h> /* for _beginthreadex */
+#include "tools.h"
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+/* make as STRESS=1 for a performance test */
+
+#ifdef NIGHTLY_REGRESSION
+/* don't ask to compute fact or fib too high */
+# define MAX_FACT_FIB 20
+# define ITERS (2*MAX_FACT_FIB)
+#else
+/* PERF or STRESS */
+/* very low computation here */
+# define MAX_FACT_FIB 8
+# define ITERS 4000
+#endif
+#define VERBOSE 0
+
+/* we create a 2nd thread to complicate DR cache flushing */
+int WINAPI
+run_func(void * arg)
+{
+    WaitForSingleObject((HANDLE)arg, INFINITE);
+    return 0;
+}
+
+static SIZE_T
+get_mem_usage()
+{
+    VM_COUNTERS mem;
+    get_process_mem_stats(GetCurrentProcess(), &mem);
+#if VERBOSE
+    print("Process Memory Statistics:\n");
+    print("\tPeak virtual size:         %6d KB\n",
+          mem.PeakVirtualSize/1024);
+    print("\tPeak working set size:     %6d KB\n",
+          mem.PeakWorkingSetSize/1024);
+    print("\tPeak paged pool usage:     %6d KB\n",
+          mem.QuotaPeakPagedPoolUsage/1024);
+    print("\tPeak non-paged pool usage: %6d KB\n",
+          mem.QuotaPeakNonPagedPoolUsage/1024);
+    print("\tPeak pagefile usage:       %6d KB\n",
+          mem.PeakPagefileUsage/1024);
+#endif
+    return mem.PeakPagefileUsage;
+}
+
+static void
+check_mem_usage(SIZE_T peakpage)
+{
+    VM_COUNTERS mem;
+    SIZE_T newpeak = get_mem_usage();
+#if VERBOSE
+    print("Pagefile usage increase is %d KB\n", (newpeak - peakpage) / 1024);
+#endif
+    /* Since under trace threshold increase shouldn't be too much -- I
+     * see 20KB w/ TOT 11/20/06 vs 336KB for -no_free_unmapped_futures
+     * and 88KB for -rct_sticky.
+     * If we went up into the 100's of iters we could keep it even lower
+     * but we want a short test.
+
+     FIXME: coarse units makes the increase larger, and traces do make
+     a big difference:
+      % for i in "" "-disable_traces" "-no_coarse_units" "-no_coarse_units -disable_traces"; do useops $i; (for j in 40 60 80 100; do rundr rel win32/reload-newaddr.exe $j; done) 2>&1 | grep '^Pagefile usage increase'; done
+      DYNAMORIO_OPTIONS=
+      Pagefile usage increase is 68 KB
+      Pagefile usage increase is 112 KB
+      Pagefile usage increase is 136 KB
+      Pagefile usage increase is 152 KB
+      DYNAMORIO_OPTIONS=-disable_traces
+      Pagefile usage increase is 48 KB
+      Pagefile usage increase is 52 KB
+      Pagefile usage increase is 56 KB
+      Pagefile usage increase is 52 KB
+      DYNAMORIO_OPTIONS=-no_coarse_units
+      Pagefile usage increase is 20 KB
+      Pagefile usage increase is 72 KB
+      Pagefile usage increase is 108 KB
+      Pagefile usage increase is 96 KB
+      DYNAMORIO_OPTIONS=-no_coarse_units -disable_traces
+      Pagefile usage increase is 28 KB
+      Pagefile usage increase is 32 KB
+      Pagefile usage increase is 36 KB
+      Pagefile usage increase is 32 KB
+
+    Strangely debug build uses less memory for coarse:
+      % for i in "" "-disable_traces" "-no_coarse_units" "-no_coarse_units -disable_traces"; do useops $i; (for j in 40; do rundr dbg win32/reload-newaddr.exe $j; done) 2>&1 | grep '^Pagefile usage increase'; done
+      DYNAMORIO_OPTIONS=
+      Pagefile usage increase is 56 KB
+      DYNAMORIO_OPTIONS=-disable_traces
+      Pagefile usage increase is 40 KB
+      DYNAMORIO_OPTIONS=-no_coarse_units
+      Pagefile usage increase is 24 KB
+      DYNAMORIO_OPTIONS=-no_coarse_units -disable_traces
+      Pagefile usage increase is 32 KB
+
+     But it seems that we have to run a LOT of iters to really prove there's no leak:
+       % for i in 10 100 200 500 1000 2000; do echo $i; (norio win32/reload-newaddr.exe $i; usetree coarse; useops -no_coarse_units; rio win32/reload-newaddr.exe $i; useops -desktop; rio win32/reload-newaddr.exe $i; useops; rio win32/reload-newaddr.exe $i) 2>&1 | grep 'Peak pagefile' | sed 's/Peak pagefile usage://' | xargs | awk '{printf "native %4d; nocoarse %4d +%3d; desktop %4d +%3d; tot %4d +%3d\n", $1, $3, $3-$1, $5, $5-$1, $7, $7-$1}'; done
+      10
+      native  428; nocoarse 1588 +1160; desktop 1424 +996; tot 1516 +1088
+      100
+      native  436; nocoarse 1740 +1304; desktop 1448 +1012; tot 1712 +1276
+      200
+      native  448; nocoarse 1760 +1312; desktop 1460 +1012; tot 1756 +1308
+      500
+      native  488; nocoarse 1804 +1316; desktop 1500 +1012; tot 1804 +1316
+      1000
+      native  548; nocoarse 1864 +1316; desktop 1560 +1012; tot 1868 +1320
+      2000
+      native  676; nocoarse 1988 +1312; desktop 1688 +1012; tot 1992 +1316
+
+    So for now for short regression we allow the larger values we've seen.
+    FIXME: have a long regr test that does hundreds of iters!
+    */
+    /* typical DR */
+    /* FIXME: I used to have a <32KB category but I seem to have nondet
+     * behavior, make vs raw being different, etc. etc. so just checking
+     * for under 80KB
+     */
+    if (newpeak - peakpage < 80*1024)
+        print("Memory check: pagefile usage increase is < 80 KB\n");
+    /* detect_dangling_fcache doesn't free fcache */
+    else if (newpeak - peakpage < 120*1024)
+        print("Memory check: pagefile usage increase is >= 80 KB, < 120 KB\n");
+    else {
+        /* give actual number here so we can see how high it is */
+        print("Memory check: pagefile usage increase is %d KB >= 120 KB\n",
+              (newpeak - peakpage)/1024);
+    }
+}
+
+static int
+doload(int iter)
+{
+    HANDLE lib;
+    int sum = 0;
+    void *p;
+    lib = LoadLibrary("reload-newaddr.dll.dll");
+    if (lib == NULL) {
+        print("error loading library\n");
+        return 0;
+    } else {
+        /* don't ask to compute fact or fib too high */
+        BOOL (WINAPI *proc)(DWORD);
+        proc = (BOOL (WINAPI *)(DWORD))
+            GetProcAddress(lib, (LPCSTR)"import_me");
+        sum += (*proc)(iter % MAX_FACT_FIB);
+        FreeLibrary(lib);
+        /* prevent loading at the same address again */
+        p = VirtualAlloc(lib, 4*1024, MEM_RESERVE, PAGE_EXECUTE_READWRITE);
+        /* deliberately do not free */
+#if VERBOSE
+        print("alloced "PFX" @ last loaded slot "PFX"\n", p, lib);
+#endif
+    }
+    return sum;
+}
+
+int main(int argc, char** argv)
+{
+    HANDLE event;
+    int i, res;
+    unsigned long hThread;
+    uint iters = 0;
+    int sum = 0;
+    SIZE_T peakpage;
+
+    USE_USER32();
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    if (argc != 2) {
+        iters = ITERS;
+    } else {
+        iters = atoi(argv[1]);
+    }
+    fprintf(stderr, "iters is %d\n", iters);
+
+    event = CreateEvent(NULL, TRUE, FALSE, NULL);
+    hThread = _beginthreadex(NULL, 0, run_func, event, 0, &i);
+
+    for (i=0; i<iters/2; i++) {
+        res = doload(i);
+        if (res == 0)
+            break;
+        sum += res;
+    }
+
+    peakpage = get_mem_usage();
+
+    for (i=iters/2; i<iters; i++) {
+        res = doload(i);
+        if (res == 0)
+            break;
+        sum += res;
+    }
+
+    check_mem_usage(peakpage);
+
+    SetEvent(event);
+    WaitForSingleObject((HANDLE)hThread, INFINITE);
+
+    print("sum=%d\n", sum);
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
diff --git a/suite/tests/win32/reload-newaddr.dll.c b/suite/tests/win32/reload-newaddr.dll.c
new file mode 100644
index 000000000..7112eda05
--- /dev/null
+++ b/suite/tests/win32/reload-newaddr.dll.c
@@ -0,0 +1,73 @@
+/* **********************************************************
+ * Copyright (c) 2006 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "tools.h"
+
+#define VERBOSE 0
+
+int
+fib(int n) {
+    if (n <= 1)
+        return 1;
+    return fib(n-1) + fib(n-2);
+}
+
+int
+fact(int n) {
+    if (n <= 1)
+        return 1;
+    return n * fact(n-1);
+}
+
+int  __declspec(dllexport)
+import_me(int x)
+{
+#if VERBOSE
+    print("in import %d\n", x);
+#endif
+    if (x % 2 == 0)
+        return fib(x);
+    else
+        return fact(x);
+}
+
+BOOL APIENTRY 
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    switch (reason_for_call) {
+    case DLL_PROCESS_ATTACH:
+        break;
+    case DLL_PROCESS_DETACH:
+        break;        
+    }
+    return TRUE;
+}
diff --git a/suite/tests/win32/reload-newaddr.template b/suite/tests/win32/reload-newaddr.template
new file mode 100755
index 000000000..38fa5dee9
--- /dev/null
+++ b/suite/tests/win32/reload-newaddr.template
@@ -0,0 +1,24 @@
+// we don't expect anyone to run w/ -throw_exception or -kill_thread in
+// addition to -no_executable_if_rx_text
+#if +PROGRAM_SHEPHERDING : +-security +-no_executable_if_rx_text
+# if : --detect_mode
+#  SEC_VIO_AUTO_STOP
+#  if : --internal_detach_mask
+#   STOP
+#  endif
+# else
+#  SEC_VIO_CONT
+# endif
+#endif
+iters is 40
+// FIXME: strange behavior: giving up on smaller categories
+#if : +-desktop || : +-disable_traces --no_coarse_units
+Memory check: pagefile usage increase is < 80 KB
+#else
+#if : +-no_coarse_units
+Memory check: pagefile usage increase is < 80 KB
+#else
+Memory check: pagefile usage increase is < 80 KB
+#endif
+#endif
+sum=-994385352
diff --git a/suite/tests/win32/reload-race.c b/suite/tests/win32/reload-race.c
new file mode 100644
index 000000000..5fe58a5c2
--- /dev/null
+++ b/suite/tests/win32/reload-race.c
@@ -0,0 +1,336 @@
+/* **********************************************************
+ * Copyright (c) 2006-2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* reload-race.c: tests repeatedly loads and unloads a dll, and
+ * executes from it in between in different tests - case 6293
+ */
+
+#include <windows.h>
+#include <process.h> /* for _beginthreadex */
+#include "tools.h"
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#include <setjmp.h>
+
+#define VERBOSE 0
+
+#define UNIPROC
+
+/* FIXME: we may even want to have a sleep */
+
+/* for .B race we'd want to reach the unloaded module, note we provide
+ * a .C opportunity by having the unloaded module call thread_yield()
+ * and .E and .F violations on our IAT vs PLT calls.  (No explicit
+ * function pointers necessary.)
+ */
+
+#ifdef UNIPROC
+#  define YIELD() thread_yield()    /* or do nothing on a multi processor */
+#else
+#  define YIELD() /* or do nothing on a multi processor */
+#endif
+
+
+/* Need to ensure we're doing the same amount of work - so we need to
+ * count our races - we need edge detector whether we were in unloaded
+ * area or in a good module NUM_TRANSITIONS counts how many should be
+ * reproduced to give the race a chance.  (We can't detect whether a
+ * B/C/E/F violation was suppressed though.)
+ */
+
+/* make as STRESS=1 for a performance test */
+
+#ifdef NIGHTLY_REGRESSION
+/* don't ask to compute fact or fib too high */
+#define MAX_FACT_FIB 8
+#define NUM_TRANSITIONS 10
+#else
+/* PERF or STRESS */
+/* very low computation here */
+#define MAX_FACT_FIB 8
+#define NUM_TRANSITIONS 100
+#endif
+
+
+enum {LAST_OK, LAST_FAULT, LAST_START};
+
+typedef int (*funptr)();
+
+/* note we must have dynamically linked DLLs */
+funptr import1;
+funptr import2;
+/* Note we may not want to use GetProcAddress() in target thread if
+ * that is serialized with loader (and to allow for running -aslr 1 we
+ * can't grab once only */
+
+/* If a DLL shows up at the same address yet not its preferred base
+ * and it has to be rebased, we'll also have bad execution before it
+ * gets rebased, which is not likely to happen in real apps
+ */
+
+int transitions; /* atomic writes necessary */
+
+jmp_buf mark;
+int where; /* 0 = normal, 1 = segfault longjmp */
+    
+
+int sum1 = 0;
+int sum2 = 0;
+int done1 = 0;
+int done2 = 0;
+
+/* we create a 2nd thread to complicate DR cache flushing */
+int WINAPI
+run_func(void * arg)
+{
+    int last_good = LAST_START; /* so we can count proper transitions */
+    int same_run = 0;
+
+    /* need to run as long as necessary */
+    while (transitions < NUM_TRANSITIONS) {
+#if VERBOSE
+        print("about to call\n");
+#endif
+        /* try to write to every single page */
+        where = setjmp(mark);
+        if (where == 0) {
+            /* .B for target if it is unloaded in a race, as well as
+             * .E .F on the way there should all be suppressed */
+
+            /* note that the target itself may get a .C and a .B on
+             * its way back from thread_yield()
+             */
+
+            if(same_run % 2 == 0)
+                YIELD();
+            /* .E shouldn't be seen here */
+            sum1 += import1(transitions % MAX_FACT_FIB);
+            done1++;
+            if(same_run % 3 == 0)
+                YIELD();
+            /* .F shouldn't be seen here */
+            sum2 += import2(transitions % MAX_FACT_FIB);
+            done2++;
+            /* note the above will be racy - e.g. not all transitions
+             * will exercise both */
+#if VERBOSE
+            print("made it in and out on %d, same_run %d\n", transitions, same_run);
+#endif
+            if (last_good != LAST_OK) {
+                last_good = LAST_OK;
+                InterlockedIncrement(&transitions);
+                same_run = 0;
+            } else {
+                same_run++;
+            }
+        } else {
+            /* fault */
+            if (last_good != LAST_FAULT) {
+                last_good = LAST_FAULT;
+                InterlockedIncrement(&transitions);
+                same_run = 0;
+            } else {
+                same_run++;
+            }
+        }
+    }
+
+    print("made it in and out %d transitions\n", transitions);
+#if VERBOSE
+    print("import_me1 ran %d, sum %d\n", done1, sum1);
+    print("import_me2 ran %d, sum %d\n", done2, sum2);
+#endif
+
+    WaitForSingleObject((HANDLE)arg, INFINITE);
+    return 0;
+}
+
+/* make sure DR isn't using too much memory and is actually freeing fragments */
+static void
+check_mem_usage()
+{
+    VM_COUNTERS mem;
+    get_process_mem_stats(GetCurrentProcess(), &mem);
+#if VERBOSE
+    print("Process Memory Statistics:\n");
+    print("\tPeak virtual size:         %6d KB\n",
+          mem.PeakVirtualSize/1024);
+    print("\tPeak working set size:     %6d KB\n",
+          mem.PeakWorkingSetSize/1024);
+    print("\tPeak paged pool usage:     %6d KB\n",
+          mem.QuotaPeakPagedPoolUsage/1024);
+    print("\tPeak non-paged pool usage: %6d KB\n",
+          mem.QuotaPeakNonPagedPoolUsage/1024);
+    print("\tPeak pagefile usage:       %6d KB\n",
+          mem.PeakPagefileUsage/1024);
+#endif
+/*
+  native:
+        Peak virtual size:           7772 KB
+        Peak working set size:        996 KB
+        Peak paged pool usage:          8 KB
+        Peak non-paged pool usage:      1 KB
+        Peak pagefile usage:          352 KB
+DR results with debug build, where library takes up more WSS (one
+reason we use pagefile usage as our discerning factor):
+  -no_shared_deletion:
+        Peak virtual size:         140388 KB
+        Peak working set size:       7948 KB
+        Peak paged pool usage:          9 KB
+        Peak non-paged pool usage:      1 KB
+        Peak pagefile usage:         6536 KB
+  -no_syscalls_synch_flush:
+        Peak virtual size:         140388 KB
+        Peak working set size:       7784 KB
+        Peak paged pool usage:          9 KB
+        Peak non-paged pool usage:      1 KB
+        Peak pagefile usage:         6368 KB
+  -no_cache_shared_free_list:
+        Peak virtual size:         140388 KB
+        Peak working set size:       5156 KB
+        Peak paged pool usage:          9 KB
+        Peak non-paged pool usage:      1 KB
+        Peak pagefile usage:         3736 KB
+  defaults:
+        Peak virtual size:         140388 KB
+        Peak working set size:       3352 KB
+        Peak paged pool usage:          9 KB
+        Peak non-paged pool usage:      1 KB
+        Peak pagefile usage:         1680 KB
+*/
+    /* native */
+#if VERBOSE
+    print("Pagefile usage is %d KB\n", mem.PeakPagefileUsage/1024);
+#endif
+    if (mem.PeakPagefileUsage < 900*1024)
+        print("Memory check: pagefile usage is < 900 KB\n");
+    /* typical DR */
+    else if (mem.PeakPagefileUsage < 2816*1024)
+        print("Memory check: pagefile usage is >= 900 KB, < 2816 KB\n");
+    /* prof_pcs uses a buffer the size of DR.dll */
+    else if (mem.PeakPagefileUsage < 6000*1024)
+        print("Memory check: pagefile usage is >= 2816 KB, < 6000 KB\n");
+    /* detect_dangling_fcache doesn't free fcache */
+    else if (mem.PeakPagefileUsage < 16384*1024)
+        print("Memory check: pagefile usage is >= 6000 KB, < 16384 KB\n");
+    else {
+        /* give actual number here so we can see how high it is */
+        print("Memory check: pagefile usage is %d KB >= 16384 KB\n",
+              mem.PeakPagefileUsage/1024);
+    }
+}
+
+/* top-level exception handler */
+static LONG
+our_top_handler(struct _EXCEPTION_POINTERS * pExceptionInfo)
+{
+    if (pExceptionInfo->ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION) {
+#if VERBOSE
+        print("Got segfault\n");
+#endif
+        longjmp(mark, 1);
+    }
+# if VERBOSE
+    print("Exception occurred, process about to die silently\n");
+# endif
+    return EXCEPTION_EXECUTE_HANDLER; /* => global unwind and silent death */
+}
+
+
+int main(int argc, char** argv)
+{
+    HANDLE lib;
+    HANDLE event;
+    unsigned long hThread;
+    int reloaded = 0;
+
+    USE_USER32();
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+#ifdef LINUX
+    /* though win32/ */
+    intercept_signal(SIGSEGV, signal_handler);
+#else
+    /* note that can't normally if we have a debugger attached this
+     * will not get this executed */
+    SetUnhandledExceptionFilter((LPTOP_LEVEL_EXCEPTION_FILTER) our_top_handler);
+#endif
+
+    event = CreateEvent(NULL, TRUE, FALSE, NULL);
+    hThread = _beginthreadex(NULL, 0, run_func, event, 0, NULL);
+
+    /* need to run as long as necessary to hit the required faults */
+    while (transitions < NUM_TRANSITIONS) {
+        lib = LoadLibrary("reload-race.dll.dll");
+        if (lib == NULL) {
+            print("error loading library\n");
+            break;
+        } else {
+            reloaded++;
+#if VERBOSE
+    print("reloaded %d times %d\n", reloaded);
+#endif
+            import1 = (funptr)
+                GetProcAddress(lib, (LPCSTR)"import_me1");
+            import2 = (funptr)
+                GetProcAddress(lib, (LPCSTR)"import_me2");
+            /* may want to explicitly sleep here but that may be slower */
+            if (reloaded % 2 == 0)
+                YIELD();
+            FreeLibrary(lib);
+            if (reloaded % 3 == 0)
+                YIELD();
+        }
+    }
+
+    SetEvent(event);
+    WaitForSingleObject((HANDLE)hThread, INFINITE);
+    print("main loop done\n");
+    check_mem_usage();
+#if VERBOSE
+    print("reloaded %d times %d\n", reloaded);
+#endif
+
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
diff --git a/suite/tests/win32/reload-race.dll.c b/suite/tests/win32/reload-race.dll.c
new file mode 100644
index 000000000..3b9c77f7c
--- /dev/null
+++ b/suite/tests/win32/reload-race.dll.c
@@ -0,0 +1,104 @@
+/* **********************************************************
+ * Copyright (c) 2006 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "tools.h"
+
+int
+fib(int n) {
+    if (n <= 1)
+        return 1;
+    return fib(n-1) + fib(n-2);
+}
+
+int
+fact(int n) {
+    if (n <= 1)
+        return 1;
+    return n * fact(n-1);
+}
+
+int  __declspec(dllexport)
+import_me1(int x)
+{
+    /* any printing here can not be matched */
+#ifdef VERBOSE
+    print("in import_me1 %d\n", x);
+    if (!properly_initalized) {
+        print("app race - DLL in invalid state\n");
+    }
+#endif
+    if (x % 2 == 0)
+        return fib(x);
+    else
+        return fact(x);
+}
+
+int  __declspec(dllexport)
+import_me2(int x)
+{
+    /* any printing here can not be matched */
+#ifdef VERBOSE
+    print("in import_me2 %d\n", x);
+    if (!properly_initalized) {
+        print("app race - DLL in invalid state\n");
+    }
+#endif
+    /* note that our IAT may not be properly initialized 
+     * (if DLL is not bound to kernel32.dll 
+     * 100271dc  000273b6
+     * 0:001> da reload_race_dll+000273b6
+     * 100273b6  "..Sleep"
+     */
+    /* so we may target the RVA here */
+    thread_yield();
+    /* .C shouldn't be shown on the way back */
+    if (x % 2 == 0)
+        return fib(x);
+    else
+        return fact(x);
+}
+
+int properly_initalized;
+
+BOOL APIENTRY 
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    switch (reason_for_call) {
+    case DLL_PROCESS_ATTACH:
+        properly_initalized = 1;
+        break;
+    case DLL_PROCESS_DETACH:
+        properly_initalized = 0;
+        break;        
+    }
+    return TRUE;
+}
diff --git a/suite/tests/win32/reload-race.template b/suite/tests/win32/reload-race.template
new file mode 100644
index 000000000..b469bee2c
--- /dev/null
+++ b/suite/tests/win32/reload-race.template
@@ -0,0 +1,7 @@
+made it in and out 10 transitions
+main loop done
+#if : --low --thin_client
+Memory check: pagefile usage is >= 900 KB, < 2816 KB
+#else
+Memory check: pagefile usage is < 900 KB
+#endif
diff --git a/suite/tests/win32/reload.c b/suite/tests/win32/reload.c
new file mode 100644
index 000000000..e2aabe30d
--- /dev/null
+++ b/suite/tests/win32/reload.c
@@ -0,0 +1,186 @@
+/* **********************************************************
+ * Copyright (c) 2005-2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* reload.c: tests repeatedly loads and unloads a dll, and executes from it
+ * in between, to stress our cache management
+ */
+
+#include <windows.h>
+#include <process.h> /* for _beginthreadex */
+#include "tools.h"
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+/* make as STRESS=1 for a performance test */
+
+#ifdef NIGHTLY_REGRESSION
+/* don't ask to compute fact or fib too high */
+#define MAX_FACT_FIB 30
+# define ITERS (2*MAX_FACT_FIB)
+#else
+/* PERF or STRESS */
+/* very low computation here */
+#define MAX_FACT_FIB 8
+# define ITERS 4000
+#endif
+#define VERBOSE 0
+
+/* we create a 2nd thread to complicate DR cache flushing */
+int WINAPI
+run_func(void * arg)
+{
+    WaitForSingleObject((HANDLE)arg, INFINITE);
+    return 0;
+}
+
+/* make sure DR isn't using too much memory and is actually freeing fragments */
+static void
+check_mem_usage()
+{
+    VM_COUNTERS mem;
+    get_process_mem_stats(GetCurrentProcess(), &mem);
+#if VERBOSE
+    print("Process Memory Statistics:\n");
+    print("\tPeak virtual size:         %6d KB\n",
+          mem.PeakVirtualSize/1024);
+    print("\tPeak working set size:     %6d KB\n",
+          mem.PeakWorkingSetSize/1024);
+    print("\tPeak paged pool usage:     %6d KB\n",
+          mem.QuotaPeakPagedPoolUsage/1024);
+    print("\tPeak non-paged pool usage: %6d KB\n",
+          mem.QuotaPeakNonPagedPoolUsage/1024);
+    print("\tPeak pagefile usage:       %6d KB\n",
+          mem.PeakPagefileUsage/1024);
+#endif
+/*
+  native:
+        Peak virtual size:           7772 KB
+        Peak working set size:        996 KB
+        Peak paged pool usage:          8 KB
+        Peak non-paged pool usage:      1 KB
+        Peak pagefile usage:          352 KB
+DR results with debug build, where library takes up more WSS (one
+reason we use pagefile usage as our discerning factor):
+  -no_shared_deletion:
+        Peak virtual size:         140388 KB
+        Peak working set size:       7948 KB
+        Peak paged pool usage:          9 KB
+        Peak non-paged pool usage:      1 KB
+        Peak pagefile usage:         6536 KB
+  -no_syscalls_synch_flush:
+        Peak virtual size:         140388 KB
+        Peak working set size:       7784 KB
+        Peak paged pool usage:          9 KB
+        Peak non-paged pool usage:      1 KB
+        Peak pagefile usage:         6368 KB
+  -no_cache_shared_free_list:
+        Peak virtual size:         140388 KB
+        Peak working set size:       5156 KB
+        Peak paged pool usage:          9 KB
+        Peak non-paged pool usage:      1 KB
+        Peak pagefile usage:         3736 KB
+  defaults:
+        Peak virtual size:         140388 KB
+        Peak working set size:       3352 KB
+        Peak paged pool usage:          9 KB
+        Peak non-paged pool usage:      1 KB
+        Peak pagefile usage:         1680 KB
+*/
+    /* native */
+#if VERBOSE
+    print("Pagefile usage is %d KB\n", mem.PeakPagefileUsage/1024);
+#endif
+    if (mem.PeakPagefileUsage < 900*1024)
+        print("Memory check: pagefile usage is < 900 KB\n");
+    /* typical DR */
+    else if (mem.PeakPagefileUsage < 2816*1024)
+        print("Memory check: pagefile usage is >= 900 KB, < 2816 KB\n");
+    /* prof_pcs uses a buffer the size of DR.dll */
+    else if (mem.PeakPagefileUsage < 6000*1024)
+        print("Memory check: pagefile usage is >= 2816 KB, < 6000 KB\n");
+    /* detect_dangling_fcache doesn't free fcache */
+    else if (mem.PeakPagefileUsage < 16384*1024)
+        print("Memory check: pagefile usage is >= 6000 KB, < 16384 KB\n");
+    else {
+        /* give actual number here so we can see how high it is */
+        print("Memory check: pagefile usage is %d KB >= 16384 KB\n",
+              mem.PeakPagefileUsage/1024);
+    }
+}
+
+int main(int argc, char** argv)
+{
+    HANDLE lib;
+    HANDLE event;
+    int i;
+    unsigned long hThread;
+    int sum = 0;
+
+    USE_USER32();
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    event = CreateEvent(NULL, TRUE, FALSE, NULL);
+    hThread = _beginthreadex(NULL, 0, run_func, event, 0, &i);
+
+    for (i=0; i<ITERS; i++) {
+        lib = LoadLibrary("reload.dll.dll");
+        if (lib == NULL) {
+            print("error loading library\n");
+            break;
+        } else {
+            /* don't ask to compute fact or fib too high */
+            BOOL (WINAPI *proc)(DWORD);
+            proc = (BOOL (WINAPI *)(DWORD))
+                GetProcAddress(lib, (LPCSTR)"import_me");
+            sum += (*proc)(i % MAX_FACT_FIB);
+            FreeLibrary(lib);
+        }
+    }
+
+    SetEvent(event);
+    WaitForSingleObject((HANDLE)hThread, INFINITE);
+
+    print("sum=%d\n", sum);
+    check_mem_usage();
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
diff --git a/suite/tests/win32/reload.dll.c b/suite/tests/win32/reload.dll.c
new file mode 100644
index 000000000..c4c67420e
--- /dev/null
+++ b/suite/tests/win32/reload.dll.c
@@ -0,0 +1,71 @@
+/* **********************************************************
+ * Copyright (c) 2005-2006 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "tools.h"
+
+int
+fib(int n) {
+    if (n <= 1)
+        return 1;
+    return fib(n-1) + fib(n-2);
+}
+
+int
+fact(int n) {
+    if (n <= 1)
+        return 1;
+    return n * fact(n-1);
+}
+
+int  __declspec(dllexport)
+import_me(int x)
+{
+#ifdef NIGHTLY_REGRESSION
+    print("in import %d\n", x);
+#endif
+    if (x % 2 == 0)
+        return fib(x);
+    else
+        return fact(x);
+}
+
+BOOL APIENTRY 
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    switch (reason_for_call) {
+    case DLL_PROCESS_ATTACH:
+        break;
+    case DLL_PROCESS_DETACH:
+        break;        
+    }
+    return TRUE;
+}
diff --git a/suite/tests/win32/reload.template b/suite/tests/win32/reload.template
new file mode 100755
index 000000000..a516814ed
--- /dev/null
+++ b/suite/tests/win32/reload.template
@@ -0,0 +1,77 @@
+in import 0
+in import 1
+in import 2
+in import 3
+in import 4
+in import 5
+in import 6
+in import 7
+in import 8
+in import 9
+in import 10
+in import 11
+in import 12
+in import 13
+in import 14
+in import 15
+in import 16
+in import 17
+in import 18
+in import 19
+in import 20
+in import 21
+in import 22
+in import 23
+in import 24
+in import 25
+in import 26
+in import 27
+in import 28
+in import 29
+in import 0
+in import 1
+in import 2
+in import 3
+in import 4
+in import 5
+in import 6
+in import 7
+in import 8
+in import 9
+in import 10
+in import 11
+in import 12
+in import 13
+in import 14
+in import 15
+in import 16
+in import 17
+in import 18
+in import 19
+in import 20
+in import 21
+in import 22
+in import 23
+in import 24
+in import 25
+in import 26
+in import 27
+in import 28
+in import 29
+sum=-1314123346
+#if -WINDOWS : || +WINDOWS : +-thin_client || +WINDOWS : +-low 
+// native
+Memory check: pagefile usage is < 900 KB
+#else
+# if +DEBUG : +-detect_dangling_fcache
+// not freeing fcache => more usage
+Memory check: pagefile usage is >= 6000 KB, < 16384 KB
+# else
+#  if +DEBUG : +-prof_pcs
+// DR profile bucket == DR dll size!
+Memory check: pagefile usage is >= 2816 KB, < 6000 KB
+#  else
+Memory check: pagefile usage is >= 900 KB, < 2816 KB
+#  endif // -detect_dangling_fcache
+# endif // -prof_pcs
+#endif // native
diff --git a/suite/tests/win32/rsbtest.c b/suite/tests/win32/rsbtest.c
new file mode 100644
index 000000000..f8bed8858
--- /dev/null
+++ b/suite/tests/win32/rsbtest.c
@@ -0,0 +1,265 @@
+/* **********************************************************
+ * Copyright (c) 2005-2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* single RET - many targets */
+/* 
+ * ok - we have 10s for PLAIN_RET
+ * - and 26s for the pop ecx/jmp ecx scheme
+ * for a single call bar
+ *  
+ *  How to make the difference worse - consider this is going to a single place only?
+ * for 3 consecutive call bar
+ *   16s RET, vs 41s on POP/JMP ECX = 2.56 same ratio
+ *   17s POP/PUSH/RET;  foo_with_extras
+
+*    68s wow a PUSH/JMP paired with RET is 1m8.571s  PLAIN_RET but fancy_CALL
+*    48s for a PUSH/JMP paired with a POP/JMP   0m48.597s  -- same ratio here - 
+        so an extra CALL doesn't hurt as bad as an extra RET!
+
+
+ * that was for #define ITER 500*100000 // *100
+
+$ cl /O2  /Zi foo.c -I.. /link /incremental:no user32.lib
+
+ */
+
+
+/* undefine this for a performance test */
+#define NIGHTLY_REGRESSION
+
+#include "tools.h"
+
+#include <stdio.h>
+
+#ifdef WINDOWS
+/* and compile with user32.lib */
+#include <windows.h>
+#endif
+
+#define GOAL 32 /* recursive fib of course is exponential here */
+
+#ifdef NIGHTLY_REGRESSION
+#  define ITER 10*1000
+#else
+#  define ITER 500*100000
+#endif
+
+#define DEPTH 10
+
+#define PLAIN_RET
+//#define JMP_ESP
+
+#define PLAIN_CALL
+
+
+int 
+foo(int n) {                    /* CFI */
+    __asm {
+        /* pay ecx penalty for both cases */
+        push ecx
+#ifdef PLAIN_CALL
+        call bar
+ac1:
+        call bar
+ac2:
+        call bar
+ac3:
+#else  /* CALL=PUSH/JMP */
+        push offset ac1
+        jmp bar
+ac1:
+        push offset ac2
+        jmp bar
+ac2:
+        push offset ac3
+        jmp bar
+ac3:
+#endif
+        pop ecx
+        jmp done
+
+bar:
+        mov eax, 5
+#ifdef PLAIN_RET
+        ret
+#else
+#ifdef JMP_ESP
+        add esp, 4  // is it safe to use the stack in-between these two ops?
+        jmp dword ptr [esp-4]
+#else
+        pop ecx
+        jmp ecx
+#endif
+#endif
+
+     done:
+    }
+    return 5;
+}
+
+int 
+foo_second(int n) {
+    __asm {
+        /* pay ecx penalty for both cases */
+        push ecx
+#ifdef PLAIN_CALL
+        call bar
+ac1:
+        call bar
+ac2:
+        call bar
+ac3:
+#else  /* CALL=PUSH/JMP */
+        push offset ac1
+        jmp bar
+ac1:
+        push offset ac2
+        jmp bar
+ac2:
+        push offset ac3
+        jmp bar
+ac3:
+#endif
+        pop ecx
+        jmp done
+
+bar:
+        mov eax, 5
+#ifdef PLAIN_RET
+        ret
+#else
+        pop ecx
+        jmp ecx
+#endif
+
+     done:
+    }
+    return 5;
+}
+
+int 
+foo_with_extras(int n) {
+    __asm {
+        /* pay ecx penalty for both cases */
+        push ecx
+        call bar
+        call bar
+        call bar
+ac:
+        pop ecx
+        jmp done
+
+bar:
+        mov eax, 5
+#ifdef PLAIN_RET
+        push ecx                /* actually adding one more instruction that the pop/jmp*/
+        pop ecx
+        ret
+#else
+        pop ecx
+        jmp ecx
+#endif
+
+     done:
+    }
+    return 5;
+}
+
+
+int 
+foo_first(int n) {
+    __asm {
+        /* pay ecx penalty for both cases */
+        push ecx
+        call bar
+        call bar
+        call bar
+ac:
+        pop ecx
+        jmp done
+
+bar:
+        mov eax, 5
+#ifdef PLAIN_RET
+        ret
+#else
+        pop ecx
+        jmp ecx
+#endif
+
+     done:
+    }
+    return 5;
+}
+
+
+int
+main(int argc, char** argv) 
+{
+    int i, t;
+    
+    /* now a little more realistic depths that fit in the RSB */
+    for (i=0; i<=ITER; i++) {
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+	t = foo(DEPTH);
+    }
+
+    print("foo(%d)=%d\n", DEPTH, t);
+
+    if (argc > 5) 
+        MessageBeep(0);
+}
diff --git a/suite/tests/win32/rsbtest.template b/suite/tests/win32/rsbtest.template
new file mode 100644
index 000000000..02c59ee9a
--- /dev/null
+++ b/suite/tests/win32/rsbtest.template
@@ -0,0 +1 @@
+foo(10)=5
diff --git a/suite/tests/win32/section-max.c b/suite/tests/win32/section-max.c
new file mode 100644
index 000000000..a90d36c91
--- /dev/null
+++ b/suite/tests/win32/section-max.c
@@ -0,0 +1,63 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include <stdio.h>
+#include "tools.h"
+
+#define VERBOSE 0
+
+HMODULE
+myload(char* lib) 
+{
+    HMODULE hm = LoadLibrary(lib);
+    if (hm == NULL) {
+        print("error %d loading library %s\n", GetLastError(), lib);
+    } else {
+        BOOL res;
+        BOOL (WINAPI *proc)(DWORD);
+        print("loaded %s\n", lib);
+#if VERBOSE
+        print("library is at "PFX"\n", hm);
+#endif
+    }
+    return hm;
+}
+
+int main()
+{
+    HMODULE lib;
+    lib = myload("section-max.dll.dll");
+    FreeLibrary(lib);
+
+    return 0;
+}
diff --git a/suite/tests/win32/section-max.dll.c b/suite/tests/win32/section-max.dll.c
new file mode 100644
index 000000000..8723a3301
--- /dev/null
+++ b/suite/tests/win32/section-max.dll.c
@@ -0,0 +1,655 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* case 8514 testing many sections - only up to 96 are allowed by the
+ *       XP SP2 loader
+ *
+ * case 6772 testing funny flags - though not really excercising the
+ * implied change in functionality: should see if they are acted upon
+ * - e.g. is discard in memory, is .shared indeed shared between
+ * processes, etc.
+ *
+ *
+ * note we don't have too many code sections
+ */
+
+#include "tools.h"
+#include <windows.h>
+
+/* also preserving alignment test from secalign-fixed.dll.c */
+
+/* Linker requires /driver if specifying /align.
+ * Documentation says that "The linker will perform some special
+ * optimizations if this option is selected." -- not sure if any other changes.
+ */
+#pragma comment(linker, "/align:0x2000 /driver")
+
+#pragma comment(linker, "/SECTION:.shared,RWS")
+#pragma data_seg(".shared")
+int shared1 = 1;
+int shared2;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.discard,RWD")
+#pragma data_seg(".discard")
+int discard1 = 2;
+int discard2;
+#pragma data_seg() 
+
+/* FIXME: case 8677 keeps track of PAGE_NOCACHE problems with ASLR !K*/
+#pragma comment(linker, "/SECTION:.nocache,RK")
+#pragma data_seg(".nocache")
+int nocache1 = 3;
+int nocache2;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.nopage,RW!P")
+#pragma data_seg(".nopage")
+int nopage1 = 4;
+int nopage2;
+#pragma data_seg() 
+
+
+/*
+E Execute The section is executable 
+R Read Allows read operations on data 
+W Write Allows write operations on data 
+
+S Shared Shares the section among all processes that load the image 
+D Discardable Marks the section as discardable 
+K Cacheable Marks the section as not cacheable 
+P Pageable Marks the section as not pageable 
+
+L Preload VxD only; marks the section as preload 
+X Memory-resident VxD only; marks the section as memory-resident 
+*/
+
+#pragma comment(linker, "/SECTION:.erw0,ERW")
+#pragma data_seg(".erw0") 
+int erw1 = 5;
+int erw2;
+#pragma data_seg() 
+
+/* FIXME: we can't use _Pragma as we do in core in ACTUAL_PRAGMA since we're
+ * not using gcc as a preprocessor here! */
+/* Use special C99 operator _Pragma to generate a pragma from a macro */
+#define ACTUAL_PRAGMA(p) _Pragma ( #p )
+#define START_DATA_SECTION(name, prot) ACTUAL_PRAGMA( data_seg(name) )
+#define VAR_IN_SECTION(name) /* nothing */
+#define END_DATA_SECTION() ACTUAL_PRAGMA( data_seg() )
+
+#define SECTION_SET(id)                         \
+        START_DATA_SECTION(".rx" #id, "rx")     \
+        int rx##id;                             \
+        END_DATA_SECTION()                      \
+        START_DATA_SECTION(".rwx" #id, "rwx")   \
+        int rwx##id;                            \
+        END_DATA_SECTION()                      \
+        START_DATA_SECTION(".r" #id, "r")       \
+        int r##id;                              \
+        END_DATA_SECTION()
+
+/* FIXME: plan was to just use SECTION_SET(1) ... 
+ * but for now doing manually
+ */
+
+
+/* add alignment - has to be smaller than the /ALIGN option above */
+#pragma comment(linker, "/SECTION:.awer5,WER,ALIGN:0x1000")
+#pragma data_seg(".awer5") 
+int awer5 = 5;
+#pragma data_seg() 
+
+/* ================= */
+/* iterate through ER and ERW (though with high alignment already separate allocations) */
+#pragma comment(linker, "/SECTION:.cer1,ER")
+#pragma code_seg(".cer1") 
+int cer1 = 5;
+/* this is supposed to crash if ever run since not writable */
+void funcer1() { cer1 = 1;}
+#pragma code_seg() 
+
+/* does it matter what is what? */
+#pragma comment(linker, "/SECTION:.cwer1,WER")
+#pragma data_seg(".cwer1") 
+int cwer1 = 5;
+/* forcing a relocation */
+void funccwer1() { cwer1 = 1;}
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.cer2,ER")
+#pragma code_seg(".cer2") 
+int cer2 = 5;
+void funcer2() { cer2 = 2;}
+#pragma code_seg() 
+
+#pragma comment(linker, "/SECTION:.cer3,ER")
+#pragma code_seg(".cer3") 
+int cer3 = 5;
+void funcer3() { cer3 = 3;}
+#pragma code_seg() 
+/* FIXME: could add more code sections, but to avoid triggering the
+ * curiosity in add_rct_module() not adding many code sections */
+
+#pragma bss_seg(".bss1") 
+int bss1;
+int bss2;
+#pragma bss_seg() 
+
+#pragma const_seg(".rdata1")
+const char hello1[] = "hello world";
+const char hello2[] = "hello world";
+
+const char hello3[] = "hello new world";
+#pragma const_seg() 
+
+/* iterate through ER and ERW (though with high alignment already separate allocations) */
+#pragma comment(linker, "/SECTION:.er1,ER")
+#pragma data_seg(".er1") 
+int er1 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer1,WER")
+#pragma data_seg(".wer1") 
+int wer1 = 5;
+#pragma data_seg() 
+
+
+/* FIXME: just doing manually in emacs for now 
+ *  (query-replace "er1" "er2" nil nil nil)
+ */
+#pragma comment(linker, "/SECTION:.er2,ER")
+#pragma data_seg(".er2") 
+int er2 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer2,WER")
+#pragma data_seg(".wer2") 
+int wer2 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er3,ER")
+#pragma data_seg(".er3") 
+int er3 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer3,WER")
+#pragma data_seg(".wer3") 
+int wer3 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er4,ER")
+#pragma data_seg(".er4") 
+int er4 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer4,WER")
+#pragma data_seg(".wer4") 
+int wer4 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er5,ER")
+#pragma data_seg(".er5") 
+int er5 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer5,WER")
+#pragma data_seg(".wer5") 
+int wer5 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er6,ER")
+#pragma data_seg(".er6") 
+int er6 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer6,WER")
+#pragma data_seg(".wer6") 
+int wer6 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er7,ER")
+#pragma data_seg(".er7") 
+int er7 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer7,WER")
+#pragma data_seg(".wer7") 
+int wer7 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er8,ER")
+#pragma data_seg(".er8") 
+int er8 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer8,WER")
+#pragma data_seg(".wer8") 
+int wer8 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er9,ER")
+#pragma data_seg(".er9") 
+int er9 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer9,WER")
+#pragma data_seg(".wer9") 
+int wer9 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er10,ER")
+#pragma data_seg(".er10") 
+int er10 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer10,WER")
+#pragma data_seg(".wer10") 
+int wer10 = 5;
+#pragma data_seg() 
+
+
+/* now replacing er with er20 */
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er201,ER")
+#pragma data_seg(".er201") 
+int er201 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer201,WER")
+#pragma data_seg(".wer201") 
+int wer201 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.er202,ER")
+#pragma data_seg(".er202") 
+int er202 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer202,WER")
+#pragma data_seg(".wer202") 
+int wer202 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er203,ER")
+#pragma data_seg(".er203") 
+int er203 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer203,WER")
+#pragma data_seg(".wer203") 
+int wer203 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er204,ER")
+#pragma data_seg(".er204") 
+int er204 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer204,WER")
+#pragma data_seg(".wer204") 
+int wer204 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er205,ER")
+#pragma data_seg(".er205") 
+int er205 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer205,WER")
+#pragma data_seg(".wer205") 
+int wer205 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er206,ER")
+#pragma data_seg(".er206") 
+int er206 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer206,WER")
+#pragma data_seg(".wer206") 
+int wer206 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er207,ER")
+#pragma data_seg(".er207") 
+int er207 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer207,WER")
+#pragma data_seg(".wer207") 
+int wer207 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er208,ER")
+#pragma data_seg(".er208") 
+int er208 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer208,WER")
+#pragma data_seg(".wer208") 
+int wer208 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er209,ER")
+#pragma data_seg(".er209") 
+int er209 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer209,WER")
+#pragma data_seg(".wer209") 
+int wer209 = 5;
+#pragma data_seg() 
+
+
+/* now replacing er20 with er30 */
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er301,ER")
+#pragma data_seg(".er301") 
+int er301 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer301,WER")
+#pragma data_seg(".wer301") 
+int wer301 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.er302,ER")
+#pragma data_seg(".er302") 
+int er302 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer302,WER")
+#pragma data_seg(".wer302") 
+int wer302 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er303,ER")
+#pragma data_seg(".er303") 
+int er303 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer303,WER")
+#pragma data_seg(".wer303") 
+int wer303 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er304,ER")
+#pragma data_seg(".er304") 
+int er304 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer304,WER")
+#pragma data_seg(".wer304") 
+int wer304 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er305,ER")
+#pragma data_seg(".er305") 
+int er305 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer305,WER")
+#pragma data_seg(".wer305") 
+int wer305 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er306,ER")
+#pragma data_seg(".er306") 
+int er306 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer306,WER")
+#pragma data_seg(".wer306") 
+int wer306 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er307,ER")
+#pragma data_seg(".er307") 
+int er307 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer307,WER")
+#pragma data_seg(".wer307") 
+int wer307 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er308,ER")
+#pragma data_seg(".er308") 
+int er308 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer308,WER")
+#pragma data_seg(".wer308") 
+int wer308 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er309,ER")
+#pragma data_seg(".er309") 
+int er309 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer309,WER")
+#pragma data_seg(".wer309") 
+int wer309 = 5;
+#pragma data_seg() 
+
+/* now replacing er20 with er40 */
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er401,ER")
+#pragma data_seg(".er401") 
+int er401 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer401,WER")
+#pragma data_seg(".wer401") 
+int wer401 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.er402,ER")
+#pragma data_seg(".er402") 
+int er402 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer402,WER")
+#pragma data_seg(".wer402") 
+int wer402 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er403,ER")
+#pragma data_seg(".er403") 
+int er403 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer403,WER")
+#pragma data_seg(".wer403") 
+int wer403 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er404,ER")
+#pragma data_seg(".er404") 
+int er404 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer404,WER")
+#pragma data_seg(".wer404") 
+int wer404 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er405,ER")
+#pragma data_seg(".er405") 
+int er405 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer405,WER")
+#pragma data_seg(".wer405") 
+int wer405 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er406,ER")
+#pragma data_seg(".er406") 
+int er406 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer406,WER")
+#pragma data_seg(".wer406") 
+int wer406 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er407,ER")
+#pragma data_seg(".er407") 
+int er407 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer407,WER")
+#pragma data_seg(".wer407") 
+int wer407 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er408,ER")
+#pragma data_seg(".er408") 
+int er408 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer408,WER")
+#pragma data_seg(".wer408") 
+int wer408 = 5;
+#pragma data_seg() 
+
+/* --- */
+#pragma comment(linker, "/SECTION:.er409,ER")
+#pragma data_seg(".er409") 
+int er409 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer409,WER")
+#pragma data_seg(".wer409") 
+int wer409 = 5;
+#pragma data_seg() 
+
+/* now replacing er20 with er50 */
+/* --- */
+#pragma comment(linker, "/SECTION:.er501,ER")
+#pragma data_seg(".er501") 
+int er501 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer501,WER")
+#pragma data_seg(".wer501") 
+int wer501 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.er502,ER")
+#pragma data_seg(".er502") 
+int er502 = 5;
+#pragma data_seg() 
+
+#pragma comment(linker, "/SECTION:.wer502,WER")
+#pragma data_seg(".wer502") 
+int wer502 = 5;
+#pragma data_seg() 
+
+#if 0 /* leaving this in goes over the WOW64 xp loader limit so removing */
+/* --- */
+#pragma comment(linker, "/SECTION:.er503,ER")
+#pragma data_seg(".er503") 
+int er503 = 5;
+#pragma data_seg() 
+#endif
+
+/* 96 if we stop adding .er50x here */
+
+
+/* although dumpbin has no problems with 107 sections */
+/* The windows XP SP2 loader still maintains this limit
+ * ---------------------------
+ * section-max.exe - Bad Image
+ * ---------------------------
+ * The application or DLL i:\vlk\trees\tot\suite\tests\security-win32\section-max.dll.dll is not a valid Windows image. Please check this against your installation diskette. 
+ * ---------------------------
+ * OK   
+ * ---------------------------
+ * > error loading library section-max.dll.dll
+ *
+ */
+
+
+
+int
+__declspec(dllexport)
+make_a_lib(int arg)
+{
+    shared2 = 101;
+    return shared1 + discard1 + nocache1 + nopage1 + erw1;
+}
+
+BOOL APIENTRY 
+DllMain(HANDLE hModule, DWORD reason_for_call, LPVOID Reserved)
+{
+    switch (reason_for_call) {
+    case DLL_PROCESS_ATTACH:
+        print("in section max dll\n");
+        break;
+    }
+    return TRUE;
+}
diff --git a/suite/tests/win32/section-max.template b/suite/tests/win32/section-max.template
new file mode 100644
index 000000000..b36b93b2b
--- /dev/null
+++ b/suite/tests/win32/section-max.template
@@ -0,0 +1,2 @@
+in section max dll
+loaded section-max.dll.dll
diff --git a/suite/tests/win32/setcxtsyscall.c b/suite/tests/win32/setcxtsyscall.c
new file mode 100644
index 000000000..b7d21666c
--- /dev/null
+++ b/suite/tests/win32/setcxtsyscall.c
@@ -0,0 +1,195 @@
+/* **********************************************************
+ * Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* a test for calling NtSetContextThread on a thread at a system call 
+
+interestingly here's what happens to the registers (not outputing since not
+machine-independent):
+
+suspended@: 00000000 00000000 00334cd0 00420c78 00000000 00000000 0052ff88 0052ffb8
+setting to: ffffffff ffffffff ffffffff ffffffff ffffffff ffffffff 0052ff88 ffffffff
+result:     00000102 ffffffff 0052ff88 004161db ffffffff ffffffff 0052ff88 ffffffff
+*/
+
+#include "tools.h"
+#include "Windows.h"
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+#define VERBOSE 0
+
+DWORD control;
+DWORD transfer_addr;
+
+static int reg_eax;
+static int reg_ebx;
+static int reg_ecx;
+static int reg_edx;
+static int reg_edi;
+static int reg_esi;
+static int reg_ebp;
+static int reg_esp;
+
+#define TIMER_UNITS_PER_MILLISECOND (1000 * 10) /* 100ns intervals */
+
+static DWORD WINAPI
+ThreadProc1(LPVOID parm) 
+{
+    LARGE_INTEGER waittime;
+    NTSTATUS res;
+    HANDLE e;
+    GET_NTDLL(NtWaitForSingleObject, (IN HANDLE ObjectHandle, 
+                                      IN BOOLEAN Alertable, 
+                                      IN PLARGE_INTEGER TimeOut ));
+    print("starting thread...\n");
+
+    e = CreateEvent(NULL, FALSE, FALSE, "foo");
+    waittime.QuadPart = -((int)500 * TIMER_UNITS_PER_MILLISECOND);
+    control = 1;
+    do {
+        res = NtWaitForSingleObject(e, false /* not alertable */, &waittime);
+    } while (control);
+    __asm {
+        mov   reg_eax, eax
+        mov   reg_ebx, ebx
+        mov   reg_ecx, ecx
+        mov   reg_edx, edx
+        mov   reg_edi, edi
+        mov   reg_esi, esi
+        mov   reg_esp, esp
+        mov   reg_ebp, ebp
+    }
+    print("res is "PFMT" but shouldn't get here!!!\n", res);
+#if VERBOSE
+    print("registers: "PFMT" "PFMT" "PFMT" "PFMT" "PFMT" "PFMT" "PFMT" "PFMT"\n",
+           reg_eax, reg_ebx, reg_ecx, reg_edx, reg_edi, reg_esi, reg_esp, reg_ebp);
+#endif
+    CloseHandle(e);
+
+    print("exiting thread\n");
+    return -1;
+}
+
+static void
+transferProc()
+{
+    __asm {
+        call next_instr
+      next_instr:
+        cmp  control, 0
+        jne  transferout
+
+        pop  edx
+        mov  transfer_addr, edx
+    }
+
+    print("&next_instr recorded\n");
+    return;
+
+ transferout:
+    __asm {
+        mov   reg_eax, eax
+        mov   reg_ebx, ebx
+        mov   reg_ecx, ecx
+        mov   reg_edx, edx
+        mov   reg_edi, edi
+        mov   reg_esi, esi
+        mov   reg_esp, esp
+        mov   reg_ebp, ebp
+    }
+#if VERBOSE
+    print("result:     "PFMT" "PFMT" "PFMT" "PFMT" "PFMT" "PFMT" "PFMT" "PFMT"\n",
+           reg_eax, reg_ebx, reg_ecx, reg_edx, reg_edi, reg_esi, reg_esp, reg_ebp);
+#endif
+    print("control has been redirected.\n");
+    /* don't try to restore stack across sysenter, etc. */
+    ExitThread(0);
+}
+
+static int 
+main(void) 
+{
+    HANDLE ht;
+    DWORD tid;
+    CONTEXT tc;
+    
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    control = 0;
+    transfer_addr = 0;
+
+    /* call this to obtain the address of next_instr */
+    transferProc();
+
+    ht = CreateThread(NULL, 0, &ThreadProc1, NULL, 0, &tid);
+
+    while (control == 0)
+        ; /* wait for thread to set control */
+ 
+    SuspendThread(ht);
+    print("thread suspended.\n");
+
+    tc.ContextFlags = CONTEXT_CONTROL | CONTEXT_INTEGER;
+    GetThreadContext(ht, &tc);
+#if VERBOSE
+    print("suspended@: "PFMT" "PFMT" "PFMT" "PFMT" "PFMT" "PFMT" "PFMT" "PFMT"\n",
+           tc.CXT_XAX, tc.CXT_XBX, tc.CXT_XCX, tc.CXT_XDX, tc.CXT_XDI, tc.CXT_XSI, tc.CXT_XSP, tc.CXT_XBP);
+#endif
+    tc.CXT_XIP = transfer_addr;
+    tc.CXT_XAX = 0xffffffff;
+    tc.CXT_XBX = 0xffffffff;
+    tc.CXT_XCX = 0xffffffff;
+    tc.CXT_XDX = 0xffffffff;
+    tc.CXT_XDI = 0xffffffff;
+    tc.CXT_XSI = 0xffffffff;
+    tc.CXT_XBP = 0xffffffff;
+#if VERBOSE
+    print("setting to: "PFMT" "PFMT" "PFMT" "PFMT" "PFMT" "PFMT" "PFMT" "PFMT"\n",
+           tc.CXT_XAX, tc.CXT_XBX, tc.CXT_XCX, tc.CXT_XDX, tc.CXT_XDI, tc.CXT_XSI, tc.CXT_XSP, tc.CXT_XBP);
+#endif
+    SetThreadContext(ht, &tc);
+
+    ResumeThread(ht);
+    WaitForSingleObject(ht, INFINITE);
+    
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+
+    return 0;
+}
diff --git a/suite/tests/win32/setcxtsyscall.expect b/suite/tests/win32/setcxtsyscall.expect
new file mode 100755
index 000000000..269f8c551
--- /dev/null
+++ b/suite/tests/win32/setcxtsyscall.expect
@@ -0,0 +1,4 @@
+&next_instr recorded
+starting thread...
+thread suspended.
+control has been redirected.
diff --git a/suite/tests/win32/setthreadcontext.c b/suite/tests/win32/setthreadcontext.c
new file mode 100644
index 000000000..cb8a5dd9c
--- /dev/null
+++ b/suite/tests/win32/setthreadcontext.c
@@ -0,0 +1,156 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include "tools.h"
+#include <windows.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+DWORD control;
+DWORD transfer_addr;
+
+static DWORD WINAPI
+ThreadProc1(LPVOID parm) 
+{
+    print("starting thread...\n");
+
+    control = 1;
+
+    __asm {
+      onemoretime:
+        cmp  control, 1
+        je   onemoretime
+    }
+
+    print("exiting thread\n");
+    return -1;
+}
+
+static void
+transferProc()
+{
+    __asm {
+        call next_instr
+      next_instr:
+        cmp  control, 0
+        jne  transferout
+
+        pop  edx
+        mov  transfer_addr, edx
+    }
+
+    print("&next_instr recorded\n");
+    return;
+
+ transferout:
+    print("control has been redirected.\n");
+    return;
+}
+
+static DWORD WINAPI
+setregproc(LPVOID parm)
+{
+    control = 1;
+    __asm {
+        mov  ecx, 0xaaaaaaaa        
+      loopagain:
+        cmp  ecx, 0
+        jne  loopagain
+    }
+
+    print("ecx was set, exiting\n");
+    return;
+}
+
+static int 
+main(void) 
+{
+    HANDLE ht;
+    DWORD tid;
+    CONTEXT tc;
+    
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    control = 0;
+    transfer_addr = 0;
+
+    /* call this to obtain the address of next_instr */
+    transferProc();
+
+    ht = CreateThread(NULL, 0, &ThreadProc1, NULL, 0, &tid);
+
+    while (control == 0)
+        ; /* wait for thread to set control */
+ 
+    SuspendThread(ht);
+    print("thread suspended.\n");
+
+    tc.ContextFlags = CONTEXT_CONTROL;
+    GetThreadContext(ht, &tc);
+
+    tc.CXT_XIP = transfer_addr;
+    SetThreadContext(ht, &tc);
+
+    ResumeThread(ht);
+    WaitForSingleObject(ht, INFINITE);
+    
+    /* all over again for CONTEXT_INTEGER */
+    control = 0;
+    ht = CreateThread(NULL, 0, &setregproc, NULL, 0, &tid);
+
+    while (control == 0)
+        ; /* wait for thread to set control */
+ 
+    SuspendThread(ht);
+    print("thread suspended.\n");
+
+    tc.ContextFlags = CONTEXT_INTEGER;
+    GetThreadContext(ht, &tc);
+
+    tc.CXT_XCX = 0;
+    SetThreadContext(ht, &tc);
+
+    ResumeThread(ht);
+    WaitForSingleObject(ht, INFINITE);
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+
+    return 0;
+}
diff --git a/suite/tests/win32/setthreadcontext.expect b/suite/tests/win32/setthreadcontext.expect
new file mode 100644
index 000000000..33069c0cb
--- /dev/null
+++ b/suite/tests/win32/setthreadcontext.expect
@@ -0,0 +1,6 @@
+&next_instr recorded
+starting thread...
+thread suspended.
+control has been redirected.
+thread suspended.
+ecx was set, exiting
diff --git a/suite/tests/win32/suspend.c b/suite/tests/win32/suspend.c
new file mode 100644
index 000000000..8f25a116c
--- /dev/null
+++ b/suite/tests/win32/suspend.c
@@ -0,0 +1,190 @@
+/* **********************************************************
+ * Copyright (c) 2003-2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define _WIN32_WINNT 0x0400
+#include <windows.h>
+#include <stdio.h>
+
+#define DO_SIMPLE_SUSPEND_TEST 1
+/* output = 
+ * starting thread...suspended(count = 0)...resuming...exiting thread...resumed(count = 1)
+ */
+
+#define DO_SYNCH_WITH_SUSPEND_SELF_TEST 1
+/* output = 
+ * suspended(count = 1)...resumed,exiting
+ */
+
+#define DO_SYNCH_WITH_ALL_SUSPEND_SELF_TEST 1
+/* ouput = 
+ * Testing exiting with self suspended thread.
+ */
+
+#define SLEEP_FOR_NUDGE 0
+/* Sleep for 20 sec while one thread has suspended itself in
+ * DO_SYNCH_WITH_SUSPEND_SELF_TEST so can manually test nudges that synch with the app
+ * in this state (detach, reset, hotp_update etc.). */
+
+/* All combinations finish with :
+ * done
+ */
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+BOOL synch_1 = TRUE;
+BOOL synch_2 = TRUE;
+
+DWORD WINAPI
+ThreadProc1(LPVOID param) 
+{
+    printf("starting thread...");
+    fflush(stdout);
+    synch_2 = FALSE;
+    while (synch_1) {
+        SwitchToThread();
+    }
+    printf("exiting thread...");
+    fflush(stdout);
+}
+
+DWORD WINAPI
+ThreadProc2(LPVOID param)
+{
+    SuspendThread(GetCurrentThread());
+    printf("resumed,exiting\n");
+    fflush(stdout);
+}
+
+#define CHECK_SUSPEND_COUNT(val, expect) do {                                    \
+        if ((val) != (expect)) {                                                 \
+            printf("\nfailure, suspend count is %d instead of %d on line %d\n",  \
+                   (val), (expect), __LINE__);                                   \
+        }                                                                        \
+    } while (0)
+
+int 
+main(void) 
+{
+    HANDLE ht;
+    DWORD tid;
+    DWORD res;
+    
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+    
+#if DO_SIMPLE_SUSPEND_TEST
+    ht=CreateThread(NULL, 0, &ThreadProc1, NULL, 0, &tid);
+    
+    while (synch_2) {
+        SwitchToThread();
+    }
+  
+    res = SuspendThread(ht);
+    printf("suspended(count = %d)...", res);
+    fflush(stdout);
+
+    synch_1 = FALSE;
+
+    printf("resuming...");
+    fflush(stdout);
+    SwitchToThread();
+    res = ResumeThread(ht);
+
+    WaitForSingleObject(ht, INFINITE);
+    printf("resumed(count = %d)\n", res);
+    fflush(stdout);
+    CloseHandle(ht);
+#endif
+
+#if DO_SYNCH_WITH_SUSPEND_SELF_TEST
+    res = 0;
+    /* First we test suspending a new thread that hasn't been initialized by dr yet. */
+    ht=CreateThread(NULL, 0, &ThreadProc2, NULL, CREATE_SUSPENDED, &tid);
+    res = SuspendThread(ht);
+    CHECK_SUSPEND_COUNT(res, 1);
+    res = ResumeThread(ht);
+    CHECK_SUSPEND_COUNT(res, 2);
+    res = ResumeThread(ht);
+    CHECK_SUSPEND_COUNT(res, 1);
+    res = 0;
+    /* Thread is now running and should suspend itself.  We want to test suspending
+     * an already self suspended thread (xref 9333 for why this is a special case). */
+    while (res == 0) {
+        res = SuspendThread(ht);
+        if (res == 0) {
+            /* Thread might not yet have gotten around to suspending itself */
+            ResumeThread(ht);
+            /* short sleep to wait */
+            Sleep(200);
+        }
+    }
+    CHECK_SUSPEND_COUNT(res, 1);
+    printf("suspended(count = %d)...", res);
+    fflush(stdout);
+# if SLEEP_FOR_NUDGE
+    Sleep(20000);
+# endif
+    res = ResumeThread(ht);
+    CHECK_SUSPEND_COUNT(res, 2);
+    res = ResumeThread(ht);
+    CHECK_SUSPEND_COUNT(res, 1);
+    WaitForSingleObject(ht, INFINITE);
+    CloseHandle(ht);
+#endif
+
+#if DO_SYNCH_WITH_ALL_SUSPEND_SELF_TEST
+    /* xref case 9333, our new thread will suspend itself and we then want to trigger
+     * a synch_with_all_threads, will use the process exit one */
+    ht=CreateThread(NULL, 0, &ThreadProc2, NULL, 0, &tid);
+    SwitchToThread();
+    /* FIXME - this is racy, we can't be sure thread has suspended itself without
+     * suspending it ourselves, we'll just sleep a little to try and be sure. (We could
+     * use the same loop as DO_SYNCH_WITH_SUSPEND_SELF_TEST but is nice to keep the
+     * synch_with_thread and synch_with_all_threads tests separate (though I guess that's
+     * mainly because only one of them worked when the test was written). */
+    Sleep(1000);
+    printf("Testing exiting with self suspended thread.\n");
+#endif
+
+    printf("done\n");
+    
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+
+    return 0;
+}
diff --git a/suite/tests/win32/suspend.expect b/suite/tests/win32/suspend.expect
new file mode 100644
index 000000000..cd3572ac8
--- /dev/null
+++ b/suite/tests/win32/suspend.expect
@@ -0,0 +1,4 @@
+starting thread...suspended(count = 0)...resuming...exiting thread...resumed(count = 1)
+suspended(count = 1)...resumed,exiting
+Testing exiting with self suspended thread.
+done
diff --git a/suite/tests/win32/threadchurn.c b/suite/tests/win32/threadchurn.c
new file mode 100644
index 000000000..0740837c0
--- /dev/null
+++ b/suite/tests/win32/threadchurn.c
@@ -0,0 +1,216 @@
+/* **********************************************************
+ * Copyright (c) 2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* case 4660 test cases for thread churn */
+
+/* undefine this for a performance test */
+#define NIGHTLY_REGRESSION
+
+#include "tools.h"
+
+typedef unsigned char* app_pc;
+
+#ifdef USE_DYNAMO
+# include "dynamorio.h"
+#endif
+
+#ifdef WINDOWS
+#  include <process.h> /* for _beginthreadex */
+#else
+#endif
+
+#define SILENT                  /* no writes at all */
+
+/* observe page faults and private size growth, see what happens if I
+ * set the Working set size to a small number
+ */
+
+enum {SWAP_OUT_AFTER_THREAD = 1}; /* trims working set down after each thread */
+enum {SWAP_OUT_AFTER_BATCH = 1}; /* trims working set down after batch  */
+
+/* 
+ * native (0,0) pf delta is 15k,   time 30s
+ * native (0,1) pf delta is 50k    time 32s
+ * native (1,1) pf delta is 168k   time 45s
+ *  native peak private 400k,
+ *  dr (0,0) - 2MB peak private, time 12m
+ */
+
+#ifdef NIGHTLY_REGRESSION
+enum {TOTAL_THREADS = 40};
+#else
+enum {TOTAL_THREADS = 20000};
+#endif
+
+enum {BATCH_SIZE = 10};
+
+enum {ROUNDS = 10};
+
+#define UNIPROC
+
+enum {LOOP_WORK = 100};
+
+/* --------- */
+int thread[TOTAL_THREADS];
+
+long global_started = 0;                   /* unsynchronized */
+long global_finished = 0;                   /* unsynchronized */
+
+#ifdef UNIPROC
+#  define YIELD() thread_yield()    /* or do nothing on a multi processor */
+#else
+#  define YIELD() /* or do nothing on a multi processor */
+#endif
+
+int compare( const void *arg1, const void *arg2 )
+{
+   /* Compare all of both strings: */
+   return _stricmp( * ( char** ) arg1, * ( char** ) arg2 );
+}
+
+void
+sort()
+{
+    int argc = 5;
+    char *argv[] = {"one", "two", "three","five", "six", "unsorted"};
+    int i;
+
+   /* Sort remaining args using Quicksort algorithm: */
+   qsort( (void *)argv, (size_t)argc, sizeof( char * ), compare );
+
+#ifdef VERY_VERBOSE
+   /* Output sorted list: */
+   for( i = 0; i < argc; ++i )
+       print( " %s", argv[i] );
+   print( "\n" );
+#endif
+}
+
+void
+executor(void)
+{
+    int w;
+    sort();                 /* do some work */
+    InterlockedIncrement(&global_started);
+
+    for (w = 0; w < LOOP_WORK; w++) {
+        sort();                 /* do more work */
+        if (w % 10 == 0)
+            YIELD();
+    }
+    InterlockedIncrement(&global_finished);
+
+    /* TODO: could thread_suspend(a paired thread) */
+}
+
+enum {MINSIZE_KB = 500,
+      MAXSIZE_KB = 1000};
+
+int
+main()
+{
+    int round;
+    int b, t;
+    int res;
+
+    INIT();
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+
+    /* this doesn't do much in fact */
+    SetProcessWorkingSetSize(GetCurrentProcess(),
+                             MINSIZE_KB*1024,
+                             MAXSIZE_KB*1024);
+    // on Win2003 there is a SetProcessWorkingSetSizeEx that sets QUOTA_LIMITS_HARDWS_ENABLE
+
+
+    if (res == 0) 
+        print("SetProcessWorkingSetSize failed GLE: %d\n", GetLastError());
+
+    for (round = 0; round < ROUNDS; round++) {
+#ifdef VERBOSE
+        print("round %d\n", round);
+#endif        
+        global_started = 0;
+        global_finished = 0;
+
+        /* do in a batch */
+        for (b = 0; b < TOTAL_THREADS / BATCH_SIZE; b++) {
+            for (t = 0; t < BATCH_SIZE; t++) {
+                thread[t] = create_thread(executor);
+                if (thread[t] == NULL) 
+                    print("GLE: %d\n", GetLastError());
+                assert(thread[t] != NULL);
+            }
+            /* now synchronize with all of them - or maybe some? */
+
+#ifdef VERBOSE
+            print("started %d threads\n", TOTAL_THREADS);
+#else
+#ifndef SILENT
+            print("started some threads\n");
+#endif
+#endif
+            for (t = 0; t < BATCH_SIZE; t++) {
+                assert(thread[t] != NULL);
+                join_thread(thread[t]);
+                CloseHandle((HANDLE)thread[t]);
+                thread[t] = NULL; /* in case want to synch with some in a batch, but with all at the end */
+
+                if (SWAP_OUT_AFTER_THREAD) {
+                    res = SetProcessWorkingSetSize(GetCurrentProcess(),
+                                                   -1, -1);
+                }
+            }
+#ifdef VERBOSE
+            print("some %d work, done %d\n", global_started, global_finished);
+#endif
+            if (SWAP_OUT_AFTER_BATCH) {
+                res = SetProcessWorkingSetSize(GetCurrentProcess(),
+                                               -1, -1);
+            }
+        }
+    }
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+
+    print("done\n");
+    return 0;
+}
+
diff --git a/suite/tests/win32/threadchurn.expect b/suite/tests/win32/threadchurn.expect
new file mode 100644
index 000000000..19f86f493
--- /dev/null
+++ b/suite/tests/win32/threadchurn.expect
@@ -0,0 +1 @@
+done
diff --git a/suite/tests/win32/threadexit.c b/suite/tests/win32/threadexit.c
new file mode 100644
index 000000000..396a1bda0
--- /dev/null
+++ b/suite/tests/win32/threadexit.c
@@ -0,0 +1,241 @@
+/* **********************************************************
+ * Copyright (c) 2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* case 3105 test cases for rapidly starting and terminating threads */
+#include "tools.h"
+
+typedef unsigned char* app_pc;
+
+#ifdef USE_DYNAMO
+# include "dynamorio.h"
+#endif
+
+#ifdef WINDOWS
+#  include <process.h> /* for _beginthreadex */
+#else
+#endif
+
+/* not including main thread */
+#define NIGHTLY_REGRESSION
+
+#ifdef NIGHTLY_REGRESSION
+#  define SAFE_NATIVE 
+enum {TOTAL_THREADS = 2};
+#else
+// #define SAFE_NATIVE  // brutal NtTerminateProcess(0
+enum {TOTAL_THREADS = 200};
+#endif
+
+// #define VERBOSE
+
+#define ALL_RACES /* anything goes: start/stop */
+
+#ifdef SAFE_NATIVE
+/* nothing fancy: 
+ * start up some threads and then exit the process 
+ */
+enum {ROUNDS = 1};
+#else
+/* calling NtTerminateProcess(0) is in very unsafe - although often works well */
+enum {ROUNDS = 10};
+#endif
+
+#ifdef ALL_RACES
+/* for thread start races terminate early */
+enum {WAIT_TO_START = 1,
+      WAIT_TO_FINISH = TOTAL_THREADS/10,
+};
+#else
+/* for thread stop races wait for everyone to start */
+enum {WAIT_TO_START = TOTAL_THREADS,
+      WAIT_TO_FINISH = TOTAL_THREADS/10,
+};
+#endif
+
+#define UNIPROC
+
+enum {LOOP_WORK = 100};
+
+/* --------- */
+HANDLE thread[TOTAL_THREADS];
+
+long global_started = 0;                   /* unsynchronized */
+long global_finished = 0;                   /* unsynchronized */
+
+
+#ifdef UNIPROC
+#  define YIELD() thread_yield()    /* or do nothing on a multi processor */
+#else
+#  define YIELD() /* or do nothing on a multi processor */
+#endif
+
+typedef void (*fptr)();
+
+int compare( const void *arg1, const void *arg2 )
+{
+   /* Compare all of both strings: */
+   return _stricmp( * ( char** ) arg1, * ( char** ) arg2 );
+}
+
+void
+sort()
+{
+    int argc = 5;
+    char *argv[] = {"one", "two", "three","five", "six", "unsorted"};
+    int i;
+
+   /* Sort remaining args using Quicksort algorithm: */
+   qsort( (void *)argv, (size_t)argc, sizeof( char * ), compare );
+
+#ifdef VERY_VERBOSE
+   /* Output sorted list: */
+   for( i = 0; i < argc; ++i )
+       print( " %s", argv[i] );
+   print( "\n" );
+#endif
+}
+
+void
+executor(void)
+{
+    int w;
+    sort();                 /* do some work */
+    InterlockedIncrement(&global_started);
+
+    for (w = 0; w < LOOP_WORK; w++) {
+        sort();                 /* do more work */
+        if (w % 10 == 0)
+            YIELD();
+    }
+    InterlockedIncrement(&global_finished);
+
+    /* TODO: could thread_suspend(a paired thread) */
+}
+
+int
+main()
+{
+    int round;
+    int t;
+
+    GET_NTDLL(NtTerminateProcess, (IN HANDLE ProcessHandle OPTIONAL,
+                                   IN NTSTATUS ExitStatus));
+
+    INIT();
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    for (round = 0; round < ROUNDS; round++) {
+#ifdef VERBOSE
+        print("round %d\n", round);
+#endif        
+        if (round > 0) {
+            /* clean up first */
+            NtTerminateProcess(0 /* everyone but me */, 666);
+#ifdef VERBOSE
+            print("all alone again %d\n", round);
+#endif VERBOSE
+        }
+
+        global_started = 0;
+        global_finished = 0;
+
+        for (t = 0; t < TOTAL_THREADS; t++) {
+            thread[t] = (HANDLE)create_thread(executor);
+            if (thread[t] == NULL) 
+                print("GLE: %d\n", GetLastError());
+            assert(thread[t] != NULL);
+        }
+
+#ifdef VERBOSE
+        print("started %d threads\n", TOTAL_THREADS);
+#else
+        print("started some threads\n");
+#endif
+
+        /* wait for some to start */
+        while (global_started < WAIT_TO_START)
+            YIELD();
+
+        /* wait for some work to get done */
+        while (global_finished < WAIT_TO_FINISH)
+            YIELD();
+#ifdef VERBOSE
+        print("some %d work, done %d\n", global_started, global_finished);
+#endif
+    }
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+
+    print("done\n");
+    return 0;
+}
+
+/*
+
+What is this about?
+
+The variable pointed to by the Addend parameter must be aligned on a
+32-bit boundary; otherwise, this function will fail on multiprocessor
+x86 systems and any non-x86 systems.
+
+0:000> u kernel32!interlockedincrement
+kernel32!InterlockedIncrement:
+7c80977b 8b4c2404         mov     ecx,[esp+0x4]
+7c80977f b801000000       mov     eax,0x1
+7c809784 f00fc101         lock    xadd [ecx],eax
+7c809788 40               inc     eax
+7c809789 c20400           ret     0x4
+7c80978c 8d4900           lea     ecx,[ecx]
+
+
+when setting THREADS=1000 
+---------------------------
+threadexit.exe - Illegal System DLL Relocation
+---------------------------
+The system DLL user32.dll was relocated in memory. The application will not run properly. The relocation occurred because the DLL Dynamically Allocated Memory occupied an address range reserved for Windows system DLLs. The vendor supplying the DLL should be contacted for a new DLL. 
+---------------------------
+OK   
+---------------------------
+Assertion failed: thread[t] != NULL, file win32/threadexit.c, line 180
+
+LastStatusValue: (NTSTATUS) 0xc0000017 - {Not Enough Quota}  Not enough virtual memory or paging file quota is available to complete the specified operation.
+
+
+
+*/
diff --git a/suite/tests/win32/threadexit.expect b/suite/tests/win32/threadexit.expect
new file mode 100644
index 000000000..c40648b8f
--- /dev/null
+++ b/suite/tests/win32/threadexit.expect
@@ -0,0 +1,2 @@
+started some threads
+done
diff --git a/suite/tests/win32/threadinjection.c b/suite/tests/win32/threadinjection.c
new file mode 100644
index 000000000..db11c6d50
--- /dev/null
+++ b/suite/tests/win32/threadinjection.c
@@ -0,0 +1,457 @@
+/* **********************************************************
+ * Copyright (c) 2003-2006 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <process.h>
+#include "tools.h"
+
+#define MAX_THREADS  32
+//////////////////////////////////////////////////////////////////
+// Prototypes
+int   main(int argc, char *argv[]);      // Thread 1: main 
+DWORD WINAPI ThreadProc(LPVOID lpParam); // Thread 2: Sleep 
+DWORD WINAPI ThreadProc2(LPVOID lpParam);// Thread 3: Exit immediately 
+//////////////////////////////////////////////////////////////////
+
+typedef struct _Parameters
+{
+    BOOL bAll;
+    BOOL bGetContext;
+    BOOL bSetContext;
+    BOOL bSuspend;
+    BOOL bVerbose;
+    int  nPID;
+    int  nSleepTime;
+} PARAMETERS, *LPPARAMETERS;
+
+#define YIELD() Sleep(0)
+
+int    ThreadNr;                        /* Number of threads started */
+
+BOOL thread_proc_wait = FALSE;
+BOOL thread_proc_waiting = FALSE;
+
+DWORD WINAPI ThreadProc( LPVOID lpParam )
+{
+
+    int nSleepTime;
+
+    nSleepTime = (*(int *) lpParam);
+
+    /* a simple spinning synchronization for the TerminateThread test */
+    while (thread_proc_wait) {
+        thread_proc_waiting = TRUE;
+        YIELD();
+    }
+    thread_proc_waiting = FALSE;
+
+    if (nSleepTime < 5000)
+    {
+        Sleep(nSleepTime);
+    }
+	
+    return 0;
+
+}
+
+DWORD WINAPI ThreadProc2( LPVOID lpParam )
+{
+
+    ExitThread(-1);
+
+    // You should never get here
+    return 0;
+
+}
+
+void LaunchTest
+( 
+	char szCommandLine[],
+	PARAMETERS myParameters
+)
+{
+    STARTUPINFO si;
+    PROCESS_INFORMATION pi;
+
+    ZeroMemory( &si, sizeof(si) );
+    si.cb = sizeof(si);
+    ZeroMemory( &pi, sizeof(pi) );
+
+	if (myParameters.bVerbose == TRUE)
+	{
+		print("------------------------------------------------------------\n");
+		print("Test begining with options: %s\n",szCommandLine);
+		print("------------------------------------------------------------\n");
+	}
+	
+	if(0 == CreateProcess(NULL, szCommandLine, NULL, NULL, FALSE, 0, NULL, ".", &si, &pi))
+	{       
+		print("Error creating process:\n\"%s\"\n", szCommandLine);		
+	}
+	else if (myParameters.bVerbose == TRUE)
+	{
+	    print("------------------------------------------------------------\n");
+	    print("Test completed: %s options executed\n",szCommandLine);
+	    print("------------------------------------------------------------\n");
+	}
+
+    WaitForSingleObject(pi.hProcess, INFINITE);
+}
+
+void LaunchAllTests
+(
+    char *argv[],
+    PARAMETERS myParameters
+)
+{
+    char   szCommandLine[512];
+
+// Remote thread test
+// Launch a separate process that injects threads into this one
+// Need to pass this process ID in the arguments        
+
+    sprintf(szCommandLine, "%s /PID=%d\n", argv[0], GetCurrentProcessId());
+    LaunchTest(szCommandLine, myParameters);
+
+// GetContextThread()
+// Launch a separate process that creates threads and attempts to GetContextThread()
+// without suspending or pausing
+
+    sprintf(szCommandLine, "%s /PID=%d /GETCONTEXT\n", argv[0], GetCurrentProcessId());
+    LaunchTest(szCommandLine, myParameters);
+
+// SetContextThread()
+// Launch a separate process that creates threads and attempts to SetContextThread()
+// without suspending or pausing
+
+    sprintf(szCommandLine, "%s /PID=%d /SETCONTEXT\n", argv[0], GetCurrentProcessId());
+    LaunchTest(szCommandLine, myParameters);
+
+// SuspendThread()
+// Launch a separate process that creates threads and attempts to SuspendThread()
+// and ResumeThread() without pausing
+
+    sprintf(szCommandLine, "%s /PID=%d /SUSPEND\n", argv[0], GetCurrentProcessId());
+    LaunchTest(szCommandLine, myParameters);
+
+// GetContextThread() with SuspendThread()
+// Launch a separate process that creates threads, suspends them, and attempts 
+// to GetContextThread() without pausing
+
+    sprintf(szCommandLine, "%s /PID=%d /GETCONTEXT /SUSPEND\n", argv[0], GetCurrentProcessId());
+    LaunchTest(szCommandLine, myParameters);
+
+// SetContextThread() with SuspendThread()
+// Launch a separate process that creates threads, suspends them, and attempts 
+// to SetContextThread() without pausing
+
+    sprintf(szCommandLine, "%s /PID=%d /SETCONTEXT /SUSPEND\n", argv[0], GetCurrentProcessId());
+    LaunchTest(szCommandLine, myParameters);
+
+// SuspendThread() with Sleep()
+// Launch a separate process that creates threads, pauses for 5 seconds, and 
+// attempts to SuspendThread() and ResumeThread() 
+
+    sprintf(szCommandLine, "%s /PID=%d /SUSPEND /SLEEP=10\n", argv[0], GetCurrentProcessId());
+    LaunchTest(szCommandLine, myParameters);
+
+// GetContextThread() with Sleep()
+// Launch a separate process that creates threads, pauses for 5 seconds, and 
+// attempts to GetContextThread()
+
+    sprintf(szCommandLine, "%s /PID=%d /GETCONTEXT /SLEEP=10", argv[0], GetCurrentProcessId());
+    LaunchTest(szCommandLine, myParameters);
+
+// SetContextThread() with Sleep()
+// Launch a separate process that creates threads, pauses for 5 seconds, and 
+// attempts to SetContextThread()
+
+    sprintf(szCommandLine, "%s /PID=%d /SETCONTEXT /SLEEP=10", argv[0], GetCurrentProcessId());
+    LaunchTest(szCommandLine, myParameters);
+
+// GetContextThread() with Sleep() & SuspendThread()
+// Launch a separate process that creates threads, pauses for 5 seconds,
+// suspends the thread & attempts to GetContextThread()
+
+    sprintf(szCommandLine, "%s /PID=%d /GETCONTEXT /SLEEP=10 /SUSPEND", argv[0], GetCurrentProcessId());
+    LaunchTest(szCommandLine, myParameters);
+
+// SetContextThread() with Sleep() & SuspendThread()
+// Launch a separate process that creates threads, pauses for 5 seconds,
+// suspends the thread & attempts to SetContextThread()
+
+    sprintf(szCommandLine, "%s /PID=%d /SETCONTEXT /SLEEP=10 /SUSPEND", argv[0], GetCurrentProcessId());
+    LaunchTest(szCommandLine, myParameters);
+
+}
+
+void Usage()
+{
+    print("\nInjectionTests");
+    print("\nCreates process, threads & remote threads and tests thread corner cases");
+    print("\n");
+    print("\nUsage: InjectionTests [Options]");
+    print("\n/ALL          - Run all tests");
+    print("\n/GETCONTEXT   - Call GetContextThread() after thread creation");
+    print("\n/SETCONTEXT   - Call SetContextThread() after thread creation");
+    print("\n/SLEEP=n      - Sleep for n milliseconds immediately after thread creation");
+    print("\n/SUSPEND      - Call SuspendThread() prior to other thread functions");
+    print("\n/PID=n        - Inject threads into remote process with PID n");
+    print("\n/VERBOSE      - Enable individual test logging");
+    print("\n/HELP, /?     - Prints this message");
+    exit(0);
+}
+
+void InitializeArguments
+(
+    LPPARAMETERS myParameters
+)
+{
+    myParameters->bAll          = FALSE;
+    myParameters->bGetContext   = FALSE;
+    myParameters->bSetContext   = FALSE;
+    myParameters->bSuspend      = FALSE;
+    myParameters->bVerbose      = FALSE;
+    myParameters->nPID          = 0;
+    myParameters->nSleepTime    = 0;
+}
+
+void ParseArguments
+(
+    int argc,
+    char *argv[],
+    LPPARAMETERS myParameters
+)
+{
+    int i;
+
+    for (i=1;i<argc;i++)
+    {
+        strupr(argv[i]);
+
+        if (!strcmp(argv[i],"/?") ||
+            !strcmp(argv[i],"/HELP"))
+        {
+            Usage();
+        }
+
+        if (!strncmp(argv[i],"/PID=",5))
+        {
+            // Skip over argument, save PID
+            myParameters->nPID = atoi(argv[i] + 5);
+        }
+        else if (!strncmp(argv[i],"/SLEEP=",7))
+        {
+            // Skip over argument, save SleepTime
+            myParameters->nSleepTime = atoi(argv[i] + 7);
+        }
+        else if (!strncmp(argv[i],"/GETCONTEXT",11))
+        {
+            myParameters->bGetContext = TRUE;
+        }
+        else if (!strncmp(argv[i],"/SETCONTEXT",11))
+        {
+            myParameters->bSetContext = TRUE;
+        }
+        else if (!strncmp(argv[i],"/SUSPEND",8))
+        {
+            myParameters->bSuspend = TRUE;
+        }
+        else if (!strncmp(argv[i],"/VERBOSE",8))
+        {
+            myParameters->bVerbose = TRUE;
+        }
+        else if (!strncmp(argv[i],"/ALL",4))
+        {
+            myParameters->bAll = TRUE;
+        }
+    }
+}
+
+void ExerciseThread
+(
+    HANDLE     hThread,
+    PARAMETERS myParameters
+)
+{
+
+    CONTEXT Context;
+    BOOL bError;
+
+    if (myParameters.nSleepTime != 0)
+    {
+        Sleep(myParameters.nSleepTime);
+    }
+
+    if (myParameters.bSuspend == TRUE)
+    {
+        if ((SuspendThread(hThread) == -1) && (myParameters.bVerbose == TRUE))
+        {
+            print("Error in SuspendThread()(Code %d)\n",GetLastError());
+        }
+        if ((ResumeThread(hThread) == -1) && (myParameters.bVerbose == TRUE))
+        {
+            print("Error in ResumeThread() (Code %d)\n",GetLastError());			
+        }
+    }
+    if (myParameters.bGetContext == TRUE)
+    {
+        bError = GetThreadContext(hThread, &Context);
+        if ((bError == 0) && (myParameters.bVerbose == TRUE))
+        {
+            print("Error in GetThreadContext (Code %d)\n",GetLastError());			
+        }
+    }
+    if ((myParameters.bSetContext == TRUE) && (myParameters.bVerbose == TRUE))
+    {
+        bError = SetThreadContext(hThread, &Context);
+        if ((bError == 0)  && (myParameters.bVerbose == TRUE))
+        {
+            print("Error in SetThreadContext (Code %d)\n",GetLastError());			
+        }
+    }
+}
+
+int main( int argc, char *argv[])       /* Thread One */
+{
+
+    DWORD  dwThreadID;
+    HANDLE hProcess;
+    HANDLE hThread1, hThread2, hThread3, hThread4;
+	char   szCommandLine[1024];
+	int    i;
+    PARAMETERS myParameters;
+
+    INIT();
+    /* set exception handler */
+
+    strcpy(szCommandLine, "\0");
+    ThreadNr = 0;
+
+    InitializeArguments(&myParameters);
+    ParseArguments(argc, argv, &myParameters);
+
+    if (argc == 1)
+    {
+        myParameters.bAll = TRUE;
+    }
+
+    // On initial call, no args are present; execute each subtest below
+    if (myParameters.bAll == TRUE)
+    {
+        LaunchAllTests(argv, myParameters);
+    }
+    else
+    {
+		print("Entering thread with options:\n");
+		for (i = 1; i < argc; i++)
+		{
+			if (!strncmp(argv[i], "/PID", 4))
+			{
+				strcat(szCommandLine, "/PID");
+			}
+			else
+			{
+				strcat(szCommandLine, argv[i]);
+			}
+		}
+		print("%s\n", szCommandLine);
+
+        do
+        {  
+            hThread1 = CreateThread(NULL, 0, &ThreadProc, &(myParameters.nSleepTime), 0, &dwThreadID );
+            ExerciseThread(hThread1, myParameters);        
+            WaitForSingleObject(hThread1, INFINITE);
+            ThreadNr++;
+  
+            thread_proc_wait = TRUE;
+            hThread2 = CreateThread(NULL, 0, &ThreadProc, &(myParameters.nSleepTime), 0, &dwThreadID );
+            while (!thread_proc_waiting) {
+                YIELD();
+            }
+            TerminateThread(hThread2, -1);
+            thread_proc_wait = FALSE;
+            thread_proc_waiting = FALSE;
+            ThreadNr++;
+
+            // ThreadProc2 calls ExitThread() immediately
+            hThread3 = CreateThread(NULL, 0, &ThreadProc2, &ThreadNr, 0, &dwThreadID );
+            WaitForSingleObject(hThread3, INFINITE);
+            ThreadNr++;
+
+            if (hThread1 != NULL)
+            {
+                CloseHandle(hThread1);
+            }
+            if (hThread2 != NULL)
+            {
+                CloseHandle(hThread2);
+            }
+            if (hThread3 != NULL)
+            {
+                CloseHandle(hThread3);
+            }
+
+		} while(ThreadNr < MAX_THREADS);
+
+        if (myParameters.nPID != 0)
+        {
+            // Prints out results in host PID
+            hProcess = OpenProcess(PROCESS_ALL_ACCESS, TRUE, myParameters.nPID);
+            if ((hProcess == NULL) && (myParameters.bVerbose == TRUE))
+            {
+                print("Error in OpenProcess(Code %d)\n",GetLastError());		
+            }
+
+            hThread4 = CreateRemoteThread(hProcess, 0, 0, &ThreadProc, &(myParameters.nSleepTime), 0, &dwThreadID );
+
+            if ((hThread4 == NULL) && (myParameters.bVerbose == TRUE))
+            {
+                print("Error in CreateRemoteThread(Code %d)\n",GetLastError());				
+            }
+            WaitForSingleObject(hThread4, INFINITE);
+
+            if (hThread4 != NULL)
+            {
+                CloseHandle(hThread4);
+            }
+        }
+		print("Exiting thread with options:\n");
+		print("%s\n", szCommandLine);
+    }
+    
+    return 0;
+}
+
diff --git a/suite/tests/win32/threadinjection.template b/suite/tests/win32/threadinjection.template
new file mode 100755
index 000000000..0d8218374
--- /dev/null
+++ b/suite/tests/win32/threadinjection.template
@@ -0,0 +1,56 @@
+Entering thread with options:
+/PID
+Exiting thread with options:
+/PID
+Entering thread with options:
+/PID/GETCONTEXT
+
+Exiting thread with options:
+/PID/GETCONTEXT
+
+Entering thread with options:
+/PID/SETCONTEXT
+
+Exiting thread with options:
+/PID/SETCONTEXT
+
+Entering thread with options:
+/PID/SUSPEND
+
+Exiting thread with options:
+/PID/SUSPEND
+
+Entering thread with options:
+/PID/GETCONTEXT/SUSPEND
+
+Exiting thread with options:
+/PID/GETCONTEXT/SUSPEND
+
+Entering thread with options:
+/PID/SETCONTEXT/SUSPEND
+
+Exiting thread with options:
+/PID/SETCONTEXT/SUSPEND
+
+Entering thread with options:
+/PID/SUSPEND/SLEEP=10
+
+Exiting thread with options:
+/PID/SUSPEND/SLEEP=10
+
+Entering thread with options:
+/PID/GETCONTEXT/SLEEP=10
+Exiting thread with options:
+/PID/GETCONTEXT/SLEEP=10
+Entering thread with options:
+/PID/SETCONTEXT/SLEEP=10
+Exiting thread with options:
+/PID/SETCONTEXT/SLEEP=10
+Entering thread with options:
+/PID/GETCONTEXT/SLEEP=10/SUSPEND
+Exiting thread with options:
+/PID/GETCONTEXT/SLEEP=10/SUSPEND
+Entering thread with options:
+/PID/SETCONTEXT/SLEEP=10/SUSPEND
+Exiting thread with options:
+/PID/SETCONTEXT/SLEEP=10/SUSPEND
diff --git a/suite/tests/win32/tls.c b/suite/tests/win32/tls.c
new file mode 100644
index 000000000..f6e20fef2
--- /dev/null
+++ b/suite/tests/win32/tls.c
@@ -0,0 +1,453 @@
+/* **********************************************************
+ * Copyright (c) 2006-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include <process.h> /* for _beginthreadex */
+#include <stdio.h>
+#include <AccCtrl.h> /* for SE_KERNEL_OBJECT */
+#include <Aclapi.h>
+#include "tools.h"
+
+#define VERBOSE 0
+
+/***************************************************************************/
+/* This is all copied from ntdll.h, share/detach.c, and lib/globals_shared.h
+ * FIXME - would be nicer to find a way to include these directly. */
+
+typedef struct _UNICODE_STRING {
+    /* Length field is size in bytes not counting final 0 */
+    USHORT Length;
+    USHORT MaximumLength;
+    PWSTR  Buffer;
+} UNICODE_STRING;
+typedef UNICODE_STRING *PUNICODE_STRING;
+
+typedef struct _OBJECT_ATTRIBUTES {
+    ULONG Length;
+    HANDLE RootDirectory;
+    PUNICODE_STRING ObjectName;
+    ULONG Attributes;
+    PVOID SecurityDescriptor;        // Points to type SECURITY_DESCRIPTOR
+    PVOID SecurityQualityOfService;  // Points to type SECURITY_QUALITY_OF_SERVICE
+} OBJECT_ATTRIBUTES;
+typedef OBJECT_ATTRIBUTES *POBJECT_ATTRIBUTES;
+
+#define InitializeObjectAttributes( p, n, a, r, s ) { \
+    (p)->Length = sizeof( OBJECT_ATTRIBUTES );          \
+    (p)->RootDirectory = r;                             \
+    (p)->Attributes = a;                                \
+    (p)->ObjectName = n;                                \
+    (p)->SecurityDescriptor = s;                        \
+    (p)->SecurityQualityOfService = NULL;               \
+    }
+
+#define OBJ_CASE_INSENSITIVE    0x00000040L
+/* N.B.: this is an invalid parameter on NT4! */
+#define OBJ_KERNEL_HANDLE       0x00000200L
+typedef ULONG ACCESS_MASK;
+
+typedef struct _CLIENT_ID {
+    HANDLE UniqueProcess;
+    HANDLE UniqueThread;
+} CLIENT_ID;
+typedef CLIENT_ID *PCLIENT_ID;
+
+typedef struct _USER_STACK {
+    PVOID FixedStackBase;
+    PVOID FixedStackLimit;
+    PVOID ExpandableStackBase;
+    PVOID ExpandableStackLimit;
+    PVOID ExpandableStackBottom;
+} USER_STACK, *PUSER_STACK;
+
+/* 64kb, same as allocation granularity so is as small as we can get */
+#define STACK_RESERVE 0x10000
+/* 12kb, matches current core stack size, note can expand to 
+ * STACK_RESERVE - (5 * PAGE_SIZE), i.e. 44kb */
+#define STACK_COMMIT 0x3000
+
+/* returns NULL on error */
+/* FIXME - is similar to core create_thread, but uses API routines where
+ * possible, could try to share. */
+/* stack_reserve and stack commit must be multiples of PAGE_SIZE and reserve
+ * should be at least 5 pages larger then commit */
+/* NOTE - For !target_kernel32 : 
+ *  target thread routine can't exit by by returning, instead it must call
+ *    ExitThread or the like
+ *  caller or target thread routine is responsible for informing csrss (if
+ *    neccessary) and freeing the the thread stack 
+ */
+static HANDLE
+nt_create_thread(HANDLE hProcess, PTHREAD_START_ROUTINE start_addr, void *arg,
+                 uint stack_reserve, uint stack_commit, bool suspended,
+                 uint *tid, bool target_kernel32)
+{
+    HANDLE hThread = NULL;
+    USER_STACK stack = {0};
+    OBJECT_ATTRIBUTES oa;
+    CLIENT_ID cid;
+    CONTEXT context = {0};
+    uint num_commit_bytes, code;
+    unsigned long old_prot;
+    void *p;
+    SECURITY_DESCRIPTOR *sd = NULL;
+
+    GET_NTDLL(NtCreateThread, (OUT PHANDLE ThreadHandle,
+                               IN ACCESS_MASK DesiredAccess,
+                               IN POBJECT_ATTRIBUTES ObjectAttributes,
+                               IN HANDLE ProcessHandle,
+                               OUT PCLIENT_ID ClientId,
+                               IN PCONTEXT ThreadContext,
+                               IN PUSER_STACK UserStack,
+                               IN BOOLEAN CreateSuspended));
+
+    /* both stack size and stack reserve must be multiples of PAGE_SIZE */
+    assert((stack_reserve & (PAGE_SIZE - 1)) == 0);
+    assert((stack_commit & (PAGE_SIZE - 1)) == 0);
+    /* We stick a non-committed page on each end just to be safe and windows
+     * needs three pages at the end to properly handle end of expandable stack
+     * case (wants to pass exception back to the app on overflow, so needs some
+     * stack for that). */
+    assert(stack_reserve >= stack_commit + (5 * PAGE_SIZE));
+
+    /* Use the security descriptor from the target process for creating the
+     * thread so that once created the thread will be able to open a full
+     * access handle to itself (xref case 2096). */
+    /* NOTES - tried many ways to impersonate based on target process token
+     * so we could just use the default and was unable to get anywhere with
+     * that.  Easiest thing to do here is just create a new security descriptor
+     * with a NULL (not empty) DACL [just InitializeSecurityDescriptor();
+     * SetSecurityDescriptorDacl()], but that's a privilege escalation
+     * problem (allows anybody full access to the thread)].  If we instead get
+     * the full security descriptor from the target process and try to use that
+     * the kernel complains that its a bad choice of owner.  What we do instead
+     * is get just the DACL and leave the rest empty (will be filled in with
+     * defaults during create thread).  Thus the security descriptor for the
+     * thread will end up having the owner, group, and SACL from this
+     * process and the DACL from the target.  Upshot is the the thread pseudo
+     * handle will have full permissions (from the DACL), but the owner will be
+     * us and, even though the handle we get back from CreateThread will be
+     * fully permissioned as we request, any subsequent attempts by us to
+     * OpenThread will fail since we aren't on the DACL.  We could always add
+     * ourselves to the DACL later or we can use the SE_DEBUG_PRIVILEGE to
+     * allow us to open it anyways.  Note if for some reason we want to view the
+     * SACL we need to enable the ACCESS_SYSTEM_SECURITY privilege when opening
+     * the handle.
+     * FIXME - we could instead build our own DACL combining the two, we could
+     * also try setting the owner/group after the thread is created if we
+     * really wanted to look like the target process thread, and could also
+     * start with a NULL sd and set the DACL later if want to match
+     * CreateThread as closely as possible.  If we do anything post system
+     * call should be sure to always create the thread suspended.
+     */
+    code = GetSecurityInfo(hProcess, SE_KERNEL_OBJECT,
+                           DACL_SECURITY_INFORMATION,
+                           NULL, NULL, NULL, NULL, &sd);
+    assert(code == ERROR_SUCCESS);
+ 
+    InitializeObjectAttributes(&oa, NULL, OBJ_CASE_INSENSITIVE, NULL, sd);
+  
+    stack.ExpandableStackBottom = VirtualAllocEx(hProcess, NULL,
+                                                 stack_reserve - PAGE_SIZE,
+                                                 MEM_RESERVE, PAGE_READWRITE);
+    if (stack.ExpandableStackBottom == NULL)
+        goto error;
+
+    /* We provide non-committed boundary page on each side of the stack just to
+     * be safe (note we will get a stack overflow exception if stack grows to
+     * 3rd to last page of this region (xpsp2)). */
+    stack.ExpandableStackBottom =
+        ((byte *)stack.ExpandableStackBottom) + PAGE_SIZE;
+    stack.ExpandableStackBase = 
+        ((byte *)stack.ExpandableStackBottom) + stack_reserve - (2*PAGE_SIZE);
+
+    stack.ExpandableStackLimit = 
+        ((byte *)stack.ExpandableStackBase) - stack_commit;
+    num_commit_bytes = stack_commit + PAGE_SIZE;
+    p = ((byte *)stack.ExpandableStackBase) - num_commit_bytes;
+    p = VirtualAllocEx(hProcess, p, num_commit_bytes, MEM_COMMIT, PAGE_READWRITE);
+    if (p == NULL)
+        goto error;
+    if (!VirtualProtectEx(hProcess, p, PAGE_SIZE, PAGE_READWRITE|PAGE_GUARD, &old_prot))
+        goto error;
+
+    /* set the context: initialize with our own */
+    context.ContextFlags = CONTEXT_FULL;
+    GetThreadContext(GetCurrentThread(), &context);
+    if (target_kernel32) {
+        assert(false);
+#if 0 /* not implemented here */
+        /* For kernel32!BaseThreadStartThunk CXT_XAX contains the address of the
+         * thread routine and CXT_XBX the arg */
+        context.CXT_XSP = (ptr_uint_t)stack.ExpandableStackBase;
+        context.CXT_XIP = (ptr_uint_t)get_kernel_thread_start_thunk();
+        context.CXT_XAX = (ptr_uint_t)start_addr;
+        context.CXT_XBX = (ptr_uint_t)arg;
+#endif
+    } else {
+        ptr_uint_t buf[2];
+        bool res;
+        SIZE_T written;
+        IF_X64(assert(false)); /* FIXME won't work for x64: need a SetContext */
+        /* directly targeting the start_address */
+        context.CXT_XSP = ((ptr_uint_t)stack.ExpandableStackBase) - sizeof(buf);
+        context.CXT_XIP = (ptr_uint_t)start_addr;
+        /* set up arg on stack, give NULL return address */
+        buf[1] = (ptr_uint_t)arg;
+        buf[0] = 0;
+        res = WriteProcessMemory(hProcess, (void *)context.CXT_XSP, &buf,
+                                 sizeof(buf), &written);
+        if (!res || written != sizeof(buf)) {
+            goto error;
+        }
+    }                           
+    if (context.CXT_XIP == 0) {
+        goto error;
+    }
+
+    /* NOTE - CreateThread passes NULL for object attributes so despite Nebbet
+     * must be optional (checked NTsp6a, XPsp2). We don't pass NULL so we can
+     * specify the security descriptor. */
+    if (!NT_SUCCESS(NtCreateThread(&hThread, THREAD_ALL_ACCESS, &oa, 
+                                   hProcess, &cid, &context, &stack, 
+                                   (byte)(suspended ? TRUE : FALSE)))) {
+        goto error;
+    }
+
+    if (tid != NULL)
+        *tid = (ptr_uint_t)cid.UniqueThread;
+
+ exit:
+    if (sd != NULL) {
+        /* Free the security descriptor. */
+        LocalFree(sd);
+    }
+
+    return hThread;
+
+ error:
+    if (stack.ExpandableStackBottom != NULL) {
+        /* Free remote stack on error. */
+        VirtualFreeEx(hProcess, stack.ExpandableStackBottom, 0, MEM_RELEASE);
+    }
+    assert(hThread == NULL);
+    hThread = NULL; /* just to be safe */
+    goto exit;
+}
+
+#define NUDGE_DEFINITIONS()                                                     \
+    /* Control nudges */                                                        \
+    NUDGE_DEF(opt, "Synchronize dynamic options")                               \
+    NUDGE_DEF(reset, "Reset code caches") /* flush & delete */                  \
+    NUDGE_DEF(detach, "Detach")                                                 \
+    NUDGE_DEF(mode, "Liveshield mode update")                                   \
+    NUDGE_DEF(policy, "Liveshield policy update")                               \
+    NUDGE_DEF(lstats, "Liveshield statistics NYI")                              \
+    NUDGE_DEF(process_control, "Process control nudge") /* Case 8594. */        \
+    NUDGE_DEF(upgrade, "DR upgrade NYI case 4179")                              \
+    NUDGE_DEF(kstats, "Dump kstats in log or kstat file NYI")                   \
+    /* internal options */                                                      \
+    NUDGE_DEF(stats, "Dump internal stats in logfiles NYI")                     \
+    NUDGE_DEF(invalidate, "Invalidate code caches NYI") /* flush */             \
+    /* stress testing */                                                        \
+    NUDGE_DEF(recreate_pc, "Recreate PC NYI")                                   \
+    NUDGE_DEF(recreate_state, "Recreate state NYI")                             \
+    NUDGE_DEF(reattach, "Reattach - almost detach, NYI")                        \
+    /* diagnostics */                                                           \
+    NUDGE_DEF(diagnose, "Request diagnostic file NYI")                          \
+    NUDGE_DEF(ldmp, "Dump core")                                                \
+    NUDGE_DEF(freeze, "Freeze coarse units")                                    \
+    NUDGE_DEF(persist, "Persist coarse units")                                  \
+    /* client nudge */                                                          \
+    NUDGE_DEF(client, "Client nudge")                                           \
+    /* security testing */                                                      \
+    NUDGE_DEF(violation, "Simulate a security violation")                       \
+    /* ADD NEW NUDGE_DEFs only immediately above this line  */                  \
+    /* since these are used as a bitmask only 30 types can be supported */
+
+typedef enum {
+    NUDGE_DR_PARAMETRIZED_START,
+#define NUDGE_DEF(name, comment) NUDGE_DR_##name, 
+    NUDGE_DEFINITIONS()
+#undef NUDGE_DEF
+    NUDGE_DR_PARAMETRIZED_END
+} nudge_generic_type_t;
+
+/* note that these are bitmask values */
+#define NUDGE_GENERIC(name) (1 << (NUDGE_DR_##name))
+
+#define NUDGE_ARG_VERSION_1 1
+#define NUDGE_ARG_CURRENT_VERSION NUDGE_ARG_VERSION_1
+
+/* nudge_arg_t flags */
+enum {
+    NUDGE_IS_INTERNAL       = 0x01, /* nudge is internally generated */
+    NUDGE_NUDGER_FREE_STACK = 0x02, /* nudger will free the nudge thread's stack so the
+                                     * nudge thread itself shouldn't */
+    NUDGE_FREE_ARG          = 0x04, /* nudge arg is in a separate allocation and should
+                                     * be freed by the nudge thread */
+};
+
+typedef struct {
+    uint version; /* version number for future proofing */
+    uint nudge_action_mask; /* drawn from NUDGE_DEFS above */
+    uint flags; /* flags drawn from above enum */
+    void *client_arg; /* argument for a client nudge */
+    /* Add future arguments for nudge actions here. */
+} nudge_arg_t;
+
+/***************************************************************************/
+
+byte *
+get_nudge_target()
+{
+    /* read DR marker
+     * just hardcode the offsets for now
+     */
+    static const uint DR_NUDGE_FUNC_OFFSET = 0x20;
+    return get_drmarker_field(DR_NUDGE_FUNC_OFFSET);
+}
+
+typedef unsigned int (__stdcall * threadfunc_t)(void *);
+
+static bool test_tls = false;
+#define TLS_SLOTS 64
+static bool tls_own[TLS_SLOTS];
+
+byte *
+get_own_teb()
+{
+    byte *teb;
+    __asm {
+        mov  eax, fs:[0x18]
+        mov  teb, eax
+    }
+    return teb;
+}
+
+int WINAPI
+thread_func(void * arg)
+{
+    int i;
+    while (!test_tls)
+        ; /* spin */
+    for (i=0; i<TLS_SLOTS; i++) {
+        if (tls_own[i]) {
+            if (TlsGetValue(i) != 0) {
+                print("TLS slot %d is "PFX" when it should be 0!\n",
+                      i, TlsGetValue(i));
+            }
+        }
+    }
+    print("Done testing tls slots\n");
+    return 0;
+}
+
+int main()
+{
+    int tid;
+    HANDLE detach_thread;
+    HANDLE my_thread;
+    HANDLE lib;
+    byte *nudge_target;
+    int i;
+
+#if DYNAMO
+    dynamo_init();
+    dynamo_start();
+#endif
+
+    my_thread = (HANDLE) _beginthreadex(NULL, 0, thread_func, NULL, 0, &tid);
+
+    nudge_target = get_nudge_target();
+    if (nudge_target == NULL) {
+        print("Cannot find DRmarker -- are you running natively?\n");
+    } else {
+        /* This test needs to do some work after detaching.  We exploit a
+         * hole in DR by creating a thread that directly targets the DR
+         * detach routine.  Hopefully this will motivate us to close
+         * the hole (case 552) :)
+         * The alternative is to create a new runall type that detaches from
+         * the outside and then waits a while, but would be hard to time.
+         */
+        nudge_arg_t *arg = (nudge_arg_t *)VirtualAlloc(NULL, sizeof(nudge_arg_t),
+                                                       MEM_COMMIT, PAGE_READWRITE);
+        arg->version = NUDGE_ARG_CURRENT_VERSION;
+        arg->nudge_action_mask = NUDGE_GENERIC(detach);
+        arg->flags = 0;
+        arg->client_arg = NULL;
+        print("About to detach using underhanded methods\n");
+        detach_thread = (HANDLE)
+            nt_create_thread(GetCurrentProcess(), (threadfunc_t) nudge_target,
+                             arg, STACK_RESERVE, STACK_COMMIT, false, NULL, false);
+        WaitForSingleObject(detach_thread, INFINITE);
+        
+        assert(get_nudge_target() == NULL);
+        print("Running natively now\n");
+    }
+
+    for (i=0; i<TLS_SLOTS; i++) {
+        /* case 8143: a runtime-loaded dll calling TlsAlloc needs to set
+         * a value for already-existing threads.  The "official" method
+         * is to directly TlsGetValue() and if not NULL assume that dll
+         * has already set that value.  Our detach needs to clear values
+         * to ensure this.  We have the simplest test possible here, of course.
+         * We do need another thread as TlsAlloc seems to clear the slot for
+         * the current thread.
+         * http://msdn.microsoft.com/library/default.asp?url=/library/en-us/dllproc/base/using_thread_local_storage_in_a_dynamic_link_library.asp
+         */
+        unsigned long tlshandle = 0UL;
+        tlshandle = TlsAlloc();
+        if (tlshandle == TLS_OUT_OF_INDEXES) {
+            break;
+        }
+#if VERBOSE
+        print("handle %d is %d\n", i, tlshandle);
+#endif
+        assert(tlshandle >= 0);
+        /* we only want teb slots */
+        if (tlshandle >= TLS_SLOTS)
+            break;
+        tls_own[tlshandle] = true;
+    }
+
+    /* tell thread to GO */
+    test_tls = true;
+
+    WaitForSingleObject(my_thread, INFINITE);
+
+#if DYNAMO
+    dynamo_stop();
+    dynamo_exit();
+#endif
+    return 0;
+}
diff --git a/suite/tests/win32/tls.expect b/suite/tests/win32/tls.expect
new file mode 100755
index 000000000..778d7e0a2
--- /dev/null
+++ b/suite/tests/win32/tls.expect
@@ -0,0 +1,3 @@
+About to detach using underhanded methods
+Running natively now
+Done testing tls slots
diff --git a/suite/tests/win32/virtualfree.c b/suite/tests/win32/virtualfree.c
new file mode 100644
index 000000000..bc0c95f4f
--- /dev/null
+++ b/suite/tests/win32/virtualfree.c
@@ -0,0 +1,62 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include <assert.h>
+
+#if USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+int
+main()
+{
+    void *p;
+    BOOL res;
+    int size = 0x04000;
+
+#if USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    p = VirtualAlloc(0, size, MEM_RESERVE|MEM_COMMIT, PAGE_EXECUTE_READWRITE);
+    assert(p);
+    res = VirtualFree(p, 0, MEM_RELEASE);
+    assert(res);
+    printf("Successful\n");
+
+#if USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
diff --git a/suite/tests/win32/virtualfree.expect b/suite/tests/win32/virtualfree.expect
new file mode 100644
index 000000000..628d78702
--- /dev/null
+++ b/suite/tests/win32/virtualfree.expect
@@ -0,0 +1 @@
+Successful
diff --git a/suite/tests/win32/virtualreserve.c b/suite/tests/win32/virtualreserve.c
new file mode 100644
index 000000000..cd6330654
--- /dev/null
+++ b/suite/tests/win32/virtualreserve.c
@@ -0,0 +1,134 @@
+/* **********************************************************
+ * Copyright (c) 2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* case 4175 had this sequence of Flush */
+#include "tools.h"
+
+int
+main()
+{
+    void *p;
+    BOOL res;
+    int size = 0x04000;
+    int commit = 0;
+    int version;
+
+    INIT();
+    version = get_windows_version();
+    assert(version != 0);
+
+    p = VirtualAlloc(0, size, MEM_RESERVE, PAGE_EXECUTE_READWRITE);
+    assert(p);
+    print("alloced %d\n",size);
+
+    commit += 0x1000;
+    p = VirtualAlloc(p, commit, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
+    assert(p);
+    print("committed %d\n",commit);
+    
+    commit += 0x1000;
+    p = VirtualAlloc(p, commit, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
+    assert(p);
+    print("committed %d\n",commit);
+
+    /* in fact nothing prevents a flush on non-reserved memory */
+    if (1) commit += 0x1000;
+    res = FlushInstructionCache(GetCurrentProcess(), p, commit);
+    assert(res);
+    print("flushed %d\n",commit);
+
+    commit += 0x1000;
+    p = VirtualAlloc(p, commit, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
+    assert(p);
+    print("committed %d\n",commit);
+
+    /* FIXME: should add VirtualQuery calls here to verify it all */
+    /* case 4494 - VirtualFree special cases */
+    res = VirtualFree((char*)p+0x2000-1, 3, MEM_DECOMMIT);
+    print("attempting to decommit 3 byte cross-page 0 - should decommit two pages\n",commit);
+    assert(res);
+
+    /* LastErrorValue: (Win32) 0x1e7 (487) - Attempt to access invalid
+     * address.  LastStatusValue: (NTSTATUS) 0xc000009f - Virtual
+     * memory cannot be freed as base address is not the base of the
+     * region and a region size of zero was specified.
+     */
+
+    res = VirtualFree((char*)p+0x3040, 0, MEM_DECOMMIT);
+    print("attempting to decommitted 3 byte cross-page 0 - should fail\n");
+    assert(res == 0 && GetLastError()==487);
+
+    res = VirtualFree((char*)p+0x10, commit, MEM_DECOMMIT);
+    print("decommitting (p+0x10, %d) (gets backwards aligned) - should hopefully fail\n", commit);
+    /*  LastErrorValue: (Win32) 0x57 (87) - The parameter is incorrect. */
+    /* LastStatusValue: (NTSTATUS) 0xc000001a - Virtual memory cannot be freed. */
+    assert(res == 0 && GetLastError()==87);
+
+    res = VirtualFree((char*)p+0x10, 0, MEM_DECOMMIT);
+    print("decommitted size 0 and p (gets backwards aligned) - should decommit whole region\n");
+    if (version == WINDOWS_VERSION_NT) {
+        /* on NT NtFreeVirtualMemory does NOT back-align the base and fails instead */
+        /* FIXME: change message above -- but then have to change template */
+        assert(res == 0 && GetLastError()==487);
+    } else {
+        assert(res);
+    }
+
+    res = VirtualFree((char*)p+0, commit, MEM_DECOMMIT);
+    print("decommitting (p+0x0, %d) - should be ok\n", commit);
+    assert(res);
+
+    /* MEM_RELEASE tests */
+    res = VirtualFree((char*)p+0x3010, 0, MEM_RELEASE);
+    print("releasing p+0x3010 - should fail\n");
+    assert(res == 0 && GetLastError()==487);
+
+    res = VirtualFree((char*)p+0x10, 0, MEM_RELEASE);
+    print("releasing p+0x10 - will actually free\n");
+    if (version == WINDOWS_VERSION_NT) {
+        /* on NT NtFreeVirtualMemory does NOT back-align the base and fails instead */
+        /* FIXME: change message above -- but then have to change template */
+        assert(res == 0 && GetLastError()==487);
+    } else {
+        assert(res);
+    }
+
+    p = VirtualAlloc(0, size, MEM_RESERVE, PAGE_EXECUTE_READWRITE);
+    assert(p);
+    print("alloced again %d\n",size);
+
+    res = VirtualFree(p, 0, MEM_RELEASE);
+    print("released p\n");
+    assert(res);
+
+    print("Successful\n");
+}
diff --git a/suite/tests/win32/virtualreserve.expect b/suite/tests/win32/virtualreserve.expect
new file mode 100644
index 000000000..aca6dfa3c
--- /dev/null
+++ b/suite/tests/win32/virtualreserve.expect
@@ -0,0 +1,15 @@
+alloced 16384
+committed 4096
+committed 8192
+flushed 12288
+committed 16384
+attempting to decommit 3 byte cross-page 0 - should decommit two pages
+attempting to decommitted 3 byte cross-page 0 - should fail
+decommitting (p+0x10, 16384) (gets backwards aligned) - should hopefully fail
+decommitted size 0 and p (gets backwards aligned) - should decommit whole region
+decommitting (p+0x0, 16384) - should be ok
+releasing p+0x3010 - should fail
+releasing p+0x10 - will actually free
+alloced again 16384
+released p
+Successful
diff --git a/suite/tests/win32/winapc.c b/suite/tests/win32/winapc.c
new file mode 100644
index 000000000..eb767afd1
--- /dev/null
+++ b/suite/tests/win32/winapc.c
@@ -0,0 +1,111 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define _WIN32_WINNT 0x0400
+#include <windows.h>
+#include <winbase.h> /* for QueueUserAPC */
+#include <process.h> /* for _beginthreadex */
+#include <stdio.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+static BOOL synch_1 = TRUE;
+static BOOL synch_2 = TRUE;
+static int result = 0;
+static int apc_arg = 0;
+
+unsigned int WINAPI
+run_func(void * arg)
+{
+    int res;
+    synch_2 = FALSE;
+    while (synch_1) {
+        /* need non alertable thread yield function */
+        SwitchToThread();
+    }
+    /* now the alertable system call */
+    res = SleepEx(100, 1);
+    /* is going to return 192 since recieved apc during sleep call 
+     * well technically 192 is io completion interruption, but seems to 
+     * report that for any interrupting APC */
+    printf("SleepEx returned %d\n", res);
+    printf("Apc arg = %d\n", apc_arg);
+    printf("Result = %d\n", result);
+    fflush(stdout);
+    return 0;
+}
+
+void WINAPI
+apc_func(unsigned long arg)
+{
+    result += 100;
+    apc_arg = arg;
+}
+
+int
+main()
+{
+    unsigned int tid;
+    unsigned long hThread;
+    int res;
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    printf("Before _beginthreadex\n");
+    fflush(stdout);
+    hThread = _beginthreadex(NULL, 0, run_func, NULL, 0, &tid);
+
+    while (synch_2) {
+        SwitchToThread();
+    }
+
+    res = QueueUserAPC(apc_func, (HANDLE)hThread, 37);
+    printf("QueueUserAPC returned %d\n", res);
+    fflush(stdout);
+    
+    synch_1 = FALSE;
+
+    WaitForSingleObject((HANDLE)hThread, INFINITE);
+
+    printf("After _beginthreadex\n");
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+    return 0;
+}
diff --git a/suite/tests/win32/winapc.expect b/suite/tests/win32/winapc.expect
new file mode 100644
index 000000000..aef9ee373
--- /dev/null
+++ b/suite/tests/win32/winapc.expect
@@ -0,0 +1,6 @@
+Before _beginthreadex
+QueueUserAPC returned 1
+SleepEx returned 192
+Apc arg = 37
+Result = 100
+After _beginthreadex
diff --git a/suite/tests/win32/winthread.c b/suite/tests/win32/winthread.c
new file mode 100644
index 000000000..23d9c3b86
--- /dev/null
+++ b/suite/tests/win32/winthread.c
@@ -0,0 +1,72 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include <process.h> /* for _beginthreadex */
+#include <stdio.h>
+
+#ifdef USE_DYNAMO
+#include "dynamorio.h"
+#endif
+
+int WINAPI
+run_func(void * arg)
+{
+    printf("In run_func\n");
+    fflush(stdout);
+    return 0;
+}
+
+int
+main()
+{
+    int tid;
+    unsigned long hThread;
+
+#ifdef USE_DYNAMO
+    dynamorio_app_init();
+    dynamorio_app_start();
+#endif
+
+    printf("Before _beginthreadex\n");
+    fflush(stdout);
+    hThread = _beginthreadex(NULL, 0, run_func, NULL, 0, &tid);
+
+    WaitForSingleObject((HANDLE)hThread, INFINITE);
+
+    printf("After _beginthreadex\n");
+
+#ifdef USE_DYNAMO
+    dynamorio_app_stop();
+    dynamorio_app_exit();
+#endif
+}
diff --git a/suite/tests/win32/winthread.expect b/suite/tests/win32/winthread.expect
new file mode 100644
index 000000000..9b670b43b
--- /dev/null
+++ b/suite/tests/win32/winthread.expect
@@ -0,0 +1,3 @@
+Before _beginthreadex
+In run_func
+After _beginthreadex
diff --git a/suite/win32-regression-expect b/suite/win32-regression-expect
new file mode 100644
index 000000000..d7eb7e38c
--- /dev/null
+++ b/suite/win32-regression-expect
@@ -0,0 +1,253 @@
+# **********************************************************
+# Copyright (c) 2003 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+============================================================
+Summary of results:
+
+Build 1: "GEN_DEFINES="/DINLINE_IND_BR_LOOKUP /DTRACE_HEAD_CACHE_INCR""
+       <no errors>
+  Run 1: DYNAMORIO_OPTIONS=\"\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 2: DYNAMORIO_OPTIONS=\"-stats\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 3: DYNAMORIO_OPTIONS=\"-loglevel 1\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 4: DYNAMORIO_OPTIONS=\"-loglevel 1 -prof_pcs\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 5: DYNAMORIO_OPTIONS=\"-loglevel 1 -stats\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 6: DYNAMORIO_OPTIONS=\"-loglevel 1 -disable_traces\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 7: DYNAMORIO_OPTIONS=\"-loglevel 1 -tracedump_binary -tracedump_origins\" TEST_MINS=20
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 8: DYNAMORIO_OPTIONS=\"-loglevel 1 -tracedump_text -tracedump_origins\" TEST_MINS=20
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 9: DYNAMORIO_OPTIONS=\"-loglevel 1 -cache_bb_max 128\" TEST_MINS=360
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 10: DYNAMORIO_OPTIONS=\"-loglevel 1 -cache_trace_max 128\" TEST_MINS=240
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 11: DYNAMORIO_OPTIONS=\"-loglevel 1 -rlr\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 12: DYNAMORIO_OPTIONS=\"-loglevel 1 -remove_dead_code 1\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 13: DYNAMORIO_OPTIONS=\"-loglevel 1 -constant_prop 1\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 14: DYNAMORIO_OPTIONS=\"-loglevel 1 -call_return_matching\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 15: DYNAMORIO_OPTIONS=\"-loglevel 1 -remove_unnecessary_zeroing\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 16: DYNAMORIO_OPTIONS=\"-loglevel 1 -peephole\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+Build 2: "CLIENT_INTERFACE=1"
+       <no errors>
+  Run 17: DYNAMORIO_OPTIONS=\"-loglevel 1\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+Build 3: "CLIENT_INTERFACE=1 EXTERNAL_RELEASE=1"
+       <no errors>
+  Run 18: DYNAMORIO_OPTIONS=\"\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 19: DYNAMORIO_OPTIONS=\"-stats\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 20: DYNAMORIO_OPTIONS=\"-loglevel 1\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 21: DYNAMORIO_OPTIONS=\"-loglevel 1 -prof_pcs\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 22: DYNAMORIO_OPTIONS=\"-loglevel 1 -stats\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 23: DYNAMORIO_OPTIONS=\"-loglevel 1 -tracedump_binary -tracedump_origins\" TEST_MINS=20
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 24: DYNAMORIO_OPTIONS=\"-loglevel 1 -tracedump_text -tracedump_origins\" TEST_MINS=20
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 25: DYNAMORIO_OPTIONS=\"-loglevel 1 -cache_bb_max 128\" TEST_MINS=360
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 26: DYNAMORIO_OPTIONS=\"-loglevel 1 -cache_trace_max 128\" TEST_MINS=240
+       SPEC: <no failures>
+       TESTS: <no failures>
+Build 4: "CLIENT_INTERFACE=1 EXTERNAL_RELEASE=1 EXTERNAL_RELEASE_PROFILE=1"
+       <no errors>
+  Run 27: DYNAMORIO_OPTIONS=\"\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 28: DYNAMORIO_OPTIONS=\"-loglevel 1 -prof_counts -tracedump_binary\" TEST_MINS=20
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 29: DYNAMORIO_OPTIONS=\"-loglevel 1 -prof_counts -tracedump_text\" TEST_MINS=20
+       SPEC: <no failures>
+       TESTS: <no failures>
+Build 5: "GEN_DEFINES="/DINLINE_IND_BR_LOOKUP /DTRACE_HEAD_CACHE_INCR /DHASHTABLE_OPEN_ADDRESS /DHASHTABLE_NOCHASING /DHASHTABLE_OPEN_ADDRESS_SENTINEL""
+       <no errors>
+  Run 30: DYNAMORIO_OPTIONS=\"-loglevel 1\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+Build 6: "GEN_DEFINES="/DTRACE_HEAD_CACHE_INCR /DHASHTABLE_OPEN_ADDRESS /DHASHTABLE_NOCHASING /DHASHTABLE_OPEN_ADDRESS_SENTINEL""
+       <no errors>
+  Run 31: DYNAMORIO_OPTIONS=\"-loglevel 1\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+Build 7: "GEN_DEFINES="/DINLINE_IND_BR_LOOKUP /DTRACE_HEAD_CACHE_INCR /DADAPTIVE_WORKING_SET""
+       <no errors>
+  Run 32: DYNAMORIO_OPTIONS=\"-loglevel 1\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+Build 8: "GEN_DEFINES="/DPROFILE_LINKCOUNT""
+       <no errors>
+  Run 33: DYNAMORIO_OPTIONS=\"-loglevel 1 -prof_counts -tracedump_binary\" TEST_MINS=20
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 34: DYNAMORIO_OPTIONS=\"-loglevel 1 -prof_counts -tracedump_text\" TEST_MINS=20
+       SPEC: <no failures>
+       TESTS: <no failures>
+Build 9: "GEN_DEFINES="/DPROFILE_LINKCOUNT /DLINKCOUNT_64_BITS""
+       <no errors>
+  Run 35: DYNAMORIO_OPTIONS=\"-loglevel 1 -prof_counts -tracedump_binary\" TEST_MINS=20
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 36: DYNAMORIO_OPTIONS=\"-loglevel 1 -prof_counts -tracedump_text\" TEST_MINS=20
+       SPEC: <no failures>
+       TESTS: <no failures>
+Build 10: "GEN_DEFINES="/DCHECK_CODE_ORIGINS""
+       <no errors>
+  Run 37: DYNAMORIO_OPTIONS=\"-loglevel 1\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 38: DYNAMORIO_OPTIONS=\"-loglevel 1 -executable_heap\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 39: DYNAMORIO_OPTIONS=\"-loglevel 1 -executable_stack\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 40: DYNAMORIO_OPTIONS=\"-loglevel 1 -executable_data\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+Build 11: "GEN_DEFINES="/DCHECK_CODE_ORIGINS /DCODE_ORIGIN_STATISTICS""
+       <no errors>
+  Run 41: DYNAMORIO_OPTIONS=\"-loglevel 1\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+Build 12: "GEN_DEFINES="/DPROTECT_FROM_APP""
+       <no errors>
+  Run 42: DYNAMORIO_OPTIONS=\"-loglevel 1\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 43: DYNAMORIO_OPTIONS=\"-loglevel 1\" TIMER= TEST_TIME=
+       SPEC: <no failures>
+       TESTS: <no failures>
+Build 13: "GEN_DEFINES="/DINLINE_IND_BR_LOOKUP /DTRACE_HEAD_CACHE_INCR /DASSUME_NORMAL_EFLAGS""
+       <no errors>
+  Run 44: DYNAMORIO_OPTIONS=\"-loglevel 1\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+Build 14: "GEN_DEFINES="/DFRAGMENT_SIZES_STUDY""
+       <no errors>
+  Run 45: DYNAMORIO_OPTIONS=\"-loglevel 1\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+Build 15: "GEN_DEFINES="/DFOOL_CPUID""
+       <no errors>
+  Run 46: DYNAMORIO_OPTIONS=\"-loglevel 1\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+Build 16: "GEN_DEFINES="/DINLINE_IND_BR_LOOKUP /DTRACE_HEAD_CACHE_INCR /DNATIVE_RETURN /DASSUME_NORMAL_EFLAGS /DCUSTOM_EXIT_STUBS""
+       <no errors>
+  Run 47: DYNAMORIO_OPTIONS=\"-loglevel 1\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+Build 17: "GEN_DEFINES="""
+       <no errors>
+  Run 48: DYNAMORIO_OPTIONS=\"-loglevel 1\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+Build 18: "DEBUG=0"
+       <no errors>
+  Run 49: DYNAMORIO_OPTIONS=\"\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 50: DYNAMORIO_OPTIONS=\"-prof_pcs\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 51: DYNAMORIO_OPTIONS=\"-tracedump_binary -tracedump_origins\" TEST_MINS=20
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 52: DYNAMORIO_OPTIONS=\"-tracedump_text -tracedump_origins\" TEST_MINS=20
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 53: DYNAMORIO_OPTIONS=\"-cache_bb_max 128\" TEST_MINS=360
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 54: DYNAMORIO_OPTIONS=\"-cache_trace_max 128\" TEST_MINS=240
+       SPEC: <no failures>
+       TESTS: <no failures>
+Build 19: "CLIENT_INTERFACE=1 EXTERNAL_RELEASE=1 DEBUG=0"
+       <no errors>
+  Run 55: DYNAMORIO_OPTIONS=\"\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 56: DYNAMORIO_OPTIONS=\"-prof_pcs\"
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 57: DYNAMORIO_OPTIONS=\"-tracedump_binary -tracedump_origins\" TEST_MINS=20
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 58: DYNAMORIO_OPTIONS=\"-tracedump_text -tracedump_origins\" TEST_MINS=20
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 59: DYNAMORIO_OPTIONS=\"-cache_bb_max 128\" TEST_MINS=360
+       SPEC: <no failures>
+       TESTS: <no failures>
+  Run 60: DYNAMORIO_OPTIONS=\"-cache_trace_max 128\" TEST_MINS=240
+       SPEC: <no failures>
+       TESTS: <no failures>
+
+============================================================
diff --git a/tools/CombineRuns b/tools/CombineRuns
new file mode 100755
index 000000000..10dc266f9
--- /dev/null
+++ b/tools/CombineRuns
@@ -0,0 +1,415 @@
+#!/usr/bin/perl -w
+
+# **********************************************************
+# Copyright (c) 2002 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+#This program parses the ouptut of Summary.perfctr and condenses multiple runs 
+#of the same benchmark into a single run (by summing all the totals)
+
+
+$usage = "Usage: $0 inputfilename -ratio <rio> <native> [-include includestring]  [-details] [-latex]\n";
+@perfctrnames=("traces","/","frags","cache_used","cache_total","flush",
+	       "minutes",
+	       "Cycles",
+	       "Inst. Decoded",
+	       "Inst. Retired",
+	       "Micro-ops Retired",
+	       "data mem refs",
+	       "DCU miss outstanding",
+	       "Inst. Fetches",
+	       "IFU misses",
+	       "ITLB misses",
+	       "inst fetch stall cycles",
+	       "inst fetch decode stalled cycles",
+	       "L2 inst fetches",
+	       "L2 loads",
+	       "L2 stores",
+	       "branches decoded",
+	       "branches",
+	       "branches mispredicted",
+	       "branches taken",
+	       "branches taken, mispredicted",
+	       "BTB misses",
+	       "Bogus branches",
+	       "resource stalls",
+	       "BACLEAR asserted",
+	       "DCU lines allocated",
+	       "L2 lines allocated",
+	       "prefetch NTA dispatched",
+	       "prefetch NTA miss all caches");
+    
+$infilename=$ARGV[0];
+open INFILE,$ARGV[0];
+shift @ARGV;
+$ratio=0;
+$details=0;
+$latex=0;
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq '-ratio') {
+	if ($#ARGV <= 0) { print $usage; exit; }
+	shift;
+	$riostring= $ARGV[0];
+	if ($#ARGV <= 0) { print $usage; exit; }
+	shift;
+	$nativestring= $ARGV[0];
+	$ratio=1;
+    } 
+    elsif ($ARGV[0] eq '-include') {
+	if ($#ARGV <= 0) { print $usage; exit; }
+	shift;
+	push @include, $ARGV[0];
+    }
+    elsif ($ARGV[0] eq '-details') {
+	$details=1;
+    }
+    elsif ($ARGV[0] eq '-latex') {
+	$latex=1;
+	$details=1;
+    } else {
+	print $usage;
+	exit;
+    }
+    shift;
+}
+
+if ($ratio==0){
+    print $usage;
+    exit;
+}
+
+$datanum=0;
+
+#skip and output first 6 lines
+for (1..6) {
+    $l=<INFILE>;
+    if ($details==0) {print $l};
+} 
+if ($details==0){
+    print "including lines w/ string: ";
+    print join " ",@include;
+    print "\n";
+}
+
+while (<INFILE>) {
+    $wholeline=$_;
+    $good=1;
+
+    foreach $searchstring (@include){
+
+	$num=($wholeline=~ s/$searchstring/$searchstring/);
+	if ($num ==0){
+	    $good=0;
+	}
+    }
+
+	
+    $wholeline=~ s/\s\/\s/ /g; 
+    $wholeline=~ s/Error/ /g; 
+   
+    if ($good==1){
+	$_= $wholeline;
+	@line=split; 
+	
+	$line[0] =~ /(.+)\/\d*$/;
+	$curname=$1;
+	shift @line;
+	push @line, 1;
+	$datalen = scalar(@line);
+
+	$bench_already_found=0;
+	
+	for $i (0..($datanum-1)){
+	    if ($runnames[$i] eq $curname){
+		for $ctrnum (0..($datalen-1)){
+		    if ($line[$ctrnum]>0){
+			$data[$i][$ctrnum]+=$line[$ctrnum];
+		    }
+		}
+		$bench_already_found=1;
+
+	    }
+	}
+
+	if($bench_already_found==0){
+	    $runnames[$datanum]=$curname;
+	    for $ctrnum (0..($datalen-1)){
+		$data[$datanum][$ctrnum]=$line[$ctrnum];
+	    }
+	    $datanum++;
+	}
+
+    }
+}
+
+if ($latex){
+    open(LATEXFILE,">$infilename.latex");
+    print "writing output to $infilename.latex\n";
+
+    print LATEXFILE "\\documentclass{article}\n";
+    print LATEXFILE "\\setlength{\\topmargin}{0.5in}\n";
+    print LATEXFILE "\\setlength{\\textheight}{10in}\n";
+    print LATEXFILE "\\setlength{\\evensidemargin}{0.0in}\n";
+    print LATEXFILE "\\setlength{\\oddsidemargin}{0.0in}\n";
+    print LATEXFILE "\\setlength{\\textwidth}{7in}\n";
+    print LATEXFILE "\\parindent=0in\n";
+    print LATEXFILE "\\pagestyle{empty}\n";
+    print LATEXFILE "\\usepackage{colortbl}\n";
+    print LATEXFILE "\\begin{document}\n";
+
+
+
+}
+
+foreach $nativenamepos (0..($datanum-1)){
+    foreach $rionamepos (0..($datanum-1)){
+	$nativename=$runnames[$nativenamepos];
+	$rioname=$runnames[$rionamepos];
+	
+	if (($nativename=~s/$nativestring/"blah"/) and
+	    ($rioname=~s/$riostring/"blah"/) and
+	    ($rioname eq $nativename)){
+	    if ($details==0){
+		#if this point is reached, then they're ok to compare
+		printdataline($nativenamepos);
+		printdataline($rionamepos);
+		
+		print "Ratio_$riostring/$nativestring\t";
+		for $ctrnum (0..($datalen-1)){
+		    if ($data[$nativenamepos][$ctrnum] == 0) {
+			$slowdown=0;
+		    } else 
+		    {
+			$slowdown=$data[$rionamepos][$ctrnum]/$data[$nativenamepos][$ctrnum];
+		    }
+		    printf("%.3f\t",$slowdown);
+		}
+		print "\n";
+	    }
+	    else{
+		printdetailedratio($riostring,$nativestring,$rionamepos,$nativenamepos);
+	    }
+	}
+	
+    }
+}
+
+if ($latex){
+    print LATEXFILE "\\end{document}\n";
+    close(LATEXFILE);
+    system("latex $infilename.latex");
+    system("dvips $infilename.dvi -o $infilename.ps");
+    system("rm $infilename.dvi $infilename.aux $infilename.log");
+    system("rm $infilename.latex");
+}
+
+sub printdetailedratio{
+    my ($riostring,$nativestring,$riopos,$nativepos)=@_;
+    
+    if ($latex){
+	print LATEXFILE "\\begin{tabular}{|l|r|r|r|}\n";
+	print LATEXFILE "\\hline\n";
+	print LATEXFILE "Comparing: &&&\\\\\n";
+	print LATEXFILE "$riostring: &&&\\\\\n$runnames[$riopos] & & & \\\\\n";
+	print LATEXFILE "$nativestring: &&&\\\\\n$runnames[$nativepos] & &&\\\\\n";
+	print LATEXFILE "&$riostring: & $nativestring & ratio \\\\\n";
+	print LATEXFILE "\\hline\n";
+    } else {
+	print "Comparing:\n$riostring: $runnames[$riopos]\n";
+	print "$nativestring: $runnames[$nativepos]\n";
+	print "\t$riostring\t$nativestring\tratio\n";
+    }
+
+    printdetailedcounter($riopos,$nativepos,7); #cycles
+    printdetailedcounter($riopos,$nativepos,8); #insts decoded
+    printdetailedcounter($riopos,$nativepos,9); #insts retired
+
+    printperfctrratio("",$riopos,$nativepos,9,7); #IPC
+
+
+    printdetailedcounter($riopos,$nativepos,10); #micro-ops
+    printperfctrratio("",$riopos,$nativepos,11,9); # % inst data mem accesses
+    printperfctrratio("Average data L1 miss latency (cycles)",
+		      $riopos,$nativepos,12,30); #dcu_miss outstanding (cycles)
+                                                 # / number dcu misses
+
+    printperfctrratio("L1 instruction miss rate",$riopos,$nativepos,14,13); #L1 inst miss / total inst fetch
+    printperfctrratio("L1 data miss rate",$riopos,$nativepos,30,11); 
+
+    printperfctrratio2("l2 miss rate",$riopos,$nativepos,31,11,13); #l2 allocated / fetches+data accesses
+
+    printperfctrratio("NTA prefetch cache miss rate",$riopos,$nativepos,33,32);
+
+    printperfctrratio("",$riopos,$nativepos,15,13); #ITLB miss / total inst fetch
+
+
+    printperfctrratio("",$riopos,$nativepos,16,7); #cycles inst. fetchstalled / total cycles
+    
+    printperfctrratio("",$riopos,$nativepos,17,7); #ifdecode cycles / cycles
+
+   
+    printdetailedcounter($riopos,$nativepos,21); #branches decoded
+    printdetailedcounter($riopos,$nativepos,22); #branches retired
+    printperfctrratio("",$riopos,$nativepos,22,9); #branches retired / insts retired
+
+    printperfctrratio("",$riopos,$nativepos,23,22); #branches mispredicted / branches retired
+
+    printperfctrratio("",$riopos,$nativepos,24,22); #branches taken / branches retired
+
+    printperfctrratio("",$riopos,$nativepos,25,22); #branches taken,mispredicted / branches 
+    printperfctrratio("",$riopos,$nativepos,26,22); #BTB misses / branches retired
+
+    printperfctrratio("",$riopos,$nativepos,27,22); #bogus branches / branches retired
+
+    printperfctrratio("",$riopos,$nativepos,28,7); #resource stalls / cycles
+    
+    printperfctrratio("",$riopos,$nativepos,29,22); #baclears / branches
+
+    printperfctrratio("",$riopos,$nativepos,11,32);
+
+    if ($latex){
+	print LATEXFILE "\\hline\n";
+	print LATEXFILE "\\end{tabular}\n\\\\\n";
+	print LATEXFILE "\\clearpage\n";
+    } else {
+	print "\n";
+    }
+}    
+
+sub printperfctrratio{
+    my ($string,$riopos,$nativepos,$ctr1,$ctr2)=@_;
+    if ($data[$riopos][$ctr2]>0){
+	$rioratio=$data[$riopos][$ctr1]/$data[$riopos][$ctr2];
+    }else{
+	$rioratio=0;
+    }
+
+    if ($data[$nativepos][$ctr2]>0){
+	$nativeratio=$data[$nativepos][$ctr1]/$data[$nativepos][$ctr2];
+    }else{
+	$nativeratio=0;
+    }
+
+    if ($nativeratio>0){
+	$ratio=$rioratio/$nativeratio;
+    }else{
+	$ratio=0;
+    }
+    
+    if ($string eq ""){
+	$string=sprintf($perfctrnames[$ctr1]."/".$perfctrnames[$ctr2]);
+    }
+
+    if ($latex){
+	printlatexline($string,$rioratio,$nativeratio,$ratio);
+    }else{
+	printf  "%s\t%.5f\t%.5f\t%.5f\n",$string,
+	$rioratio,$nativeratio,$ratio;
+    }
+}
+sub printperfctrratio2{
+    my ($string,$riopos,$nativepos,$ctr1,$ctr20,$ctr21)=@_;
+    
+    $rioratio=$data[$riopos][$ctr1]/($data[$riopos][$ctr20]+
+				     $data[$riopos][$ctr21]);
+    $nativeratio=$data[$nativepos][$ctr1]/($data[$nativepos][$ctr20]+
+					   $data[$nativepos][$ctr21]);
+    
+    if ($nativeratio>0){
+	$ratio=$rioratio/$nativeratio;
+    }else{
+	$ratio=0;
+    }
+    
+    if ($string eq ""){
+	$string=sprintf($perfctrnames[$ctr1]."/(".$perfctrnames[$ctr20]."+".$perfctrnames[$ctr21].")");
+    }
+
+    if ($latex){
+	printlatexline($string,$rioratio,$nativeratio,$ratio);
+    }else{
+	printf  "%s\t%.5f\t%.5f\t%.5f\n",$string,
+	$rioratio,$nativeratio,$ratio;
+    }
+}
+
+sub printlatexline{
+    my ($name,$rionum,$nativenum,$ratio)=@_;
+    $whitetext=0;
+    if ($ratio<.95){
+	$shading=0;
+	$whitetext=1;
+    }elsif($ratio>1.05){
+	$shading=.7;
+	$whitetext=0;
+    }else{ #greater than 1.05
+	$shading=1;
+    }
+    print LATEXFILE "\\rowcolor[gray]{$shading}\n";
+    if ($whitetext){
+	printf LATEXFILE "\\color{white}%s & \\color{white}%g & \\color{white}%g & \\color{white}%.3f \\\\\n",$name,$rionum,$nativenum,$ratio;
+    }else{
+	printf LATEXFILE "%s & %g & %g & %.3f \\\\\n",$name,$rionum,$nativenum,$ratio;
+    }
+
+}
+
+sub printdataline{
+    my ($number)=@_;
+    printf "$runnames[$number]\t";
+	
+	for $ctrnum (0..($datalen-1)){
+	    print "$data[$number][$ctrnum]";
+	 print "\t";
+	}
+	print "\n";
+}
+
+
+
+sub printdetailedcounter{
+    my ($riopos,$nativepos,$ctrnum)=@_;
+  
+
+    if ($data[$nativepos][$ctrnum]>0){
+	$slowdown=$data[$riopos][$ctrnum]/$data[$nativepos][$ctrnum];
+    }
+    else{
+	$slowdown=0;
+    }
+
+    if ($latex){	
+	printlatexline($perfctrnames[$ctrnum],$data[$riopos][$ctrnum],
+		       $data[$nativepos][$ctrnum],$slowdown);
+    } else{
+	print  "$perfctrnames[$ctrnum]\t$data[$riopos][$ctrnum]\t$data[$nativepos][$ctrnum]\t";
+	printf("%.3f\n",$slowdown);
+    }
+}
+
diff --git a/tools/DRcontrol.c b/tools/DRcontrol.c
new file mode 100644
index 000000000..990317a50
--- /dev/null
+++ b/tools/DRcontrol.c
@@ -0,0 +1,820 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * DRcontrol.c
+ *
+ * command-line tool for changing registry options
+ *
+ *
+ * (c) araksha, inc. all rights reserved
+ */
+#include <wchar.h>
+
+#include "share.h"
+#include "config.h"
+#include "processes.h"
+
+#include <stdio.h>
+
+/* convenience macros for secure string buffer operations */
+#define BUFFER_SIZE_BYTES(buf)      sizeof(buf)
+#define BUFFER_SIZE_ELEMENTS(buf)   (BUFFER_SIZE_BYTES(buf) / sizeof(buf[0]))
+#define BUFFER_LAST_ELEMENT(buf)    buf[BUFFER_SIZE_ELEMENTS(buf) - 1]
+#define NULL_TERMINATE_BUFFER(buf)  BUFFER_LAST_ELEMENT(buf) = 0
+#define BUFFER_ROOM_LEFT_W(wbuf)    (BUFFER_SIZE_ELEMENTS(wbuf) - wcslen(wbuf) - 1)
+#define BUFFER_ROOM_LEFT(abuf)      (BUFFER_SIZE_ELEMENTS(abuf) - strlen(abuf) - 1)
+
+typedef unsigned char byte;
+
+//helper from config.c, not defined in confg.h
+void dump_config_group(char *, char *, ConfigGroup *, BOOL);
+
+void 
+usage() 
+{
+    fprintf(stderr,
+            "Usage:\nDRControl [-help] [-create path] [-destroy] [-exists] [-reset] \n" 
+            "\t[-app name] [-add name] [-remove name] [-run N] [-options string] \n"
+            "\t[-detach pid] [-detachexe name] [-detachall] [-drlib dll]\n"
+            "\t[-preinject {ON|OFF|CLEAR|LIST|REPORT|LOAD_ON|LOAD_OFF|dll}]\n"
+            "\t[-create_eventlog path] [-destroy_eventlog] [-logdir path]\n"
+            "\t[-sharedcache path]\n"
+            "\t[-dump] [-fulldump] [-appdump name] [-load file] [-save file]\n"
+            "\t[-drhome path] [-modes path] [-defs path] \n"
+            "\t[-v] [-hot_patch_nudge pid] [-hot_patch_modes_nudge pid]\n"
+            "\t[-hot_patch_nudge_all] [-hot_patch_modes_nudge_all]\n"
+            "\t[-pid pid] [-all] [-timeout ms] [-delay ms]\n"
+            "\t[-drpop] [-nudge type] [-client_nudge arg] [-verbose] [-canary_default]\n"
+            "\t[-canary <full path of canary.exe> <path to a scratch folder]\n"
+            "\t[-canary_run <run_flags>] [-canary_fault <run_flag> <ops>]\n"
+            "\t[-32] [-64]\n");
+    exit(1);
+}
+
+void
+help() 
+{
+    fprintf(stderr, "Configuration Options:\n");
+    fprintf(stderr, " -create path\t\tcreate a registry and log dir setup, using 'path' as\n\t\t\tDYNAMORIO_HOME (does not change an existing setup)\n");
+    fprintf(stderr, " -destroy\t\tdelete entire product registry key\n");
+    fprintf(stderr, " -exists\t\tdisplays whether or not product reg key exists\n\t\t\t(returns 0 if exists, not 0 otherwises)\n");
+    fprintf(stderr, " -reset\t\t\tremove all app-specific keys\n");
+
+    fprintf(stderr, " -app name\t\tset app-specific options\n");
+    fprintf(stderr, " -add name\t\tadd a new application to the configuration (if not there already)\n");
+    fprintf(stderr, " -remove name\t\tremove the apps from the configuration\n");
+    fprintf(stderr, " -run N\t\t\tset global or app-specific RUNUNDER=N\n");
+    fprintf(stderr, " -options string\tsets the options string to 'string'\n");
+    fprintf(stderr, " -drlib dll\t\tsets the SC library to use; must be a fully\n");
+    fprintf(stderr, "\t\t\tqualified pathname\n");
+
+    fprintf(stderr, " -preinject {ON|OFF|CLEAR|LIST|REPORT|dll}\n\t\t\tON=set to default, OFF=remove, CLEAR=wipe out list,\n\t\t\tLIST=display current AppInit_DLLs setting;,\n\t\t\tREPORT=display current preinject setting, if any;\n\t\t\tLOAD_OFF=(vista only) turns off loading preinject library;\n\t\t\tLOAD_ON=(vista ionly) turns on loading preinject library;\n\t\t\tdll sets the preinject library to use, must be a\n\t\t\tfully qualified pathname\n");
+    fprintf(stderr, " -create_eventlog path\t\tinitializes eventlog for "PRODUCT_NAME" library at path\n");
+    fprintf(stderr, " -destroy_eventlog\t\tfrees our eventlog\n");
+    fprintf(stderr, " -drhome path\t\tsets DYNAMORIO_HOME to path\n");
+    fprintf(stderr, " -modes path\t\tsets the modes file directory to path\n");
+    fprintf(stderr, " -defs path\t\tsets the defs file directory to path\n");
+    fprintf(stderr, " -logdir path\t\tsets the logging directory\n");
+    fprintf(stderr, " -sharedcache path\t\tsets the shared DLL cache directory\n");
+    fprintf(stderr, " -dump\t\t\tdisplays the current configuration\n");
+    fprintf(stderr, " -fulldump\t\tsame as -dump (deprecated)\n");
+    fprintf(stderr, " -appdump\t\tdisplays the current app configuration\n");
+
+    fprintf(stderr, " -load file\t\tloads configuration from the specified file\n");
+    fprintf(stderr, " -save file\t\tsaves current configuration to the specified file\n");
+    fprintf(stderr, " -v\t\t\tdisplay version information\n\n");
+    fprintf(stderr, "all options/lists/strings should be enclosed in \"'s\n");
+    fprintf(stderr, "by 'app', we mean the name of an executable (eg notepad.exe), \n");
+    fprintf(stderr, "\tor the name of a svchost group (eg svchost.exe-netsvcs).\n");
+    fprintf(stderr, "when not using -app, all settings apply to the globals\n\n");
+    fprintf(stderr, " -32\t\tconfigure for 32-bit (WOW64) applications\n");
+    fprintf(stderr, " -64\t\tconfigure for 64-bit (non-WOW64) applications\n");
+
+    fprintf(stderr, "Control Options:\n");
+    fprintf(stderr, " -detach pid\t\tdetaches from indicated pid\n");
+    fprintf(stderr, " -detachexe name\tdetaches from all processes with given .exe name\n");
+    fprintf(stderr, " -detachall\t\tdetaches from all processes\n");
+    fprintf(stderr, " -hot_patch_nudge\tforces hot patch defs and modes information to be re-read for pid\n");
+    fprintf(stderr, " -hot_patch_modes_nudge\tforces hot patch modes information to be re-read for pid\n");
+    fprintf(stderr, " -hot_patch_nudge_all\tforces hot patch defs and modes information to be re-read for all processes\n");
+    fprintf(stderr, " -hot_patch_modes_nudge_all\tforces hot patch modes information to be re-read for all processes\n");
+    fprintf(stderr, " -pid pid\t\tpid to be nudged\n");
+    fprintf(stderr, " -all or -pid -1\t\tnudge all DR processes\n");
+    fprintf(stderr, " -delay ms\t\tdelay between nudges\n");
+    fprintf(stderr, " -timeout ms\t\texpected time for nudge completion\n");
+    fprintf(stderr, " -nudge type\t\tnudge action, can be repeated\n");
+    fprintf(stderr, " -client_nudge arg\t\tsend client nudge with specifed hex arg\n");
+    /* same as -nudge reset -nudge opt, but NOT the same as -reset which wipes out the registry!  */
+    fprintf(stderr, " -drpop\t\t\tcache reset\n");
+    fprintf(stderr, " -canary_default\trun canary test as PE would using registry implict setup, returns canary code\n");
+    fprintf(stderr, " -canary path_canary path_scratch_folder\trun customized canary test, returns canary code\n");
+    fprintf(stderr, " -canary_run run_flags\toverride the runs flags for the canary run\n");
+    fprintf(stderr, " -canary_fault run_flag ops\tinject a fault at canary run run_flag by setting canry options to ops\n");
+
+    exit(1);
+}
+
+void
+checked_operation(char *label, DWORD res)
+{
+    if (res != ERROR_SUCCESS) {
+        fprintf(stderr, "Error %d on operation: %s\n", res, label);
+        exit(-1);
+    }
+}
+
+
+int 
+main(int argc, char **argv)
+{
+    int res=0,
+        run=0,
+        dump=0,
+        reset=0,
+        detachall=0,
+        detachpid=0,
+        all=0,                  /* applies to all running processes */
+        pid=-1,                 /* applies to pid, -1 means -all */
+        hotp_nudge_pid=0,
+        hotp_modes_nudge_pid=0,
+        hotp_nudge_all=0,
+        hotp_modes_nudge_all=0,
+        nudge=0,                /* generic nudge with argument */
+        nudge_action_mask=0,    /* generic nudge action mask */
+        delay_ms_all=           /* delay between acting on processes */
+                NUDGE_NO_DELAY,         
+        timeout_ms=             /* timeout for finishing a nudge on a single process */
+                DETACH_RECOMMENDED_TIMEOUT,
+        runval=0,
+        canary_default=0,
+        canary_run = CANARY_RUN_FLAGS_DEFAULT,
+        canary_fault_run = 0,
+        exists = 0,
+        destroy = 0,
+        free_eventlog = 0;
+
+    uint64 nudge_client_arg=0;     /* client nudge argument */
+
+    int verbose = 0;
+
+    char *create=NULL,
+        *addapp=NULL, 
+        *appdump=NULL, 
+        *removeapp=NULL, 
+        *opstring=NULL,
+        *drdll=NULL,
+        *preinject=NULL,
+        *logdir=NULL,
+        *sharedcache=NULL,
+        *appname=NULL,
+        *drhome=NULL,
+        *modes=NULL,
+        *defs=NULL,
+        *detach_exename=NULL,
+        *load=NULL,
+        *save=NULL,
+        *eventlog=NULL,
+        *canary_process=NULL,
+        *scratch_folder=NULL,
+        *canary_fault_ops=NULL;
+
+    dr_platform_t dr_platform = DR_PLATFORM_DEFAULT;
+    
+    int argidx=1;
+
+    WCHAR wbuf[MAX_PATH];
+    ConfigGroup *policy, *working_group;
+
+    if (argc < 2) 
+        usage();
+
+    while (argidx < argc) {
+
+        if (!strcmp(argv[argidx], "-help")) {
+  	    help();
+	}
+        /* ******************** actions on active processes ******************** */
+	else if (!strcmp(argv[argidx], "-detachall")) {
+	    detachall=1;
+	}
+	else if (!strcmp(argv[argidx], "-detach")) {
+            if (++argidx >= argc)
+                usage();
+	    detachpid=atoi(argv[argidx]);
+	}
+	else if (!strcmp(argv[argidx], "-detachexe")) {
+            if (++argidx >= argc)
+                usage();
+	    detach_exename=argv[argidx];
+	}
+        else if (!strcmp(argv[argidx], "-pid") || !strcmp(argv[argidx], "-p")) {
+            if (++argidx >= argc)
+                usage();
+            pid=atoi(argv[argidx]);
+        }
+	else if (!strcmp(argv[argidx], "-all")) {
+	    all=1;
+	}
+        else if (!strcmp(argv[argidx], "-delay")) {
+            /* in milliseconds */
+            if (++argidx >= argc)
+                usage();
+            delay_ms_all=atoi(argv[argidx]);
+        }
+        else if (!strcmp(argv[argidx], "-timeout")) {
+            /* in milliseconds */
+            if (++argidx >= argc)
+                usage();
+            timeout_ms=atoi(argv[argidx]);
+        }
+	else if (!strcmp(argv[argidx], "-hot_patch_nudge")) {
+            if (++argidx >= argc)
+                usage();
+	    hotp_nudge_pid=atoi(argv[argidx]);
+	}
+	else if (!strcmp(argv[argidx], "-hot_patch_modes_nudge")) {
+            if (++argidx >= argc)
+                usage();
+	    hotp_modes_nudge_pid=atoi(argv[argidx]);
+	}
+	else if (!strcmp(argv[argidx], "-hot_patch_nudge_all")) {
+	    hotp_nudge_all = 1;
+	}
+	else if (!strcmp(argv[argidx], "-verbose")) {
+	    verbose = 1;
+	}
+	else if (!strcmp(argv[argidx], "-hot_patch_modes_nudge_all")) {
+	    hotp_modes_nudge_all = 1;
+	}
+	else if (!strcmp(argv[argidx], "-drpop")) {
+	    nudge = 1;
+            /* allow composition */
+	    nudge_action_mask |= NUDGE_GENERIC(opt) | NUDGE_GENERIC(reset);
+	}
+	else if (!strcmp(argv[argidx], "-nudge")) {
+            int nudge_numeric;
+            if (++argidx >= argc)
+                usage();
+            nudge_numeric = atoi(argv[argidx]); /* 0 if fails */
+            nudge_action_mask |= nudge_numeric;
+
+            /* compare against numeric new code, or against symbolic names */
+            /* -nudge opt -nudge reset -nudge stats -nudge 30000 */
+            {
+                int found = 0;
+#define NUDGE_DEF(name, comment) if (strcmp(#name, argv[argidx]) == 0) { found = 1; nudge_action_mask |= NUDGE_GENERIC(name);}
+                NUDGE_DEFINITIONS();
+#undef NUDGE_DEF
+                if (!found && nudge_numeric == 0) {
+                    printf("unknown -nudge %s\n", argv[argidx]);
+                    usage();
+                }
+            }   
+
+	    nudge=1;
+	}
+        else if (!strcmp(argv[argidx], "-client_nudge")) {
+            if (++argidx >= argc)
+                usage();
+            nudge_client_arg = _strtoui64(argv[argidx], NULL, 16);
+            nudge_action_mask |= NUDGE_GENERIC(client);
+            nudge = 1;
+        }
+        /* ******************** configuration actions ******************** */
+	else if (!strcmp(argv[argidx], "-reset")) {
+	    reset=1;
+	}
+	else if (!strcmp(argv[argidx], "-create")) {
+            if (++argidx >= argc)
+                usage();
+	    create = argv[argidx];
+	}
+	else if (!strcmp(argv[argidx], "-destroy")) {
+            destroy = 1;
+	}
+	else if (!strcmp(argv[argidx], "-exists")) {
+            exists = 1;
+	}
+	else if (!strcmp(argv[argidx], "-run")) {
+            run = 1;
+            if (++argidx >= argc)
+                usage();
+            runval = atoi(argv[argidx]);
+	}
+	else if (!strcmp(argv[argidx], "-app")) {
+            if (++argidx >= argc)
+                usage();
+            appname = argv[argidx];
+	}
+	else if (!strcmp(argv[argidx], "-add")) {
+            if (++argidx >= argc)
+                usage();
+	    addapp = argv[argidx];
+	}
+	else if (!strcmp(argv[argidx], "-remove")) {
+            if (++argidx >= argc)
+                usage();
+	    removeapp = argv[argidx];
+	}
+	else if (!strcmp(argv[argidx], "-options")) {
+            if (++argidx >= argc)
+                usage();
+	    opstring = argv[argidx];
+	}
+	else if (!strcmp(argv[argidx], "-drlib")) {
+            if (++argidx >= argc)
+                usage();
+	    drdll = argv[argidx];
+	}
+	else if (!strcmp(argv[argidx], "-preinject")) {
+            if (++argidx >= argc)
+                usage();
+            preinject = argv[argidx];
+	}
+        else if (!strcmp(argv[argidx], "-create_eventlog")) {
+            if (++argidx >= argc)
+                usage();
+            eventlog = argv[argidx];
+        }
+	else if (!strcmp(argv[argidx], "-destroy_eventlog")) {
+            free_eventlog = 1;
+	}
+	else if (!strcmp(argv[argidx], "-drhome")) {
+            if (++argidx >= argc)
+                usage();
+            drhome = argv[argidx];
+	}
+	else if (!strcmp(argv[argidx], "-modes")) {
+            if (++argidx >= argc)
+                usage();
+            modes = argv[argidx];
+	}
+	else if (!strcmp(argv[argidx], "-defs")) {
+            if (++argidx >= argc)
+                usage();
+            defs = argv[argidx];
+	}
+	else if (!strcmp(argv[argidx], "-logdir")) {
+            if (++argidx >= argc)
+                usage();
+	    logdir = argv[argidx];
+	}
+	else if (!strcmp(argv[argidx], "-sharedcache")) {
+            if (++argidx >= argc)
+                usage();
+	    sharedcache = argv[argidx];
+	}
+	else if (!strcmp(argv[argidx], "-load")) {
+            if (++argidx >= argc)
+                usage();
+	    load = argv[argidx];
+	}
+        else if (!strcmp(argv[argidx], "-save")) {
+            if (++argidx >= argc)
+                usage();
+            save = argv[argidx];
+        }
+	else if (!strcmp(argv[argidx], "-dump")) {
+	    dump = 1;
+	}
+	else if (!strcmp(argv[argidx], "-appdump")) {
+            if (++argidx >= argc)
+                usage();
+            appdump = argv[argidx];	    
+	}
+	else if (!strcmp(argv[argidx], "-fulldump")) {
+	    dump = 1;
+	}
+	else if (!strcmp(argv[argidx], "-v")) {
+#ifdef BUILD_NUMBER
+	  printf("DRcontrol.exe build %d -- %s", BUILD_NUMBER, __DATE__);
+#else
+	  printf("DRcontrol.exe custom build -- %s, %s", __DATE__, __TIME__);
+#endif
+	} else if (!strcmp(argv[argidx], "-canary_default")) {
+            canary_default = 1;
+	} else if (!strcmp(argv[argidx], "-canary")) {
+            if (++argidx >= argc)
+                usage();
+	    canary_process=argv[argidx];
+            if (++argidx >= argc)
+                usage();
+	    scratch_folder=argv[argidx];
+	} else if (!strcmp(argv[argidx], "-canary_run")) {
+            if (++argidx >= argc)
+                usage();
+	    canary_run = strtol(argv[argidx], NULL, 0);
+	} else if (!strcmp(argv[argidx], "-canary_fault")) {
+            char *dummy;
+            if (++argidx >= argc)
+                usage();
+            canary_fault_run = strtol(argv[argidx], &dummy, 0);
+            if (++argidx >= argc)
+                usage();
+            canary_fault_ops = argv[argidx];
+	} else if (!strcmp(argv[argidx], "-32")) {
+	    dr_platform = DR_PLATFORM_32BIT;
+	} else if (!strcmp(argv[argidx], "-64")) {
+	    dr_platform = DR_PLATFORM_64BIT;
+	} else {
+	    fprintf(stderr, "Unknown option: %s\n", argv[argidx]);
+	    usage();
+	}
+	argidx++;
+    }
+  
+    /* PR 244206: set the registry view before any registry access */
+    set_dr_platform(dr_platform);
+
+    if (canary_process != NULL || canary_default != 0) {
+        BOOL result = TRUE;
+        WCHAR canary_fault_args[MAX_PATH];
+        CANARY_INFO info = {0};
+
+        info.run_flags = canary_run;
+        info.info_flags = CANARY_INFO_FLAGS_DEFAULT;
+        info.fault_run = canary_fault_run;
+        _snwprintf(canary_fault_args, BUFFER_SIZE_ELEMENTS(canary_fault_args),
+                   L"%S", canary_fault_ops);
+        NULL_TERMINATE_BUFFER(canary_fault_args);
+        info.canary_fault_args = canary_fault_args;
+
+        if (canary_process != NULL && *canary_process != '\0' &&
+            scratch_folder != NULL && *scratch_folder != '\0') {
+            wchar_t canary[MAX_PATH], scratch[MAX_PATH], out[MAX_PATH];
+            FILE *out_file;
+            _snwprintf(canary, BUFFER_SIZE_ELEMENTS(canary), L"%S", canary_process);
+            NULL_TERMINATE_BUFFER(canary);
+            _snwprintf(scratch, BUFFER_SIZE_ELEMENTS(scratch), L"%S\\canary_test",
+                       scratch_folder);
+            NULL_TERMINATE_BUFFER(scratch);
+            CreateDirectory(scratch, NULL);
+            _snwprintf(out, BUFFER_SIZE_ELEMENTS(out), L"%S\\canary_report.crep",
+                       scratch_folder);
+            out_file = _wfopen(out, L"wb");
+            /* FIXME - check directory, out_file, and canary proc exist */
+            result = run_canary_test_ex(out_file, &info, scratch, canary);
+        } else if (canary_default != 0) {
+            result = run_canary_test(&info, L_EXPAND_LEVEL(STRINGIFY(BUILD_NUMBER)));
+            printf("See report file \"%S\"\n", info.report);
+        }
+        printf("Canary test - %s enable protection - code 0x%08x\n"
+               "  msg=\"%S\"\n  url=\"%S\"\n", result ? "do" : "don\'t",
+               info.canary_code, info.msg, info.url);
+        return info.canary_code;
+    }
+
+    if (exists) {
+        if (get_dynamorio_home() != NULL) {
+            printf("Registry setup exists\n");
+            return 0;
+        }
+        printf("Registry setup doesn't exist\n");
+        return 1;
+    }    
+
+    if (save) {
+        _snwprintf(wbuf, MAX_PATH, L"%S", save);
+        NULL_TERMINATE_BUFFER(wbuf);
+        checked_operation("save policy", save_policy(wbuf));
+    }
+
+    if (destroy) {
+        checked_operation("delete product key", destroy_root_key());
+        if (!load && create == NULL)
+            return 0;
+    }
+
+    if (load) {
+        _snwprintf(wbuf, MAX_PATH, L"%S", load);
+        NULL_TERMINATE_BUFFER(wbuf);
+        checked_operation("load policy", load_policy(wbuf, FALSE, NULL));
+    }
+
+    if (create != NULL) {
+        _snwprintf(wbuf, MAX_PATH, L"%S", create);
+        NULL_TERMINATE_BUFFER(wbuf);
+        /* FALSE: do not overwrite (preserves old behavior) */
+        checked_operation("create registry", setup_installation(wbuf, FALSE));
+    }
+
+    checked_operation("read config",
+                      read_config_group(&policy, L_PRODUCT_NAME, TRUE));
+    
+    /* ensure we init dynamorio_home, case 4009 */
+    get_dynamorio_home(); /* ignore return value */
+
+    if (reset) {
+        remove_children(policy);
+        policy->should_clear = TRUE;
+        checked_operation("write registry", write_config_group(policy));
+    }
+
+    working_group = policy;
+
+    if (dump || appdump)
+        goto dumponly;
+
+    if (preinject) {
+        if (0 == strcmp(preinject, "OFF")) {
+            checked_operation("unset autoinject", unset_autoinjection());
+        }
+        else if (0 == strcmp(preinject, "ON")) {
+            checked_operation("set autoinject", set_autoinjection());
+        }
+        else if (0 == strcmp(preinject, "CLEAR")) {
+            checked_operation("clear autoinject", 
+                              set_autoinjection_ex(FALSE, 
+                                                   APPINIT_USE_WHITELIST,
+                                                   NULL,
+                                                   L"",
+                                                   NULL,
+                                                   NULL,
+                                                   NULL,
+                                                   0));
+        }
+        else if (0 == strcmp(preinject, "LIST")) {
+            WCHAR list[MAX_PARAM_LEN];
+            checked_operation("read appinit",
+                              get_config_parameter(INJECT_ALL_KEY_L, 
+                                                   TRUE, 
+                                                   INJECT_ALL_SUBKEY_L, 
+                                                   list, 
+                                                   MAX_PARAM_LEN));
+            printf("%S\n", list);
+            if (is_vista()) {
+                printf("LoadAppInit is %s\n",
+                       is_loadappinit_set() ? "on" : "off");
+            }
+        }
+        else if (0 == strcmp(preinject, "REPORT")) {
+            WCHAR list[MAX_PARAM_LEN], *entry, *sep;
+            checked_operation("read appinit",
+                              get_config_parameter(INJECT_ALL_KEY_L, 
+                                                   TRUE, 
+                                                   INJECT_ALL_SUBKEY_L, 
+                                                   list, 
+                                                   MAX_PARAM_LEN));
+            entry = get_entry_location(list, L_EXPAND_LEVEL(INJECT_DLL_8_3_NAME),
+                                       APPINIT_SEPARATOR_CHAR);
+            if (NULL != entry) {
+                sep = wcschr(entry, APPINIT_SEPARATOR_CHAR);
+                if (NULL != sep)
+                    *sep = L'\0';
+                printf("%S\n", entry);
+                if (is_vista()) {
+                    printf("LoadAppInit is %s\n",
+                           is_loadappinit_set() ? "on" : "off");
+                }
+            }
+        }
+        else if (0 == strcmp(preinject, "LOAD_OFF")) {
+            checked_operation("unset load autoinject", unset_loadappinit());
+        }
+        else if (0 == strcmp(preinject, "LOAD_ON")) {
+            checked_operation("set load autoinject", set_loadappinit());
+        }
+        else {
+            _snwprintf(wbuf, MAX_PATH, L"%S", preinject);
+            NULL_TERMINATE_BUFFER(wbuf);
+            checked_operation("set custom autoinject", 
+                              set_autoinjection_ex(TRUE, APPINIT_OVERWRITE,
+                                                   NULL, NULL, NULL, wbuf,
+                                                   NULL, 0));
+        }
+
+        if (0 != strcmp(preinject, "LIST") &&
+            0 != strcmp(preinject, "REPORT") &&
+            using_system32_for_preinject(NULL)) {
+            DWORD platform;
+            if (get_platform(&platform) == ERROR_SUCCESS &&
+                platform == PLATFORM_WIN_NT_4) {
+                fprintf(stderr, "Warning! On NT4, new AppInit_DLLs setting will not take effect until reboot!\n");
+            }
+        }
+
+    }
+
+    if (free_eventlog) {
+        checked_operation("free eventlog", destroy_eventlog());
+    }
+
+    if (eventlog) {
+        _snwprintf(wbuf, BUFFER_SIZE_ELEMENTS(wbuf), L"%S", eventlog);
+        NULL_TERMINATE_BUFFER(wbuf);
+        checked_operation("create eventlog", create_eventlog(wbuf));
+    }
+
+    if (nudge) {
+        if (verbose)
+            printf("-nudge %d -pid %d %s\n", nudge_action_mask, pid, all ? "all" : "");
+        if (pid == -1)           /* explicitly set */
+            all = 1;
+
+        if (all)
+            checked_operation("nudge all", 
+                              generic_nudge_all(nudge_action_mask, nudge_client_arg,
+                                                timeout_ms, delay_ms_all));
+        else
+            checked_operation("nudge", 
+                              generic_nudge(pid, TRUE,
+                                            nudge_action_mask,
+                                            0, /* client ID (ignored here) */
+                                            nudge_client_arg,
+                                            timeout_ms));
+        goto finished;
+    }
+
+    if (detachall) {
+        checked_operation("detach all", 
+                          detach_all(timeout_ms));
+        goto finished;
+    }
+    if (detachpid) {
+        checked_operation("detach", 
+                          detach(detachpid, TRUE, timeout_ms));
+        goto finished;
+    }
+
+    if (detach_exename) {
+        _snwprintf(wbuf, MAX_PATH, L"%S", detach_exename);
+        NULL_TERMINATE_BUFFER(wbuf);
+        checked_operation("detach-exe", 
+                          detach_exe(wbuf, timeout_ms));
+        goto finished;
+    }
+
+    
+    if (hotp_nudge_pid) {
+        checked_operation("hot patch update", 
+                          hotp_notify_defs_update(hotp_nudge_pid, TRUE,
+                                                  timeout_ms));
+        goto finished;
+    }
+
+    if (hotp_modes_nudge_pid) {
+        checked_operation("hot patch modes update", 
+                          hotp_notify_modes_update(hotp_modes_nudge_pid, TRUE,
+                                                   timeout_ms));
+        goto finished;
+    }
+
+    if (hotp_nudge_all) {
+        checked_operation("hot patch nudge all", 
+                          hotp_notify_all_defs_update(timeout_ms));
+        goto finished;
+    }
+
+    if (hotp_modes_nudge_all) {
+        checked_operation("hot patch modes nudge all", 
+                          hotp_notify_all_modes_update(timeout_ms));
+        goto finished;
+    }
+
+    /* configuration */
+
+
+    if (addapp) {
+        _snwprintf(wbuf, MAX_PATH, L"%S", addapp);
+        NULL_TERMINATE_BUFFER(wbuf);
+        if (NULL == get_child(wbuf, policy)) {
+            add_config_group(policy, new_config_group(wbuf));
+        }
+    }
+
+    if (removeapp) {
+        _snwprintf(wbuf, MAX_PATH, L"%S", removeapp);
+        NULL_TERMINATE_BUFFER(wbuf);
+        remove_child(wbuf, policy);
+        policy->should_clear = TRUE;
+    }
+
+    if (appname) {
+        _snwprintf(wbuf, MAX_PATH, L"%S", appname);
+        NULL_TERMINATE_BUFFER(wbuf);
+        working_group = get_child(wbuf, policy);
+
+        if (NULL == working_group) {
+            working_group = new_config_group(wbuf);
+            add_config_group(policy, working_group);
+        }
+    }
+    
+    if (run) {
+        _snwprintf(wbuf, MAX_PATH, L"%d", runval);
+        NULL_TERMINATE_BUFFER(wbuf);
+        set_config_group_parameter(working_group, 
+                                   L_DYNAMORIO_VAR_RUNUNDER, wbuf);
+    }
+
+    if (opstring) {
+        _snwprintf(wbuf, MAX_PATH, L"%S", opstring);
+        NULL_TERMINATE_BUFFER(wbuf);
+        set_config_group_parameter(working_group,
+                                   L_DYNAMORIO_VAR_OPTIONS, wbuf);
+    }
+    
+    if (drdll) {
+        _snwprintf(wbuf, MAX_PATH, L"%S", drdll);
+        NULL_TERMINATE_BUFFER(wbuf);
+        set_config_group_parameter(working_group, 
+                                   L_DYNAMORIO_VAR_AUTOINJECT, wbuf);
+    }
+
+    if (drhome) {
+        _snwprintf(wbuf, MAX_PATH, L"%S", drhome);
+        NULL_TERMINATE_BUFFER(wbuf);
+        set_config_group_parameter(working_group, 
+                                   L_DYNAMORIO_VAR_HOME, wbuf);
+    }
+
+    if (modes) {
+        _snwprintf(wbuf, MAX_PATH, L"%S", modes);
+        NULL_TERMINATE_BUFFER(wbuf);
+        set_config_group_parameter(working_group, 
+                                   L_DYNAMORIO_VAR_HOT_PATCH_MODES, wbuf);
+    }
+
+    if (defs) {
+        _snwprintf(wbuf, MAX_PATH, L"%S", defs);
+        NULL_TERMINATE_BUFFER(wbuf);
+        set_config_group_parameter(working_group, 
+                                   L_DYNAMORIO_VAR_HOT_PATCH_POLICIES, wbuf);
+    }
+
+    if (logdir) {
+        _snwprintf(wbuf, MAX_PATH, L"%S", logdir);
+        NULL_TERMINATE_BUFFER(wbuf);
+        set_config_group_parameter(working_group, 
+                                   L_DYNAMORIO_VAR_LOGDIR, wbuf);
+    }
+
+    if (sharedcache) {
+        /* note if the sharedcache root directory doesn't exist it should be
+         * created before calling this function */
+        _snwprintf(wbuf, MAX_PATH, L"%S", sharedcache);
+        NULL_TERMINATE_BUFFER(wbuf);
+
+        res = setup_cache_shared_directories(wbuf);
+        if (res != ERROR_SUCCESS) {
+            fprintf(stderr, "error %d creating directories!\n", res);
+        }
+        setup_cache_shared_registry(wbuf, working_group);
+    }
+
+    checked_operation("write policy", write_config_group(policy));
+
+ dumponly:
+
+    if (appdump) {
+        _snwprintf(wbuf, MAX_PATH, L"%S", appdump);
+        NULL_TERMINATE_BUFFER(wbuf);
+        working_group = get_child(wbuf, policy);
+    }
+    else {
+        working_group = policy;
+    }
+
+    if (dump || appdump) {
+        if (NULL == working_group)
+            fprintf(stderr, "No Configuration Exists!\n");
+        else
+            dump_config_group("","  ",working_group,FALSE);
+    }
+
+ finished:
+    free_config_group(policy);
+
+    return 0;
+
+}
diff --git a/tools/DRgui/CmdlineDlg.cpp b/tools/DRgui/CmdlineDlg.cpp
new file mode 100644
index 000000000..7e2a7df8f
--- /dev/null
+++ b/tools/DRgui/CmdlineDlg.cpp
@@ -0,0 +1,138 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// CmdlineDlg.cpp : implementation file
+//
+
+#ifndef DYNAMORIO_DEMO /* around whole file */
+
+#include "stdafx.h"
+#include "DynamoRIO.h"
+#include "CmdlineDlg.h"
+
+#ifdef _DEBUG
+#define new DEBUG_NEW
+#undef THIS_FILE
+static char THIS_FILE[] = __FILE__;
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+// CCmdlineDlg dialog
+
+
+CCmdlineDlg::CCmdlineDlg(CWnd* pParent /*=NULL*/)
+    : CDialog(CCmdlineDlg::IDD, pParent)
+{
+    //{{AFX_DATA_INIT(CCmdlineDlg)
+    m_CmdLine = _T("");
+    m_WorkingDir = _T("");
+    //}}AFX_DATA_INIT
+}
+
+CCmdlineDlg::CCmdlineDlg(CString wdir, CWnd* pParent /*=NULL*/)
+    : CDialog(CCmdlineDlg::IDD, pParent)
+{
+    //{{AFX_DATA_INIT(CCmdlineDlg)
+    m_CmdLine = _T("");
+    //}}AFX_DATA_INIT
+    m_WorkingDir = wdir;
+}
+
+
+void CCmdlineDlg::DoDataExchange(CDataExchange* pDX)
+{
+    CDialog::DoDataExchange(pDX);
+    //{{AFX_DATA_MAP(CCmdlineDlg)
+    DDX_Text(pDX, IDC_CMDLINE, m_CmdLine);
+    DDX_Text(pDX, IDC_WORKING_DIR, m_WorkingDir);
+    //}}AFX_DATA_MAP
+}
+
+
+BEGIN_MESSAGE_MAP(CCmdlineDlg, CDialog)
+    //{{AFX_MSG_MAP(CCmdlineDlg)
+    ON_BN_CLICKED(IDC_WORKING_DIR_BROWSE, OnWorkingDirBrowse)
+    //}}AFX_MSG_MAP
+    END_MESSAGE_MAP()
+
+    /////////////////////////////////////////////////////////////////////////////
+    // CCmdlineDlg message handlers
+
+    BOOL CCmdlineDlg::OnInitDialog()
+{
+    CDialog::OnInitDialog();
+    UpdateData(FALSE); // write to controls
+    return TRUE;
+}
+
+static TCHAR szFilter[] = _T("Directories (*)|*|All Files (*.*)|*.*||");
+
+void CCmdlineDlg::OnWorkingDirBrowse() 
+{
+    TCHAR folder[MAX_PATH];
+    BROWSEINFO bi;
+    bi.hwndOwner = m_hWnd;
+    bi.pidlRoot = NULL;
+    bi.pszDisplayName = folder;
+    bi.lpszTitle = _T("Select Working Directory");
+    bi.ulFlags = 0;
+    bi.lpfn = NULL;
+    bi.lParam = NULL;
+    bi.iImage = 0;
+    ITEMIDLIST *id = SHBrowseForFolder(&bi);
+    if (id == NULL) // cancelled
+        return;
+    SHGetPathFromIDList(id, folder);
+    m_WorkingDir.Format(_T("%s"), folder);
+    UpdateData(FALSE);
+}
+
+CString CCmdlineDlg::GetWorkingDir()
+{
+    return m_WorkingDir;
+}
+
+CString CCmdlineDlg::GetArguments()
+{
+    return m_CmdLine;
+}
+
+void CCmdlineDlg::OnOK() 
+{
+    CDialog::OnOK();
+}
+
+#endif /* !DYNAMORIO_DEMO */ /* around whole file */
diff --git a/tools/DRgui/CmdlineDlg.h b/tools/DRgui/CmdlineDlg.h
new file mode 100644
index 000000000..cb227c65d
--- /dev/null
+++ b/tools/DRgui/CmdlineDlg.h
@@ -0,0 +1,88 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+#if !defined(AFX_CMDLINEDLG_H__762C8A3E_B123_480F_9CF7_56ABA0FA6CC6__INCLUDED_)
+#define AFX_CMDLINEDLG_H__762C8A3E_B123_480F_9CF7_56ABA0FA6CC6__INCLUDED_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif // _MSC_VER > 1000
+// CmdlineDlg.h : header file
+//
+
+/////////////////////////////////////////////////////////////////////////////
+// CCmdlineDlg dialog
+
+class CCmdlineDlg : public CDialog
+{
+    // Construction
+public:
+    CString GetArguments();
+    CString GetWorkingDir();
+    BOOL OnInitDialog();
+    CCmdlineDlg(CString wdir, CWnd* pParent = NULL);
+
+    // Dialog Data
+    //{{AFX_DATA(CCmdlineDlg)
+    enum { IDD = IDD_CMDLINE };
+    CString m_CmdLine;
+    CString m_WorkingDir;
+    //}}AFX_DATA
+
+
+    // Overrides
+    // ClassWizard generated virtual function overrides
+    //{{AFX_VIRTUAL(CCmdlineDlg)
+protected:
+    virtual void DoDataExchange(CDataExchange* pDX);    // DDX/DDV support
+    //}}AFX_VIRTUAL
+
+    // Implementation
+protected:
+    CCmdlineDlg(CWnd* pParent = NULL);   // standard constructor
+
+    // Generated message map functions
+    //{{AFX_MSG(CCmdlineDlg)
+    afx_msg void OnWorkingDirBrowse();
+    virtual void OnOK();
+    //}}AFX_MSG
+    DECLARE_MESSAGE_MAP()
+        };
+
+//{{AFX_INSERT_LOCATION}}
+// Microsoft Visual C++ will insert additional declarations immediately before the previous line.
+
+#endif // !defined(AFX_CMDLINEDLG_H__762C8A3E_B123_480F_9CF7_56ABA0FA6CC6__INCLUDED_)
diff --git a/tools/DRgui/DynamoRIO.cpp b/tools/DRgui/DynamoRIO.cpp
new file mode 100644
index 000000000..e769a8cf2
--- /dev/null
+++ b/tools/DRgui/DynamoRIO.cpp
@@ -0,0 +1,920 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// DynamoRIO.cpp : Defines the class behaviors for the application.
+//
+
+#include "stdafx.h"
+#include "DynamoRIO.h"
+
+#include "MainFrm.h"
+#include "DynamoRIODoc.h"
+#include "DynamoRIOView.h"
+#include "OptionsDlg.h"
+#include "IgnoreDlg.h"
+#include "SyswideDlg.h"
+
+#include <assert.h>
+#include <process.h> // for system()
+
+#ifdef _DEBUG
+#define new DEBUG_NEW
+#undef THIS_FILE
+static char THIS_FILE[] = __FILE__;
+#endif
+
+#define HELP_PATH          _T("\\doc\\index.html")
+
+// FIXME: these are duplicated with the installation wizard, so
+// that the GUI can set env vars for new users (installer can only 
+// set for user installing)
+#define INITIAL_OPTIONS      _T("-stats -loglevel 1")
+#define INITIAL_SYSTEMWIDE   _T("\\lib\\debug\\dynamorio.dll")
+#define INITIAL_IGNORE       _T("drinject.exe;DynamoRIO.exe")
+//#define INITIAL_IGNORE       _T("drinject.exe;DynamoRIO.exe;CSRSS.EXE;WINLOGON.EXE;SERVICES.EXE;LSASS.EXE;svchost.exe;SPOOLSV.EXE;taskmgr.exe;jconfigdnt.exe;explorer.exe")
+
+/////////////////////////////////////////////////////////////////////////////
+// CDynamoRIOApp
+
+BEGIN_MESSAGE_MAP(CDynamoRIOApp, CWinApp)
+    //{{AFX_MSG_MAP(CDynamoRIOApp)
+    ON_COMMAND(ID_APP_ABOUT, OnAppAbout)
+    ON_COMMAND(ID_APP_EXIT, OnAppExit)
+#ifndef DYNAMORIO_DEMO
+    ON_COMMAND(ID_EDIT_OPTIONS, OnEditOptions)
+    ON_COMMAND(ID_LIBRARY_RELEASE, OnLibraryRelease)
+    ON_COMMAND(ID_LIBRARY_PROFILE, OnLibraryProfile)
+    ON_COMMAND(ID_LIBRARY_DEBUG, OnLibraryDebug)
+    ON_COMMAND(ID_FILE_RUN, OnFileRun)
+    ON_COMMAND(ID_FILE_SYSTEMWIDE, OnFileSystemwide)
+    ON_COMMAND(ID_HELP_HELP, OnHelpHelp)
+    ON_COMMAND(ID_EDIT_IGNORELIST, OnEditIgnorelist)
+#endif
+    //}}AFX_MSG_MAP
+#ifndef DYNAMORIO_DEMO
+    // Standard file based document commands
+    ON_COMMAND(ID_FILE_NEW, CWinApp::OnFileNew)
+    ON_COMMAND(ID_FILE_OPEN, CWinApp::OnFileOpen)
+    // Standard print setup command
+    ON_COMMAND(ID_FILE_PRINT_SETUP, CWinApp::OnFilePrintSetup)
+#endif
+    END_MESSAGE_MAP()
+
+    /////////////////////////////////////////////////////////////////////////////
+    // CDynamoRIOApp construction
+
+    CDynamoRIOApp::CDynamoRIOApp()
+{
+    // Place all significant initialization in InitInstance
+}
+
+/////////////////////////////////////////////////////////////////////////////
+// The one and only CDynamoRIOApp object
+
+CDynamoRIOApp theApp;
+
+// FIXME: find better way to have other classes access this one
+
+/* static */ CDynamoRIOView * CDynamoRIOApp::GetActiveView()
+{
+    return (CDynamoRIOView *) theApp.m_pMainFrame->GetActiveView();
+}
+
+#ifndef DYNAMORIO_DEMO
+/* static */ BOOL CDynamoRIOApp::SystemwideSet()
+{
+    return theApp.m_bInjectAll;
+}
+
+/*static */TCHAR * CDynamoRIOApp::GetDllPath()
+{
+    return theApp.m_dll_path;
+}
+
+/*static */DLL_TYPE CDynamoRIOApp::GetDllType()
+{
+    return theApp.m_dll_type;
+}
+
+/*static */void CDynamoRIOApp::SetSystemwideSetting(int val)
+{
+    theApp.WriteProfileInt(_T("Settings"), _T("Confirm Systemwide"), val);
+}
+
+/*static */void CDynamoRIOApp::AboutToExit()
+{
+    theApp.PreExit();
+}
+#endif /* !DYNAMORIO_DEMO */
+
+/*static */void CDynamoRIOApp::SetStatusbarText(TCHAR *txt)
+{
+    theApp.m_pMainFrame->SetStatusBarText(0, txt);
+}
+
+/////////////////////////////////////////////////////////////////////////////
+// CDynamoRIOApp initialization
+
+BOOL CDynamoRIOApp::InitInstance()
+{
+#ifndef DYNAMORIO_DEMO
+    TCHAR msg[MAX_PATH*2];
+#endif
+    BOOL windows_NT;
+
+    // Standard initialization
+    // If you are not using these features and wish to reduce the size
+    //  of your final executable, you should remove from the following
+    //  the specific initialization routines you do not need.
+
+    if (!CheckWindowsVersion(windows_NT)) {
+        // abort
+        return FALSE;
+    }
+
+#if 0 // warning C4996: 'CWinApp::Enable3dControlsStatic': CWinApp::Enable3dControlsStatic is no longer needed. You should remove this call.
+#ifdef _AFXDLL
+    Enable3dControls();                     // Call this when using MFC in a shared DLL
+#else
+    Enable3dControlsStatic();       // Call this when linking to MFC statically
+#endif
+#endif
+
+#ifndef DYNAMORIO_DEMO
+    // Change the registry key under which our settings are stored,
+    // including MRU
+    SetRegistryKey(L_DYNAMORIO_REGISTRY_KEY);
+
+    LoadStdProfileSettings(12);  // Load standard INI file options (including MRU)
+#endif
+
+    // Register the application's document templates.  Document templates
+    //  serve as the connection between documents, frame windows and views.
+
+    CSingleDocTemplate* pDocTemplate;
+    pDocTemplate = new CSingleDocTemplate(
+#ifdef DYNAMORIO_DEMO
+                                          IDR_MAINFRAME_DEMO,
+#else
+                                          IDR_MAINFRAME,
+#endif
+                                          RUNTIME_CLASS(CDynamoRIODoc),
+                                          RUNTIME_CLASS(CMainFrame),       // main SDI frame window
+                                          RUNTIME_CLASS(CDynamoRIOView));
+    AddDocTemplate(pDocTemplate);
+
+    // Parse command line for standard shell commands, DDE, file open
+    CCommandLineInfo cmdInfo;
+    ParseCommandLine(cmdInfo);
+
+    // Dispatch commands specified on the command line
+    if (!ProcessShellCommand(cmdInfo))
+        return FALSE;
+
+    // enable file manager drag/drop and DDE Execute open
+    m_pMainWnd->DragAcceptFiles();
+
+    // The one and only window has been initialized, so show and update it.
+    m_pMainWnd->ShowWindow(SW_SHOW);
+    m_pMainWnd->UpdateWindow();
+
+    // I can't find any other way to access the main frame
+    m_pMainFrame = (CMainFrame *) m_pMainWnd;
+
+    // I can't figure out how to disable a menu item that has a command
+    // handler when this var is set, so I disable it:
+    m_pMainFrame->m_bAutoMenuEnable = FALSE;        
+
+#ifndef DYNAMORIO_DEMO
+    m_bSystemwideAllowed = TRUE;
+
+    // set the string we'll put into the registry key to inject system-wide
+    TCHAR data[1024];
+    int len = GetEnvironmentVariable(_T("DYNAMORIO_HOME"), m_dynamorio_home, _MAX_DIR);
+#if 1 //NOCHECKIN
+    if (len == 0)
+        m_dynamorio_home[0] = _T('\0');
+#else
+    if (len == 0) {
+        int res = MessageBox(NULL, 
+                             _T("DYNAMORIO_HOME environment variable not found.\n")
+                             _T("Set all the DynamoRIO environment variables to their default values?\n")
+                             _T("(Otherwise this GUI cannot operate and must exit.)"),
+                             _T("DynamoRIO Not Configured for Current User"), MB_YESNO | MYMBFLAGS);
+        if (res == IDYES) {
+            if (!ConfigureForNewUser())
+                return FALSE;
+        } else {
+            // abort
+            return FALSE;
+        }
+    }
+#endif
+
+    if (windows_NT) {
+        // we don't support systemwide on NT
+        // hack: use "confirm systemwide" setting to decide whether to notify user
+        if (GetProfileInt(_T("Settings"), _T("Confirm Systemwide"), 1) == 1) {
+            MessageBox(NULL, _T("Run All is not supported on Windows NT, it will be disabled"),
+                       _T("Notice"), MB_OK | MYMBFLAGS);
+            // but then how does user turn off, if can't do Run All?
+            // just turn it off now:
+            SetSystemwideSetting(0);
+        }
+        m_bSystemwideAllowed = FALSE; // so key won't be cleared
+        DisableSystemwideInject();
+    } else {
+        //NOCHECKIN assert(len > 0 && len < _MAX_DIR && len + _tcslen(L_INJECT_ALL_DLL_SUBPATH) < MAX_PATH);
+        _stprintf(data, _T("%s%s"), m_dynamorio_home, L_INJECT_ALL_DLL_SUBPATH);
+
+        // make sure it exists
+        CFile check;
+        if (!check.Open(data, CFile::modeRead|CFile::shareDenyNone)) {
+#if 0 // I'm disabling this dialog until we decide to support running apps
+            _stprintf(msg, _T("Library %s does not exist"), data);
+            MessageBox(NULL, msg, _T("Error"), MB_OK | MYMBFLAGS);
+#endif
+            m_bSystemwideAllowed = FALSE; // so key won't be cleared
+            DisableSystemwideInject();
+        } else {
+            if (_tcschr(data, _T(' ')) != NULL) {
+                // registry key cannot handle spaces in names!
+                // must get 8.3 alias -- and some volumes do not support such an alias!
+                len = GetShortPathName(data, m_inject_all_value, MAX_PATH);
+                if (len == 0) {
+                    _stprintf(msg, _T("Cannot find 8.3 alias for space-containing path \"%s\"!\nDisabling Run All"),
+                              data);
+                    MessageBox(NULL, msg, _T("Error"), MB_OK | MYMBFLAGS);
+                    m_bSystemwideAllowed = FALSE; // so key won't be cleared
+                    DisableSystemwideInject();
+                }
+            } else {
+                _tcscpy(m_inject_all_value, data);
+            }
+            check.Close();
+        }
+    }
+
+    // find current status of system-wide injection
+    if (m_bSystemwideAllowed) {
+        HKEY hk;
+        unsigned long size = 1024;
+        int     res = RegOpenKeyEx(INJECT_ALL_HIVE, INJECT_ALL_KEY_L, 0, KEY_READ, &hk);
+        assert(res == ERROR_SUCCESS);
+        res = RegQueryValueEx(hk, INJECT_ALL_SUBKEY_L, 0, NULL, (LPBYTE) data, &size);
+        assert(res == ERROR_SUCCESS);
+        RegCloseKey(hk);
+
+        // WARNING: do not use Unicode build!
+        // if ever get size==2, it's b/c of Unicode build!
+        // Plus, stats viewing doesn't work w/ Unicode build
+        if (size > sizeof(TCHAR)) {
+            // make sure we're the ones who set this value
+            if (_tcscmp(m_inject_all_value, data) != 0) {
+                // FIXME: have user notify us of conflict?
+                int res = MessageBox(NULL, 
+                                     _T("DynamoRIO's RunAll system-wide injection method is ")
+                                     _T("being used by some other program.\n")
+                                     _T("DynamoRIO can attempt to override the other program.\n")
+                                     _T("Otherwise, system-wide injection will be disabled.\n")
+                                     _T("Override?"),
+                                     _T("DynamoRIO Conflict"),
+                                     MB_YESNO | MYMBFLAGS);
+                if (res == IDYES) {
+                    // now clear the registry key
+                    SetSystemwideInject(_T(""));
+                    // if the call fails, callee calls Disable for us
+                } else {
+                    m_bSystemwideAllowed = FALSE; // so key won't be cleared
+                    DisableSystemwideInject();
+                }
+            } else {
+                m_bInjectAll = TRUE;
+            }
+        } else {
+            // empty value: no injection
+            m_bInjectAll = FALSE;
+        }
+    }
+    if (m_bSystemwideAllowed) {
+        m_pMainWnd->GetMenu()->CheckMenuItem(ID_FILE_SYSTEMWIDE, 
+                                             MF_BYCOMMAND | ((m_bInjectAll) ? MF_CHECKED : MF_UNCHECKED));
+    }
+
+    // make sure preinject dll exists
+    if (m_bSystemwideAllowed) {
+        CFile check;
+        if (!check.Open(m_inject_all_value, CFile::modeRead|CFile::shareDenyNone)) {
+#if 0 // I'm disabling this dialog until we decide to support running apps
+            _stprintf(msg, _T("Library %s does not exist!\nDisabling Run All"), m_inject_all_value);
+            MessageBox(NULL, msg, _T("DynamoRIO Configuration Error"), MB_OK | MYMBFLAGS);
+#endif
+            DisableSystemwideInject();
+        } else
+            check.Close();
+    }
+
+    m_dll_path[0] = _T('\0');
+
+    DisableMissingLibraries(TRUE);
+
+    // now select the previously checked library
+    int lib = GetProfileInt(_T("Settings"), _T("Library"), 1);
+    int tried = 0;
+    while (tried < 3) {
+        if (lib == 0 && SwitchLibraries(L_DLLPATH_RELEASE, FALSE)) {
+            OnLibraryRelease();
+            break;
+        } else if (lib == 1 && SwitchLibraries(L_DLLPATH_DEBUG, FALSE)) {
+            OnLibraryDebug();
+            break;
+        } else if (lib == 2 && SwitchLibraries(L_DLLPATH_PROFILE, FALSE)) {
+            OnLibraryProfile();
+            break;
+        } 
+        lib = (lib + 1) % 3;
+        tried++;
+    }
+    if (tried == 3) {
+#if 0 // Disabling
+        MessageBox(NULL, _T("Cannot find any DynamoRIO libraries!\n")
+                   _T("Running of applications will be disabled."), _T("Error"), MB_OK | MYMBFLAGS);
+#endif
+        // disable Run and libraries
+        m_pMainWnd->GetMenu()->EnableMenuItem(ID_FILE_RUN, MF_BYCOMMAND|MF_GRAYED);
+        DisableSystemwideInject();
+        m_pMainWnd->GetMenu()->EnableMenuItem(ID_LIBRARY_RELEASE, MF_BYCOMMAND|MF_GRAYED);
+        m_pMainWnd->GetMenu()->EnableMenuItem(ID_LIBRARY_DEBUG, MF_BYCOMMAND|MF_GRAYED);
+        m_pMainWnd->GetMenu()->EnableMenuItem(ID_LIBRARY_PROFILE, MF_BYCOMMAND|MF_GRAYED);
+        m_pMainWnd->GetMenu()->CheckMenuItem(ID_LIBRARY_RELEASE, MF_BYCOMMAND|MF_UNCHECKED);
+        m_pMainWnd->GetMenu()->CheckMenuItem(ID_LIBRARY_DEBUG,   MF_BYCOMMAND|MF_UNCHECKED);
+        m_pMainWnd->GetMenu()->CheckMenuItem(ID_LIBRARY_PROFILE, MF_BYCOMMAND|MF_UNCHECKED);
+    }
+
+    if (m_bInjectAll) {
+        MessageBox(NULL, _T("Run All is already set!"),
+                   _T("Warning"), MB_OK | MYMBFLAGS);
+
+        // FIXME: share this code with OnFileSystemwide
+        SetEnvVarPermanently(_T("DYNAMORIO_SYSTEMWIDE"), m_dll_path);
+
+        // disable changing the library
+        m_pMainWnd->GetMenu()->EnableMenuItem(ID_LIBRARY_RELEASE, MF_BYCOMMAND|MF_GRAYED);
+        m_pMainWnd->GetMenu()->EnableMenuItem(ID_LIBRARY_DEBUG, MF_BYCOMMAND|MF_GRAYED);
+        m_pMainWnd->GetMenu()->EnableMenuItem(ID_LIBRARY_PROFILE, MF_BYCOMMAND|MF_GRAYED);
+    }
+#endif /*! DYNAMORIO_DEMO */
+
+    return TRUE;
+}
+
+BOOL CDynamoRIOApp::CheckWindowsVersion(BOOL &windows_NT)
+{
+    // make sure we're on an NT-based system
+    windows_NT = FALSE;
+    TCHAR bad_os[64];
+    OSVERSIONINFO version;
+    version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
+    int res = GetVersionEx(&version);
+    assert(res != 0);
+    if (version.dwPlatformId == VER_PLATFORM_WIN32_NT) {
+        // WinNT or descendents: rather than continually update
+        // the list of known versions here we assume they're all ok.
+        if (version.dwMajorVersion == 4) {
+            // Windows NT
+            windows_NT = TRUE;
+        } else {
+            // 2K or later
+        }
+        return TRUE;
+    } else if (version.dwPlatformId == VER_PLATFORM_WIN32_WINDOWS) {
+        /* Win95 or Win98 */
+        unsigned ver_high = (version.dwBuildNumber >> 24) & 0xff;
+        unsigned ver_low = (version.dwBuildNumber >> 16) & 0xff;
+        if (ver_low >= 90 || ver_high >= 5)
+            _stprintf(bad_os, _T("Windows ME"));
+        else if (ver_low >= 10 && ver_low < 90)
+            _stprintf(bad_os, _T("Windows 98"));
+        else if (ver_low < 5)
+            _stprintf(bad_os, _T("Windows 31 / WfWg"));
+        else if (ver_low < 10)
+            _stprintf(bad_os, _T("Windows 98"));
+        else
+            assert(false);
+    } else {
+        /* Win32S on Windows 3.1 */
+        _stprintf(bad_os, _T("Win32s"));
+    }
+    TCHAR msg[128];
+    _stprintf(msg, _T("DynamoRIO does not support %s"), bad_os);
+    MessageBox(m_pMainFrame->m_hWnd, msg, _T("Fatal Error"), MB_OK | MYMBFLAGS);
+    return FALSE;
+}
+
+/////////////////////////////////////////////////////////////////////////////
+// CAboutDlg dialog used for App About
+
+class CAboutDlg : public CDialog
+{
+public:
+    CAboutDlg();
+
+    // Dialog Data
+    //{{AFX_DATA(CAboutDlg)
+    enum { IDD = IDD_ABOUTBOX };
+    //}}AFX_DATA
+
+    // ClassWizard generated virtual function overrides
+    //{{AFX_VIRTUAL(CAboutDlg)
+protected:
+    virtual void DoDataExchange(CDataExchange* pDX);    // DDX/DDV support
+    //}}AFX_VIRTUAL
+
+    // Implementation
+protected:
+    //{{AFX_MSG(CAboutDlg)
+    //}}AFX_MSG
+    DECLARE_MESSAGE_MAP()
+        };
+
+CAboutDlg::CAboutDlg() : CDialog(CAboutDlg::IDD)
+{
+    //{{AFX_DATA_INIT(CAboutDlg)
+    //}}AFX_DATA_INIT
+}
+
+void CAboutDlg::DoDataExchange(CDataExchange* pDX)
+{
+    CDialog::DoDataExchange(pDX);
+    //{{AFX_DATA_MAP(CAboutDlg)
+    //}}AFX_DATA_MAP
+}
+
+BEGIN_MESSAGE_MAP(CAboutDlg, CDialog)
+    //{{AFX_MSG_MAP(CAboutDlg)
+    //}}AFX_MSG_MAP
+    END_MESSAGE_MAP()
+
+    // App command to run the dialog
+    void CDynamoRIOApp::OnAppAbout()
+{
+    CAboutDlg aboutDlg;
+    aboutDlg.DoModal();
+}
+
+/////////////////////////////////////////////////////////////////////////////
+// CDynamoRIOApp message handlers
+
+
+CDocument * CDynamoRIOApp::OpenDocumentFile(LPCTSTR
+#ifndef DYNAMORIO_DEMO /* avoid warning */
+                                            lpszFileName
+#endif
+                                            )
+{
+#ifndef DYNAMORIO_DEMO
+    RunNewApp(lpszFileName);
+#endif
+    return m_pMainFrame->GetActiveView()->GetDocument();
+}
+
+#ifndef DYNAMORIO_DEMO
+
+static TCHAR szFilter[] = _T("Executable Files (*.exe)|*.exe|All Files (*.*)|*.*||");
+
+void CDynamoRIOApp::OnFileRun() 
+{
+    CFileDialog fileDlg(TRUE, _T(".exe"), NULL, 
+                        // hide the "open as read-only" checkbox
+                        OFN_FILEMUSTEXIST | OFN_PATHMUSTEXIST | OFN_HIDEREADONLY,
+                        szFilter);
+    int res = fileDlg.DoModal();
+    if (res == IDCANCEL)
+        return;
+    CString file = fileDlg.GetPathName();
+    RunNewApp(file);
+}
+
+BOOL CDynamoRIOApp::RunNewApp(LPCTSTR lpszFileName)
+{
+    m_pMainFrame->SetStatusBarText(0, lpszFileName);
+    AddToRecentFileList(lpszFileName);
+    CDynamoRIODoc *doc = (CDynamoRIODoc *) m_pMainFrame->GetActiveView()->GetDocument();
+    doc->RunApplication(lpszFileName);
+    return TRUE;
+}
+
+void CDynamoRIOApp::OnFileSystemwide() 
+{
+    assert(m_bSystemwideAllowed);
+
+    if (!m_bInjectAll && GetProfileInt(_T("Settings"), _T("Confirm Systemwide"), 1) == 1) {
+        // confirm with user the gravity of this setting
+        CSyswideDlg dlg;
+        int res = dlg.DoModal();
+        if (res == IDCANCEL)
+            return;
+    }
+
+    TCHAR * val;
+    if (m_bInjectAll) {
+        m_bInjectAll = FALSE;
+        val = _T("");
+
+        // enable changing existing libraries
+        DisableMissingLibraries(FALSE);
+    } else {
+        m_bInjectAll = TRUE;
+        val = m_inject_all_value;
+
+        // point DYNAMORIO_SYSTEMWIDE to the currently selected library
+        SetEnvVarPermanently(_T("DYNAMORIO_SYSTEMWIDE"), m_dll_path);
+
+        // disable changing the library
+        m_pMainWnd->GetMenu()->EnableMenuItem(ID_LIBRARY_RELEASE, MF_BYCOMMAND|MF_GRAYED);
+        m_pMainWnd->GetMenu()->EnableMenuItem(ID_LIBRARY_DEBUG, MF_BYCOMMAND|MF_GRAYED);
+        m_pMainWnd->GetMenu()->EnableMenuItem(ID_LIBRARY_PROFILE, MF_BYCOMMAND|MF_GRAYED);
+    }
+    if (!SetSystemwideInject(val))
+        return;
+    m_pMainWnd->GetMenu()->CheckMenuItem(ID_FILE_SYSTEMWIDE, 
+                                         MF_BYCOMMAND | ((m_bInjectAll) ? MF_CHECKED : MF_UNCHECKED));
+}
+
+BOOL CDynamoRIOApp::SetSystemwideInject(TCHAR *val)
+{
+    assert(m_bSystemwideAllowed);
+    HKEY hk;
+    // must have administrative privilegs to write this key!
+    int     res = RegOpenKeyEx(INJECT_ALL_HIVE, INJECT_ALL_KEY_L, 0, KEY_WRITE, &hk);
+    if (res != ERROR_SUCCESS) {
+        MessageBox(NULL, 
+                   _T("DynamoRIO's system-wide injection method requires administrative privileges.\n")
+                   _T("You must restart this program with such privileges to use this feature."),
+                   _T("Lack of Privileges"), MB_OK | MYMBFLAGS);
+        // now disable systemwide injection
+        m_bSystemwideAllowed = FALSE; // prevent infinite loop
+        DisableSystemwideInject();
+        return FALSE;
+    }
+    res = RegSetValueEx(hk, INJECT_ALL_SUBKEY_L, 0, REG_SZ, (LPBYTE) val, _tcslen(val)+1);
+    assert(res == ERROR_SUCCESS);
+    RegCloseKey(hk);
+    return TRUE;
+}
+
+void CDynamoRIOApp::DisableSystemwideInject()
+{
+    if (m_bSystemwideAllowed) {
+        // clear key
+        SetSystemwideInject(_T(""));
+    }
+    // disable systemwide altogether
+    m_bSystemwideAllowed = FALSE;
+    m_bInjectAll = FALSE;
+    // disable checkbox, but allow editing ignore list
+    m_pMainWnd->GetMenu()->EnableMenuItem(ID_FILE_SYSTEMWIDE, 
+                                          MF_BYCOMMAND|MF_GRAYED);
+    m_pMainFrame->SetStatusBarText(0, _T("Disabled system-wide injection"));
+}
+
+void CDynamoRIOApp::OnEditOptions() 
+{
+    COptionsDlg ops;
+    ops.DoModal();
+}
+
+
+void CDynamoRIOApp::OnEditIgnorelist() 
+{
+    CIgnoreDlg dlg;
+    dlg.DoModal();
+}
+
+void CDynamoRIOApp::DisableMissingLibraries(BOOL notify)
+{
+    BOOL release_ok, debug_ok, profile_ok;
+    if (SwitchLibraries(L_DLLPATH_RELEASE, notify)) {
+        release_ok = TRUE;
+        m_pMainWnd->GetMenu()->EnableMenuItem(ID_LIBRARY_RELEASE, MF_BYCOMMAND);
+    } else {
+        release_ok = FALSE;
+        m_pMainWnd->GetMenu()->EnableMenuItem(ID_LIBRARY_RELEASE, MF_BYCOMMAND|MF_GRAYED);
+    }
+    if (SwitchLibraries(L_DLLPATH_DEBUG, notify)) {
+        debug_ok = TRUE;
+        m_pMainWnd->GetMenu()->EnableMenuItem(ID_LIBRARY_DEBUG, MF_BYCOMMAND);
+    } else {
+        debug_ok = FALSE;
+        m_pMainWnd->GetMenu()->EnableMenuItem(ID_LIBRARY_DEBUG, MF_BYCOMMAND|MF_GRAYED);
+    }
+    if (SwitchLibraries(L_DLLPATH_PROFILE, notify)) {
+        profile_ok = TRUE;
+        m_pMainWnd->GetMenu()->EnableMenuItem(ID_LIBRARY_PROFILE, MF_BYCOMMAND);
+    } else {
+        profile_ok = FALSE;
+        m_pMainWnd->GetMenu()->EnableMenuItem(ID_LIBRARY_PROFILE, MF_BYCOMMAND|MF_GRAYED);
+    }
+}
+
+BOOL CDynamoRIOApp::SwitchLibraries(TCHAR *newdllpath, BOOL notify)
+{
+    // first see if new dll exists
+    CFile check;
+    TCHAR msg[MAX_PATH*2];
+    TCHAR file[MAX_PATH];
+    assert(_tcslen(m_dynamorio_home) + _tcslen(newdllpath) < MAX_PATH);
+    _stprintf(file, _T("%s%s"), m_dynamorio_home, newdllpath);
+    if (!check.Open(file, CFile::modeRead|CFile::shareDenyNone)) {
+        if (notify) {
+            _stprintf(msg, _T("Library %s does not exist"), file);
+#if 0 // I'm disabling this dialog until we decide to support running apps
+            MessageBox(NULL, msg, _T("DynamoRIO Configuration Error"), MB_OK | MYMBFLAGS);
+#endif
+        }
+        return FALSE;
+    }
+    check.Close();
+
+    // set dll pointer for single-app injection
+    _tcscpy(m_dll_path, file);
+
+    return TRUE;
+}
+
+void CDynamoRIOApp::OnLibraryRelease() 
+{
+    if (!SwitchLibraries(L_DLLPATH_RELEASE, TRUE))
+        return;
+
+    m_dll_type = DLL_RELEASE;
+    COptionsDlg::CheckOptionsVersusDllType(m_dll_type);
+    m_pMainWnd->GetMenu()->CheckMenuItem(ID_LIBRARY_RELEASE, MF_BYCOMMAND | MF_CHECKED);
+    m_pMainWnd->GetMenu()->CheckMenuItem(ID_LIBRARY_DEBUG,   MF_BYCOMMAND | MF_UNCHECKED);
+    m_pMainWnd->GetMenu()->CheckMenuItem(ID_LIBRARY_PROFILE, MF_BYCOMMAND | MF_UNCHECKED);
+}
+
+void CDynamoRIOApp::OnLibraryDebug() 
+{
+    if (!SwitchLibraries(L_DLLPATH_DEBUG, TRUE))
+        return;
+
+    m_dll_type = DLL_DEBUG;
+    COptionsDlg::CheckOptionsVersusDllType(m_dll_type);
+    m_pMainWnd->GetMenu()->CheckMenuItem(ID_LIBRARY_RELEASE, MF_BYCOMMAND | MF_UNCHECKED);
+    m_pMainWnd->GetMenu()->CheckMenuItem(ID_LIBRARY_DEBUG,   MF_BYCOMMAND | MF_CHECKED);
+    m_pMainWnd->GetMenu()->CheckMenuItem(ID_LIBRARY_PROFILE, MF_BYCOMMAND | MF_UNCHECKED);
+}
+
+void CDynamoRIOApp::OnLibraryProfile() 
+{
+    if (!SwitchLibraries(L_DLLPATH_PROFILE, TRUE))
+        return;
+
+    m_dll_type = DLL_PROFILE;
+    COptionsDlg::CheckOptionsVersusDllType(m_dll_type);
+    m_pMainWnd->GetMenu()->CheckMenuItem(ID_LIBRARY_RELEASE, MF_BYCOMMAND | MF_UNCHECKED);
+    m_pMainWnd->GetMenu()->CheckMenuItem(ID_LIBRARY_DEBUG,   MF_BYCOMMAND | MF_UNCHECKED);
+    m_pMainWnd->GetMenu()->CheckMenuItem(ID_LIBRARY_PROFILE, MF_BYCOMMAND | MF_CHECKED);
+}
+
+
+void CDynamoRIOApp::OnHelpHelp() 
+{
+    TCHAR helppath[MAX_PATH];
+    _stprintf(helppath, _T("%s%s"), m_dynamorio_home, HELP_PATH);
+
+    TCHAR cwd[MAX_PATH];
+    int cwd_res = GetCurrentDirectory(MAX_PATH, cwd);
+    assert(cwd_res > 0);
+
+    // launch browser on documentation file!
+    HINSTANCE res = ShellExecute(m_pMainWnd->m_hWnd, _T("open"), 
+                                 helppath, NULL, cwd, SW_SHOWNORMAL);
+    // FIXME: I get back 2 == SE_ERR_FNF == file not found, but netscape finds and
+    // displays it fine...
+    if ((int)res <= 32) {
+        TCHAR msg[MAX_PATH*2];
+        _stprintf(msg, _T("Error browsing help document %s"), helppath);
+        MessageBox(m_pMainWnd->m_hWnd, msg,     _T("Error"), MB_OK | MYMBFLAGS);
+    }
+}
+
+// MessageBox crashes in ExitInstance so I added this PreExit routine, called
+// using the framework's notion of "save unsaved data?" for CDocument
+void CDynamoRIOApp::PreExit()
+{
+    BOOL ok = TRUE; // libraries may have been disabled
+    if (_tcsstr(m_dll_path, L_DLLPATH_RELEASE) != NULL)
+        ok = WriteProfileInt(_T("Settings"), _T("Library"), 0);
+    else if (_tcsstr(m_dll_path, L_DLLPATH_DEBUG) != NULL)
+        ok = WriteProfileInt(_T("Settings"), _T("Library"), 1);
+    else if (_tcsstr(m_dll_path, L_DLLPATH_PROFILE) != NULL)
+        ok = WriteProfileInt(_T("Settings"), _T("Library"), 2);
+    assert(ok);
+
+        
+    if (m_bSystemwideAllowed && m_bInjectAll) {
+        // FIXME: MessageBox crashes...try to get this code into
+        // MFC's auto-framework for saving unsaved documents!
+        int res = MessageBox(NULL, // m_pMainWnd is NULL now!
+                             _T("Run All is currently set.  Turn it off?"),
+                             _T("Confirmation"), MB_YESNO | MYMBFLAGS);
+        if (res == IDYES) {
+            // clear key
+            SetSystemwideInject(_T(""));
+        }
+    }
+}
+
+void CDynamoRIOApp::SetEnvVarPermanently(TCHAR *var, TCHAR *val)
+{
+    // it takes a while to broadcast the "we've changed env vars" message,
+    // so set a wait cursor
+    HCURSOR prev_cursor = SetCursor(LoadCursor(IDC_WAIT));
+
+    TCHAR msg[MAX_PATH];
+    HKEY hk;
+    int res;
+        
+    // current user only
+    res = RegOpenKeyEx(HKEY_CURRENT_USER, _T("Environment"), 0, KEY_WRITE, &hk);
+    if (res != ERROR_SUCCESS) {
+        _stprintf(msg, _T("Error writing to HKEY_CURRENT_USER\\Environment"));
+        MessageBox(NULL, msg, _T("Error"), MB_OK | MYMBFLAGS);
+    }
+
+    res = RegSetValueEx(hk, var, 0, REG_SZ, (LPBYTE) val, _tcslen(val)+1);
+    assert(res == ERROR_SUCCESS);
+
+    RegCloseKey(hk);
+
+    // tell system that we've changed environment (else won't take effect until
+    // user logs out and back in)
+    DWORD dwReturnValue;
+    // Code I copied this from used an ANSI string...I'm leaving
+    // it like that FIXME
+    SendMessageTimeout(HWND_BROADCAST, WM_SETTINGCHANGE, 0,
+                       (LPARAM) "Environment", SMTO_ABORTIFHUNG, 5000, &dwReturnValue);
+
+    // set local var too
+    BOOL ok = SetEnvironmentVariable(var, val);
+    assert(ok);
+
+    SetCursor(prev_cursor);
+}
+
+BOOL CDynamoRIOApp::ConfigureForNewUser()
+{
+    // Need to find location of dynamorio install
+    BOOL ok = FALSE;
+    TCHAR msg[MAX_PATH*2];
+    int res;
+    // First attempt: look at gui's executable path
+    GetModuleFileName(NULL, m_dynamorio_home, MAX_PATH);
+    TCHAR *sub = _tcsstr(m_dynamorio_home, _T("\\bin\\DynamoRIO.exe"));
+    if (sub != NULL) {
+        *sub = _T('\0');
+        _stprintf(msg, _T("Is this the location of the DynamoRIO installation?\n%s\n"),
+                  m_dynamorio_home);
+        res = MessageBox(NULL, msg, _T("Confirmation"), MB_YESNO | MYMBFLAGS);
+        if (res == IDNO)
+            ok = FALSE;
+        else
+            ok = TRUE;
+    }
+    if (!ok) {
+        // Second attempt: read Start Menu's link files
+        // FIXME: code that up
+
+        // Last resort: ask user to locate directory
+        BROWSEINFO bi;
+        bi.hwndOwner = m_pMainFrame->m_hWnd;
+        bi.pidlRoot = NULL;
+        bi.pszDisplayName = m_dynamorio_home;
+        bi.lpszTitle = _T("Locate Root of DynamoRIO Installation");
+        bi.ulFlags = 0;
+        bi.lpfn = NULL;
+        bi.lParam = NULL;
+        bi.iImage = 0;
+        ITEMIDLIST *id = SHBrowseForFolder(&bi);
+        if (id == NULL) // cancelled
+            return FALSE;
+        SHGetPathFromIDList(id, m_dynamorio_home);
+    }
+
+    // it takes a while to broadcast the "we've changed env vars" message,
+    // so set a wait cursor
+    HCURSOR prev_cursor = SetCursor(LoadCursor(IDC_WAIT));
+
+    HKEY hk;
+        
+    // current user only
+    res = RegOpenKeyEx(HKEY_CURRENT_USER, _T("Environment"), 0, KEY_WRITE, &hk);
+    if (res != ERROR_SUCCESS) {
+        _stprintf(msg, _T("Error writing to HKEY_CURRENT_USER\\Environment"));
+        MessageBox(NULL, msg, _T("Error"), MB_OK | MYMBFLAGS);
+    }
+
+    // DYNAMORIO_HOME
+    TCHAR *val = m_dynamorio_home;
+    res = RegSetValueEx(hk, _T("DYNAMORIO_HOME"), 0, REG_SZ, (LPBYTE) val, _tcslen(val)+1);
+    assert(res == ERROR_SUCCESS);
+    // set local var too
+    ok = SetEnvironmentVariable(_T("DYNAMORIO_HOME"), val);
+    assert(ok);
+
+    // DYNAMORIO_OPTIONS
+    val = INITIAL_OPTIONS;
+    res = RegSetValueEx(hk, _T("DYNAMORIO_OPTIONS"), 0, REG_SZ, (LPBYTE) val, _tcslen(val)+1);
+    assert(res == ERROR_SUCCESS);
+    // set local var too
+    ok = SetEnvironmentVariable(_T("DYNAMORIO_OPTIONS"), val);
+    assert(ok);
+
+    // DYNAMORIO_SYSTEMWIDE
+    TCHAR buf[MAX_PATH];
+    _stprintf(buf, _T("%s%s"), m_dynamorio_home, INITIAL_SYSTEMWIDE);
+    val = buf;
+    res = RegSetValueEx(hk, _T("DYNAMORIO_SYSTEMWIDE"), 0, REG_SZ, (LPBYTE) val, _tcslen(val)+1);
+    assert(res == ERROR_SUCCESS);
+    // set local var too
+    ok = SetEnvironmentVariable(_T("DYNAMORIO_SYSTEMWIDE"), val);
+    assert(ok);
+
+    // DYNAMORIO_IGNORE
+    val = INITIAL_IGNORE;
+    res = RegSetValueEx(hk, _T("DYNAMORIO_IGNORE"), 0, REG_SZ, (LPBYTE) val, _tcslen(val)+1);
+    assert(res == ERROR_SUCCESS);
+    // set local var too
+    ok = SetEnvironmentVariable(_T("DYNAMORIO_IGNORE"), val);
+    assert(ok);
+
+    RegCloseKey(hk);
+
+    // tell system that we've changed environment (else won't take effect until
+    // user logs out and back in)
+    DWORD dwReturnValue;
+    // Code I copied this from used an ANSI string...I'm leaving
+    // it like that FIXME
+    SendMessageTimeout(HWND_BROADCAST, WM_SETTINGCHANGE, 0,
+                       (LPARAM) "Environment", SMTO_ABORTIFHUNG, 5000, &dwReturnValue);
+
+    // FIXME: Doc keeps its own internal var w/ home, must update here,
+    // better to have Doc grab ours!
+    CDynamoRIODoc *doc = (CDynamoRIODoc *) m_pMainFrame->GetActiveView()->GetDocument();
+    doc->InitPaths();
+
+    SetCursor(prev_cursor);
+
+    return TRUE;
+}
+
+#endif /* !DYNAMORIO_DEMO */
+
+void CDynamoRIOApp::OnAppExit() 
+{
+    // Same as double-clicking on main window close box.
+    ASSERT(AfxGetMainWnd() != NULL);
+    AfxGetMainWnd()->SendMessage(WM_CLOSE);
+}
+
+int CDynamoRIOApp::ExitInstance() 
+{
+    // no longer need to do anything special -- PreExit does it now
+    return CWinApp::ExitInstance();
+}
+
diff --git a/tools/DRgui/DynamoRIO.h b/tools/DRgui/DynamoRIO.h
new file mode 100644
index 000000000..b5d68fa96
--- /dev/null
+++ b/tools/DRgui/DynamoRIO.h
@@ -0,0 +1,147 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// DynamoRIO.h : main header file for the DYNAMORIO application
+//
+
+#if !defined(AFX_DYNAMORIO_H__23EBE600_BBF9_4F81_B49E_8CA6E806E7F8__INCLUDED_)
+#define AFX_DYNAMORIO_H__23EBE600_BBF9_4F81_B49E_8CA6E806E7F8__INCLUDED_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif // _MSC_VER > 1000
+
+#ifndef __AFXWIN_H__
+#error include 'stdafx.h' before including this file for PCH
+#endif
+
+#include "resource.h"       // main symbols
+#include "MainFrm.h"    // Added by ClassView
+#include "DynamoRIODoc.h"
+#include "DynamoRIOView.h"
+
+#define MYMBFLAGS   MB_TOPMOST | MB_ICONEXCLAMATION
+
+#ifdef UNICODE
+# define ASCII_PRINTF _T("S")
+# define UNICODE_PRINTF _T("s")
+#else
+# define ASCII_PRINTF _T("s")
+# define UNICODE_PRINTF _T("S")
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+// CDynamoRIOApp:
+// See DynamoRIO.cpp for the implementation of this class
+//
+
+class CDynamoRIOApp : public CWinApp
+{
+public:
+#ifndef DYNAMORIO_DEMO
+    BOOL ConfigureForNewUser();
+    void SetEnvVarPermanently(TCHAR *var, TCHAR *val);
+    void PreExit();
+    BOOL RunNewApp(LPCTSTR lpszFileName);
+#endif
+    CDocument * OpenDocumentFile(LPCTSTR lpszFileName);
+    CDynamoRIOApp();
+
+    // global function for TimerProc inside view
+    static CDynamoRIOView * GetActiveView();
+#ifndef DYNAMORIO_DEMO
+    // global function for systemwide status
+    static BOOL SystemwideSet();
+    // global function for current library to use
+    static TCHAR * GetDllPath();
+    // global function for current library type
+    static DLL_TYPE GetDllType();
+    // global function for registry setting
+    static void SetSystemwideSetting(int val);
+    // global function for clean up at exit (InstanceExit is too late!)
+    static void AboutToExit();
+#endif
+    // global function for setting status bar text
+    static void SetStatusbarText(TCHAR *txt);
+
+    // Overrides
+    // ClassWizard generated virtual function overrides
+    //{{AFX_VIRTUAL(CDynamoRIOApp)
+public:
+    virtual BOOL InitInstance();
+    virtual int ExitInstance();
+    //}}AFX_VIRTUAL
+
+    // Implementation
+    //{{AFX_MSG(CDynamoRIOApp)
+    afx_msg void OnAppAbout();
+    afx_msg void OnAppExit();
+#ifndef DYNAMORIO_DEMO
+    afx_msg void OnEditOptions();
+    afx_msg void OnLibraryRelease();
+    afx_msg void OnLibraryProfile();
+    afx_msg void OnLibraryDebug();
+    afx_msg void OnFileRun();
+    afx_msg void OnFileSystemwide();
+    afx_msg void OnHelpHelp();
+    afx_msg void OnEditIgnorelist();
+#endif
+    //}}AFX_MSG
+    DECLARE_MESSAGE_MAP()
+
+protected:
+    BOOL CheckWindowsVersion(BOOL &windows_NT);
+    CMainFrame *m_pMainFrame;
+#ifndef DYNAMORIO_DEMO
+    void DisableMissingLibraries(BOOL notify);
+    void DisableSystemwideInject();
+    BOOL m_bSystemwideAllowed;
+    BOOL SetSystemwideInject(TCHAR *val);
+    BOOL m_bInjectAll;
+    BOOL SwitchLibraries(TCHAR *newdllpath, BOOL notify);
+    TCHAR m_dynamorio_home[_MAX_DIR];
+    TCHAR m_inject_all_value[MAX_PATH];
+    TCHAR m_dll_path[MAX_PATH];
+    DLL_TYPE m_dll_type;
+#endif
+};
+
+/////////////////////////////////////////////////////////////////////////////
+
+//{{AFX_INSERT_LOCATION}}
+// Microsoft Visual C++ will insert additional declarations immediately before the previous line.
+
+#endif // !defined(AFX_DYNAMORIO_H__23EBE600_BBF9_4F81_B49E_8CA6E806E7F8__INCLUDED_)
diff --git a/tools/DRgui/DynamoRIO.rc b/tools/DRgui/DynamoRIO.rc
new file mode 100644
index 000000000..c3c0be00a
--- /dev/null
+++ b/tools/DRgui/DynamoRIO.rc
@@ -0,0 +1,736 @@
+// **********************************************************
+// Copyright (c) 2007-2009 VMware, Inc.  All rights reserved.
+// **********************************************************
+
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// 
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+// 
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the following disclaimer in the documentation
+//   and/or other materials provided with the distribution.
+// 
+// * Neither the name of VMware, Inc. nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+// DAMAGE.
+
+//Microsoft Developer Studio generated resource script.
+//
+#include "resource.h"
+
+#define APSTUDIO_READONLY_SYMBOLS
+/////////////////////////////////////////////////////////////////////////////
+//
+// Generated from the TEXTINCLUDE 2 resource.
+//
+#include "afxres.h"
+
+/////////////////////////////////////////////////////////////////////////////
+#undef APSTUDIO_READONLY_SYMBOLS
+
+/////////////////////////////////////////////////////////////////////////////
+// English (U.S.) resources
+
+#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
+#ifdef _WIN32
+LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
+#pragma code_page(1252)
+#endif //_WIN32
+
+#ifdef APSTUDIO_INVOKED
+/////////////////////////////////////////////////////////////////////////////
+//
+// TEXTINCLUDE
+//
+
+1 TEXTINCLUDE MOVEABLE PURE 
+BEGIN
+    "resource.h\0"
+END
+
+2 TEXTINCLUDE MOVEABLE PURE 
+BEGIN
+    "#include ""afxres.h""\r\n"
+    "\0"
+END
+
+3 TEXTINCLUDE MOVEABLE PURE 
+BEGIN
+    "#define _AFX_NO_SPLITTER_RESOURCES\r\n"
+    "#define _AFX_NO_OLE_RESOURCES\r\n"
+    "#define _AFX_NO_TRACKER_RESOURCES\r\n"
+    "#define _AFX_NO_PROPERTY_RESOURCES\r\n"
+    "\r\n"
+    "#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)\r\n"
+    "#ifdef _WIN32\r\n"
+    "LANGUAGE 9, 1\r\n"
+    "#pragma code_page(1252)\r\n"
+    "#endif //_WIN32\r\n"
+    "#include ""res\\DynamoRIO.rc2""  // non-Microsoft Visual C++ edited resources\r\n"
+    "#include ""afxres.rc""         // Standard components\r\n"
+    "#include ""afxprint.rc""       // printing/print preview resources\r\n"
+    "#endif\r\n"
+    "\0"
+END
+
+#endif    // APSTUDIO_INVOKED
+
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Icon
+//
+
+// Icon with lowest ID value placed first to ensure application icon
+// remains consistent on all systems.
+IDR_ABOUT_ICON          ICON    DISCARDABLE     "res\\DynamoRIO.ico"
+IDR_MAINFRAME           ICON    DISCARDABLE     "res\\DynamoRIO.ico"
+IDR_MAINFRAME_DEMO      ICON    DISCARDABLE     "res\\DynamoRIO.ico"
+IDR_DYNAMOTYPE          ICON    DISCARDABLE     "res\\DynamoRIODoc.ico"
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Bitmap
+//
+
+IDR_MAINFRAME           BITMAP  MOVEABLE PURE   "res\\Toolbar.bmp"
+IDR_MAINFRAME_DEMO      BITMAP  MOVEABLE PURE   "res\\Toolbar.bmp"
+IDB_LOGO_VMWARE         BITMAP  MOVEABLE PURE   "res\\vmware-logo.bmp"
+IDB_LOGO_DYNAMORIO      BITMAP  MOVEABLE PURE   "res\\drlogo.bmp"
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Toolbar
+//
+
+IDR_MAINFRAME TOOLBAR MOVEABLE PURE  16, 15
+BEGIN
+    BUTTON      ID_FILE_NEW
+    BUTTON      ID_FILE_OPEN
+    BUTTON      ID_FILE_SAVE
+    SEPARATOR
+    BUTTON      ID_EDIT_CUT
+    BUTTON      ID_EDIT_COPY
+    BUTTON      ID_EDIT_PASTE
+    SEPARATOR
+    BUTTON      ID_FILE_PRINT
+    SEPARATOR
+    BUTTON      ID_APP_ABOUT
+END
+
+IDR_MAINFRAME_DEMO TOOLBAR MOVEABLE PURE  16, 15
+BEGIN
+    BUTTON      ID_APP_ABOUT
+END
+
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Menu
+//
+
+IDR_MAINFRAME MENU DISCARDABLE 
+BEGIN
+    POPUP "&Run"
+    BEGIN
+        MENUITEM "&Run...\tCtrl+R",             ID_FILE_RUN
+        MENUITEM "Run &All",                    ID_FILE_SYSTEMWIDE
+        MENUITEM SEPARATOR
+        MENUITEM "Recent File",                 ID_FILE_MRU_FILE1, GRAYED
+        MENUITEM SEPARATOR
+        MENUITEM "E&xit",                       ID_APP_EXIT
+    END
+    POPUP "&Edit"
+    BEGIN
+        MENUITEM "&Copy statistics\tCtrl+C",    ID_EDIT_COPYSTATS
+        MENUITEM SEPARATOR
+        MENUITEM "&Options...",                 ID_EDIT_OPTIONS
+        MENUITEM "&Ignore List...",             ID_EDIT_IGNORELIST
+    END
+    POPUP "&Library"
+    BEGIN
+        MENUITEM "&Release build",              ID_LIBRARY_RELEASE, CHECKED
+        MENUITEM "&Debug build",                ID_LIBRARY_DEBUG, CHECKED
+        MENUITEM "&Profile build",              ID_LIBRARY_PROFILE, CHECKED
+    END
+    POPUP "&Help"
+    BEGIN
+        MENUITEM "&Documentation...",           ID_HELP_HELP
+        MENUITEM "&About DynamoRIO...",         ID_APP_ABOUT
+    END
+END
+
+IDR_MAINFRAME_DEMO MENU DISCARDABLE 
+BEGIN
+    POPUP "&File"
+    BEGIN
+        MENUITEM "E&xit",                       ID_APP_EXIT
+    END
+    POPUP "&Edit"
+    BEGIN
+        MENUITEM "&Copy statistics\tCtrl+C",    ID_EDIT_COPYSTATS
+    END
+    POPUP "&Help"
+    BEGIN
+        MENUITEM "&About DynamoRIO...",         ID_APP_ABOUT
+    END
+END
+
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Accelerator
+//
+
+IDR_MAINFRAME ACCELERATORS MOVEABLE PURE 
+BEGIN
+    "C",            ID_EDIT_COPYSTATS,      VIRTKEY, CONTROL, NOINVERT
+    "N",            ID_FILE_NEW,            VIRTKEY, CONTROL, NOINVERT
+    "O",            ID_FILE_OPEN,           VIRTKEY, CONTROL, NOINVERT
+    "P",            ID_FILE_PRINT,          VIRTKEY, CONTROL, NOINVERT
+    "R",            ID_FILE_RUN,            VIRTKEY, CONTROL, NOINVERT
+    "S",            ID_FILE_SAVE,           VIRTKEY, CONTROL, NOINVERT
+    "V",            ID_EDIT_PASTE,          VIRTKEY, CONTROL, NOINVERT
+    VK_BACK,        ID_EDIT_UNDO,           VIRTKEY, ALT, NOINVERT
+    VK_DELETE,      ID_EDIT_CUT,            VIRTKEY, SHIFT, NOINVERT
+    VK_F6,          ID_NEXT_PANE,           VIRTKEY, NOINVERT
+    VK_F6,          ID_PREV_PANE,           VIRTKEY, SHIFT, NOINVERT
+    VK_INSERT,      ID_EDIT_COPY,           VIRTKEY, CONTROL, NOINVERT
+    VK_INSERT,      ID_EDIT_PASTE,          VIRTKEY, SHIFT, NOINVERT
+    "X",            ID_EDIT_CUT,            VIRTKEY, CONTROL, NOINVERT
+    "Z",            ID_EDIT_UNDO,           VIRTKEY, CONTROL, NOINVERT
+END
+
+IDR_MAINFRAME_DEMO ACCELERATORS MOVEABLE PURE 
+BEGIN
+    "C",            ID_EDIT_COPYSTATS,      VIRTKEY, CONTROL, NOINVERT
+END
+
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Dialog
+//
+
+IDD_ABOUTBOX DIALOG DISCARDABLE  0, 0, 211, 193
+STYLE DS_MODALFRAME | WS_POPUP | WS_CAPTION | WS_SYSMENU
+CAPTION "About DynamoRIO"
+FONT 8, "MS Sans Serif"
+BEGIN
+    LTEXT           "DynamoRIO Version 1.3.1",IDC_STATIC,57,82,102,11,
+                    SS_NOPREFIX
+    LTEXT           "Copyright (C) 2007-2009",IDC_STATIC,67,99,78,8
+    DEFPUSHBUTTON   "OK",IDOK,81,172,50,14,WS_GROUP
+    CONTROL         153,IDC_STATIC,"Static",SS_BITMAP,43,117,125,44
+    CONTROL         157,IDC_STATIC,"Static",SS_BITMAP,70,7,71,66
+END
+
+IDD_DYNAMORIO_FORM DIALOGEX 0, 0, 289, 317
+STYLE WS_CHILD
+FONT 11, "Lucida Console", 400
+BEGIN
+    COMBOBOX        IDC_PROCESS_LIST,7,7,188,204,CBS_DROPDOWNLIST | 
+                    CBS_AUTOHSCROLL | WS_VSCROLL | WS_TABSTOP
+    EDITTEXT        IDC_STATS,7,25,265,186,ES_MULTILINE | ES_AUTOVSCROLL | 
+                    ES_AUTOHSCROLL | ES_READONLY | ES_WANTRETURN | 
+                    WS_HSCROLL
+    SCROLLBAR       IDC_STATS_SCROLLBAR,272,24,9,187,SBS_VERT | WS_TABSTOP
+    EDITTEXT        IDC_CLIENTSTATS,7,216,274,37,ES_MULTILINE | 
+                    ES_AUTOHSCROLL | ES_READONLY | ES_WANTRETURN | 
+                    WS_VSCROLL | WS_HSCROLL
+    PUSHBUTTON      "Modify logging...",IDC_CHANGE_LOGGING,208,261,73,17
+    PUSHBUTTON      "Explore",IDC_LOGDIR_EXPLORE,7,297,35,13
+    EDITTEXT        IDC_LOGDIR,115,286,166,24,ES_MULTILINE | ES_READONLY | 
+                    WS_HSCROLL
+    LTEXT           "Log level:",IDC_STATIC,7,272,50,10
+    RTEXT           "0",IDC_LOGLEVEL_VALUE,68,272,38,9
+    LTEXT           "Running",IDC_EXITED,240,9,39,8
+    LTEXT           "Status:",IDC_STATIC,203,9,33,10
+    LTEXT           "Log mask:",IDC_STATIC,7,259,44,8
+    RTEXT           "0",IDC_LOGMASK_VALUE,61,259,45,9
+    LTEXT           "Log dir:",IDC_STATIC,7,286,42,10
+END
+
+IDD_DYNAMORIO_DEMO_FORM DIALOGEX 0, 0, 289, 313
+STYLE WS_CHILD
+FONT 11, "Lucida Console", 400
+BEGIN
+    COMBOBOX        IDC_PROCESS_LIST,7,7,188,204,CBS_DROPDOWNLIST | 
+                    CBS_AUTOHSCROLL | WS_VSCROLL | WS_TABSTOP
+    EDITTEXT        IDC_STATS,7,25,265,210,ES_MULTILINE | ES_AUTOVSCROLL | 
+                    ES_AUTOHSCROLL | ES_READONLY | ES_WANTRETURN | 
+                    WS_HSCROLL
+    SCROLLBAR       IDC_STATS_SCROLLBAR,272,24,9,211,SBS_VERT | WS_TABSTOP
+    EDITTEXT        IDC_CLIENTSTATS,7,240,274,63,ES_MULTILINE | 
+                    ES_AUTOHSCROLL | ES_READONLY | ES_WANTRETURN | 
+                    WS_VSCROLL | WS_HSCROLL
+    LTEXT           "Running",IDC_EXITED,240,9,39,8
+    LTEXT           "Status:",IDC_STATIC,203,9,33,10
+END
+
+IDD_OPTIONS DIALOG DISCARDABLE  0, 0, 294, 300
+STYLE DS_MODALFRAME | WS_POPUP | WS_CAPTION | WS_SYSMENU
+CAPTION "DynamoRIO Options"
+FONT 8, "MS Sans Serif"
+BEGIN
+    CONTROL         "Instrumentation library",IDC_INSTRLIBNAME,"Button",
+                    BS_AUTOCHECKBOX | WS_TABSTOP,7,7,82,12
+    EDITTEXT        IDC_EDIT_INSTRLIBNAME,94,7,143,12,ES_AUTOHSCROLL
+    PUSHBUTTON      "Browse...",IDC_BROWSE_INSTRLIBNAME,243,7,44,12
+    GROUPBOX        "Logging",IDC_STATIC,7,22,280,114
+    CONTROL         "Export statistics",IDC_STATS,"Button",BS_AUTOCHECKBOX | 
+                    WS_TABSTOP,14,35,64,12
+    CONTROL         "Logging verbosity level",IDC_LOGLEVEL,"Button",
+                    BS_AUTOCHECKBOX | WS_TABSTOP,14,48,86,12
+    EDITTEXT        IDC_EDIT_LOGLEVEL,110,48,35,12,ES_AUTOHSCROLL | 
+                    ES_READONLY
+    PUSHBUTTON      "Set level and mask...",IDC_LOGGING_BUTTON,152,53,92,15
+    CONTROL         "Logging mask",IDC_LOGMASK,"Button",BS_AUTOCHECKBOX | 
+                    WS_TABSTOP,14,62,71,12
+    EDITTEXT        IDC_EDIT_LOGMASK,110,62,35,12,ES_AUTOHSCROLL | 
+                    ES_READONLY
+    CONTROL         "Trace dump, text format",IDC_TRACEDUMP_TEXT,"Button",
+                    BS_AUTOCHECKBOX | WS_TABSTOP,14,91,102,12
+    CONTROL         "Trace dump, binary format",IDC_TRACEDUMP_BINARY,"Button",
+                    BS_AUTOCHECKBOX | WS_TABSTOP,14,105,100,12
+    CONTROL         "Trace dump: trace origins",IDC_TRACEDUMP_ORIGINS,"Button",
+                    BS_AUTOCHECKBOX | WS_TABSTOP,14,119,99,12
+    GROUPBOX        "Code Cache",IDC_STATIC,7,142,131,44
+    CONTROL         "Max basic block KB",IDC_CACHE_BB_MAX,"Button",
+                    BS_AUTOCHECKBOX | WS_TABSTOP,14,153,76,12
+    EDITTEXT        IDC_EDIT_CACHE_BB_MAX,95,153,35,12,ES_AUTOHSCROLL | 
+                    ES_NUMBER
+    CONTROL         "Max trace KB",IDC_CACHE_TRACE_MAX,"Button",
+                    BS_AUTOCHECKBOX | WS_TABSTOP,14,169,71,12
+    EDITTEXT        IDC_EDIT_CACHE_TRACE_MAX,95,169,35,12,ES_AUTOHSCROLL | 
+                    ES_NUMBER
+    CONTROL         "Trace threshold",IDC_HOT_THRESHOLD,"Button",
+                    BS_AUTOCHECKBOX | WS_TABSTOP,165,151,64,12
+    EDITTEXT        IDC_EDIT_HOT_THRESHOLD,239,151,35,12,ES_AUTOHSCROLL | 
+                    ES_NUMBER
+    GROUPBOX        "Profiling",IDC_STATIC,7,191,131,50
+    CONTROL         "Profiling: exit counters (profile build only)",
+                    IDC_PROF_COUNTS,"Button",BS_AUTOCHECKBOX | BS_MULTILINE | 
+                    WS_TABSTOP,15,199,85,22
+    CONTROL         "Profiling: PC sampling (Linux only)",IDC_PROF_PCS,
+                    "Button",BS_AUTOCHECKBOX | BS_MULTILINE | WS_TABSTOP,15,
+                    220,85,19
+    GROUPBOX        "Disable DynamoRIO Features",IDC_STATIC,158,174,129,67
+    CONTROL         "Do not intercept Windows asynchronous control flow",
+                    IDC_NOASYNCH,"Button",BS_AUTOCHECKBOX | BS_MULTILINE | 
+                    WS_TABSTOP,165,185,102,20
+    CONTROL         "Never link",IDC_NOLINK,"Button",BS_AUTOCHECKBOX | 
+                    WS_TABSTOP,165,205,64,12
+    CONTROL         "Disable DynamoRIO (run applications natively)",
+                    IDC_NULLCALLS,"Button",BS_AUTOCHECKBOX | BS_MULTILINE | 
+                    WS_TABSTOP,165,214,112,24
+    LTEXT           "Resulting DYNAMORIO_OPTIONS environment variable:",
+                    IDC_STATIC,7,249,278,11
+    EDITTEXT        IDC_OPTIONS_EDIT,7,260,280,13,ES_AUTOHSCROLL
+    DEFPUSHBUTTON   "OK",IDOK,46,279,50,14
+    PUSHBUTTON      "Cancel",IDCANCEL,112,279,50,14
+    PUSHBUTTON      "Set Permanently",IDC_SET_PERMANENT,178,279,69,14
+    CONTROL         "Logfile notification to stderr",IDC_NOTIFY,"Button",
+                    BS_AUTOCHECKBOX | WS_TABSTOP,14,77,100,12
+END
+
+IDD_CMDLINE DIALOG DISCARDABLE  0, 0, 280, 94
+STYLE DS_MODALFRAME | WS_POPUP | WS_CAPTION | WS_SYSMENU
+CAPTION "Command Line and Working Directory"
+FONT 8, "MS Sans Serif"
+BEGIN
+    EDITTEXT        IDC_CMDLINE,7,19,266,12,ES_AUTOHSCROLL
+    EDITTEXT        IDC_WORKING_DIR,7,50,212,12,ES_AUTOHSCROLL
+    PUSHBUTTON      "Browse...",IDC_WORKING_DIR_BROWSE,229,49,44,14
+    DEFPUSHBUTTON   "OK",IDOK,82,73,50,14
+    PUSHBUTTON      "Cancel",IDCANCEL,148,73,50,14
+    LTEXT           "Command-line arguments:",IDC_STATIC,7,7,134,10
+    LTEXT           "Working directory:",IDC_STATIC,7,39,116,9
+END
+
+IDD_LOGGING DIALOG DISCARDABLE  0, 0, 244, 165
+STYLE DS_MODALFRAME | WS_POPUP | WS_CAPTION | WS_SYSMENU
+CAPTION "Set Logging"
+FONT 8, "MS Sans Serif"
+BEGIN
+    LTEXT           "Verbose Level:",IDC_STATIC,7,50,28,19
+    COMBOBOX        IDC_VERBOSITY,37,53,28,62,CBS_DROPDOWNLIST | CBS_SORT | 
+                    WS_VSCROLL | WS_TABSTOP
+    GROUPBOX        "Select Components to Produce Log Output",IDC_STATIC,76,
+                    6,161,130
+    CONTROL         "Statistics",IDC_LOG_STATS,"Button",BS_AUTOCHECKBOX | 
+                    WS_TABSTOP,86,19,51,10
+    CONTROL         "Top-level",IDC_LOG_TOP,"Button",BS_AUTOCHECKBOX | 
+                    WS_TABSTOP,86,30,51,10
+    CONTROL         "Threads",IDC_LOG_THREADS,"Button",BS_AUTOCHECKBOX | 
+                    WS_TABSTOP,86,41,51,10
+    CONTROL         "System calls",IDC_LOG_SYSCALLS,"Button",BS_AUTOCHECKBOX | 
+                    WS_TABSTOP,86,52,51,10
+    CONTROL         "Asynchronous",IDC_LOG_ASYNCH,"Button",BS_AUTOCHECKBOX | 
+                    WS_TABSTOP,86,63,61,10
+    CONTROL         "Interpreter",IDC_LOG_INTERP,"Button",BS_AUTOCHECKBOX | 
+                    WS_TABSTOP,86,74,51,10
+    CONTROL         "Emitting",IDC_LOG_EMIT,"Button",BS_AUTOCHECKBOX | 
+                    WS_TABSTOP,86,85,51,10
+    CONTROL         "Linking",IDC_LOG_LINKS,"Button",BS_AUTOCHECKBOX | 
+                    WS_TABSTOP,165,19,51,10
+    CONTROL         "Code cache",IDC_LOG_CACHE,"Button",BS_AUTOCHECKBOX | 
+                    WS_TABSTOP,165,30,51,10
+    CONTROL         "Fragments",IDC_LOG_FRAGMENT,"Button",BS_AUTOCHECKBOX | 
+                    WS_TABSTOP,165,41,51,10
+    CONTROL         "Dispatch",IDC_LOG_DISPATCH,"Button",BS_AUTOCHECKBOX | 
+                    WS_TABSTOP,165,52,51,10
+    CONTROL         "Monitor",IDC_LOG_MONITOR,"Button",BS_AUTOCHECKBOX | 
+                    WS_TABSTOP,165,63,51,10
+    CONTROL         "Heap",IDC_LOG_HEAP,"Button",BS_AUTOCHECKBOX | 
+                    WS_TABSTOP,165,74,51,10
+    CONTROL         "Memory areas",IDC_LOG_VMAREAS,"Button",BS_AUTOCHECKBOX | 
+                    WS_TABSTOP,165,85,61,10
+    PUSHBUTTON      "&Everything",IDC_LOG_ALL,97,99,52,14
+    PUSHBUTTON      "&Nothing",IDC_LOG_NONE,160,99,52,14
+    LTEXT           "Mask:",IDC_STATIC,121,120,21,10
+    LTEXT           "0x",IDC_STATIC,151,120,9,8
+    EDITTEXT        IDC_EDIT_MASK,160,118,28,12,ES_AUTOHSCROLL
+    DEFPUSHBUTTON   "OK",IDOK,64,144,50,14
+    PUSHBUTTON      "Cancel",IDCANCEL,130,144,50,14
+END
+
+IDD_IGNORELIST DIALOG DISCARDABLE  0, 0, 335, 98
+STYLE DS_MODALFRAME | WS_POPUP | WS_CAPTION | WS_SYSMENU
+CAPTION "Systemwide Ignore List"
+FONT 8, "MS Sans Serif"
+BEGIN
+    DEFPUSHBUTTON   "OK",IDOK,66,77,50,14
+    PUSHBUTTON      "Cancel",IDCANCEL,132,77,50,14
+    EDITTEXT        IDC_IGNORELIST,7,33,321,14,ES_AUTOHSCROLL
+    LTEXT           "Semicolon-separated list of executable names that DynamoRIO should NOT take control of.\n(This list only applies when Run All is set.)",
+                    IDC_STATIC,7,7,321,20
+    LTEXT           "This list is kept in the environment variable DYNAMORIO_IGNORE.",
+                    IDC_STATIC,7,53,321,11
+    PUSHBUTTON      "Set Permanently",IDC_SET_PERMANENT,198,77,70,14
+END
+
+IDD_SYSTEMWIDE_CONFIRMATION DIALOG DISCARDABLE  0, 0, 227, 82
+STYLE DS_MODALFRAME | WS_POPUP | WS_CAPTION | WS_SYSMENU
+CAPTION "Confirmation"
+FONT 8, "MS Sans Serif"
+BEGIN
+    DEFPUSHBUTTON   "&Yes",IDOK,55,61,50,14
+    PUSHBUTTON      "&No",IDCANCEL,121,61,50,14
+    CONTROL         "&Do not ask again",IDC_NOT_AGAIN,"Button",
+                    BS_AUTOCHECKBOX | WS_TABSTOP,78,41,71,9
+    CTEXT           "Running every process in the system under DynamoRIO is a potentially dangerous operation, as DynamoRIO has not been tested on every program.  Do you want to continue?",
+                    IDC_STATIC,7,7,213,33
+END
+
+
+#ifndef _MAC
+/////////////////////////////////////////////////////////////////////////////
+//
+// Version
+//
+
+VS_VERSION_INFO VERSIONINFO
+ FILEVERSION 1,0,0,1
+ PRODUCTVERSION 1,0,0,1
+ FILEFLAGSMASK 0x3fL
+#ifdef _DEBUG
+ FILEFLAGS 0x1L
+#else
+ FILEFLAGS 0x0L
+#endif
+ FILEOS 0x4L
+ FILETYPE 0x1L
+ FILESUBTYPE 0x0L
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+        BLOCK "040904b0"
+        BEGIN
+            VALUE "CompanyName", "VMware\0"
+            VALUE "FileDescription", "DynamoRIO GUI\0"
+            VALUE "FileVersion", "1, 0, 0, 1\0"
+            VALUE "InternalName", "DynamoRIO GUI\0"
+            VALUE "LegalCopyright", "Copyright (C) 2002-2008\0"
+            VALUE "OriginalFilename", "DRgui.EXE\0"
+            VALUE "ProductName", "DynamoRIO GUI\0"
+            VALUE "ProductVersion", "1, 0, 0, 1\0"
+        END
+    END
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Translation", 0x409, 1200
+    END
+END
+
+#endif    // !_MAC
+
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// DESIGNINFO
+//
+
+#ifdef APSTUDIO_INVOKED
+GUIDELINES DESIGNINFO MOVEABLE PURE 
+BEGIN
+    IDD_ABOUTBOX, DIALOG
+    BEGIN
+        LEFTMARGIN, 7
+        RIGHTMARGIN, 204
+        TOPMARGIN, 7
+        BOTTOMMARGIN, 186
+    END
+
+    IDD_DYNAMORIO_FORM, DIALOG
+    BEGIN
+        LEFTMARGIN, 7
+        RIGHTMARGIN, 281
+        TOPMARGIN, 7
+        BOTTOMMARGIN, 310
+    END
+
+    IDD_OPTIONS, DIALOG
+    BEGIN
+        LEFTMARGIN, 7
+        RIGHTMARGIN, 287
+        TOPMARGIN, 7
+        BOTTOMMARGIN, 293
+    END
+
+    IDD_CMDLINE, DIALOG
+    BEGIN
+        LEFTMARGIN, 7
+        RIGHTMARGIN, 273
+        TOPMARGIN, 7
+        BOTTOMMARGIN, 87
+    END
+
+    IDD_LOGGING, DIALOG
+    BEGIN
+        LEFTMARGIN, 7
+        RIGHTMARGIN, 237
+        TOPMARGIN, 7
+        BOTTOMMARGIN, 158
+    END
+
+    IDD_IGNORELIST, DIALOG
+    BEGIN
+        LEFTMARGIN, 7
+        RIGHTMARGIN, 328
+        TOPMARGIN, 7
+        BOTTOMMARGIN, 91
+    END
+
+    IDD_SYSTEMWIDE_CONFIRMATION, DIALOG
+    BEGIN
+        LEFTMARGIN, 7
+        RIGHTMARGIN, 220
+        TOPMARGIN, 7
+        BOTTOMMARGIN, 75
+    END
+END
+#endif    // APSTUDIO_INVOKED
+
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Dialog Info
+//
+
+IDD_LOGGING DLGINIT
+BEGIN
+    IDC_VERBOSITY, 0x403, 2, 0
+0x0030, 
+    IDC_VERBOSITY, 0x403, 2, 0
+0x0031, 
+    IDC_VERBOSITY, 0x403, 2, 0
+0x0032, 
+    IDC_VERBOSITY, 0x403, 2, 0
+0x0033, 
+    IDC_VERBOSITY, 0x403, 2, 0
+0x0034, 
+    0
+END
+
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// String Table
+//
+
+STRINGTABLE DISCARDABLE 
+BEGIN
+    IDR_MAINFRAME           "DynamoRIO\n\nDynamo\n\n\nDynamoRIO.Document\nDynamoRIO Document"
+END
+
+STRINGTABLE DISCARDABLE 
+BEGIN
+    IDR_MAINFRAME_DEMO      "DynamoRIO\n\nDynamo\n\n\nDynamoRIO.Document\nDynamoRIO Document"
+END
+
+STRINGTABLE DISCARDABLE 
+BEGIN
+    AFX_IDS_APP_TITLE       "DynamoRIO"
+    AFX_IDS_IDLEMESSAGE     "Ready"
+END
+
+STRINGTABLE DISCARDABLE 
+BEGIN
+    ID_INDICATOR_EXT        "EXT"
+    ID_INDICATOR_CAPS       "CAP"
+    ID_INDICATOR_NUM        "NUM"
+    ID_INDICATOR_SCRL       "SCRL"
+    ID_INDICATOR_OVR        "OVR"
+    ID_INDICATOR_REC        "REC"
+END
+
+STRINGTABLE DISCARDABLE 
+BEGIN
+    ID_FILE_NEW             "Create a new document\nNew"
+    ID_FILE_OPEN            "Run an application under DynamoRIO\nRun an application"
+    ID_FILE_CLOSE           "Close the active document\nClose"
+    ID_FILE_SAVE            "Save the active document\nSave"
+    ID_FILE_SAVE_AS         "Save the active document with a new name\nSave As"
+    ID_FILE_PAGE_SETUP      "Change the printing options\nPage Setup"
+    ID_FILE_PRINT_SETUP     "Change the printer and printing options\nPrint Setup"
+    ID_FILE_PRINT           "Print the active document\nPrint"
+    ID_FILE_PRINT_PREVIEW   "Display full pages\nPrint Preview"
+END
+
+STRINGTABLE DISCARDABLE 
+BEGIN
+    ID_APP_ABOUT            "Display program information, version number and copyright\nAbout"
+    ID_APP_EXIT             "Quit the application; prompts to save documents\nExit"
+END
+
+STRINGTABLE DISCARDABLE 
+BEGIN
+    ID_FILE_MRU_FILE1       "Open this document"
+    ID_FILE_MRU_FILE2       "Open this document"
+    ID_FILE_MRU_FILE3       "Open this document"
+    ID_FILE_MRU_FILE4       "Open this document"
+    ID_FILE_MRU_FILE5       "Open this document"
+    ID_FILE_MRU_FILE6       "Open this document"
+    ID_FILE_MRU_FILE7       "Open this document"
+    ID_FILE_MRU_FILE8       "Open this document"
+    ID_FILE_MRU_FILE9       "Open this document"
+    ID_FILE_MRU_FILE10      "Open this document"
+    ID_FILE_MRU_FILE11      "Open this document"
+    ID_FILE_MRU_FILE12      "Open this document"
+    ID_FILE_MRU_FILE13      "Open this document"
+    ID_FILE_MRU_FILE14      "Open this document"
+    ID_FILE_MRU_FILE15      "Open this document"
+    ID_FILE_MRU_FILE16      "Open this document"
+END
+
+STRINGTABLE DISCARDABLE 
+BEGIN
+    ID_NEXT_PANE            "Switch to the next window pane\nNext Pane"
+    ID_PREV_PANE            "Switch back to the previous window pane\nPrevious Pane"
+END
+
+STRINGTABLE DISCARDABLE 
+BEGIN
+    ID_WINDOW_SPLIT         "Split the active window into panes\nSplit"
+END
+
+STRINGTABLE DISCARDABLE 
+BEGIN
+    ID_EDIT_CLEAR           "Erase the selection\nErase"
+    ID_EDIT_CLEAR_ALL       "Erase everything\nErase All"
+    ID_EDIT_COPY            "Copy the selection and put it on the Clipboard\nCopy"
+    ID_EDIT_CUT             "Cut the selection and put it on the Clipboard\nCut"
+    ID_EDIT_FIND            "Find the specified text\nFind"
+    ID_EDIT_PASTE           "Insert Clipboard contents\nPaste"
+    ID_EDIT_REPEAT          "Repeat the last action\nRepeat"
+    ID_EDIT_REPLACE         "Replace specific text with different text\nReplace"
+    ID_EDIT_SELECT_ALL      "Select the entire document\nSelect All"
+    ID_EDIT_UNDO            "Undo the last action\nUndo"
+    ID_EDIT_REDO            "Redo the previously undone action\nRedo"
+END
+
+STRINGTABLE DISCARDABLE 
+BEGIN
+    ID_VIEW_TOOLBAR         "Show or hide the toolbar\nToggle ToolBar"
+    ID_VIEW_STATUS_BAR      "Show or hide the status bar\nToggle StatusBar"
+END
+
+STRINGTABLE DISCARDABLE 
+BEGIN
+    AFX_IDS_SCSIZE          "Change the window size"
+    AFX_IDS_SCMOVE          "Change the window position"
+    AFX_IDS_SCMINIMIZE      "Reduce the window to an icon"
+    AFX_IDS_SCMAXIMIZE      "Enlarge the window to full size"
+    AFX_IDS_SCNEXTWINDOW    "Switch to the next document window"
+    AFX_IDS_SCPREVWINDOW    "Switch to the previous document window"
+    AFX_IDS_SCCLOSE         "Close the active window and prompts to save the documents"
+END
+
+STRINGTABLE DISCARDABLE 
+BEGIN
+    AFX_IDS_SCRESTORE       "Restore the window to normal size"
+    AFX_IDS_SCTASKLIST      "Activate Task List"
+END
+
+STRINGTABLE DISCARDABLE 
+BEGIN
+    AFX_IDS_PREVIEW_CLOSE   "Close print preview mode\nCancel Preview"
+END
+
+STRINGTABLE DISCARDABLE 
+BEGIN
+    ID_SYSTEMWIDE_INJECT    "Enable or disable system-wide injection\nToggle system-wide injection"
+    ID_SYSTEMWIDE_LIST      "Edit list of ""do-not-inject-into"" programs\nEdit ignore list"
+    ID_EDIT_OPTIONS         "Set DynamoRIO runtime parameters\nSet options"
+    ID_EDIT_COPYSTATS       "Copy statistics to clipboard\nCopy statistics"
+    ID_VIEW_PROCESS         "Select new process to view\nView new process"
+    ID_LIBRARY_RELEASE      "Use release build of DynamoRIO library\nUse release library"
+    ID_LIBRARY_DEBUG        "Use debug build of DynamoRIO library\nUse debug library"
+    ID_LIBRARY_PROFILE      "Use profile build of DynamoRIO library\nUse profile library"
+    ID_FILE_SYSTEMWIDE      "Run all applications under DynamoRIO\nRun all"
+    ID_FILE_RUN             "Run an application under DynamoRIO\nRun an application"
+END
+
+#endif    // English (U.S.) resources
+/////////////////////////////////////////////////////////////////////////////
+
+
+
+#ifndef APSTUDIO_INVOKED
+/////////////////////////////////////////////////////////////////////////////
+//
+// Generated from the TEXTINCLUDE 3 resource.
+//
+#define _AFX_NO_SPLITTER_RESOURCES
+#define _AFX_NO_OLE_RESOURCES
+#define _AFX_NO_TRACKER_RESOURCES
+#define _AFX_NO_PROPERTY_RESOURCES
+
+#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
+#ifdef _WIN32
+LANGUAGE 9, 1
+#pragma code_page(1252)
+#endif //_WIN32
+#include "res\DynamoRIO.rc2"  // non-Microsoft Visual C++ edited resources
+#include "afxres.rc"         // Standard components
+#include "afxprint.rc"       // printing/print preview resources
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+#endif    // not APSTUDIO_INVOKED
+
diff --git a/tools/DRgui/DynamoRIODoc.cpp b/tools/DRgui/DynamoRIODoc.cpp
new file mode 100644
index 000000000..aff2c8133
--- /dev/null
+++ b/tools/DRgui/DynamoRIODoc.cpp
@@ -0,0 +1,416 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// DynamoRIODoc.cpp : implementation of the CDynamoRIODoc class
+//
+
+#include "stdafx.h"
+#include <assert.h>
+#include <commdlg.h>
+#include <imagehlp.h>
+#include <direct.h>
+#include "DynamoRIO.h"
+#include "DynamoRIODoc.h"
+#include "ShellInterface.h"
+#include "CmdlineDlg.h"
+
+#ifdef _DEBUG
+#define new DEBUG_NEW
+#undef THIS_FILE
+static char THIS_FILE[] = __FILE__;
+#endif
+
+#define INJECTOR_SUBPATH   _T("\\bin\\drinject.exe")
+
+/////////////////////////////////////////////////////////////////////////////
+// CDynamoRIODoc
+
+IMPLEMENT_DYNCREATE(CDynamoRIODoc, CDocument)
+
+    BEGIN_MESSAGE_MAP(CDynamoRIODoc, CDocument)
+    //{{AFX_MSG_MAP(CDynamoRIODoc)
+    // NOTE - the ClassWizard will add and remove mapping macros here.
+    //    DO NOT EDIT what you see in these blocks of generated code!
+    //}}AFX_MSG_MAP
+    END_MESSAGE_MAP()
+
+    /////////////////////////////////////////////////////////////////////////////
+    // CDynamoRIODoc construction/destruction
+
+    CDynamoRIODoc::CDynamoRIODoc()
+{
+#ifndef DYNAMORIO_DEMO
+    InitPaths();
+
+    CShellInterface::Initialize();
+#endif
+
+    // make sure framework prompts to "save unsaved work"
+    SetModifiedFlag(TRUE);
+}
+
+#ifndef DYNAMORIO_DEMO
+void CDynamoRIODoc::InitPaths()
+{
+    // limit to 200 to give room for rest of injector path
+    int len = GetEnvironmentVariable(_T("DYNAMORIO_HOME"), m_dynamorio_home, _MAX_DIR);
+    //NOCHECKIN:
+    //    assert(len > 0 && len < _MAX_DIR && len + _tcslen(INJECTOR_SUBPATH) < MAX_PATH);
+    if (len == 0)
+        m_dynamorio_home[0] = _T('\0');
+    else
+        NULL_TERMINATE_BUFFER(m_dynamorio_home);
+    _stprintf(m_injector_path, _T("%s%s"), m_dynamorio_home, INJECTOR_SUBPATH);
+
+#if 0
+    // there are issues with privileges to write to Program Files, so we no longer do this:
+    _stprintf(m_logs_dir, _T("%s\\logs"), m_dynamorio_home);
+#else
+    // instead we do $USERPROFILE\\Application Data\\DynamoRIO
+    TCHAR userprof[MAX_PATH];
+    CString profile_dir;
+    len = GetEnvironmentVariable(_T("USERPROFILE"), userprof, MAX_PATH);
+    if (len <= 0) {
+        // on NT, use $SYSTEMROOT\\Profiles
+        len = GetEnvironmentVariable(_T("SYSTEMROOT"), userprof, MAX_PATH);
+        assert(len > 0);
+        profile_dir.Format(_T("%s\\Profiles"), userprof);
+    } else {
+        profile_dir.Format(_T("%s"), userprof);
+    }
+    _stprintf(m_logs_dir, _T("%s\\Application Data\\DynamoRIO"), profile_dir);
+    if (!SetCurrentDirectory(m_logs_dir)) {
+        if (!CreateDirectory(m_logs_dir, NULL)) {
+            _stprintf(userprof, _T("Cannot create default working directory %s"), m_logs_dir);
+            MessageBox(NULL, userprof, _T("Error"), MB_OK | MYMBFLAGS);
+            _stprintf(m_logs_dir, _T("c:\\")); // any better ideas?
+        }
+    }
+
+#endif
+}
+#endif /* !DYNAMORIO_DEMO */
+
+CDynamoRIODoc::~CDynamoRIODoc()
+{
+#ifndef DYNAMORIO_DEMO
+    CShellInterface::Uninitialize();
+#endif
+}
+
+BOOL CDynamoRIODoc::OnNewDocument()
+{
+    if (!CDocument::OnNewDocument())
+        return FALSE;
+
+    // TODO: add reinitialization code here
+    // (SDI documents will reuse this document)
+
+    // blank title initially
+    SetTitle(_T(""));
+
+    return TRUE;
+}
+
+BOOL CDynamoRIODoc::OnOpenDocument(LPCTSTR lpszPathName)
+{
+    if (!CDocument::OnOpenDocument(lpszPathName))
+        return FALSE;
+
+    // initialization that doesn't happen in Serialize
+    // for us, that's all of it
+
+    return TRUE;
+}
+
+#ifndef DYNAMORIO_DEMO
+BOOL CDynamoRIODoc::RunApplication(LPCTSTR lpszPathName)
+{
+    // application name (may be different than lpszPathName, for shortcuts)
+    TCHAR app_name[MAX_PATH];
+    // working directory
+    TCHAR rundir[_MAX_DIR];
+    // entire command line = app_name + arguments
+    TCHAR app_cmdline[MAX_PATH*2];
+    // buffer for error messages
+    TCHAR msg[MAX_PATH*3];
+
+    // default working directory
+    TCHAR default_rundir[_MAX_DIR];
+    _stprintf(default_rundir, _T("%s"), m_logs_dir);
+
+    CString app_args; // will be used to build app_cmdline
+
+    // if app is a shortcut, resolve it now
+    int len = _tcslen(lpszPathName);
+    TCHAR * last4 = (TCHAR *)lpszPathName + len - 4;
+    if (_tcscmp(last4, _T(".lnk")) == 0) {
+        TCHAR params[MAX_PATH];
+        if (CShellInterface::ResolveLinkFile((TCHAR *)lpszPathName, app_name,
+                                             params, rundir, CDynamoRIOApp::GetActiveView()->m_hWnd)) {
+            _stprintf(msg, _T("Resolved link file to %s %s\nin directory %s\n"),
+                      app_name, params, rundir);
+            MessageBox(NULL, msg, _T("Link File"), MB_OK | MYMBFLAGS);
+            app_args.Format(_T("%s"), params);
+        } else {
+            _stprintf(msg, _T("Failed to resolve link file"));
+            MessageBox(NULL, msg, _T("Error"), MB_OK | MYMBFLAGS);
+            return FALSE;
+        }       
+    } else {
+        _stprintf(app_name, _T("%s"), lpszPathName);
+
+        // ask for arguments and working dir
+        // pass in default working dir
+        CString defwdir(default_rundir);
+        CCmdlineDlg dlg(defwdir);
+        int res = dlg.DoModal();
+        if (res == IDCANCEL)
+            return FALSE;
+        CString wdir = dlg.GetWorkingDir();
+        _tcscpy(rundir, wdir.GetBuffer(0));
+        app_args = dlg.GetArguments();
+    }
+
+    // construct app_cmdline
+    // if passing app name to injector as argument, must quote it
+    // if running natively, do not quote it
+    if (CDynamoRIOApp::SystemwideSet()) {
+        _stprintf(app_cmdline, _T("%s %s"), app_name, app_args);
+    } else {
+        _stprintf(app_cmdline, _T("\"%s\" %s"), app_name, app_args);
+    }
+
+    // if rundir has environment vars, resolve them
+    CString dir(rundir);
+    CString realdir(_T(""));
+    int start = dir.Find(_T('%'));
+    int old_end = 0;
+    while (start != -1) {
+        realdir += dir.Mid(old_end, start - old_end);
+        int end = dir.Find(_T('%'), start+1);
+        if (end == -1) {
+            realdir += dir.Right(dir.GetLength() - start);
+            break;
+        }
+#if 0 // debugging stuff
+        _stprintf(msg, _T("from %d to %d"), start, end);
+        MessageBox(NULL, msg, _T("env var value"), MB_OK);
+#endif
+        CString name = dir.Mid(start + 1, end - start - 1);
+#if 0 // debugging stuff
+        MessageBox(NULL, name.GetBuffer(0), _T("env var name"), MB_OK);
+#endif
+        int len = GetEnvironmentVariable(name.GetBuffer(0), msg, MAX_PATH);
+        if (len == 0)
+            msg[0] = _T('\0');
+#if 0 // debugging stuff
+        MessageBox(NULL, msg, _T("env var value"), MB_OK);
+#endif
+        realdir += msg;
+        old_end = end + 1;
+
+        start = dir.Find(_T('%'), end+1);
+    }
+    if (!realdir.IsEmpty())
+        _stprintf(rundir, _T("%s"), realdir.GetBuffer(0));
+
+    // if rundir is empty, use default
+    // FIXME: have option to set default?
+    if (rundir[0] == _T('\0'))
+        _tcscpy(rundir, default_rundir);
+
+    // be robust -- make sure app file exists
+    CFile check;
+    if (!check.Open(app_name, CFile::modeRead|CFile::shareDenyNone)) {
+        _stprintf(msg, _T("Application %s does not exist"), app_name);
+        MessageBox(NULL, msg, _T("Error"), MB_OK | MYMBFLAGS);
+        return FALSE;
+    }
+    check.Close();
+
+    // now go to run directory
+    if (!SetCurrentDirectory(rundir)) {
+        _stprintf(msg, _T("Error changing to working directory %s"), rundir);
+        MessageBox(NULL, msg, _T("Error"), MB_OK | MYMBFLAGS);
+        return FALSE;
+    }
+
+    // now launch application
+    TCHAR *launch_name;
+    TCHAR launch_cmdline[MAX_PATH*2];
+
+    if (CDynamoRIOApp::SystemwideSet()) {
+        // just launch app natively, systemwide injector will catch it
+        launch_name = app_name;
+        // note that we want target app name as part of cmd line
+        _tcscpy(launch_cmdline, app_cmdline);
+    } else {
+        // explicitly launch app under injector
+        launch_name = m_injector_path;
+        // note that we want target app name as part of cmd line
+        TCHAR * dll_path = CDynamoRIOApp::GetDllPath();
+        _stprintf(launch_cmdline, _T("\"%s\" \"%s\" %s"), 
+                  m_injector_path, dll_path, app_cmdline);
+
+        // be robust
+        if (!check.Open(m_injector_path, CFile::modeRead|CFile::shareDenyNone)) {
+            _stprintf(msg, _T("DynamoRIO injector %s does not exist"), m_injector_path);
+            MessageBox(NULL, msg, _T("DynamoRIO Configuration Error"), MB_OK | MYMBFLAGS);
+            return FALSE;
+        }
+        check.Close();
+        if (!check.Open(dll_path, CFile::modeRead|CFile::shareDenyNone)) {
+            _stprintf(msg, _T("DynamoRIO library %s does not exist"), dll_path);
+            MessageBox(NULL, msg, _T("DynamoRIO Configuration Error"), MB_OK | MYMBFLAGS);
+            return FALSE;
+        }
+        check.Close();
+    }
+
+    assert(_tcslen(rundir) + _tcslen(launch_cmdline) + _tcslen(_T("\nin directory\n")) < MAX_PATH*2);
+
+#if 0 // I'm disabling this dialog, usually just annoying
+    _stprintf(msg, _T("%s\nin directory\n%s"), launch_cmdline, rundir);
+    MessageBox(NULL, msg, _T("About to run"), MB_OK | MB_TOPMOST);
+#endif // 0
+
+#if 0 // I'm taking this out, it requires an extra library and doesn't add much
+    // FIXME: do this up above, instead of checking app_name exists?
+    PLOADED_IMAGE       li;
+#ifdef UNICODE
+    // ImageLoad does not take unicode strings
+    char ansi_launch_name[MAX_PATH];
+    WideCharToMultiByte(CP_ACP, 0, launch_name, _tcslen(launch_name),
+                        ansi_launch_name, MAX_PATH, NULL, NULL);
+    if (li = ImageLoad(ansi_launch_name, NULL))
+#else
+    if (li = ImageLoad(launch_name, NULL))
+#endif
+    {
+        ImageUnload(li);
+    } else {
+        _stprintf(msg, _T("Failed to load %s"), launch_name);
+        MessageBox(NULL, msg, _T("Error"), MB_OK | MYMBFLAGS);
+        return FALSE;
+    }
+#endif // 0
+    
+    // Launch the application process
+    STARTUPINFO         si;
+    PROCESS_INFORMATION pi;
+    STARTUPINFO mysi;
+    GetStartupInfo(&mysi);
+    ZeroMemory(&si, sizeof(si));
+    si.cb = sizeof(si);
+    si.dwFlags = STARTF_USESTDHANDLES;
+    si.hStdInput = mysi.hStdInput;
+    si.hStdOutput = mysi.hStdOutput;
+    si.hStdError = mysi.hStdError;
+
+    /* Must specify TRUE for bInheritHandles so child inherits stdin! */
+    if (!CreateProcess(launch_name, launch_cmdline, NULL, NULL, TRUE,
+                       DETACHED_PROCESS, // to avoid console creation!
+                       NULL, NULL, &si, &pi)) {
+        _stprintf(msg, _T("Failed to load %s"), app_name);
+        MessageBox(NULL, msg, _T("Error"), MB_OK | MYMBFLAGS);
+        return FALSE;
+    }
+
+#if 0
+        // FIXME: We have the injector's pid, not the child's pid!
+    if (!CDynamoRIOApp::GetActiveView()->SelectProcess(pi.dwProcessId))
+        MessageBox(NULL, _T("Failed to select"), _T("Error"),  MB_OK | MYMBFLAGS);
+#endif
+    // it takes some time for the new process to start up and its shared memory
+    // to become visible, so wait for it, but don't wait forever -- there could be
+    // a usage error or the new process may crash
+    int i = 0;
+    // wait 10ms*200 = 2 seconds
+    while (i < 200) {
+        // change the <no instances found> entry
+        if (CDynamoRIOApp::GetActiveView()->UpdateProcessList())
+            break;
+        Sleep(10);
+        i++;
+    }
+    
+    // don't call SetTitle -- View sets it to what's being viewed
+    // SetTitle(lpszPathName);
+    
+    return TRUE;
+}
+#endif /* !DYNAMORIO_DEMO */
+
+/////////////////////////////////////////////////////////////////////////////
+// CDynamoRIODoc serialization
+
+void CDynamoRIODoc::Serialize(CArchive& ar)
+{
+    if (ar.IsStoring()) {
+        // TODO: add storing code here
+    } else {
+        // TODO: add loading code here
+    }
+}
+
+/////////////////////////////////////////////////////////////////////////////
+// CDynamoRIODoc diagnostics
+
+#ifdef _DEBUG
+void CDynamoRIODoc::AssertValid() const
+{
+    CDocument::AssertValid();
+}
+
+void CDynamoRIODoc::Dump(CDumpContext& dc) const
+{
+    CDocument::Dump(dc);
+}
+#endif //_DEBUG
+
+/////////////////////////////////////////////////////////////////////////////
+// CDynamoRIODoc commands
+
+
+BOOL CDynamoRIODoc::SaveModified() 
+{
+#ifndef DYNAMORIO_DEMO
+    CDynamoRIOApp::AboutToExit();
+#endif
+    return CDocument::SaveModified();
+}
+
diff --git a/tools/DRgui/DynamoRIODoc.h b/tools/DRgui/DynamoRIODoc.h
new file mode 100644
index 000000000..a8236dc3b
--- /dev/null
+++ b/tools/DRgui/DynamoRIODoc.h
@@ -0,0 +1,105 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// DynamoRIODoc.h : interface of the CDynamoRIODoc class
+//
+/////////////////////////////////////////////////////////////////////////////
+
+#if !defined(AFX_DYNAMORIODOC_H__832A85FF_15EB_41D0_86C5_BE2CDD746729__INCLUDED_)
+#define AFX_DYNAMORIODOC_H__832A85FF_15EB_41D0_86C5_BE2CDD746729__INCLUDED_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif // _MSC_VER > 1000
+
+#define DYNAMORIO_SHARED_MEMORY_KEY "DynamoRIOStatistics" 
+
+class CDynamoRIODoc : public CDocument
+{
+protected: // create from serialization only
+    CDynamoRIODoc();
+    DECLARE_DYNCREATE(CDynamoRIODoc)
+
+    // Attributes
+public:
+
+    // Operations
+public:
+
+    // Overrides
+    // ClassWizard generated virtual function overrides
+    //{{AFX_VIRTUAL(CDynamoRIODoc)
+public:
+    virtual BOOL OnNewDocument();
+    virtual void Serialize(CArchive& ar);
+protected:
+    virtual BOOL SaveModified();
+    //}}AFX_VIRTUAL
+
+    // Implementation
+public:
+#ifndef DYNAMORIO_DEMO
+    void InitPaths();
+    TCHAR m_dynamorio_home[_MAX_DIR];
+    TCHAR m_injector_path[MAX_PATH];
+    TCHAR m_logs_dir[MAX_PATH];
+#endif
+
+    BOOL OnOpenDocument(LPCTSTR lpszPathName);
+    BOOL RunApplication(LPCTSTR lpszPathName);
+    virtual ~CDynamoRIODoc();
+#ifdef _DEBUG
+    virtual void AssertValid() const;
+    virtual void Dump(CDumpContext& dc) const;
+#endif
+
+protected:
+
+    // Generated message map functions
+protected:
+    //{{AFX_MSG(CDynamoRIODoc)
+    // NOTE - the ClassWizard will add and remove member functions here.
+    //    DO NOT EDIT what you see in these blocks of generated code !
+    //}}AFX_MSG
+    DECLARE_MESSAGE_MAP()
+};
+
+/////////////////////////////////////////////////////////////////////////////
+
+//{{AFX_INSERT_LOCATION}}
+// Microsoft Visual C++ will insert additional declarations immediately before the previous line.
+
+#endif // !defined(AFX_DYNAMORIODOC_H__832A85FF_15EB_41D0_86C5_BE2CDD746729__INCLUDED_)
diff --git a/tools/DRgui/DynamoRIOView.cpp b/tools/DRgui/DynamoRIOView.cpp
new file mode 100644
index 000000000..33511d239
--- /dev/null
+++ b/tools/DRgui/DynamoRIOView.cpp
@@ -0,0 +1,745 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// DynamoRIOView.cpp : implementation of the CDynamoRIOView class
+//
+
+#include "stdafx.h"
+#include "DynamoRIO.h"
+
+#include "DynamoRIODoc.h"
+#include "DynamoRIOView.h"
+#include "DynamoRIO.h" // for global GetActiveView
+#include "LoggingDlg.h"
+#include <assert.h>
+
+#ifdef _DEBUG
+#define new DEBUG_NEW
+#undef THIS_FILE
+static char THIS_FILE[] = __FILE__;
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+// CDynamoRIOView
+
+IMPLEMENT_DYNCREATE(CDynamoRIOView, CFormView)
+
+BEGIN_MESSAGE_MAP(CDynamoRIOView, CFormView)
+    //{{AFX_MSG_MAP(CDynamoRIOView)
+#ifndef DYNAMORIO_DEMO
+    ON_BN_CLICKED(IDC_CHANGE_LOGGING, OnChangeLogging)
+    ON_BN_CLICKED(IDC_LOGDIR_EXPLORE, OnLogDirExplore)
+#endif
+    ON_COMMAND(ID_EDIT_COPYSTATS, OnEditCopystats)
+    ON_WM_VSCROLL()
+    //}}AFX_MSG_MAP
+    // Standard printing commands
+    ON_COMMAND(ID_FILE_PRINT, CFormView::OnFilePrint)
+    ON_COMMAND(ID_FILE_PRINT_DIRECT, CFormView::OnFilePrint)
+    ON_COMMAND(ID_FILE_PRINT_PREVIEW, CFormView::OnFilePrintPreview)
+    // process list
+    ON_CBN_SELCHANGE(IDC_PROCESS_LIST, OnSelchangeList)
+    ON_CBN_DROPDOWN(IDC_PROCESS_LIST, OnDropdownList)
+END_MESSAGE_MAP()
+
+    /////////////////////////////////////////////////////////////////////////////
+    // CDynamoRIOView construction/destruction
+
+
+VOID CALLBACK TimerProc(
+                        HWND /*hwnd*/,     // handle of window for timer messages
+                        UINT /*uMsg*/,     // WM_TIMER message
+                        UINT idEvent,  // timer identifier
+                        DWORD /*dwTime*/  // current system time
+                        )
+{
+    CDynamoRIOView * view = CDynamoRIOApp::GetActiveView();
+    if (view == NULL) {
+        KillTimer(NULL, idEvent);
+        return;
+    }
+    BOOL cont = view->Refresh();
+    if (!cont) {
+        KillTimer(NULL, idEvent);
+    }
+}
+
+void CDynamoRIOView::ZeroStrings()
+{
+    m_Exited = _T("");
+    m_ClientStats = _T("");
+#ifndef DYNAMORIO_DEMO
+    m_LogLevel = _T("0");
+    m_LogMask = _T("0x0000");
+    m_LogDir = _T("");
+#endif
+}
+
+CDynamoRIOView::CDynamoRIOView()
+    : CFormView(CDynamoRIOView::IDD)
+{
+    //{{AFX_DATA_INIT(CDynamoRIOView)
+    //}}AFX_DATA_INIT
+    // TODO: add construction code here
+
+    ZeroStrings();
+    m_stats = NULL;
+    m_clientMap = NULL;
+    m_clientView = NULL;
+    m_clientStats = NULL;
+    m_list_pos = 0;
+    m_selected_pid = 0;
+    m_StatsViewLines = 0;
+
+    // FIXME: share w/ CDynamoRIOApp::CheckWindowsVersion
+    OSVERSIONINFO version;
+    version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
+    int res = GetVersionEx(&version);
+    assert(res != 0);
+    if (version.dwPlatformId == VER_PLATFORM_WIN32_NT && version.dwMajorVersion == 4)
+        m_windows_NT = TRUE;
+    else
+        m_windows_NT = FALSE;
+}
+
+void CDynamoRIOView::ClearData()
+{
+   if (m_stats != NULL) {
+        free_dynamorio_stats(m_stats);
+        m_stats = NULL;
+    }
+    if (m_clientMap != NULL) {
+        assert(m_clientView != NULL);
+        UnmapViewOfFile(m_clientView);
+        CloseHandle(m_clientMap);
+        m_clientMap = m_clientView = m_clientStats = NULL;
+    }
+}
+
+CDynamoRIOView::~CDynamoRIOView()
+{
+    ClearData();
+}
+
+void CDynamoRIOView::DoDataExchange(CDataExchange* pDX)
+{
+    CFormView::DoDataExchange(pDX);
+    //{{AFX_DATA_MAP(CDynamoRIOView)
+    DDX_Control(pDX, IDC_PROCESS_LIST, m_ProcessList);
+    DDX_Control(pDX, IDC_STATS, m_StatsCtl);
+    DDX_Control(pDX, IDC_STATS_SCROLLBAR, m_StatsSB);
+    DDX_Text(pDX, IDC_CLIENTSTATS, m_ClientStats);
+    DDX_Text(pDX, IDC_EXITED, m_Exited);
+#ifndef DYNAMORIO_DEMO
+    DDX_Text(pDX, IDC_LOGLEVEL_VALUE, m_LogLevel);
+    DDX_Text(pDX, IDC_LOGMASK_VALUE, m_LogMask);
+    DDX_Text(pDX, IDC_LOGDIR, m_LogDir);
+#endif
+    //}}AFX_DATA_MAP
+}
+
+BOOL CDynamoRIOView::PreCreateWindow(CREATESTRUCT& cs)
+{
+    // TODO: Modify the Window class or styles here by modifying
+    //  the CREATESTRUCT cs
+
+    return CFormView::PreCreateWindow(cs);
+}
+
+void CDynamoRIOView::OnInitialUpdate()
+{
+    CFormView::OnInitialUpdate();
+    GetParentFrame()->RecalcLayout();
+    ResizeParentToFit();
+    OnDropdownList();
+    // 100 flashes too much with long stats list
+    //      ::SetTimer(NULL, NULL, 100, TimerProc);
+    ::SetTimer(NULL, NULL, 200, TimerProc);
+}
+
+
+/////////////////////////////////////////////////////////////////////////////
+// CDynamoRIOView printing
+
+BOOL CDynamoRIOView::OnPreparePrinting(CPrintInfo* pInfo)
+{
+    // default preparation
+    return DoPreparePrinting(pInfo);
+}
+
+void CDynamoRIOView::OnBeginPrinting(CDC* /*pDC*/, CPrintInfo* /*pInfo*/)
+{
+    // TODO: add extra initialization before printing
+}
+
+void CDynamoRIOView::OnEndPrinting(CDC* /*pDC*/, CPrintInfo* /*pInfo*/)
+{
+    // TODO: add cleanup after printing
+}
+
+void CDynamoRIOView::OnPrint(CDC* /*pDC*/, CPrintInfo* /*pInfo*/)
+{
+    // TODO: add customized printing code here
+}
+
+/////////////////////////////////////////////////////////////////////////////
+// CDynamoRIOView diagnostics
+
+#ifdef _DEBUG
+void CDynamoRIOView::AssertValid() const
+{
+    CFormView::AssertValid();
+}
+
+void CDynamoRIOView::Dump(CDumpContext& dc) const
+{
+    CFormView::Dump(dc);
+}
+
+CDynamoRIODoc* CDynamoRIOView::GetDocument() // non-debug version is inline
+{
+    ASSERT(m_pDocument->IsKindOf(RUNTIME_CLASS(CDynamoRIODoc)));
+    return (CDynamoRIODoc*)m_pDocument;
+}
+#endif //_DEBUG
+
+/////////////////////////////////////////////////////////////////////////////
+// CDynamoRIOView message handlers
+
+/////////////////////////////////////////////////////////////////////////////
+
+BOOL
+pw_callback_under_dr(process_info_t *pi, void **param)
+{
+    TCHAR *resstr;
+    TCHAR reschar;
+    int res;
+    TCHAR buf[MAXIMUM_PATH];
+    DWORD version;
+    BOOL under_dr;
+    CDynamoRIOView *view = (CDynamoRIOView *) param;
+
+    version = 0;
+    res = under_dynamorio_ex(pi->ProcessID, &version);
+    switch (res) {
+    case DLL_PROFILE : resstr=_T("SC profile"); reschar=_T('P'); break;
+    case DLL_RELEASE : resstr=_T("SC release"); reschar=_T('R'); break;
+    case DLL_DEBUG : resstr=_T("SC debug"); reschar=_T('D'); break;
+    case DLL_CUSTOM : resstr=_T("SC custom"); reschar=_T('C'); break;
+    case DLL_NONE : resstr=_T("native"); reschar=_T('N'); break;
+    case DLL_UNKNOWN : 
+    default : resstr=_T("<error>"); reschar=_T('?');
+    }
+    under_dr = !(res == DLL_NONE || res == DLL_UNKNOWN);
+    if (under_dr) {
+        _stprintf(buf, _T("%5d  %c   %") UNICODE_PRINTF,
+                  pi->ProcessID, reschar, pi->ProcessName);
+        view->m_ProcessList.InsertString(view->m_list_pos, buf);
+        view->m_ProcessList.SetItemData(view->m_list_pos, pi->ProcessID);
+        view->m_list_pos++;
+    }
+    return TRUE;
+}
+
+void CDynamoRIOView::EnumerateInstances()
+{
+    // clear all old data
+    ClearData();
+    m_ProcessList.ResetContent();
+
+    m_list_pos = 0;
+    process_walk(&pw_callback_under_dr, (void **)this);
+
+    // now insert the 0th entry, the default selection
+    if (m_ProcessList.GetCount() > 0) {
+        m_ProcessList.InsertString(0, _T("<select an instance>"));
+    } else {
+        m_ProcessList.InsertString(0, _T("<no instances to view>"));
+    }
+    m_ProcessList.SetItemData(0, 0);
+    m_ProcessList.SetCurSel(0);
+    CString pname;
+    m_ProcessList.GetLBText(m_ProcessList.GetCurSel(), pname);
+    GetDocument()->SetTitle(pname);
+}
+
+void CDynamoRIOView::OnSelchangeList() 
+{
+    TCHAR buf[128];
+
+    ClearData();
+
+    m_selected_pid = m_ProcessList.GetItemData(m_ProcessList.GetCurSel());
+
+    // find the client stats shared memory that corresponds to this process
+    HANDLE statsCountMap =
+        OpenFileMapping(FILE_MAP_READ, FALSE,
+                        m_windows_NT ? TEXT(CLIENT_SHMEM_KEY_NT) :
+                        TEXT(CLIENT_SHMEM_KEY));
+    if (statsCountMap != NULL) {
+        PVOID statsCountView = MapViewOfFile(statsCountMap, FILE_MAP_READ, 0, 0, 0);
+        assert(statsCountView != NULL);
+        int statsCount = *((int *)statsCountView);
+        int num = 0;
+        // see flakiness comments above on main stats count: we go to +20 as workaround
+        while (num < statsCount+20) {
+            _stprintf(buf, _T("%")ASCII_PRINTF _T(".%03d"),
+                      m_windows_NT ? CLIENT_SHMEM_KEY_NT : CLIENT_SHMEM_KEY, num);
+            m_clientMap = OpenFileMapping(FILE_MAP_READ, FALSE, buf);
+            if (m_clientMap != NULL) {
+                m_clientView = MapViewOfFile(m_clientMap, FILE_MAP_READ, 0, 0, 0);
+                assert(m_clientView != NULL);
+                m_clientStats = (client_stats *) m_clientView;
+                if (m_clientStats->pid == m_selected_pid)
+                    break;
+                UnmapViewOfFile(m_clientView);
+                CloseHandle(m_clientMap);
+                m_clientMap = m_clientView = m_clientStats = NULL;
+            }
+            num++;
+        }
+        UnmapViewOfFile(statsCountView);
+        CloseHandle(statsCountMap);
+    }
+
+    m_StatsSB.SetScrollPos(0);
+
+    Refresh();
+}
+
+void CDynamoRIOView::OnDropdownList()
+{
+    EnumerateInstances();
+}
+
+BOOL CDynamoRIOView::SelectProcess(int pid)
+{
+    EnumerateInstances();
+
+    TCHAR prefix[16];
+    _stprintf(prefix, _T("%5d"), pid);
+    int num = m_ProcessList.FindString(-1, prefix);
+    if (num == CB_ERR)
+        return FALSE;
+    m_ProcessList.SetCurSel(num);
+    OnSelchangeList();
+
+    return TRUE;
+}
+
+BOOL CDynamoRIOView::UpdateProcessList()
+{
+    EnumerateInstances();
+    CString first;
+    m_ProcessList.GetLBText(0, first);
+    if (first.Compare(_T("<select an instance>")) != 0)
+        return FALSE;
+    // Incredibly, VC6.0 says "error unreachable code" if I use an else here
+    return TRUE;
+}
+
+#ifdef X64
+/* FIXME: need to resize window for this */
+# define STAT_PFMT _T("%19")_T(SZFC)
+#else
+# define STAT_PFMT _T("%10")_T(SZFC)
+#endif
+
+uint CDynamoRIOView::PrintStat(TCHAR *c, uint i, BOOL filter)
+{
+    if (filter) {
+#if 0
+        // Filter out persisted cache stat for now
+        // FIXME: have "show 0 values" checkbox
+        // FIXME: need to count up stats that match filter and use that
+        // for scrollbar max, else get flicker and empty space at bottom
+        if (strncmp(m_stats->stats[i].name, "Persisted caches",
+                    strlen("Persisted caches")) == 0)
+            return 0;
+#endif
+    }
+    return _stprintf(c, _T("%*.*") ASCII_PRINTF _T(" = ") STAT_PFMT _T("\r\n"),
+                     BUFFER_SIZE_ELEMENTS(m_stats->stats[i].name),
+                     BUFFER_SIZE_ELEMENTS(m_stats->stats[i].name),
+                     m_stats->stats[i].name, m_stats->stats[i].value);
+}
+
+uint CDynamoRIOView::PrintClientStats(TCHAR *c, TCHAR *max)
+{
+    if (m_clientStats == NULL)
+        return 0;
+#define CLIENTSTAT_NAME_MAX_SHOW CLIENTSTAT_NAME_MAX_LEN
+    uint i;
+    TCHAR *start = c;
+    char (*names)[CLIENTSTAT_NAME_MAX_LEN] =
+        (char (*)[CLIENTSTAT_NAME_MAX_LEN]) m_clientStats->data;
+    uint *vals = (uint *)
+        ((char*)m_clientStats->data + 
+         m_clientStats->num_stats*CLIENTSTAT_NAME_MAX_LEN*sizeof(char));
+    /* account for struct alignment */
+    vals = (uint *) ALIGN_FORWARD(vals, sizeof(uint));
+    for (i=0; i<m_clientStats->num_stats; i++) {
+        if (c >= max - CLIENTSTAT_NAME_MAX_SHOW*2 - 3)
+            break;
+        c += _stprintf(c, _T("%*.*") ASCII_PRINTF _T(" = ") STAT_PFMT _T("\r\n"),
+                       CLIENTSTAT_NAME_MAX_SHOW, CLIENTSTAT_NAME_MAX_SHOW,
+                       names[i], vals[i]);
+        assert(c < max);
+    }
+    return c - start;
+}
+
+// NOCHECKIN: resize stats boxes w/ dialog resize:
+// http://www.codeguru.com/forum/showthread.php?t=79384
+
+BOOL CDynamoRIOView::Refresh()
+{
+    if (m_selected_pid <= 0) {
+        ZeroStrings();
+        return TRUE;
+    }
+    // We have to grab new stats every refresh
+    dr_statistics_t *new_stats = get_dynamorio_stats(m_selected_pid);
+    if (new_stats == NULL) {
+        if (m_stats != NULL) {
+            // Leave the stats for an exited process in place, for viewing
+            // and copying
+        } else
+            return TRUE;
+    } else {
+        if (m_stats != NULL)
+            free_dynamorio_stats(m_stats);
+        m_stats = new_stats;
+    }
+
+    uint i;
+#define MAX_VISIBLE_STATS 75
+#define STATS_BUFSZ (MAX_VISIBLE_STATS*(sizeof(single_stat_t)*2/*cover %10u, etc.*/))
+    TCHAR buf[STATS_BUFSZ];
+    TCHAR *c = buf;
+    buf[0] = _T('\0');
+
+    // If I just use a CString hooked up via DDX, re-setting the
+    // text causes the top of the visible box to show the top of
+    // the stats string; plus the scrollbar goes to the top,
+    // unless held in place; probably any reset of the string
+    // does an auto-reset of the display and scrollbar position.
+    // To have all the text in there do this:
+    //      int scroll_pos = m_StatsCtl.GetScrollPos(SB_VERT);
+    //      m_StatsCtl.SetWindowText(buf);
+    //      m_StatsCtl.LineScroll(scroll_pos);
+    // But then there's a lot of flickering: too much.
+    // We only put visible text lines in the CEdit box, to reduce the
+    // flicker.  It's too hard to do it w/ the CEdit's own scrollbar,
+    // which sets itself based on the actual text, so we have a
+    // separate one.
+
+    // HACK: I don't know how to find out how many lines will
+    // fit w/o putting actual text in there
+    if (m_StatsViewLines == 0) {
+        for (i = 0; i < m_stats->num_stats &&
+                 i < MAX_VISIBLE_STATS /* can't be more than this */; i++) {
+            if (c >= &buf[STATS_BUFSZ-STAT_NAME_MAX_LEN*2])
+                break;
+            c += PrintStat(c, i, FALSE/*no filter*/);
+            assert(c < &buf[STATS_BUFSZ-1]);
+        }
+        m_StatsCtl.SetWindowText(buf);
+        UpdateData(FALSE);  // write to screen
+        // I tried having only one screenful of string there, and
+        // setting the scrollbar range to be larger, but it doesn't
+        // seem to support that.
+        RECT rect;
+        m_StatsCtl.GetRect(&rect);
+        CPoint pos(rect.right, rect.bottom);
+        m_StatsViewLines = HIWORD(m_StatsCtl.CharFromPos(pos));
+        assert(m_StatsViewLines > 0);
+        m_StatsSB.SetScrollRange(0, m_stats->num_stats-1, TRUE/*redraw*/);
+        c = buf;
+
+        SCROLLINFO info;
+        info.cbSize = sizeof(SCROLLINFO);
+        info.fMask = SIF_PAGE;
+        info.nPage = m_StatsViewLines;
+        m_StatsSB.SetScrollInfo(&info);
+    }
+
+    int scroll_pos = m_StatsSB.GetScrollPos();
+    DWORD shown = 0;
+    uint printed;
+    for (i = scroll_pos;
+         i < m_stats->num_stats && shown < m_StatsViewLines; i++) {
+        if (c >= &buf[STATS_BUFSZ-STAT_NAME_MAX_LEN*2])
+            break;
+        printed = PrintStat(c, i, TRUE/*filter*/);
+        c += printed;
+        assert(c < &buf[STATS_BUFSZ-1]);
+        if (printed > 0)
+            shown++;
+    }
+    m_StatsCtl.SetWindowText(buf);
+    // num_stats could have changed so update
+    m_StatsSB.SetScrollRange(0, m_stats->num_stats-1, TRUE/*redraw*/);
+
+    if (new_stats == NULL)
+        m_Exited = _T("  Exited"); // line right end up with Running
+    else
+        m_Exited = _T("Running");
+#ifndef DYNAMORIO_DEMO
+    m_LogLevel.Format(_T("%d"), m_stats->loglevel);
+    m_LogMask.Format(_T("0x%05X"), m_stats->logmask);
+    m_LogDir.Format(_T("%") ASCII_PRINTF, m_stats->logdir);
+#endif
+
+    if (m_clientStats != NULL) {
+#define CLIENTSTATS_BUFSZ USHRT_MAX
+        TCHAR buf[CLIENTSTATS_BUFSZ];
+        PrintClientStats(buf, &buf[CLIENTSTATS_BUFSZ-1]);
+        m_ClientStats.Format(_T("%s"), buf);
+    } else
+        m_ClientStats.Format(_T(""));
+
+    UpdateData(FALSE);  // write to screen
+    return TRUE;
+}
+
+#ifndef DYNAMORIO_DEMO
+void CDynamoRIOView::OnChangeLogging() 
+{
+    if (m_stats == NULL) {
+        MessageBox(_T("No instance is selected"), _T("Error"), MB_OK | MYMBFLAGS);
+        return;
+    }
+    if (m_stats->loglevel == 0) {
+        MessageBox(_T("If the application began with log level 0, its logging\n")
+                   _T("cannot be changed.\n"),
+                   _T("Notification"), MB_OK | MYMBFLAGS);
+        return;
+    }
+    CLoggingDlg dlg(m_stats->loglevel, m_stats->logmask);
+    int res = dlg.DoModal();
+    if (res == IDCANCEL)
+        return;
+
+    int level = dlg.GetLevel();
+    int mask = dlg.GetMask();
+
+    // NOCHECKIN: need to write via drmarker
+    m_stats->loglevel = level;
+    m_stats->logmask = mask;
+    m_LogLevel.Format(_T("%d"), m_stats->loglevel);
+    m_LogMask.Format(_T("0x%04X"), m_stats->logmask);
+    UpdateData(FALSE);  // write to screen
+}
+
+void CDynamoRIOView::OnLogDirExplore() 
+{
+    if (m_stats == NULL) {
+        MessageBox(_T("No instance is selected"), _T("Error"), MB_OK | MYMBFLAGS);
+        return;
+    }
+
+    UpdateData(TRUE); // get data from controls
+
+    if (m_LogDir.Find(_T("<none")) == 0) {
+        MessageBox(_T("There is no log dir because the loglevel was 0 when the application started"),
+                   _T("Error"), MB_OK | MYMBFLAGS);
+        return;
+    }
+
+    HINSTANCE res = ShellExecute(m_hWnd, _T("explore"), 
+                                 m_LogDir, NULL, NULL, SW_SHOWNORMAL);
+    if ((int)res <= 32) {
+        TCHAR msg[MAX_PATH*2];
+        _stprintf(msg, _T("Error exploring %s"), m_LogDir);
+        MessageBox(msg, _T("Error"), MB_OK | MYMBFLAGS);
+    }
+}
+#endif /* DYNAMORIO_DEMO */
+
+void CDynamoRIOView::OnEditCopystats() 
+{
+    if (m_ProcessList.GetCurSel() == 0) {
+        MessageBox(_T("No instance selected"), _T("Error"), MB_OK | MYMBFLAGS);
+        return;
+    }
+    if (!OpenClipboard()) {
+        MessageBox(_T("Error opening clipboard"), _T("Error"),
+                   MB_OK | MYMBFLAGS);
+        return;
+    }
+    EmptyClipboard();
+
+#define CLIPBOARD_BUFSZ USHRT_MAX
+    TCHAR buf[CLIPBOARD_BUFSZ];
+    TCHAR *pos = buf;
+
+    if (m_selected_pid > 0) {
+        if (m_stats != NULL) {
+            pos += _stprintf(pos, _T("Process id                  = %d\r\n"),
+                             m_stats->process_id);
+            pos += _stprintf(pos, _T("Process name                = %") ASCII_PRINTF
+                             _T("\r\n"), m_stats->process_name);
+            pos += _stprintf(pos, _T("Status                      = %s\r\n"),
+                             m_Exited.GetBuffer(0));
+#ifndef DYNAMORIO_DEMO
+            pos += _stprintf(pos, _T("Log mask                    = %s\r\n"),
+                             m_LogMask.GetBuffer(0));
+            pos += _stprintf(pos, _T("Log level                   = %s\r\n"),
+                             m_LogLevel.GetBuffer(0));
+            pos += _stprintf(pos, _T("Log file                    = %s\r\n"),
+                             m_LogDir.GetBuffer(0));
+#endif
+            pos += _stprintf(pos, _T("\r\nSTATS\r\n"));
+            uint i;
+            for (i = 0; i < m_stats->num_stats; i++) {
+                if (pos >= &buf[STATS_BUFSZ-STAT_NAME_MAX_LEN*2])
+                    break;
+                // FIXME: Filter here too
+                pos += PrintStat(pos, i, TRUE/*filter*/);
+                assert(pos < &buf[STATS_BUFSZ-1]);
+            }
+        } else {
+            CString pname;
+            m_ProcessList.GetLBText(m_ProcessList.GetCurSel(), pname);
+            _stprintf(pos, _T("%s"), pname);
+            pos += _tcslen(pos);
+        }
+    }
+    if (m_clientStats != NULL) {
+        pos += PrintClientStats(pos, &buf[STATS_BUFSZ-1]);
+    }
+
+    int len = _tcslen(buf);
+
+    // Allocate a global memory object for the text. 
+    HGLOBAL hglbCopy = GlobalAlloc(GMEM_DDESHARE, 
+                                   (len + 1) * sizeof(TCHAR)); 
+    if (hglbCopy == NULL) { 
+        CloseClipboard(); 
+        return;
+    } 
+
+    // Lock the handle and copy the text to the buffer. 
+    LPTSTR lptstrCopy = (LPTSTR) GlobalLock(hglbCopy); 
+    memcpy(lptstrCopy, buf, (len + 1) * sizeof(TCHAR)); 
+    lptstrCopy[len] = (TCHAR) 0;    // null TCHARacter 
+    GlobalUnlock(hglbCopy); 
+
+    // Place the handle on the clipboard. 
+    SetClipboardData(
+#ifdef UNICODE
+                     CF_UNICODETEXT,
+#else
+                     CF_TEXT,
+#endif
+                     hglbCopy); 
+
+    CloseClipboard();
+}
+
+void CDynamoRIOView::OnVScroll(UINT nSBCode, UINT nPos, CScrollBar *pScrollBar) 
+{
+    int minpos;
+    int maxpos;
+    if (pScrollBar == NULL)
+        return;
+    pScrollBar->GetScrollRange(&minpos, &maxpos); 
+    maxpos = pScrollBar->GetScrollLimit();
+    int curpos = pScrollBar->GetScrollPos();
+
+    // Determine the new position
+    switch (nSBCode) {
+    case SB_TOP:         // Scroll to far top.
+        curpos = minpos;
+        break;
+
+    case SB_BOTTOM:      // Scroll to far bottom.
+        curpos = maxpos;
+        break;
+
+    case SB_ENDSCROLL:   // End scroll.
+        break;
+
+    case SB_LINEUP:      // Scroll up.
+        if (curpos > minpos)
+            curpos--;
+        break;
+
+    case SB_LINEDOWN:   // Scroll down.
+        if (curpos < maxpos)
+            curpos++;
+        break;
+
+    case SB_PAGEUP:    // Scroll one page up.
+        {
+            // Get the page size. 
+            SCROLLINFO   info;
+            pScrollBar->GetScrollInfo(&info, SIF_ALL);
+            if (curpos > minpos)
+                curpos = max(minpos, curpos - (int) info.nPage);
+        }
+        break;
+
+    case SB_PAGEDOWN:      // Scroll one page down.
+        {
+            // Get the page size. 
+            SCROLLINFO   info;
+            pScrollBar->GetScrollInfo(&info, SIF_ALL);
+            if (curpos < maxpos)
+                curpos = min(maxpos, curpos + (int) info.nPage);
+        }
+        break;
+
+    case SB_THUMBPOSITION: // Scroll to absolute position. nPos is the position
+        curpos = nPos;     // of the scroll box at the end of the drag operation.
+        break;
+
+    case SB_THUMBTRACK:   // Drag scroll box to specified position. nPos is the
+        curpos = nPos;    // position that the scroll box has been dragged to.
+        break;
+    }
+
+    // Set the new position of the thumb (scroll box).
+    pScrollBar->SetScrollPos(curpos);
+
+    CView::OnVScroll(nSBCode, nPos, pScrollBar);
+
+    // FIXME: we could try to more smoothly scroll w/ the last-copied m_stats
+    // instead of getting all new values
+    Refresh();
+}
+
diff --git a/tools/DRgui/DynamoRIOView.h b/tools/DRgui/DynamoRIOView.h
new file mode 100644
index 000000000..96309811b
--- /dev/null
+++ b/tools/DRgui/DynamoRIOView.h
@@ -0,0 +1,180 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// DynamoRIOView.h : interface of the CDynamoRIOView class
+//
+/////////////////////////////////////////////////////////////////////////////
+
+#if !defined(AFX_DYNAMORIOVIEW_H__93D53B0D_7233_4E40_8240_C733CC5C6DF8__INCLUDED_)
+#define AFX_DYNAMORIOVIEW_H__93D53B0D_7233_4E40_8240_C733CC5C6DF8__INCLUDED_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif // _MSC_VER > 1000
+
+extern "C" {
+#include "share.h"
+#include "processes.h"
+};
+
+/* client stats -- FIXME: put this stuff in shared header file, for
+ * now duplicated here and in client
+ */
+#include "afxwin.h"
+/* We prefix "Global\" to ensure this is visible across sessions, but that
+ * prefix is not supported on NT
+ */
+#define CLIENT_SHMEM_KEY_NT "DynamoRIO_Client_Statistics" 
+#define CLIENT_SHMEM_KEY "Global\\DynamoRIO_Client_Statistics" 
+#define CLIENTSTAT_NAME_MAX_LEN 50
+
+/* we allocate this struct in the shared memory: */
+typedef struct _client_stats {
+    uint num_stats;
+    bool exited;
+    uint pid;
+    /* num_stats strings, each STAT_NAME_MAX_LEN chars, followed by
+     * num_stats values
+     */
+    char data[CLIENTSTAT_NAME_MAX_LEN];
+} client_stats;
+
+
+class CDynamoRIOView : public CFormView
+{
+protected: // create from serialization only
+    CDynamoRIOView();
+    DECLARE_DYNCREATE(CDynamoRIOView)
+
+        public:
+    //{{AFX_DATA(CDynamoRIOView)
+    enum { 
+#ifdef DYNAMORIO_DEMO
+        IDD = IDD_DYNAMORIO_DEMO_FORM 
+#else
+        IDD = IDD_DYNAMORIO_FORM 
+#endif
+    };
+    //}}AFX_DATA
+
+    // Attributes
+public:
+    CDynamoRIODoc* GetDocument();
+    BOOL SelectProcess(int pid);
+
+    // Operations
+public:
+
+    // Overrides
+    // ClassWizard generated virtual function overrides
+    //{{AFX_VIRTUAL(CDynamoRIOView)
+public:
+    virtual BOOL PreCreateWindow(CREATESTRUCT& cs);
+protected:
+    virtual void DoDataExchange(CDataExchange* pDX);    // DDX/DDV support
+    virtual void OnInitialUpdate(); // called first time after construct
+    virtual BOOL OnPreparePrinting(CPrintInfo* pInfo);
+    virtual void OnBeginPrinting(CDC* pDC, CPrintInfo* pInfo);
+    virtual void OnEndPrinting(CDC* pDC, CPrintInfo* pInfo);
+    virtual void OnPrint(CDC* pDC, CPrintInfo* pInfo);
+    //}}AFX_VIRTUAL
+
+    // Implementation
+public:
+    BOOL UpdateProcessList();
+    BOOL Refresh();
+    virtual ~CDynamoRIOView();
+#ifdef _DEBUG
+    virtual void AssertValid() const;
+    virtual void Dump(CDumpContext& dc) const;
+#endif
+
+protected:
+    void ZeroStrings();
+    void ClearData();
+    void EnumerateInstances();
+    uint PrintStat(TCHAR *c, uint i, BOOL filter);
+    uint PrintClientStats(TCHAR *c, TCHAR *max);
+
+    dr_statistics_t *m_stats;
+    CComboBox m_ProcessList;
+    HANDLE m_clientMap;
+    PVOID m_clientView;
+    client_stats *m_clientStats;
+    DWORD m_list_pos;
+    DWORD m_selected_pid;
+    BOOL m_windows_NT;
+
+    // statistics
+    CString m_Exited;
+#ifndef DYNAMORIO_DEMO
+    CString m_LogLevel;
+    CString m_LogMask;
+    CString m_LogDir;
+#endif
+    CEdit m_StatsCtl;
+    CScrollBar m_StatsSB;
+    CSliderCtrl m_StatsSlider;
+    DWORD m_StatsViewLines;
+    CString m_ClientStats;
+
+    // Generated message map functions
+protected:
+    //{{AFX_MSG(CDynamoRIOView)
+    afx_msg void OnSelchangeList();
+    afx_msg void OnDropdownList();
+#ifndef DYNAMORIO_DEMO
+    afx_msg void OnChangeLogging();
+    afx_msg void OnLogDirExplore();
+#endif
+    afx_msg void OnEditCopystats();
+    afx_msg void OnVScroll(UINT nSBCode, UINT nPos, CScrollBar* pScrollBar);
+    //}}AFX_MSG
+    DECLARE_MESSAGE_MAP()
+    friend BOOL pw_callback_under_dr(process_info_t *pi, void **param);
+};
+
+#ifndef _DEBUG  // debug version in DynamoRIOView.cpp
+inline CDynamoRIODoc* CDynamoRIOView::GetDocument()
+{ return (CDynamoRIODoc*)m_pDocument; }
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+
+//{{AFX_INSERT_LOCATION}}
+// Microsoft Visual C++ will insert additional declarations immediately before the previous line.
+
+#endif // !defined(AFX_DYNAMORIOVIEW_H__93D53B0D_7233_4E40_8240_C733CC5C6DF8__INCLUDED_)
diff --git a/tools/DRgui/IgnoreDlg.cpp b/tools/DRgui/IgnoreDlg.cpp
new file mode 100644
index 000000000..38e97e677
--- /dev/null
+++ b/tools/DRgui/IgnoreDlg.cpp
@@ -0,0 +1,145 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// IgnoreDlg.cpp : implementation file
+//
+
+#ifndef DYNAMORIO_DEMO /* around whole file */
+
+#include "stdafx.h"
+#include "DynamoRIO.h"
+#include "IgnoreDlg.h"
+#include <assert.h>
+
+#ifdef _DEBUG
+#define new DEBUG_NEW
+#undef THIS_FILE
+static char THIS_FILE[] = __FILE__;
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+// CIgnoreDlg dialog
+
+
+CIgnoreDlg::CIgnoreDlg(CWnd* pParent /*=NULL*/)
+    : CDialog(CIgnoreDlg::IDD, pParent)
+{
+    //{{AFX_DATA_INIT(CIgnoreDlg)
+    m_IgnoreList = _T("");
+    //}}AFX_DATA_INIT
+}
+
+
+void CIgnoreDlg::DoDataExchange(CDataExchange* pDX)
+{
+    CDialog::DoDataExchange(pDX);
+    //{{AFX_DATA_MAP(CIgnoreDlg)
+    DDX_Text(pDX, IDC_IGNORELIST, m_IgnoreList);
+    //}}AFX_DATA_MAP
+}
+
+
+BEGIN_MESSAGE_MAP(CIgnoreDlg, CDialog)
+    //{{AFX_MSG_MAP(CIgnoreDlg)
+    ON_BN_CLICKED(IDC_SET_PERMANENT, OnSetPermanent)
+    //}}AFX_MSG_MAP
+    END_MESSAGE_MAP()
+
+    /////////////////////////////////////////////////////////////////////////////
+    // CIgnoreDlg message handlers
+
+    BOOL CIgnoreDlg::OnInitDialog()
+{
+    CDialog::OnInitDialog();
+
+    TCHAR path[MAX_PATH];
+    int len = GetEnvironmentVariable(_T("DYNAMORIO_IGNORE"), path, MAX_PATH);
+    if (len > 0)
+        m_IgnoreList = path; // makes new storage, right?
+    UpdateData(FALSE); // FALSE means set controls
+
+    return TRUE;
+}
+
+void CIgnoreDlg::OnOK() 
+{
+    UpdateData(TRUE); // read from controls
+    int res = SetEnvironmentVariable(_T("DYNAMORIO_IGNORE"), m_IgnoreList);
+    assert(res);
+    CDialog::OnOK();
+}
+
+void CIgnoreDlg::OnSetPermanent() 
+{
+    // it takes a while to broadcast the "we've changed env vars" message,
+    // so set a wait cursor
+    HCURSOR prev_cursor = SetCursor(LoadCursor(0, IDC_WAIT));
+
+    TCHAR msg[MAX_PATH];
+    HKEY hk;
+    int res;
+        
+    // current user only
+    res = RegOpenKeyEx(HKEY_CURRENT_USER, _T("Environment"), 0, KEY_WRITE, &hk);
+    if (res != ERROR_SUCCESS) {
+        _stprintf(msg, _T("Error writing to HKEY_CURRENT_USER\\Environment"));
+        MessageBox(msg, _T("Error"), MB_OK | MYMBFLAGS);
+    }
+
+    UpdateData(TRUE); // get values from controls
+    TCHAR *val = m_IgnoreList.GetBuffer(0);
+    res = RegSetValueEx(hk, _T("DYNAMORIO_IGNORE"), 0, REG_SZ, 
+                        (LPBYTE) val, _tcslen(val)+1);
+    assert(res == ERROR_SUCCESS);
+
+    RegCloseKey(hk);
+
+    // tell system that we've changed environment (else won't take effect until
+    // user logs out and back in)
+    DWORD dwReturnValue;
+    // Code I copied this from used an ANSI string...I'm leaving
+    // it like that FIXME
+    SendMessageTimeout(HWND_BROADCAST, WM_SETTINGCHANGE, 0,
+                       (LPARAM) "Environment", SMTO_ABORTIFHUNG, 5000, &dwReturnValue);
+
+    SetCursor(prev_cursor);
+
+    // set local env var too, and avoid questions of ability to
+    // cancel the permanent operation
+    OnOK();
+}
+
+#endif /* !DYNAMORIO_DEMO */ /* around whole file */
diff --git a/tools/DRgui/IgnoreDlg.h b/tools/DRgui/IgnoreDlg.h
new file mode 100644
index 000000000..e95fd469b
--- /dev/null
+++ b/tools/DRgui/IgnoreDlg.h
@@ -0,0 +1,84 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+#if !defined(AFX_IGNOREDLG_H__42A2FDDE_31BD_40C1_B7B3_44E955FDC7AD__INCLUDED_)
+#define AFX_IGNOREDLG_H__42A2FDDE_31BD_40C1_B7B3_44E955FDC7AD__INCLUDED_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif // _MSC_VER > 1000
+// IgnoreDlg.h : header file
+//
+
+/////////////////////////////////////////////////////////////////////////////
+// CIgnoreDlg dialog
+
+class CIgnoreDlg : public CDialog
+{
+    // Construction
+public:
+    BOOL OnInitDialog();
+    CIgnoreDlg(CWnd* pParent = NULL);   // standard constructor
+
+    // Dialog Data
+    //{{AFX_DATA(CIgnoreDlg)
+    enum { IDD = IDD_IGNORELIST };
+    CString m_IgnoreList;
+    //}}AFX_DATA
+
+
+    // Overrides
+    // ClassWizard generated virtual function overrides
+    //{{AFX_VIRTUAL(CIgnoreDlg)
+protected:
+    virtual void DoDataExchange(CDataExchange* pDX);    // DDX/DDV support
+    //}}AFX_VIRTUAL
+
+    // Implementation
+protected:
+
+    // Generated message map functions
+    //{{AFX_MSG(CIgnoreDlg)
+    virtual void OnOK();
+    afx_msg void OnSetPermanent();
+    //}}AFX_MSG
+    DECLARE_MESSAGE_MAP()
+};
+
+//{{AFX_INSERT_LOCATION}}
+// Microsoft Visual C++ will insert additional declarations immediately before the previous line.
+
+#endif // !defined(AFX_IGNOREDLG_H__42A2FDDE_31BD_40C1_B7B3_44E955FDC7AD__INCLUDED_)
diff --git a/tools/DRgui/LoggingDlg.cpp b/tools/DRgui/LoggingDlg.cpp
new file mode 100644
index 000000000..0470f04ea
--- /dev/null
+++ b/tools/DRgui/LoggingDlg.cpp
@@ -0,0 +1,314 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// LoggingDlg.cpp : implementation file
+//
+
+#ifndef DYNAMORIO_DEMO /* around whole file */
+
+#include "stdafx.h"
+#include "DynamoRIO.h"
+#include "LoggingDlg.h"
+#include <assert.h>
+
+#ifdef _DEBUG
+#define new DEBUG_NEW
+#undef THIS_FILE
+static char THIS_FILE[] = __FILE__;
+#endif
+
+// FIXME: duplicate information with utils.h...
+enum {
+    LOG_NONE = 0,
+    LOG_STATS,
+    LOG_TOP,
+    LOG_THREADS,
+    LOG_SYSCALLS,
+    LOG_ASYNCH,
+    LOG_INTERP,
+    LOG_EMIT,
+    LOG_LINKS,
+    LOG_CACHE,
+    LOG_FRAGMENT,
+    LOG_DISPATCH,
+    LOG_MONITOR,
+    LOG_HEAP,
+    LOG_VMAREAS,
+    LOG_ALL,
+};
+static const int checkboxes[] = {
+    IDC_LOG_NONE,
+    IDC_LOG_STATS,
+    IDC_LOG_TOP,
+    IDC_LOG_THREADS,
+    IDC_LOG_SYSCALLS,
+    IDC_LOG_ASYNCH,
+    IDC_LOG_INTERP,
+    IDC_LOG_EMIT,
+    IDC_LOG_LINKS,
+    IDC_LOG_CACHE,
+    IDC_LOG_FRAGMENT,
+    IDC_LOG_DISPATCH,
+    IDC_LOG_MONITOR,
+    IDC_LOG_HEAP,
+    IDC_LOG_VMAREAS,
+    IDC_LOG_ALL,
+};
+static const int masks[] = {
+    0x00000000,  // LOG_NONE
+    0x00000001,  // LOG_STATS
+    0x00000002,  // LOG_TOP
+    0x00000004,  // LOG_THREADS
+    0x00000008,  // LOG_SYSCALLS
+    0x00000010,  // LOG_ASYNCH
+    0x00000020,  // LOG_INTERP
+    0x00000040,  // LOG_EMIT
+    0x00000080,  // LOG_LINKS
+    0x00000100,  // LOG_CACHE
+    0x00000200,  // LOG_FRAGMENT
+    0x00000400,  // LOG_DISPATCH
+    0x00000800,  // LOG_MONITOR
+    0x00001000,  // LOG_HEAP
+    0x00002000,  // LOG_VMAREAS
+    0x00003fff,  // LOG_ALL
+};
+static const int num_options = sizeof(checkboxes) / sizeof(int);
+
+/////////////////////////////////////////////////////////////////////////////
+// CLoggingDlg dialog
+
+
+CLoggingDlg::CLoggingDlg(CWnd* pParent /*=NULL*/)
+    : CDialog(CLoggingDlg::IDD, pParent)
+{
+    //{{AFX_DATA_INIT(CLoggingDlg)
+    m_Mask = _T("3FFF");
+    //}}AFX_DATA_INIT
+    m_Level = 1;
+}
+
+ 
+CLoggingDlg::CLoggingDlg(int level, int mask, CWnd* pParent /*=NULL*/)
+    : CDialog(CLoggingDlg::IDD, pParent)
+{
+    //{{AFX_DATA_INIT(CLoggingDlg)
+    //}}AFX_DATA_INIT
+    if ((mask & 0xcfff0000) != 0) {
+ ::MessageBox(NULL, _T("Mask must be between 0x0000 and 0x3fff"),
+              _T("Warning"), MB_OK | MYMBFLAGS);
+       mask = 0x3fff;
+           }
+    m_Mask.Format(_T("%04X"), mask);
+    m_Level = level;
+}
+
+void CLoggingDlg::DoDataExchange(CDataExchange* pDX)
+{
+    CDialog::DoDataExchange(pDX);
+    //{{AFX_DATA_MAP(CLoggingDlg)
+    DDX_Control(pDX, IDOK, m_OKButton);
+    DDX_Control(pDX, IDC_VERBOSITY, m_Verbosity);
+    DDX_Text(pDX, IDC_EDIT_MASK, m_Mask);
+    DDV_MaxChars(pDX, m_Mask, 4);
+    //}}AFX_DATA_MAP
+}
+
+
+BEGIN_MESSAGE_MAP(CLoggingDlg, CDialog)
+    //{{AFX_MSG_MAP(CLoggingDlg)
+    ON_EN_CHANGE(IDC_EDIT_MASK, OnChangeEditMask)
+    ON_BN_CLICKED(IDC_LOG_ALL, OnLogAll)
+    ON_BN_CLICKED(IDC_LOG_VMAREAS, OnLogVmareas)
+    ON_BN_CLICKED(IDC_LOG_ASYNCH, OnLogAsynch)
+    ON_BN_CLICKED(IDC_LOG_CACHE, OnLogCache)
+    ON_BN_CLICKED(IDC_LOG_DISPATCH, OnLogDispatch)
+    ON_BN_CLICKED(IDC_LOG_EMIT, OnLogEmit)
+    ON_BN_CLICKED(IDC_LOG_FRAGMENT, OnLogFragment)
+    ON_BN_CLICKED(IDC_LOG_HEAP, OnLogHeap)
+    ON_BN_CLICKED(IDC_LOG_INTERP, OnLogInterp)
+    ON_BN_CLICKED(IDC_LOG_LINKS, OnLogLinks)
+    ON_BN_CLICKED(IDC_LOG_MONITOR, OnLogMonitor)
+    ON_BN_CLICKED(IDC_LOG_STATS, OnLogStats)
+    ON_BN_CLICKED(IDC_LOG_SYSCALLS, OnLogSyscalls)
+    ON_BN_CLICKED(IDC_LOG_THREADS, OnLogThreads)
+    ON_BN_CLICKED(IDC_LOG_TOP, OnLogTop)
+    ON_BN_CLICKED(IDC_LOG_NONE, OnLogNone)
+    //}}AFX_MSG_MAP
+    END_MESSAGE_MAP()
+
+    /////////////////////////////////////////////////////////////////////////////
+    // CLoggingDlg message handlers
+
+    BOOL CLoggingDlg::OnInitDialog()
+{
+    CDialog::OnInitDialog();
+
+    VerifyMaskString();
+
+    m_Verbosity.SetCurSel(m_Level);
+
+    return TRUE;
+}
+
+// returns -1 on error
+int CLoggingDlg::GetMaskValue()
+{
+    UpdateData(TRUE); // get values from controls
+    if (m_Mask.IsEmpty())
+        return 0;
+    int mask;
+    int res = _stscanf(m_Mask, _T("%X"), &mask);
+    // sscanf will return 3 and success on "3ugh", so manually scan it
+    if (!_istxdigit(m_Mask.GetAt(0)) ||
+        (m_Mask.GetLength() > 1 && !_istxdigit(m_Mask.GetAt(1))) ||
+        (m_Mask.GetLength() > 2 && !_istxdigit(m_Mask.GetAt(2))) ||
+        (m_Mask.GetLength() > 3 && !_istxdigit(m_Mask.GetAt(3)))) {
+        return -1;
+    }
+    if (res <= 0 || res == EOF)
+        return -1;
+    else
+        return mask;
+}
+
+BOOL CLoggingDlg::SetMaskValue(int mask)
+{
+    m_Mask.Format(_T("%04X"), mask);
+    UpdateData(FALSE); // write values to controls
+    return TRUE;
+}
+
+void CLoggingDlg::VerifyMaskString()
+{
+    int mask = GetMaskValue();
+    if (mask == -1)
+        mask = 0;
+    int i;
+    for (i=1; i<num_options-1; i++) { // skip ALL and NONE
+        CButton *button = (CButton *) GetDlgItem(checkboxes[i]);
+        assert(button != NULL);
+        if ((mask & masks[i]) != 0) {
+            if (button->GetCheck() == 0)
+                button->SetCheck(1);
+        } else {
+            if (button->GetCheck() != 0)
+                button->SetCheck(0);
+        }
+    }
+}
+
+void CLoggingDlg::OnChangeEditMask() 
+{
+    UpdateData(TRUE); // get values from controls
+    int mask = GetMaskValue();
+    if (mask < 0 || mask > masks[LOG_ALL]) {
+        m_OKButton.EnableWindow(FALSE);
+    } else {
+        m_OKButton.EnableWindow(TRUE);
+        VerifyMaskString();
+    }
+}
+
+void CLoggingDlg::CheckboxChange(int id)
+{
+    CButton *button = (CButton *) GetDlgItem(checkboxes[id]);
+    assert(button != NULL);
+    int mask = GetMaskValue();
+    if (mask == -1) {
+        // don't allow checkbox change
+        button->SetCheck(1 - button->GetCheck());
+        return;
+    }
+    if (button->GetCheck() > 0) {
+        mask |= masks[id];
+    } else {
+        mask &= ~(masks[id]);
+    }
+    SetMaskValue(mask);
+}
+
+void CLoggingDlg::OnLogVmareas() { CheckboxChange(LOG_VMAREAS); } 
+void CLoggingDlg::OnLogAsynch() { CheckboxChange(LOG_ASYNCH); } 
+void CLoggingDlg::OnLogCache() { CheckboxChange(LOG_CACHE); } 
+void CLoggingDlg::OnLogDispatch() { CheckboxChange(LOG_DISPATCH); } 
+void CLoggingDlg::OnLogEmit() { CheckboxChange(LOG_EMIT); } 
+void CLoggingDlg::OnLogFragment() { CheckboxChange(LOG_FRAGMENT); } 
+void CLoggingDlg::OnLogHeap() { CheckboxChange(LOG_HEAP); } 
+void CLoggingDlg::OnLogInterp() { CheckboxChange(LOG_INTERP); } 
+void CLoggingDlg::OnLogLinks() { CheckboxChange(LOG_LINKS); } 
+void CLoggingDlg::OnLogMonitor() { CheckboxChange(LOG_MONITOR); } 
+void CLoggingDlg::OnLogStats() { CheckboxChange(LOG_STATS); } 
+void CLoggingDlg::OnLogSyscalls() { CheckboxChange(LOG_SYSCALLS); } 
+void CLoggingDlg::OnLogThreads() { CheckboxChange(LOG_THREADS); } 
+void CLoggingDlg::OnLogTop() { CheckboxChange(LOG_TOP); } 
+
+void CLoggingDlg::OnLogAll() { 
+    SetMaskValue(masks[LOG_ALL]);
+    VerifyMaskString();
+    m_OKButton.EnableWindow(TRUE);
+} 
+
+void CLoggingDlg::OnLogNone() { 
+    SetMaskValue(masks[LOG_NONE]);
+    VerifyMaskString();
+    m_OKButton.EnableWindow(TRUE);
+} 
+
+
+int CLoggingDlg::GetLevel()
+{
+    return final_level;
+}
+
+int CLoggingDlg::GetMask()
+{
+    return final_mask;      
+}
+
+void CLoggingDlg::OnOK() 
+{
+    UpdateData(TRUE); // get values from controls
+
+    final_mask = GetMaskValue();
+    if (final_mask == -1)
+        final_mask = 0;
+        
+    final_level = m_Verbosity.GetCurSel();
+
+    CDialog::OnOK();
+}
+
+#endif /* !DYNAMORIO_DEMO */ /* around whole file */
diff --git a/tools/DRgui/LoggingDlg.h b/tools/DRgui/LoggingDlg.h
new file mode 100644
index 000000000..8b0eea00f
--- /dev/null
+++ b/tools/DRgui/LoggingDlg.h
@@ -0,0 +1,113 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+#if !defined(AFX_LOGGINGDLG_H__A05FE00B_E413_49FE_AD2F_2062AE912991__INCLUDED_)
+#define AFX_LOGGINGDLG_H__A05FE00B_E413_49FE_AD2F_2062AE912991__INCLUDED_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif // _MSC_VER > 1000
+// LoggingDlg.h : header file
+//
+
+/////////////////////////////////////////////////////////////////////////////
+// CLoggingDlg dialog
+
+class CLoggingDlg : public CDialog
+{
+    // Construction
+public:
+    int GetMask();
+    int GetLevel();
+    BOOL OnInitDialog();
+    CLoggingDlg(int level, int mask, CWnd* pParent = NULL);
+
+    // Dialog Data
+    //{{AFX_DATA(CLoggingDlg)
+    enum { IDD = IDD_LOGGING };
+    CButton m_OKButton;
+    CComboBox       m_Verbosity;
+    CString m_Mask;
+    //}}AFX_DATA
+
+
+    // Overrides
+    // ClassWizard generated virtual function overrides
+    //{{AFX_VIRTUAL(CLoggingDlg)
+protected:
+    virtual void DoDataExchange(CDataExchange* pDX);    // DDX/DDV support
+    //}}AFX_VIRTUAL
+
+    // Implementation
+protected:
+    CLoggingDlg(CWnd* pParent = NULL);   // standard constructor
+    BOOL SetMaskValue(int mask);
+    int GetMaskValue();
+    void VerifyMaskString();
+    int m_Level;
+    void CheckboxChange(int id);
+    // hold final values
+    int final_level;
+    int final_mask;
+
+    // Generated message map functions
+    //{{AFX_MSG(CLoggingDlg)
+    afx_msg void OnChangeEditMask();
+    afx_msg void OnLogAll();
+    afx_msg void OnLogVmareas();
+    afx_msg void OnLogAsynch();
+    afx_msg void OnLogCache();
+    afx_msg void OnLogDispatch();
+    afx_msg void OnLogEmit();
+    afx_msg void OnLogFragment();
+    afx_msg void OnLogHeap();
+    afx_msg void OnLogInterp();
+    afx_msg void OnLogLinks();
+    afx_msg void OnLogMonitor();
+    afx_msg void OnLogStats();
+    afx_msg void OnLogSyscalls();
+    afx_msg void OnLogThreads();
+    afx_msg void OnLogTop();
+    afx_msg void OnLogNone();
+    virtual void OnOK();
+    //}}AFX_MSG
+    DECLARE_MESSAGE_MAP()
+};
+
+//{{AFX_INSERT_LOCATION}}
+// Microsoft Visual C++ will insert additional declarations immediately before the previous line.
+
+#endif // !defined(AFX_LOGGINGDLG_H__A05FE00B_E413_49FE_AD2F_2062AE912991__INCLUDED_)
diff --git a/tools/DRgui/MainFrm.cpp b/tools/DRgui/MainFrm.cpp
new file mode 100644
index 000000000..bcb19c553
--- /dev/null
+++ b/tools/DRgui/MainFrm.cpp
@@ -0,0 +1,151 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// MainFrm.cpp : implementation of the CMainFrame class
+//
+
+#include "stdafx.h"
+#include "DynamoRIO.h"
+
+#include "MainFrm.h"
+
+#ifdef _DEBUG
+#define new DEBUG_NEW
+#undef THIS_FILE
+static char THIS_FILE[] = __FILE__;
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+// CMainFrame
+
+IMPLEMENT_DYNCREATE(CMainFrame, CFrameWnd)
+
+    BEGIN_MESSAGE_MAP(CMainFrame, CFrameWnd)
+    //{{AFX_MSG_MAP(CMainFrame)
+    ON_WM_CREATE()
+    //}}AFX_MSG_MAP
+    END_MESSAGE_MAP()
+
+    static UINT indicators[] =
+{
+    ID_SEPARATOR,           // status line indicator
+    ID_INDICATOR_CAPS,
+    ID_INDICATOR_NUM,
+    ID_INDICATOR_SCRL,
+};
+
+/////////////////////////////////////////////////////////////////////////////
+// CMainFrame construction/destruction
+
+CMainFrame::CMainFrame()
+{
+    // TODO: add member initialization code here
+        
+}
+
+CMainFrame::~CMainFrame()
+{
+}
+
+int CMainFrame::OnCreate(LPCREATESTRUCT lpCreateStruct)
+{
+    if (CFrameWnd::OnCreate(lpCreateStruct) == -1)
+        return -1;
+        
+#if 0
+    // no toolbar!
+    if (!m_wndToolBar.CreateEx(this, TBSTYLE_FLAT, WS_CHILD | WS_VISIBLE | CBRS_TOP
+                               | CBRS_GRIPPER | CBRS_TOOLTIPS | CBRS_FLYBY | CBRS_SIZE_DYNAMIC) ||
+        !m_wndToolBar.LoadToolBar(IDR_MAINFRAME))
+        {
+            TRACE0("Failed to create toolbar\n");
+            return -1;      // fail to create
+        }
+#endif
+
+    if (!m_wndStatusBar.Create(this) ||
+        !m_wndStatusBar.SetIndicators(indicators,
+                                      sizeof(indicators)/sizeof(UINT)))
+        {
+            TRACE0("Failed to create status bar\n");
+            return -1;      // fail to create
+        }
+
+#if 0
+    // TODO: Delete these three lines if you don't want the toolbar to
+    //  be dockable
+    m_wndToolBar.EnableDocking(CBRS_ALIGN_ANY);
+    EnableDocking(CBRS_ALIGN_ANY);
+    DockControlBar(&m_wndToolBar);
+#endif
+
+    return 0;
+}
+
+BOOL CMainFrame::PreCreateWindow(CREATESTRUCT& cs)
+{
+    if( !CFrameWnd::PreCreateWindow(cs) )
+        return FALSE;
+    // TODO: Modify the Window class or styles here by modifying
+    //  the CREATESTRUCT cs
+
+    return TRUE;
+}
+
+/////////////////////////////////////////////////////////////////////////////
+// CMainFrame diagnostics
+
+#ifdef _DEBUG
+void CMainFrame::AssertValid() const
+{
+    CFrameWnd::AssertValid();
+}
+
+void CMainFrame::Dump(CDumpContext& dc) const
+{
+    CFrameWnd::Dump(dc);
+}
+
+#endif //_DEBUG
+
+/////////////////////////////////////////////////////////////////////////////
+// CMainFrame message handlers
+
+
+void CMainFrame::SetStatusBarText(int /*pane*/, CString txt)
+{
+    m_wndStatusBar.SetPaneText(0, txt);
+}
diff --git a/tools/DRgui/MainFrm.h b/tools/DRgui/MainFrm.h
new file mode 100644
index 000000000..57715259e
--- /dev/null
+++ b/tools/DRgui/MainFrm.h
@@ -0,0 +1,94 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// MainFrm.h : interface of the CMainFrame class
+//
+/////////////////////////////////////////////////////////////////////////////
+
+#if !defined(AFX_MAINFRM_H__0047A767_863E_41D0_A4F7_82BD7928C23B__INCLUDED_)
+#define AFX_MAINFRM_H__0047A767_863E_41D0_A4F7_82BD7928C23B__INCLUDED_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif // _MSC_VER > 1000
+
+class CMainFrame : public CFrameWnd
+{
+        
+protected: // create from serialization only
+    CMainFrame();
+    DECLARE_DYNCREATE(CMainFrame)
+
+    // Attributes
+public:
+
+    // Operations
+public:
+
+    // Overrides
+    // ClassWizard generated virtual function overrides
+    //{{AFX_VIRTUAL(CMainFrame)
+    virtual BOOL PreCreateWindow(CREATESTRUCT& cs);
+    //}}AFX_VIRTUAL
+
+    // Implementation
+public:
+    CDialogBar m_wndDialogBar;
+    void SetStatusBarText(int pane, CString txt);
+    virtual ~CMainFrame();
+#ifdef _DEBUG
+    virtual void AssertValid() const;
+    virtual void Dump(CDumpContext& dc) const;
+#endif
+
+protected:  // control bar embedded members
+    CStatusBar  m_wndStatusBar;
+    CToolBar    m_wndToolBar;
+
+    // Generated message map functions
+protected:
+    //{{AFX_MSG(CMainFrame)
+    afx_msg int OnCreate(LPCREATESTRUCT lpCreateStruct);
+    //}}AFX_MSG
+    DECLARE_MESSAGE_MAP()
+};
+
+/////////////////////////////////////////////////////////////////////////////
+
+//{{AFX_INSERT_LOCATION}}
+// Microsoft Visual C++ will insert additional declarations immediately before the previous line.
+
+#endif // !defined(AFX_MAINFRM_H__0047A767_863E_41D0_A4F7_82BD7928C23B__INCLUDED_)
diff --git a/tools/DRgui/Makefile b/tools/DRgui/Makefile
new file mode 100644
index 000000000..bee0af776
--- /dev/null
+++ b/tools/DRgui/Makefile
@@ -0,0 +1,141 @@
+# **********************************************************
+# Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+# **********************************************************/
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+ifndef ARCH
+# default is x86 instead of x64
+  ARCH := x86
+endif
+
+ifndef DYNAMORIO_MAKE
+  DYNAMORIO_MAKE := ../../make
+endif
+
+# PR 232385: we need DRgui.exe to run on NT, which means we cannot
+# build using VC71 or VC80, as their MFC versions add dependencies on
+# libraries like shlwapi.dll and ole*.dll that are not present on NT.
+# Unfortunately with VC60 we can't make a debug build as libutil
+# uses VC71 and we can't incorporate the policy_staticd pdbs.
+USE_VC60 := 1
+
+include $(DYNAMORIO_MAKE)/compiler.mk
+
+# use DYNAMORIO_HOME if possible
+ifndef DYNAMORIO_HOME
+  DYNAMORIO_HOME := ../..
+endif
+WIN_DYNAMORIO_HOME := $(shell $(CYGPATH) -ma ${DYNAMORIO_HOME})
+
+ifndef DYNAMORIO_LIBUTIL
+  DYNAMORIO_LIBUTIL := ../../libutil
+endif
+WIN_DYNAMORIO_LIBUTIL := $(shell $(CYGPATH) -ma ${DYNAMORIO_LIBUTIL})
+
+CORE_DIRNAME := core
+
+# VisualStudio commandline options:
+# cl /Od /I "%DYNAMORIO_HOME%\exports\x86_win32_dbg" /D "WIN32" /D "_DEBUG" /D "_WINDOWS" /D "UNICODE" /D "NOT_DYNAMORIO_CORE" /D "_VC80_UPGRADE=0x0600" /D "_UNICODE" /Gm /EHsc /RTC1 /MTd /Yu"stdafx.h" /Fp".\Debug_Unicode/DynamoRIO.pch" /Fo".\Debug_Unicode/" /Fd".\Debug_Unicode/" /FR".\Debug_Unicode\\" /W3 /nologo /c /ZI /TP /errorReport:prompt
+# link /OUT:".\Debug_Unicode/DynamoRIO.exe" /INCREMENTAL /NOLOGO /MANIFEST /MANIFESTFILE:".\Debug_Unicode\DynamoRIO.exe.intermediate.manifest" /DELAYLOAD:"OleAcc.dll" /DEBUG /PDB:".\Debug_Unicode/DynamoRIO.pdb" /SUBSYSTEM:WINDOWS /ENTRY:"wWinMainCRTStartup" /MACHINE:X86 /ERRORREPORT:PROMPT  DelayImp.lib
+
+DEFINES := /D"WIN32" /D"WINDOWS" /D"_DEBUG" /D"_WINDOWS" /D"UNICODE" /D"_UNICODE" \
+	   /D"NOT_DYNAMORIO_CORE" /D"_VC80_UPGRADE=0x0600"
+
+DEMO := 1
+
+ifeq ($(DEMO), 1)
+  DEFINES += /D"DYNAMORIO_DEMO"
+endif
+
+ifeq ($(ARCH), x64)
+  DEFINES += /D"X64"
+endif
+
+INCLUDES := /I"$(WIN_DYNAMORIO_HOME)/$(CORE_DIRNAME)/lib" \
+	    /I"$(WIN_DYNAMORIO_HOME)/$(CORE_DIRNAME)" \
+	    /I"$(WIN_DYNAMORIO_LIBUTIL)" \
+	    $(MSINCLUDE)
+
+WARN := /W4 /WX
+
+CFLAGS += $(WARN) $(DEFINES) $(INCLUDES)
+
+# FIXME: sort out the reasons why I need these nodefaults
+ifeq ($(USE_VC60),1)
+# still get warning about libcmt but I do need it
+NODEF := /NODEFAULTLIB:libc.lib
+else
+NODEF := /NODEFAULTLIB:libcmt.lib
+endif
+
+LDFLAGS := $(MSLIB) advapi32.lib $(NODEF) \
+	   /SUBSYSTEM:WINDOWS /ENTRY:"wWinMainCRTStartup" \
+	   /MACHINE:X86
+
+ifdef DEBUG
+  CFLAGS += /Zi /Od
+  LDFLAGS += /debug
+  POLICY_LIB := $(DYNAMORIO_LIBUTIL)/policy_staticd.lib
+else
+  CFLAGS += /O2
+  POLICY_LIB := $(DYNAMORIO_LIBUTIL)/policy_static.lib
+endif
+
+WIN_POLICY_LIB := "$(shell $(CYGPATH) -ma $(POLICY_LIB))"
+
+ifeq ($(DEMO), 1)
+  BUILDDIR := build_dbg_demo
+else
+  BUILDDIR := build_dbg_internal
+endif
+
+EXEBASE := DRgui
+HEADERS := $(wildcard *.h)
+SRCS := $(wildcard *.cpp)
+OBJS := $(patsubst %.cpp,$(BUILDDIR)/%.obj,$(SRCS)) 
+RES_SRCS := $(wildcard *.rc)
+RES_OBJS := $(patsubst %.rc,$(BUILDDIR)/%.res,$(RES_SRCS)) 
+
+.PHONY: clean
+
+all: $(BUILDDIR) $(BUILDDIR)/$(EXEBASE).exe
+
+$(BUILDDIR): 
+	@(if ! test -d $(BUILDDIR); then $(MKDIR) -p $(BUILDDIR); fi)
+
+$(BUILDDIR)/$(EXEBASE).exe: $(OBJS) $(RES_OBJS) $(POLICY_LIB)
+	$(LD) $(LDFLAGS) $(OBJS) $(RES_OBJS) $(WIN_POLICY_LIB) /out:$@ /pdb:$(BUILDDIR)/$(EXEBASE).pdb
+
+$(BUILDDIR)/%.obj: %.cpp $(HEADERS)
+	$(CC) $(CFLAGS) /c $< /Fo$@ /Fd$(subst .obj,.pdb,$@)
+
+$(BUILDDIR)/%.res: %.rc
+	$(RC) /r /fo $@ $(INCLUDES) $<
+
+clean: 
+	$(RM) -r $(BUILDDIR)
diff --git a/tools/DRgui/OptionsDlg.cpp b/tools/DRgui/OptionsDlg.cpp
new file mode 100644
index 000000000..4f1cf8d8e
--- /dev/null
+++ b/tools/DRgui/OptionsDlg.cpp
@@ -0,0 +1,982 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// OptionsDlg.cpp : implementation file
+//
+
+#ifndef DYNAMORIO_DEMO /* around whole file */
+
+#include "stdafx.h"
+#include "DynamoRIO.h"
+#include "OptionsDlg.h"
+#include "LoggingDlg.h"
+#include <assert.h>
+
+#ifdef _DEBUG
+#define new DEBUG_NEW
+#undef THIS_FILE
+static char THIS_FILE[] = __FILE__;
+#endif
+
+enum {
+    HOT_THRESHOLD = 0,
+    INSTRLIBNAME,
+    CACHE_BB_MAX,
+    CACHE_TRACE_MAX,
+    LOGLEVEL,
+    LOGMASK,
+    PROF_COUNTS,
+    PROF_PCS,
+    NOASYNCH,
+    NOLINK,
+    NULLCALLS,
+    STATS,
+
+    NOTIFY,
+    TRACEDUMP_TEXT,
+
+    TRACEDUMP_BINARY,
+    TRACEDUMP_ORIGINS,
+};
+static const TCHAR * names[] = {
+    _T("-hot_threshold"),
+    _T("-client_lib"),
+    _T("-cache_bb_max"),
+    _T("-cache_trace_max"),
+    _T("-loglevel"),
+    _T("-logmask"),
+    _T("-prof_counts"),
+    _T("-prof_pcs"),
+    _T("-noasynch"),
+    _T("-nolink"),
+    _T("-nullcalls"),
+    _T("-stats"),
+    _T("-notify"),
+
+    _T("-tracedump_text"),
+    _T("-tracedump_binary"),
+
+    _T("-tracedump_origins"),
+};
+// numeric values are always positive!  (no leading - assumed)
+enum value_type {
+    NOVALUE = 0,
+    NUM_DECIMAL,
+    NUM_HEX,
+    STRING
+};
+static const value_type hasvalue[] = {
+    NUM_DECIMAL,
+    STRING,
+    NUM_DECIMAL,
+    NUM_DECIMAL,
+    NUM_DECIMAL, 
+    NUM_HEX,  // logmask
+    NOVALUE,
+    NOVALUE,
+    NOVALUE, // noasynch
+    NOVALUE,
+    NOVALUE,
+    NOVALUE, // stats
+    NOVALUE, // notify
+    NOVALUE, // tracedump_text
+    NOVALUE,
+
+    NOVALUE,
+
+};
+static const int checkboxes[] = {
+    IDC_HOT_THRESHOLD,
+    IDC_INSTRLIBNAME,
+    IDC_CACHE_BB_MAX,
+    IDC_CACHE_TRACE_MAX,
+    IDC_LOGLEVEL,
+    IDC_LOGMASK,
+    IDC_PROF_COUNTS,
+    IDC_PROF_PCS,
+    IDC_NOASYNCH,
+    IDC_NOLINK,
+    IDC_NULLCALLS,
+    IDC_STATS,
+    IDC_NOTIFY,
+
+    IDC_TRACEDUMP_TEXT,
+    IDC_TRACEDUMP_BINARY,
+
+    IDC_TRACEDUMP_ORIGINS,
+};
+static const BOOL ok_with_release[] = {
+    TRUE,
+    TRUE,
+    TRUE,
+    TRUE,
+    FALSE, // loglevel
+    FALSE, // logmask
+    TRUE,
+    TRUE,
+    TRUE,
+    TRUE,
+    TRUE,
+    FALSE, // stats
+    FALSE, // notify
+
+    TRUE,
+    TRUE,
+
+    TRUE,
+};
+static const int num_options = sizeof(checkboxes) / sizeof(int);
+static CString * values[num_options];
+
+/////////////////////////////////////////////////////////////////////////////
+// COptionsDlg dialog
+
+
+COptionsDlg::COptionsDlg(CWnd* pParent /*=NULL*/)
+    : CDialog(COptionsDlg::IDD, pParent)
+{
+    //{{AFX_DATA_INIT(COptionsDlg)
+    m_opstring = _T("");
+    m_HotThreshold = _T("50");
+    m_InstrLibName = _T("");
+    m_CacheBBMax = _T("0");
+    m_CacheTraceMax = _T("0");
+    m_LogLevel = _T("0");
+    m_LogMask = _T("0x3FFF");
+    //}}AFX_DATA_INIT
+    values[HOT_THRESHOLD] = &m_HotThreshold;
+    values[INSTRLIBNAME] = &m_InstrLibName;
+    values[CACHE_BB_MAX] = &m_CacheBBMax;
+    values[CACHE_TRACE_MAX] = &m_CacheTraceMax;
+    values[LOGLEVEL] = &m_LogLevel;
+    values[LOGMASK] = &m_LogMask;
+
+    // rest of initialization goes in OnInitDialog, once controls are built
+}
+
+void COptionsDlg::DoDataExchange(CDataExchange* pDX)
+{
+    CDialog::DoDataExchange(pDX);
+    //{{AFX_DATA_MAP(COptionsDlg)
+    DDX_Control(pDX, IDOK, m_OKButton);
+    DDX_Text(pDX, IDC_OPTIONS_EDIT, m_opstring);
+    DDX_Text(pDX, IDC_EDIT_HOT_THRESHOLD, m_HotThreshold);
+    DDX_Text(pDX, IDC_EDIT_INSTRLIBNAME, m_InstrLibName);
+    DDX_Text(pDX, IDC_EDIT_CACHE_BB_MAX, m_CacheBBMax);
+    DDX_Text(pDX, IDC_EDIT_CACHE_TRACE_MAX, m_CacheTraceMax);
+    DDX_Text(pDX, IDC_EDIT_LOGLEVEL, m_LogLevel);
+    DDX_Text(pDX, IDC_EDIT_LOGMASK, m_LogMask);
+    //}}AFX_DATA_MAP
+}
+
+
+BEGIN_MESSAGE_MAP(COptionsDlg, CDialog)
+    //{{AFX_MSG_MAP(COptionsDlg)
+    ON_BN_CLICKED(IDC_HOT_THRESHOLD, OnHotThreshold)
+    ON_BN_CLICKED(IDC_BROWSE_INSTRLIBNAME, OnBrowseInstrlibname)
+    ON_BN_CLICKED(IDC_INSTRLIBNAME, OnInstrlibname)
+    ON_BN_CLICKED(IDC_LOGGING_BUTTON, OnLoggingButton)
+    ON_EN_CHANGE(IDC_OPTIONS_EDIT, OnChangeOptionsEdit)
+    ON_BN_CLICKED(IDC_CACHE_BB_MAX, OnCacheBBMax)
+    ON_BN_CLICKED(IDC_CACHE_TRACE_MAX, OnCacheTraceMax)
+    ON_BN_CLICKED(IDC_LOGLEVEL, OnLoglevel)
+    ON_BN_CLICKED(IDC_LOGMASK, OnLogmask)
+    ON_BN_CLICKED(IDC_PROF_PCS, OnProfPcs)
+    ON_BN_CLICKED(IDC_PROF_COUNTS, OnProfCounts)
+    ON_BN_CLICKED(IDC_STATS, OnStats)
+    ON_BN_CLICKED(IDC_NOTIFY, OnNotify)
+
+    ON_BN_CLICKED(IDC_NULLCALLS, OnNullcalls)
+    ON_BN_CLICKED(IDC_NOLINK, OnNolink)
+    ON_BN_CLICKED(IDC_NOASYNCH, OnNoasynch)
+    ON_BN_CLICKED(IDC_TRACEDUMP_TEXT, OnTraceDumpText)
+    ON_BN_CLICKED(IDC_TRACEDUMP_BINARY, OnTraceDumpBinary)
+
+    ON_BN_CLICKED(IDC_TRACEDUMP_ORIGINS, OnTraceDumpOrigins)
+
+    ON_EN_CHANGE(IDC_EDIT_INSTRLIBNAME, OnChangeEditInstrlibname)
+    ON_BN_CLICKED(IDC_SET_PERMANENT, OnSetPermanent)
+    ON_EN_CHANGE(IDC_EDIT_CACHE_BB_MAX, OnChangeEditCacheBbMax)
+    ON_EN_CHANGE(IDC_EDIT_CACHE_TRACE_MAX, OnChangeEditCacheTraceMax)
+    ON_EN_CHANGE(IDC_EDIT_HOT_THRESHOLD, OnChangeEditHotThreshold)
+    //}}AFX_MSG_MAP
+    END_MESSAGE_MAP()
+
+    /////////////////////////////////////////////////////////////////////////////
+    // COptionsDlg message handlers
+
+    BOOL COptionsDlg::OnInitDialog()
+{
+    CDialog::OnInitDialog();
+
+    // get current options string
+    TCHAR path[MAX_PATH];
+    int len = GetEnvironmentVariable(_T("DYNAMORIO_OPTIONS"), path, MAX_PATH);
+    if (len > 0)
+        m_opstring = path; // makes new storage, right?
+    UpdateData(FALSE); // FALSE means set controls
+
+    // set controls based on opstring
+    if (!CheckOpstring()) {
+        MessageBox(_T("Invalid DYNAMORIO_OPTIONS string!\nThis dialog ")
+                   _T("may not work properly with it."), _T("Error"), 
+                   MB_OK | MYMBFLAGS);
+    }
+
+    // now disable appropriate options
+
+    // always disabled, only there so can see all options
+    DisableCheckbox(IDC_PROF_PCS);
+    DLL_TYPE dll_type = CDynamoRIOApp::GetDllType();
+    if (dll_type != DLL_PROFILE) {
+        DisableCheckbox(IDC_PROF_COUNTS);
+    }
+    if (dll_type == DLL_RELEASE) {
+        int i;
+        for (i=0; i<num_options; i++) {
+            if (!ok_with_release[i]) {
+                DisableCheckbox(checkboxes[i]);
+            }
+        }
+    }
+
+    UpdateData(FALSE); // FALSE means set controls
+
+    return TRUE;
+}
+
+void COptionsDlg::OnOK() 
+{
+    UpdateData(TRUE); // TRUE means read from controls
+        
+    // set options string
+    int res = SetEnvironmentVariable(_T("DYNAMORIO_OPTIONS"), m_opstring);
+    assert(res != 0);
+
+    CDialog::OnOK();
+}
+
+void COptionsDlg::OnChangeOptionsEdit() 
+{
+    if (CheckOpstring()) {
+        m_OKButton.EnableWindow(TRUE);
+    } else {
+        m_OKButton.EnableWindow(FALSE);
+    }
+}
+
+static BOOL
+is_quote(TCHAR ch)
+{
+    return (ch == _T('\'') || ch == _T('\"') || ch == _T('`'));
+}
+
+static BOOL
+is_whitespace(TCHAR ch)
+{
+    return (ch == _T(' ') || ch == _T('\t') || 
+            ch == _T('\n') || ch == _T('\r'));
+}
+
+// returns the space delimited or quote-delimited word
+// starting at strpos in the string str 
+// maximum word length: MAX_PATH*2
+static BOOL
+getword(TCHAR *str, TCHAR **strpos, TCHAR *result)
+{
+    result[0] = _T('\0');
+    int i = 0;
+    TCHAR *pos = *strpos;
+    TCHAR quote = _T('\0');
+
+    if (pos < str || pos >= str + _tcslen(str))
+        return FALSE;
+
+    if (*pos == _T('\0'))
+        return FALSE; /* no more words */
+
+    /* eat leading spaces */
+    while (is_whitespace(*pos)) {
+        pos++;
+    }
+
+    /* extract the word */
+    if (is_quote(*pos)) {
+        quote = *pos;
+        pos++; /* don't include surrounding quotes in word */
+    }
+    while (*pos) {
+        if (quote != _T('\0')) {
+            /* if quoted, only end on matching quote */
+            if (*pos == quote) {
+                pos++; /* include the quote */
+                break;
+            }
+        } else {
+            /* if not quoted, end on whitespace */
+            if (is_whitespace(*pos))
+                break;
+        }
+        result[i++] = *pos;
+        pos++;
+        assert(i < MAX_PATH*2);
+    }
+    if (i == 0)
+        return NULL; /* no more words */
+
+    result[i] = _T('\0');
+    *strpos = pos;
+
+    return TRUE;
+}
+
+// returns the beginning and ending positions of the
+// parameter names[id] in the string str, including its value,
+// if it has one (based on has_value[id]).
+// beginning position includes leading whitespace!
+// start is inclusive, end is exclusive
+static BOOL
+find_param(TCHAR *str, int id, int &start, int &end)
+{
+    TCHAR *pos = str;
+    TCHAR *prev_pos = str;
+    TCHAR word[MAX_PATH*2];
+    while (true) {
+        if (!getword(str, &pos, word))
+            break;
+        if (_tcscmp(word, names[id]) == 0) {
+            if (hasvalue[id] != NOVALUE) {
+                TCHAR *last_pos = pos;
+                if (!getword(str, &pos, word)) {
+                    // don't return FALSE, just return empty value
+                } else if ((hasvalue[id] == NUM_DECIMAL || hasvalue[id] == NUM_HEX) &&
+                           word[0] == '-') {
+                    // assumption: no numeric value begins with - (no negative numbers)
+                    pos = last_pos;
+                }
+            }
+            start = (int) (prev_pos - str);
+            end = (int) (pos - str);
+            return TRUE;
+        }
+        prev_pos = pos;
+    }
+    return FALSE;
+}
+
+static void
+expand_ws_quotes(CString str, int &start, int &end)
+{
+    // start is inclusive, end is exclusive
+
+    // remove preceding quote, if present
+    while (start > 0 && is_quote(str.GetAt(start-1))) {
+        start--;
+    }
+    // now remove preceding spaces
+    while (start > 0 && is_whitespace(str.GetAt(start-1))) {
+        start--;
+    }
+    // remove following quote, if present
+    while (end < str.GetLength() && is_quote(str.GetAt(end))) {
+        end++;
+    }
+    // remove following spaces beyond a single space
+    while (end + 1 < str.GetLength() &&
+           (is_whitespace(str.GetAt(end)) || is_quote(str.GetAt(end))) &&
+           (is_whitespace(str.GetAt(end+1)) || is_quote(str.GetAt(end+1)))) {
+        end++;
+    }
+    // remove following space if at start
+    if (start == 0 && end < str.GetLength() &&
+        (is_whitespace(str.GetAt(end)) || is_quote(str.GetAt(end)))) {
+        end++;
+    }
+}
+
+// FIXME: share parsing code with non-static routines?
+/*static*/ BOOL COptionsDlg::CheckOptionsVersusDllType(DLL_TYPE dll_type)
+{
+    TCHAR msg[MAX_PATH];
+    // this is independent of dialog box -- so grab string separately
+    int len = GetEnvironmentVariable(_T("DYNAMORIO_OPTIONS"), msg, MAX_PATH);
+    if (len <= 0)
+        msg[0] = _T('\0');
+    CString opstring = msg;
+
+    // now parse string
+    int i;
+    TCHAR param[MAX_PATH];
+    TCHAR value[MAX_PATH];
+    TCHAR *pos = opstring.GetBuffer(0);
+    TCHAR *prev_pos = pos;
+    while (TRUE) {
+        if (!getword(opstring.GetBuffer(0), &pos, param))
+            break;
+        for (i=0; i<num_options; i++) {
+            if (_tcscmp(param, names[i]) == 0) {
+                if (hasvalue[i] != NOVALUE) {
+                    // grab next space-delimited word
+                    getword(opstring.GetBuffer(0), &pos, value);
+                } else
+                    value[0] = _T('\0');
+                if ((dll_type == DLL_RELEASE && !ok_with_release[i]) ||
+                    (dll_type != DLL_PROFILE && i == PROF_COUNTS)) {
+                    _stprintf(msg, _T("Option \"%s%s%s\" is incompatible with the selected library.\n")
+                              _T("Remove it?  Incompatible options cause failure.\n"),
+                              param, (hasvalue[i] != NOVALUE) ? _T(" ") : _T(""), value);
+                    int res = ::MessageBox(NULL, msg, _T("Confirmation"), MB_YESNO | MYMBFLAGS);
+                    if (res == IDYES) {
+                        int start = (prev_pos - opstring.GetBuffer(0));
+                        int end = (pos - opstring.GetBuffer(0));
+                        expand_ws_quotes(opstring, start, end);
+                        opstring = opstring.Left(start) + 
+                            opstring.Right(opstring.GetLength() - end);
+                        // update pos
+                        pos = opstring.GetBuffer(0) + start;
+
+                        res = SetEnvironmentVariable(_T("DYNAMORIO_OPTIONS"), opstring);
+                        assert(res != 0);
+                    }
+                }
+            }
+        }
+        prev_pos = pos;
+    }
+    return TRUE;
+}
+
+// examines m_opstring, sets checkboxes and edit boxes from it, and returns
+// false if an error is found (doesn't finish reading string if error is found)
+BOOL COptionsDlg::CheckOpstring()
+{
+    UpdateData(TRUE); // TRUE means read from controls
+
+    // now parse string
+    TCHAR param[MAX_PATH];
+    TCHAR value[MAX_PATH];
+    TCHAR *pos = m_opstring.GetBuffer(0);
+    TCHAR *prev_pos;
+    BOOL valid = TRUE;
+    BOOL matched_any;
+    BOOL match[num_options];
+    int i;
+    for (i=0; i<num_options; i++)
+        match[i] = FALSE;
+    DLL_TYPE dll_type = CDynamoRIOApp::GetDllType();
+    while (TRUE) {
+        if (!getword(m_opstring.GetBuffer(0), &pos, param))
+            break;
+        matched_any = FALSE;
+        for (i=0; i<num_options; i++) {
+            if (_tcscmp(param, names[i]) == 0) {
+                CButton *button = (CButton *) GetDlgItem(checkboxes[i]);
+                assert(button != NULL);
+                if (button->GetCheck() == 0)
+                    button->SetCheck(1);
+                match[i] = TRUE;
+                matched_any = TRUE;
+                if (hasvalue[i] != NOVALUE) {
+                    prev_pos = pos;
+                    if (!getword(m_opstring.GetBuffer(0), &pos, value)) {
+                        match[i] = FALSE;
+                        valid = FALSE;
+                        *values[i] = _T("");
+                    } else {
+                        *values[i] = value;
+                        if (i == INSTRLIBNAME) {
+                            if (!CheckLibraryExists(value, FALSE)) {
+                                match[i] = FALSE;
+                                valid = FALSE;
+                            }
+                        }
+                        if (hasvalue[i] == NUM_DECIMAL || hasvalue[i] == NUM_HEX) {
+                            if (value[0] == _T('-')) {
+                                // value is missing, don't claim next param as value!
+                                // (for string (non-numeric) values, go ahead and do so)
+                                pos = prev_pos;
+                                *values[i] = _T("");
+                                match[i] = FALSE;
+                                valid = FALSE;
+                                break;
+                            }
+                            int j;
+                            int len = _tcslen(value);
+                            for (j=0; j<len; j++) {
+                                if ((hasvalue[i] == NUM_DECIMAL && !_istdigit(value[j])) ||
+                                    (hasvalue[i] == NUM_HEX && 
+                                     ((j == 0 && value[j] != _T('0')) ||
+                                      (j == 1 && value[j] != _T('x') && value[j] != _T('X')) ||
+                                      (j > 1 && !_istxdigit(value[j]))))) {
+                                    match[i] = FALSE;
+                                    valid = FALSE;
+                                    break;
+                                }
+                            }
+                        }
+                    }
+                } else {
+                    match[i] = TRUE;
+                    matched_any = TRUE;
+                }
+                if (match[i]) {
+                    // if made it this far, now check vs. dll type
+                    if ((dll_type == DLL_RELEASE && !ok_with_release[i]) ||
+                        (dll_type != DLL_PROFILE && i == PROF_COUNTS)) {
+                        // leave it checked
+                        valid = FALSE;
+                    }
+                }
+                break;
+            }
+        }
+        if (!matched_any)
+            valid = FALSE;
+    }
+    for (i=0; i<num_options; i++) {
+        if (!match[i]) {
+            CButton *button = (CButton *) GetDlgItem(checkboxes[i]);
+            assert(button != NULL);
+            if (button->GetCheck() > 0)
+                button->SetCheck(0);
+        }
+    }
+
+    UpdateData(FALSE); // FALSE means set controls
+    return valid;
+}
+
+// common code for all the checkboxes
+void COptionsDlg::CheckOption(int param)
+{
+    CButton *button = (CButton *) GetDlgItem(checkboxes[param]);
+    assert(button != NULL);
+    BOOL has_value = (hasvalue[param] != NOVALUE);
+    CString *value = values[param];
+    assert(!has_value || value != NULL);
+
+    UpdateData(TRUE); // TRUE means read from controls
+    int start, end;
+    BOOL found = find_param(m_opstring.GetBuffer(0), param, start, end);
+    int checked = button->GetCheck();
+    if (checked) {
+        if (has_value && value->IsEmpty()) {
+            button->SetCheck(0);
+            return;
+        }
+        if (found) {
+            // occurs if things get out of sync...try to recover
+            RemoveOption(param);
+            OnChangeOptionsEdit();
+        }
+        if (!m_opstring.IsEmpty() && 
+            m_opstring.GetAt(m_opstring.GetLength()-1) != _T(' ')) {
+            m_opstring += CString(_T(" "));
+        }
+        m_opstring += CString(names[param]);
+        if (has_value) {
+            // if has spaces, put quotes around
+            CString val = *value;
+            if (val.Find(_T(' '), 0) != -1)
+                val.Format(_T("\"%s\""), *value);
+            m_opstring += CString(_T(" ")) + val;
+        }
+    } else {
+        assert(found);
+        RemoveOption(param);
+    }
+    UpdateData(FALSE); // FALSE means set controls
+}
+
+void COptionsDlg::RemoveOption(int param)
+{
+    UpdateData(TRUE); // TRUE means read from controls
+    int start, end;
+    BOOL found = find_param(m_opstring.GetBuffer(0), param, start, end);
+    if (found) {
+        expand_ws_quotes(m_opstring, start, end);
+        m_opstring = m_opstring.Left(start) + 
+            m_opstring.Right(m_opstring.GetLength() - end);
+        UpdateData(FALSE); // FALSE means set controls
+    }
+}
+
+BOOL COptionsDlg::UpdateValue(int param)
+{
+    if (hasvalue[param] == NOVALUE)
+        return FALSE;
+
+    UpdateData(TRUE); // TRUE means read from controls
+
+    CString *value = values[param];
+    // if has spaces, put quotes around
+    CString newval = *value;
+    if (newval.Find(_T(' '), 0) != -1)
+        newval.Format(_T("\"%s\""), *value);
+
+    int start, end;
+    BOOL found = find_param(m_opstring.GetBuffer(0), param, start, end);
+    if (!found)
+        return FALSE;
+    TCHAR *buf = m_opstring.GetBuffer(0);
+    TCHAR *pos = buf + start;
+    TCHAR val[MAX_PATH];
+    getword(buf, &pos, val); // param
+    TCHAR *val_pos = pos;
+    getword(buf, &pos, val); // value
+    int val_start = (val_pos - buf);
+    expand_ws_quotes(m_opstring, val_start, end);
+    m_opstring = m_opstring.Left(val_start) + CString(_T(" ")) + newval +
+        m_opstring.Right(m_opstring.GetLength() - end);
+    UpdateData(FALSE); // FALSE means set controls
+    return TRUE;
+}
+
+static TCHAR szFilter[] = _T("Dynamic-Linked Libraries (*.dll)|*.dll|All Files (*.*)|*.*||");
+
+void COptionsDlg::OnBrowseInstrlibname() 
+{
+    CFileDialog fileDlg(TRUE, _T(".dll"), NULL, 
+                        // hide the "open as read-only" checkbox
+                        OFN_FILEMUSTEXIST | OFN_PATHMUSTEXIST | OFN_HIDEREADONLY,
+                        szFilter);
+    int res = fileDlg.DoModal();
+    if (res == IDCANCEL)
+        return;
+    CButton *button = (CButton *) GetDlgItem(checkboxes[INSTRLIBNAME]);
+    assert(button != NULL);
+    if (button->GetCheck() > 0 && _tcscmp(m_InstrLibName, fileDlg.GetPathName()) != 0) {
+        m_InstrLibName = fileDlg.GetPathName();
+        UpdateData(FALSE); // FALSE means set controls
+        RemoveOption(INSTRLIBNAME);
+        CheckOption(INSTRLIBNAME);
+    } else {
+        m_InstrLibName = fileDlg.GetPathName();
+        UpdateData(FALSE); // FALSE means set controls
+    }
+}
+
+void COptionsDlg::OnInstrlibname() 
+{
+    UpdateData(TRUE); // TRUE means read from controls
+    if (CheckLibraryExists(m_InstrLibName.GetBuffer(0), TRUE))
+        CheckOption(INSTRLIBNAME);
+    else {
+        CButton *button = (CButton *) GetDlgItem(IDC_INSTRLIBNAME);
+        button->SetCheck(0);
+    }
+}
+
+void COptionsDlg::OnChangeEditInstrlibname() 
+{
+    if (UpdateValue(INSTRLIBNAME)) {
+        OnChangeOptionsEdit();
+    }
+}
+
+BOOL COptionsDlg::CheckLibraryExists(TCHAR *libname, BOOL notify)
+{
+    // make sure file exists
+    TCHAR msg[MAX_PATH*2];
+    CFile check;
+    if (!check.Open(libname, CFile::modeRead|CFile::shareDenyNone)) {
+        if (notify) {
+            _stprintf(msg, _T("Library %s does not exist"), libname);
+            MessageBox(msg, _T("Error"), MB_OK | MYMBFLAGS);
+        }
+        return FALSE;
+    }
+    check.Close();
+    return TRUE;
+}
+
+void COptionsDlg::OnLoggingButton() 
+{
+    UpdateData(TRUE); // get values from controls
+    int level = _ttoi(m_LogLevel);
+    if (level < 0)
+        level = 0;
+    if (level > 4)
+        level = 4;
+    int mask;
+    int res = _stscanf(m_LogMask, _T("%X"), &mask);
+    if (res <= 0 || res == EOF)
+        mask = 0;
+    CLoggingDlg dlg(level, mask);
+    res = dlg.DoModal();
+    if (res == IDCANCEL)
+        return;
+    m_LogLevel.Format(_T("%d"), dlg.GetLevel());
+    m_LogMask.Format(_T("0x%04X"), dlg.GetMask());
+    UpdateData(FALSE); // write to controls
+
+#if 1
+    UpdateValue(LOGLEVEL);
+    UpdateValue(LOGMASK);
+#else
+    // FIXME: change in place...for now we just remove and re-add
+    CButton *button = (CButton *) GetDlgItem(checkboxes[LOGLEVEL]);
+    assert(button != NULL);
+    if (button->GetCheck() > 0) {
+        RemoveOption(LOGLEVEL);
+        CheckOption(LOGLEVEL);
+    }
+    button = (CButton *) GetDlgItem(checkboxes[LOGMASK]);
+    assert(button != NULL);
+    if (button->GetCheck() > 0) {
+        RemoveOption(LOGMASK);
+        CheckOption(LOGMASK);
+    }
+#endif
+}
+
+void COptionsDlg::DisableCheckbox(int id)
+{
+    CButton *button = (CButton *) GetDlgItem(id);
+    assert(button != NULL);
+    button->EnableWindow(FALSE);
+}
+
+void COptionsDlg::OnHotThreshold() { CheckOption(HOT_THRESHOLD); }
+void COptionsDlg::OnCacheBBMax() { CheckOption(CACHE_BB_MAX); }
+void COptionsDlg::OnCacheTraceMax() { CheckOption(CACHE_TRACE_MAX); }
+void COptionsDlg::OnLoglevel() { CheckOption(LOGLEVEL); }
+void COptionsDlg::OnLogmask() { CheckOption(LOGMASK); } 
+void COptionsDlg::OnProfPcs() { CheckOption(PROF_PCS); } 
+void COptionsDlg::OnStats() { CheckOption(STATS); } 
+void COptionsDlg::OnNullcalls() { CheckOption(NULLCALLS); } 
+void COptionsDlg::OnNolink() { CheckOption(NOLINK); } 
+void COptionsDlg::OnNoasynch() { CheckOption(NOASYNCH); } 
+void COptionsDlg::OnTraceDumpOrigins() { CheckOption(TRACEDUMP_ORIGINS); } 
+
+
+
+// remind to pick trace dump too
+
+void COptionsDlg::OnProfCounts() { 
+
+    CButton *button = (CButton *) GetDlgItem(IDC_NOTIFY);
+
+    assert(button != NULL);
+
+    if (button->GetCheck() == 1) {
+
+        CButton *text = (CButton *) GetDlgItem(IDC_TRACEDUMP_TEXT);
+
+        assert(text != NULL);
+
+        CButton *binary = (CButton *) GetDlgItem(IDC_TRACEDUMP_BINARY);
+
+        assert(binary != NULL);
+
+        if (text->GetCheck() == 0 && binary->GetCheck() == 0) {
+
+            ::MessageBox(NULL, _T("Count profiling results are only visible in a trace dump.\n")
+
+                         _T("Don't forget to select either a text or binary trace dump."),
+
+                         _T("Reminder"), MB_OK | MYMBFLAGS);
+
+        }
+
+    }
+
+    CheckOption(PROF_COUNTS); 
+
+} 
+
+
+
+// mutually exclusive
+void COptionsDlg::OnTraceDumpText() { 
+
+    CButton *text = (CButton *) GetDlgItem(IDC_TRACEDUMP_TEXT);
+
+    assert(text != NULL);
+
+    CButton *binary = (CButton *) GetDlgItem(IDC_TRACEDUMP_BINARY);
+
+    assert(binary != NULL);
+
+    if (text->GetCheck() == 1 && binary->GetCheck() == 1) {
+
+        ::MessageBox(NULL, _T("Trace dump must be either text or binary, not both"),
+
+                     _T("Mutually Exclusive"), MB_OK | MYMBFLAGS);
+
+        text->SetCheck(0);
+
+    } else {
+
+        CheckOption(TRACEDUMP_TEXT); 
+
+    }
+
+} 
+
+void COptionsDlg::OnTraceDumpBinary() { 
+
+    CButton *text = (CButton *) GetDlgItem(IDC_TRACEDUMP_TEXT);
+
+    assert(text != NULL);
+
+    CButton *binary = (CButton *) GetDlgItem(IDC_TRACEDUMP_BINARY);
+
+    assert(binary != NULL);
+
+    if (text->GetCheck() == 1 && binary->GetCheck() == 1) {
+
+        ::MessageBox(NULL, _T("Trace dump must be either text or binary, not both"),
+
+                     _T("Mutually Exclusive"), MB_OK | MYMBFLAGS);
+
+        binary->SetCheck(0);
+
+    } else {
+
+        CheckOption(TRACEDUMP_BINARY); 
+
+    }
+
+} 
+
+
+
+// ask for confirmation since stderr sticky issue
+
+// in fact, could not even let them set notify within GUI,
+
+// but would be nice if env var brought in from cmd line showed up
+
+void COptionsDlg::OnNotify() { 
+
+    TCHAR msg[MAX_PATH];
+
+    CButton *button = (CButton *) GetDlgItem(IDC_NOTIFY);
+
+    assert(button != NULL);
+
+    if (button->GetCheck() == 1) {
+
+        _stprintf(msg, _T("Printing to stderr can cause unexpected failures.\n")
+
+                  _T("Are you sure you want to set this option?\n"));
+
+        int res = ::MessageBox(NULL, msg, _T("Confirmation"), MB_YESNO | MYMBFLAGS);
+
+        if (res == IDYES) {
+
+            CheckOption(NOTIFY); 
+
+        } else {
+
+            button->SetCheck(0);
+
+        }
+
+    } else {
+
+        CheckOption(NOTIFY); 
+
+    }
+
+} 
+
+
+
+// set env var not just for this process but permanently for this user
+void COptionsDlg::OnSetPermanent() 
+{
+    // it takes a while to broadcast the "we've changed env vars" message,
+    // so set a wait cursor
+    HCURSOR prev_cursor = SetCursor(LoadCursor(0, IDC_WAIT));
+
+    TCHAR msg[MAX_PATH];
+    HKEY hk;
+    int res;
+        
+    // current user only
+    res = RegOpenKeyEx(HKEY_CURRENT_USER, _T("Environment"), 0, KEY_WRITE, &hk);
+    if (res != ERROR_SUCCESS) {
+        _stprintf(msg, _T("Error writing to HKEY_CURRENT_USER\\Environment"));
+        MessageBox(msg, _T("Error"), MB_OK | MYMBFLAGS);
+    }
+
+    UpdateData(TRUE); // get values from controls
+    TCHAR *val = m_opstring.GetBuffer(0);
+    res = RegSetValueEx(hk, _T("DYNAMORIO_OPTIONS"), 0, REG_SZ, 
+                        (LPBYTE) val, _tcslen(val)+1);
+    assert(res == ERROR_SUCCESS);
+
+    RegCloseKey(hk);
+
+    // tell system that we've changed environment (else won't take effect until
+    // user logs out and back in)
+    DWORD dwReturnValue;
+    // Code I copied this from used an ANSI string...I'm leaving
+    // it like that FIXME
+    SendMessageTimeout(HWND_BROADCAST, WM_SETTINGCHANGE, 0,
+                       (LPARAM) "Environment", SMTO_ABORTIFHUNG, 5000, &dwReturnValue);
+
+    SetCursor(prev_cursor);
+
+    // set local env var too, and avoid questions of ability to
+    // cancel the permanent operation
+    OnOK();
+}
+
+void COptionsDlg::OnChangeEditCacheBbMax() 
+{
+    if (UpdateValue(CACHE_BB_MAX)) {
+        OnChangeOptionsEdit();
+    }
+}
+
+void COptionsDlg::OnChangeEditCacheTraceMax() 
+{
+    if (UpdateValue(CACHE_TRACE_MAX)) {
+        OnChangeOptionsEdit();
+    }
+}
+
+void COptionsDlg::OnChangeEditHotThreshold() 
+{
+    if (UpdateValue(HOT_THRESHOLD)) {
+        OnChangeOptionsEdit();
+    }
+}
+
+#endif /* !DYNAMORIO_DEMO */ /* around whole file */
diff --git a/tools/DRgui/OptionsDlg.h b/tools/DRgui/OptionsDlg.h
new file mode 100644
index 000000000..0fe22b92d
--- /dev/null
+++ b/tools/DRgui/OptionsDlg.h
@@ -0,0 +1,121 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+#if !defined(AFX_OPTIONSDLG_H__0BDD6058_E38D_4CEA_805C_DE47B633EEF9__INCLUDED_)
+#define AFX_OPTIONSDLG_H__0BDD6058_E38D_4CEA_805C_DE47B633EEF9__INCLUDED_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif // _MSC_VER > 1000
+// OptionsDlg.h : header file
+//
+
+/////////////////////////////////////////////////////////////////////////////
+// COptionsDlg dialog
+
+class COptionsDlg : public CDialog
+{
+    // Construction
+public:
+    BOOL CheckLibraryExists(TCHAR *libname, BOOL notify);
+    static BOOL CheckOptionsVersusDllType(DLL_TYPE dll_type);
+    COptionsDlg(CWnd* pParent = NULL);   // standard constructor
+    BOOL OnInitDialog();
+
+    // Dialog Data
+    //{{AFX_DATA(COptionsDlg)
+    enum { IDD = IDD_OPTIONS };
+    CButton m_OKButton;
+    CString m_opstring;
+    CString m_HotThreshold;
+    CString m_InstrLibName;
+    CString m_CacheBBMax;
+    CString m_CacheTraceMax;
+    CString m_LogLevel;
+    CString m_LogMask;
+    //}}AFX_DATA
+
+
+    // Overrides
+    // ClassWizard generated virtual function overrides
+    //{{AFX_VIRTUAL(COptionsDlg)
+protected:
+    virtual void DoDataExchange(CDataExchange* pDX);    // DDX/DDV support
+    //}}AFX_VIRTUAL
+
+    // Implementation
+protected:
+    BOOL UpdateValue(int param);
+    void DisableCheckbox(int id);
+    BOOL CheckOpstring();
+    void RemoveOption(int param);
+    void CheckOption(int param);
+
+    // Generated message map functions
+    //{{AFX_MSG(COptionsDlg)
+    virtual void OnOK();
+    afx_msg void OnHotThreshold();
+    afx_msg void OnBrowseInstrlibname();
+    afx_msg void OnInstrlibname();
+    afx_msg void OnLoggingButton();
+    afx_msg void OnChangeOptionsEdit();
+    afx_msg void OnCacheBBMax();
+    afx_msg void OnCacheTraceMax();
+    afx_msg void OnLoglevel();
+    afx_msg void OnLogmask();
+    afx_msg void OnProfPcs();
+    afx_msg void OnProfCounts();
+    afx_msg void OnStats();
+    afx_msg void OnNotify();
+    afx_msg void OnNullcalls();
+    afx_msg void OnNolink();
+    afx_msg void OnNoasynch();
+    afx_msg void OnTraceDumpText();
+    afx_msg void OnTraceDumpBinary();
+    afx_msg void OnTraceDumpOrigins();
+    afx_msg void OnChangeEditInstrlibname();
+    afx_msg void OnSetPermanent();
+    afx_msg void OnChangeEditCacheBbMax();
+    afx_msg void OnChangeEditCacheTraceMax();
+    afx_msg void OnChangeEditHotThreshold();
+    //}}AFX_MSG
+    DECLARE_MESSAGE_MAP()
+};
+
+//{{AFX_INSERT_LOCATION}}
+// Microsoft Visual C++ will insert additional declarations immediately before the previous line.
+
+#endif // !defined(AFX_OPTIONSDLG_H__0BDD6058_E38D_4CEA_805C_DE47B633EEF9__INCLUDED_)
diff --git a/tools/DRgui/ShellInterface.cpp b/tools/DRgui/ShellInterface.cpp
new file mode 100644
index 000000000..86ad32975
--- /dev/null
+++ b/tools/DRgui/ShellInterface.cpp
@@ -0,0 +1,359 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// ShellInterface.cpp: implementation of the CShellInterface class.
+//
+//////////////////////////////////////////////////////////////////////
+
+#ifndef DYNAMORIO_DEMO /* around whole file */
+
+#include "stdafx.h"
+#include "DynamoRIO.h"
+#include "ShellInterface.h"
+
+#include <shlobj.h>
+#include <objbase.h>
+#include <shlwapi.h>
+#include <assert.h>
+
+
+#ifdef _DEBUG
+#undef THIS_FILE
+static char THIS_FILE[]=__FILE__;
+#define new DEBUG_NEW
+#endif
+
+//////////////////////////////////////////////////////////////////////
+// Construction/Destruction
+//////////////////////////////////////////////////////////////////////
+
+CShellInterface::CShellInterface()
+{
+
+}
+
+CShellInterface::~CShellInterface()
+{
+
+}
+
+/*static */void CShellInterface::Initialize()
+{
+    if (m_bInitialized)
+        return;
+    int res = CoInitialize(NULL);
+    assert(res == S_OK);
+    m_bInitialized = TRUE;
+}
+
+/*static */void CShellInterface::Uninitialize()
+{
+    if (!m_bInitialized)
+        return;
+    CoUninitialize();
+    m_bInitialized = FALSE;
+}
+
+/*static */BOOL CShellInterface::m_bInitialized = FALSE;
+
+/*static */BOOL CShellInterface::ResolveLinkFile(TCHAR *name, TCHAR *resolution, 
+                                                 TCHAR *arguments, TCHAR *working_dir, HWND hwnd)
+{
+    assert(m_bInitialized);
+    HRESULT hres; 
+    IShellLink* psl; 
+    WIN32_FIND_DATA wfd; 
+ 
+    *resolution = 0; // assume failure 
+ 
+    // Get a pointer to the IShellLink interface.
+    hres = CoCreateInstance (CLSID_ShellLink, NULL, CLSCTX_INPROC_SERVER,
+                             IID_IShellLink, (void **)&psl);
+    if (SUCCEEDED (hres)) {
+        IPersistFile *ppf;
+                
+        // Get a pointer to the IPersistFile interface.
+        hres = psl->QueryInterface (IID_IPersistFile, (void **)&ppf);
+                
+        if (SUCCEEDED (hres)) {
+#ifndef UNICODE
+            WORD wsz [MAX_PATH]; // buffer for Unicode string
+                        
+            // Ensure that the string consists of Unicode TCHARacters.
+            MultiByteToWideChar(CP_ACP, 0, name, -1, wsz, MAX_PATH);
+#else
+            TCHAR *wsz = name;
+#endif
+                        
+            // Load the shortcut.
+            hres = ppf->Load(wsz, STGM_READ);
+                        
+            if (SUCCEEDED(hres)) {
+                // Resolve the shortcut.
+                hres = psl->Resolve(hwnd, SLR_ANY_MATCH);
+                if (SUCCEEDED(hres)) {
+                    _tcscpy(resolution, name);
+                    // Get the path to the shortcut target.
+                    hres = psl->GetPath(resolution, MAX_PATH,
+                                        (WIN32_FIND_DATA *)&wfd, SLGP_SHORTPATH);
+                    if (! SUCCEEDED(hres))
+                        goto cleanup;
+                    hres = psl->GetArguments(arguments, MAX_PATH);
+                    if (! SUCCEEDED(hres))
+                        goto cleanup;
+                    hres = psl->GetWorkingDirectory(working_dir, MAX_PATH);
+                    if (! SUCCEEDED(hres))
+                        goto cleanup;
+                }
+            }
+        cleanup:
+            // Release the pointer to IPersistFile.
+            ppf->Release ();
+        }
+        // Release the pointer to IShellLink.
+        psl->Release ();
+    }
+    return (SUCCEEDED(hres) ? TRUE : FALSE);
+}
+
+
+
+#if 0
+// from "Shell Links" in MSDN library:
+/*
+The application-defined ResolveIt function in the following example
+resolves a shortcut. Its parameters include a window handle, a pointer
+to the path of the shortcut, and the address of a buffer that receives
+the new path to the object. The window handle identifies the parent
+window for any message boxes that the shell may need to display. For
+example, the shell can display a message box if the link is on unshared
+media, if network problems occur, if the user needs to insert a floppy
+disk, and so on.
+
+The ResolveIt function calls theCoCreateInstance function and assumes
+that theCoInitialize function has already been called. Note that
+ResolveIt needs to use theIPersistFile interface to store the link
+information. IPersistFile is implemented by the IShellLink object. The
+link information must be loaded before the path information is
+retrieved, which is shown later in the example. Failing to load the link
+information causes the calls to the IShellLink::GetPath and
+IShellLink::GetDescription member functions to fail.
+ */
+HRESULT ResolveIt(HWND hwnd, LPCSTR lpszLinkFile, LPSTR lpszPath) 
+{ 
+    HRESULT hres; 
+    IShellLink* psl; 
+    TCHAR szGotPath[MAX_PATH]; 
+    TCHAR szDescription[MAX_PATH]; 
+    WIN32_FIND_DATA wfd; 
+    
+    *lpszPath = 0; // assume failure 
+    
+    // Get a pointer to the IShellLink interface. 
+    hres = CoCreateInstance(&CLSID_ShellLink, NULL, 
+                            CLSCTX_INPROC_SERVER, &IID_IShellLink, &psl); 
+    if (SUCCEEDED(hres)) { 
+        IPersistFile* ppf; 
+        
+        // Get a pointer to the IPersistFile interface. 
+        hres = psl->lpVtbl->QueryInterface(psl, &IID_IPersistFile, 
+                                           &ppf); 
+        if (SUCCEEDED(hres)) { 
+            WORD wsz[MAX_PATH]; 
+            
+            // Ensure that the string is Unicode. 
+            MultiByteToWideChar(CP_ACP, 0, lpszLinkFile, -1, wsz, 
+                                MAX_PATH); 
+ 
+            // Load the shortcut. 
+            hres = ppf->lpVtbl->Load(ppf, wsz, STGM_READ); 
+            if (SUCCEEDED(hres)) { 
+ 
+                // Resolve the link. 
+                hres = psl->lpVtbl->Resolve(psl, hwnd, SLR_ANY_MATCH); 
+                if (SUCCEEDED(hres)) { 
+                    
+                    // Get the path to the link target. 
+                    hres = psl->lpVtbl->GetPath(psl, szGotPath, 
+                                                MAX_PATH, (WIN32_FIND_DATA *)&wfd, 
+                                                SLGP_SHORTPATH ); 
+                    if (!SUCCEEDED(hres))
+                        HandleErr(hres); // application-defined function 
+ 
+                    // Get the description of the target. 
+                    hres = psl->lpVtbl->GetDescription(psl, 
+                                                       szDescription, MAX_PATH); 
+                    if (!SUCCEEDED(hres)) 
+                        HandleErr(hres); 
+                    lstrcpy(lpszPath, szGotPath); 
+                } 
+            } 
+            // Release the pointer to the IPersistFile interface. 
+            ppf->lpVtbl->Release(ppf); 
+        } 
+        // Release the pointer to the IShellLink interface. 
+        psl->lpVtbl->Release(psl); 
+    } 
+    return hres; 
+} 
+
+/* From MSDN's MFC sample:
+"SHORTCUT: A SampleThat Manipulates Shortcuts"
+*/
+
+HRESULT CreateShortCut::CreateIt (LPCSTR pszShortcutFile, LPSTR pszLink, 
+                                                                  LPSTR pszDesc)
+{
+    HRESULT hres;
+    IShellLink *psl;
+    
+    // Create an IShellLink object and get a pointer to the IShellLink 
+    // interface (returned from CoCreateInstance).
+    hres = CoCreateInstance (CLSID_ShellLink, NULL, CLSCTX_INPROC_SERVER,
+                             IID_IShellLink, (void **)&psl);
+    if (SUCCEEDED (hres)) {
+        IPersistFile *ppf;
+        
+        // Query IShellLink for the IPersistFile interface for 
+        // saving the shortcut in persistent storage.
+        hres = psl->QueryInterface (IID_IPersistFile, (void **)&ppf);
+        if (SUCCEEDED (hres)) { 
+            WORD wsz [MAX_PATH]; // buffer for Unicode string
+            
+            // Set the path to the shortcut target.
+            hres = psl->SetPath (pszShortcutFile);
+            
+            if (! SUCCEEDED (hres))
+                AfxMessageBox ("SetPath failed!");
+            
+            // Set the description of the shortcut.
+            hres = psl->SetDescription (pszDesc);
+            
+            if (! SUCCEEDED (hres))
+                AfxMessageBox ("SetDescription failed!");
+            
+            // Ensure that the string consists of ANSI TCHARacters.
+            MultiByteToWideChar (CP_ACP, 0, pszLink, -1, wsz, MAX_PATH);
+            
+            // Save the shortcut via the IPersistFile::Save member function.
+            hres = ppf->Save (wsz, TRUE);
+            
+            if (! SUCCEEDED (hres))
+                AfxMessageBox (“Save failed!”);
+            
+            // Release the pointer to IPersistFile.
+            ppf->Release ();
+        }
+        // Release the pointer to IShellLink.
+        psl->Release ();
+    }
+    return hres;
+} 
+
+
+void ResolveShortCut::OnOK () 
+{
+    TCHAR szFile [MAX_PATH];
+    
+    // Get the selected item in the list box.
+    DlgDirSelect (szFile, IDC_LIST1);
+    
+    // Find out whether it is a LNK file.
+    if (_tcsstr (szFile, ".lnk") != NULL)
+        // Make the call to ResolveShortcut::ResolveIt here.
+        ResolveIt (m_hWnd, szFile);
+    
+    CDialog::OnOK ();
+} 
+
+HRESULT ResolveShortCut::ResolveIt (HWND hwnd, LPCSTR pszShortcutFile)
+{
+    HRESULT hres;
+    IShellLink *psl;
+    TCHAR szGotPath [MAX_PATH];
+    TCHAR szDescription [MAX_PATH];
+    WIN32_FIND_DATA wfd;
+        
+    // Get a pointer to the IShellLink interface.
+    hres = CoCreateInstance (CLSID_ShellLink, NULL, CLSCTX_INPROC_SERVER,
+                             IID_IShellLink, (void **)&psl);
+    if (SUCCEEDED (hres)) {
+        IPersistFile *ppf;
+                
+        // Get a pointer to the IPersistFile interface.
+        hres = psl->QueryInterface (IID_IPersistFile, (void **)&ppf);
+                
+        if (SUCCEEDED (hres)) {
+            WORD wsz [MAX_PATH]; // buffer for Unicode string
+                        
+            // Ensure that the string consists of Unicode TCHARacters.
+            MultiByteToWideChar (CP_ACP, 0, pszShortcutFile, -1, wsz,
+                                 MAX_PATH);
+                        
+            // Load the shortcut.
+            hres = ppf->Load (wsz, STGM_READ);
+                        
+            if (SUCCEEDED (hres)) {
+                // Resolve the shortcut.
+                hres = psl->Resolve (hwnd, SLR_ANY_MATCH);
+                if (SUCCEEDED (hres)) {
+                    _tcscpy (szGotPath, pszShortcutFile);
+                    // Get the path to the shortcut target.
+                    hres = psl->GetPath (szGotPath, MAX_PATH,
+                                         (WIN32_FIND_DATA *)&wfd, SLGP_SHORTPATH);
+                    if (! SUCCEEDED (hres))
+                        AfxMessageBox ("GetPath failed!");
+                    else
+                        AfxMessageBox (szGotPath);
+                    // Get the description of the target.
+                    hres = psl->GetDescription (szDescription, MAX_PATH);
+                    if (! SUCCEEDED (hres))
+                        AfxMessageBox ("GetDescription failed!");
+                    else
+                        AfxMessageBox (szDescription);
+                }
+            }
+            // Release the pointer to IPersistFile.
+            ppf->Release ();
+        }
+        // Release the pointer to IShellLink.
+        psl->Release ();
+    }
+    return hres;
+}
+#endif /* 0 */
+
+#endif /* !DYNAMORIO_DEMO */ /* around whole file */
diff --git a/tools/DRgui/ShellInterface.h b/tools/DRgui/ShellInterface.h
new file mode 100644
index 000000000..8723870e9
--- /dev/null
+++ b/tools/DRgui/ShellInterface.h
@@ -0,0 +1,62 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// ShellInterface.h: interface for the CShellInterface class.
+//
+//////////////////////////////////////////////////////////////////////
+
+#if !defined(AFX_SHELLINTERFACE_H__9BEBA793_2785_4664_8854_201A76160E2C__INCLUDED_)
+#define AFX_SHELLINTERFACE_H__9BEBA793_2785_4664_8854_201A76160E2C__INCLUDED_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif // _MSC_VER > 1000
+
+class CShellInterface  
+{
+public:
+    CShellInterface();
+    virtual ~CShellInterface();
+
+    static void Initialize();
+    static void Uninitialize();
+    static BOOL ResolveLinkFile(TCHAR *name, TCHAR *resolution, TCHAR *arguments,
+                                TCHAR *working_dir, HWND hwnd);
+
+    static BOOL m_bInitialized;
+};
+
+#endif // !defined(AFX_SHELLINTERFACE_H__9BEBA793_2785_4664_8854_201A76160E2C__INCLUDED_)
diff --git a/tools/DRgui/StdAfx.cpp b/tools/DRgui/StdAfx.cpp
new file mode 100644
index 000000000..d938d1a74
--- /dev/null
+++ b/tools/DRgui/StdAfx.cpp
@@ -0,0 +1,44 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// stdafx.cpp : source file that includes just the standard includes
+//	DynamoRIO.pch will be the pre-compiled header
+//	stdafx.obj will contain the pre-compiled type information
+
+#include "stdafx.h"
+
+
+
diff --git a/tools/DRgui/StdAfx.h b/tools/DRgui/StdAfx.h
new file mode 100644
index 000000000..0b51dcba0
--- /dev/null
+++ b/tools/DRgui/StdAfx.h
@@ -0,0 +1,71 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// stdafx.h : include file for standard system include files,
+//  or project specific include files that are used frequently, but
+//      are changed infrequently
+//
+
+#define _WIN32_WINNT 0x0400 //_WIN32_WINNT_NT4 /* ==0x0400; NTDDI_VERSION is set from this */
+
+// use _s secure versions
+#define _CRT_SECURE_CPP_OVERLOAD_STANDARD_NAMES 1
+// somehow the above define doesn't apply to DynamoRIOView.cpp,
+// so we use both of these to avoid the warnings:
+#define _CRT_SECURE_NO_DEPRECATE 1
+#define _CRT_SECURE_NO_WARNINGS 1
+
+#if !defined(AFX_STDAFX_H__E6657A4C_7B90_457B_A64A_935DB0D375FE__INCLUDED_)
+#define AFX_STDAFX_H__E6657A4C_7B90_457B_A64A_935DB0D375FE__INCLUDED_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif // _MSC_VER > 1000
+
+#define VC_EXTRALEAN		// Exclude rarely-used stuff from Windows headers
+
+#include <afxwin.h>         // MFC core and standard components
+#include <afxext.h>         // MFC extensions
+#include <afxdtctl.h>		// MFC support for Internet Explorer 4 Common Controls
+#ifndef _AFX_NO_AFXCMN_SUPPORT
+#include <afxcmn.h>			// MFC support for Windows Common Controls
+#endif // _AFX_NO_AFXCMN_SUPPORT
+
+
+//{{AFX_INSERT_LOCATION}}
+// Microsoft Visual C++ will insert additional declarations immediately before the previous line.
+
+#endif // !defined(AFX_STDAFX_H__E6657A4C_7B90_457B_A64A_935DB0D375FE__INCLUDED_)
diff --git a/tools/DRgui/SyswideDlg.cpp b/tools/DRgui/SyswideDlg.cpp
new file mode 100644
index 000000000..676f2c29f
--- /dev/null
+++ b/tools/DRgui/SyswideDlg.cpp
@@ -0,0 +1,94 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// SyswideDlg.cpp : implementation file
+//
+
+#ifndef DYNAMORIO_DEMO /* around whole file */
+
+#include "stdafx.h"
+#include "DynamoRIO.h"
+#include "SyswideDlg.h"
+#include <assert.h>
+
+#ifdef _DEBUG
+#define new DEBUG_NEW
+#undef THIS_FILE
+static char THIS_FILE[] = __FILE__;
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+// CSyswideDlg dialog
+
+
+CSyswideDlg::CSyswideDlg(CWnd* pParent /*=NULL*/)
+    : CDialog(CSyswideDlg::IDD, pParent)
+{
+    //{{AFX_DATA_INIT(CSyswideDlg)
+    // NOTE: the ClassWizard will add member initialization here
+    //}}AFX_DATA_INIT
+}
+
+
+void CSyswideDlg::DoDataExchange(CDataExchange* pDX)
+{
+    CDialog::DoDataExchange(pDX);
+    //{{AFX_DATA_MAP(CSyswideDlg)
+    // NOTE: the ClassWizard will add DDX and DDV calls here
+    //}}AFX_DATA_MAP
+}
+
+
+BEGIN_MESSAGE_MAP(CSyswideDlg, CDialog)
+    //{{AFX_MSG_MAP(CSyswideDlg)
+    //}}AFX_MSG_MAP
+END_MESSAGE_MAP()
+
+////////////////////////////////////////////////////////////////////////////
+// CSyswideDlg message handlers
+
+void CSyswideDlg::OnOK() 
+{
+    CButton *check = (CButton *) GetDlgItem(IDC_NOT_AGAIN);
+    assert(check != NULL);
+    if (check->GetCheck() == 1) {
+        CDynamoRIOApp::SetSystemwideSetting(0);
+    }
+        
+    CDialog::OnOK();
+}
+
+#endif /* !DYNAMORIO_DEMO */ /* around whole file */
diff --git a/tools/DRgui/SyswideDlg.h b/tools/DRgui/SyswideDlg.h
new file mode 100644
index 000000000..9b79cc5d9
--- /dev/null
+++ b/tools/DRgui/SyswideDlg.h
@@ -0,0 +1,82 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+#if !defined(AFX_SYSWIDEDLG_H__7611241D_B288_4915_AA69_890788F0D63D__INCLUDED_)
+#define AFX_SYSWIDEDLG_H__7611241D_B288_4915_AA69_890788F0D63D__INCLUDED_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif // _MSC_VER > 1000
+// SyswideDlg.h : header file
+//
+
+/////////////////////////////////////////////////////////////////////////////
+// CSyswideDlg dialog
+
+class CSyswideDlg : public CDialog
+{
+    // Construction
+public:
+    CSyswideDlg(CWnd* pParent = NULL);   // standard constructor
+
+    // Dialog Data
+    //{{AFX_DATA(CSyswideDlg)
+    enum { IDD = IDD_SYSTEMWIDE_CONFIRMATION };
+    // NOTE: the ClassWizard will add data members here
+    //}}AFX_DATA
+
+
+    // Overrides
+    // ClassWizard generated virtual function overrides
+    //{{AFX_VIRTUAL(CSyswideDlg)
+protected:
+    virtual void DoDataExchange(CDataExchange* pDX);    // DDX/DDV support
+    //}}AFX_VIRTUAL
+
+    // Implementation
+protected:
+
+    // Generated message map functions
+    //{{AFX_MSG(CSyswideDlg)
+    virtual void OnOK();
+    //}}AFX_MSG
+    DECLARE_MESSAGE_MAP()
+};
+
+//{{AFX_INSERT_LOCATION}}
+// Microsoft Visual C++ will insert additional declarations immediately before the previous line.
+
+#endif // !defined(AFX_SYSWIDEDLG_H__7611241D_B288_4915_AA69_890788F0D63D__INCLUDED_)
diff --git a/tools/DRgui/res/DynamoRIO.ico b/tools/DRgui/res/DynamoRIO.ico
new file mode 100755
index 000000000..6bb986237
Binary files /dev/null and b/tools/DRgui/res/DynamoRIO.ico differ
diff --git a/tools/DRgui/res/DynamoRIO.rc2 b/tools/DRgui/res/DynamoRIO.rc2
new file mode 100644
index 000000000..77ee997f1
--- /dev/null
+++ b/tools/DRgui/res/DynamoRIO.rc2
@@ -0,0 +1,47 @@
+// **********************************************************
+// Copyright (c) 2007-2009 VMware, Inc.  All rights reserved.
+// **********************************************************/
+
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// 
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+// 
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the following disclaimer in the documentation
+//   and/or other materials provided with the distribution.
+// 
+// * Neither the name of VMware, Inc. nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+// DAMAGE.
+
+// Copyright (c) 2003-2007 Determina Corp.
+// Copyright (c) 2002-2003 Massachusetts Institute of Technology
+// Copyright (c) 2002 Hewlett-Packard Company
+
+//
+// DYNAMORIO.RC2 - resources Microsoft Visual C++ does not edit directly
+//
+
+#ifdef APSTUDIO_INVOKED
+	#error this file is not editable by Microsoft Visual C++
+#endif //APSTUDIO_INVOKED
+
+
+/////////////////////////////////////////////////////////////////////////////
+// Add manually edited resources here...
+
+/////////////////////////////////////////////////////////////////////////////
diff --git a/tools/DRgui/res/DynamoRIODoc.ico b/tools/DRgui/res/DynamoRIODoc.ico
new file mode 100755
index 000000000..2a1f1ae6e
Binary files /dev/null and b/tools/DRgui/res/DynamoRIODoc.ico differ
diff --git a/tools/DRgui/res/Toolbar.bmp b/tools/DRgui/res/Toolbar.bmp
new file mode 100755
index 000000000..d501723c1
Binary files /dev/null and b/tools/DRgui/res/Toolbar.bmp differ
diff --git a/tools/DRgui/res/drlogo.bmp b/tools/DRgui/res/drlogo.bmp
new file mode 100644
index 000000000..3b051a80b
Binary files /dev/null and b/tools/DRgui/res/drlogo.bmp differ
diff --git a/tools/DRgui/res/vmware-logo.bmp b/tools/DRgui/res/vmware-logo.bmp
new file mode 100755
index 000000000..d77ebdaec
Binary files /dev/null and b/tools/DRgui/res/vmware-logo.bmp differ
diff --git a/tools/DRgui/res/vmware-logo.png b/tools/DRgui/res/vmware-logo.png
new file mode 100644
index 000000000..5ac4282fb
Binary files /dev/null and b/tools/DRgui/res/vmware-logo.png differ
diff --git a/tools/DRgui/resource.h b/tools/DRgui/resource.h
new file mode 100644
index 000000000..620083320
--- /dev/null
+++ b/tools/DRgui/resource.h
@@ -0,0 +1,206 @@
+/* **********************************************************
+ * Copyright (c) 2007-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+//{{NO_DEPENDENCIES}}
+// Microsoft Developer Studio generated include file.
+// Used by DynamoRIO.rc
+//
+#define IDD_ABOUTBOX                    100
+#define IDD_DYNAMORIO_FORM              101
+#define IDD_DYNAMORIO_DEMO_FORM         102
+#define IDR_ABOUT_ICON                  126
+#define IDR_MAINFRAME_DEMO              127
+#define IDR_MAINFRAME                   128
+#define IDR_DYNAMOTYPE                  129
+#define IDD_OPTIONS                     130
+#define IDD_CMDLINE                     134
+#define IDD_LOGGING                     135
+#define IDD_IGNORELIST                  140
+#define IDD_SYSTEMWIDE_CONFIRMATION     141
+#define IDB_BITMAP1                     146
+#define IDB_BITMAP2                     147
+#define IDB_LOGO_VMWARE                 153
+#define IDB_LOGO_DYNAMORIO              157
+#define IDC_OPTIONS_EDIT                1000
+#define IDC_OPTIONS_LIST                1001
+#define IDC_PROCESS_LIST                1002
+#define IDC_CHECK1                      1004
+#define IDC_LOG_STATS                   1004
+#define IDC_NOT_AGAIN                   1004
+#define IDC_VERBOSE_SLIDER              1005
+#define IDC_CHECK2                      1005
+#define IDC_LOG_TOP                     1005
+#define IDC_NUM_THREADS                 1006
+#define IDC_EDIT1                       1006
+#define IDC_EDIT_HOT_THRESHOLD          1006
+#define IDC_IGNORELIST                  1006
+#define IDC_NUM_FRAGMENTS               1007
+#define IDC_CHECK3                      1007
+#define IDC_EDIT9                       1007
+#define IDC_INSTRLIBNAME                1007
+#define IDC_EDIT_MASK                   1007
+#define IDC_NUM_TRACES                  1008
+#define IDC_EDIT2                       1008
+#define IDC_CHECK20                     1008
+#define IDC_EDIT_INSTRLIBNAME           1008
+#define IDC_LOG_THREADS                 1008
+#define IDC_HEAP_USED                   1009
+#define IDC_BUTTON1                     1009
+#define IDC_CHECK21                     1009
+#define IDC_BROWSE_INSTRLIBNAME         1009
+#define IDC_LOG_SYSCALLS                1009
+#define IDC_NUM_DELETED                 1010
+#define IDC_CHECK4                      1010
+#define IDC_CMDLINE                     1010
+#define IDC_LOG_ASYNCH                  1010
+#define IDC_CACHE_USED                  1011
+#define IDC_CHECK5                      1011
+#define IDC_WORKING_DIR                 1011
+#define IDC_LOG_INTERP                  1011
+#define IDC_NUM_CALLBACKS               1012
+#define IDC_CHECK6                      1012
+#define IDC_WORKING_DIR_BROWSE          1012
+#define IDC_CHECK22                     1012
+#define IDC_PROF_PCS                    1012
+#define IDC_LOG_EMIT                    1012
+#define IDC_NUM_EXCEPTIONS              1013
+#define IDC_CHECK7                      1013
+#define IDC_CHECK23                     1013
+#define IDC_NOASYNCH                    1013
+#define IDC_LOG_LINKS                   1013
+#define IDC_NUM_APCS                    1014
+#define IDC_CHECK8                      1014
+#define IDC_NOLINK                      1014
+#define IDC_NUM_BASIC_BLOCKS            1015
+#define IDC_CHECK9                      1015
+#define IDC_COMBO1                      1015
+#define IDC_VERBOSITY                   1015
+#define IDC_LOGMASK_VALUE               1016
+#define IDC_CHECK10                     1016
+#define IDC_DUMP_TRACES                 1016
+#define IDC_TRACEDUMP_BINARY            1016
+#define IDC_NUM_FLUSHES2                1017
+#define IDC_CHECK11                     1017
+#define IDC_DUMP_TRACE_ORIGINS          1017
+#define IDC_TRACEDUMP_ORIGINS           1017
+#define IDC_MAX_HEAP                    1018
+#define IDC_CHECK12                     1018
+#define IDC_LOG_CACHE                   1018
+#define IDC_NUM_FLUSHES4                1019
+#define IDC_EDIT3                       1019
+#define IDC_CHECK24                     1019
+#define IDC_EDIT_CACHE_BB_MAX           1019
+#define IDC_LOG_FRAGMENT                1019
+#define IDC_MAX_CACHE                   1020
+#define IDC_CHECK13                     1020
+#define IDC_CHECK25                     1020
+#define IDC_CACHE_TRACE_MAX             1020
+#define IDC_LOG_DISPATCH                1020
+#define IDC_APP_CODE_SEEN               1021
+#define IDC_EDIT4                       1021
+#define IDC_CHECK26                     1021
+#define IDC_EDIT_CACHE_TRACE_MAX        1021
+#define IDC_LOG_MONITOR                 1021
+#define IDC_VERBOSE_VALUE               1022
+#define IDC_LOGLEVEL_VALUE              1022
+#define IDC_CHECK14                     1022
+#define IDC_LOG_HEAP                    1022
+#define IDC_EDIT5                       1023
+#define IDC_CHECK27                     1023
+#define IDC_CHANGE_LOGGING              1023
+#define IDC_EDIT_CACHE_SIZE             1023
+#define IDC_LOG_VMAREAS                 1023
+#define IDC_CHECK15                     1024
+#define IDC_CHECK28                     1024
+#define IDC_HEAP_SIZE                   1024
+#define IDC_LOG_ALL                     1024
+#define IDC_LIST                        1025
+#define IDC_EDIT6                       1025
+#define IDC_EDIT_HEAP_SIZE              1025
+#define IDC_LOG_NONE                    1025
+#define IDC_CHECK16                     1026
+#define IDC_LOGTHREADS                  1026
+#define IDC_TRACEDUMP_TEXT              1026
+#define IDC_CHECK17                     1027
+#define IDC_STATS                       1027
+#define IDC_CHECK18                     1028
+#define IDC_LOGLEVEL                    1028
+#define IDC_BUTTON2                     1029
+#define IDC_LOGGING_BUTTON              1029
+#define IDC_LOGFILE_EXPLORE             1029
+#define IDC_LOGDIR_EXPLORE              1029
+#define IDC_CHECK19                     1030
+#define IDC_LOGMASK                     1030
+#define IDC_EXITED                      1031
+#define IDC_HOT_THRESHOLD               1031
+#define IDC_CACHE_BB_MAX                1032
+#define IDC_CACHE_SIZE                  1033
+#define IDC_NOTIFY                      1033
+#define IDC_LOGFILE                     1034
+#define IDC_LOGDIR                      1034
+#define IDC_EDIT_LOGLEVEL               1035
+#define IDC_EDIT_LOGMASK                1036
+#define IDC_PROF_COUNTS                 1037
+#define IDC_NULLCALLS                   1038
+#define IDC_SET_PERMANENT               1040
+#define IDC_CLIENTSTATS                 1043
+#define IDC_CLIENTSTATS2                1044
+#define IDC_STATS_SCROLLBAR             1044
+#define IDC_SLIDER1                     1045
+#define IDC_STATS_SLIDER                1045
+#define IDC_LIST1                       1046
+#define IDC_SELECT_BAR                  1046
+#define IDC_COMBO2                      1047
+#define ID_SYSTEMWIDE_INJECT            32771
+#define ID_SYSTEMWIDE_LIST              32772
+#define ID_EDIT_OPTIONS                 32773
+#define ID_EDIT_COPYSTATS               32774
+#define ID_VIEW_PROCESS                 32775
+#define ID_LIBRARY_RELEASE              32776
+#define ID_LIBRARY_DEBUG                32777
+#define ID_LIBRARY_PROFILE              32778
+#define ID_FILE_SYSTEMWIDE              32779
+#define ID_FILE_RUN                     32781
+#define ID_HELP_HELP                    32782
+#define ID_EDIT_IGNORELIST              32783
+
+// Next default values for new objects
+// 
+#ifdef APSTUDIO_INVOKED
+#ifndef APSTUDIO_READONLY_SYMBOLS
+#define _APS_3D_CONTROLS                     1
+#define _APS_NEXT_RESOURCE_VALUE        158
+#define _APS_NEXT_COMMAND_VALUE         32784
+#define _APS_NEXT_CONTROL_VALUE         1048
+#define _APS_NEXT_SYMED_VALUE           101
+#endif
+#endif
diff --git a/tools/DRinstall/CopyDlg.cpp b/tools/DRinstall/CopyDlg.cpp
new file mode 100644
index 000000000..cfb9dda71
--- /dev/null
+++ b/tools/DRinstall/CopyDlg.cpp
@@ -0,0 +1,301 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+#include "stdafx.h"
+#include "Wizard.h"
+#include "CopyDlg.h"
+#include <assert.h>
+#include <direct.h> // for _chdir
+#include "WizSheet.h"
+#include "ShellInterface.h"
+
+#ifdef _DEBUG
+#define new DEBUG_NEW
+#undef THIS_FILE
+static char THIS_FILE[] = __FILE__;
+#endif
+
+// for CGO 2008 tutorial we create a zip file from data in our data section
+static const unsigned char zipdata[] = {
+#include "zipdump.h"
+};
+#define ZIPNAME _T("DynamoRIO.zip")
+
+// we copy the tree rooted at DynamoRIO into here, so only specify parent dir:
+#define DEFAULT_DIR _T("VMware\\DynamoRIO")
+
+// MB of space we take up
+#define DISK_SPACE_REQUIRED  14
+
+/////////////////////////////////////////////////////////////////////////////
+// CCopyDlg property page
+
+IMPLEMENT_DYNCREATE(CCopyDlg, CPropertyPage)
+
+    CCopyDlg::CCopyDlg() : CPropertyPage(CCopyDlg::IDD, 0)
+{
+    //{{AFX_DATA_INIT(CCopyDlg)
+    m_SpaceAvailable = _T("");
+    m_SpaceRequired = _T("");
+    m_Target = _T("");
+    //}}AFX_DATA_INIT
+
+    m_DefaultDir[0] = _T('\0');
+    m_psp.dwFlags |= PSP_HIDEHEADER;
+    m_psp.dwFlags &= ~(PSP_HASHELP);
+}
+
+CCopyDlg::~CCopyDlg()
+{
+}
+
+void CCopyDlg::DoDataExchange(CDataExchange* pDX)
+{
+    CPropertyPage::DoDataExchange(pDX);
+    //{{AFX_DATA_MAP(CCopyDlg)
+    DDX_Control(pDX, IDC_TARGET, m_TargetEdit);
+    DDX_Text(pDX, IDC_SPACE_AVAILABLE, m_SpaceAvailable);
+    DDX_Text(pDX, IDC_SPACE_REQUIRED, m_SpaceRequired);
+    DDX_Text(pDX, IDC_TARGET, m_Target);
+    //}}AFX_DATA_MAP
+}
+
+
+BEGIN_MESSAGE_MAP(CCopyDlg, CPropertyPage)
+    //{{AFX_MSG_MAP(CCopyDlg)
+    ON_BN_CLICKED(IDC_BROWSE, OnBrowse)
+    //}}AFX_MSG_MAP
+    END_MESSAGE_MAP()
+
+    /////////////////////////////////////////////////////////////////////////////
+    // CCopyDlg message handlers
+
+    BOOL CCopyDlg::OnInitDialog()
+{
+    CDialog::OnInitDialog();
+
+    m_SpaceRequired.Format(_T("%8d"), DISK_SPACE_REQUIRED);
+    ULARGE_INTEGER userbytes, totalbytes, freebytes;
+    // WARNING: GetDiskFreeSpaceEx is not available on NT < 4.0
+    if (!GetDiskFreeSpaceEx(_T("C:\\"), &userbytes, &totalbytes, &freebytes))
+        assert(false);
+    __int64 bytes = (freebytes.QuadPart)/(1024*1024);
+    m_SpaceAvailable.Format(_T("%8I64u"), bytes);
+
+    int res = GetCurrentDirectory(MAX_PATH, m_CWD);
+    assert(res > 0);
+
+    TCHAR drive[8];
+    int len = GetEnvironmentVariable(_T("SYSTEMDRIVE"), drive, 7);
+    if (len <= 0)
+        _stprintf(drive, _T("C:"));
+
+    TCHAR buf[MAX_PATH];
+    len = GetEnvironmentVariable(_T("PROGRAMFILES"), buf, MAX_PATH);
+    if (len <= 0) {
+        // NT doesn't have this env var...so just hardcode it
+        _stprintf(buf, _T("%s\\Program Files"), drive);
+        len = _tcslen(buf);
+    }
+    if (!SetCurrentDirectory(buf)) {
+        // directory does not exist or is not accessible!
+        _stprintf(buf, _T("%s\\"), drive);
+        if (!SetCurrentDirectory(buf)) {
+            assert(FALSE);
+        }
+    }
+
+    assert(len + _tcslen(DEFAULT_DIR) + 1 < MAX_PATH);
+    _stprintf(m_DefaultDir, _T("%s\\%s"), buf, DEFAULT_DIR);
+    m_Target.Format(_T("%s"), m_DefaultDir);
+        
+    // must go back to wd
+    if (!SetCurrentDirectory(m_CWD)) {
+        assert(FALSE);
+    }
+
+    UpdateData(FALSE);
+
+    return TRUE;
+}
+
+BOOL CCopyDlg::OnSetActive()
+{
+    CPropertySheet* pSheet = (CPropertySheet*)GetParent();
+    ASSERT_KINDOF(CPropertySheet, pSheet);
+    // our installer doesn't support Back!
+    //      pSheet->SetWizardButtons( PSWIZB_BACK | PSWIZB_NEXT);
+    pSheet->SetWizardButtons( PSWIZB_NEXT);
+    return CPropertyPage::OnSetActive();
+}
+
+
+LRESULT CCopyDlg::OnWizardNext() 
+{
+    if (!CopyFiles()) {
+        // select entire directory
+        m_TargetEdit.SetSel(0, -1);
+        m_TargetEdit.SetFocus();
+        // return to this same wizard page
+        // (we assume problem is with target dir)
+        return -1;
+    }
+
+    // need to communicate install dir to later pages
+    // use parent Sheet
+    CWizardSheet* pSheet = (CWizardSheet*)GetParent();
+    // add the DynamoRIO on
+    pSheet->m_InstallDir.Format(_T("%s\\DynamoRIO"), m_Target);
+
+    return CPropertyPage::OnWizardNext();
+}
+
+BOOL CCopyDlg::CopyFiles()
+{
+    TCHAR msg[MAX_PATH*2];
+    TCHAR cmd[MAX_PATH*2];
+
+    UpdateData(TRUE); // get target dir string
+
+    // make the target directory
+    // have to build each new directory one at a time
+    CString newdir;
+    int pos = 0;
+    int slash = m_Target.Find('\\', pos);
+
+    // first see if need to clean out existing dir
+    if (SetCurrentDirectory(m_Target)) {
+        _stprintf(msg, _T("Directory %s already exists.\n")
+                  _T("Continuing will delete all its existing files.\nContinue?"), m_Target);
+        int res = MessageBox(msg, _T("Confirmation"), MB_OKCANCEL | MYMBFLAGS);
+        if (res == IDCANCEL)
+            return FALSE;
+        // get out of the dir
+        BOOL ok = SetCurrentDirectory(m_CWD);
+        assert(ok);
+        _stprintf(cmd, _T("%s"), m_Target);
+        if (!CShellInterface::DeleteFile(cmd, GetParent()->m_hWnd)) {
+            _stprintf(msg, _T("Error removing existing directory %s"), m_Target);
+            MessageBox(msg, _T("Error Deleting Files"), MB_OK | MYMBFLAGS);
+            return FALSE;
+        }
+    }
+
+    while (TRUE) {
+        if (slash == -1)
+            newdir = m_Target;
+        else
+            newdir = m_Target.Mid(0, slash);
+        if (!SetCurrentDirectory(newdir)) {
+            _stprintf(msg, _T("Create directory %s?"), newdir);
+            int res = MessageBox(msg, _T("Confirmation"), MB_OKCANCEL | MYMBFLAGS);
+            if (res == IDCANCEL)
+                return FALSE;
+            if (!CreateDirectory(newdir, NULL)) {
+                _stprintf(msg, _T("Could not create directory %s"), newdir);
+                MessageBox(msg, _T("Error Copying Files"), MB_OK | MYMBFLAGS);
+                return FALSE;
+            }
+        }
+        if (slash == -1)
+            break;
+        pos = slash + 1;
+        slash = m_Target.Find('\\', pos);
+    }
+
+    // now copy the files
+#if 1 /* CGO 2008 tutorial: create zip file */
+    DWORD written;
+    BOOL success;
+    TCHAR to[MAX_PATH];
+    _stprintf(to, _T("%s\\%s"), m_Target, ZIPNAME);
+    HANDLE h = CreateFile(to,                    // file to open
+                          GENERIC_WRITE,         // no read access needed
+                          0,                     // no sharing
+                          NULL,                  // default security
+                          CREATE_ALWAYS,         // clobber existing file
+                          FILE_ATTRIBUTE_NORMAL, // normal file
+                          NULL);                 // no attr. template
+    if (h == NULL)
+        success = FALSE;
+    else {
+        success = WriteFile(h, zipdata, sizeof(zipdata), &written, NULL);
+        CloseHandle(h);
+    }
+    if (!success || written != sizeof(zipdata)) {
+        _stprintf(msg, _T("Error copying file to %s"), to);
+        MessageBox(msg, _T("Error Copying Files"), MB_OK | MYMBFLAGS);
+        return FALSE;
+    }
+#else
+    TCHAR from[MAX_PATH];
+    TCHAR to[MAX_PATH];
+# if 0
+    _stprintf(from, _T("%s"), _T("c:\\iye\\rio\\install\\DynamoRIO"));
+    //_stprintf(from, _T("%s"), _T("d:\\bruening\\dynamo\\install\\DynamoRIO"));
+# else
+    _stprintf(from, _T("%s\\DynamoRIO"), m_CWD);
+# endif
+    _stprintf(to, _T("%s"), m_Target);
+    if (!CShellInterface::CopyDir(from, to, GetParent()->m_hWnd)) {
+        _stprintf(msg, _T("Error copying files from %s to %s"), from, to);
+        MessageBox(msg, _T("Error Copying Files"), MB_OK | MYMBFLAGS);
+        return FALSE;
+    }
+#endif
+
+    return TRUE;
+}
+
+void CCopyDlg::OnBrowse() 
+{
+    TCHAR folder[MAX_PATH];
+    BROWSEINFO bi;
+    bi.hwndOwner = m_hWnd;
+    bi.pidlRoot = NULL;
+    bi.pszDisplayName = folder;
+    bi.lpszTitle = _T("Select folder to install into");
+    bi.ulFlags = 0;
+    bi.lpfn = NULL;
+    bi.lParam = NULL;
+    bi.iImage = 0;
+    ITEMIDLIST *id = SHBrowseForFolder(&bi);
+    if (id == NULL) // cancelled
+        return;
+    SHGetPathFromIDList(id, folder);
+    m_Target.Format(_T("%s"), folder);
+    UpdateData(FALSE);
+}
diff --git a/tools/DRinstall/CopyDlg.h b/tools/DRinstall/CopyDlg.h
new file mode 100644
index 000000000..8e912ef0f
--- /dev/null
+++ b/tools/DRinstall/CopyDlg.h
@@ -0,0 +1,99 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+#ifndef _COPY_DLG_H_
+#define _COPY_DLG_H_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif // _MSC_VER > 1000
+
+class CWizardSheet;
+
+/////////////////////////////////////////////////////////////////////////////
+// CCopyDlg dialog
+
+class CCopyDlg : public CPropertyPage
+{
+    DECLARE_DYNCREATE(CCopyDlg)
+
+// Construction
+public:
+    BOOL OnInitDialog();
+    CCopyDlg();
+    ~CCopyDlg();
+
+// Dialog Data
+    //{{AFX_DATA(CCopyDlg)
+    enum { IDD = IDD_COPY };
+    CEdit m_TargetEdit;
+    CString m_SpaceAvailable;
+    CString m_SpaceRequired;
+    CString m_Target;
+    //}}AFX_DATA
+
+
+// Overrides
+    // ClassWizard generate virtual function overrides
+    //{{AFX_VIRTUAL(CCopyDlg)
+public:
+    virtual BOOL OnSetActive();
+    virtual LRESULT OnWizardNext();
+protected:
+    virtual void DoDataExchange(CDataExchange* pDX);    // DDX/DDV support
+    //}}AFX_VIRTUAL
+
+// Implementation
+protected:
+    BOOL SetEnvironmentVars();
+    BOOL AddToStartMenu();
+    BOOL InitializeRegistry();
+    BOOL CopyFiles();
+    // Generated message map functions
+    //{{AFX_MSG(CCopyDlg)
+    afx_msg void OnBrowse();
+    //}}AFX_MSG
+    DECLARE_MESSAGE_MAP()
+        
+    TCHAR m_DefaultDir[MAX_PATH];
+    TCHAR m_StartMenu[MAX_PATH];
+    TCHAR m_CWD[MAX_PATH];
+};
+
+//{{AFX_INSERT_LOCATION}}
+// Microsoft Developer Studio will insert additional declarations immediately before the previous line.
+
+#endif // _COPY_DLG_H_
diff --git a/tools/DRinstall/FinalDlg.cpp b/tools/DRinstall/FinalDlg.cpp
new file mode 100644
index 000000000..d6fbca79e
--- /dev/null
+++ b/tools/DRinstall/FinalDlg.cpp
@@ -0,0 +1,91 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+#include "stdafx.h"
+#include "Wizard.h"
+#include "FinalDlg.h"
+
+#ifdef _DEBUG
+#define new DEBUG_NEW
+#undef THIS_FILE
+static char THIS_FILE[] = __FILE__;
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+// CFinalDlg property page
+
+IMPLEMENT_DYNCREATE(CFinalDlg, CPropertyPage)
+
+CFinalDlg::CFinalDlg() : CPropertyPage(CFinalDlg::IDD)
+{
+    //{{AFX_DATA_INIT(CFinalDlg)
+    // NOTE: the ClassWizard will add member initialization here
+    //}}AFX_DATA_INIT
+    m_psp.dwFlags |= PSP_HIDEHEADER;
+    m_psp.dwFlags &= ~(PSP_HASHELP);
+}
+
+CFinalDlg::~CFinalDlg()
+{
+}
+
+void CFinalDlg::DoDataExchange(CDataExchange* pDX)
+{
+    CPropertyPage::DoDataExchange(pDX);
+    //{{AFX_DATA_MAP(CFinalDlg)
+    // NOTE: the ClassWizard will add DDX and DDV calls here
+    //}}AFX_DATA_MAP
+}
+
+
+BEGIN_MESSAGE_MAP(CFinalDlg, CPropertyPage)
+    //{{AFX_MSG_MAP(CFinalDlg)
+    // NOTE: the ClassWizard will add message map macros here
+    //}}AFX_MSG_MAP
+END_MESSAGE_MAP()
+
+/////////////////////////////////////////////////////////////////////////////
+// CFinalDlg message handlers
+
+BOOL CFinalDlg::OnSetActive()
+{
+    CPropertySheet* pSheet = (CPropertySheet*)GetParent();
+    ASSERT_KINDOF(CPropertySheet, pSheet);
+    // our installer doesn't support Back!
+    //      pSheet->SetWizardButtons( PSWIZB_BACK | PSWIZB_NEXT | PSWIZB_FINISH);
+    pSheet->SetWizardButtons( PSWIZB_NEXT | PSWIZB_FINISH);
+    return CPropertyPage::OnSetActive();
+}
diff --git a/tools/DRinstall/FinalDlg.h b/tools/DRinstall/FinalDlg.h
new file mode 100644
index 000000000..8560e9f47
--- /dev/null
+++ b/tools/DRinstall/FinalDlg.h
@@ -0,0 +1,86 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+#ifndef _FINAL_DLG_H_
+#define _FINAL_DLG_H_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif // _MSC_VER > 1000
+
+/////////////////////////////////////////////////////////////////////////////
+// CFinalDlg dialog
+
+class CFinalDlg : public CPropertyPage
+{
+    DECLARE_DYNCREATE(CFinalDlg)
+
+// Construction
+public:
+    CFinalDlg();
+    ~CFinalDlg();
+
+// Dialog Data
+    //{{AFX_DATA(CFinalDlg)
+    enum { IDD = IDD_FINAL };
+    // NOTE - ClassWizard will add data members here.
+    //    DO NOT EDIT what you see in these blocks of generated code !
+    //}}AFX_DATA
+
+
+// Overrides
+    // ClassWizard generate virtual function overrides
+    //{{AFX_VIRTUAL(CFinalDlg)
+public:
+    virtual BOOL OnSetActive();
+protected:
+    virtual void DoDataExchange(CDataExchange* pDX);    // DDX/DDV support
+    //}}AFX_VIRTUAL
+
+// Implementation
+protected:
+    // Generated message map functions
+    //{{AFX_MSG(CFinalDlg)
+    // NOTE: the ClassWizard will add member functions here
+    //}}AFX_MSG
+    DECLARE_MESSAGE_MAP()
+
+};
+
+//{{AFX_INSERT_LOCATION}}
+// Microsoft Developer Studio will insert additional declarations immediately before the previous line.
+
+#endif // _FINAL_DLG_H_
diff --git a/tools/DRinstall/LicenseDlg.cpp b/tools/DRinstall/LicenseDlg.cpp
new file mode 100644
index 000000000..ad78b0b5e
--- /dev/null
+++ b/tools/DRinstall/LicenseDlg.cpp
@@ -0,0 +1,522 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// LicenseDlg.cpp : implementation file
+//
+// This is a part of the Microsoft Foundation Classes C++ library.
+// Copyright (C) 1992-1998 Microsoft Corporation
+// All rights reserved.
+//
+// This source code is only intended as a supplement to the
+// Microsoft Foundation Classes Reference and related
+// electronic documentation provided with the library.
+// See these sources for detailed information regarding the
+// Microsoft Foundation Classes product.
+
+#include "stdafx.h"
+#include "Wizard.h"
+#include "LicenseDlg.h"
+#include <assert.h>
+
+#ifdef _DEBUG
+#define new DEBUG_NEW
+#undef THIS_FILE
+static char THIS_FILE[] = __FILE__;
+#endif
+
+/* This text is verbatim from the modified PRBA provided by David Howard 4/2/08 */
+static TCHAR LICENSE[] = _T("")
+_T("NOTICE: BY DOWNLOADING AND INSTALLING, COPYING OR OTHERWISE USING \r\n")
+_T("THE INTERFACES, SOFTWARE AND OTHER MATERIALS ON THIS CD, YOU \r\n")
+_T("AGREE TO BE BOUND BY THE TERMS OF THIS AGREEMENT.  IF YOU DO NOT \r\n")
+_T("AGREE TO THE TERMS OF THIS EULA, YOU MAY NOT DOWNLOAD, INSTALL, \r\n")
+_T("COPY OR USE THE INTERFACES, SOFTWARE OR OTHER MATERIALS ON THIS \r\n")
+_T("CD. \"YOU\" MEANS THE NATURAL PERSON OR THE ENTITY THAT IS \r\n")
+_T("AGREEING TO BE BOUND BY THIS EULA, THEIR EMPLOYEES AND THIRD \r\n")
+_T("PARTY CONTRACTORS THAT PROVIDE SERVICES TO YOU. YOU SHALL BE \r\n")
+_T("LIABLE FOR ANY FAILURE BY SUCH EMPLOYEES AND THIRD PARTY \r\n")
+_T("CONTRACTORS TO COMPLY WITH THE TERMS OF THIS AGREEMENT. \r\n")
+_T("  \r\n")
+_T("1. DEFINITIONS. \r\n")
+_T(" \r\n")
+_T("(a) \"Software\" shall mean the VMware software, in object code only, as set \r\n")
+_T("forth in Exhibit A attached hereto. \r\n")
+_T(" \r\n")
+_T("(b) \"Documentation\" shall mean the printed or online written reference \r\n")
+_T("material, if any, that may be furnished to Licensee in conjunction with the \r\n")
+_T("Software, including, without limitation, instructions, testing guidelines, \r\n")
+_T("and end user guides. \r\n")
+_T(" \r\n")
+_T("(c) \"Intellectual Property Rights\" shall mean all intellectual property \r\n")
+_T("rights, including, without limitation, patent, copyright, trademark, and \r\n")
+_T("trade secret. \r\n")
+_T(" \r\n")
+_T("(d) \"Open Source Software\" shall mean various open source software \r\n")
+_T("components provided with the Software that are licensed to Licensee under \r\n")
+_T("the terms of the applicable license agreements included with such open \r\n")
+_T("source software components or other materials for the Software. \r\n")
+_T(" \r\n")
+_T("(e) \"Update(s)\" shall mean any modification, error correction, bug fix, \r\n")
+_T("patch or other update to or for the Software. \r\n")
+_T(" \r\n")
+_T("2. LICENSE GRANT, USE AND OWNERSHIP \r\n")
+_T(" \r\n")
+_T("(a) License.  Subject to the terms and conditions of this Agreement, VMware \r\n")
+_T("grants to Licensee a non-exclusive, revocable, non-transferable license \r\n")
+_T("(without the right to sublicense) to use the Software and Documentation \r\n")
+_T("solely for purposes of internal testing and evaluation, as well as \r\n")
+_T("development of Licensee products that communicate with the Software. \r\n")
+_T(" \r\n")
+_T("(b) Feedback. The purpose of this license is the internal testing and \r\n")
+_T("evaluation of the Software by Licensee and development by Licensee of \r\n")
+_T("Licensee products that communicate with the Software.  In furtherance of \r\n")
+_T("this purpose, Licensee shall from time to time provide feedback to VMware \r\n")
+_T("concerning the functionality and performance of the Software including, \r\n")
+_T("without limitation, identifying potential errors and improvements. \r\n")
+_T("Notwithstanding the foregoing, prior to Licensee disclosing to VMware any \r\n")
+_T("information in connection with this Agreement which Licensee considers \r\n")
+_T("proprietary or confidential, Licensee shall obtain VMware's prior written \r\n")
+_T("approval to disclose such information to VMware, and without such prior \r\n")
+_T("written approval from VMware, Licensee shall not disclose any such \r\n")
+_T("information to VMware.  Feedback and other information which is provided by \r\n")
+_T("Licensee to VMware in connection with the Software, Documentation, or this \r\n")
+_T("Agreement may be used by VMware to improve or enhance its products and, \r\n")
+_T("accordingly, VMware shall have a non-exclusive, perpetual, irrevocable, \r\n")
+_T("royalty-free, worldwide right and license to use, reproduce, disclose, \r\n")
+_T("sublicense, modify, make, have made, distribute, sell, offer for sale, \r\n")
+_T("display, perform, create derivative works, permit unmodified binary \r\n")
+_T("distribution and otherwise exploit such feedback and information without \r\n")
+_T("restriction. \r\n")
+_T(" \r\n")
+_T("(c) Restrictions.  Licensee shall not copy or use the Software (including \r\n")
+_T("the Documentation) or disseminate Confidential Information, as defined \r\n")
+_T("below, to any third party except as expressly permitted in this Agreement. \r\n")
+_T("Licensee will not, and will not permit any third party to, sublicense, \r\n")
+_T("rent, copy, modify, create derivative works of, translate, reverse \r\n")
+_T("engineer, decompile, disassemble, or otherwise reduce to human perceivable \r\n")
+_T("form any portion of the Software or Documentation.  In no event shall \r\n")
+_T("Licensee use the Software or Documentation for any commercial purpose \r\n")
+_T("except as expressly set forth in this Agreement.  The Software, \r\n")
+_T("Documentation, and all performance data and test results, including without \r\n")
+_T("limitation, benchmark test results (collectively \"Performance Data\"), \r\n")
+_T("relating to the Software are the Confidential Information of VMware, and \r\n")
+_T("will be treated in accordance with the terms of Section 4 of this \r\n")
+_T("Agreement.  Accordingly, Licensee shall not publish or disclose to any \r\n")
+_T("third party any Performance Data relating to the Software.  Licensee shall \r\n")
+_T("immediately cease all use of the Software and Documentation, upon notice \r\n")
+_T("from VMware. \r\n")
+_T(" \r\n")
+_T("(d) Ownership.  VMware shall own and retain all right, title and interest \r\n")
+_T("in and to the Intellectual Property Rights in the Software, Documentation, \r\n")
+_T("and any derivative works thereof, subject only to the license expressly set \r\n")
+_T("forth in Section 2(a) hereof.  Licensee does not acquire any other rights, \r\n")
+_T("express or implied, in the Software or Documentation.  VMWARE RESERVES ALL \r\n")
+_T("RIGHTS NOT EXPRESSLY GRANTED HEREUNDER. \r\n")
+_T(" \r\n")
+_T("(e) No Support Services.  VMware is under no obligation to support the \r\n")
+_T("Software in any way or to provide any Updates to Licensee.  In the event \r\n")
+_T("VMware, in its sole discretion, supplies any Update to Licensee, such \r\n")
+_T("Update shall be deemed Software hereunder and shall be subject to the terms \r\n")
+_T("and conditions of this Agreement. Upon VMware's release of any Update, \r\n")
+_T("Licensee shall immediately cease all use of the former version of the \r\n")
+_T("Software. \r\n")
+_T(" \r\n")
+_T("(f) Third-Party Software.  The Software enables a computer to run multiple \r\n")
+_T("instances of third-party guest operating systems and application programs. \r\n")
+_T("Licensee acknowledges that Licensee is responsible for obtaining any \r\n")
+_T("licenses necessary to operate any such third-party software, including \r\n")
+_T("guest operating systems. \r\n")
+_T(" \r\n")
+_T("(g) Open Source Software.  The terms and conditions of this Agreement shall \r\n")
+_T("not apply to any Open Source Software accompanying the Software.  Any such \r\n")
+_T("Open Source Software is provided under the terms of the open source license \r\n")
+_T("agreement or copyright notice accompanying such Open Source Software or in \r\n")
+_T("the open source licenses file accompanying the Software. \r\n")
+_T(" \r\n")
+_T("(h) Demonstration. Subject to the terms and conditions of this Agreement, \r\n")
+_T("VMware grants to Licensee a non-exclusive, revocable, non-transferable \r\n")
+_T("license (without the right to sublicense) to use the Software for \r\n")
+_T("demonstration to third parties, provided Licensee (i) has received prior \r\n")
+_T("written authorization from VMware for same, (ii) has had such third parties \r\n")
+_T("first sign a confidentiality agreement that contains nondisclosure \r\n")
+_T("restrictions substantially similar to those set forth in this Agreement, \r\n")
+_T("(iii) includes in such confidentiality agreement an acknowledgement that \r\n")
+_T("VMware does not promise or guarantee that features, functionality and/or \r\n")
+_T("modules in the Software will be included in any generally available version \r\n")
+_T("of the Software, or will be marketed separately for additional fees, and \r\n")
+_T("(iv) conducts such demonstration solely on Licensee's hardware and such \r\n")
+_T("hardware remains at all times in Licensee's possession and control. \r\n")
+_T(" \r\n")
+_T("3. TERM AND TERMINATION. This Agreement is effective as of the Effective \r\n")
+_T("Date and will continue for a one (1) year period (\"Initial Term\"), unless \r\n")
+_T("amended to establish a later expiration date (\"Subsequent Term\") by a \r\n")
+_T("written agreement signed by both parties, or until terminated as provided \r\n")
+_T("in this Agreement.  Either party may terminate this Agreement at any time \r\n")
+_T("for any reason or no reason by providing the other party advance written \r\n")
+_T("notice thereof.  Upon any expiration or termination of this Agreement, the \r\n")
+_T("rights and licenses granted to Licensee under this Agreement shall \r\n")
+_T("immediately terminate, and Licensee shall immediately cease using, and will \r\n")
+_T("return to VMware (or, at VMware's request, destroy), the Software, \r\n")
+_T("Documentation and all other tangible items in Licensee's possession or \r\n")
+_T("control that are proprietary to or contain Confidential Information.  The \r\n")
+_T("rights and obligations of the parties set forth in Sections 1, 2(b) 2(c), \r\n")
+_T("2(d), 2(e), 3, 4, 5, 6 and 7 shall survive termination or expiration of \r\n")
+_T("this Agreement for any reason. \r\n")
+_T(" \r\n")
+_T("4. CONFIDENTIALITY. \"Confidential Information\" shall mean all trade \r\n")
+_T("secrets, know-how, inventions, techniques, processes, algorithms, software \r\n")
+_T("programs, hardware, schematics, planned product features, functionality, \r\n")
+_T("methodology, performance and software source documents relating to the \r\n")
+_T("Software, and other information provided by VMware, whether disclosed \r\n")
+_T("orally, in writing, or by examination or inspection, other than information \r\n")
+_T("which Licensee can demonstrate (i) was already known to Licensee, other \r\n")
+_T("than under an obligation of confidentiality, at the time of disclosure; \r\n")
+_T("(ii) was generally available in the public domain at the time of disclosure \r\n")
+_T("to Licensee; (iii) became generally available in the public domain after \r\n")
+_T("disclosure other than through any act or omission of Licensee; (iv) was \r\n")
+_T("subsequently lawfully disclosed to Licensee by a third party without any \r\n")
+_T("obligation of confidentiality; or (v) was independently developed by \r\n")
+_T("Licensee without use of or reference to any information or materials \r\n")
+_T("disclosed by VMware or its suppliers.  Confidential Information shall \r\n")
+_T("include without limitation the Software, Documentation, Performance Data, \r\n")
+_T("any Updates, information relating to VMware products, product roadmaps, and \r\n")
+_T("other technical, business, financial and product development plans, \r\n")
+_T("forecasts and strategies.  Licensee shall not use any Confidential \r\n")
+_T("Information for any purpose other than as expressly authorized under this \r\n")
+_T("Agreement.  Except as otherwise set forth in this Agreement, in no event \r\n")
+_T("shall Licensee use the Software, Documentation or any other Confidential \r\n")
+_T("Information to develop, manufacture, market, sell, or distribute any \r\n")
+_T("product or service.  Licensee shall limit dissemination of Confidential \r\n")
+_T("Information to its employees who have a need to know such Confidential \r\n")
+_T("Information for purposes expressly authorized under this Agreement.  Except \r\n")
+_T("as otherwise set forth in this Agreement, in no event shall Licensee \r\n")
+_T("disclose any Confidential Information to any third party. Without limiting \r\n")
+_T("the foregoing, Licensee shall use at least the same degree of care that it \r\n")
+_T("uses to prevent the disclosure of its own confidential information of like \r\n")
+_T("importance, but in no event less than reasonable care, to prevent the \r\n")
+_T("disclosure of Confidential Information. \r\n")
+_T(" \r\n")
+_T("5. LIMITATION OF LIABILITY.  IT IS UNDERSTOOD THAT THE SOFTWARE, \r\n")
+_T("DOCUMENTATION AND ANY UPDATES ARE PROVIDED WITHOUT CHARGE FOR THE \r\n")
+_T("PURPOSES OF THIS AGREEMENT ONLY.  ACCORDINGLY, THE TOTAL \r\n")
+_T("LIABILITY OF VMWARE AND ITS SUPPLIERS ARISING OUT OF OR RELATED \r\n")
+_T("TO THIS AGREEMENT SHALL NOT EXCEED $100.  IN NO EVENT SHALL \r\n")
+_T("VMWARE OR ITS SUPPLIERS HAVE LIABILITY FOR ANY INDIRECT, \r\n")
+_T("INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES (INCLUDING, WITHOUT \r\n")
+_T("LIMITATION, DAMAGES FOR LOSS OF BUSINESS PROFITS, BUSINESS \r\n")
+_T("INTERRUPTION, OR LOSS OF BUSINESS INFORMATION), HOWEVER CAUSED \r\n")
+_T("AND ON ANY THEORY OF LIABILITY, EVEN IF VMWARE OR ITS SUPPLIERS \r\n")
+_T("HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.  THESE \r\n")
+_T("LIMITATIONS SHALL APPLY NOTWITHSTANDING ANY FAILURE OF ESSENTIAL \r\n")
+_T("PURPOSE OF ANY LIMITED REMEDY. \r\n")
+_T(" \r\n")
+_T("6. WARRANTY DISCLAIMER.  IT IS UNDERSTOOD THAT THE SOFTWARE, \r\n")
+_T("DOCUMENTATION, AND ANY UPDATES MAY CONTAIN ERRORS AND ARE \r\n")
+_T("PROVIDED FOR THE PURPOSES OF THIS AGREEMENT ONLY.  THE SOFTWARE, \r\n")
+_T("DOCUMENTATION, AND ANY UPDATES ARE PROVIDED \"AS IS\" WITHOUT \r\n")
+_T("WARRANTY OF ANY KIND, WHETHER EXPRESS, IMPLIED, STATUTORY, OR \r\n")
+_T("OTHERWISE. VMWARE AND ITS SUPPLIERS SPECIFICALLY DISCLAIM ALL \r\n")
+_T("IMPLIED WARRANTIES OF MERCHANTABILITY, NONINFRINGEMENT, AND \r\n")
+_T("FITNESS FOR A PARTICULAR PURPOSE.  \r\n")
+_T("Licensee acknowledges that VMware has not publicly \r\n")
+_T("announced the availability of the Software and/or Documentation, that \r\n")
+_T("VMware has not promised or guaranteed to Licensee that such Software and/or \r\n")
+_T("Documentation will be announced or made available to anyone in the future, \r\n")
+_T("that VMware has no express or implied obligation to Licensee to announce or \r\n")
+_T("introduce the Software and/or Documentation, and that VMware may not \r\n")
+_T("introduce a product similar or compatible with the Software and/or \r\n")
+_T("Documentation. Accordingly, Licensee acknowledges that any research or \r\n")
+_T("development that it performs regarding the Software or any product \r\n")
+_T("associated with the Software is done entirely at Licensee's own risk. \r\n")
+_T("Specifically, the Software may contain features, functionality or modules \r\n")
+_T("that may not be included in the generally available version of the Software \r\n")
+_T("and/or Documentation, or that may be marketed separately for additional \r\n")
+_T("fees. \r\n")
+_T(" \r\n")
+_T("7. OTHER PROVISIONS \r\n")
+_T(" \r\n")
+_T("(a) Governing Law.  This Agreement, and all disputes arising out of or \r\n")
+_T("related thereto, shall be governed by and construed under the laws of the \r\n")
+_T("State of California without reference to conflict of laws principles.  All \r\n")
+_T("such disputes shall be subject to the exclusive jurisdiction of the state \r\n")
+_T("and federal courts located in Santa Clara County, California, and the \r\n")
+_T("parties agree and submit to the personal and exclusive jurisdiction and \r\n")
+_T("venue of these courts. \r\n")
+_T(" \r\n")
+_T("(b) Assignment.  Licensee shall not assign this Agreement or any rights or \r\n")
+_T("obligations hereunder, directly or indirectly, by operation of law, merger, \r\n")
+_T("acquisition of stock or assets, or otherwise, without the prior written \r\n")
+_T("consent of VMware.  Subject to the foregoing, this Agreement shall inure to \r\n")
+_T("the benefit of and be binding upon the parties and their respective \r\n")
+_T("successors and permitted assigns. \r\n")
+_T(" \r\n")
+_T("(c) Export Regulations.  Licensee understands that VMware is subject to \r\n")
+_T("regulation by the U.S. government and its agencies, which prohibit export \r\n")
+_T("or diversion of certain technical products and information to certain \r\n")
+_T("countries and individuals.  Licensee warrants that it will comply in all \r\n")
+_T("respects with all export and re-export restrictions applicable to the \r\n")
+_T("technology and documentation provided hereunder. \r\n")
+_T(" \r\n")
+_T("(d) Entire Agreement.  This is the entire agreement between the parties \r\n")
+_T("relating to the subject matter hereof and all other terms are rejected. \r\n")
+_T("This Agreement supersedes all previous communications, representations, \r\n")
+_T("understandings and agreements, either oral or written, between the parties \r\n")
+_T("with respect to said subject matter.  The terms of this Agreement supersede \r\n")
+_T("any VMware end user license agreement that may accompany the Software \r\n")
+_T("and/or Documentation.  No waiver or modification of this Agreement shall be \r\n")
+_T("valid unless made in a writing signed by both parties.  The waiver of a \r\n")
+_T("breach of any term hereof shall in no way be construed as a waiver of any \r\n")
+_T("term or other breach hereof.  If any provision of this Agreement is held by \r\n")
+_T("a court of competent jurisdiction to be contrary to law the remaining \r\n")
+_T("provisions of this Agreement shall remain in full force and effect. \r\n")
+_T(" \r\n")
+_T("(e) Notices. All notices must be sent by (a) registered or certified mail, \r\n")
+_T("return receipt requested, (b) reputable overnight air courier, (c) \r\n")
+_T("facsimile with a confirmation copy sent by registered or certified mail, \r\n")
+_T("return receipt requested, or (d) served personally.  Notices are effective \r\n")
+_T("immediately when served personally, five (5) days after posting if sent by \r\n")
+_T("registered or certified mail, two (2) days after being sent by overnight \r\n")
+_T("courier, or one (1) day after being transmitted by facsimile.  Notices to \r\n")
+_T("either party shall be directed to the party's address set forth in this \r\n")
+_T("Agreement.  Either party may change its address for notification under this \r\n")
+_T("Agreement, by notifying the other party in accordance with this Section. \r\n")
+_T(" \r\n");
+
+
+/////////////////////////////////////////////////////////////////////////////
+// CLicenseDlg property page
+
+IMPLEMENT_DYNCREATE(CLicenseDlg, CPropertyPage)
+
+CLicenseDlg::CLicenseDlg() : CPropertyPage(CLicenseDlg::IDD)
+{
+    //{{AFX_DATA_INIT(CLicenseDlg)
+    m_License = _T("");
+    //}}AFX_DATA_INIT
+    m_psp.dwFlags |= PSP_HIDEHEADER;
+    m_psp.dwFlags &= ~(PSP_HASHELP);
+}
+
+CLicenseDlg::~CLicenseDlg()
+{
+}
+
+void CLicenseDlg::DoDataExchange(CDataExchange* pDX)
+{
+    CPropertyPage::DoDataExchange(pDX);
+    //{{AFX_DATA_MAP(CLicenseDlg)
+    DDX_Control(pDX, IDC_LICENSE, m_LicenseEdit);
+    DDX_Text(pDX, IDC_LICENSE, m_License);
+    //}}AFX_DATA_MAP
+}
+
+
+BEGIN_MESSAGE_MAP(CLicenseDlg, CPropertyPage)
+    //{{AFX_MSG_MAP(CLicenseDlg)
+    ON_BN_CLICKED(IDC_AGREE, OnAgree)
+    ON_BN_CLICKED(IDC_DISAGREE, OnDisagree)
+    ON_BN_CLICKED(IDC_UNDERSTAND, OnUnderstand)
+    //}}AFX_MSG_MAP
+END_MESSAGE_MAP()
+
+/////////////////////////////////////////////////////////////////////////////
+// CLicenseDlg message handlers
+
+
+BOOL CLicenseDlg::OnInitDialog()
+{
+    CDialog::OnInitDialog();
+    m_FirstTime = TRUE;
+    
+    m_License.Format(_T("%s"), LICENSE);
+    m_LicenseEdit.SetSel(0,0);
+    UpdateData(FALSE); // write to screen
+
+    /* I had a hard time setting up the scroll-to-bottom-before-can-accept
+     * functionality: I tried using a separate CScrollBar widget, like I ended
+     * up doing for the DRgui stats viewer, and I got that working for scrolling
+     * with the mouse (had to manually scroll the license text via LineScroll(),
+     * and set the scroll parameters after calculating using a RECT for the
+     * CEdit dimensions),
+     * but I never was able to intercept keyboard scrolling, even trying:
+        ON_WM_KEYUP()
+        ON_WM_KEYDOWN()
+        ON_EN_UPDATE(IDC_LICENSE, OnUpdateLicense)
+     * But I found a hint online that all you have to do is subclass CEdit, so that's
+     * what I do now, using LicEdit.
+     */
+
+    return TRUE;
+}
+
+BOOL CLicenseDlg::OnSetActive()
+{
+    if (!CheckWindowsVersion()) {
+        CPropertyPage::EndDialog(IDCANCEL);
+    }
+    
+    CPropertySheet* pSheet = (CPropertySheet*)GetParent();
+    ASSERT_KINDOF(CPropertySheet, pSheet);
+    if (m_FirstTime) {
+        m_FirstTime = FALSE;
+        
+        //  Disable everything until the user scrolls the license text
+        CButton *radio_all = (CButton *) GetDlgItem(IDC_DISAGREE);
+        radio_all->SetCheck(1);
+        CButton *understand = (CButton *) GetDlgItem(IDC_UNDERSTAND);
+        understand->SetCheck(BST_UNCHECKED);
+        OnUnderstand();
+        understand->EnableWindow(FALSE);
+        pSheet->SetWizardButtons(0);
+        UpdateData(FALSE); // write to screen
+    } else {
+        pSheet->SetWizardButtons(PSWIZB_NEXT);
+    }
+    return CPropertyPage::OnSetActive();
+}
+
+BOOL CLicenseDlg::CheckWindowsVersion()
+{
+    // make sure we're on an NT-based system
+    TCHAR bad_os[64];
+    OSVERSIONINFO version;
+    version.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
+    int res = GetVersionEx(&version);
+    assert(res != 0);
+    if (version.dwPlatformId == VER_PLATFORM_WIN32_NT) {
+        // WinNT or descendents: rather than continually update
+        // the list of known versions here we assume they're all ok.
+        if (version.dwMajorVersion == 4) {
+            // Windows NT
+        } else {
+            // 2K or later
+        }
+        return TRUE;
+    } else if (version.dwPlatformId == VER_PLATFORM_WIN32_WINDOWS) {
+        /* Win95 or Win98 */
+        unsigned ver_high = (version.dwBuildNumber >> 24) & 0xff;
+        unsigned ver_low = (version.dwBuildNumber >> 16) & 0xff;
+        if (ver_low >= 90 || ver_high >= 5)
+            _stprintf(bad_os, _T("Windows ME"));
+        else if (ver_low >= 10 && ver_low < 90)
+            _stprintf(bad_os, _T("Windows 98"));
+        else if (ver_low < 5)
+            _stprintf(bad_os, _T("Windows 31 / WfWg"));
+        else if (ver_low < 10)
+            _stprintf(bad_os, _T("Windows 98"));
+        else
+            assert(false);
+    } else {
+        /* Win32S on Windows 3.1 */
+        _stprintf(bad_os, _T("Win32s"));
+    }
+    TCHAR msg[128];
+    _stprintf(msg, _T("DynamoRIO does not support %s"), bad_os);
+    MessageBox(msg, _T("Fatal Error"), MB_OK | MYMBFLAGS);
+    return FALSE;
+}
+
+void CLicenseDlg::OnAgree() 
+{
+    CPropertySheet* pSheet = (CPropertySheet*)GetParent();
+    ASSERT_KINDOF(CPropertySheet, pSheet);
+    // enable Next
+    pSheet->SetWizardButtons(PSWIZB_NEXT);
+}
+
+void CLicenseDlg::OnDisagree() 
+{
+    CPropertySheet* pSheet = (CPropertySheet*)GetParent();
+    ASSERT_KINDOF(CPropertySheet, pSheet);
+    // disable Next
+    pSheet->SetWizardButtons(0);
+}
+
+void CLicenseDlg::OnUnderstand() 
+{
+    CPropertySheet* pSheet = (CPropertySheet*)GetParent();
+    ASSERT_KINDOF(CPropertySheet, pSheet);
+    CButton *button = (CButton *) GetDlgItem(IDC_UNDERSTAND);
+    if (button->GetCheck()) {
+        // enable Radio Buttons
+        GetDlgItem(IDC_AGREE)->EnableWindow(TRUE);
+        GetDlgItem(IDC_DISAGREE)->EnableWindow(TRUE);
+    } else {
+        // disable Next and Radio Buttons
+        pSheet->SetWizardButtons(0);
+        // disable Radio Buttons
+        GetDlgItem(IDC_AGREE)->EnableWindow(FALSE);
+        GetDlgItem(IDC_DISAGREE)->EnableWindow(FALSE);
+    }
+}
+
+
+/////////////////////////////////////////////////////////////////////////////
+// LicEdit
+
+BEGIN_MESSAGE_MAP(LicEdit, CEdit)
+//{{AFX_MSG_MAP(LicEdit)
+ON_WM_VSCROLL()
+ON_CONTROL_REFLECT(EN_VSCROLL, OnEnVscroll)
+//}}AFX_MSG_MAP
+END_MESSAGE_MAP()
+
+void LicEdit::OnVScroll(UINT nSBCode, UINT nPos, CScrollBar* pScrollBar)
+{
+    // TODO: Add your message handler code here and/or call default
+    SCROLLINFO info;
+    info.cbSize = sizeof(SCROLLINFO);
+    GetScrollInfo(SB_VERT, &info, SIF_RANGE | SIF_POS | SIF_PAGE);
+
+    if (info.nPos + info.nPage > (unsigned int) info.nMax) {
+        // If user scrolls to bottom, enable moving on
+        GetParent()->GetDlgItem(IDC_UNDERSTAND)->EnableWindow(TRUE);
+    }
+
+    CEdit::OnVScroll(nSBCode, nPos, pScrollBar);
+}
+
+void LicEdit::OnEnVscroll( )
+{
+    SCROLLINFO info;
+    info.cbSize = sizeof(SCROLLINFO);
+    GetScrollInfo(SB_VERT, &info, SIF_RANGE | SIF_POS | SIF_PAGE);
+
+    if (info.nPos + info.nPage > (unsigned int) info.nMax) {
+        // If user scrolls to bottom, enable moving on
+        GetParent()->GetDlgItem(IDC_UNDERSTAND)->EnableWindow(TRUE);
+    }
+}
diff --git a/tools/DRinstall/LicenseDlg.h b/tools/DRinstall/LicenseDlg.h
new file mode 100644
index 000000000..69cd6f0e2
--- /dev/null
+++ b/tools/DRinstall/LicenseDlg.h
@@ -0,0 +1,102 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+#ifndef _LICENSE_DLG_H_
+#define _LICENSE_DLG_H_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif // _MSC_VER > 1000
+
+/////////////////////////////////////////////////////////////////////////////
+// LicEdit
+
+class LicEdit : public CEdit
+{
+    void OnVScroll(UINT nSBCode, UINT nPos, CScrollBar* pScrollBar);
+    void OnEnVscroll();
+    DECLARE_MESSAGE_MAP()
+};
+
+/////////////////////////////////////////////////////////////////////////////
+// CLicenseDlg dialog
+
+class CLicenseDlg : public CPropertyPage
+{
+    DECLARE_DYNCREATE(CLicenseDlg)
+
+// Construction
+public:
+    BOOL CheckWindowsVersion();
+    BOOL OnInitDialog();
+    CLicenseDlg();
+    ~CLicenseDlg();
+
+// Dialog Data
+    //{{AFX_DATA(CLicenseDlg)
+    enum { IDD = IDD_LICENSE };
+    LicEdit m_LicenseEdit;
+    CString m_License;
+    //}}AFX_DATA
+
+
+// Overrides
+    // ClassWizard generate virtual function overrides
+    //{{AFX_VIRTUAL(CLicenseDlg)
+public:
+    virtual BOOL OnSetActive();
+protected:
+    virtual void DoDataExchange(CDataExchange* pDX);    // DDX/DDV support
+    //}}AFX_VIRTUAL
+
+// Implementation
+protected:
+    BOOL m_FirstTime;
+    void UpdateScrollbar();
+    // Generated message map functions
+    //{{AFX_MSG(CLicenseDlg)
+    afx_msg void OnAgree();
+    afx_msg void OnDisagree();
+    afx_msg void OnUnderstand();
+    //}}AFX_MSG
+    DECLARE_MESSAGE_MAP()
+
+};
+
+//{{AFX_INSERT_LOCATION}}
+// Microsoft Developer Studio will insert additional declarations immediately before the previous line.
+
+#endif // _LICENSE_DLG_H_
diff --git a/tools/DRinstall/Makefile b/tools/DRinstall/Makefile
new file mode 100644
index 000000000..34c1900de
--- /dev/null
+++ b/tools/DRinstall/Makefile
@@ -0,0 +1,127 @@
+# **********************************************************
+# Copyright (c) 2008 VMware, Inc.  All rights reserved.
+# **********************************************************/
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+ifndef ARCH
+# default is x86 instead of x64
+  ARCH := x86
+endif
+
+ifndef DYNAMORIO_MAKE
+  DYNAMORIO_MAKE := ../../make
+endif
+
+# We would stick w/ VC60 for compatibility w/ NT, to match DRgui.exe (PR 232385),
+# but it runs out of memory processing zipdump.h, and /Zm1000 doesn't even
+# get halfway (/Zm1500 and higher don't seem to work).
+# We go w/ 7.1 instead of 8 to avoid fixing the warnings that 8 has.
+# Using static libs (/MT, and no _AFXDLL define), this does run on 2K,
+# so only NT is out of luck.
+USE_VC71 := 1
+
+include $(DYNAMORIO_MAKE)/compiler.mk
+
+# use DYNAMORIO_HOME if possible
+ifndef DYNAMORIO_HOME
+  DYNAMORIO_HOME := ../..
+endif
+WIN_DYNAMORIO_HOME := $(shell $(CYGPATH) -ma ${DYNAMORIO_HOME})
+
+ifndef DYNAMORIO_LIBUTIL
+  DYNAMORIO_LIBUTIL := ../../libutil
+endif
+WIN_DYNAMORIO_LIBUTIL := $(shell $(CYGPATH) -ma ${DYNAMORIO_LIBUTIL})
+
+DEFINES := /D"WIN32" /D"WINDOWS" /D"_WINDOWS"
+
+ifeq ($(ARCH), x64)
+  DEFINES += /D"X64"
+endif
+
+LDFLAGS := $(MSLIB) advapi32.lib /SUBSYSTEM:WINDOWS /MACHINE:X86
+
+ifdef USE_VC60
+  DEFINES += /D"USE_VC60"
+endif
+
+ifdef DEBUG
+  CFLAGS += /Zi /Od /MTd
+  DEFINES += /D"_DEBUG"
+  LDFLAGS += /debug
+  BUILDDIR := build_dbg
+else
+  CFLAGS += /O2 /MT
+  BUILDDIR := build_rel
+endif
+
+INCLUDES := $(MSINCLUDE) /I$(BUILDDIR)
+
+WARN := /W4 /WX
+
+CFLAGS += $(WARN) $(DEFINES) $(INCLUDES)
+
+EXEBASE := DRinstall
+HEADERS := $(wildcard *.h)
+SRCS := $(wildcard *.cpp)
+OBJS := $(patsubst %.cpp,$(BUILDDIR)/%.obj,$(SRCS)) 
+RES_SRCS := $(wildcard *.rc)
+RES_OBJS := $(patsubst %.rc,$(BUILDDIR)/%.res,$(RES_SRCS)) 
+
+# To have the files not accessible until the license is viewed, and to
+# avoid distributing a zip utility or building a self-extracting zip
+# file that has password support, we simply hide the zip file inside
+# our executable data section.
+ZIPFILE := dynamorio.zip
+ZIPDUMP := $(BUILDDIR)/zipdump.h
+
+.PHONY: clean all check
+
+all: $(BUILDDIR) $(ZIPDUMP) $(BUILDDIR)/$(EXEBASE).exe
+
+$(ZIPDUMP): $(ZIPFILE)
+	$(OD) -v -t x1 $(ZIPFILE) | $(PERL) -p -e 's|^\w+\s+(.*)$$|\1 |; s|(\w+) +|0x\1,|g;' > $@
+
+$(ZIPFILE):
+	@(if ! test -e $(ZIPFILE); then \
+	  $(ECHO) "ERROR: must point at zip file with ZIPFILE="; false; fi)
+
+$(BUILDDIR): 
+	@(if ! test -d $(BUILDDIR); then $(MKDIR) -p $(BUILDDIR); fi)
+
+$(BUILDDIR)/$(EXEBASE).exe: $(OBJS) $(RES_OBJS) $(POLICY_LIB)
+	$(LD) $(LDFLAGS) $(OBJS) $(RES_OBJS) $(WIN_POLICY_LIB) /out:$@ /pdb:$(BUILDDIR)/$(EXEBASE).pdb
+
+$(BUILDDIR)/%.obj: %.cpp $(HEADERS)
+	$(CC) $(CFLAGS) /c $< /Fo$@ /Fd$(subst .obj,.pdb,$@)
+
+$(BUILDDIR)/%.res: %.rc
+	$(RC) /r /fo $@ $(INCLUDES) $<
+
+clean: 
+	$(RM) -r $(BUILDDIR)
diff --git a/tools/DRinstall/ShellInterface.cpp b/tools/DRinstall/ShellInterface.cpp
new file mode 100644
index 000000000..6d670bd4e
--- /dev/null
+++ b/tools/DRinstall/ShellInterface.cpp
@@ -0,0 +1,232 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+#include "stdafx.h"
+#include "wizard.h"
+#include "ShellInterface.h"
+#include "assert.h"
+
+#include <shlobj.h>
+#include <objbase.h>
+#include <shlwapi.h>
+
+#ifdef _DEBUG
+#undef THIS_FILE
+static char THIS_FILE[]=__FILE__;
+#define new DEBUG_NEW
+#endif
+
+//////////////////////////////////////////////////////////////////////
+// Construction/Destruction
+//////////////////////////////////////////////////////////////////////
+
+CShellInterface::CShellInterface()
+{
+
+}
+
+CShellInterface::~CShellInterface()
+{
+
+}
+
+/*static */BOOL CShellInterface::m_bInitialized = FALSE;
+
+/*static */void CShellInterface::Initialize()
+{
+    if (m_bInitialized)
+        return;
+    int res = CoInitialize(NULL);
+    assert(res == S_OK);
+    m_bInitialized = TRUE;
+}
+
+/*static */void CShellInterface::Uninitialize()
+{
+    if (!m_bInitialized)
+        return;
+    CoUninitialize();
+    m_bInitialized = FALSE;
+}
+
+/*static*/ BOOL CShellInterface::CreateLinkFile(LPCTSTR pszShortcutFile, 
+                                                LPTSTR pszLink, LPTSTR pszDesc)
+{
+    HRESULT hres;
+    IShellLink *psl;
+        
+    // Create an IShellLink object and get a pointer to the IShellLink 
+    // interface (returned from CoCreateInstance).
+    hres = CoCreateInstance (CLSID_ShellLink, NULL, CLSCTX_INPROC_SERVER,
+                             IID_IShellLink, (void **)&psl);
+    if (SUCCEEDED (hres)) {
+        IPersistFile *ppf;
+                
+        // Query IShellLink for the IPersistFile interface for 
+        // saving the shortcut in persistent storage.
+        hres = psl->QueryInterface (IID_IPersistFile, (void **)&ppf);
+        if (SUCCEEDED (hres)) { 
+            // Set the path to the shortcut target.
+            hres = psl->SetPath (pszShortcutFile);
+                        
+            if (! SUCCEEDED (hres))
+                AfxMessageBox (_T("SetPath failed!"));
+                        
+            // Set the description of the shortcut.
+            hres = psl->SetDescription (pszDesc);
+                        
+            if (! SUCCEEDED (hres))
+                AfxMessageBox (_T("SetDescription failed!"));
+                        
+#ifndef UNICODE
+            // Ensure that the string consists of ANSI characters.
+            WORD wsz [MAX_PATH]; // buffer for Unicode string
+            MultiByteToWideChar (CP_ACP, 0, pszLink, -1, wsz, MAX_PATH);
+            // Save the shortcut via the IPersistFile::Save member function.
+            hres = ppf->Save (wsz, TRUE);
+#else
+            // Save the shortcut via the IPersistFile::Save member function.
+            hres = ppf->Save (pszLink, TRUE);
+#endif
+
+                        
+            if (! SUCCEEDED (hres))
+                AfxMessageBox (_T("Save failed!"));
+                        
+            // Release the pointer to IPersistFile.
+            ppf->Release ();
+        }
+        // Release the pointer to IShellLink.
+        psl->Release ();
+    }
+    return (SUCCEEDED(hres));
+}
+
+
+/*static*/ BOOL CShellInterface::CopyDir(LPCTSTR from, LPCTSTR to, HWND hwnd)
+{
+    assert(_tcslen(from) < MAX_PATH-1);
+    assert(_tcslen(to) < MAX_PATH-1);
+    // strings must end with pair of nulls
+    TCHAR myfrom[MAX_PATH+2];
+    _tcscpy(myfrom, from);
+    myfrom[_tcslen(myfrom)+1] = _T('\0');
+    TCHAR myto[MAX_PATH+2];
+    _tcscpy(myto, to);
+    myto[_tcslen(myto)+1] = _T('\0');
+    SHFILEOPSTRUCT fileop;
+    fileop.hwnd = hwnd;
+    fileop.wFunc = FO_COPY;
+    fileop.pFrom = myfrom;
+    fileop.pTo = myto;
+    fileop.fFlags = FOF_NOCONFIRMMKDIR | FOF_NOCONFIRMATION | FOF_SILENT;
+    int res = SHFileOperation(&fileop);
+    return (res == 0);
+}
+
+/*static*/ BOOL CShellInterface::DeleteFile(LPCTSTR name, HWND hwnd)
+{
+    assert(_tcslen(name) < MAX_PATH-1);
+    // string must end with pair of nulls
+    TCHAR myname[MAX_PATH+2];
+    _tcscpy(myname, name);
+    myname[_tcslen(myname)+1] = _T('\0');
+    SHFILEOPSTRUCT fileop;
+    fileop.hwnd = hwnd;
+    fileop.wFunc = FO_DELETE;
+    fileop.pFrom = myname;
+    fileop.pTo = NULL;
+    fileop.fFlags = FOF_ALLOWUNDO | FOF_NOCONFIRMATION | FOF_SILENT;
+    int res = SHFileOperation(&fileop);
+    return (res == 0);
+}
+
+
+#if 0
+/* From MSDN's MFC sample:
+   "SHORTCUT: A SampleThat Manipulates Shortcuts"
+   except that sample had ansi strings
+*/
+
+HRESULT CreateShortCut::CreateIt (LPCSTR pszShortcutFile, LPSTR pszLink, 
+                                  LPSTR pszDesc)
+{
+    HRESULT hres;
+    IShellLink *psl;
+        
+    // Create an IShellLink object and get a pointer to the IShellLink 
+    // interface (returned from CoCreateInstance).
+    hres = CoCreateInstance (CLSID_ShellLink, NULL, CLSCTX_INPROC_SERVER,
+                             IID_IShellLink, (void **)&psl);
+    if (SUCCEEDED (hres)) {
+        IPersistFile *ppf;
+                
+        // Query IShellLink for the IPersistFile interface for 
+        // saving the shortcut in persistent storage.
+        hres = psl->QueryInterface (IID_IPersistFile, (void **)&ppf);
+        if (SUCCEEDED (hres)) { 
+            WORD wsz [MAX_PATH]; // buffer for Unicode string
+                        
+            // Set the path to the shortcut target.
+            hres = psl->SetPath (pszShortcutFile);
+                        
+            if (! SUCCEEDED (hres))
+                AfxMessageBox (_T("SetPath failed!"));
+                        
+            // Set the description of the shortcut.
+            hres = psl->SetDescription (pszDesc);
+                        
+            if (! SUCCEEDED (hres))
+                AfxMessageBox (_T("SetDescription failed!"));
+                        
+            // Ensure that the string consists of ANSI characters.
+            MultiByteToWideChar (CP_ACP, 0, pszLink, -1, wsz, MAX_PATH);
+                        
+            // Save the shortcut via the IPersistFile::Save member function.
+            hres = ppf->Save (wsz, TRUE);
+                        
+            if (! SUCCEEDED (hres))
+                AfxMessageBox (“Save failed!”);
+                        
+            // Release the pointer to IPersistFile.
+            ppf->Release ();
+        }
+        // Release the pointer to IShellLink.
+        psl->Release ();
+    }
+    return hres;
+} 
+#endif
diff --git a/tools/DRinstall/ShellInterface.h b/tools/DRinstall/ShellInterface.h
new file mode 100644
index 000000000..c28fb41c5
--- /dev/null
+++ b/tools/DRinstall/ShellInterface.h
@@ -0,0 +1,59 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+#ifndef _SHELL_INTERFACE_H
+#define _SHELL_INTERFACE_H
+
+#if _MSC_VER > 1000
+#pragma once
+#endif // _MSC_VER > 1000
+
+class CShellInterface  
+{
+public:
+    CShellInterface();
+    virtual ~CShellInterface();
+    
+    static void Initialize();
+    static void Uninitialize();
+    static BOOL CreateLinkFile(LPCTSTR pszShortcutFile, LPTSTR pszLink, LPTSTR pszDesc);
+    static BOOL CopyDir(LPCTSTR from, LPCTSTR to, HWND hwnd);
+    static BOOL DeleteFile(LPCTSTR name, HWND hwnd);
+    
+    static BOOL m_bInitialized;
+};
+
+#endif // _SHELL_INTERFACE_H
diff --git a/tools/DRinstall/StdAfx.cpp b/tools/DRinstall/StdAfx.cpp
new file mode 100644
index 000000000..7f3aec600
--- /dev/null
+++ b/tools/DRinstall/StdAfx.cpp
@@ -0,0 +1,41 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// stdafx.cpp : source file that includes just the standard includes
+//  Wizard.pch will be the pre-compiled header
+//  stdafx.obj will contain the pre-compiled type information
+
+#include "stdafx.h"
diff --git a/tools/DRinstall/StdAfx.h b/tools/DRinstall/StdAfx.h
new file mode 100644
index 000000000..b8c8dff00
--- /dev/null
+++ b/tools/DRinstall/StdAfx.h
@@ -0,0 +1,64 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+// stdafx.h : include file for standard system include files,
+//  or project specific include files that are used frequently, but
+//      are changed infrequently
+//
+
+#ifndef _STDAFX_H_
+#define _STDAFX_H_
+
+#define _WIN32_WINNT 0x0400 
+
+#if _MSC_VER > 1000
+#pragma once
+#endif // _MSC_VER > 1000
+
+#define VC_EXTRALEAN        // Exclude rarely-used stuff from Windows headers
+
+#include <afxwin.h>         // MFC core and standard components
+#include <afxext.h>         // MFC extensions
+#include <afxdtctl.h>       // MFC support for Internet Explorer 4 Comon Controls
+#ifndef _AFX_NO_AFXCMN_SUPPORT
+#include <afxcmn.h>         // MFC support for Windows Common Controls
+#endif // _AFX_NO_AFXCMN_SUPPORT
+
+
+//{{AFX_INSERT_LOCATION}}
+// Microsoft Visual C++ will insert additional declarations immediately before the previous line.
+
+#endif // _STDAFX_H_
diff --git a/tools/DRinstall/WizSheet.cpp b/tools/DRinstall/WizSheet.cpp
new file mode 100644
index 000000000..aee89f3de
--- /dev/null
+++ b/tools/DRinstall/WizSheet.cpp
@@ -0,0 +1,89 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+#include "stdafx.h"
+#include "Wizard.h"
+#include "WizSheet.h"
+
+#ifdef _DEBUG
+#define new DEBUG_NEW
+#undef THIS_FILE
+static char THIS_FILE[] = __FILE__;
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+// CSampWizP
+
+IMPLEMENT_DYNAMIC(CWizardSheet, CPropertySheet)
+
+CWizardSheet::CWizardSheet(UINT nIDCaption, CWnd* pParentWnd,
+                           UINT iSelectPage)
+    : CPropertySheet(nIDCaption, pParentWnd, iSelectPage)
+{
+    // add all the pages of the wizard
+    AddPage(&m_License);
+    AddPage(&m_Copy);
+    AddPage(&m_Final);
+    m_psh.dwFlags |= PSH_WIZARD;
+    m_psh.dwFlags &= ~(PSH_HASHELP);
+}
+
+CWizardSheet::CWizardSheet(LPCTSTR pszCaption, CWnd* pParentWnd,
+                           UINT iSelectPage)
+    : CPropertySheet(pszCaption, pParentWnd, iSelectPage)
+
+{
+    // add all the pages of the wizard
+    AddPage(&m_License);
+    AddPage(&m_Copy);
+    AddPage(&m_Final);
+    m_psh.dwFlags |= PSH_WIZARD;
+    m_psh.dwFlags &= ~(PSH_HASHELP);
+}
+
+CWizardSheet::~CWizardSheet()
+{
+}
+
+
+BEGIN_MESSAGE_MAP(CWizardSheet, CPropertySheet)
+    //{{AFX_MSG_MAP(CWizardSheet)
+    // NOTE - the ClassWizard will add and remove mapping macros here.
+    //}}AFX_MSG_MAP
+END_MESSAGE_MAP()
+
+/////////////////////////////////////////////////////////////////////////////
+// CWizardSheet message handlers
diff --git a/tools/DRinstall/WizSheet.h b/tools/DRinstall/WizSheet.h
new file mode 100644
index 000000000..8663a3735
--- /dev/null
+++ b/tools/DRinstall/WizSheet.h
@@ -0,0 +1,95 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+#ifndef _WIZ_SHEET_H_
+#define _WIZ_SHEET_H_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif // _MSC_VER > 1000
+
+#include "LicenseDlg.h"
+#include "CopyDlg.h"
+#include "FinalDlg.h"
+
+/////////////////////////////////////////////////////////////////////////////
+// CWizardSheet
+
+class CWizardSheet : public CPropertySheet
+{
+    DECLARE_DYNAMIC(CWizardSheet)
+
+// Construction
+public:
+    CWizardSheet(UINT nIDCaption, CWnd* pParentWnd = NULL,
+                 UINT iSelectPage = 0);
+    CWizardSheet(LPCTSTR pszCaption, CWnd* pParentWnd = NULL,
+                 UINT iSelectPage = 0);
+
+// Attributes
+public:
+    CLicenseDlg m_License;
+    CCopyDlg m_Copy;
+    CFinalDlg m_Final;
+
+// Operations
+public:
+
+// Overrides
+    // ClassWizard generated virtual function overrides
+    //{{AFX_VIRTUAL(CWizardSheet)
+    //}}AFX_VIRTUAL
+
+// Implementation
+public:
+    // used for communication of install dir between pages
+    CString m_InstallDir;
+    virtual ~CWizardSheet();
+
+    // Generated message map functions
+protected:
+    //{{AFX_MSG(CWizardSheet)
+    // NOTE - the ClassWizard will add and remove member functions here.
+    //}}AFX_MSG
+    DECLARE_MESSAGE_MAP()
+};
+
+/////////////////////////////////////////////////////////////////////////////
+
+//{{AFX_INSERT_LOCATION}}
+// Microsoft Developer Studio will insert additional declarations immediately before the previous line.
+
+#endif // _WIZ_SHEET_H_
diff --git a/tools/DRinstall/Wizard.cpp b/tools/DRinstall/Wizard.cpp
new file mode 100644
index 000000000..ddade4c72
--- /dev/null
+++ b/tools/DRinstall/Wizard.cpp
@@ -0,0 +1,106 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+#include "stdafx.h"
+#include "Wizard.h"
+#include "WizSheet.h"
+
+#ifdef _DEBUG
+#define new DEBUG_NEW
+#undef THIS_FILE
+static char THIS_FILE[] = __FILE__;
+#endif
+
+/////////////////////////////////////////////////////////////////////////////
+// CWizardApp
+
+BEGIN_MESSAGE_MAP(CWizardApp, CWinApp)
+    //{{AFX_MSG_MAP(CWizardApp)
+    // NOTE - the ClassWizard will add and remove mapping macros here.
+    //    DO NOT EDIT what you see in these blocks of generated code!
+    //}}AFX_MSG
+    ON_COMMAND(ID_HELP, CWinApp::OnHelp)
+END_MESSAGE_MAP()
+
+/////////////////////////////////////////////////////////////////////////////
+// CWizardApp construction
+
+CWizardApp::CWizardApp()
+{
+    // TODO: add construction code here,
+    // Place all significant initialization in InitInstance
+}
+
+/////////////////////////////////////////////////////////////////////////////
+// The one and only CWizardApp object
+
+CWizardApp theApp;
+
+/////////////////////////////////////////////////////////////////////////////
+// CWizardApp initialization
+
+BOOL CWizardApp::InitInstance()
+{
+    // Standard initialization
+    // If you are not using these features and wish to reduce the size
+    //  of your final executable, you should remove from the following
+    //  the specific initialization routines you do not need.
+
+#ifdef USE_VC60 /* deprecated in VC7.1: not needed anymore */
+# ifdef _AFXDLL
+    Enable3dControls();         // Call this when using MFC in a shared DLL
+# else
+    Enable3dControlsStatic();   // Call this when linking to MFC statically
+# endif
+#endif
+
+    CWizardSheet dlg(IDS_SAMPLEWIZARD, NULL, 0);
+    dlg.m_psh.hInstance = ::GetModuleHandle(NULL);
+    m_pMainWnd = &dlg;
+    int nResponse = dlg.DoModal();
+    if (nResponse == IDOK) {
+        // TODO: Place code here to handle when the dialog is
+        //  dismissed with OK
+    }
+    else if (nResponse == IDCANCEL) {
+        // TODO: Place code here to handle when the dialog is
+        //  dismissed with Cancel
+    }
+
+    // Since the dialog has been closed, return FALSE so that we exit the
+    //  application, rather than start the application's message pump.
+    return FALSE;
+}
diff --git a/tools/DRinstall/Wizard.h b/tools/DRinstall/Wizard.h
new file mode 100644
index 000000000..fd0decee8
--- /dev/null
+++ b/tools/DRinstall/Wizard.h
@@ -0,0 +1,84 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2003-2007 Determina Corp. */
+/* Copyright (c) 2002-2003 Massachusetts Institute of Technology */
+/* Copyright (c) 2002 Hewlett-Packard Company */
+
+#ifndef _WIZARD_H_
+#define _WIZARD_H_
+
+#if _MSC_VER > 1000
+#pragma once
+#endif // _MSC_VER > 1000
+
+#ifndef __AFXWIN_H__
+        #error include 'stdafx.h' before including this file for PCH
+#endif
+
+#include "resource.h"       // main symbols
+
+#define MYMBFLAGS   MB_TOPMOST | MB_ICONEXCLAMATION
+
+/////////////////////////////////////////////////////////////////////////////
+// CWizardApp:
+// See Wizard.cpp for the implementation of this class
+//
+
+class CWizardApp : public CWinApp
+{
+public:
+    CWizardApp();
+
+// Overrides
+    // ClassWizard generated virtual function overrides
+    //{{AFX_VIRTUAL(CWizardApp)
+public:
+    virtual BOOL InitInstance();
+    //}}AFX_VIRTUAL
+
+// Implementation
+
+    //{{AFX_MSG(CWizardApp)
+    // NOTE - the ClassWizard will add and remove member functions here.
+    //    DO NOT EDIT what you see in these blocks of generated code !
+    //}}AFX_MSG
+    DECLARE_MESSAGE_MAP()
+};
+
+
+/////////////////////////////////////////////////////////////////////////////
+
+//{{AFX_INSERT_LOCATION}}
+// Microsoft Visual C++ will insert additional declarations immediately before the previous line.
+
+#endif // _WIZARD_H_
diff --git a/tools/DRinstall/Wizard.rc b/tools/DRinstall/Wizard.rc
new file mode 100644
index 000000000..141f9688d
--- /dev/null
+++ b/tools/DRinstall/Wizard.rc
@@ -0,0 +1,269 @@
+// **********************************************************
+// Copyright (c) 2008 VMware, Inc.  All rights reserved.
+// **********************************************************
+
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// 
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+// 
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the following disclaimer in the documentation
+//   and/or other materials provided with the distribution.
+// 
+// * Neither the name of VMware, Inc. nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+// DAMAGE.
+
+//Microsoft Developer Studio generated resource script.
+//
+#include "resource.h"
+
+#define APSTUDIO_READONLY_SYMBOLS
+/////////////////////////////////////////////////////////////////////////////
+//
+// Generated from the TEXTINCLUDE 2 resource.
+//
+#include "afxres.h"
+
+/////////////////////////////////////////////////////////////////////////////
+#undef APSTUDIO_READONLY_SYMBOLS
+
+/////////////////////////////////////////////////////////////////////////////
+// English (U.S.) resources
+
+#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
+#ifdef _WIN32
+LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US
+#pragma code_page(1252)
+#endif //_WIN32
+
+#ifdef APSTUDIO_INVOKED
+/////////////////////////////////////////////////////////////////////////////
+//
+// TEXTINCLUDE
+//
+
+1 TEXTINCLUDE DISCARDABLE 
+BEGIN
+    "resource.h\0"
+END
+
+2 TEXTINCLUDE DISCARDABLE 
+BEGIN
+    "#include ""afxres.h""\r\n"
+    "\0"
+END
+
+3 TEXTINCLUDE DISCARDABLE 
+BEGIN
+    "#define _AFX_NO_SPLITTER_RESOURCES\r\n"
+    "#define _AFX_NO_OLE_RESOURCES\r\n"
+    "#define _AFX_NO_TRACKER_RESOURCES\r\n"
+    "#define _AFX_NO_PROPERTY_RESOURCES\r\n"
+    "\r\n"
+    "#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)\r\n"
+    "#ifdef _WIN32\r\n"
+    "LANGUAGE 9, 1\r\n"
+    "#pragma code_page(1252)\r\n"
+    "#endif //_WIN32\r\n"
+    "#include ""res\\Wizard.rc2""  // non-Microsoft Visual C++ edited resources\r\n"
+    "#include ""afxres.rc""         // Standard components\r\n"
+    "#endif\0"
+END
+
+#endif    // APSTUDIO_INVOKED
+
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Icon
+//
+
+// Icon with lowest ID value placed first to ensure application icon
+// remains consistent on all systems.
+IDR_MAINFRAME           ICON    DISCARDABLE     "res\\Wizard.ico"
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// Dialog
+//
+
+IDD_ABOUTBOX DIALOG DISCARDABLE  0, 0, 217, 55
+STYLE DS_MODALFRAME | WS_POPUP | WS_CAPTION | WS_SYSMENU
+CAPTION "About DynamoRIO Installer"
+FONT 8, "MS Sans Serif"
+BEGIN
+    ICON            IDR_MAINFRAME,IDC_STATIC,11,17,20,20
+    LTEXT           "DynamoRIO Installer Version 1.0",IDC_STATIC,40,10,119,8,
+                    SS_NOPREFIX
+    LTEXT           "Copyright (C) 1998",IDC_STATIC,40,25,119,8
+    DEFPUSHBUTTON   "OK",IDOK,178,7,32,14,WS_GROUP
+END
+
+IDD_FINAL DIALOG DISCARDABLE  0, 0, 272, 114
+STYLE WS_CHILD | WS_DISABLED | WS_CAPTION
+CAPTION "DynamoRIO 0.9.5 Installer: Setup Complete"
+FONT 8, "MS Shell Dlg"
+BEGIN
+    CTEXT           "Setup is now complete!",IDC_STATIC,23,37,226,12
+    CTEXT           "You now need to unzip the file that was copied to the directory you chose.",
+                    IDC_STATIC,45,60,188,32
+END
+
+IDD_COPY DIALOG DISCARDABLE  0, 0, 317, 113
+STYLE DS_MODALFRAME | WS_POPUP | WS_VISIBLE | WS_CAPTION | WS_SYSMENU
+CAPTION "DynamoRIO 0.9.5 Installer: Copy Files"
+FONT 8, "MS Shell Dlg"
+BEGIN
+    LTEXT           "Copy files to:",IDC_STATIC,8,23,80,10
+    EDITTEXT        IDC_TARGET,9,36,244,14,ES_AUTOHSCROLL
+    LTEXT           "Disk space required:",IDC_STATIC,10,64,85,8
+    RTEXT           "0",IDC_SPACE_REQUIRED,99,64,37,10
+    LTEXT           "Disk space available:",IDC_STATIC,10,80,85,8
+    RTEXT           "0",IDC_SPACE_AVAILABLE,94,80,42,10
+    LTEXT           "MB",IDC_STATIC,147,64,14,9
+    LTEXT           "MB",IDC_STATIC,147,80,14,9
+    PUSHBUTTON      "Browse...",IDC_BROWSE,260,36,49,14
+END
+
+IDD_LICENSE DIALOG DISCARDABLE  0, 0, 315, 223
+STYLE DS_MODALFRAME | WS_POPUP | WS_VISIBLE | WS_CAPTION | WS_SYSMENU
+CAPTION "DynamoRIO 0.9.5 Installer: License"
+FONT 8, "MS Shell Dlg"
+BEGIN
+    CONTROL         "I &Accept",IDC_AGREE,"Button",BS_AUTORADIOBUTTON,80,205,
+                    58,12
+    CONTROL         "I Do &Not Accept",IDC_DISAGREE,"Button",
+                    BS_AUTORADIOBUTTON,150,205,72,12
+    EDITTEXT        IDC_LICENSE,10,44,296,141,ES_MULTILINE | ES_READONLY | 
+                    WS_VSCROLL
+    LTEXT           "This wizard installs DynamoRIO version 0.9.5 on your computer.",
+                    IDC_BOLDTITLE,12,6,250,10
+    CONTROL         "I have read and &understand the terms of this agreement",
+                    IDC_UNDERSTAND,"Button",BS_AUTOCHECKBOX | WS_TABSTOP,80,
+                    190,226,11
+    LTEXT           "You must scroll through the entire license before continuing.",
+                    IDC_STATIC,13,18,214,11
+    LTEXT           "Note that this is NOT the same license as in earlier releases, and should be read carefully.",
+                    IDC_STATIC,13,31,292,13
+END
+
+
+#ifndef _MAC
+/////////////////////////////////////////////////////////////////////////////
+//
+// Version
+//
+
+VS_VERSION_INFO VERSIONINFO
+ FILEVERSION 1,0,0,1
+ PRODUCTVERSION 1,0,0,1
+ FILEFLAGSMASK 0x3fL
+#ifdef _DEBUG
+ FILEFLAGS 0x1L
+#else
+ FILEFLAGS 0x0L
+#endif
+ FILEOS 0x4L
+ FILETYPE 0x1L
+ FILESUBTYPE 0x0L
+BEGIN
+    BLOCK "StringFileInfo"
+    BEGIN
+        BLOCK "040904B0"
+        BEGIN
+            VALUE "CompanyName", "VMware\0"
+            VALUE "FileDescription", "DynamoRIO Installer\0"
+            VALUE "FileVersion", "1, 0, 0, 1\0"
+            VALUE "InternalName", "DRinstall\0"
+            VALUE "LegalCopyright", "Copyright (C) 2002-2008\0"
+            VALUE "LegalTrademarks", "\0"
+            VALUE "OriginalFilename", "DRinstall.EXE\0"
+            VALUE "ProductName", "DynamoRIO Installer\0"
+            VALUE "ProductVersion", "1, 0, 0, 1\0"
+        END
+    END
+    BLOCK "VarFileInfo"
+    BEGIN
+        VALUE "Translation", 0x409, 1200
+    END
+END
+
+#endif    // !_MAC
+
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// DESIGNINFO
+//
+
+#ifdef APSTUDIO_INVOKED
+GUIDELINES DESIGNINFO DISCARDABLE 
+BEGIN
+    IDD_ABOUTBOX, DIALOG
+    BEGIN
+        LEFTMARGIN, 7
+        RIGHTMARGIN, 210
+        TOPMARGIN, 7
+        BOTTOMMARGIN, 48
+    END
+END
+#endif    // APSTUDIO_INVOKED
+
+
+/////////////////////////////////////////////////////////////////////////////
+//
+// String Table
+//
+
+STRINGTABLE DISCARDABLE 
+BEGIN
+    IDS_ABOUTBOX            "&About DynamoRIO Installer..."
+END
+
+STRINGTABLE DISCARDABLE 
+BEGIN
+    IDS_HEADERTITLE         "Header Title"
+    IDS_HEADERSUBTITLE      "Header Sub Title"
+END
+
+#endif    // English (U.S.) resources
+/////////////////////////////////////////////////////////////////////////////
+
+
+
+#ifndef APSTUDIO_INVOKED
+/////////////////////////////////////////////////////////////////////////////
+//
+// Generated from the TEXTINCLUDE 3 resource.
+//
+#define _AFX_NO_SPLITTER_RESOURCES
+#define _AFX_NO_OLE_RESOURCES
+#define _AFX_NO_TRACKER_RESOURCES
+#define _AFX_NO_PROPERTY_RESOURCES
+
+#if !defined(AFX_RESOURCE_DLL) || defined(AFX_TARG_ENU)
+#ifdef _WIN32
+LANGUAGE 9, 1
+#pragma code_page(1252)
+#endif //_WIN32
+#include "res\Wizard.rc2"  // non-Microsoft Visual C++ edited resources
+#include "afxres.rc"         // Standard components
+#endif
+/////////////////////////////////////////////////////////////////////////////
+#endif    // not APSTUDIO_INVOKED
+
diff --git a/tools/DRinstall/res/Wizard.rc2 b/tools/DRinstall/res/Wizard.rc2
new file mode 100644
index 000000000..12be71557
--- /dev/null
+++ b/tools/DRinstall/res/Wizard.rc2
@@ -0,0 +1,43 @@
+// **********************************************************
+// Copyright (c) 2008 VMware, Inc.  All rights reserved.
+// **********************************************************
+
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+// 
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+// 
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the following disclaimer in the documentation
+//   and/or other materials provided with the distribution.
+// 
+// * Neither the name of VMware, Inc. nor the names of its contributors may be
+//   used to endorse or promote products derived from this software without
+//   specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+// OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+// DAMAGE.
+
+//
+// WIZARD97.RC2 - resources Microsoft Visual C++ does not edit directly
+//
+
+#ifdef APSTUDIO_INVOKED
+	#error this file is not editable by Microsoft Visual C++
+#endif //APSTUDIO_INVOKED
+
+
+/////////////////////////////////////////////////////////////////////////////
+// Add manually edited resources here...
+
+/////////////////////////////////////////////////////////////////////////////
diff --git a/tools/DRinstall/resource.h b/tools/DRinstall/resource.h
new file mode 100644
index 000000000..368ca23f3
--- /dev/null
+++ b/tools/DRinstall/resource.h
@@ -0,0 +1,93 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+//{{NO_DEPENDENCIES}}
+// Microsoft Developer Studio generated include file.
+// Used by Wizard.rc
+//
+#define IDS_SAMPLEWIZARD                1
+#define IDS_HEADERTITLE                 2
+#define IDS_HEADERSUBTITLE              3
+#define IDM_ABOUTBOX                    0x0010
+#define IDD_ABOUTBOX                    100
+#define IDS_ABOUTBOX                    101
+#define IDD_WIZARD97_DIALOG             102
+#define IDB_BANNER256                   103
+#define IDB_BANNER16                    104
+#define IDB_WATERMARK16                 105
+#define IDB_WATERMARK256                106
+#define IDC_BIGBOLDTITLE                107
+#define IDC_BOLDTITLE                   108
+#define IDC_TRANSPARENT1                109
+#define IDC_TRANSPARENT2                110
+#define IDC_TRANSPARENT3                111
+#define IDC_LIST1                       112
+#define IDC_BUTTON1                     113
+#define IDC_BROWSE                      113
+#define IDC_RADIO1                      114
+#define IDC_AGREE                       114
+#define IDC_RADIO2                      115
+#define IDC_DISAGREE                    115
+#define IDD_INTRO_NEW                   116
+#define IDD_LICENSE                     116
+#define IDD_INTERIOR1_NEW               117
+#define IDD_COPY                        117
+#define IDD_INTERIOR2_NEW               118
+#define IDD_COMPLETION_NEW              119
+#define IDD_FINAL                       119
+#define IDR_MAINFRAME                   128
+#define IDD_CONFIGURE                   129
+#define IDD_PLEASE_WAIT                 130
+#define IDC_EDIT1                       1000
+#define IDC_LICENSE                     1000
+#define IDC_TARGET                      1000
+#define IDC_BUTTON_ACCEPT               1001
+#define IDC_BUTTON_CANCEL               1002
+#define IDC_SPACE_REQUIRED              1003
+#define IDC_SPACE_AVAILABLE             1004
+#define IDC_USERS_ALL                   1006
+#define IDC_USERS_CURRENT               1007
+#define IDC_PLEASE_WAIT_MESSAGE         1007
+#define IDC_CHECK1                      1009
+#define IDC_UNDERSTAND                  1009
+#define IDC_SCROLLBAR                   1011
+
+// Next default values for new objects
+// 
+#ifdef APSTUDIO_INVOKED
+#ifndef APSTUDIO_READONLY_SYMBOLS
+#define _APS_NEXT_RESOURCE_VALUE        131
+#define _APS_NEXT_COMMAND_VALUE         32771
+#define _APS_NEXT_CONTROL_VALUE         1012
+#define _APS_NEXT_SYMED_VALUE           101
+#endif
+#endif
diff --git a/tools/DRkill.c b/tools/DRkill.c
new file mode 100644
index 000000000..510f34133
--- /dev/null
+++ b/tools/DRkill.c
@@ -0,0 +1,160 @@
+/* **********************************************************
+ * Copyright (c) 2003-2006 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * DRkill.c
+ *
+ * 'kill' that works on win32
+ *
+ * to build:
+ *  cl DRkill.c registry.c advapi32.lib /o DRkill.exe
+ *
+ * jbrink, 8/03
+ *
+ * (c) araksha, inc. all rights reserved
+ */
+
+
+#include "share.h"
+#include "processes.h"
+
+#include <stdio.h>
+
+
+void procwalk();
+
+
+void
+usage()
+{
+    fprintf(stderr, "Usage:\n");
+    fprintf(stderr, "DRkill [-help] [-quiet] [-pid n] [-exe name] [-underdr] [-v]\n");
+    exit(1);
+}
+
+void
+help()
+{
+    fprintf(stderr, "Options:\n");
+    fprintf(stderr, " -quiet n\t\t\tdon't report what is killed\n");
+    fprintf(stderr, " -pid n\t\t\tkills process with pid 'n'\n");
+    fprintf(stderr, " -exe name\t\tkills all processes whose executable matches\n");
+    fprintf(stderr, "\t\t\t'name'\n");
+    fprintf(stderr, " -underdr\t\tkills all processes running under SC\n");
+    fprintf(stderr, " -v\t\t\tdisplay version information\n\n");
+
+    exit(1);
+}
+
+BOOL underdr=FALSE, killquietly=FALSE;
+uint pid=0;
+WCHAR exe[MAX_PATH];
+
+int
+main(int argc, char **argv)
+{
+    int argidx=1;
+    exe[0]=0;
+    if (argc < 2) 
+        usage();
+
+    while (argidx < argc) {
+
+	if (!strcmp(argv[argidx], "-help")) {
+  	    help();
+	}
+	else if (!strcmp(argv[argidx], "-quiet")) {
+	    killquietly=TRUE;
+	}
+	else if (!strcmp(argv[argidx], "-pid")) {
+	    pid=atoi(argv[++argidx]);
+	}
+	else if (!strcmp(argv[argidx], "-exe")) {
+	    _snwprintf(exe, MAX_PATH, L"%S", argv[++argidx]);
+	}
+	else if (!strcmp(argv[argidx], "-underdr")) {
+            underdr=TRUE;
+	}
+	else if (!strcmp(argv[argidx], "-v")) {
+#ifdef BUILD_NUMBER
+	  printf("DRkill.exe build %d -- %s\n", BUILD_NUMBER, __DATE__);
+#else
+	  printf("DRkill.exe custom build -- %s, %s\n", __DATE__, __TIME__);
+#endif
+	}
+	else {
+	    fprintf(stderr,"Unknown option: %s\n", argv[argidx]);
+	    usage();
+	}
+	argidx++;
+    }
+    
+    if(pid) {
+        if (!killquietly)
+            printf("killing process %d\n", pid);
+
+        terminate_process(pid);       
+    }
+    else {
+        procwalk();
+    }
+}
+
+BOOL
+pw_callback(process_info_t *pi, void **param)
+{
+    int res;
+
+    if ((pid != 0 && pi->ProcessID == pid) ||
+        (exe[0] != '\0' && !wcsicmp(exe, pi->ProcessName)) || 
+        underdr) {
+
+        res = under_dynamorio(pi->ProcessID);
+        
+        if (exe[0] != '\0' || pid != 0 ||
+            (underdr && (res != DLL_NONE && res != DLL_UNKNOWN))) {
+
+            if (!killquietly)
+                printf("killing process %d=%S\n",
+                       pi->ProcessID, pi->ProcessName);
+            
+            terminate_process(pi->ProcessID);
+        }
+    } 
+    return TRUE;
+}
+
+void
+procwalk()
+{
+    process_walk(&pw_callback, NULL);
+}
diff --git a/tools/DRload.c b/tools/DRload.c
new file mode 100644
index 000000000..f4c425c06
--- /dev/null
+++ b/tools/DRload.c
@@ -0,0 +1,313 @@
+/* **********************************************************
+ * Copyright (c) 2004-2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include <stdio.h>
+
+/* This is a tool whose sole purpose is to provide an in-memory image
+ * of an initialized dynamorio.dll, so we can query it with a debugger.
+ *
+ * If you launch this executable through a debugger, use -debugbreak;
+ * otherwise, use -loop and then attach.
+ *
+ * example usage:
+ *   ./cdb -g -c "ln 7004e660; ln 77f830e7; q" c:\\derek\\dr\\tools\\DRload.exe -debugbreak c:\\derek\\dr\\builds\\11087\\exports\\x86_win32_rel\\dynamorio.dll | grep '|'
+ * =>
+ *   (7004deb8)   dynamorio!shared_code+0x7a8   |  (7004edc4)   dynamorio!features
+ *   (77f830c2)   ntdll!RtlIsDosDeviceName_U+0x25   |  (77faebf2)   ntdll!RtlpAllocateHeapUsageEntry
+ */
+
+typedef int (*int_func_t) ();
+typedef void (*void_func_t) ();
+
+int
+usage(char *exec)
+{
+    fprintf(stderr, "Usage: %s [-help] [-debugbreak] [-loop] [-key] [-no_init]\n"
+            "        [-call_to_offset <hex offset>] [-find_safe_offset] [-no_resolve]\n"
+            "        [-map <filename>] [-base <hex addr>] <DR/other dll path>\n", exec);
+    return 1;
+}
+
+int
+help(char *exec)
+{
+    usage(exec);
+    fprintf(stderr, "   -help : print this message\n");
+    fprintf(stderr, "   -debugbreak : for launching under a debugger, trigger a "
+            "debugbreak once dll is loaded\n");
+    fprintf(stderr, "   -loop : for attaching a debugger, loop infinitely once dll is"
+            " loaded\n");
+    fprintf(stderr, "   -key : for attaching a debugger, wait for keypress once dll is"
+            " loaded\n");
+    fprintf(stderr, "   -no_init : don't call dynamorio init function after dll is"
+            " loaded (use for non-dr dll)\n");
+    fprintf(stderr, "   -call_to_offset <hex offset> : once dll is loaded call this "
+            "offset to the dll base");
+    fprintf(stderr, "   -find_safe_offset : if -call_to_offset is set, finds the first"
+            "return instr in\n      the same mem region as the supplied offset and calls"
+            "it instead.");
+    fprintf(stderr, "   -no_resolve : pass DONT_RESOLVE_DLL_REFERENCES to the ldr when"
+            " loading the dll\n      (prevents dependent dlls from being loaded)\n");
+    fprintf(stderr, "   -map <filename> <hex address> : map filename at address\n");
+    fprintf(stderr, "   -base <address> : maps dynamorio.dll at address\n");
+    fprintf(stderr, "   -preferred <hex address> : makes -base usable for other dlls\n");
+    fprintf(stderr, "   <DR/other dll path> : path to dll to load\n");
+    return 0;
+}
+
+int
+map_file(const char *filename, void *addr, int image)
+{
+    HANDLE map;
+    PBYTE view;
+    /* Must specify FILE_SHARE_READ to open if -persist_lock_file is in use */
+    HANDLE file = CreateFileA(filename, GENERIC_READ, FILE_SHARE_READ, NULL,
+                              OPEN_EXISTING,
+                              FILE_ATTRIBUTE_NORMAL, NULL);
+    if (file == NULL) {
+        int err = GetLastError();
+        printf("Error %d opening %s\n", err, filename);
+        return 0;
+    }
+    /* FIXME: to map as image also need to adjust file and mapping to be executable */
+    map = CreateFileMapping(file, NULL, image ? SEC_IMAGE : PAGE_READONLY, 
+                            0, 0, NULL);
+    if (map == NULL) {
+        int err = GetLastError();
+        CloseHandle(file);
+        printf("Error %d mapping %s\n", err, filename);
+        return 0;
+    }
+    view = MapViewOfFileEx(map, FILE_MAP_READ, 0, 0, 0, addr);
+    if (view == NULL) {
+        int err = GetLastError();
+        CloseHandle(map);
+        CloseHandle(file);
+        printf("Error %d mapping view of %s\n", err, filename);
+        return 0;
+    }
+    return 1;
+}
+
+int
+main(int argc, char *argv[])
+{
+    char *DRpath;
+    HANDLE dll;
+    int_func_t init_func;
+    void_func_t take_over_func;
+    int res = 0;
+    BOOL debugbreak = FALSE;
+    BOOL infinite = FALSE;
+    BOOL keypress = FALSE;
+    BOOL initialize_dr = TRUE;
+    BOOL use_dont_resolve = FALSE;
+    int arg_offs = 1;
+    void *force_base = NULL;
+    void *preferred_base = NULL;
+    int call_offset = -1;
+    BOOL find_safe_offset = FALSE;
+
+    /* Link user32.dll for easier running under dr */
+    do { if (argc > 1000) MessageBeep(0); } while (0);
+
+    if (argc < 2)
+        return usage(argv[0]);
+    while (argv[arg_offs][0] == '-') {
+        if (strcmp(argv[arg_offs], "-help") == 0) {
+            return help(argv[0]);
+        } else if (strcmp(argv[arg_offs], "-debugbreak") == 0) {
+            debugbreak = TRUE;
+            arg_offs += 1;
+        } else if (strcmp(argv[arg_offs], "-loop") == 0) {
+            infinite = TRUE;
+            arg_offs += 1;
+        } else if (strcmp(argv[arg_offs], "-key") == 0) {
+            keypress = TRUE;
+            arg_offs += 1;
+        } else if (strcmp(argv[arg_offs], "-no_init") == 0) {
+            initialize_dr = FALSE;
+            arg_offs += 1;
+        } else if (strcmp(argv[arg_offs], "-call_to_offset") == 0) {
+            int len;
+            arg_offs += 1;
+            if (argc - (arg_offs) < 1)
+                return usage(argv[0]);
+            len = sscanf(argv[arg_offs], "%08x", &call_offset);
+            if (len != 1 || call_offset == -1)
+                return usage(argv[0]);
+            arg_offs += 1;
+        } else if (strcmp(argv[arg_offs], "-find_safe_offset") == 0) {
+            find_safe_offset = TRUE;
+            arg_offs += 1;
+        } else if (strcmp(argv[arg_offs], "-no_resolve") == 0) {
+            use_dont_resolve = TRUE;
+            arg_offs += 1;
+        } else if (strcmp(argv[arg_offs], "-map") == 0) {
+            void *addr = NULL;
+            int len;
+            arg_offs += 1;
+            if (argc - (arg_offs+1) < 1)
+                return usage(argv[0]);
+            len = sscanf(argv[arg_offs+1], "%08x", &addr);
+            if (len != 1 || addr == NULL)
+                return usage(argv[0]);
+            map_file(argv[arg_offs], addr, 0 /* mapped */);
+            arg_offs += 2;
+        } else if (strcmp(argv[arg_offs], "-base") == 0) {
+            int len;
+            arg_offs += 1;
+            if (argc - (arg_offs) < 1)
+                return usage(argv[0]);
+            len = sscanf(argv[arg_offs], "%08x", &force_base);
+            if (len != 1 || force_base == NULL)
+                return usage(argv[0]);
+            arg_offs += 1;
+        } else if (strcmp(argv[arg_offs], "-preferred") == 0) {
+            int len;
+            arg_offs += 1;
+            if (argc - (arg_offs) < 1)
+                return usage(argv[0]);
+            len = sscanf(argv[arg_offs], "%08x", &preferred_base);
+            if (len != 1 || preferred_base == NULL)
+                return usage(argv[0]);
+            arg_offs += 1;
+        } else
+            return usage(argv[0]);
+        if (argc - arg_offs < 1)
+            return usage(argv[0]);
+    }
+    DRpath = argv[arg_offs];
+
+    if (force_base != NULL) {
+        /* add load blocks at the expected base addresses */
+        void *base;
+        if (preferred_base != NULL)
+            base = preferred_base;
+        else /* assume DR dll */
+            base = (void *)0x71000000;
+        VirtualAllocEx(GetCurrentProcess(), 
+                       base, 0x1000, MEM_RESERVE, PAGE_NOACCESS);
+        if (preferred_base == NULL) {
+            /* also do debug build base */
+            base = (void*)0x15000000;
+            VirtualAllocEx(GetCurrentProcess(), 
+                           base, 0x1000, MEM_RESERVE, PAGE_NOACCESS);
+        }
+        base = force_base;
+        /* to ensure we fill all cavities we loop through */
+        while (base > (void*)0x10000) {
+            base = (void*)((int)base - 0x10000);
+            VirtualAllocEx(GetCurrentProcess(), 
+                           base, 0x1000, MEM_RESERVE, PAGE_NOACCESS);
+        }
+
+#if 0
+        map_file(DRpath, force_base, 1 /* image */);
+        /* FIXME: note that the DLL will not be relocated! */
+        /* we can't really initialize */
+#endif
+    }
+
+
+    if (use_dont_resolve) {
+        dll = LoadLibraryExA(DRpath, NULL, DONT_RESOLVE_DLL_REFERENCES);
+    } else {
+        dll = LoadLibraryA(DRpath);
+    }
+
+    if (dll == NULL) {
+        int err = GetLastError();
+        printf("Error %d loading %s\n", err, DRpath);
+        return 1;
+    }
+
+    if (initialize_dr) {
+        init_func = (int_func_t) GetProcAddress(dll, "dynamorio_app_init");
+        take_over_func = (void_func_t) GetProcAddress(dll, "dynamorio_app_take_over");
+        if (init_func == NULL || take_over_func == NULL) {
+            printf("Error finding DR init routines\n");
+            res = 1;
+            goto done;
+        }
+        res = (*init_func)();
+        /* FIXME: ASSERT(res) */
+        (*take_over_func)();
+        res = 0;
+    }
+
+    if (call_offset != -1) {
+        unsigned char *call_location = (char *)dll+call_offset;
+        if (find_safe_offset) {
+            MEMORY_BASIC_INFORMATION mbi;
+            if (VirtualQuery(call_location, &mbi, sizeof(mbi)) != sizeof(mbi) ||
+                mbi.State == MEM_FREE || mbi.State == MEM_RESERVE) {
+                printf("Call offset invalid, leaving as is\n");
+            } else {
+                /* find safe place to call, we just look for 0xc3 though could in theory
+                 * use other types of rets too */
+                unsigned char *test;
+                for (test = call_location;
+                     test < (char *)mbi.BaseAddress+mbi.RegionSize; test++) {
+                    if (*test == 0xc3 /* plain ret */) {
+                        printf("Found safe call target at offset 0x%08x\n",
+                               test - (char *)dll);
+                        call_location = test;
+                        break;
+                    }
+                }
+                if (call_location != test) {
+                    printf("Unable to find safe call target\n");
+                }
+            }
+        }
+        printf("Calling base(0x%08x) + offset(0x%08x) = 0x%08x\n",
+               dll, call_location-(char *)dll, call_location);
+        (*(int (*) ())(call_location))();
+    }
+
+ done:
+    if (keypress) {
+        printf("press any key or attach a debugger...\n");
+        fflush(stdout);
+        getchar();
+    }
+    if (debugbreak) {
+        __asm _emit 0xcc;
+    }
+    if (infinite) {
+        while (1)
+            ;
+    }
+    return res;
+}
diff --git a/tools/DRview.c b/tools/DRview.c
new file mode 100644
index 000000000..48945fe3b
--- /dev/null
+++ b/tools/DRview.c
@@ -0,0 +1,791 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * DRview.c
+ *
+ * command-line tool for determining what is running under DR
+ *
+ * (c) araksha, inc. all rights reserved
+ */
+
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <assert.h>
+#include <tchar.h>
+
+/* The order here now matters: share.h needs to be the first to include
+ * globals_shared.h, as it has to set HOT_PATCHING_INTERFACE first;
+ * but ntdll.h now includes globals_shared.h, so we put it second.
+ * But, processes.h duplicates _VM_COUNTERS from ntdll.h but can handle
+ * going 2nd but not first: so it must go last.
+ */
+#include "share.h"
+#include "ntdll.h"
+#include "processes.h"
+
+#ifdef EXTERNAL_DRVIEW
+# define NAME "DR"
+#else
+# define NAME "SC"
+#endif
+
+int procwalk();
+void dllwalk();
+
+void
+usage()
+{
+    fprintf(stderr, "Usage:\n");
+#ifdef EXTERNAL_DRVIEW
+    fprintf(stderr, "drview [-help] [-pid n] [-exe name] [-listdr] [-listall] [-v]\n");
+#else
+    fprintf(stderr, "DRview [-help] [-pid n] [-exe name] [-listdr] [-listall] [-listdlls] [-showdlls] [-nopid] [-out file] [-cmdline] [-showmem] [-showtime] [-nobuildnum] [-qname strip] [-noqnames] [-hot_patch] [-s n] [-tillidle] [-idlecpu c] [-showmemfreq f] [-idleafter s] [-v]\n");
+#endif
+    exit(1);
+}
+
+void
+help()
+{
+    fprintf(stderr, "Options:\n");
+    fprintf(stderr, " -pid n\t\t\tdisplays whether the process is injected into\n");
+    fprintf(stderr, " -exe name\t\tfinds all processes whose executable matches\n");
+    fprintf(stderr, "\t\t\t'name', shows whether injected into\n");
+    fprintf(stderr, " -listdr\t\tlist all processes injected into\n");
+    fprintf(stderr, " -listall\t\tlist all processes on the system, show whether injected_into\n");
+#ifndef EXTERNAL_DRVIEW
+    fprintf(stderr, " -listdlls\t\tlist all DLLs [short] for a specific pid or executable\n");
+    fprintf(stderr, " -showdlls\t\tlist all DLLs [long] for a specific pid or executable\n");
+    fprintf(stderr, " -nopid\t\t\tdoes not display PIDs of processes (useful for expect files)\n");
+    fprintf(stderr, " -onlypid\t\tonly shows PID\n");
+    fprintf(stderr, " -out file\t\toutput to file instead of stdout\n");
+    fprintf(stderr, " -cmdline\t\tshow process command lines\n");
+    fprintf(stderr, " -showmem\t\tshow memory stats\n");
+    fprintf(stderr, " -showtime\t\tshow scheduled time for each process (needs -showmem)\n");
+    fprintf(stderr, " -showstats\t\tshow internal stats\n");
+    fprintf(stderr, " -nobuildnum\t\tdoes not display build number of SC (useful for expect files)\n");
+    fprintf(stderr, " -qname strip\t\tshow qualified names; set strip to 0 or 1\n");
+    fprintf(stderr, " -noqnames\t\tdon\'t show qualified names\n");
+    fprintf(stderr, " -hot_patch\t\tshow hot patch status\n");
+    fprintf(stderr, " -s n\t\t\tsample every n millis (default: 500ms, e.g. 1s=1000, 5min=300000 5*60*1000)\n");
+    fprintf(stderr, " -tillidle\t\tsample until idle (default: < 3%% cpu for 3s)\n");
+    fprintf(stderr, " -idlecpu c\t\tconsider < c%% cpu utilization idle (default: 3%%)\n");
+    /* Sometimes -listall -showmem causes lsass %cpu to remain high, so
+     * we provide an option to sample frequently but query lsass less frequently.
+     */
+    fprintf(stderr, " -showmemfreq f\t\tfor -tillidle -showmem, show -showmem every f samples (default: 1)\n");
+    fprintf(stderr, " -idleafter s\t\tflag machine is idle after s seconds (default: 3s)\n");
+#endif /* !EXTERNAL_DRVIEW */
+    fprintf(stderr, " -v\t\t\tdisplay version information\n\n");
+
+    exit(1);
+}
+
+BOOL listdr=FALSE, listall=FALSE, nopid=FALSE, onlypid=FALSE,
+     listdlls=FALSE, showdlls=FALSE,
+     qname=FALSE, strip=FALSE, noqnames=FALSE;
+     cmdline=FALSE,
+     hotp=FALSE,
+     showtime=FALSE,
+     sampling=FALSE,
+     tillidle=FALSE,
+     idle=FALSE,
+     skip=FALSE,
+     showmem=FALSE, showbuild=TRUE,
+     showstats=FALSE;
+uint pid=0;
+uint millis=0;
+uint idlecpu=0;
+uint showmemfreq=1;
+uint flag_after_ms=0;
+char *exe=NULL, *outf=NULL;
+uint save_pid = 0;
+WCHAR save_module[MAX_PATH];
+WCHAR wexe[MAX_PATH];
+FILE *fp;
+LONGLONG total_user = 0;
+LONGLONG total_kernel = 0;
+
+#define MAX_CMDLINE 1024
+
+#define BUFFER_SIZE_BYTES(buf)      sizeof(buf)
+#define BUFFER_SIZE_ELEMENTS(buf)   (BUFFER_SIZE_BYTES(buf) / sizeof(buf[0]))
+#define BUFFER_LAST_ELEMENT(buf)    buf[BUFFER_SIZE_ELEMENTS(buf) - 1]
+#define NULL_TERMINATE_BUFFER(buf)  BUFFER_LAST_ELEMENT(buf) = 0
+#define NULL_TERMINATE_SIZED_BUFFER(buf, size) (buf)[(size)-1] = 0
+
+static void
+generate_process_name(process_info_t *pi, WCHAR *name_buf /* OUT */,
+                      uint name_buf_length /* elements */)
+{
+    WCHAR qual_name[MAX_CMDLINE];
+    WCHAR qual_args[MAX_CMDLINE];
+    int res;
+    BOOL use_args = FALSE;
+
+    /* hack: we assume only need qualified names for these hardcoded apps
+     * FIXME: read registry to see whether need qualification
+     */
+    if (wcsicmp(pi->ProcessName, L"svchost.exe") == 0 ||
+        wcsicmp(pi->ProcessName, L"msiexec.exe") == 0 ||
+        wcsicmp(pi->ProcessName, L"tomcat.exe") == 0 ||
+        wcsicmp(pi->ProcessName, L"dllhost.exe") == 0) {
+        /* use qualified names */
+        res = get_process_cmdline(pi->ProcessID, qual_name,
+                                  BUFFER_SIZE_ELEMENTS(qual_name));
+        NULL_TERMINATE_BUFFER(qual_name);
+        if (res == ERROR_SUCCESS) {
+            if (get_commandline_qualifier(qual_name, qual_args,
+                                          BUFFER_SIZE_ELEMENTS(qual_args),
+                                          /* hack: we assume we only strip svchost,
+                                           * and we also strip dllhost here to
+                                           * fit more of the GUI in and avoid the
+                                           * "Processid" string.
+                                           * FIXME: read from registry.
+                                           */
+                                          (wcsicmp(pi->ProcessName, L"svchost.exe") != 0 &&
+                                           wcsicmp(pi->ProcessName, L"dllhost.exe") != 0))) {
+                NULL_TERMINATE_BUFFER(qual_args);
+                use_args = TRUE;
+            } /* else not an error, just no args. e.g.: plain sqlservr.exe */
+        } else {
+            /* this is an error => notify user */
+            _snwprintf(qual_args, BUFFER_SIZE_ELEMENTS(qual_args), L"<error>");
+            NULL_TERMINATE_BUFFER(qual_args);
+            use_args = TRUE;
+        }
+    }
+    if (use_args) {
+        _snwprintf(name_buf, name_buf_length, L"%s-%s",
+                   pi->ProcessName, qual_args);
+    } else {
+        _snwprintf(name_buf, name_buf_length, L"%s", pi->ProcessName);
+    }
+    NULL_TERMINATE_SIZED_BUFFER(name_buf, name_buf_length);
+}
+
+static void
+print_mem_stats(process_info_t *pi, char reschar, int version)
+{
+    WCHAR qual_name[MAX_CMDLINE];
+    int cpu = -1, user = -1;
+    LONGLONG wallclock_time = get_system_time() - pi->CreateTime.QuadPart;
+    LONGLONG scheduled_time = pi->UserTime.QuadPart + pi->KernelTime.QuadPart;
+    static LONGLONG firstproc_time = 0;
+    if (wallclock_time != (LONGLONG) 0) {
+        cpu = (int) ((100 * scheduled_time) / wallclock_time);
+    }
+    if (scheduled_time != (LONGLONG) 0) {
+        user = (int) ((100 * pi->UserTime.QuadPart) / scheduled_time);
+    }
+
+    /* Total user and kernel time scheduled for all processes.  Don't include idle
+     * process, and if -skip option is specified don't include DRview.exe */
+    if (wcsicmp(pi->ProcessName, L"") != 0
+        && (wcsicmp(pi->ProcessName, L"drview.exe") != 0 || !skip)) {
+        total_user   += pi->UserTime.QuadPart;
+        total_kernel += pi->KernelTime.QuadPart;
+    }
+
+    /* CreateTime.QuadPart is a counter since 1916.  Both idle process and
+     * System have create time of 0, so report create time, in ms, relative to
+     * smss.exe */
+    if (wcsicmp(pi->ProcessName, L"") == 0
+        || wcsicmp(pi->ProcessName, L"system") == 0
+        || wcsicmp(pi->ProcessName, L"smss.exe") == 0) {
+        firstproc_time = pi->CreateTime.QuadPart;
+    }
+
+    generate_process_name(pi, qual_name, BUFFER_SIZE_ELEMENTS(qual_name));
+
+    /* single line best so can line up columns and process output easily */
+    fprintf(fp, "%-23.23S %5d %c %5d %2d%% %3d%% %5d %3d %7d %7d %8d %7d %7d %7d %7d %5d %5d %5d %5d %5d",
+            qual_name, pi->ProcessID, reschar, version,
+            cpu, user,
+            pi->HandleCount, pi->ThreadCount,
+            pi->VmCounters.PeakVirtualSize/1024,
+            pi->VmCounters.VirtualSize/1024,
+            pi->VmCounters.PeakPagefileUsage/1024,
+            pi->VmCounters.PagefileUsage/1024, /* aka Private */
+            pi->VmCounters.PeakWorkingSetSize/1024,
+            pi->VmCounters.WorkingSetSize/1024,
+            pi->VmCounters.PageFaultCount,
+            pi->VmCounters.QuotaPeakPagedPoolUsage/1024,
+            pi->VmCounters.QuotaPagedPoolUsage/1024,
+            pi->VmCounters.QuotaPeakNonPagedPoolUsage/1024,
+            pi->VmCounters.QuotaNonPagedPoolUsage/1024,
+            pi->InheritedFromProcessID);
+    if (showtime) {
+        /* QuadPart is in ticks 1 tick = 100 nano secs. In ms = n * 100/(1000*1000) */
+        fprintf(fp, " %15I64d %15I64d %15I64d",
+                    (pi->UserTime.QuadPart/10000),
+                    (pi->KernelTime.QuadPart/10000),
+                    ((pi->CreateTime.QuadPart - firstproc_time)/10000));
+    }
+    fprintf(fp, "\n");
+}
+
+int
+main(int argc, char **argv)
+{
+    int nprocs, argidx=1;
+
+    if (argc < 2) 
+        usage();
+
+    while (argidx < argc) {
+
+        if (!strcmp(argv[argidx], "-help")) {
+            help();
+        }
+        else if (!strcmp(argv[argidx], "-pid") || !strcmp(argv[argidx], "-p")) {
+            if (argidx+1 >= argc)
+                usage();
+            pid=atoi(argv[++argidx]);
+        }
+        else if (!strcmp(argv[argidx], "-exe")) {
+            if (argidx+1 >= argc)
+                usage();
+            exe=argv[++argidx];
+            _snwprintf(wexe, MAX_PATH, L"%S", exe);
+            wexe[MAX_PATH - 1] = L'\0';
+        }
+        else if (!strcmp(argv[argidx], "-listall")) {
+            listall=TRUE;
+        }
+        else if (!strcmp(argv[argidx], "-listdr")) {
+            listdr=TRUE;
+        }
+        else if (!strcmp(argv[argidx], "-listdlls")) {
+	  listdlls=TRUE;
+        }
+#ifndef EXTERNAL_DRVIEW
+        else if (!strcmp(argv[argidx], "-showdlls")) {
+	  listdlls=TRUE;
+          showdlls=TRUE;
+        }
+        else if (!strcmp(argv[argidx], "-nopid")) {
+            nopid=TRUE;
+        }
+        else if (!strcmp(argv[argidx], "-onlypid")) {
+            onlypid=TRUE;
+        }
+        else if (!strcmp(argv[argidx], "-out")) {
+            if (argidx+1 >= argc)
+                usage();
+            outf=argv[++argidx];
+        }
+        else if (!strcmp(argv[argidx], "-cmdline")) {
+            cmdline=TRUE;
+        }
+        else if (!strcmp(argv[argidx], "-showstats")) {
+            showstats=TRUE;
+        }
+        else if (!strcmp(argv[argidx], "-showmem")) {
+            showmem=TRUE;
+        }
+        else if (!strcmp(argv[argidx], "-showtime")) {
+            if (!showmem)
+                usage();
+            showtime=TRUE;
+        }
+        else if (!strcmp(argv[argidx], "-nobuildnum")) {
+            showbuild=FALSE;
+        }
+        else if (!strcmp(argv[argidx], "-qname")) {
+            if (argidx+1 >= argc)
+                usage();
+            qname = TRUE;
+            strip = atoi(argv[++argidx]);
+        }
+        else if (!strcmp(argv[argidx], "-noqnames")) {
+            noqnames=TRUE;
+        }
+        else if (!strcmp(argv[argidx], "-hot_patch")) {
+            hotp=TRUE;
+        }
+        else if (!strcmp(argv[argidx], "-s")) {
+            millis = (uint)atoi(argv[argidx+1]);
+            if (millis == 0) {
+                millis = 500;
+            }
+            else {
+                argidx++;
+            }
+            sampling=TRUE;
+        }
+        else if (!strcmp(argv[argidx], "-tillidle")) {
+            millis = (millis <= 0) ? 500 : millis;
+            idlecpu = (idlecpu <= 0) ? 3 : idlecpu;
+            showmemfreq = (showmemfreq <= 1) ? 1 : showmemfreq;
+            flag_after_ms = (flag_after_ms <= 0) ? 3000 : flag_after_ms;
+            tillidle = TRUE;
+            sampling = TRUE;
+        }
+        else if (!strcmp(argv[argidx], "-idlecpu")) {
+            if (argidx+1 >= argc)
+                usage();
+            idlecpu = (uint)atoi(argv[++argidx]);
+            idlecpu = (idlecpu <= 0) ? 3 : idlecpu;
+            millis = (millis <= 0) ? 500 : millis;
+            flag_after_ms = (flag_after_ms <= 0) ? 3000 : flag_after_ms;
+            tillidle = TRUE;
+            sampling = TRUE;
+        }
+        else if (!strcmp(argv[argidx], "-showmemfreq")) {
+            if (argidx+1 >= argc)
+                usage();
+            showmemfreq = (uint)atoi(argv[++argidx]);
+            showmemfreq = (showmemfreq <= 1) ? 1 : showmemfreq;
+            showmem = TRUE;
+            tillidle = TRUE;
+            sampling = TRUE;
+        }
+        else if (!strcmp(argv[argidx], "-idleafter")) {
+            if (argidx+1 >= argc)
+                usage();
+            flag_after_ms = (uint)atoi(argv[++argidx]);
+            flag_after_ms = (flag_after_ms <= 0) ? 3000 : flag_after_ms;
+            idlecpu = (idlecpu <= 0) ? 3 : idlecpu;
+            millis = (millis <= 0) ? 500 : millis;
+            sampling = TRUE;
+            tillidle = TRUE;
+        }
+        else if (!strcmp(argv[argidx], "-skip")) {
+            /* internal option: skip drview's contribution to total scheduled time */
+            skip=TRUE;
+        }
+#endif /* !EXTERNAL_DRVIEW */
+        else if (!strcmp(argv[argidx], "-v")) {
+#if defined(BUILD_NUMBER) && defined(VERSION_NUMBER)
+          printf("drview.exe version %s -- build %d\n", STRINGIFY(VERSION_NUMBER), BUILD_NUMBER);
+#elif defined(BUILD_NUMBER)
+          printf("drview.exe custom build %d -- %s\n", BUILD_NUMBER, __DATE__);
+#else
+          printf("drview.exe custom build -- %s, %s\n", __DATE__, __TIME__);
+#endif
+          exit(0);
+        }
+        else {
+            fprintf(stderr, "Unknown option: %s\n", argv[argidx]);
+            usage();
+        }
+        argidx++;
+    }
+
+    if (listdlls && !pid && !exe && !listall && !listdr)
+    {
+      fprintf(stderr, "-listdlls option should be combined with a specific -pid, -exe, -listdr or -listall option\n");
+      usage();
+    }
+
+    if (outf) {
+        fp = fopen(outf,"w");
+        if(fp == NULL) {
+            fprintf(stderr, "Error opening %s for output.\n");
+            exit(-1);
+        }
+    }
+    else
+        fp=stdout;
+ 
+    /* Invoke DLL routine when -listdlls combined with either -listall or -pid or -exe */
+    if (listdlls && (listall || pid || exe || listdr))
+    {
+      dllwalk();
+    }
+    else
+    {
+        do {
+            nprocs = procwalk();
+            fflush(fp);
+            if (sampling)
+                Sleep(millis);
+            /* FIXME: looping infinitely, make sure we are not leaking anything */
+        } while (sampling && !idle);
+    }
+
+    if (outf)
+        fclose(fp);
+
+    return nprocs;
+}
+
+int count;
+
+/* returns FALSE if status is effectively disabled, otherwise returns
+ *  TRUE (indicating status should be displayed. */
+BOOL
+get_status_string(char *buf, UINT maxchars,
+                  hotp_inject_status_t status,
+                  hotp_policy_mode_t mode)
+{
+    char *statptr = NULL;
+    char *modeptr = NULL;
+
+    if (status == HOTP_INJECT_NO_MATCH && mode == HOTP_MODE_OFF)
+        return FALSE;
+
+    switch (status) {
+    case HOTP_INJECT_ERROR :
+        statptr = "Inject Error!";
+        break;
+    case HOTP_INJECT_PROTECT :
+        statptr = "Injected Protector";
+        break;
+    case HOTP_INJECT_DETECT :
+        statptr = "Injected Detector";
+        break;
+    case HOTP_INJECT_IN_PROGRESS : 
+        statptr = "Injection in progress";
+        break;
+    case HOTP_INJECT_PENDING :
+        statptr = "Inject point not yet executed";
+        break;
+    case HOTP_INJECT_NO_MATCH :
+        statptr = "Not matched";
+        break;
+    case HOTP_INJECT_OFF :
+        /* not currently used. */
+    default :
+        statptr = NULL;
+    }
+
+    switch (mode) {
+    case HOTP_MODE_OFF :
+        modeptr = "Off";
+        break;
+    case HOTP_MODE_DETECT :
+        modeptr = "Detect";
+        break;
+    case HOTP_MODE_PROTECT :
+        modeptr = "Protect";
+        break;
+    default:
+        modeptr = NULL;
+    }
+
+    if (statptr == NULL || modeptr == NULL) {
+        _snprintf(buf, maxchars, 
+                  "[Status ERROR: Status=%d, Mode=%d]", status, mode);
+    }
+    else {
+        _snprintf(buf, maxchars, "%s [%s]", modeptr, statptr);
+    }
+
+    buf[maxchars - 1] = '\0';
+
+    return TRUE;
+}
+
+
+BOOL
+pw_callback(process_info_t *pi, void **param)
+{
+    char *resstr;
+    char reschar;
+    int res;
+    WCHAR buf[MAX_CMDLINE];
+    DWORD version;
+    BOOL under_dr;
+    
+    WCHAR qual_name[MAX_CMDLINE];
+    if (exe)
+        generate_process_name(pi, qual_name, BUFFER_SIZE_ELEMENTS(qual_name));
+
+    if ( (pid && pi->ProcessID == pid) ||
+         (exe && (!wcsicmp(wexe, pi->ProcessName) || !wcsicmp(wexe,  qual_name))) ||
+         listall || listdr)
+        {
+            version = -1;
+            res = under_dynamorio_ex(pi->ProcessID, &version);
+            switch (res) {
+            case DLL_PROFILE : resstr=NAME" profile"; reschar='P'; break;
+            case DLL_RELEASE : resstr=NAME" release"; reschar='R'; break;
+            case DLL_DEBUG : resstr=NAME" debug"; reschar='D'; break;
+            case DLL_CUSTOM : resstr=NAME" custom"; reschar='C'; break;
+            case DLL_NONE : resstr="native"; reschar='N'; break;
+            case DLL_UNKNOWN : 
+            default : resstr="<error>"; reschar='?';
+            }
+
+            under_dr = !(res == DLL_NONE || res == DLL_UNKNOWN);
+
+            if (!listdr || under_dr) {
+                if (!nopid && !showmem) {
+                    if (onlypid)
+                        fprintf(fp, "%d\n", pi->ProcessID);
+                    else
+                        fprintf(fp, "PID %d, ", pi->ProcessID);
+                }
+                if (!showmem && !onlypid) {
+                    WCHAR qual_name[MAX_CMDLINE];
+                    WCHAR *name_to_use = pi->ProcessName;
+                    if (!noqnames) {
+                        generate_process_name(pi, qual_name,
+                                              BUFFER_SIZE_ELEMENTS(qual_name));
+                        name_to_use = qual_name;
+                    }
+                    if (version == -1 || !showbuild) {
+                        fprintf(fp, "Process %S, running %s\n",
+                                name_to_use, resstr);
+                    }
+                    else {
+                        fprintf(fp, "Process %S, running %s (build %d)\n",
+                                name_to_use, resstr, version);
+                    }
+                }
+                if (cmdline) {
+                    res = get_process_cmdline(pi->ProcessID, buf,
+                                              BUFFER_SIZE_ELEMENTS(buf));
+                    NULL_TERMINATE_BUFFER(buf);
+                    if (res == ERROR_SUCCESS) {
+                        fprintf(fp, "\tCmdline: %S\n", buf);
+                    }
+                    else fprintf(fp, "\t<Cmdline err %d>\n", res);
+                }
+                if (qname) {
+                    WCHAR cmdline[MAX_CMDLINE];
+                    res = get_process_cmdline(pi->ProcessID, 
+                                              cmdline, BUFFER_SIZE_ELEMENTS(cmdline));
+                    NULL_TERMINATE_BUFFER(cmdline);
+                    if (res == ERROR_SUCCESS) {
+                        if (!get_commandline_qualifier(cmdline, buf,
+                                                       BUFFER_SIZE_ELEMENTS(buf), !strip))
+                            buf[0] = L'\0'; /* no args */
+                        NULL_TERMINATE_BUFFER(buf);
+                    }
+                    if (res == ERROR_SUCCESS)
+                        fprintf(fp, "\tQname: %S%s%S\n", pi->ProcessName,
+                                buf[0] == L'\0' ? "" : "-", buf);
+                    else 
+                        fprintf(fp, "\t<Qname err %d>\n", res);
+                    
+                }
+                if (under_dr && hotp) {
+                    hotp_policy_status_table_t *status_tbl = NULL;
+                    res = get_hotp_status(pi->ProcessID, &status_tbl);
+                    if (res == ERROR_SUCCESS) {
+                        uint j;
+                        hotp_policy_status_t *cur;
+                        fprintf(fp, "\tHotpatching:\n", res); 
+                        for (j = 0; j < status_tbl->num_policies; j++) {
+                            char status_buf[MAX_PATH];
+                            cur = &(status_tbl->policy_status_array[j]);
+                            if (get_status_string(status_buf, MAX_PATH,
+                                                  cur->inject_status,
+                                                  cur->mode))
+                                fprintf(fp, "\t  Patch %s: %s\n",
+                                        cur->policy_id, status_buf);
+                        }
+                    }
+                    else if (res == ERROR_DRMARKER_ERROR) {
+                        fprintf(fp, "\tHot Patching Not Enabled\n", res);
+                    }
+                    else {
+                        fprintf(fp, "\t<Hotpatch Query Error %d>\n", res);
+                    }
+                }
+                if (under_dr && showstats) {
+                    dr_statistics_t *stats = get_dynamorio_stats(pi->ProcessID);
+                    if (stats != NULL) {
+                        uint i;
+                        fprintf(fp, "\t%.*s\n",
+                                BUFFER_SIZE_ELEMENTS(stats->process_name),
+                                stats->process_name);
+                        for (i = 0; i < stats->num_stats; i++) {
+                            fprintf(fp, "\t%*.*s :%9d\n",
+                                    BUFFER_SIZE_ELEMENTS(stats->stats[i].name),
+                                    BUFFER_SIZE_ELEMENTS(stats->stats[i].name),
+                                    stats->stats[i].name, stats->stats[i].value);
+                        }
+                    }
+                    free_dynamorio_stats(stats);
+                }
+                if (showmem) {
+                    print_mem_stats(pi, reschar, version);
+                }
+                count++;
+            }
+        } 
+    return TRUE;
+}
+
+BOOL
+dllw_callback(module_info_t *mi, void **param)
+{
+  char *resstr;
+  char reschar;
+  int res;
+  DWORD version;
+  BOOL exe_match = (exe && wcsstr(mi->BaseDllName, wexe)) ? TRUE : FALSE;
+  BOOL exe_present = (wcsstr(mi->BaseDllName, L".exe") || wcsstr(mi->BaseDllName, L".EXE")) ? TRUE : FALSE;
+
+  /* -listdlls option is set && any of below 3 conditions hold true.
+     (1) -pid value is set and current threadID matches.
+     (2) -exe is set and all DLLs with matching threadIDs (a fake comparison) since WINDOWS provides no other handle.
+     (3) -listall option is set
+  */
+
+  if ( listdlls && ((pid && mi->ProcessID == pid) ||
+                    ((exe_match || 
+                      (exe && save_pid == mi->ProcessID && 
+                       (!exe_present || wcscmp(save_module, mi->BaseDllName) == 0)))) ||
+                    (listall || listdr)))
+    {
+        version = -1;
+        res = under_dynamorio_ex (mi->ProcessID, &version);
+        switch (res) {
+        case DLL_PROFILE : resstr=NAME" profile"; reschar='P'; break;
+        case DLL_RELEASE : resstr=NAME" release"; reschar='R'; break;
+        case DLL_DEBUG : resstr=NAME" debug"; reschar='D'; break;
+        case DLL_CUSTOM : resstr=NAME" custom"; reschar='C'; break;
+        case DLL_NONE : resstr="native"; reschar='N'; break;
+        case DLL_UNKNOWN :
+        default : resstr="<error>"; reschar='?';
+        }
+
+        save_pid = mi->ProcessID;
+
+        if (!(listdr && (res==DLL_NONE || res==DLL_UNKNOWN))) {
+            if (wcsstr(mi->BaseDllName, L".exe") || wcsstr(mi->BaseDllName, L".EXE"))
+            {
+                wcsncpy(save_module, mi->BaseDllName, 
+                        sizeof(save_module)/sizeof(save_module[0]));
+                save_module[(sizeof(save_module)/sizeof(save_module[0]))-1] = L'\0';
+                if (!nopid && !showmem)
+                    fprintf(fp, "\n\nPID %d ", mi->ProcessID);
+                if (!showmem) {
+                    if (version == -1 || !showbuild) {
+                        fprintf(fp, "\t\tProcess %S, running %s\n",
+                                mi->BaseDllName,
+                                resstr);
+                    }
+                    else {
+                        fprintf(fp, "\t\tProcess %S, running %s (build %d)\n",
+                                mi->BaseDllName,
+                                resstr,
+                                version);
+                    }
+                }
+                count = 0;
+            }
+            else
+            {
+                if (!showmem) {
+                    if (showdlls) {
+                        /* long format */
+                        fprintf(fp, "  %08x-%08x  %-16S Stamp=%x Count=%d\n    %S\n",
+                                mi->BaseAddress, (uint)mi->BaseAddress+mi->SizeOfImage,
+                                mi->BaseDllName, mi->TimeDateStamp,
+                                mi->LoadCount, mi->FullDllName);
+                    } else {
+                        fprintf(fp, "\t%-16S",
+                                mi->BaseDllName);
+                        count++;
+                        if ((count % 3) == 0) {
+                            fprintf(fp, "\n");
+                            count = 0;
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+  return TRUE;
+}
+
+int
+procwalk()
+{
+    static int pwalk_per = 0;
+    uint system_load = get_system_load(sampling);
+    if (tillidle) {
+        static uint idle_for_ms = 0;
+        idle_for_ms = (system_load < idlecpu) ? idle_for_ms + millis : 0;
+        idle = (idle_for_ms >= flag_after_ms);
+    }
+
+    count = 0;
+    if (showmem) {
+        SYSTEM_PERFORMANCE_INFORMATION sperf_info;
+        if (get_system_performance_info(&sperf_info)) {
+            fprintf (fp, "System committed memory (KB): %d / %d peak %d\n",
+                     /* in pages so x4==KB */
+                     4*sperf_info.TotalCommittedPages,
+                     4*sperf_info.TotalCommitLimit,
+                     4*sperf_info.PeakCommitment);
+            /* FIXME: add physical memory, kernel memory, etc. */
+        }
+        fprintf (fp, "System load: %d%%\t\tUptime: %lu ms\n",
+                 system_load, get_uptime());
+
+        /* %user reaches 100% so we give it the extra column over %cpu */
+        fprintf (fp, "%-23s %5s %7s %3s %4s %5s %3s %7s %7s %8s %7s %7s %7s %7s %5s %5s %5s %5s %5s",
+                 "Name-Qualification", "PID", "DR  Bld", "CPU", "User", "Hndl", "Thr",
+                 "PVSz", "VSz", "PPriv", "Priv",
+                 "PWSS", "WSS", "Fault", "PPage", "Page", "PNonP", "NonP", "PPID");
+        if (showtime)
+            fprintf (fp, " %15s %15s %15s", "User Time(ms)", "Kernel Time(ms)",
+                                            "Create Time(ms)");
+        fprintf (fp, "\n");
+
+    }
+    if (pwalk_per == 0 || pwalk_per % showmemfreq == 0) {
+        process_walk (&pw_callback, NULL);
+        if (!count)
+            fprintf (fp, "No such process found.\n");
+        else {
+            if (showmem && showtime) {
+                fprintf (fp, "Total scheduled user/kernel time: %I64d/%I64d ms\n",
+                         total_user/10000, total_kernel/10000);
+            }
+        }
+    }
+    pwalk_per++;
+
+    total_user = 0;
+    total_kernel = 0;
+
+    return count;
+}
+
+void
+dllwalk()
+{
+  count = 0;
+  dll_walk_all(&dllw_callback, NULL);
+}
+
diff --git a/tools/Makefile b/tools/Makefile
new file mode 100644
index 000000000..107b9e643
--- /dev/null
+++ b/tools/Makefile
@@ -0,0 +1,278 @@
+# **********************************************************
+# Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+ifndef DYNAMORIO_MAKE
+  DYNAMORIO_MAKE := ../make
+endif
+
+include $(DYNAMORIO_MAKE)/compiler.mk
+
+# 
+# Linux
+#
+ifeq ($(MACHINE), linux)
+
+.PHONY: all
+
+all: runstats
+
+runstats: runstats.c
+	$(CC) $< -o $@
+
+clean:
+	$(RM) runstats
+
+#
+# Windows
+#
+else
+
+# use DYNAMORIO_HOME and LIBUTIL if defined
+ifndef DYNAMORIO_HOME
+  DYNAMORIO_HOME := ..
+endif
+ifndef DYNAMORIO_LIBUTIL
+  DYNAMORIO_LIBUTIL := $(DYNAMORIO_HOME)/libutil
+endif
+
+WIN_DYNAMORIO_HOME := $(shell $(CYGPATH) -ma $(DYNAMORIO_HOME))
+WIN_DYNAMORIO_LIBUTIL := $(shell $(CYGPATH) -ma $(DYNAMORIO_LIBUTIL))
+
+CORE_DIRNAME := core
+
+# MSINCLUDE intentionally last so we don't get VisualStudio's share.h
+INCLUDES := /I"$(WIN_DYNAMORIO_HOME)/$(CORE_DIRNAME)" \
+	    /I"$(WIN_DYNAMORIO_HOME)/$(CORE_DIRNAME)/lib" \
+	    /I"$(WIN_DYNAMORIO_LIBUTIL)" \
+	    $(MSINCLUDE)
+
+# We get resource defines and the build number from
+# Makefile.custom_build. (DIFFNUM is referenced there)
+ifndef DIFFNUM
+  DIFFNUM := 1
+endif
+include $(DYNAMORIO_HOME)/$(CORE_DIRNAME)/Makefile.custom_build
+
+BNFLAGS := /DBUILD_NUMBER=$(BUILD_NUMBER)
+ifneq ($(VERSION_NUMBER),)
+  BNFLAGS += /DVERSION_NUMBER=$(VERSION_NUMBER)
+endif
+ifneq ($(VERSION_COMMA_DELIMITED),)
+  BNFLAGS += /DVERSION_COMMA_DELIMITED=$(VERSION_COMMA_DELIMITED)
+endif
+
+LDFLAGS += $(MSLIB) /release /opt:ref /opt:icf /machine:IX86 \
+	   /LIBPATH:"$(WIN_DYNAMORIO_LIBUTIL)" /mapinfo:exports \
+	   user32.lib advapi32.lib pdh.lib imagehlp.lib Shell32.lib
+
+# FIXME eventually upgrade to W4 with pragma exceptions.
+# W2 is default (we're using W3).  We should also replace
+# all references to unsafe functions (e.g., fopen) and
+# remove /wd4996
+CFLAGS += /W3 /WX /wd4996 $(INCLUDES) /DNOT_DYNAMORIO_CORE /DWIN32 $(BNFLAGS)
+ifeq ($(EXTERNAL_DRVIEW), 1)
+CFLAGS += /DEXTERNAL_DRVIEW=1
+endif
+
+# xref PR 192750 - runregression overrides this
+DBG = /Zi
+LINK_DBG = /debug
+
+ifdef DEBUG
+  CFLAGS += $(DBG) /Od /MTd
+  LDFLAGS += $(LINK_DBG)
+  POLICY_LIB := $(DYNAMORIO_LIBUTIL)/policy_staticd.lib
+else
+# use $(DBG) & $(LINK_DBG) so we get a pdb file (like we do for core release)
+  CFLAGS += $(DBG) /O2 /MT
+  LDFLAGS += $(LINK_DBG)
+  POLICY_LIB := $(DYNAMORIO_LIBUTIL)/policy_static.lib
+endif
+
+CONFIG_DLL  := drconfig.dll
+CONFIG_LIB  := drconfig.lib
+CONFIG_PATH := '$(shell $(CYGPATH) -ma $(DYNAMORIO_LIBUTIL))'
+
+CFLAGS_NO_UNICODE := $(CFLAGS)
+CFLAGS += /DUNICODE /D_UNICODE 
+
+CFLAGS_DRDEPLOY := $(CFLAGS)
+ifndef VMAP
+# We default to VMSAFE configuration
+CFLAGS_DRDEPLOY += /DPROGRAM_SHEPHERDING /DMF_API /DHOT_PATCHING_INTERFACE /DPROBE_API
+endif
+
+WIN_POLICY_LIB := "$(shell $(CYGPATH) -ma $(POLICY_LIB))"
+
+RCDEFS  := $(RESOURCE_DEFINES) /DFILE_TYPE=VFT_APP
+RCFLAGS := -r $(INCLUDES) $(RCDEFS) $(BNFLAGS)
+
+RCFILE := $(DYNAMORIO_HOME)/$(CORE_DIRNAME)/win32/resources.rc
+WIN_RCFILE := "$(shell $(CYGPATH) -ma $(RCFILE))"
+
+TGTS := DRview.exe DRcontrol.exe DRkill.exe closewnd.exe \
+	winstats.exe svccntrl.exe DRload.exe \
+	balloon.exe dummy.exe ldmp.exe \
+	drdeploy.exe
+
+.PHONY: DRgui policy_lib_check
+
+all: $(TGTS) DRgui
+
+%.exe: %.obj %.res $(POLICY_LIB)
+	$(LD) $*.obj $*.res $(LDFLAGS) $(WIN_POLICY_LIB)
+
+%.obj: %.c 
+	$(CC) $(CFLAGS) /c $<
+
+$(POLICY_LIB):
+	@(if [ ! -e $(POLICY_LIB) ]; then \
+	$(ECHO) "$(POLICY_LIB) does not exist; you may need to build it."; \
+	false; fi)
+
+# drdeploy.exe links to libutil/drconfig.dll.  When we build drdeploy,
+# we'll copy the DLL to the tools directory so users won't need to
+# update their path.
+drdeploy.exe: $(CONFIG_DLL) drdeploy.res drdeploy.c
+	$(CC) $(CFLAGS_DRDEPLOY) /c drdeploy.c
+	$(LD) drdeploy.obj drdeploy.res $(LDFLAGS) $(CONFIG_LIB) $(WIN_POLICY_LIB)
+# We could add a manifest so that we can request elevation on Vista (xref PR 233520).
+# Works fine for double clicking the app, but running from a cmd shell just gives access
+# denied if the cmd shell isn't runas admin which isn't quite as nice.  Disable for now. 
+# $(MT) -manifest drdeploy.exe.manifest -outputresource:drdeploy.exe
+
+$(CONFIG_DLL): $(DYNAMORIO_LIBUTIL)/$(CONFIG_DLL)
+	$(CP) $< ./
+
+$(DYNAMORIO_LIBUTIL)/$(CONFIG_DLL):
+	@(if [ ! -e $@ ]; then \
+	$(ECHO) "$@ does not exist; you may need to build it."; \
+	false; fi)
+
+closewnd.obj: closewnd.c
+	$(CC) $(CFLAGS_NO_UNICODE) /c $< 
+
+winstats.obj: winstats.c
+	$(CC) $(CFLAGS_NO_UNICODE) /c $< 
+
+# need ntdll.lib
+ifeq ($(ARCH), x64)
+NTDLL_LIBPATH := $(NTDLL_LIBPATH_X64)
+else
+NTDLL_LIBPATH := $(NTDLL_LIBPATH_X86)
+endif
+ldmp.exe: ldmp.c ldmp.res
+	$(CC) $(CFLAGS) $< /link ldmp.res $(LDFLAGS) /LIBPATH:"$(subst /,\\,$(NTDLL_LIBPATH))" ntdll.lib
+
+DRview.res: $(RCFILE)
+	$(RC) /Fo$@ $(RCFLAGS) /DFILE_NAME=\"DRview.exe\" /DFILE_DESCRIPTION="PRODUCT_NAME \" cmd line status viewer\"" $(WIN_RCFILE)
+
+DRcontrol.res: $(RCFILE)
+	$(RC) /Fo$@ $(RCFLAGS) /DFILE_NAME=\"DRcontrol.exe\" /DFILE_DESCRIPTION="PRODUCT_NAME \" cmd line configuration tool\"" $(WIN_RCFILE)
+
+DRkill.res: $(RCFILE)
+	$(RC) /Fo$@ $(RCFLAGS) /DFILE_NAME=\"DRkill.exe\" /DFILE_DESCRIPTION="PRODUCT_NAME \" process terminating utility\"" $(WIN_RCFILE)
+
+drdeploy.res: $(RCFILE)
+	$(RC) /Fo$@ $(RCFLAGS) /DFILE_NAME=\"drdeploy.exe\" /DFILE_DESCRIPTION="PRODUCT_NAME \" DynamoRIO deployment command-line utility\"" $(WIN_RCFILE)
+
+closewnd.res: $(RCFILE)
+	$(RC) /Fo$@ $(RCFLAGS) /DFILE_NAME=\"closewnd.exe\" /DFILE_DESCRIPTION="\"closes window app\"" /DPRERELEASE_BUILD $(WIN_RCFILE)
+
+winstats.res: $(RCFILE)
+	$(RC) /Fo$@ $(RCFLAGS) /DFILE_NAME=\"winstats.exe\" /DFILE_DESCRIPTION="\"gathers memory stats\"" /DPRERELEASE_BUILD $(WIN_RCFILE)
+
+svccntrl.res: $(RCFILE)
+	$(RC) /Fo$@ $(RCFLAGS) /DFILE_NAME=\"svccntrl.exe\" /DFILE_DESCRIPTION="PRODUCT_NAME \" cmd line services configuration utility\"" $(WIN_RCFILE)
+
+DRload.res: $(RCFILE)
+	$(RC) /Fo$@ $(RCFLAGS) /DFILE_NAME=\"DRload.exe\" /DFILE_DESCRIPTION="\"loads a dll for analysis\"" /DPRERELEASE_BUILD $(WIN_RCFILE)
+
+balloon.res: $(RCFILE)
+	$(RC) /Fo$@ $(RCFLAGS) /DFILE_NAME=\"balloon.exe\" /DFILE_DESCRIPTION="\"out of memory testing tool\"" /DPRERELEASE_BUILD $(WIN_RCFILE)
+
+runjob.res: $(RCFILE)
+	$(RC) /Fo$@ $(RCFLAGS) /DFILE_NAME=\"runjob.exe\" /DFILE_DESCRIPTION="\"job limit testing tool\"" /DPRERELEASE_BUILD $(WIN_RCFILE)
+
+readproc.res: $(RCFILE)
+	$(RC) /Fo$@ $(RCFLAGS) /DFILE_NAME=\"readproc.exe\" /DFILE_DESCRIPTION="\"dump process memory\"" /DPRERELEASE_BUILD $(WIN_RCFILE)
+
+drdel.res: $(RCFILE)
+	$(RC) /Fo$@ $(RCFLAGS) /DFILE_NAME=\"drdel.exe\" /DFILE_DESCRIPTION="\"delete from file cache\"" /DPRERELEASE_BUILD $(WIN_RCFILE)
+
+dummy.res: $(RCFILE)
+	$(RC) /Fo$@ $(RCFLAGS) /DFILE_NAME=\"dummy.exe\" /DFILE_DESCRIPTION="\"dummy process for ldmp creation\"" /DPRERELEASE_BUILD $(WIN_RCFILE)
+
+ldmp.res: $(RCFILE)
+	$(RC) /Fo$@ $(RCFLAGS) /DFILE_NAME=\"ldmp.exe\" /DFILE_DESCRIPTION="\".ldmp reading tool\"" /DPRERELEASE_BUILD $(WIN_RCFILE)
+
+
+DRgui:
+# case 192750: propagate whether to build pdb files
+	$(MAKE) -C DRgui DBG=$(DBG) LINK_DBG=$(LINK_DBG)
+
+clean:
+	$(RM) *.obj *.ilk *.pdb *.map *.exp *.res *.dll
+	$(RM) $(TGTS)
+	$(MAKE) -C DRgui clean
+
+
+# Support for adding symbols to a symbol server
+# you can do this only on Windows
+
+# You need of course double backslashes if you want to pass a name, 
+# so \\\\fileserver\\symbols\\product
+
+#FIXME: set the IP properly for fileserver, right now it is going to evlk-w2kas 
+SYMBOL_SERVER=\\\\fileserver\\symbols
+
+# I also like my VMs to have host only network so then I do
+#   subst y: z:\shared\symbols
+# which would be host shared with /mnt/fileserver/symbols if that were writable
+#SYMBOL_SERVER=y:\\
+
+SYMBOL_STORE=${SYMBOL_SERVER}\\product
+
+# Note: you'd need EXPORTS=..\\..\\exports if you don't have DYNAMORIO_HOME setup
+EXPORTS=${WIN_DYNAMORIO_HOME}\\exports
+GUI_EXPORTS=${WIN_DYNAMORIO_HOME}\\mmcgui\\Release
+PRODUCT_NAME="SecureCore"
+GUI_NAME="SecureCore GUI"
+DATE='$(shell $(DATE) +"Built at %Y-%m-%d %T")'
+COMMENT="Build ${BUILD_NUMBER} by ${USER}@${HOSTNAME}"
+
+symbols:
+	cd external; ./symstore add /r /f ${EXPORTS} /s ${SYMBOL_STORE} /t ${PRODUCT_NAME} /v ${COMMENT} /c ${DATE}
+
+guisymbols:
+	cd external; ./symstore add /r /f ${GUI_EXPORTS} /s ${SYMBOL_STORE} /t ${GUI_NAME} /v ${COMMENT} /c ${DATE}
+# FIXME: should tail ${SYMBOL_STORE}/000Admin/history to check if the file was really added
+
+endif
diff --git a/tools/Makefile.common b/tools/Makefile.common
new file mode 100644
index 000000000..acba74b37
--- /dev/null
+++ b/tools/Makefile.common
@@ -0,0 +1,84 @@
+# **********************************************************
+# Copyright (c) 2007 VMware, Inc.  All rights reserved.
+# **********************************************************/
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# toolchain is the only supported mechanism
+# see http://wiki.eng.vmware.com/Build/Toolchain/Access for toolchain setup
+
+I       =-I
+
+# common fragment that should be shared across all modules
+USE_SDK60 = 1
+ifeq ($(ARCH), x64)
+  BINSUBDIR=/x86_x64
+  LIBSUBDIR=/x64
+  # should rename this one: toolchain bug
+  MFCLIBSUBDIR=/amd64
+else
+  BINSUBDIR=
+  LIBSUBDIR=
+  MFCLIBSUBDIR=
+endif
+# xref bora/make/mk/defs-compilersWin32.mk
+ifeq ($(USE_SDK60),1)
+  WROOT_BASE := $(TCROOT)/win32/winsdk-6.1.6000
+  WROOT_VC := $(WROOT_BASE)/VC
+  WROOT_PSDK := $(WROOT_BASE)
+  WROOT_SDKTOOLS := $(WROOT_PSDK)/Bin
+else
+  ifeq ($(USE_VC71),1)
+    WROOT_BASE := $(TCROOT)/win32/visualstudio-2003/
+    WROOT_VC := $(WROOT_BASE)/VC7
+    WROOT_PSDK := $(WROOT_VC)/PlatformSDK
+    WROOT_SDKTOOLS := $(WROOT_BASE)/Common7/Tools/Bin
+  else # VC6
+    WROOT_BASE := $(TCROOT)/win32/visual-studio-98
+    WROOT_VC := $(WROOT_BASE)/vc98
+    # no SDK; FIXME: missing Common/MSDev98/Bin for rc.exe, ml.exe!
+    WROOT_PSDK := $(WROOT_VC)
+    WROOT_SDKTOOLS := $(WROOT_VC)/Bin
+  endif
+endif
+# we are not setting INCLUDE or LIB env vars, so we set all paths here
+MSINCLUDE = $(I)'$(shell cygpath -wa "$(WROOT_PSDK)/Include")' \
+	    $(I)'$(shell cygpath -wa "$(WROOT_VC)/INCLUDE")' \
+	    $(I)'$(shell cygpath -wa "$(WROOT_VC)/atlmfc/include")'
+MSLIB     = $(L)'$(shell cygpath -wa "$(WROOT_PSDK)/Lib$(LIBSUBDIR)")' \
+	    $(L)'$(shell cygpath -wa "$(WROOT_VC)/LIB$(LIBSUBDIR)")' \
+	    $(L)'$(shell cygpath -wa "$(WROOT_VC)/atlmfc/lib$(MFCLIBSUBDIR)")'
+# we do not set PATH and rely on mspdb80.dll and mspdbsrv.exe being
+# in same dir as cl.exe.
+CC :=  $(WROOT_VC)/Bin$(BINSUBDIR)/cl.exe
+CFLAGS := /nologo
+LD := $(WROOT_VC)/Bin$(BINSUBDIR)/link.exe
+# FIXME - use LDFLAGS where needed 
+LD += /nologo
+MC :=  $(WROOT_SDKTOOLS)/mc.exe
+RC :=  $(WROOT_SDKTOOLS)/rc.exe
+ML :=  $(WROOT_VC)/Bin/ml.exe
diff --git a/tools/PCProf b/tools/PCProf
new file mode 100644
index 000000000..2f3c6725c
--- /dev/null
+++ b/tools/PCProf
@@ -0,0 +1,245 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2001-2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### PCProf
+### author: Derek Bruening   February 2001
+###
+### Analyzes dynamorio_pcsamples.log
+
+$usage = "Usage: $0 [-file fragfile] [-exe exename]\n";
+
+#defaults
+$infile = "dynamorio_pcsample.log";
+$have_exe = 0;
+
+# get optional params
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq '-exe') {
+	if ($#ARGV <= 0) { print $usage; exit; }
+	shift;
+	$exe = $ARGV[0];
+	$have_exe = 1;
+    } elsif ($ARGV[0] eq '-file') {
+	if ($#ARGV <= 0) { print $usage; exit; }
+	shift;
+	$infile = $ARGV[0];
+    } else {
+	print $usage;
+	exit;
+    }
+    shift;
+}
+
+if (!(-s $infile)) {
+    print STDERR "*** $infile is empty, no output produced\n";
+    exit;
+}
+
+if ($have_exe) {
+    # see if dynamorio is in shared library or is statically linked in
+    $sharedlib = 0;
+    $dis_offs = 0;
+    $dis_exe = $exe;
+    open(LDD, "ldd $exe |") || die "Error: Couldn't run ldd $exe\n";
+    while (<LDD>) {
+	if ($_ =~ /libdynamorio.so => (\S+) \((0x[0-9A-Fa-f]+)\)/) {
+	    $sharedlib = 1;
+	    $dis_offs = hex($2);
+	    $dis_exe = $1;
+	    last;
+	}
+    }
+    close(LDD);
+    # disassemble exe, read in assembly code to get function pc delimiters
+    system("objdump -d $dis_exe > $exe.dis");
+    system("echo \"\" >> $exe.dis"); # blank line to get end of last func!
+    $i = 0;
+    $in_func = 0;
+    open(OBJ, "< $exe.dis") || die "Error: Couldn't open $exe.dis for input\n";
+    while (<OBJ>) {
+	chop;
+	$l = $_;
+	if ($in_func) {
+	    if ($l =~ /^\s*([0-9A-Fa-f]+)/) {
+		$last_addr = hex($1) + $dis_offs;
+	    } else {
+		$last{$faddr} = $last_addr;
+		$in_func = 0;
+		#printf("Func $func{$faddr} = 0x%08x ... 0x%08x\n",
+		#       $first{$faddr}, $last{$faddr});
+	    }
+	}
+	if ($l =~ /^([0-9A-Fa-f]+) <([^>]+)>/) {
+	    # can have multiple func names (if static) so key off of
+	    #   function address, not func name!
+	    $faddr = hex($1) + $dis_offs;
+	    $func{$faddr} = $2;
+	    $first{$faddr} = $faddr;
+	    $in_func = 1;
+	}
+    }
+    close(OBJ);
+    system("rm -f $exe.dis");
+}
+
+$total_hits = 0;
+$trace_hits = 0;
+$fragment_hits = 0;
+$in_header = 1;
+$header_cnt = 0;
+open(IN, "< $infile") || die "Error: Couldn't open $infile for input\n";
+while (<IN>) {
+    chop;
+    # handle DOS end-of-line:
+    if ($_ =~ /
+$/) { chop; };
+    $l = $_;
+    if ($in_header) {
+	if ($l =~ /^\s*$/ || $l =~ /^PC/) {
+	    $in_header = 0;
+	} else {
+	    $header[$header_cnt++] = $l;
+	}
+    }
+    if ($l =~ /^pc=/) {
+	if ($l =~ /trace/) {
+	    if ($l =~ /trace \#/) {
+		$l =~ /^[^\#]+\#=([0-9]+)[^\#]+\# *([0-9]+) @([x0-9a-f]+)[^0]+([x0-9a-f]+)/;
+		# $1 = count, $2 = id, $3 = tag, $4 = offs
+		$count = $1;
+		$id = $2;
+		$tag{$id} = $3;
+	    } else {
+		# release run: no trace id!
+		$l =~ /^[^\#]+\#=([0-9]+)[^@]+@([x0-9a-f]+)[^0]+([x0-9a-f]+)/;
+		# $1 = count, $2 = tag, $3 = offs
+		$count = $1;
+		$id = $2;
+		$tag{$id} = $2;
+	    }
+	    $hits{$id} += $count;
+	    $total_hits += $count;
+	    $trace_hits += $count;
+	    $trace{$id} = 1;
+	} elsif ($l =~ /fragment/) {
+	    if ($l =~ /fragment \#/) {
+		$l =~ /^[^\#]+\#=([0-9]+)[^\#]+\# *([0-9]+) @([x0-9a-f]+)[^0]+([x0-9a-f]+)/;
+		# $1 = count, $2 = id, $3 = tag, $4 = offs
+		$count = $1;
+		$id = $2;
+		$tag{$id} = $3;
+	    } else {
+		# release run: no fragment id!
+		$l =~ /^[^\#]+\#=([0-9]+)[^@]+@([x0-9a-f]+)[^0]+([x0-9a-f]+)/;
+		# $1 = count, $2 = tag, $3 = offs
+		$count = $1;
+		$id = $2;
+		$tag{$id} = $2;
+	    }
+	    $hits{$id} += $count;
+	    $total_hits += $count;
+	    $fragment_hits += $count;
+	    $trace{$id} = 0;
+	} else {
+	    $l =~ /^pc=([x0-9a-f]+)[^\#]+\#=([0-9]+)[^i]+in (.+)/;
+	    $pc = $1;
+	    $id = $3;
+	    $num_hits = $2;
+	    if ($have_exe) {
+		$fun = &lookup_func($pc);
+		# print "$id, $pc = $fun\n";
+		$id = "$id: $fun";
+	    }
+	    $hits{$id} += $num_hits;
+	    $total_hits += $num_hits;
+	    $tag{$id} = $id;
+	    $trace{$id} = -1;
+	}
+    }
+}
+close(IN);
+
+if ($total_hits == 0) {
+    print STDERR "*** $infile has no information, no output produced\n";
+    exit;
+}
+
+#foreach $h (keys %hits) {
+#    print "Trace $h ($tag{$h}) has $hits{$h} hits\n";
+#}
+
+for ($i=0; $i<$header_cnt; $i++) {
+    print "$header[$i]\n";
+}
+print "\nBREAKDOWN:\n";
+
+@ids = keys(%hits);
+@sorted = sort({$hits{$a} <=> $hits{$b}} @ids);
+
+foreach $s (@sorted) {
+    $h = $hits{$s};
+    $p = 100.0 * ($h / $total_hits);
+    if ($trace{$s} == -1) {
+	$out = sprintf("%5d hit(s) = %4.1f%% for %s\n",
+		       $h, $p, $tag{$s});
+    } elsif ($trace{$s} == 0) {
+	$out = sprintf("%5d hit(s) = %4.1f%% for fragment %s (%s)\n",
+		       $h, $p, $s, $tag{$s});
+    } else {
+	$out = sprintf("%5d hit(s) = %4.1f%% for trace %s (%s)\n",
+		       $h, $p, $s, $tag{$s});
+    }
+    print $out;
+}
+
+print "\n";
+$p = 100.0 * ($trace_hits / $total_hits);
+$out = sprintf("%5d hit(s) = %4.1f%% for ALL TRACES\n", $trace_hits, $p);
+print $out;
+$p = 100.0 * ($fragment_hits / $total_hits);
+$out = sprintf("%5d hit(s) = %4.1f%% for ALL FRAGMENTS\n", $fragment_hits, $p);
+print $out;
+
+sub lookup_func($) {
+    my ($in_pc) = @_;
+    my ($f, $pc);
+    $pc = hex($in_pc);
+    # start & end guaranteed to be in same function
+    # just do linear search, performance not an issue
+    foreach $f (keys %first) {
+	if ($pc >= $first{$f} && $pc <= $last{$f}) {
+	    # print "$start...$end is in $func{$f}:\n";
+	    return $func{$f};
+	}
+    }
+    return "<couldn't find it>";
+}
diff --git a/tools/RunInPar b/tools/RunInPar
new file mode 100644
index 000000000..d9d1032c8
--- /dev/null
+++ b/tools/RunInPar
@@ -0,0 +1,77 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2001 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### RunInPar
+### Runs shell commands in a text file in parallel
+### author: Derek Bruening   April 2001
+###
+### Must place "-SPLIT" on its own line to separate processors' work
+
+$usage = "Usage: $0 <file>\n";
+if ($#ARGV < 0) {
+    print $usage;
+    exit;
+}
+$infile = shift;
+
+$proc = 0;
+$line = 0;
+open(IN, "< $infile") || die "Error: Couldn't open $infile for input\n";
+open(OUT, "> .#mp-$proc") || die "Error in opening output file\n";
+while (<IN>) {
+    next if /^\#/;
+    next if /^$/;
+    if ($_ =~ /^-SPLIT$/) {
+	close(OUT);
+	$proc++;
+	open(OUT, "> .#mp-$proc") || die "Error in opening output file\n";
+    } else {
+	print OUT $_;
+    }
+}
+close(IN);
+close(OUT);
+
+@subs = `ls .#mp-*`;
+foreach $f (@subs) {
+    chop $f; # this affects @subs, only do it once!
+    unless (fork()) {
+	exec("/bin/csh -s < $f");
+	exit(0);
+    }
+}
+foreach $f (@subs) {
+    wait;
+}
+
+# clean up
+system("rm .#mp-*");
diff --git a/tools/Summary b/tools/Summary
new file mode 100755
index 000000000..97c0b2b60
--- /dev/null
+++ b/tools/Summary
@@ -0,0 +1,284 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2001-2003 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### Summary
+### author: Derek Bruening   April 2001
+###
+### Produces a table summarizing info in the dynamorio log files
+### in some directory tree.
+
+$usage = "Usage: $0 [-times] [-find findcmd] [-noperfctr]\n";
+
+@perfctrnames=("Cycles",
+	       "Inst._Decoded",
+	       "Inst._Retired",
+	       "Micro-ops_Retired",
+	       "data_mem_refs",
+	       "DCU_misses",
+	       "Inst._Fetches",
+	       "IFU_misses",
+	       "ITLB_misses",
+	       "cycles_inst_fetch_stalled",
+	       "cycles_inst_fetch_decode_stalled",
+	       "L2_inst_fetches",
+	       "L2_loads",
+	       "L2_stores",
+	       "branches_decoded",
+	       "branches_retired",
+	       "branches_mispredicted_retired",
+	       "branches_taken_retired",
+	       "branches_taken_mispredicted_retired",
+	       "BTB_misses",
+	       "Bogus_branches",
+	       "misc_resource_stalls",
+	       "BACLEAR_asserted",
+	       "DCU_lines_allocated",
+	       "L2_lines_allocated",
+	       "prefetch_NTA_dispatched",
+	       "prefetch_NTA_miss_all_caches"
+	       );
+$numperfctrs=scalar(@perfctrnames);
+
+@args = @ARGV;
+$times_only = 0;
+$find = "find . -type f -name mainlog.\\* | grep 'dynamorio\\.'";
+$noperfctr = 0;
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq '-find') {
+	if ($#ARGV <= 0) { print $usage; exit; }
+	shift;
+	$find = $ARGV[0];
+	print "$0 @args\n\n";
+    } elsif ($ARGV[0] eq '-times') {
+	$times_only = 1;
+    } elsif ($ARGV[0] eq '-noperfctr') {
+	$noperfctr = 1;
+    } else {
+	print $usage;
+	exit;
+    }
+    shift;
+}
+
+unless ($times_only) {
+    printf("%-43s  Traces / Frags  Heap  BB Used/Cap   Trace U/Cap  Minutes\n",
+	   "Path");
+    if (!$noperfctr) {
+	$have_perfctr = `$find | head -1 | xargs grep \"^Counter 1 \"`;
+	$noperfctr = ($have_perfctr eq "");
+    }
+    if (!$noperfctr) {
+	print join "\t",@perfctrnames;
+	print "\n";
+    }
+    printf("%-43s  --------------  ---- ------------- ------------- -------\n",
+	   "--------------------------------------");
+    if (!$noperfctr) {
+	for (1..$numperfctrs) {print "----------- ";}
+	print "\n";
+    }
+}
+
+$perfctr_num = 0;
+
+open(FIND, "$find |") || die "Error running $find\n";
+while (<FIND>) {
+    $have_elapsed = 0;
+    $elapsed = 0;
+    if (!$noperfctr) {
+	@perfctr = (-1)x$numperfctrs;
+    }
+    chop;
+    $path = $_;
+    if ($path =~ /(.+\/)dynamorio\.([0-9]*)\/mainlog\.[0-9]*/) {
+	$dir = $1;
+	$num = $2;
+	$logpath = "$dir"."dynamorio."."$num";
+	$errs = ""; # use dynamorio log file for elapsed time
+    } else {
+	# no log files, just want total time
+	if (!$times_only) {
+	    print "Error: no dynamorio log files found\n";
+	    exit 1;
+	}
+	if (-d $path) {
+	    $dir = "$path/";
+	} else {
+	    $path =~ /(.+\/)[^\/]*$/;
+	    $dir = $1;
+	}
+	$errs = `ls $dir*.err`;
+	if ($errs eq "") {
+	    print "Error: no dynamorio log files and no .err files found\n";
+	    next;
+	}
+    }
+    if ($times_only) {
+	print "Processing $dir\n";
+    }
+
+    unless ($times_only && $errs ne "") {
+	# read dynamorio log files
+	$error = 0;
+	$num_frags = -1;
+	$num_traces = -1;
+	$heap_used = 0;
+	$fcache_bb_used = 0;
+	$fcache_bb_cap = 0;
+	$fcache_trace_used = 0;
+	$fcache_trace_cap = 0;
+	# must read mainlog and all thread log files
+	open(LOGS, "find $logpath -name \\*log\\* |") || die "Error running $find\n";
+	while (<LOGS>) {
+	    open(IN, "< $_") || die "Error: Couldn't open $path for input\n";
+	    while (<IN>) {
+		chop;
+		$l = $_;
+
+		# skip log files for runstats
+		if ($l =~ /Running: runstats/) {
+		    $runstats = 1;
+		} elsif ($l =~ /execve/) {
+		    $runstats = 0;
+		}
+
+		if ($l =~ /^Fragments generated: ([0-9]+)/) {
+		    $num_frags = $1;
+		} elsif ($l =~ /^Trace fragments generated: ([0-9]+)/) {
+		    $num_traces = $1;
+		} elsif ($l =~ /(.+)cache: capacity ([0-9]+) KB, used ([0-9]+) KB/) {
+		    $which = $1;
+		    $used = $2;
+		    $cap = $3;
+		    if ($which =~ /Basic/) {
+			$fcache_bb_used += $used;
+			$fcache_bb_cap += $cap;
+		    } else {
+			$fcache_trace_used += $used;
+			$fcache_trace_cap += $cap;
+		    }
+#		} elsif ($l =~ /^Fcache space reserved: ([0-9]+) bytes/) {
+#		    $fcache_tot = $1;
+		} elsif ($l =~ /Total heap used: ([0-9]+) KB/) {
+		    $heap_used += $1;
+		} elsif (!$noperfctr && $l =~ /^Counter ([0-9]+) = ([0-9]+)/) {
+		    $perfctr[$1] = $2;
+		    if ($1 > $perfctr_num) {
+			$perfctr_num = $1;
+		    }
+		} elsif ($l =~ /Error/) {
+		    $error = 1;
+		} elsif ($l =~ /^Total running time: ([0-9]+) seconds/) {
+		    $elapsed += $1/60.;
+		    $have_elapsed = 1;
+		    if ($times_only) {
+			print "\tTime from $path\n";
+		    }
+		}
+	    }
+	    close(IN);
+	    if ($num_frags == -1) {
+		$error = 1;
+	    }
+	}
+    }
+
+    # skip log files for runstats
+    goto nextdir if ($runstats);
+
+    if (!$have_elapsed && $errs ne "") {
+	# get elapsed time from .err files
+	open(ELAP, "ls $dir*.err |") || die "Error finding $dir*.err\n";
+	while (<ELAP>) {
+	    chop;
+	    if ($times_only) {
+		print "\tTime from $_\n";
+	    }
+	    open(IN, "< $_") || die "Error: Couldn't open $_ for input\n";
+	    while (<IN>) {
+		chop;
+		if ($_ =~ /Elapsed: ([0-9]+):([0-9]+):([0-9\.]+)/) {
+		    $elapsed += $1*60 + $2 + $3/60.; # minutes
+		} elsif ($_ =~ /Timeout after ([0-9]+) seconds/) {
+		    $elapsed += $1/60.; # minutes
+		}
+	    }
+	    close(IN);
+	}
+	close(ELAP);
+	$have_elapsed = 1;
+    }
+
+    if ($times_only) {
+	if ($dir =~ /-[on]o?pt-[tr].+\//) {
+	    $dir =~ m|\./[^/]+/([^/]+)/[^/]+-([on])o?pt-([tr]).+/|;
+	    $bmark = $1;
+	    $opt = $2;
+	    $dset = $3;
+	} else {
+	    $dir =~ m|\./[^/]+/([^/]+)/[^/]+/|;
+	    $bmark = $1;
+	    # assume opt ref!
+	    $opt = 'o';
+	    $dset = 'r';
+	}
+	$val{$bmark,$opt,$dset} += $elapsed;
+	$bm{$bmark} = 1;
+    } else {
+	printf("%-43s %6d / %6d %5d %5d / %5d %5d / %5d  %5.1f",
+	       "$dir$num", $num_traces, $num_frags, $heap_used,
+	       $fcache_bb_used, $fcache_bb_cap, 
+	       $fcache_trace_used, $fcache_trace_cap, 
+	       $elapsed);
+	if (!$noperfctr) {
+	    for ($i = 1; $i < $perfctr_num; $i++) {
+		printf("\t%15.0f", $perfctr[$i]);
+	    }
+	}
+	if ($error) {
+	    print "  Error";
+	}
+	print "\n";
+    }
+  nextdir:
+}
+close(FIND);
+
+if ($times_only) {
+    print("\nBenchmark\tRef,n\tRef,o\tTest,n\tTest,o\n");
+    foreach $b (sort (keys %bm)) {
+	printf("%10s\t%6.2f\t%6.2f\t%6.2f\t%6.2f\n",$b,
+	       $val{$b,"n","r"},$val{$b,"o","r"},
+	       $val{$b,"n","t"},$val{$b,"o","t"});
+    }
+}
+
diff --git a/tools/address_query.pl b/tools/address_query.pl
new file mode 100755
index 000000000..ad67c165e
--- /dev/null
+++ b/tools/address_query.pl
@@ -0,0 +1,278 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2004-2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### address_query.pl
+### Derek Bruening, June 2004
+###
+### launches a debugger and runs the "ln" command to get the nearest
+###   symbol to each address in question
+### works best if you have symbols -- for DR dll, have pdb in same dir,
+###   for OS set _NT_SYMBOL_PATH 
+### sample usage:
+###   address_query.pl c:/builds/11087/exports/x86_win32_rel/dynamorio.dll 7004e660 77f830e7
+### it also accepts stdin or a file with -f <file>
+###
+### REQUIRES that DYNAMORIO_TOOLS is set and that it points to 
+###   an already-built DRload.exe
+###
+### note about -i interactive use feature:
+###   cdb within rxvt has some problems: interactive typing loses the last
+###   char or more, so put spaces after everything.
+###   for interactive should launch ntsd, or run cdb from within a cygwin-in-cmd shell
+
+# default debugger paths to try
+# we first try cdb from Debugging Tools For Windows
+#   (so we can get stdout and don't need a temp logfile)
+# else we use the ntsd on everyone's machines and a logfile,
+#   which we clobber every time
+$sysroot = &win32path2unix($ENV{'SYSTEMROOT'});
+$progroot = &win32path2unix($ENV{'PROGRAMFILES'});
+# try to find installation of debugging tools for windows
+@debugtools = glob("$progroot/Debug*/cdb.exe");
+@try_dbgs = ("$debugtools[0]", "$sysroot/system32/ntsd.exe");
+
+# parameters
+$usage = "Usage: $0 [-raw] [-i] [-f <addrfile>] [-d <debuggerpath>]
+<DRdllpath> [<addr1> ... <addrN>]\n";
+
+$debugger = "";
+$verbose = 0;
+$raw = 0;
+$addrfile = "";
+$DRdll = "";
+$havedll = 0;
+$numaddrs = 0;
+$interactive = 0;
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq '-f') {
+        die $usage if ($#ARGV <= 0);
+        shift;
+        $addrfile = $ARGV[0];
+        unless (-f "$addrfile") {
+            die "$err No file $addrfile\n";
+        }
+    } elsif ($ARGV[0] eq '-d') {
+        die $usage if ($#ARGV <= 0);
+        shift;
+        $debugger = $ARGV[0];
+        # checked below for existence
+    } elsif ($ARGV[0] eq '-raw') {
+        $raw = 1;
+    } elsif ($ARGV[0] eq '-i') {
+        $interactive = 1;
+    } elsif ($ARGV[0] =~ /^-/) {
+        die $usage;
+    } else {
+        if (!$havedll) {
+            $DRdll = $ARGV[0];
+            $havedll = 1;
+        } else {
+            # can't use both -f and cmdline params
+            die $usage if ($addrfile ne "");
+            $addrs[$numaddrs++] = $ARGV[0];
+        }
+    }
+    shift;
+}
+
+if ($numaddrs == 0) { # no cmdline addrs
+    if ($addrfile ne "") {
+        open(ADDRS, "< $addrfile") || die "Couldn't open $addrfile\n";
+    } else {
+        open(ADDRS, "< -") || die "Couldn't open stdin\n";
+    }
+    while (<ADDRS>) {
+        # remove \n, \r, spaces, etc.
+        s|\s*||g;
+        $addrs[$numaddrs++] = $_;
+    }
+    close(ADDRS);
+}
+
+die $usage if ($DRdll eq "");
+$DRdllwin32 = &unixpath2win32($DRdll);
+# try both -- different perls like different paths
+unless (-f "$DRdll" || -f "$DRdllwin32") {
+    die "Cannot find dll at $DRdll or $DRdllwin32\n";
+}
+
+# see if user-specified debugger exists
+if ($debugger ne "") {
+    $debugger = &win32path2unix($debugger);
+    print "Trying debugger at $debugger\n" if ($verbose);
+    if (! -f "$debugger") { # -x seems to require o+x so we don't check
+        die "Cannot find debugger at $debugger\n";
+    }
+} else {
+    for ($i = 0; $i <= $#try_dbgs; $i++) {
+        $debugger = &win32path2unix($try_dbgs[$i]);
+        print "Trying debugger at $debugger\n" if ($verbose);
+        last if (-f "$debugger");
+    }
+    die "Cannot find a debugger: @try_dbgs\n" if ($i > $#try_dbgs);
+}
+if ($debugger !~ /cdb/) {
+    $use_logfile = 1;
+}
+
+$DRtools = $ENV{'DYNAMORIO_TOOLS'};
+if ($DRtools eq "") {
+    # try same dir this script sits in
+    $DRtools = $0;
+    if ($DRtools =~ /[\\\/]/) {
+        $DRtools =~ s/^(.+)[\\\/][\w\.]+$/\1/;
+    } else {
+        $DRtools = "./";
+    }
+}
+$DRload = "$DRtools/DRload.exe";
+# need win32 path with double backslashes
+# e.g., c:\\\\derek\\\\dr\\\\tools\\\\DRload.exe
+$DRload = &unixpath2win32($DRload);
+unless (-f "$DRload") { # -x seems to require o+x so we don't check
+    die "Cannot find \$DYNAMORIO_TOOLS/DRload.exe at $DRload\n";
+}
+
+die $usage if ($numaddrs == 0);
+$queries = "";
+$marker = 'eeeeeeee';
+for ($i=0; $i<=$#addrs; $i++) {
+    # use a marker to indicate boundaries of output for each command
+    # (some have empty output)
+    $queries .= "? $marker; ln $addrs[$i]; ";
+}
+# put marker at the end too
+$queries .= "? $marker;";
+
+# FIXME: win2k ntsd does not load symbols by following loaded dll's path!
+# setting _NT_SYMBOL_PATH ENV here or exporting in shell doesn't work,
+# and -y not available on win2k ntsd, and .sympath takes all chars following
+# (even \r, \n, ", ;!), no aS available to break cmd -- giving up
+
+# enable line numbers (off by default in cdb)!
+$cmd = ".lines -e; l+l; $queries";
+
+if ($use_logfile) {
+    # must go through logfile as debugger won't write to accessible stdout
+    my $logdir = $ENV{'TMPDIR'} || $ENV{'TEMP'} || $ENV{'TMP'} || '/tmp';
+    die "Cannot find temp directory $logdir" if (! -d $logdir);
+    $logfile = "$logdir/address_query-$$.log";
+    $logfile = &unixpath2win32($logfile);
+    print "Log file is $logfile\n";
+    # start fresh -- no stale results
+    if (-f $logfile) {
+        system("rm $logfile");
+    }
+    $cmd = ".logopen $logfile; $cmd; .logclose;";
+}
+
+$cmd .= " q" unless ($interactive);
+
+# put quotes around debugger to handle spaces
+$cmdline = "\"$debugger\" -g -c \"$cmd\" $DRload -debugbreak $DRdllwin32";
+print $cmdline if ($verbose);
+if ($interactive) {
+    system("$cmdline") && die "Error: couldn't run $cmdline\n";
+} elsif ($raw) {
+    system("$cmdline") && die "Error: couldn't run $cmdline\n";
+    if ($use_logfile) {
+        system("cat $logfile");
+    }
+} else {
+    # prefix each ln output with the query (debugger doesn't re-print -c cmds)
+    $i = -1;
+    $output = 0;
+    if ($use_logfile) {
+        system("$cmdline") && die "Error: couldn't run $cmdline\n";
+        open(DBGOUT, "< $logfile") || die "Error: couldn't open $logfile\n";
+    } else {
+        open(DBGOUT, "$cmdline |") || die "Error: couldn't run $cmdline\n";
+    }
+    while (<DBGOUT>) {
+        # look for our marker
+        if (/= $marker/) {
+            $i++;
+            if ($i <= $#addrs) {
+                print "\n$addrs[$i]:\n";
+                $output = 1;
+            } else {
+                $output = 0; # at end
+            }
+        } elsif ($output) {
+            print $_;
+        }
+        # I used to match the addresses printed but these debuggers sometimes
+        # give weird output -- the right-hand is not always a larger address
+        # than the left-hand!!!  FIXME: what's going on?
+        # example:
+        # w/ export symbols:
+        #   0:001> ln 77f8d96b
+        #   (77f8d02e)   ntdll!RtlNtStatusToDosError+0x93d   |  (77f8db68)   ntdll!aulldiv
+        # w/ pdb:
+        #   0:001> ln 77f8d96b
+        #   (77f8d7c8)   ntdll!RtlpRunTable+0x1a3   |  (77f81ec1)   ntdll!RtlSetUserValueHeap
+    }
+    close(DBGOUT);
+}
+
+sub unixpath2win32($) {
+    my ($p) = @_;
+    # use cygpath if available, it will convert /home to c:\cygwin\home, etc.
+    $cp = `cygpath -wi \"$p\"`;
+    chomp $cp;
+    if ($cp eq "") {
+        # do it by hand
+        # add support for /x => x:
+        $p =~ s|^/([a-z])/|\1:/|;
+        # forward slash becomes backslash since going through another shell
+        $p =~ s|/|\\|g;
+    } else {
+        $p = $cp;
+    }
+    # make single backslashes doubles
+    $p =~ s|\\{1}|\\\\|g;
+    return $p;
+}
+
+sub win32path2unix($) {
+    my ($p) = @_;
+    # avoid issues with spaces by using 8.3 names
+    $cp = `cygpath -dmi \"$p\"`;
+    chomp $cp;
+    if ($cp eq "") {
+        # do it by hand
+        $p =~ s|\\|/|g;
+    } else {
+        $p = $cp;
+    }
+    return $p;
+}
diff --git a/tools/balloon.c b/tools/balloon.c
new file mode 100644
index 000000000..d1fefec15
--- /dev/null
+++ b/tools/balloon.c
@@ -0,0 +1,210 @@
+/* **********************************************************
+ * Copyright (c) 2004-2006 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include <assert.h>
+#include <stdio.h>
+
+#include "share.h"
+
+int usage(char *us)
+{
+    fprintf(stderr, "\
+Usage: %s\n\
+balloon -p <PID> [-v <MB or KB to reserve>] [-c <MB or KB to commit>] [-f] [-t]\n\
+        -f frees memory after allocation\n\
+        -t grabs memory top down\n\
+        -kb uses KB instead of MB as allocation unit\n\
+        -r repeat until failure\n\
+        -w wait\n\
+    Note that reserved and committed memory are in separate regions.\n\
+    Defaults are -p current -v 256MB -c 0MB\n", us);
+    return 0;
+// To check values see $ ./DRview.exe -p 416 -showmem | tail -1 | awk {'print "virtual peak " $9'}
+}
+
+enum {LAST_STATUS_VALUE_OFFSET = 0xbf4}; /* on Win2000+ case 6789 */
+int
+get_last_status()
+{
+    int laststatus;
+    /* get_own_teb()->LastStatusValue */
+    __asm {
+        mov  eax, fs:[LAST_STATUS_VALUE_OFFSET]
+        mov  laststatus, eax
+    }
+    return laststatus;
+}
+
+int
+main(int argc, char *argv[])
+{
+    int result;
+    int vsize=256;
+    int csize=0;
+    LPVOID *pv = 0;
+    LPVOID *pc = 0;
+
+    int argidx = 1;
+    int pid = 0;
+    HANDLE phandle;
+    int topdown = 0;
+    int protection = PAGE_NOACCESS;
+
+    int free_them = 0;          /* just probe the process, free them after you're done */
+    int allocation_unit = 1024 * 1024; /* default in MB */
+    int repeat = 0;
+    int fail = 0;
+    int wait = 0;
+
+    if (argc < 2) 
+        usage(argv[0]);
+
+    while (argidx < argc) {
+
+        if (!strcmp(argv[argidx], "-help")) {
+            usage(argv[0]);
+            exit(0);
+        }
+        else if (!strcmp(argv[argidx], "-p")) {
+            pid=atoi(argv[++argidx]);
+        }
+        else if (!strcmp(argv[argidx], "-c")) {
+            csize=atoi(argv[++argidx]);
+        }
+        else if (!strcmp(argv[argidx], "-v")) {
+            if (++argidx >= argc) {
+                usage(argv[0]);
+                exit(0);
+            }
+            vsize=atoi(argv[argidx]);
+        }
+        else if (!strcmp(argv[argidx], "-t")) {
+            topdown=MEM_TOP_DOWN;
+        }
+        else if (!strcmp(argv[argidx], "-f")) {
+            free_them=1;
+        }
+        else if (!strcmp(argv[argidx], "-kb")) {
+            allocation_unit=1024;
+        }
+        else if (!strcmp(argv[argidx], "-w")) {
+            wait = 1;
+        }
+        else if (!strcmp(argv[argidx], "-r")) {
+            /* if last undefinitely */
+            if (++argidx >= argc) {
+                repeat=1000000;
+            } else {
+                repeat=atoi(argv[argidx]);
+            }
+        }
+        else {
+            fprintf(stderr, "Unknown option: %s\n", argv[argidx]);
+            usage(argv[0]);
+            exit(0);
+        }
+        argidx++;
+    }
+
+    if (pid) {
+        acquire_privileges();
+        phandle = OpenProcess(PROCESS_ALL_ACCESS,
+                              FALSE, pid);
+        /* FIXME: we need PROCESS_VM_OPERATION access to the process */
+        /* should grab permissions */
+        assert(phandle);
+        release_privileges();
+    } else 
+        phandle = GetCurrentProcess();
+        
+    do {
+        if (vsize) {
+            int alloc = vsize * allocation_unit;
+            int flags = MEM_RESERVE | topdown;
+            pv = VirtualAllocEx(phandle, NULL, alloc, flags, protection);
+            printf("%s 0x%08x virtual bytes == %d%s flags=0x%08x prot=0x%08x\n"
+                   "  res = %08x %d\n", 
+                   pv ? "Just reserved" : "Failed to reserve",
+                   alloc, vsize, allocation_unit == 1024 ? "KB" : "MB",
+                   flags, protection, pv ? (int)pv : get_last_status(), 
+                   pc ? "success:" : "GLE", GetLastError());
+            if (pv == NULL)
+                fail = 1;
+        }
+
+        if (csize) {
+            int alloc = csize * allocation_unit;
+            int flags = MEM_RESERVE | MEM_COMMIT | topdown;
+            pc = VirtualAllocEx(phandle, NULL, alloc, flags, protection);
+            printf("%s 0x%08x bytes == %d%s flags=0x%08x prot=0x%08x\n"
+                   "  res = %08x, %s %x\n", 
+                   pc ? "Just committed" : "Failed to commit",
+                   alloc, csize, allocation_unit == 1024 ? "KB" : "MB",
+                   flags, protection, pc ? (int)pc : get_last_status(), 
+                   pc ? "success:" : "GLE", GetLastError());
+            if (pc == NULL)
+                fail = 1;
+        }
+        
+        fflush(stdout);
+        if (free_them) {
+            if (wait) {
+                printf("Press <enter> to free allocations...\n");
+                fflush(stdout);
+                getchar();
+            }
+            if (pc) {
+                result = VirtualFreeEx(phandle, pc, 0, MEM_RELEASE);
+                assert(result);
+            }
+            if (pv) {
+                result = VirtualFreeEx(phandle, pv, 0, MEM_RELEASE);
+                assert(result);
+            }
+
+            printf("Just freed those bytes\n");
+        }
+        fflush(stdout);
+        --repeat;
+        if (repeat > 0)
+            printf("Repeating until failure %d\n", repeat);
+    } while (repeat > 0 && !fail);
+
+    if (wait) {
+        printf("Press <enter> to exit...\n");
+        fflush(stdout);
+        getchar();
+    }
+
+    return 0;
+}
diff --git a/tools/bmcmp.pl b/tools/bmcmp.pl
new file mode 100755
index 000000000..79222339e
--- /dev/null
+++ b/tools/bmcmp.pl
@@ -0,0 +1,174 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2003-2004 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### bmcmp.pl
+### author: Derek Bruening   April 2003
+###
+### compares benchmark results produced with bmtable.pl
+
+$usage = "Usage: $0 <table1> <table2>\n";
+
+$ignore_errors = 0;
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq "-k") {
+        $ignore_errors = 1;
+        shift;
+    } else {
+        last;
+    }
+}
+if ($#ARGV != 1) {
+    print $usage;
+    exit;
+}
+$file1 = $ARGV[0];
+$file2 = $ARGV[1];
+$PWD = $ENV{'PWD'};
+open(FILE1, "< $file1") || die "Error: Couldn't open $file1 for input\n";
+open(FILE2, "< $file2") || die "Error: Couldn't open $file2 for input\n";
+print "# in dir $PWD:\n#\t$file1 / $file2\n";
+print "-------------------------------------------------\n";
+$num_bmarks = 0;
+while (<FILE1>) {
+    if ($_ =~ /^\s*(\w+)\s+\d+\/\d+\s+(\S+)\s+(\S+)\s+([\d\.]+)\s+(\d+)\s+(\d+)/) {
+	$bmark = $1;
+	$status1{$bmark} = $2;
+	$cpu1{$bmark} = $3;
+	$time1{$bmark} = $4;
+	$rss1{$bmark} = $5;
+	$vsz1{$bmark} = $6;
+	$name_bmarks[$num_bmarks] = $bmark;
+	$num_bmarks++;
+    }
+}
+close(FILE1);
+while (<FILE2>) {
+    if ($_ =~ /^\s*(\w+)\s+\d+\/\d+\s+(\S+)\s+(\S+)\s+([\d\.]+)\s+(\d+)\s+(\d+)/) {
+	$bmark = $1;
+	$status2{$bmark} = $2;
+	$cpu2{$bmark} = $3;
+	$time2{$bmark} = $4;
+	$rss2{$bmark} = $5;
+	$vsz2{$bmark} = $6;
+	$name_bmarks[$num_bmarks] = $bmark;
+	$num_bmarks++;
+    }
+}
+close(FILE2);
+
+printf("%14s   %-6s      %-7s  %-7s  %-7s\n",
+       "Benchmark", "Status", "Time", "RSS", "VSz");
+
+@sorted = sort(@name_bmarks);
+$last = "";
+$harsum{"time"} = 0;
+$harnum{"time"} = 0;
+$harsum{"rss"} = 0;
+$harnum{"rss"} = 0;
+$harsum{"vsz"} = 0;
+$harnum{"vsz"} = 0;
+foreach $s (@sorted) {
+    if ($s eq $last) {
+	next;
+    }
+    $last = $s;
+    
+    printf "%14s  %4s/%-4s   ", $s, &stat($status1{$s}), &stat($status2{$s});
+
+    # both must be ok
+    # we only look at status -- we assume table filtered out bad %CPU
+    if ($ignore_errors || $status1{$s} =~ /ok/ && $status2{$s} =~ /ok/) {
+	$bad = 0;
+    } else {
+	$bad = 1;
+    }
+
+    if (!$bad && defined($time1{$s}) && defined($time2{$s}) &&
+	$time1{$s} > 0 && $time2{$s} > 0) {
+	$ratio = $time1{$s} / $time2{$s};
+	$harsum{"time"} += 1/$ratio;
+	$harnum{"time"}++;
+	printf "%4.3f    ", $ratio;
+    } else {
+	printf "%5s    ", "-----";
+    }
+    if (!$bad && defined($rss1{$s}) && defined($rss2{$s}) &&
+	$rss1{$s} > 0 && $rss2{$s} > 0) {
+	$ratio = $rss1{$s} / $rss2{$s};
+	$harsum{"rss"} += 1/$ratio;
+	$harnum{"rss"}++;
+	printf "%4.3f    ", $ratio;
+    } else {
+	printf "%5s    ", "-----";
+    }
+    if (!$bad && defined($vsz1{$s}) && defined($vsz2{$s}) &&
+	$vsz1{$s} > 0 && $vsz2{$s} > 0) {
+	$ratio = $vsz1{$s} / $vsz2{$s};
+	$harsum{"vsz"} += 1/$ratio;
+	$harnum{"vsz"}++;
+	printf "%4.3f    ", $ratio;
+    } else {
+	printf "%5s    ", "-----";
+    }
+    print "\n";
+}
+
+print "---------------------------------------------------\n";
+printf "%14s  %4s %-4s   ", "harmonic mean", "", "";
+if ($harsum{"time"} > 0) {
+    $hmean = $harnum{"time"} / $harsum{"time"};
+    printf "%4.3f    ", $hmean;
+} else {
+    printf "%5s    ", "-----";
+}
+if ($harsum{"rss"} > 0) {
+    $hmean = $harnum{"rss"} / $harsum{"rss"};
+    printf "%4.3f    ", $hmean;
+} else {
+    printf "%5s    ", "-----";
+}
+if ($harsum{"vsz"} > 0) {
+    $hmean = $harnum{"vsz"} / $harsum{"vsz"};
+    printf "%4.3f    ", $hmean;
+} else {
+    printf "%5s    ", "-----";
+}
+print "\n";
+
+sub stat($) {
+    my ($status) = @_;
+    if ($status eq "") {
+	return "--";
+    } else {
+	return $status;
+    }
+}
diff --git a/tools/bmmemabs.pl b/tools/bmmemabs.pl
new file mode 100755
index 000000000..df0261b52
--- /dev/null
+++ b/tools/bmmemabs.pl
@@ -0,0 +1,174 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2004-2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### bmmemabs.pl
+### author: Derek Bruening   April 2003
+###
+### compares benchmark results produced with bmtable.pl
+### computes absolute memory usage difference between two runs
+
+$usage = "Usage: $0 [-k] <table1> <table2>\n";
+
+if ($#ARGV < 0) {
+    print $usage;
+    exit;
+}
+if ($ARGV[0] eq "-k") {
+    $ignore_errors = 1;
+    shift;
+}
+if ($#ARGV != 1) {
+    print $usage;
+    exit;
+}
+$file1 = $ARGV[0];
+$file2 = $ARGV[1];
+$PWD = $ENV{'PWD'};
+open(FILE1, "< $file1") || die "Error: Couldn't open $file1 for input\n";
+open(FILE2, "< $file2") || die "Error: Couldn't open $file2 for input\n";
+print "# in dir $PWD:\n#\t$file1 - $file2\n";
+print "-------------------------------------------------\n";
+$num_bmarks = 0;
+while (<FILE1>) {
+    if ($_ =~ /^\s*(\w+)\s+\d+\/\d+\s+(\S+)\s+(\S+)\s+([\d\.]+)\s+(\d+)\s+(\d+)/) {
+	$bmark = $1;
+	$status1{$bmark} = $2;
+	$cpu1{$bmark} = $3;
+	$time1{$bmark} = $4;
+	$rss1{$bmark} = $5;
+	$vsz1{$bmark} = $6;
+	$name_bmarks[$num_bmarks] = $bmark;
+	$num_bmarks++;
+    }
+}
+close(FILE1);
+while (<FILE2>) {
+    if ($_ =~ /^\s*(\w+)\s+\d+\/\d+\s+(\S+)\s+(\S+)\s+([\d\.]+)\s+(\d+)\s+(\d+)/) {
+	$bmark = $1;
+	$status2{$bmark} = $2;
+	$cpu2{$bmark} = $3;
+	$time2{$bmark} = $4;
+	$rss2{$bmark} = $5;
+	$vsz2{$bmark} = $6;
+	$name_bmarks[$num_bmarks] = $bmark;
+	$num_bmarks++;
+    }
+}
+close(FILE2);
+
+printf("%10s   %-6s  %-9s  %-7s  %-7s\n",
+       "Benchmark", "Status", "Time(min)", "RSS(KB)", "VSz(KB)");
+
+@sorted = sort(@name_bmarks);
+$last = "";
+$harsum{"time"} = 0;
+$harnum{"time"} = 0;
+$harsum{"rss"} = 0;
+$harnum{"rss"} = 0;
+$harsum{"vsz"} = 0;
+$harnum{"vsz"} = 0;
+foreach $s (@sorted) {
+    if ($s eq $last) {
+	next;
+    }
+    $last = $s;
+    
+    printf "%10s  %4s/%-4s   ", $s, &stat($status1{$s}), &stat($status2{$s});
+
+    # both must be ok
+    # we only look at status -- we assume table filtered out bad %CPU
+    if ($ignore_errors || $status1{$s} =~ /ok/ && $status2{$s} =~ /ok/) {
+	$bad = 0;
+    } else {
+	$bad = 1;
+    }
+
+    if (!$bad && defined($time1{$s}) && defined($time2{$s}) &&
+	$time1{$s} > 0 && $time2{$s} > 0) {
+	$ratio = $time1{$s} / $time2{$s};
+	$harsum{"time"} += 1/$ratio;
+	$harnum{"time"}++;
+	printf "%4.3f    ", $ratio;
+    } else {
+	printf "%5s    ", "-----";
+    }
+    if (!$bad && defined($rss1{$s}) && defined($rss2{$s}) &&
+	$rss1{$s} > 0 && $rss2{$s} > 0) {
+	$diff = $rss1{$s} - $rss2{$s};
+	$harsum{"rss"} += $diff;
+	$harnum{"rss"}++;
+	printf "%6d    ", $diff;
+    } else {
+	printf "%5s    ", "-----";
+    }
+    if (!$bad && defined($vsz1{$s}) && defined($vsz2{$s}) &&
+	$vsz1{$s} > 0 && $vsz2{$s} > 0) {
+	$diff = $vsz1{$s} - $vsz2{$s};
+	$harsum{"vsz"} += $diff;
+	$harnum{"vsz"}++;
+	printf "%6d    ", $diff;
+    } else {
+	printf "%5s    ", "-----";
+    }
+    print "\n";
+}
+
+print "-------------------------------------------------\n";
+printf "%12s     ", "harmonic/arith mean";
+if ($harsum{"time"} > 0) {
+    $hmean = $harnum{"time"} / $harsum{"time"};
+    printf "%4.3f    ", $hmean;
+} else {
+    printf "%5s    ", "-----";
+}
+if ($harsum{"rss"} > 0) {
+    $hmean = $harsum{"rss"} / $harnum{"rss"};
+    printf "%6d    ", $hmean;
+} else {
+    printf "%5s    ", "-----";
+}
+if ($harsum{"vsz"} > 0) {
+    $hmean = $harsum{"vsz"} / $harnum{"vsz"};
+    printf "%6d    ", $hmean;
+} else {
+    printf "%5s    ", "-----";
+}
+print "\n";
+
+sub stat($) {
+    my ($status) = @_;
+    if ($status eq "") {
+	return "--";
+    } else {
+	return $status;
+    }
+}
diff --git a/tools/bmmemcmp.pl b/tools/bmmemcmp.pl
new file mode 100755
index 000000000..8a66f41d9
--- /dev/null
+++ b/tools/bmmemcmp.pl
@@ -0,0 +1,198 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2004-2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### bmmemcmp.pl
+### author: Derek Bruening   April 2003
+###
+### compares benchmark memory usage results produced with bmtable.pl versus
+### a third native run -- calculates the reduction in additional memory
+### taken vs the native run
+
+$usage = "Usage: $0 [-k] <table1> <table2> <nativetable>\n";
+
+if ($#ARGV < 0) {
+    print $usage;
+    exit;
+}
+if ($ARGV[0] eq "-k") {
+    $ignore_errors = 1;
+    shift;
+}
+if ($#ARGV != 2) {
+    print $usage;
+    exit;
+}
+$file1 = $ARGV[0];
+$file2 = $ARGV[1];
+$file3 = $ARGV[2];
+$PWD = $ENV{'PWD'};
+open(FILE1, "< $file1") || die "Error: Couldn't open $file1 for input\n";
+open(FILE2, "< $file2") || die "Error: Couldn't open $file2 for input\n";
+open(FILE3, "< $file3") || die "Error: Couldn't open $file3 for input\n";
+print "# in dir $PWD:\n#\t$file1 / $file2 wrt native $file3\n";
+print "-------------------------------------------------\n";
+$num_bmarks = 0;
+while (<FILE1>) {
+    if ($_ =~ /^\s*(\w+)\s+\d+\/\d+\s+(\S+)\s+(\S+)\s+([\d\.]+)\s+(\d+)\s+(\d+)/) {
+	$bmark = $1;
+	$status1{$bmark} = $2;
+	$cpu1{$bmark} = $3;
+	$time1{$bmark} = $4;
+	$rss1{$bmark} = $5;
+	$vsz1{$bmark} = $6;
+	$name_bmarks[$num_bmarks] = $bmark;
+	$num_bmarks++;
+    }
+}
+close(FILE1);
+while (<FILE2>) {
+    if ($_ =~ /^\s*(\w+)\s+\d+\/\d+\s+(\S+)\s+(\S+)\s+([\d\.]+)\s+(\d+)\s+(\d+)/) {
+	$bmark = $1;
+	$status2{$bmark} = $2;
+	$cpu2{$bmark} = $3;
+	$time2{$bmark} = $4;
+	$rss2{$bmark} = $5;
+	$vsz2{$bmark} = $6;
+	$name_bmarks[$num_bmarks] = $bmark;
+	$num_bmarks++;
+    }
+}
+close(FILE2);
+while (<FILE3>) {
+    if ($_ =~ /^\s*(\w+)\s+\d+\/\d+\s+(\S+)\s+(\S+)\s+([\d\.]+)\s+(\d+)\s+(\d+)/) {
+	$bmark = $1;
+	$statusN{$bmark} = $2;
+	$cpuN{$bmark} = $3;
+	$timeN{$bmark} = $4;
+	$rssN{$bmark} = $5;
+	$vszN{$bmark} = $6;
+	$name_bmarks[$num_bmarks] = $bmark;
+	$num_bmarks++;
+    }
+}
+close(FILE3);
+
+printf("%10s   %-6s  %-9s  %-7s  %-7s\n",
+       "Benchmark", "Status", "Time(min)", "RSS(KB)", "VSz(KB)");
+
+@sorted = sort(@name_bmarks);
+$last = "";
+$harsum{"time"} = 0;
+$harnum{"time"} = 0;
+$harsum{"rss"} = 0;
+$harnum{"rss"} = 0;
+$harsum{"vsz"} = 0;
+$harnum{"vsz"} = 0;
+foreach $s (@sorted) {
+    if ($s eq $last) {
+	next;
+    }
+    $last = $s;
+    
+    printf "%10s  %4s/%-4s   ", $s, &stat($status1{$s}), &stat($status2{$s});
+
+    # both must be ok
+    # we only look at status -- we assume table filtered out bad %CPU
+    if ($ignore_errors || ($status1{$s} =~ /ok/ && $status2{$s} =~ /ok/)) {
+	$bad = 0;
+    } else {
+	$bad = 1;
+    }
+
+    if (!$bad && defined($time1{$s}) && defined($time2{$s}) &&
+	$time1{$s} > 0 && $time2{$s} > 0) {
+	$ratio = $time1{$s} / $time2{$s};
+	$harsum{"time"} += 1/$ratio;
+	$harnum{"time"}++;
+	printf "%4.3f    ", $ratio;
+    } else {
+	printf "%5s    ", "-----";
+    }
+    if (!$bad && defined($rss1{$s}) && defined($rss2{$s}) &&
+	defined($rssN{$s}) && $rss1{$s} > 0 && $rss2{$s} > 0 &&
+	$rssN{$s} < $rss1{$s} && $rssN{$s} < $rss2{$s}) {
+	# compute rss we added to native
+	$added1 = $rss1{$s} - $rssN{$s};
+	$added2 = $rss2{$s} - $rssN{$s};
+	$ratio = $added1 / $added2;
+	$harsum{"rss"} += 1/$ratio;
+	$harnum{"rss"}++;
+	printf "%4.3f    ", $ratio;
+    } else {
+	printf "%5s    ", "-----";
+    }
+    if (!$bad && defined($vsz1{$s}) && defined($vsz2{$s}) &&
+	defined($vszN{$s}) && $vsz1{$s} > 0 && $vsz2{$s} > 0 &&
+	$vszN{$s} < $vsz1{$s} && $vszN{$s} < $vsz2{$s}) {
+	# compute vsz we added to native
+	$added1 = $vsz1{$s} - $vszN{$s};
+	$added2 = $vsz2{$s} - $vszN{$s};
+	$ratio = $added1 / $added2;
+	$harsum{"vsz"} += 1/$ratio;
+	$harnum{"vsz"}++;
+	printf "%4.3f    ", $ratio;
+    } else {
+	printf "%5s    ", "-----";
+    }
+    print "\n";
+}
+
+print "-------------------------------------------------\n";
+printf "%12s           ", "harmonic mean";
+if ($harsum{"time"} > 0) {
+    $hmean = $harnum{"time"} / $harsum{"time"};
+    printf "%4.3f    ", $hmean;
+} else {
+    printf "%5s    ", "-----";
+}
+if ($harsum{"rss"} > 0) {
+    $hmean = $harnum{"rss"} / $harsum{"rss"};
+    printf "%4.3f    ", $hmean;
+} else {
+    printf "%5s    ", "-----";
+}
+if ($harsum{"vsz"} > 0) {
+    $hmean = $harnum{"vsz"} / $harsum{"vsz"};
+    printf "%4.3f    ", $hmean;
+} else {
+    printf "%5s    ", "-----";
+}
+print "\n";
+
+sub stat($) {
+    my ($status) = @_;
+    if ($status eq "") {
+	return "--";
+    } else {
+	return $status;
+    }
+}
diff --git a/tools/bmtable.pl b/tools/bmtable.pl
new file mode 100755
index 000000000..23037c1e5
--- /dev/null
+++ b/tools/bmtable.pl
@@ -0,0 +1,294 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2003-2004 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### bmtable.pl
+### author: Derek Bruening   April 2001
+###
+### Produces a table summarizing times from one or more benchmark runs
+### For times, the default is to use the median, 
+###   can also calculate min w/ -min parameter
+### For memory, the default is to use the max,
+###   can also calculate average or use first run w/ -memave or -mem1
+
+$usage = "Usage: $0 [-v] [-k] [-c] [-min] [-mem1] [-memave] <logfile> [logfile2 ...]\n";
+
+# constants
+$debug = 0;
+
+# parameters
+$verbose = 0;
+$ignore_errors = 0;
+$ignore_cpu_reqt = 0;
+$min = 0;
+$files = 0;
+
+# memory: use max of all runs, or:
+$mem_first = 0;
+$mem_ave = 0;
+
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq "-v") {
+        $verbose = 1;
+    } elsif ($ARGV[0] eq "-k") {
+        $ignore_errors = 1;
+    } elsif ($ARGV[0] eq "-c") {
+        $ignore_cpu_reqt = 1;
+    } elsif ($ARGV[0] eq "-mem1") {
+        $mem_first = 1;
+    } elsif ($ARGV[0] eq "-memave") {
+        $mem_ave = 1;
+    } elsif ($ARGV[0] eq "-min") {
+        $min = 1;
+    } else {
+        $logfile[$files] = $ARGV[0];
+        $files++;
+    }
+    shift;
+}
+if (!defined($logfile[0])) {
+    print $usage;
+    exit;
+}
+
+if ($mem_ave) {
+    print "RSS and VSz are average of runs for those with multiple runs\n";
+} elsif ($mem_first) {
+    print "RSS and VSz are from first run for those with multiple runs\n";
+} else {
+    print "RSS and VSz are max of runs for those with multiple runs\n";
+}
+
+$cwd = `pwd`;
+chop $cwd;
+foreach $file (@logfile) {
+    if (!($file =~ m+^/+ || $file =~ m+^\\+)) {
+        # make it absolute
+        $file = "$cwd/$file";
+    }
+    print "Processing $file\n";
+    &process_file($file);
+}
+
+if ($verbose) {
+    print "Key: - = invalid, + = valid, ** = " . ($min ? "min" : "median") . "\n";
+}
+print "----------------------------------------------------------------------\n";
+    
+# only single type of run!
+# (this script adapted from wbmtable.pl, which has multiple types)
+$run{'all'} = 'all';
+foreach $r (sort (keys %run)) {
+    if (defined($run{$r})) {
+        &print_cur_stats($r);
+    }
+}
+
+sub process_file($) {
+    my ($logfile) = @_;
+    open(LOG, "< $logfile") || die "Error opening $logfile\n";
+    while (<LOG>) {
+        chop;
+        # support old logfile formats that don't print name w/ Verify
+        if ($_ =~ m|^%%%% /.+/([^/]+)| ||
+            $_ =~ /Verify for (\S+):/ ||
+            # support for specjvm
+            $_ =~ /======= (\S+) Starting/) {   
+            print "Name is $_\n" if ($debug);
+            $bmark = $1;
+            $bmark =~ s/\.exe$//;
+            if (!defined($name{$bmark})) {
+                $name{$bmark} = $bmark;
+                $iters{$bmark} = 0;
+                print "New bmark $bmark\n" if ($debug);
+            }
+            # have to allow two names in a row if 1st is runsuite comment,
+            # and second is the Verify for the 1st iter
+            if ($_ =~ m|^%%%% /.+/([^/]+)|) {
+                $runsuite_name = 1;
+            }
+            $iter = $iters{$bmark};
+            print "  New iter $iter for $bmark\n" if ($debug);
+            # two differently-triggered names in a row: count as one iter
+            if ($runsuite_name && $_ =~ /Verify for (\S+):/) {
+                $iter--;
+                print "\tDouble-dip: iter back to $iter for $bmark\n" if ($debug);
+                $runsuite_name = 0;
+            }
+            if (!defined($time{$bmark,$iter})) {
+                $iters{$bmark}++;
+                $time{$bmark,$iter} = 0;
+                $rss{$bmark,$iter} = 0;
+                $vsz{$bmark,$iter} = 0;
+                $cpu{$bmark,$iter} = 0;
+                $status{$bmark,$iter} = " ?  ";
+            }
+        }
+        if ($_ =~ /make:/ || $_ =~ /[Ee]rror:/ || $_ =~ /FAILED/) {
+            $status{$bmark,$iter} = "FAIL";
+        } elsif ($_ =~ /Verify.* correct/ ||
+                 # support for specjvm
+                 $_ =~ /======= (\S+) Finished/) {
+            $status{$bmark,$iter} = " ok ";
+        } elsif ($_ =~ /Elapsed: (\d+):(\d+):([\d\.]+)/) {
+            # texec format
+            $elapsed = $1*60 + $2 + $3/60.; # minutes
+            $time{$bmark,$iter} += $elapsed;
+        } elsif ($_ =~ /real\t([0-9]+)m([0-9\.]+)s/) {
+            # /usr/bin/time simple format
+            # FIXME: what do hours look like?
+            $elapsed = $1 + $2/60.; # minutes
+            $time{$bmark,$iter} += $elapsed;
+            printf "\t$bmark, $iter: elapsed %5.2f @$cpu%%cpu\n", $elapsed
+                if ($debug);
+        } elsif ($_ =~ /(\d+):(\d+):(\d+)elapsed (\d+)%CPU/) {
+            # /usr/bin/time and runstats compressed format
+            $elapsed = $1*60 + $2 + $3/60.; # minutes
+            $time{$bmark,$iter} += $elapsed;
+            $cpu{$bmark,$iter} = $4 if ($4 > 0);
+            printf "\t$bmark, $iter: elapsed %5.2f @$cpu%%cpu\n", $elapsed
+                if ($debug);
+        } elsif ($_ =~ /(\d+):(\d+).(\d+)elapsed (\d+)%CPU/) {
+            # /usr/bin/time and runstats compressed format, with hours
+            $elapsed = $1 + $2/60. + $3/6000.; # minutes
+            $time{$bmark,$iter} += $elapsed;
+            $try_cpu = $4;
+            $cpu{$bmark,$iter} = $4 if ($4 > 0);
+            printf "\t$bmark, $iter: elapsed %5.2f @$cpu%%cpu\n", $elapsed
+                if ($debug);
+        } elsif ($_ =~ /(\d+) tot, (\d+) RSS/) {
+            # runstats memory usage
+            # use max of all runs by default
+            if ($mem_first) {
+                $vsz{$bmark,$iter} = $1 if ($vsz{$bmark,$iter} == 0);
+                $rss{$bmark,$iter} = $2 if ($rss{$bmark,$iter} == 0);
+            } elsif ($mem_ave) {
+                $vsz{$bmark,$iter} = ($1 + $vsz{$bmark,$iter}*$memruns{$bmark,$iter}) /
+                    ($memruns{$bmark,$iter}+1);
+                $rss{$bmark,$iter} = ($1 + $rss{$bmark,$iter}*$memruns{$bmark,$iter}) /
+                    ($memruns{$bmark,$iter}+1);
+                $memruns{$bmark,$iter}++;
+            } else {
+                $vsz{$bmark,$iter} = $1 if ($1 > $vsz{$bmark,$iter});
+                $rss{$bmark,$iter} = $2 if ($2 > $rss{$bmark,$iter});
+            }
+        }
+    }
+    close(LOG);
+}
+
+sub print_cur_stats($) {
+# run name is holdover from wbmtable.pl, so disabling this
+#    my ($runname) = @_;
+#    print "\nRun $runname\n";
+    printf("%14s      #    %-6s  %-4s  %-9s   %-7s   %-7s\n",
+           "Benchmark", "Status", "%CPU", "Time(min)", "RSS(KB)", "VSz(KB)");
+    $format       = "  %2d/%-2d   %4s    %4s   %6.2f   %7d   %7d\n";
+    $empty_format = "  %2d/%-2d   %4s    %4s   %6s   %7s   %7s\n";
+    foreach $bm (sort (keys %name)) {
+        # ignore FORTRAN90
+        next if ($bm =~ /facerec/ ||
+                 $bm =~ /fma3d/ ||
+                 $bm =~ /galgel/ ||
+                 $bm =~ /lucas/);
+        # find median time, assume memory doesn't change much, just
+        # use rss and vsz from median time run
+        $num_ok = 0;
+        $#ints = -1; # clear from last time
+        for ($i=0; $i<$iters{$bm}; $i++) {
+            if ($ignore_errors || &valid($status{$bm,$i}, $cpu{$bm,$i})) {
+                $ints[$num_ok] = $i;
+                $num_ok++;
+            }
+        }
+        if ($num_ok == 0) {
+            $median = -1;
+        } else {
+            @sorted = sort({$time{$bm,$a} <=> $time{$bm,$b}} @ints);
+            if ($min) {
+                $median = $sorted[0];
+            } else {
+                $median = $sorted[$num_ok/2]; # if even number, take smaller!
+            }
+        }
+        if ($verbose) {
+            for ($i=0; $i<$iters{$bm}; $i++) {
+                if ($i == $median) {
+                    $str = "%14s**";
+                } elsif (&valid($status{$bm,$i}, $cpu{$bm,$i})) {
+                    $str = "%14s+ ";
+                } else {
+                    $str = "%14s- ";
+                }
+                printf($str.$format,
+                       $bm, $i+1, $iters{$bm},
+                       $status{$bm,$i}, &cpustr($cpu{$bm,$i}),
+                       $time{$bm,$i}, $rss{$bm,$i}, $vsz{$bm,$i});
+            }
+        } else {
+            if ($median == -1) {
+                printf("%14s  ".$empty_format,
+                       $bm, 0, $iters{$bm}, $status{$bm,0}, " -- ",
+                       "----", "----", "----");
+            } else {
+                if ($num_ok == $iters{$bm}) {
+                    $stat = $status{$bm,$median};
+                } else {
+                    $stat = "<ok>";
+                }
+                printf("%14s  ".$format,
+                       $bm, $num_ok, $iters{$bm},
+                       $stat, &cpustr($cpu{$bm,$median}),
+                       $time{$bm,$median}, $rss{$bm,$median},
+                       $vsz{$bm,$median});
+            }
+        }
+    }
+}
+
+
+sub cpustr($) {
+    my ($cpu) = @_;
+    if ($cpu == 0) {
+        return " -- ";
+    } elsif ($cpu < 99) {
+        return sprintf "*%2d*", $cpu;
+    } else {
+        return sprintf "%3d ", $cpu;
+    }
+}
+
+sub valid($, $) {
+    my ($status, $cpu) = @_;
+    return ($status eq " ok " &&
+            # passed as param -> defined to 0, so check for 0
+            ($ignore_cpu_reqt || !defined($cpu) || $cpu == 0 || $cpu >= 99));
+}
diff --git a/tools/bugcrash.pl b/tools/bugcrash.pl
new file mode 100755
index 000000000..520661074
--- /dev/null
+++ b/tools/bugcrash.pl
@@ -0,0 +1,339 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2007 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+## 
+## FILE: bugcrash.pl:
+##   
+## Author: Sam Larsen
+## Date: Thu Mar 22 19:07:51 2007
+##
+## Reads a forensics file and:
+## - Launches windbg with DRload using the correct base address
+## - Fills in the DR stack and dcontext dumps from the forensics
+## - Locates the CONTEXT in the stack dump and sets it
+##
+## Preferred usage is just to supply a forensics file, and the script
+## will try to find windbg and the proper dynamorio.dll from the
+## nightlies.  However, you can explicitly specify these locations
+## with the options below.
+##
+
+$usage = "Usage: $0 [-v] [-g <debuggerpath>] [-b <build_dir_base>] [-d DRdllpath] <forensic file>\n";
+
+##
+## Get options
+##
+use Getopt::Long;
+@optl = ("v", "d=s", "b=s", "p=s", "g=s");
+if (!(GetOptions @optl) || $#ARGV != 0) {
+    die $usage;
+}
+
+$verbose = $opt_v;
+$build_dir = $opt_b ? $opt_b : "";
+$forensics = $ARGV[0];
+
+
+##
+## Locate debugger executable
+##
+if (!$opt_g) {
+    # try to find installation of debugging tools for windows
+    $progroot = &win32path2unix($ENV{'PROGRAMFILES'});
+    @debugtools = glob("$progroot/Debug*/windbg.exe");
+    $debugger = $debugtools[0];
+} else {
+    if (-d $opt_g) {
+        $debugger = "$opt_g/windbg.exe";
+    } else {
+        $debugger = $opt_g;
+    }
+}
+
+# -x seems to require o+x so we don't check
+die "ERROR: can't find debugger $debugger\n" if (!-f "$debugger");
+printf "Using debugger at $debugger\n" if ($verbose);
+
+
+##
+## Locate DRload
+##
+$DRtools = $ENV{'DYNAMORIO_TOOLS'};
+if ($DRtools eq "") {
+    # try same dir this script sits in
+    $DRtools = $0;
+    if ($DRtools =~ /[\\\/]/) {
+        $DRtools =~ s/^(.+)[\\\/][\w\.]+$/\1/;
+    } else {
+        $DRtools = "./";
+    }
+}
+$DRload = "$DRtools/DRload.exe";
+# need win32 path with double backslashes
+# e.g., c:\\\\derek\\\\dr\\\\tools\\\\DRload.exe
+$DRload = &unixpath2win32($DRload);
+die "ERROR: can't find DRload.exe" if (!-f "$DRload");
+
+
+##
+## Create a script file
+##
+$tmpdir = $ENV{'TMPDIR'} || $ENV{'TEMP'} || $ENV{'TMP'} || '/tmp';
+die "Cannot find temp directory $tmpdir" if (! -d $tmpdir);
+$script = "$tmpdir/bugcrash-$$.script";
+$winscript = &unixpath2win32($script);
+printf "script file is $script\n" if ($verbose);
+open (OUT, "> $script") || die "ERROR: couldn't open $script";
+
+
+##
+## Parse forensics file
+##
+open (IN, "$forensics") ||
+    die "ERROR: can't open forensics file $forensics\n";
+
+$build_num = "";
+$dr_base = "";
+$lib = "";
+
+$context = "";
+$esp = "";
+
+while (<IN>) {
+    if (!$opt_d && /^Generated by SecureCore [^,]+,\s*(.+)/) {
+        my $build_string = $1;
+        if ($build_string =~ /build ([0-9]+)/) {
+            $build_num = $1;
+        } else {
+            die "ERROR parsing dynamorio build number\n";
+        }
+    }
+    elsif (!$opt_d && /^Unrecoverable error at PC 0x([0-9A-Za-z]+)/) {
+        my $addr = $1;
+        if ($addr =~ /^15/) {
+            $lib = "debug";
+        } elsif ($addr =~ /^71/) {
+            $lib = "release";
+        } else {
+            die "ERROR: can't guess build from PC 0x$addr\n";
+        }
+    }
+    elsif ($_ =~ /^dynamorio\.dll=(0x[0-9a-f]{8})/) {
+        die "ERROR locating dynamorio.dll base\n" if ($dr_base ne "");
+        $dr_base = $1;
+    }
+    elsif ($_ =~ /ebp=0x[0-9a-f]{8} esp=(0x[0-9a-f]{8})/) {
+        die "ERROR locating esp\n" if ($esp ne "");
+        $esp = $1;
+    }
+    elsif ($_ =~ /^DR Stack: 0x([0-9a-f]{8})/) {
+        my $addr = hex($1);
+        my $base = $addr;
+
+        printf "DR stack is at 0x%x\n", $base;
+        printf OUT ".dvalloc /b 0x%x 0x1000\n", $base;
+
+        while (<IN>) {
+            my @bits = split(' ', $_);
+            if ($bits[0] =~ /^([0-9a-f]{8})$/) {
+                die "ERROR reading DR stack\n", if (hex($bits[0]) != $addr);
+                for (my $i=1; $i <= $#bits; $i++) {
+                    die "ERROR reading DR stack\n" if ($bits[$i] !~ /^[0-9a-f]{8}$/);
+                    printf OUT "ed 0x%08x 0x%08x\n", $addr, hex($bits[$i]);
+
+                    # look for the context
+                    if ($bits[$i] eq "0001003f") {
+                        die "ERROR: found > 1 instances of 0x0001003f on the stack. " .
+                            "Can't decipher the context\n" if ($context ne "");
+                        $context = $addr;
+                    }
+                    $addr += 4;
+                }
+            }
+            else {
+                my $size = $addr - $base;
+                die "ERROR: DR stack is $size bytes; expecting 0x1000\n"
+                    if ($size != 0x1000);
+                last;
+            }
+        }
+    }
+    elsif ($_ =~ /^current dcontext: 0x([0-9a-f]{8})/) {
+        my $addr = hex($1);
+        my $base = $addr;
+        
+        # The size of the dcontext could vary with the build, so
+        # read it first and then output the .dvalloc command
+        my $dcontext = "";
+        while (<IN>) {
+            my @bits = split(' ', $_);
+            if ($#bits == 0 && $bits[0] eq "<![CDATA[") {
+                next;
+            }
+            elsif ($bits[0] =~ /^([0-9a-f]{8})$/) {
+                die "ERROR reading dcontext\n", if (hex($bits[0]) != $addr);
+                for (my $i=1; $i <= $#bits; $i++) {
+                    die "ERROR reading dcontext\n" if ($bits[$i] !~ /^[0-9a-f]{8}$/);
+                    $dcontext .= sprintf("ed 0x%08x 0x%08x\n", $addr, hex($bits[$i]));
+                    $addr += 4;
+                }
+            }
+            else {
+                my $size = $addr - $base;
+                printf "dcontext is %d bytes at 0x%x\n", $size, $base;
+                printf OUT ".dvalloc /b 0x%x 0x%x\n", $base, $size;
+                printf OUT $dcontext;
+                last;
+            }
+        }
+    }
+    elsif ($_ =~ /^\<hotpatching\-information\>/) {
+        last;
+    }
+}
+die "ERROR: didn't find dynamorio base\n" if ($dr_base eq "");
+die "ERROR: didn't find context\n" if ($context eq "");
+die "ERROR: didn't find esp\n" if ($esp eq "");
+close (IN);
+
+
+##
+## Fill in context
+##
+printf OUT "r esp=$esp\n";
+printf OUT ".cxr 0x%08x\n", $context;
+
+
+##
+## Create a debugger log file
+##
+$logfile = "$tmpdir/bugcrash-$$.log";
+# start fresh -- no stale results
+system("rm -f $logfile") if (-e $logfile);
+printf "Log file is $logfile\n";
+$win32logfile = &unixpath2win32($logfile);
+printf OUT ".logopen $win32logfile\n";
+
+close (OUT);
+
+
+##
+## Locate dynamorio.dll
+##
+if ($opt_d) {
+    $DRdll = $opt_d;
+} else {
+    $DRdll = &findDRdll($build_dir, $build_num, $lib);
+}
+die "ERROR: can't find dynamorio.dll\n" if (!-e "$DRdll");
+$DRdllwin32 = &unixpath2win32($DRdll);
+
+
+##
+## Run the debugger
+##
+$cmd = "\$><$winscript";
+$cmdline = "\"$debugger\" -g -c \"$cmd\" $DRload -base $dr_base -debugbreak $DRdllwin32";
+printf "$cmdline\n" if ($verbose);
+system("$cmdline") && die "ERROR: couldn't run $cmdline\n";
+
+
+sub findDRdll($,$,$) {
+    my ($build_dir, $build_num, $lib) = @_;
+    # would be nice to use symbol server, but cannot locate
+    # based solely on build number w/o querying every dll, or having
+    # a symlink set up externally or something.
+    my @try = ( "//10.1.5.15/nightly",
+                "//10.1.5.15/nightly_archive" );
+    if ($build_dir ne "") {
+        unshift @try, $build_dir;
+    }
+    print "searching these bases: @try\n" if ($verbose);
+    # match ver_2_1, ver_2_5, etc., as well as the new dist
+    # we avoid {ver*,dist} since that gives two entries regardless of existence
+    my $ver = "[vd][ei][rs]*";
+    my $missed_at_all = 0;
+    foreach $base (@try) {
+        my $glob_path = "$base/$build_num/$ver/lib/$lib/dynamorio.dll";
+        my @wildcard = glob($glob_path);
+        if (@wildcard == 0) {
+            print STDERR "couldn't find $glob_path\n" if (!$quiet);
+            $missed_at_all = 1;
+        }
+        foreach $w (@wildcard) {
+            print "looking for $w\n" if ($verbose);
+            if (-f $w) {
+                # avoid found message if did it 1st try
+                print STDERR "found $w\n" if (!$quiet && $missed_at_all);
+                return $w;
+            }
+            print STDERR "couldn't find $w\n" if (!$quiet);
+            $missed_at_all = 1;
+        }
+    }
+    return "";
+}
+
+
+sub unixpath2win32($) {
+    my ($p) = @_;
+    # use cygpath if available, it will convert /home to c:\cygwin\home, etc.
+    $cp = `cygpath -wi \"$p\"`;
+    chomp $cp;
+    if ($cp eq "") {
+        # do it by hand
+        # add support for /x => x:
+        $p =~ s|^/([a-z])/|\1:/|;
+        # forward slash becomes backslash since going through another shell
+        $p =~ s|/|\\|g;
+    } else {
+        $p = $cp;
+    }
+    # make single backslashes doubles
+    $p =~ s|\\{1}|\\\\|g;
+    return $p;
+}
+
+
+sub win32path2unix($) {
+    my ($p) = @_;
+    # avoid issues with spaces by using 8.3 names
+    $cp = `cygpath -dmi \"$p\"`;
+    chomp $cp;
+    if ($cp eq "") {
+        # do it by hand
+        $p =~ s|\\|/|g;
+    } else {
+        $p = $cp;
+    }
+    return $p;
+}
diff --git a/tools/bugdis.pl b/tools/bugdis.pl
new file mode 100755
index 000000000..c986a2190
--- /dev/null
+++ b/tools/bugdis.pl
@@ -0,0 +1,302 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2006-2007 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### bugdis.pl
+### Derek Bruening, December 2006
+###
+### disassembles code from forensic files
+###
+### note about -i interactive use feature:
+###   cdb within rxvt has some problems: interactive typing loses the last
+###   char or more, so put spaces after everything.
+###   for interactive should launch ntsd, or run cdb from within a cygwin-in-cmd shell
+
+# default debugger paths to try
+# we first try cdb from Debugging Tools For Windows
+#   (so we can get stdout and don't need a temp logfile)
+# else we use the ntsd on everyone's machines and a logfile,
+#   which we clobber every time
+$sysroot = &win32path2unix($ENV{'SYSTEMROOT'});
+$progroot = &win32path2unix($ENV{'PROGRAMFILES'});
+# try to find installation of debugging tools for windows
+@debugtools = glob("$progroot/Debug*/cdb.exe");
+@try_dbgs = ("$debugtools[0]", "$sysroot/system32/ntsd.exe");
+$app = "cmd"; # we assume on path
+
+# parameters
+$usage = "Usage: $0 [-raw] [-v] [-i] [-u <lines>] [-b <backward bytes>]" .
+"[-d <debuggerpath>] <forensic files>\n";
+
+$file = "";
+$debugger = "";
+$verbose = 0;
+$raw = 0;
+$interactive = 0;
+# lines to eb 
+$u_lines = 3;
+$backward = 0;
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq '-d') {
+        die $usage if ($#ARGV <= 0);
+        shift;
+        $debugger = $ARGV[0];
+        # checked below for existence
+    } elsif ($ARGV[0] eq '-raw') {
+        $raw = 1;
+    } elsif ($ARGV[0] eq '-i') {
+        $interactive = 1;
+    } elsif ($ARGV[0] eq '-v') {
+        $verbose = 1;
+    } elsif ($ARGV[0] eq '-u') {
+        die $usage if ($#ARGV <= 0);
+        shift;
+        $u_lines = $ARGV[0];
+    } elsif ($ARGV[0] eq '-b') {
+        die $usage if ($#ARGV <= 0);
+        shift;
+        $backward = $ARGV[0];
+    } elsif ($ARGV[0] =~ /^-/) {
+        die $usage;
+    } else {
+        # rest are forensic files
+        last;
+    }
+    shift;
+}
+
+# see if user-specified debugger exists
+if ($debugger ne "") {
+    $debugger = &win32path2unix($debugger);
+    print "Trying debugger at $debugger\n" if ($verbose);
+    if (! -f "$debugger") { # -x seems to require o+x so we don't check
+        die "Cannot find debugger at $debugger\n";
+    }
+} else {
+    for ($i = 0; $i <= $#try_dbgs; $i++) {
+        $debugger = &win32path2unix($try_dbgs[$i]);
+        print "Trying debugger at $debugger\n" if ($verbose);
+        last if (-f "$debugger");
+    }
+    die "Cannot find a debugger: @try_dbgs\n" if ($i > $#try_dbgs);
+}
+if ($debugger !~ /cdb/) {
+    # FIXME: does ntsd not support $>< ?
+    # will have to pass it entire block of cmds on cmdline
+    die "ntsd not yet supported: you need cdb\n";
+    $use_logfile = 1;
+}
+
+$tmpdir = $ENV{'TMPDIR'} || $ENV{'TEMP'} || $ENV{'TMP'} || '/tmp';
+die "Cannot find temp directory $tmpdir" if (! -d $tmpdir);
+$script = "$tmpdir/bugdis-$$.script";
+$winscript = &unixpath2win32($script);
+
+die $usage if ($#ARGV < 0);
+$multiple = ($#ARGV > 0);
+while ($#ARGV >= 0) {
+    my $filearg = $ARGV[0];
+    # make it easy for users w/ non-globbing shell and non-globbing perl to
+    # pass *, w/o double-globbing and messing up escaped chars for other users
+    if (!$noglob && ($globall || $filearg =~ /\*/)) {
+        # FIXME: must convert from \ to / BEFORE the glob -- yet don't
+        # want to ruin escaped chars!  It's a tradeoff: is \* a directory delimiter
+        # followed by metachar *, or is it an escaped * trying to be literal?
+        # In everyday usage we assume no escaped *'s!
+        # If we didn't care for \* directory delim we might do:
+        #  $filearg =~ s|\\\\|//|g; # next rule won't get both slashes so need this rule
+        #  $filearg =~ s|\\([\w\d])|/\1|g; # \ to / only if before a normal char
+        $filearg =~ s|\\|/|g; # assume \ is dir delim, no support for \ as escape char
+        @files = glob($filearg);
+        print "globbing: \"$filearg\" => \"@files\"\n" if ($verbose);
+        if ($#files < 0) {
+            # if glob comes up empty, don't use its results (to get error msg)
+            @files = ($filearg);
+        }
+    } else {
+        @files = ($filearg);
+    }
+    $multiple = $multiple || $#files > 0;
+    foreach $file (@files) {
+        print "processing \"$file\"\n" if ($verbose);
+        $file =~ s|\\|/|g; # normalize
+        unless (-f "$file") {
+            die "Error: No forensic file $file\n";
+        }
+        if ($multiple) {
+            print "\n$file:\n";
+        }
+        process_file($file);
+    }
+    shift;
+}
+
+sub process_file($) {
+    my ($file) = @_;
+    $processing = 0;
+
+    print "script file is $script\n" if ($verbose);
+    open(OUT, "> $script") || die "Error: couldn't open $script";
+
+    if ($use_logfile) {
+        # must go through logfile as debugger won't write to accessible stdout
+        $logfile = "$tmpdir/bugdis-$$.log";
+        $logfile = &unixpath2win32($logfile);
+        print "Log file is $logfile\n";
+        # start fresh -- no stale results
+        if (-f $logfile) {
+            system("rm $logfile");
+        }
+        print OUT ".logopen $logfile\n";
+    }
+    
+    open(IN, "< $file") || die "Error: couldn't open $file\n";
+    while (<IN>) {
+        if (/^((source)|(target)): 0x([0-9a-fA-F]+)/) {
+            my $label = "$1";
+            $addr{$label} = $4;
+            my $val = hex($addr{$label});
+            my $start = sprintf("%08x", ($val - $backward) & 0xfffffff0);
+            my $dis = sprintf("%08x", ($val - $backward));
+            print "Going to disassemble $label $addr{$label} $dis $start\n"
+                if ($verbose);
+
+            print OUT ".dvalloc /b $addr{$label} 1000\n";
+            my $ebout = 0;
+            while (<IN>) {
+                if (/^$start /) {
+                    $ebout = 1;
+                }
+                if ($ebout > 0) {
+                    $ebout++;
+                    last if ($ebout > ($u_lines+2)); # +2 since we back-aligned
+                    # clear out ascii append
+                    s/  .{16}$//;
+                    print OUT "eb $_";
+                }
+            }
+            # let's assume 5-byte instrs
+            $num_instrs = $u_lines*3;
+            print OUT "u $dis L$num_instrs\n";
+        }
+    }
+    if ($use_logfile) {
+        print OUT ".logclose\n";
+    }
+    # get syntax error if put newline after q!
+    print OUT "q" unless ($interactive);
+    &run_debugger($script, $addr{'source'}, $addr{'target'});
+}
+
+system("rm $script");
+
+sub run_debugger($) {
+    my ($runfile, $tgt1, $tgt2) = @_;
+
+    $cmd = "\$><$winscript";
+  
+    # put quotes around debugger to handle spaces
+    $cmdline = "\"$debugger\" -c \"$cmd\" $app";
+    print "$cmdline\n" if ($verbose);
+    if ($interactive) {
+        system("$cmdline") && die "Error: couldn't run $cmdline\n";
+    } elsif ($raw) {
+        system("$cmdline") && die "Error: couldn't run $cmdline\n";
+        if ($use_logfile) {
+            system("cat $logfile");
+        }
+    } else {
+        # prefix each ln output with the query (debugger doesn't re-print -c cmds)
+        $output = 0;
+        if ($use_logfile) {
+            system("$cmdline") && die "Error: couldn't run $cmdline\n";
+            open(DBGOUT, "< $logfile") || die "Error: couldn't open $logfile\n";
+        } else {
+            open(DBGOUT, "$cmdline |") || die "Error: couldn't run $cmdline\n";
+        }
+        while (<DBGOUT>) {
+            if (/^Allocat/) {
+                $output = 1;
+                print "\n";
+            } elsif ($output) {
+                if (/^([0-9A-Fa-f]+)/) {
+                    if ($1 eq $tgt1 || $1 eq $tgt2) {
+                        print "=> $_";
+                    } else {
+                        print "   $_";
+                    }
+                } elsif (/Attempt to access/) {
+                    # We deliberately continue past this message:
+                    #  Allocation failed, Win32 error 487
+                    #      "Attempt to access invalid address."
+                    # as it often means we tried to allocate within our 1st alloc
+                    print "WARNING: alloc failure (may simply mean 2nd within 1st)\n";
+                } else {
+                    $output = 0;
+                }
+            }
+        }
+        close(DBGOUT);
+    }
+}
+
+sub unixpath2win32($) {
+    my ($p) = @_;
+    # use cygpath if available, it will convert /home to c:\cygwin\home, etc.
+    $cp = `cygpath -wi \"$p\"`;
+    chomp $cp;
+    if ($cp eq "") {
+        # do it by hand
+        # add support for /x => x:
+        $p =~ s|^/([a-z])/|\1:/|;
+        # forward slash becomes backslash since going through another shell
+        $p =~ s|/|\\|g;
+    } else {
+        $p = $cp;
+    }
+    # make single backslashes doubles
+    $p =~ s|\\{1}|\\\\|g;
+    return $p;
+}
+
+sub win32path2unix($) {
+    my ($p) = @_;
+    # avoid issues with spaces by using 8.3 names
+    $cp = `cygpath -dmi \"$p\"`;
+    chomp $cp;
+    if ($cp eq "") {
+        # do it by hand
+        $p =~ s|\\|/|g;
+    } else {
+        $p = $cp;
+    }
+    return $p;
+}
diff --git a/tools/bugtitle.pl b/tools/bugtitle.pl
new file mode 100755
index 000000000..b537a9741
--- /dev/null
+++ b/tools/bugtitle.pl
@@ -0,0 +1,532 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2005-2007 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+## bugtitle.pl
+## by Derek Bruening
+## March 2005
+##
+## produces a bug title from a forensic file
+## uses address_query.pl and wld.pl as helpers
+##
+## by default does an explicit glob if input filename as a * in it
+## (since we have users in shells and perls that don't glob!)
+## -noglob disables that; -glob causes explicit glob of all input filenames
+##
+## if updated copy to \\10.1.5.11\QA-Shares\QA\tools
+###########################################################################
+## Examples of type of output:
+##
+## > bugtitle.pl bug_4106_dllhost/dllhost.exe.2964.00000000.html
+## CRASH 7100d818 (21113 dllhost.exe) place_fragment:fcache.c(1476)
+## 
+## > bugtitle.pl bug_3749_connon/CanoScan_N650U_N656U_CSUv571a.exe.3424.00000000.html
+## VIOLATION NZHB.5166.B (21131 CanoScan_N650U_N656U_CSUv571a.exe)  symcjit.dll --->  HEAP section
+## 
+## > bugtitle.pl SmaService.exe.2104.00000000.xml
+## VIOLATION NGSU.4493.B (30216 SmaService.exe) 0x7c2e4e8d (ADVAPI32.dll .text rx) => 0x0017b356 (heap rw)
+##
+## > bugtitle.pl codemod.exe.1416.00000000.xml  
+## VIOLATION QXMA.5800.A (custom codemod.exe) 0x004013ee (codemod.exe .text rx) => 0x0012ff60 (stack rw)
+## 
+## > bugtitle.pl /work/studies/dgc/detectmode-nonativeexec/dotnet-serialization.exe/serialization.exe.136.00000172.xml
+## VIOLATION QKNR.0769.C (custom serialization.exe) 0x791d721e (mscorwks.dll .text rx) => 0x7999cb28 (NULL|mscorlib.dll .text rx)
+## 
+## > bugtitle.pl bug_4243_deadlock/inetinfo.exe.2020.00000000.html
+## TIMEOUT (21129 inetinfo.exe)
+## 
+## > bugtitle.pl sqlservr.exe.3048.00000000.xml 
+## OUT OF MEMORY (30118 sqlservr.exe) MB: 2047 RAM, 8 avail, 2169(^2215)/3940 commit
+## 
+## > bugtitle.pl bug_3749_connon/build_21132/CanoScan_N650U_N656U_CSUv571a.exe.4064.00000000.html
+## ASSERT (21132 CanoScan_N650U_N656U_CSUv571a.exe) win32/os.c:3263 is_native_thread_state_valid(trec->dcontext, (app_pc)cxt->Eip, (byte *)cxt->Esp)
+##
+###########################################################################
+## Examples showing that type of slash doesn't matter (though backslashes
+## must be doubled within cygwin):
+##
+## > bugtitle.pl \\\\10.1.5.10\\bugs\\bug_3978_w3wpcrash\\w3wp.exe.1124.00000000.html
+## CRASH 7001588e (21102 w3wp.exe) vm_area_add_fragment:vmareas.c(4466)
+##
+## > bugtitle.pl //10.1.5.10/bugs/bug_3978_w3wpcrash/w3wp.exe.1124.00000000.html
+## CRASH 7001588e (21102 w3wp.exe) vm_area_add_fragment:vmareas.c(4466)
+##
+###########################################################################
+## Examples from cmd shell:
+##
+## C:\derek\dr\tools>perl bugtitle.pl \\10.1.5.10\bugs\bug_3978_w3wpcrash\w3wp.exe.1124.00000000.html
+## CRASH 7001588e (21102 w3wp.exe) vm_area_add_fragment:vmareas.c(4466)
+##
+## c:\>perl c:/derek/dr/tools/bugtitle.pl \\10.1.5.10\bugs\bug_3749_connon\CanoScan_N650U_N656U_CSUv571a.exe.3424.00000000.html
+## VIOLATION NZHB.5166.B (21131 CanoScan_N650U_N656U_CSUv571a.exe)  symcjit.dll --->  HEAP section
+###########################################################################
+
+$tools = $ENV{'DYNAMORIO_TOOLS'};
+if ($tools eq "") {
+    # use same dir this script sits in
+    $tools = $0;
+    if ($tools =~ /[\\\/]/) {
+        $tools =~ s/^(.+)[\\\/][\w\.]+$/\1/;
+        $tools =~ s/\\/\\\\/g;
+    } else {
+        $tools = "./";
+    }
+    # address_query needs DYNAMORIO_TOOLS set to find DRload.exe
+    $ENV{'DYNAMORIO_TOOLS'} = $tools;
+}
+# may not be on cygwin, so explicitly invoke perl
+$addrquery = "perl $tools/address_query.pl";
+$vioquery = "perl $tools/wld/wld.pl -security_violation";
+
+# parameters
+$usage = "Usage: $0 [-b <build_dir_base>] [-d <path-to-dynamorio.dll>] [-v] [-a] [-q] [-glob] [-noglob] <forensic_file>\n";
+
+$verbose = 0;
+$quiet = 0;
+$DRdll = "";
+$builddir = "";
+$globall = 0;
+$noglob = 0;
+$analyze = 0;
+
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq '-v') {
+        $verbose = 1;
+    } elsif ($ARGV[0] eq '-q') {
+        $quiet = 1;
+    } elsif ($ARGV[0] eq '-a') {
+        $analyze = 1;
+    } elsif ($ARGV[0] eq '-glob') {
+        $globall = 1;
+    } elsif ($ARGV[0] eq '-noglob') {
+        $noglob = 1;
+    } elsif ($ARGV[0] eq '-b') {
+        die $usage if ($#ARGV <= 0);
+        shift;
+        $builddir = $ARGV[0];
+        $builddir =~ s|\\|/|g; # normalize
+        unless (-d "$builddir") {
+            die "$err No directory $builddir\n";
+        }
+    } elsif ($ARGV[0] eq '-d') {
+        die $usage if ($#ARGV <= 0);
+        shift;
+        $DRdll = $ARGV[0];
+        $DRdll =~ s|\\|/|g; # normalize
+        unless (-f "$DRdll") {
+            die "$err No file $DRdll\n";
+        }
+    } elsif ($ARGV[0] =~ /^-/) {
+        die $usage;
+    } else {
+        # rest should be forensic file names
+        last;
+    }
+    shift;
+}
+
+die $usage if ($#ARGV < 0);
+$multiple = ($#ARGV > 0);
+while ($#ARGV >= 0) {
+    my $filearg = $ARGV[0];
+    # make it easy for users w/ non-globbing shell and non-globbing perl to
+    # pass *, w/o double-globbing and messing up escaped chars for other users
+    if (!$noglob && ($globall || $filearg =~ /\*/)) {
+        # FIXME: must convert from \ to / BEFORE the glob -- yet don't
+        # want to ruin escaped chars!  It's a tradeoff: is \* a directory delimiter
+        # followed by metachar *, or is it an escaped * trying to be literal?
+        # In everyday usage we assume no escaped *'s!
+        # If we didn't care for \* directory delim we might do:
+        #  $filearg =~ s|\\\\|//|g; # next rule won't get both slashes so need this rule
+        #  $filearg =~ s|\\([\w\d])|/\1|g; # \ to / only if before a normal char
+        $filearg =~ s|\\|/|g; # assume \ is dir delim, no support for \ as escape char
+        @files = glob($filearg);
+        print "globbing: \"$filearg\" => \"@files\"\n" if ($verbose);
+        if ($#files < 0) {
+            # if glob comes up empty, don't use its results (to get error msg)
+            @files = ($filearg);
+        }
+    } else {
+        @files = ($filearg);
+    }
+    $multiple = $multiple || $#files > 0;
+    foreach $file (@files) {
+        print "processing \"$file\"\n" if ($verbose);
+        $file =~ s|\\|/|g; # normalize
+        unless (-f "$file") {
+            die "Error: No forensic file $file\n";
+        }
+        if ($multiple) {
+            print "\n$file:\n";
+        }
+        process_file($file);
+    }
+    shift;
+}
+
+sub process_file($) {
+    my ($file) = @_;
+    my $prefix = "<unknown type>";
+    my $suffix = ""; # ok to be empty
+    my $build_num = "<unknown build>";
+    my $app = "<unknown app>";
+
+    open(IN, "< $file") || die "Error: couldn't open $file\n";
+    while (<IN>) {
+        # Handle CR
+        s/\r//;
+        # Type of bug
+        if (/^Execution security violation/) {
+            $prefix = "VIOLATION";
+            if ($file =~ /\.html/) {
+                # use wld for html files
+                $vioinfo = `$vioquery $file`;
+                # example: Security violation detected from  symcjit.dll --->  HEAP section
+                $vioinfo =~ /Security violation detected from (.+)/m;
+                $suffix = $1;
+                # wld.pl likes to put extra spaces in!
+                $suffix =~ s/(\s)\s+/\1/g;
+                $suffix =~ s/^\s+//;
+            } # else we hit the source and target-properties below
+        } 
+        elsif (/Threat ID:\s*([^\s\<]+)/) {
+            $threat = $1;
+            $prefix .= " $threat";
+            if ($threat =~ /c0000017/) {
+                $prefix .= " V";
+            }
+            elsif ($threat =~ /c0000012d/) {
+                $prefix .= " P";
+            }
+        }
+        elsif (/<dll\s+range=\"0x(\w+)-0x(\w+)\"\s*name=\"([^\"]+)\"/) {
+            $dll_name[$num_dlls] = $3;
+            $dll_start[$num_dlls] = hex($1);
+            $dll_end[$num_dlls] = hex($2);
+            $num_dlls++;
+        }
+        elsif (/<source-properties/) {
+            # for hot patch violations, source isn't relevant; see case 7686
+            if (!($threat =~ /[PH]$/)) {
+                $suffix = &address_properties();
+            }
+        }
+        elsif (/<target-properties/) {
+            # for hot patch violations, target isn't relevant; see case 7686
+            if (!($threat =~ /[PH]$/)) {
+                $suffix .= " => " . &address_properties();
+            }
+        }
+        elsif (/^Unrecoverable error at PC 0x([0-9A-Za-z]+)/) {
+            # w/ multiple files, need local here so we look at each
+            # crash separately
+            my $symdll;
+            $addr = $1;
+            if ($addr =~ /^15/) {
+                $lib = "debug";
+            } else {
+                $lib = "release";
+            }
+            if ($DRdll eq "") {
+                $symdll = &findDRdll($builddir, $build_num, $lib);
+            } else {
+                # if dr dll passed to us, use it for every file
+                $symdll = $DRdll;
+            }
+            $prefix = "CRASH $addr";
+            $suffix = "[error acquiring line number]";
+            if ($symdll ne "") {
+                my $query = "$addrquery";
+                $query .= " -raw" if ($verbose);
+                print "invoking \"$query $symdll $addr\"\n" if ($verbose);
+                $sym = `$query $symdll $addr`;
+                print "addrquery returned: $sym\n" if ($verbose);
+                if ($sym =~ m|([\w\./]+\(\d+\))|m) {
+                    $line = $1;
+                    $sym =~ /dynamorio!(.+)\+/m;
+                    $func = $1;
+                    $suffix = "$func:$line";
+                } else {
+                    print "Unable to find address in $symdll\n";
+                    # We found the dll, but couldn't find the address.
+                    # Is likely an exception in generated code.
+                    $suffix = "<exception in generated code>";
+                }
+            } else {
+                print "Error locating dynamorio.dll $build_num\n";
+            }
+        }
+        elsif (/^Internal SecureCore Error: (.*)/) {
+            $prefix = "ASSERT";
+            $suffix = $1;
+        }
+        elsif (/^CURIOSITY : (.*)/) {
+            $prefix = "CURIOSITY";
+            $suffix = $1;
+        }
+        elsif (/^Timeout expired/) {
+            $prefix = "TIMEOUT";
+        } 
+        elsif (/^Out of memory/) {
+            $prefix = "OUT OF MEMORY";
+        } 
+        elsif (($analyze || $prefix =~ /MEMORY/) && /<basic-information>/) {
+            $_ = <IN>;
+            /\d+ \d+ \d+ (\d+) \d+ \d+ \d+/;
+            $ram = $1 * 4/1024;
+            $suffix .= sprintf("%u RAM, ", $ram);
+        }
+        elsif (($analyze || $prefix =~ /MEMORY/) && /num_threads:/) {
+            /num_threads: (\d+)/;
+            $ThreadCount = $1;
+            $suffix .= sprintf(", %d Thr", $ThreadCount);
+            if ($ThreadCount > 1000) {
+                $suffix .= "ead PILE!";
+            }
+        }
+        elsif (($analyze || $prefix =~ /MEMORY/) && /<vm-counters>/) {
+            $_ = <IN>;
+            ($PeakVirtualSize,
+             $VirtualSize,
+             $PageFaultCount,
+             $PeakWorkingSetSize,
+             $WorkingSetSize,
+             $QuotaPeakPagedPoolUsage,
+             $QuotaPagedPoolUsage)
+                = split / /, $_;
+
+            $_ = <IN>;
+            ($QuotaPeakNonPagedPoolUsage,
+             $QuotaNonPagedPoolUsage,
+             $PagefileUsage,
+             $PeakPagefileUsage)
+                = split / /, $_;
+
+            # matching drview names (e.g. PagefileUsage is Priv) */
+            print "KB: " .
+                $PeakVirtualSize/1024 . " PVSz," .
+                $VirtualSize/1024 . " VSz," . 
+                $PeakPagefileUsage/1024 . " PPriv," . 
+                $PagefileUsage/1024 . " Priv," . 
+                $PeakWorkingSetSize/1024 . " PWSS," . 
+                $WorkingSetSize/1024 . " WSS," . 
+                $PageFaultCount/1 . " Fault," . 
+                $QuotaPeakPagedPoolUsage/1024 . " PPage,"  .
+                $QuotaPagedPoolUsage/1024 . " Page," .
+                $QuotaPeakNonPagedPoolUsage/1024 . " PNonP," . 
+                $QuotaNonPagedPoolUsage/1024 . " NonP" . 
+                "\n" if ($verbose);
+
+            # note that pre 4.3 core writes these counters as %d  (case 10481)
+            # whlie we want to convert them to uint's here
+            if ($VirtualSize < 0) {
+                $VirtualSize += 0x100000000; # 2^32
+            }
+            if ($PagefileUsage < 0) {
+                $PagefileUsage += 0x100000000; # 2^32
+            }
+            if ($PeakWorkingSetSize < 0) {
+                $PeakWorkingSetSize += 0x100000000; # 2^32
+            }
+
+            # all in MB
+            $suffix .= sprintf("MB: %u VSz, %u Priv, %u PWSS; ", 
+                               $VirtualSize/1024/1024, 
+                               $PagefileUsage/1024/1024,
+                               $PeakWorkingSetSize/1024/1024);
+        }
+        elsif (($analyze || $prefix =~ /MEMORY/) && /<performance-information>/) {
+            # skip 1st line
+            <IN>;
+            # this line has what we want
+            $_ = <IN>;
+            /-?\d+ -?\d+ -?\d+ (\d+) (\d+) (\d+) (\d+)/;
+            # don't expect these to be negative
+            $avail = $1 * 4/1024;
+            $commit = $2 * 4/1024;
+            $limit = $3 * 4/1024;
+            $peak = $4 * 4/1024;
+            $suffix .= sprintf("%u avail, %u(^%u)/%u commit",
+                               $avail, $commit, $peak, $limit);
+        }
+
+        # Build version
+        elsif (/^Generated by SecureCore [^,]+,\s*(.+)/) {
+            $build_string = $1;
+            if ($build_string =~ /build ([0-9]+)/) {
+                $build_num = $1;
+            } else {
+                # FIXME: grab path from reg keys and try that path on local host?
+                $build_num = "custom";
+            }
+            
+        }
+        
+        # Process Name
+        #
+        # violation
+        elsif (/^Short[\s\-]Qualified[\s\-]Name[:=]\s*\"?([^\"\n]+)\"?/i) {
+            $app = $1; # short qualified name
+        }
+        # crash
+        elsif (/^\s*Name\s?=\s*\"?([^\"\n]+)\"?/i) {
+            $app = $1; # full qualified name
+            $app =~ s/.*[\\\/]//; # remove path
+        }
+        # 2.1 compatibility, let better name finding (above) override since we make
+        # assumptions (that may not always hold) about name mangling here
+        elsif ($app =~ /^<unknown app>$/) {
+            if (/^Command Line:\s*(.+)/) {
+                $app = $1; # full cmdline
+                if ($app !~ /svchost.exe/) {
+                    # remove args
+                    $app =~ s/(\.exe)\W+.*$/\1/;
+                    # remove quotes
+                    $app =~ s/\"//g;
+                }
+                $app =~ s/.*[\\\/]//; # remove path
+            }
+        }
+    }
+    close(IN);
+
+    print "$prefix ($build_num $app) $suffix\n";
+}
+
+sub findDRdll($,$,$) {
+    my ($builddir, $build_num, $lib) = @_;
+    # would be nice to use symbol server, but cannot locate
+    # based solely on build number w/o querying every dll, or having
+    # a symlink set up externally or something.
+    my @try = ( "//10.1.5.15/nightly",
+                "//10.1.5.15/nightly_archive" );
+    if ($builddir ne "") {
+        unshift @try, $builddir;
+    }
+    print "searching these bases: @try\n" if ($verbose);
+    # match ver_2_1, ver_2_5, etc., as well as the new dist
+    # we avoid {ver*,dist} since that gives two entries regardless of existence
+    my $ver = "[vd][ei][rs]*";
+    my $missed_at_all = 0;
+    foreach $base (@try) {
+        my $glob_path = "$base/$build_num/$ver/lib/$lib/dynamorio.dll";
+        my @wildcard = glob($glob_path);
+        if (@wildcard == 0) {
+            print STDERR "couldn't find $glob_path\n" if (!$quiet);
+            $missed_at_all = 1;
+        }
+        foreach $w (@wildcard) {
+            print "looking for $w\n" if ($verbose);
+            if (-f $w) {
+                # avoid found message if did it 1st try
+                print STDERR "found $w\n" if (!$quiet && $missed_at_all);
+                return $w;
+            }
+            print STDERR "couldn't find $w\n" if (!$quiet);
+            $missed_at_all = 1;
+        }
+    }
+    return "";
+}
+
+sub unixpath2win32($) {
+    my ($p) = @_;
+    # use cygpath if available, it will convert /home to c:\cygwin\home, etc.
+    $cp = `cygpath -wi \"$p\"`;
+    chomp $cp;
+    if ($cp eq "") {
+        # do it by hand
+        # add support for /x => x:
+        $p =~ s|^/([a-z])/|\1:/|;
+        # forward slash becomes backslash since going through another shell
+        $p =~ s|/|\\|g;
+    } else {
+        $p = $cp;
+    }
+    # make single backslashes doubles
+    $p =~ s|\\{1}|\\\\|g;
+    return $p;
+}
+
+# examples:
+# 0x04932af4 (stack rx)
+# 0x003b2f76 (heap rw)
+# 0x79b99773 (NULL|mscorlib.dll .data rwx)"
+sub address_properties() {
+    my $address = "";
+    my $module = "";
+    my $section = "";
+    my $protect = "";
+    my $type = "";
+    while (<IN>) {
+        # Handle CR
+        s/\r//;
+        if (/address=.*\"([^\"]+)\"/) {
+            $address = $1;
+        } elsif (/module=.*\"([^\"]+)\"/) {
+            $module = $1;
+        } elsif (/sec_name=.*\"([^\"]+)\"/) {
+            $section = $1;
+        } elsif (/State=.* FREE/) {
+            $protect = 'free';
+        } elsif ($protect ne 'free' && /Protect=.*\"\w+ ([^\"]+)\"/) {
+            $protect = $1;
+            $protect =~ s/-//g;
+        } elsif (/Type=.*\"\w+ ([^\"]+)\"/) {
+            $type = $1;
+            if ($type =~ /IMAGE/i) {
+                # if no export name get file name from module list
+                if ($module =~ /\(none\)/) {
+                    # brute force
+                    # FIXME: use nice interval data structure
+                    $address =~ /0x(\w+)/;
+                    my $addrval = hex($1);
+                    for ($i=0; $i<$num_dlls; $i++) {
+                        if ($addrval >= $dll_start[$i] &&
+                            $addrval < $dll_end[$i]) {
+                            $module = "NULL|" . $dll_name[$i];
+                        }
+                    }
+                }
+                $module .= " $section";
+            } else {
+                if ($threat =~ /\.A$/) {
+                    $module = "stack";
+                } else {
+                    $module = "heap";
+                }
+            }
+            # we assume we'll see a Type line else we'll loop to the end!
+            last;
+        }
+    }
+    return "$address ($module $protect)";
+}
diff --git a/tools/caller-profile.pl b/tools/caller-profile.pl
new file mode 100755
index 000000000..62f7574b2
--- /dev/null
+++ b/tools/caller-profile.pl
@@ -0,0 +1,136 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2006-2007 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### caller-profile.pl
+### author: Derek Bruening   April 2006
+###
+### Analyzes the output of crude caller profiling that
+### looks like this:
+###   0x710282fc 0x7102826c 0x710283a2 0x7103569a 242
+###   0x710282fc 0x7102b25e 0x71035934 0x710299a5 31537
+###   0x710282fc 0x7102826c 0x710283e3 0x710358f6 36271
+
+$usage = "Usage: $0 [-build [dbg|rel|intr|prof]] [-base <DRdllbase>] [-dll <DRdllpath>] <callproffile>\n";
+
+$bld = "dbg";
+$DR = $ENV{'DYNAMORIO_HOME'};
+$DRdll = "";
+$infile = "";
+$base = 0;
+$reloc = 0;
+
+# get optional params
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq '-build') {
+	if ($#ARGV <= 0) { print $usage; exit; }
+	shift;
+	$bld = $ARGV[0];
+    } elsif ($ARGV[0] eq '-base') {
+	if ($#ARGV <= 0) { print $usage; exit; }
+	shift;
+	$base = $ARGV[0];
+    } elsif ($ARGV[0] eq '-dll') {
+	if ($#ARGV <= 0) { print $usage; exit; }
+	shift;
+	$DRdll = $ARGV[0];
+    } else {
+        $infile = $ARGV[0];
+        last;
+    }
+    shift;
+}
+
+die $usage if ($infile eq "");
+
+if ($DRdll eq "") {
+    $DRdll = "$DR/exports/x86_win32_$bld/dynamorio.dll";
+}
+die "Cannot find $DRdll" unless (-f $DRdll);
+
+if ($base != 0) {
+    if ($DRdll =~ /dbg/ || $DRdll =~ /debug/) {
+        $normbase = 0x15000000;
+    } else {
+        $normbase = 0x71000000;
+    }
+    $reloc = $normbase - hex($base);
+}
+
+open(IN, "< $infile") || die "Error: Couldn't open $infile for input\n";
+while (<IN>) {
+    if (/^([0-9xa-fA-F ]+) (\d+)$/) {
+        $addrs = $1;
+        if ($reloc != 0) {
+            $newaddrs = "";
+            foreach $a (split(' ', $addrs)) {
+                $newaddrs .= sprintf("0x%08x ", hex($a) + $reloc);
+            }
+            $addrs = $newaddrs;
+        }
+        $cnt=$2;
+        $hexes[$num]=$addrs;
+        # FIXME: ensure no symbol path set that will have network delays --
+        # perhaps even have address_query.pl clear the symbol path every time?
+        # FIXME: would be faster to batch up all addresses and then feed
+        # through a single invocation of address_query.pl
+        open(ADDR, "address_query.pl $DRdll $addrs |") ||
+            die "Error running address_query.pl\n";
+        $out[$num]=sprintf("count=$cnt\n");
+        while (<ADDR>) {
+            $out[$num] .= $_;
+            if (/^[\w\.\/]+\((\d+)\)/) {
+                $line = $1;
+            } elsif (/^\(\w+\)\s+dynamorio!([^\+]+)/) {
+                $syms[$num] .= "$1:$line ";
+            }
+        }
+        $sortme[$num]=$num;
+        $count[$num++]=$cnt;
+        $sum+=$cnt;
+    } 
+}
+close(IN);
+
+@sortme=sort({$count[$a]<=>$count[$b]} @sortme);
+
+# summary
+print "Total calls: $sum\n\n";
+for ($i=$num-1; $i>=0; $i--) {
+    $n=$sortme[$i];
+    printf "%5.2f%% %s\n", 100*$count[$n]/$sum, $syms[$n];
+}
+
+# details
+for ($i=$num-1; $i>=0; $i--) {
+    $n=$sortme[$i];
+    printf "\n--------\n%5.2f%%\n%s", 100*$count[$n]/$sum, $out[$n];
+}
diff --git a/tools/closewnd.c b/tools/closewnd.c
new file mode 100644
index 000000000..56a4cd60a
--- /dev/null
+++ b/tools/closewnd.c
@@ -0,0 +1,85 @@
+/* **********************************************************
+ * Copyright (c) 2003-2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+
+#include "Windows.h"
+#include "stdio.h"
+
+/* make this pretty long, some regression runs are pretty slow and we just
+ * want to keep the suite from completely hanging */
+#define TIMEOUT_MS 60000
+
+#define SLEEP_INTERVAL_MS 500
+
+int
+main(int argc, char **argv)
+{
+    HANDLE hw;
+    DWORD res;
+    UINT wait_tot, wait_max;
+
+    if (argc != 3) {
+        fprintf(stderr, "usage: %s <window caption> <timeout (sec)>\n", argv[0]);
+        exit(1);
+    }
+    
+    wait_tot = 0;
+    wait_max = atoi(argv[2]) * 1000;
+
+    while (wait_tot < wait_max) {
+        hw = FindWindow(NULL, argv[1]);
+        if (hw == NULL) {
+            wait_tot += SLEEP_INTERVAL_MS;
+            Sleep(SLEEP_INTERVAL_MS);
+        }
+        else {
+            res = SendMessageTimeout(hw, WM_CLOSE, 0, 0, SMTO_NORMAL, 
+                                     TIMEOUT_MS, NULL);
+            printf("Close message sent.\n");
+            if (res == 0) {
+                /* error case */
+                res = GetLastError();
+                /* msdn lies! claims GetLastError returns 0 for timeout case, yet
+                 * it in fact returns ERROR_TIMEOUT (no suprise) just check for
+                 * both, not too suprising, I suppose, as I think they have some 
+                 * typos in the flag descriptions for the function also */
+                if (res == 0 || res == ERROR_TIMEOUT) {
+                    printf("Window timed out without response\n");
+                } else {
+                    printf("Error sending close message %d\n", res);
+                }
+            }
+
+            break;
+        }
+    }
+}
diff --git a/tools/commitmsg.pl b/tools/commitmsg.pl
new file mode 100755
index 000000000..8650c8708
--- /dev/null
+++ b/tools/commitmsg.pl
@@ -0,0 +1,213 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2004-2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### commitmsg.pl
+### author: Derek Bruening   March 2004
+###
+### Commits files with per-file messages listed in a control file.
+### The control file format is arbitrary and of my personal
+### preference, of course :)
+###
+### Each entry has the format:
+###
+###   [subdir/]<filename>:
+###   message line 1
+###   message line 2
+###   must end with a dot on a line by itself
+###   . 
+###
+### Stops parsing the file when it sees a line beginning w/ "=====", so
+### put global comments at the bottom -- and w/ the new "make diff*" targets
+### you should put a toemail line there as well and use the file as your
+### diff notes file.
+###
+### Here's a real-world example:
+###
+###   Makefile:
+###   + STANDALONE_UNIT_TEST support for building executable rather than library
+###   .
+###   dynamo.c:
+###   + STANDALONE_UNIT_TEST support by sharing standalone_init() w/
+###   CLIENT_INTERFACE and augmenting it to support log files
+###   .
+###   globals.h:
+###   + STANDALONE_UNIT_TEST support in export of standalone usage
+###   .
+###   vmareas.c:
+###   fixed vector split/merge bugs by redoing split code
+###  
+###   + VMAREAS_UNIT_TEST vector add/merge/split/delete tests
+###   .
+###   linux/os.c:
+###   no _init() or _fini() for STANDALONE_UNIT_TEST
+###   .
+###   ===========================================================================
+###   toemail:
+###   Two items in this diff:
+###  
+###   1) Fixes case 3415
+###  
+###   2) Adds support for easily building a standalone core unit test (case 3136)
+###  
+###   To build the vmareas vector test, add this to Makefile.mydefines:
+###  
+###   STANDALONE_UNIT_TEST=1
+###   ADD_DEFINES += $(D)VMAREAS_UNIT_TEST
+###  
+###   Now the make default target will build ../build/x86_linux_dbg/drtest
+###   (will add .exe suffix on windows).  Could copy that somewhere more
+###   convenient but I didn't bother.
+###  
+###   I added support for logfiles for easier debugging.
+
+$usage = "Usage: $0 [-v] [-n] [-q] <msgfile>\n";
+$verbose = 0;
+$nop = 0;
+$interactive = 1;
+
+# cannot commit until reviewed!
+$reviewed = 0;
+
+# just assume these temp file names are ok
+$cvs_msgfile = "_cvs_msg";
+$cvs_output = "_cvs_out";
+
+$verbose = 0;
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq "-v") {
+	$verbose = 1;
+    } elsif ($ARGV[0] eq "-n") {
+	$nop = 1;
+    } elsif ($ARGV[0] eq "-q") {
+	$interactive = 0;
+    } else {
+	$msgfile = $ARGV[0];
+    }
+    shift;
+}
+if (!defined($msgfile)) {
+    print $usage;
+    exit;
+}
+
+# first ensure that we're up-to-date
+# look for anything other than these results from cvs status:
+# Up-to-date
+# Locally {Modified,Added,Removed}
+#
+$uptodate = system("cvs status | grep Status | grep -vE 'Up-to-date|Locally|File had conflicts'");
+if (!$uptodate) {
+    print "Not up to date!  Merge, re-test, and try again\n";
+    exit -1;
+}
+
+open(MSGFILE, "< $msgfile") || die "Error opening $msgfile\n";
+$vers = "";
+
+while (<MSGFILE>) {
+    if (!$in_msg) {
+        if (/^reviewed by/i) {
+            $reviewed = 1;
+            $reviewer = $_;
+        } elsif (/(\S+):$/) {
+            if (!$reviewed) {
+                die "Cannot commit until reviewed!\n";
+            }
+            $file = $1;
+            $in_msg = 1;
+            $msg = "";
+            next;
+        } elsif (/^=====/) {
+            last;
+        } else {
+            print "Ignoring line: $_" unless /^$/;
+        }
+    } else {
+        if (/^\.$/) {
+            open(MSG, "> $cvs_msgfile") || die "Couldn't open $cvs_msgfile\n";
+            print MSG $msg;
+            close(MSG);
+            print "\n-----------------------------\n";
+            print "message for $file is: \n$msg\n";
+            if ($interactive) {
+                print "hit <enter> to continue, ^C to abort\n";
+                <STDIN>;
+            }
+            if ($nop) {
+                $vers .= "$file:<NOT COMMITTED>\n";
+            } else {
+                print "cvs commit -F $cvs_msgfile $file\n";
+                # record output so we can print a version summary at the end
+                
+                # somehow pipe ends up sitting there for ~40 seconds so we use a tmp file
+                #open(COM, "cvs commit -F $cvs_msgfile $file 2>&1 |") || die "Couldn't run cvs commit\n";
+                system("cvs commit -F $cvs_msgfile $file 2>&1 > $cvs_output");
+                open(COM, "< $cvs_output") || die "Couldn't open cvs output $cvs_output\n";
+                while (<COM>) {
+                    # look for:
+                    #   /home/dr/cvsroot/src/win32/os_exports.h,v  <--  os_exports.h
+                    #   new revision: 1.99; previous revision: 1.98
+                    # or for:
+                    #   RCS file: /home/dr/cvsroot/src/win32/os_private.h,v
+                    #   /home/dr/cvsroot/src/win32/os_private.h,v  <--  os_private.h
+                    #   initial revision: 1.1
+                    if (m|/cvs[^/]*/[^/]+/([^,]+)| && !/^RCS file/) {
+                        $vers .= "$1:";
+                    } elsif (/^\w+ revision: ([\d.]+)/) {
+                        $vers .= "$1\n";
+                    }
+                    print $_;
+                }
+                close(COM);
+            }
+
+            $in_msg = 0;
+            $msg = "";
+        } else {
+            $msg .= $_;
+        }
+    }
+}
+close(MSGFILE);
+
+print "\n-----------------------------\n";
+print "Summary:\n\n${reviewer}\nCommitted:\n${vers}\n";
+
+# finally, ensure that we didn't miss any files
+# look for these results from cvs status:
+# Locally {Modified,Added,Removed}
+#
+$alldone = system("cvs status | grep Status | grep Locally");
+if (!$alldone) {
+    print "Did you forget to commit these files?\n";
+    exit -1;
+}
diff --git a/tools/cvs-snapshot.sh b/tools/cvs-snapshot.sh
new file mode 100644
index 000000000..9f39ef911
--- /dev/null
+++ b/tools/cvs-snapshot.sh
@@ -0,0 +1,89 @@
+# **********************************************************
+# Copyright (c) 2007 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# FIXME: use a real tag in CVS when this is done for real
+
+# -kv doesn't handle properly binary files
+
+mkdir dynamorio
+cd dynamorio
+
+#export CO=checkout -D 2007-09-28
+# note a date in the future should work fine
+
+# thanksgiving_cvs_snapshot is last CVS checkin
+# look for the code at its new home at perforce-tiger://depot/dynamorio/...
+
+# all modules in both repositories should be tagged with cvs rtag thanksgiving_cvs_snapshot
+
+export TAG=thanksgiving_cvs_snapshot
+
+# -D 2007-11-21
+export CO="export -r $TAG -kv"
+
+#rename src -> core
+cvs -d /repo/cvseast $CO -d core src
+cvs -d /repo/cvseast $CO tools
+#rename share -> libutil
+cvs -d /repo/cvs $CO -d libutil share
+#rename liveshield-update -> liveshield
+cvs -d /repo/cvs $CO -d liveshield liveshield-update
+
+cvs -d /repo/cvseast $CO suite
+cvs -d /repo/cvseast $CO api
+cvs -d /repo/cvseast $CO clients
+cvs -d /repo/cvseast $CO win32lore
+cvs -d /repo/cvseast $CO docs
+cvs -d /repo/cvseast $CO papers
+
+#remove some old stuff
+rm -rf docs/status
+
+cvs -d /repo/cvs $CO nightly
+cvs -d /repo/cvs $CO nodemgr
+cvs -d /repo/cvs $CO threatinfo
+
+# these take a little more time, comment out if testing
+# EAST coast repository
+cvs -d /repo/cvseast $CO exploits
+cvs -d /repo/cvs $CO -d exploits exploits # merge from west coast repository exploits/antihips
+cvs -d /repo/cvseast $CO benchmarks
+cvs -d /repo/cvseast $CO spec2kdata
+
+# adding config temporarily
+cvs -d /repo/cvs $CO config
+
+### below the line
+# detertray
+
+# do we want the exploits from the west coast repository??
+
+cd ..
+tar czvf dynamorio-main.tgz dynamorio
diff --git a/tools/cygwin-bash_profile b/tools/cygwin-bash_profile
new file mode 100644
index 000000000..5aae3eea0
--- /dev/null
+++ b/tools/cygwin-bash_profile
@@ -0,0 +1,36 @@
+# **********************************************************
+# Copyright (c) 2005 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# .bash_profile
+
+# Get the aliases and functions
+if [ -f ~/.bashrc ]; then
+    . ~/.bashrc
+fi
diff --git a/tools/cygwin-bashrc b/tools/cygwin-bashrc
new file mode 100755
index 000000000..d33978359
--- /dev/null
+++ b/tools/cygwin-bashrc
@@ -0,0 +1,332 @@
+# **********************************************************
+# Copyright (c) 2004-2007 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# only echo if interactive
+function iecho {
+    if [ "$PS1" ]; then # interactive
+        echo $*;
+    fi
+}
+
+iecho "sourcing .bashrc"
+
+# HOME should already be set up to, e.g., c:/derek, but cygwin
+# will change to /c/derek, so:
+export COLONHOME=`cygpath -m ${HOME}`
+
+# USERNAME should be set up, make sure it's also the username
+# on the cvs server
+export CVSSERVER=menlo
+
+export CYGSYSROOT=`cygpath -u $SYSTEMROOT`
+
+# Once-only setup things:
+# for each drive, follow example here with c:
+#   mkdir /c
+#   mount c: /c
+# ln -s /usr/bin/perl /usr/bin/perl
+
+if [ "$PS1" ]; then # interactive
+    # set primary prompt to put host in window title
+    PS1="\n\[\033]0;\h: \w\007\][\@] \u@\h \w\n% "
+fi
+
+alias l='ls -Fs'
+alias ll='ls -lg'
+alias m='less'
+alias all='ls -a'
+alias mv='mv -i'
+alias cp='cp -i'
+alias which='type'
+alias grep='grep -d skip'
+alias updatebm='pushd ..; make relink; popd; cp -f ../build-opt/*.exe .'
+alias tarbak='tar czf bak.tgz --exclude src/lib/papi/libpapi.a'
+alias ssh='ssh -x'
+alias dclick='~/dclick.exe'
+alias mapfile='/c/Program\ Files/MapFileExplorer/MapFileExplorer.exe'
+
+export DISPLAY='localhost:0.0'
+export MAKE_MODE=UNIX
+
+export PATH="/usr/local/bin:/usr/bin:/bin:/usr/X11R6/bin:${CYGSYSROOT}/system32:${CYGSYSROOT}:/c/program files/emacs-21.3/bin:${HOME}/dr/tools"
+
+export CVS_RSH=ssh
+export CVSROOT=:ext:${USERNAME}@${CVSSERVER}:/mnt/data/cvseast
+export EDITOR=/usr/bin/emacs
+
+# get around problem of not having a common ancestor running under ssh-agent
+# by sharing one ssh-agent
+export SSH_ENV_VARS="${HOME}/.ssh/env_vars"
+function ssh-setup {
+    if [ ${SSH_AGENT_PID} ]; then
+        echo "export SSH_AGENT_PID=${SSH_AGENT_PID}" > ${SSH_ENV_VARS}
+        echo "export SSH_AUTH_SOCK=${SSH_AUTH_SOCK}" >> ${SSH_ENV_VARS}
+        # add a little security
+        chmod go-rwx ${SSH_ENV_VARS}
+        echo "source ~/.ssh/env_vars in any other shell to use this agent";
+        # clean up on exit so no stale env vars file
+        trap "rm ${SSH_ENV_VARS}" EXIT
+        # can be re-tried, so ssh-add success independent of env var file
+        ssh-add
+    else
+        echo "error: no ssh-agent ancestor is running"
+    fi
+}
+if [ -f ${SSH_ENV_VARS} ]; then
+    source ${SSH_ENV_VARS}
+    iecho "ssh agent is ${SSH_AGENT_PID}"
+fi
+
+###########################################################################
+# DYNAMORIO
+
+export DYNAMORIO_TOOLS=${COLONHOME}/dr/tools
+export DYNAMORIO_SHARE=${COLONHOME}/dr/share
+export DYNAMORIO_BENCHMARKS=${COLONHOME}/dr/benchmarks
+export DYNAMORIO_TREEROOT=${COLONHOME}/dr/trees
+export DYNAMORIO_HOME=${DYNAMORIO_TREEROOT}/tot
+export DYNAMORIO_LOGDIR=${COLONHOME}/dr/logs
+export DYNAMORIO_OPTIONS=''
+
+alias dhome='echo $DYNAMORIO_HOME'
+alias dops='echo $DYNAMORIO_OPTIONS'
+function usetree {
+    export DYNAMORIO_HOME=$DYNAMORIO_TREEROOT/$1;
+    iecho "DYNAMORIO_HOME=$DYNAMORIO_HOME"; 
+}
+function useops {
+    export DYNAMORIO_OPTIONS="$*";
+    iecho "DYNAMORIO_OPTIONS=$DYNAMORIO_OPTIONS"; 
+}
+
+function synchin { 
+    mkdir -p $DYNAMORIO_TREEROOT/$1/src/;
+    rsync -auzv ${USERNAME}@${CVSSERVER}:~/dr/trees/$1/src/ $DYNAMORIO_TREEROOT/$1/src/;
+    cdt $1;
+}
+function synchout {
+    cdt $1;
+    rsync -auzv . ${USERNAME}@${CVSSERVER}:~/dr/trees/$1/src/; 
+}
+
+function runreg {
+    synchin $1;
+    cd ${HOME}/dr/suite;
+    usetree $1;
+    ./runregression ${DYNAMORIO_BENCHMARKS} $2;
+}
+
+function cdt { cd $DYNAMORIO_TREEROOT/$1/src; }
+function cdb { cd ${DYNAMORIO_BENCHMARKS}/spec2000/*/$1; }
+function cdlog { cd `ls -1dt ${DYNAMORIO_LOGDIR}/*[0-9] | head -1`; }
+function dtail {
+    echo `ls -1dt ${DYNAMORIO_LOGDIR}/*[0-9] | head -1`;
+    tail -$1 `ls -1dt ${DYNAMORIO_LOGDIR}/*[0-9] | head -1`/*[Ee][Xx][Ee]*; 
+}
+
+function diag2line { 
+    dhome; 
+    grep -E 'Call stack|frame ptr|Unrecoverable' $1 | grep -B 50 'the application' | grep -v 'Call stack' | awk '{print $NF}' | xargs address_query.pl $DYNAMORIO_HOME/exports/x86_win32_dbg/dynamorio.dll; 
+}
+function diag2linelast { diag2line `ls -t1 ${DYNAMORIO_LOGDIR}/*.html | head -1`; }
+
+function cvslatest {
+    one=`cvs log $1 | grep '^revision' | head -1 | awk '{print $2}'`;
+    two=`cvs log $1 | grep '^revision' | head -2 | tail -1 | awk '{print $2}'`;
+    cvs diff -r $two -r $one $1; 
+}
+function cvslatestr {
+    one=`cvs log -r$1 $2 | grep '^revision' | head -1 | awk '{print $2}'`;
+    two=`cvs log -r$1 $2 | grep '^revision' | head -2 | tail -1 | awk '{print $2}'`;
+    cvs diff -r $two -r $one $2; 
+}
+
+function ldump { ldmp $1 `cygpath -w "${DYNAMORIO_TOOLS}/dummy.exe"`; }
+
+alias injector='echo "DYNAMORIO_HOME=$DYNAMORIO_HOME"; time $DYNAMORIO_HOME/exports/x86_win32_dbg/drinject.exe ${DYNAMORIO_HOME//\//\\\\}\\exports\\x86_win32_dbg\\dynamorio.dll'
+alias relinjector='echo "DYNAMORIO_HOME=$DYNAMORIO_HOME"; time $DYNAMORIO_HOME/exports/x86_win32_rel/drinject.exe ${DYNAMORIO_HOME//\//\\\\}\\exports\\x86_win32_rel\\dynamorio.dll'
+
+# match linux aliases
+alias rio=relinjector
+alias drio=injector
+
+alias meminjector='echo "DYNAMORIO_HOME=$DYNAMORIO_HOME"; time $DYNAMORIO_HOME/exports/x86_win32_dbg/drinject.exe -mem ${DYNAMORIO_HOME//\//\\\\}\\exports\\x86_win32_dbg\\dynamorio.dll'
+alias memrelinjector='echo "DYNAMORIO_HOME=$DYNAMORIO_HOME"; time $DYNAMORIO_HOME/exports/x86_win32_rel/drinject.exe -mem ${DYNAMORIO_HOME//\//\\\\}\\exports\\x86_win32_rel\\dynamorio.dll'
+
+alias straceops='usetree strace; useops -stats -loglevel 2 -logmask 0x1201f -stracelibname c:\\derek\\dr\\trees\\strace\\exports\\x86_win32_dbg\\stracedr.dll'
+
+alias drsynch='\cp $DYNAMORIO_HOME/exports/x86_win32_dbg/dynamorio.* /c/Progra~1/Determina/SecureCore/lib/debug; chmod ogu+rwx /c/Progra~1/Determina/SecureCore/lib/debug/dynamorio.dll'
+alias drrelsynch='\cp $DYNAMORIO_HOME/exports/x86_win32_rel/dynamorio.* /c/Progra~1/Determina/SecureCore/lib/release; chmod ogu+rwx /c/Progra~1/Determina/SecureCore/lib/release/dynamorio.dll'
+
+function setkey {
+    if [ -z $2 ]; then
+        debug="dbg";
+    else
+        debug=$2;
+    fi
+    # stop agent and set to not restart on reboot
+    if ( svccntrl scnodemgr -show | grep -iq node ) then 
+        net stop scnodemgr;
+        svccntrl scnodemgr -manual;
+    fi
+    drcontrol -app $1 -drlib `cygpath -w "${DYNAMORIO_HOME}/exports/x86_win32_${debug}/dynamorio.dll"`;
+    drcontrol -app $1 -logdir `cygpath -w ${DYNAMORIO_LOGDIR}`;
+    drcontrol -app $1 -options "${DYNAMORIO_OPTIONS}";
+    # avoid double entries -- assume not testing compat w/ AppInit
+    drcontrol -preinject CLEAR
+    # "drcontrol -preinject ON" uses DYNAMORIO_HOME in SecureCore reg key, 
+    # which is sometimes messed up from regression suite runs:
+    PRELIB=$DYNAMORIO_HOME/lib/drpreinject.dll
+    if test -e $PRELIB ; then
+        drcontrol -preinject `cygpath -w $PRELIB`
+    else
+        drcontrol -preinject ON
+    fi
+    echo "using preinject: "`drcontrol -preinject LIST`
+}
+
+function resetreg {
+    if [ -z $1 ]; then
+        debug="dbg";
+    else
+        debug=$1;
+    fi
+    # stop agent and set to not restart on reboot
+    if ( svccntrl scnodemgr -show | grep -iq node ) then 
+        net stop scnodemgr;
+        svccntrl scnodemgr -manual;
+    fi
+    # may as well set global key though we're setting every app key below
+    drcontrol -drlib `cygpath -w "${DYNAMORIO_HOME}/exports/x86_win32_${debug}/dynamorio.dll"`;
+    drcontrol -logdir `cygpath -w ${DYNAMORIO_LOGDIR}`;
+    drcontrol -options "${DYNAMORIO_OPTIONS}";
+    # drcontrol -load not implemented, and no way to remove a string value,
+    # so our only options are to reproduce the slist parsing and get the right
+    # rununder values, or set the values we want for each app, which is a pain
+    # to edit manually later but oh well:
+    for app in `drcontrol -dump | grep Group | awk '{print $NF}'`; do
+        drcontrol -app $app -drlib `cygpath -w "${DYNAMORIO_HOME}/exports/x86_win32_${debug}/dynamorio.dll"`;
+        drcontrol -app $app -logdir `cygpath -w ${DYNAMORIO_LOGDIR}`;
+        drcontrol -app $app -options "${DYNAMORIO_OPTIONS}";
+    done;
+    ### if ignore RUNUNDER this will re-make reg keys from scc file:
+    ###    # now re-build reg setup
+    ###    drcontrol -reset;
+    ###    for i in `grep -i exe /c/Program\ Files/Determina/SecureCore/config/default.scc`; do 
+    ###        # pull exe name out of [HKEY_LOCAL_MACHINE\<...>\inetinfo.exe]
+    ###        j=${i##*\\}; 
+    ###        drcontrol -add ${j/]/}; 
+    ###    done;
+    # avoid double entries -- assume not testing compat w/ AppInit
+    drcontrol -preinject CLEAR
+    # "drcontrol -preinject ON" uses DYNAMORIO_HOME in SecureCore reg key, 
+    # which is sometimes messed up from regression suite runs:
+    PRELIB=$DYNAMORIO_HOME/lib/drpreinject.dll
+    if test -e $PRELIB ; then
+        drcontrol -preinject `cygpath -w $PRELIB`
+    else
+        drcontrol -preinject ON
+    fi
+    echo "using preinject: "`drcontrol -preinject LIST`
+}
+
+###########################################################################
+## VC++ Environment Variables
+## From C:\Program Files\Microsoft Visual Studio\VC98\Bin\VCVARS32.BAT
+##
+VSBaseDirTemp=`cygpath -ms /c/Program\ Files/Microsoft\ Visual\ Studio`
+VSBaseDirBASH=`cygpath -u $VSBaseDirTemp`
+
+# Root of Visual Developer Studio Common files.
+VSCommonDirBASH="${VSBaseDirBASH}/Common"
+VSCommonDir=`cygpath -d ${VSCommonDirBASH}`
+
+# Root of Visual Developer Studio installed files.
+MSDevDirBASH="${VSCommonDirBASH}/MSDev98"
+MSDevDir=`cygpath -d ${MSDevDirBASH}`
+
+# Root of Visual C++ installed files.
+MSVCDirBASH="${VSBaseDirBASH}/VC98"
+MSVCDir=`cygpath -d ${MSVCDirBASH}`
+
+# VcOsDir is used to help create either a Windows 95 or Windows NT specific path.
+if [ "$OS" = "Windows_NT" ]; then export VcOsDir=WINNT; else export VcOsDir=WIN95; fi
+
+# Set environment for using Microsoft Visual C++ tools.
+if [ "$OS" = "Windows_NT" ]; then PATH="$PATH:$MSDevDirBASH/BIN:$MSVCDirBASH/BIN:$VSCommonDirBASH/TOOLS/$VcOsDir:$VSCommonDirBASH/TOOLS"; fi
+if [ "$OS" = "" ]; then PATH="$PATH:$MSDevDirBASH/BIN:$MSVCDirBASH/BIN:$VSCommonDirBASH/TOOLS/$VcOsDir:$VSCommonDirBASH/TOOLS:$windir/SYSTEM"; fi
+export INCLUDE="$MSVCDir\\ATL\\INCLUDE;$MSVCDir\\INCLUDE;$MSVCDir\\MFC\\INCLUDE;$INCLUDE"
+export LIB="$MSVCDir\\LIB;$MSVCDir\\MFC\\LIB;$LIB"
+
+unset VSBaseDirTemp
+unset VSBaseDirBASH
+unset VSCommonDirBASH
+unset VSCommonDir
+unset MSDevDirBASH
+unset MSDevDir
+unset MSVCDirBASH
+unset MSVCDir
+#
+###########################################################################
+
+###########################################################################
+# quick launching of windows apps:
+#
+alias notepad='injector ${SYSTEMROOT}\\system32\\notepad.exe'
+alias calc='injector ${SYSTEMROOT}\\system32\\calc.exe'
+alias powerpoint='injector c:\\Program\ Files\\Microsoft\ Office\\Office\\POWERPNT.EXE'
+alias excel='injector c:\\Program\ Files\\Microsoft\ Office\\Office\\EXCEL.EXE'
+alias access='injector c:\\Program\ Files\\Microsoft\ Office\\Office\\MSACCESS.EXE'
+alias frontpage='injector c:\\Program\ Files\\Microsoft\ Office\\Office\\FRONTPG.EXE'
+alias word='injector c:\\Program\ Files\\Microsoft\ Office\\Office\\WINWORD.EXE'
+alias photoshop='injector C:\\Program\ Files\\Adobe\\Photoshop\ 6.0\\Photoshp.exe'
+alias illustrator='injector C:\\Program\ Files\\Adobe\\Illustrator\ 9.0.1\\Illustrator.exe'
+alias mediaplayer='injector C:\\Program\ Files\\Windows\ Media\ Player\\mplayer2.exe'
+alias photoed='injector c:\\Program\ Files\\Common\ Files\\Microsoft\ Shared\\PhotoEd\\PhotoEd.exe'
+alias netscape='injector C:\\Program\ Files\\Netscape\\Communicator\\Program\\netscape.exe -browser'
+alias acroread='injector C:\\Program\ Files\\Adobe\\Acrobat\ 4.0\\Reader\\AcroRd32.exe'
+alias iexplore='injector C:\\Program\ Files\\Internet\ Explorer\\IEXPLORE.EXE'
+alias explorer='injector ${SYSTEMROOT}\\explorer.exe'
+alias paint='injector ${SYSTEMROOT}\\System32\\mspaint.exe'
+alias wordpad='injector C:\\Program\ Files\\Windows\ NT\\Accessories\\wordpad.exe'
+alias freecell='injector ${SYSTEMROOT}\\System32\\freecell.exe'
+alias solitaire='injector ${SYSTEMROOT}\\System32\\sol.exe'
+alias minesweeper='injector ${SYSTEMROOT}\\System32\\winmine.exe'
+alias pinball='injector C:\\Program\ Files\\Windows\ NT\\Pinball\\PINBALL.EXE'
+alias premiere='injector C:\\Program\ Files\\Adobe\\Premiere\ 6.0\\premiere.exe'
+alias vstudio='injector C:\\Program\ Files\\Microsoft\ Visual\ Studio\\Common\\MSDev98\\Bin\\MSDEV.EXE'
+alias eventvwr='injector ${SYSTEMROOT}\\System32\\eventvwr.exe'
+alias mozilla='injector C:\\Program\ Files\\mozilla.org\\Mozilla\\mozilla.exe'
+alias defrag='injector ${SYSTEMROOT}\\System32\\mmc.exe ${SYSTEMROOT}\\System32\\dfrg.msc'
+alias oocalc='injector C:\\Program\ Files\\OpenOffice.org1.0.3\\program\\ooocalc.exe'
+#
+###########################################################################
+
+cd ${HOME}
diff --git a/tools/dclick.cpp b/tools/dclick.cpp
new file mode 100644
index 000000000..5435924cc
--- /dev/null
+++ b/tools/dclick.cpp
@@ -0,0 +1,99 @@
+/* **********************************************************
+ * Copyright (c) 2004 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* dclick.cpp
+ * by Derek Bruening
+ * May 2002
+ *
+ * equivalent of DOS shell "start" command: launches a file using the
+ * shell as though it were double-clicked in explorer
+ *
+ * to build, must link in shell32.lib:
+ * cl /nologo dclick.cpp shell32.lib
+ */
+
+#include <windows.h>
+#include <tchar.h>
+#include <stdio.h>
+#include <assert.h>
+
+int main(int argc, char *argv[])
+{
+    if (argc < 2) {
+        fprintf(stderr, "Usage: %s <filename> <args...>\n", argv[0]);
+        return 1;
+    }
+
+    TCHAR cwd[MAX_PATH];
+    int cwd_res = GetCurrentDirectory(MAX_PATH, cwd);
+    assert(cwd_res > 0);
+
+    TCHAR *params = (TCHAR *) malloc(sizeof(TCHAR) * (argc-1) * MAX_PATH);
+    TCHAR *cur = params;
+    int i;
+    int len, left = (argc-1) * MAX_PATH;
+    for (i = 2; i < argc; i++) {
+        len = _sntprintf(cur, left, "%s ", argv[i]);
+        if (len < 0) {
+            /* hit max */
+            cur += left - 1;
+            fprintf(stderr, "Warning: hit parameter buffer limit\n");
+            break;
+        }
+        left -= len;
+        cur += len;
+    }
+    *cur = _T('\0');
+
+    fprintf(stderr, "Opening \"%s\" with parameters \"%s\"\n", argv[1], params);
+    // tell explorer to "open" the file
+    HINSTANCE res = ShellExecute(NULL, _T("open"), 
+                                 argv[1], (LPCTSTR) params, cwd, SW_SHOWNORMAL);
+
+    free(params);
+    if ((int)res <= 32) {
+        int errnum = GetLastError();
+        LPVOID lpMsgBuf;
+        FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | 
+                      FORMAT_MESSAGE_FROM_SYSTEM | 
+                      FORMAT_MESSAGE_IGNORE_INSERTS,
+                      NULL,  errnum,
+                      MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language
+                      (LPTSTR) &lpMsgBuf, 0, NULL);
+        // Display the string.
+        fprintf(stderr, "Error opening \"%s\":\n\t%s\n", argv[1], lpMsgBuf);
+        // Free the buffer.
+        LocalFree(lpMsgBuf);
+        return 1;
+    }
+    return 0;
+}
diff --git a/tools/desktop-scripts/cpu_idle.pl b/tools/desktop-scripts/cpu_idle.pl
new file mode 100644
index 000000000..ce9fce69e
--- /dev/null
+++ b/tools/desktop-scripts/cpu_idle.pl
@@ -0,0 +1,103 @@
+# **********************************************************
+# Copyright (c) 2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# cpu_idle.pl
+#
+# NOTE: ASSUMES drview is in path
+
+use strict;
+
+my $PROG = "cpu_idle.pl";
+
+my @drview_out = `drview -listall -showmem`;
+
+my $cpu_idx = -1;
+my $wss_idx = -1;
+
+($cpu_idx, $wss_idx) = get_idx(@drview_out);
+
+my @data;
+my $total_cpu = 0;
+my $total_wss = 0;
+my $parse = 0;
+
+foreach $line (@drview_out) {
+    $_ = $line;
+    # skip lines till we see memory data
+    if (m/PID.*Bld.*CPU/) {
+        $parse = 1;
+        next; 
+    }
+
+    # skip cpu time spent running drview, shells etc.
+    # (add others as needed)
+    next if m/(DRview|cmd|bash|perl)/i;
+
+    if ($parse) {
+        @data = split(/\s+/, $line);
+        $data[$cpu_idx] =~ s/\%//;
+        $total_cpu += $data[$cpu_idx] ;
+    }
+}
+
+print "$PROG: cpu almost idle ($total_cpu% total CPU utilization)\n" if ($total_cpu < 3);
+
+exit ($total_cpu < 3);
+
+#------------------------------------------------------------------------------
+# To keep it a little more robust, find index for CPU, WSS etc.
+#
+sub get_idx() {
+    my $line;
+    my @titles;
+    # get the title line
+    foreach $line (@drview_out) {
+        $_ = $line;
+        if (m/PID.*CPU.*WSS/i) {
+            @titles = split(/\s+/, $line);
+            last;
+        }
+    }
+
+    die "CPU column needed.." unless defined @titles;
+
+    # little robust, get the index CPU, so this will work 
+    # even if showmem columns change (overboard?)
+    my $idx = 0;
+    my $cpu_idx = -1;
+    my $wss_idx = -1;
+    foreach $_ (@titles) {
+        $cpu_idx = $idx if (m/CPU/i);
+        $wss_idx = $idx if (m/WSS/i);
+        $idx++;
+    }
+
+    return ($cpu_idx, $wss_idx);
+}
diff --git a/tools/desktop-scripts/desktop_nightly.bat b/tools/desktop-scripts/desktop_nightly.bat
new file mode 100755
index 000000000..150a8c2b7
--- /dev/null
+++ b/tools/desktop-scripts/desktop_nightly.bat
@@ -0,0 +1,37 @@
+REM **********************************************************
+REM Copyright (c) 2006 VMware, Inc.  All rights reserved.
+REM **********************************************************
+
+REM Redistribution and use in source and binary forms, with or without
+REM modification, are permitted provided that the following conditions are met:
+REM 
+REM * Redistributions of source code must retain the above copyright notice,
+REM   this list of conditions and the following disclaimer.
+REM 
+REM * Redistributions in binary form must reproduce the above copyright notice,
+REM   this list of conditions and the following disclaimer in the documentation
+REM   and/or other materials provided with the distribution.
+REM 
+REM * Neither the name of VMware, Inc. nor the names of its contributors may be
+REM   used to endorse or promote products derived from this software without
+REM   specific prior written permission.
+REM 
+REM THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+REM AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+REM IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+REM ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+REM FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+REM DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+REM SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+REM CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+REM LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+REM OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+REM DAMAGE.
+
+rem reboot, measure memory but dont send email
+c:\perl\bin\perl c:\desktop-nightly\boot_data_scripts\desktop_nightly.pl
+
+@echo off
+rem reboot, measure memory and send email
+rem c:\perl\bin\perl c:\desktop-nightly\boot_data_scripts\desktop_nightly.pl -email
+
diff --git a/tools/desktop-scripts/desktop_nightly.pl b/tools/desktop-scripts/desktop_nightly.pl
new file mode 100644
index 000000000..031d0458e
--- /dev/null
+++ b/tools/desktop-scripts/desktop_nightly.pl
@@ -0,0 +1,127 @@
+# **********************************************************
+# Copyright (c) 2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# desktop_nightly.pl
+#
+# NOTE: ASSUMES drview, drcontrol, svccntrl and perl are in path
+# NOTE: ASSUMES c:\desktop-nightly\boot_data_scripts and
+#               c:\desktop-nightly\boot_data exist  #TODO do a mkdir
+# NOTE: ASSUMES this script is in c:\desktop-nightly\boot_data_scripts #FIXME
+#
+# To run:
+#   1. Setup $NIGHTLY_DIR and $NIGHTLY_SUMMARY to your liking
+#   2. run the script: perl desktop_nightly.pl
+#      or 
+#      run desktop_nightly.bat
+#          [NOTE: put it in Programs->Startup to run at reboot/login]
+#
+
+use strict;
+use Getopt::Long;
+use Cwd;
+
+my $PROG = "desktop_nightly.pl"; # TODO: use cwd? basename etc.
+
+# TODO: if cpu utilization hits <= 2%
+#   write all sums to log file
+
+my $ROOT      = qq(C:\\desktop-nightly);
+my $SCRIPTDIR = qq($ROOT\\boot_data_scripts);
+my $OUTDIR    = qq($ROOT\\boot_data);
+
+# get the boot time before doing anything else
+my ($boot_time, @ignore) = grep s/uptime in seconds: //i, 
+                                `cscript -nologo $SCRIPTDIR\\uptime.vbs`;
+
+my %opt;
+&GetOptions(\%opt,
+    "email",  # send email when done
+    "help",
+    "usage",
+    "debug",
+);
+#TODO: print usage, verify options?
+
+#my $DETERMINA = qq("$ENV{'PROGRAMFILES'}\\Determina\\Determina Agent");
+
+my $DRVER = &get_build();
+# list $determina lib dir 30xxx builds
+#my @builds = `dir /b $DETERMINA\\lib\\302??r`;
+# get the last element from that list
+#chomp(my $DRVER  = @builds[-1]);
+
+print "$PROG: running $DRVER\n";
+print "$PROG: measuring memory until cpu utilization is <= 2%\n";
+
+# regular nightly dir
+#my $NIGHTLY_DIR = qq(\\\\10.1.5.85\\g_shares\\desktop-nightly\\results\\boot_data);
+
+# run it three times to see noise
+my $NIGHTLY_DIR = qq(\\\\10.1.5.85\\g_shares\\desktop-nightly\\results\\boot_data\\nightly3);
+
+#FIXME this works?
+#mkdir $NIGHTLY_DIR unless (-d $NIGHTLY_DIR)
+
+my $NIGHTLY_SUMMARY = qq(qatest_xp_nightly_summary.txt);
+
+#for boot time alone
+#my $MEASURE_MEM_CMD = qq($SCRIPTDIR\\measure_mem.pl --times 1 --name "$DRVER-def-opt" --nightly $NIGHTLY_DIR --summary $NIGHTLY_SUMMARY --b $boot_time);
+
+my $MEASURE_MEM_CMD = qq($SCRIPTDIR\\measure_mem.pl --tillidle --name "$DRVER-def-opt" --nightly $NIGHTLY_DIR --summary $NIGHTLY_SUMMARY --b $boot_time);
+
+print "$PROG: $MEASURE_MEM_CMD\n";
+system("perl $MEASURE_MEM_CMD");
+
+if (defined $opt{'email'}) {
+    my $EMAIL_CMD = qq($SCRIPTDIR\\email_mem_summary.pl --summary "$NIGHTLY_DIR\\$NIGHTLY_SUMMARY");
+
+    print "$PROG: $EMAIL_CMD\n";
+    system("perl $EMAIL_CMD");
+}
+
+
+#--------------------------------------------------------------------------
+sub get_build() {
+    my @dump = `drcontrol -dump 2> $OUTDIR\\drcontrol_err.txt`;
+
+    # drcontrol -dump prints 
+    #   error message, if core is not installed
+    #   data for SC config, if disabled from DMC
+    # if we have any app instrumented, then we care about under DR
+    my ($ignore_sc, $build, @ignore_rest) = 
+        grep s/DYNAMORIO_AUTOINJECT.*Agent.lib.//i, @dump;
+
+    chomp($build);
+
+    $build = "native" unless ($build);
+    $build =~ s/\s*([0-9]+[a-z]).dynamorio.dll/$1/i;
+
+    return $build;
+}
diff --git a/tools/desktop-scripts/email_mem_summary.pl b/tools/desktop-scripts/email_mem_summary.pl
new file mode 100644
index 000000000..d183a9fd8
--- /dev/null
+++ b/tools/desktop-scripts/email_mem_summary.pl
@@ -0,0 +1,100 @@
+# **********************************************************
+# Copyright (c) 2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# email_mem_summary.pl
+#
+# NOTE:  ASSUMES Net::smtp module exists
+# FIXME: Header string specifes hard coded path
+
+
+use strict;
+use Getopt::Long;
+use Net::SMTP;
+
+my $PROG = "email_mem_summary.pl";
+
+my %opt;
+&GetOptions(\%opt,
+    "summary=s",
+    "help",
+    "usage",
+    "debug",
+);
+
+my $summary_file = qq($opt{'summary'});
+
+my $ServerName = "DHOST002-63.DEX002.intermedia.net" ;
+
+open (FD,$summary_file) or die "Cannot open";
+my @summary = <FD>;
+close FD;
+
+my $msg;
+my $header =
+    "\n\n".
+    "Desktop memory usage summary: \n".
+    "    for lionfish default configuration (only core sevices under DR)\n".
+    "    on qatest (xp)\n".
+    "\n".
+    "Run data and summary file at:\n".
+    "\\\\10.1.5.85\\g_shares\\desktop-nightly\\results\\boot_data \n".
+    "\n\n".
+    $summary[0];  # header
+
+my $line;
+# reverse sorted order (latest first) of summary
+foreach $line (@summary) {
+    $msg = $line . $msg;
+}
+
+# Now prepend the header information
+$msg = $header . $msg;
+
+#print $msg;
+
+# Connect to the server
+my $smtp = Net::SMTP->new($ServerName);
+die "Couldn't connect to server" unless $smtp;
+
+my $MailFrom = "eng-nightly-results\@determina.com";
+my $MailTo = "eng-nightly-results\@determina.com";
+#my $MailTo = "srini\@determina.com";
+
+print "$PROG: sending email summary\n";
+$smtp->mail( $MailFrom );
+$smtp->to( $MailTo );
+$smtp->data();
+
+# Send the message
+$smtp->datasend("Subject: Memory summary: lionfish desktop configuration\n");
+$smtp->datasend("$msg\n\n");
+# Send the termination string
+$smtp->dataend();
+$smtp->quit() ;
diff --git a/tools/desktop-scripts/getmem5.pl b/tools/desktop-scripts/getmem5.pl
new file mode 100644
index 000000000..322a2aa75
--- /dev/null
+++ b/tools/desktop-scripts/getmem5.pl
@@ -0,0 +1,61 @@
+# **********************************************************
+# Copyright (c) 2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# perl script
+
+my $prog = "getmem5.pl";
+
+if  (($#ARGV+1)  < 1 ) {
+	print "Usage : $prog <path to dir boot_data>\n";
+	exit;
+}
+
+$ScriptPath=$ARGV[0];
+print "$prog: path=$ARGV[0]\n";  
+
+for (my $i = 5; $i < 20; $i +=5) {
+	print "$prog: sleeping-5-min..\n";
+	sleep(300);
+	write_to_summary("mem-after-$i-min\n");
+	system("perl.exe $ScriptPath\\getmeminfo_automate.pl $ScriptPath");
+}
+
+sub write_to_summary {
+    my $text = shift;
+
+    my $OutputPath = "Z:\\desktop-benchmarks\\results\\boot_data";
+    my $hostname = "xp_qatest";  #TODO: fix this
+    my $sumfile="Boot_Summary_$hostname.txt";
+    my $fullpath="$OutputPath\\$hostname\\$sumfile";
+
+    open(OUT,">>$fullpath") or die "cannot find $fullpath: $!";
+    print OUT "$text";
+    close OUT;
+}
diff --git a/tools/desktop-scripts/getmeminfo_automate.pl b/tools/desktop-scripts/getmeminfo_automate.pl
new file mode 100644
index 000000000..264baa535
--- /dev/null
+++ b/tools/desktop-scripts/getmeminfo_automate.pl
@@ -0,0 +1,238 @@
+# **********************************************************
+# Copyright (c) 2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# borrowed from Lina
+#
+use Date::EzDate;
+use Time::Local;
+
+my $build;
+my $output;
+my $hostname;
+my %Totals;
+my $titles;
+my $output_tag;
+my $TimeNow;
+my $LastBoot;
+my $TimeToBoot;
+my $ScriptPath;
+my $OutputPath;
+
+if (($#ARGV+1)  < 1 ) {
+    print " Usage : <path to dir boot_data>\n";
+    exit;
+}
+
+
+$ScriptPath=$ARGV[0];
+print "path=$ARGV[0]\n";  
+
+# first get boot time
+$TimeToBoot=&GetLastBootTime();
+
+$OutputPath = "Z:\\desktop-benchmarks\\results\\boot_data";
+&GetNetworkDrive();
+
+&GetBuild();
+&CollectData();
+
+#--------------------------------------------------------------------------
+
+sub GetNetworkDrive {
+    if (-e $OutputPath) {
+	# use it 
+	return;
+    }
+    else {
+	system("net use Z: \/del");
+	system("net use Z: \\\\10.1.5.85\\g_shares \"\" /USER:guest");
+    }
+}
+
+sub CollectData {} {
+	&GetDate();
+	&GetHost();
+	&go(); 
+	&PrintTotals();
+	system("del $ScriptPath\\$output");
+	system("del $ScriptPath\\build.out");
+	system("del $ScriptPath\\OS.out");
+	system("del $ScriptPath\\host.out");
+	system("del $ScriptPath\\lastboot.txt");
+}
+
+sub GetBuild() {
+	system("drview -listdr > $ScriptPath\\build.out");
+	open(IN,"$ScriptPath\\build.out") or die "Cannot find the build #\n";
+        while( $line=<IN> ) {				
+		if ($line =~ /found/i ) {
+		   $build="Native"; 
+		   print "NATIVE!!!!!!\n"; 
+		   return;
+		}
+
+		$line=~ s/\)//g;
+		my @build=split(' ',$line);
+		$build="DR_". $build[$#build];
+		last;
+	}
+}
+
+
+sub GetDate() {
+	my $mydate = Date::EzDate->new();
+	$mydate->set_format('myformat', '{monthnumbase1}{day of month}{yearshort}{hour}{min}{sec}');
+	$mytoday=$mydate->{'myformat'};
+	$output=$build . "_" . $mytoday . "\.txt";  ###$output="jnk";
+	$output_tag=$build . "_" . $mytoday;
+	print "your output file ==>> $output\n";
+}
+
+sub GetHost() {
+	system("hostname > $ScriptPath\\host.out");
+	system("ver > $ScriptPath\\OS.out"); #get the OS 
+	
+	open(IN,"$ScriptPath\\host.out") or die "Cannot find the build #\n";
+        $hostname=<IN>;
+        chomp;
+	s/\n//g;
+        #print "host22 = $hostname";
+	close IN;
+
+	open(IN,"$ScriptPath\\OS.out") or die "Cannot find the build #\n";
+        $line=<IN>;
+	while ($line=<IN>) {
+		print "line= $line";
+		if ($line =~ /microsoft/gi) {
+		        if ($line =~ /XP/gi) { $myos= "xp_"; last;}
+			elsif ($line =~ /2000/ ) {$myos = "2k_"; last}
+			else { $myos = "2k3_";last} 
+		}
+	}
+
+	
+	my $name= $myos . $hostname;
+	$hostname=$name;
+        print "host = $hostname";	
+
+	system("mkdir $OutputPath\\$hostname");
+}
+
+sub go() {
+	system("drview -listall -showmem > $ScriptPath\\$output");	
+	system("copy $ScriptPath\\$output $OutputPath\\$hostname");
+	close IN;
+}
+
+sub GetTotal(){
+	open(IN,"<$ScriptPath\\$output");
+	chomp(my $committed = <IN>);
+	$committed =~ s/.*KB.: //;
+	$line=<IN>; #dont need second line
+	$titles_tmp=<IN>;
+	$titles_tmp=~ s/\s+/,/g;
+	@titles=split(',',$titles_tmp);
+
+	while ($line=<IN>) {
+
+		$line=~ s/\s+/,/g;
+		#print "line1= $line \n";
+		$line=~ s/%//g;
+
+		#print "line= $line, count=$#titles\n";
+		@line2=split(',',$line); 		#print "line2= @line2\n";
+		for ($i=4; $i < ($#titles+1);$i++) {
+			$Totals{$titles[$i]}+=$line2[$i];
+		}
+	}
+	$Totals{"Committed"} = $committed;
+	close IN;
+}		
+
+sub PrintTotals() {		
+	$hostname=~s/\n//g;
+	my $sumfile="Boot_Summary_" . $hostname. "\.txt";
+	my $fullpath="$OutputPath\\$hostname\\$sumfile";
+        print "Summary= $sumfile\n";
+	my @arr;
+	format OUT=
+@>>>>>>> @>>>>>>> @>>>>>>> @>>>>>>> @>>>>>>> @>>>>>>> @>>>>>>> @>>>>>>> @>>>>>>> @>>>>>>> @>>>>>>> @>>>>>>> @>>>>>>> @>>>>>>> @>>>>>>> @>>>>>>> @>>>>>>> @>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
+	$Bld,$Totals{"CPU"},$Totals{"User"} ,$Totals{"Hndl"},$Totals{"Thr"},$Totals{"PVSz"},$Totals{"VSz"},$Totals{"PPriv"},$Totals{"Priv"}$Totals{"PWSS"},$Totals{"WSS"},$Totals{"Fault"},$Totals{"PPage"},$Totals{"Page"},$Totals{"PNonP"},$Totals{"NonP"},$Totals{"BootTime"},$Totals{"Committed"}
+.
+
+
+	if ( -e $fullpath) {		
+		open(OUT,">>$fullpath") or die "cannot open existing summary: $!\n";
+	}
+	else {  
+		print "Cannot find a summary file starting a new one..\n";		
+		open(OUT,">$fullpath") or die "cannot open a new summary: $!\n";
+		my $j=1;
+		$Bld="Build";
+		$Totals{"CPU"}="CPU";
+		$Totals{"User"}="User";
+		$Totals{"Hndl"}="Hndl";
+		$Totals{"Thr"}="Thr";
+		$Totals{"PVSz"}="PVSz";
+		$Totals{"VSz"}="VSz";
+		$Totals{"PPriv"}="PPriv";
+		$Totals{"Priv"}="Priv";
+		$Totals{"PWSS"}="PWSS";
+		$Totals{"WSS"}="WSS";
+		$Totals{"Fault"}="Fault";
+		$Totals{"PPage"}="PPage";
+		$Totals{"Page"}="Page";
+		$Totals{"PNonP"}="PNonP";
+		$Totals{"NonP"}="NonP";
+		$Totals{"BootTime"}="BootTime";
+		$Totals{"Committed"}="Commit Charge";
+		write(OUT);
+		
+	}
+	
+	&GetTotal();
+	$Bld=$build;
+	$Totals{"BootTime"}=$TimeToBoot;
+	write(OUT);
+}
+
+sub GetLastBootTime(){
+	system("cscript $ScriptPath\\uptime.vbs > $ScriptPath\\lastboot.txt");
+	open(IN,"<$ScriptPath\\lastboot.txt") or die "Cannot open sys time file:$!\n";
+	while ($line=<IN>) {
+		if ($line =~ /uptime in seconds/i) { 
+			print "line=$line \n";
+			@arr=split(':',$line); #ex:Uptime in seconds: 15764
+			close IN;
+			return($arr[1]);
+
+		}
+	}
+}
diff --git a/tools/desktop-scripts/getservices.pl b/tools/desktop-scripts/getservices.pl
new file mode 100644
index 000000000..dbe73e74b
--- /dev/null
+++ b/tools/desktop-scripts/getservices.pl
@@ -0,0 +1,362 @@
+# **********************************************************
+# Copyright (c) 2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+use Date::EzDate;
+use Time::Local;
+
+my $build;
+my $output;
+my $hostname;
+my %Totals;
+my $titles;
+my $output_tag;
+my $ScriptPath;
+my $sumfile;
+my $fullpath;
+my %Slist='';
+my %DrView;
+my %STAT; #holds the current exe status in DR or native?
+my $OutputPath;
+$ScriptPath=$ARGV[0];
+
+system("drcontrol -fulldump > $ScriptPath\\drcontrol_full.txt");
+ 
+
+&GetBuild();
+
+system("del \*\.parsed");
+if ($build  =~ /native/i) {
+	print "Running native, nothing to parse..\n";
+	exit;
+}
+
+#system("net use Z: \/del");
+#system("net use Z: \\\\10.1.5.85\\g_shares \"\" /USER:guest");
+$OutputPath = "Z:\\desktop-benchmarks\\results\\boot_data";
+(-e $OutputPath) or die "Can't find $OutputPath\n";
+
+
+&GetDate();
+&GetHost(); 
+&go(); 
+
+
+#this section added to identify which services are in DR which are not
+&CreateSlist();
+&ParseDrview();
+&Compare();
+
+#-------------------------------------------------------------------------
+
+my $AUTOINJECT=0;
+sub GetBuild() {
+	my $DR=0;
+	open IN,"< $ScriptPath\\drcontrol_full.txt";
+	while(<IN>) {
+	        $line=$_;
+		print "reading: $line";
+		if ($line =~ /AUTOINJECT/i ) {	#DYNAMORIO_AUTOINJECT=c:\builds\25093\debug\dynamorio.dll
+			print "FOUND Autoinject.........\n";
+			$line=~ s/\\+/}/g;
+			@tmp=split('}',$line);
+			for ($i=0; $i < $#tmp ; $i++) {
+				if ($tmp[$i] =~ /^\d\d\d\d./) {
+					$build2=$tmp[$i];
+					$build="DR_". $build2;
+					$AUTOINJECT=1;
+					$i=$#tmp; 
+				}
+			}
+
+	   	}
+		if ($line =~ /Config Group:/gi) {
+			print "Found a registry entry: $line\n";
+			if ($line =~ /\.exe/) {
+				$DR=1;
+				last;
+			}
+
+		}
+	 }
+	 if  ((!$DR) | (!$AUTOINJECT))  {
+		print "Running native........\n";
+		 system("$ScriptPath\\psservice.exe > $ScriptPath\\native.txt");
+		 $build="Native"; 
+	}
+	print "build=$build....\n";
+}
+
+
+sub GetDate() {
+	my $mydate = Date::EzDate->new();
+	$mydate->set_format('myformat', '{monthnumbase1}{day of month}{yearshort}{hour}{min}{sec}');
+	$mytoday=$mydate->{'myformat'};
+	$output=$build . "_" . $mytoday . "\.txt";
+	$output_tag=$build . "_" . $mytoday;
+	print "your output file ==>> $output\n";
+}
+
+sub GetHost() {
+	system("hostname > $ScriptPath\\host.out");
+	system("ver > $ScriptPath\\OS.out"); #get the OS 
+
+	open(IN,"$ScriptPath\\host.out") or die "Cannot find the build #\n";
+        $hostname=<IN>;
+        chomp;
+	s/\n//g;
+        #print "host22 = $hostname";
+	close IN;
+
+	open(IN,"$ScriptPath\\OS.out") or die "Cannot find the build #\n";
+        $line=<IN>;
+	while ($line=<IN>) {
+		print "line= $line";
+		if ($line =~ /microsoft/gi) {
+		        if ($line =~ /XP/gi) { $myos= "xp_"; last;}
+			elsif ($line =~ /2000/ ) {$myos = "2k_"; last}
+			else { $myos = "2k3_";last} 
+		}
+	}
+
+
+	my $name= $myos . $hostname;
+	$hostname=$name;
+        print "host = $hostname";
+
+	system("mkdir $OutputPath\\$hostname");
+}
+
+sub Parse(){ # parses the output of psservice.exe
+	my $PsOut=shift;
+	open (IN,"<$PsOut") or die "cannot open $PsOut: $!\n";
+	$temp=$PsOut . "\.parsed";
+	open (OUT,">$temp");
+
+	while ($line=<IN>) {
+   		#print "line= $line\n";
+   		if ($line =~ /SERVICE_NAME/) {
+        		@lines=split(':',$line);
+			chomp $lines[1];
+			print OUT $lines[1],"\t";
+    		}
+    		elsif($line =~ /STATE/) {
+     	   		@lines=split(':',$line);
+			chomp $lines[1];
+			print OUT "=";
+			print OUT $lines[1],"\n";
+    		}
+       
+	}
+}
+
+
+
+sub go() {
+
+	system("$ScriptPath\\psservice > $ScriptPath\\$build.txt");
+	&Parse("$ScriptPath\\native.txt");
+	&Parse("$ScriptPath\\$build.txt");
+	&ComparePs("$ScriptPath\\native.txt.parsed","$ScriptPath\\$build.txt.parsed");
+	#move *NOTFOUND* results
+	#system("del DR_psservice_%2");
+	#system("del DR_psservice_%2\.parsed");
+	#system("copy $ScriptPath\\$output \\\\10.1.5.11\\qa-results\\boot_data\\$hostname");
+}
+
+sub ComparePs(){
+	my $native=shift;
+	my $dr=shift;
+
+	open (IN,"<$native") or die "cannot open $native: $!\n";
+	open (DR,"<$dr") or die "cannot open $dr: $!\n";
+
+	#$temp= "$ScriptPath\\" . "compare" . "\.NOTFOUND" . $output_tag . ".txt";
+
+	$hostname=~s/\n//g;
+	$sumfile="Services_Summary_" . $hostname. "\.txt";
+	$fullpath="$OutputPath\\$hostname\\$sumfile"; 
+        print "Summary= $sumfile\n";
+
+	if ( -e $fullpath) {
+		open(OUT,">>$fullpath") or die "cannot open existing summary: $!\n";
+	}
+
+	else {
+		open(OUT,">$fullpath") or die "cannot create summary: $!\n";}
+
+	while ($line=<DR>) {
+   		 if ( $line =~ /run/i) {
+   			#print "line222=$line\n";
+    			@temp=split(' ',$line);
+    			$DR{$temp[0]} =4;
+     		}
+	}
+
+        print OUT "\n********************************************************************\n";
+	print OUT "\n------ Services stopped when booting in ";
+        print OUT "$output_tag";
+        print OUT "----------------\n";
+	print  "\n------ Services stopped when booting in ---------------\n";
+
+	while ($line=<IN>) {
+   		#print "line=$line\n";
+    		if ( $line =~ /run/i) {
+       			#print "line222=$line\n";
+       			@temp1=split(' ',$line); 
+       			if (! $DR{$temp1[0]} ) {
+          			print OUT "$temp1[0], ";
+    	  			print "$temp1[0] stopped when booting in DR\n" ;
+       			}
+    		}
+	}
+
+	print "\n------- Services stopped when booting in Native ---------------\n";
+	print OUT "\n------- Services stopped when booting in Native ---------------------------\n";
+	@temp1='';
+
+	while ($line=<OUT>) {
+  		 #print "line=$line\n";
+    		if ( $line =~ /run/i) {
+       			#print "line222=$line\n";
+       			@temp1=split(' ',$line); 
+       				if (! $Native{$temp1[0]} ) {
+    	  				print OUT "$temp1[0], ";
+	  				print "$temp1[0] stopped when booting in NATIVE\n";
+       				}
+    		}
+	}
+	close OUT;
+
+}
+
+sub Compare(){
+	my $val;
+	my $ShouldNotBeInDR;
+	my $Warning=0;
+	open(OUT,">>$fullpath") or die "cannot open existing summary: $!\n";
+
+	foreach $val (keys %STAT) {
+		if  (($STAT{$val} =~ /native/) && (exists($Slist{$val})) ) { 
+			#print "$val : $Slist{$val} native\n";
+			print OUT "Warning: $val.exe=> should be running in DR!!!\n";
+			print "Warning: $val.exe=> should be running in DR!!!\n";
+			$Warning=1;
+		}
+		if (($STAT{$val} =~ /sc/i) && (!exists($Slist{$val})) ) { 
+			#print "$val : $Slist{$val} native\n";
+			print OUT "Warning: $val.exe=> should NOT be running in DR!!!\n";
+			print "Warning: $val.exe=> should NOT be running in DR!!!\n";
+			$Warning=1;
+		}
+
+	}
+	#print the list that I am not sure it should be in DR and it is
+	#print "The following should not be in DR (it is ok if BITS) ?...\n";
+	#print "\t=>$ShouldNotBeInDR\n";
+
+	if (!$Warning) {
+		print OUT "\n\tAll services that should be in DR are running in DR\n";
+		print "\n\tAll services that should be in DR are running in DR\n";
+	}
+	print "Done -----------------------------------------------\n";
+	print "fullpath=$fullpath\nhostname=$hostname\n";
+	system("copy /Y $fullpath \"$OutputPath\\$hostname\" " );
+
+}
+
+
+sub CreateSlist() {
+	my @arr1;
+	my $line;
+	my $val;
+
+
+ 
+	open CONT,"<$ScriptPath\\drcontrol_full.txt" or die "cannot open drcontrol_full.txt :$!\n";
+	while ($line=<CONT>) {
+		if ($line =~ /Config Group/i) {
+			$line =~ s/\n//g;
+			$line =~ s/\s+//g;
+			@arr1=split(':',$line);
+
+			#print "service=$arr1[1]\n";
+			if ($arr1[1]=~ /-/) {
+				@tmpexe=split('-',$arr1[1]); #dllhost.exe-Processid02D4B3F1FD
+				$exe=$tmpexe[0];				 
+				$Slist{lc $exe}=1;
+			}
+			else {
+				$Slist{lc $arr1[1]}= 1;
+			}
+		}
+
+	}
+
+	#foreach $val (keys %Slist) {
+	#	print "$val=>>$Slist{$val}\n";
+	#}
+		 
+}
+
+sub ParseDrview(){ # set to 0 for native and 1 for DR
+	my @arr1;
+	my $Native=1;
+	my $process;
+	my $line;
+	my $val;
+
+	my @ServiceList;
+	my @ServiceList2;
+ 	system("drview -listall  > $ScriptPath\\drview_all.txt");
+	open VIEW,"<$ScriptPath\\drview_all.txt" or die "cannot open drview_all.txt :$!\n";
+	while ($line=<VIEW>) {
+		if ($line =~ /Process System/i) { next};
+		my $i=0;
+		if ($line =~ /PID/) {
+			$line=~ s/\s+/ /g;
+			$line=~ s/,//g;
+			#print " line111 = $line \n";
+			@ServiceList=split(' ',$line);
+			$EXE=$ServiceList[3];
+			$DRNative=$ServiceList[5];
+			$STAT{lc $EXE}=$DRNative;
+
+		}
+	}
+
+	#foreach $val (keys %STAT) {
+	#	print "|$val=>>$STAT{$val}\n";
+	#}
+	 
+}
+
+
+
+
diff --git a/tools/desktop-scripts/go.bat b/tools/desktop-scripts/go.bat
new file mode 100755
index 000000000..2e1e3c547
--- /dev/null
+++ b/tools/desktop-scripts/go.bat
@@ -0,0 +1,31 @@
+REM **********************************************************
+REM Copyright (c) 2006 VMware, Inc.  All rights reserved.
+REM **********************************************************
+
+REM Redistribution and use in source and binary forms, with or without
+REM modification, are permitted provided that the following conditions are met:
+REM 
+REM * Redistributions of source code must retain the above copyright notice,
+REM   this list of conditions and the following disclaimer.
+REM 
+REM * Redistributions in binary form must reproduce the above copyright notice,
+REM   this list of conditions and the following disclaimer in the documentation
+REM   and/or other materials provided with the distribution.
+REM 
+REM * Neither the name of VMware, Inc. nor the names of its contributors may be
+REM   used to endorse or promote products derived from this software without
+REM   specific prior written permission.
+REM 
+REM THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+REM AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+REM IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+REM ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+REM FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+REM DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+REM SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+REM CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+REM LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+REM OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+REM DAMAGE.
+
+FOR /L %%a IN (1,1,5) DO perl reboot_qatest.pl
diff --git a/tools/desktop-scripts/ie/README b/tools/desktop-scripts/ie/README
new file mode 100644
index 000000000..5539f1bbf
--- /dev/null
+++ b/tools/desktop-scripts/ie/README
@@ -0,0 +1,10 @@
+The following files and directories were removed due to concerns about
+redistributability:
+
+bbc_files/
+bbc.htm
+cnn_files/
+cnn.htm
+csstest.html
+Javascript Libraries_files/
+Javascript Libraries.htm
diff --git a/tools/desktop-scripts/ie/launch-ie.pl b/tools/desktop-scripts/ie/launch-ie.pl
new file mode 100644
index 000000000..7367cf470
--- /dev/null
+++ b/tools/desktop-scripts/ie/launch-ie.pl
@@ -0,0 +1,75 @@
+# **********************************************************
+# Copyright (c) 2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+
+# perl script
+
+# Usage: perl launch-ie.pl [-mp] [htmlpage]
+#
+# -mp creates multiple IE processes
+# otherwise creates one IE process with 10 windows
+# assumes c:\desktop-benchmarks\ie as directory for html files
+
+#
+# ASSUMES IE is in the path
+#
+
+# TODO: use get options, and print usage/help
+# FIXME: hardcodes cnn.htm
+#
+# usage: launch-ie [-mp] [10]
+#
+
+
+my $ie = "";
+my $prog = "launch-ie.pl";
+
+$_ = $ARGV[0];
+if (m/-mp/) {
+    $ie = "IEXPLORE.EXE";
+    print "$prog: Launching multiple IE processes\n";
+    shift;
+}
+
+my $NUM_IE = 10;
+
+$NUM_IE = $ARGV[0] if (defined $ARGV[0]);
+
+# TODO: get from command line/current dir
+my $htmldir = "c:\\desktop-benchmarks\\boot_data_scripts\\ie";
+#my $file = $ARGV[0];
+my $file;
+
+$file = "cnn.htm" unless defined $file;
+
+my $i = 0;
+for ($i = 0; $i < $NUM_IE; $i++) {
+    system("start $ie $htmldir\\$file");
+}
diff --git a/tools/desktop-scripts/ie/run.pl b/tools/desktop-scripts/ie/run.pl
new file mode 100644
index 000000000..ee7350d49
--- /dev/null
+++ b/tools/desktop-scripts/ie/run.pl
@@ -0,0 +1,86 @@
+# **********************************************************
+# Copyright (c) 2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+
+# perl script
+
+my $prog = "run.pl";
+
+if  (($#ARGV+1)  < 1 ) {
+	print "Usage : <path to dir boot_data>\n";
+	exit;
+}
+
+$ScriptPath=$ARGV[0];
+print "$prog: path=$ARGV[0]\n";  
+
+#system("net use Z: \/del");
+#system("net use Z: \\\\10.1.5.85\\g_shares \"\" /USER:guest");
+
+doit("");
+doit("-mp");
+
+#-------------------------------------------------------------------------
+sub doit {
+    my $mp = shift;
+    my $getmeminfo = "perl $ScriptPath\\getmeminfo_automate.pl $ScriptPath";
+    my $launch_ie = "perl $ScriptPath\\ie\\launch-ie.pl";
+    my $procs = "10-processes";
+    $_ = $mp;
+    $procs = "windows-1-process" unless (m/-mp/i);
+
+    for (my $i = 0; $i < 3; $i++) {
+	write_to_summary("before-launching-10-IE-$procs\n");
+	system("$getmeminfo");
+	system("$launch_ie $mp");
+	print "$prog: sleeping 120 secs..\n";
+	sleep(120);
+	write_to_summary("after-launching-10-IE-$procs\n");
+	system("$getmeminfo");
+	print "$prog: Killing IE windows\n";
+	system("drkill -exe iexplore.exe");
+	print "$prog: sleeping 120 secs..\n";
+	sleep(120);
+    }
+    write_to_summary("--------- end of test run ----------\n");
+}
+
+sub write_to_summary {
+    my $text = shift;
+
+    my $OutputPath = "Z:\\desktop-benchmarks\\results\\boot_data";
+    my $hostname = "xp_qatest";  #TODO: fix this
+    my $sumfile="Boot_Summary_$hostname.txt";
+    my $fullpath="$OutputPath\\$hostname\\$sumfile";
+
+    open(OUT,">>$fullpath") or die "$prog: Cannot find $fullpath: $!";
+    print OUT "$text";
+    close OUT;
+}
diff --git a/tools/desktop-scripts/install_cores.pl b/tools/desktop-scripts/install_cores.pl
new file mode 100644
index 000000000..525038997
--- /dev/null
+++ b/tools/desktop-scripts/install_cores.pl
@@ -0,0 +1,77 @@
+# **********************************************************
+# Copyright (c) 2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# install_cores.pl 
+#
+# Installs all cores since blowfish (Adapted from QA's scripts)
+#
+
+use strict;
+use Cwd 'chdir';
+
+system("net use n: \\\\10.1.5.15\\nightly \"\" /USER:guest") unless (-e "n:");
+my $NIGHTLY_DIR = qq(n:);
+my $dir;
+
+
+system("net stop tomcatsc");
+foreach $dir (`cmd /c dir /on /b $NIGHTLY_DIR\\302??`) {
+    chomp($dir);
+    next if ($dir < "30259");  # optionally skip ones already installed
+    &InstallNewCore($dir);
+    #print "installing $dir\n";
+}
+system("net start tomcatsc");
+
+#--------------------------------------------------
+sub InstallNewCore() {
+    my $build_dir = shift;
+
+    # lib path
+    my $lib_r_path = qq($NIGHTLY_DIR\\$build_dir\\ver_2_5\\lib\\release);
+
+    # install new core to controller
+    my $controller_path = qq("C:\\Program Files\\Determina\\Determina Management Console");
+
+    chdir("C:\\Program Files\\Determina\\Determina Management Console\\conf\\tool") or 
+       die "Couldn't change to conf dir: $!\n";
+
+    my $ver_r  = $build_dir . "r";
+    my $desc_r = $build_dir . "_release_build";
+
+    print "Installing $ver_r\n";
+    system("admintool -installCore braksha $ver_r $ver_r $desc_r $lib_r_path\\dynamorio.dll");
+
+    # make this new release core the default for all policies
+    #system("\"C:\\Program Files\\Determina\\Determina Management Console\\Support Tools\\MySQL\\bin\\mysql\" -D emv-1 -u root --password=braksha -e \"UPDATE systemconfig SET DefaultCoreVersion='$ver_r'\"");
+
+    # nightly core upgrade" use the new release build
+    #system("\"C:\\Program Files\\Determina\\Determina Management Console\\Support Tools\\MySQL\\bin\\mysql\" -D emv-1 -u root --password=braksha -e \"UPDATE policies SET CoreVersion='$ver_r' WHERE Name='qatest configuration'\"");
+}
diff --git a/tools/desktop-scripts/install_new_core b/tools/desktop-scripts/install_new_core
new file mode 100755
index 000000000..2d52bfa60
--- /dev/null
+++ b/tools/desktop-scripts/install_new_core
@@ -0,0 +1,30 @@
+# **********************************************************
+# Copyright (c) 2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
diff --git a/tools/desktop-scripts/install_new_core.bat b/tools/desktop-scripts/install_new_core.bat
new file mode 100755
index 000000000..700437040
--- /dev/null
+++ b/tools/desktop-scripts/install_new_core.bat
@@ -0,0 +1,32 @@
+REM **********************************************************
+REM Copyright (c) 2006 VMware, Inc.  All rights reserved.
+REM **********************************************************
+
+REM Redistribution and use in source and binary forms, with or without
+REM modification, are permitted provided that the following conditions are met:
+REM 
+REM * Redistributions of source code must retain the above copyright notice,
+REM   this list of conditions and the following disclaimer.
+REM 
+REM * Redistributions in binary form must reproduce the above copyright notice,
+REM   this list of conditions and the following disclaimer in the documentation
+REM   and/or other materials provided with the distribution.
+REM 
+REM * Neither the name of VMware, Inc. nor the names of its contributors may be
+REM   used to endorse or promote products derived from this software without
+REM   specific prior written permission.
+REM 
+REM THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+REM AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+REM IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+REM ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+REM FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+REM DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+REM SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+REM CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+REM LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+REM OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+REM DAMAGE.
+
+cd c:\work
+c:\perl\bin\perl install_new_core.pl
diff --git a/tools/desktop-scripts/install_new_core.pl b/tools/desktop-scripts/install_new_core.pl
new file mode 100644
index 000000000..68af7987f
--- /dev/null
+++ b/tools/desktop-scripts/install_new_core.pl
@@ -0,0 +1,113 @@
+# **********************************************************
+# Copyright (c) 2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+#
+# adapted from QA's scripts
+#
+#FIXME remove dependency on last_build_num.txt
+
+system("net use n: \\\\10.1.5.15\\nightly \"\" /USER:guest") unless (-e "n:");
+
+my $NIGHTLY_DIR = qq(n:);
+
+my $latest_build;
+my $last_build;
+
+
+&GetLatestBuildNum();
+&InstallNewCore();
+&UpdateLastBuildNum();
+
+
+
+sub GetLatestBuildNum() {
+
+# retrieve the latest build number on nightly
+    my @builds = `cmd /c dir /on /b $NIGHTLY_DIR\\302??`;
+    $latest_build = @builds[-1];
+    chomp($latest_build);
+
+# retrieve last build number on controller
+
+    open(IN,"C:\\work\\last_build_num.txt") or die "Cannot find the file last_build_num.txt\n";
+    $last_build = <IN>;
+    chomp($last_build);
+    close IN;
+
+# do we have a new build?
+
+    if ($latest_build <= $last_build) {  # no new build; do nothing
+       print "Build $latest_build is older or equal to the latest core available in the controller - Installcore will abort now\n";
+       exit;
+    }
+
+}
+
+
+sub InstallNewCore() {
+
+# copy the latest lib, skip debug and profile for now
+    #$lib_d_path = "\\\\10.1.5.15\\nightly\\$latest_build\\ver_2_5\\lib\\debug";
+    #$lib_p_path = "\\\\10.1.5.15\\nightly\\$latest_build\\ver_2_5\\lib\\profile";
+    
+    $lib_r_path = "\\\\10.1.5.15\\nightly\\$latest_build\\ver_2_5\\lib\\release";
+
+# install new core to controller
+     my $controller_path = "\"C:\\Program Files\\Determina\\Determina Management Console\"";
+
+    system("net stop tomcatsc");
+    chdir("C:\\Program Files\\Determina\\Determina Management Console\\conf\\tool");
+
+# skip debug and profile for now
+    #my $ver_d = $latest_build . "d";
+    #my $desc_d = $latest_build . "_debug_build";
+    #system("admintool -installCore braksha $ver_d $ver_d $desc_d $lib_d_path\\dynamorio.dll");
+
+    #my $ver_p = $latest_build . "p";
+    #my $desc_p = $latest_build . "_profile_build";
+    #system("admintool -installCore braksha $ver_p $ver_p $desc_p $lib_p_path\\dynamorio.dll");
+
+    my $ver_r = $latest_build . "r";
+    my $desc_r = $latest_build . "_release_build";
+    system("admintool -installCore braksha $ver_r $ver_r $desc_r $lib_r_path\\dynamorio.dll");
+
+    # make this new release core the default for all policies
+    # system("\"C:\\Program Files\\Determina\\Determina Management Console\\Support Tools\\MySQL\\bin\\mysql\" -D emv-1 -u root --password=braksha -e \"UPDATE systemconfig SET DefaultCoreVersion='$ver_r'\"");
+
+    # nightly core upgrade" use the new release build
+    system("\"C:\\Program Files\\Determina\\Determina Management Console\\Support Tools\\MySQL\\bin\\mysql\" -D emv-1 -u root --password=braksha -e \"UPDATE policies SET CoreVersion='$ver_r' WHERE Name='qatest configuration'\"");
+    system("net start tomcatsc");
+}
+
+sub UpdateLastBuildNum() {
+    open OUT,">C:\\work\\last_build_num.txt";
+    print OUT "$latest_build";
+    close OUT;
+}
diff --git a/tools/desktop-scripts/last_build_num.txt b/tools/desktop-scripts/last_build_num.txt
new file mode 100644
index 000000000..c09a36115
--- /dev/null
+++ b/tools/desktop-scripts/last_build_num.txt
@@ -0,0 +1 @@
+30252
\ No newline at end of file
diff --git a/tools/desktop-scripts/lionfish-desktop-services-all.bat b/tools/desktop-scripts/lionfish-desktop-services-all.bat
new file mode 100755
index 000000000..19a5a153b
--- /dev/null
+++ b/tools/desktop-scripts/lionfish-desktop-services-all.bat
@@ -0,0 +1,126 @@
+REM **********************************************************
+REM Copyright (c) 2006 VMware, Inc.  All rights reserved.
+REM **********************************************************
+
+REM Redistribution and use in source and binary forms, with or without
+REM modification, are permitted provided that the following conditions are met:
+REM 
+REM * Redistributions of source code must retain the above copyright notice,
+REM   this list of conditions and the following disclaimer.
+REM 
+REM * Redistributions in binary form must reproduce the above copyright notice,
+REM   this list of conditions and the following disclaimer in the documentation
+REM   and/or other materials provided with the distribution.
+REM 
+REM * Neither the name of VMware, Inc. nor the names of its contributors may be
+REM   used to endorse or promote products derived from this software without
+REM   specific prior written permission.
+REM 
+REM THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+REM AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+REM IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+REM ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+REM FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+REM DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+REM SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+REM CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+REM LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+REM OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+REM DAMAGE.
+
+rem "Set up machine for full load (all services listed in Lionfish Desktop"
+rem "configuration"
+
+rem "make specified services automatic"
+svccntrl "Alerter" -auto
+svccntrl "Application Layer Gateway Service" -auto
+svccntrl "Application Management" -auto
+svccntrl "Background Intelligent Transfer Service" -auto
+svccntrl "Client Service for NetWare" -auto
+svccntrl "Clipbook" -auto
+svccntrl "COM+ System Application" -auto
+svccntrl "Distributed Transaction Coordinator" -auto
+svccntrl "Fast User Switching Compatibility" -auto
+svccntrl "Fax" -auto
+svccntrl "HTTP SSL" -auto
+svccntrl "Human Interface Device Access" -auto
+svccntrl "IMAPI CD-Burning COM Service" -auto
+svccntrl "Indexing Service" -auto
+svccntrl "Infrared Monitor" -auto
+svccntrl "IPv6 Helper Service" -auto
+svccntrl "Logical Disk Manager Administrative Service" -auto
+svccntrl "Microsoft Software Shadow Copy Provider" -auto
+svccntrl "Net Logon" -auto
+svccntrl "NetMeeting Remote Desktop Sharing" -auto
+svccntrl "Network DDE" -auto
+svccntrl "Network DDE DSDM" -auto
+svccntrl "NT LM Security Support Provider" -auto
+svccntrl "Performance Logs and Alerts" -auto
+svccntrl "QoS RSVP" -auto
+svccntrl "Remote Access Auto Connection Manager" -auto
+svccntrl "Remote Desktop Help Session Manager" -auto
+svccntrl "Remote Procedure Call (RPC) Locator" -auto
+svccntrl "Removable Storage" -auto
+svccntrl "RIP Listener" -auto
+svccntrl "Routing and Remote Access" -auto
+svccntrl "SAP Agent" -auto
+svccntrl "Simple TCP/IP Services" -auto
+svccntrl "Smart Card" -auto
+svccntrl "TCP/IP Print Server" -auto
+svccntrl "Telnet" -auto
+svccntrl "Uninterruptible Power Supply" -auto
+svccntrl "Universal Plug and Play Device Host" -auto
+svccntrl "Volume Shadow Copy" -auto
+svccntrl "Windows Management Instrumentation Driver Extensions" -auto
+svccntrl "WMI Performance Adapter" -auto
+svccntrl "World Wide Web Publishing Service" -auto
+svccntrl "Portable Media Serial Number Service" -auto
+svccntrl "Network Provisioning" -auto
+
+
+svccntrl "Automatic Updates" -auto
+svccntrl "COM+ Event System" -auto
+svccntrl "Computer Browser" -auto
+svccntrl "Cryptographic Services" -auto
+svccntrl "DHCP Client" -auto
+svccntrl "Distributed Link Tracking Client" -auto
+svccntrl "DNS Client" -auto
+svccntrl "Error Reporting Service" -auto
+svccntrl "Event Log" -auto
+svccntrl "Help and Support" -auto
+svccntrl "IPSEC Services" -auto
+svccntrl "Logical Disk Manager" -auto
+svccntrl "Messenger" -auto
+svccntrl "Network Connections" -auto
+svccntrl "Network Location Awareness (NLA)" -auto
+svccntrl "Plug and Play" -auto
+svccntrl "Portable Media Serial Number" -auto
+svccntrl "Print Spooler" -auto
+svccntrl "Protected Storage" -auto
+svccntrl "Remote Access Connection Manager" -auto
+svccntrl "Remote Procedure Call (RPC)" -auto
+svccntrl "Remote Registry" -auto
+svccntrl "Secondary Logon" -auto
+svccntrl "Security Accounts Manager" -auto
+svccntrl "Server" -auto
+svccntrl "Shell Hardware Detection" -auto
+svccntrl "SSDP Discovery Service" -auto
+svccntrl "System Event Notification" -auto
+svccntrl "System Restore Service" -auto
+svccntrl "Task Scheduler" -auto
+svccntrl "TCP/IP NetBIOS Helper" -auto
+svccntrl "Telephony" -auto
+svccntrl "Terminal Services" -auto
+svccntrl "Themes" -auto
+svccntrl "Upload Manager" -auto
+svccntrl "WebClient" -auto
+svccntrl "Windows Audio" -auto
+svccntrl "Windows Image Acquisition (WIA)" -auto
+svccntrl "Windows Installer" -auto
+svccntrl "Windows Management Instrumentation" -auto
+svccntrl "Windows Time" -auto
+svccntrl "Wireless Configuration" -auto
+svccntrl "Workstation" -auto
+svccntrl "DCOM Server Process Launcher" -auto
+svccntrl "Windows Firewall/Internet Connection Sharing (ICS)" -auto
+svccntrl "Security Center" -auto
diff --git a/tools/desktop-scripts/lionfish-desktop-services-def.bat b/tools/desktop-scripts/lionfish-desktop-services-def.bat
new file mode 100755
index 000000000..56ca2fcb2
--- /dev/null
+++ b/tools/desktop-scripts/lionfish-desktop-services-def.bat
@@ -0,0 +1,127 @@
+REM **********************************************************
+REM Copyright (c) 2006 VMware, Inc.  All rights reserved.
+REM **********************************************************
+
+REM Redistribution and use in source and binary forms, with or without
+REM modification, are permitted provided that the following conditions are met:
+REM 
+REM * Redistributions of source code must retain the above copyright notice,
+REM   this list of conditions and the following disclaimer.
+REM 
+REM * Redistributions in binary form must reproduce the above copyright notice,
+REM   this list of conditions and the following disclaimer in the documentation
+REM   and/or other materials provided with the distribution.
+REM 
+REM * Neither the name of VMware, Inc. nor the names of its contributors may be
+REM   used to endorse or promote products derived from this software without
+REM   specific prior written permission.
+REM 
+REM THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+REM AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+REM IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+REM ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+REM FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+REM DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+REM SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+REM CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+REM LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+REM OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+REM DAMAGE.
+
+rem "Set up machine for Lionfish Desktop configuration as specifed by PM"
+
+rem "disabled following services"
+svccntrl "Alerter" -disabled
+svccntrl "Application Layer Gateway Service" -disabled
+svccntrl "Application Management" -disabled
+svccntrl "Background Intelligent Transfer Service" -disabled
+svccntrl "Client Service for NetWare" -disabled
+svccntrl "Clipbook" -disabled
+svccntrl "COM+ System Application" -disabled
+svccntrl "Distributed Transaction Coordinator" -disabled
+svccntrl "Fast User Switching Compatibility" -disabled
+svccntrl "Fax" -disabled
+svccntrl "HTTP SSL" -disabled
+svccntrl "Human Interface Device Access" -disabled
+svccntrl "IMAPI CD-Burning COM Service" -disabled
+svccntrl "Indexing Service" -disabled
+svccntrl "Infrared Monitor" -disabled
+svccntrl "IPv6 Helper Service" -disabled
+svccntrl "Logical Disk Manager Administrative Service" -disabled
+svccntrl "Microsoft Software Shadow Copy Provider" -disabled
+svccntrl "Net Logon" -disabled
+svccntrl "NetMeeting Remote Desktop Sharing" -disabled
+svccntrl "Network DDE" -disabled
+svccntrl "Network DDE DSDM" -disabled
+svccntrl "NT LM Security Support Provider" -disabled
+svccntrl "Performance Logs and Alerts" -disabled
+svccntrl "QoS RSVP" -disabled
+svccntrl "Remote Access Auto Connection Manager" -disabled
+svccntrl "Remote Desktop Help Session Manager" -disabled
+svccntrl "Remote Procedure Call (RPC) Locator" -disabled
+svccntrl "Removable Storage" -disabled
+svccntrl "RIP Listener" -disabled
+svccntrl "Routing and Remote Access" -disabled
+svccntrl "SAP Agent" -disabled
+svccntrl "Simple TCP/IP Services" -disabled
+svccntrl "Smart Card" -disabled
+svccntrl "TCP/IP Print Server" -disabled
+svccntrl "Telnet" -disabled
+svccntrl "Uninterruptible Power Supply" -disabled
+svccntrl "Universal Plug and Play Device Host" -disabled
+svccntrl "Volume Shadow Copy" -disabled
+svccntrl "Windows Management Instrumentation Driver Extensions" -disabled
+svccntrl "WMI Performance Adapter" -disabled
+svccntrl "World Wide Web Publishing Service" -disabled
+svccntrl "Portable Media Serial Number Service" -disabled
+svccntrl "Network Provisioning" -disabled
+
+
+rem "make services in defult configuration automatic"
+
+svccntrl "Automatic Updates" -auto
+svccntrl "COM+ Event System" -auto
+svccntrl "Computer Browser" -auto
+svccntrl "Cryptographic Services" -auto
+svccntrl "DHCP Client" -auto
+svccntrl "Distributed Link Tracking Client" -auto
+svccntrl "DNS Client" -auto
+svccntrl "Error Reporting Service" -auto
+svccntrl "Event Log" -auto
+svccntrl "Help and Support" -auto
+svccntrl "IPSEC Services" -auto
+svccntrl "Logical Disk Manager" -auto
+svccntrl "Messenger" -auto
+svccntrl "Network Connections" -auto
+svccntrl "Network Location Awareness (NLA)" -auto
+svccntrl "Plug and Play" -auto
+svccntrl "Portable Media Serial Number" -auto
+svccntrl "Print Spooler" -auto
+svccntrl "Protected Storage" -auto
+svccntrl "Remote Access Connection Manager" -auto
+svccntrl "Remote Procedure Call (RPC)" -auto
+svccntrl "Remote Registry" -auto
+svccntrl "Secondary Logon" -auto
+svccntrl "Security Accounts Manager" -auto
+svccntrl "Server" -auto
+svccntrl "Shell Hardware Detection" -auto
+svccntrl "SSDP Discovery Service" -auto
+svccntrl "System Event Notification" -auto
+svccntrl "System Restore Service" -auto
+svccntrl "Task Scheduler" -auto
+svccntrl "TCP/IP NetBIOS Helper" -auto
+svccntrl "Telephony" -auto
+svccntrl "Terminal Services" -auto
+svccntrl "Themes" -auto
+svccntrl "Upload Manager" -auto
+svccntrl "WebClient" -auto
+svccntrl "Windows Audio" -auto
+svccntrl "Windows Image Acquisition (WIA)" -auto
+svccntrl "Windows Installer" -auto
+svccntrl "Windows Management Instrumentation" -auto
+svccntrl "Windows Time" -auto
+svccntrl "Wireless Configuration" -auto
+svccntrl "Workstation" -auto
+svccntrl "DCOM Server Process Launcher" -auto
+svccntrl "Windows Firewall/Internet Connection Sharing (ICS)" -auto
+svccntrl "Security Center" -auto
diff --git a/tools/desktop-scripts/lionfish-desktop-services-med.bat b/tools/desktop-scripts/lionfish-desktop-services-med.bat
new file mode 100755
index 000000000..9d4d6b0c8
--- /dev/null
+++ b/tools/desktop-scripts/lionfish-desktop-services-med.bat
@@ -0,0 +1,128 @@
+REM **********************************************************
+REM Copyright (c) 2006 VMware, Inc.  All rights reserved.
+REM **********************************************************
+
+REM Redistribution and use in source and binary forms, with or without
+REM modification, are permitted provided that the following conditions are met:
+REM 
+REM * Redistributions of source code must retain the above copyright notice,
+REM   this list of conditions and the following disclaimer.
+REM 
+REM * Redistributions in binary form must reproduce the above copyright notice,
+REM   this list of conditions and the following disclaimer in the documentation
+REM   and/or other materials provided with the distribution.
+REM 
+REM * Neither the name of VMware, Inc. nor the names of its contributors may be
+REM   used to endorse or promote products derived from this software without
+REM   specific prior written permission.
+REM 
+REM THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+REM AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+REM IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+REM ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+REM FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+REM DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+REM SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+REM CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+REM LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+REM OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+REM DAMAGE.
+
+rem "Set up machine for medium load (beyond Lionfish Desktop configuration"
+rem "default specifed by PM"
+rem "make 26 more services listed in PM's spreadsheet automatic"
+
+rem "default list of services enabled (see below)"
+svccntrl "Alerter" -auto
+svccntrl "Application Layer Gateway Service" -auto
+svccntrl "Application Management" -auto
+svccntrl "Background Intelligent Transfer Service" -auto
+svccntrl "Client Service for NetWare" -disabled
+svccntrl "Clipbook" -auto
+svccntrl "COM+ System Application" -auto
+svccntrl "Distributed Transaction Coordinator" -auto
+svccntrl "Fast User Switching Compatibility" -auto
+svccntrl "Fax" -auto
+svccntrl "HTTP SSL" -auto
+svccntrl "Human Interface Device Access" -disabled
+svccntrl "IMAPI CD-Burning COM Service" -auto
+svccntrl "Indexing Service" -auto
+svccntrl "Infrared Monitor" -disabled
+svccntrl "IPv6 Helper Service" -disabled
+svccntrl "Logical Disk Manager Administrative Service" -disabled
+svccntrl "Microsoft Software Shadow Copy Provider" -disabled
+svccntrl "Net Logon" -auto
+svccntrl "NetMeeting Remote Desktop Sharing" -disabled
+svccntrl "Network DDE" -disabled
+svccntrl "Network DDE DSDM" -disabled
+svccntrl "NT LM Security Support Provider" -disabled
+svccntrl "Performance Logs and Alerts" -disabled
+svccntrl "QoS RSVP" -disabled
+svccntrl "Remote Access Auto Connection Manager" -auto
+svccntrl "Remote Desktop Help Session Manager" -auto
+svccntrl "Remote Procedure Call (RPC) Locator" -auto
+svccntrl "Removable Storage" -disabled
+svccntrl "RIP Listener" -disabled
+svccntrl "Routing and Remote Access" -disabled
+svccntrl "SAP Agent" -disabled
+svccntrl "Simple TCP/IP Services" -auto
+svccntrl "Smart Card" -auto
+svccntrl "TCP/IP Print Server" -auto
+svccntrl "Telnet" -auto
+svccntrl "Uninterruptible Power Supply" -disabled
+svccntrl "Universal Plug and Play Device Host" -auto
+svccntrl "Volume Shadow Copy" -disabled
+svccntrl "Windows Management Instrumentation Driver Extensions" -auto
+svccntrl "WMI Performance Adapter" -auto
+svccntrl "World Wide Web Publishing Service" -auto
+svccntrl "Portable Media Serial Number Service" -auto
+svccntrl "Network Provisioning" -auto
+
+rem "make services in defult configuration automatic"
+
+svccntrl "Automatic Updates" -auto
+svccntrl "COM+ Event System" -auto
+svccntrl "Computer Browser" -auto
+svccntrl "Cryptographic Services" -auto
+svccntrl "DHCP Client" -auto
+svccntrl "Distributed Link Tracking Client" -auto
+svccntrl "DNS Client" -auto
+svccntrl "Error Reporting Service" -auto
+svccntrl "Event Log" -auto
+svccntrl "Help and Support" -auto
+svccntrl "IPSEC Services" -auto
+svccntrl "Logical Disk Manager" -auto
+svccntrl "Messenger" -auto
+svccntrl "Network Connections" -auto
+svccntrl "Network Location Awareness (NLA)" -auto
+svccntrl "Plug and Play" -auto
+svccntrl "Portable Media Serial Number" -auto
+svccntrl "Print Spooler" -auto
+svccntrl "Protected Storage" -auto
+svccntrl "Remote Access Connection Manager" -auto
+svccntrl "Remote Procedure Call (RPC)" -auto
+svccntrl "Remote Registry" -auto
+svccntrl "Secondary Logon" -auto
+svccntrl "Security Accounts Manager" -auto
+svccntrl "Server" -auto
+svccntrl "Shell Hardware Detection" -auto
+svccntrl "SSDP Discovery Service" -auto
+svccntrl "System Event Notification" -auto
+svccntrl "System Restore Service" -auto
+svccntrl "Task Scheduler" -auto
+svccntrl "TCP/IP NetBIOS Helper" -auto
+svccntrl "Telephony" -auto
+svccntrl "Terminal Services" -auto
+svccntrl "Themes" -auto
+svccntrl "Upload Manager" -auto
+svccntrl "WebClient" -auto
+svccntrl "Windows Audio" -auto
+svccntrl "Windows Image Acquisition (WIA)" -auto
+svccntrl "Windows Installer" -auto
+svccntrl "Windows Management Instrumentation" -auto
+svccntrl "Windows Time" -auto
+svccntrl "Wireless Configuration" -auto
+svccntrl "Workstation" -auto
+svccntrl "DCOM Server Process Launcher" -auto
+svccntrl "Windows Firewall/Internet Connection Sharing (ICS)" -auto
+svccntrl "Security Center" -auto
diff --git a/tools/desktop-scripts/lionfish_core_srv_exes.txt b/tools/desktop-scripts/lionfish_core_srv_exes.txt
new file mode 100644
index 000000000..585cf2c86
--- /dev/null
+++ b/tools/desktop-scripts/lionfish_core_srv_exes.txt
@@ -0,0 +1,6 @@
+lsass.exe
+spoolsv.exe
+svchost.exe
+winlogon.exe
+services.exe
+nodemgr.exe
diff --git a/tools/desktop-scripts/measure_cores.pl b/tools/desktop-scripts/measure_cores.pl
new file mode 100644
index 000000000..596437c44
--- /dev/null
+++ b/tools/desktop-scripts/measure_cores.pl
@@ -0,0 +1,77 @@
+# **********************************************************
+# Copyright (c) 2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# measure_cores.pl
+#
+# Catchup on memory measurements for all the core since blowfish
+# optionall skip ones already done
+#
+# NOTE: setup the new core for qatest configuration, need to change this to
+#       use for a different config
+
+# FIXME: clean up hardcoded dir srini
+#
+
+use strict;
+
+my $PROG = "measure_30xxx_core.pl";
+
+my $dir;
+my $core_dir = qq("C:\\Program Files\\Determina\\Determina Management Console\\data\\core");
+
+# for each core available, 
+#   install it and reboot qatest
+#   qatest will run desktop-nightly.bat which will send out an email
+foreach $dir (`cmd /c dir /on /b $core_dir\\30???r`) {
+    chomp($dir);
+    # optionally skip the ones that were done
+    #next if ($dir < "30259r");
+    system("net stop tomcatsc");
+    print "$PROG: Installing using core: $dir\n";
+    &InstallNewCore($dir);
+    system("net start tomcatsc");
+    print "$PROG: Sleeping 180s\n";
+    sleep 180;
+    print "$PROG: Rebooting qatest to run desktop-nightly\n";
+
+    # run this 3 times
+    for (my $i = 0; $i < 3; $i++) {
+        # give it about 30 min to complete and start the next cycle
+        system("c:\\perl\\bin\\perl c:\\srini\\boot_data_scripts\\reboot_qatest.pl 1400");
+    }
+}
+
+#--------------------------------------------------
+sub InstallNewCore() {
+    my $ver_r  = shift;
+
+    # nightly core upgrade" use the new release build
+    system("\"C:\\Program Files\\Determina\\Determina Management Console\\Support Tools\\MySQL\\bin\\mysql\" -D emv-1 -u root --password=braksha -e \"UPDATE policies SET CoreVersion='$ver_r' WHERE Name='qatest configuration'\"");
+}
diff --git a/tools/desktop-scripts/measure_mem.pl b/tools/desktop-scripts/measure_mem.pl
new file mode 100644
index 000000000..ec639a412
--- /dev/null
+++ b/tools/desktop-scripts/measure_mem.pl
@@ -0,0 +1,275 @@
+# **********************************************************
+# Copyright (c) 2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# measure_mem.pl
+#
+# NOTE: ASSUMES drview is in path
+# NOTE: ASSUMES c:\desktop-nightly\boot_data_scripts and
+#               c:\desktop-nightly\boot_data exist  #TODO do a mkdir
+# NOTE: ASSUMES this script is in c:\desktop-nightly\boot_data_scripts #FIXME
+# NOTE: USES: lionfish_core_srv_exes.txt
+#
+# usage: perl measure_mem.pl --int 3 --boot 57 --summary zzz --nightly $(cygpath /tmp) --name test8r
+
+#TODO: exe list from file should be same as procs under DR, if not we have
+#      more running on the machine, warn?
+#TODO: print usage
+
+use strict;
+use Getopt::Long;
+
+my $PROG = "measure_mem.pl";
+
+my %opt;
+&GetOptions(\%opt,
+    "int:i",
+    "times=i",
+    "tillidle",
+    "boot=i",
+    "name=s",
+    "nightly=s",
+    "summary=s",
+    "help",
+    "usage",
+    "debug",
+);
+
+$opt{'int'} = 30 if $opt{'int'} == 0;
+
+# just paranoid about spaces
+my $name            = qq($opt{'name'});
+my $nightly         = qq($opt{'nightly'});
+my $summary         = qq($opt{'summary'});
+my $nightly_summary = qq($nightly\\$summary);
+
+my $ROOT      = qq(C:\\desktop-nightly);
+my $scriptdir = qq($ROOT\\boot_data_scripts);
+my $outdir    = qq($ROOT\\boot_data);
+
+if ($opt{'debug'}) {
+    print "scrdir: $scriptdir\n";
+    print "outdir: $outdir\n";
+    print "boot: $opt{'boot'}\n";
+    print "name: $name\n";
+    print "nightly: $nightly\n";
+    print "summary: $summary\n";
+    print "nightly_summary: $nightly_summary\n";
+}
+
+# log file
+my $stamp         = &timestamp();
+my $drtop_file    = qq($outdir\\drtop_$stamp.txt);
+my $drtop_summary = qq($outdir\\drtop_$stamp).qq(_summary.txt);
+
+# exes we care about for summary
+my $exes_file = qq($scriptdir\\lionfish_core_srv_exes.txt);
+my @exes = &read_file($exes_file);
+
+my $title_string = &title_string();
+my $summary_string;
+my $count = 0;
+my $CPU_IDX = 1; #TODO: ok?
+
+print "$PROG: measuring memorty till idle\n";
+
+# do drview -showmem every 30s and compute summation for exes
+# in lionfish configuration.  Stop when the %cpu is <= 2%
+for (my $iter = 1; 1 ; $iter++) {
+    my @drtop_out = `drview -listall -showmem`;
+
+    &append_to_file($drtop_file, @drtop_out);
+
+    &append_to_file($drtop_summary, ($title_string)) if $iter == 1;
+
+    $summary_string = 
+        &compute_summary($name, $opt{'boot'}, $iter, @drtop_out);
+
+    &append_to_file($drtop_summary, ($summary_string));
+
+    if (defined $opt{'tillidle'}) {
+        # double work
+        my @sums = split /\s+/, $summary_string;
+
+        $sums[$CPU_IDX] =~ s/%//;
+        my $cpu = $sums[$CPU_IDX];
+
+        # increment if cpu is <= 2%, otherwise reset to 0
+        $count = ($cpu < 3) ? $count + 1 : $count - $count;
+
+        # exit if cpu utilization is <= 2%, 3 times in a row
+        last if ($count > 2);
+    }
+    else {
+        last if (defined $opt{'times'} && $iter > $opt{'times'});
+    }
+
+    sleep($opt{'int'});
+}
+
+print "$PROG: copying data to $nightly\n";
+# copy results to network
+system("copy /y $drtop_file    $nightly");
+system("copy /y $drtop_summary $nightly");
+
+print "$PROG: Writing summary to $nightly_summary\n";
+# write memory summary when machine is idle to nightly dir summary file
+&append_to_file($nightly_summary, ($title_string)) unless (-e $nightly_summary);
+&append_to_file($nightly_summary, ($summary_string));
+
+#--------------------------------------------------------------------------
+sub compute_summary() {
+    my $name      = shift;
+    my $boot      = shift;
+    my $iter      = shift;
+    my @drtop_out = @_;
+
+    my @totals   = 0 x 15;
+    my @cols     = ( "\n" );
+    my $name_idx = 0;
+    my $num_exes = 0;
+    my $charge   = "0";
+
+    my $line;
+    foreach $line (@drtop_out) {
+        chomp($line);
+
+        # skip uninteresting lines
+        next if $line =~ /^$/i;
+        next if $line =~ /System load is/i;
+        if ($line =~ s/System committed memory.*: //i) {
+            $charge = $line;
+            next;
+        }
+
+        # skip the title line
+        next if ($line =~ m/Name.*PID.*Bld.*CPU.*WSS.*/i);
+
+        # split each column (Priv, WSS etc.)
+        @cols = split (/\s+/, $line);
+
+        # skip the blank process
+        next if $cols[$name_idx] =~ m/^\s*$/;
+
+        # just get the name of the exe so we can grep for it
+        # for e.g svchost.exe-netsvcs -> svchost
+        $cols[$name_idx] =~ s/\.exe.*//i;
+
+        # we are only interested in exes listed in the configuration file
+        next unless (scalar grep(m/$cols[$name_idx]/i, @exes));
+
+        # list the number of procs in summation
+        $num_exes++;
+
+        # add the various columns of data up
+        for(my $col = 4; $col < 19; $col++) {
+            $cols[$col] =~ s/%//;
+            $totals[$col - 4] += $cols[$col];
+        }
+    }
+
+    # return a line of summary
+    return &summary_string($name, $boot, $iter, $num_exes, $charge, @totals);
+}
+
+sub title_string {
+    my $title = sprintf "Build           CPU User  Thr    PVSz     VSz    PPriv    Priv    PWSS     WSS   Fault PPage  Page PNonP  NonP Commit charge                 Boot #Exes *30s\n";
+    return $title;
+}
+
+sub summary_string {
+    my $name     = shift;
+    my $boot     = shift;
+    my $iter     = shift;
+    my $num_exes = shift;
+    my $charge   = shift;
+    my @totals   = @_;
+
+    my $summary = sprintf(
+        "%-14.14s %3d%% %3d%% %4d %7d %7d %8d %7d %7d %7d %7d %5d %5d %5d %5d %s %5d %5d %4d\n",
+        $name,
+        $totals[ 0],      # CPU %
+        $totals[ 1],      # User %
+                          # totals[2] handles, not printed
+        $totals[ 3],      # Threads
+        $totals[ 4],      # Peak Vsz
+        $totals[ 5],      # VSz
+        $totals[ 6],      # Peak Priv
+        $totals[ 7],      # Priv
+        $totals[ 8],      # Peak WSS
+        $totals[ 9],      # WSS
+        $totals[10],      # Fault
+        $totals[11],      # Peak Page pool
+        $totals[12],      # Page pool
+        $totals[13],      # Peak Non-Paged pool
+        $totals[14],      # Non-paged
+        $charge,          # commit charge
+        $boot,            # boot time
+        $num_exes,        # number of exes in summation
+        $iter,            # iter
+    );
+
+    return $summary;
+}
+
+sub append_to_file {
+    my $file = shift;
+    my @data = @_;
+
+    open (FD, ">>$file") or
+        die "$PROG: cannot open $file: $!";
+    print FD @data; 
+    close FD;
+}
+
+sub read_file {
+    my $file = shift;
+    my @data = ();
+
+    open (FD, "<$file") or
+        die "$PROG: cannot open $file: $!";
+    @data = <FD>;
+    close FD;
+
+    return @data;
+}
+
+sub timestamp() {
+    # ignore wday - day of the week, $yday - day of the year, and 
+    # $is_dst - is daylight savings time
+    my ($sec,  $min,  $hour,
+        $mday, $mon,  $year,
+        $wday, $yday, $is_dst) = localtime(time);
+
+    # last 2 digits of year
+    $year = ($year + 1900) % 100;
+    my $stamp = sprintf("%02d%02d%02d_%02d%02d%02d",
+                            $year,$mon+1,$mday, $hour,$min,$sec);
+    return $stamp;
+}
diff --git a/tools/desktop-scripts/measure_options.pl b/tools/desktop-scripts/measure_options.pl
new file mode 100644
index 000000000..5638b6a34
--- /dev/null
+++ b/tools/desktop-scripts/measure_options.pl
@@ -0,0 +1,403 @@
+# **********************************************************
+# Copyright (c) 2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# measure_options.pl
+#
+# NOTE: ASSUMES drview, drcontrol, svccntrl and perl are in path
+# NOTE: ASSUMES c:\desktop-nightly\boot_data_scripts and
+#               c:\desktop-nightly\boot_data exist  #TODO do a mkdir
+# NOTE: ASSUMES this script is in c:\desktop-nightly\boot_data_scripts #FIXME
+#
+# NOTE: USES: BOOTNUM_FILE, uptime.vbs
+# 
+# NOTE: Set up BOOTNUM_FILE (see below) to 0 (when done it will set it up to
+#       0 and should be ready to use, but if runs were interrupted you'd
+#       need to do this manually)
+#
+# NOTE: Appends options specified in @dropts to current options provided by
+#       nodemgr
+#
+# To run:
+#   1. Just write 0 to BOOTNUM_FILE
+#   2. Setup @dropts array with option to be run
+#   3. Setup $RESULTS_DIR and $SUMMARY_FILE to your liking
+#   4. run the script: perl measure_options.pl
+#      or
+#      call it from a batch file
+#      [NOTE: put batch file in Programs->Startup to run at reboot/login]
+#
+
+use strict;
+use Getopt::Long;
+
+my $PROG = "measure_options.pl"; # TODO: use cwd? basename etc.
+
+# calculate boot time and log it
+#
+# disable/enable node manager
+#
+# run drtop every boot every 30s
+#   sum all cols from -showmem o/p
+# if cpu utilization hits <= 2%
+#   write all sums to log file
+#
+# for each boot do
+#   set up registry keys for each app with
+#     appropriate options for this run
+#   do the run
+#
+# different runs
+#   reset from 1 to nth thread
+#   set -hotp_nudge_is_reset
+#   -C
+#   -E
+#   -F
+#   shared bbs
+#   shared traces
+#   private traces with diff cache sizes
+#   different set of services loaded
+#     default set
+#     medium set
+#     all set
+#
+# move all logfiles to \\fileserver\g_shares\desktop_nightly\results
+#
+# enable node manager
+#   some other script installs latest core
+
+my $ROOT      = qq(C:\\desktop-nightly);
+my $SCRIPTDIR = qq($ROOT\\boot_data_scripts);
+my $OUTDIR    = qq($ROOT\\boot_data);
+
+# get the boot time before doing anything else
+my ($boot_time, @ignore) = grep s/uptime in seconds: //i,
+                                `cscript -nologo $SCRIPTDIR\\uptime.vbs`;
+
+my %opt;
+&GetOptions(\%opt,
+    "debug",
+    "default",
+    "help",
+    "usage",
+);
+#TODO: print usage, verify options
+
+my $DRVER = &get_build();
+
+system("net use m: \\\\10.1.5.85\\g_shares\\desktop-benchmarks \"\" /USER:guest") unless (-e "m:");
+
+my $date = &datestamp();
+my $RESULTS_DIR = qq(m:\\results\\boot_data\\qatest_$DRVER).qq(_mem_$date);
+mkdir $RESULTS_DIR unless -e $RESULTS_DIR;
+
+my $SUMMARY_FILE = qq(qatest_$DRVER).qq(_mem_$date.txt);
+
+# List of options for each run.
+# Just enable/put the options you want run here.
+my @dropts = (
+     "",   # dont change this, let this be boot 0
+
+#    "",                      # default
+#    "",                      # default
+#    "",                      # default
+#    "-no_enable_reset",
+#    "-no_enable_reset",
+#    "-no_enable_reset",
+#    "-hotp_nudge_is_reset",
+#    "-hotp_nudge_is_reset",
+#    "-hotp_nudge_is_reset",
+#    "-C -hotp_nudge_is_reset",
+#    "-C -hotp_nudge_is_reset",
+#    "-C -hotp_nudge_is_reset",
+#    "-E -hotp_nudge_is_reset",
+#    "-E -hotp_nudge_is_reset",
+#    "-E -hotp_nudge_is_reset",
+#    "-F -hotp_nudge_is_reset",
+#    "-F -hotp_nudge_is_reset",
+#    "-F -hotp_nudge_is_reset",
+#    "-C -E -F -hotp_nudge_is_reset",
+#    "-C -E -F -hotp_nudge_is_reset",
+#    "-C -E -F -hotp_nudge_is_reset",
+    # -reset_at_nth_thread is added to this list in a loop (see below)
+
+#    # see memory impact of two new features I added:
+#    "-no_remove_shared_trace_heads",
+#    "-remove_trace_components",
+#
+#    # shared ibt tables experimental
+#    "-shared_ibt_tables",
+#
+#    # shared bbs, no traces
+#    "-shared_bbs_only",
+#    "-shared_bbs_only",
+#    "-shared_bbs_only",
+#
+#    # shared bbs, private traces
+#    "-no_shared_traces",
+#    "-no_shared_traces -cache_trace_regen 20",
+#    "-no_shared_traces -cache_trace_regen 30",
+#    "-no_shared_traces -cache_trace_regen 40",
+#    "-no_shared_traces -no_finite_trace_cache",
+#
+#    # private bbs, no traces
+#    "-disable_traces -no_shared_bbs",
+#    "-disable_traces -no_shared_bbs -cache_bb_regen 20",
+#    "-disable_traces -no_shared_bbs -cache_bb_regen 30",
+#    "-disable_traces -no_shared_bbs -cache_bb_regen 40",
+#    "-disable_traces -no_shared_bbs -no_finite_bb_cache",
+#
+#    # private bbs, shared traces
+#    "-no_shared_bbs",
+#    "-no_shared_bbs -cache_bb_regen 20",
+#    "-no_shared_bbs -cache_bb_regen 30",
+#    "-no_shared_bbs -cache_bb_regen 40",
+#    "-no_shared_bbs -no_finite_bb_cache",
+#
+#    # all private
+#    # bb regen 10 (default), vary trace regen 10,20,30,40,infinite
+#    "-no_shared_traces -no_shared_bbs",
+#    "-no_shared_traces -no_shared_bbs -cache_trace_regen 20",
+#    "-no_shared_traces -no_shared_bbs -cache_trace_regen 30",
+#    "-no_shared_traces -no_shared_bbs -cache_trace_regen 40",
+#    "-no_shared_traces -no_shared_bbs -no_finite_trace_cache",
+#    # bb regen 20, vary trace regen 10,20,30,40,infinite
+#    "-no_shared_traces -no_shared_bbs -cache_bb_regen 20",
+#    "-no_shared_traces -no_shared_bbs -cache_bb_regen 20 -cache_trace_regen 20",
+#    "-no_shared_traces -no_shared_bbs -cache_bb_regen 20 -cache_trace_regen 30",
+#    "-no_shared_traces -no_shared_bbs -cache_bb_regen 20 -cache_trace_regen 40",
+#    "-no_shared_traces -no_shared_bbs -cache_bb_regen 20 -no_finite_trace_cache",
+#    # bb regen 30, vary trace regen 10,20,30,40,infinite
+#    "-no_shared_traces -no_shared_bbs -cache_bb_regen 30",
+#    "-no_shared_traces -no_shared_bbs -cache_bb_regen 30 -cache_trace_regen 20",
+#    "-no_shared_traces -no_shared_bbs -cache_bb_regen 30 -cache_trace_regen 30",
+#    "-no_shared_traces -no_shared_bbs -cache_bb_regen 30 -cache_trace_regen 40l",
+#    "-no_shared_traces -no_shared_bbs -cache_bb_regen 30 -no_finite_trace_cache",
+#    # bb regen 40, vary trace regen 10,20,30,40,infinite
+#    "-no_shared_traces -no_shared_bbs -cache_bb_regen 40",
+#    "-no_shared_traces -no_shared_bbs -cache_bb_regen 40 -cache_trace_regen 20",
+#    "-no_shared_traces -no_shared_bbs -cache_bb_regen 40 -cache_trace_regen 30",
+#    "-no_shared_traces -no_shared_bbs -cache_bb_regen 40 -cache_trace_regen 40",
+#    "-no_shared_traces -no_shared_bbs -cache_bb_regen 40 -no_finite_trace_cache",
+#    # bb infinite, vary trace regen 10,20,30,40,infinite
+#    "-no_shared_traces -no_shared_bbs -no_finite_bb_cache",
+#    "-no_shared_traces -no_shared_bbs -no_finite_bb_cache -cache_trace_regen 20",
+#    "-no_shared_traces -no_shared_bbs -no_finite_bb_cache -cache_trace_regen 30",
+#    "-no_shared_traces -no_shared_bbs -no_finite_bb_cache -cache_trace_regen 40",
+#    "-no_shared_traces -no_shared_bbs -no_finite_bb_cache -no_finite_trace_cache",
+
+);
+
+# enable to run reset_at_nth_thread 1..20
+#for (my $thr = 1; $thr < 21; $thr++) {
+#    push @dropts, "reset_at_nth_thread $thr";
+#}
+
+
+# run with just the default set of options
+@dropts = ( "" ) if defined $opt{'default'};
+
+# file to save registry settings
+my $SAVE_CFG = qq($SCRIPTDIR\\qatest_determina_keys.cfg);
+
+# file that holds the boot number
+my $BOOTNUM_FILE = qq($SCRIPTDIR\\bootnum);
+my $bootnum = &get_bootnum();
+print "$PROG: Boot number: $bootnum\n";
+
+# at each boot, pick a set of options to run with
+
+&disable_node_manager();
+
+# writes 0 to BOOTNUM_FILE
+&cleanup_and_exit() if ($bootnum >= scalar @dropts);
+
+print "$PROG: running options: $dropts[$bootnum+1]\n";
+
+# save the registry settings the first time
+system("drcontrol -save $SAVE_CFG") if ($bootnum == 0);
+system("drcontrol -dump > $RESULTS_DIR\\drcontrol-dump.$bootnum");
+
+my $MEASURE_MEM_CMD = qq($SCRIPTDIR\\measure_mem.pl -tillidle --name "$DRVER-$bootnum" --nightly $RESULTS_DIR --summary $SUMMARY_FILE --b $boot_time);
+
+print "$PROG: $MEASURE_MEM_CMD\n";
+system("perl $MEASURE_MEM_CMD");
+
+&do_hotp_nudge() if ($dropts[$bootnum] =~ m/hotp_nudge_is_reset/i);
+
+# now set up registry keys for next measurement
+# restore the registry settings for our agent
+system("drcontrol -detachall");
+system("drcontrol -load $SAVE_CFG");
+&resetreg($dropts[$bootnum+1]);
+
+print "$PROG: rebooting..\n";
+
+reboot($bootnum);
+
+
+
+#-----------------------------------------------------------------------------
+
+sub resetreg {
+    my $append_dropt = shift;
+
+    #TODO: use %PROGRAM FILES% ?
+    my $DETERMINA = qq("$ENV{'PROGRAMFILES'}\\Determina\\Determina Agent");
+    my $DRHOME = $DETERMINA;
+    #my $DRHOME = qq("C:\\Program Files\\Determina\\Determina Agent");
+    my $LIBDIR = "lib";
+    my $LOGDIR = "logs";
+
+    my $prev_dropt = "";
+    my $dropt = "";
+
+    my @ignore;
+
+    # disable node manager if needed
+    &disable_node_manager() unless
+        scalar grep m/scnodemgr.*disabled/i, `svccntrl scnodemgr -show`;
+
+    my $drdll = qq($DRHOME\\$LIBDIR\\$DRVER\\dynamorio.dll);
+
+    # may as well set global key though we're setting every app key below
+    system("drcontrol -drlib $drdll");
+    system("drcontrol -logdir $DRHOME\\$LOGDIR");
+
+    ($prev_dropt, @ignore) =
+            grep s/\s*DYNAMORIO_OPTIONS=\s*//, `drcontrol -dump`;
+    chomp($prev_dropt);
+    print "$PROG: global dropt: $prev_dropt\n";
+
+    $dropt = qq("$prev_dropt $append_dropt");
+    system("drcontrol -options $dropt");
+
+    # drcontrol -load not implemented, and no way to remove a string value,
+    # so our only options are to reproduce the slist parsing and get the
+    # right rununder values, or set the values we want for each app, which
+    # is a pain to edit manually later but oh well:
+    my @apps = grep s/\s*Config\s*Group\s*: \s*//, `drcontrol -dump`;
+    my $app;
+    foreach $app (@apps) {
+        system("drcontrol -app $app -drlib $drdll");
+        system("drcontrol -app $app -logdir $DRHOME\\$LOGDIR");
+        ($prev_dropt, @ignore) =
+            grep s/\s*DYNAMORIO_OPTIONS=\s*//, `drcontrol -appdump $app`;
+        next unless $prev_dropt;
+        chomp($prev_dropt);
+        $dropt = qq("$prev_dropt $append_dropt");
+        system("drcontrol -app $app -options $dropt");
+    }
+
+    ### if ignore RUNUNDER this will re-make reg keys from scc file:
+    ###    # now re-build reg setup
+    ###    drcontrol -reset;
+    ###    for i in `grep -i exe /c/Program\ Files/Determina/SecureCore/config/default.scc`; do
+    ###        # pull exe name out of [HKEY_LOCAL_MACHINE\<...>\inetinfo.exe]
+    ###        j=${i##*\\};
+    ###        drcontrol -add ${j/]/};
+    ###    done;
+    system("drcontrol -preinject ON");
+}
+
+sub do_hotp_nudge {
+    # should be idle now, so nudge
+    system("drcontrol -hot_patch_nudge_all");
+    # sleep for 10 more minutes to see what happens to memory
+    sleep 600;
+}
+
+sub cleanup_and_exit {
+    # clean up after all runs
+    system("drcontrol -load $SAVE_CFG");
+    system("echo 0 > $BOOTNUM_FILE");
+
+    &enable_node_manager();
+
+    exit 0;
+}
+
+sub disable_node_manager {
+    #TODO: manual is fine or do we need to disable
+    system("svccntrl scnodemgr -disabled");
+    system("net stop scnodemgr");
+}
+
+sub enable_node_manager {
+    system("svccntrl scnodemgr -auto");
+    system("net start scnodemgr");
+}
+
+sub get_bootnum {
+    open FD, "<$BOOTNUM_FILE" or die "Cannot open $BOOTNUM_FILE: $!";
+    chomp(my $num = <FD>);
+    close FD;
+    return $num;
+}
+
+sub reboot {
+    my $bootnum = shift;
+    $bootnum++;
+    system("echo $bootnum > $BOOTNUM_FILE");
+    system("shutdown -r -t 0 -f");
+}
+
+
+#--------------------------------------------------------------------------
+sub get_build() {
+    my @dump = `drcontrol -dump 2> $OUTDIR\\drcontrol_err.txt`;
+
+    # drcontrol -dump prints
+    #   error message, if core is not installed
+    #   data for SC config, if disabled from DMC
+    # if we have any app instrumented, then we care about under DR
+    my ($ignore_sc, $build, @ignore_rest) =
+        grep s/DYNAMORIO_AUTOINJECT.*Agent.lib.//i, @dump;
+
+    chomp($build);
+
+    $build = "native" unless ($build);
+    $build =~ s/\s*([0-9]+[a-z]).dynamorio.dll/$1/i;
+
+    return $build;
+}
+
+sub datestamp() {
+    # ignore wday - day of the week, $yday - day of the year, and 
+    # $is_dst - is daylight savings time
+    my ($sec,  $min,  $hour,
+        $mday, $mon,  $year,
+        $wday, $yday, $is_dst) = localtime(time);
+
+    # last 2 digits of year
+    $year = ($year + 1900) % 100;
+    my $stamp = sprintf("%02d%02d%02d",
+                            $year,$mon+1,$mday);
+    return $stamp;
+}
diff --git a/tools/desktop-scripts/measure_reset.pl b/tools/desktop-scripts/measure_reset.pl
new file mode 100644
index 000000000..870bbf53a
--- /dev/null
+++ b/tools/desktop-scripts/measure_reset.pl
@@ -0,0 +1,142 @@
+# **********************************************************
+# Copyright (c) 2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# measure_reset.pl
+#
+# NOTE: ASSUMES drcontrol and drkill are in path
+# NOTE: USES: getmeminfo_automate.pl
+#
+# TODO: has hard coded OutputPath
+
+my $prog = "measure_reset.pl";
+
+if (($#ARGV+1)  < 1 ) {
+    print "Usage : $prog <path to dir boot_data>\n";
+    exit;
+}
+
+$ScriptPath=$ARGV[0];
+
+my $BOOTWAIT = 600;
+my $WAITFOR  = 300;
+
+waitfor($BOOTWAIT);
+
+nudge_reset("core-srv\n");
+
+waitfor($WAITFOR);
+
+launch_ie("10-IE-windows-1-proc\n","", 10);
+nudge_reset("srv-10-IE-windows-1-proc\n");
+
+# do some work after reset
+waitfor($WAITFOR);
+launch_ie("2-more-IE-windows\n", "", 2);
+
+print "$prog: killing IE\n";
+system("drkill -exe iexplore.exe");
+waitfor($WAITFOR);
+
+
+launch_ie("10-IE-procs\n","-mp", 10);
+nudge_reset("srv-10-IE-procs\n");
+
+# do some work after reset, some process get it
+waitfor($WAITFOR);
+launch_ie("2-more-IE-windows\n", "", 2);
+
+print "$prog: killing IE\n";
+system("drkill -exe iexplore.exe");
+waitfor($WAITFOR);
+
+
+system("perl $ScriptPath\\measure_word.pl $ScriptPath 3 0");
+nudge_reset("srv-3-word-docs\n");
+
+# do some work after reset
+waitfor($WAITFOR);
+system("perl $ScriptPath\\measure_word.pl $ScriptPath 0 1");
+
+print "$prog: killing word\n";
+system("drkill -exe winword.exe");
+waitfor($WAITFOR);
+getmem("after-kill-word\n");
+
+
+
+
+#--------------------------------------------------------------------
+
+sub launch_ie {
+    my $text = shift;
+    my $arg1 = shift;
+    my $arg2 = shift;
+
+    getmem("before-$text");
+    print "$prog: launching $text..\n";
+    system("perl $ScriptPath\\ie\\launch-ie.pl $arg1 $arg2");
+    waitfor($WAITFOR);
+    getmem("after-$text");
+}
+
+sub nudge_reset {
+    my $text = shift;
+
+    getmem("before-reset-$text");
+    print "$prog: nudging to reset $text..\n";
+    system("drcontrol -hot_patch_nudge_all");
+    waitfor($WAITFOR);
+    getmem("after-reset-$text");
+}
+
+sub waitfor {
+    my $waitfor = shift;
+    print "$prog: Waiting $waitfor secs..\n";
+    sleep($waitfor);
+}
+
+sub getmem {
+    my $text = shift;
+    write_to_summary($text);
+    system("perl.exe $ScriptPath\\getmeminfo_automate.pl $ScriptPath");
+}
+
+sub write_to_summary {
+    my $text = shift;
+
+    my $OutputPath = "Z:\\desktop-benchmarks\\results\\boot_data";
+    my $hostname = "xp_qatest";  #TODO: fix this
+    my $sumfile="Boot_Summary_$hostname.txt";
+    my $fullpath="$OutputPath\\$hostname\\$sumfile";
+
+    open(OUT,">>$fullpath") or die "$prog: Cannot find $fullpath: $!";
+    print OUT "$text";
+    close OUT;
+}
diff --git a/tools/desktop-scripts/measure_word.pl b/tools/desktop-scripts/measure_word.pl
new file mode 100644
index 000000000..780517156
--- /dev/null
+++ b/tools/desktop-scripts/measure_word.pl
@@ -0,0 +1,100 @@
+# **********************************************************
+# Copyright (c) 2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+
+# perl script
+
+# NOTE: ASSUMES word and drcontrol are in the path
+# NOTE: USES: getmeminfo_automate.pl
+# NOTE: ASSUME hardcoded locations for $masterdoc and $bmarkdoc:
+#       $masterdoc = "c:\\desktop-benchmarks\\derek-win32-bmarks\\master.doc";
+#       $bmarkdoc = "c:\\desktop-benchmarks\\derek-win32-bmarks\\master-bmark-noclose.doc"; # FIXME YUCK!
+# FIXME: has hard coded OutputPath
+
+
+my $prog = "measure-word.pl";
+my $app  = "winword.exe";
+
+if (($#ARGV+1)  < 1 ) {
+    print "Usage : $prog <path to dir boot_data> [3 num word] [1 do-bmark]\n";
+    exit;
+}
+
+my $NUM_WORD;
+my $do_bmark;
+
+($ScriptPath, $NUM_WORD, $do_bmark)=($ARGV[0], $ARGV[1], $ARGV[2]);
+
+$NUM_WORD = 3 unless (defined $NUM_WORD);
+$do_bmark = 0 unless (defined $do_bmark);
+
+my $masterdoc = "c:\\desktop-benchmarks\\derek-win32-bmarks\\master.doc";
+my $bmarkdoc  = "c:\\desktop-benchmarks\\derek-win32-bmarks\\master-bmark-noclose.doc";
+
+if ($NUM_WORD > 0) {
+    write_to_summary("before-$NUM_WORD-word-docs\n");
+    system("perl.exe $ScriptPath\\getmeminfo_automate.pl $ScriptPath");
+
+    for (my $i = 1; $i < $NUM_WORD+1; $i++) {
+        system("start $app $masterdoc");
+        print "$prog: sleeping 60 secs\n";
+        sleep(60);
+        write_to_summary("after-$i-word\n");
+        system("perl.exe $ScriptPath\\getmeminfo_automate.pl $ScriptPath");
+    }
+
+    print "$prog: sleeping 300 secs\n";
+    sleep(300);
+}
+
+if ($do_bmark) {
+    write_to_summary("before-1-bmark\n");
+    system("perl.exe $ScriptPath\\getmeminfo_automate.pl $ScriptPath");
+    system("start $app $bmarkdoc");
+    print "$prog: sleeping 300 secs\n";
+    sleep(300);
+    write_to_summary("after-1-bmark\n");
+    system("perl.exe $ScriptPath\\getmeminfo_automate.pl $ScriptPath");
+}
+
+#--------------------------------------------------------------------
+
+sub write_to_summary {
+    my $text = shift;
+
+    my $OutputPath = "Z:\\desktop-benchmarks\\results\\boot_data";
+    my $hostname = "xp_qatest";  #TODO: fix this
+    my $sumfile="Boot_Summary_$hostname.txt";
+    my $fullpath="$OutputPath\\$hostname\\$sumfile";
+
+    open(OUT,">>$fullpath") or die "$prog: Cannot find $fullpath: $!";
+    print OUT "$text";
+    close OUT;
+}
diff --git a/tools/desktop-scripts/reboot_qatest.pl b/tools/desktop-scripts/reboot_qatest.pl
new file mode 100644
index 000000000..0b0fbe9a4
--- /dev/null
+++ b/tools/desktop-scripts/reboot_qatest.pl
@@ -0,0 +1,38 @@
+# **********************************************************
+# Copyright (c) 2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+my $PROG = "reboot_qatest.pl"
+my $sec = 0;
+$sec = $ARGV[0] if defined $ARGV[0];
+
+print "$PROG: rebooting qatest\n";
+system("psexec \\\\192.168.3.77 -u Administrator -p braksha shutdown -r -f -t 0");
+print "$PROG: sleeping $sec sec\n";
+sleep($sec);
diff --git a/tools/desktop-scripts/sleep.pl b/tools/desktop-scripts/sleep.pl
new file mode 100644
index 000000000..2c4ba3f7b
--- /dev/null
+++ b/tools/desktop-scripts/sleep.pl
@@ -0,0 +1,35 @@
+# **********************************************************
+# Copyright (c) 2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# sleep for sometime
+
+my $s = $ARGV[0];
+$s = 1 unless defined $s;
+sleep($s);
diff --git a/tools/desktop-scripts/sysmark_automate.bat b/tools/desktop-scripts/sysmark_automate.bat
new file mode 100755
index 000000000..27697bebe
--- /dev/null
+++ b/tools/desktop-scripts/sysmark_automate.bat
@@ -0,0 +1,33 @@
+REM **********************************************************
+REM Copyright (c) 2006 VMware, Inc.  All rights reserved.
+REM **********************************************************
+
+REM Redistribution and use in source and binary forms, with or without
+REM modification, are permitted provided that the following conditions are met:
+REM 
+REM * Redistributions of source code must retain the above copyright notice,
+REM   this list of conditions and the following disclaimer.
+REM 
+REM * Redistributions in binary form must reproduce the above copyright notice,
+REM   this list of conditions and the following disclaimer in the documentation
+REM   and/or other materials provided with the distribution.
+REM 
+REM * Neither the name of VMware, Inc. nor the names of its contributors may be
+REM   used to endorse or promote products derived from this software without
+REM   specific prior written permission.
+REM 
+REM THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+REM AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+REM IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+REM ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+REM FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+REM DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+REM SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+REM CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+REM LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+REM OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+REM DAMAGE.
+
+ipconfig /release
+"C:\Documents and Settings\tester\Desktop\tools\svccntrl.exe" scnodemgr -disabled
+"C:\Program Files\BAPCo\SYSmark 2004\Benchmgr\Sysmgr.exe" OPSUITE=1 PROJNAME=r2d2_dr30088r_apps_exempt_051025
diff --git a/tools/desktop-scripts/uptime.vbs b/tools/desktop-scripts/uptime.vbs
new file mode 100644
index 000000000..0833cf8e7
--- /dev/null
+++ b/tools/desktop-scripts/uptime.vbs
@@ -0,0 +1,72 @@
+' **********************************************************
+' Copyright (c) 2006 VMware, Inc.  All rights reserved.
+' **********************************************************
+
+' Redistribution and use in source and binary forms, with or without
+' modification, are permitted provided that the following conditions are met:
+' 
+' * Redistributions of source code must retain the above copyright notice,
+'   this list of conditions and the following disclaimer.
+' 
+' * Redistributions in binary form must reproduce the above copyright notice,
+'   this list of conditions and the following disclaimer in the documentation
+'   and/or other materials provided with the distribution.
+' 
+' * Neither the name of VMware, Inc. nor the names of its contributors may be
+'   used to endorse or promote products derived from this software without
+'   specific prior written permission.
+' 
+' THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+' IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+' ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+' FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+' DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+' SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+' CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+' LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+' OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+' DAMAGE.
+
+strComputer = "."   ' "." for local computer
+
+On Error Resume Next
+Set objWMIService = GetObject("winmgmts:" _
+        & "{impersonationLevel=impersonate}!\\" & strComputer _
+        & "\root\cimv2")
+
+If Err.Number = 0 Then
+     On Error Goto 0
+     strQuery = "select * from Win32_PerfRawData_PerfOS_System"
+     Set colObjects = objWMIService.ExecQuery(strQuery)
+
+     For Each objWmiObject In colObjects
+       intPerfTimeStamp = objWmiObject.Timestamp_Object
+       intPerfTimeFreq = objWmiObject.Frequency_Object
+       intCounter = objWmiObject.SystemUpTime
+     Next
+
+     ' Calculation in seconds:
+     'Calculations for Raw Counter Data:PERF_ELAPSED_TIME
+     'http://msdn.microsoft.com/library/en-us/perfmon/base/perf_elapsed_time.asp
+     iUptimeInSec = (intPerfTimeStamp - intCounter)/intPerfTimeFreq
+     WScript.Echo "Uptime in seconds: " & iUptimeInSec \ 1
+
+     ' convert the seconds
+     sUptime = ConvertTime(iUptimeInSec)
+     WScript.Echo "Uptime: " & sUptime
+
+Else
+     Wscript.Echo "Could not connect to computer with WMI: " _
+         & strComputer
+End If
+
+
+Function ConvertTime(seconds)
+     ConvSec = seconds Mod 60
+     ConvMin = (seconds Mod 3600) \ 60
+     ConvHour =  (seconds Mod (3600 * 24)) \ 3600
+     ConvDays =  seconds \ (3600 * 24)
+     ConvertTime = ConvDays & " days " & ConvHour & " hours " _
+                 & ConvMin & " minutes " & ConvSec & " seconds "
+End Function
diff --git a/tools/dgcstats.pl b/tools/dgcstats.pl
new file mode 100755
index 000000000..7e4b149b5
--- /dev/null
+++ b/tools/dgcstats.pl
@@ -0,0 +1,233 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2004 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### dgcstats.pl
+### author: Derek Bruening   August 2003
+###
+### Analyzes DGC_DIAGNOSTICS results -- point it at a log directory
+
+$usage = "Usage: $0 [root]\n";
+
+$verbose = 0;
+
+$dir = ".";
+if ($#ARGV >= 0) {
+    $dir = $ARGV[0];
+}
+
+chdir $dir;
+$main = `ls *[Ee][Xx][Ee].[0-9]*`;
+chop $main;
+print "Reading $main\n";
+open(MAIN, "< $main") || die "Error: Couldn't open $main for input\n";
+while (<MAIN>) {
+    next unless /^DYNGEN/;
+    if (/=> (0x[0-9a-fA-F]+)-(0x[0-9a-fA-F]+) (.*)$/) {
+        $start = hex($1);
+        $end = hex($2);
+        $comment = $3;
+        if (!defined($dg_start{$start})) {
+            $dg_start{$start} = $start;
+            $dg_end{$start} = $end;
+            $dg_comment{$start} = $comment;
+        }
+    }
+}
+close(MAIN);
+
+$find = "find . -name log.\\*";
+open(FIND, "$find |") || die "Error running $find\n";
+while (<FIND>) {
+    chop;
+    print "Reading $_\n";
+    open(LOG, "< $_") || die "Error: Couldn't open $_ for input\n";
+    while (<LOG>) {
+        if ($in_alloc) {
+            # old log files: /^Currently in (0x[0-9a-fA-F]+) (.*)/
+            if (/current pc = (0x[0-9a-fA-F]+)/) {
+                # we could always append and for multiple allocations to
+                # same address get an accumulated callstack...but we don't
+                $prot_stack{$addr} = "\t$1";
+            # old log files: /frame ptr.*ret = (0x[0-9a-fA-F]+) (.*)/
+            } elsif (/frame ptr.*ret = (0x[0-9a-fA-F]+)/) {
+                $prot_stack{$addr} .= "\t$1";
+            } elsif (/\s+(\[.*\])$/) {
+                $prot_stack{$addr} .= " $1\n";
+            } elsif (!/^Call stack/) {
+                $in_alloc = 0;
+            }
+        }
+        # FIXME: log.* order could matter...but don't have synch info so who cares
+        # old log files: /NtAllocateVirtualMemory base=(0x[0-9a-fA-F]+) size=(0x[0-9a-fA-F]+)/
+        if (/NtAllocateVirtualMemory.*@(0x[0-9a-fA-F]+) sz=(0x[0-9a-fA-F]+)/) {
+            $addr = hex($1);
+            $end = $addr + hex($2);
+            $prot_start{$addr} = $addr;
+            $prot_end{$addr} = $end;
+            $prot_line{$addr} = $_;
+            $in_alloc = $addr;
+        }
+        # FIXME: try to watch NtFree as well?  don't have synch info though
+        if (/^Entry into dyngen F[0-9]+\((0x[0-9a-fA-F]+)(.*)\) via: +(0x[0-9a-fA-F]+).*  ([<A-Za-z]:?([->!,+a-zA-Z~_\\0-9\.]| {1})+)/) {
+            $dst = hex($1) & 0xfffff000;
+            $src = hex($3) & 0xfffff000;
+            $desc = $2; # not used since we search the .exe. file
+            $dll = $4;
+            $found = 0;
+            for ($i=0; $i<$num_entries{$dst}; $i++) {
+                if ($src == $entry_list{$dst}[$i]) {
+                    $entry_count{$dst}[$i]++;
+                    $found = 1;
+                    break;
+                }
+            }
+            if (!$found) {
+                $entry_list{$dst}[$num_entries{$dst}] = $src;
+                $entry_dll{$dst}[$num_entries{$dst}] = $dll;
+                $entry_count{$dst}[$num_entries{$dst}] = 1;
+                if ($entry_desc{$dst} eq "") {
+                    foreach $s (sort (keys %dg_start)) {
+                        if ($dst >= $s && $dst < $dg_end{$s}) {
+                            $entry_desc{$dst} = $dg_comment{$s};
+                            break;
+                        }
+                    }
+                }
+                $num_entries{$dst}++;
+            }
+            if ($verbose && $dst != $src) {
+                printf "0x%08x => 0x%08x $3\n", $src, $dst;
+            }
+        }
+        elsif (/^Exit from dyngen F[0-9]+\((0x[0-9a-fA-F]+).*targeting +(0x[0-9a-fA-F]+) ([<A-Za-z]:?([->!,+a-zA-Z_~\\0-9\.]| {1}[A-Za-z0-9])+)/) {
+            $dst = hex($1) & 0xfffff000;
+            $src = hex($2) & 0xfffff000;
+            $dll = $3;
+            $found = 0;
+            for ($i=0; $i<$num_exits{$dst}; $i++) {
+                if ($src == $exit_list{$dst}[$i]) {
+                    $exit_count{$dst}[$i]++;
+                    $found = 1;
+                    break;
+                }
+            }
+            if (!$found) {
+                $exit_list{$dst}[$num_exits{$dst}] = $src;
+                $exit_dll{$dst}[$num_exits{$dst}] = $dll;
+                $exit_count{$dst}[$num_exits{$dst}] = 1;
+                $num_exits{$dst}++;
+            }
+            if ($verbose && $dst != $src) {
+                printf "0x%08x => 0x%08x $dll\n", $src, $dst;
+            }
+        }
+        # old log files: /region (0x[0-9a-fA-F]+)-(0x[0-9a-fA-F]+) written @(0x[0-9a-fA-F]+)/
+        elsif (/Exec ([0-9a-fA-F]+)-([0-9a-fA-F]+) written @(0x[0-9a-fA-F]+)/) {
+            $at = hex($3);
+            foreach $s (sort (keys %dg_start)) {
+                if ($at >= $s && $at < $dg_end{$s}) {
+                    $entry_write{$s}++;
+                    break;
+                }
+            }
+        }
+        elsif (/^Write target is actually inside app bb @(0x[0-9a-fA-F]+)/) {
+            $at = hex($3);
+            foreach $s (sort (keys %dg_start)) {
+                if ($at >= $s && $at < $dg_end{$s}) {
+                    $entry_write_frag{$s}++;
+                    break;
+                }
+            }
+        }
+    }
+    close(LOG);
+}
+close(FIND);
+
+foreach $dst (sort (keys %entry_list)) {
+    printf "0x%08x %s\n", $dst, $entry_desc{$dst};
+    if ($entry_write{$dst} > 0) {
+        if (!defined($entry_write_frag{$dst})) {
+            $entry_write_frag{$dst} = 0;
+        }
+        print "  written $entry_write{$dst} times, fragments $entry_write_frag{$dst} times\n";
+    }
+    foreach $addr (sort (keys %prot_start)) {
+        if ($dst >= $addr && $dst < $prot_end{$addr}) {
+            print "  " . $prot_line{$addr};
+            print $prot_stack{$addr};
+        }
+    }
+
+    # hack for sorting: sort list of ints as nice indirection
+    undef @sortme;
+    for ($i=0; $i<$num_entries{$dst}; $i++) {
+        $sortme[$i] = $i;
+    }
+    @sortme = sort { $entry_list{$dst}[$a] <=> $entry_list{$dst}[$b] } @sortme;
+    printf "  entries:\n";
+    for ($i=0; $i<$num_entries{$dst}; $i++) {
+        printf("\t%2d: #%4d  0x%08x $entry_dll{$dst}[$sortme[$i]]\n", $i,
+               $entry_count{$dst}[$sortme[$i]], $entry_list{$dst}[$sortme[$i]]);
+    }
+
+    if (defined($num_exits{$dst})) {
+        printf "  exits:\n";
+        # hack for sorting: sort list of ints as nice indirection
+        undef @sortme;
+        for ($i=0; $i<$num_exits{$dst}; $i++) {
+            $sortme[$i] = $i;
+        }
+        @sortme = sort { $exit_list{$dst}[$a] <=> $exit_list{$dst}[$b] } @sortme;
+        for ($i=0; $i<$num_exits{$dst}; $i++) {
+            printf("\t%2d: #%4d  0x%08x $exit_dll{$dst}[$sortme[$i]]\n", $i,
+                   $exit_count{$dst}[$sortme[$i]], $exit_list{$dst}[$sortme[$i]]);
+        }
+    }
+}
+foreach $dst (sort (keys %exit_list)) {
+    if (!defined($num_entries{$dst})) {
+        # hack for sorting: sort list of ints as nice indirection
+        undef @sortme;
+        for ($i=0; $i<$num_exits{$dst}; $i++) {
+            $sortme[$i] = $i;
+        }
+        @sortme = sort { $exit_list{$dst}[$a] <=> $exit_list{$dst}[$b] } @sortme;
+        printf "0x%08x exits:\n", $dst;
+        for ($i=0; $i<$num_exits{$dst}; $i++) {
+            printf("\t%2d: #%4d  0x%08x $exit_dll{$dst}[$sortme[$i]]\n", $i,
+                   $exit_count{$dst}[$sortme[$i]], $exit_list{$dst}[$sortme[$i]]);
+        }
+    }
+}
+
diff --git a/tools/dos2ux b/tools/dos2ux
new file mode 100644
index 000000000..5809fe5f1
--- /dev/null
+++ b/tools/dos2ux
@@ -0,0 +1,37 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2001 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+while (<STDIN>) {
+    s/
+//;
+    print "$_";
+}
diff --git a/tools/drdel.c b/tools/drdel.c
new file mode 100644
index 000000000..7bba73f50
--- /dev/null
+++ b/tools/drdel.c
@@ -0,0 +1,370 @@
+/* **********************************************************
+ * Copyright (c) 2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* DRdel - clean ASLR or persistent cache files and directories */
+
+/* most of the functionality used in nodemgr should be in share/ while
+ * this tool may be a good place for experimentation and test
+ * scaffolding
+ */
+#include <windows.h>
+#include <assert.h>
+#include <stdio.h>
+
+#include "share.h"
+
+#define DEBUG
+
+
+enum {LAST_STATUS_VALUE_OFFSET = 0xbf4}; /* on Win2000+ case 6789 */
+int
+get_last_status()
+{
+    int laststatus;
+    /* get_own_teb()->LastStatusValue */
+    __asm {
+        mov  eax, fs:[LAST_STATUS_VALUE_OFFSET]
+        mov  laststatus, eax
+    }
+    return laststatus;
+}
+
+
+#ifdef DEBUG
+int verbose = 0;
+
+void
+print_status(bool ok)
+{
+    printf("%s 0x%08x %s %d\n",
+           ok ? "" : "NTSTATUS", 
+           ok ? 0 : get_last_status(), 
+           ok ? "success:" : "GLE", GetLastError());
+}
+#endif
+
+
+int usage(char *us)
+{
+    fprintf(stderr, "\
+Usage: %s\n\
+drdel -f <file> -d <directory> -r \n\
+    -ms <size> target min size available\n\
+    -us <size> target max size used\n\
+    -f <file>  work on one file only\n\
+    -d <directory>  work on a specified directory\n\
+    -r use cache directories from registry\n\
+\n\
+    -c         check if in use and skip\n\
+    -m         mark for deletion when closed\n\
+    -t         .tmp renaming\n\
+    -o         on close delete\n\
+    -b         delete on next boot\n\
+\n\
+    -v         verbose\n\
+\n", us);
+    return 0;
+}
+
+/* FIXME: should move this to share/utils.c */
+/* and merge/reuse or at least cf. with  */
+/* file_exists() */
+/* get_unique_filename() */
+/* delete_file_rename_in_use() */
+/* copy_file_permissions() 
+ * delete_tree()
+ */
+
+DWORD
+is_file_in_use(WCHAR *filename)
+{
+    HANDLE hFile; 
+ 
+    hFile = CreateFile(filename,              // file to open
+                       GENERIC_READ,          // open for reading
+                       0,                     // EXCLUSIVE access, should fail if in use
+                                              // note will fail anyone trying to use at this time
+                       NULL,                  // default security
+                       OPEN_EXISTING,         // existing file only
+                       FILE_ATTRIBUTE_NORMAL, // normal file
+                       NULL);                 // no attr. template
+
+    /* FIXME: is there a better way to check if a file is currently in
+     * use that doesn't get in the way of others.  Admittedly very
+     * short race.  Can we count on the handle properties, e.g. if
+     * QueryObject SYSTEM_OBJECT_INFORMATION.HandleCount and
+     * PointerCount are one number vs another.
+     */
+
+/*
+0:000> !handle 10 f
+Handle 10
+  Type         	File
+  Attributes   	0
+  GrantedAccess	0x120089:
+         ReadControl,Synch
+         Read/List,ReadEA,ReadAttr
+  HandleCount  	2
+  PointerCount 	3
+  No Object Specific Information available
+*/
+
+    if (verbose) {
+        print_status(hFile != INVALID_HANDLE_VALUE);
+    }
+
+    if (hFile == INVALID_HANDLE_VALUE) 
+        return true;            /* in use */
+    CloseHandle(hFile);
+    return false;
+}
+
+
+/* see in utils.c file_exists() that one cannot
+ * open the root directory (and in fact "\\remote\share" as well)
+ */
+bool
+is_file_present(WCHAR *filename)
+{
+    HANDLE hFile; 
+ 
+    hFile = CreateFile(filename,              // file to open
+                       0,                     // just existence check
+                       FILE_SHARE_READ,       // share for reading FIXME: do we need this
+                       NULL,                  // default security
+                       OPEN_EXISTING,         // existing file only
+                       FILE_ATTRIBUTE_NORMAL, // normal file
+                       NULL);                 // no attr. template
+    if (verbose) {
+        print_status(hFile != INVALID_HANDLE_VALUE);
+    }
+
+    if (hFile == INVALID_HANDLE_VALUE) 
+        return false;
+    CloseHandle(hFile);
+    return true;
+}
+
+
+bool
+delete_file_on_close(WCHAR *filename)
+{
+    HANDLE hFile; 
+ 
+    hFile = CreateFile(filename,              // file to open
+                       0,                     // just existence check
+                       FILE_SHARE_READ|FILE_SHARE_DELETE, // share for reading FIXME: do we need this
+                       NULL,                  // default security
+                       OPEN_EXISTING,         // existing file only
+                       FILE_ATTRIBUTE_NORMAL | FILE_FLAG_DELETE_ON_CLOSE, // normal file, delete on close
+                       NULL);                 // no attr. template
+    if (verbose) {
+        print_status(hFile != INVALID_HANDLE_VALUE);
+    }
+
+    if (hFile == INVALID_HANDLE_VALUE) 
+        return false;
+    CloseHandle(hFile);
+    return true;
+}
+
+DWORD
+delete_file(WCHAR *filename)
+{
+    BOOL success = DeleteFile(filename);
+    if (verbose) {
+        print_status(success);
+    }
+
+    if (success)
+        return ERROR_SUCCESS;
+    else {
+        return GetLastError();
+        /* 
+         * For memory mapped files - e.g. after an NtCreateSection() we get
+         * (NTSTATUS) 0xc0000121 (3221225761) - An attempt has been made to
+         * remove a file or directory that cannot be deleted.
+         */
+    }
+}
+
+int check_in_use = 0;
+int delete = 0;
+int temprename = 0;
+int onboot = 0;
+int onclose = 0;
+
+/* 
+ * Possible states of a file
+ * {existing, not existing} x {in use, or not} x {DELETED, or not} 
+ *        x {.used, or not} x {pending reboot removal, or not}
+ * 
+reboot removal - MoveFileEx(MOVEFILE_DELAY_UNTIL_REBOOT) adds to PendingFileRenameOperations 
+HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Session Manager\PendingFileRenameOperations
+
+ * - not existing
+ * - existing, hard link x {in use, not in use}
+ * - existing, sym link x {in use, not in use} (Longhorn)
+ * - existing, not in use
+ * - existing, not in use, previously renamed to .used
+ * - existing, in use
+ * - existing, DELETED, still in use
+ * - existing, in use, renamed to .used
+ * - existing, in use, DELETED, renamed to .used
+ * - existing, in use, DELETED, listed for reboot removal
+ * - existing, in use, DELETED, listed for reboot removal, renamed to .used
+ */
+
+int
+process_file(WCHAR *filename)
+{
+    int in_use = 0;
+    int deleted = 0;
+
+    if (verbose) {
+        printf("processing %S\n", filename);
+
+        if (is_file_present(filename)) {
+            if (is_file_in_use(filename)) {
+                printf("file %ls is in use\n", filename);
+            } else {
+                printf("file %ls exists and is not in use\n", filename);
+            }
+        } else {
+            printf("file %ls doesn't exists\n", filename);
+        }
+    }
+
+    if (check_in_use) {
+        in_use = is_file_in_use(filename);
+        if (verbose && in_use) {
+            printf("file %ls is in use\n", filename);
+        }
+        return 1;
+    }
+
+    if (delete) {
+        deleted = delete_file(filename) == ERROR_SUCCESS;
+    }
+
+    if (onboot) {
+        delete_file_on_boot(filename);
+    }
+
+    if (onclose) {
+        delete_file_on_close(filename);
+    }
+
+    if (!deleted) {
+        if (temprename) {
+            delete_file_rename_in_use(filename);
+            /* note this will also put in PendingFileRenameOperations */
+        }
+    }
+
+    return 0;
+}
+
+int
+process_directory(WCHAR *dirname)
+{
+
+    if (verbose) {
+        printf("delete_tree %S\n", dirname);
+    }
+
+    delete_tree(dirname);
+    return 0;
+}
+
+int
+main(int argc, char *argv[])
+{
+    LPVOID *pv = 0;
+    LPVOID *pc = 0;
+
+    int argidx = 1;
+
+    int file = 0;
+    int dir = 0;
+    WCHAR filebuf[MAX_PATH];
+    WCHAR dirbuf[MAX_PATH];
+
+    if (argc < 2) 
+        usage(argv[0]);
+
+    while (argidx < argc) {
+        if (!strcmp(argv[argidx], "-help")) {
+            usage(argv[0]);
+            exit(0);
+        }
+        else if (!strcmp(argv[argidx], "-f")) {
+            _snwprintf(filebuf, MAX_PATH, L"%S", argv[++argidx]);
+            file = 1;
+        }
+        else if (!strcmp(argv[argidx], "-d")) {
+            _snwprintf(dirbuf, MAX_PATH, L"%S", argv[++argidx]);
+            dir = 1;
+        }
+        else if (!strcmp(argv[argidx], "-m")) {
+            delete = 1;
+        }
+        else if (!strcmp(argv[argidx], "-o")) {
+            onclose = 1;
+        }
+        else if (!strcmp(argv[argidx], "-b")) {
+            onboot = 1;
+        }
+        else if (!strcmp(argv[argidx], "-t")) {
+            temprename = 1;
+        }
+        else if (!strcmp(argv[argidx], "-c")) {
+            check_in_use = 1;
+        }
+        else if (!strcmp(argv[argidx], "-v")) {
+            verbose = 1;
+        }
+        else {
+            fprintf(stderr, "Unknown option: %s\n", argv[argidx]);
+            usage(argv[0]);
+            exit(0);
+        }
+        argidx++;
+    }
+
+    if (file) {
+        process_file(filebuf);
+    }
+    if (dir) {
+        process_directory(dirbuf);
+    }
+}
diff --git a/tools/drdeploy.c b/tools/drdeploy.c
new file mode 100644
index 000000000..b3ae51b2e
--- /dev/null
+++ b/tools/drdeploy.c
@@ -0,0 +1,501 @@
+/* **********************************************************
+ * Copyright (c) 2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* compile with make VMAP=1 for a vmap version (makefile defaults to VMSAFE version) */
+
+#ifdef X64
+#error X64 not yet supported
+#endif
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <io.h>
+#include "globals_shared.h"
+#include "dr_config.h" /* MUST be before share.h (it sets HOT_PATCHING_INTERFACE) */
+#include "config.h"
+#include "share.h"
+
+typedef enum _action_t {
+    action_none,
+    action_nudge,
+    action_register,
+    action_unregister,
+    action_list,
+} action_t;
+
+#define fatal(msg, ...) \
+    fprintf(stderr, "ERROR: " msg "\n", __VA_ARGS__);    \
+    exit(1);
+
+#define warn(msg, ...) \
+    fprintf(stderr, "WARNING: " msg "\n", __VA_ARGS__);
+
+const char *usage_str = 
+    "usage: drdeploy [options]\n"
+    "       -v                 Display version information\n"
+    "       -reg <process>     Register <process> to run under DR\n"
+    "       -unreg <process>   Unregister <process> from running under DR\n"
+    "       -isreg <process>   Display whether <process> is registered and if so its\n"
+    "                          configuration\n"
+    "       -list_registered   Display all registered processes and their configuration\n"
+    "       -root <root>       DR root directory\n"
+#if defined(MF_API) && defined(PROBE_API)
+    "       -mode <mode>       DR mode (code, probe, or security)\n"
+#elif defined(PROBE_API)
+    "       -mode <mode>       DR mode (code or probe)\n"
+#elif defined(MF_API)
+    "       -mode <mode>       DR mode (code or security)\n"
+#else
+    /* No mode argument, is always code. */
+#endif
+    "       -debug             Use the DR debug library\n"
+    "       -32                Target 32-bit or WOW64 applications\n"
+    "       -64                Target 64-bit (non-WOW64) applications\n"
+    "\n"
+    "       -ops \"<options>\"   Additional DR control options.  When specifying\n"
+    "                          multiple options, enclose the entire list of\n"
+    "                          options in quotes.\n"
+    "\n"
+    "       -client <path> <ID> \"<options>\"\n"
+    "                          Register one or more clients to run alongside DR.\n"
+    "                          This option is only valid when registering a\n"
+    "                          process.  The -client option takes three arguments:\n"
+    "                          the full path to a client library, a unique 8-digit\n"
+    "                          hex ID, and an optional list of client options\n"
+    "                          (use \"\" to specify no options).  Multiple clients\n"
+    "                          can be installed via multiple -client options.  In\n"
+    "                          this case, clients specified first on the command\n"
+    "                          line have higher priority.  Neither the path nor\n"
+    "                          the options may contain semicolon characters.\n"
+    "\n"
+    "       Note that nudging 64-bit processes is not yet supported.\n"
+    "       -nudge <process> <client ID> <argument>\n"
+    "                          Nudge the client with ID <client ID> in all running\n"
+    "                          processes with name <process>, and pass <argument>\n"
+    "                          to the nudge callback.  <client ID> must be the\n"
+    "                          8-digit hex ID of the target client.  <argument>\n"
+    "                          should be a hex literal (0, 1, 3f etc.).\n"
+    "       -nudge_pid <process_id> <client ID> <argument>\n"
+    "                          Nudge the client with ID <client ID> in the process with\n"
+    "                          id <process_id>, and pass <argument> to the nudge\n"
+    "                          callback.  <client ID> must be the 8-digit hex ID\n"
+    "                          of the target client.  <argument> should be a hex\n"
+    "                          literal (0, 1, 3f etc.).\n"
+    "       -nudge_all <client ID> <argument>\n"
+    "                          Nudge the client with ID <client ID> in all running\n"
+    "                          processes and pass <argument> to the nudge callback.\n"
+    "                          <client ID> must be the 8-digit hex ID of the target\n"
+    "                          client.  <argument> should be a hex literal\n"
+    "                          (0, 1, 3f etc.)\n"
+    "       -nudge_timeout <ms> Max time (in milliseconds) to wait for a nudge to\n"
+    "                          finish before continuing.  The default is an infinite\n"
+    "                          wait.  A value of 0 means don't wait for nudges to\n"
+    "                          complete.";
+
+#define usage(msg, ...)                                         \
+    fprintf(stderr, "\n");                                      \
+    fprintf(stderr, "ERROR: " msg "\n\n", __VA_ARGS__);         \
+    fprintf(stderr, "%s\n", usage_str);                         \
+    exit(1);
+
+
+/* Unregister a process */
+void unregister_proc(const char *process, dr_platform_t dr_platform)
+{
+    dr_config_status_t status = dr_unregister_process(process, dr_platform);
+    if (status == DR_PROC_REG_INVALID) {
+        fatal("no existing registration");
+    }
+    else if (status == DR_FAILURE) {
+        fatal("unregistration failed");
+    }
+}
+
+
+/* Check if the provided root directory actually has the files we
+ * expect.
+ */
+void check_dr_root(const char *dr_root)
+{
+    int i;
+    char buf[MAX_PATH];
+
+    const char *checked_files[] = {
+        "lib32\\drpreinject.dll",
+        "lib32\\release\\dynamorio.dll",
+        "lib32\\debug\\dynamorio.dll",
+        "lib64\\drpreinject.dll",
+        "lib64\\release\\dynamorio.dll",
+        "lib64\\debug\\dynamorio.dll"
+    };
+
+    for (i=0; i<_countof(checked_files); i++) {
+        sprintf_s(buf, _countof(buf), "%s/%s", dr_root, checked_files[i]);
+        if (_access(buf, 0) == -1) {
+            warn("%s does not appear to be a valid DynamoRIO root", dr_root);
+            return;
+        }
+    }
+}
+
+/* Register a process to run under DR */
+void register_proc(const char *process,
+                   const char *dr_root,
+                   const dr_operation_mode_t dr_mode,
+                   bool debug,
+                   dr_platform_t dr_platform,
+                   const char *extra_ops)
+{
+    dr_config_status_t status;
+
+    if (dr_root == NULL) {
+        usage("you must provide the DynamoRIO root directory");
+    }
+    if (dr_mode == DR_MODE_NONE) {
+        usage("you must provide a DynamoRIO mode");
+    }
+
+    /* If this is the first setting of AppInit on NT, warn about reboot */
+    if (!is_autoinjection_set()) {
+        DWORD platform;
+        if (get_platform(&platform) == ERROR_SUCCESS &&
+            platform == PLATFORM_WIN_NT_4) {
+            warn("on Windows NT, applications will not be taken over until reboot");
+        }
+    }
+
+    if (dr_process_is_registered(process, dr_platform, NULL, NULL, NULL, NULL)) {
+        warn("overriding existing registration");
+        unregister_proc(process, dr_platform);
+    }
+
+    status = dr_register_process(process, dr_root, dr_mode,
+                                 debug, dr_platform, extra_ops);
+
+    if (status != DR_SUCCESS) {
+        /* PR 233108: try to give more info on whether a privilege failure */
+        if (create_root_key() == ERROR_ACCESS_DENIED)
+            warn("insufficient privileges: re-run as administrator");
+        fatal("process registration failed");
+    }
+    
+    /* warn if the DR root directory doesn't look right */
+    check_dr_root(dr_root);
+}
+
+
+/* Check if the specified client library actually exists. */
+void check_client_lib(const char *client_lib)
+{
+    if (_access(client_lib, 0) == -1) {
+        warn("%s does not exist", client_lib);
+    }
+}
+
+
+void register_client(const char *process_name,
+                     dr_platform_t dr_platform,
+                     client_id_t client_id,
+                     const char *path,
+                     const char *options)
+{
+    size_t priority;
+    dr_config_status_t status;
+
+    if (!dr_process_is_registered(process_name, dr_platform,
+                                  NULL, NULL, NULL, NULL)) {
+        fatal("can't register client: process is not registered");
+    }
+
+    check_client_lib(path);
+
+    /* just append to the existing client list */
+    priority = dr_num_registered_clients(process_name, dr_platform);
+
+    status = dr_register_client(process_name, dr_platform, client_id,
+                                priority, path, options);
+
+    if (status != DR_SUCCESS) {
+        fatal("client registration failed");
+    }
+}
+
+static void
+list_process(char *name, dr_platform_t platform, dr_registered_process_iterator_t *iter)
+{
+    char name_buf[MAX_PATH] = {0};
+    char root_dir_buf[MAX_PATH] = {0};
+    dr_operation_mode_t dr_mode;
+    bool debug;
+    char dr_options[DR_MAX_OPTIONS_LENGTH] = {0};
+    dr_client_iterator_t *c_iter;
+
+    if (name == NULL) {
+        dr_registered_process_iterator_next(iter, name_buf, root_dir_buf, &dr_mode,
+                                            &debug, dr_options);
+        name = name_buf;
+    } else if (!dr_process_is_registered(name, platform, root_dir_buf, &dr_mode,
+                                         &debug, dr_options)) {
+        printf("Process %s not registered for %s\n", name,
+               platform == DR_PLATFORM_64BIT ? "64-bit" : "32-bit/WOW64");
+        return;
+    }
+
+    printf("Process %s registered for %s\n", name,
+           platform == DR_PLATFORM_64BIT ? "64-bit" : "32-bit/WOW64");
+    printf("\tRoot=\"%s\" Debug=%s\n\tOptions=\"%s\"\n",
+           root_dir_buf, debug ? "yes" : "no", dr_options);
+    
+    c_iter = dr_client_iterator_start(name, platform);
+    while (dr_client_iterator_hasnext(c_iter)) {
+        client_id_t id;
+        size_t client_pri;
+        char client_path[MAX_PATH] = {0};
+        char client_opts[DR_MAX_OPTIONS_LENGTH] = {0};
+        
+        dr_client_iterator_next(c_iter, &id, &client_pri, client_path, client_opts);
+        
+        printf("\tClient=0x%08x Priority=%d\n\t\tPath=\"%s\"\n\t\tOptions=\"%s\"\n",
+               id, (uint)client_pri, client_path, client_opts);
+    }
+    dr_client_iterator_stop(c_iter);
+}
+
+
+int main(int argc, char *argv[])
+{
+    char *process = NULL;
+    char *dr_root = NULL;
+    char *client_paths[MAX_CLIENT_LIBS] = {NULL,};
+    char *client_options[MAX_CLIENT_LIBS] = {NULL,};
+    client_id_t client_ids[MAX_CLIENT_LIBS] = {0,};
+    size_t num_clients = 0;
+#if defined(MF_API) || defined(PROBE_API)
+    /* must set -mode */
+    dr_operation_mode_t dr_mode = DR_MODE_NONE;
+#else
+    /* only one choice so no -mode */
+    dr_operation_mode_t dr_mode = DR_MODE_CODE_MANIPULATION;
+#endif
+    char *extra_ops = NULL;
+    action_t action = action_none;
+    bool use_debug = false;
+    dr_platform_t dr_platform = DR_PLATFORM_DEFAULT;
+    bool nudge_all = false;
+    process_id_t nudge_pid = 0;
+    client_id_t nudge_id = 0;
+    uint64 nudge_arg = 0;
+    bool list_registered = false;
+    uint nudge_timeout = INFINITE;
+    
+    int i;
+
+    /* parse command line */
+    for (i=1; i<argc; i++) {
+        if (strcmp(argv[i], "-debug") == 0) {
+            use_debug = true;
+            continue;
+        }
+        else if (!strcmp(argv[i], "-v")) {
+#if defined(BUILD_NUMBER) && defined(VERSION_NUMBER)
+          printf("drdeploy.exe version %s -- build %d\n", STRINGIFY(VERSION_NUMBER), BUILD_NUMBER);
+#elif defined(BUILD_NUMBER)
+          printf("drdeploy.exe custom build %d -- %s\n", BUILD_NUMBER, __DATE__);
+#else
+          printf("drdeploy.exe custom build -- %s, %s\n", __DATE__, __TIME__);
+#endif
+          exit(0);
+        } else if (!strcmp(argv[i], "-list_registered")) {
+            action = action_list;
+            list_registered = true;
+            continue;
+        }
+        /* all other flags have an argument -- make sure it exists */
+        else if (i == argc - 1) {
+            usage("invalid arguments");
+        }
+
+        if (strcmp(argv[i], "-reg") == 0) {
+            if (action != action_none) {
+                usage("more than one action specified");
+            }
+            action = action_register;
+            process = argv[++i];
+        }
+        else if (strcmp(argv[i], "-unreg") == 0) {
+            if (action != action_none) {
+                usage("more than one action specified");
+            }
+            action = action_unregister;
+            process = argv[++i];
+        }
+        else if (strcmp(argv[i], "-isreg") == 0) {
+            if (action != action_none) {
+                usage("more than one action specified");
+            }
+            action = action_list;
+            process = argv[++i];
+        }
+        else if (strcmp(argv[i], "-nudge_timeout") == 0) {
+            nudge_timeout = strtoul(argv[++i], NULL, 10);
+        }
+        else if (strcmp(argv[i], "-nudge") == 0 ||
+                 strcmp(argv[i], "-nudge_pid") == 0 ||
+                 strcmp(argv[i], "-nudge_all") == 0){
+            if (action != action_none) {
+                usage("more than one action specified");
+            }
+            if (i + 2 >= argc || 
+                (strcmp(argv[i], "-nudge_all") != 0 && i + 3 >= argc)) {
+                usage("too few arguments to -nudge");
+            }
+            action = action_nudge;
+            if (strcmp(argv[i], "-nudge") == 0)
+                process = argv[++i];
+            else if (strcmp(argv[i], "-nudge_pid") == 0)
+                nudge_pid = strtoul(argv[++i], NULL, 10);
+            else
+                nudge_all = true;
+            nudge_id = strtoul(argv[++i], NULL, 16);
+            nudge_arg = _strtoui64(argv[++i], NULL, 16);
+        }        
+        else if (strcmp(argv[i], "-root") == 0) {
+            dr_root = argv[++i];
+        }
+#if defined(MF_API) || defined(PROBE_API)
+        else if (strcmp(argv[i], "-mode") == 0) {
+            char *mode_str = argv[++i];
+            if (strcmp(mode_str, "code") == 0) {
+                dr_mode = DR_MODE_CODE_MANIPULATION;
+            }
+# ifdef MF_API
+            else if (strcmp(mode_str, "security") == 0) {
+                dr_mode = DR_MODE_MEMORY_FIREWALL;
+            }
+# endif
+# ifdef PROBE_API
+            else if (strcmp(mode_str, "probe") == 0) {
+                dr_mode = DR_MODE_PROBE;
+            }
+# endif
+            else {
+                usage("unknown mode: %s", mode_str);
+            }
+        }
+#endif
+        else if (strcmp(argv[i], "-client") == 0) {
+            if (num_clients == MAX_CLIENT_LIBS) {
+                fatal("Maximum number of clients is %d", MAX_CLIENT_LIBS);
+            }
+            else {
+                if (i + 3 >= argc) {
+                    usage("too few arguments to -client");
+                }
+
+                client_paths[num_clients] = argv[++i];
+                client_ids[num_clients] = strtoul(argv[++i], NULL, 16);
+                client_options[num_clients] = argv[++i];
+                num_clients++;
+            }
+        }
+        else if (strcmp(argv[i], "-ops") == 0) {
+            extra_ops = argv[++i];
+	}
+        else if (strcmp(argv[i], "-32") == 0) {
+	    dr_platform = DR_PLATFORM_32BIT;
+	}
+        else if (strcmp(argv[i], "-64") == 0) {
+	    dr_platform = DR_PLATFORM_64BIT;
+        }
+        else {
+            usage("unknown option: %s", argv[i]);
+        }
+    }
+
+    /* PR 244206: set the registry view before any registry access */
+    set_dr_platform(dr_platform);
+
+    if (action == action_register) {
+        size_t i;
+        register_proc(process, dr_root, dr_mode, use_debug, dr_platform, extra_ops);
+        for (i=0; i<num_clients; i++) {
+            register_client(process, dr_platform, client_ids[i],
+                            client_paths[i], client_options[i]);
+        }
+    }
+    else if (action == action_unregister) {
+        unregister_proc(process, dr_platform);
+    }
+    else if (action == action_nudge) {
+        int count = 1;
+        dr_config_status_t res = DR_SUCCESS;
+        if (nudge_all)
+            res = dr_nudge_all(nudge_id, nudge_arg, nudge_timeout, &count);
+        else if (nudge_pid != 0) {
+            res = dr_nudge_pid(nudge_pid, nudge_id, nudge_arg, nudge_timeout);
+            if (res == DR_NUDGE_PID_NOT_INJECTED)
+                printf("process %d is not running under DR\n", nudge_pid);
+            if (res != DR_SUCCESS && res != DR_NUDGE_TIMEOUT) {
+                count = 0;
+                res = ERROR_SUCCESS;
+            }
+        } else
+            res = dr_nudge_process(process, nudge_id, nudge_arg, nudge_timeout, &count);
+
+        printf("%d processes nudged\n", count);
+        if (res == DR_NUDGE_TIMEOUT)
+            printf("timed out waiting for nudge to complete");
+        else if (res != DR_SUCCESS)
+            printf("nudge operation failed, verify adequate permissions for this operation.");
+    }
+    else if (action == action_list) {
+        if (!list_registered)
+            list_process(process, dr_platform, NULL);
+        else /* list all */ {
+            dr_registered_process_iterator_t *iter =
+                dr_registered_process_iterator_start(dr_platform);
+            printf("Registered processes for %s\n", dr_platform == DR_PLATFORM_64BIT ?
+                   "64-bit" : "32-bit/WOW64");
+            while (dr_registered_process_iterator_hasnext(iter))
+                list_process(NULL, dr_platform, iter);
+            dr_registered_process_iterator_stop(iter);
+        }
+    }
+    else  {
+        usage("no action specified");
+    }
+
+    return 0;
+}
diff --git a/tools/drdeploy.exe.manifest b/tools/drdeploy.exe.manifest
new file mode 100644
index 000000000..5e60d1133
--- /dev/null
+++ b/tools/drdeploy.exe.manifest
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0"> 
+   <assemblyIdentity version="1.0.0.0" processorArchitecture="X86" name="drdeploy" type="win32"/>
+      <trustInfo xmlns="urn:schemas-microsoft-com:asm.v3">
+      <security>
+         <requestedPrivileges>
+            <requestedExecutionLevel level="requireAdministrator"/> 
+         </requestedPrivileges>
+      </security>
+   </trustInfo>
+</assembly>
diff --git a/tools/drview-analyze.pl b/tools/drview-analyze.pl
new file mode 100755
index 000000000..89c3d2863
--- /dev/null
+++ b/tools/drview-analyze.pl
@@ -0,0 +1,187 @@
+#!/usr/bin/perl -w
+
+# **********************************************************
+# Copyright (c) 2005 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# drview-analyze.pl
+#    analysis of continous drview snapshosts
+#
+# assuming DRview output of
+# Name          PID DR  Bld CPU User  Hndl Thr    PVSz     VSz    PPriv    Priv    PWSS     WSS   Fault PPage  Page PNonP  NonP
+# System          8 ?    -1  0%   0%   353  55    5836    5772      640      36     856     232   48566     0     0     0     0
+# smss.exe      216 N    -1  0%   0%    33   6    8456    4296     1720     156    1972     400     999    13     5    15     1
+#
+
+# takes variable and current value, 
+# returns result in global last and delta
+sub calculate_delta
+{
+    my ($var, $value) = @_;
+
+    if (!$last{$exe,$pid,$var}) {
+        $last{$exe,$pid,$var} = 0;
+        $delta{$var} = 0;
+    } else {
+        $delta{$var} = $value - $last{$exe,$pid,$var};
+    }
+    $last{$exe,$pid,$var} = $value;
+
+    if (!$churn{$exe,$pid,$var}) {
+        $churn{$exe,$pid,$var} = 0;
+    }
+
+    # we should count only positive values
+    # this way monotonically growing functions will also 
+    if ($delta{$var} > 0 ) {
+        $churn{$exe,$pid,$var} += $delta{$var};
+    }
+
+#TODO: count delta edges - so we know how many flex points we've had
+
+}
+
+$showall = 0;
+
+sub header
+{
+
+    if (!$showall) {
+        print "Process       PID     Threads             WorkSet/KB                 Page Faults            PrivateVSz/KB\n";
+        print "                  cur dlt  chrn     cur     dlt     churn       cum     dlt      chrn     cur     dlt      chrn\n";
+# For current value metrics like threads and workset delta is last difference, churn is total positive difference
+# For cumulative value metrics like page faults, churn is simply the difference from the starting point
+    }
+}
+
+header();
+
+$errors = 0;
+$ignored = 0;
+$verbose = 0;
+
+while ($line = <>) {
+    chomp ($line);
+
+    if ($line =~ "    Could not open.*") {
+        $errors++;
+    }
+    elsif ($line =~ "--------------------------------------------------") {
+        $ignored++;
+    }
+    elsif ($line eq "\r") { # blank
+        $ignored++;
+    }
+    elsif ($line eq "") { # blank
+        $ignored++;
+    }
+    elsif ($line =~ /\w+ \w+ \d+ (\d+):(\d+):(\d+) (\d+)/) {
+        # Wed Mar 23 10:27:26 2005
+        $hour = $1;
+        $min = $2;
+        $sec = $3;
+        # TODO: need smaller units as well
+        if ($verbose) {
+            print "$hour:$min:$sec\n";
+        }
+    } 
+    elsif ($line =~ /System load is -?(\d+)%/) {
+        # TODO: fix the -
+        $sysload = $1;
+        if ($verbose) {
+            print "Sysload $sysload\n";
+        }
+    } 
+    elsif ($line =~
+/Name          PID DR  Bld CPU User  Hndl Thr    PVSz     VSz    PPriv    Priv    PWSS     WSS   Fault PPage  Page PNonP  NonP/
+               ) {
+        $newset = 1;
+    }
+    else {
+        if ($newset) {
+            print "\n";
+        }
+        
+        ($exe, $pid, $underdr, $build, 
+         $cpuPCT, $userPCT, 
+         $handles, $threads, 
+         $peak_virtsz, $virtsz, 
+         $peak_privsz, $privsz, 
+         $peak_workset, $workset, 
+         $faults, 
+         $peak_paged_pool, $paged_pool, 
+         $peak_nonpaged_pool, $nonpaged_pool)
+            = split(/\s+/, $line);
+
+        # TODO: make sure nonpaged_pool is indeed a number to make sure this is a complete line
+
+#    print "$exe, $pid, $underdr, $build, $cpuPCT, $userPCT, $handles, $threads, $peak_virtsz, $virtsz, $peak_privsz, $privsz, ";
+#    print "$peak_workset, $workset, $faults, $peak_paged_pool, $paged_pool, $peak_nonpaged_pool, $nonpaged_pool\n";
+
+#    /* single line best so can line up columns and process output easily */
+        if ($showall) {
+            printf "%-11.11s %5d %s %5d %s %s %5d %3d %7d %7d %8d %7d %7d %7d %7d %5d %5d %5d %5d\n",
+            $exe, $pid, $underdr, $build, $cpuPCT, $userPCT, $handles, $threads, $peak_virtsz, $virtsz, $peak_privsz, $privsz,
+            $peak_workset, $workset, $faults, $peak_paged_pool, $paged_pool, $peak_nonpaged_pool, $nonpaged_pool;
+        }
+
+# extract Thread handle rate and calculate difference
+
+        calculate_delta('threads', $threads);
+        calculate_delta('workset', $workset);
+        calculate_delta('faults', $faults);
+        calculate_delta('privsz', $privsz);
+
+        printf "%-11.11s %5d", $exe, $pid;
+        printf " %3d", $threads;
+        printf " %3d", $delta{'threads'};
+        printf " %5d", $churn{$exe,$pid,'threads'};
+
+        printf " %7d", $workset;
+        printf " %7d", $delta{'workset'};
+        printf " %9d", $churn{$exe,$pid,'workset'};
+
+        printf " %9d", $faults;
+        printf " %7d", $delta{'faults'};
+        printf " %9d", $churn{$exe,$pid,'faults'};
+
+        printf " %7d", $privsz;
+        printf " %7d", $delta{'privsz'};
+        printf " %9d", $churn{$exe,$pid,'privsz'};
+#TODO: print all other data - don't lose information!
+
+        print "\n";
+        $newset = 0;
+    }
+}
+
+header();
+
+# with absolute of negatives
+#winmgmt.exe  1880  10   0  121   3132   -808   409996 5559853   3943  662978
diff --git a/tools/dummy.c b/tools/dummy.c
new file mode 100644
index 000000000..bd7192dc3
--- /dev/null
+++ b/tools/dummy.c
@@ -0,0 +1,40 @@
+/* **********************************************************
+ * Copyright (c) 2004 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+
+/* does nothing */
+DWORD
+main(DWORD argc, char *argv[])
+{
+    return 0;
+}
diff --git a/tools/dynamorio b/tools/dynamorio
new file mode 100755
index 000000000..18eddf22f
--- /dev/null
+++ b/tools/dynamorio
@@ -0,0 +1,241 @@
+#!/bin/sh
+
+# **********************************************************
+# Copyright (c) 2002-2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# Copyright (c) 2001-2002 Massachusetts Institute of Technology
+# Copyright (c) 2001-2002 Hewlett-Packard Company
+
+###########################################################################
+##
+## DynamoRIO injector script for Linux
+## 
+## usage: drdeploy [-v] [-debug] [-dr_home <path>] [-ops <dr_options>]
+##                 [-logdir <path>] -client <path> <executable> [args ...]
+##
+##  [-v]                   Verbose output
+##  [-debug]               Use debug build of DynamoRIO (release is default)
+##  [-64]                  Use 64-bit DynamoRIO (default is 32-bit).  This
+##                           should match what the client library and executable
+##                           being run were compiled for.
+##  [-dr_home <path>]      Path to the DynamoRIO install.
+##                           Defaults to .. from drdeploy; may also be omitted
+##                           if DYNAMORIO_HOME is defined in the environment.
+##  [-ops <dr_options>]    DynamoRIO options (such as -stderr_mask 0xc
+##                           or -opt_memory: see docs). Can also be specified
+##                           with DYNAMORIO_OPTIONS in the environment.
+##  [-logdir <path>]       Path to logging directory.  Defaults to ../logs from
+##                           drdeploy; may also be omitted if DYNAMORIO_LOGDIR
+##                           is defined in the environment.
+##  [-client <path> <ID> "<options>"]
+##                         Register one or more clients to run alongside DR.
+##                           This option is only valid when registering a
+##                           process.  The -client option takes three arguments:
+##                           the full path to a client library, a unique 8-digit
+##                           hex ID, and an optional list of client options
+##                           (use "" to specify no options).  Multiple clients
+##                           can be installed via multiple -client options.  In
+##                           this case, clients specified first on the command
+##                           line have higher priority.  Neither the path nor
+##                           the options may contain semicolon characters.
+##  <executable> [args...] The executable to run under DynamoRIO and the args
+##                           to pass to it.
+
+lib=release
+arch=32
+
+usage="usage: $0 [-v] [-debug] [-dr_home <path>] [-ops <dr_options>] [-logdir <path>] [-client <path> <ID> \"<options>\"]* <executable> [args ...]"
+
+default_home_rel="`dirname $0`/.."
+default_home=`cd $default_home_rel ; pwd`
+client_list=""
+
+if [ "$1" = "" ] || [ "$1" = "-h" ] || [ "$1" = "-help" ] || [ "$1" = "--help" ]
+then
+    echo $usage
+    echo
+    head -n 44 $0 | tail -n 36
+    exit
+fi
+
+# Read options
+while [ "$1" = "-v" ] || [ "$1" = "-debug" ] || [ "$1" = "-dr_home" ] || \
+      [ "$1" = "-ops" ] || [ "$1" = "-logdir" ] || [ "$1" = "-client" ] || \
+      [ "$1" = "-64" ]
+do
+
+if [ "$1" = "-v" ] 
+then
+    verbose=yes
+    shift
+fi
+
+if [ "$1" = "-debug" ]
+then
+    lib=debug
+    shift
+fi
+
+if [ "$1" = "-64" ]
+then
+    arch=64
+    shift
+fi
+
+if [ "$1" = "-dr_home" ]
+then
+    shift
+    DYNAMORIO_HOME="$1"
+    shift
+fi
+
+if [ "$1" = "-ops" ]
+then
+    shift
+    specified_ops=yes
+    ops=$1
+    shift
+fi
+
+if [ "$1" = "-logdir" ]
+then
+    shift
+    logdir=$1
+    shift
+fi
+
+if [ "$1" = "-client" ]
+then
+    shift
+    client="$1"
+    shift
+    client_id="$1"
+    shift
+    client_ops="$1"
+    shift
+    # setup client
+    if [ ! -n "$client" ] || [ ! -n "$client_id" ]
+    then
+        echo "ERROR - client not fully specified"
+        echo $usage
+        exit
+    fi
+    if [ ! -e $client ]
+    then
+        echo "ERROR - client lib $client not found"
+        echo $usage
+        exit
+    fi
+    # convert to absolute path
+    client=`cd \`dirname $client\` ; pwd`/`basename $client`
+    # add to list now in case multiple
+    if [ -n "$client_list" ]
+    then
+        client_list="${client_list};${client};${client_id};${client_ops}"
+    else
+        client_list="${client};${client_id};${client_ops}"
+    fi
+fi
+
+done
+
+# Setup lib path
+if [ ! -n "$dr_lib_path" ]
+then
+    if [ -n "$DYNAMORIO_HOME" ]
+    then
+        dr_lib_path=$DYNAMORIO_HOME/lib$arch
+    else
+        dr_lib_path=$default_home/lib$arch
+    fi
+fi
+if [ ! -e $dr_lib_path/$lib/libdynamorio.so ]
+then
+    echo "ERROR - libdynamorio.so not found in $dr_lib_path/$lib"
+    echo $usage
+    exit
+fi
+# convert to absolute path
+dr_lib_path=`cd $dr_lib_path ; pwd`
+
+if [ ! -n "$*" ]
+then
+    echo "ERROR - no application specified"
+    echo $usage
+    exit
+fi
+
+# setup logdir
+if [ ! -n "$logdir" ]
+then
+    if [ -n "$DYNAMORIO_LOGDIR" ]
+    then
+        logdir=$DYNAMORIO_LOGDIR
+    else
+        logdir=$default_home/logs
+    fi
+fi
+if [ ! -d $logdir ]
+then
+    echo "ERROR - logdir $logdir not found"
+    echo $usage
+    exit
+fi
+# convert to absolute path and export
+export DYNAMORIO_LOGDIR=`cd $logdir ; pwd`
+
+# client options
+if [ -n "$client_list" ]
+then
+    client_op_string="-code_api -client_lib \"$client_list\""
+else
+    client_op_string=""
+fi
+
+# setup options
+if [ "$specified_ops" = "yes" ]
+then
+    export DYNAMORIO_OPTIONS="${client_op_string} $ops"
+else
+    export DYNAMORIO_OPTIONS="${client_op_string} $DYNAMORIO_OPTIONS"
+fi
+
+if [ "$verbose" = "yes" ]
+then
+    echo "Using $lib DynamoRIO ($dr_lib_path/$lib/libdynamorio.so)"
+    echo "With DynamoRIO options=\"$DYNAMORIO_OPTIONS\""
+    echo "Using logging directory=\"$DYNAMORIO_LOGDIR\""
+    echo "Running \"$*\""
+fi
+
+export LD_LIBRARY_PATH=$dr_lib_path:$dr_lib_path/$lib:$LD_LIBRARY_PATH
+export LD_PRELOAD="libdynamorio.so libdrpreload.so"
+
+$*
diff --git a/tools/extract-kstats.pl b/tools/extract-kstats.pl
new file mode 100755
index 000000000..3309be7c6
--- /dev/null
+++ b/tools/extract-kstats.pl
@@ -0,0 +1,118 @@
+#!/usr/bin/perl -w
+
+# **********************************************************
+# Copyright (c) 2005-2007 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### extract-stats.pl
+###
+### Summarize data gathered from DR logs under the various loglevels
+
+
+$usage = "Usage: $0 <kstats file>\n";
+
+if ($#ARGV + 1 == 1) {
+    $logfile = $ARGV[0];
+}
+
+if (!defined($logfile)) {
+    print $usage;
+    exit;
+}
+
+open(PROFILE, "< $logfile") || die "Error opening $logfile\n";
+
+print "Kstats from logfile $logfile\n(All times are in msec)\n";
+
+@array = ();
+$total_time = 0;
+while (<PROFILE>) {
+
+    chop($_);
+
+    if (/[ ]*(\w+):[ ]*(\d+) totc/ && /,[ ]*(\d+) self,/) {
+        /[ ]*(\w+):[ ]*(\d+) totc/;
+        $node = $1;
+        $tot_time = $2;
+        /,[ ]*(\d+) self,/;
+        $self_time = $1;
+        $array[$#array + 1] = "$node $tot_time $self_time";
+        if ($node eq "thread_measured") {
+            $total_time = $tot_time;
+        }
+    }
+
+}
+close(PROFILE);
+
+# tot time is the 2nd element in the entry
+sub by_tot {
+    ($trash1, $a_time, $trash2) = split / /, $a;
+    ($trash1, $b_time, $trash2) = split / /, $b;
+    $b_time <=> $a_time;
+}
+
+# self time is the 3rd element in the entry
+sub by_self {
+    ($trash1, $trash2, $a_time) = split / /, $a;
+    ($trash1, $trash2, $b_time) = split / /, $b;
+    $b_time <=> $a_time;
+}
+
+@sorted_kstats = sort by_self @array;
+if ($#sorted_kstats > 0) {
+    printf "\nSorted by self time\n";
+    printf "%50s  %15s  %10s\n", "Node", "Time", "%-age";
+    printf "%50s  %15s  %10s\n", "====", "====", "=====";
+    for ($i = 0; $i <= $#sorted_kstats; $i++) {
+        ($node, $trash, $time) = split / /, $sorted_kstats[$i];
+        printf "%50s  %15.0f", $node, $time;
+        $percent = 0;
+        if ($total_time > 0) {
+            $percent = 100 * $time / $total_time;
+        }
+        printf "  %10.2f\n", $percent;
+    }
+}
+
+@sorted_kstats = sort by_tot @array;
+if ($#sorted_kstats > 0) {
+    printf "\nSorted by total time\n";
+    printf "%50s  %15s  %10s\n", "Node", "Time", "%-age";
+    printf "%50s  %15s  %10s\n", "====", "====", "=====";
+    for ($i = 0; $i <= $#sorted_kstats; $i++) {
+        ($node, $time, $trash) = split / /, $sorted_kstats[$i];
+        printf "%50s  %15.0f", $node, $time;
+        $percent = 0;
+        if ($total_time > 0) {
+            $percent = 100 * $time / $total_time;
+        }
+        printf "  %10.2f\n", $percent;
+    }
+}
diff --git a/tools/extract-profile.pl b/tools/extract-profile.pl
new file mode 100755
index 000000000..9f8328746
--- /dev/null
+++ b/tools/extract-profile.pl
@@ -0,0 +1,387 @@
+#!/usr/bin/perl -w
+
+# **********************************************************
+# Copyright (c) 2004-2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### extract-profile.pl
+###
+### Summarize profile info gathered from DR-based PC sampling
+
+$verbose = 0;
+$dr_profile = 0;
+$dr_dll_hits = 0;
+$export_addresses = 0;
+$EXPORT = "exported-addresses";
+$dump_addresses = 0;
+$DUMP = "dumped-addresses";
+$three_gigs = 0;
+$summary = 0;
+
+# addressquery.pl can handle ~150 address max so bound the
+# value of export_limit;
+$export_limit = 150;
+
+$usage = "Usage: $0 [-h] [-drprofile] [-num_addrs <num>] <profile file>\n" .
+    "   -summary      Produce a summary of code cache hit rate & thread count\n" .
+    "   -drprofile    Writes the addresses sampled within the DR DLL to files\n" .
+    "                 named $EXPORT and $DUMP.\n" .
+    "                 $EXPORT is in a format suitable for the\n" .
+    "                 '-f' arg of the address_query script. $DUMP is\n" .
+    "                 in a format suitable for the '-address_file' arg to the\n" .
+    "                 extract-samples script.\n" .
+    "   -num_addrs <num>\n" .
+    "                 Specify the max # of addresses to write to the\n" .
+    "                 $EXPORT and $DUMP files. The current\n" .
+    "                 max is $export_limit as address_query struggles with a\n" .
+    "                 larger # of addresses.\n" .
+    "   -3GB          Assume user address space is 3GB.\n";
+
+
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq "-v") {
+	$verbose = 1;
+    } elsif ($ARGV[0] eq "-drprofile") {
+        $dr_profile = 1;
+        $export_addresses = 1;
+        $dump_addresses = 1;
+        @dr_profile_array = ();
+    } elsif ($ARGV[0] eq "-num_addrs") {
+        shift;
+        # addressquery.pl can handle ~150 address max so bound the
+        # value of export_limit;
+        $export_limit = $ARGV[0] > 150 ? 150 : $ARGV[0];
+    } elsif ($ARGV[0] eq "-summary") {
+        $summary = 1;
+    } elsif ($ARGV[0] eq "-3GB") {
+        $three_gigs = 1;
+    } elsif ($ARGV[0] eq "-h") {
+        print $usage;
+        exit;
+    } else {
+	$infile = $ARGV[0];
+    }
+    shift;
+}
+if (!defined($infile)) {
+    print $usage;
+    exit;
+}
+
+open(PROFILE, "< $infile") or die "Error opening $infile\n";
+
+$marker_count = 0;
+$DR_dll = "dynamorio.dll";
+
+# This is where new entries should be added. Simply add the string that
+# is printed in the log file suffixed with the '=> $marker_count++' glob.
+%marker_hash =
+    (
+     # The IBL profile region strings need to be kept in sync with the labels
+     # that are dumped in x86/arch.c/arch_process_profile().
+     "trace IBL code ret" => $marker_count++,
+     "trace IBL code indcall" => $marker_count++,
+     "trace IBL code indjmp" => $marker_count++,
+     "BB IBL code ret" => $marker_count++,
+     "BB IBL code indcall" => $marker_count++,
+     "BB IBL code indjmp" => $marker_count++,
+     "coarse IBL code ret" => $marker_count++,
+     "coarse IBL code indcall" => $marker_count++,
+     "coarse IBL code indjmp" => $marker_count++,
+
+     "cache enter/exit code" => $marker_count++,
+     "fcache bb unit" => $marker_count++,
+     "fcache trace unit" => $marker_count++,
+     "special heap unit" => $marker_count++,
+     "generated code" => $marker_count++,
+     $DR_dll => $marker_count++,
+     "kernel32.dll" => $marker_count++,
+     "ntdll.dll" => $marker_count++,
+     "global" => $marker_count++,
+     );
+
+# Init the counters
+for ($i = 0; $i < $marker_count; $i++) {
+    $counter_array[ $i ] = 0;
+}
+
+$reported_hits = 1;
+$kernel_hits_one = 0;
+$kernel_hits_two = 0;
+$false_kernel_hits = 0;
+$global_profile = 0;
+$thread_count = 0;
+$thread_with_hits = 0;
+$dll_base = 0;
+$delta = 0;
+$dr_pref_base = hex("0x15000000");
+while (<PROFILE>) {
+
+    chop($_);
+
+    if ($verbose) {
+        print STDERR "Process $.: <$_>\n";
+    }
+
+    if (/^.*built with: -D.*-DDEBUG/) {
+        $dr_pref_base = hex("0x15000000");
+    }
+
+    if (/^Profile for thread (\d+)/) {
+        $thread_count++;
+    }
+    $found = 0;
+    while (($key, $value) = each %marker_hash) {
+        if ($found) {
+            if ($verbose) {
+                print STDERR "Tag match already found skip...\n";
+            }
+            next;
+        }
+        if (/^Dumping ${key} profile/) {
+
+            if ($verbose) {
+                print STDERR "Match for $key\n";
+            }
+
+            # Track # threads
+            if (/Thread (\d+)/ || /thread (\d+)/) {
+                # Have we seen this thread before?
+                if (!defined($thread_hash{$1})) {
+                    $thread_hash{$1} = 1;
+                    $thread_with_hits++;
+                }
+            }
+            
+            # Read the next line
+            $_ = <PROFILE>;
+            if (/^(\d+)[ ]hits/) {
+                $count = $1;
+                if ($verbose) {
+                    print STDERR "Found hit info -- $count\n";
+                }
+            } else {
+                print "Expecting hit info, found $_\n";
+                exit;
+            }
+            if ($key eq "global") {
+                if ($verbose) {
+                    print STDERR "Found global profile $_...\n";
+                }
+                $global_profile = 1;
+                $reported_hits = $count;
+                if ($verbose) {
+                    print STDERR "Reported hits $reported_hits\n";
+                }
+                # Read the next 3 lines, which are like:
+                # Profile Dump
+                # Range 0x00000000-0xffffffff
+                # Step 0x40000000 (0-3)
+                $_ = <PROFILE>;
+                $_ = <PROFILE>;
+                $_ = <PROFILE>;
+                # Validate the step.
+                if (!(/^Step 0x40000000 \(0-3\)/)) {
+                    die "Error: unexpected step in global profile\n";
+                }
+                # Read & process the next 4 lines, which are like:
+                # 0x00000000       1821
+                # 0x40000000       7570
+                # 0x80000000        671
+                # 0xc0000000         99
+                # But may not be present if no hits!
+                while ($_ !~ /^0x8/ && $_ !~ /0xc/) {
+                    last if (!($_ = <PROFILE>));
+                    last if (/^Finished/);
+                }
+                # Read the first set of possible kernel space hits
+                if (!$three_gigs) {
+                    if (/^0x80000000[ ]+(\d+)/) {
+                        $kernel_hits_one = $1;
+                    } else {
+                        $kernel_hits_one = 0;
+                    }
+                }
+                $_ = <PROFILE>;
+                # Read the second set of possible kernel space hits
+                if (/^0xc0000000[ ]+(\d+)/) {
+                    $kernel_hits_two = $1;
+                } else {
+                    $kernel_hits_two = 0;
+                }
+            }
+            else {
+                $counter_array[$value] += $count;
+                # Read the next 2 lines, which are like:
+                # Profile Dump
+                # Range 0x85331000-0x8533efff
+                $_ = <PROFILE>;
+                $_ = <PROFILE>;
+                if (!($key =~ /dll/)) {
+                    if (/^Range 0x[89abcdef]/) {
+                        $false_kernel_hits += $count;
+                    }
+                }
+                else {
+                    if (/^Range 0x([\d|a-f]+)-0x([\d|a-f]+)/) {
+                        $dll_base = $1;
+                    }
+                }
+                if ($dr_profile && $key eq "dynamorio.dll") {
+                    $delta = hex($dll_base) - $dr_pref_base;
+                    do {
+                        my $hit_addr = "";
+                        $_ = <PROFILE>;
+                        if (/^0x([\d|a-f]+)[ ]+(\d+)/) {
+                            $hit_addr = sprintf("%x", hex($1) - $delta);
+                            $dr_profile_array[$#dr_profile_array + 1] = "$hit_addr $2";
+                            $dr_dll_hits += $2;
+                        }
+                    } while ( !(/^Finished Profile Dump/) );
+                }
+            }
+
+            # We don't 'last' out of the loop because we'd have
+            # to reset the hash iter using 'keys' on the next
+            # key match. And keys could be a sort underneath, which
+            # is costlier than continuing the loop, which is
+            # linear.
+            $found = 1;
+        }
+    }
+}
+
+if (!$global_profile) {
+    print "Global profile dump not found -- no summary available\n";
+    exit;
+}
+
+sub by_hits {
+    ($trash, $a_hits) = split / /, $a;
+    ($trash, $b_hits) = split / /, $b;
+    $b_hits <=> $a_hits;
+}
+
+if ($dr_profile && $#dr_profile_array > 0) {
+
+    @sorted_by_hits = sort by_hits @dr_profile_array;
+    if ($export_addresses) {
+        open(EXPORT, "> $EXPORT") or
+            die "Error opening address export file\n";
+    }
+    if ($dump_addresses) {
+        open(DUMP, "> $DUMP") or
+            die "Error opening address dump file\n";
+    }
+    print "Writing up to $export_limit addresses to $EXPORT and $DUMP\n";
+    for ($i = 0; $i <= $#sorted_by_hits; $i++) {
+        if ($i == $export_limit) {
+            last;
+        }
+        ($address, $hits) = split / /, $sorted_by_hits[$i];
+        if ($export_addresses) {
+            print EXPORT "$address\n";
+        }
+        if ($dump_addresses) {
+            printf DUMP "$address %8d %4.2f\n", $hits, 100 * $hits/$dr_dll_hits;
+        }
+    }
+    if ($export_addresses) {
+        close(EXPORT);
+    }
+    if ($dump_addresses) {
+        close(DUMP);
+    }
+}
+
+if ($thread_count == 0) {
+    # Must be an old core that produced the profile.
+    $thread_count = $thread_with_hits;
+}
+if (!$summary) {
+    print "Profile from file $infile ($thread_count threads total, $thread_with_hits w/profiling hits)\n\n";
+}
+
+if (!$summary) {
+    printf "%35s  %10s %10s\n",
+    "Segment", "Hits", "%-age:";
+    printf "%35s  %10s %10s\n",
+    "==========", "=======", "=====";
+}
+$classified_hits = 0;
+$cache_hits = 0;
+$DR_hits = 0;
+$aggregate_reported = 0;
+foreach $key (sort keys %marker_hash) {
+
+    $value = $marker_hash{$key};
+
+    # Bypass the global profile
+    if ($key eq "global") {
+        next;
+    }
+    # Bypass region w/no hits
+    if ($counter_array[$value] == 0) {
+        next;
+    }
+    $classified_hits += $counter_array[$value];
+    $aggregate_reported += ($counter_array[$value] / $reported_hits) * 100;
+    if ($summary) {
+        if ($key =~ /fcache/ || $key =~ /IBL/) {
+            $cache_hits += $counter_array[$value];
+        } elsif ($key =~ /$DR_dll/) {
+            $DR_hits = $counter_array[$value];
+        }
+    }
+    if (!$summary) {
+        printf "%35s  %10d %10.2f\n",
+        $key, $counter_array[$value],
+        ($counter_array[$value] / $reported_hits) * 100;
+    }
+}
+$kernel_hits = $kernel_hits_one + $kernel_hits_two;
+$unexplained_hits = $reported_hits - $classified_hits - $kernel_hits;
+if (!$summary) {
+    printf "%35s  %10d %10.2f\n",
+    "os kernel", $kernel_hits, ($kernel_hits / $reported_hits) * 100;
+    printf "%35s  %10d %10.2f\n",
+    "unexplained", $unexplained_hits, ($unexplained_hits / $reported_hits) * 100;
+}
+$aggregate_reported += ($kernel_hits / $reported_hits) * 100;
+
+if (!$summary) {
+    printf "%35s  %10s %10s\n",
+    "==========", "=======", "=====";
+    printf "%35s  %10d %10.2f\n",
+    "Totals", "$reported_hits", "$aggregate_reported";
+} else {
+    printf "SUMMARY: %s -- cache hit rate %.2f%% (hits %d), thread count %d, thread w/profiling hits %d\n",
+    $infile, 100 * $cache_hits / ($cache_hits + $DR_hits), $cache_hits,
+    $thread_count, $thread_with_hits;
+}
diff --git a/tools/extract-samples.pl b/tools/extract-samples.pl
new file mode 100755
index 000000000..7e00e2604
--- /dev/null
+++ b/tools/extract-samples.pl
@@ -0,0 +1,256 @@
+#!/usr/bin/perl -w
+
+# **********************************************************
+# Copyright (c) 2004-2005 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### extract-samples.pl
+###
+### Merge output from addressquery and extract-profile to report
+### DR hotspots.
+
+$verbose = 0;
+$num_routines = 10;
+
+$usage = "Usage: $0 [-h] -symbol-file <file> -addr-file <file>\n" .
+    "   -symbol-file   A file produced by address_query.pl containing\n" .
+    "                  address to symbol mappings.\n" .
+    "   -addr-file     An address file produced by extract-profile script.\n" .
+    "   -num-routines <num>\n" .
+    "                  The max # of routines to dump sample info for. The\n" .
+    "                  default is $num_routines\n.";
+
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq "-v") {
+	$verbose = 1;
+    } elsif ($ARGV[0] eq "-h") {
+        print $usage;
+        exit;
+    } elsif ($ARGV[0] eq "-num-routines") {
+        shift;
+        $num_routines = $ARGV[0];
+    } elsif ($ARGV[0] eq "-symbol-file") {
+        shift;
+        $SYMBOL_FILE = $ARGV[0];
+        unless (-f "$SYMBOL_FILE") {
+            die "No file $SYMBOL_FILE\n";
+        }
+    } elsif ($ARGV[0] eq "-addr-file") {
+        shift;
+        $ADDRESS_FILE = $ARGV[0];
+        unless (-f "$ADDRESS_FILE") {
+            die "No file $ADDRESS_FILE\n";
+        }
+    }
+    else {
+        print $usage;
+        exit;
+    }
+    shift;
+}
+
+if (!defined($SYMBOL_FILE) || !defined($ADDRESS_FILE)) {
+    print $usage;
+    exit;
+}
+
+%addr_to_symbol = ();
+%symbol_hits = ();
+%addr_to_srcline = ();
+%srcline_hits = ();
+$find_symbol = 0;
+
+open(SYMBOL_FILE, "< $SYMBOL_FILE") or die "Error opening $SYMBOL_FILE\n";
+while (<SYMBOL_FILE>) {
+
+    chop($_);
+
+    if ($verbose) {
+        print STDERR "Process $.: <$_>\n";
+    }
+
+    # We expect lines of the form to follow:
+    # 70007a70:
+    # utils.c(916)
+    # (700079b0)   dynamorio!read_lock_fragment_lookup+0xc0   |  (70007aa0)   dynamorio!write_lock
+    if (/^[0x]??([\d|a-f]+):/) {
+
+        $address = $1;
+        if ($verbose) {
+            print STDERR "Match for addr $address\n";
+        }
+        $find_symbol = 1;
+
+        if ( !($_ = <SYMBOL_FILE>) ) {
+            print STDERR "ERROR -- unexpectedly out of data!\n";
+            exit;
+        }
+        chop($_);
+        if ($verbose) {
+            print STDERR "Src line is $_\n";
+        }
+        # The src line is something like:
+        # utils.c(549)+0x2
+        if (/^(\w+)\.(\w+)\((\d+)\)/) {
+            $srcline = "$1.$2:$3";
+            $addr_to_srcline{$address} = $srcline;
+            $srcline_hits{$srcline} = 0;
+            if ($verbose) {
+                print STDERR "Addr $address -> $addr_to_srcline{$address}\n";
+            }
+        }
+
+        if ( !($_ = <SYMBOL_FILE>) ) {
+            print STDERR "ERROR -- out of data, expecting src info!\n";
+            exit;
+        }
+        chop($_);
+        if ($verbose) {
+            print STDERR "Symbol line is $_\n";
+        }
+
+        # The symbol line is something like:
+        # (70006bb0)   dynamorio!mutex_trylock+0x1c   |  (70006be0)   dynamorio!mutex_unlock
+        if (/^\(([\d|a-f]+)\)[ ]+dynamorio!(\w+)/) {
+            # Add the symbol name to the map
+            $addr_to_symbol{$address} = $2;
+            $symbol_hits{$2} = 0;
+            if ($verbose) {
+                print STDERR "Addr $address -> $addr_to_symbol{$address}\n";
+            }
+            $find_symbol = 0;
+        } else {
+            print STDERR "ERROR -- out of data at line $., expecting symbol info!: $_\n";
+            # exit;
+        }
+    }
+}
+
+close(SYMBOL_FILE);
+
+# The addr->symbol mappings are set. Reconcile these with the addr->hits
+# mappings.
+
+open(ADDRESS_FILE, "< $ADDRESS_FILE") or die "Error opening $ADDRESS_FILE\n";
+$total_hits = 0;
+while (<ADDRESS_FILE>) {
+
+    chop($_);
+
+    if ($verbose) {
+        print STDERR "Process $.: <$_>\n";
+    }
+    if (/^[0x]??([\d|a-f]+)[ ]+(\d+)+[ ]+(\d|'.')+/) {
+        $address = $1;
+        $hits = $2;
+        $total_hits += $hits;
+        if ($verbose) {
+            print STDERR "Match for addr $address -- $hits\n";
+        }
+
+        # Find the symbol name for the address.
+        if (defined($addr_to_symbol{$address})) {
+            $symbol = $addr_to_symbol{$address};
+            $symbol_hits{$symbol} += $hits;
+            if ($verbose) {
+                print STDERR "Update $symbol ($address) to " .
+                    " $symbol_hits{$symbol} hits\n";
+            }
+        } else {
+            if ($verbose) {
+                print STDERR "Addr $address not present in symbol file\n";
+            }
+        }
+
+        # Find the src line for the address.
+        if (defined($addr_to_srcline{$address})) {
+            $srcline = $addr_to_srcline{$address};
+            $srcline_hits{$srcline} += $hits;
+            if ($verbose) {
+                print STDERR "Update $srcline ($address) to " .
+                    " $srcline_hits{$srcline} hits\n";
+            }
+        } else {
+            if ($verbose) {
+                print STDERR "Addr $address not present in symbol file\n";
+            }
+        }
+    }
+}
+close(ADDRESS_FILE);
+
+@array = ();
+while (($key, $value) = each %symbol_hits) {
+    $array[$#array + 1] = "$key $value";
+    if ($verbose) {
+        printf STDERR "Symbol array[$#array] -- %s\n", $array[$#array];
+    }
+}
+
+sub by_hits {
+    ($trash, $a_hits) = split / /, $a;
+    ($trash, $b_hits) = split / /, $b;
+    $b_hits <=> $a_hits;
+}
+
+@sorted_symbols = sort by_hits @array;
+if ($#sorted_symbols > 0) {
+    printf "\n\tHottest symbols\n\n";
+    printf "%30s  %15s  %8s\n", "Symbol name", "Hits", "%-age";
+    printf "%30s  %15s  %8s\n", "===========", "====", "=====";
+    for ($i = 0; $i <= $#sorted_symbols; $i++) {
+        if ($i > $num_routines) {
+            last;
+        }
+        ($symbol, $hits) = split / /, $sorted_symbols[$i];
+        printf "%30s  %15d  %8.2f\n", $symbol, $hits, 100 * $hits / $total_hits;
+    }
+}
+
+@array = ();
+while (($key, $value) = each %srcline_hits) {
+    $array[$#array + 1] = "$key $value";
+    if ($verbose) {
+        printf STDERR "Srcline array[$#array] -- %s\n", $array[$#array];
+    }
+}
+
+@sorted_srclines = sort by_hits @array;
+if ($#sorted_srclines > 0) {
+    printf "\n\tHottest src lines\n\n";
+    printf "%30s  %15s  %8s\n", "Src line", "Hits", "%-age";
+    printf "%30s  %15s  %8s\n", "=========", "====", "=====";
+    for ($i = 0; $i <= $#sorted_srclines; $i++) {
+        if ($i > $num_routines) {
+            last;
+        }
+        ($srcline, $hits) = split / /, $sorted_srclines[$i];
+        printf "%30s  %15d  %8.2f\n", $srcline, $hits, 100 * $hits / $total_hits;
+    }
+}
diff --git a/tools/extract-stats.pl b/tools/extract-stats.pl
new file mode 100755
index 000000000..c862c03f9
--- /dev/null
+++ b/tools/extract-stats.pl
@@ -0,0 +1,671 @@
+#!/usr/bin/perl -w
+
+# **********************************************************
+# Copyright (c) 2004 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### extract-stats.pl
+###
+### Summarize data gathered from DR logs under the various loglevels
+
+
+sub init_tag($$) {
+
+    my($tag, $rate) = @_;
+
+    $marker_hash{$tag} = $marker_count++;
+    $rate_hash{$tag} = $rate;
+}
+
+
+sub init_syscall_tags() {
+
+    init_tag("System calls, pre", 1);
+    init_tag("System calls, post", 1);
+
+}
+
+
+sub init_security_tags() {
+
+    init_tag("Return after call total validations", 1);
+
+}
+
+
+sub init_cache_exits_tags() {
+
+    init_tag("Fcache exits, total", 1);
+    init_tag("Fcache exits, from traces", 1);
+    init_tag("Fcache exits, from BBs", 1);
+    init_tag("Fcache exits, total indirect branches", 1);
+    init_tag("Fcache exits, total direct branches", 1);
+    init_tag("Fcache exits, non-trace indirect branches", 1);
+    init_tag("Fcache exits, ind target not in cache", 1);
+    init_tag("Fcache exits, ind target extending a trace, BAD", 1);
+    init_tag("Fcache exits, ind target in cache but not table", 1);
+    init_tag("Fcache exits, from BB, ind target ...", 1);
+    init_tag("Fcache exits, BB->BB, ind target ...", 1);
+    init_tag("Fcache exits, BB->BB, ind back target ...", 1);
+    init_tag("Fcache exits, BB->trace, ind target ...", 1);
+    init_tag("Fcache exits, from trace, ind target ...", 1);
+    init_tag("Fcache exits, trace->trace, ind target ...", 1);
+    init_tag("Fcache exits, trace->BB, ind target ...", 1);
+    init_tag("Fcache exits, indirect calls", 1);
+    init_tag("Fcache exits, indirect jmps", 1);
+    init_tag("Fcache exits, returns", 1);
+    init_tag("Fcache exits, returns from traces", 1);
+    init_tag("Fcache exits, dir target not in cache", 1);
+    init_tag("Fcache exits, link not allowed", 1);
+    init_tag("Fcache exits, target trace head", 1);
+    init_tag("Fcache exits, extending a trace", 1);
+    init_tag("Fcache exits, no link shared <-> private", 1);
+    init_tag("Fragments generated, bb and trace", 1);
+    init_tag("Trace fragments generated", 1);
+    init_tag("Trace wannabes prevented from being traces", 1);
+    init_tag("Future fragments generated", 1);
+    init_tag("Shared fragments generated", 1);
+    init_tag("Private fragments generated", 1);
+    init_tag("Shared future fragments generated", 1);
+    init_tag("Unique fragments generated", 1);
+    init_tag("Fragments deleted for any reason", 1);
+    init_tag("Fragments deleted due to capacity conflicts", 1);
+    init_tag("Fragments deleted and replaced with traces", 1);
+    init_tag("Fragments regenerated in cache", 1);
+    init_tag("Trace fragments extended", 1);
+    init_tag("Trace fragments extended, ibl exits updated", 1);
+    init_tag("Patched fragments", 1);
+    init_tag("Patched relocation slots", 1);
+    init_tag("Branches linked, direct", 1);
+    init_tag("Branches linked, indirect", 1);
+
+}
+
+
+sub init_cache_memory_tags() {
+
+    init_tag("Fcache trace space claimed \\(bytes\\)", 0);
+    init_tag("Fcache trace space used \\(bytes\\)", 0);
+    init_tag("Fcache trace peak used \\(bytes\\)", 0);
+    init_tag("Fcache trace headers \\(bytes\\)", 0);
+    init_tag("Fcache trace fragment bodies \\(bytes\\)", 0);
+    init_tag("Fcache trace direct exit stubs \\(bytes\\)", 0);
+    init_tag("Fcache trace indirect exit stubs \\(bytes\\)", 0);
+    init_tag("Fcache trace fragment prefixes \\(bytes\\)", 0);
+    init_tag("Fcache trace align space \\(bytes\\)", 0);
+    init_tag("Fcache bb capacity \\(bytes\\)", 0);
+    init_tag("Fcache bb space claimed \\(bytes\\)", 0);
+    init_tag("Fcache bb space used \\(bytes\\)", 0);
+    init_tag("Fcache bb peak used \\(bytes\\)", 0);
+    init_tag("Fcache bb headers \\(bytes\\)", 0);
+    init_tag("Fcache bb fragment bodies \\(bytes\\)", 0);
+    init_tag("Fcache bb indirect exit stubs \\(bytes\\)", 0);
+    init_tag("Fcache bb align space \\(bytes\\)", 0);
+    init_tag("Fcache shared capacity \\(bytes\\)", 0);
+    init_tag("Fcache shared space claimed \\(bytes\\)", 0);
+    init_tag("Fcache shared space used \\(bytes\\)", 0);
+    init_tag("Fcache shared peak used \\(bytes\\)", 0);
+    init_tag("Fcache shared headers \\(bytes\\)", 0);
+    init_tag("Fcache shared fragment bodies \\(bytes\\)", 0);
+    init_tag("Fcache shared direct exit stubs \\(bytes\\)", 0);
+    init_tag("Fcache shared indirect exit stubs \\(bytes\\)", 0);
+    init_tag("Fcache shared align space \\(bytes\\)", 0);
+    init_tag("Fcache combined claimed \\(bytes\\)", 0);
+    init_tag("Fcache combined capacity \\(bytes\\)", 0);
+    init_tag("Peak fcache combined capacity \\(bytes\\)", 0);
+
+}
+
+
+sub init_memory_tags() {
+
+    init_tag("Heap headers \\(bytes\\)", 0);
+    init_tag("Total memory from OS", 0);
+    init_tag("Peak total memory from OS", 0);
+    init_tag("Stack capacity \\(bytes\\)", 0);
+    init_tag("Peak stack capacity \\(bytes\\)", 0);
+    init_tag("Mmap capacity \\(bytes\\)", 0);
+    init_tag("Peak mmap capacity \\(bytes\\)", 0);
+    init_tag("Heap capacity \\(bytes\\)", 0);
+    init_tag("Peak heap capacity \\(bytes\\)", 0);
+    init_tag("Total memory from OS", 0);
+    init_tag("Peak total memory from OS", 0);
+    init_tag("Our library space \\(bytes\\)", 0);
+    init_tag("Application reserved-only capacity \\(bytes\\)", 0);
+    init_tag("Peak application reserved-only capacity \\(bytes\\)", 0);
+    init_tag("Application committed capacity \\(bytes\\)", 0);
+    init_tag("Peak application committed capacity \\(bytes\\)", 0);
+    init_tag("Application stack capacity \\(bytes\\)", 0);
+    init_tag("Peak application stack capacity \\(bytes\\)", 0);
+    init_tag("Application heap capacity \\(bytes\\)", 0);
+    init_tag("Peak application heap capacity \\(bytes\\)", 0);
+    init_tag("Application image capacity \\(bytes\\)", 0);
+    init_tag("Peak application image capacity \\(bytes\\)", 0);
+    init_tag("Application mmap capacity \\(bytes\\)", 0);
+    init_tag("Peak application mmap capacity \\(bytes\\)", 0);
+    init_tag("Application executable capacity \\(bytes\\)", 0);
+    init_tag("Peak application executable capacity \\(bytes\\)", 0);
+    init_tag("Application read-only capacity \\(bytes\\)", 0);
+    init_tag("Peak application read-only capacity \\(bytes\\)", 0);
+    init_tag("Application writable capacity \\(bytes\\)", 0);
+    init_tag("Peak application writable capacity \\(bytes\\)", 0);
+    init_tag("Total \\(app + us\\) virtual size \\(bytes\\)", 0);
+    init_tag("Peak total \\(app + us\\) virtual size \\(bytes\\)", 0);
+    init_tag("Application virtual size \\(bytes\\)", 0);
+    init_tag("Peak application virtual size \\(bytes\\)", 0);
+    init_tag("Our additional virtual size \\(bytes\\)", 0);
+    init_tag("Peak our additional virtual size \\(bytes\\)", 0);
+    init_tag("Our commited capacity \\(bytes\\)", 0);
+    init_tag("Our peak commited capacity \\(bytes\\)", 0);
+    init_tag("Our reserved capacity \\(bytes\\)", 0);
+    init_tag("Our peak reserved capacity \\(bytes\\)", 0);
+    init_tag("App unallocatable free space", 0);
+    init_tag("Peak app unallocatable free space", 0);
+    init_tag("Our unallocatable free space", 0);
+    init_tag("Our peak unallocatable free space", 0);
+    init_tag("Total unallocatable free space", 0);
+    init_tag("Peak total unallocatable free space", 0);
+    init_tag("Number of unaligned allocations \\(TEB's etc.\\)", 0);
+    init_tag("Peak unaligned allocations", 0);
+}
+
+
+sub process_snapshot($$) {
+
+    my($verbose, $line) = @_;
+
+    if ($verbose) {
+        print STDERR "Digest $line...\n";
+    }
+
+    # Bypass the "** statistics @<fragment count>" line.
+    if ($line =~ /^(All|Thread) statistics @[0-9]/) {
+        if ($verbose) {
+            print STDERR "Bypass $line\n";
+        }
+        return;
+    }
+
+    my $found = 0;
+    my $key;
+    my $value;
+
+    $lines[ $line_count++ ] = $line;
+    while (($key, $value) = each %marker_hash) {
+        if ($found == 1) {
+            next;
+        }
+        if ($verbose) {
+            print STDERR "Check for \"$key\" in line $.: $line\n";
+        }
+
+        # XXX Match thread stats also...
+        #if ($line =~ /^[ ]*${key}[ ]+:[ ]+(\d+)/) {
+        if ($line =~ /^[ ]*${key}[ ]+(\(thread\))?:[ ]+(\d+)/) {
+        #if ($line =~ /^[ ]*${key}/ && /:[ ]+(\d+)/) {
+
+            # Adjust for processing thread-private log files.
+            my $counter = defined($2) ? $2 : $1;
+            if ($verbose) {
+                print STDERR "Count for $key is $counter\n";
+            }
+
+            # XXX Tried to put the substr call in the parens but
+            # it wouldn't work.
+            #if ($line eq "(") {
+            #    if ($verbose) {
+            #        print STDERR "THREAD log\n";
+            #    }
+            #}
+            # Now update the table
+            $counter_array[$value][$snapshot_opens] = $counter;
+            $valid{$key} = 1;
+            if ($verbose) {
+                print STDERR "Update counter for $key => $value, " .
+                    "$snapshot_opens with $counter\n";
+            }
+            # We don't 'last' out of the loop because we'd have
+            # to reset the hash iter using 'keys' on the next
+            # key match. And keys could be a sort underneath, which
+            # is costlier than continuing the loop, which is
+            # linear.
+            $found = 1;
+        }
+    }
+}
+
+sub prepare_stats($$$) {
+
+    my $i;
+    my($limit, $value, $rate) = @_;
+
+    # Clean up undefined values.
+    for ($i = 0; $i < $limit - 1; $i++) {
+        if (!defined($counter_array[$value][$i])) {
+            $counter_array[$value][$i] = 0;
+        }
+    }
+
+    # We don't track non-rate stats incrementally.
+    if ($rate) {
+        $avg_rate = $counter_array[$value][$limit - 1] /
+            int($snapshot_times[$limit - 1] / 1000000);
+        for ($i = $limit - 1; $i > 0; $i--) {
+            $counter_array[$value][$i] -= $counter_array[$value][$i-1];
+            $rate_array[$value][$i] =
+                ($counter_array[$value][$i] - $counter_array[$value][$i-1]) /
+                $snapshot_times[$i];
+        }
+    }
+}
+
+sub numerically { $a <=> $b; }
+
+sub compute_stats($$$$) {
+
+    my $i;
+    my($limit, $key, $value, $rate) = @_;
+    my $start_index = $limit > 2 ? 1 : 0;
+
+    # Sort the counter array and store the max and median. Don't count
+    # the first counter as it's often artificially high.
+    @sorted_counters = sort numerically
+        @{ $counter_array[$value] } [$start_index..$limit - 1];
+    $max_val = $sorted_counters[$limit - ($limit > 2 ? 2 : 1)];
+    $median_val = $sorted_counters[$median_index];
+
+    # XXX Figure out how to sort this and use an array of structs
+    $prev_time = $max_rate = $max_time = $max_rate_time = 0;
+
+    @rate_array = ();
+
+    for ($i = $start_index; $i < $limit; $i++) {
+        $curr_time = $snapshot_times[$i];
+        $count = $counter_array[$value][$i];
+
+        # Find the largest value.
+        if ($count == $max_val) {
+            $max_time = $curr_time;
+        }
+
+        # Find the largest rate.
+        if ($rate) {
+            # If the delta is too small, don't compute a rate.
+            if ($curr_time - $prev_time == 0) {
+                next;
+            }
+            $current_rate = int(1000000 * $count / ($curr_time - $prev_time));
+            if ($current_rate > $max_rate) {
+                $max_rate = $current_rate;
+                $max_rate_time = $curr_time;
+            }
+            $rate_array[$i] = $current_rate;
+            $prev_time = $curr_time;
+        }
+    }
+
+    # Sort the rate array and store the max and median. Don't count
+    # the first counter as it's not computed.
+    for ($i = $start_index; $i < $limit; $i++) {
+        if (!defined($rate_array[$i])) {
+            $rate_array[$i] = 0;
+        }
+    }
+
+    if ($rate) {
+        @sorted_rates = sort numerically
+            @rate_array [$start_index..$limit - 1];
+        $max_rate = $sorted_rates[$limit - 2];
+        $median_rate = $sorted_rates[$median_index];
+    }
+}
+
+sub print_stats($$$$$) {
+
+    my $i;
+    my $currtime;
+    my $prevtime = 0;
+    my($rate, $value, $details, $limit, $raw) = @_;
+
+    if (!$raw && $rate) {
+        printf "\tmax rate = $max_rate [snapshot = " .
+            int($max_rate_time / 1000000) . "], median rate = $median_rate" .
+            ", avg rate = " . int($avg_rate) . "\n"
+    }
+    elsif ($raw) {
+        print "$key raw data\n";
+    }
+
+    if ($details || $raw) {
+        if ($rate) {
+            printf "%15s %15s %15s\n", "Snapshot (us)", "Value", "Rate (/s)";
+            printf "%15s %15s %15s\n", "==========", "==========",
+            "==========";
+        }
+        else {
+            printf "%15s %15s\n", "Snapshot (us)", "Value";
+            printf "%15s %15s\n", "==========", "==========";
+        }
+        for ($i = 0; $i < $limit; $i++) {
+            $currtime = $snapshot_times[$i];
+            if ($raw && !defined($counter_array[$value][$i])) {
+                $counter_array[$value][$i] = 0;
+            }
+            printf "%15d %15d", "$currtime", "$counter_array[$value][$i]";
+            if ($rate) {
+                printf " %15.2f", $counter_array[$value][$i] /
+                    (($currtime - $prevtime)/1000000);
+            }
+            printf "\n";
+            $prevtime = $currtime;
+        }
+    }
+    print "\n";
+}
+
+
+################## main() ################
+
+$usage = "Usage: $0 -v -d -cache_exits -cache_memory -memory <stats file>\n";
+
+$verbose = 0;
+$warning = 0;
+$details = 0;
+%marker_hash = ();
+%valid = ();
+%rate_hash = (); # tracks if a tag is a rate or not
+
+@counter_array = ();
+@rate_array = ();
+
+$catch_cache_exits = 1;
+$catch_syscalls = 1;
+$catch_security = 1;
+$catch_cache_memory = 0;
+$catch_memory = 0;
+
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq "-v") {
+	$verbose = 1;
+    }
+    elsif ($ARGV[0] eq "-w") {
+	$warning = 1;
+    }
+    elsif ($ARGV[0] eq "-d") {
+	$details = 1;
+    }
+    elsif ($ARGV[0] eq "-catch_syscalls") {
+        $catch_syscalls = 1;
+    }
+    elsif ($ARGV[0] eq "-cache_exits") {
+        $catch_cache_exits = 1;
+    }
+    elsif ($ARGV[0] eq "-cache_memory") {
+        $catch_cache_memory = 1;
+    }
+    elsif ($ARGV[0] eq "-memory") {
+        init_memory_tags();
+    }
+    else {
+	$logfile = $ARGV[0];
+    }
+    shift;
+}
+if (!defined($logfile)) {
+    print $usage;
+    exit;
+}
+
+open(PROFILE, "< $logfile") || die "Error opening $logfile\n";
+
+if ($catch_security) {
+    init_security_tags();
+}
+if ($catch_syscalls) {
+    init_syscall_tags();
+}
+if ($catch_cache_exits) {
+    init_cache_exits_tags();
+}
+if ($catch_cache_memory) {
+    init_cache_memory_tags();
+}
+if ($catch_memory) {
+    init_memory_tags();
+}
+
+if ($verbose) {
+    print STDERR "# stats to match: $marker_count...\n";
+}
+
+print "Stats from logfile $logfile\n(All rates are in per s)\n";
+
+$marker_count = 0;
+
+$snapshot_opens = 0;
+$time_snapshot_start = 0;
+$fragment_snapshot_start = 0;
+
+# Buffer the lines pertaining to memory data
+@lines = ();
+$line_count = 0;
+
+while (<PROFILE>) {
+
+    chop($_);
+
+    if ($verbose) {
+        print STDERR "Check line $. >>$_<<\n";
+    }
+
+    if (/^Begin stats @([0-9]+):([0-9]+):([0-9]+)$/) {
+        if ($time_snapshot_start || $fragment_snapshot_start) {
+            if ($warning) {
+                print STDERR "WARNING: Nested time stats detected, " .
+                    "line $.!\n";
+            }
+            $snapshot_times[$snapshot_opens++] = $currtime;
+        }
+        $currtime = ($1 * 60 + $2) * 1000000 + $3;
+        $time_snapshot_start = 1;
+        $fragment_snapshot_start = 0;
+        @lines = ();
+        $line_count = 0;
+        if ($verbose) {
+            print STDERR "Begin time-based snapshot for $currtime " .
+                "\@line $.\n";
+        }
+    }
+    # An alternative to pull out only the fragment count is
+    # /^All statistics @([0-9]+)+ \([0-9]+:[0-9]+\.[0-9]+\)/
+    elsif (/^All statistics @([0-9]+) \(([0-9]+):([0-9]+)\.([0-9]+)\)/) {
+        if ($1 == 0 && $2 == 0 && $3 == 0 && $4 == 0) {
+            if ($verbose) {
+                print STDERR "Skip 0-time 'All stats...'\n";
+            }
+            next;
+        }
+        elsif ($time_snapshot_start) { # time-based snapshot begin-marker
+            if ($verbose) {
+                print STDERR "Skip 'All stats...', gathering time stats\n";
+            }
+            next;
+        }
+        else { # Open a snapshot
+            if ($fragment_snapshot_start) {
+                if ($warning) {
+                    print STDERR "WARNING: Nested fragment stats detected, " .
+                        "line $.!\n";
+                }
+                $snapshot_times[$snapshot_opens++] = $currtime;
+            }
+            $currtime = ($2 * 60 + $3) * 1000000 + $4;
+            $fragment_snapshot_start = 1;
+            $time_snapshot_start = 0;
+            @lines = ();
+            $line_count = 0;
+            if ($verbose) {
+                print STDERR "Begin fragment-based snapshot $currtime " .
+                    "\@line $.\n";
+            }
+        }
+    }
+    elsif (/^Thread statistics @[0-9]+ global, ([0-9]+) thread fragments \(([0-9]+):([0-9]+)\.([0-9]+)\)/) { # Begin thread-private stats
+        if ($1 == 0 && $2 == 0 && $3 == 0 && $4 == 0) {
+            if ($verbose) {
+                print STDERR "Skip 0-time \"Thread statistics...\"\n";
+            }
+            next;
+        }
+        elsif ($time_snapshot_start) { # time-based snapshot begin-marker
+            if ($verbose) {
+                print STDERR "Skip \"Thread statistics...\", " .
+                    "gathering time stats\n";
+            }
+            next;
+        }
+        else { # Open a snapshot
+            if ($fragment_snapshot_start) {
+                if ($warning) {
+                    print STDERR "WARNING: Nested fragment stats detected, " .
+                        "line $.!\n";
+                }
+                $snapshot_times[$snapshot_opens++] = $currtime;
+            }
+            $currtime = ($2 * 60 + $3) * 1000000 + $4;
+            $fragment_snapshot_start = 1;
+            $time_snapshot_start = 0;
+            @lines = ();
+            $line_count = 0;
+            if ($verbose) {
+                print STDERR "Begin fragment-based snapshot $currtime " .
+                    "\@line $.\n";
+            }
+        }
+    }
+    elsif (/^End stats/) { # We see a time-based snapshot end-marker
+        if ($fragment_snapshot_start) {
+            print STDERR "ERROR: fragment-based stats open, " .
+                "time-based trying to close, line $.!\n";
+            die;
+        }
+        elsif (!$time_snapshot_start) {
+            print STDERR "ERROR: no time-based stats open, " .
+                "time-based trying to close, line $.!\n";
+            die;
+        }
+        else { # All is well, close this snapshot
+            $snapshot_times[$snapshot_opens++] = $currtime;
+            $time_snapshot_start = 0;
+            if ($verbose) {
+                print STDERR "End time-based snapshot for $currtime\n";
+            }
+        }
+    }
+    # XXX Sometimes "/^[ ]+Peak our additional virtual size/" works??
+    elsif (/^(Process|Thread) heap breakdown:$/) {
+        #if (!$fragment_snapshot_start && !$time_snapshot_start) {
+        #    print STDERR "ERROR: no stats window open, " .
+        #        "fragment-based trying to close, line $.!\n";
+        #    die;
+        #}
+        if ($fragment_snapshot_start) { # All is well, close this snapshot
+            $snapshot_times[$snapshot_opens++] = $currtime;
+            $fragment_snapshot_start = 0;
+            if ($verbose) {
+                print STDERR "End fragment-based snapshot for $currtime\n";
+            }
+        }
+    }
+
+    elsif (/^(Process|Thread) heap breakdown:$/) {
+        #if (!$fragment_snapshot_start && !$time_snapshot_start) {
+        #    print STDERR "ERROR: no stats window open, " .
+        #        "fragment-based trying to close, line $.!\n";
+        #    die;
+        #}
+        if ($fragment_snapshot_start) { # All is well, close this snapshot
+            $snapshot_times[$snapshot_opens++] = $currtime;
+            $fragment_snapshot_start = 0;
+            if ($verbose) {
+                print STDERR "End fragment-based snapshot for $currtime\n";
+            }
+        }
+    }
+
+    # We are reading a stats snapshot
+    elsif ($fragment_snapshot_start || $time_snapshot_start) {
+        if ($verbose) {
+            print STDERR "Process line $_\n";
+        }
+        process_snapshot($verbose, $_);
+    }
+}
+
+$median_index = $snapshot_opens / 2;
+
+# The times are in microseconds. Let's group by milliseconds.
+printf "# snapshots $snapshot_opens, median snapshot time " .
+    int($snapshot_times[$median_index]/1000000) . "s\n\n";
+
+if ($snapshot_opens == 0) {
+    print "No snapshots collected!\n";
+    print STDERR "No snapshots collected!\n";
+    exit;
+}
+
+foreach $key (sort keys %marker_hash) {
+
+    if (!defined($valid{$key})) {
+        if ($verbose) {
+            print STDERR "Bypass $key, no stats gathered...\n";
+        }
+        next;
+    }
+
+    # Is this a rate-oriented stat? What's the index into the
+    # counter table?
+    $rate = $rate_hash{$key};
+    $value = $marker_hash{$key};
+
+    # print_stats($rate, $value, $details, $snapshot_opens, 1);
+
+    prepare_stats($snapshot_opens, $value, $rate);
+    compute_stats($snapshot_opens, $key, $value, $rate);
+
+    if ($verbose) {
+        print STDERR "Print summary for $key...\n";
+    }
+
+    print "$key: median = $median_val\n" .
+        "max = $max_val [snapshot time = " . int($max_time / 1000000) .
+        "s]\n";
+
+    print_stats($rate, $value, $details, $snapshot_opens, 0);
+}
diff --git a/tools/graph.pl b/tools/graph.pl
new file mode 100755
index 000000000..8607f6ace
--- /dev/null
+++ b/tools/graph.pl
@@ -0,0 +1,257 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2002 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# Derek's Graph Builder
+# Need to have: perl-GD and perl-GIFgraph
+
+# For linux, on balrog (RH7.2), I had to do:
+# rpm -Uvh gd-2.0.1-4.i386.rpm gd-devel-2.0.1-4.i386.rpm 
+# rpm -ivh perl-5.6.1-14.i686.rpm 
+# rpm -Uvh perl-GD-1.33-3.i686.rpm 
+# rpm -ivh --oldpackage db3-3.1.17-7.i386.rpm 
+# rpm -ivh perl-modules-5.6.1-14.i686.rpm 
+# rpm -Uvh perl-GD-TextUtil-0.80-2.i386.rpm 
+# rpm -Uvh perl-GD-Graph-1.33-3.i686.rpm 
+# rpm -ivh perl-GIFgraph-1.20-4.i386.rpm
+
+# Installing GIFgraph For win32:
+# Got FreeType library from www.freetype.org, built & installed it
+# Got gd library from http://www.boutell.com/gd/, built and installed it
+# GIFgraph: gifs are no longer supported b/c of unisys patent on lzw!
+# -> use GD::Graph
+# Got GD-1.33, GDGraph-1.33, and GDTextUtil-0.80 from:
+#   ftp://cpan.valueclick.com/pub/CPAN/modules/by-module/
+# Installing perl packages: "perl Makefile.PL; make; make install"
+
+use GD::Graph::bars;
+use GD;
+
+# Datafile example:
+#  type=clustered
+#  size=600x400
+#  title=Dynamo Speedup
+#  x_label=Benchmark
+#  y_label=Speedup
+#  legend="-O0" "-O3"
+#  y_max_value=1.4
+#  y_tick_number=14
+#  y_number_format=%3.1f
+#  
+#  apsi    0.994019        0.994626
+#  art     1.23069         1.08014
+#  bzip2   0.928           0.825924
+#  crafty  0.210147        0.305814
+#  eon     0.310073        0.34035
+#  gap     0.268882        0.283897
+#  gcc     0.147219        0.14813
+#  gzip    0.643855        0.648045
+#  mcf     1.1145          1.12565
+#  parser  0.425933        0.48503
+#  perlbmk 0.193682        0.287931
+#  swim    0.997532        0.974938
+#  twolf   0.847986        0.805139
+#  vortex  0.207531        0.261248
+#  vpr     0.877838        0.926712
+
+$usage = "Usage: $0 <datafile>\n";
+die $usage if (@ARGV == 0);
+$infile = shift;
+
+# default parameters
+$multiple_runs = 0;
+$sep_slim = 0;
+$overwrite = 0; # clustered
+$sizex = 600; $sizey = 500;
+$title = "Default Title";
+$x_label = "X LABEL";
+$y_label = "Y LABEL";
+$y_max_value = 10;
+$y_tick_number = 1;
+$y_number_format = "%3.0f";
+$legend_placement = 'RL';
+$transparent = 1;
+# lg_cols default is undefined
+
+$n = 0;
+$d = 0;
+$data_sets = 0;
+open(IN, "< $infile") || die "Error: Couldn't open $infile for input\n";
+while (<IN>) {
+    next if /^\s*$/; # skip blank lines
+    next if /^\#/; # skip comments
+    if ($_ =~ /\S*=/) {
+	# parameter line
+	if ($_ =~ /multiple_runs/) {
+	    $multiple_runs = 1;
+	} elsif ($_ =~ /sep_slim/) {
+	    $sep_slim = 1;
+	} elsif ($_ =~ /type=(\w+)/) {
+	    $type = $1;
+	    if ($type eq "clustered") {
+		$overwrite = 0;
+	    } else {
+		$overwrite = 2; # stacked
+	    }
+	} elsif ($_ =~ /size=(\d+)x(\d+)/) {
+	    $sizex = $1;
+	    $sizey = $2;
+	} elsif ($_ =~ /title=(.+)/) {
+	    $title = $1;
+	} elsif ($_ =~ /x_label=(.+)/) {
+	    $x_label = $1;
+	} elsif ($_ =~ /y_label=(.+)/) {
+	    $y_label = $1;
+	} elsif ($_ =~ /y_max_value=([\d\.]+)/) {
+	    $y_max_value = $1;
+	} elsif ($_ =~ /y_tick_number=([\d\.]+)/) {
+	    $y_tick_number = $1;
+	} elsif ($_ =~ /y_number_format=\"?(.*)\"?/) {
+	    $y_number_format = $1;
+	} elsif ($_ =~ /legend_placement=(.+)/) {
+	    $legend_placement = $1;
+	} elsif ($_ =~ /transparent=(.+)/) {
+	    $transparent = $1;
+	} elsif ($_ =~ /lg_cols=([\d]+)/) {
+	    $lg_cols = $1;
+	} elsif ($_ =~ /legend=/) {
+	    $_ =~ s/legend=//;
+	    chop;
+	    @legend = split("\" ", $_);
+	    # now remove quotes
+	    foreach $l (@legend) {
+		$l =~ s/\"//g;
+	    }
+	    if ($sep_slim) {
+		# add blank legend for separator dataset
+		@legend = ("", @legend);
+	    }
+	} else {
+	    $_ =~ /(.*)=/;
+	    die "Unknown parameter: $1\n";
+	}
+    } else {
+	# data line
+	# first get label
+	if ($multiple_runs) {
+	    $_ =~ s/(\w+)\s+(\d+)\s+//;
+	    $bm[$n] = "$1 $2";
+	    $this_bm = $1;
+	} else {
+	    $_ =~ s/(\w+)\s+//;
+	    $bm[$n] = $1;
+	    $this_bm = $1;
+	}
+	if ($sep_slim) {
+	    # add empty data set for separator
+	    $val[0][$n] = "";
+	    $d++;
+	} else {
+	    # add entire empty entry for separator
+	    if ($n > 0 && $this_bm ne $last_bm) {
+		# separator
+		$temp = $bm[$n];
+		$bm[$n] = "";
+		for ($i=0; $i<$data_sets; $i++) {
+		    $val[$i][$n] = "";
+		}
+		$n++;
+		$bm[$n] = $temp;
+	    }
+	}
+	while (1) {
+	    if ($_ =~ /^\s*$/) {
+		last;
+	    }
+	    $_ =~ s/([\d\.]+)\s*//;
+	    $data = $1;
+	    if ($data ne "") {
+		$val[$d][$n] = $data;
+		$d++;
+	    } else {
+		last;
+	    }
+	}
+	if ($n == 0) {
+	    $data_sets = $d;
+	}
+	$d = 0;
+	$last_bm = $this_bm;
+	$n++;
+    }
+}
+
+@data = ([@bm]);
+for ($d=0; $d<$data_sets; $d++) {
+    push(@data, @val[$d]);
+}
+$my_graph = new GD::Graph::bars($sizex, $sizey);
+if (defined(@legend)) {
+    $my_graph->set_legend(@legend);
+}
+$my_graph->set_title_font(gdLargeFont);
+$my_graph->set_x_axis_font(gdMediumBoldFont);
+$my_graph->set_y_axis_font(gdMediumBoldFont);
+$my_graph->set_x_label_font(gdMediumBoldFont);
+$my_graph->set_y_label_font(gdMediumBoldFont);
+$my_graph->set_legend_font(gdMediumBoldFont);
+if ($sep_slim) {
+    $my_graph->set(dclrs => 
+	   [ qw(pink lred lgreen lblue lyellow lpurple cyan lorange) ]);
+}
+if (defined($lg_cols)) {
+    $my_graph->set(lg_cols           => $lg_cols);
+}
+$my_graph->set(
+	       overwrite         => $overwrite,
+	       title             => $title,
+	       x_label           => $x_label,
+	       y_label           => $y_label,
+	       long_ticks        => 1,
+	       axis_space        => 5,
+	       y_max_value       => $y_max_value,
+	       y_tick_number     => $y_tick_number,
+	       y_label_skip      => 1,
+	       y_number_format   => $y_number_format,
+	       x_ticks           => 0,
+	       x_labels_vertical => 1,
+	       x_label_position  => 0.5,
+	       legend_placement  => $legend_placement,
+	       transparent       => $transparent,
+	       );
+
+## can't do gifs anymore!
+## $my_graph->plot_to_gif( "$infile.gif", \@data );
+
+open(IMG, "> $infile.png") or die $!;
+binmode IMG;
+print IMG $my_graph->plot(\@data)->png;
+close(IMG);
diff --git a/tools/hang_my_machine.c b/tools/hang_my_machine.c
new file mode 100644
index 000000000..7b00e7dc8
--- /dev/null
+++ b/tools/hang_my_machine.c
@@ -0,0 +1,50 @@
+/* **********************************************************
+ * Copyright (c) 2004-2005 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include <tchar.h>
+#include <assert.h>
+
+#define MB_SERVICE_NOTIFICATION 0x00200000L
+
+/* calls detach for the passed in process id */
+DWORD __cdecl
+main(DWORD argc, char *argv[], char *envp[])
+{
+    int i;
+    for (i = 0; i < 100; i++) {
+    MessageBoxW(NULL, L"\\??\\c:\foo.txt", L"none", MB_OK | MB_TOPMOST | MB_SERVICE_NOTIFICATION);
+    MessageBoxW(NULL, L"\\??\\r",  L"\\??\\c:\foo.txt", MB_OK | MB_TOPMOST | MB_SERVICE_NOTIFICATION);
+    MessageBoxW(NULL, L"\\??\\r:", L"none", MB_OK | MB_TOPMOST | MB_SERVICE_NOTIFICATION);
+    }
+    return 0;
+}
diff --git a/tools/heapacct.pl b/tools/heapacct.pl
new file mode 100644
index 000000000..599041a23
--- /dev/null
+++ b/tools/heapacct.pl
@@ -0,0 +1,158 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2005-2007 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# Prints out summary of memory statistics from a global log file
+# This script is a merger of old heapacct.pl and new memsummary.sh
+
+# FIXME: add cmd line arg -v
+$verbose = 0;
+$debug = 0;
+
+if ($#ARGV < 0) {
+    print "Usage: $0 <file>\n";
+    exit 0;
+}
+$file = $ARGV[0];
+
+open(IN, "< $file") || die "Couldn't open $file\n";
+while (<IN>) {
+    # heap acct: use final printed stats, since using max
+    if (/Updated-at-end Process.*heap breakdown/) {
+        while (<IN>) {
+            if (/^([^:]+): cur=.*max=\s*(\d+)K, \#=\s*(\d+)/){
+                $hnm[$hnms++]=$1;
+                $hmax{$1}=$2;
+                $hnum{$1}=$3;
+            } else {
+                last;
+            }
+        }
+    }
+    
+    if (/^\(Begin\) All statistics/) {
+        # reset
+        $hnms = 0;
+        $pnms = 0;
+    } elsif (/Peak heap cap|Peak heap claim|Peak heap align|Peak stack|Peak mmap|Peak file|peak cap|Peak spec/) {
+        if (/^\s*(.*) \(bytes\) :\s*(\d+)/){
+            print "Matched \"$1\" \"$2\" \"$3\"\n" if ($debug);
+            $nm=$1;
+            $val=$2;
+            $pnm[$pnms++]=$nm;
+            $p{$nm}=$val;
+        } else {
+            print "Error: $_" unless (/Peak special heap units/);
+        }
+    } elsif (/Peak total memory from OS :\s*(\d+)/){
+        $reserved_tot=$1;
+    } elsif (/Our peak commited capacity \(bytes\) :\s*(\d+)/){
+        $tot=$1;
+    }
+
+    # stats we don't or haven't always had peaks for
+    if (/Heap align space \(bytes\) :\s*(\d+)/) {
+        # old logs don't have this as a peak, so take max printed
+        if ($1 > $align) {
+            $align = $1;
+        }
+    } elsif (/Heap headers \(bytes\) :\s*(\d+)/) {
+        # not a peak, so take max printed
+        if ($1 > $header) {
+            $header = $1;
+        }
+    } elsif (/Heap claimed \(bytes\) :\s*(\d+)/) {
+        # old logs don't have this as a peak, so take max printed
+        if ($1 > $claimed) {
+            $claimed = $1;
+        }
+    }
+
+    if ($verbose) {
+        # completely separate numbers: variable-sized frequency
+        # last printed is fine since these never decrease
+        if (/Heap allocs in buckets :\s*(\d+)/) {
+            $bucket = $1;
+        } elsif (/Heap allocs variable-sized :\s*(\d+)/) {
+            $variable = $1;
+        }
+    }
+}
+close(IN);
+
+# things we don't have peak for
+if (defined($p{'Peak heap claimed'})) {
+    $claimed = $p{'Peak heap claimed'};
+}
+$tot_cap = $p{'Peak heap capacity'};
+$pnm[$pnms++]='Unused heap (capacity - claimed)';
+$p{$pnm[$pnms-1]} = $p{'Peak heap capacity'} - $claimed;
+
+if (!defined($p{'Peak heap align space'})) {
+    $pnm[$pnms++]='(Estimated) Peak heap align space';
+    $p{$pnm[$pnms-1]} = $align;
+}
+# FIXME: don't have peak
+$pnm[$pnms++]='(Estimated) Peak heap headers';
+$p{$pnm[$pnms-1]} = $header;
+
+
+printf "%35s 100.00%% %10d KB\n", "TOTAL COMMITTED MEMORY", $tot/1024;
+printf "%35s %6.2f%% %10d KB\n", "TOTAL RESERVED (non-vmm) MEMORY",
+    100*$reserved_tot/$tot, $reserved_tot/1024;
+for ($i=0; $i<$pnms; $i++) {
+    printf "%35s %6.2f%% %10d KB\n", $pnm[$i],
+    100*$p{$pnm[$i]}/$tot, $p{$pnm[$i]}/1024;
+}
+for ($i=0; $i<$hnms; $i++) {
+    printf "%35s %6.2f%% %10d KB\n", $hnm[$i],
+    100*1024*$hmax{$hnm[$i]}/$tot, $hmax{$hnm[$i]};
+    $hsum += $hmax{$hnm[$i]};
+}
+
+if ($verbose) {
+    printf "Sum of heap acct is %10d KB\n", $hsum;
+    printf "Adding special %10d\n", $p{'Peak special heap capacity'}/1024;
+    printf "Adding align %10d\n", $p{'Peak heap align space'}/1024;
+    $hsum += $p{'Peak heap align space'}/1024;
+    $hsum += $p{'Peak special heap capacity'}/1024;
+    # FIXME: this often comes out negative, presumably b/c the heapacct
+    # maxes are not all simultaneous
+    printf "Rest of heap: %10d KB\n", ($p{'Peak heap capacity'}/1024) - $hsum;
+}
+
+if ($verbose) {
+    # FIXME: could print out detailed bucket info w/ new stats in logfiles
+    $tot_alloc = $bucket + $variable;
+    printf "Tot alloc: %10d\n", $tot_alloc;
+    printf "Buckets:   %10d => %7.3f%%\n", $bucket, 100*$bucket/$tot_alloc;
+    printf "Variable:  %10d => %7.3f%%\n", $variable, 100*$variable/$tot_alloc;
+}
diff --git a/tools/laststats.sh b/tools/laststats.sh
new file mode 100755
index 000000000..5d7a969fc
--- /dev/null
+++ b/tools/laststats.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# **********************************************************
+# Copyright (c) 2007 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+## laststats.sh
+## by Derek Bruening
+## December 2005
+##
+## prints the tail of a log file starting at the last global stats dump
+
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 <logfile>"
+    exit 127
+fi
+
+log=$1
+
+if ! test -e $log ; then
+    echo "Error: $log not found"
+    exit 127
+fi
+
+start=`grep -n '^(Begin) All statistics' $log | tail -1 | cut -d : -f 1`
+total=`wc -l $log | awk '{print $1}'`
+tail -$((total-start+1)) $log 
diff --git a/tools/ldmp.c b/tools/ldmp.c
new file mode 100644
index 000000000..6584f738d
--- /dev/null
+++ b/tools/ldmp.c
@@ -0,0 +1,1159 @@
+/* **********************************************************
+ * Copyright (c) 2004-2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#include <windows.h>
+#include <stdio.h>
+#include <assert.h>
+
+#define RC1_HACK 1
+#define VERBOSE 0
+
+/* alignment helpers */
+#define ALIGNED(x, alignment) ((((uint)x) & ((alignment)-1)) == 0)
+#define ALIGN_FORWARD(x, alignment) ((((uint)x) + ((alignment)-1)) & (~((alignment)-1)))
+#define ALIGN_BACKWARD(x, alignment) (((uint)x) & (~((alignment)-1)))
+#define PAD(length, alignment) (ALIGN_FORWARD((length), (alignment)) - (length))
+
+/* check if all bits in mask are set in var */
+#define TESTALL(mask, var) (((mask) & (var)) == (mask))
+/* check if any bit in mask is set in var */
+#define TESTANY(mask, var) (((mask) & (var)) != 0)
+/* check if a single bit is set in var */
+#define TEST TESTANY
+
+#define BUFFER_SIZE_BYTES(buf)      sizeof(buf)
+#define BUFFER_SIZE_ELEMENTS(buf)   (BUFFER_SIZE_BYTES(buf) / sizeof(buf[0]))
+#define BUFFER_LAST_ELEMENT(buf)    buf[BUFFER_SIZE_ELEMENTS(buf) - 1]
+#define NULL_TERMINATE_BUFFER(buf)  BUFFER_LAST_ELEMENT(buf) = 0
+
+typedef int bool;
+typedef unsigned int uint;
+typedef LONG NTSTATUS;
+#define NT_SUCCESS(Status) ((NTSTATUS)(Status) >= 0)
+#define STATUS_SUCCESS ((NTSTATUS)0x00000000L)
+
+#define false FALSE
+#define true TRUE
+#define PAGE_SIZE 0x1000
+#define ALLOCATION_GRANULARITY 0x10000
+#define INVALID_File (NULL)
+
+#define GET_NTDLL(NtFunction, signature)                             \
+  NTSYSAPI NTSTATUS NTAPI NtFunction signature
+
+typedef struct _UNICODE_STRING {
+    /* Length field is size in bytes not counting final 0 */
+    USHORT Length;
+    USHORT MaximumLength;
+    PWSTR  Buffer;
+} UNICODE_STRING;
+typedef UNICODE_STRING *PUNICODE_STRING;
+
+typedef struct _OBJECT_ATTRIBUTES {
+    ULONG Length;
+    HANDLE RootDirectory;
+    PUNICODE_STRING ObjectName;
+    ULONG Attributes;
+    PVOID SecurityDescriptor;        // Points to type SECURITY_DESCRIPTOR
+    PVOID SecurityQualityOfService;  // Points to type SECURITY_QUALITY_OF_SERVICE
+} OBJECT_ATTRIBUTES;
+typedef OBJECT_ATTRIBUTES *POBJECT_ATTRIBUTES;
+
+#define InitializeObjectAttributes( p, n, a, r, s ) { \
+    (p)->Length = sizeof( OBJECT_ATTRIBUTES );          \
+    (p)->RootDirectory = r;                             \
+    (p)->Attributes = a;                                \
+    (p)->ObjectName = n;                                \
+    (p)->SecurityDescriptor = s;                        \
+    (p)->SecurityQualityOfService = NULL;               \
+    }
+
+#define OBJ_CASE_INSENSITIVE    0x00000040L
+#define OBJ_KERNEL_HANDLE       0x00000200L
+typedef ULONG ACCESS_MASK;
+
+#ifndef _W64
+# define _W64
+#endif
+typedef _W64 long LONG_PTR, *PLONG_PTR;
+typedef _W64 unsigned long ULONG_PTR, *PULONG_PTR;
+typedef LONG KPRIORITY;
+typedef ULONG KAFFINITY;
+
+typedef struct _PROCESS_BASIC_INFORMATION {
+    NTSTATUS ExitStatus;
+    char *PebBaseAddress;
+    ULONG_PTR AffinityMask;
+    KPRIORITY BasePriority;
+    ULONG_PTR UniqueProcessId;
+    ULONG_PTR InheritedFromUniqueProcessId;
+} PROCESS_BASIC_INFORMATION;
+typedef PROCESS_BASIC_INFORMATION *PPROCESS_BASIC_INFORMATION;
+
+typedef struct _CLIENT_ID {
+    HANDLE UniqueProcess;
+    HANDLE UniqueThread;
+} CLIENT_ID;
+typedef CLIENT_ID *PCLIENT_ID;
+
+typedef struct _THREAD_BASIC_INFORMATION { // Information Class 0
+    NTSTATUS ExitStatus;
+    PNT_TIB TebBaseAddress;
+    CLIENT_ID ClientId;
+    KAFFINITY AffinityMask;
+    KPRIORITY Priority;
+    KPRIORITY BasePriority;
+} THREAD_BASIC_INFORMATION, *PTHREAD_BASIC_INFORMATION;
+
+typedef enum {MEMORY_RESERVE_ONLY = MEM_RESERVE, 
+              MEMORY_COMMIT = MEM_RESERVE|MEM_COMMIT
+} memory_commit_status_t;
+
+typedef struct _IO_STATUS_BLOCK {
+    NTSTATUS Status;
+    // PVOID Pointer - Reserved, supposedly in a union with Status. see MSDN
+    ULONG Information;          // MSDN says it is ULONG_PTR
+} IO_STATUS_BLOCK, *PIO_STATUS_BLOCK;
+
+typedef struct _USER_STACK {
+    PVOID FixedStackBase;
+    PVOID FixedStackLimit;
+    PVOID ExpandableStackBase;
+    PVOID ExpandableStackLimit;
+    PVOID ExpandableStackBottom;
+} USER_STACK, *PUSER_STACK;
+
+#define FILE_SYNCHRONOUS_IO_NONALERT            0x00000020
+
+typedef enum _THREADINFOCLASS {
+    ThreadBasicInformation,
+    ThreadTimes,
+    ThreadPriority,
+    ThreadBasePriority,
+    ThreadAffinityMask,
+    ThreadImpersonationToken,
+    ThreadDescriptorTableEntry,
+    ThreadEnableAlignmentFaultFixup,
+    ThreadEventPair_Reusable,
+    ThreadQuerySetWin32StartAddress,
+    ThreadZeroTlsCell,
+    ThreadPerformanceCount,
+    ThreadAmILastThread,
+    ThreadIdealProcessor,
+    ThreadPriorityBoost,
+    ThreadSetTlsArrayAddress,
+    ThreadIsIoPending,
+    ThreadHideFromDebugger,
+    MaxThreadInfoClass
+    } THREADINFOCLASS;
+
+typedef enum _PROCESSINFOCLASS {
+    ProcessBasicInformation,
+    ProcessQuotaLimits,
+    ProcessIoCounters,
+    ProcessVmCounters,
+    ProcessTimes,
+    ProcessBasePriority,
+    ProcessRaisePriority,
+    ProcessDebugPort,
+    ProcessExceptionPort,
+    ProcessAccessToken,
+    ProcessLdtInformation,
+    ProcessLdtSize,
+    ProcessDefaultHardErrorMode,
+    ProcessIoPortHandlers,          // Note: this is kernel mode only
+    ProcessPooledUsageAndLimits,
+    ProcessWorkingSetWatch,
+    ProcessUserModeIOPL,
+    ProcessEnableAlignmentFaultFixup,
+    ProcessPriorityClass,
+    ProcessWx86Information,
+    ProcessHandleCount,
+    ProcessAffinityMask,
+    ProcessPriorityBoost,
+    ProcessDeviceMap,
+    ProcessSessionInformation,
+    ProcessForegroundInformation,
+    ProcessWow64Information,
+    MaxProcessInfoClass
+} PROCESSINFOCLASS;
+
+typedef struct _DESCRIPTOR_TABLE_ENTRY {
+    ULONG Selector;
+    LDT_ENTRY Descriptor;
+} DESCRIPTOR_TABLE_ENTRY, *PDESCRIPTOR_TABLE_ENTRY;
+
+
+/************************************************************************/
+static bool
+nt_remote_allocate_virtual_memory(HANDLE process, void **base, uint size, 
+                                  uint prot, memory_commit_status_t commit)
+{
+    NTSTATUS res;
+    GET_NTDLL(NtAllocateVirtualMemory, (IN HANDLE ProcessHandle,
+                                        IN OUT PVOID *BaseAddress,
+                                        IN ULONG ZeroBits,
+                                        IN OUT PULONG AllocationSize,
+                                        IN ULONG AllocationType,
+                                        IN ULONG Protect));
+    uint sz = size;
+    assert(ALIGNED(*base, PAGE_SIZE));
+    res = NtAllocateVirtualMemory(process, base, 0 /* zero bits */, &sz, 
+                                  commit, prot);
+    assert(sz >= size);
+    return NT_SUCCESS(res);
+}
+
+static bool
+query_thread_info(HANDLE h, THREAD_BASIC_INFORMATION *info)
+{
+    NTSTATUS res;
+    GET_NTDLL(NtQueryInformationThread, (IN HANDLE ThreadHandle,
+                                         IN THREADINFOCLASS ThreadInformationClass,
+                                         OUT PVOID ThreadInformation,
+                                         IN ULONG ThreadInformationLength,
+                                         OUT PULONG ReturnLength OPTIONAL));
+    int got;
+    memset(info, 0, sizeof(THREAD_BASIC_INFORMATION));
+    res = NtQueryInformationThread(h, ThreadBasicInformation,
+                                   info, sizeof(THREAD_BASIC_INFORMATION), &got);
+    assert(!NT_SUCCESS(res) || got == sizeof(THREAD_BASIC_INFORMATION));
+    return NT_SUCCESS(res);
+}
+
+static bool
+query_process_info(HANDLE h, PROCESS_BASIC_INFORMATION *info)
+{
+    NTSTATUS res;
+    GET_NTDLL(NtQueryInformationProcess, (IN HANDLE ProcessHandle,
+                                          IN PROCESSINFOCLASS ProcessInformationClass,
+                                          OUT PVOID ProcessInformation,
+                                          IN ULONG ProcessInformationLength,
+                                          OUT PULONG ReturnLength OPTIONAL));
+    int got;
+    memset(info, 0, sizeof(PROCESS_BASIC_INFORMATION));
+    res = NtQueryInformationProcess(h, ProcessBasicInformation, info, 
+                                    sizeof(PROCESS_BASIC_INFORMATION), &got);
+    assert(!NT_SUCCESS(res) || got == sizeof(PROCESS_BASIC_INFORMATION));
+    return NT_SUCCESS(res);
+}
+
+static bool
+set_win32_start_addr(HANDLE h, uint *start_addr)
+{
+    NTSTATUS res;
+    GET_NTDLL(NtSetInformationThread, (IN HANDLE ThreadHandle,
+                                       IN THREADINFOCLASS ThreadInfoClass,
+                                       IN PVOID ThreadInformation,
+                                       IN ULONG ThreadInformationLength));
+    res = NtSetInformationThread(h, ThreadQuerySetWin32StartAddress,
+                                 start_addr, sizeof(*start_addr));
+    return NT_SUCCESS(res);
+}
+
+static bool
+wchar_to_unicode(PUNICODE_STRING dst, PCWSTR src)
+{
+    NTSTATUS res;
+    GET_NTDLL(RtlInitUnicodeString, (IN OUT PUNICODE_STRING DestinationString,
+                                     IN PCWSTR SourceString));
+    res = RtlInitUnicodeString(dst, src);
+    return NT_SUCCESS(res);
+}
+
+static uint
+remove_writecopy(uint prot) 
+{
+    uint other = prot & (PAGE_GUARD | PAGE_NOCACHE | PAGE_WRITECOMBINE);
+    prot &= ~(PAGE_GUARD | PAGE_NOCACHE | PAGE_WRITECOMBINE);
+
+    switch (prot) {
+    case PAGE_READONLY:         
+    case PAGE_READWRITE:                
+    case PAGE_EXECUTE:          
+    case PAGE_EXECUTE_READ:     
+    case PAGE_EXECUTE_READWRITE:
+        break;
+    case PAGE_WRITECOPY:
+        prot = PAGE_READWRITE;
+        break;
+    case PAGE_EXECUTE_WRITECOPY:
+        prot = PAGE_EXECUTE_READWRITE;
+        break;
+    }
+    return (prot | other);
+}
+
+static bool
+prot_is_readable(uint prot) 
+{
+    prot &= ~(PAGE_GUARD | PAGE_NOCACHE | PAGE_WRITECOMBINE);
+
+    switch (prot) {
+    case PAGE_READONLY:         
+    case PAGE_READWRITE:        
+    case PAGE_WRITECOPY:        
+    case PAGE_EXECUTE:          
+    case PAGE_EXECUTE_READ:     
+    case PAGE_EXECUTE_READWRITE:
+    case PAGE_EXECUTE_WRITECOPY: return true;
+    }
+    return false;
+}
+
+static bool
+prot_is_writable(uint prot) 
+{
+    prot &= ~(PAGE_GUARD | PAGE_NOCACHE | PAGE_WRITECOMBINE);
+    return (prot == PAGE_READWRITE || prot == PAGE_WRITECOPY ||
+            prot == PAGE_EXECUTE_READWRITE || prot == PAGE_EXECUTE_WRITECOPY);
+}
+
+static char *
+mem_state_string(DWORD state)
+{
+    switch (state) {
+    case 0:            return "none";
+    case MEM_COMMIT:   return "COMMIT";
+    case MEM_FREE:     return "FREE";
+    case MEM_RESERVE:  return "RESERVE";
+    }
+    return "<error>";
+}
+
+static char *
+mem_type_string(DWORD type)
+{
+    switch (type) {
+    case 0:            return "none";
+    case MEM_IMAGE:    return "IMAGE";
+    case MEM_MAPPED:   return "MAPPED";
+    case MEM_PRIVATE:  return "PRIVATE";
+    }
+    return "<error>";
+}
+
+static char *
+prot_string(DWORD prot)
+{
+    DWORD ignore_extras = prot & ~(PAGE_GUARD | PAGE_NOCACHE | PAGE_WRITECOMBINE);
+    switch (ignore_extras) {
+    case PAGE_NOACCESS:          return "----";
+    case PAGE_READONLY:          return "r---";
+    case PAGE_READWRITE:         return "rw--";
+    case PAGE_WRITECOPY:         return "rw-c";
+    case PAGE_EXECUTE:           return "--x-";
+    case PAGE_EXECUTE_READ:      return "r-x-";
+    case PAGE_EXECUTE_READWRITE: return "rwx-";
+    case PAGE_EXECUTE_WRITECOPY: return "rwxc";
+    }
+    return "<error>";
+}
+
+void
+dump_mbi(MEMORY_BASIC_INFORMATION *mbi)
+{
+    printf("BaseAddress:       %08x\n"
+           "AllocationBase:    %08x\n"
+           "AllocationProtect: %08x %s\n"
+           "RegionSize:        %08x\n"
+           "State:             %08x %s\n"
+           "Protect:           %08x %s\n"
+           "Type:              %08x %s\n",
+           mbi->BaseAddress,
+           mbi->AllocationBase,
+           mbi->AllocationProtect, prot_string(mbi->AllocationProtect),
+           mbi->RegionSize,
+           mbi->State, mem_state_string(mbi->State),
+           mbi->Protect, prot_string(mbi->Protect),
+           mbi->Type, mem_type_string(mbi->Type));
+}
+
+/********************************************************************************/
+
+typedef struct _mem_map {
+    char *old;
+    char *new;
+} mem_map;
+
+static int map_count = 0;
+static mem_map map[4097];
+static int pending_map_count = 0;
+static char * pending_map[4097];
+
+static void
+add_mapped_addr(char *old, char *new)
+{
+    assert(map_count < BUFFER_SIZE_ELEMENTS(map));
+    map[map_count].new = new;
+    map[map_count].old = old;
+    map_count++;
+}
+
+static void
+add_pending_mapped_addr(char *addr)
+{
+    assert(pending_map_count < BUFFER_SIZE_ELEMENTS(map));
+    pending_map[pending_map_count++] = addr;
+}
+
+static bool
+is_pending_mapped_addr(char *addr)
+{
+    int i;
+    for (i = 0; i < pending_map_count; i++) {
+        if (pending_map[i] == addr)
+            return true;
+    }
+    return false;
+}
+
+/* we can get NULL from a bad TEB so use this instead */
+#define FAIL ((char *)-1)
+
+static char *
+get_mapped_addr(char *old) 
+{
+    int i;
+    for (i = 0; i < map_count; i++) {
+        if (map[i].old == old)
+            return map[i].new;
+    }
+    return FAIL;
+}
+
+static char *
+get_original_addr(char *new)
+{
+    int i;
+    for (i = 0; i < map_count; i++) {
+        if (map[i].new == new)
+            return map[i].old;
+    }
+    return FAIL;
+}
+
+/* NOTE - sum users implicitly assume this is exactly 1 page */
+static char buf[4096];
+
+/* Note, because of my sloppy file format, type may or may not have a 
+ * description string, so we skip it and just eat the line, we can't do
+ * anything with the type information anyways */
+static bool
+read_mbi(FILE *file, MEMORY_BASIC_INFORMATION *mbi) 
+{
+    int res;
+    char *resc;
+    char dummy_buf[128];
+    memset(mbi, 0, sizeof(MEMORY_BASIC_INFORMATION));
+    res = fscanf(file, "\nBaseAddress=0x%08x\nAllocationBase=0x%08x\n"
+                 "AllocationProtect=0x%08x %*s\nRegionSize=0x%08x\n"
+                 "State=0x%08x %*s\nProtect=0x%08x %*s\n",
+                 &(mbi->BaseAddress), &(mbi->AllocationBase), 
+                 &(mbi->AllocationProtect), &(mbi->RegionSize), 
+                 &(mbi->State), &(mbi->Protect));
+    if (res != 6)
+        return false;
+    /* eat the type line */
+    resc = fgets(dummy_buf, sizeof(dummy_buf), file);
+    assert(resc != NULL);
+    return true;
+}
+
+static void
+print_descriptor(DESCRIPTOR_TABLE_ENTRY *entry)
+{
+    /* IA-32 SDM Volume 3A, section 3.4.5, "Segment Descriptors".
+     * The segment type is determined by the S bit (0=system,
+     * 1=code or data) and the 4-bit type field:
+     */
+    static const char *types[] = {"Data RO             ",
+                                  "Data RO, acc        ",
+                                  "Data R/W,           ",
+                                  "Data R/W, acc       ",
+                                  "Data RO, down       ",
+                                  "Data RO, down, acc  ",
+                                  "Data R/W, down      ",
+                                  "Data R/W, down, acc ",
+                                  "Code EO             ",
+                                  "Code EO, acc        ",
+                                  "Code E/R            ",
+                                  "Code E/R, acc       ",
+                                  "Code EO, conf       ",
+                                  "Code EO, conf, acc  ",
+                                  "Code E/RO, conf     ",
+                                  "Code E/RO, conf, acc"};
+
+    LDT_ENTRY *descr = &entry->Descriptor;
+    
+    uint base = (descr->BaseLow | 
+                 (descr->HighWord.Bits.BaseMid << 16) | 
+                 (descr->HighWord.Bits.BaseHi << 24));
+            
+    uint limit = descr->LimitLow | (descr->HighWord.Bits.LimitHi << 16);
+    if (descr->HighWord.Bits.Granularity == 1) {
+        limit = limit << 12 | 0xfff;
+    }
+    
+    printf("\t%04x ", entry->Selector);
+    printf("%08x ", base);
+    printf("%08x ", limit);
+
+    /* The definition for LDT_ENTRY combines the S and type fields 
+     * into a five bit field. */
+    if (TEST(0x10, descr->HighWord.Bits.Type)) {
+        printf("%s ", types[descr->HighWord.Bits.Type & 0xf]);
+    } else {
+        printf("System               ");
+    }
+
+    printf(" %x  ", descr->HighWord.Bits.Dpl);
+    printf(" %x  ", descr->HighWord.Bits.Default_Big);
+    printf("%s  ", descr->HighWord.Bits.Granularity == 1 ? "4kb" : " 1b");
+    printf(" %x   ", descr->HighWord.Bits.Pres);
+    printf("%x ", descr->HighWord.Bits.Reserved_0);
+    printf(" %x\n", descr->HighWord.Bits.Sys);
+}
+
+static void
+print_descriptors(DESCRIPTOR_TABLE_ENTRY entries[6])
+{
+    int i;
+    printf("\n\tSel    Base    Limit            Type        Dpl D/B Gran Pres L Sys\n"
+           "\t---- -------- -------- -------------------- --- --- ---- ---- - ---\n");
+
+    for (i=0; i<6; i++) {
+        /* Intel SDM vol. 3A, section 3.4.2: the first entry in the LDT
+         * is null and not used by the processor */
+        if (entries[i].Selector != 0) {
+            print_descriptor(&entries[i]);
+        }
+    }
+    printf("\n");
+}
+
+static void
+insert_entry(uint selector, uint word1, uint word2, DESCRIPTOR_TABLE_ENTRY entries[6])
+{
+    int i;
+
+    /* Intel SDM vol. 3A, section 3.4.2: the first entry in the LDT
+     * is null and not used by the processor */
+    if (selector != 0) {
+        for (i=0; i<6; i++) {
+            /* don't insert duplicates */
+            if (entries[i].Selector == selector)
+                return;
+
+            /* 0 indicates empty slot in entries[] */
+            if (entries[i].Selector == 0) {
+                entries[i].Selector = selector;
+                *((PULONG)&entries[i].Descriptor) = word1;
+                *((PULONG)&entries[i].Descriptor.HighWord) = word2;
+                return;
+            }
+        }
+    }       
+}
+
+/* macro to read descriptor info and insert it in a table */
+#define read_descriptor_and_insert(name, reg) {                                         \
+        fgets(line, sizeof(line), file);                                                \
+        res = sscanf(line, name "=0x%04x (0x%08x 0x%08x)\n", &reg, &word1, &word2);     \
+        assert(res == 3 || res == 1);                                                   \
+        if (res == 3)                                                                   \
+            insert_entry(reg, word1, word2, entries);                                   \
+        }
+
+static void
+read_threads(FILE *file, bool create, HANDLE hProc) {
+    HANDLE hThread;
+    DWORD thread_id, new_id, res;
+    THREAD_BASIC_INFORMATION thread_info;
+    CLIENT_ID cid;
+    OBJECT_ATTRIBUTES oa;
+    USER_STACK stack = {0};
+    CONTEXT cxt;
+    GET_NTDLL(NtCreateThread, (OUT PHANDLE ThreadHandle,
+                               IN ACCESS_MASK DesiredAccess,
+                               IN POBJECT_ATTRIBUTES ObjectAttributes,
+                               IN HANDLE ProcessHandle,
+                               OUT PCLIENT_ID ClientId,
+                               IN PCONTEXT ThreadContext,
+                               IN PUSER_STACK UserStack,
+                               IN BOOLEAN CreateSuspended));
+
+    if (create) {
+        cxt.ContextFlags = CONTEXT_INTEGER|CONTEXT_CONTROL;
+        /* initialize, unsafe but should be ok right? */
+        GetThreadContext(GetCurrentThread(), &cxt);
+        InitializeObjectAttributes(&oa, NULL, OBJ_CASE_INSENSITIVE, 
+                                   NULL, NULL);
+    }
+
+    while (fscanf(file, "Thread=0x%08x\n", &thread_id)) {
+        bool valid_state = true, valid_handle_state = false;
+        uint handle_rights;
+        char *teb;
+        bool valid_selectors = false;
+        uint Cs, Ss, Ds, Es, Fs, Gs;
+        DESCRIPTOR_TABLE_ENTRY entries[6] = {0,};
+        uint word1, word2;
+        char line[64];
+        long pos;
+        uint win32_start_addr = 0;
+
+        res = fscanf(file, "TEB=0x%08x\n", &teb);
+        assert(res == 1);
+
+        res = fscanf(file, "HandleRights=0x%08x\n", &handle_rights);
+        if (res == 1) {
+            valid_handle_state = true;
+        }
+
+        res = fscanf(file, "Eax=0x%08x, Ebx=0x%08x, Ecx=0x%08x, Edx=0x%08x\n",
+                     &cxt.Eax, &cxt.Ebx, &cxt.Ecx, &cxt.Edx);
+        if (res == 4) {
+            res = fscanf(file, "Esi=0x%08x, Edi=0x%08x, Esp=0x%08x, Ebp=0x%08x\n",
+                         &cxt.Esi, &cxt.Edi, &cxt.Esp, &cxt.Ebp);
+            assert(res == 4);
+            res = fscanf(file, "EFlags=0x%08x, Eip=0x%08x\n\n",
+                         &cxt.EFlags, &cxt.Eip);
+            assert(res == 2);
+
+            /* look for segment registers and associated descriptors */
+            pos = ftell(file);
+            fgets(line, sizeof(line), file);
+            res = sscanf(line, "Cs=0x%04x (0x%08x 0x%08x)\n", &Cs, &word1, &word2);
+            if (res == 3 || res == 1) {
+                valid_selectors = true;
+                if (res == 3)
+                    insert_entry(Cs, word1, word2, entries);
+
+                read_descriptor_and_insert("Ss", Ss);
+                read_descriptor_and_insert("Ds", Ds);
+                read_descriptor_and_insert("Es", Es);
+                read_descriptor_and_insert("Fs", Fs);
+                read_descriptor_and_insert("Gs", Gs);
+            } else {
+                /* put file position back a line */
+                fseek(file, pos, SEEK_SET);
+            }
+
+            /* look for the win32 start address */
+            pos = ftell(file);
+            res = fscanf(file, "Win32StartAddr=0x%08x\n", &win32_start_addr);
+            if (res != 1) {
+                /* set start addr to 0 so we know right away if the thread 
+                 * doesn't have a start address in the ldmp.
+                 */
+                win32_start_addr = 0;
+
+                /* put file position back a line */
+                fseek(file, pos, SEEK_SET);
+            }
+        } else {
+            valid_state = false;
+            fgets(buf, sizeof(buf), file);
+            /* only show error when creating to avoid duplicate messages */
+            if (create) {
+                printf("\nError reading thread state for original thread tid=0x%04x\n",
+                       thread_id);
+                printf("%s", buf);
+            }
+            fgets(buf, sizeof(buf), file); /* read in the newline */
+            /* clear context integer registers */
+            cxt.Eax = 0; cxt.Ebx = 0; cxt.Ecx = 0; cxt.Edx = 0; cxt.EFlags = 0;
+            cxt.Edi = 0; cxt.Esi = 0; cxt.Esp = 0; cxt.Ebp = 0; cxt.Eip = 0;
+        }
+
+        if (create) {
+            res = NtCreateThread(&hThread, THREAD_ALL_ACCESS, &oa, hProc,
+                                 &cid, &cxt, &stack, TRUE);
+            assert(NT_SUCCESS(res));
+            res = set_win32_start_addr(hThread, &win32_start_addr);
+            assert(res);
+            res = query_thread_info(hThread, &thread_info);
+            assert(res);
+            new_id = (uint)thread_info.ClientId.UniqueThread;
+            printf("created thread tid=0x%04x with TEB=0x%08x original tid=0x%04x with TEB=0x%08x\n",
+                   new_id, thread_info.TebBaseAddress, thread_id, teb);
+            if (valid_selectors) {
+                printf("\tcs=%04x ss=%04x ds=%04x es=%04x fs=%04x gs=%04x\n",
+                       Cs, Ss, Ds, Es, Fs, Gs);
+            }
+            if (entries[0].Selector != 0) {
+                print_descriptors(entries);
+            }
+            if (valid_handle_state) {
+                printf("\tHandleRights=0x%08x\n", handle_rights);
+            }
+            if (teb == NULL) {
+                printf("\twill be unable to copy over TEB\n");
+            } else {
+                add_mapped_addr(teb, (char *)thread_info.TebBaseAddress);
+            }
+            if (!valid_state)
+                printf("\tnew thread's register state is invalid\n\n");
+            CloseHandle(hThread);
+        } else {
+            /* prevent this region from being copied over till we know 
+             * where to put it */
+            add_pending_mapped_addr(teb);
+        }
+    }
+}
+
+static HANDLE
+create_process(char *path)
+{
+    HANDLE hProc, hSection, hFile;
+    UNICODE_STRING uexe;
+    OBJECT_ATTRIBUTES oa;
+    IO_STATUS_BLOCK iosb;
+    wchar_t wpath[MAX_PATH];
+    GET_NTDLL(NtCreateProcess, (OUT PHANDLE ProcessHandle,
+                                IN ACCESS_MASK DesiredAccess,
+                                IN POBJECT_ATTRIBUTES ObjectAttributes,
+                                IN HANDLE InheritFromProcessHandle,
+                                IN BOOLEAN InheritHandles,
+                                IN HANDLE SectionHandle OPTIONAL,
+                                IN HANDLE DebugPort OPTIONAL,
+                                IN HANDLE ExceptionPort OPTIONAL));
+    GET_NTDLL(NtOpenFile, (OUT PHANDLE FileHandle,
+                           IN ACCESS_MASK DesiredAccess,
+                           IN POBJECT_ATTRIBUTES ObjectAttributes,
+                           OUT PIO_STATUS_BLOCK IoStatusBlock,
+                           IN ULONG ShareAccess,
+                           IN ULONG OpenOptions));
+    GET_NTDLL(NtCreateSection, (OUT PHANDLE SectionHandle,
+                                IN ACCESS_MASK DesiredAccess,
+                                IN POBJECT_ATTRIBUTES ObjectAttributes,
+                                IN PLARGE_INTEGER SectionSize OPTIONAL,
+                                IN ULONG Protect,
+                                IN ULONG Attributes,
+                                IN HANDLE FileHandle));
+    
+    /* create dummy process */
+    _snwprintf(wpath, BUFFER_SIZE_ELEMENTS(wpath), L"\\??\\%hs", path);
+    NULL_TERMINATE_BUFFER(wpath);
+#if VERBOSE
+    printf("dummy exe path = %ls\n", wpath);
+#endif
+    wchar_to_unicode(&uexe, wpath);
+    InitializeObjectAttributes(&oa, &uexe, OBJ_CASE_INSENSITIVE, NULL, NULL);
+    if (!NT_SUCCESS(NtOpenFile(&hFile, FILE_EXECUTE | SYNCHRONIZE, &oa, &iosb,
+                               FILE_SHARE_READ, FILE_SYNCHRONOUS_IO_NONALERT))) {
+        printf("failed to open dummy process exe file\n");
+        exit(0);
+    }
+    oa.ObjectName = 0;
+    if (!NT_SUCCESS(NtCreateSection(&hSection, SECTION_ALL_ACCESS, &oa,
+                                    0, PAGE_EXECUTE, SEC_IMAGE, hFile))) {
+        printf("failed to create section\n");
+    }
+    if (!NT_SUCCESS(NtCreateProcess(&hProc, PROCESS_ALL_ACCESS, &oa,
+                                    GetCurrentProcess(), true, hSection, 0, 0))) {
+        printf("failed to create dummy process\n");
+        exit(0);
+    }
+    return hProc;
+}
+
+static void *highest_address_copied = NULL;
+static bool reached_vsyscall_page = false;
+
+/* if just_mapped then only copies over mapped regions, otherwise copies over
+ * non-mapped regions only */
+static void
+copy_memory(FILE *file, bool just_mapped, HANDLE hProc)
+{
+    MEMORY_BASIC_INFORMATION mbi;
+    DWORD res;
+    fpos_t pos;
+    bool do_copy;
+    while (true) {
+        res = fgetpos(file, &pos);
+        assert(res == 0);
+        res = read_mbi(file, &mbi);
+        if (!res)
+            break;
+        if (mbi.State != MEM_FREE) {
+            char *allocation_base = (char *)mbi.AllocationBase;
+            uint allocation_protect = mbi.AllocationProtect;
+            uint allocation_size = 0;
+            char *target;
+            fpos_t last_mbi_pos; /* for unknown TEB backing out */
+            
+            /* we can't handle write copy flag! remove it */
+            allocation_protect = remove_writecopy(allocation_protect);
+#if VERBOSE
+            printf("allocation base = 0x%08x, protect = 0x%08x\n", 
+                   allocation_base, allocation_protect);
+#endif
+
+            do {
+                allocation_size += mbi.RegionSize;
+                /* scan past data */
+                if (prot_is_readable(mbi.Protect)) {
+                    /* FIXME : hack, rc1 core doesn't check for guard page so
+                     * sometimes memory is copied and sometimes it isn't if
+                     * is guard! (why?), post rc1 should check 
+                     * MEM_COMMIT && !guard && is_readable */
+                    assert(mbi.State == MEM_COMMIT);
+                    if (!TEST(PAGE_GUARD, mbi.Protect)) {
+                        res = fseek(file, mbi.RegionSize, SEEK_CUR);
+                        assert(res == 0);
+                    } else {
+#if RC1_HACK
+                        fpos_t hack_pos;
+                        MEMORY_BASIC_INFORMATION hack_mbi;
+                        res = fgetpos(file, &hack_pos);
+                        assert(res == 0);
+                        if (!read_mbi(file, &hack_mbi)) {
+                            res = fsetpos(file, &hack_pos);
+                            assert(res == 0);
+                            res = fseek(file, mbi.RegionSize, SEEK_CUR);
+                            assert(res == 0);
+                        } else {
+                            res = fsetpos(file, &hack_pos);
+                            assert(res == 0);
+                        }
+#endif
+                    }
+                }
+                res = fgetpos(file, &last_mbi_pos);
+                assert(res == 0);
+            } while (read_mbi(file, &mbi) && 
+                     mbi.State != MEM_FREE &&
+                     mbi.AllocationBase == allocation_base);
+            
+            /* see if we should copy over this allocation */
+            target = get_mapped_addr(allocation_base);
+            do_copy = (just_mapped && target != FAIL) ||
+                (!just_mapped && target == FAIL && 
+                 !is_pending_mapped_addr(allocation_base));
+            if (!do_copy) {
+                res = fsetpos(file, &last_mbi_pos);
+                assert(res == 0);
+                continue;
+            }
+
+            /* get allocation */    
+            if (target == FAIL) {
+                if (!ALIGNED(allocation_base, ALLOCATION_GRANULARITY)) {
+                    /* probably a TEB for a thread that wasn't in the all
+                     * threads list at time of ldmp */
+                    printf("Probable TEB for unknown thread region "
+                           "addr 0x%08x size 0x%08x\n", 
+                           allocation_base, allocation_size);
+                }
+                target = allocation_base;
+                /* FIXME : why can't we use VirtualAllocEx? it fails with
+                 * code 487 == INVALID_ADDRESS */
+                res = nt_remote_allocate_virtual_memory(hProc, &target, 
+                                                        allocation_size,
+                                                        allocation_protect, 
+                                                        MEMORY_COMMIT);
+                if (!res) {
+                    target = NULL;
+                    res = nt_remote_allocate_virtual_memory(hProc, &target, 
+                                                            allocation_size,
+                                                            allocation_protect,
+                                                            MEMORY_COMMIT);
+                }
+                if (!res) {
+                    printf("ERROR: unable to allocate memory at 0x%08x size 0x%08x, SKIPPING\n", 
+                           allocation_base, allocation_size);
+                    res = fsetpos(file, &last_mbi_pos);
+                    assert(res == 0);
+                    continue;
+                }
+
+                assert(target != NULL);
+                if (target != allocation_base) {
+                    printf("ERROR: unable to allocate memory at 0x%08x size 0x%08x\n\t will be copied to 0x%08x instead\n", 
+                           allocation_base, allocation_size, target);
+                }
+#if VERBOSE
+                printf("target = 0x%08x, base = 0x%08x\n", target, allocation_base);
+#endif
+            }
+#if VERBOSE
+            printf("size=0x%08x\n", allocation_size);
+            printf("target=0x%08x\n", target);
+#endif
+
+            /* reset file pointer */
+            res = fsetpos(file, &pos);
+            assert(res == 0);
+
+            /* walk memory */
+#define TARGET_ADDR (target + ((char *)mbi.BaseAddress - allocation_base))
+            res = fgetpos(file, &pos);
+            assert(res == 0);
+            while (read_mbi(file, &mbi) && mbi.State != MEM_FREE &&
+                   mbi.AllocationBase == allocation_base) {
+                if ((uint)mbi.BaseAddress > (uint)highest_address_copied)
+                    highest_address_copied = mbi.BaseAddress;
+                if (mbi.State == MEM_RESERVE) {
+                    res = VirtualFreeEx(hProc, TARGET_ADDR, mbi.RegionSize, 
+                                        MEM_DECOMMIT);
+                    if (!res && TARGET_ADDR == (char *)0x7ffe1000) {
+                        printf("unable to make post vsyscall/shared user data page 0x7ffe1000 reserve, skipping\n");
+                        /* in case break out, save position */
+                        res = fgetpos(file, &pos);
+                        assert(res == 0);
+                        continue;
+                    }
+                    assert(res);
+                } else {
+                    uint old_prot;
+                    assert(mbi.State = MEM_COMMIT);
+                    if (!TEST(PAGE_GUARD, mbi.Protect) && 
+                        prot_is_readable(mbi.Protect)) {
+                        uint i;
+                        /* copy over memory */
+                        res = VirtualProtectEx(hProc, TARGET_ADDR, 
+                                               mbi.RegionSize, PAGE_READWRITE, 
+                                               &old_prot);
+                        if (!res && TARGET_ADDR == (char *)0x7ffe0000) {
+                            printf("unable to copy over vsyscall/shared user data page 0x7ffe0000, skipping\n");
+                            reached_vsyscall_page = true;
+                            res = fseek(file, mbi.RegionSize, SEEK_CUR);
+                            assert(res == 0);
+                            /* in case break out, save position */
+                            res = fgetpos(file, &pos);
+                            assert(res == 0);
+                            continue;
+                        }
+                        assert(res);
+                        assert(mbi.RegionSize % sizeof(buf) == 0);
+                        for (i = 0; i < mbi.RegionSize; i += sizeof(buf)) {
+                            uint written;
+                            res = fread(buf, 1, sizeof(buf), file);
+                            assert(res == sizeof(buf));
+                            res = WriteProcessMemory(hProc, TARGET_ADDR+i, buf,
+                                                     sizeof(buf), &written);
+                            assert(res && written == sizeof(buf));
+                        }
+                    } else {
+#if RC1_HACK
+                        /* FIXME : rc1 hack, might have memory from guard page
+                         * somehow */
+                        fpos_t hack_pos;
+                        MEMORY_BASIC_INFORMATION hack_mbi;
+                        res = fgetpos(file, &hack_pos);
+                        assert(res == 0);
+                        if (!read_mbi(file, &hack_mbi)) {
+                            res = fsetpos(file, &hack_pos);
+                            assert(res == 0);
+                            res = fseek(file, mbi.RegionSize, SEEK_CUR);
+                            assert(res == 0);
+                        } else {
+                            res = fsetpos(file, &hack_pos);
+                            assert(res == 0);
+                        }
+#endif
+                    }
+                    res = VirtualProtectEx(hProc, TARGET_ADDR, mbi.RegionSize, 
+                                           remove_writecopy(mbi.Protect), 
+                                           &old_prot);
+                    assert(res);
+                }
+                /* in case break out, save position */
+                res = fgetpos(file, &pos);
+                assert(res == 0);
+            }
+            /* roll back last mbi */
+            res = fsetpos(file, &pos);
+            assert(res == 0);
+#undef TARGET_ADDR
+        }
+    }
+}
+
+/* creates a debuggable process that matches (roughly) the process that was 
+ * used to generate the .ldmp file, prints out a mapping of thread_ids from 
+ * the dump to the new process.
+ * ldmp.exe <.ldmp file> <windows path to dummy.exe (absolute, local drive)> */
+DWORD __cdecl
+main(DWORD argc, char *argv[], char *envp[])
+{
+    FILE *file;
+    HANDLE hProc;
+    PROCESS_BASIC_INFORMATION info;
+    MEMORY_BASIC_INFORMATION mbi;
+    char *pb;
+    uint drbase = 0;
+
+    DWORD res;
+    fpos_t thread_start_pos;
+
+    if (argc != 3) {
+        printf("usage : ldmp <.ldmp file> <dummy executable, absolute local \\ path>\n");
+        printf("ex from cygwin\n"
+               "ldmp ../logs/nspmon.exe.5124.0000000.ldmp c:\\\\tools\\\\dummy.exe");
+        exit(-1);
+    }
+    if (GetProcAddress(GetModuleHandle(L"ntdll.dll"), "NtCreateThreadEx")) {
+        printf("ERROR - ldmp.exe does not yet support running on Vista, please use\n\t"
+               "a 2k/XP/2k3 machine to open this ldmp file (xref case 10990).\n");
+        exit(-1);
+    }
+#if VERBOSE
+    printf("opening file\n");
+#endif
+    file = fopen(argv[1], "rb");
+    if (file == NULL) {
+        printf("unable to find file %s\n", argv[1]);
+        exit(-1);
+    }
+
+    hProc = create_process(argv[2]);
+
+    /* read peb (& message if there is one) */
+    res = fscanf(file, "PEB=0x%08x\n", &pb);
+    if (res != 1) {
+        uint length = 0;
+        /* newer format, starts with a message prefaced with number of bytes
+         * in message */
+        /* case 8912: don't read a newline here as it will gobble the next newline
+         * if the msg is empty; instead we add 1 to the length and read the newline
+         * w/ the msg.
+         */
+        res = fscanf(file, "0x%08x", &length);
+        assert(res == 1);
+        length = length + 1; /* newline after length, see comment above */
+        /* FIXME - should never happen, but we could handle by doing this in
+         * pieces */
+        assert(length < sizeof(buf));
+        if (length >= sizeof(buf))
+            length = sizeof(buf)-1;
+        res = fread(buf, 1, length, file); 
+        assert(res == length);
+        buf[length] = '\0';
+        /* Remember that buf has a newline as its first char */
+        printf("Message -%s- End Message\n", buf);
+        fflush(stdout);
+        res = fscanf(file, "PEB=0x%08x\n", &pb);
+    }
+    assert(res == 1);
+    res = query_process_info(hProc, &info);
+    assert(res);
+    printf("\ncreated dummy process pid=%d with PEB=0x%08x original PEB=0x%08x\n",
+           info.UniqueProcessId, info.PebBaseAddress, pb);
+
+    /* read dynamorio.dll base, present only in format after case 5366 */
+    if (fscanf(file, "dynamorio.dll=0x%08x\n", &drbase) == 1) {
+        /* for custom scripting */
+        printf("\ndynamorio.dll=0x%08x\n", drbase);
+        /* or more likely */
+        printf("\nwindbg -pv -p %d -c '.reload dynamorio.dll=0x%08x'\n", 
+               info.UniqueProcessId, drbase);
+    }
+
+    add_mapped_addr(pb, (char *)info.PebBaseAddress);
+    /* add shared user data (win2k) / syscall page (xp/03) to map list */
+    add_mapped_addr((char *)0x7ffe0000, (char *)0x7ffe0000);
+
+    /* iterate through threads without creating to build up pending
+     * teb mappings */
+    res = fgetpos(file, &thread_start_pos);
+    assert(res == 0);
+    read_threads(file, false, hProc);
+
+    printf("\n");
+    /* free memory in dummy process */
+    pb = NULL;
+    while(VirtualQueryEx(hProc, pb, &mbi, sizeof(mbi)) == sizeof(mbi)) {
+        if (mbi.State == MEM_FREE || get_original_addr(mbi.AllocationBase) != FAIL) {
+            if ((char *)mbi.BaseAddress + mbi.RegionSize < (char *)mbi.BaseAddress)
+                break;
+            pb = (char *)mbi.BaseAddress + mbi.RegionSize;
+        } else {
+            if (!VirtualFreeEx(hProc, mbi.AllocationBase, 0, MEM_RELEASE)) {
+                /* try an unmap */
+                GET_NTDLL(NtUnmapViewOfSection, (IN HANDLE ProcessHandle,
+                                                 IN PVOID BaseAddress));
+                if (!NT_SUCCESS(NtUnmapViewOfSection(hProc, mbi.AllocationBase))) {
+                    if (get_original_addr(mbi.AllocationBase) == FAIL) {
+                        printf("Unable to free memory region :\n");
+                        dump_mbi(&mbi);
+                    }
+                    pb += mbi.RegionSize;
+                } else {
+#if VERBOSE
+                    printf("unmapped allocation at 0x%08x\n", mbi.AllocationBase);
+#endif
+                }
+            } else {
+#if VERBOSE
+                printf("freed memory at 0x%08x\n", mbi.AllocationBase);
+#endif
+            }
+        }
+    }
+#if VERBOSE
+    printf("finished freeing memory, starting copy over\n");
+#endif
+
+    /* chomp any error line (such as all threads list freed) */
+    if (fscanf(file, "<%512[a-zA-Z1-9 ]>", &buf))
+        printf("%s\n", buf);
+
+    /* copy over directly corresponding (non-mapped) memory regions */
+    copy_memory(file, false, hProc);
+    
+    /* now create threads (can't do this before since we have no control over 
+     * where the teb's are placed and they might conflict with one of the 
+     * copied over regions above */
+    res = fsetpos(file, &thread_start_pos);
+    assert(res == 0);
+    read_threads(file, true, hProc);
+    printf("\n");
+
+    /* now copy over mapped allocations */
+    /* first chomp off any error line, we already printed it once above */
+    fscanf(file, "<%512[a-zA-Z1-9 ]>", &buf);
+    copy_memory(file, true, hProc);
+
+    if (!reached_vsyscall_page) {
+        printf("ERROR: failed to reach shared_user_data/vsyscall page, ldmp likely truncated.\n"
+               "       Memory above 0x%08x is likely unavailable or incorrect.\n\n",
+               highest_address_copied);
+    }
+
+    printf("finished\n");
+    CloseHandle(hProc);
+    return 0;
+}
diff --git a/tools/ldmp.notes b/tools/ldmp.notes
new file mode 100755
index 000000000..34cf20c2c
--- /dev/null
+++ b/tools/ldmp.notes
@@ -0,0 +1,11 @@
+to use ldmp.exe to view an ldmp file
+
+ldmp <ldmp file> <dummy executable> > map_file
+once finished open the map file (this file has the mappings of old thread ids and TEB's to the new one (the contents of the TEB are copied over, just the location changes) and also the PEB as well as any error msgs), we expect an error aroudn the vsyscall page since the os won't let us write there, for this reason I recommend running ldmp on the same platform (xpsp2, 03, 2000 etc.) that the .ldmp file was created on.  Read the process id and attach windbg NON-INVASIVELY (otherwise will crash) to that process.  At this point you can take a dump with 
+.dump /mFfu <out file>
+and continue on your merry way.
+
+Note that vadump Type information is lost as well as all writecopy bits on protection regions.  Note also that all handle information is lost and some other state (such as suspend counts etc.) is lost as well.  May be other things that change that we don't know about yet too.  Don't forget to use the map_file to translate thread ids, peb and teb addresses (TODO write a perl script to do this for you on demand).  Also be aware that the vsyscall page might not match the original applications, esp. if you ran ldmp.exe on a different platform then the dump was created on.  Also todo shell script to turn a .ldmp into a .dmp using ldmp and ntsd.  Note also that only CONTEXT_INTEGER, Eip, and EFlags registers are valid (if demand warrants we can add segement registers/debug registers without much work, float/sse is a little harder, but could also be done).
+
+RC1 notes
+Also, because of when the callstack is grabbed vs. the dumping of memory you'll have to adjust ebp for the main thread to see the main threads callstack.  The adjustment is the esp adjustment from os_dump_core_live_dump (currently 0x304 in debug builds) + 4 for the argument to get_own_context + 8 for the two push esp's (in os_dump_core_live_dump and get_own_context).  So for debug builds the offset is currently 0x310 (should be similar for release, just look at the callstack, isn't to hard to spot the return address to dr code).  This will be fixed on the core side post rc1.
diff --git a/tools/linux_install.sh b/tools/linux_install.sh
new file mode 100644
index 000000000..4f700bc24
--- /dev/null
+++ b/tools/linux_install.sh
@@ -0,0 +1,401 @@
+#!/bin/sh
+
+# **********************************************************
+# Copyright (c) 2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+echo "This script will install DynamoRIO to the current directory."
+echo "Continue [y/n]"
+
+read do_install
+
+if [ "$do_install" = "n" ] && [ "$do_install" = "N" ]
+then
+    exit
+fi
+
+echo "You must read and accept the licensing agreement before installing."
+echo "Press enter to read the licensing agreement"
+read tmp
+echo ""
+
+# This text is verbatim from the modified PRBA provided by David Howard 4/2/08
+echo "NOTICE: BY DOWNLOADING AND INSTALLING, COPYING OR OTHERWISE USING "
+echo "THE INTERFACES, SOFTWARE AND OTHER MATERIALS ON THIS CD, YOU "
+echo "AGREE TO BE BOUND BY THE TERMS OF THIS AGREEMENT.  IF YOU DO NOT "
+echo "AGREE TO THE TERMS OF THIS EULA, YOU MAY NOT DOWNLOAD, INSTALL, "
+echo "COPY OR USE THE INTERFACES, SOFTWARE OR OTHER MATERIALS ON THIS "
+echo "CD. \"YOU\" MEANS THE NATURAL PERSON OR THE ENTITY THAT IS "
+echo "AGREEING TO BE BOUND BY THIS EULA, THEIR EMPLOYEES AND THIRD "
+echo "PARTY CONTRACTORS THAT PROVIDE SERVICES TO YOU. YOU SHALL BE "
+echo "LIABLE FOR ANY FAILURE BY SUCH EMPLOYEES AND THIRD PARTY "
+echo "CONTRACTORS TO COMPLY WITH THE TERMS OF THIS AGREEMENT. "
+echo "  "
+echo "1. DEFINITIONS. "
+echo " "
+echo "(a \"Software\" shall mean the VMware software, in object code only, as set "
+echo "forth in Exhibit A attached hereto. "
+echo " "
+echo "(b \"Documentation\" shall mean the printed or online written reference "
+echo "material, if any, that may be furnished to Licensee in conjunction with the "
+echo "Software, including, without limitation, instructions, testing guidelines, "
+echo "and end user guides. "
+echo " "
+
+echo "--"
+echo "Press Enter to Continue"
+read tmp
+echo "--"
+
+echo "(c \"Intellectual Property Rights\" shall mean all intellectual property "
+echo "rights, including, without limitation, patent, copyright, trademark, and "
+echo "trade secret. "
+echo " "
+echo "(d \"Open Source Software\" shall mean various open source software "
+echo "components provided with the Software that are licensed to Licensee under "
+echo "the terms of the applicable license agreements included with such open "
+echo "source software components or other materials for the Software. "
+echo " "
+echo "(e \"Update(s\" shall mean any modification, error correction, bug fix, "
+echo "patch or other update to or for the Software. "
+echo " "
+echo "2. LICENSE GRANT, USE AND OWNERSHIP "
+echo " "
+echo "(a License.  Subject to the terms and conditions of this Agreement, VMware "
+echo "grants to Licensee a non-exclusive, revocable, non-transferable license "
+echo "(without the right to sublicense to use the Software and Documentation "
+echo "solely for purposes of internal testing and evaluation, as well as "
+echo "development of Licensee products that communicate with the Software. "
+echo " "
+
+echo "--"
+echo "Press Enter to Continue"
+read tmp
+echo "--"
+
+echo "(b Feedback. The purpose of this license is the internal testing and "
+echo "evaluation of the Software by Licensee and development by Licensee of "
+echo "Licensee products that communicate with the Software.  In furtherance of "
+echo "this purpose, Licensee shall from time to time provide feedback to VMware "
+echo "concerning the functionality and performance of the Software including, "
+echo "without limitation, identifying potential errors and improvements. "
+echo "Notwithstanding the foregoing, prior to Licensee disclosing to VMware any "
+echo "information in connection with this Agreement which Licensee considers "
+echo "proprietary or confidential, Licensee shall obtain VMware's prior written "
+echo "approval to disclose such information to VMware, and without such prior "
+echo "written approval from VMware, Licensee shall not disclose any such "
+echo "information to VMware.  Feedback and other information which is provided by "
+echo "Licensee to VMware in connection with the Software, Documentation, or this "
+echo "Agreement may be used by VMware to improve or enhance its products and, "
+echo "accordingly, VMware shall have a non-exclusive, perpetual, irrevocable, "
+echo "royalty-free, worldwide right and license to use, reproduce, disclose, "
+echo "sublicense, modify, make, have made, distribute, sell, offer for sale, "
+echo "display, perform, create derivative works, permit unmodified binary "
+echo "distribution and otherwise exploit such feedback and information without "
+echo "restriction. "
+echo " "
+
+echo "--"
+echo "Press Enter to Continue"
+read tmp
+echo "--"
+
+echo "(c Restrictions.  Licensee shall not copy or use the Software (including "
+echo "the Documentation or disseminate Confidential Information, as defined "
+echo "below, to any third party except as expressly permitted in this Agreement. "
+echo "Licensee will not, and will not permit any third party to, sublicense, "
+echo "rent, copy, modify, create derivative works of, translate, reverse "
+echo "engineer, decompile, disassemble, or otherwise reduce to human perceivable "
+echo "form any portion of the Software or Documentation.  In no event shall "
+echo "Licensee use the Software or Documentation for any commercial purpose "
+echo "except as expressly set forth in this Agreement.  The Software, "
+echo "Documentation, and all performance data and test results, including without "
+echo "limitation, benchmark test results (collectively \"Performance Data\", "
+echo "relating to the Software are the Confidential Information of VMware, and "
+echo "will be treated in accordance with the terms of Section 4 of this "
+echo "Agreement.  Accordingly, Licensee shall not publish or disclose to any "
+echo "third party any Performance Data relating to the Software.  Licensee shall "
+echo "immediately cease all use of the Software and Documentation, upon notice "
+echo "from VMware. "
+echo " "
+echo "(d Ownership.  VMware shall own and retain all right, title and interest "
+echo "in and to the Intellectual Property Rights in the Software, Documentation, "
+echo "and any derivative works thereof, subject only to the license expressly set "
+
+echo "--"
+echo "Press Enter to Continue"
+read tmp
+echo "--"
+
+echo "forth in Section 2(a hereof.  Licensee does not acquire any other rights, "
+echo "express or implied, in the Software or Documentation.  VMWARE RESERVES ALL "
+echo "RIGHTS NOT EXPRESSLY GRANTED HEREUNDER. "
+echo " "
+echo "(e No Support Services.  VMware is under no obligation to support the "
+echo "Software in any way or to provide any Updates to Licensee.  In the event "
+echo "VMware, in its sole discretion, supplies any Update to Licensee, such "
+echo "Update shall be deemed Software hereunder and shall be subject to the terms "
+echo "and conditions of this Agreement. Upon VMware's release of any Update, "
+echo "Licensee shall immediately cease all use of the former version of the "
+echo "Software. "
+echo " "
+echo "(f Third-Party Software.  The Software enables a computer to run multiple "
+echo "instances of third-party guest operating systems and application programs. "
+echo "Licensee acknowledges that Licensee is responsible for obtaining any "
+echo "licenses necessary to operate any such third-party software, including "
+echo "guest operating systems. "
+echo " "
+echo "(g Open Source Software.  The terms and conditions of this Agreement shall "
+echo "not apply to any Open Source Software accompanying the Software.  Any such "
+echo "Open Source Software is provided under the terms of the open source license "
+
+echo "--"
+echo "Press Enter to Continue"
+read tmp
+echo "--"
+
+echo "agreement or copyright notice accompanying such Open Source Software or in "
+echo "the open source licenses file accompanying the Software. "
+echo " "
+echo "(h Demonstration. Subject to the terms and conditions of this Agreement, "
+echo "VMware grants to Licensee a non-exclusive, revocable, non-transferable "
+echo "license (without the right to sublicense to use the Software for "
+echo "demonstration to third parties, provided Licensee (i has received prior "
+echo "written authorization from VMware for same, (ii has had such third parties "
+echo "first sign a confidentiality agreement that contains nondisclosure "
+echo "restrictions substantially similar to those set forth in this Agreement, "
+echo "(iii includes in such confidentiality agreement an acknowledgement that "
+echo "VMware does not promise or guarantee that features, functionality and/or "
+echo "modules in the Software will be included in any generally available version "
+echo "of the Software, or will be marketed separately for additional fees, and "
+echo "(iv conducts such demonstration solely on Licensee's hardware and such "
+echo "hardware remains at all times in Licensee's possession and control. "
+echo " "
+echo "3. TERM AND TERMINATION. This Agreement is effective as of the Effective "
+echo "Date and will continue for a one (1 year period (\"Initial Term\", unless "
+echo "amended to establish a later expiration date (\"Subsequent Term\" by a "
+echo "written agreement signed by both parties, or until terminated as provided "
+
+echo "--"
+echo "Press Enter to Continue"
+read tmp
+echo "--"
+
+echo "in this Agreement.  Either party may terminate this Agreement at any time "
+echo "for any reason or no reason by providing the other party advance written "
+echo "notice thereof.  Upon any expiration or termination of this Agreement, the "
+echo "rights and licenses granted to Licensee under this Agreement shall "
+echo "immediately terminate, and Licensee shall immediately cease using, and will "
+echo "return to VMware (or, at VMware's request, destroy, the Software, "
+echo "Documentation and all other tangible items in Licensee's possession or "
+echo "control that are proprietary to or contain Confidential Information.  The "
+echo "rights and obligations of the parties set forth in Sections 1, 2(b 2(c, "
+echo "2(d, 2(e, 3, 4, 5, 6 and 7 shall survive termination or expiration of "
+echo "this Agreement for any reason. "
+echo " "
+echo "4. CONFIDENTIALITY. \"Confidential Information\" shall mean all trade "
+echo "secrets, know-how, inventions, techniques, processes, algorithms, software "
+echo "programs, hardware, schematics, planned product features, functionality, "
+echo "methodology, performance and software source documents relating to the "
+echo "Software, and other information provided by VMware, whether disclosed "
+echo "orally, in writing, or by examination or inspection, other than information "
+echo "which Licensee can demonstrate (i was already known to Licensee, other "
+echo "than under an obligation of confidentiality, at the time of disclosure; "
+echo "(ii was generally available in the public domain at the time of disclosure "
+
+echo "--"
+echo "Press Enter to Continue"
+read tmp
+echo "--"
+
+echo "to Licensee; (iii became generally available in the public domain after "
+echo "disclosure other than through any act or omission of Licensee; (iv was "
+echo "subsequently lawfully disclosed to Licensee by a third party without any "
+echo "obligation of confidentiality; or (v was independently developed by "
+echo "Licensee without use of or reference to any information or materials "
+echo "disclosed by VMware or its suppliers.  Confidential Information shall "
+echo "include without limitation the Software, Documentation, Performance Data, "
+echo "any Updates, information relating to VMware products, product roadmaps, and "
+echo "other technical, business, financial and product development plans, "
+echo "forecasts and strategies.  Licensee shall not use any Confidential "
+echo "Information for any purpose other than as expressly authorized under this "
+echo "Agreement.  Except as otherwise set forth in this Agreement, in no event "
+echo "shall Licensee use the Software, Documentation or any other Confidential "
+echo "Information to develop, manufacture, market, sell, or distribute any "
+echo "product or service.  Licensee shall limit dissemination of Confidential "
+echo "Information to its employees who have a need to know such Confidential "
+echo "Information for purposes expressly authorized under this Agreement.  Except "
+echo "as otherwise set forth in this Agreement, in no event shall Licensee "
+echo "disclose any Confidential Information to any third party. Without limiting "
+echo "the foregoing, Licensee shall use at least the same degree of care that it "
+echo "uses to prevent the disclosure of its own confidential information of like "
+
+echo "--"
+echo "Press Enter to Continue"
+read tmp
+echo "--"
+
+echo "importance, but in no event less than reasonable care, to prevent the "
+echo "disclosure of Confidential Information. "
+echo " "
+echo "5. LIMITATION OF LIABILITY.  IT IS UNDERSTOOD THAT THE SOFTWARE, "
+echo "DOCUMENTATION AND ANY UPDATES ARE PROVIDED WITHOUT CHARGE FOR THE "
+echo "PURPOSES OF THIS AGREEMENT ONLY.  ACCORDINGLY, THE TOTAL "
+echo "LIABILITY OF VMWARE AND ITS SUPPLIERS ARISING OUT OF OR RELATED "
+echo "TO THIS AGREEMENT SHALL NOT EXCEED $100.  IN NO EVENT SHALL "
+echo "VMWARE OR ITS SUPPLIERS HAVE LIABILITY FOR ANY INDIRECT, "
+echo "INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES (INCLUDING, WITHOUT "
+echo "LIMITATION, DAMAGES FOR LOSS OF BUSINESS PROFITS, BUSINESS "
+echo "INTERRUPTION, OR LOSS OF BUSINESS INFORMATION, HOWEVER CAUSED "
+echo "AND ON ANY THEORY OF LIABILITY, EVEN IF VMWARE OR ITS SUPPLIERS "
+echo "HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.  THESE "
+echo "LIMITATIONS SHALL APPLY NOTWITHSTANDING ANY FAILURE OF ESSENTIAL "
+echo "PURPOSE OF ANY LIMITED REMEDY. "
+echo " "
+echo "6. WARRANTY DISCLAIMER.  IT IS UNDERSTOOD THAT THE SOFTWARE, "
+echo "DOCUMENTATION, AND ANY UPDATES MAY CONTAIN ERRORS AND ARE "
+echo "PROVIDED FOR THE PURPOSES OF THIS AGREEMENT ONLY.  THE SOFTWARE, "
+echo "DOCUMENTATION, AND ANY UPDATES ARE PROVIDED \"AS IS\" WITHOUT "
+
+echo "--"
+echo "Press Enter to Continue"
+read tmp
+echo "--"
+
+echo "WARRANTY OF ANY KIND, WHETHER EXPRESS, IMPLIED, STATUTORY, OR "
+echo "OTHERWISE. VMWARE AND ITS SUPPLIERS SPECIFICALLY DISCLAIM ALL "
+echo "IMPLIED WARRANTIES OF MERCHANTABILITY, NONINFRINGEMENT, AND "
+echo "FITNESS FOR A PARTICULAR PURPOSE.  "
+echo "Licensee acknowledges that VMware has not publicly "
+echo "announced the availability of the Software and/or Documentation, that "
+echo "VMware has not promised or guaranteed to Licensee that such Software and/or "
+echo "Documentation will be announced or made available to anyone in the future, "
+echo "that VMware has no express or implied obligation to Licensee to announce or "
+echo "introduce the Software and/or Documentation, and that VMware may not "
+echo "introduce a product similar or compatible with the Software and/or "
+echo "Documentation. Accordingly, Licensee acknowledges that any research or "
+echo "development that it performs regarding the Software or any product "
+echo "associated with the Software is done entirely at Licensee's own risk. "
+echo "Specifically, the Software may contain features, functionality or modules "
+echo "that may not be included in the generally available version of the Software "
+echo "and/or Documentation, or that may be marketed separately for additional "
+echo "fees. "
+echo " "
+
+echo "--"
+echo "Press Enter to Continue"
+read tmp
+echo "--"
+
+echo "7. OTHER PROVISIONS "
+echo " "
+echo "(a Governing Law.  This Agreement, and all disputes arising out of or "
+echo "related thereto, shall be governed by and construed under the laws of the "
+echo "State of California without reference to conflict of laws principles.  All "
+echo "such disputes shall be subject to the exclusive jurisdiction of the state "
+echo "and federal courts located in Santa Clara County, California, and the "
+echo "parties agree and submit to the personal and exclusive jurisdiction and "
+echo "venue of these courts. "
+echo " "
+echo "(b Assignment.  Licensee shall not assign this Agreement or any rights or "
+echo "obligations hereunder, directly or indirectly, by operation of law, merger, "
+echo "acquisition of stock or assets, or otherwise, without the prior written "
+echo "consent of VMware.  Subject to the foregoing, this Agreement shall inure to "
+echo "the benefit of and be binding upon the parties and their respective "
+echo "successors and permitted assigns. "
+echo " "
+echo "(c Export Regulations.  Licensee understands that VMware is subject to "
+echo "regulation by the U.S. government and its agencies, which prohibit export "
+echo "or diversion of certain technical products and information to certain "
+echo "countries and individuals.  Licensee warrants that it will comply in all "
+
+echo "--"
+echo "Press Enter to Continue"
+read tmp
+echo "--"
+
+echo "respects with all export and re-export restrictions applicable to the "
+echo "technology and documentation provided hereunder. "
+echo " "
+echo "(d Entire Agreement.  This is the entire agreement between the parties "
+echo "relating to the subject matter hereof and all other terms are rejected. "
+echo "This Agreement supersedes all previous communications, representations, "
+echo "understandings and agreements, either oral or written, between the parties "
+echo "with respect to said subject matter.  The terms of this Agreement supersede "
+echo "any VMware end user license agreement that may accompany the Software "
+echo "and/or Documentation.  No waiver or modification of this Agreement shall be "
+echo "valid unless made in a writing signed by both parties.  The waiver of a "
+echo "breach of any term hereof shall in no way be construed as a waiver of any "
+echo "term or other breach hereof.  If any provision of this Agreement is held by "
+echo "a court of competent jurisdiction to be contrary to law the remaining "
+echo "provisions of this Agreement shall remain in full force and effect. "
+echo " "
+echo "(e Notices. All notices must be sent by (a registered or certified mail, "
+echo "return receipt requested, (b reputable overnight air courier, (c "
+echo "facsimile with a confirmation copy sent by registered or certified mail, "
+echo "return receipt requested, or (d served personally.  Notices are effective "
+echo "immediately when served personally, five (5 days after posting if sent by "
+
+echo "--"
+echo "Press Enter to Continue"
+read tmp
+echo "--"
+
+echo "registered or certified mail, two (2 days after being sent by overnight "
+echo "courier, or one (1 day after being transmitted by facsimile.  Notices to "
+echo "either party shall be directed to the party's address set forth in this "
+echo "Agreement.  Either party may change its address for notification under this "
+echo "Agreement, by notifying the other party in accordance with this Section. "
+echo ""
+
+# This is pretty mean - just exiting if they're just hitting enter through the license
+# but this way they will have to pay some attention at least.  If this is too mean
+# switch to a while loop to check for yes.
+echo "--"
+echo "Have you read and understood the terms of this agreement? [yes/no]"
+read understood
+if [ "$understood" != "yes" ] && [ "$understood" != "Yes" ] && [ "$understood" != "YES" ]
+then
+    echo "You must read and understand the terms of the licensing agreement"
+    echo "before installation can continue."
+    exit
+fi
+echo "Do you accept the terms of this agreement? Type \"I Accept\" to accept."
+read acceptance
+if [ "$acceptance" != "I Accept" ] && [ "$acceptance" != "i accept" ] && \
+   [ "$acceptance" != "I accept" ] && [ "$acceptance" != "I ACCEPT" ]
+then
+    echo "You must accept the licensing agreement before installation can continue."
+    exit
+fi
+
+# Okay ready to install
+unzip -P UId3E4c2WPSiFlOzJDQx dynamorio.zip
+
+echo "Finished"
\ No newline at end of file
diff --git a/tools/make-times b/tools/make-times
new file mode 100755
index 000000000..29c7da8db
--- /dev/null
+++ b/tools/make-times
@@ -0,0 +1,48 @@
+#!/bin/tcsh
+
+# **********************************************************
+# Copyright (c) 2002 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# example of usage: make-times 22 t-dy t-nody
+
+echo "# in dir" `pwd`":" $2 "/" $3
+# tail on # of benchmarks
+paste $2 $3 | tail -$1 | awk '{print $1 "\t" $2/$7}' > ._sn
+paste $2 $3 | tail -$1 | awk '{print $1 "\t" $3/$8}' > ._so
+awk '{print $2}' < ._so > ._so1
+cat header
+paste ._sn ._so1
+# harmonic mean = num / sum(1/datum):
+awk '{sum += 1/$2; num+=1} END {print num/sum}' < ._so > ._so_ave
+awk '{sum += 1/$2; num+=1} END {print num/sum}' < ._sn > ._sn_ave
+echo "H_MEAN"  `cat ._sn_ave`  `cat ._so_ave` 
+rm ._so ._sn ._so1 ._so_ave ._sn_ave
+
+
diff --git a/tools/make-times-optonly b/tools/make-times-optonly
new file mode 100755
index 000000000..1a6c3066b
--- /dev/null
+++ b/tools/make-times-optonly
@@ -0,0 +1,42 @@
+#!/bin/tcsh
+
+# **********************************************************
+# Copyright (c) 2002 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# example of usage: make-times 22 t-dy t-nody
+
+echo "# in dir" `pwd`":" $2 "/" $3
+# tail on # of benchmarks
+paste $2 $3 | tail -$1 | awk '{print $1 "\t" $3/$8}' > ._so
+cat ._so
+# harmonic mean = num / sum(1/datum):
+awk '{sum += 1/$2; num+=1} END {print num/sum}' < ._so > ._so_ave
+echo "H_MEAN " `cat ._so_ave` 
+rm ._so ._so_ave
diff --git a/tools/maketar b/tools/maketar
new file mode 100755
index 000000000..fa8c06540
--- /dev/null
+++ b/tools/maketar
@@ -0,0 +1,101 @@
+#!/bin/bash
+
+# **********************************************************
+# Copyright (c) 2002-2003 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# Builds dynamorio.tar for Linux release of DynamoRIO
+
+usage="<absolute path for new install dir>";
+
+# mandatory 1st argument
+if [ $# -lt 1 ]; then
+    echo "Usage: $0 $usage"
+    exit 127
+fi
+base=$1
+if test -e $base ; then 
+    echo "Error: directory $base already exists"
+    exit 127
+fi
+mkdir $base
+mkdir $base/dynamorio
+mkdir $base/dynamorio/logs
+mkdir $base/dynamorio/doc
+mkdir $base/dynamorio/include
+mkdir $base/dynamorio/bin
+mkdir $base/dynamorio/lib
+mkdir $base/dynamorio/lib/debug
+mkdir $base/dynamorio/lib/release
+mkdir $base/dynamorio/lib/profile
+
+cd $DYNAMORIO_HOME
+
+cp tools/dynamorio $base/dynamorio/bin
+
+cd src
+if test -e Makefile.mydefines ; then 
+    mv Makefile.mydefines .ignoreme
+fi
+make clear
+make CLIENT_INTERFACE=1 EXTERNAL_RELEASE=1 install
+cp ../lib/debug/* $base/dynamorio/lib/debug
+cp ../lib/*.so $base/dynamorio/lib
+cp ../include/* $base/dynamorio/include
+
+# linuxgui, now that header files are there
+cd ../linuxgui
+CPPFLAGS="-I$DYNAMORIO_HOME/include -DLINUX" ./autogen.sh
+make
+cp src/linuxgui $base/dynamorio/bin/dynamorio-gui
+cd ../src
+
+make clear
+make CLIENT_INTERFACE=1 EXTERNAL_RELEASE=1 EXTERNAL_RELEASE_PROFILE=1 install
+cp ../lib/debug/* $base/dynamorio/lib/profile
+
+make clear
+make CLIENT_INTERFACE=1 EXTERNAL_RELEASE=1 DEBUG=0 install
+cp ../lib/release/* $base/dynamorio/lib/release
+if test -e .ignoreme ; then 
+    mv .ignoreme Makefile.mydefines
+fi
+
+cp ../docs/LICENSE.TXT $base/dynamorio/doc
+cp ../docs/LICENSE.TXT $base/dynamorio
+cp ../docs/linux-release-README $base/README
+
+cp -r ../docs/release/* $base/dynamorio/doc
+find $base/dynamorio/doc -name \*~ -exec rm {} \;
+find $base/dynamorio/doc -name CVS -exec rm -rf {} \;
+rm $base/dynamorio/doc/todo
+
+cd $base
+tar cf dynamorio.tar README dynamorio/
+bzip2 dynamorio.tar
diff --git a/tools/maketimes b/tools/maketimes
new file mode 100755
index 000000000..9d9350b9b
--- /dev/null
+++ b/tools/maketimes
@@ -0,0 +1,164 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2002 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### maketimes
+### author: Derek Bruening   July 2002
+###
+
+$usage = "Usage: $0 <file1> <file2> [res1 res2]\n";
+
+if ($#ARGV != 1 && $#ARGV != 3) {
+    print $usage;
+    exit;
+}
+$file1 = $ARGV[0];
+$file2 = $ARGV[1];
+if ($#ARGV == 3) {
+    $have_res = 1;
+    $res1 = $ARGV[2];
+    $res2 = $ARGV[3];
+} else {
+    $have_res = 0;
+}
+$PWD = $ENV{'PWD'};
+open(FILE1, "< $file1") || die "Error: Couldn't open $file1 for input\n";
+open(FILE2, "< $file2") || die "Error: Couldn't open $file2 for input\n";
+print "# in dir $PWD:\n#\t$file1 / $file2\n";
+if ($have_res) {
+    print "# results: $res1, $res2\n";
+} else {
+    print "# not verifying results\n";
+}
+print "---------------------------------------------\n";
+
+if ($have_res) {
+    open(RES1, "< $res1") || die "Error: Couldn't open $res1 for input\n";
+    $bmark = "";
+    while (<RES1>) {
+	if ($_ =~ /^%%%% .*\/(\w+)$/) {
+	    $bmark = $1;
+	    $ok1{$bmark} = 0;
+	}
+	if ($_ =~ /^Verify: correct/) {
+	    $ok1{$bmark} = 1;
+	}
+    }
+    close(RES1);
+    open(RES2, "< $res2") || die "Error: Couldn't open $res2 for input\n";
+    $bmark = "";
+    while (<RES2>) {
+	if ($_ =~ /^%%%% .*\/(\w+)$/) {
+	    $bmark = $1;
+	    $ok2{$bmark} = 0;
+	}
+	if ($_ =~ /^Verify: correct/) {
+	    $ok2{$bmark} = 1;
+	}
+    }
+    close(RES2);
+}
+
+$num_bmarks = 0;
+while (<FILE1>) {
+    if ($_ =~ /^\s*(\w+)\s+([\d\.]+)\s+([\d\.]+)\s+([\d\.]+)\s+([\d\.]+)/) {
+	$bmark = $1;
+	$ref_nopt = $2;
+	$ref_opt = $3;
+	$test_nopt = $4;
+	$test_opt = $5;
+	$t1{$bmark} = $ref_opt;
+	$name_bmarks[$num_bmarks] = $bmark;
+	$num_bmarks++;
+    }
+}
+close(FILE1);
+while (<FILE2>) {
+    if ($_ =~ /^\s*(\w+)\s+([\d\.]+)\s+([\d\.]+)\s+([\d\.]+)\s+([\d\.]+)/) {
+	$bmark = $1;
+	$ref_nopt = $2;
+	$ref_opt = $3;
+	$test_nopt = $4;
+	$test_opt = $5;
+	$t2{$bmark} = $ref_opt;
+	$name_bmarks[$num_bmarks] = $bmark;
+	$num_bmarks++;
+    }
+}
+close(FILE2);
+
+@sorted = sort(@name_bmarks);
+$last = "";
+$ratio_sum = 0;
+$ratio_num = 0;
+foreach $s (@sorted) {
+    if ($s eq $last) {
+	next;
+    }
+    $last = $s;
+
+    printf "%12s", $s;
+    if (defined($t1{$s})) {
+	printf "%9.2f", $t1{$s};
+	if ($have_res) {
+	    printf "%s", $ok1{$s} ? " " : "*";
+	}
+	$div = $t1{$s};
+    } else {
+	printf "%9s", "-----";
+	$div = 0;
+    }
+
+    if (defined($t2{$s})) {
+	printf "%9.2f", $t2{$s};
+	if ($have_res) {
+	    printf "%s", $ok2{$s} ? " " : "*";
+	}
+	if ($div > 0 && $t2{$s} > 0) {
+	    $ratio = $div / $t2{$s};
+	    $ratio_sum += 1/$ratio;
+	    $ratio_num++;
+	    printf "%8s%4.3f", "=> ", $ratio;
+	}
+    } else {
+	printf "%9s", "-----";
+    }
+    print "\n";
+}
+
+print "---------------------------------------------\n";
+if ($have_res) {
+    printf "%12s%27s", "harmonic mean", "";
+} else {
+    printf "%12s%25s", "harmonic mean", "";
+}
+$hmean = $ratio_num / $ratio_sum;
+printf "%4.3f\n", $hmean;
diff --git a/tools/makezip.sh b/tools/makezip.sh
new file mode 100755
index 000000000..a65cd7613
--- /dev/null
+++ b/tools/makezip.sh
@@ -0,0 +1,420 @@
+#!/bin/bash
+
+# **********************************************************
+# Copyright (c) 2008-2009 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# Copyright (c) 2003-2007 Determina Corp.
+# Copyright (c) 2001-2003 Massachusetts Institute of Technology
+# Copyright (c) 2001-2002 Hewlett Packard Company
+
+# PR 202095: packages up a DynamoRIO release
+
+# The first build# is the exported version build# and must be <64K
+# The second uniquebuild# can be larger
+usage="<viper|vmap> <absolute build path> <build#> <uniquebuild#> <ver#> [publish path]";
+
+projname=$1
+
+# mandatory 1st 5 arguments
+if [ $# -ne 6 ] && [ $# -ne 5 ]; then
+    echo "Usage: $0 $usage"
+    exit 127
+fi
+if [ $projname != "viper" -a $projname != "vmap" ]; then
+    echo "USAGE: $0 $usage"
+    exit 127
+fi
+if [ $projname == "vmap" ]; then
+    # we want the directory and zip file to have the known name "DynamoRIO"
+    projname="dynamorio";
+    pkgname="DynamoRIO";
+else
+    pkgname=$projname
+fi
+base=$2
+if test -e $base ; then 
+    echo "Error: directory $base already exists"
+    exit 127
+fi
+buildnum=$3
+# internal build number guaranteed to be unique but may be >64K
+unique_buildnum=$4
+vernum=$5
+publishdir=$6
+vercomma=${vernum//./,}
+defines="BUILD_NUMBER=$buildnum VERSION_NUMBER=$vernum VERSION_COMMA_DELIMITED=$vercomma UNIQUE_BUILD_NUMBER=$unique_buildnum"
+
+if [ $projname == "viper" ]; then
+target_config="VMSAFE=1"
+else
+target_config="VMAP=1"
+fi
+
+# this is how compiler.mk checks for Linux vs. Windows - only windows should have OS set
+if [ -z "$OS" ]; then
+    linux=1
+    windows=0
+    echo Linux
+    CP="cp -a"
+    os="Linux"
+    # PR 236973: there is no "g++" binary we can put on path so we have
+    # to explicitly reference the binary.  This path needs ARCH
+    # replaced with "i686" or "x86_64".
+    gxx=/build/toolchain/lin32/gcc-4.1.2-2/bin/ARCH-linux-g++
+else
+    linux=0
+    windows=1
+    echo Windows
+    CP="cp -a"
+    os="Windows"
+fi
+
+mkdir $base
+mkdir $base/$projname
+mkdir $base/$projname/include
+mkdir $base/$projname/bin
+if [ $windows -eq 1 ]; then
+    mkdir $base/$projname/bin/bin32
+    mkdir $base/$projname/bin/bin64
+fi
+mkdir $base/$projname/lib32
+mkdir $base/$projname/lib32/debug
+mkdir $base/$projname/lib32/release
+mkdir $base/$projname/lib64
+mkdir $base/$projname/lib64/debug
+mkdir $base/$projname/lib64/release
+mkdir $base/$projname/docs
+mkdir $base/$projname/logs
+
+# not part of zip file, pdbs on windows, non-stripped binaries on Linux
+mkdir $base/symbols
+mkdir $base/symbols/lib32
+mkdir $base/symbols/lib32/release
+mkdir $base/symbols/lib32/debug
+mkdir $base/symbols/lib64
+mkdir $base/symbols/lib64/release
+mkdir $base/symbols/lib64/debug
+mkdir $base/symbols/bin
+if [ $windows -eq 1 ]; then
+    mkdir $base/symbols/bin/bin32
+    mkdir $base/symbols/bin/bin64
+fi
+
+mkdir $base/include_test
+
+##################################################
+# defaults for DYNAMORIO_ env vars
+
+if [ -z "$DYNAMORIO_HOME" ] ; then
+    makezip_dir=`dirname $0`
+    DYNAMORIO_HOME=`cd $makezip_dir/../ ; pwd`
+fi
+if [ -z "$DYNAMORIO_MAKE" ] ; then
+    DYNAMORIO_MAKE=$DYNAMORIO_HOME/make
+fi
+if [ -z "$DYNAMORIO_LIBUTIL" ] ; then
+    DYNAMORIO_LIBUTIL=$DYNAMORIO_HOME/libutil
+fi
+if [ -z "$DYNAMORIO_TOOLS" ] ; then
+    DYNAMORIO_TOOLS=$DYNAMORIO_HOME/tools
+fi
+if [ -z "$DYNAMORIO_API" ] ; then
+    DYNAMORIO_API=$DYNAMORIO_HOME/api
+fi
+
+##################################################
+# core
+
+cd $DYNAMORIO_HOME/core
+if test -e Makefile.mydefines ; then 
+    mv Makefile.mydefines .ignoreme
+fi
+
+check_includes () {
+    if [ $# -ne 0 ]; then
+        echo "Internal error calling check_includes()"
+        exit 127
+    fi
+    if diff -r $base/include_test $base/$projname/include; then
+        rm -f $base/include_test/*.h
+    else
+        echo "Error: generated include files aren't identical to those previously generated"
+        exit 127
+    fi
+}
+
+#args are <extra_defines> <86|64> <rel|dbg>
+run_build () {
+    if [ $# -ne 3 ]; then
+        echo "Internal error calling run_build()"
+        exit 127
+    fi
+    if [ $3 == "rel" ]; then
+        lib_sub="release"
+    else
+        lib_sub="debug"
+    fi
+    if [ $2 == "86" ]; then
+        lib="lib32"
+        bin="bin32"
+    else
+        lib="lib64"
+        bin="bin64"
+    fi
+
+    make $target_config $1 $defines clean install
+    if [ $linux -eq 1 ]; then
+        $CP ../exports/x${2}_linux_${3}/libdynamorio.so $base/$projname/$lib/$lib_sub
+        $CP ../exports/x${2}_linux_${3}/libdynamorio.so $base/symbols/$lib/$lib_sub
+        if [ $3 == "rel" ]; then
+            $CP ../exports/x${2}_linux_${3}/libdrpreload.so $base/$projname/$lib
+            $CP ../exports/x${2}_linux_${3}/libdrpreload.so $base/symbols/$lib
+        fi
+        $CP ../exports/x${2}_linux_${3}/*.h $base/include_test
+    else
+        $CP ../exports/x${2}_win32_${3}/dynamorio.{dll,lib} $base/$projname/$lib/$lib_sub
+        $CP ../exports/x${2}_win32_${3}/dynamorio.{pdb,map} $base/symbols/$lib/$lib_sub
+        if [ $3 == "rel" ]; then
+            $CP ../exports/x${2}_win32_${3}/{drpreinject,drearlyhelp1,drearlyhelp2}.dll $base/$projname/$lib
+            $CP ../exports/x${2}_win32_${3}/{drpreinject.{pdb,map},drearlyhelp1.{pdb,map},drearlyhelp2.{pdb,map}} $base/symbols/$lib
+            $CP ../exports/x${2}_win32_${3}/drinject.exe $base/$projname/bin/$bin
+            $CP ../exports/x${2}_win32_${3}/drinject.{pdb,map} $base/symbols/bin/$bin
+        fi
+        $CP ../exports/x${2}_win32_${3}/*.h $base/include_test
+    fi
+}
+
+make clear
+
+run_build "DEBUG=0 INTERNAL=0 EXTERNAL_INJECTOR=1" 86 rel
+$CP $base/include_test/*.h $base/$projname/include
+rm -f $base/include_test/*.h
+
+run_build "DEBUG=1 INTERNAL=0" 86 dbg
+check_includes
+
+run_build "DEBUG=0 INTERNAL=0 ARCH=x64 EXTERNAL_INJECTOR=1" 64 rel
+check_includes
+
+run_build "DEBUG=1 INTERNAL=0 ARCH=x64" 64 dbg
+check_includes
+
+rm -rf $base/include_test
+
+if [ $linux -eq 1 ]; then
+    # strip debug and local symbols (leave only global exports and imports)
+    stripcmd='strip -p --strip-debug --discard-all'
+    find $base/$projname/lib* -name \*.so -exec $stripcmd {} \;
+fi
+
+if test -e .ignoreme ; then 
+    mv .ignoreme Makefile.mydefines
+fi
+
+##################################################
+# tools
+
+if [ $linux -eq 1 ]; then
+    $CP $DYNAMORIO_TOOLS/dynamorio $base/$projname/bin/drdeploy
+
+    #Removed since no longer need license/installer
+    #$CP $DYNAMORIO_TOOLS/linux_install.sh $base/install.sh
+    #gzexe $base/install.sh
+    #rm -f $base/install.sh~
+
+    # if we ever have binaries here we should strip them as well.
+    # we could do a find at the end over all dirs except the samples.
+else
+    cd $DYNAMORIO_LIBUTIL
+    make $target_config $defines clean all
+    $CP drconfig.{dll,lib} $base/$projname/lib32
+    $CP drconfig.{pdb,map} $base/symbols/lib32
+    cd $DYNAMORIO_TOOLS
+    make $target_config $defines EXTERNAL_DRVIEW=1 clean all
+    $CP drdeploy.exe $base/$projname/bin
+    $CP drdeploy.{pdb,map} $base/symbols/bin
+    # drdeploy needs this in same dir: make dup copy
+    $CP drconfig.dll $base/$projname/bin
+    $CP DRview.exe $base/$projname/bin/bin32/drview.exe
+    $CP DRview.{pdb,map} $base/symbols/bin/bin32
+
+    if [ $projname != "viper" ]; then
+        cd DRgui
+        make $defines clean all
+        $CP build_dbg_demo/DRgui.exe $base/$projname/bin/bin32/drgui.exe
+    fi
+fi
+
+##################################################
+# docs
+
+# release-build headers were built above; libutil/dr_config.h should be there
+if [ -z $DYNAMORIO_API ] ; then
+    cd $DYNAMORIO_HOME/api
+else
+    cd $DYNAMORIO_API
+fi
+cd docs
+# do separate clean to avoid cygwin bug where dir is not fully deleted
+# and so it can't create it when do "make clean pdf/all"
+make clean
+if [ $projname == "viper" ]; then
+    make $target_config pdf
+else
+    # see comments below: we're not building pdf
+    make $target_config all
+fi
+if [ $projname == "viper" ]; then
+    # replace DR, DynamoRIO, and client in header files w/ VIPER and VIPA
+    make VMSAFE=1 HEADER_DIR=$base/$projname/include update_headers
+fi
+$CP -r html $base/$projname/docs
+if [ $projname == "viper" ]; then
+    # The pdf output is messed up (more so on Linux) xref PR 314339 about fixing it.
+    $CP latex/refman.pdf $base/$projname/docs/$projname.pdf
+fi
+
+##################################################
+# provide pre-compiled dlls for samples
+# test our setup so far by using it to build
+
+if [ -z $DYNAMORIO_API ] ; then
+    cd $DYNAMORIO_HOME/api
+else
+    cd $DYNAMORIO_API
+fi
+cd samples
+
+# args are <defines> <subdir> <arch>
+build_samples() {
+    if [ $# -ne 3 ]; then
+        echo "Internal error calling build_samples()"
+        exit 127
+    fi
+    # no stats sample for viper and no security sample for vmap
+    if [ $projname == "viper" ]; then
+        DYNAMORIO_HOME=$base/$projname make EXCLUDE="stats.c tracedump.c" $1 clean all
+    else
+        # PR 236973: there is no "g++" toolchain binary so we have to include whole path
+        DYNAMORIO_HOME=$base/$projname make GXX=${gxx/ARCH/$3} EXCLUDE=MF_moduledb.c $1 clean all
+    fi
+    mkdir -p $base/$projname/docs/html/samples/build$2
+    if [ $linux -eq 1 ]; then
+        $CP *.so $base/$projname/docs/html/samples/build$2
+        # FIXME - better way to handle executables
+        $CP tracedump $base/$projname/docs/html/samples/build$2
+    else
+        $CP *.{pdb,dll,exe} $base/$projname/docs/html/samples/build$2
+        if [ $projname != "viper" ]; then
+        # we don't want security api sample app
+            rm -f $base/$projname/docs/html/samples/build$2/VIPA_test.exe
+        fi
+        # PR 232490: ensure services can read these
+        chmod ugo+rx $base/$projname/docs/html/samples/build$2/*.{dll,exe}
+    fi
+}
+
+mkdir -p $base/$projname/docs/html/samples
+# must setup environment for samples Makefile
+# we assume for Linux that something reasonable is already set up
+if [ $windows -eq 1 ]; then
+    export INCLUDE=`cygpath -wa "${TCROOT}/win32/winsdk-6.1.6000/VC/INCLUDE"`\;`cygpath -wa "${TCROOT}/win32/winsdk-6.1.6000/Include"`;
+    export LIB=`cygpath -wa "${TCROOT}/win32/winsdk-6.1.6000/VC/LIB"`\;`cygpath -wa "${TCROOT}/win32/winsdk-6.1.6000/Lib"`;
+    export PATH=`cygpath -ua "${TCROOT}/win32/winsdk-6.1.6000/VC/Bin"`:$PATH;
+fi
+build_samples "ARCH=x86 GCC=/build/toolchain/lin32/gcc-4.1.2-2/bin/i686-linux-gcc" 32 i686
+if [ $windows -eq 1 ]; then
+    export LIB=`cygpath -wa "${TCROOT}/win32/winsdk-6.1.6000/VC/LIB/x64"`\;`cygpath -wa "${TCROOT}/win32/winsdk-6.1.6000/Lib/x64"`;
+    export PATH=`cygpath -ua "${TCROOT}/win32/winsdk-6.1.6000/VC/Bin/x86_x64"`:$PATH;
+fi
+build_samples "ARCH=x64 GCC=/build/toolchain/lin32/gcc-4.1.2-2/bin/x86_64-linux-gcc" 64 x86_64
+
+
+##################################################
+# README file
+
+cd $DYNAMORIO_HOME
+$CP License.txt $base/$projname/
+
+# FIXME PR 232127: once gets larger we won't want to auto-generate but for now:
+echo -e 'QUICKSTART\n' > $base/$projname/README
+if [ $linux -eq 1 ]; then
+    echo -e "% bin/drdeploy -client docs/html/samples/build32/bbsize.so 0x1 \"\" ls" >> $base/$projname/README
+    echo -e "Or if on 64-bit Linux" >> $base/$projname/README
+    echo -e "% bin/drdeploy -64 -client docs/html/samples/build64/bbsize.so 0x1 \"\" ls" >> $base/$projname/README
+    echo -e "" >> $base/$projname/README
+    echo -e "% bin/drdeploy" >> $base/$projname/README
+    echo -e "For additional options." >> $base/$projname/README
+else
+    if [ $projname == "viper" ]; then
+        echo -e '% bin/drdeploy.exe -32 -reg calc.exe -root `cygpath -wa .` -mode code -client `cygpath -wa ./docs/html/samples/build32/bbsize.dll` 0x1 ""' >> $base/$projname/README
+        echo -e 'Or for 64-bit' >> $base/$projname/README
+        echo -e '% bin/drdeploy.exe -64 -reg calc.exe -root `cygpath -wa .` -mode code -client `cygpath -wa ./docs/html/samples/build64/bbsize.dll` 0x1 ""' >> $base/$projname/README
+    else
+        # no -mode arg for vmap
+        echo -e '% bin/drdeploy.exe -32 -reg calc.exe -root `cygpath -wa .` -client `cygpath -wa ./docs/html/samples/build32/bbsize.dll` 0x1 ""' >> $base/$projname/README
+        echo -e 'Or for 64-bit' >> $base/$projname/README
+        echo -e '% bin/drdeploy.exe -64 -reg calc.exe -root `cygpath -wa .` -client `cygpath -wa ./docs/html/samples/build64/bbsize.dll` 0x1 ""' >> $base/$projname/README
+    fi
+    echo -e 'Then:' >> $base/$projname/README
+    echo -e '% cmd /c start calc' >> $base/$projname/README
+    echo -e '\nNow close calc and a messagebox should pop up.' >> $base/$projname/README
+fi
+
+
+cd $base
+if [ $linux -eq 1 ]; then
+    tar -c -z -v -f $pkgname-$os-$vernum-$buildnum.tgz $projname
+    tar -c -z -v -f $pkgname-$os-$vernum-$buildnum-symbols.tgz symbols
+
+    #Removed since no longer need license/installer
+    #the zip file must not have the version in it since install.sh hardcodes its name
+    #zip -r -P UId3E4c2WPSiFlOzJDQx $projname.zip $projname.tgz 
+    #tar -cf $projname-$vernum-$buildnum.tar $projname.zip install.sh
+else
+    zip -r $pkgname-$os-$vernum-$buildnum.zip $projname/
+    zip -r $pkgname-$os-$vernum-$buildnum-symbols.zip symbols/
+
+    #Removed since no longer need license/installer
+    #if [ $projname != "viper" ]; then
+    #    # PR 259103: create a setup.exe that includes the zip file inside it
+    #    cd $DYNAMORIO_TOOLS/DRinstall
+    #    make ZIPFILE=$base/$projname-$vernum-$buildnum.zip clean all
+    #    $CP build_rel/DRinstall.exe $base/setup-$vernum-$buildnum.exe
+    #fi
+fi
+    
+if [ -n "$publishdir" ]; then
+    cd $DYNAMORIO_HOME
+    if ! test -e $publishdir; then
+        mkdir -p $publishdir
+    fi
+    $CP $base/$pkgname-$os-$vernum-$buildnum* $publishdir
+fi
diff --git a/tools/mangle-ret.pl b/tools/mangle-ret.pl
new file mode 100755
index 000000000..9a16fa129
--- /dev/null
+++ b/tools/mangle-ret.pl
@@ -0,0 +1,83 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2004 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### mangle-ret.pl
+### for converting ret into jmp* in a .s file,
+### to see impact of rsb on native apps (independent of DynamoRIO)
+###
+### author: Derek Bruening   August 2002
+###
+
+$usage = "Usage: $0 <file>\n";
+
+if ($#ARGV < 0) {
+    print $usage;
+    exit 1;
+}
+$file = $ARGV[0];
+$file =~ /(.*)\.s$/;
+if ($1 eq "") {
+    print "Error: input file must be .s file\n";
+    exit 1;
+}
+$out = "$1.mangle.s";
+open(FILE, "< $file") || die "Error: Couldn't open $file for input\n";
+open(OUT, "> $out") || die "Error: Couldn't open $out for output\n";
+$rets = 0;
+while (<FILE>) {
+    if ($_ =~ /^\s*ret\s*$/) {
+	$rets++;
+	print OUT "# ---- translation of ret -----\n";
+	print OUT "	addl \$4,%esp\n";
+	print OUT "	jmp *0xfffffffc(%esp)\n";
+    } elsif ($_ =~ /^\s*ret\s+\$([0-9a-fA-Fx]+)\s*$/) {
+	print "Found ret with immed operand: $_";
+	# extra stack space is removed after return address is popped
+	$rets++;
+	$extra = $1;
+	if ($extra =~ /0x([0-9a-fA-F]+)/) {
+	    $remove = 4 + hex($1);
+	} else {
+	    $remove = 4 + $extra;
+	}
+	$offs = -$remove;
+	print  OUT "# ---- translation of ret -----\n";
+	print  OUT "	addl \$$remove,%esp\n";
+	printf OUT "	jmp *0x%08x(%%esp)\n", $offs;
+    } elsif ($_ =~ /^\s*ret\s+/) {
+	print "Error: Found unidentified ret: $_";
+    } else {
+	print OUT $_;
+    }
+}
+close(FILE);
+print "Translated $rets returns in $file to $out\n";
diff --git a/tools/memsummary.sh b/tools/memsummary.sh
new file mode 100755
index 000000000..37354ceac
--- /dev/null
+++ b/tools/memsummary.sh
@@ -0,0 +1,49 @@
+#!/bin/bash
+
+# **********************************************************
+# Copyright (c) 2005-2007 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 <logfile>"
+    exit 127
+fi
+
+log=$1
+
+if ! test -e $log ; then
+    echo "Error: $log not found"
+    exit 127
+fi
+
+tail -n $((`wc -l $log | awk '{print $1}'` - `grep -n -o breakdown $log | tail -1 | cut -f 1 -d :`)) $log | grep -v locks
+start=`grep -n '^(Begin) All statistics' $log | tail -1 | cut -d : -f 1`
+total=`wc -l $log | awk '{print $1}'`
+tail -$((total-start+1)) $log | grep -E 'Our peak com|Peak total m|Peak heap cap|Peak heap align|Peak stack|Peak mmap|Peak file|peak cap|Peak spec|Peak file'
+grep -A 5 '^Process Memory' $log
diff --git a/tools/moduledb_study.pl b/tools/moduledb_study.pl
new file mode 100644
index 000000000..b85c4f93c
--- /dev/null
+++ b/tools/moduledb_study.pl
@@ -0,0 +1,138 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# processes a file created by moduledb_study.sh
+
+$file = $ARGV[0];
+open(FILE, "< $file") || die "Error: Couldn't open $file\n";
+
+$tools = $ENV{'DYNAMORIO_TOOLS'};
+if ($tools eq "") {
+    # use same dir this script sits in
+    $tools = $0;
+    if ($tools =~ /[\\\/]/) {
+        $tools =~ s/^(.+)[\\\/][\w\.]+$/\1/;
+        $tools =~ s/\\/\\\\/g;
+    } else {
+        $tools = "./";
+    }
+    # address_query needs DYNAMORIO_TOOLS set to find DRload.exe
+    $ENV{'DYNAMORIO_TOOLS'} = $tools;
+}
+
+$path = ".";
+if ($ARGV[0] =~ /^(.*)\/[^\/]*$/) {
+    $path = $1;
+}
+
+while (<FILE>) {
+    $root = "-";
+    $case_num = 0;
+    $build = "-";
+    $app = "-";
+    $source = "-";
+    $source_addr = "-";
+    $source_prop = "-";
+    $target = "-";
+    $target_addr = "-";
+    $target_prop = "-";
+    $id = "-";
+    
+    if ($_ =~ /^(bug|case)-?_?\s?(\d+)/i) {
+        $case_num = $2;
+    } elsif ($_ =~ /^([^\/]*)\//) {
+        $root = $1;
+    }
+
+    $bugtitle_info = `perl $tools/bugtitle.pl \Q$path/$_\E`;
+    if ($bugtitle_info =~ /^VIOLATION\s([^\s]*)\s\((<unknown build>|custom|\d*)\s([^\)]*)\)(.*)$/) {
+        $id = $1;
+        $build = $2;
+        $app = $3;
+        $rest = $4;
+        if ($rest =~ /^\s*(.*)\s--->\s(.*)$/) {
+            $source = $1;
+            $target = $2;
+        } elsif ($rest =~ /^\s*(0x[\da-fA-F]*)\s\(([^\s]*)\s(.*)\)\s=>\s(0x[\da-fA-F]*)\s\(([^\s]*)\s(.*)\).*$/) {
+            $source_addr = $1;
+            $source = $2;
+            $source_prop = $3;
+            $target_addr = $4;
+            $target = $5;
+            $target_prop = $6;
+        }
+    }
+
+    if ($case_num == 0) {
+        print "$root";
+    } else {
+        print "$case_num";
+    }
+    print " $id \"$app\" $source $target\n";
+}
+
+# use this | sort | uniq > file
+# grep -iE " ....\.....\.A " results_uniq.txt > results_a.txt
+# ...
+# grep -iEv " ....\.....\.[abcef] " results_uniq.txt > results_other.txt
+
+# 1129 unique violations (however many are core bugs!)
+# gathered from determina bugs folder, does not include zipped up forensics files 
+# 37 .O or .P
+# 31 .A (stack) (mostly bugs)
+# 471 .C/E/F
+# 590 .B
+
+# of .B (number may not add up due to some truncated forensics not being included)
+# grep -iE " heap heap$" results_b.txt > results_b_heap_to_heap.txt
+# grep -iEv " (heap|unknown) heap$" results_b.txt | grep -iE " heap$" > results_b_mod_to_
+# 56 heap -> heap
+# 50 unknown -> heap
+# 408 module -> heap [potential dll2heap relaxation]
+#   grep -iEv "(ntdll.dll|kernel32.dll|advapi32.dll|user32.dll|rpcrt4.dll|ole32.dll)" results_b_mod_to_heap.txt > res
+#   118 not common dll (most of the remainder are bugs, from before we got the hook policy nailed down, or bad hookers)
+#   even of these most are microsoft (need to get company name)
+# 76 module -> module
+#   grep -iEv " kernel32.dll$" results_b_mod_to_mod.txt > results_b_mod_to_mod_less_commo
+#   54 less targeting kernel32 (bugs and bad hookers)
+#     27 unique target dlls (~2/3 MS? need to run through my company name script)
+
+# of rct - note there tend to be more unique violations per dll
+# grep -Ev "dynamorio.dll" results_rct.txt > results_rct2.txt
+# grep -iEv "unknown heap$" results_rct2.txt | grep  -iE " heap$" > results_rct2_mod_to_h
+# (need to lean to select columns in emacs, for now used excel)
+# 386 ignoring to dynamorio.dll (a bug)
+#  15 unknown to heap
+#  73 module to heap (.C) [currently not easily addressable, should add rct from list? limit to just heap? maybe stopping some of the auto-attacks?]
+#  298 module to module
+#   93 unique targets (~2/3 MS? need to run through company name script)
+#   61 unique src modules ""
diff --git a/tools/moduledb_study.sh b/tools/moduledb_study.sh
new file mode 100644
index 000000000..12a59eca8
--- /dev/null
+++ b/tools/moduledb_study.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# **********************************************************
+# Copyright (c) 2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# This shell script gathers all the forensics files in <arg 1> and copies them to <arg 2>
+# preserving directory paths.  It then creates a file <arg 3> with a list of all the
+# security violation forensics files (with full paths rooted at <arg2>
+# For ex.
+# ./moduledb_study.sh /mnt/bugs/ /c/tmp_forensics file_list.txt
+
+if [ $# -ne 3 ]; then
+    echo "Usage: $0 <root of bug repository> <place to copy forensics files> <out file name>"
+    exit 1
+fi
+
+cd $1
+mkdir $2
+find . -iname '*.xml' | grep "00000...\.xml$" | rsync -v --files-from=- . $2
+find . -iname '*.html' | grep "00000...\.html$" | rsync -v --files-from=- . $2
+cd $2
+grep -ril "security violation" * > $3
+exit 0
\ No newline at end of file
diff --git a/tools/ntdll.h b/tools/ntdll.h
new file mode 100644
index 000000000..d150ef548
--- /dev/null
+++ b/tools/ntdll.h
@@ -0,0 +1,219 @@
+/* **********************************************************
+ * Copyright (c) 2004-2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Shared ntdll functionality for tools */
+
+#ifndef NTDLL_H
+#define NTDLL_H
+
+#include "win32/ntdll.h"
+#undef GET_NTDLL
+
+/* ntdll.dll interface 
+ * cleaner to use ntdll.lib but too lazy to add to build process
+ *
+ * WARNING: the Native API is an undocumented API and
+ * could change without warning with a new version of Windows.
+ */
+static HANDLE ntdll_handle = NULL;
+
+#define GET_PROC_ADDR(func, type, name) do { \
+    if (ntdll_handle == NULL) \
+        ntdll_handle = GetModuleHandle((LPCTSTR)_T("ntdll.dll")); \
+    if (func == NULL) {                      \
+        assert(ntdll_handle != NULL);        \
+        func = (type) GetProcAddress(ntdll_handle, (LPCSTR)name); \
+        assert(func != NULL);                \
+    }                                        \
+} while (0)
+
+/* A wrapper to define kernel entry point in a static function */
+/* In C use only at the end of a block prologue! */
+#define GET_NTDLL(NtFunction, type)                             \
+  typedef int (WINAPI *NtFunction##Type) type;                  \
+  static NtFunction##Type NtFunction;                           \
+  GET_PROC_ADDR(NtFunction, NtFunction##Type, #NtFunction);
+
+static LONGLONG
+get_system_time()
+{
+    int i, len = 0;
+    LARGE_INTEGER time;
+    GET_NTDLL(NtQuerySystemTime, (OUT PLARGE_INTEGER time));
+    i = NtQuerySystemTime(&time);
+    if (i != 0) /* function failed */
+        return 0;
+    return (LONGLONG) time.QuadPart;
+}
+
+static unsigned long
+get_uptime()
+{
+    LARGE_INTEGER counter;
+    LARGE_INTEGER frequency;
+    NTSTATUS res;
+    GET_NTDLL(NtQueryPerformanceCounter, (OUT PLARGE_INTEGER PerformanceCount,
+                                          OUT PLARGE_INTEGER PerformanceFrequency OPTIONAL
+                                          ));
+    res = NtQueryPerformanceCounter(&counter, &frequency);
+    if (!NT_SUCCESS(res) || frequency.QuadPart == 0)
+        return 0;
+    return (unsigned long)(counter.QuadPart*1000/(frequency.QuadPart));
+}
+
+
+static uint
+get_system_load(bool sampled)
+{
+    int i, len = 0;
+    SYSTEM_PROCESSOR_TIMES times;
+    LONGLONG clock1, clock2;
+    LONGLONG idle1, idle2;
+    int idle_load = 0;
+    GET_NTDLL(NtQuerySystemInformation,
+              (IN SYSTEM_INFORMATION_CLASS SystemInformationClass,
+               OUT PVOID SystemInformation,
+               IN ULONG SystemInformationLength,
+               OUT PULONG ReturnLength OPTIONAL));
+    i = NtQuerySystemInformation(SystemProcessorTimes, &times, sizeof(times), &len);
+    if (i != 0) /* function failed */
+        return -1;
+    /* return length not trustworthy, according to Nebbett, so we don't test it */
+    /* actually cannot get load from kernel:
+     * can only get idle time since boot, so to get average over
+     * an interval must take samples yourself -- this is what taskmgr and
+     * perfmon do (perfmon help even says so)
+     */
+    clock1 = get_system_time();
+    idle1 = times.IdleTime.QuadPart;
+    /* if sampling, use the clock and idle from prev iteration, otherwise average load
+     * over 100 ms */
+    if (sampled) {
+        static LONGLONG prev_clock = 0, prev_idle = 0;
+        if (clock1 > prev_clock) {
+            idle_load = (int) ((100 * (idle1 - prev_idle)) / (clock1 - prev_clock));
+            prev_clock = clock1;
+            prev_idle = idle1;
+        }
+    } else {
+        Sleep(100);
+        i = NtQuerySystemInformation(SystemProcessorTimes, &times, sizeof(times), &len);
+        if (i != 0) /* function failed */
+            return -1;
+        idle2 = times.IdleTime.QuadPart;
+        clock2 = get_system_time();
+        if (clock2 > clock1)
+            idle_load = (int) ((100 * (idle2 - idle1)) / (clock2 - clock1));
+    }
+
+    idle_load = (idle_load >= 100) ? 100 : idle_load;
+
+    /* we want %CPU == 100 - idle load */
+    return 100 - idle_load;
+}
+
+static BOOL
+get_system_performance_info(SYSTEM_PERFORMANCE_INFORMATION *sperf_info)
+{
+    int i, len = 0;
+    GET_NTDLL(NtQuerySystemInformation,
+              (IN SYSTEM_INFORMATION_CLASS SystemInformationClass,
+               OUT PVOID SystemInformation,
+               IN ULONG SystemInformationLength,
+               OUT PULONG ReturnLength OPTIONAL));
+    i = NtQuerySystemInformation(SystemPerformanceInformation, sperf_info,
+                                 sizeof(*sperf_info), &len);
+    return (i == 0);    
+}
+
+/* returns both %CPU and %user time */
+static int
+get_process_load_ex(HANDLE h, int *cpu, int *user)
+{
+    KERNEL_USER_TIMES times;
+    LONGLONG scheduled_time;
+    LONGLONG wallclock_time;
+    int i, len = 0;
+    /* could share w/ other process info routines... */
+    GET_NTDLL(NtQueryInformationProcess,
+              (IN HANDLE ProcessHandle,
+               IN PROCESSINFOCLASS ProcessInformationClass,
+               OUT PVOID ProcessInformation,
+               IN ULONG ProcessInformationLength,
+               OUT PULONG ReturnLength OPTIONAL));
+    i = NtQueryInformationProcess((HANDLE) h, ProcessTimes,
+                                  &times, sizeof(times), &len);
+    if (i != 0) /* function failed */
+        return 0;
+    /* return length not trustworthy, according to Nebbett, so we don't test it */
+    scheduled_time = times.UserTime.QuadPart + times.KernelTime.QuadPart;
+    if (cpu != NULL) {
+        /* %CPU == (scheduled time) / (wall clock time) */
+        wallclock_time = get_system_time() - times.CreateTime.QuadPart;
+        if (wallclock_time == (LONGLONG) 0)
+            *cpu = 0;
+        else
+            *cpu = (int) ((100 * scheduled_time) / wallclock_time);
+    }
+    if (user != NULL) {
+        /* %user == (user time) / (scheduled time) */
+        if (scheduled_time == (LONGLONG) 0)
+            *user = 0;
+        else
+            *user = (int) ((100 * times.UserTime.QuadPart) / scheduled_time);
+    }
+    return 1;
+}
+
+static int
+get_process_mem_stats(HANDLE h, VM_COUNTERS *info)
+{
+    int i, len = 0;
+    /* could share w/ other process info routines... */
+    GET_NTDLL(NtQueryInformationProcess,
+              (IN HANDLE ProcessHandle,
+               IN PROCESSINFOCLASS ProcessInformationClass,
+               OUT PVOID ProcessInformation,
+               IN ULONG ProcessInformationLength,
+               OUT PULONG ReturnLength OPTIONAL));
+    i = NtQueryInformationProcess((HANDLE) h, ProcessVmCounters,
+                                  info, sizeof(VM_COUNTERS), &len);
+    if (i != 0) {
+        /* function failed */
+            memset(info, 0, sizeof(VM_COUNTERS));
+    }
+    else
+        assert(len == sizeof(VM_COUNTERS));
+    return TRUE;
+}
+
+#endif /* NTDLL_H */
diff --git a/tools/pcache-htable.pl b/tools/pcache-htable.pl
new file mode 100755
index 000000000..fb87b49c4
--- /dev/null
+++ b/tools/pcache-htable.pl
@@ -0,0 +1,233 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2007 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+## pcache.pl
+## by Derek Bruening
+## February 2007
+
+# default debugger paths to try
+# we first try cdb from Debugging Tools For Windows
+#   (so we can get stdout and don't need a temp logfile)
+# else we use the ntsd on everyone's machines and a logfile,
+#   which we clobber every time
+$sysroot = &win32path2unix($ENV{'SYSTEMROOT'});
+$progroot = &win32path2unix($ENV{'PROGRAMFILES'});
+# try to find installation of debugging tools for windows
+@debugtools = glob("$progroot/Debug*/cdb.exe");
+@try_dbgs = ("$debugtools[0]", "$sysroot/system32/ntsd.exe");
+
+# parameters
+$usage = "Usage: $0 [-raw] [-i] [-d <debuggerpath>]" .
+    "[-dr <DRdllpath>] <pcache>\n";
+
+$debugger = "";
+$verbose = 0;
+$raw = 0;
+$DRdll = "";
+$interactive = 0;
+$pfile = "";
+die $usage if ($#ARGV < 0);
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq '-dr') {
+        die $usage if ($#ARGV <= 0);
+        shift;
+        $DRdll = $ARGV[0];
+        unless (-f "$DRdll") {
+            die "$err No file $DRdll\n";
+        }
+    } elsif ($ARGV[0] eq '-d') {
+        die $usage if ($#ARGV <= 0);
+        shift;
+        $debugger = $ARGV[0];
+        # checked below for existence
+    } elsif ($ARGV[0] eq '-raw') {
+        $raw = 1;
+    } elsif ($ARGV[0] eq '-i') {
+        $interactive = 1;
+    } elsif ($ARGV[0] =~ /^-/) {
+        die $usage;
+    } else {
+        $pfile = $ARGV[0];
+        die $usage if ($#ARGV > 0);
+    }
+    shift;
+}
+
+$pfile_win32 = &unixpath2win32($pfile);
+# try both -- different perls like different paths
+unless (-f "$pfile" || -f "$pfile_win32") {
+    die "Cannot find pcache file at $pfile or $pfile_win32\n";
+}
+
+if ($DRdll eq "") {
+    $DRhome = $ENV{'DYNAMORIO_HOME'};
+    if ($pfile =~ /-dbg/) {
+        $suffix = "dbg";
+    } else {
+        $suffix = "rel";
+    }
+    $DRdll = "$DRhome/exports/x86_win32_$suffix/dynamorio.dll";
+}
+$DRdllwin32 = &unixpath2win32($DRdll);
+# try both -- different perls like different paths
+unless (-f "$DRdll" || -f "$DRdllwin32") {
+    die "Cannot find dll at $DRdll or $DRdllwin32\n";
+}
+
+# see if user-specified debugger exists
+if ($debugger ne "") {
+    $debugger = &win32path2unix($debugger);
+    print "Trying debugger at $debugger\n" if ($verbose);
+    if (! -f "$debugger") { # -x seems to require o+x so we don't check
+        die "Cannot find debugger at $debugger\n";
+    }
+} else {
+    for ($i = 0; $i <= $#try_dbgs; $i++) {
+        $debugger = &win32path2unix($try_dbgs[$i]);
+        print "Trying debugger at $debugger\n" if ($verbose);
+        last if (-f "$debugger");
+    }
+    die "Cannot find a debugger: @try_dbgs\n" if ($i > $#try_dbgs);
+}
+if ($debugger !~ /cdb/) {
+    $use_logfile = 1;
+}
+
+$DRtools = $ENV{'DYNAMORIO_TOOLS'};
+if ($DRtools eq "") {
+    # try same dir this script sits in
+    $DRtools = $0;
+    if ($DRtools =~ /[\\\/]/) {
+        $DRtools =~ s/^(.+)[\\\/][\w\.]+$/\1/;
+    } else {
+        $DRtools = "./";
+    }
+}
+$DRload = "$DRtools/DRload.exe";
+# need win32 path with double backslashes
+# e.g., c:\\\\derek\\\\dr\\\\tools\\\\DRload.exe
+$DRload = &unixpath2win32($DRload);
+unless (-f "$DRload") { # -x seems to require o+x so we don't check
+    die "Cannot find \$DYNAMORIO_TOOLS/DRload.exe at $DRload\n";
+}
+
+# FIXME: win2k ntsd does not load symbols by following loaded dll's path!
+# setting _NT_SYMBOL_PATH ENV here or exporting in shell doesn't work,
+# and -y not available on win2k ntsd, and .sympath takes all chars following
+# (even \r, \n, ", ;!), no aS available to break cmd -- giving up
+
+$cmd = "";
+$marker = 'eeeeeeee';
+$cmd .= "? $marker; ";
+
+$map_pc = "13000000"; # arbitrary
+$htable_header_len = "5c";
+$cmd .= "dd @@(0x$map_pc + ((_coarse_persisted_info_t *)0x$map_pc)->header_len + ((_coarse_persisted_info_t *)0x$map_pc)->option_string_len + ((_coarse_persisted_info_t *)0x$map_pc)->hotp_patch_list_len + ((_coarse_persisted_info_t *)0x$map_pc)->reloc_len + ((_coarse_persisted_info_t *)0x$map_pc)->rac_htable_len + ((_coarse_persisted_info_t *)0x$map_pc)->rct_htable_len + 0x$htable_header_len) @@(0x$map_pc + ((_coarse_persisted_info_t *)0x$map_pc)->header_len + ((_coarse_persisted_info_t *)0x$map_pc)->option_string_len + ((_coarse_persisted_info_t *)0x$map_pc)->hotp_patch_list_len + ((_coarse_persisted_info_t *)0x$map_pc)->reloc_len + ((_coarse_persisted_info_t *)0x$map_pc)->rac_htable_len + ((_coarse_persisted_info_t *)0x$map_pc)->rct_htable_len + ((_coarse_persisted_info_t *)0x$map_pc)->cache_htable_len - 1); ";
+
+if ($use_logfile) {
+    my $logdir = $ENV{'TMPDIR'} || $ENV{'TEMP'} || $ENV{'TMP'} || '/tmp';
+    die "Cannot find temp directory $logdir" if (! -d $logdir);
+    # must go through logfile as debugger won't write to accessible stdout
+    $logfile = "$logdir/pcache.pl-$$.log";
+    $logfile = &unixpath2win32($logfile);
+    print "Log file is $logfile\n";
+    # start fresh -- no stale results
+    if (-f $logfile) {
+        system("rm $logfile");
+    }
+    $cmd = ".logopen $logfile; $cmd; .logclose;";
+}
+
+$cmd .= " q" unless ($interactive);
+
+# put quotes around debugger to handle spaces
+$cmdline = "\"$debugger\" -g -c \"$cmd\" $DRload -debugbreak " .
+    "-map $pfile_win32 $map_pc $DRdllwin32";
+print $cmdline if ($verbose);
+if ($interactive) {
+    system("$cmdline") && die "Error: couldn't run $cmdline\n";
+} elsif ($raw) {
+    system("$cmdline") && die "Error: couldn't run $cmdline\n";
+    if ($use_logfile) {
+        system("cat $logfile");
+    }
+} else {
+    # prefix each ln output with the query (debugger doesn't re-print -c cmds)
+    $i = -1;
+    $output = 0;
+    if ($use_logfile) {
+        system("$cmdline") && die "Error: couldn't run $cmdline\n";
+        open(DBGOUT, "< $logfile") || die "Error: couldn't open $logfile\n";
+    } else {
+        open(DBGOUT, "$cmdline |") || die "Error: couldn't run $cmdline\n";
+    }
+    while (<DBGOUT>) {
+        # look for our marker
+        $saw_marker = 1 if (/= $marker/);
+        next unless ($saw_marker);
+        print $_ unless (/= $marker/ || /^quit/ || /^\*\*\* ERROR: Symbol/);
+    }
+    close(DBGOUT);
+}
+
+sub unixpath2win32($) {
+    my ($p) = @_;
+    # use cygpath if available, it will convert /home to c:\cygwin\home, etc.
+    $cp = `cygpath -wi \"$p\"`;
+    chomp $cp;
+    if ($cp eq "") {
+        # do it by hand
+        # add support for /x => x:
+        $p =~ s|^/([a-z])/|\1:/|;
+        # forward slash becomes backslash since going through another shell
+        $p =~ s|/|\\|g;
+    } else {
+        $p = $cp;
+    }
+    # make single backslashes doubles
+    $p =~ s|\\{1}|\\\\|g;
+    return $p;
+}
+
+sub win32path2unix($) {
+    my ($p) = @_;
+    # avoid issues with spaces by using 8.3 names
+    $cp = `cygpath -dmi \"$p\"`;
+    chomp $cp;
+    if ($cp eq "") {
+        # do it by hand
+        $p =~ s|\\|/|g;
+    } else {
+        $p = $cp;
+    }
+    return $p;
+}
diff --git a/tools/pcache-ws.pl b/tools/pcache-ws.pl
new file mode 100755
index 000000000..f7b5e70af
--- /dev/null
+++ b/tools/pcache-ws.pl
@@ -0,0 +1,116 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2007 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+## pcache-ws.pl
+## by Derek Bruening
+## February 2007
+##
+## Lists persistent cache files and modules in a process's working set,
+## using vadump.exe.
+
+$usage = "Usage: $0 <pid>";
+$tools = $ENV{'DYNAMORIO_TOOLS'};
+if ($tools eq "") {
+    # use same dir this script sits in
+    $tools = $0;
+    if ($tools =~ /[\\\/]/) {
+        $tools =~ s/^(.+)[\\\/][^\\\/]+$/\1/;
+        $tools =~ s/\\/\\\\/g;
+    } else {
+        $tools = "./";
+    }
+    # address_query needs DYNAMORIO_TOOLS set to find DRload.exe
+    $ENV{'DYNAMORIO_TOOLS'} = $tools;
+}
+$vadump = `cygpath -u $tools/external/vadump`;
+chomp $vadump;
+
+die $usage if ($#ARGV < 0);
+$pid = $ARGV[0];
+
+$cmd = "$vadump -o -p $pid";
+print "Running $cmd\n" if ($verbose);
+
+# FIXME: win32 perl (non-cygwin) doesn't need the pipe-hang workaround;
+# we try to distinguish via the interpreter path: no guarantees though!
+if ($^X =~ /\/usr/) {
+    # FIXME: a direct pipe hangs, but passing through something else works
+    open(VADUMP, "$cmd 2>&1 | cat |") || die "Error running $cmd\n";
+} else {
+    # FIXME: can't get stderr redirection working for win32 perl
+    open(VADUMP, "$cmd |") || die "Error running $cmd\n";
+}
+while (<VADUMP>) {
+    chomp;
+    s/\r//; # in case DOS files get in here!
+    if (/^OpenProcess/) {
+        die "No such process with pid $pid\n";
+    }
+    next unless (/^0x[0-9A-F]{8} \((\d+)\) (.*)$/);
+    $count = $1;
+    $descr = $2;
+    print "Got $count $descr\n" if ($verbose);
+    next if ($descr =~ /^PRIVATE/i || $descr =~ /^UNKNOWN/ ||
+             $descr =~ /^Process/ || $descr =~ /^TEB/ ||
+             $descr =~ /^Stack/ || $descr =~ /^DATAFILE.*\.nls$/);
+    $descr =~ s/DATAFILE_MAPPED Base 0x[0-9A-F]{8} //;
+    $descr = lc($descr);
+    $raw_descr = $descr;
+    if ($raw_descr ne $last_raw_descr) {
+        $finished{$last_descr} = 1;
+        # handle duplicate names
+        if ($finished{$descr}) {
+            $dupnum = 2;
+            do {
+                $dupdescr = "$descr ($dupnum)";
+                $dupnum++;
+            } while ($finished{$dupdescr});
+            $descr = $dupdescr;
+        }
+    } else {
+        $descr = $last_descr;
+    }
+    $private{$descr}++ if ($count == 0);
+    $shareable{$descr}++ if ($count == 1);
+    $shared{$descr}++ if ($count > 1);
+    $mem{$descr} = $descr;
+    $last_descr = $descr;
+    $last_raw_descr = $raw_descr;
+}
+close(VADUMP);
+
+die "Error running $cmd\n" if (scalar(keys %mem) == 0);
+printf "%6s %6s %6s %6s %s \n", "Priv", "S-able", "Shared", "Total";
+foreach $i (sort (keys %mem)) {
+    printf "%6d %6d %6d %6d  %s \n", $private{$i}, $shareable{$i}, $shared{$i},
+    $private{$i} + $shareable{$i} + $shared{$i}, $i;
+}
diff --git a/tools/pcache.pl b/tools/pcache.pl
new file mode 100755
index 000000000..71d8ebfbf
--- /dev/null
+++ b/tools/pcache.pl
@@ -0,0 +1,257 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2007 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+## pcache.pl
+## by Derek Bruening
+## February 2007
+
+# default debugger paths to try
+# we first try cdb from Debugging Tools For Windows
+#   (so we can get stdout and don't need a temp logfile)
+# else we use the ntsd on everyone's machines and a logfile,
+#   which we clobber every time
+$sysroot = &win32path2unix($ENV{'SYSTEMROOT'});
+$progroot = &win32path2unix($ENV{'PROGRAMFILES'});
+# try to find installation of debugging tools for windows
+@debugtools = glob("$progroot/Debug*/cdb.exe");
+@try_dbgs = ("$debugtools[0]", "$sysroot/system32/ntsd.exe");
+
+# parameters
+$usage = "Usage: $0 [-raw] [-i] [-d <debuggerpath>]" .
+    "[-dr <DRdllpath>] <pcache>\n";
+
+$debugger = "";
+$verbose = 0;
+$raw = 0;
+$DRdll = "";
+$interactive = 0;
+$pfile = "";
+die $usage if ($#ARGV < 0);
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq '-dr') {
+        die $usage if ($#ARGV <= 0);
+        shift;
+        $DRdll = $ARGV[0];
+        unless (-f "$DRdll") {
+            die "$err No file $DRdll\n";
+        }
+    } elsif ($ARGV[0] eq '-d') {
+        die $usage if ($#ARGV <= 0);
+        shift;
+        $debugger = $ARGV[0];
+        # checked below for existence
+    } elsif ($ARGV[0] eq '-raw') {
+        $raw = 1;
+    } elsif ($ARGV[0] eq '-i') {
+        $interactive = 1;
+    } elsif ($ARGV[0] =~ /^-/) {
+        die $usage;
+    } else {
+        $pfile = $ARGV[0];
+        die $usage if ($#ARGV > 0);
+    }
+    shift;
+}
+
+$pfile_win32 = &unixpath2win32($pfile);
+# try both -- different perls like different paths
+unless (-f "$pfile" || -f "$pfile_win32") {
+    die "Cannot find pcache file at $pfile or $pfile_win32\n";
+}
+
+if ($DRdll eq "") {
+    $DRhome = $ENV{'DYNAMORIO_HOME'};
+    if ($pfile =~ /-dbg/) {
+        $suffix = "dbg";
+    } else {
+        $suffix = "rel";
+    }
+    $DRdll = "$DRhome/exports/x86_win32_$suffix/dynamorio.dll";
+}
+$DRdllwin32 = &unixpath2win32($DRdll);
+# try both -- different perls like different paths
+unless (-f "$DRdll" || -f "$DRdllwin32") {
+    die "Cannot find dll at $DRdll or $DRdllwin32\n";
+}
+
+# see if user-specified debugger exists
+if ($debugger ne "") {
+    $debugger = &win32path2unix($debugger);
+    print "Trying debugger at $debugger\n" if ($verbose);
+    if (! -f "$debugger") { # -x seems to require o+x so we don't check
+        die "Cannot find debugger at $debugger\n";
+    }
+} else {
+    for ($i = 0; $i <= $#try_dbgs; $i++) {
+        $debugger = &win32path2unix($try_dbgs[$i]);
+        print "Trying debugger at $debugger\n" if ($verbose);
+        last if (-f "$debugger");
+    }
+    die "Cannot find a debugger: @try_dbgs\n" if ($i > $#try_dbgs);
+}
+if ($debugger !~ /cdb/) {
+    $use_logfile = 1;
+}
+
+$DRtools = $ENV{'DYNAMORIO_TOOLS'};
+if ($DRtools eq "") {
+    # try same dir this script sits in
+    $DRtools = $0;
+    if ($DRtools =~ /[\\\/]/) {
+        $DRtools =~ s/^(.+)[\\\/][\w\.]+$/\1/;
+    } else {
+        $DRtools = "./";
+    }
+}
+$DRload = "$DRtools/DRload.exe";
+# need win32 path with double backslashes
+# e.g., c:\\\\derek\\\\dr\\\\tools\\\\DRload.exe
+$DRload = &unixpath2win32($DRload);
+unless (-f "$DRload") { # -x seems to require o+x so we don't check
+    die "Cannot find \$DYNAMORIO_TOOLS/DRload.exe at $DRload\n";
+}
+
+# FIXME: win2k ntsd does not load symbols by following loaded dll's path!
+# setting _NT_SYMBOL_PATH ENV here or exporting in shell doesn't work,
+# and -y not available on win2k ntsd, and .sympath takes all chars following
+# (even \r, \n, ", ;!), no aS available to break cmd -- giving up
+
+$cmd = "";
+$marker = 'eeeeeeee';
+$cmd .= "? $marker; ";
+
+$map_pc = "13000000"; # arbitrary
+$cmd .= ".echo \"Flags:\"; ";
+$cmd .= ".if ((poi($map_pc+10) & 0x000001)==0x000001) {.echo \"X86_32\";}; ";
+$cmd .= ".if ((poi($map_pc+10) & 0x000002)==0x000002) {.echo \"X86_64\";}; ";
+$cmd .= ".if ((poi($map_pc+10) & 0x000004)==0x000004) {.echo \"SEEN_BORLAND_SEH\";}; ";
+$cmd .= ".if ((poi($map_pc+10) & 0x000008)==0x000008) {.echo \"ELIDED_UBR\";}; ";
+$cmd .= ".if ((poi($map_pc+10) & 0x000010)==0x000010) {.echo \"SUPPORT_RAC\";}; ";
+$cmd .= ".if ((poi($map_pc+10) & 0x000020)==0x000020) {.echo \"SUPPORT_RCT\";}; ";
+$cmd .= ".if ((poi($map_pc+10) & 0x000040)==0x000040) {.echo \"ENTIRE_MODULE_RCT\";}; ";
+$cmd .= ".if ((poi($map_pc+10) & 0x000080)==0x000080) {.echo \"SUPPORT_TRACES\";}; ";
+$cmd .= ".if ((poi($map_pc+10) & 0x000100)==0x000100) {.echo \"MAP_RW_SEPARATE\";}; ";
+$cmd .= ".if ((poi($map_pc+10) & 0x000200)==0x000200) {.echo \"EXEMPTION_OPTIONS\";}; ";
+$cmd .= "dt -r1 _coarse_persisted_info_t $map_pc; ";
+$cmd .= ".echo Build number in decimal:; ";
+$cmd .= "n10; ?? (uint)((_coarse_persisted_info_t *)0x$map_pc)->build_number; n16; ";
+$cmd .= ".echo Options:; ";
+$cmd .= "da @@(0x$map_pc + ((_coarse_persisted_info_t *)0x$map_pc)->header_len); ";
+$cmd .= ".echo Hotpatch list:; ";
+$cmd .= "dd @@(0x$map_pc + ((_coarse_persisted_info_t *)0x$map_pc)->header_len + ((_coarse_persisted_info_t *)0x$map_pc)->option_string_len) L@@(((_coarse_persisted_info_t *)0x$map_pc)->hotp_patch_list_len / 4); ";
+$cmd .= ".echo Main hashtable:; ";
+$cmd .= "?? *((_coarse_table_t *) (0x$map_pc + ((_coarse_persisted_info_t *)0x$map_pc)->header_len + ((_coarse_persisted_info_t *)0x$map_pc)->option_string_len + ((_coarse_persisted_info_t *)0x$map_pc)->hotp_patch_list_len + ((_coarse_persisted_info_t *)0x$map_pc)->reloc_len + ((_coarse_persisted_info_t *)0x$map_pc)->rac_htable_len + ((_coarse_persisted_info_t *)0x$map_pc)->rct_htable_len)); ";
+$cmd .= ".echo Stub hashtable:; ";
+$cmd .= "?? *((_coarse_table_t *) (0x$map_pc + ((_coarse_persisted_info_t *)0x$map_pc)->header_len + ((_coarse_persisted_info_t *)0x$map_pc)->option_string_len + ((_coarse_persisted_info_t *)0x$map_pc)->hotp_patch_list_len + ((_coarse_persisted_info_t *)0x$map_pc)->reloc_len + ((_coarse_persisted_info_t *)0x$map_pc)->rac_htable_len + ((_coarse_persisted_info_t *)0x$map_pc)->rct_htable_len + ((_coarse_persisted_info_t *)0x$map_pc)->cache_htable_len)); ";
+$cmd .= ".echo RCT hashtable:; ";
+$cmd .= "?? *((_app_pc_table_t *) (0x$map_pc + ((_coarse_persisted_info_t *)0x$map_pc)->header_len + ((_coarse_persisted_info_t *)0x$map_pc)->option_string_len + ((_coarse_persisted_info_t *)0x$map_pc)->hotp_patch_list_len + ((_coarse_persisted_info_t *)0x$map_pc)->reloc_len + ((_coarse_persisted_info_t *)0x$map_pc)->rac_htable_len)); ";
+$cmd .= ".echo RAC hashtable:; ";
+$cmd .= "?? *((_app_pc_table_t *) (0x$map_pc + ((_coarse_persisted_info_t *)0x$map_pc)->header_len + ((_coarse_persisted_info_t *)0x$map_pc)->option_string_len + ((_coarse_persisted_info_t *)0x$map_pc)->hotp_patch_list_len + ((_coarse_persisted_info_t *)0x$map_pc)->reloc_len)); ";
+
+if ($use_logfile) {
+    my $logdir = $ENV{'TMPDIR'} || $ENV{'TEMP'} || $ENV{'TMP'} || '/tmp';
+    die "Cannot find temp directory $logdir" if (! -d $logdir);
+    # must go through logfile as debugger won't write to accessible stdout
+    $logfile = "$logdir/pcache.pl-$$.log";
+    $logfile = &unixpath2win32($logfile);
+    print "Log file is $logfile\n";
+    # start fresh -- no stale results
+    if (-f $logfile) {
+        system("rm $logfile");
+    }
+    $cmd = ".logopen $logfile; $cmd; .logclose;";
+}
+
+$cmd .= " q" unless ($interactive);
+
+# put quotes around debugger to handle spaces
+$cmdline = "\"$debugger\" -g -c \"$cmd\" $DRload -debugbreak " .
+    "-map $pfile_win32 $map_pc $DRdllwin32";
+print $cmdline if ($verbose);
+if ($interactive) {
+    system("$cmdline") && die "Error: couldn't run $cmdline\n";
+} elsif ($raw) {
+    system("$cmdline") && die "Error: couldn't run $cmdline\n";
+    if ($use_logfile) {
+        system("cat $logfile");
+    }
+} else {
+    # prefix each ln output with the query (debugger doesn't re-print -c cmds)
+    $i = -1;
+    $output = 0;
+    if ($use_logfile) {
+        system("$cmdline") && die "Error: couldn't run $cmdline\n";
+        open(DBGOUT, "< $logfile") || die "Error: couldn't open $logfile\n";
+    } else {
+        open(DBGOUT, "$cmdline |") || die "Error: couldn't run $cmdline\n";
+    }
+    while (<DBGOUT>) {
+        # look for our marker
+        $saw_marker = 1 if (/= $marker/);
+        next unless ($saw_marker);
+        print $_ unless (/= $marker/ || /^quit/ || /^\*\*\* ERROR: Symbol/);
+    }
+    close(DBGOUT);
+}
+
+sub unixpath2win32($) {
+    my ($p) = @_;
+    # use cygpath if available, it will convert /home to c:\cygwin\home, etc.
+    $cp = `cygpath -wi \"$p\"`;
+    chomp $cp;
+    if ($cp eq "") {
+        # do it by hand
+        # add support for /x => x:
+        $p =~ s|^/([a-z])/|\1:/|;
+        # forward slash becomes backslash since going through another shell
+        $p =~ s|/|\\|g;
+    } else {
+        $p = $cp;
+    }
+    # make single backslashes doubles
+    $p =~ s|\\{1}|\\\\|g;
+    return $p;
+}
+
+sub win32path2unix($) {
+    my ($p) = @_;
+    # avoid issues with spaces by using 8.3 names
+    $cp = `cygpath -dmi \"$p\"`;
+    chomp $cp;
+    if ($cp eq "") {
+        # do it by hand
+        $p =~ s|\\|/|g;
+    } else {
+        $p = $cp;
+    }
+    return $p;
+}
diff --git a/tools/procdb.pl b/tools/procdb.pl
new file mode 100755
index 000000000..597cc10f8
--- /dev/null
+++ b/tools/procdb.pl
@@ -0,0 +1,266 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+## bugtitle.pl
+## by Vladimir Kiriansky
+## August 2006
+##
+## find odd processes and show description of all found
+###########################################################################
+
+## Requires wget
+
+## example use:
+# just collect all links to select a few suspicious ones
+# $ ./procdb.pl -s bug_8704/iexplore.exe.1608.00000000.xml 
+#   ...
+#   wget -O -  http://www.processlibrary.com/directory/files/wscntfy/ | grep -A 5 'Description:' 
+
+# normal use:
+# $ ./procdb.pl bug_8704/iexplore.exe.1608.00000000.xml > processes.out
+# $ cat process.out
+#   ..
+#   <td width="390">regscan.exe is added by
+#           Trojan.W32.Rbot. It is a worm which attemps to spread via
+#           network shares. It also contains backdoor Trojan
+#           capabilities allowing unauthorised remote access to the
+#           infected computer. If found on your system make sure that
+#           you have downloaded the latest update for your antivirus
+#           application. This process is a security risk and should be
+#           removed from your system.<br>
+
+
+# copy&paste from bugtitle.pl
+
+# FIXME: should merge with the logic from wld/wld.pl
+# about having a whitelist database on DLLs
+
+
+# adding only exe's that should be seen only once!
+# so that we can detect a c:\temp\services.exe as rogue
+
+# FIXME: not yet used
+%seen_known = ();
+
+# should add more entries from config/applications.html or even have a
+# generator from config/application_data.h
+%known_exe_description = {
+    'lsass' => 'lsass',
+    'services' => 'services'};
+
+$tools = $ENV{'DYNAMORIO_TOOLS'};
+if ($tools eq "") {
+    # use same dir this script sits in
+    $tools = $0;
+    if ($tools =~ /[\\\/]/) {
+        $tools =~ s/^(.+)[\\\/][\w\.]+$/$1/;
+        $tools =~ s/\\/\\\\/g;
+    } else {
+        $tools = "./";
+    }
+    # address_query needs DYNAMORIO_TOOLS set to find DRload.exe
+    $ENV{'DYNAMORIO_TOOLS'} = $tools;
+}
+
+# parameters
+$usage = "Usage: $0 [-v] [-q] [-glob] [-noglob] <forensic_file>\n";
+
+$just_show = 0;
+$verbose = 0;
+$quiet = 0;
+
+$globall = 0;
+$noglob = 0;
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq '-v') {
+        $verbose = 1;
+    } elsif ($ARGV[0] eq '-q') {
+        $quiet = 1;
+    } elsif ($ARGV[0] eq '-s') {
+        $just_show = 1;
+    } elsif ($ARGV[0] eq '-glob') {
+        $globall = 1;
+    } elsif ($ARGV[0] eq '-noglob') {
+        $noglob = 1;
+    } elsif ($ARGV[0] =~ /^-/) {
+        die $usage;
+    } else {
+        # rest should be forensic file names
+        last;
+    }
+    shift;
+}
+
+die $usage if ($#ARGV < 0);
+$multiple = ($#ARGV > 0);
+while ($#ARGV >= 0) {
+    my $filearg = $ARGV[0];
+    # make it easy for users w/ non-globbing shell and non-globbing perl to
+    # pass *, w/o double-globbing and messing up escaped chars for other users
+    if (!$noglob && ($globall || $filearg =~ /\*/)) {
+        # FIXME: must convert from \ to / BEFORE the glob -- yet don't
+        # want to ruin escaped chars!  It's a tradeoff: is \* a directory delimiter
+        # followed by metachar *, or is it an escaped * trying to be literal?
+        # In everyday usage we assume no escaped *'s!
+        # If we didn't care for \* directory delim we might do:
+        #  $filearg =~ s|\\\\|//|g; # next rule won't get both slashes so need this rule
+        #  $filearg =~ s|\\([\w\d])|/\1|g; # \ to / only if before a normal char
+        $filearg =~ s|\\|/|g; # assume \ is dir delim, no support for \ as escape char
+        @files = glob($filearg);
+        print "globbing: \"$filearg\" => \"@files\"\n" if ($verbose);
+        if ($#files < 0) {
+            # if glob comes up empty, don't use its results (to get error msg)
+            @files = ($filearg);
+        }
+    } else {
+        @files = ($filearg);
+    }
+    $multiple = $multiple || $#files > 0;
+    foreach $file (@files) {
+        print "processing \"$file\"\n" if ($verbose);
+        $file =~ s|\\|/|g; # normalize
+        unless (-f "$file") {
+            die "Error: No forensic file $file\n";
+        }
+        if ($multiple) {
+            print "\n$file:\n";
+        }
+        process_file($file);
+    }
+    shift;
+}
+
+sub is_known_exe
+{
+    my ($exe) = @_;
+
+    return 0;
+
+    return 0 unless exists $known_exe_description { $exe };
+
+    # known should be only once
+    if (defined $seen_known{ $exe } && 
+        $seen_known{ $exe } == 1) {
+        # test with svchost.exe which may also be the Welchia worm
+        print "WARNING: second appearance of $exe\n";
+    }
+    $seen_known{ $exe } = 1;
+
+    return 1;
+}
+
+sub get_external_data
+{
+ # wget -q http://www.processlibrary.com/directory/files/defwatch -O - | grep -A 5 "Description:"
+    my ($exe) = @_;
+
+    my $base = "http://www.processlibrary.com/directory/files/";
+    my $url = "$base$exe/";
+    my $wget = "wget -q -O - ";
+    my $postprocess = "grep -A 5 'Description:'";
+    my $desc = "";
+
+    my $command = "$wget $url | $postprocess";
+
+    $desc = `$command` if (!$just_show);
+    print "$command \n" if ($just_show);
+
+# FIXME: note processlibrary tries to find a substring if they can't find an exact match
+# so we have to verify that the process name they are returning is the one we really asked for
+# for the time being leaving users to worry about it by being more verbose
+    return $desc;
+}
+
+
+#    http://www.processlibrary.com/directory/files/rtvscan/index.html
+
+sub process_file($) {
+    my ($file) = @_;
+    $processing = 0;
+
+    open(IN, "< $file") || die "Error: couldn't open $file\n";
+    while (<IN>) {
+        # Handle CR
+        s/\r//;
+
+        # Start of process list
+        if (/^<process-list> <!\[CDATA\[/) {
+            $processing = 1;
+        } 
+        elsif (/]]> <\/process-list>/) {
+            $processing = 0;
+        } elsif ($processing) {
+            $skip = 0;
+            if (/^([^\s\<]+).exe$/i) {
+                $process=$1;
+            } else {
+                $process=$_;
+                chomp $process;
+                if ($process ne "System") {
+                    print "not an EXE '$process!'\n";
+                } else {
+                    $skip = 1;
+                }
+            }
+
+            if (!$skip) {
+                print "Querying '$process'\n" if ($verbose);
+                $skip = is_known_exe($exe);
+            }
+
+            if (!$skip) {
+                print "Querying Externally '$process'\n";
+                print &get_external_data($process), "\n";
+            }
+        }
+    }
+    close(IN);
+}
+
+sub unixpath2win32($) {
+    my ($p) = @_;
+    # use cygpath if available, it will convert /home to c:\cygwin\home, etc.
+    $cp = `cygpath -wi \"$p\"`;
+    chomp $cp;
+    if ($cp eq "") {
+        # do it by hand
+        # add support for /x => x:
+        $p =~ s|^/([a-z])/|$1:/|;
+        # forward slash becomes backslash since going through another shell
+        $p =~ s|/|\\|g;
+    } else {
+        $p = $cp;
+    }
+    # make single backslashes doubles
+    $p =~ s|\\{1}|\\\\|g;
+    return $p;
+}
diff --git a/tools/process-kstats.pl b/tools/process-kstats.pl
new file mode 100755
index 000000000..9576e30bc
--- /dev/null
+++ b/tools/process-kstats.pl
@@ -0,0 +1,142 @@
+#!/usr/bin/perl -w
+
+# **********************************************************
+# Copyright (c) 2004 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### process-kstats.pl
+###
+### Summarize kstat info
+
+$verbose = 0;
+
+$usage = "Usage: $0 [-h] <kstats file>\n";
+
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq "-v") {
+	$verbose = 1;
+    } elsif ($ARGV[0] eq "-h") {
+        print $usage;
+        exit;
+    } else {
+	$infile = $ARGV[0];
+    }
+    shift;
+}
+if (!defined($infile)) {
+    print $usage;
+    exit;
+}
+
+open(PROFILE, "< $infile") or die "Error opening $infile\n";
+
+$read_kstats = 0;
+@hit_array = ();
+$total_cycles = 0;
+
+while (<PROFILE>) {
+
+    chop($_);
+
+    if ($verbose) {
+        print STDERR "Process $.: <$_>\n";
+    }
+
+    if (/^Process KSTATS:/ || /^Thread \d+ KSTATS \{/ ) {
+        
+        if ($verbose) {
+            print STDERR "Found kstats\n";
+        }
+        $read_kstats = 1;
+    } elsif ($read_kstats) {
+        # Kstats output for a single ktats nodes is 2 lines and looks like:
+        # interp:   6559513 totc,       1 num,   6559513 minc,   6559513 avg,   6559513 maxc,   6559513 self,         0 sub,
+        # 2 ms,         0 ms out,in interp loop
+        if ( /\w+:[ ]*(\d+) totc,[ ]*(\d+) num,/ ) {
+            $totc = $1;
+            $num = $2;
+            # /(\w+):(\d)+ totc,[ ]*(\d+) num,[ ]*(\d+) minc,[ ]*(\d+) avg,[ ]*/
+            /,[ ]*\d+ maxc,[ ]*(\d+) self,[ ]*(\d+) sub/;
+            $self = $1;
+            $sub_time = $2;
+
+            # Read the next line to get the name.
+            $_ = <PROFILE>;
+            /(\d+) ms,[ ]*(\d+) ms out,[ ]*(.+)/;
+            $ms = $1;
+            $outliers = $2 == 0 ? 1 : $2;
+            $name = $3;
+
+            # The total # cyles are attributed to thread_measured, which is also
+            # "total measured and propagated in thread"
+            if ($name eq "total measured and propagated in thread") {
+                $total_cycles = $totc;
+            }
+            $name =~ s/ /-/g;
+            if (length($name) > 30) {
+                $name = substr($name, 0, 30);
+            }
+            $data = "$num $totc $self $sub_time $outliers $ms $name";
+            if ($verbose) {
+                print STDERR "Data -- $data\n";
+            }
+            push @hit_array, $data;
+        }
+    }
+}
+
+# Sorts elements of the form "$num $totc $self $sub_time $outliers $ms $name"
+sub by_time {
+    @a_fields = split / /, $a;
+    @b_fields = split / /, $b;
+    # Sort based on the 2nd field, the totc
+    $b_fields[1] <=> $a_fields[1];
+}
+
+@sorted_hits = sort by_time @hit_array;
+if ($#sorted_hits == 0) {
+   printf "No edges!\n";
+   exit;
+}
+
+if ($total_cycles == 0) {
+    printf "WARNING: total cycles could not be calculated\n";
+}
+printf "%10s %10s %30s %20s %20s %20s %10s\n", "%-age", "# calls", "Name", "Totc", "Self", "Sub", "Outlier";;
+printf "%10s %30s %20s %20s %20s %10s\n", "====", "====", "====", "====", "====", "===", "===";
+for ($i = 0; $i < $#sorted_hits; $i++) {
+    @fields = split / /, $sorted_hits[$i];
+    $outlier = $fields[5] == 0 ? 0 : $fields[4]/$fields[5];
+    $percentage = $total_cycles == 0 ? 0 : 100 * $fields[1] / $total_cycles;
+    # Elements are of the form "$num $totc $self $sub_time $outliers $ms $name"
+    printf "%10.2f %10d %30s %20d %20d %20d %10.2f\n", $percentage, $fields[0],
+    $fields[6], $fields[1], $fields[2], $fields[3], $outlier;
+    #printf "%10d %30s %20d %20d %20d %10.2f\n", $num, $name, $totc, $self,
+    #$sub_time, $outliers / $ms;
+}
diff --git a/tools/procstats.c b/tools/procstats.c
new file mode 100644
index 000000000..6a36f3ded
--- /dev/null
+++ b/tools/procstats.c
@@ -0,0 +1,327 @@
+/* **********************************************************
+ * Copyright (c) 2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+#define _WIN32_WINNT 0x0400
+
+#include <Windows.h>
+#include <stdio.h>
+#include <Tlhelp32.h>
+#include <tchar.h>
+#include <pdh.h>
+
+// a.s.
+#include <stdlib.h>
+
+#define MAXPATH 80
+#define NCOUNTERS 27
+#define MCOUNTERS 3
+
+TCHAR *counters[NCOUNTERS] = {
+    _T("Thread Count"),
+    _T("Working Set"),
+    _T("Page Faults/sec"),
+    _T("Page File Bytes"),
+    _T("% User Time"),
+    _T("% Privileged Time"),
+    _T("% Processor Time"),
+    _T("Creating Process ID"),
+    _T("Elapsed Time"),
+    _T("Handle Count"),
+    _T("ID Process"),
+    _T("IO Data Bytes/sec"),
+    _T("IO Data Operations/sec"),
+    _T("IO Other Bytes/sec"),
+    _T("IO Other Operations/sec"),
+    _T("IO Read Bytes/sec"),
+    _T("IO Read Operations/sec"),
+    _T("IO Write Bytes/sec"),
+    _T("IO Write Operations/sec"),
+    _T("Page File Bytes Peak"),
+    _T("Pool Nonpaged Bytes"),
+    _T("Pool Paged Bytes"),
+    _T("Priority Base"),
+    _T("Private Bytes"),
+    _T("Virtual Bytes"),
+    _T("Virtual Bytes Peak"),
+    _T("Working Set Peak"),
+};
+
+TCHAR *shortnames[NCOUNTERS] = {
+    _T("tc"),
+    _T("wss"),
+    _T("pgflts"),
+    _T("pgfileK"),
+    _T("utimes"),
+    _T("ktimes"),
+    _T("ttimes"),
+    _T("ppid"),
+    _T("realtim"),
+    _T("handles"),
+    _T("pid"),
+    _T("IOdataK"),
+    _T("IOdataO"),
+    _T("IOothrK"),
+    _T("IOothrO"),
+    _T("IOreadK"),
+    _T("IOreadO"),
+    _T("IOwritK"),
+    _T("IOwritO"),
+    _T("pgfpeak"),
+    _T("poolnpK"),
+    _T("poolpK"),
+    _T("priorty"),
+    _T("privK"),
+    _T("vmK"),
+    _T("vmpeak"),
+    _T("wsspeak"),
+};
+
+TCHAR *formatstrings[NCOUNTERS] = {
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+    _T("%.2f\t"),
+    _T("%.2f\t"),
+    _T("%.2f\t"),
+    _T("%.0f\t"),
+    _T("%.2f\t"),
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+    _T("%.0f\t"),
+};
+
+BOOL use_kb[NCOUNTERS] = {
+    0,
+    1,
+    0,
+    1,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    1,
+    0,
+    1,
+    0,
+    1,
+    0,
+    1,
+    0,
+    1,
+    1,
+    1,
+    0,
+    1,
+    1,
+    1,
+    1,
+};
+
+int waitforprocess(TCHAR *name) {
+    HANDLE processes;
+    PROCESSENTRY32 pe = { sizeof(pe) };
+    BOOL fOK;
+    while(1) {
+        processes = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
+        fOK = Process32First(processes, &pe);
+        for (; fOK; fOK = Process32Next(processes, &pe)) {
+            if(!_tcsicmp(pe.szExeFile,name))
+                return pe.th32ProcessID;
+        }
+        CloseHandle(processes);
+        Sleep(181);
+    }
+}
+
+// a.s.
+void GetProcNameFromId( int pid, TCHAR* pName )
+{
+    HANDLE processes;
+    PROCESSENTRY32 pe = { sizeof(pe) };
+    BOOL fOK;
+    while(1) {
+        processes = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0);
+        fOK = Process32First(processes, &pe);
+        for (; fOK; fOK = Process32Next(processes, &pe)) {
+            if( (int)pe.th32ProcessID == pid)
+			{
+				_tcscpy( pName, pe.szExeFile );
+				return;
+			}
+        }
+        CloseHandle(processes);
+        Sleep(181);
+    }
+
+}
+// eof a.s.
+
+int __cdecl _tmain (int argc, TCHAR **argv)
+{
+   HQUERY hQuery;
+   HCOUNTER *pCounterHandle;
+   PDH_STATUS pdhStatus;
+   PDH_FMT_COUNTERVALUE fmtValue;
+   DWORD ctrType;
+   TCHAR szPathBuffer[MAXPATH];
+   int i, samples=0;
+   int NUM_SAMPLES=1000, INTERVAL_MS=1000;
+   int pid;
+   FILE *fp = stdout;
+   char fn[MAXPATH];
+   TCHAR basename[MAXPATH];
+   HANDLE htimer;
+   LARGE_INTEGER duetime;
+
+   if(argc<2) {
+       _ftprintf(stderr,_T("Usage: %s [<exeName> | <pid> | all] [num_samples] [interval] [outputfile]\n"), argv[0]);	
+       return -1;
+   }
+
+   if(_tcscmp(argv[1],_T("all"))) {
+       if( isalpha( *argv[1] ) ) {
+           printf("waiting...\n");
+           pid = waitforprocess(argv[1]);
+           //remove .exe
+           _tcscpy(basename, argv[1]);
+           basename[_tcslen(argv[1]) - 4] = _T('\0');
+       }
+       else {
+           pid = _ttoi( argv[1] );
+           GetProcNameFromId( pid, basename );
+           basename[_tcslen(basename) - 4] = _T('\0');
+       }
+       // eof a.s.
+   }
+   else {
+       pid = 0;
+       _tcscpy(basename, _T("_Total"));
+   }
+   
+   if(argc>2) {
+       NUM_SAMPLES=_ttoi(argv[2]);
+       if(argc>3)
+           INTERVAL_MS=_ttoi(argv[3]);
+       if(argc>4) {
+           sprintf(fn, "%S", argv[4]);
+           fp = fopen(fn,"w");
+           if(fp==NULL) {
+               _ftprintf(stderr, _T("error opening output file %s\n"),
+                         argv[4]);
+               return -1;
+           }
+       }           
+   }
+
+   _ftprintf(stderr,
+             _T("Monitoring %s, pid=%d. Using %d samples at %dms interval\n"),
+             basename, pid, NUM_SAMPLES, INTERVAL_MS);
+   
+   pdhStatus = PdhOpenQuery (0, 0, &hQuery);
+   pCounterHandle = (HCOUNTER *)malloc(NCOUNTERS*sizeof(HCOUNTER));
+
+   for(i=0; i<NCOUNTERS; ++i) {
+       _stprintf(szPathBuffer,_T("\\Process(%s)\\%s"), basename, counters[i]);
+       pdhStatus = PdhAddCounter (hQuery,
+                                  szPathBuffer,
+                                  0, 
+                                  &pCounterHandle[i]);
+   }
+
+   // "Prime" counters that need two values to display a formatted value.
+   pdhStatus = PdhCollectQueryData (hQuery);
+
+   //disp titles
+   for(i=0; i<NCOUNTERS; ++i)
+       _ftprintf(fp, _T("%s\t"), shortnames[i]);
+   _ftprintf(fp, _T("\n"));
+
+   htimer = CreateWaitableTimer(NULL, FALSE, NULL);
+   duetime.LowPart = 0;
+   duetime.HighPart = 0;
+   SetWaitableTimer(htimer, &duetime, INTERVAL_MS, NULL, NULL, FALSE);
+
+   while (samples++ < NUM_SAMPLES) {
+
+       //first wait is instantaneous
+       WaitForSingleObject(htimer, INTERVAL_MS * 2);
+       
+       pdhStatus = PdhCollectQueryData (hQuery);
+       
+       for(i=0; i<NCOUNTERS; ++i) {
+           // Get the current value of this counter.
+           pdhStatus = PdhGetFormattedCounterValue (pCounterHandle[i],
+                                                    PDH_FMT_DOUBLE,
+                                                    &ctrType,
+                                                    &fmtValue);
+           
+           if (pdhStatus == ERROR_SUCCESS) {
+               _ftprintf (fp, formatstrings[i], use_kb[i] ? 
+                          fmtValue.doubleValue / 1024 : fmtValue.doubleValue);
+           } else {
+               goto processgone;
+           }
+       }
+       _ftprintf(fp, _T("\n"));       
+   }
+   
+ processgone:
+
+   pdhStatus = PdhCloseQuery (hQuery);
+
+   if(fp != stdout)
+       fclose(fp);
+
+   free(pCounterHandle);
+
+   return 0;
+}
+
diff --git a/tools/restimes b/tools/restimes
new file mode 100755
index 000000000..d34dd7c45
--- /dev/null
+++ b/tools/restimes
@@ -0,0 +1,79 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2002-2003 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### restimes
+### author: Derek Bruening   April 2001
+###
+### Produces a table summarizing times in a .res file
+
+$usage = "Usage: $0 <.res file>\n";
+
+if ($#ARGV < 0) {
+    print $usage;
+    exit;
+}
+$resfile = $ARGV[0];
+open(FIND, "< $resfile") || die "Error running $resfile\n";
+
+while (<FIND>) {
+    chop;
+    if ($_ =~ m|.+/([^/]+)|) {
+	$bmark = $1;
+	$elapsed = 0;
+	$val{$bmark,$opt,$dset} = 0;
+	$bm{$bmark} = 1;
+    } elsif ($_ =~ /Elapsed: ([0-9]+):([0-9]+):([0-9\.]+)/) {
+	# texec format
+	$elapsed = $1*60 + $2 + $3/60.; # minutes
+	$val{$bmark,$opt,$dset} += $elapsed;
+    } elsif ($_ =~ /real\t([0-9]+)m([0-9\.]+)s/) {
+	# /usr/bin/time simple format
+	# FIXME: what do hours look like?
+	$elapsed = $1 + $2/60.; # minutes
+	$val{$bmark,$opt,$dset} += $elapsed;
+    } elsif ($_ =~ /(\d+):(\d+).(\d+)elapsed (\d+)%CPU/) {
+	# /usr/bin/time and runstats compressed format
+	# FIXME: what do hours look like?
+	$elapsed = $1 + $2/60.; # minutes
+	$val{$bmark,$opt,$dset} += $elapsed;
+	
+    }
+}
+close(FIND);
+
+print("Benchmark\tTime\tRSS\tVSz\t%CPU\n");
+foreach $b (sort (keys %bm)) {
+    printf("%10s\t%6.2f\t%6.2f\t%6.2f\t%6.2f\n",$b,
+	   $val{$b,"n","r"},$val{$b,"o","r"},
+	   $val{$b,"n","t"},$val{$b,"o","t"});
+}
+
diff --git a/tools/rio b/tools/rio
new file mode 100755
index 000000000..929ff92e9
--- /dev/null
+++ b/tools/rio
@@ -0,0 +1,65 @@
+#!/bin/sh
+
+# **********************************************************
+# Copyright (c) 2001-2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+#$Id: rio 9862 2009-02-10 17:24:10Z derek.bruening $
+#rio injector
+#Vladimir Kiriansky, October 2001
+
+#uses environment variables:
+#requires DYNAMORIO_HOME
+#set DYNAMORIO_OPTIONS to control DynamoRIO behavior
+#  to see feedback on a run, you might set DYNAMORIO_OPTIONS to
+#  "-stats -notify -loglevel 1"
+
+debug=0
+
+if [ "$1" = "" ]
+then
+    echo "usage: rio [-debug] executable [args ...]"
+    exit
+elif [ "$1" = "-debug" ]
+then
+    debug=1
+    shift
+fi
+
+# If release version exists, use it, unless -debug is passed in
+if [ $debug = 0 ] && [ -d $DYNAMORIO_HOME/exports/x86_linux_rel ]
+then
+    export LD_LIBRARY_PATH=$DYNAMORIO_HOME/exports/x86_linux_rel:$LD_LIBRARY_PATH
+else
+    export LD_LIBRARY_PATH=$DYNAMORIO_HOME/exports/x86_linux_dbg:$LD_LIBRARY_PATH
+fi
+
+export LD_PRELOAD="libdrpreload.so libdynamorio.so"
+
+$*
diff --git a/tools/rmshm.sh b/tools/rmshm.sh
new file mode 100755
index 000000000..c164aa028
--- /dev/null
+++ b/tools/rmshm.sh
@@ -0,0 +1,37 @@
+#!/bin/csh
+
+# **********************************************************
+# Copyright (c) 2001 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+foreach thing (`ipcs -m |grep 0x0| awk '{print $2}'`)
+echo $thing >/dev/null
+ipcrm shm $thing>/dev/null
+end
+
diff --git a/tools/runjob.c b/tools/runjob.c
new file mode 100644
index 000000000..8a153a2e7
--- /dev/null
+++ b/tools/runjob.c
@@ -0,0 +1,241 @@
+/* **********************************************************
+ * Copyright (c) 2007 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+
+#define _WIN32_WINNT 0x0500
+/* for Jobs on Win2000+ */
+
+#include <windows.h>
+#include <assert.h>
+#include <stdio.h>
+
+#include "share.h"
+
+#define FP stderr
+
+#ifndef USING_SDK
+
+typedef struct _IO_COUNTERS {
+    ULONGLONG  ReadOperationCount;
+    ULONGLONG  WriteOperationCount;
+    ULONGLONG  OtherOperationCount;
+    ULONGLONG ReadTransferCount;
+    ULONGLONG WriteTransferCount;
+    ULONGLONG OtherTransferCount;
+} IO_COUNTERS;
+typedef IO_COUNTERS *PIO_COUNTERS;
+
+/* FIXME: this is really screwed up: they have added a field in the
+ * object and 
+ * LastErrorValue: (Win32) 0x18 (24) - The program issued a command but the command length is incorrect.
+ */
+typedef struct _NEW_JOBOBJECT_BASIC_LIMIT_INFORMATION {
+    LARGE_INTEGER PerProcessUserTimeLimit;
+    LARGE_INTEGER PerJobUserTimeLimit;
+    DWORD LimitFlags;
+    SIZE_T MinimumWorkingSetSize;
+    SIZE_T MaximumWorkingSetSize;
+    DWORD ActiveProcessLimit;
+    ULONG* Affinity;
+    DWORD PriorityClass;
+    /* VC98 has only the above fields */
+
+    DWORD SchedulingClass;
+} NEW_JOBOBJECT_BASIC_LIMIT_INFORMATION;
+
+typedef struct _JOBOBJECT_EXTENDED_LIMIT_INFORMATION {
+    NEW_JOBOBJECT_BASIC_LIMIT_INFORMATION BasicLimitInformation;
+    IO_COUNTERS IoInfo;
+    SIZE_T ProcessMemoryLimit;
+    SIZE_T JobMemoryLimit;
+    SIZE_T PeakProcessMemoryUsed;
+    SIZE_T PeakJobMemoryUsed;
+} JOBOBJECT_EXTENDED_LIMIT_INFORMATION, *PJOBOBJECT_EXTENDED_LIMIT_INFORMATION;
+
+#endif
+
+void 
+StartRestrictedProcess(LPTSTR app_name, LPTSTR app_cmdline,
+                            SIZE_T pagefile_limit)
+{
+    // based on http://www.microsoft.com/msj/0399/jobkernelobj/jobkernelobj.aspx
+
+    // Create a job kernel object
+    HANDLE hjob = CreateJobObject(NULL, NULL);
+
+    NEW_JOBOBJECT_BASIC_LIMIT_INFORMATION jobli = { 0 };
+    JOBOBJECT_BASIC_UI_RESTRICTIONS jobuir;
+
+    STARTUPINFO si = { sizeof(si) };
+    PROCESS_INFORMATION pi;
+
+    JOBOBJECT_EXTENDED_LIMIT_INFORMATION jobext = { 0 };
+    BOOL ok;
+
+    // Place some restrictions on processes in the job
+
+    // First, set some basic restrictions
+
+    // The process always runs in the idle priority class
+    jobli.PriorityClass = IDLE_PRIORITY_CLASS;
+
+    // The job cannot use more than 10 second of CPU time
+    jobli.PerJobUserTimeLimit.QuadPart = 10*10000000;    // 1 second in 100-ns intervals
+
+    // These are two 2 restrictions I want placed on the job (process)
+    jobli.LimitFlags = JOB_OBJECT_LIMIT_PRIORITY_CLASS | JOB_OBJECT_LIMIT_JOB_TIME;
+    ok = SetInformationJobObject(hjob, JobObjectBasicLimitInformation, &jobli, sizeof(jobli));
+    if (!ok) {
+        fprintf(FP, "GLE %d", GetLastError());
+    }
+
+    // Second, set some UI restrictions
+    jobuir.UIRestrictionsClass  = JOB_OBJECT_UILIMIT_NONE;    // A fancy 0
+
+    // The process can’t logoff the system
+    jobuir.UIRestrictionsClass |= JOB_OBJECT_UILIMIT_EXITWINDOWS;
+
+    // The process can’t access USER object (like other windows) in the system
+    jobuir.UIRestrictionsClass |= JOB_OBJECT_UILIMIT_HANDLES;
+
+    ok = SetInformationJobObject(hjob, JobObjectBasicUIRestrictions, &jobuir, sizeof(jobuir));
+    if (!ok) {
+        fprintf(FP, "GLE %d", GetLastError());
+    }
+
+    // actually we really want to set page file limits as well
+    // and we'll set separately
+    // The process always runs in the idle priority class
+    jobext.ProcessMemoryLimit = pagefile_limit;
+    jobext.BasicLimitInformation.LimitFlags = 0x00000100 /* JOB_OBJECT_LIMIT_PROCESS_MEMORY */;
+
+    ok = SetInformationJobObject(hjob, 
+                                 9 /* FIXME: JobObjectExtendedLimitInformation */,
+                                 &jobext, sizeof(jobext));
+    if (!ok) {
+        fprintf(FP, "GLE %d", GetLastError());
+    }
+
+    // Spawn the process that is to be in the job
+    // Note: You must first spawn the process and then place the process in the job.
+    //        This means that the process's thread must be initially suspended so that 
+    //        it can't execute any code outside of the job's restrictions.
+    CreateProcess(app_name, app_cmdline, 
+                  NULL, NULL, FALSE, 
+                  CREATE_SUSPENDED, NULL, NULL, &si, &pi);
+
+    // Place the process in the job.
+    // Note: if this process spawns any children, the children are 
+    //        automatically associated with the same job
+    AssignProcessToJobObject(hjob, pi.hProcess);
+
+    // Now, we can allow the child process’s thread to execute code.
+    ResumeThread(pi.hThread);
+    CloseHandle(pi.hThread);
+
+    // Wait for the process to terminate or for all the job’s allotted CPU time to be used
+    {
+        DWORD dw;
+        HANDLE h[2];
+        h[0] = pi.hProcess;
+        h[1] = hjob;
+        dw = WaitForMultipleObjects(2, h, FALSE, INFINITE);
+        switch (dw - WAIT_OBJECT_0) {
+        case 0:
+            // The process has terminated
+            break;
+        case 1:
+            // All of the job's allotted CPU time or other limits were used
+            fprintf(stderr, "job limit reached\n");
+            break;
+        }
+    }
+    // Clean-up properly
+    CloseHandle(pi.hProcess);
+    CloseHandle(hjob);
+}
+
+// FIXME: note that we can use a Job limit to precisely alot the CPU time
+int usage(char *us)
+{
+    fprintf(FP, "Usage: %s [-p page_limit in MB] [-kb] <program> <args...>\n", us);
+    fflush(FP);
+    return 0;
+}
+
+int __cdecl
+main(int argc, char *argv[], char *envp[])
+{
+    LPTSTR              app_name = NULL;
+    TCHAR               full_app_name[2*MAX_PATH];
+    TCHAR               app_cmdline[2*MAX_PATH];
+    int                 arg_offs = 1;
+
+    int allocation_unit = 1024*1024;
+    int pagelimit = 10*allocation_unit; /* 10 MB */
+
+    if (argc < 2) {
+        return usage(argv[0]);
+    }
+    while (argv[arg_offs][0] == '-') {
+        if (strcmp(argv[arg_offs], "-p") == 0) {
+            if (argc <= arg_offs+1)
+                return usage(argv[0]);
+            pagelimit = atoi(argv[arg_offs+1]) * allocation_unit;
+            arg_offs += 2;
+        }
+        else if (strcmp(argv[arg_offs], "-kb") == 0) {
+            allocation_unit=1024;
+            arg_offs += 1;
+        }
+
+    }
+
+    /* FIXME: watch out for the usual /Program BUG */
+    _snwprintf(full_app_name, BUFFER_SIZE_ELEMENTS(full_app_name), L"%hs", 
+              argv[arg_offs]);
+
+    /* FIXME: only one argument taken */
+    _snwprintf(app_cmdline, BUFFER_SIZE_ELEMENTS(app_cmdline), L"%hs %hs", 
+              argv[arg_offs], argv[arg_offs + 1]);
+
+    StartRestrictedProcess(NULL/* FIXME: full_app_name */, 
+                           app_cmdline,
+                           pagelimit);
+    fprintf(FP, "done\n");
+    fflush(FP);
+    return 0;
+}
+
+
+
+
diff --git a/tools/runstats.c b/tools/runstats.c
new file mode 100644
index 000000000..d06b088ca
--- /dev/null
+++ b/tools/runstats.c
@@ -0,0 +1,415 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* runstats.c
+ * by Derek Bruening
+ *
+ * similar to /usr/bin/time, but adds maximum memory
+ * usage statistics, implemented via sampling every 500ms
+ *
+ * also has texec functionality: will kill child after specified
+ * time limit expires
+ */
+
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/types.h> /* for wait and mmap */
+#include <sys/wait.h>  /* for wait */
+#include <assert.h>
+#include <stdio.h>
+#include <sys/time.h> /* itimer */
+#include <stdlib.h> /* getenv */
+#include <string.h> /* strstr */
+
+#define VERBOSE 0
+
+#define FP stderr
+
+/* just use single-arg handlers */
+typedef void (*handler_t)(int);
+typedef void (*handler_3_t)(int, struct siginfo *, void *);
+
+typedef struct {
+    unsigned long VmSize;
+    unsigned long VmLck;
+    unsigned long VmRSS;
+    unsigned long VmData;
+    unsigned long VmStk;
+    unsigned long VmExe;
+    unsigned long VmLib;
+} vmstats_t;
+
+/* global vars for access in signal handler */
+static pid_t child;
+static vmstats_t vmstats;
+static int limit; /* in seconds */
+struct timeval start, end;
+static int silent; /* whether to print anything */
+
+/***************************************************************************/
+/* /proc/self/status reader */
+
+/* these are defined in /usr/src/linux/fs/proc/array.c */
+#define STATUS_LINE_LENGTH        4096
+#define STATUS_LINE_FORMAT         "%s %d"
+
+void
+get_mem_stats(pid_t pid)
+{
+    char proc_pid_status[64];      /* file name */
+    FILE *status;
+    char line[STATUS_LINE_LENGTH];
+    // open file's /proc/id/status file
+    int n = snprintf(proc_pid_status, sizeof(proc_pid_status),
+                     "/proc/%d/status", pid);
+    if (n<0 || n==sizeof(proc_pid_status))
+        assert(0); /* paranoia */
+    status = fopen(proc_pid_status,"r");
+    if (status == NULL)
+        return; /* child already exited */
+    while(!feof(status)){
+        unsigned long val = 0;
+        char prefix[STATUS_LINE_LENGTH];
+        int len;
+        if (NULL==fgets(line, sizeof(line), status))
+            break;
+        if (strstr(line, "VmSize") != 0) {
+            len = sscanf(line, STATUS_LINE_FORMAT, prefix, &val);
+            assert(len == 2);
+            if (val > vmstats.VmSize)
+                vmstats.VmSize = val;
+#if VERBOSE
+            fprintf(FP, "VmSize is %d kB\n", val);
+#endif
+        } else if (strstr(line, "VmLck") != 0) {
+            len = sscanf(line, STATUS_LINE_FORMAT, prefix, &val);
+            assert(len == 2);
+            if (val > vmstats.VmLck)
+                vmstats.VmLck = val;
+#if VERBOSE
+            fprintf(FP, "VmLck is %d kB\n", val);
+#endif
+        } else if (strstr(line, "VmRSS") != 0) {
+            len = sscanf(line, STATUS_LINE_FORMAT, prefix, &val);
+            assert(len == 2);
+            if (val > vmstats.VmRSS)
+                vmstats.VmRSS = val;
+#if VERBOSE
+            fprintf(FP, "VmRSS is %d kB\n", val);
+#endif
+        } else if (strstr(line, "VmData") != 0) {
+            len = sscanf(line, STATUS_LINE_FORMAT, prefix, &val);
+            assert(len == 2);
+            if (val > vmstats.VmData)
+                vmstats.VmData = val;
+#if VERBOSE
+            fprintf(FP, "VmData is %d kB\n", val);
+#endif
+        } else if (strstr(line, "VmStk") != 0) {
+            len = sscanf(line, STATUS_LINE_FORMAT, prefix, &val);
+            assert(len == 2);
+            if (val > vmstats.VmStk)
+                vmstats.VmStk = val;
+#if VERBOSE
+            fprintf(FP, "VmStk is %d kB\n", val);
+#endif
+        } else if (strstr(line, "VmExe") != 0) {
+            len = sscanf(line, STATUS_LINE_FORMAT, prefix, &val);
+            assert(len == 2);
+            if (val > vmstats.VmExe)
+                vmstats.VmExe = val;
+#if VERBOSE
+            fprintf(FP, "VmExe is %d kB\n", val);
+#endif
+        } else if (strstr(line, "VmLib") != 0) {
+            len = sscanf(line, STATUS_LINE_FORMAT, prefix, &val);
+            assert(len == 2);
+            if (val > vmstats.VmLib)
+                vmstats.VmLib = val;
+#if VERBOSE
+            fprintf(FP, "VmLib is %d kB\n", val);
+#endif
+            /* final one, so break */
+            break;
+        }
+    }
+    fclose(status);
+}
+
+/***************************************************************************/
+
+static void
+signal_handler(int sig)
+{
+    if (sig == SIGALRM) {
+#if VERBOSE
+        fprintf(FP, "just got SIGALRM for %d  =>  ", child);
+#endif
+        if (limit > 0) {
+            gettimeofday(&end, (struct timezone *) 0);
+#if VERBOSE
+            fprintf(FP, "SIGALRM: comparing limit %d s vs. elapsed %d s\n",
+                    limit, end.tv_sec - start.tv_sec);
+#endif
+            /* for limit, ignore fractions of second */
+            if (end.tv_sec - start.tv_sec > limit) {
+                /* could use SIGTERM, but on win32 cygwin need SIGKILL */
+                kill(child, SIGKILL);
+                fprintf(FP, "Timeout after %d seconds\n", limit);
+                exit(-1);
+            }
+        }
+        get_mem_stats(child);
+    } else if (sig == SIGCHLD) {
+#if VERBOSE
+        fprintf(FP, "just got SIGCHLD for %d\n", child);
+#endif
+    }
+}
+
+/* set up signal_handler as the handler for signal "sig" */
+static void
+intercept_signal(int sig, handler_t handler)
+{
+    int rc;
+    struct sigaction act;
+
+    act.sa_sigaction = (handler_3_t) handler;
+#if BLOCK_IN_HANDLER
+    rc = sigfillset(&act.sa_mask); /* block all signals within handler */
+#else
+    rc = sigemptyset(&act.sa_mask); /* no signals are blocked within handler */
+#endif
+    assert(rc == 0);
+    act.sa_flags = SA_ONSTACK;
+    
+    /* arm the signal */
+    rc = sigaction(sig, &act, NULL);
+    assert(rc == 0);
+}
+
+#define TV_MSEC tv_usec / 1000
+
+/* some of this code is based on /usr/bin/time source code */
+void
+print_stats(struct timeval *start, struct timeval *end,
+            struct rusage *ru, int status)
+{
+    unsigned long r;            /* Elapsed real milliseconds.  */
+    unsigned long v;            /* Elapsed virtual (CPU) milliseconds.  */
+    end->tv_sec -= start->tv_sec;
+    if (end->tv_usec < start->tv_usec) {
+        /* Manually carry a one from the seconds field.  */
+        end->tv_usec += 1000000;
+        --end->tv_sec;
+    }
+    end->tv_usec -= start->tv_usec;
+
+    if (WIFSTOPPED (status)) {
+        fprintf(FP, "Command stopped by signal %d\n",
+                WSTOPSIG (status));
+    } else if (WIFSIGNALED (status)) {
+        fprintf(FP, "Command terminated by signal %d\n",
+                WTERMSIG (status));
+    } else if (WIFEXITED (status) && WEXITSTATUS (status)) {
+        fprintf(FP, "Command exited with non-zero status %d\n",
+                WEXITSTATUS (status));
+    }
+
+    /* Convert all times to milliseconds.  Occasionally, one of these values
+       comes out as zero.  Dividing by zero causes problems, so we first
+       check the time value.  If it is zero, then we take `evasive action'
+       instead of calculating a value.  */
+    
+    r = end->tv_sec * 1000 + end->tv_usec / 1000;
+    
+    v = ru->ru_utime.tv_sec * 1000 + ru->ru_utime.TV_MSEC +
+        ru->ru_stime.tv_sec * 1000 + ru->ru_stime.TV_MSEC;
+
+    /* Elapsed real (wall clock) time.  */
+    if (end->tv_sec >= 3600) {  /* One hour -> h:m:s.  */
+        fprintf(FP, "%ld:%02ld:%02ldelapsed ",
+                 end->tv_sec / 3600,
+                 (end->tv_sec % 3600) / 60,
+                 end->tv_sec % 60);
+    } else {
+        fprintf(FP, "%ld:%02ld.%02ldelapsed ",  /* -> m:s.  */
+                 end->tv_sec / 60,
+                 end->tv_sec % 60,
+                 end->tv_usec / 10000);
+    }
+    /* % cpu is (total cpu time)/(elapsed time).  */
+    if (r > 0)
+        fprintf(FP, "%lu%%CPU ", (v * 100 / r));
+    else
+        fprintf(FP, "?%%CPU ");
+    fprintf(FP, "%ld.%02lduser ",
+            ru->ru_utime.tv_sec, ru->ru_utime.TV_MSEC / 10);
+    fprintf(FP, "%ld.%02ldsystem ",
+            ru->ru_stime.tv_sec, ru->ru_stime.TV_MSEC / 10);
+    fprintf(FP, "(%ldmajor+%ldminor)pagefaults %ldswaps\n",
+            ru->ru_majflt,/* Major page faults.  */
+            ru->ru_minflt,/* Minor page faults.  */
+            ru->ru_nswap); /* times swapped out */
+    fprintf(FP, "(%lu tot, %lu RSS, %lu data, %lu stk, %lu exe, %lu lib)k\n",
+            vmstats.VmSize, vmstats.VmRSS, vmstats.VmData,
+            vmstats.VmStk, vmstats.VmExe, vmstats.VmLib);
+
+#if VERBOSE
+    fprintf(FP, "Memory usage:\n");
+    fprintf(FP, "\tVmSize: %d kB\n", vmstats.VmSize);
+    fprintf(FP, "\tVmLck: %d kB\n", vmstats.VmLck);
+    fprintf(FP, "\tVmRSS: %d kB\n", vmstats.VmRSS);
+    fprintf(FP, "\tVmData: %d kB\n", vmstats.VmData);
+    fprintf(FP, "\tVmStk: %d kB\n", vmstats.VmStk);
+    fprintf(FP, "\tVmExe: %d kB\n", vmstats.VmExe);
+    fprintf(FP, "\tVmLib: %d kB\n", vmstats.VmLib);
+#endif
+}
+
+int usage(char *us)
+{
+    fprintf(FP, "Usage: %s [-s limit_sec | -m limit_min | -h limit_hr]\n"
+                "       [-silent] [-env var value] <program> <args...>\n", us);
+    return 0;
+}
+
+int main(int argc, char *argv[])
+{
+    int rc;
+    struct itimerval t;
+    struct rusage ru;
+    int arg_offs = 1;
+
+    if (argc < 2) {
+        return usage(argv[0]);
+    }
+    while (argv[arg_offs][0] == '-') {
+        if (strcmp(argv[arg_offs], "-s") == 0) {
+            if (argc <= arg_offs+1)
+                return usage(argv[0]);
+            limit = atoi(argv[arg_offs+1]);
+            arg_offs += 2;
+        } else if (strcmp(argv[arg_offs], "-m") == 0) {
+            if (argc <= arg_offs+1)
+                return usage(argv[0]);
+            limit = atoi(argv[arg_offs+1])*60;
+            arg_offs += 2;
+        } else if (strcmp(argv[arg_offs], "-h") == 0) {
+            if (argc <= arg_offs+1)
+                return usage(argv[0]);
+            limit = atoi(argv[arg_offs+1])*3600;
+            arg_offs += 2;
+        } else if (strcmp(argv[arg_offs], "-v") == 0) {
+            /* just ignore -v, only here for compatibility with texec */
+            arg_offs += 1;
+        } else if (strcmp(argv[arg_offs], "-silent") == 0) {
+            silent = 1;
+            arg_offs += 1;
+        } else if (strcmp(argv[arg_offs], "-env") == 0) {
+            if (argc <= arg_offs+2)
+                return usage(argv[0]);
+#if VERBOSE
+            fprintf(FP, "setting env var \"%s\" to \"%s\"\n",
+                    argv[arg_offs+1], argv[arg_offs+2]);
+#endif
+            rc = setenv(argv[arg_offs+1], argv[arg_offs+2], 1/*overwrite*/);
+            if (rc != 0 ||
+                strcmp(getenv(argv[arg_offs+1]), argv[arg_offs+2]) != 0) {
+                fprintf(FP, "error in setenv of \"%s\" to \"%s\"\n",
+                        argv[arg_offs+1], argv[arg_offs+2]);
+                fprintf(FP, "setenv returned %d\n", rc);
+                fprintf(FP, "env var \"%s\" is now \"%s\"\n",
+                        argv[arg_offs+1], getenv(argv[arg_offs+1]));
+                exit(-1);
+            }
+            arg_offs += 3;
+        } else {
+            return usage(argv[0]);
+        }
+        if (limit < 0) {
+            return usage(argv[0]);
+        }
+        if (argc - arg_offs < 1)
+            return usage(argv[0]);
+    }
+
+    gettimeofday(&start, (struct timezone *) 0);
+
+    child = fork();
+    if (child < 0) {
+        perror("ERROR on fork");
+    } else if (child > 0) {
+        pid_t result;
+        int status;
+
+        get_mem_stats(child);
+
+        intercept_signal(SIGALRM, (handler_t) signal_handler);
+        intercept_signal(SIGCHLD, (handler_t) signal_handler);
+
+        t.it_interval.tv_sec = 0;
+        t.it_interval.tv_usec = 500000;
+        t.it_value.tv_sec = 0;
+        t.it_value.tv_usec = 500000;
+        /* must use real timer so timer decrements while process suspended */
+        rc = setitimer(ITIMER_REAL, &t, NULL);
+        assert(rc == 0);
+
+#if VERBOSE
+        fprintf(FP, "parent waiting for child\n");
+#endif
+        /* need loop since SIGALRM will interrupt us */
+        do {
+            result = wait3(&status, 0, &ru);
+        } while (result != child);
+        gettimeofday(&end, (struct timezone *) 0);
+#if VERBOSE
+        fprintf(FP, "child has exited\n");
+#endif
+    
+        /* turn off timer */
+        t.it_interval.tv_usec = 0;
+        t.it_value.tv_usec = 0;
+        rc = setitimer(ITIMER_REAL, &t, NULL);
+        assert(rc == 0);
+        
+        if (!silent)
+            print_stats(&start, &end, &ru, status);
+    } else {
+        int result;
+        result = execvp(argv[arg_offs], argv+arg_offs);
+        if (result < 0)
+            perror("ERROR in execvp");
+    }
+    return 0;
+}
diff --git a/tools/shared-stats.pl b/tools/shared-stats.pl
new file mode 100755
index 000000000..88e34e3cc
--- /dev/null
+++ b/tools/shared-stats.pl
@@ -0,0 +1,122 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2004-2005 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# shared-stats.pl
+# quick script to process thread sharing study results from a
+# SHARING_STUDY DynamoRIO build
+# by Derek Bruening, Dec 2003
+
+# usage: give it the process logfile as stdin
+
+# will see many copies of each stat, just want last one
+
+$who = 0;
+$have_traces = 0;
+while (<>) {
+    if (/^Shared basic block statistics/) {
+        $who = 0;
+        $print_bb = $_; # start over each time so get last
+    } elsif (/^Shared trace statistics/) {
+        $who = 1;
+        $have_traces = 1;
+        $print_bb .= $_;
+    } elsif (/total blocks:\s*(\d+)/) {
+        $tot[$who]=$1;
+        $print_bb .= $_;
+    } elsif (/shared count:\s*(\d+)/) {
+        $cnt[$who]=$1;
+        $print_bb .= $_;
+    } elsif(/shared blocks:\s*(\d+)/) {
+        $shar[$who]=$1;
+        $print_bb .= $_;
+    } elsif (/shared heap:\s*(\d+)/) {
+        $heap[$who]=$1;
+        $print_bb .= $_;
+    } elsif(/shared cache:\s*(\d+)/) {
+        $cache[$who]=$1;
+        $print_bb .= $_;
+    } elsif(/Peak stack/) {
+        $print_st .= $_;
+    } elsif(/Peak fcache.*:\s*(\d+)/) {
+        $peak_cache=$1;
+        $print_st .= $_;
+    } elsif(/Peak heap.*:\s*(\d+)/) {
+        $peak_heap=$1;
+        $print_st .= $_;
+    } elsif(/Peak total mem.*:\s*(\d+)/) {
+        $peak_mem=$1;
+        $print_st .= $_;
+    } elsif(/Threads ever/) {
+        $print_st = $_; # start over each time so get last
+    } elsif(/Threads under.*:\s*(\d+)/) {
+        $threads_under += $1;
+        $threads_under_num ++;
+    }
+}
+
+print $print_st;
+
+print $print_bb;
+
+printf  "Ave # threads under DR control: %9.2f\n", $threads_under/$threads_under_num;
+printf  "  <ave over stats dumps, not time!>\n";
+
+for ($i=0; $i<2; $i++) {
+    if ($i == 0) {
+        printf "\n*** Basic blocks ***\n";
+    } else {
+        if (!$have_traces) {
+            last;
+        }
+        printf "\n*** Traces ***\n";
+    }
+    if ($shar[$i] == 0) {
+        printf "No sharing!\n";
+        printf "%% peak mem saved:           %9.2f\n", 0;
+        next;
+    }
+    $per = ($cnt[$i] - ($tot[$i] - $shar[$i])) / $shar[$i];
+    printf  "Unshared blocks:                %9d\n", $tot[$i] - $shar[$i];
+    printf "%% blocks that are shared:       %9.2f\n", 100*$shar[$i]/$tot[$i];
+    printf  "Ave # threads per shared block: %9.2f\n", $per;
+        printf "%% peak heap that's shared:      %9.2f\n", 100*$heap[$i] / $peak_heap;
+    printf "%% peak cache that's shared:     %9.2f\n", 100*$cache[$i] / $peak_cache;
+    printf "%% peak mem that's shared:       %9.2f\n", 100*($cache[$i]+$heap[$i]) / $peak_mem;
+# if we assume even split among threads, w/ shared bb cache should
+# save $heap * (1 - 1/$per)
+    $save_heap = $heap[$i] * (1 - 1/$per);
+    $save_cache = $cache[$i] * (1 - 1/$per);
+    printf "%% peak heap saved:              %9.2f\n", 100*$save_heap / $peak_heap;
+    printf "%% peak cache saved:             %9.2f\n", 100*$save_cache / $peak_cache;
+    printf "%% peak mem saved:               %9.2f\n", 100*($save_cache+$save_heap) / $peak_mem;
+}
+
diff --git a/tools/stripCR b/tools/stripCR
new file mode 100755
index 000000000..ecc7e9608
--- /dev/null
+++ b/tools/stripCR
@@ -0,0 +1,36 @@
+#!/bin/csh
+
+# **********************************************************
+# Copyright (c) 2001-2003 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# Remove \r from the file given as argument
+mv $1 "/tmp/._#$1:t"
+tr -d '\r' < "/tmp/._#$1:t" > $1
+rm "/tmp/._#$1:t"
diff --git a/tools/svccntrl.c b/tools/svccntrl.c
new file mode 100644
index 000000000..b160db827
--- /dev/null
+++ b/tools/svccntrl.c
@@ -0,0 +1,246 @@
+/* **********************************************************
+ * Copyright (c) 2003-2006 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/*
+ * svccntrl.c
+ *
+ * command-line tool for manipulating services
+ *
+ * (c) determina, inc. all rights reserved
+ */
+
+#include "mfapi.h"
+#include "services.h"
+
+#include <stdio.h>
+
+WCHAR *longbuff;
+ServiceHandle foundsvc = INVALID_SERVICE_HANDLE;
+
+BOOL show_svcs_cb(ServiceHandle service, void **param)
+{
+    static const char *automatic="auto";
+    static const char *manual="manual";
+    static const char *disabled="disabled";
+    static const char *unknown="<unknown>";
+    const char *typename;
+    DWORD type = get_service_start_type(service);
+    typename = (type == SERVICE_AUTO_START ? automatic : 
+                            (type == SERVICE_DEMAND_START ? manual : 
+                             (type == SERVICE_DISABLED ? disabled :
+                              unknown)));
+    printf("%S %s\n", get_service_name(service), typename);
+    return TRUE;
+}
+
+BOOL 
+finddispname_cb(ServiceHandle svc, void **param)
+{
+    if (!wcsicmp(longbuff, get_service_display_name(svc))) {
+        foundsvc = svc;
+        return FALSE;
+    }
+    return TRUE;
+}
+
+void 
+usage() 
+{
+    fprintf(stderr,"Usage:\nsvcctrl svcname [ -auto | -manual | -restart | -disabled ] [-help] [-show] [-dep svc2] [-depreset] [-v]\n");
+    exit(1);
+}
+
+void
+help() 
+{
+    
+    fprintf(stderr, "Options:\n");
+    fprintf(stderr, " -auto\\tt\tset the service to run automatically\n");
+    fprintf(stderr, " -manual\t\tset the service to manual control\n");
+    fprintf(stderr, " -restart\t\tset the service to auto-restart\n");
+    fprintf(stderr, " -disabled\t\tdisable the service\n");
+    fprintf(stderr, " -show\t\t\tshow all installed services and start types\n");
+    fprintf(stderr, " -dep svc2\t\tmake the service dependent on svc2\n");
+    fprintf(stderr, " -depreset\t\treset service dependencies\n");
+    fprintf(stderr, " -v\t\t\tdisplay version information\n\n");
+    exit(1);
+}
+
+
+ServiceHandle
+get_svc(char *buf)
+{
+    ServiceHandle svc;
+    WCHAR w_buf[MAX_PATH];
+
+    _snwprintf(w_buf, MAX_PATH, L"%S", buf);
+    svc = get_service_by_name(w_buf);
+    if (svc == INVALID_SERVICE_HANDLE) {
+        longbuff = w_buf;
+        enumerate_services(&finddispname_cb, NULL);
+        svc = foundsvc;
+    }
+    return svc;
+}
+
+int 
+main(int argc, char **argv)
+{
+    int res=0,
+        sauto=0,
+        sman=0,
+        sdis=0,
+        show=0,
+        depreset=0,
+        srestart=0;
+    char *dep = NULL;
+
+    char *svcname=NULL;
+    int argidx=1;
+    ServiceHandle svc;
+    WCHAR w_buf[MAX_PATH];
+
+    svcname = argv[argidx++];
+
+    while (argidx < argc) {
+
+        if (!strcmp(argv[argidx], "-help")) {
+  	    help();
+	}
+	else if (!strcmp(argv[argidx], "-auto")) {
+	    sauto=1;
+	}
+	else if (!strcmp(argv[argidx], "-manual")) {
+	    sman=1;
+	}
+	else if (!strcmp(argv[argidx], "-restart")) {
+	    srestart=1;
+	}
+	else if (!strcmp(argv[argidx], "-show")) {
+	    show=1;
+	}
+	else if (!strcmp(argv[argidx], "-depreset")) {
+	    depreset=1;
+	}
+	else if (!strcmp(argv[argidx], "-dep")) {
+            if (argidx + 1 >= argc)
+                usage();
+            dep = argv[++argidx];
+	}
+	else if (!strcmp(argv[argidx], "-disabled")) {
+	    sdis=1;
+	}
+	else if (!strcmp(argv[argidx], "-v")) {
+#ifdef BUILD_NUMBER
+	  printf("svccntrl.exe build %d -- %s\n", BUILD_NUMBER, __DATE__);
+#else
+	  printf("svccntrl.exe custom build -- %s, %s\n", __DATE__, __TIME__);
+#endif
+	}
+	else {
+	    fprintf(stderr, "Unknown option: %s\n", argv[argidx]);
+	    usage();
+	}
+	argidx++;
+    }
+
+    if (argc < 3) 
+        usage();
+  
+    if ( sauto + sman + sdis + show + srestart != 1 && 
+         dep == NULL && !depreset) {
+        fprintf(stderr, "Bad combination of options.\n");
+        usage();
+    }
+
+    services_init();
+
+    if (show) {
+        enumerate_services(&show_svcs_cb, NULL);
+    }
+    else {
+       
+        _snwprintf(w_buf, MAX_PATH, L"%S", svcname);
+
+        if (srestart) {
+            res = set_service_restart_type(w_buf, FALSE);
+            if (res != ERROR_SUCCESS) {
+                fprintf(stderr, "Error %d updating the configuration\n", res);
+            }
+        }
+        else if (dep) {
+            ServiceHandle svc2;
+            svc = get_svc(svcname);
+            svc2 = get_svc(dep);
+            if (svc == INVALID_SERVICE_HANDLE ||
+                svc2 == INVALID_SERVICE_HANDLE) {
+                fprintf(stderr, "Invalid services: %s, %s\n", svcname, dep);
+            }
+            else {
+                res = add_dependent_service(svc, svc2);
+                if (res != ERROR_SUCCESS)
+                    fprintf(stderr, "Error %d setting dependencies\n", res);
+            }
+        }
+        else if (depreset) {
+            svc = get_svc(svcname);
+            if (svc == INVALID_SERVICE_HANDLE) {
+                fprintf(stderr, "Invalid service: %s\n", svcname);
+            }
+            else {
+                res = reset_dependent_services(svc);
+                if (res != ERROR_SUCCESS)
+                    fprintf(stderr, "Error %d resetting dependencies\n", res);
+            }
+        }
+        else {
+            svc = get_svc(svcname);
+            
+            if (svc == INVALID_SERVICE_HANDLE) {
+                fprintf(stderr, "Invalid service: %s\n", svcname);
+            } else {
+                res = set_service_start_type(svc, sauto ? SERVICE_AUTO_START :
+                                             sman ? SERVICE_DEMAND_START :
+                                             sdis ? SERVICE_DISABLED :
+                                             SERVICE_NO_CHANGE);
+                if (res != ERROR_SUCCESS)
+                    fprintf(stderr, "There was an error setting the configuration\n");
+            }
+        }
+        
+    }
+
+    services_cleanup();
+
+    return 0;
+
+}
diff --git a/tools/syscall.c b/tools/syscall.c
new file mode 100644
index 000000000..8dc7aef65
--- /dev/null
+++ b/tools/syscall.c
@@ -0,0 +1,86 @@
+/* **********************************************************
+ * Copyright (c) 2004 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+
+#include <windows.h>
+#include <stdio.h>
+
+
+int
+main(int argc, char **argv)
+{
+    int   trip_count;
+    int   syscall_count = 0;
+    int   i;
+    int   tid = GetCurrentThreadId();
+    void *addr;
+    int   old_flags;
+    int   prot_mask = PAGE_EXECUTE;
+    int   prot_flags = PAGE_EXECUTE_READ;
+    BOOL  double_syscall = FALSE;
+
+
+    if ( !(argc == 2 || argc == 3) ||
+         (argc == 3 && strcmp(argv[2], "-double_syscall"))) {
+        fprintf(stderr, "Usage: %s <loop trip count> [-double-syscall]\n",
+                argv[0]);
+        return -1;
+    }
+    if (argc == 3 && !strcmp(argv[2], "-double_syscall"))
+     double_syscall = TRUE;
+    trip_count = atoi(argv[1]);
+    addr = VirtualAlloc(NULL,
+                        65536,
+                        MEM_RESERVE,
+                        prot_flags);
+    for (i = 0; i < trip_count;
+         i++, syscall_count++, prot_flags ^= prot_mask) {
+        VirtualProtect(addr,
+                       65536,
+                       prot_flags,
+                       &old_flags);
+        if (double_syscall) {
+            VirtualProtect(addr,
+                           65536,
+                           prot_flags,
+                           &old_flags);
+            syscall_count++;
+            prot_flags ^= prot_mask;
+        }
+    }
+    VirtualFree(addr,
+                0,
+                MEM_RELEASE);
+    fprintf(stderr, "Loop trips completed -- %d, syscalls completed -- %d\n",
+            i, syscall_count);
+    return 0;
+}
diff --git a/tools/tracestats.pl b/tools/tracestats.pl
new file mode 100755
index 000000000..59a03f928
--- /dev/null
+++ b/tools/tracestats.pl
@@ -0,0 +1,1362 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2004-2006 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### tracestats.pl
+### formerly called Frags, with fewer options
+### author: Derek Bruening   February 2001
+###
+### Analyzes DynamoRIO trace output: sorts traces based
+### on counts or time spent in them (from DynamoRIO profiling info)
+###
+### -brief: lists summary table instead of full info for each trace
+### -stats: lists completion stats instead of full info
+### -prefix_stats: lists counts of prefixes that need to restore eflags
+### -links: will print % of time each exit from a trace was used
+###   (only with Dynamo's -prof_counts)
+### -groups: attempts to group traces based on their links
+### -exe exename: will look up line numbers in original
+###   code that traces came from 
+###   (only with Dynamo's -dump_trace_origins flag)
+###
+### Several scalars below could overflow a 32-bit int, I assume
+### perl always uses floats internally and won't overflow
+
+# FIXME: some of these are mutually exclusive, indicate which
+$usage = "Usage: $0 [-brief] [-stats] [-prefix_stats] [-links] [-groups]
+\t[-sideline] [-exe exename] [-bbpcs] [-pcs pcfile] <tracefile>
+
+-stats and -bbpcs: work best with -tracedump_origins output in the tracefile
+-links and -groups: only work with -prof_counts output in the tracefile
+";
+
+# defaults
+$have_exe = 0;
+$src = 0;
+$brief = 0;
+$links = 0;
+$groups = 0;
+$pcs = 0;
+$linux_pcs = 0;
+
+# win32 pcs only
+$win32_pcs_step = 0;
+$overcount_instrs = 0;
+$overcount_samples = 0;
+
+$stats = 0;
+$prefix_stats = 0;
+$infile = "";
+
+# get optional params
+while ($#ARGV >= 0) {
+    if ($ARGV[0] eq '-exe') {
+        if ($#ARGV <= 0) { print $usage; exit; }
+        shift;
+        $exe = $ARGV[0];
+        unless (-f "$exe" && -x "$exe" ) {
+            die "$err No executable $exe\n";
+        }
+        $have_exe = 1;
+        $src = 1; # src is not a separate option anymore
+    } elsif ($ARGV[0] eq '-in') {
+        if ($#ARGV <= 0) { print $usage; exit; }
+        shift;
+        $infile = $ARGV[0];
+    } elsif ($ARGV[0] eq '-pcs') {
+        if ($#ARGV <= 0) { print $usage; exit; }
+        shift;
+        $pcs = 1;
+        $pcfile = $ARGV[0];
+    } elsif ($ARGV[0] eq '-stats') {
+        $stats = 1;
+    } elsif ($ARGV[0] eq '-prefix_stats') {
+        # study of static and dynamic count of prefixes that need to restore eflags
+        $prefix_stats = 1;
+        $prefix_noeflags = 0;
+        $prefix_eflags = 0;
+        $prefix_noeflags_count = 0;
+        $prefix_eflags_count = 0;
+    } elsif ($ARGV[0] eq '-brief') {
+        $brief = 1;
+    } elsif ($ARGV[0] eq '-sideline') {
+        $brief = 1;
+        $sideline = 1;
+    } elsif ($ARGV[0] eq '-links') {
+        $brief = 1;
+        $links = 1;
+    } elsif ($ARGV[0] eq '-groups') {
+        $groups = 1;
+        $brief = 1;
+        $links = 1;
+    } elsif ($ARGV[0] eq '-bbpcs') {
+        $pcs_by_orig_bb = 1;
+    } elsif ($ARGV[0] =~ /^-/) {
+        print $usage;
+        exit;
+    } else {
+        $infile = $ARGV[0];
+        last;
+    }
+    shift;
+}
+
+die $usage if ($infile eq "");
+
+if ($src && !$have_exe) {
+    print "Error: must specify executable name with -exe when using -src\n";
+    exit 0;
+}
+
+if ($pcs) {
+    # read in all pcs into a table
+    open(PCS, "< $pcfile") || die "Error: Couldn't open $pcfile for input\n";
+    # Linux format:
+    #    ITIMER distribution (74):
+    #       74.3% of time in INTERPRETER (55)
+    #    ...
+    #
+    #    PC PROFILING RESULTS
+    #    pc=0xf6e8c129   #=1     in DynamoRIO dispatch
+    #    pc=0x179d102c   #=1     in trace #   251 @0x08048470 w/ offs 0x00000028
+    #    pc=0x002c1192   #=1     in DynamoRIO interpreter
+    #    ...     
+    # Windows format:
+    #    ...
+    #    Dumping dynamorio.dll profile
+    #    127 hits out of 596, 21.30%
+    #    ...     
+    #    Dumping global profile
+    #    596 hits
+    #    ...
+    #    Dumping fcache trace unit profile (Shared)
+    #    134 hits
+    #    Profile Dump
+    #    Range 0x182f1000-0x182fefff
+    #    Step 0x00000008 (0-7167)
+    #    0x182f1c78          1
+    #    0x182f1d60          1
+    #    0x182f1d78          1
+    #    ...
+    #    Finished Profile Dump
+    my $in_fcache_pcs = 0;
+    while (<PCS>) {
+        chop;
+        # handle DOS end-of-line:
+        if ($_ =~ /
+$/) { chop; };
+        if ($_ =~ /ITIMER distribution/) {
+            $linux_pcs = 1;
+        }
+        if ($_ =~ /ITIMER distribution \(([0-9]+)\)/ ||
+            $_ =~ /hits out of ([0-9]+)/) {
+            $total_samples = $1;
+        }
+        $in_fcache_pcs = 1 if (/trace unit/);
+        next unless ((!$linux_pcs && $in_fcache_pcs) || ($linux_pcs && $_ =~ /^pc/));
+        $l = $_;
+        if ($linux_pcs) {
+            # only care about traces and fragments
+            # can be multiple entries for a single pc: must key by pc AND tag
+            if ($l =~ /trace/ || $l =~ /fragment/) {
+                if ($l =~ /trace \#/ || $l =~ /fragment \#/) {
+                    $l =~ /^pc=([x0-9a-f]+)[^\#]+\#=([0-9]+)[^\#]+\# *([0-9]+) @([x0-9a-f]+)/;
+                    # $1 = pc, $2 = count, $3 = id, $4 = tag
+                    $pc_samples{$4, $1} = $2;
+                } else {
+                    # release run: no trace id!
+                    $l =~ /^pc=([x0-9a-f]+)[^\#]+\#=([0-9]+)[^@]+@([x0-9a-f]+)/;
+                    # $1 = pc, $2 = count, $3 = tag
+                    $pc_samples{$3, $1} = $2;
+                }
+            }
+        } else {
+            if (/Finished Profile Dump/) {
+                $in_fcache_pcs = 0;
+            } elsif (/Step 0x0*([1-9]+)/) {
+                # bucket size
+                $win32_pcs_step = hex($1);
+            } else {
+                next unless ($l =~ /^0x/);
+                $l =~ /([x0-9a-f]+)\s+([0-9]+)/;
+                # FIXME: no tag info available so will double-count for traces
+                # that re-use same fcache slots
+                $pc_samples{$1} = $2;
+                $pc_sample_taken{$1} = 0;
+            }
+        }
+    }
+    close(PCS);
+}
+
+if ($src) {
+    # get info on shared libraries
+    $shared_num = 0;
+    open(LDD, "ldd $exe |") || die "Error: Couldn't run ldd $exe\n";
+    while (<LDD>) {
+        chop;
+        $l = $_;
+        if ($l =~ /^\s*([\/a-zA-Z_\.0-9-]+) => .* \((0x[0-9A-Fa-f]+)\)/) {
+            $shared_name[$shared_num] = $1;
+            $shared_start[$shared_num] = $2;
+            $shared_sort[$shared_num] = $shared_num;
+            $shared_num++;
+        }
+    }
+    close(LDD);
+
+    # now sort by starting address
+    @shared_sort = sort({hex($shared_start[$b]) <=> hex($shared_start[$a])}
+                        @shared_sort);
+    print "Shared library assumptions:\n";
+    for ($i=$shared_num-1; $i>=0; $i--) {
+        $addr1 = $shared_start[$shared_sort[$i]];
+        if ($i == 0) {
+            $addr2 = "above";
+        } else {
+            $addr2 = sprintf("0x%08x",
+                             hex($shared_start[$shared_sort[$i-1]]) - 1);
+        }
+        print "\t$addr1 .. $addr2 == $shared_name[$shared_sort[$i]]\n";
+    }
+    print "\n";
+}
+
+# read in and store all lines from file
+$in_trace = 0;
+$in_stub_summary = 0;
+$line = 0;
+$tnum = 0;
+$linked_to = 0;
+$have_times = 0;
+$max_stub = 0;
+$stub_64bit = 0;
+$body_start = 0;
+$direct_total = 0;
+$indirect_total = 0;
+$exitstub_boundary = -1;
+$flushed = 0;
+$stub_num = 0;
+$in_original = 0;
+$unseen_count = 0;
+
+# for pc samples
+$stubs_start = 0;
+$stub_samples = 0;
+$ind_samples = 0;
+$in_cmp = 0;
+$samples = 0;
+$dcontext_ecx = 0;
+$ret_samples = 0;
+$indjc_samples = 0; # indirect jumps & calls
+$branch_samples = 0;
+
+open(IN, "< $infile") || die "Error: Couldn't open $infile for input\n";
+while (<IN>) {
+    chop;
+    # handle DOS end-of-line:
+    if ($_ =~ /
+$/) { chop; };
+    $l = $_;
+    if ($l =~ /^TRACE \# ([0-9]+)/) {
+        # start of trace
+        $in_trace = 1;
+        # in case no Fragment# data
+        $id[$tnum] = $1;
+        if ($pcs && $pcs_by_orig_bb) {
+            # only keep src bb tag info for the current trace
+            $cur_bbnum = 0;
+            $cur_stubnum = 0;
+        }
+        if ($prefix_stats) {
+            $trace_total_exec = 0;
+        }
+    }
+    if ($in_trace) {
+        if ($l =~ /^Fragment \# ([0-9]+)/) {
+            # get fragment id
+            $id[$tnum] = $1;
+            # now initialize everything using fragment id:
+            # for reverse-lookup
+            $tnum_from_id{$1} = $tnum;
+            $tag[$tnum] = -1;
+            $size[$tnum] = -1;
+            if ($pcs) {
+                $time[$tnum] = 0;
+            } else {
+                $time[$tnum] = -1;
+            }
+            $count[$tnum] = -1;
+            $sideline_pre[$tnum] = -1;
+            $sideline_during[$tnum] = -1;
+            $sideline_post[$tnum] = -1;
+            # trace completion stats
+            $bbs[$tnum] = 0;
+            $bbs_noelide[$tnum] = 0;
+            $ind_branches[$tnum] = 0;
+            $bb_instrs[$tnum] = 0;
+            $bb_bytes[$tnum] = 0;
+            $end_in_cbr[$tnum] = 0;
+            $completion_all[$tnum] = 0;
+            $completion_half[$tnum] = 0;
+            $last_instr = "<not init>";
+        }
+        if (!$in_original && $l =~ /^ORIGINAL CODE/) {
+            $in_original = 1;
+        }
+        if ($in_original) {
+            if ($l =~ /^END ORIGINAL CODE/) {
+                $in_original = 0;
+                if ($pcs && $pcs_by_orig_bb) {
+                    # now count over again via exit jmps from trace
+                    $cur_bbnum = 0;
+                }
+                if ($stats || ($pcs && $pcs_by_orig_bb)) {
+                    # does trace end with conditional branch?
+                    if ($last_instr =~ /  0x[0-9a-fA-F]+   7/ ||        # jcc short
+                        $last_instr =~ /  0x[0-9a-fA-F]+   0[fF] 8/ ||  # jcc
+                        $last_instr =~ /  0x[0-9a-fA-F]+   e[0123]/) {  # loop*, jecxz
+                        $end_in_cbr[$tnum] = 1;
+                    }
+                    #printf("Frag \# %2d: ends in %s => %d cbr\n",
+                    #      $id[$tnum], $last_instr, $end_in_cbr[$tnum]);
+                }
+            }
+            if ($pcs && $pcs_by_orig_bb) {
+                if ($l =~ /^basic block.*start_pc = (0x[0-9a-fA-F]+)/ ||
+                    # tracedump.exe output: arbitrarily different, should fix
+                    $l =~ /^Basic block.*tag (0x[0-9a-fA-F]+)/) {
+                    $bbtag[$cur_bbnum++] = $1;
+                }
+                # handle elision
+                # easy for tracedump_text:
+                #    if ($l =~ /continuing in .* at (0x[0-9a-fA-F]+)/) {
+                # but rather than doing that and having to distinguish text
+                # and binary we calculate ourselves from call and jmp instrs:
+                if ($l =~ /  0x([0-9a-fA-F]+)\s+(e8|e9) (([0-9a-fA-F]+ ){4})\s*(call|jmp)/) {
+                    my $instraddr = $1;
+                    my $disp = $3;
+                    $disp =~ /(\w\w)\s+(\w\w)\s+(\w\w)\s+(\w\w)/;
+                    my $first = $4;
+                    $disp = hex("$4$3$2$1");
+                    # perl doesn't consider leading 1 to be negative
+                    if ($first =~ /^[89a-fA-F]/) {
+                        $disp -= 0xffffffff;
+                    }
+                    my $target = hex($instraddr) + 5 # len of instr
+                        + $disp;
+                    $target = sprintf("0x%08x", $target);
+                    # no exit for it so instead of making a new bb
+                    # just list it with the original tag
+                    $bbtag[$cur_bbnum-1] .= ",$target";
+                }
+                if ($l =~ /^  0x[0-9a-fA-F]+/) {
+                    $last_instr = $l;
+                }
+            }
+            if ($stats) {
+                if ($l =~ /^basic block \# ([0-9]+)/ ||
+                    # tracedump.exe output: arbitrarily different, should fix
+                    $l =~ /^Basic block ([0-9]+):/) {
+                    $bbs[$tnum] = $1 + 1;
+                    $bbs_noelide[$tnum]++;
+                } elsif ($l =~ /continuing/) {
+                    $bbs_noelide[$tnum]++;
+                } elsif ($l =~ /^  0x[0-9a-fA-F]+   (([0-9a-fA-F][0-9a-fA-F] )+)/) {
+                    $bb_instrs[$tnum]++;
+                    $bb_bytes[$tnum] += (length($1)/3);
+                    $last_instr = $l;
+                } elsif ($l =~ /^              (( [0-9a-fA-F][0-9a-fA-F])+)/) {
+                    $bb_bytes[$tnum] += (length($1)/3);
+                }
+            }
+        }
+        # even if brief, if pcs, we need to traverse
+        if (!$brief || $pcs) {
+            if ($prefix_stats && !$in_original && $l =~ /prefix entry/) {
+                # was prefix simply restore of eax?
+                if ($trace{$tnum,$line-2} =~ /indirect branch target entry/) {
+                    $prefix_noeflags++;
+                    $prefix_noeflags_count += $trace_total_exec;
+                } else {
+                    $prefix_eflags++;
+                    $prefix_eflags_count += $trace_total_exec;
+                }
+            }
+            if (!$in_original && ($pcs || $stats) && $dcontext_ecx == 0 &&
+                $l =~ /normal entry/) {
+                # get dcontext addr by examining prefix
+                # final instruction is always a restore to ecx
+                $trace{$tnum,$line-1} =~ /mov\s+(0x[0-9a-fA-F]*) -> \%ecx/;
+                $dcontext_ecx = hex($1); # 4 = ecx_offs - ebx_offs 
+            }
+            if (!$in_original && ($pcs || $stats) && $l =~ /^  (0x[0-9a-fA-F]+)/) {
+                $addr = $1; # keep in string form
+                if ($linux_pcs) {
+                    $samples = $pc_samples{$tag[$tnum], $addr};
+                } else {
+                    $addrnum = $addr;
+                    $addrnum =~ s/0x//;
+                    $addrnum = hex($addrnum);
+                    # Round it down to bucket size
+                    $addrnum = $addrnum & (~($win32_pcs_step - 1));
+                    # Now back to hex
+                    $addrnum = sprintf("0x%08x", $addrnum);
+                    # FIMXE: no tag so will dup in every trace that occupied that spot!
+                    $samples = $pc_samples{$addrnum};
+                    $pc_sample_taken{$addrnum}++;
+                    if ($pc_sample_taken{$addrnum} > 1) {
+                        $overcount_instrs++;
+                        $overcount_samples += $samples;
+                        # FIXME: rather than attributing samples to EVERY instr
+                        # that rounds into this bucket (and thus have summary stats
+                        # massively over-count), we instead assign samples to the
+                        # first instr in the bucket only, potentially missing
+                        # samples that actually happened on DR-overhead instrs.
+                        $samples = 0;
+                    }
+                    $time[$tnum] += $samples;
+                    if ($pcs && $pcs_by_orig_bb) {
+                        if ($stubs_start) {
+                            $stubsamples[$cur_stubnum] += $samples;
+                        } else {
+                            $bbsamples[$cur_bbnum] += $samples;
+                        }
+                    }
+                }
+                if ($l =~ / j[a-z]+\s*\$0x/ && !$stubs_start && !$in_cmp) {
+                    # direct branch
+                    $branch_samples += $samples;
+                }
+                # look for ind br overhead by watching for "save ecx"
+                if (is_save_ecx($l)) {
+                    $in_cmp = 1;
+                } 
+                if ($in_cmp) {
+                    # assume ind br is a return if "save ecx" followed by a pop
+                    if ($in_cmp == 2) {
+                        if ($stats) {
+                            $ind_branches[$tnum]++;
+                        }
+                        if ($pcs && $pcs_by_orig_bb) {
+                            $bbib[$cur_bbnum] = 1;
+                        }
+                        if ($l =~ /pop /) {
+                            $ret_samples += $samples;
+                        } else {
+                            $indjc_samples += $samples;
+                        }
+                    }
+                    $in_cmp++;
+                    # we count from save ecx through either restore ecx
+                    # (inlined) or jmp to exit stub, we do include jmp
+                    $ind_samples += $samples;
+                    if ($l =~ /jmp.*exit stub/) {
+                        $in_cmp = 0;
+                    }
+                    if (is_restore_ecx($l)) {
+                        $in_cmp = 0;
+                    }
+                }
+                if ($stubs_start) {
+                    $stub_samples += $samples;
+                }
+                if (!$brief) {
+                    $trace{$tnum,$line++} = sprintf("%3d%s", $samples, $l);
+                }
+                if ($pcs && $pcs_by_orig_bb) {
+                    if (!$stubs_start && is_exit_cti($l)) {
+                        $cur_bbnum++;
+                    }
+                    # $stubs_start case is handled below where we look for "exit stub
+                    # n", as looking at jmps is not enough for -prof_counts
+                }
+            } else {
+                if (!$brief) {
+                    $trace{$tnum,$line++} = $l;
+                }
+            }
+            if (!$in_original && $pcs && $pcs_by_orig_bb && $stubs_start &&
+                $l =~ /-- exit stub /) {
+                # $stubs_start is set below, which is what we want: don't inc until #1
+                $cur_stubnum++;
+            }
+        }
+        if ($in_original) {
+            # no more processing
+            next;
+        }
+        if ($in_stub_summary) {
+            if ($l =~ /^\s*\#/) {
+                if ($l =~ /\#([0-9]+): target = 0x([0-9a-fA-F]+) \(F\#([0-9-]+)\), count = ([0-9]+)(.*)/) {
+                    $stub_target{$tnum,$num_stubs[$tnum]} = $3;
+                    $cnt = $4;
+                } elsif ($l =~ /\#([0-9]+): target = 0x([0-9a-fA-F]+).*count = ([0-9]+)(.*)/) {
+                    $stub_target{$tnum,$num_stubs[$tnum]} = 0;
+                    $cnt = $3;
+                }
+                # count total direct & indirect
+                if ($prefix_stats) {
+                    $trace_total_exec += $cnt;
+                }
+                if ($2 eq "0") {
+                    $indirect_total += $cnt;
+                } else {
+                    $direct_total += $cnt;
+                }
+                $linked = $5;
+                if ($linked ne "") {
+                    if (!($linked =~ /linked/)) {
+                        die "Weird exit stub string: $linked\n";
+                    }
+                    $stub_linked{$tnum,$num_stubs[$tnum]} = 1;
+                } else {
+                    $stub_linked{$tnum,$num_stubs[$tnum]} = 0;
+                }
+                # old-style (before %U): 0 prior to real number
+                if ($cnt =~ /^0([0-9]+)/) {
+                    $cnt = $1;
+                }
+                if ($cnt > hex("0xffffffff")) {
+                    $stub_64bit = 1;
+                }
+            
+                if ($cnt > $max_stub) {
+                    $max_stub = $cnt;
+                }
+                $stub_count{$tnum,$num_stubs[$tnum]} = $cnt;
+                # set to -1 so can tell if gets set later
+                $stub_eflags{$tnum,$num_stubs[$tnum]} = -1;
+                $num_stubs[$tnum]++;
+            } else {
+                $in_stub_summary = 0;
+            }
+        }
+        if ($have_stubs) {
+            # find out if direct linked exit stub saves eflags or not
+            if ($exitstub_boundary > -1) {
+                if ($stub_linked{$tnum,$exitstub_boundary} &&
+                    $stub_target{$tnum,$exitstub_boundary} > -1) {
+                    # this is first instr of stub
+                    if ($l =~ /mov    \%eax -> /) {
+                        # saves eflags
+                        $stub_eflags{$tnum,$exitstub_boundary} = 1;
+                    } else {
+                        # does not save eflags
+                        $stub_eflags{$tnum,$exitstub_boundary} = 0;
+                    }
+                }
+                $exitstub_boundary = -1;
+            } elsif ($l =~ /-- exit stub ([0-9]+)/) {
+                $exitstub_boundary = $1;
+            }
+        }
+        if ($linked_to) {
+            if ($l =~ /^\s/) {
+                $l =~ /^\sFragment \# ([0-9]+)/;
+                if ($1 eq $id[$tnum]) {
+                    $trace{$tnum,$line++} = "\t  ==> Self-loop";
+                }
+            } else {
+                $linked_to = 0;
+            }
+        }
+        if ($body_start && $l =~ /^  (0x[0-9a-fA-F]+)/) {
+            $start_addr = hex($1);
+            $body_start = 0;
+        }
+        if ($l =~ /^Tag = (0x[0-9a-fA-F]+)/) {
+            $tag[$tnum] = $1;
+        } elsif ($l =~ /normal entry/) {
+            $body_start = 1;
+        } elsif ($l =~ /Flushed/) {
+            $flushed++;
+        } elsif ($l =~ /^Size = ([0-9]+)/) {
+            $size[$tnum] = $1;
+        } elsif ($l =~ /pre-opt count = ([0-9]+)/) {
+            $sideline_pre[$tnum] = $1;
+        } elsif ($l =~ /post-opt count = ([0-9]+)/) {
+            $sideline_during[$tnum] = $1;
+        } elsif ($l =~ /new trace count = ([0-9]+)/) {
+            $sideline_post[$tnum] = $1;
+        } elsif ($l =~ /^\stime/) {
+            $have_times = 1;
+            $l =~ /[^0-9]([0-9.]+)/;
+            $time[$tnum] = $1;
+            if ($count[$tnum] == 0) {
+                $per = 0.;
+            } else {
+                $per = 1000.0 * ($time[$tnum] / $count[$tnum]);
+            }
+            $trace{$tnum,$line++} = "\t  => us per execution: $per";
+        } elsif ($l =~ /^\scount/) {
+            $l =~ /[^0-9]([0-9]+)/;
+            $count[$tnum] = $1;
+        } elsif ($l =~/^Exit stubs/) {
+            $have_stubs = 1;
+            $in_stub_summary = 1;
+            $num_stubs[$tnum] = 0;
+        } elsif ($l =~/^Linked to/) {
+            $linked_to = 1;
+        } elsif ($l =~ /^  (0x[0-9a-fA-F]+)\s.+</ ||
+                 # workaround for bug where <exit stub N> is not printed:
+                 (!$stubs_start && $l =~ /^  (0x[0-9a-fA-F]+)\s.+j.+/)) {
+            # measure # of instr bytes from top until this exit from trace
+            # we assume all entires were to normal entry, not ibt entry
+            # we also do not count the # bytes of the exit instr itself
+            # neither of these assumptions is that egregious -- we're only
+            # using this estimate as a relative weight among different exits!
+            $addr = hex($1);
+            $stub_size{$tnum,$stub_num} = ($addr - $start_addr);
+            $stub_num++;
+        } elsif ($l =~ /-- exit stub 0/) {
+            $stubs_start = 1;
+        } elsif ($l =~ /^END TRACE/) {
+            # end of trace
+            $trace{$tnum,$line++} = "Total samples: $time[$tnum]";
+            if ($pcs && $pcs_by_orig_bb) {
+                $cur_stubnum++;
+                $trace{$tnum,$line++} = "BB samples breakdown:";
+                die "Error: trace $tnum tag $tag[$tnum]: $cur_stubnum stubs != $cur_bbnum bbs in trace $tnum"
+                    if ($cur_stubnum != $cur_bbnum);
+                if ($end_in_cbr[$tnum]) {
+                    # last jmp is fall-through, count as part of penultimate
+                    $cur_bbnum--;
+                    $bbsamples[$cur_bbnum-1] += $bbsamples[$cur_bbnum];
+                    $stubsamples[$cur_bbnum-1] += $stubsamples[$cur_bbnum];
+                }
+                for ($i=0; $i<$cur_bbnum; $i++) {
+                    $trace{$tnum,$line++} =
+                        sprintf("\tbb# %2d %-20s = %4d samples, %4d in %8s exit => %4d total",
+                                $i, $bbtag[$i], $bbsamples[$i], $stubsamples[$i],
+                                $bbib[$i] ? "indirect" : "direct",
+                                $bbsamples[$i] + $stubsamples[$i]);
+                    $bbtag[$i] = "";
+                    $bbib[$i] = 0;
+                    $bbsamples[$i] = 0;
+                    $stubsamples[$i] = 0;
+                }
+            }
+            $sortme[$tnum] = $tnum;
+            $length[$tnum++] = $line;
+            $in_trace = 0;
+            $line = 0;
+            $stub_num = 0;
+            $stubs_start = 0;
+        }
+    } else { # !in_trace
+        if ($l =~ /^Traces below dump threshold of (\d+): (\d+)/) {
+            $unseen_threshold = $1;
+            $unseen_num = $2;
+        } elsif ($l =~ /^Total count below dump threshold: (\d+)/) {
+            $unseen_count = $1;
+        }
+    }
+}
+close(IN);
+
+if ($prefix_stats) {
+    printf("Prefixes that restore eflags vs. those that do not:\n");
+    printf("Static restore:  %12d, do not: %12d, => %5.2f%% restore\n",
+           $prefix_eflags, $prefix_noeflags,
+           $prefix_eflags*100/($prefix_eflags+$prefix_noeflags));
+    printf("Dynamic restore: %12.0f, do not: %12.0f, => %5.2f%% restore\n",
+           $prefix_eflags_count, $prefix_noeflags_count,
+           $prefix_eflags_count*100/($prefix_eflags_count+$prefix_noeflags_count));
+    exit 0;
+}
+
+print "Total traces dumped: $tnum\n";
+print "\tTraces prematurely kicked out of cache: $flushed\n";
+print "Total traces not dumped: $unseen_num\n";
+$total_traces = $tnum + $unseen_num;
+print "=> Total traces: $total_traces\n\n";
+
+if ($pcs) {
+    print "Total PC samples: $total_samples\n";
+    if (!$linux_pcs) {
+        print "\tOverlapping instrs: $overcount_instrs\n";
+        print "\tOverlapping samples: $overcount_samples\n";
+    }
+    print "PC samples inside exit stubs: $stub_samples\n";
+    print "PC samples in indirect branch overhead: $ind_samples\n";
+    $tot = $stub_samples + $ind_samples;
+    $tot_frac = $tot / $total_samples * 100.;
+    printf("Total of previous 2 lines = %d = %3.1f%% of total samples\n",
+           $tot, $tot_frac);
+    print "PC samples on return instructions: $ret_samples\n";
+    print "PC samples on other indirect branches: $indjc_samples\n";
+    $tot = $ret_samples + $indjc_samples;
+    if ($tot == 0) {
+        $tot_frac = 0;
+    } else {
+        $tot_frac = $ret_samples / $tot * 100.;
+    }
+    printf("=> %% indirect branches that are returns: %3.1f%%\n", $tot_frac);
+    print "PC samples on direct branches: $branch_samples\n";
+    if ($branch_samples + $tot == 0) {
+        $tot_frac = 0;
+    } else {
+        $tot_frac = $tot / ($branch_samples + $tot) * 100.;
+    }
+    printf("=> %% branches that are indirect: %3.1f%%\n", $tot_frac);
+    print "\n";
+}
+
+# if have both stubs and times, sort by stubs
+if ($have_stubs) {
+    if (!$groups) {
+        print "Max stub count is $max_stub\n";
+        if ($stub_64bit) {
+            print "Needed 64-bit stub counters!\n";
+        }
+        $grand_total = $direct_total + $indirect_total;
+        print "Indirect=$indirect_total, direct=$direct_total, total=$grand_total\n";
+        $direct_frac = 100. * ($direct_total / $grand_total);
+        $indirect_frac = 100. * ($indirect_total / $grand_total);
+        printf("\t=> %4.1f%% direct, %4.1f%% indirect\n",
+               $direct_frac, $indirect_frac);
+
+        if ($unseen_count > 0) {
+            $grand_grand_total = $grand_total + $unseen_count;
+            print "\nUnseen traces below $unseen_threshold threshold: $unseen_num traces\n";
+            $unseen_cnt_frac = 100. * $unseen_count / $grand_grand_total;
+            printf("\ttotal count=$unseen_count (%4.1f%%) => grand total=$grand_grand_total\n\n",
+            $unseen_cnt_frac);
+        }
+
+        # eflags saving stats
+        $eflags_save_count = 0;
+        $eflags_save_stubs = 0;
+        $eflags_nosave_count = 0;
+        $eflags_nosave_stubs = 0;
+        for ($t=0; $t<$tnum; $t++) {
+            for ($e=0; $e<$num_stubs[$t]; $e++) {
+                # only look at linked direct branches
+                if ($stub_linked{$t,$e} && $stub_target{$t,$e} > -1) {
+                    if ($stub_eflags{$t,$e} == -1) {
+                        print "Error: eflags not set for trace $id[$t] exit $e\n";
+                    } elsif ($stub_eflags{$t,$e}) {
+                        $eflags_save_stubs++;
+                        $eflags_save_count += $stub_count{$t,$e};
+                    } else {
+                        $eflags_nosave_stubs++;
+                        $eflags_nosave_count += $stub_count{$t,$e};
+                    }
+                }
+            }
+        }
+        if ($eflags_save_stubs + $eflags_nosave_stubs > 0) {
+            $eflags_stub_frac = 100. * $eflags_save_stubs /
+                ($eflags_save_stubs + $eflags_nosave_stubs);
+            print "Eflags stubs: save=$eflags_save_stubs";
+            printf(" (%4.1f%%), nosave=$eflags_nosave_stubs\n",
+                   $eflags_stub_frac);
+            $eflags_count_frac = 100. * $eflags_save_count /
+                ($eflags_save_count + $eflags_nosave_count);
+            print "Eflags counts: save=$eflags_save_count";
+            printf(" (%4.1f%%), nosave=$eflags_nosave_count\n",
+                   $eflags_count_frac);
+        } else {
+            print "Error obtaining eflags stubs stats\n";
+        }
+
+        print "\n";
+    }
+    print "----------------------------------------------\n\n";
+    # estimate times
+    $total_estimate = 0;
+    if ($unseen_count > 0) {
+        # FIXME: can we do better than this average number of instrs
+        # in a trace, *3/4 since half exit at halfway point?
+        # ave # instrs across spec & win32 is 29
+        $unseen_size = 29 * 0.75;
+        $unseen_estimate = $unseen_count / 1000000. * $unseen_size;
+        $total_estimate += $unseen_estimate;
+    }
+    for ($t=0; $t<$tnum; $t++) {
+        # estimate time spent inside
+        $estimate[$t] = 0.;
+        $entry_count[$t] = 0.;
+        for ($e=0; $e<$num_stubs[$t]; $e++) {
+            # scale down to avoid overflow and for easier visual comparisons
+            $cnt = $stub_count{$t,$e} / 1000000.;
+            if (! defined($stub_size{$t,$e})) {
+                die "Error: size of trace $id[$t]'s stub $e not defined\n";
+            }
+            $entry_count[$t] += $cnt;
+            $estimate[$t] += $cnt * $stub_size{$t,$e};
+        }
+        $total_estimate += $estimate[$t];
+    }
+    for ($t=0; $t<$tnum; $t++) {
+        $estimate_frac[$t] = 100. * $estimate[$t] / $total_estimate;
+    }
+    if ($unseen_count > 0) {
+        $unseen_frac = 100. * $unseen_estimate / $total_estimate;
+        print "Unseen traces, time based only on general averages:\n";
+        printf("%6d below %8d threshold traces: count %7.2f M, time %9.2f (%4.1f%%)\n\n",
+               $unseen_num, $unseen_threshold,
+               $unseen_count/1000000., $unseen_estimate, $unseen_frac);
+        print "Dumped traces:\n";
+    }
+    @sortme = sort({$estimate[$a] <=> $estimate[$b]} @sortme);
+}
+elsif ($have_times) {
+    @sortme = sort({$time[$a] <=> $time[$b]} @sortme);
+
+    # get total time
+    $total_time = 0;
+    for ($t=0; $t<$tnum; $t++) {
+        $total_time += $time[$t];
+    }
+}
+elsif ($pcs) {
+    @sortme = sort({$time[$a] <=> $time[$b]} @sortme);
+}
+else {
+    # leave in original order == reverse order
+    @sortme = sort({$b <=> $a} @sortme);
+}
+
+# now output in sorted order
+if ($stats) {
+    $max_bbs = 0;
+    $ave_bbs = 0;
+    $max_bbs_noelide = 0;
+    $ave_bbs_noelide = 0;
+    $max_ibs = 0;
+    $ave_ibs = 0;
+    $max_ins = 0;
+    $ave_ins = 0;
+    $max_bytes = 0;
+    $ave_bytes = 0;
+    $ave_comp_all = 0;
+    $ave_comp_half = 0;
+    $max10_bbs = 0;
+    $ave10_bbs = 0;
+    $max10_bbs_noelide = 0;
+    $ave10_bbs_noelide = 0;
+    $max10_ibs = 0;
+    $ave10_ibs = 0;
+    $max10_ins = 0;
+    $ave10_ins = 0;
+    $max10_bytes = 0;
+    $ave10_bytes = 0;
+    $ave10_comp_all = 0;
+    $ave10_comp_half = 0;
+    $coverage5 = 0;
+    $coverage10 = 0;
+    $coverage50 = 0;
+}
+for ($t=$tnum-1; $t>=0; $t--) {
+    $n = $sortme[$t];
+
+    if (($brief && !$pcs_by_orig_bb) || $stats) {
+        if (!$groups && !$stats) {
+            # if have both stubs and times, just use stubs
+            if ($have_stubs) {
+                $s = sprintf(", count %7.2f M, time %9.2f (%4.1f%%)",
+                             $entry_count[$n], $estimate[$n], $estimate_frac[$n]);
+                $frac = $estimate_frac[$n];
+            } elsif ($have_times) {
+                $frac = 100 * $time[$n] / $total_time;
+                my $cnt = $count[$n] / 1000000;
+                $s = sprintf(", count %7.2f M, time %9.2f ms (%4.1f%%)",
+                             $cnt, $time[$n], $frac);
+            } elsif ($pcs) {
+                $frac = 100 * $time[$n] / $total_samples;
+                $s = sprintf(", samples %8d (%4.1f%%)",
+                             $time[$n], $frac);
+            } else {
+                $s = "";
+            }
+            if ($sideline) {
+                if ($sideline_pre[$n] == -1) {
+                    printf("Frag \# %5d ($tag[$n]) (%4.1f%%) = not sideline-optimized\n",
+                           $id[$n], $frac);
+                } else {
+                    $post_frac = 100. * $sideline_post[$n] /
+                        ($sideline_pre[$n] + $sideline_during[$n] +
+                         $sideline_post[$n]);
+                    printf("Frag \# %5d ($tag[$n]) (%4.1f%%) = pre %6d, dur %6d, post %8d (%5.1f%%)\n",
+                           $id[$n], $frac, $sideline_pre[$n], $sideline_during[$n],
+                           $sideline_post[$n], $post_frac);
+                }
+            } else {
+                printf("Frag \# %5d ($tag[$n]) = size %5d$s\n",
+                       $id[$n], $size[$n]);
+            }
+        }
+
+        if (($links || $stats) && $have_stubs) {
+            $link_total = 0;
+            for ($e=0; $e<$num_stubs[$n]; $e++) {
+                $link_total += $stub_count{$n,$e};
+            }
+            if ($link_total == 0) {
+                for ($e=0; $e<$num_stubs[$n]; $e++) {
+                    $stub_percent{$n,$e} = 0;
+                }
+                next;
+            }
+            if ($stats) {
+                # trying to get % of time completed trace (went all the way to
+                # end bb) and % of time got halfway through trace
+                if ($end_in_cbr[$n]) {
+                    # for purposes of completion, final 2 exits are identical
+                    # (just different directions of final cbr)
+                    $end_stub = $num_stubs[$n] - 2; # index of penultimate
+                } else {
+                    $end_stub = $num_stubs[$n] - 1; # index of final
+                }
+            }
+            for ($e=0; $e<$num_stubs[$n]; $e++) {
+                $stub_frac = 100. * ($stub_count{$n,$e} / $link_total);
+                if (!$groups && !$stats) {
+                    if ($stub_frac > 1.0) {
+                        printf("\tStub \#%2d: %4.1f%% => %s\n",
+                               $e, $stub_frac, $stub_target{$n,$e});
+                    }
+                }
+                # store for use in finding groups
+                $stub_percent{$n,$e} = $stub_frac;
+                if ($stats && $e < $end_stub/2) {
+                    # add up through halfway point
+                    $completion_half[$n] += $stub_frac;
+                }
+            }
+            # store for use in finding groups
+            $stub_total[$n] = $link_total;
+            if ($stats) {
+                $completion_all[$n] = $stub_percent{$n,$num_stubs[$n]-1};
+                $completion_half[$n] = 100. - $completion_half[$n];
+            }
+        }
+        next if (!$stats);
+    }
+
+    if ($stats) {
+        printf("Frag \# %5d = %2d/%2d bbs, %3d ibs, %5d ins, %5d bytes; %5.1f%% all, %5.1f%% half\n",
+               
+               $id[$n], $bbs[$n], $bbs_noelide[$n], $ind_branches[$n], $bb_instrs[$n], $bb_bytes[$n],
+               $completion_all[$n], $completion_half[$n]);
+        if ($stats) {
+            $max_bbs = $bbs[$n] if ($bbs[$n] > $max_bbs);
+            $ave_bbs += $bbs[$n];
+            $max_bbs_noelide = $bbs_noelide[$n] if ($bbs_noelide[$n] > $max_bbs_noelide);
+            $ave_bbs_noelide += $bbs_noelide[$n];
+            $max_ibs = $ind_branches[$n] if ($ind_branches[$n] > $max_ibs);
+            $ave_ibs += $ind_branches[$n];
+            $max_ins = $bb_instrs[$n] if ($bb_instrs[$n] > $max_ins);
+            $ave_ins += $bb_instrs[$n];
+            $max_bytes = $bb_bytes[$n] if ($bb_bytes[$n] > $max_bytes);
+            $ave_bytes += $bb_bytes[$n];
+            $ave_comp_all += $completion_all[$n];
+            $ave_comp_half += $completion_half[$n];
+            if ($t >= $tnum-10) {
+                # stats for top 10 traces
+                $max10_bbs = $bbs[$n] if ($bbs[$n] > $max10_bbs);
+                $ave10_bbs += $bbs[$n];
+                $max10_bbs_noelide = $bbs_noelide[$n] if ($bbs_noelide[$n] > $max10_bbs_noelide);
+                $ave10_bbs_noelide += $bbs_noelide[$n];
+                $max10_ibs = $ind_branches[$n] if ($ind_branches[$n] > $max10_ibs);
+                $ave10_ibs += $ind_branches[$n];
+                $max10_ins = $bb_instrs[$n] if ($bb_instrs[$n] > $max10_ins);
+                $ave10_ins += $bb_instrs[$n];
+                $max10_bytes = $bb_bytes[$n] if ($bb_bytes[$n] > $max10_bytes);
+                $ave10_bytes += $bb_bytes[$n];
+                $ave10_comp_all += $completion_all[$n];
+                $ave10_comp_half += $completion_half[$n];
+                $coverage10 += $estimate_frac[$n];
+            }
+            if ($t >= $tnum-5) {
+                # stats for top 5 traces
+                $coverage5 += $estimate_frac[$n];
+            }
+            if ($t >= $tnum-50) {
+                # stats for top 50 traces
+                $coverage50 += $estimate_frac[$n];
+            }
+        }
+        next;
+    }
+
+    print "===========================================================================\n\n";
+    $in_origins = 0;
+    $last_srcline = "";
+    for ($o=0; $o<$length[$n]; $o++) {
+        $l = $trace{$n,$o};
+
+        if ($in_origins) {
+            if ($src && $l =~ /^\s*(0x[0-9A-Fa-f]+)/) {
+                $addr = $1;
+                $instr = substr($l, 36);
+                $srcline = lookup_line($exe, $addr);
+                if ($srcline =~ /\?\?\(\) @ \?\?:0/) {
+                    $i = 0;
+                    while ($i < $shared_num) {
+                        if (hex($addr) >= $shared_start[$i] &&
+                            ($i == $shared_num-1 ||
+                             hex($addr) < $shared_start[$i+1])) {
+                            $srcline = "[somewhere in $shared_name[$i]]";
+                        }
+                        $i++;
+                    }
+                    if ($srcline eq $last_srcline) {
+                        $srcline = "";
+                    } else {
+                        $last_srcline = $srcline;
+                    }
+                }
+                ## get actual code
+                if ($srcline =~ /@ ([\w.]+):(\d+)/) {
+                    $file = $1;
+                    $lineno = $2;
+                    if ($srcline eq $last_srcline) {
+                        $srcline = "";
+                    } else {
+                        $last_srcline = $srcline;
+                        $actual = `sed -n '$lineno p' $file`;
+                        chomp $actual;
+                        ## strip leading WS: ($actual) = ($actual =~ /\s*(.*)$/);
+                        $srcline = sprintf("[%s]%s", $srcline, $actual);
+                    }
+                }
+                ## print sprintf("%-35s %s\n", $instr, $srcline);
+                if ($srcline ne "") {
+                    print "$srcline\n";
+                }
+            }
+            if ($l =~ /^END ORIGINAL CODE/) {
+                $in_origins = 0;
+            }
+        }       
+        print "$l\n";
+        if ($have_times && $o==1) {
+            $frac = 100 * $time[$n] / $total_time;
+            printf("Time Recorded: %9.2f ms (%4.1f%%)\n", $time[$n], $frac);
+        }
+        if ($have_stubs && $o==1) {
+            printf("Time Estimate: %9.2f (%4.1f%%)\n",
+                   $estimate[$n], $estimate_frac[$n]);
+            for ($e=0; $e<$num_stubs[$n]; $e++) {
+                print "\tStub \#$e: size $stub_size{$n,$e}\n";
+            }
+        }
+        if ($l =~ /^ORIGINAL CODE/ && $have_exe) {
+            $in_origins = 1;
+        }
+    }
+    print "\n";
+}
+
+if ($stats) {
+    printf("Max all      = %3d bbs, %3d ibs, %5d ins, %5d bytes\n",            
+           $max_bbs, $max_ibs, $max_ins, $max_bytes);
+    printf("Max top 10   = %3d bbs, %3d ibs, %5d ins, %5d bytes\n",            
+           $max10_bbs, $max10_ibs, $max10_ins, $max10_bytes);
+    printf("Ave all    = %4.1f bbs, %4.1f ibs, %5d ins, %5d bytes; %5.1f%% all, %5.1f%% half\n",
+           $ave_bbs/$tnum, $ave_ibs/$tnum, $ave_ins/$tnum, $ave_bytes/$tnum,
+           $ave_comp_all/$tnum, $ave_comp_half/$tnum);
+    printf("Ave top 10 = %4.1f bbs, %4.1f ibs, %5d ins, %5d bytes; %5.1f%% all, %5.1f%% half\n",
+           $ave10_bbs/10, $ave10_ibs/10, $ave10_ins/10, $ave10_bytes/10,
+           $ave10_comp_all/10, $ave10_comp_half/10);
+    printf("No elide bbs: all: %3d max, %4.1f ave; top10: %3d max, %4.1f ave\n",
+           $max_bbs_noelide, $ave_bbs_noelide/$tnum, $max10_bbs_noelide, $ave10_bbs_noelide/10);
+    printf("Coverage: top 5 = %5.1f%%, top 10 = %5.1f%%, top 50 = %5.1f%%\n",
+           $coverage5, $coverage10, $coverage50);
+    exit;
+}
+
+# Find groups
+# My method: go through traces in sorted order
+# For each trace: recursively walk exits w/ exit freq >= 10%
+#   If hit an already-examined trace, stop
+#   Add est time % to "group %" unless trace is already part of another group
+#   (this way total group %'s will add to 100, and by going through in
+#    sorted order this shouldn't cause us to miss larger groups in
+#    favor of smaller)
+if ($groups && $have_stubs) {
+    @todo = {};
+    @tabs = {};
+    @frac = {};
+    for ($t=$0; $t<$tnum; $t++) {
+        $n = $sortme[$t];
+        push(@todo, $n);
+        push(@tabs, 0);
+        push(@frac, -1);
+        $group_done[$t] = 0;
+    }
+    $gnum = 0;
+    $group_percent[0] = 0;
+    $group_out[0] = "";
+    $in_group = 0;
+    while ($#todo > 0) {
+        $t = pop(@todo);
+        $indent = pop(@tabs);
+        $wt = pop(@frac);
+        if ($indent == 0) {
+            if ($in_group) {
+                $group_out[$gnum] .= sprintf("\tTotal for group = %4.1f%%\n\n",
+                                             $group_percent[$gnum]);
+                $sort_group[$gnum] = $gnum;
+                $gnum++;
+                $group_percent[$gnum] = 0;
+                for ($n=$0; $n<$tnum; $n++) {
+                    $group_done[$n] = 0;
+                }
+            }
+            if ($done[$t]) {
+                $in_group = 0;
+                next;
+            } else {
+                $in_group = 1;
+                $group_top[$gnum] = $t;
+            }
+        }
+        # yes group_done is not needed, I'm leaving it in so I can go
+        # back to "double counting" that may find bigger/better groups
+        # (but I doubt it)
+        if ($in_group && $t != -1 && !$done[$t] && !$group_done[$t]) {
+            $group_percent[$gnum] += $estimate_frac[$t];
+            $group_done[$t] = 1;
+        }
+        for ($i=0; $i<$indent; $i++) {
+            $group_out[$gnum] .= "\t";
+        }
+        if ($t == -1) {
+            $name = sprintf("%-6s", "*");
+        } else {
+            $name = sprintf("%-6d", $id[$t]);
+        }
+        $group_out[$gnum] .= "$name";
+        if ($indent > 0) {
+            $group_out[$gnum] .= sprintf("(%4.1f%%)", $wt);
+        }
+        if ($t == -1) {
+            $group_out[$gnum] .= "\n";
+            next;
+        }
+        if ($done[$t]) {
+            $out = sprintf("  [%9.2f (%4.1f%%)] +",
+                           $estimate[$t], $estimate_frac[$t]);
+            if ($group_num[$t] != $gnum) {
+                if ($t == $group_top[$group_num[$t]]) {
+                    $out .= " ==> top of group $group_num[$t]";
+                } else {
+                    $out .= " ==> middle of group $group_num[$t]";
+                }
+            } else {
+                $out .= " ==> loop";
+            }
+        } else {
+            $out = sprintf("  [%9.2f (%4.1f%%)]",
+                           $estimate[$t], $estimate_frac[$t]);
+            if ($t > -1) {
+                # gather exits above the cutoff to be sorted
+                $num_sort_stub = 0;
+                undef @sort_stub;
+                for ($e=0; $e<$num_stubs[$t]; $e++) {
+                    # cutoff is 10%!
+                    if ($stub_percent{$t,$e} > 9.999) {
+                        $sort_stub[$num_sort_stub++] = $e;
+                    }
+                }
+                @sort_stub = sort( { $stub_percent{$t,$a} <=> $stub_percent{$t,$b} }
+                                   @sort_stub );
+                # now push exits, in reverse sorted order so do highest first
+                for ($i=0; $i<$num_sort_stub; $i++) {
+                    $e = $sort_stub[$i];
+                    if ($stub_target{$t,$e} == -1) {
+                        $n = -1;
+                    } else {
+                        $n = $tnum_from_id{$stub_target{$t,$e}};
+                    }
+                    push(@todo, $n);
+                    push(@tabs, ($indent+1));
+                    push(@frac, $stub_percent{$t,$e});
+                }
+            }
+            $done[$t] = 1;
+            $group_num[$t] = $gnum;
+        }
+        $group_out[$gnum] .= "$out\n";
+    }
+    @sort_group = sort({$group_percent[$a] <=> $group_percent[$b]} @sort_group);
+    for ($g=$gnum-1; $g>=0; $g--) {
+        $gp = $sort_group[$g];
+        print "Group $gp:\n$group_out[$gp]";
+    }
+}
+
+
+sub compare_numbers() {
+    $a <=> $b;
+# $a <=> $b is equivalent to:
+# if ($a<$b){return -1;} elsif ($a>$b){return 1;} else{return 0;}
+}
+
+sub lookup_line($, $) {
+    my ($exe, $addr) = @_;
+    my ($i, $func, $loc);
+    open(FOO, "addr2line -e $exe -f $addr |") ||
+        die "Error running addr2line\n";
+    $i = 0;
+    while (<FOO>) {
+        chop;
+        if ($i == 0) {
+            $func = $_;
+        } else {
+            $loc = $_;
+        }
+        $i++;
+    }
+    close(FOO);
+    $loc =~ s/\/.+\///;
+    $res = "$func() @ $loc";
+    return $res;
+}
+
+sub is_save_ecx($) {
+    my ($l) = @_;
+    # save ecx can either be a move to dcontext as absolute
+    # address, or a move to a tls slot in fs:
+    return (($l =~ /mov\s+\%ecx -> (0x[0-9a-fA-F]*)/ &&
+             hex($1) == $dcontext_ecx) ||
+            ($l =~ /mov\s+\%ecx -> \%fs:0x[0-9a-fA-F]*/));
+}
+
+sub is_restore_ecx($) {
+    my ($l) = @_;
+    # save ecx can either be a move to dcontext as absolute
+    # address, or a move to a tls slot in fs:
+    return (($l =~ /mov\s+(0x[0-9a-fA-F]*) -> \%ecx/ &&
+             hex($1) == $dcontext_ecx) ||
+            ($l =~ /mov\s+\%fs:0x[0-9a-fA-F]* -> \%ecx/));
+}
+
+sub is_exit_cti($) {
+    my ($l) = @_;
+    # 32-bit jmps only to exclude jmp part of jecxz mangle
+    return ($l =~ /(e9|(0f 8[0-9a-fA-F])) ([0-9a-fA-F]+ ){4}\s*j/);
+}
+
+### OBSOLETE CODE
+### This code ran objdump, read in resulting assembly code, and then
+### used it to print out disassembly of trace:
+###   
+###   if ($have_exe && $dis) {
+###       # disassemble exe, read in assembly code
+###       system("objdump -d $exe > $exe.dis");
+###       system("echo \"\" >> $exe.dis"); # blank line to get end of last func!
+###       $i = 0;
+###       $in_func = 0;
+###       open(OBJ, "< $exe.dis") || die "Error: Couldn't open $exe.dis for input\n";
+###       while (<OBJ>) {
+###     chop;
+###     $l = $_;
+###     if ($in_func) {
+###         if ($l =~ /^\s*([0-9A-Fa-f]+)/) {
+###             $addr{$faddr,$i} = hex($1);
+###             $instr{$faddr,$i} = $l;
+###             $num_instrs{$faddr}++;
+###             $i++;
+###         } else {
+###             $last{$faddr} = $addr{$faddr,($i-1)};
+###             $in_func = 0;
+###             $i = 0;
+###             # print "Function $func{$faddr} = $first{$faddr}...$last{$faddr}\n";
+###         }
+###     }
+###     if ($l =~ /^([0-9A-Fa-f]+) <([^>]+)>/) {
+###         # can have multiple func names (if static) so key off of
+###         #   function address, not func name!
+###         $faddr = hex($1);
+###         $func{$faddr} = $2;
+###         $first{$faddr} = $faddr;
+###         $in_func = 1;
+###     }
+###       }
+###       close(OBJ);
+###       system("rm -f $exe.dis");
+###   }
+###   
+###             $l =~ /^\s(0x[0-9A-Fa-f]+)\.\.\.(0x[0-9A-Fa-f]+)/;
+###             $start = $1;
+###             $end = $2;
+###             if ($src && 0) {
+###                 $start_line = lookup_line($exe, $start);
+###                 $end_line = lookup_line($exe, $end);
+###                 print "\t  == $start_line ... $end_line\n";
+###             }
+###             if ($dis) {
+###                 $start =~ s/0x//;
+###                 $end =~ s/0x//;
+###                 $found = lookup_dis($start, $end);
+###                 if (!$found) {
+###                     print "Couldn't find $start...$end in $exe\n";
+###                 }
+###                 print "\n";
+###             }
+###   
+###   sub lookup_dis($, $) {
+###       my ($start, $end) = @_;
+###       my ($f, $i);
+###       $num_start = hex($start);
+###       $num_end = hex($end);
+###       # start & end guaranteed to be in same function
+###       # just do linear search, performance not an issue
+###       foreach $f (keys %first) {
+###     if ($num_start >= $first{$f} &&
+###         $num_end <= $last{$f}) {
+###         print "$start...$end is in $func{$f}:\n";
+###         $between = 0;
+###         for ($i=0; $i<$num_instrs{$f}; $i++) {
+###             if ($num_start == $addr{$f,$i}) {
+###                 $between = 1;
+###             }
+###             # end addr is past the last addr
+###             if ($num_end == $addr{$f,$i}) {
+###                 $between = 0;
+###             }
+###             if ($between) {
+###                 print "$instr{$f,$i}\n";
+###             }
+###         }
+###         return 1;
+###     }
+###       }
+###       return 0;
+###   }
+###   
+### END OBSOLETE CODE
+
diff --git a/tools/turn_on_all_services.sh b/tools/turn_on_all_services.sh
new file mode 100755
index 000000000..12cf9c10f
--- /dev/null
+++ b/tools/turn_on_all_services.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+
+# **********************************************************
+# Copyright (c) 2005 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+# try to start every service and set it to auto-start
+
+old_IFS="${IFS}"
+# handle spaces in names
+IFS=$'\n'
+for i in `sc query state= all | grep SERVICE_NAME | cut -c 15-`; do 
+    echo ""; echo $i
+    start=`sc qc "$i" | grep START_TYPE | awk '{print $NF}'`
+    if [ $start = "AUTO_START" ]; then
+        echo "$i already set to auto-start"
+    else
+        # will refuse to start if currently disabled so we set to auto first
+        sc config "$i" start= auto > /dev/null
+    fi
+    state=`sc query "$i" | grep STATE | awk '{print $4}'`
+    echo state is $state
+    if [ "$state" != "RUNNING" ]; then
+        echo "trying to start $i"
+        sc start "$i" > /dev/null
+        # no exit code here -- have to wait and query status
+        sleep 3
+        state=`sc query "$i" | grep STATE | awk '{print $4}'`
+        if [ "$state" != "RUNNING" ]; then
+            if [ "$start" = "DISABLED" ]; then
+                echo "unable to start $i -- disabling"
+                sc config "$i" start= disabled > /dev/null
+            else
+                echo "unable to start $i -- setting to manual"
+                sc config "$i" start= demand > /dev/null
+            fi
+        else
+            echo "successfully started $i and set to auto"
+        fi
+    fi
+done
+IFS="${old_IFS}"
diff --git a/tools/vista_prelim_cfg.drcontrol.cfg b/tools/vista_prelim_cfg.drcontrol.cfg
new file mode 100755
index 000000000..ee0bc6389
--- /dev/null
+++ b/tools/vista_prelim_cfg.drcontrol.cfg
@@ -0,0 +1,394 @@
+POLICY_VERSION=30000
+
+BEGIN_BLOCK
+
+GLOBAL
+
+DYNAMORIO_OPTIONS=
+
+DYNAMORIO_RUNUNDER=1
+
+DYNAMORIO_UNSUPPORTED=
+
+DYNAMORIO_AUTOINJECT=\dynamorio.dll
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=alg.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=araktest.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=calc.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=cisvc.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=detertray.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=dwm.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=explorer.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=fxssvc.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=inetinfo.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=locator.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=logonui.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=lsass.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=lsm.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=mqsvc.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=mqtgsvc.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=msascui.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=msdtc.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=msiexec.exe
+
+DYNAMORIO_RUNUNDER=48
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=msiexec.exe-v
+
+DYNAMORIO_RUNUNDER=49
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=mysqld-nt.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=nmupgrade.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=nodemgr.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=openssl.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=searchindexer.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=services.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=slsvc.exe
+
+DYNAMORIO_OPTIONS=-exempt_rct_to_list "slsvc.exe" -exempt_rct_from_list "slsvc.exe"
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=smsvchost.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=snmp.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=snmptrap.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=spoolsv.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=svchost.exe
+
+DYNAMORIO_RUNUNDER=16
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=svchost.exe-dcomlaunch
+
+DYNAMORIO_RUNUNDER=17
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=svchost.exe-iissvcs
+
+DYNAMORIO_RUNUNDER=17
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=svchost.exe-ipripsvc
+
+DYNAMORIO_RUNUNDER=17
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=svchost.exe-localservice
+
+DYNAMORIO_RUNUNDER=17
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=svchost.exe-localservicenetworkrestricted
+
+DYNAMORIO_RUNUNDER=17
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=svchost.exe-localservicenonetwork
+
+DYNAMORIO_RUNUNDER=17
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=svchost.exe-localsystemnetworkrestricted
+
+DYNAMORIO_RUNUNDER=17
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=svchost.exe-lpdservice
+
+DYNAMORIO_RUNUNDER=17
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=svchost.exe-netsvcs
+
+DYNAMORIO_RUNUNDER=17
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=svchost.exe-networkservice
+
+DYNAMORIO_RUNUNDER=17
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=svchost.exe-networkservicenetworkrestricted
+
+DYNAMORIO_RUNUNDER=17
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=svchost.exe-rpcss
+
+DYNAMORIO_RUNUNDER=17
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=svchost.exe-secsvcs
+
+DYNAMORIO_RUNUNDER=17
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=svchost.exe-wersvcgroup
+
+DYNAMORIO_RUNUNDER=17
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=tcpsvcs.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=tlntsvr.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=tomcat.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=vssvc.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=w3wp.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=wininit.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=winlogon.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=wmiapsrv.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=wmiprvse.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=wmpnetwk.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=wmpnscfg.exe
+
+END_BLOCK
+
+BEGIN_BLOCK
+
+APP_NAME=wuauclt.exe
+
+END_BLOCK
+
diff --git a/tools/vista_setup.notes b/tools/vista_setup.notes
new file mode 100755
index 000000000..f68ff2675
--- /dev/null
+++ b/tools/vista_setup.notes
@@ -0,0 +1,31 @@
+to set up on vista
+
+copy over the following files
+drpreinject.dll
+dynamorio.dll
+drcontrol.exe
+drview.exe
+vista_prelim_cfg.drcontrol.cfg
+
+Be sure the folder(s) containing dynamorio.dll and drpreinject.dll are readable by all users (i.e. readable by everyone) and the that log folder specified below is writable by everyone.
+
+open a runas admin cmd shell (not the same as running from an admin account;
+as an admin user create a link to cmd.exe, right click and select runas admin)
+
+in the cmd shell run (examples assume files copied to c:\dr\setup and are in [] )
+[cd c:\dr\setup]
+drcontrol -create <path to folder holding dynamorio.dll [ex. c:\dr\setup ] >
+drcontrol -load vista_prelim_cfg.drcontrol.cfg
+drcontrol -logdir <path to desired log directory [ex. c:\dr\logs ] >
+drcontrol -drlib <path to dynamorio.dll [ex. c:\dr\setup\dynamorio.dll ] >
+drcontrol -create_eventlog <path to dynamorio.dll [ex. c:\dr\setup\dynamorio.dll ] >
+drcontrol -preinject LOAD_ON
+drcontrol -preinject <path to drpreinject.dll [ex. c:\dr\setup\drpreinject.dll ] >
+
+use regedit or drcontrol to adjust options, processes under dr, hotpatch
+configuration, etc. as desired
+
+note that the prelim configuration uses the global options for all process
+except slsvc.exe
+
+note that we can be pretty slow in vista (esp. explorer.exe under mf, which also has the known bug 9717)
diff --git a/tools/win32injectall.c b/tools/win32injectall.c
new file mode 100644
index 000000000..73164854c
--- /dev/null
+++ b/tools/win32injectall.c
@@ -0,0 +1,124 @@
+/* **********************************************************
+ * Copyright (c) 2001-2003 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* build like this:
+ *   cl win32injectall.c advapi32.lib
+ */
+
+#include <windows.h>
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+/* This routine assumes an environment variable named
+ * DYNAMO_AUTO_INJECT is set and that it points to something like:
+ * c:\iye\rio\inject\win32dynamo.dll
+ */
+
+#define INJECT_ALL_HIVE    HKEY_LOCAL_MACHINE
+#define INJECT_ALL_KEY     "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Windows"
+#define INJECT_ALL_SUBKEY  "AppInit_DLLs"
+
+void
+print_error(int error)
+{
+    LPVOID lpMsgBuf;
+    FormatMessage( 
+		  FORMAT_MESSAGE_ALLOCATE_BUFFER | 
+		  FORMAT_MESSAGE_FROM_SYSTEM | 
+		  FORMAT_MESSAGE_IGNORE_INSERTS,
+		  NULL,
+		  error,
+		  MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language
+		  (LPTSTR) &lpMsgBuf,
+		  0,
+		  NULL 
+		  );
+    // Display the string.
+    fprintf(stderr, "%s\n", (LPCTSTR)lpMsgBuf);
+    // Free the buffer.
+    LocalFree( lpMsgBuf );
+}
+
+int
+main(int argc, char *argv[])
+{
+    int res;
+    HKEY hk;
+    char * val;
+    char data[1024];
+    unsigned long size = 0;
+
+    if (argc != 2) {
+	printf("Usage: %s set|unset|view\n", argv[0]);
+	return 0;
+    }
+
+    if (strcmp(argv[1], "view") == 0) {
+	res = RegOpenKeyEx(INJECT_ALL_HIVE, INJECT_ALL_KEY, 0, KEY_READ, &hk);
+	assert(res == ERROR_SUCCESS);
+	res = RegQueryValueEx(hk, INJECT_ALL_SUBKEY, 0, NULL, (LPBYTE) data,
+			      &size);
+	/* for some reason we get error code "More data is available" here
+	 * (doing exact same thing in win32gui always gets ERROR_SUCCESS)
+	 */
+	/* assert(res == ERROR_SUCCESS); */
+	RegCloseKey(hk);
+	// assumption: only we use this key, so if it is non-empty it must be us!
+	if (size > 1)
+	    printf("Inject all is set to %s\n", data);
+	else
+	    printf("Inject all is NOT set\n");
+    } else {
+	if (strcmp(argv[1], "set") == 0) {
+	    const char *subdir = "\\bin\\drpreinject.dll";
+	    size =
+		GetEnvironmentVariable((LPCTSTR)"DYNAMORIO_HOME", data, 1023 - strlen(subdir));
+	    strcpy(data + strlen(data), subdir);
+	    val = data;
+	    printf("Setting key to %s\n", val);
+	} else if (strcmp(argv[1], "unset") == 0) {
+	    val = "";
+	} else {
+	    printf("Usage: %s set|unset|view\n", argv[0]);
+	    return 0;
+	}
+	res = RegOpenKeyEx(INJECT_ALL_HIVE, INJECT_ALL_KEY, 0, KEY_WRITE, &hk);
+	assert(res == ERROR_SUCCESS);
+	res = RegSetValueEx(hk, INJECT_ALL_SUBKEY, 0, REG_SZ, (LPBYTE) val,
+			    strlen(val)+1);
+	assert(res == ERROR_SUCCESS);
+	RegCloseKey(hk);
+    }
+
+    return 0;
+}
diff --git a/tools/windbg-scripts/acl_thread b/tools/windbg-scripts/acl_thread
new file mode 100755
index 000000000..12277c8a6
--- /dev/null
+++ b/tools/windbg-scripts/acl_thread
@@ -0,0 +1,51 @@
+$$ **********************************************************
+$$ Copyright (c) 2006 VMware, Inc.  All rights reserved.
+$$ **********************************************************
+
+$$ Redistribution and use in source and binary forms, with or without
+$$ modification, are permitted provided that the following conditions are met:
+$$ 
+$$ * Redistributions of source code must retain the above copyright notice,
+$$   this list of conditions and the following disclaimer.
+$$ 
+$$ * Redistributions in binary form must reproduce the above copyright notice,
+$$   this list of conditions and the following disclaimer in the documentation
+$$   and/or other materials provided with the distribution.
+$$ 
+$$ * Neither the name of VMware, Inc. nor the names of its contributors may be
+$$   used to endorse or promote products derived from this software without
+$$   specific prior written permission.
+$$ 
+$$ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+$$ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+$$ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+$$ ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+$$ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+$$ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+$$ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+$$ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+$$ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+$$ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+$$ DAMAGE.
+
+$$ usage: set $t0 to a bitmask of access privileges
+$$ values are from winnt.h and http://msdn.microsoft.com/library/default.asp?url=/library/en-us/dllproc/base/thread_security_and_access_rights.asp
+
+? @$t0
+.if ((@$t0 & 0x0001)==0x0001) {.echo "THREAD_TERMINATE";};  
+.if ((@$t0 & 0x0002)==0x0002) {.echo "THREAD_SUSPEND_RESUME";};  
+.if ((@$t0 & 0x0008)==0x0008) {.echo "THREAD_GET_CONTEXT";};  
+.if ((@$t0 & 0x0010)==0x0010) {.echo "THREAD_SET_CONTEXT";};  
+.if ((@$t0 & 0x0020)==0x0020) {.echo "THREAD_SET_INFORMATION";};  
+.if ((@$t0 & 0x0040)==0x0040) {.echo "THREAD_QUERY_INFORMATION";};  
+.if ((@$t0 & 0x0080)==0x0080) {.echo "THREAD_SET_THREAD_TOKEN";};
+.if ((@$t0 & 0x0100)==0x0100) {.echo "THREAD_IMPERSONATE";};
+.if ((@$t0 & 0x0200)==0x0200) {.echo "THREAD_DIRECT_IMPERSONATION";};
+.if ((@$t0 & 0x0400)==0x0400) {.echo "THREAD_SET_LIMITED_INFORMATION";};
+.if ((@$t0 & 0x0800)==0x0800) {.echo "THREAD_QUERY_LIMITED_INFORMATION";};
+.if ((@$t0 & 0x1f03ff)==0x1f03ff) {.echo "(==THREAD_ALL_ACCESS)";};
+.if ((@$t0 & 0x00010000)==0x00010000) {.echo "DELETE";};
+.if ((@$t0 & 0x00020000)==0x00020000) {.echo "READ_CONTROL";};
+.if ((@$t0 & 0x00040000)==0x00040000) {.echo "WRITE_DAC";};
+.if ((@$t0 & 0x00080000)==0x00080000) {.echo "WRITE_OWNER";};
+.if ((@$t0 & 0x00100000)==0x00100000) {.echo "SYNCHRONIZE";};
diff --git a/tools/windbg-scripts/cache_fifo_walk b/tools/windbg-scripts/cache_fifo_walk
new file mode 100644
index 000000000..e68be147a
--- /dev/null
+++ b/tools/windbg-scripts/cache_fifo_walk
@@ -0,0 +1,62 @@
+$$ **********************************************************
+$$ Copyright (c) 2005 VMware, Inc.  All rights reserved.
+$$ **********************************************************
+
+$$ Redistribution and use in source and binary forms, with or without
+$$ modification, are permitted provided that the following conditions are met:
+$$ 
+$$ * Redistributions of source code must retain the above copyright notice,
+$$   this list of conditions and the following disclaimer.
+$$ 
+$$ * Redistributions in binary form must reproduce the above copyright notice,
+$$   this list of conditions and the following disclaimer in the documentation
+$$   and/or other materials provided with the distribution.
+$$ 
+$$ * Neither the name of VMware, Inc. nor the names of its contributors may be
+$$   used to endorse or promote products derived from this software without
+$$   specific prior written permission.
+$$ 
+$$ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+$$ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+$$ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+$$ ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+$$ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+$$ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+$$ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+$$ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+$$ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+$$ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+$$ DAMAGE.
+
+$$ walk fcache fifo list, with next_fcache at a different offset
+$$ for an EmptySlot vs a Fragment.
+
+$$ usage: store fifo value into t0:
+$$ r $t0 = 27ece6a8
+
+$$ field offsets valid as of fragment.h:1.163, fcache.c:1.165
+$$ note that this only applies to private fragments which now have
+$$ a different layout than shared (private_fragment_t), as shared do
+$$ not use a fifo list
+
+$$ must use pseudo-register, not something like u1, which doesn't work
+$$ due to literal textual expansion rather than value replacement.
+$$ @ makes it MUCH faster since won't try to look up as symbol.
+
+.for (; @$t0 != 0; ) {
+    .echo "---------------------------------------------"
+    .if ((poi(@$t0+4) & 020000) = 020000) {
+        .echo _EmptySlot; ? @$t0
+        .echo start_pc; ? poi(@$t0)
+        .echo flags; ? poi(@$t0+4)
+        .echo next_fcache; ? poi(@$t0+8)
+        .echo prev_fcache; ? poi(@$t0+c)
+        .echo fcache_size; ? poi(@$t0+10)
+        .echo fcache_owner; ? poi(@$t0+14)
+        r $t0 = poi(@$t0+8)
+    }
+    .else {
+        dt theFragment @$t0
+        r $t0 = poi(@$t0+24)
+    }
+}
diff --git a/tools/windbg-scripts/coarse_incoming b/tools/windbg-scripts/coarse_incoming
new file mode 100755
index 000000000..580c033dd
--- /dev/null
+++ b/tools/windbg-scripts/coarse_incoming
@@ -0,0 +1,50 @@
+$$ **********************************************************
+$$ Copyright (c) 2006 VMware, Inc.  All rights reserved.
+$$ **********************************************************
+
+$$ Redistribution and use in source and binary forms, with or without
+$$ modification, are permitted provided that the following conditions are met:
+$$ 
+$$ * Redistributions of source code must retain the above copyright notice,
+$$   this list of conditions and the following disclaimer.
+$$ 
+$$ * Redistributions in binary form must reproduce the above copyright notice,
+$$   this list of conditions and the following disclaimer in the documentation
+$$   and/or other materials provided with the distribution.
+$$ 
+$$ * Neither the name of VMware, Inc. nor the names of its contributors may be
+$$   used to endorse or promote products derived from this software without
+$$   specific prior written permission.
+$$ 
+$$ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+$$ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+$$ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+$$ ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+$$ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+$$ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+$$ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+$$ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+$$ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+$$ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+$$ DAMAGE.
+
+$$ list coarse unit incoming links, including expanding fine lists
+
+$$ usage: store unit's incoming field into t0:
+$$ r $t0 = 27ece6a8
+
+.for (r $t1 = @$t0; @$t1 != 0; ) {
+    ? @$t1;
+    dt -b dynamorio!_coarse_incoming_t @$t1;
+    .if (poi(@$t1+4)=0) {
+        .echo "expanding fine"; 
+        .for (r $t2 = poi(@$t1); @$t2 != 0; ) {
+            ? @$t2;
+            dt -b _common_direct_linkstub_t @$t2;
+            r $t2 = poi(@$t2+4); 
+        }
+        .echo "done expanding"; 
+        .echo ""
+    }
+    r $t1 = poi(@$t1+8);
+}
diff --git a/tools/windbg-scripts/coarse_lookup b/tools/windbg-scripts/coarse_lookup
new file mode 100755
index 000000000..4d92b6f32
--- /dev/null
+++ b/tools/windbg-scripts/coarse_lookup
@@ -0,0 +1,47 @@
+$$ **********************************************************
+$$ Copyright (c) 2006 VMware, Inc.  All rights reserved.
+$$ **********************************************************
+
+$$ Redistribution and use in source and binary forms, with or without
+$$ modification, are permitted provided that the following conditions are met:
+$$ 
+$$ * Redistributions of source code must retain the above copyright notice,
+$$   this list of conditions and the following disclaimer.
+$$ 
+$$ * Redistributions in binary form must reproduce the above copyright notice,
+$$   this list of conditions and the following disclaimer in the documentation
+$$   and/or other materials provided with the distribution.
+$$ 
+$$ * Neither the name of VMware, Inc. nor the names of its contributors may be
+$$   used to endorse or promote products derived from this software without
+$$   specific prior written permission.
+$$ 
+$$ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+$$ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+$$ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+$$ ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+$$ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+$$ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+$$ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+$$ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+$$ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+$$ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+$$ DAMAGE.
+
+$$ lookup a tag in a coarse htable, assuming it uses phi
+
+$$ usage: store tag into t0 and the _coarse_table_t pointer into t1
+
+r $t2 = @@(&((_coarse_table_t *)@$t1)->table[((@$t0 * 2654435769) >> (32 - ((_coarse_table_t *)@$t1)->hash_bits)) & ((_coarse_table_t *)@$t1)->hash_mask])
+$$ keep going until find match or null
+.for (; poi(@$t2) != 0; ) {
+    .if (poi(@$t2) = @$t0) {
+        .echo **** found tag in coarse htable; ? @$t2
+        dt _app_to_cache_t @$t2
+        .break
+    }
+    .else {
+        r $t2 = @$t2 + 8
+    }
+}
+.echo finished searching coarse htable; .echo;
diff --git a/tools/windbg-scripts/dump_all_trace_heads b/tools/windbg-scripts/dump_all_trace_heads
new file mode 100755
index 000000000..d2967a0c6
--- /dev/null
+++ b/tools/windbg-scripts/dump_all_trace_heads
@@ -0,0 +1,46 @@
+$$ **********************************************************
+$$ Copyright (c) 2005 VMware, Inc.  All rights reserved.
+$$ **********************************************************
+
+$$ Redistribution and use in source and binary forms, with or without
+$$ modification, are permitted provided that the following conditions are met:
+$$ 
+$$ * Redistributions of source code must retain the above copyright notice,
+$$   this list of conditions and the following disclaimer.
+$$ 
+$$ * Redistributions in binary form must reproduce the above copyright notice,
+$$   this list of conditions and the following disclaimer in the documentation
+$$   and/or other materials provided with the distribution.
+$$ 
+$$ * Neither the name of VMware, Inc. nor the names of its contributors may be
+$$   used to endorse or promote products derived from this software without
+$$   specific prior written permission.
+$$ 
+$$ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+$$ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+$$ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+$$ ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+$$ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+$$ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+$$ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+$$ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+$$ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+$$ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+$$ DAMAGE.
+
+$$ diassemble every single shared trace head
+$$ FIXME: would be nice to use struct fields instead of hardcoded
+$$ offsets -- but having trouble doing that in @@ (works in ??)
+
+r $t0 = @@(shared_bb.table)
+r $t1 = @@(&shared_bb.table[shared_bb.capacity])
+.for (; @$t0 < @$t1; ) {
+    r $t2=poi(@$t0)
+    .if ((poi(@$t2+4) & 8) == 8) {
+        ? @$t0
+        ? @$t2
+        dt theFragment @$t2
+        U poi(@$t2+8) poi(@$t2+8)+@@(*(unsigned short *)(@$t2+0xc))
+    }
+    r $t0 = @$t0 + 4
+}
diff --git a/tools/windbg-scripts/find_dr b/tools/windbg-scripts/find_dr
new file mode 100644
index 000000000..2f3dd18b7
--- /dev/null
+++ b/tools/windbg-scripts/find_dr
@@ -0,0 +1,41 @@
+$$ **********************************************************
+$$ Copyright (c) 2006 VMware, Inc.  All rights reserved.
+$$ **********************************************************
+
+$$ Redistribution and use in source and binary forms, with or without
+$$ modification, are permitted provided that the following conditions are met:
+$$ 
+$$ * Redistributions of source code must retain the above copyright notice,
+$$   this list of conditions and the following disclaimer.
+$$ 
+$$ * Redistributions in binary form must reproduce the above copyright notice,
+$$   this list of conditions and the following disclaimer in the documentation
+$$   and/or other materials provided with the distribution.
+$$ 
+$$ * Neither the name of VMware, Inc. nor the names of its contributors may be
+$$   used to endorse or promote products derived from this software without
+$$   specific prior written permission.
+$$ 
+$$ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+$$ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+$$ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+$$ ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+$$ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+$$ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+$$ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+$$ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+$$ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+$$ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+$$ DAMAGE.
+
+* FIXME: currently both are to be used as commands, 
+* e.g. finddr_slow or finddr_fast, preferrably should be arguments to .reload
+
+* use for analyzing processes too early or too late for following drmarker with finddr_fast
+as finddr_slow .foreach ( base {s -[1]a 0 L?80000000 ", build" } ) { !vprot base }
+
+* follow drmarker from ntdll!KiUserCallbackDispatcher
+* 0:000> dt theDrMarker dr_base_addr
+*   +0x020 dr_base_addr : Ptr32 Void
+* follow jmp to hook func, 
+as ? finddr_fast poi((( poi(ntdll!KiUserCallbackDispatcher+1) + ntdll!KiUserCallbackDispatcher+5) & fffff000) + 0x20)
diff --git a/tools/windbg-scripts/fragment_flags b/tools/windbg-scripts/fragment_flags
new file mode 100644
index 000000000..62c3426dc
--- /dev/null
+++ b/tools/windbg-scripts/fragment_flags
@@ -0,0 +1,67 @@
+$$ **********************************************************
+$$ Copyright (c) 2005-2008 VMware, Inc.  All rights reserved.
+$$ **********************************************************
+
+$$ Redistribution and use in source and binary forms, with or without
+$$ modification, are permitted provided that the following conditions are met:
+$$ 
+$$ * Redistributions of source code must retain the above copyright notice,
+$$   this list of conditions and the following disclaimer.
+$$ 
+$$ * Redistributions in binary form must reproduce the above copyright notice,
+$$   this list of conditions and the following disclaimer in the documentation
+$$   and/or other materials provided with the distribution.
+$$ 
+$$ * Neither the name of VMware, Inc. nor the names of its contributors may be
+$$   used to endorse or promote products derived from this software without
+$$   specific prior written permission.
+$$ 
+$$ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+$$ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+$$ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+$$ ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+$$ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+$$ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+$$ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+$$ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+$$ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+$$ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+$$ DAMAGE.
+
+$$ usage: set $t0 to point to a Fragment
+
+$$ flags as of fragment.h 1.178
+
+? @$t0; .echo =>; ? poi(@$t0+4);
+.if ((poi(@$t0+4) & 0x000001)==0x000001) {.echo "IS_FUTURE";};
+.if ((poi(@$t0+4) & 0x000002)==0x000002) {.echo "TRACE_LINKS_SHIFTED";};
+.if ((poi(@$t0+4) & 0x000004)==0x000004) {.echo "IS_TRACE";};
+.if ((poi(@$t0+4) & 0x000008)==0x000008) {.echo "IS_TRACE_HEAD";};
+.if ((poi(@$t0+4) & 0x000010)==0x000010) {.echo "LINKED_OUTGOING";};
+.if ((poi(@$t0+4) & 0x000020)==0x000020) {.echo "LINKED_INCOMING";};
+.if ((poi(@$t0+4) & 0x000040)==0x000040) {.echo "CANNOT_DELETE";};
+.if ((poi(@$t0+4) & 0x000080)==0x000080) {.echo "CANNOT_BE_TRACE";};
+.if ((poi(@$t0+4) & 0x000100)==0x000100) {.echo "FAKE";};
+.if ((poi(@$t0+4) & 0x000200)==0x000200) {.echo "WRITES_EFLAGS_6";};
+.if ((poi(@$t0+4) & 0x000400)==0x000400) {.echo "WRITES_EFLAGS_OF";};
+.if ((poi(@$t0+4) & 0x000800)==0x000800) {.echo "FCACHE_FREE_LIST";};
+.if ((poi(@$t0+4) & 0x001000)==0x001000) {.echo "HAS_SYSCALL";};
+.if ((poi(@$t0+4) & 0x002000)==0x002000) {.echo "TRACE_BUILDING";};
+.if ((poi(@$t0+4) & 0x004000)==0x004000) {.echo "WAS_DELETED";};
+.if ((poi(@$t0+4) & 0x008000)==0x008000) {.echo "SELFMOD_SANDBOXED";};
+.if ((poi(@$t0+4) & 0x010000)==0x010000) {.echo "HAS_DIRECT_CTI";};
+.if ((poi(@$t0+4) & 0x020000)==0x020000) {.echo "IS_EMPTY_SLOT";};
+.if ((poi(@$t0+4) & 0x040000)==0x040000) {.echo "IS_EXTRA_VMAREA";};
+.if ((poi(@$t0+4) & 0x080000)==0x080000) {.echo "IS_EXTRA_VMAREA_INIT";};
+.if ((poi(@$t0+4) & 0x100000)==0x100000) {.echo "DYNGEN";};
+.if ((poi(@$t0+4) & 0x200000)==0x200000) {.echo "DYNGEN_RESTRICTED";};
+.if ((poi(@$t0+4) & 0x400000)==0x400000) {.echo "ENDS_WITH_RETURN";};
+.if ((poi(@$t0+4) & 0x800000)==0x800000) {.echo "MUST_END_TRACE";};
+.if ((poi(@$t0+4) & 0x1000000)==0x1000000) {.echo "SHARED";};
+.if ((poi(@$t0+4) & 0x2000000)==0x2000000) {.echo "TEMP_PRIVATE";};
+.if ((poi(@$t0+4) & 0x4000000)==0x4000000) {.echo "TRACE_OUTPUT";};
+.if ((poi(@$t0+4) & 0x8000000)==0x8000000) {.echo "CBR_FALLTHROUGH_SHORT";};
+.if ((poi(@$t0+4) & 0x10000000)==0x10000000) {.echo "COARSE_GRAIN";};
+.if ((poi(@$t0+4) & 0x20000000)==0x20000000) {.echo "IBL_TARGETED";}
+.if ((poi(@$t0+4) & 0x40000000)==0x40000000) {.echo "DO_NOT_SIDELINE";};
+.if ((poi(@$t0+4) & 0x80000000)==0x80000000) {.echo "FOLLOWS_FREE_ENTRY";};
diff --git a/tools/windbg-scripts/fragment_lookup b/tools/windbg-scripts/fragment_lookup
new file mode 100644
index 000000000..277575973
--- /dev/null
+++ b/tools/windbg-scripts/fragment_lookup
@@ -0,0 +1,156 @@
+$$ **********************************************************
+$$ Copyright (c) 2005-2006 VMware, Inc.  All rights reserved.
+$$ **********************************************************
+
+$$ Redistribution and use in source and binary forms, with or without
+$$ modification, are permitted provided that the following conditions are met:
+$$ 
+$$ * Redistributions of source code must retain the above copyright notice,
+$$   this list of conditions and the following disclaimer.
+$$ 
+$$ * Redistributions in binary form must reproduce the above copyright notice,
+$$   this list of conditions and the following disclaimer in the documentation
+$$   and/or other materials provided with the distribution.
+$$ 
+$$ * Neither the name of VMware, Inc. nor the names of its contributors may be
+$$   used to endorse or promote products derived from this software without
+$$   specific prior written permission.
+$$ 
+$$ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+$$ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+$$ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+$$ ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+$$ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+$$ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+$$ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+$$ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+$$ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+$$ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+$$ DAMAGE.
+
+$$ lookup a tag in every available fragment table
+$$ uses the current thread's dcontext from the teb to find the private tables
+
+$$ assumption: shared_{bb,trace,future} and private future use phi,
+$$ rest use plain mask hash func
+
+$$ usage: store tag into t0:
+$$ r $t0 = 27ece6a8
+
+$$ must use pseudo-register, not something like u1, which doesn't work
+$$ due to literal textual expansion rather than value replacement.
+$$ @ makes it MUCH faster since won't try to look up as symbol.
+
+$$ first, look in shared tables
+r $t1 = @@(&shared_bb->table[((@$t0 * 2654435769) >> (32 - shared_bb->hash_bits)) & shared_bb->hash_mask])
+$$ keep going until find match or null_fragment
+.for (; poi(poi(@$t1)) != 0; ) {
+    .if (poi(poi(@$t1)) = @$t0) {
+        .echo **** found fragment in shared bb table; ? poi(@$t1)
+        dt theFragment poi(@$t1)
+        .break
+    }
+    .else {
+        r $t1 = @$t1 + 4
+    }
+}
+.echo finished searching shared bb table; .echo;
+
+r $t1 = @@(&shared_trace->table[((@$t0 * 2654435769) >> (32 - shared_trace->hash_bits)) & shared_trace->hash_mask])
+$$ keep going until find match or null_fragment
+.for (; poi(poi(@$t1)) != 0; ) {
+    .if (poi(poi(@$t1)) = @$t0) {
+        .echo **** found fragment in shared trace table; ? poi(@$t1)
+        dt theFragment poi(@$t1)
+        .break
+    }
+    .else {
+        r $t1 = @$t1 + 4
+    }
+}
+.echo finished searching shared trace table; .echo;
+
+r $t1 = @@(&shared_future->table[((@$t0 * 2654435769) >> (32 - shared_future->hash_bits)) & shared_future->hash_mask])
+$$ keep going until find match or null_fragment
+.for (; poi(poi(@$t1)) != 0; ) {
+    .if (poi(poi(@$t1)) = @$t0) {
+        .echo **** found fragment in shared future table; ? poi(@$t1)
+        dt theFutureFragment poi(@$t1)
+        .break
+    }
+    .else {
+        r $t1 = @$t1 + 4
+    }
+}
+.echo finished searching shared future table; .echo;
+
+$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$
+$$ try local tables
+
+r $t2 = poi(@$teb+@@(dynamorio!tls_dcontext_offs))
+r $t2 = @@(((theDynamoContext *)@$t2)->fragment_field)
+
+r $t1 = @@(& (((thePerThread *)@$t2)->bb).table[@$t0 & (((thePerThread*)@$t2)->bb).hash_mask] )
+$$ keep going until find match or null_fragment
+.for (; poi(poi(@$t1)) != 0; ) {
+    .if (poi(poi(@$t1)) = @$t0) {
+        .echo **** found fragment in private bb table; ? poi(@$t1)
+        dt theFragment poi(@$t1)
+        .break
+    }
+    .else {
+        r $t1 = @$t1 + 4
+    }
+}
+.echo finished searching private bb table; .echo;
+
+r $t1 = @@(& (((thePerThread *)@$t2)->future).table[((@$t0 * 2654435769) >> (32 - (((thePerThread*)@$t2)->future).hash_bits)) & (((thePerThread*)@$t2)->future).hash_mask] )
+$$ keep going until find match or null_fragment
+.for (; poi(poi(@$t1)) != 0; ) {
+    .if (poi(poi(@$t1)) = @$t0) {
+        .echo **** found fragment in private future table; ? poi(@$t1)
+        dt theFragment poi(@$t1)
+        .break
+    }
+    .else {
+        r $t1 = @$t1 + 4
+    }
+}
+.echo finished searching private future table; .echo;
+
+$$ unfortunately no way to tell in release build whether trace table is
+$$ initialized or not...we'll just get a mem access error
+.if (@@( (((thePerThread *)@$t2)->trace).table = 0xabababab)) {
+    .echo (no private trace table); .echo;
+}
+.else {
+    r $t1 = @@(& (((thePerThread *)@$t2)->trace).table[@$t0 & (((thePerThread*)@$t2)->trace).hash_mask] )
+    $$ keep going until find match or null_fragment
+    .for (; poi(poi(@$t1)) != 0; ) {
+        .if (poi(poi(@$t1)) = @$t0) {
+            .echo **** found fragment in private trace table; ? poi(@$t1)
+            dt theFragment poi(@$t1)
+            .break
+        }
+        .else {
+            r $t1 = @$t1 + 4
+        }
+    }   
+    .echo finished searching private trace table; .echo;
+}
+
+.for (r $t3=0; @$t3<3; r $t3=@$t3+1) {
+    r $t1 = @@(& (((thePerThread *)@$t2)->trace_ibt[@$t3]).table[@$t0 & (((thePerThread*)@$t2)->trace_ibt[@$t3]).hash_mask] )
+    $$ keep going until find match or null_fragment
+    .for (; poi(poi(@$t1)) != 0; ) {
+        .if (poi(poi(@$t1)) = @$t0) {
+            .echo **** found fragment in private trace_ibt table #; ? @$t3; ? poi(@$t1)
+            dt theFragment poi(@$t1)
+            .break
+        }
+        .else {
+            r $t1 = @$t1 + 4
+        }
+    }   
+    .echo finished searching private trace_ibt table #; ? @$t3; .echo;
+}
diff --git a/tools/windbg-scripts/linkstub_flags b/tools/windbg-scripts/linkstub_flags
new file mode 100644
index 000000000..9b9dd6a7d
--- /dev/null
+++ b/tools/windbg-scripts/linkstub_flags
@@ -0,0 +1,55 @@
+$$ **********************************************************
+$$ Copyright (c) 2005 VMware, Inc.  All rights reserved.
+$$ **********************************************************
+
+$$ Redistribution and use in source and binary forms, with or without
+$$ modification, are permitted provided that the following conditions are met:
+$$ 
+$$ * Redistributions of source code must retain the above copyright notice,
+$$   this list of conditions and the following disclaimer.
+$$ 
+$$ * Redistributions in binary form must reproduce the above copyright notice,
+$$   this list of conditions and the following disclaimer in the documentation
+$$   and/or other materials provided with the distribution.
+$$ 
+$$ * Neither the name of VMware, Inc. nor the names of its contributors may be
+$$   used to endorse or promote products derived from this software without
+$$   specific prior written permission.
+$$ 
+$$ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+$$ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+$$ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+$$ ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+$$ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+$$ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+$$ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+$$ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+$$ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+$$ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+$$ DAMAGE.
+
+$$ usage: set $t1 to linkstub pointer
+$$ example: r $t1=@@(dcontext->last_exit)
+
+$$ flags as of link.h 1.81
+
+? @$t1; .echo =>; ? low(poi(@$t1+4));
+.if ((poi(@$t1+4) & 0x000001)==0x000001) {.echo "DIRECT";};
+.if ((poi(@$t1+4) & 0x000002)==0x000002) {.echo "INDIRECT";};
+.if ((poi(@$t1+4) & 0x000004)==0x000004) {.echo "RETURN";};
+.if ((poi(@$t1+4) & 0x000008)==0x000008) {.echo "CALL";};
+.if ((poi(@$t1+4) & 0x000010)==0x000010) {.echo "JMP";};
+.if ((poi(@$t1+4) & 0x000020)==0x000020) {.echo "IND_JMP_PLT";};
+.if ((poi(@$t1+4) & 0x000040)==0x000040) {.echo "SELFMOD_EXIT";};
+.if ((poi(@$t1+4) & 0x000080)==0x000080) {.echo "TARGET_PREFIX";};
+.if ((poi(@$t1+4) & 0x000100)==0x000100) {.echo "*** INVALID / CUSTOM_TRACES ENDS_BLOCK";};
+.if ((poi(@$t1+4) & 0x000200)==0x000200) {.echo "CALLBACK_RETURN";};
+.if ((poi(@$t1+4) & 0x000400)==0x000400) {.echo "NI_SYSCALL";};
+$$ START_OF_LIST
+$$ END_OF_LIST
+.if ((poi(@$t1+4) & 0x001000)==0x002000) {.echo "NEVER_LINKED";};
+.if ((poi(@$t1+4) & 0x004000)==0x004000) {.echo "LINKED";};
+.if ((poi(@$t1+4) & 0x008000)==0x008000) {.echo "SEPARATE_STUB";};
+
+.if ((poi(@$t1+4) & 0x000800)==0x000800) {.echo "START_OF_LIST";};
+.if ((poi(@$t1+4) & 0x000800)==0x001000) {.echo "END_OF_LIST";};
diff --git a/tools/windbg-scripts/rank_order b/tools/windbg-scripts/rank_order
new file mode 100644
index 000000000..7eb7f88d3
--- /dev/null
+++ b/tools/windbg-scripts/rank_order
@@ -0,0 +1,33 @@
+$$ **********************************************************
+$$ Copyright (c) 2007 VMware, Inc.  All rights reserved.
+$$ **********************************************************
+
+$$ Redistribution and use in source and binary forms, with or without
+$$ modification, are permitted provided that the following conditions are met:
+$$ 
+$$ * Redistributions of source code must retain the above copyright notice,
+$$   this list of conditions and the following disclaimer.
+$$ 
+$$ * Redistributions in binary form must reproduce the above copyright notice,
+$$   this list of conditions and the following disclaimer in the documentation
+$$   and/or other materials provided with the distribution.
+$$ 
+$$ * Neither the name of VMware, Inc. nor the names of its contributors may be
+$$   used to endorse or promote products derived from this software without
+$$   specific prior written permission.
+$$ 
+$$ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+$$ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+$$ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+$$ ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+$$ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+$$ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+$$ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+$$ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+$$ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+$$ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+$$ DAMAGE.
+
+* list owned locks from a core dump with Message rank order violation
+as thread_owned_locks .frame 1; dt @@(old_thread_owned_locks->last_lock) theMutex -l prev_owned_lock name
+thread_owned_locks
diff --git a/tools/windbg-scripts/shared_delete_entries_walk b/tools/windbg-scripts/shared_delete_entries_walk
new file mode 100755
index 000000000..9aab357fe
--- /dev/null
+++ b/tools/windbg-scripts/shared_delete_entries_walk
@@ -0,0 +1,66 @@
+$$ **********************************************************
+$$ Copyright (c) 2006 VMware, Inc.  All rights reserved.
+$$ **********************************************************
+
+$$ Redistribution and use in source and binary forms, with or without
+$$ modification, are permitted provided that the following conditions are met:
+$$ 
+$$ * Redistributions of source code must retain the above copyright notice,
+$$   this list of conditions and the following disclaimer.
+$$ 
+$$ * Redistributions in binary form must reproduce the above copyright notice,
+$$   this list of conditions and the following disclaimer in the documentation
+$$   and/or other materials provided with the distribution.
+$$ 
+$$ * Neither the name of VMware, Inc. nor the names of its contributors may be
+$$   used to endorse or promote products derived from this software without
+$$   specific prior written permission.
+$$ 
+$$ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+$$ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+$$ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+$$ ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+$$ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+$$ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+$$ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+$$ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+$$ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+$$ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+$$ DAMAGE.
+
+$$ walk pending deletion list
+
+$$ usage: just run it
+
+$$ field offsets valid as of vmareas.c:1.267
+
+$$ must use pseudo-register, not something like u1, which doesn't work
+$$ due to literal textual expansion rather than value replacement.
+$$ @ makes it MUCH faster since won't try to look up as symbol.
+
+r $t0 = @@(shared_delete)
+r $t1 = 0
+r $t2 = 0
+.for (; @$t0 != 0; ) {
+    .echo "---------------------------------------------"
+    .echo frags; ? poi(@$t0)
+    .echo ref_count; ? poi(@$t0+4)
+    .echo flushtime_deleted; ? poi(@$t0+8)
+    $$ Now walk all the frags in the region
+    r $t3 = poi(@$t0)
+    r $t4 = 0
+    .for (; @$t3 != 0; ) {
+        .echo     Ftag; ? poi(@$t3)
+        .echo     Flags; ? poi(@$t3+4)
+        .echo     Cache PC; ? poi(@$t3+8)
+        r $t4 = @$t4 + 1
+        r $t3 = poi(@$t3+10)
+    }
+    .echo Region pending delete frags; ? @$t4
+    r $t2 = @$t2 + @$t4
+
+    r $t0 = poi(@$t0+c)
+    r $t1 = @$t1 + 1
+}
+.echo Pending delete regions; ? @$t1
+.echo Pending delete frags; ? @$t2
diff --git a/tools/windbg-scripts/vmareas_frags_walk b/tools/windbg-scripts/vmareas_frags_walk
new file mode 100644
index 000000000..113b8344f
--- /dev/null
+++ b/tools/windbg-scripts/vmareas_frags_walk
@@ -0,0 +1,59 @@
+$$ **********************************************************
+$$ Copyright (c) 2005 VMware, Inc.  All rights reserved.
+$$ **********************************************************
+
+$$ Redistribution and use in source and binary forms, with or without
+$$ modification, are permitted provided that the following conditions are met:
+$$ 
+$$ * Redistributions of source code must retain the above copyright notice,
+$$   this list of conditions and the following disclaimer.
+$$ 
+$$ * Redistributions in binary form must reproduce the above copyright notice,
+$$   this list of conditions and the following disclaimer in the documentation
+$$   and/or other materials provided with the distribution.
+$$ 
+$$ * Neither the name of VMware, Inc. nor the names of its contributors may be
+$$   used to endorse or promote products derived from this software without
+$$   specific prior written permission.
+$$ 
+$$ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+$$ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+$$ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+$$ ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+$$ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+$$ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+$$ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+$$ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+$$ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+$$ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+$$ DAMAGE.
+
+$$ walk vmareas frags list, with next_vmarea at a different offset
+$$ for a MultiEntry vs a Fragment.
+
+$$ usage: store frags value into t0:
+$$ r $t0 = 27ece6a8
+
+$$ field offsets valid as of fragment.h:1.163, vmareas.c:1.258
+
+$$ must use pseudo-register, not something like u1, which doesn't work
+$$ due to literal textual expansion rather than value replacement.
+$$ @ makes it MUCH faster since won't try to look up as symbol.
+
+.for (; @$t0 != 0; ) {
+    .echo "---------------------------------------------"
+    .if ((poi(@$t0+4) & 040000) = 040000) {
+        .echo _MultiEntry; ? @$t0
+        .echo f; ? poi(@$t0)
+        .echo flags; ? poi(@$t0+4)
+        .echo next_vmarea; ? poi(@$t0+8)
+        .echo prev_vmarea; ? poi(@$t0+c)
+        .echo also_vmarea; ? poi(@$t0+10)
+        .echo pc; ? poi(@$t0+14)
+        r $t0 = poi(@$t0+8)
+    }
+    .else {
+        dt theFragment @$t0
+        r $t0 = poi(@$t0+14)
+    }
+}
diff --git a/tools/winstats.c b/tools/winstats.c
new file mode 100644
index 000000000..0f4eeb21c
--- /dev/null
+++ b/tools/winstats.c
@@ -0,0 +1,344 @@
+/* **********************************************************
+ * Copyright (c) 2003-2008 VMware, Inc.  All rights reserved.
+ * **********************************************************/
+
+/*
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ * 
+ * * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ * 
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ * 
+ * * Neither the name of VMware, Inc. nor the names of its contributors may be
+ *   used to endorse or promote products derived from this software without
+ *   specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+ * DAMAGE.
+ */
+
+/* Copyright (c) 2001-2002 Massachusetts Institute of Technology
+ *
+ * This file is part of RIO, a research tool that is being supplied
+ * "as is" without any accompanying services or improvements from MIT.
+ * MIT makes no representations or warranties, express or implied.
+ * Permission is hereby granted for non-commercial use of RIO provided 
+ * that you preserve this copyright notice, and you do not mention 
+ * the copyright holders in advertising related to RIO without 
+ * their permission.
+ */
+/*
+ * winstats.c -- runs a target executable and gives memory stats at completion
+ * build like so: cl /nologo winstats.c user32.lib imagehlp.lib
+ */
+
+/* FIXME: Unicode support?!?! */
+
+/* xref PR 231843, we target 2003 to get a version of PROCESS_ALL_ACCESS that
+ * works for all platforms. */
+#define _WIN32_WINNT _WIN32_WINNT_WS03
+
+#include <windows.h>
+#include <commdlg.h>
+#include <imagehlp.h>
+#include <stdio.h>
+#include <assert.h>
+#include <time.h>
+#include <tchar.h>
+
+#include "globals_shared.h"
+
+#include "ntdll.h"
+
+/* global vars */
+static int limit; /* in seconds */
+static BOOL showmem;
+static double wallclock; /* in seconds */
+
+#define FP stderr
+
+static void
+print_mem_stats(HANDLE h)
+{
+    long secs = (long) wallclock;
+    VM_COUNTERS mem;
+    int cpu;
+    if (!get_process_load_ex(h, &cpu, NULL))
+        cpu = -1;
+    get_process_mem_stats(h, &mem);
+#if VERBOSE
+    printf("Process Memory Statistics:\n");
+    printf("\tPeak virtual size:         %6d KB\n",
+	mem.PeakVirtualSize/1024);
+    printf("\tPeak working set size:     %6d KB\n",
+	mem.PeakWorkingSetSize/1024);
+    printf("\tPeak paged pool usage:     %6d KB\n",
+	mem.QuotaPeakPagedPoolUsage/1024);
+    printf("\tPeak non-paged pool usage: %6d KB\n",
+	mem.QuotaPeakNonPagedPoolUsage/1024);
+    printf("\tPeak pagefile usage:       %6d KB\n",
+	mem.PeakPagefileUsage/1024);
+#endif
+    /* Elapsed real (wall clock) time.  */
+    if (secs >= 3600) {	/* One hour -> h:m:s.  */
+	fprintf(FP, "%ld:%02ld:%02ldelapsed ",
+		secs / 3600,
+		(secs % 3600) / 60,
+		secs % 60);
+    } else {
+	fprintf(FP, "%ld:%02ld.%02ldelapsed ",	/* -> m:s.  */
+		secs / 60,
+		secs % 60,
+		0 /* for now */);
+    }
+    fprintf(FP, "%d%%CPU \n", cpu);
+    fprintf(FP, "(%lu tot, %lu RSS, %lu paged, %lu non, %lu swap)k\n",
+	    mem.PeakVirtualSize/1024,
+	    mem.PeakWorkingSetSize/1024,
+	    mem.QuotaPeakPagedPoolUsage/1024,
+	    mem.QuotaPeakNonPagedPoolUsage/1024,
+	    mem.PeakPagefileUsage/1024);
+}
+
+#define DEBUGPRINT 0
+
+/* FIXME: would like ^C to kill child process, it doesn't.
+ * also, child process seems able to read stdin but not to write
+ * to stdout or stderr (in fact it dies if it tries)
+ */
+#define HANDLE_CONTROL_C 0
+
+static void
+debugbox(char *msg)
+{
+    MessageBox(NULL, TEXT(msg), TEXT("Inject Progress"), MB_OK | MB_TOPMOST);
+}
+
+#if HANDLE_CONTROL_C
+BOOL WINAPI HandlerRoutine(DWORD dwCtrlType   //  control signal type
+			   )
+{
+    printf("Inside HandlerRoutine!\n");
+    fflush(stdout);
+    /*    GenerateConsoleCtrlEvent(dwCtrlType, phandle);*/
+    return TRUE;
+}
+#endif 
+
+int usage(char *us)
+{
+    printf("Usage: %s [-s limit_sec | -m limit_min | -h limit_hr]\n"
+	    "      [-silent] <program> <args...>\n", us);
+    return 0;
+}
+
+#define MAX_CMDLINE 2048
+
+/* we must be a console application in order to have the process
+ * we launch NOT get a brand new console window!
+ */
+int __cdecl
+main(int argc, char *argv[], char *envp[])
+{
+    LPTSTR              app_name = NULL;
+    TCHAR               app_cmdline[MAX_CMDLINE];
+    LPTSTR              ch;
+    int                 i;
+    LPTSTR              pszCmdLine = GetCommandLine();
+    PLOADED_IMAGE       li;
+    STARTUPINFO         si;
+    PROCESS_INFORMATION pi;
+    HANDLE              phandle;
+    BOOL                success;
+    DWORD               app_entry;
+    DWORD               wait_result;
+    int arg_offs = 1;
+    time_t start_time, end_time;
+
+    STARTUPINFO mysi;
+    GetStartupInfo(&mysi);
+
+    /* parse args */
+    limit = 0;
+    showmem = TRUE;
+    if (argc < 2) {
+	return usage(argv[0]);
+    }
+    while (argv[arg_offs][0] == '-') {
+	if (strcmp(argv[arg_offs], "-s") == 0) {
+	    if (argc <= arg_offs+1)
+		return usage(argv[0]);
+	    limit = atoi(argv[arg_offs+1]);
+	    arg_offs += 2;
+	} else if (strcmp(argv[arg_offs], "-m") == 0) {
+	    if (argc <= arg_offs+1)
+		return usage(argv[0]);
+	    limit = atoi(argv[arg_offs+1])*60;
+	    arg_offs += 2;
+	} else if (strcmp(argv[arg_offs], "-h") == 0) {
+	    if (argc <= arg_offs+1)
+		return usage(argv[0]);
+	    limit = atoi(argv[arg_offs+1])*3600;
+	    arg_offs += 2;
+	} else if (strcmp(argv[arg_offs], "-v") == 0) {
+	    /* just ignore -v, only here for compatibility with texec */
+	    arg_offs += 1;
+	} else if (strcmp(argv[arg_offs], "-silent") == 0) {
+	    showmem = FALSE;
+	    arg_offs += 1;
+	} else {
+	    return usage(argv[0]);
+	}
+	if (limit < 0) {
+	    return usage(argv[0]);
+	}
+	if (argc - arg_offs < 1) {
+	    return usage(argv[0]);
+	}
+    }
+
+    /* we don't want quotes included in our
+     * application path string (app_name), but we do want to put
+     * quotes around every member of the command line
+     */
+    app_name = argv[arg_offs];
+    if (app_name[0] == '\"' || app_name[0] == '\'' || app_name[0] == '`') {
+	/* remove quotes */
+	app_name++;
+	app_name[strlen(app_name)-1] = '\0';
+    }
+    /* note that we want target app name as part of cmd line
+     * (FYI: if we were using WinMain, the pzsCmdLine passed in
+     *  does not have our own app name in it)
+     * since cygwin shell ends up putting extra quote characters, etc.
+     * in pszCmdLine ('"foo"' => "\"foo\""), we can't easily walk past
+     * the 1st 2 args (our path and the dll path), so we reconstruct
+     * the cmd line from scratch:
+     */
+    ch = app_cmdline;
+    ch += _snprintf(app_cmdline, MAX_CMDLINE, "\"%s\"", app_name);
+    for (i = arg_offs+1; i < argc; i++)
+	ch += _snprintf(ch, MAX_CMDLINE - (ch - app_cmdline), " \"%s\"", argv[i]);
+    assert(ch - app_cmdline < MAX_CMDLINE);
+#if DEBUGPRINT
+    printf("Running \"%s\"\n", app_cmdline);
+    fflush(stdout);
+#endif
+
+#if HANDLE_CONTROL_C
+    if (!SetConsoleCtrlHandler(HandlerRoutine, TRUE)) {
+	debugbox("SetConsoleCtrlHandler failed");
+	return 1;
+    }
+    {
+	int flags;
+	/* FIXME: hStdInput apparently is not the right handle...how do I get
+	 * the right handle?!?
+	 */
+	if (!GetConsoleMode(mysi.hStdInput, &flags)) {
+	    debugbox("GetConsoleMode failed");
+	    return 1;
+	}
+	printf("console mode flags are: 0x%08x\n", flags);
+    }
+#endif
+
+    if (li = ImageLoad(app_name, NULL)) {
+	app_entry = (DWORD)
+	    (li->MappedAddress +
+	     li->FileHeader->OptionalHeader.AddressOfEntryPoint);
+	ImageUnload(li);
+    }
+    else {
+        _snprintf(app_cmdline, MAX_CMDLINE, "Failed to load executable image \"%s\"",
+                 app_name);
+        debugbox(app_cmdline);
+        return 1;
+    }
+    
+    /* Launch the application process. */
+    ZeroMemory(&si, sizeof(si));
+    si.cb = sizeof(si);
+    si.dwFlags = STARTF_USESTDHANDLES;
+    si.hStdInput = mysi.hStdInput;
+    si.hStdOutput = mysi.hStdOutput;
+    si.hStdError = mysi.hStdError;
+
+    /* Must specify TRUE for bInheritHandles so child inherits stdin! */
+    if (!CreateProcess(app_name, app_cmdline, NULL, NULL, TRUE,
+		       CREATE_SUSPENDED,
+		       NULL, NULL, &si, &pi)) {
+	debugbox("Failed to launch application");
+        return 1;
+    }
+
+    if ((phandle = OpenProcess(PROCESS_ALL_ACCESS,
+			       FALSE, pi.dwProcessId)) == NULL) {
+        debugbox("Cannot open application process");
+        TerminateProcess(pi.hProcess, 0);
+        return 1;
+    }
+
+#if DEBUGPRINT
+    printf("Successful at starting process\n");
+    fflush(stdout);
+#endif
+
+    success = FALSE;
+    __try {
+	start_time = time(NULL);
+
+	/* resume the suspended app process so its main thread can run */
+	ResumeThread(pi.hThread);
+    
+	/* detach from the app process */
+	CloseHandle(pi.hThread);
+	CloseHandle(pi.hProcess);
+
+	/* now wait for app process to finish */
+	wait_result = WaitForSingleObject(phandle,
+					  (limit==0) ? INFINITE : (limit*1000));
+	end_time = time(NULL);
+	wallclock = difftime(end_time, start_time);
+	if (wait_result == WAIT_OBJECT_0)
+	    success = TRUE;
+	else
+	    printf("Timeout after %d seconds\n", limit);
+    }
+    __finally {
+	/* FIXME: this is my attempt to get ^C, but it doesn't work... */
+#if HANDLE_CONTROL_C
+	if (!success) {
+	    printf("Terminating child process!\n");
+	    fflush(stdout);
+	    TerminateProcess(phandle, 0);
+	} else
+	    printf("Injector exiting peacefully\n");
+#endif
+	if (showmem)
+	    print_mem_stats(phandle);
+	if (!success) {
+	    /* kill the child */
+	    TerminateProcess(phandle, 0);
+	}
+	CloseHandle(phandle);
+#if DEBUGPRINT
+	fflush(stdout);
+#endif
+    }
+
+    return 0;
+}
diff --git a/tools/x86opnums.pl b/tools/x86opnums.pl
new file mode 100755
index 000000000..58b029adc
--- /dev/null
+++ b/tools/x86opnums.pl
@@ -0,0 +1,52 @@
+#!/usr/bin/perl
+
+# **********************************************************
+# Copyright (c) 2004-2008 VMware, Inc.  All rights reserved.
+# **********************************************************
+
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+# 
+# * Redistributions of source code must retain the above copyright notice,
+#   this list of conditions and the following disclaimer.
+# 
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# 
+# * Neither the name of VMware, Inc. nor the names of its contributors may be
+#   used to endorse or promote products derived from this software without
+#   specific prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL VMWARE, INC. OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+# DAMAGE.
+
+### I use this script to add the numbers in comments in the OP_
+### enum in src/x86/instr.h
+### Instructions: send the modified enum from decode_table.c
+### as stdin for this script, it sends to standard out the enum
+### with the numbers re-calculated.
+
+$num = 0;
+while (<>) {
+    chop;
+    # handle DOS end-of-line:
+    if ($_ =~ /\r$/) { chop; };
+    $l = $_;
+    if ($l =~ /OP_/) {
+	$l =~ /^\s*\/\*\s*OP_([a-zA-Z0-9_]*)(\s*)\*\/\s*(.*)$/;
+	printf "/* %3d */     OP_$1,$2 /* $3 */ /**< $1 opcode */\n", $num;
+	$num++;
+    } else {
+	print "$l\n";
+    }
+}